chronicle-etl 0.4.3 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2b6fdca3723ec52287c070a0dd08d0cfaf825f5e8f46da0d5a34172c0008573
-  data.tar.gz: e15181ba7edc1698404af8ff8c05d5367786ea809360393e825ca5ee5eef6c75
+  metadata.gz: 2e9f565004fce3539bdcfec9b98f64e33d8d4f1dde532f573adf54bfb61eadfa
+  data.tar.gz: 81da054f627ae084d45b4fe53480d5ef26fadf9cfd3ad4eb53d908741d8214e4
 SHA512:
-  metadata.gz: c5508dcfc5e1367122ebbc191dc60a76cc7b1f088ae9ebd52bae41e07420f54ea7977a35f2b25839933b06d4673e2a120feecbd958efee07e0d313eaa7a5d167
-  data.tar.gz: adcb90549af364189c5ae3b0811c039277aba7dc6fbf2fbd6de8b89c572948d0dc00ba8da59db03252f4770737423e62c3a1175ef018742ef8bd7aee14837f63
+  metadata.gz: ffe35b9c55a442610b9832b7f918f16dff0ccad3245248f76c52afbd88c43d671e8724f72a8d181a13e3cafffae6bc97f5963f65eb5c2a0ddb7080d9b6f1d145
+  data.tar.gz: c063698040b906f99e57cc3f9e829ff8b8ca1d4e5314a3a4179dc965c17ba70da04155d01b0b1fadf573e6a110a69862e9651435bdcdb1160bfe84cb8a55a10a

data/.rubocop.yml

@@ -11,6 +11,9 @@ Style/StringLiterals:
 Layout/MultilineAssignmentLayout:
   Enabled: false
 
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
 Layout/RedundantLineBreak:
   Enabled: false
 

data/README.md

@@ -2,7 +2,7 @@
 
 ![chronicle-etl-banner](https://user-images.githubusercontent.com/6291/157330518-0f934c9a-9ec4-43d9-9cc2-12f156d09b37.png)
 
-[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl) [![Ruby](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml/badge.svg)](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml)
+[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl) [![Ruby](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml/badge.svg)](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml) [![Docs](https://img.shields.io/badge/docs-rubydoc.info-blue)](https://www.rubydoc.info/gems/chronicle-etl/)
 
 Are you trying to archive your digital history or incorporate it into your own projects? You’ve probably discovered how frustrating it is to get machine-readable access to your own data. While [building a memex](https://hyfen.net/memex/), I learned first-hand what great efforts must be made before you can begin using the data in interesting ways.
 
@@ -34,28 +34,41 @@ $ chronicle-etl --extractor NAME --transformer NAME --loader NAME
 
 # Read test.csv and display it to stdout as a table 
 $ chronicle-etl --extractor csv --input ./data.csv --loader table
+
+# Retrieve shell commands run in the last 5 hours
+$ chronicle-etl -e shell --since 5h
+
+# Get email senders from an .mbox email archive file
+$ chronicle-etl --extractor email:mbox -i sample-email-archive.mbox -t email --fields actor.slug
+
+# Save an access token as a secret and use it in a job
+$ chronicle-etl secrets:set pinboard access_token username:foo123
+$ chronicle-etl secrets:list # Verify that's it's available
+$ chronicle-etl -e pinboard --since 1mo # Used automatically based on plugin name
 ```
 
 ### Common options
 ```sh
 Options:
-  -j, [--name=NAME]                    # Job configuration name
-  -e, [--extractor=EXTRACTOR-NAME]     # Extractor class. Default: stdin
-      [--extractor-opts=key:value]     # Extractor options
-  -t, [--transformer=TRANFORMER-NAME]  # Transformer class. Default: null
-      [--transformer-opts=key:value]   # Transformer options
-  -l, [--loader=LOADER-NAME]           # Loader class. Default: stdout
-      [--loader-opts=key:value]        # Loader options
-  -i, [--input=FILENAME]               # Input filename or directory
-      [--since=DATE]                   # Load records SINCE this date. Overrides job's `load_since` configuration option in extractor's options
-      [--until=DATE]                   # Load records UNTIL this date
-      [--limit=N]                      # Only extract the first LIMIT records
-  -o, [--output=OUTPUT]                # Output filename
-      [--fields=field1 field2 ...]     # Output only these fields
-      [--log-level=LOG_LEVEL]          # Log level (debug, info, warn, error, fatal)
-                                       # Default: info
-  -v, [--verbose], [--no-verbose]      # Set log level to verbose
-      [--silent], [--no-silent]        # Silence all output
+  -j, [--name=NAME]                      # Job configuration name
+  -e, [--extractor=NAME]                 # Extractor class. Default: stdin
+      [--extractor-opts=key:value]       # Extractor options
+  -t, [--transformer=NAME]               # Transformer class. Default: null
+      [--transformer-opts=key:value]     # Transformer options
+  -l, [--loader=NAME]                    # Loader class. Default: table
+      [--loader-opts=key:value]          # Loader options
+  -i, [--input=FILENAME]                 # Input filename or directory
+      [--since=DATE]                     # Load records SINCE this date (or fuzzy time duration)
+      [--until=DATE]                     # Load records UNTIL this date (or fuzzy time duration)
+      [--limit=N]                        # Only extract the first LIMIT records
+  -o, [--output=OUTPUT]                  # Output filename
+      [--fields=field1 field2 ...]       # Output only these fields
+      [--header-row], [--no-header-row]  # Output the header row of tabular output
+
+      [--log-level=LOG_LEVEL]            # Log level (debug, info, warn, error, fatal)
+                                         # Default: info
+  -v, [--verbose], [--no-verbose]        # Set log level to verbose
+      [--silent], [--no-silent]          # Silence all output
 ```
 
 ## Connectors
@@ -83,58 +96,50 @@ $ chronicle-etl connectors:list
 - [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/json_loader.rb) - Load records serialized as JSON
 - [`rest`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/rest_loader.rb) - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
 
-### Plugins
-Plugins provide access to data from third-party platforms, services, or formats. 
+## Chronicle Plugins
+Plugins provide access to data from third-party platforms, services, or formats. Plugins are packaged as separate rubygems and can be installed through `$ gem install` or through the CLI itself.
+
+### Plugin usage
 
 ```bash
 # Install a plugin
 $ chronicle-etl plugins:install NAME
 
-# Install the imessage plugin
-$ chronicle-etl plugins:install imessage
-
 # List installed plugins
 $ chronicle-etl plugins:list
 
+# Use a plugin
+$ chronicle-etl plugins:install shell
+$ chronicle-etl --extractor shell:history --limit 10
+
 # Uninstall a plugin
 $ chronicle-etl plugins:uninstall NAME
 ```
 
-A few dozen importers exist [in my Memex project](https://hyfen.net/memex/) and they’re being ported over to the Chronicle system. This table shows what’s available now and what’s coming. Rows are sorted in very rough order of priority.
+### Status
+
+A few dozen importers exist [in my Memex project](https://hyfen.net/memex/) and I'm porting them over to the Chronicle system. The [Chronicle Plugin Tracker](https://github.com/orgs/chronicle-app/projects/1/views/1) lets you keep track what's available and what's coming soon.
 
-If you want to work together on a connector, please [get in touch](#get-in-touch)!
+If you don't see a plugin for a third-party provider or data source that you're interested in using with `chronicle-etl`, [please open an issue](https://github.com/chronicle-app/chronicle-etl/issues/new). If you want to work together on a plugin, please [get in touch](#get-in-touch)!
+
+#### Currently available
 
 | Name                                                            | Description                                                                                 | Availability                     |
 |-----------------------------------------------------------------|---------------------------------------------------------------------------------------------|----------------------------------|
 | [imessage](https://github.com/chronicle-app/chronicle-imessage) | iMessage messages and attachments                                                           | Available                        |
-| [shell](https://github.com/chronicle-app/chronicle-shell)       | Shell command history                                                                       | Available (zsh support pending)  |
-| [email](https://github.com/chronicle-app/chronicle-email)       | Emails and attachments from IMAP or .mbox files                                             | Available (imap support pending) |
+| [shell](https://github.com/chronicle-app/chronicle-shell)       | Shell command history                                                                       | Available (still needs zsh support)  |
+| [email](https://github.com/chronicle-app/chronicle-email)       | Emails and attachments from IMAP or .mbox files                                             | Available (still needs IMAP support) |
 | [pinboard](https://github.com/chronicle-app/chronicle-email)    | Bookmarks and tags                                                                          | Available                        |
 | [safari](https://github.com/chronicle-app/chronicle-safari)     | Browser history from local sqlite db                                                        | Available                        |
-| github                                                          | Github user and repo activity                                                               | In progress                      |
-| chrome                                                          | Browser history from local sqlite db                                                        | Needs porting                    |
-| whatsapp                                                        | Messaging history (via individual chat exports) or reverse-engineered local desktop install | Unstarted                        |
-| anki                                                            | Studying and card creation history                                                          | Needs porting                    |
-| facebook                                                        | Messaging and history posting via data export files                                         | Needs porting                    |
-| twitter                                                         | History via API or export data files                                                        | Needs porting                    |
-| foursquare                                                      | Location history via API                                                                    | Needs porting                    |
-| goodreads                                                       | Reading history via export csv (RIP goodreads API)                                          | Needs porting                    |
-| lastfm                                                          | Listening history via API                                                                   | Needs porting                    |
-| images                                                          | Process image files                                                                         | Needs porting                    |
-| arc                                                             | Location history from synced icloud backup files                                            | Needs porting                    |
-| firefox                                                         | Browser history from local sqlite db                                                        | Needs porting                    |
-| fitbit                                                          | Personal analytics via API                                                                  | Needs porting                    |
-| git                                                             | Commit history on a repo                                                                    | Needs porting                    |
-| google-calendar                                                 | Calendar events via API                                                                     | Needs porting                    |
-| instagram                                                       | Posting and messaging history via export data                                               | Needs porting                    |
-| shazam                                                          | Song tags via reverse-engineered API                                                        | Needs porting                    |
-| slack                                                           | Messaging history via API                                                                   | Need rethinking                  |
-| strava                                                          | Activity history via API                                                                    | Needs porting                    |
-| things                                                          | Task activity via local sqlite db                                                           | Needs porting                    |
-| bear                                                            | Note taking activity via local sqlite db                                                    | Needs porting                    |
-| youtube                                                         | Video activity via takeout data and API                                                     | Needs porting                    |
-
-### Writing your own connector
+
+#### Coming soon
+
+In summary, the following **are coming soon**:
+anki, arc, bear, chrome, facebook, firefox, fitbit, foursquare, git, github, goodreads, google-calendar, images, instagram, lastfm, shazam, slack, strava, things, twitter, whatsapp, youtube.
+
+Please check the [Chronicle Plugin Tracker](https://github.com/orgs/chronicle-app/projects/1/views/1) for details.
+
+### Writing your own plugin
 
 Additional connectors are packaged as separate ruby gems. You can view the [iMessage plugin](https://github.com/chronicle-app/chronicle-imessage) for an example.
 
@@ -149,7 +154,7 @@ module Chronicle
     class FooExtractor < Chronicle::ETL::Extractor
       register_connector do |r|
         r.identifier = 'foo'
-        r.description = 'From foo.com'
+        r.description = 'from foo.com'
       end
 
       setting :access_token, required: true
@@ -168,6 +173,45 @@ module Chronicle
 end
 ```
 
+## Secrets Management
+
+If your job needs secrets such as access tokens or passwords, `chronicle-etl` has a built-in secret management system. 
+
+Secrets are organized in namespaces. Typically, you use one namespace per plugin (`pinboard` secrets for the `pinboard` plugin). When you run a job that uses the `pinboard` plugin extractor, for example, the secrets from that namespace will automatically be included in the extractor's options. To override which secrets get included, you can use do it in the connector options with `secrets: ALT-NAMESPACE`.
+
+Under the hood, secrets are stored in `~/.config/chronicle/etl/secrets/NAMESPACE.yml` with 0600 permissions on each file.
+
+### Using the secret manager
+
+```sh
+# Save a secret under the 'pinboard' namespace
+$ chronicle-etl secrets:set pinboard access_token username:foo123
+
+# Set a secret using stdin
+$ echo -n "username:foo123" | chronicle-etl secrets:set pinboard access_token
+
+# List available secretes
+$ chronicle-etl secrets:list
+
+# Use 'pinboard' secrets in the pinboard extractor's options (happens automatically)
+$ chronicle-etl -e pinboard --since 1mo
+
+# Use a custom secrets namespace
+$ chronicle-etl secrets:set pinboard-alt access_token different-username:foo123
+$ chronicle-etl -e pinboard --extractor-opts secrets:pinboard-alt --since 1mo
+
+# Remove a secret
+$ chronicle-etl secrets:unset pinboard access_token
+```
+
+## Roadmap
+
+- Add **homebrew formula** for easier installation. #13
+- Keep tackling **new plugins**. See: [Chronicle Plugin Tracker](https://github.com/orgs/chronicle-app/projects/1)
+- Add support for **incremental extractions** #37
+- **Improve stdin extractor and shell command transformer** (#5) so that users can easily integrate their own scripts/tools into jobs
+- **Add documentation for Chronicle Schema**. It's found throughout this project but never explained.
+
 ## Development
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 

data/chronicle-etl.gemspec

@@ -43,22 +43,23 @@ Gem::Specification.new do |spec|
   spec.add_dependency "marcel", "~> 1.0.2"
   spec.add_dependency "mini_exiftool", "~> 2.10"
   spec.add_dependency "nokogiri", "~> 1.13"
-  spec.add_dependency "runcom", ">= 6.0"
   spec.add_dependency "sequel", "~> 5.35"
   spec.add_dependency "sqlite3", "~> 1.4"
   spec.add_dependency "thor", "~> 1.2"
   spec.add_dependency "thor-hollaback", "~> 0.2"
   spec.add_dependency "tty-progressbar", "~> 0.17"
+  spec.add_dependency "tty-prompt", "~> 0.23"
   spec.add_dependency "tty-spinner"
   spec.add_dependency "tty-table", "~> 0.11"
-  spec.add_dependency "tty-prompt", "~> 0.23"
+  spec.add_dependency "xdg", ">= 4.0"
 
   spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "guard-rspec", "~> 4.7.3"
+  spec.add_development_dependency "fakefs"
   spec.add_development_dependency "pry-byebug", "~> 3.9"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_development_dependency "rubocop", "~> 1.25.1"
   spec.add_development_dependency "simplecov", "~> 0.21"
-  spec.add_development_dependency "guard-rspec", "~> 4.7.3"
   spec.add_development_dependency "yard", "~> 0.9.7"
-  spec.add_development_dependency "rubocop", "~> 1.25.1"
 end

data/lib/chronicle/etl/cli/connectors.rb

@@ -4,6 +4,8 @@ module Chronicle
   module ETL
     module CLI
       # CLI commands for working with ETL connectors
+      #
+      # @todo make this work with new plugin system (i.e. no loading of all plugins)
       class Connectors < SubcommandBase
         default_task 'list'
         namespace :connectors
@@ -11,8 +13,6 @@ module Chronicle
         desc "list", "Lists available connectors"
         # Display all available connectors that chronicle-etl has access to
         def list
-          Chronicle::ETL::Registry.load_all!
-
           connector_info = Chronicle::ETL::Registry.connectors.map do |connector_registration|
             {
               identifier: connector_registration.identifier,

data/lib/chronicle/etl/cli/jobs.rb

@@ -20,8 +20,8 @@ module Chronicle
 
         # This is an array to deal with shell globbing
         class_option :input, aliases: '-i', desc: 'Input filename or directory', default: [], type: 'array', banner: 'FILENAME'
-        class_option :since, desc: "Load records SINCE this date", banner: 'DATE'
-        class_option :until, desc: "Load records UNTIL this date", banner: 'DATE'
+        class_option :since, desc: "Load records SINCE this date (or fuzzy time duration)", banner: 'DATE'
+        class_option :until, desc: "Load records UNTIL this date (or fuzzy time duration)", banner: 'DATE'
         class_option :limit, desc: "Only extract the first LIMIT records", banner: 'N'
 
         class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
@@ -49,7 +49,7 @@ LONG_DESC
 
           if job_definition.plugins_missing?
             missing_plugins = job_definition.errors[:plugins]
-              .select { |error| error.is_a?(Chronicle::ETL::PluginLoadError) }
+              .select { |error| error.is_a?(Chronicle::ETL::PluginNotInstalledError) }
               .map(&:name)
               .uniq
             install_missing_plugins(missing_plugins)
@@ -57,7 +57,11 @@ LONG_DESC
 
           run_job(job_definition)
         rescue Chronicle::ETL::JobDefinitionError => e
-          cli_fail(message: "Error running job.\n#{job_definition.errors}", exception: e)
+          message = ""
+          job_definition.errors.each_pair do |category, errors|
+            message << "Problem with #{category}:\n  - #{errors.map(&:to_s).join("\n  - ")}"
+          end
+          cli_fail(message: "Error running job.\n#{message}", exception: e)
         end
 
         desc "create", "Create a job"
@@ -66,8 +70,7 @@ LONG_DESC
           job_definition = build_job_definition(options)
           job_definition.validate!
 
-          path = File.join('chronicle', 'etl', 'jobs', options[:name])
-          Chronicle::ETL::Config.write(path, job_definition.definition)
+          Chronicle::ETL::Config.write("jobs", options[:name], job_definition.definition)
         rescue Chronicle::ETL::JobDefinitionError => e
           cli_fail(message: "Job definition error", exception: e)
         end
@@ -88,7 +91,7 @@ LONG_DESC
           jobs = Chronicle::ETL::Config.available_jobs
 
           job_details = jobs.map do |job|
-            r = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
+            r = Chronicle::ETL::Config.load("jobs", job)
 
             extractor = r[:extractor][:name] if r[:extractor]
             transformer = r[:transformer][:name] if r[:transformer]
@@ -109,6 +112,11 @@ LONG_DESC
         private
 
         def run_job(job_definition)
+          # FIXME: have to validate here so next method can work. This is clumsy
+          job_definition.validate!
+          # FIXME: clumsy to make CLI responsible for setting secrets here. Think about a better way to do this
+          job_definition.apply_default_secrets
+
           job = Chronicle::ETL::Job.new(job_definition)
           runner = Chronicle::ETL::Runner.new(job)
           runner.run!
@@ -136,21 +144,22 @@ LONG_DESC
         end
 
         def load_job_config name
-          Chronicle::ETL::Config.load_job_from_config(name)
+          Chronicle::ETL::Config.read_job(name)
         end
 
         # Takes flag options and turns them into a runner config
+        # TODO: this needs a lot of refactoring
         def process_flag_options options
-          extractor_options = options[:'extractor-opts'].merge({
+          extractor_options = options[:'extractor-opts'].transform_keys(&:to_sym).merge({
             input: (options[:input] if options[:input].any?),
             since: options[:since],
             until: options[:until],
-            limit: options[:limit],
+            limit: options[:limit]
           }.compact)
 
-          transformer_options = options[:'transformer-opts']
+          transformer_options = options[:'transformer-opts'].transform_keys(&:to_sym)
 
-          loader_options = options[:'loader-opts'].merge({
+          loader_options = options[:'loader-opts'].transform_keys(&:to_sym).merge({
             output: options[:output],
             header_row: options[:header_row],
             fields: options[:fields]

data/lib/chronicle/etl/cli/main.rb

@@ -24,6 +24,9 @@ module Chronicle
         desc 'plugins:COMMAND', 'Configure plugins', hide: true
         subcommand 'plugins', Plugins
 
+        desc 'secrets:COMMAND', 'Manage secrets', hide: true
+        subcommand 'secrets', Secrets
+
         # Entrypoint for the CLI
         def self.start(given_args = ARGV, config = {})
           # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class

data/lib/chronicle/etl/cli/plugins.rb

@@ -15,15 +15,25 @@ module Chronicle
         def install(*plugins)
           cli_fail(message: "Please specify a plugin to install") unless plugins.any?
 
-          spinner = TTY::Spinner.new("[:spinner] Installing #{plugins.join(", ")}...", format: :dots_2)
+          installed, not_installed = plugins.partition do |plugin|
+            Chronicle::ETL::Registry::PluginRegistry.installed?(plugin)
+          end
+
+          puts "Already installed: #{installed.join(", ")}" if installed.any?
+          cli_exit unless not_installed.any?
+
+          spinner = TTY::Spinner.new("[:spinner] Installing #{not_installed.join(", ")}...", format: :dots_2)
           spinner.auto_spin
-          plugins.each do |plugin|
+
+          not_installed.each do |plugin|
             spinner.update(title: "Installing #{plugin}")
             Chronicle::ETL::Registry::PluginRegistry.install(plugin)
+
           rescue Chronicle::ETL::PluginError => e
             spinner.error("Error".red)
             cli_fail(message: "Plugin '#{plugin}' could not be installed", exception: e)
           end
+
           spinner.success("(#{'successful'.green})")
         end
 

data/lib/chronicle/etl/cli/secrets.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "tty-prompt"
+
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL plugins
+      class Secrets < SubcommandBase
+        default_task 'list'
+        namespace :secrets
+
+        desc "set NAMESPACE KEY [VALUE]", "Add a secret. VALUE can be set as argument or from stdin"
+        def set(namespace, key, value=nil)
+          validate_namespace(namespace)
+
+          if value
+            # came as argument
+          elsif $stdin.respond_to?(:stat) && $stdin.stat.pipe?
+            value = $stdin.read
+          else
+            prompt = TTY::Prompt.new
+            value = prompt.mask("Please enter #{key} for #{namespace}:")
+          end
+
+          Chronicle::ETL::Secrets.set(namespace, key, value.strip)
+          cli_exit(message: "Secret set")
+        rescue TTY::Reader::InputInterrupt
+          cli_fail(message: "\nSecret not set")
+        end
+
+        desc "unset NAMESPACE KEY", "Remove a secret"
+        def unset(namespace, key)
+          validate_namespace(namespace)
+
+          Chronicle::ETL::Secrets.unset(namespace, key)
+          cli_exit(message: "Secret unset")
+        end
+
+        desc "list", "List available secrets"
+        def list(namespace=nil)
+          all_secrets = Chronicle::ETL::Secrets.all(namespace)
+          cli_exit(message: "No secrets are stored") unless all_secrets.any?
+
+          rows = []
+          all_secrets.each do |namespace, secrets|
+            rows += secrets.map do |key, value|
+              # hidden_value = (value[0..5] + ("*" * [0, [value.length - 5, 30].min].max)).truncate(30)
+              truncated_value = value.truncate(30)
+              [namespace, key, truncated_value]
+            end
+          end
+
+          headers = ['namespace', 'key', 'value'].map { |h| h.upcase.bold }
+
+          puts "Available secrets:"
+          table = TTY::Table.new(headers, rows)
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+
+        private
+
+        def validate_namespace(namespace)
+          cli_fail(message: "'#{namespace}' is not a valid namespace") unless Chronicle::ETL::Secrets.valid_namespace_name?(namespace)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli.rb

@@ -7,4 +7,5 @@ require 'chronicle/etl/cli/subcommand_base'
 require 'chronicle/etl/cli/connectors'
 require 'chronicle/etl/cli/jobs'
 require 'chronicle/etl/cli/plugins'
+require 'chronicle/etl/cli/secrets'
 require 'chronicle/etl/cli/main'

data/lib/chronicle/etl/config.rb

@@ -1,55 +1,67 @@
-require 'runcom'
+require 'fileutils'
+require 'yaml'
 
 module Chronicle
   module ETL
     # Utility methods to read, write, and access config files
     module Config
-      module_function
+      extend self
 
-      # Loads a yml config file
-      def load(path)
-        config = Runcom::Config.new(path)
-        # FIXME: hack to deeply symbolize keys
-        JSON.parse(config.to_h.to_json, symbolize_names: true)
+      attr_accessor :xdg_environment
+
+      def load(type, identifier)
+        base = config_pathname_for_type(type)
+        path = base.join("#{identifier}.yml")
+        return {} unless path.exist?
+
+        YAML.safe_load(File.read(path), symbolize_names: true, permitted_classes: [Symbol, Date, Time])
       end
 
       # Writes a hash as a yml config file
-      def write(path, data)
-        config = Runcom::Config.new(path)
-        filename = config.all[0].to_s + '.yml'
-        File.open(filename, 'w') do |f|
-          f << data.to_yaml
+      def write(type, identifier, data)
+        base = config_pathname_for_type(type)
+        path = base.join("#{identifier}.yml")
+        FileUtils.mkdir_p(File.dirname(path))
+        File.open(path, 'w', 0o600) do |f|
+          # Ruby likes to add --- separators when writing yaml files
+          f << data.to_yaml.gsub(/^-+\n/, '')
         end
       end
 
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
       def available_jobs
-        Dir.glob(File.join(config_directory("jobs"), "*.yml")).map do |filename|
+        Dir.glob(File.join(config_pathname_for_type("jobs"), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
 
-      # Returns all available credentials available in ~/.config/chronicle/etl/credentials/*.yml
-      def available_credentials
-        Dir.glob(File.join(config_directory("credentials"), "*.yml")).map do |filename|
+      def available_configs(type)
+        Dir.glob(File.join(config_pathname_for_type(type), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
 
       # Load a job definition from job config directory
-      def load_job_from_config(job_name)
-        definition = self.load("chronicle/etl/jobs/#{job_name}.yml")
-        definition[:name] = job_name
-        definition
+      def read_job(job_name)
+        load('jobs', job_name)
       end
 
-      def load_credentials(name)
-        config = self.load("chronicle/etl/credentials/#{name}.yml")
+      def config_pathname
+        base = Pathname.new(xdg_config.config_home)
+        base.join('chronicle', 'etl')
       end
 
-      def config_directory(type)
-        path = "chronicle/etl/#{type}"
-        Runcom::Config.new(path).current || raise(Chronicle::ETL::ConfigError, "Could not access config directory (#{path})")
+      def config_pathname_for_type(type)
+        config_pathname.join(type)
+      end
+
+      def xdg_config
+        # Only used for overriding ENV['HOME'] for XDG-related specs
+        if @xdg_environment
+          XDG::Environment.new(environment: @xdg_environment)
+        else
+          XDG::Environment.new
+        end
       end
     end
   end

data/lib/chronicle/etl/configurable.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "ostruct"
+require "chronic_duration"
 
 module Chronicle
   module ETL
@@ -57,7 +58,9 @@ module Chronicle
 
           options.each do |name, value|
             setting = self.class.all_settings[name]
-            raise(Chronicle::ETL::ConnectorConfigurationError, "Unrecognized setting: #{name}") unless setting
+
+            # Do nothing with a given option if it's not a connector setting
+            next unless setting
 
             @config[name] = coerced_value(setting, value)
           end
@@ -83,6 +86,8 @@ module Chronicle
 
         def coerced_value(setting, value)
           setting.type ? __send__("coerce_#{setting.type}", value) : value
+        rescue StandardError
+          raise(Chronicle::ETL::ConnectorConfigurationError, "Could not coerce #{value} into a #{setting.type}")
         end
 
         def coerce_string(value)
@@ -103,11 +108,15 @@ module Chronicle
         end
 
         def coerce_time(value)
-          # TODO: handle durations like '3h'
-          if value.is_a?(String)
-            Time.parse(value)
+          return value unless value.is_a?(String)
+
+          # Hacky check for duration strings like "60m"
+          if value.match(/[a-z]+/)
+            ChronicDuration.raise_exceptions = true
+            duration_ago = ChronicDuration.parse(value)
+            Time.now - duration_ago
           else
-            value
+            Time.parse(value)
           end
         end
       end

data/lib/chronicle/etl/exceptions.rb

@@ -2,6 +2,8 @@ module Chronicle
   module ETL
     class Error < StandardError; end
 
+    class SecretsError < Error; end
+
     class ConfigError < Error; end
 
     class RunnerTypeError < Error; end
@@ -23,6 +25,7 @@ module Chronicle
       end
     end
 
+    class PluginNotInstalledError < PluginError; end
     class PluginConflictError < PluginError; end
     class PluginNotAvailableError < PluginError; end
     class PluginLoadError < PluginError; end
@@ -45,14 +48,7 @@ module Chronicle
 
     class SerializationError < Error; end
 
-    class TransformationError < Error
-      attr_reader :transformation
-
-      def initialize(message=nil, transformation:)
-        super(message)
-        @transformation = transformation
-      end
-    end
+    class TransformationError < Error; end
 
     class UntransformableRecordError < TransformationError; end
   end

data/lib/chronicle/etl/job_definition.rb

@@ -45,8 +45,10 @@ module Chronicle
       def plugins_missing?
         validate
 
-        @errors[:plugins] || []
-          .filter { |e| e.instance_of?(Chronicle::ETL::PluginLoadError) }
+        return false unless @errors[:plugins]&.any?
+
+        @errors[:plugins]
+          .filter { |e| e.instance_of?(Chronicle::ETL::PluginNotInstalledError) }
           .any?
       end
 
@@ -62,6 +64,30 @@ module Chronicle
         load_credentials
       end
 
+      # For each connector in this job, mix in secrets into the options
+      def apply_default_secrets
+        Chronicle::ETL::Registry::PHASES.each do |phase|
+          # If the option have a `secrets` key, we look up those secrets and
+          # mix them in. If not, use the connector's plugin name and look up 
+          # secrets with the same namespace
+          if @definition[phase][:options][:secrets]
+            namespace = @definition[phase][:options][:secrets]
+          else
+            # We don't want to do this lookup for built-in connectors
+            next if __send__("#{phase}_klass".to_sym).connector_registration.built_in?
+
+            # infer plugin name from connector name and use it for secrets
+            # namesepace
+            namespace = @definition[phase][:name].split(":").first
+          end
+
+          # Reverse merge secrets into connector's options (we want to preserve
+          # options that came from job file or CLI options)
+          secrets = Chronicle::ETL::Secrets.read(namespace)
+          @definition[phase][:options] = secrets.merge(@definition[phase][:options])
+        end
+      end
+
       # Is this job continuing from a previous run?
       def incremental?
         @definition[:incremental]

data/lib/chronicle/etl/job_logger.rb

@@ -1,5 +1,6 @@
-require 'sequel'
 require 'forwardable'
+require 'sequel'
+require 'xdg'
 
 module Chronicle
   module ETL
@@ -35,8 +36,8 @@ module Chronicle
       end
 
       def self.db_filename
-        data = Runcom::Data.new "chronicle/etl/job_log.db"
-        filename = data.all[0].to_s
+        base = Pathname.new(XDG::Data.new.home)
+        base.join('job_log.db')
       end
 
       def self.initialize_db

data/lib/chronicle/etl/loaders/json_loader.rb

@@ -9,10 +9,13 @@ module Chronicle
       setting :output, default: $stdout
 
       def start
-        if @config.output == $stdout
-          @output = @config.output
+        if @config.output.is_a?(IO)
+          # This might seem like a duplication of the default value ($stdout)
+          # but it's because rspec overwrites $stdout (in helper #capture) to
+          # capture output.
+          @output = $stdout.dup
         else
-          @output = File.open(@config.output, "w")
+          @output = File.open(@config.output, "w+")
         end
       end
 
@@ -31,7 +34,7 @@ module Chronicle
       end
 
       def finish
-        @output.close
+        @output.close if @output.is_a?(IO)
       end
 
       private

data/lib/chronicle/etl/registry/plugin_registry.rb

@@ -13,8 +13,8 @@ module Chronicle
       module PluginRegistry
         # Does this plugin exist?
         def self.exists?(name)
-          # TODO: implement this. Could query rubygems.org or have a
-          # hardcoded approved list
+          # TODO: implement this. Could query rubygems.org or use a hardcoded
+          # list somewhere
           true
         end
 
@@ -31,6 +31,12 @@ module Chronicle
             .values
         end
 
+        # Check whether a given plugin is installed
+        def self.installed?(name)
+          gem_name = "chronicle-#{name}"
+          all_installed.map(&:name).include?(gem_name)
+        end
+
         # Activate a plugin with given name by `require`ing it
         def self.activate(name)
           # By default, activates the latest available version of a gem
@@ -39,14 +45,17 @@ module Chronicle
         rescue Gem::ConflictError => e
           # TODO: figure out if there's more we can do here
           raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
-        rescue LoadError => e
-          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded" if exists?(name)
-
-          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist"
+        rescue StandardError, LoadError => e
+          # StandardError to catch random non-loading problems that might occur
+          # when requiring the plugin (eg class macro invoked the wrong way)
+          # TODO: decide if this should be separated
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded"
         end
 
         # Install a plugin to local gems
         def self.install(name)
+          return if installed?(name)
+
           gem_name = "chronicle-#{name}"
           raise(Chronicle::ETL::PluginNotAvailableError.new(gem_name), "Plugin #{name} doesn't exist") unless exists?(gem_name)
 

data/lib/chronicle/etl/registry/registry.rb

@@ -9,18 +9,7 @@ module Chronicle
       class << self
         attr_accessor :connectors
 
-        def load_all!
-          load_connectors_from_gems
-        end
-
-        def load_connectors_from_gems
-          Gem::Specification.filter{|s| s.name.match(/^chronicle/) }.each do |gem|
-            require_str = gem.name.gsub('chronicle-', 'chronicle/')
-            require require_str rescue LoadError
-          end
-        end
-
-        def register connector
+        def register(connector)
           connectors << connector
         end
 
@@ -28,9 +17,14 @@ module Chronicle
           @connectors ||= []
         end
 
-        def find_by_phase_and_identifier(phase, identifier)
-          # Simple case: built in connector
+        # Find connector from amongst those currently loaded
+        def find_by_phase_and_identifier_local(phase, identifier)
           connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
+        end
+
+        # Find connector and load relevant plugin to find it if necessary
+        def find_by_phase_and_identifier(phase, identifier)
+          connector = find_by_phase_and_identifier_local(phase, identifier)
           return connector if connector
 
           # if not available in built-in connectors, try to activate a
@@ -44,6 +38,8 @@ module Chronicle
             plugin = identifier
           end
 
+          raise(Chronicle::ETL::PluginNotInstalledError.new(plugin)) unless PluginRegistry.installed?(plugin)
+
           PluginRegistry.activate(plugin)
 
           candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }

data/lib/chronicle/etl/runner.rb

@@ -50,12 +50,12 @@ class Chronicle::ETL::Runner
       transformer = @job.instantiate_transformer(extraction)
       record = transformer.transform
 
-      Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
+      Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
       @job_logger.log_transformation(transformer)
 
       @loader.load(record) unless @job.dry_run?
     rescue Chronicle::ETL::TransformationError => e
-      Chronicle::ETL::Logger.error(tty_log_transformation_failure(e))
+      Chronicle::ETL::Logger.error(tty_log_transformation_failure(e, transformer))
     ensure
       @progress_bar.increment
     end
@@ -83,14 +83,14 @@ class Chronicle::ETL::Runner
     output
   end
 
-  def tty_log_transformation transformer
+  def tty_log_transformation(transformer)
     output = "  ✓".green
     output += " #{transformer}"
   end
 
-  def tty_log_transformation_failure exception
+  def tty_log_transformation_failure(exception, transformer)
     output = "  ✖".red
-    output += " Failed to build #{exception.transformation}. #{exception.message}"
+    output += " Failed to build #{transformer}. #{exception.message}"
   end
 
   def tty_log_completion

data/lib/chronicle/etl/secrets.rb

@@ -0,0 +1,55 @@
+module Chronicle
+  module ETL
+    # Secret management module
+    module Secrets
+      module_function
+
+      # Save a setting to a namespaced config file
+      def set(namespace, key, value)
+        config = read(namespace)
+        config[key.to_sym] = value
+        write(namespace, config)
+      end
+
+      # Remove a setting from a namespaced config file
+      def unset(namespace, key)
+        config = read(namespace)
+        config.delete(key.to_sym)
+        write(namespace, config)
+      end
+
+      # Retrieve all secrets from all namespaces
+      def all(namespace = nil)
+        namespaces = namespace.nil? ? available_secrets : [namespace]
+        namespaces
+          .to_h { |namespace| [namespace.to_sym, read(namespace)] }
+          .delete_if { |_, v| v.empty? }
+      end
+
+      # Return whether a namespace name is valid (lowercase alphanumeric and -)
+      def valid_namespace_name?(namespace)
+        namespace.match(/^[a-z0-9\-]+$/)
+      end
+
+      # Read secrets from a config file
+      def read(namespace)
+        definition = Chronicle::ETL::Config.load("secrets", namespace)
+        definition[:secrets] || {}
+      end
+
+      # Write secrets to a config file
+      def write(namespace, secrets)
+        data = {
+          secrets: (secrets || {}).transform_keys(&:to_s),
+          chronicle_etl_version: Chronicle::ETL::VERSION
+        }.transform_keys(&:to_s) # Should I implement deeply_transform_keys...?
+        Chronicle::ETL::Config.write("secrets", namespace, data)
+      end
+
+      # Which config files are available in ~/.config/chronicle/etl/secrets
+      def available_secrets
+        Chronicle::ETL::Config.available_configs('secrets')
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/image_file_transformer.rb

@@ -43,7 +43,7 @@ module Chronicle
       def id
         @id ||= begin
           id = build_with_strategy(field: :id, strategy: @config.id_strategy)
-          raise UntransformableRecordError.new("Could not build id", transformation: self) unless id
+          raise(UntransformableRecordError, "Could not build id") unless id
 
           id
         end
@@ -52,7 +52,7 @@ module Chronicle
       def timestamp
         @timestamp ||= begin
           ts = build_with_strategy(field: :timestamp, strategy: @config.timestamp_strategy)
-          raise UntransformableRecordError.new("Could not build timestamp", transformation: self) unless ts
+          raise(UntransformableRecordError, "Could not build timestamp") unless ts
 
           ts
         end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.4.3"
+    VERSION = "0.5.1"
   end
 end

data/lib/chronicle/etl.rb

@@ -14,6 +14,7 @@ require_relative 'etl/models/base'
 require_relative 'etl/models/raw'
 require_relative 'etl/models/entity'
 require_relative 'etl/runner'
+require_relative 'etl/secrets'
 require_relative 'etl/serializers/serializer'
 require_relative 'etl/utils/binary_attachments'
 require_relative 'etl/utils/hash_utilities'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.1
 platform: ruby
 authors:
 - Andrew Louis
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-14 00:00:00.000000000 Z
+date: 2022-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -94,20 +94,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.13'
-- !ruby/object:Gem::Dependency
-  name: runcom
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '6.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: tty-prompt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.23'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.23'
 - !ruby/object:Gem::Dependency
   name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
@@ -207,19 +207,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.11'
 - !ruby/object:Gem::Dependency
-  name: tty-prompt
+  name: xdg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.23'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.23'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +234,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -277,33 +305,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.9'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.21'
+        version: 1.25.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.21'
+        version: 1.25.1
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.7.3
+        version: '0.21'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.7.3
+        version: '0.21'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -318,20 +346,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.7
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.25.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.25.1
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -364,6 +378,7 @@ files:
 - lib/chronicle/etl/cli/jobs.rb
 - lib/chronicle/etl/cli/main.rb
 - lib/chronicle/etl/cli/plugins.rb
+- lib/chronicle/etl/cli/secrets.rb
 - lib/chronicle/etl/cli/subcommand_base.rb
 - lib/chronicle/etl/config.rb
 - lib/chronicle/etl/configurable.rb
@@ -396,6 +411,7 @@ files:
 - lib/chronicle/etl/registry/registry.rb
 - lib/chronicle/etl/registry/self_registering.rb
 - lib/chronicle/etl/runner.rb
+- lib/chronicle/etl/secrets.rb
 - lib/chronicle/etl/serializers/jsonapi_serializer.rb
 - lib/chronicle/etl/serializers/raw_serializer.rb
 - lib/chronicle/etl/serializers/serializer.rb
@@ -415,7 +431,7 @@ metadata:
   homepage_uri: https://github.com/chronicle-app
   source_code_uri: https://github.com/chronicle-app/chronicle-etl
   changelog_uri: https://github.com/chronicle-app/chronicle-etl/releases
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -430,8 +446,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.9
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: ETL tool for personal data
 test_files: []