chronicle-etl 0.1.1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f364392d44aab353504c3daeb62c9e19b9a3bbc8c0e7d7efc0e3324457bb2921
-  data.tar.gz: e79ba859fa67c6ddb2e7112fe213a23a4e0d692063aab9a8f9e86fd970e2b44c
+  metadata.gz: 3d9be4b073385d9758a8b709bb6726d8dae648b8c4dbef474343840c011d1178
+  data.tar.gz: 0d95395407d37f7e322287a0920bba60e1b4f81eb8649190d021e13580604a2d
 SHA512:
-  metadata.gz: a0f0cbae079c6f1afef6b0b4f67cbfdd28b041b3247457913f6a07dfb6a21aef185286002d869293f1984b7c9806d57a788c58e9d92c2f7ae05ed61c7322633c
-  data.tar.gz: c36e3c54539540406e5dcde64ac4b3ceccc070b4f81639ae337dccdcf6b1dc0f193966ce02291a568be22f8ffb0207225a5598f8f2e11d0790e3b130ff92815c
+  metadata.gz: 386c96518aa2d2810ae2a93bbe3af5bb08e26e132608b4e6ed8a278da076783e453854a2120c5016b6d02cd5dea406146d10ef3c7c1e77d854acd8ff2608eaf7
+  data.tar.gz: eb14402be5d6db44a6f06e6ec930acc5103b36e5c2e5a13e89137c9ee45f5f11c1e9e6ab13d4d44e6ee06bd9b02309e02ac33e81256645ef71d5b431c97eb199

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  EnabledByDefault: true
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/MethodCallWithArgsParentheses:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.7.1

data/.yardopts

@@ -0,0 +1 @@
+--markup=markdown

data/CHANGELOG.md

@@ -0,0 +1,23 @@
+# Changelog
+
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.4] - 2020-08-18
+### Updated
+- Better display of available ETL classes
+- Updated documentation
+
+## [0.1.3] - 2020-08-13
+### Added
+- Ability to list all available ETL classes
+- Refactored E, T, L module and class structure
+- Better progress bar
+
+## [0.1.2] - 2020-08-02
+### Added
+- This changelog
+- Ability to use extractors, transformers, and loaders from other gems
+
+## [0.1.0] - 2020-08-01
+### Added
+- Basic job runner and ETL classes

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.1.1)
+    chronicle-etl (0.2.1)
       colorize (~> 0.8.1)
-      ruby-progressbar (~> 1.10)
-      table_print
       thor (~> 0.20)
+      tty-progressbar (~> 0.17)
+      tty-table (~> 0.11)
 
 GEM
   remote: https://rubygems.org/
@@ -14,14 +14,21 @@ GEM
     coderay (1.1.3)
     colorize (0.8.1)
     diff-lcs (1.4.4)
+    equatable (0.6.1)
     method_source (1.0.0)
+    necromancer (0.6.0)
+    pastel (0.7.4)
+      equatable (~> 0.6)
+      tty-color (~> 0.5)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.9.0)
       byebug (~> 11.0)
       pry (~> 0.13.0)
-    rake (10.5.0)
+    rake (13.0.1)
+    redcarpet (3.5.0)
+    refinements (7.7.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -35,19 +42,44 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    ruby-progressbar (1.10.1)
-    table_print (1.5.7)
+    runcom (6.2.0)
+      refinements (~> 7.4)
+      xdg (~> 4.0)
+    strings (0.1.8)
+      strings-ansi (~> 0.1)
+      unicode-display_width (~> 1.5)
+      unicode_utils (~> 1.4)
+    strings-ansi (0.1.0)
     thor (0.20.3)
+    tty-color (0.5.2)
+    tty-cursor (0.7.1)
+    tty-progressbar (0.17.0)
+      strings-ansi (~> 0.1.0)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.7)
+      unicode-display_width (~> 1.6)
+    tty-screen (0.8.1)
+    tty-table (0.11.0)
+      equatable (~> 0.6)
+      necromancer (~> 0.5)
+      pastel (~> 0.7.2)
+      strings (~> 0.1.5)
+      tty-screen (~> 0.7)
+    unicode-display_width (1.7.0)
+    unicode_utils (1.4.0)
+    xdg (4.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
+  bundler (~> 2.1)
   chronicle-etl!
   pry-byebug (~> 3.9)
-  rake (~> 10.0)
-  rspec (~> 3.0)
+  rake (~> 13.0)
+  redcarpet (~> 3.5)
+  rspec (~> 3.9)
+  runcom (~> 6.2)
 
 BUNDLED WITH
-   1.17.2
+   2.1.4

data/README.md

@@ -1,6 +1,10 @@
-# Chronicle::Etl
+# Chronicle::ETL
 
-Chronicle ETL is a utility tool for manipulating personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
+
+Chronicle ETL is a utility tool for archiving and processing personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+
+This project is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex).
 
 ## Installation
 
@@ -8,28 +12,77 @@ Chronicle ETL is a utility tool for manipulating personal data. You can extract
 $ gem install chronicle-etl
 ```
 
-## Examples
+## Usage
 
 After installing the gem, `chronicle-etl` is available to run in your shell.
 
+```bash
+# read test.csv and display it as a table
+$ chronicle-etl jobs:run --extractor csv --extractor-opts filename:test.csv --loader table
+
+# Display help for the jobs:run command
+$ chronicle-etl jobs help run
 ```
-chronicle-etl --extractor csv --extractor-opts filename:test.csv --loader table
-cat test.csv | chronicle-etl --extractor csv --loader table
+
+## Connectors
+
+Connectors are available to read, process, and load data from different formats or external services.
+
+```bash
+# List all available connectors
+$ chronicle-etl connectors:list
 ```
 
-## Full usage
+Built in connectors:
+
+### Extractors
+- `stdin` - (default) Load records from line-separated stdin
+- `csv`
+- `file` - load from a single file or directory (with a glob pattern)
+
+### Transformers
+- `null` - (default) Don't do anything
+
+### Loaders
+- `stdout` - (default) output transformed records to stdount
+- `csv` - Load records to a csv file
+- `table` - Output an ascii table of records. Useful for debugging.
+
+### Provider-specific importers
+
+In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
+
+- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` files. Transformers for chronicle schema
+- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
+
+To install any of these, run `gem install chronicle-PROVIDER`. 
+
+If you don't want to use the available rubygem importers, `chronicle-etl` can use `stdin` as an Extractor source (newline separated records). You can also use `stdout` as a loader — transformed records will be outputted separated by newlines.
+
+I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.net) to chat about what will be available!
+
+### Full commands
 
 ```
-Commands:
-  chronicle-etl help [COMMAND]  # Describe available commands or one specific command
-  chronicle-etl job             # Runs an ETL job
+$ chronicle-etl help 
+
+ALL COMMANDS
+  help                       # This help menu
+  connectors help [COMMAND]  # Describe subcommands or one specific subcommand
+  connectors:install NAME    # Installs connector NAME
+  connectors:list            # Lists available connectors
+  jobs help [COMMAND]        # Describe subcommands or one specific subcommand
+  jobs:create                # Create a job
+  jobs:list                  # List all available jobs
+  jobs:run                   # Start a job
+  jobs:show                  # Show a job
 ```
 
 ### Job options
 
 ```
 Usage:
-  chronicle-etl job
+  chronicle-etl jobs:run
 
 Options:
   -e, [--extractor=extractor-name]      # Extractor class (available: stdin, csv, file)
@@ -62,4 +115,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Chronicle::Etl project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).

data/bin/console

@@ -7,8 +7,20 @@ require "chronicle/etl"
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
+require "pry"
+Pry.start
+
+def reload!(print = true)
+  puts 'Reloading ...' if print
+  # Main project directory.
+  root_dir = File.expand_path('..', __dir__)
+  # Directories within the project that should be reloaded.
+  reload_dirs = %w{lib}
+  # Loop through and reload every file in all relevant project directories.
+  reload_dirs.each do |dir|
+    Dir.glob("#{root_dir}/#{dir}/**/*.rb").each { |f| load(f) }
+  end
+  # Return true when complete.
+  true
+end
 
-require "irb"
-IRB.start(__FILE__)

data/chronicle-etl.gemspec

@@ -5,7 +5,7 @@ require "chronicle/etl/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "chronicle-etl"
-  spec.version       = Chronicle::Etl::VERSION
+  spec.version       = Chronicle::ETL::VERSION
   spec.authors       = ["Andrew Louis"]
   spec.email         = ["andrew@hyfen.net"]
 
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
 
     spec.metadata["homepage_uri"] = spec.homepage
     spec.metadata["source_code_uri"] = "https://github.com/chronicle-app/chronicle-etl"
-    spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-etl"
+    spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-etl/blob/master/CHANGELOG.md"
   else
     raise "RubyGems 2.0 or newer is required to protect against " \
       "public gem pushes."
@@ -38,11 +38,13 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "thor", "~> 0.20"
   spec.add_dependency "colorize", "~> 0.8.1"
-  spec.add_dependency "table_print"
-  spec.add_dependency "ruby-progressbar", "~> 1.10"
+  spec.add_dependency "tty-table", "~> 0.11"
+  spec.add_dependency "tty-progressbar", "~> 0.17"
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "pry-byebug", "~> 3.9"
+  spec.add_development_dependency 'runcom', '~> 6.2'
+  spec.add_development_dependency 'redcarpet', '~> 3.5'
 end

data/exe/chronicle-etl

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require "chronicle/etl/cli"
+require "chronicle/etl/cli/main"
 
-Chronicle::Etl::CLI.start(ARGV)
+Chronicle::ETL::CLI::Main.start(ARGV)

data/lib/chronicle/etl.rb

@@ -1,5 +1,8 @@
+require_relative 'etl/catalog'
+require_relative 'etl/config'
 require_relative 'etl/extractors/extractor'
-require_relative 'etl/transformers/transformer'
 require_relative 'etl/loaders/loader'
-require_relative 'etl/utils/progress_bar_wrapper'
 require_relative 'etl/runner'
+require_relative 'etl/transformers/transformer'
+require_relative 'etl/utils/progress_bar'
+require_relative 'etl/version'

data/lib/chronicle/etl/catalog.rb

@@ -0,0 +1,62 @@
+module Chronicle
+  module ETL
+    # Utility methods to catalogue which Extractor, Transformer, and
+    # Loader connector classes are available to chronicle-etl
+    module Catalog
+      PLUGINS = ['email', 'bash']
+
+      # Return which ETL connectors are available, both built in and externally-defined
+      def self.available_classes
+        # TODO: have a registry of plugins
+
+        # Attempt to load each chronicle plugin that we might know about so
+        # that we can later search for subclasses to build our list of
+        # available classes
+        PLUGINS.each do |plugin|
+          require "chronicle/#{plugin}"
+        rescue LoadError
+          # this will happen if the gem isn't available globally 
+        end
+
+        parent_klasses = [
+          ::Chronicle::ETL::Extractor,
+          ::Chronicle::ETL::Transformer,
+          ::Chronicle::ETL::Loader
+        ]
+        klasses = []
+        parent_klasses.map do |parent|
+          klasses += ::ObjectSpace.each_object(::Class).select { |klass| klass < parent }
+        end
+
+        klasses.map do |klass|
+          {
+            name: klass.name,
+            built_in: klass.built_in?,
+            provider: klass.provider,
+            phase: klass.phase
+          }
+        end
+      end
+
+      # Returns whether a class is an Extractor, Transformer, or Loader
+      def phase
+        ancestors = self.ancestors
+        return :extractor if ancestors.include? Chronicle::ETL::Extractor
+        return :transformer if ancestors.include? Chronicle::ETL::Transformer
+        return :loader if ancestors.include? Chronicle::ETL::Loader
+      end
+
+      # Returns which third-party provider this connector is associated wtih
+      def provider
+        # TODO: needs better convention for a gem reporting its provider name
+        provider = to_s.split('::')[1].downcase
+        provider == 'etl' ? 'chronicle' : provider
+      end
+
+      # Returns whether this connector is a built-in one
+      def built_in?
+        to_s.include? 'Chronicle::ETL'
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/connectors.rb

@@ -0,0 +1,32 @@
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL connectors
+      class Connectors < SubcommandBase
+        default_task 'list'
+        namespace :connectors
+
+        desc "install NAME", "Installs connector NAME"
+        def install
+          puts "Installing"
+        end
+
+        desc "list", "Lists available connectors"
+        # Display all available connectors that chronicle-etl has access to
+        def list
+          klasses = Chronicle::ETL::Catalog.available_classes
+          klasses = klasses.sort_by do |a|
+            [a[:built_in].to_s, a[:provider], a[:phase]]
+          end
+
+          headers = klasses.first.keys.map do |key|
+            key.to_s.upcase.bold
+          end
+
+          table = TTY::Table.new(headers, klasses.map(&:values))
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/jobs.rb

@@ -0,0 +1,111 @@
+require 'pp'
+require 'pry'
+
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL jobs
+      class Jobs < SubcommandBase
+        default_task "start"
+        namespace :jobs
+
+        class_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
+        class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
+        class_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
+        class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
+        class_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
+        class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
+        class_option :job, aliases: '-j', desc: 'Job configuration name (or filename)'
+
+        map run: :start  # Thor doesn't like `run` as a command name
+        desc "run", "Start a job"
+        long_desc <<-LONG_DESC
+          This will run an ETL job. Each job needs three parts:
+
+            1. #{'Extractor'.underline}: pulls data from an external source. By default, this is stdout. Other common options including pulling data from an API or reading JSON from a file.
+
+            2. #{'Transformer'.underline}: transforms data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
+
+            3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout.
+
+            If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
+LONG_DESC
+        # Run an ETL job
+        def start
+          runner_options = build_runner_options(options)
+          runner = Chronicle::ETL::Runner.new(runner_options)
+          runner.run!
+        end
+
+        desc "create", "Create a job"
+        # Create an ETL job
+        def create
+          runner_options = build_runner_options(options)
+          path = File.join('chronicle', 'etl', 'jobs', options[:job])
+          Chronicle::ETL::Config.write(path, runner_options)
+        end
+
+        desc "show", "Show details about a job"
+        # Show an ETL job
+        def show
+          runner_options = build_runner_options(options)
+          pp runner_options
+        end
+
+        desc "list", "List all available jobs"
+        # List available ETL jobs
+        def list
+          jobs = Chronicle::ETL::Config.jobs
+
+          job_details = jobs.map do |job|
+            r = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
+
+            extractor = r[:extractor][:name] if r[:extractor]
+            transformer = r[:transformer][:name] if r[:transformer]
+            loader = r[:loader][:name] if r[:loader]
+
+            [job, extractor, transformer, loader]
+          end
+
+          headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
+
+          table = TTY::Table.new(headers, job_details)
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+
+        private
+
+        # Create runner options by reading config file and then overwriting with flag options
+        def build_runner_options options
+          flag_options = process_flag_options(options)
+          job_options = load_job(options[:job])
+          flag_options.merge(job_options)
+        end
+
+        def load_job job
+          yml_config = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
+          # FIXME: use better trick to depely symbolize keys
+          JSON.parse(yml_config.to_json, symbolize_names: true)
+        end
+
+        # Takes flag options and turns them into a runner config
+        def process_flag_options options
+          {
+            extractor: {
+              name: options[:extractor],
+              options: options[:'extractor-opts']
+            },
+            transformer: {
+              name: options[:transformer],
+              options: options[:'transformer-opts']
+            },
+            loader: {
+              name: options[:loader],
+              options: options[:'loader-opts']
+            }
+          }
+        end
+      end
+    end
+  end
+end