chronicle-etl 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ab7fe84d09764034f061236d75fdf7403af9e1835d568831ae4896b90fc39b5
-  data.tar.gz: c4c7a1ff47ecaf7d364e35d3439eef1345f90fee5a6610735124592109e2c02c
+  metadata.gz: 77abdc6f98e01300e0cf0ff4c8737fdc3bfe395754e9ddba1b6d3de86f2d6be8
+  data.tar.gz: 4a76565cfe9448b8ee7a6aa253a98923d7beccf02cc1ea7c8bacf3e7f7ab88ab
 SHA512:
-  metadata.gz: a5a4b0e3769cd6063cb78843828ab15766df9d1ac1cd5d65fb81145d1415e95e46583c578312418fca91bfe02b05a8db3277a41a8a6b420f12263135deb49022
-  data.tar.gz: e6d6e23d3c164d6fc5b9283d468fe9c9a450329d2d778e3ad49962f42fbe0316299111cdd02d2abc92eb52dc46823c5cbaf1c195203fc4e78129e9b28528c5ef
+  metadata.gz: cc6aaafde633a7316e26a23984f6a6965977e7bba388d2bb09751d3ef8ba0ed6a1ff78e50816af3a9806d5b03720fc524a270e228be0cc2894298acbbc342155
+  data.tar.gz: 32dd999d57c307b9e57db3b877362b0e780f8a618d5c573a5295241ec47dee65472b1e98414e4cb0959d978f83cb7cfde1177c9fb1d11c54e5edf1ca5ba4419a

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  EnabledByDefault: true
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/MethodCallWithArgsParentheses:
+  Enabled: false

data/.yardopts

@@ -0,0 +1 @@
+--markup=markdown

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.1.4)
+    chronicle-etl (0.2.0)
       colorize (~> 0.8.1)
       thor (~> 0.20)
       tty-progressbar (~> 0.17)
@@ -27,6 +27,8 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.13.0)
     rake (13.0.1)
+    redcarpet (3.5.0)
+    refinements (7.7.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -40,6 +42,9 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    runcom (6.2.0)
+      refinements (~> 7.4)
+      xdg (~> 4.0)
     strings (0.1.8)
       strings-ansi (~> 0.1)
       unicode-display_width (~> 1.5)
@@ -62,6 +67,7 @@ GEM
       tty-screen (~> 0.7)
     unicode-display_width (1.7.0)
     unicode_utils (1.4.0)
+    xdg (4.2.0)
 
 PLATFORMS
   ruby
@@ -71,7 +77,9 @@ DEPENDENCIES
   chronicle-etl!
   pry-byebug (~> 3.9)
   rake (~> 13.0)
+  redcarpet (~> 3.5)
   rspec (~> 3.9)
+  runcom (~> 6.2)
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -1,4 +1,4 @@
-# Chronicle::Etl
+# Chronicle::ETL
 
 [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
 
@@ -16,12 +16,24 @@ $ gem install chronicle-etl
 
 After installing the gem, `chronicle-etl` is available to run in your shell.
 
+```bash
+# read test.csv and display it as a table
+$ chronicle-etl jobs:run --extractor csv --extractor-opts filename:test.csv --loader table
+
+# Display help for the jobs:run command
+$ chronicle-etl jobs help run
 ```
-chronicle-etl --extractor csv --extractor-opts filename:test.csv --loader table
-cat test.csv | chronicle-etl --extractor csv --loader table
+
+## Connectors
+
+Connectors are available to read, process, and load data from different formats or external services.
+
+```bash
+# List all available connectors
+$ chronicle-etl connectors:list
 ```
 
-## Available importers
+Built in connectors:
 
 ### Extractors
 - `stdin` - (default) Load records from line-separated stdin
@@ -54,17 +66,23 @@ I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.ne
 ```
 $ chronicle-etl help 
 
-Commands:
-  chronicle-etl help [COMMAND]  # Describe available commands or one specific command
-  chronicle-etl job             # Runs an ETL job
-  chronicle-etl list            # List all ETL classes
+ALL COMMANDS
+  help                       # This help menu
+  connectors help [COMMAND]  # Describe subcommands or one specific subcommand
+  connectors:install NAME    # Installs connector NAME
+  connectors:list            # Lists available connectors
+  jobs help [COMMAND]        # Describe subcommands or one specific subcommand
+  jobs:create                # Create a job
+  jobs:list                  # List all available jobs
+  jobs:run                   # Start a job
+  jobs:show                  # Show a job
 ```
 
 ### Job options
 
 ```
 Usage:
-  chronicle-etl job
+  chronicle-etl jobs:run
 
 Options:
   -e, [--extractor=extractor-name]      # Extractor class (available: stdin, csv, file)
@@ -97,4 +115,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Chronicle::Etl project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).

data/chronicle-etl.gemspec

@@ -5,7 +5,7 @@ require "chronicle/etl/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "chronicle-etl"
-  spec.version       = Chronicle::Etl::VERSION
+  spec.version       = Chronicle::ETL::VERSION
   spec.authors       = ["Andrew Louis"]
   spec.email         = ["andrew@hyfen.net"]
 
@@ -45,4 +45,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "pry-byebug", "~> 3.9"
+  spec.add_development_dependency 'runcom', '~> 6.2'
+  spec.add_development_dependency 'redcarpet', '~> 3.5'
 end

data/exe/chronicle-etl

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require "chronicle/etl/cli"
+require "chronicle/etl/cli/main"
 
-Chronicle::Etl::CLI.start(ARGV)
+Chronicle::ETL::CLI::Main.start(ARGV)

data/lib/chronicle/etl.rb

@@ -1,6 +1,8 @@
 require_relative 'etl/catalog'
+require_relative 'etl/config'
 require_relative 'etl/extractors/extractor'
-require_relative 'etl/transformers/transformer'
 require_relative 'etl/loaders/loader'
-require_relative 'etl/utils/progress_bar'
 require_relative 'etl/runner'
+require_relative 'etl/transformers/transformer'
+require_relative 'etl/utils/progress_bar'
+require_relative 'etl/version'

data/lib/chronicle/etl/catalog.rb

@@ -1,30 +1,31 @@
 module Chronicle
-  module Etl
+  module ETL
     # Utility methods to catalogue which Extractor, Transformer, and
-    # Loader classes are available to chronicle-etl
+    # Loader connector classes are available to chronicle-etl
     module Catalog
-      def self.available_classes
-        parent_klasses = [
-          Chronicle::Etl::Extractor,
-          Chronicle::Etl::Transformer,
-          Chronicle::Etl::Loader
-        ]
+      PLUGINS = ['email', 'bash']
 
+      # Return which ETL connectors are available, both built in and externally-defined
+      def self.available_classes
         # TODO: have a registry of plugins
-        plugins = ['email', 'bash']
 
         # Attempt to load each chronicle plugin that we might know about so
         # that we can later search for subclasses to build our list of
         # available classes
-        plugins.each do |plugin|
+        PLUGINS.each do |plugin|
           require "chronicle/#{plugin}"
         rescue LoadError
           # this will happen if the gem isn't available globally 
         end
 
+        parent_klasses = [
+          ::Chronicle::ETL::Extractor,
+          ::Chronicle::ETL::Transformer,
+          ::Chronicle::ETL::Loader
+        ]
         klasses = []
-        parent_klasses.each do |parent|
-          klasses += ObjectSpace.each_object(Class).select { |klass| klass < parent }
+        parent_klasses.map do |parent|
+          klasses += ::ObjectSpace.each_object(::Class).select { |klass| klass < parent }
         end
 
         klasses.map do |klass|
@@ -37,21 +38,24 @@ module Chronicle
         end
       end
 
+      # Returns whether a class is an Extractor, Transformer, or Loader
       def phase
         ancestors = self.ancestors
-        return :extractor if ancestors.include? Chronicle::Etl::Extractor
-        return :transformer if ancestors.include? Chronicle::Etl::Transformer
-        return :loader if ancestors.include? Chronicle::Etl::Loader
+        return :extractor if ancestors.include? Chronicle::ETL::Extractor
+        return :transformer if ancestors.include? Chronicle::ETL::Transformer
+        return :loader if ancestors.include? Chronicle::ETL::Loader
       end
 
+      # Returns which third-party provider this connector is associated wtih
       def provider
         # TODO: needs better convention for a gem reporting its provider name
         provider = to_s.split('::')[1].downcase
         provider == 'etl' ? 'chronicle' : provider
       end
 
+      # Returns whether this connector is a built-in one
       def built_in?
-        to_s.include? 'Chronicle::Etl'
+        to_s.include? 'Chronicle::ETL'
       end
     end
   end

data/lib/chronicle/etl/cli/connectors.rb

@@ -0,0 +1,32 @@
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL connectors
+      class Connectors < SubcommandBase
+        default_task 'list'
+        namespace :connectors
+
+        desc "install NAME", "Installs connector NAME"
+        def install
+          puts "Installing"
+        end
+
+        desc "list", "Lists available connectors"
+        # Display all available connectors that chronicle-etl has access to
+        def list
+          klasses = Chronicle::ETL::Catalog.available_classes
+          klasses = klasses.sort_by do |a|
+            [a[:built_in].to_s, a[:provider], a[:phase]]
+          end
+
+          headers = klasses.first.keys.map do |key|
+            key.to_s.upcase.bold
+          end
+
+          table = TTY::Table.new(headers, klasses.map(&:values))
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/jobs.rb

@@ -0,0 +1,111 @@
+require 'pp'
+require 'pry'
+
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL jobs
+      class Jobs < SubcommandBase
+        default_task "start"
+        namespace :jobs
+
+        class_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
+        class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
+        class_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
+        class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
+        class_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
+        class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
+        class_option :job, aliases: '-j', desc: 'Job configuration name (or filename)'
+
+        map run: :start  # Thor doesn't like `run` as a command name
+        desc "run", "Start a job"
+        long_desc <<-LONG_DESC
+          This will run an ETL job. Each job needs three parts:
+
+            1. #{'Extractor'.underline}: pulls data from an external source. By default, this is stdout. Other common options including pulling data from an API or reading JSON from a file.
+
+            2. #{'Transformer'.underline}: transforms data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
+
+            3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout.
+
+            If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
+LONG_DESC
+        # Run an ETL job
+        def start
+          runner_options = build_runner_options(options)
+          runner = Chronicle::ETL::Runner.new(runner_options)
+          runner.run!
+        end
+
+        desc "create", "Create a job"
+        # Create an ETL job
+        def create
+          runner_options = build_runner_options(options)
+          path = File.join('chronicle', 'etl', 'jobs', options[:job])
+          Chronicle::ETL::Config.write(path, runner_options)
+        end
+
+        desc "show", "Show details about a job"
+        # Show an ETL job
+        def show
+          runner_options = build_runner_options(options)
+          pp runner_options
+        end
+
+        desc "list", "List all available jobs"
+        # List available ETL jobs
+        def list
+          jobs = Chronicle::ETL::Config.jobs
+
+          job_details = jobs.map do |job|
+            r = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
+
+            extractor = r[:extractor][:name] if r[:extractor]
+            transformer = r[:transformer][:name] if r[:transformer]
+            loader = r[:loader][:name] if r[:loader]
+
+            [job, extractor, transformer, loader]
+          end
+
+          headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
+
+          table = TTY::Table.new(headers, job_details)
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+
+        private
+
+        # Create runner options by reading config file and then overwriting with flag options
+        def build_runner_options options
+          flag_options = process_flag_options(options)
+          job_options = load_job(options[:job])
+          flag_options.merge(job_options)
+        end
+
+        def load_job job
+          yml_config = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
+          # FIXME: use better trick to depely symbolize keys
+          JSON.parse(yml_config.to_json, symbolize_names: true)
+        end
+
+        # Takes flag options and turns them into a runner config
+        def process_flag_options options
+          {
+            extractor: {
+              name: options[:extractor],
+              options: options[:'extractor-opts']
+            },
+            transformer: {
+              name: options[:transformer],
+              options: options[:'transformer-opts']
+            },
+            loader: {
+              name: options[:loader],
+              options: options[:'loader-opts']
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/main.rb

@@ -0,0 +1,83 @@
+require 'thor'
+require 'chronicle/etl'
+require 'colorize'
+
+require 'chronicle/etl/cli/subcommand_base'
+require 'chronicle/etl/cli/connectors'
+require 'chronicle/etl/cli/jobs'
+
+module Chronicle
+  module ETL
+    module CLI
+      # Main entrypoint for CLI app
+      class Main < Thor
+        class_option "verbose", type: :boolean, default: false
+        default_task "jobs"
+
+        desc 'connectors:COMMAND', 'Connectors available for ETL jobs', hide: true
+        subcommand 'connectors', Connectors
+
+        desc 'jobs:COMMAND', 'Configure and run jobs', hide: true
+        subcommand 'jobs', Jobs
+
+        # Entrypoint for the CLI
+        def self.start(given_args = ARGV, config = {})
+          if given_args.none?
+            abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
+          end
+
+          # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class
+          if given_args.any? && given_args[0].include?(':')
+            commands = given_args.shift.split(':')
+            given_args = given_args.unshift(commands).flatten
+          end
+
+          super(given_args, config)
+        end
+
+        # Displays help options for chronicle-etl
+        def help(meth = nil, subcommand = false)
+          if meth && !respond_to?(meth)
+            klass, task = Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
+            klass.start(['-h', task].compact, shell: shell)
+          else
+            shell.say "ABOUT".bold
+            shell.say "  #{'chronicle-etl'.italic} is a utility tool for #{'extracting'.underline}, #{'transforming'.underline}, and #{'loading'.underline} personal data."
+            shell.say
+            shell.say "USAGE".bold
+            shell.say "  $ chronicle-etl COMMAND"
+            shell.say
+            shell.say "EXAMPLES".bold
+            shell.say "  Show available connectors:".italic.light_black
+            shell.say "  $ chronicle-etl connectors:list"
+            shell.say
+            shell.say "  Run a simple job:".italic.light_black
+            shell.say "  $ chronicle-etl jobs:start --extractor stdin --transformer null --loader stdout"
+            shell.say
+            shell.say "  Show full job options:".italic.light_black
+            shell.say "  $ chronicle-etl jobs help start"
+
+            list = []
+
+            Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
+              list += thor_class.printable_tasks(false)
+            end
+            list.sort! { |a, b| a[0] <=> b[0] }
+            list.unshift ["help", "# This help menu"]
+
+            shell.say
+            shell.say 'ALL COMMANDS'.bold
+            shell.print_table(list, indent: 2, truncate: true)
+            shell.say
+            shell.say "VERSION".bold
+            shell.say "  #{Chronicle::ETL::VERSION}"
+            shell.say
+            shell.say "FULL DOCUMENTATION".bold
+            shell.say "  https://github.com/chronicle-app/chronicle-etl".blue
+            shell.say
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -0,0 +1,37 @@
+module Chronicle
+  module ETL
+    module CLI
+      # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
+      class SubcommandBase < Thor
+        # Print usage instructions for a subcommand
+        def self.help(shell, subcommand = false)
+          list = printable_commands(true, subcommand)
+          Thor::Util.thor_classes_in(self).each do |klass|
+            list += klass.printable_commands(false)
+          end
+          list.sort! { |a, b| a[0] <=> b[0] }
+
+          shell.say "COMMANDS".bold
+          shell.print_table(list, indent: 2, truncate: true)
+          shell.say
+          class_options_help(shell)
+        end
+
+        # Show docs with command:subcommand pattern.
+        # For `help` command, don't use colon
+        def self.banner(command, namespace = nil, subcommand = false)
+          if command.name == 'help'
+            "#{subcommand_prefix} #{command.usage}"
+          else
+            "#{subcommand_prefix}:#{command.usage}"
+          end
+        end
+
+        # Use subcommand classname to derive display name for subcommand
+        def self.subcommand_prefix
+          self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/config.rb

@@ -0,0 +1,32 @@
+require 'runcom'
+
+module Chronicle
+  module ETL
+    # Utility methods to read, write, and access config files
+    module Config
+      # Loads a yml config file
+      def self.load(path)
+        config = Runcom::Config.new(path)
+        # FIXME: hack to deeply symbolize keys
+        JSON.parse(config.to_h.to_json, symbolize_names: true)
+      end
+
+      # Writes a hash as a yml config file
+      def self.write(path, data)
+        config = Runcom::Config.new(path)
+        filename = config.all[0].to_s + '.yml'
+        File.open(filename, 'w') do |f|
+          f << data.to_yaml
+        end
+      end
+
+      # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
+      def self.jobs
+        job_directory = Runcom::Config.new('chronicle/etl/jobs').current
+        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+          File.basename(filename, ".*")
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -1,5 +1,5 @@
 require 'csv'
-class Chronicle::Etl::CsvExtractor < Chronicle::Etl::Extractor
+class Chronicle::ETL::CsvExtractor < Chronicle::ETL::Extractor
   DEFAULT_OPTIONS = {
     headers: true,
     filename: $stdin

data/lib/chronicle/etl/extractors/extractor.rb

@@ -1,20 +1,26 @@
 require 'chronicle/etl'
 
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing an Extractor for an ETL job
     class Extractor
-      extend Chronicle::Etl::Catalog
-
-      ETL_PHASE = :extractor
+      extend Chronicle::ETL::Catalog
 
+      # Construct a new instance of this extractor. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Extractor
       def initialize(options = {})
-        @options = options.transform_keys!(&:to_sym) 
+        @options = options.transform_keys!(&:to_sym)
       end
 
+      # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
       def extract
         raise NotImplementedError
       end
 
+      # An optional method to calculate how many records there are to extract. Used primarily for
+      # building the progress bar
       def results_count; end
     end
   end

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -1,8 +1,8 @@
 require 'pathname'
 
 module Chronicle
-  module Etl
-    class FileExtractor < Chronicle::Etl::Extractor
+  module ETL
+    class FileExtractor < Chronicle::ETL::Extractor
       def extract
         if file?
           extract_file do |data, metadata|

data/lib/chronicle/etl/extractors/stdin_extractor.rb

@@ -1,6 +1,6 @@
 module Chronicle
-  module Etl
-    class StdinExtractor < Chronicle::Etl::Extractor
+  module ETL
+    class StdinExtractor < Chronicle::ETL::Extractor
       def extract
         $stdin.read.each_line do |line|
           yield line

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -1,8 +1,8 @@
 require 'csv'
 
 module Chronicle
-  module Etl
-    class CsvLoader < Chronicle::Etl::Loader
+  module ETL
+    class CsvLoader < Chronicle::ETL::Loader
       def initialize(options={})
         super(options)
         @rows = []

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,18 +1,26 @@
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing a Loader for an ETL job
     class Loader
-      extend Chronicle::Etl::Catalog
+      extend Chronicle::ETL::Catalog
 
+      # Construct a new instance of this loader. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Loader
       def initialize(options = {})
         @options = options
       end
 
+      # Called once before processing records
       def start; end
 
+      # Load a single record
       def load
         raise NotImplementedError
       end
 
+      # Called once there are no more records to process
       def finish; end
     end
   end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,6 +1,6 @@
 module Chronicle
-  module Etl
-    class StdoutLoader < Chronicle::Etl::Loader
+  module ETL
+    class StdoutLoader < Chronicle::ETL::Loader
       def load(result)
         puts result.inspect
       end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,8 +1,8 @@
 require 'tty/table'
 
 module Chronicle
-  module Etl
-    class TableLoader < Chronicle::Etl::Loader
+  module ETL
+    class TableLoader < Chronicle::ETL::Loader
       def initialize(options)
         super(options)
       end

data/lib/chronicle/etl/runner.rb

@@ -1,4 +1,4 @@
-class Chronicle::Etl::Runner
+class Chronicle::ETL::Runner
   BUILTIN = {
     extractor: ['stdin', 'json', 'csv', 'file'],
     transformer: ['null'],
@@ -13,7 +13,7 @@ class Chronicle::Etl::Runner
 
   def run!
     total = @extractor.results_count
-    progress_bar = Chronicle::Etl::Utils::ProgressBar.new(title: 'Running job', total: total)
+    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
     count = 0
 
     @loader.start
@@ -40,7 +40,7 @@ class Chronicle::Etl::Runner
 
   def load_etl_class(phase, x)
     if BUILTIN[phase].include? x
-      klass_name = "Chronicle::Etl::#{x.capitalize}#{phase.to_s.capitalize}"
+      klass_name = "Chronicle::ETL::#{x.capitalize}#{phase.to_s.capitalize}"
     else
       # TODO: come up with syntax for specifying a particular extractor in a provider library
       provider, name = x.split(":")
@@ -48,7 +48,7 @@ class Chronicle::Etl::Runner
       begin
         require "chronicle/#{provider}"
       rescue LoadError => e
-        warn("Error loading #{phase} '#{provider}'")
+        warn("Error loading #{phase} '#{provider}'".red)
         warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
         exit(false)
       end

data/lib/chronicle/etl/transformers/json_transformer.rb

@@ -1,8 +1,8 @@
 require 'json'
 
 module Chronicle
-  module Etl
-    class JsonTransformer < Chronicle::Etl::Transformer
+  module ETL
+    class JsonTransformer < Chronicle::ETL::Transformer
       def transform data
         return JSON.parse(data)
       end

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -1,6 +1,6 @@
 module Chronicle
-  module Etl
-    class NullTransformer < Chronicle::Etl::Transformer
+  module ETL
+    class NullTransformer < Chronicle::ETL::Transformer
       def transform data
         return data
       end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -1,15 +1,31 @@
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing an Transformer for an ETL job
     class Transformer
-      extend Chronicle::Etl::Catalog
+      extend Chronicle::ETL::Catalog
 
+      # Construct a new instance of this transformer. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Transformer
       def initialize(options = {})
         @options = options
       end
 
+      # The main entrypoint for transforming a record. Called by a Runner on each extracted record
       def transform data
         raise NotImplementedError
       end
+
+      # The domain or provider-specific id of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def id; end
+
+      # The domain or provider-specific timestamp of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def timestamp; end
     end
   end
 end

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -2,7 +2,7 @@ require 'tty/progressbar'
 require 'colorize'
 
 module Chronicle
-  module Etl
+  module ETL
     module Utils
 
       class ProgressBar

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
-  module Etl
-    VERSION = "0.1.4"
+  module ETL
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-18 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -122,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: runcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -133,8 +161,10 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
+- ".yardopts"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -148,7 +178,11 @@ files:
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
 - lib/chronicle/etl/catalog.rb
-- lib/chronicle/etl/cli.rb
+- lib/chronicle/etl/cli/connectors.rb
+- lib/chronicle/etl/cli/jobs.rb
+- lib/chronicle/etl/cli/main.rb
+- lib/chronicle/etl/cli/subcommand_base.rb
+- lib/chronicle/etl/config.rb
 - lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
 - lib/chronicle/etl/extractors/file_extractor.rb

data/lib/chronicle/etl/cli.rb

@@ -1,56 +0,0 @@
-require 'thor'
-require 'chronicle/etl'
-require 'colorize'
-
-module Chronicle
-  module Etl
-    class CLI < Thor
-      default_task :job
-
-      desc 'job', 'Runs an ETL job'
-      method_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
-      method_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
-      method_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
-      method_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
-      method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
-      method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-      method_option :job, aliases: '-j', desc: 'Job configuration file'
-
-      def job
-        runner_options = {
-          extractor: {
-            name: options[:extractor],
-            options: options[:'extractor-opts']
-          },
-          transformer: {
-            name: options[:transformer],
-            options: options[:'transformer-opts']
-          },
-          loader: {
-            name: options[:loader],
-            options: options[:'loader-opts']
-          }
-        }
-
-        runner = Runner.new(runner_options)
-        runner.run!
-      end
-
-      # FIXME: namespace this differently
-      desc 'list', 'List all ETL classes'
-      def list
-        klasses = Chronicle::Etl::Catalog.available_classes
-        klasses = klasses.sort_by do |a|
-          [a[:built_in].to_s, a[:provider], a[:phase]]
-        end
-
-        headers = klasses.first.keys.map do |key|
-          key.to_s.capitalize.light_white
-        end
-
-        table = TTY::Table.new(headers, klasses.map(&:values))
-        puts table.render(padding: [0, 2])
-      end
-    end
-  end
-end