chronicle-etl 0.1.4 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ab7fe84d09764034f061236d75fdf7403af9e1835d568831ae4896b90fc39b5
-  data.tar.gz: c4c7a1ff47ecaf7d364e35d3439eef1345f90fee5a6610735124592109e2c02c
+  metadata.gz: 7a02a2377d0e8d4135f3b931bc73641eac28058d736d9c1dba0a97107c1d4c0e
+  data.tar.gz: 810d5bff80e852fa08ef9824ed6b313aa309bb69e84228bc1fbb7595069e043b
 SHA512:
-  metadata.gz: a5a4b0e3769cd6063cb78843828ab15766df9d1ac1cd5d65fb81145d1415e95e46583c578312418fca91bfe02b05a8db3277a41a8a6b420f12263135deb49022
-  data.tar.gz: e6d6e23d3c164d6fc5b9283d468fe9c9a450329d2d778e3ad49962f42fbe0316299111cdd02d2abc92eb52dc46823c5cbaf1c195203fc4e78129e9b28528c5ef
+  metadata.gz: 0d5fbea3c63349bb3f566e6137755f6cc8a4060d0e401abf5a0e7d8b44a4c4278089c10ffb8bb9cf2d783a238449140e5e54d90f3ad158aa362c6335eedca5aa
+  data.tar.gz: bf6fa83b1d5e55760e62d3cc090bf09bb69a7c761ae4a9358fb4d82192c7efc7500b6db361f39adac3581982862654aa4603a78dfbb3aed53b51d01137ffd736

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  EnabledByDefault: true
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/MethodCallWithArgsParentheses:
+  Enabled: false

data/.yardopts

@@ -0,0 +1 @@
+--markup=markdown

data/Gemfile.lock

@@ -1,8 +1,10 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.1.4)
+    chronicle-etl (0.2.4)
       colorize (~> 0.8.1)
+      deep_merge (~> 1.2)
+      sequel (~> 5.35)
       thor (~> 0.20)
       tty-progressbar (~> 0.17)
       tty-table (~> 0.11)
@@ -13,6 +15,7 @@ GEM
     byebug (11.1.3)
     coderay (1.1.3)
     colorize (0.8.1)
+    deep_merge (1.2.1)
     diff-lcs (1.4.4)
     equatable (0.6.1)
     method_source (1.0.0)
@@ -27,6 +30,8 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.13.0)
     rake (13.0.1)
+    redcarpet (3.5.0)
+    refinements (7.7.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -40,6 +45,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    runcom (6.2.0)
+      refinements (~> 7.4)
+      xdg (~> 4.0)
+    sequel (5.36.0)
+    sqlite3 (1.4.2)
     strings (0.1.8)
       strings-ansi (~> 0.1)
       unicode-display_width (~> 1.5)
@@ -62,6 +72,7 @@ GEM
       tty-screen (~> 0.7)
     unicode-display_width (1.7.0)
     unicode_utils (1.4.0)
+    xdg (4.2.0)
 
 PLATFORMS
   ruby
@@ -71,7 +82,10 @@ DEPENDENCIES
   chronicle-etl!
   pry-byebug (~> 3.9)
   rake (~> 13.0)
+  redcarpet (~> 3.5)
   rspec (~> 3.9)
+  runcom (~> 6.2)
+  sqlite3 (~> 1.4)
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -1,10 +1,10 @@
-# Chronicle::Etl
+# Chronicle::ETL
 
 [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
 
-Chronicle ETL is a utility tool for archiving and processing personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+Chronicle ETL is a utility that helps you archive and processes personal data. You can *extract* it from a variety of sources, *transform* it, and *load* it to an external API, file, or stdout.
 
-This project is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex).
+This tool is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex) and the dozens of existing importers are being migrated to Chronicle.
 
 ## Installation
 
@@ -16,12 +16,24 @@ $ gem install chronicle-etl
 
 After installing the gem, `chronicle-etl` is available to run in your shell.
 
+```bash
+# read test.csv and display it as a table
+$ chronicle-etl jobs:run --extractor csv --extractor-opts filename:test.csv --loader table
+
+# Display help for the jobs:run command
+$ chronicle-etl jobs help run
 ```
-chronicle-etl --extractor csv --extractor-opts filename:test.csv --loader table
-cat test.csv | chronicle-etl --extractor csv --loader table
+
+## Connectors
+
+Connectors are available to read, process, and load data from different formats or external services.
+
+```bash
+# List all available connectors
+$ chronicle-etl connectors:list
 ```
 
-## Available importers
+Built in connectors:
 
 ### Extractors
 - `stdin` - (default) Load records from line-separated stdin
@@ -40,7 +52,7 @@ cat test.csv | chronicle-etl --extractor csv --loader table
 
 In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
 
-- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` files. Transformers for chronicle schema
+- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files. Transformers for chronicle schema
 - [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
 
 To install any of these, run `gem install chronicle-PROVIDER`. 
@@ -54,17 +66,23 @@ I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.ne
 ```
 $ chronicle-etl help 
 
-Commands:
-  chronicle-etl help [COMMAND]  # Describe available commands or one specific command
-  chronicle-etl job             # Runs an ETL job
-  chronicle-etl list            # List all ETL classes
+ALL COMMANDS
+  help                       # This help menu
+  connectors help [COMMAND]  # Describe subcommands or one specific subcommand
+  connectors:install NAME    # Installs connector NAME
+  connectors:list            # Lists available connectors
+  jobs help [COMMAND]        # Describe subcommands or one specific subcommand
+  jobs:create                # Create a job
+  jobs:list                  # List all available jobs
+  jobs:run                   # Start a job
+  jobs:show                  # Show a job
 ```
 
 ### Job options
 
 ```
 Usage:
-  chronicle-etl job
+  chronicle-etl jobs:run
 
 Options:
   -e, [--extractor=extractor-name]      # Extractor class (available: stdin, csv, file)
@@ -97,4 +115,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Chronicle::Etl project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).

data/chronicle-etl.gemspec

@@ -5,7 +5,7 @@ require "chronicle/etl/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "chronicle-etl"
-  spec.version       = Chronicle::Etl::VERSION
+  spec.version       = Chronicle::ETL::VERSION
   spec.authors       = ["Andrew Louis"]
   spec.email         = ["andrew@hyfen.net"]
 
@@ -40,9 +40,14 @@ Gem::Specification.new do |spec|
   spec.add_dependency "colorize", "~> 0.8.1"
   spec.add_dependency "tty-table", "~> 0.11"
   spec.add_dependency "tty-progressbar", "~> 0.17"
+  spec.add_dependency 'sequel', '~> 5.35'
+  spec.add_dependency 'deep_merge', '~> 1.2'
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "pry-byebug", "~> 3.9"
+  spec.add_development_dependency 'runcom', '~> 6.2'
+  spec.add_development_dependency 'redcarpet', '~> 3.5'
+  spec.add_development_dependency 'sqlite3', '~> 1.4'
 end

data/exe/chronicle-etl

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require "chronicle/etl/cli"
+require "chronicle/etl/cli/main"
 
-Chronicle::Etl::CLI.start(ARGV)
+Chronicle::ETL::CLI::Main.start(ARGV)

data/lib/chronicle/etl.rb

@@ -1,6 +1,19 @@
 require_relative 'etl/catalog'
+require_relative 'etl/config'
+require_relative 'etl/exceptions'
 require_relative 'etl/extractors/extractor'
-require_relative 'etl/transformers/transformer'
+require_relative 'etl/job_definition'
+require_relative 'etl/job_log'
+require_relative 'etl/job_logger'
+require_relative 'etl/job'
 require_relative 'etl/loaders/loader'
-require_relative 'etl/utils/progress_bar'
+require_relative 'etl/models/activity'
+require_relative 'etl/models/base'
+require_relative 'etl/models/entity'
+require_relative 'etl/models/generic'
 require_relative 'etl/runner'
+require_relative 'etl/transformers/transformer'
+require_relative 'etl/utils/hash_utilities'
+require_relative 'etl/utils/jsonapi'
+require_relative 'etl/utils/progress_bar'
+require_relative 'etl/version'

data/lib/chronicle/etl/catalog.rb

@@ -1,30 +1,37 @@
 module Chronicle
-  module Etl
+  module ETL
     # Utility methods to catalogue which Extractor, Transformer, and
-    # Loader classes are available to chronicle-etl
+    # Loader connector classes are available to chronicle-etl
     module Catalog
-      def self.available_classes
-        parent_klasses = [
-          Chronicle::Etl::Extractor,
-          Chronicle::Etl::Transformer,
-          Chronicle::Etl::Loader
-        ]
+      PHASES = [:extractor, :transformer, :loader]
+      PLUGINS = ['email', 'bash']
+      BUILTIN = {
+        extractor: ['stdin', 'json', 'csv', 'file'],
+        transformer: ['null'],
+        loader: ['stdout', 'csv', 'table', 'rest']
+      }.freeze
 
+      # Return which ETL connectors are available, both built in and externally-defined
+      def self.available_classes
         # TODO: have a registry of plugins
-        plugins = ['email', 'bash']
 
         # Attempt to load each chronicle plugin that we might know about so
         # that we can later search for subclasses to build our list of
         # available classes
-        plugins.each do |plugin|
+        PLUGINS.each do |plugin|
           require "chronicle/#{plugin}"
         rescue LoadError
-          # this will happen if the gem isn't available globally 
+          # this will happen if the gem isn't available globally
         end
 
+        parent_klasses = [
+          ::Chronicle::ETL::Extractor,
+          ::Chronicle::ETL::Transformer,
+          ::Chronicle::ETL::Loader
+        ]
         klasses = []
-        parent_klasses.each do |parent|
-          klasses += ObjectSpace.each_object(Class).select { |klass| klass < parent }
+        parent_klasses.map do |parent|
+          klasses += ::ObjectSpace.each_object(::Class).select { |klass| klass < parent }
         end
 
         klasses.map do |klass|
@@ -37,21 +44,64 @@ module Chronicle
         end
       end
 
+      # Take a phase (e, t, or l) and an identifier and return the right class
+      def self.phase_and_identifier_to_klass(phase, identifier)
+        Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
+      end
+
+      # For a given connector identifier, return the class (either builtin, or from a 
+      # external chronicle gem)
+      def self.identifier_to_klass(identifier:, phase:)
+        if BUILTIN[phase].include? identifier
+          load_builtin_klass(name: identifier, phase: phase)
+        else
+          provider, name = identifier.split(':')
+          name ||= ''
+          load_provider_klass(provider: provider, name: name, phase: phase)
+        end
+      end
+
+      # Returns whether a class is an Extractor, Transformer, or Loader
       def phase
         ancestors = self.ancestors
-        return :extractor if ancestors.include? Chronicle::Etl::Extractor
-        return :transformer if ancestors.include? Chronicle::Etl::Transformer
-        return :loader if ancestors.include? Chronicle::Etl::Loader
+        return :extractor if ancestors.include? Chronicle::ETL::Extractor
+        return :transformer if ancestors.include? Chronicle::ETL::Transformer
+        return :loader if ancestors.include? Chronicle::ETL::Loader
       end
 
+      # Returns which third-party provider this connector is associated wtih
       def provider
         # TODO: needs better convention for a gem reporting its provider name
         provider = to_s.split('::')[1].downcase
         provider == 'etl' ? 'chronicle' : provider
       end
 
+      # Returns whether this connector is a built-in one
       def built_in?
-        to_s.include? 'Chronicle::Etl'
+        to_s.include? 'Chronicle::ETL'
+      end
+
+      private
+
+      def self.load_builtin_klass(name:, phase:)
+        klass_str = "Chronicle::ETL::#{name.capitalize}#{phase.capitalize}"
+        begin
+          Object.const_get(klass_str)
+        rescue NameError => e
+          raise ConnectorNotAvailableError.new("Connector not found", name: name)
+        end
+      end
+
+      def self.load_provider_klass(name: '', phase:, provider:)
+        begin
+          require "chronicle/#{provider}"
+          klass_str = "Chronicle::#{provider.capitalize}::#{name.capitalize}#{phase.capitalize}"
+          Object.const_get(klass_str)
+        rescue LoadError => e
+          raise ProviderNotAvailableError.new("Provider '#{provider.capitalize}' could not be loaded", provider: provider)
+        rescue NameError => e
+          raise ProviderConnectorNotAvailableError.new("Connector '#{name}' in '#{provider}' could not be found", provider: provider, name: name)
+        end
       end
     end
   end

data/lib/chronicle/etl/cli/connectors.rb

@@ -0,0 +1,32 @@
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL connectors
+      class Connectors < SubcommandBase
+        default_task 'list'
+        namespace :connectors
+
+        desc "install NAME", "Installs connector NAME"
+        def install
+          puts "Installing"
+        end
+
+        desc "list", "Lists available connectors"
+        # Display all available connectors that chronicle-etl has access to
+        def list
+          klasses = Chronicle::ETL::Catalog.available_classes
+          klasses = klasses.sort_by do |a|
+            [a[:built_in].to_s, a[:provider], a[:phase]]
+          end
+
+          headers = klasses.first.keys.map do |key|
+            key.to_s.upcase.bold
+          end
+
+          table = TTY::Table.new(headers, klasses.map(&:values))
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/jobs.rb

@@ -0,0 +1,116 @@
+require 'pp'
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL jobs
+      class Jobs < SubcommandBase
+        default_task "start"
+        namespace :jobs
+
+        class_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
+        class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
+        class_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
+        class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
+        class_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
+        class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
+        class_option :name, aliases: '-j', desc: 'Job configuration name'
+
+        map run: :start  # Thor doesn't like `run` as a command name
+        desc "run", "Start a job"
+        long_desc <<-LONG_DESC
+          This will run an ETL job. Each job needs three parts:
+
+            1. #{'Extractor'.underline}: pulls data from an external source. By default, this is stdout. Other common options including pulling data from an API or reading JSON from a file.
+
+            2. #{'Transformer'.underline}: transforms data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
+
+            3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout.
+
+            If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
+LONG_DESC
+        # Run an ETL job
+        def start
+          job_definition = build_job_definition(options)
+          job = Chronicle::ETL::Job.new(job_definition)
+          runner = Chronicle::ETL::Runner.new(job)
+          runner.run!
+        rescue Chronicle::ETL::ProviderNotAvailableError => e
+          warn(e.message.red)
+          warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
+          exit(false)
+        rescue Chronicle::ETL::ConnectorNotAvailableError => e
+          warn(e.message.red)
+          exit(false)
+        end
+
+        desc "create", "Create a job"
+        # Create an ETL job
+        def create
+          job_definition = build_job_definition(options)
+          path = File.join('chronicle', 'etl', 'jobs', options[:name])
+          Chronicle::ETL::Config.write(path, job_definition)
+        end
+
+        desc "show", "Show details about a job"
+        # Show an ETL job
+        def show
+          job_config = build_job_definition(options)
+          pp job_config
+        end
+
+        desc "list", "List all available jobs"
+        # List available ETL jobs
+        def list
+          jobs = Chronicle::ETL::Config.available_jobs
+
+          job_details = jobs.map do |job|
+            r = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
+
+            extractor = r[:extractor][:name] if r[:extractor]
+            transformer = r[:transformer][:name] if r[:transformer]
+            loader = r[:loader][:name] if r[:loader]
+
+            [job, extractor, transformer, loader]
+          end
+
+          headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
+
+          table = TTY::Table.new(headers, job_details)
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+
+        private
+
+        # Create job definition by reading config file and then overwriting with flag options
+        def build_job_definition(options)
+          definition = Chronicle::ETL::JobDefinition.new
+          definition.add_config(process_flag_options(options))
+          definition.add_config(load_job_config(options[:name]))
+          definition
+        end
+
+        def load_job_config name
+          Chronicle::ETL::Config.load_job_from_config(name)
+        end
+
+        # Takes flag options and turns them into a runner config
+        def process_flag_options options
+          {
+            extractor: {
+              name: options[:extractor],
+              options: options[:'extractor-opts']
+            },
+            transformer: {
+              name: options[:transformer],
+              options: options[:'transformer-opts']
+            },
+            loader: {
+              name: options[:loader],
+              options: options[:'loader-opts']
+            }
+          }
+        end
+      end
+    end
+  end
+end