chronicle-etl 0.1.2 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a8f688143a1057176324084919170a3351007cf619489d3ab76914c527bf15a
-  data.tar.gz: 2c7cfeeb279274f66b9eaccbe8a5f17fdf67453b82590d55a0df8f1cde8e748b
+  metadata.gz: e1c08bc4f71c807525090abbf1701be19ab72cce08a99cc3bbec9b0db7150a02
+  data.tar.gz: 172a5d7e7ba7a9424ef7b5ab4da2b8c44defdb4e0a34c833248ff1b63f40407e
 SHA512:
-  metadata.gz: 2882b8daef2dc427fdf0ea517504f35fe286f19a9040319f6884461577d978d963c92aa19c613b92dc55e5a8edc0271396450addaf772dceeb0ff4d5ed38a30f
-  data.tar.gz: 2f45c5b4d1e896e82a514215032cfb6b0a9802dacd2be0b9e57b3bd27a5f13b40a392e1577081d61aa219aa4c4e5c7e97744d8f371031fb2b871b27f68094dd3
+  metadata.gz: 0f671c00928b15f9c0f6fa159ac106ff9c4f65a8bd16048e5d0cab82d680945317f7680e7796e98c665bb5cc757e0657f1a36d773d89e3e1587d9eebc12abdd8
+  data.tar.gz: 449d1368e0054f39006c7903218300b9b97ca839d6eff43b6b7bd659e5146d443a31c53325c4769ae7a56db9d42417020ccde17362ae024c01aca2ed63029044

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  EnabledByDefault: true
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/MethodCallWithArgsParentheses:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.7.1

data/.yardopts

@@ -0,0 +1 @@
+--markup=markdown

data/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.4] - 2020-08-18
+### Updated
+- Better display of available ETL classes
+- Updated documentation
+
+## [0.1.3] - 2020-08-13
+### Added
+- Ability to list all available ETL classes
+- Refactored E, T, L module and class structure
+- Better progress bar
+
 ## [0.1.2] - 2020-08-02
 ### Added
 - This changelog

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.1.2)
+    chronicle-etl (0.2.2)
       colorize (~> 0.8.1)
-      ruby-progressbar (~> 1.10)
-      table_print
       thor (~> 0.20)
+      tty-progressbar (~> 0.17)
+      tty-table (~> 0.11)
 
 GEM
   remote: https://rubygems.org/
@@ -14,14 +14,21 @@ GEM
     coderay (1.1.3)
     colorize (0.8.1)
     diff-lcs (1.4.4)
+    equatable (0.6.1)
     method_source (1.0.0)
+    necromancer (0.6.0)
+    pastel (0.7.4)
+      equatable (~> 0.6)
+      tty-color (~> 0.5)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.9.0)
       byebug (~> 11.0)
       pry (~> 0.13.0)
-    rake (10.5.0)
+    rake (13.0.1)
+    redcarpet (3.5.0)
+    refinements (7.7.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -35,19 +42,44 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    ruby-progressbar (1.10.1)
-    table_print (1.5.7)
+    runcom (6.2.0)
+      refinements (~> 7.4)
+      xdg (~> 4.0)
+    strings (0.1.8)
+      strings-ansi (~> 0.1)
+      unicode-display_width (~> 1.5)
+      unicode_utils (~> 1.4)
+    strings-ansi (0.1.0)
     thor (0.20.3)
+    tty-color (0.5.2)
+    tty-cursor (0.7.1)
+    tty-progressbar (0.17.0)
+      strings-ansi (~> 0.1.0)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.7)
+      unicode-display_width (~> 1.6)
+    tty-screen (0.8.1)
+    tty-table (0.11.0)
+      equatable (~> 0.6)
+      necromancer (~> 0.5)
+      pastel (~> 0.7.2)
+      strings (~> 0.1.5)
+      tty-screen (~> 0.7)
+    unicode-display_width (1.7.0)
+    unicode_utils (1.4.0)
+    xdg (4.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
+  bundler (~> 2.1)
   chronicle-etl!
   pry-byebug (~> 3.9)
-  rake (~> 10.0)
-  rspec (~> 3.0)
+  rake (~> 13.0)
+  redcarpet (~> 3.5)
+  rspec (~> 3.9)
+  runcom (~> 6.2)
 
 BUNDLED WITH
-   1.17.2
+   2.1.4

data/README.md

@@ -1,6 +1,10 @@
-# Chronicle::Etl
+# Chronicle::ETL
 
-Chronicle ETL is a utility tool for manipulating personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
+
+Chronicle ETL is a utility tool for archiving and processing personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+
+This project is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex).
 
 ## Installation
 
@@ -8,28 +12,77 @@ Chronicle ETL is a utility tool for manipulating personal data. You can extract
 $ gem install chronicle-etl
 ```
 
-## Examples
+## Usage
 
 After installing the gem, `chronicle-etl` is available to run in your shell.
 
+```bash
+# read test.csv and display it as a table
+$ chronicle-etl jobs:run --extractor csv --extractor-opts filename:test.csv --loader table
+
+# Display help for the jobs:run command
+$ chronicle-etl jobs help run
 ```
-chronicle-etl --extractor csv --extractor-opts filename:test.csv --loader table
-cat test.csv | chronicle-etl --extractor csv --loader table
+
+## Connectors
+
+Connectors are available to read, process, and load data from different formats or external services.
+
+```bash
+# List all available connectors
+$ chronicle-etl connectors:list
 ```
 
-## Full usage
+Built in connectors:
+
+### Extractors
+- `stdin` - (default) Load records from line-separated stdin
+- `csv`
+- `file` - load from a single file or directory (with a glob pattern)
+
+### Transformers
+- `null` - (default) Don't do anything
+
+### Loaders
+- `stdout` - (default) output transformed records to stdount
+- `csv` - Load records to a csv file
+- `table` - Output an ascii table of records. Useful for debugging.
+
+### Provider-specific importers
+
+In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
+
+- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` files. Transformers for chronicle schema
+- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
+
+To install any of these, run `gem install chronicle-PROVIDER`. 
+
+If you don't want to use the available rubygem importers, `chronicle-etl` can use `stdin` as an Extractor source (newline separated records). You can also use `stdout` as a loader — transformed records will be outputted separated by newlines.
+
+I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.net) to chat about what will be available!
+
+### Full commands
 
 ```
-Commands:
-  chronicle-etl help [COMMAND]  # Describe available commands or one specific command
-  chronicle-etl job             # Runs an ETL job
+$ chronicle-etl help 
+
+ALL COMMANDS
+  help                       # This help menu
+  connectors help [COMMAND]  # Describe subcommands or one specific subcommand
+  connectors:install NAME    # Installs connector NAME
+  connectors:list            # Lists available connectors
+  jobs help [COMMAND]        # Describe subcommands or one specific subcommand
+  jobs:create                # Create a job
+  jobs:list                  # List all available jobs
+  jobs:run                   # Start a job
+  jobs:show                  # Show a job
 ```
 
 ### Job options
 
 ```
 Usage:
-  chronicle-etl job
+  chronicle-etl jobs:run
 
 Options:
   -e, [--extractor=extractor-name]      # Extractor class (available: stdin, csv, file)
@@ -62,4 +115,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Chronicle::Etl project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).

data/bin/console

@@ -7,8 +7,20 @@ require "chronicle/etl"
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
+require "pry"
+Pry.start
+
+def reload!(print = true)
+  puts 'Reloading ...' if print
+  # Main project directory.
+  root_dir = File.expand_path('..', __dir__)
+  # Directories within the project that should be reloaded.
+  reload_dirs = %w{lib}
+  # Loop through and reload every file in all relevant project directories.
+  reload_dirs.each do |dir|
+    Dir.glob("#{root_dir}/#{dir}/**/*.rb").each { |f| load(f) }
+  end
+  # Return true when complete.
+  true
+end
 
-require "irb"
-IRB.start(__FILE__)

data/chronicle-etl.gemspec

@@ -5,7 +5,7 @@ require "chronicle/etl/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "chronicle-etl"
-  spec.version       = Chronicle::Etl::VERSION
+  spec.version       = Chronicle::ETL::VERSION
   spec.authors       = ["Andrew Louis"]
   spec.email         = ["andrew@hyfen.net"]
 
@@ -38,11 +38,13 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "thor", "~> 0.20"
   spec.add_dependency "colorize", "~> 0.8.1"
-  spec.add_dependency "table_print"
-  spec.add_dependency "ruby-progressbar", "~> 1.10"
+  spec.add_dependency "tty-table", "~> 0.11"
+  spec.add_dependency "tty-progressbar", "~> 0.17"
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "pry-byebug", "~> 3.9"
+  spec.add_development_dependency 'runcom', '~> 6.2'
+  spec.add_development_dependency 'redcarpet', '~> 3.5'
 end

data/exe/chronicle-etl

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require "chronicle/etl/cli"
+require "chronicle/etl/cli/main"
 
-Chronicle::Etl::CLI.start(ARGV)
+Chronicle::ETL::CLI::Main.start(ARGV)

data/lib/chronicle/etl.rb

@@ -1,5 +1,9 @@
+require_relative 'etl/catalog'
+require_relative 'etl/config'
+require_relative 'etl/exceptions'
 require_relative 'etl/extractors/extractor'
-require_relative 'etl/transformers/transformer'
 require_relative 'etl/loaders/loader'
-require_relative 'etl/utils/progress_bar_wrapper'
 require_relative 'etl/runner'
+require_relative 'etl/transformers/transformer'
+require_relative 'etl/utils/progress_bar'
+require_relative 'etl/version'

data/lib/chronicle/etl/catalog.rb

@@ -0,0 +1,102 @@
+module Chronicle
+  module ETL
+    # Utility methods to catalogue which Extractor, Transformer, and
+    # Loader connector classes are available to chronicle-etl
+    module Catalog
+      PLUGINS = ['email', 'bash']
+      BUILTIN = {
+        extractor: ['stdin', 'json', 'csv', 'file'],
+        transformer: ['null'],
+        loader: ['stdout', 'csv', 'table', 'rest']
+      }.freeze
+
+      # Return which ETL connectors are available, both built in and externally-defined
+      def self.available_classes
+        # TODO: have a registry of plugins
+
+        # Attempt to load each chronicle plugin that we might know about so
+        # that we can later search for subclasses to build our list of
+        # available classes
+        PLUGINS.each do |plugin|
+          require "chronicle/#{plugin}"
+        rescue LoadError
+          # this will happen if the gem isn't available globally
+        end
+
+        parent_klasses = [
+          ::Chronicle::ETL::Extractor,
+          ::Chronicle::ETL::Transformer,
+          ::Chronicle::ETL::Loader
+        ]
+        klasses = []
+        parent_klasses.map do |parent|
+          klasses += ::ObjectSpace.each_object(::Class).select { |klass| klass < parent }
+        end
+
+        klasses.map do |klass|
+          {
+            name: klass.name,
+            built_in: klass.built_in?,
+            provider: klass.provider,
+            phase: klass.phase
+          }
+        end
+      end
+
+      # For a given connector identifier, return the class (either builtin, or from a 
+      # external chronicle gem)
+      def self.identifier_to_klass(identifier:, phase:)
+        if BUILTIN[phase].include? identifier
+          load_builtin_klass(name: identifier, phase: phase)
+        else
+          provider, name = identifier.split(':')
+          name ||= ''
+          load_provider_klass(provider: provider, name: name, phase: phase)
+        end
+      end
+
+      # Returns whether a class is an Extractor, Transformer, or Loader
+      def phase
+        ancestors = self.ancestors
+        return :extractor if ancestors.include? Chronicle::ETL::Extractor
+        return :transformer if ancestors.include? Chronicle::ETL::Transformer
+        return :loader if ancestors.include? Chronicle::ETL::Loader
+      end
+
+      # Returns which third-party provider this connector is associated wtih
+      def provider
+        # TODO: needs better convention for a gem reporting its provider name
+        provider = to_s.split('::')[1].downcase
+        provider == 'etl' ? 'chronicle' : provider
+      end
+
+      # Returns whether this connector is a built-in one
+      def built_in?
+        to_s.include? 'Chronicle::ETL'
+      end
+
+      private
+
+      def self.load_builtin_klass(name:, phase:)
+        klass_str = "Chronicle::ETL::#{name.capitalize}#{phase.capitalize}"
+        begin
+          Object.const_get(klass_str)
+        rescue NameError => e
+          raise ConnectorNotAvailableError.new("Connector not found", name: name)
+        end
+      end
+
+      def self.load_provider_klass(name: '', phase:, provider:)
+        begin
+          require "chronicle/#{provider}"
+          klass_str = "Chronicle::#{provider.capitalize}::#{name.capitalize}#{phase.capitalize}"
+          Object.const_get(klass_str)
+        rescue LoadError => e
+          raise ProviderNotAvailableError.new("Provider '#{provider.capitalize}' could not be loaded", provider: provider)
+        rescue NameError => e
+          raise ProviderConnectorNotAvailableError.new("Connector '#{name}' in '#{provider}' could not be found", provider: provider, name: name)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/connectors.rb

@@ -0,0 +1,32 @@
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL connectors
+      class Connectors < SubcommandBase
+        default_task 'list'
+        namespace :connectors
+
+        desc "install NAME", "Installs connector NAME"
+        def install
+          puts "Installing"
+        end
+
+        desc "list", "Lists available connectors"
+        # Display all available connectors that chronicle-etl has access to
+        def list
+          klasses = Chronicle::ETL::Catalog.available_classes
+          klasses = klasses.sort_by do |a|
+            [a[:built_in].to_s, a[:provider], a[:phase]]
+          end
+
+          headers = klasses.first.keys.map do |key|
+            key.to_s.upcase.bold
+          end
+
+          table = TTY::Table.new(headers, klasses.map(&:values))
+          puts table.render(indent: 0, padding: [0, 2])
+        end
+      end
+    end
+  end
+end