chronicle-etl 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf96bda84a74a4b43166060ba7fc5ec9a0aa3f129a7717c4d47d2672b9d0b654
-  data.tar.gz: eedf76e4684970f1c5e45fc68a86086e4e81be3080489c86410c8eb255f9eb68
+  metadata.gz: 3ab7fe84d09764034f061236d75fdf7403af9e1835d568831ae4896b90fc39b5
+  data.tar.gz: c4c7a1ff47ecaf7d364e35d3439eef1345f90fee5a6610735124592109e2c02c
 SHA512:
-  metadata.gz: ad274453c44b94be793025bdd96767f9db2d84f1ae769d41bab8af4280dc95379dd4b710024c1519403b4cab13a55fb6f1657b7ba1f77ef4e4701159a80c0460
-  data.tar.gz: 8e3c68d4f29e2e35b6bda9dce7e6d7f79e4a6d15fec93154a66cf18cc13045fe45b2abe7b37e370ab413cd2ff561f4dcaad6c18bb09de4f60da8527509b2e94a
+  metadata.gz: a5a4b0e3769cd6063cb78843828ab15766df9d1ac1cd5d65fb81145d1415e95e46583c578312418fca91bfe02b05a8db3277a41a8a6b420f12263135deb49022
+  data.tar.gz: e6d6e23d3c164d6fc5b9283d468fe9c9a450329d2d778e3ad49962f42fbe0316299111cdd02d2abc92eb52dc46823c5cbaf1c195203fc4e78129e9b28528c5ef

data/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.4] - 2020-08-18
+### Updated
+- Better display of available ETL classes
+- Updated documentation
+
 ## [0.1.3] - 2020-08-13
 ### Added
 - Ability to list all available ETL classes

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.1.3)
+    chronicle-etl (0.1.4)
       colorize (~> 0.8.1)
       thor (~> 0.20)
       tty-progressbar (~> 0.17)

data/README.md

@@ -2,7 +2,9 @@
 
 [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
 
-Chronicle ETL is a utility tool for manipulating personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+Chronicle ETL is a utility tool for archiving and processing personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+
+This project is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex).
 
 ## Installation
 
@@ -10,7 +12,7 @@ Chronicle ETL is a utility tool for manipulating personal data. You can extract
 $ gem install chronicle-etl
 ```
 
-## Examples
+## Usage
 
 After installing the gem, `chronicle-etl` is available to run in your shell.
 
@@ -19,12 +21,43 @@ chronicle-etl --extractor csv --extractor-opts filename:test.csv --loader table
 cat test.csv | chronicle-etl --extractor csv --loader table
 ```
 
-## Full usage
+## Available importers
+
+### Extractors
+- `stdin` - (default) Load records from line-separated stdin
+- `csv`
+- `file` - load from a single file or directory (with a glob pattern)
+
+### Transformers
+- `null` - (default) Don't do anything
+
+### Loaders
+- `stdout` - (default) output transformed records to stdount
+- `csv` - Load records to a csv file
+- `table` - Output an ascii table of records. Useful for debugging.
+
+### Provider-specific importers
+
+In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
+
+- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` files. Transformers for chronicle schema
+- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
+
+To install any of these, run `gem install chronicle-PROVIDER`. 
+
+If you don't want to use the available rubygem importers, `chronicle-etl` can use `stdin` as an Extractor source (newline separated records). You can also use `stdout` as a loader — transformed records will be outputted separated by newlines.
+
+I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.net) to chat about what will be available!
+
+### Full commands
 
 ```
+$ chronicle-etl help 
+
 Commands:
   chronicle-etl help [COMMAND]  # Describe available commands or one specific command
   chronicle-etl job             # Runs an ETL job
+  chronicle-etl list            # List all ETL classes
 ```
 
 ### Job options

data/lib/chronicle/etl/catalog.rb

@@ -47,7 +47,7 @@ module Chronicle
       def provider
         # TODO: needs better convention for a gem reporting its provider name
         provider = to_s.split('::')[1].downcase
-        return provider unless provider == 'etl'
+        provider == 'etl' ? 'chronicle' : provider
       end
 
       def built_in?

data/lib/chronicle/etl/cli.rb

@@ -1,5 +1,6 @@
 require 'thor'
 require 'chronicle/etl'
+require 'colorize'
 
 module Chronicle
   module Etl
@@ -39,9 +40,16 @@ module Chronicle
       desc 'list', 'List all ETL classes'
       def list
         klasses = Chronicle::Etl::Catalog.available_classes
+        klasses = klasses.sort_by do |a|
+          [a[:built_in].to_s, a[:provider], a[:phase]]
+        end
 
-        table = TTY::Table.new(['class_name', 'built_in?', 'provider', 'phase'], klasses.map(&:values))
-        puts table.render
+        headers = klasses.first.keys.map do |key|
+          key.to_s.capitalize.light_white
+        end
+
+        table = TTY::Table.new(headers, klasses.map(&:values))
+        puts table.render(padding: [0, 2])
       end
     end
   end

data/lib/chronicle/etl/loaders/loader.rb

@@ -2,15 +2,13 @@ module Chronicle
   module Etl
     class Loader
       extend Chronicle::Etl::Catalog
-      
+
       def initialize(options = {})
         @options = options
       end
 
       def start; end
 
-      def first_load result; end
-
       def load
         raise NotImplementedError
       end
@@ -22,4 +20,4 @@ end
 
 require_relative 'csv_loader'
 require_relative 'stdout_loader'
-require_relative 'table_loader'
+require_relative 'table_loader'

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -7,18 +7,14 @@ module Chronicle
         super(options)
       end
 
-      # defer creating table until we get first result and can determine headers
-      def first_load(result)
-        headers = result.keys
-        @table = TTY::Table.new(header: headers)
-      end
-
       def load(result)
-        @table << result
+        @table ||= TTY::Table.new(header: result.keys)
+        values = result.values.map{|x| x.to_s[0..30]}
+        @table << values
       end
 
       def finish
-        puts @table.render(:ascii)
+        puts @table.render(:ascii, padding: [0, 1])
       end
     end
   end

data/lib/chronicle/etl/runner.rb

@@ -12,23 +12,18 @@ class Chronicle::Etl::Runner
   end
 
   def run!
-    progress_bar = Chronicle::Etl::Utils::ProgressBar.new(title: "Running job", total: @extractor.results_count)
+    total = @extractor.results_count
+    progress_bar = Chronicle::Etl::Utils::ProgressBar.new(title: 'Running job', total: total)
     count = 0
 
     @loader.start
 
     @extractor.extract do |data, metadata|
       transformed_data = @transformer.transform(data)
-      
-      @loader.first_load(transformed_data) if count == 0
       @loader.load(transformed_data)
 
       progress_bar.increment
       count += 1
-    # rescue StandardError => e
-    #   require 'pry'
-    #   binding.pry
-    #   progress_bar.log "Error processing; #{e.inspect}"
     end
 
     progress_bar.finish
@@ -43,13 +38,13 @@ class Chronicle::Etl::Runner
     @loader = load_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
   end
 
-  def load_etl_class(phase, name)
-    if BUILTIN[phase].include? name
-      klass_name = "Chronicle::Etl::#{name.capitalize}#{phase.to_s.capitalize}"
+  def load_etl_class(phase, x)
+    if BUILTIN[phase].include? x
+      klass_name = "Chronicle::Etl::#{x.capitalize}#{phase.to_s.capitalize}"
     else
       # TODO: come up with syntax for specifying a particular extractor in a provider library
-      # provider, extractor = name.split(":")
-      provider = name
+      provider, name = x.split(":")
+      provider = x unless provider
       begin
         require "chronicle/#{provider}"
       rescue LoadError => e
@@ -57,7 +52,7 @@ class Chronicle::Etl::Runner
         warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
         exit(false)
       end
-      klass_name = "Chronicle::#{name.capitalize}::ChronicleTransformer"
+      klass_name = "Chronicle::#{provider.capitalize}::#{name&.capitalize}#{phase.capitalize}"
     end
     Object.const_get(klass_name)
   end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module Etl
-    VERSION = "0.1.3"
+    VERSION = "0.1.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor