chronicle-etl 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d9be4b073385d9758a8b709bb6726d8dae648b8c4dbef474343840c011d1178
-  data.tar.gz: 0d95395407d37f7e322287a0920bba60e1b4f81eb8649190d021e13580604a2d
+  metadata.gz: bfd4e081bfeda7d097a5a5eee6ccf28baf0a9b3878968d74c9a604013d0b55a6
+  data.tar.gz: 003ebd2ffe2b1220c7f43a4875043cb5500aa1b7a6327b84c9be10f04e0e8d40
 SHA512:
-  metadata.gz: 386c96518aa2d2810ae2a93bbe3af5bb08e26e132608b4e6ed8a278da076783e453854a2120c5016b6d02cd5dea406146d10ef3c7c1e77d854acd8ff2608eaf7
-  data.tar.gz: eb14402be5d6db44a6f06e6ec930acc5103b36e5c2e5a13e89137c9ee45f5f11c1e9e6ab13d4d44e6ee06bd9b02309e02ac33e81256645ef71d5b431c97eb199
+  metadata.gz: 3d786fb4acf8d0b03e65262209def310ca25b92646847f6e96791e6491e9b159ab11db7fa35f785f6782fbc0b9e3daebb625e2353fce2422f7fc79aed7a4d6bc
+  data.tar.gz: 87771745b9df2160966299f1d73eb568b46080ed217a1952e1dd938fff7758432f2cc6f449036d24951660b405e18083f1adac26d00243a0fab9003a96eb569d

data/.gitignore

@@ -7,6 +7,9 @@
 /spec/reports/
 /tmp/
 
+# https://yehudakatz.com/2010/12/16/clarifying-the-roles-of-the-gemspec-and-gemfile/
+Gemfile.lock
+
 # rspec failure tracking
 .rspec_status
 .DS_Store

data/.rubocop.yml

@@ -5,4 +5,7 @@ Style/StringLiterals:
   Enabled: false
 
 Style/MethodCallWithArgsParentheses:
+  Enabled: false
+
+Lint/ConstantResolution:
   Enabled: false

data/README.md

@@ -2,9 +2,9 @@
 
 [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
 
-Chronicle ETL is a utility tool for archiving and processing personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+Chronicle ETL is a utility that helps you archive and processes personal data. You can *extract* it from a variety of sources, *transform* it, and *load* it to an external API, file, or stdout.
 
-This project is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex).
+This tool is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex) and the dozens of existing importers are being migrated to Chronicle.
 
 ## Installation
 
@@ -31,6 +31,9 @@ Connectors are available to read, process, and load data from different formats
 ```bash
 # List all available connectors
 $ chronicle-etl connectors:list
+
+# Install a connector
+$ chronicle-etl connectors:install imessage
 ```
 
 Built in connectors:
@@ -44,16 +47,18 @@ Built in connectors:
 - `null` - (default) Don't do anything
 
 ### Loaders
-- `stdout` - (default) output transformed records to stdount
+- `stdout` - (default) output records to stdout serialized as JSON
 - `csv` - Load records to a csv file
+- `rest` - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
 - `table` - Output an ascii table of records. Useful for debugging.
 
 ### Provider-specific importers
 
 In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
 
-- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` files. Transformers for chronicle schema
-- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
+- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files
+- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`
+- [imessage](https://github.com/chronicle-app/chronicle-imessage). Extract iMessage messages from a local macOS installation
 
 To install any of these, run `gem install chronicle-PROVIDER`. 
 
@@ -61,7 +66,7 @@ If you don't want to use the available rubygem importers, `chronicle-etl` can us
 
 I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.net) to chat about what will be available!
 
-### Full commands
+## Full commands
 
 ```
 $ chronicle-etl help 
@@ -75,26 +80,28 @@ ALL COMMANDS
   jobs:create                # Create a job
   jobs:list                  # List all available jobs
   jobs:run                   # Start a job
-  jobs:show                  # Show a job
+  jobs:show                  # Show details about a job
 ```
 
-### Job options
+### Running a job
 
 ```
 Usage:
   chronicle-etl jobs:run
 
 Options:
-  -e, [--extractor=extractor-name]      # Extractor class (available: stdin, csv, file)
-                                        # Default: stdin
+      [--log-level=LOG_LEVEL]           # Log level (debug, info, warn, error, fatal)
+                                        # Default: info
+  -v, [--verbose], [--no-verbose]       # Set log level to verbose
+      [--dry-run], [--no-dry-run]       # Only run the extraction and transform steps, not the loading
+  -e, [--extractor=extractor-name]      # Extractor class. Default: stdin
       [--extractor-opts=key:value]      # Extractor options
-  -t, [--transformer=transformer-name]  # Transformer class (available: null)
-                                        # Default: null
+  -t, [--transformer=transformer-name]  # Transformer class. Default: null
       [--transformer-opts=key:value]    # Transformer options
-  -l, [--loader=loader-name]            # Loader class (available: stdout, csv, table)
-                                        # Default: stdout
+  -l, [--loader=loader-name]            # Loader class. Default: stdout
       [--loader-opts=key:value]         # Loader options
-  -j, [--job=JOB]                       # Job configuration file
+  -j, [--name=NAME]                     # Job configuration name
+
 
 Runs an ETL job
 ```

data/chronicle-etl.gemspec

@@ -36,15 +36,21 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "chronic_duration", "~> 0.10.6"
   spec.add_dependency "colorize", "~> 0.8.1"
-  spec.add_dependency "tty-table", "~> 0.11"
+  spec.add_dependency "marcel", "~> 1.0.2"
+  spec.add_dependency "mini_exiftool", "~> 2.10"
+  spec.add_dependency "nokogiri", "~> 1.13"
+  spec.add_dependency "runcom", "~> 6.2"
+  spec.add_dependency "sequel", "~> 5.35"
+  spec.add_dependency "sqlite3", "~> 1.4"
+  spec.add_dependency "thor", "~> 0.20"
   spec.add_dependency "tty-progressbar", "~> 0.17"
+  spec.add_dependency "tty-table", "~> 0.11"
 
   spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "pry-byebug", "~> 3.9"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "pry-byebug", "~> 3.9"
-  spec.add_development_dependency 'runcom', '~> 6.2'
-  spec.add_development_dependency 'redcarpet', '~> 3.5'
 end

data/lib/chronicle/etl/cli/connectors.rb

@@ -7,23 +7,35 @@ module Chronicle
         namespace :connectors
 
         desc "install NAME", "Installs connector NAME"
-        def install
-          puts "Installing"
+        def install(name)
+          Chronicle::ETL::Registry.install_connector(name)
         end
 
         desc "list", "Lists available connectors"
         # Display all available connectors that chronicle-etl has access to
         def list
-          klasses = Chronicle::ETL::Catalog.available_classes
-          klasses = klasses.sort_by do |a|
-            [a[:built_in].to_s, a[:provider], a[:phase]]
+          Chronicle::ETL::Registry.load_all!
+
+          connector_info = Chronicle::ETL::Registry.connectors.map do |connector_registration|
+            {
+              identifier: connector_registration.identifier,
+              phase: connector_registration.phase,
+              description: connector_registration.descriptive_phrase,
+              provider: connector_registration.provider,
+              core: connector_registration.built_in? ? '✓' : '',
+              class: connector_registration.klass_name
+            }
+          end
+
+          connector_info = connector_info.sort_by do |a|
+            [a[:core].to_s, a[:provider], a[:phase], a[:identifier]]
           end
 
-          headers = klasses.first.keys.map do |key|
+          headers = connector_info.first.keys.map do |key|
             key.to_s.upcase.bold
           end
 
-          table = TTY::Table.new(headers, klasses.map(&:values))
+          table = TTY::Table.new(headers, connector_info.map(&:values))
           puts table.render(indent: 0, padding: [0, 2])
         end
       end

data/lib/chronicle/etl/cli/jobs.rb

@@ -1,6 +1,4 @@
 require 'pp'
-require 'pry'
-
 module Chronicle
   module ETL
     module CLI
@@ -9,16 +7,19 @@ module Chronicle
         default_task "start"
         namespace :jobs
 
-        class_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
+        class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'extractor-name'
         class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
-        class_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
+        class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'transformer-name'
         class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
-        class_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
+        class_option :loader, aliases: '-l', desc: 'Loader class. Default: stdout', banner: 'loader-name'
         class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-        class_option :job, aliases: '-j', desc: 'Job configuration name (or filename)'
+        class_option :name, aliases: '-j', desc: 'Job configuration name'
 
         map run: :start  # Thor doesn't like `run` as a command name
         desc "run", "Start a job"
+        option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
+        option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
+        option :dry_run, desc: 'Only run the extraction and transform steps, not the loading', type: :boolean
         long_desc <<-LONG_DESC
           This will run an ETL job. Each job needs three parts:
 
@@ -26,36 +27,37 @@ module Chronicle
 
             2. #{'Transformer'.underline}: transforms data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
 
-            3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout.
+            3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout. With the --dry-run option, this step won't be run.
 
             If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
 LONG_DESC
         # Run an ETL job
         def start
-          runner_options = build_runner_options(options)
-          runner = Chronicle::ETL::Runner.new(runner_options)
+          setup_log_level
+          job_definition = build_job_definition(options)
+          job = Chronicle::ETL::Job.new(job_definition)
+          runner = Chronicle::ETL::Runner.new(job)
           runner.run!
         end
 
         desc "create", "Create a job"
         # Create an ETL job
         def create
-          runner_options = build_runner_options(options)
-          path = File.join('chronicle', 'etl', 'jobs', options[:job])
-          Chronicle::ETL::Config.write(path, runner_options)
+          job_definition = build_job_definition(options)
+          path = File.join('chronicle', 'etl', 'jobs', options[:name])
+          Chronicle::ETL::Config.write(path, job_definition.definition)
         end
 
         desc "show", "Show details about a job"
         # Show an ETL job
         def show
-          runner_options = build_runner_options(options)
-          pp runner_options
+          puts Chronicle::ETL::Job.new(build_job_definition(options))
         end
 
         desc "list", "List all available jobs"
         # List available ETL jobs
         def list
-          jobs = Chronicle::ETL::Config.jobs
+          jobs = Chronicle::ETL::Config.available_jobs
 
           job_details = jobs.map do |job|
             r = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
@@ -75,34 +77,43 @@ LONG_DESC
 
         private
 
-        # Create runner options by reading config file and then overwriting with flag options
-        def build_runner_options options
-          flag_options = process_flag_options(options)
-          job_options = load_job(options[:job])
-          flag_options.merge(job_options)
+        def setup_log_level
+          if options[:verbose]
+            Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::DEBUG
+          elsif options[:log_level]
+            level = Chronicle::ETL::Logger.const_get(options[:log_level].upcase)
+            Chronicle::ETL::Logger.log_level = level 
+          end
+        end
+
+        # Create job definition by reading config file and then overwriting with flag options
+        def build_job_definition(options)
+          definition = Chronicle::ETL::JobDefinition.new
+          definition.add_config(load_job_config(options[:name]))
+          definition.add_config(process_flag_options(options))
+          definition
         end
 
-        def load_job job
-          yml_config = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
-          # FIXME: use better trick to depely symbolize keys
-          JSON.parse(yml_config.to_json, symbolize_names: true)
+        def load_job_config name
+          Chronicle::ETL::Config.load_job_from_config(name)
         end
 
         # Takes flag options and turns them into a runner config
         def process_flag_options options
           {
+            dry_run: options[:dry_run],
             extractor: {
               name: options[:extractor],
               options: options[:'extractor-opts']
-            },
+            }.compact,
             transformer: {
               name: options[:transformer],
               options: options[:'transformer-opts']
-            },
+            }.compact,
             loader: {
               name: options[:loader],
               options: options[:'loader-opts']
-            }
+            }.compact
           }
         end
       end

data/lib/chronicle/etl/cli/main.rb

@@ -22,6 +22,11 @@ module Chronicle
 
         # Entrypoint for the CLI
         def self.start(given_args = ARGV, config = {})
+          if given_args[0] == "--version"
+            puts "#{Chronicle::ETL::VERSION}"
+            exit
+          end
+
           if given_args.none?
             abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
           end
@@ -52,10 +57,10 @@ module Chronicle
             shell.say "  $ chronicle-etl connectors:list"
             shell.say
             shell.say "  Run a simple job:".italic.light_black
-            shell.say "  $ chronicle-etl jobs:start --extractor stdin --transformer null --loader stdout"
+            shell.say "  $ chronicle-etl jobs:run --extractor stdin --transformer null --loader stdout"
             shell.say
             shell.say "  Show full job options:".italic.light_black
-            shell.say "  $ chronicle-etl jobs help start"
+            shell.say "  $ chronicle-etl jobs help run"
 
             list = []
 
@@ -72,6 +77,9 @@ module Chronicle
             shell.say "VERSION".bold
             shell.say "  #{Chronicle::ETL::VERSION}"
             shell.say
+            shell.say "  Display current version:".italic.light_black
+            shell.say "  $ chronicle-etl --version"
+            shell.say
             shell.say "FULL DOCUMENTATION".bold
             shell.say "  https://github.com/chronicle-app/chronicle-etl".blue
             shell.say

data/lib/chronicle/etl/config.rb

@@ -4,15 +4,17 @@ module Chronicle
   module ETL
     # Utility methods to read, write, and access config files
     module Config
+      module_function
+
       # Loads a yml config file
-      def self.load(path)
+      def load(path)
         config = Runcom::Config.new(path)
         # FIXME: hack to deeply symbolize keys
         JSON.parse(config.to_h.to_json, symbolize_names: true)
       end
 
       # Writes a hash as a yml config file
-      def self.write(path, data)
+      def write(path, data)
         config = Runcom::Config.new(path)
         filename = config.all[0].to_s + '.yml'
         File.open(filename, 'w') do |f|
@@ -21,12 +23,31 @@ module Chronicle
       end
 
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
-      def self.jobs
+      def available_jobs
         job_directory = Runcom::Config.new('chronicle/etl/jobs').current
         Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
+
+      # Returns all available credentials available in ~/.config/chronicle/etl/credentials/*.yml
+      def available_credentials
+        job_directory = Runcom::Config.new('chronicle/etl/credentials').current
+        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+          File.basename(filename, ".*")
+        end
+      end
+
+      # Load a job definition from job config directory
+      def load_job_from_config(job_name)
+        definition = self.load("chronicle/etl/jobs/#{job_name}.yml")
+        definition[:name] = job_name
+        definition
+      end
+
+      def load_credentials(name)
+        config = self.load("chronicle/etl/credentials/#{name}.yml")
+      end
     end
   end
 end

data/lib/chronicle/etl/exceptions.rb

@@ -0,0 +1,30 @@
+module Chronicle
+  module ETL
+    class Error < StandardError; end;
+
+    class RunnerTypeError < Error; end
+
+    class ConnectorNotAvailableError < Error
+      def initialize(message, provider: nil, name: nil)
+        super(message)
+        @provider = provider
+        @name = name
+      end
+      attr_reader :name, :provider
+    end
+
+    class ProviderNotAvailableError < ConnectorNotAvailableError; end
+    class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
+
+    class TransformationError < Error
+      attr_reader :transformation
+
+      def initialize(message=nil, transformation:)
+        super(message)
+        @transformation = transformation
+      end
+    end
+
+    class UntransformableRecordError < TransformationError; end
+  end
+end

data/lib/chronicle/etl/extraction.rb

@@ -0,0 +1,12 @@
+module Chronicle
+  module ETL
+    class Extraction
+      attr_accessor :data, :meta
+
+      def initialize(data: {}, meta: {})
+        @data = data
+        @meta = meta
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -1,42 +1,48 @@
 require 'csv'
-class Chronicle::ETL::CsvExtractor < Chronicle::ETL::Extractor
-  DEFAULT_OPTIONS = {
-    headers: true,
-    filename: $stdin
-  }.freeze
-
-  def initialize(options = {})
-    super(DEFAULT_OPTIONS.merge(options))
-  end
 
-  def extract
-    csv = initialize_csv
-    csv.each do |row|
-      result = row.to_h
-      yield result
+module Chronicle
+  module ETL
+    class CsvExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::FilesystemReader
+
+      register_connector do |r|
+        r.description = 'input as CSV'
+      end
+
+      DEFAULT_OPTIONS = {
+        headers: true,
+        filename: $stdin
+      }.freeze
+
+      def initialize(options = {})
+        super(DEFAULT_OPTIONS.merge(options))
+      end
+
+      def extract
+        csv = initialize_csv
+        csv.each do |row|
+          yield Chronicle::ETL::Extraction.new(data: row.to_h)
+        end
+      end
+
+      def results_count
+        CSV.read(@options[:filename], headers: @options[:headers]).count unless stdin?(@options[:filename])
+      end
+
+      private
+
+      def initialize_csv
+        headers = @options[:headers].is_a?(String) ? @options[:headers].split(',') : @options[:headers]
+
+        csv_options = {
+          headers: headers,
+          converters: :all
+        }
+
+        open_from_filesystem(filename: @options[:filename]) do |file|
+          return CSV.new(file, **csv_options)
+        end
+      end
     end
   end
-
-  def results_count
-    CSV.read(@options[:filename], headers: @options[:headers]).count if read_from_file?
-  end
-
-  private
-
-  def initialize_csv
-    headers = @options[:headers].is_a?(String) ? @options[:headers].split(',') : @options[:headers]
-
-    csv_options = {
-      headers: headers,
-      header_converters: :symbol,
-      converters: [:all]
-    }
-
-    stream = read_from_file? ? File.open(@options[:filename]) : @options[:filename]
-    CSV.new(stream, **csv_options)
-  end
-
-  def read_from_file?
-    @options[:filename] != $stdin
-  end
 end

data/lib/chronicle/etl/extractors/extractor.rb

@@ -4,7 +4,7 @@ module Chronicle
   module ETL
     # Abstract class representing an Extractor for an ETL job
     class Extractor
-      extend Chronicle::ETL::Catalog
+      extend Chronicle::ETL::Registry::SelfRegistering
 
       # Construct a new instance of this extractor. Options are passed in from a Runner
       # == Paramters:
@@ -12,6 +12,8 @@ module Chronicle
       #   Options for configuring this Extractor
       def initialize(options = {})
         @options = options.transform_keys!(&:to_sym)
+        sanitize_options
+        handle_continuation
       end
 
       # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
@@ -22,10 +24,26 @@ module Chronicle
       # An optional method to calculate how many records there are to extract. Used primarily for
       # building the progress bar
       def results_count; end
+
+      private
+
+      def sanitize_options
+        @options[:load_since] = Time.parse(@options[:load_since]) if @options[:load_since] && @options[:load_since].is_a?(String)
+        @options[:load_until] = Time.parse(@options[:load_until]) if @options[:load_until] && @options[:load_until].is_a?(String)
+      end
+
+      def handle_continuation
+        return unless @options[:continuation]
+
+        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation].highest_timestamp
+        @options[:load_after_id] = @options[:continuation].last_id if @options[:continuation].last_id
+      end
     end
   end
 end
 
+require_relative 'helpers/filesystem_reader'
 require_relative 'csv_extractor'
 require_relative 'file_extractor'
+require_relative 'json_extractor'
 require_relative 'stdin_extractor'

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -3,49 +3,31 @@ require 'pathname'
 module Chronicle
   module ETL
     class FileExtractor < Chronicle::ETL::Extractor
-      def extract
-        if file?
-          extract_file do |data, metadata|
-            yield(data, metadata)
-          end
-        elsif directory?
-          extract_from_directory do |data, metadata|
-            yield(data, metadata)
-          end
-        end
-      end
+      include Extractors::Helpers::FilesystemReader
 
-      def results_count
-        if file?
-          return 1
-        else
-          search_pattern = File.join(@options[:filename], '**/*.eml')
-          Dir.glob(search_pattern).count
-        end
+      register_connector do |r|
+        r.description = 'file or directory of files'
       end
 
-      private
-
-      def extract_from_directory
-        search_pattern = File.join(@options[:filename], '**/*.eml')
-        filenames = Dir.glob(search_pattern)
+      def extract
         filenames.each do |filename|
-          file = File.open(filename)
-          yield(file.read, {filename: file})
+          yield Chronicle::ETL::Extraction.new(data: filename)
         end
       end
 
-      def extract_file
-        file = File.open(@options[:filename])
-        yield(file.read, {filename: @options[:filename]})
+      def results_count
+        filenames.count
       end
 
-      def directory?
-        Pathname.new(@options[:filename]).directory?
-      end
+      private
 
-      def file?
-        Pathname.new(@options[:filename]).file?
+      def filenames
+        @filenames ||= filenames_in_directory(
+          path: @options[:filename],
+          dir_glob_pattern: @options[:dir_glob_pattern],
+          load_since: @options[:load_since],
+          load_until: @options[:load_until]
+        )
       end
     end
   end