chronicle-etl 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b74c4a7782c1ab31173e628b3e5ccb8743fe21f29d6f48d739b0e3cc2dfda22e
-  data.tar.gz: 7ea44638b08f6da12c0a5386f3d852600f50336ce0bb57347114804770f75691
+  metadata.gz: 5fd411a9a41a645b85780230c79b09f361e121d0e8ca7f3270ca8eba55a76ca8
+  data.tar.gz: c09053715910ab4f027fbdc3a5b7d10c042eee962f7fa93c6571ce8359f51009
 SHA512:
-  metadata.gz: efb23677c731a54b0382c3095dc9bb5f98a97365c1daf031bbc8c20335e7bd146b76b3a50486971e48192e7540bc0ae1b09f232590a590257203ae3560396767
-  data.tar.gz: cba40b71a7e8c0b17a286ecd3db3724bff290fdd79b3fdf55ab89967f6af14228911c0e6928a949b8dd899acd6ad396b8a21fb03162a8561247c97b1200bac29
+  metadata.gz: 2c9ec14b6c0a51f1c5ec77ee8d9a7f016d16bdc35db5634f9fa5d38aabc30dec201cd4b8bef06a31b86773a0c1cda2d271d7008dcb247a86d956c094919f3c0f
+  data.tar.gz: 0dca41e1654e5b2b98a148f853492a67126cdac767000b3c5f97c5c8ff88b77464e17a2fab38b72c1f014f3515c911e5f3f391eaf68d64e73dcfcff5d8e6cb6a

data/.github/workflows/ruby.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.rubocop.yml

@@ -1,11 +1,38 @@
 AllCops:
   EnabledByDefault: true
+  TargetRubyVersion: 2.7
+
+Style/FrozenStringLiteralComment:
+  SafeAutoCorrect: true
 
 Style/StringLiterals:
   Enabled: false
 
+Layout/MultilineAssignmentLayout:
+  Enabled: false
+
+Layout/RedundantLineBreak:
+  Enabled: false
+
 Style/MethodCallWithArgsParentheses:
   Enabled: false
 
+Style/MethodCalledOnDoEndBlock:
+  Exclude:
+    - 'spec/**/*'
+
+Style/OpenStructUse:
+  Enabled: false
+
+Style/Copyright:
+  Enabled: false
+
+Style/SymbolArray:
+  EnforcedStyle: brackets
+
+Style/WordArray:
+  EnforcedStyle: brackets
+
 Lint/ConstantResolution:
-  Enabled: false
+  Enabled: false
+

data/Guardfile

@@ -0,0 +1,7 @@
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/README.md

@@ -1,6 +1,6 @@
 # Chronicle::ETL
 
-[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
+[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl) [![Ruby](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml/badge.svg)](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml)
 
 Chronicle ETL is a utility that helps you archive and processes personal data. You can *extract* it from a variety of sources, *transform* it, and *load* it to an external API, file, or stdout.
 
@@ -57,7 +57,7 @@ Built in connectors:
 In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
 
 - [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files
-- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`
+- [shell](https://github.com/chronicle-app/chronicle-shell). Extract shell history from Bash or Zsh`
 - [imessage](https://github.com/chronicle-app/chronicle-imessage). Extract iMessage messages from a local macOS installation
 
 To install any of these, run `gem install chronicle-PROVIDER`. 

data/Rakefile

@@ -1,6 +1,8 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
-
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+require 'yard'
+YARD::Rake::YardocTask.new
+
+task default: :spec

data/chronicle-etl.gemspec

@@ -35,17 +35,18 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+  spec.required_ruby_version = ">= 2.7"
 
-  spec.add_dependency "activesupport"
+  spec.add_dependency "activesupport", "~> 7.0"
   spec.add_dependency "chronic_duration", "~> 0.10.6"
   spec.add_dependency "colorize", "~> 0.8.1"
   spec.add_dependency "marcel", "~> 1.0.2"
   spec.add_dependency "mini_exiftool", "~> 2.10"
   spec.add_dependency "nokogiri", "~> 1.13"
-  spec.add_dependency "runcom", "~> 6.2"
+  spec.add_dependency "runcom", ">= 6.0"
   spec.add_dependency "sequel", "~> 5.35"
   spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "thor", "~> 1.2"
   spec.add_dependency "tty-progressbar", "~> 0.17"
   spec.add_dependency "tty-table", "~> 0.11"
 
@@ -53,4 +54,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug", "~> 3.9"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_development_dependency "simplecov", "~> 0.21"
+  spec.add_development_dependency "guard-rspec", "~> 4.7.3"
+  spec.add_development_dependency "yard", "~> 0.9.7"
+  spec.add_development_dependency "rubocop", "~> 1.25.1"
 end

data/exe/chronicle-etl

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require "chronicle/etl/cli/main"
+require "chronicle/etl/cli"
 
 Chronicle::ETL::CLI::Main.start(ARGV)

data/lib/chronicle/etl/cli/connectors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     module CLI
@@ -38,6 +40,38 @@ module Chronicle
           table = TTY::Table.new(headers, connector_info.map(&:values))
           puts table.render(indent: 0, padding: [0, 2])
         end
+
+        desc "show PHASE IDENTIFIER", "Show information about a connector"
+        def show(phase, identifier)
+          unless ['extractor', 'transformer', 'loader'].include?(phase)
+            puts "phase argument must be one of: [extractor, transformer, loader]"
+            return
+          end
+
+          begin
+            connector = Chronicle::ETL::Registry.find_by_phase_and_identifier(phase.to_sym, identifier)
+          rescue Chronicle::ETL::ConnectorNotAvailableError
+            puts "Could not find #{phase} #{identifier}"
+            return
+          end
+
+          puts connector.klass.to_s.bold
+          puts "  #{connector.descriptive_phrase}"
+          puts
+          puts "OPTIONS"
+
+          headers = ['name', 'default', 'required'].map{ |h| h.to_s.upcase.bold }
+
+          settings = connector.klass.settings.map do |name, setting|
+            [
+              name,
+              setting.default,
+              setting.required ? 'yes' : 'no'
+            ]
+          end
+          table = TTY::Table.new(headers, settings)
+          puts table.render(indent: 0, padding: [0, 2])
+        end
       end
     end
   end

data/lib/chronicle/etl/cli/jobs.rb

@@ -1,4 +1,5 @@
 require 'pp'
+
 module Chronicle
   module ETL
     module CLI
@@ -7,15 +8,29 @@ module Chronicle
         default_task "start"
         namespace :jobs
 
+        class_option :name, aliases: '-j', desc: 'Job configuration name'
+
         class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'extractor-name'
         class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
         class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'transformer-name'
         class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
         class_option :loader, aliases: '-l', desc: 'Loader class. Default: stdout', banner: 'loader-name'
         class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-        class_option :name, aliases: '-j', desc: 'Job configuration name'
 
-        map run: :start  # Thor doesn't like `run` as a command name
+        # This is an array to deal with shell globbing
+        class_option :input, aliases: '-i', desc: 'Input filename or directory', default: [], type: 'array', banner: 'FILENAME'
+        class_option :since, desc: "Load records SINCE this date. Overrides job's `load_since` configuration option in extractor's options", banner: 'DATE'
+        class_option :until, desc: "Load records UNTIL this date", banner: 'DATE'
+        class_option :limit, desc: "Only extract the first LIMIT records", banner: 'N'
+
+        class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
+        class_option :fields, desc: 'Output only these fields', type: 'array', banner: 'field1 field2 ...'
+
+        class_option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
+        class_option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
+
+        # Thor doesn't like `run` as a command name
+        map run: :start
         desc "run", "Start a job"
         option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
         option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
@@ -69,7 +84,7 @@ LONG_DESC
             [job, extractor, transformer, loader]
           end
 
-          headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
+          headers = ['name', 'extractor', 'transformer', 'loader'].map { |h| h.upcase.bold }
 
           table = TTY::Table.new(headers, job_details)
           puts table.render(indent: 0, padding: [0, 2])
@@ -90,7 +105,7 @@ LONG_DESC
         def build_job_definition(options)
           definition = Chronicle::ETL::JobDefinition.new
           definition.add_config(load_job_config(options[:name]))
-          definition.add_config(process_flag_options(options))
+          definition.add_config(process_flag_options(options).transform_keys(&:to_sym))
           definition
         end
 
@@ -100,19 +115,33 @@ LONG_DESC
 
         # Takes flag options and turns them into a runner config
         def process_flag_options options
+          extractor_options = options[:'extractor-opts'].merge({
+            filename: (options[:input] if options[:input].any?),
+            since: options[:since],
+            until: options[:until],
+            limit: options[:limit],
+          }.compact)
+
+          transformer_options = options[:'transformer-opts']
+
+          loader_options = options[:'loader-opts'].merge({
+            output: options[:output],
+            fields: options[:fields]
+          }.compact)
+
           {
             dry_run: options[:dry_run],
             extractor: {
               name: options[:extractor],
-              options: options[:'extractor-opts']
+              options: extractor_options
             }.compact,
             transformer: {
               name: options[:transformer],
-              options: options[:'transformer-opts']
+              options: transformer_options
             }.compact,
             loader: {
               name: options[:loader],
-              options: options[:'loader-opts']
+              options: loader_options
             }.compact
           }
         end

data/lib/chronicle/etl/cli/main.rb

@@ -1,17 +1,10 @@
-require 'thor'
-require 'chronicle/etl'
 require 'colorize'
 
-require 'chronicle/etl/cli/subcommand_base'
-require 'chronicle/etl/cli/connectors'
-require 'chronicle/etl/cli/jobs'
-
 module Chronicle
   module ETL
     module CLI
       # Main entrypoint for CLI app
-      class Main < Thor
-        class_option "verbose", type: :boolean, default: false
+      class Main < ::Thor
         default_task "jobs"
 
         desc 'connectors:COMMAND', 'Connectors available for ETL jobs', hide: true
@@ -22,15 +15,6 @@ module Chronicle
 
         # Entrypoint for the CLI
         def self.start(given_args = ARGV, config = {})
-          if given_args[0] == "--version"
-            puts "#{Chronicle::ETL::VERSION}"
-            exit
-          end
-
-          if given_args.none?
-            abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
-          end
-
           # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class
           if given_args.any? && given_args[0].include?(':')
             commands = given_args.shift.split(':')
@@ -40,10 +24,20 @@ module Chronicle
           super(given_args, config)
         end
 
+        def self.exit_on_failure?
+          true
+        end
+
+        desc "version", "Show version"
+        map %w(--version -v) => :version
+        def version
+          shell.say "chronicle-etl #{Chronicle::ETL::VERSION}"
+        end
+
         # Displays help options for chronicle-etl
         def help(meth = nil, subcommand = false)
           if meth && !respond_to?(meth)
-            klass, task = Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
+            klass, task = ::Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
             klass.start(['-h', task].compact, shell: shell)
           else
             shell.say "ABOUT".bold
@@ -64,7 +58,7 @@ module Chronicle
 
             list = []
 
-            Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
+            ::Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
               list += thor_class.printable_tasks(false)
             end
             list.sort! { |a, b| a[0] <=> b[0] }

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -2,11 +2,11 @@ module Chronicle
   module ETL
     module CLI
       # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
-      class SubcommandBase < Thor
+      class SubcommandBase < ::Thor
         # Print usage instructions for a subcommand
         def self.help(shell, subcommand = false)
           list = printable_commands(true, subcommand)
-          Thor::Util.thor_classes_in(self).each do |klass|
+          ::Thor::Util.thor_classes_in(self).each do |klass|
             list += klass.printable_commands(false)
           end
           list.sort! { |a, b| a[0] <=> b[0] }

data/lib/chronicle/etl/cli.rb

@@ -0,0 +1,7 @@
+require 'thor'
+require 'chronicle/etl'
+
+require 'chronicle/etl/cli/subcommand_base'
+require 'chronicle/etl/cli/connectors'
+require 'chronicle/etl/cli/jobs'
+require 'chronicle/etl/cli/main'

data/lib/chronicle/etl/configurable.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "ostruct"
+
+module Chronicle
+  module ETL
+    # A mixin that gives a class
+    # a {Chronicle::ETL::Configurable::ClassMethods#setting} macro to define
+    # settings and their properties (require, type, etc)
+    #
+    # @example Basic usage
+    #   class Test < Chronicle::ETL::Extractor
+    #     include Chronicle::ETL::Configurable
+    #     setting :when, type: :date, required: true
+    #   end
+    #
+    #   t = Test.new(when: '2022-02-24')
+    #   t.config.when
+    module Configurable
+      # An individual setting for this Configurable
+      Setting = Struct.new(:default, :required, :type)
+      private_constant :Setting
+
+      # Collection of user-supplied options for this Configurable
+      class Config < OpenStruct
+        # Config values that aren't nil, as a hash
+        def compacted_h
+          to_h.compact
+        end
+      end
+
+      # @private
+      def self.included(klass)
+        klass.extend(ClassMethods)
+        klass.include(InstanceMethods)
+        klass.prepend(Initializer)
+      end
+
+      # Initializer method for classes that have Configurable mixed in
+      module Initializer
+        # Make sure this class has a default @config ready to use
+        def initialize(*args)
+          @config = initialize_default_config
+          super
+        end
+      end
+
+      # Instance methods for classes that have Configurable mixed in
+      module InstanceMethods
+        attr_reader :config
+
+        # Take given options and apply them to this class's settings
+        # and make them available in @config and validates that they
+        # conform to setting rules
+        def apply_options(options)
+          options.transform_keys!(&:to_sym)
+
+          options.each do |name, value|
+            setting = self.class.all_settings[name]
+            raise(Chronicle::ETL::ConfigurationError, "Unrecognized setting: #{name}") unless setting
+
+            @config[name] = coerced_value(setting, value)
+          end
+          validate_config
+          options
+        end
+
+        # Name of all settings available to this class
+        def self.settings
+          self.class.all_settings.keys
+        end
+
+        private
+
+        def initialize_default_config
+          self.class.config_with_defaults
+        end
+
+        def validate_config
+          missing = (self.class.all_required_settings.keys - @config.compacted_h.keys)
+          raise Chronicle::ETL::ConfigurationError, "Missing options: #{missing}" if missing.count.positive?
+        end
+
+        def coerced_value(setting, value)
+          setting.type ? __send__("coerce_#{setting.type}", value) : value
+        end
+
+        def coerce_string(value)
+          value.to_s
+        end
+
+        def coerce_time(value)
+          # TODO: handle durations like '3h'
+          if value.is_a?(String)
+            Time.parse(value)
+          else
+            value
+          end
+        end
+      end
+
+      # Class methods for classes that have Configurable mixed in
+      module ClassMethods
+        # Macro for creating a setting on a class {::Chronicle::ETL::Configurable}
+        #
+        # @param [String] name Name of the setting
+        # @param [Boolean] required whether setting is required
+        # @param [Object] default Default value
+        # @param [Symbol] type Type
+        #
+        # @example Basic usage
+        #   setting :when, type: :date, required: true
+        #
+        # @see ::Chronicle::ETL::Configurable
+        def setting(name, default: nil, required: false, type: nil)
+          s = Setting.new(default, required, type)
+          settings[name] = s
+        end
+
+        # Collect all settings defined on this class and its ancestors (that
+        # have Configurable mixin included)
+        def all_settings
+          if superclass.include?(Chronicle::ETL::Configurable)
+            superclass.all_settings.merge(settings)
+          else
+            settings
+          end
+        end
+
+        # Filters settings to those that are required.
+        def all_required_settings
+          all_settings.select { |_name, setting| setting.required } || {}
+        end
+
+        def settings
+          @settings ||= {}
+        end
+
+        def setting_exists?(name)
+          all_settings.keys.include? name
+        end
+
+        def config_with_defaults
+          s = all_settings.transform_values(&:default)
+          Config.new(s)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/exceptions.rb

@@ -2,6 +2,8 @@ module Chronicle
   module ETL
     class Error < StandardError; end;
 
+    class ConfigurationError < Error; end;
+
     class RunnerTypeError < Error; end
 
     class ConnectorNotAvailableError < Error

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -2,21 +2,15 @@ require 'csv'
 
 module Chronicle
   module ETL
-    class CsvExtractor < Chronicle::ETL::Extractor
+    class CSVExtractor < Chronicle::ETL::Extractor
       include Extractors::Helpers::FilesystemReader
 
       register_connector do |r|
         r.description = 'input as CSV'
       end
 
-      DEFAULT_OPTIONS = {
-        headers: true,
-        filename: $stdin
-      }.freeze
-
-      def initialize(options = {})
-        super(DEFAULT_OPTIONS.merge(options))
-      end
+      setting :headers, default: true
+      setting :filename, default: $stdin
 
       def extract
         csv = initialize_csv
@@ -26,20 +20,20 @@ module Chronicle
       end
 
       def results_count
-        CSV.read(@options[:filename], headers: @options[:headers]).count unless stdin?(@options[:filename])
+        CSV.read(@config.filename, headers: @config.headers).count unless stdin?(@config.filename)
       end
 
       private
 
       def initialize_csv
-        headers = @options[:headers].is_a?(String) ? @options[:headers].split(',') : @options[:headers]
+        headers = @config.headers.is_a?(String) ? @config.headers.split(',') : @config.headers
 
         csv_options = {
           headers: headers,
           converters: :all
         }
 
-        open_from_filesystem(filename: @options[:filename]) do |file|
+        open_from_filesystem(filename: @config.filename) do |file|
           return CSV.new(file, **csv_options)
         end
       end

data/lib/chronicle/etl/extractors/extractor.rb

@@ -5,15 +5,20 @@ module Chronicle
     # Abstract class representing an Extractor for an ETL job
     class Extractor
       extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
+
+      setting :since, type: :date
+      setting :until, type: :date
+      setting :limit
+      setting :load_after_id
+      setting :filename
 
       # Construct a new instance of this extractor. Options are passed in from a Runner
-      # == Paramters:
+      # == Parameters:
       # options::
       #   Options for configuring this Extractor
       def initialize(options = {})
-        @options = options.transform_keys!(&:to_sym)
-        sanitize_options
-        handle_continuation
+        apply_options(options)
       end
 
       # Hook called before #extract. Useful for gathering data, initailizing proxies, etc
@@ -30,17 +35,13 @@ module Chronicle
 
       private
 
-      def sanitize_options
-        @options[:load_since] = Time.parse(@options[:load_since]) if @options[:load_since] && @options[:load_since].is_a?(String)
-        @options[:load_until] = Time.parse(@options[:load_until]) if @options[:load_until] && @options[:load_until].is_a?(String)
-      end
-
-      def handle_continuation
-        return unless @options[:continuation]
+      # TODO: reimplemenet this
+      # def handle_continuation
+      #   return unless @config.continuation
 
-        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation].highest_timestamp
-        @options[:load_after_id] = @options[:continuation].last_id if @options[:continuation].last_id
-      end
+      #   @config.since = @config.continuation.highest_timestamp if @config.continuation.highest_timestamp
+      #   @config.load_after_id = @config.continuation.last_id if @config.continuation.last_id
+      # end
     end
   end
 end

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -9,6 +9,9 @@ module Chronicle
         r.description = 'file or directory of files'
       end
 
+      # TODO: consolidate this with @config.filename
+      setting :dir_glob_pattern
+
       def extract
         filenames.each do |filename|
           yield Chronicle::ETL::Extraction.new(data: filename)
@@ -23,10 +26,10 @@ module Chronicle
 
       def filenames
         @filenames ||= filenames_in_directory(
-          path: @options[:filename],
-          dir_glob_pattern: @options[:dir_glob_pattern],
-          load_since: @options[:load_since],
-          load_until: @options[:load_until]
+          path: @config.filename,
+          dir_glob_pattern: @config.dir_glob_pattern,
+          load_since: @config.since,
+          load_until: @config.until
         )
       end
     end

data/lib/chronicle/etl/extractors/json_extractor.rb

@@ -7,16 +7,8 @@ module Chronicle
         r.description = 'input as JSON'
       end
 
-      DEFAULT_OPTIONS = {
-        filename: $stdin,
-
-        # We're expecting line-separated json objects
-        jsonl: true
-      }.freeze
-
-      def initialize(options = {})
-        super(DEFAULT_OPTIONS.merge(options))
-      end
+      setting :filename, default: $stdin
+      setting :jsonl, default: true
 
       def extract
         load_input do |input|

data/lib/chronicle/etl/job.rb

@@ -35,7 +35,7 @@ module Chronicle
 
       def instantiate_transformer(extraction)
         @transformer_klass = @job_definition.transformer_klass
-        @transformer_klass.new(@transformer_options, extraction)
+        @transformer_klass.new(extraction, @transformer_options)
       end
 
       def instantiate_loader

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -2,7 +2,7 @@ require 'csv'
 
 module Chronicle
   module ETL
-    class CsvLoader < Chronicle::ETL::Loader
+    class CSVLoader < Chronicle::ETL::Loader
       register_connector do |r|
         r.description = 'CSV'
       end

data/lib/chronicle/etl/loaders/loader.rb

@@ -3,13 +3,16 @@ module Chronicle
     # Abstract class representing a Loader for an ETL job
     class Loader
       extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
+
+      setting :output
 
       # Construct a new instance of this loader. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Loader
       def initialize(options = {})
-        @options = options
+        apply_options(options)
       end
 
       # Called once before processing records

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -9,19 +9,19 @@ module Chronicle
         r.description = 'a REST endpoint'
       end
 
-      def initialize( options={} )
-        super(options)
-      end
+      setting :hostname, required: true
+      setting :endpoint, required: true
+      setting :access_token
 
       def load(record)
         payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
         # have the outer data key that json-api expects
         payload = { data: payload } unless payload[:data]
 
-        uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
+        uri = URI.parse("#{@config.hostname}#{@config.endpoint}")
 
         header = {
-          "Authorization" => "Bearer #{@options[:access_token]}",
+          "Authorization" => "Bearer #{@config.access_token}",
           "Content-Type": 'application/json'
         }
         use_ssl = uri.scheme == 'https'

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -9,20 +9,14 @@ module Chronicle
         r.description = 'an ASCII table'
       end
 
-      DEFAULT_OPTIONS = {
-        fields_limit: nil,
-        fields_exclude: ['lids', 'type'],
-        fields_include: [],
-        truncate_values_at: nil,
-        table_renderer: :basic
-      }.freeze
-
-      def initialize(options={})
-        @options = options.reverse_merge(DEFAULT_OPTIONS)
-        @records = []
-      end
+      setting :fields_limit, default: nil
+      setting :fields_exclude, default: ['lids', 'type']
+      setting :fields_include, default: []
+      setting :truncate_values_at, default: 40
+      setting :table_renderer, default: :basic
 
       def load(record)
+        @records ||= []
         @records << record.to_h_flattened
       end
 
@@ -34,7 +28,7 @@ module Chronicle
 
         @table = TTY::Table.new(header: headers, rows: rows)
         puts @table.render(
-          @options[:table_renderer].to_sym,
+          @config.table_renderer.to_sym,
           padding: [0, 2, 0, 0]
         )
       end
@@ -43,15 +37,15 @@ module Chronicle
 
       def build_headers(records)
         headers =
-          if @options[:fields_include].any?
-            Set[*@options[:fields_include]]
+          if @config.fields_include.any?
+            Set[*@config.fields_include]
           else
             # use all the keys of the flattened record hash
             Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
           end
 
-        headers = headers.delete_if { |header| header.end_with?(*@options[:fields_exclude]) } if @options[:fields_exclude].any?
-        headers = headers.first(@options[:fields_limit]) if @options[:fields_limit]
+        headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) } if @config.fields_exclude.any?
+        headers = headers.first(@config.fields_limit) if @config.fields_limit
 
         headers.to_a.map(&:to_sym)
       end
@@ -60,8 +54,8 @@ module Chronicle
         records.map do |record|
           values = record.values_at(*headers).map{|value| value.to_s }
 
-          if @options[:truncate_values_at]
-            values = values.map{ |value| value.truncate(@options[:truncate_values_at]) }
+          if @config.truncate_values_at
+            values = values.map{ |value| value.truncate(@config.truncate_values_at) }
           end
 
           values

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -3,6 +3,7 @@ module Chronicle
     module Registry
       # Records details about a connector such as its provider and a description
       class ConnectorRegistration
+        # FIXME: refactor custom accessor methods later in file
         attr_accessor :identifier, :provider, :klass, :description
 
         def initialize(klass)

data/lib/chronicle/etl/transformers/image_file_transformer.rb

@@ -19,20 +19,14 @@ module Chronicle
         r.description = 'an image file'
       end
 
-      DEFAULT_OPTIONS = {
-        timestamp_strategy: 'file_mtime',
-        id_strategy: 'file_hash',
-        verb: 'photographed',
-
-        # EXIF tags often don't have timezones
-        timezone_default: 'Eastern Time (US & Canada)',
-        include_image_data: true
-      }.freeze
-
-      def initialize(*args)
-        super(*args)
-        @options = @options.reverse_merge(DEFAULT_OPTIONS)
-      end
+      setting :timestamp_strategy, default: 'file_mtime'
+      setting :id_strategy, default: 'file_hash'
+      setting :verb, default: 'photographed'
+      # EXIF tags often don't have timezones
+      setting :timezone_default, default: 'Eastern Time (US & Canada)'
+      setting :include_image_data, default: true
+      setting :actor
+      setting :involved
 
       def transform
         # FIXME: set @filename; use block for reading file when necessary
@@ -48,7 +42,7 @@ module Chronicle
 
       def id
         @id ||= begin
-          id = build_with_strategy(field: :id, strategy: @options[:id_strategy])
+          id = build_with_strategy(field: :id, strategy: @config.id_strategy)
           raise UntransformableRecordError.new("Could not build id", transformation: self) unless id
 
           id
@@ -57,7 +51,7 @@ module Chronicle
 
       def timestamp
         @timestamp ||= begin
-          ts = build_with_strategy(field: :timestamp, strategy: @options[:timestamp_strategy])
+          ts = build_with_strategy(field: :timestamp, strategy: @config.timestamp_strategy)
           raise UntransformableRecordError.new("Could not build timestamp", transformation: self) unless ts
 
           ts
@@ -68,8 +62,8 @@ module Chronicle
 
       def build_created(file)
         record = ::Chronicle::ETL::Models::Activity.new
-        record.verb = @options[:verb]
-        record.provider = @options[:provider]
+        record.verb = @config.verb
+        record.provider = @config.provider
         record.provider_id = id
         record.end_at = timestamp
         record.dedupe_on = [[:provider_id, :verb, :provider]]
@@ -84,24 +78,24 @@ module Chronicle
       def build_actor
         actor = ::Chronicle::ETL::Models::Entity.new
         actor.represents = 'identity'
-        actor.provider = @options[:actor][:provider]
-        actor.slug = @options[:actor][:slug]
+        actor.provider = @config.actor[:provider]
+        actor.slug = @config.actor[:slug]
         actor.dedupe_on = [[:provider, :slug, :represents]]
         actor
       end
 
       def build_image
         image = ::Chronicle::ETL::Models::Entity.new
-        image.represents = @options[:involved][:represents]
+        image.represents = @config.involved[:represents]
         image.title = build_title
         image.body = exif['Description']
-        image.provider = @options[:involved][:provider]
+        image.provider = @config.involved[:provider]
         image.provider_id = id
         image.assign_attributes(build_gps)
         image.dedupe_on = [[:provider, :provider_id, :represents]]
 
-        if @options[:ocr_strategy]
-          ocr_text = build_with_strategy(field: :ocr, strategy: @options[:ocr_strategy])
+        if @config.ocr_strategy
+          ocr_text = build_with_strategy(field: :ocr, strategy: @config.ocr_strategy)
           image.metadata[:ocr_text] = ocr_text if ocr_text
         end
 
@@ -111,7 +105,7 @@ module Chronicle
         image.depicts = build_people_depicted(names)
         image.abouts = build_keywords(tags)
 
-        if @options[:include_image_data]
+        if @config.include_image_data
           attachment = ::Chronicle::ETL::Models::Attachment.new
           attachment.data = build_image_data
           image.attachments = [attachment]
@@ -124,7 +118,7 @@ module Chronicle
         topics.map do |topic|
           t = ::Chronicle::ETL::Models::Entity.new
           t.represents = 'topic'
-          t.provider = @options[:involved][:provider]
+          t.provider = @config.involved[:provider]
           t.title = topic
           t.slug = topic.parameterize
           t.dedupe_on = [[:provider, :represents, :slug]]
@@ -136,7 +130,7 @@ module Chronicle
         names.map do |name|
           identity = ::Chronicle::ETL::Models::Entity.new
           identity.represents = 'identity'
-          identity.provider = @options[:involved][:provider]
+          identity.provider = @config.involved[:provider]
           identity.slug = name.parameterize
           identity.title = name
           identity.dedupe_on = [[:provider, :represents, :slug]]
@@ -199,7 +193,7 @@ module Chronicle
         elsif false
           # TODO: support option of using GPS coordinates to determine timezone
         else
-          zone = ActiveSupport::TimeZone.new(@options[:timezone_default])
+          zone = ActiveSupport::TimeZone.new(@config.timezone_default)
           timestamp = zone.parse(timestamp.asctime)
         end
 

data/lib/chronicle/etl/transformers/transformer.rb

@@ -3,14 +3,15 @@ module Chronicle
     # Abstract class representing an Transformer for an ETL job
     class Transformer
       extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
 
       # Construct a new instance of this transformer. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Transformer
-      def initialize(options = {}, extraction)
-        @options = options
+      def initialize(extraction, options = {})
         @extraction = extraction
+        apply_options(options)
       end
 
       # @abstract Subclass is expected to implement #transform

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.3.1"
+    VERSION = "0.4.0"
   end
 end

data/lib/chronicle/etl.rb

@@ -1,5 +1,6 @@
 require_relative 'etl/registry/registry'
 require_relative 'etl/config'
+require_relative 'etl/configurable'
 require_relative 'etl/exceptions'
 require_relative 'etl/extraction'
 require_relative 'etl/extractors/extractor'

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Andrew Louis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-10 00:00:00.000000000 Z
+date: 2022-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: chronic_duration
   requirement: !ruby/object:Gem::Requirement
@@ -98,16 +98,16 @@ dependencies:
   name: runcom
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: tty-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +234,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.25.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.25.1
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -243,14 +299,15 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".ruby-version"
 - ".travis.yml"
 - ".yardopts"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -259,11 +316,13 @@ files:
 - chronicle-etl.gemspec
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
+- lib/chronicle/etl/cli.rb
 - lib/chronicle/etl/cli/connectors.rb
 - lib/chronicle/etl/cli/jobs.rb
 - lib/chronicle/etl/cli/main.rb
 - lib/chronicle/etl/cli/subcommand_base.rb
 - lib/chronicle/etl/config.rb
+- lib/chronicle/etl/configurable.rb
 - lib/chronicle/etl/exceptions.rb
 - lib/chronicle/etl/extraction.rb
 - lib/chronicle/etl/extractors/csv_extractor.rb
@@ -317,14 +376,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: ETL tool for personal data

data/.ruby-version

@@ -1 +0,0 @@
-2.7.1