chronicle-etl 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a02a2377d0e8d4135f3b931bc73641eac28058d736d9c1dba0a97107c1d4c0e
-  data.tar.gz: 810d5bff80e852fa08ef9824ed6b313aa309bb69e84228bc1fbb7595069e043b
+  metadata.gz: bfd4e081bfeda7d097a5a5eee6ccf28baf0a9b3878968d74c9a604013d0b55a6
+  data.tar.gz: 003ebd2ffe2b1220c7f43a4875043cb5500aa1b7a6327b84c9be10f04e0e8d40
 SHA512:
-  metadata.gz: 0d5fbea3c63349bb3f566e6137755f6cc8a4060d0e401abf5a0e7d8b44a4c4278089c10ffb8bb9cf2d783a238449140e5e54d90f3ad158aa362c6335eedca5aa
-  data.tar.gz: bf6fa83b1d5e55760e62d3cc090bf09bb69a7c761ae4a9358fb4d82192c7efc7500b6db361f39adac3581982862654aa4603a78dfbb3aed53b51d01137ffd736
+  metadata.gz: 3d786fb4acf8d0b03e65262209def310ca25b92646847f6e96791e6491e9b159ab11db7fa35f785f6782fbc0b9e3daebb625e2353fce2422f7fc79aed7a4d6bc
+  data.tar.gz: 87771745b9df2160966299f1d73eb568b46080ed217a1952e1dd938fff7758432f2cc6f449036d24951660b405e18083f1adac26d00243a0fab9003a96eb569d

data/.gitignore

@@ -7,6 +7,9 @@
 /spec/reports/
 /tmp/
 
+# https://yehudakatz.com/2010/12/16/clarifying-the-roles-of-the-gemspec-and-gemfile/
+Gemfile.lock
+
 # rspec failure tracking
 .rspec_status
 .DS_Store

data/.rubocop.yml

@@ -5,4 +5,7 @@ Style/StringLiterals:
   Enabled: false
 
 Style/MethodCallWithArgsParentheses:
+  Enabled: false
+
+Lint/ConstantResolution:
   Enabled: false

data/README.md

@@ -31,6 +31,9 @@ Connectors are available to read, process, and load data from different formats
 ```bash
 # List all available connectors
 $ chronicle-etl connectors:list
+
+# Install a connector
+$ chronicle-etl connectors:install imessage
 ```
 
 Built in connectors:
@@ -44,16 +47,18 @@ Built in connectors:
 - `null` - (default) Don't do anything
 
 ### Loaders
-- `stdout` - (default) output transformed records to stdount
+- `stdout` - (default) output records to stdout serialized as JSON
 - `csv` - Load records to a csv file
+- `rest` - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
 - `table` - Output an ascii table of records. Useful for debugging.
 
 ### Provider-specific importers
 
 In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
 
-- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files. Transformers for chronicle schema
-- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
+- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files
+- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`
+- [imessage](https://github.com/chronicle-app/chronicle-imessage). Extract iMessage messages from a local macOS installation
 
 To install any of these, run `gem install chronicle-PROVIDER`. 
 
@@ -61,7 +66,7 @@ If you don't want to use the available rubygem importers, `chronicle-etl` can us
 
 I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.net) to chat about what will be available!
 
-### Full commands
+## Full commands
 
 ```
 $ chronicle-etl help 
@@ -75,26 +80,28 @@ ALL COMMANDS
   jobs:create                # Create a job
   jobs:list                  # List all available jobs
   jobs:run                   # Start a job
-  jobs:show                  # Show a job
+  jobs:show                  # Show details about a job
 ```
 
-### Job options
+### Running a job
 
 ```
 Usage:
   chronicle-etl jobs:run
 
 Options:
-  -e, [--extractor=extractor-name]      # Extractor class (available: stdin, csv, file)
-                                        # Default: stdin
+      [--log-level=LOG_LEVEL]           # Log level (debug, info, warn, error, fatal)
+                                        # Default: info
+  -v, [--verbose], [--no-verbose]       # Set log level to verbose
+      [--dry-run], [--no-dry-run]       # Only run the extraction and transform steps, not the loading
+  -e, [--extractor=extractor-name]      # Extractor class. Default: stdin
       [--extractor-opts=key:value]      # Extractor options
-  -t, [--transformer=transformer-name]  # Transformer class (available: null)
-                                        # Default: null
+  -t, [--transformer=transformer-name]  # Transformer class. Default: null
       [--transformer-opts=key:value]    # Transformer options
-  -l, [--loader=loader-name]            # Loader class (available: stdout, csv, table)
-                                        # Default: stdout
+  -l, [--loader=loader-name]            # Loader class. Default: stdout
       [--loader-opts=key:value]         # Loader options
-  -j, [--job=JOB]                       # Job configuration file
+  -j, [--name=NAME]                     # Job configuration name
+
 
 Runs an ETL job
 ```

data/chronicle-etl.gemspec

@@ -36,18 +36,21 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "chronic_duration", "~> 0.10.6"
   spec.add_dependency "colorize", "~> 0.8.1"
-  spec.add_dependency "tty-table", "~> 0.11"
+  spec.add_dependency "marcel", "~> 1.0.2"
+  spec.add_dependency "mini_exiftool", "~> 2.10"
+  spec.add_dependency "nokogiri", "~> 1.13"
+  spec.add_dependency "runcom", "~> 6.2"
+  spec.add_dependency "sequel", "~> 5.35"
+  spec.add_dependency "sqlite3", "~> 1.4"
+  spec.add_dependency "thor", "~> 0.20"
   spec.add_dependency "tty-progressbar", "~> 0.17"
-  spec.add_dependency 'sequel', '~> 5.35'
-  spec.add_dependency 'deep_merge', '~> 1.2'
+  spec.add_dependency "tty-table", "~> 0.11"
 
   spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "pry-byebug", "~> 3.9"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "pry-byebug", "~> 3.9"
-  spec.add_development_dependency 'runcom', '~> 6.2'
-  spec.add_development_dependency 'redcarpet', '~> 3.5'
-  spec.add_development_dependency 'sqlite3', '~> 1.4'
 end

data/lib/chronicle/etl/cli/connectors.rb

@@ -7,23 +7,35 @@ module Chronicle
         namespace :connectors
 
         desc "install NAME", "Installs connector NAME"
-        def install
-          puts "Installing"
+        def install(name)
+          Chronicle::ETL::Registry.install_connector(name)
         end
 
         desc "list", "Lists available connectors"
         # Display all available connectors that chronicle-etl has access to
         def list
-          klasses = Chronicle::ETL::Catalog.available_classes
-          klasses = klasses.sort_by do |a|
-            [a[:built_in].to_s, a[:provider], a[:phase]]
+          Chronicle::ETL::Registry.load_all!
+
+          connector_info = Chronicle::ETL::Registry.connectors.map do |connector_registration|
+            {
+              identifier: connector_registration.identifier,
+              phase: connector_registration.phase,
+              description: connector_registration.descriptive_phrase,
+              provider: connector_registration.provider,
+              core: connector_registration.built_in? ? '✓' : '',
+              class: connector_registration.klass_name
+            }
+          end
+
+          connector_info = connector_info.sort_by do |a|
+            [a[:core].to_s, a[:provider], a[:phase], a[:identifier]]
           end
 
-          headers = klasses.first.keys.map do |key|
+          headers = connector_info.first.keys.map do |key|
             key.to_s.upcase.bold
           end
 
-          table = TTY::Table.new(headers, klasses.map(&:values))
+          table = TTY::Table.new(headers, connector_info.map(&:values))
           puts table.render(indent: 0, padding: [0, 2])
         end
       end

data/lib/chronicle/etl/cli/jobs.rb

@@ -7,16 +7,19 @@ module Chronicle
         default_task "start"
         namespace :jobs
 
-        class_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
+        class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'extractor-name'
         class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
-        class_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
+        class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'transformer-name'
         class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
-        class_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
+        class_option :loader, aliases: '-l', desc: 'Loader class. Default: stdout', banner: 'loader-name'
         class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
         class_option :name, aliases: '-j', desc: 'Job configuration name'
 
         map run: :start  # Thor doesn't like `run` as a command name
         desc "run", "Start a job"
+        option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
+        option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
+        option :dry_run, desc: 'Only run the extraction and transform steps, not the loading', type: :boolean
         long_desc <<-LONG_DESC
           This will run an ETL job. Each job needs three parts:
 
@@ -24,23 +27,17 @@ module Chronicle
 
             2. #{'Transformer'.underline}: transforms data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
 
-            3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout.
+            3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout. With the --dry-run option, this step won't be run.
 
             If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
 LONG_DESC
         # Run an ETL job
         def start
+          setup_log_level
           job_definition = build_job_definition(options)
           job = Chronicle::ETL::Job.new(job_definition)
           runner = Chronicle::ETL::Runner.new(job)
           runner.run!
-        rescue Chronicle::ETL::ProviderNotAvailableError => e
-          warn(e.message.red)
-          warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
-          exit(false)
-        rescue Chronicle::ETL::ConnectorNotAvailableError => e
-          warn(e.message.red)
-          exit(false)
         end
 
         desc "create", "Create a job"
@@ -48,14 +45,13 @@ LONG_DESC
         def create
           job_definition = build_job_definition(options)
           path = File.join('chronicle', 'etl', 'jobs', options[:name])
-          Chronicle::ETL::Config.write(path, job_definition)
+          Chronicle::ETL::Config.write(path, job_definition.definition)
         end
 
         desc "show", "Show details about a job"
         # Show an ETL job
         def show
-          job_config = build_job_definition(options)
-          pp job_config
+          puts Chronicle::ETL::Job.new(build_job_definition(options))
         end
 
         desc "list", "List all available jobs"
@@ -81,11 +77,20 @@ LONG_DESC
 
         private
 
+        def setup_log_level
+          if options[:verbose]
+            Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::DEBUG
+          elsif options[:log_level]
+            level = Chronicle::ETL::Logger.const_get(options[:log_level].upcase)
+            Chronicle::ETL::Logger.log_level = level 
+          end
+        end
+
         # Create job definition by reading config file and then overwriting with flag options
         def build_job_definition(options)
           definition = Chronicle::ETL::JobDefinition.new
-          definition.add_config(process_flag_options(options))
           definition.add_config(load_job_config(options[:name]))
+          definition.add_config(process_flag_options(options))
           definition
         end
 
@@ -96,18 +101,19 @@ LONG_DESC
         # Takes flag options and turns them into a runner config
         def process_flag_options options
           {
+            dry_run: options[:dry_run],
             extractor: {
               name: options[:extractor],
               options: options[:'extractor-opts']
-            },
+            }.compact,
             transformer: {
               name: options[:transformer],
               options: options[:'transformer-opts']
-            },
+            }.compact,
             loader: {
               name: options[:loader],
               options: options[:'loader-opts']
-            }
+            }.compact
           }
         end
       end

data/lib/chronicle/etl/cli/main.rb

@@ -22,6 +22,11 @@ module Chronicle
 
         # Entrypoint for the CLI
         def self.start(given_args = ARGV, config = {})
+          if given_args[0] == "--version"
+            puts "#{Chronicle::ETL::VERSION}"
+            exit
+          end
+
           if given_args.none?
             abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
           end
@@ -52,10 +57,10 @@ module Chronicle
             shell.say "  $ chronicle-etl connectors:list"
             shell.say
             shell.say "  Run a simple job:".italic.light_black
-            shell.say "  $ chronicle-etl jobs:start --extractor stdin --transformer null --loader stdout"
+            shell.say "  $ chronicle-etl jobs:run --extractor stdin --transformer null --loader stdout"
             shell.say
             shell.say "  Show full job options:".italic.light_black
-            shell.say "  $ chronicle-etl jobs help start"
+            shell.say "  $ chronicle-etl jobs help run"
 
             list = []
 
@@ -72,6 +77,9 @@ module Chronicle
             shell.say "VERSION".bold
             shell.say "  #{Chronicle::ETL::VERSION}"
             shell.say
+            shell.say "  Display current version:".italic.light_black
+            shell.say "  $ chronicle-etl --version"
+            shell.say
             shell.say "FULL DOCUMENTATION".bold
             shell.say "  https://github.com/chronicle-app/chronicle-etl".blue
             shell.say

data/lib/chronicle/etl/config.rb

@@ -30,7 +30,7 @@ module Chronicle
         end
       end
 
-      # Returns all available credentials available in ~/.config/chronilce/etl/credenetials/*.yml
+      # Returns all available credentials available in ~/.config/chronicle/etl/credentials/*.yml
       def available_credentials
         job_directory = Runcom::Config.new('chronicle/etl/credentials').current
         Dir.glob(File.join(job_directory, "*.yml")).map do |filename|

data/lib/chronicle/etl/exceptions.rb

@@ -2,7 +2,7 @@ module Chronicle
   module ETL
     class Error < StandardError; end;
 
-    class InvalidTransformedRecordError < Error; end
+    class RunnerTypeError < Error; end
 
     class ConnectorNotAvailableError < Error
       def initialize(message, provider: nil, name: nil)
@@ -15,5 +15,16 @@ module Chronicle
 
     class ProviderNotAvailableError < ConnectorNotAvailableError; end
     class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
+
+    class TransformationError < Error
+      attr_reader :transformation
+
+      def initialize(message=nil, transformation:)
+        super(message)
+        @transformation = transformation
+      end
+    end
+
+    class UntransformableRecordError < TransformationError; end
   end
 end

data/lib/chronicle/etl/extraction.rb

@@ -0,0 +1,12 @@
+module Chronicle
+  module ETL
+    class Extraction
+      attr_accessor :data, :meta
+
+      def initialize(data: {}, meta: {})
+        @data = data
+        @meta = meta
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -1,41 +1,48 @@
 require 'csv'
-class Chronicle::ETL::CsvExtractor < Chronicle::ETL::Extractor
-  DEFAULT_OPTIONS = {
-    headers: true,
-    filename: $stdin
-  }.freeze
-
-  def initialize(options = {})
-    super(DEFAULT_OPTIONS.merge(options))
-  end
 
-  def extract
-    csv = initialize_csv
-    csv.each do |row|
-      result = row.to_h
-      yield result
+module Chronicle
+  module ETL
+    class CsvExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::FilesystemReader
+
+      register_connector do |r|
+        r.description = 'input as CSV'
+      end
+
+      DEFAULT_OPTIONS = {
+        headers: true,
+        filename: $stdin
+      }.freeze
+
+      def initialize(options = {})
+        super(DEFAULT_OPTIONS.merge(options))
+      end
+
+      def extract
+        csv = initialize_csv
+        csv.each do |row|
+          yield Chronicle::ETL::Extraction.new(data: row.to_h)
+        end
+      end
+
+      def results_count
+        CSV.read(@options[:filename], headers: @options[:headers]).count unless stdin?(@options[:filename])
+      end
+
+      private
+
+      def initialize_csv
+        headers = @options[:headers].is_a?(String) ? @options[:headers].split(',') : @options[:headers]
+
+        csv_options = {
+          headers: headers,
+          converters: :all
+        }
+
+        open_from_filesystem(filename: @options[:filename]) do |file|
+          return CSV.new(file, **csv_options)
+        end
+      end
     end
   end
-
-  def results_count
-    CSV.read(@options[:filename], headers: @options[:headers]).count if read_from_file?
-  end
-
-  private
-
-  def initialize_csv
-    headers = @options[:headers].is_a?(String) ? @options[:headers].split(',') : @options[:headers]
-
-    csv_options = {
-      headers: headers,
-      converters: :all
-    }
-
-    stream = read_from_file? ? File.open(@options[:filename]) : @options[:filename]
-    CSV.new(stream, **csv_options)
-  end
-
-  def read_from_file?
-    @options[:filename] != $stdin
-  end
 end

data/lib/chronicle/etl/extractors/extractor.rb

@@ -4,7 +4,7 @@ module Chronicle
   module ETL
     # Abstract class representing an Extractor for an ETL job
     class Extractor
-      extend Chronicle::ETL::Catalog
+      extend Chronicle::ETL::Registry::SelfRegistering
 
       # Construct a new instance of this extractor. Options are passed in from a Runner
       # == Paramters:
@@ -12,6 +12,7 @@ module Chronicle
       #   Options for configuring this Extractor
       def initialize(options = {})
         @options = options.transform_keys!(&:to_sym)
+        sanitize_options
         handle_continuation
       end
 
@@ -26,6 +27,11 @@ module Chronicle
 
       private
 
+      def sanitize_options
+        @options[:load_since] = Time.parse(@options[:load_since]) if @options[:load_since] && @options[:load_since].is_a?(String)
+        @options[:load_until] = Time.parse(@options[:load_until]) if @options[:load_until] && @options[:load_until].is_a?(String)
+      end
+
       def handle_continuation
         return unless @options[:continuation]
 
@@ -36,6 +42,8 @@ module Chronicle
   end
 end
 
+require_relative 'helpers/filesystem_reader'
 require_relative 'csv_extractor'
 require_relative 'file_extractor'
+require_relative 'json_extractor'
 require_relative 'stdin_extractor'

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -3,49 +3,31 @@ require 'pathname'
 module Chronicle
   module ETL
     class FileExtractor < Chronicle::ETL::Extractor
-      def extract
-        if file?
-          extract_file do |data, metadata|
-            yield(data, metadata)
-          end
-        elsif directory?
-          extract_from_directory do |data, metadata|
-            yield(data, metadata)
-          end
-        end
-      end
+      include Extractors::Helpers::FilesystemReader
 
-      def results_count
-        if file?
-          return 1
-        else
-          search_pattern = File.join(@options[:filename], '**/*.eml')
-          Dir.glob(search_pattern).count
-        end
+      register_connector do |r|
+        r.description = 'file or directory of files'
       end
 
-      private
-
-      def extract_from_directory
-        search_pattern = File.join(@options[:filename], '**/*.eml')
-        filenames = Dir.glob(search_pattern)
+      def extract
         filenames.each do |filename|
-          file = File.open(filename)
-          yield(file.read, {filename: file})
+          yield Chronicle::ETL::Extraction.new(data: filename)
         end
       end
 
-      def extract_file
-        file = File.open(@options[:filename])
-        yield(file.read, {filename: @options[:filename]})
+      def results_count
+        filenames.count
       end
 
-      def directory?
-        Pathname.new(@options[:filename]).directory?
-      end
+      private
 
-      def file?
-        Pathname.new(@options[:filename]).file?
+      def filenames
+        @filenames ||= filenames_in_directory(
+          path: @options[:filename],
+          dir_glob_pattern: @options[:dir_glob_pattern],
+          load_since: @options[:load_since],
+          load_until: @options[:load_until]
+        )
       end
     end
   end

data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb

@@ -0,0 +1,104 @@
+require 'pathname'
+
+module Chronicle
+  module ETL
+    module Extractors
+      module Helpers
+        module FilesystemReader
+
+          def filenames_in_directory(...)
+            filenames = gather_files(...)
+            if block_given?
+              filenames.each do |filename|
+                yield filename
+              end
+            else
+              filenames
+            end
+          end
+
+          def read_from_filesystem(filename:, yield_each_line: true, dir_glob_pattern: '**/*')
+            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
+              if yield_each_line
+                file.each_line do |line|
+                  yield line
+                end
+              else
+                yield file.read
+              end
+            end
+          end
+
+          def open_from_filesystem(filename:, dir_glob_pattern: '**/*')
+            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
+              yield file
+            end
+          end
+
+          def results_count
+            raise NotImplementedError
+            # if file?
+            #   return 1
+            # else
+            #   search_pattern = File.join(@options[:filename], '**/*')
+            #   Dir.glob(search_pattern).count
+            # end
+          end
+
+          private
+
+          def gather_files(path:, dir_glob_pattern: '**/*', load_since: nil, load_until: nil, smaller_than: nil, larger_than: nil, sort: :mtime)
+            search_pattern = File.join(path, '**', dir_glob_pattern)
+            files = Dir.glob(search_pattern)
+
+            files = files.keep_if {|f| (File.mtime(f) > load_since)} if load_since
+            files = files.keep_if {|f| (File.mtime(f) < load_until)} if load_until
+
+            # pass in file sizes in bytes
+            files = files.keep_if {|f| (File.size(f) < smaller_than)} if smaller_than
+            files = files.keep_if {|f| (File.size(f) > larger_than)} if larger_than
+
+            # TODO: incorporate sort argument
+            files.sort_by{ |f| File.mtime(f) }
+          end
+
+          def select_files_in_directory(path:, dir_glob_pattern: '**/*')
+            raise IOError.new("#{path} is not a directory.") unless directory?(path)
+
+            search_pattern = File.join(path, dir_glob_pattern)
+            Dir.glob(search_pattern).each do |filename|
+              yield(filename)
+            end
+          end
+
+          def open_files(filename:, dir_glob_pattern:)
+            if stdin?(filename)
+              yield $stdin
+            elsif directory?(filename)
+              search_pattern = File.join(filename, dir_glob_pattern)
+              filenames = Dir.glob(search_pattern)
+              filenames.each do |filename|
+                file = File.open(filename)
+                yield(file)
+              end
+            elsif file?(filename)
+              yield File.open(filename)
+            end
+          end
+
+          def stdin?(filename)
+            filename == $stdin
+          end
+
+          def directory?(filename)
+            Pathname.new(filename).directory?
+          end
+
+          def file?(filename)
+            Pathname.new(filename).file?
+          end
+        end
+      end
+    end
+  end
+end