chronicle-etl 0.2.4 → 0.4.0

data/lib/chronicle/etl/configurable.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "ostruct"
+
+module Chronicle
+  module ETL
+    # A mixin that gives a class
+    # a {Chronicle::ETL::Configurable::ClassMethods#setting} macro to define
+    # settings and their properties (require, type, etc)
+    #
+    # @example Basic usage
+    #   class Test < Chronicle::ETL::Extractor
+    #     include Chronicle::ETL::Configurable
+    #     setting :when, type: :date, required: true
+    #   end
+    #
+    #   t = Test.new(when: '2022-02-24')
+    #   t.config.when
+    module Configurable
+      # An individual setting for this Configurable
+      Setting = Struct.new(:default, :required, :type)
+      private_constant :Setting
+
+      # Collection of user-supplied options for this Configurable
+      class Config < OpenStruct
+        # Config values that aren't nil, as a hash
+        def compacted_h
+          to_h.compact
+        end
+      end
+
+      # @private
+      def self.included(klass)
+        klass.extend(ClassMethods)
+        klass.include(InstanceMethods)
+        klass.prepend(Initializer)
+      end
+
+      # Initializer method for classes that have Configurable mixed in
+      module Initializer
+        # Make sure this class has a default @config ready to use
+        def initialize(*args)
+          @config = initialize_default_config
+          super
+        end
+      end
+
+      # Instance methods for classes that have Configurable mixed in
+      module InstanceMethods
+        attr_reader :config
+
+        # Take given options and apply them to this class's settings
+        # and make them available in @config and validates that they
+        # conform to setting rules
+        def apply_options(options)
+          options.transform_keys!(&:to_sym)
+
+          options.each do |name, value|
+            setting = self.class.all_settings[name]
+            raise(Chronicle::ETL::ConfigurationError, "Unrecognized setting: #{name}") unless setting
+
+            @config[name] = coerced_value(setting, value)
+          end
+          validate_config
+          options
+        end
+
+        # Name of all settings available to this class
+        def self.settings
+          self.class.all_settings.keys
+        end
+
+        private
+
+        def initialize_default_config
+          self.class.config_with_defaults
+        end
+
+        def validate_config
+          missing = (self.class.all_required_settings.keys - @config.compacted_h.keys)
+          raise Chronicle::ETL::ConfigurationError, "Missing options: #{missing}" if missing.count.positive?
+        end
+
+        def coerced_value(setting, value)
+          setting.type ? __send__("coerce_#{setting.type}", value) : value
+        end
+
+        def coerce_string(value)
+          value.to_s
+        end
+
+        def coerce_time(value)
+          # TODO: handle durations like '3h'
+          if value.is_a?(String)
+            Time.parse(value)
+          else
+            value
+          end
+        end
+      end
+
+      # Class methods for classes that have Configurable mixed in
+      module ClassMethods
+        # Macro for creating a setting on a class {::Chronicle::ETL::Configurable}
+        #
+        # @param [String] name Name of the setting
+        # @param [Boolean] required whether setting is required
+        # @param [Object] default Default value
+        # @param [Symbol] type Type
+        #
+        # @example Basic usage
+        #   setting :when, type: :date, required: true
+        #
+        # @see ::Chronicle::ETL::Configurable
+        def setting(name, default: nil, required: false, type: nil)
+          s = Setting.new(default, required, type)
+          settings[name] = s
+        end
+
+        # Collect all settings defined on this class and its ancestors (that
+        # have Configurable mixin included)
+        def all_settings
+          if superclass.include?(Chronicle::ETL::Configurable)
+            superclass.all_settings.merge(settings)
+          else
+            settings
+          end
+        end
+
+        # Filters settings to those that are required.
+        def all_required_settings
+          all_settings.select { |_name, setting| setting.required } || {}
+        end
+
+        def settings
+          @settings ||= {}
+        end
+
+        def setting_exists?(name)
+          all_settings.keys.include? name
+        end
+
+        def config_with_defaults
+          s = all_settings.transform_values(&:default)
+          Config.new(s)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/exceptions.rb

@@ -2,7 +2,9 @@ module Chronicle
   module ETL
     class Error < StandardError; end;
 
-    class InvalidTransformedRecordError < Error; end
+    class ConfigurationError < Error; end;
+
+    class RunnerTypeError < Error; end
 
     class ConnectorNotAvailableError < Error
       def initialize(message, provider: nil, name: nil)
@@ -15,5 +17,16 @@ module Chronicle
 
     class ProviderNotAvailableError < ConnectorNotAvailableError; end
     class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
+
+    class TransformationError < Error
+      attr_reader :transformation
+
+      def initialize(message=nil, transformation:)
+        super(message)
+        @transformation = transformation
+      end
+    end
+
+    class UntransformableRecordError < TransformationError; end
   end
 end

data/lib/chronicle/etl/extraction.rb

@@ -0,0 +1,12 @@
+module Chronicle
+  module ETL
+    class Extraction
+      attr_accessor :data, :meta
+
+      def initialize(data: {}, meta: {})
+        @data = data
+        @meta = meta
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -1,41 +1,42 @@
 require 'csv'
-class Chronicle::ETL::CsvExtractor < Chronicle::ETL::Extractor
-  DEFAULT_OPTIONS = {
-    headers: true,
-    filename: $stdin
-  }.freeze
-
-  def initialize(options = {})
-    super(DEFAULT_OPTIONS.merge(options))
-  end
 
-  def extract
-    csv = initialize_csv
-    csv.each do |row|
-      result = row.to_h
-      yield result
-    end
-  end
+module Chronicle
+  module ETL
+    class CSVExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::FilesystemReader
 
-  def results_count
-    CSV.read(@options[:filename], headers: @options[:headers]).count if read_from_file?
-  end
+      register_connector do |r|
+        r.description = 'input as CSV'
+      end
 
-  private
+      setting :headers, default: true
+      setting :filename, default: $stdin
 
-  def initialize_csv
-    headers = @options[:headers].is_a?(String) ? @options[:headers].split(',') : @options[:headers]
+      def extract
+        csv = initialize_csv
+        csv.each do |row|
+          yield Chronicle::ETL::Extraction.new(data: row.to_h)
+        end
+      end
 
-    csv_options = {
-      headers: headers,
-      converters: :all
-    }
+      def results_count
+        CSV.read(@config.filename, headers: @config.headers).count unless stdin?(@config.filename)
+      end
 
-    stream = read_from_file? ? File.open(@options[:filename]) : @options[:filename]
-    CSV.new(stream, **csv_options)
-  end
+      private
+
+      def initialize_csv
+        headers = @config.headers.is_a?(String) ? @config.headers.split(',') : @config.headers
 
-  def read_from_file?
-    @options[:filename] != $stdin
+        csv_options = {
+          headers: headers,
+          converters: :all
+        }
+
+        open_from_filesystem(filename: @config.filename) do |file|
+          return CSV.new(file, **csv_options)
+        end
+      end
+    end
   end
 end

data/lib/chronicle/etl/extractors/extractor.rb

@@ -4,38 +4,50 @@ module Chronicle
   module ETL
     # Abstract class representing an Extractor for an ETL job
     class Extractor
-      extend Chronicle::ETL::Catalog
+      extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
+
+      setting :since, type: :date
+      setting :until, type: :date
+      setting :limit
+      setting :load_after_id
+      setting :filename
 
       # Construct a new instance of this extractor. Options are passed in from a Runner
-      # == Paramters:
+      # == Parameters:
       # options::
       #   Options for configuring this Extractor
       def initialize(options = {})
-        @options = options.transform_keys!(&:to_sym)
-        handle_continuation
+        apply_options(options)
       end
 
-      # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
-      def extract
-        raise NotImplementedError
-      end
+      # Hook called before #extract. Useful for gathering data, initailizing proxies, etc
+      def prepare; end
 
       # An optional method to calculate how many records there are to extract. Used primarily for
       # building the progress bar
       def results_count; end
 
+      # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
+      def extract
+        raise NotImplementedError
+      end
+
       private
 
-      def handle_continuation
-        return unless @options[:continuation]
+      # TODO: reimplemenet this
+      # def handle_continuation
+      #   return unless @config.continuation
 
-        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation].highest_timestamp
-        @options[:load_after_id] = @options[:continuation].last_id if @options[:continuation].last_id
-      end
+      #   @config.since = @config.continuation.highest_timestamp if @config.continuation.highest_timestamp
+      #   @config.load_after_id = @config.continuation.last_id if @config.continuation.last_id
+      # end
     end
   end
 end
 
+require_relative 'helpers/filesystem_reader'
 require_relative 'csv_extractor'
 require_relative 'file_extractor'
+require_relative 'json_extractor'
 require_relative 'stdin_extractor'

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -3,49 +3,34 @@ require 'pathname'
 module Chronicle
   module ETL
     class FileExtractor < Chronicle::ETL::Extractor
-      def extract
-        if file?
-          extract_file do |data, metadata|
-            yield(data, metadata)
-          end
-        elsif directory?
-          extract_from_directory do |data, metadata|
-            yield(data, metadata)
-          end
-        end
-      end
+      include Extractors::Helpers::FilesystemReader
 
-      def results_count
-        if file?
-          return 1
-        else
-          search_pattern = File.join(@options[:filename], '**/*.eml')
-          Dir.glob(search_pattern).count
-        end
+      register_connector do |r|
+        r.description = 'file or directory of files'
       end
 
-      private
+      # TODO: consolidate this with @config.filename
+      setting :dir_glob_pattern
 
-      def extract_from_directory
-        search_pattern = File.join(@options[:filename], '**/*.eml')
-        filenames = Dir.glob(search_pattern)
+      def extract
         filenames.each do |filename|
-          file = File.open(filename)
-          yield(file.read, {filename: file})
+          yield Chronicle::ETL::Extraction.new(data: filename)
         end
       end
 
-      def extract_file
-        file = File.open(@options[:filename])
-        yield(file.read, {filename: @options[:filename]})
+      def results_count
+        filenames.count
       end
 
-      def directory?
-        Pathname.new(@options[:filename]).directory?
-      end
+      private
 
-      def file?
-        Pathname.new(@options[:filename]).file?
+      def filenames
+        @filenames ||= filenames_in_directory(
+          path: @config.filename,
+          dir_glob_pattern: @config.dir_glob_pattern,
+          load_since: @config.since,
+          load_until: @config.until
+        )
       end
     end
   end

data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb

@@ -0,0 +1,104 @@
+require 'pathname'
+
+module Chronicle
+  module ETL
+    module Extractors
+      module Helpers
+        module FilesystemReader
+
+          def filenames_in_directory(...)
+            filenames = gather_files(...)
+            if block_given?
+              filenames.each do |filename|
+                yield filename
+              end
+            else
+              filenames
+            end
+          end
+
+          def read_from_filesystem(filename:, yield_each_line: true, dir_glob_pattern: '**/*')
+            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
+              if yield_each_line
+                file.each_line do |line|
+                  yield line
+                end
+              else
+                yield file.read
+              end
+            end
+          end
+
+          def open_from_filesystem(filename:, dir_glob_pattern: '**/*')
+            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
+              yield file
+            end
+          end
+
+          def results_count
+            raise NotImplementedError
+            # if file?
+            #   return 1
+            # else
+            #   search_pattern = File.join(@options[:filename], '**/*')
+            #   Dir.glob(search_pattern).count
+            # end
+          end
+
+          private
+
+          def gather_files(path:, dir_glob_pattern: '**/*', load_since: nil, load_until: nil, smaller_than: nil, larger_than: nil, sort: :mtime)
+            search_pattern = File.join(path, '**', dir_glob_pattern)
+            files = Dir.glob(search_pattern)
+
+            files = files.keep_if {|f| (File.mtime(f) > load_since)} if load_since
+            files = files.keep_if {|f| (File.mtime(f) < load_until)} if load_until
+
+            # pass in file sizes in bytes
+            files = files.keep_if {|f| (File.size(f) < smaller_than)} if smaller_than
+            files = files.keep_if {|f| (File.size(f) > larger_than)} if larger_than
+
+            # TODO: incorporate sort argument
+            files.sort_by{ |f| File.mtime(f) }
+          end
+
+          def select_files_in_directory(path:, dir_glob_pattern: '**/*')
+            raise IOError.new("#{path} is not a directory.") unless directory?(path)
+
+            search_pattern = File.join(path, dir_glob_pattern)
+            Dir.glob(search_pattern).each do |filename|
+              yield(filename)
+            end
+          end
+
+          def open_files(filename:, dir_glob_pattern:)
+            if stdin?(filename)
+              yield $stdin
+            elsif directory?(filename)
+              search_pattern = File.join(filename, dir_glob_pattern)
+              filenames = Dir.glob(search_pattern)
+              filenames.each do |filename|
+                file = File.open(filename)
+                yield(file)
+              end
+            elsif file?(filename)
+              yield File.open(filename)
+            end
+          end
+
+          def stdin?(filename)
+            filename == $stdin
+          end
+
+          def directory?(filename)
+            Pathname.new(filename).directory?
+          end
+
+          def file?(filename)
+            Pathname.new(filename).file?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/json_extractor.rb

@@ -0,0 +1,37 @@
+module Chronicle
+  module ETL
+    class JsonExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::FilesystemReader
+
+      register_connector do |r|
+        r.description = 'input as JSON'
+      end
+
+      setting :filename, default: $stdin
+      setting :jsonl, default: true
+
+      def extract
+        load_input do |input|
+          parsed_data = parse_data(input)
+          yield Chronicle::ETL::Extraction.new(data: parsed_data) if parsed_data
+        end
+      end
+
+      def results_count
+      end
+
+      private
+
+      def parse_data data
+        JSON.parse(data)
+      rescue JSON::ParserError => e
+      end
+
+      def load_input
+        read_from_filesystem(filename: @options[:filename]) do |data|
+          yield data
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/stdin_extractor.rb

@@ -1,9 +1,14 @@
 module Chronicle
   module ETL
     class StdinExtractor < Chronicle::ETL::Extractor
+      register_connector do |r|
+        r.description = 'stdin'
+      end
+
       def extract
         $stdin.read.each_line do |line|
-          yield line
+          data = { line: line.strip }
+          yield Chronicle::ETL::Extraction.new(data: data)
         end
       end
     end

data/lib/chronicle/etl/job.rb

@@ -1,6 +1,11 @@
+require 'forwardable'
 module Chronicle
   module ETL
     class Job
+      extend Forwardable
+
+      def_delegators :@job_definition, :dry_run?
+
       attr_accessor :name,
                     :extractor_klass,
                     :extractor_options,
@@ -12,32 +17,30 @@ module Chronicle
       # TODO: build a proper id system
       alias id name
 
-      def initialize(definition)
-        definition = definition.definition # FIXME
-        @name = definition[:name]
-        @extractor_klass = load_klass(:extractor, definition[:extractor][:name])
-        @extractor_options = definition[:extractor][:options] || {}
-
-        @transformer_klass = load_klass(:transformer, definition[:transformer][:name])
-        @transformer_options = definition[:transformer][:options] || {}
-
-        @loader_klass = load_klass(:loader, definition[:loader][:name])
-        @loader_options = definition[:loader][:options] || {}
+      def initialize(job_definition)
+        @job_definition = job_definition
+        @name = @job_definition.definition[:name]
+        @extractor_options = @job_definition.extractor_options
+        @transformer_options = @job_definition.transformer_options
+        @loader_options = @job_definition.loader_options
 
-        set_continuation if load_continuation?
+        set_continuation if use_continuation?
         yield self if block_given?
       end
 
       def instantiate_extractor
-        instantiate_klass(:extractor)
+        @extractor_klass = @job_definition.extractor_klass
+        @extractor_klass.new(@extractor_options)
       end
 
-      def instantiate_transformer(data)
-        instantiate_klass(:transformer, data)
+      def instantiate_transformer(extraction)
+        @transformer_klass = @job_definition.transformer_klass
+        @transformer_klass.new(extraction, @transformer_options)
       end
 
       def instantiate_loader
-        instantiate_klass(:loader)
+        @loader_klass = @job_definition.loader_klass
+        @loader_klass.new(@loader_options)
       end
 
       def save_log?
@@ -45,26 +48,24 @@ module Chronicle
         return !id.nil?
       end
 
-      private
-
-      def instantiate_klass(phase, *args)
-        options = self.send("#{phase.to_s}_options")
-        args = args.unshift(options)
-        klass = self.send("#{phase.to_s}_klass")
-        klass.new(*args)
+      def to_s
+        output = "Job"
+        output += " '#{name}'".bold if name
+        output += "\n"
+        output += "  → Extracting from #{@job_definition.extractor_klass.description}\n"
+        output += "  → Transforming #{@job_definition.transformer_klass.description}\n"
+        output += "  → Loading to #{@job_definition.loader_klass.description}\n"
       end
 
-      def load_klass phase, identifier
-        Chronicle::ETL::Catalog.phase_and_identifier_to_klass(phase, identifier)
-      end
+      private
 
       def set_continuation
-        continuation = Chronicle::ETL::JobLogger.load_latest(@job_id)
+        continuation = Chronicle::ETL::JobLogger.load_latest(@id)
         @extractor_options[:continuation] = continuation
       end
 
-      def load_continuation?
-        save_log?
+      def use_continuation?
+        @job_definition.incremental?
       end
     end
   end