chronicle-etl 0.3.0 → 0.4.1

data/lib/chronicle/etl/cli/main.rb

@@ -1,17 +1,10 @@
-require 'thor'
-require 'chronicle/etl'
 require 'colorize'
 
-require 'chronicle/etl/cli/subcommand_base'
-require 'chronicle/etl/cli/connectors'
-require 'chronicle/etl/cli/jobs'
-
 module Chronicle
   module ETL
     module CLI
       # Main entrypoint for CLI app
-      class Main < Thor
-        class_option "verbose", type: :boolean, default: false
+      class Main < ::Thor
         default_task "jobs"
 
         desc 'connectors:COMMAND', 'Connectors available for ETL jobs', hide: true
@@ -22,15 +15,6 @@ module Chronicle
 
         # Entrypoint for the CLI
         def self.start(given_args = ARGV, config = {})
-          if given_args[0] == "--version"
-            puts "#{Chronicle::ETL::VERSION}"
-            exit
-          end
-
-          if given_args.none?
-            abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
-          end
-
           # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class
           if given_args.any? && given_args[0].include?(':')
             commands = given_args.shift.split(':')
@@ -40,10 +24,20 @@ module Chronicle
           super(given_args, config)
         end
 
+        def self.exit_on_failure?
+          true
+        end
+
+        desc "version", "Show version"
+        map %w(--version -v) => :version
+        def version
+          shell.say "chronicle-etl #{Chronicle::ETL::VERSION}"
+        end
+
         # Displays help options for chronicle-etl
         def help(meth = nil, subcommand = false)
           if meth && !respond_to?(meth)
-            klass, task = Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
+            klass, task = ::Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
             klass.start(['-h', task].compact, shell: shell)
           else
             shell.say "ABOUT".bold
@@ -64,7 +58,7 @@ module Chronicle
 
             list = []
 
-            Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
+            ::Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
               list += thor_class.printable_tasks(false)
             end
             list.sort! { |a, b| a[0] <=> b[0] }

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -2,11 +2,11 @@ module Chronicle
   module ETL
     module CLI
       # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
-      class SubcommandBase < Thor
+      class SubcommandBase < ::Thor
         # Print usage instructions for a subcommand
         def self.help(shell, subcommand = false)
           list = printable_commands(true, subcommand)
-          Thor::Util.thor_classes_in(self).each do |klass|
+          ::Thor::Util.thor_classes_in(self).each do |klass|
             list += klass.printable_commands(false)
           end
           list.sort! { |a, b| a[0] <=> b[0] }

data/lib/chronicle/etl/cli.rb

@@ -0,0 +1,7 @@
+require 'thor'
+require 'chronicle/etl'
+
+require 'chronicle/etl/cli/subcommand_base'
+require 'chronicle/etl/cli/connectors'
+require 'chronicle/etl/cli/jobs'
+require 'chronicle/etl/cli/main'

data/lib/chronicle/etl/configurable.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "ostruct"
+
+module Chronicle
+  module ETL
+    # A mixin that gives a class
+    # a {Chronicle::ETL::Configurable::ClassMethods#setting} macro to define
+    # settings and their properties (require, type, etc)
+    #
+    # @example Basic usage
+    #   class Test < Chronicle::ETL::Extractor
+    #     include Chronicle::ETL::Configurable
+    #     setting :when, type: :date, required: true
+    #   end
+    #
+    #   t = Test.new(when: '2022-02-24')
+    #   t.config.when
+    module Configurable
+      # An individual setting for this Configurable
+      Setting = Struct.new(:default, :required, :type)
+      private_constant :Setting
+
+      # Collection of user-supplied options for this Configurable
+      class Config < OpenStruct
+        # Config values that aren't nil, as a hash
+        def compacted_h
+          to_h.compact
+        end
+      end
+
+      # @private
+      def self.included(klass)
+        klass.extend(ClassMethods)
+        klass.include(InstanceMethods)
+        klass.prepend(Initializer)
+      end
+
+      # Initializer method for classes that have Configurable mixed in
+      module Initializer
+        # Make sure this class has a default @config ready to use
+        def initialize(*args)
+          @config = initialize_default_config
+          super
+        end
+      end
+
+      # Instance methods for classes that have Configurable mixed in
+      module InstanceMethods
+        attr_reader :config
+
+        # Take given options and apply them to this class's settings
+        # and make them available in @config and validates that they
+        # conform to setting rules
+        def apply_options(options)
+          options.transform_keys!(&:to_sym)
+
+          options.each do |name, value|
+            setting = self.class.all_settings[name]
+            raise(Chronicle::ETL::ConfigurationError, "Unrecognized setting: #{name}") unless setting
+
+            @config[name] = coerced_value(setting, value)
+          end
+          validate_config
+          options
+        end
+
+        # Name of all settings available to this class
+        def self.settings
+          self.class.all_settings.keys
+        end
+
+        private
+
+        def initialize_default_config
+          self.class.config_with_defaults
+        end
+
+        def validate_config
+          missing = (self.class.all_required_settings.keys - @config.compacted_h.keys)
+          raise Chronicle::ETL::ConfigurationError, "Missing options: #{missing}" if missing.count.positive?
+        end
+
+        def coerced_value(setting, value)
+          setting.type ? __send__("coerce_#{setting.type}", value) : value
+        end
+
+        def coerce_string(value)
+          value.to_s
+        end
+
+        def coerce_boolean(value)
+          if value.is_a?(String)
+            value.downcase == "true"
+          else
+            value
+          end
+        end
+
+        def coerce_time(value)
+          # TODO: handle durations like '3h'
+          if value.is_a?(String)
+            Time.parse(value)
+          else
+            value
+          end
+        end
+      end
+
+      # Class methods for classes that have Configurable mixed in
+      module ClassMethods
+        # Macro for creating a setting on a class {::Chronicle::ETL::Configurable}
+        #
+        # @param [String] name Name of the setting
+        # @param [Boolean] required whether setting is required
+        # @param [Object] default Default value
+        # @param [Symbol] type Type
+        #
+        # @example Basic usage
+        #   setting :when, type: :date, required: true
+        #
+        # @see ::Chronicle::ETL::Configurable
+        def setting(name, default: nil, required: false, type: nil)
+          s = Setting.new(default, required, type)
+          settings[name] = s
+        end
+
+        # Collect all settings defined on this class and its ancestors (that
+        # have Configurable mixin included)
+        def all_settings
+          if superclass.include?(Chronicle::ETL::Configurable)
+            superclass.all_settings.merge(settings)
+          else
+            settings
+          end
+        end
+
+        # Filters settings to those that are required.
+        def all_required_settings
+          all_settings.select { |_name, setting| setting.required } || {}
+        end
+
+        def settings
+          @settings ||= {}
+        end
+
+        def setting_exists?(name)
+          all_settings.keys.include? name
+        end
+
+        def config_with_defaults
+          s = all_settings.transform_values(&:default)
+          Config.new(s)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/exceptions.rb

@@ -1,6 +1,8 @@
 module Chronicle
   module ETL
-    class Error < StandardError; end;
+    class Error < StandardError; end
+
+    class ConfigurationError < Error; end
 
     class RunnerTypeError < Error; end
 
@@ -16,6 +18,10 @@ module Chronicle
     class ProviderNotAvailableError < ConnectorNotAvailableError; end
     class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
 
+    class ExtractionError < Error; end
+
+    class SerializationError < Error; end
+
     class TransformationError < Error
       attr_reader :transformation
 

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -2,46 +2,47 @@ require 'csv'
 
 module Chronicle
   module ETL
-    class CsvExtractor < Chronicle::ETL::Extractor
-      include Extractors::Helpers::FilesystemReader
+    class CSVExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::InputReader
 
       register_connector do |r|
-        r.description = 'input as CSV'
+        r.description = 'CSV'
       end
 
-      DEFAULT_OPTIONS = {
-        headers: true,
-        filename: $stdin
-      }.freeze
+      setting :headers, default: true
 
-      def initialize(options = {})
-        super(DEFAULT_OPTIONS.merge(options))
+      def prepare
+        @csvs = prepare_sources
       end
 
       def extract
-        csv = initialize_csv
-        csv.each do |row|
-          yield Chronicle::ETL::Extraction.new(data: row.to_h)
+        @csvs.each do |csv|
+          csv.read.each do |row|
+            yield Chronicle::ETL::Extraction.new(data: row.to_h)
+          end
         end
       end
 
       def results_count
-        CSV.read(@options[:filename], headers: @options[:headers]).count unless stdin?(@options[:filename])
+        @csvs.reduce(0) do |total_rows, csv|
+          row_count = csv.readlines.size
+          csv.rewind
+          total_rows + row_count
+        end
       end
 
       private
 
-      def initialize_csv
-        headers = @options[:headers].is_a?(String) ? @options[:headers].split(',') : @options[:headers]
-
-        csv_options = {
-          headers: headers,
-          converters: :all
-        }
-
-        open_from_filesystem(filename: @options[:filename]) do |file|
-          return CSV.new(file, **csv_options)
+      def prepare_sources
+        @csvs = []
+        read_input do |csv_data|
+          csv_options = {
+            headers: @config.headers.is_a?(String) ? @config.headers.split(',') : @config.headers,
+            converters: :all
+          }
+          @csvs << CSV.new(csv_data, **csv_options)
         end
+        @csvs
       end
     end
   end

data/lib/chronicle/etl/extractors/extractor.rb

@@ -5,44 +5,48 @@ module Chronicle
     # Abstract class representing an Extractor for an ETL job
     class Extractor
       extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
+
+      setting :since, type: :time
+      setting :until, type: :time
+      setting :limit
+      setting :load_after_id
+      setting :input
 
       # Construct a new instance of this extractor. Options are passed in from a Runner
-      # == Paramters:
+      # == Parameters:
       # options::
       #   Options for configuring this Extractor
       def initialize(options = {})
-        @options = options.transform_keys!(&:to_sym)
-        sanitize_options
-        handle_continuation
+        apply_options(options)
       end
 
-      # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
-      def extract
-        raise NotImplementedError
-      end
+      # Hook called before #extract. Useful for gathering data, initailizing proxies, etc
+      def prepare; end
 
       # An optional method to calculate how many records there are to extract. Used primarily for
       # building the progress bar
       def results_count; end
 
-      private
-
-      def sanitize_options
-        @options[:load_since] = Time.parse(@options[:load_since]) if @options[:load_since] && @options[:load_since].is_a?(String)
-        @options[:load_until] = Time.parse(@options[:load_until]) if @options[:load_until] && @options[:load_until].is_a?(String)
+      # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
+      def extract
+        raise NotImplementedError
       end
 
-      def handle_continuation
-        return unless @options[:continuation]
+      private
 
-        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation].highest_timestamp
-        @options[:load_after_id] = @options[:continuation].last_id if @options[:continuation].last_id
-      end
+      # TODO: reimplemenet this
+      # def handle_continuation
+      #   return unless @config.continuation
+
+      #   @config.since = @config.continuation.highest_timestamp if @config.continuation.highest_timestamp
+      #   @config.load_after_id = @config.continuation.last_id if @config.continuation.last_id
+      # end
     end
   end
 end
 
-require_relative 'helpers/filesystem_reader'
+require_relative 'helpers/input_reader'
 require_relative 'csv_extractor'
 require_relative 'file_extractor'
 require_relative 'json_extractor'

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -2,32 +2,55 @@ require 'pathname'
 
 module Chronicle
   module ETL
+    # Return filenames that match a pattern in a directory
     class FileExtractor < Chronicle::ETL::Extractor
-      include Extractors::Helpers::FilesystemReader
 
       register_connector do |r|
         r.description = 'file or directory of files'
       end
 
+      setting :input, default: ['.']
+      setting :dir_glob_pattern, default: "**/*"
+      setting :larger_than
+      setting :smaller_than
+
+      def prepare
+        @pathnames = gather_files
+      end
+
       def extract
-        filenames.each do |filename|
-          yield Chronicle::ETL::Extraction.new(data: filename)
+        @pathnames.each do |pathname|
+          yield Chronicle::ETL::Extraction.new(data: pathname.to_path)
         end
       end
 
       def results_count
-        filenames.count
+        @pathnames.count
       end
 
       private
 
-      def filenames
-        @filenames ||= filenames_in_directory(
-          path: @options[:filename],
-          dir_glob_pattern: @options[:dir_glob_pattern],
-          load_since: @options[:load_since],
-          load_until: @options[:load_until]
-        )
+      def gather_files
+        roots = [@config.input].flatten.map { |filename| Pathname.new(filename) }
+        raise(ExtractionError, "Input must exist") unless roots.all?(&:exist?)
+
+        directories, files = roots.partition(&:directory?)
+
+        directories.each do |directory|
+          files += Dir.glob(File.join(directory, @config.dir_glob_pattern)).map { |filename| Pathname.new(filename) }
+        end
+
+        files = files.uniq
+
+        files = files.keep_if { |f| (f.mtime > @config.since) } if @config.since
+        files = files.keep_if { |f| (f.mtime < @config.until) } if @config.until
+
+        # pass in file sizes in bytes
+        files = files.keep_if { |f| (f.size < @config.smaller_than) } if @config.smaller_than
+        files = files.keep_if { |f| (f.size > @config.larger_than) } if @config.larger_than
+
+        # # TODO: incorporate sort argument
+        files.sort_by(&:mtime)
       end
     end
   end

data/lib/chronicle/etl/extractors/helpers/input_reader.rb

@@ -0,0 +1,76 @@
+require 'pathname'
+
+module Chronicle
+  module ETL
+    module Extractors
+      module Helpers
+        module InputReader
+          # Return an array of input filenames; converts a single string
+          # to an array if necessary
+          def filenames
+            [@config.input].flatten.map
+          end
+
+          # Filenames as an array of pathnames
+          def pathnames
+            filenames.map { |filename| Pathname.new(filename) }
+          end
+
+          # Whether we're reading from files
+          def read_from_files?
+            filenames.any?
+          end
+
+          # Whether we're reading input from stdin
+          def read_from_stdin?
+            !read_from_files? && $stdin.stat.pipe?
+          end
+
+          # Read input sources and yield each content
+          def read_input
+            if read_from_files?
+              pathnames.each do |pathname|
+                File.open(pathname) do |file|
+                  yield file.read, pathname.to_path
+                end
+              end
+            elsif read_from_stdin?
+              yield $stdin.read, $stdin
+            else
+              raise ExtractionError, "No input files or stdin provided"
+            end
+          end
+
+          # Read input sources line by line
+          def read_input_as_lines(&block)
+            if read_from_files?
+              lines_from_files(&block)
+            elsif read_from_stdin?
+              lines_from_stdin(&block)
+            else
+              raise ExtractionError, "No input files or stdin provided"
+            end
+          end
+
+          private
+
+          def lines_from_files(&block)
+            pathnames.each do |pathname|
+              File.open(pathname) do |file|
+                lines_from_io(file, &block)
+              end
+            end
+          end
+
+          def lines_from_stdin(&block)
+            lines_from_io($stdin, &block)
+          end
+
+          def lines_from_io(io, &block)
+            io.each_line(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/json_extractor.rb

@@ -1,43 +1,44 @@
 module Chronicle
   module ETL
-    class JsonExtractor < Chronicle::ETL::Extractor
-      include Extractors::Helpers::FilesystemReader
+    class JSONExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::InputReader
 
       register_connector do |r|
-        r.description = 'input as JSON'
+        r.description = 'JSON'
       end
 
-      DEFAULT_OPTIONS = {
-        filename: $stdin,
+      setting :jsonl, default: true, type: :boolean
 
-        # We're expecting line-separated json objects
-        jsonl: true
-      }.freeze
-
-      def initialize(options = {})
-        super(DEFAULT_OPTIONS.merge(options))
+      def prepare
+        @jsons = []
+        load_input do |input|
+          @jsons << parse_data(input)
+        end
       end
 
       def extract
-        load_input do |input|
-          parsed_data = parse_data(input)
-          yield Chronicle::ETL::Extraction.new(data: parsed_data) if parsed_data
+        @jsons.each do |json|
+          yield Chronicle::ETL::Extraction.new(data: json)
         end
       end
 
       def results_count
+        @jsons.count
       end
 
       private
 
       def parse_data data
         JSON.parse(data)
-      rescue JSON::ParserError => e
+      rescue JSON::ParserError
+        raise Chronicle::ETL::ExtractionError, "Could not parse JSON"
       end
 
-      def load_input
-        read_from_filesystem(filename: @options[:filename]) do |data|
-          yield data
+      def load_input(&block)
+        if @config.jsonl
+          read_input_as_lines(&block)
+        else
+          read_input(&block)
         end
       end
     end

data/lib/chronicle/etl/job.rb

@@ -35,7 +35,7 @@ module Chronicle
 
       def instantiate_transformer(extraction)
         @transformer_klass = @job_definition.transformer_klass
-        @transformer_klass.new(@transformer_options, extraction)
+        @transformer_klass.new(extraction, @transformer_options)
       end
 
       def instantiate_loader

data/lib/chronicle/etl/job_definition.rb

@@ -14,7 +14,7 @@ module Chronicle
           options: {}
         },
         loader: {
-          name: 'stdout',
+          name: 'table',
           options: {}
         }
       }.freeze

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -2,7 +2,7 @@ require 'csv'
 
 module Chronicle
   module ETL
-    class CsvLoader < Chronicle::ETL::Loader
+    class CSVLoader < Chronicle::ETL::Loader
       register_connector do |r|
         r.description = 'CSV'
       end