chronicle-etl 0.5.5 → 0.6.1

data/lib/chronicle/etl/cli/secrets.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "tty-prompt"
+require 'tty-prompt'
 
 module Chronicle
   module ETL
@@ -10,8 +10,8 @@ module Chronicle
         default_task 'list'
         namespace :secrets
 
-        desc "set NAMESPACE KEY [VALUE]", "Add a secret. VALUE can be set as argument or from stdin"
-        def set(namespace, key, value=nil)
+        desc 'set NAMESPACE KEY [VALUE]', 'Add a secret. VALUE can be set as argument or from stdin'
+        def set(namespace, key, value = nil)
           validate_namespace(namespace)
 
           if value
@@ -24,23 +24,23 @@ module Chronicle
           end
 
           Chronicle::ETL::Secrets.set(namespace, key, value.strip)
-          cli_exit(message: "Secret set")
+          cli_exit(message: 'Secret set')
         rescue TTY::Reader::InputInterrupt
           cli_fail(message: "\nSecret not set")
         end
 
-        desc "unset NAMESPACE KEY", "Remove a secret"
+        desc 'unset NAMESPACE KEY', 'Remove a secret'
         def unset(namespace, key)
           validate_namespace(namespace)
 
           Chronicle::ETL::Secrets.unset(namespace, key)
-          cli_exit(message: "Secret unset")
+          cli_exit(message: 'Secret unset')
         end
 
-        desc "list", "List available secrets"
-        def list(namespace=nil)
+        desc 'list', 'List available secrets'
+        def list(namespace = nil)
           all_secrets = Chronicle::ETL::Secrets.all(namespace)
-          cli_exit(message: "No secrets are stored") unless all_secrets.any?
+          cli_exit(message: 'No secrets are stored') unless all_secrets.any?
 
           rows = []
           all_secrets.each do |namespace, secrets|
@@ -51,9 +51,9 @@ module Chronicle
             end
           end
 
-          headers = ['namespace', 'key', 'value'].map { |h| h.upcase.bold }
+          headers = %w[namespace key value].map { |h| h.upcase.bold }
 
-          puts "Available secrets:"
+          puts 'Available secrets:'
           table = TTY::Table.new(headers, rows)
           puts table.render(indent: 0, padding: [0, 2])
         end
@@ -61,7 +61,9 @@ module Chronicle
         private
 
         def validate_namespace(namespace)
-          cli_fail(message: "'#{namespace}' is not a valid namespace") unless Chronicle::ETL::Secrets.valid_namespace_name?(namespace)
+          return if Chronicle::ETL::Secrets.valid_namespace_name?(namespace)
+
+          cli_fail(message: "'#{namespace}' is not a valid namespace")
         end
       end
     end

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -11,7 +11,7 @@ module Chronicle
           end
           list.sort! { |a, b| a[0] <=> b[0] }
 
-          shell.say "COMMANDS".bold
+          shell.say 'COMMANDS'.bold
           shell.print_table(list, indent: 2, truncate: true)
           shell.say
           class_options_help(shell)
@@ -19,7 +19,7 @@ module Chronicle
 
         # Show docs with command:subcommand pattern.
         # For `help` command, don't use colon
-        def self.banner(command, namespace = nil, subcommand = false)
+        def self.banner(command, _namespace = nil, _subcommand = false)
           if command.name == 'help'
             "#{subcommand_prefix} #{command.usage}"
           else
@@ -29,7 +29,9 @@ module Chronicle
 
         # Use subcommand classname to derive display name for subcommand
         def self.subcommand_prefix
-          self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+          name.gsub(/.*::/, '').gsub(/^[A-Z]/) do |match|
+            match[0].downcase
+          end.gsub(/[A-Z]/) { |match| "-#{match[0].downcase}" }
         end
       end
     end

data/lib/chronicle/etl/config.rb

@@ -1,4 +1,4 @@
-require "active_support/core_ext/hash/keys"
+require 'active_support/core_ext/hash/keys'
 require 'fileutils'
 require 'yaml'
 
@@ -15,7 +15,7 @@ module Chronicle
         path = base.join("#{identifier}.yml")
         return {} unless path.exist?
 
-        YAML.safe_load(File.read(path), symbolize_names: true, permitted_classes: [Symbol, Date, Time])
+        YAML.safe_load_file(path, symbolize_names: true, permitted_classes: [Symbol, Date, Time])
       end
 
       # Writes a hash as a yml config file
@@ -31,28 +31,38 @@ module Chronicle
         end
       end
 
+      # Returns path for a given config type and identifier
+      def path(type, identifier)
+        base = config_pathname_for_type(type)
+        base.join("#{identifier}.yml")
+      end
+
+      # Whether a config exists for a given type and identifier
       def exists?(type, identifier)
         base = config_pathname_for_type(type)
         path = base.join("#{identifier}.yml")
-        return path.exist?
+        path.exist?
       end
 
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
       def available_jobs
-        Dir.glob(File.join(config_pathname_for_type("jobs"), "*.yml")).map do |filename|
-          File.basename(filename, ".*")
+        Dir.glob(File.join(config_pathname_for_type('jobs'), '*.yml')).map do |filename|
+          File.basename(filename, '.*')
         end
       end
 
+      # Returns all configs available for a given type
       def available_configs(type)
-        Dir.glob(File.join(config_pathname_for_type(type), "*.yml")).map do |filename|
-          File.basename(filename, ".*")
+        Dir.glob(File.join(config_pathname_for_type(type), '*.yml')).map do |filename|
+          File.basename(filename, '.*')
         end
       end
 
       # Load a job definition from job config directory
       def read_job(job_name)
-        load('jobs', job_name)
+        definition = load('jobs', job_name)
+        definition[:name] ||= job_name
+        definition
       end
 
       def config_pathname

data/lib/chronicle/etl/configurable.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require "ostruct"
-require "chronic_duration"
+require 'ostruct'
+require 'chronic_duration'
 
 module Chronicle
   module ETL
@@ -19,7 +19,7 @@ module Chronicle
     #   t.config.when
     module Configurable
       # An individual setting for this Configurable
-      Setting = Struct.new(:default, :required, :type)
+      Setting = Struct.new(:default, :required, :type, :description)
       private_constant :Setting
 
       # Collection of user-supplied options for this Configurable
@@ -62,7 +62,7 @@ module Chronicle
             # Do nothing with a given option if it's not a connector setting
             next unless setting
 
-            @config[name] = coerced_value(setting, value)
+            @config[name] = coerced_value(setting, name, value)
           end
           validate_config
           options
@@ -84,10 +84,17 @@ module Chronicle
           raise Chronicle::ETL::ConnectorConfigurationError, "Missing options: #{missing}" if missing.count.positive?
         end
 
-        def coerced_value(setting, value)
+        def coerced_value(setting, name, value)
           setting.type ? __send__("coerce_#{setting.type}", value) : value
         rescue StandardError
-          raise(Chronicle::ETL::ConnectorConfigurationError, "Could not coerce #{value} into a #{setting.type}")
+          raise(
+            Chronicle::ETL::ConnectorConfigurationError,
+            "Could not convert value '#{value}' into a #{setting.type} for setting '#{name}'"
+          )
+        end
+
+        def coerce_hash(value)
+          value.is_a?(Hash) ? value : {}
         end
 
         def coerce_string(value)
@@ -101,12 +108,16 @@ module Chronicle
 
         def coerce_boolean(value)
           if value.is_a?(String)
-            value.downcase == "true"
+            value.downcase == 'true'
           else
             value
           end
         end
 
+        def coerce_array(value)
+          value.is_a?(Array) ? value : [value]
+        end
+
         def coerce_time(value)
           # parsing yml files might result in us getting Date objects
           # we convert to DateTime first to to ensure UTC
@@ -138,8 +149,8 @@ module Chronicle
         #   setting :when, type: :date, required: true
         #
         # @see ::Chronicle::ETL::Configurable
-        def setting(name, default: nil, required: false, type: nil)
-          s = Setting.new(default, required, type)
+        def setting(name, default: nil, required: false, type: nil, description: nil)
+          s = Setting.new(default, required, type, description)
           settings[name] = s
         end
 

data/lib/chronicle/etl/exceptions.rb

@@ -26,6 +26,7 @@ module Chronicle
       attr_reader :name
 
       def initialize(name)
+        super
         @name = name
       end
     end
@@ -51,10 +52,9 @@ module Chronicle
 
     class ExtractionError < Error; end
 
-    class SerializationError < Error; end
-
     class TransformationError < Error; end
-
     class UntransformableRecordError < TransformationError; end
+
+    class LoaderError < Error; end
   end
 end

data/lib/chronicle/etl/extraction.rb

@@ -1,11 +1,21 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     class Extraction
-      attr_accessor :data, :meta
+      attr_accessor :data, :meta, :source, :type, :strategy, :extractor
 
-      def initialize(data: {}, meta: {})
+      def initialize(data: {}, meta: {}, source: nil, type: nil, strategy: nil, extractor: nil)
         @data = data
         @meta = meta
+        @source = source
+        @type = type
+        @strategy = strategy
+        @extractor = extractor
+      end
+
+      def to_h
+        { data: @data, meta: @meta, source: @source }
       end
     end
   end

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'csv'
 
 module Chronicle
@@ -6,6 +8,7 @@ module Chronicle
       include Extractors::Helpers::InputReader
 
       register_connector do |r|
+        r.identifier = :csv
         r.description = 'CSV'
       end
 
@@ -33,6 +36,12 @@ module Chronicle
 
       private
 
+      def all_rows
+        @csvs.reduce([]) do |all_rows, csv|
+          all_rows + csv.to_a.map(&:to_h)
+        end
+      end
+
       def prepare_sources
         @csvs = []
         read_input do |csv_data|

data/lib/chronicle/etl/extractors/extractor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'chronicle/etl'
 
 module Chronicle
@@ -21,7 +23,7 @@ module Chronicle
         apply_options(options)
       end
 
-      # Hook called before #extract. Useful for gathering data, initailizing proxies, etc
+      # Hook called before #extract. Useful for gathering data, initializing proxies, etc
       def prepare; end
 
       # An optional method to calculate how many records there are to extract. Used primarily for
@@ -33,7 +35,18 @@ module Chronicle
         raise NotImplementedError
       end
 
-      private
+      protected
+
+      def build_extraction(data:, meta: nil, source: nil, type: nil, strategy: nil)
+        Extraction.new(
+          extractor: self.class,
+          data: data,
+          meta: meta,
+          source: source || self.class.connector_registration.source,
+          type: type || self.class.connector_registration.type,
+          strategy: strategy || self.class.connector_registration.strategy
+        )
+      end
 
       # TODO: reimplemenet this
       # def handle_continuation

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -1,16 +1,18 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 module Chronicle
   module ETL
     # Return filenames that match a pattern in a directory
     class FileExtractor < Chronicle::ETL::Extractor
-
       register_connector do |r|
+        r.identifier = :file
         r.description = 'file or directory of files'
       end
 
       setting :input, default: ['.']
-      setting :dir_glob_pattern, default: "**/*"
+      setting :dir_glob_pattern, default: '**/*'
       setting :larger_than
       setting :smaller_than
 
@@ -32,7 +34,7 @@ module Chronicle
 
       def gather_files
         roots = [@config.input].flatten.map { |filename| Pathname.new(filename) }
-        raise(ExtractionError, "Input must exist") unless roots.all?(&:exist?)
+        raise(ExtractionError, 'Input must exist') unless roots.all?(&:exist?)
 
         directories, files = roots.partition(&:directory?)
 

data/lib/chronicle/etl/extractors/helpers/input_reader.rb

@@ -37,7 +37,7 @@ module Chronicle
             elsif read_from_stdin?
               yield $stdin.read, $stdin
             else
-              raise ExtractionError, "No input files or stdin provided"
+              raise ExtractionError, 'No input files or stdin provided'
             end
           end
 
@@ -48,7 +48,7 @@ module Chronicle
             elsif read_from_stdin?
               lines_from_stdin(&block)
             else
-              raise ExtractionError, "No input files or stdin provided"
+              raise ExtractionError, 'No input files or stdin provided'
             end
           end
 

data/lib/chronicle/etl/extractors/json_extractor.rb

@@ -1,18 +1,23 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     class JSONExtractor < Chronicle::ETL::Extractor
       include Extractors::Helpers::InputReader
 
       register_connector do |r|
+        r.identifier = :json
         r.description = 'JSON'
       end
 
       setting :jsonl, default: true, type: :boolean
+      setting :path, default: nil, type: :string
 
       def prepare
         @jsons = []
         load_input do |input|
-          @jsons << parse_data(input)
+          data = parse_data(input)
+          @jsons += [data].flatten
         end
       end
 
@@ -28,10 +33,15 @@ module Chronicle
 
       private
 
-      def parse_data data
-        JSON.parse(data)
+      def parse_data(data)
+        parsed_data = JSON.parse(data)
+        if @config.path
+          parsed_data.dig(*@config.path.split('.'))
+        else
+          parsed_data
+        end
       rescue JSON::ParserError
-        raise Chronicle::ETL::ExtractionError, "Could not parse JSON"
+        raise Chronicle::ETL::ExtractionError, 'Could not parse JSON'
       end
 
       def load_input(&block)

data/lib/chronicle/etl/extractors/stdin_extractor.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     class StdinExtractor < Chronicle::ETL::Extractor
       register_connector do |r|
+        r.identifier = :stdin
         r.description = 'stdin'
       end
 

data/lib/chronicle/etl/job.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'forwardable'
 
 module Chronicle
@@ -12,13 +14,13 @@ module Chronicle
       def_delegators :@job_definition, :dry_run?
 
       attr_accessor :name,
-                    :extractor_klass,
-                    :extractor_options,
-                    :transformer_klass,
-                    :transformer_options,
-                    :loader_klass,
-                    :loader_options,
-                    :job_definition
+        :extractor_klass,
+        :extractor_options,
+        :transformer_klasses,
+        :transformer_options,
+        :loader_klass,
+        :loader_options,
+        :job_definition
 
       # TODO: build a proper id system
       alias id name
@@ -39,9 +41,10 @@ module Chronicle
         @extractor_klass.new(@extractor_options)
       end
 
-      def instantiate_transformer(extraction)
-        @transformer_klass = @job_definition.transformer_klass
-        @transformer_klass.new(extraction, @transformer_options)
+      def instantiate_transformers
+        @job_definition.transformer_klasses.each_with_index.map do |klass, i|
+          klass.new(@transformer_options[i] || {})
+        end
       end
 
       def instantiate_loader
@@ -51,20 +54,35 @@ module Chronicle
 
       def save_log?
         # TODO: this needs more nuance
-        return !id.nil?
+        !id.nil?
       end
 
       def to_s
-        output = "Job"
-        output += " '#{name}'".bold if name
-        output += "\n"
-        output += "  → Extracting from #{@job_definition.extractor_klass.description}\n"
-        output += "  → Transforming #{@job_definition.transformer_klass.description}\n"
-        output += "  → Loading to #{@job_definition.loader_klass.description}\n"
+        output = "Job summary\n".upcase.bold
+        # output = ""
+        output += "#{name}:\n" if name
+        output += "→ Extracting from #{@job_definition.extractor_klass.description}\n"
+        output += options_to_s(@extractor_options)
+
+        @job_definition.transformer_klasses.each do |klass|
+          output += "→ Transforming #{klass.description}\n"
+        end
+        # TODO: transformer options
+        output += "→ Loading to #{@job_definition.loader_klass.description}\n"
+        output += options_to_s(@loader_options)
+        output
       end
 
       private
 
+      def options_to_s(options, indent: 4)
+        output = ''
+        options.each do |k, v|
+          output += "#{' ' * indent}#{k.to_s.light_blue}: #{v}\n"
+        end
+        output
+      end
+
       def set_continuation
         continuation = Chronicle::ETL::JobLogger.load_latest(@id)
         @extractor_options[:continuation] = continuation

data/lib/chronicle/etl/job_definition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/core_ext/hash/deep_merge'
 
 module Chronicle
@@ -9,12 +11,14 @@ module Chronicle
           name: 'stdin',
           options: {}
         },
-        transformer: {
-          name: 'null',
-          options: {}
-        },
+        transformers: [
+          {
+            name: 'null',
+            options: {}
+          }
+        ],
         loader: {
-          name: 'table',
+          name: 'json',
           options: {}
         }
       }.freeze
@@ -22,7 +26,7 @@ module Chronicle
       attr_reader :errors
       attr_accessor :definition
 
-      def initialize()
+      def initialize
         @definition = SKELETON_DEFINITION
       end
 
@@ -34,12 +38,12 @@ module Chronicle
       def validate
         @errors = {}
 
-        Chronicle::ETL::Registry::Connectors::PHASES.each do |phase|
-          __send__("#{phase}_klass".to_sym)
-        rescue Chronicle::ETL::PluginError => e
-          @errors[:plugins] ||= []
-          @errors[:plugins] << e
-        end
+        extractor_klass
+        transformer_klasses
+        loader_klass
+      rescue Chronicle::ETL::PluginError => e
+        @errors[:plugins] ||= []
+        @errors[:plugins] << e
       end
 
       def plugins_missing?
@@ -48,12 +52,11 @@ module Chronicle
         return false unless @errors[:plugins]&.any?
 
         @errors[:plugins]
-          .filter { |e| e.instance_of?(Chronicle::ETL::PluginNotInstalledError) }
-          .any?
+          .any? { |e| e.instance_of?(Chronicle::ETL::PluginNotInstalledError) }
       end
 
       def validate!
-        raise(Chronicle::ETL::JobDefinitionError.new(self), "Job definition is invalid") unless valid?
+        raise(Chronicle::ETL::JobDefinitionError.new(self), 'Job definition is invalid') unless valid?
 
         true
       end
@@ -66,19 +69,20 @@ module Chronicle
 
       # For each connector in this job, mix in secrets into the options
       def apply_default_secrets
-        Chronicle::ETL::Registry::Connectors::PHASES.each do |phase|
+        # FIXME: handle transformer secrets
+        %i[extractor loader].each do |phase|
           # If the option have a `secrets` key, we look up those secrets and
-          # mix them in. If not, use the connector's plugin name and look up 
+          # mix them in. If not, use the connector's plugin name and look up
           # secrets with the same namespace
           if @definition[phase][:options][:secrets]
             namespace = @definition[phase][:options][:secrets]
           else
             # We don't want to do this lookup for built-in connectors
-            next if __send__("#{phase}_klass".to_sym).connector_registration.built_in?
+            next if __send__(:"#{phase}_klass").connector_registration.built_in?
 
             # infer plugin name from connector name and use it for secrets
             # namesepace
-            namespace = @definition[phase][:name].split(":").first
+            namespace = @definition[phase][:name].split(':').first
           end
 
           # Reverse merge secrets into connector's options (we want to preserve
@@ -98,15 +102,17 @@ module Chronicle
       end
 
       def extractor_klass
-        load_klass(:extractor, @definition[:extractor][:name])
+        find_connector_klass(:extractor, @definition[:extractor][:name])
       end
 
-      def transformer_klass
-        load_klass(:transformer, @definition[:transformer][:name])
+      def transformer_klasses
+        @definition[:transformers].map do |transformer|
+          find_connector_klass(:transformer, transformer[:name])
+        end
       end
 
       def loader_klass
-        load_klass(:loader, @definition[:loader][:name])
+        find_connector_klass(:loader, @definition[:loader][:name])
       end
 
       def extractor_options
@@ -114,7 +120,9 @@ module Chronicle
       end
 
       def transformer_options
-        @definition[:transformer][:options]
+        @definition[:transformers].map do |transformer|
+          transformer[:options]
+        end
       end
 
       def loader_options
@@ -123,12 +131,16 @@ module Chronicle
 
       private
 
-      def load_klass(phase, identifier)
+      def find_schema_transformer_klass(source_klass, target)
+        Chronicle::ETL::Registry::Connectors.find_converter_for_source(source_klass, target).klass
+      end
+
+      def find_connector_klass(phase, identifier)
         Chronicle::ETL::Registry::Connectors.find_by_phase_and_identifier(phase, identifier).klass
       end
 
       def load_credentials
-        Chronicle::ETL::Registry::Connectors::PHASES.each do |phase|
+        %i[extractor loader].each do |phase|
           credentials_name = @definition[phase].dig(:options, :credentials)
           if credentials_name
             credentials = Chronicle::ETL::Config.load_credentials(credentials_name)