chronicle-etl 0.5.4 → 0.6.1

data/lib/chronicle/etl/job_definition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/core_ext/hash/deep_merge'
 
 module Chronicle
@@ -9,12 +11,14 @@ module Chronicle
           name: 'stdin',
           options: {}
         },
-        transformer: {
-          name: 'null',
-          options: {}
-        },
+        transformers: [
+          {
+            name: 'null',
+            options: {}
+          }
+        ],
         loader: {
-          name: 'table',
+          name: 'json',
           options: {}
         }
       }.freeze
@@ -22,7 +26,7 @@ module Chronicle
       attr_reader :errors
       attr_accessor :definition
 
-      def initialize()
+      def initialize
         @definition = SKELETON_DEFINITION
       end
 
@@ -34,12 +38,12 @@ module Chronicle
       def validate
         @errors = {}
 
-        Chronicle::ETL::Registry::PHASES.each do |phase|
-          __send__("#{phase}_klass".to_sym)
-        rescue Chronicle::ETL::PluginError => e
-          @errors[:plugins] ||= []
-          @errors[:plugins] << e
-        end
+        extractor_klass
+        transformer_klasses
+        loader_klass
+      rescue Chronicle::ETL::PluginError => e
+        @errors[:plugins] ||= []
+        @errors[:plugins] << e
       end
 
       def plugins_missing?
@@ -48,12 +52,11 @@ module Chronicle
         return false unless @errors[:plugins]&.any?
 
         @errors[:plugins]
-          .filter { |e| e.instance_of?(Chronicle::ETL::PluginNotInstalledError) }
-          .any?
+          .any? { |e| e.instance_of?(Chronicle::ETL::PluginNotInstalledError) }
       end
 
       def validate!
-        raise(Chronicle::ETL::JobDefinitionError.new(self), "Job definition is invalid") unless valid?
+        raise(Chronicle::ETL::JobDefinitionError.new(self), 'Job definition is invalid') unless valid?
 
         true
       end
@@ -66,19 +69,20 @@ module Chronicle
 
       # For each connector in this job, mix in secrets into the options
       def apply_default_secrets
-        Chronicle::ETL::Registry::PHASES.each do |phase|
+        # FIXME: handle transformer secrets
+        %i[extractor loader].each do |phase|
           # If the option have a `secrets` key, we look up those secrets and
-          # mix them in. If not, use the connector's plugin name and look up 
+          # mix them in. If not, use the connector's plugin name and look up
           # secrets with the same namespace
           if @definition[phase][:options][:secrets]
             namespace = @definition[phase][:options][:secrets]
           else
             # We don't want to do this lookup for built-in connectors
-            next if __send__("#{phase}_klass".to_sym).connector_registration.built_in?
+            next if __send__(:"#{phase}_klass").connector_registration.built_in?
 
             # infer plugin name from connector name and use it for secrets
             # namesepace
-            namespace = @definition[phase][:name].split(":").first
+            namespace = @definition[phase][:name].split(':').first
           end
 
           # Reverse merge secrets into connector's options (we want to preserve
@@ -98,15 +102,17 @@ module Chronicle
       end
 
       def extractor_klass
-        load_klass(:extractor, @definition[:extractor][:name])
+        find_connector_klass(:extractor, @definition[:extractor][:name])
       end
 
-      def transformer_klass
-        load_klass(:transformer, @definition[:transformer][:name])
+      def transformer_klasses
+        @definition[:transformers].map do |transformer|
+          find_connector_klass(:transformer, transformer[:name])
+        end
       end
 
       def loader_klass
-        load_klass(:loader, @definition[:loader][:name])
+        find_connector_klass(:loader, @definition[:loader][:name])
       end
 
       def extractor_options
@@ -114,7 +120,9 @@ module Chronicle
       end
 
       def transformer_options
-        @definition[:transformer][:options]
+        @definition[:transformers].map do |transformer|
+          transformer[:options]
+        end
       end
 
       def loader_options
@@ -123,12 +131,16 @@ module Chronicle
 
       private
 
-      def load_klass(phase, identifier)
-        Chronicle::ETL::Registry.find_by_phase_and_identifier(phase, identifier).klass
+      def find_schema_transformer_klass(source_klass, target)
+        Chronicle::ETL::Registry::Connectors.find_converter_for_source(source_klass, target).klass
+      end
+
+      def find_connector_klass(phase, identifier)
+        Chronicle::ETL::Registry::Connectors.find_by_phase_and_identifier(phase, identifier).klass
       end
 
       def load_credentials
-        Chronicle::ETL::Registry::PHASES.each do |phase|
+        %i[extractor loader].each do |phase|
           credentials_name = @definition[phase].dig(:options, :credentials)
           if credentials_name
             credentials = Chronicle::ETL::Config.load_credentials(credentials_name)

data/lib/chronicle/etl/job_log.rb

@@ -9,13 +9,13 @@ module Chronicle
       extend Forwardable
 
       attr_accessor :job,
-                    :job_id,
-                    :last_id,
-                    :highest_timestamp,
-                    :num_records_processed,
-                    :started_at,
-                    :finished_at,
-                    :success
+        :job_id,
+        :last_id,
+        :highest_timestamp,
+        :num_records_processed,
+        :started_at,
+        :finished_at,
+        :success
 
       def_delegators :@job, :save_log?
 
@@ -28,11 +28,11 @@ module Chronicle
 
       # Log the result of a single transformation in a job
       # @param transformer [Chronicle::ETL::Tranformer] The transformer that ran
-      def log_transformation(transformer)
-        @last_id = transformer.id if transformer.id
+      def log_transformation(_transformer)
+        # @last_id = transformer.id if transformer.id
 
         # Save the highest timestamp that we've encountered so far
-        @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
+        # @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
 
         # TODO: a transformer might yield nil. We might also want certain transformers to explode
         # records into multiple new ones. Therefore, this this variable will need more subtle behaviour
@@ -54,13 +54,13 @@ module Chronicle
         @finished_at = Time.now
       end
 
-      def job= job
+      def job=(job)
         @job = job
         @job_id = job.id
       end
 
       def duration
-        return unless @finished_at
+        return unless @finished_at && @started_at
 
         @finished_at - @started_at
       end
@@ -78,14 +78,12 @@ module Chronicle
         }
       end
 
-      private
-
       # Create a new JobLog and set its instance variables from a serialized hash
-      def self.build_from_serialized attrs
+      def self.build_from_serialized(attrs)
         attrs.delete(:id)
         new do |job_log|
           attrs.each do |key, value|
-            setter = "#{key.to_s}=".to_sym
+            setter = :"#{key}="
             job_log.send(setter, value)
           end
         end

data/lib/chronicle/etl/job_logger.rb

@@ -12,7 +12,7 @@ module Chronicle
       attr_accessor :job_log
 
       # For a given `job_id`, return the last successful log
-      def self.load_latest(job_id)
+      def self.load_latest(_job_id)
         with_db_connection do |db|
           attrs = db[:job_logs].reverse_order(:finished_at).where(success: true).first
           JobLog.build_from_serialized(attrs) if attrs
@@ -28,11 +28,11 @@ module Chronicle
       end
 
       def self.db_exists?
-        File.exists?(db_filename)
+        File.exist?(db_filename)
       end
 
       def self.schema_exists?(db)
-        return db.tables.include? :job_logs
+        db.tables.include? :job_logs
       end
 
       def self.db_filename
@@ -44,7 +44,7 @@ module Chronicle
         FileUtils.mkdir_p(File.dirname(db_filename))
       end
 
-      def self.initialize_schema db
+      def self.initialize_schema(db)
         db.create_table :job_logs do
           primary_key :id
           String :job_id, null: false

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'csv'
+require 'chronicle/utils/hash_utils'
 
 module Chronicle
   module ETL
@@ -6,6 +9,7 @@ module Chronicle
       include Chronicle::ETL::Loaders::Helpers::StdoutHelper
 
       register_connector do |r|
+        r.identifier = :csv
         r.description = 'CSV'
       end
 
@@ -18,13 +22,14 @@ module Chronicle
       end
 
       def load(record)
-        records << record.to_h_flattened
+        records << record
       end
 
       def finish
         return unless records.any?
 
-        headers = build_headers(records)
+        # headers = filtered_headers(records)
+        headers = gather_headers(records)
 
         csv_options = {}
         if @config.headers
@@ -34,8 +39,7 @@ module Chronicle
 
         csv_output = CSV.generate(**csv_options) do |csv|
           records.each do |record|
-            csv << record
-              .transform_keys(&:to_sym)
+            csv << Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
               .values_at(*headers)
               .map { |value| force_utf8(value) }
           end
@@ -48,6 +52,15 @@ module Chronicle
           File.write(@config.output, csv_output)
         end
       end
+
+      private
+
+      def gather_headers(records)
+        records_flattened = records.map do |record|
+          Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
+        end
+        records_flattened.flat_map(&:keys).uniq
+      end
     end
   end
 end

data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tempfile'
 
 module Chronicle
@@ -5,6 +7,8 @@ module Chronicle
     module Loaders
       module Helpers
         module StdoutHelper
+          # TODO: have option to immediately output to stdout
+
           # TODO: let users use "stdout" as an option for the `output` setting
           # Assume we're using stdout if no output is specified
           def output_to_stdout?

data/lib/chronicle/etl/loaders/json_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tempfile'
 
 module Chronicle
@@ -6,10 +8,10 @@ module Chronicle
       include Chronicle::ETL::Loaders::Helpers::StdoutHelper
 
       register_connector do |r|
+        r.identifier = :json
         r.description = 'json'
       end
 
-      setting :serializer
       setting :output
 
       # If true, one JSON record per line. If false, output a single json
@@ -26,23 +28,24 @@ module Chronicle
           if output_to_stdout?
             create_stdout_temp_file
           else
-            File.open(@config.output, "w+")
+            File.open(@config.output, 'w+')
           end
 
         @output_file.puts("[\n") unless @config.line_separated
       end
 
       def load(record)
-        serialized = serializer.serialize(record)
+        serialized = record.to_h
 
         # When dealing with raw data, we can get improperly encoded strings
         # (eg from sqlite database columns). We force conversion to UTF-8
         # before converting into JSON
-        encoded = serialized.transform_values do |value|
-          next value unless value.is_a?(String)
+        # encoded = serialized.transform_values do |value|
+        #   next value unless value.is_a?(String)
 
-          force_utf8(value)
-        end
+        #   force_utf8(value)
+        # end
+        encoded = deeply_force_utf8(serialized)
 
         line = encoded.to_json
         # For line-separated output, we just put json + newline
@@ -57,6 +60,8 @@ module Chronicle
         @output_file.write(line)
 
         @first_line = false
+        # rescue StandardError => e
+        #   binding.pry
       end
 
       def finish
@@ -70,9 +75,24 @@ module Chronicle
 
       private
 
-      # TODO: implement this
-      def serializer
-        @config.serializer || Chronicle::ETL::RawSerializer
+      # TODO: Move this to a helper module
+      def deeply_force_utf8(hash)
+        # FIXME: probably shouldn't happen but it does
+        return hash.map { |x| force_utf8(x) } if hash.is_a?(Array)
+        return force_utf8(hash) unless hash.is_a?(Hash)
+
+        hash.transform_values do |value|
+          case value
+          when String
+            force_utf8(value)
+          when Hash
+            deeply_force_utf8(value)
+          when Array
+            value.map { |v| deeply_force_utf8(v) }
+          else
+            value
+          end
+        end
       end
     end
   end

data/lib/chronicle/etl/loaders/loader.rb

@@ -32,23 +32,6 @@ module Chronicle
 
       # Called once there are no more records to process
       def finish; end
-
-      private
-
-      def build_headers(records)
-        headers =
-          if @config.fields && @config.fields.any?
-            Set[*@config.fields]
-          else
-            # use all the keys of the flattened record hash
-            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
-          end
-
-        headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) }
-        headers = headers.first(@config.fields_limit) if @config.fields_limit
-
-        headers.to_a.map(&:to_sym)
-      end
     end
   end
 end

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -1,11 +1,15 @@
+# frozen_string_literal: true
+
 require 'net/http'
 require 'uri'
 require 'json'
+require 'chronicle/serialization'
 
 module Chronicle
   module ETL
     class RestLoader < Chronicle::ETL::Loader
       register_connector do |r|
+        r.identifier = :rest
         r.description = 'a REST endpoint'
       end
 
@@ -13,16 +17,12 @@ module Chronicle
       setting :endpoint, required: true
       setting :access_token
 
-      def load(record)
-        payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
-        # have the outer data key that json-api expects
-        payload = { data: payload } unless payload[:data]
-
+      def load(payload)
         uri = URI.parse("#{@config.hostname}#{@config.endpoint}")
 
         header = {
-          "Authorization" => "Bearer #{@config.access_token}",
-          "Content-Type": 'application/json'
+          'Authorization' => "Bearer #{@config.access_token}",
+          'Content-Type': 'application/json'
         }
         use_ssl = uri.scheme == 'https'
 

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,49 +1,74 @@
+# frozen_string_literal: true
+
 require 'tty/table'
+require 'chronicle/utils/hash_utils'
 require 'active_support/core_ext/string/filters'
 require 'active_support/core_ext/hash/reverse_merge'
 
 module Chronicle
   module ETL
     class TableLoader < Chronicle::ETL::Loader
+
       register_connector do |r|
+        r.identifier = :table
         r.description = 'an ASCII table'
       end
 
       setting :truncate_values_at, default: 40
       setting :table_renderer, default: :basic
-      setting :fields_exclude, default: ['lids', 'type']
+      setting :fields_exclude, default: ['type']
       setting :header_row, default: true
 
       def load(record)
-        records << record.to_h_flattened
+        records << record
       end
 
       def finish
         return if records.empty?
 
-        headers = build_headers(records)
+        headers = gather_headers(records)
         rows = build_rows(records, headers)
 
+        render_table(headers, rows)
+      end
+
+      def records
+        @records ||= []
+      end
+
+      private
+
+      def render_table(headers, rows)
         @table = TTY::Table.new(header: (headers if @config.header_row), rows: rows)
         puts @table.render(
           @config.table_renderer.to_sym,
           padding: [0, 2, 0, 0]
         )
+      rescue TTY::Table::ResizeError
+        # The library throws this error before trying to render the table
+        # vertically. These options seem to work.
+        puts @table.render(
+          @config.table_renderer.to_sym,
+          padding: [0, 2, 0, 0],
+          width: 10_000,
+          resize: false
+        )
       end
 
-      def records
-        @records ||= []
+      def gather_headers(records)
+        records_flattened = records.map do |record|
+          Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
+        end
+        records_flattened.flat_map(&:keys).uniq
       end
 
-      private
-
       def build_rows(records, headers)
         records.map do |record|
-          values = record.transform_keys(&:to_sym).values_at(*headers).map{|value| value.to_s }
-          values = values.map { |value| force_utf8(value) }
-          if @config.truncate_values_at
-            values = values.map{ |value| value.truncate(@config.truncate_values_at) }
-          end
+          values = Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
+            .values_at(*headers)
+            .map { |value| force_utf8(value.to_s) }
+
+          values = values.map { |value| value.truncate(@config.truncate_values_at) } if @config.truncate_values_at
 
           values
         end

data/lib/chronicle/etl/logger.rb

@@ -14,13 +14,13 @@ module Chronicle
 
       @log_level = INFO
 
-      def output message, level
+      def output(message, level)
         return unless level >= @log_level
 
         if @ui_element
           @ui_element.log(message)
         else
-          $stderr.puts(message)
+          warn(message)
         end
       end
 
@@ -41,7 +41,7 @@ module Chronicle
       end
 
       def attach_to_ui(ui_element)
-        @ui_elemenet = ui_element
+        @ui_element = ui_element
       end
 
       def detach_from_ui

data/lib/chronicle/etl/oauth_authorizer.rb

@@ -49,21 +49,19 @@ module Chronicle
       def authorize!
         associate_oauth_credentials
         @server = load_server
-        spinner = TTY::Spinner.new(":spinner :title", format: :dots_2)
-        Chronicle::ETL::Logger.attach_to_ui(spinner)
+        spinner = TTY::Spinner.new(':spinner :title', format: :dots_2)
         spinner.auto_spin
-        spinner.update(title: "Starting temporary authorization server on port #{@port}""")
+        spinner.update(title: "Starting temporary authorization server on port #{@port}"'')
 
         server_thread = start_authorization_server(port: @port)
         start_oauth_flow
 
-        spinner.update(title: "Waiting for authorization to complete in your browser")
+        spinner.update(title: 'Waiting for authorization to complete in your browser')
         sleep 0.1 while authorization_pending?(server_thread)
 
         @server.quit!
         server_thread.join
         spinner.success("(#{'successful'.green})")
-        Chronicle::ETL::Logger.detach_from_ui
 
         # TODO: properly handle failed authorizations
         raise Chronicle::ETL::AuthorizationError unless @server.latest_authorization
@@ -87,7 +85,7 @@ module Chronicle
       def load_server
         # Load at runtime so that we can set omniauth strategies based on
         # which chronicle plugin has been loaded.
-        require_relative './authorization_server'
+        require_relative 'authorization_server'
         Chronicle::ETL::AuthorizationServer
       end
 
@@ -99,7 +97,7 @@ module Chronicle
 
         Thread.new do
           @server.run!({ port: @port }) do |s|
-            s.silent = true if s.class.to_s == "Thin::Server"
+            s.silent = true if defined?(::Thin::Server) && s.instance_of?(::Thin::Server)
           end
         end
       end
@@ -119,7 +117,7 @@ module Chronicle
             AccessLog: [],
             # TODO: make this windows friendly
             # https://github.com/winton/stasis/commit/77da36f43285fda129300e382f18dfaff48571b0
-            Logger: WEBrick::Log::new("/dev/null")
+            Logger: WEBrick::Log.new('/dev/null')
           }
         )
       rescue LoadError
@@ -129,8 +127,8 @@ module Chronicle
       def extract_secrets(authorization:, pluck_values:)
         return authorization unless pluck_values&.any?
 
-        pluck_values.each_with_object({}) do |(key, identifiers), secrets|
-          secrets[key] = authorization.dig(*identifiers)
+        pluck_values.transform_values do |identifiers|
+          authorization.dig(*identifiers)
         end
       end
 

data/lib/chronicle/etl/record.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# TODO: move this into chronicle-core after figuring out what to do about data vs properties
+module Chronicle
+  module ETL
+    class Record
+      attr_accessor :data, :extraction
+
+      def initialize(data: {}, extraction: nil)
+        @data = data
+        @extraction = extraction
+      end
+    end
+  end
+end