chronicle-etl 0.5.5 → 0.6.1

data/lib/chronicle/etl/job_log.rb

@@ -9,13 +9,13 @@ module Chronicle
       extend Forwardable
 
       attr_accessor :job,
-                    :job_id,
-                    :last_id,
-                    :highest_timestamp,
-                    :num_records_processed,
-                    :started_at,
-                    :finished_at,
-                    :success
+        :job_id,
+        :last_id,
+        :highest_timestamp,
+        :num_records_processed,
+        :started_at,
+        :finished_at,
+        :success
 
       def_delegators :@job, :save_log?
 
@@ -28,11 +28,11 @@ module Chronicle
 
       # Log the result of a single transformation in a job
       # @param transformer [Chronicle::ETL::Tranformer] The transformer that ran
-      def log_transformation(transformer)
-        @last_id = transformer.id if transformer.id
+      def log_transformation(_transformer)
+        # @last_id = transformer.id if transformer.id
 
         # Save the highest timestamp that we've encountered so far
-        @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
+        # @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
 
         # TODO: a transformer might yield nil. We might also want certain transformers to explode
         # records into multiple new ones. Therefore, this this variable will need more subtle behaviour
@@ -54,13 +54,13 @@ module Chronicle
         @finished_at = Time.now
       end
 
-      def job= job
+      def job=(job)
         @job = job
         @job_id = job.id
       end
 
       def duration
-        return unless @finished_at
+        return unless @finished_at && @started_at
 
         @finished_at - @started_at
       end
@@ -78,14 +78,12 @@ module Chronicle
         }
       end
 
-      private
-
       # Create a new JobLog and set its instance variables from a serialized hash
-      def self.build_from_serialized attrs
+      def self.build_from_serialized(attrs)
         attrs.delete(:id)
         new do |job_log|
           attrs.each do |key, value|
-            setter = "#{key.to_s}=".to_sym
+            setter = :"#{key}="
             job_log.send(setter, value)
           end
         end

data/lib/chronicle/etl/job_logger.rb

@@ -12,7 +12,7 @@ module Chronicle
       attr_accessor :job_log
 
       # For a given `job_id`, return the last successful log
-      def self.load_latest(job_id)
+      def self.load_latest(_job_id)
         with_db_connection do |db|
           attrs = db[:job_logs].reverse_order(:finished_at).where(success: true).first
           JobLog.build_from_serialized(attrs) if attrs
@@ -28,11 +28,11 @@ module Chronicle
       end
 
       def self.db_exists?
-        File.exists?(db_filename)
+        File.exist?(db_filename)
       end
 
       def self.schema_exists?(db)
-        return db.tables.include? :job_logs
+        db.tables.include? :job_logs
       end
 
       def self.db_filename
@@ -44,7 +44,7 @@ module Chronicle
         FileUtils.mkdir_p(File.dirname(db_filename))
       end
 
-      def self.initialize_schema db
+      def self.initialize_schema(db)
         db.create_table :job_logs do
           primary_key :id
           String :job_id, null: false

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'csv'
+require 'chronicle/utils/hash_utils'
 
 module Chronicle
   module ETL
@@ -6,6 +9,7 @@ module Chronicle
       include Chronicle::ETL::Loaders::Helpers::StdoutHelper
 
       register_connector do |r|
+        r.identifier = :csv
         r.description = 'CSV'
       end
 
@@ -18,13 +22,14 @@ module Chronicle
       end
 
       def load(record)
-        records << record.to_h_flattened
+        records << record
       end
 
       def finish
         return unless records.any?
 
-        headers = build_headers(records)
+        # headers = filtered_headers(records)
+        headers = gather_headers(records)
 
         csv_options = {}
         if @config.headers
@@ -34,8 +39,7 @@ module Chronicle
 
         csv_output = CSV.generate(**csv_options) do |csv|
           records.each do |record|
-            csv << record
-              .transform_keys(&:to_sym)
+            csv << Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
               .values_at(*headers)
               .map { |value| force_utf8(value) }
           end
@@ -48,6 +52,15 @@ module Chronicle
           File.write(@config.output, csv_output)
         end
       end
+
+      private
+
+      def gather_headers(records)
+        records_flattened = records.map do |record|
+          Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
+        end
+        records_flattened.flat_map(&:keys).uniq
+      end
     end
   end
 end

data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tempfile'
 
 module Chronicle
@@ -5,6 +7,8 @@ module Chronicle
     module Loaders
       module Helpers
         module StdoutHelper
+          # TODO: have option to immediately output to stdout
+
           # TODO: let users use "stdout" as an option for the `output` setting
           # Assume we're using stdout if no output is specified
           def output_to_stdout?

data/lib/chronicle/etl/loaders/json_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tempfile'
 
 module Chronicle
@@ -6,10 +8,10 @@ module Chronicle
       include Chronicle::ETL::Loaders::Helpers::StdoutHelper
 
       register_connector do |r|
+        r.identifier = :json
         r.description = 'json'
       end
 
-      setting :serializer
       setting :output
 
       # If true, one JSON record per line. If false, output a single json
@@ -26,23 +28,24 @@ module Chronicle
           if output_to_stdout?
             create_stdout_temp_file
           else
-            File.open(@config.output, "w+")
+            File.open(@config.output, 'w+')
           end
 
         @output_file.puts("[\n") unless @config.line_separated
       end
 
       def load(record)
-        serialized = serializer.serialize(record)
+        serialized = record.to_h
 
         # When dealing with raw data, we can get improperly encoded strings
         # (eg from sqlite database columns). We force conversion to UTF-8
         # before converting into JSON
-        encoded = serialized.transform_values do |value|
-          next value unless value.is_a?(String)
+        # encoded = serialized.transform_values do |value|
+        #   next value unless value.is_a?(String)
 
-          force_utf8(value)
-        end
+        #   force_utf8(value)
+        # end
+        encoded = deeply_force_utf8(serialized)
 
         line = encoded.to_json
         # For line-separated output, we just put json + newline
@@ -57,6 +60,8 @@ module Chronicle
         @output_file.write(line)
 
         @first_line = false
+        # rescue StandardError => e
+        #   binding.pry
       end
 
       def finish
@@ -70,9 +75,24 @@ module Chronicle
 
       private
 
-      # TODO: implement this
-      def serializer
-        @config.serializer || Chronicle::ETL::RawSerializer
+      # TODO: Move this to a helper module
+      def deeply_force_utf8(hash)
+        # FIXME: probably shouldn't happen but it does
+        return hash.map { |x| force_utf8(x) } if hash.is_a?(Array)
+        return force_utf8(hash) unless hash.is_a?(Hash)
+
+        hash.transform_values do |value|
+          case value
+          when String
+            force_utf8(value)
+          when Hash
+            deeply_force_utf8(value)
+          when Array
+            value.map { |v| deeply_force_utf8(v) }
+          else
+            value
+          end
+        end
       end
     end
   end

data/lib/chronicle/etl/loaders/loader.rb

@@ -32,23 +32,6 @@ module Chronicle
 
       # Called once there are no more records to process
       def finish; end
-
-      private
-
-      def build_headers(records)
-        headers =
-          if @config.fields && @config.fields.any?
-            Set[*@config.fields]
-          else
-            # use all the keys of the flattened record hash
-            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
-          end
-
-        headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) }
-        headers = headers.first(@config.fields_limit) if @config.fields_limit
-
-        headers.to_a.map(&:to_sym)
-      end
     end
   end
 end

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -1,11 +1,15 @@
+# frozen_string_literal: true
+
 require 'net/http'
 require 'uri'
 require 'json'
+require 'chronicle/serialization'
 
 module Chronicle
   module ETL
     class RestLoader < Chronicle::ETL::Loader
       register_connector do |r|
+        r.identifier = :rest
         r.description = 'a REST endpoint'
       end
 
@@ -13,16 +17,12 @@ module Chronicle
       setting :endpoint, required: true
       setting :access_token
 
-      def load(record)
-        payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
-        # have the outer data key that json-api expects
-        payload = { data: payload } unless payload[:data]
-
+      def load(payload)
         uri = URI.parse("#{@config.hostname}#{@config.endpoint}")
 
         header = {
-          "Authorization" => "Bearer #{@config.access_token}",
-          "Content-Type": 'application/json'
+          'Authorization' => "Bearer #{@config.access_token}",
+          'Content-Type': 'application/json'
         }
         use_ssl = uri.scheme == 'https'
 

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,49 +1,74 @@
+# frozen_string_literal: true
+
 require 'tty/table'
+require 'chronicle/utils/hash_utils'
 require 'active_support/core_ext/string/filters'
 require 'active_support/core_ext/hash/reverse_merge'
 
 module Chronicle
   module ETL
     class TableLoader < Chronicle::ETL::Loader
+
       register_connector do |r|
+        r.identifier = :table
         r.description = 'an ASCII table'
       end
 
       setting :truncate_values_at, default: 40
       setting :table_renderer, default: :basic
-      setting :fields_exclude, default: ['lids', 'type']
+      setting :fields_exclude, default: ['type']
       setting :header_row, default: true
 
       def load(record)
-        records << record.to_h_flattened
+        records << record
       end
 
       def finish
         return if records.empty?
 
-        headers = build_headers(records)
+        headers = gather_headers(records)
         rows = build_rows(records, headers)
 
+        render_table(headers, rows)
+      end
+
+      def records
+        @records ||= []
+      end
+
+      private
+
+      def render_table(headers, rows)
         @table = TTY::Table.new(header: (headers if @config.header_row), rows: rows)
         puts @table.render(
           @config.table_renderer.to_sym,
           padding: [0, 2, 0, 0]
         )
+      rescue TTY::Table::ResizeError
+        # The library throws this error before trying to render the table
+        # vertically. These options seem to work.
+        puts @table.render(
+          @config.table_renderer.to_sym,
+          padding: [0, 2, 0, 0],
+          width: 10_000,
+          resize: false
+        )
       end
 
-      def records
-        @records ||= []
+      def gather_headers(records)
+        records_flattened = records.map do |record|
+          Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
+        end
+        records_flattened.flat_map(&:keys).uniq
       end
 
-      private
-
       def build_rows(records, headers)
         records.map do |record|
-          values = record.transform_keys(&:to_sym).values_at(*headers).map{|value| value.to_s }
-          values = values.map { |value| force_utf8(value) }
-          if @config.truncate_values_at
-            values = values.map{ |value| value.truncate(@config.truncate_values_at) }
-          end
+          values = Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
+            .values_at(*headers)
+            .map { |value| force_utf8(value.to_s) }
+
+          values = values.map { |value| value.truncate(@config.truncate_values_at) } if @config.truncate_values_at
 
           values
         end

data/lib/chronicle/etl/logger.rb

@@ -14,13 +14,13 @@ module Chronicle
 
       @log_level = INFO
 
-      def output message, level
+      def output(message, level)
         return unless level >= @log_level
 
         if @ui_element
           @ui_element.log(message)
         else
-          $stderr.puts(message)
+          warn(message)
         end
       end
 

data/lib/chronicle/etl/oauth_authorizer.rb

@@ -49,14 +49,14 @@ module Chronicle
       def authorize!
         associate_oauth_credentials
         @server = load_server
-        spinner = TTY::Spinner.new(":spinner :title", format: :dots_2)
+        spinner = TTY::Spinner.new(':spinner :title', format: :dots_2)
         spinner.auto_spin
-        spinner.update(title: "Starting temporary authorization server on port #{@port}""")
+        spinner.update(title: "Starting temporary authorization server on port #{@port}"'')
 
         server_thread = start_authorization_server(port: @port)
         start_oauth_flow
 
-        spinner.update(title: "Waiting for authorization to complete in your browser")
+        spinner.update(title: 'Waiting for authorization to complete in your browser')
         sleep 0.1 while authorization_pending?(server_thread)
 
         @server.quit!
@@ -85,7 +85,7 @@ module Chronicle
       def load_server
         # Load at runtime so that we can set omniauth strategies based on
         # which chronicle plugin has been loaded.
-        require_relative './authorization_server'
+        require_relative 'authorization_server'
         Chronicle::ETL::AuthorizationServer
       end
 
@@ -97,7 +97,7 @@ module Chronicle
 
         Thread.new do
           @server.run!({ port: @port }) do |s|
-            s.silent = true if s.class.to_s == "Thin::Server"
+            s.silent = true if defined?(::Thin::Server) && s.instance_of?(::Thin::Server)
           end
         end
       end
@@ -117,7 +117,7 @@ module Chronicle
             AccessLog: [],
             # TODO: make this windows friendly
             # https://github.com/winton/stasis/commit/77da36f43285fda129300e382f18dfaff48571b0
-            Logger: WEBrick::Log::new("/dev/null")
+            Logger: WEBrick::Log.new('/dev/null')
           }
         )
       rescue LoadError
@@ -127,8 +127,8 @@ module Chronicle
       def extract_secrets(authorization:, pluck_values:)
         return authorization unless pluck_values&.any?
 
-        pluck_values.each_with_object({}) do |(key, identifiers), secrets|
-          secrets[key] = authorization.dig(*identifiers)
+        pluck_values.transform_values do |identifiers|
+          authorization.dig(*identifiers)
         end
       end
 

data/lib/chronicle/etl/record.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# TODO: move this into chronicle-core after figuring out what to do about data vs properties
+module Chronicle
+  module ETL
+    class Record
+      attr_accessor :data, :extraction
+
+      def initialize(data: {}, extraction: nil)
+        @data = data
+        @extraction = extraction
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -1,15 +1,18 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     module Registry
-      # Records details about a connector such as its provider and a description
+      # Records details about a connector such as its source provider and a description
       class ConnectorRegistration
-        # FIXME: refactor custom accessor methods later in file
-        attr_accessor :identifier, :provider, :klass, :description
+        attr_accessor :klass, :identifier, :source, :strategy, :type, :description, :from_schema, :to_schema
 
+        # Create a new connector registration
         def initialize(klass)
           @klass = klass
         end
 
+        # The ETL phase of this connector
         def phase
           if klass.ancestors.include? Chronicle::ETL::Extractor
             :extractor
@@ -24,6 +27,7 @@ module Chronicle
           "#{phase}-#{identifier}"
         end
 
+        # Whether this connector is built-in to Chronicle
         def built_in?
           @klass.to_s.include? 'Chronicle::ETL'
         end
@@ -32,32 +36,20 @@ module Chronicle
           @klass.to_s
         end
 
-        def identifier
-          @identifier || @klass.to_s.split('::').last.gsub!(/(Extractor$|Loader$|Transformer$)/, '').downcase
-        end
-
-        def description
-          @description || @klass.to_s.split('::').last
-        end
-
-        def provider
-          @provider || (built_in? ? 'chronicle' : '')
-        end
-
         # TODO: allow overriding here. Maybe through self-registration process
         def plugin
-          @provider
+          @source
         end
 
         def descriptive_phrase
           prefix = case phase
-          when :extractor
-            "Extracts from"
-          when :transformer
-            "Transforms"
-          when :loader
-            "Loads to"
-          end
+                   when :extractor
+                     'Extracts from'
+                   when :transformer
+                     'Transforms'
+                   when :loader
+                     'Loads to'
+                   end
 
           "#{prefix} #{description}"
         end