chronicle-etl 0.5.4 → 0.6.1

data/lib/chronicle/etl/runner.rb

@@ -1,133 +1,163 @@
+# frozen_string_literal: true
+
 require 'colorize'
 require 'chronic_duration'
-require "tty-spinner"
-
-class Chronicle::ETL::Runner
-  def initialize(job)
-    @job = job
-    @job_logger = Chronicle::ETL::JobLogger.new(@job)
-  end
-
-  def run!
-    begin_job
-    validate_job
-    instantiate_connectors
-    prepare_job
-    prepare_ui
-    run_extraction
-  rescue Chronicle::ETL::ExtractionError => e
-    @job_logger&.error
-    raise(Chronicle::ETL::RunnerError, "Extraction failed. #{e.message}")
-  rescue Interrupt
-    @job_logger&.error
-    raise(Chronicle::ETL::RunInterruptedError, "Job interrupted.")
-  rescue StandardError => e
-    # Just throwing this in here until we have better exception handling in
-    # loaders, etc
-    @job_logger&.error
-    raise(Chronicle::ETL::RunnerError, "Error running job. #{e.message}")
-  ensure
-    finish_job
-  end
-
-  private
-
-  def begin_job
-    Chronicle::ETL::Logger.info(tty_log_job_initialize)
-    @initialization_spinner = TTY::Spinner.new(":spinner :title", format: :dots_2)
-  end
-
-  def validate_job
-    @initialization_spinner.update(title: "Validating job")
-    @job.job_definition.validate!
-  end
-
-  def instantiate_connectors
-    @initialization_spinner.update(title: "Initializing connectors")
-    @extractor = @job.instantiate_extractor
-    @loader = @job.instantiate_loader
-  end
-
-  def prepare_job
-    @initialization_spinner.update(title: "Preparing job")
-    @job_logger.start
-    @loader.start
-
-    @initialization_spinner.update(title: "Preparing extraction")
-    @initialization_spinner.auto_spin
-    @extractor.prepare
-    @initialization_spinner.success("(#{'successful'.green})")
-    Chronicle::ETL::Logger.info("\n")
-  end
-
-  def prepare_ui
-    total = @extractor.results_count
-    @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
-    Chronicle::ETL::Logger.attach_to_ui(@progress_bar)
-  end
-
-  def run_extraction
-    @extractor.extract do |extraction|
-      process_extraction(extraction)
-      @progress_bar.increment
+require 'tty-spinner'
+
+module Chronicle
+  module ETL
+    class Runner
+      def initialize(job)
+        @job = job
+        @job_logger = Chronicle::ETL::JobLogger.new(@job)
+      end
+
+      def run!
+        begin_job
+        validate_job
+        instantiate_connectors
+        prepare_job
+        prepare_ui
+        run_extraction
+      rescue Chronicle::ETL::ExtractionError => e
+        @job_logger&.error
+        raise(Chronicle::ETL::RunnerError, "Extraction failed. #{e.message}")
+      rescue Interrupt
+        @job_logger&.error
+        raise(Chronicle::ETL::RunInterruptedError, 'Job interrupted.')
+      # rescue StandardError => e
+      #   # Just throwing this in here until we have better exception handling in
+      #   # loaders, etc
+      #   @job_logger&.error
+      #   raise(Chronicle::ETL::RunnerError, "Error running job. #{e.message}")
+      ensure
+        finish_job
+      end
+
+      private
+
+      def begin_job
+        Chronicle::ETL::Logger.info(tty_log_job_initialize)
+        @initialization_spinner = TTY::Spinner.new(':spinner :title', format: :dots_2)
+      end
+
+      def validate_job
+        @initialization_spinner.update(title: 'Validating job')
+        @job.job_definition.validate!
+      end
+
+      def instantiate_connectors
+        @initialization_spinner.update(title: 'Initializing connectors')
+        @extractor = @job.instantiate_extractor
+        @transformers = @job.instantiate_transformers
+        @loader = @job.instantiate_loader
+      end
+
+      def prepare_job
+        @initialization_spinner.update(title: 'Preparing job')
+        @job_logger.start
+        @loader.start
+
+        @initialization_spinner.update(title: 'Preparing extraction')
+        @initialization_spinner.auto_spin
+        @extractor.prepare
+        @initialization_spinner.success("(#{'successful'.green})")
+        Chronicle::ETL::Logger.info("\n")
+      end
+
+      def prepare_ui
+        total = @extractor.results_count
+        @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+        Chronicle::ETL::Logger.attach_to_ui(@progress_bar)
+      end
+
+      def run_extraction
+        # Pattern based on Kiba's StreamingRunner
+        # https://github.com/thbar/kiba/blob/master/lib/kiba/streaming_runner.rb
+        stream = extractor_stream
+        recurser = ->(s, t) { transform_stream(s, t) }
+        @transformers.reduce(stream, &recurser).each do |record|
+          Chronicle::ETL::Logger.debug(tty_log_transformation(record))
+          @job_logger.log_transformation(record)
+          @progress_bar.increment
+          load_record(record)
+        end
+
+        @progress_bar.finish
+
+        # This is typically a slow method (writing to stdout, writing a big file, etc)
+        # TODO: consider adding a spinner?
+        @loader.finish
+        @job_logger.finish
+      end
+
+      # Initial steam of extracted data, wrapped in a Record class
+      def extractor_stream
+        Enumerator.new do |y|
+          @extractor.extract do |extraction|
+            record = Chronicle::ETL::Record.new(data: extraction.data, extraction: extraction)
+            y << record
+          end
+        end
+      end
+
+      # For a given stream of records and a given transformer,
+      # returns a new stream of transformed records and finally
+      # calls the finish method on the transformer
+      def transform_stream(stream, transformer)
+        Enumerator.new do |y|
+          stream.each do |record|
+            transformer.call(record) do |transformed_record|
+              y << transformed_record
+            end
+          end
+
+          transformer.call_finish do |transformed_record|
+            y << transformed_record
+          end
+        end
+      end
+
+      def load_record(record)
+        @loader.load(record.data) unless @job.dry_run?
+      end
+
+      def finish_job
+        @job_logger.save
+        @progress_bar&.finish
+        Chronicle::ETL::Logger.detach_from_ui
+        Chronicle::ETL::Logger.info(tty_log_completion)
+      end
+
+      def tty_log_job_initialize
+        output = 'Beginning job '
+        output += "'#{@job.name}'".bold if @job.name
+        output
+      end
+
+      def tty_log_transformation(record)
+        output = '  ✓'.green
+        output + " #{record}"
+      end
+
+      def tty_log_transformation_failure(exception, transformer)
+        output = '  ✖'.red
+        output + " Failed to transform #{transformer}. #{exception.message}"
+      end
+
+      def tty_log_completion
+        status = @job_logger.success ? 'Success' : 'Failed'
+        job_completion = @job_logger.success ? 'Completed' : 'Partially completed'
+        output = "\n#{job_completion} job"
+        output += " '#{@job.name}'".bold if @job.name
+        output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
+        output += "\n  Status:\t".light_black + status
+        output += "\n  Completed:\t".light_black + @job_logger.job_log.num_records_processed.to_s
+        if @job_logger.job_log.highest_timestamp
+          output += "\n  Latest:\t".light_black + @job_logger.job_log.highest_timestamp.iso8601.to_s
+        end
+        output
+      end
     end
-
-    @progress_bar.finish
-
-    # This is typically a slow method (writing to stdout, writing a big file, etc)
-    # TODO: consider adding a spinner?
-    @loader.finish
-    @job_logger.finish
-  end
-
-  def process_extraction(extraction)
-    # For each extraction from our extractor, we create a new tarnsformer
-    transformer = @job.instantiate_transformer(extraction)
-
-    # And then transform that record, logging it if we're in debug log level
-    record = transformer.transform
-    Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
-    @job_logger.log_transformation(transformer)
-
-    # Then send the results to the loader
-    @loader.load(record) unless @job.dry_run?
-  rescue Chronicle::ETL::TransformationError => e
-    # TODO: have an option to cancel job if we encounter an error
-    Chronicle::ETL::Logger.error(tty_log_transformation_failure(e, transformer))
-  end
-
-  def finish_job
-    @job_logger.save
-    @progress_bar&.finish
-    Chronicle::ETL::Logger.detach_from_ui
-    Chronicle::ETL::Logger.info(tty_log_completion)
-  end
-
-  def tty_log_job_initialize
-    output = "Beginning job "
-    output += "'#{@job.name}'".bold if @job.name
-    output
-  end
-
-  def tty_log_transformation(transformer)
-    output = "  ✓".green
-    output += " #{transformer}"
-  end
-
-  def tty_log_transformation_failure(exception, transformer)
-    output = "  ✖".red
-    output += " Failed to build #{transformer}. #{exception.message}"
-  end
-
-  def tty_log_completion
-    status = @job_logger.success ? 'Success' : 'Failed'
-    job_completion = @job_logger.success ? 'Completed' : 'Partially completed'
-    output = "\n#{job_completion} job"
-    output += " '#{@job.name}'".bold if @job.name
-    output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
-    output += "\n  Status:\t".light_black + status
-    output += "\n  Completed:\t".light_black + "#{@job_logger.job_log.num_records_processed}"
-    output += "\n  Latest:\t".light_black + "#{@job_logger.job_log.highest_timestamp.iso8601}" if @job_logger.job_log.highest_timestamp
-    output
   end
 end

data/lib/chronicle/etl/secrets.rb

@@ -1,4 +1,4 @@
-require "active_support/core_ext/hash/keys"
+require 'active_support/core_ext/hash/keys'
 
 module Chronicle
   module ETL
@@ -8,7 +8,7 @@ module Chronicle
 
       # Whether a given namespace exists
       def exists?(namespace)
-        Chronicle::ETL::Config.exists?("secrets", namespace)
+        Chronicle::ETL::Config.exists?('secrets', namespace)
       end
 
       # Save a setting to a namespaced config file
@@ -47,7 +47,7 @@ module Chronicle
 
       # Read secrets from a config file
       def read(namespace)
-        definition = Chronicle::ETL::Config.load("secrets", namespace)
+        definition = Chronicle::ETL::Config.load('secrets', namespace)
         definition[:secrets] || {}
       end
 
@@ -56,8 +56,8 @@ module Chronicle
         data = {
           secrets: (secrets || {}).transform_keys(&:to_s),
           chronicle_etl_version: Chronicle::ETL::VERSION
-        }.deep_stringify_keys
-        Chronicle::ETL::Config.write("secrets", namespace, data)
+        }
+        Chronicle::ETL::Config.write('secrets', namespace, data)
       end
 
       # Which config files are available in ~/.config/chronicle/etl/secrets

data/lib/chronicle/etl/transformers/buffer_transformer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class BufferTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :buffer
+        r.description = 'by buffering'
+      end
+
+      setting :size, default: 10, description: 'The size of the buffer'
+
+      def transform(record)
+        stash_record(record)
+
+        # FIXME: this doesn't seem to be working with the runner
+        return if @stashed_records.size < @config.size
+
+        # FIXME: this will result in the wrong extraction being associated with
+        # the batch of flushed records
+        flush_stashed_records.map(&:data)
+      end
+
+      def finish
+        flush_stashed_records
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/chronicle_transformer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class ChronicleTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :chronicle
+        r.description = 'records to Chronicle schema'
+      end
+
+      def transform(record)
+        converter_klass = find_converter(record.extraction)
+        # TODO: handle missing converter
+
+        converter_klass.new.call(record) do |transformed_record|
+          yield transformed_record.data
+        end
+      end
+
+      private
+
+      def find_converter(extraction)
+        Chronicle::ETL::Registry::Connectors.find_converter_for_source(
+          source: extraction.source,
+          type: extraction.type,
+          strategy: extraction.strategy,
+          target: :chronicle
+        )&.klass
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/chronobase_transformer.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class ChronobaseTransformer < Chronicle::ETL::Transformer
+      PROPERTY_MAP = {
+        source: :provider,
+        source_id: :provider_id,
+        url: :provider_url,
+        end_time: :end_at,
+        start_time: :start_at,
+
+        name: :title,
+        description: :body,
+        text: :body,
+
+        recipient: :consumers,
+        agent: :actor,
+        object: :involved,
+
+        # music ones
+        by_artist: :creators,
+        in_album: :containers
+      }.freeze
+
+      VERB_MAP = {
+        ListenAction: 'listened',
+        CommunicateAction: 'messaged'
+      }.freeze
+
+      ENTITY_MAP = {
+        MusicRecording: 'song',
+        MusicAlbum: 'album',
+        MusicGroup: 'musicartist',
+        Message: 'message',
+        Person: 'person'
+      }.freeze
+
+      register_connector do |r|
+        r.identifier = :chronobase
+        r.description = 'records to chronobase schema'
+      end
+
+      def transform(record)
+        deeply_convert_record(record.data)
+      end
+
+      private
+
+      def deeply_convert_record(record)
+        type = activity?(record) ? 'activity' : 'entity'
+
+        properties = record.properties.compact.each_with_object({}) do |(k, v), h|
+          key = PROPERTY_MAP[k.to_sym] || k
+          h[key] = v
+        end
+
+        properties[:verb] = VERB_MAP[record.type_id.to_sym] if VERB_MAP.key?(record.type_id.to_sym)
+        properties[:represents] = ENTITY_MAP[record.type_id.to_sym] if ENTITY_MAP.key?(record.type_id.to_sym)
+
+        properties.transform_values! do |v|
+          case v
+          when Chronicle::Models::Base
+            deeply_convert_record(v)
+          when Array
+            v.map { |e| e.is_a?(Chronicle::Models::Base) ? deeply_convert_record(e) : e }
+          else
+            v
+          end
+        end
+
+        Chronicle::Serialization::Record.new(
+          id: record.id,
+          type: type,
+          properties: properties.compact,
+          meta: {
+            dedupe_on: transform_dedupe_on(record)
+          },
+          schema: 'chronobase'
+        )
+      end
+
+      def activity?(record)
+        record.type_id.end_with?('Action')
+      end
+
+      def transform_dedupe_on(record)
+        property_map_with_type = PROPERTY_MAP.merge({
+          type: activity?(record) ? :verb : :represents
+        })
+
+        record.dedupe_on.map do |set|
+          set.map do |d|
+            property_map_with_type[d] || d
+          end.join(',')
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/fields_limit_transformer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'chronicle/utils/hash_utils'
+
+module Chronicle
+  module ETL
+    # A transformer that filters the fields of a record and returns a new hash with only the specified fields.
+    class FieldsLimitTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :fields_limit
+        r.description = 'by taking first N fields'
+      end
+
+      setting :limit, type: :numeric, default: 10
+
+      def transform(record)
+        # flattern hash and then take the first limit fields
+
+        Chronicle::Utils::HashUtils.flatten_hash(record.data.to_h).first(@config.limit).to_h
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/filter_fields_transformer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    # A transformer that filters the fields of a record and returns a new hash with only the specified fields.
+    class FilterFieldsTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :filter_fields
+        r.description = 'by taking a subset of the fields'
+      end
+
+      setting :fields, type: :array, default: []
+
+      def transform(record)
+        hash = record.data.to_h.deep_transform_keys(&:to_sym)
+        filter_hash(hash, @config.fields.map)
+      end
+
+      private
+
+      def access_nested_value(data, path)
+        keys = path.split('.')
+        keys.reduce(data) do |acc, key|
+          if acc.is_a?(Array)
+            acc.map do |item|
+              item[key.to_sym]
+            rescue StandardError
+              nil
+            end
+              .compact
+          elsif key.include?('[')
+            key, index = key.split(/\[|\]/).reject(&:empty?)
+            acc = acc[key.to_sym] if acc
+            acc.is_a?(Array) ? acc[index.to_i] : nil
+          else
+            acc&.dig(key.to_sym)
+          end
+        end
+      end
+
+      def filter_hash(original_hash, fields)
+        fields.each_with_object({}) do |field, result|
+          value = access_nested_value(original_hash, field)
+          keys = field.split('.')
+          last_key = keys.pop.to_sym
+
+          current = result
+          keys.each do |key|
+            key = key.to_sym
+            key, = key.to_s.split(/\[|\]/) if key.to_s.include?('[')
+            current[key] ||= {}
+            current = current[key]
+          end
+
+          current[last_key] = value
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/filter_transformer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    # Return only records that match all the conditions of the filters
+    # setting.
+    class FilterTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :filter
+        r.description = 'by only accepting records that match conditions'
+      end
+
+      setting :filters, type: :hash
+
+      def transform(record)
+        record_hash = record.data.to_h
+
+        @config.filters.each do |key, value|
+          path = key.split('.').map do |k|
+            k.match?(/^\d+$/) ? k.to_i : k.to_sym
+          end
+
+          return nil unless record_hash.dig(*path) == value
+        end
+
+        record.data
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/format_transformer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class FormatTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :format
+        r.description = 'records to a differnet hash/json format'
+      end
+
+      setting :format, default: nil
+
+      def transform(record)
+        serializer = find_serializer(@config.format)
+        serializer.serialize(record.data)
+      end
+
+      private
+
+      def find_serializer(format)
+        case format
+        when 'jsonld'
+          Chronicle::Serialization::JSONLDSerializer
+        when 'jsonapi'
+          Chronicle::Serialization::JSONAPISerializer
+        else
+          raise 'unknown format'
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/merge_meta_transformer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class MergeMetaTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :merge_meta
+        r.description = 'merge extraction meta fields into the record'
+      end
+
+      def transform(record)
+        record.data unless record.extraction&.meta
+
+        record.data[:_meta] = record.extraction.meta
+        record.data
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/multiply_transformer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class MultiplyTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :multiply
+        r.description = 'by taking a sample'
+      end
+
+      setting :n, default: 2, type: :numeric
+
+      # return the result, sample_size percentage of the time. otherwise nil
+      def transform(record)
+        @config.n.to_i.times do
+          yield record.data
+        end
+      end
+    end
+  end
+end