chronicle-etl 0.5.5 → 0.6.1

data/lib/chronicle/etl/transformers/chronobase_transformer.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class ChronobaseTransformer < Chronicle::ETL::Transformer
+      PROPERTY_MAP = {
+        source: :provider,
+        source_id: :provider_id,
+        url: :provider_url,
+        end_time: :end_at,
+        start_time: :start_at,
+
+        name: :title,
+        description: :body,
+        text: :body,
+
+        recipient: :consumers,
+        agent: :actor,
+        object: :involved,
+
+        # music ones
+        by_artist: :creators,
+        in_album: :containers
+      }.freeze
+
+      VERB_MAP = {
+        ListenAction: 'listened',
+        CommunicateAction: 'messaged'
+      }.freeze
+
+      ENTITY_MAP = {
+        MusicRecording: 'song',
+        MusicAlbum: 'album',
+        MusicGroup: 'musicartist',
+        Message: 'message',
+        Person: 'person'
+      }.freeze
+
+      register_connector do |r|
+        r.identifier = :chronobase
+        r.description = 'records to chronobase schema'
+      end
+
+      def transform(record)
+        deeply_convert_record(record.data)
+      end
+
+      private
+
+      def deeply_convert_record(record)
+        type = activity?(record) ? 'activity' : 'entity'
+
+        properties = record.properties.compact.each_with_object({}) do |(k, v), h|
+          key = PROPERTY_MAP[k.to_sym] || k
+          h[key] = v
+        end
+
+        properties[:verb] = VERB_MAP[record.type_id.to_sym] if VERB_MAP.key?(record.type_id.to_sym)
+        properties[:represents] = ENTITY_MAP[record.type_id.to_sym] if ENTITY_MAP.key?(record.type_id.to_sym)
+
+        properties.transform_values! do |v|
+          case v
+          when Chronicle::Models::Base
+            deeply_convert_record(v)
+          when Array
+            v.map { |e| e.is_a?(Chronicle::Models::Base) ? deeply_convert_record(e) : e }
+          else
+            v
+          end
+        end
+
+        Chronicle::Serialization::Record.new(
+          id: record.id,
+          type: type,
+          properties: properties.compact,
+          meta: {
+            dedupe_on: transform_dedupe_on(record)
+          },
+          schema: 'chronobase'
+        )
+      end
+
+      def activity?(record)
+        record.type_id.end_with?('Action')
+      end
+
+      def transform_dedupe_on(record)
+        property_map_with_type = PROPERTY_MAP.merge({
+          type: activity?(record) ? :verb : :represents
+        })
+
+        record.dedupe_on.map do |set|
+          set.map do |d|
+            property_map_with_type[d] || d
+          end.join(',')
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/fields_limit_transformer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'chronicle/utils/hash_utils'
+
+module Chronicle
+  module ETL
+    # A transformer that filters the fields of a record and returns a new hash with only the specified fields.
+    class FieldsLimitTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :fields_limit
+        r.description = 'by taking first N fields'
+      end
+
+      setting :limit, type: :numeric, default: 10
+
+      def transform(record)
+        # flattern hash and then take the first limit fields
+
+        Chronicle::Utils::HashUtils.flatten_hash(record.data.to_h).first(@config.limit).to_h
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/filter_fields_transformer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    # A transformer that filters the fields of a record and returns a new hash with only the specified fields.
+    class FilterFieldsTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :filter_fields
+        r.description = 'by taking a subset of the fields'
+      end
+
+      setting :fields, type: :array, default: []
+
+      def transform(record)
+        hash = record.data.to_h.deep_transform_keys(&:to_sym)
+        filter_hash(hash, @config.fields.map)
+      end
+
+      private
+
+      def access_nested_value(data, path)
+        keys = path.split('.')
+        keys.reduce(data) do |acc, key|
+          if acc.is_a?(Array)
+            acc.map do |item|
+              item[key.to_sym]
+            rescue StandardError
+              nil
+            end
+              .compact
+          elsif key.include?('[')
+            key, index = key.split(/\[|\]/).reject(&:empty?)
+            acc = acc[key.to_sym] if acc
+            acc.is_a?(Array) ? acc[index.to_i] : nil
+          else
+            acc&.dig(key.to_sym)
+          end
+        end
+      end
+
+      def filter_hash(original_hash, fields)
+        fields.each_with_object({}) do |field, result|
+          value = access_nested_value(original_hash, field)
+          keys = field.split('.')
+          last_key = keys.pop.to_sym
+
+          current = result
+          keys.each do |key|
+            key = key.to_sym
+            key, = key.to_s.split(/\[|\]/) if key.to_s.include?('[')
+            current[key] ||= {}
+            current = current[key]
+          end
+
+          current[last_key] = value
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/filter_transformer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    # Return only records that match all the conditions of the filters
+    # setting.
+    class FilterTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :filter
+        r.description = 'by only accepting records that match conditions'
+      end
+
+      setting :filters, type: :hash
+
+      def transform(record)
+        record_hash = record.data.to_h
+
+        @config.filters.each do |key, value|
+          path = key.split('.').map do |k|
+            k.match?(/^\d+$/) ? k.to_i : k.to_sym
+          end
+
+          return nil unless record_hash.dig(*path) == value
+        end
+
+        record.data
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/format_transformer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class FormatTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :format
+        r.description = 'records to a differnet hash/json format'
+      end
+
+      setting :format, default: nil
+
+      def transform(record)
+        serializer = find_serializer(@config.format)
+        serializer.serialize(record.data)
+      end
+
+      private
+
+      def find_serializer(format)
+        case format
+        when 'jsonld'
+          Chronicle::Serialization::JSONLDSerializer
+        when 'jsonapi'
+          Chronicle::Serialization::JSONAPISerializer
+        else
+          raise 'unknown format'
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/merge_meta_transformer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class MergeMetaTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :merge_meta
+        r.description = 'merge extraction meta fields into the record'
+      end
+
+      def transform(record)
+        record.data unless record.extraction&.meta
+
+        record.data[:_meta] = record.extraction.meta
+        record.data
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/multiply_transformer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class MultiplyTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :multiply
+        r.description = 'by taking a sample'
+      end
+
+      setting :n, default: 2, type: :numeric
+
+      # return the result, sample_size percentage of the time. otherwise nil
+      def transform(record)
+        @config.n.to_i.times do
+          yield record.data
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -1,18 +1,16 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     class NullTransformer < Chronicle::ETL::Transformer
       register_connector do |r|
-        r.identifier = 'null'
+        r.identifier = :null
         r.description = 'in no way'
       end
 
-      def transform
-        Chronicle::ETL::Models::Raw.new(@extraction.data)
+      def transform(record)
+        yield record.data
       end
-
-      def timestamp; end
-
-      def id; end
     end
   end
 end

data/lib/chronicle/etl/transformers/sampler_transformer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class SamplerTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :sampler
+        r.description = 'by taking a sample'
+      end
+
+      setting :percent, default: 10, type: :numeric
+
+      # return the result, `percent` percentage of the time. otherwise nil
+      def transform(record)
+        return unless rand(100) < @config.percent
+
+        record.data
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/sort_transformer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class SortTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :sort
+        r.description = 'sorts records by a given field'
+      end
+
+      setting :key, required: true, default: 'id'
+      setting :direction, required: false, default: 'desc'
+
+      def transform(record)
+        stash_record(record)
+      end
+
+      def finish
+        return unless @stashed_records&.any?
+
+        sorted = @stashed_records.sort_by do |record|
+          value = record.data[@config.key]
+          value.nil? ? [1] : [0, value]
+        end
+
+        sorted.reverse! if @config.direction == 'desc'
+        sorted
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     # Abstract class representing an Transformer for an ETL job
@@ -5,65 +7,85 @@ module Chronicle
       extend Chronicle::ETL::Registry::SelfRegistering
       include Chronicle::ETL::Configurable
 
+      attr_reader :stashed_records
+
       # Construct a new instance of this transformer. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Transformer
-      def initialize(extraction, options = {})
-        unless extraction.is_a?(Chronicle::ETL::Extraction)
-          raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
-        end
-
-        @extraction = extraction
+      def initialize(options = {})
         apply_options(options)
       end
 
-      # @abstract Subclass is expected to implement #transform
-      # @!method transform
-      #   The main entrypoint for transforming a record. Called by a Runner on each extracted record
-
-      # The domain or provider-specific id of the record this transformer is working on.
-      # It is useful for: 
-      # - de-duping records that might exist in the loader's destination
-      # - building a cursor so an extractor doesn't have to start from the beginning of a 
-      #   a source 
-      def id
-        raise NotImplementedError
+      # Called once for each extracted record. Can return 0 or more transformed records.
+      def call(record, &block)
+        raise ArgumentError, 'Input must be a Chronicle::ETL::Record' unless record.is_a?(Record)
+
+        yielded = false
+
+        transformed_data = transform(record) do |data|
+          new_record = update_data(record, data)
+          block.call(new_record)
+
+          yielded = true
+        end
+
+        return if yielded
+
+        # Handle transformers that don't yield anything and return
+        # transformed data directly. Skip nil values.
+        [transformed_data].flatten.compact.each do |data|
+          new_record = update_data(record, data)
+          block.call(new_record)
+        end
       end
 
-      # The domain or provider-specific timestamp of the record this transformer is working on.
-      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
-      # data source from the beginning.
-      def timestamp
-        raise NotImplementedError
+      def call_finish(&block)
+        remaining_records = finish
+        return if remaining_records.nil?
+
+        remaining_records.each do |record|
+          block.call(record)
+        end
       end
 
-      # An optional, human-readable identifier for a transformation, intended for debugging or logging.
-      # By default, it is just the id.
-      def friendly_identifier
-        id
+      def transform(_record)
+        raise NotImplementedError, 'You must implement the transform method'
       end
 
-      def to_s
-        ts = begin
-          unknown = "???"
-          timestamp&.iso8601 || unknown
-        rescue TransformationError, NotImplementedError
-          unknown
-        end
+      # Called once after runner has processed all records
+      def finish; end
 
-        identifier = begin
-          unknown = self.class.to_s
-          friendly_identifier || self.class.to_s
-        rescue TransformationError, NotImplementedError
-          unknown
-        end
+      protected
+
+      def stash_record(record)
+        @stashed_records ||= []
+        @stashed_records << record
+        nil
+      end
+
+      def flush_stashed_records
+        @stashed_records.tap(&:clear)
+      end
 
-        "[#{ts}] #{identifier}"
+      def update_data(record, new_data)
+        new_record = record.clone
+        new_record.data = new_data
+        new_record
       end
     end
   end
 end
 
 require_relative 'null_transformer'
-require_relative 'image_file_transformer'
+require_relative 'sampler_transformer'
+require_relative 'buffer_transformer'
+require_relative 'multiply_transformer'
+require_relative 'sort_transformer'
+require_relative 'chronicle_transformer'
+require_relative 'format_transformer'
+require_relative 'filter_fields_transformer'
+require_relative 'fields_limit_transformer'
+require_relative 'merge_meta_transformer'
+require_relative 'filter_transformer'
+require_relative 'chronobase_transformer'

data/lib/chronicle/etl/utils/binary_attachments.rb

@@ -7,7 +7,7 @@ module Chronicle
       # Utility methods for dealing with binary files
       module BinaryAttachments
         def self.filename_to_base64(filename:, mimetype: nil)
-          mimetype = mimetype || guess_mimetype(filename: filename)
+          mimetype ||= guess_mimetype(filename: filename)
 
           "data:#{mimetype};base64,#{Base64.strict_encode64(File.read(filename))}"
         end

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -4,7 +4,6 @@ require 'colorize'
 module Chronicle
   module ETL
     module Utils
-
       class ProgressBar
         FORMAT_WITH_TOTAL = [
           ':bar ',
@@ -37,7 +36,7 @@ module Chronicle
           '/s) '.light_black
         ].join.freeze
 
-        def initialize(title: 'Loading', total:)
+        def initialize(total:, title: 'Loading')
           opts = {
             clear: true,
             complete: '▓'.light_blue,
@@ -64,7 +63,7 @@ module Chronicle
         end
 
         def log(message)
-          message.split("\n").each do |line|
+          message.split("\n").each do |_line|
             @pbar.log message
           end
         end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.5.5"
+    VERSION = '0.6.1'.freeze
   end
 end

data/lib/chronicle/etl.rb

@@ -1,25 +1,23 @@
+# frozen_string_literal: true
+
+require 'chronicle/schema'
+require 'chronicle/models/base'
+
 require_relative 'etl/registry/registry'
 require_relative 'etl/authorizer'
 require_relative 'etl/config'
 require_relative 'etl/configurable'
 require_relative 'etl/exceptions'
 require_relative 'etl/extraction'
+require_relative 'etl/record'
 require_relative 'etl/job_definition'
 require_relative 'etl/job_log'
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
 require_relative 'etl/logger'
-require_relative 'etl/models/activity'
-require_relative 'etl/models/attachment'
-require_relative 'etl/models/base'
-require_relative 'etl/models/raw'
-require_relative 'etl/models/entity'
 require_relative 'etl/runner'
 require_relative 'etl/secrets'
-require_relative 'etl/serializers/serializer'
 require_relative 'etl/utils/binary_attachments'
-require_relative 'etl/utils/hash_utilities'
-require_relative 'etl/utils/text_recognition'
 require_relative 'etl/utils/progress_bar'
 require_relative 'etl/version'