chronicle-etl 0.5.4 → 0.6.1

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -1,18 +1,16 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     class NullTransformer < Chronicle::ETL::Transformer
       register_connector do |r|
-        r.identifier = 'null'
+        r.identifier = :null
         r.description = 'in no way'
       end
 
-      def transform
-        Chronicle::ETL::Models::Raw.new(@extraction.data)
+      def transform(record)
+        yield record.data
       end
-
-      def timestamp; end
-
-      def id; end
     end
   end
 end

data/lib/chronicle/etl/transformers/sampler_transformer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class SamplerTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :sampler
+        r.description = 'by taking a sample'
+      end
+
+      setting :percent, default: 10, type: :numeric
+
+      # return the result, `percent` percentage of the time. otherwise nil
+      def transform(record)
+        return unless rand(100) < @config.percent
+
+        record.data
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/sort_transformer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class SortTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :sort
+        r.description = 'sorts records by a given field'
+      end
+
+      setting :key, required: true, default: 'id'
+      setting :direction, required: false, default: 'desc'
+
+      def transform(record)
+        stash_record(record)
+      end
+
+      def finish
+        return unless @stashed_records&.any?
+
+        sorted = @stashed_records.sort_by do |record|
+          value = record.data[@config.key]
+          value.nil? ? [1] : [0, value]
+        end
+
+        sorted.reverse! if @config.direction == 'desc'
+        sorted
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     # Abstract class representing an Transformer for an ETL job
@@ -5,65 +7,85 @@ module Chronicle
       extend Chronicle::ETL::Registry::SelfRegistering
       include Chronicle::ETL::Configurable
 
+      attr_reader :stashed_records
+
       # Construct a new instance of this transformer. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Transformer
-      def initialize(extraction, options = {})
-        unless extraction.is_a?(Chronicle::ETL::Extraction)
-          raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
-        end
-
-        @extraction = extraction
+      def initialize(options = {})
         apply_options(options)
       end
 
-      # @abstract Subclass is expected to implement #transform
-      # @!method transform
-      #   The main entrypoint for transforming a record. Called by a Runner on each extracted record
-
-      # The domain or provider-specific id of the record this transformer is working on.
-      # It is useful for: 
-      # - de-duping records that might exist in the loader's destination
-      # - building a cursor so an extractor doesn't have to start from the beginning of a 
-      #   a source 
-      def id
-        raise NotImplementedError
+      # Called once for each extracted record. Can return 0 or more transformed records.
+      def call(record, &block)
+        raise ArgumentError, 'Input must be a Chronicle::ETL::Record' unless record.is_a?(Record)
+
+        yielded = false
+
+        transformed_data = transform(record) do |data|
+          new_record = update_data(record, data)
+          block.call(new_record)
+
+          yielded = true
+        end
+
+        return if yielded
+
+        # Handle transformers that don't yield anything and return
+        # transformed data directly. Skip nil values.
+        [transformed_data].flatten.compact.each do |data|
+          new_record = update_data(record, data)
+          block.call(new_record)
+        end
       end
 
-      # The domain or provider-specific timestamp of the record this transformer is working on.
-      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
-      # data source from the beginning.
-      def timestamp
-        raise NotImplementedError
+      def call_finish(&block)
+        remaining_records = finish
+        return if remaining_records.nil?
+
+        remaining_records.each do |record|
+          block.call(record)
+        end
       end
 
-      # An optional, human-readable identifier for a transformation, intended for debugging or logging.
-      # By default, it is just the id.
-      def friendly_identifier
-        id
+      def transform(_record)
+        raise NotImplementedError, 'You must implement the transform method'
       end
 
-      def to_s
-        ts = begin
-          unknown = "???"
-          timestamp&.iso8601 || unknown
-        rescue TransformationError, NotImplementedError
-          unknown
-        end
+      # Called once after runner has processed all records
+      def finish; end
 
-        identifier = begin
-          unknown = self.class.to_s
-          friendly_identifier || self.class.to_s
-        rescue TransformationError, NotImplementedError
-          unknown
-        end
+      protected
+
+      def stash_record(record)
+        @stashed_records ||= []
+        @stashed_records << record
+        nil
+      end
+
+      def flush_stashed_records
+        @stashed_records.tap(&:clear)
+      end
 
-        "[#{ts}] #{identifier}"
+      def update_data(record, new_data)
+        new_record = record.clone
+        new_record.data = new_data
+        new_record
       end
     end
   end
 end
 
 require_relative 'null_transformer'
-require_relative 'image_file_transformer'
+require_relative 'sampler_transformer'
+require_relative 'buffer_transformer'
+require_relative 'multiply_transformer'
+require_relative 'sort_transformer'
+require_relative 'chronicle_transformer'
+require_relative 'format_transformer'
+require_relative 'filter_fields_transformer'
+require_relative 'fields_limit_transformer'
+require_relative 'merge_meta_transformer'
+require_relative 'filter_transformer'
+require_relative 'chronobase_transformer'

data/lib/chronicle/etl/utils/binary_attachments.rb

@@ -7,7 +7,7 @@ module Chronicle
       # Utility methods for dealing with binary files
       module BinaryAttachments
         def self.filename_to_base64(filename:, mimetype: nil)
-          mimetype = mimetype || guess_mimetype(filename: filename)
+          mimetype ||= guess_mimetype(filename: filename)
 
           "data:#{mimetype};base64,#{Base64.strict_encode64(File.read(filename))}"
         end

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -4,7 +4,6 @@ require 'colorize'
 module Chronicle
   module ETL
     module Utils
-
       class ProgressBar
         FORMAT_WITH_TOTAL = [
           ':bar ',
@@ -37,7 +36,7 @@ module Chronicle
           '/s) '.light_black
         ].join.freeze
 
-        def initialize(title: 'Loading', total:)
+        def initialize(total:, title: 'Loading')
           opts = {
             clear: true,
             complete: '▓'.light_blue,
@@ -64,7 +63,7 @@ module Chronicle
         end
 
         def log(message)
-          message.split("\n").each do |line|
+          message.split("\n").each do |_line|
             @pbar.log message
           end
         end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.5.4"
+    VERSION = '0.6.1'.freeze
   end
 end

data/lib/chronicle/etl.rb

@@ -1,25 +1,23 @@
+# frozen_string_literal: true
+
+require 'chronicle/schema'
+require 'chronicle/models/base'
+
 require_relative 'etl/registry/registry'
 require_relative 'etl/authorizer'
 require_relative 'etl/config'
 require_relative 'etl/configurable'
 require_relative 'etl/exceptions'
 require_relative 'etl/extraction'
+require_relative 'etl/record'
 require_relative 'etl/job_definition'
 require_relative 'etl/job_log'
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
 require_relative 'etl/logger'
-require_relative 'etl/models/activity'
-require_relative 'etl/models/attachment'
-require_relative 'etl/models/base'
-require_relative 'etl/models/raw'
-require_relative 'etl/models/entity'
 require_relative 'etl/runner'
 require_relative 'etl/secrets'
-require_relative 'etl/serializers/serializer'
 require_relative 'etl/utils/binary_attachments'
-require_relative 'etl/utils/hash_utilities'
-require_relative 'etl/utils/text_recognition'
 require_relative 'etl/utils/progress_bar'
 require_relative 'etl/version'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.1
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-23 00:00:00.000000000 Z
+date: 2024-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -39,75 +39,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.10.6
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: chronicle-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: launchy
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
-  name: marcel
+  name: gems
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: mini_exiftool
+  name: launchy
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: marcel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: 1.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: omniauth
   requirement: !ruby/object:Gem::Requirement
@@ -240,14 +240,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: xdg
   requirement: !ruby/object:Gem::Requirement
@@ -280,16 +280,16 @@ dependencies:
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -352,14 +352,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.25.1
+        version: '1.57'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.25.1
+        version: '1.57'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -374,6 +374,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -447,28 +489,31 @@ files:
 - lib/chronicle/etl/loaders/rest_loader.rb
 - lib/chronicle/etl/loaders/table_loader.rb
 - lib/chronicle/etl/logger.rb
-- lib/chronicle/etl/models/activity.rb
-- lib/chronicle/etl/models/attachment.rb
-- lib/chronicle/etl/models/base.rb
-- lib/chronicle/etl/models/entity.rb
-- lib/chronicle/etl/models/raw.rb
 - lib/chronicle/etl/oauth_authorizer.rb
+- lib/chronicle/etl/record.rb
 - lib/chronicle/etl/registry/connector_registration.rb
-- lib/chronicle/etl/registry/plugin_registry.rb
+- lib/chronicle/etl/registry/connectors.rb
+- lib/chronicle/etl/registry/plugin_registration.rb
+- lib/chronicle/etl/registry/plugins.rb
 - lib/chronicle/etl/registry/registry.rb
 - lib/chronicle/etl/registry/self_registering.rb
 - lib/chronicle/etl/runner.rb
 - lib/chronicle/etl/secrets.rb
-- lib/chronicle/etl/serializers/jsonapi_serializer.rb
-- lib/chronicle/etl/serializers/raw_serializer.rb
-- lib/chronicle/etl/serializers/serializer.rb
-- lib/chronicle/etl/transformers/image_file_transformer.rb
+- lib/chronicle/etl/transformers/buffer_transformer.rb
+- lib/chronicle/etl/transformers/chronicle_transformer.rb
+- lib/chronicle/etl/transformers/chronobase_transformer.rb
+- lib/chronicle/etl/transformers/fields_limit_transformer.rb
+- lib/chronicle/etl/transformers/filter_fields_transformer.rb
+- lib/chronicle/etl/transformers/filter_transformer.rb
+- lib/chronicle/etl/transformers/format_transformer.rb
+- lib/chronicle/etl/transformers/merge_meta_transformer.rb
+- lib/chronicle/etl/transformers/multiply_transformer.rb
 - lib/chronicle/etl/transformers/null_transformer.rb
+- lib/chronicle/etl/transformers/sampler_transformer.rb
+- lib/chronicle/etl/transformers/sort_transformer.rb
 - lib/chronicle/etl/transformers/transformer.rb
 - lib/chronicle/etl/utils/binary_attachments.rb
-- lib/chronicle/etl/utils/hash_utilities.rb
 - lib/chronicle/etl/utils/progress_bar.rb
-- lib/chronicle/etl/utils/text_recognition.rb
 - lib/chronicle/etl/version.rb
 homepage: https://github.com/chronicle-app
 licenses:
@@ -478,6 +523,7 @@ metadata:
   homepage_uri: https://github.com/chronicle-app
   source_code_uri: https://github.com/chronicle-app/chronicle-etl
   changelog_uri: https://github.com/chronicle-app/chronicle-etl/releases
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -486,14 +532,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: ETL tool for personal data

data/lib/chronicle/etl/models/activity.rb

@@ -1,15 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      class Activity < Chronicle::ETL::Models::Base
-        TYPE = 'activities'.freeze
-        ATTRIBUTES = [:verb, :start_at, :end_at].freeze
-        ASSOCIATIONS = [:involved, :actor].freeze
-
-        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/attachment.rb

@@ -1,14 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      class Attachment < Chronicle::ETL::Models::Base
-        TYPE = 'attachments'.freeze
-        ATTRIBUTES = [:url_original, :data].freeze
-
-        attr_accessor(*ATTRIBUTES)
-      end
-    end
-  end
-end