chronicle-etl 0.5.5 → 0.6.1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.6.1
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-19 00:00:00.000000000 Z
+date: 2024-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.6
+- !ruby/object:Gem::Dependency
+  name: chronicle-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -94,34 +108,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.2
-- !ruby/object:Gem::Dependency
-  name: mini_exiftool
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: omniauth
   requirement: !ruby/object:Gem::Requirement
@@ -254,14 +240,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: xdg
   requirement: !ruby/object:Gem::Requirement
@@ -366,14 +352,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.25.1
+        version: '1.57'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.25.1
+        version: '1.57'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -416,6 +402,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -489,12 +489,8 @@ files:
 - lib/chronicle/etl/loaders/rest_loader.rb
 - lib/chronicle/etl/loaders/table_loader.rb
 - lib/chronicle/etl/logger.rb
-- lib/chronicle/etl/models/activity.rb
-- lib/chronicle/etl/models/attachment.rb
-- lib/chronicle/etl/models/base.rb
-- lib/chronicle/etl/models/entity.rb
-- lib/chronicle/etl/models/raw.rb
 - lib/chronicle/etl/oauth_authorizer.rb
+- lib/chronicle/etl/record.rb
 - lib/chronicle/etl/registry/connector_registration.rb
 - lib/chronicle/etl/registry/connectors.rb
 - lib/chronicle/etl/registry/plugin_registration.rb
@@ -503,16 +499,21 @@ files:
 - lib/chronicle/etl/registry/self_registering.rb
 - lib/chronicle/etl/runner.rb
 - lib/chronicle/etl/secrets.rb
-- lib/chronicle/etl/serializers/jsonapi_serializer.rb
-- lib/chronicle/etl/serializers/raw_serializer.rb
-- lib/chronicle/etl/serializers/serializer.rb
-- lib/chronicle/etl/transformers/image_file_transformer.rb
+- lib/chronicle/etl/transformers/buffer_transformer.rb
+- lib/chronicle/etl/transformers/chronicle_transformer.rb
+- lib/chronicle/etl/transformers/chronobase_transformer.rb
+- lib/chronicle/etl/transformers/fields_limit_transformer.rb
+- lib/chronicle/etl/transformers/filter_fields_transformer.rb
+- lib/chronicle/etl/transformers/filter_transformer.rb
+- lib/chronicle/etl/transformers/format_transformer.rb
+- lib/chronicle/etl/transformers/merge_meta_transformer.rb
+- lib/chronicle/etl/transformers/multiply_transformer.rb
 - lib/chronicle/etl/transformers/null_transformer.rb
+- lib/chronicle/etl/transformers/sampler_transformer.rb
+- lib/chronicle/etl/transformers/sort_transformer.rb
 - lib/chronicle/etl/transformers/transformer.rb
 - lib/chronicle/etl/utils/binary_attachments.rb
-- lib/chronicle/etl/utils/hash_utilities.rb
 - lib/chronicle/etl/utils/progress_bar.rb
-- lib/chronicle/etl/utils/text_recognition.rb
 - lib/chronicle/etl/version.rb
 homepage: https://github.com/chronicle-app
 licenses:
@@ -522,6 +523,7 @@ metadata:
   homepage_uri: https://github.com/chronicle-app
   source_code_uri: https://github.com/chronicle-app/chronicle-etl
   changelog_uri: https://github.com/chronicle-app/chronicle-etl/releases
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -530,14 +532,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: ETL tool for personal data

data/lib/chronicle/etl/models/activity.rb

@@ -1,15 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      class Activity < Chronicle::ETL::Models::Base
-        TYPE = 'activities'.freeze
-        ATTRIBUTES = [:verb, :start_at, :end_at].freeze
-        ASSOCIATIONS = [:involved, :actor].freeze
-
-        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/attachment.rb

@@ -1,14 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      class Attachment < Chronicle::ETL::Models::Base
-        TYPE = 'attachments'.freeze
-        ATTRIBUTES = [:url_original, :data].freeze
-
-        attr_accessor(*ATTRIBUTES)
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/base.rb

@@ -1,122 +0,0 @@
-require 'digest'
-
-module Chronicle
-  module ETL
-    module Models
-      # Represents a record that's been transformed by a Transformer and
-      # ready to be loaded. Loosely based on ActiveModel.
-      #
-      # @todo Experiment with just mixing in ActiveModel instead of this
-      #   this reimplementation
-      class Base
-        ATTRIBUTES = [:provider, :provider_id, :provider_namespace, :lat, :lng, :metadata].freeze
-        ASSOCIATIONS = [].freeze
-
-        attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
-
-        def initialize(attributes = {})
-          assign_attributes(attributes) if attributes
-          @dedupe_on = []
-          @metadata = {}
-        end
-
-        # A unique identifier for this model is formed from a type
-        # and either an id or lids.
-        def identifier_hash
-          {
-            type: self.class::TYPE,
-            id: @id,
-            lids: lids
-          }.compact
-        end
-
-        # Array of local ids that uniquely identify this record
-        def lids
-          @dedupe_on.map do |fields|
-            generate_lid(fields)
-          end.compact.uniq
-        end
-
-        # For a given set of fields of this model, generate a
-        # unique local id by hashing the field values
-        def generate_lid fields
-          raise ArgumentError.new("Must provide an array of symbolized fields") unless fields.is_a?(Array)
-
-          values = fields.sort.map do |field|
-            instance_variable = "@#{field.to_s}"
-            self.instance_variable_get(instance_variable)
-          end
-
-          return if values.any? { |e| e.nil? }
-
-          Digest::SHA256.hexdigest(values.join(","))
-        end
-
-        # Set of attribute names that this model has is Base's shared
-        # attributes combined with the child class's
-        def attribute_list
-          (ATTRIBUTES + self.class::ATTRIBUTES).uniq
-        end
-
-        # All of this record's attributes
-        def attributes
-          attributes = {}
-          attribute_list.each do |attribute|
-            instance_variable = "@#{attribute.to_s}"
-            attributes[attribute] = self.instance_variable_get(instance_variable)
-          end
-          attributes.compact
-        end
-
-        # All of this record's associations
-        def associations
-          association_list = ASSOCIATIONS + self.class::ASSOCIATIONS
-          attributes = {}
-          association_list.each do |attribute|
-            instance_variable = "@#{attribute.to_s}"
-            association = self.instance_variable_get(instance_variable)
-            attributes[attribute] = association if association
-          end
-          attributes.compact
-        end
-
-        def associations_hash
-          associations.map do |k, v|
-            if v.is_a?(Array)
-              [k, v.map(&:to_h)]
-            else
-              [k, v.to_h]
-            end
-          end.to_h
-        end
-
-        def meta_hash
-          {
-            meta: {
-              dedupe_on: @dedupe_on.map{|d| d.map(&:to_s).join(",")}
-            }
-          }
-        end
-
-        # FIXME: move this to a Utils module
-        def to_h_flattened
-          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
-        end
-
-        def to_h
-          identifier_hash
-            .merge(attributes)
-            .merge(associations_hash)
-            .merge(meta_hash)
-        end
-
-        def assign_attributes attributes
-          attributes.each do |k, v|
-            setter = :"#{k}="
-            public_send(setter, v) if respond_to? setter
-          end
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/entity.rb

@@ -1,29 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      class Entity < Chronicle::ETL::Models::Base
-        TYPE = 'entities'.freeze
-        ATTRIBUTES = [:title, :body, :provider_url, :represents, :slug, :myself, :metadata].freeze
-
-        # TODO: This desperately needs a validation system
-        ASSOCIATIONS = [
-          :involvements, # inverse of activity's `involved`
-          :analogous,
-          :attachments,
-          :abouts,
-          :aboutables, # inverse of above
-          :depicts,
-          :consumers,
-          :creators,
-          :creations,
-          :contains,
-          :containers # inverse of above
-        ].freeze  # TODO: add these to reflect Chronicle Schema
-
-        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/raw.rb

@@ -1,26 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      # A record from an extraction with no processing or normalization applied
-      class Raw
-        TYPE = 'raw'
-
-        attr_accessor :raw_data
-
-        def initialize(raw_data)
-          @raw_data = raw_data
-        end
-
-        def to_h
-          @raw_data.to_h
-        end
-
-        def to_h_flattened
-          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -1,31 +0,0 @@
-module Chronicle
-  module ETL
-    class JSONAPISerializer < Chronicle::ETL::Serializer
-      def initialize(*args)
-        super
-
-        raise(SerializationError, "Record must be a subclass of Chronicle::ETL::Model::Base") unless @record.is_a?(Chronicle::ETL::Models::Base)
-      end
-
-      def serializable_hash
-        @record
-          .identifier_hash
-          .merge({ attributes: @record.attributes })
-          .merge({ relationships: build_associations })
-          .merge(@record.meta_hash)
-      end
-
-      def build_associations
-        @record.associations.transform_values do |value|
-          association_data =
-            if value.is_a?(Array)
-              value.map { |record| JSONAPISerializer.new(record).serializable_hash }
-            else
-              JSONAPISerializer.new(value).serializable_hash
-            end
-          { data: association_data }
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/serializers/raw_serializer.rb

@@ -1,10 +0,0 @@
-module Chronicle
-  module ETL
-    # Take a Raw model and output `raw_data` as a hash
-    class RawSerializer < Chronicle::ETL::Serializer
-      def serializable_hash
-        @record.to_h
-      end
-    end
-  end
-end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -1,28 +0,0 @@
-module Chronicle
-  module ETL
-    # Abstract class representing a Serializer for an ETL record
-    class Serializer
-      # Construct a new instance of this serializer.
-      # == Parameters:
-      # options::
-      #   Options for configuring this Serializers
-      def initialize(record, options = {})
-        @record = record
-        @options = options
-      end
-
-      # Serialize a record as a hash
-      def serializable_hash
-        raise NotImplementedError
-      end
-
-      def self.serialize(record)
-        serializer = self.new(record)
-        serializer.serializable_hash
-      end
-    end
-  end
-end
-
-require_relative 'jsonapi_serializer'
-require_relative 'raw_serializer'