chronicle-etl 0.2.4 → 0.3.0

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Louis
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-23 00:00:00.000000000 Z
+date: 2022-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: chronic_duration
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: 0.10.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: 0.10.6
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -39,33 +53,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.8.1
 - !ruby/object:Gem::Dependency
-  name: tty-table
+  name: marcel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 1.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
-  name: tty-progressbar
+  name: mini_exiftool
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.17'
+        version: '2.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.17'
+        version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: runcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -81,117 +123,117 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5.35'
 - !ruby/object:Gem::Dependency
-  name: deep_merge
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
-  type: :development
+        version: '0.20'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '0.20'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: tty-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
+        version: '0.17'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: tty-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
-  type: :development
+        version: '0.11'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
-  name: pry-byebug
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: runcom
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
-  name: redcarpet
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '3.9'
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -209,7 +251,6 @@ files:
 - ".yardopts"
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -218,16 +259,18 @@ files:
 - chronicle-etl.gemspec
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
-- lib/chronicle/etl/catalog.rb
 - lib/chronicle/etl/cli/connectors.rb
 - lib/chronicle/etl/cli/jobs.rb
 - lib/chronicle/etl/cli/main.rb
 - lib/chronicle/etl/cli/subcommand_base.rb
 - lib/chronicle/etl/config.rb
 - lib/chronicle/etl/exceptions.rb
+- lib/chronicle/etl/extraction.rb
 - lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
 - lib/chronicle/etl/extractors/file_extractor.rb
+- lib/chronicle/etl/extractors/helpers/filesystem_reader.rb
+- lib/chronicle/etl/extractors/json_extractor.rb
 - lib/chronicle/etl/extractors/stdin_extractor.rb
 - lib/chronicle/etl/job.rb
 - lib/chronicle/etl/job_definition.rb
@@ -238,16 +281,25 @@ files:
 - lib/chronicle/etl/loaders/rest_loader.rb
 - lib/chronicle/etl/loaders/stdout_loader.rb
 - lib/chronicle/etl/loaders/table_loader.rb
+- lib/chronicle/etl/logger.rb
 - lib/chronicle/etl/models/activity.rb
+- lib/chronicle/etl/models/attachment.rb
 - lib/chronicle/etl/models/base.rb
 - lib/chronicle/etl/models/entity.rb
 - lib/chronicle/etl/models/generic.rb
+- lib/chronicle/etl/registry/connector_registration.rb
+- lib/chronicle/etl/registry/registry.rb
+- lib/chronicle/etl/registry/self_registering.rb
 - lib/chronicle/etl/runner.rb
+- lib/chronicle/etl/serializers/jsonapi_serializer.rb
+- lib/chronicle/etl/serializers/serializer.rb
+- lib/chronicle/etl/transformers/image_file_transformer.rb
 - lib/chronicle/etl/transformers/null_transformer.rb
 - lib/chronicle/etl/transformers/transformer.rb
+- lib/chronicle/etl/utils/binary_attachments.rb
 - lib/chronicle/etl/utils/hash_utilities.rb
-- lib/chronicle/etl/utils/jsonapi.rb
 - lib/chronicle/etl/utils/progress_bar.rb
+- lib/chronicle/etl/utils/text_recognition.rb
 - lib/chronicle/etl/version.rb
 homepage: https://github.com/chronicle-app
 licenses:
@@ -256,7 +308,7 @@ metadata:
   homepage_uri: https://github.com/chronicle-app
   source_code_uri: https://github.com/chronicle-app/chronicle-etl
   changelog_uri: https://github.com/chronicle-app/chronicle-etl/blob/master/CHANGELOG.md
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -272,7 +324,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key:
+signing_key: 
 specification_version: 4
 summary: ETL tool for personal data
 test_files: []

data/Gemfile.lock

@@ -1,91 +0,0 @@
-PATH
-  remote: .
-  specs:
-    chronicle-etl (0.2.4)
-      colorize (~> 0.8.1)
-      deep_merge (~> 1.2)
-      sequel (~> 5.35)
-      thor (~> 0.20)
-      tty-progressbar (~> 0.17)
-      tty-table (~> 0.11)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    byebug (11.1.3)
-    coderay (1.1.3)
-    colorize (0.8.1)
-    deep_merge (1.2.1)
-    diff-lcs (1.4.4)
-    equatable (0.6.1)
-    method_source (1.0.0)
-    necromancer (0.6.0)
-    pastel (0.7.4)
-      equatable (~> 0.6)
-      tty-color (~> 0.5)
-    pry (0.13.1)
-      coderay (~> 1.1)
-      method_source (~> 1.0)
-    pry-byebug (3.9.0)
-      byebug (~> 11.0)
-      pry (~> 0.13.0)
-    rake (13.0.1)
-    redcarpet (3.5.0)
-    refinements (7.7.0)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    runcom (6.2.0)
-      refinements (~> 7.4)
-      xdg (~> 4.0)
-    sequel (5.36.0)
-    sqlite3 (1.4.2)
-    strings (0.1.8)
-      strings-ansi (~> 0.1)
-      unicode-display_width (~> 1.5)
-      unicode_utils (~> 1.4)
-    strings-ansi (0.1.0)
-    thor (0.20.3)
-    tty-color (0.5.2)
-    tty-cursor (0.7.1)
-    tty-progressbar (0.17.0)
-      strings-ansi (~> 0.1.0)
-      tty-cursor (~> 0.7)
-      tty-screen (~> 0.7)
-      unicode-display_width (~> 1.6)
-    tty-screen (0.8.1)
-    tty-table (0.11.0)
-      equatable (~> 0.6)
-      necromancer (~> 0.5)
-      pastel (~> 0.7.2)
-      strings (~> 0.1.5)
-      tty-screen (~> 0.7)
-    unicode-display_width (1.7.0)
-    unicode_utils (1.4.0)
-    xdg (4.2.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 2.1)
-  chronicle-etl!
-  pry-byebug (~> 3.9)
-  rake (~> 13.0)
-  redcarpet (~> 3.5)
-  rspec (~> 3.9)
-  runcom (~> 6.2)
-  sqlite3 (~> 1.4)
-
-BUNDLED WITH
-   2.1.4

data/lib/chronicle/etl/catalog.rb

@@ -1,108 +0,0 @@
-module Chronicle
-  module ETL
-    # Utility methods to catalogue which Extractor, Transformer, and
-    # Loader connector classes are available to chronicle-etl
-    module Catalog
-      PHASES = [:extractor, :transformer, :loader]
-      PLUGINS = ['email', 'bash']
-      BUILTIN = {
-        extractor: ['stdin', 'json', 'csv', 'file'],
-        transformer: ['null'],
-        loader: ['stdout', 'csv', 'table', 'rest']
-      }.freeze
-
-      # Return which ETL connectors are available, both built in and externally-defined
-      def self.available_classes
-        # TODO: have a registry of plugins
-
-        # Attempt to load each chronicle plugin that we might know about so
-        # that we can later search for subclasses to build our list of
-        # available classes
-        PLUGINS.each do |plugin|
-          require "chronicle/#{plugin}"
-        rescue LoadError
-          # this will happen if the gem isn't available globally
-        end
-
-        parent_klasses = [
-          ::Chronicle::ETL::Extractor,
-          ::Chronicle::ETL::Transformer,
-          ::Chronicle::ETL::Loader
-        ]
-        klasses = []
-        parent_klasses.map do |parent|
-          klasses += ::ObjectSpace.each_object(::Class).select { |klass| klass < parent }
-        end
-
-        klasses.map do |klass|
-          {
-            name: klass.name,
-            built_in: klass.built_in?,
-            provider: klass.provider,
-            phase: klass.phase
-          }
-        end
-      end
-
-      # Take a phase (e, t, or l) and an identifier and return the right class
-      def self.phase_and_identifier_to_klass(phase, identifier)
-        Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
-      end
-
-      # For a given connector identifier, return the class (either builtin, or from a 
-      # external chronicle gem)
-      def self.identifier_to_klass(identifier:, phase:)
-        if BUILTIN[phase].include? identifier
-          load_builtin_klass(name: identifier, phase: phase)
-        else
-          provider, name = identifier.split(':')
-          name ||= ''
-          load_provider_klass(provider: provider, name: name, phase: phase)
-        end
-      end
-
-      # Returns whether a class is an Extractor, Transformer, or Loader
-      def phase
-        ancestors = self.ancestors
-        return :extractor if ancestors.include? Chronicle::ETL::Extractor
-        return :transformer if ancestors.include? Chronicle::ETL::Transformer
-        return :loader if ancestors.include? Chronicle::ETL::Loader
-      end
-
-      # Returns which third-party provider this connector is associated wtih
-      def provider
-        # TODO: needs better convention for a gem reporting its provider name
-        provider = to_s.split('::')[1].downcase
-        provider == 'etl' ? 'chronicle' : provider
-      end
-
-      # Returns whether this connector is a built-in one
-      def built_in?
-        to_s.include? 'Chronicle::ETL'
-      end
-
-      private
-
-      def self.load_builtin_klass(name:, phase:)
-        klass_str = "Chronicle::ETL::#{name.capitalize}#{phase.capitalize}"
-        begin
-          Object.const_get(klass_str)
-        rescue NameError => e
-          raise ConnectorNotAvailableError.new("Connector not found", name: name)
-        end
-      end
-
-      def self.load_provider_klass(name: '', phase:, provider:)
-        begin
-          require "chronicle/#{provider}"
-          klass_str = "Chronicle::#{provider.capitalize}::#{name.capitalize}#{phase.capitalize}"
-          Object.const_get(klass_str)
-        rescue LoadError => e
-          raise ProviderNotAvailableError.new("Provider '#{provider.capitalize}' could not be loaded", provider: provider)
-        rescue NameError => e
-          raise ProviderConnectorNotAvailableError.new("Connector '#{name}' in '#{provider}' could not be found", provider: provider, name: name)
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/utils/jsonapi.rb

@@ -1,28 +0,0 @@
-module Chronicle
-  module ETL
-    module Utils
-      module JSONAPI
-        # For a given Chronicle::ETL::Model, serialize it as jsonapi
-        def self.serialize(record)
-          return unless record.is_a? Chronicle::ETL::Models::Base
-
-          obj = record.identifier_hash
-          obj[:attributes] = record.attributes
-
-          relationships = Hash[record.associations.map do |k, v|
-            if v.is_a?(Array)
-              data = { data: v.map{ |association| serialize(association) } }
-            else
-              data = { data: serialize(v) }
-            end
-
-            [k, data]
-          end]
-
-          obj[:relationships] = relationships if relationships.any?
-          obj
-        end
-      end
-    end
-  end
-end