chronicle-etl 0.3.1 → 0.4.2

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Andrew Louis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-10 00:00:00.000000000 Z
+date: 2022-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: chronic_duration
   requirement: !ruby/object:Gem::Requirement
@@ -98,16 +98,16 @@ dependencies:
   name: runcom
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: thor-hollaback
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: tty-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tty-table
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: tty-prompt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.23'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.23'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +276,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.25.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.25.1
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -243,14 +341,15 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".ruby-version"
 - ".travis.yml"
 - ".yardopts"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -259,17 +358,20 @@ files:
 - chronicle-etl.gemspec
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
+- lib/chronicle/etl/cli.rb
 - lib/chronicle/etl/cli/connectors.rb
 - lib/chronicle/etl/cli/jobs.rb
 - lib/chronicle/etl/cli/main.rb
+- lib/chronicle/etl/cli/plugins.rb
 - lib/chronicle/etl/cli/subcommand_base.rb
 - lib/chronicle/etl/config.rb
+- lib/chronicle/etl/configurable.rb
 - lib/chronicle/etl/exceptions.rb
 - lib/chronicle/etl/extraction.rb
 - lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
 - lib/chronicle/etl/extractors/file_extractor.rb
-- lib/chronicle/etl/extractors/helpers/filesystem_reader.rb
+- lib/chronicle/etl/extractors/helpers/input_reader.rb
 - lib/chronicle/etl/extractors/json_extractor.rb
 - lib/chronicle/etl/extractors/stdin_extractor.rb
 - lib/chronicle/etl/job.rb
@@ -277,21 +379,24 @@ files:
 - lib/chronicle/etl/job_log.rb
 - lib/chronicle/etl/job_logger.rb
 - lib/chronicle/etl/loaders/csv_loader.rb
+- lib/chronicle/etl/loaders/helpers/encoding_helper.rb
+- lib/chronicle/etl/loaders/json_loader.rb
 - lib/chronicle/etl/loaders/loader.rb
 - lib/chronicle/etl/loaders/rest_loader.rb
-- lib/chronicle/etl/loaders/stdout_loader.rb
 - lib/chronicle/etl/loaders/table_loader.rb
 - lib/chronicle/etl/logger.rb
 - lib/chronicle/etl/models/activity.rb
 - lib/chronicle/etl/models/attachment.rb
 - lib/chronicle/etl/models/base.rb
 - lib/chronicle/etl/models/entity.rb
-- lib/chronicle/etl/models/generic.rb
+- lib/chronicle/etl/models/raw.rb
 - lib/chronicle/etl/registry/connector_registration.rb
+- lib/chronicle/etl/registry/plugin_registry.rb
 - lib/chronicle/etl/registry/registry.rb
 - lib/chronicle/etl/registry/self_registering.rb
 - lib/chronicle/etl/runner.rb
 - lib/chronicle/etl/serializers/jsonapi_serializer.rb
+- lib/chronicle/etl/serializers/raw_serializer.rb
 - lib/chronicle/etl/serializers/serializer.rb
 - lib/chronicle/etl/transformers/image_file_transformer.rb
 - lib/chronicle/etl/transformers/null_transformer.rb
@@ -317,14 +422,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.9
 signing_key: 
 specification_version: 4
 summary: ETL tool for personal data

data/.ruby-version

@@ -1 +0,0 @@
-2.7.1

data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb

@@ -1,104 +0,0 @@
-require 'pathname'
-
-module Chronicle
-  module ETL
-    module Extractors
-      module Helpers
-        module FilesystemReader
-
-          def filenames_in_directory(...)
-            filenames = gather_files(...)
-            if block_given?
-              filenames.each do |filename|
-                yield filename
-              end
-            else
-              filenames
-            end
-          end
-
-          def read_from_filesystem(filename:, yield_each_line: true, dir_glob_pattern: '**/*')
-            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
-              if yield_each_line
-                file.each_line do |line|
-                  yield line
-                end
-              else
-                yield file.read
-              end
-            end
-          end
-
-          def open_from_filesystem(filename:, dir_glob_pattern: '**/*')
-            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
-              yield file
-            end
-          end
-
-          def results_count
-            raise NotImplementedError
-            # if file?
-            #   return 1
-            # else
-            #   search_pattern = File.join(@options[:filename], '**/*')
-            #   Dir.glob(search_pattern).count
-            # end
-          end
-
-          private
-
-          def gather_files(path:, dir_glob_pattern: '**/*', load_since: nil, load_until: nil, smaller_than: nil, larger_than: nil, sort: :mtime)
-            search_pattern = File.join(path, '**', dir_glob_pattern)
-            files = Dir.glob(search_pattern)
-
-            files = files.keep_if {|f| (File.mtime(f) > load_since)} if load_since
-            files = files.keep_if {|f| (File.mtime(f) < load_until)} if load_until
-
-            # pass in file sizes in bytes
-            files = files.keep_if {|f| (File.size(f) < smaller_than)} if smaller_than
-            files = files.keep_if {|f| (File.size(f) > larger_than)} if larger_than
-
-            # TODO: incorporate sort argument
-            files.sort_by{ |f| File.mtime(f) }
-          end
-
-          def select_files_in_directory(path:, dir_glob_pattern: '**/*')
-            raise IOError.new("#{path} is not a directory.") unless directory?(path)
-
-            search_pattern = File.join(path, dir_glob_pattern)
-            Dir.glob(search_pattern).each do |filename|
-              yield(filename)
-            end
-          end
-
-          def open_files(filename:, dir_glob_pattern:)
-            if stdin?(filename)
-              yield $stdin
-            elsif directory?(filename)
-              search_pattern = File.join(filename, dir_glob_pattern)
-              filenames = Dir.glob(search_pattern)
-              filenames.each do |filename|
-                file = File.open(filename)
-                yield(file)
-              end
-            elsif file?(filename)
-              yield File.open(filename)
-            end
-          end
-
-          def stdin?(filename)
-            filename == $stdin
-          end
-
-          def directory?(filename)
-            Pathname.new(filename).directory?
-          end
-
-          def file?(filename)
-            Pathname.new(filename).file?
-          end
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,14 +0,0 @@
-module Chronicle
-  module ETL
-    class StdoutLoader < Chronicle::ETL::Loader
-      register_connector do |r|
-        r.description = 'stdout'
-      end
-
-      def load(record)
-        serializer = Chronicle::ETL::JSONAPISerializer.new(record)
-        puts serializer.serializable_hash.to_json
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/generic.rb

@@ -1,23 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      class Generic < Chronicle::ETL::Models::Base
-        TYPE = 'generic'
-
-        attr_accessor :properties
-
-        def initialize(properties = {})
-          @properties = properties
-          super
-        end
-
-        # Generic models have arbitrary attributes stored in @properties
-        def attributes
-          @properties.transform_keys(&:to_sym)
-        end
-      end
-    end
-  end
-end