chronicle-etl 0.1.4 → 0.2.4

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -1,10 +1,9 @@
 module Chronicle
-  module Etl
-    class NullTransformer < Chronicle::Etl::Transformer
-      def transform data
-        return data
+  module ETL
+    class NullTransformer < Chronicle::ETL::Transformer
+      def transform
+        Chronicle::ETL::Models::Generic.new(@data)
       end
     end
-
   end
-end
+end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -1,18 +1,34 @@
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing an Transformer for an ETL job
     class Transformer
-      extend Chronicle::Etl::Catalog
+      extend Chronicle::ETL::Catalog
 
-      def initialize(options = {})
+      # Construct a new instance of this transformer. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Transformer
+      def initialize(options = {}, data)
         @options = options
+        @data = data
+        @record = Chronicle::ETL::Models::Activity.new
       end
 
-      def transform data
-        raise NotImplementedError
-      end
+      # @abstract Subclass is expected to implement #transform
+      # @!method transform
+      #   The main entrypoint for transforming a record. Called by a Runner on each extracted record
+
+      # The domain or provider-specific id of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def id; end
+
+      # The domain or provider-specific timestamp of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def timestamp; end
     end
   end
 end
 
-require_relative 'json_transformer'
 require_relative 'null_transformer'

data/lib/chronicle/etl/utils/hash_utilities.rb

@@ -0,0 +1,19 @@
+module Chronicle
+  module ETL
+    module Utils
+      module HashUtilities
+        def self.flatten_hash(hash)
+          hash.each_with_object({}) do |(k, v), h|
+            if v.is_a? Hash
+              flatten_hash(v).map do |h_k, h_v|
+                h["#{k}.#{h_k}".to_sym] = h_v
+              end
+            else 
+              h[k] = v
+            end
+           end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/utils/jsonapi.rb

@@ -0,0 +1,28 @@
+module Chronicle
+  module ETL
+    module Utils
+      module JSONAPI
+        # For a given Chronicle::ETL::Model, serialize it as jsonapi
+        def self.serialize(record)
+          return unless record.is_a? Chronicle::ETL::Models::Base
+
+          obj = record.identifier_hash
+          obj[:attributes] = record.attributes
+
+          relationships = Hash[record.associations.map do |k, v|
+            if v.is_a?(Array)
+              data = { data: v.map{ |association| serialize(association) } }
+            else
+              data = { data: serialize(v) }
+            end
+
+            [k, data]
+          end]
+
+          obj[:relationships] = relationships if relationships.any?
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -2,7 +2,7 @@ require 'tty/progressbar'
 require 'colorize'
 
 module Chronicle
-  module Etl
+  module ETL
     module Utils
 
       class ProgressBar
@@ -64,7 +64,7 @@ module Chronicle
         end
 
         def log(message)
-          @pbar.log message.inspect
+          @pbar.log message
         end
 
         def finish

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
-  module Etl
-    VERSION = "0.1.4"
+  module ETL
+    VERSION = "0.2.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.4
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-18 00:00:00.000000000 Z
+date: 2020-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -66,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.35'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.35'
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +150,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: runcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -133,9 +203,10 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
-- CHANGELOG.md
+- ".yardopts"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -148,19 +219,34 @@ files:
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
 - lib/chronicle/etl/catalog.rb
-- lib/chronicle/etl/cli.rb
+- lib/chronicle/etl/cli/connectors.rb
+- lib/chronicle/etl/cli/jobs.rb
+- lib/chronicle/etl/cli/main.rb
+- lib/chronicle/etl/cli/subcommand_base.rb
+- lib/chronicle/etl/config.rb
+- lib/chronicle/etl/exceptions.rb
 - lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
 - lib/chronicle/etl/extractors/file_extractor.rb
 - lib/chronicle/etl/extractors/stdin_extractor.rb
+- lib/chronicle/etl/job.rb
+- lib/chronicle/etl/job_definition.rb
+- lib/chronicle/etl/job_log.rb
+- lib/chronicle/etl/job_logger.rb
 - lib/chronicle/etl/loaders/csv_loader.rb
 - lib/chronicle/etl/loaders/loader.rb
+- lib/chronicle/etl/loaders/rest_loader.rb
 - lib/chronicle/etl/loaders/stdout_loader.rb
 - lib/chronicle/etl/loaders/table_loader.rb
+- lib/chronicle/etl/models/activity.rb
+- lib/chronicle/etl/models/base.rb
+- lib/chronicle/etl/models/entity.rb
+- lib/chronicle/etl/models/generic.rb
 - lib/chronicle/etl/runner.rb
-- lib/chronicle/etl/transformers/json_transformer.rb
 - lib/chronicle/etl/transformers/null_transformer.rb
 - lib/chronicle/etl/transformers/transformer.rb
+- lib/chronicle/etl/utils/hash_utilities.rb
+- lib/chronicle/etl/utils/jsonapi.rb
 - lib/chronicle/etl/utils/progress_bar.rb
 - lib/chronicle/etl/version.rb
 homepage: https://github.com/chronicle-app

data/CHANGELOG.md

@@ -1,23 +0,0 @@
-# Changelog
-
-This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.4] - 2020-08-18
-### Updated
-- Better display of available ETL classes
-- Updated documentation
-
-## [0.1.3] - 2020-08-13
-### Added
-- Ability to list all available ETL classes
-- Refactored E, T, L module and class structure
-- Better progress bar
-
-## [0.1.2] - 2020-08-02
-### Added
-- This changelog
-- Ability to use extractors, transformers, and loaders from other gems
-
-## [0.1.0] - 2020-08-01
-### Added
-- Basic job runner and ETL classes

data/lib/chronicle/etl/cli.rb

@@ -1,56 +0,0 @@
-require 'thor'
-require 'chronicle/etl'
-require 'colorize'
-
-module Chronicle
-  module Etl
-    class CLI < Thor
-      default_task :job
-
-      desc 'job', 'Runs an ETL job'
-      method_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
-      method_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
-      method_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
-      method_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
-      method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
-      method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-      method_option :job, aliases: '-j', desc: 'Job configuration file'
-
-      def job
-        runner_options = {
-          extractor: {
-            name: options[:extractor],
-            options: options[:'extractor-opts']
-          },
-          transformer: {
-            name: options[:transformer],
-            options: options[:'transformer-opts']
-          },
-          loader: {
-            name: options[:loader],
-            options: options[:'loader-opts']
-          }
-        }
-
-        runner = Runner.new(runner_options)
-        runner.run!
-      end
-
-      # FIXME: namespace this differently
-      desc 'list', 'List all ETL classes'
-      def list
-        klasses = Chronicle::Etl::Catalog.available_classes
-        klasses = klasses.sort_by do |a|
-          [a[:built_in].to_s, a[:provider], a[:phase]]
-        end
-
-        headers = klasses.first.keys.map do |key|
-          key.to_s.capitalize.light_white
-        end
-
-        table = TTY::Table.new(headers, klasses.map(&:values))
-        puts table.render(padding: [0, 2])
-      end
-    end
-  end
-end

data/lib/chronicle/etl/transformers/json_transformer.rb

@@ -1,11 +0,0 @@
-require 'json'
-
-module Chronicle
-  module Etl
-    class JsonTransformer < Chronicle::Etl::Transformer
-      def transform data
-        return JSON.parse(data)
-      end
-    end
-  end
-end