chronicle-etl 0.1.1 → 0.2.1

data/lib/chronicle/etl/transformers/json_transformer.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+module Chronicle
+  module ETL
+    class JsonTransformer < Chronicle::ETL::Transformer
+      def transform data
+        return JSON.parse(data)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -0,0 +1,10 @@
+module Chronicle
+  module ETL
+    class NullTransformer < Chronicle::ETL::Transformer
+      def transform data
+        return data
+      end
+    end
+
+  end
+end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -1,18 +1,34 @@
 module Chronicle
-  module Etl
-    module Transformers
-      class Transformer
-        def initialize(options = {})
-          @options = options
-        end
+  module ETL
+    # Abstract class representing an Transformer for an ETL job
+    class Transformer
+      extend Chronicle::ETL::Catalog
 
-        def transform data
-          raise NotImplementedError
-        end
+      # Construct a new instance of this transformer. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Transformer
+      def initialize(options = {})
+        @options = options
       end
+
+      # The main entrypoint for transforming a record. Called by a Runner on each extracted record
+      def transform data
+        raise NotImplementedError
+      end
+
+      # The domain or provider-specific id of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def id; end
+
+      # The domain or provider-specific timestamp of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def timestamp; end
     end
   end
 end
 
-require_relative 'null'
-require_relative 'json'
+require_relative 'json_transformer'
+require_relative 'null_transformer'

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -0,0 +1,76 @@
+require 'tty/progressbar'
+require 'colorize'
+
+module Chronicle
+  module ETL
+    module Utils
+
+      class ProgressBar
+        FORMAT_WITH_TOTAL = [
+          ':bar ',
+          ':percent'.light_white,
+          ' | '.light_black,
+          ':current'.light_white,
+          '/'.light_black,
+          ':total'.light_white,
+          ' ('.light_black,
+          'ELAPSED:'.light_black,
+          ':elapsed'.light_white,
+          ' | ETA:'.light_black,
+          ':eta'.light_white,
+          ' | RATE: '.light_black,
+          ':mean_rate'.light_white,
+          '/s) '.light_black
+        ].join.freeze
+
+        FORMAT_WITHOUT_TOTAL = [
+          ':current'.light_white,
+          '/'.light_black,
+          '???'.light_white,
+          ' ('.light_black,
+          'ELAPSED:'.light_black,
+          ':elapsed'.light_white,
+          ' | ETA:'.light_black,
+          '??:??'.light_white,
+          ' | RATE: '.light_black,
+          ':mean_rate'.light_white,
+          '/s) '.light_black
+        ].join.freeze
+
+        def initialize(title: 'Loading', total:)
+          opts = {
+            clear: true,
+            complete: '▓'.light_blue,
+            incomplete: '░'.blue,
+            frequency: 10
+          }
+
+          if total
+            opts[:total] = total
+            format_str = "#{title} #{FORMAT_WITH_TOTAL}"
+            @pbar = TTY::ProgressBar.new(FORMAT_WITH_TOTAL, opts)
+          else
+            format_str = "#{title} #{FORMAT_WITHOUT_TOTAL}"
+            opts[:no_width] = true
+          end
+
+          @pbar = TTY::ProgressBar.new(format_str, opts)
+
+          @pbar.resize
+        end
+
+        def increment
+          @pbar.advance(1)
+        end
+
+        def log(message)
+          @pbar.log message.inspect
+        end
+
+        def finish
+          @pbar.finish
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
-  module Etl
-    VERSION = "0.1.1"
+  module ETL
+    VERSION = "0.2.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Louis
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-02 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -39,75 +39,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.8.1
 - !ruby/object:Gem::Dependency
-  name: table_print
+  name: tty-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
-  name: ruby-progressbar
+  name: tty-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: runcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -133,7 +161,11 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
 - ".travis.yml"
+- ".yardopts"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -145,19 +177,26 @@ files:
 - chronicle-etl.gemspec
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
-- lib/chronicle/etl/cli.rb
-- lib/chronicle/etl/extractors/csv.rb
+- lib/chronicle/etl/catalog.rb
+- lib/chronicle/etl/cli/connectors.rb
+- lib/chronicle/etl/cli/jobs.rb
+- lib/chronicle/etl/cli/main.rb
+- lib/chronicle/etl/cli/subcommand_base.rb
+- lib/chronicle/etl/config.rb
+- lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
-- lib/chronicle/etl/extractors/stdin.rb
-- lib/chronicle/etl/loaders/csv.rb
+- lib/chronicle/etl/extractors/file_extractor.rb
+- lib/chronicle/etl/extractors/stdin_extractor.rb
+- lib/chronicle/etl/loaders/csv_loader.rb
 - lib/chronicle/etl/loaders/loader.rb
-- lib/chronicle/etl/loaders/stdout.rb
-- lib/chronicle/etl/loaders/table.rb
+- lib/chronicle/etl/loaders/rest_loader.rb
+- lib/chronicle/etl/loaders/stdout_loader.rb
+- lib/chronicle/etl/loaders/table_loader.rb
 - lib/chronicle/etl/runner.rb
-- lib/chronicle/etl/transformers/json.rb
-- lib/chronicle/etl/transformers/null.rb
+- lib/chronicle/etl/transformers/json_transformer.rb
+- lib/chronicle/etl/transformers/null_transformer.rb
 - lib/chronicle/etl/transformers/transformer.rb
-- lib/chronicle/etl/utils/progress_bar_wrapper.rb
+- lib/chronicle/etl/utils/progress_bar.rb
 - lib/chronicle/etl/version.rb
 homepage: https://github.com/chronicle-app
 licenses:
@@ -165,8 +204,8 @@ licenses:
 metadata:
   homepage_uri: https://github.com/chronicle-app
   source_code_uri: https://github.com/chronicle-app/chronicle-etl
-  changelog_uri: https://github.com/chronicle-app/chronicle-etl
-post_install_message: 
+  changelog_uri: https://github.com/chronicle-app/chronicle-etl/blob/master/CHANGELOG.md
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -181,8 +220,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: ETL tool for personal data
 test_files: []

data/lib/chronicle/etl/cli.rb

@@ -1,38 +0,0 @@
-require 'thor'
-require 'chronicle/etl'
-
-module Chronicle
-  module Etl
-    class CLI < Thor
-      default_task :job
-
-      desc 'job', 'Runs an ETL job'
-      method_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
-      method_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
-      method_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
-      method_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
-      method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
-      method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-      method_option :job, aliases: '-j', desc: 'Job configuration file'
-      def job
-        runner_options = {
-          extractor: {
-            name: options[:extractor],
-            options: options[:'extractor-opts']
-          },
-          transformer: {
-            name: options[:transformer],
-            options: options[:'transformer-opts']
-          },
-          loader: {
-            name: options[:loader],
-            options: options[:'loader-opts']
-          }
-        }
-
-        runner = Runner.new(runner_options)
-        runner.run!
-      end
-    end
-  end
-end

data/lib/chronicle/etl/extractors/stdin.rb

@@ -1,13 +0,0 @@
-module Chronicle
-  module Etl
-    module Extractors
-      class Stdin < Chronicle::Etl::Extractors::Extractor
-        def extract
-          $stdin.read.each_line do |line|
-            yield line
-          end
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/loaders/csv.rb

@@ -1,31 +0,0 @@
-require 'csv'
-
-module Chronicle
-  module Etl
-    module Loaders
-      class Csv < Chronicle::Etl::Loaders::Loader
-        def initialize(options={})
-          super(options)
-          @rows = []
-        end
-
-        def load(result)
-          if (result.values)
-            @rows << result.values
-          else
-            @rows << result
-          end
-        end
-
-        def finish
-          z = $stdout
-          CSV(z) do |csv|
-            @rows.each do |row|
-              csv << row
-            end
-          end
-        end
-      end
-    end
-  end
-end