chronicle-etl 0.1.2 → 0.2.2

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,25 +1,32 @@
 module Chronicle
-  module Etl
-    module Loaders
-      class Loader
-        def initialize(options = {})
-          @options = options
-        end
+  module ETL
+    # Abstract class representing a Loader for an ETL job
+    class Loader
+      extend Chronicle::ETL::Catalog
 
-        def start; end
-
-        def first_load; end
+      # Construct a new instance of this loader. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Loader
+      def initialize(options = {})
+        @options = options
+      end
 
-        def load
-          raise NotImplementedError
-        end
+      # Called once before processing records
+      def start; end
 
-        def finish; end
+      # Load a single record
+      def load
+        raise NotImplementedError
       end
+
+      # Called once there are no more records to process
+      def finish; end
     end
   end
 end
 
-require_relative 'stdout'
-require_relative 'csv'
-require_relative 'table'
+require_relative 'csv_loader'
+require_relative 'rest_loader'
+require_relative 'stdout_loader'
+require_relative 'table_loader'

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -0,0 +1,30 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Chronicle
+  module ETL
+    class RestLoader < Chronicle::ETL::Loader
+      def initialize(options={})
+        super(options)
+      end
+
+      def load(result)
+        uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
+
+        header = {
+          "Authorization" => "Bearer #{@options[:access_token]}",
+          "Content-Type": 'application/json'
+        }
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        request = Net::HTTP::Post.new(uri.request_uri, header)
+
+        obj = {data: result} unless result[:data]
+        request.body = obj.to_json
+
+        response = http.request(request)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -0,0 +1,9 @@
+module Chronicle
+  module ETL
+    class StdoutLoader < Chronicle::ETL::Loader
+      def load(result)
+        puts result.inspect
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -0,0 +1,21 @@
+require 'tty/table'
+
+module Chronicle
+  module ETL
+    class TableLoader < Chronicle::ETL::Loader
+      def initialize(options)
+        super(options)
+      end
+
+      def load(result)
+        @table ||= TTY::Table.new(header: result.keys)
+        values = result.values.map{|x| x.to_s[0..30]}
+        @table << values
+      end
+
+      def finish
+        puts @table.render(:ascii, padding: [0, 1])
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,57 +1,46 @@
-class Chronicle::Etl::Runner
-  BUILTIN = {
-    extractor: ['stdin', 'json', 'csv'],
-    transformer: ['null'],
-    loader: ['stdout', 'csv', 'table']
-  }.freeze
-
-  def initialize(options)
-    @options = options
+require 'colorize'
 
-    instantiate_etl_classes
+class Chronicle::ETL::Runner
+  def initialize(options = {})
+    @options = options
   end
 
   def run!
-    progress_bar = Chronicle::Etl::Utils::ProgressBarWrapper.new(@extractor.results_count)
-    @loader.start
+    extractor = instantiate_klass(:extractor)
+    loader = instantiate_klass(:loader)
 
-    @extractor.extract do |result, i|
-      @loader.first_load(result) if i == 0
+    total = extractor.results_count
+    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
 
-      transformed_data = @transformer.transform(result)
-      @loader.load(transformed_data)
+    loader.start
 
+    extractor.extract do |data, metadata|
+      transformer = instantiate_klass(:transformer, data)
+      transformed_data = transformer.transform
+
+      loader.load(transformed_data)
       progress_bar.increment
     end
 
     progress_bar.finish
-    @loader.finish
+    loader.finish
   end
 
   private
 
-  def instantiate_etl_classes
-    @extractor = load_etl_class(:extractor, @options[:extractor][:name]).new(@options[:extractor][:options])
-    @transformer = load_etl_class(:transformer, @options[:transformer][:name]).new(@options[:transformer][:options])
-    @loader = load_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
+  def instantiate_klass(phase, *args)
+    klass = load_etl_class(phase, @options[phase][:name])
+    klass.new(@options[phase][:options], *args)
   end
 
-  def load_etl_class(phase, name)
-    if BUILTIN[phase].include? name
-      klass_name = "Chronicle::Etl::#{phase.to_s.capitalize}s::#{name.capitalize}"
-    else
-      # TODO: come up with syntax for specifying a particular extractor in a provider library
-      # provider, extractor = name.split(":")
-      provider = name
-      begin
-        require "chronicle/#{provider}"
-      rescue LoadError => e
-        warn("Error loading #{phase} '#{provider}'")
-        warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
-        exit(false)
-      end
-      klass_name = "Chronicle::#{name.capitalize}::Extractor"
-    end
-    Object.const_get(klass_name)
+  def load_etl_class(phase, identifier)
+    Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
+  rescue Chronicle::ETL::ProviderNotAvailableError => e
+    warn(e.message.red)
+    warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
+    exit(false)
+  rescue Chronicle::ETL::ConnectorNotAvailableError => e
+    warn(e.message.red)
+    exit(false)
   end
 end

data/lib/chronicle/etl/transformers/json_transformer.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+module Chronicle
+  module ETL
+    class JsonTransformer < Chronicle::ETL::Transformer
+      def transform data
+        return JSON.parse(data)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -0,0 +1,10 @@
+module Chronicle
+  module ETL
+    class NullTransformer < Chronicle::ETL::Transformer
+      def transform
+        return @data
+      end
+    end
+
+  end
+end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -1,18 +1,35 @@
 module Chronicle
-  module Etl
-    module Transformers
-      class Transformer
-        def initialize(options = {})
-          @options = options
-        end
+  module ETL
+    # Abstract class representing an Transformer for an ETL job
+    class Transformer
+      extend Chronicle::ETL::Catalog
 
-        def transform data
-          raise NotImplementedError
-        end
+      # Construct a new instance of this transformer. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Transformer
+      def initialize(options = {}, data)
+        @options = options
+        @data = data
       end
+
+      # The main entrypoint for transforming a record. Called by a Runner on each extracted record
+      def transform
+        raise NotImplementedError
+      end
+
+      # The domain or provider-specific id of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def id; end
+
+      # The domain or provider-specific timestamp of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def timestamp; end
     end
   end
 end
 
-require_relative 'null'
-require_relative 'json'
+require_relative 'json_transformer'
+require_relative 'null_transformer'

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -0,0 +1,76 @@
+require 'tty/progressbar'
+require 'colorize'
+
+module Chronicle
+  module ETL
+    module Utils
+
+      class ProgressBar
+        FORMAT_WITH_TOTAL = [
+          ':bar ',
+          ':percent'.light_white,
+          ' | '.light_black,
+          ':current'.light_white,
+          '/'.light_black,
+          ':total'.light_white,
+          ' ('.light_black,
+          'ELAPSED:'.light_black,
+          ':elapsed'.light_white,
+          ' | ETA:'.light_black,
+          ':eta'.light_white,
+          ' | RATE: '.light_black,
+          ':mean_rate'.light_white,
+          '/s) '.light_black
+        ].join.freeze
+
+        FORMAT_WITHOUT_TOTAL = [
+          ':current'.light_white,
+          '/'.light_black,
+          '???'.light_white,
+          ' ('.light_black,
+          'ELAPSED:'.light_black,
+          ':elapsed'.light_white,
+          ' | ETA:'.light_black,
+          '??:??'.light_white,
+          ' | RATE: '.light_black,
+          ':mean_rate'.light_white,
+          '/s) '.light_black
+        ].join.freeze
+
+        def initialize(title: 'Loading', total:)
+          opts = {
+            clear: true,
+            complete: '▓'.light_blue,
+            incomplete: '░'.blue,
+            frequency: 10
+          }
+
+          if total
+            opts[:total] = total
+            format_str = "#{title} #{FORMAT_WITH_TOTAL}"
+            @pbar = TTY::ProgressBar.new(FORMAT_WITH_TOTAL, opts)
+          else
+            format_str = "#{title} #{FORMAT_WITHOUT_TOTAL}"
+            opts[:no_width] = true
+          end
+
+          @pbar = TTY::ProgressBar.new(format_str, opts)
+
+          @pbar.resize
+        end
+
+        def increment
+          @pbar.advance(1)
+        end
+
+        def log(message)
+          @pbar.log message.inspect
+        end
+
+        def finish
+          @pbar.finish
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
-  module Etl
-    VERSION = "0.1.2"
+  module ETL
+    VERSION = "0.2.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.2
 platform: ruby
 authors:
 - Andrew Louis
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-02 00:00:00.000000000 Z
+date: 2020-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -39,75 +39,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.8.1
 - !ruby/object:Gem::Dependency
-  name: table_print
+  name: tty-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
-  name: ruby-progressbar
+  name: tty-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: runcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -133,7 +161,10 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
 - ".travis.yml"
+- ".yardopts"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -146,19 +177,27 @@ files:
 - chronicle-etl.gemspec
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
-- lib/chronicle/etl/cli.rb
-- lib/chronicle/etl/extractors/csv.rb
+- lib/chronicle/etl/catalog.rb
+- lib/chronicle/etl/cli/connectors.rb
+- lib/chronicle/etl/cli/jobs.rb
+- lib/chronicle/etl/cli/main.rb
+- lib/chronicle/etl/cli/subcommand_base.rb
+- lib/chronicle/etl/config.rb
+- lib/chronicle/etl/exceptions.rb
+- lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
-- lib/chronicle/etl/extractors/stdin.rb
-- lib/chronicle/etl/loaders/csv.rb
+- lib/chronicle/etl/extractors/file_extractor.rb
+- lib/chronicle/etl/extractors/stdin_extractor.rb
+- lib/chronicle/etl/loaders/csv_loader.rb
 - lib/chronicle/etl/loaders/loader.rb
-- lib/chronicle/etl/loaders/stdout.rb
-- lib/chronicle/etl/loaders/table.rb
+- lib/chronicle/etl/loaders/rest_loader.rb
+- lib/chronicle/etl/loaders/stdout_loader.rb
+- lib/chronicle/etl/loaders/table_loader.rb
 - lib/chronicle/etl/runner.rb
-- lib/chronicle/etl/transformers/json.rb
-- lib/chronicle/etl/transformers/null.rb
+- lib/chronicle/etl/transformers/json_transformer.rb
+- lib/chronicle/etl/transformers/null_transformer.rb
 - lib/chronicle/etl/transformers/transformer.rb
-- lib/chronicle/etl/utils/progress_bar_wrapper.rb
+- lib/chronicle/etl/utils/progress_bar.rb
 - lib/chronicle/etl/version.rb
 homepage: https://github.com/chronicle-app
 licenses:
@@ -167,7 +206,7 @@ metadata:
   homepage_uri: https://github.com/chronicle-app
   source_code_uri: https://github.com/chronicle-app/chronicle-etl
   changelog_uri: https://github.com/chronicle-app/chronicle-etl/blob/master/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -182,8 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: ETL tool for personal data
 test_files: []