chronicle-etl 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d9be4b073385d9758a8b709bb6726d8dae648b8c4dbef474343840c011d1178
-  data.tar.gz: 0d95395407d37f7e322287a0920bba60e1b4f81eb8649190d021e13580604a2d
+  metadata.gz: e1c08bc4f71c807525090abbf1701be19ab72cce08a99cc3bbec9b0db7150a02
+  data.tar.gz: 172a5d7e7ba7a9424ef7b5ab4da2b8c44defdb4e0a34c833248ff1b63f40407e
 SHA512:
-  metadata.gz: 386c96518aa2d2810ae2a93bbe3af5bb08e26e132608b4e6ed8a278da076783e453854a2120c5016b6d02cd5dea406146d10ef3c7c1e77d854acd8ff2608eaf7
-  data.tar.gz: eb14402be5d6db44a6f06e6ec930acc5103b36e5c2e5a13e89137c9ee45f5f11c1e9e6ab13d4d44e6ee06bd9b02309e02ac33e81256645ef71d5b431c97eb199
+  metadata.gz: 0f671c00928b15f9c0f6fa159ac106ff9c4f65a8bd16048e5d0cab82d680945317f7680e7796e98c665bb5cc757e0657f1a36d773d89e3e1587d9eebc12abdd8
+  data.tar.gz: 449d1368e0054f39006c7903218300b9b97ca839d6eff43b6b7bd659e5146d443a31c53325c4769ae7a56db9d42417020ccde17362ae024c01aca2ed63029044

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.2.1)
+    chronicle-etl (0.2.2)
       colorize (~> 0.8.1)
       thor (~> 0.20)
       tty-progressbar (~> 0.17)

data/lib/chronicle/etl.rb

@@ -1,5 +1,6 @@
 require_relative 'etl/catalog'
 require_relative 'etl/config'
+require_relative 'etl/exceptions'
 require_relative 'etl/extractors/extractor'
 require_relative 'etl/loaders/loader'
 require_relative 'etl/runner'

data/lib/chronicle/etl/catalog.rb

@@ -4,6 +4,11 @@ module Chronicle
     # Loader connector classes are available to chronicle-etl
     module Catalog
       PLUGINS = ['email', 'bash']
+      BUILTIN = {
+        extractor: ['stdin', 'json', 'csv', 'file'],
+        transformer: ['null'],
+        loader: ['stdout', 'csv', 'table', 'rest']
+      }.freeze
 
       # Return which ETL connectors are available, both built in and externally-defined
       def self.available_classes
@@ -15,7 +20,7 @@ module Chronicle
         PLUGINS.each do |plugin|
           require "chronicle/#{plugin}"
         rescue LoadError
-          # this will happen if the gem isn't available globally 
+          # this will happen if the gem isn't available globally
         end
 
         parent_klasses = [
@@ -38,6 +43,18 @@ module Chronicle
         end
       end
 
+      # For a given connector identifier, return the class (either builtin, or from a 
+      # external chronicle gem)
+      def self.identifier_to_klass(identifier:, phase:)
+        if BUILTIN[phase].include? identifier
+          load_builtin_klass(name: identifier, phase: phase)
+        else
+          provider, name = identifier.split(':')
+          name ||= ''
+          load_provider_klass(provider: provider, name: name, phase: phase)
+        end
+      end
+
       # Returns whether a class is an Extractor, Transformer, or Loader
       def phase
         ancestors = self.ancestors
@@ -57,6 +74,29 @@ module Chronicle
       def built_in?
         to_s.include? 'Chronicle::ETL'
       end
+
+      private
+
+      def self.load_builtin_klass(name:, phase:)
+        klass_str = "Chronicle::ETL::#{name.capitalize}#{phase.capitalize}"
+        begin
+          Object.const_get(klass_str)
+        rescue NameError => e
+          raise ConnectorNotAvailableError.new("Connector not found", name: name)
+        end
+      end
+
+      def self.load_provider_klass(name: '', phase:, provider:)
+        begin
+          require "chronicle/#{provider}"
+          klass_str = "Chronicle::#{provider.capitalize}::#{name.capitalize}#{phase.capitalize}"
+          Object.const_get(klass_str)
+        rescue LoadError => e
+          raise ProviderNotAvailableError.new("Provider '#{provider.capitalize}' could not be loaded", provider: provider)
+        rescue NameError => e
+          raise ProviderConnectorNotAvailableError.new("Connector '#{name}' in '#{provider}' could not be found", provider: provider, name: name)
+        end
+      end
     end
   end
 end

data/lib/chronicle/etl/cli/jobs.rb

@@ -1,5 +1,4 @@
 require 'pp'
-require 'pry'
 
 module Chronicle
   module ETL

data/lib/chronicle/etl/exceptions.rb

@@ -0,0 +1,17 @@
+module Chronicle
+  module ETL
+    class Error < StandardError; end;
+
+    class ConnectorNotAvailableError < Error
+      def initialize(message, provider: nil, name: nil)
+        super(message)
+        @provider = provider
+        @name = name
+      end
+      attr_reader :name, :provider
+    end
+
+    class ProviderNotAvailableError < ConnectorNotAvailableError; end
+    class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,59 +1,46 @@
-class Chronicle::ETL::Runner
-  BUILTIN = {
-    extractor: ['stdin', 'json', 'csv', 'file'],
-    transformer: ['null'],
-    loader: ['stdout', 'csv', 'table']
-  }.freeze
+require 'colorize'
 
-  def initialize(options)
+class Chronicle::ETL::Runner
+  def initialize(options = {})
     @options = options
-
-    instantiate_etl_classes
   end
 
   def run!
-    total = @extractor.results_count
+    extractor = instantiate_klass(:extractor)
+    loader = instantiate_klass(:loader)
+
+    total = extractor.results_count
     progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
-    count = 0
 
-    @loader.start
+    loader.start
 
-    @extractor.extract do |data, metadata|
-      transformed_data = @transformer.transform(data)
-      @loader.load(transformed_data)
+    extractor.extract do |data, metadata|
+      transformer = instantiate_klass(:transformer, data)
+      transformed_data = transformer.transform
 
+      loader.load(transformed_data)
       progress_bar.increment
-      count += 1
     end
 
     progress_bar.finish
-    @loader.finish
+    loader.finish
   end
 
   private
 
-  def instantiate_etl_classes
-    @extractor = load_etl_class(:extractor, @options[:extractor][:name]).new(@options[:extractor][:options])
-    @transformer = load_etl_class(:transformer, @options[:transformer][:name]).new(@options[:transformer][:options])
-    @loader = load_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
+  def instantiate_klass(phase, *args)
+    klass = load_etl_class(phase, @options[phase][:name])
+    klass.new(@options[phase][:options], *args)
   end
 
-  def load_etl_class(phase, x)
-    if BUILTIN[phase].include? x
-      klass_name = "Chronicle::ETL::#{x.capitalize}#{phase.to_s.capitalize}"
-    else
-      # TODO: come up with syntax for specifying a particular extractor in a provider library
-      provider, name = x.split(":")
-      provider = x unless provider
-      begin
-        require "chronicle/#{provider}"
-      rescue LoadError => e
-        warn("Error loading #{phase} '#{provider}'".red)
-        warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
-        exit(false)
-      end
-      klass_name = "Chronicle::#{provider.capitalize}::#{name&.capitalize}#{phase.capitalize}"
-    end
-    Object.const_get(klass_name)
+  def load_etl_class(phase, identifier)
+    Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
+  rescue Chronicle::ETL::ProviderNotAvailableError => e
+    warn(e.message.red)
+    warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
+    exit(false)
+  rescue Chronicle::ETL::ConnectorNotAvailableError => e
+    warn(e.message.red)
+    exit(false)
   end
 end

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -1,8 +1,8 @@
 module Chronicle
   module ETL
     class NullTransformer < Chronicle::ETL::Transformer
-      def transform data
-        return data
+      def transform
+        return @data
       end
     end
 

data/lib/chronicle/etl/transformers/transformer.rb

@@ -8,12 +8,13 @@ module Chronicle
       # == Paramters:
       # options::
       #   Options for configuring this Transformer
-      def initialize(options = {})
+      def initialize(options = {}, data)
         @options = options
+        @data = data
       end
 
       # The main entrypoint for transforming a record. Called by a Runner on each extracted record
-      def transform data
+      def transform
         raise NotImplementedError
       end
 

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.2.1"
+    VERSION = "0.2.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-27 00:00:00.000000000 Z
+date: 2020-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -183,6 +183,7 @@ files:
 - lib/chronicle/etl/cli/main.rb
 - lib/chronicle/etl/cli/subcommand_base.rb
 - lib/chronicle/etl/config.rb
+- lib/chronicle/etl/exceptions.rb
 - lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
 - lib/chronicle/etl/extractors/file_extractor.rb