chronicle-etl 0.5.2 → 0.5.5

data/lib/chronicle/etl/runner.rb

@@ -1,5 +1,6 @@
 require 'colorize'
 require 'chronic_duration'
+require "tty-spinner"
 
 class Chronicle::ETL::Runner
   def initialize(job)
@@ -8,76 +9,101 @@ class Chronicle::ETL::Runner
   end
 
   def run!
+    begin_job
     validate_job
     instantiate_connectors
     prepare_job
     prepare_ui
     run_extraction
+  rescue Chronicle::ETL::ExtractionError => e
+    @job_logger&.error
+    raise(Chronicle::ETL::RunnerError, "Extraction failed. #{e.message}")
+  rescue Interrupt
+    @job_logger&.error
+    raise(Chronicle::ETL::RunInterruptedError, "Job interrupted.")
+  rescue StandardError => e
+    # Just throwing this in here until we have better exception handling in
+    # loaders, etc
+    @job_logger&.error
+    raise(Chronicle::ETL::RunnerError, "Error running job. #{e.message}")
+  ensure
     finish_job
   end
 
   private
 
+  def begin_job
+    Chronicle::ETL::Logger.info(tty_log_job_initialize)
+    @initialization_spinner = TTY::Spinner.new(":spinner :title", format: :dots_2)
+  end
+
   def validate_job
+    @initialization_spinner.update(title: "Validating job")
     @job.job_definition.validate!
   end
 
   def instantiate_connectors
+    @initialization_spinner.update(title: "Initializing connectors")
     @extractor = @job.instantiate_extractor
     @loader = @job.instantiate_loader
   end
 
   def prepare_job
-    Chronicle::ETL::Logger.info(tty_log_job_start)
+    @initialization_spinner.update(title: "Preparing job")
     @job_logger.start
     @loader.start
+
+    @initialization_spinner.update(title: "Preparing extraction")
+    @initialization_spinner.auto_spin
     @extractor.prepare
+    @initialization_spinner.success("(#{'successful'.green})")
+    Chronicle::ETL::Logger.info("\n")
   end
 
   def prepare_ui
     total = @extractor.results_count
     @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
-    Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
+    Chronicle::ETL::Logger.attach_to_ui(@progress_bar)
   end
 
-  # TODO: refactor this further
   def run_extraction
     @extractor.extract do |extraction|
-      unless extraction.is_a?(Chronicle::ETL::Extraction)
-        raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
-      end
-
-      transformer = @job.instantiate_transformer(extraction)
-      record = transformer.transform
-
-      Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
-      @job_logger.log_transformation(transformer)
-
-      @loader.load(record) unless @job.dry_run?
-    rescue Chronicle::ETL::TransformationError => e
-      Chronicle::ETL::Logger.error(tty_log_transformation_failure(e, transformer))
-    ensure
+      process_extraction(extraction)
       @progress_bar.increment
     end
 
     @progress_bar.finish
+
+    # This is typically a slow method (writing to stdout, writing a big file, etc)
+    # TODO: consider adding a spinner?
     @loader.finish
     @job_logger.finish
-  rescue Interrupt
-    Chronicle::ETL::Logger.error("\n#{'Job interrupted'.red}")
-    @job_logger.error
-  rescue StandardError => e
-    raise e
+  end
+
+  def process_extraction(extraction)
+    # For each extraction from our extractor, we create a new tarnsformer
+    transformer = @job.instantiate_transformer(extraction)
+
+    # And then transform that record, logging it if we're in debug log level
+    record = transformer.transform
+    Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
+    @job_logger.log_transformation(transformer)
+
+    # Then send the results to the loader
+    @loader.load(record) unless @job.dry_run?
+  rescue Chronicle::ETL::TransformationError => e
+    # TODO: have an option to cancel job if we encounter an error
+    Chronicle::ETL::Logger.error(tty_log_transformation_failure(e, transformer))
   end
 
   def finish_job
     @job_logger.save
     @progress_bar&.finish
-    Chronicle::ETL::Logger.detach_from_progress_bar
+    Chronicle::ETL::Logger.detach_from_ui
     Chronicle::ETL::Logger.info(tty_log_completion)
   end
 
-  def tty_log_job_start
+  def tty_log_job_initialize
     output = "Beginning job "
     output += "'#{@job.name}'".bold if @job.name
     output
@@ -95,8 +121,9 @@ class Chronicle::ETL::Runner
 
   def tty_log_completion
     status = @job_logger.success ? 'Success' : 'Failed'
-    output = "\nCompleted job "
-    output += "'#{@job.name}'".bold if @job.name
+    job_completion = @job_logger.success ? 'Completed' : 'Partially completed'
+    output = "\n#{job_completion} job"
+    output += " '#{@job.name}'".bold if @job.name
     output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
     output += "\n  Status:\t".light_black + status
     output += "\n  Completed:\t".light_black + "#{@job_logger.job_log.num_records_processed}"

data/lib/chronicle/etl/secrets.rb

@@ -1,9 +1,16 @@
+require "active_support/core_ext/hash/keys"
+
 module Chronicle
   module ETL
     # Secret management module
     module Secrets
       module_function
 
+      # Whether a given namespace exists
+      def exists?(namespace)
+        Chronicle::ETL::Config.exists?("secrets", namespace)
+      end
+
       # Save a setting to a namespaced config file
       def set(namespace, key, value)
         config = read(namespace)
@@ -11,6 +18,13 @@ module Chronicle
         write(namespace, config)
       end
 
+      # Save a hash to a secrets namespace
+      def set_all(namespace, secrets)
+        config = read(namespace)
+        config = config.merge(secrets.deep_stringify_keys)
+        write(namespace, config)
+      end
+
       # Remove a setting from a namespaced config file
       def unset(namespace, key)
         config = read(namespace)
@@ -42,7 +56,7 @@ module Chronicle
         data = {
           secrets: (secrets || {}).transform_keys(&:to_s),
           chronicle_etl_version: Chronicle::ETL::VERSION
-        }.transform_keys(&:to_s) # Should I implement deeply_transform_keys...?
+        }
         Chronicle::ETL::Config.write("secrets", namespace, data)
       end
 

data/lib/chronicle/etl/transformers/transformer.rb

@@ -10,6 +10,10 @@ module Chronicle
       # options::
       #   Options for configuring this Transformer
       def initialize(extraction, options = {})
+        unless extraction.is_a?(Chronicle::ETL::Extraction)
+          raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
+        end
+
         @extraction = extraction
         apply_options(options)
       end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.5.2"
+    VERSION = "0.5.5"
   end
 end

data/lib/chronicle/etl.rb

@@ -1,4 +1,5 @@
 require_relative 'etl/registry/registry'
+require_relative 'etl/authorizer'
 require_relative 'etl/config'
 require_relative 'etl/configurable'
 require_relative 'etl/exceptions'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.5
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-30 00:00:00.000000000 Z
+date: 2022-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: gems
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: launchy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: marcel
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: omniauth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.35'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -235,33 +291,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.7.3
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.7.3
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: fakefs
+  name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.7.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.7.3
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -332,6 +388,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -372,7 +456,10 @@ files:
 - chronicle-etl.gemspec
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
+- lib/chronicle/etl/authorization_server.rb
+- lib/chronicle/etl/authorizer.rb
 - lib/chronicle/etl/cli.rb
+- lib/chronicle/etl/cli/authorizations.rb
 - lib/chronicle/etl/cli/cli_base.rb
 - lib/chronicle/etl/cli/connectors.rb
 - lib/chronicle/etl/cli/jobs.rb
@@ -407,8 +494,11 @@ files:
 - lib/chronicle/etl/models/base.rb
 - lib/chronicle/etl/models/entity.rb
 - lib/chronicle/etl/models/raw.rb
+- lib/chronicle/etl/oauth_authorizer.rb
 - lib/chronicle/etl/registry/connector_registration.rb
-- lib/chronicle/etl/registry/plugin_registry.rb
+- lib/chronicle/etl/registry/connectors.rb
+- lib/chronicle/etl/registry/plugin_registration.rb
+- lib/chronicle/etl/registry/plugins.rb
 - lib/chronicle/etl/registry/registry.rb
 - lib/chronicle/etl/registry/self_registering.rb
 - lib/chronicle/etl/runner.rb

data/lib/chronicle/etl/registry/plugin_registry.rb

@@ -1,84 +0,0 @@
-require 'rubygems'
-require 'rubygems/command'
-require 'rubygems/commands/install_command'
-require 'rubygems/uninstaller'
-
-module Chronicle
-  module ETL
-    module Registry
-      # Responsible for managing plugins available to chronicle-etl
-      #
-      # @todo Better validation for whether a gem is actually a plugin
-      # @todo Add ways to load a plugin that don't require a gem on rubygems.org
-      module PluginRegistry
-        # Does this plugin exist?
-        def self.exists?(name)
-          # TODO: implement this. Could query rubygems.org or use a hardcoded
-          # list somewhere
-          true
-        end
-
-        # All versions of all plugins currently installed
-        def self.all_installed
-          # TODO: add check for chronicle-etl dependency
-          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" }
-        end
-
-        # Latest version of each installed plugin
-        def self.all_installed_latest
-          all_installed.group_by(&:name)
-            .transform_values { |versions| versions.sort_by(&:version).reverse.first }
-            .values
-        end
-
-        # Check whether a given plugin is installed
-        def self.installed?(name)
-          gem_name = "chronicle-#{name}"
-          all_installed.map(&:name).include?(gem_name)
-        end
-
-        # Activate a plugin with given name by `require`ing it
-        def self.activate(name)
-          # By default, activates the latest available version of a gem
-          # so don't have to run Kernel#gem separately
-          require "chronicle/#{name}"
-        rescue Gem::ConflictError => e
-          # TODO: figure out if there's more we can do here
-          raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
-        rescue StandardError, LoadError => e
-          # StandardError to catch random non-loading problems that might occur
-          # when requiring the plugin (eg class macro invoked the wrong way)
-          # TODO: decide if this should be separated
-          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded"
-        end
-
-        # Install a plugin to local gems
-        def self.install(name)
-          return if installed?(name)
-
-          gem_name = "chronicle-#{name}"
-          raise(Chronicle::ETL::PluginNotAvailableError.new(gem_name), "Plugin #{name} doesn't exist") unless exists?(gem_name)
-
-          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
-          Gem.install(gem_name)
-
-          activate(name)
-        rescue Gem::UnsatisfiableDependencyError
-          # TODO: we need to catch a lot more than this here
-          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} could not be installed."
-        end
-
-        # Uninstall a plugin
-        def self.uninstall(name)
-          gem_name = "chronicle-#{name}"
-          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
-          uninstaller = Gem::Uninstaller.new(gem_name)
-          uninstaller.uninstall
-        rescue Gem::InstallError
-          # TODO: strengthen this exception handling
-          raise(Chronicle::ETL::PluginError.new(name), "Plugin #{name} wasn't uninstalled")
-        end
-      end
-    end
-  end
-end