chronicle-etl 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f041e90fc6019ecbc2424e8e75e7f21fa5ba715c2cdbde73d61c298e9ca82e07
-  data.tar.gz: 2feda7669f8fefc7ad80fe7918530a86ad2ced431f75e70ad42653c871b67d90
+  metadata.gz: f2b6fdca3723ec52287c070a0dd08d0cfaf825f5e8f46da0d5a34172c0008573
+  data.tar.gz: e15181ba7edc1698404af8ff8c05d5367786ea809360393e825ca5ee5eef6c75
 SHA512:
-  metadata.gz: 4a40c72dcb6514037c6e53214dc0af3bfba20c272b959c3e83496658b4b8dc3f841d7399aa215a5fcc5c5cd494278223666b8b881f645ed2c61b667351ccde94
-  data.tar.gz: 5b5450b76a8c03d7cb8405888b2e059390c1585a66524dd7403b458c828a1936422e03a9654ec6d270b634bf6bfe17d6dafda54e05aca8a8732207366d03ffd2
+  metadata.gz: c5508dcfc5e1367122ebbc191dc60a76cc7b1f088ae9ebd52bae41e07420f54ea7977a35f2b25839933b06d4673e2a120feecbd958efee07e0d313eaa7a5d167
+  data.tar.gz: adcb90549af364189c5ae3b0811c039277aba7dc6fbf2fbd6de8b89c572948d0dc00ba8da59db03252f4770737423e62c3a1175ef018742ef8bd7aee14837f63

data/lib/chronicle/etl/cli/cli_base.rb

@@ -0,0 +1,31 @@
+module Chronicle
+  module ETL
+    module CLI
+      # Base class for CLI commands
+      class CLIBase < ::Thor
+        no_commands do
+          # Shorthand for cli_exit(status: :failure)
+          def cli_fail(message: nil, exception: nil)
+            cli_exit(status: :failure, message: message, exception: exception)
+          end
+
+          # Exit from CLI
+          #
+          # @params status Can be eitiher :success or :failure
+          # @params message to print
+          # @params exception stacktrace if log_level is set to debug
+          def cli_exit(status: :success, message: nil, exception: nil)
+            exit_code = status == :success ? 0 : 1
+            log_level = status == :success ? :info : :fatal
+
+            message = message.red if status != :success
+
+            Chronicle::ETL::Logger.debug(exception.full_message) if exception
+            Chronicle::ETL::Logger.send(log_level, message) if message
+            exit(exit_code)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/connectors.rb

@@ -39,15 +39,13 @@ module Chronicle
         desc "show PHASE IDENTIFIER", "Show information about a connector"
         def show(phase, identifier)
           unless ['extractor', 'transformer', 'loader'].include?(phase)
-            Chronicle::ETL::Logger.fatal("Phase argument must be one of: [extractor, transformer, loader]")
-            exit 1
+            cli_fail(message: "Phase argument must be one of: [extractor, transformer, loader]")
           end
 
           begin
             connector = Chronicle::ETL::Registry.find_by_phase_and_identifier(phase.to_sym, identifier)
-          rescue Chronicle::ETL::ConnectorNotAvailableError, Chronicle::ETL::PluginError
-            Chronicle::ETL::Logger.fatal("Could not find #{phase} #{identifier}")
-            exit 1
+          rescue Chronicle::ETL::ConnectorNotAvailableError, Chronicle::ETL::PluginError => e
+            cli_fail(message: "Could not find #{phase} #{identifier}", exception: e)
           end
 
           puts connector.klass.to_s.bold

data/lib/chronicle/etl/cli/jobs.rb

@@ -45,15 +45,19 @@ module Chronicle
 LONG_DESC
         # Run an ETL job
         def start
-          run_job(options)
-        rescue Chronicle::ETL::JobDefinitionError => e
-          missing_plugins = e.job_definition.errors
-            .select { |error| error.is_a?(Chronicle::ETL::PluginLoadError) }
-            .map(&:name)
-            .uniq
+          job_definition = build_job_definition(options)
+
+          if job_definition.plugins_missing?
+            missing_plugins = job_definition.errors[:plugins]
+              .select { |error| error.is_a?(Chronicle::ETL::PluginLoadError) }
+              .map(&:name)
+              .uniq
+            install_missing_plugins(missing_plugins)
+          end
 
-          install_missing_plugins(missing_plugins)
-          run_job(options)
+          run_job(job_definition)
+        rescue Chronicle::ETL::JobDefinitionError => e
+          cli_fail(message: "Error running job.\n#{job_definition.errors}", exception: e)
         end
 
         desc "create", "Create a job"
@@ -65,8 +69,7 @@ LONG_DESC
           path = File.join('chronicle', 'etl', 'jobs', options[:name])
           Chronicle::ETL::Config.write(path, job_definition.definition)
         rescue Chronicle::ETL::JobDefinitionError => e
-          Chronicle::ETL::Logger.debug(e.full_message)
-          Chronicle::ETL::Logger.fatal("Job definition error".red)
+          cli_fail(message: "Job definition error", exception: e)
         end
 
         desc "show", "Show details about a job"
@@ -76,9 +79,7 @@ LONG_DESC
           job_definition.validate!
           puts Chronicle::ETL::Job.new(job_definition)
         rescue Chronicle::ETL::JobDefinitionError => e
-          Chronicle::ETL::Logger.debug(e.full_message)
-          Chronicle::ETL::Logger.fatal("Job definition error".red)
-          exit 1
+          cli_fail(message: "Job definition error", exception: e)
         end
 
         desc "list", "List all available jobs"
@@ -102,15 +103,12 @@ LONG_DESC
           table = TTY::Table.new(headers, job_details)
           puts table.render(indent: 0, padding: [0, 2])
         rescue Chronicle::ETL::ConfigError => e
-          Chronicle::ETL::Logger.debug(e.full_message)
-          Chronicle::ETL::Logger.fatal("Error reading config. #{e.message}".red)
-          exit 1
+          cli_fail(message: "Config error. #{e.message}", exception: e)
         end
 
         private
 
-        def run_job(options)
-          job_definition = build_job_definition(options)
+        def run_job(job_definition)
           job = Chronicle::ETL::Job.new(job_definition)
           runner = Chronicle::ETL::Runner.new(job)
           runner.run!
@@ -123,18 +121,10 @@ LONG_DESC
           message += "Do you want to install "
           message += missing_plugins.map { |name| "chronicle-#{name}".bold}.join(", ")
           message += " and start the job?"
-          install = prompt.yes?(message)
-          return unless install
+          will_install = prompt.yes?(message)
+          cli_fail(message: "Must install #{missing_plugins.join(", ")} plugin to run job") unless will_install
 
-          spinner = TTY::Spinner.new("[:spinner] Installing plugins...", format: :dots_2)
-          spinner.auto_spin
-          missing_plugins.each do |plugin|
-            Chronicle::ETL::Registry::PluginRegistry.install(plugin)
-          end
-          spinner.success("(#{'successful'.green})")
-        rescue Chronicle::ETL::PluginNotAvailableError => e
-          spinner.error("Error".red)
-          Chronicle::ETL::Logger.fatal("Plugin '#{e.name}' could not be installed".red)
+          Chronicle::ETL::CLI::Plugins.new.install(*missing_plugins)
         end
 
         # Create job definition by reading config file and then overwriting with flag options

data/lib/chronicle/etl/cli/main.rb

@@ -4,7 +4,7 @@ module Chronicle
   module ETL
     module CLI
       # Main entrypoint for CLI app
-      class Main < ::Thor
+      class Main < Chronicle::ETL::CLI::CLIBase
         class_before :set_log_level
         class_before :set_color_output
 
@@ -92,6 +92,9 @@ module Chronicle
         end
 
         no_commands do
+          def testb
+            puts "hi"
+          end
           def set_color_output
             String.disable_colorization true if options[:'no-color'] || ENV['NO_COLOR']
           end

data/lib/chronicle/etl/cli/plugins.rb

@@ -3,7 +3,6 @@
 require "tty-prompt"
 require "tty-spinner"
 
-
 module Chronicle
   module ETL
     module CLI
@@ -13,16 +12,19 @@ module Chronicle
         namespace :plugins
 
         desc "install", "Install a plugin"
-        def install(name)
-          spinner = TTY::Spinner.new("[:spinner] Installing plugin #{name}...", format: :dots_2)
+        def install(*plugins)
+          cli_fail(message: "Please specify a plugin to install") unless plugins.any?
+
+          spinner = TTY::Spinner.new("[:spinner] Installing #{plugins.join(", ")}...", format: :dots_2)
           spinner.auto_spin
-          Chronicle::ETL::Registry::PluginRegistry.install(name)
+          plugins.each do |plugin|
+            spinner.update(title: "Installing #{plugin}")
+            Chronicle::ETL::Registry::PluginRegistry.install(plugin)
+          rescue Chronicle::ETL::PluginError => e
+            spinner.error("Error".red)
+            cli_fail(message: "Plugin '#{plugin}' could not be installed", exception: e)
+          end
           spinner.success("(#{'successful'.green})")
-        rescue Chronicle::ETL::PluginError => e
-          spinner.error("Error".red)
-          Chronicle::ETL::Logger.debug(e.full_message)
-          Chronicle::ETL::Logger.fatal("Plugin '#{name}' could not be installed".red)
-          exit 1
         end
 
         desc "uninstall", "Unintall a plugin"
@@ -33,9 +35,7 @@ module Chronicle
           spinner.success("(#{'successful'.green})")
         rescue Chronicle::ETL::PluginError => e
           spinner.error("Error".red)
-          Chronicle::ETL::Logger.debug(e.full_message)
-          Chronicle::ETL::Logger.fatal("Plugin '#{name}' could not be uninstalled (was it installed?)".red)
-          exit 1
+          cli_fail(message: "Plugin '#{name}' could not be uninstalled (was it installed?)", exception: e)
         end
 
         desc "list", "Lists available plugins"

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -2,7 +2,7 @@ module Chronicle
   module ETL
     module CLI
       # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
-      class SubcommandBase < ::Thor
+      class SubcommandBase < Chronicle::ETL::CLI::CLIBase
         # Print usage instructions for a subcommand
         def self.help(shell, subcommand = false)
           list = printable_commands(true, subcommand)

data/lib/chronicle/etl/cli.rb

@@ -2,6 +2,7 @@ require 'thor'
 require 'thor/hollaback'
 require 'chronicle/etl'
 
+require 'chronicle/etl/cli/cli_base'
 require 'chronicle/etl/cli/subcommand_base'
 require 'chronicle/etl/cli/connectors'
 require 'chronicle/etl/cli/jobs'

data/lib/chronicle/etl/exceptions.rb

@@ -23,6 +23,7 @@ module Chronicle
       end
     end
 
+    class PluginConflictError < PluginError; end
     class PluginNotAvailableError < PluginError; end
     class PluginLoadError < PluginError; end
 

data/lib/chronicle/etl/job_definition.rb

@@ -26,20 +26,32 @@ module Chronicle
         @definition = SKELETON_DEFINITION
       end
 
+      def valid?
+        validate
+        @errors.empty?
+      end
+
       def validate
-        @errors = []
+        @errors = {}
 
         Chronicle::ETL::Registry::PHASES.each do |phase|
           __send__("#{phase}_klass".to_sym)
         rescue Chronicle::ETL::PluginError => e
-          @errors << e
+          @errors[:plugins] ||= []
+          @errors[:plugins] << e
         end
+      end
 
-        @errors.empty?
+      def plugins_missing?
+        validate
+
+        @errors[:plugins] || []
+          .filter { |e| e.instance_of?(Chronicle::ETL::PluginLoadError) }
+          .any?
       end
 
       def validate!
-        raise(Chronicle::ETL::JobDefinitionError.new(self), "Job definition is invalid") unless validate
+        raise(Chronicle::ETL::JobDefinitionError.new(self), "Job definition is invalid") unless valid?
 
         true
       end
@@ -48,7 +60,6 @@ module Chronicle
       def add_config(config = {})
         @definition = @definition.deep_merge(config)
         load_credentials
-        validate
       end
 
       # Is this job continuing from a previous run?

data/lib/chronicle/etl/registry/plugin_registry.rb

@@ -36,8 +36,11 @@ module Chronicle
           # By default, activates the latest available version of a gem
           # so don't have to run Kernel#gem separately
           require "chronicle/#{name}"
-        rescue LoadError
-          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin #{name} couldn't be loaded" if exists?(name)
+        rescue Gem::ConflictError => e
+          # TODO: figure out if there's more we can do here
+          raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
+        rescue LoadError => e
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded" if exists?(name)
 
           raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist"
         end
@@ -49,9 +52,11 @@ module Chronicle
 
           Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
           Gem.install(gem_name)
+
+          activate(name)
         rescue Gem::UnsatisfiableDependencyError
           # TODO: we need to catch a lot more than this here
-          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist"
+          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} could not be installed."
         end
 
         # Uninstall a plugin

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.4.2"
+    VERSION = "0.4.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Andrew Louis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-03-12 00:00:00.000000000 Z
+date: 2022-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -359,6 +359,7 @@ files:
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
 - lib/chronicle/etl/cli.rb
+- lib/chronicle/etl/cli/cli_base.rb
 - lib/chronicle/etl/cli/connectors.rb
 - lib/chronicle/etl/cli/jobs.rb
 - lib/chronicle/etl/cli/main.rb