chronicle-etl 0.3.1 → 0.4.2

data/lib/chronicle/etl/cli/jobs.rb

@@ -1,4 +1,6 @@
 require 'pp'
+require 'tty-prompt'
+
 module Chronicle
   module ETL
     module CLI
@@ -7,18 +9,28 @@ module Chronicle
         default_task "start"
         namespace :jobs
 
-        class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'extractor-name'
+        class_option :name, aliases: '-j', desc: 'Job configuration name'
+
+        class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'NAME'
         class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
-        class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'transformer-name'
+        class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'NAME'
         class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
-        class_option :loader, aliases: '-l', desc: 'Loader class. Default: stdout', banner: 'loader-name'
+        class_option :loader, aliases: '-l', desc: 'Loader class. Default: table', banner: 'NAME'
         class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-        class_option :name, aliases: '-j', desc: 'Job configuration name'
 
-        map run: :start  # Thor doesn't like `run` as a command name
+        # This is an array to deal with shell globbing
+        class_option :input, aliases: '-i', desc: 'Input filename or directory', default: [], type: 'array', banner: 'FILENAME'
+        class_option :since, desc: "Load records SINCE this date", banner: 'DATE'
+        class_option :until, desc: "Load records UNTIL this date", banner: 'DATE'
+        class_option :limit, desc: "Only extract the first LIMIT records", banner: 'N'
+
+        class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
+        class_option :fields, desc: 'Output only these fields', type: 'array', banner: 'field1 field2 ...'
+        class_option :header_row, desc: 'Output the header row of tabular output', type: 'boolean'
+
+        # Thor doesn't like `run` as a command name
+        map run: :start
         desc "run", "Start a job"
-        option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
-        option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
         option :dry_run, desc: 'Only run the extraction and transform steps, not the loading', type: :boolean
         long_desc <<-LONG_DESC
           This will run an ETL job. Each job needs three parts:
@@ -33,25 +45,40 @@ module Chronicle
 LONG_DESC
         # Run an ETL job
         def start
-          setup_log_level
-          job_definition = build_job_definition(options)
-          job = Chronicle::ETL::Job.new(job_definition)
-          runner = Chronicle::ETL::Runner.new(job)
-          runner.run!
+          run_job(options)
+        rescue Chronicle::ETL::JobDefinitionError => e
+          missing_plugins = e.job_definition.errors
+            .select { |error| error.is_a?(Chronicle::ETL::PluginLoadError) }
+            .map(&:name)
+            .uniq
+
+          install_missing_plugins(missing_plugins)
+          run_job(options)
         end
 
         desc "create", "Create a job"
         # Create an ETL job
         def create
           job_definition = build_job_definition(options)
+          job_definition.validate!
+
           path = File.join('chronicle', 'etl', 'jobs', options[:name])
           Chronicle::ETL::Config.write(path, job_definition.definition)
+        rescue Chronicle::ETL::JobDefinitionError => e
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Job definition error".red)
         end
 
         desc "show", "Show details about a job"
         # Show an ETL job
         def show
-          puts Chronicle::ETL::Job.new(build_job_definition(options))
+          job_definition = build_job_definition(options)
+          job_definition.validate!
+          puts Chronicle::ETL::Job.new(job_definition)
+        rescue Chronicle::ETL::JobDefinitionError => e
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Job definition error".red)
+          exit 1
         end
 
         desc "list", "List all available jobs"
@@ -69,28 +96,52 @@ LONG_DESC
             [job, extractor, transformer, loader]
           end
 
-          headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
+          headers = ['name', 'extractor', 'transformer', 'loader'].map { |h| h.upcase.bold }
 
+          puts "Available jobs:"
           table = TTY::Table.new(headers, job_details)
           puts table.render(indent: 0, padding: [0, 2])
+        rescue Chronicle::ETL::ConfigError => e
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Error reading config. #{e.message}".red)
+          exit 1
         end
 
         private
 
-        def setup_log_level
-          if options[:verbose]
-            Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::DEBUG
-          elsif options[:log_level]
-            level = Chronicle::ETL::Logger.const_get(options[:log_level].upcase)
-            Chronicle::ETL::Logger.log_level = level 
+        def run_job(options)
+          job_definition = build_job_definition(options)
+          job = Chronicle::ETL::Job.new(job_definition)
+          runner = Chronicle::ETL::Runner.new(job)
+          runner.run!
+        end
+
+        # TODO: probably could merge this with something in cli/plugin
+        def install_missing_plugins(missing_plugins)
+          prompt = TTY::Prompt.new
+          message = "Plugin#{'s' if missing_plugins.count > 1} specified by job not installed.\n"
+          message += "Do you want to install "
+          message += missing_plugins.map { |name| "chronicle-#{name}".bold}.join(", ")
+          message += " and start the job?"
+          install = prompt.yes?(message)
+          return unless install
+
+          spinner = TTY::Spinner.new("[:spinner] Installing plugins...", format: :dots_2)
+          spinner.auto_spin
+          missing_plugins.each do |plugin|
+            Chronicle::ETL::Registry::PluginRegistry.install(plugin)
           end
+          spinner.success("(#{'successful'.green})")
+        rescue Chronicle::ETL::PluginNotAvailableError => e
+          spinner.error("Error".red)
+          Chronicle::ETL::Logger.fatal("Plugin '#{e.name}' could not be installed".red)
         end
 
         # Create job definition by reading config file and then overwriting with flag options
         def build_job_definition(options)
           definition = Chronicle::ETL::JobDefinition.new
           definition.add_config(load_job_config(options[:name]))
-          definition.add_config(process_flag_options(options))
+          definition.add_config(process_flag_options(options).transform_keys(&:to_sym))
           definition
         end
 
@@ -100,19 +151,34 @@ LONG_DESC
 
         # Takes flag options and turns them into a runner config
         def process_flag_options options
+          extractor_options = options[:'extractor-opts'].merge({
+            input: (options[:input] if options[:input].any?),
+            since: options[:since],
+            until: options[:until],
+            limit: options[:limit],
+          }.compact)
+
+          transformer_options = options[:'transformer-opts']
+
+          loader_options = options[:'loader-opts'].merge({
+            output: options[:output],
+            header_row: options[:header_row],
+            fields: options[:fields]
+          }.compact)
+
           {
             dry_run: options[:dry_run],
             extractor: {
               name: options[:extractor],
-              options: options[:'extractor-opts']
+              options: extractor_options
             }.compact,
             transformer: {
               name: options[:transformer],
-              options: options[:'transformer-opts']
+              options: transformer_options
             }.compact,
             loader: {
               name: options[:loader],
-              options: options[:'loader-opts']
+              options: loader_options
             }.compact
           }
         end

data/lib/chronicle/etl/cli/main.rb

@@ -1,17 +1,18 @@
-require 'thor'
-require 'chronicle/etl'
 require 'colorize'
 
-require 'chronicle/etl/cli/subcommand_base'
-require 'chronicle/etl/cli/connectors'
-require 'chronicle/etl/cli/jobs'
-
 module Chronicle
   module ETL
     module CLI
       # Main entrypoint for CLI app
-      class Main < Thor
-        class_option "verbose", type: :boolean, default: false
+      class Main < ::Thor
+        class_before :set_log_level
+        class_before :set_color_output
+
+        class_option :log_level, desc: 'Log level (debug, info, warn, error, fatal, silent)', default: 'info'
+        class_option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
+        class_option :silent, desc: 'Silence all output', type: :boolean
+        class_option :'no-color', desc: 'Disable colour output', type: :boolean
+
         default_task "jobs"
 
         desc 'connectors:COMMAND', 'Connectors available for ETL jobs', hide: true
@@ -20,17 +21,11 @@ module Chronicle
         desc 'jobs:COMMAND', 'Configure and run jobs', hide: true
         subcommand 'jobs', Jobs
 
+        desc 'plugins:COMMAND', 'Configure plugins', hide: true
+        subcommand 'plugins', Plugins
+
         # Entrypoint for the CLI
         def self.start(given_args = ARGV, config = {})
-          if given_args[0] == "--version"
-            puts "#{Chronicle::ETL::VERSION}"
-            exit
-          end
-
-          if given_args.none?
-            abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
-          end
-
           # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class
           if given_args.any? && given_args[0].include?(':')
             commands = given_args.shift.split(':')
@@ -40,10 +35,20 @@ module Chronicle
           super(given_args, config)
         end
 
+        def self.exit_on_failure?
+          true
+        end
+
+        desc "version", "Show version"
+        map %w(--version -v) => :version
+        def version
+          shell.say "chronicle-etl #{Chronicle::ETL::VERSION}"
+        end
+
         # Displays help options for chronicle-etl
         def help(meth = nil, subcommand = false)
           if meth && !respond_to?(meth)
-            klass, task = Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
+            klass, task = ::Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
             klass.start(['-h', task].compact, shell: shell)
           else
             shell.say "ABOUT".bold
@@ -64,7 +69,7 @@ module Chronicle
 
             list = []
 
-            Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
+            ::Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
               list += thor_class.printable_tasks(false)
             end
             list.sort! { |a, b| a[0] <=> b[0] }
@@ -85,6 +90,23 @@ module Chronicle
             shell.say
           end
         end
+
+        no_commands do
+          def set_color_output
+            String.disable_colorization true if options[:'no-color'] || ENV['NO_COLOR']
+          end
+
+          def set_log_level
+            if options[:silent]
+              Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::SILENT
+            elsif options[:verbose]
+              Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::DEBUG
+            elsif options[:log_level]
+              level = Chronicle::ETL::Logger.const_get(options[:log_level].upcase)
+              Chronicle::ETL::Logger.log_level = level
+            end
+          end
+        end
       end
     end
   end

data/lib/chronicle/etl/cli/plugins.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "tty-prompt"
+require "tty-spinner"
+
+
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL plugins
+      class Plugins < SubcommandBase
+        default_task 'list'
+        namespace :plugins
+
+        desc "install", "Install a plugin"
+        def install(name)
+          spinner = TTY::Spinner.new("[:spinner] Installing plugin #{name}...", format: :dots_2)
+          spinner.auto_spin
+          Chronicle::ETL::Registry::PluginRegistry.install(name)
+          spinner.success("(#{'successful'.green})")
+        rescue Chronicle::ETL::PluginError => e
+          spinner.error("Error".red)
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Plugin '#{name}' could not be installed".red)
+          exit 1
+        end
+
+        desc "uninstall", "Unintall a plugin"
+        def uninstall(name)
+          spinner = TTY::Spinner.new("[:spinner] Uninstalling plugin #{name}...", format: :dots_2)
+          spinner.auto_spin
+          Chronicle::ETL::Registry::PluginRegistry.uninstall(name)
+          spinner.success("(#{'successful'.green})")
+        rescue Chronicle::ETL::PluginError => e
+          spinner.error("Error".red)
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Plugin '#{name}' could not be uninstalled (was it installed?)".red)
+          exit 1
+        end
+
+        desc "list", "Lists available plugins"
+        # Display all available plugins that chronicle-etl has access to
+        def list
+          plugins = Chronicle::ETL::Registry::PluginRegistry.all_installed_latest
+
+          info = plugins.map do |plugin|
+            {
+              name: plugin.name.sub("chronicle-", ""),
+              description: plugin.description,
+              version: plugin.version
+            }
+          end
+
+          headers = ['name', 'description', 'latest version'].map{ |h| h.to_s.upcase.bold }
+          table = TTY::Table.new(headers, info.map(&:values))
+          puts "Installed plugins:"
+          puts table.render(indent: 2, padding: [0, 0])
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -2,11 +2,11 @@ module Chronicle
   module ETL
     module CLI
       # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
-      class SubcommandBase < Thor
+      class SubcommandBase < ::Thor
         # Print usage instructions for a subcommand
         def self.help(shell, subcommand = false)
           list = printable_commands(true, subcommand)
-          Thor::Util.thor_classes_in(self).each do |klass|
+          ::Thor::Util.thor_classes_in(self).each do |klass|
             list += klass.printable_commands(false)
           end
           list.sort! { |a, b| a[0] <=> b[0] }

data/lib/chronicle/etl/cli.rb

@@ -0,0 +1,9 @@
+require 'thor'
+require 'thor/hollaback'
+require 'chronicle/etl'
+
+require 'chronicle/etl/cli/subcommand_base'
+require 'chronicle/etl/cli/connectors'
+require 'chronicle/etl/cli/jobs'
+require 'chronicle/etl/cli/plugins'
+require 'chronicle/etl/cli/main'

data/lib/chronicle/etl/config.rb

@@ -24,16 +24,14 @@ module Chronicle
 
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
       def available_jobs
-        job_directory = Runcom::Config.new('chronicle/etl/jobs').current
-        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+        Dir.glob(File.join(config_directory("jobs"), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
 
       # Returns all available credentials available in ~/.config/chronicle/etl/credentials/*.yml
       def available_credentials
-        job_directory = Runcom::Config.new('chronicle/etl/credentials').current
-        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+        Dir.glob(File.join(config_directory("credentials"), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
@@ -48,6 +46,11 @@ module Chronicle
       def load_credentials(name)
         config = self.load("chronicle/etl/credentials/#{name}.yml")
       end
+
+      def config_directory(type)
+        path = "chronicle/etl/#{type}"
+        Runcom::Config.new(path).current || raise(Chronicle::ETL::ConfigError, "Could not access config directory (#{path})")
+      end
     end
   end
 end

data/lib/chronicle/etl/configurable.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "ostruct"
+
+module Chronicle
+  module ETL
+    # A mixin that gives a class
+    # a {Chronicle::ETL::Configurable::ClassMethods#setting} macro to define
+    # settings and their properties (require, type, etc)
+    #
+    # @example Basic usage
+    #   class Test < Chronicle::ETL::Extractor
+    #     include Chronicle::ETL::Configurable
+    #     setting :when, type: :date, required: true
+    #   end
+    #
+    #   t = Test.new(when: '2022-02-24')
+    #   t.config.when
+    module Configurable
+      # An individual setting for this Configurable
+      Setting = Struct.new(:default, :required, :type)
+      private_constant :Setting
+
+      # Collection of user-supplied options for this Configurable
+      class Config < OpenStruct
+        # Config values that aren't nil, as a hash
+        def compacted_h
+          to_h.compact
+        end
+      end
+
+      # @private
+      def self.included(klass)
+        klass.extend(ClassMethods)
+        klass.include(InstanceMethods)
+        klass.prepend(Initializer)
+      end
+
+      # Initializer method for classes that have Configurable mixed in
+      module Initializer
+        # Make sure this class has a default @config ready to use
+        def initialize(*args)
+          @config = initialize_default_config
+          super
+        end
+      end
+
+      # Instance methods for classes that have Configurable mixed in
+      module InstanceMethods
+        attr_reader :config
+
+        # Take given options and apply them to this class's settings
+        # and make them available in @config and validates that they
+        # conform to setting rules
+        def apply_options(options)
+          options.transform_keys!(&:to_sym)
+
+          options.each do |name, value|
+            setting = self.class.all_settings[name]
+            raise(Chronicle::ETL::ConnectorConfigurationError, "Unrecognized setting: #{name}") unless setting
+
+            @config[name] = coerced_value(setting, value)
+          end
+          validate_config
+          options
+        end
+
+        # Name of all settings available to this class
+        def self.settings
+          self.class.all_settings.keys
+        end
+
+        private
+
+        def initialize_default_config
+          self.class.config_with_defaults
+        end
+
+        def validate_config
+          missing = (self.class.all_required_settings.keys - @config.compacted_h.keys)
+          raise Chronicle::ETL::ConnectorConfigurationError, "Missing options: #{missing}" if missing.count.positive?
+        end
+
+        def coerced_value(setting, value)
+          setting.type ? __send__("coerce_#{setting.type}", value) : value
+        end
+
+        def coerce_string(value)
+          value.to_s
+        end
+
+        # TODO: think about whether to split up float, integer
+        def coerce_numeric(value)
+          value.to_f
+        end
+
+        def coerce_boolean(value)
+          if value.is_a?(String)
+            value.downcase == "true"
+          else
+            value
+          end
+        end
+
+        def coerce_time(value)
+          # TODO: handle durations like '3h'
+          if value.is_a?(String)
+            Time.parse(value)
+          else
+            value
+          end
+        end
+      end
+
+      # Class methods for classes that have Configurable mixed in
+      module ClassMethods
+        # Macro for creating a setting on a class {::Chronicle::ETL::Configurable}
+        #
+        # @param [String] name Name of the setting
+        # @param [Boolean] required whether setting is required
+        # @param [Object] default Default value
+        # @param [Symbol] type Type
+        #
+        # @example Basic usage
+        #   setting :when, type: :date, required: true
+        #
+        # @see ::Chronicle::ETL::Configurable
+        def setting(name, default: nil, required: false, type: nil)
+          s = Setting.new(default, required, type)
+          settings[name] = s
+        end
+
+        # Collect all settings defined on this class and its ancestors (that
+        # have Configurable mixin included)
+        def all_settings
+          if superclass.include?(Chronicle::ETL::Configurable)
+            superclass.all_settings.merge(settings)
+          else
+            settings
+          end
+        end
+
+        # Filters settings to those that are required.
+        def all_required_settings
+          all_settings.select { |_name, setting| setting.required } || {}
+        end
+
+        def settings
+          @settings ||= {}
+        end
+
+        def setting_exists?(name)
+          all_settings.keys.include? name
+        end
+
+        def config_with_defaults
+          s = all_settings.transform_values(&:default)
+          Config.new(s)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/exceptions.rb

@@ -1,9 +1,33 @@
 module Chronicle
   module ETL
-    class Error < StandardError; end;
+    class Error < StandardError; end
+
+    class ConfigError < Error; end
 
     class RunnerTypeError < Error; end
 
+    class JobDefinitionError < Error
+      attr_reader :job_definition
+
+      def initialize(job_definition)
+        @job_definition = job_definition
+        super
+      end
+    end
+
+    class PluginError < Error
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+      end
+    end
+
+    class PluginNotAvailableError < PluginError; end
+    class PluginLoadError < PluginError; end
+
+    class ConnectorConfigurationError < Error; end
+
     class ConnectorNotAvailableError < Error
       def initialize(message, provider: nil, name: nil)
         super(message)
@@ -16,6 +40,10 @@ module Chronicle
     class ProviderNotAvailableError < ConnectorNotAvailableError; end
     class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
 
+    class ExtractionError < Error; end
+
+    class SerializationError < Error; end
+
     class TransformationError < Error
       attr_reader :transformation