chronicle-etl 0.4.0 → 0.4.3

data/lib/chronicle/etl/cli/plugins.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "tty-prompt"
+require "tty-spinner"
+
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL plugins
+      class Plugins < SubcommandBase
+        default_task 'list'
+        namespace :plugins
+
+        desc "install", "Install a plugin"
+        def install(*plugins)
+          cli_fail(message: "Please specify a plugin to install") unless plugins.any?
+
+          spinner = TTY::Spinner.new("[:spinner] Installing #{plugins.join(", ")}...", format: :dots_2)
+          spinner.auto_spin
+          plugins.each do |plugin|
+            spinner.update(title: "Installing #{plugin}")
+            Chronicle::ETL::Registry::PluginRegistry.install(plugin)
+          rescue Chronicle::ETL::PluginError => e
+            spinner.error("Error".red)
+            cli_fail(message: "Plugin '#{plugin}' could not be installed", exception: e)
+          end
+          spinner.success("(#{'successful'.green})")
+        end
+
+        desc "uninstall", "Unintall a plugin"
+        def uninstall(name)
+          spinner = TTY::Spinner.new("[:spinner] Uninstalling plugin #{name}...", format: :dots_2)
+          spinner.auto_spin
+          Chronicle::ETL::Registry::PluginRegistry.uninstall(name)
+          spinner.success("(#{'successful'.green})")
+        rescue Chronicle::ETL::PluginError => e
+          spinner.error("Error".red)
+          cli_fail(message: "Plugin '#{name}' could not be uninstalled (was it installed?)", exception: e)
+        end
+
+        desc "list", "Lists available plugins"
+        # Display all available plugins that chronicle-etl has access to
+        def list
+          plugins = Chronicle::ETL::Registry::PluginRegistry.all_installed_latest
+
+          info = plugins.map do |plugin|
+            {
+              name: plugin.name.sub("chronicle-", ""),
+              description: plugin.description,
+              version: plugin.version
+            }
+          end
+
+          headers = ['name', 'description', 'latest version'].map{ |h| h.to_s.upcase.bold }
+          table = TTY::Table.new(headers, info.map(&:values))
+          puts "Installed plugins:"
+          puts table.render(indent: 2, padding: [0, 0])
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -2,7 +2,7 @@ module Chronicle
   module ETL
     module CLI
       # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
-      class SubcommandBase < ::Thor
+      class SubcommandBase < Chronicle::ETL::CLI::CLIBase
         # Print usage instructions for a subcommand
         def self.help(shell, subcommand = false)
           list = printable_commands(true, subcommand)

data/lib/chronicle/etl/cli.rb

@@ -1,7 +1,10 @@
 require 'thor'
+require 'thor/hollaback'
 require 'chronicle/etl'
 
+require 'chronicle/etl/cli/cli_base'
 require 'chronicle/etl/cli/subcommand_base'
 require 'chronicle/etl/cli/connectors'
 require 'chronicle/etl/cli/jobs'
+require 'chronicle/etl/cli/plugins'
 require 'chronicle/etl/cli/main'

data/lib/chronicle/etl/config.rb

@@ -24,16 +24,14 @@ module Chronicle
 
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
       def available_jobs
-        job_directory = Runcom::Config.new('chronicle/etl/jobs').current
-        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+        Dir.glob(File.join(config_directory("jobs"), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
 
       # Returns all available credentials available in ~/.config/chronicle/etl/credentials/*.yml
       def available_credentials
-        job_directory = Runcom::Config.new('chronicle/etl/credentials').current
-        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+        Dir.glob(File.join(config_directory("credentials"), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
@@ -48,6 +46,11 @@ module Chronicle
       def load_credentials(name)
         config = self.load("chronicle/etl/credentials/#{name}.yml")
       end
+
+      def config_directory(type)
+        path = "chronicle/etl/#{type}"
+        Runcom::Config.new(path).current || raise(Chronicle::ETL::ConfigError, "Could not access config directory (#{path})")
+      end
     end
   end
 end

data/lib/chronicle/etl/configurable.rb

@@ -57,7 +57,7 @@ module Chronicle
 
           options.each do |name, value|
             setting = self.class.all_settings[name]
-            raise(Chronicle::ETL::ConfigurationError, "Unrecognized setting: #{name}") unless setting
+            raise(Chronicle::ETL::ConnectorConfigurationError, "Unrecognized setting: #{name}") unless setting
 
             @config[name] = coerced_value(setting, value)
           end
@@ -78,7 +78,7 @@ module Chronicle
 
         def validate_config
           missing = (self.class.all_required_settings.keys - @config.compacted_h.keys)
-          raise Chronicle::ETL::ConfigurationError, "Missing options: #{missing}" if missing.count.positive?
+          raise Chronicle::ETL::ConnectorConfigurationError, "Missing options: #{missing}" if missing.count.positive?
         end
 
         def coerced_value(setting, value)
@@ -89,6 +89,19 @@ module Chronicle
           value.to_s
         end
 
+        # TODO: think about whether to split up float, integer
+        def coerce_numeric(value)
+          value.to_f
+        end
+
+        def coerce_boolean(value)
+          if value.is_a?(String)
+            value.downcase == "true"
+          else
+            value
+          end
+        end
+
         def coerce_time(value)
           # TODO: handle durations like '3h'
           if value.is_a?(String)

data/lib/chronicle/etl/exceptions.rb

@@ -1,11 +1,34 @@
 module Chronicle
   module ETL
-    class Error < StandardError; end;
+    class Error < StandardError; end
 
-    class ConfigurationError < Error; end;
+    class ConfigError < Error; end
 
     class RunnerTypeError < Error; end
 
+    class JobDefinitionError < Error
+      attr_reader :job_definition
+
+      def initialize(job_definition)
+        @job_definition = job_definition
+        super
+      end
+    end
+
+    class PluginError < Error
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+      end
+    end
+
+    class PluginConflictError < PluginError; end
+    class PluginNotAvailableError < PluginError; end
+    class PluginLoadError < PluginError; end
+
+    class ConnectorConfigurationError < Error; end
+
     class ConnectorNotAvailableError < Error
       def initialize(message, provider: nil, name: nil)
         super(message)
@@ -18,6 +41,10 @@ module Chronicle
     class ProviderNotAvailableError < ConnectorNotAvailableError; end
     class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
 
+    class ExtractionError < Error; end
+
+    class SerializationError < Error; end
+
     class TransformationError < Error
       attr_reader :transformation
 

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -3,39 +3,46 @@ require 'csv'
 module Chronicle
   module ETL
     class CSVExtractor < Chronicle::ETL::Extractor
-      include Extractors::Helpers::FilesystemReader
+      include Extractors::Helpers::InputReader
 
       register_connector do |r|
-        r.description = 'input as CSV'
+        r.description = 'CSV'
       end
 
       setting :headers, default: true
-      setting :filename, default: $stdin
+
+      def prepare
+        @csvs = prepare_sources
+      end
 
       def extract
-        csv = initialize_csv
-        csv.each do |row|
-          yield Chronicle::ETL::Extraction.new(data: row.to_h)
+        @csvs.each do |csv|
+          csv.read.each do |row|
+            yield Chronicle::ETL::Extraction.new(data: row.to_h)
+          end
         end
       end
 
       def results_count
-        CSV.read(@config.filename, headers: @config.headers).count unless stdin?(@config.filename)
+        @csvs.reduce(0) do |total_rows, csv|
+          row_count = csv.readlines.size
+          csv.rewind
+          total_rows + row_count
+        end
       end
 
       private
 
-      def initialize_csv
-        headers = @config.headers.is_a?(String) ? @config.headers.split(',') : @config.headers
-
-        csv_options = {
-          headers: headers,
-          converters: :all
-        }
-
-        open_from_filesystem(filename: @config.filename) do |file|
-          return CSV.new(file, **csv_options)
+      def prepare_sources
+        @csvs = []
+        read_input do |csv_data|
+          csv_options = {
+            headers: @config.headers.is_a?(String) ? @config.headers.split(',') : @config.headers,
+            converters: :all
+          }
+          @csvs << CSV.new(csv_data, **csv_options)
         end
+        @csvs
       end
     end
   end

data/lib/chronicle/etl/extractors/extractor.rb

@@ -7,11 +7,11 @@ module Chronicle
       extend Chronicle::ETL::Registry::SelfRegistering
       include Chronicle::ETL::Configurable
 
-      setting :since, type: :date
-      setting :until, type: :date
-      setting :limit
+      setting :since, type: :time
+      setting :until, type: :time
+      setting :limit, type: :numeric
       setting :load_after_id
-      setting :filename
+      setting :input
 
       # Construct a new instance of this extractor. Options are passed in from a Runner
       # == Parameters:
@@ -46,7 +46,7 @@ module Chronicle
   end
 end
 
-require_relative 'helpers/filesystem_reader'
+require_relative 'helpers/input_reader'
 require_relative 'csv_extractor'
 require_relative 'file_extractor'
 require_relative 'json_extractor'

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -2,35 +2,55 @@ require 'pathname'
 
 module Chronicle
   module ETL
+    # Return filenames that match a pattern in a directory
     class FileExtractor < Chronicle::ETL::Extractor
-      include Extractors::Helpers::FilesystemReader
 
       register_connector do |r|
         r.description = 'file or directory of files'
       end
 
-      # TODO: consolidate this with @config.filename
-      setting :dir_glob_pattern
+      setting :input, default: ['.']
+      setting :dir_glob_pattern, default: "**/*"
+      setting :larger_than
+      setting :smaller_than
+
+      def prepare
+        @pathnames = gather_files
+      end
 
       def extract
-        filenames.each do |filename|
-          yield Chronicle::ETL::Extraction.new(data: filename)
+        @pathnames.each do |pathname|
+          yield Chronicle::ETL::Extraction.new(data: pathname.to_path)
         end
       end
 
       def results_count
-        filenames.count
+        @pathnames.count
       end
 
       private
 
-      def filenames
-        @filenames ||= filenames_in_directory(
-          path: @config.filename,
-          dir_glob_pattern: @config.dir_glob_pattern,
-          load_since: @config.since,
-          load_until: @config.until
-        )
+      def gather_files
+        roots = [@config.input].flatten.map { |filename| Pathname.new(filename) }
+        raise(ExtractionError, "Input must exist") unless roots.all?(&:exist?)
+
+        directories, files = roots.partition(&:directory?)
+
+        directories.each do |directory|
+          files += Dir.glob(File.join(directory, @config.dir_glob_pattern)).map { |filename| Pathname.new(filename) }
+        end
+
+        files = files.uniq
+
+        files = files.keep_if { |f| (f.mtime > @config.since) } if @config.since
+        files = files.keep_if { |f| (f.mtime < @config.until) } if @config.until
+
+        # pass in file sizes in bytes
+        files = files.keep_if { |f| (f.size < @config.smaller_than) } if @config.smaller_than
+        files = files.keep_if { |f| (f.size > @config.larger_than) } if @config.larger_than
+
+        # # TODO: incorporate sort argument
+        files.sort_by(&:mtime)
       end
     end
   end

data/lib/chronicle/etl/extractors/helpers/input_reader.rb

@@ -0,0 +1,76 @@
+require 'pathname'
+
+module Chronicle
+  module ETL
+    module Extractors
+      module Helpers
+        module InputReader
+          # Return an array of input filenames; converts a single string
+          # to an array if necessary
+          def filenames
+            [@config.input].flatten.map
+          end
+
+          # Filenames as an array of pathnames
+          def pathnames
+            filenames.map { |filename| Pathname.new(filename) }
+          end
+
+          # Whether we're reading from files
+          def read_from_files?
+            filenames.any?
+          end
+
+          # Whether we're reading input from stdin
+          def read_from_stdin?
+            !read_from_files? && $stdin.stat.pipe?
+          end
+
+          # Read input sources and yield each content
+          def read_input
+            if read_from_files?
+              pathnames.each do |pathname|
+                File.open(pathname) do |file|
+                  yield file.read, pathname.to_path
+                end
+              end
+            elsif read_from_stdin?
+              yield $stdin.read, $stdin
+            else
+              raise ExtractionError, "No input files or stdin provided"
+            end
+          end
+
+          # Read input sources line by line
+          def read_input_as_lines(&block)
+            if read_from_files?
+              lines_from_files(&block)
+            elsif read_from_stdin?
+              lines_from_stdin(&block)
+            else
+              raise ExtractionError, "No input files or stdin provided"
+            end
+          end
+
+          private
+
+          def lines_from_files(&block)
+            pathnames.each do |pathname|
+              File.open(pathname) do |file|
+                lines_from_io(file, &block)
+              end
+            end
+          end
+
+          def lines_from_stdin(&block)
+            lines_from_io($stdin, &block)
+          end
+
+          def lines_from_io(io, &block)
+            io.each_line(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/json_extractor.rb

@@ -1,35 +1,44 @@
 module Chronicle
   module ETL
-    class JsonExtractor < Chronicle::ETL::Extractor
-      include Extractors::Helpers::FilesystemReader
+    class JSONExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::InputReader
 
       register_connector do |r|
-        r.description = 'input as JSON'
+        r.description = 'JSON'
       end
 
-      setting :filename, default: $stdin
-      setting :jsonl, default: true
+      setting :jsonl, default: true, type: :boolean
 
-      def extract
+      def prepare
+        @jsons = []
         load_input do |input|
-          parsed_data = parse_data(input)
-          yield Chronicle::ETL::Extraction.new(data: parsed_data) if parsed_data
+          @jsons << parse_data(input)
+        end
+      end
+
+      def extract
+        @jsons.each do |json|
+          yield Chronicle::ETL::Extraction.new(data: json)
         end
       end
 
       def results_count
+        @jsons.count
       end
 
       private
 
       def parse_data data
         JSON.parse(data)
-      rescue JSON::ParserError => e
+      rescue JSON::ParserError
+        raise Chronicle::ETL::ExtractionError, "Could not parse JSON"
       end
 
-      def load_input
-        read_from_filesystem(filename: @options[:filename]) do |data|
-          yield data
+      def load_input(&block)
+        if @config.jsonl
+          read_input_as_lines(&block)
+        else
+          read_input(&block)
         end
       end
     end

data/lib/chronicle/etl/job.rb

@@ -1,6 +1,11 @@
 require 'forwardable'
+
 module Chronicle
   module ETL
+    # A runner job
+    #
+    # TODO: this can probably be merged with JobDefinition. Not clear
+    # where the boundaries are
     class Job
       extend Forwardable
 
@@ -12,7 +17,8 @@ module Chronicle
                     :transformer_klass,
                     :transformer_options,
                     :loader_klass,
-                    :loader_options
+                    :loader_options,
+                    :job_definition
 
       # TODO: build a proper id system
       alias id name

data/lib/chronicle/etl/job_definition.rb

@@ -14,22 +14,52 @@ module Chronicle
           options: {}
         },
         loader: {
-          name: 'stdout',
+          name: 'table',
           options: {}
         }
       }.freeze
 
+      attr_reader :errors
       attr_accessor :definition
 
       def initialize()
         @definition = SKELETON_DEFINITION
       end
 
+      def valid?
+        validate
+        @errors.empty?
+      end
+
+      def validate
+        @errors = {}
+
+        Chronicle::ETL::Registry::PHASES.each do |phase|
+          __send__("#{phase}_klass".to_sym)
+        rescue Chronicle::ETL::PluginError => e
+          @errors[:plugins] ||= []
+          @errors[:plugins] << e
+        end
+      end
+
+      def plugins_missing?
+        validate
+
+        @errors[:plugins] || []
+          .filter { |e| e.instance_of?(Chronicle::ETL::PluginLoadError) }
+          .any?
+      end
+
+      def validate!
+        raise(Chronicle::ETL::JobDefinitionError.new(self), "Job definition is invalid") unless valid?
+
+        true
+      end
+
       # Add config hash to this definition
       def add_config(config = {})
         @definition = @definition.deep_merge(config)
         load_credentials
-        validate
       end
 
       # Is this job continuing from a previous run?
@@ -80,10 +110,6 @@ module Chronicle
           end
         end
       end
-
-      def validate
-        return true   # TODO
-      end
     end
   end
 end

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -7,22 +7,49 @@ module Chronicle
         r.description = 'CSV'
       end
 
-      def initialize(options={})
-        super(options)
-        @rows = []
+      setting :output, default: $stdout
+      setting :headers, default: true
+      setting :header_row, default: true
+
+      def records
+        @records ||= []
       end
 
       def load(record)
-        @rows << record.to_h_flattened.values
+        records << record.to_h_flattened
       end
 
       def finish
-        z = $stdout
-        CSV(z) do |csv|
-          @rows.each do |row|
-            csv << row
+        return unless records.any?
+
+        headers = build_headers(records)
+
+        csv_options = {}
+        if @config.headers
+          csv_options[:write_headers] = @config.header_row
+          csv_options[:headers] = headers
+        end
+
+        if @config.output.is_a?(IO)
+          # This might seem like a duplication of the default value ($stdout)
+          # but it's because rspec overwrites $stdout (in helper #capture) to
+          # capture output.
+          io = $stdout.dup
+        else
+          io = File.open(@config.output, "w+")
+        end
+
+        output = CSV.generate(**csv_options) do |csv|
+          records.each do |record|
+            csv << record
+              .transform_keys(&:to_sym)
+              .values_at(*headers)
+              .map { |value| force_utf8(value) }
           end
         end
+
+        io.write(output)
+        io.close
       end
     end
   end

data/lib/chronicle/etl/loaders/helpers/encoding_helper.rb

@@ -0,0 +1,18 @@
+require 'pathname'
+
+module Chronicle
+  module ETL
+    module Loaders
+      module Helpers
+        module EncodingHelper
+          # Mostly useful for handling loading with binary data from a raw extraction
+          def force_utf8(value)
+            return value unless value.is_a?(String)
+
+            value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
+          end
+        end
+      end
+    end
+  end
+end