chronicle-etl 0.5.5 → 0.6.1

data/lib/chronicle/etl/registry/connectors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 
 module Chronicle
@@ -5,54 +7,109 @@ module Chronicle
     module Registry
       # A singleton class that acts as a registry of connector classes available for ETL jobs
       module Connectors
-        PHASES = [:extractor, :transformer, :loader].freeze
+        PHASES = %i[extractor transformer loader].freeze
         public_constant :PHASES
 
         class << self
           attr_accessor :connectors
+        end
+
+        def self.register(connector)
+          connectors << connector
+        end
 
-          def register(connector)
-            connectors << connector
+        def self.connectors
+          @connectors ||= []
+        end
+
+        def self.ancestor_for_phase(phase)
+          case phase
+          when :extractor
+            Chronicle::ETL::Extractor
+          when :transformer
+            Chronicle::ETL::Transformer
+          when :loader
+            Chronicle::ETL::Loader
           end
+        end
 
-          def connectors
-            @connectors ||= []
+        def self.find_converter_for_source(source:, type: nil, strategy: nil, target: nil)
+          # FIXME: we're assuming extractor plugin has been loaded already
+          # This may not be the case if the schema converter is running
+          # off a json dump off extraction data.
+          # plugin = source_klass.connector_registration.source
+          # type = source_klass.connector_registration.type
+          # strategy = source_klass.connector_registration.strategy
+
+          connectors.find do |c|
+            c.phase == :transformer &&
+              c.source == source &&
+              (type.nil? || c.type == type) &&
+              (strategy.nil? || c.strategy == strategy || c.strategy.nil?) &&
+              (target.nil? || c.to_schema == target)
           end
+        end
+
+        # Find connector from amongst those currently loaded
+        def self.find_by_phase_and_identifier_built_in(phase, identifier)
+          connectors.find { |c| c.phase == phase.to_sym && c.identifier == identifier.to_sym }
+        end
+
+        # Find connector and load relevant plugin to find it if necessary
+        def self.find_by_phase_and_identifier(phase, identifier)
+          connector = find_by_phase_and_identifier_built_in(phase, identifier)
+          return connector if connector
+
+          # determine if we need to try to load a local file. if it has a dot in the identifier, we treat it as a file
+          return find_by_phase_and_identifier_local(phase, identifier) if identifier.to_s.include?('.')
+
+          # Example identifier: lastfm:listens:api
+          plugin, type, strategy = identifier.split(':')
+            .map { |part| part.gsub('-', '_') }
+            .map(&:to_sym)
 
-          # Find connector from amongst those currently loaded
-          def find_by_phase_and_identifier_local(phase, identifier)
-            connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
+          plugin_identifier = plugin.to_s.gsub('_', '-')
+
+          unless Chronicle::ETL::Registry::Plugins.installed?(plugin_identifier)
+            raise Chronicle::ETL::PluginNotInstalledError, plugin_identifier
           end
 
-          # Find connector and load relevant plugin to find it if necessary
-          def find_by_phase_and_identifier(phase, identifier)
-            connector = find_by_phase_and_identifier_local(phase, identifier)
-            return connector if connector
-
-            # if not available in built-in connectors, try to activate a
-            # relevant plugin and try again
-            if identifier.include?(":")
-              plugin, name = identifier.split(":")
-            else
-              # This case handles the case where the identifier is a 
-              # shorthand (ie `imessage`) because there's only one default
-              # connector.
-              plugin = identifier
-            end
-
-            raise(Chronicle::ETL::PluginNotInstalledError.new(plugin)) unless Chronicle::ETL::Registry::Plugins.installed?(plugin)
-
-            Chronicle::ETL::Registry::Plugins.activate(plugin)
-
-            candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }
-            # if no name given, just use first connector with right phase/plugin
-            # TODO: set up a property for connectors to specify that they're the
-            # default connector for the plugin
-            candidates = candidates.select { |c| c.identifier == name } if name
-            connector = candidates.first
-
-            connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
+          Chronicle::ETL::Registry::Plugins.activate(plugin_identifier)
+
+          # find most specific connector that matches the identifier
+          connector = connectors.find do |c|
+            c.plugin == plugin && (type.nil? || c.type == type) && (strategy.nil? || c.strategy == strategy)
           end
+
+          connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
+        end
+
+        # Load a plugin from local file system
+        def self.find_by_phase_and_identifier_local(phase, identifier)
+          script = File.read(identifier)
+          raise ConnectorNotAvailableError, "Connector '#{identifier}' not found" if script.nil?
+
+          # load the file by evaluating the contents
+          eval(script, TOPLEVEL_BINDING, __FILE__, __LINE__) # rubocop:disable Security/Eval
+
+          # read the file and look for all class definitions in the ruby script.
+          class_names = script.scan(/class (\w+)/).flatten
+
+          class_names.each do |class_name|
+            klass = Object.const_get(class_name)
+
+            next unless klass.ancestors.include?(ancestor_for_phase(phase))
+
+            registration = ::Chronicle::ETL::Registry::ConnectorRegistration.new(klass)
+
+            klass.connector_registration = registration
+            return registration
+            # return klass
+          rescue NameError
+            # ignore
+          end
+
+          raise ConnectorNotAvailableError, "Connector '#{identifier}' not found"
         end
       end
     end

data/lib/chronicle/etl/registry/plugin_registration.rb

@@ -4,7 +4,7 @@ module Chronicle
       class PluginRegistration
         attr_accessor :name, :description, :gem, :version, :installed, :gemspec
 
-        def initialize(name=nil)
+        def initialize(name = nil)
           @installed = false
           @name = name
           yield self if block_given?

data/lib/chronicle/etl/registry/plugins.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 require 'rubygems/command'
 require 'rubygems/commands/install_command'
@@ -13,16 +15,17 @@ module Chronicle
       # @todo Better validation for whether a gem is actually a plugin
       # @todo Add ways to load a plugin that don't require a gem on rubygems.org
       module Plugins
-        KNOWN_PLUGINS = [
-          'email',
-          'foursquare',
-          'github',
-          'imessage',
-          'pinboard',
-          'safari',
-          'shell',
-          'spotify',
-          'zulip'
+        KNOWN_PLUGINS = %w[
+          apple-podcasts
+          email
+          foursquare
+          github
+          imessage
+          pinboard
+          safari
+          shell
+          spotify
+          zulip
         ].freeze
         public_constant :KNOWN_PLUGINS
 
@@ -30,7 +33,7 @@ module Chronicle
         # make registry aware of existence of name of non-gem plugin
         def self.register_standalone(name:)
           plugin = Chronicle::ETL::Registry::PluginRegistration.new do |p|
-            p.name = name
+            p.name = name.to_sym
             p.installed = true
           end
 
@@ -44,19 +47,19 @@ module Chronicle
 
         # Check whether a given plugin is installed
         def self.installed?(name)
-          installed.map(&:name).include?(name)
+          installed.map(&:name).include?(name.to_sym)
         end
 
         # List of plugins installed as standalone
         def self.installed_standalone
-          @standalones ||= []
+          @installed_standalone ||= []
         end
 
         # List of plugins installed as gems
         def self.installed_as_gem
           installed_gemspecs_latest.map do |gem|
             Chronicle::ETL::Registry::PluginRegistration.new do |p|
-              p.name = gem.name.sub("chronicle-", "")
+              p.name = gem.name.sub('chronicle-', '').to_sym
               p.gem = gem.name
               p.description = gem.description
               p.version = gem.version.to_s
@@ -106,7 +109,9 @@ module Chronicle
         # All versions of all plugins currently installed
         def self.installed_gemspecs
           # TODO: add check for chronicle-etl dependency
-          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" }
+          Gem::Specification.filter do |s|
+            s.name.match(/^chronicle-/) && s.name != 'chronicle-etl' && s.name != 'chronicle-core'
+          end
         end
 
         # Latest version of each installed plugin
@@ -120,15 +125,18 @@ module Chronicle
         def self.activate(name)
           # By default, activates the latest available version of a gem
           # so don't have to run Kernel#gem separately
-          require "chronicle/#{name}"
+
+          plugin_require_name = name.to_s.gsub('-', '_')
+          require "chronicle/#{plugin_require_name}"
         rescue Gem::ConflictError => e
           # TODO: figure out if there's more we can do here
-          raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
-        rescue StandardError, LoadError => e
+          raise Chronicle::ETL::PluginConflictError.new(name),
+            "Plugin '#{plugin_require_name}' couldn't be loaded. #{e.message}"
+        rescue StandardError, LoadError
           # StandardError to catch random non-loading problems that might occur
           # when requiring the plugin (eg class macro invoked the wrong way)
           # TODO: decide if this should be separated
-          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded"
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{plugin_require_name}' couldn't be loaded"
         end
 
         # Install a plugin to local gems

data/lib/chronicle/etl/runner.rb

@@ -1,133 +1,163 @@
+# frozen_string_literal: true
+
 require 'colorize'
 require 'chronic_duration'
-require "tty-spinner"
-
-class Chronicle::ETL::Runner
-  def initialize(job)
-    @job = job
-    @job_logger = Chronicle::ETL::JobLogger.new(@job)
-  end
-
-  def run!
-    begin_job
-    validate_job
-    instantiate_connectors
-    prepare_job
-    prepare_ui
-    run_extraction
-  rescue Chronicle::ETL::ExtractionError => e
-    @job_logger&.error
-    raise(Chronicle::ETL::RunnerError, "Extraction failed. #{e.message}")
-  rescue Interrupt
-    @job_logger&.error
-    raise(Chronicle::ETL::RunInterruptedError, "Job interrupted.")
-  rescue StandardError => e
-    # Just throwing this in here until we have better exception handling in
-    # loaders, etc
-    @job_logger&.error
-    raise(Chronicle::ETL::RunnerError, "Error running job. #{e.message}")
-  ensure
-    finish_job
-  end
-
-  private
-
-  def begin_job
-    Chronicle::ETL::Logger.info(tty_log_job_initialize)
-    @initialization_spinner = TTY::Spinner.new(":spinner :title", format: :dots_2)
-  end
-
-  def validate_job
-    @initialization_spinner.update(title: "Validating job")
-    @job.job_definition.validate!
-  end
-
-  def instantiate_connectors
-    @initialization_spinner.update(title: "Initializing connectors")
-    @extractor = @job.instantiate_extractor
-    @loader = @job.instantiate_loader
-  end
-
-  def prepare_job
-    @initialization_spinner.update(title: "Preparing job")
-    @job_logger.start
-    @loader.start
-
-    @initialization_spinner.update(title: "Preparing extraction")
-    @initialization_spinner.auto_spin
-    @extractor.prepare
-    @initialization_spinner.success("(#{'successful'.green})")
-    Chronicle::ETL::Logger.info("\n")
-  end
-
-  def prepare_ui
-    total = @extractor.results_count
-    @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
-    Chronicle::ETL::Logger.attach_to_ui(@progress_bar)
-  end
-
-  def run_extraction
-    @extractor.extract do |extraction|
-      process_extraction(extraction)
-      @progress_bar.increment
+require 'tty-spinner'
+
+module Chronicle
+  module ETL
+    class Runner
+      def initialize(job)
+        @job = job
+        @job_logger = Chronicle::ETL::JobLogger.new(@job)
+      end
+
+      def run!
+        begin_job
+        validate_job
+        instantiate_connectors
+        prepare_job
+        prepare_ui
+        run_extraction
+      rescue Chronicle::ETL::ExtractionError => e
+        @job_logger&.error
+        raise(Chronicle::ETL::RunnerError, "Extraction failed. #{e.message}")
+      rescue Interrupt
+        @job_logger&.error
+        raise(Chronicle::ETL::RunInterruptedError, 'Job interrupted.')
+      # rescue StandardError => e
+      #   # Just throwing this in here until we have better exception handling in
+      #   # loaders, etc
+      #   @job_logger&.error
+      #   raise(Chronicle::ETL::RunnerError, "Error running job. #{e.message}")
+      ensure
+        finish_job
+      end
+
+      private
+
+      def begin_job
+        Chronicle::ETL::Logger.info(tty_log_job_initialize)
+        @initialization_spinner = TTY::Spinner.new(':spinner :title', format: :dots_2)
+      end
+
+      def validate_job
+        @initialization_spinner.update(title: 'Validating job')
+        @job.job_definition.validate!
+      end
+
+      def instantiate_connectors
+        @initialization_spinner.update(title: 'Initializing connectors')
+        @extractor = @job.instantiate_extractor
+        @transformers = @job.instantiate_transformers
+        @loader = @job.instantiate_loader
+      end
+
+      def prepare_job
+        @initialization_spinner.update(title: 'Preparing job')
+        @job_logger.start
+        @loader.start
+
+        @initialization_spinner.update(title: 'Preparing extraction')
+        @initialization_spinner.auto_spin
+        @extractor.prepare
+        @initialization_spinner.success("(#{'successful'.green})")
+        Chronicle::ETL::Logger.info("\n")
+      end
+
+      def prepare_ui
+        total = @extractor.results_count
+        @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+        Chronicle::ETL::Logger.attach_to_ui(@progress_bar)
+      end
+
+      def run_extraction
+        # Pattern based on Kiba's StreamingRunner
+        # https://github.com/thbar/kiba/blob/master/lib/kiba/streaming_runner.rb
+        stream = extractor_stream
+        recurser = ->(s, t) { transform_stream(s, t) }
+        @transformers.reduce(stream, &recurser).each do |record|
+          Chronicle::ETL::Logger.debug(tty_log_transformation(record))
+          @job_logger.log_transformation(record)
+          @progress_bar.increment
+          load_record(record)
+        end
+
+        @progress_bar.finish
+
+        # This is typically a slow method (writing to stdout, writing a big file, etc)
+        # TODO: consider adding a spinner?
+        @loader.finish
+        @job_logger.finish
+      end
+
+      # Initial steam of extracted data, wrapped in a Record class
+      def extractor_stream
+        Enumerator.new do |y|
+          @extractor.extract do |extraction|
+            record = Chronicle::ETL::Record.new(data: extraction.data, extraction: extraction)
+            y << record
+          end
+        end
+      end
+
+      # For a given stream of records and a given transformer,
+      # returns a new stream of transformed records and finally
+      # calls the finish method on the transformer
+      def transform_stream(stream, transformer)
+        Enumerator.new do |y|
+          stream.each do |record|
+            transformer.call(record) do |transformed_record|
+              y << transformed_record
+            end
+          end
+
+          transformer.call_finish do |transformed_record|
+            y << transformed_record
+          end
+        end
+      end
+
+      def load_record(record)
+        @loader.load(record.data) unless @job.dry_run?
+      end
+
+      def finish_job
+        @job_logger.save
+        @progress_bar&.finish
+        Chronicle::ETL::Logger.detach_from_ui
+        Chronicle::ETL::Logger.info(tty_log_completion)
+      end
+
+      def tty_log_job_initialize
+        output = 'Beginning job '
+        output += "'#{@job.name}'".bold if @job.name
+        output
+      end
+
+      def tty_log_transformation(record)
+        output = '  ✓'.green
+        output + " #{record}"
+      end
+
+      def tty_log_transformation_failure(exception, transformer)
+        output = '  ✖'.red
+        output + " Failed to transform #{transformer}. #{exception.message}"
+      end
+
+      def tty_log_completion
+        status = @job_logger.success ? 'Success' : 'Failed'
+        job_completion = @job_logger.success ? 'Completed' : 'Partially completed'
+        output = "\n#{job_completion} job"
+        output += " '#{@job.name}'".bold if @job.name
+        output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
+        output += "\n  Status:\t".light_black + status
+        output += "\n  Completed:\t".light_black + @job_logger.job_log.num_records_processed.to_s
+        if @job_logger.job_log.highest_timestamp
+          output += "\n  Latest:\t".light_black + @job_logger.job_log.highest_timestamp.iso8601.to_s
+        end
+        output
+      end
     end
-
-    @progress_bar.finish
-
-    # This is typically a slow method (writing to stdout, writing a big file, etc)
-    # TODO: consider adding a spinner?
-    @loader.finish
-    @job_logger.finish
-  end
-
-  def process_extraction(extraction)
-    # For each extraction from our extractor, we create a new tarnsformer
-    transformer = @job.instantiate_transformer(extraction)
-
-    # And then transform that record, logging it if we're in debug log level
-    record = transformer.transform
-    Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
-    @job_logger.log_transformation(transformer)
-
-    # Then send the results to the loader
-    @loader.load(record) unless @job.dry_run?
-  rescue Chronicle::ETL::TransformationError => e
-    # TODO: have an option to cancel job if we encounter an error
-    Chronicle::ETL::Logger.error(tty_log_transformation_failure(e, transformer))
-  end
-
-  def finish_job
-    @job_logger.save
-    @progress_bar&.finish
-    Chronicle::ETL::Logger.detach_from_ui
-    Chronicle::ETL::Logger.info(tty_log_completion)
-  end
-
-  def tty_log_job_initialize
-    output = "Beginning job "
-    output += "'#{@job.name}'".bold if @job.name
-    output
-  end
-
-  def tty_log_transformation(transformer)
-    output = "  ✓".green
-    output += " #{transformer}"
-  end
-
-  def tty_log_transformation_failure(exception, transformer)
-    output = "  ✖".red
-    output += " Failed to build #{transformer}. #{exception.message}"
-  end
-
-  def tty_log_completion
-    status = @job_logger.success ? 'Success' : 'Failed'
-    job_completion = @job_logger.success ? 'Completed' : 'Partially completed'
-    output = "\n#{job_completion} job"
-    output += " '#{@job.name}'".bold if @job.name
-    output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
-    output += "\n  Status:\t".light_black + status
-    output += "\n  Completed:\t".light_black + "#{@job_logger.job_log.num_records_processed}"
-    output += "\n  Latest:\t".light_black + "#{@job_logger.job_log.highest_timestamp.iso8601}" if @job_logger.job_log.highest_timestamp
-    output
   end
 end

data/lib/chronicle/etl/secrets.rb

@@ -1,4 +1,4 @@
-require "active_support/core_ext/hash/keys"
+require 'active_support/core_ext/hash/keys'
 
 module Chronicle
   module ETL
@@ -8,7 +8,7 @@ module Chronicle
 
       # Whether a given namespace exists
       def exists?(namespace)
-        Chronicle::ETL::Config.exists?("secrets", namespace)
+        Chronicle::ETL::Config.exists?('secrets', namespace)
       end
 
       # Save a setting to a namespaced config file
@@ -47,7 +47,7 @@ module Chronicle
 
       # Read secrets from a config file
       def read(namespace)
-        definition = Chronicle::ETL::Config.load("secrets", namespace)
+        definition = Chronicle::ETL::Config.load('secrets', namespace)
         definition[:secrets] || {}
       end
 
@@ -57,7 +57,7 @@ module Chronicle
           secrets: (secrets || {}).transform_keys(&:to_s),
           chronicle_etl_version: Chronicle::ETL::VERSION
         }
-        Chronicle::ETL::Config.write("secrets", namespace, data)
+        Chronicle::ETL::Config.write('secrets', namespace, data)
       end
 
       # Which config files are available in ~/.config/chronicle/etl/secrets

data/lib/chronicle/etl/transformers/buffer_transformer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class BufferTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :buffer
+        r.description = 'by buffering'
+      end
+
+      setting :size, default: 10, description: 'The size of the buffer'
+
+      def transform(record)
+        stash_record(record)
+
+        # FIXME: this doesn't seem to be working with the runner
+        return if @stashed_records.size < @config.size
+
+        # FIXME: this will result in the wrong extraction being associated with
+        # the batch of flushed records
+        flush_stashed_records.map(&:data)
+      end
+
+      def finish
+        flush_stashed_records
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/chronicle_transformer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Chronicle
+  module ETL
+    class ChronicleTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = :chronicle
+        r.description = 'records to Chronicle schema'
+      end
+
+      def transform(record)
+        converter_klass = find_converter(record.extraction)
+        # TODO: handle missing converter
+
+        converter_klass.new.call(record) do |transformed_record|
+          yield transformed_record.data
+        end
+      end
+
+      private
+
+      def find_converter(extraction)
+        Chronicle::ETL::Registry::Connectors.find_converter_for_source(
+          source: extraction.source,
+          type: extraction.type,
+          strategy: extraction.strategy,
+          target: :chronicle
+        )&.klass
+      end
+    end
+  end
+end