chronicle-etl 0.4.0 → 0.4.3

data/lib/chronicle/etl/loaders/json_loader.rb

@@ -0,0 +1,44 @@
+module Chronicle
+  module ETL
+    class JSONLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'json'
+      end
+
+      setting :serializer
+      setting :output, default: $stdout
+
+      def start
+        if @config.output == $stdout
+          @output = @config.output
+        else
+          @output = File.open(@config.output, "w")
+        end
+      end
+
+      def load(record)
+        serialized = serializer.serialize(record)
+
+        # When dealing with raw data, we can get improperly encoded strings
+        # (eg from sqlite database columns). We force conversion to UTF-8
+        # before converting into JSON
+        encoded = serialized.transform_values do |value|
+          next value unless value.is_a?(String)
+
+          force_utf8(value)
+        end
+        @output.puts encoded.to_json
+      end
+
+      def finish
+        @output.close
+      end
+
+      private
+
+      def serializer
+        @config.serializer || Chronicle::ETL::RawSerializer
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,11 +1,17 @@
+require_relative 'helpers/encoding_helper'
+
 module Chronicle
   module ETL
     # Abstract class representing a Loader for an ETL job
     class Loader
       extend Chronicle::ETL::Registry::SelfRegistering
       include Chronicle::ETL::Configurable
+      include Chronicle::ETL::Loaders::Helpers::EncodingHelper
 
       setting :output
+      setting :fields
+      setting :fields_limit, default: nil
+      setting :fields_exclude
 
       # Construct a new instance of this loader. Options are passed in from a Runner
       # == Parameters:
@@ -25,11 +31,28 @@ module Chronicle
 
       # Called once there are no more records to process
       def finish; end
+
+      private
+
+      def build_headers(records)
+        headers =
+          if @config.fields && @config.fields.any?
+            Set[*@config.fields]
+          else
+            # use all the keys of the flattened record hash
+            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
+          end
+
+        headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) }
+        headers = headers.first(@config.fields_limit) if @config.fields_limit
+
+        headers.to_a.map(&:to_sym)
+      end
     end
   end
 end
 
 require_relative 'csv_loader'
+require_relative 'json_loader'
 require_relative 'rest_loader'
-require_relative 'stdout_loader'
 require_relative 'table_loader'

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -9,51 +9,38 @@ module Chronicle
         r.description = 'an ASCII table'
       end
 
-      setting :fields_limit, default: nil
-      setting :fields_exclude, default: ['lids', 'type']
-      setting :fields_include, default: []
       setting :truncate_values_at, default: 40
       setting :table_renderer, default: :basic
+      setting :fields_exclude, default: ['lids', 'type']
+      setting :header_row, default: true
 
       def load(record)
-        @records ||= []
-        @records << record.to_h_flattened
+        records << record.to_h_flattened
       end
 
       def finish
-        return if @records.empty?
+        return if records.empty?
 
-        headers = build_headers(@records)
-        rows = build_rows(@records, headers)
+        headers = build_headers(records)
+        rows = build_rows(records, headers)
 
-        @table = TTY::Table.new(header: headers, rows: rows)
+        @table = TTY::Table.new(header: (headers if @config.header_row), rows: rows)
         puts @table.render(
           @config.table_renderer.to_sym,
           padding: [0, 2, 0, 0]
         )
       end
 
-      private
-
-      def build_headers(records)
-        headers =
-          if @config.fields_include.any?
-            Set[*@config.fields_include]
-          else
-            # use all the keys of the flattened record hash
-            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
-          end
-
-        headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) } if @config.fields_exclude.any?
-        headers = headers.first(@config.fields_limit) if @config.fields_limit
-
-        headers.to_a.map(&:to_sym)
+      def records
+        @records ||= []
       end
 
+      private
+
       def build_rows(records, headers)
         records.map do |record|
-          values = record.values_at(*headers).map{|value| value.to_s }
-
+          values = record.transform_keys(&:to_sym).values_at(*headers).map{|value| value.to_s }
+          values = values.map { |value| force_utf8(value) }
           if @config.truncate_values_at
             values = values.map{ |value| value.truncate(@config.truncate_values_at) }
           end

data/lib/chronicle/etl/logger.rb

@@ -8,11 +8,11 @@ module Chronicle
       WARN = 2
       ERROR = 3
       FATAL = 4
+      SILENT = 5
 
       attr_accessor :log_level
 
       @log_level = INFO
-      @destination = $stderr
 
       def output message, level
         return unless level >= @log_level
@@ -20,10 +20,14 @@ module Chronicle
         if @progress_bar
           @progress_bar.log(message)
         else
-          @destination.puts(message)
+          $stderr.puts(message)
         end
       end
 
+      def fatal(message)
+        output(message, FATAL)
+      end
+
       def error(message)
         output(message, ERROR)
       end

data/lib/chronicle/etl/models/base.rb

@@ -5,6 +5,9 @@ module Chronicle
     module Models
       # Represents a record that's been transformed by a Transformer and
       # ready to be loaded. Loosely based on ActiveModel.
+      #
+      # @todo Experiment with just mixing in ActiveModel instead of this
+      #   this reimplementation
       class Base
         ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
         ASSOCIATIONS = [].freeze

data/lib/chronicle/etl/models/entity.rb

@@ -5,13 +5,19 @@ module Chronicle
     module Models
       class Entity < Chronicle::ETL::Models::Base
         TYPE = 'entities'.freeze
-        ATTRIBUTES = [:title, :body, :represents, :slug, :myself, :metadata].freeze
+        ATTRIBUTES = [:title, :body, :provider_url, :represents, :slug, :myself, :metadata].freeze
+
+        # TODO: This desperately needs a validation system
         ASSOCIATIONS = [
+          :involvements, # inverse of activity's `involved`
+
           :attachments,
           :abouts,
+          :aboutables, # inverse of above
           :depicts,
           :consumers,
-          :contains
+          :contains,
+          :containers # inverse of above
         ].freeze  # TODO: add these to reflect Chronicle Schema
 
         attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)

data/lib/chronicle/etl/models/raw.rb

@@ -0,0 +1,26 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      # A record from an extraction with no processing or normalization applied
+      class Raw
+        TYPE = 'raw'
+
+        attr_accessor :raw_data
+
+        def initialize(raw_data)
+          @raw_data = raw_data
+        end
+
+        def to_h
+          @raw_data.to_h
+        end
+
+        def to_h_flattened
+          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -44,6 +44,11 @@ module Chronicle
           @provider || (built_in? ? 'chronicle' : '')
         end
 
+        # TODO: allow overriding here. Maybe through self-registration process
+        def plugin
+          @provider
+        end
+
         def descriptive_phrase
           prefix = case phase
           when :extractor

data/lib/chronicle/etl/registry/plugin_registry.rb

@@ -0,0 +1,75 @@
+require 'rubygems'
+require 'rubygems/command'
+require 'rubygems/commands/install_command'
+require 'rubygems/uninstaller'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Responsible for managing plugins available to chronicle-etl
+      #
+      # @todo Better validation for whether a gem is actually a plugin
+      # @todo Add ways to load a plugin that don't require a gem on rubygems.org
+      module PluginRegistry
+        # Does this plugin exist?
+        def self.exists?(name)
+          # TODO: implement this. Could query rubygems.org or have a
+          # hardcoded approved list
+          true
+        end
+
+        # All versions of all plugins currently installed
+        def self.all_installed
+          # TODO: add check for chronicle-etl dependency
+          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" }
+        end
+
+        # Latest version of each installed plugin
+        def self.all_installed_latest
+          all_installed.group_by(&:name)
+            .transform_values { |versions| versions.sort_by(&:version).reverse.first }
+            .values
+        end
+
+        # Activate a plugin with given name by `require`ing it
+        def self.activate(name)
+          # By default, activates the latest available version of a gem
+          # so don't have to run Kernel#gem separately
+          require "chronicle/#{name}"
+        rescue Gem::ConflictError => e
+          # TODO: figure out if there's more we can do here
+          raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
+        rescue LoadError => e
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded" if exists?(name)
+
+          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist"
+        end
+
+        # Install a plugin to local gems
+        def self.install(name)
+          gem_name = "chronicle-#{name}"
+          raise(Chronicle::ETL::PluginNotAvailableError.new(gem_name), "Plugin #{name} doesn't exist") unless exists?(gem_name)
+
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          Gem.install(gem_name)
+
+          activate(name)
+        rescue Gem::UnsatisfiableDependencyError
+          # TODO: we need to catch a lot more than this here
+          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} could not be installed."
+        end
+
+        # Uninstall a plugin
+        def self.uninstall(name)
+          gem_name = "chronicle-#{name}"
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          uninstaller = Gem::Uninstaller.new(gem_name)
+          uninstaller.uninstall
+        rescue Gem::InstallError
+          # TODO: strengthen this exception handling
+          raise(Chronicle::ETL::PluginError.new(name), "Plugin #{name} wasn't uninstalled")
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/registry.rb

@@ -20,28 +20,40 @@ module Chronicle
           end
         end
 
-        def install_connector name
-          gem_name = "chronicle-#{name}"
-          Gem.install(gem_name)
+        def register connector
+          connectors << connector
         end
 
-        def register connector
+        def connectors
           @connectors ||= []
-          @connectors << connector
         end
 
         def find_by_phase_and_identifier(phase, identifier)
-          connector = find_within_loaded_connectors(phase, identifier)
-          unless connector
-            # Only load external connectors (slow) if not found in built-in connectors
-            load_all!
-            connector = find_within_loaded_connectors(phase, identifier)
+          # Simple case: built in connector
+          connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
+          return connector if connector
+
+          # if not available in built-in connectors, try to activate a
+          # relevant plugin and try again
+          if identifier.include?(":")
+            plugin, name = identifier.split(":")
+          else
+            # This case handles the case where the identifier is a 
+            # shorthand (ie `imessage`) because there's only one default
+            # connector.
+            plugin = identifier
           end
-          connector || raise(ConnectorNotAvailableError.new("Connector '#{identifier}' not found"))
-        end
 
-        def find_within_loaded_connectors(phase, identifier)
-          @connectors.find { |c| c.phase == phase && c.identifier == identifier }
+          PluginRegistry.activate(plugin)
+
+          candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }
+          # if no name given, just use first connector with right phase/plugin
+          # TODO: set up a property for connectors to specify that they're the
+          # default connector for the plugin
+          candidates = candidates.select { |c| c.identifier == name } if name
+          connector = candidates.first
+
+          connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
         end
       end
     end
@@ -50,3 +62,4 @@ end
 
 require_relative 'self_registering'
 require_relative 'connector_registration'
+require_relative 'plugin_registry'

data/lib/chronicle/etl/runner.rb

@@ -8,19 +8,41 @@ class Chronicle::ETL::Runner
   end
 
   def run!
-    extractor = @job.instantiate_extractor
-    loader = @job.instantiate_loader
+    validate_job
+    instantiate_connectors
+    prepare_job
+    prepare_ui
+    run_extraction
+    finish_job
+  end
+
+  private
+
+  def validate_job
+    @job.job_definition.validate!
+  end
 
+  def instantiate_connectors
+    @extractor = @job.instantiate_extractor
+    @loader = @job.instantiate_loader
+  end
+
+  def prepare_job
+    Chronicle::ETL::Logger.info(tty_log_job_start)
     @job_logger.start
-    loader.start
+    @loader.start
+    @extractor.prepare
+  end
 
-    extractor.prepare
-    total = extractor.results_count
+  def prepare_ui
+    total = @extractor.results_count
     @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
     Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
+  end
 
-    Chronicle::ETL::Logger.info(tty_log_job_start)
-    extractor.extract do |extraction|
+  # TODO: refactor this further
+  def run_extraction
+    @extractor.extract do |extraction|
       unless extraction.is_a?(Chronicle::ETL::Extraction)
         raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
       end
@@ -28,14 +50,10 @@ class Chronicle::ETL::Runner
       transformer = @job.instantiate_transformer(extraction)
       record = transformer.transform
 
-      unless record.is_a?(Chronicle::ETL::Models::Base)
-        raise Chronicle::ETL::RunnerTypeError, "Transformed data should be a type of Chronicle::ETL::Models"
-      end
-
       Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
       @job_logger.log_transformation(transformer)
 
-      loader.load(record) unless @job.dry_run?
+      @loader.load(record) unless @job.dry_run?
     rescue Chronicle::ETL::TransformationError => e
       Chronicle::ETL::Logger.error(tty_log_transformation_failure(e))
     ensure
@@ -43,22 +61,22 @@ class Chronicle::ETL::Runner
     end
 
     @progress_bar.finish
-    loader.finish
+    @loader.finish
     @job_logger.finish
   rescue Interrupt
     Chronicle::ETL::Logger.error("\n#{'Job interrupted'.red}")
     @job_logger.error
   rescue StandardError => e
     raise e
-  ensure
+  end
+
+  def finish_job
     @job_logger.save
-    @progress_bar.finish
+    @progress_bar&.finish
     Chronicle::ETL::Logger.detach_from_progress_bar
     Chronicle::ETL::Logger.info(tty_log_completion)
   end
 
-  private
-
   def tty_log_job_start
     output = "Beginning job "
     output += "'#{@job.name}'".bold if @job.name

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -1,6 +1,12 @@
 module Chronicle
   module ETL
     class JSONAPISerializer < Chronicle::ETL::Serializer
+      def initialize(*args)
+        super
+
+        raise(SerializationError, "Record must be a subclass of Chronicle::ETL::Model::Base") unless @record.is_a?(Chronicle::ETL::Models::Base)
+      end
+
       def serializable_hash
         @record
           .identifier_hash

data/lib/chronicle/etl/serializers/raw_serializer.rb

@@ -0,0 +1,10 @@
+module Chronicle
+  module ETL
+    # Take a Raw model and output `raw_data` as a hash
+    class RawSerializer < Chronicle::ETL::Serializer
+      def serializable_hash
+        @record.to_h
+      end
+    end
+  end
+end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -24,4 +24,5 @@ module Chronicle
   end
 end
 
-require_relative 'jsonapi_serializer'
+require_relative 'jsonapi_serializer'
+require_relative 'raw_serializer'

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -7,7 +7,7 @@ module Chronicle
       end
 
       def transform
-        Chronicle::ETL::Models::Generic.new(@extraction.data)
+        Chronicle::ETL::Models::Raw.new(@extraction.data)
       end
 
       def timestamp; end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.4.0"
+    VERSION = "0.4.3"
   end
 end

data/lib/chronicle/etl.rb

@@ -3,23 +3,30 @@ require_relative 'etl/config'
 require_relative 'etl/configurable'
 require_relative 'etl/exceptions'
 require_relative 'etl/extraction'
-require_relative 'etl/extractors/extractor'
 require_relative 'etl/job_definition'
 require_relative 'etl/job_log'
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
-require_relative 'etl/loaders/loader'
 require_relative 'etl/logger'
 require_relative 'etl/models/activity'
 require_relative 'etl/models/attachment'
 require_relative 'etl/models/base'
+require_relative 'etl/models/raw'
 require_relative 'etl/models/entity'
-require_relative 'etl/models/generic'
 require_relative 'etl/runner'
 require_relative 'etl/serializers/serializer'
-require_relative 'etl/transformers/transformer'
 require_relative 'etl/utils/binary_attachments'
 require_relative 'etl/utils/hash_utilities'
 require_relative 'etl/utils/text_recognition'
 require_relative 'etl/utils/progress_bar'
 require_relative 'etl/version'
+
+require_relative 'etl/extractors/extractor'
+require_relative 'etl/loaders/loader'
+require_relative 'etl/transformers/transformer'
+
+begin
+  require 'pry'
+rescue LoadError
+  # Pry not available
+end