chronicle-etl 0.3.1 → 0.4.2

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -9,59 +9,40 @@ module Chronicle
         r.description = 'an ASCII table'
       end
 
-      DEFAULT_OPTIONS = {
-        fields_limit: nil,
-        fields_exclude: ['lids', 'type'],
-        fields_include: [],
-        truncate_values_at: nil,
-        table_renderer: :basic
-      }.freeze
-
-      def initialize(options={})
-        @options = options.reverse_merge(DEFAULT_OPTIONS)
-        @records = []
-      end
+      setting :truncate_values_at, default: 40
+      setting :table_renderer, default: :basic
+      setting :fields_exclude, default: ['lids', 'type']
+      setting :header_row, default: true
 
       def load(record)
-        @records << record.to_h_flattened
+        records << record.to_h_flattened
       end
 
       def finish
-        return if @records.empty?
+        return if records.empty?
 
-        headers = build_headers(@records)
-        rows = build_rows(@records, headers)
+        headers = build_headers(records)
+        rows = build_rows(records, headers)
 
-        @table = TTY::Table.new(header: headers, rows: rows)
+        @table = TTY::Table.new(header: (headers if @config.header_row), rows: rows)
         puts @table.render(
-          @options[:table_renderer].to_sym,
+          @config.table_renderer.to_sym,
           padding: [0, 2, 0, 0]
         )
       end
 
-      private
-
-      def build_headers(records)
-        headers =
-          if @options[:fields_include].any?
-            Set[*@options[:fields_include]]
-          else
-            # use all the keys of the flattened record hash
-            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
-          end
-
-        headers = headers.delete_if { |header| header.end_with?(*@options[:fields_exclude]) } if @options[:fields_exclude].any?
-        headers = headers.first(@options[:fields_limit]) if @options[:fields_limit]
-
-        headers.to_a.map(&:to_sym)
+      def records
+        @records ||= []
       end
 
+      private
+
       def build_rows(records, headers)
         records.map do |record|
-          values = record.values_at(*headers).map{|value| value.to_s }
-
-          if @options[:truncate_values_at]
-            values = values.map{ |value| value.truncate(@options[:truncate_values_at]) }
+          values = record.transform_keys(&:to_sym).values_at(*headers).map{|value| value.to_s }
+          values = values.map { |value| force_utf8(value) }
+          if @config.truncate_values_at
+            values = values.map{ |value| value.truncate(@config.truncate_values_at) }
           end
 
           values

data/lib/chronicle/etl/logger.rb

@@ -8,11 +8,11 @@ module Chronicle
       WARN = 2
       ERROR = 3
       FATAL = 4
+      SILENT = 5
 
       attr_accessor :log_level
 
       @log_level = INFO
-      @destination = $stderr
 
       def output message, level
         return unless level >= @log_level
@@ -20,10 +20,14 @@ module Chronicle
         if @progress_bar
           @progress_bar.log(message)
         else
-          @destination.puts(message)
+          $stderr.puts(message)
         end
       end
 
+      def fatal(message)
+        output(message, FATAL)
+      end
+
       def error(message)
         output(message, ERROR)
       end

data/lib/chronicle/etl/models/base.rb

@@ -5,6 +5,9 @@ module Chronicle
     module Models
       # Represents a record that's been transformed by a Transformer and
       # ready to be loaded. Loosely based on ActiveModel.
+      #
+      # @todo Experiment with just mixing in ActiveModel instead of this
+      #   this reimplementation
       class Base
         ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
         ASSOCIATIONS = [].freeze

data/lib/chronicle/etl/models/entity.rb

@@ -5,13 +5,19 @@ module Chronicle
     module Models
       class Entity < Chronicle::ETL::Models::Base
         TYPE = 'entities'.freeze
-        ATTRIBUTES = [:title, :body, :represents, :slug, :myself, :metadata].freeze
+        ATTRIBUTES = [:title, :body, :provider_url, :represents, :slug, :myself, :metadata].freeze
+
+        # TODO: This desperately needs a validation system
         ASSOCIATIONS = [
+          :involvements, # inverse of activity's `involved`
+
           :attachments,
           :abouts,
+          :aboutables, # inverse of above
           :depicts,
           :consumers,
-          :contains
+          :contains,
+          :containers # inverse of above
         ].freeze  # TODO: add these to reflect Chronicle Schema
 
         attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)

data/lib/chronicle/etl/models/raw.rb

@@ -0,0 +1,26 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      # A record from an extraction with no processing or normalization applied
+      class Raw
+        TYPE = 'raw'
+
+        attr_accessor :raw_data
+
+        def initialize(raw_data)
+          @raw_data = raw_data
+        end
+
+        def to_h
+          @raw_data.to_h
+        end
+
+        def to_h_flattened
+          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -3,6 +3,7 @@ module Chronicle
     module Registry
       # Records details about a connector such as its provider and a description
       class ConnectorRegistration
+        # FIXME: refactor custom accessor methods later in file
         attr_accessor :identifier, :provider, :klass, :description
 
         def initialize(klass)
@@ -43,6 +44,11 @@ module Chronicle
           @provider || (built_in? ? 'chronicle' : '')
         end
 
+        # TODO: allow overriding here. Maybe through self-registration process
+        def plugin
+          @provider
+        end
+
         def descriptive_phrase
           prefix = case phase
           when :extractor

data/lib/chronicle/etl/registry/plugin_registry.rb

@@ -0,0 +1,70 @@
+require 'rubygems'
+require 'rubygems/command'
+require 'rubygems/commands/install_command'
+require 'rubygems/uninstaller'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Responsible for managing plugins available to chronicle-etl
+      #
+      # @todo Better validation for whether a gem is actually a plugin
+      # @todo Add ways to load a plugin that don't require a gem on rubygems.org
+      module PluginRegistry
+        # Does this plugin exist?
+        def self.exists?(name)
+          # TODO: implement this. Could query rubygems.org or have a
+          # hardcoded approved list
+          true
+        end
+
+        # All versions of all plugins currently installed
+        def self.all_installed
+          # TODO: add check for chronicle-etl dependency
+          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" }
+        end
+
+        # Latest version of each installed plugin
+        def self.all_installed_latest
+          all_installed.group_by(&:name)
+            .transform_values { |versions| versions.sort_by(&:version).reverse.first }
+            .values
+        end
+
+        # Activate a plugin with given name by `require`ing it
+        def self.activate(name)
+          # By default, activates the latest available version of a gem
+          # so don't have to run Kernel#gem separately
+          require "chronicle/#{name}"
+        rescue LoadError
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin #{name} couldn't be loaded" if exists?(name)
+
+          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist"
+        end
+
+        # Install a plugin to local gems
+        def self.install(name)
+          gem_name = "chronicle-#{name}"
+          raise(Chronicle::ETL::PluginNotAvailableError.new(gem_name), "Plugin #{name} doesn't exist") unless exists?(gem_name)
+
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          Gem.install(gem_name)
+        rescue Gem::UnsatisfiableDependencyError
+          # TODO: we need to catch a lot more than this here
+          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist"
+        end
+
+        # Uninstall a plugin
+        def self.uninstall(name)
+          gem_name = "chronicle-#{name}"
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          uninstaller = Gem::Uninstaller.new(gem_name)
+          uninstaller.uninstall
+        rescue Gem::InstallError
+          # TODO: strengthen this exception handling
+          raise(Chronicle::ETL::PluginError.new(name), "Plugin #{name} wasn't uninstalled")
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/registry.rb

@@ -20,28 +20,40 @@ module Chronicle
           end
         end
 
-        def install_connector name
-          gem_name = "chronicle-#{name}"
-          Gem.install(gem_name)
+        def register connector
+          connectors << connector
         end
 
-        def register connector
+        def connectors
           @connectors ||= []
-          @connectors << connector
         end
 
         def find_by_phase_and_identifier(phase, identifier)
-          connector = find_within_loaded_connectors(phase, identifier)
-          unless connector
-            # Only load external connectors (slow) if not found in built-in connectors
-            load_all!
-            connector = find_within_loaded_connectors(phase, identifier)
+          # Simple case: built in connector
+          connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
+          return connector if connector
+
+          # if not available in built-in connectors, try to activate a
+          # relevant plugin and try again
+          if identifier.include?(":")
+            plugin, name = identifier.split(":")
+          else
+            # This case handles the case where the identifier is a 
+            # shorthand (ie `imessage`) because there's only one default
+            # connector.
+            plugin = identifier
           end
-          connector || raise(ConnectorNotAvailableError.new("Connector '#{identifier}' not found"))
-        end
 
-        def find_within_loaded_connectors(phase, identifier)
-          @connectors.find { |c| c.phase == phase && c.identifier == identifier }
+          PluginRegistry.activate(plugin)
+
+          candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }
+          # if no name given, just use first connector with right phase/plugin
+          # TODO: set up a property for connectors to specify that they're the
+          # default connector for the plugin
+          candidates = candidates.select { |c| c.identifier == name } if name
+          connector = candidates.first
+
+          connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
         end
       end
     end
@@ -50,3 +62,4 @@ end
 
 require_relative 'self_registering'
 require_relative 'connector_registration'
+require_relative 'plugin_registry'

data/lib/chronicle/etl/runner.rb

@@ -8,19 +8,41 @@ class Chronicle::ETL::Runner
   end
 
   def run!
-    extractor = @job.instantiate_extractor
-    loader = @job.instantiate_loader
+    validate_job
+    instantiate_connectors
+    prepare_job
+    prepare_ui
+    run_extraction
+    finish_job
+  end
+
+  private
+
+  def validate_job
+    @job.job_definition.validate!
+  end
 
+  def instantiate_connectors
+    @extractor = @job.instantiate_extractor
+    @loader = @job.instantiate_loader
+  end
+
+  def prepare_job
+    Chronicle::ETL::Logger.info(tty_log_job_start)
     @job_logger.start
-    loader.start
+    @loader.start
+    @extractor.prepare
+  end
 
-    extractor.prepare
-    total = extractor.results_count
+  def prepare_ui
+    total = @extractor.results_count
     @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
     Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
+  end
 
-    Chronicle::ETL::Logger.info(tty_log_job_start)
-    extractor.extract do |extraction|
+  # TODO: refactor this further
+  def run_extraction
+    @extractor.extract do |extraction|
       unless extraction.is_a?(Chronicle::ETL::Extraction)
         raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
       end
@@ -28,14 +50,10 @@ class Chronicle::ETL::Runner
       transformer = @job.instantiate_transformer(extraction)
       record = transformer.transform
 
-      unless record.is_a?(Chronicle::ETL::Models::Base)
-        raise Chronicle::ETL::RunnerTypeError, "Transformed data should be a type of Chronicle::ETL::Models"
-      end
-
       Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
       @job_logger.log_transformation(transformer)
 
-      loader.load(record) unless @job.dry_run?
+      @loader.load(record) unless @job.dry_run?
     rescue Chronicle::ETL::TransformationError => e
       Chronicle::ETL::Logger.error(tty_log_transformation_failure(e))
     ensure
@@ -43,22 +61,22 @@ class Chronicle::ETL::Runner
     end
 
     @progress_bar.finish
-    loader.finish
+    @loader.finish
     @job_logger.finish
   rescue Interrupt
     Chronicle::ETL::Logger.error("\n#{'Job interrupted'.red}")
     @job_logger.error
   rescue StandardError => e
     raise e
-  ensure
+  end
+
+  def finish_job
     @job_logger.save
-    @progress_bar.finish
+    @progress_bar&.finish
     Chronicle::ETL::Logger.detach_from_progress_bar
     Chronicle::ETL::Logger.info(tty_log_completion)
   end
 
-  private
-
   def tty_log_job_start
     output = "Beginning job "
     output += "'#{@job.name}'".bold if @job.name

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -1,6 +1,12 @@
 module Chronicle
   module ETL
     class JSONAPISerializer < Chronicle::ETL::Serializer
+      def initialize(*args)
+        super
+
+        raise(SerializationError, "Record must be a subclass of Chronicle::ETL::Model::Base") unless @record.is_a?(Chronicle::ETL::Models::Base)
+      end
+
       def serializable_hash
         @record
           .identifier_hash

data/lib/chronicle/etl/serializers/raw_serializer.rb

@@ -0,0 +1,10 @@
+module Chronicle
+  module ETL
+    # Take a Raw model and output `raw_data` as a hash
+    class RawSerializer < Chronicle::ETL::Serializer
+      def serializable_hash
+        @record.to_h
+      end
+    end
+  end
+end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -24,4 +24,5 @@ module Chronicle
   end
 end
 
-require_relative 'jsonapi_serializer'
+require_relative 'jsonapi_serializer'
+require_relative 'raw_serializer'

data/lib/chronicle/etl/transformers/image_file_transformer.rb

@@ -19,20 +19,14 @@ module Chronicle
         r.description = 'an image file'
       end
 
-      DEFAULT_OPTIONS = {
-        timestamp_strategy: 'file_mtime',
-        id_strategy: 'file_hash',
-        verb: 'photographed',
-
-        # EXIF tags often don't have timezones
-        timezone_default: 'Eastern Time (US & Canada)',
-        include_image_data: true
-      }.freeze
-
-      def initialize(*args)
-        super(*args)
-        @options = @options.reverse_merge(DEFAULT_OPTIONS)
-      end
+      setting :timestamp_strategy, default: 'file_mtime'
+      setting :id_strategy, default: 'file_hash'
+      setting :verb, default: 'photographed'
+      # EXIF tags often don't have timezones
+      setting :timezone_default, default: 'Eastern Time (US & Canada)'
+      setting :include_image_data, default: true
+      setting :actor
+      setting :involved
 
       def transform
         # FIXME: set @filename; use block for reading file when necessary
@@ -48,7 +42,7 @@ module Chronicle
 
       def id
         @id ||= begin
-          id = build_with_strategy(field: :id, strategy: @options[:id_strategy])
+          id = build_with_strategy(field: :id, strategy: @config.id_strategy)
           raise UntransformableRecordError.new("Could not build id", transformation: self) unless id
 
           id
@@ -57,7 +51,7 @@ module Chronicle
 
       def timestamp
         @timestamp ||= begin
-          ts = build_with_strategy(field: :timestamp, strategy: @options[:timestamp_strategy])
+          ts = build_with_strategy(field: :timestamp, strategy: @config.timestamp_strategy)
           raise UntransformableRecordError.new("Could not build timestamp", transformation: self) unless ts
 
           ts
@@ -68,8 +62,8 @@ module Chronicle
 
       def build_created(file)
         record = ::Chronicle::ETL::Models::Activity.new
-        record.verb = @options[:verb]
-        record.provider = @options[:provider]
+        record.verb = @config.verb
+        record.provider = @config.provider
         record.provider_id = id
         record.end_at = timestamp
         record.dedupe_on = [[:provider_id, :verb, :provider]]
@@ -84,24 +78,24 @@ module Chronicle
       def build_actor
         actor = ::Chronicle::ETL::Models::Entity.new
         actor.represents = 'identity'
-        actor.provider = @options[:actor][:provider]
-        actor.slug = @options[:actor][:slug]
+        actor.provider = @config.actor[:provider]
+        actor.slug = @config.actor[:slug]
         actor.dedupe_on = [[:provider, :slug, :represents]]
         actor
       end
 
       def build_image
         image = ::Chronicle::ETL::Models::Entity.new
-        image.represents = @options[:involved][:represents]
+        image.represents = @config.involved[:represents]
         image.title = build_title
         image.body = exif['Description']
-        image.provider = @options[:involved][:provider]
+        image.provider = @config.involved[:provider]
         image.provider_id = id
         image.assign_attributes(build_gps)
         image.dedupe_on = [[:provider, :provider_id, :represents]]
 
-        if @options[:ocr_strategy]
-          ocr_text = build_with_strategy(field: :ocr, strategy: @options[:ocr_strategy])
+        if @config.ocr_strategy
+          ocr_text = build_with_strategy(field: :ocr, strategy: @config.ocr_strategy)
           image.metadata[:ocr_text] = ocr_text if ocr_text
         end
 
@@ -111,7 +105,7 @@ module Chronicle
         image.depicts = build_people_depicted(names)
         image.abouts = build_keywords(tags)
 
-        if @options[:include_image_data]
+        if @config.include_image_data
           attachment = ::Chronicle::ETL::Models::Attachment.new
           attachment.data = build_image_data
           image.attachments = [attachment]
@@ -124,7 +118,7 @@ module Chronicle
         topics.map do |topic|
           t = ::Chronicle::ETL::Models::Entity.new
           t.represents = 'topic'
-          t.provider = @options[:involved][:provider]
+          t.provider = @config.involved[:provider]
           t.title = topic
           t.slug = topic.parameterize
           t.dedupe_on = [[:provider, :represents, :slug]]
@@ -136,7 +130,7 @@ module Chronicle
         names.map do |name|
           identity = ::Chronicle::ETL::Models::Entity.new
           identity.represents = 'identity'
-          identity.provider = @options[:involved][:provider]
+          identity.provider = @config.involved[:provider]
           identity.slug = name.parameterize
           identity.title = name
           identity.dedupe_on = [[:provider, :represents, :slug]]
@@ -199,7 +193,7 @@ module Chronicle
         elsif false
           # TODO: support option of using GPS coordinates to determine timezone
         else
-          zone = ActiveSupport::TimeZone.new(@options[:timezone_default])
+          zone = ActiveSupport::TimeZone.new(@config.timezone_default)
           timestamp = zone.parse(timestamp.asctime)
         end
 

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -7,7 +7,7 @@ module Chronicle
       end
 
       def transform
-        Chronicle::ETL::Models::Generic.new(@extraction.data)
+        Chronicle::ETL::Models::Raw.new(@extraction.data)
       end
 
       def timestamp; end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -3,14 +3,15 @@ module Chronicle
     # Abstract class representing an Transformer for an ETL job
     class Transformer
       extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
 
       # Construct a new instance of this transformer. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Transformer
-      def initialize(options = {}, extraction)
-        @options = options
+      def initialize(extraction, options = {})
         @extraction = extraction
+        apply_options(options)
       end
 
       # @abstract Subclass is expected to implement #transform

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.3.1"
+    VERSION = "0.4.2"
   end
 end

data/lib/chronicle/etl.rb

@@ -1,24 +1,32 @@
 require_relative 'etl/registry/registry'
 require_relative 'etl/config'
+require_relative 'etl/configurable'
 require_relative 'etl/exceptions'
 require_relative 'etl/extraction'
-require_relative 'etl/extractors/extractor'
 require_relative 'etl/job_definition'
 require_relative 'etl/job_log'
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
-require_relative 'etl/loaders/loader'
 require_relative 'etl/logger'
 require_relative 'etl/models/activity'
 require_relative 'etl/models/attachment'
 require_relative 'etl/models/base'
+require_relative 'etl/models/raw'
 require_relative 'etl/models/entity'
-require_relative 'etl/models/generic'
 require_relative 'etl/runner'
 require_relative 'etl/serializers/serializer'
-require_relative 'etl/transformers/transformer'
 require_relative 'etl/utils/binary_attachments'
 require_relative 'etl/utils/hash_utilities'
 require_relative 'etl/utils/text_recognition'
 require_relative 'etl/utils/progress_bar'
 require_relative 'etl/version'
+
+require_relative 'etl/extractors/extractor'
+require_relative 'etl/loaders/loader'
+require_relative 'etl/transformers/transformer'
+
+begin
+  require 'pry'
+rescue LoadError
+  # Pry not available
+end