chronicle-etl 0.2.2 → 0.3.1

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,20 +1,71 @@
 require 'tty/table'
+require 'active_support/core_ext/string/filters'
+require 'active_support/core_ext/hash/reverse_merge'
 
 module Chronicle
   module ETL
     class TableLoader < Chronicle::ETL::Loader
-      def initialize(options)
-        super(options)
+      register_connector do |r|
+        r.description = 'an ASCII table'
       end
 
-      def load(result)
-        @table ||= TTY::Table.new(header: result.keys)
-        values = result.values.map{|x| x.to_s[0..30]}
-        @table << values
+      DEFAULT_OPTIONS = {
+        fields_limit: nil,
+        fields_exclude: ['lids', 'type'],
+        fields_include: [],
+        truncate_values_at: nil,
+        table_renderer: :basic
+      }.freeze
+
+      def initialize(options={})
+        @options = options.reverse_merge(DEFAULT_OPTIONS)
+        @records = []
+      end
+
+      def load(record)
+        @records << record.to_h_flattened
       end
 
       def finish
-        puts @table.render(:ascii, padding: [0, 1])
+        return if @records.empty?
+
+        headers = build_headers(@records)
+        rows = build_rows(@records, headers)
+
+        @table = TTY::Table.new(header: headers, rows: rows)
+        puts @table.render(
+          @options[:table_renderer].to_sym,
+          padding: [0, 2, 0, 0]
+        )
+      end
+
+      private
+
+      def build_headers(records)
+        headers =
+          if @options[:fields_include].any?
+            Set[*@options[:fields_include]]
+          else
+            # use all the keys of the flattened record hash
+            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
+          end
+
+        headers = headers.delete_if { |header| header.end_with?(*@options[:fields_exclude]) } if @options[:fields_exclude].any?
+        headers = headers.first(@options[:fields_limit]) if @options[:fields_limit]
+
+        headers.to_a.map(&:to_sym)
+      end
+
+      def build_rows(records, headers)
+        records.map do |record|
+          values = record.values_at(*headers).map{|value| value.to_s }
+
+          if @options[:truncate_values_at]
+            values = values.map{ |value| value.truncate(@options[:truncate_values_at]) }
+          end
+
+          values
+        end
       end
     end
   end

data/lib/chronicle/etl/logger.rb

@@ -0,0 +1,48 @@
+module Chronicle
+  module ETL
+    module Logger
+      extend self
+
+      DEBUG = 0
+      INFO = 1
+      WARN = 2
+      ERROR = 3
+      FATAL = 4
+
+      attr_accessor :log_level
+
+      @log_level = INFO
+      @destination = $stderr
+
+      def output message, level
+        return unless level >= @log_level
+
+        if @progress_bar
+          @progress_bar.log(message)
+        else
+          @destination.puts(message)
+        end
+      end
+
+      def error(message)
+        output(message, ERROR)
+      end
+
+      def info(message)
+        output(message, INFO)
+      end
+
+      def debug(message)
+        output(message, DEBUG)
+      end
+
+      def attach_to_progress_bar(progress_bar)
+        @progress_bar = progress_bar
+      end
+
+      def detach_from_progress_bar
+        @progress_bar = nil
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/activity.rb

@@ -0,0 +1,15 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Activity < Chronicle::ETL::Models::Base
+        TYPE = 'activities'.freeze
+        ATTRIBUTES = [:verb, :start_at, :end_at].freeze
+        ASSOCIATIONS = [:involved, :actor].freeze
+
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/attachment.rb

@@ -0,0 +1,14 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Attachment < Chronicle::ETL::Models::Base
+        TYPE = 'attachments'.freeze
+        ATTRIBUTES = [:url_original, :data].freeze
+
+        attr_accessor(*ATTRIBUTES)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/base.rb

@@ -0,0 +1,119 @@
+require 'digest'
+
+module Chronicle
+  module ETL
+    module Models
+      # Represents a record that's been transformed by a Transformer and
+      # ready to be loaded. Loosely based on ActiveModel.
+      class Base
+        ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
+        ASSOCIATIONS = [].freeze
+
+        attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
+
+        def initialize(attributes = {})
+          assign_attributes(attributes) if attributes
+          @dedupe_on = []
+          @metadata = {}
+        end
+
+        # A unique identifier for this model is formed from a type
+        # and either an id or lids.
+        def identifier_hash
+          {
+            type: self.class::TYPE,
+            id: @id,
+            lids: lids
+          }.compact
+        end
+
+        # Array of local ids that uniquely identify this record
+        def lids
+          @dedupe_on.map do |fields|
+            generate_lid(fields)
+          end.compact.uniq
+        end
+
+        # For a given set of fields of this model, generate a
+        # unique local id by hashing the field values
+        def generate_lid fields
+          raise ArgumentError.new("Must provide an array of symbolized fields") unless fields.is_a?(Array)
+
+          values = fields.sort.map do |field|
+            instance_variable = "@#{field.to_s}"
+            self.instance_variable_get(instance_variable)
+          end
+
+          return if values.any? { |e| e.nil? }
+
+          Digest::SHA256.hexdigest(values.join(","))
+        end
+
+        # Set of attribute names that this model has is Base's shared
+        # attributes combined with the child class's
+        def attribute_list
+          (ATTRIBUTES + self.class::ATTRIBUTES).uniq
+        end
+
+        # All of this record's attributes
+        def attributes
+          attributes = {}
+          attribute_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            attributes[attribute] = self.instance_variable_get(instance_variable)
+          end
+          attributes.compact
+        end
+
+        # All of this record's associations
+        def associations
+          association_list = ASSOCIATIONS + self.class::ASSOCIATIONS
+          attributes = {}
+          association_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            association = self.instance_variable_get(instance_variable)
+            attributes[attribute] = association if association
+          end
+          attributes.compact
+        end
+
+        def associations_hash
+          associations.map do |k, v|
+            if v.is_a?(Array)
+              [k, v.map(&:to_h)]
+            else
+              [k, v.to_h]
+            end
+          end.to_h
+        end
+
+        def meta_hash
+          {
+            meta: {
+              dedupe_on: @dedupe_on.map{|d| d.map(&:to_s).join(",")}
+            }
+          }
+        end
+
+        # FIXME: move this to a Utils module
+        def to_h_flattened
+          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
+        end
+
+        def to_h
+          identifier_hash
+            .merge(attributes)
+            .merge(associations_hash)
+            .merge(meta_hash)
+        end
+
+        def assign_attributes attributes
+          attributes.each do |k, v|
+            setter = :"#{k}="
+            public_send(setter, v) if respond_to? setter
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/entity.rb

@@ -0,0 +1,21 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Entity < Chronicle::ETL::Models::Base
+        TYPE = 'entities'.freeze
+        ATTRIBUTES = [:title, :body, :represents, :slug, :myself, :metadata].freeze
+        ASSOCIATIONS = [
+          :attachments,
+          :abouts,
+          :depicts,
+          :consumers,
+          :contains
+        ].freeze  # TODO: add these to reflect Chronicle Schema
+
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/generic.rb

@@ -0,0 +1,23 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Generic < Chronicle::ETL::Models::Base
+        TYPE = 'generic'
+
+        attr_accessor :properties
+
+        def initialize(properties = {})
+          @properties = properties
+          super
+        end
+
+        # Generic models have arbitrary attributes stored in @properties
+        def attributes
+          @properties.transform_keys(&:to_sym)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -0,0 +1,61 @@
+module Chronicle
+  module ETL
+    module Registry
+      # Records details about a connector such as its provider and a description
+      class ConnectorRegistration
+        attr_accessor :identifier, :provider, :klass, :description
+
+        def initialize(klass)
+          @klass = klass
+        end
+
+        def phase
+          if klass.ancestors.include? Chronicle::ETL::Extractor
+            :extractor
+          elsif klass.ancestors.include? Chronicle::ETL::Transformer
+            :transformer
+          elsif klass.ancestors.include? Chronicle::ETL::Loader
+            :loader
+          end
+        end
+
+        def to_s
+          "#{phase}-#{identifier}"
+        end
+
+        def built_in?
+          @klass.to_s.include? 'Chronicle::ETL'
+        end
+
+        def klass_name
+          @klass.to_s
+        end
+
+        def identifier
+          @identifier || @klass.to_s.split('::').last.gsub!(/(Extractor$|Loader$|Transformer$)/, '').downcase
+        end
+
+        def description
+          @description || @klass.to_s.split('::').last
+        end
+
+        def provider
+          @provider || (built_in? ? 'chronicle' : '')
+        end
+
+        def descriptive_phrase
+          prefix = case phase
+          when :extractor
+            "Extracts from"
+          when :transformer
+            "Transforms"
+          when :loader
+            "Loads to"
+          end
+
+          "#{prefix} #{description}"
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/registry.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+
+module Chronicle
+  module ETL
+    # A singleton class that acts as a registry of connector classes available for ETL jobs
+    module Registry
+      PHASES = [:extractor, :transformer, :loader]
+
+      class << self
+        attr_accessor :connectors
+
+        def load_all!
+          load_connectors_from_gems
+        end
+
+        def load_connectors_from_gems
+          Gem::Specification.filter{|s| s.name.match(/^chronicle/) }.each do |gem|
+            require_str = gem.name.gsub('chronicle-', 'chronicle/')
+            require require_str rescue LoadError
+          end
+        end
+
+        def install_connector name
+          gem_name = "chronicle-#{name}"
+          Gem.install(gem_name)
+        end
+
+        def register connector
+          @connectors ||= []
+          @connectors << connector
+        end
+
+        def find_by_phase_and_identifier(phase, identifier)
+          connector = find_within_loaded_connectors(phase, identifier)
+          unless connector
+            # Only load external connectors (slow) if not found in built-in connectors
+            load_all!
+            connector = find_within_loaded_connectors(phase, identifier)
+          end
+          connector || raise(ConnectorNotAvailableError.new("Connector '#{identifier}' not found"))
+        end
+
+        def find_within_loaded_connectors(phase, identifier)
+          @connectors.find { |c| c.phase == phase && c.identifier == identifier }
+        end
+      end
+    end
+  end
+end
+
+require_relative 'self_registering'
+require_relative 'connector_registration'

data/lib/chronicle/etl/registry/self_registering.rb

@@ -0,0 +1,25 @@
+require 'forwardable'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Gives a connector class the ability to let the Chronicle::ETL::Registry
+      # know about itself
+      module SelfRegistering
+        extend Forwardable
+
+        attr_accessor :connector_registration
+
+        def_delegators :@connector_registration, :description, :provider, :identifier
+
+        # Creates a ConnectorRegistration for this connector's details and register's it
+        # into the Registry
+        def register_connector
+          @connector_registration ||= ::Chronicle::ETL::Registry::ConnectorRegistration.new(self)
+          yield @connector_registration if block_given?
+          ::Chronicle::ETL::Registry.register(@connector_registration)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,46 +1,88 @@
 require 'colorize'
+require 'chronic_duration'
 
 class Chronicle::ETL::Runner
-  def initialize(options = {})
-    @options = options
+  def initialize(job)
+    @job = job
+    @job_logger = Chronicle::ETL::JobLogger.new(@job)
   end
 
   def run!
-    extractor = instantiate_klass(:extractor)
-    loader = instantiate_klass(:loader)
+    extractor = @job.instantiate_extractor
+    loader = @job.instantiate_loader
 
+    @job_logger.start
+    loader.start
+
+    extractor.prepare
     total = extractor.results_count
-    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+    @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+    Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
 
-    loader.start
+    Chronicle::ETL::Logger.info(tty_log_job_start)
+    extractor.extract do |extraction|
+      unless extraction.is_a?(Chronicle::ETL::Extraction)
+        raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
+      end
+
+      transformer = @job.instantiate_transformer(extraction)
+      record = transformer.transform
+
+      unless record.is_a?(Chronicle::ETL::Models::Base)
+        raise Chronicle::ETL::RunnerTypeError, "Transformed data should be a type of Chronicle::ETL::Models"
+      end
 
-    extractor.extract do |data, metadata|
-      transformer = instantiate_klass(:transformer, data)
-      transformed_data = transformer.transform
+      Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
+      @job_logger.log_transformation(transformer)
 
-      loader.load(transformed_data)
-      progress_bar.increment
+      loader.load(record) unless @job.dry_run?
+    rescue Chronicle::ETL::TransformationError => e
+      Chronicle::ETL::Logger.error(tty_log_transformation_failure(e))
+    ensure
+      @progress_bar.increment
     end
 
-    progress_bar.finish
+    @progress_bar.finish
     loader.finish
+    @job_logger.finish
+  rescue Interrupt
+    Chronicle::ETL::Logger.error("\n#{'Job interrupted'.red}")
+    @job_logger.error
+  rescue StandardError => e
+    raise e
+  ensure
+    @job_logger.save
+    @progress_bar.finish
+    Chronicle::ETL::Logger.detach_from_progress_bar
+    Chronicle::ETL::Logger.info(tty_log_completion)
   end
 
   private
 
-  def instantiate_klass(phase, *args)
-    klass = load_etl_class(phase, @options[phase][:name])
-    klass.new(@options[phase][:options], *args)
+  def tty_log_job_start
+    output = "Beginning job "
+    output += "'#{@job.name}'".bold if @job.name
+    output
+  end
+
+  def tty_log_transformation transformer
+    output = "  ✓".green
+    output += " #{transformer}"
+  end
+
+  def tty_log_transformation_failure exception
+    output = "  ✖".red
+    output += " Failed to build #{exception.transformation}. #{exception.message}"
   end
 
-  def load_etl_class(phase, identifier)
-    Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
-  rescue Chronicle::ETL::ProviderNotAvailableError => e
-    warn(e.message.red)
-    warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
-    exit(false)
-  rescue Chronicle::ETL::ConnectorNotAvailableError => e
-    warn(e.message.red)
-    exit(false)
+  def tty_log_completion
+    status = @job_logger.success ? 'Success' : 'Failed'
+    output = "\nCompleted job "
+    output += "'#{@job.name}'".bold if @job.name
+    output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
+    output += "\n  Status:\t".light_black + status
+    output += "\n  Completed:\t".light_black + "#{@job_logger.job_log.num_records_processed}"
+    output += "\n  Latest:\t".light_black + "#{@job_logger.job_log.highest_timestamp.iso8601}" if @job_logger.job_log.highest_timestamp
+    output
   end
 end

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -0,0 +1,25 @@
+module Chronicle
+  module ETL
+    class JSONAPISerializer < Chronicle::ETL::Serializer
+      def serializable_hash
+        @record
+          .identifier_hash
+          .merge({ attributes: @record.attributes })
+          .merge({ relationships: build_associations })
+          .merge(@record.meta_hash)
+      end
+
+      def build_associations
+        @record.associations.transform_values do |value|
+          association_data =
+            if value.is_a?(Array)
+              value.map { |record| JSONAPISerializer.new(record).serializable_hash }
+            else
+              JSONAPISerializer.new(value).serializable_hash
+            end
+          { data: association_data }
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -0,0 +1,27 @@
+module Chronicle
+  module ETL
+    # Abstract class representing a Serializer for an ETL record
+    class Serializer
+      # Construct a new instance of this serializer.
+      # == Parameters:
+      # options::
+      #   Options for configuring this Serializers
+      def initialize(record, options = {})
+        @record = record
+        @options = options
+      end
+
+      # Serialize a record as a hash
+      def serializable_hash
+        raise NotImplementedError
+      end
+
+      def self.serialize(record)
+        serializer = self.new(record)
+        serializer.serializable_hash
+      end
+    end
+  end
+end
+
+require_relative 'jsonapi_serializer'