chronicle-etl 0.2.1 → 0.3.0

data/lib/chronicle/etl/logger.rb

@@ -0,0 +1,48 @@
+module Chronicle
+  module ETL
+    module Logger
+      extend self
+
+      DEBUG = 0
+      INFO = 1
+      WARN = 2
+      ERROR = 3
+      FATAL = 4
+
+      attr_accessor :log_level
+
+      @log_level = INFO
+      @destination = $stderr
+
+      def output message, level
+        return unless level >= @log_level
+
+        if @progress_bar
+          @progress_bar.log(message)
+        else
+          @destination.puts(message)
+        end
+      end
+
+      def error(message)
+        output(message, ERROR)
+      end
+
+      def info(message)
+        output(message, INFO)
+      end
+
+      def debug(message)
+        output(message, DEBUG)
+      end
+
+      def attach_to_progress_bar(progress_bar)
+        @progress_bar = progress_bar
+      end
+
+      def detach_from_progress_bar
+        @progress_bar = nil
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/activity.rb

@@ -0,0 +1,15 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Activity < Chronicle::ETL::Models::Base
+        TYPE = 'activities'.freeze
+        ATTRIBUTES = [:verb, :start_at, :end_at].freeze
+        ASSOCIATIONS = [:involved, :actor].freeze
+
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/attachment.rb

@@ -0,0 +1,14 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Attachment < Chronicle::ETL::Models::Base
+        TYPE = 'attachments'.freeze
+        ATTRIBUTES = [:url_original, :data].freeze
+
+        attr_accessor(*ATTRIBUTES)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/base.rb

@@ -0,0 +1,119 @@
+require 'digest'
+
+module Chronicle
+  module ETL
+    module Models
+      # Represents a record that's been transformed by a Transformer and
+      # ready to be loaded. Loosely based on ActiveModel.
+      class Base
+        ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
+        ASSOCIATIONS = [].freeze
+
+        attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
+
+        def initialize(attributes = {})
+          assign_attributes(attributes) if attributes
+          @dedupe_on = []
+          @metadata = {}
+        end
+
+        # A unique identifier for this model is formed from a type
+        # and either an id or lids.
+        def identifier_hash
+          {
+            type: self.class::TYPE,
+            id: @id,
+            lids: lids
+          }.compact
+        end
+
+        # Array of local ids that uniquely identify this record
+        def lids
+          @dedupe_on.map do |fields|
+            generate_lid(fields)
+          end.compact.uniq
+        end
+
+        # For a given set of fields of this model, generate a
+        # unique local id by hashing the field values
+        def generate_lid fields
+          raise ArgumentError.new("Must provide an array of symbolized fields") unless fields.is_a?(Array)
+
+          values = fields.sort.map do |field|
+            instance_variable = "@#{field.to_s}"
+            self.instance_variable_get(instance_variable)
+          end
+
+          return if values.any? { |e| e.nil? }
+
+          Digest::SHA256.hexdigest(values.join(","))
+        end
+
+        # Set of attribute names that this model has is Base's shared
+        # attributes combined with the child class's
+        def attribute_list
+          (ATTRIBUTES + self.class::ATTRIBUTES).uniq
+        end
+
+        # All of this record's attributes
+        def attributes
+          attributes = {}
+          attribute_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            attributes[attribute] = self.instance_variable_get(instance_variable)
+          end
+          attributes.compact
+        end
+
+        # All of this record's associations
+        def associations
+          association_list = ASSOCIATIONS + self.class::ASSOCIATIONS
+          attributes = {}
+          association_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            association = self.instance_variable_get(instance_variable)
+            attributes[attribute] = association if association
+          end
+          attributes.compact
+        end
+
+        def associations_hash
+          associations.map do |k, v|
+            if v.is_a?(Array)
+              [k, v.map(&:to_h)]
+            else
+              [k, v.to_h]
+            end
+          end.to_h
+        end
+
+        def meta_hash
+          {
+            meta: {
+              dedupe_on: @dedupe_on.map{|d| d.map(&:to_s).join(",")}
+            }
+          }
+        end
+
+        # FIXME: move this to a Utils module
+        def to_h_flattened
+          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
+        end
+
+        def to_h
+          identifier_hash
+            .merge(attributes)
+            .merge(associations_hash)
+            .merge(meta_hash)
+        end
+
+        def assign_attributes attributes
+          attributes.each do |k, v|
+            setter = :"#{k}="
+            public_send(setter, v) if respond_to? setter
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/entity.rb

@@ -0,0 +1,21 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Entity < Chronicle::ETL::Models::Base
+        TYPE = 'entities'.freeze
+        ATTRIBUTES = [:title, :body, :represents, :slug, :myself, :metadata].freeze
+        ASSOCIATIONS = [
+          :attachments,
+          :abouts,
+          :depicts,
+          :consumers,
+          :contains
+        ].freeze  # TODO: add these to reflect Chronicle Schema
+
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/generic.rb

@@ -0,0 +1,23 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Generic < Chronicle::ETL::Models::Base
+        TYPE = 'generic'
+
+        attr_accessor :properties
+
+        def initialize(properties = {})
+          @properties = properties
+          super
+        end
+
+        # Generic models have arbitrary attributes stored in @properties
+        def attributes
+          @properties.transform_keys(&:to_sym)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -0,0 +1,61 @@
+module Chronicle
+  module ETL
+    module Registry
+      # Records details about a connector such as its provider and a description
+      class ConnectorRegistration
+        attr_accessor :identifier, :provider, :klass, :description
+
+        def initialize(klass)
+          @klass = klass
+        end
+
+        def phase
+          if klass.ancestors.include? Chronicle::ETL::Extractor
+            :extractor
+          elsif klass.ancestors.include? Chronicle::ETL::Transformer
+            :transformer
+          elsif klass.ancestors.include? Chronicle::ETL::Loader
+            :loader
+          end
+        end
+
+        def to_s
+          "#{phase}-#{identifier}"
+        end
+
+        def built_in?
+          @klass.to_s.include? 'Chronicle::ETL'
+        end
+
+        def klass_name
+          @klass.to_s
+        end
+
+        def identifier
+          @identifier || @klass.to_s.split('::').last.gsub!(/(Extractor$|Loader$|Transformer$)/, '').downcase
+        end
+
+        def description
+          @description || @klass.to_s.split('::').last
+        end
+
+        def provider
+          @provider || (built_in? ? 'chronicle' : '')
+        end
+
+        def descriptive_phrase
+          prefix = case phase
+          when :extractor
+            "Extracts from"
+          when :transformer
+            "Transforms"
+          when :loader
+            "Loads to"
+          end
+
+          "#{prefix} #{description}"
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/registry.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+
+module Chronicle
+  module ETL
+    # A singleton class that acts as a registry of connector classes available for ETL jobs
+    module Registry
+      PHASES = [:extractor, :transformer, :loader]
+
+      class << self
+        attr_accessor :connectors
+
+        def load_all!
+          load_connectors_from_gems
+        end
+
+        def load_connectors_from_gems
+          Gem::Specification.filter{|s| s.name.match(/^chronicle/) }.each do |gem|
+            require_str = gem.name.gsub('chronicle-', 'chronicle/')
+            require require_str rescue LoadError
+          end
+        end
+
+        def install_connector name
+          gem_name = "chronicle-#{name}"
+          Gem.install(gem_name)
+        end
+
+        def register connector
+          @connectors ||= []
+          @connectors << connector
+        end
+
+        def find_by_phase_and_identifier(phase, identifier)
+          connector = find_within_loaded_connectors(phase, identifier)
+          unless connector
+            # Only load external connectors (slow) if not found in built-in connectors
+            load_all!
+            connector = find_within_loaded_connectors(phase, identifier)
+          end
+          connector || raise(ConnectorNotAvailableError.new("Connector '#{identifier}' not found"))
+        end
+
+        def find_within_loaded_connectors(phase, identifier)
+          @connectors.find { |c| c.phase == phase && c.identifier == identifier }
+        end
+      end
+    end
+  end
+end
+
+require_relative 'self_registering'
+require_relative 'connector_registration'

data/lib/chronicle/etl/registry/self_registering.rb

@@ -0,0 +1,25 @@
+require 'forwardable'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Gives a connector class the ability to let the Chronicle::ETL::Registry
+      # know about itself
+      module SelfRegistering
+        extend Forwardable
+
+        attr_accessor :connector_registration
+
+        def_delegators :@connector_registration, :description, :provider, :identifier
+
+        # Creates a ConnectorRegistration for this connector's details and register's it
+        # into the Registry
+        def register_connector
+          @connector_registration ||= ::Chronicle::ETL::Registry::ConnectorRegistration.new(self)
+          yield @connector_registration if block_given?
+          ::Chronicle::ETL::Registry.register(@connector_registration)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,59 +1,87 @@
-class Chronicle::ETL::Runner
-  BUILTIN = {
-    extractor: ['stdin', 'json', 'csv', 'file'],
-    transformer: ['null'],
-    loader: ['stdout', 'csv', 'table']
-  }.freeze
-
-  def initialize(options)
-    @options = options
+require 'colorize'
+require 'chronic_duration'
 
-    instantiate_etl_classes
+class Chronicle::ETL::Runner
+  def initialize(job)
+    @job = job
+    @job_logger = Chronicle::ETL::JobLogger.new(@job)
   end
 
   def run!
-    total = @extractor.results_count
-    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
-    count = 0
+    extractor = @job.instantiate_extractor
+    loader = @job.instantiate_loader
+
+    @job_logger.start
+    loader.start
 
-    @loader.start
+    total = extractor.results_count
+    @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+    Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
 
-    @extractor.extract do |data, metadata|
-      transformed_data = @transformer.transform(data)
-      @loader.load(transformed_data)
+    Chronicle::ETL::Logger.info(tty_log_job_start)
+    extractor.extract do |extraction|
+      unless extraction.is_a?(Chronicle::ETL::Extraction)
+        raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
+      end
+
+      transformer = @job.instantiate_transformer(extraction)
+      record = transformer.transform
+
+      unless record.is_a?(Chronicle::ETL::Models::Base)
+        raise Chronicle::ETL::RunnerTypeError, "Transformed data should be a type of Chronicle::ETL::Models"
+      end
 
-      progress_bar.increment
-      count += 1
+      Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
+      @job_logger.log_transformation(transformer)
+
+      loader.load(record) unless @job.dry_run?
+    rescue Chronicle::ETL::TransformationError => e
+      Chronicle::ETL::Logger.error(tty_log_transformation_failure(e))
+    ensure
+      @progress_bar.increment
     end
 
-    progress_bar.finish
-    @loader.finish
+    @progress_bar.finish
+    loader.finish
+    @job_logger.finish
+  rescue Interrupt
+    Chronicle::ETL::Logger.error("\n#{'Job interrupted'.red}")
+    @job_logger.error
+  rescue StandardError => e
+    raise e
+  ensure
+    @job_logger.save
+    @progress_bar.finish
+    Chronicle::ETL::Logger.detach_from_progress_bar
+    Chronicle::ETL::Logger.info(tty_log_completion)
   end
 
   private
 
-  def instantiate_etl_classes
-    @extractor = load_etl_class(:extractor, @options[:extractor][:name]).new(@options[:extractor][:options])
-    @transformer = load_etl_class(:transformer, @options[:transformer][:name]).new(@options[:transformer][:options])
-    @loader = load_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
+  def tty_log_job_start
+    output = "Beginning job "
+    output += "'#{@job.name}'".bold if @job.name
+    output
   end
 
-  def load_etl_class(phase, x)
-    if BUILTIN[phase].include? x
-      klass_name = "Chronicle::ETL::#{x.capitalize}#{phase.to_s.capitalize}"
-    else
-      # TODO: come up with syntax for specifying a particular extractor in a provider library
-      provider, name = x.split(":")
-      provider = x unless provider
-      begin
-        require "chronicle/#{provider}"
-      rescue LoadError => e
-        warn("Error loading #{phase} '#{provider}'".red)
-        warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
-        exit(false)
-      end
-      klass_name = "Chronicle::#{provider.capitalize}::#{name&.capitalize}#{phase.capitalize}"
-    end
-    Object.const_get(klass_name)
+  def tty_log_transformation transformer
+    output = "  ✓".green
+    output += " #{transformer}"
+  end
+
+  def tty_log_transformation_failure exception
+    output = "  ✖".red
+    output += " Failed to build #{exception.transformation}. #{exception.message}"
+  end
+
+  def tty_log_completion
+    status = @job_logger.success ? 'Success' : 'Failed'
+    output = "\nCompleted job "
+    output += "'#{@job.name}'".bold if @job.name
+    output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
+    output += "\n  Status:\t".light_black + status
+    output += "\n  Completed:\t".light_black + "#{@job_logger.job_log.num_records_processed}"
+    output += "\n  Latest:\t".light_black + "#{@job_logger.job_log.highest_timestamp.iso8601}" if @job_logger.job_log.highest_timestamp
+    output
   end
 end

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -0,0 +1,25 @@
+module Chronicle
+  module ETL
+    class JSONAPISerializer < Chronicle::ETL::Serializer
+      def serializable_hash
+        @record
+          .identifier_hash
+          .merge({ attributes: @record.attributes })
+          .merge({ relationships: build_associations })
+          .merge(@record.meta_hash)
+      end
+
+      def build_associations
+        @record.associations.transform_values do |value|
+          association_data =
+            if value.is_a?(Array)
+              value.map { |record| JSONAPISerializer.new(record).serializable_hash }
+            else
+              JSONAPISerializer.new(value).serializable_hash
+            end
+          { data: association_data }
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -0,0 +1,27 @@
+module Chronicle
+  module ETL
+    # Abstract class representing a Serializer for an ETL record
+    class Serializer
+      # Construct a new instance of this serializer.
+      # == Parameters:
+      # options::
+      #   Options for configuring this Serializers
+      def initialize(record, options = {})
+        @record = record
+        @options = options
+      end
+
+      # Serialize a record as a hash
+      def serializable_hash
+        raise NotImplementedError
+      end
+
+      def self.serialize(record)
+        serializer = self.new(record)
+        serializer.serializable_hash
+      end
+    end
+  end
+end
+
+require_relative 'jsonapi_serializer'