chronicle-etl 0.2.4 → 0.4.0

data/lib/chronicle/etl/job_definition.rb

@@ -1,19 +1,20 @@
-require 'deep_merge'
+require 'active_support/core_ext/hash/deep_merge'
 
 module Chronicle
   module ETL
     class JobDefinition
       SKELETON_DEFINITION = {
+        incremental: false,
         extractor: {
-          name: nil,
+          name: 'stdin',
           options: {}
         },
         transformer: {
-          name: nil,
+          name: 'null',
           options: {}
         },
         loader: {
-          name: nil,
+          name: 'stdout',
           options: {}
         }
       }.freeze
@@ -26,16 +27,53 @@ module Chronicle
 
       # Add config hash to this definition
       def add_config(config = {})
-        @definition = config.deep_merge(@definition)
+        @definition = @definition.deep_merge(config)
         load_credentials
         validate
       end
 
+      # Is this job continuing from a previous run?
+      def incremental?
+        @definition[:incremental]
+      end
+
+      def dry_run?
+        @definition[:dry_run]
+      end
+
+      def extractor_klass
+        load_klass(:extractor, @definition[:extractor][:name])
+      end
+
+      def transformer_klass
+        load_klass(:transformer, @definition[:transformer][:name])
+      end
+
+      def loader_klass
+        load_klass(:loader, @definition[:loader][:name])
+      end
+
+      def extractor_options
+        @definition[:extractor][:options]
+      end
+
+      def transformer_options
+        @definition[:transformer][:options]
+      end
+
+      def loader_options
+        @definition[:loader][:options]
+      end
+
       private
 
+      def load_klass(phase, identifier)
+        Chronicle::ETL::Registry.find_by_phase_and_identifier(phase, identifier).klass
+      end
+
       def load_credentials
-        Chronicle::ETL::Catalog::PHASES.each do |phase|
-          credentials_name = @definition[phase][:options][:credentials]
+        Chronicle::ETL::Registry::PHASES.each do |phase|
+          credentials_name = @definition[phase].dig(:options, :credentials)
           if credentials_name
             credentials = Chronicle::ETL::Config.load_credentials(credentials_name)
             @definition[phase][:options].deep_merge(credentials)

data/lib/chronicle/etl/job_log.rb

@@ -50,11 +50,21 @@ module Chronicle
         @success = true
       end
 
+      def error
+        @finished_at = Time.now
+      end
+
       def job= job
         @job = job
         @job_id = job.id
       end
 
+      def duration
+        return unless @finished_at
+
+        @finished_at - @started_at
+      end
+
       # Take a JobLog's instance variables and turn them into a hash representation
       def serialize
         {

data/lib/chronicle/etl/job_logger.rb

@@ -1,32 +1,14 @@
 require 'sequel'
 require 'forwardable'
 
-require 'pry'
-
 module Chronicle
   module ETL
     # Saves JobLogs to db and loads previous ones
     class JobLogger
       extend Forwardable
 
-      def_delegators :@job_log, :start, :finish, :log_transformation
-
-      # Create a new JobLogger
-      def initialize(job)
-        @job_log = JobLog.new do |job_log|
-          job_log.job = job
-        end
-      end
-
-      # Save this JobLogger's JobLog to db
-      def save
-        return unless @job_log.save_log?
-
-        JobLogger.with_db_connection do |db|
-          dataset = db[:job_logs]
-          dataset.insert(@job_log.serialize)
-        end
-      end
+      def_delegators :@job_log, :start, :finish, :error, :log_transformation, :duration, :success
+      attr_accessor :job_log
 
       # For a given `job_id`, return the last successful log
       def self.load_latest(job_id)
@@ -73,6 +55,27 @@ module Chronicle
           Time :finished_at
         end
       end
+
+      # Create a new JobLogger
+      def initialize(job)
+        @job_log = JobLog.new do |job_log|
+          job_log.job = job
+        end
+      end
+
+      # Save this JobLogger's JobLog to db
+      def save
+        return unless @job_log.save_log?
+
+        JobLogger.with_db_connection do |db|
+          dataset = db[:job_logs]
+          dataset.insert(@job_log.serialize)
+        end
+      end
+
+      def summarize
+        @job_log.inspect
+      end
     end
   end
 end

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -2,7 +2,11 @@ require 'csv'
 
 module Chronicle
   module ETL
-    class CsvLoader < Chronicle::ETL::Loader
+    class CSVLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'CSV'
+      end
+
       def initialize(options={})
         super(options)
         @rows = []

data/lib/chronicle/etl/loaders/loader.rb

@@ -2,14 +2,17 @@ module Chronicle
   module ETL
     # Abstract class representing a Loader for an ETL job
     class Loader
-      extend Chronicle::ETL::Catalog
+      extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
+
+      setting :output
 
       # Construct a new instance of this loader. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Loader
       def initialize(options = {})
-        @options = options
+        apply_options(options)
       end
 
       # Called once before processing records

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -5,19 +5,23 @@ require 'json'
 module Chronicle
   module ETL
     class RestLoader < Chronicle::ETL::Loader
-      def initialize( options={} )
-        super(options)
+      register_connector do |r|
+        r.description = 'a REST endpoint'
       end
 
+      setting :hostname, required: true
+      setting :endpoint, required: true
+      setting :access_token
+
       def load(record)
-        payload = Chronicle::ETL::Utils::JSONAPI.serialize(record)
+        payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
         # have the outer data key that json-api expects
         payload = { data: payload } unless payload[:data]
 
-        uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
+        uri = URI.parse("#{@config.hostname}#{@config.endpoint}")
 
         header = {
-          "Authorization" => "Bearer #{@options[:access_token]}",
+          "Authorization" => "Bearer #{@config.access_token}",
           "Content-Type": 'application/json'
         }
         use_ssl = uri.scheme == 'https'

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,8 +1,13 @@
 module Chronicle
   module ETL
     class StdoutLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'stdout'
+      end
+
       def load(record)
-        puts record.to_h
+        serializer = Chronicle::ETL::JSONAPISerializer.new(record)
+        puts serializer.serializable_hash.to_json
       end
     end
   end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,21 +1,65 @@
 require 'tty/table'
+require 'active_support/core_ext/string/filters'
+require 'active_support/core_ext/hash/reverse_merge'
 
 module Chronicle
   module ETL
     class TableLoader < Chronicle::ETL::Loader
-      def initialize(options)
-        super(options)
+      register_connector do |r|
+        r.description = 'an ASCII table'
       end
 
+      setting :fields_limit, default: nil
+      setting :fields_exclude, default: ['lids', 'type']
+      setting :fields_include, default: []
+      setting :truncate_values_at, default: 40
+      setting :table_renderer, default: :basic
+
       def load(record)
-        record_hash = record.to_h_flattened
-        @table ||= TTY::Table.new(header: record_hash.keys)
-        values = record_hash.values.map{|x| x.to_s[0..30]}
-        @table << values
+        @records ||= []
+        @records << record.to_h_flattened
       end
 
       def finish
-        puts @table.render(:ascii, padding: [0, 1]) if @table
+        return if @records.empty?
+
+        headers = build_headers(@records)
+        rows = build_rows(@records, headers)
+
+        @table = TTY::Table.new(header: headers, rows: rows)
+        puts @table.render(
+          @config.table_renderer.to_sym,
+          padding: [0, 2, 0, 0]
+        )
+      end
+
+      private
+
+      def build_headers(records)
+        headers =
+          if @config.fields_include.any?
+            Set[*@config.fields_include]
+          else
+            # use all the keys of the flattened record hash
+            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
+          end
+
+        headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) } if @config.fields_exclude.any?
+        headers = headers.first(@config.fields_limit) if @config.fields_limit
+
+        headers.to_a.map(&:to_sym)
+      end
+
+      def build_rows(records, headers)
+        records.map do |record|
+          values = record.values_at(*headers).map{|value| value.to_s }
+
+          if @config.truncate_values_at
+            values = values.map{ |value| value.truncate(@config.truncate_values_at) }
+          end
+
+          values
+        end
       end
     end
   end

data/lib/chronicle/etl/logger.rb

@@ -0,0 +1,48 @@
+module Chronicle
+  module ETL
+    module Logger
+      extend self
+
+      DEBUG = 0
+      INFO = 1
+      WARN = 2
+      ERROR = 3
+      FATAL = 4
+
+      attr_accessor :log_level
+
+      @log_level = INFO
+      @destination = $stderr
+
+      def output message, level
+        return unless level >= @log_level
+
+        if @progress_bar
+          @progress_bar.log(message)
+        else
+          @destination.puts(message)
+        end
+      end
+
+      def error(message)
+        output(message, ERROR)
+      end
+
+      def info(message)
+        output(message, INFO)
+      end
+
+      def debug(message)
+        output(message, DEBUG)
+      end
+
+      def attach_to_progress_bar(progress_bar)
+        @progress_bar = progress_bar
+      end
+
+      def detach_from_progress_bar
+        @progress_bar = nil
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/attachment.rb

@@ -0,0 +1,14 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Attachment < Chronicle::ETL::Models::Base
+        TYPE = 'attachments'.freeze
+        ATTRIBUTES = [:url_original, :data].freeze
+
+        attr_accessor(*ATTRIBUTES)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/base.rb

@@ -6,7 +6,7 @@ module Chronicle
       # Represents a record that's been transformed by a Transformer and
       # ready to be loaded. Loosely based on ActiveModel.
       class Base
-        ATTRIBUTES = [:provider, :provider_id, :lat, :lng].freeze
+        ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
         ASSOCIATIONS = [].freeze
 
         attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
@@ -14,6 +14,7 @@ module Chronicle
         def initialize(attributes = {})
           assign_attributes(attributes) if attributes
           @dedupe_on = []
+          @metadata = {}
         end
 
         # A unique identifier for this model is formed from a type
@@ -36,6 +37,8 @@ module Chronicle
         # For a given set of fields of this model, generate a
         # unique local id by hashing the field values
         def generate_lid fields
+          raise ArgumentError.new("Must provide an array of symbolized fields") unless fields.is_a?(Array)
+
           values = fields.sort.map do |field|
             instance_variable = "@#{field.to_s}"
             self.instance_variable_get(instance_variable)
@@ -75,9 +78,21 @@ module Chronicle
         end
 
         def associations_hash
-          Hash[associations.map do |k, v|
-            [k, v.to_h]
-          end]
+          associations.map do |k, v|
+            if v.is_a?(Array)
+              [k, v.map(&:to_h)]
+            else
+              [k, v.to_h]
+            end
+          end.to_h
+        end
+
+        def meta_hash
+          {
+            meta: {
+              dedupe_on: @dedupe_on.map{|d| d.map(&:to_s).join(",")}
+            }
+          }
         end
 
         # FIXME: move this to a Utils module
@@ -86,11 +101,12 @@ module Chronicle
         end
 
         def to_h
-          identifier_hash.merge(attributes).merge(associations_hash)
+          identifier_hash
+            .merge(attributes)
+            .merge(associations_hash)
+            .merge(meta_hash)
         end
 
-        private
-
         def assign_attributes attributes
           attributes.each do |k, v|
             setter = :"#{k}="

data/lib/chronicle/etl/models/entity.rb

@@ -5,10 +5,16 @@ module Chronicle
     module Models
       class Entity < Chronicle::ETL::Models::Base
         TYPE = 'entities'.freeze
-        ATTRIBUTES = [:title, :body, :represents, :slug].freeze
-        ASSOCIATIONS = [].freeze  # TODO: add these to reflect Chronicle Schema
+        ATTRIBUTES = [:title, :body, :represents, :slug, :myself, :metadata].freeze
+        ASSOCIATIONS = [
+          :attachments,
+          :abouts,
+          :depicts,
+          :consumers,
+          :contains
+        ].freeze  # TODO: add these to reflect Chronicle Schema
 
-        attr_accessor(*ATTRIBUTES)
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
       end
     end
   end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -0,0 +1,62 @@
+module Chronicle
+  module ETL
+    module Registry
+      # Records details about a connector such as its provider and a description
+      class ConnectorRegistration
+        # FIXME: refactor custom accessor methods later in file
+        attr_accessor :identifier, :provider, :klass, :description
+
+        def initialize(klass)
+          @klass = klass
+        end
+
+        def phase
+          if klass.ancestors.include? Chronicle::ETL::Extractor
+            :extractor
+          elsif klass.ancestors.include? Chronicle::ETL::Transformer
+            :transformer
+          elsif klass.ancestors.include? Chronicle::ETL::Loader
+            :loader
+          end
+        end
+
+        def to_s
+          "#{phase}-#{identifier}"
+        end
+
+        def built_in?
+          @klass.to_s.include? 'Chronicle::ETL'
+        end
+
+        def klass_name
+          @klass.to_s
+        end
+
+        def identifier
+          @identifier || @klass.to_s.split('::').last.gsub!(/(Extractor$|Loader$|Transformer$)/, '').downcase
+        end
+
+        def description
+          @description || @klass.to_s.split('::').last
+        end
+
+        def provider
+          @provider || (built_in? ? 'chronicle' : '')
+        end
+
+        def descriptive_phrase
+          prefix = case phase
+          when :extractor
+            "Extracts from"
+          when :transformer
+            "Transforms"
+          when :loader
+            "Loads to"
+          end
+
+          "#{prefix} #{description}"
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/registry.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+
+module Chronicle
+  module ETL
+    # A singleton class that acts as a registry of connector classes available for ETL jobs
+    module Registry
+      PHASES = [:extractor, :transformer, :loader]
+
+      class << self
+        attr_accessor :connectors
+
+        def load_all!
+          load_connectors_from_gems
+        end
+
+        def load_connectors_from_gems
+          Gem::Specification.filter{|s| s.name.match(/^chronicle/) }.each do |gem|
+            require_str = gem.name.gsub('chronicle-', 'chronicle/')
+            require require_str rescue LoadError
+          end
+        end
+
+        def install_connector name
+          gem_name = "chronicle-#{name}"
+          Gem.install(gem_name)
+        end
+
+        def register connector
+          @connectors ||= []
+          @connectors << connector
+        end
+
+        def find_by_phase_and_identifier(phase, identifier)
+          connector = find_within_loaded_connectors(phase, identifier)
+          unless connector
+            # Only load external connectors (slow) if not found in built-in connectors
+            load_all!
+            connector = find_within_loaded_connectors(phase, identifier)
+          end
+          connector || raise(ConnectorNotAvailableError.new("Connector '#{identifier}' not found"))
+        end
+
+        def find_within_loaded_connectors(phase, identifier)
+          @connectors.find { |c| c.phase == phase && c.identifier == identifier }
+        end
+      end
+    end
+  end
+end
+
+require_relative 'self_registering'
+require_relative 'connector_registration'

data/lib/chronicle/etl/registry/self_registering.rb

@@ -0,0 +1,25 @@
+require 'forwardable'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Gives a connector class the ability to let the Chronicle::ETL::Registry
+      # know about itself
+      module SelfRegistering
+        extend Forwardable
+
+        attr_accessor :connector_registration
+
+        def_delegators :@connector_registration, :description, :provider, :identifier
+
+        # Creates a ConnectorRegistration for this connector's details and register's it
+        # into the Registry
+        def register_connector
+          @connector_registration ||= ::Chronicle::ETL::Registry::ConnectorRegistration.new(self)
+          yield @connector_registration if block_given?
+          ::Chronicle::ETL::Registry.register(@connector_registration)
+        end
+      end
+    end
+  end
+end