chronicle-etl 0.2.4 → 0.3.0

data/lib/chronicle/etl/extractors/json_extractor.rb

@@ -0,0 +1,45 @@
+module Chronicle
+  module ETL
+    class JsonExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::FilesystemReader
+
+      register_connector do |r|
+        r.description = 'input as JSON'
+      end
+
+      DEFAULT_OPTIONS = {
+        filename: $stdin,
+
+        # We're expecting line-separated json objects
+        jsonl: true
+      }.freeze
+
+      def initialize(options = {})
+        super(DEFAULT_OPTIONS.merge(options))
+      end
+
+      def extract
+        load_input do |input|
+          parsed_data = parse_data(input)
+          yield Chronicle::ETL::Extraction.new(data: parsed_data) if parsed_data
+        end
+      end
+
+      def results_count
+      end
+
+      private
+
+      def parse_data data
+        JSON.parse(data)
+      rescue JSON::ParserError => e
+      end
+
+      def load_input
+        read_from_filesystem(filename: @options[:filename]) do |data|
+          yield data
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/stdin_extractor.rb

@@ -1,9 +1,14 @@
 module Chronicle
   module ETL
     class StdinExtractor < Chronicle::ETL::Extractor
+      register_connector do |r|
+        r.description = 'stdin'
+      end
+
       def extract
         $stdin.read.each_line do |line|
-          yield line
+          data = { line: line.strip }
+          yield Chronicle::ETL::Extraction.new(data: data)
         end
       end
     end

data/lib/chronicle/etl/job.rb

@@ -1,6 +1,11 @@
+require 'forwardable'
 module Chronicle
   module ETL
     class Job
+      extend Forwardable
+
+      def_delegators :@job_definition, :dry_run?
+
       attr_accessor :name,
                     :extractor_klass,
                     :extractor_options,
@@ -12,32 +17,30 @@ module Chronicle
       # TODO: build a proper id system
       alias id name
 
-      def initialize(definition)
-        definition = definition.definition # FIXME
-        @name = definition[:name]
-        @extractor_klass = load_klass(:extractor, definition[:extractor][:name])
-        @extractor_options = definition[:extractor][:options] || {}
-
-        @transformer_klass = load_klass(:transformer, definition[:transformer][:name])
-        @transformer_options = definition[:transformer][:options] || {}
-
-        @loader_klass = load_klass(:loader, definition[:loader][:name])
-        @loader_options = definition[:loader][:options] || {}
+      def initialize(job_definition)
+        @job_definition = job_definition
+        @name = @job_definition.definition[:name]
+        @extractor_options = @job_definition.extractor_options
+        @transformer_options = @job_definition.transformer_options
+        @loader_options = @job_definition.loader_options
 
-        set_continuation if load_continuation?
+        set_continuation if use_continuation?
         yield self if block_given?
       end
 
       def instantiate_extractor
-        instantiate_klass(:extractor)
+        @extractor_klass = @job_definition.extractor_klass
+        @extractor_klass.new(@extractor_options)
       end
 
-      def instantiate_transformer(data)
-        instantiate_klass(:transformer, data)
+      def instantiate_transformer(extraction)
+        @transformer_klass = @job_definition.transformer_klass
+        @transformer_klass.new(@transformer_options, extraction)
       end
 
       def instantiate_loader
-        instantiate_klass(:loader)
+        @loader_klass = @job_definition.loader_klass
+        @loader_klass.new(@loader_options)
       end
 
       def save_log?
@@ -45,26 +48,24 @@ module Chronicle
         return !id.nil?
       end
 
-      private
-
-      def instantiate_klass(phase, *args)
-        options = self.send("#{phase.to_s}_options")
-        args = args.unshift(options)
-        klass = self.send("#{phase.to_s}_klass")
-        klass.new(*args)
+      def to_s
+        output = "Job"
+        output += " '#{name}'".bold if name
+        output += "\n"
+        output += "  → Extracting from #{@job_definition.extractor_klass.description}\n"
+        output += "  → Transforming #{@job_definition.transformer_klass.description}\n"
+        output += "  → Loading to #{@job_definition.loader_klass.description}\n"
       end
 
-      def load_klass phase, identifier
-        Chronicle::ETL::Catalog.phase_and_identifier_to_klass(phase, identifier)
-      end
+      private
 
       def set_continuation
-        continuation = Chronicle::ETL::JobLogger.load_latest(@job_id)
+        continuation = Chronicle::ETL::JobLogger.load_latest(@id)
         @extractor_options[:continuation] = continuation
       end
 
-      def load_continuation?
-        save_log?
+      def use_continuation?
+        @job_definition.incremental?
       end
     end
   end

data/lib/chronicle/etl/job_definition.rb

@@ -1,19 +1,20 @@
-require 'deep_merge'
+require 'active_support/core_ext/hash/deep_merge'
 
 module Chronicle
   module ETL
     class JobDefinition
       SKELETON_DEFINITION = {
+        incremental: false,
         extractor: {
-          name: nil,
+          name: 'stdin',
           options: {}
         },
         transformer: {
-          name: nil,
+          name: 'null',
           options: {}
         },
         loader: {
-          name: nil,
+          name: 'stdout',
           options: {}
         }
       }.freeze
@@ -26,16 +27,53 @@ module Chronicle
 
       # Add config hash to this definition
       def add_config(config = {})
-        @definition = config.deep_merge(@definition)
+        @definition = @definition.deep_merge(config)
         load_credentials
         validate
       end
 
+      # Is this job continuing from a previous run?
+      def incremental?
+        @definition[:incremental]
+      end
+
+      def dry_run?
+        @definition[:dry_run]
+      end
+
+      def extractor_klass
+        load_klass(:extractor, @definition[:extractor][:name])
+      end
+
+      def transformer_klass
+        load_klass(:transformer, @definition[:transformer][:name])
+      end
+
+      def loader_klass
+        load_klass(:loader, @definition[:loader][:name])
+      end
+
+      def extractor_options
+        @definition[:extractor][:options]
+      end
+
+      def transformer_options
+        @definition[:transformer][:options]
+      end
+
+      def loader_options
+        @definition[:loader][:options]
+      end
+
       private
 
+      def load_klass(phase, identifier)
+        Chronicle::ETL::Registry.find_by_phase_and_identifier(phase, identifier).klass
+      end
+
       def load_credentials
-        Chronicle::ETL::Catalog::PHASES.each do |phase|
-          credentials_name = @definition[phase][:options][:credentials]
+        Chronicle::ETL::Registry::PHASES.each do |phase|
+          credentials_name = @definition[phase].dig(:options, :credentials)
           if credentials_name
             credentials = Chronicle::ETL::Config.load_credentials(credentials_name)
             @definition[phase][:options].deep_merge(credentials)

data/lib/chronicle/etl/job_log.rb

@@ -50,11 +50,21 @@ module Chronicle
         @success = true
       end
 
+      def error
+        @finished_at = Time.now
+      end
+
       def job= job
         @job = job
         @job_id = job.id
       end
 
+      def duration
+        return unless @finished_at
+
+        @finished_at - @started_at
+      end
+
       # Take a JobLog's instance variables and turn them into a hash representation
       def serialize
         {

data/lib/chronicle/etl/job_logger.rb

@@ -1,32 +1,14 @@
 require 'sequel'
 require 'forwardable'
 
-require 'pry'
-
 module Chronicle
   module ETL
     # Saves JobLogs to db and loads previous ones
     class JobLogger
       extend Forwardable
 
-      def_delegators :@job_log, :start, :finish, :log_transformation
-
-      # Create a new JobLogger
-      def initialize(job)
-        @job_log = JobLog.new do |job_log|
-          job_log.job = job
-        end
-      end
-
-      # Save this JobLogger's JobLog to db
-      def save
-        return unless @job_log.save_log?
-
-        JobLogger.with_db_connection do |db|
-          dataset = db[:job_logs]
-          dataset.insert(@job_log.serialize)
-        end
-      end
+      def_delegators :@job_log, :start, :finish, :error, :log_transformation, :duration, :success
+      attr_accessor :job_log
 
       # For a given `job_id`, return the last successful log
       def self.load_latest(job_id)
@@ -73,6 +55,27 @@ module Chronicle
           Time :finished_at
         end
       end
+
+      # Create a new JobLogger
+      def initialize(job)
+        @job_log = JobLog.new do |job_log|
+          job_log.job = job
+        end
+      end
+
+      # Save this JobLogger's JobLog to db
+      def save
+        return unless @job_log.save_log?
+
+        JobLogger.with_db_connection do |db|
+          dataset = db[:job_logs]
+          dataset.insert(@job_log.serialize)
+        end
+      end
+
+      def summarize
+        @job_log.inspect
+      end
     end
   end
 end

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -3,6 +3,10 @@ require 'csv'
 module Chronicle
   module ETL
     class CsvLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'CSV'
+      end
+
       def initialize(options={})
         super(options)
         @rows = []

data/lib/chronicle/etl/loaders/loader.rb

@@ -2,7 +2,7 @@ module Chronicle
   module ETL
     # Abstract class representing a Loader for an ETL job
     class Loader
-      extend Chronicle::ETL::Catalog
+      extend Chronicle::ETL::Registry::SelfRegistering
 
       # Construct a new instance of this loader. Options are passed in from a Runner
       # == Parameters:

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -5,12 +5,16 @@ require 'json'
 module Chronicle
   module ETL
     class RestLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'a REST endpoint'
+      end
+
       def initialize( options={} )
         super(options)
       end
 
       def load(record)
-        payload = Chronicle::ETL::Utils::JSONAPI.serialize(record)
+        payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
         # have the outer data key that json-api expects
         payload = { data: payload } unless payload[:data]
 

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,8 +1,13 @@
 module Chronicle
   module ETL
     class StdoutLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'stdout'
+      end
+
       def load(record)
-        puts record.to_h
+        serializer = Chronicle::ETL::JSONAPISerializer.new(record)
+        puts serializer.serializable_hash.to_json
       end
     end
   end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,21 +1,71 @@
 require 'tty/table'
+require 'active_support/core_ext/string/filters'
+require 'active_support/core_ext/hash/reverse_merge'
 
 module Chronicle
   module ETL
     class TableLoader < Chronicle::ETL::Loader
-      def initialize(options)
-        super(options)
+      register_connector do |r|
+        r.description = 'an ASCII table'
+      end
+
+      DEFAULT_OPTIONS = {
+        fields_limit: nil,
+        fields_exclude: ['lids', 'type'],
+        fields_include: [],
+        truncate_values_at: nil,
+        table_renderer: :basic
+      }.freeze
+
+      def initialize(options={})
+        @options = options.reverse_merge(DEFAULT_OPTIONS)
+        @records = []
       end
 
       def load(record)
-        record_hash = record.to_h_flattened
-        @table ||= TTY::Table.new(header: record_hash.keys)
-        values = record_hash.values.map{|x| x.to_s[0..30]}
-        @table << values
+        @records << record.to_h_flattened
       end
 
       def finish
-        puts @table.render(:ascii, padding: [0, 1]) if @table
+        return if @records.empty?
+
+        headers = build_headers(@records)
+        rows = build_rows(@records, headers)
+
+        @table = TTY::Table.new(header: headers, rows: rows)
+        puts @table.render(
+          @options[:table_renderer].to_sym,
+          padding: [0, 2, 0, 0]
+        )
+      end
+
+      private
+
+      def build_headers(records)
+        headers =
+          if @options[:fields_include].any?
+            Set[*@options[:fields_include]]
+          else
+            # use all the keys of the flattened record hash
+            Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
+          end
+
+        headers = headers.delete_if { |header| header.end_with?(*@options[:fields_exclude]) } if @options[:fields_exclude].any?
+        headers = headers.first(@options[:fields_limit]) if @options[:fields_limit]
+
+        headers.to_a.map(&:to_sym)
+      end
+
+      def build_rows(records, headers)
+        records.map do |record|
+          values = record.values_at(*headers).map{|value| value.to_s }
+
+          if @options[:truncate_values_at]
+            values = values.map{ |value| value.truncate(@options[:truncate_values_at]) }
+          end
+
+          values
+        end
       end
     end
   end

data/lib/chronicle/etl/logger.rb

@@ -0,0 +1,48 @@
+module Chronicle
+  module ETL
+    module Logger
+      extend self
+
+      DEBUG = 0
+      INFO = 1
+      WARN = 2
+      ERROR = 3
+      FATAL = 4
+
+      attr_accessor :log_level
+
+      @log_level = INFO
+      @destination = $stderr
+
+      def output message, level
+        return unless level >= @log_level
+
+        if @progress_bar
+          @progress_bar.log(message)
+        else
+          @destination.puts(message)
+        end
+      end
+
+      def error(message)
+        output(message, ERROR)
+      end
+
+      def info(message)
+        output(message, INFO)
+      end
+
+      def debug(message)
+        output(message, DEBUG)
+      end
+
+      def attach_to_progress_bar(progress_bar)
+        @progress_bar = progress_bar
+      end
+
+      def detach_from_progress_bar
+        @progress_bar = nil
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/attachment.rb

@@ -0,0 +1,14 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Attachment < Chronicle::ETL::Models::Base
+        TYPE = 'attachments'.freeze
+        ATTRIBUTES = [:url_original, :data].freeze
+
+        attr_accessor(*ATTRIBUTES)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/base.rb

@@ -6,7 +6,7 @@ module Chronicle
       # Represents a record that's been transformed by a Transformer and
       # ready to be loaded. Loosely based on ActiveModel.
       class Base
-        ATTRIBUTES = [:provider, :provider_id, :lat, :lng].freeze
+        ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
         ASSOCIATIONS = [].freeze
 
         attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
@@ -14,6 +14,7 @@ module Chronicle
         def initialize(attributes = {})
           assign_attributes(attributes) if attributes
           @dedupe_on = []
+          @metadata = {}
         end
 
         # A unique identifier for this model is formed from a type
@@ -36,6 +37,8 @@ module Chronicle
         # For a given set of fields of this model, generate a
         # unique local id by hashing the field values
         def generate_lid fields
+          raise ArgumentError.new("Must provide an array of symbolized fields") unless fields.is_a?(Array)
+
           values = fields.sort.map do |field|
             instance_variable = "@#{field.to_s}"
             self.instance_variable_get(instance_variable)
@@ -75,9 +78,21 @@ module Chronicle
         end
 
         def associations_hash
-          Hash[associations.map do |k, v|
-            [k, v.to_h]
-          end]
+          associations.map do |k, v|
+            if v.is_a?(Array)
+              [k, v.map(&:to_h)]
+            else
+              [k, v.to_h]
+            end
+          end.to_h
+        end
+
+        def meta_hash
+          {
+            meta: {
+              dedupe_on: @dedupe_on.map{|d| d.map(&:to_s).join(",")}
+            }
+          }
         end
 
         # FIXME: move this to a Utils module
@@ -86,11 +101,12 @@ module Chronicle
         end
 
         def to_h
-          identifier_hash.merge(attributes).merge(associations_hash)
+          identifier_hash
+            .merge(attributes)
+            .merge(associations_hash)
+            .merge(meta_hash)
         end
 
-        private
-
         def assign_attributes attributes
           attributes.each do |k, v|
             setter = :"#{k}="

data/lib/chronicle/etl/models/entity.rb

@@ -5,10 +5,16 @@ module Chronicle
     module Models
       class Entity < Chronicle::ETL::Models::Base
         TYPE = 'entities'.freeze
-        ATTRIBUTES = [:title, :body, :represents, :slug].freeze
-        ASSOCIATIONS = [].freeze  # TODO: add these to reflect Chronicle Schema
+        ATTRIBUTES = [:title, :body, :represents, :slug, :myself, :metadata].freeze
+        ASSOCIATIONS = [
+          :attachments,
+          :abouts,
+          :depicts,
+          :consumers,
+          :contains
+        ].freeze  # TODO: add these to reflect Chronicle Schema
 
-        attr_accessor(*ATTRIBUTES)
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
       end
     end
   end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -0,0 +1,61 @@
+module Chronicle
+  module ETL
+    module Registry
+      # Records details about a connector such as its provider and a description
+      class ConnectorRegistration
+        attr_accessor :identifier, :provider, :klass, :description
+
+        def initialize(klass)
+          @klass = klass
+        end
+
+        def phase
+          if klass.ancestors.include? Chronicle::ETL::Extractor
+            :extractor
+          elsif klass.ancestors.include? Chronicle::ETL::Transformer
+            :transformer
+          elsif klass.ancestors.include? Chronicle::ETL::Loader
+            :loader
+          end
+        end
+
+        def to_s
+          "#{phase}-#{identifier}"
+        end
+
+        def built_in?
+          @klass.to_s.include? 'Chronicle::ETL'
+        end
+
+        def klass_name
+          @klass.to_s
+        end
+
+        def identifier
+          @identifier || @klass.to_s.split('::').last.gsub!(/(Extractor$|Loader$|Transformer$)/, '').downcase
+        end
+
+        def description
+          @description || @klass.to_s.split('::').last
+        end
+
+        def provider
+          @provider || (built_in? ? 'chronicle' : '')
+        end
+
+        def descriptive_phrase
+          prefix = case phase
+          when :extractor
+            "Extracts from"
+          when :transformer
+            "Transforms"
+          when :loader
+            "Loads to"
+          end
+
+          "#{prefix} #{description}"
+        end
+      end
+    end
+  end
+end