chronicle-etl 0.1.4 → 0.2.4

data/lib/chronicle/etl/job_logger.rb

@@ -0,0 +1,78 @@
+require 'sequel'
+require 'forwardable'
+
+require 'pry'
+
+module Chronicle
+  module ETL
+    # Saves JobLogs to db and loads previous ones
+    class JobLogger
+      extend Forwardable
+
+      def_delegators :@job_log, :start, :finish, :log_transformation
+
+      # Create a new JobLogger
+      def initialize(job)
+        @job_log = JobLog.new do |job_log|
+          job_log.job = job
+        end
+      end
+
+      # Save this JobLogger's JobLog to db
+      def save
+        return unless @job_log.save_log?
+
+        JobLogger.with_db_connection do |db|
+          dataset = db[:job_logs]
+          dataset.insert(@job_log.serialize)
+        end
+      end
+
+      # For a given `job_id`, return the last successful log
+      def self.load_latest(job_id)
+        with_db_connection do |db|
+          attrs = db[:job_logs].reverse_order(:finished_at).where(success: true).first
+          JobLog.build_from_serialized(attrs) if attrs
+        end
+      end
+
+      def self.with_db_connection
+        initialize_db unless db_exists?
+        Sequel.connect("sqlite://#{db_filename}") do |db|
+          initialize_schema(db) unless schema_exists?(db)
+          yield db
+        end
+      end
+
+      def self.db_exists?
+        File.exists?(db_filename)
+      end
+
+      def self.schema_exists?(db)
+        return db.tables.include? :job_logs
+      end
+
+      def self.db_filename
+        data = Runcom::Data.new "chronicle/etl/job_log.db"
+        filename = data.all[0].to_s
+      end
+
+      def self.initialize_db
+        FileUtils.mkdir_p(File.dirname(db_filename))
+      end
+
+      def self.initialize_schema db
+        db.create_table :job_logs do
+          primary_key :id
+          String :job_id, null: false
+          String :last_id
+          Time :highest_timestamp
+          Integer :num_records_processed
+          boolean :success, default: false
+          Time :started_at
+          Time :finished_at
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -1,19 +1,15 @@
 require 'csv'
 
 module Chronicle
-  module Etl
-    class CsvLoader < Chronicle::Etl::Loader
+  module ETL
+    class CsvLoader < Chronicle::ETL::Loader
       def initialize(options={})
         super(options)
         @rows = []
       end
 
-      def load(result)
-        if (result.is_a? Hash)
-          @rows << result.values
-        else
-          @rows << result
-        end
+      def load(record)
+        @rows << record.to_h_flattened.values
       end
 
       def finish

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,23 +1,32 @@
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing a Loader for an ETL job
     class Loader
-      extend Chronicle::Etl::Catalog
+      extend Chronicle::ETL::Catalog
 
+      # Construct a new instance of this loader. Options are passed in from a Runner
+      # == Parameters:
+      # options::
+      #   Options for configuring this Loader
       def initialize(options = {})
         @options = options
       end
 
+      # Called once before processing records
       def start; end
 
+      # Load a single record
       def load
         raise NotImplementedError
       end
 
+      # Called once there are no more records to process
       def finish; end
     end
   end
 end
 
 require_relative 'csv_loader'
+require_relative 'rest_loader'
 require_relative 'stdout_loader'
 require_relative 'table_loader'

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -0,0 +1,33 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Chronicle
+  module ETL
+    class RestLoader < Chronicle::ETL::Loader
+      def initialize( options={} )
+        super(options)
+      end
+
+      def load(record)
+        payload = Chronicle::ETL::Utils::JSONAPI.serialize(record)
+        # have the outer data key that json-api expects
+        payload = { data: payload } unless payload[:data]
+
+        uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
+
+        header = {
+          "Authorization" => "Bearer #{@options[:access_token]}",
+          "Content-Type": 'application/json'
+        }
+        use_ssl = uri.scheme == 'https'
+
+        Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
+          request = Net::HTTP::Post.new(uri.request_uri, header)
+          request.body = payload.to_json
+          http.request(request)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,9 +1,9 @@
 module Chronicle
-  module Etl
-    class StdoutLoader < Chronicle::Etl::Loader
-      def load(result)
-        puts result.inspect
+  module ETL
+    class StdoutLoader < Chronicle::ETL::Loader
+      def load(record)
+        puts record.to_h
       end
     end
   end
-end
+end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,20 +1,21 @@
 require 'tty/table'
 
 module Chronicle
-  module Etl
-    class TableLoader < Chronicle::Etl::Loader
+  module ETL
+    class TableLoader < Chronicle::ETL::Loader
       def initialize(options)
         super(options)
       end
 
-      def load(result)
-        @table ||= TTY::Table.new(header: result.keys)
-        values = result.values.map{|x| x.to_s[0..30]}
+      def load(record)
+        record_hash = record.to_h_flattened
+        @table ||= TTY::Table.new(header: record_hash.keys)
+        values = record_hash.values.map{|x| x.to_s[0..30]}
         @table << values
       end
 
       def finish
-        puts @table.render(:ascii, padding: [0, 1])
+        puts @table.render(:ascii, padding: [0, 1]) if @table
       end
     end
   end

data/lib/chronicle/etl/models/activity.rb

@@ -0,0 +1,15 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Activity < Chronicle::ETL::Models::Base
+        TYPE = 'activities'.freeze
+        ATTRIBUTES = [:verb, :start_at, :end_at].freeze
+        ASSOCIATIONS = [:involved, :actor].freeze
+
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/base.rb

@@ -0,0 +1,103 @@
+require 'digest'
+
+module Chronicle
+  module ETL
+    module Models
+      # Represents a record that's been transformed by a Transformer and
+      # ready to be loaded. Loosely based on ActiveModel.
+      class Base
+        ATTRIBUTES = [:provider, :provider_id, :lat, :lng].freeze
+        ASSOCIATIONS = [].freeze
+
+        attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
+
+        def initialize(attributes = {})
+          assign_attributes(attributes) if attributes
+          @dedupe_on = []
+        end
+
+        # A unique identifier for this model is formed from a type
+        # and either an id or lids.
+        def identifier_hash
+          {
+            type: self.class::TYPE,
+            id: @id,
+            lids: lids
+          }.compact
+        end
+
+        # Array of local ids that uniquely identify this record
+        def lids
+          @dedupe_on.map do |fields|
+            generate_lid(fields)
+          end.compact.uniq
+        end
+
+        # For a given set of fields of this model, generate a
+        # unique local id by hashing the field values
+        def generate_lid fields
+          values = fields.sort.map do |field|
+            instance_variable = "@#{field.to_s}"
+            self.instance_variable_get(instance_variable)
+          end
+
+          return if values.any? { |e| e.nil? }
+
+          Digest::SHA256.hexdigest(values.join(","))
+        end
+
+        # Set of attribute names that this model has is Base's shared
+        # attributes combined with the child class's
+        def attribute_list
+          (ATTRIBUTES + self.class::ATTRIBUTES).uniq
+        end
+
+        # All of this record's attributes
+        def attributes
+          attributes = {}
+          attribute_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            attributes[attribute] = self.instance_variable_get(instance_variable)
+          end
+          attributes.compact
+        end
+
+        # All of this record's associations
+        def associations
+          association_list = ASSOCIATIONS + self.class::ASSOCIATIONS
+          attributes = {}
+          association_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            association = self.instance_variable_get(instance_variable)
+            attributes[attribute] = association if association
+          end
+          attributes.compact
+        end
+
+        def associations_hash
+          Hash[associations.map do |k, v|
+            [k, v.to_h]
+          end]
+        end
+
+        # FIXME: move this to a Utils module
+        def to_h_flattened
+          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
+        end
+
+        def to_h
+          identifier_hash.merge(attributes).merge(associations_hash)
+        end
+
+        private
+
+        def assign_attributes attributes
+          attributes.each do |k, v|
+            setter = :"#{k}="
+            public_send(setter, v) if respond_to? setter
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/entity.rb

@@ -0,0 +1,15 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Entity < Chronicle::ETL::Models::Base
+        TYPE = 'entities'.freeze
+        ATTRIBUTES = [:title, :body, :represents, :slug].freeze
+        ASSOCIATIONS = [].freeze  # TODO: add these to reflect Chronicle Schema
+
+        attr_accessor(*ATTRIBUTES)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/generic.rb

@@ -0,0 +1,23 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Generic < Chronicle::ETL::Models::Base
+        TYPE = 'generic'
+
+        attr_accessor :properties
+
+        def initialize(properties = {})
+          @properties = properties
+          super
+        end
+
+        # Generic models have arbitrary attributes stored in @properties
+        def attributes
+          @properties.transform_keys(&:to_sym)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,59 +1,37 @@
-class Chronicle::Etl::Runner
-  BUILTIN = {
-    extractor: ['stdin', 'json', 'csv', 'file'],
-    transformer: ['null'],
-    loader: ['stdout', 'csv', 'table']
-  }.freeze
+require 'colorize'
 
-  def initialize(options)
-    @options = options
-
-    instantiate_etl_classes
+class Chronicle::ETL::Runner
+  def initialize(job)
+    @job = job
+    @job_logger = Chronicle::ETL::JobLogger.new(@job)
   end
 
   def run!
-    total = @extractor.results_count
-    progress_bar = Chronicle::Etl::Utils::ProgressBar.new(title: 'Running job', total: total)
-    count = 0
+    extractor = @job.instantiate_extractor
+    loader = @job.instantiate_loader
+
+    @job_logger.start
+    loader.start
+
+    total = extractor.results_count
+    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
 
-    @loader.start
+    extractor.extract do |data, metadata|
+      transformer = @job.instantiate_transformer(data)
+      record = transformer.transform
 
-    @extractor.extract do |data, metadata|
-      transformed_data = @transformer.transform(data)
-      @loader.load(transformed_data)
+      unless record.is_a?(Chronicle::ETL::Models::Base)
+        raise Chronicle::ETL::InvalidTransformedRecordError, "Transformed data is not a type of Chronicle::ETL::Models"
+      end
 
+      @job_logger.log_transformation(transformer)
+      loader.load(record)
       progress_bar.increment
-      count += 1
     end
 
     progress_bar.finish
-    @loader.finish
-  end
-
-  private
-
-  def instantiate_etl_classes
-    @extractor = load_etl_class(:extractor, @options[:extractor][:name]).new(@options[:extractor][:options])
-    @transformer = load_etl_class(:transformer, @options[:transformer][:name]).new(@options[:transformer][:options])
-    @loader = load_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
-  end
-
-  def load_etl_class(phase, x)
-    if BUILTIN[phase].include? x
-      klass_name = "Chronicle::Etl::#{x.capitalize}#{phase.to_s.capitalize}"
-    else
-      # TODO: come up with syntax for specifying a particular extractor in a provider library
-      provider, name = x.split(":")
-      provider = x unless provider
-      begin
-        require "chronicle/#{provider}"
-      rescue LoadError => e
-        warn("Error loading #{phase} '#{provider}'")
-        warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
-        exit(false)
-      end
-      klass_name = "Chronicle::#{provider.capitalize}::#{name&.capitalize}#{phase.capitalize}"
-    end
-    Object.const_get(klass_name)
+    loader.finish
+    @job_logger.finish
+    @job_logger.save
   end
 end