chronicle-etl 0.3.0 → 0.4.1

data/lib/chronicle/etl/loaders/json_loader.rb

@@ -0,0 +1,44 @@
+module Chronicle
+  module ETL
+    class JSONLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'json'
+      end
+
+      setting :serializer
+      setting :output, default: $stdout
+
+      def start
+        if @config.output == $stdout
+          @output = @config.output
+        else
+          @output = File.open(@config.output, "w")
+        end
+      end
+
+      def load(record)
+        serialized = serializer.serialize(record)
+
+        # When dealing with raw data, we can get improperly encoded strings
+        # (eg from sqlite database columns). We force conversion to UTF-8
+        # before converting into JSON
+        encoded = serialized.transform_values do |value|
+          next value unless value.is_a?(String)
+
+          value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        end
+        @output.puts encoded.to_json
+      end
+
+      def finish
+        @output.close
+      end
+
+      private
+
+      def serializer
+        @config.serializer || Chronicle::ETL::RawSerializer
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/loader.rb

@@ -3,13 +3,16 @@ module Chronicle
     # Abstract class representing a Loader for an ETL job
     class Loader
       extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
+
+      setting :output
 
       # Construct a new instance of this loader. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Loader
       def initialize(options = {})
-        @options = options
+        apply_options(options)
       end
 
       # Called once before processing records
@@ -27,6 +30,6 @@ module Chronicle
 end
 
 require_relative 'csv_loader'
+require_relative 'json_loader'
 require_relative 'rest_loader'
-require_relative 'stdout_loader'
 require_relative 'table_loader'

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -9,19 +9,19 @@ module Chronicle
         r.description = 'a REST endpoint'
       end
 
-      def initialize( options={} )
-        super(options)
-      end
+      setting :hostname, required: true
+      setting :endpoint, required: true
+      setting :access_token
 
       def load(record)
         payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
         # have the outer data key that json-api expects
         payload = { data: payload } unless payload[:data]
 
-        uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
+        uri = URI.parse("#{@config.hostname}#{@config.endpoint}")
 
         header = {
-          "Authorization" => "Bearer #{@options[:access_token]}",
+          "Authorization" => "Bearer #{@config.access_token}",
           "Content-Type": 'application/json'
         }
         use_ssl = uri.scheme == 'https'

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -9,59 +9,56 @@ module Chronicle
         r.description = 'an ASCII table'
       end
 
-      DEFAULT_OPTIONS = {
-        fields_limit: nil,
-        fields_exclude: ['lids', 'type'],
-        fields_include: [],
-        truncate_values_at: nil,
-        table_renderer: :basic
-      }.freeze
-
-      def initialize(options={})
-        @options = options.reverse_merge(DEFAULT_OPTIONS)
-        @records = []
-      end
+      setting :fields_limit, default: nil
+      setting :fields_exclude, default: ['lids', 'type']
+      setting :fields, default: []
+      setting :truncate_values_at, default: 40
+      setting :table_renderer, default: :basic
 
       def load(record)
-        @records << record.to_h_flattened
+        records << record.to_h_flattened
       end
 
       def finish
-        return if @records.empty?
+        return if records.empty?
 
-        headers = build_headers(@records)
-        rows = build_rows(@records, headers)
+        headers = build_headers(records)
+        rows = build_rows(records, headers)
 
         @table = TTY::Table.new(header: headers, rows: rows)
         puts @table.render(
-          @options[:table_renderer].to_sym,
+          @config.table_renderer.to_sym,
           padding: [0, 2, 0, 0]
         )
       end
 
+      def records
+        @records ||= []
+      end
+
       private
 
       def build_headers(records)
         headers =
-          if @options[:fields_include].any?
-            Set[*@options[:fields_include]]
+          if @config.fields.any?
+            Set[*@config.fields]
           else
             # use all the keys of the flattened record hash
             Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
           end
 
-        headers = headers.delete_if { |header| header.end_with?(*@options[:fields_exclude]) } if @options[:fields_exclude].any?
-        headers = headers.first(@options[:fields_limit]) if @options[:fields_limit]
+        headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) } if @config.fields_exclude.any?
+        headers = headers.first(@config.fields_limit) if @config.fields_limit
 
         headers.to_a.map(&:to_sym)
       end
 
       def build_rows(records, headers)
         records.map do |record|
-          values = record.values_at(*headers).map{|value| value.to_s }
+          values = record.transform_keys(&:to_sym).values_at(*headers).map{|value| value.to_s }
 
-          if @options[:truncate_values_at]
-            values = values.map{ |value| value.truncate(@options[:truncate_values_at]) }
+          if @config.truncate_values_at
+            values = values.map{ |value| value.truncate(@config.truncate_values_at) }
           end
 
           values

data/lib/chronicle/etl/logger.rb

@@ -8,6 +8,7 @@ module Chronicle
       WARN = 2
       ERROR = 3
       FATAL = 4
+      SILENT = 5
 
       attr_accessor :log_level
 

data/lib/chronicle/etl/models/base.rb

@@ -5,6 +5,9 @@ module Chronicle
     module Models
       # Represents a record that's been transformed by a Transformer and
       # ready to be loaded. Loosely based on ActiveModel.
+      #
+      # @todo Experiment with just mixing in ActiveModel instead of this
+      #   this reimplementation
       class Base
         ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
         ASSOCIATIONS = [].freeze

data/lib/chronicle/etl/models/entity.rb

@@ -5,13 +5,19 @@ module Chronicle
     module Models
       class Entity < Chronicle::ETL::Models::Base
         TYPE = 'entities'.freeze
-        ATTRIBUTES = [:title, :body, :represents, :slug, :myself, :metadata].freeze
+        ATTRIBUTES = [:title, :body, :provider_url, :represents, :slug, :myself, :metadata].freeze
+
+        # TODO: This desperately needs a validation system
         ASSOCIATIONS = [
+          :involvements, # inverse of activity's `involved`
+
           :attachments,
           :abouts,
+          :aboutables, # inverse of above
           :depicts,
           :consumers,
-          :contains
+          :contains,
+          :containers # inverse of above
         ].freeze  # TODO: add these to reflect Chronicle Schema
 
         attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)

data/lib/chronicle/etl/models/raw.rb

@@ -0,0 +1,26 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      # A record from an extraction with no processing or normalization applied
+      class Raw
+        TYPE = 'raw'
+
+        attr_accessor :raw_data
+
+        def initialize(raw_data)
+          @raw_data = raw_data
+        end
+
+        def to_h
+          @raw_data.to_h
+        end
+
+        def to_h_flattened
+          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -3,6 +3,7 @@ module Chronicle
     module Registry
       # Records details about a connector such as its provider and a description
       class ConnectorRegistration
+        # FIXME: refactor custom accessor methods later in file
         attr_accessor :identifier, :provider, :klass, :description
 
         def initialize(klass)

data/lib/chronicle/etl/runner.rb

@@ -14,6 +14,7 @@ class Chronicle::ETL::Runner
     @job_logger.start
     loader.start
 
+    extractor.prepare
     total = extractor.results_count
     @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
     Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
@@ -27,9 +28,10 @@ class Chronicle::ETL::Runner
       transformer = @job.instantiate_transformer(extraction)
       record = transformer.transform
 
-      unless record.is_a?(Chronicle::ETL::Models::Base)
-        raise Chronicle::ETL::RunnerTypeError, "Transformed data should be a type of Chronicle::ETL::Models"
-      end
+      # TODO: rethink this
+      # unless record.is_a?(Chronicle::ETL::Models)
+      #   raise Chronicle::ETL::RunnerTypeError, "Transformed data should be a type of Chronicle::ETL::Models"
+      # end
 
       Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
       @job_logger.log_transformation(transformer)
@@ -51,7 +53,7 @@ class Chronicle::ETL::Runner
     raise e
   ensure
     @job_logger.save
-    @progress_bar.finish
+    @progress_bar&.finish
     Chronicle::ETL::Logger.detach_from_progress_bar
     Chronicle::ETL::Logger.info(tty_log_completion)
   end

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -1,6 +1,12 @@
 module Chronicle
   module ETL
     class JSONAPISerializer < Chronicle::ETL::Serializer
+      def initialize(*args)
+        super
+
+        raise(SerializationError, "Record must be a subclass of Chronicle::ETL::Model::Base") unless @record.is_a?(Chronicle::ETL::Models::Base)
+      end
+
       def serializable_hash
         @record
           .identifier_hash

data/lib/chronicle/etl/serializers/raw_serializer.rb

@@ -0,0 +1,10 @@
+module Chronicle
+  module ETL
+    # Take a Raw model and output `raw_data` as a hash
+    class RawSerializer < Chronicle::ETL::Serializer
+      def serializable_hash
+        @record.to_h
+      end
+    end
+  end
+end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -24,4 +24,5 @@ module Chronicle
   end
 end
 
-require_relative 'jsonapi_serializer'
+require_relative 'jsonapi_serializer'
+require_relative 'raw_serializer'

data/lib/chronicle/etl/transformers/image_file_transformer.rb

@@ -19,20 +19,14 @@ module Chronicle
         r.description = 'an image file'
       end
 
-      DEFAULT_OPTIONS = {
-        timestamp_strategy: 'file_mtime',
-        id_strategy: 'file_hash',
-        verb: 'photographed',
-
-        # EXIF tags often don't have timezones
-        timezone_default: 'Eastern Time (US & Canada)',
-        include_image_data: true
-      }.freeze
-
-      def initialize(*args)
-        super(*args)
-        @options = @options.reverse_merge(DEFAULT_OPTIONS)
-      end
+      setting :timestamp_strategy, default: 'file_mtime'
+      setting :id_strategy, default: 'file_hash'
+      setting :verb, default: 'photographed'
+      # EXIF tags often don't have timezones
+      setting :timezone_default, default: 'Eastern Time (US & Canada)'
+      setting :include_image_data, default: true
+      setting :actor
+      setting :involved
 
       def transform
         # FIXME: set @filename; use block for reading file when necessary
@@ -48,7 +42,7 @@ module Chronicle
 
       def id
         @id ||= begin
-          id = build_with_strategy(field: :id, strategy: @options[:id_strategy])
+          id = build_with_strategy(field: :id, strategy: @config.id_strategy)
           raise UntransformableRecordError.new("Could not build id", transformation: self) unless id
 
           id
@@ -57,7 +51,7 @@ module Chronicle
 
       def timestamp
         @timestamp ||= begin
-          ts = build_with_strategy(field: :timestamp, strategy: @options[:timestamp_strategy])
+          ts = build_with_strategy(field: :timestamp, strategy: @config.timestamp_strategy)
           raise UntransformableRecordError.new("Could not build timestamp", transformation: self) unless ts
 
           ts
@@ -68,8 +62,8 @@ module Chronicle
 
       def build_created(file)
         record = ::Chronicle::ETL::Models::Activity.new
-        record.verb = @options[:verb]
-        record.provider = @options[:provider]
+        record.verb = @config.verb
+        record.provider = @config.provider
         record.provider_id = id
         record.end_at = timestamp
         record.dedupe_on = [[:provider_id, :verb, :provider]]
@@ -84,24 +78,24 @@ module Chronicle
       def build_actor
         actor = ::Chronicle::ETL::Models::Entity.new
         actor.represents = 'identity'
-        actor.provider = @options[:actor][:provider]
-        actor.slug = @options[:actor][:slug]
+        actor.provider = @config.actor[:provider]
+        actor.slug = @config.actor[:slug]
         actor.dedupe_on = [[:provider, :slug, :represents]]
         actor
       end
 
       def build_image
         image = ::Chronicle::ETL::Models::Entity.new
-        image.represents = @options[:involved][:represents]
+        image.represents = @config.involved[:represents]
         image.title = build_title
         image.body = exif['Description']
-        image.provider = @options[:involved][:provider]
+        image.provider = @config.involved[:provider]
         image.provider_id = id
         image.assign_attributes(build_gps)
         image.dedupe_on = [[:provider, :provider_id, :represents]]
 
-        if @options[:ocr_strategy]
-          ocr_text = build_with_strategy(field: :ocr, strategy: @options[:ocr_strategy])
+        if @config.ocr_strategy
+          ocr_text = build_with_strategy(field: :ocr, strategy: @config.ocr_strategy)
           image.metadata[:ocr_text] = ocr_text if ocr_text
         end
 
@@ -111,7 +105,7 @@ module Chronicle
         image.depicts = build_people_depicted(names)
         image.abouts = build_keywords(tags)
 
-        if @options[:include_image_data]
+        if @config.include_image_data
           attachment = ::Chronicle::ETL::Models::Attachment.new
           attachment.data = build_image_data
           image.attachments = [attachment]
@@ -124,7 +118,7 @@ module Chronicle
         topics.map do |topic|
           t = ::Chronicle::ETL::Models::Entity.new
           t.represents = 'topic'
-          t.provider = @options[:involved][:provider]
+          t.provider = @config.involved[:provider]
           t.title = topic
           t.slug = topic.parameterize
           t.dedupe_on = [[:provider, :represents, :slug]]
@@ -136,7 +130,7 @@ module Chronicle
         names.map do |name|
           identity = ::Chronicle::ETL::Models::Entity.new
           identity.represents = 'identity'
-          identity.provider = @options[:involved][:provider]
+          identity.provider = @config.involved[:provider]
           identity.slug = name.parameterize
           identity.title = name
           identity.dedupe_on = [[:provider, :represents, :slug]]
@@ -199,7 +193,7 @@ module Chronicle
         elsif false
           # TODO: support option of using GPS coordinates to determine timezone
         else
-          zone = ActiveSupport::TimeZone.new(@options[:timezone_default])
+          zone = ActiveSupport::TimeZone.new(@config.timezone_default)
           timestamp = zone.parse(timestamp.asctime)
         end
 

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -7,7 +7,7 @@ module Chronicle
       end
 
       def transform
-        Chronicle::ETL::Models::Generic.new(@extraction.data)
+        Chronicle::ETL::Models::Raw.new(@extraction.data)
       end
 
       def timestamp; end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -3,14 +3,15 @@ module Chronicle
     # Abstract class representing an Transformer for an ETL job
     class Transformer
       extend Chronicle::ETL::Registry::SelfRegistering
+      include Chronicle::ETL::Configurable
 
       # Construct a new instance of this transformer. Options are passed in from a Runner
       # == Parameters:
       # options::
       #   Options for configuring this Transformer
-      def initialize(options = {}, extraction)
-        @options = options
+      def initialize(extraction, options = {})
         @extraction = extraction
+        apply_options(options)
       end
 
       # @abstract Subclass is expected to implement #transform

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.3.0"
+    VERSION = "0.4.1"
   end
 end

data/lib/chronicle/etl.rb

@@ -1,24 +1,32 @@
 require_relative 'etl/registry/registry'
 require_relative 'etl/config'
+require_relative 'etl/configurable'
 require_relative 'etl/exceptions'
 require_relative 'etl/extraction'
-require_relative 'etl/extractors/extractor'
 require_relative 'etl/job_definition'
 require_relative 'etl/job_log'
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
-require_relative 'etl/loaders/loader'
 require_relative 'etl/logger'
 require_relative 'etl/models/activity'
 require_relative 'etl/models/attachment'
 require_relative 'etl/models/base'
+require_relative 'etl/models/raw'
 require_relative 'etl/models/entity'
-require_relative 'etl/models/generic'
 require_relative 'etl/runner'
 require_relative 'etl/serializers/serializer'
-require_relative 'etl/transformers/transformer'
 require_relative 'etl/utils/binary_attachments'
 require_relative 'etl/utils/hash_utilities'
 require_relative 'etl/utils/text_recognition'
 require_relative 'etl/utils/progress_bar'
 require_relative 'etl/version'
+
+require_relative 'etl/extractors/extractor'
+require_relative 'etl/loaders/loader'
+require_relative 'etl/transformers/transformer'
+
+begin
+  require 'pry'
+rescue LoadError
+  # Pry not available
+end