chronicle-etl 0.2.4 → 0.3.0

data/lib/chronicle/etl/registry/registry.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+
+module Chronicle
+  module ETL
+    # A singleton class that acts as a registry of connector classes available for ETL jobs
+    module Registry
+      PHASES = [:extractor, :transformer, :loader]
+
+      class << self
+        attr_accessor :connectors
+
+        def load_all!
+          load_connectors_from_gems
+        end
+
+        def load_connectors_from_gems
+          Gem::Specification.filter{|s| s.name.match(/^chronicle/) }.each do |gem|
+            require_str = gem.name.gsub('chronicle-', 'chronicle/')
+            require require_str rescue LoadError
+          end
+        end
+
+        def install_connector name
+          gem_name = "chronicle-#{name}"
+          Gem.install(gem_name)
+        end
+
+        def register connector
+          @connectors ||= []
+          @connectors << connector
+        end
+
+        def find_by_phase_and_identifier(phase, identifier)
+          connector = find_within_loaded_connectors(phase, identifier)
+          unless connector
+            # Only load external connectors (slow) if not found in built-in connectors
+            load_all!
+            connector = find_within_loaded_connectors(phase, identifier)
+          end
+          connector || raise(ConnectorNotAvailableError.new("Connector '#{identifier}' not found"))
+        end
+
+        def find_within_loaded_connectors(phase, identifier)
+          @connectors.find { |c| c.phase == phase && c.identifier == identifier }
+        end
+      end
+    end
+  end
+end
+
+require_relative 'self_registering'
+require_relative 'connector_registration'

data/lib/chronicle/etl/registry/self_registering.rb

@@ -0,0 +1,25 @@
+require 'forwardable'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Gives a connector class the ability to let the Chronicle::ETL::Registry
+      # know about itself
+      module SelfRegistering
+        extend Forwardable
+
+        attr_accessor :connector_registration
+
+        def_delegators :@connector_registration, :description, :provider, :identifier
+
+        # Creates a ConnectorRegistration for this connector's details and register's it
+        # into the Registry
+        def register_connector
+          @connector_registration ||= ::Chronicle::ETL::Registry::ConnectorRegistration.new(self)
+          yield @connector_registration if block_given?
+          ::Chronicle::ETL::Registry.register(@connector_registration)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,4 +1,5 @@
 require 'colorize'
+require 'chronic_duration'
 
 class Chronicle::ETL::Runner
   def initialize(job)
@@ -14,24 +15,73 @@ class Chronicle::ETL::Runner
     loader.start
 
     total = extractor.results_count
-    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+    @progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+    Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
 
-    extractor.extract do |data, metadata|
-      transformer = @job.instantiate_transformer(data)
+    Chronicle::ETL::Logger.info(tty_log_job_start)
+    extractor.extract do |extraction|
+      unless extraction.is_a?(Chronicle::ETL::Extraction)
+        raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
+      end
+
+      transformer = @job.instantiate_transformer(extraction)
       record = transformer.transform
 
       unless record.is_a?(Chronicle::ETL::Models::Base)
-        raise Chronicle::ETL::InvalidTransformedRecordError, "Transformed data is not a type of Chronicle::ETL::Models"
+        raise Chronicle::ETL::RunnerTypeError, "Transformed data should be a type of Chronicle::ETL::Models"
       end
 
+      Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
       @job_logger.log_transformation(transformer)
-      loader.load(record)
-      progress_bar.increment
+
+      loader.load(record) unless @job.dry_run?
+    rescue Chronicle::ETL::TransformationError => e
+      Chronicle::ETL::Logger.error(tty_log_transformation_failure(e))
+    ensure
+      @progress_bar.increment
     end
 
-    progress_bar.finish
+    @progress_bar.finish
     loader.finish
     @job_logger.finish
+  rescue Interrupt
+    Chronicle::ETL::Logger.error("\n#{'Job interrupted'.red}")
+    @job_logger.error
+  rescue StandardError => e
+    raise e
+  ensure
     @job_logger.save
+    @progress_bar.finish
+    Chronicle::ETL::Logger.detach_from_progress_bar
+    Chronicle::ETL::Logger.info(tty_log_completion)
+  end
+
+  private
+
+  def tty_log_job_start
+    output = "Beginning job "
+    output += "'#{@job.name}'".bold if @job.name
+    output
+  end
+
+  def tty_log_transformation transformer
+    output = "  ✓".green
+    output += " #{transformer}"
+  end
+
+  def tty_log_transformation_failure exception
+    output = "  ✖".red
+    output += " Failed to build #{exception.transformation}. #{exception.message}"
+  end
+
+  def tty_log_completion
+    status = @job_logger.success ? 'Success' : 'Failed'
+    output = "\nCompleted job "
+    output += "'#{@job.name}'".bold if @job.name
+    output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
+    output += "\n  Status:\t".light_black + status
+    output += "\n  Completed:\t".light_black + "#{@job_logger.job_log.num_records_processed}"
+    output += "\n  Latest:\t".light_black + "#{@job_logger.job_log.highest_timestamp.iso8601}" if @job_logger.job_log.highest_timestamp
+    output
   end
 end

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -0,0 +1,25 @@
+module Chronicle
+  module ETL
+    class JSONAPISerializer < Chronicle::ETL::Serializer
+      def serializable_hash
+        @record
+          .identifier_hash
+          .merge({ attributes: @record.attributes })
+          .merge({ relationships: build_associations })
+          .merge(@record.meta_hash)
+      end
+
+      def build_associations
+        @record.associations.transform_values do |value|
+          association_data =
+            if value.is_a?(Array)
+              value.map { |record| JSONAPISerializer.new(record).serializable_hash }
+            else
+              JSONAPISerializer.new(value).serializable_hash
+            end
+          { data: association_data }
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -0,0 +1,27 @@
+module Chronicle
+  module ETL
+    # Abstract class representing a Serializer for an ETL record
+    class Serializer
+      # Construct a new instance of this serializer.
+      # == Parameters:
+      # options::
+      #   Options for configuring this Serializers
+      def initialize(record, options = {})
+        @record = record
+        @options = options
+      end
+
+      # Serialize a record as a hash
+      def serializable_hash
+        raise NotImplementedError
+      end
+
+      def self.serialize(record)
+        serializer = self.new(record)
+        serializer.serializable_hash
+      end
+    end
+  end
+end
+
+require_relative 'jsonapi_serializer'

data/lib/chronicle/etl/transformers/image_file_transformer.rb

@@ -0,0 +1,253 @@
+require 'mini_exiftool'
+require 'active_support'
+require 'active_support/core_ext/object'
+require 'active_support/core_ext/time'
+require 'active_support/core_ext/hash/reverse_merge'
+require 'active_support/core_ext/string/inflections'
+
+module Chronicle
+  module ETL
+    # Transform a JPEG or other image file into a record.
+    # By default, file mtime and a hash of the file content is used to build
+    # the timestamp and ID respectively but other options are available (such
+    # as reading EXIF tags or extended attributes from the filesystem).
+    #
+    # TODO: This should be extracted into its own plugin
+    class ImageFileTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = 'image-file'
+        r.description = 'an image file'
+      end
+
+      DEFAULT_OPTIONS = {
+        timestamp_strategy: 'file_mtime',
+        id_strategy: 'file_hash',
+        verb: 'photographed',
+
+        # EXIF tags often don't have timezones
+        timezone_default: 'Eastern Time (US & Canada)',
+        include_image_data: true
+      }.freeze
+
+      def initialize(*args)
+        super(*args)
+        @options = @options.reverse_merge(DEFAULT_OPTIONS)
+      end
+
+      def transform
+        # FIXME: set @filename; use block for reading file when necessary
+        @file = File.open(@extraction.data)
+        record = build_created(@file)
+        @file.close
+        record
+      end
+
+      def friendly_identifier
+        @file.path
+      end
+
+      def id
+        @id ||= begin
+          id = build_with_strategy(field: :id, strategy: @options[:id_strategy])
+          raise UntransformableRecordError.new("Could not build id", transformation: self) unless id
+
+          id
+        end
+      end
+
+      def timestamp
+        @timestamp ||= begin
+          ts = build_with_strategy(field: :timestamp, strategy: @options[:timestamp_strategy])
+          raise UntransformableRecordError.new("Could not build timestamp", transformation: self) unless ts
+
+          ts
+        end
+      end
+
+      private
+
+      def build_created(file)
+        record = ::Chronicle::ETL::Models::Activity.new
+        record.verb = @options[:verb]
+        record.provider = @options[:provider]
+        record.provider_id = id
+        record.end_at = timestamp
+        record.dedupe_on = [[:provider_id, :verb, :provider]]
+
+        record.involved = build_image
+        record.actor = build_actor
+
+        record.assign_attributes(build_gps)
+        record
+      end
+
+      def build_actor
+        actor = ::Chronicle::ETL::Models::Entity.new
+        actor.represents = 'identity'
+        actor.provider = @options[:actor][:provider]
+        actor.slug = @options[:actor][:slug]
+        actor.dedupe_on = [[:provider, :slug, :represents]]
+        actor
+      end
+
+      def build_image
+        image = ::Chronicle::ETL::Models::Entity.new
+        image.represents = @options[:involved][:represents]
+        image.title = build_title
+        image.body = exif['Description']
+        image.provider = @options[:involved][:provider]
+        image.provider_id = id
+        image.assign_attributes(build_gps)
+        image.dedupe_on = [[:provider, :provider_id, :represents]]
+
+        if @options[:ocr_strategy]
+          ocr_text = build_with_strategy(field: :ocr, strategy: @options[:ocr_strategy])
+          image.metadata[:ocr_text] = ocr_text if ocr_text
+        end
+
+        names = extract_people_depicted
+        tags = extract_keywords(names)
+
+        image.depicts = build_people_depicted(names)
+        image.abouts = build_keywords(tags)
+
+        if @options[:include_image_data]
+          attachment = ::Chronicle::ETL::Models::Attachment.new
+          attachment.data = build_image_data
+          image.attachments = [attachment]
+        end
+
+        image
+      end
+
+      def build_keywords(topics)
+        topics.map do |topic|
+          t = ::Chronicle::ETL::Models::Entity.new
+          t.represents = 'topic'
+          t.provider = @options[:involved][:provider]
+          t.title = topic
+          t.slug = topic.parameterize
+          t.dedupe_on = [[:provider, :represents, :slug]]
+          t
+        end
+      end
+
+      def build_people_depicted(names)
+        names.map do |name|
+          identity = ::Chronicle::ETL::Models::Entity.new
+          identity.represents = 'identity'
+          identity.provider = @options[:involved][:provider]
+          identity.slug = name.parameterize
+          identity.title = name
+          identity.dedupe_on = [[:provider, :represents, :slug]]
+          identity
+        end
+      end
+
+      def build_gps
+        return {} unless exif['GPSLatitude']
+
+        {
+          lat: exif['GPSLatitude'],
+          lng: exif['GPSLongitude'],
+          elevation: exif['GPSAltitude']
+        }
+      end
+
+      def build_image_data
+        ::Chronicle::ETL::Utils::BinaryAttachments.filename_to_base64(filename: @file.path)
+      end
+
+      def build_title
+        File.basename(@file)
+      end
+
+      def build_with_strategy(field:, strategy:[])
+        strategies = [strategy].flatten.compact
+        strategies.each do |s|
+          builder_method = "build_#{field}_using_#{s}"
+          result = send(builder_method.to_sym)
+          return result if result
+        end
+        return
+      end
+
+      def build_id_using_file_hash
+        Digest::SHA256.hexdigest(File.read(@file))
+      end
+
+      def build_id_using_xattr_version
+        load_value_from_xattr_plist("com.apple.metadata:kMDItemVersion")
+      end
+
+      def build_id_using_xmp_document_id
+        exif['OriginalDocumentID'] || exif['DerivedFromDocumentID']
+      end
+
+      def build_timestamp_using_file_mtime
+        File.mtime(@file)
+      end
+
+      def build_timestamp_using_exif_datetimeoriginal
+        # EXIF tags don't have timezone information. This is a DateTime in UTC
+        timestamp = exif['DateTimeOriginal'] || return
+
+        if exif['OffsetTimeOriginal']
+          # Offset tags are only available in newer EXIF tags. If it exists, we
+          # use it instead of UTC
+          timestamp = timestamp.change(offset: exif['OffsetTimeOriginal'])
+        elsif false
+          # TODO: support option of using GPS coordinates to determine timezone
+        else
+          zone = ActiveSupport::TimeZone.new(@options[:timezone_default])
+          timestamp = zone.parse(timestamp.asctime)
+        end
+
+        timestamp
+      end
+
+      # TODO: add documentation for how to set up `macocr`
+      def build_ocr_using_macocr
+        `macocr "#{@file.path}" 2>/dev/null`.presence
+      end
+
+      def exif
+        @exif ||= MiniExiftool.new(
+          @file.path,
+          numerical: true,
+
+          # EXIF timestamps don't have timezone information. MiniExifTool uses Time
+          # by default which parses timestamps in local time zone. Using DateTime
+          # parses dates as UTC and then we can apply a timezone offset if the optional
+          # EXIF timezone offset fields are available.
+          # https://github.com/janfri/mini_exiftool/issues/39#issuecomment-832587649
+          timestamps: DateTime
+        )
+      end
+
+      # Figure out which faces are tagged as regions and return a list of their names
+      def extract_people_depicted
+        return [] unless exif['RegionName']
+
+        names = [exif['RegionName']].flatten
+        types = [exif['RegionType']].flatten
+
+        names.zip(types).select{|x| x[1] == 'Face'}.map{|x| x[0]}.uniq
+      end
+
+      # Extract image keywords from EXIF/IPTC tag and subtract out those of which are
+      # tagged people (determiend by looking at face regions)
+      def extract_keywords(people_names = [])
+        [exif['Keywords'] || []].flatten - people_names
+      end
+
+      def load_value_from_xattr_plist attribute
+        require 'nokogiri'
+        xml = `xattr -p #{attribute} \"#{@file.path}\" | xxd -r -p | plutil -convert xml1 -o - -- - 2>/dev/null`
+        return unless xml
+        value = Nokogiri::XML.parse(r).xpath("//string").text
+        return value.presence
+      end
+    end
+  end
+end

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -1,9 +1,18 @@
 module Chronicle
   module ETL
     class NullTransformer < Chronicle::ETL::Transformer
+      register_connector do |r|
+        r.identifier = 'null'
+        r.description = 'in no way'
+      end
+
       def transform
-        Chronicle::ETL::Models::Generic.new(@data)
+        Chronicle::ETL::Models::Generic.new(@extraction.data)
       end
+
+      def timestamp; end
+
+      def id; end
     end
   end
 end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -2,16 +2,15 @@ module Chronicle
   module ETL
     # Abstract class representing an Transformer for an ETL job
     class Transformer
-      extend Chronicle::ETL::Catalog
+      extend Chronicle::ETL::Registry::SelfRegistering
 
       # Construct a new instance of this transformer. Options are passed in from a Runner
-      # == Paramters:
+      # == Parameters:
       # options::
       #   Options for configuring this Transformer
-      def initialize(options = {}, data)
+      def initialize(options = {}, extraction)
         @options = options
-        @data = data
-        @record = Chronicle::ETL::Models::Activity.new
+        @extraction = extraction
       end
 
       # @abstract Subclass is expected to implement #transform
@@ -19,16 +18,47 @@ module Chronicle
       #   The main entrypoint for transforming a record. Called by a Runner on each extracted record
 
       # The domain or provider-specific id of the record this transformer is working on.
-      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
-      # data source from the beginning.
-      def id; end
+      # It is useful for: 
+      # - de-duping records that might exist in the loader's destination
+      # - building a cursor so an extractor doesn't have to start from the beginning of a 
+      #   a source 
+      def id
+        raise NotImplementedError
+      end
 
       # The domain or provider-specific timestamp of the record this transformer is working on.
       # Used for building a cursor so an extractor doesn't have to start from the beginning of a
       # data source from the beginning.
-      def timestamp; end
+      def timestamp
+        raise NotImplementedError
+      end
+
+      # An optional, human-readable identifier for a transformation, intended for debugging or logging.
+      # By default, it is just the id.
+      def friendly_identifier
+        id
+      end
+
+      def to_s
+        ts = begin
+          unknown = "???"
+          timestamp&.iso8601 || unknown
+        rescue TransformationError, NotImplementedError
+          unknown
+        end
+
+        identifier = begin
+          unknown = self.class.to_s
+          friendly_identifier || self.class.to_s
+        rescue TransformationError, NotImplementedError
+          unknown
+        end
+
+        "[#{ts}] #{identifier}"
+      end
     end
   end
 end
 
 require_relative 'null_transformer'
+require_relative 'image_file_transformer'

data/lib/chronicle/etl/utils/binary_attachments.rb

@@ -0,0 +1,21 @@
+require 'marcel'
+require 'base64'
+
+module Chronicle
+  module ETL
+    module Utils
+      # Utility methods for dealing with binary files
+      module BinaryAttachments
+        def self.filename_to_base64(filename:, mimetype: nil)
+          mimetype = mimetype || guess_mimetype(filename: filename)
+
+          "data:#{mimetype};base64,#{Base64.strict_encode64(File.read(filename))}"
+        end
+
+        def self.guess_mimetype(filename:)
+          Marcel::MimeType.for(filename)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -64,7 +64,9 @@ module Chronicle
         end
 
         def log(message)
-          @pbar.log message
+          message.split("\n").each do |line|
+            @pbar.log message
+          end
         end
 
         def finish

data/lib/chronicle/etl/utils/text_recognition.rb

@@ -0,0 +1,15 @@
+require 'active_support/core_ext/object/blank'
+
+module Chronicle
+  module ETL
+    module Utils
+      # OCR for image files
+      # TODO: add other strategies and document `macocr`
+      module TextRecognition
+        def self.recognize_in_image(filename:)
+          `macocr "#{filename}" 2>/dev/null`.presence
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.2.4"
+    VERSION = "0.3.0"
   end
 end

data/lib/chronicle/etl.rb

@@ -1,19 +1,24 @@
-require_relative 'etl/catalog'
+require_relative 'etl/registry/registry'
 require_relative 'etl/config'
 require_relative 'etl/exceptions'
+require_relative 'etl/extraction'
 require_relative 'etl/extractors/extractor'
 require_relative 'etl/job_definition'
 require_relative 'etl/job_log'
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
 require_relative 'etl/loaders/loader'
+require_relative 'etl/logger'
 require_relative 'etl/models/activity'
+require_relative 'etl/models/attachment'
 require_relative 'etl/models/base'
 require_relative 'etl/models/entity'
 require_relative 'etl/models/generic'
 require_relative 'etl/runner'
+require_relative 'etl/serializers/serializer'
 require_relative 'etl/transformers/transformer'
+require_relative 'etl/utils/binary_attachments'
 require_relative 'etl/utils/hash_utilities'
-require_relative 'etl/utils/jsonapi'
+require_relative 'etl/utils/text_recognition'
 require_relative 'etl/utils/progress_bar'
 require_relative 'etl/version'