chronicle-etl 0.5.4 → 0.6.1

data/lib/chronicle/etl/models/base.rb

@@ -1,122 +0,0 @@
-require 'digest'
-
-module Chronicle
-  module ETL
-    module Models
-      # Represents a record that's been transformed by a Transformer and
-      # ready to be loaded. Loosely based on ActiveModel.
-      #
-      # @todo Experiment with just mixing in ActiveModel instead of this
-      #   this reimplementation
-      class Base
-        ATTRIBUTES = [:provider, :provider_id, :provider_namespace, :lat, :lng, :metadata].freeze
-        ASSOCIATIONS = [].freeze
-
-        attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
-
-        def initialize(attributes = {})
-          assign_attributes(attributes) if attributes
-          @dedupe_on = []
-          @metadata = {}
-        end
-
-        # A unique identifier for this model is formed from a type
-        # and either an id or lids.
-        def identifier_hash
-          {
-            type: self.class::TYPE,
-            id: @id,
-            lids: lids
-          }.compact
-        end
-
-        # Array of local ids that uniquely identify this record
-        def lids
-          @dedupe_on.map do |fields|
-            generate_lid(fields)
-          end.compact.uniq
-        end
-
-        # For a given set of fields of this model, generate a
-        # unique local id by hashing the field values
-        def generate_lid fields
-          raise ArgumentError.new("Must provide an array of symbolized fields") unless fields.is_a?(Array)
-
-          values = fields.sort.map do |field|
-            instance_variable = "@#{field.to_s}"
-            self.instance_variable_get(instance_variable)
-          end
-
-          return if values.any? { |e| e.nil? }
-
-          Digest::SHA256.hexdigest(values.join(","))
-        end
-
-        # Set of attribute names that this model has is Base's shared
-        # attributes combined with the child class's
-        def attribute_list
-          (ATTRIBUTES + self.class::ATTRIBUTES).uniq
-        end
-
-        # All of this record's attributes
-        def attributes
-          attributes = {}
-          attribute_list.each do |attribute|
-            instance_variable = "@#{attribute.to_s}"
-            attributes[attribute] = self.instance_variable_get(instance_variable)
-          end
-          attributes.compact
-        end
-
-        # All of this record's associations
-        def associations
-          association_list = ASSOCIATIONS + self.class::ASSOCIATIONS
-          attributes = {}
-          association_list.each do |attribute|
-            instance_variable = "@#{attribute.to_s}"
-            association = self.instance_variable_get(instance_variable)
-            attributes[attribute] = association if association
-          end
-          attributes.compact
-        end
-
-        def associations_hash
-          associations.map do |k, v|
-            if v.is_a?(Array)
-              [k, v.map(&:to_h)]
-            else
-              [k, v.to_h]
-            end
-          end.to_h
-        end
-
-        def meta_hash
-          {
-            meta: {
-              dedupe_on: @dedupe_on.map{|d| d.map(&:to_s).join(",")}
-            }
-          }
-        end
-
-        # FIXME: move this to a Utils module
-        def to_h_flattened
-          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
-        end
-
-        def to_h
-          identifier_hash
-            .merge(attributes)
-            .merge(associations_hash)
-            .merge(meta_hash)
-        end
-
-        def assign_attributes attributes
-          attributes.each do |k, v|
-            setter = :"#{k}="
-            public_send(setter, v) if respond_to? setter
-          end
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/entity.rb

@@ -1,29 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      class Entity < Chronicle::ETL::Models::Base
-        TYPE = 'entities'.freeze
-        ATTRIBUTES = [:title, :body, :provider_url, :represents, :slug, :myself, :metadata].freeze
-
-        # TODO: This desperately needs a validation system
-        ASSOCIATIONS = [
-          :involvements, # inverse of activity's `involved`
-          :analogous,
-          :attachments,
-          :abouts,
-          :aboutables, # inverse of above
-          :depicts,
-          :consumers,
-          :creators,
-          :creations,
-          :contains,
-          :containers # inverse of above
-        ].freeze  # TODO: add these to reflect Chronicle Schema
-
-        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
-      end
-    end
-  end
-end

data/lib/chronicle/etl/models/raw.rb

@@ -1,26 +0,0 @@
-require 'chronicle/etl/models/base'
-
-module Chronicle
-  module ETL
-    module Models
-      # A record from an extraction with no processing or normalization applied
-      class Raw
-        TYPE = 'raw'
-
-        attr_accessor :raw_data
-
-        def initialize(raw_data)
-          @raw_data = raw_data
-        end
-
-        def to_h
-          @raw_data.to_h
-        end
-
-        def to_h_flattened
-          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/registry/plugin_registry.rb

@@ -1,95 +0,0 @@
-require 'rubygems'
-require 'rubygems/command'
-require 'rubygems/commands/install_command'
-require 'rubygems/uninstaller'
-
-module Chronicle
-  module ETL
-    module Registry
-      # Responsible for managing plugins available to chronicle-etl
-      #
-      # @todo Better validation for whether a gem is actually a plugin
-      # @todo Add ways to load a plugin that don't require a gem on rubygems.org
-      module PluginRegistry
-        class << self
-          # Start of a system for having non-gem plugins. Right now, we just
-          # make registry aware of existenc of name of non-gem plugin
-          def register_standalone(name)
-            standalones << name
-          end
-
-          def standalones
-            @standalones ||= []
-          end
-        end
-
-        # Does this plugin exist?
-        def self.exists?(name)
-          # TODO: implement this. Could query rubygems.org or use a hardcoded
-          # list somewhere
-          true
-        end
-
-        # All versions of all plugins currently installed
-        def self.all_installed
-          # TODO: add check for chronicle-etl dependency
-          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" }
-        end
-
-        # Latest version of each installed plugin
-        def self.all_installed_latest
-          all_installed.group_by(&:name)
-            .transform_values { |versions| versions.sort_by(&:version).reverse.first }
-            .values
-        end
-
-        # Check whether a given plugin is installed
-        def self.installed?(name)
-          (standalones + all_installed.map { |gem| gem.name.gsub("chronicle-", "") }).include?(name)
-        end
-
-        # Activate a plugin with given name by `require`ing it
-        def self.activate(name)
-          # By default, activates the latest available version of a gem
-          # so don't have to run Kernel#gem separately
-          require "chronicle/#{name}"
-        rescue Gem::ConflictError => e
-          # TODO: figure out if there's more we can do here
-          raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
-        rescue StandardError, LoadError => e
-          # StandardError to catch random non-loading problems that might occur
-          # when requiring the plugin (eg class macro invoked the wrong way)
-          # TODO: decide if this should be separated
-          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded"
-        end
-
-        # Install a plugin to local gems
-        def self.install(name)
-          return if installed?(name)
-
-          gem_name = "chronicle-#{name}"
-          raise(Chronicle::ETL::PluginNotAvailableError.new(gem_name), "Plugin #{name} doesn't exist") unless exists?(gem_name)
-
-          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
-          Gem.install(gem_name)
-
-          activate(name)
-        rescue Gem::UnsatisfiableDependencyError
-          # TODO: we need to catch a lot more than this here
-          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} could not be installed."
-        end
-
-        # Uninstall a plugin
-        def self.uninstall(name)
-          gem_name = "chronicle-#{name}"
-          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
-          uninstaller = Gem::Uninstaller.new(gem_name)
-          uninstaller.uninstall
-        rescue Gem::InstallError
-          # TODO: strengthen this exception handling
-          raise(Chronicle::ETL::PluginError.new(name), "Plugin #{name} wasn't uninstalled")
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/serializers/jsonapi_serializer.rb

@@ -1,31 +0,0 @@
-module Chronicle
-  module ETL
-    class JSONAPISerializer < Chronicle::ETL::Serializer
-      def initialize(*args)
-        super
-
-        raise(SerializationError, "Record must be a subclass of Chronicle::ETL::Model::Base") unless @record.is_a?(Chronicle::ETL::Models::Base)
-      end
-
-      def serializable_hash
-        @record
-          .identifier_hash
-          .merge({ attributes: @record.attributes })
-          .merge({ relationships: build_associations })
-          .merge(@record.meta_hash)
-      end
-
-      def build_associations
-        @record.associations.transform_values do |value|
-          association_data =
-            if value.is_a?(Array)
-              value.map { |record| JSONAPISerializer.new(record).serializable_hash }
-            else
-              JSONAPISerializer.new(value).serializable_hash
-            end
-          { data: association_data }
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/serializers/raw_serializer.rb

@@ -1,10 +0,0 @@
-module Chronicle
-  module ETL
-    # Take a Raw model and output `raw_data` as a hash
-    class RawSerializer < Chronicle::ETL::Serializer
-      def serializable_hash
-        @record.to_h
-      end
-    end
-  end
-end

data/lib/chronicle/etl/serializers/serializer.rb

@@ -1,28 +0,0 @@
-module Chronicle
-  module ETL
-    # Abstract class representing a Serializer for an ETL record
-    class Serializer
-      # Construct a new instance of this serializer.
-      # == Parameters:
-      # options::
-      #   Options for configuring this Serializers
-      def initialize(record, options = {})
-        @record = record
-        @options = options
-      end
-
-      # Serialize a record as a hash
-      def serializable_hash
-        raise NotImplementedError
-      end
-
-      def self.serialize(record)
-        serializer = self.new(record)
-        serializer.serializable_hash
-      end
-    end
-  end
-end
-
-require_relative 'jsonapi_serializer'
-require_relative 'raw_serializer'

data/lib/chronicle/etl/transformers/image_file_transformer.rb

@@ -1,247 +0,0 @@
-require 'mini_exiftool'
-require 'active_support'
-require 'active_support/core_ext/object'
-require 'active_support/core_ext/time'
-require 'active_support/core_ext/hash/reverse_merge'
-require 'active_support/core_ext/string/inflections'
-
-module Chronicle
-  module ETL
-    # Transform a JPEG or other image file into a record.
-    # By default, file mtime and a hash of the file content is used to build
-    # the timestamp and ID respectively but other options are available (such
-    # as reading EXIF tags or extended attributes from the filesystem).
-    #
-    # TODO: This should be extracted into its own plugin
-    class ImageFileTransformer < Chronicle::ETL::Transformer
-      register_connector do |r|
-        r.identifier = 'image-file'
-        r.description = 'an image file'
-      end
-
-      setting :timestamp_strategy, default: 'file_mtime'
-      setting :id_strategy, default: 'file_hash'
-      setting :verb, default: 'photographed'
-      # EXIF tags often don't have timezones
-      setting :timezone_default, default: 'Eastern Time (US & Canada)'
-      setting :include_image_data, default: true
-      setting :actor
-      setting :involved
-
-      def transform
-        # FIXME: set @filename; use block for reading file when necessary
-        @file = File.open(@extraction.data)
-        record = build_created(@file)
-        @file.close
-        record
-      end
-
-      def friendly_identifier
-        @file.path
-      end
-
-      def id
-        @id ||= begin
-          id = build_with_strategy(field: :id, strategy: @config.id_strategy)
-          raise(UntransformableRecordError, "Could not build id") unless id
-
-          id
-        end
-      end
-
-      def timestamp
-        @timestamp ||= begin
-          ts = build_with_strategy(field: :timestamp, strategy: @config.timestamp_strategy)
-          raise(UntransformableRecordError, "Could not build timestamp") unless ts
-
-          ts
-        end
-      end
-
-      private
-
-      def build_created(file)
-        record = ::Chronicle::ETL::Models::Activity.new
-        record.verb = @config.verb
-        record.provider = @config.provider
-        record.provider_id = id
-        record.end_at = timestamp
-        record.dedupe_on = [[:provider_id, :verb, :provider]]
-
-        record.involved = build_image
-        record.actor = build_actor
-
-        record.assign_attributes(build_gps)
-        record
-      end
-
-      def build_actor
-        actor = ::Chronicle::ETL::Models::Entity.new
-        actor.represents = 'identity'
-        actor.provider = @config.actor[:provider]
-        actor.slug = @config.actor[:slug]
-        actor.dedupe_on = [[:provider, :slug, :represents]]
-        actor
-      end
-
-      def build_image
-        image = ::Chronicle::ETL::Models::Entity.new
-        image.represents = @config.involved[:represents]
-        image.title = build_title
-        image.body = exif['Description']
-        image.provider = @config.involved[:provider]
-        image.provider_id = id
-        image.assign_attributes(build_gps)
-        image.dedupe_on = [[:provider, :provider_id, :represents]]
-
-        if @config.ocr_strategy
-          ocr_text = build_with_strategy(field: :ocr, strategy: @config.ocr_strategy)
-          image.metadata[:ocr_text] = ocr_text if ocr_text
-        end
-
-        names = extract_people_depicted
-        tags = extract_keywords(names)
-
-        image.depicts = build_people_depicted(names)
-        image.abouts = build_keywords(tags)
-
-        if @config.include_image_data
-          attachment = ::Chronicle::ETL::Models::Attachment.new
-          attachment.data = build_image_data
-          image.attachments = [attachment]
-        end
-
-        image
-      end
-
-      def build_keywords(topics)
-        topics.map do |topic|
-          t = ::Chronicle::ETL::Models::Entity.new
-          t.represents = 'topic'
-          t.provider = @config.involved[:provider]
-          t.title = topic
-          t.slug = topic.parameterize
-          t.dedupe_on = [[:provider, :represents, :slug]]
-          t
-        end
-      end
-
-      def build_people_depicted(names)
-        names.map do |name|
-          identity = ::Chronicle::ETL::Models::Entity.new
-          identity.represents = 'identity'
-          identity.provider = @config.involved[:provider]
-          identity.slug = name.parameterize
-          identity.title = name
-          identity.dedupe_on = [[:provider, :represents, :slug]]
-          identity
-        end
-      end
-
-      def build_gps
-        return {} unless exif['GPSLatitude']
-
-        {
-          lat: exif['GPSLatitude'],
-          lng: exif['GPSLongitude'],
-          elevation: exif['GPSAltitude']
-        }
-      end
-
-      def build_image_data
-        ::Chronicle::ETL::Utils::BinaryAttachments.filename_to_base64(filename: @file.path)
-      end
-
-      def build_title
-        File.basename(@file)
-      end
-
-      def build_with_strategy(field:, strategy:[])
-        strategies = [strategy].flatten.compact
-        strategies.each do |s|
-          builder_method = "build_#{field}_using_#{s}"
-          result = send(builder_method.to_sym)
-          return result if result
-        end
-        return
-      end
-
-      def build_id_using_file_hash
-        Digest::SHA256.hexdigest(File.read(@file))
-      end
-
-      def build_id_using_xattr_version
-        load_value_from_xattr_plist("com.apple.metadata:kMDItemVersion")
-      end
-
-      def build_id_using_xmp_document_id
-        exif['OriginalDocumentID'] || exif['DerivedFromDocumentID']
-      end
-
-      def build_timestamp_using_file_mtime
-        File.mtime(@file)
-      end
-
-      def build_timestamp_using_exif_datetimeoriginal
-        # EXIF tags don't have timezone information. This is a DateTime in UTC
-        timestamp = exif['DateTimeOriginal'] || return
-
-        if exif['OffsetTimeOriginal']
-          # Offset tags are only available in newer EXIF tags. If it exists, we
-          # use it instead of UTC
-          timestamp = timestamp.change(offset: exif['OffsetTimeOriginal'])
-        elsif false
-          # TODO: support option of using GPS coordinates to determine timezone
-        else
-          zone = ActiveSupport::TimeZone.new(@config.timezone_default)
-          timestamp = zone.parse(timestamp.asctime)
-        end
-
-        timestamp
-      end
-
-      # TODO: add documentation for how to set up `macocr`
-      def build_ocr_using_macocr
-        `macocr "#{@file.path}" 2>/dev/null`.presence
-      end
-
-      def exif
-        @exif ||= MiniExiftool.new(
-          @file.path,
-          numerical: true,
-
-          # EXIF timestamps don't have timezone information. MiniExifTool uses Time
-          # by default which parses timestamps in local time zone. Using DateTime
-          # parses dates as UTC and then we can apply a timezone offset if the optional
-          # EXIF timezone offset fields are available.
-          # https://github.com/janfri/mini_exiftool/issues/39#issuecomment-832587649
-          timestamps: DateTime
-        )
-      end
-
-      # Figure out which faces are tagged as regions and return a list of their names
-      def extract_people_depicted
-        return [] unless exif['RegionName']
-
-        names = [exif['RegionName']].flatten
-        types = [exif['RegionType']].flatten
-
-        names.zip(types).select{|x| x[1] == 'Face'}.map{|x| x[0]}.uniq
-      end
-
-      # Extract image keywords from EXIF/IPTC tag and subtract out those of which are
-      # tagged people (determiend by looking at face regions)
-      def extract_keywords(people_names = [])
-        [exif['Keywords'] || []].flatten - people_names
-      end
-
-      def load_value_from_xattr_plist attribute
-        require 'nokogiri'
-        xml = `xattr -p #{attribute} \"#{@file.path}\" | xxd -r -p | plutil -convert xml1 -o - -- - 2>/dev/null`
-        return unless xml
-        value = Nokogiri::XML.parse(r).xpath("//string").text
-        return value.presence
-      end
-    end
-  end
-end

data/lib/chronicle/etl/utils/hash_utilities.rb

@@ -1,19 +0,0 @@
-module Chronicle
-  module ETL
-    module Utils
-      module HashUtilities
-        def self.flatten_hash(hash)
-          hash.each_with_object({}) do |(k, v), h|
-            if v.is_a? Hash
-              flatten_hash(v).map do |h_k, h_v|
-                h["#{k}.#{h_k}".to_sym] = h_v
-              end
-            else 
-              h[k] = v
-            end
-           end
-        end
-      end
-    end
-  end
-end

data/lib/chronicle/etl/utils/text_recognition.rb

@@ -1,15 +0,0 @@
-require 'active_support/core_ext/object/blank'
-
-module Chronicle
-  module ETL
-    module Utils
-      # OCR for image files
-      # TODO: add other strategies and document `macocr`
-      module TextRecognition
-        def self.recognize_in_image(filename:)
-          `macocr "#{filename}" 2>/dev/null`.presence
-        end
-      end
-    end
-  end
-end