cdmdexer 0.17.7

data/lib/cdmdexer/field_mapping.rb

@@ -0,0 +1,28 @@
+module CDMDEXER
+  class FieldMapping
+    attr_reader :config
+    def initialize(config: {})
+      @config = symbolize(config)
+    end
+
+    def origin_path
+      config.fetch(:origin_path)
+    end
+
+    def dest_path
+      config.fetch(:dest_path)
+    end
+
+    def formatters
+      config.fetch(:formatters, [DefaultFormatter]).map do |formatter|
+        formatter.is_a?(String) ? Object.const_get(formatter) : formatter
+      end
+    end
+
+    private
+
+    def symbolize(config)
+      config.inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo }
+    end
+  end
+end

data/lib/cdmdexer/field_transformer.rb

@@ -0,0 +1,41 @@
+require 'hash_at_path'
+
+module CDMDEXER
+  class FieldTransformer
+    extend Forwardable
+    def_delegators :@field_mapping, :origin_path, :dest_path, :formatters
+    attr_reader :field_value, :field_mapping, :formatter_klass
+    def initialize(field_mapping: FieldMapping.new,
+                   record: {},
+                   formatter_klass: FieldFormatter)
+      @field_mapping   = field_mapping
+      @field_value     = compact(record.at_path(origin_path))
+      @formatter_klass = formatter_klass
+    end
+
+    def reduce
+      (blank?(value)) ? {} : { "#{dest_path}" => value }
+    end
+
+    def value
+      @value ||= (!blank?(field_value)) ? transform_field : nil
+    end
+
+    private
+
+    def compact(record)
+      (record.respond_to?(:compact)) ? record.compact : record
+    end
+
+    # File activesupport/lib/active_support/core_ext/object/blank.rb, line 14
+    def blank?(val)
+      val.respond_to?(:empty?) ? !!val.empty? : !val
+    end
+
+    def transform_field
+      formatter_klass.new(value: field_value, formatters: formatters).format!
+    rescue StandardError => e
+      raise "Mapping Error:#{field_mapping.config} Error:#{e.message}"
+    end
+  end
+end

data/lib/cdmdexer/filtered_set_specs.rb

@@ -0,0 +1,41 @@
+module CDMDEXER
+  module DefaultFilterSetCallback
+    def valid?(set: {})
+      true
+    end
+  end
+
+  class FilteredSetSpecs
+    attr_reader :oai_base_url,
+                :oai_client,
+                :callback
+
+    def initialize(oai_base_url: :missing_oai_base_url,
+                   oai_client: OaiClient,
+                   callback: CDMDEXER::DefaultSetFilterCallback.new)
+      @oai_base_url = oai_base_url
+      @oai_client   = oai_client
+      @callback     = callback
+    end
+
+    def set_specs
+      filtered_sets.map { |set| set['setSpec'] }
+    end
+
+    def filtered_sets
+      @filtered_sets ||= sets.select do |set|
+        callback.valid?(set: set)
+      end
+    end
+
+    private
+
+    def sets
+      @sets ||= list_sets['OAI_PMH']['ListSets']['set']
+    end
+
+    def list_sets
+      @list_sets ||= oai_client.new(base_url: oai_base_url).request('verb=ListSets')
+    end
+  end
+end

data/lib/cdmdexer/formatters.rb

@@ -0,0 +1,169 @@
+require 'titleize'
+require 'json'
+require 'net/http'
+# A handful of very simple formatters to clean up CONTENTdm API metadata
+module CDMDEXER
+
+
+  class GeoNameID
+    def self.format(value)
+      value.split('/').last
+    end
+  end
+
+  class GeoNameIDToJson
+    URL = "http://ws.geonames.net/getJSON?username=#{ENV['GEONAMES_USER']}&token=#{ENV['GEONAMES_TOKEN']}"
+    def self.format(value)
+      JSON.parse(Net::HTTP.get_response(URI.parse("#{URL}&geonameId=#{value}")).body)
+    end
+  end
+
+  class GeoNameToLocation
+    def self.format(value)
+      return if !value.respond_to?(:fetch)
+      return if !value['lat'] || !value['lng']
+      "#{value['lat']},#{value['lng']}"
+    end
+  end
+
+  class GeoNameToPlaceName
+    def self.format(value)
+      return if !value.respond_to?(:fetch)
+      [
+        value['name'],
+        value['adminName1'],
+        value['adminName2']
+      ].select { |place| place != 'Minnesota'}.compact.uniq
+    end
+  end
+
+  class DefaultFormatter
+    def self.format(value)
+      value
+    end
+  end
+
+  class KeywordFormatter
+    def self.format(value)
+      vals = []
+      vals << value['genera'].split(';') if value['genera'].respond_to?(:split)
+      vals << value['specif'].split(';') if value['specif'].respond_to?(:split)
+      vals << value['subjec'].split(';') if value['subjec'].respond_to?(:split)
+      vals.flatten.uniq.sort
+    end
+  end
+
+  class UniqueFormatter
+    def self.format(value)
+      if value.respond_to?(:uniq)
+        value.uniq
+      else
+        value.titleize
+      end
+    end
+  end
+
+  class Titlieze
+    def self.format(value)
+      if value.respond_to?(:map)
+        value.map {|value| value.titleize }
+      else
+        value.titleize
+      end
+    end
+  end
+
+  class ImageId
+    def self.format(value)
+      value
+    end
+  end
+
+  class ToJsonFormatter
+    def self.format(values)
+      values.to_json if values.respond_to?(:to_json)
+    end
+  end
+
+  class StripSemicolonFormatter
+    def self.format(values)
+      if values.respond_to?(:map)
+        values.map {|value| value.gsub(/;/, '') }
+      else
+        values.gsub(/;/, '')
+      end
+    end
+  end
+
+  class StripFormatter
+    def self.format(values)
+      return '' if values.nil?
+      if values.respond_to?(:map)
+        values.map {|value| value.strip }
+      else
+        values.strip
+      end
+    end
+  end
+
+  class SplitFormatter
+    def self.format(value)
+      (value.respond_to?(:split)) ? value.split(';') : value
+    end
+  end
+
+  class JoinFormatter
+    def self.format(value)
+      (value.respond_to?(:join)) ? value.join('; ') : value
+    end
+  end
+
+  class AddSetSpecFormatter
+    def self.format(value)
+      value.merge('setSpec' => value['id'].split('/').first)
+    end
+  end
+
+  class SetSpecFormatter
+    def self.format(value)
+      value['setSpec']
+    end
+  end
+
+  class CollectionNameFormatter
+    def self.format(value)
+      value['oai_sets'].fetch(value['setSpec'], {})
+                       .fetch(:name, '')
+    end
+  end
+
+  class CollectionDescriptionFormatter
+    def self.format(value)
+      value['oai_sets'].fetch(value['setSpec'], {})
+                       .fetch(:description, '')
+    end
+  end
+
+  class FilterBadCollections
+    def self.format(value)
+      (/Collection information undefined/i =~ value) ? '' : value
+    end
+  end
+
+  class ToIFormatter
+    def self.format(value)
+      value.to_i if value.respond_to?(:to_i)
+    end
+  end
+
+  class LocationFormatter
+    def self.format(record)
+      if record['latitu'] && record['longit'] && record['latitu'] != '' && record['longit'] != '' && record['latitu'] != {}
+        "#{record['latitu']}, #{record['longit']}"
+      else
+        nil
+      end
+    end
+  end
+
+end

data/lib/cdmdexer/hooks.rb

@@ -0,0 +1,31 @@
+module CDMDEXER
+  def self.const_missing(name)
+    if name.to_s == 'Solr'
+      hook(pattern: name.to_s, default: DefaultSolr)
+    elsif name.to_s == 'CompletedCallback'
+      hook(pattern: name.to_s, default: DefaultCompletedCallback)
+    elsif name.to_s == 'OaiNotification'
+      hook(pattern: name.to_s, default: DefaultOaiNotification)
+    elsif name.to_s == 'LoaderNotification'
+      hook(pattern: name.to_s, default: DefaultLoaderNotification)
+    elsif name.to_s == 'CdmNotification'
+      hook(pattern: name.to_s, default: DefaultCdmNotification)
+    end
+  end
+
+  def self.hook(pattern: '', default: false)
+    if find_hook(pattern, default)
+      Object.const_get("CDMDEXER::#{find_hook(pattern, default)}")
+    else
+      default
+    end
+  end
+
+  def self.find_hook(pattern, default)
+    CDMDEXER.constants.find do |konst|
+      if Object.const_get("CDMDEXER::#{konst}") != default
+        /#{pattern}/ =~ konst.to_s
+      end
+    end
+  end
+end

data/lib/cdmdexer/load_worker.rb

@@ -0,0 +1,36 @@
+require 'sidekiq'
+module CDMDEXER
+  # Load Records into a solr index
+  class LoadWorker
+    include Sidekiq::Worker
+    sidekiq_options queue: 'critical'
+    attr_reader :solr_config, :records, :deletables
+    attr_writer :loader_klass, :solr_klass
+    def perform(records = [], deletables = [], solr_config = {})
+      @solr_config = solr_config.symbolize_keys
+      @records     = records
+      @deletables  = deletables
+      load!
+    end
+
+    def loader_klass
+      @loader_klass ||= Loader
+    end
+
+    def solr_klass
+      @solr_klass ||= DefaultSolr
+    end
+
+    def load!
+      loader_klass.new(records: records,
+                       deletable_ids: deletables,
+                       solr_client: solr_client).load!
+    end
+
+    private
+
+    def solr_client
+      @solr_client ||= solr_klass.new(solr_config)
+    end
+  end
+end

data/lib/cdmdexer/loader.rb

@@ -0,0 +1,19 @@
+module CDMDEXER
+
+  class Loader
+    attr_reader :solr_client, :records, :deletable_ids
+
+    def initialize(records: [],
+                   deletable_ids: [],
+                   solr_client: CDMDEXER::DefaultSolr)
+      @solr_client   = solr_client
+      @records       = records
+      @deletable_ids = deletable_ids
+    end
+
+    def load!
+      solr_client.delete deletable_ids unless deletable_ids.empty?
+      solr_client.add records
+    end
+  end
+end

data/lib/cdmdexer/oai_client.rb

@@ -0,0 +1,26 @@
+
+require 'json'
+require 'http'
+module CDMDEXER
+    class OaiClient
+      attr_reader :base_url, :client
+      def initialize(base_url: '', client: HTTP)
+        @base_url    = base_url
+        @client = client
+      end
+
+      def request(query)
+        hashify get("#{base_url}?#{query}")
+      end
+
+      private
+
+      def get(url)
+        client.get(url).to_s
+      end
+
+      def hashify(xml)
+        Hash.from_xml(xml)
+      end
+    end
+end

data/lib/cdmdexer/oai_request.rb

@@ -0,0 +1,100 @@
+require 'hash_at_path'
+require 'json'
+
+module CDMDEXER
+  # Light wrapper around OAI requests
+  # Enhances OAI responses with handles sets, records, etc and adds a little
+  # extra value to their data with a keyed set lookup, filters for deleted
+  # and non-deleted records
+  class OaiRequest
+    attr_reader :endpoint_url,
+                :resumption_token,
+                :client,
+                :set_spec
+
+    def initialize(endpoint_url: '',
+                   resumption_token: nil,
+                   set_spec: nil,
+                   client: Net::HTTP)
+      @endpoint_url     = endpoint_url
+      @resumption_token = resumption_token
+      @client           = client
+      @set_spec         = set_spec ? "&set=#{set_spec}" : ''
+    end
+
+    def records
+      headers.map do |header|
+        header.merge(
+          id: header['identifier'].split(':').last.split('/').join(':')
+        )
+      end
+    end
+
+    def sets
+      # Ensure a result of one set is still an array
+      @sets ||= force_array request(sets_endpoint_url).at_path('OAI_PMH/ListSets/set')
+    end
+
+    def set_lookup
+      sets.inject({}) { |memo, set| memo.merge(to_key(set)) }
+    end
+
+    def next_resumption_token
+      identifier_request.at_path('OAI_PMH/ListIdentifiers/resumptionToken')
+    end
+
+    def deletable_ids
+      records.select { |record| record['status'] == 'deleted' }
+             .map { |record| record[:id] }
+    end
+
+    def updatables
+      records.reject { |record| record['status'] == 'deleted' }
+    end
+
+    private
+
+    # TODO: Add some error handling if this ever turns up empty
+    def headers
+      force_array identifier_request.at_path('OAI_PMH/ListIdentifiers/header')
+    end
+
+    # Ensure results are a single level array
+    # (single row sets, records, etc)
+    def force_array(result)
+      [result].flatten
+    end
+
+    def to_key(set)
+      {
+        set['setSpec'] =>
+          {
+            name: set['setName'],
+            description: set.at_path('setDescription/dc/description')
+          }
+      }
+    end
+
+    def identifier_request
+      @identifier_request ||=
+        resumption_token ? request(batch_endpoint_url) : request(first_batch_endpoint_url)
+    end
+
+    def first_batch_endpoint_url
+      "#{endpoint_url}?verb=ListIdentifiers&metadataPrefix=oai_dc#{set_spec}"
+    end
+
+    def batch_endpoint_url
+      "#{endpoint_url}?verb=ListIdentifiers&resumptionToken=#{resumption_token}"
+    end
+
+    def sets_endpoint_url
+      "#{endpoint_url}?verb=ListSets"
+    end
+
+    def request(location)
+      CDMDEXER::OaiNotification.call!(location)
+      Hash.from_xml(client.get_response(URI(location)).body)
+    end
+  end
+end

data/lib/cdmdexer/rake_task.rb

@@ -0,0 +1,6 @@
+require_relative '../cdmdexer'
+module CDMDEXER
+  module RakeTask
+    Dir[File.expand_path(File.join(File.dirname(__FILE__),"tasks/*.rake"))].each { |ext| load ext } if defined?(Rake)
+  end
+end

data/lib/cdmdexer/record_transformer.rb

@@ -0,0 +1,25 @@
+module CDMDEXER
+  class RecordTransformer
+    attr_reader :record, :field_mappings, :field_transformer
+    def initialize(record: {},
+                   field_mappings: [],
+                   field_transformer: FieldTransformer)
+      @record            = record
+      @field_mappings    = field_mappings
+      @field_transformer = field_transformer
+    end
+
+    def transform!
+      field_mappings.inject({}) do |dest_record, field_mapping|
+        dest_record.merge(transform_field(record, field_mapping))
+      end
+    end
+
+    private
+
+    def transform_field(record, field_mapping)
+      field_transformer.new(field_mapping: field_mapping,
+                            record: record).reduce
+    end
+  end
+end

data/lib/cdmdexer/regex_filter_callback.rb

@@ -0,0 +1,19 @@
+module CDMDEXER
+  # Search an OAI ListSets field using a regular expression
+  class RegexFilterCallback
+    attr_reader :field, :pattern, :inclusive
+    def initialize(field: 'setName', pattern: /.*/, inclusive: true)
+      @field     = field
+      @pattern   = pattern
+      @inclusive = inclusive
+    end
+
+    def valid?(set: {})
+      inclusive ? matches?(set) : !matches?(set)
+    end
+
+    def matches?(set)
+      pattern.match?(set[field])
+    end
+  end
+end

data/lib/cdmdexer/tasks/delete.rake

@@ -0,0 +1,12 @@
+require 'cdmdexer'
+
+namespace :cdmdexer do
+  desc "delete all records that aren't in a given OAI endpoint"
+  task :delete_batch, [:start, :prefix, :oai_url, :solr_url] do |t, args|
+        CDMDEXER::BatchDeleterWorker.perform_async(args[:start].to_i,
+                                         args[:prefix],
+                                         args[:oai_url],
+                                         args[:solr_url])
+  end
+end
+

data/lib/cdmdexer/tasks/etl.rake

@@ -0,0 +1,96 @@
+require 'cdmdexer'
+
+namespace :cdmdexer do
+
+
+
+  desc 'Ingest a Collection Syncronously'
+  task :collection_sync do
+    # config = etl.config
+    # raise etl.config.keys.inspect
+    CDMDEXER::ETLWorker.new.perform(
+      'solr_config' => {:url=>"http://solr:8983/solr/mdl-1"},
+      'oai_endpoint' => 'http://cdm16022.contentdm.oclc.org/oai/oai.php',
+      'cdm_endpoint' => 'https://server16022.contentdm.oclc.org/dmwebservices/index.php',
+      'set_spec' => 'mpls',
+      'batch_size' => 10,
+      'max_compounds' => 10
+    )
+  end
+
+
+  desc 'Launch a background job to index metadata from CONTENTdm to Solr.'
+  task :batch, [
+    :solr_url,
+    :oai_endpoint,
+    :cdm_endpoint,
+    :set_spec,
+    :batch_size,
+    :max_compounds
+  ] do |t, args|
+    CDMDEXER::ETLWorker.perform_async(
+      solr_config: { url: args.fetch(:solr_url) },
+      oai_endpoint: args.fetch(:oai_endpoint),
+      cdm_endpoint: args.fetch(:cdm_endpoint),
+      set_spec: args[:set_spec] != '""' ? args[:set_spec] : nil,
+      batch_size: args.fetch(:batch_size, 10),
+      max_compounds: args.fetch(:max_compounds, 10)
+    )
+  end
+
+  desc 'Launch an indexing worker for each collection with an optional regex
+        pattern to match setSpec. Patterns can be inclusive or exclusive.'
+  task :by_collections, [
+    :solr_url,
+    :oai_endpoint,
+    :cdm_endpoint,
+    :set_spec_pattern,
+    :inclusive,
+    :batch_size
+  ] do |t, args|
+    oai_endpoint = args.fetch(:oai_endpoint)
+    # Optional args
+    pattern = args.fetch(:set_spec_pattern, false)
+    inclusive = args.fetch(:inclusive, 'true') == 'true'
+    # Define your own callback if you want to use other set related fields
+    # Use the RegexFilterCallback as an example of how to build your own filter
+    set_specs =
+      if pattern
+        filter = CDMDEXER::RegexFilterCallback.new(field: 'setName',
+                                                pattern: Regexp.new(pattern),
+                                                inclusive: inclusive)
+        CDMDEXER::FilteredSetSpecs.new(oai_base_url: oai_endpoint,
+                                    callback: filter).set_specs
+      else
+        CDMDEXER::FilteredSetSpecs.new(oai_base_url: oai_endpoint).set_specs
+      end
+
+    puts "Indexing Sets: '#{set_specs.join(', ')}'"
+
+    etl_config = {
+      solr_config: { url: args.fetch(:solr_url) },
+      oai_endpoint: args.fetch(:oai_endpoint),
+      cdm_endpoint: args.fetch(:cdm_endpoint),
+      batch_size: args.fetch(:batch_size, 5),
+      max_compounds: args.fetch(:max_compounds, 10)
+    }
+
+    CDMDEXER::ETLBySetSpecs.new(set_specs: set_specs, etl_config: etl_config).run!
+  end
+
+  desc 'Launch a background job to index a single record.'
+  task :record, [
+    :collection,
+    :id,
+    :solr_url,
+    :cdm_endpoint,
+    :oai_endpoint
+  ] do |t, args|
+    CDMDEXER::TransformWorker.perform_async(
+      [[args.fetch(:collection), args.fetch(:id)]],
+      { url: args.fetch(:solr_url) },
+      args.fetch(:cdm_endpoint),
+      args.fetch(:oai_endpoint)
+    )
+  end
+end