krikri 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6e3513b7b316e31b65b2779273dec1985fadf37
-  data.tar.gz: d094a7b90fcdcf4e96ce47ab36a3f0db8474a293
+  metadata.gz: 8ba2b247b82e5f0d8a3f52adb0e3242d41deff3b
+  data.tar.gz: 429288310bd3531d4fbb20881d7a17983c69a015
 SHA512:
-  metadata.gz: 215e8a8e7b6e2118b444c8699b03c37ba67aed42a8b04cd94569dfb11be91d2cff230deb668885a120f3203f38dd865c77878e27d60782ea7bddc36d8f57a6d1
-  data.tar.gz: cb1e47c7e94ff570492d12fb4ba21c4d383ffff45bbf167d16aa1f252559fd166c3cc0ab457737bf314685bd2dcdc10cd18bec562cf6f9c0f5029c860abaa2ea
+  metadata.gz: 783c38573ddf5cac189f56220981aeba2768c1eba0a5e4d873d36642f4e633ca9e3ef449ba14ceae49d58b0bc05e426c9b5b84708f0ed7e9788ec305f3416464
+  data.tar.gz: 817df4eee63756335abc562d8badc40160d32d83a92aa5fde93e32ba17ca2938b2b311eed93cf8469756939f6551afd3b0bca25ca2b269f504d6b21b258457b8

data/app/controllers/krikri/records_controller.rb

@@ -80,5 +80,19 @@ module Krikri
 
       config.solr_document_model = Krikri::SearchIndexDocument
     end
+
+    ##
+    # Construct a valid item URI from a local name, and use it to fetch a single
+    # document from the search index.
+    # Override method in Blacklight::SolrHelper.
+    # TODO: This method is depreciated in Blacklight v5.10.
+    # TODO: Write appropriate test for this functionality after it is updated
+    # with Blacklight v5.10.
+    # @param String id is a local name.
+    def get_solr_response_for_doc_id(id=nil, extra_controller_params={})
+      id_uri = Krikri::Settings.marmotta.item_container << '/' << id
+      solr_response = solr_repository.find(id_uri, extra_controller_params)
+      [solr_response, solr_response.documents.first]
+    end
   end
 end

data/app/models/krikri/search_index_document.rb

@@ -4,6 +4,16 @@ module Krikri
   # Represents a single document returned from a query to the search index.
   class SearchIndexDocument < SolrDocument
 
+    ##
+    # Use local name instead of full item id URI in route.  For example, a
+    # document with the id 'http://dp.la/marmotta/ldp/items/123ab' will have an
+    # id param of '123ab'.  This is necessary because routes that contain '.' 
+    # are not valid.
+    # @return String
+    def to_param
+      self[self.class.unique_key].match(/[\/]([^\/]*)\z/)[1]
+    end
+
     ##
     # Get the aggregation, populated with data from Marmotta, which corresponds
     # to this SearchIndexDocument

data/lib/krikri/engine.rb

@@ -5,6 +5,7 @@ require 'krikri/ldp'
 require 'dpla/map'
 require 'rdf/marmotta'
 require 'oai/client'
+require 'rest-client'
 require 'edtf'
 
 require 'resque'

data/lib/krikri/enricher.rb

@@ -0,0 +1,156 @@
+module Krikri
+  ##
+  # A SoftwareAgent that runs enrichment processes.
+  #
+  # @example
+  #
+  #   To enrich records that were mapped by the mapping activity with ID 3:
+  #
+  #   # Define which enrichments are run, and thier parameters:
+  #   chain = {
+  #     'Krikri::Enrichments::StripHtml' => {
+  #       input_fields: [{sourceResource: :title}]
+  #     },
+  #     'Krikri::Enrichments::StripWhitespace' => {
+  #       input_fields: [{sourceResource: :title}]
+  #     }
+  #   }
+  #   Krikri::Enricher.enqueue({
+  #     generator_uri: 'http://ldp.local.dp.la/ldp/activity/3',
+  #     chain: chain
+  #   })
+  #
+  # @see Krikri::SoftwareAgent#enqueue
+  # @see Krikri::Enrichment
+  #
+  class Enricher
+    include SoftwareAgent
+
+    attr_reader :chain, :generator_uri
+
+    def self.queue_name
+      :enrichment
+    end
+
+    ##
+    # Create a new Enricher, given a hash of options:
+    #   generator_uri:  the LDP URI of the Activity that generated the mapped
+    #      records that this one will enrich.
+    #   chain:  a hash specifying the input_fields and output_fields, as
+    #      illustrated above, which will be passed to the Enrichment.
+    #
+    # @see Krikri::Enrichment
+    # @param opts [Hash] a hash of options
+    def initialize(opts = {})
+      @generator_uri = RDF::URI(opts.fetch(:generator_uri))
+      @chain = deep_sym(opts.fetch(:chain) { {} })
+    end
+
+    ##
+    # Run the enrichmnt.
+    #
+    # Take each record that was affected by the activity defined by our
+    # instantiation, and apply each enrichment from the enrichment chain.
+    #
+    def run(activity_uri = nil)
+      log :info, 'enricher is running'
+      # see TODO below
+      target_aggregations.each do |agg|
+        begin
+          chain_enrichments!(agg)
+          activity_uri ? agg.save_with_provenance(activity_uri) : agg.save
+        rescue => e
+          log :error, "Enrichment error: #{e.message}\n#{e.backtrace}"
+        end
+      end
+      log :info, 'enricher is done'
+    end
+
+    # TODO:  remove this when the current topic branch that introduces the
+    # EntityConsumer mixin has been merged.
+    def target_aggregations
+      query = Krikri::ProvenanceQueryClient.find_by_activity(generator_uri)
+      query.execute.lazy.flat_map do |solution|
+        agg = DPLA::MAP::Aggregation.new(solution.record.to_s)
+        agg.get
+        agg
+      end
+    end
+
+    ##
+    # Given an aggregation, take each enrichment specified by the `chain'
+    # given in our instantiation, and apply that enrichment, with the given
+    # options, modifying the aggregation in-place.
+    #
+    def chain_enrichments!(agg)
+      chain.keys.each do |e|
+        enrichment = e.to_s.constantize.new
+        if enrichment.is_a? Krikri::FieldEnrichment
+          agg = do_field_enrichment(agg, enrichment, chain[e])
+        else
+          agg = do_basic_enrichment(agg, enrichment, chain[e])
+        end
+      end
+    end
+
+    private
+
+    ##
+    # Perform a default enrichment, using Enrichment#enrichment or a derived
+    # class that expects the same arguments.
+    #
+    # @param agg [DPLA::MAP::Aggregation]
+    # @param enrichment [Krikri::Enrichment]
+    # @param options [Hash]
+    #
+    # @see Krikri::Enrichment
+    #
+    def do_basic_enrichment(agg, enrichment, options)
+      enrichment.enrich(
+        agg, options[:input_fields], options[:output_fields]
+      )
+    end
+
+    ##
+    # Perform a FieldEnrichment enrichment on the given aggregation.
+    #
+    # With FieldEnrichment#enrich, the input_fields option parameter is passed
+    # as a variable arguments list
+    #
+    # @param agg [DPLA::MAP::Aggregation]
+    # @param enrichment [Krikri::FieldEnrichment]
+    # @param options [Hash]  Hash with :input_fields containing variable
+    #                        arguments list
+    #
+    # @see Krikri::FieldEnrichment
+    #
+    def do_field_enrichment(agg, enrichment, options)
+      enrichment.enrich(agg, *options[:input_fields])
+    end
+
+    ##
+    # Transform the given hash recursively by turning all of its string keys
+    # and values into symbols.
+    #
+    # Symbols are expected in the enrichment classes, and we will usually be
+    # dealing with values that have been deserialized from JSON.
+    #
+    def deep_sym(obj)
+      if obj.is_a? Hash
+        return obj.inject({}) do |memo, (k, v)|
+          memo[k.to_sym] = deep_sym(v)
+          memo
+        end
+      elsif obj.is_a? Array
+        return obj.inject([]) do |memo, el|
+          memo << deep_sym(el)
+          memo
+        end
+      elsif obj.respond_to? :to_sym
+        return obj.to_sym
+      else
+        return nil
+      end
+    end
+  end
+end

data/lib/krikri/enrichment.rb

@@ -3,8 +3,6 @@ module Krikri
   # Mixin module for enriching a set of input_fields and setting the resulting
   # values to a set of output fields.
   module Enrichment
-    extend SoftwareAgent
-
     ##
     # The main enrichment method; passes specified input fields to
     # #enrich_values, which must return an array of values with length equal to

data/lib/krikri/enrichments/{deduplication.rb~ → dedup_values.rb~}

@@ -1,10 +1,7 @@
 module Krikri::Enrichments
   ##
-  # Enrichment to remove duplicate values
   #
-  #
-  class Deduplication
+  class DedupValues
     include Krikri::FieldEnrichment
-
   end
 end

data/lib/krikri/enrichments/genre_filter.rb

@@ -0,0 +1,45 @@
+module Krikri::Enrichments
+  ##
+  # Enrichment to remove non-genre fields from
+  #
+  #   StripHtml.new.enrich_value('Book') => 'Book'
+  #   StripHtml.new.enrich_value('not a book') => nil
+  #
+  # Allowed genre terms are:
+  #
+  #  - Book
+  #  - Film/Video
+  #  - Manuscript
+  #  - Maps
+  #  - Music
+  #  - Musical Score
+  #  - Newspapers
+  #  - Nonmusic
+  #  - Photograph/Pictorial Works
+  #  - Serial
+  #
+  # Removes all non-string values
+  class GenreFilter
+    include Krikri::FieldEnrichment
+
+    TERMS = ['Book',
+             'Film/Video',
+             'Manuscript',
+             'Maps',
+             'Music',
+             'Musical Score',
+             'Newspapers',
+             'Nonmusic',
+             'Photograph/Pictorial Works',
+             'Serial']
+
+    def enrich_value(value)
+      return nil unless value.is_a? String
+      term = TERMS.select do |t|
+        t.downcase.gsub(/[^a-zA-Z]/, '') ==
+          value.downcase.gsub(/[^a-zA-Z]/, '')
+      end
+      term.empty? ? nil : term.first
+    end
+  end
+end

data/lib/krikri/enrichments/strip_ending_punctuation.rb

@@ -0,0 +1,21 @@
+module Krikri::Enrichments
+  ##
+  # Strip ending punctuation
+  #
+  #   StripEndingPunctuation.new
+  #     .enrich_value("moomin!...!;,.",)
+  #   # => "moomin"
+  #
+  # Leaves quotation marks and closing parentheses & brackets. Also
+  # leaves periods when they follow a one or two letter abbreviation.
+  class StripEndingPunctuation
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.is_a? String
+      value.gsub!(/[^\p{Alnum}\'\"\)\]\}]*$/, '') unless
+        value.match /\s*[a-z]{1,2}\.$/i
+      value
+    end
+  end
+end

data/lib/krikri/enrichments/strip_ending_punctuation.rb~

@@ -0,0 +1,18 @@
+module Krikri::Enrichments
+  ##
+  # Strip Ending punctuation
+  #
+  #   StripPunctuation.new
+  #     .enrich_value("([!.;:\tmoominpapa;:;:; moominmama! ...\n")
+  #   # => "\tmoominpapa;:;:; moominmama! ...\n"
+  #
+  # Leaves quotation marks.
+  class StripLeadingPunctuation
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.is_a? String
+      value.gsub(/[^\p{Alnum}\'\"\s]*/, '')
+    end
+  end
+end

data/lib/krikri/enrichments/strip_leading_colons.rb

@@ -0,0 +1,15 @@
+module Krikri::Enrichments
+  ##
+  # Strip leading colons
+  #
+  #   StripLeadingColons.new.enrich_value(";:\tmoominpa()pa;;;")
+  #   # => "\tmoominpa()pa;;;"
+  class StripLeadingColons
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.is_a? String
+      value.gsub(/^[\;\:]*/, '')
+    end
+  end
+end

data/lib/krikri/enrichments/strip_leading_colons.rb~

@@ -0,0 +1,15 @@
+module Krikri::Enrichments
+  ##
+  # Strip leading colons
+  #
+  #   StripPunctuation.new.enrich_value(";:\tmoominpa()pa;;;")
+  #   # => "\tmoominpa()pa;;;"
+  class StripLeadingColons
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.is_a? String
+      value.gsub(/^[\;\:]*/, '')
+    end
+  end
+end

data/lib/krikri/enrichments/strip_leading_punctuation.rb

@@ -0,0 +1,18 @@
+module Krikri::Enrichments
+  ##
+  # Strip leading punctuation
+  #
+  #   StripLeadingPunctuation.new
+  #     .enrich_value("([!.;:\tmoominpapa;:;:; moominmama! ...\n")
+  #   # => "\tmoominpapa;:;:; moominmama! ...\n"
+  #
+  # Leaves quotation marks.
+  class StripLeadingPunctuation
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.is_a? String
+      value.gsub(/^[^\p{Alnum}\'\"\s]*/, '')
+    end
+  end
+end

data/lib/krikri/enrichments/strip_leading_punctuation.rb~

@@ -0,0 +1,18 @@
+module Krikri::Enrichments
+  ##
+  # Strip leading punctuation
+  #
+  #   StripPunctuation.new
+  #     .enrich_value("([!.;:\tmoominpapa;:;:; moominmama! ...\n")
+  #   # => "\tmoominpapa;:;:; moominmama! ...\n"
+  #
+  # Leaves quotation marks.
+  class StripLeadingPunctuation
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.is_a? String
+      value.gsub(/^[^\p{Alnum}\'\"\s]*/, '')
+    end
+  end
+end

data/lib/krikri/enrichments/timespan_split.rb

@@ -28,18 +28,40 @@ module Krikri::Enrichments
   class TimespanSplit
     include Krikri::FieldEnrichment
 
+    ##
+    # Enrich a `DPLA::MAP::TimeSpan` object or string value with `begin` and
+    # `end` values.
+    #
+    # @param value [DPLA::MAP::TimeSpan, String, Object]
+    #
+    # @return [Object] a new `TimeSpan` object containing the providedLabel
+    #   and the enriched begin/end; if given a value other than a `TimeSpan`
+    #   or `String` returns that value.
     def enrich_value(value)
       value = timespan_from_string(value) if value.is_a? String
       return value unless value.is_a? DPLA::MAP::TimeSpan
       populate_timespan(value)
     end
 
+    ##
+    # Converts a string to a `DPLA::MAP::TimeSpan` with the string as
+    # `providedLabel`.
+    #
+    # @param [String] a string value containing a date, time, or timespan
+    #
+    # @return [DPLA::MAP::TimeSpan] a new, empty timespan with `providedLabel`
     def timespan_from_string(value)
       timespan = DPLA::MAP::TimeSpan.new
       timespan.providedLabel = value
       timespan
     end
 
+    ##
+    # Populates a timespan with a begin and end date.
+    #
+    # @param timespan [DPLA::MAP::TimeSpan]
+    #
+    # @return [DPLA::MAP::TimeSpan]
     def populate_timespan(timespan)
       return timespan unless (timespan.begin.empty? || timespan.end.empty?) &&
         !timespan.providedLabel.empty?
@@ -55,17 +77,37 @@ module Krikri::Enrichments
       return timespan
     end
 
+    ##
+    # @return [Array<Date, EDTF::Interval>]
     def parse_labels(labels)
       labels.map { |l| Krikri::Util::ExtendedDateParser.parse(l, true) }.compact
     end
 
+    ##
+    # Converts an EDTF date to a begin and end date.
+    #
+    # @param date [Date, DateTime, EDTF::Interval] a date, with or without EDTF
+    #   precision features; or an interval.
+    #
+    # @return [Array<Date, DateTime>] an array of two elements containing the
+    #   begin and end dates.
     def span_from_date(date)
       return [nil, nil] if date.nil?
-      return [date, date] if date.is_a? Date
+      if date.is_a?(Date)
+        return [date, date] if date.precision == :day
+        return [date, (date.succ - 1)]
+      end
       [(date.respond_to?(:first) ? date.first : date.from),
        (date.respond_to?(:last) ? date.last : date.to)]
     end
 
+    ##
+    # Reduces a timespan with multiple begin or end dates to a single earliest
+    # begin date and a single latest end date.
+    #
+    # @param timespan [DPLA::MAP::TimeSpan] the timespan to reduce
+    #
+    # @return [DPLA::MAP::TimeSpan] an updated timespan
     def reduce_to_largest_span(timespan)
       timespan.begin = timespan.begin.sort.first
       timespan.end = timespan.end.sort.last