krikri 0.4.0 → 0.5.0

data/lib/krikri/entity_behavior.rb

@@ -0,0 +1,40 @@
+
+module Krikri
+  ##
+  # Base class for behaviors related to entities that are generated or revised
+  # by activities.
+  #
+  # A SoftwareAgent implements #entity_behavior, which returns an appropriate
+  # subclass of EntityBehavior.  When an Activity is queried for its entities,
+  # it instantiates an instance of its particular SoftwareAgent, and then
+  # calls the #entities method of the agent's entity behavior.
+  #
+  # @see Krikri::Activity#entities
+  # @see lib/krikri/entity_behaviors
+  #
+  class EntityBehavior
+    attr_reader :activity
+    def initialize(activity)
+      @activity = activity
+    end
+
+    ##
+    # Return an Enumerator of objects that have been affected by our @activity.
+    #
+    # @return [Enumerator] objects
+    # @see lib/krikri/entity_behaviors
+    # @see Krikri::Activity#entities
+    #
+    def entities
+      raise NotImplementedError 
+    end
+
+    ##
+    # @see Krikri::Activity#entities
+    # @see Krikri::EntityBehavior#entities
+    #
+    def self.entities(activity)
+      new(activity).entities
+    end
+  end
+end

data/lib/krikri/entity_behaviors/aggregation_entity_behavior.rb

@@ -0,0 +1,29 @@
+
+module Krikri
+  ##
+  # A behavior that provides methods for working with aggregations
+  # (as defined by DPLA::MAP::Aggregation), in the context of entities that
+  # are generated by activities.
+  #
+  # @see Krikri::EntityBehavior
+  #
+  class AggregationEntityBehavior < Krikri::EntityBehavior
+
+    ##
+    # @param load [Boolean]  Whether to load the whole record from the LDP
+    #                        server.  Default: true.
+    #                        DPLA::MAP::Aggregation#get is slow, because it
+    #                        results in a network request, so this provides the
+    #                        possibility of avoiding it.
+    # @see Krikri::EntityBehavior::entities
+    # @return [Enumerator] DPLA::MAP::Aggregation objects
+    #
+    def entities(load = true)
+      @activity.entity_uris.lazy.map do |uri|
+        agg = DPLA::MAP::Aggregation.new(uri)
+        agg.get if load
+        agg
+      end
+    end
+  end
+end

data/lib/krikri/entity_behaviors/original_record_entity_behavior.rb

@@ -0,0 +1,26 @@
+
+module Krikri
+  ##
+  # A behavior that provides methods for working with original records
+  # (as defined by Krikri::OriginalRecord), in the context of entities that
+  # are generated by activities.
+  #
+  # @see Krikri::EntityBehavior
+  #
+  class OriginalRecordEntityBehavior < Krikri::EntityBehavior
+
+    ##
+    # @param load [Boolean]  Whether to load the whole record from the LDP
+    #                        server.  Default: true.  OriginalRecord.load is
+    #                        slow, because it results in a network request, so
+    #                        this provides the possibility of avoiding it.
+    # @see Krikri::EntityBehavior::entities
+    # @return [Enumerator] OriginalRecord objects
+    #
+    def entities(load = true)
+      @activity.entity_uris.lazy.map do |uri|
+        load ? OriginalRecord.load(uri) : OriginalRecord.new(uri)
+      end
+    end
+  end
+end

data/lib/krikri/entity_consumer.rb

@@ -0,0 +1,48 @@
+module Krikri
+  ##
+  # A mixin for classes, like certain software agents, that generates entities.
+  # For example, a mapper usually generates RDF aggregations, such that
+  # Mapper::Agent includes EntityConsumer, and a mapper agent is instantiated
+  # with
+  #   a = Krikri::Mapper::Agent.new({
+  #     generator_uri: 'http://some.org/activity/1'
+  #   })
+  # such that
+  #   a.generator_activity
+  # returns a Krikri::Activity
+  #
+  module EntityConsumer
+    extend ActiveSupport::Concern
+
+    included { attr_reader :generator_activity }
+
+    ##
+    # Store this agent's generator activity, which is the activity that
+    # produced the target entities upon which the current agent will operate.
+    #
+    # It is assumed that the agent class will define #entity_behavior, which
+    # returns the class of the appropriate behavior.
+    #
+    # `generator_uri' can be a string or RDF::URI.
+    #
+    # In the future, we might want to take a `generator_activity' parameter,
+    # because not every activity will modify its entities with provenance
+    # messages; an indexing activity, in particular.  In this case an LDP URI
+    # representing the activity is not relevant.
+    #
+    # @see Krikri::Mapper::Agent
+    # @see Krikri::Harvester
+    #
+    def assign_generator_activity!(opts)
+      if opts.include?(:generator_uri)
+        generator_uri = opts.delete(:generator_uri)
+        # allow generator_uri to be string or RDF::URI with `to_s' ...
+        activity_id = generator_uri.to_s[/\d+$/].to_i  # 0 if no match
+        fail "Can not determine ID for #{generator_uri}" if activity_id == 0
+        @generator_activity = Krikri::Activity.find_by_id(activity_id)
+        raise "Generator activity not found for id #{activity_id}" \
+          unless @generator_activity
+      end
+    end
+  end
+end

data/lib/krikri/harvester.rb

@@ -46,6 +46,14 @@ module Krikri
       end
     end
 
+    ##
+    # @see Krikri::Activity#entities
+    # @see Krikri::EntityBehavior
+    # @see Krikri::SoftwareAgent#entity_behavior
+    def entity_behavior
+      @entity_behavior ||= Krikri::OriginalRecordEntityBehavior
+    end
+
     ##
     # Accepts options for a generic harvester:
     #   uri: a URI for the harvest endpoint or provider

data/lib/krikri/indexer.rb

@@ -0,0 +1,46 @@
+module Krikri
+  ##
+  # A SoftwareAgent to run indexing processes.
+  #
+  # @example
+  #
+  #   To index records enriched by the enrichment activity with ID 3:
+  #
+  #   Krikri::Indexer.enqueue({
+  #     index_class: 'Krikri::QASearchIndex',
+  #     generator_uri: 'http://ldp.local.dp.la/ldp/activity/3'
+  #     some_option_for_index_class: 'abc'
+  #   })
+  #
+  #   The options hash contains options for the Indexer as well as the
+  #   SearchIndex.
+  #
+  # @see Krikri::SoftwareAgent#enqueue
+  #
+  class Indexer
+    include SoftwareAgent
+    include EntityConsumer
+
+    attr_reader :index_class, :index_opts
+
+    def self.queue_name
+      :indexing
+    end
+
+    def initialize(opts = {})
+      assign_generator_activity!(opts)
+      @index_class = opts.delete(:index_class)
+      @index_opts = opts
+    end
+
+    def run
+      log :info, 'indexer is running'
+      search_index = index_class.constantize.new(index_opts)
+      search_index.update_from_activity(generator_activity)
+    rescue => e
+      Rails.logger.error("Indexing error: #{e.message}\n#{e.backtrace}")
+    ensure
+      log :info, 'indexer is done'
+    end
+  end
+end

data/lib/krikri/mapper.rb

@@ -90,23 +90,45 @@ module Krikri
     ##
     # A SoftwareAgent to run mapping processes.
     #
+    # @example
+    #
+    #   To map the records harvested by the harvest activity with ID 1:
+    #
+    #   Krikri::Mapper::Agent.enqueue(
+    #     :mapping,
+    #     opts = {
+    #       name: 'scdl_qdc',
+    #       generator_uri: 'http://ldp.local.dp.la/ldp/activity/1'
+    #     }
+    #   )
+    #
     # @see: Krikri::SoftwareAgent, Krikri::Activity
     class Agent
       include SoftwareAgent
+      include EntityConsumer
 
-      attr_reader :name, :generator_uri
+      attr_reader :name
 
       def self.queue_name
         :mapping
       end
 
+      ##
+      # @see Krikri::Activity#entities
+      # @see Krikri::EntityBehavior
+      # @see Krikri::SoftwareAgent#entity_behavior
+      def entity_behavior
+        @entity_behavior ||= Krikri::AggregationEntityBehavior
+      end
+
       def initialize(opts = {})
         @name = opts.fetch(:name).to_sym
-        @generator_uri = RDF::URI(opts.fetch(:generator_uri))
+        assign_generator_activity!(opts)
       end
 
       def run(activity_uri = nil)
-        Krikri::Mapper.map(name, records).each do |rec|
+        harvest_records = generator_activity.entities
+        Krikri::Mapper.map(name, harvest_records).each do |rec|
           begin
             rec.mint_id! if rec.node?
             activity_uri ? rec.save_with_provenance(activity_uri) : rec.save
@@ -117,12 +139,6 @@ module Krikri
         end
       end
 
-      def records
-        Krikri::ProvenanceQueryClient.find_by_activity(generator_uri)
-          .execute.lazy.flat_map do |solution|
-          OriginalRecord.load(solution.record.to_s)
-        end
-      end
     end
   end
 end

data/lib/krikri/qa_query_client.rb

@@ -0,0 +1,154 @@
+module Krikri
+  ##
+  # Implements SPARQL queries for finding values and counts of values in
+  # RDF Resources by edm:provider.
+  module QAQueryClient
+    SPARQL_CLIENT = Repository.query_client
+    TYPE = :aggregation
+    VALUE = :value
+
+    module_function
+
+    ##
+    # Writes a {SPARQL::Client::Query} against the Repository for values
+    # matching at the end of a chain of predicates.
+    #
+    # @example
+    #   QAQueryClient.values_for_predicate(RDF::EDM.hasView,
+    #     RDF::URI('http://example.org/moomin'))
+    #   => #<SPARQL::Client::Query:0x3fb814e241e8(
+    #        SELECT ?aggregation ?value ?isShownAt WHERE {
+    #          ?aggregation <http://www.europeana.eu/schemas/edm/provider>
+    #            <http://example.org/moomin> .
+    #          ?aggregation a
+    #            <http://www.openarchives.org/ore/terms/Aggregation> .
+    #          OPTIONAL {
+    #            ?aggregation <http://www.europeana.eu/schemas/edm/hasView>
+    #              ?value .
+    #            ?aggregation <http://www.europeana.eu/schemas/edm/isShownAt>
+    #              ?isShownAt .
+    #          }
+    #        } ORDER BY ?aggregation)>
+    #
+    # @example
+    #   QAQueryClient.values_for_predicate([RDF::EDM.aggregatedCHO,
+    #                                       RDF::DC.title],
+    #     RDF::URI('http://example.org/moomin').execute.first.value
+    #   => #<SPARQL::Client::Query:0x3fb814e8379c(
+    #        SELECT ?aggregation ?value ?isShownAt WHERE {
+    #          ?aggregation <http://www.europeana.eu/schemas/edm/provider>
+    #            <http://example.org/moomin> .
+    #          ?aggregation a
+    #            <http://www.openarchives.org/ore/terms/Aggregation> .
+    #          OPTIONAL {
+    #            ?aggregation
+    #              <http://www.europeana.eu/schemas/edm/aggregatedCHO> ?obj0 .
+    #            ?obj0 <http://purl.org/dc/terms/title> ?value .
+    #            ?aggregation <http://www.europeana.eu/schemas/edm/isShownAt>
+    #              ?isShownAt .
+    #          }
+    #        } ORDER BY ?aggregation)>
+    #
+    # @example
+    #   QAQueryClient.values_for_predicate([RDF::EDM.aggregatedCHO,
+    #                                       RDF::DC.title],
+    #     RDF::URI('http://example.org/moomin').execute.first.value
+    #   # => "Stonewall Inn [2]"
+    #
+    # @param predicates [#to_uri, Array<#to_uri>] a predicate or list of
+    #   predicates to query
+    # @param provider_uri [#to_uri] a URI for an edm:provider value. Results
+    #   will be filtered to Resources that have this provider.
+    #
+    # @return [SPARQL::Client::Query] a query object that will give solutions
+    #   with `:value`, `:aggregation`, and `:isShownAt` variables.
+    def values_for_predicate(predicates, provider_uri = nil)
+      raise ArgumentError, 'provider_uri must be an RDF::URI' unless
+        provider_uri.respond_to? :to_uri
+
+      optional_patterns = build_optional_patterns(predicates)
+
+      optional_patterns << [TYPE, RDF::EDM.isShownAt, :isShownAt]
+
+      SPARQL_CLIENT.select(TYPE, VALUE, :isShownAt)
+        .where(*where_patterns(provider_uri))
+        .optional(*optional_patterns)
+        .order_by(TYPE)
+    end
+
+    ##
+    # Writes a {SPARQL::Client::Query} against the Repository for counts of
+    # values matching at the end of a chain of predicates.
+    #
+    # @example
+    #   QAQueryClient.counts_for_predicate(RDF::EDM.hasView,
+    #     RDF::URI('http://example.org/moomin')
+    #   => #<SPARQL::Client::Query:0x3fa596c6f068(
+    #        SELECT ?value ( COUNT(*) AS ?count ) WHERE {
+    #          ?aggregation <http://www.europeana.eu/schemas/edm/provider>
+    #            <http://example.org/moomin> .
+    #          ?aggregation a
+    #            <http://www.openarchives.org/ore/terms/Aggregation> .
+    #          OPTIONAL {
+    #            ?aggregation <http://www.europeana.eu/schemas/edm/hasView>
+    #              ?value .
+    #          }
+    #        } GROUP BY ?value ORDER BY DESC(?count))>
+    #
+    # @param predicates [#to_uri, Array<#to_uri>] a predicate or list of
+    #   predicates to query
+    # @param provider_uri [#to_uri] a URI for an edm:provider value. Results
+    #   will be filtered to Resources that have this provider.
+    #
+    # @return [SPARQL::Client::Query] a query object that will give solutions
+    #   with `:value` and `:count` variables.
+    def counts_for_predicate(predicates, provider_uri = nil)
+      raise ArgumentError, 'provider_uri must be an RDF::URI' unless
+        provider_uri.respond_to? :to_uri
+
+      optional_patterns = build_optional_patterns(predicates)
+
+      SPARQL_CLIENT.select(VALUE, count: { '*' => :count })
+        .where(*where_patterns(provider_uri))
+        .optional(*optional_patterns)
+        .group_by(VALUE)
+        .order_by('DESC(?count)')
+    end
+
+    ##
+    # @param provider_uri [#to_uri] a URI for an edm:provider value.
+    #
+    # @return [Array<Array<#to_term>>] An array of pattern arrays that match
+    #   ore:Aggregations with the given provider_uri
+    # @see RDF::Query
+    # @see RDF::Query::Pattern
+    def where_patterns(provider_uri)
+      [[TYPE, RDF::EDM.provider, provider_uri],
+       [TYPE, RDF.type, RDF::ORE.Aggregation]]
+    end
+
+    ##
+    # Builds patterns matching a predicate or chain of predicates given,
+    # assigning an unbound variable to each set of matches and passing it to
+    # the next pattern.
+    #
+    # @param predicates [#to_uri, Array<#to_uri>] a predicate or list of
+    #   predicates to build patterns against.
+    #
+    # @return [Array<Array<#to_term>>] An array of pattern arrays
+    # @see RDF::Query
+    # @see RDF::Query::Pattern
+    def build_optional_patterns(predicates)
+      return [[TYPE, predicates, VALUE]] unless
+        predicates.is_a? Enumerable
+
+      var1 = TYPE
+      patterns = predicates.each_with_object([]) do |predicate, ps|
+        var2 =
+          (ps.count == predicates.size - 1) ? VALUE : "obj#{ps.count}".to_sym
+        ps << [var1, predicate, var2]
+        var1 = var2
+      end
+    end
+  end
+end

data/lib/krikri/random_search_index_document_builder.rb

@@ -0,0 +1,41 @@
+module Krikri
+  # Gets random records from the search index
+  class RandomSearchIndexDocumentBuilder
+    attr_accessor :provider_id
+
+    ##
+    # @param &block may contain provider_id
+    #   Sample use:
+    #     RandomSearchIndexDocumentBuilder.new do
+    #       self.provider_id = '0123'
+    #     end
+    def initialize(&block)
+      # set values from block
+      instance_eval &block if block_given?
+    end
+
+    # @return Krikri::SearchIndexDocument
+    def document
+      solr_response = Krikri::SolrResponseBuilder.new(query_params)
+      return nil if solr_response.response.docs.empty?
+
+      Krikri::SearchIndexDocument.new(solr_response.response.docs.first)
+    end
+
+    private
+
+    ##
+    # Parameters for the Solr request.
+    # Limits search by @provider_id if it has been specified.
+    def query_params
+      params = { :id => '*:*',
+                 :sort => "random_#{rand(9999)} desc",
+                 :rows => 1 }
+      return params unless provider_id.present?
+
+      provider = RDF::URI(Krikri::Provider.base_uri) / provider_id
+      params[:fq] = "provider_id:\"#{provider}\""
+      params
+    end
+  end
+end