krikri 0.12.0 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ac6156f3d3dfd9164e4598d0b236bff739885ed
-  data.tar.gz: 437640a56aa024e4bcb4b58a58a70e02d3f8edb5
+  metadata.gz: 66d34d2d357a32a8406cd36a09e0333abec7ac0f
+  data.tar.gz: af0783e05398a667a1380bd0cad675ec546b489d
 SHA512:
-  metadata.gz: 9bdc4b60c7c533c4ed27d635c14fa11bef703037729edc5e4b31c80376f450289f1fb369c6f5e36d5de3ced9edd8162591d2f651bff01b6c63a1505c33861217
-  data.tar.gz: f2cc707b635d75265947eccc27617310199721186831e84b6d75ab64dab4178471631df760a5db5f01f8dff2dd42c305d8c47876216858ab0295b8035d9d6666
+  metadata.gz: 9806d3d2968791b32b743c49bf5b4ab9bda09bf44533680f194560b51adcab7948067f3d8df067dec6b74567c93c3648d3ca531c8c0be279ed7adcbfee13c3b9
+  data.tar.gz: 97087dbff34ec494908a678f8b02d37b61c3110ae8aef07c769dbe09b3092e116dc814f0dd020f50d53c1d0d777abe9dfffbadb169077f8b4a99eb489e553275

data/app/models/krikri/activity.rb

@@ -10,16 +10,42 @@ module Krikri
   class Activity < ActiveRecord::Base
     # @!attribute agent
     #    @return [String] a string representing the Krikri::SoftwareAgent
-    #                     responsible for the activity.
+    #      responsible for the activity.
+    # @!attribute start_time
+    #    @return [DateTime] a datestamp marking the activity's start
     # @!attribute end_time
     #    @return [DateTime] a datestamp marking the activity's competion
     # @!attribute opts
     #    @return [JSON] the options to pass to the #agent class when running
-    #                   the activity
-    # @!attribute start_time
-    #    @return [DateTime] a datestamp marking the activity's start
+    #      the activity
+    
     validate :agent_must_be_a_software_agent
 
+    ##
+    # @example building a valid URI from the base
+    #   Krikri::Activity.base_uri / 1
+    #   
+    # @return [RDF::URI] the configured base URI for this class
+    def self.base_uri
+      RDF::URI.intern(Krikri::Settings['marmotta']['ldp']) /
+        Krikri::Settings['prov']['activity']
+    end
+
+    ##
+    # @param uri [#to_s] a uri for this activity
+    #
+    # @return [Krikri::Activity] the activity with the given uri
+    #
+    # @raise [RuntimeError] if the URI form does not match the activity
+    # @raise [ActiveRecord::RecordNotFound] if no activity is found
+    def self.from_uri(uri)
+      raise "Cannot find #{self} from URI: #{uri}; " \
+            "the requested uri does not match #{base_uri}" unless 
+        uri.start_with? base_uri
+
+      find(uri.to_s.sub(base_uri.to_s, '').sub('/', ''))
+    end
+
     def agent_must_be_a_software_agent
       errors.add(:agent, 'does not represent a SoftwareAgent') unless
         agent.constantize < Krikri::SoftwareAgent
@@ -81,7 +107,9 @@ module Krikri
     def agent_instance
       @agent_instance ||= agent.constantize.new(parsed_opts)
     end
-
+    
+    ##
+    # @return [Hash] the options parsed as JSON
     def parsed_opts
       JSON.parse(opts, symbolize_names: true)
     end
@@ -89,11 +117,9 @@ module Krikri
     ##
     # @return [RDF::URI] the uri for this activity
     def rdf_subject
-      RDF::URI(Krikri::Settings['marmotta']['ldp']) /
-        Krikri::Settings['prov']['activity'] / id.to_s
+      self.class.base_uri / id.to_s
     end
     alias_method :to_term, :rdf_subject
-    
 
     ##
     # @return [String] a string reprerestation of the activity

data/lib/generators/krikri/templates/schema.xml

@@ -0,0 +1 @@
+../../../../solr_conf/schema.xml

data/lib/generators/krikri/templates/solrconfig.xml

@@ -0,0 +1 @@
+../../../../solr_conf/solrconfig.xml

data/lib/krikri/async_uri_getter.rb

@@ -51,6 +51,8 @@ module Krikri
     #   redirects.
     # @option opts [Integer] :max_redirects Number of redirects to follow before
     #   giving up. (default: 10)
+    # @option opts [Boolean] :inline_exceptions If true, pass exceptions as a
+    #   5xx response with the exception string in the body. (default: false)
     def initialize(opts: {})
       @default_opts = { max_redirects: MAX_REDIRECTS }.merge(opts)
     end
@@ -80,16 +82,41 @@ module Krikri
       # Wait for the request thread to complete
       def join
         @request_thread.join
+      rescue => e
+        # If the join throws an exception, the thread is dead anyway.  The
+        # subsequent call to `with_response` will propagate the exception to the
+        # calling thread.
+        raise e unless inline_exceptions?
       end
 
       ##
       # @yield [Faraday::Response] the response returned for the request
       def with_response
         yield @request_thread.value
+      rescue => e
+        if inline_exceptions?
+          # Deliver an error response to the caller to allow uniform access
+          msg = e.message + "\n\n" + e.backtrace.join("\n")
+          yield Faraday::Response.new(status: 500,
+                                      body: msg,
+                                      response_headers: {
+                                        'X-Exception' => e,
+                                        'X-Exception-Message' => e.message,
+                                        'X-Internal-Response' => 'true'
+                                      })
+        else
+          raise e
+        end
       end
 
       private
 
+      ##
+      # True if we are using inline exceptions
+      def inline_exceptions?
+        opts.fetch(:inline_exceptions, false)
+      end
+
       ##
       # Run the Faraday request in a new thread
       def start_request

data/lib/krikri/engine.rb

@@ -129,13 +129,13 @@ module Krikri
         # Get the persisted original record for this Aggregation.
         # @return [Krikri::OriginalRecord, nil]
         #
-        # @raise [NameError] when the original record id is not a URI or the
-        #   the original record has not been persisted to Marmotta.
+        # @raise [NameError] when the original record is empty or is a blank
+        #   node.
         #
         # @raise [Faraday::ConnectionError] when there is a connection problem
         #   with Marmotta.
         def original_record
-          raise NameError, "#{originalRecord.first} is not an OriginalRecord" if
+          raise NameError, no_origrec_message if
             originalRecord.empty? || originalRecord.first.node?
           Krikri::OriginalRecord.load(originalRecord.first.rdf_subject
             .to_s)
@@ -164,10 +164,25 @@ module Krikri
 
         def mint_id_fragment(seed = nil)
           return seed if seed
-          return SecureRandom.hex if
+          # We rely on originalRecord for consistent ID minting, so we have to
+          # raise an exception if it's empty or is a blank node; for example,
+          # if a mapping failed to map it.
+          raise NameError, no_origrec_message if
             originalRecord.empty? || originalRecord.first.node?
           local_name_from_original_record
         end
+
+        def no_origrec_message
+          "#{dpla_id} #{no_origrec_cond}"
+        end
+
+        def no_origrec_cond
+          if originalRecord.empty?
+            "has an empty originalRecord"
+          else
+            "has a blank node for its originalRecord"
+          end
+        end
       end
     end
 

data/lib/krikri/enricher.rb

@@ -2,9 +2,8 @@ module Krikri
   ##
   # A SoftwareAgent that runs enrichment processes.
   #
-  # @example
-  #
-  #   To enrich records that were mapped by the mapping activity with ID 3:
+  # @example to enrich records that were mapped by the mapping activity 
+  #   with ID 3:
   #
   #   # Define which enrichments are run, and thier parameters:
   #   chain = {
@@ -43,7 +42,6 @@ module Krikri
     # @see Audumbla::Enrichment
     # @param opts [Hash] a hash of options
     def initialize(opts = {})
-      @generator_uri = RDF::URI(opts.fetch(:generator_uri))
       @chain = deep_sym(opts.fetch(:chain) { {} })
       @entity_behavior = self.class.entity_behavior
       assign_generator_activity!(opts)
@@ -56,8 +54,7 @@ module Krikri
     # instantiation, and apply each enrichment from the enrichment chain.
     #
     def run(activity_uri = nil)
-      mapped_records = generator_activity.entities
-      mapped_records.each do |rec|
+      entities.each do |rec|
         begin
           chain_enrichments!(rec)
           activity_uri ? rec.save_with_provenance(activity_uri) : rec.save

data/lib/krikri/entity_behavior.rb

@@ -1,19 +1,36 @@
 
 module Krikri
   ##
-  # Base class for behaviors related to entities that are generated or revised
-  # by activities.
+  # Base class for retrieval behaviors related to entities that were generated 
+  # or revised by a `Krikri::Activity`.
   #
-  # A SoftwareAgent implements #entity_behavior, which returns an appropriate
-  # subclass of EntityBehavior.  When an Activity is queried for its entities,
-  # it instantiates an instance of its particular SoftwareAgent, and then
-  # calls the #entities method of the agent's entity behavior.
+  # @example implementing an entity behavior
+  #   class CustomBehavior < Krikri::EntityBehavior
+  #     def entities(load = true, include_invalidated = false)
+  #       activity_uris(include_invalidated) do |uri|
+  #         # some behavior over URIs to return initialized entities
+  #       end
+  #     end
+  #   end
+  #
+  # @example retrieving entities with a behavior
+  #   Krikri::Activity.find(activity_id)
+  #   CustomBehavor.entities(activity)
+  #
+  # A `SoftwareAgent` implements `#entity_behavior`, which returns an appropriate
+  # subclass of `EntityBehavior`.  When an Activity is queried for its entities,
+  # it instantiates an instance of its particular `SoftwareAgent`, and then
+  # calls the `#entities` method of the agent's entity behavior.
   #
   # @see Krikri::Activity#entities
   # @see lib/krikri/entity_behaviors
-  #
   class EntityBehavior
+    # @!attribute [r] activity
+    #   @return [Krikri::Activity]
     attr_reader :activity
+
+    ##
+    # @param activity [Krikri::Activity]
     def initialize(activity)
       @activity = activity
     end
@@ -21,20 +38,49 @@ module Krikri
     ##
     # Return an Enumerator of objects that have been affected by our @activity.
     #
-    # @return [Enumerator] objects
+    # @param load [Boolean] `true` to force load the entities from the datastore
+    #   on access
+    # @param include_invalidated [Boolean] `true` to include entities marked as
+    #   invalid.
+    #
+    # @return [Enumerator] the entities. When possible, they should be 
+    #   initialized & retrieved lazily.
+    #   
+    #
     # @see lib/krikri/entity_behaviors
     # @see Krikri::Activity#entities
     #
+    # @see Krikri::LDP::Invalidatable for more about "invalidated" entities
     def entities(*args)
-      raise NotImplementedError 
+      raise NotImplementedError
     end
 
     ##
-    # @see Krikri::Activity#entities
-    # @see Krikri::EntityBehavior#entities
+    # Initializes an instance of this class with the given `Activity` and
+    # returns an enumerator of the associated entities.
     #
+    # @param activity   [Krikri::Activity]
+    # @param load                [Boolean]
+    # @param include_invalidated [Boolean]
+    #
+    # @see Krikri::EntityBehavior#entities
+    # @see Krikri::Activity#entities
     def self.entities(activity, *args)
       new(activity).entities(*args)
     end
+
+    private
+    
+    ##
+    # Private utility method capturing common logic for applying entity logic to
+    # uris gathered from the instance's `Krikri::Activity`.
+    #
+    # @param include_invalided [Boolean]
+    #
+    # @return [Enumerator::Lazy] the uris, lazily mapped to the behavior in the
+    #   given block
+    def activity_uris(include_invalidated, &block)
+      activity.entity_uris(include_invalidated).lazy.map(&block)
+    end
   end
 end

data/lib/krikri/entity_behaviors/aggregation_entity_behavior.rb

@@ -22,7 +22,7 @@ module Krikri
     # @return [Enumerator] DPLA::MAP::Aggregation objects
     #
     def entities(load = true, include_invalidated = false)
-      @activity.entity_uris(include_invalidated).lazy.map do |uri|
+      activity_uris(include_invalidated) do |uri|
         agg = DPLA::MAP::Aggregation.new(uri)
         agg.get if load
         agg

data/lib/krikri/entity_behaviors/original_record_entity_behavior.rb

@@ -23,7 +23,7 @@ module Krikri
     # @return [Enumerator] OriginalRecord objects
     #
     def entities(load = true, include_invalidated = false)
-      @activity.entity_uris(include_invalidated).lazy.map do |uri|
+      activity_uris(include_invalidated) do |uri|
         load ? OriginalRecord.load(uri) : OriginalRecord.new(uri)
       end
     end

data/lib/krikri/entity_consumer.rb

@@ -1,15 +1,21 @@
 module Krikri
   ##
-  # A mixin for classes, like certain software agents, that generates entities.
-  # For example, a mapper usually generates RDF aggregations, such that
-  # Mapper::Agent includes EntityConsumer, and a mapper agent is instantiated
-  # with
-  #   a = Krikri::Mapper::Agent.new({
-  #     generator_uri: 'http://some.org/activity/1'
-  #   })
-  # such that
-  #   a.generator_activity
-  # returns a Krikri::Activity
+  # A mixin for `Krikri::SoftwareAgent`s that use entities. Provides a
+  # mechanism for setting an `#entity_source` and consuming entities.
+  #
+  # For backwards compatability, this supports an older interface where entities
+  # are selected based on a `generator_activity`.
+  #
+  # @example the deprecated interface
+  #   class AnAgent
+  #     include Krikri::EntityConsumer
+  #   end
+  #
+  #   agent = AnAgent.new
+  #   agent.assign_generator_activity!(generator_uri:
+  #     Krikri::Activity.find(1).rdf_subject)
+  #
+  #   agent.generator_activity.entities
   #
   module EntityConsumer
     extend ActiveSupport::Concern
@@ -32,17 +38,34 @@ module Krikri
     #
     # @see Krikri::Mapper::Agent
     # @see Krikri::Harvester
-    #
     def assign_generator_activity!(opts)
       if opts.include?(:generator_uri)
         generator_uri = opts.delete(:generator_uri)
-        # allow generator_uri to be string or RDF::URI with `to_s' ...
-        activity_id = generator_uri.to_s[/\d+$/].to_i  # 0 if no match
-        fail "Can not determine ID for #{generator_uri}" if activity_id == 0
-        @generator_activity = Krikri::Activity.find_by_id(activity_id)
-        raise "Generator activity not found for id #{activity_id}" \
-          unless @generator_activity
+        @entity_source =
+          @generator_activity = Krikri::Activity.from_uri(generator_uri)
       end
     end
+
+    ##
+    # @return [Enumerator<Krikri::LDP::Resource>] entities this agent will use
+    def entities
+      entity_source ? entity_source.entities : []
+    end
+
+    ##
+    # @return [#entities, nil]
+    def entity_source
+      @entity_source
+    end
+
+    ##
+    # Sets the entity source to a new instance of the provided class,
+    # initialized with the provided arguments and block.
+    #
+    # @param klass [Class] a class with an instance method `#entities`
+    # @return [void]
+    def set_entity_source!(klass, *args, &block)
+      @entity_source = klass.new(*args, &block)
+    end
   end
 end

data/lib/krikri/indexer.rb

@@ -2,21 +2,21 @@ module Krikri
   ##
   # A SoftwareAgent to run indexing processes.
   #
-  # @example
-  #
-  #   To index records enriched by the enrichment activity with ID 3:
-  #
+  # @example To index records enriched by the enrichment activity with ID 3:
   #   Krikri::Indexer.enqueue({
   #     index_class: 'Krikri::QASearchIndex',
   #     generator_uri: 'http://ldp.local.dp.la/ldp/activity/3'
   #     some_option_for_index_class: 'abc'
   #   })
   #
-  #   The options hash contains options for the Indexer as well as the
-  #   SearchIndex.
+  # The options hash contains options for the `Indexer` as well as the
+  # `SearchIndex`.
   #
+  # @todo use generalized `EntityConsumer` interface that is independent from
+  #   `Activity`. `#index` needs to be largely rewritten due to 
+  #   `#update_from_activity`, which is tighly bound to `Activity` rather than
+  #   `Enumerator<#entities>`.
   # @see Krikri::SoftwareAgent#enqueue
-  #
   class Indexer
     include SoftwareAgent
     include EntityConsumer

data/lib/krikri/mapper.rb

@@ -75,38 +75,48 @@ module Krikri
     ##
     # A SoftwareAgent to run mapping processes.
     #
-    # @example
-    #
-    #   To map the records harvested by the harvest activity with ID 1:
-    #
-    #   Krikri::Mapper::Agent.enqueue(
-    #     :mapping,
-    #     opts = {
-    #       name: 'scdl_qdc',
-    #       generator_uri: 'http://ldp.local.dp.la/ldp/activity/1'
-    #     }
-    #   )
+    # @example to map the records harvested by the harvest activity with ID 1:
+    #   Krikri::Mapper::Agent.enqueue(name: :scdl_qdc,
+    #     generator_uri: 'http://ldp.local.dp.la/ldp/activity/1')
     #
     # @see: Krikri::SoftwareAgent, Krikri::Activity
     class Agent
       include SoftwareAgent
       include EntityConsumer
 
+      # @!attribute [r] name
+      #   @return [Symbol]
       attr_reader :name
 
+      ##
+      # @return [Symbol] the default queue for jobs using this agent
       def self.queue_name
         :mapping
       end
 
+      ##
+      # @see Krikri::Activity#entities
+      # @see Krikri::EntityBehavior
+      # @see Krikri::SoftwareAgent#entity_behavior
+      def entity_behavior
+        @entity_behavior ||= Krikri::AggregationEntityBehavior
+      end
+
+      ##
+      # @param opts [Hash]
+      # @option opts [#to_sym] name  the symbol naming the mapping to use
       def initialize(opts = {})
         @name = opts.fetch(:name).to_sym
         @entity_behavior = self.class.entity_behavior
         assign_generator_activity!(opts)
       end
 
+      ##
+      # @param activity_uri [RDF::URI] the uri of the activity to attribute
+      #   for provenance purposes (default: nil)
+      # @see SoftwareAgent#run
       def run(activity_uri = nil)
-        harvest_records = generator_activity.entities
-        Krikri::Mapper.map(name, harvest_records).each do |rec|
+        Krikri::Mapper.map(name, entities).each do |rec|
           begin
             rec.mint_id! if rec.node?
             activity_uri ? rec.save_with_provenance(activity_uri) : rec.save
@@ -116,7 +126,6 @@ module Krikri
           end
         end
       end
-
     end
   end
 end

data/lib/krikri/search_index.rb

@@ -6,6 +6,10 @@ module Krikri
   ##
   # Search index base class that gets extended by QA and Production index
   # classes
+  #
+  # @todo rewrite to use generalized `EntityConsumer` interface & avoid 
+  #   `#update_from_activity`, which is tighly bound to `Activity` rather than
+  #   `Enumerator<#entities>`.
   class SearchIndex
     def initialize(opts)
       @bulk_update_size = opts.delete(:bulk_update_size) { 10 }

data/lib/krikri/software_agent.rb

@@ -1,13 +1,17 @@
 module Krikri
   ##
-  # SoftwareAgent is a mixin for logic common to code that generates a
-  # `Krikri::Activity`.
+  # SoftwareAgent is a mixin for logic common to classes that carry out the 
+  # work involved in a `Krikri::Activity`. This corresponds to a 
+  # prov:SoftwareAgent within PROV-O & PROV-DM.
   #
   # Software Agents should handle internal errors that do not result in full
   # activity failure, and raise a `RuntimeError` when the job fails. `Activity`
   # handles logging of activity start/stop, and failure status.
   #
+  # Implementers must provide a `#run` method.
+  #
   # @see Krikri::Activity
+  # @see https://www.w3.org/TR/prov-dm/#concept-software-agent
   module SoftwareAgent
     extend ActiveSupport::Concern
 
@@ -30,13 +34,13 @@ module Krikri
 
     ##
     # @abstract Perform this agent's work. The method may accept an
-    #   `activity_uri` to record as the Activity in provenance metadata.
+    #   `activity_uri` to record as the prov:Activity in provenance metadata.
     #   If so, the implementation must be optional and handle `nil` values by
     #   declining to record provenance
     #
     # @return [Boolean] `true` if the run has succeeded; otherwise `false`
     #
-    # @raise [RuntimeError] when the software agent's activity fails
+    # @raise [RuntimeError] when the software agent's process fails
     #
     # @see Krirkri::Activity
     # @see Krikri::Job.run

data/lib/krikri/version.rb

@@ -1,3 +1,3 @@
 module Krikri
-  VERSION = '0.12.0'.freeze
+  VERSION = '0.12.1'.freeze
 end

data/spec/internal/Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 
 # Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
-gem 'rails', '4.1.9'
+gem 'rails', '4.1.14.1'
 # Use sqlite3 as the database for Active Record
 gem 'sqlite3'
 # Use SCSS for stylesheets
@@ -11,7 +11,7 @@ gem 'sass-rails', '~> 4.0.3'
 gem 'uglifier', '>= 1.3.0'
 # Use CoffeeScript for .js.coffee assets and views
 gem 'coffee-rails', '~> 4.0.0'
-# See https://github.com/sstephenson/execjs#readme for more supported runtimes
+# See https://github.com/rails/execjs#readme for more supported runtimes
 # gem 'therubyracer',  platforms: :ruby
 
 # Use jquery as the JavaScript library
@@ -36,7 +36,7 @@ gem 'sdoc', '~> 0.4.0',          group: :doc
 # gem 'debugger', group: [:development, :test]
 
         
-        gem 'krikri', :path => '/vagrant'
+        gem 'krikri', :path => '/Users/mark/Documents/Code/dpla/KriKri'
 
 gem "factory_girl_rails", "~> 4.4.0", group: :development
 gem "jettywrapper", "~> 2.0", group: :development