krikri 0.3.3 → 0.4.0

data/lib/krikri/harvesters/api_harvester.rb

@@ -0,0 +1,164 @@
+module Krikri::Harvesters
+  ##
+  # A harvester implementation for REST APIs. The default ApiHarvester expects
+  # Solr-like JSON responses/records.
+  #
+  # An internal interface is provided for easier subclassing. A new API
+  # harvester may reimplement:
+  #   - #get_docs (to retrieve record docs from a response)
+  #   - #get_count (to determine total record count from a response)
+  #   - #get_identifier (to retrieve an indentifier from a record document)
+  #   - #get_content (to retrieve a content string from a record document)
+  #   - #next_options` (to generate the parameters for the next request)
+  #
+  # If the content type of the records is other than JSON, you will also want
+  # to override `#content_type`.
+  class ApiHarvester
+    include Krikri::Harvester
+
+    attr_reader :opts
+
+    ##
+    # @param opts [Hash] options for the harvester
+    # @see .expected_opts
+    def initialize(opts = {})
+      super
+      @opts = opts.fetch(:api, {})
+    end
+
+    ##
+    # @return [Hash] A hash documenting the allowable options to pass to
+    #   initializers.
+    #
+    # @see Krikri::Harvester::expected_opts
+    def self.expected_opts
+      {
+        key: :api,
+        opts: {
+          params: { type: :string, required: false }
+        }
+      }
+    end
+
+    ##
+    # @see Krikri::Harvester#count
+    def count
+      get_count(request(opts))
+    end
+
+    ##
+    # @return [Enumerator::Lazy] an enumerator of the records targeted by this
+    #   harvester.
+    def records
+      enumerate_records.lazy.map { |rec| build_record(rec) }
+    end
+
+    ##
+    # Gets a single record with the given identifier from the API
+    #
+    # @return [Enumerator::Lazy] an enumerator over the ids for the records
+    #   targeted by this harvester.
+    def record_ids
+      enumerate_records.lazy.map { |r| get_identifier(r) }
+    end
+
+    ##
+    # @param identifier [#to_s] the identifier of the record to get
+    # @return [#to_s] the record
+    def get_record(identifier)
+      response = request(:params => { :q => "id:#{identifier.to_s}" })
+      build_record(get_docs(response).first)
+    end
+
+    ##
+    # @return [String] the content type for the records generated by this
+    #   harvester
+    def content_type
+      'application/json'
+    end
+
+    private
+
+    ##
+    # @param doc [#to_s] a raw record document with an identifier
+    #
+    # @return [String] the provider's identifier for the document
+    def get_identifier(doc)
+      doc['record_id']
+    end
+
+    ##
+    # @param response [#to_s] a response from the REST API
+    #
+    # @return [Integer] a count of the total records found by the request
+    def get_count(response)
+      response['response']['numFound']
+    end
+
+    ##
+    # @param response [#to_s] a response from the REST API
+    #
+    # @return [Array] an array of record documents from the response
+    def get_docs(response)
+      response['response']['docs']
+    end
+
+    ##
+    # @param doc [#to_s] a raw record document
+    #
+    # @return [String] the record content
+    def get_content(doc)
+      doc.to_json
+    end
+
+    ##
+    # Send a request via `RestClient`, and parse the result as JSON
+    def request(request_opts)
+      JSON.parse(RestClient.get(uri, request_opts))
+    end
+
+    ##
+    # Given a current set of options and a number of records from the last
+    # request, generate the options for the next request.
+    #
+    # @param opts [Hash] an options hash from the previous request
+    # @param record_count [#to_i]
+    #
+    # @return [Hash] the next request's options hash
+    def next_options(opts, record_count)
+      old_start = opts['params'].fetch('start', 0)
+      opts['params']['start'] = old_start.to_i + record_count
+      opts
+    end
+
+    ##
+    # @return [Enumerator] an enumerator over the records
+    def enumerate_records
+      Enumerator.new do |yielder|
+        request_opts = opts.deep_dup
+        loop do
+          break if request_opts.nil?
+          docs = get_docs(request(request_opts.dup))
+          break if docs.empty?
+
+          docs.each { |r| yielder << r }
+
+          request_opts = next_options(request_opts, docs.count)
+        end
+      end
+    end
+
+    ##
+    # Builds an instance of `@record_class` with the given doc's JSON as
+    # content.
+    #
+    # @param doc [#to_json] the content to serialize as JSON in `#content`
+    # @return [#to_s] an instance of @record_class with a minted id and
+    #   content the given content
+    def build_record(doc)
+      @record_class.build(mint_id(get_identifier(doc)),
+                          get_content(doc),
+                          content_type)
+    end
+  end
+end

data/lib/krikri/harvesters/api_harvester.rb~

@@ -0,0 +1,44 @@
+module Krikri::Harvesters
+  ##
+  # A harvester implementation for REST APIs
+  class ApiHarvester
+    include Krikri::Harvester
+
+    def initialize(opts = {})
+      super
+      @opts = opts.fetch(:api, {})
+    end
+
+    ##
+    # @return [Hash] A hash documenting the allowable options to pass to
+    #   initializers.
+    #
+    # @see Krikri::Harvester::expected_opts
+    def self.expected_opts
+      {
+        key: :api,
+        opts: {
+          params: { type: :string, required: false }
+        }
+      }
+    end
+
+    ##
+    # @see Krikri::Harvester#count
+    def count
+      JSON.parse(request(opts['params']))['response']['numFound']
+    end
+
+    ##
+    # @return [Enumerator::Lazy] an enumerator of the records targeted by this
+    #   harvester.
+    def records
+    end
+
+    private
+
+    def request(opts)
+      RestClient.get uri, opts
+    end
+  end
+end

data/lib/krikri/harvesters/mdl_api_harvester.rb~

@@ -0,0 +1,7 @@
+module Krikri::Harvesters
+  ##
+  # A harvester implementation for Minnesota Digital Library's API
+  class MdlApiHarvester
+
+  end
+end

data/lib/krikri/harvesters/mdl_harvester.rb~

@@ -0,0 +1,4 @@
+module Krikri::Harvesters
+  ##
+  # A harvester implementation for MDL's API
+  class MdlHarvester

data/lib/krikri/harvesters/oai_harvester.rb

@@ -156,6 +156,14 @@ module Krikri::Harvesters
     ##
     # Runs the request in the given block against the sets specified in `opts`.
     # Results are concatenated into a single enumerator
+    #
+    # @param opts [Hash] the options to pass, including all sets to process.
+    # @yield [set_opts] gives options to the block once for each set. The
+    #   block should run the harvest action with the options and give an
+    #   Enumerable.
+    #
+    # @return [Enumerator::Lazy] A lazy enumerator concatenating the results
+    #   of the block with each set.
     def request_with_sets(opts, &block)
       sets = Array(opts.delete(:set))
       if opts[:skip_set]

data/lib/krikri/ldp/rdf_source.rb

@@ -2,6 +2,8 @@ module Krikri::LDP
   ##
   # Adds simple LDP persistence to ActiveTriples::Resource classes
   # @see ActiveTriples::Resource
+  #
+  # @see http://www.w3.org/TR/ldp/#ldprs
   module RdfSource
     extend ActiveSupport::Concern
     include Krikri::LDP::Resource
@@ -30,6 +32,34 @@ module Krikri::LDP
       result
     end
 
+    ##
+    # Adds an appropritate provenance statement with the given URI and saves
+    # the resource.
+    #
+    # This method treats RDFSources as stateful resources. This is in conflict
+    # with the PROV model, which assumes each revision is its own Resource. The
+    # internal predicate `dpla:wasRevisedBy` is used for non-generating
+    # revisions of stateful RDFSources.
+    #
+    # @todo Assuming a Marmotta LDP server, there are version URIs available
+    #   (via Memento) which could be used for a direct PROV implementation.
+    #   Consider options for doing that either alongside or in place of this
+    #   approach.
+    #
+    # @param activity_uri [#to_term] the URI of the prov:Activity to mark as
+    #   generating or revising the saved resource.
+    #
+    # @see #save
+    #
+    # @see http://www.w3.org/TR/prov-primer/
+    # @see http://www.w3.org/TR/2013/REC-prov-o-20130430/
+    def save_with_provenance(activity_uri)
+      predicate =
+        exists? ? RDF::DPLA.wasRevisedBy : RDF::PROV.wasGeneratedBy
+      self << RDF::Statement(self, predicate, activity_uri)
+      save
+    end
+
     private
 
     ##

data/lib/krikri/mapper.rb

@@ -109,9 +109,7 @@ module Krikri
         Krikri::Mapper.map(name, records).each do |rec|
           begin
             rec.mint_id! if rec.node?
-            rec << RDF::Statement(rec, RDF::PROV.wasGeneratedBy, activity_uri) if
-              activity_uri
-            rec.save
+            activity_uri ? rec.save_with_provenance(activity_uri) : rec.save
           rescue => e
             Rails.logger.error("Error saving record: #{rec.rdf_subject}\n" \
                                "#{e.message}\n#{e.backtrace}")

data/lib/krikri/provenance_query_client.rb

@@ -6,11 +6,21 @@ module Krikri
 
     module_function
 
+    ##
+    # Finds all entities generated or revised by the activity whose URI is
+    # given.
+    #
+    # @param activity_uri [#to_uri] the URI of the activity to search
+    #
+    # @return [RDF::SPARQL::Query] a query object that, when executed, will
+    #   give solutions containing the URIs for the resources in `#record`.
     def find_by_activity(activity_uri)
       raise ArgumentError, 'activity_uri must be an RDF::URI' unless
         activity_uri.respond_to? :to_uri
       SPARQL_CLIENT.select(:record)
-        .where([:record, RDF::PROV.wasGeneratedBy, activity_uri])
+        .where([:record,
+                [RDF::PROV.wasGeneratedBy, '|', RDF::DPLA.wasRevisedBy],
+                activity_uri])
     end
   end
 end

data/lib/krikri/util/extended_date_parser.rb

@@ -1,4 +1,6 @@
 module Krikri::Util
+  ##
+  # Utilities to parse string values into EDTF dates or Intervals.
   module ExtendedDateParser
     module_function
 
@@ -20,18 +22,21 @@ module Krikri::Util
     #   #range_match to look for range values.
     #
     # @return [Date, EDTF::Epoch, EDTF::Interval, nil] the date parsed or nil
+    # @see http://www.loc.gov/standards/datetime/
     def parse(date_str, allow_interval = false)
-      date_str.strip!
-      date_str.gsub!(/\s+/, ' ')
-      date = parse_interval(date_str) if allow_interval
-      date ||= parse_m_d_y(date_str)
-      date ||= Date.edtf(date_str.gsub('.', '-'))
-      date ||= partial_edtf(date_str)
-      date ||= decade_hyphen(date_str)
-      date ||= month_year(date_str)
-      date ||= decade_s(date_str)
-      date ||= hyphenated_partial_range(date_str)
-      date ||= parse_date(date_str)
+      str = preprocess(date_str.dup)
+      date = parse_interval(str) if allow_interval
+      date ||= parse_m_d_y(str)
+      date ||= Date.edtf(str.gsub('.', '-'))
+      date ||= partial_edtf(str)
+      date ||= decade_hyphen(str)
+      date ||= month_year(str)
+      date ||= decade_s(str)
+      date ||= hyphenated_partial_range(str)
+      date ||= parse_date(str)
+      # Only do this if certian letters are present to avoid infinite loops.
+      date ||= circa(str) if str.match(/[circabout]/i)
+      date = date.first if date.is_a? EDTF::Set
       date || nil
     end
 
@@ -55,6 +60,36 @@ module Krikri::Util
       end
     end
 
+    ##
+    # Preprocess the date string to remove extra whitespace and convert ad hoc
+    # formatting to equivalent EDTF.
+    #
+    # @todo should '-` be intepreted as 'x' or '?'
+    # @see http://www.loc.gov/standards/datetime/pre-submission.html#maskedprecision
+    def preprocess(str)
+      str.gsub!(/late/i, '')
+      str.gsub!(/early/i, '')
+      str.strip!
+      str.gsub!(/\s+/, ' ')
+      str.gsub!('0s', 'x') if str.match(/^[1-9]+0s$/)
+      str.gsub!('-', 'x') if str.match(/^[1-9]+\-+$/)
+      str
+    end
+
+    ##
+    # Remove 'circa' or 'about' or variations and return an uncertian ETDF
+    # dates.
+    #
+    # @param str [String]
+    # @return [Date, nil] an EDTF date, marked uncertian; or `nil`
+    # @see #parse
+    def circa(str)
+      run = str.gsub!(/.*c[irca\.]*/i, '')
+      run ||= str.gsub!(/.*about/i, '')
+      date = parse(str) if run
+      date.nil? ? nil : date.uncertain!
+    end
+
     ##
     # Creates an EDTF::Interval from a string
     #

data/lib/krikri/version.rb

@@ -1,3 +1,3 @@
 module Krikri
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

data/spec/internal/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: /home/tjohnson/src/dpla/krikri
   specs:
-    krikri (0.3.2)
+    krikri (0.3.3)
       analysand (= 4.0.0)
       blacklight (~> 5.8.0)
       devise (~> 3.4.1)
-      dpla-map (= 4.0.0.0.pre.9)
+      dpla-map (= 4.0.0.0.pre.10)
       edtf
       jsonpath
       oai
@@ -61,7 +61,7 @@ GEM
       rack
       yajl-ruby
     arel (5.0.1.20140414130214)
-    autoprefixer-rails (5.1.7)
+    autoprefixer-rails (5.1.7.1)
       execjs
       json
     bcrypt (3.1.10)
@@ -72,7 +72,7 @@ GEM
       nokogiri (~> 1.6)
       rails (>= 3.2.6, < 5)
       rsolr (~> 1.0.6)
-    bootstrap-sass (3.3.3)
+    bootstrap-sass (3.3.4.1)
       autoprefixer-rails (>= 5.0.0.1)
       sass (>= 3.2.19)
     builder (3.2.2)
@@ -100,10 +100,10 @@ GEM
       responders
       thread_safe (~> 0.1)
       warden (~> 1.2.3)
-    dpla-map (4.0.0.0.pre.9)
-      active-triples (~> 0.3)
-      linked_vocabs (~> 0.1)
-    ebnf (0.3.6)
+    dpla-map (4.0.0.0.pre.10)
+      active-triples (~> 0.6.0)
+      linked_vocabs (~> 0.2.0)
+    ebnf (0.3.7)
       haml (~> 4.0)
       rdf (~> 1.1)
       sxp (~> 0.1, >= 0.1.3)
@@ -130,7 +130,7 @@ GEM
     htmlentities (4.3.3)
     http_parser.rb (0.6.0)
     i18n (0.7.0)
-    jbuilder (2.2.11)
+    jbuilder (2.2.12)
       activesupport (>= 3.0.0, < 5)
       multi_json (~> 1.2)
     jettywrapper (2.0.3)
@@ -153,8 +153,8 @@ GEM
       activesupport (>= 3.0.0)
     libv8 (3.16.14.7)
     link_header (0.0.8)
-    linked_vocabs (0.1.0)
-      active-triples (>= 0.1.0)
+    linked_vocabs (0.2.0)
+      active-triples (>= 0.6.0)
       rake
       rdf (>= 1.1.2.1)
       sparql
@@ -329,7 +329,7 @@ GEM
       libv8 (~> 3.16.14.0)
       ref
     thor (0.19.1)
-    thread_safe (0.3.4)
+    thread_safe (0.3.5)
     tilt (1.4.1)
     timers (4.0.1)
       hitimes