krikri 0.2.1 → 0.3.1

data/lib/krikri/harvesters/basic_save_behavior.rb

@@ -0,0 +1,9 @@
+module Krikri::Harvesters
+  ##
+  # Harvest behavior that call a simple save on a record
+  class BasicSaveBehavior < HarvestBehavior
+    def process_record
+      record.save(activity_uri)
+    end
+  end
+end

data/lib/krikri/harvesters/basic_save_behavior.rb~

@@ -0,0 +1 @@
+module Krikri

data/lib/krikri/harvesters/harvest_behavior.rb

@@ -0,0 +1,40 @@
+module Krikri::Harvesters
+  ##
+  # Defines an interface for handling records during the harvest process.
+  # Subclasses specify behavior by implementing `#process_record`.
+  #
+  # Behaviors should be implemented idempotently so they can be safely
+  # retried on errors.
+  #
+  # @example
+  #   behavior = MyHarvestBehavior.new(record, activity_uri)
+  #   behavior.process_record
+  #
+  # @example
+  #   MyHarvestBehavior.process_record(record, activity_uri)
+  #
+  # @see Krirki::Harvester#run
+  class HarvestBehavior
+    # @!attribute activity_uri [r]
+    #   a URI identifying the activity responsible for invoking the behavior
+    # @!attribute record [r]
+    #   the record to process with this behavior
+    attr_reader :activity_uri, :record
+
+    def initialize(record, activity_uri)
+      @record = record
+      @activity_uri = activity_uri
+    end
+
+    ##
+    # Creates a new instance of this behavior with the given arguments
+    # and calls `#process_record`.
+    #
+    # @param activity_uri
+    # @param record
+    # @see self#record, self#activity_uri for parameter usage
+    def self.process_record(record, activity_uri)
+      new(record, activity_uri).process_record
+    end
+  end
+end

data/lib/krikri/harvesters/harvest_behavior.rb~

@@ -0,0 +1,14 @@
+module Krikri::Harvesters
+  class HarvestBehavior
+    attr_reader :activity_uri, :record
+
+    def initialize(record, activity_uri)
+      @record = record
+      @activity_uri = activity_uri
+    end
+
+    def self.process_record(record, activity_uri)
+      new(record, activity_uri).process_record
+    end
+  end
+end

data/lib/krikri/harvesters/oai_harvester.rb

@@ -1,6 +1,29 @@
 module Krikri::Harvesters
   ##
   # A harvester implementation for OAI-PMH
+  #
+  # Accepts options to pass to OAI client as `:oai => opts`
+  #
+  # @example
+  #
+  #   OAIHarvester.new(:uri => endpoint,
+  #     :oai => { :set => 'my_set', :metadata_prefix => 'oai_dc' }
+  #
+  # Options allowed are:
+  #
+  #   - set: A string or array of strings specifying the sets to harvest.
+  #          If multiple sets are given, they will be lazily requested from
+  #          `OAI::Client#list_records` in turn and combined into a single
+  #          enumerator.
+  #   - skip_set: A string or array of strings specifying the sets to skip.
+  #               If both `set` and `skip_set` are given, sets specified as
+  #               skip are excluded from the harvest. Otherwise, all sets
+  #               returned by `#set` except those skipped will be harvested.
+  #   - metadata_prefix: A string specifying the metadata prefix. e.g. 'oai_dc'.
+  #   - from: The begin date for the harvest.
+  #   - until: The end date for the harvest.
+  #
+  # @see http://www.rubydoc.info/gems/oai/OAI/Client
   class OAIHarvester
     include Krikri::Harvester
     attr_accessor :client
@@ -12,6 +35,7 @@ module Krikri::Harvesters
     # @see OAI::Client
     # @see #expected_opts
     def initialize(opts = {})
+      opts[:harvest_behavior] ||= OAISkipDeletedBehavior
       super
       @opts = opts.fetch(:oai, {})
 
@@ -32,9 +56,13 @@ module Krikri::Harvesters
     #
     #     record_ids.take(1000)
     #
+    # @param opts [Hash] opts to pass to OAI::Client
+    # @see #expected_opts
     def record_ids(opts = {})
-      opts = opts.merge(@opts)
-      client.list_identifiers(opts).full.lazy.flat_map(&:identifier)
+      opts = @opts.merge(opts)
+      request_with_sets(opts) do |set_opts|
+        client.list_identifiers(set_opts).full.lazy.flat_map(&:identifier)
+      end
     end
 
     # Count on record_ids will request all ids and load them into memory
@@ -51,38 +79,106 @@ module Krikri::Harvesters
     #
     #     records.take(1000)
     #
+    # @param opts [Hash] opts to pass to OAI::Client
+    # @see #expected_opts
     def records(opts = {})
-      opts = opts.merge(@opts)
-      client.list_records(opts).full.lazy.flat_map do |rec|
-        @record_class.build(mint_id(rec.header.identifier),
-                            record_xml(rec))
-
+      opts = @opts.merge(opts)
+      request_with_sets(opts) do |set_opts|
+        client.list_records(set_opts).full.lazy.flat_map do |rec|
+          @record_class.build(mint_id(rec.header.identifier),
+                              record_xml(rec))
+        end
       end
     end
 
-    # TODO: normalize records; there will be differences in XML
-    # for different requests
+    ##
+    # Gets a single record with the given identifier from the OAI endpoint
+    #
+    # @param identifier [#to_s] the identifier of the record to get
+    # @param opts [Hash] options to pass to the OAI client
     def get_record(identifier, opts = {})
       opts[:identifier] = identifier
-      opts = opts.merge(@opts)
+      opts = @opts.merge(opts)
       @record_class.build(mint_id(identifier),
                           record_xml(client.get_record(opts).record))
     end
 
     ##
+    # Lists the sets available from the OAI endpoint. Accepts a block to
+    # pass to `#map` on the resulting array.
+    #
+    # @example:
+    #
+    #   sets(&:spec)
+    #
+    # @param opts [Hash] options to pass to the OAI client
+    # @return [Array<OAI::Set>] an array of sets.
+    #
+    # @see OAI::Set
+    def sets(opts = {}, &block)
+      arry = client.list_sets.full.to_a
+      return arry unless block_given?
+      arry.map(&block)
+    end
+
+    ##
+    # @return [Hash] A hash documenting the allowable options to pass to
+    #   initializers.
+    #
     # @see Krikri::Harvester::expected_opts
     def self.expected_opts
       {
         key: :oai,
         opts: {
           set: {type: :string, required: false, multiple_ok: true},
-          metadata_prefix: {type: :string, required: true}
+          skip_set: {type: :string, required: false, multiple_ok: true},
+          metadata_prefix: {type: :string, required: false},
+          from: {type: :string, required: false},
+          until: {type: :string, required: false}
         }
       }
     end
 
+    ##
+    # Concatinates two enumerators
+    # @todo find a better home for this. Reopen Enumerable? or use the
+    #    `Enumerating` gem: https://github.com/mdub/enumerating
+    def concat_enum(*enums)
+      Enumerator.new do |yielder|
+        enums.each do |enum|
+          enum.each { |i| yielder << i }
+        end
+      end
+    end
+
     private
 
+    ##
+    # Runs the request in the given block against the sets specified in `opts`.
+    # Results are concatenated into a single enumerator
+    def request_with_sets(opts, &block)
+      sets = Array(opts.delete(:set))
+      if opts[:skip_set]
+        sets = self.sets(&:spec) if sets.empty?
+        skips = Array(opts.delete(:skip_set))
+        sets.reject! { |s| skips.include? s }
+      end
+      sets = [nil] if sets.empty?
+
+      set_enums = sets.map do |set|
+        set_opts = opts.dup
+        set_opts[:set] = set unless set.nil?
+        yield(set_opts) if block_given?
+      end
+      concat_enum(*set_enums).lazy
+    end
+
+    ##
+    # Transforms an OAI::Record to xml suitable for saving with the
+    # OriginalRecord
+    #
+    # @param rec [OAI::Record]
+    # @return [String] an xml string
     def record_xml(rec)
       doc = Nokogiri::XML::Builder.new do |xml|
         xml.record('xmlns' => 'http://www.openarchives.org/OAI/2.0/') {
@@ -93,10 +189,12 @@ module Krikri::Harvesters
               xml.set_spec set.text
             end
           }
-          xml << rec.metadata.to_s
+          xml << rec.metadata.to_s unless rec.metadata.nil?
           xml << rec.about.to_s unless rec.about.nil?
         }
       end
+      doc.doc.at_css('header')['status'] = rec.header.status if
+        rec.header.status
       doc.to_xml
     end
   end

data/lib/krikri/harvesters/oai_skip_deleted_behavior.rb

@@ -0,0 +1,20 @@
+module Krikri::Harvesters
+  ##
+  # Harvest behavior that skips OAI records marked as deleted
+  class OAISkipDeletedBehavior < BasicSaveBehavior
+    def process_record
+      return if deleted?(record)
+      super
+    end
+
+    private
+
+    def deleted?(record)
+      header = Nokogiri::XML(record.content).xpath('//xmlns:header')
+      return false if header.empty?
+      status = header.first['status']
+      return true if status.to_s.downcase.include? 'deleted'
+      false
+    end
+  end
+end

data/lib/krikri/harvesters/oai_skip_deleted_behavior.rb~

@@ -0,0 +1,17 @@
+module Krikri::Harvesters
+  ##
+  # Harvest behavior that skips OAI records marked as deleted
+  class OAISkipDeletedBehavior < BasicSaveBehavior
+    def process_record
+      return if deleted?(record)
+      super
+    end
+
+    private
+
+    def deleted?(record)
+      require 'pry'
+      binding.pry
+    end
+  end
+end

data/lib/krikri/id_minter.rb~

@@ -0,0 +1,8 @@
+module Krikri
+  class IdMinter
+    class << self
+      def mint
+      end
+    end
+  end
+end

data/lib/krikri/id_service.rb~

@@ -0,0 +1,4 @@
+module Krikri
+  module IdService
+  end
+end

data/lib/krikri/index_service.rb

@@ -4,8 +4,25 @@ require 'rsolr'
 module Krikri
   ##
   # Generates flattened Solr documents and manages indexing of DPLA MAP models.
+  #
+  # @example
+  #
+  #   indexer = Krikri::IndexService.new
+  #   agg = Krikri::Aggregation.new
+  #   json_doc = agg.to_jsonld['@graph'].first.to_json
+  #
+  #   indexer.add(json_doc)
+  #   indexer.commit
+  #
   class IndexService
-    @solr = RSolr.connect
+    attr_reader :solr
+
+    ##
+    # @param opts [Hash] options to pass to RSolr
+    # @see RSolr.connect
+    def initialize(opts = Krikri::Settings.solr.to_h)
+      @solr = RSolr.connect(opts)
+    end
 
     # TODO: Assure that the following metacharacters are escaped:
     # + - && || ! ( ) { } [ ] ^ " ~ * ? : \
@@ -13,22 +30,22 @@ module Krikri
     ##
     # Adds a single JSON document to Solr
     # @param JSON
-    def self.add(doc)
-      @solr.add solr_doc(doc)
+    def add(doc)
+      solr.add solr_doc(doc)
     end
 
     ##
     # Deletes an item from Solr
     # @param String or Array
-    def self.delete_by_id(id)
-      @solr.delete_by_id id
+    def delete_by_id(id)
+      solr.delete_by_id id
     end
 
     ##
     # Deletes items from Solr that match query
     # @param String or Array
-    def self.delete_by_query(query)
-      @solr.delete_by_query query
+    def delete_by_query(query)
+      solr.delete_by_query query
     end
 
     ##
@@ -36,32 +53,23 @@ module Krikri
     # Should be run after self.add and self.delete
     # Okay to add or delete multiple docs and commit them all with
     # a single self.commit
-    def self.commit
-      @solr.commit
+    def commit
+      solr.commit
     end
 
     ##
     # Converts JSON document into a Hash that complies with Solr schema
     # @param [JSON]
     # @return [Hash]
-    def self.solr_doc(doc)
+    def solr_doc(doc)
       remove_invalid_keys(flat_hash(JSON.parse(doc)))
     end
 
-    ##
-    # Remove keys (ie. fields) that are not in the Solr schema.
-    # @param [Hash]
-    # @return [Hash]
-    def self.remove_invalid_keys(solr_doc)
-      valid_keys = schema_keys
-      solr_doc.delete_if { |key, _| !key.in? valid_keys }
-    end
-
     ##
     # Get field names from Solr schema in host application.
     # Will raise exception if file not found.
     # @return [Array]
-    def self.schema_keys
+    def schema_keys
       schema_file = File.join(Rails.root, 'solr_conf', 'schema.xml')
       file = File.open(schema_file)
       doc = Nokogiri::XML(file)
@@ -69,13 +77,15 @@ module Krikri
       doc.xpath('//fields/field').map { |f| f.attr('name') }
     end
 
+    private
+
     ##
     # Flattens a nested hash
     # Joins keys with "_" and removes "@" symbols
     # Example:
     #   flat_hash( {"a"=>"1", "b"=>{"c"=>"2", "d"=>"3"} )
     #   => {"a"=>"1", "b_c"=>"2", "b_d"=>"3"}
-    def self.flat_hash(hash, keys = [])
+    def flat_hash(hash, keys = [])
       new_hash = {}
 
       hash.each do |key, val|
@@ -99,14 +109,20 @@ module Krikri
     # @param Array
     #
     # TODO: Revisit this to make it more generalizable
-    def self.format_key(keys)
+    def format_key(keys)
       keys.join('_')
         .gsub('@', '')
         .gsub('http://www.geonames.org/ontology#', '')
         .gsub('http://www.w3.org/2003/01/geo/wgs84_pos#', '')
     end
 
-    private_class_method(:solr_doc, :remove_invalid_keys, :flat_hash,
-                         :format_key)
+    ##
+    # Remove keys (ie. fields) that are not in the Solr schema.
+    # @param [Hash]
+    # @return [Hash]
+    def remove_invalid_keys(solr_doc)
+      valid_keys = schema_keys
+      solr_doc.delete_if { |key, _| !key.in? valid_keys }
+    end
   end
 end

data/lib/krikri/ldp/container.rb~

@@ -0,0 +1,22 @@
+module Krikri::LDP
+  class Container < ActiveTriples::Resource
+    include Krikri::LDP::Resource 
+
+    configure :type => RDF::URI('http://www.w3.org/ns/ldp#Container')
+    
+    def initialize(uri, slug)
+      @ldp_uri = uri
+      @container_slug = slug
+      super("#{uri}/#{slug}")
+    end
+
+    def save(body = nil, opts = {})
+      opts[:Link] = "<#{type.first.to_s}>; rel='type'"
+      opts[:Slug] = @container_slug
+      opts[:method] = :post unless exists?
+      body ||= (dump :ttl)
+      opts['Content-Type'] ||= 'text/turtle'
+      super(body, opts)
+    end
+  end
+end