krikri 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85a1cc695e659e9b4ccc25b8864d4a767bda8327
-  data.tar.gz: cd1d7d097a711712357d70f107f7d7019b034da0
+  metadata.gz: 5ba0502820a606ae6570cb7e683f35e4e7802536
+  data.tar.gz: 43cfd2802c166d2953038007c02d76146a72b5bb
 SHA512:
-  metadata.gz: 72f31b3ffb3bae41116d2359b8deef556884ac362ca204f4259444c46bfed27dd50f41b336f7fa4dee8abb197c9c18600d3e4cca3a7bd2ca6ed5fa55361d6984
-  data.tar.gz: 58ceef4e5c312d4417838d412327cd38ed2b4f85d67b867b3f2e2b284c391539fa969b687d6fda4df5b7178822a5210946ae4bdb4c1e6892cf363b42f04e2238
+  metadata.gz: eea171ecd705736afdeb184d612e21ebf46e6f39336a7c6d0f10d80cc7d80d44d9a9a9aa15b03483e98f1ee6493cc2dc5eff817ed1d889fb4d5747e29b21a732
+  data.tar.gz: 7f81bfc7389b026eb30fb6c784bc30f1a5c1933c6f975bd45e71b866defa403fa847741900da0207779c1164b54c8bbfbce6a768ac5011a9648fdd81311344b0

data/app/models/krikri/harvesters/oai_harvester.rb~

@@ -0,0 +1,10 @@
+module Krikri::Harvesters
+  class OAIHarvester < Krikri::Harvester
+
+    attr_accessor :client
+
+    def initialize(opts={})
+      @client = OAI::Client.new(opts[:endpoint])
+    end
+  end
+end

data/app/models/krikri/original_record_desc.rb~

@@ -0,0 +1,6 @@
+module Krikri
+  ##
+  # Descriptive Metadata for OriginalRecord
+  class OriginalRecordMetadata < ActiveTriples::Resource
+  end
+end

data/app/models/krikri/original_record_metadata.rb~

@@ -0,0 +1,9 @@
+module Krikri
+  ##
+  # Descriptive Metadata for OriginalRecord
+  class OriginalRecordMetadata < ActiveTriples::Resource
+    include Krikri::Ldp::RdfSource
+
+    property :wasGeneratedBy, predicate: RDF::PROV.wasGeneratedBy
+  end
+end

data/app/models/krikri/parser.rb~

@@ -0,0 +1,5 @@
+module Krikri
+  class Parser
+    
+  end
+end

data/app/models/krikri/parsers/xml_parser.rb~

@@ -0,0 +1,5 @@
+module Krikri::Parsers
+
+  class XmlParser
+  end
+end

data/config/initializers/config.rb~

@@ -0,0 +1,19 @@
+setting_files = [self.root.join('config', 'settings.yml'),
+                 self.root.join('config', 'settings', "#{Rails.env}.yml"),
+                 self.root.join('config', 'environments', "#{Rails.env}.yml"),
+                 self.root.join('config', 'settings.local.yml'),
+                 self.root.join('config', 'settings', "#{Rails.env}.local.yml"),
+                 self.root.join('config', 'environments', 
+                                "#{Rails.env}.local.yml")
+                ].map { |path| path.to_s }
+
+settings_const = Kernel.const_get(RailsConfig.const_name)
+
+source_paths = settings_const.add_source!('nil')[0..-2].map do |source| 
+  source.path 
+end
+
+source_paths = setting_files + source_paths
+
+RailsConfig.load_and_set_settings(source_paths)
+Krikri::Settings = Kernel.const_get(RailsConfig.const_name)

data/config/settings/test.yml~

@@ -0,0 +1,3 @@
+---
+marmotta:
+  base: 'http://localhost:8080/marmotta/'

data/lib/aggregation.rb~

@@ -0,0 +1,4 @@
+class DPLA::MAP::Aggregation
+  include Krikri::LDP::RdfSource
+  configure :base_uri => Krikri::Settings['marmotta']['item_container']
+end

data/lib/blacklight.rb~

@@ -0,0 +1,8 @@
+module Blacklight
+  def self.connection_config
+    @connection_config ||= begin
+      raise "The #{::Rails.env} environment settings were not found in the blacklight.yml config" unless blacklight_yml[::Rails.env]
+      blacklight_yml[::Rails.env].symbolize_keys
+    end
+  end
+end

data/lib/krikri/engine.rb

@@ -15,6 +15,12 @@ module Krikri
   class Engine < ::Rails::Engine
     isolate_namespace Krikri
 
+    def configure_blacklight!
+      krikri_solr = Krikri::Settings.solr
+      Blacklight.solr_config = Blacklight.solr_config.merge(krikri_solr) unless
+        krikri_solr.nil?
+    end
+
     # Autoload various classes and modules in lib
     config.autoload_paths += Dir["#{config.root}/lib/**/"]
 
@@ -65,6 +71,10 @@ module Krikri
         RDF::Marmotta.new(Krikri::Settings['marmotta']['base'])
     end
 
+    initializer :blacklight_settings do
+      configure_blacklight!
+    end
+
     initializer :aggregation do
       DPLA::MAP::Aggregation.class_eval do
         include Krikri::LDP::RdfSource
@@ -72,8 +82,25 @@ module Krikri
 
         def mint_id!(seed = nil)
           set_subject!(mint_id_fragment(seed))
-          sourceResource.first.set_subject!(rdf_subject / '#sourceResource') \
-            unless sourceResource.empty?
+          update_source_resource_subject
+        end
+
+        ##
+        # Forceably update the subject for the dpla:SourceResource to use a
+        # fragment URI. This is necessary because of an issue in
+        # ActiveTriples.
+        #
+        # @see https://github.com/ActiveTriples/ActiveTriples/issues/107
+        def update_source_resource_subject
+          unless sourceResource.empty?
+            sr = get_values(RDF::EDM::aggregatedCHO).first
+            orig = sourceResource_ids.first
+            # Q: should we have a check like `orig.node?` here?
+            sr.set_subject!(rdf_subject / '#sourceResource')
+            sr.persist!
+            update([self, RDF::EDM::aggregatedCHO, sr])
+            delete([orig, nil, nil])
+          end
         end
 
         ##

data/lib/krikri/enrichment.rb~

@@ -0,0 +1,5 @@
+module Krikri
+  class Enrichment
+    extend SoftwareAgent
+  end
+end

data/lib/krikri/enrichment_agent.rb~

@@ -0,0 +1,11 @@
+module Krikri
+  ##
+  # EnrichmentAgent
+  class EnrichmentAgent
+    include SoftwareAgent
+
+    def initialize(opts = {})
+      @generator_uri = RDF::URI(opts.fetch(:generator_uri))
+    end
+  end
+end

data/lib/krikri/enrichment_agent_spec.rb~

@@ -0,0 +1,2 @@
+describe Krikri::Enrichment::Agent do
+  it_behaves_like 'a software agent', opts

data/lib/krikri/enrichment_job.rb~

@@ -0,0 +1,13 @@
+module Krikri
+  ##
+  # MappingJob: perform enqueued Krikri::Mapper::Agent
+  #
+  # @see: Krikri::Mapper, Krikri::Mapper::Agent
+  class EnrichmentJob < Krikri::Job
+    @queue = :enrichment
+
+    def self.run(mapper, activity_uri = nil)
+      mapper.run(activity_uri)
+    end
+  end
+end

data/lib/krikri/enrichments.rb~

@@ -0,0 +1,4 @@
+module Krikri
+  module Enrichments
+  end
+end

data/lib/krikri/enrichments/capitalize.rb~

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+module Krikri::Enrichments
+  ##
+  # Enrichment to capitalize sentences in a text field
+  #
+  #   Capitalize.new.enrich_value(
+  #     'THE MOOMIN BOOKS MAKE FOR BOTH SPLENDID BEDTIME READ-ALOUDS AND ' \
+  #     "SOLITARY SAVORING. IT’S MORE THAN FORTY YEARS SINCE JANSSON’S " \
+  #     'MOOMINTROLLS FIRST APPEARED. I FOUND THE WRITING AND INVENTION ' \
+  #     'AS APPEALING AS EVER. SHE HAS A THISTLEDOWN TOUCH.'
+  #   => 'The moomin books make for both splendid bedtime read-alouds and ' \
+  #     "solitary savoring. It’s more than forty years since jansson’s " \
+  #     'moomintrolls first appeared. I found the writing and invention ' \
+  #     'as appealing as ever. She has a thistledown touch.'
+  #
+  class Capitalize
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.respond_to? :capitalize
+      value.captialize
+    end
+  end
+end

data/lib/krikri/enrichments/deduplication.rb~

@@ -0,0 +1,10 @@
+module Krikri::Enrichments
+  ##
+  # Enrichment to remove duplicate values
+  #
+  #
+  class Deduplication
+    include Krikri::FieldEnrichment
+
+  end
+end

data/lib/krikri/enrichments/genre_filter.rb~

@@ -0,0 +1,9 @@
+module Krikri::Enrichments
+  ##
+  # Enrichment to remove non-genre fields from
+  #
+  #   StripHtml.new.enrich_value('Book') => 'Book'
+  #   StripHtml.new.enrich_value('book') => 'book'
+  #   StripHtml.new.enrich_value('not a book') => nil
+  #
+  class StripHtml

data/lib/krikri/enrichments/iso_enrcich.rb~

@@ -0,0 +1,8 @@
+module Krikri::Enrichments
+  ##
+  # Converts text fields to ISO 689-3 URIs (Lexvo)
+  #
+  class StripHtml
+    include Krikri::FieldEnrichment
+  end
+end

data/lib/krikri/enrichments/iso_enrich.rb~

@@ -0,0 +1,13 @@
+module Krikri::Enrichments
+  ##
+  # Converts text fields to ISO 689-3 URIs (Lexvo)
+  #
+  class IsoEnrich
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      require 'pry'
+      binding.pry
+    end
+  end
+end

data/lib/krikri/enrichments/parse_date.rb~

@@ -0,0 +1,10 @@
+module Krikri::Enrichments
+  ##
+  # Normalizes date strings to edm:TimeSpans
+  #
+  #   StripPunctuation.new.enrich_value("\tmo!ominpa)(pa  \n .$%^ moominmama  ")
+  #   # => "\tmoominpapa  \n  moominmama  "
+  class ParseDate
+
+  end
+end

data/lib/krikri/enrichments/strip_html.rb~

@@ -0,0 +1,16 @@
+module Krikri::Enrichments
+  ##
+  # Enrichment to capitalize sentences in a text field
+  #
+  #   StripHtml.new.enrich_value('<html>Moomin <i><b>Valley</i></b>')
+  #   => 'Moomin Valley'
+  #
+  class StripHtml
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      return value unless value.respond_to? :capitalize
+      value.captialize
+    end
+  end
+end

data/lib/krikri/enrichments/timespan.rb~

@@ -0,0 +1,6 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::Timespan do
+  it_behaves_like 'a field enrichment'
+
+end

data/lib/krikri/enrichments/timespan_split.rb~

@@ -0,0 +1,83 @@
+module Krikri::Enrichments
+  ##
+  # Splits edm:TimeSpan labels and populates `#begin` and `#end`. Any values
+  # generated from string values are left as strings to be normalized by
+  # another enrichment.
+  #
+  # Ignores values that are neither a String or an edm:TimeSpan
+  #
+  # Converts string values to a timespan with the value as providedLabel
+  # and enriches as below.
+  #
+  # Populates the `#providedLabel`, `#begin`, and `#end` properties of TimeSpan
+  # objects, and:
+  #   - If a `#providedLabel` is present, but begin and end are missing:
+  #     - Attempts to parse `#providedLabel` as an EDTF interval and populates
+  #       begin and end with their respective values.
+  #     - Splits dates
+  #   - If no
+  #   - Returns the TimeSpan unaltered if all three fields are present.
+  class TimespanSplit
+    include Krikri::FieldEnrichment
+
+    def enrich_value(value)
+      value = timespan_from_string(value) if value.is_a? String
+      return value unless value.is_a? DPLA::MAP::TimeSpan
+      populate_timespan(value)
+    end
+
+    def timespan_from_string(value)
+      timespan = DPLA::MAP::TimeSpan.new
+      timespan.providedLabel = value
+      timespan
+    end
+
+    def populate_timespan(timespan)
+      return timespan unless timespan.providedLabel.empty? ||
+        timespan.begin.empty? || timespan.end.empty?
+      parse_label(timespan.providedLabel)
+
+      return timespan
+    end
+
+    ##
+    # e.g. 1970-08-01/02 or 1970-12/10
+    def partial_edtf(str)
+      /^(\d{4}(-\d{2})*)-(\d{2})\/(\d{2})$/.match(str) do |m|
+        Date.edtf("#{m[1]}-#{m[3]}/#{m[1]}-#{m[4]}")
+      end
+    end
+
+    ##
+    # e.g. 01-2045
+    def month_year(str)
+      /\d{2}-(\d{4})/.match(str) do |m|
+        Date.edtf("#{m[2]}-#{m[1]}")
+      end
+    end
+
+    ##
+    # e.g. 1990-92
+    def hyphenated_partial_range(str)
+      /^(\d{2})(\d{2})-(\d{2})$/.match(str) do |m|
+        Date.edtf("#{m[1]}#{m[2]}/#{m[1]}#{m[3]}")
+      end
+    end
+
+    ##
+    # e.g. 1990s
+    def decade_s(str)
+      /^(\d{3})0s)$/.match(str) do |m|
+        Date.edtf("#{m}x")
+      end
+    end
+
+    ##
+    # e.g. 199-
+    def decade_hyphen
+      /^(\d{3})-)$/.match(str) do |m|
+        Date.edtf("#{m}x")
+      end
+    end
+  end
+end

data/lib/krikri/field_enrichment.rb~

@@ -0,0 +1,3 @@
+module Krikri
+  module FieldEnrichment
+    extend SoftwareAgent

data/lib/krikri/harvester.rb

@@ -22,14 +22,14 @@ module Krikri
   #      recommended to override this method in your subclass with
   #      an efficient implementation.
   #    - #run. Wraps persistence of each record returned by #records.
-  #      Runs a full harvest, inserting Original Records into the database,
-  #      given the options passed to #initialize.
+  #      Runs a full harvest, processing the created `record_class`
+  #      instances through `harvest_behavior`, given the options passed
+  #      to #initialize.
   #
   # When including, add a call to Krikri::Harvester::Registry.register() to
   # put it in the registry so that it can be looked up.
-  # See lib/krikri/engine.rb.
-  # @see Krikri::Engine
   #
+  # @see Krikri::Engine
   module Harvester
     extend ActiveSupport::Concern
     include SoftwareAgent
@@ -55,6 +55,7 @@ module Krikri
     #                 Krikri::OriginalRecord).
     #   id_minter: Module to create identifiers for generated records (optional;
     #              defaults to Krikri::Md5Minter)
+    #   harvest_behavior: A behavior object implementing `#process_record`
     #
     # Pass harvester specific options to inheriting classes under a key for
     # that harvester. E.g. { uri: my_uri, oai: { metadata_prefix: :oai_dc } }
@@ -64,9 +65,16 @@ module Krikri
       @uri = opts.fetch(:uri)
       @name = opts.delete(:name)
       @record_class = opts.delete(:record_class) { Krikri::OriginalRecord }
+                      .to_s.constantize
       @id_minter = opts.delete(:id_minter) { Krikri::Md5Minter }
+                   .to_s.constantize
+      @harvest_behavior = opts.delete(:harvest_behavior) do
+        Krikri::Harvesters::BasicSaveBehavior
+      end.to_s.constantize
     end
 
+    delegate :process_record, :to => :@harvest_behavior
+
     ##
     # @abstract Provide a low-memory, lazy enumerable for record ids.
     #
@@ -107,16 +115,20 @@ module Krikri
 
     ##
     # Run the harvest.
-    # This should be idempotent so it can be safely retried on errors.
+    #
+    # Individual records are processed through `#process_record` which is
+    # delegated to the harvester's `@harvest_behavior` by default.
     #
     # @return [Boolean]
+    # @see Krirki::Harvesters:HarvestBehavior
     def run(activity_uri = nil)
       log :info, 'harvest is running'
       records.each do |rec|
         begin
-          rec.save(activity_uri)
+          process_record(rec, activity_uri)
         rescue => e
-          log :error, "Error harvesting record:\n#{rec.content}\n\twith message:\n#{e.message}"
+          log :error, "Error harvesting record:\n#{rec.content}\n\t" \
+                      "with message:\n#{e.message}"
           next
         end
       end