krikri 0.4.0 → 0.5.0

data/lib/krikri/search_index.rb

@@ -0,0 +1,326 @@
+require 'rubygems'
+require 'rsolr'
+require 'elasticsearch'
+
+module Krikri
+  ##
+  # Search index base class that gets extended by QA and Production index
+  # classes
+  class SearchIndex
+    def initialize(opts)
+      @bulk_update_size = opts.delete(:bulk_update_size) { 10 }
+    end
+
+    ##
+    # Add a single JSON document to the search index.
+    # Implemented in a child class.
+    #
+    # @param _ [Hash] Hash that can be serialized to JSON with #to_json
+    def add(_)
+      fail NotImplementedError
+    end
+
+    ##
+    # Add a number of JSON documents to the search index at once.
+    # Implemented in a child class.
+    #
+    # @param _ [Array] Hashes that can be serialized to JSON with #to_json
+    def bulk_add(_)
+      fail NotImplementedError
+    end
+
+    ##
+    # Shim that determines, for a particular type of index, which strategy
+    # to use, adding a single document, or adding them in bulk.  Intended
+    # to be overridden as necessary.
+    # @see #add
+    # @see #bulk_add
+    def update_from_activity(activity)
+      fail "#{activity} is not an Activity" \
+        unless activity.class == Krikri::Activity
+      incremental_update_from_activity(activity)
+    end
+
+    protected
+
+    ##
+    # Given an activity, use the bulk-update method to load its revised
+    # entities into the search index.
+    #
+    # @param activity [Krikri::Activity]
+    def bulk_update_from_activity(activity)
+      all_aggs = entities_as_json_hashes(activity)
+      agg_batches = bulk_update_batches(all_aggs)
+      agg_batches.each do |batch|
+        bulk_add(batch)
+      end
+    end
+
+    ##
+    # Enumerate arrays of JSON strings, one array per batch that is supposed
+    # to be loaded into the search index.
+    #
+    # @param aggregations [Enumerator]
+    # @return [Enumerator] Each array of JSON strings
+    def bulk_update_batches(aggregations)
+      en = Enumerator.new do |e|
+        i = 1
+        batch = []
+        aggregations.each do |agg|
+          batch << agg
+          if i % @bulk_update_size == 0
+            e.yield batch
+            batch = []
+          end
+          i += 1
+        end
+        e.yield batch if batch.count > 0  # last one
+      end
+      en.lazy
+    end
+
+    ##
+    # Given an activity, load its revised entities into the search index one
+    # at a time.
+    #
+    # @param activity [Krikri::Activity]
+    def incremental_update_from_activity(activity)
+      entities_as_json_hashes(activity).each do |h|
+        add(h)
+      end
+    end
+
+    ##
+    # Given an activity, enumerate over revised entities, represented as
+    # hashes that can be serialized to JSON.
+    #
+    # @param activity [Krikri::Activity]
+    # @return [Enumerator]
+    def entities_as_json_hashes(activity)
+      activity.entities.lazy.map do |agg|
+        hash_for_index_schema(agg)
+      end
+    end
+
+    ##
+    # Return a JSON string from the given aggregation in a format suitable for
+    # the search index.
+    #
+    # The default behavior is to turn out the MAPv4 JSON-LD straight from
+    # the aggregation.
+    #
+    # This can be overridden to convert this to MAPv3 JSON-LD or whatever.
+    #
+    # @param aggregation [DPLA::MAP::Aggregation] The aggregation
+    # @return [Hash] Hash that can respond to #to_json for serialization
+    def hash_for_index_schema(aggregation)
+      aggregation.to_jsonld['@graph'][0]
+    end
+  end
+
+
+  ##
+  # Generates flattened Solr documents and manages indexing of DPLA MAP models.
+  #
+  # @example
+  #
+  #   indexer = Krikri::QASearchIndex.new
+  #   agg = Krikri::Aggregation.new
+  #   doc = agg.to_jsonld['@graph'].first
+  #
+  #   indexer.add(doc)
+  #   indexer.commit
+  #
+  class QASearchIndex < Krikri::SearchIndex
+    attr_reader :solr
+
+    ##
+    # @param opts [Hash] options to pass to RSolr
+    # @see RSolr.connect
+    def initialize(opts = Krikri::Settings.solr.to_h)
+      @solr = RSolr.connect(opts)
+      super(opts)
+    end
+
+    # TODO: Assure that the following metacharacters are escaped:
+    # + - && || ! ( ) { } [ ] ^ " ~ * ? : \
+
+    ##
+    # Adds a single JSON document to Solr
+    # @param doc [Hash]  A hash that complies with the Solr schema
+    def add(doc)
+      solr.add solr_doc(doc)
+    end
+
+    ##
+    # @see Krikri::SearchIndex#update_from_activity
+    def update_from_activity(activity)
+      fail "#{activity} is not an Activity" \
+        unless activity.class == Krikri::Activity
+      bulk_update_from_activity(activity)
+    end
+
+    ##
+    # Add multiple documents to Solr
+    # @param docs [Array]  Array of hashes that comply with the Solr schema
+    def bulk_add(docs)
+      solr.add(docs.map { |d| solr_doc(d) })
+    end
+
+    ##
+    # Deletes an item from Solr
+    # @param String or Array
+    def delete_by_id(id)
+      solr.delete_by_id id
+    end
+
+    ##
+    # Deletes items from Solr that match query
+    # @param String or Array
+    def delete_by_query(query)
+      solr.delete_by_query query
+    end
+
+    ##
+    # Commits changes to Solr, making them visible to new requests
+    # Should be run after self.add and self.delete
+    # Okay to add or delete multiple docs and commit them all with
+    # a single self.commit
+    def commit
+      solr.commit
+    end
+
+    ##
+    # Converts JSON document into a Hash that complies with Solr schema
+    # @param [JSON]
+    # @return [Hash]
+    def solr_doc(doc)
+      remove_invalid_keys(flat_hash(doc))
+    end
+
+    ##
+    # Get field names from Solr schema in host application.
+    # Will raise exception if file not found.
+    # @return [Array]
+    def schema_keys
+      schema_file = File.join(Rails.root, 'solr_conf', 'schema.xml')
+      file = File.open(schema_file)
+      doc = Nokogiri::XML(file)
+      file.close
+      doc.xpath('//fields/field').map { |f| f.attr('name') }
+    end
+
+    private
+
+    ##
+    # Flattens a nested hash
+    # Joins keys with "_" and removes "@" symbols
+    # Example:
+    #   flat_hash( {"a"=>"1", "b"=>{"c"=>"2", "d"=>"3"} )
+    #   => {"a"=>"1", "b_c"=>"2", "b_d"=>"3"}
+    def flat_hash(hash, keys = [])
+      new_hash = {}
+
+      hash.each do |key, val|
+        new_hash[format_key(keys + [key])] = val unless
+          val.is_a?(Array) || val.is_a?(Hash)
+        new_hash.merge!(flat_hash(val, keys + [key])) if val.is_a? Hash
+
+        if val.is_a? Array
+          val.each do |v|
+            if v.is_a? Hash
+              new_hash.merge!(flat_hash(v, keys + [key])) do |key, f, s|
+                Array(f) << s
+              end
+            else
+              formatted_key = format_key(keys + [key])
+              new_hash[formatted_key] =
+                new_hash[formatted_key] ? (Array(new_hash[formatted_key]) << v) : v
+            end
+          end
+        end
+      end
+
+      new_hash
+    end
+
+    ##
+    # Formats a key to match a field name in the Solr schema
+    #
+    # Removes unnecessary special character strings that would
+    # require special treatment in Solr
+    #
+    # @param Array
+    #
+    # TODO: Revisit this to make it more generalizable
+    def format_key(keys)
+      keys.join('_')
+        .gsub('@', '')
+        .gsub('http://www.geonames.org/ontology#', '')
+        .gsub('http://www.w3.org/2003/01/geo/wgs84_pos#', '')
+    end
+
+    ##
+    # Remove keys (ie. fields) that are not in the Solr schema.
+    # @param [Hash]
+    # @return [Hash]
+    def remove_invalid_keys(solr_doc)
+      valid_keys = schema_keys
+      solr_doc.delete_if { |key, _| !key.in? valid_keys }
+    end
+  end
+
+
+  ##
+  # Production ElasticSearch search index class
+  class ProdSearchIndex < Krikri::SearchIndex
+    attr_reader :elasticsearch, :index_name
+
+    ##
+    # @param [Hash] opts
+    #
+    # Options used by this class:
+    #   - index_name [String]  The name of the ElasticSearch index
+    # Other options are passed along to Elasticsearch::Client.
+    #
+    def initialize(opts = Krikri::Settings.elasticsearch.to_h)
+      super(opts)
+      @index_name = opts.delete(:index_name) { 'dpla_alias' }
+      @elasticsearch = Elasticsearch::Client.new(opts)
+    end
+
+    ##
+    # Add a number of JSON documents to the search index at once.
+    # @param docs [Array] Array of hashes that can be serialized with #to_json
+    def bulk_add(docs)
+      body = docs.map do |doc|
+        {
+          index: {
+            _index: @index_name,
+            _type: doc[:ingestType],
+            _id:  doc[:id],
+            data: doc
+          }
+        }
+      end
+      @elasticsearch.bulk body: body
+    end
+
+    ##
+    # @see Krikri::SearchIndex#update_from_activity
+    def update_from_activity(activity)
+      fail "#{activity} is not an Activity" \
+        unless activity.class == Krikri::Activity
+      bulk_update_from_activity(activity)
+    end
+
+    ##
+    # @see Krikri::SearchIndex#hash_for_index_schema
+    def hash_for_index_schema(aggregation)
+      graph = aggregation.to_jsonld['@graph'][0]
+      # TODO: munge graph to MAPv3
+      #       update search_index_spec.rb when that's done.
+      graph
+    end
+  end
+end

data/lib/krikri/search_results_helper_behavior.rb

@@ -10,31 +10,8 @@ module Krikri
 
     # Override method in Blacklight::UrlHelperBehavior.
     def link_to_document(document, field_or_opts = nil, opts = {})
-      link_to field_or_opts, url_for_document(document)
-    end
-
-    # This method is used to make display fields into hyperlinks.
-    def make_this_a_link(options = {})
-      # options[:document] # the original document
-      # options[:field] # the field to render
-      # options[:value] # the value of the field
-
-      link_to options[:value], options[:value]
-    end
-
-    # This method is used to link display fields to the document's show path.
-    def link_to_show(options = {})
-      # options[:document] # the original document
-      # options[:field] # the field to render
-      # options[:value] # the value of the field
-
-      link_to_document options[:document], options[:value]
-    end
-
-    def render_document_field_data(document, key)
-      if document[key]
-        render_index_field_value document, :field => key
-      end
+      link_to field_or_opts, url_for_document(document, { :provider =>
+        params[:provider] })
     end
 
     # Disable bookmarks.
@@ -97,9 +74,5 @@ module Krikri
     def error_msg(message = '')
       "There was a problem getting the record.\n\n#{message}"
     end
-
-    def random_record_url
-      url_for_document(Krikri::RandomRecordGenerator.new.record)
-    end
   end
 end

data/lib/krikri/software_agent.rb

@@ -6,8 +6,23 @@ module Krikri
   module SoftwareAgent
     extend ActiveSupport::Concern
 
+    included do
+      attr_writer :entity_behavior
+    end
+
     Logger = ActiveSupport::TaggedLogging.new(Rails.logger)
 
+    ##
+    # Return the EntityBehavior associated with the SoftwareAgent.
+    # Meant to be overridden as necessary.
+    #
+    # @see Krikri::Activity#entities
+    # @see Krikri::EntityBehavior
+    #
+    def entity_behavior
+      @entity_behavior ||= nil
+    end
+
     ##
     # Return an agent name suitable for saving in an Activity.
     # This is the name of the most-derived class upon which this is invoked.

data/lib/krikri/solr_response_builder.rb

@@ -0,0 +1,14 @@
+module Krikri
+  class SolrResponseBuilder
+    attr_accessor :response
+    ##
+    # @param query_terms [Hash] of terms for a Solr query
+    # Sample use:
+    #   SolrResponseBuilder.new({ :q => 'dogs' }).response
+    #   @return Blacklight::SolrResponse
+    def initialize(query_params)
+      self.response = Blacklight::SolrResponse.new(Blacklight::SolrRepository
+        .new(Blacklight::Configuration.new).search(query_params), query_params)
+    end
+  end
+end

data/lib/krikri/version.rb

@@ -1,3 +1,3 @@
 module Krikri
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/krikri.rb

@@ -13,4 +13,9 @@ module Krikri
   autoload :ModsParser,       'krikri/parsers/mods_parser'
   autoload :QdcParser,        'krikri/parsers/qdc_parser'
   autoload :OaiParserHeaders, 'krikri/parsers/oai_parser_headers'
+  autoload :AggregationEntityBehavior,
+           'krikri/entity_behaviors/aggregation_entity_behavior'
+  autoload :OriginalRecordEntityBehavior,
+           'krikri/entity_behaviors/original_record_entity_behavior'
+
 end

data/lib/tasks/krikri.rake

@@ -4,7 +4,7 @@ require 'dpla/map/factories'
 require 'open-uri'
 require 'resque/tasks'
 
-require 'krikri/index_service'
+require 'krikri/search_index'
 
 krikri_dir = Gem::Specification.find_by_name('krikri').gem_dir
 require "#{krikri_dir}/app/models/krikri/original_record"
@@ -13,9 +13,8 @@ require "#{krikri_dir}/spec/factories/krikri_original_record"
 namespace :krikri do
 
   def index_aggregation(agg)
-    graph = agg.to_jsonld['@graph'].first
-    indexer = Krikri::IndexService.new
-    indexer.add graph.to_json
+    indexer = Krikri::QASearchIndex.new
+    indexer.add agg.to_jsonld['@graph'].first
     indexer.commit
   end
 
@@ -28,12 +27,12 @@ namespace :krikri do
       original_record = build(:oai_dc_record)
       original_record.save unless original_record.exists?
 
-      # Make a DPLA::MAP::Aggregation with an original record instantiated as an
-      # ActiveTriples::Resource object.
-      agg = build(:aggregation,
-                  :originalRecord => ActiveTriples::Resource
-                                  .new(original_record.rdf_subject)
-            )
+      provider = Krikri::Provider.new('123')
+      provider.label = 'Moomin valley Historical Society'
+
+      agg = build(:aggregation)
+      agg.originalRecord = original_record.rdf_source
+      agg.provider = provider
 
       agg.mint_id!('krikri_sample')
 
@@ -81,7 +80,7 @@ namespace :krikri do
       original_record.delete! if original_record.exists?
 
       # Delete all sample records from Solr
-      indexer = Krikri::IndexService.new
+      indexer = Krikri::QASearchIndex.new
       indexer.delete_by_query 'id:*krikri_sample*'
       indexer.commit
     end

data/spec/controllers/application_controller_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Krikri::ApplicationController, :type => :controller do
+
+end

data/spec/controllers/providers_controller_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe Krikri::ProvidersController, :type => :controller do
+
+  routes { Krikri::Engine.routes }
+
+  describe 'GET #show' do
+    login_user
+
+    it 'sets provider variable' do
+      expect { get :show, id: 'moomin' }
+        .to change { assigns[:current_provider] }
+             .to an_instance_of(Krikri::Provider)
+    end
+
+    it 'renders the :show view' do
+      get :show, id: 'moomin'
+      expect(response).to render_template('krikri/providers/show')
+    end
+  end
+
+  describe 'GET #index' do
+    login_user
+
+    it 'renders the :show view' do
+      get :index
+      expect(response).to render_template('krikri/providers/index')
+    end
+
+    it 'sets providers variable' do
+      providers = [instance_double(Krikri::Provider)]
+      allow(Krikri::Provider).to receive(:all).and_return(providers)
+
+      expect { get :index }.to change { assigns[:providers] }.to(providers)
+    end
+  end
+end

data/spec/controllers/qa_reports_controller_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+
+describe Krikri::QaReportsController, :type => :controller do
+  routes { Krikri::Engine.routes }
+
+  let(:report) { instance_double(Krikri::QAReport) }
+
+  describe 'GET #index' do
+    login_user
+
+    let(:reports) { [report] }
+
+    before { allow(Krikri::QAReport).to receive(:all).and_return(reports) }
+
+    it 'renders the index view' do
+      get :index
+      expect(response).to render_template('krikri/qa_reports/index')
+    end
+
+    it 'populates the reports list' do
+      get :index
+      expect(assigns(:reports)).to eq(reports)
+    end
+  end
+
+  describe 'GET #show' do
+    login_user
+
+    before do
+      allow(Krikri::QAReport)
+        .to receive(:find).with(report_id.to_s).and_return(report)
+      allow(report).to receive(:field_csv).and_return(csv)
+    end
+
+    let(:report_id) { 123 }
+
+    let(:csv) do
+      CSV::Table.new([CSV::Row.new(['field_name', 'aggregation', 'isShownAt'],
+                                   ['moomin', '123', 'http://example.org/123'])])
+    end
+
+    let(:count_csv) do
+      CSV::Table.new([CSV::Row.new(['name', 'count'], ['moomin', '1'])])
+    end
+
+    it 'accepts a type ' do
+      get :show, id: report_id, type: 'count'
+      expect(assigns(:type)).to eq :count
+    end
+
+    context 'CSV' do
+      it 'gives CSV result' do
+        get :show, id: report_id, format: 'csv'
+        expect(response.content_type).to eq 'text/csv'
+      end
+
+      it 'gives correct text in CSV result' do
+        get :show, id: report_id, format: 'csv'
+        expect(response.body).to eq csv.to_s
+      end
+
+      it 'gives count text in CSV result' do
+        allow(report).to receive(:count_csv).and_return(count_csv)
+        get :show, id: report_id, type: 'count', format: 'csv'
+        expect(response.body).to eq count_csv.to_s
+      end
+    end
+  end
+end

data/spec/controllers/records_controller_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+
+describe Krikri::RecordsController, :type => :controller do
+  include_context 'with indexed item'
+
+  routes { Krikri::Engine.routes }
+
+  let(:doc_id) { agg.rdf_subject.to_s.split('/').last }
+
+  describe 'GET #index' do
+    login_user
+
+    let(:user_query) { 'history' }
+
+    it 'renders the index view' do
+      get :index, q: user_query
+      expect(response).to render_template('krikri/records/index')
+    end
+
+    it 'gives a solr response' do
+      get :index, q: user_query
+      expect(assigns[:response]).to be_a Blacklight::SolrResponse
+    end
+
+    it 'responds to json requests' do
+      get :index, q: user_query, format: 'json'
+      expect { JSON.parse(response.body) }.not_to raise_error
+    end
+
+    context 'with provider filter' do
+      it 'sets provider id' do
+        get :index, provider: provider.id
+        expect(assigns[:provider_id]).to eq provider.id
+      end
+
+      it 'finds records with provider' do
+        get :index, provider: provider.id
+        ids = assigns[:response][:response][:docs].map { |doc| doc[:id] }
+        expect(ids).to include agg.rdf_subject.to_s
+      end
+
+      it 'filters records by provider' do
+        get :index, provider: 'abc'
+        expect(assigns[:response][:response][:docs]).to be_empty
+      end
+    end
+  end
+
+  describe 'GET #show' do
+    login_user
+
+    it 'renders show view' do
+      get :show, id: doc_id
+      require 'pry'
+      binding.pry unless assigns[:document]
+
+      expect(response).to render_template('krikri/records/show')
+    end
+
+    it 'gets document by local name' do
+      get :show, id: doc_id
+
+      expect(assigns[:document][:id]).to eq agg.rdf_subject.to_s
+    end
+
+    context 'with provider filter' do
+      it 'sets provider id' do
+        get :show, id: doc_id, provider: provider.id
+        expect(assigns[:provider_id]).to eq provider.id
+      end
+    end
+  end
+end