spotlight-oaipmh-resources 0.3.6

data/app/services/spotlight/resources/solr_harvesting_builder.rb

@@ -0,0 +1,115 @@
+module Spotlight
+  module Resources
+    # transforms a OaipmhHarvester into solr documents
+    class SolrHarvestingBuilder < Spotlight::SolrDocumentBuilder
+      
+      def to_solr
+        begin
+        return to_enum(:to_solr) { 0 } unless block_given?
+
+        base_doc = super
+           
+        mapping_file = nil
+        if (!resource.data[:solr_mapping_file].eql?("Default Mapping File") && !resource.data[:solr_mapping_file].eql?("New Mapping File"))
+          mapping_file = resource.data[:mapping_file]
+        end
+
+        max_batch_count = Spotlight::Oaipmh::Resources::Engine.config.solr_harvest_batch_max
+        
+        @solr_converter = SolrConverter.new(resource.data[:set], resource.exhibit.slug, mapping_file)
+        @solr_converter.parse_mapping_file(@solr_converter.mapping_file) 
+
+        unique_id_field = nil 
+        if (!@solr_converter.get_unique_id_field.nil?)
+          unique_id_field = @solr_converter.get_unique_id_field
+        end
+                 
+        count = 0
+        totalrecords = 0
+        
+        #If the resumption token was stored, begin there.
+        if (resource.data.include?(:cursor) && !resource.data[:cursor].blank?)
+          page = resource.data[:cursor]
+          harvests = resource.paginate(page)
+        else
+          page = 1
+          harvests = resource.harvests
+        end
+
+        if (resource.data.include?(:count) && !resource.data[:count].blank?)
+          totalrecords = resource.data[:count]
+        end
+
+        last_page_evaluated = harvests['response']['docs'].blank?
+
+        while (!last_page_evaluated)
+          harvests['response']['docs'].each do |record|
+            @item = SolrHarvestingItem.new(exhibit, @solr_converter)
+            
+            @item.metadata = record
+            @item.parse_record(unique_id_field)
+            begin
+              @item_solr = @item.to_solr
+              @item_sidecar = @item.sidecar_data
+              
+              #Add the sidecar info for editing
+              sidecar ||= resource.document_model.new(id: @item.id).sidecar(resource.exhibit) 
+              sidecar.update(data: @item_sidecar)
+              yield base_doc.merge(@item_solr) if @item_solr.present?
+
+              count = count + 1
+              totalrecords = totalrecords + 1
+              curtime = Time.zone.now
+              resource.get_job_entry.update(job_item_count: totalrecords, end_time: curtime)
+
+            rescue Exception => e
+              Delayed::Worker.logger.add(Logger::ERROR, @item.id + ' did not index successfully')
+              Delayed::Worker.logger.add(Logger::ERROR, e.message)
+              Delayed::Worker.logger.add(Logger::ERROR, e.backtrace)
+            end
+          end #End of each loop
+          page = page + 1
+          #Stop harvesting if the batch has reached the maximum allowed value
+          if (!last_page_evaluated) 
+            if (max_batch_count != -1 && count >= max_batch_count)
+              schedule_next_batch(page, totalrecords)
+              break
+            else
+             harvests = resource.paginate(page)
+             
+            end
+          end
+          
+                    
+          #Terminate the loop if it is empty        
+          if (harvests['response']['docs'].blank?)
+            last_page_evaluated = true
+          end
+        end #End of while loop
+        rescue
+          resource.get_job_entry.failed!
+          raise
+        end
+        if (last_page_evaluated)
+          resource.get_job_entry.succeeded!
+        end
+
+      end
+      
+      def get_unique_id_field_name(mapping_file)
+        mapping_config = YAML.load_file(mapping_file)
+        mapping_config.each do |field|
+          if (!field.key?("spotlight-field") || field['spotlight-field'].blank?)
+            raise InvalidMappingFile, "spotlight-field is required for each entry"
+          end
+        end
+      end
+      
+      def schedule_next_batch(cursor, count)
+        Spotlight::Resources::PerformHarvestsJob.perform_later(resource.data[:type], resource.data[:base_url], resource.data[:set], resource.data[:mapping_file], resource.exhibit, resource.data[:user], resource.data[:job_entry], cursor, count)
+      end
+
+
+    end
+  end
+end

data/app/views/catalog/_show.html.erb

@@ -0,0 +1,10 @@
+<% doc_presenter = show_presenter(document) %>
+<%# default partial to display solr document fields in catalog show view -%>
+<dl class="dl-horizontal  dl-invert ddtest">
+  <% document_show_fields(document).each do |field_name, field| -%>
+    <% if should_render_show_field? document, field %>
+	    <dt class="blacklight-<%= field_name.parameterize %> col-md-3"><%= render_document_show_field_label document, field: field_name %></dt>
+	    <dd class="blacklight-<%= field_name.parameterize %> col-md-9"><%= doc_presenter.field_value field_name %></dd>
+    <% end -%>
+  <% end -%>
+</dl>

data/app/views/spotlight/harvesting_complete_mailer/harvest_failed.html.erb

@@ -0,0 +1,6 @@
+<p><%= t :".title" %></p>
+
+<p><%= t :".body", set: @set, title: @exhibit.title %></p>
+<% if (!@message.blank?) %>
+<p>The failure message given was '<%=@message%>'</p>
+<% end %>

data/app/views/spotlight/harvesting_complete_mailer/harvest_indexed.html.erb

@@ -0,0 +1,13 @@
+<p><%= t :".title" %></p>
+
+<p><%= t :".body", set: @set, title: @exhibit.title %></p>
+<% if (!@failed_items.blank?) %>
+<p>The following items did not index properly and were not added:</p>
+<p>
+<% 
+     @failed_items.each do |item| %>
+     	<%=item %></br>
+<%  end %>
+</p>
+<%  end  %>
+

data/app/views/spotlight/resources/harvester/_form.html.erb

@@ -0,0 +1,36 @@
+<%= bootstrap_form_for([current_exhibit, @resource.becomes(Spotlight::Resources::Harvester)], url: spotlight_oaipmh_resources_engine.exhibit_harvester_path(exhibit_id: current_exhibit), layout: :horizontal, label_col: 'col-md-2', control_col: 'col-sm-6 col-md-6' ) do |f| %>
+  <%= f.select :type, Spotlight::Resources::HarvestType::HARVEST_TYPES, {}, :onChange=>"toggleHarvesters(this.value);" %>
+  <%= f.text_field :url, help: t('.url-field.help'), label: t('.url') %>
+  <%= f.text_field :set, help: t('.set-field.help'), label: t('.set') %>
+  <span id="oaiharvester"><%= f.select :mapping_file, Spotlight::Resources::OaipmhHarvester.mapping_files, {}, :onChange=>"toggleUploadArea(this.value);" %></span>
+  <span id="solrharvester" style="display:none"><%= f.select :solr_mapping_file, Spotlight::Resources::SolrHarvester.mapping_files, {}, :onChange=>"toggleUploadArea(this.value);" %></span>
+  <div id="custom_mapping_file_div" style="display:none"><%= f.file_field :custom_mapping, help: t('.custom-mapping-field.help'), label: t('.custom-mapping') %></div>
+  <div class="form-actions">
+    <div class="primary-actions">
+      <%= cancel_link @resource, :back, class: 'btn btn-default' %>
+      <%= f.submit t('.add_item'), class: 'btn btn-primary' %>
+    </div>
+  </div>
+<% end %>
+
+<%= javascript_tag do %>
+  function toggleUploadArea(value) {
+	  if(value=='New Mapping File') {
+	  	document.getElementById("custom_mapping_file_div").style.display = "block";
+	  }
+	  else {
+	  	document.getElementById("custom_mapping_file_div").style.display = "none";
+	  }
+  }
+  
+  function toggleHarvesters(value) {
+  	if(value=='MODS') {
+	  	document.getElementById("oaiharvester").style.display = "block";
+	  	document.getElementById("solrharvester").style.display = "none";
+	  }
+	  else {
+	  	document.getElementById("oaiharvester").style.display = "none";
+	  	document.getElementById("solrharvester").style.display = "block";
+	  }
+  }
+<% end %>

data/config/default_solr_mapping.yml

@@ -0,0 +1,20 @@
+#Structure:
+#
+#  - spotlight-field: xxx (field names should be separated with dashes except for the suffix: firstpart-secondpart_ssim or _tesim)
+#    multivalue-breaks: "yes" (optional) - use this for splitting out multiple values to be broken on (and faceted on) individually (ex - subjects)
+#    default-value: xxx (optional)
+#    delimiter: xxx (optional, what to separate all path values with.  Defaults to a space)
+#    solr-field:
+#        - field-name: xxx (repeatable - all path fields will be concatenated)
+#
+# Make sure to set unique-id_tesim if you want to choose an identifier for your record.
+# Otherwise, reharvesting will result in the addition of more records rather than overwriting
+# existing records.
+
+- spotlight-field: unique-id_tesim
+  solr-field:
+     - field-name: "hashKey"
+     
+- spotlight-field: full_title_tesim
+  solr-field:
+     - field-name: "artist.romanizedName"

data/config/locales/en.yml

@@ -0,0 +1,32 @@
+en:
+  spotlight:
+    resources:
+        harvester:
+            form:
+              title: 'Metadata Harvesting'
+              type: "Harvesting Type"
+              type-field:
+                help: "Choose where to harvest from (eg - Solr, OAI)"
+              url: "Base URL"
+              add_item: "Harvest items"
+              url-field:
+                help: "Add the base URL of the data to be harvested."
+              set: "Set spec"
+              set-field:
+                help: "Type in the set name to be harvested."
+              mapping-file: "Select Mapping File"
+              mapping-file-field:
+                help: "Select Mapping File to Use (default is mapping.yml)"
+              custom-mapping: "Custom Mapping File"
+              custom-mapping-field:
+                help: "Upload an optional mapping file (YML format)."
+            performharvest:
+              success: "Successfully scheduled '%{set}' for harvesting."
+    harvesting_complete_mailer:
+       harvest_indexed:
+          title: "Your harvest has just finished being processed."
+          body: "The %{set} set has been indexed from the harvest and added to the exhibit %{title}."
+       harvest_failed:
+          title: "Your harvest has failed to import."
+          body: "The %{set} harvest for the exhibit %{title} has failed."
+        

data/config/mapping.yml

@@ -0,0 +1,172 @@
+#Structure:
+
+#  - spotlight-field: xxx
+#    default-value: xxx (optional)
+#    delimiter: xxx (optional, what to separate all path values with.  Defaults to a space)
+#    mods:
+#        - path: xxx (repeatable - all path fields will be concatenated)
+#          delimiter: " "
+#          attribute: displayLabel
+#          attribute-value: collection
+#          mods-path: xxx 
+#          mods-value: xx
+#          subpaths: 
+#            - subpath: xxx
+#            - subpath: xxx
+#
+- spotlight-field: unique-id_tesim
+  mods:
+      - path: recordInfo/recordIdentifier
+- spotlight-field: unique-id-sort_tesi
+  mods:
+      - path: recordInfo/recordIdentifier
+- spotlight-field: full_title_tesim
+  delimiter: ": "
+  mods:
+      - path: titleInfo
+        delimiter: " "
+        subpaths:
+          - subpath: nonSort
+          - subpath: title
+      - path: titleInfo/subTitle
+- spotlight-field: citation-title_tesim
+  mods:
+      - path: titleInfo/title
+- spotlight-field: creator_tesim
+  mods:
+      - path: plain_name
+        delimiter: " , "
+        mods-path: role/roleTerm
+        mods-value: creator
+        subpaths:
+          - subpath: namePart
+- spotlight-field: start-date_tesim
+  mods:
+      - path: originInfo/dateCreated
+        delimiter: "|"
+        attribute: point
+        attribute-value: start
+- spotlight-field: end-date_tesim
+  mods:
+      - path: originInfo/dateCreated
+        delimiter: "|"
+        attribute: point
+        attribute-value: end
+- spotlight-field: date_tesim
+  mods:
+      - path: originInfo/dateCreated
+        attribute: '!point'
+        attribute-value: 
+- spotlight-field: contributer_tesim
+  delimiter: " , "
+  mods:
+      - path: plain_name
+        delimiter: " , "
+        mods-path: role/roleTerm
+        mods-value: '!creator'
+        subpaths:
+          - subpath: namePart
+- spotlight-field: spotlight_upload_description_tesim
+  delimiter: "|"
+  mods: 
+      - path: abstract
+      - path: note
+        attribute: type
+        attribute-value: 
+- spotlight-field: format_tesim
+  delimiter: "|"
+  mods:
+      - path: physicalDescription/extent
+- spotlight-field: language_ssim
+  default-value: English
+  delimiter: "|"
+  mods: 
+      - path: language/languageTerm
+- spotlight-field: repository_ssim
+  delimiter: "|"
+  mods:
+      - path: location/physicalLocation
+        attribute: type
+        attribute-value: repository
+      - path: relatedItem
+        subpaths:
+          - subpath: location/physicalLocation
+        attribute: displayLabel
+        attribute-value: collection
+- spotlight-field: subjects_ssim
+  delimiter: "|"
+  mods:
+      - path: subject
+        delimiter: "--"
+        subpaths:
+          - subpath: name/namePart
+          - subpath: topic
+          - subpath: geographic
+          - subpath: genre
+- spotlight-field: type_ssim
+  delimiter: "|"
+  mods:
+      - path: genre
+- spotlight-field: origin_ssim
+  mods:
+      - path: originInfo/place/placeTerm
+- spotlight-field: biography_tesim
+  delimiter: "|"
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: biographical/historical
+- spotlight-field: statement-of-responsibility_tesim
+  delimiter: "|"
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: statement of responsibility
+- spotlight-field: citation_tesim
+  delimiter: "|"
+  mods:
+       - path: note
+         attribute: type
+         attribute-value: preferred citation
+- spotlight-field: physical-form_tesim
+  delimiter: "|"
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: additional physical form
+- spotlight-field: language-info_tesim
+  delimiter: "|"
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: language
+- spotlight-field: publications_tesim
+  delimiter: "|"
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: publications
+- spotlight-field: funding_tesim
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: funding
+- spotlight-field: funding_ssim
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: funding
+- spotlight-field: thumbnail_url_ssm
+  mods:
+      - path: location/url
+        attribute: access
+        attribute-value: preview
+- spotlight-field: full_image_url_ssm
+  mods:
+      - path: location/url
+        attribute: access
+        attribute-value: raw object
+- spotlight-field: rights_tesim
+  mods:
+      - path: accessCondition
+         

data/config/marc_mapping.yml

@@ -0,0 +1,190 @@
+#Structure:
+
+#  - spotlight-field: xxx
+#    default-value: xxx (optional)
+#    delimiter: xxx (optional, what to separate all path values with.  Defaults to a space)
+#    mods:
+#        - path: xxx (repeatable - all path fields will be concatenated)
+#          delimiter: " "
+#          attribute: displayLabel
+#          attribute-value: collection
+#          mods-path: xxx 
+#          mods-value: xx
+#          subpaths: 
+#            - subpath: xxx
+#            - subpath: xxx
+#
+- spotlight-field: unique-id_tesim
+  mods:
+      - path: recordInfo/recordIdentifier
+- spotlight-field: full_title_tesim
+  delimiter: ": "
+  mods:
+      - path: titleInfo
+        delimiter: " "
+        subpaths:
+          - subpath: nonSort
+          - subpath: title
+      - path: titleInfo/subTitle
+- spotlight-field: citation-title_tesim
+  mods:
+      - path: titleInfo/title
+- spotlight-field: creator_tesim
+  mods:
+      - path: plain_name
+        delimiter: " , "
+        mods-path: role/roleTerm
+        mods-value: creator
+        subpaths:
+          - subpath: namePart
+- spotlight-field: start-date_tesim
+  mods:
+      - path: originInfo/dateCreated
+        delimiter: "|"
+        attribute: point
+        attribute-value: start
+- spotlight-field: end-date_tesim
+  mods:
+      - path: originInfo/dateCreated
+        delimiter: "|"
+        attribute: point
+        attribute-value: end
+- spotlight-field: publisher_tesim
+  mods:
+      - path: originInfo
+        subpaths:
+          - subpath: place/placeTerm
+          - subpath: publisher
+          - subpath: dateIssued
+- spotlight-field: date_tesim
+  mods:
+      - path: originInfo/dateCreated
+        attribute: '!point'
+        attribute-value: 
+- spotlight-field: classification_ssim
+  mods:
+      - path: classification
+- spotlight-field: classification_tesim
+  mods:
+      - path: classification
+- spotlight-field: contributer_tesim
+  delimiter: "|"
+  mods:
+      - path: plain_name
+        delimiter: ", "
+        mods-path: role/roleTerm
+        mods-value: '!creator'
+        subpaths:
+          - subpath: namePart
+          - subpath: role/roleTerm
+- spotlight-field: spotlight_upload_description_tesim
+  delimiter: "|"
+  mods: 
+      - path: abstract
+- spotlight-field: format_tesim
+  delimiter: "|"
+  mods:
+      - path: physicalDescription/extent
+- spotlight-field: physical-description_tesim
+  delimiter: "|"
+  mods:
+      - path: physicalDescription/note
+- spotlight-field: language_ssim
+  default-value: English
+  delimiter: "|"
+  mods: 
+      - path: language/languageTerm
+- spotlight-field: language_tesim
+  default-value: English
+  delimiter: "|"
+  mods: 
+      - path: language/languageTerm
+- spotlight-field: repository_ssim
+  delimiter: "|"
+  mods:
+      - path: location/physicalLocation
+        attribute: type
+        attribute-value: repository
+      - path: relatedItem
+        subpaths:
+          - subpath: location/physicalLocation
+        attribute: displayLabel
+        attribute-value: collection
+- spotlight-field: repository_tesim
+  delimiter: "|"
+  mods:
+      - path: location/physicalLocation
+        attribute: type
+        attribute-value: repository
+      - path: relatedItem
+        subpaths:
+          - subpath: location/physicalLocation
+        attribute: displayLabel
+        attribute-value: collection
+- spotlight-field: subjects_ssim
+  delimiter: "|"
+  mods:
+      - path: subject
+        delimiter: "--"
+        subpaths:
+          - subpath: name/namePart
+          - subpath: topic
+          - subpath: genre
+- spotlight-field: subjects_tesim
+  delimiter: "|"
+  mods:
+      - path: subject
+        delimiter: "--"
+        subpaths:
+          - subpath: name/namePart
+          - subpath: topic
+          - subpath: genre
+- spotlight-field: genre_tesim
+  delimiter: ";"
+  mods:
+      - path: genre
+- spotlight-field: place_tesim
+  delimiter: "|"
+  mods:
+      - path: subject/hierarchicalGeographic
+        delimiter: "--"
+        subpaths:
+          - subpath: country
+          - subpath: state
+          - subpath: city
+- spotlight-field: origin_ssim
+  mods:
+      - path: originInfo/place/placeTerm
+- spotlight-field: origin_tesim
+  mods:
+      - path: originInfo/place/placeTerm
+- spotlight-field: attribution_tesim
+  delimiter: "|"
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: statement of responsibility
+- spotlight-field: note_tesim
+  delimiter: "|"
+  mods:
+      - path: note
+        attribute: type
+        attribute-value: '!statement of responsibility'
+- spotlight-field: thumbnail_url_ssm
+  mods:
+      - path: location/url
+        attribute: access
+        attribute-value: preview
+- spotlight-field: full_image_url_ssm
+  mods:
+      - path: location/url
+        attribute: access
+        attribute-value: raw object
+- spotlight-field: rights_tesim
+  mods:
+      - path: accessCondition
+- spotlight-field: creation_date_tesim
+  mods:
+      - path: originInfo/dateIssued
+      - path: originInfo/dateCreated
+