spotlight-oaipmh-resources 0.3.6

data/app/models/spotlight/resources/oaipmh_mods_item.rb

@@ -0,0 +1,61 @@
+require 'oai'
+require 'mods'
+
+include OAI::XPath
+include Spotlight::Resources::Exceptions
+module Spotlight::Resources
+  class OaipmhModsItem
+    attr_reader :titles, :id
+    attr_accessor :metadata, :sidecar_data
+    def initialize(exhibit, converter)
+      @solr_hash = {}
+      @exhibit = exhibit
+      @converter = converter
+    end
+    
+    def to_solr
+      add_document_id
+      solr_hash
+    end
+    
+    def parse_mods_record()
+        
+      @modsrecord = Mods::Record.new.from_str(metadata.elements.to_a[0].to_s)
+          
+      if (@modsrecord.mods_ng_xml.record_info && @modsrecord.mods_ng_xml.record_info.recordIdentifier)
+        @id = @modsrecord.mods_ng_xml.record_info.recordIdentifier.text 
+        #Strip out all of the decimals
+        @id = @id.gsub('.', '')
+        @id = @exhibit.id.to_s + "-" + @id.to_s
+      end
+      
+      begin
+        @titles = @modsrecord.full_titles
+      rescue NoMethodError
+        @titles = nil
+      end
+      
+      if (@titles.blank? && @id.blank?)
+        raise InvalidModsRecord, "A mods record was found that has no title and no identifier."
+      elsif (@titles.blank?)
+        raise InvalidModsRecord, "Mods record " + @id + " must have a title.  This mods record was not updated in Spotlight."
+      elsif (@id.blank?)
+        raise InvalidModsRecord, "Mods record " + @titles[0] + "must have a title. This mods record was not updated in Spotlight."
+      end  
+      
+      @solr_hash = @converter.convert(@modsrecord)
+      @sidecar_data = @converter.sidecar_hash
+   end
+  
+   # private
+    
+    attr_reader :solr_hash, :exhibit
+    
+    
+    
+    def add_document_id
+      solr_hash[:id] = @id.to_s
+    end
+  
+  end
+end

data/app/models/spotlight/resources/solr_converter.rb

@@ -0,0 +1,180 @@
+include Spotlight::Resources::Exceptions
+module Spotlight::Resources
+  
+  class SolrEntry
+     attr_accessor :solr_field
+   end
+  class ConverterItem
+    attr_accessor :spotlight_field, :solr_items, :default_value, :delimiter, :multivalue_facets
+        
+    def initialize()
+      delimiter = ", "
+    end
+    
+    def extract_values(solrmd)
+      
+      values = extract_solr_values(solrmd)
+      
+      #Remove duplicates
+      values = values.uniq
+      
+      finalvalue = nil
+      if (!values.empty?)
+        #if multiple values, allow for faceting on each item by keeping it as an array
+        if (!multivalue_facets.nil? && (multivalue_facets.eql?("yes") || multivalue_facets))
+          
+          finalvalue = values;
+        else
+          finalvalue = values.join(delimiter)
+        end
+      end 
+      finalvalue
+    end
+    
+private
+    
+    def extract_solr_values(solrmd)
+      values = Array.new
+      if (!solr_items.nil?)
+        retvals = Array.new
+        solr_items.each do |item|
+          myretval = solrmd[item.solr_field]
+          
+          if (myretval.blank? && !default_value.blank?)
+            value = default_value
+            values << value
+          elsif (!myretval.blank?)
+            values << myretval
+          end
+        end
+      end
+      values
+    end
+
+end
+  
+  class SolrConverter
+    STANDARD_SPOTLIGHT_FIELDS = ['unique-id_tesim', 'full_title_tesim', 'spotlight_upload_description_tesim', 'thumbnail_url_ssm', 'full_image_url_ssm', 'spotlight_upload_date_tesim"', 'spotlight_upload_attribution_tesim']
+      
+    attr_accessor :sidecar_hash
+       
+    #Initialize with the name of the set being converted
+    def initialize(set, exhibitslug, mapping_file)
+      @set = set
+      @exhibitslug = exhibitslug
+      @mapping_file = mapping_file   
+      @converter_items = Array.new  
+      @sidecar_hash = {}
+    end
+    
+    def convert(solrrecord)
+      if (@converter_items.empty?)
+        parse_mapping_file(mapping_file)
+      end
+      
+      solr_hash = {}
+        
+      @converter_items.each do |item|
+        value = item.extract_values(solrrecord)
+          
+      #Not sure why but if a value isn't assigned, the last existing value for the field gets
+      #placed in all non-existing values
+       solr_hash[get_spotlight_field_name(item.spotlight_field)] = value
+       @sidecar_hash[item.spotlight_field] = value
+
+      end
+      solr_hash
+    end
+    
+    #Some spotlight fields use the exhibit slug, others do not
+    def get_spotlight_field_name(spotlight_field)      
+      if (!STANDARD_SPOTLIGHT_FIELDS.include?(spotlight_field))
+        spotlight_field = 'exhibit_' + @exhibitslug + '_' + spotlight_field
+      end
+      spotlight_field
+    end
+    
+
+    #Retrieves the mapping file for the set, if one exists, otherwise uses the generic mapping file
+    def mapping_file
+      if (@mapping_file == nil)
+        engine_root = Spotlight::Oaipmh::Resources::Engine.root
+        @mapping_file = File.join(engine_root, 'config', 'default_solr_mapping.yml')
+      else
+        @mapping_file = Rails.root.join("public/uploads/solrmapping", @mapping_file)
+      end
+      @mapping_file        
+    end
+
+  
+  #private
+  
+   #parses the mapping file into a model
+  def parse_mapping_file(file)
+    
+    mapping_config = YAML.load_file(file)
+    mapping_config.each do |field|
+      
+      item = ConverterItem.new
+      #validate the spotlight-field is not null
+      if (!field.key?("spotlight-field") || field['spotlight-field'].blank?)
+        raise InvalidMappingFile, "spotlight-field is required for each entry"
+      end
+      item.spotlight_field = field['spotlight-field']
+      
+      if (field.key?("delimiter"))
+        item.delimiter = field["delimiter"]
+      end
+      if (field.key?("default-value"))
+        item.default_value = field["default-value"]
+      end
+
+      if (field.key?("multivalue-breaks"))
+        item.multivalue_facets = field["multivalue-breaks"]
+      end
+      
+      #must have a solr-field value 
+      if (!field.key?("solr-field"))
+        raise InvalidMappingFile, "solr-field is required for each entry"
+      end
+      
+      #if using xpath, then add the values from xpath
+      if (field.key?('solr-field'))
+        item.solr_items = Array.new
+        field['solr-field'].each do |solr_field|
+          if (!solr_field.key?("field-name") || solr_field['field-name'].blank?)
+            raise InvalidMappingFile, "field-name is required for each solr-field entry"
+          end
+          
+          solritem = SolrEntry.new
+          solritem.solr_field = solr_field['field-name']
+          item.solr_items << solritem
+
+        end
+      end
+      
+      #If it is the unique field, set it
+      if (field['spotlight-field'].eql?("unique-id_tesim"))
+        delimiter = ""
+        if (!field["delimiter"].blank?)
+          delimiter = field["delimiter"]
+        end
+        
+        fields = Array.new
+        item.solr_items.each do |solr_item|
+          fields << solr_item.solr_field
+        end
+        @unique_id_field = fields.join(delimiter)
+      end
+      
+      @converter_items << item
+    end
+    @converter_items
+  end
+  
+  def get_unique_id_field()
+    @unique_id_field
+  end
+  
+  end
+end

data/app/models/spotlight/resources/solr_harvester.rb

@@ -0,0 +1,42 @@
+require 'net/http'
+require 'uri'
+  
+module Spotlight::Resources
+  class SolrHarvester
+    ROW_COUNT = 50
+        
+    def initialize(base_url, set)
+      @url = base_url + set
+      @base_url = base_url
+      @set = set
+    end
+            
+    def get_harvests
+      @solr_connection = RSolr.connect :url => @url  
+      response = @solr_connection.paginate 0, ROW_COUNT, 'select', :params => {:q => '*:*', :wt => 'json'}
+    end
+    
+    def paginate (page)
+      if (@solr_connection.nil?)
+        @solr_connection = RSolr.connect :url => @url  
+      end
+      response = @solr_connection.paginate page, ROW_COUNT, 'select', :params => {:q => '*:*', :wt => 'json'}
+    end
+    
+    
+    def self.mapping_files
+      if (Dir.exist?('public/uploads/solrmapping'))
+        files = Dir.entries('public/uploads/solrmapping')
+        files.delete(".")
+        files.delete("..")
+      else
+        files = Array.new
+      end
+      
+      files.insert(0, "New Mapping File")
+      files.insert(0, "Default Mapping File")
+      files
+    end
+
+  end
+end

data/app/models/spotlight/resources/solr_harvesting_item.rb

@@ -0,0 +1,50 @@
+
+include Spotlight::Resources::Exceptions
+module Spotlight::Resources
+  class SolrHarvestingItem
+    attr_reader :titles, :id
+    attr_accessor :metadata, :sidecar_data
+    def initialize(exhibit, converter)
+      @solr_hash = {}
+      @exhibit = exhibit
+      @converter = converter
+    end
+    
+    def to_solr
+      add_document_id
+      solr_hash
+    end
+    
+    def parse_record(unique_id_field)
+      if (!metadata[unique_id_field].blank?)
+        if (metadata[unique_id_field].kind_of?(Array))
+          @id = metadata[unique_id_field][0]
+        else
+          @id = metadata[unique_id_field] 
+        end
+        
+        #Strip out all of the decimals
+        @id = @id.gsub('.', '')
+        @id = @exhibit.id.to_s + "-" + @id.to_s
+      end
+      
+      @solr_hash = @converter.convert(metadata)
+      @sidecar_data = @converter.sidecar_hash
+   end
+  
+   # private
+    
+    attr_reader :solr_hash, :exhibit
+
+    
+    def add_document_id
+      if (!@id.blank?)
+        solr_hash[:id] = @id.to_s
+      else
+        #Generate a random number if no unique id is supplied.
+        solr_hash[:id] = rand.to_s[2..11] 
+      end
+    end
+  
+  end
+end

data/app/services/spotlight/resources/oaipmh_builder.rb

@@ -0,0 +1,166 @@
+module Spotlight
+  module Resources
+    # transforms a OaipmhHarvester into solr documents
+    class OaipmhBuilder < Spotlight::SolrDocumentBuilder
+      
+      def to_solr
+        begin
+        return to_enum(:to_solr) { 0 } unless block_given?
+
+        base_doc = super
+          
+           
+        mapping_file = nil
+        if (!resource.data[:mapping_file].eql?("Default Mapping File") && !resource.data[:mapping_file].eql?("New Mapping File"))
+          mapping_file = resource.data[:mapping_file]
+        end
+ 
+        max_batch_count = Spotlight::Oaipmh::Resources::Engine.config.oai_harvest_batch_max
+                   
+        @oai_mods_converter = OaipmhModsConverter.new(resource.data[:set], resource.exhibit.slug, mapping_file)
+        
+        count = 0
+        totalrecords = 0
+        failed_items = nil
+
+    		#If the resumption token was stored, begin there.
+        if (resource.data.include?(:cursor) && !resource.data[:cursor].blank?)
+          cursor = resource.data[:cursor]
+          harvests = resource.paginate(cursor)
+          
+   			else
+          harvests = resource.harvests
+        end
+     		
+     		resumption_token = harvests.resumption_token
+     		
+     		if (resource.data.include?(:count) && !resource.data[:count].blank?)
+          totalrecords = resource.data[:count]
+        end
+        
+        last_page_evaluated = false
+        while (!last_page_evaluated)
+          if (resumption_token.nil?)
+            last_page_evaluated = true
+          end
+          harvests.each do |record|
+            @item = OaipmhModsItem.new(exhibit, @oai_mods_converter)
+            @item.metadata = record.metadata
+            @item.parse_mods_record()
+            begin
+              @item_solr = @item.to_solr
+              @item_sidecar = @item.sidecar_data
+              
+              repository_field_name = @oai_mods_converter.get_spotlight_field_name("repository_ssim")
+              
+              process_images()
+
+              #Add the sidecar info for editing
+              sidecar ||= resource.document_model.new(id: @item.id).sidecar(resource.exhibit) 
+              sidecar.update(data: @item_sidecar)
+              yield base_doc.merge(@item_solr) if @item_solr.present?
+              
+              count = count + 1
+              totalrecords = totalrecords + 1
+              curtime = Time.zone.now
+              resource.get_job_entry.update(job_item_count: totalrecords, end_time: curtime)
+
+            rescue Exception => e
+              Delayed::Worker.logger.add(Logger::ERROR, @item.id + ' did not index successfully')
+              Delayed::Worker.logger.add(Logger::ERROR, e.message)
+              Delayed::Worker.logger.add(Logger::ERROR, e.backtrace)
+              if (failed_items.nil?)
+                failed_items = Array.new
+              end
+              failed_items << @item.id
+            end
+          end
+
+          #Stop harvesting if the batch has reached the maximum allowed value
+          if (!resumption_token.nil?) 
+        		if (max_batch_count != -1 && count >= max_batch_count)
+              schedule_next_batch(resumption_token, totalrecords, failed_items)
+        		  break
+        		else
+             harvests = resource.paginate(resumption_token)
+             resumption_token = harvests.resumption_token
+            end
+          end
+        
+        end
+        rescue Exception => e
+          resource.get_job_entry.failed!
+          Delayed::Worker.logger.add(Logger::ERROR, resource.data[:set] + ' harvest failed')
+          Delayed::Worker.logger.add(Logger::ERROR, e.message)
+          Delayed::Worker.logger.add(Logger::ERROR, e.backtrace)
+          Spotlight::HarvestingCompleteMailer.harvest_failed(resource.data[:set], resource.exhibit, resource.data[:user], e.message).deliver_now
+          raise
+        end
+        if (last_page_evaluated)
+        	resource.get_job_entry.succeeded!
+        	#Send job message
+          Spotlight::HarvestingCompleteMailer.harvest_indexed(resource.data[:set], resource.exhibit, resource.data[:user], failed_items).deliver_now
+       	end
+      end
+
+private   
+      def schedule_next_batch(cursor, count, failed_items)
+        Spotlight::Resources::PerformHarvestsJob.perform_later(resource.data[:type], resource.data[:base_url], resource.data[:set], resource.data[:mapping_file], resource.exhibit, resource.data[:user], resource.data[:job_entry], cursor, count, failed_items)
+      end
+            
+      def process_images()
+        if (@item_solr.key?('thumbnail_url_ssm') && !@item_solr['thumbnail_url_ssm'].blank? && !@item_solr['thumbnail_url_ssm'].eql?('null'))           
+          thumburl = fetch_ids_uri(@item_solr['thumbnail_url_ssm'])
+          if (!thumburl.blank? && !thumburl.eql?('null'))
+            thumburl = transform_ids_uri_to_iiif(thumburl)
+            @item_solr['thumbnail_url_ssm'] =  thumburl
+          end    
+        end
+        if (@item_solr.key?('full_image_url_ssm') && !@item_solr['full_image_url_ssm'].blank? && !@item_solr['full_image_url_ssm'].eql?('null'))           
+          
+          fullurl = fetch_ids_uri(@item_solr['full_image_url_ssm'])
+          if (!fullurl.blank?)
+
+            #If it is http, make it https            
+            if (fullurl.include?('http://'))
+              fullurl = fullurl.sub(/http:\/\//, "https://")
+            end
+            #if it is IDS, then add ?buttons=y so that mirador works
+            if (fullurl.include?('https://ids') && !fullurl.include?('?buttons=y'))
+              fullurl = fullurl + '?buttons=y'
+            end
+            @item_solr['full_image_url_ssm'] =  fullurl
+          end
+        end
+      end
+
+      
+      #Resolves urn-3 uris
+      def fetch_ids_uri(uri_str)
+        if (uri_str =~ /urn-3/)
+          response = Net::HTTP.get_response(URI.parse(URI.encode(uri_str)))['location']
+        elsif (uri_str.include?('?'))
+          uri_str = uri_str.slice(0..(uri_str.index('?')-1))
+        else
+          uri_str
+        end
+      end
+    
+      #Returns the uri for the iiif
+      def transform_ids_uri_to_iiif(ids_uri)
+        #Strip of parameters
+        uri = ids_uri.sub(/\?.+/, "")
+        #Change /view/ to /iiif/
+        uri = uri.sub(%r|/view/|, "/iiif/")
+        #Append /default.jpg to end if it doesn't exist
+        if (!uri.include?('default.jpg') && !uri.include?('native.jpg'))
+          uri = uri + "/full/180,/0/default.jpg"
+        elsif (uri.include?("full/,150/"))
+          uri = uri.sub(/full\/,150\//,"full/180,/")
+        end    
+        uri
+      end
+
+    end
+  end
+end