discovery-indexer 0.3 → 0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f58e08463571857d8a8469e31f38efabe2aed260
-  data.tar.gz: a72bbe36942e3858721ef47f55986ef00dffb4fd
+  metadata.gz: 7839ece7ccc93abe604fbc2e88c7735da73e3593
+  data.tar.gz: 5713715eac6372293c4e7c50cf4169b3566c873e
 SHA512:
-  metadata.gz: 6072df1b1424f5adfd5e6d181ff99e0e47c91ae1991f821cb539fd340017b981f599b3b726a6a5acebaa09cb673f3d024bc1cf2532dd724aea35a2e64d8d7db5
-  data.tar.gz: 4c756b97eb2d8256c17ff8111cea52ad49e697fbfa286032dcdf72a85e303f0489a367e6197c38186cbdd06f8225db392dc05dd96aa8947207cffb4b31aa6ba4
+  metadata.gz: 980c089b86b8dc005f92e858e3a2eec5f64cbad358510432241928db201fd452511581f71aa57b42d2f124d143f9e43c344ce4072f49568629801a1da31e55d5
+  data.tar.gz: ded188f8a0d2b06c3702c89070af6000c7d943fb98f3ce35d3cf4189a708acfcc5ce69de8fb213128490dd6927ed47589d5dc5203b6d2bf387425c701636af08

data/lib/discovery-indexer.rb

@@ -11,7 +11,6 @@ require 'reader/modsxml'
 require 'reader/modsxml_reader'
 
 require 'mapper/general_mapper'
-require 'mapper/index_mapper'
 
 require 'writer/solr_client'
 require 'writer/solr_writer'

data/lib/mapper/general_mapper.rb

@@ -2,6 +2,12 @@ module DiscoveryIndexer
   module Mapper
     class GeneralMapper
       
+      # Initializes an instance from IndexMapper
+      # @param [String] druid e.g. ab123cd4567
+      # @param [Stanford::Mods::Record] modsxml represents the MODS xml for the druid
+      # @param [DiscoveryIndexer::Reader::PurlxmlModel] purlxml represents the purlxml model
+      # @param [Hash] collection_names represents a hash of collection_druid and 
+      #  collection_name !{"aa111aa1111"=>"First Collection", "bb123bb1234"=>"Second Collection"}
       def initialize(druid, modsxml, purlxml, collection_names={})
         @druid = druid
         @modsxml = modsxml
@@ -9,9 +15,164 @@ module DiscoveryIndexer
         @collection_names = collection_names
       end
 
+      # Create a Hash representing a Solr doc, with all MODS related fields populated.  
+      # @return [Hash] Hash representing the Solr document
       def map()
+        solr_doc = {}
+        solr_doc[:id] = @druid
+        solr_doc.update mods_to_title_fields
+        solr_doc.update mods_to_author_fields
+        solr_doc.update mods_to_subject_search_fields
+        solr_doc.update mods_to_publication_fields
+        solr_doc.update mods_to_pub_date
+        solr_doc.update mods_to_others
+        
+        solr_doc[:all_search] = @modsxml.text.gsub(/\s+/, ' ')
+        return solr_doc
       end
-          
+
+      # @return [Hash] Hash representing the title fields
+      def mods_to_title_fields
+        # title fields
+        doc_hash = { 
+          :title_245a_search => @modsxml.sw_short_title,
+          :title_245_search => @modsxml.sw_full_title,
+          :title_variant_search => @modsxml.sw_addl_titles,
+          :title_sort => @modsxml.sw_sort_title,
+          :title_245a_display => @modsxml.sw_short_title,
+          :title_display => @modsxml.sw_title_display,
+          :title_full_display => @modsxml.sw_full_title,
+        }
+        doc_hash
+      end
+      
+      # @return [Hash] Hash representing the author fields
+      def mods_to_author_fields
+        doc_hash = { 
+          # author fields
+          :author_1xx_search => @modsxml.sw_main_author,
+          :author_7xx_search => @modsxml.sw_addl_authors,
+          :author_person_facet => @modsxml.sw_person_authors,
+          :author_other_facet => @modsxml.sw_impersonal_authors,
+          :author_sort => @modsxml.sw_sort_author[1..-1],
+          :author_corp_display => @modsxml.sw_corporate_authors,
+          :author_meeting_display => @modsxml.sw_meeting_authors,
+          :author_person_display => @modsxml.sw_person_authors,
+          :author_person_full_display => @modsxml.sw_person_authors,
+        }
+        doc_hash
+      end
+      
+      # @return [Hash] Hash representing the search fields
+      def mods_to_subject_search_fields
+        doc_hash = { 
+          # subject search fields
+          :topic_search => @modsxml.topic_search, 
+          :geographic_search => @modsxml.geographic_search,
+          :subject_other_search => @modsxml.subject_other_search, 
+          :subject_other_subvy_search => @modsxml.subject_other_subvy_search,
+          :subject_all_search => @modsxml.subject_all_search, 
+          :topic_facet => @modsxml.topic_facet,
+          :geographic_facet => @modsxml.geographic_facet,
+          :era_facet => @modsxml.era_facet,
+        }
+      end
+      
+      # @return [Hash] Hash representing the publication fields
+      def mods_to_publication_fields
+        doc_hash = { 
+          # publication fields
+          :pub_search =>  @modsxml.place,
+          :pub_date_sort =>  @modsxml.pub_date_sort,
+          :imprint_display =>  @modsxml.pub_date_display,
+          :pub_date =>  @modsxml.pub_date_facet,
+          :pub_date_display =>  @modsxml.pub_date_display, # pub_date_display may be deprecated
+        }
+      end
+      
+      # @return [Hash] Hash representing the pub date
+      def mods_to_pub_date
+        doc_hash = {}
+        pub_date_sort = @modsxml.pub_date_sort
+        if is_positive_int? pub_date_sort
+          doc_hash[:pub_year_tisim] =  pub_date_sort # for date slider
+          # put the displayable year in the correct field, :creation_year_isi for example
+          doc_hash[date_type_sym] =  @modsxml.pub_date_sort  if date_type_sym
+        end
+        return doc_hash
+      end    
+        
+      # @return [Hash] Hash representing some fields 
+      def mods_to_others
+        doc_hash = { 
+          :format_main_ssim => format_main_ssim,
+          :format => format, # for backwards compatibility
+          :language => @modsxml.sw_language_facet,
+          :physical =>  @modsxml.term_values([:physical_description, :extent]),
+          :summary_search => @modsxml.term_values(:abstract),
+          :toc_search => @modsxml.term_values(:tableOfContents),
+          :url_suppl => @modsxml.term_values([:related_item, :location, :url]),
+        }
+        return doc_hash
+      end
+    
+      # select one or more format values from the controlled vocabulary here:
+      #   http://searchworks-solr-lb.stanford.edu:8983/solr/select?facet.field=format&rows=0&facet.sort=index
+      # via stanford-mods gem
+      # @return [Array<String>] value(s) in the SearchWorks controlled vocabulary, or []
+      def format
+        vals = @modsxml.format
+        if vals.empty?
+          puts "#{@druid} has no SearchWorks format from MODS - check <typeOfResource> and other implicated MODS elements"
+        end
+        vals
+      end
+      
+      # call stanford-mods format_main to get results
+      # @return [Array<String>] value(s) in the SearchWorks controlled vocabulary, or []
+      def format_main_ssim
+        vals = @modsxml.format_main
+        if vals.empty?
+          puts "#{@druid} has no SearchWorks Resource Type from MODS - check <typeOfResource> and other implicated MODS elements"
+        end
+        vals
+      end
+    
+      # call stanford-mods sw_genre to get results
+      # @return [Array<String>] value(s) 
+      def genre_ssim
+        @modsxml.sw_genre
+      end
+    
+    protected
+    
+      # @return true if the string parses into an int, and if so, the int is >= 0
+      def is_positive_int? str
+        begin
+          if str.to_i >= 0
+            return true
+          else
+            return false
+          end
+        rescue
+        end
+        return false
+      end
+    
+      # determines particular flavor of displayable publication year field 
+      # @return Solr field name as a symbol
+      def date_type_sym
+        vals = @modsxml.term_values([:origin_info,:dateIssued])
+        if vals and vals.length > 0
+          return :publication_year_isi
+        end
+        vals = @modsxml.term_values([:origin_info,:dateCreated])  
+        if vals and vals.length > 0
+          return :creation_year_isi
+        end
+        nil
+      end
+      
     end
   end
 end

data/lib/reader/purlxml_parser_strict.rb

@@ -172,7 +172,6 @@ module DiscoveryIndexer
       
       def parse_copyright
         node=@purlxml_ng_doc.css('//rightsMetadata/copyright/human[type="copyright"]')
-        puts node.inspect
         copyright_text = node.first.content if node && node.first
         return copyright_text
       end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module DiscoveryIndexer
-    VERSION = '0.3'
+    VERSION = '0.4'
 end

data/lib/writer/solr_client.rb

@@ -11,8 +11,8 @@ module DiscoveryIndexer
       # @param [Hash] solr_doc a Hash representation of the solr document
       # @param [RSolr::Client] solr_connector is an open connection with the solr core
       # @param [Integer] max_retries the maximum number of tries before fail
-      def self.add(solr_doc, solr_connector, max_retries = 10)
-        process(solr_doc, solr_connector, max_retries, is_delete=false)  
+      def self.add(id, solr_doc, solr_connector, max_retries = 10)
+        process(id, solr_doc, solr_connector, max_retries, is_delete=false)  
       end
 
       # Add the document to solr, retry if an error occurs.
@@ -20,13 +20,11 @@ module DiscoveryIndexer
       # @param [Hash] solr_doc that has only the id !{:id=>"ab123cd4567"}
       # @param [RSolr::Client] solr_connector is an open connection with the solr core
       # @param [Integer] max_retries the maximum number of tries before fail
-      def self.delete(solr_doc, solr_connector, max_retries = 10)
-        process(solr_doc, solr_connector, max_retries, is_delete=true)
+      def self.delete(id, solr_doc, solr_connector, max_retries = 10)
+        process(id, solr_doc, solr_connector, max_retries, is_delete=true)
       end
       
-      def self.process(solr_doc, solr_connector, max_retries, is_delete=false)
-        id = solr_doc[:id]
-        puts id
+      def self.process(id, solr_doc, solr_connector, max_retries, is_delete=false)
         handler = Proc.new do |exception, attempt_number, total_delay|
           DiscoveryIndexer::Logging.logger.debug "#{exception.class} on attempt #{attempt_number} for #{id}"
         end

data/lib/writer/solr_writer.rb

@@ -6,7 +6,7 @@ module DiscoveryIndexer
     class SolrWriter
       include DiscoveryIndexer::Logging
       
-      def process(druid, index_doc, targets, solr_targets_configs)
+      def process(id, index_doc, targets, solr_targets_configs)
         @solr_targets_configs = solr_targets_configs
         index_targets = []
         delete_targets = []
@@ -20,29 +20,29 @@ module DiscoveryIndexer
         end
         
         # get targets with true
-        solr_index_client(index_doc, index_targets)
+        solr_index_client(id, index_doc, index_targets)
         # get targets with false
-        solr_delete_client(druid, delete_targets)
+        solr_delete_client(id, delete_targets)
       end
       
-      def solr_delete_from_all(druid, solr_targets_configs)
+      def solr_delete_from_all(id, solr_targets_configs)
         # Get a list of all registered targets
         @solr_targets_configs=solr_targets_configs
         targets = @solr_targets_configs.keys()
-        solr_delete_client(druid, targets)
+        solr_delete_client(id, targets)
       end
             
-      def solr_index_client(index_doc, targets)
+      def solr_index_client(id, index_doc, targets)
         targets.each do |solr_target|
           solr_connector = get_connector_for_target(solr_target)     
-           SolrClient.add(index_doc, solr_connector)
+           SolrClient.add(id, index_doc, solr_connector)
         end          
       end
       
-      def solr_delete_client(druid, targets)
+      def solr_delete_client(id, targets)
         targets.each do |solr_target|
           solr_connector = get_connector_for_target(solr_target)     
-          SolrClient.delete({:id=>druid}, solr_connector)
+          SolrClient.delete(id,{}, solr_connector)
         end         
       end
 

metadata

@@ -1,125 +1,125 @@
 --- !ruby/object:Gem::Specification
 name: discovery-indexer
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: '0.4'
 platform: ruby
 authors:
 - Ahmed AlSum
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-24 00:00:00.000000000 Z
+date: 2015-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: stanford-mods
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: retries
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rsolr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: This library manages the core operations for the discovery indexing such
@@ -133,7 +133,6 @@ files:
 - lib/errors.rb
 - lib/logging.rb
 - lib/mapper/general_mapper.rb
-- lib/mapper/index_mapper.rb
 - lib/reader/modsxml.rb
 - lib/reader/modsxml_reader.rb
 - lib/reader/purlxml.rb
@@ -155,12 +154,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []

data/lib/mapper/index_mapper.rb

@@ -1,179 +0,0 @@
-module DiscoveryIndexer
-  module Mapper
-    
-    # This class is responsible for creating the solr_doc hash based on the input
-    # of druid_id, modsxml, purlxml, and optional hash of collection_names
-    class IndexMapper < GeneralMapper
-      
-      # Initializes an instance from IndexMapper
-      # @param [String] druid e.g. ab123cd4567
-      # @param [Stanford::Mods::Record] modsxml represents the MODS xml for the druid
-      # @param [DiscoveryIndexer::Reader::PurlxmlModel] purlxml represents the purlxml model
-      # @param [Hash] collection_names represents a hash of collection_druid and 
-      #  collection_name !{"aa111aa1111"=>"First Collection", "bb123bb1234"=>"Second Collection"}
-      def initialize(druid, modsxml, purlxml, collection_names={})
-        super druid, modsxml, purlxml, collection_names
-      end  
-      
-      # Create a Hash representing a Solr doc, with all MODS related fields populated.  
-      # @return [Hash] Hash representing the Solr document
-      def map()
-        solr_doc = {}
-        solr_doc[:id] = @druid
-        solr_doc.update mods_to_title_fields
-        solr_doc.update mods_to_author_fields
-        solr_doc.update mods_to_subject_search_fields
-        solr_doc.update mods_to_publication_fields
-        solr_doc.update mods_to_pub_date
-        solr_doc.update mods_to_others
-        
-        solr_doc[:all_search] = @modsxml.text.gsub(/\s+/, ' ')
-        return solr_doc
-      end
-
-      # @return [Hash] Hash representing the title fields
-      def mods_to_title_fields
-        # title fields
-        doc_hash = { 
-          :title_245a_search => @modsxml.sw_short_title,
-          :title_245_search => @modsxml.sw_full_title,
-          :title_variant_search => @modsxml.sw_addl_titles,
-          :title_sort => @modsxml.sw_sort_title,
-          :title_245a_display => @modsxml.sw_short_title,
-          :title_display => @modsxml.sw_title_display,
-          :title_full_display => @modsxml.sw_full_title,
-        }
-        doc_hash
-      end
-      
-      # @return [Hash] Hash representing the author fields
-      def mods_to_author_fields
-        doc_hash = { 
-          # author fields
-          :author_1xx_search => @modsxml.sw_main_author,
-          :author_7xx_search => @modsxml.sw_addl_authors,
-          :author_person_facet => @modsxml.sw_person_authors,
-          :author_other_facet => @modsxml.sw_impersonal_authors,
-          :author_sort => @modsxml.sw_sort_author[1..-1],
-          :author_corp_display => @modsxml.sw_corporate_authors,
-          :author_meeting_display => @modsxml.sw_meeting_authors,
-          :author_person_display => @modsxml.sw_person_authors,
-          :author_person_full_display => @modsxml.sw_person_authors,
-        }
-        doc_hash
-      end
-      
-      # @return [Hash] Hash representing the search fields
-      def mods_to_subject_search_fields
-        doc_hash = { 
-          # subject search fields
-          :topic_search => @modsxml.topic_search, 
-          :geographic_search => @modsxml.geographic_search,
-          :subject_other_search => @modsxml.subject_other_search, 
-          :subject_other_subvy_search => @modsxml.subject_other_subvy_search,
-          :subject_all_search => @modsxml.subject_all_search, 
-          :topic_facet => @modsxml.topic_facet,
-          :geographic_facet => @modsxml.geographic_facet,
-          :era_facet => @modsxml.era_facet,
-        }
-      end
-      
-      # @return [Hash] Hash representing the publication fields
-      def mods_to_publication_fields
-        doc_hash = { 
-          # publication fields
-          :pub_search =>  @modsxml.place,
-          :pub_date_sort =>  @modsxml.pub_date_sort,
-          :imprint_display =>  @modsxml.pub_date_display,
-          :pub_date =>  @modsxml.pub_date_facet,
-          :pub_date_display =>  @modsxml.pub_date_display, # pub_date_display may be deprecated
-        }
-      end
-      
-      # @return [Hash] Hash representing the pub date
-      def mods_to_pub_date
-        doc_hash = {}
-        pub_date_sort = @modsxml.pub_date_sort
-        if is_positive_int? pub_date_sort
-          doc_hash[:pub_year_tisim] =  pub_date_sort # for date slider
-          # put the displayable year in the correct field, :creation_year_isi for example
-          doc_hash[date_type_sym] =  @modsxml.pub_date_sort  if date_type_sym
-        end
-        return doc_hash
-      end    
-        
-      # @return [Hash] Hash representing some fields 
-      def mods_to_others
-        doc_hash = { 
-          :format_main_ssim => format_main_ssim,
-          :format => format, # for backwards compatibility
-          :language => @modsxml.sw_language_facet,
-          :physical =>  @modsxml.term_values([:physical_description, :extent]),
-          :summary_search => @modsxml.term_values(:abstract),
-          :toc_search => @modsxml.term_values(:tableOfContents),
-          :url_suppl => @modsxml.term_values([:related_item, :location, :url]),
-        }
-        return doc_hash
-      end
-    
-      # select one or more format values from the controlled vocabulary here:
-      #   http://searchworks-solr-lb.stanford.edu:8983/solr/select?facet.field=format&rows=0&facet.sort=index
-      # via stanford-mods gem
-      # @return [Array<String>] value(s) in the SearchWorks controlled vocabulary, or []
-      def format
-        vals = @modsxml.format
-        if vals.empty?
-          puts "#{@druid} has no SearchWorks format from MODS - check <typeOfResource> and other implicated MODS elements"
-        end
-        vals
-      end
-      
-      # call stanford-mods format_main to get results
-      # @return [Array<String>] value(s) in the SearchWorks controlled vocabulary, or []
-      def format_main_ssim
-        vals = @modsxml.format_main
-        if vals.empty?
-          puts "#{@druid} has no SearchWorks Resource Type from MODS - check <typeOfResource> and other implicated MODS elements"
-        end
-        vals
-      end
-    
-      # call stanford-mods sw_genre to get results
-      # @return [Array<String>] value(s) 
-      def genre_ssim
-        @modsxml.sw_genre
-      end
-    
-    protected
-    
-      # @return true if the string parses into an int, and if so, the int is >= 0
-      def is_positive_int? str
-        begin
-          if str.to_i >= 0
-            return true
-          else
-            return false
-          end
-        rescue
-        end
-        return false
-      end
-    
-      # determines particular flavor of displayable publication year field 
-      # @return Solr field name as a symbol
-      def date_type_sym
-        vals = @modsxml.term_values([:origin_info,:dateIssued])
-        if vals and vals.length > 0
-          return :publication_year_isi
-        end
-        vals = @modsxml.term_values([:origin_info,:dateCreated])  
-        if vals and vals.length > 0
-          return :creation_year_isi
-        end
-        nil
-      end
-      
-    end
-  end
-end
-