discovery-indexer 0.6 → 0.6.1

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    M2FhNzdjZDM4NDlhNjUzYzc3YTMyODRjNjQyMDBlYmU0NThmZmMwZg==
-  data.tar.gz: !binary |-
-    NjBhYzU1N2VkNjMzMWM0YjA1NGI2MzQ5YWFjY2FkOThjYjVjMTZlMQ==
+SHA1:
+  metadata.gz: f6c25de0587885161b4a8fc2882808e21ecaa915
+  data.tar.gz: 74bc83405539956a12749de6de5af8e39b8e2d8a
 SHA512:
-  metadata.gz: !binary |-
-    MjRiNWUzYzBmNDAyNGY3YzJmNmU0MzIzYjgzYzEzMWYzMzZjYWI3ZTQ4MWIz
-    Y2Q5MzAxMGFmMDFhZjQ4YzI5ZTkwMTM3NjEyMGJhMGNjMGE2NmRlN2VhYzM5
-    ZjQ1ZTFkMjg3MWZiNzdkYjczNDU4OTczY2RkODRkNmI0YWE5YTg=
-  data.tar.gz: !binary |-
-    ZDA5NGIxYzUyOGVkNzU2NmQ1ZDQ3NWUwN2NhZWFkOGRiYzk4NzhjOTE0M2Mz
-    NTM1NjMyYjhmMDg1OTdkZTA2NzY4NWM2Njg3Y2JiMjZhODM2Mzg1MWYyYWEx
-    ZjQzODgxYzEwOGY3ZWQzNTRiZThkZGJhN2ZiMTdiYjczZTIwMGQ=
+  metadata.gz: 012c0f66b119167e989aa977fea213fb61cf4c8b89b96cce59016cdc7918d1b6205e9429cbdeb717f64e1c540b17e54967c59584b8421750bcdf95a2be6e0a56
+  data.tar.gz: 38470801e1d4558ddfd27e8e4dd9933b1617e6a93cd727e897cf668e0b5acb26ba0858538af1fe66c001fc24cbdeebb963eb983d559638183e81e10b89c1e9d1

data/lib/utilities/extract_sub_targets.rb

@@ -0,0 +1,13 @@
+module DiscoveryIndexer
+  module Utilities
+    class ExtractSubTargets
+      
+      def self.by_name target_name, release_tags
+        if release_tags.keys.include?(target_name) then
+          return release_tags[target_name]
+        end
+      end
+      
+    end
+  end
+end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module DiscoveryIndexer
-    VERSION = '0.6'
+    VERSION = '0.6.1'
 end

data/lib/writer/solr_client.rb

@@ -1,6 +1,6 @@
 require 'retries'
 require 'rsolr'
-
+require 'rest-client'
 module DiscoveryIndexer
   module Writer
     class SolrClient
@@ -8,42 +8,85 @@ module DiscoveryIndexer
 
       # Add the document to solr, retry if an error occurs.
       # See https://github.com/ooyala/retries for docs on with_retries.
-      # @param [Hash] solr_doc a Hash representation of the solr document
-      # @param [RSolr::Client] solr_connector is an open connection with the solr core
-      # @param [Integer] max_retries the maximum number of tries before fail
+      # @param id [String] the document id, usually it will be druid.
+      # @param solr_doc [Hash] a Hash representation of the solr document
+      # @param solr_connector [RSolr::Client]  is an open connection with the solr core
+      # @param max_retries [Integer] the maximum number of tries before fail
       def self.add(id, solr_doc, solr_connector, max_retries = 10)
         process(id, solr_doc, solr_connector, max_retries, is_delete=false)  
       end
 
       # Add the document to solr, retry if an error occurs.
       # See https://github.com/ooyala/retries for docs on with_retries.
-      # @param [Hash] solr_doc that has only the id !{:id=>"ab123cd4567"}
-      # @param [RSolr::Client] solr_connector is an open connection with the solr core
-      # @param [Integer] max_retries the maximum number of tries before fail
-      def self.delete(id, solr_doc, solr_connector, max_retries = 10)
-        process(id, solr_doc, solr_connector, max_retries, is_delete=true)
+      # @param id [String] the document id, usually it will be druid.
+      # @param solr_connector[RSolr::Client]  is an open connection with the solr core
+      # @param max_retries [Integer] the maximum number of tries before fail
+      def self.delete(id, solr_connector, max_retries = 10)
+        process(id, {}, solr_connector, max_retries, is_delete=true)
       end
-      
+
+      # It's an internal method that receives all the requests and deal with
+      # SOLR core. This method can call add, delete, or update
+      #
+      # @param id [String] the document id, usually it will be druid.
+      # @param solr_doc [Hash] is the solr doc in hash format 
+      # @param solr_connector [RSolr::Client]  is an open connection with the solr core
+      # @param max_retries [Integer] the maximum number of tries before fail
       def self.process(id, solr_doc, solr_connector, max_retries, is_delete=false)
         handler = Proc.new do |exception, attempt_number, total_delay|
           DiscoveryIndexer::Logging.logger.debug "#{exception.class} on attempt #{attempt_number} for #{id}"
         end
-        
+          
         with_retries(:max_tries => max_retries, :handler => handler, :base_sleep_seconds => 1, :max_sleep_seconds => 5) do |attempt|
           DiscoveryIndexer::Logging.logger.debug "Attempt #{attempt} for #{id}"
-          
+        
           if is_delete
             solr_connector.delete_by_id(id)
             DiscoveryIndexer::Logging.logger.info "Successfully deleted #{id} on attempt #{attempt}"
+          elsif allow_update?(solr_connector) && doc_exists?(id,solr_connector)
+            update_solr_doc(id,solr_doc,solr_connector)
+            DiscoveryIndexer::Logging.logger.info "Successfully updated #{id} on attempt #{attempt}"
           else
             solr_connector.add(solr_doc)
             DiscoveryIndexer::Logging.logger.info "Successfully indexed #{id} on attempt #{attempt}"
           end
-          
+          solr_connector.commit
         end
-        solr_connector.commit
       end
-     
+
+      # @param solr_connector [RSolr::Client]  is an open connection with the solr core
+      # @return [Boolean] true if the solr core allowing update feature
+      def self.allow_update?(solr_connector)
+        return solr_connector.options.include?(:allow_update) ? solr_connector.options[:allow_update] : false
+      end
+
+      # @param id [String] the document id, usually it will be druid.
+      # @param solr_connector [RSolr::Client]  is an open connection with the solr core
+      # @return [Boolean] true if the solr doc defined by this id exists
+      def self.doc_exists?(id,solr_connector)
+        response=solr_connector.get 'select', :params=>{:q=>'id:"' + id + '"'}  
+        response['response']['numFound'] == 1
+      end
+      
+      # It is an internal method that updates the solr doc instead of adding a new one.
+      def self.update_solr_doc(id,solr_doc,solr_connector)
+        # update_solr_doc can't used RSolr because updating hash doc is not supported
+        #  so we need to build the json input manually
+        url="#{solr_connector.options[:url]}update?commit=true"
+        params="[{\"id\":\"#{id}\","
+        solr_doc.each do |field_name,new_values|
+          unless field_name == :id
+            params+="\"#{field_name}\":"
+            new_values=[new_values] unless new_values.class==Array
+            new_values = new_values.map {|s| s.to_s.gsub("\\","\\\\\\").gsub('"','\"').strip} # strip leading/trailing spaces and escape quotes for each value
+            params+="{\"set\":[\"#{new_values.join('","')}\"]},"
+          end
+        end
+        params.chomp!(',')
+        params+="}]"
+        RestClient.post url, params,:content_type => :json, :accept=>:json
+      end
+      
     end
   end
 end

data/lib/writer/solr_writer.rb

@@ -42,7 +42,7 @@ module DiscoveryIndexer
       def solr_delete_client(id, targets)
         targets.each do |solr_target|
           solr_connector = get_connector_for_target(solr_target)     
-          SolrClient.delete(id,{}, solr_connector)
+          SolrClient.delete(id, solr_connector)
         end         
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discovery-indexer
 version: !ruby/object:Gem::Version
-  version: '0.6'
+  version: 0.6.1
 platform: ruby
 authors:
 - Ahmed AlSum
@@ -14,112 +14,126 @@ dependencies:
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: stanford-mods
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: retries
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rsolr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This library manages the core operations for the discovery indexing such
@@ -140,6 +154,7 @@ files:
 - lib/reader/purlxml_parser.rb
 - lib/reader/purlxml_parser_strict.rb
 - lib/reader/purlxml_reader.rb
+- lib/utilities/extract_sub_targets.rb
 - lib/version.rb
 - lib/writer/solr_client.rb
 - lib/writer/solr_writer.rb
@@ -153,17 +168,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.2
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Shared library for the basic discovery indexing operation for Stanford DLSS.