harvestdor-indexer 1.0.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 45d033dffd56d8c3abd90ccafd0fa4666b4379da
-  data.tar.gz: f579c955fb390f2d7ed497aa237dbe763361d60b
+  metadata.gz: 8eb186d19cd3f8c6051dce74feaffa9d8bafcbe8
+  data.tar.gz: f8be614289308a8752c615b66a286139f45a684b
 SHA512:
-  metadata.gz: a7e9213e27145df273eaee8384b690e9ae720aeffd7f781f64dec9bc13948c6716b942499305037c7210c59ae9b5f07747f5ff67e4fe315577a78bce7d530a81
-  data.tar.gz: 896d1c26f157087e2bd6f935e0106024e9a7840cdb5c5db05e6a348ff1fe0579410bc70895b26b3db60a6461883bfd5bfe4a8ee43483d0f8821bf73cad4c2b1c
+  metadata.gz: 9949e15b1ebe035ee7317cbc9832e2b8fb3b8feaff155f5143f366a7d47829f8402f991fd78d5167d2a94c0f38994ff366eaff1f2b1d9bfa652b9ee2f8801d73
+  data.tar.gz: a382a7f2698a1a3001a4cb82a17fc5683d5a7d5305ba212717b1b92c758f4edc0c6564000f4ade2872ee8f0acc1dc7b552f9ccdf02f6bdc3749c2bcdc5deae79

data/README.rdoc

@@ -118,6 +118,7 @@ I suggest you run your code on harvestdor-dev, as it is already set up to be abl
 
 == Releases
 
+* <b>1.0.3</b> Implemented class level config so anything that inherits from Harvestdor::Indexer can share configuration settings
 * <b>1.0.0</b> Replaced OAI harvesting mechanism with dor-fetcher
 * <b>0.0.13</b> Upgrade to latest faraday HTTP client syntax; Use retries gem (https://github.com/ooyala/retries) to make retrying of index process more robust
 * <b>0.0.12</b> fix total_object nil error

data/harvestdor-indexer.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'retries'
   gem.add_dependency 'harvestdor', '>=0.0.14'
   gem.add_dependency 'stanford-mods'
-  gem.add_dependency 'dor-fetcher', '>=1.0.0'
+  gem.add_dependency 'dor-fetcher', '=1.0.5'
   
   # Runtime dependencies
   gem.add_runtime_dependency 'confstruct'

data/lib/harvestdor-indexer.rb

@@ -22,6 +22,10 @@ module Harvestdor
     attr_accessor :total_time_to_parse,:total_time_to_solr
     attr_accessor :dor_fetcher_client, :client_config
 
+
+    # Class level config variable
+    @@config ||= Confstruct::Configuration.new()
+
     def initialize yml_path, client_config_path, options = {}
       @success_count=0    # the number of objects successfully indexed
       @error_count=0      # the number of objects that failed
@@ -36,8 +40,14 @@ module Harvestdor
       @dor_fetcher_client=DorFetcher::Client.new({:service_url => client_config["dor_fetcher_service_url"]})
     end
 
+    # to allow class level access to config variables for record_merger and solr_doc_builder
+    #  (rather than passing a lot of params to constructor)
+    def self.config
+      @@config ||= Confstruct::Configuration.new()
+    end
+
     def config
-      @config ||= Confstruct::Configuration.new()
+      Indexer.config
     end
 
     def logger
@@ -232,7 +242,7 @@ module Harvestdor
     # Get only the druid from the end of the default_set string
     # from the yml file
     def strip_default_set_string()
-      @config.default_set.split('_').last
+      Indexer.config.default_set.split('_').last
     end
 
     protected #---------------------------------------------------------------------

data/lib/harvestdor-indexer/version.rb

@@ -1,6 +1,6 @@
 module Harvestdor
   class Indexer
     # this is the Ruby Gem version
-    VERSION = "1.0.0"
+    VERSION = "1.0.3"
   end
 end

data/spec/fixtures/vcr_cassettes/before_all_call.yml

@@ -0,0 +1,48 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*; q=0.5, application/xml'
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/xml; charset=utf-8
+      Etag:
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - e72ddd14-0ca2-4b1b-bad6-1ff7d4fc90d6
+      X-Runtime:
+      - '0.186248'
+      Connection:
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 18:46:07 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/cant_find_whitelist_call.yml

@@ -0,0 +1,48 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*; q=0.5, application/xml'
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/xml; charset=utf-8
+      Etag:
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - 3ee1c4c2-6bc7-41f3-a19a-6466c6f3ee6b
+      X-Runtime:
+      - '0.004152'
+      Connection:
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 18:48:31 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/empty_array_no_blacklist_config_call.yml

@@ -0,0 +1,48 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*; q=0.5, application/xml'
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/xml; charset=utf-8
+      Etag:
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - 1acd71cc-22d9-4c18-beda-193b44b07922
+      X-Runtime:
+      - '0.003821'
+      Connection:
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 18:53:12 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/empty_array_no_whitelist_config_call.yml

@@ -0,0 +1,48 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*; q=0.5, application/xml'
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/xml; charset=utf-8
+      Etag:
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - 02a8b8e5-2dae-469b-8613-3d5e75ea7564
+      X-Runtime:
+      - '0.006424'
+      Connection:
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 18:53:12 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/get_collection_druids_call.yml

@@ -2,7 +2,7 @@
 http_interactions:
 - request:
     method: get
-    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    uri: http://127.0.0.1:3000/collections/yg867hg1375
     body:
       encoding: US-ASCII
       string: ''
@@ -14,7 +14,7 @@ http_interactions:
   response:
     status:
       code: 200
-      message: 'OK '
+      message: OK
     headers:
       X-Frame-Options:
       - SAMEORIGIN
@@ -25,34 +25,32 @@ http_interactions:
       Content-Type:
       - application/json; charset=utf-8
       Etag:
-      - '"682afec57f678e4d153a5841b21395dd"'
+      - '"c3a375a17d64792857f21d1af227ed7b"'
       Cache-Control:
       - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
       X-Request-Id:
-      - 0954c447-9cb9-4eeb-8020-d87f13098f07
+      - 125a9964-6326-4114-9f59-fb533551d554
       X-Runtime:
-      - '0.006736'
-      Server:
-      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
-      Date:
-      - Wed, 22 Oct 2014 18:42:32 GMT
-      Content-Length:
-      - '1121'
+      - '0.011086'
       Connection:
-      - Keep-Alive
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
     body:
       encoding: US-ASCII
-      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
-        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+      string: '{"collections":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":"Francis
+        E. Stafford photographs, 1909-1933"}],"items":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":"Album
         A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
-        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        social customs, and people."},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":"Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":"Album
         D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        social customs, and people."},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":"Album
         C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
-        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        Press"},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":"Album
         B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+        social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
     http_version:
-  recorded_at: Wed, 22 Oct 2014 18:42:32 GMT
+  recorded_at: Wed, 12 Nov 2014 19:34:03 GMT
 recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/ignore_druids_in_blacklist_and_whitelist_call.yml

@@ -0,0 +1,48 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*; q=0.5, application/xml'
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/xml; charset=utf-8
+      Etag:
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - 7d9369f3-a2dc-4a1e-aa79-4d574628be4d
+      X-Runtime:
+      - '0.004594'
+      Connection:
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 18:46:07 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/ignore_druids_in_blacklist_call.yml

@@ -2,7 +2,7 @@
 http_interactions:
 - request:
     method: get
-    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    uri: http://127.0.0.1:3000/
     body:
       encoding: US-ASCII
       string: ''
@@ -14,7 +14,7 @@ http_interactions:
   response:
     status:
       code: 200
-      message: 'OK '
+      message: OK
     headers:
       X-Frame-Options:
       - SAMEORIGIN
@@ -23,36 +23,77 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       Content-Type:
-      - application/json; charset=utf-8
+      - text/html; charset=utf-8
       Etag:
-      - '"682afec57f678e4d153a5841b21395dd"'
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
       Cache-Control:
       - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
       X-Request-Id:
-      - 1e0232c6-fc39-49bf-b874-89567e225d00
+      - 994d795e-727d-4c83-aa8d-addfd5d1a01f
       X-Runtime:
-      - '0.006851'
+      - '0.010719'
+      Connection:
+      - close
       Server:
-      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
-      Date:
-      - Wed, 22 Oct 2014 18:53:15 GMT
-      Content-Length:
-      - '1121'
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 19:34:03 GMT
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/collections/yg867hg1375
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/json; charset=utf-8
+      Etag:
+      - '"c3a375a17d64792857f21d1af227ed7b"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - dc0faecb-c39d-42e4-bb1d-8f7be4b8c287
+      X-Runtime:
+      - '0.012066'
       Connection:
-      - Keep-Alive
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
     body:
       encoding: US-ASCII
-      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
-        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+      string: '{"collections":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":"Francis
+        E. Stafford photographs, 1909-1933"}],"items":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":"Album
         A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
-        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        social customs, and people."},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":"Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":"Album
         D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        social customs, and people."},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":"Album
         C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
-        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        Press"},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":"Album
         B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+        social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
     http_version:
-  recorded_at: Wed, 22 Oct 2014 18:53:15 GMT
+  recorded_at: Wed, 12 Nov 2014 19:34:03 GMT
 recorded_with: VCR 2.9.3