harvestdor-indexer 2.1.1 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -0
- data/.rubocop_todo.yml +2 -221
- data/README.rdoc +12 -40
- data/Rakefile +12 -12
- data/harvestdor-indexer.gemspec +16 -16
- data/lib/harvestdor/indexer/metrics.rb +11 -13
- data/lib/harvestdor/indexer/resource.rb +25 -19
- data/lib/harvestdor/indexer/solr.rb +1 -1
- data/lib/harvestdor/indexer/version.rb +1 -1
- data/lib/harvestdor/indexer.rb +38 -24
- data/spec/fixtures/vcr_cassettes/get_collection_druids_call.yml +112 -3
- data/spec/fixtures/vcr_cassettes/process_druids_whitelist_call.yml +113 -4
- data/spec/fixtures/vcr_cassettes/single_rsolr_connection_call.yml +112 -3
- data/spec/unit/harvestdor/indexer/metrics_spec.rb +13 -13
- data/spec/unit/harvestdor-indexer-resource_spec.rb +75 -47
- data/spec/unit/harvestdor-indexer-solr_spec.rb +11 -11
- data/spec/unit/harvestdor-indexer_spec.rb +67 -53
- metadata +2 -2
data/lib/harvestdor/indexer.rb
CHANGED
|
@@ -13,21 +13,21 @@ require 'dor-fetcher'
|
|
|
13
13
|
# stdlib
|
|
14
14
|
require 'logger'
|
|
15
15
|
|
|
16
|
-
require
|
|
16
|
+
require 'harvestdor/indexer/version'
|
|
17
17
|
|
|
18
18
|
require 'active_support/benchmarkable'
|
|
19
19
|
module Harvestdor
|
|
20
20
|
# Base class to harvest from DOR via harvestdor gem and then index
|
|
21
21
|
class Indexer
|
|
22
|
-
require
|
|
23
|
-
require
|
|
24
|
-
require
|
|
22
|
+
require 'harvestdor/indexer/metrics'
|
|
23
|
+
require 'harvestdor/indexer/resource'
|
|
24
|
+
require 'harvestdor/indexer/solr'
|
|
25
25
|
|
|
26
26
|
include ActiveSupport::Benchmarkable
|
|
27
27
|
|
|
28
28
|
attr_accessor :metrics, :logger
|
|
29
29
|
|
|
30
|
-
def initialize
|
|
30
|
+
def initialize(options = {})
|
|
31
31
|
config.configure(options)
|
|
32
32
|
yield(config) if block_given?
|
|
33
33
|
@metrics = Harvestdor::Indexer::Metrics.new logger: logger
|
|
@@ -52,8 +52,8 @@ module Harvestdor
|
|
|
52
52
|
# harvest the druids via DorFetcher
|
|
53
53
|
# create a Solr profiling document for each druid
|
|
54
54
|
# write the result to the Solr index
|
|
55
|
-
def harvest_and_index
|
|
56
|
-
benchmark
|
|
55
|
+
def harvest_and_index(each_options = { in_threads: 4 })
|
|
56
|
+
benchmark 'Harvest and Indexing' do
|
|
57
57
|
each_resource(each_options) do |resource|
|
|
58
58
|
index resource
|
|
59
59
|
end
|
|
@@ -62,16 +62,34 @@ module Harvestdor
|
|
|
62
62
|
end
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
+
##
|
|
66
|
+
# Collect all the explicitly identified resources (e.g. from the whitelist), and all
|
|
67
|
+
# members of any collections in that list, and offer them as an Enumerator.
|
|
68
|
+
#
|
|
69
|
+
# Using enumerators allows us to lazy-fetch and correctly garbage collect resources after
|
|
70
|
+
# a downstream consumer is finished processing them. If a consumer needs to use this resource
|
|
71
|
+
# list multiple times (and is confident all the resources will fit in available memory!), they
|
|
72
|
+
# could memoize the result of e.g. `#to_a` for their own use.
|
|
73
|
+
#
|
|
74
|
+
# @return [Enumerator] an enumerator of Harvestdor::Indexer::Resources for the druid whitelist,
|
|
75
|
+
# and all the items belonging to each collection id in druids.
|
|
65
76
|
def resources
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
77
|
+
return to_enum(:resources) unless block_given?
|
|
78
|
+
|
|
79
|
+
druids.each do |x|
|
|
80
|
+
# Include the named resource in the enumerable
|
|
81
|
+
resource = Harvestdor::Indexer::Resource.new(self, x)
|
|
82
|
+
yield resource
|
|
83
|
+
|
|
84
|
+
# And also yield any members of that resources
|
|
85
|
+
resource.items.each do |coll_member|
|
|
86
|
+
yield coll_member
|
|
87
|
+
end
|
|
88
|
+
end
|
|
71
89
|
end
|
|
72
90
|
|
|
73
|
-
def each_resource
|
|
74
|
-
benchmark
|
|
91
|
+
def each_resource(options = {}, &_block)
|
|
92
|
+
benchmark '' do
|
|
75
93
|
Parallel.each(resources, options) do |resource|
|
|
76
94
|
metrics.tally on_error: method(:resource_error) do
|
|
77
95
|
yield resource
|
|
@@ -84,9 +102,9 @@ module Harvestdor
|
|
|
84
102
|
logger.info("Total records processed: #{metrics.total}")
|
|
85
103
|
end
|
|
86
104
|
|
|
87
|
-
def resource_error
|
|
88
|
-
if e.instance_of?
|
|
89
|
-
|
|
105
|
+
def resource_error(e)
|
|
106
|
+
if e.instance_of?(Parallel::Break) || e.instance_of?(Parallel::Kill)
|
|
107
|
+
fail e
|
|
90
108
|
end
|
|
91
109
|
end
|
|
92
110
|
|
|
@@ -98,8 +116,7 @@ module Harvestdor
|
|
|
98
116
|
|
|
99
117
|
# create Solr doc for the druid and add it to Solr
|
|
100
118
|
# NOTE: don't forget to send commit to Solr, either once at end (already in harvest_and_index), or for each add, or ...
|
|
101
|
-
def index
|
|
102
|
-
|
|
119
|
+
def index(resource)
|
|
103
120
|
benchmark "Indexing #{resource.druid}" do
|
|
104
121
|
logger.debug "About to index #{resource.druid}"
|
|
105
122
|
doc_hash = {}
|
|
@@ -145,12 +162,9 @@ module Harvestdor
|
|
|
145
162
|
# @param [String] path - path of file containing a list of druids
|
|
146
163
|
# @return [Array<String>] an Array of druids
|
|
147
164
|
def load_id_list(path)
|
|
148
|
-
list = File.open(path).each_line
|
|
149
|
-
.map { |line| line.strip }
|
|
150
|
-
.reject { |line| line.strip.start_with?('#') }
|
|
151
|
-
.reject { |line| line.empty? }
|
|
165
|
+
list = File.open(path).each_line.map(&:strip).reject { |line| line.strip.start_with?('#') }.reject(&:empty?)
|
|
152
166
|
rescue
|
|
153
|
-
msg =
|
|
167
|
+
msg = 'Unable to find list of druids at ' + path
|
|
154
168
|
logger.fatal msg
|
|
155
169
|
raise msg
|
|
156
170
|
end
|
|
@@ -8,7 +8,7 @@ http_interactions:
|
|
|
8
8
|
string: ''
|
|
9
9
|
headers:
|
|
10
10
|
Accept:
|
|
11
|
-
-
|
|
11
|
+
- "*/*"
|
|
12
12
|
User-Agent:
|
|
13
13
|
- Ruby
|
|
14
14
|
response:
|
|
@@ -51,7 +51,7 @@ http_interactions:
|
|
|
51
51
|
Press"},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":"Album
|
|
52
52
|
B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
|
|
53
53
|
social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
|
|
54
|
-
http_version:
|
|
54
|
+
http_version:
|
|
55
55
|
recorded_at: Wed, 12 Nov 2014 19:34:03 GMT
|
|
56
56
|
- request:
|
|
57
57
|
method: get
|
|
@@ -147,6 +147,115 @@ http_interactions:
|
|
|
147
147
|
<dc:coverage>China</dc:coverage>
|
|
148
148
|
</oai_dc:dc>
|
|
149
149
|
</publicObject>
|
|
150
|
-
http_version:
|
|
150
|
+
http_version:
|
|
151
151
|
recorded_at: Wed, 17 Dec 2014 19:39:38 GMT
|
|
152
|
+
- request:
|
|
153
|
+
method: get
|
|
154
|
+
uri: https://purl.stanford.edu/yg867hg1375.xml
|
|
155
|
+
body:
|
|
156
|
+
encoding: US-ASCII
|
|
157
|
+
string: ''
|
|
158
|
+
headers:
|
|
159
|
+
Accept-Encoding:
|
|
160
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
|
161
|
+
Accept:
|
|
162
|
+
- "*/*"
|
|
163
|
+
User-Agent:
|
|
164
|
+
- Ruby
|
|
165
|
+
response:
|
|
166
|
+
status:
|
|
167
|
+
code: 200
|
|
168
|
+
message: OK
|
|
169
|
+
headers:
|
|
170
|
+
Date:
|
|
171
|
+
- Fri, 30 Oct 2015 14:51:05 GMT
|
|
172
|
+
Server:
|
|
173
|
+
- Apache/2.2.15 (CentOS)
|
|
174
|
+
Cache-Control:
|
|
175
|
+
- max-age=0, private, must-revalidate
|
|
176
|
+
Etag:
|
|
177
|
+
- '"fc30e6f59dacce0a53e4578a4be37a3c"'
|
|
178
|
+
X-Frame-Options:
|
|
179
|
+
- SAMEORIGIN
|
|
180
|
+
X-Xss-Protection:
|
|
181
|
+
- 1; mode=block
|
|
182
|
+
X-Content-Type-Options:
|
|
183
|
+
- nosniff
|
|
184
|
+
X-Runtime:
|
|
185
|
+
- '0.009128'
|
|
186
|
+
X-Request-Id:
|
|
187
|
+
- 413224ab-b387-4d91-bbcc-f44d6cfb7273
|
|
188
|
+
X-Powered-By:
|
|
189
|
+
- Phusion Passenger 5.0.16
|
|
190
|
+
Strict-Transport-Security:
|
|
191
|
+
- max-age=31536000; includeSubDomains
|
|
192
|
+
Last-Modified:
|
|
193
|
+
- Thu, 01 Oct 2015 19:20:04 GMT
|
|
194
|
+
Status:
|
|
195
|
+
- 200 OK
|
|
196
|
+
Transfer-Encoding:
|
|
197
|
+
- chunked
|
|
198
|
+
Content-Type:
|
|
199
|
+
- application/xml; charset=utf-8
|
|
200
|
+
body:
|
|
201
|
+
encoding: UTF-8
|
|
202
|
+
string: |
|
|
203
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
204
|
+
<publicObject id="druid:yg867hg1375" published="2015-10-01T12:20:04-07:00">
|
|
205
|
+
<identityMetadata>
|
|
206
|
+
<objectId>druid:yg867hg1375</objectId>
|
|
207
|
+
<objectCreator>DOR</objectCreator>
|
|
208
|
+
<objectLabel>Francis E. Stafford photographs, 1909-1933</objectLabel>
|
|
209
|
+
<objectType>collection</objectType>
|
|
210
|
+
<adminPolicy>druid:vb546ms7107</adminPolicy>
|
|
211
|
+
<otherId name="catkey">9615156</otherId>
|
|
212
|
+
<otherId name="uuid">8f1feb20-4b29-11e3-8e31-0050569b3c3c</otherId>
|
|
213
|
+
<tag>Remediated By : 4.17.1</tag>
|
|
214
|
+
<displayType>image</displayType>
|
|
215
|
+
<release to="Searchworks">true</release>
|
|
216
|
+
</identityMetadata>
|
|
217
|
+
<xml/>
|
|
218
|
+
<rightsMetadata>
|
|
219
|
+
<access type="discover">
|
|
220
|
+
<machine>
|
|
221
|
+
<world/>
|
|
222
|
+
</machine>
|
|
223
|
+
</access>
|
|
224
|
+
<access type="read">
|
|
225
|
+
<machine>
|
|
226
|
+
<world/>
|
|
227
|
+
</machine>
|
|
228
|
+
</access>
|
|
229
|
+
<use>
|
|
230
|
+
<human type="useAndReproduction"/>
|
|
231
|
+
<human type="creativeCommons"/>
|
|
232
|
+
<machine type="creativeCommons"/>
|
|
233
|
+
</use>
|
|
234
|
+
<copyright>
|
|
235
|
+
<human type="copyright">For copyright status, please contact the Hoover Institution Archives at archives@hoover.stanford.edu.</human>
|
|
236
|
+
</copyright>
|
|
237
|
+
</rightsMetadata>
|
|
238
|
+
<rdf:RDF xmlns:fedora-model="info:fedora/fedora-system:def/model#" xmlns:hydra="http://projecthydra.org/ns/relations#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
|
239
|
+
<rdf:Description rdf:about="info:fedora/druid:yg867hg1375">
|
|
240
|
+
</rdf:Description>
|
|
241
|
+
</rdf:RDF>
|
|
242
|
+
<oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:srw_dc="info:srw/schema/1/dc-schema" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
|
|
243
|
+
<dc:title>Francis E. Stafford photographs, 1909-1933</dc:title>
|
|
244
|
+
<dc:contributor>Stafford, Francis E., 1884-1938</dc:contributor>
|
|
245
|
+
<dc:type>Collection</dc:type>
|
|
246
|
+
<dc:date>1909-1933</dc:date>
|
|
247
|
+
<dc:language>und</dc:language>
|
|
248
|
+
<dc:format>3 oversize boxes.</dc:format>
|
|
249
|
+
<dc:description>Photographs of scenes in China.</dc:description>
|
|
250
|
+
<dc:rights>Closed. Digital use copies available.</dc:rights>
|
|
251
|
+
<dc:description type="biographical/historical">American missionary in China, 1909-1915 and 1932-1933.</dc:description>
|
|
252
|
+
<dc:description>Inventory.</dc:description>
|
|
253
|
+
<dc:coverage>China</dc:coverage>
|
|
254
|
+
</oai_dc:dc>
|
|
255
|
+
<releaseData>
|
|
256
|
+
<release to="Searchworks">true</release>
|
|
257
|
+
</releaseData>
|
|
258
|
+
</publicObject>
|
|
259
|
+
http_version:
|
|
260
|
+
recorded_at: Fri, 30 Oct 2015 14:51:05 GMT
|
|
152
261
|
recorded_with: VCR 2.9.3
|
|
@@ -94,7 +94,7 @@ http_interactions:
|
|
|
94
94
|
<dc:coverage>China</dc:coverage>
|
|
95
95
|
</oai_dc:dc>
|
|
96
96
|
</publicObject>
|
|
97
|
-
http_version:
|
|
97
|
+
http_version:
|
|
98
98
|
recorded_at: Wed, 17 Dec 2014 19:44:13 GMT
|
|
99
99
|
- request:
|
|
100
100
|
method: get
|
|
@@ -155,7 +155,7 @@ http_interactions:
|
|
|
155
155
|
social customs, and people."},{"druid":"druid:jf275fd6276","latest_change":"2013-11-26T00:11:17Z","title":"Album
|
|
156
156
|
A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
|
|
157
157
|
social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
|
|
158
|
-
http_version:
|
|
158
|
+
http_version:
|
|
159
159
|
recorded_at: Wed, 17 Dec 2014 19:44:14 GMT
|
|
160
160
|
- request:
|
|
161
161
|
method: get
|
|
@@ -853,7 +853,7 @@ http_interactions:
|
|
|
853
853
|
<dc:relation type="collection">Francis E. Stafford photographs, 1909-1933</dc:relation>
|
|
854
854
|
</oai_dc:dc>
|
|
855
855
|
</publicObject>
|
|
856
|
-
http_version:
|
|
856
|
+
http_version:
|
|
857
857
|
recorded_at: Wed, 17 Dec 2014 19:44:14 GMT
|
|
858
858
|
- request:
|
|
859
859
|
method: get
|
|
@@ -1521,6 +1521,115 @@ http_interactions:
|
|
|
1521
1521
|
<dc:relation type="collection">Francis E. Stafford photographs, 1909-1933</dc:relation>
|
|
1522
1522
|
</oai_dc:dc>
|
|
1523
1523
|
</publicObject>
|
|
1524
|
-
http_version:
|
|
1524
|
+
http_version:
|
|
1525
1525
|
recorded_at: Wed, 17 Dec 2014 19:44:14 GMT
|
|
1526
|
+
- request:
|
|
1527
|
+
method: get
|
|
1528
|
+
uri: https://purl.stanford.edu/yg867hg1375.xml
|
|
1529
|
+
body:
|
|
1530
|
+
encoding: US-ASCII
|
|
1531
|
+
string: ''
|
|
1532
|
+
headers:
|
|
1533
|
+
Accept-Encoding:
|
|
1534
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
|
1535
|
+
Accept:
|
|
1536
|
+
- "*/*"
|
|
1537
|
+
User-Agent:
|
|
1538
|
+
- Ruby
|
|
1539
|
+
response:
|
|
1540
|
+
status:
|
|
1541
|
+
code: 200
|
|
1542
|
+
message: OK
|
|
1543
|
+
headers:
|
|
1544
|
+
Date:
|
|
1545
|
+
- Fri, 30 Oct 2015 14:50:39 GMT
|
|
1546
|
+
Server:
|
|
1547
|
+
- Apache/2.2.15 (CentOS)
|
|
1548
|
+
Cache-Control:
|
|
1549
|
+
- max-age=0, private, must-revalidate
|
|
1550
|
+
Etag:
|
|
1551
|
+
- '"fc30e6f59dacce0a53e4578a4be37a3c"'
|
|
1552
|
+
X-Frame-Options:
|
|
1553
|
+
- SAMEORIGIN
|
|
1554
|
+
X-Xss-Protection:
|
|
1555
|
+
- 1; mode=block
|
|
1556
|
+
X-Content-Type-Options:
|
|
1557
|
+
- nosniff
|
|
1558
|
+
X-Runtime:
|
|
1559
|
+
- '0.026943'
|
|
1560
|
+
X-Request-Id:
|
|
1561
|
+
- 55c754b2-524c-4302-9c92-50d673e8988f
|
|
1562
|
+
X-Powered-By:
|
|
1563
|
+
- Phusion Passenger 5.0.16
|
|
1564
|
+
Strict-Transport-Security:
|
|
1565
|
+
- max-age=31536000; includeSubDomains
|
|
1566
|
+
Last-Modified:
|
|
1567
|
+
- Thu, 01 Oct 2015 19:20:04 GMT
|
|
1568
|
+
Status:
|
|
1569
|
+
- 200 OK
|
|
1570
|
+
Transfer-Encoding:
|
|
1571
|
+
- chunked
|
|
1572
|
+
Content-Type:
|
|
1573
|
+
- application/xml; charset=utf-8
|
|
1574
|
+
body:
|
|
1575
|
+
encoding: UTF-8
|
|
1576
|
+
string: |
|
|
1577
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
1578
|
+
<publicObject id="druid:yg867hg1375" published="2015-10-01T12:20:04-07:00">
|
|
1579
|
+
<identityMetadata>
|
|
1580
|
+
<objectId>druid:yg867hg1375</objectId>
|
|
1581
|
+
<objectCreator>DOR</objectCreator>
|
|
1582
|
+
<objectLabel>Francis E. Stafford photographs, 1909-1933</objectLabel>
|
|
1583
|
+
<objectType>collection</objectType>
|
|
1584
|
+
<adminPolicy>druid:vb546ms7107</adminPolicy>
|
|
1585
|
+
<otherId name="catkey">9615156</otherId>
|
|
1586
|
+
<otherId name="uuid">8f1feb20-4b29-11e3-8e31-0050569b3c3c</otherId>
|
|
1587
|
+
<tag>Remediated By : 4.17.1</tag>
|
|
1588
|
+
<displayType>image</displayType>
|
|
1589
|
+
<release to="Searchworks">true</release>
|
|
1590
|
+
</identityMetadata>
|
|
1591
|
+
<xml/>
|
|
1592
|
+
<rightsMetadata>
|
|
1593
|
+
<access type="discover">
|
|
1594
|
+
<machine>
|
|
1595
|
+
<world/>
|
|
1596
|
+
</machine>
|
|
1597
|
+
</access>
|
|
1598
|
+
<access type="read">
|
|
1599
|
+
<machine>
|
|
1600
|
+
<world/>
|
|
1601
|
+
</machine>
|
|
1602
|
+
</access>
|
|
1603
|
+
<use>
|
|
1604
|
+
<human type="useAndReproduction"/>
|
|
1605
|
+
<human type="creativeCommons"/>
|
|
1606
|
+
<machine type="creativeCommons"/>
|
|
1607
|
+
</use>
|
|
1608
|
+
<copyright>
|
|
1609
|
+
<human type="copyright">For copyright status, please contact the Hoover Institution Archives at archives@hoover.stanford.edu.</human>
|
|
1610
|
+
</copyright>
|
|
1611
|
+
</rightsMetadata>
|
|
1612
|
+
<rdf:RDF xmlns:fedora-model="info:fedora/fedora-system:def/model#" xmlns:hydra="http://projecthydra.org/ns/relations#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
|
1613
|
+
<rdf:Description rdf:about="info:fedora/druid:yg867hg1375">
|
|
1614
|
+
</rdf:Description>
|
|
1615
|
+
</rdf:RDF>
|
|
1616
|
+
<oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:srw_dc="info:srw/schema/1/dc-schema" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
|
|
1617
|
+
<dc:title>Francis E. Stafford photographs, 1909-1933</dc:title>
|
|
1618
|
+
<dc:contributor>Stafford, Francis E., 1884-1938</dc:contributor>
|
|
1619
|
+
<dc:type>Collection</dc:type>
|
|
1620
|
+
<dc:date>1909-1933</dc:date>
|
|
1621
|
+
<dc:language>und</dc:language>
|
|
1622
|
+
<dc:format>3 oversize boxes.</dc:format>
|
|
1623
|
+
<dc:description>Photographs of scenes in China.</dc:description>
|
|
1624
|
+
<dc:rights>Closed. Digital use copies available.</dc:rights>
|
|
1625
|
+
<dc:description type="biographical/historical">American missionary in China, 1909-1915 and 1932-1933.</dc:description>
|
|
1626
|
+
<dc:description>Inventory.</dc:description>
|
|
1627
|
+
<dc:coverage>China</dc:coverage>
|
|
1628
|
+
</oai_dc:dc>
|
|
1629
|
+
<releaseData>
|
|
1630
|
+
<release to="Searchworks">true</release>
|
|
1631
|
+
</releaseData>
|
|
1632
|
+
</publicObject>
|
|
1633
|
+
http_version:
|
|
1634
|
+
recorded_at: Fri, 30 Oct 2015 14:50:39 GMT
|
|
1526
1635
|
recorded_with: VCR 2.9.3
|
|
@@ -94,7 +94,7 @@ http_interactions:
|
|
|
94
94
|
<dc:coverage>China</dc:coverage>
|
|
95
95
|
</oai_dc:dc>
|
|
96
96
|
</publicObject>
|
|
97
|
-
http_version:
|
|
97
|
+
http_version:
|
|
98
98
|
recorded_at: Wed, 17 Dec 2014 19:39:38 GMT
|
|
99
99
|
- request:
|
|
100
100
|
method: get
|
|
@@ -104,7 +104,7 @@ http_interactions:
|
|
|
104
104
|
string: ''
|
|
105
105
|
headers:
|
|
106
106
|
Accept:
|
|
107
|
-
-
|
|
107
|
+
- "*/*"
|
|
108
108
|
User-Agent:
|
|
109
109
|
- Ruby
|
|
110
110
|
response:
|
|
@@ -147,6 +147,115 @@ http_interactions:
|
|
|
147
147
|
Press"},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":"Album
|
|
148
148
|
B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
|
|
149
149
|
social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
|
|
150
|
-
http_version:
|
|
150
|
+
http_version:
|
|
151
151
|
recorded_at: Wed, 12 Nov 2014 19:34:03 GMT
|
|
152
|
+
- request:
|
|
153
|
+
method: get
|
|
154
|
+
uri: https://purl.stanford.edu/yg867hg1375.xml
|
|
155
|
+
body:
|
|
156
|
+
encoding: US-ASCII
|
|
157
|
+
string: ''
|
|
158
|
+
headers:
|
|
159
|
+
Accept-Encoding:
|
|
160
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
|
161
|
+
Accept:
|
|
162
|
+
- "*/*"
|
|
163
|
+
User-Agent:
|
|
164
|
+
- Ruby
|
|
165
|
+
response:
|
|
166
|
+
status:
|
|
167
|
+
code: 200
|
|
168
|
+
message: OK
|
|
169
|
+
headers:
|
|
170
|
+
Date:
|
|
171
|
+
- Fri, 30 Oct 2015 14:50:57 GMT
|
|
172
|
+
Server:
|
|
173
|
+
- Apache/2.2.15 (CentOS)
|
|
174
|
+
Cache-Control:
|
|
175
|
+
- max-age=0, private, must-revalidate
|
|
176
|
+
Etag:
|
|
177
|
+
- '"fc30e6f59dacce0a53e4578a4be37a3c"'
|
|
178
|
+
X-Frame-Options:
|
|
179
|
+
- SAMEORIGIN
|
|
180
|
+
X-Xss-Protection:
|
|
181
|
+
- 1; mode=block
|
|
182
|
+
X-Content-Type-Options:
|
|
183
|
+
- nosniff
|
|
184
|
+
X-Runtime:
|
|
185
|
+
- '0.022172'
|
|
186
|
+
X-Request-Id:
|
|
187
|
+
- 264e8877-5ed0-46cb-ae15-a0fc6f0bffd6
|
|
188
|
+
X-Powered-By:
|
|
189
|
+
- Phusion Passenger 5.0.16
|
|
190
|
+
Strict-Transport-Security:
|
|
191
|
+
- max-age=31536000; includeSubDomains
|
|
192
|
+
Last-Modified:
|
|
193
|
+
- Thu, 01 Oct 2015 19:20:04 GMT
|
|
194
|
+
Status:
|
|
195
|
+
- 200 OK
|
|
196
|
+
Transfer-Encoding:
|
|
197
|
+
- chunked
|
|
198
|
+
Content-Type:
|
|
199
|
+
- application/xml; charset=utf-8
|
|
200
|
+
body:
|
|
201
|
+
encoding: UTF-8
|
|
202
|
+
string: |
|
|
203
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
204
|
+
<publicObject id="druid:yg867hg1375" published="2015-10-01T12:20:04-07:00">
|
|
205
|
+
<identityMetadata>
|
|
206
|
+
<objectId>druid:yg867hg1375</objectId>
|
|
207
|
+
<objectCreator>DOR</objectCreator>
|
|
208
|
+
<objectLabel>Francis E. Stafford photographs, 1909-1933</objectLabel>
|
|
209
|
+
<objectType>collection</objectType>
|
|
210
|
+
<adminPolicy>druid:vb546ms7107</adminPolicy>
|
|
211
|
+
<otherId name="catkey">9615156</otherId>
|
|
212
|
+
<otherId name="uuid">8f1feb20-4b29-11e3-8e31-0050569b3c3c</otherId>
|
|
213
|
+
<tag>Remediated By : 4.17.1</tag>
|
|
214
|
+
<displayType>image</displayType>
|
|
215
|
+
<release to="Searchworks">true</release>
|
|
216
|
+
</identityMetadata>
|
|
217
|
+
<xml/>
|
|
218
|
+
<rightsMetadata>
|
|
219
|
+
<access type="discover">
|
|
220
|
+
<machine>
|
|
221
|
+
<world/>
|
|
222
|
+
</machine>
|
|
223
|
+
</access>
|
|
224
|
+
<access type="read">
|
|
225
|
+
<machine>
|
|
226
|
+
<world/>
|
|
227
|
+
</machine>
|
|
228
|
+
</access>
|
|
229
|
+
<use>
|
|
230
|
+
<human type="useAndReproduction"/>
|
|
231
|
+
<human type="creativeCommons"/>
|
|
232
|
+
<machine type="creativeCommons"/>
|
|
233
|
+
</use>
|
|
234
|
+
<copyright>
|
|
235
|
+
<human type="copyright">For copyright status, please contact the Hoover Institution Archives at archives@hoover.stanford.edu.</human>
|
|
236
|
+
</copyright>
|
|
237
|
+
</rightsMetadata>
|
|
238
|
+
<rdf:RDF xmlns:fedora-model="info:fedora/fedora-system:def/model#" xmlns:hydra="http://projecthydra.org/ns/relations#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
|
239
|
+
<rdf:Description rdf:about="info:fedora/druid:yg867hg1375">
|
|
240
|
+
</rdf:Description>
|
|
241
|
+
</rdf:RDF>
|
|
242
|
+
<oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:srw_dc="info:srw/schema/1/dc-schema" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
|
|
243
|
+
<dc:title>Francis E. Stafford photographs, 1909-1933</dc:title>
|
|
244
|
+
<dc:contributor>Stafford, Francis E., 1884-1938</dc:contributor>
|
|
245
|
+
<dc:type>Collection</dc:type>
|
|
246
|
+
<dc:date>1909-1933</dc:date>
|
|
247
|
+
<dc:language>und</dc:language>
|
|
248
|
+
<dc:format>3 oversize boxes.</dc:format>
|
|
249
|
+
<dc:description>Photographs of scenes in China.</dc:description>
|
|
250
|
+
<dc:rights>Closed. Digital use copies available.</dc:rights>
|
|
251
|
+
<dc:description type="biographical/historical">American missionary in China, 1909-1915 and 1932-1933.</dc:description>
|
|
252
|
+
<dc:description>Inventory.</dc:description>
|
|
253
|
+
<dc:coverage>China</dc:coverage>
|
|
254
|
+
</oai_dc:dc>
|
|
255
|
+
<releaseData>
|
|
256
|
+
<release to="Searchworks">true</release>
|
|
257
|
+
</releaseData>
|
|
258
|
+
</publicObject>
|
|
259
|
+
http_version:
|
|
260
|
+
recorded_at: Fri, 30 Oct 2015 14:50:57 GMT
|
|
152
261
|
recorded_with: VCR 2.9.3
|
|
@@ -1,45 +1,45 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe Harvestdor::Indexer::Metrics do
|
|
4
|
-
it
|
|
4
|
+
it 'records successes' do
|
|
5
5
|
expect { subject.success! }.to change { subject.success_count }.from(0).to(1)
|
|
6
6
|
end
|
|
7
7
|
|
|
8
|
-
it
|
|
8
|
+
it 'records errors' do
|
|
9
9
|
expect { subject.error! }.to change { subject.error_count }.from(0).to(1)
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
-
describe
|
|
13
|
-
it
|
|
14
|
-
expect do
|
|
12
|
+
describe '#total' do
|
|
13
|
+
it 'is the sum of the successes and errors' do
|
|
14
|
+
expect do
|
|
15
15
|
subject.error!
|
|
16
16
|
subject.success!
|
|
17
17
|
end.to change { subject.total }.from(0).to(2)
|
|
18
18
|
end
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
describe
|
|
22
|
-
it "
|
|
21
|
+
describe '#tally' do
|
|
22
|
+
it "records a success if the block doesn't fail" do
|
|
23
23
|
expect do
|
|
24
24
|
subject.tally do
|
|
25
|
-
#noop
|
|
25
|
+
# noop
|
|
26
26
|
end
|
|
27
27
|
end.to change { subject.success_count }.from(0).to(1)
|
|
28
28
|
end
|
|
29
|
-
|
|
30
|
-
it
|
|
29
|
+
|
|
30
|
+
it 'records an error if the block fails' do
|
|
31
31
|
expect do
|
|
32
32
|
subject.tally do
|
|
33
|
-
|
|
33
|
+
fail 'Broken'
|
|
34
34
|
end
|
|
35
35
|
end.to change { subject.error_count }.from(0).to(1)
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
-
it
|
|
38
|
+
it 'allows an error handler to be provided' do
|
|
39
39
|
x = double
|
|
40
40
|
expect(x).to receive(:call).with(kind_of(RuntimeError))
|
|
41
41
|
subject.tally(on_error: x) do
|
|
42
|
-
|
|
42
|
+
fail 'Broken'
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
45
|
end
|