relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/lib/relaton/3gpp/bibliography.rb +19 -6
  3. data/lib/relaton/3gpp/processor.rb +6 -0
  4. data/lib/relaton/adobe/processor.rb +9 -0
  5. data/lib/relaton/bib/converter/csl.rb +110 -0
  6. data/lib/relaton/bib/converter/ris.rb +104 -0
  7. data/lib/relaton/bib/converter/titles.rb +24 -0
  8. data/lib/relaton/bib/item_data.rb +23 -0
  9. data/lib/relaton/bib/model/item.rb +6 -6
  10. data/lib/relaton/bib/sanitizer.rb +71 -30
  11. data/lib/relaton/bib.rb +10 -0
  12. data/lib/relaton/bipm/bibliography.rb +24 -21
  13. data/lib/relaton/bipm/processor.rb +1 -0
  14. data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +6 -6
  15. data/lib/relaton/bipm/si_brochure_parser.rb +2 -2
  16. data/lib/relaton/bsi/bibliography.rb +26 -23
  17. data/lib/relaton/bsi/processor.rb +5 -0
  18. data/lib/relaton/calconnect/bibliography.rb +6 -6
  19. data/lib/relaton/calconnect/hit_collection.rb +4 -1
  20. data/lib/relaton/ccsds/bibliography.rb +16 -10
  21. data/lib/relaton/ccsds/hit_collection.rb +1 -1
  22. data/lib/relaton/ccsds/processor.rb +13 -0
  23. data/lib/relaton/cen/bibliography.rb +27 -24
  24. data/lib/relaton/cen/hit_collection.rb +1 -1
  25. data/lib/relaton/cen/processor.rb +5 -0
  26. data/lib/relaton/cen/scraper.rb +9 -9
  27. data/lib/relaton/cie/bibliography.rb +11 -9
  28. data/lib/relaton/cie/data_fetcher.rb +18 -18
  29. data/lib/relaton/cie/processor.rb +1 -0
  30. data/lib/relaton/cie/scrapper.rb +15 -8
  31. data/lib/relaton/cie.rb +1 -1
  32. data/lib/relaton/cloud.rb +127 -0
  33. data/lib/relaton/core/hit_collection.rb +6 -10
  34. data/lib/relaton/core/processor.rb +121 -0
  35. data/lib/relaton/core/request_error.rb +21 -3
  36. data/lib/relaton/db/cache.rb +474 -148
  37. data/lib/relaton/db/cache_entry.rb +42 -0
  38. data/lib/relaton/db/registry.rb +152 -0
  39. data/lib/relaton/db.rb +275 -102
  40. data/lib/relaton/doi/crossref.rb +23 -6
  41. data/lib/relaton/doi/processor.rb +1 -0
  42. data/lib/relaton/easc/processor.rb +1 -0
  43. data/lib/relaton/ecma/bibliography.rb +18 -13
  44. data/lib/relaton/ecma/data_fetcher.rb +1 -1
  45. data/lib/relaton/ecma/data_parser.rb +1 -1
  46. data/lib/relaton/ecma/edition_parser.rb +2 -2
  47. data/lib/relaton/ecma/memento_parser.rb +4 -4
  48. data/lib/relaton/ecma/standard_parser.rb +6 -6
  49. data/lib/relaton/etsi/bibliography.rb +8 -7
  50. data/lib/relaton/etsi/processor.rb +1 -0
  51. data/lib/relaton/gb/bibliography.rb +20 -12
  52. data/lib/relaton/gb/gb_scraper.rb +5 -5
  53. data/lib/relaton/gb/scraper.rb +16 -16
  54. data/lib/relaton/gb/sec_scraper.rb +8 -8
  55. data/lib/relaton/gb/t_scraper.rb +5 -5
  56. data/lib/relaton/gost/processor.rb +1 -0
  57. data/lib/relaton/iala/bibliography.rb +15 -12
  58. data/lib/relaton/iala/processor.rb +1 -0
  59. data/lib/relaton/iana/bibliography.rb +12 -12
  60. data/lib/relaton/iana/data_fetcher.rb +3 -3
  61. data/lib/relaton/iana/parser.rb +12 -5
  62. data/lib/relaton/iana/processor.rb +9 -0
  63. data/lib/relaton/iec/bibliography.rb +18 -7
  64. data/lib/relaton/iec/data_parser.rb +24 -9
  65. data/lib/relaton/iec/processor.rb +11 -0
  66. data/lib/relaton/iec.rb +1 -1
  67. data/lib/relaton/ieee/bibliography.rb +19 -13
  68. data/lib/relaton/ieee/data_fetcher.rb +1 -1
  69. data/lib/relaton/ieee/processor.rb +24 -0
  70. data/lib/relaton/ieee/rawbib_id_parser.rb +2 -2
  71. data/lib/relaton/ietf/bibliography.rb +11 -9
  72. data/lib/relaton/ietf/data_fetcher.rb +1 -1
  73. data/lib/relaton/ietf/processor.rb +1 -0
  74. data/lib/relaton/ietf/rfc/entry.rb +15 -19
  75. data/lib/relaton/ietf/scraper.rb +5 -4
  76. data/lib/relaton/iho/processor.rb +1 -0
  77. data/lib/relaton/index/file_io.rb +2 -2
  78. data/lib/relaton/index/pool.rb +4 -3
  79. data/lib/relaton/index/shard_source.rb +1 -1
  80. data/lib/relaton/index/type.rb +2 -5
  81. data/lib/relaton/isbn/open_library.rb +11 -7
  82. data/lib/relaton/isbn/processor.rb +10 -0
  83. data/lib/relaton/iso/bibliography.rb +11 -9
  84. data/lib/relaton/iso/data_parser.rb +2 -2
  85. data/lib/relaton/iso/processor.rb +5 -0
  86. data/lib/relaton/iso/scraper.rb +20 -20
  87. data/lib/relaton/itu/bibliography.rb +107 -16
  88. data/lib/relaton/itu/data_crawler_r.rb +2 -2
  89. data/lib/relaton/itu/hit_collection.rb +64 -52
  90. data/lib/relaton/itu/processor.rb +1 -0
  91. data/lib/relaton/itu/scraper.rb +13 -3
  92. data/lib/relaton/itu.rb +0 -2
  93. data/lib/relaton/jis/bibliography.rb +22 -11
  94. data/lib/relaton/jis/data_fetcher.rb +4 -4
  95. data/lib/relaton/jis/processor.rb +1 -0
  96. data/lib/relaton/jis/scraper.rb +8 -8
  97. data/lib/relaton/nist/bibliography.rb +25 -21
  98. data/lib/relaton/oasis/bibliography.rb +16 -13
  99. data/lib/relaton/oasis/browser_agent.rb +2 -2
  100. data/lib/relaton/oasis/data_parser.rb +5 -5
  101. data/lib/relaton/oasis/data_parser_utils.rb +2 -2
  102. data/lib/relaton/oasis/data_part_parser.rb +7 -7
  103. data/lib/relaton/ogc/bibliography.rb +15 -14
  104. data/lib/relaton/ogc/hit_collection.rb +9 -7
  105. data/lib/relaton/ogc/processor.rb +13 -0
  106. data/lib/relaton/oiml/processor.rb +1 -0
  107. data/lib/relaton/omg/bibliography.rb +9 -9
  108. data/lib/relaton/omg/scraper.rb +14 -13
  109. data/lib/relaton/omg.rb +1 -1
  110. data/lib/relaton/plateau/bibliography.rb +9 -7
  111. data/lib/relaton/plateau/hit_collection.rb +2 -1
  112. data/lib/relaton/plateau/processor.rb +1 -0
  113. data/lib/relaton/un/bibliography.rb +21 -10
  114. data/lib/relaton/un/processor.rb +1 -0
  115. data/lib/relaton/version.rb +1 -1
  116. data/lib/relaton/w3c/processor.rb +1 -0
  117. data/lib/relaton/xsf/bibliography.rb +11 -6
  118. data/lib/relaton.rb +13 -0
  119. metadata +60 -14
  120. data/lib/relaton/itu/pubid.rb +0 -199
data/lib/relaton/db.rb CHANGED
@@ -1,11 +1,13 @@
1
1
  require "relaton/bib"
2
2
  require "yaml"
3
3
  require "net/http"
4
- require "nokogiri"
4
+ require "moxml"
5
5
  require "fileutils"
6
6
  require "date"
7
7
 
8
8
  module Relaton
9
+ autoload :Cloud, "relaton/cloud"
10
+
9
11
  class Db
10
12
  # @param global_cache [String] directory of global DB
11
13
  # @param local_cache [String] directory of local DB
@@ -61,18 +63,23 @@ module Relaton
61
63
  # or "YYYY-MM-DD")
62
64
  #
63
65
  # @return [nil, RelatonBib::BibliographicItem, ...]
66
+ # @raise [Relaton::UnknownReferenceError] no flavor recognizes the
67
+ # reference (see Registry#route)
64
68
  ##
65
- def fetch(text, year = nil, opts = {})
69
+ def fetch(text, year = nil, opts = {}) # rubocop:disable Metrics/MethodLength
66
70
  reference = text.strip
67
- stdclass = @registry.class_by_ref(reference) || return
71
+ stdclass, pubid = @registry.route(reference)
68
72
  processor = @registry[stdclass]
69
73
  ref = if processor.respond_to?(:urn_to_code)
70
74
  processor.urn_to_code(reference)&.first
71
75
  else reference
72
76
  end
73
77
  ref ||= reference
78
+ # The routed pubid is the parse of `reference`; a URN rewritten to a
79
+ # code is parsed again by the flavor.
80
+ pubid = nil unless ref == reference
74
81
  result = combine_doc ref, year, opts, stdclass
75
- result || check_bibliocache(ref, year, opts, stdclass)
82
+ result || fetch_routed(ref, year, opts, stdclass, pubid)
76
83
  end
77
84
 
78
85
  # @see Relaton::Db#fetch
@@ -90,8 +97,8 @@ module Relaton
90
97
  result = []
91
98
  db = @db || @local_db
92
99
  if db
93
- result += db.all do |file, xml|
94
- search_xml file, xml, text, edition, year
100
+ result += db.all do |processor, xml|
101
+ search_xml processor, xml, text, edition, year
95
102
  end.compact
96
103
  end
97
104
  result
@@ -110,7 +117,7 @@ module Relaton
110
117
  # nil if document not found
111
118
  #
112
119
  def fetch_async(ref, year = nil, opts = {}, &block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
113
- stdclass = @registry.class_by_ref ref
120
+ stdclass = async_route ref
114
121
  if stdclass
115
122
  unless @queues[stdclass]
116
123
  processor = @registry[stdclass]
@@ -132,23 +139,32 @@ module Relaton
132
139
  end
133
140
  end
134
141
 
142
+ #
143
+ # Fetch with the flavor the caller names, else the routed one.
144
+ #
145
+ # @param stdclass [Symbol, String, nil] a processor short name
146
+ # (`:relaton_iso`, as relaton-cli passes it) or a prefix (`"ISO"`)
147
+ # @raise [Relaton::UnknownReferenceError] no flavor is named and none
148
+ # recognizes the reference
149
+ #
135
150
  def fetch_std(code, year = nil, stdclass = nil, opts = {})
136
- std = nil
137
- @registry.processors.each do |name, processor|
138
- std = name if processor.prefix == stdclass
139
- end
140
- std = @registry.class_by_ref(code) or return nil unless std
151
+ std = named_class(stdclass)
152
+ # A flavor the caller names is the one asked: no co-publisher.
153
+ return check_bibliocache(code, year, opts, std) if std
141
154
 
142
- check_bibliocache(code, year, opts, std)
155
+ std, pubid = @registry.route(code)
156
+ fetch_routed(code, year, opts, std, pubid)
143
157
  end
144
158
 
145
159
  # The document identifier class corresponding to the given code
146
160
  # @param code [String]
147
161
  # @return [Array]
148
162
  def docid_type(code)
149
- stdclass = @registry.class_by_ref(code) or return [nil, code]
163
+ stdclass, = @registry.route(code)
150
164
  _, code = strip_id_wrapper(code, stdclass)
151
165
  [@registry[stdclass].idtype, code]
166
+ rescue UnknownReferenceError
167
+ [nil, code]
152
168
  end
153
169
 
154
170
  # @param key [String]
@@ -169,43 +185,65 @@ module Relaton
169
185
  # @return [String]
170
186
  def to_xml
171
187
  db = @local_db || @db || return
172
- Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
173
- xml.documents { xml.parent.add_child db.all.join(" ") }
174
- end.to_xml
188
+ parts = db.all.join(" ")
189
+ Moxml.parse(
190
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<documents>#{parts}</documents>",
191
+ ).to_xml(indent: 0, expand_empty: false)
175
192
  end
176
193
 
177
194
  private
178
195
 
179
- def fetch_doc(code, year, opts, processor)
180
- if Db.configuration.use_api then fetch_api(code, year, opts, processor)
181
- else processor.get(code, year, opts)
196
+ # The flavor routing picks for `fetch_async`'s per-flavor queue; nil
197
+ # (logged) when no flavor recognizes the reference.
198
+ def async_route(ref)
199
+ @registry.route(ref).first
200
+ rescue UnknownReferenceError => e
201
+ Util.info e.message, key: ref
202
+ nil
203
+ end
204
+
205
+ # The processor a caller names by short name (`:relaton_iso`) or prefix.
206
+ def named_class(stdclass)
207
+ return unless stdclass
208
+
209
+ short = stdclass.to_sym
210
+ return short if @registry.processors.key?(short)
211
+
212
+ @registry.processors.detect { |_, p| p.prefix == stdclass.to_s }&.first
213
+ end
214
+
215
+ # The flavor's `get` receives the parsed pubid when there is one
216
+ # (relaton#205), else the String. The API request sends the String the
217
+ # caller wrote.
218
+ def fetch_doc(code, year, opts, processor, query = nil)
219
+ if Db.configuration.use_api
220
+ fetch_api(code, year, opts, processor, query)
221
+ else processor.get(query || code, year, opts)
182
222
  end
183
223
  end
184
224
 
185
- def fetch_api(code, year, opts, processor)
225
+ def fetch_api(code, year, opts, processor, query = nil)
186
226
  url = "#{Db.configuration.api_host}" \
187
227
  "/api/v1/document?#{params(code, year, opts)}"
188
228
  rsp = Net::HTTP.get_response URI(url)
189
229
  processor.from_xml rsp.body if rsp.code == "200"
190
230
  rescue Errno::ECONNREFUSED
191
- processor.get(code, year, opts)
231
+ processor.get(query || code, year, opts)
192
232
  end
193
233
 
194
234
  def params(code, year, opts)
195
235
  opts.merge(code: code, year: year).map { |k, v| "#{k}=#{v}" }.join "&"
196
236
  end
197
237
 
198
- def search_xml(file, xml, text, edition, year)
238
+ def search_xml(processor, xml, text, edition, year)
239
+ return unless processor
199
240
  return unless text.nil? || match_xml_text?(xml, text)
200
241
 
201
- search_edition_year(file, xml, edition, year)
242
+ search_edition_year(processor, xml, edition, year)
202
243
  end
203
244
 
204
- def search_edition_year(file, content, edition, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
205
- processor = @registry.processor_by_ref(file.split("/")[-2])
206
- item = if file.match?(/xml$/) then processor.from_xml(content)
207
- else processor.from_yaml(content)
208
- end
245
+ def search_edition_year(processor, content, edition, year) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
246
+ item = processor.from_xml(content)
209
247
  item if (edition.nil? || item.edition.content == edition) && (year.nil? ||
210
248
  item.date.detect do |d|
211
249
  d.type == "published" && d.at.to_date.year.to_s == year.to_s
@@ -240,7 +278,9 @@ module Relaton
240
278
  if updates
241
279
  doc.relation << Bib::Relation.new(bibitem: updates, type: "updates")
242
280
  end
243
- divider = stdclass == :relaton_itu ? " " : "/"
281
+ # The supplement joins its base as the flavor's identifier spells it:
282
+ # `NIST SP 800-38A Add`, not `/Add`, which pubid does not parse.
283
+ divider = %i[relaton_itu relaton_nist].include?(stdclass) ? " " : "/"
244
284
  refs[1..].each_with_object(doc) do |c, d|
245
285
  bib = check_bibliocache(ref + divider + c, year, opts, stdclass)
246
286
  if bib
@@ -251,18 +291,73 @@ module Relaton
251
291
  end
252
292
  end
253
293
 
294
+ # The legacy string cache key, for a processor with no pubid class. The
295
+ # publication date range is not part of it: it filters, it is not
296
+ # identity.
254
297
  def std_id(code, year, opts, stdclass)
255
298
  prefix, code = strip_id_wrapper(code, stdclass)
256
299
  ret = code
257
300
  ret += (stdclass == :relaton_gb ? "-" : ":") + year if year
258
301
  ret += " (all parts)" if opts[:all_parts]
259
- after = opts[:publication_date_after]
260
- ret += " after-#{after}" if after
261
- before = opts[:publication_date_before]
262
- ret += " before-#{before}" if before
263
302
  ["#{prefix}(#{ret.strip})", code]
264
303
  end
265
304
 
305
+ #
306
+ # The cache key of a query: the flavor's parsed pubid, with the `year`
307
+ # and `all_parts` options folded in. A reference the flavor cannot parse
308
+ # raises `Pubid::Errors::ParseError`. Nil when the flavor has a pubid
309
+ # class but gives no key for this query (a miss by the flavor's own rule,
310
+ # or a query whose answer the cache cannot hold, such as a CCSDS format):
311
+ # that query is not cached. A processor with no pubid class gets the
312
+ # legacy string key.
313
+ #
314
+ # @param pubid [Pubid::Identifier, nil] the reference as routing parsed
315
+ # it, so the processor does not parse it again
316
+ # @return [Pubid::Identifier, String, nil]
317
+ #
318
+ def cache_key(code, year, opts, stdclass, pubid = nil)
319
+ processor = @registry[stdclass]
320
+ unless processor.pubid_class
321
+ return std_id(code, year, opts, stdclass).first
322
+ end
323
+
324
+ processor.cache_key(code, year, opts, pubid)
325
+ end
326
+
327
+ #
328
+ # The pubid `get` receives, and the cache key folded from it: one parse
329
+ # of the reference serves both (relaton#205). A processor with no pubid
330
+ # class gets the String and the legacy string key.
331
+ #
332
+ # @param pubid [Pubid::Identifier, nil] routing's parse
333
+ # @return [Array(Pubid::Identifier, Object)] the query pubid (nil: `get`
334
+ # gets the String) and the key (nil: not cached)
335
+ #
336
+ def query_and_key(code, year, opts, stdclass, pubid)
337
+ processor = @registry[stdclass]
338
+ unless processor.pubid_class
339
+ return [nil, std_id(code, year, opts, stdclass).first]
340
+ end
341
+
342
+ query = processor.query_pubid(code, opts, pubid)
343
+ [query, query && processor.cache_key(code, year, opts, query)]
344
+ end
345
+
346
+ # The key of a fetched document, from its primary identifier. The
347
+ # identifier is data, so an unparseable one gives no key.
348
+ def item_key(bib, stdclass)
349
+ docid = bib.docidentifier.detect(&:primary) || bib.docidentifier.first
350
+ return unless docid&.content
351
+
352
+ cache_key docid.content, nil, {}, stdclass
353
+ rescue ::Pubid::Errors::Error, Parslet::ParseFailed
354
+ nil
355
+ end
356
+
357
+ def date_range?(opts)
358
+ opts[:publication_date_before] || opts[:publication_date_after]
359
+ end
360
+
266
361
  def strip_id_wrapper(code, stdclass)
267
362
  prefix = @registry[stdclass].prefix
268
363
  code =
@@ -273,66 +368,112 @@ module Relaton
273
368
  [prefix, code]
274
369
  end
275
370
 
276
- def bib_retval(entry, stdclass)
277
- if entry && !entry.match?(/^not_found/)
278
- @registry[stdclass].from_xml(entry)
371
+ # The routed flavor, then its co-publishers' flavors (relaton#205).
372
+ def fetch_routed(code, year, opts, stdclass, pubid)
373
+ check_bibliocache(code, year, opts, stdclass, pubid: pubid) ||
374
+ copublisher_fetch(code, year, opts, stdclass, pubid)
375
+ end
376
+
377
+ #
378
+ # A co-published document the lead flavor's catalog does not have:
379
+ # ask the co-publishers' flavors, in the order the parsed pubid holds
380
+ # them (relaton#205; pubid#469, #472). Each one keys, caches and fetches
381
+ # with its own flavor, so a second fetch is answered from the caches.
382
+ # A co-publisher whose flavor gives no pubid for the lead's printed form
383
+ # is skipped. Only that probe parse is rescued: an error from inside the
384
+ # co-publisher's fetch propagates.
385
+ #
386
+ # @return [Relaton::Bib::ItemData, nil]
387
+ #
388
+ def copublisher_fetch(code, year, opts, stdclass, pubid)
389
+ # The reference as the lead read it: no `PREFIX(...)` wrapper, no en dash.
390
+ _, code = strip_id_wrapper(code, stdclass)
391
+ copublisher_classes(code, opts, stdclass, pubid).each do |publisher, co|
392
+ Util.info "Not found; trying co-publisher `#{publisher}`", key: code
393
+ query = copublisher_query(code, opts, co, publisher) or next
394
+ bib = check_bibliocache(code, year, opts, co, pubid: query)
395
+ return bib if bib
279
396
  end
397
+ nil
280
398
  end
281
399
 
282
- def check_bibliocache(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
283
- if opts[:publication_date_before] || opts[:publication_date_after]
284
- base_opts = opts.except(
285
- :publication_date_before, :publication_date_after
286
- )
287
- base_id, = std_id(code, year, base_opts, stdclass)
288
- db = @local_db || @db
289
- if db&.valid_entry?(base_id, year)
290
- entry = db[base_id]
291
- if entry && !entry.match?(/^not_found/) &&
292
- pub_date_in_range?(entry, opts)
293
- return bib_retval(entry, stdclass)
294
- end
295
- end
400
+ # @return [Array<Array(String, Symbol)>] each co-publisher that a flavor
401
+ # other than the lead serves, with that flavor
402
+ def copublisher_classes(code, opts, stdclass, pubid)
403
+ processor = @registry[stdclass]
404
+ query = processor.query_pubid(code, opts, pubid) or return []
405
+
406
+ processor.copublishers(query).filter_map do |publisher|
407
+ co = @registry.class_by_publisher(publisher)
408
+ [publisher, co] if co && co != stdclass
409
+ end.uniq(&:last)
410
+ end
411
+
412
+ # The co-publisher flavor's own parse of the reference, or nil when it
413
+ # gives none (a parse error, or a flavor that reads it as a miss: IEEE),
414
+ # which is logged.
415
+ #
416
+ # @return [Pubid::Identifier, nil]
417
+ def copublisher_query(code, opts, stdclass, publisher)
418
+ query = begin
419
+ @registry[stdclass].query_pubid(code, opts)
420
+ rescue ::Pubid::Errors::Error, Parslet::ParseFailed
421
+ nil
296
422
  end
423
+ return query if query
424
+
425
+ Util.info "Not found; co-publisher `#{publisher}` cannot read it",
426
+ key: code
427
+ nil
428
+ end
429
+
430
+ def bib_retval(entry, stdclass)
431
+ @registry[stdclass].from_xml(entry) if entry.is_a?(String)
432
+ end
297
433
 
298
- id, searchcode = std_id(code, year, opts, stdclass)
299
- db = @local_db || @db
434
+ def check_bibliocache(code, year, opts, stdclass, pubid: nil) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
435
+ _, searchcode = strip_id_wrapper(code, stdclass)
436
+ query, id = query_and_key(searchcode, year, opts, stdclass, pubid)
437
+ db = id && (@local_db || @db)
300
438
  altdb = @local_db && @db ? @db : nil
301
439
  if db.nil?
302
440
  return if opts[:fetch_db]
303
441
 
304
- bibentry = new_bib_entry(searchcode, year, opts, stdclass)
442
+ bibentry = new_bib_entry(searchcode, year, opts, stdclass, query: query)
305
443
  return bib_retval(bibentry, stdclass)
306
444
  end
307
-
308
- @semaphore.synchronize do
309
- db.delete(id) unless db.valid_entry?(id, year)
445
+ if date_range?(opts)
446
+ return check_date_range(searchcode, id, year, opts, stdclass, query)
310
447
  end
448
+
449
+ @semaphore.synchronize { db.expire id, year }
311
450
  if altdb
312
- return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
451
+ return bib_retval(altdb.read(id), stdclass) if opts[:fetch_db]
313
452
 
314
453
  @semaphore.synchronize do
315
454
  db.clone_entry id, altdb if altdb.valid_entry? id, year
316
455
  end
317
- new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
456
+ new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
457
+ query: query)
318
458
  @semaphore.synchronize do
319
459
  altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
320
460
  end
321
461
  else
322
- return bib_retval(db[id], stdclass) if opts[:fetch_db]
462
+ return bib_retval(db.read(id), stdclass) if opts[:fetch_db]
323
463
 
324
- new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
464
+ new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
465
+ query: query)
325
466
  end
326
- bib_retval(db[id], stdclass)
467
+ bib_retval(db.read(id), stdclass)
327
468
  end
328
469
 
329
470
  def new_bib_entry(code, year, opts, stdclass, **args)
330
- entry = @semaphore.synchronize { args[:db] && args[:db][args[:id]] }
471
+ entry = @semaphore.synchronize { args[:db]&.read(args[:id]) }
331
472
  if !entry || opts[:no_cache]
332
473
  return fetch_entry(code, year, opts, stdclass, **args)
333
474
  end
334
475
 
335
- if entry&.match?(/^not_found/)
476
+ if entry.is_a?(NotFound)
336
477
  Util.info "not found in cache, if you wish to " \
337
478
  "ignore cache please use `no-cache` option.", key: code
338
479
  return
@@ -340,59 +481,101 @@ module Relaton
340
481
  entry
341
482
  end
342
483
 
343
- def fetch_entry(code, year, opts, stdclass, **args)
484
+ def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize
344
485
  processor = @registry[stdclass]
345
- bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1))
346
-
347
- entry = check_entry(bib, stdclass, **args)
486
+ bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
487
+ args[:query])
488
+ entry = bib_entry bib
348
489
  return entry if args[:db].nil?
349
490
 
350
- @semaphore.synchronize { args[:db][args[:id]] ||= entry }
351
- args[:no_cache] ? bib_entry(bib) : entry
491
+ # `no_cache` refreshes a cached entry, but a failed fetch does not
492
+ # replace a cached document with a not-found entry.
493
+ refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
494
+ @semaphore.synchronize do
495
+ if refresh || !args[:db].read(args[:id])
496
+ save_bib args[:db], args[:id], bib, entry, stdclass
497
+ end
498
+ end
499
+ entry
500
+ end
501
+
502
+ #
503
+ # Cache a fetched document. The document's own identifier gets a row;
504
+ # when the query key differs from it (an undated or incomplete query),
505
+ # the query gets a row that points to the same document.
506
+ #
507
+ def save_bib(db, key, bib, entry, stdclass)
508
+ item = bib.respond_to?(:docidentifier) && item_key(bib, stdclass)
509
+ db.store key, entry, item_key: item || nil
352
510
  end
353
511
 
354
- def check_entry(bib, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
355
- bib_id = bib && bib.docidentifier.first&.content
512
+ #
513
+ # A publication date range selects among the cached editions of the
514
+ # reference, so it is not part of the key. On a miss the flavor is asked
515
+ # with the range, and its answer is cached under its own identifier only:
516
+ # the query row keeps pointing to the latest edition.
517
+ #
518
+ def check_date_range(code, key, year, opts, stdclass, query = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
519
+ caches = [@local_db, @db].compact
520
+ cached = caches.flat_map { |c| c.candidates(key) }.filter_map do |_, xml|
521
+ date = published_date(xml)
522
+ [date, xml] if date && pub_date_in_range?(xml, opts)
523
+ end.max_by(&:first)
524
+ return bib_retval(cached.last, stdclass) if cached && !opts[:no_cache]
525
+ return if opts[:fetch_db]
526
+
527
+ processor = @registry[stdclass]
528
+ bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
529
+ query)
530
+ return unless bib.respond_to?(:to_xml)
356
531
 
357
- quoted = Regexp.quote("(#{bib_id})")
358
- if args[:db] && args[:id] && bib_id &&
359
- args[:id] !~ /#{quoted}/
360
- bid = std_id(bib.docidentifier.first.content, nil, {}, stdclass).first
361
- @semaphore.synchronize { args[:db][bid] ||= bib_entry bib }
362
- "redirection #{bid}"
363
- else bib_entry bib
532
+ entry = bib_entry bib
533
+ item = item_key(bib, stdclass)
534
+ if item
535
+ @semaphore.synchronize do
536
+ [@local_db, @db].compact.each { |c| c.store item, entry }
537
+ end
364
538
  end
539
+ bib_retval entry, stdclass
365
540
  end
366
541
 
367
- def net_retry(code, year, opts, processor, retries)
368
- fetch_doc code, year, opts, processor
542
+ # @param query [Pubid::Identifier, nil] what `get` receives in place of
543
+ # the String +code+ (Db#query_and_key)
544
+ def net_retry(code, year, opts, processor, retries, query = nil)
545
+ fetch_doc code, year, opts, processor, query
369
546
  rescue Relaton::RequestError => e
370
547
  raise e unless retries > 1
371
548
 
372
- net_retry(code, year, opts, processor, retries - 1)
549
+ net_retry(code, year, opts, processor, retries - 1, query)
373
550
  end
374
551
 
375
552
  def bib_entry(bib)
376
553
  if bib.respond_to?(:to_xml)
377
554
  bib.to_xml(bibdata: true)
378
555
  else
379
- "not_found #{Date.today}"
556
+ NotFound.new(fetched: Date.today.to_s)
380
557
  end
381
558
  end
382
559
 
383
- def pub_date_in_range?(entry, opts) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
384
- doc = Nokogiri::XML(entry)
385
- date_str = doc.at("//date[@type='published']/on")&.text
386
- return false unless date_str
560
+ # @param entry [String] document XML
561
+ # @return [Date, nil] the published date
562
+ def published_date(entry)
563
+ date_str = Moxml.parse(entry)
564
+ .at_xpath("//date[@type='published']/on")&.text
565
+ date_str && parse_pub_date(date_str)
566
+ end
387
567
 
388
- date = parse_pub_date(date_str)
568
+ def pub_date_in_range?(entry, opts) # rubocop:disable Metrics/CyclomaticComplexity
569
+ date = published_date(entry)
389
570
  return false unless date
390
571
 
572
+ # `parse_pub_date`, not `Date.parse`: a bound may be "YYYY" or
573
+ # "YYYY-MM", which `Date.parse` rejects.
391
574
  after = opts[:publication_date_after]
392
- return false if after && date < Date.parse(after.to_s)
575
+ return false if after && date < parse_pub_date(after.to_s)
393
576
 
394
577
  before = opts[:publication_date_before]
395
- return false if before && date >= Date.parse(before.to_s)
578
+ return false if before && date >= parse_pub_date(before.to_s)
396
579
 
397
580
  true
398
581
  end
@@ -407,18 +590,8 @@ module Relaton
407
590
  nil
408
591
  end
409
592
 
410
- def open_cache_biblio(dir) # rubocop:disable Metrics/MethodLength
411
- return nil if dir.nil?
412
-
413
- db = Cache.new dir
414
-
415
- Dir["#{dir}/*/"].each do |fdir|
416
- next if db.check_version?(fdir)
417
-
418
- FileUtils.rm_rf(fdir, secure: true)
419
- Util.info "cache #{fdir}: version is obsolete and cache is cleared."
420
- end
421
- db
593
+ def open_cache_biblio(dir)
594
+ dir && Cache.new(dir)
422
595
  end
423
596
 
424
597
  def process_queue(qwp)
@@ -10,25 +10,42 @@ module Relaton
10
10
  #
11
11
  # Get a document by DOI from the CrossRef API.
12
12
  #
13
- # @param [String] doi The DOI.
13
+ # @param [String, Pubid::Doi::Identifier] doi The DOI.
14
14
  #
15
15
  # @return [RelatonBib::BibliographicItem, RelatonIetf::IetfBibliographicItem,
16
16
  # RelatonBipm::BipmBibliographicItem, RelatonIeee::IeeeBibliographicItem,
17
17
  # RelatonNist::NistBibliographicItem] The bibitem.
18
18
  #
19
19
  def get(doi)
20
- Util.info "Fetching from search.crossref.org ...", key: doi
21
- id = doi.sub(%r{^doi:}, "")
22
- message = get_by_id id
20
+ key = doi.to_s
21
+ Util.info "Fetching from search.crossref.org ...", key: key
22
+ message = get_by_id doi_of(doi)
23
23
  if message
24
- Util.info "Found: `#{message['DOI']}`", key: doi
24
+ Util.info "Found: `#{message['DOI']}`", key: key
25
25
  Parser.parse message
26
26
  else
27
- Util.info "Not found.", key: doi
27
+ Util.info "Not found.", key: key
28
28
  nil
29
29
  end
30
30
  end
31
31
 
32
+ #
33
+ # The DOI itself (`<prefix>/<suffix>`), as the Crossref API takes it.
34
+ #
35
+ # A parsed pubid gives it from its components. A String may carry a
36
+ # `doi:` scheme (in any case) or a `doi.org` URL: Relaton::Db keys every
37
+ # form Pubid::Doi reads as one DOI, so all of them must reach the same
38
+ # DOI here, or a miss on one form is cached for the others.
39
+ #
40
+ # @param [String, Pubid::Doi::Identifier] doi
41
+ # @return [String]
42
+ #
43
+ def doi_of(doi)
44
+ return "#{doi.prefix}/#{doi.suffix}" unless doi.is_a?(String)
45
+
46
+ doi.sub(%r{\A(?:doi:|https?://(?:dx\.)?doi\.org/)}i, "")
47
+ end
48
+
32
49
  #
33
50
  # Get a document by DOI from the CrossRef API.
34
51
  #
@@ -8,6 +8,7 @@ module Relaton
8
8
  def initialize # rubocop:disable Lint/MissingSuper
9
9
  @short = :relaton_doi
10
10
  @prefix = "DOI"
11
+ @pubid_identifier = :Doi # Db cache key
11
12
  @defaultprefix = %r{^doi:}
12
13
  @idtype = "DOI"
13
14
  end
@@ -12,6 +12,7 @@ module Relaton
12
12
  def initialize
13
13
  @short = :relaton_easc
14
14
  @prefix = "EASC"
15
+ @pubid_identifier = :Easc # Db cache key
15
16
  # Both Cyrillic and Latin series prefixes route here.
16
17
  @defaultprefix = %r{^(?:ПМГ|РМГ|PMG|RMG)\b}
17
18
  @idtype = "EASC"