relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/relaton/3gpp/bibliography.rb +19 -6
- data/lib/relaton/3gpp/processor.rb +6 -0
- data/lib/relaton/adobe/processor.rb +9 -0
- data/lib/relaton/bib/converter/csl.rb +110 -0
- data/lib/relaton/bib/converter/ris.rb +104 -0
- data/lib/relaton/bib/converter/titles.rb +24 -0
- data/lib/relaton/bib/item_data.rb +23 -0
- data/lib/relaton/bib/model/item.rb +6 -6
- data/lib/relaton/bib/sanitizer.rb +71 -30
- data/lib/relaton/bib.rb +10 -0
- data/lib/relaton/bipm/bibliography.rb +24 -21
- data/lib/relaton/bipm/processor.rb +1 -0
- data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +6 -6
- data/lib/relaton/bipm/si_brochure_parser.rb +2 -2
- data/lib/relaton/bsi/bibliography.rb +26 -23
- data/lib/relaton/bsi/processor.rb +5 -0
- data/lib/relaton/calconnect/bibliography.rb +6 -6
- data/lib/relaton/calconnect/hit_collection.rb +4 -1
- data/lib/relaton/ccsds/bibliography.rb +16 -10
- data/lib/relaton/ccsds/hit_collection.rb +1 -1
- data/lib/relaton/ccsds/processor.rb +13 -0
- data/lib/relaton/cen/bibliography.rb +27 -24
- data/lib/relaton/cen/hit_collection.rb +1 -1
- data/lib/relaton/cen/processor.rb +5 -0
- data/lib/relaton/cen/scraper.rb +9 -9
- data/lib/relaton/cie/bibliography.rb +11 -9
- data/lib/relaton/cie/data_fetcher.rb +18 -18
- data/lib/relaton/cie/processor.rb +1 -0
- data/lib/relaton/cie/scrapper.rb +15 -8
- data/lib/relaton/cie.rb +1 -1
- data/lib/relaton/cloud.rb +127 -0
- data/lib/relaton/core/hit_collection.rb +6 -10
- data/lib/relaton/core/processor.rb +121 -0
- data/lib/relaton/core/request_error.rb +21 -3
- data/lib/relaton/db/cache.rb +474 -148
- data/lib/relaton/db/cache_entry.rb +42 -0
- data/lib/relaton/db/registry.rb +152 -0
- data/lib/relaton/db.rb +275 -102
- data/lib/relaton/doi/crossref.rb +23 -6
- data/lib/relaton/doi/processor.rb +1 -0
- data/lib/relaton/easc/processor.rb +1 -0
- data/lib/relaton/ecma/bibliography.rb +18 -13
- data/lib/relaton/ecma/data_fetcher.rb +1 -1
- data/lib/relaton/ecma/data_parser.rb +1 -1
- data/lib/relaton/ecma/edition_parser.rb +2 -2
- data/lib/relaton/ecma/memento_parser.rb +4 -4
- data/lib/relaton/ecma/standard_parser.rb +6 -6
- data/lib/relaton/etsi/bibliography.rb +8 -7
- data/lib/relaton/etsi/processor.rb +1 -0
- data/lib/relaton/gb/bibliography.rb +20 -12
- data/lib/relaton/gb/gb_scraper.rb +5 -5
- data/lib/relaton/gb/scraper.rb +16 -16
- data/lib/relaton/gb/sec_scraper.rb +8 -8
- data/lib/relaton/gb/t_scraper.rb +5 -5
- data/lib/relaton/gost/processor.rb +1 -0
- data/lib/relaton/iala/bibliography.rb +15 -12
- data/lib/relaton/iala/processor.rb +1 -0
- data/lib/relaton/iana/bibliography.rb +12 -12
- data/lib/relaton/iana/data_fetcher.rb +3 -3
- data/lib/relaton/iana/parser.rb +12 -5
- data/lib/relaton/iana/processor.rb +9 -0
- data/lib/relaton/iec/bibliography.rb +18 -7
- data/lib/relaton/iec/data_parser.rb +24 -9
- data/lib/relaton/iec/processor.rb +11 -0
- data/lib/relaton/iec.rb +1 -1
- data/lib/relaton/ieee/bibliography.rb +19 -13
- data/lib/relaton/ieee/data_fetcher.rb +1 -1
- data/lib/relaton/ieee/processor.rb +24 -0
- data/lib/relaton/ieee/rawbib_id_parser.rb +2 -2
- data/lib/relaton/ietf/bibliography.rb +11 -9
- data/lib/relaton/ietf/data_fetcher.rb +1 -1
- data/lib/relaton/ietf/processor.rb +1 -0
- data/lib/relaton/ietf/rfc/entry.rb +15 -19
- data/lib/relaton/ietf/scraper.rb +5 -4
- data/lib/relaton/iho/processor.rb +1 -0
- data/lib/relaton/index/file_io.rb +2 -2
- data/lib/relaton/index/pool.rb +4 -3
- data/lib/relaton/index/shard_source.rb +1 -1
- data/lib/relaton/index/type.rb +2 -5
- data/lib/relaton/isbn/open_library.rb +11 -7
- data/lib/relaton/isbn/processor.rb +10 -0
- data/lib/relaton/iso/bibliography.rb +11 -9
- data/lib/relaton/iso/data_parser.rb +2 -2
- data/lib/relaton/iso/processor.rb +5 -0
- data/lib/relaton/iso/scraper.rb +20 -20
- data/lib/relaton/itu/bibliography.rb +107 -16
- data/lib/relaton/itu/data_crawler_r.rb +2 -2
- data/lib/relaton/itu/hit_collection.rb +64 -52
- data/lib/relaton/itu/processor.rb +1 -0
- data/lib/relaton/itu/scraper.rb +13 -3
- data/lib/relaton/itu.rb +0 -2
- data/lib/relaton/jis/bibliography.rb +22 -11
- data/lib/relaton/jis/data_fetcher.rb +4 -4
- data/lib/relaton/jis/processor.rb +1 -0
- data/lib/relaton/jis/scraper.rb +8 -8
- data/lib/relaton/nist/bibliography.rb +25 -21
- data/lib/relaton/oasis/bibliography.rb +16 -13
- data/lib/relaton/oasis/browser_agent.rb +2 -2
- data/lib/relaton/oasis/data_parser.rb +5 -5
- data/lib/relaton/oasis/data_parser_utils.rb +2 -2
- data/lib/relaton/oasis/data_part_parser.rb +7 -7
- data/lib/relaton/ogc/bibliography.rb +15 -14
- data/lib/relaton/ogc/hit_collection.rb +9 -7
- data/lib/relaton/ogc/processor.rb +13 -0
- data/lib/relaton/oiml/processor.rb +1 -0
- data/lib/relaton/omg/bibliography.rb +9 -9
- data/lib/relaton/omg/scraper.rb +14 -13
- data/lib/relaton/omg.rb +1 -1
- data/lib/relaton/plateau/bibliography.rb +9 -7
- data/lib/relaton/plateau/hit_collection.rb +2 -1
- data/lib/relaton/plateau/processor.rb +1 -0
- data/lib/relaton/un/bibliography.rb +21 -10
- data/lib/relaton/un/processor.rb +1 -0
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/processor.rb +1 -0
- data/lib/relaton/xsf/bibliography.rb +11 -6
- data/lib/relaton.rb +13 -0
- metadata +60 -14
- data/lib/relaton/itu/pubid.rb +0 -199
data/lib/relaton/db.rb
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
require "relaton/bib"
|
|
2
2
|
require "yaml"
|
|
3
3
|
require "net/http"
|
|
4
|
-
require "
|
|
4
|
+
require "moxml"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
require "date"
|
|
7
7
|
|
|
8
8
|
module Relaton
|
|
9
|
+
autoload :Cloud, "relaton/cloud"
|
|
10
|
+
|
|
9
11
|
class Db
|
|
10
12
|
# @param global_cache [String] directory of global DB
|
|
11
13
|
# @param local_cache [String] directory of local DB
|
|
@@ -61,18 +63,23 @@ module Relaton
|
|
|
61
63
|
# or "YYYY-MM-DD")
|
|
62
64
|
#
|
|
63
65
|
# @return [nil, RelatonBib::BibliographicItem, ...]
|
|
66
|
+
# @raise [Relaton::UnknownReferenceError] no flavor recognizes the
|
|
67
|
+
# reference (see Registry#route)
|
|
64
68
|
##
|
|
65
|
-
def fetch(text, year = nil, opts = {})
|
|
69
|
+
def fetch(text, year = nil, opts = {}) # rubocop:disable Metrics/MethodLength
|
|
66
70
|
reference = text.strip
|
|
67
|
-
stdclass = @registry.
|
|
71
|
+
stdclass, pubid = @registry.route(reference)
|
|
68
72
|
processor = @registry[stdclass]
|
|
69
73
|
ref = if processor.respond_to?(:urn_to_code)
|
|
70
74
|
processor.urn_to_code(reference)&.first
|
|
71
75
|
else reference
|
|
72
76
|
end
|
|
73
77
|
ref ||= reference
|
|
78
|
+
# The routed pubid is the parse of `reference`; a URN rewritten to a
|
|
79
|
+
# code is parsed again by the flavor.
|
|
80
|
+
pubid = nil unless ref == reference
|
|
74
81
|
result = combine_doc ref, year, opts, stdclass
|
|
75
|
-
result ||
|
|
82
|
+
result || fetch_routed(ref, year, opts, stdclass, pubid)
|
|
76
83
|
end
|
|
77
84
|
|
|
78
85
|
# @see Relaton::Db#fetch
|
|
@@ -90,8 +97,8 @@ module Relaton
|
|
|
90
97
|
result = []
|
|
91
98
|
db = @db || @local_db
|
|
92
99
|
if db
|
|
93
|
-
result += db.all do |
|
|
94
|
-
search_xml
|
|
100
|
+
result += db.all do |processor, xml|
|
|
101
|
+
search_xml processor, xml, text, edition, year
|
|
95
102
|
end.compact
|
|
96
103
|
end
|
|
97
104
|
result
|
|
@@ -110,7 +117,7 @@ module Relaton
|
|
|
110
117
|
# nil if document not found
|
|
111
118
|
#
|
|
112
119
|
def fetch_async(ref, year = nil, opts = {}, &block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
113
|
-
stdclass =
|
|
120
|
+
stdclass = async_route ref
|
|
114
121
|
if stdclass
|
|
115
122
|
unless @queues[stdclass]
|
|
116
123
|
processor = @registry[stdclass]
|
|
@@ -132,23 +139,32 @@ module Relaton
|
|
|
132
139
|
end
|
|
133
140
|
end
|
|
134
141
|
|
|
142
|
+
#
|
|
143
|
+
# Fetch with the flavor the caller names, else the routed one.
|
|
144
|
+
#
|
|
145
|
+
# @param stdclass [Symbol, String, nil] a processor short name
|
|
146
|
+
# (`:relaton_iso`, as relaton-cli passes it) or a prefix (`"ISO"`)
|
|
147
|
+
# @raise [Relaton::UnknownReferenceError] no flavor is named and none
|
|
148
|
+
# recognizes the reference
|
|
149
|
+
#
|
|
135
150
|
def fetch_std(code, year = nil, stdclass = nil, opts = {})
|
|
136
|
-
std =
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
end
|
|
140
|
-
std = @registry.class_by_ref(code) or return nil unless std
|
|
151
|
+
std = named_class(stdclass)
|
|
152
|
+
# A flavor the caller names is the one asked: no co-publisher.
|
|
153
|
+
return check_bibliocache(code, year, opts, std) if std
|
|
141
154
|
|
|
142
|
-
|
|
155
|
+
std, pubid = @registry.route(code)
|
|
156
|
+
fetch_routed(code, year, opts, std, pubid)
|
|
143
157
|
end
|
|
144
158
|
|
|
145
159
|
# The document identifier class corresponding to the given code
|
|
146
160
|
# @param code [String]
|
|
147
161
|
# @return [Array]
|
|
148
162
|
def docid_type(code)
|
|
149
|
-
stdclass = @registry.
|
|
163
|
+
stdclass, = @registry.route(code)
|
|
150
164
|
_, code = strip_id_wrapper(code, stdclass)
|
|
151
165
|
[@registry[stdclass].idtype, code]
|
|
166
|
+
rescue UnknownReferenceError
|
|
167
|
+
[nil, code]
|
|
152
168
|
end
|
|
153
169
|
|
|
154
170
|
# @param key [String]
|
|
@@ -169,43 +185,65 @@ module Relaton
|
|
|
169
185
|
# @return [String]
|
|
170
186
|
def to_xml
|
|
171
187
|
db = @local_db || @db || return
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
188
|
+
parts = db.all.join(" ")
|
|
189
|
+
Moxml.parse(
|
|
190
|
+
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<documents>#{parts}</documents>",
|
|
191
|
+
).to_xml(indent: 0, expand_empty: false)
|
|
175
192
|
end
|
|
176
193
|
|
|
177
194
|
private
|
|
178
195
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
196
|
+
# The flavor routing picks for `fetch_async`'s per-flavor queue; nil
|
|
197
|
+
# (logged) when no flavor recognizes the reference.
|
|
198
|
+
def async_route(ref)
|
|
199
|
+
@registry.route(ref).first
|
|
200
|
+
rescue UnknownReferenceError => e
|
|
201
|
+
Util.info e.message, key: ref
|
|
202
|
+
nil
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# The processor a caller names by short name (`:relaton_iso`) or prefix.
|
|
206
|
+
def named_class(stdclass)
|
|
207
|
+
return unless stdclass
|
|
208
|
+
|
|
209
|
+
short = stdclass.to_sym
|
|
210
|
+
return short if @registry.processors.key?(short)
|
|
211
|
+
|
|
212
|
+
@registry.processors.detect { |_, p| p.prefix == stdclass.to_s }&.first
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# The flavor's `get` receives the parsed pubid when there is one
|
|
216
|
+
# (relaton#205), else the String. The API request sends the String the
|
|
217
|
+
# caller wrote.
|
|
218
|
+
def fetch_doc(code, year, opts, processor, query = nil)
|
|
219
|
+
if Db.configuration.use_api
|
|
220
|
+
fetch_api(code, year, opts, processor, query)
|
|
221
|
+
else processor.get(query || code, year, opts)
|
|
182
222
|
end
|
|
183
223
|
end
|
|
184
224
|
|
|
185
|
-
def fetch_api(code, year, opts, processor)
|
|
225
|
+
def fetch_api(code, year, opts, processor, query = nil)
|
|
186
226
|
url = "#{Db.configuration.api_host}" \
|
|
187
227
|
"/api/v1/document?#{params(code, year, opts)}"
|
|
188
228
|
rsp = Net::HTTP.get_response URI(url)
|
|
189
229
|
processor.from_xml rsp.body if rsp.code == "200"
|
|
190
230
|
rescue Errno::ECONNREFUSED
|
|
191
|
-
processor.get(code, year, opts)
|
|
231
|
+
processor.get(query || code, year, opts)
|
|
192
232
|
end
|
|
193
233
|
|
|
194
234
|
def params(code, year, opts)
|
|
195
235
|
opts.merge(code: code, year: year).map { |k, v| "#{k}=#{v}" }.join "&"
|
|
196
236
|
end
|
|
197
237
|
|
|
198
|
-
def search_xml(
|
|
238
|
+
def search_xml(processor, xml, text, edition, year)
|
|
239
|
+
return unless processor
|
|
199
240
|
return unless text.nil? || match_xml_text?(xml, text)
|
|
200
241
|
|
|
201
|
-
search_edition_year(
|
|
242
|
+
search_edition_year(processor, xml, edition, year)
|
|
202
243
|
end
|
|
203
244
|
|
|
204
|
-
def search_edition_year(
|
|
205
|
-
|
|
206
|
-
item = if file.match?(/xml$/) then processor.from_xml(content)
|
|
207
|
-
else processor.from_yaml(content)
|
|
208
|
-
end
|
|
245
|
+
def search_edition_year(processor, content, edition, year) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
|
246
|
+
item = processor.from_xml(content)
|
|
209
247
|
item if (edition.nil? || item.edition.content == edition) && (year.nil? ||
|
|
210
248
|
item.date.detect do |d|
|
|
211
249
|
d.type == "published" && d.at.to_date.year.to_s == year.to_s
|
|
@@ -240,7 +278,9 @@ module Relaton
|
|
|
240
278
|
if updates
|
|
241
279
|
doc.relation << Bib::Relation.new(bibitem: updates, type: "updates")
|
|
242
280
|
end
|
|
243
|
-
|
|
281
|
+
# The supplement joins its base as the flavor's identifier spells it:
|
|
282
|
+
# `NIST SP 800-38A Add`, not `/Add`, which pubid does not parse.
|
|
283
|
+
divider = %i[relaton_itu relaton_nist].include?(stdclass) ? " " : "/"
|
|
244
284
|
refs[1..].each_with_object(doc) do |c, d|
|
|
245
285
|
bib = check_bibliocache(ref + divider + c, year, opts, stdclass)
|
|
246
286
|
if bib
|
|
@@ -251,18 +291,73 @@ module Relaton
|
|
|
251
291
|
end
|
|
252
292
|
end
|
|
253
293
|
|
|
294
|
+
# The legacy string cache key, for a processor with no pubid class. The
|
|
295
|
+
# publication date range is not part of it: it filters, it is not
|
|
296
|
+
# identity.
|
|
254
297
|
def std_id(code, year, opts, stdclass)
|
|
255
298
|
prefix, code = strip_id_wrapper(code, stdclass)
|
|
256
299
|
ret = code
|
|
257
300
|
ret += (stdclass == :relaton_gb ? "-" : ":") + year if year
|
|
258
301
|
ret += " (all parts)" if opts[:all_parts]
|
|
259
|
-
after = opts[:publication_date_after]
|
|
260
|
-
ret += " after-#{after}" if after
|
|
261
|
-
before = opts[:publication_date_before]
|
|
262
|
-
ret += " before-#{before}" if before
|
|
263
302
|
["#{prefix}(#{ret.strip})", code]
|
|
264
303
|
end
|
|
265
304
|
|
|
305
|
+
#
|
|
306
|
+
# The cache key of a query: the flavor's parsed pubid, with the `year`
|
|
307
|
+
# and `all_parts` options folded in. A reference the flavor cannot parse
|
|
308
|
+
# raises `Pubid::Errors::ParseError`. Nil when the flavor has a pubid
|
|
309
|
+
# class but gives no key for this query (a miss by the flavor's own rule,
|
|
310
|
+
# or a query whose answer the cache cannot hold, such as a CCSDS format):
|
|
311
|
+
# that query is not cached. A processor with no pubid class gets the
|
|
312
|
+
# legacy string key.
|
|
313
|
+
#
|
|
314
|
+
# @param pubid [Pubid::Identifier, nil] the reference as routing parsed
|
|
315
|
+
# it, so the processor does not parse it again
|
|
316
|
+
# @return [Pubid::Identifier, String, nil]
|
|
317
|
+
#
|
|
318
|
+
def cache_key(code, year, opts, stdclass, pubid = nil)
|
|
319
|
+
processor = @registry[stdclass]
|
|
320
|
+
unless processor.pubid_class
|
|
321
|
+
return std_id(code, year, opts, stdclass).first
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
processor.cache_key(code, year, opts, pubid)
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
#
|
|
328
|
+
# The pubid `get` receives, and the cache key folded from it: one parse
|
|
329
|
+
# of the reference serves both (relaton#205). A processor with no pubid
|
|
330
|
+
# class gets the String and the legacy string key.
|
|
331
|
+
#
|
|
332
|
+
# @param pubid [Pubid::Identifier, nil] routing's parse
|
|
333
|
+
# @return [Array(Pubid::Identifier, Object)] the query pubid (nil: `get`
|
|
334
|
+
# gets the String) and the key (nil: not cached)
|
|
335
|
+
#
|
|
336
|
+
def query_and_key(code, year, opts, stdclass, pubid)
|
|
337
|
+
processor = @registry[stdclass]
|
|
338
|
+
unless processor.pubid_class
|
|
339
|
+
return [nil, std_id(code, year, opts, stdclass).first]
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
query = processor.query_pubid(code, opts, pubid)
|
|
343
|
+
[query, query && processor.cache_key(code, year, opts, query)]
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# The key of a fetched document, from its primary identifier. The
|
|
347
|
+
# identifier is data, so an unparseable one gives no key.
|
|
348
|
+
def item_key(bib, stdclass)
|
|
349
|
+
docid = bib.docidentifier.detect(&:primary) || bib.docidentifier.first
|
|
350
|
+
return unless docid&.content
|
|
351
|
+
|
|
352
|
+
cache_key docid.content, nil, {}, stdclass
|
|
353
|
+
rescue ::Pubid::Errors::Error, Parslet::ParseFailed
|
|
354
|
+
nil
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
def date_range?(opts)
|
|
358
|
+
opts[:publication_date_before] || opts[:publication_date_after]
|
|
359
|
+
end
|
|
360
|
+
|
|
266
361
|
def strip_id_wrapper(code, stdclass)
|
|
267
362
|
prefix = @registry[stdclass].prefix
|
|
268
363
|
code =
|
|
@@ -273,66 +368,112 @@ module Relaton
|
|
|
273
368
|
[prefix, code]
|
|
274
369
|
end
|
|
275
370
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
371
|
+
# The routed flavor, then its co-publishers' flavors (relaton#205).
|
|
372
|
+
def fetch_routed(code, year, opts, stdclass, pubid)
|
|
373
|
+
check_bibliocache(code, year, opts, stdclass, pubid: pubid) ||
|
|
374
|
+
copublisher_fetch(code, year, opts, stdclass, pubid)
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
#
|
|
378
|
+
# A co-published document the lead flavor's catalog does not have:
|
|
379
|
+
# ask the co-publishers' flavors, in the order the parsed pubid holds
|
|
380
|
+
# them (relaton#205; pubid#469, #472). Each one keys, caches and fetches
|
|
381
|
+
# with its own flavor, so a second fetch is answered from the caches.
|
|
382
|
+
# A co-publisher whose flavor gives no pubid for the lead's printed form
|
|
383
|
+
# is skipped. Only that probe parse is rescued: an error from inside the
|
|
384
|
+
# co-publisher's fetch propagates.
|
|
385
|
+
#
|
|
386
|
+
# @return [Relaton::Bib::ItemData, nil]
|
|
387
|
+
#
|
|
388
|
+
def copublisher_fetch(code, year, opts, stdclass, pubid)
|
|
389
|
+
# The reference as the lead read it: no `PREFIX(...)` wrapper, no en dash.
|
|
390
|
+
_, code = strip_id_wrapper(code, stdclass)
|
|
391
|
+
copublisher_classes(code, opts, stdclass, pubid).each do |publisher, co|
|
|
392
|
+
Util.info "Not found; trying co-publisher `#{publisher}`", key: code
|
|
393
|
+
query = copublisher_query(code, opts, co, publisher) or next
|
|
394
|
+
bib = check_bibliocache(code, year, opts, co, pubid: query)
|
|
395
|
+
return bib if bib
|
|
279
396
|
end
|
|
397
|
+
nil
|
|
280
398
|
end
|
|
281
399
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
400
|
+
# @return [Array<Array(String, Symbol)>] each co-publisher that a flavor
|
|
401
|
+
# other than the lead serves, with that flavor
|
|
402
|
+
def copublisher_classes(code, opts, stdclass, pubid)
|
|
403
|
+
processor = @registry[stdclass]
|
|
404
|
+
query = processor.query_pubid(code, opts, pubid) or return []
|
|
405
|
+
|
|
406
|
+
processor.copublishers(query).filter_map do |publisher|
|
|
407
|
+
co = @registry.class_by_publisher(publisher)
|
|
408
|
+
[publisher, co] if co && co != stdclass
|
|
409
|
+
end.uniq(&:last)
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# The co-publisher flavor's own parse of the reference, or nil when it
|
|
413
|
+
# gives none (a parse error, or a flavor that reads it as a miss: IEEE),
|
|
414
|
+
# which is logged.
|
|
415
|
+
#
|
|
416
|
+
# @return [Pubid::Identifier, nil]
|
|
417
|
+
def copublisher_query(code, opts, stdclass, publisher)
|
|
418
|
+
query = begin
|
|
419
|
+
@registry[stdclass].query_pubid(code, opts)
|
|
420
|
+
rescue ::Pubid::Errors::Error, Parslet::ParseFailed
|
|
421
|
+
nil
|
|
296
422
|
end
|
|
423
|
+
return query if query
|
|
424
|
+
|
|
425
|
+
Util.info "Not found; co-publisher `#{publisher}` cannot read it",
|
|
426
|
+
key: code
|
|
427
|
+
nil
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
def bib_retval(entry, stdclass)
|
|
431
|
+
@registry[stdclass].from_xml(entry) if entry.is_a?(String)
|
|
432
|
+
end
|
|
297
433
|
|
|
298
|
-
|
|
299
|
-
|
|
434
|
+
def check_bibliocache(code, year, opts, stdclass, pubid: nil) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
|
|
435
|
+
_, searchcode = strip_id_wrapper(code, stdclass)
|
|
436
|
+
query, id = query_and_key(searchcode, year, opts, stdclass, pubid)
|
|
437
|
+
db = id && (@local_db || @db)
|
|
300
438
|
altdb = @local_db && @db ? @db : nil
|
|
301
439
|
if db.nil?
|
|
302
440
|
return if opts[:fetch_db]
|
|
303
441
|
|
|
304
|
-
bibentry = new_bib_entry(searchcode, year, opts, stdclass)
|
|
442
|
+
bibentry = new_bib_entry(searchcode, year, opts, stdclass, query: query)
|
|
305
443
|
return bib_retval(bibentry, stdclass)
|
|
306
444
|
end
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
db.delete(id) unless db.valid_entry?(id, year)
|
|
445
|
+
if date_range?(opts)
|
|
446
|
+
return check_date_range(searchcode, id, year, opts, stdclass, query)
|
|
310
447
|
end
|
|
448
|
+
|
|
449
|
+
@semaphore.synchronize { db.expire id, year }
|
|
311
450
|
if altdb
|
|
312
|
-
return bib_retval(altdb
|
|
451
|
+
return bib_retval(altdb.read(id), stdclass) if opts[:fetch_db]
|
|
313
452
|
|
|
314
453
|
@semaphore.synchronize do
|
|
315
454
|
db.clone_entry id, altdb if altdb.valid_entry? id, year
|
|
316
455
|
end
|
|
317
|
-
new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id
|
|
456
|
+
new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
|
|
457
|
+
query: query)
|
|
318
458
|
@semaphore.synchronize do
|
|
319
459
|
altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
|
|
320
460
|
end
|
|
321
461
|
else
|
|
322
|
-
return bib_retval(db
|
|
462
|
+
return bib_retval(db.read(id), stdclass) if opts[:fetch_db]
|
|
323
463
|
|
|
324
|
-
new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id
|
|
464
|
+
new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
|
|
465
|
+
query: query)
|
|
325
466
|
end
|
|
326
|
-
bib_retval(db
|
|
467
|
+
bib_retval(db.read(id), stdclass)
|
|
327
468
|
end
|
|
328
469
|
|
|
329
470
|
def new_bib_entry(code, year, opts, stdclass, **args)
|
|
330
|
-
entry = @semaphore.synchronize { args[:db]
|
|
471
|
+
entry = @semaphore.synchronize { args[:db]&.read(args[:id]) }
|
|
331
472
|
if !entry || opts[:no_cache]
|
|
332
473
|
return fetch_entry(code, year, opts, stdclass, **args)
|
|
333
474
|
end
|
|
334
475
|
|
|
335
|
-
if entry
|
|
476
|
+
if entry.is_a?(NotFound)
|
|
336
477
|
Util.info "not found in cache, if you wish to " \
|
|
337
478
|
"ignore cache please use `no-cache` option.", key: code
|
|
338
479
|
return
|
|
@@ -340,59 +481,101 @@ module Relaton
|
|
|
340
481
|
entry
|
|
341
482
|
end
|
|
342
483
|
|
|
343
|
-
def fetch_entry(code, year, opts, stdclass, **args)
|
|
484
|
+
def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize
|
|
344
485
|
processor = @registry[stdclass]
|
|
345
|
-
bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1)
|
|
346
|
-
|
|
347
|
-
entry =
|
|
486
|
+
bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
|
|
487
|
+
args[:query])
|
|
488
|
+
entry = bib_entry bib
|
|
348
489
|
return entry if args[:db].nil?
|
|
349
490
|
|
|
350
|
-
|
|
351
|
-
|
|
491
|
+
# `no_cache` refreshes a cached entry, but a failed fetch does not
|
|
492
|
+
# replace a cached document with a not-found entry.
|
|
493
|
+
refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
|
|
494
|
+
@semaphore.synchronize do
|
|
495
|
+
if refresh || !args[:db].read(args[:id])
|
|
496
|
+
save_bib args[:db], args[:id], bib, entry, stdclass
|
|
497
|
+
end
|
|
498
|
+
end
|
|
499
|
+
entry
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
#
|
|
503
|
+
# Cache a fetched document. The document's own identifier gets a row;
|
|
504
|
+
# when the query key differs from it (an undated or incomplete query),
|
|
505
|
+
# the query gets a row that points to the same document.
|
|
506
|
+
#
|
|
507
|
+
def save_bib(db, key, bib, entry, stdclass)
|
|
508
|
+
item = bib.respond_to?(:docidentifier) && item_key(bib, stdclass)
|
|
509
|
+
db.store key, entry, item_key: item || nil
|
|
352
510
|
end
|
|
353
511
|
|
|
354
|
-
|
|
355
|
-
|
|
512
|
+
#
|
|
513
|
+
# A publication date range selects among the cached editions of the
|
|
514
|
+
# reference, so it is not part of the key. On a miss the flavor is asked
|
|
515
|
+
# with the range, and its answer is cached under its own identifier only:
|
|
516
|
+
# the query row keeps pointing to the latest edition.
|
|
517
|
+
#
|
|
518
|
+
def check_date_range(code, key, year, opts, stdclass, query = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
|
519
|
+
caches = [@local_db, @db].compact
|
|
520
|
+
cached = caches.flat_map { |c| c.candidates(key) }.filter_map do |_, xml|
|
|
521
|
+
date = published_date(xml)
|
|
522
|
+
[date, xml] if date && pub_date_in_range?(xml, opts)
|
|
523
|
+
end.max_by(&:first)
|
|
524
|
+
return bib_retval(cached.last, stdclass) if cached && !opts[:no_cache]
|
|
525
|
+
return if opts[:fetch_db]
|
|
526
|
+
|
|
527
|
+
processor = @registry[stdclass]
|
|
528
|
+
bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
|
|
529
|
+
query)
|
|
530
|
+
return unless bib.respond_to?(:to_xml)
|
|
356
531
|
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
else bib_entry bib
|
|
532
|
+
entry = bib_entry bib
|
|
533
|
+
item = item_key(bib, stdclass)
|
|
534
|
+
if item
|
|
535
|
+
@semaphore.synchronize do
|
|
536
|
+
[@local_db, @db].compact.each { |c| c.store item, entry }
|
|
537
|
+
end
|
|
364
538
|
end
|
|
539
|
+
bib_retval entry, stdclass
|
|
365
540
|
end
|
|
366
541
|
|
|
367
|
-
|
|
368
|
-
|
|
542
|
+
# @param query [Pubid::Identifier, nil] what `get` receives in place of
|
|
543
|
+
# the String +code+ (Db#query_and_key)
|
|
544
|
+
def net_retry(code, year, opts, processor, retries, query = nil)
|
|
545
|
+
fetch_doc code, year, opts, processor, query
|
|
369
546
|
rescue Relaton::RequestError => e
|
|
370
547
|
raise e unless retries > 1
|
|
371
548
|
|
|
372
|
-
net_retry(code, year, opts, processor, retries - 1)
|
|
549
|
+
net_retry(code, year, opts, processor, retries - 1, query)
|
|
373
550
|
end
|
|
374
551
|
|
|
375
552
|
def bib_entry(bib)
|
|
376
553
|
if bib.respond_to?(:to_xml)
|
|
377
554
|
bib.to_xml(bibdata: true)
|
|
378
555
|
else
|
|
379
|
-
|
|
556
|
+
NotFound.new(fetched: Date.today.to_s)
|
|
380
557
|
end
|
|
381
558
|
end
|
|
382
559
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
560
|
+
# @param entry [String] document XML
|
|
561
|
+
# @return [Date, nil] the published date
|
|
562
|
+
def published_date(entry)
|
|
563
|
+
date_str = Moxml.parse(entry)
|
|
564
|
+
.at_xpath("//date[@type='published']/on")&.text
|
|
565
|
+
date_str && parse_pub_date(date_str)
|
|
566
|
+
end
|
|
387
567
|
|
|
388
|
-
|
|
568
|
+
def pub_date_in_range?(entry, opts) # rubocop:disable Metrics/CyclomaticComplexity
|
|
569
|
+
date = published_date(entry)
|
|
389
570
|
return false unless date
|
|
390
571
|
|
|
572
|
+
# `parse_pub_date`, not `Date.parse`: a bound may be "YYYY" or
|
|
573
|
+
# "YYYY-MM", which `Date.parse` rejects.
|
|
391
574
|
after = opts[:publication_date_after]
|
|
392
|
-
return false if after && date <
|
|
575
|
+
return false if after && date < parse_pub_date(after.to_s)
|
|
393
576
|
|
|
394
577
|
before = opts[:publication_date_before]
|
|
395
|
-
return false if before && date >=
|
|
578
|
+
return false if before && date >= parse_pub_date(before.to_s)
|
|
396
579
|
|
|
397
580
|
true
|
|
398
581
|
end
|
|
@@ -407,18 +590,8 @@ module Relaton
|
|
|
407
590
|
nil
|
|
408
591
|
end
|
|
409
592
|
|
|
410
|
-
def open_cache_biblio(dir)
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
db = Cache.new dir
|
|
414
|
-
|
|
415
|
-
Dir["#{dir}/*/"].each do |fdir|
|
|
416
|
-
next if db.check_version?(fdir)
|
|
417
|
-
|
|
418
|
-
FileUtils.rm_rf(fdir, secure: true)
|
|
419
|
-
Util.info "cache #{fdir}: version is obsolete and cache is cleared."
|
|
420
|
-
end
|
|
421
|
-
db
|
|
593
|
+
def open_cache_biblio(dir)
|
|
594
|
+
dir && Cache.new(dir)
|
|
422
595
|
end
|
|
423
596
|
|
|
424
597
|
def process_queue(qwp)
|
data/lib/relaton/doi/crossref.rb
CHANGED
|
@@ -10,25 +10,42 @@ module Relaton
|
|
|
10
10
|
#
|
|
11
11
|
# Get a document by DOI from the CrossRef API.
|
|
12
12
|
#
|
|
13
|
-
# @param [String] doi The DOI.
|
|
13
|
+
# @param [String, Pubid::Doi::Identifier] doi The DOI.
|
|
14
14
|
#
|
|
15
15
|
# @return [RelatonBib::BibliographicItem, RelatonIetf::IetfBibliographicItem,
|
|
16
16
|
# RelatonBipm::BipmBibliographicItem, RelatonIeee::IeeeBibliographicItem,
|
|
17
17
|
# RelatonNist::NistBibliographicItem] The bibitem.
|
|
18
18
|
#
|
|
19
19
|
def get(doi)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
message = get_by_id
|
|
20
|
+
key = doi.to_s
|
|
21
|
+
Util.info "Fetching from search.crossref.org ...", key: key
|
|
22
|
+
message = get_by_id doi_of(doi)
|
|
23
23
|
if message
|
|
24
|
-
Util.info "Found: `#{message['DOI']}`", key:
|
|
24
|
+
Util.info "Found: `#{message['DOI']}`", key: key
|
|
25
25
|
Parser.parse message
|
|
26
26
|
else
|
|
27
|
-
Util.info "Not found.", key:
|
|
27
|
+
Util.info "Not found.", key: key
|
|
28
28
|
nil
|
|
29
29
|
end
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
+
#
|
|
33
|
+
# The DOI itself (`<prefix>/<suffix>`), as the Crossref API takes it.
|
|
34
|
+
#
|
|
35
|
+
# A parsed pubid gives it from its components. A String may carry a
|
|
36
|
+
# `doi:` scheme (in any case) or a `doi.org` URL: Relaton::Db keys every
|
|
37
|
+
# form Pubid::Doi reads as one DOI, so all of them must reach the same
|
|
38
|
+
# DOI here, or a miss on one form is cached for the others.
|
|
39
|
+
#
|
|
40
|
+
# @param [String, Pubid::Doi::Identifier] doi
|
|
41
|
+
# @return [String]
|
|
42
|
+
#
|
|
43
|
+
def doi_of(doi)
|
|
44
|
+
return "#{doi.prefix}/#{doi.suffix}" unless doi.is_a?(String)
|
|
45
|
+
|
|
46
|
+
doi.sub(%r{\A(?:doi:|https?://(?:dx\.)?doi\.org/)}i, "")
|
|
47
|
+
end
|
|
48
|
+
|
|
32
49
|
#
|
|
33
50
|
# Get a document by DOI from the CrossRef API.
|
|
34
51
|
#
|