relaton 3.1.3 → 3.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +13 -3
- data/lib/relaton/bib/converter/endnote/from_endnote.rb +118 -0
- data/lib/relaton/bib/converter/endnote/to_endnote.rb +107 -0
- data/lib/relaton/bib/converter/endnote.rb +30 -0
- data/lib/relaton/bib/converter/ris/from_ris.rb +144 -0
- data/lib/relaton/bib/converter/ris/to_ris.rb +92 -0
- data/lib/relaton/bib/converter/ris.rb +7 -81
- data/lib/relaton/bib/item_data.rb +4 -0
- data/lib/relaton/bib.rb +1 -0
- data/lib/relaton/bibcollection.rb +4 -1
- data/lib/relaton/cli/base_convertor.rb +19 -3
- data/lib/relaton/cli/collection_store.rb +171 -0
- data/lib/relaton/cli/command.rb +39 -2
- data/lib/relaton/cli/subcommand_collection.rb +56 -63
- data/lib/relaton/cli/util.rb +6 -0
- data/lib/relaton/cli/xml_to_html_renderer.rb +14 -7
- data/lib/relaton/db/api_resolver.rb +115 -0
- data/lib/relaton/db/cache.rb +59 -13
- data/lib/relaton/db/cache_entry.rb +5 -1
- data/lib/relaton/db/config.rb +50 -5
- data/lib/relaton/db.rb +97 -23
- data/lib/relaton/itu/hit_collection.rb +4 -3
- data/lib/relaton/itu/scraper.rb +55 -26
- data/lib/relaton/render/iso690/styles/jis-en.yml +21 -5
- data/lib/relaton/render/iso690/styles/jis-ja.yml +11 -4
- data/lib/relaton/version.rb +1 -1
- metadata +13 -6
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
require "net/http"
|
|
2
|
+
require "uri"
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "zlib"
|
|
5
|
+
require_relative "../version"
|
|
6
|
+
|
|
7
|
+
module Relaton
|
|
8
|
+
class Db
|
|
9
|
+
#
|
|
10
|
+
# The api.relaton.org tier of a `Db` fetch (relaton#535): local cache,
|
|
11
|
+
# then this, then the flavor's own `get`.
|
|
12
|
+
#
|
|
13
|
+
# It reads `GET <api_host>/v3/documents/<key>` as Relaton YAML, where the
|
|
14
|
+
# key is one percent-encoded path segment. The API is not authoritative
|
|
15
|
+
# while its coverage is verified per flavor, so every answer it cannot
|
|
16
|
+
# give with certainty is nil, and `Db#fetch_doc` then asks the flavor:
|
|
17
|
+
# a 404, any other status, a transport failure, and a body the flavor
|
|
18
|
+
# cannot read. Nothing here raises, and nothing here touches the cache:
|
|
19
|
+
# `Db` stores the answer as it stores a flavor's, with its ETag, and
|
|
20
|
+
# revalidates a stale row with that ETag.
|
|
21
|
+
#
|
|
22
|
+
module ApiResolver
|
|
23
|
+
ACCEPT = "application/yaml".freeze
|
|
24
|
+
# The transport failures that mean "ask the flavor" (the list of
|
|
25
|
+
# `Index::ShardSource::NET_ERRORS`; not required from there, so that
|
|
26
|
+
# `require "relaton/db"` opens no partial `Relaton::Index`).
|
|
27
|
+
NET_ERRORS = [
|
|
28
|
+
SocketError, Timeout::Error, IOError, SystemCallError,
|
|
29
|
+
OpenSSL::SSL::SSLError, Net::HTTPBadResponse,
|
|
30
|
+
Net::HTTPHeaderSyntaxError, Net::ProtocolError, Zlib::Error
|
|
31
|
+
].freeze
|
|
32
|
+
|
|
33
|
+
#
|
|
34
|
+
# What the API answered. `item` is the document of a 200, with its
|
|
35
|
+
# `etag`; `not_modified` is a 304 to a revalidation, whose `etag` is
|
|
36
|
+
# the one sent (the cached document is still the API's).
|
|
37
|
+
#
|
|
38
|
+
Answer = Data.define(:item, :etag, :not_modified) do
|
|
39
|
+
def initialize(item: nil, etag: nil, not_modified: false)
|
|
40
|
+
super
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
module_function
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
# @param processor [Relaton::Core::Processor] the routed flavor
|
|
48
|
+
# @param key [Pubid::Identifier] the query's cache key; the API is
|
|
49
|
+
# asked for its rendering
|
|
50
|
+
# @param etag [String, nil] the ETag of a stale cached answer: sent as
|
|
51
|
+
# `If-None-Match`, so an unchanged document answers 304 with no body.
|
|
52
|
+
# The ETag is of the YAML representation, and the cache holds XML,
|
|
53
|
+
# so it is the header value as received, never computed here.
|
|
54
|
+
# @return [Answer, nil] nil: ask the flavor
|
|
55
|
+
#
|
|
56
|
+
def fetch(processor, key, etag: nil)
|
|
57
|
+
rsp = get(key.to_s, etag) or return
|
|
58
|
+
return Answer.new(etag: etag, not_modified: true) if rsp.code == "304"
|
|
59
|
+
|
|
60
|
+
item = read(processor, rsp.body, key) or return
|
|
61
|
+
Answer.new(item: item, etag: rsp["ETag"])
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @return [Net::HTTPResponse, nil] a 200, or a 304 to a revalidation;
|
|
65
|
+
# else nil
|
|
66
|
+
def get(ref, etag = nil)
|
|
67
|
+
host = Db.configuration.api_host.chomp("/")
|
|
68
|
+
uri = URI("#{host}/v3/documents/#{escape(ref)}")
|
|
69
|
+
rsp = request(uri, etag)
|
|
70
|
+
return rsp if rsp.code == "200" || (etag && rsp.code == "304")
|
|
71
|
+
|
|
72
|
+
info "HTTP #{rsp.code}", ref
|
|
73
|
+
nil
|
|
74
|
+
rescue *NET_ERRORS => e
|
|
75
|
+
info "#{e.class}: #{e.message}", ref
|
|
76
|
+
nil
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def request(uri, etag = nil)
|
|
80
|
+
timeout = Db.configuration.api_timeout
|
|
81
|
+
headers = { "Accept" => ACCEPT,
|
|
82
|
+
"User-Agent" => "relaton/#{Relaton::VERSION}" }
|
|
83
|
+
headers["If-None-Match"] = etag if etag
|
|
84
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
|
|
85
|
+
open_timeout: timeout,
|
|
86
|
+
read_timeout: timeout) do |http|
|
|
87
|
+
http.get(uri.request_uri, headers)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The API's body is data: a body the flavor's YAML model rejects, or
|
|
92
|
+
# one that is not a document (`--- []` reads as an Array), is a miss,
|
|
93
|
+
# whatever the model raises for it.
|
|
94
|
+
def read(processor, body, ref)
|
|
95
|
+
item = processor.from_yaml body
|
|
96
|
+
return item if item.is_a?(Relaton::Bib::ItemData)
|
|
97
|
+
|
|
98
|
+
info "answer is not a document (#{item.class})", ref
|
|
99
|
+
nil
|
|
100
|
+
rescue StandardError => e
|
|
101
|
+
info "unreadable answer (#{e.class}: #{e.message})", ref
|
|
102
|
+
nil
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# One path segment: `/` and `:` are escaped, a space is `%20`.
|
|
106
|
+
def escape(ref)
|
|
107
|
+
URI.encode_www_form_component(ref).gsub("+", "%20")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def info(reason, ref)
|
|
111
|
+
Util.info "api.relaton.org: #{reason}; asking the flavor", key: ref
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
data/lib/relaton/db/cache.rb
CHANGED
|
@@ -25,6 +25,13 @@ module Relaton
|
|
|
25
25
|
class Cache
|
|
26
26
|
LAYOUT = "v2".freeze
|
|
27
27
|
VERSIONS_KEY = "_versions".freeze
|
|
28
|
+
# The rows key that records the one-shot v1 import (relaton#240). It
|
|
29
|
+
# lives in the store, not in a marker file beside it (relaton#548).
|
|
30
|
+
V1_IMPORTED_KEY = "_v1_imported".freeze
|
|
31
|
+
# The legacy marker file of that import, read but no longer written.
|
|
32
|
+
V1_IMPORTED_FILE = ".v1-imported".freeze
|
|
33
|
+
# Rows keys that hold no bucket.
|
|
34
|
+
RESERVED_KEYS = [VERSIONS_KEY, V1_IMPORTED_KEY].freeze
|
|
28
35
|
STRING_FLAVOR = "_key".freeze
|
|
29
36
|
PUBID_FLAVOR = "_pubid".freeze
|
|
30
37
|
# The legacy string form of a not-found entry (`not_found <date>`).
|
|
@@ -75,10 +82,15 @@ module Relaton
|
|
|
75
82
|
@dir
|
|
76
83
|
end
|
|
77
84
|
|
|
78
|
-
# Remove every row and document
|
|
85
|
+
# Remove every row and document, under one rows transaction (lock
|
|
86
|
+
# order rows, then docs). The v1 import marker is never deleted, not
|
|
87
|
+
# even for a moment, so a cleared cache is not filled again from
|
|
88
|
+
# `<dir>-v1.bak` by this or another process.
|
|
79
89
|
def clear
|
|
80
|
-
@rows.
|
|
81
|
-
|
|
90
|
+
@rows.adapter.transaction do
|
|
91
|
+
@rows.each_key { |k| @rows.delete k unless k == V1_IMPORTED_KEY }
|
|
92
|
+
@docs.clear
|
|
93
|
+
end
|
|
82
94
|
@versions = {}
|
|
83
95
|
end
|
|
84
96
|
|
|
@@ -118,9 +130,11 @@ module Relaton
|
|
|
118
130
|
# @param value [String, Relaton::Db::NotFound, nil] document XML, a
|
|
119
131
|
# not-found entry (also as `not_found <date>`); nil deletes the row
|
|
120
132
|
# @param item_key [Pubid::Identifier, String, nil] the document's key
|
|
133
|
+
# @param etag [String, nil] the ETag of the API answer the document
|
|
134
|
+
# came from (relaton#535), kept on both rows; nil for a flavor answer
|
|
121
135
|
# @return [String, Relaton::Db::NotFound, nil] value
|
|
122
136
|
#
|
|
123
|
-
def store(key, value, item_key: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
137
|
+
def store(key, value, item_key: nil, etag: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
124
138
|
return delete(key) if value.nil?
|
|
125
139
|
|
|
126
140
|
bucket, key = resolve key
|
|
@@ -137,9 +151,10 @@ module Relaton
|
|
|
137
151
|
record_version item_bucket
|
|
138
152
|
@rows.adapter.transaction do
|
|
139
153
|
@docs.set file, value
|
|
140
|
-
upsert item_bucket,
|
|
154
|
+
upsert item_bucket,
|
|
155
|
+
new_row(item_key, CacheEntry::DOC, file, fetched, etag)
|
|
141
156
|
unless canonical(key) == canonical(item_key)
|
|
142
|
-
upsert bucket, new_row(key, CacheEntry::DOC, file, fetched)
|
|
157
|
+
upsert bucket, new_row(key, CacheEntry::DOC, file, fetched, etag)
|
|
143
158
|
end
|
|
144
159
|
end
|
|
145
160
|
value
|
|
@@ -183,7 +198,7 @@ module Relaton
|
|
|
183
198
|
|
|
184
199
|
found = other.lookup(key) or return
|
|
185
200
|
row_id, row = found
|
|
186
|
-
store row_id, other.read_entry(row)
|
|
201
|
+
store row_id, other.read_entry(row), etag: row["etag"]
|
|
187
202
|
end
|
|
188
203
|
|
|
189
204
|
# @param key [Pubid::Identifier, String]
|
|
@@ -192,6 +207,30 @@ module Relaton
|
|
|
192
207
|
lookup(key)&.last&.dig("fetched")
|
|
193
208
|
end
|
|
194
209
|
|
|
210
|
+
# @param key [Pubid::Identifier, String]
|
|
211
|
+
# @return [String, nil] the ETag of the API answer the entry came from
|
|
212
|
+
def etag(key)
|
|
213
|
+
lookup(key)&.last&.dig("etag")
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
#
|
|
217
|
+
# The document and the ETag to revalidate a key with, read from one
|
|
218
|
+
# row in one lookup, so the two belong to one answer. Only the key's
|
|
219
|
+
# own row counts: a dated key's lookup can also read a row that adds a
|
|
220
|
+
# language (`#lookup`), and that row's ETag was issued for another URL.
|
|
221
|
+
#
|
|
222
|
+
# @param key [Pubid::Identifier, String]
|
|
223
|
+
# @return [Array(String, String), nil] the document XML and its ETag
|
|
224
|
+
#
|
|
225
|
+
def revalidation(key)
|
|
226
|
+
row = lookup(key)&.last
|
|
227
|
+
own = canonical resolve(key).last
|
|
228
|
+
return unless row && row["etag"] && row_key(row) == own
|
|
229
|
+
|
|
230
|
+
xml = read_entry row
|
|
231
|
+
[xml, row["etag"]] if xml.is_a?(String)
|
|
232
|
+
end
|
|
233
|
+
|
|
195
234
|
# An undated entry expires after 60 days, a dated document never. A
|
|
196
235
|
# not_found entry expires after 60 days even for a dated query: the
|
|
197
236
|
# document can be published later.
|
|
@@ -319,8 +358,7 @@ module Relaton
|
|
|
319
358
|
# fetch, and redirects were dropped from the v2 model).
|
|
320
359
|
def import_old_layout
|
|
321
360
|
bak = "#{dir}-v1.bak"
|
|
322
|
-
|
|
323
|
-
return if File.exist?(marker) || !Dir.exist?(bak)
|
|
361
|
+
return if v1_imported? || !Dir.exist?(bak)
|
|
324
362
|
|
|
325
363
|
imported = 0
|
|
326
364
|
Dir.glob("#{bak}/**/*.xml").each do |f|
|
|
@@ -331,10 +369,17 @@ module Relaton
|
|
|
331
369
|
self[key] = value
|
|
332
370
|
imported += 1
|
|
333
371
|
end
|
|
334
|
-
|
|
372
|
+
@rows.set V1_IMPORTED_KEY, Date.today.to_s
|
|
335
373
|
Util.info "cache #{dir}: imported #{imported} entries from #{bak}"
|
|
336
374
|
end
|
|
337
375
|
|
|
376
|
+
# An older relaton wrote the marker as a file; it still counts. `get`,
|
|
377
|
+
# not `exists?`: it takes the shared lock, so it waits for a write.
|
|
378
|
+
def v1_imported?
|
|
379
|
+
!@rows.get(V1_IMPORTED_KEY).nil? ||
|
|
380
|
+
File.exist?(File.join(dir, LAYOUT, V1_IMPORTED_FILE))
|
|
381
|
+
end
|
|
382
|
+
|
|
338
383
|
# The wrapped string key of an old-layout document: its own primary
|
|
339
384
|
# docidentifier under its flavor prefix, resolved through the same
|
|
340
385
|
# `wrapped_key` path a string cache read takes. A value no parser reads
|
|
@@ -433,7 +478,7 @@ module Relaton
|
|
|
433
478
|
|
|
434
479
|
def each_row
|
|
435
480
|
@rows.each_key do |bucket|
|
|
436
|
-
next if bucket
|
|
481
|
+
next if RESERVED_KEYS.include? bucket
|
|
437
482
|
|
|
438
483
|
read_bucket(bucket).each { |row| yield bucket, row }
|
|
439
484
|
end
|
|
@@ -510,8 +555,9 @@ module Relaton
|
|
|
510
555
|
end
|
|
511
556
|
end
|
|
512
557
|
|
|
513
|
-
def new_row(key, status, file, fetched)
|
|
514
|
-
row = CacheEntry.new(status: status, file: file, fetched: fetched
|
|
558
|
+
def new_row(key, status, file, fetched, etag = nil)
|
|
559
|
+
row = CacheEntry.new(status: status, file: file, fetched: fetched,
|
|
560
|
+
etag: etag)
|
|
515
561
|
if key.is_a?(String) then row.key = key
|
|
516
562
|
else row.id = canonical(key)
|
|
517
563
|
end
|
|
@@ -15,7 +15,9 @@ module Relaton
|
|
|
15
15
|
# `to_hash`) or, for a processor with no pubid class, by a string (`key`).
|
|
16
16
|
# `file` names the document in the doc store; several rows can share one
|
|
17
17
|
# file (a query row and the row of the document it returned). A
|
|
18
|
-
# `not_found` row has no file.
|
|
18
|
+
# `not_found` row has no file. `etag` is the ETag of the api.relaton.org
|
|
19
|
+
# answer the document came from (relaton#535), for revalidation; a
|
|
20
|
+
# flavor answer, and a row an older relaton wrote, have none.
|
|
19
21
|
class CacheEntry < Lutaml::Model::Serializable
|
|
20
22
|
DOC = "doc".freeze
|
|
21
23
|
NOT_FOUND = "not_found".freeze
|
|
@@ -25,6 +27,7 @@ module Relaton
|
|
|
25
27
|
attribute :status, :string
|
|
26
28
|
attribute :file, :string
|
|
27
29
|
attribute :fetched, :string
|
|
30
|
+
attribute :etag, :string
|
|
28
31
|
|
|
29
32
|
key_value do
|
|
30
33
|
map "id", to: :id
|
|
@@ -32,6 +35,7 @@ module Relaton
|
|
|
32
35
|
map "status", to: :status
|
|
33
36
|
map "file", to: :file
|
|
34
37
|
map "fetched", to: :fetched
|
|
38
|
+
map "etag", to: :etag
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
def not_found?
|
data/lib/relaton/db/config.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
1
3
|
module Relaton
|
|
2
4
|
class Db
|
|
3
5
|
module Config
|
|
@@ -11,19 +13,62 @@ module Relaton
|
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
class Configuration
|
|
16
|
+
# Values of a `RELATON_*` switch (any case) that turn it on.
|
|
17
|
+
SWITCH_ON = %w[1 true yes on].freeze
|
|
14
18
|
# Values of RELATON_OFFLINE that turn offline mode on.
|
|
15
|
-
OFFLINE_ON =
|
|
19
|
+
OFFLINE_ON = SWITCH_ON
|
|
20
|
+
DEFAULT_API_HOST = "https://api.relaton.org".freeze
|
|
16
21
|
|
|
17
22
|
# @!attribute offline
|
|
18
23
|
# @return [Boolean] true: every fetch is answered from the local
|
|
19
24
|
# store only, and a miss raises Relaton::NotInLocalStoreError
|
|
20
25
|
# (relaton#536). Defaults to RELATON_OFFLINE.
|
|
21
|
-
|
|
26
|
+
#
|
|
27
|
+
# @!attribute use_api
|
|
28
|
+
# @return [Boolean] true: a cache miss asks api.relaton.org first,
|
|
29
|
+
# and the flavor answers what the API does not (relaton#535).
|
|
30
|
+
# Defaults to RELATON_USE_API, else false. The CLI's `--api` /
|
|
31
|
+
# `--no-api` sets it for one run.
|
|
32
|
+
# @!attribute api_host
|
|
33
|
+
# @return [String] the API's base URL. Defaults to RELATON_API_HOST,
|
|
34
|
+
# else https://api.relaton.org.
|
|
35
|
+
# @!attribute api_timeout
|
|
36
|
+
# @return [Numeric] open and read timeout of an API request, seconds
|
|
37
|
+
attr_accessor :use_api, :api_host, :api_timeout, :offline
|
|
22
38
|
|
|
23
39
|
def initialize
|
|
24
|
-
@use_api =
|
|
25
|
-
@api_host =
|
|
26
|
-
@
|
|
40
|
+
@use_api = switch? "RELATON_USE_API"
|
|
41
|
+
@api_host = env_api_host || DEFAULT_API_HOST
|
|
42
|
+
@api_timeout = 10
|
|
43
|
+
@offline = switch? "RELATON_OFFLINE"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def switch?(name)
|
|
49
|
+
SWITCH_ON.include? ENV[name].to_s.downcase
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# RELATON_API_HOST, when it is an http(s) URL with a host. A shell
|
|
53
|
+
# value is easy to write without its scheme (`api.example.org`), and
|
|
54
|
+
# the resolver would then ask a host-less URI; that is warned about,
|
|
55
|
+
# not used.
|
|
56
|
+
def env_api_host
|
|
57
|
+
host = ENV["RELATON_API_HOST"].to_s.strip
|
|
58
|
+
return if host.empty?
|
|
59
|
+
|
|
60
|
+
uri = URI.parse(host)
|
|
61
|
+
return host if uri.is_a?(URI::HTTP) && uri.host && !uri.host.empty?
|
|
62
|
+
|
|
63
|
+
invalid_api_host host
|
|
64
|
+
rescue URI::InvalidURIError
|
|
65
|
+
invalid_api_host host
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def invalid_api_host(host)
|
|
69
|
+
Util.warn "RELATON_API_HOST #{host.inspect} is not an http(s) URL; " \
|
|
70
|
+
"using #{DEFAULT_API_HOST}"
|
|
71
|
+
nil
|
|
27
72
|
end
|
|
28
73
|
end
|
|
29
74
|
|
data/lib/relaton/db.rb
CHANGED
|
@@ -287,26 +287,94 @@ module Relaton
|
|
|
287
287
|
end
|
|
288
288
|
|
|
289
289
|
# The flavor's `get` receives the parsed pubid when there is one
|
|
290
|
-
# (relaton#205), else the String.
|
|
291
|
-
#
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
290
|
+
# (relaton#205), else the String. With `use_api`, api.relaton.org is
|
|
291
|
+
# asked first (relaton#535), and the flavor answers whatever it does not.
|
|
292
|
+
#
|
|
293
|
+
# @param api [Hash] the API tier's state for one fetch: in,
|
|
294
|
+
# `:stale_etag`/`:stale` (a stale row's ETag and its document, to
|
|
295
|
+
# revalidate); out, `:etag` (the ETag of an API answer, nil for a
|
|
296
|
+
# flavor answer), which Db#fetch_entry stores with the row
|
|
297
|
+
def fetch_doc(code, year, opts, processor, query = nil, api: {})
|
|
298
|
+
api_answer(code, year, opts, processor, query, api) ||
|
|
299
|
+
processor.get(query || code, year, opts)
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
#
|
|
303
|
+
# The API's document for the query, or nil. The API is asked only for a
|
|
304
|
+
# query that has a pubid cache key, is not all-parts and has no
|
|
305
|
+
# publication date range: the API has no all-parts answer, and a range
|
|
306
|
+
# is not in the key. Its answer is kept only when the document's own
|
|
307
|
+
# cache key is the query's key: the API has no undated or edition logic
|
|
308
|
+
# (`ISO 19115-1` answers the 2014 record), and the flavor's rule for
|
|
309
|
+
# such a query is the one `Db` caches.
|
|
310
|
+
#
|
|
311
|
+
# A stale row the API wrote is revalidated with its ETag. A 304 answers
|
|
312
|
+
# the cached document, fetched today: `Cache#store` reads the row's
|
|
313
|
+
# `fetched` from the document, so the document is stored again, not
|
|
314
|
+
# only its row. It needs no identity check: the row was written from an
|
|
315
|
+
# answer that passed one, for this key.
|
|
316
|
+
#
|
|
317
|
+
def api_answer(code, year, opts, processor, query, api) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists
|
|
318
|
+
api[:etag] = nil
|
|
319
|
+
return unless Db.configuration.use_api
|
|
320
|
+
return if opts[:all_parts] || date_range?(opts)
|
|
321
|
+
|
|
322
|
+
key = query && processor.cache_key(code, year, opts, query)
|
|
323
|
+
return unless key.is_a?(::Pubid::Identifier)
|
|
324
|
+
|
|
325
|
+
answer = ApiResolver.fetch(processor, key, etag: api[:stale_etag])
|
|
326
|
+
return unless answer
|
|
327
|
+
return revalidated(processor, api, answer, key) if answer.not_modified
|
|
328
|
+
|
|
329
|
+
bib = answer.item
|
|
330
|
+
if api_identity?(bib, key, processor)
|
|
331
|
+
api[:etag] = answer.etag
|
|
332
|
+
return bib
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
Util.info "api.relaton.org: answered another document " \
|
|
336
|
+
"(#{bib.docidentifier.first&.content}); asking the flavor",
|
|
337
|
+
key: key.to_s
|
|
338
|
+
nil
|
|
297
339
|
end
|
|
298
340
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
processor.from_xml
|
|
304
|
-
|
|
305
|
-
|
|
341
|
+
# The stale document a 304 kept, fetched today, with its ETag. It is
|
|
342
|
+
# cached data: one that does not read back is a miss (the flavor
|
|
343
|
+
# answers), whatever the model raises for it.
|
|
344
|
+
def revalidated(processor, api, answer, key)
|
|
345
|
+
bib = processor.from_xml api[:stale]
|
|
346
|
+
bib.fetched = Date.today.to_s
|
|
347
|
+
api[:etag] = answer.etag
|
|
348
|
+
bib
|
|
349
|
+
rescue StandardError => e
|
|
350
|
+
Util.info "api.relaton.org: cached document unreadable " \
|
|
351
|
+
"(#{e.class}); asking the flavor", key: key.to_s
|
|
352
|
+
nil
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# The API state Db#fetch_doc starts from: a stale document that the API
|
|
356
|
+
# answered carries the ETag to revalidate it with. Both come from one
|
|
357
|
+
# row of the stale cache (Cache#revalidation), read again here: the
|
|
358
|
+
# stale entry `with_stale` read is not used, since another worker may
|
|
359
|
+
# have replaced the row since, and its ETag would then confirm an older
|
|
360
|
+
# document.
|
|
361
|
+
def api_state(args)
|
|
362
|
+
return {} unless args[:stale].is_a?(String) && args[:stale_db]
|
|
363
|
+
|
|
364
|
+
xml, etag = @semaphore.synchronize do
|
|
365
|
+
args[:stale_db].revalidation(args[:id])
|
|
366
|
+
end
|
|
367
|
+
etag ? { stale_etag: etag, stale: xml } : {}
|
|
306
368
|
end
|
|
307
369
|
|
|
308
|
-
|
|
309
|
-
|
|
370
|
+
# The API's document is data: an identifier the flavor cannot key is a
|
|
371
|
+
# mismatch, whatever the key raises for it.
|
|
372
|
+
def api_identity?(bib, key, processor)
|
|
373
|
+
item_key(bib, @registry.processors.key(processor)) == key
|
|
374
|
+
rescue StandardError => e
|
|
375
|
+
Util.info "api.relaton.org: unkeyable answer (#{e.class}); " \
|
|
376
|
+
"asking the flavor", key: key.to_s
|
|
377
|
+
false
|
|
310
378
|
end
|
|
311
379
|
|
|
312
380
|
def search_xml(processor, xml, text, edition, year)
|
|
@@ -600,9 +668,10 @@ module Relaton
|
|
|
600
668
|
#
|
|
601
669
|
def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
602
670
|
processor = @registry[stdclass]
|
|
671
|
+
api = api_state args
|
|
603
672
|
bib = begin
|
|
604
673
|
net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
|
|
605
|
-
args[:query])
|
|
674
|
+
args[:query], api: api)
|
|
606
675
|
rescue Relaton::RequestError => e
|
|
607
676
|
raise unless args[:stale].is_a?(String)
|
|
608
677
|
|
|
@@ -621,7 +690,8 @@ module Relaton
|
|
|
621
690
|
@semaphore.synchronize do
|
|
622
691
|
args[:db].expire args[:id], year if args[:stale]
|
|
623
692
|
if args[:stale] || refresh || !args[:db].read(args[:id])
|
|
624
|
-
save_bib args[:db], args[:id], bib, entry, stdclass
|
|
693
|
+
save_bib args[:db], args[:id], bib, entry, stdclass,
|
|
694
|
+
etag: api[:etag]
|
|
625
695
|
end
|
|
626
696
|
end
|
|
627
697
|
entry
|
|
@@ -632,9 +702,11 @@ module Relaton
|
|
|
632
702
|
# when the query key differs from it (an undated or incomplete query),
|
|
633
703
|
# the query gets a row that points to the same document.
|
|
634
704
|
#
|
|
635
|
-
|
|
705
|
+
# `etag` is the ETag of the API answer (Db#fetch_doc), nil for a flavor
|
|
706
|
+
# answer.
|
|
707
|
+
def save_bib(db, key, bib, entry, stdclass, etag: nil) # rubocop:disable Metrics/ParameterLists
|
|
636
708
|
item = bib.respond_to?(:docidentifier) && item_key(bib, stdclass)
|
|
637
|
-
db.store key, entry, item_key: item || nil
|
|
709
|
+
db.store key, entry, item_key: item || nil, etag: etag
|
|
638
710
|
end
|
|
639
711
|
|
|
640
712
|
#
|
|
@@ -686,12 +758,13 @@ module Relaton
|
|
|
686
758
|
|
|
687
759
|
# @param query [Pubid::Identifier, nil] what `get` receives in place of
|
|
688
760
|
# the String +code+ (Db#query_and_key)
|
|
689
|
-
|
|
690
|
-
|
|
761
|
+
# @param api [Hash] the API tier's state (Db#fetch_doc)
|
|
762
|
+
def net_retry(code, year, opts, processor, retries, query = nil, api: {}) # rubocop:disable Metrics/ParameterLists
|
|
763
|
+
fetch_doc code, year, opts, processor, query, api: api
|
|
691
764
|
rescue Relaton::RequestError => e
|
|
692
765
|
raise e unless retries > 1
|
|
693
766
|
|
|
694
|
-
net_retry(code, year, opts, processor, retries - 1, query)
|
|
767
|
+
net_retry(code, year, opts, processor, retries - 1, query, api: api)
|
|
695
768
|
end
|
|
696
769
|
|
|
697
770
|
def bib_entry(bib)
|
|
@@ -782,6 +855,7 @@ end
|
|
|
782
855
|
|
|
783
856
|
require_relative "db/util"
|
|
784
857
|
require_relative "db/config"
|
|
858
|
+
require_relative "db/api_resolver"
|
|
785
859
|
require_relative "db/workers_pool"
|
|
786
860
|
require_relative "db/cache"
|
|
787
861
|
require_relative "db/registry"
|
|
@@ -296,14 +296,15 @@ module Relaton
|
|
|
296
296
|
"T-SP-OB.#{ref.number}-#{ref.year}" if bulletin?
|
|
297
297
|
end
|
|
298
298
|
|
|
299
|
-
# The docidentifier (year only, no month or day). A bulletin
|
|
300
|
-
# sector even when the reference is the
|
|
299
|
+
# The docidentifier (year only, no month or day). A bulletin carries no
|
|
300
|
+
# sector even when the reference is the TSB spelling `ITU-T OB.1096`
|
|
301
|
+
# (pubid#696); `Scraper#bureau` still makes TSB its author.
|
|
301
302
|
#
|
|
302
303
|
# @return [String]
|
|
303
304
|
def publication_code
|
|
304
305
|
return "ITU-R RR (#{ref.year})" if radio_regulations?
|
|
305
306
|
|
|
306
|
-
"ITU
|
|
307
|
+
"ITU OB No. #{ref.number} (#{ref.year})"
|
|
307
308
|
end
|
|
308
309
|
|
|
309
310
|
def radio_regulations?
|
data/lib/relaton/itu/scraper.rb
CHANGED
|
@@ -141,30 +141,70 @@ module Relaton
|
|
|
141
141
|
def fetch_publisher_contributors
|
|
142
142
|
return [] unless hit.hit[:code]
|
|
143
143
|
|
|
144
|
-
abbrev = hit.hit[:code].sub(/-\w\s.*/, "")
|
|
145
|
-
case abbrev
|
|
146
|
-
when "ITU"
|
|
147
|
-
name = "International Telecommunication Union"
|
|
148
|
-
url = "www.itu.int"
|
|
149
|
-
end
|
|
150
|
-
org = Relaton::Bib::Organization.new(
|
|
151
|
-
name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
|
|
152
|
-
abbreviation: Relaton::Bib::LocalizedString.new(content: abbrev),
|
|
153
|
-
uri: [Relaton::Bib::Uri.new(content: url)],
|
|
154
|
-
)
|
|
155
144
|
role = Relaton::Bib::Contributor::Role.new(type: "publisher")
|
|
156
|
-
[Relaton::Bib::Contributor.new(organization:
|
|
145
|
+
[Relaton::Bib::Contributor.new(organization: publisher_org,
|
|
146
|
+
role: [role])]
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The publishing organization. Every ITU code names ITU, so a code pubid
|
|
150
|
+
# cannot parse is still ITU's.
|
|
151
|
+
#
|
|
152
|
+
# @return [Relaton::Bib::Organization]
|
|
153
|
+
def publisher_org
|
|
154
|
+
Relaton::Bib::Organization.new(
|
|
155
|
+
name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
|
|
156
|
+
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
157
|
+
content: code_pubid&.publisher || "ITU",
|
|
158
|
+
),
|
|
159
|
+
uri: [Relaton::Bib::Uri.new(content: "www.itu.int")],
|
|
160
|
+
)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# The pubid parse of the hit's (first) code. nil when the hit has no
|
|
164
|
+
# code, or when ITU spells it in a form pubid cannot identify.
|
|
165
|
+
#
|
|
166
|
+
# @return [Pubid::Itu::Identifiers::Base, nil]
|
|
167
|
+
def code_pubid
|
|
168
|
+
return @code_pubid if defined? @code_pubid
|
|
169
|
+
|
|
170
|
+
code = hit.hit[:code].to_s.split(" | ").first
|
|
171
|
+
@code_pubid = code && ::Pubid::Itu.parse(code)
|
|
172
|
+
rescue ::Pubid::Errors::ParseError
|
|
173
|
+
@code_pubid = nil
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# The bureau letter (T, R or D) of the code: its sector, or the sector
|
|
177
|
+
# of the document a wrapper (an annex, an appendix) belongs to. An
|
|
178
|
+
# Operational Bulletin prints no sector (pubid#696), but TSB produces it.
|
|
179
|
+
#
|
|
180
|
+
# @return [String, nil]
|
|
181
|
+
def bureau
|
|
182
|
+
id = code_pubid
|
|
183
|
+
return "T" if id.is_a? ::Pubid::Itu::Identifiers::SpecialPublication
|
|
184
|
+
|
|
185
|
+
while id
|
|
186
|
+
return id.sector.sector if id.sector
|
|
187
|
+
|
|
188
|
+
id = wrapped(id)
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# The document a wrapper identifier belongs to, or nil.
|
|
193
|
+
def wrapped(id)
|
|
194
|
+
base = id.base if id.respond_to?(:base)
|
|
195
|
+
base if base.is_a?(::Pubid::Identifier)
|
|
157
196
|
end
|
|
158
197
|
|
|
159
198
|
# @return [Relaton::Bib::Contributor, nil]
|
|
160
199
|
def fetch_editorial_contributor
|
|
161
200
|
wg_name = parser.fetch_workgroup
|
|
162
|
-
bureau = hit.hit[:code]&.match(/(?<=-)\w/)&.to_s
|
|
163
201
|
return unless bureau
|
|
164
202
|
|
|
165
203
|
org = Relaton::Bib::Organization.new(
|
|
166
204
|
name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
|
|
167
|
-
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
205
|
+
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
206
|
+
content: "ITU-#{bureau}",
|
|
207
|
+
),
|
|
168
208
|
subdivision: group_subdivision(wg_name),
|
|
169
209
|
)
|
|
170
210
|
role = Relaton::Bib::Contributor::Role.new(
|
|
@@ -193,18 +233,7 @@ module Relaton
|
|
|
193
233
|
|
|
194
234
|
# @return [Array<Relaton::Bib::Copyright>]
|
|
195
235
|
def fetch_copyright
|
|
196
|
-
|
|
197
|
-
case abbreviation
|
|
198
|
-
when "ITU"
|
|
199
|
-
name = "International Telecommunication Union"
|
|
200
|
-
url = "www.itu.int"
|
|
201
|
-
end
|
|
202
|
-
org = Relaton::Bib::Organization.new(
|
|
203
|
-
name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
|
|
204
|
-
abbreviation: Relaton::Bib::LocalizedString.new(content: abbreviation),
|
|
205
|
-
uri: [Relaton::Bib::Uri.new(content: url)],
|
|
206
|
-
)
|
|
207
|
-
owner = Relaton::Bib::ContributionInfo.new(organization: org)
|
|
236
|
+
owner = Relaton::Bib::ContributionInfo.new(organization: publisher_org)
|
|
208
237
|
year = parser.doc_date&.match(/\d{4}/)&.to_s
|
|
209
238
|
[Relaton::Bib::Copyright.new(from: year, owner: [owner])]
|
|
210
239
|
end
|