relaton 3.1.2 → 3.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +13 -3
- data/lib/relaton/bib/converter/csl/from_csl.rb +195 -0
- data/lib/relaton/bib/converter/csl.rb +13 -1
- data/lib/relaton/bibcollection.rb +4 -1
- data/lib/relaton/cli/collection_store.rb +171 -0
- data/lib/relaton/cli/command.rb +30 -1
- data/lib/relaton/cli/subcommand_collection.rb +61 -64
- data/lib/relaton/cli/util.rb +6 -0
- data/lib/relaton/core/request_error.rb +18 -0
- data/lib/relaton/db/api_resolver.rb +115 -0
- data/lib/relaton/db/cache.rb +59 -13
- data/lib/relaton/db/cache_entry.rb +5 -1
- data/lib/relaton/db/config.rb +56 -3
- data/lib/relaton/db/workers_pool.rb +11 -1
- data/lib/relaton/db.rb +224 -65
- data/lib/relaton/etsi/data_fetcher.rb +15 -5
- data/lib/relaton/itu/hit_collection.rb +4 -3
- data/lib/relaton/itu/scraper.rb +55 -26
- data/lib/relaton/jis/README.adoc +2 -0
- data/lib/relaton/jis/data_fetcher.rb +74 -13
- data/lib/relaton/jis/hit_collection.rb +2 -1
- data/lib/relaton/jis/processor.rb +1 -1
- data/lib/relaton/jis/scraper.rb +46 -5
- data/lib/relaton/plateau/data_fetcher.rb +10 -1
- data/lib/relaton/plateau/handbook_parser.rb +8 -1
- data/lib/relaton/plateau/hit_collection.rb +19 -4
- data/lib/relaton/version.rb +1 -1
- metadata +7 -4
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
require "pubid"
|
|
2
|
+
require_relative "util"
|
|
2
3
|
require_relative "full_text_search"
|
|
4
|
+
require_relative "collection_store"
|
|
3
5
|
|
|
4
6
|
module Relaton
|
|
5
7
|
module Cli
|
|
@@ -15,15 +17,10 @@ module Relaton
|
|
|
15
17
|
option :doctype, desc: "Documents type"
|
|
16
18
|
|
|
17
19
|
def create(file)
|
|
18
|
-
dir = directory
|
|
19
|
-
file_path = File.join dir, file
|
|
20
20
|
col = Relaton::Bibcollection.new options
|
|
21
|
-
if
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
FileUtils.mkdir_p dir # unless Dir.exist? dir
|
|
25
|
-
File.write file_path, col.to_yaml, encoding: "UTF-8"
|
|
26
|
-
end
|
|
21
|
+
return if store.create file, col
|
|
22
|
+
|
|
23
|
+
Util.warn "Collection `#{file}` aready exist"
|
|
27
24
|
end
|
|
28
25
|
|
|
29
26
|
desc "info COLLECTION", "View collection information"
|
|
@@ -31,11 +28,12 @@ module Relaton
|
|
|
31
28
|
"Default is $HOME/.relaton/collections."
|
|
32
29
|
|
|
33
30
|
def info(file) # rubocop:disable Metrics/AbcSize
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
puts "
|
|
38
|
-
|
|
31
|
+
col = store.get file
|
|
32
|
+
return Util.warn "Collection `#{file}` does not exist" unless col
|
|
33
|
+
|
|
34
|
+
puts "Collection: #{file}"
|
|
35
|
+
puts "Last updated: #{store.updated(file)&.localtime}"
|
|
36
|
+
puts "File size: #{store.size file}"
|
|
39
37
|
puts "Number of items: #{col.items.size}"
|
|
40
38
|
puts "Author: #{col.author}"
|
|
41
39
|
puts "Title: #{col.title}"
|
|
@@ -47,12 +45,9 @@ module Relaton
|
|
|
47
45
|
option :entries, aliases: :e, type: :boolean, desc: "Show entries"
|
|
48
46
|
|
|
49
47
|
def list
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
puts File.basename f
|
|
54
|
-
puts_entries yml
|
|
55
|
-
end
|
|
48
|
+
store.names.each do |name|
|
|
49
|
+
puts name
|
|
50
|
+
puts_entries store.get(name)
|
|
56
51
|
end
|
|
57
52
|
end
|
|
58
53
|
|
|
@@ -90,7 +85,7 @@ module Relaton
|
|
|
90
85
|
searcher = Relaton::FullTextSeatch.new(col[:collection])
|
|
91
86
|
searcher.search text
|
|
92
87
|
if searcher.any?
|
|
93
|
-
puts "Collection: #{
|
|
88
|
+
puts "Collection: #{col[:file]}"
|
|
94
89
|
searcher.print_results
|
|
95
90
|
end
|
|
96
91
|
end
|
|
@@ -113,8 +108,10 @@ module Relaton
|
|
|
113
108
|
desc: "Collection to store a document"
|
|
114
109
|
option :dir, aliases: :d, desc: "Directory with collections. Default is " \
|
|
115
110
|
"$HOME/.relaton/collections."
|
|
111
|
+
option :api, type: :boolean, desc: Relaton::Cli::API_OPTION_DESC
|
|
116
112
|
|
|
117
113
|
def fetch(code) # rubocop:disable Metrics/AbcSize
|
|
114
|
+
api_was = apply_api_option options
|
|
118
115
|
opts = {}
|
|
119
116
|
if options[:"publication-date-before"]
|
|
120
117
|
opts[:publication_date_before] = parse_date_option(options[:"publication-date-before"], "--publication-date-before")
|
|
@@ -124,17 +121,19 @@ module Relaton
|
|
|
124
121
|
end
|
|
125
122
|
validate_date_range opts[:publication_date_after], opts[:publication_date_before]
|
|
126
123
|
doc = Relaton.db.fetch(code, options[:year]&.to_s, **opts)
|
|
127
|
-
if doc
|
|
128
|
-
colfile = File.join directory, options[:collection]
|
|
129
|
-
coll = read_collection colfile
|
|
130
|
-
coll << doc
|
|
131
|
-
File.write colfile, coll.to_yaml, encoding: "UTF-8"
|
|
124
|
+
if doc then add_to_collection doc
|
|
132
125
|
else Util.info "No matching bibliographic entry found"
|
|
133
126
|
end
|
|
134
|
-
rescue Relaton::UnknownReferenceError
|
|
127
|
+
rescue Relaton::UnknownReferenceError, Relaton::NotInLocalStoreError,
|
|
128
|
+
Relaton::RequestError => e
|
|
129
|
+
# An unknown reference, offline mode with no stored entry
|
|
130
|
+
# (relaton#536) or a transport failure: log the reason, and do not
|
|
131
|
+
# write the collection.
|
|
135
132
|
Util.warn e.message
|
|
136
133
|
rescue Pubid::Errors::Error
|
|
137
134
|
Util.error %("#{code}" is not a recognized standards identifier)
|
|
135
|
+
ensure
|
|
136
|
+
restore_api_option api_was
|
|
138
137
|
end
|
|
139
138
|
|
|
140
139
|
desc "import FILE", "Import document or collection from an XML file " \
|
|
@@ -145,23 +144,19 @@ module Relaton
|
|
|
145
144
|
option :dir, aliases: :d, desc: "Directory with collections. Default is " \
|
|
146
145
|
"$HOME/.relaton/collections."
|
|
147
146
|
|
|
148
|
-
def import(file)
|
|
149
|
-
|
|
150
|
-
coll = read_collection collfile
|
|
151
|
-
xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
|
|
152
|
-
xml.remove_namespaces!
|
|
147
|
+
def import(file)
|
|
148
|
+
xml = read_xml file
|
|
153
149
|
if xml.at "relaton-collection"
|
|
154
150
|
imported = import_collection(xml)
|
|
155
|
-
|
|
151
|
+
store.update(options[:collection]) do |coll|
|
|
152
|
+
next imported unless coll # a new collection is the imported one
|
|
153
|
+
|
|
156
154
|
coll << imported
|
|
157
|
-
|
|
158
|
-
coll = imported
|
|
155
|
+
coll
|
|
159
156
|
end
|
|
160
157
|
else
|
|
161
|
-
|
|
162
|
-
coll << Relaton::Bibdata.from_xml(xml)
|
|
158
|
+
add_to_collection Relaton::Bibdata.from_xml(xml)
|
|
163
159
|
end
|
|
164
|
-
File.write collfile, coll.to_yaml, encoding: "UTF-8"
|
|
165
160
|
end
|
|
166
161
|
|
|
167
162
|
desc "export COLLECTION", "Export collection into XML file"
|
|
@@ -169,7 +164,9 @@ module Relaton
|
|
|
169
164
|
"$HOME/.relaton/collections."
|
|
170
165
|
|
|
171
166
|
def export(file)
|
|
172
|
-
coll =
|
|
167
|
+
coll = store.get file
|
|
168
|
+
return Util.warn "Collection `#{file}` does not exist" unless coll
|
|
169
|
+
|
|
173
170
|
outfile = "#{file.sub(/\.\w+$/, '')}.xml"
|
|
174
171
|
File.write outfile, coll.to_xml(bibdata: true), encoding: "UTF-8"
|
|
175
172
|
end
|
|
@@ -197,44 +194,44 @@ module Relaton
|
|
|
197
194
|
options.fetch :dir, File.join(Dir.home, ".relaton/collections")
|
|
198
195
|
end
|
|
199
196
|
|
|
200
|
-
#
|
|
201
|
-
# @return [
|
|
202
|
-
def
|
|
203
|
-
|
|
204
|
-
rescue Psych::SyntaxError
|
|
205
|
-
Util.error "the file `#{file}` isn't a collection."
|
|
206
|
-
nil
|
|
197
|
+
# The collections, in a lutaml-store store under `<dir>/v2` (#549).
|
|
198
|
+
# @return [Relaton::Cli::CollectionStore]
|
|
199
|
+
def store
|
|
200
|
+
@store ||= CollectionStore.new directory
|
|
207
201
|
end
|
|
208
202
|
|
|
209
|
-
#
|
|
210
|
-
#
|
|
211
|
-
def
|
|
212
|
-
|
|
203
|
+
# Adds an item to the `--collection` in one atomic read-modify-write,
|
|
204
|
+
# and creates the collection when it is missing.
|
|
205
|
+
def add_to_collection(item)
|
|
206
|
+
store.update(options[:collection]) do |coll|
|
|
207
|
+
coll ||= Relaton::Bibcollection.new({})
|
|
208
|
+
coll << item
|
|
209
|
+
coll
|
|
210
|
+
end
|
|
211
|
+
end
|
|
213
212
|
|
|
214
|
-
|
|
213
|
+
# The XML file `import` reads: user input, not a collection.
|
|
214
|
+
# @return [Nokogiri::XML::Document]
|
|
215
|
+
def read_xml(file)
|
|
216
|
+
xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
|
|
217
|
+
xml.remove_namespaces!
|
|
218
|
+
xml
|
|
215
219
|
end
|
|
216
220
|
|
|
217
221
|
# @return [Array<Hash>]
|
|
218
222
|
def collections
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
if yml && yml["root"]
|
|
223
|
-
m << { collection: Relaton::Bibcollection.new(yml["root"]),
|
|
224
|
-
file: f }
|
|
225
|
-
end
|
|
226
|
-
m
|
|
223
|
+
store.names(options.fetch(:collection, "*")).filter_map do |name|
|
|
224
|
+
coll = store.get name
|
|
225
|
+
{ collection: coll, file: name } if coll
|
|
227
226
|
end
|
|
228
227
|
end
|
|
229
228
|
|
|
230
|
-
# Puts document IDs for each item in
|
|
231
|
-
# @param
|
|
232
|
-
def puts_entries(
|
|
229
|
+
# Puts document IDs for each item in the collection
|
|
230
|
+
# @param coll [Relaton::Bibcollection]
|
|
231
|
+
def puts_entries(coll)
|
|
233
232
|
return unless options[:entries]
|
|
234
233
|
|
|
235
|
-
|
|
236
|
-
puts " #{b.docidentifier}"
|
|
237
|
-
end
|
|
234
|
+
coll.items.each { |b| puts " #{b.docidentifier}" }
|
|
238
235
|
end
|
|
239
236
|
|
|
240
237
|
# @param item [Relaton::Bibdata]
|
data/lib/relaton/cli/util.rb
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
module Relaton
|
|
2
2
|
module Cli
|
|
3
|
+
# The `--api` option of `fetch` and `collection fetch` (relaton#535).
|
|
4
|
+
# Here, not in command.rb: a command's class body reads it.
|
|
5
|
+
API_OPTION_DESC = "Ask api.relaton.org before the flavor's own " \
|
|
6
|
+
"source; --no-api does not (default: RELATON_USE_API, " \
|
|
7
|
+
"else off)".freeze
|
|
8
|
+
|
|
3
9
|
module Util
|
|
4
10
|
extend Relaton::Bib::Util
|
|
5
11
|
PROGNAME = "relaton-cli".freeze
|
|
@@ -17,6 +17,24 @@ module Relaton
|
|
|
17
17
|
# declare only when absent — both forms rescue as StandardError
|
|
18
18
|
class RequestError < Error; end unless defined?(Relaton::RequestError)
|
|
19
19
|
|
|
20
|
+
# Offline mode (`Relaton::Db.configuration.offline`, RELATON_OFFLINE) and
|
|
21
|
+
# the local store has no entry for the reference (relaton#536). Not a
|
|
22
|
+
# RequestError: `Relaton::Db#net_retry` must not retry it, since no
|
|
23
|
+
# request was made.
|
|
24
|
+
class NotInLocalStoreError < Error
|
|
25
|
+
# @return [String] the reference the local store lacks
|
|
26
|
+
attr_reader :reference
|
|
27
|
+
|
|
28
|
+
# @param reference [String]
|
|
29
|
+
# @param dirs [Array<String>] the cache directories that were read
|
|
30
|
+
def initialize(reference, dirs = [])
|
|
31
|
+
@reference = reference
|
|
32
|
+
where = dirs.empty? ? "no cache directory" : dirs.join(", ")
|
|
33
|
+
super("`#{reference}` is not in the local store (#{where}); " \
|
|
34
|
+
"offline mode makes no network request")
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
20
38
|
# `Relaton::Db` found no flavor for a reference: no flavor's pubid grammar
|
|
21
39
|
# reads it exactly, and no flavor's prefix matches it. relaton-cli rescues
|
|
22
40
|
# it, logs the message and returns no document.
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
require "net/http"
|
|
2
|
+
require "uri"
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "zlib"
|
|
5
|
+
require_relative "../version"
|
|
6
|
+
|
|
7
|
+
module Relaton
|
|
8
|
+
class Db
|
|
9
|
+
#
|
|
10
|
+
# The api.relaton.org tier of a `Db` fetch (relaton#535): local cache,
|
|
11
|
+
# then this, then the flavor's own `get`.
|
|
12
|
+
#
|
|
13
|
+
# It reads `GET <api_host>/v3/documents/<key>` as Relaton YAML, where the
|
|
14
|
+
# key is one percent-encoded path segment. The API is not authoritative
|
|
15
|
+
# while its coverage is verified per flavor, so every answer it cannot
|
|
16
|
+
# give with certainty is nil, and `Db#fetch_doc` then asks the flavor:
|
|
17
|
+
# a 404, any other status, a transport failure, and a body the flavor
|
|
18
|
+
# cannot read. Nothing here raises, and nothing here touches the cache:
|
|
19
|
+
# `Db` stores the answer as it stores a flavor's, with its ETag, and
|
|
20
|
+
# revalidates a stale row with that ETag.
|
|
21
|
+
#
|
|
22
|
+
module ApiResolver
|
|
23
|
+
ACCEPT = "application/yaml".freeze
|
|
24
|
+
# The transport failures that mean "ask the flavor" (the list of
|
|
25
|
+
# `Index::ShardSource::NET_ERRORS`; not required from there, so that
|
|
26
|
+
# `require "relaton/db"` opens no partial `Relaton::Index`).
|
|
27
|
+
NET_ERRORS = [
|
|
28
|
+
SocketError, Timeout::Error, IOError, SystemCallError,
|
|
29
|
+
OpenSSL::SSL::SSLError, Net::HTTPBadResponse,
|
|
30
|
+
Net::HTTPHeaderSyntaxError, Net::ProtocolError, Zlib::Error
|
|
31
|
+
].freeze
|
|
32
|
+
|
|
33
|
+
#
|
|
34
|
+
# What the API answered. `item` is the document of a 200, with its
|
|
35
|
+
# `etag`; `not_modified` is a 304 to a revalidation, whose `etag` is
|
|
36
|
+
# the one sent (the cached document is still the API's).
|
|
37
|
+
#
|
|
38
|
+
Answer = Data.define(:item, :etag, :not_modified) do
|
|
39
|
+
def initialize(item: nil, etag: nil, not_modified: false)
|
|
40
|
+
super
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
module_function
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
# @param processor [Relaton::Core::Processor] the routed flavor
|
|
48
|
+
# @param key [Pubid::Identifier] the query's cache key; the API is
|
|
49
|
+
# asked for its rendering
|
|
50
|
+
# @param etag [String, nil] the ETag of a stale cached answer: sent as
|
|
51
|
+
# `If-None-Match`, so an unchanged document answers 304 with no body.
|
|
52
|
+
# The ETag is of the YAML representation, and the cache holds XML,
|
|
53
|
+
# so it is the header value as received, never computed here.
|
|
54
|
+
# @return [Answer, nil] nil: ask the flavor
|
|
55
|
+
#
|
|
56
|
+
def fetch(processor, key, etag: nil)
|
|
57
|
+
rsp = get(key.to_s, etag) or return
|
|
58
|
+
return Answer.new(etag: etag, not_modified: true) if rsp.code == "304"
|
|
59
|
+
|
|
60
|
+
item = read(processor, rsp.body, key) or return
|
|
61
|
+
Answer.new(item: item, etag: rsp["ETag"])
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @return [Net::HTTPResponse, nil] a 200, or a 304 to a revalidation;
|
|
65
|
+
# else nil
|
|
66
|
+
def get(ref, etag = nil)
|
|
67
|
+
host = Db.configuration.api_host.chomp("/")
|
|
68
|
+
uri = URI("#{host}/v3/documents/#{escape(ref)}")
|
|
69
|
+
rsp = request(uri, etag)
|
|
70
|
+
return rsp if rsp.code == "200" || (etag && rsp.code == "304")
|
|
71
|
+
|
|
72
|
+
info "HTTP #{rsp.code}", ref
|
|
73
|
+
nil
|
|
74
|
+
rescue *NET_ERRORS => e
|
|
75
|
+
info "#{e.class}: #{e.message}", ref
|
|
76
|
+
nil
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def request(uri, etag = nil)
|
|
80
|
+
timeout = Db.configuration.api_timeout
|
|
81
|
+
headers = { "Accept" => ACCEPT,
|
|
82
|
+
"User-Agent" => "relaton/#{Relaton::VERSION}" }
|
|
83
|
+
headers["If-None-Match"] = etag if etag
|
|
84
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
|
|
85
|
+
open_timeout: timeout,
|
|
86
|
+
read_timeout: timeout) do |http|
|
|
87
|
+
http.get(uri.request_uri, headers)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The API's body is data: a body the flavor's YAML model rejects, or
|
|
92
|
+
# one that is not a document (`--- []` reads as an Array), is a miss,
|
|
93
|
+
# whatever the model raises for it.
|
|
94
|
+
def read(processor, body, ref)
|
|
95
|
+
item = processor.from_yaml body
|
|
96
|
+
return item if item.is_a?(Relaton::Bib::ItemData)
|
|
97
|
+
|
|
98
|
+
info "answer is not a document (#{item.class})", ref
|
|
99
|
+
nil
|
|
100
|
+
rescue StandardError => e
|
|
101
|
+
info "unreadable answer (#{e.class}: #{e.message})", ref
|
|
102
|
+
nil
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# One path segment: `/` and `:` are escaped, a space is `%20`.
|
|
106
|
+
def escape(ref)
|
|
107
|
+
URI.encode_www_form_component(ref).gsub("+", "%20")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def info(reason, ref)
|
|
111
|
+
Util.info "api.relaton.org: #{reason}; asking the flavor", key: ref
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
data/lib/relaton/db/cache.rb
CHANGED
|
@@ -25,6 +25,13 @@ module Relaton
|
|
|
25
25
|
class Cache
|
|
26
26
|
LAYOUT = "v2".freeze
|
|
27
27
|
VERSIONS_KEY = "_versions".freeze
|
|
28
|
+
# The rows key that records the one-shot v1 import (relaton#240). It
|
|
29
|
+
# lives in the store, not in a marker file beside it (relaton#548).
|
|
30
|
+
V1_IMPORTED_KEY = "_v1_imported".freeze
|
|
31
|
+
# The legacy marker file of that import, read but no longer written.
|
|
32
|
+
V1_IMPORTED_FILE = ".v1-imported".freeze
|
|
33
|
+
# Rows keys that hold no bucket.
|
|
34
|
+
RESERVED_KEYS = [VERSIONS_KEY, V1_IMPORTED_KEY].freeze
|
|
28
35
|
STRING_FLAVOR = "_key".freeze
|
|
29
36
|
PUBID_FLAVOR = "_pubid".freeze
|
|
30
37
|
# The legacy string form of a not-found entry (`not_found <date>`).
|
|
@@ -75,10 +82,15 @@ module Relaton
|
|
|
75
82
|
@dir
|
|
76
83
|
end
|
|
77
84
|
|
|
78
|
-
# Remove every row and document
|
|
85
|
+
# Remove every row and document, under one rows transaction (lock
|
|
86
|
+
# order rows, then docs). The v1 import marker is never deleted, not
|
|
87
|
+
# even for a moment, so a cleared cache is not filled again from
|
|
88
|
+
# `<dir>-v1.bak` by this or another process.
|
|
79
89
|
def clear
|
|
80
|
-
@rows.
|
|
81
|
-
|
|
90
|
+
@rows.adapter.transaction do
|
|
91
|
+
@rows.each_key { |k| @rows.delete k unless k == V1_IMPORTED_KEY }
|
|
92
|
+
@docs.clear
|
|
93
|
+
end
|
|
82
94
|
@versions = {}
|
|
83
95
|
end
|
|
84
96
|
|
|
@@ -118,9 +130,11 @@ module Relaton
|
|
|
118
130
|
# @param value [String, Relaton::Db::NotFound, nil] document XML, a
|
|
119
131
|
# not-found entry (also as `not_found <date>`); nil deletes the row
|
|
120
132
|
# @param item_key [Pubid::Identifier, String, nil] the document's key
|
|
133
|
+
# @param etag [String, nil] the ETag of the API answer the document
|
|
134
|
+
# came from (relaton#535), kept on both rows; nil for a flavor answer
|
|
121
135
|
# @return [String, Relaton::Db::NotFound, nil] value
|
|
122
136
|
#
|
|
123
|
-
def store(key, value, item_key: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
137
|
+
def store(key, value, item_key: nil, etag: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
124
138
|
return delete(key) if value.nil?
|
|
125
139
|
|
|
126
140
|
bucket, key = resolve key
|
|
@@ -137,9 +151,10 @@ module Relaton
|
|
|
137
151
|
record_version item_bucket
|
|
138
152
|
@rows.adapter.transaction do
|
|
139
153
|
@docs.set file, value
|
|
140
|
-
upsert item_bucket,
|
|
154
|
+
upsert item_bucket,
|
|
155
|
+
new_row(item_key, CacheEntry::DOC, file, fetched, etag)
|
|
141
156
|
unless canonical(key) == canonical(item_key)
|
|
142
|
-
upsert bucket, new_row(key, CacheEntry::DOC, file, fetched)
|
|
157
|
+
upsert bucket, new_row(key, CacheEntry::DOC, file, fetched, etag)
|
|
143
158
|
end
|
|
144
159
|
end
|
|
145
160
|
value
|
|
@@ -183,7 +198,7 @@ module Relaton
|
|
|
183
198
|
|
|
184
199
|
found = other.lookup(key) or return
|
|
185
200
|
row_id, row = found
|
|
186
|
-
store row_id, other.read_entry(row)
|
|
201
|
+
store row_id, other.read_entry(row), etag: row["etag"]
|
|
187
202
|
end
|
|
188
203
|
|
|
189
204
|
# @param key [Pubid::Identifier, String]
|
|
@@ -192,6 +207,30 @@ module Relaton
|
|
|
192
207
|
lookup(key)&.last&.dig("fetched")
|
|
193
208
|
end
|
|
194
209
|
|
|
210
|
+
# @param key [Pubid::Identifier, String]
|
|
211
|
+
# @return [String, nil] the ETag of the API answer the entry came from
|
|
212
|
+
def etag(key)
|
|
213
|
+
lookup(key)&.last&.dig("etag")
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
#
|
|
217
|
+
# The document and the ETag to revalidate a key with, read from one
|
|
218
|
+
# row in one lookup, so the two belong to one answer. Only the key's
|
|
219
|
+
# own row counts: a dated key's lookup can also read a row that adds a
|
|
220
|
+
# language (`#lookup`), and that row's ETag was issued for another URL.
|
|
221
|
+
#
|
|
222
|
+
# @param key [Pubid::Identifier, String]
|
|
223
|
+
# @return [Array(String, String), nil] the document XML and its ETag
|
|
224
|
+
#
|
|
225
|
+
def revalidation(key)
|
|
226
|
+
row = lookup(key)&.last
|
|
227
|
+
own = canonical resolve(key).last
|
|
228
|
+
return unless row && row["etag"] && row_key(row) == own
|
|
229
|
+
|
|
230
|
+
xml = read_entry row
|
|
231
|
+
[xml, row["etag"]] if xml.is_a?(String)
|
|
232
|
+
end
|
|
233
|
+
|
|
195
234
|
# An undated entry expires after 60 days, a dated document never. A
|
|
196
235
|
# not_found entry expires after 60 days even for a dated query: the
|
|
197
236
|
# document can be published later.
|
|
@@ -319,8 +358,7 @@ module Relaton
|
|
|
319
358
|
# fetch, and redirects were dropped from the v2 model).
|
|
320
359
|
def import_old_layout
|
|
321
360
|
bak = "#{dir}-v1.bak"
|
|
322
|
-
|
|
323
|
-
return if File.exist?(marker) || !Dir.exist?(bak)
|
|
361
|
+
return if v1_imported? || !Dir.exist?(bak)
|
|
324
362
|
|
|
325
363
|
imported = 0
|
|
326
364
|
Dir.glob("#{bak}/**/*.xml").each do |f|
|
|
@@ -331,10 +369,17 @@ module Relaton
|
|
|
331
369
|
self[key] = value
|
|
332
370
|
imported += 1
|
|
333
371
|
end
|
|
334
|
-
|
|
372
|
+
@rows.set V1_IMPORTED_KEY, Date.today.to_s
|
|
335
373
|
Util.info "cache #{dir}: imported #{imported} entries from #{bak}"
|
|
336
374
|
end
|
|
337
375
|
|
|
376
|
+
# An older relaton wrote the marker as a file; it still counts. `get`,
|
|
377
|
+
# not `exists?`: it takes the shared lock, so it waits for a write.
|
|
378
|
+
def v1_imported?
|
|
379
|
+
!@rows.get(V1_IMPORTED_KEY).nil? ||
|
|
380
|
+
File.exist?(File.join(dir, LAYOUT, V1_IMPORTED_FILE))
|
|
381
|
+
end
|
|
382
|
+
|
|
338
383
|
# The wrapped string key of an old-layout document: its own primary
|
|
339
384
|
# docidentifier under its flavor prefix, resolved through the same
|
|
340
385
|
# `wrapped_key` path a string cache read takes. A value no parser reads
|
|
@@ -433,7 +478,7 @@ module Relaton
|
|
|
433
478
|
|
|
434
479
|
def each_row
|
|
435
480
|
@rows.each_key do |bucket|
|
|
436
|
-
next if bucket
|
|
481
|
+
next if RESERVED_KEYS.include? bucket
|
|
437
482
|
|
|
438
483
|
read_bucket(bucket).each { |row| yield bucket, row }
|
|
439
484
|
end
|
|
@@ -510,8 +555,9 @@ module Relaton
|
|
|
510
555
|
end
|
|
511
556
|
end
|
|
512
557
|
|
|
513
|
-
def new_row(key, status, file, fetched)
|
|
514
|
-
row = CacheEntry.new(status: status, file: file, fetched: fetched
|
|
558
|
+
def new_row(key, status, file, fetched, etag = nil)
|
|
559
|
+
row = CacheEntry.new(status: status, file: file, fetched: fetched,
|
|
560
|
+
etag: etag)
|
|
515
561
|
if key.is_a?(String) then row.key = key
|
|
516
562
|
else row.id = canonical(key)
|
|
517
563
|
end
|
|
@@ -15,7 +15,9 @@ module Relaton
|
|
|
15
15
|
# `to_hash`) or, for a processor with no pubid class, by a string (`key`).
|
|
16
16
|
# `file` names the document in the doc store; several rows can share one
|
|
17
17
|
# file (a query row and the row of the document it returned). A
|
|
18
|
-
# `not_found` row has no file.
|
|
18
|
+
# `not_found` row has no file. `etag` is the ETag of the api.relaton.org
|
|
19
|
+
# answer the document came from (relaton#535), for revalidation; a
|
|
20
|
+
# flavor answer, and a row an older relaton wrote, have none.
|
|
19
21
|
class CacheEntry < Lutaml::Model::Serializable
|
|
20
22
|
DOC = "doc".freeze
|
|
21
23
|
NOT_FOUND = "not_found".freeze
|
|
@@ -25,6 +27,7 @@ module Relaton
|
|
|
25
27
|
attribute :status, :string
|
|
26
28
|
attribute :file, :string
|
|
27
29
|
attribute :fetched, :string
|
|
30
|
+
attribute :etag, :string
|
|
28
31
|
|
|
29
32
|
key_value do
|
|
30
33
|
map "id", to: :id
|
|
@@ -32,6 +35,7 @@ module Relaton
|
|
|
32
35
|
map "status", to: :status
|
|
33
36
|
map "file", to: :file
|
|
34
37
|
map "fetched", to: :fetched
|
|
38
|
+
map "etag", to: :etag
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
def not_found?
|
data/lib/relaton/db/config.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
1
3
|
module Relaton
|
|
2
4
|
class Db
|
|
3
5
|
module Config
|
|
@@ -11,11 +13,62 @@ module Relaton
|
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
class Configuration
|
|
14
|
-
|
|
16
|
+
# Values of a `RELATON_*` switch (any case) that turn it on.
|
|
17
|
+
SWITCH_ON = %w[1 true yes on].freeze
|
|
18
|
+
# Values of RELATON_OFFLINE that turn offline mode on.
|
|
19
|
+
OFFLINE_ON = SWITCH_ON
|
|
20
|
+
DEFAULT_API_HOST = "https://api.relaton.org".freeze
|
|
21
|
+
|
|
22
|
+
# @!attribute offline
|
|
23
|
+
# @return [Boolean] true: every fetch is answered from the local
|
|
24
|
+
# store only, and a miss raises Relaton::NotInLocalStoreError
|
|
25
|
+
# (relaton#536). Defaults to RELATON_OFFLINE.
|
|
26
|
+
#
|
|
27
|
+
# @!attribute use_api
|
|
28
|
+
# @return [Boolean] true: a cache miss asks api.relaton.org first,
|
|
29
|
+
# and the flavor answers what the API does not (relaton#535).
|
|
30
|
+
# Defaults to RELATON_USE_API, else false. The CLI's `--api` /
|
|
31
|
+
# `--no-api` sets it for one run.
|
|
32
|
+
# @!attribute api_host
|
|
33
|
+
# @return [String] the API's base URL. Defaults to RELATON_API_HOST,
|
|
34
|
+
# else https://api.relaton.org.
|
|
35
|
+
# @!attribute api_timeout
|
|
36
|
+
# @return [Numeric] open and read timeout of an API request, seconds
|
|
37
|
+
attr_accessor :use_api, :api_host, :api_timeout, :offline
|
|
15
38
|
|
|
16
39
|
def initialize
|
|
17
|
-
@use_api =
|
|
18
|
-
@api_host =
|
|
40
|
+
@use_api = switch? "RELATON_USE_API"
|
|
41
|
+
@api_host = env_api_host || DEFAULT_API_HOST
|
|
42
|
+
@api_timeout = 10
|
|
43
|
+
@offline = switch? "RELATON_OFFLINE"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def switch?(name)
|
|
49
|
+
SWITCH_ON.include? ENV[name].to_s.downcase
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# RELATON_API_HOST, when it is an http(s) URL with a host. A shell
|
|
53
|
+
# value is easy to write without its scheme (`api.example.org`), and
|
|
54
|
+
# the resolver would then ask a host-less URI; that is warned about,
|
|
55
|
+
# not used.
|
|
56
|
+
def env_api_host
|
|
57
|
+
host = ENV["RELATON_API_HOST"].to_s.strip
|
|
58
|
+
return if host.empty?
|
|
59
|
+
|
|
60
|
+
uri = URI.parse(host)
|
|
61
|
+
return host if uri.is_a?(URI::HTTP) && uri.host && !uri.host.empty?
|
|
62
|
+
|
|
63
|
+
invalid_api_host host
|
|
64
|
+
rescue URI::InvalidURIError
|
|
65
|
+
invalid_api_host host
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def invalid_api_host(host)
|
|
69
|
+
Util.warn "RELATON_API_HOST #{host.inspect} is not an http(s) URL; " \
|
|
70
|
+
"using #{DEFAULT_API_HOST}"
|
|
71
|
+
nil
|
|
19
72
|
end
|
|
20
73
|
end
|
|
21
74
|
|
|
@@ -8,7 +8,17 @@ module Relaton
|
|
|
8
8
|
@queue = SizedQueue.new(workers * 2)
|
|
9
9
|
@threads = Array.new workers do
|
|
10
10
|
Thread.new do
|
|
11
|
-
|
|
11
|
+
# A worker that dies silently wedges the pool: the item it was
|
|
12
|
+
# processing never completes, and every consumer blocked on that
|
|
13
|
+
# completion blocks forever (metanorma-standoc CI deadlock,
|
|
14
|
+
# 2026-11-13). The pool outlives any single item's failure.
|
|
15
|
+
while item = @queue.pop
|
|
16
|
+
begin
|
|
17
|
+
yield(item)
|
|
18
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
19
|
+
Util.error "fetch worker item failed: #{e.class}: #{e.message}"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
12
22
|
end
|
|
13
23
|
end
|
|
14
24
|
end
|