relaton 3.1.3 → 3.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +13 -3
- data/lib/relaton/bibcollection.rb +4 -1
- data/lib/relaton/cli/collection_store.rb +171 -0
- data/lib/relaton/cli/command.rb +27 -0
- data/lib/relaton/cli/subcommand_collection.rb +56 -63
- data/lib/relaton/cli/util.rb +6 -0
- data/lib/relaton/db/api_resolver.rb +115 -0
- data/lib/relaton/db/cache.rb +59 -13
- data/lib/relaton/db/cache_entry.rb +5 -1
- data/lib/relaton/db/config.rb +50 -5
- data/lib/relaton/db.rb +97 -23
- data/lib/relaton/itu/hit_collection.rb +4 -3
- data/lib/relaton/itu/scraper.rb +55 -26
- data/lib/relaton/version.rb +1 -1
- metadata +6 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c2dcd5f1b916c7fe552e39b9376228255e2523385c8da7f91e1193b17ffb58d7
|
|
4
|
+
data.tar.gz: e7331e3f2c069cda9567b518411dcb35e866700151987d9b4d4f14e17ef0f8b5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 65781d01bf2cedf282a94e0275ebf7bcc187fe9d92f029e58768acd782fe05e04f0391ddc579bfce8d7d106242927ad54e9b371b58293dde2651f7e940e201ef
|
|
7
|
+
data.tar.gz: 3f1e0879b62d13c363fd9d34bd02ffdc764a63f7ec8197d1614df983da34bcf02db5086679b68aa391b4fb52d91930be59cc2e58e7719c81ae21a2c9d9de3fa3
|
data/README.adoc
CHANGED
|
@@ -66,15 +66,25 @@ e.g. `get("ISO 19115-1", "2014", all_parts: true)` is transformed into a referen
|
|
|
66
66
|
|
|
67
67
|
=== Configuration
|
|
68
68
|
|
|
69
|
-
* `use_api` - `true`
|
|
70
|
-
* `api_host` - URL of
|
|
69
|
+
* `use_api` - `true` to ask api.relaton.org before the flavor's own fetch on a cache miss; the flavor still answers whatever the API does not. The default value is the `RELATON_USE_API` environment variable (`1`, `true`, `yes` or `on` turn it on), else `false`.
|
|
70
|
+
* `api_host` - URL of the API. The default value is the `RELATON_API_HOST` environment variable, else https://api.relaton.org.
|
|
71
|
+
* `api_timeout` - open and read timeout of an API request, in seconds. The default value is `10`.
|
|
72
|
+
* `offline` - `true` to answer every fetch from the local cache only, with no network request. The default value is the `RELATON_OFFLINE` environment variable, else `false`.
|
|
73
|
+
|
|
74
|
+
The `relaton` command line turns the API on for one run with `--api` and off with `--no-api` (on `relaton fetch` and `relaton collection fetch`); with neither flag it follows `RELATON_USE_API`:
|
|
75
|
+
|
|
76
|
+
[source,sh]
|
|
77
|
+
----
|
|
78
|
+
relaton fetch "IETF RFC 8341" --api
|
|
79
|
+
RELATON_USE_API=1 relaton fetch "IETF RFC 8341"
|
|
80
|
+
----
|
|
71
81
|
|
|
72
82
|
[source,ruby]
|
|
73
83
|
----
|
|
74
84
|
require "relaton"
|
|
75
85
|
=> true
|
|
76
86
|
|
|
77
|
-
Relaton.configure do |conf|
|
|
87
|
+
Relaton::Db.configure do |conf|
|
|
78
88
|
conf.use_api = true
|
|
79
89
|
conf.api_host = "https://api.relaton.org"
|
|
80
90
|
end
|
|
@@ -82,7 +82,10 @@ module Relaton
|
|
|
82
82
|
# @return [Relaton::Bibdata, Relaton::Bibcollection]
|
|
83
83
|
def new_bib_item_class(item)
|
|
84
84
|
if item.is_a?(Hash)
|
|
85
|
-
|
|
85
|
+
# `#to_h` writes a nested collection as `{"root" => …}`.
|
|
86
|
+
if item["root"].is_a?(Hash)
|
|
87
|
+
::Relaton::Bibcollection.new(item["root"])
|
|
88
|
+
elsif item["items"]
|
|
86
89
|
::Relaton::Bibcollection.new(item)
|
|
87
90
|
else
|
|
88
91
|
bibitem = ::Relaton::Cli::YAMLConvertor.convert_single_file item
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
require "fileutils"
|
|
2
|
+
require "time"
|
|
3
|
+
require "yaml"
|
|
4
|
+
require "lutaml/store"
|
|
5
|
+
require_relative "../bibcollection"
|
|
6
|
+
require_relative "util"
|
|
7
|
+
|
|
8
|
+
module Relaton
|
|
9
|
+
module Cli
|
|
10
|
+
# The collections of `relaton collection` (relaton#549): one lutaml-store
|
|
11
|
+
# FileSystem store under `<dir>/v2/`, keyed by the collection name the
|
|
12
|
+
# user types. Each value is the collection's YAML (`{"root" => …}`, as
|
|
13
|
+
# `Bibcollection#to_h` makes it) with a top-level `updated` time, so the
|
|
14
|
+
# data file is the same YAML an older relaton wrote.
|
|
15
|
+
#
|
|
16
|
+
# lutaml-store is the only storage layer, as for `Db::Cache` (relaton#548):
|
|
17
|
+
# its lock makes a read-modify-write atomic between threads and processes,
|
|
18
|
+
# and its rename makes a write atomic. Nothing here builds a path from a
|
|
19
|
+
# name or takes a lock of its own. The one raw IO is the one-time move of
|
|
20
|
+
# the flat files an older relaton wrote directly in `<dir>`: each is
|
|
21
|
+
# imported, then moved to `<dir>-v1.bak/`, never deleted.
|
|
22
|
+
class CollectionStore
|
|
23
|
+
LAYOUT = "v2".freeze
|
|
24
|
+
UPDATED = "updated".freeze
|
|
25
|
+
|
|
26
|
+
attr_reader :dir
|
|
27
|
+
|
|
28
|
+
# @param dir [String] the collections directory
|
|
29
|
+
def initialize(dir)
|
|
30
|
+
@dir = dir
|
|
31
|
+
@store = Lutaml::Store::BasicStore.new(
|
|
32
|
+
adapter_type: :filesystem,
|
|
33
|
+
adapter_options: { path: File.join(dir, LAYOUT), extension: ".yaml",
|
|
34
|
+
integrity_checks: false,
|
|
35
|
+
create_directories: false },
|
|
36
|
+
cache: { enabled: false },
|
|
37
|
+
)
|
|
38
|
+
move_flat_files
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @param pattern [String] a glob over the names (`--collection`)
|
|
42
|
+
# @return [Array<String>] the matching names, sorted
|
|
43
|
+
def names(pattern = "*")
|
|
44
|
+
@store.keys.select { |name| File.fnmatch pattern, name }.sort
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# @param name [String]
|
|
48
|
+
def exist?(name)
|
|
49
|
+
@store.exists? name
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @param name [String]
|
|
53
|
+
# @return [Relaton::Bibcollection, nil]
|
|
54
|
+
def get(name)
|
|
55
|
+
hash = read name
|
|
56
|
+
Relaton::Bibcollection.new hash["root"] if hash
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# @param name [String]
|
|
60
|
+
# @return [Time, nil] when the collection was last written
|
|
61
|
+
def updated(name)
|
|
62
|
+
time = read(name)&.dig UPDATED
|
|
63
|
+
Time.iso8601 time if time
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# @param name [String]
|
|
67
|
+
# @return [Integer, nil] the size of the stored collection, in bytes
|
|
68
|
+
def size(name)
|
|
69
|
+
@store.get(name)&.bytesize
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# @param name [String]
|
|
73
|
+
# @param coll [Relaton::Bibcollection]
|
|
74
|
+
# @return [Boolean] false when the store already holds the name
|
|
75
|
+
def create(name, coll)
|
|
76
|
+
@store.adapter.transaction do
|
|
77
|
+
next false if @store.exists? name
|
|
78
|
+
|
|
79
|
+
@store.set name, encode(coll.to_h)
|
|
80
|
+
true
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Atomic read-modify-write under the store's exclusive lock.
|
|
85
|
+
#
|
|
86
|
+
# @param name [String]
|
|
87
|
+
# @yieldparam coll [Relaton::Bibcollection, nil] nil when it is missing
|
|
88
|
+
# @yieldreturn [Relaton::Bibcollection]
|
|
89
|
+
def update(name)
|
|
90
|
+
@store.update(name) do |value|
|
|
91
|
+
hash = decode value if value
|
|
92
|
+
coll = hash && Relaton::Bibcollection.new(hash["root"])
|
|
93
|
+
encode yield(coll).to_h
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
private
|
|
98
|
+
|
|
99
|
+
# @return [Hash, nil]
|
|
100
|
+
def read(name)
|
|
101
|
+
value = @store.get name
|
|
102
|
+
decode value if value
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def decode(value)
|
|
106
|
+
YAML.safe_load value, permitted_classes: [Date, Time, Symbol],
|
|
107
|
+
aliases: true
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def encode(hash, updated = Time.now)
|
|
111
|
+
hash.merge(UPDATED => updated.utc.iso8601).to_yaml
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# A flat collection file is imported unless the store already holds its
|
|
115
|
+
# name, and then moved to the backup either way, under the store's lock
|
|
116
|
+
# so two processes do not move one file. The scan runs again inside the
|
|
117
|
+
# lock: another process can have moved the files first.
|
|
118
|
+
def move_flat_files
|
|
119
|
+
return if flat_collections.empty?
|
|
120
|
+
|
|
121
|
+
@store.adapter.transaction do
|
|
122
|
+
flat_collections(warn: false).each do |name, path, hash|
|
|
123
|
+
if @store.exists? name
|
|
124
|
+
Util.warn "collections #{dir}: `#{name}` is in the store; " \
|
|
125
|
+
"the flat file is not imported, moved to #{backup path}"
|
|
126
|
+
else
|
|
127
|
+
@store.set name, encode(hash, File.mtime(path))
|
|
128
|
+
Util.info "collections #{dir}: `#{name}` imported, the flat " \
|
|
129
|
+
"file moved to #{backup path}"
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# @param warn [Boolean] warn about a file that is not valid YAML
|
|
136
|
+
# @return [Array<Array(String, String, Hash)>] name, path and YAML of
|
|
137
|
+
# each file directly in the directory that holds a collection
|
|
138
|
+
def flat_collections(warn: true)
|
|
139
|
+
return [] unless Dir.exist? dir
|
|
140
|
+
|
|
141
|
+
Dir.children(dir).sort.filter_map do |name|
|
|
142
|
+
path = File.join dir, name
|
|
143
|
+
next unless File.file?(path) && !name.start_with?(".")
|
|
144
|
+
|
|
145
|
+
hash = YAML.safe_load File.read(path, encoding: "UTF-8"),
|
|
146
|
+
permitted_classes: [Date, Time, Symbol],
|
|
147
|
+
aliases: true
|
|
148
|
+
[name, path, hash] if hash.is_a?(Hash) && hash["root"].is_a?(Hash)
|
|
149
|
+
rescue Psych::Exception, ArgumentError => e
|
|
150
|
+
Util.warn "the file `#{path}` isn't a collection: #{e.message}" if warn
|
|
151
|
+
nil
|
|
152
|
+
rescue Errno::ENOENT
|
|
153
|
+
nil # another process moved it
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Moves a file to `<dir>-v1.bak/`; a name the backup already holds
|
|
158
|
+
# takes a numeric suffix.
|
|
159
|
+
# @return [String] the backup path
|
|
160
|
+
def backup(path)
|
|
161
|
+
bak = "#{File.expand_path dir}-v1.bak" # `dir/` must not put it inside
|
|
162
|
+
FileUtils.mkdir_p bak
|
|
163
|
+
target = File.join bak, File.basename(path)
|
|
164
|
+
n = 0
|
|
165
|
+
target = File.join(bak, "#{File.basename path}.#{n += 1}") while File.exist? target
|
|
166
|
+
FileUtils.mv path, target
|
|
167
|
+
target
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
data/lib/relaton/cli/command.rb
CHANGED
|
@@ -2,6 +2,7 @@ require "date"
|
|
|
2
2
|
require "tmpdir"
|
|
3
3
|
require "shellwords"
|
|
4
4
|
require "pubid"
|
|
5
|
+
require_relative "util"
|
|
5
6
|
require_relative "relaton_file"
|
|
6
7
|
require_relative "xml_convertor"
|
|
7
8
|
require_relative "yaml_convertor"
|
|
@@ -58,10 +59,14 @@ module Relaton
|
|
|
58
59
|
option :"publication-date-after",
|
|
59
60
|
desc: "Fetch only documents published after the specified date " \
|
|
60
61
|
"(e.g. 2002, 2002-01, or 2002-01-01)"
|
|
62
|
+
option :api, type: :boolean, desc: Relaton::Cli::API_OPTION_DESC
|
|
61
63
|
|
|
62
64
|
def fetch(code)
|
|
65
|
+
api_was = apply_api_option options
|
|
63
66
|
io = IO.new($stdout.fcntl(::Fcntl::F_DUPFD), mode: "w:UTF-8")
|
|
64
67
|
io.puts(fetch_document(code, options) || supported_type_message)
|
|
68
|
+
ensure
|
|
69
|
+
restore_api_option api_was
|
|
65
70
|
end
|
|
66
71
|
|
|
67
72
|
desc "Metanorma-XML-File / Directory Relaton-XML-Directory",
|
|
@@ -343,6 +348,26 @@ module Relaton
|
|
|
343
348
|
"Invalid date range: --publication-date-after (#{date_after}) must be before --publication-date-before (#{date_before})."
|
|
344
349
|
end
|
|
345
350
|
|
|
351
|
+
# `--api` / `--no-api` (relaton#535) sets the API tier for this command;
|
|
352
|
+
# no flag keeps the configuration (RELATON_USE_API, else off). Offline
|
|
353
|
+
# mode still makes no request. The command restores the value it
|
|
354
|
+
# returns (#restore_api_option), so a caller that runs the CLI in its
|
|
355
|
+
# own process keeps its configuration.
|
|
356
|
+
#
|
|
357
|
+
# @return [Boolean, nil] the configured value, to restore
|
|
358
|
+
def apply_api_option(options)
|
|
359
|
+
config = Relaton::Db.configuration
|
|
360
|
+
was = config.use_api
|
|
361
|
+
config.use_api = options[:api] unless options[:api].nil?
|
|
362
|
+
was
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# @param was [Boolean, nil] what #apply_api_option returned; nil when
|
|
366
|
+
# the command stopped before it
|
|
367
|
+
def restore_api_option(was)
|
|
368
|
+
Relaton::Db.configuration.use_api = was unless was.nil?
|
|
369
|
+
end
|
|
370
|
+
|
|
346
371
|
# @param code [String]
|
|
347
372
|
# @param options [Hash]
|
|
348
373
|
# @option options [String] :type
|
|
@@ -351,7 +376,9 @@ module Relaton
|
|
|
351
376
|
# @return [String, nil]
|
|
352
377
|
def fetch_document(code, options) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize,Metrics/MethodLength
|
|
353
378
|
year = options[:year]&.to_s
|
|
379
|
+
# `api` is the CLI's switch (#apply_api_option), not a fetch option.
|
|
354
380
|
dup_opts = options.dup.transform_keys { |k| k.to_s.gsub("-", "_").to_sym }
|
|
381
|
+
dup_opts.delete :api
|
|
355
382
|
%i[publication_date_before publication_date_after].each do |key|
|
|
356
383
|
dup_opts[key] = parse_date_option(dup_opts[key], key.to_s.tr("_", "-").prepend("--")) if dup_opts[key]
|
|
357
384
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
require "pubid"
|
|
2
|
+
require_relative "util"
|
|
2
3
|
require_relative "full_text_search"
|
|
4
|
+
require_relative "collection_store"
|
|
3
5
|
|
|
4
6
|
module Relaton
|
|
5
7
|
module Cli
|
|
@@ -15,15 +17,10 @@ module Relaton
|
|
|
15
17
|
option :doctype, desc: "Documents type"
|
|
16
18
|
|
|
17
19
|
def create(file)
|
|
18
|
-
dir = directory
|
|
19
|
-
file_path = File.join dir, file
|
|
20
20
|
col = Relaton::Bibcollection.new options
|
|
21
|
-
if
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
FileUtils.mkdir_p dir # unless Dir.exist? dir
|
|
25
|
-
File.write file_path, col.to_yaml, encoding: "UTF-8"
|
|
26
|
-
end
|
|
21
|
+
return if store.create file, col
|
|
22
|
+
|
|
23
|
+
Util.warn "Collection `#{file}` aready exist"
|
|
27
24
|
end
|
|
28
25
|
|
|
29
26
|
desc "info COLLECTION", "View collection information"
|
|
@@ -31,11 +28,12 @@ module Relaton
|
|
|
31
28
|
"Default is $HOME/.relaton/collections."
|
|
32
29
|
|
|
33
30
|
def info(file) # rubocop:disable Metrics/AbcSize
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
puts "
|
|
38
|
-
|
|
31
|
+
col = store.get file
|
|
32
|
+
return Util.warn "Collection `#{file}` does not exist" unless col
|
|
33
|
+
|
|
34
|
+
puts "Collection: #{file}"
|
|
35
|
+
puts "Last updated: #{store.updated(file)&.localtime}"
|
|
36
|
+
puts "File size: #{store.size file}"
|
|
39
37
|
puts "Number of items: #{col.items.size}"
|
|
40
38
|
puts "Author: #{col.author}"
|
|
41
39
|
puts "Title: #{col.title}"
|
|
@@ -47,12 +45,9 @@ module Relaton
|
|
|
47
45
|
option :entries, aliases: :e, type: :boolean, desc: "Show entries"
|
|
48
46
|
|
|
49
47
|
def list
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
puts File.basename f
|
|
54
|
-
puts_entries yml
|
|
55
|
-
end
|
|
48
|
+
store.names.each do |name|
|
|
49
|
+
puts name
|
|
50
|
+
puts_entries store.get(name)
|
|
56
51
|
end
|
|
57
52
|
end
|
|
58
53
|
|
|
@@ -90,7 +85,7 @@ module Relaton
|
|
|
90
85
|
searcher = Relaton::FullTextSeatch.new(col[:collection])
|
|
91
86
|
searcher.search text
|
|
92
87
|
if searcher.any?
|
|
93
|
-
puts "Collection: #{
|
|
88
|
+
puts "Collection: #{col[:file]}"
|
|
94
89
|
searcher.print_results
|
|
95
90
|
end
|
|
96
91
|
end
|
|
@@ -113,8 +108,10 @@ module Relaton
|
|
|
113
108
|
desc: "Collection to store a document"
|
|
114
109
|
option :dir, aliases: :d, desc: "Directory with collections. Default is " \
|
|
115
110
|
"$HOME/.relaton/collections."
|
|
111
|
+
option :api, type: :boolean, desc: Relaton::Cli::API_OPTION_DESC
|
|
116
112
|
|
|
117
113
|
def fetch(code) # rubocop:disable Metrics/AbcSize
|
|
114
|
+
api_was = apply_api_option options
|
|
118
115
|
opts = {}
|
|
119
116
|
if options[:"publication-date-before"]
|
|
120
117
|
opts[:publication_date_before] = parse_date_option(options[:"publication-date-before"], "--publication-date-before")
|
|
@@ -124,11 +121,7 @@ module Relaton
|
|
|
124
121
|
end
|
|
125
122
|
validate_date_range opts[:publication_date_after], opts[:publication_date_before]
|
|
126
123
|
doc = Relaton.db.fetch(code, options[:year]&.to_s, **opts)
|
|
127
|
-
if doc
|
|
128
|
-
colfile = File.join directory, options[:collection]
|
|
129
|
-
coll = read_collection colfile
|
|
130
|
-
coll << doc
|
|
131
|
-
File.write colfile, coll.to_yaml, encoding: "UTF-8"
|
|
124
|
+
if doc then add_to_collection doc
|
|
132
125
|
else Util.info "No matching bibliographic entry found"
|
|
133
126
|
end
|
|
134
127
|
rescue Relaton::UnknownReferenceError, Relaton::NotInLocalStoreError,
|
|
@@ -139,6 +132,8 @@ module Relaton
|
|
|
139
132
|
Util.warn e.message
|
|
140
133
|
rescue Pubid::Errors::Error
|
|
141
134
|
Util.error %("#{code}" is not a recognized standards identifier)
|
|
135
|
+
ensure
|
|
136
|
+
restore_api_option api_was
|
|
142
137
|
end
|
|
143
138
|
|
|
144
139
|
desc "import FILE", "Import document or collection from an XML file " \
|
|
@@ -149,23 +144,19 @@ module Relaton
|
|
|
149
144
|
option :dir, aliases: :d, desc: "Directory with collections. Default is " \
|
|
150
145
|
"$HOME/.relaton/collections."
|
|
151
146
|
|
|
152
|
-
def import(file)
|
|
153
|
-
|
|
154
|
-
coll = read_collection collfile
|
|
155
|
-
xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
|
|
156
|
-
xml.remove_namespaces!
|
|
147
|
+
def import(file)
|
|
148
|
+
xml = read_xml file
|
|
157
149
|
if xml.at "relaton-collection"
|
|
158
150
|
imported = import_collection(xml)
|
|
159
|
-
|
|
151
|
+
store.update(options[:collection]) do |coll|
|
|
152
|
+
next imported unless coll # a new collection is the imported one
|
|
153
|
+
|
|
160
154
|
coll << imported
|
|
161
|
-
|
|
162
|
-
coll = imported
|
|
155
|
+
coll
|
|
163
156
|
end
|
|
164
157
|
else
|
|
165
|
-
|
|
166
|
-
coll << Relaton::Bibdata.from_xml(xml)
|
|
158
|
+
add_to_collection Relaton::Bibdata.from_xml(xml)
|
|
167
159
|
end
|
|
168
|
-
File.write collfile, coll.to_yaml, encoding: "UTF-8"
|
|
169
160
|
end
|
|
170
161
|
|
|
171
162
|
desc "export COLLECTION", "Export collection into XML file"
|
|
@@ -173,7 +164,9 @@ module Relaton
|
|
|
173
164
|
"$HOME/.relaton/collections."
|
|
174
165
|
|
|
175
166
|
def export(file)
|
|
176
|
-
coll =
|
|
167
|
+
coll = store.get file
|
|
168
|
+
return Util.warn "Collection `#{file}` does not exist" unless coll
|
|
169
|
+
|
|
177
170
|
outfile = "#{file.sub(/\.\w+$/, '')}.xml"
|
|
178
171
|
File.write outfile, coll.to_xml(bibdata: true), encoding: "UTF-8"
|
|
179
172
|
end
|
|
@@ -201,44 +194,44 @@ module Relaton
|
|
|
201
194
|
options.fetch :dir, File.join(Dir.home, ".relaton/collections")
|
|
202
195
|
end
|
|
203
196
|
|
|
204
|
-
#
|
|
205
|
-
# @return [
|
|
206
|
-
def
|
|
207
|
-
|
|
208
|
-
rescue Psych::SyntaxError
|
|
209
|
-
Util.error "the file `#{file}` isn't a collection."
|
|
210
|
-
nil
|
|
197
|
+
# The collections, in a lutaml-store store under `<dir>/v2` (#549).
|
|
198
|
+
# @return [Relaton::Cli::CollectionStore]
|
|
199
|
+
def store
|
|
200
|
+
@store ||= CollectionStore.new directory
|
|
211
201
|
end
|
|
212
202
|
|
|
213
|
-
#
|
|
214
|
-
#
|
|
215
|
-
def
|
|
216
|
-
|
|
203
|
+
# Adds an item to the `--collection` in one atomic read-modify-write,
|
|
204
|
+
# and creates the collection when it is missing.
|
|
205
|
+
def add_to_collection(item)
|
|
206
|
+
store.update(options[:collection]) do |coll|
|
|
207
|
+
coll ||= Relaton::Bibcollection.new({})
|
|
208
|
+
coll << item
|
|
209
|
+
coll
|
|
210
|
+
end
|
|
211
|
+
end
|
|
217
212
|
|
|
218
|
-
|
|
213
|
+
# The XML file `import` reads: user input, not a collection.
|
|
214
|
+
# @return [Nokogiri::XML::Document]
|
|
215
|
+
def read_xml(file)
|
|
216
|
+
xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
|
|
217
|
+
xml.remove_namespaces!
|
|
218
|
+
xml
|
|
219
219
|
end
|
|
220
220
|
|
|
221
221
|
# @return [Array<Hash>]
|
|
222
222
|
def collections
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
if yml && yml["root"]
|
|
227
|
-
m << { collection: Relaton::Bibcollection.new(yml["root"]),
|
|
228
|
-
file: f }
|
|
229
|
-
end
|
|
230
|
-
m
|
|
223
|
+
store.names(options.fetch(:collection, "*")).filter_map do |name|
|
|
224
|
+
coll = store.get name
|
|
225
|
+
{ collection: coll, file: name } if coll
|
|
231
226
|
end
|
|
232
227
|
end
|
|
233
228
|
|
|
234
|
-
# Puts document IDs for each item in
|
|
235
|
-
# @param
|
|
236
|
-
def puts_entries(
|
|
229
|
+
# Puts document IDs for each item in the collection
|
|
230
|
+
# @param coll [Relaton::Bibcollection]
|
|
231
|
+
def puts_entries(coll)
|
|
237
232
|
return unless options[:entries]
|
|
238
233
|
|
|
239
|
-
|
|
240
|
-
puts " #{b.docidentifier}"
|
|
241
|
-
end
|
|
234
|
+
coll.items.each { |b| puts " #{b.docidentifier}" }
|
|
242
235
|
end
|
|
243
236
|
|
|
244
237
|
# @param item [Relaton::Bibdata]
|
data/lib/relaton/cli/util.rb
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
module Relaton
|
|
2
2
|
module Cli
|
|
3
|
+
# The `--api` option of `fetch` and `collection fetch` (relaton#535).
|
|
4
|
+
# Here, not in command.rb: a command's class body reads it.
|
|
5
|
+
API_OPTION_DESC = "Ask api.relaton.org before the flavor's own " \
|
|
6
|
+
"source; --no-api does not (default: RELATON_USE_API, " \
|
|
7
|
+
"else off)".freeze
|
|
8
|
+
|
|
3
9
|
module Util
|
|
4
10
|
extend Relaton::Bib::Util
|
|
5
11
|
PROGNAME = "relaton-cli".freeze
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
require "net/http"
|
|
2
|
+
require "uri"
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "zlib"
|
|
5
|
+
require_relative "../version"
|
|
6
|
+
|
|
7
|
+
module Relaton
|
|
8
|
+
class Db
|
|
9
|
+
#
|
|
10
|
+
# The api.relaton.org tier of a `Db` fetch (relaton#535): local cache,
|
|
11
|
+
# then this, then the flavor's own `get`.
|
|
12
|
+
#
|
|
13
|
+
# It reads `GET <api_host>/v3/documents/<key>` as Relaton YAML, where the
|
|
14
|
+
# key is one percent-encoded path segment. The API is not authoritative
|
|
15
|
+
# while its coverage is verified per flavor, so every answer it cannot
|
|
16
|
+
# give with certainty is nil, and `Db#fetch_doc` then asks the flavor:
|
|
17
|
+
# a 404, any other status, a transport failure, and a body the flavor
|
|
18
|
+
# cannot read. Nothing here raises, and nothing here touches the cache:
|
|
19
|
+
# `Db` stores the answer as it stores a flavor's, with its ETag, and
|
|
20
|
+
# revalidates a stale row with that ETag.
|
|
21
|
+
#
|
|
22
|
+
module ApiResolver
|
|
23
|
+
ACCEPT = "application/yaml".freeze
|
|
24
|
+
# The transport failures that mean "ask the flavor" (the list of
|
|
25
|
+
# `Index::ShardSource::NET_ERRORS`; not required from there, so that
|
|
26
|
+
# `require "relaton/db"` opens no partial `Relaton::Index`).
|
|
27
|
+
NET_ERRORS = [
|
|
28
|
+
SocketError, Timeout::Error, IOError, SystemCallError,
|
|
29
|
+
OpenSSL::SSL::SSLError, Net::HTTPBadResponse,
|
|
30
|
+
Net::HTTPHeaderSyntaxError, Net::ProtocolError, Zlib::Error
|
|
31
|
+
].freeze
|
|
32
|
+
|
|
33
|
+
#
|
|
34
|
+
# What the API answered. `item` is the document of a 200, with its
|
|
35
|
+
# `etag`; `not_modified` is a 304 to a revalidation, whose `etag` is
|
|
36
|
+
# the one sent (the cached document is still the API's).
|
|
37
|
+
#
|
|
38
|
+
Answer = Data.define(:item, :etag, :not_modified) do
|
|
39
|
+
def initialize(item: nil, etag: nil, not_modified: false)
|
|
40
|
+
super
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
module_function
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
# @param processor [Relaton::Core::Processor] the routed flavor
|
|
48
|
+
# @param key [Pubid::Identifier] the query's cache key; the API is
|
|
49
|
+
# asked for its rendering
|
|
50
|
+
# @param etag [String, nil] the ETag of a stale cached answer: sent as
|
|
51
|
+
# `If-None-Match`, so an unchanged document answers 304 with no body.
|
|
52
|
+
# The ETag is of the YAML representation, and the cache holds XML,
|
|
53
|
+
# so it is the header value as received, never computed here.
|
|
54
|
+
# @return [Answer, nil] nil: ask the flavor
|
|
55
|
+
#
|
|
56
|
+
def fetch(processor, key, etag: nil)
|
|
57
|
+
rsp = get(key.to_s, etag) or return
|
|
58
|
+
return Answer.new(etag: etag, not_modified: true) if rsp.code == "304"
|
|
59
|
+
|
|
60
|
+
item = read(processor, rsp.body, key) or return
|
|
61
|
+
Answer.new(item: item, etag: rsp["ETag"])
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @return [Net::HTTPResponse, nil] a 200, or a 304 to a revalidation;
|
|
65
|
+
# else nil
|
|
66
|
+
def get(ref, etag = nil)
|
|
67
|
+
host = Db.configuration.api_host.chomp("/")
|
|
68
|
+
uri = URI("#{host}/v3/documents/#{escape(ref)}")
|
|
69
|
+
rsp = request(uri, etag)
|
|
70
|
+
return rsp if rsp.code == "200" || (etag && rsp.code == "304")
|
|
71
|
+
|
|
72
|
+
info "HTTP #{rsp.code}", ref
|
|
73
|
+
nil
|
|
74
|
+
rescue *NET_ERRORS => e
|
|
75
|
+
info "#{e.class}: #{e.message}", ref
|
|
76
|
+
nil
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def request(uri, etag = nil)
|
|
80
|
+
timeout = Db.configuration.api_timeout
|
|
81
|
+
headers = { "Accept" => ACCEPT,
|
|
82
|
+
"User-Agent" => "relaton/#{Relaton::VERSION}" }
|
|
83
|
+
headers["If-None-Match"] = etag if etag
|
|
84
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
|
|
85
|
+
open_timeout: timeout,
|
|
86
|
+
read_timeout: timeout) do |http|
|
|
87
|
+
http.get(uri.request_uri, headers)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The API's body is data: a body the flavor's YAML model rejects, or
|
|
92
|
+
# one that is not a document (`--- []` reads as an Array), is a miss,
|
|
93
|
+
# whatever the model raises for it.
|
|
94
|
+
def read(processor, body, ref)
|
|
95
|
+
item = processor.from_yaml body
|
|
96
|
+
return item if item.is_a?(Relaton::Bib::ItemData)
|
|
97
|
+
|
|
98
|
+
info "answer is not a document (#{item.class})", ref
|
|
99
|
+
nil
|
|
100
|
+
rescue StandardError => e
|
|
101
|
+
info "unreadable answer (#{e.class}: #{e.message})", ref
|
|
102
|
+
nil
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# One path segment: `/` and `:` are escaped, a space is `%20`.
|
|
106
|
+
def escape(ref)
|
|
107
|
+
URI.encode_www_form_component(ref).gsub("+", "%20")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def info(reason, ref)
|
|
111
|
+
Util.info "api.relaton.org: #{reason}; asking the flavor", key: ref
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
data/lib/relaton/db/cache.rb
CHANGED
|
@@ -25,6 +25,13 @@ module Relaton
|
|
|
25
25
|
class Cache
|
|
26
26
|
LAYOUT = "v2".freeze
|
|
27
27
|
VERSIONS_KEY = "_versions".freeze
|
|
28
|
+
# The rows key that records the one-shot v1 import (relaton#240). It
|
|
29
|
+
# lives in the store, not in a marker file beside it (relaton#548).
|
|
30
|
+
V1_IMPORTED_KEY = "_v1_imported".freeze
|
|
31
|
+
# The legacy marker file of that import, read but no longer written.
|
|
32
|
+
V1_IMPORTED_FILE = ".v1-imported".freeze
|
|
33
|
+
# Rows keys that hold no bucket.
|
|
34
|
+
RESERVED_KEYS = [VERSIONS_KEY, V1_IMPORTED_KEY].freeze
|
|
28
35
|
STRING_FLAVOR = "_key".freeze
|
|
29
36
|
PUBID_FLAVOR = "_pubid".freeze
|
|
30
37
|
# The legacy string form of a not-found entry (`not_found <date>`).
|
|
@@ -75,10 +82,15 @@ module Relaton
|
|
|
75
82
|
@dir
|
|
76
83
|
end
|
|
77
84
|
|
|
78
|
-
# Remove every row and document
|
|
85
|
+
# Remove every row and document, under one rows transaction (lock
|
|
86
|
+
# order rows, then docs). The v1 import marker is never deleted, not
|
|
87
|
+
# even for a moment, so a cleared cache is not filled again from
|
|
88
|
+
# `<dir>-v1.bak` by this or another process.
|
|
79
89
|
def clear
|
|
80
|
-
@rows.
|
|
81
|
-
|
|
90
|
+
@rows.adapter.transaction do
|
|
91
|
+
@rows.each_key { |k| @rows.delete k unless k == V1_IMPORTED_KEY }
|
|
92
|
+
@docs.clear
|
|
93
|
+
end
|
|
82
94
|
@versions = {}
|
|
83
95
|
end
|
|
84
96
|
|
|
@@ -118,9 +130,11 @@ module Relaton
|
|
|
118
130
|
# @param value [String, Relaton::Db::NotFound, nil] document XML, a
|
|
119
131
|
# not-found entry (also as `not_found <date>`); nil deletes the row
|
|
120
132
|
# @param item_key [Pubid::Identifier, String, nil] the document's key
|
|
133
|
+
# @param etag [String, nil] the ETag of the API answer the document
|
|
134
|
+
# came from (relaton#535), kept on both rows; nil for a flavor answer
|
|
121
135
|
# @return [String, Relaton::Db::NotFound, nil] value
|
|
122
136
|
#
|
|
123
|
-
def store(key, value, item_key: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
137
|
+
def store(key, value, item_key: nil, etag: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
124
138
|
return delete(key) if value.nil?
|
|
125
139
|
|
|
126
140
|
bucket, key = resolve key
|
|
@@ -137,9 +151,10 @@ module Relaton
|
|
|
137
151
|
record_version item_bucket
|
|
138
152
|
@rows.adapter.transaction do
|
|
139
153
|
@docs.set file, value
|
|
140
|
-
upsert item_bucket,
|
|
154
|
+
upsert item_bucket,
|
|
155
|
+
new_row(item_key, CacheEntry::DOC, file, fetched, etag)
|
|
141
156
|
unless canonical(key) == canonical(item_key)
|
|
142
|
-
upsert bucket, new_row(key, CacheEntry::DOC, file, fetched)
|
|
157
|
+
upsert bucket, new_row(key, CacheEntry::DOC, file, fetched, etag)
|
|
143
158
|
end
|
|
144
159
|
end
|
|
145
160
|
value
|
|
@@ -183,7 +198,7 @@ module Relaton
|
|
|
183
198
|
|
|
184
199
|
found = other.lookup(key) or return
|
|
185
200
|
row_id, row = found
|
|
186
|
-
store row_id, other.read_entry(row)
|
|
201
|
+
store row_id, other.read_entry(row), etag: row["etag"]
|
|
187
202
|
end
|
|
188
203
|
|
|
189
204
|
# @param key [Pubid::Identifier, String]
|
|
@@ -192,6 +207,30 @@ module Relaton
|
|
|
192
207
|
lookup(key)&.last&.dig("fetched")
|
|
193
208
|
end
|
|
194
209
|
|
|
210
|
+
# @param key [Pubid::Identifier, String]
|
|
211
|
+
# @return [String, nil] the ETag of the API answer the entry came from
|
|
212
|
+
def etag(key)
|
|
213
|
+
lookup(key)&.last&.dig("etag")
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
#
|
|
217
|
+
# The document and the ETag to revalidate a key with, read from one
|
|
218
|
+
# row in one lookup, so the two belong to one answer. Only the key's
|
|
219
|
+
# own row counts: a dated key's lookup can also read a row that adds a
|
|
220
|
+
# language (`#lookup`), and that row's ETag was issued for another URL.
|
|
221
|
+
#
|
|
222
|
+
# @param key [Pubid::Identifier, String]
|
|
223
|
+
# @return [Array(String, String), nil] the document XML and its ETag
|
|
224
|
+
#
|
|
225
|
+
def revalidation(key)
|
|
226
|
+
row = lookup(key)&.last
|
|
227
|
+
own = canonical resolve(key).last
|
|
228
|
+
return unless row && row["etag"] && row_key(row) == own
|
|
229
|
+
|
|
230
|
+
xml = read_entry row
|
|
231
|
+
[xml, row["etag"]] if xml.is_a?(String)
|
|
232
|
+
end
|
|
233
|
+
|
|
195
234
|
# An undated entry expires after 60 days, a dated document never. A
|
|
196
235
|
# not_found entry expires after 60 days even for a dated query: the
|
|
197
236
|
# document can be published later.
|
|
@@ -319,8 +358,7 @@ module Relaton
|
|
|
319
358
|
# fetch, and redirects were dropped from the v2 model).
|
|
320
359
|
def import_old_layout
|
|
321
360
|
bak = "#{dir}-v1.bak"
|
|
322
|
-
|
|
323
|
-
return if File.exist?(marker) || !Dir.exist?(bak)
|
|
361
|
+
return if v1_imported? || !Dir.exist?(bak)
|
|
324
362
|
|
|
325
363
|
imported = 0
|
|
326
364
|
Dir.glob("#{bak}/**/*.xml").each do |f|
|
|
@@ -331,10 +369,17 @@ module Relaton
|
|
|
331
369
|
self[key] = value
|
|
332
370
|
imported += 1
|
|
333
371
|
end
|
|
334
|
-
|
|
372
|
+
@rows.set V1_IMPORTED_KEY, Date.today.to_s
|
|
335
373
|
Util.info "cache #{dir}: imported #{imported} entries from #{bak}"
|
|
336
374
|
end
|
|
337
375
|
|
|
376
|
+
# An older relaton wrote the marker as a file; it still counts. `get`,
|
|
377
|
+
# not `exists?`: it takes the shared lock, so it waits for a write.
|
|
378
|
+
def v1_imported?
|
|
379
|
+
!@rows.get(V1_IMPORTED_KEY).nil? ||
|
|
380
|
+
File.exist?(File.join(dir, LAYOUT, V1_IMPORTED_FILE))
|
|
381
|
+
end
|
|
382
|
+
|
|
338
383
|
# The wrapped string key of an old-layout document: its own primary
|
|
339
384
|
# docidentifier under its flavor prefix, resolved through the same
|
|
340
385
|
# `wrapped_key` path a string cache read takes. A value no parser reads
|
|
@@ -433,7 +478,7 @@ module Relaton
|
|
|
433
478
|
|
|
434
479
|
def each_row
|
|
435
480
|
@rows.each_key do |bucket|
|
|
436
|
-
next if bucket
|
|
481
|
+
next if RESERVED_KEYS.include? bucket
|
|
437
482
|
|
|
438
483
|
read_bucket(bucket).each { |row| yield bucket, row }
|
|
439
484
|
end
|
|
@@ -510,8 +555,9 @@ module Relaton
|
|
|
510
555
|
end
|
|
511
556
|
end
|
|
512
557
|
|
|
513
|
-
def new_row(key, status, file, fetched)
|
|
514
|
-
row = CacheEntry.new(status: status, file: file, fetched: fetched
|
|
558
|
+
def new_row(key, status, file, fetched, etag = nil)
|
|
559
|
+
row = CacheEntry.new(status: status, file: file, fetched: fetched,
|
|
560
|
+
etag: etag)
|
|
515
561
|
if key.is_a?(String) then row.key = key
|
|
516
562
|
else row.id = canonical(key)
|
|
517
563
|
end
|
|
@@ -15,7 +15,9 @@ module Relaton
|
|
|
15
15
|
# `to_hash`) or, for a processor with no pubid class, by a string (`key`).
|
|
16
16
|
# `file` names the document in the doc store; several rows can share one
|
|
17
17
|
# file (a query row and the row of the document it returned). A
|
|
18
|
-
# `not_found` row has no file.
|
|
18
|
+
# `not_found` row has no file. `etag` is the ETag of the api.relaton.org
|
|
19
|
+
# answer the document came from (relaton#535), for revalidation; a
|
|
20
|
+
# flavor answer, and a row an older relaton wrote, have none.
|
|
19
21
|
class CacheEntry < Lutaml::Model::Serializable
|
|
20
22
|
DOC = "doc".freeze
|
|
21
23
|
NOT_FOUND = "not_found".freeze
|
|
@@ -25,6 +27,7 @@ module Relaton
|
|
|
25
27
|
attribute :status, :string
|
|
26
28
|
attribute :file, :string
|
|
27
29
|
attribute :fetched, :string
|
|
30
|
+
attribute :etag, :string
|
|
28
31
|
|
|
29
32
|
key_value do
|
|
30
33
|
map "id", to: :id
|
|
@@ -32,6 +35,7 @@ module Relaton
|
|
|
32
35
|
map "status", to: :status
|
|
33
36
|
map "file", to: :file
|
|
34
37
|
map "fetched", to: :fetched
|
|
38
|
+
map "etag", to: :etag
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
def not_found?
|
data/lib/relaton/db/config.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
1
3
|
module Relaton
|
|
2
4
|
class Db
|
|
3
5
|
module Config
|
|
@@ -11,19 +13,62 @@ module Relaton
|
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
class Configuration
|
|
16
|
+
# Values of a `RELATON_*` switch (any case) that turn it on.
|
|
17
|
+
SWITCH_ON = %w[1 true yes on].freeze
|
|
14
18
|
# Values of RELATON_OFFLINE that turn offline mode on.
|
|
15
|
-
OFFLINE_ON =
|
|
19
|
+
OFFLINE_ON = SWITCH_ON
|
|
20
|
+
DEFAULT_API_HOST = "https://api.relaton.org".freeze
|
|
16
21
|
|
|
17
22
|
# @!attribute offline
|
|
18
23
|
# @return [Boolean] true: every fetch is answered from the local
|
|
19
24
|
# store only, and a miss raises Relaton::NotInLocalStoreError
|
|
20
25
|
# (relaton#536). Defaults to RELATON_OFFLINE.
|
|
21
|
-
|
|
26
|
+
#
|
|
27
|
+
# @!attribute use_api
|
|
28
|
+
# @return [Boolean] true: a cache miss asks api.relaton.org first,
|
|
29
|
+
# and the flavor answers what the API does not (relaton#535).
|
|
30
|
+
# Defaults to RELATON_USE_API, else false. The CLI's `--api` /
|
|
31
|
+
# `--no-api` sets it for one run.
|
|
32
|
+
# @!attribute api_host
|
|
33
|
+
# @return [String] the API's base URL. Defaults to RELATON_API_HOST,
|
|
34
|
+
# else https://api.relaton.org.
|
|
35
|
+
# @!attribute api_timeout
|
|
36
|
+
# @return [Numeric] open and read timeout of an API request, seconds
|
|
37
|
+
attr_accessor :use_api, :api_host, :api_timeout, :offline
|
|
22
38
|
|
|
23
39
|
def initialize
|
|
24
|
-
@use_api =
|
|
25
|
-
@api_host =
|
|
26
|
-
@
|
|
40
|
+
@use_api = switch? "RELATON_USE_API"
|
|
41
|
+
@api_host = env_api_host || DEFAULT_API_HOST
|
|
42
|
+
@api_timeout = 10
|
|
43
|
+
@offline = switch? "RELATON_OFFLINE"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def switch?(name)
|
|
49
|
+
SWITCH_ON.include? ENV[name].to_s.downcase
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# RELATON_API_HOST, when it is an http(s) URL with a host. A shell
|
|
53
|
+
# value is easy to write without its scheme (`api.example.org`), and
|
|
54
|
+
# the resolver would then ask a host-less URI; that is warned about,
|
|
55
|
+
# not used.
|
|
56
|
+
def env_api_host
|
|
57
|
+
host = ENV["RELATON_API_HOST"].to_s.strip
|
|
58
|
+
return if host.empty?
|
|
59
|
+
|
|
60
|
+
uri = URI.parse(host)
|
|
61
|
+
return host if uri.is_a?(URI::HTTP) && uri.host && !uri.host.empty?
|
|
62
|
+
|
|
63
|
+
invalid_api_host host
|
|
64
|
+
rescue URI::InvalidURIError
|
|
65
|
+
invalid_api_host host
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def invalid_api_host(host)
|
|
69
|
+
Util.warn "RELATON_API_HOST #{host.inspect} is not an http(s) URL; " \
|
|
70
|
+
"using #{DEFAULT_API_HOST}"
|
|
71
|
+
nil
|
|
27
72
|
end
|
|
28
73
|
end
|
|
29
74
|
|
data/lib/relaton/db.rb
CHANGED
|
@@ -287,26 +287,94 @@ module Relaton
|
|
|
287
287
|
end
|
|
288
288
|
|
|
289
289
|
# The flavor's `get` receives the parsed pubid when there is one
|
|
290
|
-
# (relaton#205), else the String.
|
|
291
|
-
#
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
290
|
+
# (relaton#205), else the String. With `use_api`, api.relaton.org is
|
|
291
|
+
# asked first (relaton#535), and the flavor answers whatever it does not.
|
|
292
|
+
#
|
|
293
|
+
# @param api [Hash] the API tier's state for one fetch: in,
|
|
294
|
+
# `:stale_etag`/`:stale` (a stale row's ETag and its document, to
|
|
295
|
+
# revalidate); out, `:etag` (the ETag of an API answer, nil for a
|
|
296
|
+
# flavor answer), which Db#fetch_entry stores with the row
|
|
297
|
+
def fetch_doc(code, year, opts, processor, query = nil, api: {})
|
|
298
|
+
api_answer(code, year, opts, processor, query, api) ||
|
|
299
|
+
processor.get(query || code, year, opts)
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
#
|
|
303
|
+
# The API's document for the query, or nil. The API is asked only for a
|
|
304
|
+
# query that has a pubid cache key, is not all-parts and has no
|
|
305
|
+
# publication date range: the API has no all-parts answer, and a range
|
|
306
|
+
# is not in the key. Its answer is kept only when the document's own
|
|
307
|
+
# cache key is the query's key: the API has no undated or edition logic
|
|
308
|
+
# (`ISO 19115-1` answers the 2014 record), and the flavor's rule for
|
|
309
|
+
# such a query is the one `Db` caches.
|
|
310
|
+
#
|
|
311
|
+
# A stale row the API wrote is revalidated with its ETag. A 304 answers
|
|
312
|
+
# the cached document, fetched today: `Cache#store` reads the row's
|
|
313
|
+
# `fetched` from the document, so the document is stored again, not
|
|
314
|
+
# only its row. It needs no identity check: the row was written from an
|
|
315
|
+
# answer that passed one, for this key.
|
|
316
|
+
#
|
|
317
|
+
def api_answer(code, year, opts, processor, query, api) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists
|
|
318
|
+
api[:etag] = nil
|
|
319
|
+
return unless Db.configuration.use_api
|
|
320
|
+
return if opts[:all_parts] || date_range?(opts)
|
|
321
|
+
|
|
322
|
+
key = query && processor.cache_key(code, year, opts, query)
|
|
323
|
+
return unless key.is_a?(::Pubid::Identifier)
|
|
324
|
+
|
|
325
|
+
answer = ApiResolver.fetch(processor, key, etag: api[:stale_etag])
|
|
326
|
+
return unless answer
|
|
327
|
+
return revalidated(processor, api, answer, key) if answer.not_modified
|
|
328
|
+
|
|
329
|
+
bib = answer.item
|
|
330
|
+
if api_identity?(bib, key, processor)
|
|
331
|
+
api[:etag] = answer.etag
|
|
332
|
+
return bib
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
Util.info "api.relaton.org: answered another document " \
|
|
336
|
+
"(#{bib.docidentifier.first&.content}); asking the flavor",
|
|
337
|
+
key: key.to_s
|
|
338
|
+
nil
|
|
297
339
|
end
|
|
298
340
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
processor.from_xml
|
|
304
|
-
|
|
305
|
-
|
|
341
|
+
# The stale document a 304 kept, fetched today, with its ETag. It is
|
|
342
|
+
# cached data: one that does not read back is a miss (the flavor
|
|
343
|
+
# answers), whatever the model raises for it.
|
|
344
|
+
def revalidated(processor, api, answer, key)
|
|
345
|
+
bib = processor.from_xml api[:stale]
|
|
346
|
+
bib.fetched = Date.today.to_s
|
|
347
|
+
api[:etag] = answer.etag
|
|
348
|
+
bib
|
|
349
|
+
rescue StandardError => e
|
|
350
|
+
Util.info "api.relaton.org: cached document unreadable " \
|
|
351
|
+
"(#{e.class}); asking the flavor", key: key.to_s
|
|
352
|
+
nil
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# The API state Db#fetch_doc starts from: a stale document that the API
|
|
356
|
+
# answered carries the ETag to revalidate it with. Both come from one
|
|
357
|
+
# row of the stale cache (Cache#revalidation), read again here: the
|
|
358
|
+
# stale entry `with_stale` read is not used, since another worker may
|
|
359
|
+
# have replaced the row since, and its ETag would then confirm an older
|
|
360
|
+
# document.
|
|
361
|
+
def api_state(args)
|
|
362
|
+
return {} unless args[:stale].is_a?(String) && args[:stale_db]
|
|
363
|
+
|
|
364
|
+
xml, etag = @semaphore.synchronize do
|
|
365
|
+
args[:stale_db].revalidation(args[:id])
|
|
366
|
+
end
|
|
367
|
+
etag ? { stale_etag: etag, stale: xml } : {}
|
|
306
368
|
end
|
|
307
369
|
|
|
308
|
-
|
|
309
|
-
|
|
370
|
+
# The API's document is data: an identifier the flavor cannot key is a
|
|
371
|
+
# mismatch, whatever the key raises for it.
|
|
372
|
+
def api_identity?(bib, key, processor)
|
|
373
|
+
item_key(bib, @registry.processors.key(processor)) == key
|
|
374
|
+
rescue StandardError => e
|
|
375
|
+
Util.info "api.relaton.org: unkeyable answer (#{e.class}); " \
|
|
376
|
+
"asking the flavor", key: key.to_s
|
|
377
|
+
false
|
|
310
378
|
end
|
|
311
379
|
|
|
312
380
|
def search_xml(processor, xml, text, edition, year)
|
|
@@ -600,9 +668,10 @@ module Relaton
|
|
|
600
668
|
#
|
|
601
669
|
def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
602
670
|
processor = @registry[stdclass]
|
|
671
|
+
api = api_state args
|
|
603
672
|
bib = begin
|
|
604
673
|
net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
|
|
605
|
-
args[:query])
|
|
674
|
+
args[:query], api: api)
|
|
606
675
|
rescue Relaton::RequestError => e
|
|
607
676
|
raise unless args[:stale].is_a?(String)
|
|
608
677
|
|
|
@@ -621,7 +690,8 @@ module Relaton
|
|
|
621
690
|
@semaphore.synchronize do
|
|
622
691
|
args[:db].expire args[:id], year if args[:stale]
|
|
623
692
|
if args[:stale] || refresh || !args[:db].read(args[:id])
|
|
624
|
-
save_bib args[:db], args[:id], bib, entry, stdclass
|
|
693
|
+
save_bib args[:db], args[:id], bib, entry, stdclass,
|
|
694
|
+
etag: api[:etag]
|
|
625
695
|
end
|
|
626
696
|
end
|
|
627
697
|
entry
|
|
@@ -632,9 +702,11 @@ module Relaton
|
|
|
632
702
|
# when the query key differs from it (an undated or incomplete query),
|
|
633
703
|
# the query gets a row that points to the same document.
|
|
634
704
|
#
|
|
635
|
-
|
|
705
|
+
# `etag` is the ETag of the API answer (Db#fetch_doc), nil for a flavor
|
|
706
|
+
# answer.
|
|
707
|
+
def save_bib(db, key, bib, entry, stdclass, etag: nil) # rubocop:disable Metrics/ParameterLists
|
|
636
708
|
item = bib.respond_to?(:docidentifier) && item_key(bib, stdclass)
|
|
637
|
-
db.store key, entry, item_key: item || nil
|
|
709
|
+
db.store key, entry, item_key: item || nil, etag: etag
|
|
638
710
|
end
|
|
639
711
|
|
|
640
712
|
#
|
|
@@ -686,12 +758,13 @@ module Relaton
|
|
|
686
758
|
|
|
687
759
|
# @param query [Pubid::Identifier, nil] what `get` receives in place of
|
|
688
760
|
# the String +code+ (Db#query_and_key)
|
|
689
|
-
|
|
690
|
-
|
|
761
|
+
# @param api [Hash] the API tier's state (Db#fetch_doc)
|
|
762
|
+
def net_retry(code, year, opts, processor, retries, query = nil, api: {}) # rubocop:disable Metrics/ParameterLists
|
|
763
|
+
fetch_doc code, year, opts, processor, query, api: api
|
|
691
764
|
rescue Relaton::RequestError => e
|
|
692
765
|
raise e unless retries > 1
|
|
693
766
|
|
|
694
|
-
net_retry(code, year, opts, processor, retries - 1, query)
|
|
767
|
+
net_retry(code, year, opts, processor, retries - 1, query, api: api)
|
|
695
768
|
end
|
|
696
769
|
|
|
697
770
|
def bib_entry(bib)
|
|
@@ -782,6 +855,7 @@ end
|
|
|
782
855
|
|
|
783
856
|
require_relative "db/util"
|
|
784
857
|
require_relative "db/config"
|
|
858
|
+
require_relative "db/api_resolver"
|
|
785
859
|
require_relative "db/workers_pool"
|
|
786
860
|
require_relative "db/cache"
|
|
787
861
|
require_relative "db/registry"
|
|
@@ -296,14 +296,15 @@ module Relaton
|
|
|
296
296
|
"T-SP-OB.#{ref.number}-#{ref.year}" if bulletin?
|
|
297
297
|
end
|
|
298
298
|
|
|
299
|
-
# The docidentifier (year only, no month or day). A bulletin
|
|
300
|
-
# sector even when the reference is the
|
|
299
|
+
# The docidentifier (year only, no month or day). A bulletin carries no
|
|
300
|
+
# sector even when the reference is the TSB spelling `ITU-T OB.1096`
|
|
301
|
+
# (pubid#696); `Scraper#bureau` still makes TSB its author.
|
|
301
302
|
#
|
|
302
303
|
# @return [String]
|
|
303
304
|
def publication_code
|
|
304
305
|
return "ITU-R RR (#{ref.year})" if radio_regulations?
|
|
305
306
|
|
|
306
|
-
"ITU
|
|
307
|
+
"ITU OB No. #{ref.number} (#{ref.year})"
|
|
307
308
|
end
|
|
308
309
|
|
|
309
310
|
def radio_regulations?
|
data/lib/relaton/itu/scraper.rb
CHANGED
|
@@ -141,30 +141,70 @@ module Relaton
|
|
|
141
141
|
def fetch_publisher_contributors
|
|
142
142
|
return [] unless hit.hit[:code]
|
|
143
143
|
|
|
144
|
-
abbrev = hit.hit[:code].sub(/-\w\s.*/, "")
|
|
145
|
-
case abbrev
|
|
146
|
-
when "ITU"
|
|
147
|
-
name = "International Telecommunication Union"
|
|
148
|
-
url = "www.itu.int"
|
|
149
|
-
end
|
|
150
|
-
org = Relaton::Bib::Organization.new(
|
|
151
|
-
name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
|
|
152
|
-
abbreviation: Relaton::Bib::LocalizedString.new(content: abbrev),
|
|
153
|
-
uri: [Relaton::Bib::Uri.new(content: url)],
|
|
154
|
-
)
|
|
155
144
|
role = Relaton::Bib::Contributor::Role.new(type: "publisher")
|
|
156
|
-
[Relaton::Bib::Contributor.new(organization:
|
|
145
|
+
[Relaton::Bib::Contributor.new(organization: publisher_org,
|
|
146
|
+
role: [role])]
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The publishing organization. Every ITU code names ITU, so a code pubid
|
|
150
|
+
# cannot parse is still ITU's.
|
|
151
|
+
#
|
|
152
|
+
# @return [Relaton::Bib::Organization]
|
|
153
|
+
def publisher_org
|
|
154
|
+
Relaton::Bib::Organization.new(
|
|
155
|
+
name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
|
|
156
|
+
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
157
|
+
content: code_pubid&.publisher || "ITU",
|
|
158
|
+
),
|
|
159
|
+
uri: [Relaton::Bib::Uri.new(content: "www.itu.int")],
|
|
160
|
+
)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# The pubid parse of the hit's (first) code. nil when the hit has no
|
|
164
|
+
# code, or when ITU spells it in a form pubid cannot identify.
|
|
165
|
+
#
|
|
166
|
+
# @return [Pubid::Itu::Identifiers::Base, nil]
|
|
167
|
+
def code_pubid
|
|
168
|
+
return @code_pubid if defined? @code_pubid
|
|
169
|
+
|
|
170
|
+
code = hit.hit[:code].to_s.split(" | ").first
|
|
171
|
+
@code_pubid = code && ::Pubid::Itu.parse(code)
|
|
172
|
+
rescue ::Pubid::Errors::ParseError
|
|
173
|
+
@code_pubid = nil
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# The bureau letter (T, R or D) of the code: its sector, or the sector
|
|
177
|
+
# of the document a wrapper (an annex, an appendix) belongs to. An
|
|
178
|
+
# Operational Bulletin prints no sector (pubid#696), but TSB produces it.
|
|
179
|
+
#
|
|
180
|
+
# @return [String, nil]
|
|
181
|
+
def bureau
|
|
182
|
+
id = code_pubid
|
|
183
|
+
return "T" if id.is_a? ::Pubid::Itu::Identifiers::SpecialPublication
|
|
184
|
+
|
|
185
|
+
while id
|
|
186
|
+
return id.sector.sector if id.sector
|
|
187
|
+
|
|
188
|
+
id = wrapped(id)
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# The document a wrapper identifier belongs to, or nil.
|
|
193
|
+
def wrapped(id)
|
|
194
|
+
base = id.base if id.respond_to?(:base)
|
|
195
|
+
base if base.is_a?(::Pubid::Identifier)
|
|
157
196
|
end
|
|
158
197
|
|
|
159
198
|
# @return [Relaton::Bib::Contributor, nil]
|
|
160
199
|
def fetch_editorial_contributor
|
|
161
200
|
wg_name = parser.fetch_workgroup
|
|
162
|
-
bureau = hit.hit[:code]&.match(/(?<=-)\w/)&.to_s
|
|
163
201
|
return unless bureau
|
|
164
202
|
|
|
165
203
|
org = Relaton::Bib::Organization.new(
|
|
166
204
|
name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
|
|
167
|
-
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
205
|
+
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
206
|
+
content: "ITU-#{bureau}",
|
|
207
|
+
),
|
|
168
208
|
subdivision: group_subdivision(wg_name),
|
|
169
209
|
)
|
|
170
210
|
role = Relaton::Bib::Contributor::Role.new(
|
|
@@ -193,18 +233,7 @@ module Relaton
|
|
|
193
233
|
|
|
194
234
|
# @return [Array<Relaton::Bib::Copyright>]
|
|
195
235
|
def fetch_copyright
|
|
196
|
-
|
|
197
|
-
case abbreviation
|
|
198
|
-
when "ITU"
|
|
199
|
-
name = "International Telecommunication Union"
|
|
200
|
-
url = "www.itu.int"
|
|
201
|
-
end
|
|
202
|
-
org = Relaton::Bib::Organization.new(
|
|
203
|
-
name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
|
|
204
|
-
abbreviation: Relaton::Bib::LocalizedString.new(content: abbreviation),
|
|
205
|
-
uri: [Relaton::Bib::Uri.new(content: url)],
|
|
206
|
-
)
|
|
207
|
-
owner = Relaton::Bib::ContributionInfo.new(organization: org)
|
|
236
|
+
owner = Relaton::Bib::ContributionInfo.new(organization: publisher_org)
|
|
208
237
|
year = parser.doc_date&.match(/\d{4}/)&.to_s
|
|
209
238
|
[Relaton::Bib::Copyright.new(from: year, owner: [owner])]
|
|
210
239
|
end
|
data/lib/relaton/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: relaton
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.1.
|
|
4
|
+
version: 3.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: addressable
|
|
@@ -520,14 +520,14 @@ dependencies:
|
|
|
520
520
|
requirements:
|
|
521
521
|
- - "~>"
|
|
522
522
|
- !ruby/object:Gem::Version
|
|
523
|
-
version: 2.0.
|
|
523
|
+
version: 2.0.25
|
|
524
524
|
type: :runtime
|
|
525
525
|
prerelease: false
|
|
526
526
|
version_requirements: !ruby/object:Gem::Requirement
|
|
527
527
|
requirements:
|
|
528
528
|
- - "~>"
|
|
529
529
|
- !ruby/object:Gem::Version
|
|
530
|
-
version: 2.0.
|
|
530
|
+
version: 2.0.25
|
|
531
531
|
- !ruby/object:Gem::Dependency
|
|
532
532
|
name: yeptris
|
|
533
533
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -821,6 +821,7 @@ files:
|
|
|
821
821
|
- lib/relaton/cie/util.rb
|
|
822
822
|
- lib/relaton/cli.rb
|
|
823
823
|
- lib/relaton/cli/base_convertor.rb
|
|
824
|
+
- lib/relaton/cli/collection_store.rb
|
|
824
825
|
- lib/relaton/cli/command.rb
|
|
825
826
|
- lib/relaton/cli/data_fetcher.rb
|
|
826
827
|
- lib/relaton/cli/frontend/dist/404.html
|
|
@@ -887,6 +888,7 @@ files:
|
|
|
887
888
|
- lib/relaton/core/request_error.rb
|
|
888
889
|
- lib/relaton/core/workers_pool.rb
|
|
889
890
|
- lib/relaton/db.rb
|
|
891
|
+
- lib/relaton/db/api_resolver.rb
|
|
890
892
|
- lib/relaton/db/cache.rb
|
|
891
893
|
- lib/relaton/db/cache_entry.rb
|
|
892
894
|
- lib/relaton/db/config.rb
|