relaton 3.1.3 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cf214dd5828ef52d54652cd3c01d5406527e7dd0f6be2c52447a72889e7ed3c8
4
- data.tar.gz: 48c41d919e828654ae0ddfd8fc0c6dcf2dafc912b65b58b493964145eb936ca6
3
+ metadata.gz: c2dcd5f1b916c7fe552e39b9376228255e2523385c8da7f91e1193b17ffb58d7
4
+ data.tar.gz: e7331e3f2c069cda9567b518411dcb35e866700151987d9b4d4f14e17ef0f8b5
5
5
  SHA512:
6
- metadata.gz: 612e007cdc0a28b1eb0ac6ef9e0e563f067daa8532a25002d76835233e43b5e5051e2af133ad078a96be66d73c56c4ca58f5e3c675e51b8331cb58e3d842060c
7
- data.tar.gz: 55ba69a723f91403724139457801a0fa92842208d5cc06be86fd7260a8507e85ccccbe3eb454f82553247130e1e072b1def59eee9a6c54c9b2086b8c6a29bd8d
6
+ metadata.gz: 65781d01bf2cedf282a94e0275ebf7bcc187fe9d92f029e58768acd782fe05e04f0391ddc579bfce8d7d106242927ad54e9b371b58293dde2651f7e940e201ef
7
+ data.tar.gz: 3f1e0879b62d13c363fd9d34bd02ffdc764a63f7ec8197d1614df983da34bcf02db5086679b68aa391b4fb52d91930be59cc2e58e7719c81ae21a2c9d9de3fa3
data/README.adoc CHANGED
@@ -66,15 +66,25 @@ e.g. `get("ISO 19115-1", "2014", all_parts: true)` is transformed into a referen
66
66
 
67
67
  === Configuration
68
68
 
69
- * `use_api` - `true` if it needs to use an online cache, `false` if not. The default value is `true`.
70
- * `api_host` - URL of an online cache. The default value is https://api.relaton.org.
69
+ * `use_api` - `true` to ask api.relaton.org before the flavor's own fetch on a cache miss; the flavor still answers whatever the API does not. The default value is the `RELATON_USE_API` environment variable (`1`, `true`, `yes` or `on` turn it on), else `false`.
70
+ * `api_host` - URL of the API. The default value is the `RELATON_API_HOST` environment variable, else https://api.relaton.org.
71
+ * `api_timeout` - open and read timeout of an API request, in seconds. The default value is `10`.
72
+ * `offline` - `true` to answer every fetch from the local cache only, with no network request. The default value is the `RELATON_OFFLINE` environment variable, else `false`.
73
+
74
+ The `relaton` command line turns the API on for one run with `--api` and off with `--no-api` (on `relaton fetch` and `relaton collection fetch`); with neither flag it follows `RELATON_USE_API`:
75
+
76
+ [source,sh]
77
+ ----
78
+ relaton fetch "IETF RFC 8341" --api
79
+ RELATON_USE_API=1 relaton fetch "IETF RFC 8341"
80
+ ----
71
81
 
72
82
  [source,ruby]
73
83
  ----
74
84
  require "relaton"
75
85
  => true
76
86
 
77
- Relaton.configure do |conf|
87
+ Relaton::Db.configure do |conf|
78
88
  conf.use_api = true
79
89
  conf.api_host = "https://api.relaton.org"
80
90
  end
@@ -82,7 +82,10 @@ module Relaton
82
82
  # @return [Relaton::Bibdata, Relaton::Bibcollection]
83
83
  def new_bib_item_class(item)
84
84
  if item.is_a?(Hash)
85
- if item["items"]
85
+ # `#to_h` writes a nested collection as `{"root" => …}`.
86
+ if item["root"].is_a?(Hash)
87
+ ::Relaton::Bibcollection.new(item["root"])
88
+ elsif item["items"]
86
89
  ::Relaton::Bibcollection.new(item)
87
90
  else
88
91
  bibitem = ::Relaton::Cli::YAMLConvertor.convert_single_file item
@@ -0,0 +1,171 @@
1
+ require "fileutils"
2
+ require "time"
3
+ require "yaml"
4
+ require "lutaml/store"
5
+ require_relative "../bibcollection"
6
+ require_relative "util"
7
+
8
+ module Relaton
9
+ module Cli
10
+ # The collections of `relaton collection` (relaton#549): one lutaml-store
11
+ # FileSystem store under `<dir>/v2/`, keyed by the collection name the
12
+ # user types. Each value is the collection's YAML (`{"root" => …}`, as
13
+ # `Bibcollection#to_h` makes it) with a top-level `updated` time, so the
14
+ # data file is the same YAML an older relaton wrote.
15
+ #
16
+ # lutaml-store is the only storage layer, as for `Db::Cache` (relaton#548):
17
+ # its lock makes a read-modify-write atomic between threads and processes,
18
+ # and its rename makes a write atomic. Nothing here builds a path from a
19
+ # name or takes a lock of its own. The one raw IO is the one-time move of
20
+ # the flat files an older relaton wrote directly in `<dir>`: each is
21
+ # imported, then moved to `<dir>-v1.bak/`, never deleted.
22
+ class CollectionStore
23
+ LAYOUT = "v2".freeze
24
+ UPDATED = "updated".freeze
25
+
26
+ attr_reader :dir
27
+
28
+ # @param dir [String] the collections directory
29
+ def initialize(dir)
30
+ @dir = dir
31
+ @store = Lutaml::Store::BasicStore.new(
32
+ adapter_type: :filesystem,
33
+ adapter_options: { path: File.join(dir, LAYOUT), extension: ".yaml",
34
+ integrity_checks: false,
35
+ create_directories: false },
36
+ cache: { enabled: false },
37
+ )
38
+ move_flat_files
39
+ end
40
+
41
+ # @param pattern [String] a glob over the names (`--collection`)
42
+ # @return [Array<String>] the matching names, sorted
43
+ def names(pattern = "*")
44
+ @store.keys.select { |name| File.fnmatch pattern, name }.sort
45
+ end
46
+
47
+ # @param name [String]
48
+ def exist?(name)
49
+ @store.exists? name
50
+ end
51
+
52
+ # @param name [String]
53
+ # @return [Relaton::Bibcollection, nil]
54
+ def get(name)
55
+ hash = read name
56
+ Relaton::Bibcollection.new hash["root"] if hash
57
+ end
58
+
59
+ # @param name [String]
60
+ # @return [Time, nil] when the collection was last written
61
+ def updated(name)
62
+ time = read(name)&.dig UPDATED
63
+ Time.iso8601 time if time
64
+ end
65
+
66
+ # @param name [String]
67
+ # @return [Integer, nil] the size of the stored collection, in bytes
68
+ def size(name)
69
+ @store.get(name)&.bytesize
70
+ end
71
+
72
+ # @param name [String]
73
+ # @param coll [Relaton::Bibcollection]
74
+ # @return [Boolean] false when the store already holds the name
75
+ def create(name, coll)
76
+ @store.adapter.transaction do
77
+ next false if @store.exists? name
78
+
79
+ @store.set name, encode(coll.to_h)
80
+ true
81
+ end
82
+ end
83
+
84
+ # Atomic read-modify-write under the store's exclusive lock.
85
+ #
86
+ # @param name [String]
87
+ # @yieldparam coll [Relaton::Bibcollection, nil] nil when it is missing
88
+ # @yieldreturn [Relaton::Bibcollection]
89
+ def update(name)
90
+ @store.update(name) do |value|
91
+ hash = decode value if value
92
+ coll = hash && Relaton::Bibcollection.new(hash["root"])
93
+ encode yield(coll).to_h
94
+ end
95
+ end
96
+
97
+ private
98
+
99
+ # @return [Hash, nil]
100
+ def read(name)
101
+ value = @store.get name
102
+ decode value if value
103
+ end
104
+
105
+ def decode(value)
106
+ YAML.safe_load value, permitted_classes: [Date, Time, Symbol],
107
+ aliases: true
108
+ end
109
+
110
+ def encode(hash, updated = Time.now)
111
+ hash.merge(UPDATED => updated.utc.iso8601).to_yaml
112
+ end
113
+
114
+ # A flat collection file is imported unless the store already holds its
115
+ # name, and then moved to the backup either way, under the store's lock
116
+ # so two processes do not move one file. The scan runs again inside the
117
+ # lock: another process can have moved the files first.
118
+ def move_flat_files
119
+ return if flat_collections.empty?
120
+
121
+ @store.adapter.transaction do
122
+ flat_collections(warn: false).each do |name, path, hash|
123
+ if @store.exists? name
124
+ Util.warn "collections #{dir}: `#{name}` is in the store; " \
125
+ "the flat file is not imported, moved to #{backup path}"
126
+ else
127
+ @store.set name, encode(hash, File.mtime(path))
128
+ Util.info "collections #{dir}: `#{name}` imported, the flat " \
129
+ "file moved to #{backup path}"
130
+ end
131
+ end
132
+ end
133
+ end
134
+
135
+ # @param warn [Boolean] warn about a file that is not valid YAML
136
+ # @return [Array<Array(String, String, Hash)>] name, path and YAML of
137
+ # each file directly in the directory that holds a collection
138
+ def flat_collections(warn: true)
139
+ return [] unless Dir.exist? dir
140
+
141
+ Dir.children(dir).sort.filter_map do |name|
142
+ path = File.join dir, name
143
+ next unless File.file?(path) && !name.start_with?(".")
144
+
145
+ hash = YAML.safe_load File.read(path, encoding: "UTF-8"),
146
+ permitted_classes: [Date, Time, Symbol],
147
+ aliases: true
148
+ [name, path, hash] if hash.is_a?(Hash) && hash["root"].is_a?(Hash)
149
+ rescue Psych::Exception, ArgumentError => e
150
+ Util.warn "the file `#{path}` isn't a collection: #{e.message}" if warn
151
+ nil
152
+ rescue Errno::ENOENT
153
+ nil # another process moved it
154
+ end
155
+ end
156
+
157
+ # Moves a file to `<dir>-v1.bak/`; a name the backup already holds
158
+ # takes a numeric suffix.
159
+ # @return [String] the backup path
160
+ def backup(path)
161
+ bak = "#{File.expand_path dir}-v1.bak" # `dir/` must not put it inside
162
+ FileUtils.mkdir_p bak
163
+ target = File.join bak, File.basename(path)
164
+ n = 0
165
+ target = File.join(bak, "#{File.basename path}.#{n += 1}") while File.exist? target
166
+ FileUtils.mv path, target
167
+ target
168
+ end
169
+ end
170
+ end
171
+ end
@@ -2,6 +2,7 @@ require "date"
2
2
  require "tmpdir"
3
3
  require "shellwords"
4
4
  require "pubid"
5
+ require_relative "util"
5
6
  require_relative "relaton_file"
6
7
  require_relative "xml_convertor"
7
8
  require_relative "yaml_convertor"
@@ -58,10 +59,14 @@ module Relaton
58
59
  option :"publication-date-after",
59
60
  desc: "Fetch only documents published after the specified date " \
60
61
  "(e.g. 2002, 2002-01, or 2002-01-01)"
62
+ option :api, type: :boolean, desc: Relaton::Cli::API_OPTION_DESC
61
63
 
62
64
  def fetch(code)
65
+ api_was = apply_api_option options
63
66
  io = IO.new($stdout.fcntl(::Fcntl::F_DUPFD), mode: "w:UTF-8")
64
67
  io.puts(fetch_document(code, options) || supported_type_message)
68
+ ensure
69
+ restore_api_option api_was
65
70
  end
66
71
 
67
72
  desc "Metanorma-XML-File / Directory Relaton-XML-Directory",
@@ -343,6 +348,26 @@ module Relaton
343
348
  "Invalid date range: --publication-date-after (#{date_after}) must be before --publication-date-before (#{date_before})."
344
349
  end
345
350
 
351
+ # `--api` / `--no-api` (relaton#535) sets the API tier for this command;
352
+ # no flag keeps the configuration (RELATON_USE_API, else off). Offline
353
+ # mode still makes no request. The command restores the value it
354
+ # returns (#restore_api_option), so a caller that runs the CLI in its
355
+ # own process keeps its configuration.
356
+ #
357
+ # @return [Boolean, nil] the configured value, to restore
358
+ def apply_api_option(options)
359
+ config = Relaton::Db.configuration
360
+ was = config.use_api
361
+ config.use_api = options[:api] unless options[:api].nil?
362
+ was
363
+ end
364
+
365
+ # @param was [Boolean, nil] what #apply_api_option returned; nil when
366
+ # the command stopped before it
367
+ def restore_api_option(was)
368
+ Relaton::Db.configuration.use_api = was unless was.nil?
369
+ end
370
+
346
371
  # @param code [String]
347
372
  # @param options [Hash]
348
373
  # @option options [String] :type
@@ -351,7 +376,9 @@ module Relaton
351
376
  # @return [String, nil]
352
377
  def fetch_document(code, options) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize,Metrics/MethodLength
353
378
  year = options[:year]&.to_s
379
+ # `api` is the CLI's switch (#apply_api_option), not a fetch option.
354
380
  dup_opts = options.dup.transform_keys { |k| k.to_s.gsub("-", "_").to_sym }
381
+ dup_opts.delete :api
355
382
  %i[publication_date_before publication_date_after].each do |key|
356
383
  dup_opts[key] = parse_date_option(dup_opts[key], key.to_s.tr("_", "-").prepend("--")) if dup_opts[key]
357
384
  end
@@ -1,5 +1,7 @@
1
1
  require "pubid"
2
+ require_relative "util"
2
3
  require_relative "full_text_search"
4
+ require_relative "collection_store"
3
5
 
4
6
  module Relaton
5
7
  module Cli
@@ -15,15 +17,10 @@ module Relaton
15
17
  option :doctype, desc: "Documents type"
16
18
 
17
19
  def create(file)
18
- dir = directory
19
- file_path = File.join dir, file
20
20
  col = Relaton::Bibcollection.new options
21
- if File.exist? file_path
22
- Util.warn "Collection `#{file}` aready exist"
23
- else
24
- FileUtils.mkdir_p dir # unless Dir.exist? dir
25
- File.write file_path, col.to_yaml, encoding: "UTF-8"
26
- end
21
+ return if store.create file, col
22
+
23
+ Util.warn "Collection `#{file}` aready exist"
27
24
  end
28
25
 
29
26
  desc "info COLLECTION", "View collection information"
@@ -31,11 +28,12 @@ module Relaton
31
28
  "Default is $HOME/.relaton/collections."
32
29
 
33
30
  def info(file) # rubocop:disable Metrics/AbcSize
34
- path = File.join directory, file
35
- puts "Collection: #{File.basename path}"
36
- puts "Last updated: #{File.mtime path}"
37
- puts "File size: #{File.size path}"
38
- col = Relaton::Bibcollection.new YAML.load_file(path)["root"]
31
+ col = store.get file
32
+ return Util.warn "Collection `#{file}` does not exist" unless col
33
+
34
+ puts "Collection: #{file}"
35
+ puts "Last updated: #{store.updated(file)&.localtime}"
36
+ puts "File size: #{store.size file}"
39
37
  puts "Number of items: #{col.items.size}"
40
38
  puts "Author: #{col.author}"
41
39
  puts "Title: #{col.title}"
@@ -47,12 +45,9 @@ module Relaton
47
45
  option :entries, aliases: :e, type: :boolean, desc: "Show entries"
48
46
 
49
47
  def list
50
- Dir[File.join(directory, "*")].each do |f|
51
- yml = read_yaml f
52
- if yml && yml["root"]
53
- puts File.basename f
54
- puts_entries yml
55
- end
48
+ store.names.each do |name|
49
+ puts name
50
+ puts_entries store.get(name)
56
51
  end
57
52
  end
58
53
 
@@ -90,7 +85,7 @@ module Relaton
90
85
  searcher = Relaton::FullTextSeatch.new(col[:collection])
91
86
  searcher.search text
92
87
  if searcher.any?
93
- puts "Collection: #{File.basename(col[:file])}"
88
+ puts "Collection: #{col[:file]}"
94
89
  searcher.print_results
95
90
  end
96
91
  end
@@ -113,8 +108,10 @@ module Relaton
113
108
  desc: "Collection to store a document"
114
109
  option :dir, aliases: :d, desc: "Directory with collections. Default is " \
115
110
  "$HOME/.relaton/collections."
111
+ option :api, type: :boolean, desc: Relaton::Cli::API_OPTION_DESC
116
112
 
117
113
  def fetch(code) # rubocop:disable Metrics/AbcSize
114
+ api_was = apply_api_option options
118
115
  opts = {}
119
116
  if options[:"publication-date-before"]
120
117
  opts[:publication_date_before] = parse_date_option(options[:"publication-date-before"], "--publication-date-before")
@@ -124,11 +121,7 @@ module Relaton
124
121
  end
125
122
  validate_date_range opts[:publication_date_after], opts[:publication_date_before]
126
123
  doc = Relaton.db.fetch(code, options[:year]&.to_s, **opts)
127
- if doc
128
- colfile = File.join directory, options[:collection]
129
- coll = read_collection colfile
130
- coll << doc
131
- File.write colfile, coll.to_yaml, encoding: "UTF-8"
124
+ if doc then add_to_collection doc
132
125
  else Util.info "No matching bibliographic entry found"
133
126
  end
134
127
  rescue Relaton::UnknownReferenceError, Relaton::NotInLocalStoreError,
@@ -139,6 +132,8 @@ module Relaton
139
132
  Util.warn e.message
140
133
  rescue Pubid::Errors::Error
141
134
  Util.error %("#{code}" is not a recognized standards identifier)
135
+ ensure
136
+ restore_api_option api_was
142
137
  end
143
138
 
144
139
  desc "import FILE", "Import document or collection from an XML file " \
@@ -149,23 +144,19 @@ module Relaton
149
144
  option :dir, aliases: :d, desc: "Directory with collections. Default is " \
150
145
  "$HOME/.relaton/collections."
151
146
 
152
- def import(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
153
- collfile = File.join directory, options[:collection]
154
- coll = read_collection collfile
155
- xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
156
- xml.remove_namespaces!
147
+ def import(file)
148
+ xml = read_xml file
157
149
  if xml.at "relaton-collection"
158
150
  imported = import_collection(xml)
159
- if coll
151
+ store.update(options[:collection]) do |coll|
152
+ next imported unless coll # a new collection is the imported one
153
+
160
154
  coll << imported
161
- else
162
- coll = imported
155
+ coll
163
156
  end
164
157
  else
165
- coll ||= Relaton::Bibcollection.new({})
166
- coll << Relaton::Bibdata.from_xml(xml)
158
+ add_to_collection Relaton::Bibdata.from_xml(xml)
167
159
  end
168
- File.write collfile, coll.to_yaml, encoding: "UTF-8"
169
160
  end
170
161
 
171
162
  desc "export COLLECTION", "Export collection into XML file"
@@ -173,7 +164,9 @@ module Relaton
173
164
  "$HOME/.relaton/collections."
174
165
 
175
166
  def export(file)
176
- coll = read_collection File.join(directory, file)
167
+ coll = store.get file
168
+ return Util.warn "Collection `#{file}` does not exist" unless coll
169
+
177
170
  outfile = "#{file.sub(/\.\w+$/, '')}.xml"
178
171
  File.write outfile, coll.to_xml(bibdata: true), encoding: "UTF-8"
179
172
  end
@@ -201,44 +194,44 @@ module Relaton
201
194
  options.fetch :dir, File.join(Dir.home, ".relaton/collections")
202
195
  end
203
196
 
204
- # @param file [String]
205
- # @return [Hash, nil]
206
- def read_yaml(file)
207
- YAML.load_file file if File.file? file
208
- rescue Psych::SyntaxError
209
- Util.error "the file `#{file}` isn't a collection."
210
- nil
197
+ # The collections, in a lutaml-store store under `<dir>/v2` (#549).
198
+ # @return [Relaton::Cli::CollectionStore]
199
+ def store
200
+ @store ||= CollectionStore.new directory
211
201
  end
212
202
 
213
- # @param file [String]
214
- # @return [Relaton::Bibcollection, nil]
215
- def read_collection(file)
216
- return unless File.file?(file)
203
+ # Adds an item to the `--collection` in one atomic read-modify-write,
204
+ # and creates the collection when it is missing.
205
+ def add_to_collection(item)
206
+ store.update(options[:collection]) do |coll|
207
+ coll ||= Relaton::Bibcollection.new({})
208
+ coll << item
209
+ coll
210
+ end
211
+ end
217
212
 
218
- Relaton::Bibcollection.new YAML.load_file(file)["root"]
213
+ # The XML file `import` reads: user input, not a collection.
214
+ # @return [Nokogiri::XML::Document]
215
+ def read_xml(file)
216
+ xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
217
+ xml.remove_namespaces!
218
+ xml
219
219
  end
220
220
 
221
221
  # @return [Array<Hash>]
222
222
  def collections
223
- file = options.fetch :collection, "*"
224
- Dir[File.join directory, file].reduce([]) do |m, f|
225
- yml = read_yaml f
226
- if yml && yml["root"]
227
- m << { collection: Relaton::Bibcollection.new(yml["root"]),
228
- file: f }
229
- end
230
- m
223
+ store.names(options.fetch(:collection, "*")).filter_map do |name|
224
+ coll = store.get name
225
+ { collection: coll, file: name } if coll
231
226
  end
232
227
  end
233
228
 
234
- # Puts document IDs for each item in tthe cokllection
235
- # @param hash [Hash] Relaton collection
236
- def puts_entries(hash)
229
+ # Puts document IDs for each item in the collection
230
+ # @param coll [Relaton::Bibcollection]
231
+ def puts_entries(coll)
237
232
  return unless options[:entries]
238
233
 
239
- Relaton::Bibcollection.new(hash["root"]).items.each do |b|
240
- puts " #{b.docidentifier}"
241
- end
234
+ coll.items.each { |b| puts " #{b.docidentifier}" }
242
235
  end
243
236
 
244
237
  # @param item [Relaton::Bibdata]
@@ -1,5 +1,11 @@
1
1
  module Relaton
2
2
  module Cli
3
+ # The `--api` option of `fetch` and `collection fetch` (relaton#535).
4
+ # Here, not in command.rb: a command's class body reads it.
5
+ API_OPTION_DESC = "Ask api.relaton.org before the flavor's own " \
6
+ "source; --no-api does not (default: RELATON_USE_API, " \
7
+ "else off)".freeze
8
+
3
9
  module Util
4
10
  extend Relaton::Bib::Util
5
11
  PROGNAME = "relaton-cli".freeze
@@ -0,0 +1,115 @@
1
+ require "net/http"
2
+ require "uri"
3
+ require "openssl"
4
+ require "zlib"
5
+ require_relative "../version"
6
+
7
+ module Relaton
8
+ class Db
9
+ #
10
+ # The api.relaton.org tier of a `Db` fetch (relaton#535): local cache,
11
+ # then this, then the flavor's own `get`.
12
+ #
13
+ # It reads `GET <api_host>/v3/documents/<key>` as Relaton YAML, where the
14
+ # key is one percent-encoded path segment. The API is not authoritative
15
+ # while its coverage is verified per flavor, so every answer it cannot
16
+ # give with certainty is nil, and `Db#fetch_doc` then asks the flavor:
17
+ # a 404, any other status, a transport failure, and a body the flavor
18
+ # cannot read. Nothing here raises, and nothing here touches the cache:
19
+ # `Db` stores the answer as it stores a flavor's, with its ETag, and
20
+ # revalidates a stale row with that ETag.
21
+ #
22
+ module ApiResolver
23
+ ACCEPT = "application/yaml".freeze
24
+ # The transport failures that mean "ask the flavor" (the list of
25
+ # `Index::ShardSource::NET_ERRORS`; not required from there, so that
26
+ # `require "relaton/db"` opens no partial `Relaton::Index`).
27
+ NET_ERRORS = [
28
+ SocketError, Timeout::Error, IOError, SystemCallError,
29
+ OpenSSL::SSL::SSLError, Net::HTTPBadResponse,
30
+ Net::HTTPHeaderSyntaxError, Net::ProtocolError, Zlib::Error
31
+ ].freeze
32
+
33
+ #
34
+ # What the API answered. `item` is the document of a 200, with its
35
+ # `etag`; `not_modified` is a 304 to a revalidation, whose `etag` is
36
+ # the one sent (the cached document is still the API's).
37
+ #
38
+ Answer = Data.define(:item, :etag, :not_modified) do
39
+ def initialize(item: nil, etag: nil, not_modified: false)
40
+ super
41
+ end
42
+ end
43
+
44
+ module_function
45
+
46
+ #
47
+ # @param processor [Relaton::Core::Processor] the routed flavor
48
+ # @param key [Pubid::Identifier] the query's cache key; the API is
49
+ # asked for its rendering
50
+ # @param etag [String, nil] the ETag of a stale cached answer: sent as
51
+ # `If-None-Match`, so an unchanged document answers 304 with no body.
52
+ # The ETag is of the YAML representation, and the cache holds XML,
53
+ # so it is the header value as received, never computed here.
54
+ # @return [Answer, nil] nil: ask the flavor
55
+ #
56
+ def fetch(processor, key, etag: nil)
57
+ rsp = get(key.to_s, etag) or return
58
+ return Answer.new(etag: etag, not_modified: true) if rsp.code == "304"
59
+
60
+ item = read(processor, rsp.body, key) or return
61
+ Answer.new(item: item, etag: rsp["ETag"])
62
+ end
63
+
64
+ # @return [Net::HTTPResponse, nil] a 200, or a 304 to a revalidation;
65
+ # else nil
66
+ def get(ref, etag = nil)
67
+ host = Db.configuration.api_host.chomp("/")
68
+ uri = URI("#{host}/v3/documents/#{escape(ref)}")
69
+ rsp = request(uri, etag)
70
+ return rsp if rsp.code == "200" || (etag && rsp.code == "304")
71
+
72
+ info "HTTP #{rsp.code}", ref
73
+ nil
74
+ rescue *NET_ERRORS => e
75
+ info "#{e.class}: #{e.message}", ref
76
+ nil
77
+ end
78
+
79
+ def request(uri, etag = nil)
80
+ timeout = Db.configuration.api_timeout
81
+ headers = { "Accept" => ACCEPT,
82
+ "User-Agent" => "relaton/#{Relaton::VERSION}" }
83
+ headers["If-None-Match"] = etag if etag
84
+ Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
85
+ open_timeout: timeout,
86
+ read_timeout: timeout) do |http|
87
+ http.get(uri.request_uri, headers)
88
+ end
89
+ end
90
+
91
+ # The API's body is data: a body the flavor's YAML model rejects, or
92
+ # one that is not a document (`--- []` reads as an Array), is a miss,
93
+ # whatever the model raises for it.
94
+ def read(processor, body, ref)
95
+ item = processor.from_yaml body
96
+ return item if item.is_a?(Relaton::Bib::ItemData)
97
+
98
+ info "answer is not a document (#{item.class})", ref
99
+ nil
100
+ rescue StandardError => e
101
+ info "unreadable answer (#{e.class}: #{e.message})", ref
102
+ nil
103
+ end
104
+
105
+ # One path segment: `/` and `:` are escaped, a space is `%20`.
106
+ def escape(ref)
107
+ URI.encode_www_form_component(ref).gsub("+", "%20")
108
+ end
109
+
110
+ def info(reason, ref)
111
+ Util.info "api.relaton.org: #{reason}; asking the flavor", key: ref
112
+ end
113
+ end
114
+ end
115
+ end
@@ -25,6 +25,13 @@ module Relaton
25
25
  class Cache
26
26
  LAYOUT = "v2".freeze
27
27
  VERSIONS_KEY = "_versions".freeze
28
+ # The rows key that records the one-shot v1 import (relaton#240). It
29
+ # lives in the store, not in a marker file beside it (relaton#548).
30
+ V1_IMPORTED_KEY = "_v1_imported".freeze
31
+ # The legacy marker file of that import, read but no longer written.
32
+ V1_IMPORTED_FILE = ".v1-imported".freeze
33
+ # Rows keys that hold no bucket.
34
+ RESERVED_KEYS = [VERSIONS_KEY, V1_IMPORTED_KEY].freeze
28
35
  STRING_FLAVOR = "_key".freeze
29
36
  PUBID_FLAVOR = "_pubid".freeze
30
37
  # The legacy string form of a not-found entry (`not_found <date>`).
@@ -75,10 +82,15 @@ module Relaton
75
82
  @dir
76
83
  end
77
84
 
78
- # Remove every row and document.
85
+ # Remove every row and document, under one rows transaction (lock
86
+ # order rows, then docs). The v1 import marker is never deleted, not
87
+ # even for a moment, so a cleared cache is not filled again from
88
+ # `<dir>-v1.bak` by this or another process.
79
89
  def clear
80
- @rows.clear
81
- @docs.clear
90
+ @rows.adapter.transaction do
91
+ @rows.each_key { |k| @rows.delete k unless k == V1_IMPORTED_KEY }
92
+ @docs.clear
93
+ end
82
94
  @versions = {}
83
95
  end
84
96
 
@@ -118,9 +130,11 @@ module Relaton
118
130
  # @param value [String, Relaton::Db::NotFound, nil] document XML, a
119
131
  # not-found entry (also as `not_found <date>`); nil deletes the row
120
132
  # @param item_key [Pubid::Identifier, String, nil] the document's key
133
+ # @param etag [String, nil] the ETag of the API answer the document
134
+ # came from (relaton#535), kept on both rows; nil for a flavor answer
121
135
  # @return [String, Relaton::Db::NotFound, nil] value
122
136
  #
123
- def store(key, value, item_key: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
137
+ def store(key, value, item_key: nil, etag: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
124
138
  return delete(key) if value.nil?
125
139
 
126
140
  bucket, key = resolve key
@@ -137,9 +151,10 @@ module Relaton
137
151
  record_version item_bucket
138
152
  @rows.adapter.transaction do
139
153
  @docs.set file, value
140
- upsert item_bucket, new_row(item_key, CacheEntry::DOC, file, fetched)
154
+ upsert item_bucket,
155
+ new_row(item_key, CacheEntry::DOC, file, fetched, etag)
141
156
  unless canonical(key) == canonical(item_key)
142
- upsert bucket, new_row(key, CacheEntry::DOC, file, fetched)
157
+ upsert bucket, new_row(key, CacheEntry::DOC, file, fetched, etag)
143
158
  end
144
159
  end
145
160
  value
@@ -183,7 +198,7 @@ module Relaton
183
198
 
184
199
  found = other.lookup(key) or return
185
200
  row_id, row = found
186
- store row_id, other.read_entry(row)
201
+ store row_id, other.read_entry(row), etag: row["etag"]
187
202
  end
188
203
 
189
204
  # @param key [Pubid::Identifier, String]
@@ -192,6 +207,30 @@ module Relaton
192
207
  lookup(key)&.last&.dig("fetched")
193
208
  end
194
209
 
210
+ # @param key [Pubid::Identifier, String]
211
+ # @return [String, nil] the ETag of the API answer the entry came from
212
+ def etag(key)
213
+ lookup(key)&.last&.dig("etag")
214
+ end
215
+
216
+ #
217
+ # The document and the ETag to revalidate a key with, read from one
218
+ # row in one lookup, so the two belong to one answer. Only the key's
219
+ # own row counts: a dated key's lookup can also read a row that adds a
220
+ # language (`#lookup`), and that row's ETag was issued for another URL.
221
+ #
222
+ # @param key [Pubid::Identifier, String]
223
+ # @return [Array(String, String), nil] the document XML and its ETag
224
+ #
225
+ def revalidation(key)
226
+ row = lookup(key)&.last
227
+ own = canonical resolve(key).last
228
+ return unless row && row["etag"] && row_key(row) == own
229
+
230
+ xml = read_entry row
231
+ [xml, row["etag"]] if xml.is_a?(String)
232
+ end
233
+
195
234
  # An undated entry expires after 60 days, a dated document never. A
196
235
  # not_found entry expires after 60 days even for a dated query: the
197
236
  # document can be published later.
@@ -319,8 +358,7 @@ module Relaton
319
358
  # fetch, and redirects were dropped from the v2 model).
320
359
  def import_old_layout
321
360
  bak = "#{dir}-v1.bak"
322
- marker = File.join(dir, LAYOUT, ".v1-imported")
323
- return if File.exist?(marker) || !Dir.exist?(bak)
361
+ return if v1_imported? || !Dir.exist?(bak)
324
362
 
325
363
  imported = 0
326
364
  Dir.glob("#{bak}/**/*.xml").each do |f|
@@ -331,10 +369,17 @@ module Relaton
331
369
  self[key] = value
332
370
  imported += 1
333
371
  end
334
- FileUtils.touch marker
372
+ @rows.set V1_IMPORTED_KEY, Date.today.to_s
335
373
  Util.info "cache #{dir}: imported #{imported} entries from #{bak}"
336
374
  end
337
375
 
376
+ # An older relaton wrote the marker as a file; it still counts. `get`,
377
+ # not `exists?`: it takes the shared lock, so it waits for a write.
378
+ def v1_imported?
379
+ !@rows.get(V1_IMPORTED_KEY).nil? ||
380
+ File.exist?(File.join(dir, LAYOUT, V1_IMPORTED_FILE))
381
+ end
382
+
338
383
  # The wrapped string key of an old-layout document: its own primary
339
384
  # docidentifier under its flavor prefix, resolved through the same
340
385
  # `wrapped_key` path a string cache read takes. A value no parser reads
@@ -433,7 +478,7 @@ module Relaton
433
478
 
434
479
  def each_row
435
480
  @rows.each_key do |bucket|
436
- next if bucket == VERSIONS_KEY
481
+ next if RESERVED_KEYS.include? bucket
437
482
 
438
483
  read_bucket(bucket).each { |row| yield bucket, row }
439
484
  end
@@ -510,8 +555,9 @@ module Relaton
510
555
  end
511
556
  end
512
557
 
513
- def new_row(key, status, file, fetched)
514
- row = CacheEntry.new(status: status, file: file, fetched: fetched)
558
+ def new_row(key, status, file, fetched, etag = nil)
559
+ row = CacheEntry.new(status: status, file: file, fetched: fetched,
560
+ etag: etag)
515
561
  if key.is_a?(String) then row.key = key
516
562
  else row.id = canonical(key)
517
563
  end
@@ -15,7 +15,9 @@ module Relaton
15
15
  # `to_hash`) or, for a processor with no pubid class, by a string (`key`).
16
16
  # `file` names the document in the doc store; several rows can share one
17
17
  # file (a query row and the row of the document it returned). A
18
- # `not_found` row has no file.
18
+ # `not_found` row has no file. `etag` is the ETag of the api.relaton.org
19
+ # answer the document came from (relaton#535), for revalidation; a
20
+ # flavor answer, and a row an older relaton wrote, have none.
19
21
  class CacheEntry < Lutaml::Model::Serializable
20
22
  DOC = "doc".freeze
21
23
  NOT_FOUND = "not_found".freeze
@@ -25,6 +27,7 @@ module Relaton
25
27
  attribute :status, :string
26
28
  attribute :file, :string
27
29
  attribute :fetched, :string
30
+ attribute :etag, :string
28
31
 
29
32
  key_value do
30
33
  map "id", to: :id
@@ -32,6 +35,7 @@ module Relaton
32
35
  map "status", to: :status
33
36
  map "file", to: :file
34
37
  map "fetched", to: :fetched
38
+ map "etag", to: :etag
35
39
  end
36
40
 
37
41
  def not_found?
@@ -1,3 +1,5 @@
1
+ require "uri"
2
+
1
3
  module Relaton
2
4
  class Db
3
5
  module Config
@@ -11,19 +13,62 @@ module Relaton
11
13
  end
12
14
 
13
15
  class Configuration
16
+ # Values of a `RELATON_*` switch (any case) that turn it on.
17
+ SWITCH_ON = %w[1 true yes on].freeze
14
18
  # Values of RELATON_OFFLINE that turn offline mode on.
15
- OFFLINE_ON = %w[1 true yes on].freeze
19
+ OFFLINE_ON = SWITCH_ON
20
+ DEFAULT_API_HOST = "https://api.relaton.org".freeze
16
21
 
17
22
  # @!attribute offline
18
23
  # @return [Boolean] true: every fetch is answered from the local
19
24
  # store only, and a miss raises Relaton::NotInLocalStoreError
20
25
  # (relaton#536). Defaults to RELATON_OFFLINE.
21
- attr_accessor :use_api, :api_host, :offline
26
+ #
27
+ # @!attribute use_api
28
+ # @return [Boolean] true: a cache miss asks api.relaton.org first,
29
+ # and the flavor answers what the API does not (relaton#535).
30
+ # Defaults to RELATON_USE_API, else false. The CLI's `--api` /
31
+ # `--no-api` sets it for one run.
32
+ # @!attribute api_host
33
+ # @return [String] the API's base URL. Defaults to RELATON_API_HOST,
34
+ # else https://api.relaton.org.
35
+ # @!attribute api_timeout
36
+ # @return [Numeric] open and read timeout of an API request, seconds
37
+ attr_accessor :use_api, :api_host, :api_timeout, :offline
22
38
 
23
39
  def initialize
24
- @use_api = false
25
- @api_host = "https://api.relaton.org"
26
- @offline = OFFLINE_ON.include?(ENV["RELATON_OFFLINE"].to_s.downcase)
40
+ @use_api = switch? "RELATON_USE_API"
41
+ @api_host = env_api_host || DEFAULT_API_HOST
42
+ @api_timeout = 10
43
+ @offline = switch? "RELATON_OFFLINE"
44
+ end
45
+
46
+ private
47
+
48
+ def switch?(name)
49
+ SWITCH_ON.include? ENV[name].to_s.downcase
50
+ end
51
+
52
+ # RELATON_API_HOST, when it is an http(s) URL with a host. A shell
53
+ # value is easy to write without its scheme (`api.example.org`), and
54
+ # the resolver would then ask a host-less URI; that is warned about,
55
+ # not used.
56
+ def env_api_host
57
+ host = ENV["RELATON_API_HOST"].to_s.strip
58
+ return if host.empty?
59
+
60
+ uri = URI.parse(host)
61
+ return host if uri.is_a?(URI::HTTP) && uri.host && !uri.host.empty?
62
+
63
+ invalid_api_host host
64
+ rescue URI::InvalidURIError
65
+ invalid_api_host host
66
+ end
67
+
68
+ def invalid_api_host(host)
69
+ Util.warn "RELATON_API_HOST #{host.inspect} is not an http(s) URL; " \
70
+ "using #{DEFAULT_API_HOST}"
71
+ nil
27
72
  end
28
73
  end
29
74
 
data/lib/relaton/db.rb CHANGED
@@ -287,26 +287,94 @@ module Relaton
287
287
  end
288
288
 
289
289
  # The flavor's `get` receives the parsed pubid when there is one
290
- # (relaton#205), else the String. The API request sends the String the
291
- # caller wrote.
292
- def fetch_doc(code, year, opts, processor, query = nil)
293
- if Db.configuration.use_api
294
- fetch_api(code, year, opts, processor, query)
295
- else processor.get(query || code, year, opts)
296
- end
290
+ # (relaton#205), else the String. With `use_api`, api.relaton.org is
291
+ # asked first (relaton#535), and the flavor answers whatever it does not.
292
+ #
293
+ # @param api [Hash] the API tier's state for one fetch: in,
294
+ # `:stale_etag`/`:stale` (a stale row's ETag and its document, to
295
+ # revalidate); out, `:etag` (the ETag of an API answer, nil for a
296
+ # flavor answer), which Db#fetch_entry stores with the row
297
+ def fetch_doc(code, year, opts, processor, query = nil, api: {})
298
+ api_answer(code, year, opts, processor, query, api) ||
299
+ processor.get(query || code, year, opts)
300
+ end
301
+
302
+ #
303
+ # The API's document for the query, or nil. The API is asked only for a
304
+ # query that has a pubid cache key, is not all-parts and has no
305
+ # publication date range: the API has no all-parts answer, and a range
306
+ # is not in the key. Its answer is kept only when the document's own
307
+ # cache key is the query's key: the API has no undated or edition logic
308
+ # (`ISO 19115-1` answers the 2014 record), and the flavor's rule for
309
+ # such a query is the one `Db` caches.
310
+ #
311
+ # A stale row the API wrote is revalidated with its ETag. A 304 answers
312
+ # the cached document, fetched today: `Cache#store` reads the row's
313
+ # `fetched` from the document, so the document is stored again, not
314
+ # only its row. It needs no identity check: the row was written from an
315
+ # answer that passed one, for this key.
316
+ #
317
+ def api_answer(code, year, opts, processor, query, api) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists
318
+ api[:etag] = nil
319
+ return unless Db.configuration.use_api
320
+ return if opts[:all_parts] || date_range?(opts)
321
+
322
+ key = query && processor.cache_key(code, year, opts, query)
323
+ return unless key.is_a?(::Pubid::Identifier)
324
+
325
+ answer = ApiResolver.fetch(processor, key, etag: api[:stale_etag])
326
+ return unless answer
327
+ return revalidated(processor, api, answer, key) if answer.not_modified
328
+
329
+ bib = answer.item
330
+ if api_identity?(bib, key, processor)
331
+ api[:etag] = answer.etag
332
+ return bib
333
+ end
334
+
335
+ Util.info "api.relaton.org: answered another document " \
336
+ "(#{bib.docidentifier.first&.content}); asking the flavor",
337
+ key: key.to_s
338
+ nil
297
339
  end
298
340
 
299
- def fetch_api(code, year, opts, processor, query = nil)
300
- url = "#{Db.configuration.api_host}" \
301
- "/api/v1/document?#{params(code, year, opts)}"
302
- rsp = Net::HTTP.get_response URI(url)
303
- processor.from_xml rsp.body if rsp.code == "200"
304
- rescue Errno::ECONNREFUSED
305
- processor.get(query || code, year, opts)
341
+ # The stale document a 304 kept, fetched today, with its ETag. It is
342
+ # cached data: one that does not read back is a miss (the flavor
343
+ # answers), whatever the model raises for it.
344
+ def revalidated(processor, api, answer, key)
345
+ bib = processor.from_xml api[:stale]
346
+ bib.fetched = Date.today.to_s
347
+ api[:etag] = answer.etag
348
+ bib
349
+ rescue StandardError => e
350
+ Util.info "api.relaton.org: cached document unreadable " \
351
+ "(#{e.class}); asking the flavor", key: key.to_s
352
+ nil
353
+ end
354
+
355
+ # The API state Db#fetch_doc starts from: a stale document that the API
356
+ # answered carries the ETag to revalidate it with. Both come from one
357
+ # row of the stale cache (Cache#revalidation), read again here: the
358
+ # stale entry `with_stale` read is not used, since another worker may
359
+ # have replaced the row since, and its ETag would then confirm an older
360
+ # document.
361
+ def api_state(args)
362
+ return {} unless args[:stale].is_a?(String) && args[:stale_db]
363
+
364
+ xml, etag = @semaphore.synchronize do
365
+ args[:stale_db].revalidation(args[:id])
366
+ end
367
+ etag ? { stale_etag: etag, stale: xml } : {}
306
368
  end
307
369
 
308
- def params(code, year, opts)
309
- opts.merge(code: code, year: year).map { |k, v| "#{k}=#{v}" }.join "&"
370
+ # The API's document is data: an identifier the flavor cannot key is a
371
+ # mismatch, whatever the key raises for it.
372
+ def api_identity?(bib, key, processor)
373
+ item_key(bib, @registry.processors.key(processor)) == key
374
+ rescue StandardError => e
375
+ Util.info "api.relaton.org: unkeyable answer (#{e.class}); " \
376
+ "asking the flavor", key: key.to_s
377
+ false
310
378
  end
311
379
 
312
380
  def search_xml(processor, xml, text, edition, year)
@@ -600,9 +668,10 @@ module Relaton
600
668
  #
601
669
  def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
602
670
  processor = @registry[stdclass]
671
+ api = api_state args
603
672
  bib = begin
604
673
  net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
605
- args[:query])
674
+ args[:query], api: api)
606
675
  rescue Relaton::RequestError => e
607
676
  raise unless args[:stale].is_a?(String)
608
677
 
@@ -621,7 +690,8 @@ module Relaton
621
690
  @semaphore.synchronize do
622
691
  args[:db].expire args[:id], year if args[:stale]
623
692
  if args[:stale] || refresh || !args[:db].read(args[:id])
624
- save_bib args[:db], args[:id], bib, entry, stdclass
693
+ save_bib args[:db], args[:id], bib, entry, stdclass,
694
+ etag: api[:etag]
625
695
  end
626
696
  end
627
697
  entry
@@ -632,9 +702,11 @@ module Relaton
632
702
  # when the query key differs from it (an undated or incomplete query),
633
703
  # the query gets a row that points to the same document.
634
704
  #
635
- def save_bib(db, key, bib, entry, stdclass)
705
+ # `etag` is the ETag of the API answer (Db#fetch_doc), nil for a flavor
706
+ # answer.
707
+ def save_bib(db, key, bib, entry, stdclass, etag: nil) # rubocop:disable Metrics/ParameterLists
636
708
  item = bib.respond_to?(:docidentifier) && item_key(bib, stdclass)
637
- db.store key, entry, item_key: item || nil
709
+ db.store key, entry, item_key: item || nil, etag: etag
638
710
  end
639
711
 
640
712
  #
@@ -686,12 +758,13 @@ module Relaton
686
758
 
687
759
  # @param query [Pubid::Identifier, nil] what `get` receives in place of
688
760
  # the String +code+ (Db#query_and_key)
689
- def net_retry(code, year, opts, processor, retries, query = nil)
690
- fetch_doc code, year, opts, processor, query
761
+ # @param api [Hash] the API tier's state (Db#fetch_doc)
762
+ def net_retry(code, year, opts, processor, retries, query = nil, api: {}) # rubocop:disable Metrics/ParameterLists
763
+ fetch_doc code, year, opts, processor, query, api: api
691
764
  rescue Relaton::RequestError => e
692
765
  raise e unless retries > 1
693
766
 
694
- net_retry(code, year, opts, processor, retries - 1, query)
767
+ net_retry(code, year, opts, processor, retries - 1, query, api: api)
695
768
  end
696
769
 
697
770
  def bib_entry(bib)
@@ -782,6 +855,7 @@ end
782
855
 
783
856
  require_relative "db/util"
784
857
  require_relative "db/config"
858
+ require_relative "db/api_resolver"
785
859
  require_relative "db/workers_pool"
786
860
  require_relative "db/cache"
787
861
  require_relative "db/registry"
@@ -296,14 +296,15 @@ module Relaton
296
296
  "T-SP-OB.#{ref.number}-#{ref.year}" if bulletin?
297
297
  end
298
298
 
299
- # The docidentifier (year only, no month or day). A bulletin keeps its
300
- # sector even when the reference is the sector-less `ITU OB No. 1096`.
299
+ # The docidentifier (year only, no month or day). A bulletin carries no
300
+ # sector even when the reference is the TSB spelling `ITU-T OB.1096`
301
+ # (pubid#696); `Scraper#bureau` still makes TSB its author.
301
302
  #
302
303
  # @return [String]
303
304
  def publication_code
304
305
  return "ITU-R RR (#{ref.year})" if radio_regulations?
305
306
 
306
- "ITU-T OB.#{ref.number} (#{ref.year})"
307
+ "ITU OB No. #{ref.number} (#{ref.year})"
307
308
  end
308
309
 
309
310
  def radio_regulations?
@@ -141,30 +141,70 @@ module Relaton
141
141
  def fetch_publisher_contributors
142
142
  return [] unless hit.hit[:code]
143
143
 
144
- abbrev = hit.hit[:code].sub(/-\w\s.*/, "")
145
- case abbrev
146
- when "ITU"
147
- name = "International Telecommunication Union"
148
- url = "www.itu.int"
149
- end
150
- org = Relaton::Bib::Organization.new(
151
- name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
152
- abbreviation: Relaton::Bib::LocalizedString.new(content: abbrev),
153
- uri: [Relaton::Bib::Uri.new(content: url)],
154
- )
155
144
  role = Relaton::Bib::Contributor::Role.new(type: "publisher")
156
- [Relaton::Bib::Contributor.new(organization: org, role: [role])]
145
+ [Relaton::Bib::Contributor.new(organization: publisher_org,
146
+ role: [role])]
147
+ end
148
+
149
+ # The publishing organization. Every ITU code names ITU, so a code pubid
150
+ # cannot parse is still ITU's.
151
+ #
152
+ # @return [Relaton::Bib::Organization]
153
+ def publisher_org
154
+ Relaton::Bib::Organization.new(
155
+ name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
156
+ abbreviation: Relaton::Bib::LocalizedString.new(
157
+ content: code_pubid&.publisher || "ITU",
158
+ ),
159
+ uri: [Relaton::Bib::Uri.new(content: "www.itu.int")],
160
+ )
161
+ end
162
+
163
+ # The pubid parse of the hit's (first) code. nil when the hit has no
164
+ # code, or when ITU spells it in a form pubid cannot identify.
165
+ #
166
+ # @return [Pubid::Itu::Identifiers::Base, nil]
167
+ def code_pubid
168
+ return @code_pubid if defined? @code_pubid
169
+
170
+ code = hit.hit[:code].to_s.split(" | ").first
171
+ @code_pubid = code && ::Pubid::Itu.parse(code)
172
+ rescue ::Pubid::Errors::ParseError
173
+ @code_pubid = nil
174
+ end
175
+
176
+ # The bureau letter (T, R or D) of the code: its sector, or the sector
177
+ # of the document a wrapper (an annex, an appendix) belongs to. An
178
+ # Operational Bulletin prints no sector (pubid#696), but TSB produces it.
179
+ #
180
+ # @return [String, nil]
181
+ def bureau
182
+ id = code_pubid
183
+ return "T" if id.is_a? ::Pubid::Itu::Identifiers::SpecialPublication
184
+
185
+ while id
186
+ return id.sector.sector if id.sector
187
+
188
+ id = wrapped(id)
189
+ end
190
+ end
191
+
192
+ # The document a wrapper identifier belongs to, or nil.
193
+ def wrapped(id)
194
+ base = id.base if id.respond_to?(:base)
195
+ base if base.is_a?(::Pubid::Identifier)
157
196
  end
158
197
 
159
198
  # @return [Relaton::Bib::Contributor, nil]
160
199
  def fetch_editorial_contributor
161
200
  wg_name = parser.fetch_workgroup
162
- bureau = hit.hit[:code]&.match(/(?<=-)\w/)&.to_s
163
201
  return unless bureau
164
202
 
165
203
  org = Relaton::Bib::Organization.new(
166
204
  name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
167
- abbreviation: Relaton::Bib::LocalizedString.new(content: "ITU-#{bureau.upcase}"),
205
+ abbreviation: Relaton::Bib::LocalizedString.new(
206
+ content: "ITU-#{bureau}",
207
+ ),
168
208
  subdivision: group_subdivision(wg_name),
169
209
  )
170
210
  role = Relaton::Bib::Contributor::Role.new(
@@ -193,18 +233,7 @@ module Relaton
193
233
 
194
234
  # @return [Array<Relaton::Bib::Copyright>]
195
235
  def fetch_copyright
196
- abbreviation = hit.hit[:code].match(/^[^-]+/).to_s
197
- case abbreviation
198
- when "ITU"
199
- name = "International Telecommunication Union"
200
- url = "www.itu.int"
201
- end
202
- org = Relaton::Bib::Organization.new(
203
- name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
204
- abbreviation: Relaton::Bib::LocalizedString.new(content: abbreviation),
205
- uri: [Relaton::Bib::Uri.new(content: url)],
206
- )
207
- owner = Relaton::Bib::ContributionInfo.new(organization: org)
236
+ owner = Relaton::Bib::ContributionInfo.new(organization: publisher_org)
208
237
  year = parser.doc_date&.match(/\d{4}/)&.to_s
209
238
  [Relaton::Bib::Copyright.new(from: year, owner: [owner])]
210
239
  end
@@ -1,3 +1,3 @@
1
1
  module Relaton
2
- VERSION = "3.1.3".freeze
2
+ VERSION = "3.1.4".freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: relaton
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.3
4
+ version: 3.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-11-13 00:00:00.000000000 Z
11
+ date: 2026-11-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
@@ -520,14 +520,14 @@ dependencies:
520
520
  requirements:
521
521
  - - "~>"
522
522
  - !ruby/object:Gem::Version
523
- version: 2.0.22
523
+ version: 2.0.25
524
524
  type: :runtime
525
525
  prerelease: false
526
526
  version_requirements: !ruby/object:Gem::Requirement
527
527
  requirements:
528
528
  - - "~>"
529
529
  - !ruby/object:Gem::Version
530
- version: 2.0.22
530
+ version: 2.0.25
531
531
  - !ruby/object:Gem::Dependency
532
532
  name: yeptris
533
533
  requirement: !ruby/object:Gem::Requirement
@@ -821,6 +821,7 @@ files:
821
821
  - lib/relaton/cie/util.rb
822
822
  - lib/relaton/cli.rb
823
823
  - lib/relaton/cli/base_convertor.rb
824
+ - lib/relaton/cli/collection_store.rb
824
825
  - lib/relaton/cli/command.rb
825
826
  - lib/relaton/cli/data_fetcher.rb
826
827
  - lib/relaton/cli/frontend/dist/404.html
@@ -887,6 +888,7 @@ files:
887
888
  - lib/relaton/core/request_error.rb
888
889
  - lib/relaton/core/workers_pool.rb
889
890
  - lib/relaton/db.rb
891
+ - lib/relaton/db/api_resolver.rb
890
892
  - lib/relaton/db/cache.rb
891
893
  - lib/relaton/db/cache_entry.rb
892
894
  - lib/relaton/db/config.rb