relaton 3.1.2 → 3.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +13 -3
- data/lib/relaton/bib/converter/csl/from_csl.rb +195 -0
- data/lib/relaton/bib/converter/csl.rb +13 -1
- data/lib/relaton/bibcollection.rb +4 -1
- data/lib/relaton/cli/collection_store.rb +171 -0
- data/lib/relaton/cli/command.rb +30 -1
- data/lib/relaton/cli/subcommand_collection.rb +61 -64
- data/lib/relaton/cli/util.rb +6 -0
- data/lib/relaton/core/request_error.rb +18 -0
- data/lib/relaton/db/api_resolver.rb +115 -0
- data/lib/relaton/db/cache.rb +59 -13
- data/lib/relaton/db/cache_entry.rb +5 -1
- data/lib/relaton/db/config.rb +56 -3
- data/lib/relaton/db/workers_pool.rb +11 -1
- data/lib/relaton/db.rb +224 -65
- data/lib/relaton/etsi/data_fetcher.rb +15 -5
- data/lib/relaton/itu/hit_collection.rb +4 -3
- data/lib/relaton/itu/scraper.rb +55 -26
- data/lib/relaton/jis/README.adoc +2 -0
- data/lib/relaton/jis/data_fetcher.rb +74 -13
- data/lib/relaton/jis/hit_collection.rb +2 -1
- data/lib/relaton/jis/processor.rb +1 -1
- data/lib/relaton/jis/scraper.rb +46 -5
- data/lib/relaton/plateau/data_fetcher.rb +10 -1
- data/lib/relaton/plateau/handbook_parser.rb +8 -1
- data/lib/relaton/plateau/hit_collection.rb +19 -4
- data/lib/relaton/version.rb +1 -1
- metadata +7 -4
data/lib/relaton/db.rb
CHANGED
|
@@ -65,21 +65,25 @@ module Relaton
|
|
|
65
65
|
# @return [nil, RelatonBib::BibliographicItem, ...]
|
|
66
66
|
# @raise [Relaton::UnknownReferenceError] no flavor recognizes the
|
|
67
67
|
# reference (see Registry#route)
|
|
68
|
+
# @raise [Relaton::NotInLocalStoreError] offline mode, and the local
|
|
69
|
+
# store has no entry for the reference (relaton#536)
|
|
68
70
|
##
|
|
69
71
|
def fetch(text, year = nil, opts = {}) # rubocop:disable Metrics/MethodLength
|
|
70
72
|
reference = text.strip
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
73
|
+
offline_answer(reference, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
|
|
74
|
+
stdclass, pubid = @registry.route(reference)
|
|
75
|
+
processor = @registry[stdclass]
|
|
76
|
+
ref = if processor.respond_to?(:urn_to_code)
|
|
77
|
+
processor.urn_to_code(reference)&.first
|
|
78
|
+
else reference
|
|
79
|
+
end
|
|
80
|
+
ref ||= reference
|
|
81
|
+
# The routed pubid is the parse of `reference`; a URN rewritten to a
|
|
82
|
+
# code is parsed again by the flavor.
|
|
83
|
+
pubid = nil unless ref == reference
|
|
84
|
+
result = combine_doc ref, year, opts, stdclass
|
|
85
|
+
result || fetch_routed(ref, year, opts, stdclass, pubid)
|
|
86
|
+
end
|
|
83
87
|
end
|
|
84
88
|
|
|
85
89
|
# @see Relaton::Db#fetch
|
|
@@ -124,7 +128,7 @@ module Relaton
|
|
|
124
128
|
threads = ENV["RELATON_FETCH_PARALLEL"]&.to_i || processor.threads
|
|
125
129
|
wp = WorkersPool.new(threads) do |args|
|
|
126
130
|
args[3].call fetch(*args[0..2])
|
|
127
|
-
rescue Relaton::RequestError => e
|
|
131
|
+
rescue Relaton::RequestError, Relaton::NotInLocalStoreError => e
|
|
128
132
|
args[3].call e
|
|
129
133
|
rescue StandardError => e
|
|
130
134
|
# An unparseable reference ("ISO Standards Tag Set" in a
|
|
@@ -137,6 +141,14 @@ module Relaton
|
|
|
137
141
|
Util.error "`#{args[0]}` -- #{e.message}"
|
|
138
142
|
end
|
|
139
143
|
args[3].call nil
|
|
144
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
145
|
+
# The completion callback is a contract: the consumer counts
|
|
146
|
+
# exactly one delivery per fetch, and a missing one blocks it
|
|
147
|
+
# forever (metanorma-standoc CI deadlock, 2026-11-13: an
|
|
148
|
+
# RSpec::Mocks::MockExpectationError -- an Exception, not a
|
|
149
|
+
# StandardError -- escaped the rescue above).
|
|
150
|
+
Util.error "`#{args[0]}` -- #{e.class}: #{e.message}"
|
|
151
|
+
args[3].call nil
|
|
140
152
|
end
|
|
141
153
|
@queues[stdclass] =
|
|
142
154
|
{ queue: SizedQueue.new(threads * 2), workers_pool: wp }
|
|
@@ -155,14 +167,19 @@ module Relaton
|
|
|
155
167
|
# @raise [Relaton::UnknownReferenceError] no flavor is named and none
|
|
156
168
|
# recognizes the reference
|
|
157
169
|
#
|
|
170
|
+
# @raise [Relaton::NotInLocalStoreError] offline mode, and the local
|
|
171
|
+
# store has no entry for the reference (relaton#536)
|
|
172
|
+
#
|
|
158
173
|
def fetch_std(code, year = nil, stdclass = nil, opts = {})
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
174
|
+
offline_answer(code, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
|
|
175
|
+
std = named_class(stdclass)
|
|
176
|
+
# A flavor the caller names is the one asked: no co-publisher.
|
|
177
|
+
next check_bibliocache(code, year, opts, std) if std
|
|
178
|
+
|
|
179
|
+
std, pubid = @registry.route(code)
|
|
180
|
+
fetch_routed(code, year, opts, std, pubid) ||
|
|
181
|
+
fetch_equivalent(code, year, opts)
|
|
182
|
+
end
|
|
166
183
|
end
|
|
167
184
|
|
|
168
185
|
# Cross-SDO equivalence as data: on a total miss, walk the equivalence
|
|
@@ -174,6 +191,9 @@ module Relaton
|
|
|
174
191
|
#
|
|
175
192
|
# @return [Relaton::Bib::ItemData, nil]
|
|
176
193
|
def fetch_equivalent(code, year, opts)
|
|
194
|
+
# `relaton/db` alone does not declare the `Relaton::Equivalence`
|
|
195
|
+
# autoload (`relaton.rb` does), so a fetch_std miss NameErrored there.
|
|
196
|
+
require_relative "equivalence"
|
|
177
197
|
Equivalence.related_for(code).each do |candidate|
|
|
178
198
|
Util.info "Not found; trying equivalent `#{candidate}`", key: code
|
|
179
199
|
std, pubid = @registry.route(candidate)
|
|
@@ -221,6 +241,24 @@ module Relaton
|
|
|
221
241
|
|
|
222
242
|
private
|
|
223
243
|
|
|
244
|
+
#
|
|
245
|
+
# Offline mode (Configuration#offline, relaton#536) answers every fetch
|
|
246
|
+
# from the local store only: the block runs with `fetch_db` set, so no
|
|
247
|
+
# path reaches a flavor's `get`, and a miss of the whole resolution
|
|
248
|
+
# (routed flavor, co-publishers, equivalents) raises. An explicit
|
|
249
|
+
# `fetch_db` keeps its nil answer.
|
|
250
|
+
#
|
|
251
|
+
# @yieldparam opts [Hash] the options to fetch with
|
|
252
|
+
# @raise [Relaton::NotInLocalStoreError]
|
|
253
|
+
#
|
|
254
|
+
def offline_answer(reference, opts)
|
|
255
|
+
return yield(opts) if opts[:fetch_db] || !Db.configuration.offline
|
|
256
|
+
|
|
257
|
+
yield(opts.merge(fetch_db: true)) or
|
|
258
|
+
raise NotInLocalStoreError.new(reference.to_s,
|
|
259
|
+
[@local_db, @db].compact.map(&:dir))
|
|
260
|
+
end
|
|
261
|
+
|
|
224
262
|
# True when the error is a flavor's grammar refusing to read the
|
|
225
263
|
# reference. Both are raised by the flavors' pubid/parslet parses; a
|
|
226
264
|
# reference they cannot read is not-found, not a failure (relaton#412).
|
|
@@ -249,26 +287,94 @@ module Relaton
|
|
|
249
287
|
end
|
|
250
288
|
|
|
251
289
|
# The flavor's `get` receives the parsed pubid when there is one
|
|
252
|
-
# (relaton#205), else the String.
|
|
253
|
-
#
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
290
|
+
# (relaton#205), else the String. With `use_api`, api.relaton.org is
|
|
291
|
+
# asked first (relaton#535), and the flavor answers whatever it does not.
|
|
292
|
+
#
|
|
293
|
+
# @param api [Hash] the API tier's state for one fetch: in,
|
|
294
|
+
# `:stale_etag`/`:stale` (a stale row's ETag and its document, to
|
|
295
|
+
# revalidate); out, `:etag` (the ETag of an API answer, nil for a
|
|
296
|
+
# flavor answer), which Db#fetch_entry stores with the row
|
|
297
|
+
def fetch_doc(code, year, opts, processor, query = nil, api: {})
|
|
298
|
+
api_answer(code, year, opts, processor, query, api) ||
|
|
299
|
+
processor.get(query || code, year, opts)
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
#
|
|
303
|
+
# The API's document for the query, or nil. The API is asked only for a
|
|
304
|
+
# query that has a pubid cache key, is not all-parts and has no
|
|
305
|
+
# publication date range: the API has no all-parts answer, and a range
|
|
306
|
+
# is not in the key. Its answer is kept only when the document's own
|
|
307
|
+
# cache key is the query's key: the API has no undated or edition logic
|
|
308
|
+
# (`ISO 19115-1` answers the 2014 record), and the flavor's rule for
|
|
309
|
+
# such a query is the one `Db` caches.
|
|
310
|
+
#
|
|
311
|
+
# A stale row the API wrote is revalidated with its ETag. A 304 answers
|
|
312
|
+
# the cached document, fetched today: `Cache#store` reads the row's
|
|
313
|
+
# `fetched` from the document, so the document is stored again, not
|
|
314
|
+
# only its row. It needs no identity check: the row was written from an
|
|
315
|
+
# answer that passed one, for this key.
|
|
316
|
+
#
|
|
317
|
+
def api_answer(code, year, opts, processor, query, api) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists
|
|
318
|
+
api[:etag] = nil
|
|
319
|
+
return unless Db.configuration.use_api
|
|
320
|
+
return if opts[:all_parts] || date_range?(opts)
|
|
321
|
+
|
|
322
|
+
key = query && processor.cache_key(code, year, opts, query)
|
|
323
|
+
return unless key.is_a?(::Pubid::Identifier)
|
|
324
|
+
|
|
325
|
+
answer = ApiResolver.fetch(processor, key, etag: api[:stale_etag])
|
|
326
|
+
return unless answer
|
|
327
|
+
return revalidated(processor, api, answer, key) if answer.not_modified
|
|
328
|
+
|
|
329
|
+
bib = answer.item
|
|
330
|
+
if api_identity?(bib, key, processor)
|
|
331
|
+
api[:etag] = answer.etag
|
|
332
|
+
return bib
|
|
258
333
|
end
|
|
334
|
+
|
|
335
|
+
Util.info "api.relaton.org: answered another document " \
|
|
336
|
+
"(#{bib.docidentifier.first&.content}); asking the flavor",
|
|
337
|
+
key: key.to_s
|
|
338
|
+
nil
|
|
259
339
|
end
|
|
260
340
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
processor.from_xml
|
|
266
|
-
|
|
267
|
-
|
|
341
|
+
# The stale document a 304 kept, fetched today, with its ETag. It is
|
|
342
|
+
# cached data: one that does not read back is a miss (the flavor
|
|
343
|
+
# answers), whatever the model raises for it.
|
|
344
|
+
def revalidated(processor, api, answer, key)
|
|
345
|
+
bib = processor.from_xml api[:stale]
|
|
346
|
+
bib.fetched = Date.today.to_s
|
|
347
|
+
api[:etag] = answer.etag
|
|
348
|
+
bib
|
|
349
|
+
rescue StandardError => e
|
|
350
|
+
Util.info "api.relaton.org: cached document unreadable " \
|
|
351
|
+
"(#{e.class}); asking the flavor", key: key.to_s
|
|
352
|
+
nil
|
|
268
353
|
end
|
|
269
354
|
|
|
270
|
-
|
|
271
|
-
|
|
355
|
+
# The API state Db#fetch_doc starts from: a stale document that the API
|
|
356
|
+
# answered carries the ETag to revalidate it with. Both come from one
|
|
357
|
+
# row of the stale cache (Cache#revalidation), read again here: the
|
|
358
|
+
# stale entry `with_stale` read is not used, since another worker may
|
|
359
|
+
# have replaced the row since, and its ETag would then confirm an older
|
|
360
|
+
# document.
|
|
361
|
+
def api_state(args)
|
|
362
|
+
return {} unless args[:stale].is_a?(String) && args[:stale_db]
|
|
363
|
+
|
|
364
|
+
xml, etag = @semaphore.synchronize do
|
|
365
|
+
args[:stale_db].revalidation(args[:id])
|
|
366
|
+
end
|
|
367
|
+
etag ? { stale_etag: etag, stale: xml } : {}
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
# The API's document is data: an identifier the flavor cannot key is a
|
|
371
|
+
# mismatch, whatever the key raises for it.
|
|
372
|
+
def api_identity?(bib, key, processor)
|
|
373
|
+
item_key(bib, @registry.processors.key(processor)) == key
|
|
374
|
+
rescue StandardError => e
|
|
375
|
+
Util.info "api.relaton.org: unkeyable answer (#{e.class}); " \
|
|
376
|
+
"asking the flavor", key: key.to_s
|
|
377
|
+
false
|
|
272
378
|
end
|
|
273
379
|
|
|
274
380
|
def search_xml(processor, xml, text, edition, year)
|
|
@@ -489,30 +595,48 @@ module Relaton
|
|
|
489
595
|
return check_date_range(searchcode, id, year, opts, stdclass, query)
|
|
490
596
|
end
|
|
491
597
|
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
598
|
+
# The cache-only path reads, and never expires: a stale entry is
|
|
599
|
+
# still the local store's answer (relaton#536).
|
|
600
|
+
# A document in either cache answers over a not_found in the other.
|
|
601
|
+
if opts[:fetch_db]
|
|
602
|
+
entries = [db, altdb].compact.map { |c| c.read(id) }
|
|
603
|
+
return bib_retval(entries.find { |e| e.is_a?(String) }, stdclass)
|
|
604
|
+
end
|
|
495
605
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
else
|
|
505
|
-
return bib_retval(db.read(id), stdclass) if opts[:fetch_db]
|
|
606
|
+
@semaphore.synchronize { replace_stale(db, altdb, id, year) } if altdb
|
|
607
|
+
# The entry new_bib_entry answers, not a second read of `db`: a stale
|
|
608
|
+
# document served from `altdb` has no row in `db`.
|
|
609
|
+
entry = new_bib_entry(searchcode, year, opts, stdclass,
|
|
610
|
+
db: db, id: id, query: query, altdb: altdb)
|
|
611
|
+
@semaphore.synchronize { replace_stale(altdb, db, id, year) } if altdb
|
|
612
|
+
bib_retval(entry, stdclass)
|
|
613
|
+
end
|
|
506
614
|
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
615
|
+
# Replace the stale (or missing) row of `id` in `cache` with the valid
|
|
616
|
+
# row `other` holds. A valid row of `cache` is kept (local over global).
|
|
617
|
+
def replace_stale(cache, other, id, year)
|
|
618
|
+
return if cache.valid_entry?(id, year)
|
|
619
|
+
return unless other.valid_entry?(id, year)
|
|
620
|
+
|
|
621
|
+
cache.expire id, year
|
|
622
|
+
cache.clone_entry id, other
|
|
511
623
|
end
|
|
512
624
|
|
|
625
|
+
#
|
|
626
|
+
# A stale entry (Cache#valid_entry? is false) is refetched, but it is
|
|
627
|
+
# not deleted first: Db#fetch_entry serves it when the refetch fails
|
|
628
|
+
# with a transport error, and replaces it when the refetch answers.
|
|
629
|
+
# With no row of its own, the stale document of the other cache
|
|
630
|
+
# (`altdb:`) is the fallback. `no_cache` asks for a fresh answer, so it
|
|
631
|
+
# has no fallback.
|
|
632
|
+
#
|
|
513
633
|
def new_bib_entry(code, year, opts, stdclass, **args)
|
|
514
|
-
entry = @semaphore.synchronize
|
|
515
|
-
|
|
634
|
+
entry, valid = @semaphore.synchronize do
|
|
635
|
+
found = args[:db]&.read(args[:id])
|
|
636
|
+
[found, found && args[:db].valid_entry?(args[:id], year)]
|
|
637
|
+
end
|
|
638
|
+
if !entry || opts[:no_cache] || !valid
|
|
639
|
+
args = with_stale(args, entry) unless opts[:no_cache]
|
|
516
640
|
return fetch_entry(code, year, opts, stdclass, **args)
|
|
517
641
|
end
|
|
518
642
|
|
|
@@ -524,19 +648,50 @@ module Relaton
|
|
|
524
648
|
entry
|
|
525
649
|
end
|
|
526
650
|
|
|
527
|
-
|
|
651
|
+
# @return [Hash] args, with the `stale:` entry and the `stale_db:` cache
|
|
652
|
+
# that holds it, when there is one
|
|
653
|
+
def with_stale(args, entry)
|
|
654
|
+
stale_db = args[:db]
|
|
655
|
+
unless entry
|
|
656
|
+
stale_db = args[:altdb]
|
|
657
|
+
entry = @semaphore.synchronize { stale_db&.read(args[:id]) }
|
|
658
|
+
return args unless entry.is_a?(String)
|
|
659
|
+
end
|
|
660
|
+
args.merge(stale: entry, stale_db: stale_db)
|
|
661
|
+
end
|
|
662
|
+
|
|
663
|
+
#
|
|
664
|
+
# @option args [String, Relaton::Db::NotFound, nil] :stale the stale
|
|
665
|
+
# entry of the key: a transport failure answers a stale document with
|
|
666
|
+
# a warning (relaton#536), and an answer replaces it
|
|
667
|
+
# @option args [Relaton::Db::Cache] :stale_db the cache that holds it
|
|
668
|
+
#
|
|
669
|
+
def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
528
670
|
processor = @registry[stdclass]
|
|
529
|
-
|
|
530
|
-
|
|
671
|
+
api = api_state args
|
|
672
|
+
bib = begin
|
|
673
|
+
net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
|
|
674
|
+
args[:query], api: api)
|
|
675
|
+
rescue Relaton::RequestError => e
|
|
676
|
+
raise unless args[:stale].is_a?(String)
|
|
677
|
+
|
|
678
|
+
Util.warn "#{e.message}; using the cached entry fetched " \
|
|
679
|
+
"#{args[:stale_db].fetched(args[:id])}", key: code
|
|
680
|
+
return args[:stale]
|
|
681
|
+
end
|
|
531
682
|
entry = bib_entry bib
|
|
532
683
|
return entry if args[:db].nil?
|
|
533
684
|
|
|
534
685
|
# `no_cache` refreshes a cached entry, but a failed fetch does not
|
|
535
|
-
# replace a cached document with a not-found entry.
|
|
686
|
+
# replace a valid cached document with a not-found entry. A stale
|
|
687
|
+
# entry is replaced by any answer; its row goes first, so a dated
|
|
688
|
+
# query's subset row does not outlive the refetch.
|
|
536
689
|
refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
|
|
537
690
|
@semaphore.synchronize do
|
|
538
|
-
|
|
539
|
-
|
|
691
|
+
args[:db].expire args[:id], year if args[:stale]
|
|
692
|
+
if args[:stale] || refresh || !args[:db].read(args[:id])
|
|
693
|
+
save_bib args[:db], args[:id], bib, entry, stdclass,
|
|
694
|
+
etag: api[:etag]
|
|
540
695
|
end
|
|
541
696
|
end
|
|
542
697
|
entry
|
|
@@ -547,9 +702,11 @@ module Relaton
|
|
|
547
702
|
# when the query key differs from it (an undated or incomplete query),
|
|
548
703
|
# the query gets a row that points to the same document.
|
|
549
704
|
#
|
|
550
|
-
|
|
705
|
+
# `etag` is the ETag of the API answer (Db#fetch_doc), nil for a flavor
|
|
706
|
+
# answer.
|
|
707
|
+
def save_bib(db, key, bib, entry, stdclass, etag: nil) # rubocop:disable Metrics/ParameterLists
|
|
551
708
|
item = bib.respond_to?(:docidentifier) && item_key(bib, stdclass)
|
|
552
|
-
db.store key, entry, item_key: item || nil
|
|
709
|
+
db.store key, entry, item_key: item || nil, etag: etag
|
|
553
710
|
end
|
|
554
711
|
|
|
555
712
|
#
|
|
@@ -601,12 +758,13 @@ module Relaton
|
|
|
601
758
|
|
|
602
759
|
# @param query [Pubid::Identifier, nil] what `get` receives in place of
|
|
603
760
|
# the String +code+ (Db#query_and_key)
|
|
604
|
-
|
|
605
|
-
|
|
761
|
+
# @param api [Hash] the API tier's state (Db#fetch_doc)
|
|
762
|
+
def net_retry(code, year, opts, processor, retries, query = nil, api: {}) # rubocop:disable Metrics/ParameterLists
|
|
763
|
+
fetch_doc code, year, opts, processor, query, api: api
|
|
606
764
|
rescue Relaton::RequestError => e
|
|
607
765
|
raise e unless retries > 1
|
|
608
766
|
|
|
609
|
-
net_retry(code, year, opts, processor, retries - 1, query)
|
|
767
|
+
net_retry(code, year, opts, processor, retries - 1, query, api: api)
|
|
610
768
|
end
|
|
611
769
|
|
|
612
770
|
def bib_entry(bib)
|
|
@@ -697,6 +855,7 @@ end
|
|
|
697
855
|
|
|
698
856
|
require_relative "db/util"
|
|
699
857
|
require_relative "db/config"
|
|
858
|
+
require_relative "db/api_resolver"
|
|
700
859
|
require_relative "db/workers_pool"
|
|
701
860
|
require_relative "db/cache"
|
|
702
861
|
require_relative "db/registry"
|
|
@@ -166,17 +166,27 @@ module Relaton
|
|
|
166
166
|
|
|
167
167
|
def normalize(record)
|
|
168
168
|
{
|
|
169
|
-
"ETSI deliverable" => record["ETSI_DELIVERABLE"],
|
|
170
|
-
"title" => record["TITLE"],
|
|
169
|
+
"ETSI deliverable" => sanitize(record["ETSI_DELIVERABLE"]),
|
|
170
|
+
"title" => sanitize(record["TITLE"]),
|
|
171
171
|
"Details link" => "https://webapp.etsi.org/workprogram/Report_WorkItem.asp?WKI_ID=#{record['wki_id']}",
|
|
172
172
|
"PDF link" => "https://www.etsi.org/deliver/#{record['EDSpathname']}#{record['EDSPDFfilename']}",
|
|
173
173
|
"Status" => derive_status(record),
|
|
174
|
-
"Keywords" => record["Keywords"].to_s,
|
|
175
|
-
"Technical body" => record["TB"],
|
|
176
|
-
"Scope" => record["Scope"],
|
|
174
|
+
"Keywords" => sanitize(record["Keywords"]).to_s,
|
|
175
|
+
"Technical body" => sanitize(record["TB"]),
|
|
176
|
+
"Scope" => sanitize(record["Scope"]),
|
|
177
177
|
}
|
|
178
178
|
end
|
|
179
179
|
|
|
180
|
+
# The upstream ETSI metadata carries stray control characters — one
|
|
181
|
+
# 1998-era record ships a literal NUL inside "sequences", which the YAML
|
|
182
|
+
# dump surfaces as a raw \0 escape and which silently truncates the
|
|
183
|
+
# api.relaton.org D1 import stream (relaton-api-js#155,
|
|
184
|
+
# relaton-data-etsi#7). Non-structural C0 controls are dropped; \t, \n
|
|
185
|
+
# and \r stay — the Scope column is hard-wrapped upstream text.
|
|
186
|
+
def sanitize(value)
|
|
187
|
+
value&.gsub(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/, "")
|
|
188
|
+
end
|
|
189
|
+
|
|
180
190
|
def derive_status(record)
|
|
181
191
|
return "Withdrawn" if record["ACTION_TYPE"] == "WD"
|
|
182
192
|
|
|
@@ -296,14 +296,15 @@ module Relaton
|
|
|
296
296
|
"T-SP-OB.#{ref.number}-#{ref.year}" if bulletin?
|
|
297
297
|
end
|
|
298
298
|
|
|
299
|
-
# The docidentifier (year only, no month or day). A bulletin
|
|
300
|
-
# sector even when the reference is the
|
|
299
|
+
# The docidentifier (year only, no month or day). A bulletin carries no
|
|
300
|
+
# sector even when the reference is the TSB spelling `ITU-T OB.1096`
|
|
301
|
+
# (pubid#696); `Scraper#bureau` still makes TSB its author.
|
|
301
302
|
#
|
|
302
303
|
# @return [String]
|
|
303
304
|
def publication_code
|
|
304
305
|
return "ITU-R RR (#{ref.year})" if radio_regulations?
|
|
305
306
|
|
|
306
|
-
"ITU
|
|
307
|
+
"ITU OB No. #{ref.number} (#{ref.year})"
|
|
307
308
|
end
|
|
308
309
|
|
|
309
310
|
def radio_regulations?
|
data/lib/relaton/itu/scraper.rb
CHANGED
|
@@ -141,30 +141,70 @@ module Relaton
|
|
|
141
141
|
def fetch_publisher_contributors
|
|
142
142
|
return [] unless hit.hit[:code]
|
|
143
143
|
|
|
144
|
-
abbrev = hit.hit[:code].sub(/-\w\s.*/, "")
|
|
145
|
-
case abbrev
|
|
146
|
-
when "ITU"
|
|
147
|
-
name = "International Telecommunication Union"
|
|
148
|
-
url = "www.itu.int"
|
|
149
|
-
end
|
|
150
|
-
org = Relaton::Bib::Organization.new(
|
|
151
|
-
name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
|
|
152
|
-
abbreviation: Relaton::Bib::LocalizedString.new(content: abbrev),
|
|
153
|
-
uri: [Relaton::Bib::Uri.new(content: url)],
|
|
154
|
-
)
|
|
155
144
|
role = Relaton::Bib::Contributor::Role.new(type: "publisher")
|
|
156
|
-
[Relaton::Bib::Contributor.new(organization:
|
|
145
|
+
[Relaton::Bib::Contributor.new(organization: publisher_org,
|
|
146
|
+
role: [role])]
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The publishing organization. Every ITU code names ITU, so a code pubid
|
|
150
|
+
# cannot parse is still ITU's.
|
|
151
|
+
#
|
|
152
|
+
# @return [Relaton::Bib::Organization]
|
|
153
|
+
def publisher_org
|
|
154
|
+
Relaton::Bib::Organization.new(
|
|
155
|
+
name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
|
|
156
|
+
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
157
|
+
content: code_pubid&.publisher || "ITU",
|
|
158
|
+
),
|
|
159
|
+
uri: [Relaton::Bib::Uri.new(content: "www.itu.int")],
|
|
160
|
+
)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# The pubid parse of the hit's (first) code. nil when the hit has no
|
|
164
|
+
# code, or when ITU spells it in a form pubid cannot identify.
|
|
165
|
+
#
|
|
166
|
+
# @return [Pubid::Itu::Identifiers::Base, nil]
|
|
167
|
+
def code_pubid
|
|
168
|
+
return @code_pubid if defined? @code_pubid
|
|
169
|
+
|
|
170
|
+
code = hit.hit[:code].to_s.split(" | ").first
|
|
171
|
+
@code_pubid = code && ::Pubid::Itu.parse(code)
|
|
172
|
+
rescue ::Pubid::Errors::ParseError
|
|
173
|
+
@code_pubid = nil
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# The bureau letter (T, R or D) of the code: its sector, or the sector
|
|
177
|
+
# of the document a wrapper (an annex, an appendix) belongs to. An
|
|
178
|
+
# Operational Bulletin prints no sector (pubid#696), but TSB produces it.
|
|
179
|
+
#
|
|
180
|
+
# @return [String, nil]
|
|
181
|
+
def bureau
|
|
182
|
+
id = code_pubid
|
|
183
|
+
return "T" if id.is_a? ::Pubid::Itu::Identifiers::SpecialPublication
|
|
184
|
+
|
|
185
|
+
while id
|
|
186
|
+
return id.sector.sector if id.sector
|
|
187
|
+
|
|
188
|
+
id = wrapped(id)
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# The document a wrapper identifier belongs to, or nil.
|
|
193
|
+
def wrapped(id)
|
|
194
|
+
base = id.base if id.respond_to?(:base)
|
|
195
|
+
base if base.is_a?(::Pubid::Identifier)
|
|
157
196
|
end
|
|
158
197
|
|
|
159
198
|
# @return [Relaton::Bib::Contributor, nil]
|
|
160
199
|
def fetch_editorial_contributor
|
|
161
200
|
wg_name = parser.fetch_workgroup
|
|
162
|
-
bureau = hit.hit[:code]&.match(/(?<=-)\w/)&.to_s
|
|
163
201
|
return unless bureau
|
|
164
202
|
|
|
165
203
|
org = Relaton::Bib::Organization.new(
|
|
166
204
|
name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
|
|
167
|
-
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
205
|
+
abbreviation: Relaton::Bib::LocalizedString.new(
|
|
206
|
+
content: "ITU-#{bureau}",
|
|
207
|
+
),
|
|
168
208
|
subdivision: group_subdivision(wg_name),
|
|
169
209
|
)
|
|
170
210
|
role = Relaton::Bib::Contributor::Role.new(
|
|
@@ -193,18 +233,7 @@ module Relaton
|
|
|
193
233
|
|
|
194
234
|
# @return [Array<Relaton::Bib::Copyright>]
|
|
195
235
|
def fetch_copyright
|
|
196
|
-
|
|
197
|
-
case abbreviation
|
|
198
|
-
when "ITU"
|
|
199
|
-
name = "International Telecommunication Union"
|
|
200
|
-
url = "www.itu.int"
|
|
201
|
-
end
|
|
202
|
-
org = Relaton::Bib::Organization.new(
|
|
203
|
-
name: [Relaton::Bib::TypedLocalizedString.new(content: name)],
|
|
204
|
-
abbreviation: Relaton::Bib::LocalizedString.new(content: abbreviation),
|
|
205
|
-
uri: [Relaton::Bib::Uri.new(content: url)],
|
|
206
|
-
)
|
|
207
|
-
owner = Relaton::Bib::ContributionInfo.new(organization: org)
|
|
236
|
+
owner = Relaton::Bib::ContributionInfo.new(organization: publisher_org)
|
|
208
237
|
year = parser.doc_date&.match(/\d{4}/)&.to_s
|
|
209
238
|
[Relaton::Bib::Copyright.new(from: year, owner: [owner])]
|
|
210
239
|
end
|
data/lib/relaton/jis/README.adoc
CHANGED
|
@@ -4,6 +4,8 @@ RelatonJis is a Ruby gem that implements the https://github.com/metanorma/metano
|
|
|
4
4
|
|
|
5
5
|
You can use it to retrieve metadata of JIS Standards from https://webdesk.jsa.or.jp, and access such metadata through the `Relaton::Jis::ItemData` object.
|
|
6
6
|
|
|
7
|
+
The flavor also retrieves WES standards of the Japan Welding Engineering Society (JWES), which webdesk.jsa.or.jp sells next to JIS: `Relaton::Jis::Bibliography.get "WES 7700-1:2019"`. A WES record has the docidentifier type `WES` and JWES as its author and publisher. Its doctype is the JIS value of its form (`japanese-industrial-standard`, `technical-report`, `technical-specification` or `amendment`), because the JIS grammar has no WES value.
|
|
8
|
+
|
|
7
9
|
== Usage
|
|
8
10
|
|
|
9
11
|
=== Search for standards using keywords
|