relaton 3.1.1 → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +5 -3
- data/lib/relaton/3gpp.rb +1 -1
- data/lib/relaton/bib/converter/csl/from_csl.rb +195 -0
- data/lib/relaton/bib/converter/csl.rb +13 -1
- data/lib/relaton/bib/model/docidentifier.rb +66 -5
- data/lib/relaton/calconnect/hit_collection.rb +1 -1
- data/lib/relaton/cli/command.rb +3 -1
- data/lib/relaton/cli/frontend_assets.rb +7 -6
- data/lib/relaton/cli/subcommand_collection.rb +5 -1
- data/lib/relaton/core/request_error.rb +18 -0
- data/lib/relaton/db/config.rb +9 -1
- data/lib/relaton/db/workers_pool.rb +11 -1
- data/lib/relaton/db.rb +160 -58
- data/lib/relaton/ecma/bibliography.rb +2 -2
- data/lib/relaton/etsi/data_fetcher.rb +16 -6
- data/lib/relaton/iala/bibliography.rb +1 -1
- data/lib/relaton/iana.rb +1 -1
- data/lib/relaton/iec/data_parser.rb +41 -2
- data/lib/relaton/itu/README.adoc +1 -1
- data/lib/relaton/itu/data_crawler_r.rb +1 -1
- data/lib/relaton/jis/README.adoc +2 -0
- data/lib/relaton/jis/data_fetcher.rb +74 -13
- data/lib/relaton/jis/hit_collection.rb +2 -1
- data/lib/relaton/jis/processor.rb +1 -1
- data/lib/relaton/jis/scraper.rb +46 -5
- data/lib/relaton/oasis/bibliography.rb +1 -1
- data/lib/relaton/ogc/hit_collection.rb +1 -1
- data/lib/relaton/ogc.rb +1 -1
- data/lib/relaton/plateau/data_fetcher.rb +10 -1
- data/lib/relaton/plateau/handbook_parser.rb +38 -2
- data/lib/relaton/plateau/hit_collection.rb +20 -5
- data/lib/relaton/sae.rb +1 -1
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/bibliography.rb +1 -1
- data/lib/relaton/xsf/bibliography.rb +1 -1
- data/lib/relaton/xsf.rb +1 -1
- metadata +35 -48
- data/lib/relaton/3gpp/CLAUDE.md +0 -218
- data/lib/relaton/adobe/CLAUDE.md +0 -104
- data/lib/relaton/bib/CLAUDE.md +0 -150
- data/lib/relaton/bipm/CLAUDE.md +0 -187
- data/lib/relaton/bsi/CLAUDE.md +0 -35
- data/lib/relaton/calconnect/CLAUDE.md +0 -177
- data/lib/relaton/ccsds/CLAUDE.md +0 -74
- data/lib/relaton/cen/CLAUDE.md +0 -219
- data/lib/relaton/cie/CLAUDE.md +0 -90
- data/lib/relaton/core/CLAUDE.md +0 -140
- data/lib/relaton/db/CLAUDE.md +0 -385
- data/lib/relaton/doi/CLAUDE.md +0 -105
- data/lib/relaton/easc/CLAUDE.md +0 -85
- data/lib/relaton/ecma/CLAUDE.md +0 -217
- data/lib/relaton/etsi/CLAUDE.md +0 -219
- data/lib/relaton/evs/CLAUDE.md +0 -120
- data/lib/relaton/gb/CLAUDE.md +0 -47
- data/lib/relaton/gost/CLAUDE.md +0 -100
- data/lib/relaton/iala/CLAUDE.md +0 -97
- data/lib/relaton/iana/CLAUDE.md +0 -105
- data/lib/relaton/iec/CLAUDE.md +0 -13
- data/lib/relaton/ieee/CLAUDE.md +0 -202
- data/lib/relaton/ietf/CLAUDE.md +0 -382
- data/lib/relaton/iho/CLAUDE.md +0 -76
- data/lib/relaton/index/CLAUDE.md +0 -208
- data/lib/relaton/isbn/CLAUDE.md +0 -47
- data/lib/relaton/iso/CLAUDE.md +0 -38
- data/lib/relaton/itu/CLAUDE.md +0 -685
- data/lib/relaton/jcgm/CLAUDE.md +0 -107
- data/lib/relaton/jis/CLAUDE.md +0 -64
- data/lib/relaton/logger/CLAUDE.md +0 -37
- data/lib/relaton/nist/CLAUDE.md +0 -194
- data/lib/relaton/oasis/CLAUDE.md +0 -251
- data/lib/relaton/ogc/CLAUDE.md +0 -242
- data/lib/relaton/oiml/CLAUDE.md +0 -47
- data/lib/relaton/omg/CLAUDE.md +0 -137
- data/lib/relaton/plateau/CLAUDE.md +0 -126
- data/lib/relaton/sae/CLAUDE.md +0 -91
- data/lib/relaton/sdo/CLAUDE.md +0 -81
- data/lib/relaton/un/CLAUDE.md +0 -91
- data/lib/relaton/w3c/CLAUDE.md +0 -369
- data/lib/relaton/xsf/CLAUDE.md +0 -189
data/lib/relaton/db.rb
CHANGED
|
@@ -65,21 +65,25 @@ module Relaton
|
|
|
65
65
|
# @return [nil, RelatonBib::BibliographicItem, ...]
|
|
66
66
|
# @raise [Relaton::UnknownReferenceError] no flavor recognizes the
|
|
67
67
|
# reference (see Registry#route)
|
|
68
|
+
# @raise [Relaton::NotInLocalStoreError] offline mode, and the local
|
|
69
|
+
# store has no entry for the reference (relaton#536)
|
|
68
70
|
##
|
|
69
71
|
def fetch(text, year = nil, opts = {}) # rubocop:disable Metrics/MethodLength
|
|
70
72
|
reference = text.strip
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
73
|
+
offline_answer(reference, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
|
|
74
|
+
stdclass, pubid = @registry.route(reference)
|
|
75
|
+
processor = @registry[stdclass]
|
|
76
|
+
ref = if processor.respond_to?(:urn_to_code)
|
|
77
|
+
processor.urn_to_code(reference)&.first
|
|
78
|
+
else reference
|
|
79
|
+
end
|
|
80
|
+
ref ||= reference
|
|
81
|
+
# The routed pubid is the parse of `reference`; a URN rewritten to a
|
|
82
|
+
# code is parsed again by the flavor.
|
|
83
|
+
pubid = nil unless ref == reference
|
|
84
|
+
result = combine_doc ref, year, opts, stdclass
|
|
85
|
+
result || fetch_routed(ref, year, opts, stdclass, pubid)
|
|
86
|
+
end
|
|
83
87
|
end
|
|
84
88
|
|
|
85
89
|
# @see Relaton::Db#fetch
|
|
@@ -124,7 +128,7 @@ module Relaton
|
|
|
124
128
|
threads = ENV["RELATON_FETCH_PARALLEL"]&.to_i || processor.threads
|
|
125
129
|
wp = WorkersPool.new(threads) do |args|
|
|
126
130
|
args[3].call fetch(*args[0..2])
|
|
127
|
-
rescue Relaton::RequestError => e
|
|
131
|
+
rescue Relaton::RequestError, Relaton::NotInLocalStoreError => e
|
|
128
132
|
args[3].call e
|
|
129
133
|
rescue StandardError => e
|
|
130
134
|
# An unparseable reference ("ISO Standards Tag Set" in a
|
|
@@ -137,6 +141,14 @@ module Relaton
|
|
|
137
141
|
Util.error "`#{args[0]}` -- #{e.message}"
|
|
138
142
|
end
|
|
139
143
|
args[3].call nil
|
|
144
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
145
|
+
# The completion callback is a contract: the consumer counts
|
|
146
|
+
# exactly one delivery per fetch, and a missing one blocks it
|
|
147
|
+
# forever (metanorma-standoc CI deadlock, 2026-11-13: an
|
|
148
|
+
# RSpec::Mocks::MockExpectationError -- an Exception, not a
|
|
149
|
+
# StandardError -- escaped the rescue above).
|
|
150
|
+
Util.error "`#{args[0]}` -- #{e.class}: #{e.message}"
|
|
151
|
+
args[3].call nil
|
|
140
152
|
end
|
|
141
153
|
@queues[stdclass] =
|
|
142
154
|
{ queue: SizedQueue.new(threads * 2), workers_pool: wp }
|
|
@@ -155,14 +167,19 @@ module Relaton
|
|
|
155
167
|
# @raise [Relaton::UnknownReferenceError] no flavor is named and none
|
|
156
168
|
# recognizes the reference
|
|
157
169
|
#
|
|
170
|
+
# @raise [Relaton::NotInLocalStoreError] offline mode, and the local
|
|
171
|
+
# store has no entry for the reference (relaton#536)
|
|
172
|
+
#
|
|
158
173
|
def fetch_std(code, year = nil, stdclass = nil, opts = {})
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
174
|
+
offline_answer(code, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
|
|
175
|
+
std = named_class(stdclass)
|
|
176
|
+
# A flavor the caller names is the one asked: no co-publisher.
|
|
177
|
+
next check_bibliocache(code, year, opts, std) if std
|
|
178
|
+
|
|
179
|
+
std, pubid = @registry.route(code)
|
|
180
|
+
fetch_routed(code, year, opts, std, pubid) ||
|
|
181
|
+
fetch_equivalent(code, year, opts)
|
|
182
|
+
end
|
|
166
183
|
end
|
|
167
184
|
|
|
168
185
|
# Cross-SDO equivalence as data: on a total miss, walk the equivalence
|
|
@@ -174,6 +191,9 @@ module Relaton
|
|
|
174
191
|
#
|
|
175
192
|
# @return [Relaton::Bib::ItemData, nil]
|
|
176
193
|
def fetch_equivalent(code, year, opts)
|
|
194
|
+
# `relaton/db` alone does not declare the `Relaton::Equivalence`
|
|
195
|
+
# autoload (`relaton.rb` does), so a fetch_std miss NameErrored there.
|
|
196
|
+
require_relative "equivalence"
|
|
177
197
|
Equivalence.related_for(code).each do |candidate|
|
|
178
198
|
Util.info "Not found; trying equivalent `#{candidate}`", key: code
|
|
179
199
|
std, pubid = @registry.route(candidate)
|
|
@@ -221,6 +241,24 @@ module Relaton
|
|
|
221
241
|
|
|
222
242
|
private
|
|
223
243
|
|
|
244
|
+
#
|
|
245
|
+
# Offline mode (Configuration#offline, relaton#536) answers every fetch
|
|
246
|
+
# from the local store only: the block runs with `fetch_db` set, so no
|
|
247
|
+
# path reaches a flavor's `get`, and a miss of the whole resolution
|
|
248
|
+
# (routed flavor, co-publishers, equivalents) raises. An explicit
|
|
249
|
+
# `fetch_db` keeps its nil answer.
|
|
250
|
+
#
|
|
251
|
+
# @yieldparam opts [Hash] the options to fetch with
|
|
252
|
+
# @raise [Relaton::NotInLocalStoreError]
|
|
253
|
+
#
|
|
254
|
+
def offline_answer(reference, opts)
|
|
255
|
+
return yield(opts) if opts[:fetch_db] || !Db.configuration.offline
|
|
256
|
+
|
|
257
|
+
yield(opts.merge(fetch_db: true)) or
|
|
258
|
+
raise NotInLocalStoreError.new(reference.to_s,
|
|
259
|
+
[@local_db, @db].compact.map(&:dir))
|
|
260
|
+
end
|
|
261
|
+
|
|
224
262
|
# True when the error is a flavor's grammar refusing to read the
|
|
225
263
|
# reference. Both are raised by the flavors' pubid/parslet parses; a
|
|
226
264
|
# reference they cannot read is not-found, not a failure (relaton#412).
|
|
@@ -489,30 +527,48 @@ module Relaton
|
|
|
489
527
|
return check_date_range(searchcode, id, year, opts, stdclass, query)
|
|
490
528
|
end
|
|
491
529
|
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
530
|
+
# The cache-only path reads, and never expires: a stale entry is
|
|
531
|
+
# still the local store's answer (relaton#536).
|
|
532
|
+
# A document in either cache answers over a not_found in the other.
|
|
533
|
+
if opts[:fetch_db]
|
|
534
|
+
entries = [db, altdb].compact.map { |c| c.read(id) }
|
|
535
|
+
return bib_retval(entries.find { |e| e.is_a?(String) }, stdclass)
|
|
536
|
+
end
|
|
495
537
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
else
|
|
505
|
-
return bib_retval(db.read(id), stdclass) if opts[:fetch_db]
|
|
538
|
+
@semaphore.synchronize { replace_stale(db, altdb, id, year) } if altdb
|
|
539
|
+
# The entry new_bib_entry answers, not a second read of `db`: a stale
|
|
540
|
+
# document served from `altdb` has no row in `db`.
|
|
541
|
+
entry = new_bib_entry(searchcode, year, opts, stdclass,
|
|
542
|
+
db: db, id: id, query: query, altdb: altdb)
|
|
543
|
+
@semaphore.synchronize { replace_stale(altdb, db, id, year) } if altdb
|
|
544
|
+
bib_retval(entry, stdclass)
|
|
545
|
+
end
|
|
506
546
|
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
547
|
+
# Replace the stale (or missing) row of `id` in `cache` with the valid
|
|
548
|
+
# row `other` holds. A valid row of `cache` is kept (local over global).
|
|
549
|
+
def replace_stale(cache, other, id, year)
|
|
550
|
+
return if cache.valid_entry?(id, year)
|
|
551
|
+
return unless other.valid_entry?(id, year)
|
|
552
|
+
|
|
553
|
+
cache.expire id, year
|
|
554
|
+
cache.clone_entry id, other
|
|
511
555
|
end
|
|
512
556
|
|
|
557
|
+
#
|
|
558
|
+
# A stale entry (Cache#valid_entry? is false) is refetched, but it is
|
|
559
|
+
# not deleted first: Db#fetch_entry serves it when the refetch fails
|
|
560
|
+
# with a transport error, and replaces it when the refetch answers.
|
|
561
|
+
# With no row of its own, the stale document of the other cache
|
|
562
|
+
# (`altdb:`) is the fallback. `no_cache` asks for a fresh answer, so it
|
|
563
|
+
# has no fallback.
|
|
564
|
+
#
|
|
513
565
|
def new_bib_entry(code, year, opts, stdclass, **args)
|
|
514
|
-
entry = @semaphore.synchronize
|
|
515
|
-
|
|
566
|
+
entry, valid = @semaphore.synchronize do
|
|
567
|
+
found = args[:db]&.read(args[:id])
|
|
568
|
+
[found, found && args[:db].valid_entry?(args[:id], year)]
|
|
569
|
+
end
|
|
570
|
+
if !entry || opts[:no_cache] || !valid
|
|
571
|
+
args = with_stale(args, entry) unless opts[:no_cache]
|
|
516
572
|
return fetch_entry(code, year, opts, stdclass, **args)
|
|
517
573
|
end
|
|
518
574
|
|
|
@@ -524,18 +580,47 @@ module Relaton
|
|
|
524
580
|
entry
|
|
525
581
|
end
|
|
526
582
|
|
|
527
|
-
|
|
583
|
+
# @return [Hash] args, with the `stale:` entry and the `stale_db:` cache
|
|
584
|
+
# that holds it, when there is one
|
|
585
|
+
def with_stale(args, entry)
|
|
586
|
+
stale_db = args[:db]
|
|
587
|
+
unless entry
|
|
588
|
+
stale_db = args[:altdb]
|
|
589
|
+
entry = @semaphore.synchronize { stale_db&.read(args[:id]) }
|
|
590
|
+
return args unless entry.is_a?(String)
|
|
591
|
+
end
|
|
592
|
+
args.merge(stale: entry, stale_db: stale_db)
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
#
|
|
596
|
+
# @option args [String, Relaton::Db::NotFound, nil] :stale the stale
|
|
597
|
+
# entry of the key: a transport failure answers a stale document with
|
|
598
|
+
# a warning (relaton#536), and an answer replaces it
|
|
599
|
+
# @option args [Relaton::Db::Cache] :stale_db the cache that holds it
|
|
600
|
+
#
|
|
601
|
+
def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
528
602
|
processor = @registry[stdclass]
|
|
529
|
-
bib =
|
|
530
|
-
|
|
603
|
+
bib = begin
|
|
604
|
+
net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
|
|
605
|
+
args[:query])
|
|
606
|
+
rescue Relaton::RequestError => e
|
|
607
|
+
raise unless args[:stale].is_a?(String)
|
|
608
|
+
|
|
609
|
+
Util.warn "#{e.message}; using the cached entry fetched " \
|
|
610
|
+
"#{args[:stale_db].fetched(args[:id])}", key: code
|
|
611
|
+
return args[:stale]
|
|
612
|
+
end
|
|
531
613
|
entry = bib_entry bib
|
|
532
614
|
return entry if args[:db].nil?
|
|
533
615
|
|
|
534
616
|
# `no_cache` refreshes a cached entry, but a failed fetch does not
|
|
535
|
-
# replace a cached document with a not-found entry.
|
|
617
|
+
# replace a valid cached document with a not-found entry. A stale
|
|
618
|
+
# entry is replaced by any answer; its row goes first, so a dated
|
|
619
|
+
# query's subset row does not outlive the refetch.
|
|
536
620
|
refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
|
|
537
621
|
@semaphore.synchronize do
|
|
538
|
-
|
|
622
|
+
args[:db].expire args[:id], year if args[:stale]
|
|
623
|
+
if args[:stale] || refresh || !args[:db].read(args[:id])
|
|
539
624
|
save_bib args[:db], args[:id], bib, entry, stdclass
|
|
540
625
|
end
|
|
541
626
|
end
|
|
@@ -553,18 +638,29 @@ module Relaton
|
|
|
553
638
|
end
|
|
554
639
|
|
|
555
640
|
#
|
|
556
|
-
# A publication date range selects among
|
|
557
|
-
#
|
|
558
|
-
#
|
|
559
|
-
#
|
|
641
|
+
# A publication date range selects among editions of the reference. The
|
|
642
|
+
# answer must depend on the citing document's cutoff, never on the order
|
|
643
|
+
# in which parallel workers happen to populate the cache: the edition
|
|
644
|
+
# pool visible to any one worker is whatever earlier workers fetched for
|
|
645
|
+
# their own cutoffs, so answering from that pool is nondeterministic
|
|
646
|
+
# (metanorma#659). A ranged answer is therefore cached under a key that
|
|
647
|
+
# names the range and a hit on it is authoritative; the unqualified
|
|
648
|
+
# query row is never written by a ranged fetch, so it keeps pointing at
|
|
649
|
+
# the latest edition.
|
|
560
650
|
#
|
|
561
|
-
def
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
651
|
+
def range_cache_key(key, opts)
|
|
652
|
+
bounds = [opts[:publication_date_before] && "before=#{opts[:publication_date_before]}",
|
|
653
|
+
opts[:publication_date_after] && "after=#{opts[:publication_date_after]}"]
|
|
654
|
+
.compact
|
|
655
|
+
bounds.empty? ? nil : "#{key}@range:#{bounds.join(',')}"
|
|
656
|
+
end
|
|
657
|
+
|
|
658
|
+
def check_date_range(code, key, year, opts, stdclass, query = nil)
|
|
659
|
+
qualified = range_cache_key(key, opts)
|
|
660
|
+
unless opts[:no_cache]
|
|
661
|
+
hit = [@local_db, @db].compact.filter_map { |c| c.read(qualified) }.first
|
|
662
|
+
return bib_retval(hit, stdclass) if hit
|
|
663
|
+
end
|
|
568
664
|
return if opts[:fetch_db]
|
|
569
665
|
|
|
570
666
|
processor = @registry[stdclass]
|
|
@@ -574,9 +670,15 @@ module Relaton
|
|
|
574
670
|
|
|
575
671
|
entry = bib_entry bib
|
|
576
672
|
item = item_key(bib, stdclass)
|
|
577
|
-
|
|
578
|
-
@
|
|
579
|
-
|
|
673
|
+
@semaphore.synchronize do
|
|
674
|
+
[@local_db, @db].compact.each do |c|
|
|
675
|
+
# A ranged answer that carries no edition of its own is the
|
|
676
|
+
# undated form: storing it would overwrite the unqualified query
|
|
677
|
+
# row (whose contract is the latest edition), so only a genuinely
|
|
678
|
+
# dated answer may write its own row.
|
|
679
|
+
link = item if item && item != key
|
|
680
|
+
c.store item, entry if link
|
|
681
|
+
c.store qualified, entry, item_key: link if qualified
|
|
580
682
|
end
|
|
581
683
|
end
|
|
582
684
|
bib_retval entry, stdclass
|
|
@@ -38,7 +38,7 @@ module Relaton
|
|
|
38
38
|
# answer is worse than none. By the same trade-off, a reference must now
|
|
39
39
|
# parse WHOLE: the old regex was unanchored at the end, so
|
|
40
40
|
# `ECMA-6 (draft)` silently resolved to ECMA-6 and now does not. See
|
|
41
|
-
#
|
|
41
|
+
# docs/ecma.md for the measured table.
|
|
42
42
|
#
|
|
43
43
|
# The rows are selected with pubid's subset match `pubid === row`, the
|
|
44
44
|
# default of `Index::Type#search`. An `edition` or a `volume` that the
|
|
@@ -73,7 +73,7 @@ module Relaton
|
|
|
73
73
|
# An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
|
|
74
74
|
# propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
|
|
75
75
|
# `"..." is not a recognized standards identifier`
|
|
76
|
-
# (`
|
|
76
|
+
# (`lib/relaton/cli/command.rb`), and `Db#fetch`
|
|
77
77
|
# logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
|
|
78
78
|
# Rescuing here would collapse "this identifier is malformed" into "no
|
|
79
79
|
# such document", leaving a caller unable to tell them apart.
|
|
@@ -16,7 +16,7 @@ module Relaton
|
|
|
16
16
|
# status flags below are a separate axis and already permissive, so this
|
|
17
17
|
# is what makes e.g. all three editions of `ETSI EN 319 142-1` retrievable
|
|
18
18
|
# instead of one. It also makes the crawl about 2.4 times larger — see
|
|
19
|
-
#
|
|
19
|
+
# docs/etsi.md before changing it back.
|
|
20
20
|
SOURCEURL = "https://www.etsi.org/custom/standardssearch/data.php?format=json&includeScope=1&" \
|
|
21
21
|
"page=%<page>s&search=&title=1&etsiNumber=1&content=1&version=1&onApproval=1&published=1&" \
|
|
22
22
|
"withdrawn=1&historical=1&isCurrent=1&superseded=1&startDate=1988-01-15&endDate=%<date>s&" \
|
|
@@ -166,17 +166,27 @@ module Relaton
|
|
|
166
166
|
|
|
167
167
|
def normalize(record)
|
|
168
168
|
{
|
|
169
|
-
"ETSI deliverable" => record["ETSI_DELIVERABLE"],
|
|
170
|
-
"title" => record["TITLE"],
|
|
169
|
+
"ETSI deliverable" => sanitize(record["ETSI_DELIVERABLE"]),
|
|
170
|
+
"title" => sanitize(record["TITLE"]),
|
|
171
171
|
"Details link" => "https://webapp.etsi.org/workprogram/Report_WorkItem.asp?WKI_ID=#{record['wki_id']}",
|
|
172
172
|
"PDF link" => "https://www.etsi.org/deliver/#{record['EDSpathname']}#{record['EDSPDFfilename']}",
|
|
173
173
|
"Status" => derive_status(record),
|
|
174
|
-
"Keywords" => record["Keywords"].to_s,
|
|
175
|
-
"Technical body" => record["TB"],
|
|
176
|
-
"Scope" => record["Scope"],
|
|
174
|
+
"Keywords" => sanitize(record["Keywords"]).to_s,
|
|
175
|
+
"Technical body" => sanitize(record["TB"]),
|
|
176
|
+
"Scope" => sanitize(record["Scope"]),
|
|
177
177
|
}
|
|
178
178
|
end
|
|
179
179
|
|
|
180
|
+
# The upstream ETSI metadata carries stray control characters — one
|
|
181
|
+
# 1998-era record ships a literal NUL inside "sequences", which the YAML
|
|
182
|
+
# dump surfaces as a raw \0 escape and which silently truncates the
|
|
183
|
+
# api.relaton.org D1 import stream (relaton-api-js#155,
|
|
184
|
+
# relaton-data-etsi#7). Non-structural C0 controls are dropped; \t, \n
|
|
185
|
+
# and \r stay — the Scope column is hard-wrapped upstream text.
|
|
186
|
+
def sanitize(value)
|
|
187
|
+
value&.gsub(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/, "")
|
|
188
|
+
end
|
|
189
|
+
|
|
180
190
|
def derive_status(record)
|
|
181
191
|
return "Withdrawn" if record["ACTION_TYPE"] == "WD"
|
|
182
192
|
|
|
@@ -120,7 +120,7 @@ module Relaton
|
|
|
120
120
|
# An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
|
|
121
121
|
# propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
|
|
122
122
|
# `"..." is not a recognized standards identifier`
|
|
123
|
-
# (`
|
|
123
|
+
# (`lib/relaton/cli/command.rb`), and `Db#fetch`
|
|
124
124
|
# logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
|
|
125
125
|
# Rescuing here would collapse "this identifier is malformed" into "no
|
|
126
126
|
# such document", leaving a caller unable to tell them apart.
|
data/lib/relaton/iana.rb
CHANGED
|
@@ -25,7 +25,7 @@ module Relaton
|
|
|
25
25
|
# The legacy string-keyed `index-v1` (whose `:id` was the bare
|
|
26
26
|
# `registry[/sub-registry]` slug) is no longer produced or read here; it is
|
|
27
27
|
# emitted by `relaton-data-iana`'s own `crawler.rb`, derived from `index-v2`,
|
|
28
|
-
# so released gem lines keep resolving. See
|
|
28
|
+
# so released gem lines keep resolving. See docs/iana.md.
|
|
29
29
|
INDEXFILE = "index-v2".freeze
|
|
30
30
|
# The machine index on the data repo's Pages site (relaton#206) —
|
|
31
31
|
# manifest + per-shard rows read by the consumer's index; the zip stays
|
|
@@ -5,6 +5,29 @@ module Relaton
|
|
|
5
5
|
|
|
6
6
|
DOMAIN = "https://webstore.iec.ch"
|
|
7
7
|
|
|
8
|
+
# The Harmonized API serves published records with a status WORD;
|
|
9
|
+
# the harmonized stage system (ISO/IEC Directives Part 1) wants the
|
|
10
|
+
# numeric code (relaton#519). 60.60 is "International Standard
|
|
11
|
+
# published". The four end-of-life words all sit at 95.99
|
|
12
|
+
# "Withdrawal of International Standard": by the time the API
|
|
13
|
+
# serves REVISED/REPLACED/MERGED the newer record is already
|
|
14
|
+
# published and this edition is withdrawn - 90.92 ("to be
|
|
15
|
+
# revised") is a review-DECISION state that never persists into a
|
|
16
|
+
# bibliographic record, and ISO's own corpus records superseded
|
|
17
|
+
# editions at 95.99 with no stage-90 records at all
|
|
18
|
+
# (metanorma-iso#1067 wants IEC to match ISO's treatment). The
|
|
19
|
+
# original word rides along in ext.stagename, so the mapping is
|
|
20
|
+
# reversible at render time. An unmapped word raises: the table is
|
|
21
|
+
# the contract, and a new API word is a spec change to make
|
|
22
|
+
# deliberately, not silently.
|
|
23
|
+
HARMONIZED_STATUS = {
|
|
24
|
+
"PUBLISHED" => %w[60 60],
|
|
25
|
+
"REVISED" => %w[95 99],
|
|
26
|
+
"REPLACED" => %w[95 99],
|
|
27
|
+
"WITHDRAWN" => %w[95 99],
|
|
28
|
+
"MERGED" => %w[95 99],
|
|
29
|
+
}.freeze
|
|
30
|
+
|
|
8
31
|
ATTRS = %i[
|
|
9
32
|
type docidentifier language script title date contributor
|
|
10
33
|
status edition abstract copyright source relation place ext
|
|
@@ -143,8 +166,15 @@ module Relaton
|
|
|
143
166
|
|
|
144
167
|
def status
|
|
145
168
|
result = begin
|
|
146
|
-
|
|
147
|
-
|
|
169
|
+
code = HARMONIZED_STATUS.fetch(@pub["status"]) do
|
|
170
|
+
raise ArgumentError,
|
|
171
|
+
"unknown Harmonized API status word: #{@pub["status"].inspect} - " \
|
|
172
|
+
"extend DataParser::HARMONIZED_STATUS"
|
|
173
|
+
end
|
|
174
|
+
Bib::Status.new(
|
|
175
|
+
stage: Bib::Status::Stage.new(content: code[0]),
|
|
176
|
+
substage: Bib::Status::Stage.new(content: code[1]),
|
|
177
|
+
)
|
|
148
178
|
end
|
|
149
179
|
@errors[:status] &&= result.nil?
|
|
150
180
|
result
|
|
@@ -353,11 +383,20 @@ module Relaton
|
|
|
353
383
|
flavor: "iec",
|
|
354
384
|
ics: ics,
|
|
355
385
|
price_code: @pub.dig("priceInfo", "priceCode"),
|
|
386
|
+
stagename: stagename,
|
|
356
387
|
)
|
|
357
388
|
@errors[:ext] &&= result.nil?
|
|
358
389
|
result
|
|
359
390
|
end
|
|
360
391
|
|
|
392
|
+
# The Harmonized API's status word, kept beside the numeric stage
|
|
393
|
+
# code it mapped to (relaton#519) - ext.stagename is where the IHO
|
|
394
|
+
# and ISO flavors keep the SDO's own stage vocabulary too.
|
|
395
|
+
def stagename
|
|
396
|
+
word = @pub["status"]
|
|
397
|
+
StageName.new(abbreviation: word, content: word) if word
|
|
398
|
+
end
|
|
399
|
+
|
|
361
400
|
#
|
|
362
401
|
# Parse structured identifier.
|
|
363
402
|
#
|
data/lib/relaton/itu/README.adoc
CHANGED
|
@@ -175,7 +175,7 @@ RelatonItu uses the relaton-logger gem for logging. By default, it logs to STDOU
|
|
|
175
175
|
|
|
176
176
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
177
177
|
|
|
178
|
-
The index test fixture (used by tests) is a trimmed copy of the published pubid-structured `index-v2.zip` (rows are `_type: pubid:itu:…` hashes) from the `main` branch of the combined https://github.com/relaton/relaton-data-itu[relaton-data-itu] repository, kept at `spec/fixtures/index-v2.zip`. Refresh it with the regeneration snippet in `
|
|
178
|
+
The index test fixture (used by tests) is a trimmed copy of the published pubid-structured `index-v2.zip` (rows are `_type: pubid:itu:…` hashes) from the `main` branch of the combined https://github.com/relaton/relaton-data-itu[relaton-data-itu] repository, kept at `spec/fixtures/index-v2.zip`. Refresh it with the regeneration snippet in `docs/itu.md` (it must deserialize under the pinned pubid).
|
|
179
179
|
|
|
180
180
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
181
181
|
|
|
@@ -41,7 +41,7 @@ module Relaton
|
|
|
41
41
|
# the rate is set by a shared `Relaton::Core::Pacer` — `delay:` is the
|
|
42
42
|
# minimum gap between request *starts*, so ITU's own latency counts toward it
|
|
43
43
|
# instead of being added on top. A `Relaton::Itu::Governor` gives the pool one
|
|
44
|
-
# shared cooldown for a WAF block. See `#get` and
|
|
44
|
+
# shared cooldown for a WAF block. See `#get` and docs/itu.md.
|
|
45
45
|
# The crawl is parallel **within** a series only: `DataFetcher#harvest_family`
|
|
46
46
|
# rescues per series and `DataMergeR` writes per series, and the data repo's
|
|
47
47
|
# collapse guard is calibrated on that granularity.
|
data/lib/relaton/jis/README.adoc
CHANGED
|
@@ -4,6 +4,8 @@ RelatonJis is a Ruby gem that implements the https://github.com/metanorma/metano
|
|
|
4
4
|
|
|
5
5
|
You can use it to retrieve metadata of JIS Standards from https://webdesk.jsa.or.jp, and access such metadata through the `Relaton::Jis::ItemData` object.
|
|
6
6
|
|
|
7
|
+
The flavor also retrieves WES standards of the Japan Welding Engineering Society (JWES), which webdesk.jsa.or.jp sells next to JIS: `Relaton::Jis::Bibliography.get "WES 7700-1:2019"`. A WES record has the docidentifier type `WES` and JWES as its author and publisher. Its doctype is the JIS value of its form (`japanese-industrial-standard`, `technical-report`, `technical-specification` or `amendment`), because the JIS grammar has no WES value.
|
|
8
|
+
|
|
7
9
|
== Usage
|
|
8
10
|
|
|
9
11
|
=== Search for standards using keywords
|
|
@@ -9,11 +9,20 @@ module Relaton
|
|
|
9
9
|
class DataFetcher < Core::DataFetcher
|
|
10
10
|
URL = "https://webdesk.jsa.or.jp/books/"
|
|
11
11
|
|
|
12
|
+
# The Webdesk lists a crawl reads, in this order: the JIS search, then
|
|
13
|
+
# the JWES group list (WES standards, relaton-jis#18). Webdesk keeps the
|
|
14
|
+
# open list in the session, and the list page and its paging
|
|
15
|
+
# (`W11M0070`) read that list. So the lists are read one after the
|
|
16
|
+
# other, and a session refresh opens the current list again.
|
|
17
|
+
LISTS = %w[JIS JWES].freeze
|
|
18
|
+
|
|
12
19
|
def initialize(output, format)
|
|
13
20
|
super
|
|
14
21
|
@queue = SizedQueue.new 10
|
|
15
22
|
@threads = create_thread_pool 5
|
|
16
23
|
@mutex = Mutex.new
|
|
24
|
+
# URLs of the documents that did not fetch; see #prune_unwritten.
|
|
25
|
+
@failed_docs = []
|
|
17
26
|
end
|
|
18
27
|
|
|
19
28
|
def log_error(msg)
|
|
@@ -72,30 +81,63 @@ module Relaton
|
|
|
72
81
|
else
|
|
73
82
|
Util.warn "URL: #{url}"
|
|
74
83
|
Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
|
|
84
|
+
@mutex.synchronize { @failed_docs << url }
|
|
75
85
|
end
|
|
76
86
|
else
|
|
77
87
|
save_doc bib, url
|
|
78
88
|
end
|
|
79
89
|
end
|
|
80
90
|
|
|
91
|
+
# relaton-data-jis crawls with `prune: true`: it keeps data/, and after
|
|
92
|
+
# #fetch, #prune_unwritten deletes each file the run did not write. So a
|
|
93
|
+
# list that does not open, or stops part way (#get_next_page), raises:
|
|
94
|
+
# the crawl job fails before the prune and publishes nothing.
|
|
81
95
|
def fetch(_source = nil)
|
|
82
|
-
|
|
96
|
+
failed = begin
|
|
97
|
+
LISTS.find { |list| !fetch_list(list) }
|
|
98
|
+
ensure
|
|
99
|
+
end_threads_and_wait
|
|
100
|
+
end
|
|
101
|
+
raise Error, "Webdesk list #{failed} did not open" if failed
|
|
83
102
|
|
|
84
|
-
resp = agent.get "#{URL}W11M0070/index"
|
|
85
|
-
parse_page resp
|
|
86
103
|
index.save
|
|
87
104
|
report_errors
|
|
88
105
|
end
|
|
89
106
|
|
|
107
|
+
# Queue every document of one list. False if the list does not open.
|
|
108
|
+
def fetch_list(list)
|
|
109
|
+
@list = list
|
|
110
|
+
@initial_time = nil
|
|
111
|
+
return false unless initial_post
|
|
112
|
+
|
|
113
|
+
parse_page agent.get("#{URL}W11M0070/index")
|
|
114
|
+
true
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Open the current list in the session, unless it was opened less than
|
|
118
|
+
# 10 minutes ago.
|
|
90
119
|
def initial_post
|
|
91
120
|
return true if @initial_time && Time.now - @initial_time < 600
|
|
92
121
|
|
|
122
|
+
status = @list == "JWES" ? open_group_list(@list) : open_jis_search
|
|
123
|
+
@initial_time = Time.now
|
|
124
|
+
status || Util.warn("No results found for #{@list}")
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def open_jis_search
|
|
93
128
|
body = { record: 0, dantai: "JIS", searchtype2: 1,
|
|
94
129
|
status_1: 1, status_2: 2 } # rubocop:disable Naming/VariableNumber
|
|
95
130
|
resp = agent.post "#{URL}W11M0270/index", body
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
131
|
+
Relaton::Json.parse_external(resp.body)["status"]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# A group list from the "domestic group publications" page. The link
|
|
135
|
+
# redirects to the list page, which states the count of documents.
|
|
136
|
+
def open_group_list(dantai)
|
|
137
|
+
page = agent.get "#{URL}W11S0260/getSearchAllListByLink" \
|
|
138
|
+
"?dantai_cd=#{dantai}&display=KOKUNAI"
|
|
139
|
+
script = page.at_xpath('//script[contains(.,"var count =")]')
|
|
140
|
+
script&.text.to_s[/var count = (\d+);/, 1].to_i.positive?
|
|
99
141
|
end
|
|
100
142
|
|
|
101
143
|
def agent
|
|
@@ -111,7 +153,6 @@ module Relaton
|
|
|
111
153
|
|
|
112
154
|
resp = get_next_page(offset)
|
|
113
155
|
end
|
|
114
|
-
end_threads_and_wait
|
|
115
156
|
end
|
|
116
157
|
|
|
117
158
|
def parse_offset(resp) # rubocop:disable Metrics/AbcSize
|
|
@@ -135,25 +176,44 @@ module Relaton
|
|
|
135
176
|
@count.to_i
|
|
136
177
|
end
|
|
137
178
|
|
|
179
|
+
# The next page of the current list. Raises when the page does not
|
|
180
|
+
# come, because a list that stops part way leaves its unread documents
|
|
181
|
+
# unwritten, and the prune would delete them.
|
|
138
182
|
def get_next_page(offset) # rubocop:disable Metrics/MethodLength
|
|
139
183
|
attempts = 0
|
|
140
184
|
begin
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
185
|
+
unless initial_post
|
|
186
|
+
raise Error, "Webdesk list #{@list} did not open again " \
|
|
187
|
+
"at offset #{offset}"
|
|
144
188
|
end
|
|
189
|
+
|
|
190
|
+
agent.post "#{URL}W11M0070/getAddList",
|
|
191
|
+
search_type: "JIS", offset: offset
|
|
192
|
+
rescue Error
|
|
193
|
+
raise
|
|
145
194
|
rescue StandardError => e
|
|
146
195
|
attempts += 1
|
|
147
196
|
if attempts < 5
|
|
148
197
|
sleep 2
|
|
149
198
|
retry
|
|
150
|
-
else
|
|
151
|
-
Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
|
|
152
199
|
end
|
|
200
|
+
raise Error, "Webdesk list #{@list} stopped at offset " \
|
|
201
|
+
"#{offset}: #{e.message}"
|
|
153
202
|
end
|
|
154
203
|
end
|
|
155
204
|
|
|
156
|
-
|
|
205
|
+
# Prune only after a complete crawl: a document that did not fetch was
|
|
206
|
+
# not written, and the prune would delete its file.
|
|
207
|
+
def prune_unwritten(min_ratio: 0.9)
|
|
208
|
+
unless @failed_docs.empty?
|
|
209
|
+
warn "prune skipped: #{@failed_docs.size} document(s) did not fetch"
|
|
210
|
+
return []
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
super
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def save_doc(bib, url) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
157
217
|
return unless bib
|
|
158
218
|
|
|
159
219
|
id = bib.docidentifier.find(&:primary).content
|
|
@@ -175,6 +235,7 @@ module Relaton
|
|
|
175
235
|
Util.warn "File #{output_file id} already exists; writing #{file} instead" if file != output_file(id)
|
|
176
236
|
@files << file
|
|
177
237
|
File.write file, serialize(bib), encoding: "UTF-8"
|
|
238
|
+
record_written file
|
|
178
239
|
pid = pubid id
|
|
179
240
|
index.add_or_update pid, file if pid
|
|
180
241
|
end
|
|
@@ -83,7 +83,8 @@ module Relaton
|
|
|
83
83
|
|
|
84
84
|
def create_relation(hit)
|
|
85
85
|
id = hit.pubid.to_s
|
|
86
|
-
|
|
86
|
+
type = id.start_with?("WES ") ? "WES" : "JIS"
|
|
87
|
+
docid = Docidentifier.new content: id, type: type, primary: true
|
|
87
88
|
bibitem = ItemData.new(
|
|
88
89
|
formattedref: Bib::Formattedref.new(content: id),
|
|
89
90
|
docidentifier: [docid],
|