relaton 3.1.1 → 3.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/README.adoc +5 -3
  3. data/lib/relaton/3gpp.rb +1 -1
  4. data/lib/relaton/bib/converter/csl/from_csl.rb +195 -0
  5. data/lib/relaton/bib/converter/csl.rb +13 -1
  6. data/lib/relaton/bib/model/docidentifier.rb +66 -5
  7. data/lib/relaton/calconnect/hit_collection.rb +1 -1
  8. data/lib/relaton/cli/command.rb +3 -1
  9. data/lib/relaton/cli/frontend_assets.rb +7 -6
  10. data/lib/relaton/cli/subcommand_collection.rb +5 -1
  11. data/lib/relaton/core/request_error.rb +18 -0
  12. data/lib/relaton/db/config.rb +9 -1
  13. data/lib/relaton/db/workers_pool.rb +11 -1
  14. data/lib/relaton/db.rb +160 -58
  15. data/lib/relaton/ecma/bibliography.rb +2 -2
  16. data/lib/relaton/etsi/data_fetcher.rb +16 -6
  17. data/lib/relaton/iala/bibliography.rb +1 -1
  18. data/lib/relaton/iana.rb +1 -1
  19. data/lib/relaton/iec/data_parser.rb +41 -2
  20. data/lib/relaton/itu/README.adoc +1 -1
  21. data/lib/relaton/itu/data_crawler_r.rb +1 -1
  22. data/lib/relaton/jis/README.adoc +2 -0
  23. data/lib/relaton/jis/data_fetcher.rb +74 -13
  24. data/lib/relaton/jis/hit_collection.rb +2 -1
  25. data/lib/relaton/jis/processor.rb +1 -1
  26. data/lib/relaton/jis/scraper.rb +46 -5
  27. data/lib/relaton/oasis/bibliography.rb +1 -1
  28. data/lib/relaton/ogc/hit_collection.rb +1 -1
  29. data/lib/relaton/ogc.rb +1 -1
  30. data/lib/relaton/plateau/data_fetcher.rb +10 -1
  31. data/lib/relaton/plateau/handbook_parser.rb +38 -2
  32. data/lib/relaton/plateau/hit_collection.rb +20 -5
  33. data/lib/relaton/sae.rb +1 -1
  34. data/lib/relaton/version.rb +1 -1
  35. data/lib/relaton/w3c/bibliography.rb +1 -1
  36. data/lib/relaton/xsf/bibliography.rb +1 -1
  37. data/lib/relaton/xsf.rb +1 -1
  38. metadata +35 -48
  39. data/lib/relaton/3gpp/CLAUDE.md +0 -218
  40. data/lib/relaton/adobe/CLAUDE.md +0 -104
  41. data/lib/relaton/bib/CLAUDE.md +0 -150
  42. data/lib/relaton/bipm/CLAUDE.md +0 -187
  43. data/lib/relaton/bsi/CLAUDE.md +0 -35
  44. data/lib/relaton/calconnect/CLAUDE.md +0 -177
  45. data/lib/relaton/ccsds/CLAUDE.md +0 -74
  46. data/lib/relaton/cen/CLAUDE.md +0 -219
  47. data/lib/relaton/cie/CLAUDE.md +0 -90
  48. data/lib/relaton/core/CLAUDE.md +0 -140
  49. data/lib/relaton/db/CLAUDE.md +0 -385
  50. data/lib/relaton/doi/CLAUDE.md +0 -105
  51. data/lib/relaton/easc/CLAUDE.md +0 -85
  52. data/lib/relaton/ecma/CLAUDE.md +0 -217
  53. data/lib/relaton/etsi/CLAUDE.md +0 -219
  54. data/lib/relaton/evs/CLAUDE.md +0 -120
  55. data/lib/relaton/gb/CLAUDE.md +0 -47
  56. data/lib/relaton/gost/CLAUDE.md +0 -100
  57. data/lib/relaton/iala/CLAUDE.md +0 -97
  58. data/lib/relaton/iana/CLAUDE.md +0 -105
  59. data/lib/relaton/iec/CLAUDE.md +0 -13
  60. data/lib/relaton/ieee/CLAUDE.md +0 -202
  61. data/lib/relaton/ietf/CLAUDE.md +0 -382
  62. data/lib/relaton/iho/CLAUDE.md +0 -76
  63. data/lib/relaton/index/CLAUDE.md +0 -208
  64. data/lib/relaton/isbn/CLAUDE.md +0 -47
  65. data/lib/relaton/iso/CLAUDE.md +0 -38
  66. data/lib/relaton/itu/CLAUDE.md +0 -685
  67. data/lib/relaton/jcgm/CLAUDE.md +0 -107
  68. data/lib/relaton/jis/CLAUDE.md +0 -64
  69. data/lib/relaton/logger/CLAUDE.md +0 -37
  70. data/lib/relaton/nist/CLAUDE.md +0 -194
  71. data/lib/relaton/oasis/CLAUDE.md +0 -251
  72. data/lib/relaton/ogc/CLAUDE.md +0 -242
  73. data/lib/relaton/oiml/CLAUDE.md +0 -47
  74. data/lib/relaton/omg/CLAUDE.md +0 -137
  75. data/lib/relaton/plateau/CLAUDE.md +0 -126
  76. data/lib/relaton/sae/CLAUDE.md +0 -91
  77. data/lib/relaton/sdo/CLAUDE.md +0 -81
  78. data/lib/relaton/un/CLAUDE.md +0 -91
  79. data/lib/relaton/w3c/CLAUDE.md +0 -369
  80. data/lib/relaton/xsf/CLAUDE.md +0 -189
data/lib/relaton/db.rb CHANGED
@@ -65,21 +65,25 @@ module Relaton
65
65
  # @return [nil, RelatonBib::BibliographicItem, ...]
66
66
  # @raise [Relaton::UnknownReferenceError] no flavor recognizes the
67
67
  # reference (see Registry#route)
68
+ # @raise [Relaton::NotInLocalStoreError] offline mode, and the local
69
+ # store has no entry for the reference (relaton#536)
68
70
  ##
69
71
  def fetch(text, year = nil, opts = {}) # rubocop:disable Metrics/MethodLength
70
72
  reference = text.strip
71
- stdclass, pubid = @registry.route(reference)
72
- processor = @registry[stdclass]
73
- ref = if processor.respond_to?(:urn_to_code)
74
- processor.urn_to_code(reference)&.first
75
- else reference
76
- end
77
- ref ||= reference
78
- # The routed pubid is the parse of `reference`; a URN rewritten to a
79
- # code is parsed again by the flavor.
80
- pubid = nil unless ref == reference
81
- result = combine_doc ref, year, opts, stdclass
82
- result || fetch_routed(ref, year, opts, stdclass, pubid)
73
+ offline_answer(reference, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
74
+ stdclass, pubid = @registry.route(reference)
75
+ processor = @registry[stdclass]
76
+ ref = if processor.respond_to?(:urn_to_code)
77
+ processor.urn_to_code(reference)&.first
78
+ else reference
79
+ end
80
+ ref ||= reference
81
+ # The routed pubid is the parse of `reference`; a URN rewritten to a
82
+ # code is parsed again by the flavor.
83
+ pubid = nil unless ref == reference
84
+ result = combine_doc ref, year, opts, stdclass
85
+ result || fetch_routed(ref, year, opts, stdclass, pubid)
86
+ end
83
87
  end
84
88
 
85
89
  # @see Relaton::Db#fetch
@@ -124,7 +128,7 @@ module Relaton
124
128
  threads = ENV["RELATON_FETCH_PARALLEL"]&.to_i || processor.threads
125
129
  wp = WorkersPool.new(threads) do |args|
126
130
  args[3].call fetch(*args[0..2])
127
- rescue Relaton::RequestError => e
131
+ rescue Relaton::RequestError, Relaton::NotInLocalStoreError => e
128
132
  args[3].call e
129
133
  rescue StandardError => e
130
134
  # An unparseable reference ("ISO Standards Tag Set" in a
@@ -137,6 +141,14 @@ module Relaton
137
141
  Util.error "`#{args[0]}` -- #{e.message}"
138
142
  end
139
143
  args[3].call nil
144
+ rescue Exception => e # rubocop:disable Lint/RescueException
145
+ # The completion callback is a contract: the consumer counts
146
+ # exactly one delivery per fetch, and a missing one blocks it
147
+ # forever (metanorma-standoc CI deadlock, 2026-11-13: an
148
+ # RSpec::Mocks::MockExpectationError -- an Exception, not a
149
+ # StandardError -- escaped the rescue above).
150
+ Util.error "`#{args[0]}` -- #{e.class}: #{e.message}"
151
+ args[3].call nil
140
152
  end
141
153
  @queues[stdclass] =
142
154
  { queue: SizedQueue.new(threads * 2), workers_pool: wp }
@@ -155,14 +167,19 @@ module Relaton
155
167
  # @raise [Relaton::UnknownReferenceError] no flavor is named and none
156
168
  # recognizes the reference
157
169
  #
170
+ # @raise [Relaton::NotInLocalStoreError] offline mode, and the local
171
+ # store has no entry for the reference (relaton#536)
172
+ #
158
173
  def fetch_std(code, year = nil, stdclass = nil, opts = {})
159
- std = named_class(stdclass)
160
- # A flavor the caller names is the one asked: no co-publisher.
161
- return check_bibliocache(code, year, opts, std) if std
162
-
163
- std, pubid = @registry.route(code)
164
- fetch_routed(code, year, opts, std, pubid) ||
165
- fetch_equivalent(code, year, opts)
174
+ offline_answer(code, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
175
+ std = named_class(stdclass)
176
+ # A flavor the caller names is the one asked: no co-publisher.
177
+ next check_bibliocache(code, year, opts, std) if std
178
+
179
+ std, pubid = @registry.route(code)
180
+ fetch_routed(code, year, opts, std, pubid) ||
181
+ fetch_equivalent(code, year, opts)
182
+ end
166
183
  end
167
184
 
168
185
  # Cross-SDO equivalence as data: on a total miss, walk the equivalence
@@ -174,6 +191,9 @@ module Relaton
174
191
  #
175
192
  # @return [Relaton::Bib::ItemData, nil]
176
193
  def fetch_equivalent(code, year, opts)
194
+ # `relaton/db` alone does not declare the `Relaton::Equivalence`
195
+ # autoload (`relaton.rb` does), so a fetch_std miss NameErrored there.
196
+ require_relative "equivalence"
177
197
  Equivalence.related_for(code).each do |candidate|
178
198
  Util.info "Not found; trying equivalent `#{candidate}`", key: code
179
199
  std, pubid = @registry.route(candidate)
@@ -221,6 +241,24 @@ module Relaton
221
241
 
222
242
  private
223
243
 
244
+ #
245
+ # Offline mode (Configuration#offline, relaton#536) answers every fetch
246
+ # from the local store only: the block runs with `fetch_db` set, so no
247
+ # path reaches a flavor's `get`, and a miss of the whole resolution
248
+ # (routed flavor, co-publishers, equivalents) raises. An explicit
249
+ # `fetch_db` keeps its nil answer.
250
+ #
251
+ # @yieldparam opts [Hash] the options to fetch with
252
+ # @raise [Relaton::NotInLocalStoreError]
253
+ #
254
+ def offline_answer(reference, opts)
255
+ return yield(opts) if opts[:fetch_db] || !Db.configuration.offline
256
+
257
+ yield(opts.merge(fetch_db: true)) or
258
+ raise NotInLocalStoreError.new(reference.to_s,
259
+ [@local_db, @db].compact.map(&:dir))
260
+ end
261
+
224
262
  # True when the error is a flavor's grammar refusing to read the
225
263
  # reference. Both are raised by the flavors' pubid/parslet parses; a
226
264
  # reference they cannot read is not-found, not a failure (relaton#412).
@@ -489,30 +527,48 @@ module Relaton
489
527
  return check_date_range(searchcode, id, year, opts, stdclass, query)
490
528
  end
491
529
 
492
- @semaphore.synchronize { db.expire id, year }
493
- if altdb
494
- return bib_retval(altdb.read(id), stdclass) if opts[:fetch_db]
530
+ # The cache-only path reads, and never expires: a stale entry is
531
+ # still the local store's answer (relaton#536).
532
+ # A document in either cache answers over a not_found in the other.
533
+ if opts[:fetch_db]
534
+ entries = [db, altdb].compact.map { |c| c.read(id) }
535
+ return bib_retval(entries.find { |e| e.is_a?(String) }, stdclass)
536
+ end
495
537
 
496
- @semaphore.synchronize do
497
- db.clone_entry id, altdb if altdb.valid_entry? id, year
498
- end
499
- new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
500
- query: query)
501
- @semaphore.synchronize do
502
- altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
503
- end
504
- else
505
- return bib_retval(db.read(id), stdclass) if opts[:fetch_db]
538
+ @semaphore.synchronize { replace_stale(db, altdb, id, year) } if altdb
539
+ # The entry new_bib_entry answers, not a second read of `db`: a stale
540
+ # document served from `altdb` has no row in `db`.
541
+ entry = new_bib_entry(searchcode, year, opts, stdclass,
542
+ db: db, id: id, query: query, altdb: altdb)
543
+ @semaphore.synchronize { replace_stale(altdb, db, id, year) } if altdb
544
+ bib_retval(entry, stdclass)
545
+ end
506
546
 
507
- new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
508
- query: query)
509
- end
510
- bib_retval(db.read(id), stdclass)
547
+ # Replace the stale (or missing) row of `id` in `cache` with the valid
548
+ # row `other` holds. A valid row of `cache` is kept (local over global).
549
+ def replace_stale(cache, other, id, year)
550
+ return if cache.valid_entry?(id, year)
551
+ return unless other.valid_entry?(id, year)
552
+
553
+ cache.expire id, year
554
+ cache.clone_entry id, other
511
555
  end
512
556
 
557
+ #
558
+ # A stale entry (Cache#valid_entry? is false) is refetched, but it is
559
+ # not deleted first: Db#fetch_entry serves it when the refetch fails
560
+ # with a transport error, and replaces it when the refetch answers.
561
+ # With no row of its own, the stale document of the other cache
562
+ # (`altdb:`) is the fallback. `no_cache` asks for a fresh answer, so it
563
+ # has no fallback.
564
+ #
513
565
  def new_bib_entry(code, year, opts, stdclass, **args)
514
- entry = @semaphore.synchronize { args[:db]&.read(args[:id]) }
515
- if !entry || opts[:no_cache]
566
+ entry, valid = @semaphore.synchronize do
567
+ found = args[:db]&.read(args[:id])
568
+ [found, found && args[:db].valid_entry?(args[:id], year)]
569
+ end
570
+ if !entry || opts[:no_cache] || !valid
571
+ args = with_stale(args, entry) unless opts[:no_cache]
516
572
  return fetch_entry(code, year, opts, stdclass, **args)
517
573
  end
518
574
 
@@ -524,18 +580,47 @@ module Relaton
524
580
  entry
525
581
  end
526
582
 
527
- def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize
583
+ # @return [Hash] args, with the `stale:` entry and the `stale_db:` cache
584
+ # that holds it, when there is one
585
+ def with_stale(args, entry)
586
+ stale_db = args[:db]
587
+ unless entry
588
+ stale_db = args[:altdb]
589
+ entry = @semaphore.synchronize { stale_db&.read(args[:id]) }
590
+ return args unless entry.is_a?(String)
591
+ end
592
+ args.merge(stale: entry, stale_db: stale_db)
593
+ end
594
+
595
+ #
596
+ # @option args [String, Relaton::Db::NotFound, nil] :stale the stale
597
+ # entry of the key: a transport failure answers a stale document with
598
+ # a warning (relaton#536), and an answer replaces it
599
+ # @option args [Relaton::Db::Cache] :stale_db the cache that holds it
600
+ #
601
+ def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
528
602
  processor = @registry[stdclass]
529
- bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
530
- args[:query])
603
+ bib = begin
604
+ net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
605
+ args[:query])
606
+ rescue Relaton::RequestError => e
607
+ raise unless args[:stale].is_a?(String)
608
+
609
+ Util.warn "#{e.message}; using the cached entry fetched " \
610
+ "#{args[:stale_db].fetched(args[:id])}", key: code
611
+ return args[:stale]
612
+ end
531
613
  entry = bib_entry bib
532
614
  return entry if args[:db].nil?
533
615
 
534
616
  # `no_cache` refreshes a cached entry, but a failed fetch does not
535
- # replace a cached document with a not-found entry.
617
+ # replace a valid cached document with a not-found entry. A stale
618
+ # entry is replaced by any answer; its row goes first, so a dated
619
+ # query's subset row does not outlive the refetch.
536
620
  refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
537
621
  @semaphore.synchronize do
538
- if refresh || !args[:db].read(args[:id])
622
+ args[:db].expire args[:id], year if args[:stale]
623
+ if args[:stale] || refresh || !args[:db].read(args[:id])
539
624
  save_bib args[:db], args[:id], bib, entry, stdclass
540
625
  end
541
626
  end
@@ -553,18 +638,29 @@ module Relaton
553
638
  end
554
639
 
555
640
  #
556
- # A publication date range selects among the cached editions of the
557
- # reference, so it is not part of the key. On a miss the flavor is asked
558
- # with the range, and its answer is cached under its own identifier only:
559
- # the query row keeps pointing to the latest edition.
641
+ # A publication date range selects among editions of the reference. The
642
+ # answer must depend on the citing document's cutoff, never on the order
643
+ # in which parallel workers happen to populate the cache: the edition
644
+ # pool visible to any one worker is whatever earlier workers fetched for
645
+ # their own cutoffs, so answering from that pool is nondeterministic
646
+ # (metanorma#659). A ranged answer is therefore cached under a key that
647
+ # names the range and a hit on it is authoritative; the unqualified
648
+ # query row is never written by a ranged fetch, so it keeps pointing at
649
+ # the latest edition.
560
650
  #
561
- def check_date_range(code, key, year, opts, stdclass, query = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
562
- caches = [@local_db, @db].compact
563
- cached = caches.flat_map { |c| c.candidates(key) }.filter_map do |_, xml|
564
- date = published_date(xml)
565
- [date, xml] if date && pub_date_in_range?(xml, opts)
566
- end.max_by(&:first)
567
- return bib_retval(cached.last, stdclass) if cached && !opts[:no_cache]
651
+ def range_cache_key(key, opts)
652
+ bounds = [opts[:publication_date_before] && "before=#{opts[:publication_date_before]}",
653
+ opts[:publication_date_after] && "after=#{opts[:publication_date_after]}"]
654
+ .compact
655
+ bounds.empty? ? nil : "#{key}@range:#{bounds.join(',')}"
656
+ end
657
+
658
+ def check_date_range(code, key, year, opts, stdclass, query = nil)
659
+ qualified = range_cache_key(key, opts)
660
+ unless opts[:no_cache]
661
+ hit = [@local_db, @db].compact.filter_map { |c| c.read(qualified) }.first
662
+ return bib_retval(hit, stdclass) if hit
663
+ end
568
664
  return if opts[:fetch_db]
569
665
 
570
666
  processor = @registry[stdclass]
@@ -574,9 +670,15 @@ module Relaton
574
670
 
575
671
  entry = bib_entry bib
576
672
  item = item_key(bib, stdclass)
577
- if item
578
- @semaphore.synchronize do
579
- [@local_db, @db].compact.each { |c| c.store item, entry }
673
+ @semaphore.synchronize do
674
+ [@local_db, @db].compact.each do |c|
675
+ # A ranged answer that carries no edition of its own is the
676
+ # undated form: storing it would overwrite the unqualified query
677
+ # row (whose contract is the latest edition), so only a genuinely
678
+ # dated answer may write its own row.
679
+ link = item if item && item != key
680
+ c.store item, entry if link
681
+ c.store qualified, entry, item_key: link if qualified
580
682
  end
581
683
  end
582
684
  bib_retval entry, stdclass
@@ -38,7 +38,7 @@ module Relaton
38
38
  # answer is worse than none. By the same trade-off, a reference must now
39
39
  # parse WHOLE: the old regex was unanchored at the end, so
40
40
  # `ECMA-6 (draft)` silently resolved to ECMA-6 and now does not. See
41
- # lib/relaton/ecma/CLAUDE.md for the measured table.
41
+ # docs/ecma.md for the measured table.
42
42
  #
43
43
  # The rows are selected with pubid's subset match `pubid === row`, the
44
44
  # default of `Index::Type#search`. An `edition` or a `volume` that the
@@ -73,7 +73,7 @@ module Relaton
73
73
  # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
74
74
  # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
75
75
  # `"..." is not a recognized standards identifier`
76
- # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
76
+ # (`lib/relaton/cli/command.rb`), and `Db#fetch`
77
77
  # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
78
78
  # Rescuing here would collapse "this identifier is malformed" into "no
79
79
  # such document", leaving a caller unable to tell them apart.
@@ -16,7 +16,7 @@ module Relaton
16
16
  # status flags below are a separate axis and already permissive, so this
17
17
  # is what makes e.g. all three editions of `ETSI EN 319 142-1` retrievable
18
18
  # instead of one. It also makes the crawl about 2.4 times larger — see
19
- # lib/relaton/etsi/CLAUDE.md before changing it back.
19
+ # docs/etsi.md before changing it back.
20
20
  SOURCEURL = "https://www.etsi.org/custom/standardssearch/data.php?format=json&includeScope=1&" \
21
21
  "page=%<page>s&search=&title=1&etsiNumber=1&content=1&version=1&onApproval=1&published=1&" \
22
22
  "withdrawn=1&historical=1&isCurrent=1&superseded=1&startDate=1988-01-15&endDate=%<date>s&" \
@@ -166,17 +166,27 @@ module Relaton
166
166
 
167
167
  def normalize(record)
168
168
  {
169
- "ETSI deliverable" => record["ETSI_DELIVERABLE"],
170
- "title" => record["TITLE"],
169
+ "ETSI deliverable" => sanitize(record["ETSI_DELIVERABLE"]),
170
+ "title" => sanitize(record["TITLE"]),
171
171
  "Details link" => "https://webapp.etsi.org/workprogram/Report_WorkItem.asp?WKI_ID=#{record['wki_id']}",
172
172
  "PDF link" => "https://www.etsi.org/deliver/#{record['EDSpathname']}#{record['EDSPDFfilename']}",
173
173
  "Status" => derive_status(record),
174
- "Keywords" => record["Keywords"].to_s,
175
- "Technical body" => record["TB"],
176
- "Scope" => record["Scope"],
174
+ "Keywords" => sanitize(record["Keywords"]).to_s,
175
+ "Technical body" => sanitize(record["TB"]),
176
+ "Scope" => sanitize(record["Scope"]),
177
177
  }
178
178
  end
179
179
 
180
+ # The upstream ETSI metadata carries stray control characters — one
181
+ # 1998-era record ships a literal NUL inside "sequences", which the YAML
182
+ # dump surfaces as a raw \0 escape and which silently truncates the
183
+ # api.relaton.org D1 import stream (relaton-api-js#155,
184
+ # relaton-data-etsi#7). Non-structural C0 controls are dropped; \t, \n
185
+ # and \r stay — the Scope column is hard-wrapped upstream text.
186
+ def sanitize(value)
187
+ value&.gsub(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/, "")
188
+ end
189
+
180
190
  def derive_status(record)
181
191
  return "Withdrawn" if record["ACTION_TYPE"] == "WD"
182
192
 
@@ -120,7 +120,7 @@ module Relaton
120
120
  # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
121
121
  # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
122
122
  # `"..." is not a recognized standards identifier`
123
- # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
123
+ # (`lib/relaton/cli/command.rb`), and `Db#fetch`
124
124
  # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
125
125
  # Rescuing here would collapse "this identifier is malformed" into "no
126
126
  # such document", leaving a caller unable to tell them apart.
data/lib/relaton/iana.rb CHANGED
@@ -25,7 +25,7 @@ module Relaton
25
25
  # The legacy string-keyed `index-v1` (whose `:id` was the bare
26
26
  # `registry[/sub-registry]` slug) is no longer produced or read here; it is
27
27
  # emitted by `relaton-data-iana`'s own `crawler.rb`, derived from `index-v2`,
28
- # so released gem lines keep resolving. See lib/relaton/iana/CLAUDE.md.
28
+ # so released gem lines keep resolving. See docs/iana.md.
29
29
  INDEXFILE = "index-v2".freeze
30
30
  # The machine index on the data repo's Pages site (relaton#206) —
31
31
  # manifest + per-shard rows read by the consumer's index; the zip stays
@@ -5,6 +5,29 @@ module Relaton
5
5
 
6
6
  DOMAIN = "https://webstore.iec.ch"
7
7
 
8
+ # The Harmonized API serves published records with a status WORD;
9
+ # the harmonized stage system (ISO/IEC Directives Part 1) wants the
10
+ # numeric code (relaton#519). 60.60 is "International Standard
11
+ # published". The four end-of-life words all sit at 95.99
12
+ # "Withdrawal of International Standard": by the time the API
13
+ # serves REVISED/REPLACED/MERGED the newer record is already
14
+ # published and this edition is withdrawn - 90.92 ("to be
15
+ # revised") is a review-DECISION state that never persists into a
16
+ # bibliographic record, and ISO's own corpus records superseded
17
+ # editions at 95.99 with no stage-90 records at all
18
+ # (metanorma-iso#1067 wants IEC to match ISO's treatment). The
19
+ # original word rides along in ext.stagename, so the mapping is
20
+ # reversible at render time. An unmapped word raises: the table is
21
+ # the contract, and a new API word is a spec change to make
22
+ # deliberately, not silently.
23
+ HARMONIZED_STATUS = {
24
+ "PUBLISHED" => %w[60 60],
25
+ "REVISED" => %w[95 99],
26
+ "REPLACED" => %w[95 99],
27
+ "WITHDRAWN" => %w[95 99],
28
+ "MERGED" => %w[95 99],
29
+ }.freeze
30
+
8
31
  ATTRS = %i[
9
32
  type docidentifier language script title date contributor
10
33
  status edition abstract copyright source relation place ext
@@ -143,8 +166,15 @@ module Relaton
143
166
 
144
167
  def status
145
168
  result = begin
146
- stage = Bib::Status::Stage.new content: @pub["status"]
147
- Bib::Status.new stage: stage
169
+ code = HARMONIZED_STATUS.fetch(@pub["status"]) do
170
+ raise ArgumentError,
171
+ "unknown Harmonized API status word: #{@pub["status"].inspect} - " \
172
+ "extend DataParser::HARMONIZED_STATUS"
173
+ end
174
+ Bib::Status.new(
175
+ stage: Bib::Status::Stage.new(content: code[0]),
176
+ substage: Bib::Status::Stage.new(content: code[1]),
177
+ )
148
178
  end
149
179
  @errors[:status] &&= result.nil?
150
180
  result
@@ -353,11 +383,20 @@ module Relaton
353
383
  flavor: "iec",
354
384
  ics: ics,
355
385
  price_code: @pub.dig("priceInfo", "priceCode"),
386
+ stagename: stagename,
356
387
  )
357
388
  @errors[:ext] &&= result.nil?
358
389
  result
359
390
  end
360
391
 
392
+ # The Harmonized API's status word, kept beside the numeric stage
393
+ # code it mapped to (relaton#519) - ext.stagename is where the IHO
394
+ # and ISO flavors keep the SDO's own stage vocabulary too.
395
+ def stagename
396
+ word = @pub["status"]
397
+ StageName.new(abbreviation: word, content: word) if word
398
+ end
399
+
361
400
  #
362
401
  # Parse structured identifier.
363
402
  #
@@ -175,7 +175,7 @@ RelatonItu uses the relaton-logger gem for logging. By default, it logs to STDOU
175
175
 
176
176
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
177
177
 
178
- The index test fixture (used by tests) is a trimmed copy of the published pubid-structured `index-v2.zip` (rows are `_type: pubid:itu:…` hashes) from the `main` branch of the combined https://github.com/relaton/relaton-data-itu[relaton-data-itu] repository, kept at `spec/fixtures/index-v2.zip`. Refresh it with the regeneration snippet in `lib/relaton/itu/CLAUDE.md` (it must deserialize under the pinned pubid).
178
+ The index test fixture (used by tests) is a trimmed copy of the published pubid-structured `index-v2.zip` (rows are `_type: pubid:itu:…` hashes) from the `main` branch of the combined https://github.com/relaton/relaton-data-itu[relaton-data-itu] repository, kept at `spec/fixtures/index-v2.zip`. Refresh it with the regeneration snippet in `docs/itu.md` (it must deserialize under the pinned pubid).
179
179
 
180
180
  To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
181
181
 
@@ -41,7 +41,7 @@ module Relaton
41
41
  # the rate is set by a shared `Relaton::Core::Pacer` — `delay:` is the
42
42
  # minimum gap between request *starts*, so ITU's own latency counts toward it
43
43
  # instead of being added on top. A `Relaton::Itu::Governor` gives the pool one
44
- # shared cooldown for a WAF block. See `#get` and lib/relaton/itu/CLAUDE.md.
44
+ # shared cooldown for a WAF block. See `#get` and docs/itu.md.
45
45
  # The crawl is parallel **within** a series only: `DataFetcher#harvest_family`
46
46
  # rescues per series and `DataMergeR` writes per series, and the data repo's
47
47
  # collapse guard is calibrated on that granularity.
@@ -4,6 +4,8 @@ RelatonJis is a Ruby gem that implements the https://github.com/metanorma/metano
4
4
 
5
5
  You can use it to retrieve metadata of JIS Standards from https://webdesk.jsa.or.jp, and access such metadata through the `Relaton::Jis::ItemData` object.
6
6
 
7
+ The flavor also retrieves WES standards of the Japan Welding Engineering Society (JWES), which webdesk.jsa.or.jp sells next to JIS: `Relaton::Jis::Bibliography.get "WES 7700-1:2019"`. A WES record has the docidentifier type `WES` and JWES as its author and publisher. Its doctype is the JIS value of its form (`japanese-industrial-standard`, `technical-report`, `technical-specification` or `amendment`), because the JIS grammar has no WES value.
8
+
7
9
  == Usage
8
10
 
9
11
  === Search for standards using keywords
@@ -9,11 +9,20 @@ module Relaton
9
9
  class DataFetcher < Core::DataFetcher
10
10
  URL = "https://webdesk.jsa.or.jp/books/"
11
11
 
12
+ # The Webdesk lists a crawl reads, in this order: the JIS search, then
13
+ # the JWES group list (WES standards, relaton-jis#18). Webdesk keeps the
14
+ # open list in the session, and the list page and its paging
15
+ # (`W11M0070`) read that list. So the lists are read one after the
16
+ # other, and a session refresh opens the current list again.
17
+ LISTS = %w[JIS JWES].freeze
18
+
12
19
  def initialize(output, format)
13
20
  super
14
21
  @queue = SizedQueue.new 10
15
22
  @threads = create_thread_pool 5
16
23
  @mutex = Mutex.new
24
+ # URLs of the documents that did not fetch; see #prune_unwritten.
25
+ @failed_docs = []
17
26
  end
18
27
 
19
28
  def log_error(msg)
@@ -72,30 +81,63 @@ module Relaton
72
81
  else
73
82
  Util.warn "URL: #{url}"
74
83
  Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
84
+ @mutex.synchronize { @failed_docs << url }
75
85
  end
76
86
  else
77
87
  save_doc bib, url
78
88
  end
79
89
  end
80
90
 
91
+ # relaton-data-jis crawls with `prune: true`: it keeps data/, and after
92
+ # #fetch, #prune_unwritten deletes each file the run did not write. So a
93
+ # list that does not open, or stops part way (#get_next_page), raises:
94
+ # the crawl job fails before the prune and publishes nothing.
81
95
  def fetch(_source = nil)
82
- return unless initial_post
96
+ failed = begin
97
+ LISTS.find { |list| !fetch_list(list) }
98
+ ensure
99
+ end_threads_and_wait
100
+ end
101
+ raise Error, "Webdesk list #{failed} did not open" if failed
83
102
 
84
- resp = agent.get "#{URL}W11M0070/index"
85
- parse_page resp
86
103
  index.save
87
104
  report_errors
88
105
  end
89
106
 
107
+ # Queue every document of one list. False if the list does not open.
108
+ def fetch_list(list)
109
+ @list = list
110
+ @initial_time = nil
111
+ return false unless initial_post
112
+
113
+ parse_page agent.get("#{URL}W11M0070/index")
114
+ true
115
+ end
116
+
117
+ # Open the current list in the session, unless it was opened less than
118
+ # 10 minutes ago.
90
119
  def initial_post
91
120
  return true if @initial_time && Time.now - @initial_time < 600
92
121
 
122
+ status = @list == "JWES" ? open_group_list(@list) : open_jis_search
123
+ @initial_time = Time.now
124
+ status || Util.warn("No results found for #{@list}")
125
+ end
126
+
127
+ def open_jis_search
93
128
  body = { record: 0, dantai: "JIS", searchtype2: 1,
94
129
  status_1: 1, status_2: 2 } # rubocop:disable Naming/VariableNumber
95
130
  resp = agent.post "#{URL}W11M0270/index", body
96
- disp = Relaton::Json.parse_external resp.body
97
- @initial_time = Time.now
98
- disp["status"] || Util.warn("No results found for JIS")
131
+ Relaton::Json.parse_external(resp.body)["status"]
132
+ end
133
+
134
+ # A group list from the "domestic group publications" page. The link
135
+ # redirects to the list page, which states the count of documents.
136
+ def open_group_list(dantai)
137
+ page = agent.get "#{URL}W11S0260/getSearchAllListByLink" \
138
+ "?dantai_cd=#{dantai}&display=KOKUNAI"
139
+ script = page.at_xpath('//script[contains(.,"var count =")]')
140
+ script&.text.to_s[/var count = (\d+);/, 1].to_i.positive?
99
141
  end
100
142
 
101
143
  def agent
@@ -111,7 +153,6 @@ module Relaton
111
153
 
112
154
  resp = get_next_page(offset)
113
155
  end
114
- end_threads_and_wait
115
156
  end
116
157
 
117
158
  def parse_offset(resp) # rubocop:disable Metrics/AbcSize
@@ -135,25 +176,44 @@ module Relaton
135
176
  @count.to_i
136
177
  end
137
178
 
179
+ # The next page of the current list. Raises when the page does not
180
+ # come, because a list that stops part way leaves its unread documents
181
+ # unwritten, and the prune would delete them.
138
182
  def get_next_page(offset) # rubocop:disable Metrics/MethodLength
139
183
  attempts = 0
140
184
  begin
141
- if initial_post
142
- url = "#{URL}W11M0070/getAddList"
143
- agent.post url, search_type: "JIS", offset: offset
185
+ unless initial_post
186
+ raise Error, "Webdesk list #{@list} did not open again " \
187
+ "at offset #{offset}"
144
188
  end
189
+
190
+ agent.post "#{URL}W11M0070/getAddList",
191
+ search_type: "JIS", offset: offset
192
+ rescue Error
193
+ raise
145
194
  rescue StandardError => e
146
195
  attempts += 1
147
196
  if attempts < 5
148
197
  sleep 2
149
198
  retry
150
- else
151
- Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
152
199
  end
200
+ raise Error, "Webdesk list #{@list} stopped at offset " \
201
+ "#{offset}: #{e.message}"
153
202
  end
154
203
  end
155
204
 
156
- def save_doc(bib, url) # rubocop:disable Metrics/MethodLength
205
+ # Prune only after a complete crawl: a document that did not fetch was
206
+ # not written, and the prune would delete its file.
207
+ def prune_unwritten(min_ratio: 0.9)
208
+ unless @failed_docs.empty?
209
+ warn "prune skipped: #{@failed_docs.size} document(s) did not fetch"
210
+ return []
211
+ end
212
+
213
+ super
214
+ end
215
+
216
+ def save_doc(bib, url) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
157
217
  return unless bib
158
218
 
159
219
  id = bib.docidentifier.find(&:primary).content
@@ -175,6 +235,7 @@ module Relaton
175
235
  Util.warn "File #{output_file id} already exists; writing #{file} instead" if file != output_file(id)
176
236
  @files << file
177
237
  File.write file, serialize(bib), encoding: "UTF-8"
238
+ record_written file
178
239
  pid = pubid id
179
240
  index.add_or_update pid, file if pid
180
241
  end
@@ -83,7 +83,8 @@ module Relaton
83
83
 
84
84
  def create_relation(hit)
85
85
  id = hit.pubid.to_s
86
- docid = Docidentifier.new content: id, type: "JIS", primary: true
86
+ type = id.start_with?("WES ") ? "WES" : "JIS"
87
+ docid = Docidentifier.new content: id, type: type, primary: true
87
88
  bibitem = ItemData.new(
88
89
  formattedref: Bib::Formattedref.new(content: id),
89
90
  docidentifier: [docid],