relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/lib/relaton/3gpp/bibliography.rb +19 -6
  3. data/lib/relaton/3gpp/processor.rb +6 -0
  4. data/lib/relaton/adobe/processor.rb +9 -0
  5. data/lib/relaton/bib/converter/csl.rb +110 -0
  6. data/lib/relaton/bib/converter/ris.rb +104 -0
  7. data/lib/relaton/bib/converter/titles.rb +24 -0
  8. data/lib/relaton/bib/item_data.rb +23 -0
  9. data/lib/relaton/bib/model/item.rb +6 -6
  10. data/lib/relaton/bib/sanitizer.rb +71 -30
  11. data/lib/relaton/bib.rb +10 -0
  12. data/lib/relaton/bipm/bibliography.rb +24 -21
  13. data/lib/relaton/bipm/processor.rb +1 -0
  14. data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +6 -6
  15. data/lib/relaton/bipm/si_brochure_parser.rb +2 -2
  16. data/lib/relaton/bsi/bibliography.rb +26 -23
  17. data/lib/relaton/bsi/processor.rb +5 -0
  18. data/lib/relaton/calconnect/bibliography.rb +6 -6
  19. data/lib/relaton/calconnect/hit_collection.rb +4 -1
  20. data/lib/relaton/ccsds/bibliography.rb +16 -10
  21. data/lib/relaton/ccsds/hit_collection.rb +1 -1
  22. data/lib/relaton/ccsds/processor.rb +13 -0
  23. data/lib/relaton/cen/bibliography.rb +27 -24
  24. data/lib/relaton/cen/hit_collection.rb +1 -1
  25. data/lib/relaton/cen/processor.rb +5 -0
  26. data/lib/relaton/cen/scraper.rb +9 -9
  27. data/lib/relaton/cie/bibliography.rb +11 -9
  28. data/lib/relaton/cie/data_fetcher.rb +18 -18
  29. data/lib/relaton/cie/processor.rb +1 -0
  30. data/lib/relaton/cie/scrapper.rb +15 -8
  31. data/lib/relaton/cie.rb +1 -1
  32. data/lib/relaton/cloud.rb +127 -0
  33. data/lib/relaton/core/hit_collection.rb +6 -10
  34. data/lib/relaton/core/processor.rb +121 -0
  35. data/lib/relaton/core/request_error.rb +21 -3
  36. data/lib/relaton/db/cache.rb +474 -148
  37. data/lib/relaton/db/cache_entry.rb +42 -0
  38. data/lib/relaton/db/registry.rb +152 -0
  39. data/lib/relaton/db.rb +275 -102
  40. data/lib/relaton/doi/crossref.rb +23 -6
  41. data/lib/relaton/doi/processor.rb +1 -0
  42. data/lib/relaton/easc/processor.rb +1 -0
  43. data/lib/relaton/ecma/bibliography.rb +18 -13
  44. data/lib/relaton/ecma/data_fetcher.rb +1 -1
  45. data/lib/relaton/ecma/data_parser.rb +1 -1
  46. data/lib/relaton/ecma/edition_parser.rb +2 -2
  47. data/lib/relaton/ecma/memento_parser.rb +4 -4
  48. data/lib/relaton/ecma/standard_parser.rb +6 -6
  49. data/lib/relaton/etsi/bibliography.rb +8 -7
  50. data/lib/relaton/etsi/processor.rb +1 -0
  51. data/lib/relaton/gb/bibliography.rb +20 -12
  52. data/lib/relaton/gb/gb_scraper.rb +5 -5
  53. data/lib/relaton/gb/scraper.rb +16 -16
  54. data/lib/relaton/gb/sec_scraper.rb +8 -8
  55. data/lib/relaton/gb/t_scraper.rb +5 -5
  56. data/lib/relaton/gost/processor.rb +1 -0
  57. data/lib/relaton/iala/bibliography.rb +15 -12
  58. data/lib/relaton/iala/processor.rb +1 -0
  59. data/lib/relaton/iana/bibliography.rb +12 -12
  60. data/lib/relaton/iana/data_fetcher.rb +3 -3
  61. data/lib/relaton/iana/parser.rb +12 -5
  62. data/lib/relaton/iana/processor.rb +9 -0
  63. data/lib/relaton/iec/bibliography.rb +18 -7
  64. data/lib/relaton/iec/data_parser.rb +24 -9
  65. data/lib/relaton/iec/processor.rb +11 -0
  66. data/lib/relaton/iec.rb +1 -1
  67. data/lib/relaton/ieee/bibliography.rb +19 -13
  68. data/lib/relaton/ieee/data_fetcher.rb +1 -1
  69. data/lib/relaton/ieee/processor.rb +24 -0
  70. data/lib/relaton/ieee/rawbib_id_parser.rb +2 -2
  71. data/lib/relaton/ietf/bibliography.rb +11 -9
  72. data/lib/relaton/ietf/data_fetcher.rb +1 -1
  73. data/lib/relaton/ietf/processor.rb +1 -0
  74. data/lib/relaton/ietf/rfc/entry.rb +15 -19
  75. data/lib/relaton/ietf/scraper.rb +5 -4
  76. data/lib/relaton/iho/processor.rb +1 -0
  77. data/lib/relaton/index/file_io.rb +2 -2
  78. data/lib/relaton/index/pool.rb +4 -3
  79. data/lib/relaton/index/shard_source.rb +1 -1
  80. data/lib/relaton/index/type.rb +2 -5
  81. data/lib/relaton/isbn/open_library.rb +11 -7
  82. data/lib/relaton/isbn/processor.rb +10 -0
  83. data/lib/relaton/iso/bibliography.rb +11 -9
  84. data/lib/relaton/iso/data_parser.rb +2 -2
  85. data/lib/relaton/iso/processor.rb +5 -0
  86. data/lib/relaton/iso/scraper.rb +20 -20
  87. data/lib/relaton/itu/bibliography.rb +107 -16
  88. data/lib/relaton/itu/data_crawler_r.rb +2 -2
  89. data/lib/relaton/itu/hit_collection.rb +64 -52
  90. data/lib/relaton/itu/processor.rb +1 -0
  91. data/lib/relaton/itu/scraper.rb +13 -3
  92. data/lib/relaton/itu.rb +0 -2
  93. data/lib/relaton/jis/bibliography.rb +22 -11
  94. data/lib/relaton/jis/data_fetcher.rb +4 -4
  95. data/lib/relaton/jis/processor.rb +1 -0
  96. data/lib/relaton/jis/scraper.rb +8 -8
  97. data/lib/relaton/nist/bibliography.rb +25 -21
  98. data/lib/relaton/oasis/bibliography.rb +16 -13
  99. data/lib/relaton/oasis/browser_agent.rb +2 -2
  100. data/lib/relaton/oasis/data_parser.rb +5 -5
  101. data/lib/relaton/oasis/data_parser_utils.rb +2 -2
  102. data/lib/relaton/oasis/data_part_parser.rb +7 -7
  103. data/lib/relaton/ogc/bibliography.rb +15 -14
  104. data/lib/relaton/ogc/hit_collection.rb +9 -7
  105. data/lib/relaton/ogc/processor.rb +13 -0
  106. data/lib/relaton/oiml/processor.rb +1 -0
  107. data/lib/relaton/omg/bibliography.rb +9 -9
  108. data/lib/relaton/omg/scraper.rb +14 -13
  109. data/lib/relaton/omg.rb +1 -1
  110. data/lib/relaton/plateau/bibliography.rb +9 -7
  111. data/lib/relaton/plateau/hit_collection.rb +2 -1
  112. data/lib/relaton/plateau/processor.rb +1 -0
  113. data/lib/relaton/un/bibliography.rb +21 -10
  114. data/lib/relaton/un/processor.rb +1 -0
  115. data/lib/relaton/version.rb +1 -1
  116. data/lib/relaton/w3c/processor.rb +1 -0
  117. data/lib/relaton/xsf/bibliography.rb +11 -6
  118. data/lib/relaton.rb +13 -0
  119. metadata +60 -14
  120. data/lib/relaton/itu/pubid.rb +0 -199
@@ -15,10 +15,10 @@ module Relaton
15
15
  module Bibliography
16
16
  SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-iana/refs/heads/v2/"
17
17
 
18
- # @param text [String, Pubid::Iana::Identifier]
18
+ # @param ref [String, Pubid::Iana::Identifier]
19
19
  # @return [Relaton::Iana::Item, nil]
20
- def search(text)
21
- pubid = parse_ref text
20
+ def search(ref)
21
+ pubid = parse_ref ref
22
22
  return unless pubid
23
23
 
24
24
  # Pass the pubid so Relaton::Index narrows candidates by number via
@@ -34,19 +34,19 @@ module Relaton
34
34
  raise Relaton::RequestError, e.message
35
35
  end
36
36
 
37
- # @param ref [String] the IANA registry slug to look up
37
+ # @param ref [String, Pubid::Iana::Identifier] the IANA registry slug to look up
38
38
  # @param year [String, NilClass] not used
39
39
  # @param opts [Hash] options
40
40
  # @return [Relaton::Iana::Item, nil]
41
41
  def get(ref, _year = nil, _opts = {})
42
- Util.info "Fetching from Relaton repository ...", key: ref
42
+ Util.info "Fetching from Relaton repository ...", key: ref.to_s
43
43
  result = search(ref)
44
44
  unless result
45
- Util.info "Not found.", key: ref
45
+ Util.info "Not found.", key: ref.to_s
46
46
  return
47
47
  end
48
48
 
49
- Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
49
+ Util.info "Found: `#{result.docidentifier[0].content}`", key: ref.to_s
50
50
  result
51
51
  end
52
52
 
@@ -63,14 +63,14 @@ module Relaton
63
63
  # RuntimeError on anything else, and a citation relaton cannot parse is a
64
64
  # miss, not an error — `#get` reports it as "Not found.".
65
65
  #
66
- # @param text [String, Pubid::Iana::Identifier]
66
+ # @param ref [String, Pubid::Iana::Identifier]
67
67
  # @return [Pubid::Iana::Identifier, nil]
68
- def parse_ref(text)
69
- return text unless text.is_a? String
68
+ def parse_ref(ref)
69
+ return ref unless ref.is_a? String
70
70
 
71
- ::Pubid::Iana::Identifier.parse text
71
+ ::Pubid::Iana::Identifier.parse ref
72
72
  rescue StandardError => e
73
- Util.warn "Failed to parse pubid `#{text}`: #{e.message}"
73
+ Util.warn "Failed to parse pubid `#{ref}`: #{e.message}"
74
74
  nil
75
75
  end
76
76
 
@@ -51,11 +51,11 @@ module Relaton
51
51
  end
52
52
 
53
53
  def parse(content)
54
- xml = Nokogiri::XML(content)
55
- registry = xml.at("/xmlns:registry")
54
+ xml = Moxml.parse(content)
55
+ registry = xml.at_xpath("/xmlns:registry", NS)
56
56
  doc = Parser.parse registry, nil, @errors
57
57
  save_doc doc
58
- registry.xpath("./xmlns:registry").each { |r| save_doc Parser.parse(r, doc, @errors) }
58
+ registry.xpath("./xmlns:registry", NS).each { |r| save_doc Parser.parse(r, doc, @errors) }
59
59
  end
60
60
 
61
61
  #
@@ -1,10 +1,14 @@
1
1
  module Relaton
2
2
  module Iana
3
+ # The IANA registry XML uses a default namespace, and moxml (unlike raw
4
+ # nokogiri) does not bind the literal `xmlns:` prefix to it — every
5
+ # namespaced query passes this binding explicitly.
6
+ NS = { "xmlns" => "http://www.iana.org/assignments" }.freeze
3
7
  class Parser
4
8
  #
5
9
  # Document parser initalization
6
10
  #
7
- # @param [Nokogiri::XML::Element] xml
11
+ # @param [Moxml::Element] xml
8
12
  #
9
13
  def initialize(xml, rootdoc, errors = {})
10
14
  @xml = xml
@@ -15,7 +19,7 @@ module Relaton
15
19
  #
16
20
  # Initialize document parser and run it
17
21
  #
18
- # @param [Nokogiri::XML::Element] xml
22
+ # @param [Moxml::Element] xml
19
23
  #
20
24
  # @return [Relaton::Iana::ItemData, nil] bibliographic item
21
25
  #
@@ -50,7 +54,7 @@ module Relaton
50
54
  # @return [Array<Relaton::Bib::Title>] title
51
55
  #
52
56
  def parse_title
53
- content = @xml.at("./xmlns:title")&.text || @xml[:id]
57
+ content = @xml.at_xpath("./xmlns:title", NS)&.text || @xml[:id]
54
58
  result = [Bib::Title.new(content: content, language: "en", script: "Latn")]
55
59
  @errors[:title] &&= result.empty?
56
60
  result
@@ -121,8 +125,11 @@ module Relaton
121
125
  # @return [Array<Relaton::Bib::Date>] date
122
126
  #
123
127
  def parse_date
124
- d = @xml.xpath("./xmlns:created|./xmlns:published|./xmlns:updated").map do |dt|
125
- Bib::Date.new(type: dt.name, at: dt.text)
128
+ # moxml 0.5.98 drops every branch after the second in an N>=3 union
129
+ # under the reserved `xmlns` prefix (lutaml/moxml#315), which silently
130
+ # stripped `updated` dates from every crawled registry.
131
+ d = %w[created published updated].flat_map do |type|
132
+ @xml.xpath("./xmlns:#{type}", NS).map { |dt| Bib::Date.new(type: dt.name, at: dt.text) }
126
133
  end
127
134
  result = d.none? && @rootdoc ? @rootdoc.date : d
128
135
  @errors[:date] &&= result.empty?
@@ -8,6 +8,7 @@ module Relaton
8
8
  def initialize # rubocop:disable Lint/MissingSuper
9
9
  @short = :relaton_iana
10
10
  @prefix = "IANA"
11
+ @pubid_identifier = :Iana # Db cache key
11
12
  @defaultprefix = %r{^IANA\s}
12
13
  @idtype = "IANA"
13
14
  @datasets = %w[iana-registries]
@@ -35,6 +36,14 @@ module Relaton
35
36
  DataFetcher.fetch(source, **opts)
36
37
  end
37
38
 
39
+ # `get` reads a reference pubid cannot parse as a miss, so it gets no key
40
+ # here either: it is not cached.
41
+ def cache_pubid(ref)
42
+ super
43
+ rescue StandardError
44
+ nil
45
+ end
46
+
38
47
  # @param xml [String]
39
48
  # @return [Relaton::Iana::ItemData]
40
49
  def from_xml(xml)
@@ -24,18 +24,29 @@ module Relaton
24
24
  raise Relaton::RequestError, e.message
25
25
  end
26
26
 
27
- # @param code [String] the IEC standard code to look up (e.g. "IEC 8000")
27
+ # @param ref [String, Pubid::Iec::Identifier] the IEC standard code to
28
+ # look up (e.g. "IEC 8000"), or the parse that Relaton::Db routed
29
+ # with (relaton#205); a pubid is never mutated
28
30
  # @param year [String] the year the standard was published (optional)
29
31
  # @param opts [Hash] options; restricted to :all_parts if all-parts
30
32
  # reference is required
31
33
  # @return [Relaton::Iec::ItemData, nil]
32
- def get(code, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
33
- opts[:all_parts] ||= code.match?(/\s\(all parts\)/)
34
- ref = code.sub(/\s\(all parts\)/, "")
35
- return iev if ref.casecmp("IEV").zero?
36
-
37
- pubid = ::Pubid::Iec::Identifier.parse ref.upcase
34
+ def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength
35
+ if ref.is_a?(String)
36
+ opts[:all_parts] ||= ref.match?(/\s\(all parts\)/)
37
+ text = ref.sub(/\s\(all parts\)/, "")
38
+ return iev if text.casecmp("IEV").zero?
39
+
40
+ pubid = ::Pubid::Iec::Identifier.parse text.upcase
41
+ elsif ref.all_parts?
42
+ opts[:all_parts] = true
43
+ pubid = ref.identifiers.first
44
+ else
45
+ pubid = ref
46
+ end
38
47
  if year
48
+ # Copy first: a caller's pubid is also its Db cache key.
49
+ pubid = pubid.class.from_hash(pubid.to_hash)
39
50
  pubid.date = ::Pubid::Components::Date.new(year: year.to_s)
40
51
  end
41
52
 
@@ -289,38 +289,53 @@ module Relaton
289
289
  #
290
290
  # @return [Array<Relaton::Bib::Relation>] relation
291
291
  #
292
- def relation # rubocop:disable Metrics/MethodLength
292
+ def relation
293
+ result = fetch_relations
294
+ @errors[:relation] &&= result.empty?
295
+ result
296
+ end
297
+
298
+ #
299
+ # Fetch relations from the webstore. Retry on network errors only.
300
+ # A non-success response (the legacy endpoint now redirects to a 404
301
+ # HTML page) or a malformed body gives no relations.
302
+ #
303
+ # @return [Array<Relaton::Bib::Relation>] relations
304
+ #
305
+ def fetch_relations # rubocop:disable Metrics/MethodLength
293
306
  try = 0
294
- result = begin
307
+ begin
295
308
  uri = URI "#{DOMAIN}/webstore/webstore.nsf/AjaxRequestXML?Openagent&url=#{urn_id}"
296
309
  resp = Net::HTTP.get_response uri
297
- doc = Nokogiri::XML resp.body
298
- create_relations doc
299
310
  rescue StandardError => e
300
311
  try += 1
301
312
  try < 3 ? retry : raise(e)
302
313
  end
303
- @errors[:relation] &&= result.empty?
304
- result
314
+ return [] if resp.is_a?(Net::HTTPResponse) && !resp.is_a?(Net::HTTPSuccess)
315
+
316
+ create_relations Moxml.parse(resp.body)
317
+ rescue Moxml::ParseError => e
318
+ Util.warn "Failed to parse relations for `#{urn_id}`: #{e.message.lines.first&.strip}"
319
+ []
305
320
  end
306
321
 
307
322
  #
308
323
  # Create relations.
309
324
  #
310
- # @param [Nokogiri::XML::Document] doc XML document
325
+ # @param [Moxml::Document] doc XML document
311
326
  #
312
327
  # @return [Array<Relaton::Bib::Relation>] relations
313
328
  #
314
329
  def create_relations(doc) # rubocop:disable Metrics/MethodLength
315
330
  doc.xpath('//ROW[STATUS[.!="PREPARING" and .!="PUBLISHED"]]')
316
331
  .map do |r|
317
- r_type = r.at("STATUS").text.downcase
332
+ r_type = r.at_xpath("STATUS").text.downcase
318
333
  type = case r_type
319
334
  when "revised", "replaced" then "updates"
320
335
  when "withdrawn" then "obsoletes"
321
336
  else r_type
322
337
  end
323
- ref = r.at("FULL_NAME").text
338
+ ref = r.at_xpath("FULL_NAME").text
324
339
  docid = Docidentifier.new(content: ref, type: "IEC", primary: true)
325
340
  bibitem = ItemData.new(formattedref: Bib::Formattedref.new(content: ref), docidentifier: [docid])
326
341
  Relation.new type: type, bibitem: bibitem
@@ -34,6 +34,12 @@ module Relaton
34
34
  DataFetcher.fetch(source, **opts)
35
35
  end
36
36
 
37
+ # `IEV` is not a document identifier (`get` answers it with the IEV
38
+ # vocabulary), so it gets no key: it is not cached.
39
+ def cache_pubid(ref)
40
+ ref.strip.casecmp?("IEV") ? nil : super
41
+ end
42
+
37
43
  # @param xml [String]
38
44
  # @return [Relaton::Iec::ItemData]
39
45
  def from_xml(xml)
@@ -57,7 +63,12 @@ module Relaton
57
63
 
58
64
  # @param code [String]
59
65
  # @return [String, nil]
66
+ # Only a URN: `Relaton::Iec.urn_to_code` splits on `:`, so it would
67
+ # rewrite any reference with enough colons
68
+ # (`IEC 60034-1:1969+AMD1:1977+AMD2:1979+AMD3:1980 CSV`).
60
69
  def urn_to_code(code)
70
+ return unless code.match?(/\Aurn:/i)
71
+
61
72
  require_relative "../iec"
62
73
  Relaton::Iec.urn_to_code code
63
74
  end
data/lib/relaton/iec.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  require "digest/md5"
2
2
  require "net/http"
3
- require "nokogiri"
3
+ require "moxml"
4
4
  require "pubid"
5
5
  require "zip"
6
6
  require "relaton/index"
@@ -7,11 +7,12 @@ module Relaton
7
7
  #
8
8
  # Search IEEE bibliography item by reference.
9
9
  #
10
- # @param code [String]
10
+ # @param ref [String, Pubid::Ieee::Identifier] the reference, or its
11
+ # parse from Relaton::Db (relaton#205)
11
12
  #
12
13
  # @return [Relaton::Ieee::ItemData, nil]
13
14
  #
14
- def search(code) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
15
+ def search(ref) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
15
16
  index = Relaton::Index.find_or_create :ieee, url: "#{GH_URL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
16
17
  pubid_class: ::Pubid::Ieee::Identifier
17
18
  # Pass the parsed pubid (not the raw String) so index-v2 narrows
@@ -20,7 +21,7 @@ module Relaton
20
21
  # unparseable/partial ref falls back to the full-scan String search.
21
22
  # Rows are Pubid::Ieee::Identifier objects (not Comparable), so pick by
22
23
  # the string form.
23
- pubid = parse_pubid code
24
+ pubid = parse_pubid ref
24
25
  needle = pubid.to_s
25
26
  row = index.search(pubid) { |r| r[:id].to_s.include?(needle) }.min_by { |r| r[:id].to_s }
26
27
  return unless row
@@ -36,20 +37,22 @@ module Relaton
36
37
  #
37
38
  # Get IEEE bibliography item by reference.
38
39
  #
39
- # @param code [String] the IEEE standard Code to look up (e..g "528-2019")
40
+ # @param ref [String, Pubid::Ieee::Identifier] the IEEE standard
41
+ # reference to look up (e.g. "IEEE 528-2019"), or its parse from
42
+ # Relaton::Db (relaton#205)
40
43
  # @param year [String] the year the standard was published (optional)
41
44
  # @param opts [Hash] options
42
45
  #
43
46
  # @return [Relaton::Ieee::ItemData, nil]
44
47
  #
45
- def get(code, _year = nil, _opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
46
- Util.info "Fetching from Relaton repository ...", key: code
47
- item = search(code)
48
+ def get(ref, _year = nil, _opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
49
+ Util.info "Fetching from Relaton repository ...", key: ref.to_s
50
+ item = search(ref)
48
51
  if item
49
- Util.info "Found: `#{item.docidentifier.first.content}`", key: code
52
+ Util.info "Found: `#{item.docidentifier.first.content}`", key: ref.to_s
50
53
  item
51
54
  else
52
- Util.info "Not found.", key: code
55
+ Util.info "Not found.", key: ref.to_s
53
56
  nil
54
57
  end
55
58
  end
@@ -60,12 +63,15 @@ module Relaton
60
63
  # return the raw String when pubid can't parse it (e.g. a partial ref) so
61
64
  # the search falls back to the substring scan.
62
65
  #
63
- # @param code [String]
66
+ # @param ref [String, ::Pubid::Ieee::Identifier]
64
67
  # @return [::Pubid::Ieee::Identifier, String]
65
- def parse_pubid(code)
66
- ::Pubid::Ieee::Identifier.parse code
68
+ def parse_pubid(ref)
69
+ # A pubid from Relaton::Db (relaton#205) is used as it is.
70
+ return ref unless ref.is_a?(String)
71
+
72
+ ::Pubid::Ieee::Identifier.parse ref
67
73
  rescue StandardError
68
- code
74
+ ref
69
75
  end
70
76
  end
71
77
  end
@@ -147,7 +147,7 @@ module Relaton
147
147
  # IdamsParser#parse_relation, so mutates crossrefs under a mutex.
148
148
  #
149
149
  # @param [String] docnumber of main document
150
- # @param [Nokogiri::XML::Element] amsid relation data
150
+ # @param [Moxml::Element] amsid relation data
151
151
  #
152
152
  def add_crossref(docnumber, amsid)
153
153
  return if RELATION_TYPES[amsid.type] == false
@@ -36,6 +36,30 @@ module Relaton
36
36
  DataFetcher.fetch(source, **opts)
37
37
  end
38
38
 
39
+ # `get` reads a reference pubid cannot parse as a miss, so it gets no key
40
+ # here either: it is not cached.
41
+ def cache_pubid(ref)
42
+ super
43
+ rescue StandardError
44
+ nil
45
+ end
46
+
47
+ # An IEEE pubid keeps the other publishers in `publishers` (a joint
48
+ # development) or `copublisher` (a standard), not in the shared
49
+ # `copublishers`.
50
+ #
51
+ # @param pubid [Pubid::Ieee::Identifier]
52
+ # @return [Array<String>]
53
+ def copublishers(pubid)
54
+ pubid = pubid.identifiers.first if pubid.all_parts?
55
+ others = if pubid.respond_to?(:publishers) && pubid.publishers
56
+ pubid.publishers
57
+ elsif pubid.respond_to?(:copublisher)
58
+ pubid.copublisher
59
+ end
60
+ Array(others).map(&:to_s) - %w[IEEE AIEE]
61
+ end
62
+
39
63
  # @param xml [String]
40
64
  # @return [Relaton::Ieee::ItemData]
41
65
  def from_xml(xml)
@@ -138,8 +138,8 @@ module Relaton
138
138
  Renderer.new(publisher: "IEEE", number: "P802", part: "11aj", draft: "8.0", year: "2017")
139
139
  when "IEEE P802.11ajD9.0, November 2017" # "IEEE P802-11aj/D9.0.2017"
140
140
  Renderer.new(publisher: "IEEE", number: "P802", part: "11aj", draft: "9.0", year: "2017")
141
- when "ISO/IEC/IEEE P29119-4-DISMay2013" # "ISO/IEC/IEEE DIS P29119-4.2013"
142
- Renderer.new(publisher: "ISO/IEC/IEEE", stage: "DIS", number: "P29119", part: "4", year: "2013")
141
+ when "ISO/IEC/IEEE P29119-4-DISMay2013" # "ISO/IEC/IEEE DIS P29119-4.2013-05"
142
+ Renderer.new(publisher: "ISO/IEC/IEEE", stage: "DIS", number: "P29119", part: "4", year: "2013", month: "05")
143
143
  when "IEEE-P15026-3-DIS-January 2015" # "IEEE DIS P15026-3.2015"
144
144
  Renderer.new(publisher: "IEEE", stage: "DIS", number: "P15026", year: "2015")
145
145
  when "ANSI/IEEE PC63.7/D rev17, December 2014" # "ANSI/IEEE PC63-7/D/REV-17.2014"
@@ -7,25 +7,27 @@ module Relaton
7
7
  # IETF bibliography module
8
8
  module Bibliography
9
9
  class << self
10
- # @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
10
+ # @param ref [String, Pubid::Ietf::Identifier] the IETF reference
11
+ # (e.g. "RFC 8341"), or its parse from Relaton::Db (relaton#205)
11
12
  # @return [RelatonIetf::IetfBibliographicItem]
12
- def search(text)
13
- Scraper.scrape_page text
13
+ def search(ref)
14
+ Scraper.scrape_page ref
14
15
  end
15
16
 
16
- # @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
17
+ # @param ref [String, Pubid::Ietf::Identifier] the IETF reference
18
+ # (e.g. "RFC 8341"), or its parse from Relaton::Db (relaton#205)
17
19
  # @param year [String] the year the standard was published (optional)
18
20
  # @param opts [Hash] options; restricted to :all_parts if all-parts
19
21
  # reference is required
20
22
  # @return [RelatonIetf::IetfBibliographicItem] Relaton of reference
21
- def get(code, _year = nil, _opts = {})
22
- Util.info "Fetching from Relaton repository ...", key: code
23
- result = search code
23
+ def get(ref, _year = nil, _opts = {})
24
+ Util.info "Fetching from Relaton repository ...", key: ref.to_s
25
+ result = search ref
24
26
  if result
25
27
  docid = result.docidentifier.detect(&:primary) || result.docidentifier.first
26
- Util.info "Found: `#{docid.content}`", key: code
28
+ Util.info "Found: `#{docid.content}`", key: ref.to_s
27
29
  else
28
- Util.info "Not found.", key: code
30
+ Util.info "Not found.", key: ref.to_s
29
31
  end
30
32
  result
31
33
  end
@@ -332,7 +332,7 @@ module Relaton
332
332
  #
333
333
  # Get RFC index
334
334
  #
335
- # @return [Nokogiri::XML::Document] RFC index
335
+ # @return [Moxml::Document] RFC index
336
336
  #
337
337
  def rfc_index
338
338
  uri = URI "https://www.rfc-editor.org/rfc-index.xml"
@@ -6,6 +6,7 @@ module Relaton
6
6
  def initialize # rubocop:disable Lint/MissingSuper
7
7
  @short = :relaton_ietf
8
8
  @prefix = "IETF"
9
+ @pubid_identifier = :Ietf # Db cache key
9
10
  @defaultprefix = /^((IETF|RFC|BCP|FYI|STD)\s|I-D[.\s])/
10
11
  @idtype = "IETF"
11
12
  @datasets = %w[ietf-rfcsubseries ietf-internet-drafts ietf-rfc-entries]
@@ -195,8 +195,7 @@ module Relaton
195
195
  formattedref: build_formattedref,
196
196
  date: build_subseries_date(rfc_index),
197
197
  relation: build_relations(rfc_index, wg_names: wg_names),
198
- series: build_series,
199
- ext: Ext.new(doctype: Doctype.new(content: "rfc"), stream: stream, flavor: "ietf"),
198
+ ext: Ext.new(doctype: Doctype.new(content: "rfc"), stream: canonical_stream, flavor: "ietf"),
200
199
  )
201
200
  end
202
201
 
@@ -259,13 +258,6 @@ module Relaton
259
258
  ItemData.new(formattedref: Bib::Formattedref.new(content: ref), docidentifier: [docid])
260
259
  end
261
260
 
262
- def build_series
263
- return [] unless stream
264
-
265
- t = Bib::Title.new(content: stream)
266
- [Bib::Series.new(type: "stream", title: [t])]
267
- end
268
-
269
261
  # --- RFC entry builders ---
270
262
 
271
263
  def build_rfc_docid
@@ -388,7 +380,7 @@ module Relaton
388
380
  def build_rfc_series
389
381
  series = build_rfc_is_also_series
390
382
  series << Bib::Series.new(title: [Bib::Title.new(content: "RFC")], number: shortnum)
391
- series + build_rfc_stream_series
383
+ series
392
384
  end
393
385
 
394
386
  def build_rfc_is_also_series
@@ -401,27 +393,31 @@ module Relaton
401
393
  end
402
394
  end
403
395
 
404
- def build_rfc_stream_series
405
- return [] unless stream
406
-
407
- t = Bib::Title.new(content: stream)
408
- [Bib::Series.new(type: "stream", title: [t])]
409
- end
410
-
411
396
  STREAM_ORGS = {
412
397
  "IETF" => ["IETF", "Internet Engineering Task Force"],
413
398
  "IRTF" => ["IRTF", "Internet Research Task Force"],
414
399
  "IAB" => ["IAB", "Internet Architecture Board"],
415
400
  }.freeze
416
401
 
402
+ # rfc-index.xml spells the independent stream "INDEPENDENT";
403
+ # Ext.stream's declared values use "Independent". Match case-insensitively
404
+ # against the declared values so both Ext.stream and STREAM_ORGS see the
405
+ # canonical spelling.
406
+ def canonical_stream
407
+ return unless stream
408
+
409
+ values = Ext.attributes[:stream].options[:values]
410
+ values.find { |v| v.casecmp?(stream) } || stream
411
+ end
412
+
417
413
  def build_rfc_ext
418
- Ext.new(doctype: Doctype.new(content: "rfc"), stream: stream, flavor: "ietf")
414
+ Ext.new(doctype: Doctype.new(content: "rfc"), stream: canonical_stream, flavor: "ietf")
419
415
  end
420
416
 
421
417
  def build_committee_contributor(wg_names = {})
422
418
  return if wg_acronym.nil? || wg_acronym == "NON WORKING GROUP"
423
419
 
424
- abbr, name = STREAM_ORGS[stream]
420
+ abbr, name = STREAM_ORGS[canonical_stream]
425
421
  org = if abbr
426
422
  Ietf::BibXMLParser.build_org(abbr, name)
427
423
  else
@@ -15,17 +15,18 @@ module Relaton
15
15
  # publishing their `index-v1` for released relatons, untouched.
16
16
  IETF = "https://raw.githubusercontent.com/relaton/relaton-data-ietf/main/"
17
17
 
18
- # @param text [String]
18
+ # @param ref [String, Pubid::Ietf::Identifier] a reference, or its parse
19
+ # from Relaton::Db (relaton#205), which is used as it is
19
20
  # @return [Relaton::Ietf::ItemData, nil]
20
- def scrape_page(text)
21
- id = parse_id text.sub(/\AIETF\s+/, "")
21
+ def scrape_page(ref)
22
+ id = ref.is_a?(String) ? parse_id(ref.sub(/\AIETF\s+/, "")) : ref
22
23
  return unless id
23
24
 
24
25
  fetch_doc id
25
26
  rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
26
27
  Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
27
28
  Net::ProtocolError, SocketError
28
- raise Relaton::RequestError, "No document found for #{text} reference"
29
+ raise Relaton::RequestError, "No document found for #{ref} reference"
29
30
  end
30
31
 
31
32
  private
@@ -8,6 +8,7 @@ module Relaton
8
8
  def initialize
9
9
  @short = :relaton_iho
10
10
  @prefix = "IHO"
11
+ @pubid_identifier = :Iho # Db cache key
11
12
  @defaultprefix = %r{^IHO\s}
12
13
  @idtype = "IHO"
13
14
  end
@@ -28,11 +28,11 @@ module Relaton
28
28
  # if nil then the fiename is used to read and write file (used to create indes in GH actions)
29
29
  # @param [Pubid::Identifier] pubid class for deserialization
30
30
  #
31
- # `id_keys` is accepted for backward compatibility but no longer used: the
31
+ # The index format check round-trips each id through `pubid_class`;
32
32
  # index format is now validated by round-tripping a sample of ids through
33
33
  # the pubid class (see #check_serialization), which understands the pubid
34
34
  # v2 (lutaml) `_type` serialization that the old key-allowlist could not.
35
- def initialize(dir, url, filename, _id_keys = nil, pubid_class = nil)
35
+ def initialize(dir, url: nil, filename: nil, pubid_class: nil)
36
36
  @dir = dir
37
37
  @url = url
38
38
  @filename = filename
@@ -14,7 +14,6 @@ module Relaton
14
14
  # @param [String] type <description>
15
15
  # @param [String, nil] url external URL to index, used to fetch index for searching files
16
16
  # @param [String, nil] file output file name
17
- # @param [Array<Symbol>, nil] id_keys keys to check if index is correct
18
17
  # @param [String, nil] pages_url base URL of the Pages site that serves
19
18
  # the machine index (manifest + shards), see Relaton::Index::ShardSource
20
19
  #
@@ -24,9 +23,11 @@ module Relaton
24
23
  if @pool[type.upcase.to_sym]&.actual?(**args)
25
24
  @pool[type.upcase.to_sym]
26
25
  else
26
+ if args.key?(:id_keys)
27
+ Util.warn "id_keys is deprecated and ignored by Relaton::Index"
28
+ end
27
29
  @pool[type.upcase.to_sym] = Type.new(
28
- type, args[:url], args[:file], args[:id_keys], args[:pubid_class],
29
- pages_url: args[:pages_url]
30
+ type, **args.slice(:url, :file, :pubid_class, :pages_url)
30
31
  )
31
32
  end
32
33
  end
@@ -47,7 +47,7 @@ module Relaton
47
47
  @pages_url = pages_url.end_with?("/") ? pages_url : "#{pages_url}/"
48
48
  # Only the deserialization helpers are used: this FileIO reads and
49
49
  # writes no file.
50
- @file_io = FileIO.new(dir, nil, nil, nil, pubid_class)
50
+ @file_io = FileIO.new(dir, pubid_class: pubid_class)
51
51
  @mutex = Mutex.new
52
52
  @state = nil
53
53
  end
@@ -10,21 +10,18 @@ module Relaton
10
10
  # @param [String, Symbol] type type of index (ISO, IEC, etc.)
11
11
  # @param [String, nil] url external URL to index, used to fetch index for searching files
12
12
  # @param [String, nil] file output file name
13
- # @param [Array<Symbol>] id_keys keys of identifier to be used for sorting index
14
- # format of index file is checked if id_keys all is provided at least in one of the IDs
15
13
  # @param [Pubid::Identifier, nil] pubid class for deserialization
16
14
  # @param [String, nil] pages_url base URL of the Pages site that serves
17
15
  # the machine index. With it the type reads the index from there, in
18
16
  # memory (see ShardSource), and a parsed query fetches one shard only.
19
17
  #
20
- def initialize(type, url = nil, file = nil, id_keys = nil, pubid_class = nil, # rubocop:disable Metrics/ParameterLists
21
- pages_url: nil)
18
+ def initialize(type, url: nil, file: nil, pubid_class: nil, pages_url: nil)
22
19
  @file = file
23
20
  @dir = type.to_s.downcase
24
21
  @pubid_class = pubid_class
25
22
  @pages_url = pages_url
26
23
  filename = file || Index.config.filename
27
- @file_io = FileIO.new @dir, url, filename, id_keys, pubid_class
24
+ @file_io = FileIO.new @dir, url: url, filename: filename, pubid_class: pubid_class
28
25
  @source = new_source
29
26
  end
30
27