relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. checksums.yaml +4 -4
  2. data/README.adoc +57 -1
  3. data/bin/console +0 -1
  4. data/lib/relaton/3gpp/bibliography.rb +82 -7
  5. data/lib/relaton/3gpp/data_fetcher.rb +51 -3
  6. data/lib/relaton/3gpp/docidentifier.rb +114 -0
  7. data/lib/relaton/3gpp/item.rb +6 -0
  8. data/lib/relaton/3gpp/parser.rb +1 -1
  9. data/lib/relaton/3gpp/processor.rb +4 -1
  10. data/lib/relaton/3gpp.rb +5 -1
  11. data/lib/relaton/adobe/bibdata.rb +8 -0
  12. data/lib/relaton/adobe/bibitem.rb +8 -0
  13. data/lib/relaton/adobe/bibliography.rb +92 -0
  14. data/lib/relaton/adobe/docidentifier.rb +49 -0
  15. data/lib/relaton/adobe/doctype.rb +14 -0
  16. data/lib/relaton/adobe/ext.rb +32 -0
  17. data/lib/relaton/adobe/item.rb +15 -0
  18. data/lib/relaton/adobe/item_base.rb +18 -0
  19. data/lib/relaton/adobe/item_data.rb +6 -0
  20. data/lib/relaton/adobe/processor.rb +45 -0
  21. data/lib/relaton/adobe/util.rb +8 -0
  22. data/lib/relaton/adobe.rb +37 -0
  23. data/lib/relaton/bib/model/address.rb +2 -2
  24. data/lib/relaton/bib/model/docidentifier.rb +24 -9
  25. data/lib/relaton/bib/model/localized_string.rb +1 -1
  26. data/lib/relaton/bib/model/structured_identifier.rb +10 -9
  27. data/lib/relaton/bib/sanitizer.rb +202 -6
  28. data/lib/relaton/bib.rb +0 -2
  29. data/lib/relaton/bipm/bibliography.rb +159 -10
  30. data/lib/relaton/bipm/data_fetcher.rb +26 -2
  31. data/lib/relaton/bipm/data_outcomes_parser.rb +3 -4
  32. data/lib/relaton/bipm/id_parser.rb +5 -4
  33. data/lib/relaton/bipm/model/structured_identifier.rb +21 -0
  34. data/lib/relaton/bipm/processor.rb +2 -2
  35. data/lib/relaton/bipm/rawdata_bipm_metrologia/fetcher.rb +2 -4
  36. data/lib/relaton/bipm/si_brochure_parser.rb +5 -3
  37. data/lib/relaton/bipm.rb +7 -1
  38. data/lib/relaton/bsi/bibliography.rb +115 -43
  39. data/lib/relaton/bsi/hit.rb +14 -0
  40. data/lib/relaton/bsi/hit_collection.rb +15 -16
  41. data/lib/relaton/bsi/model/docidentifier.rb +99 -1
  42. data/lib/relaton/bsi/processor.rb +1 -0
  43. data/lib/relaton/calconnect/bibliography.rb +12 -14
  44. data/lib/relaton/calconnect/data_fetcher.rb +77 -9
  45. data/lib/relaton/calconnect/docidentifier.rb +80 -0
  46. data/lib/relaton/calconnect/hit_collection.rb +65 -57
  47. data/lib/relaton/calconnect/model/item.rb +7 -0
  48. data/lib/relaton/calconnect/processor.rb +7 -1
  49. data/lib/relaton/calconnect.rb +11 -1
  50. data/lib/relaton/ccsds/data/fetcher.rb +17 -12
  51. data/lib/relaton/ccsds/data/parser.rb +1 -1
  52. data/lib/relaton/ccsds/hit_collection.rb +6 -1
  53. data/lib/relaton/ccsds/model/docidentifier.rb +121 -0
  54. data/lib/relaton/ccsds/model/item.rb +2 -0
  55. data/lib/relaton/cen/bibliography.rb +75 -47
  56. data/lib/relaton/cen/hit.rb +16 -1
  57. data/lib/relaton/cen/hit_collection.rb +59 -11
  58. data/lib/relaton/cen/model/docidentifier.rb +92 -1
  59. data/lib/relaton/cen/processor.rb +13 -8
  60. data/lib/relaton/cen/scraper.rb +13 -5
  61. data/lib/relaton/cen.rb +1 -0
  62. data/lib/relaton/cie/data_fetcher.rb +215 -30
  63. data/lib/relaton/cie/processor.rb +3 -1
  64. data/lib/relaton/cie/scrapper.rb +15 -2
  65. data/lib/relaton/cie.rb +2 -1
  66. data/lib/relaton/core/data_fetcher.rb +150 -3
  67. data/lib/relaton/core/governor.rb +320 -0
  68. data/lib/relaton/core/pacer.rb +134 -0
  69. data/lib/relaton/core/processor.rb +19 -0
  70. data/lib/relaton/core/request_error.rb +14 -0
  71. data/lib/relaton/core.rb +3 -0
  72. data/lib/relaton/db/registry.rb +41 -1
  73. data/lib/relaton/doi/crossref.rb +19 -2
  74. data/lib/relaton/doi/parser.rb +109 -15
  75. data/lib/relaton/easc/bibdata.rb +8 -0
  76. data/lib/relaton/easc/bibitem.rb +8 -0
  77. data/lib/relaton/easc/bibliography.rb +95 -0
  78. data/lib/relaton/easc/docidentifier.rb +100 -0
  79. data/lib/relaton/easc/doctype.rb +14 -0
  80. data/lib/relaton/easc/ext.rb +44 -0
  81. data/lib/relaton/easc/item.rb +13 -0
  82. data/lib/relaton/easc/item_base.rb +18 -0
  83. data/lib/relaton/easc/item_data.rb +6 -0
  84. data/lib/relaton/easc/processor.rb +46 -0
  85. data/lib/relaton/easc/util.rb +8 -0
  86. data/lib/relaton/easc.rb +35 -0
  87. data/lib/relaton/ecma/bibliography.rb +93 -25
  88. data/lib/relaton/ecma/data_fetcher.rb +71 -12
  89. data/lib/relaton/ecma/docidentifier.rb +124 -0
  90. data/lib/relaton/ecma/item.rb +2 -0
  91. data/lib/relaton/ecma/memento_parser.rb +1 -1
  92. data/lib/relaton/ecma/page_fetcher.rb +15 -3
  93. data/lib/relaton/ecma/parser_common.rb +2 -2
  94. data/lib/relaton/ecma/processor.rb +4 -1
  95. data/lib/relaton/ecma/standard_parser.rb +2 -2
  96. data/lib/relaton/ecma.rb +10 -1
  97. data/lib/relaton/etsi/bibliography.rb +67 -2
  98. data/lib/relaton/etsi/data_fetcher.rb +43 -4
  99. data/lib/relaton/etsi/processor.rb +3 -1
  100. data/lib/relaton/etsi.rb +2 -1
  101. data/lib/relaton/gb/bibliography.rb +55 -29
  102. data/lib/relaton/gb/docidentifier.rb +58 -9
  103. data/lib/relaton/gb/processor.rb +3 -0
  104. data/lib/relaton/gb/scraper.rb +27 -10
  105. data/lib/relaton/gost/bibdata.rb +8 -0
  106. data/lib/relaton/gost/bibitem.rb +8 -0
  107. data/lib/relaton/gost/bibliography.rb +107 -0
  108. data/lib/relaton/gost/docidentifier.rb +80 -0
  109. data/lib/relaton/gost/doctype.rb +16 -0
  110. data/lib/relaton/gost/ext.rb +46 -0
  111. data/lib/relaton/gost/item.rb +15 -0
  112. data/lib/relaton/gost/item_base.rb +18 -0
  113. data/lib/relaton/gost/item_data.rb +6 -0
  114. data/lib/relaton/gost/processor.rb +49 -0
  115. data/lib/relaton/gost/util.rb +8 -0
  116. data/lib/relaton/gost.rb +36 -0
  117. data/lib/relaton/iala/bibdata.rb +8 -0
  118. data/lib/relaton/iala/bibitem.rb +8 -0
  119. data/lib/relaton/iala/bibliography.rb +146 -0
  120. data/lib/relaton/iala/docidentifier.rb +89 -0
  121. data/lib/relaton/iala/doctype.rb +18 -0
  122. data/lib/relaton/iala/ext.rb +32 -0
  123. data/lib/relaton/iala/item.rb +21 -0
  124. data/lib/relaton/iala/item_base.rb +18 -0
  125. data/lib/relaton/iala/item_data.rb +6 -0
  126. data/lib/relaton/iala/processor.rb +43 -0
  127. data/lib/relaton/iala/relation.rb +7 -0
  128. data/lib/relaton/iala/util.rb +8 -0
  129. data/lib/relaton/iala.rb +35 -0
  130. data/lib/relaton/iana/bibliography.rb +67 -14
  131. data/lib/relaton/iana/data_fetcher.rb +35 -5
  132. data/lib/relaton/iana/processor.rb +3 -1
  133. data/lib/relaton/iana.rb +12 -1
  134. data/lib/relaton/iec/data_fetcher.rb +7 -1
  135. data/lib/relaton/iec/hit_collection.rb +1 -1
  136. data/lib/relaton/iec/model/docidentifier.rb +9 -5
  137. data/lib/relaton/iec/model/ext.rb +2 -2
  138. data/lib/relaton/iec/processor.rb +1 -0
  139. data/lib/relaton/ieee/bibliography.rb +25 -3
  140. data/lib/relaton/ieee/data_fetcher.rb +158 -17
  141. data/lib/relaton/ieee/idams_parser.rb +18 -11
  142. data/lib/relaton/ieee/processor.rb +4 -1
  143. data/lib/relaton/ieee/rawbib_id_parser.rb +291 -86
  144. data/lib/relaton/ieee.rb +2 -1
  145. data/lib/relaton/ietf/data_fetcher.rb +295 -12
  146. data/lib/relaton/ietf/processor.rb +7 -3
  147. data/lib/relaton/ietf/rfc/entry.rb +39 -3
  148. data/lib/relaton/ietf/scraper.rb +69 -36
  149. data/lib/relaton/ietf.rb +4 -1
  150. data/lib/relaton/iho/bibliography.rb +1 -1
  151. data/lib/relaton/iho/docidentifier.rb +1 -1
  152. data/lib/relaton/index/file_io.rb +11 -11
  153. data/lib/relaton/index/file_storage.rb +6 -1
  154. data/lib/relaton/index/pool.rb +6 -1
  155. data/lib/relaton/index/shard_source.rb +201 -0
  156. data/lib/relaton/index/type.rb +63 -12
  157. data/lib/relaton/index.rb +2 -1
  158. data/lib/relaton/iso/bibliography.rb +20 -15
  159. data/lib/relaton/iso/data_fetcher.rb +3 -3
  160. data/lib/relaton/iso/data_parser.rb +17 -3
  161. data/lib/relaton/iso/hit_collection.rb +27 -15
  162. data/lib/relaton/iso/item_data.rb +22 -0
  163. data/lib/relaton/iso/model/docidentifier.rb +24 -12
  164. data/lib/relaton/iso/processor.rb +1 -0
  165. data/lib/relaton/iso/scraper.rb +19 -3
  166. data/lib/relaton/itu/bibliography.rb +9 -4
  167. data/lib/relaton/itu/data_crawler_r.rb +664 -0
  168. data/lib/relaton/itu/data_fetcher.rb +496 -50
  169. data/lib/relaton/itu/data_merge_r.rb +149 -0
  170. data/lib/relaton/itu/data_parser_r.rb +163 -89
  171. data/lib/relaton/itu/data_parser_t.rb +228 -0
  172. data/lib/relaton/itu/family_cache.rb +177 -0
  173. data/lib/relaton/itu/governor.rb +56 -0
  174. data/lib/relaton/itu/hit.rb +9 -3
  175. data/lib/relaton/itu/hit_collection.rb +258 -86
  176. data/lib/relaton/itu/model/docidentifier.rb +67 -1
  177. data/lib/relaton/itu/model/structured_identifier.rb +19 -0
  178. data/lib/relaton/itu/processor.rb +10 -4
  179. data/lib/relaton/itu/pubid.rb +27 -5
  180. data/lib/relaton/itu/recommendation_fields.rb +334 -0
  181. data/lib/relaton/itu/recommendation_parser.rb +18 -149
  182. data/lib/relaton/itu/scraper.rb +13 -3
  183. data/lib/relaton/itu.rb +2 -1
  184. data/lib/relaton/jcgm/bibdata.rb +8 -0
  185. data/lib/relaton/jcgm/bibitem.rb +8 -0
  186. data/lib/relaton/jcgm/bibliography.rb +97 -0
  187. data/lib/relaton/jcgm/data_fetcher.rb +81 -0
  188. data/lib/relaton/jcgm/docidentifier.rb +102 -0
  189. data/lib/relaton/jcgm/doctype.rb +12 -0
  190. data/lib/relaton/jcgm/ext.rb +23 -0
  191. data/lib/relaton/jcgm/item.rb +20 -0
  192. data/lib/relaton/jcgm/item_base.rb +18 -0
  193. data/lib/relaton/jcgm/item_data.rb +6 -0
  194. data/lib/relaton/jcgm/meetings_parser.rb +175 -0
  195. data/lib/relaton/jcgm/processor.rb +71 -0
  196. data/lib/relaton/jcgm/relation.rb +9 -0
  197. data/lib/relaton/jcgm/structured_identifier.rb +40 -0
  198. data/lib/relaton/jcgm/util.rb +8 -0
  199. data/lib/relaton/jcgm.rb +24 -0
  200. data/lib/relaton/jis/bibliography.rb +8 -10
  201. data/lib/relaton/jis/data_fetcher.rb +21 -19
  202. data/lib/relaton/jis/docidentifier.rb +104 -5
  203. data/lib/relaton/jis/hit.rb +18 -23
  204. data/lib/relaton/jis/hit_collection.rb +19 -18
  205. data/lib/relaton/jis/processor.rb +1 -1
  206. data/lib/relaton/jis.rb +2 -3
  207. data/lib/relaton/logger/channels/gh_issue.rb +78 -13
  208. data/lib/relaton/nist/data_fetcher.rb +63 -13
  209. data/lib/relaton/nist/docidentifier.rb +165 -0
  210. data/lib/relaton/nist/item.rb +2 -0
  211. data/lib/relaton/nist/item_base.rb +16 -0
  212. data/lib/relaton/nist/mods_parser.rb +38 -12
  213. data/lib/relaton/nist/processor.rb +2 -1
  214. data/lib/relaton/nist/relation.rb +3 -0
  215. data/lib/relaton/nist/scraper.rb +6 -3
  216. data/lib/relaton/oasis/bibliography.rb +147 -6
  217. data/lib/relaton/oasis/data_fetcher.rb +41 -5
  218. data/lib/relaton/oasis/data_parser_utils.rb +37 -3
  219. data/lib/relaton/oasis/docidentifier.rb +54 -0
  220. data/lib/relaton/oasis/item.rb +3 -0
  221. data/lib/relaton/oasis/processor.rb +7 -1
  222. data/lib/relaton/oasis.rb +14 -1
  223. data/lib/relaton/ogc/data_fetcher.rb +23 -2
  224. data/lib/relaton/ogc/docidentifier.rb +105 -0
  225. data/lib/relaton/ogc/hit_collection.rb +78 -3
  226. data/lib/relaton/ogc/processor.rb +2 -1
  227. data/lib/relaton/ogc.rb +5 -1
  228. data/lib/relaton/oiml/bibliography.rb +90 -15
  229. data/lib/relaton/oiml/docidentifier.rb +18 -3
  230. data/lib/relaton/omg/docidentifier.rb +67 -0
  231. data/lib/relaton/omg/item.rb +1 -0
  232. data/lib/relaton/omg/processor.rb +1 -0
  233. data/lib/relaton/omg/scraper.rb +61 -16
  234. data/lib/relaton/omg.rb +1 -0
  235. data/lib/relaton/plateau/bibliography.rb +10 -3
  236. data/lib/relaton/plateau/data_fetcher.rb +25 -2
  237. data/lib/relaton/plateau/handbook_parser.rb +8 -1
  238. data/lib/relaton/plateau/hit.rb +10 -2
  239. data/lib/relaton/plateau/hit_collection.rb +31 -11
  240. data/lib/relaton/plateau/processor.rb +3 -1
  241. data/lib/relaton/plateau/technical_report_parser.rb +8 -1
  242. data/lib/relaton/plateau.rb +2 -1
  243. data/lib/relaton/sdo/config.rb +34 -0
  244. data/lib/relaton/sdo/fetcher.rb +52 -0
  245. data/lib/relaton/sdo/logo.rb +95 -0
  246. data/lib/relaton/sdo/name.rb +26 -0
  247. data/lib/relaton/sdo/organization.rb +71 -0
  248. data/lib/relaton/sdo/store.rb +49 -0
  249. data/lib/relaton/sdo.rb +29 -0
  250. data/lib/relaton/version.rb +1 -1
  251. data/lib/relaton/w3c/bibliography.rb +132 -12
  252. data/lib/relaton/w3c/data_fetcher.rb +194 -16
  253. data/lib/relaton/w3c/data_parser.rb +3 -3
  254. data/lib/relaton/w3c/docidentifier.rb +48 -0
  255. data/lib/relaton/w3c/governor.rb +32 -0
  256. data/lib/relaton/w3c/item.rb +3 -0
  257. data/lib/relaton/w3c/pubid.rb +12 -0
  258. data/lib/relaton/w3c/safe_realize.rb +110 -21
  259. data/lib/relaton/w3c.rb +12 -1
  260. data/lib/relaton/xsf/bibliography.rb +61 -1
  261. data/lib/relaton/xsf/data_fetcher.rb +55 -5
  262. data/lib/relaton/xsf/docidentifier.rb +46 -0
  263. data/lib/relaton/xsf/hit_collection.rb +31 -3
  264. data/lib/relaton/xsf/item.rb +6 -0
  265. data/lib/relaton/xsf/processor.rb +1 -0
  266. data/lib/relaton/xsf.rb +5 -1
  267. data/lib/relaton.rb +42 -0
  268. metadata +135 -24
  269. data/lib/relaton/ieee/pub_id.rb +0 -161
  270. data/lib/relaton/index/id_number.rb +0 -30
@@ -2,33 +2,42 @@
2
2
 
3
3
  module Relaton
4
4
  module Iana
5
- # Methods for search IANA standards.
5
+ # Retrieval front-end for the IANA flavor. Parses the citation with
6
+ # Pubid::Iana, looks it up in the pubid-structured `relaton-data-iana`
7
+ # index-v2, and fetches the matching per-document YAML over HTTP.
8
+ #
9
+ # An IANA entry is a protocol registry, not a numbered standard: the
10
+ # identifier is a registry slug with at most one "/"
11
+ # (`IANA _6lowpan-parameters`, `IANA _6lowpan-parameters/lowpan_nhc`). The
12
+ # index rows key on the TOP-LEVEL slug (`Identifiers::Registry#number`), so
13
+ # a registry and all of its sub-registries share one bsearch bucket and the
14
+ # block below distinguishes them.
6
15
  module Bibliography
7
16
  SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-iana/refs/heads/v2/"
8
17
 
9
- # @param text [String]
10
- # @return [Relaton::Iana::ItemData, nil]
11
- def search(text) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
12
- index = Relaton::Index.find_or_create :iana, url: "#{SOURCE}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
13
- id = text.sub(/^IANA\s/, "")
14
- row = index.search(id).min_by { |i| i[:id] }
15
- return unless row
18
+ # @param text [String, Pubid::Iana::Identifier]
19
+ # @return [Relaton::Iana::Item, nil]
20
+ def search(text)
21
+ pubid = parse_ref text
22
+ return unless pubid
16
23
 
17
- url = "#{SOURCE}#{row[:file]}"
18
- resp = Net::HTTP.get_response URI(url)
19
- return unless resp.code == "200"
24
+ # Pass the pubid so Relaton::Index narrows candidates by number via
25
+ # binary search before applying the block. The match is exact, so there
26
+ # is at most one row.
27
+ row = index.search(pubid) { |r| pubid_match? r[:id], pubid }.first
28
+ return unless row
20
29
 
21
- Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
30
+ fetch_row row
22
31
  rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
23
32
  EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
24
33
  Net::ProtocolError, Errno::ETIMEDOUT => e
25
34
  raise Relaton::RequestError, e.message
26
35
  end
27
36
 
28
- # @param ref [String] the W3C standard Code to look up
37
+ # @param ref [String] the IANA registry slug to look up
29
38
  # @param year [String, NilClass] not used
30
39
  # @param opts [Hash] options
31
- # @return [Relaton::Iana::ItemData, nil]
40
+ # @return [Relaton::Iana::Item, nil]
32
41
  def get(ref, _year = nil, _opts = {})
33
42
  Util.info "Fetching from Relaton repository ...", key: ref
34
43
  result = search(ref)
@@ -41,6 +50,50 @@ module Relaton
41
50
  result
42
51
  end
43
52
 
53
+ private
54
+
55
+ def index
56
+ Relaton::Index.find_or_create :iana, url: "#{SOURCE}#{INDEXFILE}.zip",
57
+ file: "#{INDEXFILE}.yaml",
58
+ pubid_class: ::Pubid::Iana::Identifier
59
+ end
60
+
61
+ # `Pubid::Iana::Identifier.parse` accepts both the printed `IANA <slug>`
62
+ # form and the bare slug the index rows render back to. It raises a plain
63
+ # RuntimeError on anything else, and a citation relaton cannot parse is a
64
+ # miss, not an error — `#get` reports it as "Not found.".
65
+ #
66
+ # @param text [String, Pubid::Iana::Identifier]
67
+ # @return [Pubid::Iana::Identifier, nil]
68
+ def parse_ref(text)
69
+ return text unless text.is_a? String
70
+
71
+ ::Pubid::Iana::Identifier.parse text
72
+ rescue StandardError => e
73
+ Util.warn "Failed to parse pubid `#{text}`: #{e.message}"
74
+ nil
75
+ end
76
+
77
+ # Both arguments are Pubid::Iana identifiers. Compared field by field
78
+ # rather than with `==` so the two identity fields are explicit: a bare
79
+ # registry must NOT match its own sub-registries, which share its bsearch
80
+ # bucket. (index-v1 matched by substring, so `IANA rpki` also hit
81
+ # `rpki/signed-objects` and `min_by` picked the shortest id.)
82
+ def pubid_match?(row_id, query)
83
+ row_id.class == query.class &&
84
+ row_id.number == query.number &&
85
+ row_id.sub_registry.to_s == query.sub_registry.to_s
86
+ end
87
+
88
+ # @param row [Hash] an index row
89
+ # @return [Relaton::Iana::Item, nil]
90
+ def fetch_row(row)
91
+ resp = Net::HTTP.get_response URI("#{SOURCE}#{row[:file]}")
92
+ return unless resp.code == "200"
93
+
94
+ Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
95
+ end
96
+
44
97
  extend self
45
98
  end
46
99
  end
@@ -26,7 +26,28 @@ module Relaton
26
26
  private
27
27
 
28
28
  def index
29
- @index ||= Relaton::Index.find_or_create :iana, file: "#{INDEXFILE}.yaml"
29
+ @index ||= Relaton::Index.find_or_create :iana, file: "#{INDEXFILE}.yaml",
30
+ pubid_class: ::Pubid::Iana::Identifier
31
+ end
32
+
33
+ #
34
+ # Add a document to the pubid-keyed index-v2.
35
+ #
36
+ # Stores the Pubid::Iana identifier object (not its hash): with
37
+ # pubid_class set, Relaton::Index sorts the index by the id's number on
38
+ # save and serializes each id to its `_type: pubid:iana:registry` hash.
39
+ #
40
+ # The rescue is load-bearing. Relaton::Index rejects the WHOLE index if a
41
+ # single row fails to deserialize, so a slug pubid cannot parse has to be
42
+ # skipped here rather than raise and abort the crawl.
43
+ #
44
+ # @param [String] docnumber bare registry slug, `registry[/sub-registry]`
45
+ # @param [String] file path of the document file
46
+ #
47
+ def add_to_index(docnumber, file)
48
+ index.add_or_update ::Pubid::Iana::Identifier.parse(docnumber), file
49
+ rescue StandardError => e
50
+ Util.warn "Skipping index entry for `#{docnumber}` (#{file}): #{e.message}"
30
51
  end
31
52
 
32
53
  def parse(content)
@@ -45,13 +66,22 @@ module Relaton
45
66
  def save_doc(bib) # rubocop:disable Metrics/MethodLength
46
67
  return unless bib
47
68
 
48
- file = output_file(bib.docnumber)
69
+ # Two distinct registries can sanitize to one filename (`/` and `-` both
70
+ # collapse to `-`), e.g. `rpki/signed-objects` and `rpki-signed-objects`.
71
+ # Take a path of our own rather than overwriting the other document, but
72
+ # keep the clash visible in the crawl log.
73
+ file = unique_output_file bib.docnumber
49
74
  if @files.include? file
75
+ # A reserved path only ever belongs to one docnumber, so this is the
76
+ # same document again: a genuine duplicate, not a collision. Checked
77
+ # FIRST, because a disambiguated path stays != output_file forever.
50
78
  Util.warn "File #{file} already exists. Document: #{bib.docnumber}"
51
- else
52
- @files << file
79
+ elsif file != output_file(bib.docnumber)
80
+ Util.warn "File #{output_file bib.docnumber} already exists. " \
81
+ "Document: #{bib.docnumber}. Writing #{file} instead."
53
82
  end
54
- index.add_or_update bib.docnumber, file
83
+ @files << file
84
+ add_to_index bib.docnumber, file
55
85
  File.write file, serialize(bib), encoding: "UTF-8"
56
86
  end
57
87
 
@@ -61,7 +61,9 @@ module Relaton
61
61
  #
62
62
  def remove_index_file
63
63
  require_relative "../iana"
64
- Relaton::Index.find_or_create(:iana, url: true, file: "#{INDEXFILE}.yaml").remove_file
64
+ Relaton::Index.find_or_create(
65
+ :iana, url: true, file: "#{INDEXFILE}.yaml"
66
+ ).remove_file
65
67
  end
66
68
  end
67
69
  end
data/lib/relaton/iana.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require "faraday"
4
4
  require "relaton/index"
5
+ require "pubid"
5
6
  require "relaton/bib"
6
7
  require_relative "version"
7
8
  require_relative "iana/util"
@@ -15,7 +16,17 @@ require_relative "iana/bibliography"
15
16
 
16
17
  module Relaton
17
18
  module Iana
18
- INDEXFILE = "index-v1"
19
+ # The runtime index is the pubid-backed `index-v2`, whose rows are
20
+ # `Pubid::Iana::Identifiers::Registry` identifiers serialized as
21
+ # `_type: pubid:iana:registry` with `number` (the TOP-LEVEL registry slug)
22
+ # and an optional `sub_registry` — built and read via
23
+ # `pubid_class: ::Pubid::Iana::Identifier`.
24
+ #
25
+ # The legacy string-keyed `index-v1` (whose `:id` was the bare
26
+ # `registry[/sub-registry]` slug) is no longer produced or read here; it is
27
+ # emitted by `relaton-data-iana`'s own `crawler.rb`, derived from `index-v2`,
28
+ # so released gem lines keep resolving. See lib/relaton/iana/CLAUDE.md.
29
+ INDEXFILE = "index-v2".freeze
19
30
 
20
31
  # Returns hash of XML reammar
21
32
  # @return [String]
@@ -187,9 +187,15 @@ module Relaton
187
187
  def fetch_pub(pub) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
188
188
  bib = DataParser.new(pub, @errors).parse
189
189
  did = bib.docidentifier.detect(&:primary)
190
- file = output_file(did.to_s)
190
+ # Distinct docids can sanitize to one filename; take a path of our own
191
+ # rather than overwriting the other document. A reserved path only ever
192
+ # belongs to one docid, so an @files hit is the same document again.
193
+ file = unique_output_file(did.to_s)
191
194
  if @files.include? file then Util.warn "File #{file} exists."
192
195
  else
196
+ if file != output_file(did.to_s)
197
+ Util.warn "File #{output_file did.to_s} exists. Docid: #{did}. Writing #{file} instead."
198
+ end
193
199
  @files << file
194
200
  pubid = parse_pubid(did.to_s)
195
201
  index.add_or_update pubid, file if pubid
@@ -91,7 +91,7 @@ module Relaton
91
91
  # removed at every nesting level. More robust than comparing attribute
92
92
  # objects, whose derived `type`/`stage` differ between a parsed identifier
93
93
  # (component object) and a deserialized one (nil/symbol) — at the top level
94
- # and inside `base_identifier`.
94
+ # and inside `base`.
95
95
  def canonical_id(pubid, exclude)
96
96
  drop = exclude.filter_map { |e| EXCLUDE_KEYS[e] }
97
97
  prune(stringify(pubid.to_hash), drop)
@@ -50,13 +50,17 @@ module Relaton
50
50
  content.to_s
51
51
  end
52
52
 
53
+ # `remove_part!`/`remove_date!` are unnecessary here: `to_all_parts`'s
54
+ # own identity computation already excludes part/date (down the whole
55
+ # base chain, so a supplement's own year is dropped too — a disclosed
56
+ # pubid behavior change, not a bug; see spec/iec/…/docidentifier_spec.rb
57
+ # "#to_all_parts!"). `remove_stage!` stays: stage isn't one of the
58
+ # attributes pubid's `all_parts_edition_keys` strips.
53
59
  def to_all_parts!
54
60
  return unless @pubid
55
61
 
56
- remove_part!
57
- remove_date!
58
62
  remove_stage!
59
- @pubid.all_parts = true if @pubid.respond_to?(:all_parts=)
63
+ @pubid = @pubid.to_all_parts
60
64
  refresh_content!
61
65
  end
62
66
 
@@ -97,7 +101,7 @@ module Relaton
97
101
  clear_attr_on(@pubid, attr)
98
102
  end
99
103
 
100
- node = @pubid.base_identifier
104
+ node = @pubid.base
101
105
  while node
102
106
  clear_attr_on(node, attr)
103
107
  # ConsolidatedIdentifier carries a sibling collection of bundled
@@ -109,7 +113,7 @@ module Relaton
109
113
  clear_attr_on(id, attr)
110
114
  end
111
115
  end
112
- node = node.base_identifier
116
+ node = node.base
113
117
  end
114
118
  refresh_content!
115
119
  end
@@ -17,7 +17,7 @@ module Relaton
17
17
  attribute :cen_processing, :boolean
18
18
  attribute :secretary, :string
19
19
  attribute :interest_to_committees, :string
20
- attribute :tc_sc_officers_note, :string, raw: true
20
+ attribute :tc_sc_officers_note, :string
21
21
 
22
22
  xml do
23
23
  map_element "horizontal", to: :horizontal
@@ -30,7 +30,7 @@ module Relaton
30
30
  map_element "cen-processing", to: :cen_processing
31
31
  map_element "secretary", to: :secretary
32
32
  map_element "interest-to-committees", to: :interest_to_committees
33
- map_element "tc-sc-officers-note", to: :tc_sc_officers_note
33
+ map_element "tc-sc-officers-note", to: :tc_sc_officers_note, raw: :content
34
34
  end
35
35
 
36
36
  key_value do
@@ -6,6 +6,7 @@ module Relaton
6
6
  def initialize
7
7
  @short = :relaton_iec
8
8
  @prefix = "IEC"
9
+ @pubid_flavor = :Iec # global prefixes sourced from Pubid::Iec.prefixes
9
10
  @defaultprefix = %r{^(IEC\s|CISPR\s|IEV($|\s))}
10
11
  @idtype = "IEC"
11
12
  @datasets = %w[iec-harmonized-all iec-harmonized-latest]
@@ -12,9 +12,17 @@ module Relaton
12
12
  # @return [Relaton::Ieee::ItemData, nil]
13
13
  #
14
14
  def search(code) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
15
- # ref = code.sub(/Std\s/i, "") # .gsub(/[\s,:\/]/, "_").squeeze("_").upcase
16
- index = Relaton::Index.find_or_create :ieee, url: "#{GH_URL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
17
- row = index.search(code).min_by { |r| r[:id] }
15
+ index = Relaton::Index.find_or_create :ieee, url: "#{GH_URL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
16
+ pubid_class: ::Pubid::Ieee::Identifier
17
+ # Pass the parsed pubid (not the raw String) so index-v2 narrows
18
+ # candidates by number via binary search before the block runs; the
19
+ # block keeps the broad substring match the string index gave, and an
20
+ # unparseable/partial ref falls back to the full-scan String search.
21
+ # Rows are Pubid::Ieee::Identifier objects (not Comparable), so pick by
22
+ # the string form.
23
+ pubid = parse_pubid code
24
+ needle = pubid.to_s
25
+ row = index.search(pubid) { |r| r[:id].to_s.include?(needle) }.min_by { |r| r[:id].to_s }
18
26
  return unless row
19
27
 
20
28
  resp = Faraday.get "#{GH_URL}#{row[:file]}"
@@ -45,6 +53,20 @@ module Relaton
45
53
  nil
46
54
  end
47
55
  end
56
+
57
+ private
58
+
59
+ # Parse a reference into a Pubid::Ieee::Identifier for index narrowing, or
60
+ # return the raw String when pubid can't parse it (e.g. a partial ref) so
61
+ # the search falls back to the substring scan.
62
+ #
63
+ # @param code [String]
64
+ # @return [::Pubid::Ieee::Identifier, String]
65
+ def parse_pubid(code)
66
+ ::Pubid::Ieee::Identifier.parse code
67
+ rescue StandardError
68
+ code
69
+ end
48
70
  end
49
71
  end
50
72
  end
@@ -20,6 +20,90 @@ module Relaton
20
20
  "E" => false, "B" => false, "W" => false
21
21
  }.freeze
22
22
 
23
+ # Publication subtypes that IEEE does not give a designation to. Such a
24
+ # record carries a category in `stdnumber` ("White Paper", "Smart Grid
25
+ # Research: …") and a title in `normtitle`, and IEEE itself cites it by
26
+ # title. See {#no_standard_number?}.
27
+ NON_STANDARD_SUBTYPES = ["Whitepapers", "Research Documents"].freeze
28
+
29
+ class << self
30
+ #
31
+ # Build the pubid-structured `index-v2` from a directory of already
32
+ # generated per-document BibYAML files (the output of {.fetch}). This is
33
+ # a **separate step** from doc fetching: `relaton-data-ieee`'s crawler
34
+ # runs it after obtaining `data/` (a fresh crawl, or a clone of the
35
+ # published data repo). It parses each document's primary IEEE
36
+ # docidentifier into a `Pubid::Ieee::Identifier` and writes the index
37
+ # keyed by `pubid_class: ::Pubid::Ieee::Identifier`, so rows serialize to
38
+ # the `_type: pubid:ieee:*` structured form.
39
+ #
40
+ # Ids that pubid can't parse/round-trip are skipped (they'd otherwise
41
+ # break the all-pubid index that FileIO deserializes and number-sorts);
42
+ # the per-file skip and the final coverage summary are logged so the loss
43
+ # is never silent.
44
+ #
45
+ # @param dir [String] directory holding the per-document `*.yaml` files
46
+ # @param index_file [String] output index filename
47
+ # @return [Hash] `{ total:, indexed:, skipped: }` counts
48
+ #
49
+ def build_index(dir: "data", index_file: "#{INDEXFILE}.yaml")
50
+ index = Relaton::Index.find_or_create(
51
+ :ieee, file: index_file, pubid_class: ::Pubid::Ieee::Identifier
52
+ )
53
+ total = indexed = 0
54
+ Dir["#{dir}/*.yaml"].sort.each do |file|
55
+ total += 1
56
+ docid = primary_docid(file)
57
+ pid = docid && pubid(docid)
58
+ if pid
59
+ index.add_or_update pid, file
60
+ indexed += 1
61
+ else
62
+ Util.warn "Skipped (unparseable id): `#{docid || '<none>'}` (#{file})"
63
+ end
64
+ end
65
+ index.save
66
+ Util.info "IEEE #{index_file}: #{indexed}/#{total} indexed, " \
67
+ "#{total - indexed} skipped " \
68
+ "(#{total.zero? ? 0 : (100.0 * indexed / total).round(1)}% coverage)"
69
+ { total: total, indexed: indexed, skipped: total - indexed }
70
+ end
71
+
72
+ private
73
+
74
+ # Extract the primary, non-trademark IEEE docidentifier string from a
75
+ # BibYAML file, or nil when absent/unreadable.
76
+ #
77
+ # @param file [String]
78
+ # @return [String, nil]
79
+ def primary_docid(file)
80
+ yaml = YAML.safe_load(File.read(file, encoding: "UTF-8"),
81
+ permitted_classes: [Symbol, Date, Time])
82
+ id = yaml && yaml["docidentifier"]&.find do |i|
83
+ i["type"] == "IEEE" && i["primary"] == true && i["trademark"].nil?
84
+ end
85
+ id && id["content"]
86
+ rescue StandardError
87
+ nil
88
+ end
89
+
90
+ # Parse a docid string into a Pubid::Ieee::Identifier, or nil if pubid
91
+ # can't parse it or the structured id won't round-trip through the index
92
+ # (matching FileIO's `id_supported?` acceptance test).
93
+ #
94
+ # @param id [String]
95
+ # @return [::Pubid::Ieee::Identifier, nil]
96
+ def pubid(id)
97
+ pid = ::Pubid::Ieee::Identifier.parse id
98
+ hash = pid.to_hash
99
+ return nil unless ::Pubid::Ieee::Identifier.from_hash(hash).to_hash == hash
100
+
101
+ pid
102
+ rescue StandardError
103
+ nil
104
+ end
105
+ end
106
+
23
107
  #
24
108
  # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
25
109
  #
@@ -229,10 +313,22 @@ module Relaton
229
313
 
230
314
  normtitle = cheap_extract_field(xml, "normtitle")
231
315
  stdnumber = cheap_extract_field(xml, "stdnumber")
316
+ # Drop it here as well as in #parse_entry: the winner selection groups
317
+ # files by docnumber, so a white paper that reads as `IEEE Std 802`
318
+ # would join the group of the real standard and push its file out.
319
+ subtype = cheap_extract_field(xml, "publicationsubtype")
320
+ return nil if no_standard_number?(subtype, stdnumber)
321
+
232
322
  docnumber = nil
233
323
  if normtitle && stdnumber
234
324
  pubid = RawbibIdParser.parse(normtitle, stdnumber)
235
- docnumber = pubid&.to_id
325
+ # The article title (what `IdamsParser#pubid` compares against) is
326
+ # the *last* `<title>` in the file: the top-level one (first) is the
327
+ # designation, present even for a real standard.
328
+ title = cheap_extract_field(xml, "title", last: true)
329
+ isbn = xml.include?("<isbn")
330
+ fabricated = RawbibIdParser.fabricated_title_id?(pubid, title, isbn)
331
+ docnumber = pubid&.to_s unless fabricated
236
332
  end
237
333
  [idx, file, docnumber, file.include?("/updates.")]
238
334
  rescue StandardError
@@ -241,9 +337,28 @@ module Relaton
241
337
  [idx, file, nil, file.include?("/updates.")]
242
338
  end
243
339
 
244
- def cheap_extract_field(xml, tag)
245
- m = xml.match(%r{<#{tag}[^>]*?>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</#{tag}>}m)
246
- m && m[1].strip
340
+ #
341
+ # True when the record has no standard number. IEEE gives a white paper
342
+ # or a research document no designation, so `stdnumber` holds a category
343
+ # and `normtitle` holds a title. An id built from either is a title
344
+ # fragment, and records that share a fragment overwrite each other's
345
+ # output file. Both conditions are required: a record of such a subtype
346
+ # that does carry a number keeps it.
347
+ #
348
+ # @param [String, nil] subtype document element "publicationsubtype"
349
+ # @param [String, nil] stdnumber document element "stdnumber"
350
+ #
351
+ # @return [Boolean]
352
+ #
353
+ def no_standard_number?(subtype, stdnumber)
354
+ NON_STANDARD_SUBTYPES.include?(subtype) && !stdnumber.to_s.match?(/\d/)
355
+ end
356
+
357
+ def cheap_extract_field(xml, tag, last: false)
358
+ re = %r{<#{tag}[^>]*?>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</#{tag}>}m
359
+ matches = xml.scan(re)
360
+ m = last ? matches.last : matches.first
361
+ m && m[0].strip
247
362
  end
248
363
 
249
364
  def select_prefilter_winners(index, total)
@@ -326,12 +441,16 @@ module Relaton
326
441
 
327
442
  # Stragglers: any remaining staged files (losing duplicates,
328
443
  # or bib filenames that didn't end up in saved_writes due to a
329
- # crash) get cleaned up so they don't pollute `data/`.
330
- Dir.glob(File.join(@output, "*.#{@ext}.*")).each do |f|
444
+ # crash) get cleaned up so they don't pollute `data/`. Log the
445
+ # count so an unexpectedly large sweep (e.g. a worker that lost
446
+ # its state) is visible instead of silent.
447
+ stragglers = Dir.glob(File.join(@output, "*.#{@ext}.*"))
448
+ stragglers.each do |f|
331
449
  File.unlink(f)
332
450
  rescue StandardError
333
451
  # ignore — best-effort cleanup
334
452
  end
453
+ Util.info "Reconcile: cleaned #{stragglers.size} staged straggler(s)" unless stragglers.empty?
335
454
  end
336
455
 
337
456
  #
@@ -407,12 +526,7 @@ module Relaton
407
526
  pid = Process.fork do
408
527
  batch_files.each_with_index do |file, i|
409
528
  glob_idx = base_idx + i
410
- result = parse_entry(glob_idx, file)
411
- next unless result
412
-
413
- _, _, doc, bib, local_errors = result
414
- merge_errors(local_errors)
415
- commit_doc(doc, bib, file, glob_idx)
529
+ commit_entry(glob_idx, file, glob_idx)
416
530
  end
417
531
  File.binwrite(state_path, Marshal.dump(
418
532
  backrefs: backrefs,
@@ -469,12 +583,32 @@ module Relaton
469
583
  def run_shard(shard, _shard_idx)
470
584
  shard.each_with_index do |entry, i|
471
585
  idx, file = entry.is_a?(Array) ? entry : [i, entry]
472
- result = parse_entry(idx, file)
473
- next unless result
586
+ commit_entry(idx, file)
587
+ end
588
+ end
474
589
 
475
- _, _, doc, bib, local_errors = result
476
- merge_errors(local_errors)
477
- commit_doc(doc, bib, file)
590
+ #
591
+ # Parse one file and commit it, guarding the commit so a single bad
592
+ # document (e.g. a pathological docnumber that would raise
593
+ # Errno::ENAMETOOLONG) logs and is skipped rather than aborting the
594
+ # serial crawl or killing a parallel worker before it persists its
595
+ # state. Shared by the serial (`run_shard`) and parallel
596
+ # (`spawn_batch`) paths.
597
+ #
598
+ # @param [Integer] idx original glob index (preserves dedup order)
599
+ # @param [String] file path to rawbib file
600
+ # @param [Integer, nil] glob_idx staging index for parallel mode
601
+ #
602
+ def commit_entry(idx, file, glob_idx = nil)
603
+ result = parse_entry(idx, file)
604
+ return unless result
605
+
606
+ _, _, doc, bib, local_errors = result
607
+ merge_errors(local_errors)
608
+ begin
609
+ commit_doc(doc, bib, file, glob_idx)
610
+ rescue StandardError => e
611
+ Util.error "commit failed for `#{file}`: #{e.class}: #{e.message}"
478
612
  end
479
613
  end
480
614
 
@@ -501,6 +635,13 @@ module Relaton
501
635
  end
502
636
  return nil if doc.publicationinfo&.standard_id == "0"
503
637
 
638
+ info = doc.publicationinfo
639
+ if no_standard_number?(info&.publicationsubtype, info&.stdnumber)
640
+ Util.warn "No standard number. Normtitle: `#{doc.normtitle}`, " \
641
+ "file: `#{file}`"
642
+ return nil
643
+ end
644
+
504
645
  local_errors = Hash.new(true)
505
646
  bib = IdamsParser.new(doc, self, local_errors).parse
506
647
  if bib.docnumber.nil?
@@ -43,23 +43,28 @@ module Relaton
43
43
  #
44
44
  # Parse docnumber
45
45
  #
46
- # @return [String] PubID
46
+ # @return [String, nil] canonical pubid string, or nil when unparseable
47
47
  #
48
48
  def docnumber
49
- @docnumber ||= pubid&.to_id
49
+ @docnumber ||= pubid&.to_s
50
50
  end
51
51
 
52
52
  #
53
- # Create PubID
53
+ # Resolve the document's IEEE identifier (pubid-first, RawbibIdParser
54
+ # fallback).
54
55
  #
55
- # @return [Relaton::Ieee::RawbibIdParser] PubID
56
+ # @return [::Pubid::Ieee::Identifier, nil]
56
57
  #
57
58
  def pubid
58
- @pubid ||= begin
59
- normtitle = @doc.normtitle
60
- stdnumber = @doc.publicationinfo.stdnumber
61
- RawbibIdParser.parse(normtitle, stdnumber)
62
- end
59
+ return @pubid if defined?(@pubid)
60
+
61
+ normtitle = @doc.normtitle
62
+ stdnumber = @doc.publicationinfo.stdnumber
63
+ pid = RawbibIdParser.parse(normtitle, stdnumber)
64
+ title = @doc.btitle.find { |t| t[:type] == "main" }&.dig(:content)
65
+ isbn = @doc.isbn_doi.any? { |id| id[:type] == "ISBN" }
66
+ fabricated = RawbibIdParser.fabricated_title_id?(pid, title, isbn)
67
+ @pubid = fabricated ? nil : pid
63
68
  end
64
69
 
65
70
  #
@@ -92,8 +97,10 @@ module Relaton
92
97
  def parse_docidentifier # rubocop:disable Metrics/MethodLength
93
98
  ids = @doc.isbn_doi.map { |id| id[:content] = id.delete(:id); id }
94
99
 
95
- ids.unshift(content: pubid.to_s(trademark: true), scope: "trademark", type: "IEEE", primary: true)
96
- ids.unshift(content: pubid.to_s, type: "IEEE", primary: true)
100
+ if pubid
101
+ ids.unshift(content: pubid.to_s(trademark: true), scope: "trademark", type: "IEEE", primary: true)
102
+ ids.unshift(content: pubid.to_s, type: "IEEE", primary: true)
103
+ end
97
104
 
98
105
  result = ids.map { |dcid| Bib::Docidentifier.new(**dcid) }
99
106
  @errors[:docidentifier] &&= result.empty?
@@ -8,6 +8,7 @@ module Relaton
8
8
  def initialize # rubocop:disable Lint/MissingSuper
9
9
  @short = :relaton_ieee
10
10
  @prefix = "IEEE"
11
+ @pubid_flavor = :Ieee # global prefixes sourced from Pubid::Ieee.prefixes
11
12
  @defaultprefix = %r{^(?:(?:(?:ANSI|NACE)/)?IEEE|ANSI|AIEE|ASA|NACE|IRE)\s}
12
13
  @idtype = "IEEE"
13
14
  @datasets = %w[ieee-rawbib]
@@ -61,7 +62,9 @@ module Relaton
61
62
  #
62
63
  def remove_index_file
63
64
  require_relative "../ieee"
64
- Relaton::Index.find_or_create(:ieee, url: true, file: "#{INDEXFILE}.yaml").remove_file
65
+ Relaton::Index.find_or_create(
66
+ :ieee, url: true, file: "#{INDEXFILE}.yaml"
67
+ ).remove_file
65
68
  end
66
69
  end
67
70
  end