relaton 2.1.0 → 3.0.0.pre.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (625) hide show
  1. checksums.yaml +4 -4
  2. data/{docs/README.adoc → README.adoc} +109 -38
  3. data/lib/relaton/3gpp/bibdata.rb +9 -0
  4. data/lib/relaton/3gpp/bibitem.rb +9 -0
  5. data/lib/relaton/3gpp/bibliography.rb +48 -0
  6. data/lib/relaton/3gpp/data_fetcher.rb +255 -0
  7. data/lib/relaton/3gpp/doctype.rb +9 -0
  8. data/lib/relaton/3gpp/ext.rb +31 -0
  9. data/lib/relaton/3gpp/item.rb +12 -0
  10. data/lib/relaton/3gpp/item_data.rb +15 -0
  11. data/lib/relaton/3gpp/parser.rb +400 -0
  12. data/lib/relaton/3gpp/processor.rb +68 -0
  13. data/lib/relaton/3gpp/release.rb +34 -0
  14. data/lib/relaton/3gpp/util.rb +8 -0
  15. data/lib/relaton/3gpp.rb +25 -0
  16. data/lib/relaton/bib/converter/asciibib/to_asciibib.rb +663 -0
  17. data/lib/relaton/bib/converter/asciibib.rb +13 -0
  18. data/lib/relaton/bib/converter/bibtex/from_bibtex.rb +245 -0
  19. data/lib/relaton/bib/converter/bibtex/to_bibtex.rb +341 -0
  20. data/lib/relaton/bib/converter/bibtex.rb +23 -0
  21. data/lib/relaton/bib/converter/bibxml/from_rfcxml.rb +386 -0
  22. data/lib/relaton/bib/converter/bibxml/from_rfcxml_referencegroup.rb +71 -0
  23. data/lib/relaton/bib/converter/bibxml/to_rfcxml.rb +308 -0
  24. data/lib/relaton/bib/converter/bibxml/to_rfcxml_referencegroup.rb +52 -0
  25. data/lib/relaton/bib/converter/bibxml.rb +51 -0
  26. data/lib/relaton/bib/hash_parser_v1.rb +767 -0
  27. data/lib/relaton/bib/item_data.rb +229 -0
  28. data/lib/relaton/bib/model/abstract.rb +16 -0
  29. data/lib/relaton/bib/model/address.rb +22 -0
  30. data/lib/relaton/bib/model/affiliation.rb +16 -0
  31. data/lib/relaton/bib/model/bibdata.rb +11 -0
  32. data/lib/relaton/bib/model/bibdata_shared.rb +12 -0
  33. data/lib/relaton/bib/model/bibitem.rb +11 -0
  34. data/lib/relaton/bib/model/bibitem_shared.rb +12 -0
  35. data/lib/relaton/bib/model/contact.rb +18 -0
  36. data/lib/relaton/bib/model/contribution_info.rb +15 -0
  37. data/lib/relaton/bib/model/contributor.rb +29 -0
  38. data/lib/relaton/bib/model/copyright.rb +27 -0
  39. data/lib/relaton/bib/model/date.rb +31 -0
  40. data/lib/relaton/bib/model/depiction.rb +16 -0
  41. data/lib/relaton/bib/model/docidentifier.rb +34 -0
  42. data/lib/relaton/bib/model/doctype.rb +14 -0
  43. data/lib/relaton/bib/model/edition.rb +14 -0
  44. data/lib/relaton/bib/model/ext.rb +39 -0
  45. data/lib/relaton/bib/model/extent.rb +16 -0
  46. data/lib/relaton/bib/model/formattedref.rb +43 -0
  47. data/lib/relaton/bib/model/full_name_type.rb +64 -0
  48. data/lib/relaton/bib/model/fullname.rb +11 -0
  49. data/lib/relaton/bib/model/ics.rb +49 -0
  50. data/lib/relaton/bib/model/image.rb +28 -0
  51. data/lib/relaton/bib/model/item.rb +96 -0
  52. data/lib/relaton/bib/model/item_base.rb +20 -0
  53. data/lib/relaton/bib/model/item_shared.rb +88 -0
  54. data/lib/relaton/bib/model/keyword.rb +30 -0
  55. data/lib/relaton/bib/model/locality.rb +18 -0
  56. data/lib/relaton/bib/model/locality_stack.rb +14 -0
  57. data/lib/relaton/bib/model/localized_string.rb +48 -0
  58. data/lib/relaton/bib/model/localized_string_attrs.rb +24 -0
  59. data/lib/relaton/bib/model/logo.rb +14 -0
  60. data/lib/relaton/bib/model/medium.rb +22 -0
  61. data/lib/relaton/bib/model/note.rb +16 -0
  62. data/lib/relaton/bib/model/organization.rb +13 -0
  63. data/lib/relaton/bib/model/organization_type.rb +42 -0
  64. data/lib/relaton/bib/model/person.rb +36 -0
  65. data/lib/relaton/bib/model/phone.rb +14 -0
  66. data/lib/relaton/bib/model/place.rb +33 -0
  67. data/lib/relaton/bib/model/price.rb +14 -0
  68. data/lib/relaton/bib/model/relation.rb +43 -0
  69. data/lib/relaton/bib/model/series.rb +34 -0
  70. data/lib/relaton/bib/model/size.rb +23 -0
  71. data/lib/relaton/bib/model/source_locality_stack.rb +14 -0
  72. data/lib/relaton/bib/model/status.rb +27 -0
  73. data/lib/relaton/bib/model/structured_identifier.rb +48 -0
  74. data/lib/relaton/bib/model/subdivision.rb +16 -0
  75. data/lib/relaton/bib/model/title.rb +56 -0
  76. data/lib/relaton/bib/model/type/plain_date.rb +16 -0
  77. data/lib/relaton/bib/model/type/string_date.rb +48 -0
  78. data/lib/relaton/bib/model/uri.rb +18 -0
  79. data/lib/relaton/bib/model/validity.rb +16 -0
  80. data/lib/relaton/bib/model/version.rb +43 -0
  81. data/lib/relaton/bib/namespace_helper.rb +21 -0
  82. data/lib/relaton/bib/sanitizer.rb +68 -0
  83. data/lib/relaton/bib/util.rb +18 -0
  84. data/lib/relaton/bib/versions.json +35 -0
  85. data/lib/relaton/bib.rb +48 -0
  86. data/lib/relaton/bipm/bibliography.rb +82 -0
  87. data/lib/relaton/bipm/converter/asciibib.rb +64 -0
  88. data/lib/relaton/bipm/data_fetcher.rb +57 -0
  89. data/lib/relaton/bipm/data_outcomes_parser.rb +657 -0
  90. data/lib/relaton/bipm/id_parser.rb +277 -0
  91. data/lib/relaton/bipm/item_data.rb +47 -0
  92. data/lib/relaton/bipm/model/bibdata.rb +9 -0
  93. data/lib/relaton/bipm/model/bibitem.rb +9 -0
  94. data/lib/relaton/bipm/model/comment_period.rb +13 -0
  95. data/lib/relaton/bipm/model/doctype.rb +12 -0
  96. data/lib/relaton/bipm/model/ext.rb +38 -0
  97. data/lib/relaton/bipm/model/item.rb +11 -0
  98. data/lib/relaton/bipm/model/structured_identifier.rb +15 -0
  99. data/lib/relaton/bipm/processor.rb +69 -0
  100. data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +111 -0
  101. data/lib/relaton/bipm/rawdata_bipm_metrologia/fetcher.rb +174 -0
  102. data/lib/relaton/bipm/rawdata_bipm_metrologia/niso_jats_parser.rb +353 -0
  103. data/lib/relaton/bipm/si_brochure_parser.rb +186 -0
  104. data/lib/relaton/bipm/util.rb +8 -0
  105. data/lib/relaton/bipm.rb +32 -0
  106. data/lib/relaton/bsi/bibliography.rb +124 -0
  107. data/lib/relaton/bsi/hit.rb +14 -0
  108. data/lib/relaton/bsi/hit_collection.rb +114 -0
  109. data/lib/relaton/bsi/item_data.rb +13 -0
  110. data/lib/relaton/bsi/model/bibdata.rb +8 -0
  111. data/lib/relaton/bsi/model/bibitem.rb +8 -0
  112. data/lib/relaton/bsi/model/docidentifier.rb +9 -0
  113. data/lib/relaton/bsi/model/doctype.rb +14 -0
  114. data/lib/relaton/bsi/model/ext.rb +17 -0
  115. data/lib/relaton/bsi/model/item.rb +17 -0
  116. data/lib/relaton/bsi/model/item_base.rb +22 -0
  117. data/lib/relaton/bsi/model/relation.rb +9 -0
  118. data/lib/relaton/bsi/processor.rb +42 -0
  119. data/lib/relaton/bsi/schema.json +24882 -0
  120. data/lib/relaton/bsi/scraper.rb +288 -0
  121. data/lib/relaton/bsi/util.rb +8 -0
  122. data/lib/relaton/bsi.rb +25 -0
  123. data/lib/relaton/calconnect/bibliography.rb +95 -0
  124. data/lib/relaton/calconnect/data_fetcher.rb +106 -0
  125. data/lib/relaton/calconnect/hit.rb +14 -0
  126. data/lib/relaton/calconnect/hit_collection.rb +84 -0
  127. data/lib/relaton/calconnect/item_data.rb +12 -0
  128. data/lib/relaton/calconnect/model/bibdata.rb +8 -0
  129. data/lib/relaton/calconnect/model/bibitem.rb +8 -0
  130. data/lib/relaton/calconnect/model/doctype.rb +11 -0
  131. data/lib/relaton/calconnect/model/ext.rb +11 -0
  132. data/lib/relaton/calconnect/model/item.rb +13 -0
  133. data/lib/relaton/calconnect/processor.rb +66 -0
  134. data/lib/relaton/calconnect/scraper.rb +87 -0
  135. data/lib/relaton/calconnect/util.rb +8 -0
  136. data/lib/relaton/calconnect.rb +26 -0
  137. data/lib/relaton/ccsds/bibliography.rb +63 -0
  138. data/lib/relaton/ccsds/data/fetcher.rb +247 -0
  139. data/lib/relaton/ccsds/data/iso_references.rb +30 -0
  140. data/lib/relaton/ccsds/data/parser.rb +194 -0
  141. data/lib/relaton/ccsds/hit.rb +24 -0
  142. data/lib/relaton/ccsds/hit_collection.rb +42 -0
  143. data/lib/relaton/ccsds/item_data.rb +9 -0
  144. data/lib/relaton/ccsds/model/bibdata.rb +8 -0
  145. data/lib/relaton/ccsds/model/bibitem.rb +8 -0
  146. data/lib/relaton/ccsds/model/doctype.rb +9 -0
  147. data/lib/relaton/ccsds/model/ext.rb +19 -0
  148. data/lib/relaton/ccsds/model/item.rb +13 -0
  149. data/lib/relaton/ccsds/processor.rb +68 -0
  150. data/lib/relaton/ccsds/util.rb +10 -0
  151. data/lib/relaton/ccsds.rb +33 -0
  152. data/lib/relaton/cen/bibliography.rb +121 -0
  153. data/lib/relaton/cen/committees.yaml +66 -0
  154. data/lib/relaton/cen/hit.rb +16 -0
  155. data/lib/relaton/cen/hit_collection.rb +68 -0
  156. data/lib/relaton/cen/item_data.rb +7 -0
  157. data/lib/relaton/cen/model/bibdata.rb +8 -0
  158. data/lib/relaton/cen/model/bibitem.rb +8 -0
  159. data/lib/relaton/cen/model/docidentifier.rb +9 -0
  160. data/lib/relaton/cen/model/ext.rb +11 -0
  161. data/lib/relaton/cen/model/item.rb +14 -0
  162. data/lib/relaton/cen/model/structured_identifier.rb +9 -0
  163. data/lib/relaton/cen/processor.rb +40 -0
  164. data/lib/relaton/cen/scraper.rb +210 -0
  165. data/lib/relaton/cen/util.rb +8 -0
  166. data/lib/relaton/cen.rb +29 -0
  167. data/lib/relaton/cie/bibdata.rb +8 -0
  168. data/lib/relaton/cie/bibitem.rb +8 -0
  169. data/lib/relaton/cie/bibliography.rb +31 -0
  170. data/lib/relaton/cie/data_fetcher.rb +355 -0
  171. data/lib/relaton/cie/ext.rb +7 -0
  172. data/lib/relaton/cie/item.rb +11 -0
  173. data/lib/relaton/cie/item_data.rb +6 -0
  174. data/lib/relaton/cie/processor.rb +66 -0
  175. data/lib/relaton/cie/scrapper.rb +39 -0
  176. data/lib/relaton/cie/util.rb +8 -0
  177. data/lib/relaton/cie.rb +28 -0
  178. data/lib/relaton/core/array_wrapper.rb +20 -0
  179. data/lib/relaton/core/data_fetcher.rb +97 -0
  180. data/lib/relaton/core/date_parser.rb +42 -0
  181. data/lib/relaton/core/hash_keys_sybolizer.rb +19 -0
  182. data/lib/relaton/core/hit.rb +49 -0
  183. data/lib/relaton/core/hit_collection.rb +118 -0
  184. data/lib/relaton/core/processor.rb +48 -0
  185. data/lib/relaton/core/workers_pool.rb +45 -0
  186. data/lib/relaton/core.rb +9 -0
  187. data/lib/relaton/db/cache.rb +232 -0
  188. data/lib/relaton/db/config.rb +24 -0
  189. data/lib/relaton/db/registry.rb +119 -0
  190. data/lib/relaton/db/util.rb +9 -0
  191. data/lib/relaton/db/workers_pool.rb +22 -0
  192. data/lib/relaton/db.rb +20 -206
  193. data/lib/relaton/doi/crossref.rb +72 -0
  194. data/lib/relaton/doi/parser.rb +827 -0
  195. data/lib/relaton/doi/processor.rb +65 -0
  196. data/lib/relaton/doi/util.rb +8 -0
  197. data/lib/relaton/doi.rb +20 -0
  198. data/lib/relaton/ecma/bibdata.rb +8 -0
  199. data/lib/relaton/ecma/bibitem.rb +8 -0
  200. data/lib/relaton/ecma/bibliography.rb +81 -0
  201. data/lib/relaton/ecma/data_fetcher.rb +103 -0
  202. data/lib/relaton/ecma/data_parser.rb +49 -0
  203. data/lib/relaton/ecma/edition_parser.rb +80 -0
  204. data/lib/relaton/ecma/ext.rb +7 -0
  205. data/lib/relaton/ecma/item.rb +11 -0
  206. data/lib/relaton/ecma/item_data.rb +6 -0
  207. data/lib/relaton/ecma/memento_parser.rb +60 -0
  208. data/lib/relaton/ecma/page_fetcher.rb +27 -0
  209. data/lib/relaton/ecma/parser_common.rb +33 -0
  210. data/lib/relaton/ecma/processor.rb +66 -0
  211. data/lib/relaton/ecma/standard_parser.rb +134 -0
  212. data/lib/relaton/ecma/util.rb +8 -0
  213. data/lib/relaton/ecma.rb +24 -0
  214. data/lib/relaton/etsi/bibdata.rb +10 -0
  215. data/lib/relaton/etsi/bibitem.rb +10 -0
  216. data/lib/relaton/etsi/bibliography.rb +46 -0
  217. data/lib/relaton/etsi/data_fetcher.rb +128 -0
  218. data/lib/relaton/etsi/data_parser.rb +208 -0
  219. data/lib/relaton/etsi/doctype.rb +30 -0
  220. data/lib/relaton/etsi/ext.rb +31 -0
  221. data/lib/relaton/etsi/item.rb +15 -0
  222. data/lib/relaton/etsi/item_data.rb +6 -0
  223. data/lib/relaton/etsi/processor.rb +69 -0
  224. data/lib/relaton/etsi/pubid.rb +37 -0
  225. data/lib/relaton/etsi/status.rb +13 -0
  226. data/lib/relaton/etsi/util.rb +8 -0
  227. data/lib/relaton/etsi.rb +25 -0
  228. data/lib/relaton/gb/bibdata.rb +8 -0
  229. data/lib/relaton/gb/bibitem.rb +8 -0
  230. data/lib/relaton/gb/bibliography.rb +145 -0
  231. data/lib/relaton/gb/ccs.rb +14 -0
  232. data/lib/relaton/gb/committee.rb +13 -0
  233. data/lib/relaton/gb/docidentifier.rb +23 -0
  234. data/lib/relaton/gb/doctype.rb +9 -0
  235. data/lib/relaton/gb/ext.rb +36 -0
  236. data/lib/relaton/gb/gb_scraper.rb +61 -0
  237. data/lib/relaton/gb/gb_type.rb +20 -0
  238. data/lib/relaton/gb/hit.rb +48 -0
  239. data/lib/relaton/gb/hit_collection.rb +19 -0
  240. data/lib/relaton/gb/item.rb +13 -0
  241. data/lib/relaton/gb/item_data.rb +6 -0
  242. data/lib/relaton/gb/processor.rb +46 -0
  243. data/lib/relaton/gb/project_number.rb +38 -0
  244. data/lib/relaton/gb/scraper.rb +204 -0
  245. data/lib/relaton/gb/sec_scraper.rb +92 -0
  246. data/lib/relaton/gb/stage_name.rb +13 -0
  247. data/lib/relaton/gb/structured_identifier.rb +26 -0
  248. data/lib/relaton/gb/t_scraper.rb +126 -0
  249. data/lib/relaton/gb/util.rb +8 -0
  250. data/lib/relaton/gb/yaml/prefixes.yaml +200 -0
  251. data/lib/relaton/gb.rb +33 -0
  252. data/lib/relaton/iana/bibdata.rb +8 -0
  253. data/lib/relaton/iana/bibitem.rb +8 -0
  254. data/lib/relaton/iana/bibliography.rb +47 -0
  255. data/lib/relaton/iana/data_fetcher.rb +71 -0
  256. data/lib/relaton/iana/item.rb +7 -0
  257. data/lib/relaton/iana/item_data.rb +6 -0
  258. data/lib/relaton/iana/parser.rb +146 -0
  259. data/lib/relaton/iana/processor.rb +68 -0
  260. data/lib/relaton/iana/util.rb +8 -0
  261. data/lib/relaton/iana.rb +29 -0
  262. data/lib/relaton/iec/bibliography.rb +283 -0
  263. data/lib/relaton/iec/data_fetcher.rb +216 -0
  264. data/lib/relaton/iec/data_parser.rb +391 -0
  265. data/lib/relaton/iec/hit.rb +26 -0
  266. data/lib/relaton/iec/hit_collection.rb +138 -0
  267. data/lib/relaton/iec/item_data.rb +7 -0
  268. data/lib/relaton/iec/model/bibdata.rb +8 -0
  269. data/lib/relaton/iec/model/bibitem.rb +8 -0
  270. data/lib/relaton/iec/model/docidentifier.rb +131 -0
  271. data/lib/relaton/iec/model/doctype.rb +12 -0
  272. data/lib/relaton/iec/model/ext.rb +53 -0
  273. data/lib/relaton/iec/model/item.rb +20 -0
  274. data/lib/relaton/iec/model/item_base.rb +12 -0
  275. data/lib/relaton/iec/model/relation.rb +7 -0
  276. data/lib/relaton/iec/model/stage_name.rb +13 -0
  277. data/lib/relaton/iec/processor.rb +73 -0
  278. data/lib/relaton/iec/statuses.yml +199 -0
  279. data/lib/relaton/iec/util.rb +8 -0
  280. data/lib/relaton/iec.rb +98 -0
  281. data/lib/relaton/ieee/balloting_group.rb +13 -0
  282. data/lib/relaton/ieee/bibdata.rb +8 -0
  283. data/lib/relaton/ieee/bibitem.rb +8 -0
  284. data/lib/relaton/ieee/bibliography.rb +51 -0
  285. data/lib/relaton/ieee/converter/bibxml/from_rfcxml.rb +10 -0
  286. data/lib/relaton/ieee/converter/bibxml/from_rfcxml_referencegroup.rb +10 -0
  287. data/lib/relaton/ieee/converter/bibxml.rb +20 -0
  288. data/lib/relaton/ieee/data_fetcher.rb +630 -0
  289. data/lib/relaton/ieee/doctype.rb +9 -0
  290. data/lib/relaton/ieee/editorial_group.rb +19 -0
  291. data/lib/relaton/ieee/ext.rb +36 -0
  292. data/lib/relaton/ieee/idams_parser.rb +323 -0
  293. data/lib/relaton/ieee/item.rb +11 -0
  294. data/lib/relaton/ieee/item_data.rb +7 -0
  295. data/lib/relaton/ieee/processor.rb +68 -0
  296. data/lib/relaton/ieee/pub_id.rb +161 -0
  297. data/lib/relaton/ieee/rawbib_id_parser.rb +548 -0
  298. data/lib/relaton/ieee/util.rb +8 -0
  299. data/lib/relaton/ieee.rb +29 -0
  300. data/lib/relaton/ietf/bibdata.rb +8 -0
  301. data/lib/relaton/ietf/bibitem.rb +8 -0
  302. data/lib/relaton/ietf/bibliography.rb +35 -0
  303. data/lib/relaton/ietf/bibxml_parser.rb +232 -0
  304. data/lib/relaton/ietf/data_fetcher.rb +278 -0
  305. data/lib/relaton/ietf/doctype.rb +9 -0
  306. data/lib/relaton/ietf/ext.rb +63 -0
  307. data/lib/relaton/ietf/item.rb +16 -0
  308. data/lib/relaton/ietf/item_base.rb +18 -0
  309. data/lib/relaton/ietf/item_data.rb +6 -0
  310. data/lib/relaton/ietf/processing_instructions.rb +79 -0
  311. data/lib/relaton/ietf/processor.rb +68 -0
  312. data/lib/relaton/ietf/relation.rb +9 -0
  313. data/lib/relaton/ietf/rfc/abstract.rb +19 -0
  314. data/lib/relaton/ietf/rfc/author.rb +21 -0
  315. data/lib/relaton/ietf/rfc/entry.rb +410 -0
  316. data/lib/relaton/ietf/rfc/entry_date.rb +21 -0
  317. data/lib/relaton/ietf/rfc/format.rb +19 -0
  318. data/lib/relaton/ietf/rfc/index.rb +46 -0
  319. data/lib/relaton/ietf/rfc/is_also.rb +21 -0
  320. data/lib/relaton/ietf/rfc/keywords.rb +19 -0
  321. data/lib/relaton/ietf/rfc/rfc_index_namespace.rb +11 -0
  322. data/lib/relaton/ietf/scraper.rb +68 -0
  323. data/lib/relaton/ietf/util.rb +8 -0
  324. data/lib/relaton/ietf/wg_name_resolver.rb +42 -0
  325. data/lib/relaton/ietf.rb +27 -0
  326. data/lib/relaton/iho/bibdata.rb +8 -0
  327. data/lib/relaton/iho/bibitem.rb +8 -0
  328. data/lib/relaton/iho/bibliography.rb +114 -0
  329. data/lib/relaton/iho/comment_period.rb +13 -0
  330. data/lib/relaton/iho/docidentifier.rb +31 -0
  331. data/lib/relaton/iho/doctype.rb +10 -0
  332. data/lib/relaton/iho/ext.rb +20 -0
  333. data/lib/relaton/iho/hash_parser_v1.rb +144 -0
  334. data/lib/relaton/iho/item.rb +20 -0
  335. data/lib/relaton/iho/item_base.rb +18 -0
  336. data/lib/relaton/iho/item_data.rb +6 -0
  337. data/lib/relaton/iho/processor.rb +54 -0
  338. data/lib/relaton/iho/relation.rb +9 -0
  339. data/lib/relaton/iho/structured_identifier.rb +20 -0
  340. data/lib/relaton/iho/util.rb +8 -0
  341. data/lib/relaton/iho.rb +27 -0
  342. data/lib/relaton/index/config.rb +52 -0
  343. data/lib/relaton/index/file_io.rb +305 -0
  344. data/lib/relaton/index/file_storage.rb +61 -0
  345. data/lib/relaton/index/id_number.rb +30 -0
  346. data/lib/relaton/index/pool.rb +41 -0
  347. data/lib/relaton/index/type.rb +159 -0
  348. data/lib/relaton/index/util.rb +18 -0
  349. data/lib/relaton/index.rb +60 -0
  350. data/lib/relaton/isbn/isbn.rb +61 -0
  351. data/lib/relaton/isbn/open_library.rb +43 -0
  352. data/lib/relaton/isbn/parser.rb +88 -0
  353. data/lib/relaton/isbn/processor.rb +48 -0
  354. data/lib/relaton/isbn/util.rb +8 -0
  355. data/lib/relaton/isbn.rb +19 -0
  356. data/lib/relaton/iso/bibliography.rb +407 -0
  357. data/lib/relaton/iso/data_fetcher.rb +338 -0
  358. data/lib/relaton/iso/data_parser.rb +444 -0
  359. data/lib/relaton/iso/hash_parser_v1.rb +177 -0
  360. data/lib/relaton/iso/hit.rb +57 -0
  361. data/lib/relaton/iso/hit_collection.rb +184 -0
  362. data/lib/relaton/iso/item_data.rb +28 -0
  363. data/lib/relaton/iso/model/bibdata.rb +10 -0
  364. data/lib/relaton/iso/model/bibitem.rb +8 -0
  365. data/lib/relaton/iso/model/contributor.rb +6 -0
  366. data/lib/relaton/iso/model/contributor_info.rb +9 -0
  367. data/lib/relaton/iso/model/docidentifier.rb +134 -0
  368. data/lib/relaton/iso/model/doctype.rb +13 -0
  369. data/lib/relaton/iso/model/ext.rb +35 -0
  370. data/lib/relaton/iso/model/item.rb +17 -0
  371. data/lib/relaton/iso/model/item_base.rb +22 -0
  372. data/lib/relaton/iso/model/organization.rb +9 -0
  373. data/lib/relaton/iso/model/project_number.rb +22 -0
  374. data/lib/relaton/iso/model/relation.rb +9 -0
  375. data/lib/relaton/iso/model/stagename.rb +14 -0
  376. data/lib/relaton/iso/model/structured_identifier.rb +31 -0
  377. data/lib/relaton/iso/processor.rb +84 -0
  378. data/lib/relaton/iso/scraper.rb +619 -0
  379. data/lib/relaton/iso/type/pubid.rb +50 -0
  380. data/lib/relaton/iso/util.rb +8 -0
  381. data/lib/relaton/iso.rb +30 -0
  382. data/lib/relaton/itu/bibliography.rb +90 -0
  383. data/lib/relaton/itu/data_fetcher.rb +104 -0
  384. data/lib/relaton/itu/data_parser_r.rb +140 -0
  385. data/lib/relaton/itu/hit.rb +22 -0
  386. data/lib/relaton/itu/hit_collection.rb +127 -0
  387. data/lib/relaton/itu/item_data.rb +6 -0
  388. data/lib/relaton/itu/model/approval_stage.rb +13 -0
  389. data/lib/relaton/itu/model/bibdata.rb +8 -0
  390. data/lib/relaton/itu/model/bibitem.rb +8 -0
  391. data/lib/relaton/itu/model/docidentifier.rb +9 -0
  392. data/lib/relaton/itu/model/doctype.rb +13 -0
  393. data/lib/relaton/itu/model/ext.rb +47 -0
  394. data/lib/relaton/itu/model/item.rb +13 -0
  395. data/lib/relaton/itu/model/meeting.rb +13 -0
  396. data/lib/relaton/itu/model/meeting_date.rb +15 -0
  397. data/lib/relaton/itu/model/question.rb +13 -0
  398. data/lib/relaton/itu/model/recommendation_status.rb +11 -0
  399. data/lib/relaton/itu/model/structured_identifier.rb +19 -0
  400. data/lib/relaton/itu/processor.rb +66 -0
  401. data/lib/relaton/itu/pubid.rb +177 -0
  402. data/lib/relaton/itu/radio_regulations_parser.rb +70 -0
  403. data/lib/relaton/itu/recommendation_parser.rb +161 -0
  404. data/lib/relaton/itu/scraper.rb +193 -0
  405. data/lib/relaton/itu/util.rb +8 -0
  406. data/lib/relaton/itu.rb +33 -0
  407. data/lib/relaton/jis/bibdata.rb +8 -0
  408. data/lib/relaton/jis/bibitem.rb +8 -0
  409. data/lib/relaton/jis/bibliography.rb +75 -0
  410. data/lib/relaton/jis/data_fetcher.rb +181 -0
  411. data/lib/relaton/jis/docidentifier.rb +21 -0
  412. data/lib/relaton/jis/doctype.rb +9 -0
  413. data/lib/relaton/jis/ext.rb +13 -0
  414. data/lib/relaton/jis/hit.rb +93 -0
  415. data/lib/relaton/jis/hit_collection.rb +127 -0
  416. data/lib/relaton/jis/item.rb +22 -0
  417. data/lib/relaton/jis/item_base.rb +26 -0
  418. data/lib/relaton/jis/item_data.rb +8 -0
  419. data/lib/relaton/jis/processor.rb +55 -0
  420. data/lib/relaton/jis/relation.rb +11 -0
  421. data/lib/relaton/jis/scraper.rb +232 -0
  422. data/lib/relaton/jis/structured_identifier.rb +13 -0
  423. data/lib/relaton/jis/util.rb +8 -0
  424. data/lib/relaton/jis.rb +33 -0
  425. data/lib/relaton/logger/channels/gh_issue.rb +78 -0
  426. data/lib/relaton/logger/config.rb +34 -0
  427. data/lib/relaton/logger/formatter_json.rb +8 -0
  428. data/lib/relaton/logger/formatter_string.rb +12 -0
  429. data/lib/relaton/logger/log.rb +64 -0
  430. data/lib/relaton/logger/log_device.rb +27 -0
  431. data/lib/relaton/logger/pool.rb +24 -0
  432. data/lib/relaton/logger.rb +18 -0
  433. data/lib/relaton/nist/bibdata.rb +8 -0
  434. data/lib/relaton/nist/bibitem.rb +8 -0
  435. data/lib/relaton/nist/bibliography.rb +191 -0
  436. data/lib/relaton/nist/comment_period.rb +15 -0
  437. data/lib/relaton/nist/data_fetcher.rb +84 -0
  438. data/lib/relaton/nist/date.rb +7 -0
  439. data/lib/relaton/nist/doctype.rb +7 -0
  440. data/lib/relaton/nist/ext.rb +16 -0
  441. data/lib/relaton/nist/hit.rb +18 -0
  442. data/lib/relaton/nist/hit_collection.rb +362 -0
  443. data/lib/relaton/nist/item.rb +17 -0
  444. data/lib/relaton/nist/item_data.rb +6 -0
  445. data/lib/relaton/nist/mods_parser.rb +276 -0
  446. data/lib/relaton/nist/processor.rb +68 -0
  447. data/lib/relaton/nist/pubs_export.rb +69 -0
  448. data/lib/relaton/nist/relation.rb +7 -0
  449. data/lib/relaton/nist/scraper.rb +290 -0
  450. data/lib/relaton/nist/series.yaml +49 -0
  451. data/lib/relaton/nist/util.rb +8 -0
  452. data/lib/relaton/nist.rb +34 -0
  453. data/lib/relaton/oasis/bibdata.rb +8 -0
  454. data/lib/relaton/oasis/bibitem.rb +8 -0
  455. data/lib/relaton/oasis/bibliography.rb +60 -0
  456. data/lib/relaton/oasis/browser_agent.rb +71 -0
  457. data/lib/relaton/oasis/data_fetcher.rb +94 -0
  458. data/lib/relaton/oasis/data_parser.rb +305 -0
  459. data/lib/relaton/oasis/data_parser_utils.rb +296 -0
  460. data/lib/relaton/oasis/data_part_parser.rb +239 -0
  461. data/lib/relaton/oasis/doctype.rb +7 -0
  462. data/lib/relaton/oasis/ext.rb +19 -0
  463. data/lib/relaton/oasis/item.rb +11 -0
  464. data/lib/relaton/oasis/item_data.rb +6 -0
  465. data/lib/relaton/oasis/processor.rb +68 -0
  466. data/lib/relaton/oasis/util.rb +8 -0
  467. data/lib/relaton/oasis.rb +28 -0
  468. data/lib/relaton/ogc/bibdata.rb +8 -0
  469. data/lib/relaton/ogc/bibitem.rb +8 -0
  470. data/lib/relaton/ogc/bibliography.rb +64 -0
  471. data/lib/relaton/ogc/data_fetcher.rb +110 -0
  472. data/lib/relaton/ogc/docidentifier.rb +6 -0
  473. data/lib/relaton/ogc/doctype.rb +12 -0
  474. data/lib/relaton/ogc/ext.rb +15 -0
  475. data/lib/relaton/ogc/hit.rb +18 -0
  476. data/lib/relaton/ogc/hit_collection.rb +36 -0
  477. data/lib/relaton/ogc/item.rb +13 -0
  478. data/lib/relaton/ogc/item_data.rb +6 -0
  479. data/lib/relaton/ogc/processor.rb +58 -0
  480. data/lib/relaton/ogc/scraper.rb +225 -0
  481. data/lib/relaton/ogc/util.rb +8 -0
  482. data/lib/relaton/ogc.rb +26 -0
  483. data/lib/relaton/oiml/bibdata.rb +8 -0
  484. data/lib/relaton/oiml/bibitem.rb +8 -0
  485. data/lib/relaton/oiml/bibliography.rb +94 -0
  486. data/lib/relaton/oiml/docidentifier.rb +33 -0
  487. data/lib/relaton/oiml/doctype.rb +13 -0
  488. data/lib/relaton/oiml/ext.rb +42 -0
  489. data/lib/relaton/oiml/item.rb +20 -0
  490. data/lib/relaton/oiml/item_base.rb +18 -0
  491. data/lib/relaton/oiml/item_data.rb +6 -0
  492. data/lib/relaton/oiml/processor.rb +54 -0
  493. data/lib/relaton/oiml/relation.rb +9 -0
  494. data/lib/relaton/oiml/util.rb +8 -0
  495. data/lib/relaton/oiml.rb +24 -0
  496. data/lib/relaton/omg/bibdata.rb +8 -0
  497. data/lib/relaton/omg/bibitem.rb +8 -0
  498. data/lib/relaton/omg/bibliography.rb +31 -0
  499. data/lib/relaton/omg/ext.rb +7 -0
  500. data/lib/relaton/omg/item.rb +8 -0
  501. data/lib/relaton/omg/item_data.rb +6 -0
  502. data/lib/relaton/omg/processor.rb +34 -0
  503. data/lib/relaton/omg/scraper.rb +141 -0
  504. data/lib/relaton/omg/util.rb +8 -0
  505. data/lib/relaton/omg.rb +24 -0
  506. data/lib/relaton/plateau/bibdata.rb +8 -0
  507. data/lib/relaton/plateau/bibitem.rb +8 -0
  508. data/lib/relaton/plateau/bibliography.rb +24 -0
  509. data/lib/relaton/plateau/data_fetcher.rb +153 -0
  510. data/lib/relaton/plateau/doctype.rb +7 -0
  511. data/lib/relaton/plateau/ext.rb +23 -0
  512. data/lib/relaton/plateau/handbook_parser.rb +95 -0
  513. data/lib/relaton/plateau/hit.rb +13 -0
  514. data/lib/relaton/plateau/hit_collection.rb +59 -0
  515. data/lib/relaton/plateau/item.rb +12 -0
  516. data/lib/relaton/plateau/item_data.rb +6 -0
  517. data/lib/relaton/plateau/parser.rb +101 -0
  518. data/lib/relaton/plateau/processor.rb +49 -0
  519. data/lib/relaton/plateau/technical_report_parser.rb +91 -0
  520. data/lib/relaton/plateau/util.rb +8 -0
  521. data/lib/relaton/plateau.rb +28 -0
  522. data/lib/relaton/un/bibdata.rb +8 -0
  523. data/lib/relaton/un/bibitem.rb +8 -0
  524. data/lib/relaton/un/bibliography.rb +48 -0
  525. data/lib/relaton/un/doctype.rb +10 -0
  526. data/lib/relaton/un/ext.rb +30 -0
  527. data/lib/relaton/un/hit.rb +56 -0
  528. data/lib/relaton/un/hit_collection.rb +61 -0
  529. data/lib/relaton/un/item.rb +11 -0
  530. data/lib/relaton/un/item_data.rb +6 -0
  531. data/lib/relaton/un/parser.rb +108 -0
  532. data/lib/relaton/un/processor.rb +38 -0
  533. data/lib/relaton/un/session.rb +25 -0
  534. data/lib/relaton/un/token_generator.rb +105 -0
  535. data/lib/relaton/un/util.rb +8 -0
  536. data/lib/relaton/un/wasm/decoder.rb +434 -0
  537. data/lib/relaton/un/wasm/instance.rb +101 -0
  538. data/lib/relaton/un/wasm/interpreter.rb +613 -0
  539. data/lib/relaton/un/wasm/memory.rb +112 -0
  540. data/lib/relaton/un/wasm/module.rb +47 -0
  541. data/lib/relaton/un/wasm.rb +17 -0
  542. data/lib/relaton/un/wasm_v_bg.wasm +0 -0
  543. data/lib/relaton/un.rb +28 -0
  544. data/lib/relaton/version.rb +1 -1
  545. data/lib/relaton/w3c/bibdata.rb +8 -0
  546. data/lib/relaton/w3c/bibitem.rb +8 -0
  547. data/lib/relaton/w3c/bibliography.rb +54 -0
  548. data/lib/relaton/w3c/data_fetcher.rb +309 -0
  549. data/lib/relaton/w3c/data_parser.rb +379 -0
  550. data/lib/relaton/w3c/doctype.rb +7 -0
  551. data/lib/relaton/w3c/ext.rb +11 -0
  552. data/lib/relaton/w3c/item.rb +12 -0
  553. data/lib/relaton/w3c/item_data.rb +6 -0
  554. data/lib/relaton/w3c/processor.rb +49 -0
  555. data/lib/relaton/w3c/pubid.rb +73 -0
  556. data/lib/relaton/w3c/safe_realize.rb +59 -0
  557. data/lib/relaton/w3c/util.rb +8 -0
  558. data/lib/relaton/w3c.rb +25 -0
  559. data/lib/relaton/xsf/bibdata.rb +8 -0
  560. data/lib/relaton/xsf/bibitem.rb +8 -0
  561. data/lib/relaton/xsf/bibliography.rb +24 -0
  562. data/lib/relaton/xsf/data_fetcher.rb +56 -0
  563. data/lib/relaton/xsf/hit.rb +17 -0
  564. data/lib/relaton/xsf/hit_collection.rb +24 -0
  565. data/lib/relaton/xsf/item.rb +7 -0
  566. data/lib/relaton/xsf/item_data.rb +6 -0
  567. data/lib/relaton/xsf/processor.rb +50 -0
  568. data/lib/relaton/xsf/util.rb +9 -0
  569. data/lib/relaton/xsf.rb +30 -0
  570. data/lib/relaton-core.rb +1 -0
  571. data/lib/relaton.rb +44 -9
  572. metadata +708 -150
  573. data/.github/workflows/rake.yml +0 -14
  574. data/.github/workflows/release.yml +0 -24
  575. data/.github/workflows/rubocop.yml +0 -48
  576. data/.gitignore +0 -12
  577. data/.rspec +0 -3
  578. data/.rubocop.yml +0 -12
  579. data/CLAUDE.md +0 -70
  580. data/Gemfile +0 -21
  581. data/Rakefile +0 -6
  582. data/docs/CHANGELOG.adoc +0 -16
  583. data/docs/VERSIONING_POLICY.adoc +0 -38
  584. data/docs/navigation.adoc +0 -6
  585. data/lib/relaton/config.rb +0 -24
  586. data/lib/relaton/db_cache.rb +0 -230
  587. data/lib/relaton/registry.rb +0 -118
  588. data/lib/relaton/util.rb +0 -7
  589. data/lib/relaton/workers_pool.rb +0 -21
  590. data/relaton.gemspec +0 -61
  591. data/spec/relaton/config_spec.rb +0 -10
  592. data/spec/relaton/db_cache_spec.rb +0 -51
  593. data/spec/relaton/db_spec.rb +0 -556
  594. data/spec/relaton/registry_spec.rb +0 -178
  595. data/spec/relaton/util_spec.rb +0 -3
  596. data/spec/relaton_spec.rb +0 -725
  597. data/spec/spec_helper.rb +0 -45
  598. data/spec/support/gb_t_20223_2006.xml +0 -33
  599. data/spec/support/iso_111111119115_1.xml +0 -0
  600. data/spec/support/iso_19115_1.xml +0 -115
  601. data/spec/support/iso_19115_2.xml +0 -95
  602. data/spec/support/rfc_8341.xml +0 -46
  603. data/spec/vcr_cassetes/3gpp_tr_00_01u_umts_3_0_0.yml +0 -12720
  604. data/spec/vcr_cassetes/api_relaton_org.yml +0 -121
  605. data/spec/vcr_cassetes/api_relaton_org_unavailable.yml +0 -143
  606. data/spec/vcr_cassetes/cc_dir_10005_2019.yml +0 -199
  607. data/spec/vcr_cassetes/cie_001_1980.yml +0 -401
  608. data/spec/vcr_cassetes/cipm_meeting_43.yml +0 -1513
  609. data/spec/vcr_cassetes/doi_10_6028_nist_ir_8245.yml +0 -86
  610. data/spec/vcr_cassetes/ecma_6.yml +0 -112
  611. data/spec/vcr_cassetes/en_10160_1999.yml +0 -13624
  612. data/spec/vcr_cassetes/gb_t_20223_2006.yml +0 -733
  613. data/spec/vcr_cassetes/ieee_528_2019.yml +0 -2786
  614. data/spec/vcr_cassetes/iso_111111119115_1.yml +0 -12951
  615. data/spec/vcr_cassetes/iso_19115_1.yml +0 -13177
  616. data/spec/vcr_cassetes/iso_19115_1_2.yml +0 -369
  617. data/spec/vcr_cassetes/iso_19115_1_std.yml +0 -13177
  618. data/spec/vcr_cassetes/iso_19115_all_parts.yml +0 -229
  619. data/spec/vcr_cassetes/iso_19133_2005.yml +0 -148
  620. data/spec/vcr_cassetes/iso_combined_applied.yml +0 -364
  621. data/spec/vcr_cassetes/iso_combined_included.yml +0 -364
  622. data/spec/vcr_cassetes/iso_dis.yml +0 -157
  623. data/spec/vcr_cassetes/ogc_19_025r1.yml +0 -371
  624. data/spec/vcr_cassetes/omg_ami4ccm_1_0.yml +0 -317
  625. data/spec/vcr_cassetes/rfc_8341.yml +0 -1272
@@ -0,0 +1,630 @@
1
+ require "etc"
2
+ require "zip"
3
+ require_relative "../ieee"
4
+ require_relative "converter/bibxml"
5
+ require_relative "idams_parser"
6
+ require_relative "rawbib_id_parser"
7
+
8
+ module Relaton
9
+ module Ieee
10
+ class DataFetcher < Core::DataFetcher
11
+ RELATION_TYPES = {
12
+ "S" => { type: "obsoletedBy" },
13
+ "V" => { type: "updates", description: "revises" },
14
+ "T" => { type: "updates", description: "amends" },
15
+ "C" => { type: "updates", description: "corrects" },
16
+ "O" => { type: "adoptedFrom" },
17
+ "P" => { type: "complementOf", description: "supplement" },
18
+ "N" => false, "G" => false,
19
+ "F" => false, "I" => false,
20
+ "E" => false, "B" => false, "W" => false
21
+ }.freeze
22
+
23
+ #
24
+ # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
25
+ #
26
+ def log_error(msg)
27
+ Util.error msg
28
+ end
29
+
30
+ def fetch(_source = nil)
31
+ files = Dir["ieee-rawbib/**/*.{xml,zip}"].reject { |f| f["Deleted_"] }
32
+ files = prefilter_winners(files) unless ENV["IEEE_FETCH_PREFILTER"] == "0"
33
+ process_files(files)
34
+ update_relations
35
+ report_errors
36
+ end
37
+
38
+ # @return [Hash] list of AMSID => PubID
39
+ def backrefs
40
+ @backrefs ||= {}
41
+ end
42
+
43
+ # @return [Hash] list of docnumber => parsed bib (cache for update_relations)
44
+ def docs
45
+ @docs ||= {}
46
+ end
47
+
48
+ # @return [Hash] docnumber => max global glob-index whose write was
49
+ # accepted by commit_doc. Populated only when running with parallel
50
+ # workers (writes are staged to per-glob-index suffixed paths and
51
+ # reconciled into the final filename after the parsing phase).
52
+ def saved_writes
53
+ @saved_writes ||= {}
54
+ end
55
+
56
+ # Mutex guarding worker-thread mutations of shared state during parse.
57
+ def mutex
58
+ @mutex ||= Mutex.new
59
+ end
60
+
61
+ #
62
+ # Save unresolved relation reference. Called from worker threads via
63
+ # IdamsParser#parse_relation, so mutates crossrefs under a mutex.
64
+ #
65
+ # @param [String] docnumber of main document
66
+ # @param [Nokogiri::XML::Element] amsid relation data
67
+ #
68
+ def add_crossref(docnumber, amsid)
69
+ return if RELATION_TYPES[amsid.type] == false
70
+
71
+ ref = { amsid: amsid.date_string, type: amsid.type }
72
+ mutex.synchronize { crossrefs[docnumber] << ref }
73
+ end
74
+
75
+ #
76
+ # Create relation instance
77
+ #
78
+ # @param [String] type IEEE relation type
79
+ # @param [String] fref reference
80
+ #
81
+ # @return [RelatonBib::DocumentRelation]
82
+ #
83
+ def create_relation(type, fref)
84
+ unless RELATION_TYPES.key? type
85
+ Util.warn "Unknown relation type: '#{type}' for reference '#{fref}'", key: fref
86
+ return
87
+ end
88
+ return if RELATION_TYPES[type] == false
89
+
90
+ docid = Bib::Docidentifier.new(type: "IEEE", content: fref, primary: true)
91
+ bib = ItemData.new formattedref: Bib::Formattedref.new(content: fref), docidentifier: [docid]
92
+ description = create_relation_description type
93
+ Bib::Relation.new(type: RELATION_TYPES[type][:type], description: description, bibitem: bib)
94
+ end
95
+
96
+ private
97
+
98
+ def create_relation_description(type)
99
+ desc = RELATION_TYPES[type][:description] if RELATION_TYPES[type]
100
+ return unless desc
101
+
102
+ desc && Bib::LocalizedMarkedUpString.new(content: desc, language: "en", script: "Latn")
103
+ end
104
+
105
+ # @return [Hash] list of PubID => list of unresolved relations
106
+ def crossrefs
107
+ @crossrefs ||= Hash.new { |hash, key| hash[key] = [] }
108
+ end
109
+
110
+ #
111
+ # Extract XML file from zip archive
112
+ #
113
+ # @param [String] file path to achive
114
+ #
115
+ # @return [String] file content
116
+ #
117
+ def read_zip(file)
118
+ Zip::File.open(file) do |zf|
119
+ entry = zf.glob("**/*.xml").first
120
+ entry.get_input_stream.read
121
+ end
122
+ end
123
+
124
+ #
125
+ # Pre-filter the input file list down to the subset that actually
126
+ # has to be fully parsed.
127
+ #
128
+ # The IEEE rawbib dataset has ~50× duplication: every docnumber
129
+ # appears in `cache/` plus most `updates.YYYYMMDD/` folders. The
130
+ # original semantic is "latest update wins on disk", so for any
131
+ # docnumber that has at least one updates-folder file, the cache
132
+ # file's parse result is just thrown away. Pre-filter avoids
133
+ # parsing those throwaway files entirely.
134
+ #
135
+ # The cheap path here only has to extract three small XML elements
136
+ # (normtitle, stdnumber, standard_id) per file — done with
137
+ # regex on the raw XML so we skip lutaml-model's heavy DOM-to-
138
+ # object construction (which is what dominates fetch time).
139
+ #
140
+ # Selection rules:
141
+ # - For each docnumber with any updates-folder entry: keep only
142
+ # the highest-glob-idx updates-folder file.
143
+ # - For docnumbers with cache-folder entries only: keep all
144
+ # of them (commit_doc's matches-stdnumber dedup handles them).
145
+ # - Files where the cheap parse couldn't compute a docnumber
146
+ # are kept as-is — the full parse will surface any real error.
147
+ #
148
+ # Disable with IEEE_FETCH_PREFILTER=0.
149
+ #
150
+ def prefilter_winners(files)
151
+ threshold = Integer(ENV["IEEE_FETCH_PREFILTER_MIN"] || 200)
152
+ return files if files.size < threshold
153
+
154
+ procs = Integer(ENV["IEEE_FETCH_PROCESSES"] || Etc.nprocessors)
155
+ index = procs <= 1 ? prefilter_serial(files) : prefilter_parallel(files, procs)
156
+ select_prefilter_winners(index, files.size)
157
+ end
158
+
159
+ def prefilter_serial(files)
160
+ files.each_with_index.map { |f, i| extract_index_entry(i, f) }.compact
161
+ end
162
+
163
+ def prefilter_parallel(files, procs) # rubocop:disable Metrics/MethodLength
164
+ batch_size = Integer(ENV["IEEE_PREFILTER_BATCH"] || 5000)
165
+ batches = files.each_slice(batch_size).each_with_index.to_a
166
+
167
+ next_batch = 0
168
+ inflight = {}
169
+ collected = []
170
+
171
+ procs.times do
172
+ break if next_batch >= batches.size
173
+
174
+ inflight.merge!(spawn_prefilter_batch(*batches[next_batch], batch_size))
175
+ next_batch += 1
176
+ end
177
+
178
+ until inflight.empty?
179
+ pid = Process.wait
180
+ collected << inflight.delete(pid)
181
+
182
+ if next_batch < batches.size
183
+ inflight.merge!(spawn_prefilter_batch(*batches[next_batch], batch_size))
184
+ next_batch += 1
185
+ end
186
+ end
187
+
188
+ index = []
189
+ collected.each do |path|
190
+ next unless path && File.exist?(path) && File.size(path).positive?
191
+
192
+ index.concat(Marshal.load(File.binread(path)))
193
+ File.unlink(path)
194
+ end
195
+ index
196
+ end
197
+
198
+ def spawn_prefilter_batch(batch_files, batch_idx, batch_size)
199
+ require "tmpdir"
200
+ require "securerandom"
201
+ state_path = File.join(
202
+ Dir.tmpdir,
203
+ "ieee_prefilter_#{Process.pid}_#{batch_idx}_#{SecureRandom.hex(4)}.bin",
204
+ )
205
+ base_idx = batch_idx * batch_size
206
+
207
+ pid = Process.fork do
208
+ entries = batch_files.each_with_index.map do |file, i|
209
+ extract_index_entry(base_idx + i, file)
210
+ end.compact
211
+ File.binwrite(state_path, Marshal.dump(entries))
212
+ exit!(0)
213
+ end
214
+ { pid => state_path }
215
+ end
216
+
217
+ #
218
+ # Cheap-parse one file: read XML, regex-extract three fields,
219
+ # compute docnumber via the existing RawbibIdParser. Returns
220
+ # `[glob_idx, file, docnumber_or_nil, in_updates_folder?]`.
221
+ #
222
+ def extract_index_entry(idx, file)
223
+ xml = case File.extname(file)
224
+ when ".zip" then read_zip(file)
225
+ when ".xml" then File.read(file, encoding: "UTF-8")
226
+ end
227
+ return nil unless xml
228
+ return nil if cheap_extract_field(xml, "standard_id") == "0"
229
+
230
+ normtitle = cheap_extract_field(xml, "normtitle")
231
+ stdnumber = cheap_extract_field(xml, "stdnumber")
232
+ docnumber = nil
233
+ if normtitle && stdnumber
234
+ pubid = RawbibIdParser.parse(normtitle, stdnumber)
235
+ docnumber = pubid&.to_id
236
+ end
237
+ [idx, file, docnumber, file.include?("/updates.")]
238
+ rescue StandardError
239
+ # Cheap parse couldn't handle this file — keep it; full parse will
240
+ # either succeed or surface the real error.
241
+ [idx, file, nil, file.include?("/updates.")]
242
+ end
243
+
244
+ def cheap_extract_field(xml, tag)
245
+ m = xml.match(%r{<#{tag}[^>]*?>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</#{tag}>}m)
246
+ m && m[1].strip
247
+ end
248
+
249
+ def select_prefilter_winners(index, total)
250
+ unknown = index.select { |e| e[2].nil? }
251
+ by_doc = index.reject { |e| e[2].nil? }.group_by { |e| e[2] }
252
+
253
+ selected = []
254
+ by_doc.each_value do |entries|
255
+ updates = entries.select { |e| e[3] }
256
+ if updates.any?
257
+ selected << updates.max_by { |e| e[0] }
258
+ else
259
+ selected.concat(entries)
260
+ end
261
+ end
262
+
263
+ kept = (selected + unknown).sort_by { |e| e[0] }.map { |e| e[1] }
264
+ Util.warn "Prefilter: #{total} input files -> #{kept.size} winners " \
265
+ "(#{(100.0 * kept.size / total).round(1)}%)"
266
+ kept
267
+ end
268
+
269
+ #
270
+ # Parse files across a pool of short-lived forked workers. Each
271
+ # worker processes one bounded batch (IEEE_FETCH_BATCH files,
272
+ # default 5000), writes its output YAMLs to disk, marshals its
273
+ # local backrefs / crossrefs / errors to a tmp file, and exits.
274
+ # The parent keeps `procs` workers in flight; as each one exits
275
+ # it merges that worker's state and spawns the next batch.
276
+ #
277
+ # Why short-lived workers, not one long-running shard per core:
278
+ # Ruby's heap grows monotonically and the VM doesn't return
279
+ # freed memory to the OS, so a child that parses 50k files ends
280
+ # up at 1+ GB RSS even with the docs cache disabled. With ten
281
+ # such children the box swaps and slows to a crawl. Exiting a
282
+ # child after a batch of a few thousand files lets the OS
283
+ # reclaim its heap; the next fork starts fresh from the parent's
284
+ # baseline. Fork is cheap (copy-on-write), so the overhead is
285
+ # negligible compared to the memory savings.
286
+ #
287
+ # Caveats from sharding (same as the previous design):
288
+ # - Cross-batch duplicates: when the same docnumber appears in
289
+ # multiple batches, the last-finishing batch's write wins.
290
+ # Merged backrefs/crossrefs are still complete, so
291
+ # update_relations resolves cross-refs correctly.
292
+ # - "Document exists" warnings are per-batch, so cross-batch
293
+ # duplicates may not log a warning. Logging only.
294
+ #
295
+ # @param [Array<String>] files paths to rawbib XML/zip files
296
+ #
297
+ def process_files(files) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
298
+ procs = Integer(ENV["IEEE_FETCH_PROCESSES"] || Etc.nprocessors)
299
+ procs = 1 if files.empty? || procs < 2 || files.size < procs * 2
300
+
301
+ return run_shard(files, 0) if procs <= 1
302
+
303
+ batch_size = Integer(ENV["IEEE_FETCH_BATCH"] || 1000)
304
+ batches = files.each_slice(batch_size).each_with_index.to_a
305
+
306
+ state_paths = run_worker_pool(batches, procs)
307
+ merge_state_files(state_paths)
308
+ reconcile_staged_outputs
309
+ end
310
+
311
+ #
312
+ # Promote the highest-glob-index staged write per docnumber to its
313
+ # final on-disk filename, then delete any leftover staged files.
314
+ # Restores exact "latest update wins" semantics across batches:
315
+ # without this pass, a slow batch finishing late could overwrite a
316
+ # newer update that an earlier-completing batch had already saved.
317
+ #
318
+ def reconcile_staged_outputs
319
+ return if saved_writes.empty?
320
+
321
+ saved_writes.each do |docnumber, max_idx|
322
+ final = output_file(docnumber)
323
+ winner = "#{final}.#{max_idx}"
324
+ File.rename(winner, final) if File.exist?(winner)
325
+ end
326
+
327
+ # Stragglers: any remaining staged files (losing duplicates,
328
+ # or bib filenames that didn't end up in saved_writes due to a
329
+ # crash) get cleaned up so they don't pollute `data/`.
330
+ Dir.glob(File.join(@output, "*.#{@ext}.*")).each do |f|
331
+ File.unlink(f)
332
+ rescue StandardError
333
+ # ignore — best-effort cleanup
334
+ end
335
+ end
336
+
337
+ #
338
+ # Merge all batch state files into the parent's hashes. Runs once,
339
+ # after the worker pool has drained, so the parent's heap only
340
+ # has to hold the cumulative merged state (small) plus one batch's
341
+ # transient marshaled payload at a time.
342
+ #
343
+ def merge_state_files(state_paths)
344
+ state_paths.each_with_index do |path, i|
345
+ merge_batch_state(path)
346
+ # Periodic GC.start keeps the transient marshal allocations
347
+ # from piling up over hundreds of merges.
348
+ GC.start if (i % 50).zero?
349
+ end
350
+ end
351
+
352
+ #
353
+ # Maintain `procs` concurrent short-lived workers. Each Process.wait
354
+ # call blocks until any worker exits; we collect its state-file
355
+ # path and spawn the next batch (if any).
356
+ #
357
+ # Critically, we do NOT merge state into the parent's hashes here.
358
+ # Loading and merging dozens of MB of marshaled hashes per batch
359
+ # bloated the parent's heap into the multi-GB range, and every
360
+ # subsequent fork inherited that bloat via copy-on-write — driving
361
+ # the box into swap. By deferring all merging to after the parsing
362
+ # phase, the parent stays at ~baseline RSS while children are alive,
363
+ # so each fork's COW baseline is small.
364
+ #
365
+ # @return [Array<String>] state-file paths in completion order
366
+ #
367
+ def run_worker_pool(batches, procs) # rubocop:disable Metrics/MethodLength
368
+ next_batch = 0
369
+ inflight = {} # pid => state_path
370
+ collected = []
371
+
372
+ procs.times do
373
+ break if next_batch >= batches.size
374
+
375
+ inflight.merge!(spawn_batch(*batches[next_batch]))
376
+ next_batch += 1
377
+ end
378
+
379
+ until inflight.empty?
380
+ pid = Process.wait
381
+ collected << inflight.delete(pid)
382
+
383
+ if next_batch < batches.size
384
+ inflight.merge!(spawn_batch(*batches[next_batch]))
385
+ next_batch += 1
386
+ end
387
+ end
388
+
389
+ collected
390
+ end
391
+
392
+ #
393
+ # Fork one short-lived worker for a single batch. Returns a
394
+ # `{pid => state_path}` Hash. The worker writes its marshaled
395
+ # local state to `state_path` then exits; the tmp file is read
396
+ # and unlinked by the parent in `merge_batch_state`.
397
+ #
398
+ def spawn_batch(batch_files, batch_idx) # rubocop:disable Metrics/MethodLength
399
+ require "tmpdir"
400
+ require "securerandom"
401
+ state_path = File.join(
402
+ Dir.tmpdir,
403
+ "ieee_fetch_#{Process.pid}_#{batch_idx}_#{SecureRandom.hex(4)}.bin",
404
+ )
405
+ base_idx = batch_idx * Integer(ENV["IEEE_FETCH_BATCH"] || 1000)
406
+
407
+ pid = Process.fork do
408
+ batch_files.each_with_index do |file, i|
409
+ glob_idx = base_idx + i
410
+ result = parse_entry(glob_idx, file)
411
+ next unless result
412
+
413
+ _, _, doc, bib, local_errors = result
414
+ merge_errors(local_errors)
415
+ commit_doc(doc, bib, file, glob_idx)
416
+ end
417
+ File.binwrite(state_path, Marshal.dump(
418
+ backrefs: backrefs,
419
+ crossrefs: {}.merge(crossrefs),
420
+ errors: {}.merge(@errors),
421
+ saved_writes: saved_writes,
422
+ ))
423
+ exit!(0)
424
+ end
425
+
426
+ { pid => state_path }
427
+ end
428
+
429
+ #
430
+ # Read one batch's marshaled state, merge into parent state,
431
+ # remove the tmp file. Tolerates a missing/empty file (worker
432
+ # crash) by treating it as an empty merge.
433
+ #
434
+ def merge_batch_state(state_path)
435
+ if state_path && File.exist?(state_path) && File.size(state_path).positive?
436
+ payload = Marshal.load(File.binread(state_path))
437
+ merge_shard_state(payload)
438
+ end
439
+ ensure
440
+ File.unlink(state_path) if state_path && File.exist?(state_path)
441
+ end
442
+
443
+ #
444
+ # Merge one child's per-shard state into the parent's. backrefs uses
445
+ # ||= so the lowest-shard-id value wins for any amsid/docnumber pair
446
+ # that happens to appear in multiple shards (in practice they agree).
447
+ # `saved_writes` tracks the highest glob-index at which any worker
448
+ # saved a doc, so the parent can later rename the winning staged
449
+ # file to its final name.
450
+ #
451
+ def merge_shard_state(state)
452
+ state[:backrefs].each { |amsid, content| backrefs[amsid] ||= content }
453
+ state[:crossrefs].each { |dnum, refs| crossrefs[dnum].concat(refs) }
454
+ state[:errors].each { |k, v| @errors[k] &&= v }
455
+ (state[:saved_writes] || {}).each do |dnum, idx|
456
+ prev = saved_writes[dnum]
457
+ saved_writes[dnum] = idx if prev.nil? || idx > prev
458
+ end
459
+ end
460
+
461
+ #
462
+ # Process one shard sequentially. Either runs in a forked child or,
463
+ # when procs == 1, in the parent.
464
+ #
465
+ # `shard` is an array of [original_idx, file] tuples (or, when
466
+ # called from the procs==1 fallback, just the array of file paths
467
+ # — we normalize below).
468
+ #
469
+ def run_shard(shard, _shard_idx)
470
+ shard.each_with_index do |entry, i|
471
+ idx, file = entry.is_a?(Array) ? entry : [i, entry]
472
+ result = parse_entry(idx, file)
473
+ next unless result
474
+
475
+ _, _, doc, bib, local_errors = result
476
+ merge_errors(local_errors)
477
+ commit_doc(doc, bib, file)
478
+ end
479
+ end
480
+
481
+ #
482
+ # Worker-thread entry point: read file, parse XML, build bib.
483
+ # Returns nil for files we should skip; otherwise a tuple consumed
484
+ # in submission order by the main-thread commit loop.
485
+ #
486
+ # @param [Integer] idx original glob index (preserves dedup order)
487
+ # @param [String] file path to rawbib file
488
+ #
489
+ # @return [Array, nil] [idx, file, doc, bib, local_errors] or nil
490
+ #
491
+ def parse_entry(idx, file)
492
+ xml = case File.extname(file)
493
+ when ".zip" then read_zip file
494
+ when ".xml" then File.read file, encoding: "UTF-8"
495
+ end
496
+ doc = begin
497
+ ::Ieee::Idams::Publication.from_xml(xml)
498
+ rescue StandardError
499
+ Util.warn "Empty file: `#{file}`"
500
+ return nil
501
+ end
502
+ return nil if doc.publicationinfo&.standard_id == "0"
503
+
504
+ local_errors = Hash.new(true)
505
+ bib = IdamsParser.new(doc, self, local_errors).parse
506
+ if bib.docnumber.nil?
507
+ Util.warn "PubID parse error. Normtitle: `#{doc.normtitle}`, file: `#{file}`"
508
+ return nil
509
+ end
510
+ [idx, file, doc, bib, local_errors]
511
+ rescue StandardError => e
512
+ Util.error "File: #{file}\n#{e.message}\n#{e.backtrace}"
513
+ nil
514
+ end
515
+
516
+ #
517
+ # Merge a worker's local errors hash into the shared @errors hash,
518
+ # preserving the existing AND semantics (`@errors[k] &&= v`).
519
+ #
520
+ def merge_errors(local_errors)
521
+ local_errors.each { |k, v| @errors[k] &&= v }
522
+ end
523
+
524
+ #
525
+ # Dedup against backrefs and save. This runs once per parsed file —
526
+ # in the parent for the procs==1 fallback, or in each forked child
527
+ # for its shard. Same logic the old fetch_doc tail had, plus
528
+ # optional staged-output bookkeeping when `glob_idx` is provided.
529
+ #
530
+ # When `glob_idx` is given (parallel mode), save_doc writes to a
531
+ # per-glob-index suffixed path; the parent reconciles after the
532
+ # parsing phase and renames the highest-glob-index winner per
533
+ # docnumber to the final filename. This preserves the original
534
+ # "latest update wins on disk" semantic across batch boundaries
535
+ # — without it, a slow batch finishing late could overwrite a
536
+ # newer update written by an earlier-completing batch.
537
+ #
538
+ def commit_doc(doc, bib, filename, glob_idx = nil)
539
+ amsid = doc.publicationinfo.amsid
540
+ if backrefs.value?(bib.docidentifier[0].content) && /updates\.\d+/ !~ filename
541
+ oamsid = backrefs.key bib.docidentifier[0].content
542
+ Util.warn "Document exists ID: `#{bib.docidentifier[0].content}` AMSID: " \
543
+ "`#{amsid}` source: `#{filename}`. Other AMSID: `#{oamsid}`"
544
+ if bib.docidentifier.find(&:primary).content.include?(doc.publicationinfo.stdnumber)
545
+ save_doc(bib, glob_idx) # rewrite file if the PubID matches to the stdnumber
546
+ backrefs[amsid] = bib.docidentifier[0].content
547
+ track_save(bib.docnumber, glob_idx)
548
+ end
549
+ else
550
+ save_doc(bib, glob_idx)
551
+ backrefs[amsid] = bib.docidentifier[0].content
552
+ track_save(bib.docnumber, glob_idx)
553
+ end
554
+ end
555
+
556
+ #
557
+ # Record that we wrote a staged copy of `docnumber` at this
558
+ # `glob_idx`. The parent later picks the highest tracked idx
559
+ # per docnumber as the surviving on-disk version.
560
+ #
561
+ def track_save(docnumber, glob_idx)
562
+ return unless glob_idx
563
+
564
+ prev = saved_writes[docnumber]
565
+ saved_writes[docnumber] = glob_idx if prev.nil? || glob_idx > prev
566
+ end
567
+
568
+ #
569
+ # Save document to file. When `glob_idx` is provided (parallel
570
+ # mode), writes to a per-glob-index suffixed staging path so
571
+ # concurrent workers can't clobber each other's files; the parent
572
+ # reconciles after parsing. With no glob_idx, writes the final
573
+ # filename directly (sequential mode and update_relations).
574
+ #
575
+ # @param [RelatonIeee::IeeeBibliographicItem] bib
576
+ # @param [Integer, nil] glob_idx position in the original file glob
577
+ #
578
+ def save_doc(bib, glob_idx = nil)
579
+ path = output_file(bib.docnumber)
580
+ path = "#{path}.#{glob_idx}" if glob_idx
581
+ File.write path, serialize(bib), encoding: "UTF-8"
582
+ end
583
+
584
+ def to_yaml(bib) = bib.to_yaml
585
+ def to_xml(bib) = bib.to_xml(bibdata: true)
586
+ def to_bibxml(bib) = bib.to_rfcxml
587
+
588
+ #
589
+ # Resolve cross-references collected during parse. Uses the in-memory
590
+ # `docs` cache so we don't re-read+re-deserialize files from disk, and
591
+ # writes each mutated bib once instead of once per relation.
592
+ #
593
+ def update_relations # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
594
+ crossrefs.each do |dnum, rfs|
595
+ bib = nil
596
+ mutated = false
597
+ rfs.each do |rf|
598
+ if backrefs[rf[:amsid]]
599
+ rel = create_relation(rf[:type], backrefs[rf[:amsid]])
600
+ if rel
601
+ bib ||= docs[dnum] || read_bib(dnum)
602
+ bib.relation << rel
603
+ mutated = true
604
+ end
605
+ else
606
+ Util.warn "Unresolved relation: '#{rf[:amsid]}' type: '#{rf[:type]}' for '#{dnum}'"
607
+ end
608
+ end
609
+ save_doc(bib) if mutated
610
+ end
611
+ end
612
+
613
+ #
614
+ # Read document form BibXML/BibYAML file
615
+ #
616
+ # @param [String] docnumber
617
+ #
618
+ # @return [RelatonIeee::IeeeBibliographicItem]
619
+ #
620
+ def read_bib(docnumber)
621
+ c = File.read output_file(docnumber), encoding: "UTF-8"
622
+ case @format
623
+ when "xml" then Item.from_xml c
624
+ when "bibxml" then Converter::BibXml.to_item c
625
+ else Item.from_yaml c
626
+ end
627
+ end
628
+ end
629
+ end
630
+ end
@@ -0,0 +1,9 @@
1
+ module Relaton
2
+ module Ieee
3
+ class Doctype < Bib::Doctype
4
+ TYPES = %w[guide recommended-practice standard whitepaper redline other].freeze
5
+
6
+ attribute :type, :string, values: TYPES
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,19 @@
1
+ require_relative "balloting_group"
2
+
3
+ module Relaton
4
+ module Ieee
5
+ class EditorialGroup < Lutaml::Model::Serializable
6
+ attribute :society, :string
7
+ attribute :balloting_group, BallotingGroup
8
+ attribute :working_group, :string
9
+ attribute :committee, :string
10
+
11
+ xml do
12
+ map_element "society", to: :society
13
+ map_element "balloting-group", to: :balloting_group
14
+ map_element "working-group", to: :working_group
15
+ map_element "committee", to: :committee
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,36 @@
1
+ require_relative "doctype"
2
+
3
+ module Relaton
4
+ module Ieee
5
+ class Ext < Bib::Ext
6
+ attribute :doctype, Doctype
7
+ attribute :subdoctype, :string, values: %w[amendment corrigendum erratum]
8
+ attribute :trial_use, :boolean
9
+ attribute :standard_status, :string, values: %w[Inactive Active Superseded]
10
+ attribute :standard_modified, :string, values: %w[Draft Withdrawn Superseded Superseded Reserved Redline]
11
+ attribute :pubstatus, :string, values: %w[Active Inactive]
12
+ attribute :holdstatus, :string, values: %w[Hold Publish]
13
+ attribute :program, :string
14
+
15
+ xml do
16
+ map_element "trial-use", to: :trial_use
17
+ map_element "standard_status", to: :standard_status
18
+ map_element "standard_modified", to: :standard_modified
19
+ map_element "pubstatus", to: :pubstatus
20
+ map_element "holdstatus", to: :holdstatus
21
+ map_element "program", to: :program
22
+ end
23
+
24
+ key_value do
25
+ map_element "trial_use", to: :trial_use
26
+ map_element "standard_status", to: :standard_status
27
+ map_element "standard_modified", to: :standard_modified
28
+ map_element "pubstatus", to: :pubstatus
29
+ map_element "holdstatus", to: :holdstatus
30
+ map_element "program", to: :program
31
+ end
32
+
33
+ def get_schema_version = Relaton.schema_versions["relaton-model-ieee"]
34
+ end
35
+ end
36
+ end