relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (702) hide show
  1. checksums.yaml +4 -4
  2. data/{docs/README.adoc → README.adoc} +165 -38
  3. data/bin/console +0 -1
  4. data/lib/relaton/3gpp/bibdata.rb +9 -0
  5. data/lib/relaton/3gpp/bibitem.rb +9 -0
  6. data/lib/relaton/3gpp/bibliography.rb +123 -0
  7. data/lib/relaton/3gpp/data_fetcher.rb +303 -0
  8. data/lib/relaton/3gpp/docidentifier.rb +114 -0
  9. data/lib/relaton/3gpp/doctype.rb +9 -0
  10. data/lib/relaton/3gpp/ext.rb +31 -0
  11. data/lib/relaton/3gpp/item.rb +18 -0
  12. data/lib/relaton/3gpp/item_data.rb +15 -0
  13. data/lib/relaton/3gpp/parser.rb +400 -0
  14. data/lib/relaton/3gpp/processor.rb +71 -0
  15. data/lib/relaton/3gpp/release.rb +34 -0
  16. data/lib/relaton/3gpp/util.rb +8 -0
  17. data/lib/relaton/3gpp.rb +29 -0
  18. data/lib/relaton/adobe/bibdata.rb +8 -0
  19. data/lib/relaton/adobe/bibitem.rb +8 -0
  20. data/lib/relaton/adobe/bibliography.rb +92 -0
  21. data/lib/relaton/adobe/docidentifier.rb +49 -0
  22. data/lib/relaton/adobe/doctype.rb +14 -0
  23. data/lib/relaton/adobe/ext.rb +32 -0
  24. data/lib/relaton/adobe/item.rb +15 -0
  25. data/lib/relaton/adobe/item_base.rb +18 -0
  26. data/lib/relaton/adobe/item_data.rb +6 -0
  27. data/lib/relaton/adobe/processor.rb +45 -0
  28. data/lib/relaton/adobe/util.rb +8 -0
  29. data/lib/relaton/adobe.rb +37 -0
  30. data/lib/relaton/bib/converter/asciibib/to_asciibib.rb +663 -0
  31. data/lib/relaton/bib/converter/asciibib.rb +13 -0
  32. data/lib/relaton/bib/converter/bibtex/from_bibtex.rb +245 -0
  33. data/lib/relaton/bib/converter/bibtex/to_bibtex.rb +341 -0
  34. data/lib/relaton/bib/converter/bibtex.rb +23 -0
  35. data/lib/relaton/bib/converter/bibxml/from_rfcxml.rb +386 -0
  36. data/lib/relaton/bib/converter/bibxml/from_rfcxml_referencegroup.rb +71 -0
  37. data/lib/relaton/bib/converter/bibxml/to_rfcxml.rb +308 -0
  38. data/lib/relaton/bib/converter/bibxml/to_rfcxml_referencegroup.rb +52 -0
  39. data/lib/relaton/bib/converter/bibxml.rb +51 -0
  40. data/lib/relaton/bib/hash_parser_v1.rb +767 -0
  41. data/lib/relaton/bib/item_data.rb +229 -0
  42. data/lib/relaton/bib/model/abstract.rb +16 -0
  43. data/lib/relaton/bib/model/address.rb +22 -0
  44. data/lib/relaton/bib/model/affiliation.rb +16 -0
  45. data/lib/relaton/bib/model/bibdata.rb +11 -0
  46. data/lib/relaton/bib/model/bibdata_shared.rb +12 -0
  47. data/lib/relaton/bib/model/bibitem.rb +11 -0
  48. data/lib/relaton/bib/model/bibitem_shared.rb +12 -0
  49. data/lib/relaton/bib/model/contact.rb +18 -0
  50. data/lib/relaton/bib/model/contribution_info.rb +15 -0
  51. data/lib/relaton/bib/model/contributor.rb +29 -0
  52. data/lib/relaton/bib/model/copyright.rb +27 -0
  53. data/lib/relaton/bib/model/date.rb +31 -0
  54. data/lib/relaton/bib/model/depiction.rb +16 -0
  55. data/lib/relaton/bib/model/docidentifier.rb +49 -0
  56. data/lib/relaton/bib/model/doctype.rb +14 -0
  57. data/lib/relaton/bib/model/edition.rb +14 -0
  58. data/lib/relaton/bib/model/ext.rb +39 -0
  59. data/lib/relaton/bib/model/extent.rb +16 -0
  60. data/lib/relaton/bib/model/formattedref.rb +43 -0
  61. data/lib/relaton/bib/model/full_name_type.rb +64 -0
  62. data/lib/relaton/bib/model/fullname.rb +11 -0
  63. data/lib/relaton/bib/model/ics.rb +49 -0
  64. data/lib/relaton/bib/model/image.rb +28 -0
  65. data/lib/relaton/bib/model/item.rb +96 -0
  66. data/lib/relaton/bib/model/item_base.rb +20 -0
  67. data/lib/relaton/bib/model/item_shared.rb +88 -0
  68. data/lib/relaton/bib/model/keyword.rb +30 -0
  69. data/lib/relaton/bib/model/locality.rb +18 -0
  70. data/lib/relaton/bib/model/locality_stack.rb +14 -0
  71. data/lib/relaton/bib/model/localized_string.rb +48 -0
  72. data/lib/relaton/bib/model/localized_string_attrs.rb +24 -0
  73. data/lib/relaton/bib/model/logo.rb +14 -0
  74. data/lib/relaton/bib/model/medium.rb +22 -0
  75. data/lib/relaton/bib/model/note.rb +16 -0
  76. data/lib/relaton/bib/model/organization.rb +13 -0
  77. data/lib/relaton/bib/model/organization_type.rb +42 -0
  78. data/lib/relaton/bib/model/person.rb +36 -0
  79. data/lib/relaton/bib/model/phone.rb +14 -0
  80. data/lib/relaton/bib/model/place.rb +33 -0
  81. data/lib/relaton/bib/model/price.rb +14 -0
  82. data/lib/relaton/bib/model/relation.rb +43 -0
  83. data/lib/relaton/bib/model/series.rb +34 -0
  84. data/lib/relaton/bib/model/size.rb +23 -0
  85. data/lib/relaton/bib/model/source_locality_stack.rb +14 -0
  86. data/lib/relaton/bib/model/status.rb +27 -0
  87. data/lib/relaton/bib/model/structured_identifier.rb +49 -0
  88. data/lib/relaton/bib/model/subdivision.rb +16 -0
  89. data/lib/relaton/bib/model/title.rb +56 -0
  90. data/lib/relaton/bib/model/type/plain_date.rb +16 -0
  91. data/lib/relaton/bib/model/type/string_date.rb +48 -0
  92. data/lib/relaton/bib/model/uri.rb +18 -0
  93. data/lib/relaton/bib/model/validity.rb +16 -0
  94. data/lib/relaton/bib/model/version.rb +43 -0
  95. data/lib/relaton/bib/namespace_helper.rb +21 -0
  96. data/lib/relaton/bib/sanitizer.rb +264 -0
  97. data/lib/relaton/bib/util.rb +18 -0
  98. data/lib/relaton/bib/versions.json +35 -0
  99. data/lib/relaton/bib.rb +46 -0
  100. data/lib/relaton/bipm/bibliography.rb +231 -0
  101. data/lib/relaton/bipm/converter/asciibib.rb +64 -0
  102. data/lib/relaton/bipm/data_fetcher.rb +81 -0
  103. data/lib/relaton/bipm/data_outcomes_parser.rb +656 -0
  104. data/lib/relaton/bipm/id_parser.rb +278 -0
  105. data/lib/relaton/bipm/item_data.rb +47 -0
  106. data/lib/relaton/bipm/model/bibdata.rb +9 -0
  107. data/lib/relaton/bipm/model/bibitem.rb +9 -0
  108. data/lib/relaton/bipm/model/comment_period.rb +13 -0
  109. data/lib/relaton/bipm/model/doctype.rb +12 -0
  110. data/lib/relaton/bipm/model/ext.rb +38 -0
  111. data/lib/relaton/bipm/model/item.rb +11 -0
  112. data/lib/relaton/bipm/model/structured_identifier.rb +36 -0
  113. data/lib/relaton/bipm/processor.rb +69 -0
  114. data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +111 -0
  115. data/lib/relaton/bipm/rawdata_bipm_metrologia/fetcher.rb +172 -0
  116. data/lib/relaton/bipm/rawdata_bipm_metrologia/niso_jats_parser.rb +353 -0
  117. data/lib/relaton/bipm/si_brochure_parser.rb +188 -0
  118. data/lib/relaton/bipm/util.rb +8 -0
  119. data/lib/relaton/bipm.rb +38 -0
  120. data/lib/relaton/bsi/bibliography.rb +196 -0
  121. data/lib/relaton/bsi/hit.rb +28 -0
  122. data/lib/relaton/bsi/hit_collection.rb +113 -0
  123. data/lib/relaton/bsi/item_data.rb +13 -0
  124. data/lib/relaton/bsi/model/bibdata.rb +8 -0
  125. data/lib/relaton/bsi/model/bibitem.rb +8 -0
  126. data/lib/relaton/bsi/model/docidentifier.rb +107 -0
  127. data/lib/relaton/bsi/model/doctype.rb +14 -0
  128. data/lib/relaton/bsi/model/ext.rb +17 -0
  129. data/lib/relaton/bsi/model/item.rb +17 -0
  130. data/lib/relaton/bsi/model/item_base.rb +22 -0
  131. data/lib/relaton/bsi/model/relation.rb +9 -0
  132. data/lib/relaton/bsi/processor.rb +43 -0
  133. data/lib/relaton/bsi/schema.json +24882 -0
  134. data/lib/relaton/bsi/scraper.rb +288 -0
  135. data/lib/relaton/bsi/util.rb +8 -0
  136. data/lib/relaton/bsi.rb +25 -0
  137. data/lib/relaton/calconnect/bibliography.rb +93 -0
  138. data/lib/relaton/calconnect/data_fetcher.rb +174 -0
  139. data/lib/relaton/calconnect/docidentifier.rb +80 -0
  140. data/lib/relaton/calconnect/hit.rb +14 -0
  141. data/lib/relaton/calconnect/hit_collection.rb +92 -0
  142. data/lib/relaton/calconnect/item_data.rb +12 -0
  143. data/lib/relaton/calconnect/model/bibdata.rb +8 -0
  144. data/lib/relaton/calconnect/model/bibitem.rb +8 -0
  145. data/lib/relaton/calconnect/model/doctype.rb +11 -0
  146. data/lib/relaton/calconnect/model/ext.rb +11 -0
  147. data/lib/relaton/calconnect/model/item.rb +20 -0
  148. data/lib/relaton/calconnect/processor.rb +72 -0
  149. data/lib/relaton/calconnect/scraper.rb +87 -0
  150. data/lib/relaton/calconnect/util.rb +8 -0
  151. data/lib/relaton/calconnect.rb +36 -0
  152. data/lib/relaton/ccsds/bibliography.rb +63 -0
  153. data/lib/relaton/ccsds/data/fetcher.rb +252 -0
  154. data/lib/relaton/ccsds/data/iso_references.rb +30 -0
  155. data/lib/relaton/ccsds/data/parser.rb +194 -0
  156. data/lib/relaton/ccsds/hit.rb +24 -0
  157. data/lib/relaton/ccsds/hit_collection.rb +47 -0
  158. data/lib/relaton/ccsds/item_data.rb +9 -0
  159. data/lib/relaton/ccsds/model/bibdata.rb +8 -0
  160. data/lib/relaton/ccsds/model/bibitem.rb +8 -0
  161. data/lib/relaton/ccsds/model/docidentifier.rb +121 -0
  162. data/lib/relaton/ccsds/model/doctype.rb +9 -0
  163. data/lib/relaton/ccsds/model/ext.rb +19 -0
  164. data/lib/relaton/ccsds/model/item.rb +15 -0
  165. data/lib/relaton/ccsds/processor.rb +68 -0
  166. data/lib/relaton/ccsds/util.rb +10 -0
  167. data/lib/relaton/ccsds.rb +33 -0
  168. data/lib/relaton/cen/bibliography.rb +149 -0
  169. data/lib/relaton/cen/committees.yaml +66 -0
  170. data/lib/relaton/cen/hit.rb +31 -0
  171. data/lib/relaton/cen/hit_collection.rb +116 -0
  172. data/lib/relaton/cen/item_data.rb +7 -0
  173. data/lib/relaton/cen/model/bibdata.rb +8 -0
  174. data/lib/relaton/cen/model/bibitem.rb +8 -0
  175. data/lib/relaton/cen/model/docidentifier.rb +100 -0
  176. data/lib/relaton/cen/model/ext.rb +11 -0
  177. data/lib/relaton/cen/model/item.rb +14 -0
  178. data/lib/relaton/cen/model/structured_identifier.rb +9 -0
  179. data/lib/relaton/cen/processor.rb +45 -0
  180. data/lib/relaton/cen/scraper.rb +218 -0
  181. data/lib/relaton/cen/util.rb +8 -0
  182. data/lib/relaton/cen.rb +30 -0
  183. data/lib/relaton/cie/bibdata.rb +8 -0
  184. data/lib/relaton/cie/bibitem.rb +8 -0
  185. data/lib/relaton/cie/bibliography.rb +31 -0
  186. data/lib/relaton/cie/data_fetcher.rb +540 -0
  187. data/lib/relaton/cie/ext.rb +7 -0
  188. data/lib/relaton/cie/item.rb +11 -0
  189. data/lib/relaton/cie/item_data.rb +6 -0
  190. data/lib/relaton/cie/processor.rb +68 -0
  191. data/lib/relaton/cie/scrapper.rb +52 -0
  192. data/lib/relaton/cie/util.rb +8 -0
  193. data/lib/relaton/cie.rb +29 -0
  194. data/lib/relaton/core/array_wrapper.rb +20 -0
  195. data/lib/relaton/core/data_fetcher.rb +244 -0
  196. data/lib/relaton/core/date_parser.rb +42 -0
  197. data/lib/relaton/core/governor.rb +320 -0
  198. data/lib/relaton/core/hash_keys_sybolizer.rb +19 -0
  199. data/lib/relaton/core/hit.rb +49 -0
  200. data/lib/relaton/core/hit_collection.rb +118 -0
  201. data/lib/relaton/core/pacer.rb +134 -0
  202. data/lib/relaton/core/processor.rb +67 -0
  203. data/lib/relaton/core/request_error.rb +14 -0
  204. data/lib/relaton/core/workers_pool.rb +45 -0
  205. data/lib/relaton/core.rb +12 -0
  206. data/lib/relaton/db/registry.rb +44 -4
  207. data/lib/relaton/db.rb +0 -1
  208. data/lib/relaton/doi/crossref.rb +89 -0
  209. data/lib/relaton/doi/parser.rb +921 -0
  210. data/lib/relaton/doi/processor.rb +65 -0
  211. data/lib/relaton/doi/util.rb +8 -0
  212. data/lib/relaton/doi.rb +20 -0
  213. data/lib/relaton/easc/bibdata.rb +8 -0
  214. data/lib/relaton/easc/bibitem.rb +8 -0
  215. data/lib/relaton/easc/bibliography.rb +95 -0
  216. data/lib/relaton/easc/docidentifier.rb +100 -0
  217. data/lib/relaton/easc/doctype.rb +14 -0
  218. data/lib/relaton/easc/ext.rb +44 -0
  219. data/lib/relaton/easc/item.rb +13 -0
  220. data/lib/relaton/easc/item_base.rb +18 -0
  221. data/lib/relaton/easc/item_data.rb +6 -0
  222. data/lib/relaton/easc/processor.rb +46 -0
  223. data/lib/relaton/easc/util.rb +8 -0
  224. data/lib/relaton/easc.rb +35 -0
  225. data/lib/relaton/ecma/bibdata.rb +8 -0
  226. data/lib/relaton/ecma/bibitem.rb +8 -0
  227. data/lib/relaton/ecma/bibliography.rb +149 -0
  228. data/lib/relaton/ecma/data_fetcher.rb +162 -0
  229. data/lib/relaton/ecma/data_parser.rb +49 -0
  230. data/lib/relaton/ecma/docidentifier.rb +124 -0
  231. data/lib/relaton/ecma/edition_parser.rb +80 -0
  232. data/lib/relaton/ecma/ext.rb +7 -0
  233. data/lib/relaton/ecma/item.rb +13 -0
  234. data/lib/relaton/ecma/item_data.rb +6 -0
  235. data/lib/relaton/ecma/memento_parser.rb +60 -0
  236. data/lib/relaton/ecma/page_fetcher.rb +39 -0
  237. data/lib/relaton/ecma/parser_common.rb +33 -0
  238. data/lib/relaton/ecma/processor.rb +69 -0
  239. data/lib/relaton/ecma/standard_parser.rb +134 -0
  240. data/lib/relaton/ecma/util.rb +8 -0
  241. data/lib/relaton/ecma.rb +33 -0
  242. data/lib/relaton/etsi/bibdata.rb +10 -0
  243. data/lib/relaton/etsi/bibitem.rb +10 -0
  244. data/lib/relaton/etsi/bibliography.rb +111 -0
  245. data/lib/relaton/etsi/data_fetcher.rb +167 -0
  246. data/lib/relaton/etsi/data_parser.rb +208 -0
  247. data/lib/relaton/etsi/doctype.rb +30 -0
  248. data/lib/relaton/etsi/ext.rb +31 -0
  249. data/lib/relaton/etsi/item.rb +15 -0
  250. data/lib/relaton/etsi/item_data.rb +6 -0
  251. data/lib/relaton/etsi/processor.rb +71 -0
  252. data/lib/relaton/etsi/pubid.rb +37 -0
  253. data/lib/relaton/etsi/status.rb +13 -0
  254. data/lib/relaton/etsi/util.rb +8 -0
  255. data/lib/relaton/etsi.rb +26 -0
  256. data/lib/relaton/gb/bibdata.rb +8 -0
  257. data/lib/relaton/gb/bibitem.rb +8 -0
  258. data/lib/relaton/gb/bibliography.rb +171 -0
  259. data/lib/relaton/gb/ccs.rb +14 -0
  260. data/lib/relaton/gb/committee.rb +13 -0
  261. data/lib/relaton/gb/docidentifier.rb +72 -0
  262. data/lib/relaton/gb/doctype.rb +9 -0
  263. data/lib/relaton/gb/ext.rb +36 -0
  264. data/lib/relaton/gb/gb_scraper.rb +61 -0
  265. data/lib/relaton/gb/gb_type.rb +20 -0
  266. data/lib/relaton/gb/hit.rb +48 -0
  267. data/lib/relaton/gb/hit_collection.rb +19 -0
  268. data/lib/relaton/gb/item.rb +13 -0
  269. data/lib/relaton/gb/item_data.rb +6 -0
  270. data/lib/relaton/gb/processor.rb +49 -0
  271. data/lib/relaton/gb/project_number.rb +38 -0
  272. data/lib/relaton/gb/scraper.rb +221 -0
  273. data/lib/relaton/gb/sec_scraper.rb +92 -0
  274. data/lib/relaton/gb/stage_name.rb +13 -0
  275. data/lib/relaton/gb/structured_identifier.rb +26 -0
  276. data/lib/relaton/gb/t_scraper.rb +126 -0
  277. data/lib/relaton/gb/util.rb +8 -0
  278. data/lib/relaton/gb/yaml/prefixes.yaml +200 -0
  279. data/lib/relaton/gb.rb +33 -0
  280. data/lib/relaton/gost/bibdata.rb +8 -0
  281. data/lib/relaton/gost/bibitem.rb +8 -0
  282. data/lib/relaton/gost/bibliography.rb +107 -0
  283. data/lib/relaton/gost/docidentifier.rb +80 -0
  284. data/lib/relaton/gost/doctype.rb +16 -0
  285. data/lib/relaton/gost/ext.rb +46 -0
  286. data/lib/relaton/gost/item.rb +15 -0
  287. data/lib/relaton/gost/item_base.rb +18 -0
  288. data/lib/relaton/gost/item_data.rb +6 -0
  289. data/lib/relaton/gost/processor.rb +49 -0
  290. data/lib/relaton/gost/util.rb +8 -0
  291. data/lib/relaton/gost.rb +36 -0
  292. data/lib/relaton/iala/bibdata.rb +8 -0
  293. data/lib/relaton/iala/bibitem.rb +8 -0
  294. data/lib/relaton/iala/bibliography.rb +146 -0
  295. data/lib/relaton/iala/docidentifier.rb +89 -0
  296. data/lib/relaton/iala/doctype.rb +18 -0
  297. data/lib/relaton/iala/ext.rb +32 -0
  298. data/lib/relaton/iala/item.rb +21 -0
  299. data/lib/relaton/iala/item_base.rb +18 -0
  300. data/lib/relaton/iala/item_data.rb +6 -0
  301. data/lib/relaton/iala/processor.rb +43 -0
  302. data/lib/relaton/iala/relation.rb +7 -0
  303. data/lib/relaton/iala/util.rb +8 -0
  304. data/lib/relaton/iala.rb +35 -0
  305. data/lib/relaton/iana/bibdata.rb +8 -0
  306. data/lib/relaton/iana/bibitem.rb +8 -0
  307. data/lib/relaton/iana/bibliography.rb +100 -0
  308. data/lib/relaton/iana/data_fetcher.rb +101 -0
  309. data/lib/relaton/iana/item.rb +7 -0
  310. data/lib/relaton/iana/item_data.rb +6 -0
  311. data/lib/relaton/iana/parser.rb +146 -0
  312. data/lib/relaton/iana/processor.rb +70 -0
  313. data/lib/relaton/iana/util.rb +8 -0
  314. data/lib/relaton/iana.rb +40 -0
  315. data/lib/relaton/iec/bibliography.rb +283 -0
  316. data/lib/relaton/iec/data_fetcher.rb +222 -0
  317. data/lib/relaton/iec/data_parser.rb +391 -0
  318. data/lib/relaton/iec/hit.rb +26 -0
  319. data/lib/relaton/iec/hit_collection.rb +138 -0
  320. data/lib/relaton/iec/item_data.rb +7 -0
  321. data/lib/relaton/iec/model/bibdata.rb +8 -0
  322. data/lib/relaton/iec/model/bibitem.rb +8 -0
  323. data/lib/relaton/iec/model/docidentifier.rb +135 -0
  324. data/lib/relaton/iec/model/doctype.rb +12 -0
  325. data/lib/relaton/iec/model/ext.rb +53 -0
  326. data/lib/relaton/iec/model/item.rb +20 -0
  327. data/lib/relaton/iec/model/item_base.rb +12 -0
  328. data/lib/relaton/iec/model/relation.rb +7 -0
  329. data/lib/relaton/iec/model/stage_name.rb +13 -0
  330. data/lib/relaton/iec/processor.rb +74 -0
  331. data/lib/relaton/iec/statuses.yml +199 -0
  332. data/lib/relaton/iec/util.rb +8 -0
  333. data/lib/relaton/iec.rb +98 -0
  334. data/lib/relaton/ieee/balloting_group.rb +13 -0
  335. data/lib/relaton/ieee/bibdata.rb +8 -0
  336. data/lib/relaton/ieee/bibitem.rb +8 -0
  337. data/lib/relaton/ieee/bibliography.rb +73 -0
  338. data/lib/relaton/ieee/converter/bibxml/from_rfcxml.rb +10 -0
  339. data/lib/relaton/ieee/converter/bibxml/from_rfcxml_referencegroup.rb +10 -0
  340. data/lib/relaton/ieee/converter/bibxml.rb +20 -0
  341. data/lib/relaton/ieee/data_fetcher.rb +771 -0
  342. data/lib/relaton/ieee/doctype.rb +9 -0
  343. data/lib/relaton/ieee/editorial_group.rb +19 -0
  344. data/lib/relaton/ieee/ext.rb +36 -0
  345. data/lib/relaton/ieee/idams_parser.rb +330 -0
  346. data/lib/relaton/ieee/item.rb +11 -0
  347. data/lib/relaton/ieee/item_data.rb +7 -0
  348. data/lib/relaton/ieee/processor.rb +71 -0
  349. data/lib/relaton/ieee/rawbib_id_parser.rb +753 -0
  350. data/lib/relaton/ieee/util.rb +8 -0
  351. data/lib/relaton/ieee.rb +30 -0
  352. data/lib/relaton/ietf/bibdata.rb +8 -0
  353. data/lib/relaton/ietf/bibitem.rb +8 -0
  354. data/lib/relaton/ietf/bibliography.rb +35 -0
  355. data/lib/relaton/ietf/bibxml_parser.rb +232 -0
  356. data/lib/relaton/ietf/data_fetcher.rb +561 -0
  357. data/lib/relaton/ietf/doctype.rb +9 -0
  358. data/lib/relaton/ietf/ext.rb +63 -0
  359. data/lib/relaton/ietf/item.rb +16 -0
  360. data/lib/relaton/ietf/item_base.rb +18 -0
  361. data/lib/relaton/ietf/item_data.rb +6 -0
  362. data/lib/relaton/ietf/processing_instructions.rb +79 -0
  363. data/lib/relaton/ietf/processor.rb +72 -0
  364. data/lib/relaton/ietf/relation.rb +9 -0
  365. data/lib/relaton/ietf/rfc/abstract.rb +19 -0
  366. data/lib/relaton/ietf/rfc/author.rb +21 -0
  367. data/lib/relaton/ietf/rfc/entry.rb +446 -0
  368. data/lib/relaton/ietf/rfc/entry_date.rb +21 -0
  369. data/lib/relaton/ietf/rfc/format.rb +19 -0
  370. data/lib/relaton/ietf/rfc/index.rb +46 -0
  371. data/lib/relaton/ietf/rfc/is_also.rb +21 -0
  372. data/lib/relaton/ietf/rfc/keywords.rb +19 -0
  373. data/lib/relaton/ietf/rfc/rfc_index_namespace.rb +11 -0
  374. data/lib/relaton/ietf/scraper.rb +101 -0
  375. data/lib/relaton/ietf/util.rb +8 -0
  376. data/lib/relaton/ietf/wg_name_resolver.rb +42 -0
  377. data/lib/relaton/ietf.rb +30 -0
  378. data/lib/relaton/iho/bibdata.rb +8 -0
  379. data/lib/relaton/iho/bibitem.rb +8 -0
  380. data/lib/relaton/iho/bibliography.rb +114 -0
  381. data/lib/relaton/iho/comment_period.rb +13 -0
  382. data/lib/relaton/iho/docidentifier.rb +31 -0
  383. data/lib/relaton/iho/doctype.rb +10 -0
  384. data/lib/relaton/iho/ext.rb +20 -0
  385. data/lib/relaton/iho/hash_parser_v1.rb +144 -0
  386. data/lib/relaton/iho/item.rb +20 -0
  387. data/lib/relaton/iho/item_base.rb +18 -0
  388. data/lib/relaton/iho/item_data.rb +6 -0
  389. data/lib/relaton/iho/processor.rb +54 -0
  390. data/lib/relaton/iho/relation.rb +9 -0
  391. data/lib/relaton/iho/structured_identifier.rb +20 -0
  392. data/lib/relaton/iho/util.rb +8 -0
  393. data/lib/relaton/iho.rb +27 -0
  394. data/lib/relaton/index/config.rb +52 -0
  395. data/lib/relaton/index/file_io.rb +305 -0
  396. data/lib/relaton/index/file_storage.rb +66 -0
  397. data/lib/relaton/index/pool.rb +46 -0
  398. data/lib/relaton/index/shard_source.rb +201 -0
  399. data/lib/relaton/index/type.rb +210 -0
  400. data/lib/relaton/index/util.rb +18 -0
  401. data/lib/relaton/index.rb +61 -0
  402. data/lib/relaton/isbn/isbn.rb +61 -0
  403. data/lib/relaton/isbn/open_library.rb +43 -0
  404. data/lib/relaton/isbn/parser.rb +88 -0
  405. data/lib/relaton/isbn/processor.rb +48 -0
  406. data/lib/relaton/isbn/util.rb +8 -0
  407. data/lib/relaton/isbn.rb +19 -0
  408. data/lib/relaton/iso/bibliography.rb +412 -0
  409. data/lib/relaton/iso/data_fetcher.rb +338 -0
  410. data/lib/relaton/iso/data_parser.rb +458 -0
  411. data/lib/relaton/iso/hash_parser_v1.rb +177 -0
  412. data/lib/relaton/iso/hit.rb +57 -0
  413. data/lib/relaton/iso/hit_collection.rb +196 -0
  414. data/lib/relaton/iso/item_data.rb +50 -0
  415. data/lib/relaton/iso/model/bibdata.rb +10 -0
  416. data/lib/relaton/iso/model/bibitem.rb +8 -0
  417. data/lib/relaton/iso/model/contributor.rb +6 -0
  418. data/lib/relaton/iso/model/contributor_info.rb +9 -0
  419. data/lib/relaton/iso/model/docidentifier.rb +146 -0
  420. data/lib/relaton/iso/model/doctype.rb +13 -0
  421. data/lib/relaton/iso/model/ext.rb +35 -0
  422. data/lib/relaton/iso/model/item.rb +17 -0
  423. data/lib/relaton/iso/model/item_base.rb +22 -0
  424. data/lib/relaton/iso/model/organization.rb +9 -0
  425. data/lib/relaton/iso/model/project_number.rb +22 -0
  426. data/lib/relaton/iso/model/relation.rb +9 -0
  427. data/lib/relaton/iso/model/stagename.rb +14 -0
  428. data/lib/relaton/iso/model/structured_identifier.rb +31 -0
  429. data/lib/relaton/iso/processor.rb +85 -0
  430. data/lib/relaton/iso/scraper.rb +635 -0
  431. data/lib/relaton/iso/type/pubid.rb +50 -0
  432. data/lib/relaton/iso/util.rb +8 -0
  433. data/lib/relaton/iso.rb +30 -0
  434. data/lib/relaton/itu/bibliography.rb +95 -0
  435. data/lib/relaton/itu/data_crawler_r.rb +664 -0
  436. data/lib/relaton/itu/data_fetcher.rb +550 -0
  437. data/lib/relaton/itu/data_merge_r.rb +149 -0
  438. data/lib/relaton/itu/data_parser_r.rb +214 -0
  439. data/lib/relaton/itu/data_parser_t.rb +228 -0
  440. data/lib/relaton/itu/family_cache.rb +177 -0
  441. data/lib/relaton/itu/governor.rb +56 -0
  442. data/lib/relaton/itu/hit.rb +28 -0
  443. data/lib/relaton/itu/hit_collection.rb +299 -0
  444. data/lib/relaton/itu/item_data.rb +6 -0
  445. data/lib/relaton/itu/model/approval_stage.rb +13 -0
  446. data/lib/relaton/itu/model/bibdata.rb +8 -0
  447. data/lib/relaton/itu/model/bibitem.rb +8 -0
  448. data/lib/relaton/itu/model/docidentifier.rb +75 -0
  449. data/lib/relaton/itu/model/doctype.rb +13 -0
  450. data/lib/relaton/itu/model/ext.rb +47 -0
  451. data/lib/relaton/itu/model/item.rb +13 -0
  452. data/lib/relaton/itu/model/meeting.rb +13 -0
  453. data/lib/relaton/itu/model/meeting_date.rb +15 -0
  454. data/lib/relaton/itu/model/question.rb +13 -0
  455. data/lib/relaton/itu/model/recommendation_status.rb +11 -0
  456. data/lib/relaton/itu/model/structured_identifier.rb +38 -0
  457. data/lib/relaton/itu/processor.rb +72 -0
  458. data/lib/relaton/itu/pubid.rb +199 -0
  459. data/lib/relaton/itu/radio_regulations_parser.rb +70 -0
  460. data/lib/relaton/itu/recommendation_fields.rb +334 -0
  461. data/lib/relaton/itu/recommendation_parser.rb +30 -0
  462. data/lib/relaton/itu/scraper.rb +203 -0
  463. data/lib/relaton/itu/util.rb +8 -0
  464. data/lib/relaton/itu.rb +34 -0
  465. data/lib/relaton/jcgm/bibdata.rb +8 -0
  466. data/lib/relaton/jcgm/bibitem.rb +8 -0
  467. data/lib/relaton/jcgm/bibliography.rb +97 -0
  468. data/lib/relaton/jcgm/data_fetcher.rb +81 -0
  469. data/lib/relaton/jcgm/docidentifier.rb +102 -0
  470. data/lib/relaton/jcgm/doctype.rb +12 -0
  471. data/lib/relaton/jcgm/ext.rb +23 -0
  472. data/lib/relaton/jcgm/item.rb +20 -0
  473. data/lib/relaton/jcgm/item_base.rb +18 -0
  474. data/lib/relaton/jcgm/item_data.rb +6 -0
  475. data/lib/relaton/jcgm/meetings_parser.rb +175 -0
  476. data/lib/relaton/jcgm/processor.rb +71 -0
  477. data/lib/relaton/jcgm/relation.rb +9 -0
  478. data/lib/relaton/jcgm/structured_identifier.rb +40 -0
  479. data/lib/relaton/jcgm/util.rb +8 -0
  480. data/lib/relaton/jcgm.rb +24 -0
  481. data/lib/relaton/jis/bibdata.rb +8 -0
  482. data/lib/relaton/jis/bibitem.rb +8 -0
  483. data/lib/relaton/jis/bibliography.rb +73 -0
  484. data/lib/relaton/jis/data_fetcher.rb +183 -0
  485. data/lib/relaton/jis/docidentifier.rb +120 -0
  486. data/lib/relaton/jis/doctype.rb +9 -0
  487. data/lib/relaton/jis/ext.rb +13 -0
  488. data/lib/relaton/jis/hit.rb +88 -0
  489. data/lib/relaton/jis/hit_collection.rb +128 -0
  490. data/lib/relaton/jis/item.rb +22 -0
  491. data/lib/relaton/jis/item_base.rb +26 -0
  492. data/lib/relaton/jis/item_data.rb +8 -0
  493. data/lib/relaton/jis/processor.rb +55 -0
  494. data/lib/relaton/jis/relation.rb +11 -0
  495. data/lib/relaton/jis/scraper.rb +232 -0
  496. data/lib/relaton/jis/structured_identifier.rb +13 -0
  497. data/lib/relaton/jis/util.rb +8 -0
  498. data/lib/relaton/jis.rb +32 -0
  499. data/lib/relaton/logger/channels/gh_issue.rb +143 -0
  500. data/lib/relaton/logger/config.rb +34 -0
  501. data/lib/relaton/logger/formatter_json.rb +8 -0
  502. data/lib/relaton/logger/formatter_string.rb +12 -0
  503. data/lib/relaton/logger/log.rb +64 -0
  504. data/lib/relaton/logger/log_device.rb +27 -0
  505. data/lib/relaton/logger/pool.rb +24 -0
  506. data/lib/relaton/logger.rb +18 -0
  507. data/lib/relaton/nist/bibdata.rb +8 -0
  508. data/lib/relaton/nist/bibitem.rb +8 -0
  509. data/lib/relaton/nist/bibliography.rb +191 -0
  510. data/lib/relaton/nist/comment_period.rb +15 -0
  511. data/lib/relaton/nist/data_fetcher.rb +134 -0
  512. data/lib/relaton/nist/date.rb +7 -0
  513. data/lib/relaton/nist/docidentifier.rb +165 -0
  514. data/lib/relaton/nist/doctype.rb +7 -0
  515. data/lib/relaton/nist/ext.rb +16 -0
  516. data/lib/relaton/nist/hit.rb +18 -0
  517. data/lib/relaton/nist/hit_collection.rb +362 -0
  518. data/lib/relaton/nist/item.rb +19 -0
  519. data/lib/relaton/nist/item_base.rb +16 -0
  520. data/lib/relaton/nist/item_data.rb +6 -0
  521. data/lib/relaton/nist/mods_parser.rb +302 -0
  522. data/lib/relaton/nist/processor.rb +69 -0
  523. data/lib/relaton/nist/pubs_export.rb +69 -0
  524. data/lib/relaton/nist/relation.rb +10 -0
  525. data/lib/relaton/nist/scraper.rb +293 -0
  526. data/lib/relaton/nist/series.yaml +49 -0
  527. data/lib/relaton/nist/util.rb +8 -0
  528. data/lib/relaton/nist.rb +34 -0
  529. data/lib/relaton/oasis/bibdata.rb +8 -0
  530. data/lib/relaton/oasis/bibitem.rb +8 -0
  531. data/lib/relaton/oasis/bibliography.rb +201 -0
  532. data/lib/relaton/oasis/browser_agent.rb +71 -0
  533. data/lib/relaton/oasis/data_fetcher.rb +130 -0
  534. data/lib/relaton/oasis/data_parser.rb +305 -0
  535. data/lib/relaton/oasis/data_parser_utils.rb +330 -0
  536. data/lib/relaton/oasis/data_part_parser.rb +239 -0
  537. data/lib/relaton/oasis/docidentifier.rb +54 -0
  538. data/lib/relaton/oasis/doctype.rb +7 -0
  539. data/lib/relaton/oasis/ext.rb +19 -0
  540. data/lib/relaton/oasis/item.rb +14 -0
  541. data/lib/relaton/oasis/item_data.rb +6 -0
  542. data/lib/relaton/oasis/processor.rb +74 -0
  543. data/lib/relaton/oasis/util.rb +8 -0
  544. data/lib/relaton/oasis.rb +41 -0
  545. data/lib/relaton/ogc/bibdata.rb +8 -0
  546. data/lib/relaton/ogc/bibitem.rb +8 -0
  547. data/lib/relaton/ogc/bibliography.rb +64 -0
  548. data/lib/relaton/ogc/data_fetcher.rb +131 -0
  549. data/lib/relaton/ogc/docidentifier.rb +111 -0
  550. data/lib/relaton/ogc/doctype.rb +12 -0
  551. data/lib/relaton/ogc/ext.rb +15 -0
  552. data/lib/relaton/ogc/hit.rb +18 -0
  553. data/lib/relaton/ogc/hit_collection.rb +111 -0
  554. data/lib/relaton/ogc/item.rb +13 -0
  555. data/lib/relaton/ogc/item_data.rb +6 -0
  556. data/lib/relaton/ogc/processor.rb +59 -0
  557. data/lib/relaton/ogc/scraper.rb +225 -0
  558. data/lib/relaton/ogc/util.rb +8 -0
  559. data/lib/relaton/ogc.rb +30 -0
  560. data/lib/relaton/oiml/bibdata.rb +8 -0
  561. data/lib/relaton/oiml/bibitem.rb +8 -0
  562. data/lib/relaton/oiml/bibliography.rb +169 -0
  563. data/lib/relaton/oiml/docidentifier.rb +48 -0
  564. data/lib/relaton/oiml/doctype.rb +13 -0
  565. data/lib/relaton/oiml/ext.rb +42 -0
  566. data/lib/relaton/oiml/item.rb +20 -0
  567. data/lib/relaton/oiml/item_base.rb +18 -0
  568. data/lib/relaton/oiml/item_data.rb +6 -0
  569. data/lib/relaton/oiml/processor.rb +54 -0
  570. data/lib/relaton/oiml/relation.rb +9 -0
  571. data/lib/relaton/oiml/util.rb +8 -0
  572. data/lib/relaton/oiml.rb +24 -0
  573. data/lib/relaton/omg/bibdata.rb +8 -0
  574. data/lib/relaton/omg/bibitem.rb +8 -0
  575. data/lib/relaton/omg/bibliography.rb +31 -0
  576. data/lib/relaton/omg/docidentifier.rb +67 -0
  577. data/lib/relaton/omg/ext.rb +7 -0
  578. data/lib/relaton/omg/item.rb +9 -0
  579. data/lib/relaton/omg/item_data.rb +6 -0
  580. data/lib/relaton/omg/processor.rb +35 -0
  581. data/lib/relaton/omg/scraper.rb +186 -0
  582. data/lib/relaton/omg/util.rb +8 -0
  583. data/lib/relaton/omg.rb +25 -0
  584. data/lib/relaton/plateau/bibdata.rb +8 -0
  585. data/lib/relaton/plateau/bibitem.rb +8 -0
  586. data/lib/relaton/plateau/bibliography.rb +31 -0
  587. data/lib/relaton/plateau/data_fetcher.rb +176 -0
  588. data/lib/relaton/plateau/doctype.rb +7 -0
  589. data/lib/relaton/plateau/ext.rb +23 -0
  590. data/lib/relaton/plateau/handbook_parser.rb +102 -0
  591. data/lib/relaton/plateau/hit.rb +21 -0
  592. data/lib/relaton/plateau/hit_collection.rb +79 -0
  593. data/lib/relaton/plateau/item.rb +12 -0
  594. data/lib/relaton/plateau/item_data.rb +6 -0
  595. data/lib/relaton/plateau/parser.rb +101 -0
  596. data/lib/relaton/plateau/processor.rb +51 -0
  597. data/lib/relaton/plateau/technical_report_parser.rb +98 -0
  598. data/lib/relaton/plateau/util.rb +8 -0
  599. data/lib/relaton/plateau.rb +29 -0
  600. data/lib/relaton/sdo/config.rb +34 -0
  601. data/lib/relaton/sdo/fetcher.rb +52 -0
  602. data/lib/relaton/sdo/logo.rb +95 -0
  603. data/lib/relaton/sdo/name.rb +26 -0
  604. data/lib/relaton/sdo/organization.rb +71 -0
  605. data/lib/relaton/sdo/store.rb +49 -0
  606. data/lib/relaton/sdo.rb +29 -0
  607. data/lib/relaton/un/bibdata.rb +8 -0
  608. data/lib/relaton/un/bibitem.rb +8 -0
  609. data/lib/relaton/un/bibliography.rb +48 -0
  610. data/lib/relaton/un/doctype.rb +10 -0
  611. data/lib/relaton/un/ext.rb +30 -0
  612. data/lib/relaton/un/hit.rb +56 -0
  613. data/lib/relaton/un/hit_collection.rb +61 -0
  614. data/lib/relaton/un/item.rb +11 -0
  615. data/lib/relaton/un/item_data.rb +6 -0
  616. data/lib/relaton/un/parser.rb +108 -0
  617. data/lib/relaton/un/processor.rb +38 -0
  618. data/lib/relaton/un/session.rb +25 -0
  619. data/lib/relaton/un/token_generator.rb +105 -0
  620. data/lib/relaton/un/util.rb +8 -0
  621. data/lib/relaton/un/wasm/decoder.rb +434 -0
  622. data/lib/relaton/un/wasm/instance.rb +101 -0
  623. data/lib/relaton/un/wasm/interpreter.rb +613 -0
  624. data/lib/relaton/un/wasm/memory.rb +112 -0
  625. data/lib/relaton/un/wasm/module.rb +47 -0
  626. data/lib/relaton/un/wasm.rb +17 -0
  627. data/lib/relaton/un/wasm_v_bg.wasm +0 -0
  628. data/lib/relaton/un.rb +28 -0
  629. data/lib/relaton/version.rb +1 -1
  630. data/lib/relaton/w3c/bibdata.rb +8 -0
  631. data/lib/relaton/w3c/bibitem.rb +8 -0
  632. data/lib/relaton/w3c/bibliography.rb +174 -0
  633. data/lib/relaton/w3c/data_fetcher.rb +487 -0
  634. data/lib/relaton/w3c/data_parser.rb +379 -0
  635. data/lib/relaton/w3c/docidentifier.rb +48 -0
  636. data/lib/relaton/w3c/doctype.rb +7 -0
  637. data/lib/relaton/w3c/ext.rb +11 -0
  638. data/lib/relaton/w3c/governor.rb +32 -0
  639. data/lib/relaton/w3c/item.rb +15 -0
  640. data/lib/relaton/w3c/item_data.rb +6 -0
  641. data/lib/relaton/w3c/processor.rb +49 -0
  642. data/lib/relaton/w3c/pubid.rb +85 -0
  643. data/lib/relaton/w3c/safe_realize.rb +148 -0
  644. data/lib/relaton/w3c/util.rb +8 -0
  645. data/lib/relaton/w3c.rb +36 -0
  646. data/lib/relaton/xsf/bibdata.rb +8 -0
  647. data/lib/relaton/xsf/bibitem.rb +8 -0
  648. data/lib/relaton/xsf/bibliography.rb +84 -0
  649. data/lib/relaton/xsf/data_fetcher.rb +106 -0
  650. data/lib/relaton/xsf/docidentifier.rb +46 -0
  651. data/lib/relaton/xsf/hit.rb +17 -0
  652. data/lib/relaton/xsf/hit_collection.rb +52 -0
  653. data/lib/relaton/xsf/item.rb +13 -0
  654. data/lib/relaton/xsf/item_data.rb +6 -0
  655. data/lib/relaton/xsf/processor.rb +51 -0
  656. data/lib/relaton/xsf/util.rb +9 -0
  657. data/lib/relaton/xsf.rb +34 -0
  658. data/lib/relaton-core.rb +1 -0
  659. data/lib/relaton.rb +85 -0
  660. metadata +775 -161
  661. data/.github/workflows/rake.yml +0 -14
  662. data/.github/workflows/release.yml +0 -24
  663. data/.github/workflows/rubocop.yml +0 -48
  664. data/.gitignore +0 -12
  665. data/.rspec +0 -3
  666. data/CLAUDE.md +0 -78
  667. data/Gemfile +0 -29
  668. data/Rakefile +0 -6
  669. data/docs/CHANGELOG.adoc +0 -16
  670. data/docs/VERSIONING_POLICY.adoc +0 -38
  671. data/docs/navigation.adoc +0 -6
  672. data/lib/relaton/db/version.rb +0 -5
  673. data/relaton.gemspec +0 -72
  674. data/spec/relaton/config_spec.rb +0 -10
  675. data/spec/relaton/db_cache_spec.rb +0 -51
  676. data/spec/relaton/db_spec.rb +0 -567
  677. data/spec/relaton/registry_spec.rb +0 -178
  678. data/spec/relaton/util_spec.rb +0 -3
  679. data/spec/relaton_meta_spec.rb +0 -25
  680. data/spec/relaton_spec.rb +0 -800
  681. data/spec/spec_helper.rb +0 -45
  682. data/spec/support/gb_t_20223_2006.xml +0 -33
  683. data/spec/support/iso_111111119115_1.xml +0 -0
  684. data/spec/support/iso_19115_1.xml +0 -115
  685. data/spec/support/iso_19115_2.xml +0 -95
  686. data/spec/support/rfc_8341.xml +0 -46
  687. data/spec/vcr_cassetes/api_relaton_org.yml +0 -121
  688. data/spec/vcr_cassetes/api_relaton_org_unavailable.yml +0 -140
  689. data/spec/vcr_cassetes/cc_dir_10005_2019.yml +0 -202
  690. data/spec/vcr_cassetes/cipm_meeting_43.yml +0 -1505
  691. data/spec/vcr_cassetes/gb_t_20223_2006.yml +0 -729
  692. data/spec/vcr_cassetes/iso_111111119115_1.yml +0 -12951
  693. data/spec/vcr_cassetes/iso_19115_1.yml +0 -17334
  694. data/spec/vcr_cassetes/iso_19115_1_2.yml +0 -322
  695. data/spec/vcr_cassetes/iso_19115_1_std.yml +0 -17334
  696. data/spec/vcr_cassetes/iso_19115_all_parts.yml +0 -185
  697. data/spec/vcr_cassetes/iso_19133_2005.yml +0 -144
  698. data/spec/vcr_cassetes/iso_combined_applied.yml +0 -318
  699. data/spec/vcr_cassetes/iso_combined_included.yml +0 -318
  700. data/spec/vcr_cassetes/ogc_19_025r1.yml +0 -374
  701. data/spec/vcr_cassetes/omg_ami4ccm_1_0.yml +0 -317
  702. data/spec/vcr_cassetes/rfc_8341.yml +0 -1278
@@ -0,0 +1,8 @@
1
+ module Relaton
2
+ module Cie
3
+ module Util
4
+ extend Relaton::Bib::Util
5
+ PROGNAME = "relaton-cie".freeze
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,29 @@
1
+ require "nokogiri"
2
+ # require "parslet"
3
+ require "pubid"
4
+ require "relaton/index"
5
+ require "relaton/bib"
6
+ # require "relaton_bib/name_parser"
7
+ require_relative "version"
8
+ require_relative "cie/util"
9
+ require_relative "cie/item_data"
10
+ require_relative "cie/item"
11
+ require_relative "cie/bibitem"
12
+ require_relative "cie/bibdata"
13
+ require_relative "cie/bibliography"
14
+ require_relative "cie/scrapper"
15
+
16
+ module Relaton
17
+ module Cie
18
+ INDEXFILE = "index-v2".freeze
19
+
20
+ # Returns hash of XML reammar
21
+ # @return [String]
22
+ def self.grammar_hash
23
+ # gem_path = File.expand_path "..", __dir__
24
+ # grammars_path = File.join gem_path, "grammars", "*"
25
+ # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
26
+ Digest::MD5.hexdigest Relaton::VERSION # grammars
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,20 @@
1
+ module Relaton
2
+ module Core
3
+ module ArrayWrapper
4
+ #
5
+ # Wrap into Array if not Array
6
+ #
7
+ # @param [Object] content
8
+ #
9
+ # @return [Array<Object>]
10
+ #
11
+ def array(content)
12
+ case content
13
+ when Array then content
14
+ when nil then []
15
+ else [content]
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,244 @@
1
+ require "fileutils"
2
+
3
+ module Relaton
4
+ module Core
5
+ class DataFetcher
6
+ # attr_accessor :docs
7
+ #
8
+ # Initialize fetcher
9
+ #
10
+ # @param [String] output path to output directory
11
+ # @param [String] format output format (yaml, xml, bibxml)
12
+ #
13
+ def initialize(output, format)
14
+ @output = output
15
+ @format = format
16
+ @ext = format.sub "bibxml", "xml"
17
+ @files = Set.new
18
+ # path => docid that reserved it, for #unique_output_file. Distinct
19
+ # from @files, which flavors use for their own duplicate handling:
20
+ # this one has to know WHICH document owns a path, not just that it
21
+ # is taken.
22
+ @file_docids = {}
23
+ # Paths this process wrote during this run; see #write_unique.
24
+ @written = Set.new
25
+ # Set true by a fetcher that writes from forked worker processes,
26
+ # where @file_docids cannot see a peer's claim. See #write_unique.
27
+ @cross_process = false
28
+ # @docs = []
29
+ @errors = Hash.new(true)
30
+ end
31
+
32
+ # API method for external service
33
+ #
34
+ # @return the value returned by the instance `#fetch`, so callers can act
35
+ # on the outcome (e.g. relaton-iso returns whether it rebuilt).
36
+ def self.fetch(source = nil, output: "data", format: "yaml")
37
+ t1 = Time.now
38
+ puts "Started at: #{t1}"
39
+ FileUtils.mkdir_p output
40
+ result = new(output, format).fetch(source)
41
+ t2 = Time.now
42
+ puts "Stopped at: #{t2}"
43
+ puts "Done in: #{(t2 - t1).round} sec."
44
+ result
45
+ end
46
+
47
+ def fetch(source = nil)
48
+ raise NotImplementedError, "#{self.class}#fetch method must be implemented"
49
+ end
50
+
51
+ def gh_issue
52
+ return @gh_issue if defined? @gh_issue
53
+
54
+ channel = gh_issue_channel
55
+ if channel[0]
56
+ @gh_issue = Relaton::Logger::Channels::GhIssue.new(*channel)
57
+ Relaton.logger_pool[:gh_issue] = Relaton::Logger::Log.new(@gh_issue, levels: [:error])
58
+ end
59
+ @gh_issue
60
+ end
61
+
62
+ def gh_issue_channel
63
+ [ENV.fetch("GITHUB_REPOSITORY", nil), "Error fetching documents"]
64
+ end
65
+
66
+ def report_errors
67
+ gh_issue # register the channel before logging
68
+ @errors.select { |_, v| v }.each do |key, value|
69
+ # A String value IS the message: a specific, per-document failure
70
+ # such as an unparseable identifier, reported through this same
71
+ # channel without needing a per-flavor override. A boolean means
72
+ # "this field failed for every record" — what the flavors' own
73
+ # ERROR_KEYS track — and its message is derived from the key.
74
+ log_error value.is_a?(String) ? value : "Failed to fetch #{key}"
75
+ end
76
+ @gh_issue&.create_issue
77
+ end
78
+
79
+ def log_error(_msg)
80
+ raise NoMatchingPatternError, "#{self.class}#log_error method must be implemented"
81
+ end
82
+
83
+ # Most filesystems cap a single path component at 255 bytes.
84
+ MAX_BASENAME_BYTES = 255
85
+
86
+ # Create-or-fail. The failure is the point: it is how one process learns
87
+ # that another already holds a path. See #write_unique.
88
+ EXCLUSIVE = File::WRONLY | File::CREAT | File::EXCL
89
+
90
+ # @param [String] document ID
91
+ # @return [String] filename based on PubID identifier
92
+ #
93
+ # A docid can be pathologically long (pubid's `to_s` for amendment docs
94
+ # embeds the full "(Amendment to … as amended by …)" clause), which would
95
+ # make the basename exceed the OS limit and raise Errno::ENAMETOOLONG on
96
+ # write. When that happens, truncate the sanitized id and append a short
97
+ # digest of the full docid so the filename stays bounded, unique, and
98
+ # deterministic (every call site round-trips through this method).
99
+ def output_file(docid)
100
+ id = docid.downcase.gsub(/[.,\s\/:()-]+/, "-").delete_suffix("-")
101
+ ext = ".#{@ext}"
102
+ limit = MAX_BASENAME_BYTES - ext.bytesize
103
+ if id.bytesize > limit
104
+ require "digest"
105
+ suffix = "-#{Digest::SHA1.hexdigest(docid)[0, 12]}"
106
+ # `id` has no consecutive "-" (gsub collapsed runs above), so
107
+ # truncation leaves at most one trailing "-" — delete_suffix is
108
+ # enough and avoids a polynomial-ReDoS regex on the docid.
109
+ id = id.byteslice(0, limit - suffix.bytesize).scrub("").delete_suffix("-") + suffix
110
+ end
111
+ File.join @output, "#{id}#{ext}"
112
+ end
113
+
114
+ #
115
+ # Reserve a unique output path for `docid`.
116
+ #
117
+ # `output_file` sanitizes ".", ",", "/", ":", "(", ")", "-" and whitespace
118
+ # all to "-", so two DISTINCT docids can map to one path — live instance in
119
+ # relaton-data-iana: `rpki/signed-objects` and `rpki-signed-objects` both
120
+ # give `data/rpki-signed-objects.yaml`. A caller that merely warns and
121
+ # writes anyway leaves one file holding the wrong document for one of two
122
+ # index ids: a wrong answer, not a missing one.
123
+ #
124
+ # Returns `output_file(docid)` when that path is free, or when it is
125
+ # already held by this SAME docid — a genuine duplicate is the caller's
126
+ # business (skip / merge / last-wins), and this method must not turn one
127
+ # into two files. Only a real clash with a DIFFERENT docid gets a variant,
128
+ # suffixed with a digest of this docid so the name depends on the document
129
+ # and not on encounter order: adding a record never renames an existing
130
+ # file. (Which member of a clashing pair keeps the plain path does follow
131
+ # write order, which is stable for a given corpus.)
132
+ #
133
+ # @param [String] docid
134
+ # @return [String] path, reserved for this docid
135
+ #
136
+ def unique_output_file(docid)
137
+ file = output_file docid
138
+ owner = @file_docids[file]
139
+ file = digest_output_file(docid) unless owner.nil? || owner == docid
140
+ reserve file, docid
141
+ end
142
+
143
+ #
144
+ # Write `content` for `docid`, never clobbering a different document.
145
+ #
146
+ # `output_file` is not injective, so the plain path may belong to someone
147
+ # else. `unique_output_file` settles that within this process; the file is
148
+ # then created with `O_EXCL` so a peer *process* cannot be clobbered
149
+ # either. Returns the path actually written.
150
+ #
151
+ # The four outcomes, each load-bearing:
152
+ #
153
+ # * **We already wrote this path for this docid** — plain overwrite, so a
154
+ # genuine duplicate stays one file and the flavor's own duplicate
155
+ # handling (skip / merge / last-wins) still decides what happens.
156
+ # * **`O_EXCL` succeeds** — the normal case.
157
+ # * **`EEXIST`, single process** — `unique_output_file` has already proved
158
+ # that no peer of ours holds this path, so the file can only be a
159
+ # leftover from an earlier crawl. Overwrite it. Every single-process
160
+ # flavor takes this branch, and it involves no guesswork.
161
+ # * **`EEXIST`, `@cross_process`** — a peer's file and a leftover are
162
+ # indistinguishable, so take a path of our own rather than risk
163
+ # destroying a record. The caller reconciles the names afterwards, in
164
+ # the parent, where it can see every docid (see
165
+ # `Relaton::Ietf::DataFetcher#reconcile_output_files`).
166
+ #
167
+ # Deliberately NOT a wall-clock "is this file older than the crawl?" test.
168
+ # A crawl runs into a populated `data/`, so `EEXIST` is the common case on
169
+ # a re-run and such a test would decide it for every record; its
170
+ # false-"stale" direction is the silent overwrite this method exists to
171
+ # prevent, and coarse filesystem mtime granularity makes that reachable.
172
+ #
173
+ # @param [String] docid
174
+ # @param [String] content serialized document
175
+ # @return [String] path written
176
+ #
177
+ def write_unique(docid, content)
178
+ file = unique_output_file docid
179
+ return force_write(file, content) if @written.include?(file)
180
+
181
+ File.write file, content, mode: EXCLUSIVE, encoding: "UTF-8"
182
+ @written << file
183
+ file
184
+ rescue Errno::EEXIST
185
+ return force_write(file, content) unless @cross_process
186
+
187
+ force_write reserve(digest_output_file(docid), docid), content
188
+ end
189
+
190
+ #
191
+ # Serialize bibliographic item
192
+ #
193
+ # @param [RelatonCcsds::BibliographicItem] bib <description>
194
+ #
195
+ # @return [String] serialized bibliographic item
196
+ #
197
+ def serialize(bib)
198
+ send "to_#{@format}", bib
199
+ end
200
+
201
+ def to_yaml(bib)
202
+ raise NotImplementedError, "#{self.class}#to_yaml method must be implemented"
203
+ end
204
+
205
+ def to_xml(bib)
206
+ raise NotImplementedError, "#{self.class}#to_xml method must be implemented"
207
+ end
208
+
209
+ def to_bibxml(bib)
210
+ raise NotImplementedError, "#{self.class}#to_bibxml method must be implemented"
211
+ end
212
+
213
+ private
214
+
215
+ # Write, clobbering whatever is there. The caller has already established
216
+ # that the path is ours to take.
217
+ def force_write(file, content)
218
+ File.write file, content, encoding: "UTF-8"
219
+ @written << file
220
+ file
221
+ end
222
+
223
+ # Record `docid` as the owner of `file`, unless someone got there first,
224
+ # and return `file`. The reservation table must never hand one path to two
225
+ # documents; see #unique_output_file.
226
+ def reserve(file, docid)
227
+ @file_docids[file] ||= docid
228
+ file
229
+ end
230
+
231
+ # `output_file`'s path with a docid digest appended to the basename, kept
232
+ # inside the same byte cap. Deterministic: same docid, same name.
233
+ def digest_output_file(docid)
234
+ require "digest"
235
+ ext = ".#{@ext}"
236
+ suffix = "-#{Digest::SHA1.hexdigest(docid)[0, 12]}"
237
+ stem = File.basename output_file(docid), ext
238
+ limit = MAX_BASENAME_BYTES - ext.bytesize - suffix.bytesize
239
+ stem = stem.byteslice(0, limit).scrub("").delete_suffix("-") if stem.bytesize > limit
240
+ File.join @output, "#{stem}#{suffix}#{ext}"
241
+ end
242
+ end
243
+ end
244
+ end
@@ -0,0 +1,42 @@
1
+ module Relaton
2
+ module Core
3
+ module DateParser
4
+ # @param date [String, Integer, Date] date
5
+ # @param str [Boolean] return string or Date
6
+ # @return [Date, String, nil] date
7
+ def parse_date(date, str: true) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
8
+ return date if date.is_a?(Date)
9
+
10
+ case date.to_s
11
+ when /(?<date>\w+\s\d{4})/ # February 2012
12
+ format_date $~[:date], "%B %Y", str, "%Y-%m"
13
+ when /(?<date>\w+\s\d{1,2},\s\d{4})/ # February 11, 2012
14
+ format_date $~[:date], "%B %d, %Y", str, "%Y-%m-%d"
15
+ when /(?<date>\d{4}-\d{1,2}-\d{1,2})/ # 2012-02-03 or 2012-2-3
16
+ format_date $~[:date], "%Y-%m-%d", str
17
+ when /(?<date>\d{4}-\d{1,2})/ # 2012-02 or 2012-2
18
+ format_date $~[:date], "%Y-%m", str
19
+ when /(?<date>\d{4})/ # 2012
20
+ format_date $~[:date], "%Y", str
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ #
27
+ # Parse date string to Date object and format it
28
+ #
29
+ # @param [String] date date string
30
+ # @param [String] format format string
31
+ # @param [Boolean] str return string if true in other case return Date
32
+ # @param [String, nil] outformat output format
33
+ #
34
+ # @return [Date, String] date object or formatted date string
35
+ #
36
+ def format_date(date, format, str, outformat = nil)
37
+ date = Date.strptime(date, format)
38
+ str ? date.strftime(outformat || format) : date
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,320 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time" # Time.parse, for an HTTP-date Retry-After
4
+
5
+ module Relaton
6
+ module Core
7
+ #
8
+ # Process-wide, thread-safe rate-limit back-pressure for a crawl.
9
+ #
10
+ # Promoted here from `Relaton::W3c::Governor`, which is now a thin binding
11
+ # of it, when the ITU-R crawler became the second flavor to need
12
+ # cross-thread throttling. What stays with a flavor is only which errors
13
+ # mean "throttled" (THROTTLE_ERRORS / .throttle?) and which env vars tune it
14
+ # (ENV_PREFIX); the ladder, the jitter, the give-up latch and the
15
+ # Retry-After parsing are shared.
16
+ #
17
+ # The problem it solves, as first met on api.w3.org: that host sits behind
18
+ # Cloudflare, whose rate limiting is bimodal — a run either never trips it
19
+ # or trips it and stays banned for hours. Before this
20
+ # existed, each of the crawl's workers reacted to its own 429s in isolation
21
+ # with a backoff capped at ~31s — far too short to outlast the ban, and the
22
+ # retries themselves kept the limiter engaged. The Aug-2026 crawl spent 4h56m
23
+ # doing nothing but re-tripping the limiter, permanently discarding 1,412
24
+ # documents, before it aborted.
25
+ #
26
+ # The governor gives the whole worker pool one shared cooldown. When any
27
+ # thread sees a 429 it opens (or extends) that cooldown; every other thread
28
+ # observes it in #wait. Cooldowns escalate per *round* — 60s, 120s, ... to a
29
+ # 15-minute cap — and after GIVE_UP_AFTER consecutive rounds without a single
30
+ # success the crawl is declared rate-limited and gives up (~15 min) so CI
31
+ # fails fast instead of grinding for five hours.
32
+ #
33
+ # `clock:` and `sleeper:` are injectable so specs exercise the real
34
+ # arithmetic without wall-clock waits.
35
+ #
36
+ class Governor
37
+ # First cooldown, in seconds. Deliberately minute-scale: a Cloudflare ban
38
+ # window outlasts any second-scale backoff.
39
+ BASE_COOLDOWN = 60
40
+
41
+ # Ceiling for a single cooldown, including a server-supplied Retry-After
42
+ # (which must not be able to park a worker for an hour).
43
+ MAX_COOLDOWN = 900
44
+
45
+ # Consecutive throttle rounds with no intervening success before the crawl
46
+ # is considered banned. With the ladder above that is ~15 min of cooldown
47
+ # (60+120+240+480 s; the fifth round latches before its own wait).
48
+ GIVE_UP_AFTER = 5
49
+
50
+ # Fraction of the cooldown added as random jitter, so the workers do not
51
+ # all resume on the same instant and immediately re-trip the limiter.
52
+ JITTER_FRACTION = 0.1
53
+
54
+ # Which errors mean "you are being rate-limited", as opposed to "this
55
+ # resource is broken". Core ships none: the distinction is the one thing
56
+ # only the flavor knows, and getting it wrong in either direction is
57
+ # expensive — a broken resource routed here opens a pool-wide cooldown
58
+ # for one bad record, and a throttle routed the other way gets recorded
59
+ # as permanent data loss (which is exactly what the W3C crawl did before
60
+ # this class existed). Subclasses override the constant, or #throttle?
61
+ # when the signal is a status code rather than an exception class.
62
+ THROTTLE_ERRORS = [].freeze
63
+
64
+ # Prefix for the three duration knobs, so two flavors crawling in one
65
+ # process cannot share a cooldown ladder by accident. A subclass sets its
66
+ # own ("RELATON_W3C", "RELATON_ITU"); the suffixes are fixed.
67
+ ENV_PREFIX = "RELATON"
68
+
69
+ # Whether this error means the server is rate-limiting us.
70
+ #
71
+ # Class membership is the common case. A flavor whose server signals
72
+ # throttling with a *status code* on a shared exception class (ITU: 503 on
73
+ # /rec) overrides this instead of the constant.
74
+ #
75
+ # @param error [StandardError]
76
+ # @return [Boolean]
77
+ def self.throttle?(error)
78
+ self::THROTTLE_ERRORS.any? { |klass| error.is_a? klass }
79
+ end
80
+
81
+ attr_reader :base, :max, :give_up_after, :env_prefix
82
+
83
+ def initialize(base: nil, max: nil, give_up_after: nil,
84
+ clock: nil, sleeper: nil, jitter: nil, env_prefix: nil)
85
+ @env_prefix = env_prefix || self.class::ENV_PREFIX
86
+ # Remembered so #reset! can re-read the env knobs for the next crawl
87
+ # without discarding values a caller passed explicitly.
88
+ @overrides = { base: base, max: max, give_up_after: give_up_after }
89
+ load_limits
90
+ @clock = clock || -> { Time.now }
91
+ @sleeper = sleeper || ->(secs) { sleep secs }
92
+ @jitter = jitter || ->(secs) { Kernel.rand * secs * self.class::JITTER_FRACTION }
93
+ @mutex = Mutex.new
94
+ @cooldown_until = nil
95
+ @penalty = nil
96
+ @rounds = 0
97
+ @events = 0
98
+ @abandoned = false
99
+ @abandoned_rounds = nil
100
+ end
101
+
102
+ #
103
+ # Block until the shared cooldown expires. Called before every request.
104
+ #
105
+ # The mutex is never held across the sleep — the deadline is read under
106
+ # the lock and the sleeping happens outside it, so a cooling-down worker
107
+ # cannot block another thread from recording its own throttle.
108
+ #
109
+ # @return [Numeric] seconds actually slept (0.0 when there was nothing to
110
+ # wait for, including once the crawl has given up)
111
+ #
112
+ def wait
113
+ slept = 0.0
114
+ waited_for = nil
115
+ loop do
116
+ # Once we have given up there is nothing left to wait for, and stalling
117
+ # here would delay the shutdown by up to a full cooldown.
118
+ deadline, remaining = @mutex.synchronize do
119
+ gave_up? ? [nil, 0.0] : [@cooldown_until, cooldown_remaining]
120
+ end
121
+ break unless remaining.positive?
122
+ # Re-check after sleeping so a cooldown *extended* mid-round (a later
123
+ # Retry-After asking for longer) also holds back the workers already
124
+ # sleeping — they would otherwise wake on the stale deadline and
125
+ # re-trip the limiter. Looping only on a later deadline keeps a stalled
126
+ # clock from spinning here.
127
+ break if waited_for && deadline && deadline <= waited_for
128
+
129
+ waited_for = deadline
130
+ delay = remaining + @jitter.call(remaining)
131
+ @sleeper.call(delay)
132
+ slept += delay
133
+ end
134
+ slept
135
+ end
136
+
137
+ #
138
+ # Record a 429 and open/extend the shared cooldown.
139
+ #
140
+ # @param retry_after [Integer, nil] seconds from the response's Retry-After
141
+ #
142
+ # @return [Boolean] true when the crawl should give up
143
+ #
144
+ def throttled!(retry_after: nil)
145
+ @mutex.synchronize do
146
+ now = @clock.call
147
+ @events += 1
148
+ if @cooldown_until.nil? || now >= @cooldown_until
149
+ open_round(now, retry_after)
150
+ elsif retry_after
151
+ # Mid-round, the server may still ask for longer than we planned.
152
+ extend_round(now + capped(retry_after))
153
+ end
154
+ # Giving up latches. Otherwise a straggler worker succeeding after the
155
+ # producer already stopped paginating would clear @rounds and let the
156
+ # crawl save the truncated index it just decided to abandon.
157
+ #
158
+ # @rounds is snapshotted at the same moment, because #succeeded!
159
+ # resets it: read afterwards it names whatever has re-accumulated
160
+ # since, not the threshold that was crossed. The abort message is
161
+ # the only place an operator learns why the crawl stopped.
162
+ unless @abandoned
163
+ @abandoned = @rounds >= @give_up_after
164
+ @abandoned_rounds = @rounds if @abandoned
165
+ end
166
+ @abandoned
167
+ end
168
+ end
169
+
170
+ #
171
+ # Record a successful request: the limiter has released us.
172
+ #
173
+ def succeeded!
174
+ @mutex.synchronize do
175
+ @rounds = 0
176
+ @cooldown_until = nil
177
+ # Decay rather than reset, so a limiter that keeps flapping still
178
+ # climbs the ladder instead of restarting at `base` every time.
179
+ @penalty = @penalty && @penalty > @base ? [@penalty / 2.0, @base].max : nil
180
+ end
181
+ end
182
+
183
+ # @return [Boolean] give_up_after consecutive rounds passed with no success
184
+ def exhausted?
185
+ @mutex.synchronize { gave_up? }
186
+ end
187
+
188
+ # Clear all state for a fresh crawl. Resets in place rather than handing
189
+ # back a new object so callers (and specs) holding this instance keep
190
+ # observing the live governor.
191
+ def reset!
192
+ @mutex.synchronize do
193
+ load_limits
194
+ @cooldown_until = nil
195
+ @penalty = nil
196
+ @rounds = 0
197
+ @events = 0
198
+ @abandoned = false
199
+ @abandoned_rounds = nil
200
+ end
201
+ end
202
+
203
+ # @return [Integer] consecutive throttle rounds since the last success
204
+ def throttle_rounds
205
+ @mutex.synchronize { @rounds }
206
+ end
207
+
208
+ # The consecutive-round count as it stood when the give-up latched, for
209
+ # the abort message. Distinct from #throttle_rounds, which stays LIVE
210
+ # because a crawler logs it per retry as the current round (see
211
+ # Relaton::Itu::DataCrawlerR) and #succeeded! resets it.
212
+ #
213
+ # @return [Integer, nil] nil while the crawl is still running
214
+ def give_up_rounds
215
+ @mutex.synchronize { @abandoned_rounds }
216
+ end
217
+
218
+ # @return [Integer] every 429 observed, for reporting
219
+ def throttle_count
220
+ @mutex.synchronize { @events }
221
+ end
222
+
223
+ #
224
+ # Seconds requested by a rate-limited response's Retry-After header.
225
+ #
226
+ # lutaml-hal >= 0.2.5 carries a `{ status:, headers: }` hash on every error
227
+ # mapped from a response (nil for locally raised ones), so anything without
228
+ # one yields nil.
229
+ #
230
+ # @param error [StandardError]
231
+ #
232
+ # @return [Integer, nil]
233
+ #
234
+ def self.retry_after(error)
235
+ return nil unless error.respond_to?(:response)
236
+
237
+ response = error.response
238
+ headers = response.is_a?(Hash) ? (response[:headers] || response["headers"]) : nil
239
+ return nil unless headers.respond_to?(:[])
240
+
241
+ seconds_from headers["retry-after"] || headers["Retry-After"]
242
+ end
243
+
244
+ #
245
+ # Retry-After is either delta-seconds or an HTTP-date (RFC 9110). The date
246
+ # form is parsed properly rather than by scanning digits, which would pick
247
+ # up the day-of-month and back off for a wildly wrong duration; a date
248
+ # already in the past yields 0, i.e. "no extra floor".
249
+ #
250
+ def self.seconds_from(value)
251
+ str = value.to_s.strip
252
+ return nil if str.empty?
253
+ return str.to_i if str.match?(/\A\d+\z/)
254
+
255
+ begin
256
+ [(Time.parse(str) - Time.now).ceil, 0].max
257
+ rescue ArgumentError
258
+ nil
259
+ end
260
+ end
261
+
262
+ # A blank or non-positive override is treated as unset.
263
+ def self.env_int(name, default)
264
+ value = ENV[name].to_s.strip
265
+ return default if value.empty?
266
+
267
+ parsed = value.to_i
268
+ parsed.positive? ? parsed : default
269
+ end
270
+
271
+ private
272
+
273
+ # "<PREFIX>_THROTTLE_<SUFFIX>", the one thing ENV_PREFIX exists for.
274
+ def env_name(suffix)
275
+ "#{@env_prefix}_THROTTLE_#{suffix}"
276
+ end
277
+
278
+ # Read the duration knobs, letting explicit constructor args win over the
279
+ # environment. Re-read on #reset! so the env vars behave like a flavor's
280
+ # sibling knobs, which are class methods evaluated per crawl rather than
281
+ # frozen when the flavor was autoloaded.
282
+ def load_limits
283
+ @base = @overrides[:base] || self.class.env_int(env_name("BASE"), self.class::BASE_COOLDOWN)
284
+ @max = @overrides[:max] || self.class.env_int(env_name("MAX"), self.class::MAX_COOLDOWN)
285
+ @give_up_after = @overrides[:give_up_after] ||
286
+ self.class.env_int(env_name("GIVEUP"), self.class::GIVE_UP_AFTER)
287
+ end
288
+
289
+ # Start a new escalation step. Retry-After raises the floor but never
290
+ # lowers our own ladder — the server's hint is a minimum, not a discount.
291
+ def open_round(now, retry_after)
292
+ @rounds += 1
293
+ ladder = @penalty ? [@penalty * 2, @max].min : @base
294
+ hint = retry_after ? capped(retry_after) : 0
295
+ @penalty = [ladder, hint].max
296
+ @cooldown_until = now + @penalty
297
+ end
298
+
299
+ def extend_round(deadline)
300
+ @cooldown_until = deadline if deadline > @cooldown_until
301
+ end
302
+
303
+ def cooldown_remaining
304
+ return 0.0 unless @cooldown_until
305
+
306
+ [@cooldown_until - @clock.call, 0.0].max
307
+ end
308
+
309
+ def capped(seconds)
310
+ [seconds, @max].min
311
+ end
312
+
313
+ # Callers must already hold the mutex. Latched by #throttled! — never
314
+ # recomputed from @rounds, which #succeeded! resets.
315
+ def gave_up?
316
+ @abandoned
317
+ end
318
+ end
319
+ end
320
+ end
@@ -0,0 +1,19 @@
1
+ module Relaton
2
+ module Core
3
+ module HashKeysSymbolizer
4
+ def symbolize_hash_keys(obj)
5
+ case obj
6
+ when Array
7
+ obj.map { |e| symbolize_hash_keys(e) }
8
+ when Hash
9
+ obj.each_with_object({}) do |(k, v), h|
10
+ key = k.is_a?(String) ? k.to_sym : k
11
+ h[key] = symbolize_hash_keys(v)
12
+ end
13
+ else
14
+ obj
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end