relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. checksums.yaml +4 -4
  2. data/README.adoc +57 -1
  3. data/bin/console +0 -1
  4. data/lib/relaton/3gpp/bibliography.rb +82 -7
  5. data/lib/relaton/3gpp/data_fetcher.rb +51 -3
  6. data/lib/relaton/3gpp/docidentifier.rb +114 -0
  7. data/lib/relaton/3gpp/item.rb +6 -0
  8. data/lib/relaton/3gpp/parser.rb +1 -1
  9. data/lib/relaton/3gpp/processor.rb +4 -1
  10. data/lib/relaton/3gpp.rb +5 -1
  11. data/lib/relaton/adobe/bibdata.rb +8 -0
  12. data/lib/relaton/adobe/bibitem.rb +8 -0
  13. data/lib/relaton/adobe/bibliography.rb +92 -0
  14. data/lib/relaton/adobe/docidentifier.rb +49 -0
  15. data/lib/relaton/adobe/doctype.rb +14 -0
  16. data/lib/relaton/adobe/ext.rb +32 -0
  17. data/lib/relaton/adobe/item.rb +15 -0
  18. data/lib/relaton/adobe/item_base.rb +18 -0
  19. data/lib/relaton/adobe/item_data.rb +6 -0
  20. data/lib/relaton/adobe/processor.rb +45 -0
  21. data/lib/relaton/adobe/util.rb +8 -0
  22. data/lib/relaton/adobe.rb +37 -0
  23. data/lib/relaton/bib/model/address.rb +2 -2
  24. data/lib/relaton/bib/model/docidentifier.rb +24 -9
  25. data/lib/relaton/bib/model/localized_string.rb +1 -1
  26. data/lib/relaton/bib/model/structured_identifier.rb +10 -9
  27. data/lib/relaton/bib/sanitizer.rb +202 -6
  28. data/lib/relaton/bib.rb +0 -2
  29. data/lib/relaton/bipm/bibliography.rb +159 -10
  30. data/lib/relaton/bipm/data_fetcher.rb +26 -2
  31. data/lib/relaton/bipm/data_outcomes_parser.rb +3 -4
  32. data/lib/relaton/bipm/id_parser.rb +5 -4
  33. data/lib/relaton/bipm/model/structured_identifier.rb +21 -0
  34. data/lib/relaton/bipm/processor.rb +2 -2
  35. data/lib/relaton/bipm/rawdata_bipm_metrologia/fetcher.rb +2 -4
  36. data/lib/relaton/bipm/si_brochure_parser.rb +5 -3
  37. data/lib/relaton/bipm.rb +7 -1
  38. data/lib/relaton/bsi/bibliography.rb +115 -43
  39. data/lib/relaton/bsi/hit.rb +14 -0
  40. data/lib/relaton/bsi/hit_collection.rb +15 -16
  41. data/lib/relaton/bsi/model/docidentifier.rb +99 -1
  42. data/lib/relaton/bsi/processor.rb +1 -0
  43. data/lib/relaton/calconnect/bibliography.rb +12 -14
  44. data/lib/relaton/calconnect/data_fetcher.rb +77 -9
  45. data/lib/relaton/calconnect/docidentifier.rb +80 -0
  46. data/lib/relaton/calconnect/hit_collection.rb +65 -57
  47. data/lib/relaton/calconnect/model/item.rb +7 -0
  48. data/lib/relaton/calconnect/processor.rb +7 -1
  49. data/lib/relaton/calconnect.rb +11 -1
  50. data/lib/relaton/ccsds/data/fetcher.rb +17 -12
  51. data/lib/relaton/ccsds/data/parser.rb +1 -1
  52. data/lib/relaton/ccsds/hit_collection.rb +6 -1
  53. data/lib/relaton/ccsds/model/docidentifier.rb +121 -0
  54. data/lib/relaton/ccsds/model/item.rb +2 -0
  55. data/lib/relaton/cen/bibliography.rb +75 -47
  56. data/lib/relaton/cen/hit.rb +16 -1
  57. data/lib/relaton/cen/hit_collection.rb +59 -11
  58. data/lib/relaton/cen/model/docidentifier.rb +92 -1
  59. data/lib/relaton/cen/processor.rb +13 -8
  60. data/lib/relaton/cen/scraper.rb +13 -5
  61. data/lib/relaton/cen.rb +1 -0
  62. data/lib/relaton/cie/data_fetcher.rb +215 -30
  63. data/lib/relaton/cie/processor.rb +3 -1
  64. data/lib/relaton/cie/scrapper.rb +15 -2
  65. data/lib/relaton/cie.rb +2 -1
  66. data/lib/relaton/core/data_fetcher.rb +150 -3
  67. data/lib/relaton/core/governor.rb +320 -0
  68. data/lib/relaton/core/pacer.rb +134 -0
  69. data/lib/relaton/core/processor.rb +19 -0
  70. data/lib/relaton/core/request_error.rb +14 -0
  71. data/lib/relaton/core.rb +3 -0
  72. data/lib/relaton/db/registry.rb +41 -1
  73. data/lib/relaton/doi/crossref.rb +19 -2
  74. data/lib/relaton/doi/parser.rb +109 -15
  75. data/lib/relaton/easc/bibdata.rb +8 -0
  76. data/lib/relaton/easc/bibitem.rb +8 -0
  77. data/lib/relaton/easc/bibliography.rb +95 -0
  78. data/lib/relaton/easc/docidentifier.rb +100 -0
  79. data/lib/relaton/easc/doctype.rb +14 -0
  80. data/lib/relaton/easc/ext.rb +44 -0
  81. data/lib/relaton/easc/item.rb +13 -0
  82. data/lib/relaton/easc/item_base.rb +18 -0
  83. data/lib/relaton/easc/item_data.rb +6 -0
  84. data/lib/relaton/easc/processor.rb +46 -0
  85. data/lib/relaton/easc/util.rb +8 -0
  86. data/lib/relaton/easc.rb +35 -0
  87. data/lib/relaton/ecma/bibliography.rb +93 -25
  88. data/lib/relaton/ecma/data_fetcher.rb +71 -12
  89. data/lib/relaton/ecma/docidentifier.rb +124 -0
  90. data/lib/relaton/ecma/item.rb +2 -0
  91. data/lib/relaton/ecma/memento_parser.rb +1 -1
  92. data/lib/relaton/ecma/page_fetcher.rb +15 -3
  93. data/lib/relaton/ecma/parser_common.rb +2 -2
  94. data/lib/relaton/ecma/processor.rb +4 -1
  95. data/lib/relaton/ecma/standard_parser.rb +2 -2
  96. data/lib/relaton/ecma.rb +10 -1
  97. data/lib/relaton/etsi/bibliography.rb +67 -2
  98. data/lib/relaton/etsi/data_fetcher.rb +43 -4
  99. data/lib/relaton/etsi/processor.rb +3 -1
  100. data/lib/relaton/etsi.rb +2 -1
  101. data/lib/relaton/gb/bibliography.rb +55 -29
  102. data/lib/relaton/gb/docidentifier.rb +58 -9
  103. data/lib/relaton/gb/processor.rb +3 -0
  104. data/lib/relaton/gb/scraper.rb +27 -10
  105. data/lib/relaton/gost/bibdata.rb +8 -0
  106. data/lib/relaton/gost/bibitem.rb +8 -0
  107. data/lib/relaton/gost/bibliography.rb +107 -0
  108. data/lib/relaton/gost/docidentifier.rb +80 -0
  109. data/lib/relaton/gost/doctype.rb +16 -0
  110. data/lib/relaton/gost/ext.rb +46 -0
  111. data/lib/relaton/gost/item.rb +15 -0
  112. data/lib/relaton/gost/item_base.rb +18 -0
  113. data/lib/relaton/gost/item_data.rb +6 -0
  114. data/lib/relaton/gost/processor.rb +49 -0
  115. data/lib/relaton/gost/util.rb +8 -0
  116. data/lib/relaton/gost.rb +36 -0
  117. data/lib/relaton/iala/bibdata.rb +8 -0
  118. data/lib/relaton/iala/bibitem.rb +8 -0
  119. data/lib/relaton/iala/bibliography.rb +146 -0
  120. data/lib/relaton/iala/docidentifier.rb +89 -0
  121. data/lib/relaton/iala/doctype.rb +18 -0
  122. data/lib/relaton/iala/ext.rb +32 -0
  123. data/lib/relaton/iala/item.rb +21 -0
  124. data/lib/relaton/iala/item_base.rb +18 -0
  125. data/lib/relaton/iala/item_data.rb +6 -0
  126. data/lib/relaton/iala/processor.rb +43 -0
  127. data/lib/relaton/iala/relation.rb +7 -0
  128. data/lib/relaton/iala/util.rb +8 -0
  129. data/lib/relaton/iala.rb +35 -0
  130. data/lib/relaton/iana/bibliography.rb +67 -14
  131. data/lib/relaton/iana/data_fetcher.rb +35 -5
  132. data/lib/relaton/iana/processor.rb +3 -1
  133. data/lib/relaton/iana.rb +12 -1
  134. data/lib/relaton/iec/data_fetcher.rb +7 -1
  135. data/lib/relaton/iec/hit_collection.rb +1 -1
  136. data/lib/relaton/iec/model/docidentifier.rb +9 -5
  137. data/lib/relaton/iec/model/ext.rb +2 -2
  138. data/lib/relaton/iec/processor.rb +1 -0
  139. data/lib/relaton/ieee/bibliography.rb +25 -3
  140. data/lib/relaton/ieee/data_fetcher.rb +158 -17
  141. data/lib/relaton/ieee/idams_parser.rb +18 -11
  142. data/lib/relaton/ieee/processor.rb +4 -1
  143. data/lib/relaton/ieee/rawbib_id_parser.rb +291 -86
  144. data/lib/relaton/ieee.rb +2 -1
  145. data/lib/relaton/ietf/data_fetcher.rb +295 -12
  146. data/lib/relaton/ietf/processor.rb +7 -3
  147. data/lib/relaton/ietf/rfc/entry.rb +39 -3
  148. data/lib/relaton/ietf/scraper.rb +69 -36
  149. data/lib/relaton/ietf.rb +4 -1
  150. data/lib/relaton/iho/bibliography.rb +1 -1
  151. data/lib/relaton/iho/docidentifier.rb +1 -1
  152. data/lib/relaton/index/file_io.rb +11 -11
  153. data/lib/relaton/index/file_storage.rb +6 -1
  154. data/lib/relaton/index/pool.rb +6 -1
  155. data/lib/relaton/index/shard_source.rb +201 -0
  156. data/lib/relaton/index/type.rb +63 -12
  157. data/lib/relaton/index.rb +2 -1
  158. data/lib/relaton/iso/bibliography.rb +20 -15
  159. data/lib/relaton/iso/data_fetcher.rb +3 -3
  160. data/lib/relaton/iso/data_parser.rb +17 -3
  161. data/lib/relaton/iso/hit_collection.rb +27 -15
  162. data/lib/relaton/iso/item_data.rb +22 -0
  163. data/lib/relaton/iso/model/docidentifier.rb +24 -12
  164. data/lib/relaton/iso/processor.rb +1 -0
  165. data/lib/relaton/iso/scraper.rb +19 -3
  166. data/lib/relaton/itu/bibliography.rb +9 -4
  167. data/lib/relaton/itu/data_crawler_r.rb +664 -0
  168. data/lib/relaton/itu/data_fetcher.rb +496 -50
  169. data/lib/relaton/itu/data_merge_r.rb +149 -0
  170. data/lib/relaton/itu/data_parser_r.rb +163 -89
  171. data/lib/relaton/itu/data_parser_t.rb +228 -0
  172. data/lib/relaton/itu/family_cache.rb +177 -0
  173. data/lib/relaton/itu/governor.rb +56 -0
  174. data/lib/relaton/itu/hit.rb +9 -3
  175. data/lib/relaton/itu/hit_collection.rb +258 -86
  176. data/lib/relaton/itu/model/docidentifier.rb +67 -1
  177. data/lib/relaton/itu/model/structured_identifier.rb +19 -0
  178. data/lib/relaton/itu/processor.rb +10 -4
  179. data/lib/relaton/itu/pubid.rb +27 -5
  180. data/lib/relaton/itu/recommendation_fields.rb +334 -0
  181. data/lib/relaton/itu/recommendation_parser.rb +18 -149
  182. data/lib/relaton/itu/scraper.rb +13 -3
  183. data/lib/relaton/itu.rb +2 -1
  184. data/lib/relaton/jcgm/bibdata.rb +8 -0
  185. data/lib/relaton/jcgm/bibitem.rb +8 -0
  186. data/lib/relaton/jcgm/bibliography.rb +97 -0
  187. data/lib/relaton/jcgm/data_fetcher.rb +81 -0
  188. data/lib/relaton/jcgm/docidentifier.rb +102 -0
  189. data/lib/relaton/jcgm/doctype.rb +12 -0
  190. data/lib/relaton/jcgm/ext.rb +23 -0
  191. data/lib/relaton/jcgm/item.rb +20 -0
  192. data/lib/relaton/jcgm/item_base.rb +18 -0
  193. data/lib/relaton/jcgm/item_data.rb +6 -0
  194. data/lib/relaton/jcgm/meetings_parser.rb +175 -0
  195. data/lib/relaton/jcgm/processor.rb +71 -0
  196. data/lib/relaton/jcgm/relation.rb +9 -0
  197. data/lib/relaton/jcgm/structured_identifier.rb +40 -0
  198. data/lib/relaton/jcgm/util.rb +8 -0
  199. data/lib/relaton/jcgm.rb +24 -0
  200. data/lib/relaton/jis/bibliography.rb +8 -10
  201. data/lib/relaton/jis/data_fetcher.rb +21 -19
  202. data/lib/relaton/jis/docidentifier.rb +104 -5
  203. data/lib/relaton/jis/hit.rb +18 -23
  204. data/lib/relaton/jis/hit_collection.rb +19 -18
  205. data/lib/relaton/jis/processor.rb +1 -1
  206. data/lib/relaton/jis.rb +2 -3
  207. data/lib/relaton/logger/channels/gh_issue.rb +78 -13
  208. data/lib/relaton/nist/data_fetcher.rb +63 -13
  209. data/lib/relaton/nist/docidentifier.rb +165 -0
  210. data/lib/relaton/nist/item.rb +2 -0
  211. data/lib/relaton/nist/item_base.rb +16 -0
  212. data/lib/relaton/nist/mods_parser.rb +38 -12
  213. data/lib/relaton/nist/processor.rb +2 -1
  214. data/lib/relaton/nist/relation.rb +3 -0
  215. data/lib/relaton/nist/scraper.rb +6 -3
  216. data/lib/relaton/oasis/bibliography.rb +147 -6
  217. data/lib/relaton/oasis/data_fetcher.rb +41 -5
  218. data/lib/relaton/oasis/data_parser_utils.rb +37 -3
  219. data/lib/relaton/oasis/docidentifier.rb +54 -0
  220. data/lib/relaton/oasis/item.rb +3 -0
  221. data/lib/relaton/oasis/processor.rb +7 -1
  222. data/lib/relaton/oasis.rb +14 -1
  223. data/lib/relaton/ogc/data_fetcher.rb +23 -2
  224. data/lib/relaton/ogc/docidentifier.rb +105 -0
  225. data/lib/relaton/ogc/hit_collection.rb +78 -3
  226. data/lib/relaton/ogc/processor.rb +2 -1
  227. data/lib/relaton/ogc.rb +5 -1
  228. data/lib/relaton/oiml/bibliography.rb +90 -15
  229. data/lib/relaton/oiml/docidentifier.rb +18 -3
  230. data/lib/relaton/omg/docidentifier.rb +67 -0
  231. data/lib/relaton/omg/item.rb +1 -0
  232. data/lib/relaton/omg/processor.rb +1 -0
  233. data/lib/relaton/omg/scraper.rb +61 -16
  234. data/lib/relaton/omg.rb +1 -0
  235. data/lib/relaton/plateau/bibliography.rb +10 -3
  236. data/lib/relaton/plateau/data_fetcher.rb +25 -2
  237. data/lib/relaton/plateau/handbook_parser.rb +8 -1
  238. data/lib/relaton/plateau/hit.rb +10 -2
  239. data/lib/relaton/plateau/hit_collection.rb +31 -11
  240. data/lib/relaton/plateau/processor.rb +3 -1
  241. data/lib/relaton/plateau/technical_report_parser.rb +8 -1
  242. data/lib/relaton/plateau.rb +2 -1
  243. data/lib/relaton/sdo/config.rb +34 -0
  244. data/lib/relaton/sdo/fetcher.rb +52 -0
  245. data/lib/relaton/sdo/logo.rb +95 -0
  246. data/lib/relaton/sdo/name.rb +26 -0
  247. data/lib/relaton/sdo/organization.rb +71 -0
  248. data/lib/relaton/sdo/store.rb +49 -0
  249. data/lib/relaton/sdo.rb +29 -0
  250. data/lib/relaton/version.rb +1 -1
  251. data/lib/relaton/w3c/bibliography.rb +132 -12
  252. data/lib/relaton/w3c/data_fetcher.rb +194 -16
  253. data/lib/relaton/w3c/data_parser.rb +3 -3
  254. data/lib/relaton/w3c/docidentifier.rb +48 -0
  255. data/lib/relaton/w3c/governor.rb +32 -0
  256. data/lib/relaton/w3c/item.rb +3 -0
  257. data/lib/relaton/w3c/pubid.rb +12 -0
  258. data/lib/relaton/w3c/safe_realize.rb +110 -21
  259. data/lib/relaton/w3c.rb +12 -1
  260. data/lib/relaton/xsf/bibliography.rb +61 -1
  261. data/lib/relaton/xsf/data_fetcher.rb +55 -5
  262. data/lib/relaton/xsf/docidentifier.rb +46 -0
  263. data/lib/relaton/xsf/hit_collection.rb +31 -3
  264. data/lib/relaton/xsf/item.rb +6 -0
  265. data/lib/relaton/xsf/processor.rb +1 -0
  266. data/lib/relaton/xsf.rb +5 -1
  267. data/lib/relaton.rb +42 -0
  268. metadata +135 -24
  269. data/lib/relaton/ieee/pub_id.rb +0 -161
  270. data/lib/relaton/index/id_number.rb +0 -30
@@ -11,13 +11,21 @@ module Relaton
11
11
  class DataFetcher < Core::DataFetcher
12
12
  PAGE_SIZE = 50
13
13
 
14
+ # `version=1` keeps the superseded EDITIONS of a document in the result
15
+ # set; `version=0` returns only the current edition of each branch. The
16
+ # status flags below are a separate axis and already permissive, so this
17
+ # is what makes e.g. all three editions of `ETSI EN 319 142-1` retrievable
18
+ # instead of one. It also makes the crawl about 2.4 times larger — see
19
+ # lib/relaton/etsi/CLAUDE.md before changing it back.
14
20
  SOURCEURL = "https://www.etsi.org/custom/standardssearch/data.php?format=json&includeScope=1&" \
15
- "page=%<page>s&search=&title=1&etsiNumber=1&content=1&version=0&onApproval=1&published=1&" \
21
+ "page=%<page>s&search=&title=1&etsiNumber=1&content=1&version=1&onApproval=1&published=1&" \
16
22
  "withdrawn=1&historical=1&isCurrent=1&superseded=1&startDate=1988-01-15&endDate=%<date>s&" \
17
23
  "harmonized=0&keyword=&TB=&stdType=&frequency=&mandate=&collection=&sort=1&x=%<timestamp>s".freeze
18
24
 
19
25
  def index
20
- @index ||= Relaton::Index.find_or_create :etsi, file: INDEX_FILE
26
+ @index ||= Relaton::Index.find_or_create(
27
+ :etsi, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Etsi::Identifier
28
+ )
21
29
  end
22
30
 
23
31
  def log_error(msg)
@@ -107,9 +115,40 @@ module Relaton
107
115
 
108
116
  def save(bib)
109
117
  id = bib.docidentifier.first.content
110
- file = output_file id
118
+ pid = pubid(id) or return # skip ids pubid can't parse/serialize
119
+
120
+ # Distinct docids can sanitize to one filename (`output_file` collapses
121
+ # `/`, `-`, `.`, `:` and `()` alike, and ETSI ids use all of them), which
122
+ # used to overwrite silently. Take a path of our own. There is no @files
123
+ # here, so a repeat of the SAME id still resolves to one path and
124
+ # overwrites itself, as before.
125
+ file = unique_output_file id
126
+ if file != output_file(id)
127
+ Util.warn "File #{output_file id} already exists. Docid: #{id}. Writing #{file} instead."
128
+ end
111
129
  File.write file, serialize(bib), encoding: "UTF-8"
112
- index.add_or_update id, file
130
+ index.add_or_update pid, file
131
+ end
132
+
133
+ #
134
+ # Parse an ETSI docid into a Pubid::Etsi identifier, or nil when it can't
135
+ # be parsed or serialized. Storing the pubid object (not its hash) lets
136
+ # Relaton::Index sort the index by id number and serialize each id to its
137
+ # `_type: pubid:etsi:*` hash on save. A single unindexable id must never
138
+ # abort the crawl or corrupt the index, so it is skipped defensively
139
+ # (mirrors Relaton::Nist#pubid / Relaton::Jcgm#add_to_index). ETSI's whole
140
+ # published corpus parses on the pinned pubid; the guard protects against a
141
+ # future malformed record.
142
+ #
143
+ # @param [String] id docidentifier content, e.g. "ETSI GS ZSM 012 V1.1.1 (2022-12)"
144
+ # @return [::Pubid::Etsi::Identifier, nil]
145
+ #
146
+ def pubid(id)
147
+ pid = ::Pubid::Etsi.parse id
148
+ pid.to_hash # skip ids whose to_hash raises so index.save can't fail
149
+ pid
150
+ rescue StandardError
151
+ nil
113
152
  end
114
153
 
115
154
  def to_yaml(bib)
@@ -62,7 +62,9 @@ module Relaton
62
62
  #
63
63
  def remove_index_file
64
64
  require_relative "../etsi"
65
- Relaton::Index.find_or_create(:etsi, url: true, file: INDEX_FILE).remove_file
65
+ Relaton::Index.find_or_create(
66
+ :etsi, url: true, file: "#{INDEXFILE}.yaml"
67
+ ).remove_file
66
68
  end
67
69
  end
68
70
  end
data/lib/relaton/etsi.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require "net/http"
4
4
  require "mechanize"
5
+ require "pubid"
5
6
  require "relaton/index"
6
7
  require_relative "version"
7
8
  # require_relative "relaton_etsi/pubid"
@@ -12,7 +13,7 @@ require_relative "etsi/bibliography"
12
13
 
13
14
  module Relaton
14
15
  module Etsi
15
- INDEX_FILE = "index-v1.yaml"
16
+ INDEXFILE = "index-v2".freeze
16
17
 
17
18
  # Returns hash of gem versions used to generate data model.
18
19
  # @return [String]
@@ -40,19 +40,16 @@ module Relaton
40
40
  # @param code [String] the GB standard Code to look up (e..g "GB/T 20223")
41
41
  # @param year [String] the year the standard was published (optional)
42
42
  # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
43
- # @return [String] Relaton XML serialisation of reference
44
- def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
45
- if year.nil?
46
- /^(?<code1>[^-]+)-(?<year1>[^-]+)$/ =~ code
47
- unless code1.nil?
48
- code = code1
49
- year = year1
50
- end
51
- end
52
-
53
- code += ".1" if opts[:all_parts]
54
- code, year = code.split("-", 2) if code.include?("-")
55
- ret = get1(code, year, opts)
43
+ # @return [Relaton::Gb::ItemData, nil]
44
+ # @raise [Pubid::Errors::ParseError] when the code is not a GB
45
+ # identifier
46
+ def get(code, year = nil, opts = {})
47
+ require "pubid"
48
+ pubid = ::Pubid::Gb::Identifier.parse(code)
49
+ year = (year || pubid.year)&.to_s
50
+ pubid = pubid.exclude(:year)
51
+ pubid.part = "1" if opts[:all_parts]
52
+ ret = get1(pubid, year, opts)
56
53
  return nil if ret.nil?
57
54
 
58
55
  ret = ret.to_most_recent_reference unless year
@@ -62,49 +59,78 @@ module Relaton
62
59
 
63
60
  private
64
61
 
65
- def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
66
- # id = year ? "#{code}:#{year}" : code
67
- # Util.info "WARNING: No match found on the GB website for `#{id}`. " \
68
- # "The code must be exactly like it is on the website."
62
+ # @param pubid [Pubid::Gb::Identifier] the reference without a year
63
+ def fetch_ref_err(pubid, year, missed_years) # rubocop:disable Metrics/MethodLength
69
64
  unless missed_years.empty?
70
65
  Util.info "(There was no match for `#{year}`, though there " \
71
66
  "were matches found for `#{missed_years.join('`, `')}`.)"
72
67
  end
73
- if /\d-\d/.match? code
68
+ if pubid.part
74
69
  Util.info "The provided document part may not exist, or " \
75
70
  "the document may no longer be published in parts."
76
71
  else
77
72
  Util.info "If you wanted to cite all document parts for the " \
78
- "reference, use `#{code} (all parts)`.\nIf the document " \
73
+ "reference, use `#{pubid} (all parts)`.\nIf the document " \
79
74
  "is not a standard, use its document type abbreviation " \
80
75
  "(TS, TR, PAS, Guide)."
81
76
  end
82
77
  nil
83
78
  end
84
79
 
85
- def get1(code, year, _opts)
80
+ # @param pubid [Pubid::Gb::Identifier] the reference without a year
81
+ # @param year [String, nil]
82
+ def get1(pubid, year, _opts)
86
83
  # search must include year whenever available
87
- searchcode = code + (year.nil? ? "" : "-#{year}")
88
- result = search_filter(searchcode) || return
84
+ query = with_year(pubid, year)
85
+ searchcode = query.to_s
86
+ result = search_filter(query) || return
89
87
  ret = results_filter(result, year)
90
88
  if ret[:ret]
91
89
  Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: searchcode
92
90
  ret[:ret]
93
91
  else
94
92
  Util.info "Not found.", key: searchcode
95
- fetch_ref_err(code, year, ret[:years])
93
+ fetch_ref_err(pubid, year, ret[:years])
96
94
  end
97
95
  end
98
96
 
99
- def search_filter(code)
100
- # search filter needs to incorporate year
101
- docidrx = %r{^[^\s]+\s[\d.-]+}
102
- result = search(code)
103
- result.select! do |hit|
104
- hit.docref && hit.docref.match(docidrx).to_s.include?(code)
97
+ # @param pubid [Pubid::Gb::Identifier]
98
+ # @param year [String, nil]
99
+ # @return [Pubid::Gb::Identifier] a copy of pubid that carries the year
100
+ def with_year(pubid, year)
101
+ return pubid unless year
102
+
103
+ dated = pubid.exclude
104
+ dated.date = ::Pubid::Components::Date.new(year: year)
105
+ dated
106
+ end
107
+
108
+ # Keeps the hits that name the queried document. A query with no year
109
+ # matches every year. A query with no part matches only the part-less
110
+ # document: in GB, `GB/T 20223` and `GB/T 20223.1` are two documents.
111
+ #
112
+ # @param query [Pubid::Gb::Identifier]
113
+ # @return [Relaton::Gb::HitCollection, nil]
114
+ def search_filter(query)
115
+ ignore = query.year ? [] : %i[year]
116
+ result = search(query.to_s)
117
+ result&.select! do |hit|
118
+ hit_pubid = parse_docref(hit.docref)
119
+ hit_pubid && query.matches?(hit_pubid, ignore: ignore)
105
120
  end
106
121
  end
107
122
 
123
+ # A portal docref is data, so a value pubid cannot read is nil and
124
+ # drops the hit.
125
+ #
126
+ # @param docref [String, nil]
127
+ # @return [Pubid::Gb::Identifier, nil]
128
+ def parse_docref(docref)
129
+ ::Pubid::Gb::Identifier.parse(docref) if docref
130
+ rescue StandardError
131
+ nil
132
+ end
133
+
108
134
  # Sort through the results from Isobib, fetching them three at a time,
109
135
  # and return the first result that matches the code,
110
136
  # matches the year (if provided), and which # has a title (amendments do
@@ -1,22 +1,71 @@
1
1
  module Relaton
2
2
  module Gb
3
+ # A Chinese Standard identifier backed by `Pubid::Gb`.
4
+ #
5
+ # `content` stays a plain string, so serialization is unchanged; the parsed
6
+ # identifier lives beside it in `@pubid` and drives the three mutators.
7
+ # Follows the IALA, CEN and OMG shape (`lib/relaton/iala/docidentifier.rb`,
8
+ # `lib/relaton/cen/model/docidentifier.rb`).
3
9
  class Docidentifier < Bib::Docidentifier
4
- def content
5
- @all_parts ? "#{super} (all parts)" : super
6
- end
10
+ # @return [Pubid::Gb::Identifier, nil] nil when the content is not a GB
11
+ # identifier, or the grammar cannot read it
12
+ attr_reader :pubid
7
13
 
8
- def to_all_parts!
9
- remove_part!
10
- remove_date!
11
- @all_parts = true
14
+ # Capture the inherited (LocalizedMarkedUpString) content setter before
15
+ # overriding #content=, so #replace_pubid writes the re-rendered string
16
+ # back WITHOUT re-parsing it and discarding the mutation.
17
+ alias_method :store_content, :content=
18
+
19
+ def content=(value)
20
+ super
21
+ return unless value
22
+
23
+ @pubid = begin
24
+ # `pubid` is required lazily because deserialization reaches this
25
+ # class without the flavor entry file having been loaded. LoadError
26
+ # degrades to a plain string; StandardError covers a value that is
27
+ # not a GB identifier. Both are DATA and must not raise. A malformed
28
+ # *query* raises — see Bibliography.get.
29
+ require "pubid"
30
+ ::Pubid::Gb::Identifier.parse(value)
31
+ rescue LoadError, StandardError
32
+ nil
33
+ end
12
34
  end
13
35
 
14
36
  def remove_part!
15
- content.sub!(/\.\d+/, "")
37
+ return unless @pubid
38
+
39
+ replace_pubid @pubid.exclude(:part)
16
40
  end
17
41
 
18
42
  def remove_date!
19
- content.sub!(/-\d{4}$/, "")
43
+ return unless @pubid
44
+
45
+ replace_pubid @pubid.exclude(:year)
46
+ end
47
+
48
+ # Strips the part and the date, then wraps the identifier in pubid's
49
+ # `AllParts`. `Pubid::Gb::Identifiers::AllParts` renders the
50
+ # ` (all parts)` marker itself and parses the form back, so — unlike CEN
51
+ # and BSI, whose renderers have no marker — `content` is the wrapper's
52
+ # own string (`GB/T 5606 (all parts)`, which
53
+ # `spec/gb/fixtures/gbt_5606_2004_all_parts.xml` asserts).
54
+ #
55
+ # The guard keeps a second call from doubling the marker.
56
+ def to_all_parts!
57
+ return if !@pubid || @pubid.all_parts?
58
+
59
+ replace_pubid @pubid.exclude(:part, :year).to_all_parts
60
+ end
61
+
62
+ private
63
+
64
+ # `Pubid#exclude` returns a COPY, so the new identifier replaces the old
65
+ # one.
66
+ def replace_pubid(new_pubid)
67
+ @pubid = new_pubid
68
+ store_content @pubid.to_s
20
69
  end
21
70
  end
22
71
  end
@@ -8,6 +8,9 @@ module Relaton
8
8
  def initialize
9
9
  @short = :relaton_gb
10
10
  @prefix = "CN"
11
+ # Global prefixes come from Pubid::Gb.prefixes. "CN" is not one of
12
+ # them, so it leaves the register; routing still reads @prefix.
13
+ @pubid_flavor = :Gb
11
14
  @defaultprefix = %r{^(GB|GB/T|GB/Z) }
12
15
  @idtype = "Chinese Standard"
13
16
  end
@@ -45,8 +45,7 @@ module Relaton
45
45
  # @param docref [Strings]
46
46
  # @return [Array<Relaton::Bib::Contributor>]
47
47
  def get_contributors(doc, docref)
48
- name = docref.match(/^[^\s]+/).to_s
49
- name.sub!(%r{/[TZ]$}, "") unless name =~ /^GB/
48
+ name = docref_pubid(docref)&.publisher&.to_s
50
49
  gbtype = get_gbtype(doc, docref)
51
50
  org_names = %w[en zh].map { |l| create_org_name(l, name, gbtype) }.compact
52
51
  return [] unless org_names.any?
@@ -120,11 +119,14 @@ module Relaton
120
119
  end
121
120
  end
122
121
 
122
+ # The publisher of a social group standard names the group
123
+ # (`T/GZAEPI`), so only its first segment keys `yaml/prefixes.yaml`.
124
+ #
123
125
  # @param ref [String]
124
- # @return [String]
126
+ # @return [Hash{String=>String}, nil]
125
127
  def get_prefix(ref)
126
- pref = ref.match(/^[^\s]+/).to_s.split("/").first
127
- prefix pref
128
+ pref = docref_pubid(ref)&.publisher&.to_s
129
+ prefix pref.split("/").first if pref
128
130
  end
129
131
 
130
132
  # @param pref [String]
@@ -137,7 +139,7 @@ module Relaton
137
139
  # @param ref [String]
138
140
  # @return [String]
139
141
  def get_mandate(ref)
140
- case ref.match(%r{(?<=\/)[^\s]+}).to_s
142
+ case docref_pubid(ref)&.mandate
141
143
  when "T" then "recommended"
142
144
  when "Z" then "guidelines"
143
145
  else "mandatory"
@@ -180,14 +182,29 @@ module Relaton
180
182
  # @return [Relaton::Gb::GbType]
181
183
  def get_gbtype(doc, ref)
182
184
  # ref = get_ref(doc)
183
- GbType.new(scope: get_scope(doc), prefix: get_prefix(ref)["prefix"], mandate: get_mandate(ref), topic: "other")
185
+ GbType.new(scope: get_scope(doc), prefix: get_prefix(ref)&.fetch("prefix", nil), mandate: get_mandate(ref), topic: "other")
184
186
  end
185
187
 
188
+ # A portal docref is data, so a value pubid cannot read is nil rather
189
+ # than an error.
190
+ #
186
191
  # @param docref [String]
187
- # @return [Array<String>] array of matched groups [docnumber, partnumber]
192
+ # @return [Pubid::Gb::Identifier, nil]
193
+ def docref_pubid(docref)
194
+ require "pubid"
195
+ ::Pubid::Gb::Identifier.parse(docref)
196
+ rescue StandardError
197
+ nil
198
+ end
199
+
200
+ # @param docref [String]
201
+ # @return [Array<String, nil>] the id without part and year (e.g.
202
+ # `GB/T 5606`), the part, and the year
188
203
  def parse_docref(docref)
189
- m = docref.match(/^([^–—.-]*\d+)\.?((?<=\.)\d+|)(?:-(\d{4}))?/)
190
- [m[1], m[2], m[3]]
204
+ pubid = docref_pubid(docref)
205
+ return [docref, nil, nil] unless pubid
206
+
207
+ [pubid.exclude(:part, :year).to_s, pubid.part, pubid.year]
191
208
  end
192
209
 
193
210
  # @param docref [String]
@@ -0,0 +1,8 @@
1
+ module Relaton
2
+ module Gost
3
+ class Bibdata < Item
4
+ model ItemData
5
+ include Bib::BibdataShared
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ module Relaton
2
+ module Gost
3
+ class Bibitem < Item
4
+ model ItemData
5
+ include Bib::BibitemShared
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,107 @@
1
+ require "net/http"
2
+
3
+ module Relaton
4
+ module Gost
5
+ # Retrieval front-end for the GOST flavor. Parses the citation with
6
+ # Pubid::Gost, looks it up in the pubid-structured `relaton-data-gost`
7
+ # index-v2, and fetches the matching per-document YAML over HTTP.
8
+ #
9
+ # The index rows are keyed by Pubid::Gost identifiers (`_type: pubid:gost:
10
+ # {interstate,national}-standard`, number, year), so `Relaton::Index`
11
+ # narrows candidates by number via binary search before the block applies
12
+ # the precise pubid match. Both Latin "GOST"/"GOST R" and Cyrillic
13
+ # "ГОСТ"/"ГОСТ Р" surface forms parse (Pubid normalises Cyrillic to Latin).
14
+ module Bibliography
15
+ ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-gost/main/".freeze
16
+
17
+ class << self
18
+ # Search for a GOST publication by its identifier.
19
+ #
20
+ # @param text [String, Pubid::Gost::Identifier] the GOST reference to
21
+ # look up (e.g. "GOST R 34.12-2015", "GOST 14946-82", "ГОСТ 1.0")
22
+ # @param year [String, nil] the edition year (optional; may also be
23
+ # embedded in the reference)
24
+ # @param _opts [Hash] options (unused)
25
+ # @return [Relaton::Gost::Item, nil]
26
+ def search(text, year = nil, _opts = {})
27
+ pubid = text.is_a?(String) ? ::Pubid::Gost.parse(text) : text
28
+ Util.info "Fetching from Relaton repository ...", key: pubid.to_s
29
+ # Pass the pubid so Relaton::Index narrows candidates by number via
30
+ # binary search; pick the latest edition for an undated citation.
31
+ row = rows(pubid, year).max_by { |r| r[:id].year.to_i }
32
+ unless row
33
+ Util.info "Not found.", key: pubid.to_s
34
+ return
35
+ end
36
+
37
+ uri = URI("#{ENDPOINT}#{row[:file]}")
38
+ resp = Net::HTTP.get_response uri
39
+ unless resp.code == "200"
40
+ raise Relaton::RequestError, "Could not access #{uri}: HTTP #{resp.code}"
41
+ end
42
+
43
+ item = Relaton::Gost::Item.from_yaml resp.body
44
+ Util.info "Found: `#{item.docidentifier.first&.content}`", key: pubid.to_s
45
+ item.tap { |i| i.fetched = Date.today.to_s }
46
+ rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
47
+ Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
48
+ Net::ProtocolError, Net::ReadTimeout, OpenSSL::SSL::SSLError,
49
+ Errno::ETIMEDOUT => e
50
+ raise Relaton::RequestError, "Could not access #{uri}: #{e.message}"
51
+ end
52
+
53
+ # Fetch a GOST publication, suppressing the edition year for an undated
54
+ # citation so the designation renders undated (`GOST R 34.12`), the way
55
+ # the ISO/OIML fetchers do. A dated citation (`GOST R 34.12-2015`, or an
56
+ # explicit `year`) still pins that edition.
57
+ #
58
+ # @param opts [Hash] options
59
+ # @option opts [Boolean] :keep_year retain the edition year even for an
60
+ # undated citation (or, when false, strip it even for a dated one)
61
+ # @see #search
62
+ def get(ref, year = nil, opts = {})
63
+ item = search(ref, year, opts)
64
+ return item unless item
65
+
66
+ pubid = ref.is_a?(String) ? ::Pubid::Gost.parse(ref) : ref
67
+ dated = pubid.year || year
68
+ return item if (dated && opts[:keep_year].nil?) || opts[:keep_year]
69
+
70
+ item.to_most_recent_reference
71
+ end
72
+
73
+ private
74
+
75
+ def index
76
+ Relaton::Index.find_or_create(
77
+ :gost,
78
+ url: "#{ENDPOINT}#{INDEXFILE}.zip",
79
+ file: "#{INDEXFILE}.yaml",
80
+ pubid_class: ::Pubid::Gost::Identifier,
81
+ )
82
+ end
83
+
84
+ # The index rows for a citation. A dated citation matches exactly
85
+ # (`exact: true`, pubid `==`). An undated one takes pubid's subset
86
+ # match `query === row`, so it matches every edition of the document:
87
+ # the omitted year matches any value. The class must be identical, so
88
+ # interstate and national standards stay distinct, and `1.1` does not
89
+ # match `1.10`. pubid declares `copublisher` strict for GOST, so an
90
+ # omitted copublisher means "none". The `year` argument pins an edition
91
+ # the reference string omitted.
92
+ #
93
+ # @param query [Pubid::Gost::Identifier]
94
+ # @param year [String, nil]
95
+ # @return [Array<Hash>]
96
+ def rows(query, year)
97
+ return index.search(query, exact: true) if query.year
98
+
99
+ rows = index.search(query)
100
+ return rows unless year
101
+
102
+ rows.select { |r| r[:id].year.to_s == year.to_s }
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,80 @@
1
+ module Relaton
2
+ module Gost
3
+ # Structured GOST document identifier. Wraps a Pubid::Gost identifier so the
4
+ # canonical citation form (e.g. "GOST R 34.12-2015", "GOST 14946-82") stays
5
+ # in sync with `content`, and undated-reference handling (return the latest
6
+ # edition) can strip the year via the pubid. Mirrors Relaton::Oiml::Docidentifier.
7
+ class Docidentifier < Bib::Docidentifier
8
+ attr_reader :pubid
9
+
10
+ # Capture the inherited content setter before overriding #content=, so
11
+ # #refresh_content! can write the re-rendered string back WITHOUT
12
+ # re-parsing. `remove_date!` used to re-sync through `self.content=`,
13
+ # which rebuilds @pubid from the string; harmless on its own, but a
14
+ # re-parse of an all-parts wrapper's own rendering (which carries
15
+ # pubid's "(all parts)" marker) would not round-trip at all.
16
+ alias_method :store_content, :content=
17
+
18
+ def initialize(attrs = {}, options = {})
19
+ pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
20
+ attrs[:content] ||= pubid.to_s if pubid
21
+ super
22
+ @pubid = pubid if pubid
23
+ end
24
+
25
+ def content=(value)
26
+ super
27
+ @pubid = ::Pubid::Gost.parse(value) if value
28
+ rescue StandardError
29
+ @pubid = nil
30
+ end
31
+
32
+ # `remove_date!` was implemented but `remove_part!` and `to_all_parts!`
33
+ # were not, so `Bib::ItemData#to_all_parts` still raised
34
+ # `NotImplementedError` on every GOST item — and it descends from
35
+ # `ScriptError`, so a caller's `rescue => e` did not catch it.
36
+ #
37
+ # GOST pubid carries the edition as `year` (not `date`), so `remove_date!`
38
+ # clears that: `GOST R 34.12-2015` -> `GOST R 34.12`. `part`/`subpart`
39
+ # come from the pubid base class and are unused by GOST today, so
40
+ # `remove_part!` is a no-op for the rendered string — implemented so it
41
+ # never raises. All three no-op safely when `@pubid` is nil.
42
+
43
+ def remove_part!
44
+ clear_attr!(:part)
45
+ end
46
+
47
+ def remove_date!
48
+ clear_attr!(:year)
49
+ end
50
+
51
+ def to_all_parts!
52
+ return if !@pubid || @pubid.all_parts?
53
+
54
+ # `remove_part!`/`remove_date!` mutate `@pubid` in place via direct
55
+ # attribute setters, so a bare reference (or a shallow `.dup`) would
56
+ # still see those mutations. `#exclude` with no args rebuilds the
57
+ # whole identifier as new objects — a real, independent snapshot —
58
+ # so the all-parts wrapper below keeps the original part and date.
59
+ original = @pubid.exclude
60
+ remove_part!
61
+ remove_date!
62
+ @pubid = original.to_all_parts
63
+ end
64
+
65
+ private
66
+
67
+ def clear_attr!(attr)
68
+ return unless @pubid && @pubid.respond_to?("#{attr}=")
69
+
70
+ @pubid.public_send("#{attr}=", nil)
71
+ @pubid.subpart = nil if attr == :part && @pubid.respond_to?(:subpart=)
72
+ refresh_content!
73
+ end
74
+
75
+ def refresh_content!
76
+ store_content(@pubid.to_s) if @pubid
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,16 @@
1
+ module Relaton
2
+ module Gost
3
+ # GOST document types. The value is carried in the inherited
4
+ # `content` attribute. The list mirrors the categories GOST
5
+ # publishes (interstate GOST, Russian national GOST R,
6
+ # preliminary standards, methodological documents).
7
+ class Doctype < Bib::Doctype
8
+ TYPES = %w[
9
+ interstate
10
+ national
11
+ preliminary
12
+ methodological
13
+ ].freeze
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,46 @@
1
+ module Relaton
2
+ module Gost
3
+ # GOST bibliographic item extension. Adds GOST-specific structured
4
+ # metadata on top of the shared Relaton::Bib::Ext so the fields
5
+ # round-trip through both XML and YAML without per-repo merge hacks.
6
+ # Doctype is referenced lazily via autoload from lib/relaton/gost.rb.
7
+ class Ext < Bib::Ext
8
+ attribute :doctype, Doctype
9
+ attribute :urn, :string
10
+ attribute :webpage, :string
11
+ # ICS classification code (МКС / Межгосударственный классификатор
12
+ # стандартов) — e.g. "13.080.20".
13
+ attribute :ics_code, :string
14
+ # Original-language developer of the standard (РАЗРАБОТЧИК).
15
+ attribute :developer, :string
16
+ # Keyword list (Ключевые слова).
17
+ attribute :keywords, :string, collection: true, initialize_empty: true
18
+ # Page count (Страниц).
19
+ attribute :pages, :string
20
+ # Original-language designation as printed on the cover
21
+ # (e.g. "ГОСТ Р 71039— 2023") — preserves em-dash and spacing
22
+ # quirks that the parsed Pubid normalises away.
23
+ attribute :designation_original, :string
24
+
25
+ xml do
26
+ map_element "urn", to: :urn
27
+ map_element "webpage", to: :webpage
28
+ map_element "ics_code", to: :ics_code
29
+ map_element "developer", to: :developer
30
+ map_element "keywords", to: :keywords
31
+ map_element "pages", to: :pages
32
+ map_element "designation_original", to: :designation_original
33
+ end
34
+
35
+ key_value do
36
+ map_element "urn", to: :urn
37
+ map_element "webpage", to: :webpage
38
+ map_element "ics_code", to: :ics_code
39
+ map_element "developer", to: :developer
40
+ map_element "keywords", to: :keywords
41
+ map_element "pages", to: :pages
42
+ map_element "designation_original", to: :designation_original
43
+ end
44
+ end
45
+ end
46
+ end