relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. checksums.yaml +4 -4
  2. data/README.adoc +57 -1
  3. data/bin/console +0 -1
  4. data/lib/relaton/3gpp/bibliography.rb +82 -7
  5. data/lib/relaton/3gpp/data_fetcher.rb +51 -3
  6. data/lib/relaton/3gpp/docidentifier.rb +114 -0
  7. data/lib/relaton/3gpp/item.rb +6 -0
  8. data/lib/relaton/3gpp/parser.rb +1 -1
  9. data/lib/relaton/3gpp/processor.rb +4 -1
  10. data/lib/relaton/3gpp.rb +5 -1
  11. data/lib/relaton/adobe/bibdata.rb +8 -0
  12. data/lib/relaton/adobe/bibitem.rb +8 -0
  13. data/lib/relaton/adobe/bibliography.rb +92 -0
  14. data/lib/relaton/adobe/docidentifier.rb +49 -0
  15. data/lib/relaton/adobe/doctype.rb +14 -0
  16. data/lib/relaton/adobe/ext.rb +32 -0
  17. data/lib/relaton/adobe/item.rb +15 -0
  18. data/lib/relaton/adobe/item_base.rb +18 -0
  19. data/lib/relaton/adobe/item_data.rb +6 -0
  20. data/lib/relaton/adobe/processor.rb +45 -0
  21. data/lib/relaton/adobe/util.rb +8 -0
  22. data/lib/relaton/adobe.rb +37 -0
  23. data/lib/relaton/bib/model/address.rb +2 -2
  24. data/lib/relaton/bib/model/docidentifier.rb +24 -9
  25. data/lib/relaton/bib/model/localized_string.rb +1 -1
  26. data/lib/relaton/bib/model/structured_identifier.rb +10 -9
  27. data/lib/relaton/bib/sanitizer.rb +202 -6
  28. data/lib/relaton/bib.rb +0 -2
  29. data/lib/relaton/bipm/bibliography.rb +159 -10
  30. data/lib/relaton/bipm/data_fetcher.rb +26 -2
  31. data/lib/relaton/bipm/data_outcomes_parser.rb +3 -4
  32. data/lib/relaton/bipm/id_parser.rb +5 -4
  33. data/lib/relaton/bipm/model/structured_identifier.rb +21 -0
  34. data/lib/relaton/bipm/processor.rb +2 -2
  35. data/lib/relaton/bipm/rawdata_bipm_metrologia/fetcher.rb +2 -4
  36. data/lib/relaton/bipm/si_brochure_parser.rb +5 -3
  37. data/lib/relaton/bipm.rb +7 -1
  38. data/lib/relaton/bsi/bibliography.rb +115 -43
  39. data/lib/relaton/bsi/hit.rb +14 -0
  40. data/lib/relaton/bsi/hit_collection.rb +15 -16
  41. data/lib/relaton/bsi/model/docidentifier.rb +99 -1
  42. data/lib/relaton/bsi/processor.rb +1 -0
  43. data/lib/relaton/calconnect/bibliography.rb +12 -14
  44. data/lib/relaton/calconnect/data_fetcher.rb +77 -9
  45. data/lib/relaton/calconnect/docidentifier.rb +80 -0
  46. data/lib/relaton/calconnect/hit_collection.rb +65 -57
  47. data/lib/relaton/calconnect/model/item.rb +7 -0
  48. data/lib/relaton/calconnect/processor.rb +7 -1
  49. data/lib/relaton/calconnect.rb +11 -1
  50. data/lib/relaton/ccsds/data/fetcher.rb +17 -12
  51. data/lib/relaton/ccsds/data/parser.rb +1 -1
  52. data/lib/relaton/ccsds/hit_collection.rb +6 -1
  53. data/lib/relaton/ccsds/model/docidentifier.rb +121 -0
  54. data/lib/relaton/ccsds/model/item.rb +2 -0
  55. data/lib/relaton/cen/bibliography.rb +75 -47
  56. data/lib/relaton/cen/hit.rb +16 -1
  57. data/lib/relaton/cen/hit_collection.rb +59 -11
  58. data/lib/relaton/cen/model/docidentifier.rb +92 -1
  59. data/lib/relaton/cen/processor.rb +13 -8
  60. data/lib/relaton/cen/scraper.rb +13 -5
  61. data/lib/relaton/cen.rb +1 -0
  62. data/lib/relaton/cie/data_fetcher.rb +215 -30
  63. data/lib/relaton/cie/processor.rb +3 -1
  64. data/lib/relaton/cie/scrapper.rb +15 -2
  65. data/lib/relaton/cie.rb +2 -1
  66. data/lib/relaton/core/data_fetcher.rb +150 -3
  67. data/lib/relaton/core/governor.rb +320 -0
  68. data/lib/relaton/core/pacer.rb +134 -0
  69. data/lib/relaton/core/processor.rb +19 -0
  70. data/lib/relaton/core/request_error.rb +14 -0
  71. data/lib/relaton/core.rb +3 -0
  72. data/lib/relaton/db/registry.rb +41 -1
  73. data/lib/relaton/doi/crossref.rb +19 -2
  74. data/lib/relaton/doi/parser.rb +109 -15
  75. data/lib/relaton/easc/bibdata.rb +8 -0
  76. data/lib/relaton/easc/bibitem.rb +8 -0
  77. data/lib/relaton/easc/bibliography.rb +95 -0
  78. data/lib/relaton/easc/docidentifier.rb +100 -0
  79. data/lib/relaton/easc/doctype.rb +14 -0
  80. data/lib/relaton/easc/ext.rb +44 -0
  81. data/lib/relaton/easc/item.rb +13 -0
  82. data/lib/relaton/easc/item_base.rb +18 -0
  83. data/lib/relaton/easc/item_data.rb +6 -0
  84. data/lib/relaton/easc/processor.rb +46 -0
  85. data/lib/relaton/easc/util.rb +8 -0
  86. data/lib/relaton/easc.rb +35 -0
  87. data/lib/relaton/ecma/bibliography.rb +93 -25
  88. data/lib/relaton/ecma/data_fetcher.rb +71 -12
  89. data/lib/relaton/ecma/docidentifier.rb +124 -0
  90. data/lib/relaton/ecma/item.rb +2 -0
  91. data/lib/relaton/ecma/memento_parser.rb +1 -1
  92. data/lib/relaton/ecma/page_fetcher.rb +15 -3
  93. data/lib/relaton/ecma/parser_common.rb +2 -2
  94. data/lib/relaton/ecma/processor.rb +4 -1
  95. data/lib/relaton/ecma/standard_parser.rb +2 -2
  96. data/lib/relaton/ecma.rb +10 -1
  97. data/lib/relaton/etsi/bibliography.rb +67 -2
  98. data/lib/relaton/etsi/data_fetcher.rb +43 -4
  99. data/lib/relaton/etsi/processor.rb +3 -1
  100. data/lib/relaton/etsi.rb +2 -1
  101. data/lib/relaton/gb/bibliography.rb +55 -29
  102. data/lib/relaton/gb/docidentifier.rb +58 -9
  103. data/lib/relaton/gb/processor.rb +3 -0
  104. data/lib/relaton/gb/scraper.rb +27 -10
  105. data/lib/relaton/gost/bibdata.rb +8 -0
  106. data/lib/relaton/gost/bibitem.rb +8 -0
  107. data/lib/relaton/gost/bibliography.rb +107 -0
  108. data/lib/relaton/gost/docidentifier.rb +80 -0
  109. data/lib/relaton/gost/doctype.rb +16 -0
  110. data/lib/relaton/gost/ext.rb +46 -0
  111. data/lib/relaton/gost/item.rb +15 -0
  112. data/lib/relaton/gost/item_base.rb +18 -0
  113. data/lib/relaton/gost/item_data.rb +6 -0
  114. data/lib/relaton/gost/processor.rb +49 -0
  115. data/lib/relaton/gost/util.rb +8 -0
  116. data/lib/relaton/gost.rb +36 -0
  117. data/lib/relaton/iala/bibdata.rb +8 -0
  118. data/lib/relaton/iala/bibitem.rb +8 -0
  119. data/lib/relaton/iala/bibliography.rb +146 -0
  120. data/lib/relaton/iala/docidentifier.rb +89 -0
  121. data/lib/relaton/iala/doctype.rb +18 -0
  122. data/lib/relaton/iala/ext.rb +32 -0
  123. data/lib/relaton/iala/item.rb +21 -0
  124. data/lib/relaton/iala/item_base.rb +18 -0
  125. data/lib/relaton/iala/item_data.rb +6 -0
  126. data/lib/relaton/iala/processor.rb +43 -0
  127. data/lib/relaton/iala/relation.rb +7 -0
  128. data/lib/relaton/iala/util.rb +8 -0
  129. data/lib/relaton/iala.rb +35 -0
  130. data/lib/relaton/iana/bibliography.rb +67 -14
  131. data/lib/relaton/iana/data_fetcher.rb +35 -5
  132. data/lib/relaton/iana/processor.rb +3 -1
  133. data/lib/relaton/iana.rb +12 -1
  134. data/lib/relaton/iec/data_fetcher.rb +7 -1
  135. data/lib/relaton/iec/hit_collection.rb +1 -1
  136. data/lib/relaton/iec/model/docidentifier.rb +9 -5
  137. data/lib/relaton/iec/model/ext.rb +2 -2
  138. data/lib/relaton/iec/processor.rb +1 -0
  139. data/lib/relaton/ieee/bibliography.rb +25 -3
  140. data/lib/relaton/ieee/data_fetcher.rb +158 -17
  141. data/lib/relaton/ieee/idams_parser.rb +18 -11
  142. data/lib/relaton/ieee/processor.rb +4 -1
  143. data/lib/relaton/ieee/rawbib_id_parser.rb +291 -86
  144. data/lib/relaton/ieee.rb +2 -1
  145. data/lib/relaton/ietf/data_fetcher.rb +295 -12
  146. data/lib/relaton/ietf/processor.rb +7 -3
  147. data/lib/relaton/ietf/rfc/entry.rb +39 -3
  148. data/lib/relaton/ietf/scraper.rb +69 -36
  149. data/lib/relaton/ietf.rb +4 -1
  150. data/lib/relaton/iho/bibliography.rb +1 -1
  151. data/lib/relaton/iho/docidentifier.rb +1 -1
  152. data/lib/relaton/index/file_io.rb +11 -11
  153. data/lib/relaton/index/file_storage.rb +6 -1
  154. data/lib/relaton/index/pool.rb +6 -1
  155. data/lib/relaton/index/shard_source.rb +201 -0
  156. data/lib/relaton/index/type.rb +63 -12
  157. data/lib/relaton/index.rb +2 -1
  158. data/lib/relaton/iso/bibliography.rb +20 -15
  159. data/lib/relaton/iso/data_fetcher.rb +3 -3
  160. data/lib/relaton/iso/data_parser.rb +17 -3
  161. data/lib/relaton/iso/hit_collection.rb +27 -15
  162. data/lib/relaton/iso/item_data.rb +22 -0
  163. data/lib/relaton/iso/model/docidentifier.rb +24 -12
  164. data/lib/relaton/iso/processor.rb +1 -0
  165. data/lib/relaton/iso/scraper.rb +19 -3
  166. data/lib/relaton/itu/bibliography.rb +9 -4
  167. data/lib/relaton/itu/data_crawler_r.rb +664 -0
  168. data/lib/relaton/itu/data_fetcher.rb +496 -50
  169. data/lib/relaton/itu/data_merge_r.rb +149 -0
  170. data/lib/relaton/itu/data_parser_r.rb +163 -89
  171. data/lib/relaton/itu/data_parser_t.rb +228 -0
  172. data/lib/relaton/itu/family_cache.rb +177 -0
  173. data/lib/relaton/itu/governor.rb +56 -0
  174. data/lib/relaton/itu/hit.rb +9 -3
  175. data/lib/relaton/itu/hit_collection.rb +258 -86
  176. data/lib/relaton/itu/model/docidentifier.rb +67 -1
  177. data/lib/relaton/itu/model/structured_identifier.rb +19 -0
  178. data/lib/relaton/itu/processor.rb +10 -4
  179. data/lib/relaton/itu/pubid.rb +27 -5
  180. data/lib/relaton/itu/recommendation_fields.rb +334 -0
  181. data/lib/relaton/itu/recommendation_parser.rb +18 -149
  182. data/lib/relaton/itu/scraper.rb +13 -3
  183. data/lib/relaton/itu.rb +2 -1
  184. data/lib/relaton/jcgm/bibdata.rb +8 -0
  185. data/lib/relaton/jcgm/bibitem.rb +8 -0
  186. data/lib/relaton/jcgm/bibliography.rb +97 -0
  187. data/lib/relaton/jcgm/data_fetcher.rb +81 -0
  188. data/lib/relaton/jcgm/docidentifier.rb +102 -0
  189. data/lib/relaton/jcgm/doctype.rb +12 -0
  190. data/lib/relaton/jcgm/ext.rb +23 -0
  191. data/lib/relaton/jcgm/item.rb +20 -0
  192. data/lib/relaton/jcgm/item_base.rb +18 -0
  193. data/lib/relaton/jcgm/item_data.rb +6 -0
  194. data/lib/relaton/jcgm/meetings_parser.rb +175 -0
  195. data/lib/relaton/jcgm/processor.rb +71 -0
  196. data/lib/relaton/jcgm/relation.rb +9 -0
  197. data/lib/relaton/jcgm/structured_identifier.rb +40 -0
  198. data/lib/relaton/jcgm/util.rb +8 -0
  199. data/lib/relaton/jcgm.rb +24 -0
  200. data/lib/relaton/jis/bibliography.rb +8 -10
  201. data/lib/relaton/jis/data_fetcher.rb +21 -19
  202. data/lib/relaton/jis/docidentifier.rb +104 -5
  203. data/lib/relaton/jis/hit.rb +18 -23
  204. data/lib/relaton/jis/hit_collection.rb +19 -18
  205. data/lib/relaton/jis/processor.rb +1 -1
  206. data/lib/relaton/jis.rb +2 -3
  207. data/lib/relaton/logger/channels/gh_issue.rb +78 -13
  208. data/lib/relaton/nist/data_fetcher.rb +63 -13
  209. data/lib/relaton/nist/docidentifier.rb +165 -0
  210. data/lib/relaton/nist/item.rb +2 -0
  211. data/lib/relaton/nist/item_base.rb +16 -0
  212. data/lib/relaton/nist/mods_parser.rb +38 -12
  213. data/lib/relaton/nist/processor.rb +2 -1
  214. data/lib/relaton/nist/relation.rb +3 -0
  215. data/lib/relaton/nist/scraper.rb +6 -3
  216. data/lib/relaton/oasis/bibliography.rb +147 -6
  217. data/lib/relaton/oasis/data_fetcher.rb +41 -5
  218. data/lib/relaton/oasis/data_parser_utils.rb +37 -3
  219. data/lib/relaton/oasis/docidentifier.rb +54 -0
  220. data/lib/relaton/oasis/item.rb +3 -0
  221. data/lib/relaton/oasis/processor.rb +7 -1
  222. data/lib/relaton/oasis.rb +14 -1
  223. data/lib/relaton/ogc/data_fetcher.rb +23 -2
  224. data/lib/relaton/ogc/docidentifier.rb +105 -0
  225. data/lib/relaton/ogc/hit_collection.rb +78 -3
  226. data/lib/relaton/ogc/processor.rb +2 -1
  227. data/lib/relaton/ogc.rb +5 -1
  228. data/lib/relaton/oiml/bibliography.rb +90 -15
  229. data/lib/relaton/oiml/docidentifier.rb +18 -3
  230. data/lib/relaton/omg/docidentifier.rb +67 -0
  231. data/lib/relaton/omg/item.rb +1 -0
  232. data/lib/relaton/omg/processor.rb +1 -0
  233. data/lib/relaton/omg/scraper.rb +61 -16
  234. data/lib/relaton/omg.rb +1 -0
  235. data/lib/relaton/plateau/bibliography.rb +10 -3
  236. data/lib/relaton/plateau/data_fetcher.rb +25 -2
  237. data/lib/relaton/plateau/handbook_parser.rb +8 -1
  238. data/lib/relaton/plateau/hit.rb +10 -2
  239. data/lib/relaton/plateau/hit_collection.rb +31 -11
  240. data/lib/relaton/plateau/processor.rb +3 -1
  241. data/lib/relaton/plateau/technical_report_parser.rb +8 -1
  242. data/lib/relaton/plateau.rb +2 -1
  243. data/lib/relaton/sdo/config.rb +34 -0
  244. data/lib/relaton/sdo/fetcher.rb +52 -0
  245. data/lib/relaton/sdo/logo.rb +95 -0
  246. data/lib/relaton/sdo/name.rb +26 -0
  247. data/lib/relaton/sdo/organization.rb +71 -0
  248. data/lib/relaton/sdo/store.rb +49 -0
  249. data/lib/relaton/sdo.rb +29 -0
  250. data/lib/relaton/version.rb +1 -1
  251. data/lib/relaton/w3c/bibliography.rb +132 -12
  252. data/lib/relaton/w3c/data_fetcher.rb +194 -16
  253. data/lib/relaton/w3c/data_parser.rb +3 -3
  254. data/lib/relaton/w3c/docidentifier.rb +48 -0
  255. data/lib/relaton/w3c/governor.rb +32 -0
  256. data/lib/relaton/w3c/item.rb +3 -0
  257. data/lib/relaton/w3c/pubid.rb +12 -0
  258. data/lib/relaton/w3c/safe_realize.rb +110 -21
  259. data/lib/relaton/w3c.rb +12 -1
  260. data/lib/relaton/xsf/bibliography.rb +61 -1
  261. data/lib/relaton/xsf/data_fetcher.rb +55 -5
  262. data/lib/relaton/xsf/docidentifier.rb +46 -0
  263. data/lib/relaton/xsf/hit_collection.rb +31 -3
  264. data/lib/relaton/xsf/item.rb +6 -0
  265. data/lib/relaton/xsf/processor.rb +1 -0
  266. data/lib/relaton/xsf.rb +5 -1
  267. data/lib/relaton.rb +42 -0
  268. metadata +135 -24
  269. data/lib/relaton/ieee/pub_id.rb +0 -161
  270. data/lib/relaton/index/id_number.rb +0 -30
@@ -36,23 +36,15 @@ module Relaton
36
36
  # the list of available edition years when none match the requested year
37
37
  #
38
38
  def find
39
- if pubid.year
40
- find_by_year pubid.year
41
- else
42
- find_all_years
43
- end
39
+ pubid.year ? find_by_year : find_all_years
44
40
  end
45
41
 
46
- def find_by_year(ref_year)
47
- missed_years = []
48
- @array.each do |hit|
49
- next unless hit.matches?
50
-
51
- return hit.item if hit.pubid.year.to_s == ref_year.to_s
52
-
53
- missed_years << hit.pubid.year
54
- end
55
- missed_years
42
+ # The item of the requested edition. When no edition has the requested
43
+ # year, the years of the matching editions are returned.
44
+ def find_by_year
45
+ editions = @array.select(&:matches?)
46
+ hit = edition_hit editions
47
+ hit ? hit.item : editions.map { |h| h.pubid.year }
56
48
  end
57
49
 
58
50
  # The main item is the latest edition of the requested type; every other
@@ -69,9 +61,11 @@ module Relaton
69
61
 
70
62
  # The lowest-numbered part becomes the all-parts umbrella; every candidate
71
63
  # sharing the series and number is attached as an `instanceOf` relation.
64
+ # A part-less document counts as part 0. Its index row deserializes with
65
+ # `parts` nil, not [].
72
66
  def find_all_parts
73
67
  parts = @array.select { |hit| hit.matches? all_parts: true }
74
- lowest = parts.min_by { |hit| hit.pubid.parts.first.to_i }
68
+ lowest = parts.min_by { |hit| Array(hit.pubid.parts).first.to_i }
75
69
  item = lowest.item.to_all_parts
76
70
  attach_relations item, item.docidentifier.first.content
77
71
  end
@@ -101,14 +95,21 @@ module Relaton
101
95
  def index
102
96
  @index ||= Relaton::Index.find_or_create(
103
97
  :jis,
104
- url: "#{GH_URL}#{INDEXFILE_V2}.zip",
105
- file: "#{INDEXFILE_V2}.yaml",
98
+ url: "#{GH_URL}#{INDEXFILE}.zip",
99
+ file: "#{INDEXFILE}.yaml",
106
100
  pubid_class: ::Pubid::Jis::Identifier,
107
101
  )
108
102
  end
109
103
 
110
104
  private
111
105
 
106
+ # The exact printed id comes first, then the same identifier without the
107
+ # reaffirmation mark (`JIS L 4107:2000` finds `JIS L 4107:2000R`).
108
+ def edition_hit(editions)
109
+ editions.find { |h| h.pubid.to_s == pubid.to_s } ||
110
+ editions.find { |h| h.pubid.matches? pubid, ignore: [:reaffirmed] }
111
+ end
112
+
112
113
  # Broad candidate filter: same series and number as the reference. For a
113
114
  # supplement (amendment/corrigendum/explanation) the document series and
114
115
  # number live on `base`, so compare against that.
@@ -47,7 +47,7 @@ module Relaton
47
47
  def remove_index_file
48
48
  require_relative "../jis"
49
49
  Relaton::Index.find_or_create(
50
- :jis, url: true, file: "#{INDEXFILE_V2}.yaml"
50
+ :jis, url: true, file: "#{INDEXFILE}.yaml"
51
51
  ).remove_file
52
52
  end
53
53
  end
data/lib/relaton/jis.rb CHANGED
@@ -15,9 +15,8 @@ require_relative "jis/bibliography"
15
15
 
16
16
  module Relaton
17
17
  module Jis
18
- INDEXFILE = "index-v1"
19
- # Pubid-based index produced alongside index-v1 during the pubid migration.
20
- INDEXFILE_V2 = "index-v2"
18
+ # The pubid index. relaton-data-jis rebuilds the legacy index-v1 from data/.
19
+ INDEXFILE = "index-v2"
21
20
 
22
21
  class Error < StandardError; end
23
22
 
@@ -1,14 +1,15 @@
1
1
  require "net/http"
2
+ require "json"
2
3
 
3
4
  module Relaton
4
5
  module Logger
5
6
  module Channels
6
7
  #
7
- # This class is used to create a GitHub issue with the log content.
8
- # The issue will be created in the repository specified in the
9
- # initializer.
8
+ # This class is used to report the log content as a GitHub issue in the
9
+ # repository specified in the initializer.
10
10
  # The log content is stored in the issue body. Only unique log messages
11
- # are stored.
11
+ # are stored. When an open issue with the same title already exists, the
12
+ # log is added to it as a comment rather than opening a duplicate.
12
13
  # Token is required to create an issue. It should be stored in the
13
14
  # environment variable GITHUB_TOKEN.
14
15
  # To create an issue, call the create_issue method after all log messages
@@ -35,33 +36,97 @@ module Relaton
35
36
  def close
36
37
  end
37
38
 
39
+ # Report the collected log. A crawler runs on a schedule, so creating a
40
+ # fresh issue every time would pile up duplicates of the same standing
41
+ # report; when an open issue with this title already exists, add the run's
42
+ # log as a comment on it instead.
38
43
  def create_issue
39
44
  return if @log.empty? || ENV["GITHUB_TOKEN"].nil?
40
45
 
41
- responce = post_issue
42
-
43
- if responce.code.to_i == 201
44
- puts "Issue created!"
46
+ if (number = open_issue_number)
47
+ report_response post_comment(number), "Issue #{number} commented!", 201
45
48
  else
46
- puts "Failed to create issue: #{responce.code} #{responce.message}\n#{responce.body}"
49
+ report_response post_issue, "Issue created!", 201
47
50
  end
48
51
  end
49
52
 
50
53
  private
51
54
 
52
- def post_issue
53
- uri = URI("https://api.github.com/repos/#{@repo}/issues")
55
+ def report_response(responce, success_message, success_code)
56
+ if responce.code.to_i == success_code
57
+ puts success_message
58
+ else
59
+ puts "Failed to report issue: #{responce.code} #{responce.message}\n#{responce.body}"
60
+ end
61
+ end
62
+
63
+ # Number of the open issue with exactly this title, or nil. Any failure
64
+ # (network, rate limit, missing search scope) degrades to nil, so the
65
+ # worst case is the previous behaviour — a new issue.
66
+ def open_issue_number
67
+ query = "repo:#{@repo} is:issue is:open in:title \"#{@title}\""
68
+ uri = URI("https://api.github.com/search/issues")
69
+ uri.query = URI.encode_www_form(q: query, per_page: 100)
70
+ responce = get(uri)
71
+ return nil unless responce.code.to_i == 200
72
+
73
+ items = JSON.parse(responce.body)["items"] || []
74
+ items.find { |i| i["title"] == @title }&.fetch("number", nil)
75
+ rescue StandardError => e
76
+ puts "Failed to search for an open `#{@title}` issue: #{e.message}"
77
+ nil
78
+ end
79
+
80
+ def get(uri)
81
+ http = Net::HTTP.new(uri.host, uri.port)
82
+ http.use_ssl = true
83
+ http.request(Net::HTTP::Get.new(uri.request_uri, headers))
84
+ end
85
+
86
+ def post(uri, body)
54
87
  http = Net::HTTP.new(uri.host, uri.port)
55
88
  http.use_ssl = true
56
89
 
57
90
  request = Net::HTTP::Post.new(uri.request_uri, headers)
58
- request.body = issue_body.to_json
91
+ request.body = body.to_json
59
92
 
60
93
  http.request(request)
61
94
  end
62
95
 
96
+ def post_issue
97
+ post URI("https://api.github.com/repos/#{@repo}/issues"), issue_body
98
+ end
99
+
100
+ def post_comment(number)
101
+ post URI("https://api.github.com/repos/#{@repo}/issues/#{number}/comments"),
102
+ { body: body }
103
+ end
104
+
63
105
  def issue_body
64
- { title: @title, body: @log.join("\n") }
106
+ { title: @title, body: body }
107
+ end
108
+
109
+ # GitHub rejects an issue or comment body longer than this with a 422,
110
+ # which loses the whole report — and these reports get long (a crawl can
111
+ # log hundreds of ids it could not index). Keep as many whole lines as
112
+ # fit and say how many were left out.
113
+ MAX_BODY = 65_536
114
+ TRUNCATION_ROOM = 200
115
+
116
+ def body
117
+ lines = @log.to_a
118
+ text = lines.join("\n")
119
+ return text if text.length <= MAX_BODY
120
+
121
+ kept = []
122
+ size = 0
123
+ lines.each do |line|
124
+ break if size + line.length + 1 > MAX_BODY - TRUNCATION_ROOM
125
+
126
+ kept << line
127
+ size += line.length + 1
128
+ end
129
+ "#{kept.join("\n")}\n\n…and #{lines.size - kept.size} more (see the run log)"
65
130
  end
66
131
 
67
132
  def headers
@@ -9,44 +9,94 @@ require_relative "mods_parser"
9
9
  module Relaton
10
10
  module Nist
11
11
  class DataFetcher < Core::DataFetcher
12
- URL = "https://github.com/usnistgov/NIST-Tech-Pubs/releases/download/Nov2024/allrecords-MODS.xml"
12
+ RELEASES_URL = "https://github.com/usnistgov/NIST-Tech-Pubs/releases"
13
+ MODS_ASSET = "allrecords-MODS.xml"
13
14
 
14
- def fetch(_source = nil)
15
+ def fetch(source = nil)
15
16
  FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
16
- fetch_tech_pubs
17
+ fetch_tech_pubs source
17
18
  # add_static_files
18
19
  index.save
19
20
  report_errors
20
21
  end
21
22
 
22
- def fetch_tech_pubs
23
- xml_data = Mechanize.new.get(URL).body
23
+ def fetch_tech_pubs(source = nil)
24
+ xml_data = Mechanize.new.get(source_url(source)).body
24
25
  docs = LocMods::Collection.from_xml xml_data
25
26
  docs.mods.each { |doc| write_file ModsParser.new(doc, series, @errors).parse }
26
27
  end
27
28
 
29
+ # Build the MODS download URL for a NIST-Tech-Pubs release. With no tag
30
+ # (nil, blank, or "latest") use GitHub's `latest/download` redirect so new
31
+ # releases are picked up automatically by the crawler; a concrete tag
32
+ # (e.g. "June2026") pins that specific release.
33
+ def source_url(source = nil)
34
+ tag = source.to_s.strip
35
+ path = tag.empty? || tag.casecmp?("latest") ? "latest/download" : "download/#{tag}"
36
+ "#{RELEASES_URL}/#{path}/#{MODS_ASSET}"
37
+ end
38
+
28
39
  def write_file(bib)
29
40
  id = bib.docidentifier.find(&:primary) || bib.docidentifier.first
30
- file = output_file id.content.sub(/^NIST IR/, "NISTIR")
41
+ unless id
42
+ failures << "Document skipped: no identifier (#{document_label(bib)})"
43
+ return
44
+ end
45
+ # Distinct docids can sanitize to one filename; take a path of our own
46
+ # rather than overwriting the other document (Core#unique_output_file).
47
+ docid = id.content.sub(/^NIST IR/, "NISTIR")
48
+ file = unique_output_file docid
31
49
  if @files.include? file
50
+ # Same reserved path == same docid: a genuine duplicate. Checked FIRST,
51
+ # because a disambiguated path stays != output_file forever.
32
52
  Util.warn "File #{file} exists. Docid: #{id.content}"
33
- else @files << file
53
+ elsif file != output_file(docid)
54
+ Util.warn "File #{output_file docid} exists. Docid: #{id.content}. Writing #{file} instead."
34
55
  end
56
+ @files << file
35
57
  pid = pubid id.content
36
- index.add_or_update pid, file if pid
58
+ if pid
59
+ index.add_or_update pid, file
60
+ else
61
+ failures << "Unparseable id `#{id.content}` was not indexed (#{file})"
62
+ end
37
63
  File.write file, serialize(bib), encoding: "UTF-8"
38
64
  end
39
65
 
40
- # Parse a docidentifier string into a Pubid::Nist::Identifier; nil (with a
41
- # warning) if pubid can't parse it, so a single bad id never aborts the
42
- # crawl or corrupts index-v2.
66
+ # A human-readable label for a bib with no usable docidentifier, so the
67
+ # skipped record is identifiable in the reported error.
68
+ def document_label(bib)
69
+ Array(bib.title).map(&:content).compact.reject(&:empty?).first || "unknown title"
70
+ end
71
+
72
+ # Parse a docidentifier string into a Pubid::Nist::Identifier, or nil if
73
+ # pubid can't parse it — so a single bad id never aborts the crawl or
74
+ # corrupts index-v2. The caller records the failure (see #write_file) so
75
+ # #report_errors can surface it as a tracked GitHub issue.
43
76
  def pubid(id)
44
77
  ::Pubid::Nist::Identifier.parse id
45
- rescue StandardError => e
46
- Util.warn "Failed to parse `#{id}` with pubid: #{e.message}"
78
+ rescue StandardError
47
79
  nil
48
80
  end
49
81
 
82
+ # Per-document failures (unparseable ids, identifier-less records)
83
+ # collected during the crawl and surfaced by #report_errors.
84
+ def failures
85
+ @failures ||= []
86
+ end
87
+
88
+ # Surface accumulated per-document failures through the shared error
89
+ # machinery (the "Error fetching documents" GitHub issue in CI) so they
90
+ # are visible and tracked, not a hard crash or a silent stderr warning.
91
+ # The gh_issue channel is registered inside #report_errors, so emit these
92
+ # at :error (via #log_error) after it is set up and before super creates
93
+ # the issue.
94
+ def report_errors
95
+ gh_issue
96
+ failures.each { |msg| log_error msg }
97
+ super
98
+ end
99
+
50
100
  # def add_static_files
51
101
  # Dir["./static/*.yaml"].each do |file|
52
102
  # bib = Item.from_yaml(File.read(file, encoding: "UTF-8"))
@@ -0,0 +1,165 @@
1
+ module Relaton
2
+ module Nist
3
+ # NIST document identifier backed by a parsed Pubid::Nist::Identifier.
4
+ #
5
+ # Mirrors the CCSDS/IEC flavors (lib/relaton/ccsds/model/docidentifier.rb,
6
+ # lib/relaton/iec/model/docidentifier.rb): the lutaml `content` stays a plain
7
+ # string for serialization, while the parsed pubid is kept in `@pubid` so the
8
+ # base class's abstract structural mutators (`remove_part!`, `remove_date!`,
9
+ # `to_all_parts!`) can edit the identifier graph and re-render.
10
+ #
11
+ # NIST-specific caveats:
12
+ # - **A pubid is only adopted when it round-trips.** A NIST DOI such as
13
+ # `NIST.SP.800-162` is itself a valid NIST pubid in dotted MR form, but it
14
+ # must serialize verbatim (its `:human` render is `NIST SP 800-162`). So
15
+ # `content=` keeps the parsed pubid only when `pubid.to_s(:human)` equals
16
+ # the input; otherwise the raw string is preserved. This test is
17
+ # type-independent, so it holds no matter what order lutaml assigns
18
+ # `content`/`type` on `from_xml`/`from_yaml` (unlike a `type == "NIST"`
19
+ # gate, which sees a nil `type` on the YAML setter-ordering path).
20
+ # - **The date lives in the `edition` component, not a `:date` attribute.**
21
+ # NIST has no single date field (`year`/`month` scalars are nil on the
22
+ # common parse paths); a publication year is carried as the edition id
23
+ # (`FIPS 46e1977`) or its trailing `additional_text` (`NBS CIRC 11e2.1915`).
24
+ # So `remove_date!` clears the scalar date fields *and* the edition's
25
+ # year component, while preserving non-year editions/revisions (`r5`, `e2`).
26
+ # Dates carried in the `update` component (some NBS supplements) are left
27
+ # intact — `update` also encodes non-date update codes (`/Upd2`).
28
+ # - **No `(all parts)` marker in `content`.** `content` is live-derived
29
+ # from `@pubid` (see below), so `to_all_parts!` freezes the already
30
+ # part+date+stage-stripped rendering into `@raw_content` before
31
+ # wrapping `@pubid` — matching IEC/CCSDS. `#pubid` itself, read
32
+ # directly, now answers `all_parts? == true` and renders WITH pubid's
33
+ # generic "(all parts)" marker, since it's the wrapper.
34
+ class Docidentifier < Bib::Docidentifier
35
+ attribute :content, :string
36
+
37
+ attr_reader :pubid
38
+
39
+ YEAR = /\A\d{4}\z/
40
+
41
+ def initialize(arg = nil, **kwargs)
42
+ arg.is_a?(Hash) ? super(arg) : super(**kwargs)
43
+ # `super` normally runs `content=` via lutaml's attribute application; the
44
+ # re-run is a fallback for construction paths that skip it (so the value
45
+ # is parsed exactly once in the common case).
46
+ raw = arg.is_a?(Hash) ? (arg["content"] || arg[:content]) : kwargs[:content]
47
+ self.content = raw if raw && @pubid.nil? && @raw_content.nil?
48
+ end
49
+
50
+ alias_method :original_content=, :content=
51
+ alias_method :original_content, :content
52
+
53
+ def content=(value)
54
+ @pubid = nil
55
+ @raw_content = nil
56
+
57
+ parsed = value.is_a?(::Pubid::Nist::Identifier) ? value : parse_pubid(value)
58
+
59
+ # Adopt the pubid only when it renders back to the exact input, so
60
+ # non-canonical forms (DOIs) are preserved as the raw string.
61
+ if parsed && (!value.is_a?(::String) || parsed.to_s(format: :human) == value)
62
+ @pubid = parsed
63
+ elsif value.is_a?(::String)
64
+ @raw_content = value
65
+ end
66
+
67
+ send(:original_content=, to_s)
68
+ end
69
+
70
+ def content
71
+ return @raw_content if @raw_content
72
+ return @pubid.to_s(format: :human) if @pubid
73
+
74
+ original_content
75
+ end
76
+
77
+ def to_s
78
+ content.to_s
79
+ end
80
+
81
+ # `#exclude` (no args) clones the whole graph (incl. a supplement's
82
+ # base), since `walk_chain` mutates it in place. `@raw_content` freezes
83
+ # the stripped rendering — `content` is live from `@pubid`, and the
84
+ # wrapper would otherwise add pubid's generic "(all parts)" marker,
85
+ # which pubid-nist itself never has. `identifiers` then keeps the
86
+ # ORIGINAL part/date/stage, not the stripped working copy.
87
+ def to_all_parts!
88
+ return if !@pubid || @pubid.all_parts?
89
+
90
+ original = @pubid.exclude
91
+ remove_part!
92
+ remove_date!
93
+ remove_stage!
94
+ @raw_content = to_s
95
+ @pubid = original.to_all_parts
96
+ end
97
+
98
+ def remove_part!
99
+ walk_chain { |node| clear_part_on(node) }
100
+ end
101
+
102
+ def remove_stage!
103
+ walk_chain { |node| node.stage = nil if node.respond_to?(:stage=) }
104
+ end
105
+
106
+ def remove_date!
107
+ walk_chain { |node| clear_date_on(node) }
108
+ end
109
+
110
+ private
111
+
112
+ def parse_pubid(value)
113
+ return nil unless value.is_a?(::String) && !value.empty?
114
+
115
+ ::Pubid::Nist::Identifier.parse(value)
116
+ rescue StandardError
117
+ Util.warn "Failed to parse Pubid: #{value}"
118
+ nil
119
+ end
120
+
121
+ # Apply a mutation to the top pubid and every identifier down the
122
+ # `base` chain (only SupplementIdentifier exposes the chain; a
123
+ # plain identifier returns nil and the walk terminates after one pass).
124
+ def walk_chain
125
+ return unless @pubid
126
+
127
+ node = @pubid
128
+ while node
129
+ yield node
130
+ node = node.respond_to?(:base) ? node.base : nil
131
+ end
132
+ refresh_content!
133
+ end
134
+
135
+ def clear_part_on(pubid)
136
+ pubid.part = nil if pubid.respond_to?(:part=)
137
+ pubid.parts = nil if pubid.respond_to?(:parts=)
138
+ end
139
+
140
+ # NIST spreads the date across scalar fields (mostly nil in practice) and
141
+ # the edition component. Clear the scalars, then strip a year that rides on
142
+ # the edition: a trailing year suffix (`e2.1915` -> `e2`) or a whole
143
+ # bare-year edition (`e1977`/`r2001` -> gone), while leaving numbered
144
+ # editions and revisions (`e2`, `r5`) intact.
145
+ def clear_date_on(pubid)
146
+ %i[year month revision_year revision_month edition_year update_year]
147
+ .each { |f| pubid.send("#{f}=", nil) if pubid.respond_to?("#{f}=") }
148
+
149
+ return unless pubid.respond_to?(:edition) && (ed = pubid.edition)
150
+
151
+ if ed.respond_to?(:additional_text) && ed.additional_text.to_s.match?(YEAR)
152
+ ed.additional_text = nil
153
+ end
154
+ return unless ed.respond_to?(:id) && ed.id.to_s.match?(YEAR) &&
155
+ (!ed.respond_to?(:additional_text) || ed.additional_text.nil?)
156
+
157
+ pubid.edition = nil if pubid.respond_to?(:edition=)
158
+ end
159
+
160
+ def refresh_content!
161
+ send(:original_content=, to_s)
162
+ end
163
+ end
164
+ end
165
+ end
@@ -1,4 +1,5 @@
1
1
  require_relative "date"
2
+ require_relative "docidentifier"
2
3
  require_relative "ext"
3
4
 
4
5
  module Relaton
@@ -9,6 +10,7 @@ module Relaton
9
10
  class Item < Bib::Item
10
11
  model ItemData
11
12
 
13
+ attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
12
14
  attribute :date, Date, collection: true
13
15
  attribute :relation, Relation, collection: true, initialize_empty: true
14
16
  attribute :ext, Ext
@@ -0,0 +1,16 @@
1
+ require_relative "docidentifier"
2
+
3
+ module Relaton
4
+ module Nist
5
+ # Nested bibitem used inside Relation. Mirrors Iso::ItemBase / Iec::ItemBase:
6
+ # overrides the shared docidentifier/relation with the NIST-flavored types so
7
+ # a relation's cross-reference ids are Nist::Docidentifier too (keeping the
8
+ # whole tree's docidentifier collections a single, YAML-serializable type).
9
+ class ItemBase < Bib::ItemBase
10
+ model ItemData
11
+
12
+ attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
13
+ attribute :relation, Relation, collection: true, initialize_empty: true
14
+ end
15
+ end
16
+ end
@@ -26,7 +26,7 @@ module Relaton
26
26
  args = ATTRS.each_with_object({}) do |attr, hash|
27
27
  hash[attr] = send("parse_#{attr}")
28
28
  end
29
- args[:ext] = Ext.new(doctype: parse_doctype, flavor: "nilst")
29
+ args[:ext] = Ext.new(doctype: parse_doctype, flavor: "nist")
30
30
  ItemData.new(**args)
31
31
  end
32
32
 
@@ -39,7 +39,10 @@ module Relaton
39
39
  { type: "DOI", content: parse_doi },
40
40
  ].reject { |id| id[:content].nil? || id[:content].empty? }
41
41
  @errors[:docidentifier] &&= ids.empty?
42
- ids.map { |id| Bib::Docidentifier.new(**id) }
42
+ # All ids share the Nist::Docidentifier collection type (Item#docidentifier)
43
+ # so YAML serialization stays valid; the DOI is not canonical NIST pubid
44
+ # form, so the class keeps it as the raw string (see Docidentifier#content=).
45
+ ids.map { |id| Docidentifier.new(**id) }
43
46
  end
44
47
 
45
48
  # @return [String]
@@ -55,7 +58,7 @@ module Relaton
55
58
  # not any inside the number). Then force :human rendering.
56
59
  spaced = str.sub(/\A(NIST|NBS)\.([A-Z][A-Za-z]*)\./, '\1 \2 ')
57
60
  ::Pubid::Nist::Identifier.parse(spaced).to_s(format: :human)
58
- rescue Parslet::ParseFailed
61
+ rescue ::Pubid::Errors::ParseError
59
62
  str.gsub(".", " ").sub(/^[\D]+/, &:upcase)
60
63
  end
61
64
 
@@ -154,21 +157,36 @@ module Relaton
154
157
  def parse_doctype = Doctype.new(content: "standard")
155
158
 
156
159
  def parse_contributor
157
- # exclude primary contributors to avoid duplication
158
- contributor = @doc.name.reject { |n| n.usage == "primary" }.map do |name|
160
+ seen = []
161
+ contributor = @doc.name.filter_map do |name|
159
162
  entity, default_role = create_entity(name)
160
163
  next unless entity
161
164
 
165
+ # A name may appear twice — as the `usage="primary"` main entry and
166
+ # again as a plain added entry. Keep the first (primary) occurrence
167
+ # and drop the duplicate, so the lead author is retained, not doubled.
168
+ key = contributor_key(name)
169
+ next if seen.include?(key)
170
+
171
+ seen << key
162
172
  role = (name.role || []).reduce([]) do |a, r|
163
173
  a + r.role_term.map { |rt| Bib::Contributor::Role.new(type: rt.content) }
164
174
  end
165
175
  role << Bib::Contributor::Role.new(type: default_role) if role.empty?
166
176
  create_contributor(entity, role)
167
- end.compact
177
+ end
168
178
  @errors[:contributor] &&= contributor.empty?
169
179
  contributor
170
180
  end
171
181
 
182
+ # De-duplication signature for a contributor: its type, its non-date name
183
+ # parts, and its authority id — so identically-named people with distinct
184
+ # authority ids are still kept separate.
185
+ def contributor_key(name)
186
+ parts = name.name_part.reject(&:type).map { |p| p.content&.strip }
187
+ [name.type, parts, name.name_identifier&.first&.content]
188
+ end
189
+
172
190
  def create_contributor(entity, role)
173
191
  case entity
174
192
  when Bib::Person
@@ -228,17 +246,25 @@ module Relaton
228
246
  item_id = get_id_from_str related_item_id(item)
229
247
  return if item_id.nil? || item_id.empty?
230
248
 
231
- docid = Bib::Docidentifier.new(type: "NIST", content: item_id)
249
+ docid = Docidentifier.new(type: "NIST", content: item_id)
232
250
  fref = Bib::Formattedref.new(content: item_id)
233
251
  ItemData.new(docidentifier: [docid], formattedref: fref)
234
252
  end
235
253
 
254
+ # Derive the related document's DOI id. Prefer an `otherType` that carries
255
+ # a `10.6028` DOI — it may be embedded in prose (e.g. "Translation of |a
256
+ # 10.6028/NIST.SP.1299" or "Supersedes … https://doi.org/10.6028/…"), not
257
+ # only a bare DOI — otherwise fall back to the `<name><namePart>`. Returns
258
+ # nil (relation skipped by #create_related_item) when neither yields a DOI,
259
+ # so a related item with no name and no DOI never aborts the crawl.
236
260
  def related_item_id(item)
237
- if item.other_type && item.other_type[0..6] == "10.6028"
238
- item.other_type
239
- else
240
- item.name[0].name_part[0].content
241
- end => id
261
+ id = if item.other_type&.include?("10.6028")
262
+ item.other_type
263
+ else
264
+ item.name&.first&.name_part&.first&.content
265
+ end
266
+ return if id.nil?
267
+
242
268
  doi = remove_doi_prefix(id)
243
269
  return if doi.nil?
244
270
 
@@ -6,6 +6,7 @@ module Relaton
6
6
  def initialize # rubocop:disable Lint/MissingSuper
7
7
  @short = :relaton_nist
8
8
  @prefix = "NIST"
9
+ @pubid_flavor = :Nist # global prefixes sourced from Pubid::Nist.prefixes
9
10
  @defaultprefix = %r{^(NIST|NBS|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
10
11
  @idtype = "NIST"
11
12
  @datasets = %w[nist-tech-pubs]
@@ -60,7 +61,7 @@ module Relaton
60
61
  def remove_index_file
61
62
  require_relative "../nist"
62
63
  Relaton::Index.find_or_create(
63
- :nist, url: true, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Nist::Identifier
64
+ :nist, url: true, file: "#{INDEXFILE}.yaml"
64
65
  ).remove_file
65
66
  end
66
67
  end
@@ -1,7 +1,10 @@
1
+ require_relative "item_base"
2
+
1
3
  module Relaton
2
4
  module Nist
3
5
  class Relation
4
6
  attribute :type, :string, values: %w[obsoletedBy supersedes supersededBy]
7
+ attribute :bibitem, ItemBase
5
8
  end
6
9
  end
7
10
  end