relaton 3.1.1 → 3.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/README.adoc +5 -3
  3. data/lib/relaton/3gpp.rb +1 -1
  4. data/lib/relaton/bib/converter/csl/from_csl.rb +195 -0
  5. data/lib/relaton/bib/converter/csl.rb +13 -1
  6. data/lib/relaton/bib/model/docidentifier.rb +66 -5
  7. data/lib/relaton/calconnect/hit_collection.rb +1 -1
  8. data/lib/relaton/cli/command.rb +3 -1
  9. data/lib/relaton/cli/frontend_assets.rb +7 -6
  10. data/lib/relaton/cli/subcommand_collection.rb +5 -1
  11. data/lib/relaton/core/request_error.rb +18 -0
  12. data/lib/relaton/db/config.rb +9 -1
  13. data/lib/relaton/db/workers_pool.rb +11 -1
  14. data/lib/relaton/db.rb +160 -58
  15. data/lib/relaton/ecma/bibliography.rb +2 -2
  16. data/lib/relaton/etsi/data_fetcher.rb +16 -6
  17. data/lib/relaton/iala/bibliography.rb +1 -1
  18. data/lib/relaton/iana.rb +1 -1
  19. data/lib/relaton/iec/data_parser.rb +41 -2
  20. data/lib/relaton/itu/README.adoc +1 -1
  21. data/lib/relaton/itu/data_crawler_r.rb +1 -1
  22. data/lib/relaton/jis/README.adoc +2 -0
  23. data/lib/relaton/jis/data_fetcher.rb +74 -13
  24. data/lib/relaton/jis/hit_collection.rb +2 -1
  25. data/lib/relaton/jis/processor.rb +1 -1
  26. data/lib/relaton/jis/scraper.rb +46 -5
  27. data/lib/relaton/oasis/bibliography.rb +1 -1
  28. data/lib/relaton/ogc/hit_collection.rb +1 -1
  29. data/lib/relaton/ogc.rb +1 -1
  30. data/lib/relaton/plateau/data_fetcher.rb +10 -1
  31. data/lib/relaton/plateau/handbook_parser.rb +38 -2
  32. data/lib/relaton/plateau/hit_collection.rb +20 -5
  33. data/lib/relaton/sae.rb +1 -1
  34. data/lib/relaton/version.rb +1 -1
  35. data/lib/relaton/w3c/bibliography.rb +1 -1
  36. data/lib/relaton/xsf/bibliography.rb +1 -1
  37. data/lib/relaton/xsf.rb +1 -1
  38. metadata +35 -48
  39. data/lib/relaton/3gpp/CLAUDE.md +0 -218
  40. data/lib/relaton/adobe/CLAUDE.md +0 -104
  41. data/lib/relaton/bib/CLAUDE.md +0 -150
  42. data/lib/relaton/bipm/CLAUDE.md +0 -187
  43. data/lib/relaton/bsi/CLAUDE.md +0 -35
  44. data/lib/relaton/calconnect/CLAUDE.md +0 -177
  45. data/lib/relaton/ccsds/CLAUDE.md +0 -74
  46. data/lib/relaton/cen/CLAUDE.md +0 -219
  47. data/lib/relaton/cie/CLAUDE.md +0 -90
  48. data/lib/relaton/core/CLAUDE.md +0 -140
  49. data/lib/relaton/db/CLAUDE.md +0 -385
  50. data/lib/relaton/doi/CLAUDE.md +0 -105
  51. data/lib/relaton/easc/CLAUDE.md +0 -85
  52. data/lib/relaton/ecma/CLAUDE.md +0 -217
  53. data/lib/relaton/etsi/CLAUDE.md +0 -219
  54. data/lib/relaton/evs/CLAUDE.md +0 -120
  55. data/lib/relaton/gb/CLAUDE.md +0 -47
  56. data/lib/relaton/gost/CLAUDE.md +0 -100
  57. data/lib/relaton/iala/CLAUDE.md +0 -97
  58. data/lib/relaton/iana/CLAUDE.md +0 -105
  59. data/lib/relaton/iec/CLAUDE.md +0 -13
  60. data/lib/relaton/ieee/CLAUDE.md +0 -202
  61. data/lib/relaton/ietf/CLAUDE.md +0 -382
  62. data/lib/relaton/iho/CLAUDE.md +0 -76
  63. data/lib/relaton/index/CLAUDE.md +0 -208
  64. data/lib/relaton/isbn/CLAUDE.md +0 -47
  65. data/lib/relaton/iso/CLAUDE.md +0 -38
  66. data/lib/relaton/itu/CLAUDE.md +0 -685
  67. data/lib/relaton/jcgm/CLAUDE.md +0 -107
  68. data/lib/relaton/jis/CLAUDE.md +0 -64
  69. data/lib/relaton/logger/CLAUDE.md +0 -37
  70. data/lib/relaton/nist/CLAUDE.md +0 -194
  71. data/lib/relaton/oasis/CLAUDE.md +0 -251
  72. data/lib/relaton/ogc/CLAUDE.md +0 -242
  73. data/lib/relaton/oiml/CLAUDE.md +0 -47
  74. data/lib/relaton/omg/CLAUDE.md +0 -137
  75. data/lib/relaton/plateau/CLAUDE.md +0 -126
  76. data/lib/relaton/sae/CLAUDE.md +0 -91
  77. data/lib/relaton/sdo/CLAUDE.md +0 -81
  78. data/lib/relaton/un/CLAUDE.md +0 -91
  79. data/lib/relaton/w3c/CLAUDE.md +0 -369
  80. data/lib/relaton/xsf/CLAUDE.md +0 -189
@@ -11,7 +11,7 @@ module Relaton
11
11
  @short = :relaton_jis
12
12
  @prefix = "JIS"
13
13
  @pubid_identifier = :Jis # Db cache key
14
- @defaultprefix = %r{^(JIS|TR)\s}
14
+ @defaultprefix = %r{^(JIS|TR|WES)\s} # WES: relaton-jis#18
15
15
  @idtype = "JIS"
16
16
  @datasets = %w[jis-webdesk]
17
17
  end
@@ -15,6 +15,12 @@ module Relaton
15
15
  DATETYPES = { "発行年月日" => "issued", "確認年月日" => "confirmed" }.freeze
16
16
  STATUSES = { "有効" => "valid", "廃止" => "withdrawn" }.freeze
17
17
 
18
+ # WES (Japan Welding Engineering Society) standards are sold on the same
19
+ # site and kept in the JIS flavor (relaton-jis#18). Their pages name no
20
+ # drafting body, so JWES is the author and publisher.
21
+ JWES_NAMES = { "ja" => "一般社団法人日本溶接協会",
22
+ "en" => "Japan Welding Engineering Society" }.freeze
23
+
18
24
  def initialize(url, errors = {})
19
25
  @url = url
20
26
  @agent = Mechanize.new
@@ -70,13 +76,17 @@ module Relaton
70
76
  return [] if docid.nil? || docid.empty?
71
77
 
72
78
  [Docidentifier.new(
73
- content: docid, type: "JIS", primary: true,
79
+ content: docid, type: id_type, primary: true,
74
80
  )]
75
81
  end
76
82
 
83
+ # The series letter and the number: `JIS X 0208:1997` -> `X0208`,
84
+ # `TR Z 0010:2008` -> `Z0010`. A WES id has no series letter:
85
+ # `WES TR 2032:2024` -> `2032`.
77
86
  def fetch_docnumber
78
- docid = document_id
79
- match = docid&.match(/^\w+\s(\w)\s?(\d+)/)
87
+ match = document_id&.match(
88
+ /^(?:JIS\s)?(?:WES\s)?(?:T[RS]\s)?(?:([A-Z])\s?)?(\d+)/,
89
+ )
80
90
  @errors[:docnumber] &&= match.nil?
81
91
  return unless match
82
92
 
@@ -87,6 +97,15 @@ module Relaton
87
97
  @document_id ||= @doc.at_xpath("./h2/text()[1]")&.text&.strip
88
98
  end
89
99
 
100
+ def wes?
101
+ document_id.to_s.start_with? "WES "
102
+ end
103
+
104
+ # The docidentifier and structuredidentifier type.
105
+ def id_type
106
+ wes? ? "WES" : "JIS"
107
+ end
108
+
90
109
  def fetch_date
91
110
  result = DATETYPES.each_with_object([]) do |(key, type), a|
92
111
  node = @doc.at_xpath("./div/div/div/p/text()[contains(.,'#{key}')]")
@@ -141,8 +160,13 @@ module Relaton
141
160
  Bib::Status.new(stage: stage)
142
161
  end
143
162
 
163
+ # The JIS grammar requires a doctype and has no WES value, so a WES
164
+ # record takes the JIS value of its form (`WES TR`/`WES TS` match the
165
+ # TR/TS rules).
144
166
  def fetch_doctype # rubocop:disable Metrics/CyclomaticComplexity
145
167
  type = case document_id
168
+ when %r{^WES\s\d[^/]*/AMENDMENT} then "amendment"
169
+ when /^WES\s\d/ then "japanese-industrial-standard"
146
170
  when /JIS\s[A-Z]\s[\w-]+:\d{4}\/AMENDMENT/ then "amendment"
147
171
  when /JIS\s[A-Z]\s[\w-]+/ then "japanese-industrial-standard"
148
172
  when /TR[\s\/][\w-]+/ then "technical-report"
@@ -163,6 +187,8 @@ module Relaton
163
187
  end
164
188
 
165
189
  def fetch_contributor
190
+ return jwes_contributors if wes?
191
+
166
192
  authorizer = create_contrib(
167
193
  "一般財団法人 日本規格協会", "authorizer"
168
194
  )
@@ -175,6 +201,20 @@ module Relaton
175
201
  result
176
202
  end
177
203
 
204
+ def jwes_contributors
205
+ tls = Bib::TypedLocalizedString
206
+ scripts = { "ja" => "Jpan", "en" => "Latn" }
207
+ %w[author publisher].map do |role|
208
+ name = JWES_NAMES.map do |lang, content|
209
+ tls.new content: content, language: lang, script: scripts[lang]
210
+ end
211
+ org = Bib::Organization.new name: name
212
+ Bib::Contributor.new(
213
+ organization: org, role: [Bib::Contributor::Role.new(type: role)],
214
+ )
215
+ end
216
+ end
217
+
178
218
  def create_contrib(name, role)
179
219
  org = Bib::Organization.new name: create_orgname(name)
180
220
  role_obj = Bib::Contributor::Role.new(type: role)
@@ -195,7 +235,8 @@ module Relaton
195
235
 
196
236
  def fetch_editorialgroup_contributor # rubocop:disable Metrics/MethodLength
197
237
  node = @doc.at_xpath("./table/tr[th[.='原案作成団体']]/td")
198
- @errors[:editorialgroup] &&= node.nil?
238
+ # A WES page names no drafting body (see JWES_NAMES).
239
+ @errors[:editorialgroup] &&= node.nil? unless wes?
199
240
  return unless node
200
241
 
201
242
  subdivision = Bib::Subdivision.new(
@@ -215,7 +256,7 @@ module Relaton
215
256
  def fetch_structuredidentifier
216
257
  StructuredIdentifier.new(
217
258
  project_number: Iso::ProjectNumber.new(content: fetch_docnumber),
218
- type: "JIS",
259
+ type: id_type,
219
260
  )
220
261
  end
221
262
 
@@ -109,7 +109,7 @@ module Relaton
109
109
  # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
110
110
  # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
111
111
  # `"..." is not a recognized standards identifier`
112
- # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
112
+ # (`lib/relaton/cli/command.rb`), and `Db#fetch`
113
113
  # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
114
114
  # Rescuing here would collapse "this identifier is malformed" into "no
115
115
  # such document", leaving a caller unable to tell them apart.
@@ -104,7 +104,7 @@ module Relaton
104
104
  # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
105
105
  # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
106
106
  # `"..." is not a recognized standards identifier`
107
- # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
107
+ # (`lib/relaton/cli/command.rb`), and `Db#fetch`
108
108
  # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
109
109
  # Rescuing here would collapse "this identifier is malformed" into "no
110
110
  # such document", leaving a caller unable to tell them apart.
data/lib/relaton/ogc.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  # pubid loads with the flavor, for the ::Pubid::Ogc::Identifier that the
2
2
  # index code names. Processor#remove_index_file names no pubid class: the
3
- # delete never reads the index (see lib/relaton/index/CLAUDE.md).
3
+ # delete never reads the index (see docs/index.md).
4
4
  require "pubid"
5
5
  require_relative "index"
6
6
  require_relative "iso"
@@ -12,6 +12,10 @@ module Relaton
12
12
  class DataFetcher < Core::DataFetcher
13
13
  HANDBOOKS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/handbooks.json".freeze
14
14
  TECHNICAL_REPORTS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/technical-reports.json".freeze
15
+ # The Japanese volume label of a Handbook id gets an ASCII file-name
16
+ # suffix: #file_name strips non-word characters, so 本編 and 本編+付録
17
+ # of one edition would otherwise share a file.
18
+ VOLUME_SUFFIX = { "本編+付録" => "-with-appendix", "本編" => "-main" }.freeze
15
19
 
16
20
  def index
17
21
  @index ||= Relaton::Index.find_or_create(
@@ -125,6 +129,9 @@ module Relaton
125
129
  Util.warn "File #{file} already exists, skipping.", key: id
126
130
  else
127
131
  File.write(file, serialize(item))
132
+ # Recorded for #prune_unwritten: the data repo crawls into a kept
133
+ # data/, and an unrecorded write makes every prune skip.
134
+ record_written file
128
135
  @files << file
129
136
  pid = pubid id
130
137
  if pid
@@ -152,12 +159,14 @@ module Relaton
152
159
  end
153
160
 
154
161
  def file_name(id)
155
- name = id.gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
162
+ volume = id[/ (本編(?:+付録)?)\z/, 1]
163
+ name = id.delete_suffix(" #{volume}").gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
156
164
  if id.match?(/民間活用編/)
157
165
  name += "-private"
158
166
  elsif id.match?(/公共活用編/)
159
167
  name += "-public"
160
168
  end
169
+ name += VOLUME_SUFFIX[volume] if volume
161
170
  File.join(@output, "#{name}.#{@ext}")
162
171
  end
163
172
 
@@ -23,13 +23,49 @@ module Relaton
23
23
  @edition_label ||= @version["title"].to_s[/第[\d.]+版/].to_s
24
24
  end
25
25
 
26
+ # Volume label of the version title, as Pubid::Plateau renders it:
27
+ # "本編" (main volume) or "本編+付録" (main volume and appendix). Handbook
28
+ # #00 edition 5.0 is published as both. ASCII "+" becomes "+".
29
+ def volume_label
30
+ @version["title"].to_s[/第[\d.]+版[  ]+(本編(?:[++]付録)?)/, 1]&.tr("+", "+")
31
+ end
32
+
26
33
  def slug_number
27
- @slug_number ||= @entry["slug"]&.to_s&.split("_")&.first
34
+ @slug_number ||= mapped_docnumber&.[](/#(\S+)/, 1) ||
35
+ @entry["slug"]&.to_s&.split("_")&.first
28
36
  end
29
37
 
30
38
  def parse_docnumber
31
39
  @errors[:hb_docnumber] &&= @entry["slug"].nil? || @entry["slug"].to_s.empty?
32
- ["Handbook ##{slug_number}", edition_label].reject { |s| s.to_s.empty? }.join(" ")
40
+ mapped_docnumber ||
41
+ ["Handbook ##{slug_number}", edition_label, volume_label].reject { |s| s.to_s.empty? }.join(" ")
42
+ end
43
+
44
+ # Part label of a split edition, e.g. "(公共活用編)". Editions 1-3 of
45
+ # Handbook #11 were two documents each (relaton-plateau#13).
46
+ def part_label
47
+ @version["title"].to_s[/([^()]+活用編)/]
48
+ end
49
+
50
+ # The canonical docnumber of a split edition, mapped by pubid's
51
+ # update_codes ("Handbook #11 第1.0版(公共活用編)" -> "Handbook #04
52
+ # 第1.0版"), or nil for an edition that is not split.
53
+ def mapped_docnumber
54
+ return @mapped_docnumber if defined? @mapped_docnumber
55
+
56
+ @mapped_docnumber = if part_label
57
+ slug = @entry["slug"]&.to_s&.split("_")&.first
58
+ raw = "PLATEAU Handbook ##{slug} #{edition_label}#{part_label}"
59
+ map_docnumber(raw)
60
+ end
61
+ end
62
+
63
+ def map_docnumber(raw)
64
+ id = ::Pubid::Plateau.parse(raw).to_s
65
+ id.delete_prefix("PLATEAU ") unless id == raw
66
+ rescue ArgumentError, ::Pubid::Errors::Error => e
67
+ Util.warn "Cannot map split edition `#{raw}`: #{e.message}"
68
+ nil
33
69
  end
34
70
 
35
71
  def parse_abstract
@@ -9,16 +9,24 @@ module Relaton
9
9
  # legacy Latin form (`1.0`), stripped in #to_all_editions to derive the
10
10
  # edition-less family id from a relaton-bib docidentifier *string* (the
11
11
  # fetched document is not a pubid). Matching itself is done on pubid objects.
12
- EDITION_SUFFIX = / (?:第[\d.]+版|\d+\.\d+)$/
12
+ # A Handbook volume label (`本編`, `本編+付録`) follows the edition, and
13
+ # goes with it.
14
+ EDITION_SUFFIX = / (?:第[\d.]+版|\d+\.\d+)(?: 本編(?:+付録)?)?$/
13
15
 
14
16
  # An edition-less reference selects every edition of the document with
15
17
  # pubid's subset match `pubid_ref === row`: the omitted edition matches
16
18
  # any value, and pubid declares `annex` strict for PLATEAU, so
17
19
  # `PLATEAU Handbook #03` does not reach `#03-1`. A reference with an
18
- # edition selects that one row with `==`.
20
+ # edition selects that one row with `==`. pubid also declares `volume`
21
+ # strict, so the edition-less match compares each row without its
22
+ # volume (`本編`, `本編+付録`): every edition includes every volume.
19
23
  def find
20
- @array = index.search(pubid_ref, exact: !all_editions?)
21
- .map { |row| Hit.new(row, self) }
24
+ rows = if all_editions?
25
+ index.search(pubid_ref) { |row| pubid_ref === without_volume(row[:id]) }
26
+ else
27
+ index.search(pubid_ref, exact: true)
28
+ end
29
+ @array = rows.map { |row| Hit.new(row, self) }
22
30
  self
23
31
  end
24
32
 
@@ -43,12 +51,19 @@ module Relaton
43
51
  # parse — pubid normalizes Latin input to the canonical id
44
52
  # (metanorma/pubid #269) — so search accepts both forms. An unrecognized
45
53
  # reference raises; like ISO and 3GPP we let it propagate — relaton-cli
46
- # rescues Pubid::Errors::Error. See lib/relaton/plateau/CLAUDE.md.
54
+ # rescues Pubid::Errors::Error. See docs/plateau.md.
47
55
  # Relaton::Db passes the reference already parsed (relaton#205).
48
56
  def pubid_ref
49
57
  @pubid_ref ||= ref.is_a?(String) ? ::Pubid::Plateau.parse(ref) : ref
50
58
  end
51
59
 
60
+ # The row's identifier as an edition-less reference sees it. `#exclude`
61
+ # returns a copy, so the cached index row is untouched. Only a Handbook
62
+ # has a volume.
63
+ def without_volume(id)
64
+ id.respond_to?(:volume) && id.volume ? id.exclude(:volume) : id
65
+ end
66
+
52
67
  # A reference with no edition (`PLATEAU Handbook #00`, or any Technical
53
68
  # Report — TRs carry no edition) asks for all editions of the document.
54
69
  def all_editions?
data/lib/relaton/sae.rb CHANGED
@@ -10,7 +10,7 @@ require_relative "version"
10
10
  # deposits instead: every SAE standard edition has its own DOI
11
11
  # (`10.4271/j3016_202104`). `DataFetcher` crawls them into the
12
12
  # `relaton-data-sae` data repo with a pubid `index-v2`, and `Bibliography`
13
- # reads that index. See lib/relaton/sae/CLAUDE.md.
13
+ # reads that index. See docs/sae.md.
14
14
  module Relaton
15
15
  module Sae
16
16
  INDEXFILE = "index-v2".freeze
@@ -1,3 +1,3 @@
1
1
  module Relaton
2
- VERSION = "3.1.1".freeze
2
+ VERSION = "3.1.3".freeze
3
3
  end
@@ -133,7 +133,7 @@ module Relaton
133
133
  # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
134
134
  # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
135
135
  # `"..." is not a recognized standards identifier`
136
- # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
136
+ # (`lib/relaton/cli/command.rb`), and `Db#fetch`
137
137
  # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
138
138
  # Rescuing here would collapse "this identifier is malformed" into "no
139
139
  # such document", leaving a caller unable to tell them apart.
@@ -15,7 +15,7 @@ module Relaton
15
15
  # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
16
16
  # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
17
17
  # `"..." is not a recognized standards identifier`
18
- # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`,
18
+ # (`lib/relaton/cli/command.rb`,
19
19
  # `subcommand_collection.rb:134`), and `Db#fetch` logs it via the
20
20
  # `StandardError` arm at `lib/relaton/db.rb:122`. Rescuing here would
21
21
  # collapse "this identifier is malformed" into "no such document", and a
data/lib/relaton/xsf.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  require "mechanize"
4
4
  # pubid loads with the flavor, for the ::Pubid::Xsf::Identifier that the
5
5
  # index code names. Processor#remove_index_file names no pubid class: the
6
- # delete never reads the index (see lib/relaton/index/CLAUDE.md).
6
+ # delete never reads the index (see docs/index.md).
7
7
  require "pubid"
8
8
  require_relative "index"
9
9
  require_relative "bib"