relaton 3.1.2 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,11 +9,20 @@ module Relaton
9
9
  class DataFetcher < Core::DataFetcher
10
10
  URL = "https://webdesk.jsa.or.jp/books/"
11
11
 
12
+ # The Webdesk lists a crawl reads, in this order: the JIS search, then
13
+ # the JWES group list (WES standards, relaton-jis#18). Webdesk keeps the
14
+ # open list in the session, and the list page and its paging
15
+ # (`W11M0070`) read that list. So the lists are read one after the
16
+ # other, and a session refresh opens the current list again.
17
+ LISTS = %w[JIS JWES].freeze
18
+
12
19
  def initialize(output, format)
13
20
  super
14
21
  @queue = SizedQueue.new 10
15
22
  @threads = create_thread_pool 5
16
23
  @mutex = Mutex.new
24
+ # URLs of the documents that did not fetch; see #prune_unwritten.
25
+ @failed_docs = []
17
26
  end
18
27
 
19
28
  def log_error(msg)
@@ -72,30 +81,63 @@ module Relaton
72
81
  else
73
82
  Util.warn "URL: #{url}"
74
83
  Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
84
+ @mutex.synchronize { @failed_docs << url }
75
85
  end
76
86
  else
77
87
  save_doc bib, url
78
88
  end
79
89
  end
80
90
 
91
+ # relaton-data-jis crawls with `prune: true`: it keeps data/, and after
92
+ # #fetch, #prune_unwritten deletes each file the run did not write. So a
93
+ # list that does not open, or stops part way (#get_next_page), raises:
94
+ # the crawl job fails before the prune and publishes nothing.
81
95
  def fetch(_source = nil)
82
- return unless initial_post
96
+ failed = begin
97
+ LISTS.find { |list| !fetch_list(list) }
98
+ ensure
99
+ end_threads_and_wait
100
+ end
101
+ raise Error, "Webdesk list #{failed} did not open" if failed
83
102
 
84
- resp = agent.get "#{URL}W11M0070/index"
85
- parse_page resp
86
103
  index.save
87
104
  report_errors
88
105
  end
89
106
 
107
+ # Queue every document of one list. False if the list does not open.
108
+ def fetch_list(list)
109
+ @list = list
110
+ @initial_time = nil
111
+ return false unless initial_post
112
+
113
+ parse_page agent.get("#{URL}W11M0070/index")
114
+ true
115
+ end
116
+
117
+ # Open the current list in the session, unless it was opened less than
118
+ # 10 minutes ago.
90
119
  def initial_post
91
120
  return true if @initial_time && Time.now - @initial_time < 600
92
121
 
122
+ status = @list == "JWES" ? open_group_list(@list) : open_jis_search
123
+ @initial_time = Time.now
124
+ status || Util.warn("No results found for #{@list}")
125
+ end
126
+
127
+ def open_jis_search
93
128
  body = { record: 0, dantai: "JIS", searchtype2: 1,
94
129
  status_1: 1, status_2: 2 } # rubocop:disable Naming/VariableNumber
95
130
  resp = agent.post "#{URL}W11M0270/index", body
96
- disp = Relaton::Json.parse_external resp.body
97
- @initial_time = Time.now
98
- disp["status"] || Util.warn("No results found for JIS")
131
+ Relaton::Json.parse_external(resp.body)["status"]
132
+ end
133
+
134
+ # A group list from the "domestic group publications" page. The link
135
+ # redirects to the list page, which states the count of documents.
136
+ def open_group_list(dantai)
137
+ page = agent.get "#{URL}W11S0260/getSearchAllListByLink" \
138
+ "?dantai_cd=#{dantai}&display=KOKUNAI"
139
+ script = page.at_xpath('//script[contains(.,"var count =")]')
140
+ script&.text.to_s[/var count = (\d+);/, 1].to_i.positive?
99
141
  end
100
142
 
101
143
  def agent
@@ -111,7 +153,6 @@ module Relaton
111
153
 
112
154
  resp = get_next_page(offset)
113
155
  end
114
- end_threads_and_wait
115
156
  end
116
157
 
117
158
  def parse_offset(resp) # rubocop:disable Metrics/AbcSize
@@ -135,25 +176,44 @@ module Relaton
135
176
  @count.to_i
136
177
  end
137
178
 
179
+ # The next page of the current list. Raises when the page does not
180
+ # come, because a list that stops part way leaves its unread documents
181
+ # unwritten, and the prune would delete them.
138
182
  def get_next_page(offset) # rubocop:disable Metrics/MethodLength
139
183
  attempts = 0
140
184
  begin
141
- if initial_post
142
- url = "#{URL}W11M0070/getAddList"
143
- agent.post url, search_type: "JIS", offset: offset
185
+ unless initial_post
186
+ raise Error, "Webdesk list #{@list} did not open again " \
187
+ "at offset #{offset}"
144
188
  end
189
+
190
+ agent.post "#{URL}W11M0070/getAddList",
191
+ search_type: "JIS", offset: offset
192
+ rescue Error
193
+ raise
145
194
  rescue StandardError => e
146
195
  attempts += 1
147
196
  if attempts < 5
148
197
  sleep 2
149
198
  retry
150
- else
151
- Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
152
199
  end
200
+ raise Error, "Webdesk list #{@list} stopped at offset " \
201
+ "#{offset}: #{e.message}"
153
202
  end
154
203
  end
155
204
 
156
- def save_doc(bib, url) # rubocop:disable Metrics/MethodLength
205
+ # Prune only after a complete crawl: a document that did not fetch was
206
+ # not written, and the prune would delete its file.
207
+ def prune_unwritten(min_ratio: 0.9)
208
+ unless @failed_docs.empty?
209
+ warn "prune skipped: #{@failed_docs.size} document(s) did not fetch"
210
+ return []
211
+ end
212
+
213
+ super
214
+ end
215
+
216
+ def save_doc(bib, url) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
157
217
  return unless bib
158
218
 
159
219
  id = bib.docidentifier.find(&:primary).content
@@ -175,6 +235,7 @@ module Relaton
175
235
  Util.warn "File #{output_file id} already exists; writing #{file} instead" if file != output_file(id)
176
236
  @files << file
177
237
  File.write file, serialize(bib), encoding: "UTF-8"
238
+ record_written file
178
239
  pid = pubid id
179
240
  index.add_or_update pid, file if pid
180
241
  end
@@ -83,7 +83,8 @@ module Relaton
83
83
 
84
84
  def create_relation(hit)
85
85
  id = hit.pubid.to_s
86
- docid = Docidentifier.new content: id, type: "JIS", primary: true
86
+ type = id.start_with?("WES ") ? "WES" : "JIS"
87
+ docid = Docidentifier.new content: id, type: type, primary: true
87
88
  bibitem = ItemData.new(
88
89
  formattedref: Bib::Formattedref.new(content: id),
89
90
  docidentifier: [docid],
@@ -11,7 +11,7 @@ module Relaton
11
11
  @short = :relaton_jis
12
12
  @prefix = "JIS"
13
13
  @pubid_identifier = :Jis # Db cache key
14
- @defaultprefix = %r{^(JIS|TR)\s}
14
+ @defaultprefix = %r{^(JIS|TR|WES)\s} # WES: relaton-jis#18
15
15
  @idtype = "JIS"
16
16
  @datasets = %w[jis-webdesk]
17
17
  end
@@ -15,6 +15,12 @@ module Relaton
15
15
  DATETYPES = { "発行年月日" => "issued", "確認年月日" => "confirmed" }.freeze
16
16
  STATUSES = { "有効" => "valid", "廃止" => "withdrawn" }.freeze
17
17
 
18
+ # WES (Japan Welding Engineering Society) standards are sold on the same
19
+ # site and kept in the JIS flavor (relaton-jis#18). Their pages name no
20
+ # drafting body, so JWES is the author and publisher.
21
+ JWES_NAMES = { "ja" => "一般社団法人日本溶接協会",
22
+ "en" => "Japan Welding Engineering Society" }.freeze
23
+
18
24
  def initialize(url, errors = {})
19
25
  @url = url
20
26
  @agent = Mechanize.new
@@ -70,13 +76,17 @@ module Relaton
70
76
  return [] if docid.nil? || docid.empty?
71
77
 
72
78
  [Docidentifier.new(
73
- content: docid, type: "JIS", primary: true,
79
+ content: docid, type: id_type, primary: true,
74
80
  )]
75
81
  end
76
82
 
83
+ # The series letter and the number: `JIS X 0208:1997` -> `X0208`,
84
+ # `TR Z 0010:2008` -> `Z0010`. A WES id has no series letter:
85
+ # `WES TR 2032:2024` -> `2032`.
77
86
  def fetch_docnumber
78
- docid = document_id
79
- match = docid&.match(/^\w+\s(\w)\s?(\d+)/)
87
+ match = document_id&.match(
88
+ /^(?:JIS\s)?(?:WES\s)?(?:T[RS]\s)?(?:([A-Z])\s?)?(\d+)/,
89
+ )
80
90
  @errors[:docnumber] &&= match.nil?
81
91
  return unless match
82
92
 
@@ -87,6 +97,15 @@ module Relaton
87
97
  @document_id ||= @doc.at_xpath("./h2/text()[1]")&.text&.strip
88
98
  end
89
99
 
100
+ def wes?
101
+ document_id.to_s.start_with? "WES "
102
+ end
103
+
104
+ # The docidentifier and structuredidentifier type.
105
+ def id_type
106
+ wes? ? "WES" : "JIS"
107
+ end
108
+
90
109
  def fetch_date
91
110
  result = DATETYPES.each_with_object([]) do |(key, type), a|
92
111
  node = @doc.at_xpath("./div/div/div/p/text()[contains(.,'#{key}')]")
@@ -141,8 +160,13 @@ module Relaton
141
160
  Bib::Status.new(stage: stage)
142
161
  end
143
162
 
163
+ # The JIS grammar requires a doctype and has no WES value, so a WES
164
+ # record takes the JIS value of its form (`WES TR`/`WES TS` match the
165
+ # TR/TS rules).
144
166
  def fetch_doctype # rubocop:disable Metrics/CyclomaticComplexity
145
167
  type = case document_id
168
+ when %r{^WES\s\d[^/]*/AMENDMENT} then "amendment"
169
+ when /^WES\s\d/ then "japanese-industrial-standard"
146
170
  when /JIS\s[A-Z]\s[\w-]+:\d{4}\/AMENDMENT/ then "amendment"
147
171
  when /JIS\s[A-Z]\s[\w-]+/ then "japanese-industrial-standard"
148
172
  when /TR[\s\/][\w-]+/ then "technical-report"
@@ -163,6 +187,8 @@ module Relaton
163
187
  end
164
188
 
165
189
  def fetch_contributor
190
+ return jwes_contributors if wes?
191
+
166
192
  authorizer = create_contrib(
167
193
  "一般財団法人 日本規格協会", "authorizer"
168
194
  )
@@ -175,6 +201,20 @@ module Relaton
175
201
  result
176
202
  end
177
203
 
204
+ def jwes_contributors
205
+ tls = Bib::TypedLocalizedString
206
+ scripts = { "ja" => "Jpan", "en" => "Latn" }
207
+ %w[author publisher].map do |role|
208
+ name = JWES_NAMES.map do |lang, content|
209
+ tls.new content: content, language: lang, script: scripts[lang]
210
+ end
211
+ org = Bib::Organization.new name: name
212
+ Bib::Contributor.new(
213
+ organization: org, role: [Bib::Contributor::Role.new(type: role)],
214
+ )
215
+ end
216
+ end
217
+
178
218
  def create_contrib(name, role)
179
219
  org = Bib::Organization.new name: create_orgname(name)
180
220
  role_obj = Bib::Contributor::Role.new(type: role)
@@ -195,7 +235,8 @@ module Relaton
195
235
 
196
236
  def fetch_editorialgroup_contributor # rubocop:disable Metrics/MethodLength
197
237
  node = @doc.at_xpath("./table/tr[th[.='原案作成団体']]/td")
198
- @errors[:editorialgroup] &&= node.nil?
238
+ # A WES page names no drafting body (see JWES_NAMES).
239
+ @errors[:editorialgroup] &&= node.nil? unless wes?
199
240
  return unless node
200
241
 
201
242
  subdivision = Bib::Subdivision.new(
@@ -215,7 +256,7 @@ module Relaton
215
256
  def fetch_structuredidentifier
216
257
  StructuredIdentifier.new(
217
258
  project_number: Iso::ProjectNumber.new(content: fetch_docnumber),
218
- type: "JIS",
259
+ type: id_type,
219
260
  )
220
261
  end
221
262
 
@@ -12,6 +12,10 @@ module Relaton
12
12
  class DataFetcher < Core::DataFetcher
13
13
  HANDBOOKS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/handbooks.json".freeze
14
14
  TECHNICAL_REPORTS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/technical-reports.json".freeze
15
+ # The Japanese volume label of a Handbook id gets an ASCII file-name
16
+ # suffix: #file_name strips non-word characters, so 本編 and 本編+付録
17
+ # of one edition would otherwise share a file.
18
+ VOLUME_SUFFIX = { "本編+付録" => "-with-appendix", "本編" => "-main" }.freeze
15
19
 
16
20
  def index
17
21
  @index ||= Relaton::Index.find_or_create(
@@ -125,6 +129,9 @@ module Relaton
125
129
  Util.warn "File #{file} already exists, skipping.", key: id
126
130
  else
127
131
  File.write(file, serialize(item))
132
+ # Recorded for #prune_unwritten: the data repo crawls into a kept
133
+ # data/, and an unrecorded write makes every prune skip.
134
+ record_written file
128
135
  @files << file
129
136
  pid = pubid id
130
137
  if pid
@@ -152,12 +159,14 @@ module Relaton
152
159
  end
153
160
 
154
161
  def file_name(id)
155
- name = id.gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
162
+ volume = id[/ (本編(?:+付録)?)\z/, 1]
163
+ name = id.delete_suffix(" #{volume}").gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
156
164
  if id.match?(/民間活用編/)
157
165
  name += "-private"
158
166
  elsif id.match?(/公共活用編/)
159
167
  name += "-public"
160
168
  end
169
+ name += VOLUME_SUFFIX[volume] if volume
161
170
  File.join(@output, "#{name}.#{@ext}")
162
171
  end
163
172
 
@@ -23,6 +23,13 @@ module Relaton
23
23
  @edition_label ||= @version["title"].to_s[/第[\d.]+版/].to_s
24
24
  end
25
25
 
26
+ # Volume label of the version title, as Pubid::Plateau renders it:
27
+ # "本編" (main volume) or "本編+付録" (main volume and appendix). Handbook
28
+ # #00 edition 5.0 is published as both. ASCII "+" becomes "+".
29
+ def volume_label
30
+ @version["title"].to_s[/第[\d.]+版[  ]+(本編(?:[++]付録)?)/, 1]&.tr("+", "+")
31
+ end
32
+
26
33
  def slug_number
27
34
  @slug_number ||= mapped_docnumber&.[](/#(\S+)/, 1) ||
28
35
  @entry["slug"]&.to_s&.split("_")&.first
@@ -31,7 +38,7 @@ module Relaton
31
38
  def parse_docnumber
32
39
  @errors[:hb_docnumber] &&= @entry["slug"].nil? || @entry["slug"].to_s.empty?
33
40
  mapped_docnumber ||
34
- ["Handbook ##{slug_number}", edition_label].reject { |s| s.to_s.empty? }.join(" ")
41
+ ["Handbook ##{slug_number}", edition_label, volume_label].reject { |s| s.to_s.empty? }.join(" ")
35
42
  end
36
43
 
37
44
  # Part label of a split edition, e.g. "(公共活用編)". Editions 1-3 of
@@ -9,16 +9,24 @@ module Relaton
9
9
  # legacy Latin form (`1.0`), stripped in #to_all_editions to derive the
10
10
  # edition-less family id from a relaton-bib docidentifier *string* (the
11
11
  # fetched document is not a pubid). Matching itself is done on pubid objects.
12
- EDITION_SUFFIX = / (?:第[\d.]+版|\d+\.\d+)$/
12
+ # A Handbook volume label (`本編`, `本編+付録`) follows the edition, and
13
+ # goes with it.
14
+ EDITION_SUFFIX = / (?:第[\d.]+版|\d+\.\d+)(?: 本編(?:+付録)?)?$/
13
15
 
14
16
  # An edition-less reference selects every edition of the document with
15
17
  # pubid's subset match `pubid_ref === row`: the omitted edition matches
16
18
  # any value, and pubid declares `annex` strict for PLATEAU, so
17
19
  # `PLATEAU Handbook #03` does not reach `#03-1`. A reference with an
18
- # edition selects that one row with `==`.
20
+ # edition selects that one row with `==`. pubid also declares `volume`
21
+ # strict, so the edition-less match compares each row without its
22
+ # volume (`本編`, `本編+付録`): every edition includes every volume.
19
23
  def find
20
- @array = index.search(pubid_ref, exact: !all_editions?)
21
- .map { |row| Hit.new(row, self) }
24
+ rows = if all_editions?
25
+ index.search(pubid_ref) { |row| pubid_ref === without_volume(row[:id]) }
26
+ else
27
+ index.search(pubid_ref, exact: true)
28
+ end
29
+ @array = rows.map { |row| Hit.new(row, self) }
22
30
  self
23
31
  end
24
32
 
@@ -49,6 +57,13 @@ module Relaton
49
57
  @pubid_ref ||= ref.is_a?(String) ? ::Pubid::Plateau.parse(ref) : ref
50
58
  end
51
59
 
60
+ # The row's identifier as an edition-less reference sees it. `#exclude`
61
+ # returns a copy, so the cached index row is untouched. Only a Handbook
62
+ # has a volume.
63
+ def without_volume(id)
64
+ id.respond_to?(:volume) && id.volume ? id.exclude(:volume) : id
65
+ end
66
+
52
67
  # A reference with no edition (`PLATEAU Handbook #00`, or any Technical
53
68
  # Report — TRs carry no edition) asks for all editions of the document.
54
69
  def all_editions?
@@ -1,3 +1,3 @@
1
1
  module Relaton
2
- VERSION = "3.1.2".freeze
2
+ VERSION = "3.1.4".freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: relaton
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.2
4
+ version: 3.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-11-11 00:00:00.000000000 Z
11
+ date: 2026-11-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
@@ -520,14 +520,14 @@ dependencies:
520
520
  requirements:
521
521
  - - "~>"
522
522
  - !ruby/object:Gem::Version
523
- version: 2.0.9
523
+ version: 2.0.25
524
524
  type: :runtime
525
525
  prerelease: false
526
526
  version_requirements: !ruby/object:Gem::Requirement
527
527
  requirements:
528
528
  - - "~>"
529
529
  - !ruby/object:Gem::Version
530
- version: 2.0.9
530
+ version: 2.0.25
531
531
  - !ruby/object:Gem::Dependency
532
532
  name: yeptris
533
533
  requirement: !ruby/object:Gem::Requirement
@@ -654,6 +654,7 @@ files:
654
654
  - lib/relaton/bib/converter/bibxml/to_rfcxml_referencegroup_v3.rb
655
655
  - lib/relaton/bib/converter/bibxml/to_rfcxml_v3.rb
656
656
  - lib/relaton/bib/converter/csl.rb
657
+ - lib/relaton/bib/converter/csl/from_csl.rb
657
658
  - lib/relaton/bib/converter/ris.rb
658
659
  - lib/relaton/bib/converter/titles.rb
659
660
  - lib/relaton/bib/hash_parser_v1.rb
@@ -820,6 +821,7 @@ files:
820
821
  - lib/relaton/cie/util.rb
821
822
  - lib/relaton/cli.rb
822
823
  - lib/relaton/cli/base_convertor.rb
824
+ - lib/relaton/cli/collection_store.rb
823
825
  - lib/relaton/cli/command.rb
824
826
  - lib/relaton/cli/data_fetcher.rb
825
827
  - lib/relaton/cli/frontend/dist/404.html
@@ -886,6 +888,7 @@ files:
886
888
  - lib/relaton/core/request_error.rb
887
889
  - lib/relaton/core/workers_pool.rb
888
890
  - lib/relaton/db.rb
891
+ - lib/relaton/db/api_resolver.rb
889
892
  - lib/relaton/db/cache.rb
890
893
  - lib/relaton/db/cache_entry.rb
891
894
  - lib/relaton/db/config.rb