relaton 3.0.3 → 3.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/relaton/3gpp/data_fetcher.rb +3 -2
- data/lib/relaton/bib/CLAUDE.md +24 -4
- data/lib/relaton/bib/README.adoc +64 -0
- data/lib/relaton/bib/converter/bibxml/from_rfcxml.rb +12 -1
- data/lib/relaton/bib/converter/bibxml/to_rfcxml.rb +32 -8
- data/lib/relaton/bib/converter/bibxml/to_rfcxml_referencegroup.rb +10 -3
- data/lib/relaton/bib/converter/bibxml/to_rfcxml_referencegroup_v3.rb +43 -0
- data/lib/relaton/bib/converter/bibxml/to_rfcxml_v3.rb +416 -0
- data/lib/relaton/bib/converter/bibxml.rb +21 -9
- data/lib/relaton/bib/item_data.rb +19 -7
- data/lib/relaton/bib/model/abstract.rb +5 -0
- data/lib/relaton/bib/model/localized_string.rb +6 -1
- data/lib/relaton/bib/sanitizer.rb +20 -5
- data/lib/relaton/bipm/data_outcomes_parser.rb +2 -1
- data/lib/relaton/bipm/si_brochure_parser.rb +3 -2
- data/lib/relaton/core/data_fetcher.rb +3 -2
- data/lib/relaton/core/processor.rb +29 -0
- data/lib/relaton/db/CLAUDE.md +14 -2
- data/lib/relaton/db/cache.rb +5 -2
- data/lib/relaton/db.rb +7 -3
- data/lib/relaton/ecma/processor.rb +8 -0
- data/lib/relaton/equivalence.rb +2 -2
- data/lib/relaton/gb/scraper.rb +2 -2
- data/lib/relaton/ieee/data_fetcher.rb +2 -1
- data/lib/relaton/ietf/bibxml_parser.rb +6 -0
- data/lib/relaton/iho/CLAUDE.md +4 -2
- data/lib/relaton/iho/bibliography.rb +44 -13
- data/lib/relaton/iho/processor.rb +6 -0
- data/lib/relaton/index/file_io.rb +9 -8
- data/lib/relaton/index/shard_source.rb +3 -2
- data/lib/relaton/iso/hit_collection.rb +20 -3
- data/lib/relaton/jcgm/meetings_parser.rb +2 -2
- data/lib/relaton/nist/cswp.rb +2 -2
- data/lib/relaton/nist/data_fetcher.rb +3 -2
- data/lib/relaton/sdo/store.rb +3 -1
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/xsf/hit.rb +2 -1
- metadata +10 -8
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
module Relaton
|
|
2
|
+
module Bib
|
|
3
|
+
module Converter
|
|
4
|
+
module BibXml
|
|
5
|
+
#
|
|
6
|
+
# Strict RFC 7991 (xml2rfc v3) flavour of {ToRfcxml}.
|
|
7
|
+
#
|
|
8
|
+
# It differs from the BibXML output its parent produces in four ways:
|
|
9
|
+
#
|
|
10
|
+
# * the deprecated `<format>` element is never emitted;
|
|
11
|
+
# * identifiers of documents published outside the IETF go into
|
|
12
|
+
# `<refcontent>` rather than being forced into `<seriesInfo>`
|
|
13
|
+
# (RFC 7991 2.39), as do series that carry no number;
|
|
14
|
+
# * `<stream>` and the `ascii*` attributes are populated;
|
|
15
|
+
# * `<front>` is filled out enough to satisfy the v3 grammar, which
|
|
16
|
+
# requires a title and at least one author.
|
|
17
|
+
#
|
|
18
|
+
class ToRfcxmlV3 < ToRfcxml
|
|
19
|
+
# Organizations whose documents are "home" standards: their
|
|
20
|
+
# identifiers are expressible as <seriesInfo>.
|
|
21
|
+
HOME_ORGS = [
|
|
22
|
+
"IETF", "Internet Engineering Task Force", "RFC Publisher"
|
|
23
|
+
].freeze
|
|
24
|
+
|
|
25
|
+
# Identifier types that are already emitted as <seriesInfo> by the
|
|
26
|
+
# parent, or that never belong in one.
|
|
27
|
+
NON_AUTHORITATIVE_TYPES = %w[DOI URI].freeze
|
|
28
|
+
|
|
29
|
+
# Identifiers metanorma-ietf drops as internal bookkeeping.
|
|
30
|
+
IGNORED_ID_RE = /\A(rfc-anchor|Internet-Draft)/
|
|
31
|
+
|
|
32
|
+
# Series that name the document itself; handled as identifiers.
|
|
33
|
+
SUBSERIES = %w[BCP STD].freeze
|
|
34
|
+
|
|
35
|
+
HOME_ID_RE = /\A(?<name>RFC|BCP|STD|FYI)\s*(?<value>\d+)\z/i
|
|
36
|
+
# xml2rfc only resolves a draft reference when seriesInfo carries the
|
|
37
|
+
# full draft name, so keep `draft-…` intact. The `I-D.` anchor form
|
|
38
|
+
# is a citation label, not the identifier (metanorma-ietf#283).
|
|
39
|
+
DRAFT_ID_RE = /\A(?<value>draft-\S+)\z/i
|
|
40
|
+
DRAFT_ANCHOR_RE = /\AI-D[.\s]\s*\S+\z/i
|
|
41
|
+
|
|
42
|
+
# Identifier types that duplicate a human-readable identifier or are
|
|
43
|
+
# bookkeeping rather than citation content (metanorma-ietf#301).
|
|
44
|
+
EXCLUDED_ID_TYPES = %w[URN].freeze
|
|
45
|
+
|
|
46
|
+
# Title parts joined when relaton supplies no composite title.
|
|
47
|
+
TITLE_PARTS = %w[title-intro title-main title-part].freeze
|
|
48
|
+
|
|
49
|
+
# Roles that may stand in for an author when the item names none,
|
|
50
|
+
# mirroring the released renderer's `creatornames_roles_allowed`.
|
|
51
|
+
FALLBACK_ROLES = %w[performer adapter translator publisher
|
|
52
|
+
distributor authorizer].freeze
|
|
53
|
+
|
|
54
|
+
# The values RFC XML v3 allows on <stream>; "editorial" was added by
|
|
55
|
+
# RFC 9280. Keyed by the spellings the RFC Editor and relaton-ietf
|
|
56
|
+
# use, so "Independent Submission" is not silently thrown away.
|
|
57
|
+
STREAMS = {
|
|
58
|
+
"ietf" => "IETF", "iab" => "IAB", "irtf" => "IRTF",
|
|
59
|
+
"independent" => "independent",
|
|
60
|
+
"independent submission" => "independent",
|
|
61
|
+
"independent submission stream" => "independent",
|
|
62
|
+
"editorial" => "editorial", "rfc editor" => "editorial",
|
|
63
|
+
"rfc-editor" => "editorial"
|
|
64
|
+
}.freeze
|
|
65
|
+
|
|
66
|
+
# Latin letters with no NFKD decomposition, which would otherwise be
|
|
67
|
+
# dropped outright ("Sørensen" -> "Srensen").
|
|
68
|
+
TRANSLITERATIONS = {
|
|
69
|
+
"æ" => "ae", "Æ" => "AE", "œ" => "oe", "Œ" => "OE",
|
|
70
|
+
"ø" => "o", "Ø" => "O", "ß" => "ss", "ẞ" => "SS",
|
|
71
|
+
"đ" => "d", "Đ" => "D", "ð" => "d", "Ð" => "D",
|
|
72
|
+
"þ" => "th", "Þ" => "Th", "ł" => "l", "Ł" => "L",
|
|
73
|
+
"ħ" => "h", "Ħ" => "H", "ŋ" => "ng", "Ŋ" => "NG",
|
|
74
|
+
"ŧ" => "t", "Ŧ" => "T", "ı" => "i", "ĸ" => "k"
|
|
75
|
+
}.freeze
|
|
76
|
+
|
|
77
|
+
def transform
|
|
78
|
+
model = ::Rfcxml::V3::Reference.new
|
|
79
|
+
model.anchor = create_anchor
|
|
80
|
+
model.target = create_target
|
|
81
|
+
model.stream = create_stream
|
|
82
|
+
model.front = create_front
|
|
83
|
+
model.refcontent = create_refcontent
|
|
84
|
+
model
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private
|
|
88
|
+
|
|
89
|
+
# The released IETF renderer takes an HTML-typed uri as the reference
|
|
90
|
+
# target when there is no src (isodoc/ietf/references.rb), and the
|
|
91
|
+
# transformer does the same; match it here.
|
|
92
|
+
def target_types = %w[src HTML doi]
|
|
93
|
+
|
|
94
|
+
# anchor is mandatory in v3 and is an XML identifier, so fall back
|
|
95
|
+
# through every source the item has and never leave whitespace in it.
|
|
96
|
+
def create_anchor
|
|
97
|
+
sanitize_anchor(@anchor || @item.docnumber || derive_anchor ||
|
|
98
|
+
@item.id)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def sanitize_anchor(anchor)
|
|
102
|
+
return nil if anchor.nil?
|
|
103
|
+
|
|
104
|
+
anchor.to_s.strip.gsub(/\s+/, ".")
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# <front> needs a title and at least one author to be valid v3.
|
|
108
|
+
def create_front
|
|
109
|
+
front = super
|
|
110
|
+
front.title = compound_title || formattedref_title
|
|
111
|
+
front.author = [unknown_author] if Array(front.author).empty?
|
|
112
|
+
front
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# The base emitter takes title[0], which on a multipart standard is
|
|
116
|
+
# the intro alone ("IT Security techniques"). A citation needs the
|
|
117
|
+
# whole compound, so prefer the composite title relaton builds, then
|
|
118
|
+
# the intro-main-part join, then whatever is first.
|
|
119
|
+
def compound_title
|
|
120
|
+
content = composite_title || joined_title ||
|
|
121
|
+
@item.title.first&.content
|
|
122
|
+
return nil if content.nil?
|
|
123
|
+
|
|
124
|
+
::Rfcxml::V3::Title.new(content: plain_text(content))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def composite_title
|
|
128
|
+
@item.title.find { |t| t.type == "main" }&.content
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def joined_title
|
|
132
|
+
parts = TITLE_PARTS.filter_map do |type|
|
|
133
|
+
@item.title.find { |t| t.type == type }&.content
|
|
134
|
+
end
|
|
135
|
+
parts.empty? ? nil : parts.join(" - ")
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def formattedref_title
|
|
139
|
+
content = @item.formattedref&.content or return nil
|
|
140
|
+
|
|
141
|
+
::Rfcxml::V3::Title.new(content: plain_text(content))
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# v3 <title> is text-only, so inline markup a formattedref or title
|
|
145
|
+
# carries has to be flattened rather than escaped into the output.
|
|
146
|
+
def plain_text(content)
|
|
147
|
+
text = content.to_s
|
|
148
|
+
return text unless text.include?("<")
|
|
149
|
+
|
|
150
|
+
Nokogiri::XML.fragment(text).text.squeeze(" ").strip
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def unknown_author
|
|
154
|
+
::Rfcxml::V3::Author.new(surname: "Unknown")
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# One <seriesInfo> per (name, value): the same identifier can arrive
|
|
158
|
+
# from a docidentifier and a series, or from two docidentifiers that
|
|
159
|
+
# spell it differently ("RFC 2119" and "RFC2119").
|
|
160
|
+
def create_seriesinfo
|
|
161
|
+
(docidentifier_to_seriesinfo + series_to_seriesinfo +
|
|
162
|
+
identifier_to_seriesinfo).uniq { |si| [si.name, si.value] }
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Only series that carry a number can become <seriesInfo>; the rest
|
|
166
|
+
# are rendered as <refcontent>.
|
|
167
|
+
def series_to_seriesinfo
|
|
168
|
+
numbered_series.map do |ser|
|
|
169
|
+
::Rfcxml::V3::SeriesInfo.new(name: series_title(ser).content,
|
|
170
|
+
value: ser.number)
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def numbered_series
|
|
175
|
+
renderable_series.select(&:number)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Series that are neither the stream marker nor a restatement of
|
|
179
|
+
# something already rendered from the document's identifiers. Only
|
|
180
|
+
# names actually emitted elsewhere are dropped: a series such as
|
|
181
|
+
# Internet-Draft still carries its number when no docidentifier
|
|
182
|
+
# supplies one.
|
|
183
|
+
def renderable_series
|
|
184
|
+
@item.series.reject { |ser| skip_series?(ser) }
|
|
185
|
+
.uniq { |ser| series_title(ser).content }
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def skip_series?(ser)
|
|
189
|
+
return true if ser.type == "stream"
|
|
190
|
+
|
|
191
|
+
title = series_title(ser)
|
|
192
|
+
title.nil? || title.content.to_s == "DOI" ||
|
|
193
|
+
identifier_series_names.include?(title.content.to_s)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Series names already covered by <seriesInfo> or <refcontent> built
|
|
197
|
+
# from the item's identifiers.
|
|
198
|
+
def identifier_series_names
|
|
199
|
+
@identifier_series_names ||=
|
|
200
|
+
identifier_to_seriesinfo.map(&:name) +
|
|
201
|
+
subseries_identifiers.map { |id| id.split.first }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def series_title(ser)
|
|
205
|
+
ser.title.find { |title| title.content.to_s != "DOI" }
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# --- Identifiers ---
|
|
209
|
+
|
|
210
|
+
def identifier_to_seriesinfo
|
|
211
|
+
return [] unless home_standard?
|
|
212
|
+
|
|
213
|
+
authoritative_identifiers.filter_map { |id| id_to_seriesinfo(id) }
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def id_to_seriesinfo(id)
|
|
217
|
+
name, value = split_identifier(id)
|
|
218
|
+
return nil unless name
|
|
219
|
+
|
|
220
|
+
# No `stream` attribute here: in v3 the stream is carried by the
|
|
221
|
+
# <stream> element, and duplicating it only adds noise.
|
|
222
|
+
::Rfcxml::V3::SeriesInfo.new(name: name, value: value,
|
|
223
|
+
status: seriesinfo_status)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def split_identifier(id)
|
|
227
|
+
if (match = DRAFT_ID_RE.match(id))
|
|
228
|
+
["Internet-Draft", match[:value]]
|
|
229
|
+
elsif (match = HOME_ID_RE.match(id))
|
|
230
|
+
[match[:name].upcase, match[:value]]
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# An untyped docidentifier that merely restates the reference's own
|
|
235
|
+
# label ("ZELLER", "Grail") is a citation label, not a citation: keep
|
|
236
|
+
# it only when the reference would otherwise show nothing at all.
|
|
237
|
+
def label_echo?(docid)
|
|
238
|
+
return false unless docid.type.nil?
|
|
239
|
+
|
|
240
|
+
docid.content.to_s.casecmp(label_candidates.to_s).zero? &&
|
|
241
|
+
reference_has_visible_text?
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def label_candidates
|
|
245
|
+
@item.docnumber || @item.id
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def reference_has_visible_text?
|
|
249
|
+
@item.title.any? || @item.formattedref ||
|
|
250
|
+
@item.contributor.any? || @item.date.any?
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def seriesinfo_status
|
|
254
|
+
@item.status&.stage&.content
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def authoritative_identifiers
|
|
258
|
+
ids = subseries_identifiers + docidentifier_identifiers
|
|
259
|
+
ids.reject { |id| id.empty? || IGNORED_ID_RE.match?(id) }.uniq
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def docidentifier_identifiers
|
|
263
|
+
@item.docidentifier.reject { |di| non_authoritative?(di) }
|
|
264
|
+
.map { |di| di.content.to_s.strip }
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def non_authoritative?(docid)
|
|
268
|
+
NON_AUTHORITATIVE_TYPES.include?(docid.type) ||
|
|
269
|
+
EXCLUDED_ID_TYPES.include?(docid.type) ||
|
|
270
|
+
docid.scope == "trademark" ||
|
|
271
|
+
DRAFT_ANCHOR_RE.match?(docid.content.to_s) ||
|
|
272
|
+
label_echo?(docid)
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def subseries_identifiers
|
|
276
|
+
@item.series.filter_map do |ser|
|
|
277
|
+
title = ser.title.find { |t| SUBSERIES.include?(t.content.to_s) }
|
|
278
|
+
"#{title.content} #{ser.number}" if title && ser.number
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def home_standard?
|
|
283
|
+
@item.contributor.any? do |contrib|
|
|
284
|
+
org = contrib.organization or next false
|
|
285
|
+
|
|
286
|
+
([org.abbreviation&.content] + org.name.map(&:content))
|
|
287
|
+
.compact.any? { |name| HOME_ORGS.include?(name.to_s) }
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# --- refcontent ---
|
|
292
|
+
|
|
293
|
+
def create_refcontent
|
|
294
|
+
(identifier_refcontent + series_refcontent)
|
|
295
|
+
.map { |text| ::Rfcxml::V3::Refcontent.new(content: text) }
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# Identifiers that cannot be expressed as <seriesInfo> — either
|
|
299
|
+
# because the document is not an IETF one, or because the identifier
|
|
300
|
+
# does not split into a series name and number.
|
|
301
|
+
# Whatever <seriesInfo> already states must not be repeated here,
|
|
302
|
+
# whichever route put it there.
|
|
303
|
+
def identifier_refcontent
|
|
304
|
+
ids = authoritative_identifiers.reject { |id| in_seriesinfo?(id) }
|
|
305
|
+
ids.empty? ? [] : [ids.join(", ")]
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def in_seriesinfo?(id)
|
|
309
|
+
name, value = split_identifier(id)
|
|
310
|
+
return false if name.nil?
|
|
311
|
+
|
|
312
|
+
create_seriesinfo.any? { |si| si.name == name && si.value == value }
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def series_refcontent
|
|
316
|
+
(renderable_series - numbered_series).filter_map do |ser|
|
|
317
|
+
text = ser.formattedref&.content&.to_s || series_description(ser)
|
|
318
|
+
text unless text.strip.empty?
|
|
319
|
+
end
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
def series_description(ser)
|
|
323
|
+
[series_title(ser)&.content, ser.run, ser.organization,
|
|
324
|
+
ser.place&.city, series_dates(ser)]
|
|
325
|
+
.map(&:to_s).reject(&:empty?).join(", ")
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
def series_dates(ser)
|
|
329
|
+
[ser.from, ser.to].compact.map(&:to_s).reject(&:empty?).join("-")
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
# --- stream ---
|
|
333
|
+
|
|
334
|
+
# Values outside the grammar (relaton-ietf also has "Legacy") have no
|
|
335
|
+
# valid representation, so they are dropped with a warning rather
|
|
336
|
+
# than emitted as invalid XML.
|
|
337
|
+
def create_stream
|
|
338
|
+
value = ext_stream || series_stream
|
|
339
|
+
return nil if value.nil?
|
|
340
|
+
|
|
341
|
+
stream = STREAMS[value.to_s.strip.downcase]
|
|
342
|
+
unless stream
|
|
343
|
+
Util.warn "Dropping stream `#{value}`: not an RFC XML v3 stream"
|
|
344
|
+
end
|
|
345
|
+
stream
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
def ext_stream
|
|
349
|
+
@item.ext.respond_to?(:stream) ? @item.ext.stream : nil
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
def series_stream
|
|
353
|
+
ser = @item.series.find { |s| s.type == "stream" }
|
|
354
|
+
ser && ser.title.first&.content
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# --- Contributors ---
|
|
358
|
+
|
|
359
|
+
# Every relaton-ietf RFC record carries publisher and authorizer
|
|
360
|
+
# contributors, and published RFC XML never lists those as authors.
|
|
361
|
+
# So: authors and editors if there are any, and only otherwise fall
|
|
362
|
+
# back to the wider set, which keeps a translator-only monograph its
|
|
363
|
+
# translator (metanorma-ietf#301).
|
|
364
|
+
def contributors_for_authors
|
|
365
|
+
contribs = super
|
|
366
|
+
primary = contribs.select { |c| role?(c, %w[author editor]) }
|
|
367
|
+
primary.any? ? primary : contribs.select { |c| fallback_author?(c) }
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
def role?(contrib, types)
|
|
371
|
+
contrib.role.any? { |r| types.include?(r.type) }
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
# A contributor with no role at all is an author by default; that is
|
|
375
|
+
# how most non-IETF bibitems are written.
|
|
376
|
+
def fallback_author?(contrib)
|
|
377
|
+
contrib.role.empty? || role?(contrib, FALLBACK_ROLES)
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# --- ascii folding ---
|
|
381
|
+
|
|
382
|
+
def create_authors
|
|
383
|
+
super.each do |author|
|
|
384
|
+
author.ascii_fullname = ascii(author.fullname)
|
|
385
|
+
author.ascii_surname = ascii(author.surname)
|
|
386
|
+
author.ascii_initials = ascii(author.initials)
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def create_organization(contrib)
|
|
391
|
+
org = super or return nil
|
|
392
|
+
|
|
393
|
+
org.ascii = ascii(Array(org.content).join)
|
|
394
|
+
org
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
# Transliterate to bare ASCII: expand the letters above, then strip
|
|
398
|
+
# the combining marks NFKD leaves behind. "Nürk" -> "Nurk",
|
|
399
|
+
# "Ö." -> "O.", "Sørensen" -> "Sorensen".
|
|
400
|
+
def ascii(str)
|
|
401
|
+
return nil if str.nil?
|
|
402
|
+
|
|
403
|
+
folded = str.to_s.gsub(/[#{TRANSLITERATIONS.keys.join}]/,
|
|
404
|
+
TRANSLITERATIONS)
|
|
405
|
+
.unicode_normalize(:nfkd)
|
|
406
|
+
.encode("ASCII", invalid: :replace, undef: :replace, replace: "")
|
|
407
|
+
# Folding a non-Latin script leaves only the punctuation and
|
|
408
|
+
# spacing behind ("Νίκος Παπαδόπουλος" -> " "), which is worse than
|
|
409
|
+
# no attribute at all. Real transliteration is the caller's policy.
|
|
410
|
+
folded.match?(/[[:alnum:]]/) ? folded : nil
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
end
|
|
414
|
+
end
|
|
415
|
+
end
|
|
416
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
require_relative "bibxml/to_rfcxml"
|
|
2
2
|
require_relative "bibxml/to_rfcxml_referencegroup"
|
|
3
|
+
require_relative "bibxml/to_rfcxml_v3"
|
|
4
|
+
require_relative "bibxml/to_rfcxml_referencegroup_v3"
|
|
3
5
|
require_relative "bibxml/from_rfcxml"
|
|
4
6
|
require_relative "bibxml/from_rfcxml_referencegroup"
|
|
5
7
|
|
|
@@ -16,17 +18,27 @@ module Relaton
|
|
|
16
18
|
RFCPREFIXES = %w[RFC BCP FYI STD].freeze
|
|
17
19
|
|
|
18
20
|
# Forward: ItemData -> Rfcxml model
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
21
|
+
#
|
|
22
|
+
# @param v3 [Boolean] emit strict RFC 7991 (xml2rfc v3) instead of the
|
|
23
|
+
# BibXML shape the relaton data fetchers publish
|
|
24
|
+
# @param anchor [String, nil] anchor to use instead of the one derived
|
|
25
|
+
# from the item's docnumber/docidentifier
|
|
26
|
+
def self.from_item(item, include_keywords: true, v3: false, anchor: nil) # rubocop:disable Naming/MethodParameterName
|
|
27
|
+
klass = if v3 then v3_converter(item)
|
|
28
|
+
elsif bcp?(item) then ToRfcxmlReferencegroup
|
|
29
|
+
else ToRfcxml
|
|
30
|
+
end
|
|
31
|
+
klass.new(item, include_keywords: include_keywords,
|
|
32
|
+
anchor: anchor).transform
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.v3_converter(item)
|
|
36
|
+
if item.relation.any? { |rel| rel.type == "includes" }
|
|
37
|
+
ToRfcxmlReferencegroupV3
|
|
38
|
+
else ToRfcxmlV3
|
|
28
39
|
end
|
|
29
40
|
end
|
|
41
|
+
private_class_method :v3_converter
|
|
30
42
|
|
|
31
43
|
def self.bcp?(item) # rubocop:disable Metrics/CyclomaticComplexity
|
|
32
44
|
item.docnumber&.match(/^BCP/) ||
|
|
@@ -19,8 +19,18 @@ module Relaton
|
|
|
19
19
|
COLLECTION_WRITE_ONLY_ATTRIBUTES = %i[title abstract source relation].freeze
|
|
20
20
|
|
|
21
21
|
ATTRIBUTES.each { |attr| attr_accessor attr }
|
|
22
|
-
COLLECTION_ATTRBUTES.each { |attr|
|
|
23
|
-
|
|
22
|
+
COLLECTION_ATTRBUTES.each { |attr| attr_reader attr }
|
|
23
|
+
|
|
24
|
+
# Collections are always arrays. lutaml-model assigns every mapped
|
|
25
|
+
# attribute after building the instance, so an element absent from the
|
|
26
|
+
# XML would otherwise overwrite the [] set in #initialize with nil --
|
|
27
|
+
# `Item.from_xml` items then blew up in any consumer that iterated them,
|
|
28
|
+
# while `from_yaml` ones did not.
|
|
29
|
+
(COLLECTION_ATTRBUTES + COLLECTION_WRITE_ONLY_ATTRIBUTES).each do |attr|
|
|
30
|
+
define_method("#{attr}=") do |value|
|
|
31
|
+
instance_variable_set("@#{attr}", array(value))
|
|
32
|
+
end
|
|
33
|
+
end
|
|
24
34
|
|
|
25
35
|
attr_reader :id, :docidentifier
|
|
26
36
|
|
|
@@ -28,10 +38,10 @@ module Relaton
|
|
|
28
38
|
ATTRIBUTES.each { |attr| instance_variable_set("@#{attr}", args[attr]) }
|
|
29
39
|
|
|
30
40
|
(COLLECTION_ATTRBUTES + COLLECTION_WRITE_ONLY_ATTRIBUTES).each do |attr|
|
|
31
|
-
instance_variable_set("@#{attr}", args[attr]
|
|
41
|
+
instance_variable_set("@#{attr}", array(args[attr]))
|
|
32
42
|
end
|
|
33
43
|
|
|
34
|
-
@docidentifier = args[:docidentifier]
|
|
44
|
+
@docidentifier = array(args[:docidentifier])
|
|
35
45
|
self.schema_version = schema
|
|
36
46
|
@id = args[:id]
|
|
37
47
|
create_id
|
|
@@ -42,7 +52,7 @@ module Relaton
|
|
|
42
52
|
end
|
|
43
53
|
|
|
44
54
|
def docidentifier=(value)
|
|
45
|
-
@docidentifier = value
|
|
55
|
+
@docidentifier = array(value)
|
|
46
56
|
create_id
|
|
47
57
|
end
|
|
48
58
|
|
|
@@ -179,8 +189,10 @@ module Relaton
|
|
|
179
189
|
Converter::Asciibib.from_item(self)
|
|
180
190
|
end
|
|
181
191
|
|
|
182
|
-
|
|
183
|
-
|
|
192
|
+
# @param opts [Hash] passed to {Converter::BibXml.from_item}; notably
|
|
193
|
+
# `v3: true` for strict RFC 7991 output instead of BibXML.
|
|
194
|
+
def to_rfcxml(**)
|
|
195
|
+
Converter::BibXml.from_item(self, **).to_xml
|
|
184
196
|
end
|
|
185
197
|
|
|
186
198
|
def deep_clone
|
|
@@ -28,10 +28,15 @@ module Relaton
|
|
|
28
28
|
class LocalizedMarkedUpString < LocalizedStringAttrs
|
|
29
29
|
module ContentSanitization
|
|
30
30
|
def content=(value)
|
|
31
|
-
super(
|
|
31
|
+
super(Sanitizer.sanitize(value, allowed: sanitizer_allowed))
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
# The elements that content keeps; Abstract widens it to blocks.
|
|
36
|
+
def sanitizer_allowed
|
|
37
|
+
Relaton::Bib::Sanitizer::ALLOWED
|
|
38
|
+
end
|
|
39
|
+
|
|
35
40
|
# Cast at the assignment layer (lutaml-model#972): a bare scalar
|
|
36
41
|
# assigned to a singular custom-typed attribute (e.g. a relation's
|
|
37
42
|
# description) is wrapped here, so the serializer's type validation
|
|
@@ -24,12 +24,27 @@ module Relaton
|
|
|
24
24
|
# markup, and must be preserved verbatim. Without the opaque-skip,
|
|
25
25
|
# the recursive walk would unwrap MathML / AsciiMath elements down to
|
|
26
26
|
# bare text nodes — see #116 for the round-trip-loss symptom.
|
|
27
|
+
#
|
|
28
|
+
# ALLOWED_WITH_BLOCKS adds the basicdoc block elements: lists, notes,
|
|
29
|
+
# tables, figures and admonitions, with the children that their grammar
|
|
30
|
+
# requires (a caption <name>, <thead>/<tbody>/<tfoot>, a figure's
|
|
31
|
+
# <image>). Only Abstract asks for it, because <abstract> is the one
|
|
32
|
+
# field whose grammar admits blocks; a title or a note admits only
|
|
33
|
+
# PureTextElement. Before relaton#421 an abstract's list or note
|
|
34
|
+
# unwrapped in place and came back as bare paragraphs, with no error.
|
|
27
35
|
module Sanitizer
|
|
28
36
|
ALLOWED = %w[
|
|
29
37
|
em strong sub sup tt underline strike smallcap br stem
|
|
30
38
|
p eref xref fn link
|
|
31
39
|
].freeze
|
|
32
40
|
|
|
41
|
+
BLOCK = %w[
|
|
42
|
+
ul ol li dl dt dd note admonition
|
|
43
|
+
table thead tbody tfoot tr td th figure name image
|
|
44
|
+
].freeze
|
|
45
|
+
|
|
46
|
+
ALLOWED_WITH_BLOCKS = (ALLOWED + BLOCK).freeze
|
|
47
|
+
|
|
33
48
|
# Elements whose children are non-basicdoc inline notation
|
|
34
49
|
# (MathML, AsciiMath, LaTeX, …) and must be preserved verbatim
|
|
35
50
|
# rather than sanitised against ALLOWED.
|
|
@@ -63,13 +78,13 @@ module Relaton
|
|
|
63
78
|
# Reserved prefixes. XML declares both, so the content must not.
|
|
64
79
|
NS_RESERVED = %w[xml xmlns].freeze
|
|
65
80
|
|
|
66
|
-
def self.sanitize(content)
|
|
81
|
+
def self.sanitize(content, allowed: ALLOWED)
|
|
67
82
|
return content unless sanitizable?(content)
|
|
68
83
|
|
|
69
84
|
node = parse(content)
|
|
70
85
|
return content if node.nil?
|
|
71
86
|
|
|
72
|
-
sanitize_children(node)
|
|
87
|
+
sanitize_children(node, allowed)
|
|
73
88
|
node.children.map { |c| c.to_xml(encoding: "UTF-8", indent: 0, expand_empty: false) }.join
|
|
74
89
|
end
|
|
75
90
|
|
|
@@ -273,15 +288,15 @@ module Relaton
|
|
|
273
288
|
end
|
|
274
289
|
private_class_method :sanitizable?
|
|
275
290
|
|
|
276
|
-
def self.sanitize_children(node)
|
|
291
|
+
def self.sanitize_children(node, allowed)
|
|
277
292
|
node.children.to_a.each do |child|
|
|
278
293
|
next unless child.element?
|
|
279
294
|
|
|
280
295
|
child.name = RENAME[child.name] if RENAME.key?(child.name)
|
|
281
296
|
next if OPAQUE.include?(child.name)
|
|
282
297
|
|
|
283
|
-
sanitize_children(child)
|
|
284
|
-
unwrap(child) unless
|
|
298
|
+
sanitize_children(child, allowed)
|
|
299
|
+
unwrap(child) unless allowed.include?(child.name)
|
|
285
300
|
end
|
|
286
301
|
end
|
|
287
302
|
private_class_method :sanitize_children
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
require "yeptris/psych"
|
|
1
2
|
module Relaton::Bipm
|
|
2
3
|
class DataOutcomesParser
|
|
3
4
|
SHORTTYPE = {
|
|
@@ -130,7 +131,7 @@ module Relaton::Bipm
|
|
|
130
131
|
fr_file = en_file.sub "en", "fr"
|
|
131
132
|
[en_file, fr_file].map do |file|
|
|
132
133
|
if File.exist? file
|
|
133
|
-
data =
|
|
134
|
+
data = Yeptris::Psych.safe_load(File.read(file, encoding: "UTF-8"))
|
|
134
135
|
path = file
|
|
135
136
|
end
|
|
136
137
|
[path, data]
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
require "yeptris/psych"
|
|
1
2
|
require "moxml"
|
|
2
3
|
|
|
3
4
|
module Relaton::Bipm
|
|
@@ -60,8 +61,8 @@ module Relaton::Bipm
|
|
|
60
61
|
warn_duplicate = false
|
|
61
62
|
item2 = Item.from_yaml File.read(outfile, encoding: "UTF-8")
|
|
62
63
|
fix_si_brochure_id item2
|
|
63
|
-
hash1 =
|
|
64
|
-
hash2 =
|
|
64
|
+
hash1 = Yeptris::Psych.safe_load item1.to_yaml
|
|
65
|
+
hash2 = Yeptris::Psych.safe_load item2.to_yaml
|
|
65
66
|
Item.from_yaml deep_merge(hash1, hash2).to_yaml
|
|
66
67
|
else
|
|
67
68
|
warn_duplicate = true
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
require "yeptris/psych"
|
|
1
2
|
require "yeptris"
|
|
2
3
|
require "fileutils"
|
|
3
4
|
|
|
@@ -121,7 +122,7 @@ module Relaton
|
|
|
121
122
|
kept = ledger.merge(ledger_new).select do |_, row|
|
|
122
123
|
row.is_a?(Hash) && row["file"] && @written.include?(row["file"])
|
|
123
124
|
end
|
|
124
|
-
File.write ledger_path,
|
|
125
|
+
File.write ledger_path, Yeptris::Psych.dump(kept), encoding: "UTF-8"
|
|
125
126
|
@ledger = kept
|
|
126
127
|
end
|
|
127
128
|
|
|
@@ -349,7 +350,7 @@ module Relaton
|
|
|
349
350
|
|
|
350
351
|
def ledger
|
|
351
352
|
@ledger ||= if File.exist?(ledger_path)
|
|
352
|
-
|
|
353
|
+
Yeptris::Psych.safe_load_file(ledger_path) || {}
|
|
353
354
|
else
|
|
354
355
|
{}
|
|
355
356
|
end
|