metanorma 2.5.23 → 2.5.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b10fe30704e90dd64f266eccf4924730880bbd373d4edc3abcf14680e9d7bf65
4
- data.tar.gz: e558651f1522eb3f278a97da5c9a3884fe037dc9e67b6d863a3365df097a5eb4
3
+ metadata.gz: cb5842a8eea875101cbaf77d25d62d9fa5ca74f15ed8dcc69273108a95335998
4
+ data.tar.gz: 5042b060a3d433bf0852e1a2dbad3a3b44201438ce875dc7bdb30af98c79eaa3
5
5
  SHA512:
6
- metadata.gz: fd160507ec2c4095036e73fc0e59d3fddb3d904c9f3d02d583a19739591184ef6df32fb234952f516498ef0cd7e8da4b757b3f6bb416134f4657dd7e054063b8
7
- data.tar.gz: 3bd440249de25476c58639323a600100832a349d46bb37fcfed5b11c9a6db8495e7a26c95ed2ed5be074f986a8e0a5c2b6a7ba770684bc7532a4b32d727e1903
6
+ metadata.gz: ff6f596ef437df009705e085713692e51fee114676e4227d912bd7556155b1b8efbdbebb08e51d3f15b1abc1ca6f635f72abc6c245138f4077c1390523583627
7
+ data.tar.gz: 5f2d5554445a316746c08401e5ebc071e4f22132f7f252b225068bc603aff0cf014b69738b11f5341520683fab7d445b2cef7dc7af3f348d4d0d0042160ddb87
data/Gemfile.devel CHANGED
@@ -15,4 +15,4 @@ gem "lutaml-model", "~> 0.8.107"
15
15
  gem "metanorma-plugin-glossarist", "~> 0.5"
16
16
  gem "moxml", "~> 0.6.0"
17
17
 
18
- gem "parsanol", "~> 1.3.129" # relaton 3.1 floor; pubid edition-2 grammar era
18
+ gem "parsanol", "~> 1.5" # relaton 3.1.6+ string-contract parse_xml era
@@ -1,9 +1,9 @@
1
1
  require "relaton"
2
2
  require "relaton/bib"
3
- require "relaton/cli"
4
3
  require "metanorma-utils"
5
4
  require_relative "util/util"
6
5
  require_relative "util/disambig_files"
6
+ require_relative "config/bibdata_parser"
7
7
  require_relative "config/config"
8
8
  require_relative "config/manifest"
9
9
  require_relative "helpers"
@@ -0,0 +1,136 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "moxml"
4
+ require "relaton"
5
+ require "relaton/bib"
6
+
7
+ module Metanorma
8
+ class Collection
9
+ module Config
10
+ # Collection bibdata ingestion through relaton's library surface.
11
+ #
12
+ # Mirrors the flavor dispatch of the relaton Cli entry points
13
+ # (cli.rb parse_xml, YAMLConvertor convert_single_file): the
14
+ # processor is named by the first docidentifier's type, else by the
15
+ # leading word of its text. The long-term home for this logic is a
16
+ # relaton library-level API (follow-up issue, not filed from here);
17
+ # metanorma must not call the user-facing Cli layer.
18
+ module BibdataParser
19
+ class << self
20
+ # @param xml [String] serialized bibdata or bibitem XML
21
+ # @return [Relaton::Bib::ItemData, nil] nil when the source is not
22
+ # parseable XML
23
+ def from_xml(xml)
24
+ return unless xml
25
+
26
+ doc = Moxml.new.parse(xml)
27
+ plain = without_namespaces(doc)
28
+ if (processor = processor_for_xml(doc))
29
+ processor.from_xml(plain)
30
+ else
31
+ # No-flavor fallback: unparseable sources degrade to a nil
32
+ # bibdata, as in the mirrored Cli method.
33
+ Relaton::Bib::Item.from_xml(plain) rescue nil
34
+ end
35
+ rescue Moxml::ParseError
36
+ nil
37
+ end
38
+
39
+ # @param content [Hash] bibdata in YAML form
40
+ # @return [Relaton::Bib::ItemData]
41
+ def from_yaml(content)
42
+ yaml = content.to_yaml
43
+ if (processor = processor_for_yaml(content))
44
+ begin
45
+ processor.from_yaml(yaml)
46
+ rescue RuntimeError
47
+ Relaton::Bib::Item.from_yaml(yaml)
48
+ end
49
+ else
50
+ Relaton::Bib::Item.from_yaml(yaml)
51
+ end
52
+ end
53
+
54
+ private
55
+
56
+ # The processor named by the first docidentifier's type, else by
57
+ # the first word of its text.
58
+ def processor_for_xml(doc)
59
+ docid = doc.xpath("//*[local-name()='docidentifier']").first
60
+ proc = get_proc(docid)
61
+ return proc if proc
62
+
63
+ Relaton::Db::Registry.instance
64
+ .by_type(docid&.text&.match(/^\w+/)&.to_s)
65
+ end
66
+
67
+ def get_proc(docid)
68
+ type = docid&.[]("type")
69
+ return unless type
70
+
71
+ Relaton::Db::Registry.instance.by_type(type)
72
+ end
73
+
74
+ # The processor named by `ext.flavor`, else by the first
75
+ # docidentifier's type / leading word of its content.
76
+ def processor_for_yaml(content)
77
+ flavor = content.dig("ext", "flavor") ||
78
+ yaml_doctype(content["docidentifier"])
79
+ return unless flavor
80
+
81
+ Relaton::Db::Registry.instance.by_type(flavor)
82
+ end
83
+
84
+ def yaml_doctype(docid)
85
+ did = docid.is_a?(Array) ? docid.fetch(0) : docid
86
+ return unless did
87
+
88
+ did["type"] || did.fetch("content")&.match(/^\w+/)&.to_s
89
+ end
90
+
91
+ # The root element as XML, with every element and attribute in no
92
+ # namespace (`xml:` attributes keep their prefix). The lutaml-model
93
+ # mappings are namespace-free: they skip a prefixed element, and a
94
+ # nested `relation/bibitem` in a default namespace raises in
95
+ # lutaml-model 0.8.175. Moxml cannot drop a namespace in place, so
96
+ # this builds a copy. (Mirror of relaton cli.rb without_namespaces.)
97
+ def without_namespaces(doc)
98
+ plain = Moxml.new.create_document
99
+ plain.add_child plain_element(plain, doc.root)
100
+ plain.root.to_xml
101
+ end
102
+
103
+ def plain_element(doc, src)
104
+ elm = doc.create_element(src.name)
105
+ # An attribute in no namespace wins over a prefixed one of the
106
+ # same local name (`type` over `a:type`).
107
+ src.attributes.sort_by { |a| a.namespace ? 0 : 1 }.each do |attr|
108
+ elm[plain_name(attr)] = attr.value
109
+ end
110
+ src.children.each { |child| elm.add_child plain_node(doc, child) }
111
+ elm
112
+ end
113
+
114
+ # The local name; an `xml:` attribute keeps its prefix.
115
+ def plain_name(attr)
116
+ attr.namespace&.prefix == "xml" ? "xml:#{attr.name}" : attr.name
117
+ end
118
+
119
+ # An entity reference (`&nbsp;`) stays as its text: lutaml-model
120
+ # cannot read an entity it has no declaration for. A processing
121
+ # instruction becomes an empty text.
122
+ def plain_node(doc, src)
123
+ case src
124
+ when Moxml::Element then plain_element(doc, src)
125
+ when Moxml::Cdata then doc.create_cdata(src.content)
126
+ when Moxml::Text then doc.create_text(src.content)
127
+ when Moxml::Comment then doc.create_comment(src.content)
128
+ when Moxml::EntityReference then doc.create_text("&#{src.name};")
129
+ else doc.create_text("")
130
+ end
131
+ end
132
+ end
133
+ end
134
+ end
135
+ end
136
+ end
@@ -67,8 +67,9 @@ module Metanorma
67
67
 
68
68
  def bibdata_from_xml(model, node)
69
69
  node or return
70
- force_primary_docidentifier_xml(node.adapter_node.native)
71
- model.bibdata = Relaton::Cli.parse_xml(node.adapter_node.native)
70
+ native = node.adapter_node.native
71
+ force_primary_docidentifier_xml(native)
72
+ model.bibdata = BibdataParser.from_xml(native.to_xml)
72
73
  end
73
74
 
74
75
  def force_primary_docidentifier_xml(node)
@@ -1,6 +1,6 @@
1
1
  require "relaton"
2
2
  require "relaton/bib"
3
- require "relaton/cli"
3
+ require_relative "../config/bibdata_parser"
4
4
 
5
5
  module Metanorma
6
6
  class Collection
@@ -136,8 +136,9 @@ module Metanorma
136
136
  from_xml (Nokogiri::XML(File.read(file, encoding: "UTF-8"),
137
137
  &:huge))
138
138
  when :yaml
139
- yaml = File.read(file, encoding: "UTF-8")
140
- Relaton::Cli::YAMLConvertor.convert_single_file(yaml)
139
+ Config::BibdataParser.from_yaml(
140
+ YAML.safe_load_file(file, permitted_classes: [Date, Symbol]),
141
+ )
141
142
  end
142
143
  end
143
144
  end
@@ -3,20 +3,22 @@
3
3
  module Metanorma
4
4
  class Collection
5
5
  class Renderer
6
- def file_compile_formats(filename, identifier, opts)
6
+ def file_compile_formats(filename, identifier, opts, processor)
7
7
  f = @files.get(identifier, :outputs)
8
8
  format = opts[:extension_keys]
9
9
  concatenate_presentation?({ format: }) and format << :presentation
10
10
  format.each do |e|
11
11
  e == :pdf and output_filename = opts[:pdffile]
12
- file_compile_format(filename, identifier, e, f, output_filename)
12
+ file_compile_format(filename, identifier, e, f,
13
+ new_output_fname: output_filename,
14
+ processor: processor)
13
15
  end
14
16
  @files.set(identifier, :outputs, f)
15
17
  end
16
18
 
17
19
  # Generate output filename with correct extension
18
- def output_filename_with_extension(fname, format)
19
- ext = @compile.processor.output_formats[format]
20
+ def output_filename_with_extension(fname, format, processor)
21
+ ext = processor.output_formats[format]
20
22
  fname_base = File.basename(fname)
21
23
  # ONLY replace .xml or .html extensions, NEVER remove section numbers
22
24
  if /\.(xml|html)$/.match?(fname_base)
@@ -41,7 +43,7 @@ module Metanorma
41
43
 
42
44
  # Apply custom filename pattern with substitutions
43
45
  def apply_custom_filename_pattern(ident, custom_fname, output_fname,
44
- format)
46
+ ext)
45
47
  idx = @files.get(ident, :idx)
46
48
  original_file = @files.get(ident, :ref)
47
49
  basename = File.basename(original_file, ".*")
@@ -53,30 +55,34 @@ format)
53
55
  if File.dirname(custom_fname) == "."
54
56
  output_fname
55
57
  else
56
- preserve_output_dir_structure(custom_fname, output_fname,
57
- format, explicit_custom: true)
58
+ preserve_output_dir_structure(custom_fname, output_fname, ext,
59
+ explicit_custom: true)
58
60
  end
59
61
  end
60
62
 
61
63
  # Apply directory preservation for files with directory structure
62
- def apply_directory_preservation(fname, output_fname, format)
64
+ def apply_directory_preservation(fname, output_fname, ext)
63
65
  fname_dir = File.dirname(fname)
64
66
  output_fname = File.join(fname_dir, output_fname)
65
- preserve_output_dir_structure(fname, output_fname, format)
67
+ preserve_output_dir_structure(fname, output_fname, ext)
66
68
  end
67
69
 
68
- # Compile single format for a file and update outputs hash
69
- def file_compile_format(fname, ident, format, outputs, new_output_fname)
70
- ext = @compile.processor.output_formats[format]
71
- output_fname = output_filename_with_extension(fname, format)
70
+ # Compile single format for a file and update outputs hash; opts
71
+ # carries the custom pdf filename (:new_output_fname) and the flavour
72
+ # :processor resolving the format-to-extension mapping.
73
+ def file_compile_format(fname, ident, format, outputs, opts)
74
+ ext = opts[:processor].output_formats[format]
75
+ output_fname = output_filename_with_extension(fname, format,
76
+ opts[:processor])
72
77
  output_fname = handle_new_output_filename(output_fname,
73
- new_output_fname)
74
- if !new_output_fname && (custom = @files.preserve_directory_structure?(ident))
78
+ opts[:new_output_fname])
79
+ if !opts[:new_output_fname] &&
80
+ (custom = @files.preserve_directory_structure?(ident))
75
81
  output_fname = apply_custom_filename_pattern(ident, custom,
76
- output_fname, format)
77
- elsif !new_output_fname && File.dirname(fname) != "."
82
+ output_fname, ext)
83
+ elsif !opts[:new_output_fname] && File.dirname(fname) != "."
78
84
  output_fname = apply_directory_preservation(fname, output_fname,
79
- format)
85
+ ext)
80
86
  end
81
87
  should_skip = /html$/.match?(ext) && @files.get(ident, :sectionsplit)
82
88
  should_skip or outputs[format] = File.join(@outdir, output_fname)
@@ -126,14 +132,14 @@ format)
126
132
  "#{custom_basename}#{ext}"
127
133
  end
128
134
 
129
- # Preserve directory structure from input filename in output
130
- def preserve_output_dir_structure(fname, output_fname, format = nil,
135
+ # Preserve directory structure from input filename in output; ext is
136
+ # the output extension string the caller resolved for its format.
137
+ def preserve_output_dir_structure(fname, output_fname, ext = "",
131
138
  explicit_custom: false)
132
139
  fname_dir = File.dirname(fname)
133
140
  return output_fname if fname_dir == "."
134
141
 
135
142
  output_basename = File.basename(output_fname)
136
- ext = format ? @compile.processor.output_formats[format].to_s : ""
137
143
  should_move = should_move_to_subdirectory?(ext, output_basename,
138
144
  explicit_custom)
139
145
  if should_move
@@ -29,8 +29,11 @@ module Metanorma
29
29
  Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
30
30
  @isodoc, sso)
31
31
  # post (semantic only): intra-doc xref/eref turned into internal erefs;
32
- # repo bibitems copied; indirect bibliography inserted.
33
- @ncnames = {}
32
+ # repo bibitems copied; indirect bibliography inserted. The
33
+ # indirect-reference state is a per-member memo (not renderer
34
+ # instance state): members run concurrently on the render pool, and
35
+ # a shared memo would let one member's reset drop another member's
36
+ # already-updated-anchor guards mid-pass.
34
37
  @nested or update_indirect_refs_to_docs(xml, docid, internal_refs, sso)
35
38
  # post: bibitem[@type='internal'] anchors resolved to the containing
36
39
  # collection document.
@@ -191,58 +194,72 @@ presxml)
191
194
  # Resolve erefs to a container of ids in another doc,
192
195
  # to an anchor eref (direct link)
193
196
  def update_indirect_refs_to_docs(docxml, _docid, internal_refs, presxml)
194
- bib, erefs, doc_suffix, doc_type, f = update_indirect_refs_prep(docxml,
195
- presxml)
197
+ f, memo = update_indirect_refs_prep(docxml, presxml)
196
198
  internal_refs.each do |schema, ids|
197
- add_suffix = doc_suffix && doc_type && doc_type != schema
199
+ add_suffix = memo[:doc_suffix] && memo[:doc_type] &&
200
+ memo[:doc_type] != schema
198
201
  ids.each do |id, file|
199
202
  f[file] ||= { url: @files.url?(file),
200
203
  parentid: @files.get(file) && @files.get(file,
201
204
  :parentid) }
202
- k = indirect_ref_key(schema, id, doc_suffix, add_suffix)
203
- update_indirect_refs_to_docs1(f[file], k, file, bib, erefs)
205
+ k = indirect_ref_key(memo, schema, id, add_suffix)
206
+ update_indirect_refs_to_docs1(f[file], k, file, memo)
204
207
  end
205
208
  end
206
209
  end
207
210
 
211
+ # Prepares the per-member indirect-reference pass: the file container
212
+ # plus a memo hash carrying the pre-gathered bibitems and erefs, the
213
+ # document suffix/type, the schema/id key cache, and the
214
+ # already-updated-anchor guards. The memo is a local rather than
215
+ # renderer instance state, so members on the render pool cannot reset
216
+ # each other's state mid-pass.
208
217
  def update_indirect_refs_prep(docxml, presxml)
209
- @updated_anchors = {}
210
- @indirect_keys = {}
211
- [Util::gather_bibitems(docxml), Util::gather_bibitemids(docxml, presxml),
212
- docxml.root["document_suffix"], docxml.root["type"], {}]
218
+ memo = { updated_anchors: {}, indirect_keys: {},
219
+ bibitems: Util::gather_bibitems(docxml),
220
+ erefs: Util::gather_bibitemids(docxml, presxml),
221
+ doc_suffix: docxml.root["document_suffix"],
222
+ doc_type: docxml.root["type"] }
223
+ [{}, memo]
213
224
  end
214
225
 
215
- def indirect_ref_key(schema, id, doc_suffix, add_suffix)
226
+ def indirect_ref_key(memo, schema, id, add_suffix)
216
227
  /^#{schema}_/.match?(id) and return id
217
- x = @indirect_keys.dig(schema, id) and return x
218
- @indirect_keys[schema] ||= {}
219
- @indirect_keys[schema][id] = if add_suffix
220
- "#{schema}_#{id}_#{doc_suffix}"
221
- else
222
- "#{schema}_#{id}"
223
- end
228
+ keys = memo[:indirect_keys]
229
+ x = keys.dig(schema, id) and return x
230
+ keys[schema] ||= {}
231
+ keys[schema][id] = if add_suffix
232
+ "#{schema}_#{id}_#{memo[:doc_suffix]}"
233
+ else
234
+ "#{schema}_#{id}"
235
+ end
224
236
  end
225
237
 
226
- def update_indirect_refs_to_docs1(filec, key, file, bibitems, erefs)
227
- erefs[key]&.each do |e|
238
+ def update_indirect_refs_to_docs1(filec, key, file, memo)
239
+ memo[:erefs][key]&.each do |e|
228
240
  e["citeas"] = file
229
241
  update_indirect_refs_to_docs_anchor(e, file, filec[:url],
230
- filec[:parentid])
242
+ filec[:parentid], memo)
231
243
  end
232
- update_indirect_refs_to_docs_docid(bibitems[key], file)
244
+ update_indirect_refs_to_docs_docid(memo[:bibitems][key], file)
233
245
  end
234
246
 
235
- def update_indirect_refs_to_docs_anchor(eref, file, url, parentid)
247
+ def update_indirect_refs_to_docs_anchor(eref, file, url, parentid, memo)
236
248
  a = eref.at(".//#{ANCHOR_XPATH}") or return
237
249
  parentid and file = "#{parentid}_#{file}"
238
250
  existing = a.text
239
251
  anchor = if url then existing
240
- else
241
- @indirect_keys[existing] ||= {}
242
- @indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
252
+ else indirect_ncname(memo, existing, file)
243
253
  end
244
- @updated_anchors[existing] or a.children = anchor
245
- @updated_anchors[anchor] = true
254
+ memo[:updated_anchors][existing] or a.children = anchor
255
+ memo[:updated_anchors][anchor] = true
256
+ end
257
+
258
+ # The NCName anchor binding an eref's existing anchor text to its
259
+ # target file, memoised in the member pass's key cache.
260
+ def indirect_ncname(memo, existing, file)
261
+ keys = memo[:indirect_keys][existing] ||= {}
262
+ keys[file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
246
263
  end
247
264
 
248
265
  def update_indirect_refs_to_docs_docid(bib, file)
@@ -256,34 +273,28 @@ presxml)
256
273
  def update_anchors(bib, docid, erefs, erefs_no_anchor, erefs_anchors)
257
274
  @files.get(docid) or error_anchor(erefs, docid)
258
275
  has_anchors, url, ncn_docid = update_anchors_prep(docid)
276
+ concat = {} # per-member memo, as with the indirect-reference pass
259
277
  erefs_no_anchor.each do |e|
260
278
  update_anchor_create_loc(bib, e, docid)
261
279
  end
262
280
  !url && has_anchors or return
263
281
  erefs_anchors.each do |e|
264
- update_anchors1(docid, ncn_docid, e)
282
+ update_anchors1(concat, docid, ncn_docid, e)
265
283
  end
266
284
  end
267
285
 
268
- def update_anchors1(docid, ncn_docid, anchor)
269
- @concat_anchors[anchor.text] ||= "#{ncn_docid}_#{anchor.text}"
270
- if @files.get(docid).dig(:anchors_lookup, @concat_anchors[anchor.text])
271
- anchor.content = @concat_anchors[anchor.text]
286
+ def update_anchors1(concat, docid, ncn_docid, anchor)
287
+ concat[anchor.text] ||= "#{ncn_docid}_#{anchor.text}"
288
+ if @files.get(docid).dig(:anchors_lookup, concat[anchor.text])
289
+ anchor.content = concat[anchor.text]
272
290
  end
273
291
  end
274
292
 
275
293
  def update_anchors_prep(docid)
276
- @concat_anchors = {}
277
294
  [@files.get(docid)&.key?(:anchors_lookup), @files.url?(docid),
278
295
  Metanorma::Utils::to_ncname(docid)]
279
296
  end
280
297
 
281
- # encode both prefix and suffix to NCName
282
- def suffix_anchor_indirect(prefix, suffix)
283
- k = "#{prefix}_#{suffix}"
284
- @ncnames[k] ||= Metanorma::Utils::to_ncname(k)
285
- end
286
-
287
298
  # if there is a crossref to another document, with no anchor, retrieve the
288
299
  # anchor given the locality, and insert it into the crossref
289
300
  def update_anchor_create_loc(_bib, eref, docid)
@@ -2,6 +2,14 @@
2
2
 
3
3
  require "isodoc"
4
4
  require "metanorma-utils"
5
+ require "concurrent-ruby"
6
+ # eprun (via twitter_cldr, via isodoc) sets its active Unicode version as a
7
+ # thread-local at require time, on whichever thread first loads it. Load it
8
+ # here, before the member pool starts, so that require happens once on the
9
+ # main thread and the pool workers can then set their own thread-local
10
+ # (initialise_pool_thread) instead of racing over who first requires the
11
+ # gem mid-render.
12
+ require "eprun"
5
13
  require_relative "fileparse"
6
14
  require_relative "filelocation"
7
15
  require_relative "../artifact_store"
@@ -10,37 +18,76 @@ require "json"
10
18
  module Metanorma
11
19
  class Collection
12
20
  class Renderer
21
+ # One member's rendering failure, buffered until the member pool joins
22
+ # so that failures can be reported in manifest order regardless of
23
+ # completion order.
24
+ MemberFailure = Struct.new(:index, :identifier, :errors, :missing)
25
+
13
26
  # compile and output individual file in collection
14
27
  # warn "metanorma compile -x html #{f.path}"
15
- def file_compile(file, filename, identifier)
28
+ def file_compile(file, filename, identifier, index)
16
29
  sectionsplit_val = @files.get(identifier, :sectionsplit)
17
30
  sectionsplit_val and return
18
31
  opts = compile_options_base(identifier)
19
32
  .merge(compile_options_update(identifier))
33
+ .merge(compile_options_bare_after_first(index))
20
34
 
21
- err_count = @compile.errors.size
22
- @compile.compile file, opts
35
+ compile = member_compile(file, opts)
23
36
  @files.set(identifier, :outputs, {})
24
- file_compile_formats(filename, identifier, opts)
25
- file_compile_verify(identifier, @compile.errors[err_count..] || [])
37
+ file_compile_formats(filename, identifier, opts,
38
+ member_processor(compile))
39
+ file_compile_verify(identifier, index, compile.errors)
40
+ end
41
+
42
+ # Run one member's conversion on its own Compile instance
43
+ # (metanorma#705): each member's error pool collects only that
44
+ # member's failures, so attribution is order-independent under the
45
+ # member pool. The previous design sliced a shared @compile.errors by
46
+ # array-position delta, which was only sound while members compiled
47
+ # sequentially. Construction cost is negligible (~0.1ms incl. flavour
48
+ # load) next to a member render; the renderer-level @compile remains
49
+ # for flavour loading and the collection-level font/PDF handling.
50
+ def member_compile(file, opts)
51
+ compile = Compile.new
52
+ compile.load_flavor(@flavor)
53
+ compile.compile file, opts
54
+ compile
55
+ end
56
+
57
+ # The flavour processor for output-path mapping: a compile that
58
+ # returned early (unsupported extensions, unreadable input) never
59
+ # reached its processor; the registry resolves the same processor
60
+ # object the compile would, so path mapping stays deterministic and
61
+ # independent of member order.
62
+ def member_processor(compile)
63
+ compile.processor ||
64
+ @registry.find_processor(Util::taste2flavor(@flavor).to_sym)
65
+ end
66
+
67
+ # bare-after-first: the directive previously mutated the shared compile
68
+ # options as the sequential loop walked the keys; under the member pool
69
+ # it becomes a per-member option keyed by the member's position in the
70
+ # file lookup (the first key renders bare-free, all later ones bare).
71
+ def compile_options_bare_after_first(index)
72
+ bare = index.positive? &&
73
+ @directives.detect { |d| d.key == "bare-after-first" }
74
+ bare ? { bare: true } : {}
26
75
  end
27
76
 
28
77
  # A member whose rendering raised produces no output file, but the
29
78
  # collection build used to continue, exit 0, and link the absent
30
79
  # document from index.html -- silent data loss (#586). Two
31
- # complementary checks close that hole. (1) Compile pools rescued
32
- # error messages in @compile.errors for its caller to surface; the
33
- # standalone CLI does, but the collection pipeline never read the
34
- # pool. The pool is shared across members and never reset, so the
35
- # slice appended during this member's compile is this member's
36
- # errors -- sound while members compile sequentially (the current
37
- # design; @compile's own format-level parallelism is joined before
38
- # compile returns). (2) Independently of whether anything was
80
+ # complementary checks close that hole. (1) Each member compiles with
81
+ # its own Compile instance (#705), whose error pool therefore holds
82
+ # only this member's conversion failures -- attribution is independent
83
+ # of execution order (the format-level pool inside a compile joins
84
+ # before compile returns). (2) Independently of whether anything was
39
85
  # pooled, verify the outputs registered for the member exist on
40
- # disk. Failures are reported here and accumulated; the collection
41
- # renders every remaining member before Collection#render aborts,
42
- # so one run reports all failed members.
43
- def file_compile_verify(identifier, errors)
86
+ # disk. Failures are buffered here and accumulated; the collection
87
+ # renders every remaining member before finalise_member_failures
88
+ # reports them and Collection#render aborts, so one run reports all
89
+ # failed members.
90
+ def file_compile_verify(identifier, index, errors)
44
91
  # Section-split parts (marked with :parentid) register format
45
92
  # slots they never produce -- their :xml slot in particular is
46
93
  # never written, and under filename renaming it aliases to the
@@ -51,10 +98,32 @@ module Metanorma
51
98
  missing = (@files.get(identifier, :outputs) || {})
52
99
  .reject { |_fmt, path| File.exist?(path) }
53
100
  errors.empty? && missing.empty? and return
54
- file_compile_failure_log(identifier, errors, missing)
55
- msg = file_compile_failure_msg(identifier, errors, missing)
56
- ::Metanorma::Util.log("[metanorma] Error: #{msg}", :error)
57
- fatal_errors << msg
101
+ record_member_failure(index, identifier, errors, missing)
102
+ end
103
+
104
+ # Buffer a member failure under the mutex: pool workers complete out
105
+ # of manifest order, and appends from multiple threads are not a
106
+ # synchronisation point we rely on.
107
+ def record_member_failure(index, identifier, errors, missing)
108
+ @failures_mutex.synchronize do
109
+ @member_failures << MemberFailure.new(index, identifier, errors,
110
+ missing)
111
+ end
112
+ end
113
+
114
+ # Flush the buffered member failures in manifest order: the error
115
+ # lines, the log entries (and so the .err.html artifact) and the
116
+ # fatal_errors list are ordered by the manifest, never by completion
117
+ # order. Runs before Collection#render aborts on fatal_errors (#586).
118
+ def finalise_member_failures
119
+ @member_failures.sort_by(&:index).each do |failure|
120
+ file_compile_failure_log(failure.identifier, failure.errors,
121
+ failure.missing)
122
+ msg = file_compile_failure_msg(failure.identifier, failure.errors,
123
+ failure.missing)
124
+ ::Metanorma::Util.log("[metanorma] Error: #{msg}", :error)
125
+ fatal_errors << msg
126
+ end
58
127
  end
59
128
 
60
129
  def file_compile_failure_log(identifier, errors, missing)
@@ -110,14 +179,29 @@ module Metanorma
110
179
 
111
180
  # process each file in the collection
112
181
  # files are held in memory, and altered as postprocessing
182
+ #
183
+ # Members are rendered on a bounded pool (metanorma#705): members are
184
+ # independent render units, and the sequential design existed for
185
+ # error attribution, not data dependence. Shared-state audit:
186
+ # - FileLookup entries are keyed per member; the only entry a member
187
+ # writes is its own :outputs hash. Distinct keys, distinct hashes.
188
+ # - each member builds its own Compile instance, so no compile state
189
+ # (errors, processor, log) is shared between members;
190
+ # - the cross-reference passes mutate only the member's own XML tree
191
+ # and per-member memo structures (localised in fileparse.rb); the
192
+ # renderer-level memos the passes read (svg hotspot index) are pure
193
+ # functions of the file lookup, which is frozen once init completes;
194
+ # - attachment copies and the DisambigFiles nametable stay on the main
195
+ # thread;
196
+ # - failures are buffered under a mutex and flushed in manifest order
197
+ # once the pool joins, and the navigation/coverpage/concatenation
198
+ # passes are built from the manifest afterwards (navigation.rb), so
199
+ # completion order never leaks into the output.
113
200
  def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
114
201
  warn "\n\n\n\n\nRender Files: #{DateTime.now.strftime('%H:%M:%S')}"
115
202
  internal_refs = locate_internal_refs
203
+ jobs = []
116
204
  @files.keys.each_with_index do |ident, i|
117
- i.positive? && @directives.detect do |d|
118
- d.key == "bare-after-first"
119
- end and
120
- @compile_options.merge!(bare: true)
121
205
  if @files.get(ident, :attachment) then copy_file_to_dest(ident)
122
206
  else
123
207
  # Distributed reinflation: the manifest carries every member
@@ -126,24 +210,79 @@ module Metanorma
126
210
  # other members' sections are rendered by sibling runs over
127
211
  # the same store.
128
212
  @only_members and !render_member?(ident) and next
129
- file, fname = @files.targetfile_id(ident, read: true)
130
- @preserve_unresolved && member_stored?(ident, file) and next
131
- warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
132
- collection_xml = update_xrefs(file, ident, internal_refs)
133
- @preserve_unresolved and
134
- store_member_artefacts(ident, file, collection_xml)
135
- # Strip .xml or .html extension, but NOT section numbers like .0
136
- fname_base = File.basename(fname)
137
- collection_filename = fname_base
138
- /\.(xml|html)$/.match?(fname_base) and
139
- collection_filename = fname_base.sub(/\.(xml|html)$/, "")
140
- collection_xml_path = File.join(Dir.tmpdir,
141
- "#{collection_filename}.xml")
142
- File.write collection_xml_path, collection_xml, encoding: "UTF-8"
143
- file_compile(collection_xml_path, fname, ident)
144
- FileUtils.rm_f(collection_xml_path)
213
+ jobs << [i, ident]
145
214
  end
146
215
  end
216
+ render_members(jobs, internal_refs)
217
+ finalise_member_failures
218
+ end
219
+
220
+ # Render the selected members on a bounded pool. Default 4 workers,
221
+ # overridable with the :member_pool_size render option.
222
+ def render_members(jobs, internal_refs)
223
+ jobs.empty? and return
224
+ pool = Concurrent::FixedThreadPool.new(member_pool_size)
225
+ jobs.each do |(index, ident)|
226
+ pool.post { render_member(index, ident, internal_refs) }
227
+ end
228
+ pool.shutdown
229
+ pool.wait_for_termination
230
+ end
231
+
232
+ # One member render unit: cross-reference passes over the member's own
233
+ # tree, then compile and output. A member whose processing raises is
234
+ # recorded as that member's failure -- a raising job must never take
235
+ # its worker thread down or abort the pool, so the remaining members
236
+ # still render and the run reports all failures (#586).
237
+ def render_member(index, ident, internal_refs)
238
+ initialise_pool_thread
239
+ file, fname = @files.targetfile_id(ident, read: true)
240
+ @preserve_unresolved && member_stored?(ident, file) and return
241
+ warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
242
+ xml = member_collection_xml(file, ident, internal_refs)
243
+ xml_path = write_member_collection_xml(fname, xml)
244
+ file_compile(xml_path, fname, ident, index)
245
+ FileUtils.rm_f(xml_path)
246
+ rescue StandardError => e
247
+ record_member_failure(index, ident, ["#{e.class}: #{e.message}"], {})
248
+ end
249
+
250
+ # Cross-reference the member's tree against the collection; a staging
251
+ # build additionally persists the stub-bearing artefacts to the store.
252
+ def member_collection_xml(file, ident, internal_refs)
253
+ collection_xml = update_xrefs(file, ident, internal_refs)
254
+ @preserve_unresolved and
255
+ store_member_artefacts(ident, file, collection_xml)
256
+ collection_xml
257
+ end
258
+
259
+ # Write the member's cross-referenced XML to a temp file for the
260
+ # compile. The temp basename drives the compiled output filenames, so
261
+ # it stays the member's own (per-collection-disambiguated) basename --
262
+ # distinct across members, so pool workers never share a temp path.
263
+ def write_member_collection_xml(fname, collection_xml)
264
+ # Strip .xml or .html extension, but NOT section numbers like .0
265
+ fname_base = File.basename(fname)
266
+ collection_filename = fname_base
267
+ /\.(xml|html)$/.match?(fname_base) and
268
+ collection_filename = fname_base.sub(/\.(xml|html)$/, "")
269
+ collection_xml_path = File.join(Dir.tmpdir,
270
+ "#{collection_filename}.xml")
271
+ File.write(collection_xml_path, collection_xml, encoding: "UTF-8")
272
+ collection_xml_path
273
+ end
274
+
275
+ # Re-establish the per-thread gem defaults a fresh worker thread
276
+ # lacks. eprun keeps the active Unicode version in a thread-local
277
+ # that require time sets only on the loading thread (eprun.rb's
278
+ # `Eprun.current_version = Eprun::UcdVersion.latest`); on any other
279
+ # thread it reads nil and isodoc's index collation crashes with
280
+ # NoMethodError inside Eprun::UcdVersion.get. The gem itself is
281
+ # required up front (see the require block above), so every worker
282
+ # starts its job by giving its thread the same latest version the
283
+ # loading thread received.
284
+ def initialise_pool_thread
285
+ Eprun.current_version ||= Eprun::UcdVersion.latest
147
286
  end
148
287
 
149
288
  # Persist an isolated member's stub-bearing Semantic XML and its anchor
@@ -23,6 +23,9 @@ module Metanorma
23
23
  builder.ul { |b| docrefs(mnf, b) }
24
24
  end
25
25
 
26
+ # A document entry can carry nested entries of its own (publication >
27
+ # part > section, metanorma-cli#496); the block, if given, renders
28
+ # them inside the document's list item.
26
29
  def docrefs(mnf, builder)
27
30
  ident = docref_ident(mnf)
28
31
  builder.li do |li|
@@ -31,6 +34,7 @@ module Metanorma
31
34
  /[<>&]/.match?(x) ? x : @c.encode(x, :hexadecimal)
32
35
  end.join
33
36
  end
37
+ yield li if block_given?
34
38
  end
35
39
  end
36
40
 
@@ -39,6 +43,17 @@ module Metanorma
39
43
  @c.decode(@isodoc.docid_prefix(nil, ident))
40
44
  end
41
45
 
46
+ # nested entries of a document entry, as a sublist of its list item;
47
+ # all: list them regardless of their index flag
48
+ def nested_docrefs(mnf, builder, all: true)
49
+ kids = Array(mnf.entry)
50
+ all or kids = kids.select { |e| index?(e) }
51
+ kids.empty? and return
52
+ builder.ul do |u|
53
+ kids.each { |e| u << indexfile1(e, all:) }
54
+ end
55
+ end
56
+
42
57
  # Check if file has a recognized MIME type (other than XML)
43
58
  # If so, don't append .html (e.g., .svg, .png, .jpg, etc.)
44
59
  def index_link(docref, ident)
@@ -72,19 +87,23 @@ module Metanorma
72
87
  mnf.entry&.detect { |e| index?(e) }
73
88
  end
74
89
 
75
- def indexfile1(mnf)
76
- index?(mnf) or return ""
77
- cleanup_indexfile1(build_indexfile1(mnf))
90
+ # all: list the entry regardless of its index flag
91
+ def indexfile1(mnf, all: false)
92
+ all || index?(mnf) or return ""
93
+ cleanup_indexfile1(build_indexfile1(mnf, all:))
78
94
  end
79
95
 
80
- def build_indexfile1(mnf)
96
+ def build_indexfile1(mnf, all: false)
81
97
  Nokogiri::HTML::Builder.new do |b|
82
- if mnf.file then docrefs(mnf, b)
98
+ if mnf.file
99
+ docrefs(mnf, b) do |li|
100
+ nested_docrefs(mnf, li, all:)
101
+ end
83
102
  else
84
103
  b.li do |l|
85
104
  l << indexfile_title(mnf)
86
105
  l.ul do |u|
87
- Array(mnf.entry).each { |e| u << indexfile1(e) }
106
+ Array(mnf.entry).each { |e| u << indexfile1(e, all:) }
88
107
  end
89
108
  end
90
109
  end
@@ -125,7 +144,7 @@ module Metanorma
125
144
  files.empty? and return nil
126
145
  r = Nokogiri::HTML::Builder.new do |b|
127
146
  b.ul do |u|
128
- files.each { |f| docrefs(f, u) }
147
+ files.each { |f| docrefs(f, u) { |li| nested_docrefs(f, li) } }
129
148
  end
130
149
  end
131
150
  r.doc.root&.to_html&.tr("\n", " ")
@@ -137,9 +156,9 @@ module Metanorma
137
156
  ident = @c.decode(@isodoc.docid_prefix(nil, d.identifier.dup))
138
157
  m << { "identifier" => ident, "file" => index_link(d, ident),
139
158
  "title" => indexfile_title(d), "level" => d.type }
140
- else
141
- liquid_docrefs(d.entry).each { |m1| m << m1 }
142
159
  end
160
+ # entries nested under a document follow it (metanorma-cli#496)
161
+ liquid_docrefs(d.entry).each { |m1| m << m1 }
143
162
  end
144
163
  end
145
164
  end
@@ -17,6 +17,9 @@ module Metanorma
17
17
  class Renderer
18
18
  FORMATS = %i[html xml doc pdf pdf-portfolio].freeze
19
19
 
20
+ # Worker count for the member render pool (metanorma#705).
21
+ DEFAULT_MEMBER_POOL_SIZE = 4
22
+
20
23
  attr_accessor :isodoc, :isodoc_presxml
21
24
  attr_reader :xml, :compile, :compile_options, :documents, :outdir,
22
25
  :manifest, :fatal_errors
@@ -94,9 +97,12 @@ module Metanorma
94
97
  @c = HTMLEntities.new
95
98
  @files_to_delete = []
96
99
  # Per-member rendering failures (missing outputs, pooled compile
97
- # errors), accumulated by file_compile_verify; Collection#render
98
- # aborts on them once every member has rendered (#586).
100
+ # errors, raised exceptions), buffered by file_compile_verify and
101
+ # flushed in manifest order once the member pool joins; Collection
102
+ # #render aborts on them after every member has rendered (#586).
99
103
  @fatal_errors = []
104
+ @failures_mutex = Mutex.new
105
+ @member_failures = []
100
106
  @nested = options[:nested]
101
107
  # if false, this is the root instance of Renderer
102
108
  # if true, then this is not the last time Renderer will be run
@@ -130,6 +136,8 @@ module Metanorma
130
136
  # pruning superseded content-hash versions on write; keep_cache retains
131
137
  # every version (history / multiple states side by side).
132
138
  @keep_cache = options[:keep_cache]
139
+ @member_pool_size = options[:member_pool_size] ||
140
+ DEFAULT_MEMBER_POOL_SIZE
133
141
 
134
142
  @coverpage = options[:coverpage] || collection.coverpage
135
143
  @coverpage_pdf_portflio = options[:coverpage_pdf_portfolio] ||
@@ -157,6 +165,11 @@ module Metanorma
157
165
  create_non_existing_directory(@outdir)
158
166
  end
159
167
 
168
+ # Worker count for the member render pool (see DEFAULT_MEMBER_POOL_SIZE).
169
+ def member_pool_size
170
+ @member_pool_size
171
+ end
172
+
160
173
  def directives_normalise(directives)
161
174
  @coverpage_pdf_portflio and
162
175
  directives = directives_normalise_coverpage_pdf_portfolio(directives)
@@ -416,15 +416,35 @@ module Metanorma
416
416
  # name (geometry_schema.cartesian_point_ISO_10303-42_2025_...), so
417
417
  # the plain name resolves by prefix. Multiple equal-length matches
418
418
  # are ambiguous and refuse, as do differing ids at the same key.
419
+ # Memoized per prefix: the index is immutable once built, and a
420
+ # mega-member's svgmap erefs hit this per stage pass (the linear
421
+ # scan showed up at 24%+ of a 104-style render's wall).
419
422
  def hotspot_prefix_lookup(idx, prefix)
420
- return idx[prefix] if idx[prefix]
423
+ @hotspot_prefix_memo ||= {}
424
+ @hotspot_prefix_memo.fetch(prefix) do
425
+ @hotspot_prefix_memo[prefix] =
426
+ idx[prefix] || prefix_lookup_scan(idx, prefix)
427
+ end
428
+ end
421
429
 
422
- candidates = idx.select { |k, r| r && k.start_with?("#{prefix}_") }
430
+ # Candidate keys sharing the prefix, by binary search over the
431
+ # sorted index keys: keys with the stem prefix are contiguous in
432
+ # sort order, so the slice between the lower bound and the first
433
+ # non-matching key is the entire candidate set.
434
+ def prefix_lookup_scan(idx, prefix)
435
+ stem = "#{prefix}_"
436
+ keys = hotspot_sorted_keys(idx)
437
+ lo = keys.bsearch_index { |k| k >= stem } or return nil
438
+ candidates = keys[lo..].take_while { |k| k.start_with?(stem) }
423
439
  return nil if candidates.empty?
424
440
 
425
- shortest = candidates.min_by { |k, _| k.length }
426
- ties = candidates.count { |k, _| k.length == shortest[0].length }
427
- ties == 1 ? shortest[1] : nil
441
+ shortest = candidates.min_by(&:length)
442
+ ties = candidates.count { |k| k.length == shortest.length }
443
+ ties == 1 ? idx[shortest] : nil
444
+ end
445
+
446
+ def hotspot_sorted_keys(idx)
447
+ @hotspot_sorted_keys ||= idx.keys.select { |k| idx[k] }.sort
428
448
  end
429
449
  end
430
450
  end
@@ -54,7 +54,10 @@ module Metanorma
54
54
  },
55
55
  manifest: {
56
56
  level: "collection", title: "Collection",
57
- docref: files.sort_by { |f| f[:order] }.each.map do |f|
57
+ # [order, idx]: tasks complete out of order under the section
58
+ # split pool, and Ruby sorts are not guaranteed stable, so the
59
+ # split idx breaks displayorder ties deterministically.
60
+ docref: files.sort_by { |f| [f[:order], f[:idx]] }.each.map do |f|
58
61
  # XML files are always stored flat, so fileref is always the basename
59
62
  # f[:url] may include directory for HTML output, but XML is basename only
60
63
  fileref = File.basename(f[:url])
@@ -67,8 +67,13 @@ module Metanorma
67
67
  empty = empty_doc(xml)
68
68
  empty1 = empty_attachments(empty)
69
69
  @mutex = Mutex.new
70
- # @pool = Concurrent::FixedThreadPool.new(4)
71
- @pool = Concurrent::FixedThreadPool.new(1)
70
+ # Per-section work mutates only its own xml.dup and the per-section
71
+ # hidden-ref container; the shared document is read-only once
72
+ # xref_preprocess has run serially above, and the manifest consumer
73
+ # re-sorts the results by [displayorder, split idx], so completion
74
+ # order is unobservable. Throttled to 1 during the isodoc#641
75
+ # fmt-xref refactor (92bd93a) and never restored. metanorma#703.
76
+ @pool = Concurrent::FixedThreadPool.new(4)
72
77
  sectionsplit1(xml, empty, empty1, 0)
73
78
  end
74
79
 
@@ -96,7 +101,7 @@ module Metanorma
96
101
  )
97
102
  warn "Sectionsplit: #{output_filename}"
98
103
  a = sectionfile(xml, empty, output_filename, chunks,
99
- parentnode)
104
+ parentnode).merge(idx: opt[:idx])
100
105
  @mutex.synchronize { opt[:acc] << a }
101
106
  end
102
107
  end
@@ -1,3 +1,3 @@
1
1
  module Metanorma
2
- VERSION = "2.5.23".freeze
2
+ VERSION = "2.5.25".freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: metanorma
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.23
4
+ version: 2.5.25
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-11-12 00:00:00.000000000 Z
11
+ date: 2026-11-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asciidoctor
@@ -259,6 +259,7 @@ files:
259
259
  - lib/metanorma/collection/artifact_store.rb
260
260
  - lib/metanorma/collection/collection.rb
261
261
  - lib/metanorma/collection/config/bibdata.rb
262
+ - lib/metanorma/collection/config/bibdata_parser.rb
262
263
  - lib/metanorma/collection/config/compile_options.rb
263
264
  - lib/metanorma/collection/config/config.rb
264
265
  - lib/metanorma/collection/config/converters.rb