metanorma 2.5.7 → 2.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d73fd0dd05e3f13aecfe06da74c57bdba985c633ca55464d90ad8eef1d54c3ea
4
- data.tar.gz: 7b8c4a103f6ffce8d6a6234f18ef4f7e0cabf18af0c3aca9abec49e4e10255e8
3
+ metadata.gz: fac12acf3d39fe398084ba3fb04c1fa1bb12ceebfa48187ae0f7e006c7f6833c
4
+ data.tar.gz: 562c2bda6334a61d8733c6dd989a002342af492e91ea8d10c81858a4328408aa
5
5
  SHA512:
6
- metadata.gz: 8927439c5072688f6df2e001e9a97611d5d1802047e4f5b1572412c0217e8eead1255ff97fe464efc16c669887ae861de4d0364109b40202b36ff0151c8998e2
7
- data.tar.gz: d45b9e9140271b2a4cfb6e9cef1a527ba13d69dceb07c9bd4181fd1b39f0180f319cb07858161e7e8683aeced2dd8c20a1b3a5ba222bec6b648a19dc01c06996
6
+ metadata.gz: 8d09182c5e26e988445bfbfc9624a6d07eefba9d373910384eab675730a7b93976a74ecb2497f4c631f6d7d311295db954ca8e26a3868cb58082b114e6a7543f
7
+ data.tar.gz: 6cdce7f6a5cd577e338d635ab2b8fed170645ac771aa8eec675256d75ba7751be07f14a379c1b75a717f9d19200b809ba51f8e68b451ef3e50169ce831ec0969
data/Gemfile CHANGED
@@ -8,6 +8,16 @@ gemspec
8
8
 
9
9
  gem "metanorma-core", ">= 0.2.5"
10
10
 
11
+ # taste stack the compile specs exercise: the oiml taste registers the
12
+ # flavor and lazily requires metanorma-oiml (OIML STS transformers,
13
+ # unreleased); mko main carries the model-side export entry.
14
+ # Lives here, not in Gemfile.devel: consumer workflows may rewrite the
15
+ # devel overlay (fontist CI appends its path pin to it), and the repo's
16
+ # own compile specs must not depend on its survival.
17
+ gem "metanorma-taste", "~> 1.1"
18
+ gem "metanorma-oiml", github: "metanorma/metanorma-oiml", branch: "main"
19
+ gem "metanorma-mko", github: "metanorma/metanorma-mko", branch: "main"
20
+
11
21
  begin
12
22
  eval_gemfile("Gemfile.devel")
13
23
  rescue StandardError
data/Gemfile.devel CHANGED
@@ -18,9 +18,16 @@ gem "glossarist", "~> 2.13"
18
18
  gem "lutaml-model", "~> 0.8.107"
19
19
  gem "metanorma-plugin-glossarist", "~> 0.5"
20
20
  gem "moxml", "~> 0.5"
21
- # taste stack the compile specs exercise: the oiml taste registers the
22
- # flavor and lazily requires metanorma-oiml (OIML STS transformers,
23
- # unreleased); mko main carries the model-side export entry
24
- gem "metanorma-taste", "~> 1.1"
25
- gem "metanorma-oiml", github: "metanorma/metanorma-oiml", branch: "main"
26
- gem "metanorma-mko", github: "metanorma/metanorma-mko", branch: "main"
21
+
22
+ # Bridge: pubid 2.0.1 (and git main, until the PARG edition-2 migration
23
+ # lands) bakes grammars with the parsanol `mode` header alias, which
24
+ # parsanol 1.3.113+ removed; pubid main's own floor (>= 1.3.114) admits
25
+ # the breaking 1.3.120, so collection specs fail with 13 rendering
26
+ # regressions. 1.3.114 is pubid's own floor and the last alias-accepting
27
+ # compiler. Drop this pin when pubid ships edition-2 artifacts.
28
+ gem "parsanol", "1.3.114"
29
+
30
+ # Bridge: svg_conform 0.2.5 was pushed with a compact-index checksum
31
+ # mismatch (Bundler::APIResponseMismatchError on every download); pin
32
+ # 0.2.4 until a corrected push lands.
33
+ gem "svg_conform", "0.2.4"
@@ -184,17 +184,33 @@ module Metanorma
184
184
  # param filepath [String]
185
185
  # @raise [AdocFileNotFoundException]
186
186
  def compile_adoc_file(file)
187
+ warn "BASEDIR-PROBE " + base_dir_options.inspect
187
188
  f = (Pathname.new file).absolute? ? file : File.join(@dir, file)
188
189
  File.exist?(f) or raise AdocFileNotFoundException.new "#{f} not found!"
189
190
  compile_adoc_file?(file) or return
190
191
  ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...", :warning)
191
192
  ::Metanorma::Compile.new
192
- .compile(f, agree_to_terms: true, install_fonts: false,
193
- extension_keys: [:xml])
193
+ .compile(f, { agree_to_terms: true, install_fonts: false,
194
+ extension_keys: [:xml] }.merge(base_dir_options))
194
195
  ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...done!",
195
196
  :warning)
196
197
  end
197
198
 
199
+ # Opt-in collection directive `base-dir: collection`: anchor the
200
+ # AsciiDoc include jail at the collection directory instead of the
201
+ # process working directory, so members compile identically from
202
+ # any working directory (metanorma#627). The anchor spans the whole
203
+ # collection, so member includes reaching above the member
204
+ # directory (../../../templates) stay inside the jail. The default
205
+ # (no directive) keeps today's working-directory anchoring, so
206
+ # existing builds stay byte-identical.
207
+ def base_dir_options
208
+ @collection.directives.detect do |d|
209
+ d.key == "base-dir" && d.value == "collection"
210
+ end or return {}
211
+ { base_dir: @collection.dirname }
212
+ end
213
+
198
214
  def compile_adoc_file?(file)
199
215
  !File.exist?(file.sub(/\.adoc$/, ".xml")) and return true
200
216
  @collection.directives.detect do |d|
@@ -276,8 +276,10 @@ module Metanorma
276
276
  font_options)
277
277
 
278
278
  mn2pdf = {
279
- font_manifest: ::Metanorma::Util::FontistHelper
280
- .location_manifest(@compile.processor, { fonts: fonts }),
279
+ font_manifest: ::Metanorma::Util::FontistHelper.location_manifest(
280
+ @compile.processor, { fonts: fonts },
281
+ continue_without_fonts: @compile_options[:continue_without_fonts],
282
+ ),
281
283
  }
282
284
  end
283
285
  [pres, { fonts: fonts, mn2pdf: mn2pdf }.compact]
@@ -190,7 +190,10 @@ module Metanorma
190
190
  p = Metanorma::Registry.instance.find_processor(Util::taste2flavor(flavor))
191
191
  if ::Metanorma::Util::FontistHelper.has_custom_fonts?(p, options, {})
192
192
  @fonts_manifest =
193
- ::Metanorma::Util::FontistHelper.location_manifest(p, options)
193
+ ::Metanorma::Util::FontistHelper.location_manifest(
194
+ p, options,
195
+ continue_without_fonts: options[:continue_without_fonts],
196
+ )
194
197
  end
195
198
  @options = options
196
199
  end
@@ -7,6 +7,7 @@ require "yaml"
7
7
  require "fontist"
8
8
  require "fontist/manifest"
9
9
  require "metanorma-taste"
10
+ require "metanorma/plugin/lutaml"
10
11
  require_relative "writeable"
11
12
  require_relative "validator"
12
13
  require_relative "compile_options"
@@ -28,6 +29,8 @@ module Metanorma
28
29
  include Flavor
29
30
  include Writeable
30
31
 
32
+ autoload :FragmentParallel, "metanorma/compile/fragment_parallel"
33
+
31
34
  DEFAULT_NUM_WORKERS = 3
32
35
 
33
36
  # @return [Array<String>]
@@ -52,6 +55,10 @@ module Metanorma
52
55
  # Step 1: Generate Semantic XML
53
56
  semantic_result = generate_semantic_xml(filename, options) or return nil
54
57
  source_file, semantic_xml = semantic_result
58
+ # All lutaml macros have expanded by the end of conversion; the
59
+ # plugin's retained XMI graphs (LRU-bounded, gigabytes for large
60
+ # models) are dead weight for every later phase. Flush them.
61
+ Metanorma::Plugin::Lutaml::CacheRegistry.clear_all
55
62
 
56
63
  # Step 2: Prepare output paths
57
64
  xml = Nokogiri::XML(semantic_xml, &:huge)
@@ -189,8 +196,13 @@ module Metanorma
189
196
  c.process_input_adoc_overrides(attrs, options)
190
197
  end
191
198
 
192
- def process_input_adoc_includes(file, filename)
199
+ def process_input_adoc_includes(file, filename, base_dir = nil)
193
200
  dir = File.dirname(filename)
201
+ # With an explicit asset anchor (base_dir), Asciidoctor resolves
202
+ # include targets against that anchor rather than the working
203
+ # directory, so the docfile-directory prefix must be absolute for
204
+ # includes to stay docfile-anchored from any working directory.
205
+ base_dir and dir = File.expand_path(dir)
194
206
  dir != "." and
195
207
  file = file.gsub(/^include::/, "include::#{dir}/")
196
208
  .gsub(/^embed::/, "embed::#{dir}/")
@@ -198,10 +210,19 @@ module Metanorma
198
210
  end
199
211
 
200
212
  def process_input_adoc(filename, options)
213
+ if (w = ENV["METANORMA_PARALLEL_CLAUSES"].to_i) >= 2
214
+ # Fragment-based parallel semantic compile (spike, #620): split,
215
+ # compile the fragments in worker processes, stitch. The stitched
216
+ # semantic XML rejoins the ordinary output pipeline below.
217
+ Util.log("[metanorma] Processing: AsciiDoc input" \
218
+ " (fragment-parallel, #{w} workers).", :info)
219
+ return [read_file(filename),
220
+ FragmentParallel.new(filename, options, w).run]
221
+ end
201
222
  Util.log("[metanorma] Processing: AsciiDoc input.", :info)
202
223
  file = read_file(filename)
203
224
  file = process_input_adoc_hdr(file, options)
204
- file = process_input_adoc_includes(file, filename)
225
+ file = process_input_adoc_includes(file, filename, options[:base_dir])
205
226
  update_options_after_preprocessing(file, options)
206
227
  [file, @processor.input_to_isodoc(file, filename, options)]
207
228
  end
@@ -200,8 +200,10 @@ module Metanorma
200
200
 
201
201
  ext == :pdf && custom_fonts and
202
202
  ret[:mn2pdf] = {
203
- font_manifest: Util::FontistHelper
204
- .location_manifest(@processor, ret),
203
+ font_manifest: Util::FontistHelper.location_manifest(
204
+ @processor, ret,
205
+ continue_without_fonts: options[:continue_without_fonts],
206
+ ),
205
207
  }
206
208
  end
207
209
  end
@@ -0,0 +1,549 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "open3"
5
+ require "yaml"
6
+
7
+ module Metanorma
8
+ class Compile
9
+ # Spike: fragment-based parallel semantic compilation (metanorma#620).
10
+ #
11
+ # Opt in with METANORMA_PARALLEL_CLAUSES=<N> (N >= 2; default off). The
12
+ # AsciiDoc source is split into clause fragments, each fragment is
13
+ # compiled to semantic XML by its own metanorma process (isolating the
14
+ # Asciidoctor/glossarist global state), and the fragment XMLs are
15
+ # stitched back into one semantic document that the ordinary output
16
+ # pipeline consumes as today.
17
+ #
18
+ # Split rules:
19
+ # - includes are resolved inline, recursively;
20
+ # - text before the first top-level heading (document header, preamble,
21
+ # floating-title forewords) belongs to the first fragment;
22
+ # - each top-level (`==`) clause is one fragment;
23
+ # - a clause whose body is a single `[yaml2text,<file>,<var>]` block
24
+ # over an array-valued YAML file is split further: each fragment
25
+ # repeats the clause's heading chain and the block template verbatim,
26
+ # but feeds it a generated YAML file holding one chunk of entries.
27
+ # That is what makes vocabulary corpora (per-schema terms loops)
28
+ # parallelisable without understanding their expansion.
29
+ #
30
+ # Stitching merges root-level children by element name: preface,
31
+ # sections, bibliography and metanorma-extension children are merged
32
+ # (wrapper clauses carrying the same anchor or title are unified;
33
+ # non-wrapper duplicates are dropped), annex/indexsect/colophon are
34
+ # appended in source order. Auto-generated boilerplate (foreword,
35
+ # boilerplate) is taken from the first fragment only.
36
+ #
37
+ # Known spike gaps: per-fragment validation is skipped (the stitched
38
+ # document flows through the normal pipeline, where validation and
39
+ # presentation run as today); bibitem pruning that keys on citations
40
+ # sees only the owning fragment's citations; shorthand `<<bibitem>>`
41
+ # references to bibitems declared in another fragment degrade to
42
+ # xref; glossarist <concept> cross-references to terms owned by
43
+ # another fragment degrade to <errormsg> unless the reference is
44
+ # unqualified and the repair pass can reconstruct it (domain-qualified
45
+ # concept resolution needs the glossarist dataset, which no fragment
46
+ # holds for the whole document); generated element ids differ between
47
+ # any two compiles, so stitched output is canon-equivalent to a serial
48
+ # compile (identical modulo generated ids and whitespace-only text at
49
+ # splice points), not byte-identical.
50
+ class FragmentParallel
51
+ HEADING1 = /^==(?!=)/.freeze
52
+ STYLE_ATTR = /^(\[\[.+?\]\]|\[[^\[\]]+\])$/.freeze
53
+ FRAME_TITLE = /^\.[^\s]/.freeze
54
+ INCLUDE = /^include::(\S+?)\[\]\s*$/.freeze
55
+ # [yaml2text,file,var] and [data2text,var=file]
56
+ DATA2TEXT = /\[(yaml2text|data2text),(.+?)\]\s*$/.freeze
57
+ MAX_INCLUDE_DEPTH = 8
58
+
59
+ SKIP_ROOT = %w[bibdata boilerplate].freeze
60
+ MERGE_ROOT = %w[preface sections bibliography
61
+ metanorma-extension].freeze
62
+ WRAPPERS = %w[clause terms references introduction].freeze
63
+ NONBASE_PREFACE_SKIP = %w[foreword].freeze
64
+ # Canonical root-child order of metanorma semantic XML (standoc emits
65
+ # bibliography last, annexes after sections); the stitch reorders to
66
+ # match, since fragments append in source order.
67
+ ROOT_ORDER = %w[bibdata metanorma-extension boilerplate preface
68
+ sections annex indexsect colophon bibliography].freeze
69
+
70
+ CHILD_SCRIPT = <<~RUBY.freeze
71
+ require "metanorma"
72
+ (ENV["METANORMA_PARALLEL_CHILD_REQUIRES"] || "")
73
+ .split(",").each { |g| require g }
74
+ Metanorma::Compile.new.compile(
75
+ ARGV[0], format: :asciidoc, extension_keys: [:xml],
76
+ novalid: true, agree_to_terms: true
77
+ )
78
+ RUBY
79
+
80
+ def initialize(filename, options, workers)
81
+ @filename = filename
82
+ @workers = workers
83
+ @dir = File.realpath(File.dirname(File.expand_path(filename)))
84
+ @base = File.basename(filename, ".*")
85
+ @prefix = ".#{@base}.frag"
86
+ @requires = [options[:type] && "metanorma-#{options[:type]}"]
87
+ if (r = ENV["METANORMA_PARALLEL_CHILD_REQUIRES"])
88
+ @requires = r.split(",")
89
+ end
90
+ @requires.compact!
91
+ end
92
+
93
+ def run
94
+ fragments = build_fragments
95
+ warn "[fragment-parallel] #{@filename}: #{fragments.size} fragments," \
96
+ " #{@workers} workers"
97
+ compile_fragments(fragments)
98
+ t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
99
+ xml = stitch(fragments.map { |f| f[:path].sub(/\.adoc$/, ".xml") })
100
+ warn format("[fragment-parallel] stitched in %.1fs",
101
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
102
+ xml
103
+ ensure
104
+ cleanup
105
+ end
106
+
107
+ private
108
+
109
+ # -- splitting
110
+
111
+ def build_fragments
112
+ lines = resolve_includes(
113
+ File.read(@filename, encoding: "utf-8"), 0
114
+ ).split("\n")
115
+ header, segments = segmentize(lines)
116
+ fragments = []
117
+ segments.each do |seg|
118
+ if (mega = data2text_mega(seg))
119
+ mega_fragments(fragments, header, seg, mega)
120
+ else
121
+ fragments << new_fragment(fragments.size, header, seg[:lines])
122
+ end
123
+ end
124
+ fragments
125
+ end
126
+
127
+ def new_fragment(i, header, body_lines)
128
+ path = File.join(@dir, "#{@prefix}-#{i}.adoc")
129
+ File.write(path, (header + body_lines).join("\n").concat("\n"),
130
+ encoding: "UTF-8")
131
+ { path: path, lines: header + body_lines }
132
+ end
133
+
134
+ # Returns [header_lines, [{lines:}, ...]]; header is everything
135
+ # before the first segment boundary. A boundary opens at a top-level
136
+ # heading, an include, or a run of attribute/anchor/frame-title lines
137
+ # immediately preceding one; the heading itself is part of that
138
+ # segment and does not open another.
139
+ def segmentize(lines)
140
+ bounds = []
141
+ i = 0
142
+ while i < lines.size
143
+ if boundary_start(lines, i)
144
+ # the boundary opens at the attribute-run start (or the bare
145
+ # heading); blank lines separating the previous clause travel
146
+ # with the fragment that follows them, so inter-clause
147
+ # spacing survives the split
148
+ b = i
149
+ b -= 1 while b.positive? && lines[b - 1].strip.empty?
150
+ bounds << b
151
+ i = boundary_start_resume(lines, i) || lines.size
152
+ else
153
+ i += 1
154
+ end
155
+ end
156
+ return [[], [{ lines: lines }]] if bounds.empty?
157
+
158
+ header = lines[0...bounds.first]
159
+ segs = bounds.each_with_index.map do |b, k|
160
+ { lines: lines[b...(bounds[k + 1] || lines.size)] }
161
+ end
162
+ [header, segs]
163
+ end
164
+
165
+ # Index where the boundary scan resumes: past the heading/include
166
+ # that closes the attribute run opened at +i+.
167
+ def boundary_start_resume(lines, i)
168
+ return i + 1 if seg_boundary_line?(lines[i])
169
+
170
+ j = i
171
+ loop do
172
+ j += 1
173
+ j += 1 while j < lines.size && lines[j].strip.empty?
174
+ return j + 1 if j < lines.size && seg_boundary_line?(lines[j])
175
+ return nil unless j < lines.size &&
176
+ (lines[j].match?(STYLE_ATTR) ||
177
+ lines[j].match?(FRAME_TITLE))
178
+ end
179
+ end
180
+
181
+ # Index of the heading/include that closes the attribute run
182
+ # opening at line +i+, or nil when lines[i] opens no boundary.
183
+ def boundary_start(lines, i)
184
+ return i if seg_boundary_line?(lines[i])
185
+
186
+ return nil unless lines[i].match?(STYLE_ATTR) ||
187
+ lines[i].match?(FRAME_TITLE)
188
+
189
+ j = i
190
+ loop do
191
+ j += 1
192
+ j += 1 while j < lines.size && lines[j].strip.empty?
193
+ return nil if j >= lines.size
194
+ return j if seg_boundary_line?(lines[j])
195
+ return nil unless lines[j].match?(STYLE_ATTR) ||
196
+ lines[j].match?(FRAME_TITLE)
197
+ end
198
+ end
199
+
200
+ def seg_boundary_line?(line)
201
+ line.match?(HEADING1) || line.match?(INCLUDE)
202
+ end
203
+
204
+ def resolve_includes(text, depth, base = @dir)
205
+ return text if depth >= MAX_INCLUDE_DEPTH
206
+
207
+ text.gsub(INCLUDE) do
208
+ inc = Regexp.last_match[1]
209
+ path = File.expand_path(inc, base)
210
+ if File.exist?(path)
211
+ resolve_includes(File.read(path, encoding: "utf-8"), depth + 1,
212
+ File.dirname(path))
213
+ else
214
+ Regexp.last_match[0]
215
+ end
216
+ end
217
+ end
218
+
219
+ # A segment is mega-splittable when it holds exactly one
220
+ # yaml2text/data2text open block whose data file parses to an
221
+ # array of at least two entries.
222
+ def data2text_mega(seg)
223
+ attr_i = seg[:lines].each_index.find do |i|
224
+ seg[:lines][i].match?(DATA2TEXT)
225
+ end
226
+ return nil unless attr_i
227
+
228
+ m = seg[:lines][attr_i].match(DATA2TEXT)
229
+ delim = seg[:lines][attr_i + 1]
230
+ return nil unless delim && delim.match?(/^-{4,}$/)
231
+
232
+ close = ((attr_i + 2)...seg[:lines].size).find do |j|
233
+ seg[:lines][j] == delim
234
+ end
235
+ return nil unless close
236
+
237
+ file, var = parse_data2text_args(m[2])
238
+ return nil unless file && var
239
+
240
+ data_path = File.expand_path(file, @dir)
241
+ return nil unless File.exist?(data_path)
242
+
243
+ begin
244
+ entries = YAML.safe_load(File.read(data_path))
245
+ rescue Psych::SyntaxError
246
+ return nil
247
+ end
248
+ return nil unless entries.is_a?(Array) && entries.size >= 2
249
+
250
+ { attr_i: attr_i, close: close, delim: delim, kind: m[1],
251
+ template: seg[:lines][(attr_i + 2)...close],
252
+ var: var, entries: entries, data_path: data_path }
253
+ end
254
+
255
+ # "[yaml2text,]file,var" or "[data2text,]var=file" -> [file, var].
256
+ # The data file is the argument naming an existing YAML file.
257
+ def parse_data2text_args(raw)
258
+ if raw.include?(",")
259
+ a, b = raw.split(",", 2)
260
+ [[b, a], [a, b]].each do |file, var|
261
+ return [file, var] if data_file_arg?(file) && ident_arg?(var)
262
+ end
263
+ [nil, nil]
264
+ elsif (p = raw.split("=", 2)).size == 2
265
+ data_file_arg?(p[1]) && ident_arg?(p[0]) ? [p[1], p[0]] : [nil, nil]
266
+ else
267
+ [nil, nil]
268
+ end
269
+ end
270
+
271
+ def data_file_arg?(arg)
272
+ arg.match?(/\.ya?ml\z/i) && File.exist?(File.expand_path(arg, @dir))
273
+ end
274
+
275
+ def ident_arg?(arg)
276
+ arg.match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/)
277
+ end
278
+
279
+ def mega_fragments(fragments, header, seg, mega)
280
+ # one chunk per worker: each child re-loads the glossarist
281
+ # dataset, so more chunks mostly re-pay that fixed cost
282
+ chunks = [mega[:entries].size, @workers].min
283
+ size = (mega[:entries].size / chunks.to_f).ceil
284
+ head = header + seg[:lines][0...mega[:attr_i]]
285
+ base_idx = fragments.size
286
+ mega[:entries].each_slice(size).with_index do |slice, i|
287
+ idx = base_idx + i
288
+ data_name = "#{@prefix}-#{idx}.data.yaml"
289
+ data_path = File.join(@dir, data_name)
290
+ File.write(data_path, YAML.dump(slice), encoding: "UTF-8")
291
+ directive = if mega[:kind] == "data2text"
292
+ "[data2text,#{mega[:var]}=#{data_name}]"
293
+ else
294
+ "[yaml2text,#{data_name},#{mega[:var]}]"
295
+ end
296
+ path = File.join(@dir, "#{@prefix}-#{idx}.adoc")
297
+ File.write(path, [
298
+ head,
299
+ directive,
300
+ mega[:delim], mega[:template], mega[:delim],
301
+ ].flatten.join("\n").concat("\n"), encoding: "UTF-8")
302
+ fragments << { path: path, mega: true }
303
+ end
304
+ end
305
+
306
+ # -- compilation
307
+
308
+ def compile_fragments(fragments)
309
+ pool = ::Metanorma::Util::WorkersPool.new(@workers)
310
+ lock = Mutex.new
311
+ done = 0
312
+ failures = []
313
+ fragments.each_with_index do |f, i|
314
+ pool.schedule do
315
+ t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
316
+ begin
317
+ child_compile(f[:path])
318
+ rescue StandardError => e
319
+ # a raise here would kill the pool thread and deadlock
320
+ # WorkersPool#shutdown; surface it after the pool drains
321
+ lock.synchronize { failures << e }
322
+ end
323
+ dt = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
324
+ lock.synchronize do
325
+ done += 1
326
+ warn format("[fragment-parallel] [%d/%d] %s %.1fs",
327
+ done, fragments.size, File.basename(f[:path]), dt)
328
+ end
329
+ end
330
+ end
331
+ pool.shutdown
332
+ raise failures.first unless failures.empty?
333
+ end
334
+
335
+ def child_compile(path)
336
+ env = {
337
+ "METANORMA_PARALLEL_CLAUSES" => nil,
338
+ "METANORMA_PARALLEL_CHILD_REQUIRES" => @requires.join(","),
339
+ }
340
+ out, err, status = Open3.capture3(env, RbConfig.ruby, "-e",
341
+ CHILD_SCRIPT, path, chdir: @dir)
342
+ File.write("#{path}.log", [out, err].join, encoding: "UTF-8")
343
+ return if status.success?
344
+
345
+ raise "fragment compile failed (#{status.exitstatus}): #{path}," \
346
+ " see #{path}.log"
347
+ end
348
+
349
+ # -- stitching
350
+
351
+ def stitch(paths)
352
+ base = parse(paths.first)
353
+ ns = base.root.namespace&.href
354
+ paths[1..].each do |p|
355
+ stitch_fragment_root(base, parse(p), ns)
356
+ end
357
+ reorder_root(base)
358
+ repair_concept_refs(base, ns)
359
+ base.to_xml
360
+ end
361
+
362
+ # Concept cross-references whose target term lives in another
363
+ # fragment fail resolution inside that fragment and come back as
364
+ # <errormsg> payloads carrying the reference texts verbatim. This
365
+ # pass repairs the unqualified subset: when the referenced id (or
366
+ # its glossarist "term-" prefixed anchor) names a term in the
367
+ # assembled document, the resolved refterm/renderterm/xref form is
368
+ # reconstructed. Domain-qualified references (the bulk in the
369
+ # iso-10303-2 corpus) resolve through the glossarist dataset, which
370
+ # the stitched document does not carry -- those remain errormsg and
371
+ # are the spike's principal known gap.
372
+ def repair_concept_refs(base, ns)
373
+ terms = base.root.xpath(".//xmlns:term[@anchor]", xmlns: ns)
374
+ .each_with_object({}) { |t, m| m[t["anchor"]] = t }
375
+ repaired = 0
376
+ base.root.xpath(".//xmlns:concept/xmlns:errormsg", xmlns: ns)
377
+ .each do |err|
378
+ tts = err.xpath(".//xmlns:tt", xmlns: ns).map(&:text)
379
+ next unless tts.size == 3
380
+
381
+ # the error payload carries the raw concept id; the rendered
382
+ # anchor carries the glossarist "term-" prefix
383
+ id = tts[2]
384
+ id = "term-#{id}" unless terms.key?(id)
385
+ next unless terms.key?(id)
386
+
387
+ refterm = base.document.create_element("refterm")
388
+ refterm.content = tts[0]
389
+ renderterm = base.document.create_element("renderterm")
390
+ renderterm.content = tts[1]
391
+ xref = base.document.create_element("xref")
392
+ xref["target"] = id
393
+ err.replace(refterm)
394
+ refterm.next = renderterm
395
+ renderterm.next = xref
396
+ repaired += 1
397
+ end
398
+ warn "[fragment-parallel] repaired #{repaired} cross-fragment" \
399
+ " concept references" if repaired.positive?
400
+ end
401
+
402
+ # Merge one fragment's root into +base+. Layout note: standoc emits
403
+ # root children adjacent, with a newline after the root open tag and
404
+ # one before the root close; whitespace-only text travels with the
405
+ # element it precedes/follows, so appends reproduce the fragment's
406
+ # own layout at the splice point.
407
+ def stitch_fragment_root(base, d, ns)
408
+ pairs = []
409
+ ws = +""
410
+ d.root.children.each do |c|
411
+ if c.text?
412
+ ws << c.text
413
+ else
414
+ pairs << [ws, c]
415
+ ws = +""
416
+ end
417
+ end
418
+ last = nil
419
+ pairs.each do |lead, el|
420
+ name = el.name
421
+ if SKIP_ROOT.include?(name)
422
+ last = :skip
423
+ elsif MERGE_ROOT.include?(name) &&
424
+ (t = base.root.at_xpath("./xmlns:#{name}", xmlns: ns))
425
+ merge_children(t, el, ns, name)
426
+ last = :merge
427
+ else
428
+ drop_trailing_ws(base.root)
429
+ base.root.add_child(
430
+ base.root.document.create_text_node(lead)
431
+ ) unless lead.empty?
432
+ adopt(base, el, ns).each { |n| base.root.add_child(n) }
433
+ last = :append
434
+ end
435
+ end
436
+ return unless last == :append && !ws.empty?
437
+
438
+ drop_trailing_ws(base.root)
439
+ base.root.add_child(base.root.document.create_text_node(ws))
440
+ end
441
+
442
+ def drop_trailing_ws(root)
443
+ t = root.children.last
444
+ t.remove if t&.text? && t.text.strip.empty?
445
+ end
446
+
447
+ # Root children are emitted in the canonical semantic-XML order;
448
+ # each element keeps the whitespace-only text node that preceded it.
449
+ # Everything is detached and re-added in order: libxml2's in-place
450
+ # node moves do not reliably reposition text nodes.
451
+ def reorder_root(base)
452
+ pairs = []
453
+ ws = nil
454
+ base.root.children.each do |c|
455
+ if c.text? && c.text.strip.empty?
456
+ ws = c
457
+ else
458
+ pairs << [ws, c]
459
+ ws = nil
460
+ end
461
+ end
462
+ rank = ->(p, i) { [ROOT_ORDER.index(p[1].name) || ROOT_ORDER.size, i] }
463
+ ordered = pairs.sort_by.with_index { |p, i| rank.call(p, i) }
464
+ base.root.children.each(&:remove)
465
+ ordered.each do |lead, el|
466
+ base.root.add_child(
467
+ base.root.document.create_text_node(lead.text)
468
+ ) if lead
469
+ base.root.add_child(el)
470
+ end
471
+ base.root.add_child(
472
+ base.root.document.create_text_node(ws.text)
473
+ ) if ws
474
+ end
475
+
476
+ def parse(path)
477
+ Nokogiri::XML(File.read(path, encoding: "UTF-8"), &:huge)
478
+ end
479
+
480
+ # Copy `el` (from another document) into `base`'s document, dropping
481
+ # the default-namespace declaration so it inherits the target's.
482
+ def adopt(base, el, ns)
483
+ s = el.to_xml
484
+ s = s.gsub(%r{ xmlns="#{Regexp.quote(ns)}"}, "") if ns
485
+ Nokogiri::XML::DocumentFragment.new(base.document, s).children
486
+ end
487
+
488
+ def merge_children(base_el, frag_el, ns, rootname)
489
+ skip = rootname == "preface" ? NONBASE_PREFACE_SKIP : []
490
+ pending = +""
491
+ landed = false
492
+ frag_el.children.each do |child|
493
+ if child.text?
494
+ pending << child.text
495
+ next
496
+ end
497
+ twin = base_el.elements.find { |c| elem_key(c, ns) ==
498
+ elem_key(child, ns) }
499
+ if skip.include?(child.name)
500
+ pending = +""
501
+ elsif twin && WRAPPERS.include?(child.name)
502
+ base_el.add_child(
503
+ base_el.document.create_text_node(pending)
504
+ ) unless pending.empty?
505
+ pending = +""
506
+ landed = true
507
+ merge_children(twin, child, ns, rootname)
508
+ elsif twin
509
+ # duplicate generated metadata or boilerplate: keep base's
510
+ pending = +""
511
+ else
512
+ base_el.add_child(
513
+ base_el.document.create_text_node(pending)
514
+ ) unless pending.empty?
515
+ pending = +""
516
+ landed = true
517
+ adopt(base_el.document, child, ns).each do |n|
518
+ base_el.add_child(n)
519
+ end
520
+ end
521
+ end
522
+ return unless landed && !pending.empty?
523
+
524
+ # trailing whitespace of the fragment's own layout
525
+ base_el.add_child(base_el.document.create_text_node(pending))
526
+ end
527
+
528
+ # Wrapper clauses of the same mega-clause carry the same explicit
529
+ # anchor, or -- when the source names no anchor -- the same title.
530
+ def elem_key(el, ns)
531
+ return [el.name, "##{el['anchor']}"] if el["anchor"]
532
+
533
+ t = ns ? el.at_xpath("./xmlns:title", xmlns: ns) : el.at("./title")
534
+ [el.name, t&.text]
535
+ end
536
+
537
+ def cleanup
538
+ return if ENV["METANORMA_PARALLEL_KEEP"] == "1"
539
+
540
+ # every artifact of this run carries the run prefix (fragment
541
+ # sources, chunk data files, fragment XML, child logs, the
542
+ # glossarist per-document log)
543
+ Dir.glob("#{@dir}/#{@prefix}-*").each do |p|
544
+ File.unlink(p) if File.file?(p)
545
+ end
546
+ end
547
+ end
548
+ end
549
+ end
@@ -156,10 +156,10 @@ module Metanorma
156
156
  to: File.dirname(output_paths[:out]),
157
157
  presentation_xml: presentation)
158
158
  rescue LoadError => e
159
- @log.add_msg("MKO_OUTPUT" => { category: "MKO Output",
160
- error: "the mko output format requires " \
161
- "the metanorma-mko gem: %s",
162
- severity: 0 })
159
+ @log.add_msg(MKO_OUTPUT: { category: "MKO Output",
160
+ error: "the mko output format requires " \
161
+ "the metanorma-mko gem: %s",
162
+ severity: 0 })
163
163
  @log.add("MKO_OUTPUT", nil, params: [e.message])
164
164
  nil
165
165
  end
@@ -182,7 +182,7 @@ module Metanorma
182
182
  # behind; collect it before the next extension stacks its own.
183
183
  GC.start
184
184
  rescue StandardError => e
185
- isodoc_error_process(e, strict_error?(ext, isodoc_options), false)
185
+ isodoc_error_process(e)
186
186
  end
187
187
 
188
188
  # Process format directly from semantic XML
@@ -192,7 +192,7 @@ module Metanorma
192
192
  GC.start
193
193
  true # Return as Thread equivalent
194
194
  rescue StandardError => e
195
- isodoc_error_process(e, strict_error?(ext, isodoc_options), true)
195
+ isodoc_error_process(e)
196
196
  ext != :presentation
197
197
  end
198
198
 
@@ -214,31 +214,22 @@ module Metanorma
214
214
 
215
215
  private
216
216
 
217
- # A presentation failure is always strict: every downstream format
218
- # needs the presentation XML. The :strict option arrives as a
219
- # boolean from the CLI but as a string from document attributes, so
220
- # accept both (the two rescue sites previously each checked only
221
- # one form).
222
- def strict_error?(ext, isodoc_options)
223
- ext == :presentation ||
224
- [true, "true"].include?(isodoc_options[:strict])
225
- end
226
-
227
- # On the strict path the message is pooled into @errors for the
228
- # caller to surface (the standalone CLI reports the pool and
229
- # aborts; the collection renderer attributes it to the failed
230
- # member, #586). The class+message line is also printed here in all
231
- # cases, so it sits adjacent to its backtrace in the build log: a
232
- # pooled message surfaces only at end of run, and a bare backtrace
233
- # mid-log is undiagnosable. The exception class is retained because
234
- # for errors like Encoding::UndefinedConversionError it carries
235
- # half the diagnosis.
236
- def isodoc_error_process(err, strict, must_abort)
217
+ # An output stage that raises has not produced its artifact, so the
218
+ # failure must reach the caller however Compile is consumed: the
219
+ # message is pooled into @errors unconditionally (#601) — the
220
+ # standalone CLI reports the pool and aborts, so a missing artifact
221
+ # can never pair with exit code 0; the collection renderer
222
+ # attributes it to the failed member (#586). The class+message line
223
+ # is also printed here in all cases, so it sits adjacent to its
224
+ # backtrace in the build log: a pooled message surfaces only at end
225
+ # of run, and a bare backtrace mid-log is undiagnosable. The
226
+ # exception class is retained because for errors like
227
+ # Encoding::UndefinedConversionError it carries half the diagnosis.
228
+ def isodoc_error_process(err)
237
229
  msg = "#{err.class}: #{err.message}"
238
- strict || err.message.include?("Fatal:") and @errors << msg
230
+ @errors << msg
239
231
  puts msg
240
232
  puts err.backtrace.join("\n")
241
- must_abort and 1
242
233
  end
243
234
  end
244
235
  end
@@ -102,6 +102,25 @@ module Metanorma
102
102
  @@updated_formulas_repo = true
103
103
  install_fonts_safe(manifest, agree, continue, no_progress)
104
104
  end
105
+
106
+ # Fontist raises MissingFontError for any manifest font that is not
107
+ # installed locally, even after install_fonts_safe has honoured
108
+ # --continue-without-fonts for the install itself (#595): the flag's
109
+ # promise would be defeated one call later. Drop the missing font
110
+ # from the manifest with a warning, so mn2pdf renders with
111
+ # substituted fonts; without the flag the error stays fatal.
112
+ def locate_manifest(manifest, continue_without_fonts)
113
+ instance = Fontist::Manifest.from_hash(manifest, locations: true)
114
+ instance&.to_hash
115
+ rescue Fontist::Errors::MissingFontError => e
116
+ raise unless continue_without_fonts
117
+
118
+ Util.log("[fontist] '#{e.font}' font is missing. " \
119
+ "Continuing without it (--continue-without-fonts).",
120
+ :warning)
121
+ rest = manifest.reject { |name, _| name == e.font }
122
+ rest.empty? ? {} : locate_manifest(rest, continue_without_fonts)
123
+ end
105
124
  end
106
125
 
107
126
  def self.install_fonts(processor, options)
@@ -132,7 +151,8 @@ module Metanorma
132
151
  || source_attributes[:fonts]
133
152
  end
134
153
 
135
- def self.location_manifest(processor, source_attributes)
154
+ def self.location_manifest(processor, source_attributes,
155
+ continue_without_fonts: false)
136
156
  base = if processor.respond_to?(:fonts_manifest) &&
137
157
  !processor.fonts_manifest.nil?
138
158
  processor.fonts_manifest.dup
@@ -142,12 +162,8 @@ module Metanorma
142
162
 
143
163
  return nil if base.empty? && !source_attributes[:fonts]
144
164
 
145
- instance = Fontist::Manifest.from_hash(
146
- append_source_fonts(base, source_attributes),
147
- locations: true,
148
- )
149
-
150
- instance.to_hash unless instance.nil?
165
+ locate_manifest(append_source_fonts(base, source_attributes),
166
+ continue_without_fonts)
151
167
  end
152
168
 
153
169
  def self.append_source_fonts(manifest, source_attributes)
@@ -11,6 +11,8 @@ module Metanorma
11
11
  def init_vars(workers)
12
12
  @workers = workers
13
13
  @queue = SizedQueue.new(@workers)
14
+ @mutex = Mutex.new
15
+ @exceptions = []
14
16
  end
15
17
 
16
18
  def init_thread
@@ -18,21 +20,38 @@ module Metanorma
18
20
  catch(:exit) do
19
21
  loop do
20
22
  job, args = @queue.pop
21
- job.call *args
23
+ run_job job, args
22
24
  end
23
25
  end
24
26
  end
25
27
  end
26
28
 
29
+ # A job that raises must not take its worker thread down: the
30
+ # thread stays in the loop, so jobs scheduled behind the failure
31
+ # still run, and the exception is held for shutdown to re-raise
32
+ # in the calling thread (#601). A dead worker here would deadlock
33
+ # shutdown on a saturated queue and strand the remaining jobs.
34
+ def run_job(job, args)
35
+ job.call *args
36
+ rescue StandardError => e
37
+ @mutex.synchronize { @exceptions << e }
38
+ end
39
+
27
40
  def schedule(*args, &block)
28
41
  @queue << [block, args]
29
42
  end
30
43
 
44
+ # Drain the queue, join every worker, then propagate the first
45
+ # job exception to the caller: a raising job must fail the run
46
+ # that scheduled it, never exit 0.
31
47
  def shutdown
32
48
  @workers.times do
33
49
  schedule { throw :exit }
34
50
  end
35
51
  @threads.map(&:join)
52
+ return if @exceptions.empty?
53
+
54
+ raise @exceptions.first
36
55
  end
37
56
  end
38
57
  end
@@ -1,3 +1,3 @@
1
1
  module Metanorma
2
- VERSION = "2.5.7".freeze
2
+ VERSION = "2.5.8".freeze
3
3
  end
data/metanorma.gemspec CHANGED
@@ -26,12 +26,16 @@ Gem::Specification.new do |spec|
26
26
 
27
27
  spec.add_runtime_dependency "asciidoctor"
28
28
  spec.add_runtime_dependency "concurrent-ruby"
29
- spec.add_runtime_dependency "fontist", ">= 2.1.2"
29
+ spec.add_runtime_dependency "fontist", "~> 3.0"
30
30
  spec.add_runtime_dependency "htmlentities"
31
- spec.add_runtime_dependency "isodoc", ">= 3.5.0"
31
+ spec.add_runtime_dependency "isodoc", "~> 3.5"
32
32
  spec.add_runtime_dependency "marcel"
33
33
  spec.add_runtime_dependency "metanorma-core"
34
- spec.add_runtime_dependency "metanorma-standoc", ">= 3.5.0"
34
+ # metanorma-plugin-lutaml is required by Metanorma::Compile: the XMI
35
+ # macro caches are flushed at the conversion/output phase boundary
36
+ # (CacheRegistry shipped in 0.7.46)
37
+ spec.add_runtime_dependency "metanorma-plugin-lutaml", "~> 0.7.46"
38
+ spec.add_runtime_dependency "metanorma-standoc", "~> 3.5"
35
39
  spec.add_runtime_dependency "mn2pdf", "~> 2"
36
40
  spec.add_runtime_dependency "nokogiri"
37
41
  spec.add_development_dependency "canon" #, "= 0.2.3"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: metanorma
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.7
4
+ version: 2.5.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-10-30 00:00:00.000000000 Z
11
+ date: 2026-11-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asciidoctor
@@ -42,16 +42,16 @@ dependencies:
42
42
  name: fontist
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: 2.1.2
47
+ version: '3.0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ">="
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: 2.1.2
54
+ version: '3.0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: htmlentities
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -70,16 +70,16 @@ dependencies:
70
70
  name: isodoc
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ">="
73
+ - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: 3.5.0
75
+ version: '3.5'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ">="
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: 3.5.0
82
+ version: '3.5'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: marcel
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -108,20 +108,34 @@ dependencies:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: metanorma-plugin-lutaml
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: 0.7.46
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: 0.7.46
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: metanorma-standoc
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
115
- - - ">="
129
+ - - "~>"
116
130
  - !ruby/object:Gem::Version
117
- version: 3.5.0
131
+ version: '3.5'
118
132
  type: :runtime
119
133
  prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
- - - ">="
136
+ - - "~>"
123
137
  - !ruby/object:Gem::Version
124
- version: 3.5.0
138
+ version: '3.5'
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: mn2pdf
127
141
  requirement: !ruby/object:Gem::Requirement
@@ -264,6 +278,7 @@ files:
264
278
  - lib/metanorma/compile/compile_options.rb
265
279
  - lib/metanorma/compile/extract.rb
266
280
  - lib/metanorma/compile/flavor.rb
281
+ - lib/metanorma/compile/fragment_parallel.rb
267
282
  - lib/metanorma/compile/output_filename.rb
268
283
  - lib/metanorma/compile/output_filename_config.rb
269
284
  - lib/metanorma/compile/relaton_drop.rb