metanorma 2.5.17 → 2.5.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e74645aab84c8445fdcbdab1190af0c3e885aa315b987d9f608c6633d2453fd0
4
- data.tar.gz: e71224bccf1879c5356ad8420b638d31de3bfa4f602fbed210328990965c1e2b
3
+ metadata.gz: 677ac7e8316fc02998475e7d8032cdc1e33cdd114095138017460aacb8b25e56
4
+ data.tar.gz: b3ac4529a4400cd837c81360d6c59dcae4c2fa9ce87c72a1a7c645c684743e2f
5
5
  SHA512:
6
- metadata.gz: e499614dc2d81c316a92908f5bd22e1fff19b92095ccbabff70a1411bd011e4e7fbef871a490c9ed3d17d76d3ed20ceebcf6c967b7503a6ca4a2756dfe974eeb
7
- data.tar.gz: 56c95bb8a97425b3b6f3106f91fd5c92fba09383d135251fc2c4998c35f4ba88244b849b9f11a23eb08d8280699cdcb58a12cc448db91d6658b95305c3408104
6
+ metadata.gz: 3219348c35be5e4108e6fa3c167c9d2a22d720c81839e5e99189f72ea220170883306cd6db8072ad61d190dc20e892bfbe8dfb110bf0c4233bd16ab53e19ebec
7
+ data.tar.gz: 85d307b34b25264c0c2df025c66826fb6a68057026c1b41b34d4e7711725ea8c4e837c6df173f41c7c78a827214055dbf12b539fdb791b2cc49077aad0b7ae4d
data/Gemfile CHANGED
@@ -10,14 +10,14 @@ gem "relaton", "~> 3.0"
10
10
  gem "metanorma-core", ">= 0.2.5"
11
11
 
12
12
  # taste stack the compile specs exercise: the oiml taste registers the
13
- # flavor and lazily requires metanorma-oiml (OIML STS transformers,
14
- # unreleased); mko main carries the model-side export entry.
13
+ # flavor and lazily requires metanorma-oiml (OIML STS transformers);
14
+ # mko carries the model-side export entry.
15
15
  # Lives here, not in Gemfile.devel: consumer workflows may rewrite the
16
16
  # devel overlay (fontist CI appends its path pin to it), and the repo's
17
17
  # own compile specs must not depend on its survival.
18
18
  gem "metanorma-taste", "~> 1.1"
19
- gem "metanorma-oiml", github: "metanorma/metanorma-oiml", branch: "main"
20
- gem "metanorma-mko", github: "metanorma/metanorma-mko", branch: "main"
19
+ gem "metanorma-oiml", "~> 0.1"
20
+ gem "metanorma-mko", "~> 1.1"
21
21
 
22
22
  begin
23
23
  eval_gemfile("Gemfile.devel")
data/Gemfile.devel CHANGED
@@ -1,13 +1,10 @@
1
1
  # Local development pins (eval'd by Gemfile; not committed workflow state)
2
- gem "metanorma-document", github: "metanorma/metanorma-document", branch: "main"
3
- gem "metanorma-standoc", github: "metanorma/metanorma-standoc", branch: "main"
4
- gem "metanorma-iso", github: "metanorma/metanorma-iso", branch: "main"
5
- gem "isodoc", github: "metanorma/isodoc", branch: "main"
6
- # released metanorma-iho 1.4.0 still carries lib/relaton/render/general.rb,
7
- # which shadows relaton-render's iso690 autoload by load-path name; iho
8
- # main relocated its render files
9
- gem "metanorma-iho", github: "metanorma/metanorma-iho", branch: "main"
10
- gem "pubid", "~> 2.0.0.pre.alpha"
2
+ gem "metanorma-document", "~> 0.5"
3
+ gem "metanorma-standoc", "~> 3.5"
4
+ gem "metanorma-iso", "~> 3.5"
5
+ gem "isodoc", "~> 3.7"
6
+ gem "metanorma-iho", "~> 1.4"
7
+ gem "pubid", "~> 2.0"
11
8
  # The 1.9.356/0.5.13x pin pair worked around the leptris#1733 to_xml
12
9
  # regression; retired — moxml rides the 0.6 line (0.5 series gets no
13
10
  # fixes; 0.6.0 removed the Opal surface, lutaml/moxml#516), which
@@ -49,7 +49,7 @@ module Metanorma
49
49
  ids, docxml, isodoc, tag = svgmap_resolve_prep(docxml, docid, presxml)
50
50
 
51
51
  # Stage 1: Resolve EREF references to their targets
52
- resolve_svgmap_erefs(docxml, tag, isodoc, ids, presxml)
52
+ resolve_svgmap_erefs(docxml, tag, isodoc, ids, presxml, docid)
53
53
 
54
54
  # Stage 1b: Re-key the svgmap xref targets that a reinflated render
55
55
  # carries from the staging build: they name anchors in the compile-
@@ -100,9 +100,9 @@ module Metanorma
100
100
  private
101
101
 
102
102
  # Resolves EREF elements within svgmap to their actual link targets
103
- def resolve_svgmap_erefs(docxml, tag, isodoc, ids, presxml)
103
+ def resolve_svgmap_erefs(docxml, tag, isodoc, ids, presxml, docid)
104
104
  docxml.xpath(ns("//svgmap//#{tag}")).each do |e|
105
- svgmap_resolve_eref(e, isodoc, docxml, ids, presxml)
105
+ svgmap_resolve_eref(e, isodoc, docxml, ids, presxml, docid)
106
106
  end
107
107
  end
108
108
 
@@ -160,16 +160,35 @@ module Metanorma
160
160
  end
161
161
 
162
162
  # Resolves a single EREF element to its target link
163
- def svgmap_resolve_eref(eref, isodoc, _docxml, ids, presxml)
163
+ def svgmap_resolve_eref(eref, isodoc, _docxml, ids, presxml, docid)
164
164
  href = isodoc.eref_target(eref) or return
165
165
  href = href[:link]
166
166
  href == "##{eref['bibitemid']}" ||
167
167
  (href =~ /^#/ && !ids[href.sub(/^#/, "")]) and return
168
+ href = remap_cross_member_hotspot(href, docid)
168
169
  eref["target"] = href.strip
169
170
  eref.name = presxml ? "fmt-link" : "link"
170
171
  eref.elements&.remove
171
172
  end
172
173
 
174
+ # A cross-member svgmap target composes its anchor as authored
175
+ # (<hotspot>_<doc-stamp>), while the target page's SVG ids carry
176
+ # Vectory's disambiguation index (<hotspot>_<counter>) - the two
177
+ # conventions disagree exactly when the members render separately
178
+ # under staging (iso-10303 run 42346461599: 15 fragments broken).
179
+ # Re-key the anchor through the hotspot anchor index: exact, then the
180
+ # anchor with a known member-suffix stripped. Only same-file
181
+ # re-keys are taken; anything else is left untouched.
182
+ def remap_cross_member_hotspot(href, docid)
183
+ href =~ /\A([^#]+\.html)#(.+)\z/ or return href
184
+ file, anchor = Regexp.last_match(1), Regexp.last_match(2)
185
+ rec = hotspot_anchor_index[anchor] ||
186
+ hotspot_anchor_index[hotspot_anchor_base(anchor, docid)] or
187
+ return href
188
+ target = hotspot_target_href(rec, docid) or return href
189
+ target.start_with?("#{file}#") ? target : href
190
+ end
191
+
173
192
  # Re-key svgmap xref targets whose anchor lives outside the current
174
193
  # section page. The svg_xrefs pass composes these targets as
175
194
  # +<xref target='anchor_suffix'/>+ from the SVG hotspot hrefs, naming
@@ -337,16 +356,30 @@ module Metanorma
337
356
  else
338
357
  id
339
358
  end
340
- if !idx.key?(base)
341
- idx[base] = { key: key, id: final, url: url }
342
- elsif idx[base] && idx[base][:id] != final
343
- idx[base] = nil
344
- end
359
+ register_hotspot_base(idx, base, key: key, id: final, url: url)
360
+ # SVG ids additionally carry Vectory's disambiguation index
361
+ # (<orig>_<counter>): the stored id is the assigned form, while
362
+ # cross-member svgmap links compose the anchor from the authored
363
+ # original. Register the original as a second base candidate;
364
+ # the ambiguity guard still nils true collisions.
365
+ stripped = base.sub(/_\d+\z/, "")
366
+ register_hotspot_base(idx, stripped, key: key, id: final,
367
+ url: url) if stripped != base
345
368
  end
346
369
  end
347
370
  idx
348
371
  end
349
372
 
373
+ def register_hotspot_base(idx, base, record)
374
+ return if base.empty?
375
+
376
+ if !idx.key?(base)
377
+ idx[base] = record
378
+ elsif idx[base] && idx[base][:id] != record[:id]
379
+ idx[base] = nil
380
+ end
381
+ end
382
+
350
383
  # The member page's URL relative to the page being rendered; the index
351
384
  # stores it collection-root-relative.
352
385
  def hotspot_relative_url(url, docid)
@@ -29,7 +29,6 @@ module Metanorma
29
29
  include Flavor
30
30
  include Writeable
31
31
 
32
- autoload :FragmentParallel, "metanorma/compile/fragment_parallel"
33
32
 
34
33
  DEFAULT_NUM_WORKERS = 3
35
34
 
@@ -210,15 +209,6 @@ module Metanorma
210
209
  end
211
210
 
212
211
  def process_input_adoc(filename, options)
213
- if (w = ENV["METANORMA_PARALLEL_CLAUSES"].to_i) >= 2
214
- # Fragment-based parallel semantic compile (spike, #620): split,
215
- # compile the fragments in worker processes, stitch. The stitched
216
- # semantic XML rejoins the ordinary output pipeline below.
217
- Util.log("[metanorma] Processing: AsciiDoc input" \
218
- " (fragment-parallel, #{w} workers).", :info)
219
- return [read_file(filename),
220
- FragmentParallel.new(filename, options, w).run]
221
- end
222
212
  Util.log("[metanorma] Processing: AsciiDoc input.", :info)
223
213
  file = read_file(filename)
224
214
  file = process_input_adoc_hdr(file, options)
@@ -1,3 +1,3 @@
1
1
  module Metanorma
2
- VERSION = "2.5.17".freeze
2
+ VERSION = "2.5.19".freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: metanorma
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.17
4
+ version: 2.5.19
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-11-07 00:00:00.000000000 Z
11
+ date: 2026-11-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asciidoctor
@@ -278,7 +278,6 @@ files:
278
278
  - lib/metanorma/compile/compile_options.rb
279
279
  - lib/metanorma/compile/extract.rb
280
280
  - lib/metanorma/compile/flavor.rb
281
- - lib/metanorma/compile/fragment_parallel.rb
282
281
  - lib/metanorma/compile/output_filename.rb
283
282
  - lib/metanorma/compile/output_filename_config.rb
284
283
  - lib/metanorma/compile/relaton_drop.rb
@@ -1,739 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "fileutils"
4
- require "yaml"
5
-
6
- module Metanorma
7
- class Compile
8
- # Spike: fragment-based parallel semantic compilation (metanorma#620).
9
- #
10
- # Opt in with METANORMA_PARALLEL_CLAUSES=<N> (N >= 2; default off). The
11
- # AsciiDoc source is split into clause fragments, each fragment is
12
- # compiled to semantic XML by a worker forked off this process, and
13
- # the fragment XMLs are stitched back into one semantic document that
14
- # the ordinary output pipeline consumes as today.
15
- #
16
- # Forking over spawning: a fresh interpreter per fragment re-pays the
17
- # flavor boot and -- fatally for vocabulary corpora -- the glossarist
18
- # dataset load (tens of seconds per fragment). A forked worker
19
- # inherits everything the parent already loaded. Before the first
20
- # fork, the parent preloads the document's glossarist datasets once
21
- # (opt-in plugin store cache, METANORMA_GLOSSARIST_SHARED_DATASETS),
22
- # so the workers read the loaded models copy-on-write instead of
23
- # re-parsing them.
24
- #
25
- # Split rules:
26
- # - includes are resolved inline, recursively;
27
- # - text before the first top-level heading (document header, preamble,
28
- # floating-title forewords) belongs to the first fragment;
29
- # - each top-level (`==`) clause is one fragment;
30
- # - a clause whose body is a single `[yaml2text,<file>,<var>]` block
31
- # over an array-valued YAML file is split further: each fragment
32
- # repeats the clause's heading chain and the block template verbatim,
33
- # but feeds it a generated YAML file holding one chunk of entries.
34
- # That is what makes vocabulary corpora (per-schema terms loops)
35
- # parallelisable without understanding their expansion.
36
- #
37
- # Stitching merges root-level children by element name: preface,
38
- # sections, bibliography and metanorma-extension children are merged
39
- # (wrapper clauses carrying the same anchor or title are unified;
40
- # non-wrapper duplicates are dropped), annex/indexsect/colophon are
41
- # appended in source order. Auto-generated boilerplate (foreword,
42
- # boilerplate) is taken from the first fragment only.
43
- #
44
- # Known spike gaps: per-fragment validation is skipped (the stitched
45
- # document flows through the normal pipeline, where validation and
46
- # presentation run as today); bibitem pruning that keys on citations
47
- # sees only the owning fragment's citations; shorthand `<<bibitem>>`
48
- # references to bibitems declared in another fragment degrade to
49
- # xref; glossarist <concept> cross-references to terms owned by
50
- # another fragment degrade to <errormsg> unless the reference is
51
- # unqualified and the repair pass can reconstruct it (domain-qualified
52
- # concept resolution needs the glossarist dataset, which no fragment
53
- # holds for the whole document); generated element ids differ between
54
- # any two compiles, so stitched output is canon-equivalent to a serial
55
- # compile (identical modulo generated ids and whitespace-only text at
56
- # splice points), not byte-identical.
57
- class FragmentParallel
58
- HEADING1 = /^==(?!=)/.freeze
59
- STYLE_ATTR = /^(\[\[.+?\]\]|\[[^\[\]]+\])$/.freeze
60
- FRAME_TITLE = /^\.[^\s]/.freeze
61
- INCLUDE = /^include::(\S+?)\[\]\s*$/.freeze
62
- # [yaml2text,file,var] and [data2text,var=file]
63
- DATA2TEXT = /\[(yaml2text|data2text),(.+?)\]\s*$/.freeze
64
- MAX_INCLUDE_DEPTH = 8
65
-
66
- # Mirrors the plugin's DatasetPreprocessor header directive.
67
- DATASET_ATTR = /^:glossarist-dataset:\s*(.*)$/.freeze
68
-
69
- # The metanorma-plugin-glossarist store-cache gate (its
70
- # DatasetRegistry::SHARED_STORE_ENV; spelled out here because the
71
- # plugin may be absent entirely).
72
- SHARED_DATASETS_ENV = "METANORMA_GLOSSARIST_SHARED_DATASETS"
73
-
74
- SKIP_ROOT = %w[bibdata boilerplate].freeze
75
- MERGE_ROOT = %w[preface sections bibliography
76
- metanorma-extension].freeze
77
- WRAPPERS = %w[clause terms references introduction].freeze
78
- NONBASE_PREFACE_SKIP = %w[foreword].freeze
79
- # Canonical root-child order of metanorma semantic XML (standoc emits
80
- # bibliography last, annexes after sections); the stitch reorders to
81
- # match, since fragments append in source order.
82
- ROOT_ORDER = %w[bibdata metanorma-extension boilerplate preface
83
- sections annex indexsect colophon bibliography].freeze
84
-
85
- def initialize(filename, _options, workers)
86
- @filename = filename
87
- @workers = workers
88
- @dir = File.realpath(File.dirname(File.expand_path(filename)))
89
- @base = File.basename(filename, ".*")
90
- @prefix = ".#{@base}.frag"
91
- end
92
-
93
- def run
94
- text = resolve_includes(
95
- File.read(@filename, encoding: "utf-8"), 0
96
- )
97
- fragments = build_fragments(text)
98
- warn "[fragment-parallel] #{@filename}: #{fragments.size} fragments," \
99
- " #{@workers} workers"
100
- warm_shared_datasets(text)
101
- compile_fragments(fragments)
102
- t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
103
- xml = stitch(fragments.map { |f| f[:path].sub(/\.adoc$/, ".xml") })
104
- warn format("[fragment-parallel] stitched in %.1fs",
105
- Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
106
- xml
107
- ensure
108
- cleanup
109
- end
110
-
111
- private
112
-
113
- # -- splitting
114
-
115
- def build_fragments(text)
116
- lines = text.split("\n")
117
- header, segments = segmentize(lines)
118
- fragments = []
119
- packer = fragment_packer(fragments, header)
120
- segments.each do |seg|
121
- if (mega = data2text_mega(seg))
122
- packer.flush
123
- mega_fragments(fragments, header, seg, mega)
124
- else
125
- packer.feed(seg[:lines])
126
- end
127
- end
128
- packer.flush
129
- fragments
130
- end
131
-
132
- # Pack consecutive segments into fragments of at least
133
- # MIN_FRAG_BYTES source bytes: one fragment per clause multiplies
134
- # the per-fragment fixed cost (relaton init, boilerplate, YAML
135
- # loads — measured ~4s floor) by the clause count, which is a
136
- # large LOSS on clause-dense documents with cheap clauses
137
- # (jis-z-8301: 52 fragments, 2x slower than serial). Packing
138
- # targets fragments substantial enough to amortise the fixed
139
- # cost over workers x 2-3 fragments; documents whose clauses are
140
- # already large are unaffected (one segment per fragment).
141
- MIN_FRAG_BYTES = 40_000
142
-
143
- # Accumulates consecutive whole segments across clause boundaries
144
- # and emits fragments of at least MIN_FRAG_BYTES source bytes.
145
- # Segments are never split: a boundary inside a clause separates
146
- # the clause's heading from its subsections, and both halves then
147
- # compile into morphed structures that lose their nested content.
148
- class FragmentPacker
149
- def initialize(min_bytes, &emit)
150
- @min_bytes = min_bytes
151
- @emit = emit
152
- @buf = []
153
- @bytes = 0
154
- end
155
-
156
- def feed(seg_lines)
157
- seg_bytes = seg_lines.sum { |l| l.bytesize + 1 }
158
- if @bytes.positive? && @bytes >= @min_bytes
159
- flush
160
- end
161
- @buf.concat(seg_lines)
162
- @bytes += seg_bytes
163
- if @bytes >= @min_bytes
164
- flush
165
- end
166
- end
167
-
168
- def flush
169
- return if @buf.empty?
170
-
171
- @emit.call(@buf.dup)
172
- @buf = []
173
- @bytes = 0
174
- end
175
- end
176
-
177
- def fragment_packer(fragments, header, min_bytes: MIN_FRAG_BYTES)
178
- FragmentPacker.new(min_bytes) do |body_lines|
179
- fragments << new_fragment(fragments.size, header, body_lines)
180
- end
181
- end
182
-
183
- def new_fragment(i, header, body_lines)
184
- path = File.join(@dir, "#{@prefix}-#{i}.adoc")
185
- File.write(path, (header + body_lines).join("\n").concat("\n"),
186
- encoding: "UTF-8")
187
- { path: path, lines: header + body_lines }
188
- end
189
-
190
- # Returns [header_lines, [{lines:}, ...]]; header is everything
191
- # before the first segment boundary. A boundary opens at a top-level
192
- # heading, an include, or a run of attribute/anchor/frame-title lines
193
- # immediately preceding one; the heading itself is part of that
194
- # segment and does not open another.
195
- def segmentize(lines)
196
- bounds = []
197
- i = 0
198
- while i < lines.size
199
- if boundary_start(lines, i)
200
- # the boundary opens at the attribute-run start (or the bare
201
- # heading); blank lines separating the previous clause travel
202
- # with the fragment that follows them, so inter-clause
203
- # spacing survives the split
204
- b = i
205
- b -= 1 while b.positive? && lines[b - 1].strip.empty?
206
- bounds << b
207
- i = boundary_start_resume(lines, i) || lines.size
208
- else
209
- i += 1
210
- end
211
- end
212
- return [[], [{ lines: lines }]] if bounds.empty?
213
-
214
- header = lines[0...bounds.first]
215
- segs = bounds.each_with_index.map do |b, k|
216
- { lines: lines[b...(bounds[k + 1] || lines.size)] }
217
- end
218
- [header, segs]
219
- end
220
-
221
- # Index where the boundary scan resumes: past the heading/include
222
- # that closes the attribute run opened at +i+.
223
- def boundary_start_resume(lines, i)
224
- return i + 1 if seg_boundary_line?(lines[i])
225
-
226
- j = i
227
- loop do
228
- j += 1
229
- j += 1 while j < lines.size && lines[j].strip.empty?
230
- return j + 1 if j < lines.size && seg_boundary_line?(lines[j])
231
- return nil unless j < lines.size &&
232
- (lines[j].match?(STYLE_ATTR) ||
233
- lines[j].match?(FRAME_TITLE))
234
- end
235
- end
236
-
237
- # Index of the heading/include that closes the attribute run
238
- # opening at line +i+, or nil when lines[i] opens no boundary.
239
- def boundary_start(lines, i)
240
- return i if seg_boundary_line?(lines[i])
241
-
242
- return nil unless lines[i].match?(STYLE_ATTR) ||
243
- lines[i].match?(FRAME_TITLE)
244
-
245
- j = i
246
- loop do
247
- j += 1
248
- j += 1 while j < lines.size && lines[j].strip.empty?
249
- return nil if j >= lines.size
250
- return j if seg_boundary_line?(lines[j])
251
- return nil unless lines[j].match?(STYLE_ATTR) ||
252
- lines[j].match?(FRAME_TITLE)
253
- end
254
- end
255
-
256
- def seg_boundary_line?(line)
257
- line.match?(HEADING1) || line.match?(INCLUDE)
258
- end
259
-
260
- def resolve_includes(text, depth, base = @dir)
261
- return text if depth >= MAX_INCLUDE_DEPTH
262
-
263
- text.gsub(INCLUDE) do
264
- inc = Regexp.last_match[1]
265
- path = File.expand_path(inc, base)
266
- if File.exist?(path)
267
- resolve_includes(File.read(path, encoding: "utf-8"), depth + 1,
268
- File.dirname(path))
269
- else
270
- Regexp.last_match[0]
271
- end
272
- end
273
- end
274
-
275
- # A segment is mega-splittable when it holds exactly one
276
- # yaml2text/data2text open block whose data file parses to an
277
- # array of at least two entries.
278
- def data2text_mega(seg)
279
- attr_i = seg[:lines].each_index.find do |i|
280
- seg[:lines][i].match?(DATA2TEXT)
281
- end
282
- return nil unless attr_i
283
-
284
- m = seg[:lines][attr_i].match(DATA2TEXT)
285
- delim = seg[:lines][attr_i + 1]
286
- return nil unless delim && delim.match?(/^-{4,}$/)
287
-
288
- close = ((attr_i + 2)...seg[:lines].size).find do |j|
289
- seg[:lines][j] == delim
290
- end
291
- return nil unless close
292
-
293
- file, var = parse_data2text_args(m[2])
294
- return nil unless file && var
295
-
296
- data_path = File.expand_path(file, @dir)
297
- return nil unless File.exist?(data_path)
298
-
299
- begin
300
- entries = YAML.safe_load(File.read(data_path))
301
- rescue Psych::SyntaxError
302
- return nil
303
- end
304
- return nil unless entries.is_a?(Array) && entries.size >= 2
305
-
306
- { attr_i: attr_i, close: close, delim: delim, kind: m[1],
307
- template: seg[:lines][(attr_i + 2)...close],
308
- var: var, entries: entries, data_path: data_path }
309
- end
310
-
311
- # "[yaml2text,]file,var" or "[data2text,]var=file" -> [file, var].
312
- # The data file is the argument naming an existing YAML file.
313
- def parse_data2text_args(raw)
314
- if raw.include?(",")
315
- a, b = raw.split(",", 2)
316
- [[b, a], [a, b]].each do |file, var|
317
- return [file, var] if data_file_arg?(file) && ident_arg?(var)
318
- end
319
- [nil, nil]
320
- elsif (p = raw.split("=", 2)).size == 2
321
- data_file_arg?(p[1]) && ident_arg?(p[0]) ? [p[1], p[0]] : [nil, nil]
322
- else
323
- [nil, nil]
324
- end
325
- end
326
-
327
- def data_file_arg?(arg)
328
- arg.match?(/\.ya?ml\z/i) && File.exist?(File.expand_path(arg, @dir))
329
- end
330
-
331
- def ident_arg?(arg)
332
- arg.match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/)
333
- end
334
-
335
- def mega_fragments(fragments, header, seg, mega)
336
- # one chunk per worker: the chunks left over after the first wave
337
- # land on workers that just finished, keeping every worker fed
338
- chunks = [mega[:entries].size, @workers].min
339
- size = (mega[:entries].size / chunks.to_f).ceil
340
- head = header + seg[:lines][0...mega[:attr_i]]
341
- base_idx = fragments.size
342
- mega[:entries].each_slice(size).with_index do |slice, i|
343
- idx = base_idx + i
344
- data_name = "#{@prefix}-#{idx}.data.yaml"
345
- data_path = File.join(@dir, data_name)
346
- File.write(data_path, YAML.dump(slice), encoding: "UTF-8")
347
- directive = if mega[:kind] == "data2text"
348
- "[data2text,#{mega[:var]}=#{data_name}]"
349
- else
350
- "[yaml2text,#{data_name},#{mega[:var]}]"
351
- end
352
- path = File.join(@dir, "#{@prefix}-#{idx}.adoc")
353
- File.write(path, [
354
- head,
355
- directive,
356
- mega[:delim], mega[:template], mega[:delim],
357
- ].flatten.join("\n").concat("\n"), encoding: "UTF-8")
358
- fragments << { path: path, mega: true }
359
- end
360
- end
361
-
362
- # -- dataset preload
363
-
364
- # Load the document's glossarist datasets once, in this process,
365
- # before any worker is forked. The plugin's opt-in store cache
366
- # (METANORMA_GLOSSARIST_SHARED_DATASETS) memoizes the parsed
367
- # datasets process-wide; the forked workers then answer their own
368
- # registry lookups from the shared, copy-on-write models instead of
369
- # re-parsing tens of thousands of YAML files per fragment.
370
- #
371
- # Skipped without the plugin, or when the datasets cannot be
372
- # resolved: the fragments then behave exactly as on the spike --
373
- # each loads what it needs, in its own fork.
374
- def warm_shared_datasets(text)
375
- return unless defined?(Metanorma::Plugin::Glossarist::DatasetRegistry)
376
- return unless Metanorma::Plugin::Glossarist::DatasetRegistry
377
- .respond_to?(:shared_store_for)
378
-
379
- paths = glossarist_dataset_paths(text)
380
- return if paths.empty?
381
-
382
- # An explicit "0" is a user opt-out: leave the plugin gate off
383
- # and skip the preload -- the workers re-load per fork, as before.
384
- gate = ENV.fetch(SHARED_DATASETS_ENV, nil)
385
- gate = ENV[SHARED_DATASETS_ENV] = "1" if gate.nil? || gate.empty?
386
- return if gate == "0"
387
-
388
- registry = Metanorma::Plugin::Glossarist::DatasetRegistry.new
389
- t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
390
- paths.each { |p| registry.concepts_at(p) }
391
- # Give the workers a compacted, shared heap to fork from.
392
- GC.start
393
- warn format("[fragment-parallel] preloaded %d glossarist dataset(s)" \
394
- " in %.1fs; workers share them copy-on-write",
395
- paths.size,
396
- Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
397
- rescue StandardError => e
398
- warn "[fragment-parallel] glossarist dataset preload skipped:" \
399
- " #{e.message}"
400
- end
401
-
402
- # Registered dataset paths in document-header order, resolved
403
- # against the docfile directory the same way the plugin resolves
404
- # them for a fragment (:glossarist-dataset: name:path;name:path).
405
- def glossarist_dataset_paths(text)
406
- text.scan(DATASET_ATTR)
407
- .flat_map { |(contexts)| contexts.split(";") }
408
- .filter_map do |entry|
409
- _, rel = entry.split(":", 2)&.map(&:strip)
410
- next unless rel && !rel.empty?
411
-
412
- path = File.expand_path(rel, @dir)
413
- File.directory?(path) ? path : nil
414
- end.uniq
415
- end
416
-
417
- # -- compilation
418
-
419
- # Bounded fork scheduler: keep @workers children alive until the
420
- # fragments run out, reap any exited child, then raise the first
421
- # failure after all scheduled work has drained.
422
- def compile_fragments(fragments)
423
- return compile_fragments_in_process(fragments) unless fork?
424
-
425
- queue = fragments.dup
426
- running = {}
427
- failure = nil
428
- until queue.empty? && running.empty?
429
- until queue.empty? || running.size >= @workers
430
- f = queue.shift
431
- begin
432
- t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
433
- running[spawn_fragment(f)] = [f, t]
434
- rescue StandardError => e
435
- failure ||= e.message
436
- end
437
- end
438
- break if running.empty?
439
-
440
- pid, status = Process.waitpid2(-1, 0)
441
- f, t = running.delete(pid)
442
- next unless f
443
-
444
- warn format("[fragment-parallel] [%d/%d] %s %.1fs",
445
- fragments.size - queue.size - running.size,
446
- fragments.size, File.basename(f[:path]),
447
- Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
448
- unless status.success?
449
- failure ||= "fragment compile failed" \
450
- " (#{status.exitstatus || "signal #{status.termsig}"}):" \
451
- " #{f[:path]}, see #{f[:path]}.log"
452
- end
453
- end
454
- raise failure if failure
455
- end
456
-
457
- # No fork on this platform (Windows): compile serially in-process,
458
- # capturing each fragment's output in its log like a forked child's.
459
- # Slower, same artifacts and failure semantics.
460
- def compile_fragments_in_process(fragments)
461
- clauses_env = ENV["METANORMA_PARALLEL_CLAUSES"]
462
- ENV["METANORMA_PARALLEL_CLAUSES"] = nil
463
- out = $stdout.dup
464
- err = $stderr.dup
465
- fragments.each_with_index do |f, i|
466
- t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
467
- failure = nil
468
- File.open("#{f[:path]}.log", "w:UTF-8") do |log|
469
- $stdout.reopen(log)
470
- $stderr.reopen(log)
471
- begin
472
- compile_fragment(f)
473
- $stdout.flush
474
- $stderr.flush
475
- rescue StandardError => e
476
- failure = StandardError.new(
477
- "fragment compile failed: #{f[:path]}: #{e.message}",
478
- )
479
- failure.set_backtrace(e.backtrace)
480
- end
481
- end
482
- $stdout.reopen(out)
483
- $stderr.reopen(err)
484
- warn format("[fragment-parallel] [%d/%d] %s %.1fs",
485
- i + 1, fragments.size, File.basename(f[:path]),
486
- Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
487
- raise failure if failure
488
- end
489
- ensure
490
- ENV["METANORMA_PARALLEL_CLAUSES"] = clauses_env
491
- $stdout.reopen(out)
492
- $stderr.reopen(err)
493
- end
494
-
495
- def fork?
496
- Process.respond_to?(:fork)
497
- end
498
-
499
- # Fork one worker for +f+. The child rebinds stdout/stderr to the
500
- # fragment log, drops this env var so the nested compile does not
501
- # recurse into fragment-parallel again, and exits without running
502
- # the parent's at-exit handlers: the parent owns the run's
503
- # lifecycle.
504
- def spawn_fragment(f)
505
- log = File.open("#{f[:path]}.log", "w:UTF-8")
506
- if (pid = Process.fork).nil?
507
- $stdout.reopen(log)
508
- $stderr.reopen(log)
509
- log.close
510
- ENV["METANORMA_PARALLEL_CLAUSES"] = nil
511
- begin
512
- compile_fragment(f)
513
- $stdout.flush
514
- $stderr.flush
515
- exit!(0)
516
- rescue StandardError => e
517
- warn e.full_message
518
- exit!(1)
519
- end
520
- end
521
- log.close
522
- pid
523
- end
524
-
525
- # The child-side body of one fragment compile. Runs in a forked copy
526
- # of this process: the flavor libraries, the metanorma registry and
527
- # the preloaded glossarist datasets are all inherited, so the
528
- # fragment pays only its own expansion and conversion.
529
- def compile_fragment(f)
530
- Metanorma::Compile.new.compile(
531
- f[:path], format: :asciidoc, extension_keys: [:xml],
532
- novalid: true, agree_to_terms: true
533
- )
534
- end
535
-
536
- # -- stitching
537
-
538
- def stitch(paths)
539
- base = parse(paths.first)
540
- ns = base.root.namespace&.href
541
- paths[1..].each do |p|
542
- stitch_fragment_root(base, parse(p), ns)
543
- end
544
- reorder_root(base)
545
- repair_concept_refs(base, ns)
546
- base.to_xml
547
- end
548
-
549
- # Concept cross-references whose target term lives in another
550
- # fragment fail resolution inside that fragment and come back as
551
- # <errormsg> payloads carrying the reference texts verbatim. This
552
- # pass repairs the unqualified subset: when the referenced id (or
553
- # its glossarist "term-" prefixed anchor) names a term in the
554
- # assembled document, the resolved refterm/renderterm/xref form is
555
- # reconstructed. Domain-qualified references (the bulk in the
556
- # iso-10303-2 corpus) resolve through the glossarist dataset, which
557
- # the stitched document does not carry -- those remain errormsg and
558
- # are the spike's principal known gap.
559
- def repair_concept_refs(base, ns)
560
- terms = base.root.xpath(".//xmlns:term[@anchor]", xmlns: ns)
561
- .each_with_object({}) { |t, m| m[t["anchor"]] = t }
562
- repaired = 0
563
- base.root.xpath(".//xmlns:concept/xmlns:errormsg", xmlns: ns)
564
- .each do |err|
565
- tts = err.xpath(".//xmlns:tt", xmlns: ns).map(&:text)
566
- next unless tts.size == 3
567
-
568
- # the error payload carries the raw concept id; the rendered
569
- # anchor carries the glossarist "term-" prefix
570
- id = tts[2]
571
- id = "term-#{id}" unless terms.key?(id)
572
- next unless terms.key?(id)
573
-
574
- refterm = base.document.create_element("refterm")
575
- refterm.content = tts[0]
576
- renderterm = base.document.create_element("renderterm")
577
- renderterm.content = tts[1]
578
- xref = base.document.create_element("xref")
579
- xref["target"] = id
580
- err.replace(refterm)
581
- refterm.next = renderterm
582
- renderterm.next = xref
583
- repaired += 1
584
- end
585
- warn "[fragment-parallel] repaired #{repaired} cross-fragment" \
586
- " concept references" if repaired.positive?
587
- end
588
-
589
- # Merge one fragment's root into +base+. Layout note: standoc emits
590
- # root children adjacent, with a newline after the root open tag and
591
- # one before the root close; whitespace-only text travels with the
592
- # element it precedes/follows, so appends reproduce the fragment's
593
- # own layout at the splice point.
594
- def stitch_fragment_root(base, d, ns)
595
- pairs = []
596
- ws = +""
597
- d.root.children.each do |c|
598
- if c.text?
599
- ws << c.text
600
- else
601
- pairs << [ws, c]
602
- ws = +""
603
- end
604
- end
605
- last = nil
606
- pairs.each do |lead, el|
607
- name = el.name
608
- if SKIP_ROOT.include?(name)
609
- last = :skip
610
- elsif MERGE_ROOT.include?(name) &&
611
- (t = base.root.at_xpath("./xmlns:#{name}", xmlns: ns))
612
- merge_children(t, el, ns, name)
613
- last = :merge
614
- else
615
- drop_trailing_ws(base.root)
616
- base.root.add_child(
617
- base.root.document.create_text_node(lead)
618
- ) unless lead.empty?
619
- adopt(base, el, ns).each { |n| base.root.add_child(n) }
620
- last = :append
621
- end
622
- end
623
- return unless last == :append && !ws.empty?
624
-
625
- drop_trailing_ws(base.root)
626
- base.root.add_child(base.root.document.create_text_node(ws))
627
- end
628
-
629
- def drop_trailing_ws(root)
630
- t = root.children.last
631
- t.remove if t&.text? && t.text.strip.empty?
632
- end
633
-
634
- # Root children are emitted in the canonical semantic-XML order;
635
- # each element keeps the whitespace-only text node that preceded it.
636
- # Everything is detached and re-added in order: libxml2's in-place
637
- # node moves do not reliably reposition text nodes.
638
- def reorder_root(base)
639
- pairs = []
640
- ws = nil
641
- base.root.children.each do |c|
642
- if c.text? && c.text.strip.empty?
643
- ws = c
644
- else
645
- pairs << [ws, c]
646
- ws = nil
647
- end
648
- end
649
- rank = ->(p, i) { [ROOT_ORDER.index(p[1].name) || ROOT_ORDER.size, i] }
650
- ordered = pairs.sort_by.with_index { |p, i| rank.call(p, i) }
651
- base.root.children.each(&:remove)
652
- ordered.each do |lead, el|
653
- base.root.add_child(
654
- base.root.document.create_text_node(lead.text)
655
- ) if lead
656
- base.root.add_child(el)
657
- end
658
- base.root.add_child(
659
- base.root.document.create_text_node(ws.text)
660
- ) if ws
661
- end
662
-
663
- def parse(path)
664
- Nokogiri::XML(File.read(path, encoding: "UTF-8"), &:huge)
665
- end
666
-
667
- # Copy `el` (from another document) into `base`'s document, dropping
668
- # the default-namespace declaration so it inherits the target's.
669
- def adopt(base, el, ns)
670
- s = el.to_xml
671
- s = s.gsub(%r{ xmlns="#{Regexp.quote(ns)}"}, "") if ns
672
- Nokogiri::XML::DocumentFragment.new(base.document, s).children
673
- end
674
-
675
- def merge_children(base_el, frag_el, ns, rootname)
676
- skip = rootname == "preface" ? NONBASE_PREFACE_SKIP : []
677
- pending = +""
678
- landed = false
679
- frag_el.children.each do |child|
680
- if child.text?
681
- pending << child.text
682
- next
683
- end
684
- twin = base_el.elements.find { |c| elem_key(c, ns) ==
685
- elem_key(child, ns) }
686
- if skip.include?(child.name)
687
- pending = +""
688
- elsif twin && WRAPPERS.include?(child.name)
689
- base_el.add_child(
690
- base_el.document.create_text_node(pending)
691
- ) unless pending.empty?
692
- pending = +""
693
- landed = true
694
- merge_children(twin, child, ns, rootname)
695
- elsif twin
696
- # duplicate generated metadata or boilerplate: keep base's
697
- pending = +""
698
- else
699
- base_el.add_child(
700
- base_el.document.create_text_node(pending)
701
- ) unless pending.empty?
702
- pending = +""
703
- landed = true
704
- adopt(base_el.document, child, ns).each do |n|
705
- base_el.add_child(n)
706
- end
707
- end
708
- end
709
- return unless landed && !pending.empty?
710
-
711
- # trailing whitespace of the fragment's own layout
712
- base_el.add_child(base_el.document.create_text_node(pending))
713
- end
714
-
715
- # Wrapper clauses of the same mega-clause carry the same explicit
716
- # anchor, or -- when the source names no anchor -- the same title.
717
- def elem_key(el, ns)
718
- return [el.name, "##{el['anchor']}"] if el["anchor"]
719
-
720
- t = ns ? el.at_xpath("./xmlns:title", xmlns: ns) : el.at("./title")
721
- [el.name, t&.text]
722
- end
723
-
724
- def cleanup
725
- return if ENV["METANORMA_PARALLEL_KEEP"] == "1"
726
-
727
- # every artifact of this run carries the run prefix (fragment
728
- # sources, chunk data files, fragment XML, child logs, the
729
- # glossarist per-document log)
730
- Dir.glob("#{@dir}/#{@prefix}-*").each do |p|
731
- File.unlink(p) if File.file?(p)
732
- rescue Errno::EACCES
733
- # Windows can hold a just-closed file briefly (indexer/AV);
734
- # a leftover temp artifact is harmless
735
- end
736
- end
737
- end
738
- end
739
- end