metanorma 2.5.8 → 2.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fac12acf3d39fe398084ba3fb04c1fa1bb12ceebfa48187ae0f7e006c7f6833c
4
- data.tar.gz: 562c2bda6334a61d8733c6dd989a002342af492e91ea8d10c81858a4328408aa
3
+ metadata.gz: 579747ce72c82e8b994e424d242896e19f5210dc622d6deff34747f613b1965a
4
+ data.tar.gz: 54ce451903c665067260b1b6c95ae39418f0b5c9fd1c23452492c45504bc4f5e
5
5
  SHA512:
6
- metadata.gz: 8d09182c5e26e988445bfbfc9624a6d07eefba9d373910384eab675730a7b93976a74ecb2497f4c631f6d7d311295db954ca8e26a3868cb58082b114e6a7543f
7
- data.tar.gz: 6cdce7f6a5cd577e338d635ab2b8fed170645ac771aa8eec675256d75ba7751be07f14a379c1b75a717f9d19200b809ba51f8e68b451ef3e50169ce831ec0969
6
+ metadata.gz: 292f5591ffcc32fcec02f839eaf74bc12b863dd5c91d3d19848588b960df4ce89cdf6aa8db74e1ac0bf0fac5436098926e8058a55e6b7305d35e0f7faa333b5f
7
+ data.tar.gz: 297bcc082477736b93b84bfb43e877f25b225273a1f2bbadd44889f6a21f06f26fdcfd012d01d1ca5cf6056f6ef157902f872c5e52067c3a2a5ad431ff913a3c
@@ -120,6 +120,12 @@ module Metanorma
120
120
  @compile_options.merge!(bare: true)
121
121
  if @files.get(ident, :attachment) then copy_file_to_dest(ident)
122
122
  else
123
+ # Distributed reinflation: the manifest carries every member
124
+ # (their stored files feed the anchors and cross-reference
125
+ # passes) but this run renders only the selected slice; the
126
+ # other members' sections are rendered by sibling runs over
127
+ # the same store.
128
+ @only_members and !render_member?(ident) and next
123
129
  file, fname = @files.targetfile_id(ident, read: true)
124
130
  @preserve_unresolved && member_stored?(ident, file) and next
125
131
  warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
@@ -151,6 +157,16 @@ module Metanorma
151
157
  ArtifactStore.content_hash(source, ::Metanorma::VERSION)
152
158
  end
153
159
 
160
+ # Only-members selects docref identifiers; section-split outputs are
161
+ # derivative entries added at renderer init under their own keys,
162
+ # each carrying :parentid. A split output renders whenever its
163
+ # parent is selected, or a sliced reinflate ships index pages whose
164
+ # section pages are missing (iso-10303#811).
165
+ def render_member?(ident)
166
+ @only_members.include?(ident) ||
167
+ @only_members.include?(@files.get(ident, :parentid))
168
+ end
169
+
154
170
  # Resume predicate: is this member already compiled and stored for its
155
171
  # current input content? If so the isolated compile can skip it.
156
172
  def member_stored?(ident, source)
@@ -101,6 +101,10 @@ module Metanorma
101
101
  # if false, this is the root instance of Renderer
102
102
  # if true, then this is not the last time Renderer will be run
103
103
  # (e.g. this is sectionsplit)
104
+ # Distributed reinflation: restrict the rendered members to these
105
+ # identifiers. The manifest still carries every member, so the
106
+ # anchors and cross-reference passes see the whole collection.
107
+ @only_members = options[:only_members]
104
108
  # Isolated/incremental build: preserve unresolved cross-document
105
109
  # references as stubs instead of resolving or stripping them, so each
106
110
  # member compiles independently and a later reinflation pass relinks.
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "fileutils"
4
- require "open3"
5
4
  require "yaml"
6
5
 
7
6
  module Metanorma
@@ -10,10 +9,18 @@ module Metanorma
10
9
  #
11
10
  # Opt in with METANORMA_PARALLEL_CLAUSES=<N> (N >= 2; default off). The
12
11
  # AsciiDoc source is split into clause fragments, each fragment is
13
- # compiled to semantic XML by its own metanorma process (isolating the
14
- # Asciidoctor/glossarist global state), and the fragment XMLs are
15
- # stitched back into one semantic document that the ordinary output
16
- # pipeline consumes as today.
12
+ # compiled to semantic XML by a worker forked off this process, and
13
+ # the fragment XMLs are stitched back into one semantic document that
14
+ # the ordinary output pipeline consumes as today.
15
+ #
16
+ # Forking over spawning: a fresh interpreter per fragment re-pays the
17
+ # flavor boot and -- fatally for vocabulary corpora -- the glossarist
18
+ # dataset load (tens of seconds per fragment). A forked worker
19
+ # inherits everything the parent already loaded. Before the first
20
+ # fork, the parent preloads the document's glossarist datasets once
21
+ # (opt-in plugin store cache, METANORMA_GLOSSARIST_SHARED_DATASETS),
22
+ # so the workers read the loaded models copy-on-write instead of
23
+ # re-parsing them.
17
24
  #
18
25
  # Split rules:
19
26
  # - includes are resolved inline, recursively;
@@ -56,6 +63,14 @@ module Metanorma
56
63
  DATA2TEXT = /\[(yaml2text|data2text),(.+?)\]\s*$/.freeze
57
64
  MAX_INCLUDE_DEPTH = 8
58
65
 
66
+ # Mirrors the plugin's DatasetPreprocessor header directive.
67
+ DATASET_ATTR = /^:glossarist-dataset:\s*(.*)$/.freeze
68
+
69
+ # The metanorma-plugin-glossarist store-cache gate (its
70
+ # DatasetRegistry::SHARED_STORE_ENV; spelled out here because the
71
+ # plugin may be absent entirely).
72
+ SHARED_DATASETS_ENV = "METANORMA_GLOSSARIST_SHARED_DATASETS"
73
+
59
74
  SKIP_ROOT = %w[bibdata boilerplate].freeze
60
75
  MERGE_ROOT = %w[preface sections bibliography
61
76
  metanorma-extension].freeze
@@ -67,33 +82,22 @@ module Metanorma
67
82
  ROOT_ORDER = %w[bibdata metanorma-extension boilerplate preface
68
83
  sections annex indexsect colophon bibliography].freeze
69
84
 
70
- CHILD_SCRIPT = <<~RUBY.freeze
71
- require "metanorma"
72
- (ENV["METANORMA_PARALLEL_CHILD_REQUIRES"] || "")
73
- .split(",").each { |g| require g }
74
- Metanorma::Compile.new.compile(
75
- ARGV[0], format: :asciidoc, extension_keys: [:xml],
76
- novalid: true, agree_to_terms: true
77
- )
78
- RUBY
79
-
80
- def initialize(filename, options, workers)
85
+ def initialize(filename, _options, workers)
81
86
  @filename = filename
82
87
  @workers = workers
83
88
  @dir = File.realpath(File.dirname(File.expand_path(filename)))
84
89
  @base = File.basename(filename, ".*")
85
90
  @prefix = ".#{@base}.frag"
86
- @requires = [options[:type] && "metanorma-#{options[:type]}"]
87
- if (r = ENV["METANORMA_PARALLEL_CHILD_REQUIRES"])
88
- @requires = r.split(",")
89
- end
90
- @requires.compact!
91
91
  end
92
92
 
93
93
  def run
94
- fragments = build_fragments
94
+ text = resolve_includes(
95
+ File.read(@filename, encoding: "utf-8"), 0
96
+ )
97
+ fragments = build_fragments(text)
95
98
  warn "[fragment-parallel] #{@filename}: #{fragments.size} fragments," \
96
99
  " #{@workers} workers"
100
+ warm_shared_datasets(text)
97
101
  compile_fragments(fragments)
98
102
  t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
99
103
  xml = stitch(fragments.map { |f| f[:path].sub(/\.adoc$/, ".xml") })
@@ -108,10 +112,8 @@ module Metanorma
108
112
 
109
113
  # -- splitting
110
114
 
111
- def build_fragments
112
- lines = resolve_includes(
113
- File.read(@filename, encoding: "utf-8"), 0
114
- ).split("\n")
115
+ def build_fragments(text)
116
+ lines = text.split("\n")
115
117
  header, segments = segmentize(lines)
116
118
  fragments = []
117
119
  segments.each do |seg|
@@ -277,8 +279,8 @@ module Metanorma
277
279
  end
278
280
 
279
281
  def mega_fragments(fragments, header, seg, mega)
280
- # one chunk per worker: each child re-loads the glossarist
281
- # dataset, so more chunks mostly re-pay that fixed cost
282
+ # one chunk per worker: the chunks left over after the first wave
283
+ # land on workers that just finished, keeping every worker fed
282
284
  chunks = [mega[:entries].size, @workers].min
283
285
  size = (mega[:entries].size / chunks.to_f).ceil
284
286
  head = header + seg[:lines][0...mega[:attr_i]]
@@ -303,47 +305,178 @@ module Metanorma
303
305
  end
304
306
  end
305
307
 
308
+ # -- dataset preload
309
+
310
+ # Load the document's glossarist datasets once, in this process,
311
+ # before any worker is forked. The plugin's opt-in store cache
312
+ # (METANORMA_GLOSSARIST_SHARED_DATASETS) memoizes the parsed
313
+ # datasets process-wide; the forked workers then answer their own
314
+ # registry lookups from the shared, copy-on-write models instead of
315
+ # re-parsing tens of thousands of YAML files per fragment.
316
+ #
317
+ # Skipped without the plugin, or when the datasets cannot be
318
+ # resolved: the fragments then behave exactly as on the spike --
319
+ # each loads what it needs, in its own fork.
320
+ def warm_shared_datasets(text)
321
+ return unless defined?(Metanorma::Plugin::Glossarist::DatasetRegistry)
322
+ return unless Metanorma::Plugin::Glossarist::DatasetRegistry
323
+ .respond_to?(:shared_store_for)
324
+
325
+ paths = glossarist_dataset_paths(text)
326
+ return if paths.empty?
327
+
328
+ # An explicit "0" is a user opt-out: leave the plugin gate off
329
+ # and skip the preload -- the workers re-load per fork, as before.
330
+ gate = ENV.fetch(SHARED_DATASETS_ENV, nil)
331
+ gate = ENV[SHARED_DATASETS_ENV] = "1" if gate.nil? || gate.empty?
332
+ return if gate == "0"
333
+
334
+ registry = Metanorma::Plugin::Glossarist::DatasetRegistry.new
335
+ t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
336
+ paths.each { |p| registry.concepts_at(p) }
337
+ # Give the workers a compacted, shared heap to fork from.
338
+ GC.start
339
+ warn format("[fragment-parallel] preloaded %d glossarist dataset(s)" \
340
+ " in %.1fs; workers share them copy-on-write",
341
+ paths.size,
342
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
343
+ rescue StandardError => e
344
+ warn "[fragment-parallel] glossarist dataset preload skipped:" \
345
+ " #{e.message}"
346
+ end
347
+
348
+ # Registered dataset paths in document-header order, resolved
349
+ # against the docfile directory the same way the plugin resolves
350
+ # them for a fragment (:glossarist-dataset: name:path;name:path).
351
+ def glossarist_dataset_paths(text)
352
+ text.scan(DATASET_ATTR)
353
+ .flat_map { |(contexts)| contexts.split(";") }
354
+ .filter_map do |entry|
355
+ _, rel = entry.split(":", 2)&.map(&:strip)
356
+ next unless rel && !rel.empty?
357
+
358
+ path = File.expand_path(rel, @dir)
359
+ File.directory?(path) ? path : nil
360
+ end.uniq
361
+ end
362
+
306
363
  # -- compilation
307
364
 
365
+ # Bounded fork scheduler: keep @workers children alive until the
366
+ # fragments run out, reap any exited child, then raise the first
367
+ # failure after all scheduled work has drained.
308
368
  def compile_fragments(fragments)
309
- pool = ::Metanorma::Util::WorkersPool.new(@workers)
310
- lock = Mutex.new
311
- done = 0
312
- failures = []
313
- fragments.each_with_index do |f, i|
314
- pool.schedule do
315
- t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
369
+ return compile_fragments_in_process(fragments) unless fork?
370
+
371
+ queue = fragments.dup
372
+ running = {}
373
+ failure = nil
374
+ until queue.empty? && running.empty?
375
+ until queue.empty? || running.size >= @workers
376
+ f = queue.shift
316
377
  begin
317
- child_compile(f[:path])
378
+ t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
379
+ running[spawn_fragment(f)] = [f, t]
318
380
  rescue StandardError => e
319
- # a raise here would kill the pool thread and deadlock
320
- # WorkersPool#shutdown; surface it after the pool drains
321
- lock.synchronize { failures << e }
381
+ failure ||= e.message
322
382
  end
323
- dt = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
324
- lock.synchronize do
325
- done += 1
326
- warn format("[fragment-parallel] [%d/%d] %s %.1fs",
327
- done, fragments.size, File.basename(f[:path]), dt)
383
+ end
384
+ break if running.empty?
385
+
386
+ pid, status = Process.waitpid2(-1, 0)
387
+ f, t = running.delete(pid)
388
+ next unless f
389
+
390
+ warn format("[fragment-parallel] [%d/%d] %s %.1fs",
391
+ fragments.size - queue.size - running.size,
392
+ fragments.size, File.basename(f[:path]),
393
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
394
+ unless status.success?
395
+ failure ||= "fragment compile failed" \
396
+ " (#{status.exitstatus || "signal #{status.termsig}"}):" \
397
+ " #{f[:path]}, see #{f[:path]}.log"
398
+ end
399
+ end
400
+ raise failure if failure
401
+ end
402
+
403
+ # No fork on this platform (Windows): compile serially in-process,
404
+ # capturing each fragment's output in its log like a forked child's.
405
+ # Slower, same artifacts and failure semantics.
406
+ def compile_fragments_in_process(fragments)
407
+ clauses_env = ENV["METANORMA_PARALLEL_CLAUSES"]
408
+ ENV["METANORMA_PARALLEL_CLAUSES"] = nil
409
+ out = $stdout.dup
410
+ err = $stderr.dup
411
+ fragments.each_with_index do |f, i|
412
+ t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
413
+ failure = nil
414
+ File.open("#{f[:path]}.log", "w:UTF-8") do |log|
415
+ $stdout.reopen(log)
416
+ $stderr.reopen(log)
417
+ begin
418
+ compile_fragment(f)
419
+ $stdout.flush
420
+ $stderr.flush
421
+ rescue StandardError => e
422
+ failure = StandardError.new(
423
+ "fragment compile failed: #{f[:path]}: #{e.message}",
424
+ )
425
+ failure.set_backtrace(e.backtrace)
328
426
  end
329
427
  end
428
+ $stdout.reopen(out)
429
+ $stderr.reopen(err)
430
+ warn format("[fragment-parallel] [%d/%d] %s %.1fs",
431
+ i + 1, fragments.size, File.basename(f[:path]),
432
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
433
+ raise failure if failure
434
+ end
435
+ ensure
436
+ ENV["METANORMA_PARALLEL_CLAUSES"] = clauses_env
437
+ $stdout.reopen(out)
438
+ $stderr.reopen(err)
439
+ end
440
+
441
+ def fork?
442
+ Process.respond_to?(:fork)
443
+ end
444
+
445
+ # Fork one worker for +f+. The child rebinds stdout/stderr to the
446
+ # fragment log, drops this env var so the nested compile does not
447
+ # recurse into fragment-parallel again, and exits without running
448
+ # the parent's at-exit handlers: the parent owns the run's
449
+ # lifecycle.
450
+ def spawn_fragment(f)
451
+ log = File.open("#{f[:path]}.log", "w:UTF-8")
452
+ if (pid = Process.fork).nil?
453
+ $stdout.reopen(log)
454
+ $stderr.reopen(log)
455
+ log.close
456
+ ENV["METANORMA_PARALLEL_CLAUSES"] = nil
457
+ begin
458
+ compile_fragment(f)
459
+ $stdout.flush
460
+ $stderr.flush
461
+ exit!(0)
462
+ rescue StandardError => e
463
+ warn e.full_message
464
+ exit!(1)
465
+ end
330
466
  end
331
- pool.shutdown
332
- raise failures.first unless failures.empty?
467
+ log.close
468
+ pid
333
469
  end
334
470
 
335
- def child_compile(path)
336
- env = {
337
- "METANORMA_PARALLEL_CLAUSES" => nil,
338
- "METANORMA_PARALLEL_CHILD_REQUIRES" => @requires.join(","),
339
- }
340
- out, err, status = Open3.capture3(env, RbConfig.ruby, "-e",
341
- CHILD_SCRIPT, path, chdir: @dir)
342
- File.write("#{path}.log", [out, err].join, encoding: "UTF-8")
343
- return if status.success?
344
-
345
- raise "fragment compile failed (#{status.exitstatus}): #{path}," \
346
- " see #{path}.log"
471
+ # The child-side body of one fragment compile. Runs in a forked copy
472
+ # of this process: the flavor libraries, the metanorma registry and
473
+ # the preloaded glossarist datasets are all inherited, so the
474
+ # fragment pays only its own expansion and conversion.
475
+ def compile_fragment(f)
476
+ Metanorma::Compile.new.compile(
477
+ f[:path], format: :asciidoc, extension_keys: [:xml],
478
+ novalid: true, agree_to_terms: true
479
+ )
347
480
  end
348
481
 
349
482
  # -- stitching
@@ -542,6 +675,9 @@ module Metanorma
542
675
  # glossarist per-document log)
543
676
  Dir.glob("#{@dir}/#{@prefix}-*").each do |p|
544
677
  File.unlink(p) if File.file?(p)
678
+ rescue Errno::EACCES
679
+ # Windows can hold a just-closed file briefly (indexer/AV);
680
+ # a leftover temp artifact is harmless
545
681
  end
546
682
  end
547
683
  end
@@ -1,3 +1,3 @@
1
1
  module Metanorma
2
- VERSION = "2.5.8".freeze
2
+ VERSION = "2.5.10".freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: metanorma
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.8
4
+ version: 2.5.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-11-01 00:00:00.000000000 Z
11
+ date: 2026-11-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asciidoctor