metanorma 2.5.8 → 2.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5901d6f63360e485eec67b384b342a3bbd3e6e7c471a65eabd88d23dd54736b9
|
|
4
|
+
data.tar.gz: 1fdaced94e13cdef6904120007564fca55707f54a8bf1ec691b24fba8091de91
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b31a468f386fa605b3e7a97ac9e5e5d435a18898b11c22b151aa17041a108f23df0e27090a458d14d12c49b33c852e9ef5085de2d4e81b28c5a5dee3a4d7992e
|
|
7
|
+
data.tar.gz: 4520886b164a3f659ea267c519da26975a05c68141b7512f354b7ffc037a00bb43c2ce5c4540415cafe71e92be1e60686a54db9df72b3a335b625378ff429252
|
|
@@ -120,6 +120,12 @@ module Metanorma
|
|
|
120
120
|
@compile_options.merge!(bare: true)
|
|
121
121
|
if @files.get(ident, :attachment) then copy_file_to_dest(ident)
|
|
122
122
|
else
|
|
123
|
+
# Distributed reinflation: the manifest carries every member
|
|
124
|
+
# (their stored files feed the anchors and cross-reference
|
|
125
|
+
# passes) but this run renders only the selected slice; the
|
|
126
|
+
# other members' sections are rendered by sibling runs over
|
|
127
|
+
# the same store.
|
|
128
|
+
@only_members and !@only_members.include?(ident) and next
|
|
123
129
|
file, fname = @files.targetfile_id(ident, read: true)
|
|
124
130
|
@preserve_unresolved && member_stored?(ident, file) and next
|
|
125
131
|
warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
|
|
@@ -101,6 +101,10 @@ module Metanorma
|
|
|
101
101
|
# if false, this is the root instance of Renderer
|
|
102
102
|
# if true, then this is not the last time Renderer will be run
|
|
103
103
|
# (e.g. this is sectionsplit)
|
|
104
|
+
# Distributed reinflation: restrict the rendered members to these
|
|
105
|
+
# identifiers. The manifest still carries every member, so the
|
|
106
|
+
# anchors and cross-reference passes see the whole collection.
|
|
107
|
+
@only_members = options[:only_members]
|
|
104
108
|
# Isolated/incremental build: preserve unresolved cross-document
|
|
105
109
|
# references as stubs instead of resolving or stripping them, so each
|
|
106
110
|
# member compiles independently and a later reinflation pass relinks.
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "fileutils"
|
|
4
|
-
require "open3"
|
|
5
4
|
require "yaml"
|
|
6
5
|
|
|
7
6
|
module Metanorma
|
|
@@ -10,10 +9,18 @@ module Metanorma
|
|
|
10
9
|
#
|
|
11
10
|
# Opt in with METANORMA_PARALLEL_CLAUSES=<N> (N >= 2; default off). The
|
|
12
11
|
# AsciiDoc source is split into clause fragments, each fragment is
|
|
13
|
-
# compiled to semantic XML by
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
12
|
+
# compiled to semantic XML by a worker forked off this process, and
|
|
13
|
+
# the fragment XMLs are stitched back into one semantic document that
|
|
14
|
+
# the ordinary output pipeline consumes as today.
|
|
15
|
+
#
|
|
16
|
+
# Forking over spawning: a fresh interpreter per fragment re-pays the
|
|
17
|
+
# flavor boot and -- fatally for vocabulary corpora -- the glossarist
|
|
18
|
+
# dataset load (tens of seconds per fragment). A forked worker
|
|
19
|
+
# inherits everything the parent already loaded. Before the first
|
|
20
|
+
# fork, the parent preloads the document's glossarist datasets once
|
|
21
|
+
# (opt-in plugin store cache, METANORMA_GLOSSARIST_SHARED_DATASETS),
|
|
22
|
+
# so the workers read the loaded models copy-on-write instead of
|
|
23
|
+
# re-parsing them.
|
|
17
24
|
#
|
|
18
25
|
# Split rules:
|
|
19
26
|
# - includes are resolved inline, recursively;
|
|
@@ -56,6 +63,14 @@ module Metanorma
|
|
|
56
63
|
DATA2TEXT = /\[(yaml2text|data2text),(.+?)\]\s*$/.freeze
|
|
57
64
|
MAX_INCLUDE_DEPTH = 8
|
|
58
65
|
|
|
66
|
+
# Mirrors the plugin's DatasetPreprocessor header directive.
|
|
67
|
+
DATASET_ATTR = /^:glossarist-dataset:\s*(.*)$/.freeze
|
|
68
|
+
|
|
69
|
+
# The metanorma-plugin-glossarist store-cache gate (its
|
|
70
|
+
# DatasetRegistry::SHARED_STORE_ENV; spelled out here because the
|
|
71
|
+
# plugin may be absent entirely).
|
|
72
|
+
SHARED_DATASETS_ENV = "METANORMA_GLOSSARIST_SHARED_DATASETS"
|
|
73
|
+
|
|
59
74
|
SKIP_ROOT = %w[bibdata boilerplate].freeze
|
|
60
75
|
MERGE_ROOT = %w[preface sections bibliography
|
|
61
76
|
metanorma-extension].freeze
|
|
@@ -67,33 +82,22 @@ module Metanorma
|
|
|
67
82
|
ROOT_ORDER = %w[bibdata metanorma-extension boilerplate preface
|
|
68
83
|
sections annex indexsect colophon bibliography].freeze
|
|
69
84
|
|
|
70
|
-
|
|
71
|
-
require "metanorma"
|
|
72
|
-
(ENV["METANORMA_PARALLEL_CHILD_REQUIRES"] || "")
|
|
73
|
-
.split(",").each { |g| require g }
|
|
74
|
-
Metanorma::Compile.new.compile(
|
|
75
|
-
ARGV[0], format: :asciidoc, extension_keys: [:xml],
|
|
76
|
-
novalid: true, agree_to_terms: true
|
|
77
|
-
)
|
|
78
|
-
RUBY
|
|
79
|
-
|
|
80
|
-
def initialize(filename, options, workers)
|
|
85
|
+
def initialize(filename, _options, workers)
|
|
81
86
|
@filename = filename
|
|
82
87
|
@workers = workers
|
|
83
88
|
@dir = File.realpath(File.dirname(File.expand_path(filename)))
|
|
84
89
|
@base = File.basename(filename, ".*")
|
|
85
90
|
@prefix = ".#{@base}.frag"
|
|
86
|
-
@requires = [options[:type] && "metanorma-#{options[:type]}"]
|
|
87
|
-
if (r = ENV["METANORMA_PARALLEL_CHILD_REQUIRES"])
|
|
88
|
-
@requires = r.split(",")
|
|
89
|
-
end
|
|
90
|
-
@requires.compact!
|
|
91
91
|
end
|
|
92
92
|
|
|
93
93
|
def run
|
|
94
|
-
|
|
94
|
+
text = resolve_includes(
|
|
95
|
+
File.read(@filename, encoding: "utf-8"), 0
|
|
96
|
+
)
|
|
97
|
+
fragments = build_fragments(text)
|
|
95
98
|
warn "[fragment-parallel] #{@filename}: #{fragments.size} fragments," \
|
|
96
99
|
" #{@workers} workers"
|
|
100
|
+
warm_shared_datasets(text)
|
|
97
101
|
compile_fragments(fragments)
|
|
98
102
|
t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
99
103
|
xml = stitch(fragments.map { |f| f[:path].sub(/\.adoc$/, ".xml") })
|
|
@@ -108,10 +112,8 @@ module Metanorma
|
|
|
108
112
|
|
|
109
113
|
# -- splitting
|
|
110
114
|
|
|
111
|
-
def build_fragments
|
|
112
|
-
lines =
|
|
113
|
-
File.read(@filename, encoding: "utf-8"), 0
|
|
114
|
-
).split("\n")
|
|
115
|
+
def build_fragments(text)
|
|
116
|
+
lines = text.split("\n")
|
|
115
117
|
header, segments = segmentize(lines)
|
|
116
118
|
fragments = []
|
|
117
119
|
segments.each do |seg|
|
|
@@ -277,8 +279,8 @@ module Metanorma
|
|
|
277
279
|
end
|
|
278
280
|
|
|
279
281
|
def mega_fragments(fragments, header, seg, mega)
|
|
280
|
-
# one chunk per worker:
|
|
281
|
-
#
|
|
282
|
+
# one chunk per worker: the chunks left over after the first wave
|
|
283
|
+
# land on workers that just finished, keeping every worker fed
|
|
282
284
|
chunks = [mega[:entries].size, @workers].min
|
|
283
285
|
size = (mega[:entries].size / chunks.to_f).ceil
|
|
284
286
|
head = header + seg[:lines][0...mega[:attr_i]]
|
|
@@ -303,47 +305,178 @@ module Metanorma
|
|
|
303
305
|
end
|
|
304
306
|
end
|
|
305
307
|
|
|
308
|
+
# -- dataset preload
|
|
309
|
+
|
|
310
|
+
# Load the document's glossarist datasets once, in this process,
|
|
311
|
+
# before any worker is forked. The plugin's opt-in store cache
|
|
312
|
+
# (METANORMA_GLOSSARIST_SHARED_DATASETS) memoizes the parsed
|
|
313
|
+
# datasets process-wide; the forked workers then answer their own
|
|
314
|
+
# registry lookups from the shared, copy-on-write models instead of
|
|
315
|
+
# re-parsing tens of thousands of YAML files per fragment.
|
|
316
|
+
#
|
|
317
|
+
# Skipped without the plugin, or when the datasets cannot be
|
|
318
|
+
# resolved: the fragments then behave exactly as on the spike --
|
|
319
|
+
# each loads what it needs, in its own fork.
|
|
320
|
+
def warm_shared_datasets(text)
|
|
321
|
+
return unless defined?(Metanorma::Plugin::Glossarist::DatasetRegistry)
|
|
322
|
+
return unless Metanorma::Plugin::Glossarist::DatasetRegistry
|
|
323
|
+
.respond_to?(:shared_store_for)
|
|
324
|
+
|
|
325
|
+
paths = glossarist_dataset_paths(text)
|
|
326
|
+
return if paths.empty?
|
|
327
|
+
|
|
328
|
+
# An explicit "0" is a user opt-out: leave the plugin gate off
|
|
329
|
+
# and skip the preload -- the workers re-load per fork, as before.
|
|
330
|
+
gate = ENV.fetch(SHARED_DATASETS_ENV, nil)
|
|
331
|
+
gate = ENV[SHARED_DATASETS_ENV] = "1" if gate.nil? || gate.empty?
|
|
332
|
+
return if gate == "0"
|
|
333
|
+
|
|
334
|
+
registry = Metanorma::Plugin::Glossarist::DatasetRegistry.new
|
|
335
|
+
t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
336
|
+
paths.each { |p| registry.concepts_at(p) }
|
|
337
|
+
# Give the workers a compacted, shared heap to fork from.
|
|
338
|
+
GC.start
|
|
339
|
+
warn format("[fragment-parallel] preloaded %d glossarist dataset(s)" \
|
|
340
|
+
" in %.1fs; workers share them copy-on-write",
|
|
341
|
+
paths.size,
|
|
342
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
|
|
343
|
+
rescue StandardError => e
|
|
344
|
+
warn "[fragment-parallel] glossarist dataset preload skipped:" \
|
|
345
|
+
" #{e.message}"
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
# Registered dataset paths in document-header order, resolved
|
|
349
|
+
# against the docfile directory the same way the plugin resolves
|
|
350
|
+
# them for a fragment (:glossarist-dataset: name:path;name:path).
|
|
351
|
+
def glossarist_dataset_paths(text)
|
|
352
|
+
text.scan(DATASET_ATTR)
|
|
353
|
+
.flat_map { |(contexts)| contexts.split(";") }
|
|
354
|
+
.filter_map do |entry|
|
|
355
|
+
_, rel = entry.split(":", 2)&.map(&:strip)
|
|
356
|
+
next unless rel && !rel.empty?
|
|
357
|
+
|
|
358
|
+
path = File.expand_path(rel, @dir)
|
|
359
|
+
File.directory?(path) ? path : nil
|
|
360
|
+
end.uniq
|
|
361
|
+
end
|
|
362
|
+
|
|
306
363
|
# -- compilation
|
|
307
364
|
|
|
365
|
+
# Bounded fork scheduler: keep @workers children alive until the
|
|
366
|
+
# fragments run out, reap any exited child, then raise the first
|
|
367
|
+
# failure after all scheduled work has drained.
|
|
308
368
|
def compile_fragments(fragments)
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
369
|
+
return compile_fragments_in_process(fragments) unless fork?
|
|
370
|
+
|
|
371
|
+
queue = fragments.dup
|
|
372
|
+
running = {}
|
|
373
|
+
failure = nil
|
|
374
|
+
until queue.empty? && running.empty?
|
|
375
|
+
until queue.empty? || running.size >= @workers
|
|
376
|
+
f = queue.shift
|
|
316
377
|
begin
|
|
317
|
-
|
|
378
|
+
t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
379
|
+
running[spawn_fragment(f)] = [f, t]
|
|
318
380
|
rescue StandardError => e
|
|
319
|
-
|
|
320
|
-
# WorkersPool#shutdown; surface it after the pool drains
|
|
321
|
-
lock.synchronize { failures << e }
|
|
381
|
+
failure ||= e.message
|
|
322
382
|
end
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
383
|
+
end
|
|
384
|
+
break if running.empty?
|
|
385
|
+
|
|
386
|
+
pid, status = Process.waitpid2(-1, 0)
|
|
387
|
+
f, t = running.delete(pid)
|
|
388
|
+
next unless f
|
|
389
|
+
|
|
390
|
+
warn format("[fragment-parallel] [%d/%d] %s %.1fs",
|
|
391
|
+
fragments.size - queue.size - running.size,
|
|
392
|
+
fragments.size, File.basename(f[:path]),
|
|
393
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
|
|
394
|
+
unless status.success?
|
|
395
|
+
failure ||= "fragment compile failed" \
|
|
396
|
+
" (#{status.exitstatus || "signal #{status.termsig}"}):" \
|
|
397
|
+
" #{f[:path]}, see #{f[:path]}.log"
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
raise failure if failure
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
# No fork on this platform (Windows): compile serially in-process,
|
|
404
|
+
# capturing each fragment's output in its log like a forked child's.
|
|
405
|
+
# Slower, same artifacts and failure semantics.
|
|
406
|
+
def compile_fragments_in_process(fragments)
|
|
407
|
+
clauses_env = ENV["METANORMA_PARALLEL_CLAUSES"]
|
|
408
|
+
ENV["METANORMA_PARALLEL_CLAUSES"] = nil
|
|
409
|
+
out = $stdout.dup
|
|
410
|
+
err = $stderr.dup
|
|
411
|
+
fragments.each_with_index do |f, i|
|
|
412
|
+
t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
413
|
+
failure = nil
|
|
414
|
+
File.open("#{f[:path]}.log", "w:UTF-8") do |log|
|
|
415
|
+
$stdout.reopen(log)
|
|
416
|
+
$stderr.reopen(log)
|
|
417
|
+
begin
|
|
418
|
+
compile_fragment(f)
|
|
419
|
+
$stdout.flush
|
|
420
|
+
$stderr.flush
|
|
421
|
+
rescue StandardError => e
|
|
422
|
+
failure = StandardError.new(
|
|
423
|
+
"fragment compile failed: #{f[:path]}: #{e.message}",
|
|
424
|
+
)
|
|
425
|
+
failure.set_backtrace(e.backtrace)
|
|
328
426
|
end
|
|
329
427
|
end
|
|
428
|
+
$stdout.reopen(out)
|
|
429
|
+
$stderr.reopen(err)
|
|
430
|
+
warn format("[fragment-parallel] [%d/%d] %s %.1fs",
|
|
431
|
+
i + 1, fragments.size, File.basename(f[:path]),
|
|
432
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - t)
|
|
433
|
+
raise failure if failure
|
|
434
|
+
end
|
|
435
|
+
ensure
|
|
436
|
+
ENV["METANORMA_PARALLEL_CLAUSES"] = clauses_env
|
|
437
|
+
$stdout.reopen(out)
|
|
438
|
+
$stderr.reopen(err)
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
def fork?
|
|
442
|
+
Process.respond_to?(:fork)
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
# Fork one worker for +f+. The child rebinds stdout/stderr to the
|
|
446
|
+
# fragment log, drops this env var so the nested compile does not
|
|
447
|
+
# recurse into fragment-parallel again, and exits without running
|
|
448
|
+
# the parent's at-exit handlers: the parent owns the run's
|
|
449
|
+
# lifecycle.
|
|
450
|
+
def spawn_fragment(f)
|
|
451
|
+
log = File.open("#{f[:path]}.log", "w:UTF-8")
|
|
452
|
+
if (pid = Process.fork).nil?
|
|
453
|
+
$stdout.reopen(log)
|
|
454
|
+
$stderr.reopen(log)
|
|
455
|
+
log.close
|
|
456
|
+
ENV["METANORMA_PARALLEL_CLAUSES"] = nil
|
|
457
|
+
begin
|
|
458
|
+
compile_fragment(f)
|
|
459
|
+
$stdout.flush
|
|
460
|
+
$stderr.flush
|
|
461
|
+
exit!(0)
|
|
462
|
+
rescue StandardError => e
|
|
463
|
+
warn e.full_message
|
|
464
|
+
exit!(1)
|
|
465
|
+
end
|
|
330
466
|
end
|
|
331
|
-
|
|
332
|
-
|
|
467
|
+
log.close
|
|
468
|
+
pid
|
|
333
469
|
end
|
|
334
470
|
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
raise "fragment compile failed (#{status.exitstatus}): #{path}," \
|
|
346
|
-
" see #{path}.log"
|
|
471
|
+
# The child-side body of one fragment compile. Runs in a forked copy
|
|
472
|
+
# of this process: the flavor libraries, the metanorma registry and
|
|
473
|
+
# the preloaded glossarist datasets are all inherited, so the
|
|
474
|
+
# fragment pays only its own expansion and conversion.
|
|
475
|
+
def compile_fragment(f)
|
|
476
|
+
Metanorma::Compile.new.compile(
|
|
477
|
+
f[:path], format: :asciidoc, extension_keys: [:xml],
|
|
478
|
+
novalid: true, agree_to_terms: true
|
|
479
|
+
)
|
|
347
480
|
end
|
|
348
481
|
|
|
349
482
|
# -- stitching
|
|
@@ -542,6 +675,9 @@ module Metanorma
|
|
|
542
675
|
# glossarist per-document log)
|
|
543
676
|
Dir.glob("#{@dir}/#{@prefix}-*").each do |p|
|
|
544
677
|
File.unlink(p) if File.file?(p)
|
|
678
|
+
rescue Errno::EACCES
|
|
679
|
+
# Windows can hold a just-closed file briefly (indexer/AV);
|
|
680
|
+
# a leftover temp artifact is harmless
|
|
545
681
|
end
|
|
546
682
|
end
|
|
547
683
|
end
|
data/lib/metanorma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: metanorma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.5.
|
|
4
|
+
version: 2.5.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: asciidoctor
|