suma 0.5.38 → 0.5.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/suma/cli/check_svg_quality.rb +1 -1
- data/lib/suma/cli/delivery.rb +26 -12
- data/lib/suma/delivery/diagnostics.rb +136 -0
- data/lib/suma/delivery/expectations.rb +28 -2
- data/lib/suma/delivery/site_verifier.rb +2 -0
- data/lib/suma/delivery.rb +1 -0
- data/lib/suma/provenance.rb +9 -0
- data/lib/suma/staged_collection_builder/worker.rb +245 -0
- data/lib/suma/staged_collection_builder.rb +113 -146
- data/lib/suma/svg_quality/scanner.rb +14 -3
- data/lib/suma/term_extractor.rb +41 -1
- data/lib/suma/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0a7c2ce507254463b15a6f6e30ab2d0b664bc841ada2de657af4fc14aba7f012
|
|
4
|
+
data.tar.gz: f5abf56a5ca249f795793653cc43b835283d7e6685ad8f9cb013e452e041e944
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2c56115e2779608109c3301ce743ddf5e25541f9a61ede17ea95d9f7afe62e93b75d70aabc69ddc8fcc222ed24c88e66570a991576f200980097e5f78abd8eaf
|
|
7
|
+
data.tar.gz: 685e437b51483bcd56444d26c1bdf7c0aa649c0e9f0e8b43fc4b43990c754bffd85c36b997b64a35592a5cddf056adaf7e4e0847a7e6c210bb0b8eafdaacf4a4
|
|
@@ -11,7 +11,7 @@ module Suma
|
|
|
11
11
|
class CheckSvgQuality
|
|
12
12
|
DATA_PATH = "schemas"
|
|
13
13
|
DEFAULT_PATTERN = "**/*.svg"
|
|
14
|
-
DEFAULT_PROFILE =
|
|
14
|
+
DEFAULT_PROFILE = Suma::SvgQuality::Scanner.default_profile
|
|
15
15
|
|
|
16
16
|
def initialize(pattern: DEFAULT_PATTERN, profile: DEFAULT_PROFILE,
|
|
17
17
|
format: "terminal", output: nil, min_errors: nil,
|
data/lib/suma/cli/delivery.rb
CHANGED
|
@@ -26,8 +26,10 @@ module Suma
|
|
|
26
26
|
desc: "YAML manifest of site-shape thresholds " \
|
|
27
27
|
"(sections_min, index_pages, " \
|
|
28
28
|
"schema_attachments_min, schema_docs_min, " \
|
|
29
|
-
"missing_express_max, broken_links_max
|
|
30
|
-
"
|
|
29
|
+
"missing_express_max, broken_links_max, " \
|
|
30
|
+
"diagnostics_max: category => rule => " \
|
|
31
|
+
"max rows over the site's .err.html " \
|
|
32
|
+
"files); violations fail the verify"
|
|
31
33
|
|
|
32
34
|
def verify(site_dir)
|
|
33
35
|
result = Suma::Delivery::SiteVerifier.new(site_dir,
|
|
@@ -40,15 +42,19 @@ module Suma
|
|
|
40
42
|
# Without an expectations manifest the contract is zero broken
|
|
41
43
|
# links. With one, broken_links_max IS the broken-link contract:
|
|
42
44
|
# the unconditional raise would fire first and make the cap
|
|
43
|
-
# unreachable (TRThurman, iso-10303#835).
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
45
|
+
# unreachable (TRThurman, iso-10303#835). Defined inside
|
|
46
|
+
# no_commands so Thor does not register a command for a helper
|
|
47
|
+
# and warn about the missing usage/description on every run.
|
|
48
|
+
no_commands do
|
|
49
|
+
def verify_failures(result, failures)
|
|
50
|
+
if options[:expectations]
|
|
51
|
+
raise "expectations violated: #{failures.join('; ')}" unless
|
|
52
|
+
failures.empty?
|
|
53
|
+
elsif result.broken_count.zero?
|
|
54
|
+
nil
|
|
55
|
+
else
|
|
56
|
+
raise "broken links found (#{result.broken_count})"
|
|
57
|
+
end
|
|
52
58
|
end
|
|
53
59
|
end
|
|
54
60
|
|
|
@@ -171,7 +177,7 @@ module Suma
|
|
|
171
177
|
"broken; fragments: #{r['total_fragments']} total, " \
|
|
172
178
|
"#{r['broken_fragments'].size} broken; " \
|
|
173
179
|
"#{r['cross_document_links']} cross-document; " \
|
|
174
|
-
"#{inventory_summary(r)}"
|
|
180
|
+
"#{inventory_summary(r)}; #{diagnostics_summary(r)}"
|
|
175
181
|
end
|
|
176
182
|
|
|
177
183
|
def inventory_summary(report)
|
|
@@ -183,6 +189,14 @@ module Suma
|
|
|
183
189
|
"missing-express pages=#{report['missing_express_pages'].size}"
|
|
184
190
|
end
|
|
185
191
|
|
|
192
|
+
# The err.html census the verify walked: total diagnostic rows over
|
|
193
|
+
# the categories the site's error reports carry.
|
|
194
|
+
def diagnostics_summary(report)
|
|
195
|
+
census = report["diagnostics"] || {}
|
|
196
|
+
"diagnostics: #{census.sum { |_, rules| rules.values.sum }} rows " \
|
|
197
|
+
"over #{census.size} categories"
|
|
198
|
+
end
|
|
199
|
+
|
|
186
200
|
def write_report(report)
|
|
187
201
|
File.write(options[:out], JSON.pretty_generate(report))
|
|
188
202
|
end
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "leptris"
|
|
4
|
+
|
|
5
|
+
module Suma
|
|
6
|
+
module Delivery
|
|
7
|
+
# Diagnostic-row census over metanorma .err.html error reports
|
|
8
|
+
# (metanorma/suma#203): rows grouped by CATEGORY and RULE.
|
|
9
|
+
#
|
|
10
|
+
# A gate keyed on a literal code string breaks when the codes migrate
|
|
11
|
+
# (iso-10303#859: the part2 gate greps STANDOC_38 in document.err.html
|
|
12
|
+
# while metanorma-iso 3.5.14 re-emitted those rows as ISO_56, so the
|
|
13
|
+
# gate reads 0 over 476 live rows). Codes migrate; the stable identity
|
|
14
|
+
# of a diagnostic row is category + rule, so the census keys on both.
|
|
15
|
+
#
|
|
16
|
+
# The err.html writer is Metanorma::Utils::Log#write: one
|
|
17
|
+
# `<h2 id="...">Category</h2>` per category, each followed by a table
|
|
18
|
+
# whose data rows are six cells - Line, ID, Error (the rule code,
|
|
19
|
+
# e.g. STANDOC_27 or ISO_56), Message, Context, Severity.
|
|
20
|
+
class Diagnostics
|
|
21
|
+
# The Error cell's position within a data row.
|
|
22
|
+
RULE_CELL_INDEX = 2
|
|
23
|
+
|
|
24
|
+
class << self
|
|
25
|
+
# Parse one err.html given as a file path or an HTML string.
|
|
26
|
+
#
|
|
27
|
+
# @return [Counts] rows grouped by category, then rule
|
|
28
|
+
def count(path_or_html)
|
|
29
|
+
html = if File.file?(path_or_html)
|
|
30
|
+
File.read(path_or_html)
|
|
31
|
+
else
|
|
32
|
+
path_or_html
|
|
33
|
+
end
|
|
34
|
+
new(html).counts
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# The census of every err.html under a rendered site, summed:
|
|
38
|
+
# the site-level figure an expectations gate asserts against.
|
|
39
|
+
def count_site(site_dir)
|
|
40
|
+
Dir.glob(File.join(site_dir, "**", "*.err.html"))
|
|
41
|
+
.reduce(Counts.new) { |sum, path| sum.merge(count(path)) }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def initialize(html)
|
|
46
|
+
@doc = Leptris::XML.parse_html(html)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Every data row of every category table, keyed by its h2 heading
|
|
50
|
+
# (the log category) and Error cell (the rule code).
|
|
51
|
+
def counts
|
|
52
|
+
Counts.new(tally)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
# Per category, per rule, the number of data rows.
|
|
58
|
+
def tally
|
|
59
|
+
rows = Hash.new { |hash, category| hash[category] = Hash.new(0) }
|
|
60
|
+
@doc.css("h2").each do |heading|
|
|
61
|
+
rules_of(heading).each do |rule|
|
|
62
|
+
rows[heading.text.strip][rule] += 1
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
rows
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# The rule code of every data row in the heading's table.
|
|
69
|
+
def rules_of(heading)
|
|
70
|
+
table = heading.xpath("following-sibling::table[1]").first
|
|
71
|
+
return [] unless table
|
|
72
|
+
|
|
73
|
+
table.css("tr").filter_map { |row| rule_of(row) }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# The Error cell of a row: the third of Line/ID/Error/Message/
|
|
77
|
+
# Context/Severity (the ID cell renders as a `th`).
|
|
78
|
+
def rule_of(row)
|
|
79
|
+
rule = row.xpath("td|th")[RULE_CELL_INDEX]&.text&.strip
|
|
80
|
+
rule.nil? || rule.empty? ? nil : rule
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# The per-category, per-rule census of one or more err.html
|
|
84
|
+
# documents: immutable value object over
|
|
85
|
+
# { "Style" => { "STANDOC_48" => 15 } }, enumerable over
|
|
86
|
+
# (category, rule, count) triples.
|
|
87
|
+
class Counts
|
|
88
|
+
include Enumerable
|
|
89
|
+
|
|
90
|
+
def initialize(categories = {})
|
|
91
|
+
@categories = categories
|
|
92
|
+
.transform_values { |rules| rules.transform_values(&:to_i).freeze }
|
|
93
|
+
.freeze
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The raw census hash: category => (rule => row count). Frozen.
|
|
97
|
+
def to_h
|
|
98
|
+
@categories
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def each
|
|
102
|
+
return enum_for(__method__) unless block_given?
|
|
103
|
+
|
|
104
|
+
@categories.each do |category, rules|
|
|
105
|
+
rules.each { |rule, count| yield category, rule, count }
|
|
106
|
+
end
|
|
107
|
+
self
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Rows counted for one rule; an unreported rule counts zero.
|
|
111
|
+
def rule_count(category, rule)
|
|
112
|
+
@categories.dig(category, rule) || 0
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Every row counted for one category.
|
|
116
|
+
def total(category)
|
|
117
|
+
@categories.fetch(category, {}).values.sum
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Rows counted overall.
|
|
121
|
+
def rows
|
|
122
|
+
@categories.values.sum { |rules| rules.values.sum }
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# The union census: counts of both sides summed.
|
|
126
|
+
def merge(other)
|
|
127
|
+
other = self.class.new({}) unless other.is_a?(self.class)
|
|
128
|
+
summed = other.to_h.merge(to_h) do |_category, theirs, ours|
|
|
129
|
+
theirs.merge(ours) { |_rule, a, b| a + b }
|
|
130
|
+
end
|
|
131
|
+
self.class.new(summed)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
@@ -13,6 +13,10 @@ module Suma
|
|
|
13
13
|
attribute :schema_docs_min, :integer
|
|
14
14
|
attribute :missing_express_max, :integer, default: -> { 0 }
|
|
15
15
|
attribute :broken_links_max, :integer, default: -> { 0 }
|
|
16
|
+
# Category => rule => maximum live diagnostic rows: the stable
|
|
17
|
+
# identity of an err.html row is category + rule, not the code,
|
|
18
|
+
# which migrates across metanorma releases (iso-10303#859).
|
|
19
|
+
attribute :diagnostics_max, :hash
|
|
16
20
|
|
|
17
21
|
key_value do
|
|
18
22
|
map "sections_min", to: :sections_min
|
|
@@ -21,13 +25,22 @@ module Suma
|
|
|
21
25
|
map "schema_docs_min", to: :schema_docs_min
|
|
22
26
|
map "missing_express_max", to: :missing_express_max
|
|
23
27
|
map "broken_links_max", to: :broken_links_max
|
|
28
|
+
map "diagnostics_max", to: :diagnostics_max
|
|
24
29
|
end
|
|
25
30
|
|
|
26
31
|
# Human-readable violation lines for every threshold the +report+
|
|
27
32
|
# and +broken_links+ breach; unset thresholds stay silent. The name
|
|
28
33
|
# suffix is the comparison: `_min` floors, `_max` caps, a bare name
|
|
29
|
-
# (index_pages) is exact.
|
|
34
|
+
# (index_pages) is exact. The nested diagnostics_max thresholds
|
|
35
|
+
# count against the report's diagnostics census.
|
|
30
36
|
def violations(report, broken_links)
|
|
37
|
+
threshold_violations(report, broken_links) +
|
|
38
|
+
diagnostics_violations(report["diagnostics"])
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def threshold_violations(report, broken_links)
|
|
31
44
|
actuals(report, broken_links).filter_map do |name, actual|
|
|
32
45
|
limit = public_send(name) or next
|
|
33
46
|
relation = relation_of(name)
|
|
@@ -39,7 +52,20 @@ module Suma
|
|
|
39
52
|
end
|
|
40
53
|
end
|
|
41
54
|
|
|
42
|
-
|
|
55
|
+
# diagnostics_max: { "Referencing" => { "ISO_56" => 0 } }. A rule
|
|
56
|
+
# absent from the census counts zero rows.
|
|
57
|
+
def diagnostics_violations(census)
|
|
58
|
+
Array(diagnostics_max).flat_map do |category, rules|
|
|
59
|
+
rules.filter_map do |rule, limit|
|
|
60
|
+
actual = (census || {}).dig(category, rule) || 0
|
|
61
|
+
next if actual <= limit
|
|
62
|
+
|
|
63
|
+
format("%<category>s/%<rule>s: got %<actual>d, expected at " \
|
|
64
|
+
"most %<limit>d", category: category, rule: rule,
|
|
65
|
+
actual: actual, limit: limit)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
43
69
|
|
|
44
70
|
def relation_of(name)
|
|
45
71
|
return "at least" if name.end_with?("_min")
|
|
@@ -52,6 +52,7 @@ module Suma
|
|
|
52
52
|
verify_links(doc, rel_path, html_file, anchor_index, report)
|
|
53
53
|
end
|
|
54
54
|
report["inventory"] = site_inventory
|
|
55
|
+
report["diagnostics"] = Diagnostics.count_site(@site_dir).to_h
|
|
55
56
|
Report.new(report: report)
|
|
56
57
|
end
|
|
57
58
|
|
|
@@ -71,6 +72,7 @@ module Suma
|
|
|
71
72
|
"broken_intra_fragments" => [],
|
|
72
73
|
"missing_express_pages" => [],
|
|
73
74
|
"inventory" => {},
|
|
75
|
+
"diagnostics" => {},
|
|
74
76
|
}
|
|
75
77
|
end
|
|
76
78
|
|
data/lib/suma/delivery.rb
CHANGED
|
@@ -6,6 +6,7 @@ module Suma
|
|
|
6
6
|
module Delivery
|
|
7
7
|
autoload :FrameNav, "suma/delivery/frame_nav"
|
|
8
8
|
autoload :CrossPageFragments, "suma/delivery/cross_page_fragments"
|
|
9
|
+
autoload :Diagnostics, "suma/delivery/diagnostics"
|
|
9
10
|
autoload :Expectations, "suma/delivery/expectations"
|
|
10
11
|
autoload :Pack, "suma/delivery/pack"
|
|
11
12
|
autoload :Plan, "suma/delivery/plan"
|
data/lib/suma/provenance.rb
CHANGED
|
@@ -108,6 +108,15 @@ module Suma
|
|
|
108
108
|
@render_toolchain ||= toolchain.merge("render" => render_gems)
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
+
# The toolchain identity in the form cached artifacts record and
|
|
112
|
+
# compare (the staged store sidecars, the render key, the site-build
|
|
113
|
+
# key). A digest identifies the toolchain that produced a stored
|
|
114
|
+
# artifact without restating every gem version beside it.
|
|
115
|
+
def self.render_toolchain_digest
|
|
116
|
+
@render_toolchain_digest ||=
|
|
117
|
+
Digest::SHA256.hexdigest(JSON.generate(render_toolchain))
|
|
118
|
+
end
|
|
119
|
+
|
|
111
120
|
def self.render_gems
|
|
112
121
|
gem_specs.select { |spec| render_gem?(spec.name) }
|
|
113
122
|
.sort_by { |spec| [spec.name, spec.version.to_s] }
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "yaml"
|
|
5
|
+
require "fileutils"
|
|
6
|
+
require "metanorma"
|
|
7
|
+
|
|
8
|
+
module Suma
|
|
9
|
+
class StagedCollectionBuilder
|
|
10
|
+
# The code the staged builder's child processes run. Children are
|
|
11
|
+
# spawned as `ruby -e 'require "suma"; Worker.run'` with the mode and
|
|
12
|
+
# paths on ARGV and render options JSON-encoded on STDIN. All
|
|
13
|
+
# worker logic lives here as real, spec-testable methods: the previous
|
|
14
|
+
# implementation embedded this code as generated script strings, which
|
|
15
|
+
# is unauditable and shipped dead code once (suma#166 - an
|
|
16
|
+
# interpolating heredoc silently ate the regex anchors in the alias
|
|
17
|
+
# matchers, so the artefact aliases never landed and reinflate failed
|
|
18
|
+
# with "no stored artefact").
|
|
19
|
+
module Worker
|
|
20
|
+
# One batch stage's spawn parameters, threaded through the batch
|
|
21
|
+
# render methods.
|
|
22
|
+
BatchContext =
|
|
23
|
+
Struct.new(:config_dir, :batch_count, :store_dir, :out_dir,
|
|
24
|
+
:no_valid)
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
# Entry. ARGV = ["stage", manifest_path, store_dir, id_slug] with
|
|
29
|
+
# render options on STDIN (sequential stage: render one member and
|
|
30
|
+
# alias its artefacts); ARGV = ["batch", config_dir, batch_count,
|
|
31
|
+
# store_dir, out_dir, no_valid] (batch stage: claim and render
|
|
32
|
+
# batch files until none remain); ARGV = ["render", manifest_path]
|
|
33
|
+
# with options on STDIN (plain render, no aliasing - reinflate).
|
|
34
|
+
def run(argv = ARGV)
|
|
35
|
+
case mode = argv.shift
|
|
36
|
+
when "stage" then stage(*argv)
|
|
37
|
+
when "batch" then batch(*argv)
|
|
38
|
+
when "render" then render_only(*argv)
|
|
39
|
+
else abort("[suma staged worker] unknown mode #{mode.inspect}")
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Sequential stage: one fresh process per member, so peak memory
|
|
44
|
+
# stays at one member (metanorma/metanorma#576).
|
|
45
|
+
def stage(manifest_path, store_dir, id_slug)
|
|
46
|
+
collection = render_with_options(manifest_path)
|
|
47
|
+
alias_artifacts(collection, id_slug, store_dir)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Plain render (reinflate): no store aliasing - the reinflate pass
|
|
51
|
+
# consumes the store, it does not extend it.
|
|
52
|
+
def render_only(manifest_path)
|
|
53
|
+
render_with_options(manifest_path)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The render options arrive from the parent over STDIN as JSON (not
|
|
57
|
+
# Marshal: deserializing a stream is rb/unsafe-deserialization, and
|
|
58
|
+
# the payload is plain strings, booleans, arrays, and nested hashes
|
|
59
|
+
# - JSON expresses it exactly, with symbols restored below).
|
|
60
|
+
def render_with_options(manifest_path)
|
|
61
|
+
Metanorma::Collection.parse(manifest_path)
|
|
62
|
+
.render(load_options($stdin.read))
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Parent side: symbols and hash keys become strings for the wire.
|
|
66
|
+
def dump_options(opts)
|
|
67
|
+
JSON.generate(dump_value(opts))
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Child side: restore symbol top-level keys, symbol :format values,
|
|
71
|
+
# and the nested :compile hash.
|
|
72
|
+
def load_options(json)
|
|
73
|
+
opts = JSON.parse(json).to_h { |k, v| [k.to_sym, v] }
|
|
74
|
+
opts[:format] = opts[:format].map(&:to_sym) if opts[:format]
|
|
75
|
+
if opts[:compile]
|
|
76
|
+
opts[:compile] = opts[:compile].to_h do |k, v|
|
|
77
|
+
[k.to_sym, v]
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
opts
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def dump_value(value)
|
|
84
|
+
case value
|
|
85
|
+
when Hash then value.to_h { |k, v| [k.to_s, dump_value(v)] }
|
|
86
|
+
when Array then value.map { |v| dump_value(v) }
|
|
87
|
+
when Symbol then value.to_s
|
|
88
|
+
else value
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Batch stage: claim one batch file at a time through O_EXCL
|
|
93
|
+
# markers, so N concurrent children partition the batches without a
|
|
94
|
+
# coordinator; a child with no batches left exits and hands its
|
|
95
|
+
# memory back to the OS. Exits 1 when it could claim nothing while
|
|
96
|
+
# batches were announced (nothing was rendered).
|
|
97
|
+
def batch(config_dir, batch_count, store_dir, out_dir, no_valid)
|
|
98
|
+
ctx = BatchContext.new(config_dir, Integer(batch_count), store_dir,
|
|
99
|
+
out_dir, no_valid)
|
|
100
|
+
claimed = 0
|
|
101
|
+
while (idx = claim_batch(ctx.config_dir, ctx.batch_count))
|
|
102
|
+
render_batch(ctx, idx)
|
|
103
|
+
claimed += 1
|
|
104
|
+
end
|
|
105
|
+
exit 1 if claimed.zero? && ctx.batch_count.positive?
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def render_batch(ctx, idx)
|
|
109
|
+
# The batch file is written by this builder's own write_batch_bins
|
|
110
|
+
# in the same process tree - trusted, same-toolchain data, JSON-
|
|
111
|
+
# encoded member manifests.
|
|
112
|
+
manifests = JSON.parse(File.read(batch_file(ctx.config_dir, idx)))
|
|
113
|
+
manifests.each_with_index do |manifest, pos|
|
|
114
|
+
render_batch_member(ctx, idx, pos, manifest)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def render_batch_member(ctx, idx, pos, manifest)
|
|
119
|
+
one_path = File.join(ctx.config_dir, "._suma_staged_b#{idx}_#{pos}.yml")
|
|
120
|
+
out = File.join(ctx.out_dir, "._staged_out_b#{idx}_#{pos}")
|
|
121
|
+
started = Time.now
|
|
122
|
+
render_and_alias(render_member(one_path, ctx, out), manifest, ctx)
|
|
123
|
+
log_batch_ok(ctx, idx, pos, started)
|
|
124
|
+
ensure
|
|
125
|
+
FileUtils.rm_f(one_path)
|
|
126
|
+
FileUtils.rm_rf(out)
|
|
127
|
+
GC.start
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def render_and_alias(collection, manifest, ctx)
|
|
131
|
+
alias_artifacts(collection, slug(member_identifier(manifest)),
|
|
132
|
+
ctx.store_dir)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def render_member(manifest_path, ctx, out)
|
|
136
|
+
compile = { install_fonts: false }
|
|
137
|
+
compile[:novalid] = true if ctx.no_valid == "1"
|
|
138
|
+
collection = Metanorma::Collection.parse(manifest_path)
|
|
139
|
+
collection.render(
|
|
140
|
+
format: [:xml], output_folder: out,
|
|
141
|
+
preserve_unresolved: true, artifact_store_dir: ctx.store_dir,
|
|
142
|
+
compile: compile
|
|
143
|
+
)
|
|
144
|
+
collection
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def log_batch_ok(ctx, idx, pos, started)
|
|
148
|
+
warn format(
|
|
149
|
+
"[staged worker %<pid>d] batch %<idx>d/%<count>d member " \
|
|
150
|
+
"%<pos>d OK (%<seconds>d s)",
|
|
151
|
+
pid: Process.pid, idx: idx + 1, count: ctx.batch_count, pos: pos,
|
|
152
|
+
seconds: Time.now - started
|
|
153
|
+
)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The artefact aliasing both stage paths perform after a member's
|
|
157
|
+
# store write (metanorma/suma#204): the store keys on the rendered
|
|
158
|
+
# document's own docid slug, while the parent reinflates and the
|
|
159
|
+
# warm-skip look up by the docref identifier. Without these copies a
|
|
160
|
+
# render banks entries the lookups can never find - the sequential
|
|
161
|
+
# stage never warm-skipped for exactly this reason.
|
|
162
|
+
def alias_artifacts(collection, id_slug, store_dir)
|
|
163
|
+
alias_from_bibdatas(collection, id_slug, store_dir)
|
|
164
|
+
alias_from_identifier_aliases(identifier_aliases(collection),
|
|
165
|
+
store_dir)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Alias from the collection's own bibdata keys: they are the docids
|
|
169
|
+
# this render stored under. A produced-files delta is not safe here
|
|
170
|
+
# - with concurrent workers another child's render lands in the
|
|
171
|
+
# shared store mid-flight, and a delta would alias other members'
|
|
172
|
+
# documents under this docref (suma#167). bibdatas is empty for
|
|
173
|
+
# corpus manifests (suma#181) - the identifier_aliases below drive
|
|
174
|
+
# the aliasing in that case.
|
|
175
|
+
def alias_from_bibdatas(collection, id_slug, store_dir)
|
|
176
|
+
return if id_slug.empty?
|
|
177
|
+
|
|
178
|
+
collection.bibdatas&.each_key do |key|
|
|
179
|
+
src = slug(key)
|
|
180
|
+
next if src.empty? || src == id_slug
|
|
181
|
+
|
|
182
|
+
copy_store_entries(src, id_slug, store_dir)
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# identifier_aliases maps raw identifier -> resolved docid; alias
|
|
187
|
+
# the resolved-docid entries under the raw identifier's slug.
|
|
188
|
+
def alias_from_identifier_aliases(aliases, store_dir)
|
|
189
|
+
aliases.each do |original, resolved|
|
|
190
|
+
src = slug(resolved)
|
|
191
|
+
dst = slug(original)
|
|
192
|
+
next if src.empty? || dst.empty? || src == dst
|
|
193
|
+
|
|
194
|
+
copy_store_entries(src, dst, store_dir)
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def copy_store_entries(src, dst, store_dir)
|
|
199
|
+
Dir[File.join(store_dir, "#{src}.*.semantic.xml")].each do |f|
|
|
200
|
+
FileUtils.cp(f, f.sub(%r{/#{Regexp.escape(src)}\.}, "/#{dst}."))
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def identifier_aliases(collection)
|
|
205
|
+
collection.manifest.identifier_aliases || {}
|
|
206
|
+
rescue NoMethodError
|
|
207
|
+
{}
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def member_identifier(manifest)
|
|
211
|
+
manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def claim_batch(config_dir, count)
|
|
215
|
+
(0...count).find do |i|
|
|
216
|
+
try_claim(File.join(config_dir, "._suma_batch_#{i}.claim"))
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# O_EXCL create: succeeds for exactly one child, fails with
|
|
221
|
+
# EEXIST for everyone else - the claim.
|
|
222
|
+
def try_claim(path)
|
|
223
|
+
File.open(path, File::RDWR | File::CREAT | File::EXCL) do |claim|
|
|
224
|
+
claim.write("1")
|
|
225
|
+
end
|
|
226
|
+
true
|
|
227
|
+
rescue Errno::EEXIST
|
|
228
|
+
false
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def batch_file(config_dir, idx)
|
|
232
|
+
File.join(config_dir, "._suma_batch_#{idx}.bin")
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Mirrors the store's filename slug (metanorma ArtifactStore#slug).
|
|
236
|
+
# The \A/\z anchors are load-bearing: they trim leading/trailing
|
|
237
|
+
# dashes only, and losing them in generated text was the suma#166
|
|
238
|
+
# defect.
|
|
239
|
+
def slug(value)
|
|
240
|
+
value.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
|
|
241
|
+
.gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
end
|
|
@@ -37,10 +37,65 @@ module Suma
|
|
|
37
37
|
class StagedCollectionBuilder
|
|
38
38
|
STORE_DIRNAME = ".metanorma-collection-cache"
|
|
39
39
|
|
|
40
|
+
autoload :Worker, "suma/staged_collection_builder/worker"
|
|
41
|
+
|
|
42
|
+
# The child-process boot line: one require, no embedded logic. All
|
|
43
|
+
# worker code lives in Worker (lib/suma/staged_collection_builder/
|
|
44
|
+
# worker.rb) — generated script strings shipped dead code once
|
|
45
|
+
# (suma#166) and could only be spec-pinned as text.
|
|
46
|
+
CHILD_BOOT = 'require "suma"; ' \
|
|
47
|
+
"Suma::StagedCollectionBuilder::Worker.run"
|
|
48
|
+
|
|
40
49
|
# Members rendered per child process: one Metanorma boot amortised over
|
|
41
50
|
# a batch of small members keeps boot cost under a few percent.
|
|
42
51
|
DEFAULT_BATCH_SIZE = 25
|
|
43
52
|
|
|
53
|
+
# The warm-skip identity of one staged-store entry (metanorma/suma#203):
|
|
54
|
+
# the digest of the rendering toolchain that produced it, banked in a
|
|
55
|
+
# sidecar beside the stored semantic XML (raw hex, the form the
|
|
56
|
+
# SchemaCompiler render key uses). Presence-keyed reuse is unsound when
|
|
57
|
+
# a workflow cache carries a store across a toolchain change; an entry
|
|
58
|
+
# whose sidecar is missing (legacy store) or records a different
|
|
59
|
+
# toolchain is re-rendered, and every freshly rendered entry is
|
|
60
|
+
# re-banked — so a restored store converges onto the current toolchain
|
|
61
|
+
# at the cost of at most one extra render per entry.
|
|
62
|
+
class StoreEntry
|
|
63
|
+
SIDECHAR = ".toolchain"
|
|
64
|
+
|
|
65
|
+
class << self
|
|
66
|
+
# Bank +digest+ as the producing toolchain of the entry stored at
|
|
67
|
+
# +stored_path+.
|
|
68
|
+
def bank(stored_path, digest)
|
|
69
|
+
File.write(sidecar_path(stored_path), digest)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# The toolchain digest banked for the entry, or nil for a legacy
|
|
73
|
+
# entry written before banking existed.
|
|
74
|
+
def toolchain_digest(stored_path)
|
|
75
|
+
path = sidecar_path(stored_path)
|
|
76
|
+
File.file?(path) ? File.read(path) : nil
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Whether the entry was produced by the toolchain +digest+
|
|
80
|
+
# identifies. A legacy entry (no sidecar) never matches, so the
|
|
81
|
+
# first build over an old store re-renders once and re-banks.
|
|
82
|
+
def current?(stored_path, digest)
|
|
83
|
+
toolchain_digest(stored_path) == digest
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# The banked digest truncated for log lines.
|
|
87
|
+
def short_digest(stored_path)
|
|
88
|
+
(toolchain_digest(stored_path) || "unknown")[0, 8]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def sidecar_path(stored_path)
|
|
94
|
+
"#{stored_path}#{SIDECHAR}"
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
44
99
|
# +collection_config_path+ is the emitted collection manifest (the same
|
|
45
100
|
# +collection-output.yaml+ the normal build renders). Member filerefs are
|
|
46
101
|
# relative to its directory, so all per-member manifests are written there.
|
|
@@ -84,6 +139,42 @@ module Suma
|
|
|
84
139
|
reinflate(reinflate_selection(members), members)
|
|
85
140
|
end
|
|
86
141
|
|
|
142
|
+
# The warm-skip query (metanorma/suma#203): members whose stored entry
|
|
143
|
+
# was banked under the CURRENT rendering toolchain are done; every
|
|
144
|
+
# other member — no stored entry, a legacy entry without a banked
|
|
145
|
+
# digest, or an entry another toolchain produced — renders again.
|
|
146
|
+
# A workflow cache can carry a store across a toolchain or content-key
|
|
147
|
+
# change (TRThurman, iso-10303#858), and presence alone cannot see it:
|
|
148
|
+
# the banked digest is the durable guarantee the restored store cannot
|
|
149
|
+
# fake. Each member gets one provenance line: what was taken from the
|
|
150
|
+
# store and what compiled it, or that it renders.
|
|
151
|
+
def unstaged_members(members)
|
|
152
|
+
members.reject do |member|
|
|
153
|
+
identifier = member["identifier"]
|
|
154
|
+
stored = identifier && stored_semantic(identifier)
|
|
155
|
+
if stored && StoreEntry.current?(stored, Provenance.render_toolchain_digest)
|
|
156
|
+
Utils.log "[staged] #{identifier}: restored from store " \
|
|
157
|
+
"(toolchain #{StoreEntry.short_digest(stored)})"
|
|
158
|
+
true
|
|
159
|
+
else
|
|
160
|
+
Utils.log "[staged] #{identifier}: rendering"
|
|
161
|
+
false
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Record the rendering toolchain beside every freshly rendered
|
|
167
|
+
# member's store entry, so the next warm build can trust-or-recheck
|
|
168
|
+
# it. A legacy entry without a digest is re-rendered once and re-banked
|
|
169
|
+
# with one (safe migration); a warm-skipped member was already
|
|
170
|
+
# validated against the current digest and keeps its sidecar.
|
|
171
|
+
def bank_toolchains(members)
|
|
172
|
+
members.each do |member|
|
|
173
|
+
stored = stored_semantic(member["identifier"]) or next
|
|
174
|
+
StoreEntry.bank(stored, Provenance.render_toolchain_digest)
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
87
178
|
private
|
|
88
179
|
|
|
89
180
|
# The staged slice of the collection: an isolate selection wins over a
|
|
@@ -92,13 +183,16 @@ module Suma
|
|
|
92
183
|
@isolate ? isolate_members(members) : shard_members(members)
|
|
93
184
|
end
|
|
94
185
|
|
|
95
|
-
# Render the staged members, serially or across the worker pool
|
|
186
|
+
# Render the staged members, serially or across the worker pool, then
|
|
187
|
+
# bank every freshly rendered member's toolchain identity.
|
|
96
188
|
def stage_members(staged)
|
|
189
|
+
unstaged = unstaged_members(staged)
|
|
97
190
|
if @workers == 1
|
|
98
|
-
|
|
191
|
+
unstaged.each_with_index { |member, i| stage_member(member, i) }
|
|
99
192
|
else
|
|
100
|
-
stage_parallel(
|
|
193
|
+
stage_parallel(unstaged)
|
|
101
194
|
end
|
|
195
|
+
bank_toolchains(unstaged)
|
|
102
196
|
end
|
|
103
197
|
|
|
104
198
|
# The reinflated slice: all members unless a reinflate shard is given,
|
|
@@ -156,19 +250,6 @@ module Suma
|
|
|
156
250
|
"skipping reinflate"
|
|
157
251
|
end
|
|
158
252
|
|
|
159
|
-
# Members already present in the store (their docref-slug artefact
|
|
160
|
-
# exists) are not re-rendered: an interrupted or repeated build over
|
|
161
|
-
# a warm store goes straight to reinflate, so the assembly pass costs
|
|
162
|
-
# minutes instead of re-rendering the whole collection.
|
|
163
|
-
def unstaged_members(members)
|
|
164
|
-
members.reject do |member|
|
|
165
|
-
identifier = member["identifier"]
|
|
166
|
-
stored = identifier && stored_semantic(identifier)
|
|
167
|
-
Utils.log "[staged] stored, skipping #{identifier}" if stored
|
|
168
|
-
stored
|
|
169
|
-
end
|
|
170
|
-
end
|
|
171
|
-
|
|
172
253
|
# Batch the members and run +@workers+ renderer child processes
|
|
173
254
|
# concurrently. Each batch file carries fully-rendered single-member
|
|
174
255
|
# manifests; children claim batches through O_EXCL markers, so a slow
|
|
@@ -177,8 +258,9 @@ module Suma
|
|
|
177
258
|
# child exit hands its memory back to the OS. Members write disjoint,
|
|
178
259
|
# identifier-keyed store entries, so concurrent children cannot race;
|
|
179
260
|
# sectionsplit only happens at reinflate (a single sequential pass).
|
|
261
|
+
# +members+ is the warm-skip's unstaged remainder.
|
|
180
262
|
def stage_parallel(members)
|
|
181
|
-
paths, count = write_batch_files(
|
|
263
|
+
paths, count = write_batch_files(members)
|
|
182
264
|
run_batch_workers(count)
|
|
183
265
|
ensure
|
|
184
266
|
remove_batch_files(paths, count)
|
|
@@ -202,14 +284,14 @@ module Suma
|
|
|
202
284
|
def write_batch_bins(batches)
|
|
203
285
|
batches.map.with_index do |batch, i|
|
|
204
286
|
path = File.join(@config_dir, "._suma_batch_#{i}.bin")
|
|
205
|
-
File.
|
|
287
|
+
File.write(path, Worker.dump_options(batch))
|
|
206
288
|
path
|
|
207
289
|
end
|
|
208
290
|
end
|
|
209
291
|
|
|
210
292
|
def run_batch_workers(count)
|
|
211
293
|
children = Array.new([@workers, count].min) do
|
|
212
|
-
Process.spawn(RbConfig.ruby, "-e",
|
|
294
|
+
Process.spawn(RbConfig.ruby, "-e", CHILD_BOOT, "batch", @config_dir,
|
|
213
295
|
count.to_s, @store_dir, @output_directory,
|
|
214
296
|
@no_valid ? "1" : "0", chdir: @config_dir)
|
|
215
297
|
end
|
|
@@ -232,122 +314,6 @@ module Suma
|
|
|
232
314
|
# Child renderer: claims and renders batches until none remain. The
|
|
233
315
|
# Metanorma boot is paid once per child, and GC runs between members to
|
|
234
316
|
# bound the heap. Any render failure exits nonzero and fails the build.
|
|
235
|
-
def child_script
|
|
236
|
-
<<~'RUBY'
|
|
237
|
-
require "yaml"
|
|
238
|
-
require "fileutils"
|
|
239
|
-
require "metanorma"
|
|
240
|
-
config_dir, batch_count, store_dir, out_dir, no_valid = ARGV
|
|
241
|
-
count = Integer(batch_count)
|
|
242
|
-
claimed = 0
|
|
243
|
-
loop do
|
|
244
|
-
idx = (0...count).find do |i|
|
|
245
|
-
begin
|
|
246
|
-
File.open(File.join(config_dir, "._suma_batch_#{i}.claim"),
|
|
247
|
-
File::RDWR | File::CREAT | File::EXCL) {}
|
|
248
|
-
true
|
|
249
|
-
rescue Errno::EEXIST
|
|
250
|
-
false
|
|
251
|
-
end
|
|
252
|
-
end
|
|
253
|
-
break unless idx
|
|
254
|
-
|
|
255
|
-
manifests = Marshal.load(File.binread(File.join(
|
|
256
|
-
config_dir, "._suma_batch_#{idx}.bin")))
|
|
257
|
-
manifests.each_with_index do |manifest, j|
|
|
258
|
-
one_path = File.join(config_dir,
|
|
259
|
-
"._suma_staged_b#{idx}_#{j}.yml")
|
|
260
|
-
out = File.join(out_dir, "._staged_out_b#{idx}_#{j}")
|
|
261
|
-
t0 = Time.now
|
|
262
|
-
begin
|
|
263
|
-
File.write(one_path, YAML.dump(manifest))
|
|
264
|
-
# The artefact store keys on the rendered document's own
|
|
265
|
-
# docid, while the parent reinflates by the docref
|
|
266
|
-
# identifier; members whose document docid differs from the
|
|
267
|
-
# docref (docrefs carrying metanorma's GUID marker) were
|
|
268
|
-
# therefore unrecoverable ("no stored artefact"). Alias the
|
|
269
|
-
# artefacts keyed by the render's own bibdata keys under the
|
|
270
|
-
# docref slug too. Shape: single_member_manifest =>
|
|
271
|
-
# {"manifest" => {"manifest" => [{"docref" => [dr]}]}}.
|
|
272
|
-
member_id = manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
|
|
273
|
-
id_slug = member_id.to_s
|
|
274
|
-
.gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
|
|
275
|
-
.gsub(/\A-|-\z/, "")
|
|
276
|
-
compile = { install_fonts: false }
|
|
277
|
-
compile[:novalid] = true if no_valid == "1"
|
|
278
|
-
collection = Metanorma::Collection.parse(one_path)
|
|
279
|
-
collection.render(
|
|
280
|
-
format: [:xml], output_folder: out,
|
|
281
|
-
preserve_unresolved: true, artifact_store_dir: store_dir,
|
|
282
|
-
compile: compile,
|
|
283
|
-
)
|
|
284
|
-
unless id_slug.empty?
|
|
285
|
-
# Alias strictly from the collection's own bibdata keys:
|
|
286
|
-
# they are the docids this member's render stored under.
|
|
287
|
-
# A produced-files delta is not safe here - with
|
|
288
|
-
# concurrent workers another child's render lands in the
|
|
289
|
-
# shared store mid-flight, and a delta would alias other
|
|
290
|
-
# members' documents under this docref (suma#167).
|
|
291
|
-
(collection.bibdatas || {}).keys.each do |key|
|
|
292
|
-
src_slug = key.to_s
|
|
293
|
-
.gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
|
|
294
|
-
.gsub(/\A-|-\z/, "")
|
|
295
|
-
next if src_slug.empty? || src_slug == id_slug
|
|
296
|
-
|
|
297
|
-
Dir[File.join(store_dir,
|
|
298
|
-
"#{src_slug}.*.semantic.xml")].each do |f|
|
|
299
|
-
alias_name = f.sub(
|
|
300
|
-
%r{/#{Regexp.escape(src_slug)}.},
|
|
301
|
-
"/#{id_slug}.",
|
|
302
|
-
)
|
|
303
|
-
FileUtils.cp(f, alias_name)
|
|
304
|
-
end
|
|
305
|
-
end
|
|
306
|
-
end
|
|
307
|
-
|
|
308
|
-
# The corpus manifests carry no bibdata, so bibdatas is
|
|
309
|
-
# empty and the loop above never runs (suma#181): the
|
|
310
|
-
# artefacts stay keyed under the document's own docid and
|
|
311
|
-
# reinflate - looking up by the docref identifier - finds
|
|
312
|
-
# nothing on a cold store ("no stored artefact"). metanorma
|
|
313
|
-
# resolves every manifest identifier to the document docid
|
|
314
|
-
# during populate and records the mapping on the manifest
|
|
315
|
-
# itself (identifier_aliases, raw -> resolved). That map is
|
|
316
|
-
# the child's own post-render collection - reading it is
|
|
317
|
-
# race-free per the suma#167 discipline.
|
|
318
|
-
aliases = begin
|
|
319
|
-
collection.manifest.identifier_aliases || {}
|
|
320
|
-
rescue NoMethodError
|
|
321
|
-
{}
|
|
322
|
-
end
|
|
323
|
-
aliases.each do |original, resolved|
|
|
324
|
-
src = resolved.to_s
|
|
325
|
-
.gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
|
|
326
|
-
.gsub(/\A-|-\z/, "")
|
|
327
|
-
dst = original.to_s
|
|
328
|
-
.gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
|
|
329
|
-
.gsub(/\A-|-\z/, "")
|
|
330
|
-
next if src.empty? || dst.empty? || src == dst
|
|
331
|
-
|
|
332
|
-
Dir[File.join(store_dir, "#{src}.*.semantic.xml")].each do |f|
|
|
333
|
-
FileUtils.cp(
|
|
334
|
-
f, f.sub(%r{/#{Regexp.escape(src)}\.}, "/#{dst}."),
|
|
335
|
-
)
|
|
336
|
-
end
|
|
337
|
-
end
|
|
338
|
-
warn format("[staged worker %d] batch %d/%d member %d OK (%ds)",
|
|
339
|
-
Process.pid, idx + 1, count, j, Time.now - t0)
|
|
340
|
-
ensure
|
|
341
|
-
File.unlink(one_path) rescue nil
|
|
342
|
-
FileUtils.rm_rf(out) rescue nil
|
|
343
|
-
GC.start
|
|
344
|
-
end
|
|
345
|
-
end
|
|
346
|
-
claimed += 1
|
|
347
|
-
end
|
|
348
|
-
exit 1 if claimed.zero? && count.positive?
|
|
349
|
-
RUBY
|
|
350
|
-
end
|
|
351
317
|
|
|
352
318
|
def manifest
|
|
353
319
|
@manifest ||= YAML.load_file(@config_path)
|
|
@@ -388,6 +354,7 @@ module Suma
|
|
|
388
354
|
opts: { format: [:xml], output_folder: out,
|
|
389
355
|
preserve_unresolved: true, artifact_store_dir: @store_dir,
|
|
390
356
|
compile: compile_flags },
|
|
357
|
+
mode: "stage", id_slug: slug(member["identifier"])
|
|
391
358
|
)
|
|
392
359
|
ensure
|
|
393
360
|
FileUtils.rm_f(one_path)
|
|
@@ -625,20 +592,20 @@ module Suma
|
|
|
625
592
|
|
|
626
593
|
# Mirrors the store's filename slug (metanorma ArtifactStore#slug).
|
|
627
594
|
def slug(identifier)
|
|
628
|
-
|
|
629
|
-
.gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
|
|
595
|
+
Worker.slug(identifier)
|
|
630
596
|
end
|
|
631
597
|
|
|
632
598
|
# Render a manifest in a FRESH child process so its memory is reclaimed on
|
|
633
|
-
# exit. Under +bundle exec+, the child inherits the bundle. Raises on
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
599
|
+
# exit. Under +bundle exec+, the child inherits the bundle. Raises on
|
|
600
|
+
# failure. +mode+ "render" plain-renders (reinflate); "stage" additionally
|
|
601
|
+
# aliases the member's store artefacts under +id_slug+, the same aliasing
|
|
602
|
+
# the batch child path performs (suma#204 - without it the sequential
|
|
603
|
+
# stage banked docid-slug-only entries and never warm-skipped).
|
|
604
|
+
def run_render(manifest_path:, opts:, mode: "render", id_slug: nil)
|
|
605
|
+
argv = [mode, manifest_path]
|
|
606
|
+
argv += [@store_dir, id_slug] if mode == "stage"
|
|
607
|
+
IO.popen([RbConfig.ruby, "-e", CHILD_BOOT, *argv], "wb") do |io|
|
|
608
|
+
io.write(Worker.dump_options(opts))
|
|
642
609
|
io.close_write
|
|
643
610
|
end
|
|
644
611
|
return if $?.success?
|
|
@@ -13,13 +13,24 @@ module Suma
|
|
|
13
13
|
# +#call(index, total, report)+. +NullProgress+ is the default
|
|
14
14
|
# no-op; the CLI passes a lambda that writes to +$stderr+.
|
|
15
15
|
class Scanner
|
|
16
|
-
|
|
16
|
+
# The operative metanorma profile ships from metanorma-standoc by
|
|
17
|
+
# path: svg_conform's bundled copy is a sample only, and the
|
|
18
|
+
# "metanorma" engine name fails loudly by design (svg_conform#132).
|
|
19
|
+
STANDOC_PROFILE_PATH =
|
|
20
|
+
"lib/metanorma/standoc/svg_conform_profile/metanorma.yml"
|
|
21
|
+
|
|
22
|
+
def self.default_profile
|
|
23
|
+
File.join(
|
|
24
|
+
Gem::Specification.find_by_name("metanorma-standoc").gem_dir,
|
|
25
|
+
STANDOC_PROFILE_PATH,
|
|
26
|
+
)
|
|
27
|
+
end
|
|
17
28
|
|
|
18
29
|
attr_reader :profile, :progress
|
|
19
30
|
|
|
20
|
-
def initialize(profile:
|
|
31
|
+
def initialize(profile: nil,
|
|
21
32
|
progress: NullProgress.new)
|
|
22
|
-
@profile = profile
|
|
33
|
+
@profile = profile || self.class.default_profile
|
|
23
34
|
@progress = progress
|
|
24
35
|
end
|
|
25
36
|
|
data/lib/suma/term_extractor.rb
CHANGED
|
@@ -288,7 +288,7 @@ module Suma
|
|
|
288
288
|
"#{entity_name_to_text(entity.id)} #{entity_ref}"
|
|
289
289
|
else
|
|
290
290
|
entity_subtypes = subtypes.map do |e|
|
|
291
|
-
owner =
|
|
291
|
+
owner = declaring_owner(schema, e.id)
|
|
292
292
|
urn_mention(express_entity_urn("#{owner}.#{e.id}"), e.id)
|
|
293
293
|
end
|
|
294
294
|
|
|
@@ -299,6 +299,13 @@ module Suma
|
|
|
299
299
|
end
|
|
300
300
|
end
|
|
301
301
|
|
|
302
|
+
# The owner a supertype citation qualifies with: the closure declarer,
|
|
303
|
+
# else the sibling-schema declarer, else the citing schema.
|
|
304
|
+
def declaring_owner(schema, entity_id)
|
|
305
|
+
declaring_schema_id(schema, entity_id) ||
|
|
306
|
+
sibling_declaring_schema_id(entity_id) || schema.id
|
|
307
|
+
end
|
|
308
|
+
|
|
302
309
|
# The schema an imported declaration is declared in, resolved through
|
|
303
310
|
# +schema+'s USE-FROM closure — the same resolution the source remark
|
|
304
311
|
# xrefs carry: a local declaration wins; otherwise the USE FROM edges
|
|
@@ -321,6 +328,39 @@ module Suma
|
|
|
321
328
|
end
|
|
322
329
|
end
|
|
323
330
|
|
|
331
|
+
# When the closure walk finds no declarer, the declaration may sit
|
|
332
|
+
# in a sibling schema with no interface edge to the citing one: the
|
|
333
|
+
# _arm/_mim pairs cite across without USE FROM/REFERENCE FROM, so
|
|
334
|
+
# `X_arm.Y` lands on a concept declared under `X_mim` (TRThurman's
|
|
335
|
+
# 148 distinct wrong-qualifier targets, iso-10303#835 /
|
|
336
|
+
# metanorma/suma#197). Scan the manifest's schemas for the entity
|
|
337
|
+
# name; exactly one declarer resolves, several stay nil.
|
|
338
|
+
def sibling_declaring_schema_id(entity_id)
|
|
339
|
+
@sibling_cache ||= {}
|
|
340
|
+
@sibling_cache.fetch(entity_id) do
|
|
341
|
+
declarers = declarers_in_manifest(entity_id)
|
|
342
|
+
unique_declarer(declarers)
|
|
343
|
+
end
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# The manifest schemas whose entity collection declares +entity_id+.
|
|
347
|
+
def declarers_in_manifest(entity_id)
|
|
348
|
+
schema_manifest_paths.keys.filter_map do |id|
|
|
349
|
+
loaded = loaded_schema(id)
|
|
350
|
+
declares_entity?(loaded, entity_id) ? id : nil
|
|
351
|
+
end
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
def declares_entity?(schema, entity_id)
|
|
355
|
+
schema&.entities&.any? { |e| e.id == entity_id } ? true : false
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# The single distinct declarer, or nil when none or several — an
|
|
359
|
+
# ambiguous name never resolves to a guess.
|
|
360
|
+
def unique_declarer(declarers)
|
|
361
|
+
declarers.uniq.one? ? declarers.first : nil
|
|
362
|
+
end
|
|
363
|
+
|
|
324
364
|
# Distinct schema ids within +schema+'s USE-FROM closure declaring
|
|
325
365
|
# +entity_id+. An interface whose item list names the declaration
|
|
326
366
|
# claims it outright (that is what the item list means in EXPRESS) —
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.40
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: base64
|
|
@@ -243,6 +243,7 @@ files:
|
|
|
243
243
|
- lib/suma/corpus_graph.rb
|
|
244
244
|
- lib/suma/delivery.rb
|
|
245
245
|
- lib/suma/delivery/cross_page_fragments.rb
|
|
246
|
+
- lib/suma/delivery/diagnostics.rb
|
|
246
247
|
- lib/suma/delivery/expectations.rb
|
|
247
248
|
- lib/suma/delivery/frame_nav.rb
|
|
248
249
|
- lib/suma/delivery/pack.rb
|
|
@@ -292,6 +293,7 @@ files:
|
|
|
292
293
|
- lib/suma/site_build.rb
|
|
293
294
|
- lib/suma/site_config.rb
|
|
294
295
|
- lib/suma/staged_collection_builder.rb
|
|
296
|
+
- lib/suma/staged_collection_builder/worker.rb
|
|
295
297
|
- lib/suma/svg_quality.rb
|
|
296
298
|
- lib/suma/svg_quality/batch_report.rb
|
|
297
299
|
- lib/suma/svg_quality/formatters.rb
|