suma 0.5.38 → 0.5.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: afd727de6d181c16bca36f90e25d84cce00b70b3d1ff5b30190812c5dccbc4ec
4
- data.tar.gz: b75f867c8e9f2a47cf380cb076938924aaaef297546244c6c29c1e09cf23b16b
3
+ metadata.gz: 0a7c2ce507254463b15a6f6e30ab2d0b664bc841ada2de657af4fc14aba7f012
4
+ data.tar.gz: f5abf56a5ca249f795793653cc43b835283d7e6685ad8f9cb013e452e041e944
5
5
  SHA512:
6
- metadata.gz: 07bb788123a2cd052a2ccffd4deb2b1468f0cc6dc83cd4964c4af185ded66d037fb5fd2be2486c900759c8a3ac4e850ca138fcd23ecd57e7e4c2e69165751cbf
7
- data.tar.gz: 3b0cc365321032218dd12f98752bb38d4e87569fe1487228f77d46c754e60b22a65e1066086e112943fc832a10eec7804f6f868295807e6e52e0e2b5cf4d81bb
6
+ metadata.gz: 2c56115e2779608109c3301ce743ddf5e25541f9a61ede17ea95d9f7afe62e93b75d70aabc69ddc8fcc222ed24c88e66570a991576f200980097e5f78abd8eaf
7
+ data.tar.gz: 685e437b51483bcd56444d26c1bdf7c0aa649c0e9f0e8b43fc4b43990c754bffd85c36b997b64a35592a5cddf056adaf7e4e0847a7e6c210bb0b8eafdaacf4a4
@@ -11,7 +11,7 @@ module Suma
11
11
  class CheckSvgQuality
12
12
  DATA_PATH = "schemas"
13
13
  DEFAULT_PATTERN = "**/*.svg"
14
- DEFAULT_PROFILE = :metanorma
14
+ DEFAULT_PROFILE = Suma::SvgQuality::Scanner.default_profile
15
15
 
16
16
  def initialize(pattern: DEFAULT_PATTERN, profile: DEFAULT_PROFILE,
17
17
  format: "terminal", output: nil, min_errors: nil,
@@ -26,8 +26,10 @@ module Suma
26
26
  desc: "YAML manifest of site-shape thresholds " \
27
27
  "(sections_min, index_pages, " \
28
28
  "schema_attachments_min, schema_docs_min, " \
29
- "missing_express_max, broken_links_max); " \
30
- "violations fail the verify"
29
+ "missing_express_max, broken_links_max, " \
30
+ "diagnostics_max: category => rule => " \
31
+ "max rows over the site's .err.html " \
32
+ "files); violations fail the verify"
31
33
 
32
34
  def verify(site_dir)
33
35
  result = Suma::Delivery::SiteVerifier.new(site_dir,
@@ -40,15 +42,19 @@ module Suma
40
42
  # Without an expectations manifest the contract is zero broken
41
43
  # links. With one, broken_links_max IS the broken-link contract:
42
44
  # the unconditional raise would fire first and make the cap
43
- # unreachable (TRThurman, iso-10303#835).
44
- def verify_failures(result, failures)
45
- if options[:expectations]
46
- raise "expectations violated: #{failures.join('; ')}" unless
47
- failures.empty?
48
- elsif result.broken_count.zero?
49
- nil
50
- else
51
- raise "broken links found (#{result.broken_count})"
45
+ # unreachable (TRThurman, iso-10303#835). Defined inside
46
+ # no_commands so Thor does not register a command for a helper
47
+ # and warn about the missing usage/description on every run.
48
+ no_commands do
49
+ def verify_failures(result, failures)
50
+ if options[:expectations]
51
+ raise "expectations violated: #{failures.join('; ')}" unless
52
+ failures.empty?
53
+ elsif result.broken_count.zero?
54
+ nil
55
+ else
56
+ raise "broken links found (#{result.broken_count})"
57
+ end
52
58
  end
53
59
  end
54
60
 
@@ -171,7 +177,7 @@ module Suma
171
177
  "broken; fragments: #{r['total_fragments']} total, " \
172
178
  "#{r['broken_fragments'].size} broken; " \
173
179
  "#{r['cross_document_links']} cross-document; " \
174
- "#{inventory_summary(r)}"
180
+ "#{inventory_summary(r)}; #{diagnostics_summary(r)}"
175
181
  end
176
182
 
177
183
  def inventory_summary(report)
@@ -183,6 +189,14 @@ module Suma
183
189
  "missing-express pages=#{report['missing_express_pages'].size}"
184
190
  end
185
191
 
192
+ # The err.html census the verify walked: total diagnostic rows over
193
+ # the categories the site's error reports carry.
194
+ def diagnostics_summary(report)
195
+ census = report["diagnostics"] || {}
196
+ "diagnostics: #{census.sum { |_, rules| rules.values.sum }} rows " \
197
+ "over #{census.size} categories"
198
+ end
199
+
186
200
  def write_report(report)
187
201
  File.write(options[:out], JSON.pretty_generate(report))
188
202
  end
@@ -0,0 +1,136 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "leptris"
4
+
5
+ module Suma
6
+ module Delivery
7
+ # Diagnostic-row census over metanorma .err.html error reports
8
+ # (metanorma/suma#203): rows grouped by CATEGORY and RULE.
9
+ #
10
+ # A gate keyed on a literal code string breaks when the codes migrate
11
+ # (iso-10303#859: the part2 gate greps STANDOC_38 in document.err.html
12
+ # while metanorma-iso 3.5.14 re-emitted those rows as ISO_56, so the
13
+ # gate reads 0 over 476 live rows). Codes migrate; the stable identity
14
+ # of a diagnostic row is category + rule, so the census keys on both.
15
+ #
16
+ # The err.html writer is Metanorma::Utils::Log#write: one
17
+ # `<h2 id="...">Category</h2>` per category, each followed by a table
18
+ # whose data rows are six cells - Line, ID, Error (the rule code,
19
+ # e.g. STANDOC_27 or ISO_56), Message, Context, Severity.
20
+ class Diagnostics
21
+ # The Error cell's position within a data row.
22
+ RULE_CELL_INDEX = 2
23
+
24
+ class << self
25
+ # Parse one err.html given as a file path or an HTML string.
26
+ #
27
+ # @return [Counts] rows grouped by category, then rule
28
+ def count(path_or_html)
29
+ html = if File.file?(path_or_html)
30
+ File.read(path_or_html)
31
+ else
32
+ path_or_html
33
+ end
34
+ new(html).counts
35
+ end
36
+
37
+ # The census of every err.html under a rendered site, summed:
38
+ # the site-level figure an expectations gate asserts against.
39
+ def count_site(site_dir)
40
+ Dir.glob(File.join(site_dir, "**", "*.err.html"))
41
+ .reduce(Counts.new) { |sum, path| sum.merge(count(path)) }
42
+ end
43
+ end
44
+
45
+ def initialize(html)
46
+ @doc = Leptris::XML.parse_html(html)
47
+ end
48
+
49
+ # Every data row of every category table, keyed by its h2 heading
50
+ # (the log category) and Error cell (the rule code).
51
+ def counts
52
+ Counts.new(tally)
53
+ end
54
+
55
+ private
56
+
57
+ # Per category, per rule, the number of data rows.
58
+ def tally
59
+ rows = Hash.new { |hash, category| hash[category] = Hash.new(0) }
60
+ @doc.css("h2").each do |heading|
61
+ rules_of(heading).each do |rule|
62
+ rows[heading.text.strip][rule] += 1
63
+ end
64
+ end
65
+ rows
66
+ end
67
+
68
+ # The rule code of every data row in the heading's table.
69
+ def rules_of(heading)
70
+ table = heading.xpath("following-sibling::table[1]").first
71
+ return [] unless table
72
+
73
+ table.css("tr").filter_map { |row| rule_of(row) }
74
+ end
75
+
76
+ # The Error cell of a row: the third of Line/ID/Error/Message/
77
+ # Context/Severity (the ID cell renders as a `th`).
78
+ def rule_of(row)
79
+ rule = row.xpath("td|th")[RULE_CELL_INDEX]&.text&.strip
80
+ rule.nil? || rule.empty? ? nil : rule
81
+ end
82
+
83
+ # The per-category, per-rule census of one or more err.html
84
+ # documents: immutable value object over
85
+ # { "Style" => { "STANDOC_48" => 15 } }, enumerable over
86
+ # (category, rule, count) triples.
87
+ class Counts
88
+ include Enumerable
89
+
90
+ def initialize(categories = {})
91
+ @categories = categories
92
+ .transform_values { |rules| rules.transform_values(&:to_i).freeze }
93
+ .freeze
94
+ end
95
+
96
+ # The raw census hash: category => (rule => row count). Frozen.
97
+ def to_h
98
+ @categories
99
+ end
100
+
101
+ def each
102
+ return enum_for(__method__) unless block_given?
103
+
104
+ @categories.each do |category, rules|
105
+ rules.each { |rule, count| yield category, rule, count }
106
+ end
107
+ self
108
+ end
109
+
110
+ # Rows counted for one rule; an unreported rule counts zero.
111
+ def rule_count(category, rule)
112
+ @categories.dig(category, rule) || 0
113
+ end
114
+
115
+ # Every row counted for one category.
116
+ def total(category)
117
+ @categories.fetch(category, {}).values.sum
118
+ end
119
+
120
+ # Rows counted overall.
121
+ def rows
122
+ @categories.values.sum { |rules| rules.values.sum }
123
+ end
124
+
125
+ # The union census: counts of both sides summed.
126
+ def merge(other)
127
+ other = self.class.new({}) unless other.is_a?(self.class)
128
+ summed = other.to_h.merge(to_h) do |_category, theirs, ours|
129
+ theirs.merge(ours) { |_rule, a, b| a + b }
130
+ end
131
+ self.class.new(summed)
132
+ end
133
+ end
134
+ end
135
+ end
136
+ end
@@ -13,6 +13,10 @@ module Suma
13
13
  attribute :schema_docs_min, :integer
14
14
  attribute :missing_express_max, :integer, default: -> { 0 }
15
15
  attribute :broken_links_max, :integer, default: -> { 0 }
16
+ # Category => rule => maximum live diagnostic rows: the stable
17
+ # identity of an err.html row is category + rule, not the code,
18
+ # which migrates across metanorma releases (iso-10303#859).
19
+ attribute :diagnostics_max, :hash
16
20
 
17
21
  key_value do
18
22
  map "sections_min", to: :sections_min
@@ -21,13 +25,22 @@ module Suma
21
25
  map "schema_docs_min", to: :schema_docs_min
22
26
  map "missing_express_max", to: :missing_express_max
23
27
  map "broken_links_max", to: :broken_links_max
28
+ map "diagnostics_max", to: :diagnostics_max
24
29
  end
25
30
 
26
31
  # Human-readable violation lines for every threshold the +report+
27
32
  # and +broken_links+ breach; unset thresholds stay silent. The name
28
33
  # suffix is the comparison: `_min` floors, `_max` caps, a bare name
29
- # (index_pages) is exact.
34
+ # (index_pages) is exact. The nested diagnostics_max thresholds
35
+ # count against the report's diagnostics census.
30
36
  def violations(report, broken_links)
37
+ threshold_violations(report, broken_links) +
38
+ diagnostics_violations(report["diagnostics"])
39
+ end
40
+
41
+ private
42
+
43
+ def threshold_violations(report, broken_links)
31
44
  actuals(report, broken_links).filter_map do |name, actual|
32
45
  limit = public_send(name) or next
33
46
  relation = relation_of(name)
@@ -39,7 +52,20 @@ module Suma
39
52
  end
40
53
  end
41
54
 
42
- private
55
+ # diagnostics_max: { "Referencing" => { "ISO_56" => 0 } }. A rule
56
+ # absent from the census counts zero rows.
57
+ def diagnostics_violations(census)
58
+ Array(diagnostics_max).flat_map do |category, rules|
59
+ rules.filter_map do |rule, limit|
60
+ actual = (census || {}).dig(category, rule) || 0
61
+ next if actual <= limit
62
+
63
+ format("%<category>s/%<rule>s: got %<actual>d, expected at " \
64
+ "most %<limit>d", category: category, rule: rule,
65
+ actual: actual, limit: limit)
66
+ end
67
+ end
68
+ end
43
69
 
44
70
  def relation_of(name)
45
71
  return "at least" if name.end_with?("_min")
@@ -52,6 +52,7 @@ module Suma
52
52
  verify_links(doc, rel_path, html_file, anchor_index, report)
53
53
  end
54
54
  report["inventory"] = site_inventory
55
+ report["diagnostics"] = Diagnostics.count_site(@site_dir).to_h
55
56
  Report.new(report: report)
56
57
  end
57
58
 
@@ -71,6 +72,7 @@ module Suma
71
72
  "broken_intra_fragments" => [],
72
73
  "missing_express_pages" => [],
73
74
  "inventory" => {},
75
+ "diagnostics" => {},
74
76
  }
75
77
  end
76
78
 
data/lib/suma/delivery.rb CHANGED
@@ -6,6 +6,7 @@ module Suma
6
6
  module Delivery
7
7
  autoload :FrameNav, "suma/delivery/frame_nav"
8
8
  autoload :CrossPageFragments, "suma/delivery/cross_page_fragments"
9
+ autoload :Diagnostics, "suma/delivery/diagnostics"
9
10
  autoload :Expectations, "suma/delivery/expectations"
10
11
  autoload :Pack, "suma/delivery/pack"
11
12
  autoload :Plan, "suma/delivery/plan"
@@ -108,6 +108,15 @@ module Suma
108
108
  @render_toolchain ||= toolchain.merge("render" => render_gems)
109
109
  end
110
110
 
111
+ # The toolchain identity in the form cached artifacts record and
112
+ # compare (the staged store sidecars, the render key, the site-build
113
+ # key). A digest identifies the toolchain that produced a stored
114
+ # artifact without restating every gem version beside it.
115
+ def self.render_toolchain_digest
116
+ @render_toolchain_digest ||=
117
+ Digest::SHA256.hexdigest(JSON.generate(render_toolchain))
118
+ end
119
+
111
120
  def self.render_gems
112
121
  gem_specs.select { |spec| render_gem?(spec.name) }
113
122
  .sort_by { |spec| [spec.name, spec.version.to_s] }
@@ -0,0 +1,245 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "yaml"
5
+ require "fileutils"
6
+ require "metanorma"
7
+
8
+ module Suma
9
+ class StagedCollectionBuilder
10
+ # The code the staged builder's child processes run. Children are
11
+ # spawned as `ruby -e 'require "suma"; Worker.run'` with the mode and
12
+ # paths on ARGV and render options JSON-encoded on STDIN. All
13
+ # worker logic lives here as real, spec-testable methods: the previous
14
+ # implementation embedded this code as generated script strings, which
15
+ # is unauditable and shipped dead code once (suma#166 - an
16
+ # interpolating heredoc silently ate the regex anchors in the alias
17
+ # matchers, so the artefact aliases never landed and reinflate failed
18
+ # with "no stored artefact").
19
+ module Worker
20
+ # One batch stage's spawn parameters, threaded through the batch
21
+ # render methods.
22
+ BatchContext =
23
+ Struct.new(:config_dir, :batch_count, :store_dir, :out_dir,
24
+ :no_valid)
25
+
26
+ module_function
27
+
28
+ # Entry. ARGV = ["stage", manifest_path, store_dir, id_slug] with
29
+ # render options on STDIN (sequential stage: render one member and
30
+ # alias its artefacts); ARGV = ["batch", config_dir, batch_count,
31
+ # store_dir, out_dir, no_valid] (batch stage: claim and render
32
+ # batch files until none remain); ARGV = ["render", manifest_path]
33
+ # with options on STDIN (plain render, no aliasing - reinflate).
34
+ def run(argv = ARGV)
35
+ case mode = argv.shift
36
+ when "stage" then stage(*argv)
37
+ when "batch" then batch(*argv)
38
+ when "render" then render_only(*argv)
39
+ else abort("[suma staged worker] unknown mode #{mode.inspect}")
40
+ end
41
+ end
42
+
43
+ # Sequential stage: one fresh process per member, so peak memory
44
+ # stays at one member (metanorma/metanorma#576).
45
+ def stage(manifest_path, store_dir, id_slug)
46
+ collection = render_with_options(manifest_path)
47
+ alias_artifacts(collection, id_slug, store_dir)
48
+ end
49
+
50
+ # Plain render (reinflate): no store aliasing - the reinflate pass
51
+ # consumes the store, it does not extend it.
52
+ def render_only(manifest_path)
53
+ render_with_options(manifest_path)
54
+ end
55
+
56
+ # The render options arrive from the parent over STDIN as JSON (not
57
+ # Marshal: deserializing a stream is rb/unsafe-deserialization, and
58
+ # the payload is plain strings, booleans, arrays, and nested hashes
59
+ # - JSON expresses it exactly, with symbols restored below).
60
+ def render_with_options(manifest_path)
61
+ Metanorma::Collection.parse(manifest_path)
62
+ .render(load_options($stdin.read))
63
+ end
64
+
65
+ # Parent side: symbols and hash keys become strings for the wire.
66
+ def dump_options(opts)
67
+ JSON.generate(dump_value(opts))
68
+ end
69
+
70
+ # Child side: restore symbol top-level keys, symbol :format values,
71
+ # and the nested :compile hash.
72
+ def load_options(json)
73
+ opts = JSON.parse(json).to_h { |k, v| [k.to_sym, v] }
74
+ opts[:format] = opts[:format].map(&:to_sym) if opts[:format]
75
+ if opts[:compile]
76
+ opts[:compile] = opts[:compile].to_h do |k, v|
77
+ [k.to_sym, v]
78
+ end
79
+ end
80
+ opts
81
+ end
82
+
83
+ def dump_value(value)
84
+ case value
85
+ when Hash then value.to_h { |k, v| [k.to_s, dump_value(v)] }
86
+ when Array then value.map { |v| dump_value(v) }
87
+ when Symbol then value.to_s
88
+ else value
89
+ end
90
+ end
91
+
92
+ # Batch stage: claim one batch file at a time through O_EXCL
93
+ # markers, so N concurrent children partition the batches without a
94
+ # coordinator; a child with no batches left exits and hands its
95
+ # memory back to the OS. Exits 1 when it could claim nothing while
96
+ # batches were announced (nothing was rendered).
97
+ def batch(config_dir, batch_count, store_dir, out_dir, no_valid)
98
+ ctx = BatchContext.new(config_dir, Integer(batch_count), store_dir,
99
+ out_dir, no_valid)
100
+ claimed = 0
101
+ while (idx = claim_batch(ctx.config_dir, ctx.batch_count))
102
+ render_batch(ctx, idx)
103
+ claimed += 1
104
+ end
105
+ exit 1 if claimed.zero? && ctx.batch_count.positive?
106
+ end
107
+
108
+ def render_batch(ctx, idx)
109
+ # The batch file is written by this builder's own write_batch_bins
110
+ # in the same process tree - trusted, same-toolchain data, JSON-
111
+ # encoded member manifests.
112
+ manifests = JSON.parse(File.read(batch_file(ctx.config_dir, idx)))
113
+ manifests.each_with_index do |manifest, pos|
114
+ render_batch_member(ctx, idx, pos, manifest)
115
+ end
116
+ end
117
+
118
+ def render_batch_member(ctx, idx, pos, manifest)
119
+ one_path = File.join(ctx.config_dir, "._suma_staged_b#{idx}_#{pos}.yml")
120
+ out = File.join(ctx.out_dir, "._staged_out_b#{idx}_#{pos}")
121
+ started = Time.now
122
+ render_and_alias(render_member(one_path, ctx, out), manifest, ctx)
123
+ log_batch_ok(ctx, idx, pos, started)
124
+ ensure
125
+ FileUtils.rm_f(one_path)
126
+ FileUtils.rm_rf(out)
127
+ GC.start
128
+ end
129
+
130
+ def render_and_alias(collection, manifest, ctx)
131
+ alias_artifacts(collection, slug(member_identifier(manifest)),
132
+ ctx.store_dir)
133
+ end
134
+
135
+ def render_member(manifest_path, ctx, out)
136
+ compile = { install_fonts: false }
137
+ compile[:novalid] = true if ctx.no_valid == "1"
138
+ collection = Metanorma::Collection.parse(manifest_path)
139
+ collection.render(
140
+ format: [:xml], output_folder: out,
141
+ preserve_unresolved: true, artifact_store_dir: ctx.store_dir,
142
+ compile: compile
143
+ )
144
+ collection
145
+ end
146
+
147
+ def log_batch_ok(ctx, idx, pos, started)
148
+ warn format(
149
+ "[staged worker %<pid>d] batch %<idx>d/%<count>d member " \
150
+ "%<pos>d OK (%<seconds>d s)",
151
+ pid: Process.pid, idx: idx + 1, count: ctx.batch_count, pos: pos,
152
+ seconds: Time.now - started
153
+ )
154
+ end
155
+
156
+ # The artefact aliasing both stage paths perform after a member's
157
+ # store write (metanorma/suma#204): the store keys on the rendered
158
+ # document's own docid slug, while the parent reinflates and the
159
+ # warm-skip look up by the docref identifier. Without these copies a
160
+ # render banks entries the lookups can never find - the sequential
161
+ # stage never warm-skipped for exactly this reason.
162
+ def alias_artifacts(collection, id_slug, store_dir)
163
+ alias_from_bibdatas(collection, id_slug, store_dir)
164
+ alias_from_identifier_aliases(identifier_aliases(collection),
165
+ store_dir)
166
+ end
167
+
168
+ # Alias from the collection's own bibdata keys: they are the docids
169
+ # this render stored under. A produced-files delta is not safe here
170
+ # - with concurrent workers another child's render lands in the
171
+ # shared store mid-flight, and a delta would alias other members'
172
+ # documents under this docref (suma#167). bibdatas is empty for
173
+ # corpus manifests (suma#181) - the identifier_aliases below drive
174
+ # the aliasing in that case.
175
+ def alias_from_bibdatas(collection, id_slug, store_dir)
176
+ return if id_slug.empty?
177
+
178
+ collection.bibdatas&.each_key do |key|
179
+ src = slug(key)
180
+ next if src.empty? || src == id_slug
181
+
182
+ copy_store_entries(src, id_slug, store_dir)
183
+ end
184
+ end
185
+
186
+ # identifier_aliases maps raw identifier -> resolved docid; alias
187
+ # the resolved-docid entries under the raw identifier's slug.
188
+ def alias_from_identifier_aliases(aliases, store_dir)
189
+ aliases.each do |original, resolved|
190
+ src = slug(resolved)
191
+ dst = slug(original)
192
+ next if src.empty? || dst.empty? || src == dst
193
+
194
+ copy_store_entries(src, dst, store_dir)
195
+ end
196
+ end
197
+
198
+ def copy_store_entries(src, dst, store_dir)
199
+ Dir[File.join(store_dir, "#{src}.*.semantic.xml")].each do |f|
200
+ FileUtils.cp(f, f.sub(%r{/#{Regexp.escape(src)}\.}, "/#{dst}."))
201
+ end
202
+ end
203
+
204
+ def identifier_aliases(collection)
205
+ collection.manifest.identifier_aliases || {}
206
+ rescue NoMethodError
207
+ {}
208
+ end
209
+
210
+ def member_identifier(manifest)
211
+ manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
212
+ end
213
+
214
+ def claim_batch(config_dir, count)
215
+ (0...count).find do |i|
216
+ try_claim(File.join(config_dir, "._suma_batch_#{i}.claim"))
217
+ end
218
+ end
219
+
220
+ # O_EXCL create: succeeds for exactly one child, fails with
221
+ # EEXIST for everyone else - the claim.
222
+ def try_claim(path)
223
+ File.open(path, File::RDWR | File::CREAT | File::EXCL) do |claim|
224
+ claim.write("1")
225
+ end
226
+ true
227
+ rescue Errno::EEXIST
228
+ false
229
+ end
230
+
231
+ def batch_file(config_dir, idx)
232
+ File.join(config_dir, "._suma_batch_#{idx}.bin")
233
+ end
234
+
235
+ # Mirrors the store's filename slug (metanorma ArtifactStore#slug).
236
+ # The \A/\z anchors are load-bearing: they trim leading/trailing
237
+ # dashes only, and losing them in generated text was the suma#166
238
+ # defect.
239
+ def slug(value)
240
+ value.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
241
+ .gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
242
+ end
243
+ end
244
+ end
245
+ end
@@ -37,10 +37,65 @@ module Suma
37
37
  class StagedCollectionBuilder
38
38
  STORE_DIRNAME = ".metanorma-collection-cache"
39
39
 
40
+ autoload :Worker, "suma/staged_collection_builder/worker"
41
+
42
+ # The child-process boot line: one require, no embedded logic. All
43
+ # worker code lives in Worker (lib/suma/staged_collection_builder/
44
+ # worker.rb) — generated script strings shipped dead code once
45
+ # (suma#166) and could only be spec-pinned as text.
46
+ CHILD_BOOT = 'require "suma"; ' \
47
+ "Suma::StagedCollectionBuilder::Worker.run"
48
+
40
49
  # Members rendered per child process: one Metanorma boot amortised over
41
50
  # a batch of small members keeps boot cost under a few percent.
42
51
  DEFAULT_BATCH_SIZE = 25
43
52
 
53
+ # The warm-skip identity of one staged-store entry (metanorma/suma#203):
54
+ # the digest of the rendering toolchain that produced it, banked in a
55
+ # sidecar beside the stored semantic XML (raw hex, the form the
56
+ # SchemaCompiler render key uses). Presence-keyed reuse is unsound when
57
+ # a workflow cache carries a store across a toolchain change; an entry
58
+ # whose sidecar is missing (legacy store) or records a different
59
+ # toolchain is re-rendered, and every freshly rendered entry is
60
+ # re-banked — so a restored store converges onto the current toolchain
61
+ # at the cost of at most one extra render per entry.
62
+ class StoreEntry
63
+ SIDECHAR = ".toolchain"
64
+
65
+ class << self
66
+ # Bank +digest+ as the producing toolchain of the entry stored at
67
+ # +stored_path+.
68
+ def bank(stored_path, digest)
69
+ File.write(sidecar_path(stored_path), digest)
70
+ end
71
+
72
+ # The toolchain digest banked for the entry, or nil for a legacy
73
+ # entry written before banking existed.
74
+ def toolchain_digest(stored_path)
75
+ path = sidecar_path(stored_path)
76
+ File.file?(path) ? File.read(path) : nil
77
+ end
78
+
79
+ # Whether the entry was produced by the toolchain +digest+
80
+ # identifies. A legacy entry (no sidecar) never matches, so the
81
+ # first build over an old store re-renders once and re-banks.
82
+ def current?(stored_path, digest)
83
+ toolchain_digest(stored_path) == digest
84
+ end
85
+
86
+ # The banked digest truncated for log lines.
87
+ def short_digest(stored_path)
88
+ (toolchain_digest(stored_path) || "unknown")[0, 8]
89
+ end
90
+
91
+ private
92
+
93
+ def sidecar_path(stored_path)
94
+ "#{stored_path}#{SIDECHAR}"
95
+ end
96
+ end
97
+ end
98
+
44
99
  # +collection_config_path+ is the emitted collection manifest (the same
45
100
  # +collection-output.yaml+ the normal build renders). Member filerefs are
46
101
  # relative to its directory, so all per-member manifests are written there.
@@ -84,6 +139,42 @@ module Suma
84
139
  reinflate(reinflate_selection(members), members)
85
140
  end
86
141
 
142
+ # The warm-skip query (metanorma/suma#203): members whose stored entry
143
+ # was banked under the CURRENT rendering toolchain are done; every
144
+ # other member — no stored entry, a legacy entry without a banked
145
+ # digest, or an entry another toolchain produced — renders again.
146
+ # A workflow cache can carry a store across a toolchain or content-key
147
+ # change (TRThurman, iso-10303#858), and presence alone cannot see it:
148
+ # the banked digest is the durable guarantee the restored store cannot
149
+ # fake. Each member gets one provenance line: what was taken from the
150
+ # store and what compiled it, or that it renders.
151
+ def unstaged_members(members)
152
+ members.reject do |member|
153
+ identifier = member["identifier"]
154
+ stored = identifier && stored_semantic(identifier)
155
+ if stored && StoreEntry.current?(stored, Provenance.render_toolchain_digest)
156
+ Utils.log "[staged] #{identifier}: restored from store " \
157
+ "(toolchain #{StoreEntry.short_digest(stored)})"
158
+ true
159
+ else
160
+ Utils.log "[staged] #{identifier}: rendering"
161
+ false
162
+ end
163
+ end
164
+ end
165
+
166
+ # Record the rendering toolchain beside every freshly rendered
167
+ # member's store entry, so the next warm build can trust-or-recheck
168
+ # it. A legacy entry without a digest is re-rendered once and re-banked
169
+ # with one (safe migration); a warm-skipped member was already
170
+ # validated against the current digest and keeps its sidecar.
171
+ def bank_toolchains(members)
172
+ members.each do |member|
173
+ stored = stored_semantic(member["identifier"]) or next
174
+ StoreEntry.bank(stored, Provenance.render_toolchain_digest)
175
+ end
176
+ end
177
+
87
178
  private
88
179
 
89
180
  # The staged slice of the collection: an isolate selection wins over a
@@ -92,13 +183,16 @@ module Suma
92
183
  @isolate ? isolate_members(members) : shard_members(members)
93
184
  end
94
185
 
95
- # Render the staged members, serially or across the worker pool.
186
+ # Render the staged members, serially or across the worker pool, then
187
+ # bank every freshly rendered member's toolchain identity.
96
188
  def stage_members(staged)
189
+ unstaged = unstaged_members(staged)
97
190
  if @workers == 1
98
- unstaged_members(staged).each_with_index { |member, i| stage_member(member, i) }
191
+ unstaged.each_with_index { |member, i| stage_member(member, i) }
99
192
  else
100
- stage_parallel(staged)
193
+ stage_parallel(unstaged)
101
194
  end
195
+ bank_toolchains(unstaged)
102
196
  end
103
197
 
104
198
  # The reinflated slice: all members unless a reinflate shard is given,
@@ -156,19 +250,6 @@ module Suma
156
250
  "skipping reinflate"
157
251
  end
158
252
 
159
- # Members already present in the store (their docref-slug artefact
160
- # exists) are not re-rendered: an interrupted or repeated build over
161
- # a warm store goes straight to reinflate, so the assembly pass costs
162
- # minutes instead of re-rendering the whole collection.
163
- def unstaged_members(members)
164
- members.reject do |member|
165
- identifier = member["identifier"]
166
- stored = identifier && stored_semantic(identifier)
167
- Utils.log "[staged] stored, skipping #{identifier}" if stored
168
- stored
169
- end
170
- end
171
-
172
253
  # Batch the members and run +@workers+ renderer child processes
173
254
  # concurrently. Each batch file carries fully-rendered single-member
174
255
  # manifests; children claim batches through O_EXCL markers, so a slow
@@ -177,8 +258,9 @@ module Suma
177
258
  # child exit hands its memory back to the OS. Members write disjoint,
178
259
  # identifier-keyed store entries, so concurrent children cannot race;
179
260
  # sectionsplit only happens at reinflate (a single sequential pass).
261
+ # +members+ is the warm-skip's unstaged remainder.
180
262
  def stage_parallel(members)
181
- paths, count = write_batch_files(unstaged_members(members))
263
+ paths, count = write_batch_files(members)
182
264
  run_batch_workers(count)
183
265
  ensure
184
266
  remove_batch_files(paths, count)
@@ -202,14 +284,14 @@ module Suma
202
284
  def write_batch_bins(batches)
203
285
  batches.map.with_index do |batch, i|
204
286
  path = File.join(@config_dir, "._suma_batch_#{i}.bin")
205
- File.binwrite(path, Marshal.dump(batch))
287
+ File.write(path, Worker.dump_options(batch))
206
288
  path
207
289
  end
208
290
  end
209
291
 
210
292
  def run_batch_workers(count)
211
293
  children = Array.new([@workers, count].min) do
212
- Process.spawn(RbConfig.ruby, "-e", child_script, @config_dir,
294
+ Process.spawn(RbConfig.ruby, "-e", CHILD_BOOT, "batch", @config_dir,
213
295
  count.to_s, @store_dir, @output_directory,
214
296
  @no_valid ? "1" : "0", chdir: @config_dir)
215
297
  end
@@ -232,122 +314,6 @@ module Suma
232
314
  # Child renderer: claims and renders batches until none remain. The
233
315
  # Metanorma boot is paid once per child, and GC runs between members to
234
316
  # bound the heap. Any render failure exits nonzero and fails the build.
235
- def child_script
236
- <<~'RUBY'
237
- require "yaml"
238
- require "fileutils"
239
- require "metanorma"
240
- config_dir, batch_count, store_dir, out_dir, no_valid = ARGV
241
- count = Integer(batch_count)
242
- claimed = 0
243
- loop do
244
- idx = (0...count).find do |i|
245
- begin
246
- File.open(File.join(config_dir, "._suma_batch_#{i}.claim"),
247
- File::RDWR | File::CREAT | File::EXCL) {}
248
- true
249
- rescue Errno::EEXIST
250
- false
251
- end
252
- end
253
- break unless idx
254
-
255
- manifests = Marshal.load(File.binread(File.join(
256
- config_dir, "._suma_batch_#{idx}.bin")))
257
- manifests.each_with_index do |manifest, j|
258
- one_path = File.join(config_dir,
259
- "._suma_staged_b#{idx}_#{j}.yml")
260
- out = File.join(out_dir, "._staged_out_b#{idx}_#{j}")
261
- t0 = Time.now
262
- begin
263
- File.write(one_path, YAML.dump(manifest))
264
- # The artefact store keys on the rendered document's own
265
- # docid, while the parent reinflates by the docref
266
- # identifier; members whose document docid differs from the
267
- # docref (docrefs carrying metanorma's GUID marker) were
268
- # therefore unrecoverable ("no stored artefact"). Alias the
269
- # artefacts keyed by the render's own bibdata keys under the
270
- # docref slug too. Shape: single_member_manifest =>
271
- # {"manifest" => {"manifest" => [{"docref" => [dr]}]}}.
272
- member_id = manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
273
- id_slug = member_id.to_s
274
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
275
- .gsub(/\A-|-\z/, "")
276
- compile = { install_fonts: false }
277
- compile[:novalid] = true if no_valid == "1"
278
- collection = Metanorma::Collection.parse(one_path)
279
- collection.render(
280
- format: [:xml], output_folder: out,
281
- preserve_unresolved: true, artifact_store_dir: store_dir,
282
- compile: compile,
283
- )
284
- unless id_slug.empty?
285
- # Alias strictly from the collection's own bibdata keys:
286
- # they are the docids this member's render stored under.
287
- # A produced-files delta is not safe here - with
288
- # concurrent workers another child's render lands in the
289
- # shared store mid-flight, and a delta would alias other
290
- # members' documents under this docref (suma#167).
291
- (collection.bibdatas || {}).keys.each do |key|
292
- src_slug = key.to_s
293
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
294
- .gsub(/\A-|-\z/, "")
295
- next if src_slug.empty? || src_slug == id_slug
296
-
297
- Dir[File.join(store_dir,
298
- "#{src_slug}.*.semantic.xml")].each do |f|
299
- alias_name = f.sub(
300
- %r{/#{Regexp.escape(src_slug)}.},
301
- "/#{id_slug}.",
302
- )
303
- FileUtils.cp(f, alias_name)
304
- end
305
- end
306
- end
307
-
308
- # The corpus manifests carry no bibdata, so bibdatas is
309
- # empty and the loop above never runs (suma#181): the
310
- # artefacts stay keyed under the document's own docid and
311
- # reinflate - looking up by the docref identifier - finds
312
- # nothing on a cold store ("no stored artefact"). metanorma
313
- # resolves every manifest identifier to the document docid
314
- # during populate and records the mapping on the manifest
315
- # itself (identifier_aliases, raw -> resolved). That map is
316
- # the child's own post-render collection - reading it is
317
- # race-free per the suma#167 discipline.
318
- aliases = begin
319
- collection.manifest.identifier_aliases || {}
320
- rescue NoMethodError
321
- {}
322
- end
323
- aliases.each do |original, resolved|
324
- src = resolved.to_s
325
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
326
- .gsub(/\A-|-\z/, "")
327
- dst = original.to_s
328
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
329
- .gsub(/\A-|-\z/, "")
330
- next if src.empty? || dst.empty? || src == dst
331
-
332
- Dir[File.join(store_dir, "#{src}.*.semantic.xml")].each do |f|
333
- FileUtils.cp(
334
- f, f.sub(%r{/#{Regexp.escape(src)}\.}, "/#{dst}."),
335
- )
336
- end
337
- end
338
- warn format("[staged worker %d] batch %d/%d member %d OK (%ds)",
339
- Process.pid, idx + 1, count, j, Time.now - t0)
340
- ensure
341
- File.unlink(one_path) rescue nil
342
- FileUtils.rm_rf(out) rescue nil
343
- GC.start
344
- end
345
- end
346
- claimed += 1
347
- end
348
- exit 1 if claimed.zero? && count.positive?
349
- RUBY
350
- end
351
317
 
352
318
  def manifest
353
319
  @manifest ||= YAML.load_file(@config_path)
@@ -388,6 +354,7 @@ module Suma
388
354
  opts: { format: [:xml], output_folder: out,
389
355
  preserve_unresolved: true, artifact_store_dir: @store_dir,
390
356
  compile: compile_flags },
357
+ mode: "stage", id_slug: slug(member["identifier"])
391
358
  )
392
359
  ensure
393
360
  FileUtils.rm_f(one_path)
@@ -625,20 +592,20 @@ module Suma
625
592
 
626
593
  # Mirrors the store's filename slug (metanorma ArtifactStore#slug).
627
594
  def slug(identifier)
628
- identifier.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
629
- .gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
595
+ Worker.slug(identifier)
630
596
  end
631
597
 
632
598
  # Render a manifest in a FRESH child process so its memory is reclaimed on
633
- # exit. Under +bundle exec+, the child inherits the bundle. Raises on failure.
634
- def run_render(manifest_path:, opts:)
635
- script = <<~RUBY
636
- require "metanorma"
637
- opts = Marshal.load(STDIN.read)
638
- Metanorma::Collection.parse(#{manifest_path.inspect}).render(opts)
639
- RUBY
640
- IO.popen([RbConfig.ruby, "-e", script], "wb") do |io|
641
- io.write(Marshal.dump(opts))
599
+ # exit. Under +bundle exec+, the child inherits the bundle. Raises on
600
+ # failure. +mode+ "render" plain-renders (reinflate); "stage" additionally
601
+ # aliases the member's store artefacts under +id_slug+, the same aliasing
602
+ # the batch child path performs (suma#204 - without it the sequential
603
+ # stage banked docid-slug-only entries and never warm-skipped).
604
+ def run_render(manifest_path:, opts:, mode: "render", id_slug: nil)
605
+ argv = [mode, manifest_path]
606
+ argv += [@store_dir, id_slug] if mode == "stage"
607
+ IO.popen([RbConfig.ruby, "-e", CHILD_BOOT, *argv], "wb") do |io|
608
+ io.write(Worker.dump_options(opts))
642
609
  io.close_write
643
610
  end
644
611
  return if $?.success?
@@ -13,13 +13,24 @@ module Suma
13
13
  # +#call(index, total, report)+. +NullProgress+ is the default
14
14
  # no-op; the CLI passes a lambda that writes to +$stderr+.
15
15
  class Scanner
16
- DEFAULT_PROFILE = :metanorma
16
+ # The operative metanorma profile ships from metanorma-standoc by
17
+ # path: svg_conform's bundled copy is a sample only, and the
18
+ # "metanorma" engine name fails loudly by design (svg_conform#132).
19
+ STANDOC_PROFILE_PATH =
20
+ "lib/metanorma/standoc/svg_conform_profile/metanorma.yml"
21
+
22
+ def self.default_profile
23
+ File.join(
24
+ Gem::Specification.find_by_name("metanorma-standoc").gem_dir,
25
+ STANDOC_PROFILE_PATH,
26
+ )
27
+ end
17
28
 
18
29
  attr_reader :profile, :progress
19
30
 
20
- def initialize(profile: DEFAULT_PROFILE,
31
+ def initialize(profile: nil,
21
32
  progress: NullProgress.new)
22
- @profile = profile
33
+ @profile = profile || self.class.default_profile
23
34
  @progress = progress
24
35
  end
25
36
 
@@ -288,7 +288,7 @@ module Suma
288
288
  "#{entity_name_to_text(entity.id)} #{entity_ref}"
289
289
  else
290
290
  entity_subtypes = subtypes.map do |e|
291
- owner = declaring_schema_id(schema, e.id) || schema.id
291
+ owner = declaring_owner(schema, e.id)
292
292
  urn_mention(express_entity_urn("#{owner}.#{e.id}"), e.id)
293
293
  end
294
294
 
@@ -299,6 +299,13 @@ module Suma
299
299
  end
300
300
  end
301
301
 
302
+ # The owner a supertype citation qualifies with: the closure declarer,
303
+ # else the sibling-schema declarer, else the citing schema.
304
+ def declaring_owner(schema, entity_id)
305
+ declaring_schema_id(schema, entity_id) ||
306
+ sibling_declaring_schema_id(entity_id) || schema.id
307
+ end
308
+
302
309
  # The schema an imported declaration is declared in, resolved through
303
310
  # +schema+'s USE-FROM closure — the same resolution the source remark
304
311
  # xrefs carry: a local declaration wins; otherwise the USE FROM edges
@@ -321,6 +328,39 @@ module Suma
321
328
  end
322
329
  end
323
330
 
331
+ # When the closure walk finds no declarer, the declaration may sit
332
+ # in a sibling schema with no interface edge to the citing one: the
333
+ # _arm/_mim pairs cite across without USE FROM/REFERENCE FROM, so
334
+ # `X_arm.Y` lands on a concept declared under `X_mim` (TRThurman's
335
+ # 148 distinct wrong-qualifier targets, iso-10303#835 /
336
+ # metanorma/suma#197). Scan the manifest's schemas for the entity
337
+ # name; exactly one declarer resolves, several stay nil.
338
+ def sibling_declaring_schema_id(entity_id)
339
+ @sibling_cache ||= {}
340
+ @sibling_cache.fetch(entity_id) do
341
+ declarers = declarers_in_manifest(entity_id)
342
+ unique_declarer(declarers)
343
+ end
344
+ end
345
+
346
+ # The manifest schemas whose entity collection declares +entity_id+.
347
+ def declarers_in_manifest(entity_id)
348
+ schema_manifest_paths.keys.filter_map do |id|
349
+ loaded = loaded_schema(id)
350
+ declares_entity?(loaded, entity_id) ? id : nil
351
+ end
352
+ end
353
+
354
+ def declares_entity?(schema, entity_id)
355
+ schema&.entities&.any? { |e| e.id == entity_id } ? true : false
356
+ end
357
+
358
+ # The single distinct declarer, or nil when none or several — an
359
+ # ambiguous name never resolves to a guess.
360
+ def unique_declarer(declarers)
361
+ declarers.uniq.one? ? declarers.first : nil
362
+ end
363
+
324
364
  # Distinct schema ids within +schema+'s USE-FROM closure declaring
325
365
  # +entity_id+. An interface whose item list names the declaration
326
366
  # claims it outright (that is what the item list means in EXPRESS) —
data/lib/suma/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Suma
4
- VERSION = "0.5.38"
4
+ VERSION = "0.5.40"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: suma
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.38
4
+ version: 0.5.40
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2026-11-13 00:00:00.000000000 Z
11
+ date: 2026-11-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: base64
@@ -243,6 +243,7 @@ files:
243
243
  - lib/suma/corpus_graph.rb
244
244
  - lib/suma/delivery.rb
245
245
  - lib/suma/delivery/cross_page_fragments.rb
246
+ - lib/suma/delivery/diagnostics.rb
246
247
  - lib/suma/delivery/expectations.rb
247
248
  - lib/suma/delivery/frame_nav.rb
248
249
  - lib/suma/delivery/pack.rb
@@ -292,6 +293,7 @@ files:
292
293
  - lib/suma/site_build.rb
293
294
  - lib/suma/site_config.rb
294
295
  - lib/suma/staged_collection_builder.rb
296
+ - lib/suma/staged_collection_builder/worker.rb
295
297
  - lib/suma/svg_quality.rb
296
298
  - lib/suma/svg_quality/batch_report.rb
297
299
  - lib/suma/svg_quality/formatters.rb