suma 0.5.39 → 0.5.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 22f28100458fea39621ee5b70b16745f37efe2eb6cac086570da5ced507a6e16
4
- data.tar.gz: 47f7b5512cc818e306db23835c21150bccc8d92a8ee50701326eaf847a8a952d
3
+ metadata.gz: 431b3f51ec584ae707195c6f1a2f98e302bcb0cf3bef73235ab12472724ac1b3
4
+ data.tar.gz: 0e63b252d9eb5d92558919c223390ef9449acdc67394477dd743dcfe333aef2f
5
5
  SHA512:
6
- metadata.gz: '0484557c5ce603b7826f4134897122b61bc0f0940947637767376e312abb31144965ca3e3187b6124b2750e3c7b0860a007fad3ae832339c59fc67116f2b3168'
7
- data.tar.gz: 94d0c922d2bd70ba7d6ce67f78ec135b4bb276d5a0119f57a28acdd69e877fdf300dafa4d753075efea0a80840a29300b0fb9f362e1a058d9d68e5f293cbd6a6
6
+ metadata.gz: 5e1cc7f2fd9c50b683606a0a4939408900cbd740283fc3d2bfd5bd7e70fd78cc37df799f5ced4376c06b46117081ee5de80dbe3f54d4f1f6c7d0f0b506ddaf6e
7
+ data.tar.gz: 7239c2d87a4376e0c47e110e17cbdbe6280ae658d8a58940f135456d2a7b7e9b7ae32bfa59a41af5b796366ad0616559714e0aa7625f816e23808d6090358a1f
@@ -26,8 +26,10 @@ module Suma
26
26
  desc: "YAML manifest of site-shape thresholds " \
27
27
  "(sections_min, index_pages, " \
28
28
  "schema_attachments_min, schema_docs_min, " \
29
- "missing_express_max, broken_links_max); " \
30
- "violations fail the verify"
29
+ "missing_express_max, broken_links_max, " \
30
+ "diagnostics_max: category => rule => " \
31
+ "max rows over the site's .err.html " \
32
+ "files); violations fail the verify"
31
33
 
32
34
  def verify(site_dir)
33
35
  result = Suma::Delivery::SiteVerifier.new(site_dir,
@@ -175,7 +177,7 @@ module Suma
175
177
  "broken; fragments: #{r['total_fragments']} total, " \
176
178
  "#{r['broken_fragments'].size} broken; " \
177
179
  "#{r['cross_document_links']} cross-document; " \
178
- "#{inventory_summary(r)}"
180
+ "#{inventory_summary(r)}; #{diagnostics_summary(r)}"
179
181
  end
180
182
 
181
183
  def inventory_summary(report)
@@ -187,6 +189,14 @@ module Suma
187
189
  "missing-express pages=#{report['missing_express_pages'].size}"
188
190
  end
189
191
 
192
+ # The err.html census the verify walked: total diagnostic rows over
193
+ # the categories the site's error reports carry.
194
+ def diagnostics_summary(report)
195
+ census = report["diagnostics"] || {}
196
+ "diagnostics: #{census.sum { |_, rules| rules.values.sum }} rows " \
197
+ "over #{census.size} categories"
198
+ end
199
+
190
200
  def write_report(report)
191
201
  File.write(options[:out], JSON.pretty_generate(report))
192
202
  end
@@ -0,0 +1,136 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "leptris"
4
+
5
+ module Suma
6
+ module Delivery
7
+ # Diagnostic-row census over metanorma .err.html error reports
8
+ # (metanorma/suma#203): rows grouped by CATEGORY and RULE.
9
+ #
10
+ # A gate keyed on a literal code string breaks when the codes migrate
11
+ # (iso-10303#859: the part2 gate greps STANDOC_38 in document.err.html
12
+ # while metanorma-iso 3.5.14 re-emitted those rows as ISO_56, so the
13
+ # gate reads 0 over 476 live rows). Codes migrate; the stable identity
14
+ # of a diagnostic row is category + rule, so the census keys on both.
15
+ #
16
+ # The err.html writer is Metanorma::Utils::Log#write: one
17
+ # `<h2 id="...">Category</h2>` per category, each followed by a table
18
+ # whose data rows are six cells - Line, ID, Error (the rule code,
19
+ # e.g. STANDOC_27 or ISO_56), Message, Context, Severity.
20
+ class Diagnostics
21
+ # The Error cell's position within a data row.
22
+ RULE_CELL_INDEX = 2
23
+
24
+ class << self
25
+ # Parse one err.html given as a file path or an HTML string.
26
+ #
27
+ # @return [Counts] rows grouped by category, then rule
28
+ def count(path_or_html)
29
+ html = if File.file?(path_or_html)
30
+ File.read(path_or_html)
31
+ else
32
+ path_or_html
33
+ end
34
+ new(html).counts
35
+ end
36
+
37
+ # The census of every err.html under a rendered site, summed:
38
+ # the site-level figure an expectations gate asserts against.
39
+ def count_site(site_dir)
40
+ Dir.glob(File.join(site_dir, "**", "*.err.html"))
41
+ .reduce(Counts.new) { |sum, path| sum.merge(count(path)) }
42
+ end
43
+ end
44
+
45
+ def initialize(html)
46
+ @doc = Leptris::XML.parse_html(html)
47
+ end
48
+
49
+ # Every data row of every category table, keyed by its h2 heading
50
+ # (the log category) and Error cell (the rule code).
51
+ def counts
52
+ Counts.new(tally)
53
+ end
54
+
55
+ private
56
+
57
+ # Per category, per rule, the number of data rows.
58
+ def tally
59
+ rows = Hash.new { |hash, category| hash[category] = Hash.new(0) }
60
+ @doc.css("h2").each do |heading|
61
+ rules_of(heading).each do |rule|
62
+ rows[heading.text.strip][rule] += 1
63
+ end
64
+ end
65
+ rows
66
+ end
67
+
68
+ # The rule code of every data row in the heading's table.
69
+ def rules_of(heading)
70
+ table = heading.xpath("following-sibling::table[1]").first
71
+ return [] unless table
72
+
73
+ table.css("tr").filter_map { |row| rule_of(row) }
74
+ end
75
+
76
+ # The Error cell of a row: the third of Line/ID/Error/Message/
77
+ # Context/Severity (the ID cell renders as a `th`).
78
+ def rule_of(row)
79
+ rule = row.xpath("td|th")[RULE_CELL_INDEX]&.text&.strip
80
+ rule.nil? || rule.empty? ? nil : rule
81
+ end
82
+
83
+ # The per-category, per-rule census of one or more err.html
84
+ # documents: immutable value object over
85
+ # { "Style" => { "STANDOC_48" => 15 } }, enumerable over
86
+ # (category, rule, count) triples.
87
+ class Counts
88
+ include Enumerable
89
+
90
+ def initialize(categories = {})
91
+ @categories = categories
92
+ .transform_values { |rules| rules.transform_values(&:to_i).freeze }
93
+ .freeze
94
+ end
95
+
96
+ # The raw census hash: category => (rule => row count). Frozen.
97
+ def to_h
98
+ @categories
99
+ end
100
+
101
+ def each
102
+ return enum_for(__method__) unless block_given?
103
+
104
+ @categories.each do |category, rules|
105
+ rules.each { |rule, count| yield category, rule, count }
106
+ end
107
+ self
108
+ end
109
+
110
+ # Rows counted for one rule; an unreported rule counts zero.
111
+ def rule_count(category, rule)
112
+ @categories.dig(category, rule) || 0
113
+ end
114
+
115
+ # Every row counted for one category.
116
+ def total(category)
117
+ @categories.fetch(category, {}).values.sum
118
+ end
119
+
120
+ # Rows counted overall.
121
+ def rows
122
+ @categories.values.sum { |rules| rules.values.sum }
123
+ end
124
+
125
+ # The union census: counts of both sides summed.
126
+ def merge(other)
127
+ other = self.class.new({}) unless other.is_a?(self.class)
128
+ summed = other.to_h.merge(to_h) do |_category, theirs, ours|
129
+ theirs.merge(ours) { |_rule, a, b| a + b }
130
+ end
131
+ self.class.new(summed)
132
+ end
133
+ end
134
+ end
135
+ end
136
+ end
@@ -13,6 +13,10 @@ module Suma
13
13
  attribute :schema_docs_min, :integer
14
14
  attribute :missing_express_max, :integer, default: -> { 0 }
15
15
  attribute :broken_links_max, :integer, default: -> { 0 }
16
+ # Category => rule => maximum live diagnostic rows: the stable
17
+ # identity of an err.html row is category + rule, not the code,
18
+ # which migrates across metanorma releases (iso-10303#859).
19
+ attribute :diagnostics_max, :hash
16
20
 
17
21
  key_value do
18
22
  map "sections_min", to: :sections_min
@@ -21,13 +25,22 @@ module Suma
21
25
  map "schema_docs_min", to: :schema_docs_min
22
26
  map "missing_express_max", to: :missing_express_max
23
27
  map "broken_links_max", to: :broken_links_max
28
+ map "diagnostics_max", to: :diagnostics_max
24
29
  end
25
30
 
26
31
  # Human-readable violation lines for every threshold the +report+
27
32
  # and +broken_links+ breach; unset thresholds stay silent. The name
28
33
  # suffix is the comparison: `_min` floors, `_max` caps, a bare name
29
- # (index_pages) is exact.
34
+ # (index_pages) is exact. The nested diagnostics_max thresholds
35
+ # count against the report's diagnostics census.
30
36
  def violations(report, broken_links)
37
+ threshold_violations(report, broken_links) +
38
+ diagnostics_violations(report["diagnostics"])
39
+ end
40
+
41
+ private
42
+
43
+ def threshold_violations(report, broken_links)
31
44
  actuals(report, broken_links).filter_map do |name, actual|
32
45
  limit = public_send(name) or next
33
46
  relation = relation_of(name)
@@ -39,7 +52,20 @@ module Suma
39
52
  end
40
53
  end
41
54
 
42
- private
55
+ # diagnostics_max: { "Referencing" => { "ISO_56" => 0 } }. A rule
56
+ # absent from the census counts zero rows.
57
+ def diagnostics_violations(census)
58
+ Array(diagnostics_max).flat_map do |category, rules|
59
+ rules.filter_map do |rule, limit|
60
+ actual = (census || {}).dig(category, rule) || 0
61
+ next if actual <= limit
62
+
63
+ format("%<category>s/%<rule>s: got %<actual>d, expected at " \
64
+ "most %<limit>d", category: category, rule: rule,
65
+ actual: actual, limit: limit)
66
+ end
67
+ end
68
+ end
43
69
 
44
70
  def relation_of(name)
45
71
  return "at least" if name.end_with?("_min")
@@ -52,6 +52,7 @@ module Suma
52
52
  verify_links(doc, rel_path, html_file, anchor_index, report)
53
53
  end
54
54
  report["inventory"] = site_inventory
55
+ report["diagnostics"] = Diagnostics.count_site(@site_dir).to_h
55
56
  Report.new(report: report)
56
57
  end
57
58
 
@@ -71,6 +72,7 @@ module Suma
71
72
  "broken_intra_fragments" => [],
72
73
  "missing_express_pages" => [],
73
74
  "inventory" => {},
75
+ "diagnostics" => {},
74
76
  }
75
77
  end
76
78
 
data/lib/suma/delivery.rb CHANGED
@@ -6,6 +6,7 @@ module Suma
6
6
  module Delivery
7
7
  autoload :FrameNav, "suma/delivery/frame_nav"
8
8
  autoload :CrossPageFragments, "suma/delivery/cross_page_fragments"
9
+ autoload :Diagnostics, "suma/delivery/diagnostics"
9
10
  autoload :Expectations, "suma/delivery/expectations"
10
11
  autoload :Pack, "suma/delivery/pack"
11
12
  autoload :Plan, "suma/delivery/plan"
@@ -108,6 +108,15 @@ module Suma
108
108
  @render_toolchain ||= toolchain.merge("render" => render_gems)
109
109
  end
110
110
 
111
+ # The toolchain identity in the form cached artifacts record and
112
+ # compare (the staged store sidecars, the render key, the site-build
113
+ # key). A digest identifies the toolchain that produced a stored
114
+ # artifact without restating every gem version beside it.
115
+ def self.render_toolchain_digest
116
+ @render_toolchain_digest ||=
117
+ Digest::SHA256.hexdigest(JSON.generate(render_toolchain))
118
+ end
119
+
111
120
  def self.render_gems
112
121
  gem_specs.select { |spec| render_gem?(spec.name) }
113
122
  .sort_by { |spec| [spec.name, spec.version.to_s] }
@@ -0,0 +1,260 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "yaml"
5
+ require "fileutils"
6
+ require "metanorma"
7
+
8
+ module Suma
9
+ class StagedCollectionBuilder
10
+ # The code the staged builder's child processes run. Children are
11
+ # spawned as `ruby -e 'require "suma"; Worker.run'` with the mode and
12
+ # paths on ARGV and render options JSON-encoded on STDIN. All
13
+ # worker logic lives here as real, spec-testable methods: the previous
14
+ # implementation embedded this code as generated script strings, which
15
+ # is unauditable and shipped dead code once (suma#166 - an
16
+ # interpolating heredoc silently ate the regex anchors in the alias
17
+ # matchers, so the artefact aliases never landed and reinflate failed
18
+ # with "no stored artefact").
19
+ module Worker
20
+ # One batch stage's spawn parameters, threaded through the batch
21
+ # render methods.
22
+ BatchContext =
23
+ Struct.new(:config_dir, :batch_count, :store_dir, :out_dir,
24
+ :no_valid)
25
+
26
+ module_function
27
+
28
+ # Entry. ARGV = ["stage", manifest_path, store_dir, id_slug] with
29
+ # render options on STDIN (sequential stage: render one member and
30
+ # alias its artefacts); ARGV = ["batch", config_dir, batch_count,
31
+ # store_dir, out_dir, no_valid] (batch stage: claim and render
32
+ # batch files until none remain); ARGV = ["render", manifest_path]
33
+ # with options on STDIN (plain render, no aliasing - reinflate).
34
+ def run(argv = ARGV)
35
+ case mode = argv.shift
36
+ when "stage" then stage(*argv)
37
+ when "batch" then batch(*argv)
38
+ when "render" then render_only(*argv)
39
+ else abort("[suma staged worker] unknown mode #{mode.inspect}")
40
+ end
41
+ end
42
+
43
+ # Sequential stage: one fresh process per member, so peak memory
44
+ # stays at one member (metanorma/metanorma#576).
45
+ def stage(manifest_path, store_dir, id_slug)
46
+ collection = render_with_options(manifest_path)
47
+ alias_artifacts(collection, id_slug, store_dir)
48
+ end
49
+
50
+ # Plain render (reinflate): no store aliasing - the reinflate pass
51
+ # consumes the store, it does not extend it.
52
+ def render_only(manifest_path)
53
+ render_with_options(manifest_path)
54
+ end
55
+
56
+ # The render options arrive from the parent over STDIN as JSON (not
57
+ # Marshal: deserializing a stream is rb/unsafe-deserialization, and
58
+ # the payload is plain strings, booleans, arrays, and nested hashes
59
+ # - JSON expresses it exactly, with symbols restored below). Returns
60
+ # the PARSED COLLECTION, not the render's return: Collection#render
61
+ # returns the Renderer on current metanorma, and the aliasing reads
62
+ # the collection's bibdata keys and manifest identifier_aliases.
63
+ def render_with_options(manifest_path)
64
+ collection = Metanorma::Collection.parse(manifest_path)
65
+ collection.render(load_options($stdin.read))
66
+ collection
67
+ end
68
+
69
+ # Parent side: symbols and hash keys become strings for the wire.
70
+ def dump_options(opts)
71
+ JSON.generate(dump_value(opts))
72
+ end
73
+
74
+ # Child side: restore symbol top-level keys, symbol :format values,
75
+ # and the nested :compile hash.
76
+ def load_options(json)
77
+ opts = JSON.parse(json).to_h { |k, v| [k.to_sym, v] }
78
+ opts[:format] = opts[:format].map(&:to_sym) if opts[:format]
79
+ if opts[:compile]
80
+ opts[:compile] = opts[:compile].to_h do |k, v|
81
+ [k.to_sym, v]
82
+ end
83
+ end
84
+ opts
85
+ end
86
+
87
+ def dump_value(value)
88
+ case value
89
+ when Hash then value.to_h { |k, v| [k.to_s, dump_value(v)] }
90
+ when Array then value.map { |v| dump_value(v) }
91
+ when Symbol then value.to_s
92
+ else value
93
+ end
94
+ end
95
+
96
+ # Batch stage: claim one batch file at a time through O_EXCL
97
+ # markers, so N concurrent children partition the batches without a
98
+ # coordinator; a child with no batches left exits and hands its
99
+ # memory back to the OS. Exits 1 when it could claim nothing while
100
+ # batches were announced (nothing was rendered).
101
+ def batch(config_dir, batch_count, store_dir, out_dir, no_valid)
102
+ ctx = BatchContext.new(config_dir, Integer(batch_count), store_dir,
103
+ out_dir, no_valid)
104
+ claimed = 0
105
+ while (idx = claim_batch(ctx.config_dir, ctx.batch_count))
106
+ render_batch(ctx, idx)
107
+ claimed += 1
108
+ end
109
+ exit 1 if claimed.zero? && ctx.batch_count.positive?
110
+ end
111
+
112
+ def render_batch(ctx, idx)
113
+ # The batch file is written by this builder's own write_batch_bins
114
+ # in the same process tree - trusted, same-toolchain data, JSON-
115
+ # encoded member manifests.
116
+ manifests = JSON.parse(File.read(batch_file(ctx.config_dir, idx)))
117
+ manifests.each_with_index do |manifest, pos|
118
+ render_batch_member(ctx, idx, pos, manifest)
119
+ end
120
+ end
121
+
122
+ def member_paths(ctx, idx, pos)
123
+ [File.join(ctx.config_dir, "._suma_staged_b#{idx}_#{pos}.yml"),
124
+ File.join(ctx.out_dir, "._staged_out_b#{idx}_#{pos}")]
125
+ end
126
+
127
+ def cleanup_batch_files(one_path, out)
128
+ FileUtils.rm_f(one_path)
129
+ FileUtils.rm_rf(out)
130
+ GC.start
131
+ end
132
+
133
+ def render_batch_member(ctx, idx, pos, manifest)
134
+ one_path, out = member_paths(ctx, idx, pos)
135
+ started = Time.now
136
+ begin
137
+ File.write(one_path, YAML.dump(manifest))
138
+ render_and_alias(render_member(one_path, ctx, out), manifest, ctx)
139
+ log_batch_ok(ctx, idx, pos, started)
140
+ ensure
141
+ cleanup_batch_files(one_path, out)
142
+ end
143
+ end
144
+
145
+ def render_and_alias(collection, manifest, ctx)
146
+ alias_artifacts(collection, slug(member_identifier(manifest)),
147
+ ctx.store_dir)
148
+ end
149
+
150
+ def render_member(manifest_path, ctx, out)
151
+ compile = { install_fonts: false }
152
+ compile[:novalid] = true if ctx.no_valid == "1"
153
+ collection = Metanorma::Collection.parse(manifest_path)
154
+ collection.render(
155
+ format: [:xml], output_folder: out,
156
+ preserve_unresolved: true, artifact_store_dir: ctx.store_dir,
157
+ compile: compile
158
+ )
159
+ collection
160
+ end
161
+
162
+ def log_batch_ok(ctx, idx, pos, started)
163
+ warn format(
164
+ "[staged worker %<pid>d] batch %<idx>d/%<count>d member " \
165
+ "%<pos>d OK (%<seconds>d s)",
166
+ pid: Process.pid, idx: idx + 1, count: ctx.batch_count, pos: pos,
167
+ seconds: Time.now - started
168
+ )
169
+ end
170
+
171
+ # The artefact aliasing both stage paths perform after a member's
172
+ # store write (metanorma/suma#204): the store keys on the rendered
173
+ # document's own docid slug, while the parent reinflates and the
174
+ # warm-skip look up by the docref identifier. Without these copies a
175
+ # render banks entries the lookups can never find - the sequential
176
+ # stage never warm-skipped for exactly this reason.
177
+ def alias_artifacts(collection, id_slug, store_dir)
178
+ alias_from_bibdatas(collection, id_slug, store_dir)
179
+ alias_from_identifier_aliases(identifier_aliases(collection),
180
+ store_dir)
181
+ end
182
+
183
+ # Alias from the collection's own bibdata keys: they are the docids
184
+ # this render stored under. A produced-files delta is not safe here
185
+ # - with concurrent workers another child's render lands in the
186
+ # shared store mid-flight, and a delta would alias other members'
187
+ # documents under this docref (suma#167). bibdatas is empty for
188
+ # corpus manifests (suma#181) - the identifier_aliases below drive
189
+ # the aliasing in that case.
190
+ def alias_from_bibdatas(collection, id_slug, store_dir)
191
+ return if id_slug.empty?
192
+
193
+ collection.bibdatas&.each_key do |key|
194
+ src = slug(key)
195
+ next if src.empty? || src == id_slug
196
+
197
+ copy_store_entries(src, id_slug, store_dir)
198
+ end
199
+ end
200
+
201
+ # identifier_aliases maps raw identifier -> resolved docid; alias
202
+ # the resolved-docid entries under the raw identifier's slug.
203
+ def alias_from_identifier_aliases(aliases, store_dir)
204
+ aliases.each do |original, resolved|
205
+ src = slug(resolved)
206
+ dst = slug(original)
207
+ next if src.empty? || dst.empty? || src == dst
208
+
209
+ copy_store_entries(src, dst, store_dir)
210
+ end
211
+ end
212
+
213
+ def copy_store_entries(src, dst, store_dir)
214
+ Dir[File.join(store_dir, "#{src}.*.semantic.xml")].each do |f|
215
+ FileUtils.cp(f, f.sub(%r{/#{Regexp.escape(src)}\.}, "/#{dst}."))
216
+ end
217
+ end
218
+
219
+ def identifier_aliases(collection)
220
+ collection.manifest.identifier_aliases || {}
221
+ rescue NoMethodError
222
+ {}
223
+ end
224
+
225
+ def member_identifier(manifest)
226
+ manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
227
+ end
228
+
229
+ def claim_batch(config_dir, count)
230
+ (0...count).find do |i|
231
+ try_claim(File.join(config_dir, "._suma_batch_#{i}.claim"))
232
+ end
233
+ end
234
+
235
+ # O_EXCL create: succeeds for exactly one child, fails with
236
+ # EEXIST for everyone else - the claim.
237
+ def try_claim(path)
238
+ File.open(path, File::RDWR | File::CREAT | File::EXCL) do |claim|
239
+ claim.write("1")
240
+ end
241
+ true
242
+ rescue Errno::EEXIST
243
+ false
244
+ end
245
+
246
+ def batch_file(config_dir, idx)
247
+ File.join(config_dir, "._suma_batch_#{idx}.bin")
248
+ end
249
+
250
+ # Mirrors the store's filename slug (metanorma ArtifactStore#slug).
251
+ # The \A/\z anchors are load-bearing: they trim leading/trailing
252
+ # dashes only, and losing them in generated text was the suma#166
253
+ # defect.
254
+ def slug(value)
255
+ value.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
256
+ .gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
257
+ end
258
+ end
259
+ end
260
+ end
@@ -37,10 +37,65 @@ module Suma
37
37
  class StagedCollectionBuilder
38
38
  STORE_DIRNAME = ".metanorma-collection-cache"
39
39
 
40
+ autoload :Worker, "suma/staged_collection_builder/worker"
41
+
42
+ # The child-process boot line: one require, no embedded logic. All
43
+ # worker code lives in Worker (lib/suma/staged_collection_builder/
44
+ # worker.rb) — generated script strings shipped dead code once
45
+ # (suma#166) and could only be spec-pinned as text.
46
+ CHILD_BOOT = 'require "suma"; ' \
47
+ "Suma::StagedCollectionBuilder::Worker.run"
48
+
40
49
  # Members rendered per child process: one Metanorma boot amortised over
41
50
  # a batch of small members keeps boot cost under a few percent.
42
51
  DEFAULT_BATCH_SIZE = 25
43
52
 
53
+ # The warm-skip identity of one staged-store entry (metanorma/suma#203):
54
+ # the digest of the rendering toolchain that produced it, banked in a
55
+ # sidecar beside the stored semantic XML (raw hex, the form the
56
+ # SchemaCompiler render key uses). Presence-keyed reuse is unsound when
57
+ # a workflow cache carries a store across a toolchain change; an entry
58
+ # whose sidecar is missing (legacy store) or records a different
59
+ # toolchain is re-rendered, and every freshly rendered entry is
60
+ # re-banked — so a restored store converges onto the current toolchain
61
+ # at the cost of at most one extra render per entry.
62
+ class StoreEntry
63
+ SIDECHAR = ".toolchain"
64
+
65
+ class << self
66
+ # Bank +digest+ as the producing toolchain of the entry stored at
67
+ # +stored_path+.
68
+ def bank(stored_path, digest)
69
+ File.write(sidecar_path(stored_path), digest)
70
+ end
71
+
72
+ # The toolchain digest banked for the entry, or nil for a legacy
73
+ # entry written before banking existed.
74
+ def toolchain_digest(stored_path)
75
+ path = sidecar_path(stored_path)
76
+ File.file?(path) ? File.read(path) : nil
77
+ end
78
+
79
+ # Whether the entry was produced by the toolchain +digest+
80
+ # identifies. A legacy entry (no sidecar) never matches, so the
81
+ # first build over an old store re-renders once and re-banks.
82
+ def current?(stored_path, digest)
83
+ toolchain_digest(stored_path) == digest
84
+ end
85
+
86
+ # The banked digest truncated for log lines.
87
+ def short_digest(stored_path)
88
+ (toolchain_digest(stored_path) || "unknown")[0, 8]
89
+ end
90
+
91
+ private
92
+
93
+ def sidecar_path(stored_path)
94
+ "#{stored_path}#{SIDECHAR}"
95
+ end
96
+ end
97
+ end
98
+
44
99
  # +collection_config_path+ is the emitted collection manifest (the same
45
100
  # +collection-output.yaml+ the normal build renders). Member filerefs are
46
101
  # relative to its directory, so all per-member manifests are written there.
@@ -84,6 +139,42 @@ module Suma
84
139
  reinflate(reinflate_selection(members), members)
85
140
  end
86
141
 
142
+ # The warm-skip query (metanorma/suma#203): members whose stored entry
143
+ # was banked under the CURRENT rendering toolchain are done; every
144
+ # other member — no stored entry, a legacy entry without a banked
145
+ # digest, or an entry another toolchain produced — renders again.
146
+ # A workflow cache can carry a store across a toolchain or content-key
147
+ # change (TRThurman, iso-10303#858), and presence alone cannot see it:
148
+ # the banked digest is the durable guarantee the restored store cannot
149
+ # fake. Each member gets one provenance line: what was taken from the
150
+ # store and what compiled it, or that it renders.
151
+ def unstaged_members(members)
152
+ members.reject do |member|
153
+ identifier = member["identifier"]
154
+ stored = identifier && stored_semantic(identifier)
155
+ if stored && StoreEntry.current?(stored, Provenance.render_toolchain_digest)
156
+ Utils.log "[staged] #{identifier}: restored from store " \
157
+ "(toolchain #{StoreEntry.short_digest(stored)})"
158
+ true
159
+ else
160
+ Utils.log "[staged] #{identifier}: rendering"
161
+ false
162
+ end
163
+ end
164
+ end
165
+
166
+ # Record the rendering toolchain beside every freshly rendered
167
+ # member's store entry, so the next warm build can trust-or-recheck
168
+ # it. A legacy entry without a digest is re-rendered once and re-banked
169
+ # with one (safe migration); a warm-skipped member was already
170
+ # validated against the current digest and keeps its sidecar.
171
+ def bank_toolchains(members)
172
+ members.each do |member|
173
+ stored = stored_semantic(member["identifier"]) or next
174
+ StoreEntry.bank(stored, Provenance.render_toolchain_digest)
175
+ end
176
+ end
177
+
87
178
  private
88
179
 
89
180
  # The staged slice of the collection: an isolate selection wins over a
@@ -92,13 +183,16 @@ module Suma
92
183
  @isolate ? isolate_members(members) : shard_members(members)
93
184
  end
94
185
 
95
- # Render the staged members, serially or across the worker pool.
186
+ # Render the staged members, serially or across the worker pool, then
187
+ # bank every freshly rendered member's toolchain identity.
96
188
  def stage_members(staged)
189
+ unstaged = unstaged_members(staged)
97
190
  if @workers == 1
98
- unstaged_members(staged).each_with_index { |member, i| stage_member(member, i) }
191
+ unstaged.each_with_index { |member, i| stage_member(member, i) }
99
192
  else
100
- stage_parallel(staged)
193
+ stage_parallel(unstaged)
101
194
  end
195
+ bank_toolchains(unstaged)
102
196
  end
103
197
 
104
198
  # The reinflated slice: all members unless a reinflate shard is given,
@@ -156,19 +250,6 @@ module Suma
156
250
  "skipping reinflate"
157
251
  end
158
252
 
159
- # Members already present in the store (their docref-slug artefact
160
- # exists) are not re-rendered: an interrupted or repeated build over
161
- # a warm store goes straight to reinflate, so the assembly pass costs
162
- # minutes instead of re-rendering the whole collection.
163
- def unstaged_members(members)
164
- members.reject do |member|
165
- identifier = member["identifier"]
166
- stored = identifier && stored_semantic(identifier)
167
- Utils.log "[staged] stored, skipping #{identifier}" if stored
168
- stored
169
- end
170
- end
171
-
172
253
  # Batch the members and run +@workers+ renderer child processes
173
254
  # concurrently. Each batch file carries fully-rendered single-member
174
255
  # manifests; children claim batches through O_EXCL markers, so a slow
@@ -177,8 +258,9 @@ module Suma
177
258
  # child exit hands its memory back to the OS. Members write disjoint,
178
259
  # identifier-keyed store entries, so concurrent children cannot race;
179
260
  # sectionsplit only happens at reinflate (a single sequential pass).
261
+ # +members+ is the warm-skip's unstaged remainder.
180
262
  def stage_parallel(members)
181
- paths, count = write_batch_files(unstaged_members(members))
263
+ paths, count = write_batch_files(members)
182
264
  run_batch_workers(count)
183
265
  ensure
184
266
  remove_batch_files(paths, count)
@@ -202,14 +284,14 @@ module Suma
202
284
  def write_batch_bins(batches)
203
285
  batches.map.with_index do |batch, i|
204
286
  path = File.join(@config_dir, "._suma_batch_#{i}.bin")
205
- File.binwrite(path, Marshal.dump(batch))
287
+ File.write(path, Worker.dump_options(batch))
206
288
  path
207
289
  end
208
290
  end
209
291
 
210
292
  def run_batch_workers(count)
211
293
  children = Array.new([@workers, count].min) do
212
- Process.spawn(RbConfig.ruby, "-e", child_script, @config_dir,
294
+ Process.spawn(RbConfig.ruby, "-e", CHILD_BOOT, "batch", @config_dir,
213
295
  count.to_s, @store_dir, @output_directory,
214
296
  @no_valid ? "1" : "0", chdir: @config_dir)
215
297
  end
@@ -232,122 +314,6 @@ module Suma
232
314
  # Child renderer: claims and renders batches until none remain. The
233
315
  # Metanorma boot is paid once per child, and GC runs between members to
234
316
  # bound the heap. Any render failure exits nonzero and fails the build.
235
- def child_script
236
- <<~'RUBY'
237
- require "yaml"
238
- require "fileutils"
239
- require "metanorma"
240
- config_dir, batch_count, store_dir, out_dir, no_valid = ARGV
241
- count = Integer(batch_count)
242
- claimed = 0
243
- loop do
244
- idx = (0...count).find do |i|
245
- begin
246
- File.open(File.join(config_dir, "._suma_batch_#{i}.claim"),
247
- File::RDWR | File::CREAT | File::EXCL) {}
248
- true
249
- rescue Errno::EEXIST
250
- false
251
- end
252
- end
253
- break unless idx
254
-
255
- manifests = Marshal.load(File.binread(File.join(
256
- config_dir, "._suma_batch_#{idx}.bin")))
257
- manifests.each_with_index do |manifest, j|
258
- one_path = File.join(config_dir,
259
- "._suma_staged_b#{idx}_#{j}.yml")
260
- out = File.join(out_dir, "._staged_out_b#{idx}_#{j}")
261
- t0 = Time.now
262
- begin
263
- File.write(one_path, YAML.dump(manifest))
264
- # The artefact store keys on the rendered document's own
265
- # docid, while the parent reinflates by the docref
266
- # identifier; members whose document docid differs from the
267
- # docref (docrefs carrying metanorma's GUID marker) were
268
- # therefore unrecoverable ("no stored artefact"). Alias the
269
- # artefacts keyed by the render's own bibdata keys under the
270
- # docref slug too. Shape: single_member_manifest =>
271
- # {"manifest" => {"manifest" => [{"docref" => [dr]}]}}.
272
- member_id = manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
273
- id_slug = member_id.to_s
274
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
275
- .gsub(/\A-|-\z/, "")
276
- compile = { install_fonts: false }
277
- compile[:novalid] = true if no_valid == "1"
278
- collection = Metanorma::Collection.parse(one_path)
279
- collection.render(
280
- format: [:xml], output_folder: out,
281
- preserve_unresolved: true, artifact_store_dir: store_dir,
282
- compile: compile,
283
- )
284
- unless id_slug.empty?
285
- # Alias strictly from the collection's own bibdata keys:
286
- # they are the docids this member's render stored under.
287
- # A produced-files delta is not safe here - with
288
- # concurrent workers another child's render lands in the
289
- # shared store mid-flight, and a delta would alias other
290
- # members' documents under this docref (suma#167).
291
- (collection.bibdatas || {}).keys.each do |key|
292
- src_slug = key.to_s
293
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
294
- .gsub(/\A-|-\z/, "")
295
- next if src_slug.empty? || src_slug == id_slug
296
-
297
- Dir[File.join(store_dir,
298
- "#{src_slug}.*.semantic.xml")].each do |f|
299
- alias_name = f.sub(
300
- %r{/#{Regexp.escape(src_slug)}.},
301
- "/#{id_slug}.",
302
- )
303
- FileUtils.cp(f, alias_name)
304
- end
305
- end
306
- end
307
-
308
- # The corpus manifests carry no bibdata, so bibdatas is
309
- # empty and the loop above never runs (suma#181): the
310
- # artefacts stay keyed under the document's own docid and
311
- # reinflate - looking up by the docref identifier - finds
312
- # nothing on a cold store ("no stored artefact"). metanorma
313
- # resolves every manifest identifier to the document docid
314
- # during populate and records the mapping on the manifest
315
- # itself (identifier_aliases, raw -> resolved). That map is
316
- # the child's own post-render collection - reading it is
317
- # race-free per the suma#167 discipline.
318
- aliases = begin
319
- collection.manifest.identifier_aliases || {}
320
- rescue NoMethodError
321
- {}
322
- end
323
- aliases.each do |original, resolved|
324
- src = resolved.to_s
325
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
326
- .gsub(/\A-|-\z/, "")
327
- dst = original.to_s
328
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
329
- .gsub(/\A-|-\z/, "")
330
- next if src.empty? || dst.empty? || src == dst
331
-
332
- Dir[File.join(store_dir, "#{src}.*.semantic.xml")].each do |f|
333
- FileUtils.cp(
334
- f, f.sub(%r{/#{Regexp.escape(src)}\.}, "/#{dst}."),
335
- )
336
- end
337
- end
338
- warn format("[staged worker %d] batch %d/%d member %d OK (%ds)",
339
- Process.pid, idx + 1, count, j, Time.now - t0)
340
- ensure
341
- File.unlink(one_path) rescue nil
342
- FileUtils.rm_rf(out) rescue nil
343
- GC.start
344
- end
345
- end
346
- claimed += 1
347
- end
348
- exit 1 if claimed.zero? && count.positive?
349
- RUBY
350
- end
351
317
 
352
318
  def manifest
353
319
  @manifest ||= YAML.load_file(@config_path)
@@ -388,6 +354,7 @@ module Suma
388
354
  opts: { format: [:xml], output_folder: out,
389
355
  preserve_unresolved: true, artifact_store_dir: @store_dir,
390
356
  compile: compile_flags },
357
+ mode: "stage", id_slug: slug(member["identifier"])
391
358
  )
392
359
  ensure
393
360
  FileUtils.rm_f(one_path)
@@ -625,20 +592,20 @@ module Suma
625
592
 
626
593
  # Mirrors the store's filename slug (metanorma ArtifactStore#slug).
627
594
  def slug(identifier)
628
- identifier.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
629
- .gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
595
+ Worker.slug(identifier)
630
596
  end
631
597
 
632
598
  # Render a manifest in a FRESH child process so its memory is reclaimed on
633
- # exit. Under +bundle exec+, the child inherits the bundle. Raises on failure.
634
- def run_render(manifest_path:, opts:)
635
- script = <<~RUBY
636
- require "metanorma"
637
- opts = Marshal.load(STDIN.read)
638
- Metanorma::Collection.parse(#{manifest_path.inspect}).render(opts)
639
- RUBY
640
- IO.popen([RbConfig.ruby, "-e", script], "wb") do |io|
641
- io.write(Marshal.dump(opts))
599
+ # exit. Under +bundle exec+, the child inherits the bundle. Raises on
600
+ # failure. +mode+ "render" plain-renders (reinflate); "stage" additionally
601
+ # aliases the member's store artefacts under +id_slug+, the same aliasing
602
+ # the batch child path performs (suma#204 - without it the sequential
603
+ # stage banked docid-slug-only entries and never warm-skipped).
604
+ def run_render(manifest_path:, opts:, mode: "render", id_slug: nil)
605
+ argv = [mode, manifest_path]
606
+ argv += [@store_dir, id_slug] if mode == "stage"
607
+ IO.popen([RbConfig.ruby, "-e", CHILD_BOOT, *argv], "wb") do |io|
608
+ io.write(Worker.dump_options(opts))
642
609
  io.close_write
643
610
  end
644
611
  return if $?.success?
data/lib/suma/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Suma
4
- VERSION = "0.5.39"
4
+ VERSION = "0.5.41"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: suma
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.39
4
+ version: 0.5.41
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
@@ -243,6 +243,7 @@ files:
243
243
  - lib/suma/corpus_graph.rb
244
244
  - lib/suma/delivery.rb
245
245
  - lib/suma/delivery/cross_page_fragments.rb
246
+ - lib/suma/delivery/diagnostics.rb
246
247
  - lib/suma/delivery/expectations.rb
247
248
  - lib/suma/delivery/frame_nav.rb
248
249
  - lib/suma/delivery/pack.rb
@@ -292,6 +293,7 @@ files:
292
293
  - lib/suma/site_build.rb
293
294
  - lib/suma/site_config.rb
294
295
  - lib/suma/staged_collection_builder.rb
296
+ - lib/suma/staged_collection_builder/worker.rb
295
297
  - lib/suma/svg_quality.rb
296
298
  - lib/suma/svg_quality/batch_report.rb
297
299
  - lib/suma/svg_quality/formatters.rb