suma 0.5.19 → 0.5.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: df03b41b1c8ad66d68f1f6bb711f9cc49396e1b9217b2819eef5f1a9ae98b997
4
- data.tar.gz: 04f0d313dbe23ce6579b0c2736d264c7934dcc250a84c9e55f8bc39014e50a8e
3
+ metadata.gz: d57c7866a336d71b91bae0745359a3ad66a5ff465eb2f456a27d74a3d901328b
4
+ data.tar.gz: 6c84fb67e536cc4df2fed9e6a75ad39700b3c6a8162d63a6f92779e370c52be7
5
5
  SHA512:
6
- metadata.gz: 1eb98f1e8d9cd8ead0c40db5867929d74a50d440cd964848a0fbafaac2b23e0ad0d484522cc98a1ea5afc9c4cf88137ed8670af848088059c37c0c718da2dfdc
7
- data.tar.gz: 85bf0f0ea57ac912398b33264550a89f7b0118cac0b76f7a239cc33b49a78d39d9e535abdeb6a801022fb462dabb02727bbc38a055ee99b9f7ac7a88e50b8afe
6
+ metadata.gz: 9ccc76d252726749978bceee65caf1a75fc954f80b6c17652e373c6aa6b7c263d64ccb7f3e114accc4f66a564c54c1f7a9282d9914cd11ce2319fe50925ca64d
7
+ data.tar.gz: 623e14b1b637cf112bdd66d8d62ac9b54543b45498efc5117abb1f1aad2a76b5b582eaa49cc1ec66576de03d3069de05e41e1dde8d84c231ff7a1f8621247379
@@ -29,6 +29,23 @@ module Suma
29
29
  "INDEXth member and skips reinflate, so " \
30
30
  "disjoint shards can render concurrently " \
31
31
  "into a shared store (requires --staged)"
32
+ option :staged_shard_isolate, type: :string,
33
+ desc: "Stage only the single member " \
34
+ "whose fileref contains this " \
35
+ "slug (e.g. iso-10303-104) and " \
36
+ "skip reinflate: gives a " \
37
+ "mega-member a build run of its " \
38
+ "own (requires --staged)"
39
+ option :staged_reinflate_shard, type: :string,
40
+ desc: "Reinflate only this shard of " \
41
+ "the collection as INDEX/TOTAL " \
42
+ "(e.g. 2/4): stage everything " \
43
+ "(warm store = skips), render " \
44
+ "the slice's members into the " \
45
+ "site, so parallel reinflate " \
46
+ "jobs over one store write " \
47
+ "disjoint sections (requires " \
48
+ "--staged)"
32
49
  option :incremental, type: :boolean, default: false,
33
50
  desc: "Re-render only changed schemas and their " \
34
51
  "transitive dependents (state file defaults " \
@@ -86,7 +103,9 @@ module Suma
86
103
  { staged: options[:staged],
87
104
  staged_workers: options[:staged_workers],
88
105
  staged_batch_size: options[:staged_batch_size],
89
- staged_shard: parse_shard(options[:staged_shard]) }
106
+ staged_shard: parse_shard(options[:staged_shard]),
107
+ staged_shard_isolate: options[:staged_shard_isolate],
108
+ staged_reinflate_shard: parse_shard(options[:staged_reinflate_shard]) }
90
109
  end
91
110
 
92
111
  def parse_shard(value)
data/lib/suma/cli/core.rb CHANGED
@@ -37,6 +37,13 @@ module Suma
37
37
  option :staged_shard, type: :string,
38
38
  desc: "Render only this shard as INDEX/TOTAL" \
39
39
  "(requires --staged)"
40
+ option :staged_shard_isolate, type: :string,
41
+ desc: "Stage only the member whose " \
42
+ "fileref contains this slug " \
43
+ "(requires --staged)"
44
+ option :staged_reinflate_shard, type: :string,
45
+ desc: "Reinflate only this shard as " \
46
+ "INDEX/TOTAL (requires --staged)"
40
47
  option :incremental, type: :boolean, default: false,
41
48
  desc: "Re-render only changed schemas and dependents"
42
49
  option :state_file, type: :string,
@@ -14,7 +14,8 @@ module Suma
14
14
  def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
15
15
  output_directory: "_site", staged: false,
16
16
  staged_workers: 1, staged_batch_size: nil,
17
- staged_shard: nil,
17
+ staged_shard: nil, staged_shard_isolate: nil,
18
+ staged_reinflate_shard: nil,
18
19
  incremental: false, state_file: nil, jobs: 1,
19
20
  provenance_file: nil, sign_method: nil, sign_key: nil,
20
21
  no_valid: false)
@@ -28,6 +29,8 @@ module Suma
28
29
  @staged_workers = staged_workers
29
30
  @staged_batch_size = staged_batch_size
30
31
  @staged_shard = staged_shard
32
+ @staged_shard_isolate = staged_shard_isolate
33
+ @staged_reinflate_shard = staged_reinflate_shard
31
34
  @incremental = incremental
32
35
  @state_file = state_file
33
36
  @jobs = jobs
@@ -190,6 +193,8 @@ module Suma
190
193
  batch_size: @staged_batch_size ||
191
194
  StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
192
195
  shard: @staged_shard,
196
+ isolate: @staged_shard_isolate,
197
+ reinflate_shard: @staged_reinflate_shard,
193
198
  ).build
194
199
  else
195
200
  Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
@@ -3,6 +3,7 @@
3
3
  require "metanorma"
4
4
  require "yaml"
5
5
  require "fileutils"
6
+ require "pathname"
6
7
 
7
8
  module Suma
8
9
  # Memory-bounded, resumable staged collection build (metanorma/suma#94).
@@ -48,12 +49,13 @@ module Suma
48
49
  # entries, so children cannot race. +shard+ is an [index, total] pair:
49
50
  # only members whose position satisfies +index % total+ are staged and
50
51
  # reinflation is skipped, so disjoint shards of one collection can render
51
- # concurrently in separate build runs into a shared store. (a
52
- # data-declaring initializer)
52
+ # concurrently in separate build runs into a shared store. +isolate+ is
53
+ # a fileref slug with the same shard semantics for a single member.
54
+ # (a data-declaring initializer)
53
55
  def initialize(collection_config_path:, output_directory:, coverpage: nil,
54
56
  store_dir: nil, formats: %i[xml html], no_valid: false,
55
57
  workers: 1, batch_size: DEFAULT_BATCH_SIZE,
56
- shard: nil) # rubocop:disable Metrics/ParameterLists
58
+ shard: nil, isolate: nil, reinflate_shard: nil) # rubocop:disable Metrics/ParameterLists
57
59
  @config_path = collection_config_path
58
60
  @config_dir = File.dirname(File.expand_path(collection_config_path))
59
61
  @output_directory = output_directory
@@ -64,25 +66,53 @@ module Suma
64
66
  @workers = [workers.to_i, 1].max
65
67
  @batch_size = [batch_size.to_i, 1].max
66
68
  @shard = shard
69
+ @isolate = isolate
70
+ @reinflate_shard = reinflate_shard
67
71
  end
68
72
 
69
73
  def build
70
74
  FileUtils.mkdir_p(@output_directory)
71
75
  FileUtils.mkdir_p(@store_dir)
72
76
  members = collection_members
73
- staged = shard_members(members)
77
+ staged = staged_selection(members)
74
78
  Utils.log shard_log(staged.size, members.size)
79
+ stage_members(staged)
80
+ return if @shard || @isolate
81
+
82
+ reinflate(reinflate_selection(members))
83
+ end
84
+
85
+ private
86
+
87
+ # The staged slice of the collection: an isolate selection wins over a
88
+ # modulo shard when both are given.
89
+ def staged_selection(members)
90
+ @isolate ? isolate_members(members) : shard_members(members)
91
+ end
92
+
93
+ # Render the staged members, serially or across the worker pool.
94
+ def stage_members(staged)
75
95
  if @workers == 1
76
- staged.each_with_index { |member, i| stage_member(member, i) }
96
+ unstaged_members(staged).each_with_index { |member, i| stage_member(member, i) }
77
97
  else
78
98
  stage_parallel(staged)
79
99
  end
80
- return if @shard
81
-
82
- reinflate(members)
83
100
  end
84
101
 
85
- private
102
+ # The reinflated slice: all members unless a reinflate shard is given,
103
+ # in which case only the members matching +index % total+ render into
104
+ # the output directory. Disjoint reinflate slices of one collection can
105
+ # run as parallel build jobs over the same store: each renders its own
106
+ # members' sections, cross-references resolve through the restored
107
+ # anchors of the whole store, and the written files are disjoint.
108
+ def reinflate_selection(members)
109
+ return members unless @reinflate_shard
110
+
111
+ index, total = @reinflate_shard
112
+ members.each_with_index.filter_map do |member, position|
113
+ member if (position % total) == index
114
+ end
115
+ end
86
116
 
87
117
  # A shard stages the members whose position in the member order matches
88
118
  # +index % total+; the full member list is unchanged.
@@ -95,15 +125,48 @@ module Suma
95
125
  end
96
126
  end
97
127
 
128
+ # Isolation stages the single member whose fileref path contains the
129
+ # +isolate+ slug (e.g. a mega-member that needs a whole runner to
130
+ # itself, away from sibling workers), and skips reinflation like a
131
+ # shard. Empty selection means the slug matched nothing, which is a
132
+ # caller error worth failing on rather than silently rendering
133
+ # nothing.
134
+ def isolate_members(members)
135
+ staged = members.select do |member|
136
+ File.basename(member["fileref"]).include?(@isolate)
137
+ end
138
+ return staged unless staged.empty?
139
+
140
+ raise ArgumentError,
141
+ "--staged-shard-isolate matched no member for #{@isolate.inspect}"
142
+ end
143
+
98
144
  def shard_log(staged_count, total_count)
99
145
  line = "[staged] #{total_count} members; workers: #{@workers}; " \
100
146
  "batch size: #{@batch_size}; store: #{@store_dir}"
147
+ if @isolate
148
+ return "#{line}; isolate #{@isolate.inspect}: staging " \
149
+ "#{staged_count}, skipping reinflate"
150
+ end
101
151
  return line unless @shard
102
152
 
103
153
  "#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
104
154
  "skipping reinflate"
105
155
  end
106
156
 
157
+ # Members already present in the store (their docref-slug artefact
158
+ # exists) are not re-rendered: an interrupted or repeated build over
159
+ # a warm store goes straight to reinflate, so the assembly pass costs
160
+ # minutes instead of re-rendering the whole collection.
161
+ def unstaged_members(members)
162
+ members.reject do |member|
163
+ identifier = member["identifier"]
164
+ stored = identifier && stored_semantic(identifier)
165
+ Utils.log "[staged] stored, skipping #{identifier}" if stored
166
+ stored
167
+ end
168
+ end
169
+
107
170
  # Batch the members and run +@workers+ renderer child processes
108
171
  # concurrently. Each batch file carries fully-rendered single-member
109
172
  # manifests; children claim batches through O_EXCL markers, so a slow
@@ -113,7 +176,7 @@ module Suma
113
176
  # identifier-keyed store entries, so concurrent children cannot race;
114
177
  # sectionsplit only happens at reinflate (a single sequential pass).
115
178
  def stage_parallel(members)
116
- paths, count = write_batch_files(members)
179
+ paths, count = write_batch_files(unstaged_members(members))
117
180
  run_batch_workers(count)
118
181
  ensure
119
182
  remove_batch_files(paths, count)
@@ -201,15 +264,13 @@ module Suma
201
264
  # identifier; members whose document docid differs from the
202
265
  # docref (docrefs carrying metanorma's GUID marker) were
203
266
  # therefore unrecoverable ("no stored artefact"). Alias the
204
- # artefacts this render produced under the docref slug too.
205
- # Shape: single_member_manifest => {"manifest" =>
206
- # {"manifest" => [{"docref" => [dr]}]}}.
267
+ # artefacts keyed by the render's own bibdata keys under the
268
+ # docref slug too. Shape: single_member_manifest =>
269
+ # {"manifest" => {"manifest" => [{"docref" => [dr]}]}}.
207
270
  member_id = manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
208
271
  id_slug = member_id.to_s
209
272
  .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
210
273
  .gsub(/\A-|-\z/, "")
211
- semantics_before = Dir.children(store_dir)
212
- .select { |f| f.end_with?(".semantic.xml") }
213
274
  compile = { install_fonts: false }
214
275
  compile[:novalid] = true if no_valid == "1"
215
276
  collection = Metanorma::Collection.parse(one_path)
@@ -219,37 +280,25 @@ module Suma
219
280
  compile: compile,
220
281
  )
221
282
  unless id_slug.empty?
222
- # A warm artefact cache makes the render a full cache hit:
223
- # no new artefact files appear, so a produced-files delta
224
- # never fires. The collection's own keys are what the
225
- # renderer stored under, so alias every semantic artefact
226
- # keyed by them to the docref slug as well.
227
- store_keys = (collection.bibdatas || {}).keys
228
- produced = Dir.children(store_dir).select do |f|
229
- f.end_with?(".semantic.xml")
230
- end - semantics_before
231
- if produced.empty?
232
- store_keys.each do |key|
233
- src_slug = key.to_s
234
- .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
235
- .gsub(/\A-|-\z/, "")
236
- next if src_slug.empty? || src_slug == id_slug
237
-
238
- Dir[File.join(store_dir,
239
- "#{src_slug}.*.semantic.xml")].each do |f|
240
- alias_name = f.sub(
241
- %r{/#{Regexp.escape(src_slug)}.},
242
- "/#{id_slug}.",
243
- )
244
- FileUtils.cp(f, alias_name)
245
- end
246
- end
247
- end
248
- produced.each do |f|
249
- if (m = f.match(/\A(.*).([0-9a-f]+).semantic.xml\z/))
250
- alias_name = "#{id_slug}.#{m[2]}.semantic.xml"
251
- alias_path = File.join(store_dir, alias_name)
252
- FileUtils.cp(File.join(store_dir, f), alias_path)
283
+ # Alias strictly from the collection's own bibdata keys:
284
+ # they are the docids this member's render stored under.
285
+ # A produced-files delta is not safe here - with
286
+ # concurrent workers another child's render lands in the
287
+ # shared store mid-flight, and a delta would alias other
288
+ # members' documents under this docref (suma#167).
289
+ (collection.bibdatas || {}).keys.each do |key|
290
+ src_slug = key.to_s
291
+ .gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
292
+ .gsub(/\A-|-\z/, "")
293
+ next if src_slug.empty? || src_slug == id_slug
294
+
295
+ Dir[File.join(store_dir,
296
+ "#{src_slug}.*.semantic.xml")].each do |f|
297
+ alias_name = f.sub(
298
+ %r{/#{Regexp.escape(src_slug)}.},
299
+ "/#{id_slug}.",
300
+ )
301
+ FileUtils.cp(f, alias_name)
253
302
  end
254
303
  end
255
304
  end
@@ -313,21 +362,81 @@ module Suma
313
362
  end
314
363
 
315
364
  # Assemble a manifest of the stored stubs and reinflate them into the site.
365
+ # The original manifest tree is preserved (nested collections,
366
+ # attachments, per-member sectionsplit) with each member entry swapped
367
+ # for its stored stub; the render resolves paths against the
368
+ # reinflate manifest's own directory, so every path the render reads
369
+ # is absolutized against the collection root (suma#168, suma#169).
316
370
  def reinflate(members)
317
371
  reinf_dir = File.join(@output_directory, "_staged_reinflate")
318
372
  FileUtils.mkdir_p(reinf_dir)
319
373
  docrefs = stored_docrefs(members, reinf_dir)
374
+ write_reinflate_manifest(docrefs, reinf_dir)
375
+ run_render(manifest_path: File.join(reinf_dir, "reinflate.yml"),
376
+ opts: reinflate_render_options)
377
+ end
378
+
379
+ def write_reinflate_manifest(docrefs, reinf_dir)
380
+ rebuilt = with_stubbed_members(manifest["manifest"], docrefs, [0])
320
381
  reinf_path = File.join(reinf_dir, "reinflate.yml")
321
382
  File.write(reinf_path,
322
- base_manifest.merge("manifest" => collection_manifest(docrefs)).to_yaml)
383
+ base_manifest.merge("manifest" => rebuilt).to_yaml)
323
384
  Utils.log "[staged] reinflating #{docrefs.size} stored members -> " \
324
385
  "#{@output_directory}"
325
- run_render(
326
- manifest_path: reinf_path,
327
- opts: { format: @formats, output_folder: @output_directory,
328
- reinflate: true, coverpage: @coverpage,
329
- compile: compile_flags }.compact,
330
- )
386
+ end
387
+
388
+ def reinflate_render_options
389
+ {
390
+ format: @formats,
391
+ output_folder: File.expand_path(@output_directory),
392
+ reinflate: true,
393
+ coverpage: @coverpage && File.expand_path(@coverpage, @config_dir),
394
+ compile: compile_flags,
395
+ }.compact
396
+ end
397
+
398
+ # Deep-copy the manifest tree, replacing each member entry (same
399
+ # predicate and pre-order as collection_members) with its stub docref,
400
+ # and absolutizing non-member filerefs (attachments) against the
401
+ # collection root.
402
+ def with_stubbed_members(node, docrefs, counter)
403
+ case node
404
+ when Array
405
+ node.map { |child| with_stubbed_members(child, docrefs, counter) }
406
+ when Hash
407
+ with_stubbed_members_hash(node, docrefs, counter)
408
+ else
409
+ node
410
+ end
411
+ end
412
+
413
+ def with_stubbed_members_hash(node, docrefs, counter)
414
+ rewritten = node.dup
415
+ rewritten.each_key do |key|
416
+ rewritten[key] = rewrite_entry_value(key, rewritten[key], docrefs, counter)
417
+ end
418
+ stub_for(rewritten, docrefs, counter) || rewritten
419
+ end
420
+
421
+ def rewrite_entry_value(key, value, docrefs, counter)
422
+ if %w[file fileref].include?(key) && value &&
423
+ !Pathname.new(value).absolute?
424
+ File.expand_path(value, @config_dir)
425
+ else
426
+ with_stubbed_members(value, docrefs, counter)
427
+ end
428
+ end
429
+
430
+ def stub_for(node, docrefs, counter)
431
+ return unless member_entry?(node)
432
+
433
+ index = counter[0]
434
+ counter[0] += 1
435
+ docrefs[index]
436
+ end
437
+
438
+ def member_entry?(node)
439
+ (node["fileref"] || node["file"]) && !truthy?(node["attachment"])
331
440
  end
332
441
 
333
442
  def compile_flags
data/lib/suma/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Suma
4
- VERSION = "0.5.19"
4
+ VERSION = "0.5.21"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: suma
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.19
4
+ version: 0.5.21
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2026-11-01 00:00:00.000000000 Z
11
+ date: 2026-11-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: expressir