suma 0.5.19 → 0.5.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/suma/cli/build.rb +20 -1
- data/lib/suma/cli/core.rb +7 -0
- data/lib/suma/processor.rb +6 -1
- data/lib/suma/staged_collection_builder.rb +162 -53
- data/lib/suma/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d57c7866a336d71b91bae0745359a3ad66a5ff465eb2f456a27d74a3d901328b
|
|
4
|
+
data.tar.gz: 6c84fb67e536cc4df2fed9e6a75ad39700b3c6a8162d63a6f92779e370c52be7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9ccc76d252726749978bceee65caf1a75fc954f80b6c17652e373c6aa6b7c263d64ccb7f3e114accc4f66a564c54c1f7a9282d9914cd11ce2319fe50925ca64d
|
|
7
|
+
data.tar.gz: 623e14b1b637cf112bdd66d8d62ac9b54543b45498efc5117abb1f1aad2a76b5b582eaa49cc1ec66576de03d3069de05e41e1dde8d84c231ff7a1f8621247379
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -29,6 +29,23 @@ module Suma
|
|
|
29
29
|
"INDEXth member and skips reinflate, so " \
|
|
30
30
|
"disjoint shards can render concurrently " \
|
|
31
31
|
"into a shared store (requires --staged)"
|
|
32
|
+
option :staged_shard_isolate, type: :string,
|
|
33
|
+
desc: "Stage only the single member " \
|
|
34
|
+
"whose fileref contains this " \
|
|
35
|
+
"slug (e.g. iso-10303-104) and " \
|
|
36
|
+
"skip reinflate: gives a " \
|
|
37
|
+
"mega-member a build run of its " \
|
|
38
|
+
"own (requires --staged)"
|
|
39
|
+
option :staged_reinflate_shard, type: :string,
|
|
40
|
+
desc: "Reinflate only this shard of " \
|
|
41
|
+
"the collection as INDEX/TOTAL " \
|
|
42
|
+
"(e.g. 2/4): stage everything " \
|
|
43
|
+
"(warm store = skips), render " \
|
|
44
|
+
"the slice's members into the " \
|
|
45
|
+
"site, so parallel reinflate " \
|
|
46
|
+
"jobs over one store write " \
|
|
47
|
+
"disjoint sections (requires " \
|
|
48
|
+
"--staged)"
|
|
32
49
|
option :incremental, type: :boolean, default: false,
|
|
33
50
|
desc: "Re-render only changed schemas and their " \
|
|
34
51
|
"transitive dependents (state file defaults " \
|
|
@@ -86,7 +103,9 @@ module Suma
|
|
|
86
103
|
{ staged: options[:staged],
|
|
87
104
|
staged_workers: options[:staged_workers],
|
|
88
105
|
staged_batch_size: options[:staged_batch_size],
|
|
89
|
-
staged_shard: parse_shard(options[:staged_shard])
|
|
106
|
+
staged_shard: parse_shard(options[:staged_shard]),
|
|
107
|
+
staged_shard_isolate: options[:staged_shard_isolate],
|
|
108
|
+
staged_reinflate_shard: parse_shard(options[:staged_reinflate_shard]) }
|
|
90
109
|
end
|
|
91
110
|
|
|
92
111
|
def parse_shard(value)
|
data/lib/suma/cli/core.rb
CHANGED
|
@@ -37,6 +37,13 @@ module Suma
|
|
|
37
37
|
option :staged_shard, type: :string,
|
|
38
38
|
desc: "Render only this shard as INDEX/TOTAL" \
|
|
39
39
|
"(requires --staged)"
|
|
40
|
+
option :staged_shard_isolate, type: :string,
|
|
41
|
+
desc: "Stage only the member whose " \
|
|
42
|
+
"fileref contains this slug " \
|
|
43
|
+
"(requires --staged)"
|
|
44
|
+
option :staged_reinflate_shard, type: :string,
|
|
45
|
+
desc: "Reinflate only this shard as " \
|
|
46
|
+
"INDEX/TOTAL (requires --staged)"
|
|
40
47
|
option :incremental, type: :boolean, default: false,
|
|
41
48
|
desc: "Re-render only changed schemas and dependents"
|
|
42
49
|
option :state_file, type: :string,
|
data/lib/suma/processor.rb
CHANGED
|
@@ -14,7 +14,8 @@ module Suma
|
|
|
14
14
|
def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
|
|
15
15
|
output_directory: "_site", staged: false,
|
|
16
16
|
staged_workers: 1, staged_batch_size: nil,
|
|
17
|
-
staged_shard: nil,
|
|
17
|
+
staged_shard: nil, staged_shard_isolate: nil,
|
|
18
|
+
staged_reinflate_shard: nil,
|
|
18
19
|
incremental: false, state_file: nil, jobs: 1,
|
|
19
20
|
provenance_file: nil, sign_method: nil, sign_key: nil,
|
|
20
21
|
no_valid: false)
|
|
@@ -28,6 +29,8 @@ module Suma
|
|
|
28
29
|
@staged_workers = staged_workers
|
|
29
30
|
@staged_batch_size = staged_batch_size
|
|
30
31
|
@staged_shard = staged_shard
|
|
32
|
+
@staged_shard_isolate = staged_shard_isolate
|
|
33
|
+
@staged_reinflate_shard = staged_reinflate_shard
|
|
31
34
|
@incremental = incremental
|
|
32
35
|
@state_file = state_file
|
|
33
36
|
@jobs = jobs
|
|
@@ -190,6 +193,8 @@ module Suma
|
|
|
190
193
|
batch_size: @staged_batch_size ||
|
|
191
194
|
StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
|
|
192
195
|
shard: @staged_shard,
|
|
196
|
+
isolate: @staged_shard_isolate,
|
|
197
|
+
reinflate_shard: @staged_reinflate_shard,
|
|
193
198
|
).build
|
|
194
199
|
else
|
|
195
200
|
Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "metanorma"
|
|
4
4
|
require "yaml"
|
|
5
5
|
require "fileutils"
|
|
6
|
+
require "pathname"
|
|
6
7
|
|
|
7
8
|
module Suma
|
|
8
9
|
# Memory-bounded, resumable staged collection build (metanorma/suma#94).
|
|
@@ -48,12 +49,13 @@ module Suma
|
|
|
48
49
|
# entries, so children cannot race. +shard+ is an [index, total] pair:
|
|
49
50
|
# only members whose position satisfies +index % total+ are staged and
|
|
50
51
|
# reinflation is skipped, so disjoint shards of one collection can render
|
|
51
|
-
# concurrently in separate build runs into a shared store.
|
|
52
|
-
#
|
|
52
|
+
# concurrently in separate build runs into a shared store. +isolate+ is
|
|
53
|
+
# a fileref slug with the same shard semantics for a single member.
|
|
54
|
+
# (a data-declaring initializer)
|
|
53
55
|
def initialize(collection_config_path:, output_directory:, coverpage: nil,
|
|
54
56
|
store_dir: nil, formats: %i[xml html], no_valid: false,
|
|
55
57
|
workers: 1, batch_size: DEFAULT_BATCH_SIZE,
|
|
56
|
-
shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
58
|
+
shard: nil, isolate: nil, reinflate_shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
57
59
|
@config_path = collection_config_path
|
|
58
60
|
@config_dir = File.dirname(File.expand_path(collection_config_path))
|
|
59
61
|
@output_directory = output_directory
|
|
@@ -64,25 +66,53 @@ module Suma
|
|
|
64
66
|
@workers = [workers.to_i, 1].max
|
|
65
67
|
@batch_size = [batch_size.to_i, 1].max
|
|
66
68
|
@shard = shard
|
|
69
|
+
@isolate = isolate
|
|
70
|
+
@reinflate_shard = reinflate_shard
|
|
67
71
|
end
|
|
68
72
|
|
|
69
73
|
def build
|
|
70
74
|
FileUtils.mkdir_p(@output_directory)
|
|
71
75
|
FileUtils.mkdir_p(@store_dir)
|
|
72
76
|
members = collection_members
|
|
73
|
-
staged =
|
|
77
|
+
staged = staged_selection(members)
|
|
74
78
|
Utils.log shard_log(staged.size, members.size)
|
|
79
|
+
stage_members(staged)
|
|
80
|
+
return if @shard || @isolate
|
|
81
|
+
|
|
82
|
+
reinflate(reinflate_selection(members))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
# The staged slice of the collection: an isolate selection wins over a
|
|
88
|
+
# modulo shard when both are given.
|
|
89
|
+
def staged_selection(members)
|
|
90
|
+
@isolate ? isolate_members(members) : shard_members(members)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Render the staged members, serially or across the worker pool.
|
|
94
|
+
def stage_members(staged)
|
|
75
95
|
if @workers == 1
|
|
76
|
-
staged.each_with_index { |member, i| stage_member(member, i) }
|
|
96
|
+
unstaged_members(staged).each_with_index { |member, i| stage_member(member, i) }
|
|
77
97
|
else
|
|
78
98
|
stage_parallel(staged)
|
|
79
99
|
end
|
|
80
|
-
return if @shard
|
|
81
|
-
|
|
82
|
-
reinflate(members)
|
|
83
100
|
end
|
|
84
101
|
|
|
85
|
-
|
|
102
|
+
# The reinflated slice: all members unless a reinflate shard is given,
|
|
103
|
+
# in which case only the members matching +index % total+ render into
|
|
104
|
+
# the output directory. Disjoint reinflate slices of one collection can
|
|
105
|
+
# run as parallel build jobs over the same store: each renders its own
|
|
106
|
+
# members' sections, cross-references resolve through the restored
|
|
107
|
+
# anchors of the whole store, and the written files are disjoint.
|
|
108
|
+
def reinflate_selection(members)
|
|
109
|
+
return members unless @reinflate_shard
|
|
110
|
+
|
|
111
|
+
index, total = @reinflate_shard
|
|
112
|
+
members.each_with_index.filter_map do |member, position|
|
|
113
|
+
member if (position % total) == index
|
|
114
|
+
end
|
|
115
|
+
end
|
|
86
116
|
|
|
87
117
|
# A shard stages the members whose position in the member order matches
|
|
88
118
|
# +index % total+; the full member list is unchanged.
|
|
@@ -95,15 +125,48 @@ module Suma
|
|
|
95
125
|
end
|
|
96
126
|
end
|
|
97
127
|
|
|
128
|
+
# Isolation stages the single member whose fileref path contains the
|
|
129
|
+
# +isolate+ slug (e.g. a mega-member that needs a whole runner to
|
|
130
|
+
# itself, away from sibling workers), and skips reinflation like a
|
|
131
|
+
# shard. Empty selection means the slug matched nothing, which is a
|
|
132
|
+
# caller error worth failing on rather than silently rendering
|
|
133
|
+
# nothing.
|
|
134
|
+
def isolate_members(members)
|
|
135
|
+
staged = members.select do |member|
|
|
136
|
+
File.basename(member["fileref"]).include?(@isolate)
|
|
137
|
+
end
|
|
138
|
+
return staged unless staged.empty?
|
|
139
|
+
|
|
140
|
+
raise ArgumentError,
|
|
141
|
+
"--staged-shard-isolate matched no member for #{@isolate.inspect}"
|
|
142
|
+
end
|
|
143
|
+
|
|
98
144
|
def shard_log(staged_count, total_count)
|
|
99
145
|
line = "[staged] #{total_count} members; workers: #{@workers}; " \
|
|
100
146
|
"batch size: #{@batch_size}; store: #{@store_dir}"
|
|
147
|
+
if @isolate
|
|
148
|
+
return "#{line}; isolate #{@isolate.inspect}: staging " \
|
|
149
|
+
"#{staged_count}, skipping reinflate"
|
|
150
|
+
end
|
|
101
151
|
return line unless @shard
|
|
102
152
|
|
|
103
153
|
"#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
|
|
104
154
|
"skipping reinflate"
|
|
105
155
|
end
|
|
106
156
|
|
|
157
|
+
# Members already present in the store (their docref-slug artefact
|
|
158
|
+
# exists) are not re-rendered: an interrupted or repeated build over
|
|
159
|
+
# a warm store goes straight to reinflate, so the assembly pass costs
|
|
160
|
+
# minutes instead of re-rendering the whole collection.
|
|
161
|
+
def unstaged_members(members)
|
|
162
|
+
members.reject do |member|
|
|
163
|
+
identifier = member["identifier"]
|
|
164
|
+
stored = identifier && stored_semantic(identifier)
|
|
165
|
+
Utils.log "[staged] stored, skipping #{identifier}" if stored
|
|
166
|
+
stored
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
107
170
|
# Batch the members and run +@workers+ renderer child processes
|
|
108
171
|
# concurrently. Each batch file carries fully-rendered single-member
|
|
109
172
|
# manifests; children claim batches through O_EXCL markers, so a slow
|
|
@@ -113,7 +176,7 @@ module Suma
|
|
|
113
176
|
# identifier-keyed store entries, so concurrent children cannot race;
|
|
114
177
|
# sectionsplit only happens at reinflate (a single sequential pass).
|
|
115
178
|
def stage_parallel(members)
|
|
116
|
-
paths, count = write_batch_files(members)
|
|
179
|
+
paths, count = write_batch_files(unstaged_members(members))
|
|
117
180
|
run_batch_workers(count)
|
|
118
181
|
ensure
|
|
119
182
|
remove_batch_files(paths, count)
|
|
@@ -201,15 +264,13 @@ module Suma
|
|
|
201
264
|
# identifier; members whose document docid differs from the
|
|
202
265
|
# docref (docrefs carrying metanorma's GUID marker) were
|
|
203
266
|
# therefore unrecoverable ("no stored artefact"). Alias the
|
|
204
|
-
# artefacts
|
|
205
|
-
# Shape: single_member_manifest =>
|
|
206
|
-
#
|
|
267
|
+
# artefacts keyed by the render's own bibdata keys under the
|
|
268
|
+
# docref slug too. Shape: single_member_manifest =>
|
|
269
|
+
# {"manifest" => {"manifest" => [{"docref" => [dr]}]}}.
|
|
207
270
|
member_id = manifest.dig("manifest", "manifest", 0, "docref", 0, "identifier")
|
|
208
271
|
id_slug = member_id.to_s
|
|
209
272
|
.gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
|
|
210
273
|
.gsub(/\A-|-\z/, "")
|
|
211
|
-
semantics_before = Dir.children(store_dir)
|
|
212
|
-
.select { |f| f.end_with?(".semantic.xml") }
|
|
213
274
|
compile = { install_fonts: false }
|
|
214
275
|
compile[:novalid] = true if no_valid == "1"
|
|
215
276
|
collection = Metanorma::Collection.parse(one_path)
|
|
@@ -219,37 +280,25 @@ module Suma
|
|
|
219
280
|
compile: compile,
|
|
220
281
|
)
|
|
221
282
|
unless id_slug.empty?
|
|
222
|
-
#
|
|
223
|
-
#
|
|
224
|
-
#
|
|
225
|
-
#
|
|
226
|
-
#
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
%r{/#{Regexp.escape(src_slug)}.},
|
|
242
|
-
"/#{id_slug}.",
|
|
243
|
-
)
|
|
244
|
-
FileUtils.cp(f, alias_name)
|
|
245
|
-
end
|
|
246
|
-
end
|
|
247
|
-
end
|
|
248
|
-
produced.each do |f|
|
|
249
|
-
if (m = f.match(/\A(.*).([0-9a-f]+).semantic.xml\z/))
|
|
250
|
-
alias_name = "#{id_slug}.#{m[2]}.semantic.xml"
|
|
251
|
-
alias_path = File.join(store_dir, alias_name)
|
|
252
|
-
FileUtils.cp(File.join(store_dir, f), alias_path)
|
|
283
|
+
# Alias strictly from the collection's own bibdata keys:
|
|
284
|
+
# they are the docids this member's render stored under.
|
|
285
|
+
# A produced-files delta is not safe here - with
|
|
286
|
+
# concurrent workers another child's render lands in the
|
|
287
|
+
# shared store mid-flight, and a delta would alias other
|
|
288
|
+
# members' documents under this docref (suma#167).
|
|
289
|
+
(collection.bibdatas || {}).keys.each do |key|
|
|
290
|
+
src_slug = key.to_s
|
|
291
|
+
.gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
|
|
292
|
+
.gsub(/\A-|-\z/, "")
|
|
293
|
+
next if src_slug.empty? || src_slug == id_slug
|
|
294
|
+
|
|
295
|
+
Dir[File.join(store_dir,
|
|
296
|
+
"#{src_slug}.*.semantic.xml")].each do |f|
|
|
297
|
+
alias_name = f.sub(
|
|
298
|
+
%r{/#{Regexp.escape(src_slug)}.},
|
|
299
|
+
"/#{id_slug}.",
|
|
300
|
+
)
|
|
301
|
+
FileUtils.cp(f, alias_name)
|
|
253
302
|
end
|
|
254
303
|
end
|
|
255
304
|
end
|
|
@@ -313,21 +362,81 @@ module Suma
|
|
|
313
362
|
end
|
|
314
363
|
|
|
315
364
|
# Assemble a manifest of the stored stubs and reinflate them into the site.
|
|
365
|
+
# The original manifest tree is preserved (nested collections,
|
|
366
|
+
# attachments, per-member sectionsplit) with each member entry swapped
|
|
367
|
+
# for its stored stub; the render resolves paths against the
|
|
368
|
+
# reinflate manifest's own directory, so every path the render reads
|
|
369
|
+
# is absolutized against the collection root (suma#168, suma#169).
|
|
316
370
|
def reinflate(members)
|
|
317
371
|
reinf_dir = File.join(@output_directory, "_staged_reinflate")
|
|
318
372
|
FileUtils.mkdir_p(reinf_dir)
|
|
319
373
|
docrefs = stored_docrefs(members, reinf_dir)
|
|
374
|
+
write_reinflate_manifest(docrefs, reinf_dir)
|
|
375
|
+
run_render(manifest_path: File.join(reinf_dir, "reinflate.yml"),
|
|
376
|
+
opts: reinflate_render_options)
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
def write_reinflate_manifest(docrefs, reinf_dir)
|
|
380
|
+
rebuilt = with_stubbed_members(manifest["manifest"], docrefs, [0])
|
|
320
381
|
reinf_path = File.join(reinf_dir, "reinflate.yml")
|
|
321
382
|
File.write(reinf_path,
|
|
322
|
-
base_manifest.merge("manifest" =>
|
|
383
|
+
base_manifest.merge("manifest" => rebuilt).to_yaml)
|
|
323
384
|
Utils.log "[staged] reinflating #{docrefs.size} stored members -> " \
|
|
324
385
|
"#{@output_directory}"
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def reinflate_render_options
|
|
389
|
+
{
|
|
390
|
+
format: @formats,
|
|
391
|
+
output_folder: File.expand_path(@output_directory),
|
|
392
|
+
reinflate: true,
|
|
393
|
+
coverpage: @coverpage && File.expand_path(@coverpage, @config_dir),
|
|
394
|
+
compile: compile_flags,
|
|
395
|
+
}.compact
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
# Deep-copy the manifest tree, replacing each member entry (same
|
|
399
|
+
# predicate and pre-order as collection_members) with its stub docref,
|
|
400
|
+
# and absolutizing non-member filerefs (attachments) against the
|
|
401
|
+
# collection root.
|
|
402
|
+
def with_stubbed_members(node, docrefs, counter)
|
|
403
|
+
case node
|
|
404
|
+
when Array
|
|
405
|
+
node.map { |child| with_stubbed_members(child, docrefs, counter) }
|
|
406
|
+
when Hash
|
|
407
|
+
with_stubbed_members_hash(node, docrefs, counter)
|
|
408
|
+
else
|
|
409
|
+
node
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def with_stubbed_members_hash(node, docrefs, counter)
|
|
414
|
+
rewritten = node.dup
|
|
415
|
+
rewritten.each_key do |key|
|
|
416
|
+
rewritten[key] = rewrite_entry_value(key, rewritten[key], docrefs, counter)
|
|
417
|
+
end
|
|
418
|
+
stub_for(rewritten, docrefs, counter) || rewritten
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
def rewrite_entry_value(key, value, docrefs, counter)
|
|
422
|
+
if %w[file fileref].include?(key) && value &&
|
|
423
|
+
!Pathname.new(value).absolute?
|
|
424
|
+
File.expand_path(value, @config_dir)
|
|
425
|
+
else
|
|
426
|
+
with_stubbed_members(value, docrefs, counter)
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
def stub_for(node, docrefs, counter)
|
|
431
|
+
return unless member_entry?(node)
|
|
432
|
+
|
|
433
|
+
index = counter[0]
|
|
434
|
+
counter[0] += 1
|
|
435
|
+
docrefs[index]
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
def member_entry?(node)
|
|
439
|
+
(node["fileref"] || node["file"]) && !truthy?(node["attachment"])
|
|
331
440
|
end
|
|
332
441
|
|
|
333
442
|
def compile_flags
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.21
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: expressir
|