suma 0.5.20 → 0.5.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/suma/cli/build.rb +20 -1
- data/lib/suma/cli/core.rb +7 -0
- data/lib/suma/processor.rb +6 -1
- data/lib/suma/staged_collection_builder.rb +72 -14
- data/lib/suma/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0c89b77873c034c41b9b7660ec2909985af7436ae422672843e9317f4f8fc341
|
|
4
|
+
data.tar.gz: 28f6d6ae1d113a0237f54dc156ace158430df12b98761858ee01518811ae2ea9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ca38065478452e1aba7f79c52fe0a5f28a3de20d1301fb9153fa0f63f44cf9f0cf096248f4f1d1a03ce6d3e001eb47151f8bf848b5ada46f857b697c20bba1ff
|
|
7
|
+
data.tar.gz: 977e635c1e99e0a119d8c48f72f4ef763492dfe54ab3561da56485a8becb08c0cb5bc64960dc2dd5322d9a1e6e1d5fe66e39cdb1f6f6717fc11092e0201207b9
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -29,6 +29,23 @@ module Suma
|
|
|
29
29
|
"INDEXth member and skips reinflate, so " \
|
|
30
30
|
"disjoint shards can render concurrently " \
|
|
31
31
|
"into a shared store (requires --staged)"
|
|
32
|
+
option :staged_shard_isolate, type: :string,
|
|
33
|
+
desc: "Stage only the single member " \
|
|
34
|
+
"whose fileref contains this " \
|
|
35
|
+
"slug (e.g. iso-10303-104) and " \
|
|
36
|
+
"skip reinflate: gives a " \
|
|
37
|
+
"mega-member a build run of its " \
|
|
38
|
+
"own (requires --staged)"
|
|
39
|
+
option :staged_reinflate_shard, type: :string,
|
|
40
|
+
desc: "Reinflate only this shard of " \
|
|
41
|
+
"the collection as INDEX/TOTAL " \
|
|
42
|
+
"(e.g. 2/4): stage everything " \
|
|
43
|
+
"(warm store = skips), render " \
|
|
44
|
+
"the slice's members into the " \
|
|
45
|
+
"site, so parallel reinflate " \
|
|
46
|
+
"jobs over one store write " \
|
|
47
|
+
"disjoint sections (requires " \
|
|
48
|
+
"--staged)"
|
|
32
49
|
option :incremental, type: :boolean, default: false,
|
|
33
50
|
desc: "Re-render only changed schemas and their " \
|
|
34
51
|
"transitive dependents (state file defaults " \
|
|
@@ -86,7 +103,9 @@ module Suma
|
|
|
86
103
|
{ staged: options[:staged],
|
|
87
104
|
staged_workers: options[:staged_workers],
|
|
88
105
|
staged_batch_size: options[:staged_batch_size],
|
|
89
|
-
staged_shard: parse_shard(options[:staged_shard])
|
|
106
|
+
staged_shard: parse_shard(options[:staged_shard]),
|
|
107
|
+
staged_shard_isolate: options[:staged_shard_isolate],
|
|
108
|
+
staged_reinflate_shard: parse_shard(options[:staged_reinflate_shard]) }
|
|
90
109
|
end
|
|
91
110
|
|
|
92
111
|
def parse_shard(value)
|
data/lib/suma/cli/core.rb
CHANGED
|
@@ -37,6 +37,13 @@ module Suma
|
|
|
37
37
|
option :staged_shard, type: :string,
|
|
38
38
|
desc: "Render only this shard as INDEX/TOTAL" \
|
|
39
39
|
"(requires --staged)"
|
|
40
|
+
option :staged_shard_isolate, type: :string,
|
|
41
|
+
desc: "Stage only the member whose " \
|
|
42
|
+
"fileref contains this slug " \
|
|
43
|
+
"(requires --staged)"
|
|
44
|
+
option :staged_reinflate_shard, type: :string,
|
|
45
|
+
desc: "Reinflate only this shard as " \
|
|
46
|
+
"INDEX/TOTAL (requires --staged)"
|
|
40
47
|
option :incremental, type: :boolean, default: false,
|
|
41
48
|
desc: "Re-render only changed schemas and dependents"
|
|
42
49
|
option :state_file, type: :string,
|
data/lib/suma/processor.rb
CHANGED
|
@@ -14,7 +14,8 @@ module Suma
|
|
|
14
14
|
def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
|
|
15
15
|
output_directory: "_site", staged: false,
|
|
16
16
|
staged_workers: 1, staged_batch_size: nil,
|
|
17
|
-
staged_shard: nil,
|
|
17
|
+
staged_shard: nil, staged_shard_isolate: nil,
|
|
18
|
+
staged_reinflate_shard: nil,
|
|
18
19
|
incremental: false, state_file: nil, jobs: 1,
|
|
19
20
|
provenance_file: nil, sign_method: nil, sign_key: nil,
|
|
20
21
|
no_valid: false)
|
|
@@ -28,6 +29,8 @@ module Suma
|
|
|
28
29
|
@staged_workers = staged_workers
|
|
29
30
|
@staged_batch_size = staged_batch_size
|
|
30
31
|
@staged_shard = staged_shard
|
|
32
|
+
@staged_shard_isolate = staged_shard_isolate
|
|
33
|
+
@staged_reinflate_shard = staged_reinflate_shard
|
|
31
34
|
@incremental = incremental
|
|
32
35
|
@state_file = state_file
|
|
33
36
|
@jobs = jobs
|
|
@@ -190,6 +193,8 @@ module Suma
|
|
|
190
193
|
batch_size: @staged_batch_size ||
|
|
191
194
|
StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
|
|
192
195
|
shard: @staged_shard,
|
|
196
|
+
isolate: @staged_shard_isolate,
|
|
197
|
+
reinflate_shard: @staged_reinflate_shard,
|
|
193
198
|
).build
|
|
194
199
|
else
|
|
195
200
|
Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
|
|
@@ -49,12 +49,13 @@ module Suma
|
|
|
49
49
|
# entries, so children cannot race. +shard+ is an [index, total] pair:
|
|
50
50
|
# only members whose position satisfies +index % total+ are staged and
|
|
51
51
|
# reinflation is skipped, so disjoint shards of one collection can render
|
|
52
|
-
# concurrently in separate build runs into a shared store.
|
|
53
|
-
#
|
|
52
|
+
# concurrently in separate build runs into a shared store. +isolate+ is
|
|
53
|
+
# a fileref slug with the same shard semantics for a single member.
|
|
54
|
+
# (a data-declaring initializer)
|
|
54
55
|
def initialize(collection_config_path:, output_directory:, coverpage: nil,
|
|
55
56
|
store_dir: nil, formats: %i[xml html], no_valid: false,
|
|
56
57
|
workers: 1, batch_size: DEFAULT_BATCH_SIZE,
|
|
57
|
-
shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
58
|
+
shard: nil, isolate: nil, reinflate_shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
58
59
|
@config_path = collection_config_path
|
|
59
60
|
@config_dir = File.dirname(File.expand_path(collection_config_path))
|
|
60
61
|
@output_directory = output_directory
|
|
@@ -65,25 +66,53 @@ module Suma
|
|
|
65
66
|
@workers = [workers.to_i, 1].max
|
|
66
67
|
@batch_size = [batch_size.to_i, 1].max
|
|
67
68
|
@shard = shard
|
|
69
|
+
@isolate = isolate
|
|
70
|
+
@reinflate_shard = reinflate_shard
|
|
68
71
|
end
|
|
69
72
|
|
|
70
73
|
def build
|
|
71
74
|
FileUtils.mkdir_p(@output_directory)
|
|
72
75
|
FileUtils.mkdir_p(@store_dir)
|
|
73
76
|
members = collection_members
|
|
74
|
-
staged =
|
|
77
|
+
staged = staged_selection(members)
|
|
75
78
|
Utils.log shard_log(staged.size, members.size)
|
|
79
|
+
stage_members(staged)
|
|
80
|
+
return if @shard || @isolate
|
|
81
|
+
|
|
82
|
+
reinflate(reinflate_selection(members), members)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
# The staged slice of the collection: an isolate selection wins over a
|
|
88
|
+
# modulo shard when both are given.
|
|
89
|
+
def staged_selection(members)
|
|
90
|
+
@isolate ? isolate_members(members) : shard_members(members)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Render the staged members, serially or across the worker pool.
|
|
94
|
+
def stage_members(staged)
|
|
76
95
|
if @workers == 1
|
|
77
96
|
unstaged_members(staged).each_with_index { |member, i| stage_member(member, i) }
|
|
78
97
|
else
|
|
79
98
|
stage_parallel(staged)
|
|
80
99
|
end
|
|
81
|
-
return if @shard
|
|
82
|
-
|
|
83
|
-
reinflate(members)
|
|
84
100
|
end
|
|
85
101
|
|
|
86
|
-
|
|
102
|
+
# The reinflated slice: all members unless a reinflate shard is given,
|
|
103
|
+
# in which case only the members matching +index % total+ render into
|
|
104
|
+
# the output directory. Disjoint reinflate slices of one collection can
|
|
105
|
+
# run as parallel build jobs over the same store: each renders its own
|
|
106
|
+
# members' sections, cross-references resolve through the restored
|
|
107
|
+
# anchors of the whole store, and the written files are disjoint.
|
|
108
|
+
def reinflate_selection(members)
|
|
109
|
+
return members unless @reinflate_shard
|
|
110
|
+
|
|
111
|
+
index, total = @reinflate_shard
|
|
112
|
+
members.each_with_index.filter_map do |member, position|
|
|
113
|
+
member if (position % total) == index
|
|
114
|
+
end
|
|
115
|
+
end
|
|
87
116
|
|
|
88
117
|
# A shard stages the members whose position in the member order matches
|
|
89
118
|
# +index % total+; the full member list is unchanged.
|
|
@@ -96,9 +125,29 @@ module Suma
|
|
|
96
125
|
end
|
|
97
126
|
end
|
|
98
127
|
|
|
128
|
+
# Isolation stages the single member whose fileref path contains the
|
|
129
|
+
# +isolate+ slug (e.g. a mega-member that needs a whole runner to
|
|
130
|
+
# itself, away from sibling workers), and skips reinflation like a
|
|
131
|
+
# shard. Empty selection means the slug matched nothing, which is a
|
|
132
|
+
# caller error worth failing on rather than silently rendering
|
|
133
|
+
# nothing.
|
|
134
|
+
def isolate_members(members)
|
|
135
|
+
staged = members.select do |member|
|
|
136
|
+
File.basename(member["fileref"]).include?(@isolate)
|
|
137
|
+
end
|
|
138
|
+
return staged unless staged.empty?
|
|
139
|
+
|
|
140
|
+
raise ArgumentError,
|
|
141
|
+
"--staged-shard-isolate matched no member for #{@isolate.inspect}"
|
|
142
|
+
end
|
|
143
|
+
|
|
99
144
|
def shard_log(staged_count, total_count)
|
|
100
145
|
line = "[staged] #{total_count} members; workers: #{@workers}; " \
|
|
101
146
|
"batch size: #{@batch_size}; store: #{@store_dir}"
|
|
147
|
+
if @isolate
|
|
148
|
+
return "#{line}; isolate #{@isolate.inspect}: staging " \
|
|
149
|
+
"#{staged_count}, skipping reinflate"
|
|
150
|
+
end
|
|
102
151
|
return line unless @shard
|
|
103
152
|
|
|
104
153
|
"#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
|
|
@@ -318,13 +367,18 @@ module Suma
|
|
|
318
367
|
# for its stored stub; the render resolves paths against the
|
|
319
368
|
# reinflate manifest's own directory, so every path the render reads
|
|
320
369
|
# is absolutized against the collection root (suma#168, suma#169).
|
|
321
|
-
def reinflate(
|
|
370
|
+
def reinflate(render_members, all_members = render_members)
|
|
322
371
|
reinf_dir = File.join(@output_directory, "_staged_reinflate")
|
|
323
372
|
FileUtils.mkdir_p(reinf_dir)
|
|
324
|
-
|
|
373
|
+
# Stub every member: the render passes read each member's stored
|
|
374
|
+
# file for the anchors and cross-references, whatever this run is
|
|
375
|
+
# going to render. A reinflate shard additionally restricts the
|
|
376
|
+
# rendered members; sibling runs over the same store render the
|
|
377
|
+
# rest.
|
|
378
|
+
docrefs = stored_docrefs(all_members, reinf_dir)
|
|
325
379
|
write_reinflate_manifest(docrefs, reinf_dir)
|
|
326
380
|
run_render(manifest_path: File.join(reinf_dir, "reinflate.yml"),
|
|
327
|
-
opts: reinflate_render_options)
|
|
381
|
+
opts: reinflate_render_options(render_members))
|
|
328
382
|
end
|
|
329
383
|
|
|
330
384
|
def write_reinflate_manifest(docrefs, reinf_dir)
|
|
@@ -336,14 +390,18 @@ module Suma
|
|
|
336
390
|
"#{@output_directory}"
|
|
337
391
|
end
|
|
338
392
|
|
|
339
|
-
def reinflate_render_options
|
|
340
|
-
{
|
|
393
|
+
def reinflate_render_options(render_members = nil)
|
|
394
|
+
opts = {
|
|
341
395
|
format: @formats,
|
|
342
396
|
output_folder: File.expand_path(@output_directory),
|
|
343
397
|
reinflate: true,
|
|
344
398
|
coverpage: @coverpage && File.expand_path(@coverpage, @config_dir),
|
|
345
399
|
compile: compile_flags,
|
|
346
|
-
}
|
|
400
|
+
}
|
|
401
|
+
if @reinflate_shard && render_members
|
|
402
|
+
opts[:only_members] = render_members.filter_map { |m| m["identifier"] }
|
|
403
|
+
end
|
|
404
|
+
opts.compact
|
|
347
405
|
end
|
|
348
406
|
|
|
349
407
|
# Deep-copy the manifest tree, replacing each member entry (same
|
data/lib/suma/version.rb
CHANGED