suma 0.5.16 → 0.5.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/suma/cli/build.rb +21 -1
- data/lib/suma/cli/core.rb +3 -0
- data/lib/suma/processor.rb +3 -0
- data/lib/suma/staged_collection_builder.rb +59 -7
- data/lib/suma/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7aecfe621f260b1496a90b9f070404447b68e438fbeeae524de10c64cbfe55e7
|
|
4
|
+
data.tar.gz: 40809dcb0598a3d4a0b0c38ef1bb991aadddebd94c9a86d6c838c7dd6a6bad0c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1b8edfa1f0c4c1339145fc73cbaf30828cd741f2160043fb900ac3bf0102b2681cdf019984783a7f07919cbf0066d49cb17ab26773463e251fd0ea6b715fffca
|
|
7
|
+
data.tar.gz: 0df164617379d06f748db5e28436cf94ded01fe9e77532e5bc3a64a3faf15a567cb2a5de0ca5eac9f04cd07e72ee1ac926fdb084daa52479e31b9515e03e55be
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -23,6 +23,12 @@ module Suma
|
|
|
23
23
|
option :staged_batch_size, type: :numeric,
|
|
24
24
|
desc: "Members per staged render process " \
|
|
25
25
|
"(default: 25)"
|
|
26
|
+
option :staged_shard, type: :string,
|
|
27
|
+
desc: "Render only this shard of the collection " \
|
|
28
|
+
"as INDEX/TOTAL (e.g. 2/8): stages every " \
|
|
29
|
+
"INDEXth member and skips reinflate, so " \
|
|
30
|
+
"disjoint shards can render concurrently " \
|
|
31
|
+
"into a shared store (requires --staged)"
|
|
26
32
|
option :incremental, type: :boolean, default: false,
|
|
27
33
|
desc: "Re-render only changed schemas and their " \
|
|
28
34
|
"transitive dependents (state file defaults " \
|
|
@@ -79,7 +85,21 @@ module Suma
|
|
|
79
85
|
def staged_options(options)
|
|
80
86
|
{ staged: options[:staged],
|
|
81
87
|
staged_workers: options[:staged_workers],
|
|
82
|
-
staged_batch_size: options[:staged_batch_size]
|
|
88
|
+
staged_batch_size: options[:staged_batch_size],
|
|
89
|
+
staged_shard: parse_shard(options[:staged_shard]) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def parse_shard(value)
|
|
93
|
+
return nil unless value
|
|
94
|
+
|
|
95
|
+
index, total = value.split("/", 2).map { |p| Integer(p, 10) }
|
|
96
|
+
unless total && index >= 0 && index < total
|
|
97
|
+
raise ArgumentError,
|
|
98
|
+
"--staged-shard expects INDEX/TOTAL with INDEX < TOTAL, " \
|
|
99
|
+
"got #{value.inspect}"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
[index, total]
|
|
83
103
|
end
|
|
84
104
|
|
|
85
105
|
def log_error(error)
|
data/lib/suma/cli/core.rb
CHANGED
|
@@ -34,6 +34,9 @@ module Suma
|
|
|
34
34
|
desc: "Concurrent staged render processes"
|
|
35
35
|
option :staged_batch_size, type: :numeric,
|
|
36
36
|
desc: "Members per staged render process"
|
|
37
|
+
option :staged_shard, type: :string,
|
|
38
|
+
desc: "Render only this shard as INDEX/TOTAL" \
|
|
39
|
+
"(requires --staged)"
|
|
37
40
|
option :incremental, type: :boolean, default: false,
|
|
38
41
|
desc: "Re-render only changed schemas and dependents"
|
|
39
42
|
option :state_file, type: :string,
|
data/lib/suma/processor.rb
CHANGED
|
@@ -14,6 +14,7 @@ module Suma
|
|
|
14
14
|
def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
|
|
15
15
|
output_directory: "_site", staged: false,
|
|
16
16
|
staged_workers: 1, staged_batch_size: nil,
|
|
17
|
+
staged_shard: nil,
|
|
17
18
|
incremental: false, state_file: nil, jobs: 1,
|
|
18
19
|
provenance_file: nil, sign_method: nil, sign_key: nil,
|
|
19
20
|
no_valid: false)
|
|
@@ -26,6 +27,7 @@ module Suma
|
|
|
26
27
|
@staged = staged
|
|
27
28
|
@staged_workers = staged_workers
|
|
28
29
|
@staged_batch_size = staged_batch_size
|
|
30
|
+
@staged_shard = staged_shard
|
|
29
31
|
@incremental = incremental
|
|
30
32
|
@state_file = state_file
|
|
31
33
|
@jobs = jobs
|
|
@@ -187,6 +189,7 @@ module Suma
|
|
|
187
189
|
workers: @staged_workers,
|
|
188
190
|
batch_size: @staged_batch_size ||
|
|
189
191
|
StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
|
|
192
|
+
shard: @staged_shard,
|
|
190
193
|
).build
|
|
191
194
|
else
|
|
192
195
|
Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
|
|
@@ -45,10 +45,15 @@ module Suma
|
|
|
45
45
|
# relative to its directory, so all per-member manifests are written there.
|
|
46
46
|
# +workers+ > 1 runs that many renderer processes concurrently over batches
|
|
47
47
|
# of +batch_size+ members; members write disjoint, identifier-keyed store
|
|
48
|
-
# entries, so children cannot race.
|
|
48
|
+
# entries, so children cannot race. +shard+ is an [index, total] pair:
|
|
49
|
+
# only members whose position satisfies +index % total+ are staged and
|
|
50
|
+
# reinflation is skipped, so disjoint shards of one collection can render
|
|
51
|
+
# concurrently in separate build runs into a shared store. (a
|
|
52
|
+
# data-declaring initializer)
|
|
49
53
|
def initialize(collection_config_path:, output_directory:, coverpage: nil,
|
|
50
54
|
store_dir: nil, formats: %i[xml html], no_valid: false,
|
|
51
|
-
workers: 1, batch_size: DEFAULT_BATCH_SIZE
|
|
55
|
+
workers: 1, batch_size: DEFAULT_BATCH_SIZE,
|
|
56
|
+
shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
52
57
|
@config_path = collection_config_path
|
|
53
58
|
@config_dir = File.dirname(File.expand_path(collection_config_path))
|
|
54
59
|
@output_directory = output_directory
|
|
@@ -58,24 +63,47 @@ module Suma
|
|
|
58
63
|
@no_valid = no_valid
|
|
59
64
|
@workers = [workers.to_i, 1].max
|
|
60
65
|
@batch_size = [batch_size.to_i, 1].max
|
|
66
|
+
@shard = shard
|
|
61
67
|
end
|
|
62
68
|
|
|
63
69
|
def build
|
|
64
70
|
FileUtils.mkdir_p(@output_directory)
|
|
65
71
|
FileUtils.mkdir_p(@store_dir)
|
|
66
72
|
members = collection_members
|
|
67
|
-
|
|
68
|
-
|
|
73
|
+
staged = shard_members(members)
|
|
74
|
+
Utils.log shard_log(staged.size, members.size)
|
|
69
75
|
if @workers == 1
|
|
70
|
-
|
|
76
|
+
staged.each_with_index { |member, i| stage_member(member, i) }
|
|
71
77
|
else
|
|
72
|
-
stage_parallel(
|
|
78
|
+
stage_parallel(staged)
|
|
73
79
|
end
|
|
80
|
+
return if @shard
|
|
81
|
+
|
|
74
82
|
reinflate(members)
|
|
75
83
|
end
|
|
76
84
|
|
|
77
85
|
private
|
|
78
86
|
|
|
87
|
+
# A shard stages the members whose position in the member order matches
|
|
88
|
+
# +index % total+; the full member list is unchanged.
|
|
89
|
+
def shard_members(members)
|
|
90
|
+
return members unless @shard
|
|
91
|
+
|
|
92
|
+
index, total = @shard
|
|
93
|
+
members.each_with_index.filter_map do |member, position|
|
|
94
|
+
member if (position % total) == index
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def shard_log(staged_count, total_count)
|
|
99
|
+
line = "[staged] #{total_count} members; workers: #{@workers}; " \
|
|
100
|
+
"batch size: #{@batch_size}; store: #{@store_dir}"
|
|
101
|
+
return line unless @shard
|
|
102
|
+
|
|
103
|
+
"#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
|
|
104
|
+
"skipping reinflate"
|
|
105
|
+
end
|
|
106
|
+
|
|
79
107
|
# Batch the members and run +@workers+ renderer child processes
|
|
80
108
|
# concurrently. Each batch file carries fully-rendered single-member
|
|
81
109
|
# manifests; children claim batches through O_EXCL markers, so a slow
|
|
@@ -184,15 +212,39 @@ module Suma
|
|
|
184
212
|
.select { |f| f.end_with?(".semantic.xml") }
|
|
185
213
|
compile = { install_fonts: false }
|
|
186
214
|
compile[:novalid] = true if no_valid == "1"
|
|
187
|
-
Metanorma::Collection.parse(one_path)
|
|
215
|
+
collection = Metanorma::Collection.parse(one_path)
|
|
216
|
+
collection.render(
|
|
188
217
|
format: [:xml], output_folder: out,
|
|
189
218
|
preserve_unresolved: true, artifact_store_dir: store_dir,
|
|
190
219
|
compile: compile,
|
|
191
220
|
)
|
|
192
221
|
unless id_slug.empty?
|
|
222
|
+
# A warm artefact cache makes the render a full cache hit:
|
|
223
|
+
# no new artefact files appear, so a produced-files delta
|
|
224
|
+
# never fires. The collection's own keys are what the
|
|
225
|
+
# renderer stored under, so alias every semantic artefact
|
|
226
|
+
# keyed by them to the docref slug as well.
|
|
227
|
+
store_keys = (collection.bibdatas || {}).keys
|
|
193
228
|
produced = Dir.children(store_dir).select do |f|
|
|
194
229
|
f.end_with?(".semantic.xml")
|
|
195
230
|
end - semantics_before
|
|
231
|
+
if produced.empty?
|
|
232
|
+
store_keys.each do |key|
|
|
233
|
+
src_slug = key.to_s
|
|
234
|
+
.gsub(/[^A-Za-z0-9._-]+/, "-").gsub(/-{2,}/, "-")
|
|
235
|
+
.gsub(/\A-|-\z/, "")
|
|
236
|
+
next if src_slug.empty? || src_slug == id_slug
|
|
237
|
+
|
|
238
|
+
Dir[File.join(store_dir,
|
|
239
|
+
"\#{src_slug}.*.semantic.xml")].each do |f|
|
|
240
|
+
alias_name = f.sub(
|
|
241
|
+
%r{/\#{Regexp.escape(src_slug)}.},
|
|
242
|
+
"/\#{id_slug}.",
|
|
243
|
+
)
|
|
244
|
+
FileUtils.cp(f, alias_name)
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
end
|
|
196
248
|
produced.each do |f|
|
|
197
249
|
if (m = f.match(/\A(.*).([0-9a-f]+).semantic.xml\z/))
|
|
198
250
|
alias_name = "\#{id_slug}.\#{m[2]}.semantic.xml"
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.18
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: expressir
|