suma 0.5.17 → 0.5.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/suma/cli/build.rb +21 -1
- data/lib/suma/cli/core.rb +3 -0
- data/lib/suma/processor.rb +3 -0
- data/lib/suma/staged_collection_builder.rb +34 -6
- data/lib/suma/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7aecfe621f260b1496a90b9f070404447b68e438fbeeae524de10c64cbfe55e7
|
|
4
|
+
data.tar.gz: 40809dcb0598a3d4a0b0c38ef1bb991aadddebd94c9a86d6c838c7dd6a6bad0c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1b8edfa1f0c4c1339145fc73cbaf30828cd741f2160043fb900ac3bf0102b2681cdf019984783a7f07919cbf0066d49cb17ab26773463e251fd0ea6b715fffca
|
|
7
|
+
data.tar.gz: 0df164617379d06f748db5e28436cf94ded01fe9e77532e5bc3a64a3faf15a567cb2a5de0ca5eac9f04cd07e72ee1ac926fdb084daa52479e31b9515e03e55be
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -23,6 +23,12 @@ module Suma
|
|
|
23
23
|
option :staged_batch_size, type: :numeric,
|
|
24
24
|
desc: "Members per staged render process " \
|
|
25
25
|
"(default: 25)"
|
|
26
|
+
option :staged_shard, type: :string,
|
|
27
|
+
desc: "Render only this shard of the collection " \
|
|
28
|
+
"as INDEX/TOTAL (e.g. 2/8): stages every " \
|
|
29
|
+
"INDEXth member and skips reinflate, so " \
|
|
30
|
+
"disjoint shards can render concurrently " \
|
|
31
|
+
"into a shared store (requires --staged)"
|
|
26
32
|
option :incremental, type: :boolean, default: false,
|
|
27
33
|
desc: "Re-render only changed schemas and their " \
|
|
28
34
|
"transitive dependents (state file defaults " \
|
|
@@ -79,7 +85,21 @@ module Suma
|
|
|
79
85
|
def staged_options(options)
|
|
80
86
|
{ staged: options[:staged],
|
|
81
87
|
staged_workers: options[:staged_workers],
|
|
82
|
-
staged_batch_size: options[:staged_batch_size]
|
|
88
|
+
staged_batch_size: options[:staged_batch_size],
|
|
89
|
+
staged_shard: parse_shard(options[:staged_shard]) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def parse_shard(value)
|
|
93
|
+
return nil unless value
|
|
94
|
+
|
|
95
|
+
index, total = value.split("/", 2).map { |p| Integer(p, 10) }
|
|
96
|
+
unless total && index >= 0 && index < total
|
|
97
|
+
raise ArgumentError,
|
|
98
|
+
"--staged-shard expects INDEX/TOTAL with INDEX < TOTAL, " \
|
|
99
|
+
"got #{value.inspect}"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
[index, total]
|
|
83
103
|
end
|
|
84
104
|
|
|
85
105
|
def log_error(error)
|
data/lib/suma/cli/core.rb
CHANGED
|
@@ -34,6 +34,9 @@ module Suma
|
|
|
34
34
|
desc: "Concurrent staged render processes"
|
|
35
35
|
option :staged_batch_size, type: :numeric,
|
|
36
36
|
desc: "Members per staged render process"
|
|
37
|
+
option :staged_shard, type: :string,
|
|
38
|
+
desc: "Render only this shard as INDEX/TOTAL" \
|
|
39
|
+
"(requires --staged)"
|
|
37
40
|
option :incremental, type: :boolean, default: false,
|
|
38
41
|
desc: "Re-render only changed schemas and dependents"
|
|
39
42
|
option :state_file, type: :string,
|
data/lib/suma/processor.rb
CHANGED
|
@@ -14,6 +14,7 @@ module Suma
|
|
|
14
14
|
def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
|
|
15
15
|
output_directory: "_site", staged: false,
|
|
16
16
|
staged_workers: 1, staged_batch_size: nil,
|
|
17
|
+
staged_shard: nil,
|
|
17
18
|
incremental: false, state_file: nil, jobs: 1,
|
|
18
19
|
provenance_file: nil, sign_method: nil, sign_key: nil,
|
|
19
20
|
no_valid: false)
|
|
@@ -26,6 +27,7 @@ module Suma
|
|
|
26
27
|
@staged = staged
|
|
27
28
|
@staged_workers = staged_workers
|
|
28
29
|
@staged_batch_size = staged_batch_size
|
|
30
|
+
@staged_shard = staged_shard
|
|
29
31
|
@incremental = incremental
|
|
30
32
|
@state_file = state_file
|
|
31
33
|
@jobs = jobs
|
|
@@ -187,6 +189,7 @@ module Suma
|
|
|
187
189
|
workers: @staged_workers,
|
|
188
190
|
batch_size: @staged_batch_size ||
|
|
189
191
|
StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
|
|
192
|
+
shard: @staged_shard,
|
|
190
193
|
).build
|
|
191
194
|
else
|
|
192
195
|
Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
|
|
@@ -45,10 +45,15 @@ module Suma
|
|
|
45
45
|
# relative to its directory, so all per-member manifests are written there.
|
|
46
46
|
# +workers+ > 1 runs that many renderer processes concurrently over batches
|
|
47
47
|
# of +batch_size+ members; members write disjoint, identifier-keyed store
|
|
48
|
-
# entries, so children cannot race.
|
|
48
|
+
# entries, so children cannot race. +shard+ is an [index, total] pair:
|
|
49
|
+
# only members whose position satisfies +index % total+ are staged and
|
|
50
|
+
# reinflation is skipped, so disjoint shards of one collection can render
|
|
51
|
+
# concurrently in separate build runs into a shared store. (a
|
|
52
|
+
# data-declaring initializer)
|
|
49
53
|
def initialize(collection_config_path:, output_directory:, coverpage: nil,
|
|
50
54
|
store_dir: nil, formats: %i[xml html], no_valid: false,
|
|
51
|
-
workers: 1, batch_size: DEFAULT_BATCH_SIZE
|
|
55
|
+
workers: 1, batch_size: DEFAULT_BATCH_SIZE,
|
|
56
|
+
shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
52
57
|
@config_path = collection_config_path
|
|
53
58
|
@config_dir = File.dirname(File.expand_path(collection_config_path))
|
|
54
59
|
@output_directory = output_directory
|
|
@@ -58,24 +63,47 @@ module Suma
|
|
|
58
63
|
@no_valid = no_valid
|
|
59
64
|
@workers = [workers.to_i, 1].max
|
|
60
65
|
@batch_size = [batch_size.to_i, 1].max
|
|
66
|
+
@shard = shard
|
|
61
67
|
end
|
|
62
68
|
|
|
63
69
|
def build
|
|
64
70
|
FileUtils.mkdir_p(@output_directory)
|
|
65
71
|
FileUtils.mkdir_p(@store_dir)
|
|
66
72
|
members = collection_members
|
|
67
|
-
|
|
68
|
-
|
|
73
|
+
staged = shard_members(members)
|
|
74
|
+
Utils.log shard_log(staged.size, members.size)
|
|
69
75
|
if @workers == 1
|
|
70
|
-
|
|
76
|
+
staged.each_with_index { |member, i| stage_member(member, i) }
|
|
71
77
|
else
|
|
72
|
-
stage_parallel(
|
|
78
|
+
stage_parallel(staged)
|
|
73
79
|
end
|
|
80
|
+
return if @shard
|
|
81
|
+
|
|
74
82
|
reinflate(members)
|
|
75
83
|
end
|
|
76
84
|
|
|
77
85
|
private
|
|
78
86
|
|
|
87
|
+
# A shard stages the members whose position in the member order matches
|
|
88
|
+
# +index % total+; the full member list is unchanged.
|
|
89
|
+
def shard_members(members)
|
|
90
|
+
return members unless @shard
|
|
91
|
+
|
|
92
|
+
index, total = @shard
|
|
93
|
+
members.each_with_index.filter_map do |member, position|
|
|
94
|
+
member if (position % total) == index
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def shard_log(staged_count, total_count)
|
|
99
|
+
line = "[staged] #{total_count} members; workers: #{@workers}; " \
|
|
100
|
+
"batch size: #{@batch_size}; store: #{@store_dir}"
|
|
101
|
+
return line unless @shard
|
|
102
|
+
|
|
103
|
+
"#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
|
|
104
|
+
"skipping reinflate"
|
|
105
|
+
end
|
|
106
|
+
|
|
79
107
|
# Batch the members and run +@workers+ renderer child processes
|
|
80
108
|
# concurrently. Each batch file carries fully-rendered single-member
|
|
81
109
|
# manifests; children claim batches through O_EXCL markers, so a slow
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.18
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: expressir
|