suma 0.5.17 → 0.5.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/suma/cli/build.rb +21 -1
- data/lib/suma/cli/core.rb +3 -0
- data/lib/suma/processor.rb +3 -0
- data/lib/suma/staged_collection_builder.rb +43 -15
- data/lib/suma/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: df03b41b1c8ad66d68f1f6bb711f9cc49396e1b9217b2819eef5f1a9ae98b997
|
|
4
|
+
data.tar.gz: 04f0d313dbe23ce6579b0c2736d264c7934dcc250a84c9e55f8bc39014e50a8e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1eb98f1e8d9cd8ead0c40db5867929d74a50d440cd964848a0fbafaac2b23e0ad0d484522cc98a1ea5afc9c4cf88137ed8670af848088059c37c0c718da2dfdc
|
|
7
|
+
data.tar.gz: 85bf0f0ea57ac912398b33264550a89f7b0118cac0b76f7a239cc33b49a78d39d9e535abdeb6a801022fb462dabb02727bbc38a055ee99b9f7ac7a88e50b8afe
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -23,6 +23,12 @@ module Suma
|
|
|
23
23
|
option :staged_batch_size, type: :numeric,
|
|
24
24
|
desc: "Members per staged render process " \
|
|
25
25
|
"(default: 25)"
|
|
26
|
+
option :staged_shard, type: :string,
|
|
27
|
+
desc: "Render only this shard of the collection " \
|
|
28
|
+
"as INDEX/TOTAL (e.g. 2/8): stages every " \
|
|
29
|
+
"INDEXth member and skips reinflate, so " \
|
|
30
|
+
"disjoint shards can render concurrently " \
|
|
31
|
+
"into a shared store (requires --staged)"
|
|
26
32
|
option :incremental, type: :boolean, default: false,
|
|
27
33
|
desc: "Re-render only changed schemas and their " \
|
|
28
34
|
"transitive dependents (state file defaults " \
|
|
@@ -79,7 +85,21 @@ module Suma
|
|
|
79
85
|
def staged_options(options)
|
|
80
86
|
{ staged: options[:staged],
|
|
81
87
|
staged_workers: options[:staged_workers],
|
|
82
|
-
staged_batch_size: options[:staged_batch_size]
|
|
88
|
+
staged_batch_size: options[:staged_batch_size],
|
|
89
|
+
staged_shard: parse_shard(options[:staged_shard]) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def parse_shard(value)
|
|
93
|
+
return nil unless value
|
|
94
|
+
|
|
95
|
+
index, total = value.split("/", 2).map { |p| Integer(p, 10) }
|
|
96
|
+
unless total && index >= 0 && index < total
|
|
97
|
+
raise ArgumentError,
|
|
98
|
+
"--staged-shard expects INDEX/TOTAL with INDEX < TOTAL, " \
|
|
99
|
+
"got #{value.inspect}"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
[index, total]
|
|
83
103
|
end
|
|
84
104
|
|
|
85
105
|
def log_error(error)
|
data/lib/suma/cli/core.rb
CHANGED
|
@@ -34,6 +34,9 @@ module Suma
|
|
|
34
34
|
desc: "Concurrent staged render processes"
|
|
35
35
|
option :staged_batch_size, type: :numeric,
|
|
36
36
|
desc: "Members per staged render process"
|
|
37
|
+
option :staged_shard, type: :string,
|
|
38
|
+
desc: "Render only this shard as INDEX/TOTAL" \
|
|
39
|
+
"(requires --staged)"
|
|
37
40
|
option :incremental, type: :boolean, default: false,
|
|
38
41
|
desc: "Re-render only changed schemas and dependents"
|
|
39
42
|
option :state_file, type: :string,
|
data/lib/suma/processor.rb
CHANGED
|
@@ -14,6 +14,7 @@ module Suma
|
|
|
14
14
|
def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
|
|
15
15
|
output_directory: "_site", staged: false,
|
|
16
16
|
staged_workers: 1, staged_batch_size: nil,
|
|
17
|
+
staged_shard: nil,
|
|
17
18
|
incremental: false, state_file: nil, jobs: 1,
|
|
18
19
|
provenance_file: nil, sign_method: nil, sign_key: nil,
|
|
19
20
|
no_valid: false)
|
|
@@ -26,6 +27,7 @@ module Suma
|
|
|
26
27
|
@staged = staged
|
|
27
28
|
@staged_workers = staged_workers
|
|
28
29
|
@staged_batch_size = staged_batch_size
|
|
30
|
+
@staged_shard = staged_shard
|
|
29
31
|
@incremental = incremental
|
|
30
32
|
@state_file = state_file
|
|
31
33
|
@jobs = jobs
|
|
@@ -187,6 +189,7 @@ module Suma
|
|
|
187
189
|
workers: @staged_workers,
|
|
188
190
|
batch_size: @staged_batch_size ||
|
|
189
191
|
StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
|
|
192
|
+
shard: @staged_shard,
|
|
190
193
|
).build
|
|
191
194
|
else
|
|
192
195
|
Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
|
|
@@ -45,10 +45,15 @@ module Suma
|
|
|
45
45
|
# relative to its directory, so all per-member manifests are written there.
|
|
46
46
|
# +workers+ > 1 runs that many renderer processes concurrently over batches
|
|
47
47
|
# of +batch_size+ members; members write disjoint, identifier-keyed store
|
|
48
|
-
# entries, so children cannot race.
|
|
48
|
+
# entries, so children cannot race. +shard+ is an [index, total] pair:
|
|
49
|
+
# only members whose position satisfies +index % total+ are staged and
|
|
50
|
+
# reinflation is skipped, so disjoint shards of one collection can render
|
|
51
|
+
# concurrently in separate build runs into a shared store. (a
|
|
52
|
+
# data-declaring initializer)
|
|
49
53
|
def initialize(collection_config_path:, output_directory:, coverpage: nil,
|
|
50
54
|
store_dir: nil, formats: %i[xml html], no_valid: false,
|
|
51
|
-
workers: 1, batch_size: DEFAULT_BATCH_SIZE
|
|
55
|
+
workers: 1, batch_size: DEFAULT_BATCH_SIZE,
|
|
56
|
+
shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
52
57
|
@config_path = collection_config_path
|
|
53
58
|
@config_dir = File.dirname(File.expand_path(collection_config_path))
|
|
54
59
|
@output_directory = output_directory
|
|
@@ -58,24 +63,47 @@ module Suma
|
|
|
58
63
|
@no_valid = no_valid
|
|
59
64
|
@workers = [workers.to_i, 1].max
|
|
60
65
|
@batch_size = [batch_size.to_i, 1].max
|
|
66
|
+
@shard = shard
|
|
61
67
|
end
|
|
62
68
|
|
|
63
69
|
def build
|
|
64
70
|
FileUtils.mkdir_p(@output_directory)
|
|
65
71
|
FileUtils.mkdir_p(@store_dir)
|
|
66
72
|
members = collection_members
|
|
67
|
-
|
|
68
|
-
|
|
73
|
+
staged = shard_members(members)
|
|
74
|
+
Utils.log shard_log(staged.size, members.size)
|
|
69
75
|
if @workers == 1
|
|
70
|
-
|
|
76
|
+
staged.each_with_index { |member, i| stage_member(member, i) }
|
|
71
77
|
else
|
|
72
|
-
stage_parallel(
|
|
78
|
+
stage_parallel(staged)
|
|
73
79
|
end
|
|
80
|
+
return if @shard
|
|
81
|
+
|
|
74
82
|
reinflate(members)
|
|
75
83
|
end
|
|
76
84
|
|
|
77
85
|
private
|
|
78
86
|
|
|
87
|
+
# A shard stages the members whose position in the member order matches
|
|
88
|
+
# +index % total+; the full member list is unchanged.
|
|
89
|
+
def shard_members(members)
|
|
90
|
+
return members unless @shard
|
|
91
|
+
|
|
92
|
+
index, total = @shard
|
|
93
|
+
members.each_with_index.filter_map do |member, position|
|
|
94
|
+
member if (position % total) == index
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def shard_log(staged_count, total_count)
|
|
99
|
+
line = "[staged] #{total_count} members; workers: #{@workers}; " \
|
|
100
|
+
"batch size: #{@batch_size}; store: #{@store_dir}"
|
|
101
|
+
return line unless @shard
|
|
102
|
+
|
|
103
|
+
"#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
|
|
104
|
+
"skipping reinflate"
|
|
105
|
+
end
|
|
106
|
+
|
|
79
107
|
# Batch the members and run +@workers+ renderer child processes
|
|
80
108
|
# concurrently. Each batch file carries fully-rendered single-member
|
|
81
109
|
# manifests; children claim batches through O_EXCL markers, so a slow
|
|
@@ -140,7 +168,7 @@ module Suma
|
|
|
140
168
|
# Metanorma boot is paid once per child, and GC runs between members to
|
|
141
169
|
# bound the heap. Any render failure exits nonzero and fails the build.
|
|
142
170
|
def child_script
|
|
143
|
-
<<~RUBY
|
|
171
|
+
<<~'RUBY'
|
|
144
172
|
require "yaml"
|
|
145
173
|
require "fileutils"
|
|
146
174
|
require "metanorma"
|
|
@@ -150,7 +178,7 @@ module Suma
|
|
|
150
178
|
loop do
|
|
151
179
|
idx = (0...count).find do |i|
|
|
152
180
|
begin
|
|
153
|
-
File.open(File.join(config_dir, "._suma_batch_
|
|
181
|
+
File.open(File.join(config_dir, "._suma_batch_#{i}.claim"),
|
|
154
182
|
File::RDWR | File::CREAT | File::EXCL) {}
|
|
155
183
|
true
|
|
156
184
|
rescue Errno::EEXIST
|
|
@@ -160,11 +188,11 @@ module Suma
|
|
|
160
188
|
break unless idx
|
|
161
189
|
|
|
162
190
|
manifests = Marshal.load(File.binread(File.join(
|
|
163
|
-
config_dir, "._suma_batch_
|
|
191
|
+
config_dir, "._suma_batch_#{idx}.bin")))
|
|
164
192
|
manifests.each_with_index do |manifest, j|
|
|
165
193
|
one_path = File.join(config_dir,
|
|
166
|
-
"._suma_staged_b
|
|
167
|
-
out = File.join(out_dir, "._staged_out_b
|
|
194
|
+
"._suma_staged_b#{idx}_#{j}.yml")
|
|
195
|
+
out = File.join(out_dir, "._staged_out_b#{idx}_#{j}")
|
|
168
196
|
t0 = Time.now
|
|
169
197
|
begin
|
|
170
198
|
File.write(one_path, YAML.dump(manifest))
|
|
@@ -208,10 +236,10 @@ module Suma
|
|
|
208
236
|
next if src_slug.empty? || src_slug == id_slug
|
|
209
237
|
|
|
210
238
|
Dir[File.join(store_dir,
|
|
211
|
-
"
|
|
239
|
+
"#{src_slug}.*.semantic.xml")].each do |f|
|
|
212
240
|
alias_name = f.sub(
|
|
213
|
-
%r{
|
|
214
|
-
"
|
|
241
|
+
%r{/#{Regexp.escape(src_slug)}.},
|
|
242
|
+
"/#{id_slug}.",
|
|
215
243
|
)
|
|
216
244
|
FileUtils.cp(f, alias_name)
|
|
217
245
|
end
|
|
@@ -219,7 +247,7 @@ module Suma
|
|
|
219
247
|
end
|
|
220
248
|
produced.each do |f|
|
|
221
249
|
if (m = f.match(/\A(.*).([0-9a-f]+).semantic.xml\z/))
|
|
222
|
-
alias_name = "
|
|
250
|
+
alias_name = "#{id_slug}.#{m[2]}.semantic.xml"
|
|
223
251
|
alias_path = File.join(store_dir, alias_name)
|
|
224
252
|
FileUtils.cp(File.join(store_dir, f), alias_path)
|
|
225
253
|
end
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.19
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: expressir
|