suma 0.5.20 → 0.5.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/suma/cli/build.rb +20 -1
- data/lib/suma/cli/core.rb +7 -0
- data/lib/suma/processor.rb +6 -1
- data/lib/suma/staged_collection_builder.rb +57 -8
- data/lib/suma/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d57c7866a336d71b91bae0745359a3ad66a5ff465eb2f456a27d74a3d901328b
|
|
4
|
+
data.tar.gz: 6c84fb67e536cc4df2fed9e6a75ad39700b3c6a8162d63a6f92779e370c52be7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9ccc76d252726749978bceee65caf1a75fc954f80b6c17652e373c6aa6b7c263d64ccb7f3e114accc4f66a564c54c1f7a9282d9914cd11ce2319fe50925ca64d
|
|
7
|
+
data.tar.gz: 623e14b1b637cf112bdd66d8d62ac9b54543b45498efc5117abb1f1aad2a76b5b582eaa49cc1ec66576de03d3069de05e41e1dde8d84c231ff7a1f8621247379
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -29,6 +29,23 @@ module Suma
|
|
|
29
29
|
"INDEXth member and skips reinflate, so " \
|
|
30
30
|
"disjoint shards can render concurrently " \
|
|
31
31
|
"into a shared store (requires --staged)"
|
|
32
|
+
option :staged_shard_isolate, type: :string,
|
|
33
|
+
desc: "Stage only the single member " \
|
|
34
|
+
"whose fileref contains this " \
|
|
35
|
+
"slug (e.g. iso-10303-104) and " \
|
|
36
|
+
"skip reinflate: gives a " \
|
|
37
|
+
"mega-member a build run of its " \
|
|
38
|
+
"own (requires --staged)"
|
|
39
|
+
option :staged_reinflate_shard, type: :string,
|
|
40
|
+
desc: "Reinflate only this shard of " \
|
|
41
|
+
"the collection as INDEX/TOTAL " \
|
|
42
|
+
"(e.g. 2/4): stage everything " \
|
|
43
|
+
"(warm store = skips), render " \
|
|
44
|
+
"the slice's members into the " \
|
|
45
|
+
"site, so parallel reinflate " \
|
|
46
|
+
"jobs over one store write " \
|
|
47
|
+
"disjoint sections (requires " \
|
|
48
|
+
"--staged)"
|
|
32
49
|
option :incremental, type: :boolean, default: false,
|
|
33
50
|
desc: "Re-render only changed schemas and their " \
|
|
34
51
|
"transitive dependents (state file defaults " \
|
|
@@ -86,7 +103,9 @@ module Suma
|
|
|
86
103
|
{ staged: options[:staged],
|
|
87
104
|
staged_workers: options[:staged_workers],
|
|
88
105
|
staged_batch_size: options[:staged_batch_size],
|
|
89
|
-
staged_shard: parse_shard(options[:staged_shard])
|
|
106
|
+
staged_shard: parse_shard(options[:staged_shard]),
|
|
107
|
+
staged_shard_isolate: options[:staged_shard_isolate],
|
|
108
|
+
staged_reinflate_shard: parse_shard(options[:staged_reinflate_shard]) }
|
|
90
109
|
end
|
|
91
110
|
|
|
92
111
|
def parse_shard(value)
|
data/lib/suma/cli/core.rb
CHANGED
|
@@ -37,6 +37,13 @@ module Suma
|
|
|
37
37
|
option :staged_shard, type: :string,
|
|
38
38
|
desc: "Render only this shard as INDEX/TOTAL" \
|
|
39
39
|
"(requires --staged)"
|
|
40
|
+
option :staged_shard_isolate, type: :string,
|
|
41
|
+
desc: "Stage only the member whose " \
|
|
42
|
+
"fileref contains this slug " \
|
|
43
|
+
"(requires --staged)"
|
|
44
|
+
option :staged_reinflate_shard, type: :string,
|
|
45
|
+
desc: "Reinflate only this shard as " \
|
|
46
|
+
"INDEX/TOTAL (requires --staged)"
|
|
40
47
|
option :incremental, type: :boolean, default: false,
|
|
41
48
|
desc: "Re-render only changed schemas and dependents"
|
|
42
49
|
option :state_file, type: :string,
|
data/lib/suma/processor.rb
CHANGED
|
@@ -14,7 +14,8 @@ module Suma
|
|
|
14
14
|
def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
|
|
15
15
|
output_directory: "_site", staged: false,
|
|
16
16
|
staged_workers: 1, staged_batch_size: nil,
|
|
17
|
-
staged_shard: nil,
|
|
17
|
+
staged_shard: nil, staged_shard_isolate: nil,
|
|
18
|
+
staged_reinflate_shard: nil,
|
|
18
19
|
incremental: false, state_file: nil, jobs: 1,
|
|
19
20
|
provenance_file: nil, sign_method: nil, sign_key: nil,
|
|
20
21
|
no_valid: false)
|
|
@@ -28,6 +29,8 @@ module Suma
|
|
|
28
29
|
@staged_workers = staged_workers
|
|
29
30
|
@staged_batch_size = staged_batch_size
|
|
30
31
|
@staged_shard = staged_shard
|
|
32
|
+
@staged_shard_isolate = staged_shard_isolate
|
|
33
|
+
@staged_reinflate_shard = staged_reinflate_shard
|
|
31
34
|
@incremental = incremental
|
|
32
35
|
@state_file = state_file
|
|
33
36
|
@jobs = jobs
|
|
@@ -190,6 +193,8 @@ module Suma
|
|
|
190
193
|
batch_size: @staged_batch_size ||
|
|
191
194
|
StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
|
|
192
195
|
shard: @staged_shard,
|
|
196
|
+
isolate: @staged_shard_isolate,
|
|
197
|
+
reinflate_shard: @staged_reinflate_shard,
|
|
193
198
|
).build
|
|
194
199
|
else
|
|
195
200
|
Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
|
|
@@ -49,12 +49,13 @@ module Suma
|
|
|
49
49
|
# entries, so children cannot race. +shard+ is an [index, total] pair:
|
|
50
50
|
# only members whose position satisfies +index % total+ are staged and
|
|
51
51
|
# reinflation is skipped, so disjoint shards of one collection can render
|
|
52
|
-
# concurrently in separate build runs into a shared store.
|
|
53
|
-
#
|
|
52
|
+
# concurrently in separate build runs into a shared store. +isolate+ is
|
|
53
|
+
# a fileref slug with the same shard semantics for a single member.
|
|
54
|
+
# (a data-declaring initializer)
|
|
54
55
|
def initialize(collection_config_path:, output_directory:, coverpage: nil,
|
|
55
56
|
store_dir: nil, formats: %i[xml html], no_valid: false,
|
|
56
57
|
workers: 1, batch_size: DEFAULT_BATCH_SIZE,
|
|
57
|
-
shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
58
|
+
shard: nil, isolate: nil, reinflate_shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
58
59
|
@config_path = collection_config_path
|
|
59
60
|
@config_dir = File.dirname(File.expand_path(collection_config_path))
|
|
60
61
|
@output_directory = output_directory
|
|
@@ -65,25 +66,53 @@ module Suma
|
|
|
65
66
|
@workers = [workers.to_i, 1].max
|
|
66
67
|
@batch_size = [batch_size.to_i, 1].max
|
|
67
68
|
@shard = shard
|
|
69
|
+
@isolate = isolate
|
|
70
|
+
@reinflate_shard = reinflate_shard
|
|
68
71
|
end
|
|
69
72
|
|
|
70
73
|
def build
|
|
71
74
|
FileUtils.mkdir_p(@output_directory)
|
|
72
75
|
FileUtils.mkdir_p(@store_dir)
|
|
73
76
|
members = collection_members
|
|
74
|
-
staged =
|
|
77
|
+
staged = staged_selection(members)
|
|
75
78
|
Utils.log shard_log(staged.size, members.size)
|
|
79
|
+
stage_members(staged)
|
|
80
|
+
return if @shard || @isolate
|
|
81
|
+
|
|
82
|
+
reinflate(reinflate_selection(members))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
# The staged slice of the collection: an isolate selection wins over a
|
|
88
|
+
# modulo shard when both are given.
|
|
89
|
+
def staged_selection(members)
|
|
90
|
+
@isolate ? isolate_members(members) : shard_members(members)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Render the staged members, serially or across the worker pool.
|
|
94
|
+
def stage_members(staged)
|
|
76
95
|
if @workers == 1
|
|
77
96
|
unstaged_members(staged).each_with_index { |member, i| stage_member(member, i) }
|
|
78
97
|
else
|
|
79
98
|
stage_parallel(staged)
|
|
80
99
|
end
|
|
81
|
-
return if @shard
|
|
82
|
-
|
|
83
|
-
reinflate(members)
|
|
84
100
|
end
|
|
85
101
|
|
|
86
|
-
|
|
102
|
+
# The reinflated slice: all members unless a reinflate shard is given,
|
|
103
|
+
# in which case only the members matching +index % total+ render into
|
|
104
|
+
# the output directory. Disjoint reinflate slices of one collection can
|
|
105
|
+
# run as parallel build jobs over the same store: each renders its own
|
|
106
|
+
# members' sections, cross-references resolve through the restored
|
|
107
|
+
# anchors of the whole store, and the written files are disjoint.
|
|
108
|
+
def reinflate_selection(members)
|
|
109
|
+
return members unless @reinflate_shard
|
|
110
|
+
|
|
111
|
+
index, total = @reinflate_shard
|
|
112
|
+
members.each_with_index.filter_map do |member, position|
|
|
113
|
+
member if (position % total) == index
|
|
114
|
+
end
|
|
115
|
+
end
|
|
87
116
|
|
|
88
117
|
# A shard stages the members whose position in the member order matches
|
|
89
118
|
# +index % total+; the full member list is unchanged.
|
|
@@ -96,9 +125,29 @@ module Suma
|
|
|
96
125
|
end
|
|
97
126
|
end
|
|
98
127
|
|
|
128
|
+
# Isolation stages the single member whose fileref path contains the
|
|
129
|
+
# +isolate+ slug (e.g. a mega-member that needs a whole runner to
|
|
130
|
+
# itself, away from sibling workers), and skips reinflation like a
|
|
131
|
+
# shard. Empty selection means the slug matched nothing, which is a
|
|
132
|
+
# caller error worth failing on rather than silently rendering
|
|
133
|
+
# nothing.
|
|
134
|
+
def isolate_members(members)
|
|
135
|
+
staged = members.select do |member|
|
|
136
|
+
File.basename(member["fileref"]).include?(@isolate)
|
|
137
|
+
end
|
|
138
|
+
return staged unless staged.empty?
|
|
139
|
+
|
|
140
|
+
raise ArgumentError,
|
|
141
|
+
"--staged-shard-isolate matched no member for #{@isolate.inspect}"
|
|
142
|
+
end
|
|
143
|
+
|
|
99
144
|
def shard_log(staged_count, total_count)
|
|
100
145
|
line = "[staged] #{total_count} members; workers: #{@workers}; " \
|
|
101
146
|
"batch size: #{@batch_size}; store: #{@store_dir}"
|
|
147
|
+
if @isolate
|
|
148
|
+
return "#{line}; isolate #{@isolate.inspect}: staging " \
|
|
149
|
+
"#{staged_count}, skipping reinflate"
|
|
150
|
+
end
|
|
102
151
|
return line unless @shard
|
|
103
152
|
|
|
104
153
|
"#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
|
data/lib/suma/version.rb
CHANGED