suma 0.5.17 → 0.5.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e5c15270fcd8076bab49e6fdf940d0f83fb2ad3ca4b3f274ac156eef6af5a8f0
4
- data.tar.gz: b33b2aae6fbb4322fe010643b65326dad5683c265e3907d78bcaa39ad2fd0ad6
3
+ metadata.gz: df03b41b1c8ad66d68f1f6bb711f9cc49396e1b9217b2819eef5f1a9ae98b997
4
+ data.tar.gz: 04f0d313dbe23ce6579b0c2736d264c7934dcc250a84c9e55f8bc39014e50a8e
5
5
  SHA512:
6
- metadata.gz: 93696359dfff9571f9705661a1ce6f40f8f78e4eb222dec2d34f4d7a030a6aa7962e091ed9d2688352ab80b31deb730989bedde47871d2c1a7c6e73fb656fffc
7
- data.tar.gz: 2557f73a80035a6b77f7b35d7289044e5792ee78e91d8edf26b2f293f24013e995991ef079a62a78ebd612d9dcfc13f6ddce8ba0d1b0f61411820138db3bbd96
6
+ metadata.gz: 1eb98f1e8d9cd8ead0c40db5867929d74a50d440cd964848a0fbafaac2b23e0ad0d484522cc98a1ea5afc9c4cf88137ed8670af848088059c37c0c718da2dfdc
7
+ data.tar.gz: 85bf0f0ea57ac912398b33264550a89f7b0118cac0b76f7a239cc33b49a78d39d9e535abdeb6a801022fb462dabb02727bbc38a055ee99b9f7ac7a88e50b8afe
@@ -23,6 +23,12 @@ module Suma
23
23
  option :staged_batch_size, type: :numeric,
24
24
  desc: "Members per staged render process " \
25
25
  "(default: 25)"
26
+ option :staged_shard, type: :string,
27
+ desc: "Render only this shard of the collection " \
28
+ "as INDEX/TOTAL (e.g. 2/8): stages every " \
29
+ "INDEXth member and skips reinflate, so " \
30
+ "disjoint shards can render concurrently " \
31
+ "into a shared store (requires --staged)"
26
32
  option :incremental, type: :boolean, default: false,
27
33
  desc: "Re-render only changed schemas and their " \
28
34
  "transitive dependents (state file defaults " \
@@ -79,7 +85,21 @@ module Suma
79
85
  def staged_options(options)
80
86
  { staged: options[:staged],
81
87
  staged_workers: options[:staged_workers],
82
- staged_batch_size: options[:staged_batch_size] }
88
+ staged_batch_size: options[:staged_batch_size],
89
+ staged_shard: parse_shard(options[:staged_shard]) }
90
+ end
91
+
92
+ def parse_shard(value)
93
+ return nil unless value
94
+
95
+ index, total = value.split("/", 2).map { |p| Integer(p, 10) }
96
+ unless total && index >= 0 && index < total
97
+ raise ArgumentError,
98
+ "--staged-shard expects INDEX/TOTAL with INDEX < TOTAL, " \
99
+ "got #{value.inspect}"
100
+ end
101
+
102
+ [index, total]
83
103
  end
84
104
 
85
105
  def log_error(error)
data/lib/suma/cli/core.rb CHANGED
@@ -34,6 +34,9 @@ module Suma
34
34
  desc: "Concurrent staged render processes"
35
35
  option :staged_batch_size, type: :numeric,
36
36
  desc: "Members per staged render process"
37
+ option :staged_shard, type: :string,
38
+ desc: "Render only this shard as INDEX/TOTAL" \
39
+ "(requires --staged)"
37
40
  option :incremental, type: :boolean, default: false,
38
41
  desc: "Re-render only changed schemas and dependents"
39
42
  option :state_file, type: :string,
@@ -14,6 +14,7 @@ module Suma
14
14
  def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
15
15
  output_directory: "_site", staged: false,
16
16
  staged_workers: 1, staged_batch_size: nil,
17
+ staged_shard: nil,
17
18
  incremental: false, state_file: nil, jobs: 1,
18
19
  provenance_file: nil, sign_method: nil, sign_key: nil,
19
20
  no_valid: false)
@@ -26,6 +27,7 @@ module Suma
26
27
  @staged = staged
27
28
  @staged_workers = staged_workers
28
29
  @staged_batch_size = staged_batch_size
30
+ @staged_shard = staged_shard
29
31
  @incremental = incremental
30
32
  @state_file = state_file
31
33
  @jobs = jobs
@@ -187,6 +189,7 @@ module Suma
187
189
  workers: @staged_workers,
188
190
  batch_size: @staged_batch_size ||
189
191
  StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
192
+ shard: @staged_shard,
190
193
  ).build
191
194
  else
192
195
  Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
@@ -45,10 +45,15 @@ module Suma
45
45
  # relative to its directory, so all per-member manifests are written there.
46
46
  # +workers+ > 1 runs that many renderer processes concurrently over batches
47
47
  # of +batch_size+ members; members write disjoint, identifier-keyed store
48
- # entries, so children cannot race. (a data-declaring initializer)
48
+ # entries, so children cannot race. +shard+ is an [index, total] pair:
49
+ # only members whose position satisfies +index % total+ are staged and
50
+ # reinflation is skipped, so disjoint shards of one collection can render
51
+ # concurrently in separate build runs into a shared store. (a
52
+ # data-declaring initializer)
49
53
  def initialize(collection_config_path:, output_directory:, coverpage: nil,
50
54
  store_dir: nil, formats: %i[xml html], no_valid: false,
51
- workers: 1, batch_size: DEFAULT_BATCH_SIZE) # rubocop:disable Metrics/ParameterLists
55
+ workers: 1, batch_size: DEFAULT_BATCH_SIZE,
56
+ shard: nil) # rubocop:disable Metrics/ParameterLists
52
57
  @config_path = collection_config_path
53
58
  @config_dir = File.dirname(File.expand_path(collection_config_path))
54
59
  @output_directory = output_directory
@@ -58,24 +63,47 @@ module Suma
58
63
  @no_valid = no_valid
59
64
  @workers = [workers.to_i, 1].max
60
65
  @batch_size = [batch_size.to_i, 1].max
66
+ @shard = shard
61
67
  end
62
68
 
63
69
  def build
64
70
  FileUtils.mkdir_p(@output_directory)
65
71
  FileUtils.mkdir_p(@store_dir)
66
72
  members = collection_members
67
- Utils.log "[staged] #{members.size} members; workers: #{@workers}; " \
68
- "batch size: #{@batch_size}; store: #{@store_dir}"
73
+ staged = shard_members(members)
74
+ Utils.log shard_log(staged.size, members.size)
69
75
  if @workers == 1
70
- members.each_with_index { |member, i| stage_member(member, i) }
76
+ staged.each_with_index { |member, i| stage_member(member, i) }
71
77
  else
72
- stage_parallel(members)
78
+ stage_parallel(staged)
73
79
  end
80
+ return if @shard
81
+
74
82
  reinflate(members)
75
83
  end
76
84
 
77
85
  private
78
86
 
87
+ # A shard stages the members whose position in the member order matches
88
+ # +index % total+; the full member list is unchanged.
89
+ def shard_members(members)
90
+ return members unless @shard
91
+
92
+ index, total = @shard
93
+ members.each_with_index.filter_map do |member, position|
94
+ member if (position % total) == index
95
+ end
96
+ end
97
+
98
+ def shard_log(staged_count, total_count)
99
+ line = "[staged] #{total_count} members; workers: #{@workers}; " \
100
+ "batch size: #{@batch_size}; store: #{@store_dir}"
101
+ return line unless @shard
102
+
103
+ "#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
104
+ "skipping reinflate"
105
+ end
106
+
79
107
  # Batch the members and run +@workers+ renderer child processes
80
108
  # concurrently. Each batch file carries fully-rendered single-member
81
109
  # manifests; children claim batches through O_EXCL markers, so a slow
@@ -140,7 +168,7 @@ module Suma
140
168
  # Metanorma boot is paid once per child, and GC runs between members to
141
169
  # bound the heap. Any render failure exits nonzero and fails the build.
142
170
  def child_script
143
- <<~RUBY
171
+ <<~'RUBY'
144
172
  require "yaml"
145
173
  require "fileutils"
146
174
  require "metanorma"
@@ -150,7 +178,7 @@ module Suma
150
178
  loop do
151
179
  idx = (0...count).find do |i|
152
180
  begin
153
- File.open(File.join(config_dir, "._suma_batch_\#{i}.claim"),
181
+ File.open(File.join(config_dir, "._suma_batch_#{i}.claim"),
154
182
  File::RDWR | File::CREAT | File::EXCL) {}
155
183
  true
156
184
  rescue Errno::EEXIST
@@ -160,11 +188,11 @@ module Suma
160
188
  break unless idx
161
189
 
162
190
  manifests = Marshal.load(File.binread(File.join(
163
- config_dir, "._suma_batch_\#{idx}.bin")))
191
+ config_dir, "._suma_batch_#{idx}.bin")))
164
192
  manifests.each_with_index do |manifest, j|
165
193
  one_path = File.join(config_dir,
166
- "._suma_staged_b\#{idx}_\#{j}.yml")
167
- out = File.join(out_dir, "._staged_out_b\#{idx}_\#{j}")
194
+ "._suma_staged_b#{idx}_#{j}.yml")
195
+ out = File.join(out_dir, "._staged_out_b#{idx}_#{j}")
168
196
  t0 = Time.now
169
197
  begin
170
198
  File.write(one_path, YAML.dump(manifest))
@@ -208,10 +236,10 @@ module Suma
208
236
  next if src_slug.empty? || src_slug == id_slug
209
237
 
210
238
  Dir[File.join(store_dir,
211
- "\#{src_slug}.*.semantic.xml")].each do |f|
239
+ "#{src_slug}.*.semantic.xml")].each do |f|
212
240
  alias_name = f.sub(
213
- %r{/\#{Regexp.escape(src_slug)}.},
214
- "/\#{id_slug}.",
241
+ %r{/#{Regexp.escape(src_slug)}.},
242
+ "/#{id_slug}.",
215
243
  )
216
244
  FileUtils.cp(f, alias_name)
217
245
  end
@@ -219,7 +247,7 @@ module Suma
219
247
  end
220
248
  produced.each do |f|
221
249
  if (m = f.match(/\A(.*).([0-9a-f]+).semantic.xml\z/))
222
- alias_name = "\#{id_slug}.\#{m[2]}.semantic.xml"
250
+ alias_name = "#{id_slug}.#{m[2]}.semantic.xml"
223
251
  alias_path = File.join(store_dir, alias_name)
224
252
  FileUtils.cp(File.join(store_dir, f), alias_path)
225
253
  end
data/lib/suma/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Suma
4
- VERSION = "0.5.17"
4
+ VERSION = "0.5.19"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: suma
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.17
4
+ version: 0.5.19
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2026-10-31 00:00:00.000000000 Z
11
+ date: 2026-11-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: expressir