suma 0.5.20 → 0.5.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 467b9bcc21844daaac8a68fdb76b15b9e165b6fd2243d469282f44d15f69cb97
4
- data.tar.gz: 4714dc0a4e2b9b8f63f4cdb5a88db33401662633f15fb7669c3c18908dc77492
3
+ metadata.gz: d57c7866a336d71b91bae0745359a3ad66a5ff465eb2f456a27d74a3d901328b
4
+ data.tar.gz: 6c84fb67e536cc4df2fed9e6a75ad39700b3c6a8162d63a6f92779e370c52be7
5
5
  SHA512:
6
- metadata.gz: 4a2a42ab57f4d4cec318dd6232e604ae9081402d66372437abd3854efa435ad87fddd43e91fd0da24156584bfc652174b36930cc7d30b00fb727857b9cd3e66a
7
- data.tar.gz: 69822cae107459997e8ebb80499fc6f8058f8e396949a33e8971b2cc6802ffbabe1f221f51c3b132b5c32170f965920363c5f61da3d6f7d3ae49e27a1a5b0ecb
6
+ metadata.gz: 9ccc76d252726749978bceee65caf1a75fc954f80b6c17652e373c6aa6b7c263d64ccb7f3e114accc4f66a564c54c1f7a9282d9914cd11ce2319fe50925ca64d
7
+ data.tar.gz: 623e14b1b637cf112bdd66d8d62ac9b54543b45498efc5117abb1f1aad2a76b5b582eaa49cc1ec66576de03d3069de05e41e1dde8d84c231ff7a1f8621247379
@@ -29,6 +29,23 @@ module Suma
29
29
  "INDEXth member and skips reinflate, so " \
30
30
  "disjoint shards can render concurrently " \
31
31
  "into a shared store (requires --staged)"
32
+ option :staged_shard_isolate, type: :string,
33
+ desc: "Stage only the single member " \
34
+ "whose fileref contains this " \
35
+ "slug (e.g. iso-10303-104) and " \
36
+ "skip reinflate: gives a " \
37
+ "mega-member a build run of its " \
38
+ "own (requires --staged)"
39
+ option :staged_reinflate_shard, type: :string,
40
+ desc: "Reinflate only this shard of " \
41
+ "the collection as INDEX/TOTAL " \
42
+ "(e.g. 2/4): stage everything " \
43
+ "(warm store = skips), render " \
44
+ "the slice's members into the " \
45
+ "site, so parallel reinflate " \
46
+ "jobs over one store write " \
47
+ "disjoint sections (requires " \
48
+ "--staged)"
32
49
  option :incremental, type: :boolean, default: false,
33
50
  desc: "Re-render only changed schemas and their " \
34
51
  "transitive dependents (state file defaults " \
@@ -86,7 +103,9 @@ module Suma
86
103
  { staged: options[:staged],
87
104
  staged_workers: options[:staged_workers],
88
105
  staged_batch_size: options[:staged_batch_size],
89
- staged_shard: parse_shard(options[:staged_shard]) }
106
+ staged_shard: parse_shard(options[:staged_shard]),
107
+ staged_shard_isolate: options[:staged_shard_isolate],
108
+ staged_reinflate_shard: parse_shard(options[:staged_reinflate_shard]) }
90
109
  end
91
110
 
92
111
  def parse_shard(value)
data/lib/suma/cli/core.rb CHANGED
@@ -37,6 +37,13 @@ module Suma
37
37
  option :staged_shard, type: :string,
38
38
  desc: "Render only this shard as INDEX/TOTAL" \
39
39
  "(requires --staged)"
40
+ option :staged_shard_isolate, type: :string,
41
+ desc: "Stage only the member whose " \
42
+ "fileref contains this slug " \
43
+ "(requires --staged)"
44
+ option :staged_reinflate_shard, type: :string,
45
+ desc: "Reinflate only this shard as " \
46
+ "INDEX/TOTAL (requires --staged)"
40
47
  option :incremental, type: :boolean, default: false,
41
48
  desc: "Re-render only changed schemas and dependents"
42
49
  option :state_file, type: :string,
@@ -14,7 +14,8 @@ module Suma
14
14
  def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
15
15
  output_directory: "_site", staged: false,
16
16
  staged_workers: 1, staged_batch_size: nil,
17
- staged_shard: nil,
17
+ staged_shard: nil, staged_shard_isolate: nil,
18
+ staged_reinflate_shard: nil,
18
19
  incremental: false, state_file: nil, jobs: 1,
19
20
  provenance_file: nil, sign_method: nil, sign_key: nil,
20
21
  no_valid: false)
@@ -28,6 +29,8 @@ module Suma
28
29
  @staged_workers = staged_workers
29
30
  @staged_batch_size = staged_batch_size
30
31
  @staged_shard = staged_shard
32
+ @staged_shard_isolate = staged_shard_isolate
33
+ @staged_reinflate_shard = staged_reinflate_shard
31
34
  @incremental = incremental
32
35
  @state_file = state_file
33
36
  @jobs = jobs
@@ -190,6 +193,8 @@ module Suma
190
193
  batch_size: @staged_batch_size ||
191
194
  StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
192
195
  shard: @staged_shard,
196
+ isolate: @staged_shard_isolate,
197
+ reinflate_shard: @staged_reinflate_shard,
193
198
  ).build
194
199
  else
195
200
  Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
@@ -49,12 +49,13 @@ module Suma
49
49
  # entries, so children cannot race. +shard+ is an [index, total] pair:
50
50
  # only members whose position satisfies +index % total+ are staged and
51
51
  # reinflation is skipped, so disjoint shards of one collection can render
52
- # concurrently in separate build runs into a shared store. (a
53
- # data-declaring initializer)
52
+ # concurrently in separate build runs into a shared store. +isolate+ is
53
+ # a fileref slug with the same shard semantics for a single member.
54
+ # (a data-declaring initializer)
54
55
  def initialize(collection_config_path:, output_directory:, coverpage: nil,
55
56
  store_dir: nil, formats: %i[xml html], no_valid: false,
56
57
  workers: 1, batch_size: DEFAULT_BATCH_SIZE,
57
- shard: nil) # rubocop:disable Metrics/ParameterLists
58
+ shard: nil, isolate: nil, reinflate_shard: nil) # rubocop:disable Metrics/ParameterLists
58
59
  @config_path = collection_config_path
59
60
  @config_dir = File.dirname(File.expand_path(collection_config_path))
60
61
  @output_directory = output_directory
@@ -65,25 +66,53 @@ module Suma
65
66
  @workers = [workers.to_i, 1].max
66
67
  @batch_size = [batch_size.to_i, 1].max
67
68
  @shard = shard
69
+ @isolate = isolate
70
+ @reinflate_shard = reinflate_shard
68
71
  end
69
72
 
70
73
  def build
71
74
  FileUtils.mkdir_p(@output_directory)
72
75
  FileUtils.mkdir_p(@store_dir)
73
76
  members = collection_members
74
- staged = shard_members(members)
77
+ staged = staged_selection(members)
75
78
  Utils.log shard_log(staged.size, members.size)
79
+ stage_members(staged)
80
+ return if @shard || @isolate
81
+
82
+ reinflate(reinflate_selection(members))
83
+ end
84
+
85
+ private
86
+
87
+ # The staged slice of the collection: an isolate selection wins over a
88
+ # modulo shard when both are given.
89
+ def staged_selection(members)
90
+ @isolate ? isolate_members(members) : shard_members(members)
91
+ end
92
+
93
+ # Render the staged members, serially or across the worker pool.
94
+ def stage_members(staged)
76
95
  if @workers == 1
77
96
  unstaged_members(staged).each_with_index { |member, i| stage_member(member, i) }
78
97
  else
79
98
  stage_parallel(staged)
80
99
  end
81
- return if @shard
82
-
83
- reinflate(members)
84
100
  end
85
101
 
86
- private
102
+ # The reinflated slice: all members unless a reinflate shard is given,
103
+ # in which case only the members matching +index % total+ render into
104
+ # the output directory. Disjoint reinflate slices of one collection can
105
+ # run as parallel build jobs over the same store: each renders its own
106
+ # members' sections, cross-references resolve through the restored
107
+ # anchors of the whole store, and the written files are disjoint.
108
+ def reinflate_selection(members)
109
+ return members unless @reinflate_shard
110
+
111
+ index, total = @reinflate_shard
112
+ members.each_with_index.filter_map do |member, position|
113
+ member if (position % total) == index
114
+ end
115
+ end
87
116
 
88
117
  # A shard stages the members whose position in the member order matches
89
118
  # +index % total+; the full member list is unchanged.
@@ -96,9 +125,29 @@ module Suma
96
125
  end
97
126
  end
98
127
 
128
+ # Isolation stages the single member whose fileref path contains the
129
+ # +isolate+ slug (e.g. a mega-member that needs a whole runner to
130
+ # itself, away from sibling workers), and skips reinflation like a
131
+ # shard. Empty selection means the slug matched nothing, which is a
132
+ # caller error worth failing on rather than silently rendering
133
+ # nothing.
134
+ def isolate_members(members)
135
+ staged = members.select do |member|
136
+ File.basename(member["fileref"]).include?(@isolate)
137
+ end
138
+ return staged unless staged.empty?
139
+
140
+ raise ArgumentError,
141
+ "--staged-shard-isolate matched no member for #{@isolate.inspect}"
142
+ end
143
+
99
144
  def shard_log(staged_count, total_count)
100
145
  line = "[staged] #{total_count} members; workers: #{@workers}; " \
101
146
  "batch size: #{@batch_size}; store: #{@store_dir}"
147
+ if @isolate
148
+ return "#{line}; isolate #{@isolate.inspect}: staging " \
149
+ "#{staged_count}, skipping reinflate"
150
+ end
102
151
  return line unless @shard
103
152
 
104
153
  "#{line}; shard #{@shard[0]}/#{@shard[1]}: staging #{staged_count}, " \
data/lib/suma/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Suma
4
- VERSION = "0.5.20"
4
+ VERSION = "0.5.21"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: suma
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.20
4
+ version: 0.5.21
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.