binpacker 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,7 @@ module Binpacker
7
7
  def initialize(worker_count, tty: $stdout.tty?)
8
8
  @worker_count = worker_count
9
9
  @tty = tty
10
- @workers = Array.new(worker_count) { { done: 0, total: 0, file: "", elapsed: 0.0 } }
10
+ @workers = Array.new(worker_count) { { done: 0, total: 0, file: '', elapsed: 0.0 } }
11
11
  @start = Time.now
12
12
  @last_ci_output = Time.at(Time.now.to_f - CI_INTERVAL)
13
13
  @lines_written = 0
@@ -32,8 +32,9 @@ module Binpacker
32
32
  end
33
33
  end
34
34
 
35
- def finish(worker_stats = [])
35
+ def finish(_worker_stats = [])
36
36
  return unless @tty
37
+
37
38
  redraw
38
39
  $stdout.puts
39
40
  end
@@ -42,9 +43,10 @@ module Binpacker
42
43
  active = worker_stats.reject { |s| s[:files] == 0 && s[:examples] == 0 }
43
44
  return if active.empty?
44
45
 
45
- active.each_with_index do |s, i|
46
+ worker_stats.each_with_index do |s, wid|
47
+ next if s[:files] == 0 && s[:examples] == 0
48
+
46
49
  t = format_time(s[:total_time])
47
- wid = worker_stats.index(s)
48
50
  $stdout.puts " Worker #{wid}: #{s[:files]} tests, #{t} | #{s[:examples]} examples, #{s[:passed]} passed"
49
51
  end
50
52
  total_tests = active.sum { |s| s[:files] }
@@ -55,7 +57,7 @@ module Binpacker
55
57
  max_dev = times.map { |t| (t - mean).abs }.max
56
58
  dev_pct = mean > 0 ? (max_dev / mean * 100).round(1) : 0
57
59
 
58
- $stdout.puts " ──"
60
+ $stdout.puts ' ──'
59
61
  $stdout.puts " Total: #{total_tests} tests, #{format_time(total_time)} | #{total_examples} examples"
60
62
  $stdout.puts " Balance: max deviation #{format_time(max_dev)} (#{dev_pct}%)"
61
63
  end
@@ -67,7 +69,7 @@ module Binpacker
67
69
  clear_lines
68
70
  @workers.each_with_index do |w, i|
69
71
  bar = build_bar(w[:done], w[:total])
70
- status = w[:total] > 0 && w[:done] >= w[:total] ? "done" : w[:file][-50..] || ""
72
+ status = w[:total] > 0 && w[:done] >= w[:total] ? 'done' : w[:file][-50..] || ''
71
73
  $stdout.puts format_line(i, bar, w[:done], w[:total], status, w[:elapsed])
72
74
  end
73
75
  @lines_written = @worker_count
@@ -76,13 +78,15 @@ module Binpacker
76
78
 
77
79
  def clear_lines
78
80
  return if @lines_written == 0
81
+
79
82
  @lines_written.times do
80
83
  $stdout.print "\033[A\033[K"
81
84
  end
82
85
  end
83
86
 
84
87
  def build_bar(done, total)
85
- return "[----------]" if total == 0
88
+ return '[----------]' if total == 0
89
+
86
90
  width = 10
87
91
  filled = (done.to_f / total * width).round
88
92
  "[#{'█' * filled}#{'░' * (width - filled)}]"
@@ -96,10 +100,11 @@ module Binpacker
96
100
  def periodic_output
97
101
  now = Time.now
98
102
  return if now - @last_ci_output < CI_INTERVAL
103
+
99
104
  @last_ci_output = now
100
105
 
101
106
  parts = @workers.map.with_index do |w, i|
102
- ratio = w[:total] > 0 ? "#{w[:done]}/#{w[:total]}" : "0/?"
107
+ ratio = w[:total] > 0 ? "#{w[:done]}/#{w[:total]}" : '0/?'
103
108
  "W#{i}: #{ratio}"
104
109
  end
105
110
  elapsed = (now - @start).round(1)
@@ -108,7 +113,8 @@ module Binpacker
108
113
  end
109
114
 
110
115
  def format_time(seconds)
111
- return " 0.0s" if seconds < 0.001
116
+ return ' 0.0s' if seconds < 0.001
117
+
112
118
  m = (seconds / 60).floor
113
119
  s = (seconds % 60).round(1)
114
120
  m > 0 ? "#{m}m#{s.to_s.rjust(4, '0')}s" : "#{s.to_s.rjust(5)}s"
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "json"
3
+ require 'json'
4
4
 
5
5
  module Binpacker
6
6
  # Builds the machine-readable Run report: predicted versus actual per-Worker
@@ -9,13 +9,17 @@ module Binpacker
9
9
  SCHEMA = 1
10
10
  DRIFT_LIMIT = 10
11
11
 
12
- def initialize(profile:, algorithm:, predicted_loads:, worker_stats:, all_timings:, timings:)
12
+ def initialize(profile:, algorithm:, predicted_loads:, worker_stats:, all_timings:, timings:,
13
+ shard: nil, discovered: nil, selected: nil)
13
14
  @profile = profile
14
15
  @algorithm = algorithm
15
16
  @predicted_loads = predicted_loads
16
17
  @worker_stats = worker_stats
17
18
  @all_timings = all_timings
18
19
  @timings = timings
20
+ @shard = shard
21
+ @discovered = discovered
22
+ @selected = selected
19
23
  end
20
24
 
21
25
  def to_h
@@ -26,6 +30,7 @@ module Binpacker
26
30
  schema: SCHEMA,
27
31
  profile: @profile,
28
32
  algorithm: @algorithm,
33
+ shard: shard_section,
29
34
  worker_count: @worker_stats.size,
30
35
  predicted_makespan: round(predicted.max || 0.0),
31
36
  actual_makespan: round(actual.max || 0.0),
@@ -44,6 +49,29 @@ module Binpacker
44
49
 
45
50
  private
46
51
 
52
+ # The audit trail for a sharded run, and the reason `discovered` is recorded at all.
53
+ #
54
+ # Shards never talk to each other: each computes the same N-way partition and trusts the others to have
55
+ # computed it identically. They do so only while they agree on the timing data the partition is cut
56
+ # from, which in CI means every shard restoring the same timing cache. A shard that restores a
57
+ # different one — a cache miss where its siblings hit — partitions differently, and the failure is
58
+ # silent: tests land in no shard at all and the build stays green.
59
+ #
60
+ # `discovered` is the whole-suite count before slicing, so it agrees across shards that see the same
61
+ # repository. `selected` is this shard's slice. Summing `selected` over a matrix's reports and
62
+ # comparing to the shared `discovered` turns that silent skip into a failure —
63
+ # `binpacker shards-check` does exactly that.
64
+ def shard_section
65
+ return nil unless @shard
66
+
67
+ {
68
+ index: @shard.index,
69
+ total: @shard.total,
70
+ discovered_tests: @discovered,
71
+ selected_tests: @selected
72
+ }
73
+ end
74
+
47
75
  def workers
48
76
  @worker_stats.map.with_index do |s, i|
49
77
  {
@@ -84,13 +112,15 @@ module Binpacker
84
112
  end
85
113
 
86
114
  def normalize_file(file)
87
- file.to_s.sub(%r{\A\./}, "")
115
+ file.to_s.sub(%r{\A\./}, '')
88
116
  end
89
117
 
90
118
  def deviation_pct(loads)
91
119
  return 0.0 if loads.empty?
120
+
92
121
  mean = loads.sum / loads.size
93
122
  return 0.0 unless mean.positive?
123
+
94
124
  max_dev = loads.map { |t| (t - mean).abs }.max
95
125
  round(max_dev / mean * 100)
96
126
  end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Binpacker
4
+ # One slice of the suite, for splitting a run across independent machines.
5
+ #
6
+ # Workers divide a suite across the cores of ONE machine and share its wall clock; a shard divides it
7
+ # across machines that have no wall clock in common. So the two compose rather than compete: a CI matrix
8
+ # of N jobs each passing `--shard k/N` runs `worker_count` workers over its own slice, and the suite's
9
+ # wall time becomes roughly the slowest shard rather than the whole.
10
+ #
11
+ # The slice is cut by the SAME weight-balanced partitioner that assigns work to workers, over the same
12
+ # measured timings — shards are just a coarser bin-packing of the same problem, so an equal-count split
13
+ # would balance no better than round-robin. That also makes the cut deterministic: given one timing file,
14
+ # every shard computes the identical N-way partition and takes only its own bin, so no shard needs to
15
+ # know what the others decided, and every test lands in exactly one shard.
16
+ class Shard
17
+ attr_reader :index, :total
18
+
19
+ # Parses the `k/n` form used by `--shard` and BINPACKER_SHARD. `k` is 1-based, so a matrix can pass its
20
+ # own 1-based job number straight through.
21
+ #
22
+ # @param spec [String, nil]
23
+ # @return [Shard, nil] nil when `spec` is nil or empty, i.e. an unsharded run.
24
+ def self.parse(spec)
25
+ return nil if spec.nil? || spec.to_s.strip.empty?
26
+
27
+ match = %r{\A\s*(\d+)\s*/\s*(\d+)\s*\z}.match(spec.to_s)
28
+ raise ConfigError, "invalid shard #{spec.inspect}: expected the form K/N, e.g. 1/3" unless match
29
+
30
+ new(index: Integer(match[1]), total: Integer(match[2]))
31
+ end
32
+
33
+ def initialize(index:, total:)
34
+ raise ConfigError, "shard count must be at least 1, got #{total}" if total < 1
35
+ raise ConfigError, "shard index must be between 1 and #{total}, got #{index}" unless (1..total).cover?(index)
36
+
37
+ @index = index
38
+ @total = total
39
+ freeze
40
+ end
41
+
42
+ def to_s
43
+ "#{index}/#{total}"
44
+ end
45
+
46
+ # Whole-suite runs still construct no Shard, so this is only ever true for an explicit `--shard 1/1`.
47
+ def whole_suite?
48
+ total == 1
49
+ end
50
+
51
+ # @param tests [Array<Test>] every test discovered, before any slicing
52
+ # @param timings [Hash] predicted weight per test key, as `Timing#load_with_fallback` returns
53
+ # @param scheduler [Scheduler] the same partitioner the run uses for workers
54
+ # @return [Array<Test>] the tests belonging to this shard
55
+ def select(tests:, timings:, scheduler:)
56
+ return tests if whole_suite?
57
+
58
+ bins = scheduler.partition(tests: tests, worker_count: total, timings: timings)
59
+ bins[index - 1].remaining
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Binpacker
6
+ # Fan-in audit for a sharded matrix: did the shards, between them, actually run the whole suite?
7
+ #
8
+ # Shards never coordinate. Each cuts the same N-way partition from the timing data it loaded and takes
9
+ # its own bin, which is correct exactly as long as they all loaded the SAME data. In CI that means every
10
+ # shard job restoring the same timing cache — and the failure mode when one does not is the bad one:
11
+ # it partitions differently, some tests land in no shard, and every job still reports success. Nothing
12
+ # inside a single shard can notice, because a shard cannot tell "not mine" from "does not exist".
13
+ #
14
+ # So the check belongs after the matrix, over the run reports it produced. Point it at every shard's
15
+ # report and it fails unless the reports describe one coherent split of one suite.
16
+ class ShardCheck
17
+ Result = Struct.new(:ok, :problems, :summary, keyword_init: true) do
18
+ def ok? = ok
19
+ end
20
+
21
+ def self.call(paths) = new(paths).call
22
+
23
+ def initialize(paths)
24
+ @paths = Array(paths)
25
+ end
26
+
27
+ def call
28
+ return failure(['no run reports given']) if @paths.empty?
29
+
30
+ reports, unreadable = load_reports
31
+ return failure(unreadable) unless unreadable.empty?
32
+
33
+ shards = reports.filter_map { |path, data| shard_of(path, data) }
34
+ missing = reports.map(&:first) - shards.map { |s| s[:path] }
35
+ return failure(missing.map { |p| "#{p}: no `shard` section — was it run with --shard?" }) unless missing.empty?
36
+
37
+ problems = check(shards)
38
+ problems.empty? ? success(shards) : failure(problems)
39
+ end
40
+
41
+ private
42
+
43
+ def load_reports
44
+ reports = []
45
+ unreadable = []
46
+ @paths.each do |path|
47
+ reports << [path, JSON.parse(File.read(path))]
48
+ rescue Errno::ENOENT
49
+ unreadable << "#{path}: no such file"
50
+ rescue JSON::ParserError => e
51
+ unreadable << "#{path}: not valid JSON (#{e.message})"
52
+ end
53
+ [reports, unreadable]
54
+ end
55
+
56
+ def shard_of(path, data)
57
+ section = data['shard']
58
+ return nil unless section.is_a?(Hash)
59
+
60
+ {
61
+ path: path,
62
+ index: section['index'],
63
+ total: section['total'],
64
+ discovered: section['discovered_tests'],
65
+ selected: section['selected_tests']
66
+ }
67
+ end
68
+
69
+ def check(shards)
70
+ problems = []
71
+ problems.concat(agreement_problems(shards, :total, 'shard count'))
72
+ problems.concat(agreement_problems(shards, :discovered, 'discovered test count'))
73
+ problems.concat(completeness_problems(shards))
74
+ problems.concat(coverage_problems(shards))
75
+ problems
76
+ end
77
+
78
+ # Every report must describe the same matrix. Disagreement here means the reports were not produced by
79
+ # one run, and no coverage conclusion drawn from them would mean anything.
80
+ def agreement_problems(shards, field, label)
81
+ values = shards.map { |s| s[field] }.uniq
82
+ return [] if values.size <= 1
83
+
84
+ ["shards disagree on #{label}: #{values.sort_by(&:to_s).inspect}"]
85
+ end
86
+
87
+ # A matrix that lost a job silently drops that job's slice, which looks exactly like a smaller suite.
88
+ def completeness_problems(shards)
89
+ total = shards.first[:total]
90
+ return [] if total.nil?
91
+
92
+ seen = shards.map { |s| s[:index] }
93
+ duplicates = seen.tally.select { |_, n| n > 1 }.keys.sort
94
+ problems = []
95
+ problems << "shard #{duplicates.join(', ')} reported more than once" unless duplicates.empty?
96
+
97
+ absent = (1..total).to_a - seen
98
+ problems << "no report for shard #{absent.join(', ')} of #{total}" unless absent.empty?
99
+ problems
100
+ end
101
+
102
+ # The check this class exists for: the slices must add up to the suite.
103
+ def coverage_problems(shards)
104
+ discovered = shards.first[:discovered]
105
+ return [] if discovered.nil?
106
+
107
+ selected = shards.sum { |s| s[:selected].to_i }
108
+ return [] if selected == discovered
109
+
110
+ verb = selected < discovered ? 'ran no shard' : 'ran in more than one shard'
111
+ ["shards cover #{selected} of #{discovered} tests — #{(discovered - selected).abs} #{verb}. " \
112
+ 'The shards partitioned different timing data; make every shard load the same timing file.']
113
+ end
114
+
115
+ def success(shards)
116
+ Result.new(
117
+ ok: true,
118
+ problems: [],
119
+ summary: "#{shards.size} shards cover all #{shards.first[:discovered]} tests"
120
+ )
121
+ end
122
+
123
+ def failure(problems)
124
+ Result.new(ok: false, problems: problems, summary: 'shard coverage check failed')
125
+ end
126
+ end
127
+ end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "json"
3
+ require 'json'
4
4
 
5
5
  module Binpacker
6
6
  class Timing
@@ -8,40 +8,69 @@ module Binpacker
8
8
 
9
9
  DEFAULT_WEIGHT = 1.0
10
10
 
11
+ # Samples retained per test by #compact! and consulted by the
12
+ # median in #load_per_file. Three samples make a single anomalous
13
+ # run (GC pause, noisy CI neighbour) unable to move the weight.
14
+ MAX_SAMPLES_PER_TEST = 3
15
+
11
16
  def initialize(path)
12
17
  @path = Pathname(path)
13
18
  end
14
19
 
20
+ # True once any timing samples exist, i.e. the project has been
21
+ # calibrated at least once. Callers use this to tell a measured
22
+ # run (weights in seconds) from a pure cold start (fallbacks only).
23
+ def calibrated?
24
+ !samples_by_test.empty?
25
+ end
26
+
15
27
  def load_with_fallback(tests)
16
28
  per_file = load_per_file
29
+ coefficient = seconds_per_kb(per_file)
17
30
  tests.each_with_object({}) do |test, hash|
18
31
  key = normalize_path(test.file)
19
- hash[test.key] = per_file.fetch(key) { filesize_weight(test.file) }
32
+ hash[test.key] = per_file.fetch(key) { fallback_weight(test.file, coefficient) }
20
33
  end
21
34
  end
22
35
 
23
36
  def load_raw
24
- return {} unless @path.exist?
25
-
26
- @path.each_line(encoding: "UTF-8")
27
- .map { |line| parse_line(line) }
28
- .compact
29
- .group_by { |e| [normalize_path(e.file), e.name] }
30
- .transform_values { |entries| entries.last.time }
37
+ @load_raw ||= samples_by_test.transform_values(&:last)
31
38
  end
32
39
 
40
+ # Predicted weight per file: the median of each test's recent
41
+ # samples, summed per file. The append-only history must NOT be
42
+ # summed wholesale — a file present in N historical runs would
43
+ # weigh ~N times its true cost, so long-lived files dominate and
44
+ # newly added ones are starved, skewing the partition.
33
45
  def load_per_file
34
- return {} unless @path.exist?
46
+ samples_by_test.each_with_object({}) do |((file, _name), times), per_file|
47
+ weight = median(times.last(MAX_SAMPLES_PER_TEST))
48
+ per_file[file] = per_file.fetch(file, 0.0) + weight
49
+ end
50
+ end
35
51
 
36
- @path.each_line(encoding: "UTF-8")
37
- .map { |line| parse_line(line) }
38
- .compact
39
- .group_by { |e| normalize_path(e.file) }
40
- .transform_values { |entries| entries.sum(&:time) }
52
+ # Rewrites the timing file keeping only the most recent
53
+ # MAX_SAMPLES_PER_TEST samples per test, so the append-only
54
+ # history (and any CI cache built from it) stays bounded instead
55
+ # of growing by one run per invocation.
56
+ def compact!
57
+ samples = samples_by_test
58
+ return if samples.empty?
59
+
60
+ tmp = Pathname("#{@path}.tmp")
61
+ tmp.open('w', encoding: 'UTF-8') do |io|
62
+ samples.each do |(file, name), times|
63
+ times.last(MAX_SAMPLES_PER_TEST).each do |time|
64
+ io.puts JSON.generate({ file: file, name: name, time: time })
65
+ end
66
+ end
67
+ end
68
+ File.rename(tmp.to_s, @path.to_s)
69
+ invalidate
41
70
  end
42
71
 
43
72
  def normalize_path(path)
44
- Pathname(path).cleanpath.to_s.sub(/\A\.\//, "")
73
+ Pathname(path).cleanpath.to_s.sub(%r{\A\./}, '')
45
74
  end
46
75
 
47
76
  # True when a measured Weight already exists for this Test.
@@ -49,34 +78,103 @@ module Binpacker
49
78
  load_raw.key?([normalize_path(file), name])
50
79
  end
51
80
 
52
- def weight_for(file:, name:)
53
- measured = load_raw
54
- measured.fetch([file, name]) { filesize_weight(file) }
55
- end
56
-
57
81
  def append(file:, name:, time:)
58
82
  @path.dirname.mkpath unless @path.dirname.directory?
59
- @path.open("a", encoding: "UTF-8") { |io| io.puts JSON.generate({ file: file, name: name, time: time }) }
83
+ @path.open('a', encoding: 'UTF-8') { |io| io.puts JSON.generate({ file: file, name: name, time: time }) }
84
+ invalidate
60
85
  end
61
86
 
62
87
  def append_all(entries)
63
88
  return if entries.empty?
89
+
64
90
  @path.dirname.mkpath unless @path.dirname.directory?
65
- @path.open("a", encoding: "UTF-8") do |io|
91
+ @path.open('a', encoding: 'UTF-8') do |io|
66
92
  entries.each { |e| io.puts JSON.generate({ file: e[:file], name: e[:name], time: e[:time] }) }
67
93
  end
94
+ invalidate
68
95
  end
69
96
 
70
97
  private
71
98
 
99
+ # [normalized file, name] => [sample, ...] in append (= run) order.
100
+ def samples_by_test
101
+ @samples_by_test ||= begin
102
+ samples = Hash.new { |h, k| h[k] = [] }
103
+ if @path.exist?
104
+ @path.each_line(encoding: 'UTF-8') do |line|
105
+ entry = parse_line(line)
106
+ samples[[normalize_path(entry.file), entry.name]] << entry.time if entry
107
+ end
108
+ end
109
+ samples.default_proc = nil
110
+ samples
111
+ end
112
+ end
113
+
114
+ def invalidate
115
+ @samples_by_test = nil
116
+ @load_raw = nil
117
+ end
118
+
119
+ def median(values)
120
+ sorted = values.sort
121
+ mid = sorted.size / 2
122
+ sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
123
+ end
124
+
72
125
  def filesize_weight(file)
73
126
  path = Pathname(file)
74
127
  path.exist? ? [path.size / 1024.0, DEFAULT_WEIGHT].max : DEFAULT_WEIGHT
75
128
  end
76
129
 
130
+ # Seconds-per-KB coefficient used to scale filesize fallbacks onto
131
+ # the same axis as measured Weights. Measured Weights are seconds;
132
+ # the raw filesize fallback is KB, so mixing them (batch floors,
133
+ # donor selection) compares apples to oranges — 30s of predicted
134
+ # work reads as "30 KB of files". We estimate a conversion from
135
+ # the files we DO have timings for: for each measured file that
136
+ # still exists on disk with size > 0, take measured_seconds /
137
+ # size_kb, and use the median of those ratios (robust to a few
138
+ # outlier files that are unusually fast or slow for their size).
139
+ # nil when nothing can be estimated (no measurements, or none of
140
+ # the measured files exist on disk) — callers then keep raw KB.
141
+ def seconds_per_kb(per_file)
142
+ ratios = per_file.filter_map do |file, seconds|
143
+ kb = size_kb(file)
144
+ next if kb.nil? || kb <= 0
145
+
146
+ seconds / kb
147
+ end
148
+ ratios.empty? ? nil : median(ratios)
149
+ end
150
+
151
+ # Predicted Weight for an unmeasured Test. Without a coefficient we
152
+ # keep the legacy raw-KB behavior (floored at 1.0). With one we
153
+ # scale filesize into seconds: an existing file becomes
154
+ # size_kb * coefficient (floored at 0.01 to avoid degenerate zero
155
+ # weights, but NOT at 1.0 — sub-second predictions are meaningful
156
+ # now that the unit is seconds); a missing/unreadable file falls
157
+ # back to DEFAULT_WEIGHT as a plausible seconds-scale default.
158
+ def fallback_weight(file, coefficient)
159
+ return filesize_weight(file) if coefficient.nil?
160
+
161
+ kb = size_kb(file)
162
+ return DEFAULT_WEIGHT if kb.nil?
163
+
164
+ [kb * coefficient, 0.01].max
165
+ end
166
+
167
+ # File size in KB, or nil when the file does not exist / is unreadable.
168
+ def size_kb(file)
169
+ path = Pathname(file)
170
+ path.exist? ? path.size / 1024.0 : nil
171
+ rescue SystemCallError
172
+ nil
173
+ end
174
+
77
175
  def parse_line(line)
78
176
  data = JSON.parse(line.strip)
79
- Entry.new(file: data["file"], name: data["name"], time: data["time"])
177
+ Entry.new(file: data['file'], name: data['name'], time: data['time'])
80
178
  rescue JSON::ParserError
81
179
  nil
82
180
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Binpacker
4
- VERSION = "0.3.0"
4
+ VERSION = "0.5.0"
5
5
  end
@@ -37,7 +37,7 @@ module Binpacker
37
37
  end
38
38
 
39
39
  def total_weight(timings)
40
- remaining.sum { |t| timings.fetch(t.key, Timing::DEFAULT_WEIGHT) }
40
+ remaining.sum(0.0) { |t| timings.fetch(t.key, Timing::DEFAULT_WEIGHT) }
41
41
  end
42
42
 
43
43
  def push(test)
data/lib/binpacker.rb CHANGED
@@ -1,19 +1,21 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative "binpacker/version"
4
- require_relative "binpacker/config"
5
- require_relative "binpacker/timing"
6
- require_relative "binpacker/test_discovery"
7
- require_relative "binpacker/worker_queue"
8
- require_relative "binpacker/scheduler"
9
- require_relative "binpacker/worker"
10
- require_relative "binpacker/test_runner"
11
- require_relative "binpacker/calibration"
12
- require_relative "binpacker/report"
13
- require_relative "binpacker/skills"
14
- require_relative "binpacker/project_state"
15
- require_relative "binpacker/orchestrator"
16
- require_relative "binpacker/progress"
3
+ require_relative 'binpacker/version'
4
+ require_relative 'binpacker/config'
5
+ require_relative 'binpacker/timing'
6
+ require_relative 'binpacker/test_discovery'
7
+ require_relative 'binpacker/worker_queue'
8
+ require_relative 'binpacker/scheduler'
9
+ require_relative 'binpacker/shard'
10
+ require_relative 'binpacker/shard_check'
11
+ require_relative 'binpacker/worker'
12
+ require_relative 'binpacker/test_runner'
13
+ require_relative 'binpacker/calibration'
14
+ require_relative 'binpacker/report'
15
+ require_relative 'binpacker/skills'
16
+ require_relative 'binpacker/project_state'
17
+ require_relative 'binpacker/orchestrator'
18
+ require_relative 'binpacker/progress'
17
19
 
18
20
  module Binpacker
19
21
  Error = Class.new(StandardError)
@@ -1,6 +1,31 @@
1
1
  module Binpacker
2
2
  class Orchestrator
3
- def initialize: (untyped, ?passthrough: untyped, ?quiet: untyped, ?report_path: untyped) -> void
3
+ MIN_BATCH_WEIGHT: Float
4
+ COLD_START_BATCHES_PER_WORKER: Integer
5
+
6
+ @config: untyped
7
+ @passthrough: untyped
8
+ @quiet: untyped
9
+ @report_path: untyped
10
+ @shard: Binpacker::Shard?
11
+ @discovered_count: Integer
12
+ @selected_count: Integer
13
+ @timings: Hash[untyped, Float]
14
+ @predicted_loads: Array[Float]
15
+
16
+ def initialize: (untyped, ?passthrough: untyped, ?quiet: untyped, ?report_path: untyped, ?shard: Binpacker::Shard?) -> void
4
17
  def run: () -> { passed: untyped, total: untyped, passed_count: untyped, timings: Array[untyped], empty_filter: untyped }
18
+
19
+ private
20
+
21
+ def discover: () -> untyped
22
+ def run_static: (untyped, untyped, Binpacker::Timing, untyped) -> untyped
23
+ def run_dynamic: (untyped, untyped, Binpacker::Timing, untyped, Float) -> untyped
24
+ def write_report: (untyped, Array[untyped]) -> void
25
+ def min_batch_weight: (Binpacker::Timing, Hash[untyped, Float]) -> Float
26
+ def drain_batch: (Binpacker::WorkerQueue?, Float) -> Array[untyped]
27
+ def finalize: (Binpacker::Timing, Array[untyped], untyped, untyped, untyped, untyped) -> { passed: untyped, total: untyped, passed_count: untyped, timings: Array[untyped], empty_filter: untyped }
28
+ def minitest_empty_filter?: (untyped, untyped) -> bool
29
+ def minitest_include_filter?: () -> bool
5
30
  end
6
31
  end
@@ -3,7 +3,8 @@ module Binpacker
3
3
  SCHEMA: Integer
4
4
  DRIFT_LIMIT: Integer
5
5
 
6
- def initialize: (profile: untyped, algorithm: untyped, predicted_loads: untyped, worker_stats: untyped, all_timings: untyped, timings: untyped) -> void
6
+ def initialize: (profile: untyped, algorithm: untyped, predicted_loads: untyped, worker_stats: untyped, all_timings: untyped, timings: untyped, ?shard: Binpacker::Shard?, ?discovered: Integer?, ?selected: Integer?) -> void
7
+ def shard_section: () -> Hash[Symbol, untyped]?
7
8
  def to_h: () -> Hash[Symbol, untyped]
8
9
  def write: (untyped) -> untyped
9
10
  end
@@ -1,5 +1,16 @@
1
1
  module Binpacker
2
2
  class Scheduler
3
3
  def self.for: (untyped) -> (Binpacker::LptScheduler | Binpacker::MultifitScheduler)
4
+
5
+ # Declared on the base so a caller holding a `Scheduler` can partition; the base body raises
6
+ # NotImplementedError and every concrete scheduler overrides it.
7
+ def partition: (tests: Array[untyped], worker_count: Integer, timings: Hash[untyped, Float]) -> Array[Binpacker::WorkerQueue]
8
+ end
9
+
10
+ class LptScheduler < Scheduler
11
+ end
12
+
13
+ class MultifitScheduler < Scheduler
14
+ ITERATIONS: Integer
4
15
  end
5
16
  end