binpacker 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/binpacker-worker +12 -2
- data/lib/binpacker/cli.rb +85 -47
- data/lib/binpacker/orchestrator.rb +98 -40
- data/lib/binpacker/progress.rb +15 -9
- data/lib/binpacker/report.rb +33 -3
- data/lib/binpacker/shard.rb +62 -0
- data/lib/binpacker/shard_check.rb +127 -0
- data/lib/binpacker/timing.rb +122 -24
- data/lib/binpacker/version.rb +1 -1
- data/lib/binpacker/worker_queue.rb +1 -1
- data/lib/binpacker.rb +16 -14
- data/sig/binpacker/orchestrator.rbs +26 -1
- data/sig/binpacker/report.rbs +2 -1
- data/sig/binpacker/scheduler.rbs +11 -0
- data/sig/binpacker/shard.rbs +13 -0
- data/sig/binpacker/shard_check.rbs +29 -0
- data/sig/binpacker/timing.rbs +6 -0
- data/sig/binpacker/worker_queue.rbs +11 -0
- metadata +5 -1
data/lib/binpacker/progress.rb
CHANGED
|
@@ -7,7 +7,7 @@ module Binpacker
|
|
|
7
7
|
def initialize(worker_count, tty: $stdout.tty?)
|
|
8
8
|
@worker_count = worker_count
|
|
9
9
|
@tty = tty
|
|
10
|
-
@workers = Array.new(worker_count) { { done: 0, total: 0, file:
|
|
10
|
+
@workers = Array.new(worker_count) { { done: 0, total: 0, file: '', elapsed: 0.0 } }
|
|
11
11
|
@start = Time.now
|
|
12
12
|
@last_ci_output = Time.at(Time.now.to_f - CI_INTERVAL)
|
|
13
13
|
@lines_written = 0
|
|
@@ -32,8 +32,9 @@ module Binpacker
|
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
def finish(
|
|
35
|
+
def finish(_worker_stats = [])
|
|
36
36
|
return unless @tty
|
|
37
|
+
|
|
37
38
|
redraw
|
|
38
39
|
$stdout.puts
|
|
39
40
|
end
|
|
@@ -42,9 +43,10 @@ module Binpacker
|
|
|
42
43
|
active = worker_stats.reject { |s| s[:files] == 0 && s[:examples] == 0 }
|
|
43
44
|
return if active.empty?
|
|
44
45
|
|
|
45
|
-
|
|
46
|
+
worker_stats.each_with_index do |s, wid|
|
|
47
|
+
next if s[:files] == 0 && s[:examples] == 0
|
|
48
|
+
|
|
46
49
|
t = format_time(s[:total_time])
|
|
47
|
-
wid = worker_stats.index(s)
|
|
48
50
|
$stdout.puts " Worker #{wid}: #{s[:files]} tests, #{t} | #{s[:examples]} examples, #{s[:passed]} passed"
|
|
49
51
|
end
|
|
50
52
|
total_tests = active.sum { |s| s[:files] }
|
|
@@ -55,7 +57,7 @@ module Binpacker
|
|
|
55
57
|
max_dev = times.map { |t| (t - mean).abs }.max
|
|
56
58
|
dev_pct = mean > 0 ? (max_dev / mean * 100).round(1) : 0
|
|
57
59
|
|
|
58
|
-
$stdout.puts
|
|
60
|
+
$stdout.puts ' ──'
|
|
59
61
|
$stdout.puts " Total: #{total_tests} tests, #{format_time(total_time)} | #{total_examples} examples"
|
|
60
62
|
$stdout.puts " Balance: max deviation #{format_time(max_dev)} (#{dev_pct}%)"
|
|
61
63
|
end
|
|
@@ -67,7 +69,7 @@ module Binpacker
|
|
|
67
69
|
clear_lines
|
|
68
70
|
@workers.each_with_index do |w, i|
|
|
69
71
|
bar = build_bar(w[:done], w[:total])
|
|
70
|
-
status = w[:total] > 0 && w[:done] >= w[:total] ?
|
|
72
|
+
status = w[:total] > 0 && w[:done] >= w[:total] ? 'done' : w[:file][-50..] || ''
|
|
71
73
|
$stdout.puts format_line(i, bar, w[:done], w[:total], status, w[:elapsed])
|
|
72
74
|
end
|
|
73
75
|
@lines_written = @worker_count
|
|
@@ -76,13 +78,15 @@ module Binpacker
|
|
|
76
78
|
|
|
77
79
|
def clear_lines
|
|
78
80
|
return if @lines_written == 0
|
|
81
|
+
|
|
79
82
|
@lines_written.times do
|
|
80
83
|
$stdout.print "\033[A\033[K"
|
|
81
84
|
end
|
|
82
85
|
end
|
|
83
86
|
|
|
84
87
|
def build_bar(done, total)
|
|
85
|
-
return
|
|
88
|
+
return '[----------]' if total == 0
|
|
89
|
+
|
|
86
90
|
width = 10
|
|
87
91
|
filled = (done.to_f / total * width).round
|
|
88
92
|
"[#{'█' * filled}#{'░' * (width - filled)}]"
|
|
@@ -96,10 +100,11 @@ module Binpacker
|
|
|
96
100
|
def periodic_output
|
|
97
101
|
now = Time.now
|
|
98
102
|
return if now - @last_ci_output < CI_INTERVAL
|
|
103
|
+
|
|
99
104
|
@last_ci_output = now
|
|
100
105
|
|
|
101
106
|
parts = @workers.map.with_index do |w, i|
|
|
102
|
-
ratio = w[:total] > 0 ? "#{w[:done]}/#{w[:total]}" :
|
|
107
|
+
ratio = w[:total] > 0 ? "#{w[:done]}/#{w[:total]}" : '0/?'
|
|
103
108
|
"W#{i}: #{ratio}"
|
|
104
109
|
end
|
|
105
110
|
elapsed = (now - @start).round(1)
|
|
@@ -108,7 +113,8 @@ module Binpacker
|
|
|
108
113
|
end
|
|
109
114
|
|
|
110
115
|
def format_time(seconds)
|
|
111
|
-
return
|
|
116
|
+
return ' 0.0s' if seconds < 0.001
|
|
117
|
+
|
|
112
118
|
m = (seconds / 60).floor
|
|
113
119
|
s = (seconds % 60).round(1)
|
|
114
120
|
m > 0 ? "#{m}m#{s.to_s.rjust(4, '0')}s" : "#{s.to_s.rjust(5)}s"
|
data/lib/binpacker/report.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require 'json'
|
|
4
4
|
|
|
5
5
|
module Binpacker
|
|
6
6
|
# Builds the machine-readable Run report: predicted versus actual per-Worker
|
|
@@ -9,13 +9,17 @@ module Binpacker
|
|
|
9
9
|
SCHEMA = 1
|
|
10
10
|
DRIFT_LIMIT = 10
|
|
11
11
|
|
|
12
|
-
def initialize(profile:, algorithm:, predicted_loads:, worker_stats:, all_timings:, timings
|
|
12
|
+
def initialize(profile:, algorithm:, predicted_loads:, worker_stats:, all_timings:, timings:,
|
|
13
|
+
shard: nil, discovered: nil, selected: nil)
|
|
13
14
|
@profile = profile
|
|
14
15
|
@algorithm = algorithm
|
|
15
16
|
@predicted_loads = predicted_loads
|
|
16
17
|
@worker_stats = worker_stats
|
|
17
18
|
@all_timings = all_timings
|
|
18
19
|
@timings = timings
|
|
20
|
+
@shard = shard
|
|
21
|
+
@discovered = discovered
|
|
22
|
+
@selected = selected
|
|
19
23
|
end
|
|
20
24
|
|
|
21
25
|
def to_h
|
|
@@ -26,6 +30,7 @@ module Binpacker
|
|
|
26
30
|
schema: SCHEMA,
|
|
27
31
|
profile: @profile,
|
|
28
32
|
algorithm: @algorithm,
|
|
33
|
+
shard: shard_section,
|
|
29
34
|
worker_count: @worker_stats.size,
|
|
30
35
|
predicted_makespan: round(predicted.max || 0.0),
|
|
31
36
|
actual_makespan: round(actual.max || 0.0),
|
|
@@ -44,6 +49,29 @@ module Binpacker
|
|
|
44
49
|
|
|
45
50
|
private
|
|
46
51
|
|
|
52
|
+
# The audit trail for a sharded run, and the reason `discovered` is recorded at all.
|
|
53
|
+
#
|
|
54
|
+
# Shards never talk to each other: each computes the same N-way partition and trusts the others to have
|
|
55
|
+
# computed it identically. They do so only while they agree on the timing data the partition is cut
|
|
56
|
+
# from, which in CI means every shard restoring the same timing cache. A shard that restores a
|
|
57
|
+
# different one — a cache miss where its siblings hit — partitions differently, and the failure is
|
|
58
|
+
# silent: tests land in no shard at all and the build stays green.
|
|
59
|
+
#
|
|
60
|
+
# `discovered` is the whole-suite count before slicing, so it agrees across shards that see the same
|
|
61
|
+
# repository. `selected` is this shard's slice. Summing `selected` over a matrix's reports and
|
|
62
|
+
# comparing to the shared `discovered` turns that silent skip into a failure —
|
|
63
|
+
# `binpacker shards-check` does exactly that.
|
|
64
|
+
def shard_section
|
|
65
|
+
return nil unless @shard
|
|
66
|
+
|
|
67
|
+
{
|
|
68
|
+
index: @shard.index,
|
|
69
|
+
total: @shard.total,
|
|
70
|
+
discovered_tests: @discovered,
|
|
71
|
+
selected_tests: @selected
|
|
72
|
+
}
|
|
73
|
+
end
|
|
74
|
+
|
|
47
75
|
def workers
|
|
48
76
|
@worker_stats.map.with_index do |s, i|
|
|
49
77
|
{
|
|
@@ -84,13 +112,15 @@ module Binpacker
|
|
|
84
112
|
end
|
|
85
113
|
|
|
86
114
|
def normalize_file(file)
|
|
87
|
-
file.to_s.sub(%r{\A\./},
|
|
115
|
+
file.to_s.sub(%r{\A\./}, '')
|
|
88
116
|
end
|
|
89
117
|
|
|
90
118
|
def deviation_pct(loads)
|
|
91
119
|
return 0.0 if loads.empty?
|
|
120
|
+
|
|
92
121
|
mean = loads.sum / loads.size
|
|
93
122
|
return 0.0 unless mean.positive?
|
|
123
|
+
|
|
94
124
|
max_dev = loads.map { |t| (t - mean).abs }.max
|
|
95
125
|
round(max_dev / mean * 100)
|
|
96
126
|
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Binpacker
|
|
4
|
+
# One slice of the suite, for splitting a run across independent machines.
|
|
5
|
+
#
|
|
6
|
+
# Workers divide a suite across the cores of ONE machine and share its wall clock; a shard divides it
|
|
7
|
+
# across machines that have no wall clock in common. So the two compose rather than compete: a CI matrix
|
|
8
|
+
# of N jobs each passing `--shard k/N` runs `worker_count` workers over its own slice, and the suite's
|
|
9
|
+
# wall time becomes roughly the slowest shard rather than the whole.
|
|
10
|
+
#
|
|
11
|
+
# The slice is cut by the SAME weight-balanced partitioner that assigns work to workers, over the same
|
|
12
|
+
# measured timings — shards are just a coarser bin-packing of the same problem, so an equal-count split
|
|
13
|
+
# would balance no better than round-robin. That also makes the cut deterministic: given one timing file,
|
|
14
|
+
# every shard computes the identical N-way partition and takes only its own bin, so no shard needs to
|
|
15
|
+
# know what the others decided, and every test lands in exactly one shard.
|
|
16
|
+
class Shard
|
|
17
|
+
attr_reader :index, :total
|
|
18
|
+
|
|
19
|
+
# Parses the `k/n` form used by `--shard` and BINPACKER_SHARD. `k` is 1-based, so a matrix can pass its
|
|
20
|
+
# own 1-based job number straight through.
|
|
21
|
+
#
|
|
22
|
+
# @param spec [String, nil]
|
|
23
|
+
# @return [Shard, nil] nil when `spec` is nil or empty, i.e. an unsharded run.
|
|
24
|
+
def self.parse(spec)
|
|
25
|
+
return nil if spec.nil? || spec.to_s.strip.empty?
|
|
26
|
+
|
|
27
|
+
match = %r{\A\s*(\d+)\s*/\s*(\d+)\s*\z}.match(spec.to_s)
|
|
28
|
+
raise ConfigError, "invalid shard #{spec.inspect}: expected the form K/N, e.g. 1/3" unless match
|
|
29
|
+
|
|
30
|
+
new(index: Integer(match[1]), total: Integer(match[2]))
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(index:, total:)
|
|
34
|
+
raise ConfigError, "shard count must be at least 1, got #{total}" if total < 1
|
|
35
|
+
raise ConfigError, "shard index must be between 1 and #{total}, got #{index}" unless (1..total).cover?(index)
|
|
36
|
+
|
|
37
|
+
@index = index
|
|
38
|
+
@total = total
|
|
39
|
+
freeze
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def to_s
|
|
43
|
+
"#{index}/#{total}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Whole-suite runs still construct no Shard, so this is only ever true for an explicit `--shard 1/1`.
|
|
47
|
+
def whole_suite?
|
|
48
|
+
total == 1
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @param tests [Array<Test>] every test discovered, before any slicing
|
|
52
|
+
# @param timings [Hash] predicted weight per test key, as `Timing#load_with_fallback` returns
|
|
53
|
+
# @param scheduler [Scheduler] the same partitioner the run uses for workers
|
|
54
|
+
# @return [Array<Test>] the tests belonging to this shard
|
|
55
|
+
def select(tests:, timings:, scheduler:)
|
|
56
|
+
return tests if whole_suite?
|
|
57
|
+
|
|
58
|
+
bins = scheduler.partition(tests: tests, worker_count: total, timings: timings)
|
|
59
|
+
bins[index - 1].remaining
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Binpacker
|
|
6
|
+
# Fan-in audit for a sharded matrix: did the shards, between them, actually run the whole suite?
|
|
7
|
+
#
|
|
8
|
+
# Shards never coordinate. Each cuts the same N-way partition from the timing data it loaded and takes
|
|
9
|
+
# its own bin, which is correct exactly as long as they all loaded the SAME data. In CI that means every
|
|
10
|
+
# shard job restoring the same timing cache — and the failure mode when one does not is the bad one:
|
|
11
|
+
# it partitions differently, some tests land in no shard, and every job still reports success. Nothing
|
|
12
|
+
# inside a single shard can notice, because a shard cannot tell "not mine" from "does not exist".
|
|
13
|
+
#
|
|
14
|
+
# So the check belongs after the matrix, over the run reports it produced. Point it at every shard's
|
|
15
|
+
# report and it fails unless the reports describe one coherent split of one suite.
|
|
16
|
+
class ShardCheck
|
|
17
|
+
Result = Struct.new(:ok, :problems, :summary, keyword_init: true) do
|
|
18
|
+
def ok? = ok
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.call(paths) = new(paths).call
|
|
22
|
+
|
|
23
|
+
def initialize(paths)
|
|
24
|
+
@paths = Array(paths)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def call
|
|
28
|
+
return failure(['no run reports given']) if @paths.empty?
|
|
29
|
+
|
|
30
|
+
reports, unreadable = load_reports
|
|
31
|
+
return failure(unreadable) unless unreadable.empty?
|
|
32
|
+
|
|
33
|
+
shards = reports.filter_map { |path, data| shard_of(path, data) }
|
|
34
|
+
missing = reports.map(&:first) - shards.map { |s| s[:path] }
|
|
35
|
+
return failure(missing.map { |p| "#{p}: no `shard` section — was it run with --shard?" }) unless missing.empty?
|
|
36
|
+
|
|
37
|
+
problems = check(shards)
|
|
38
|
+
problems.empty? ? success(shards) : failure(problems)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def load_reports
|
|
44
|
+
reports = []
|
|
45
|
+
unreadable = []
|
|
46
|
+
@paths.each do |path|
|
|
47
|
+
reports << [path, JSON.parse(File.read(path))]
|
|
48
|
+
rescue Errno::ENOENT
|
|
49
|
+
unreadable << "#{path}: no such file"
|
|
50
|
+
rescue JSON::ParserError => e
|
|
51
|
+
unreadable << "#{path}: not valid JSON (#{e.message})"
|
|
52
|
+
end
|
|
53
|
+
[reports, unreadable]
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def shard_of(path, data)
|
|
57
|
+
section = data['shard']
|
|
58
|
+
return nil unless section.is_a?(Hash)
|
|
59
|
+
|
|
60
|
+
{
|
|
61
|
+
path: path,
|
|
62
|
+
index: section['index'],
|
|
63
|
+
total: section['total'],
|
|
64
|
+
discovered: section['discovered_tests'],
|
|
65
|
+
selected: section['selected_tests']
|
|
66
|
+
}
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def check(shards)
|
|
70
|
+
problems = []
|
|
71
|
+
problems.concat(agreement_problems(shards, :total, 'shard count'))
|
|
72
|
+
problems.concat(agreement_problems(shards, :discovered, 'discovered test count'))
|
|
73
|
+
problems.concat(completeness_problems(shards))
|
|
74
|
+
problems.concat(coverage_problems(shards))
|
|
75
|
+
problems
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Every report must describe the same matrix. Disagreement here means the reports were not produced by
|
|
79
|
+
# one run, and no coverage conclusion drawn from them would mean anything.
|
|
80
|
+
def agreement_problems(shards, field, label)
|
|
81
|
+
values = shards.map { |s| s[field] }.uniq
|
|
82
|
+
return [] if values.size <= 1
|
|
83
|
+
|
|
84
|
+
["shards disagree on #{label}: #{values.sort_by(&:to_s).inspect}"]
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# A matrix that lost a job silently drops that job's slice, which looks exactly like a smaller suite.
|
|
88
|
+
def completeness_problems(shards)
|
|
89
|
+
total = shards.first[:total]
|
|
90
|
+
return [] if total.nil?
|
|
91
|
+
|
|
92
|
+
seen = shards.map { |s| s[:index] }
|
|
93
|
+
duplicates = seen.tally.select { |_, n| n > 1 }.keys.sort
|
|
94
|
+
problems = []
|
|
95
|
+
problems << "shard #{duplicates.join(', ')} reported more than once" unless duplicates.empty?
|
|
96
|
+
|
|
97
|
+
absent = (1..total).to_a - seen
|
|
98
|
+
problems << "no report for shard #{absent.join(', ')} of #{total}" unless absent.empty?
|
|
99
|
+
problems
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# The check this class exists for: the slices must add up to the suite.
|
|
103
|
+
def coverage_problems(shards)
|
|
104
|
+
discovered = shards.first[:discovered]
|
|
105
|
+
return [] if discovered.nil?
|
|
106
|
+
|
|
107
|
+
selected = shards.sum { |s| s[:selected].to_i }
|
|
108
|
+
return [] if selected == discovered
|
|
109
|
+
|
|
110
|
+
verb = selected < discovered ? 'ran no shard' : 'ran in more than one shard'
|
|
111
|
+
["shards cover #{selected} of #{discovered} tests — #{(discovered - selected).abs} #{verb}. " \
|
|
112
|
+
'The shards partitioned different timing data; make every shard load the same timing file.']
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def success(shards)
|
|
116
|
+
Result.new(
|
|
117
|
+
ok: true,
|
|
118
|
+
problems: [],
|
|
119
|
+
summary: "#{shards.size} shards cover all #{shards.first[:discovered]} tests"
|
|
120
|
+
)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def failure(problems)
|
|
124
|
+
Result.new(ok: false, problems: problems, summary: 'shard coverage check failed')
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
data/lib/binpacker/timing.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require 'json'
|
|
4
4
|
|
|
5
5
|
module Binpacker
|
|
6
6
|
class Timing
|
|
@@ -8,40 +8,69 @@ module Binpacker
|
|
|
8
8
|
|
|
9
9
|
DEFAULT_WEIGHT = 1.0
|
|
10
10
|
|
|
11
|
+
# Samples retained per test by #compact! and consulted by the
|
|
12
|
+
# median in #load_per_file. Three samples make a single anomalous
|
|
13
|
+
# run (GC pause, noisy CI neighbour) unable to move the weight.
|
|
14
|
+
MAX_SAMPLES_PER_TEST = 3
|
|
15
|
+
|
|
11
16
|
def initialize(path)
|
|
12
17
|
@path = Pathname(path)
|
|
13
18
|
end
|
|
14
19
|
|
|
20
|
+
# True once any timing samples exist, i.e. the project has been
|
|
21
|
+
# calibrated at least once. Callers use this to tell a measured
|
|
22
|
+
# run (weights in seconds) from a pure cold start (fallbacks only).
|
|
23
|
+
def calibrated?
|
|
24
|
+
!samples_by_test.empty?
|
|
25
|
+
end
|
|
26
|
+
|
|
15
27
|
def load_with_fallback(tests)
|
|
16
28
|
per_file = load_per_file
|
|
29
|
+
coefficient = seconds_per_kb(per_file)
|
|
17
30
|
tests.each_with_object({}) do |test, hash|
|
|
18
31
|
key = normalize_path(test.file)
|
|
19
|
-
hash[test.key] = per_file.fetch(key) {
|
|
32
|
+
hash[test.key] = per_file.fetch(key) { fallback_weight(test.file, coefficient) }
|
|
20
33
|
end
|
|
21
34
|
end
|
|
22
35
|
|
|
23
36
|
def load_raw
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
@path.each_line(encoding: "UTF-8")
|
|
27
|
-
.map { |line| parse_line(line) }
|
|
28
|
-
.compact
|
|
29
|
-
.group_by { |e| [normalize_path(e.file), e.name] }
|
|
30
|
-
.transform_values { |entries| entries.last.time }
|
|
37
|
+
@load_raw ||= samples_by_test.transform_values(&:last)
|
|
31
38
|
end
|
|
32
39
|
|
|
40
|
+
# Predicted weight per file: the median of each test's recent
|
|
41
|
+
# samples, summed per file. The append-only history must NOT be
|
|
42
|
+
# summed wholesale — a file present in N historical runs would
|
|
43
|
+
# weigh ~N times its true cost, so long-lived files dominate and
|
|
44
|
+
# newly added ones are starved, skewing the partition.
|
|
33
45
|
def load_per_file
|
|
34
|
-
|
|
46
|
+
samples_by_test.each_with_object({}) do |((file, _name), times), per_file|
|
|
47
|
+
weight = median(times.last(MAX_SAMPLES_PER_TEST))
|
|
48
|
+
per_file[file] = per_file.fetch(file, 0.0) + weight
|
|
49
|
+
end
|
|
50
|
+
end
|
|
35
51
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
52
|
+
# Rewrites the timing file keeping only the most recent
|
|
53
|
+
# MAX_SAMPLES_PER_TEST samples per test, so the append-only
|
|
54
|
+
# history (and any CI cache built from it) stays bounded instead
|
|
55
|
+
# of growing by one run per invocation.
|
|
56
|
+
def compact!
|
|
57
|
+
samples = samples_by_test
|
|
58
|
+
return if samples.empty?
|
|
59
|
+
|
|
60
|
+
tmp = Pathname("#{@path}.tmp")
|
|
61
|
+
tmp.open('w', encoding: 'UTF-8') do |io|
|
|
62
|
+
samples.each do |(file, name), times|
|
|
63
|
+
times.last(MAX_SAMPLES_PER_TEST).each do |time|
|
|
64
|
+
io.puts JSON.generate({ file: file, name: name, time: time })
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
File.rename(tmp.to_s, @path.to_s)
|
|
69
|
+
invalidate
|
|
41
70
|
end
|
|
42
71
|
|
|
43
72
|
def normalize_path(path)
|
|
44
|
-
Pathname(path).cleanpath.to_s.sub(
|
|
73
|
+
Pathname(path).cleanpath.to_s.sub(%r{\A\./}, '')
|
|
45
74
|
end
|
|
46
75
|
|
|
47
76
|
# True when a measured Weight already exists for this Test.
|
|
@@ -49,34 +78,103 @@ module Binpacker
|
|
|
49
78
|
load_raw.key?([normalize_path(file), name])
|
|
50
79
|
end
|
|
51
80
|
|
|
52
|
-
def weight_for(file:, name:)
|
|
53
|
-
measured = load_raw
|
|
54
|
-
measured.fetch([file, name]) { filesize_weight(file) }
|
|
55
|
-
end
|
|
56
|
-
|
|
57
81
|
def append(file:, name:, time:)
|
|
58
82
|
@path.dirname.mkpath unless @path.dirname.directory?
|
|
59
|
-
@path.open(
|
|
83
|
+
@path.open('a', encoding: 'UTF-8') { |io| io.puts JSON.generate({ file: file, name: name, time: time }) }
|
|
84
|
+
invalidate
|
|
60
85
|
end
|
|
61
86
|
|
|
62
87
|
def append_all(entries)
|
|
63
88
|
return if entries.empty?
|
|
89
|
+
|
|
64
90
|
@path.dirname.mkpath unless @path.dirname.directory?
|
|
65
|
-
@path.open(
|
|
91
|
+
@path.open('a', encoding: 'UTF-8') do |io|
|
|
66
92
|
entries.each { |e| io.puts JSON.generate({ file: e[:file], name: e[:name], time: e[:time] }) }
|
|
67
93
|
end
|
|
94
|
+
invalidate
|
|
68
95
|
end
|
|
69
96
|
|
|
70
97
|
private
|
|
71
98
|
|
|
99
|
+
# [normalized file, name] => [sample, ...] in append (= run) order.
|
|
100
|
+
def samples_by_test
|
|
101
|
+
@samples_by_test ||= begin
|
|
102
|
+
samples = Hash.new { |h, k| h[k] = [] }
|
|
103
|
+
if @path.exist?
|
|
104
|
+
@path.each_line(encoding: 'UTF-8') do |line|
|
|
105
|
+
entry = parse_line(line)
|
|
106
|
+
samples[[normalize_path(entry.file), entry.name]] << entry.time if entry
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
samples.default_proc = nil
|
|
110
|
+
samples
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def invalidate
|
|
115
|
+
@samples_by_test = nil
|
|
116
|
+
@load_raw = nil
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def median(values)
|
|
120
|
+
sorted = values.sort
|
|
121
|
+
mid = sorted.size / 2
|
|
122
|
+
sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
|
|
123
|
+
end
|
|
124
|
+
|
|
72
125
|
def filesize_weight(file)
|
|
73
126
|
path = Pathname(file)
|
|
74
127
|
path.exist? ? [path.size / 1024.0, DEFAULT_WEIGHT].max : DEFAULT_WEIGHT
|
|
75
128
|
end
|
|
76
129
|
|
|
130
|
+
# Seconds-per-KB coefficient used to scale filesize fallbacks onto
|
|
131
|
+
# the same axis as measured Weights. Measured Weights are seconds;
|
|
132
|
+
# the raw filesize fallback is KB, so mixing them (batch floors,
|
|
133
|
+
# donor selection) compares apples to oranges — 30s of predicted
|
|
134
|
+
# work reads as "30 KB of files". We estimate a conversion from
|
|
135
|
+
# the files we DO have timings for: for each measured file that
|
|
136
|
+
# still exists on disk with size > 0, take measured_seconds /
|
|
137
|
+
# size_kb, and use the median of those ratios (robust to a few
|
|
138
|
+
# outlier files that are unusually fast or slow for their size).
|
|
139
|
+
# nil when nothing can be estimated (no measurements, or none of
|
|
140
|
+
# the measured files exist on disk) — callers then keep raw KB.
|
|
141
|
+
def seconds_per_kb(per_file)
|
|
142
|
+
ratios = per_file.filter_map do |file, seconds|
|
|
143
|
+
kb = size_kb(file)
|
|
144
|
+
next if kb.nil? || kb <= 0
|
|
145
|
+
|
|
146
|
+
seconds / kb
|
|
147
|
+
end
|
|
148
|
+
ratios.empty? ? nil : median(ratios)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Predicted Weight for an unmeasured Test. Without a coefficient we
|
|
152
|
+
# keep the legacy raw-KB behavior (floored at 1.0). With one we
|
|
153
|
+
# scale filesize into seconds: an existing file becomes
|
|
154
|
+
# size_kb * coefficient (floored at 0.01 to avoid degenerate zero
|
|
155
|
+
# weights, but NOT at 1.0 — sub-second predictions are meaningful
|
|
156
|
+
# now that the unit is seconds); a missing/unreadable file falls
|
|
157
|
+
# back to DEFAULT_WEIGHT as a plausible seconds-scale default.
|
|
158
|
+
def fallback_weight(file, coefficient)
|
|
159
|
+
return filesize_weight(file) if coefficient.nil?
|
|
160
|
+
|
|
161
|
+
kb = size_kb(file)
|
|
162
|
+
return DEFAULT_WEIGHT if kb.nil?
|
|
163
|
+
|
|
164
|
+
[kb * coefficient, 0.01].max
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# File size in KB, or nil when the file does not exist / is unreadable.
|
|
168
|
+
def size_kb(file)
|
|
169
|
+
path = Pathname(file)
|
|
170
|
+
path.exist? ? path.size / 1024.0 : nil
|
|
171
|
+
rescue SystemCallError
|
|
172
|
+
nil
|
|
173
|
+
end
|
|
174
|
+
|
|
77
175
|
def parse_line(line)
|
|
78
176
|
data = JSON.parse(line.strip)
|
|
79
|
-
Entry.new(file: data[
|
|
177
|
+
Entry.new(file: data['file'], name: data['name'], time: data['time'])
|
|
80
178
|
rescue JSON::ParserError
|
|
81
179
|
nil
|
|
82
180
|
end
|
data/lib/binpacker/version.rb
CHANGED
data/lib/binpacker.rb
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative
|
|
4
|
-
require_relative
|
|
5
|
-
require_relative
|
|
6
|
-
require_relative
|
|
7
|
-
require_relative
|
|
8
|
-
require_relative
|
|
9
|
-
require_relative
|
|
10
|
-
require_relative
|
|
11
|
-
require_relative
|
|
12
|
-
require_relative
|
|
13
|
-
require_relative
|
|
14
|
-
require_relative
|
|
15
|
-
require_relative
|
|
16
|
-
require_relative
|
|
3
|
+
require_relative 'binpacker/version'
|
|
4
|
+
require_relative 'binpacker/config'
|
|
5
|
+
require_relative 'binpacker/timing'
|
|
6
|
+
require_relative 'binpacker/test_discovery'
|
|
7
|
+
require_relative 'binpacker/worker_queue'
|
|
8
|
+
require_relative 'binpacker/scheduler'
|
|
9
|
+
require_relative 'binpacker/shard'
|
|
10
|
+
require_relative 'binpacker/shard_check'
|
|
11
|
+
require_relative 'binpacker/worker'
|
|
12
|
+
require_relative 'binpacker/test_runner'
|
|
13
|
+
require_relative 'binpacker/calibration'
|
|
14
|
+
require_relative 'binpacker/report'
|
|
15
|
+
require_relative 'binpacker/skills'
|
|
16
|
+
require_relative 'binpacker/project_state'
|
|
17
|
+
require_relative 'binpacker/orchestrator'
|
|
18
|
+
require_relative 'binpacker/progress'
|
|
17
19
|
|
|
18
20
|
module Binpacker
|
|
19
21
|
Error = Class.new(StandardError)
|
|
@@ -1,6 +1,31 @@
|
|
|
1
1
|
module Binpacker
|
|
2
2
|
class Orchestrator
|
|
3
|
-
|
|
3
|
+
MIN_BATCH_WEIGHT: Float
|
|
4
|
+
COLD_START_BATCHES_PER_WORKER: Integer
|
|
5
|
+
|
|
6
|
+
@config: untyped
|
|
7
|
+
@passthrough: untyped
|
|
8
|
+
@quiet: untyped
|
|
9
|
+
@report_path: untyped
|
|
10
|
+
@shard: Binpacker::Shard?
|
|
11
|
+
@discovered_count: Integer
|
|
12
|
+
@selected_count: Integer
|
|
13
|
+
@timings: Hash[untyped, Float]
|
|
14
|
+
@predicted_loads: Array[Float]
|
|
15
|
+
|
|
16
|
+
def initialize: (untyped, ?passthrough: untyped, ?quiet: untyped, ?report_path: untyped, ?shard: Binpacker::Shard?) -> void
|
|
4
17
|
def run: () -> { passed: untyped, total: untyped, passed_count: untyped, timings: Array[untyped], empty_filter: untyped }
|
|
18
|
+
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def discover: () -> untyped
|
|
22
|
+
def run_static: (untyped, untyped, Binpacker::Timing, untyped) -> untyped
|
|
23
|
+
def run_dynamic: (untyped, untyped, Binpacker::Timing, untyped, Float) -> untyped
|
|
24
|
+
def write_report: (untyped, Array[untyped]) -> void
|
|
25
|
+
def min_batch_weight: (Binpacker::Timing, Hash[untyped, Float]) -> Float
|
|
26
|
+
def drain_batch: (Binpacker::WorkerQueue?, Float) -> Array[untyped]
|
|
27
|
+
def finalize: (Binpacker::Timing, Array[untyped], untyped, untyped, untyped, untyped) -> { passed: untyped, total: untyped, passed_count: untyped, timings: Array[untyped], empty_filter: untyped }
|
|
28
|
+
def minitest_empty_filter?: (untyped, untyped) -> bool
|
|
29
|
+
def minitest_include_filter?: () -> bool
|
|
5
30
|
end
|
|
6
31
|
end
|
data/sig/binpacker/report.rbs
CHANGED
|
@@ -3,7 +3,8 @@ module Binpacker
|
|
|
3
3
|
SCHEMA: Integer
|
|
4
4
|
DRIFT_LIMIT: Integer
|
|
5
5
|
|
|
6
|
-
def initialize: (profile: untyped, algorithm: untyped, predicted_loads: untyped, worker_stats: untyped, all_timings: untyped, timings: untyped) -> void
|
|
6
|
+
def initialize: (profile: untyped, algorithm: untyped, predicted_loads: untyped, worker_stats: untyped, all_timings: untyped, timings: untyped, ?shard: Binpacker::Shard?, ?discovered: Integer?, ?selected: Integer?) -> void
|
|
7
|
+
def shard_section: () -> Hash[Symbol, untyped]?
|
|
7
8
|
def to_h: () -> Hash[Symbol, untyped]
|
|
8
9
|
def write: (untyped) -> untyped
|
|
9
10
|
end
|
data/sig/binpacker/scheduler.rbs
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
module Binpacker
|
|
2
2
|
class Scheduler
|
|
3
3
|
def self.for: (untyped) -> (Binpacker::LptScheduler | Binpacker::MultifitScheduler)
|
|
4
|
+
|
|
5
|
+
# Declared on the base so a caller holding a `Scheduler` can partition; the base body raises
|
|
6
|
+
# NotImplementedError and every concrete scheduler overrides it.
|
|
7
|
+
def partition: (tests: Array[untyped], worker_count: Integer, timings: Hash[untyped, Float]) -> Array[Binpacker::WorkerQueue]
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
class LptScheduler < Scheduler
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
class MultifitScheduler < Scheduler
|
|
14
|
+
ITERATIONS: Integer
|
|
4
15
|
end
|
|
5
16
|
end
|