binpacker 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: dfee21fa79b8b157afbe89bba482a2a8c2b95c4ec22dbff36117a269275f47ef
4
- data.tar.gz: 636266dc852e315dffb3e1b815f39b615e4a73be4cfa9dac445a01ef7b7d76fc
3
+ metadata.gz: 991ac5d941b69573b4f6d03c211ff862465a91ee31ddd82f5ce8d8150a2d4f0d
4
+ data.tar.gz: 8ad2f245f73a46a4fdd64272049835c36d3b2dc4a0bb5df9cf67f851789ae59a
5
5
  SHA512:
6
- metadata.gz: 3eb4035f35100358451c5efa9ba5ae178932defb326a988d9d1e6da1521c79b201e68ba075f8bd2575f36a4d0ce33d92b40fd5117afbac21068e3eaabb93daf2
7
- data.tar.gz: 788f484e43e7d9c6d77519cabe924f6080e77ebff5e4fb1125e38721c165c1bf6bbdd6e0f4a9fa1140a3b1d62da770be746cbcd4f68dce15f36d0ab556d949c9
6
+ metadata.gz: 4345168c76cd0e072ab8f0775df54fe705e13f873a1f7f8ce764b897f42f91f75c5b4b0dee07329b577200230a7707a90284e73b554a339798575e084ce93f15
7
+ data.tar.gz: 40408cf8b88b878033226df711fbf3e57c217293b85b6eb7f94f55289eb7740d02dcc66598c5f594a3cd21821b5186eb38942c7e03747838a6487ccc37e018d1
@@ -13,10 +13,13 @@ module Binpacker
13
13
  @timing = Timing.new(config.timing_file)
14
14
  end
15
15
 
16
- def run(tests)
16
+ # Runs each Test serially and appends its measured Weight to the Timing file.
17
+ # With incremental: true, Tests that already have a measured Weight are skipped.
18
+ def run(tests, incremental: false)
19
+ targets = incremental ? unmeasured(tests) : tests
17
20
  results = []
18
21
 
19
- tests.each do |test|
22
+ targets.each do |test|
20
23
  elapsed = run_single(test)
21
24
  results << { file: test.file, name: test.name, time: elapsed }
22
25
  end
@@ -27,21 +30,30 @@ module Binpacker
27
30
 
28
31
  private
29
32
 
33
+ def unmeasured(tests)
34
+ measured = @timing.load_raw
35
+ tests.reject { |test| measured.key?([@timing.normalize_path(test.file), test.name]) }
36
+ end
37
+
30
38
  def run_single(test)
31
39
  outfile = Tempfile.new("binpacker-cal")
32
40
  outfile.close
33
41
 
34
42
  start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
35
43
 
44
+ # At file granularity the Test's name is its file path (see Test
45
+ # discovery), so there is no single example/method to filter — run the
46
+ # whole file. A per-test filter would match nothing and measure only
47
+ # load time.
48
+ file_unit = test.name == test.file
49
+
36
50
  case @config.test_runner
37
51
  when "rspec"
38
- cmd = [
39
- "rspec", test.file,
40
- "--example", test.name,
41
- "--format", "json", "--out", outfile.path
42
- ]
52
+ cmd = ["rspec", test.file, "--format", "json", "--out", outfile.path]
53
+ cmd.push("--example", test.name) unless file_unit
43
54
  when "minitest"
44
- cmd = ["ruby", "-Ilib:test", test.file, "--name", "/^#{Regexp.escape(test.name)}$/"]
55
+ cmd = ["ruby", "-Ilib:test", test.file]
56
+ cmd.push("--name", "/^#{Regexp.escape(test.name)}$/") unless file_unit
45
57
  else
46
58
  raise ConfigError, "unsupported runner for calibration: #{@config.test_runner}"
47
59
  end
data/lib/binpacker/cli.rb CHANGED
@@ -15,6 +15,12 @@ module Binpacker
15
15
  @profile = nil
16
16
  @passthrough = []
17
17
  @quiet = false
18
+ @incremental = false
19
+ @report_path = nil
20
+ @skill_list = false
21
+ @skill_path = false
22
+ @skill_describe = false
23
+ @skill_name = nil
18
24
  parse!
19
25
  end
20
26
 
@@ -26,6 +32,10 @@ module Binpacker
26
32
  cmd_run
27
33
  when "init"
28
34
  cmd_init
35
+ when "skill"
36
+ cmd_skill
37
+ when "describe"
38
+ cmd_describe
29
39
  when "--version", "-v"
30
40
  puts "binpacker #{Binpacker::VERSION}"
31
41
  when "--help", "-h", nil
@@ -58,6 +68,26 @@ module Binpacker
58
68
  opts.on("--quiet", "Suppress worker output") do
59
69
  @quiet = true
60
70
  end
71
+
72
+ opts.on("--incremental", "Calibrate only tests without timing data") do
73
+ @incremental = true
74
+ end
75
+
76
+ opts.on("--report PATH", "(run) Write a JSON run report to PATH") do |v|
77
+ @report_path = v
78
+ end
79
+
80
+ opts.on("--list", "(skill) List bundled skills") do
81
+ @skill_list = true
82
+ end
83
+
84
+ opts.on("--path", "(skill) Print the SKILL.md path for <name>") do
85
+ @skill_path = true
86
+ end
87
+
88
+ opts.on("--describe", "(skill) Report project state and next skill") do
89
+ @skill_describe = true
90
+ end
61
91
  end
62
92
 
63
93
  # Extract passthrough arguments after "--"
@@ -67,7 +97,10 @@ module Binpacker
67
97
  end
68
98
 
69
99
  remaining = parser.parse(@args)
70
- @command = remaining.shift
100
+ # A positional command wins; otherwise keep any command an option set
101
+ # (e.g. --version), so `binpacker --version` isn't clobbered to nil.
102
+ @command = remaining.shift || @command
103
+ @skill_name = remaining.shift
71
104
  end
72
105
 
73
106
  def cmd_init
@@ -78,6 +111,11 @@ module Binpacker
78
111
  end
79
112
 
80
113
  framework = detect_framework
114
+ unless ProjectState::SUPPORTED_FRAMEWORKS.include?(framework)
115
+ $stderr.puts "Detected #{framework} — binpacker supports rspec and minitest only."
116
+ $stderr.puts "No binpacker.yml was created."
117
+ exit 1
118
+ end
81
119
  pattern = framework == "minitest" ? "test/**/*_test.rb" : "spec/**/*_spec.rb"
82
120
  runner = framework
83
121
 
@@ -89,7 +127,7 @@ module Binpacker
89
127
  timing_file: binpacker.timings
90
128
  test_pattern: "#{pattern}"
91
129
  scheduler:
92
- algorithm: lpt
130
+ algorithm: multifit
93
131
  steal_enabled: true
94
132
  ci:
95
133
  extends: default
@@ -110,21 +148,98 @@ module Binpacker
110
148
  discovery_klass = config.test_runner == "rspec" ? RSpecDiscovery : MinitestDiscovery
111
149
  tests = discovery_klass.new(config).enumerate
112
150
 
113
- puts "Calibrating #{tests.size} tests..."
114
151
  cal = Calibration.new(config)
115
- timings = cal.run(tests)
152
+
153
+ if @incremental
154
+ puts "Calibrating (incremental) #{tests.size} tests..."
155
+ else
156
+ puts "Calibrating #{tests.size} tests..."
157
+ end
158
+
159
+ timings = cal.run(tests, incremental: @incremental)
116
160
 
117
161
  total = timings.sum { |t| t[:time] }
118
- puts "Calibration complete: #{tests.size} tests in #{total.round(2)}s"
162
+ measured = timings.size
163
+ skipped = tests.size - measured
164
+ summary = "Calibration complete: #{measured} #{pluralize(measured, 'test')} in #{total.round(2)}s"
165
+ summary += " (#{skipped} skipped, already measured)" if @incremental && skipped.positive?
166
+ puts summary
119
167
  puts "Timing data written to #{config.timing_file}"
120
168
  end
121
169
 
170
+ def cmd_skill
171
+ return cmd_describe if @skill_describe
172
+
173
+ if @skill_path
174
+ path = Skills.path(@skill_name)
175
+ if path
176
+ puts path
177
+ else
178
+ $stderr.puts "unknown skill: #{@skill_name}"
179
+ exit 1
180
+ end
181
+ elsif @skill_name && !@skill_list
182
+ print_skill(@skill_name)
183
+ else
184
+ list_skills
185
+ end
186
+ end
187
+
188
+ def cmd_describe
189
+ state = ProjectState.new
190
+ puts "binpacker project state:"
191
+ puts " config (binpacker.yml): #{state.config_present? ? 'present' : 'missing'}"
192
+ puts " timing data: #{state.timing_present? ? 'present' : 'missing'}"
193
+ puts " test framework: #{state.framework || 'not detected'}"
194
+ puts " CI wired for binpacker: #{state.ci_wired? ? 'yes' : 'no'}"
195
+ puts ""
196
+
197
+ unless state.supported_framework?
198
+ puts "binpacker supports rspec and minitest; #{state.framework} is not supported yet."
199
+ puts "Setup cannot proceed against a #{state.framework} suite."
200
+ return
201
+ end
202
+
203
+ rec = state.recommendation
204
+ puts "Recommended next skill: #{rec}"
205
+ puts "Run `binpacker skill #{rec}` to load its instructions."
206
+ end
207
+
208
+ def list_skills
209
+ skills = Skills.list
210
+ if skills.empty?
211
+ puts "No bundled skills found."
212
+ return
213
+ end
214
+ puts "Bundled skills:"
215
+ skills.each { |s| puts " #{s[:name]}" }
216
+ puts ""
217
+ puts "Print a skill's instructions with `binpacker skill <name>`."
218
+ end
219
+
220
+ def print_skill(name)
221
+ unless Skills.exist?(name)
222
+ $stderr.puts "unknown skill: #{name}"
223
+ $stderr.puts "Run `binpacker skill` to list bundled skills."
224
+ exit 1
225
+ end
226
+
227
+ puts "# Skill: #{name}"
228
+ puts "# Source: #{Skills.path(name)}"
229
+ puts ""
230
+ puts Skills.body(name)
231
+ end
232
+
122
233
  def cmd_run
123
234
  config = Config.new(profile: @profile)
124
- orchestrator = Orchestrator.new(config, passthrough: @passthrough, quiet: @quiet)
235
+ report_path = @report_path || config.report_file
236
+ orchestrator = Orchestrator.new(
237
+ config, passthrough: @passthrough, quiet: @quiet, report_path: report_path
238
+ )
125
239
 
126
240
  puts "binpacker starting (#{config.worker_count} workers, profile: #{config.profile})"
127
241
  result = orchestrator.run
242
+ puts "Run report written to #{report_path}" if report_path
128
243
  unit = test_unit_label(config)
129
244
 
130
245
  if result[:passed]
@@ -156,23 +271,30 @@ module Binpacker
156
271
  run Execute tests across worker processes
157
272
  calibrate Run tests serially to generate timing data
158
273
  init Create binpacker.yml with auto-detected settings
274
+ describe Report project state and recommend the next skill
275
+ skill List or print bundled agent skills
159
276
 
160
277
  Options:
161
278
  --profile NAME Select profile from binpacker.yml
279
+ --incremental (calibrate) Measure only tests without timing data
280
+ --report PATH (run) Write a JSON run report to PATH
281
+ --list (skill) List bundled skills
282
+ --path (skill) Print the SKILL.md path for <name>
283
+ --describe (skill) Report project state and next skill
162
284
  --help Show this message
163
285
 
164
286
  Examples:
165
287
  binpacker init
166
288
  binpacker run --profile ci
167
289
  binpacker run -- --tag ~slow
168
- binpacker calibrate
290
+ binpacker calibrate --incremental
291
+ binpacker describe
292
+ binpacker skill binpacker-setup
169
293
  HELP
170
294
  end
171
295
 
172
296
  def detect_framework
173
- return "minitest" if Dir.glob("test*/**/*_test.rb").any?
174
- return "minitest" if Dir.glob("test*/**/test_*.rb").any?
175
- "rspec"
297
+ ProjectState.new.framework || "rspec"
176
298
  end
177
299
  end
178
300
  end
@@ -14,10 +14,11 @@ module Binpacker
14
14
  "timing_file" => "binpacker.timings",
15
15
  "test_pattern" => "spec/**/*_spec.rb",
16
16
  "test_exclude" => [],
17
+ "report_file" => nil,
17
18
  "scheduler" => {
18
19
  "strategy" => "static",
19
20
  "steal_enabled" => false,
20
- "algorithm" => "lpt"
21
+ "algorithm" => "multifit"
21
22
  }
22
23
  }.freeze
23
24
 
@@ -4,16 +4,18 @@ module Binpacker
4
4
  class Orchestrator
5
5
  BATCH_SIZE = 10
6
6
 
7
- def initialize(config, passthrough: [], quiet: false)
7
+ def initialize(config, passthrough: [], quiet: false, report_path: nil)
8
8
  @config = config
9
9
  @passthrough = passthrough
10
10
  @quiet = quiet
11
+ @report_path = report_path
11
12
  end
12
13
 
13
14
  def run
14
15
  tests = discover
15
16
  timing = Timing.new(@config.timing_file)
16
17
  timings = timing.load_with_fallback(tests)
18
+ @timings = timings
17
19
 
18
20
  scheduler = Scheduler.for(@config.scheduler["algorithm"])
19
21
  queues = scheduler.partition(
@@ -22,6 +24,9 @@ module Binpacker
22
24
  timings: timings
23
25
  )
24
26
 
27
+ # Capture predicted per-Worker loads before execution drains the queues.
28
+ @predicted_loads = queues.map { |q| q.total_weight(timings) }
29
+
25
30
  runner_class = TestRunner.for(@config.test_runner)
26
31
  workers = queues.map.with_index do |queue, idx|
27
32
  Worker.new(idx, runner_class, passthrough: @passthrough, quiet: @quiet).tap(&:start)
@@ -96,6 +101,7 @@ module Binpacker
96
101
  }
97
102
  end
98
103
  progress.summary(worker_stats)
104
+ write_report(worker_stats, all_timings)
99
105
 
100
106
  finalize(timing, all_timings, all_passed, total_examples, passed_examples, tests)
101
107
  end
@@ -199,11 +205,25 @@ module Binpacker
199
205
  }
200
206
  end
201
207
  progress.summary(worker_stats)
208
+ write_report(worker_stats, all_timings)
202
209
 
203
210
  workers.each(&:cleanup)
204
211
  finalize(timing, all_timings, all_passed, total_examples, passed_examples, tests)
205
212
  end
206
213
 
214
+ def write_report(worker_stats, all_timings)
215
+ return unless @report_path
216
+
217
+ Report.new(
218
+ profile: @config.profile,
219
+ algorithm: @config.scheduler["algorithm"],
220
+ predicted_loads: @predicted_loads,
221
+ worker_stats: worker_stats,
222
+ all_timings: all_timings,
223
+ timings: @timings
224
+ ).write(@report_path)
225
+ end
226
+
207
227
  def drain_batch(queue)
208
228
  return [] if queue.nil? || queue.empty?
209
229
  batch = []
@@ -45,9 +45,9 @@ module Binpacker
45
45
  active.each_with_index do |s, i|
46
46
  t = format_time(s[:total_time])
47
47
  wid = worker_stats.index(s)
48
- $stdout.puts " Worker #{wid}: #{s[:files]} files, #{t} | #{s[:examples]} examples, #{s[:passed]} passed"
48
+ $stdout.puts " Worker #{wid}: #{s[:files]} tests, #{t} | #{s[:examples]} examples, #{s[:passed]} passed"
49
49
  end
50
- total_files = active.sum { |s| s[:files] }
50
+ total_tests = active.sum { |s| s[:files] }
51
51
  total_time = active.sum { |s| s[:total_time] }
52
52
  total_examples = active.sum { |s| s[:examples] }
53
53
  times = active.map { |s| s[:total_time] }
@@ -56,7 +56,7 @@ module Binpacker
56
56
  dev_pct = mean > 0 ? (max_dev / mean * 100).round(1) : 0
57
57
 
58
58
  $stdout.puts " ──"
59
- $stdout.puts " Total: #{total_files} files, #{format_time(total_time)} | #{total_examples} examples"
59
+ $stdout.puts " Total: #{total_tests} tests, #{format_time(total_time)} | #{total_examples} examples"
60
60
  $stdout.puts " Balance: max deviation #{format_time(max_dev)} (#{dev_pct}%)"
61
61
  end
62
62
 
@@ -99,14 +99,12 @@ module Binpacker
99
99
  @last_ci_output = now
100
100
 
101
101
  parts = @workers.map.with_index do |w, i|
102
- if w[:total] > 0 && w[:done] >= w[:total]
103
- "W#{i}: done"
104
- else
105
- "W#{i}: #{w[:done]}/#{w[:total]}"
106
- end
102
+ ratio = w[:total] > 0 ? "#{w[:done]}/#{w[:total]}" : "0/?"
103
+ "W#{i}: #{ratio}"
107
104
  end
108
105
  elapsed = (now - @start).round(1)
109
106
  $stdout.puts "[binpacker #{elapsed}s] #{parts.join(' | ')}"
107
+ $stdout.flush
110
108
  end
111
109
 
112
110
  def format_time(seconds)
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Binpacker
4
+ # Inspects the working directory to report binpacker's setup state and
5
+ # recommend the next skill. Backs `binpacker describe`.
6
+ # See docs/design/agent-workflows.md.
7
+ class ProjectState
8
+ CONFIG_FILE = "binpacker.yml"
9
+ DEFAULT_TIMING_FILE = "binpacker.timings"
10
+ SUPPORTED_FRAMEWORKS = %w[rspec minitest].freeze
11
+ TESTUNIT_HINT = %r{test-unit|test/unit}
12
+
13
+ def config_present?
14
+ File.exist?(CONFIG_FILE)
15
+ end
16
+
17
+ def timing_present?
18
+ File.exist?(timing_file)
19
+ end
20
+
21
+ # "rspec", "minitest", "test-unit", or nil. Minitest and test-unit share
22
+ # the *_test.rb convention, so they are told apart by dependency hints.
23
+ def framework
24
+ return minitest_family if minitest_globs?
25
+ return "rspec" if Dir.glob("spec/**/*_spec.rb").any?
26
+ nil
27
+ end
28
+
29
+ # binpacker ships runners for rspec and minitest only. A detected framework
30
+ # outside that set (e.g. test-unit) is unsupported; nil is merely unknown.
31
+ def supported_framework?
32
+ fw = framework
33
+ fw.nil? || SUPPORTED_FRAMEWORKS.include?(fw)
34
+ end
35
+
36
+ def ci_wired?
37
+ Dir.glob(".github/workflows/*.{yml,yaml}").any? do |wf|
38
+ File.read(wf).include?("binpacker run")
39
+ end
40
+ end
41
+
42
+ # The skill the agent should run next.
43
+ def recommendation
44
+ return "binpacker-setup" unless config_present? && timing_present?
45
+ "binpacker-improve"
46
+ end
47
+
48
+ def to_h
49
+ {
50
+ config_present: config_present?,
51
+ timing_present: timing_present?,
52
+ framework: framework,
53
+ ci_wired: ci_wired?,
54
+ recommendation: recommendation
55
+ }
56
+ end
57
+
58
+ private
59
+
60
+ def minitest_globs?
61
+ Dir.glob("test*/**/*_test.rb").any? || Dir.glob("test*/**/test_*.rb").any?
62
+ end
63
+
64
+ def minitest_family
65
+ testunit_hinted? ? "test-unit" : "minitest"
66
+ end
67
+
68
+ def testunit_hinted?
69
+ sources = Dir.glob("Gemfile") + Dir.glob("*.gemspec") + Dir.glob("test*/**/*_test.rb").first(5)
70
+ sources.any? { |file| File.read(file).match?(TESTUNIT_HINT) }
71
+ rescue StandardError
72
+ false
73
+ end
74
+
75
+ def timing_file
76
+ return DEFAULT_TIMING_FILE unless config_present?
77
+ Config.new.timing_file
78
+ rescue StandardError
79
+ DEFAULT_TIMING_FILE
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Binpacker
6
+ # Builds the machine-readable Run report: predicted versus actual per-Worker
7
+ # durations plus the worst per-Test Drift. See docs/design/run-report-format.md.
8
+ class Report
9
+ SCHEMA = 1
10
+ DRIFT_LIMIT = 10
11
+
12
+ def initialize(profile:, algorithm:, predicted_loads:, worker_stats:, all_timings:, timings:)
13
+ @profile = profile
14
+ @algorithm = algorithm
15
+ @predicted_loads = predicted_loads
16
+ @worker_stats = worker_stats
17
+ @all_timings = all_timings
18
+ @timings = timings
19
+ end
20
+
21
+ def to_h
22
+ predicted = @predicted_loads
23
+ actual = @worker_stats.map { |s| s[:total_time] }
24
+
25
+ {
26
+ schema: SCHEMA,
27
+ profile: @profile,
28
+ algorithm: @algorithm,
29
+ worker_count: @worker_stats.size,
30
+ predicted_makespan: round(predicted.max || 0.0),
31
+ actual_makespan: round(actual.max || 0.0),
32
+ workers: workers,
33
+ balance: {
34
+ predicted_deviation_pct: deviation_pct(predicted),
35
+ actual_deviation_pct: deviation_pct(actual)
36
+ },
37
+ drift: drift
38
+ }
39
+ end
40
+
41
+ def write(path)
42
+ File.write(path, JSON.pretty_generate(to_h))
43
+ end
44
+
45
+ private
46
+
47
+ def workers
48
+ @worker_stats.map.with_index do |s, i|
49
+ {
50
+ id: i,
51
+ predicted: round(@predicted_loads[i] || 0.0),
52
+ actual: round(s[:total_time]),
53
+ tests: s[:files],
54
+ examples: s[:examples]
55
+ }
56
+ end
57
+ end
58
+
59
+ # Drift is reported per file (the scheduling unit), so predicted and actual
60
+ # are compared at the same granularity even when timings are recorded
61
+ # per example. Both sides are aggregated to normalized file paths.
62
+ def drift
63
+ actual = aggregate_by_file(@all_timings.map { |e| [e[:file], e[:time]] })
64
+ predicted = aggregate_by_file(@timings.map { |(file, _name), weight| [file, weight] })
65
+
66
+ (actual.keys | predicted.keys)
67
+ .map { |file| drift_entry(file, predicted[file], actual[file]) }
68
+ .sort_by { |d| -(d[:actual] - d[:predicted]).abs }
69
+ .first(DRIFT_LIMIT)
70
+ end
71
+
72
+ def drift_entry(file, predicted, actual)
73
+ {
74
+ file: file,
75
+ predicted: round(predicted || Timing::DEFAULT_WEIGHT),
76
+ actual: round(actual || 0.0)
77
+ }
78
+ end
79
+
80
+ def aggregate_by_file(pairs)
81
+ pairs.each_with_object(Hash.new(0.0)) do |(file, time), acc|
82
+ acc[normalize_file(file)] += time
83
+ end
84
+ end
85
+
86
+ def normalize_file(file)
87
+ file.to_s.sub(%r{\A\./}, "")
88
+ end
89
+
90
+ def deviation_pct(loads)
91
+ return 0.0 if loads.empty?
92
+ mean = loads.sum / loads.size
93
+ return 0.0 unless mean.positive?
94
+ max_dev = loads.map { |t| (t - mean).abs }.max
95
+ round(max_dev / mean * 100)
96
+ end
97
+
98
+ def round(value)
99
+ value.to_f.round(3)
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Binpacker
4
+ # Registry for the gem-shipped, user-facing agent skills. Skills live in the
5
+ # gem's top-level skills/ directory, each as skills/<name>/SKILL.md.
6
+ # See docs/adr/0003-agent-driven-install-and-skills.md.
7
+ module Skills
8
+ ROOT = File.expand_path("../../skills", __dir__)
9
+
10
+ module_function
11
+
12
+ # [{ name:, path: }], sorted by name.
13
+ def list
14
+ Dir.glob(File.join(ROOT, "*", "SKILL.md")).sort.map do |file|
15
+ { name: File.basename(File.dirname(file)), path: file }
16
+ end
17
+ end
18
+
19
+ def exist?(name)
20
+ !path(name).nil?
21
+ end
22
+
23
+ # Absolute path of a skill's SKILL.md, or nil when unknown.
24
+ def path(name)
25
+ file = File.join(ROOT, name, "SKILL.md")
26
+ File.file?(file) ? file : nil
27
+ end
28
+
29
+ # The SKILL.md body for a skill.
30
+ def body(name)
31
+ file = path(name)
32
+ raise Error, "unknown skill: #{name}" unless file
33
+ File.read(file)
34
+ end
35
+ end
36
+ end
@@ -69,7 +69,13 @@ module Binpacker
69
69
 
70
70
  class MinitestDiscovery < TestDiscovery
71
71
  def enumerate
72
- require "minitest"
72
+ begin
73
+ require "minitest"
74
+ rescue LoadError
75
+ raise DiscoveryError,
76
+ "minitest is not available. binpacker supports rspec and minitest; " \
77
+ "test-unit and other frameworks are not supported."
78
+ end
73
79
  add_project_load_paths
74
80
  def Minitest.autorun; end
75
81
  Minitest.seed = 42
@@ -44,6 +44,11 @@ module Binpacker
44
44
  Pathname(path).cleanpath.to_s.sub(/\A\.\//, "")
45
45
  end
46
46
 
47
+ # True when a measured Weight already exists for this Test.
48
+ def measured?(file:, name:)
49
+ load_raw.key?([normalize_path(file), name])
50
+ end
51
+
47
52
  def weight_for(file:, name:)
48
53
  measured = load_raw
49
54
  measured.fetch([file, name]) { filesize_weight(file) }
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Binpacker
4
- VERSION = "0.2.0"
4
+ VERSION = "0.3.0"
5
5
  end
data/lib/binpacker.rb CHANGED
@@ -9,6 +9,9 @@ require_relative "binpacker/scheduler"
9
9
  require_relative "binpacker/worker"
10
10
  require_relative "binpacker/test_runner"
11
11
  require_relative "binpacker/calibration"
12
+ require_relative "binpacker/report"
13
+ require_relative "binpacker/skills"
14
+ require_relative "binpacker/project_state"
12
15
  require_relative "binpacker/orchestrator"
13
16
  require_relative "binpacker/progress"
14
17
 
@@ -0,0 +1,6 @@
1
+ module Binpacker
2
+ class Calibration
3
+ def initialize: (untyped) -> void
4
+ def run: (untyped, ?incremental: bool) -> Array[untyped]
5
+ end
6
+ end
@@ -0,0 +1,7 @@
1
+ module Binpacker
2
+ class CLI
3
+ def self.start: (untyped) -> nil
4
+ def initialize: (untyped) -> void
5
+ def run: () -> nil
6
+ end
7
+ end
@@ -0,0 +1,13 @@
1
+ module Binpacker
2
+ class Config
3
+ def initialize: (?profile: untyped, ?config_path: untyped) -> void
4
+ def worker_count: () -> Integer
5
+ def test_runner: () -> String
6
+ def timing_file: () -> String
7
+ def test_pattern: () -> String
8
+ def test_exclude: () -> Array[String]
9
+ def test_granularity: () -> String
10
+ def report_file: () -> String?
11
+ def profile: () -> String
12
+ end
13
+ end
@@ -0,0 +1,6 @@
1
+ module Binpacker
2
+ class Orchestrator
3
+ def initialize: (untyped, ?passthrough: untyped, ?quiet: untyped, ?report_path: untyped) -> void
4
+ def run: () -> { passed: untyped, total: untyped, passed_count: untyped, timings: Array[untyped], empty_filter: untyped }
5
+ end
6
+ end
@@ -0,0 +1,5 @@
1
+ module Binpacker
2
+ class ProgressDisplay
3
+ def initialize: (untyped, ?tty: untyped) -> void
4
+ end
5
+ end
@@ -0,0 +1,16 @@
1
+ module Binpacker
2
+ class ProjectState
3
+ CONFIG_FILE: String
4
+ DEFAULT_TIMING_FILE: String
5
+ SUPPORTED_FRAMEWORKS: Array[String]
6
+ TESTUNIT_HINT: Regexp
7
+
8
+ def config_present?: () -> bool
9
+ def timing_present?: () -> bool
10
+ def framework: () -> String?
11
+ def supported_framework?: () -> bool
12
+ def ci_wired?: () -> bool
13
+ def recommendation: () -> String
14
+ def to_h: () -> Hash[Symbol, untyped]
15
+ end
16
+ end
@@ -0,0 +1,10 @@
1
+ module Binpacker
2
+ class Report
3
+ SCHEMA: Integer
4
+ DRIFT_LIMIT: Integer
5
+
6
+ def initialize: (profile: untyped, algorithm: untyped, predicted_loads: untyped, worker_stats: untyped, all_timings: untyped, timings: untyped) -> void
7
+ def to_h: () -> Hash[Symbol, untyped]
8
+ def write: (untyped) -> untyped
9
+ end
10
+ end
@@ -0,0 +1,5 @@
1
+ module Binpacker
2
+ class Scheduler
3
+ def self.for: (untyped) -> (Binpacker::LptScheduler | Binpacker::MultifitScheduler)
4
+ end
5
+ end
@@ -0,0 +1,10 @@
1
+ module Binpacker
2
+ module Skills
3
+ ROOT: String
4
+
5
+ def self.list: () -> Array[Hash[Symbol, String]]
6
+ def self.exist?: (untyped) -> bool
7
+ def self.path: (untyped) -> String?
8
+ def self.body: (untyped) -> String
9
+ end
10
+ end
@@ -0,0 +1,15 @@
1
+ module Binpacker
2
+ def key: () -> [untyped, String]
3
+ class TestDiscovery
4
+ def initialize: (untyped) -> void
5
+ def enumerate: () -> Array[Binpacker::Test]
6
+ end
7
+ class RSpecDiscovery < TestDiscovery
8
+ def enumerate: () -> Array[Binpacker::Test]
9
+ end
10
+ class MinitestDiscovery < TestDiscovery
11
+ def enumerate: () -> Array[Binpacker::Test]
12
+ end
13
+ class Test
14
+ end
15
+ end
@@ -0,0 +1,13 @@
1
+ module Binpacker
2
+ class TestRunner
3
+ def self.runner_name: () -> bot
4
+ def self.for: (untyped) -> (singleton(Binpacker::MinitestRunner) | singleton(Binpacker::RSpecRunner))
5
+ end
6
+ class RSpecRunner
7
+ def self.runner_name: () -> "rspec"
8
+ def self.discovery_command: (untyped) -> Array[untyped]
9
+ end
10
+ class MinitestRunner
11
+ def self.runner_name: () -> "minitest"
12
+ end
13
+ end
@@ -0,0 +1,11 @@
1
+ module Binpacker
2
+ class Timing
3
+ def initialize: (untyped) -> void
4
+ def load_raw: () -> (Hash[[String, untyped], untyped] | {})
5
+ def load_per_file: () -> (Hash[String, untyped] | {})
6
+ def normalize_path: (untyped) -> String
7
+ def measured?: (file: untyped, name: untyped) -> bool
8
+ class Entry
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,12 @@
1
+ module Binpacker
2
+ class Worker
3
+ def initialize: (untyped, untyped, ?passthrough: untyped, ?quiet: untyped) -> void
4
+ def start: () -> Binpacker::Worker
5
+ def send_test: (untyped) -> nil
6
+ def batch_done: () -> nil
7
+ def wait_for_batch: () -> (nil | { timings: [], exit_code: 0 | 1, examples: untyped, passed: untyped } | { timings: [], exit_code: untyped, examples: untyped, passed: untyped })
8
+ def collect_results: () -> :finished
9
+ def timings: () -> Array[untyped]
10
+ def status: () -> (:crashed | :created | :error | :finished | :ready | :running)
11
+ end
12
+ end
@@ -0,0 +1,6 @@
1
+ module Binpacker
2
+ class WorkerQueue
3
+ def initialize: (untyped, ?untyped) -> void
4
+ def empty?: () -> (false | true)
5
+ end
6
+ end
@@ -0,0 +1,52 @@
1
+ ---
2
+ name: binpacker-improve
3
+ description: Tune an already-configured binpacker project from real CI data — fetch recent run reports, analyze the gap between predicted and actual durations, and propose binpacker.yml adjustments. Use when binpacker is set up and running in CI and the user wants to reduce makespan, fix worker imbalance, or refresh stale timing data. Propose-only — never apply changes without confirmation.
4
+ ---
5
+
6
+ # Binpacker Improve
7
+
8
+ A recurring, **propose-only** workflow. It reads real CI data and proposes config changes; it never edits `binpacker.yml`, commits, or pushes without explicit user confirmation.
9
+
10
+ Run `binpacker describe` first. If it recommends `binpacker-setup`, the project isn't configured yet — use that skill instead.
11
+
12
+ ## 1. Gather CI data
13
+
14
+ Fetch recent run reports (see `docs/design/run-report-format.md` for the schema) from CI artifacts:
15
+
16
+ ```sh
17
+ gh run list --workflow <ci-workflow> --limit 10
18
+ gh run download <run-id> --name <report-artifact>
19
+ ```
20
+
21
+ Collect several recent reports so you can see trends, not just one run. Each report holds `predicted_makespan`, `actual_makespan`, per-worker predicted/actual durations, `balance`, and the worst-drift Tests.
22
+
23
+ ## 2. Analyze
24
+
25
+ Look for patterns across the reports:
26
+
27
+ - **Persistent worker skew** — the same worker is consistently slowest → the partition or worker count is off.
28
+ - **Predicted vs actual gap** — `actual_makespan` far above `predicted_makespan` → timing data is stale; the scheduler is planning against wrong Weights.
29
+ - **Recurring high-drift files** — the same files top the `drift` list every run → their timing data is stale or they are outliers.
30
+ - **Cache-miss frequency** — if CI logs show frequent timing-cache misses (e.g. after dependency bumps), the runtime cache isn't helping cold starts.
31
+
32
+ ## 3. Propose
33
+
34
+ Present a concrete `binpacker.yml` diff and/or commands, with the evidence behind each change. Typical levers:
35
+
36
+ - **`workers`** — raise/lower the `ci` worker count when actual makespan shows consistent under- or over-utilisation.
37
+ - **`scheduler.algorithm`** — `lpt` ↔ `multifit`. Prefer `multifit` for tighter balance; only suggest `lpt` if `multifit` isn't helping.
38
+ - **`scheduler.steal_enabled`** — enable when static balance is poor but runtime stealing would recover it.
39
+ - **Re-calibrate** — when drift is high or timing is stale, recommend `binpacker calibrate --incremental` to refresh the drifting Tests.
40
+ - **Commit a baseline** — when the cache misses often (dependency-bump churn) or timings have stabilised, recommend committing `binpacker.timings` as a durable baseline (cold-start floor). See `docs/adr/0005-two-tier-timing-persistence.md`.
41
+
42
+ ## 4. Confirm and apply
43
+
44
+ - Show the diff. Explain the expected effect (e.g. "should cut max deviation from 6% toward <3%").
45
+ - Apply only what the user approves. Committing a baseline or editing config both require confirmation.
46
+ - If the user wants, open a PR with `gh`; otherwise leave the working tree changes for them to review.
47
+
48
+ ## Done when
49
+
50
+ - The user has seen the predicted-vs-actual analysis with evidence.
51
+ - A concrete, confirmed set of changes (or an explicit "no change needed") has been reached.
52
+ - Any commit / PR was made only with confirmation.
@@ -0,0 +1,76 @@
1
+ ---
2
+ name: binpacker-setup
3
+ description: Bring a project from zero to a working parallel test run with binpacker — detect the framework, generate config, calibrate timing data, validate a parallel run, and wire CI. Use when binpacker is installed but not yet set up (no binpacker.yml, or no timing data), or when the user asks to set up / configure / adopt binpacker.
4
+ ---
5
+
6
+ # Binpacker Setup
7
+
8
+ A one-time workflow that takes a project from zero to a working parallel test run. Run `binpacker describe` first — if it recommends `binpacker-improve`, the project is already set up and you want that skill instead.
9
+
10
+ Assess state as you go and **ask the user at each decision point** rather than assuming. Do not run destructive or outward-facing steps (committing, opening a PR) without explicit confirmation.
11
+
12
+ ## 1. Detect
13
+
14
+ - Confirm binpacker is installed: `binpacker --version`.
15
+ - Run `binpacker describe` to read current state (config present? timing data? framework? CI wired?).
16
+ - Determine the test framework (rspec / minitest) and the test glob. If `binpacker describe` reports one, trust it; otherwise inspect the repo.
17
+
18
+ ## 2. Config
19
+
20
+ If `binpacker.yml` is absent, create it with `binpacker init` (it auto-detects the framework and writes `default` + `ci` profiles). Then review the generated file with the user and adjust:
21
+
22
+ - `workers` — `auto` locally; a fixed number (e.g. `4`) for the `ci` profile matching the CI runner.
23
+ - `test_pattern` / `test_exclude` — match the project's real layout.
24
+ - `scheduler.algorithm` — `multifit` (default) unless there's a reason to use `lpt`.
25
+ - `scheduler.steal_enabled` — `true` to let idle workers steal at runtime.
26
+ - `report_file` (on the `ci` profile) — e.g. `binpacker-report.json`, so CI always emits a run report.
27
+
28
+ ## 3. Calibrate
29
+
30
+ Seed timing data so the scheduler has real Weights:
31
+
32
+ ```sh
33
+ binpacker calibrate
34
+ ```
35
+
36
+ If some timing data already exists, measure only the gaps:
37
+
38
+ ```sh
39
+ binpacker calibrate --incremental
40
+ ```
41
+
42
+ ## 4. Validate
43
+
44
+ Run the suite in parallel and confirm it is green and balanced:
45
+
46
+ ```sh
47
+ binpacker run
48
+ ```
49
+
50
+ Check the summary's `Balance: max deviation`. A large deviation on the first run is expected before timing data accumulates; note it and move on.
51
+
52
+ ## 5. Wire CI
53
+
54
+ Add (or extend) a CI workflow so the suite runs under binpacker. The workflow should:
55
+
56
+ - Install binpacker (via the project's dependency manager or `gem install binpacker`).
57
+ - Restore and save `binpacker.timings` with `actions/cache` (key includes OS + a hash of the test files). This is the runtime timing layer.
58
+ - Run `binpacker run --profile ci` (which emits the run report via `report_file`).
59
+ - Upload the run report with `actions/upload-artifact` so `binpacker-improve` can read it later.
60
+
61
+ Propose the workflow YAML and let the user review it before writing.
62
+
63
+ ## 6. Handoff
64
+
65
+ Only after the user confirms:
66
+
67
+ - Create a branch, commit `binpacker.yml`, the CI workflow, and (if the user wants a committed baseline) `binpacker.timings`.
68
+ - Open a PR with `gh`, summarising what was set up.
69
+ - After CI runs, point the user at `binpacker-improve` to tune from real CI data.
70
+
71
+ ## Done when
72
+
73
+ - `binpacker run` passes locally across workers.
74
+ - `binpacker.yml` reflects the project's framework, layout, and CI worker count.
75
+ - CI runs `binpacker run` and uploads a run report.
76
+ - The user has confirmed any commit / PR.
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: binpacker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - megurine
@@ -9,8 +9,9 @@ bindir: exe
9
9
  cert_chain: []
10
10
  date: 1980-01-02 00:00:00.000000000 Z
11
11
  dependencies: []
12
- description: A test runner wrapper that manages worker processes and distributes tests
13
- among them using LPT scheduling with optional work-stealing.
12
+ description: A test runner wrapper that models test distribution as an identical-machines
13
+ scheduling problem and assigns tests to worker processes using LPT scheduling, with
14
+ optional work-stealing.
14
15
  executables:
15
16
  - binpacker
16
17
  extensions: []
@@ -25,13 +26,32 @@ files:
25
26
  - lib/binpacker/config.rb
26
27
  - lib/binpacker/orchestrator.rb
27
28
  - lib/binpacker/progress.rb
29
+ - lib/binpacker/project_state.rb
30
+ - lib/binpacker/report.rb
28
31
  - lib/binpacker/scheduler.rb
32
+ - lib/binpacker/skills.rb
29
33
  - lib/binpacker/test_discovery.rb
30
34
  - lib/binpacker/test_runner.rb
31
35
  - lib/binpacker/timing.rb
32
36
  - lib/binpacker/version.rb
33
37
  - lib/binpacker/worker.rb
34
38
  - lib/binpacker/worker_queue.rb
39
+ - sig/binpacker/calibration.rbs
40
+ - sig/binpacker/cli.rbs
41
+ - sig/binpacker/config.rbs
42
+ - sig/binpacker/orchestrator.rbs
43
+ - sig/binpacker/progress.rbs
44
+ - sig/binpacker/project_state.rbs
45
+ - sig/binpacker/report.rbs
46
+ - sig/binpacker/scheduler.rbs
47
+ - sig/binpacker/skills.rbs
48
+ - sig/binpacker/test_discovery.rbs
49
+ - sig/binpacker/test_runner.rbs
50
+ - sig/binpacker/timing.rbs
51
+ - sig/binpacker/worker.rbs
52
+ - sig/binpacker/worker_queue.rbs
53
+ - skills/binpacker-improve/SKILL.md
54
+ - skills/binpacker-setup/SKILL.md
35
55
  homepage: https://github.com/rigortype/binpacker
36
56
  licenses:
37
57
  - MPL-2.0
@@ -54,6 +74,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
54
74
  requirements: []
55
75
  rubygems_version: 4.0.10
56
76
  specification_version: 4
57
- summary: Minimize CI test-suite makespan by solving an identical-machines scheduling
58
- problem
77
+ summary: Reduce CI test-suite makespan via identical-machines scheduling
59
78
  test_files: []