binpacker 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfee21fa79b8b157afbe89bba482a2a8c2b95c4ec22dbff36117a269275f47ef
-  data.tar.gz: 636266dc852e315dffb3e1b815f39b615e4a73be4cfa9dac445a01ef7b7d76fc
+  metadata.gz: 98842f3601fe059ff372b62a214ee73e5e96d0b2c01cf5b651063f04e3f00184
+  data.tar.gz: 94e73ac26474bf1da1942509f35618e8576d230b685ee69fe09579d4b9387898
 SHA512:
-  metadata.gz: 3eb4035f35100358451c5efa9ba5ae178932defb326a988d9d1e6da1521c79b201e68ba075f8bd2575f36a4d0ce33d92b40fd5117afbac21068e3eaabb93daf2
-  data.tar.gz: 788f484e43e7d9c6d77519cabe924f6080e77ebff5e4fb1125e38721c165c1bf6bbdd6e0f4a9fa1140a3b1d62da770be746cbcd4f68dce15f36d0ab556d949c9
+  metadata.gz: 6a983e121b41c7a9e62dad700ee5541813b84c2553e7fc479c3dbedf996358669a9508d2fd7f3eb8df21831c583421ee91382a453ac73a60beead9bf03da0675
+  data.tar.gz: 657ac41276da7094a5ea54d53936a9237b1b3139f53b741c954427b57f2d781dca56df91746972e33e93f49edb7a93a2155d5b92561653d672f87ca73373669a

data/exe/binpacker-worker

@@ -166,6 +166,16 @@ end.parse!
 
 wid = options[:id]
 exit_code = 0
+
+# Test code must never see the orchestrator's control pipe. A test
+# that reads stdin — an LSP server loop, anything calling gets —
+# would block forever on the pipe the orchestrator keeps open
+# between dynamic batches (or, worse, consume control messages).
+# Keep the pipe on a private handle and point process stdin, which
+# spawned test runners inherit, at /dev/null so readers see EOF.
+control = $stdin.dup
+$stdin.reopen(File::NULL)
+
 $stderr.puts "[worker-#{wid}] #{options[:runner]} worker started"
 
 loop do
@@ -174,7 +184,7 @@ loop do
 
   tests = []
   done_or_eof = false
-  $stdin.each_line do |line|
+  control.each_line do |line|
     data = JSON.parse(line.strip)
     if data["type"] == "done"
       done_or_eof = true
@@ -186,7 +196,7 @@ loop do
   end
 
   break if done_or_eof && tests.empty?
-  break if tests.empty? && ($stdin.eof? || $stdin.closed?)
+  break if tests.empty? && (control.eof? || control.closed?)
 
   files = tests.map { |t| t["file"] }.uniq
   $stderr.puts "[worker-#{wid}] batch: #{files.size} files, #{tests.size} entries"

data/lib/binpacker/calibration.rb

@@ -13,10 +13,13 @@ module Binpacker
       @timing = Timing.new(config.timing_file)
     end
 
-    def run(tests)
+    # Runs each Test serially and appends its measured Weight to the Timing file.
+    # With incremental: true, Tests that already have a measured Weight are skipped.
+    def run(tests, incremental: false)
+      targets = incremental ? unmeasured(tests) : tests
       results = []
 
-      tests.each do |test|
+      targets.each do |test|
         elapsed = run_single(test)
         results << { file: test.file, name: test.name, time: elapsed }
       end
@@ -27,21 +30,30 @@ module Binpacker
 
     private
 
+    def unmeasured(tests)
+      measured = @timing.load_raw
+      tests.reject { |test| measured.key?([@timing.normalize_path(test.file), test.name]) }
+    end
+
     def run_single(test)
       outfile = Tempfile.new("binpacker-cal")
       outfile.close
 
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
+      # At file granularity the Test's name is its file path (see Test
+      # discovery), so there is no single example/method to filter — run the
+      # whole file. A per-test filter would match nothing and measure only
+      # load time.
+      file_unit = test.name == test.file
+
       case @config.test_runner
       when "rspec"
-        cmd = [
-          "rspec", test.file,
-          "--example", test.name,
-          "--format", "json", "--out", outfile.path
-        ]
+        cmd = ["rspec", test.file, "--format", "json", "--out", outfile.path]
+        cmd.push("--example", test.name) unless file_unit
       when "minitest"
-        cmd = ["ruby", "-Ilib:test", test.file, "--name", "/^#{Regexp.escape(test.name)}$/"]
+        cmd = ["ruby", "-Ilib:test", test.file]
+        cmd.push("--name", "/^#{Regexp.escape(test.name)}$/") unless file_unit
       else
         raise ConfigError, "unsupported runner for calibration: #{@config.test_runner}"
       end

data/lib/binpacker/cli.rb

@@ -15,6 +15,12 @@ module Binpacker
       @profile = nil
       @passthrough = []
       @quiet = false
+      @incremental = false
+      @report_path = nil
+      @skill_list = false
+      @skill_path = false
+      @skill_describe = false
+      @skill_name = nil
       parse!
     end
 
@@ -26,6 +32,10 @@ module Binpacker
         cmd_run
       when "init"
         cmd_init
+      when "skill"
+        cmd_skill
+      when "describe"
+        cmd_describe
       when "--version", "-v"
         puts "binpacker #{Binpacker::VERSION}"
       when "--help", "-h", nil
@@ -58,6 +68,26 @@ module Binpacker
         opts.on("--quiet", "Suppress worker output") do
           @quiet = true
         end
+
+        opts.on("--incremental", "Calibrate only tests without timing data") do
+          @incremental = true
+        end
+
+        opts.on("--report PATH", "(run) Write a JSON run report to PATH") do |v|
+          @report_path = v
+        end
+
+        opts.on("--list", "(skill) List bundled skills") do
+          @skill_list = true
+        end
+
+        opts.on("--path", "(skill) Print the SKILL.md path for <name>") do
+          @skill_path = true
+        end
+
+        opts.on("--describe", "(skill) Report project state and next skill") do
+          @skill_describe = true
+        end
       end
 
       # Extract passthrough arguments after "--"
@@ -67,7 +97,10 @@ module Binpacker
       end
 
       remaining = parser.parse(@args)
-      @command = remaining.shift
+      # A positional command wins; otherwise keep any command an option set
+      # (e.g. --version), so `binpacker --version` isn't clobbered to nil.
+      @command = remaining.shift || @command
+      @skill_name = remaining.shift
     end
 
     def cmd_init
@@ -78,6 +111,11 @@ module Binpacker
       end
 
       framework = detect_framework
+      unless ProjectState::SUPPORTED_FRAMEWORKS.include?(framework)
+        $stderr.puts "Detected #{framework} — binpacker supports rspec and minitest only."
+        $stderr.puts "No binpacker.yml was created."
+        exit 1
+      end
       pattern = framework == "minitest" ? "test/**/*_test.rb" : "spec/**/*_spec.rb"
       runner = framework
 
@@ -89,7 +127,7 @@ module Binpacker
             timing_file: binpacker.timings
             test_pattern: "#{pattern}"
             scheduler:
-              algorithm: lpt
+              algorithm: multifit
               steal_enabled: true
           ci:
             extends: default
@@ -110,21 +148,98 @@ module Binpacker
       discovery_klass = config.test_runner == "rspec" ? RSpecDiscovery : MinitestDiscovery
       tests = discovery_klass.new(config).enumerate
 
-      puts "Calibrating #{tests.size} tests..."
       cal = Calibration.new(config)
-      timings = cal.run(tests)
+
+      if @incremental
+        puts "Calibrating (incremental) #{tests.size} tests..."
+      else
+        puts "Calibrating #{tests.size} tests..."
+      end
+
+      timings = cal.run(tests, incremental: @incremental)
 
       total = timings.sum { |t| t[:time] }
-      puts "Calibration complete: #{tests.size} tests in #{total.round(2)}s"
+      measured = timings.size
+      skipped = tests.size - measured
+      summary = "Calibration complete: #{measured} #{pluralize(measured, 'test')} in #{total.round(2)}s"
+      summary += " (#{skipped} skipped, already measured)" if @incremental && skipped.positive?
+      puts summary
       puts "Timing data written to #{config.timing_file}"
     end
 
+    def cmd_skill
+      return cmd_describe if @skill_describe
+
+      if @skill_path
+        path = Skills.path(@skill_name)
+        if path
+          puts path
+        else
+          $stderr.puts "unknown skill: #{@skill_name}"
+          exit 1
+        end
+      elsif @skill_name && !@skill_list
+        print_skill(@skill_name)
+      else
+        list_skills
+      end
+    end
+
+    def cmd_describe
+      state = ProjectState.new
+      puts "binpacker project state:"
+      puts "  config (binpacker.yml): #{state.config_present? ? 'present' : 'missing'}"
+      puts "  timing data:            #{state.timing_present? ? 'present' : 'missing'}"
+      puts "  test framework:         #{state.framework || 'not detected'}"
+      puts "  CI wired for binpacker: #{state.ci_wired? ? 'yes' : 'no'}"
+      puts ""
+
+      unless state.supported_framework?
+        puts "binpacker supports rspec and minitest; #{state.framework} is not supported yet."
+        puts "Setup cannot proceed against a #{state.framework} suite."
+        return
+      end
+
+      rec = state.recommendation
+      puts "Recommended next skill: #{rec}"
+      puts "Run `binpacker skill #{rec}` to load its instructions."
+    end
+
+    def list_skills
+      skills = Skills.list
+      if skills.empty?
+        puts "No bundled skills found."
+        return
+      end
+      puts "Bundled skills:"
+      skills.each { |s| puts "  #{s[:name]}" }
+      puts ""
+      puts "Print a skill's instructions with `binpacker skill <name>`."
+    end
+
+    def print_skill(name)
+      unless Skills.exist?(name)
+        $stderr.puts "unknown skill: #{name}"
+        $stderr.puts "Run `binpacker skill` to list bundled skills."
+        exit 1
+      end
+
+      puts "# Skill: #{name}"
+      puts "# Source: #{Skills.path(name)}"
+      puts ""
+      puts Skills.body(name)
+    end
+
     def cmd_run
       config = Config.new(profile: @profile)
-      orchestrator = Orchestrator.new(config, passthrough: @passthrough, quiet: @quiet)
+      report_path = @report_path || config.report_file
+      orchestrator = Orchestrator.new(
+        config, passthrough: @passthrough, quiet: @quiet, report_path: report_path
+      )
 
       puts "binpacker starting (#{config.worker_count} workers, profile: #{config.profile})"
       result = orchestrator.run
+      puts "Run report written to #{report_path}" if report_path
       unit = test_unit_label(config)
 
       if result[:passed]
@@ -156,23 +271,30 @@ module Binpacker
           run          Execute tests across worker processes
           calibrate    Run tests serially to generate timing data
           init         Create binpacker.yml with auto-detected settings
+          describe     Report project state and recommend the next skill
+          skill        List or print bundled agent skills
 
         Options:
           --profile NAME   Select profile from binpacker.yml
+          --incremental    (calibrate) Measure only tests without timing data
+          --report PATH    (run) Write a JSON run report to PATH
+          --list           (skill) List bundled skills
+          --path           (skill) Print the SKILL.md path for <name>
+          --describe       (skill) Report project state and next skill
           --help           Show this message
 
         Examples:
           binpacker init
           binpacker run --profile ci
           binpacker run -- --tag ~slow
-          binpacker calibrate
+          binpacker calibrate --incremental
+          binpacker describe
+          binpacker skill binpacker-setup
       HELP
     end
 
     def detect_framework
-      return "minitest" if Dir.glob("test*/**/*_test.rb").any?
-      return "minitest" if Dir.glob("test*/**/test_*.rb").any?
-      "rspec"
+      ProjectState.new.framework || "rspec"
     end
   end
 end

data/lib/binpacker/config.rb

@@ -14,10 +14,11 @@ module Binpacker
       "timing_file" => "binpacker.timings",
       "test_pattern" => "spec/**/*_spec.rb",
       "test_exclude" => [],
+      "report_file" => nil,
       "scheduler" => {
         "strategy" => "static",
         "steal_enabled" => false,
-        "algorithm" => "lpt"
+        "algorithm" => "multifit"
       }
     }.freeze
 

data/lib/binpacker/orchestrator.rb

@@ -2,33 +2,58 @@
 
 module Binpacker
   class Orchestrator
-    BATCH_SIZE = 10
-
-    def initialize(config, passthrough: [], quiet: false)
+    # Dynamic batches halve the queue's remaining predicted weight,
+    # floored at a per-run minimum: early batches are large to amortize
+    # the per-batch test-runner boot (each batch is a fresh rspec
+    # process), tail batches stay small so workers finish together.
+    #
+    # The floor comes in two regimes (computed by #min_batch_weight in
+    # #run and threaded into #drain_batch):
+    #   - Calibrated: weights are measured seconds, so the floor is a
+    #     fixed MIN_BATCH_WEIGHT (~30s), an order of magnitude above the
+    #     ~3s runner boot — boot overhead stays in the noise.
+    #   - Cold start: weights are filesize KB, not seconds, so a fixed
+    #     30 is meaningless. We instead derive a floor that targets
+    #     ~COLD_START_BATCHES_PER_WORKER batches per worker (below).
+    MIN_BATCH_WEIGHT = 30.0
+
+    # On a pure cold start predicted weights are filesize KB, not
+    # seconds, so the 30s MIN_BATCH_WEIGHT floor has no meaning. Instead
+    # we size the floor so the total predicted weight divides into about
+    # this many batches per worker — matching the ~5 runner boots per
+    # worker that gave the best makespan in the PR #12 simulation.
+    COLD_START_BATCHES_PER_WORKER = 5
+
+    def initialize(config, passthrough: [], quiet: false, report_path: nil)
       @config = config
       @passthrough = passthrough
       @quiet = quiet
+      @report_path = report_path
     end
 
     def run
       tests = discover
       timing = Timing.new(@config.timing_file)
       timings = timing.load_with_fallback(tests)
+      @timings = timings
 
-      scheduler = Scheduler.for(@config.scheduler["algorithm"])
+      scheduler = Scheduler.for(@config.scheduler['algorithm'])
       queues = scheduler.partition(
         tests: tests,
         worker_count: @config.worker_count,
         timings: timings
       )
 
+      # Capture predicted per-Worker loads before execution drains the queues.
+      @predicted_loads = queues.map { |q| q.total_weight(timings) }
+
       runner_class = TestRunner.for(@config.test_runner)
-      workers = queues.map.with_index do |queue, idx|
+      workers = queues.map.with_index do |_queue, idx|
         Worker.new(idx, runner_class, passthrough: @passthrough, quiet: @quiet).tap(&:start)
       end
 
-      if @config.scheduler["steal_enabled"]
-        run_dynamic(workers, queues, timing, tests)
+      if @config.scheduler['steal_enabled']
+        run_dynamic(workers, queues, timing, tests, min_batch_weight(timing, timings))
       else
         run_static(workers, queues, timing, tests)
       end
@@ -38,9 +63,9 @@ module Binpacker
 
     def discover
       case @config.test_runner
-      when "rspec"
+      when 'rspec'
         RSpecDiscovery.new(@config).enumerate
-      when "minitest"
+      when 'minitest'
         MinitestDiscovery.new(@config).enumerate
       else
         raise ConfigError, "unsupported runner: #{@config.test_runner}"
@@ -53,7 +78,7 @@ module Binpacker
 
       workers.zip(queues).each do |worker, queue|
         worker.send_tests(queue.remaining)
-        progress.update(worker.id, done: 0, total: queue_totals[worker.id], file: queue.remaining.first&.file || "")
+        progress.update(worker.id, done: 0, total: queue_totals[worker.id], file: queue.remaining.first&.file || '')
       end
       progress.refresh
 
@@ -76,10 +101,10 @@ module Binpacker
         worker_examples[worker.id] = worker.example_count
         worker_passed[worker.id] = worker.passed_count
         worker_time[worker.id] = worker.timings.sum { |t| t[:time] }
-        progress.update(worker.id, done: queue_totals[worker.id], total: queue_totals[worker.id], file: "done")
+        progress.update(worker.id, done: queue_totals[worker.id], total: queue_totals[worker.id], file: 'done')
         progress.refresh
       rescue WorkerError => e
-        $stderr.puts "worker #{worker.id} error: #{e.message}"
+        warn "worker #{worker.id} error: #{e.message}"
         all_passed = false
       ensure
         worker.cleanup
@@ -87,7 +112,7 @@ module Binpacker
 
       progress.finish
 
-      worker_stats = workers.map.with_index do |w, i|
+      worker_stats = workers.map.with_index do |_w, i|
         {
           files: queue_totals[i],
           total_time: worker_time[i],
@@ -96,15 +121,14 @@ module Binpacker
         }
       end
       progress.summary(worker_stats)
+      write_report(worker_stats, all_timings)
 
       finalize(timing, all_timings, all_passed, total_examples, passed_examples, tests)
     end
 
-    def run_dynamic(workers, queues, timing, tests)
+    def run_dynamic(workers, queues, timing, tests, floor)
       all_timings = []
       all_passed = true
-      total_examples = 0
-      passed_examples = 0
       active = []
 
       queue_totals = queues.map(&:size)
@@ -117,25 +141,23 @@ module Binpacker
       progress = ProgressDisplay.new(workers.size)
 
       workers.zip(queues).each do |worker, queue|
-        batch = drain_batch(queue)
+        batch = drain_batch(queue, floor)
         if batch.empty?
           worker.signal_done
           worker.collect_results
           all_timings.concat(worker.timings)
           all_passed &&= worker.success?
-          total_examples += worker.example_count
-          passed_examples += worker.passed_count
           worker_examples[worker.id] = worker.example_count
           worker_passed[worker.id] = worker.passed_count
           worker.cleanup
           worker_done[worker.id] = queue_totals[worker.id]
-          progress.update(worker.id, done: worker_done[worker.id], total: queue_totals[worker.id], file: "done")
+          progress.update(worker.id, done: worker_done[worker.id], total: queue_totals[worker.id], file: 'done')
         else
           worker.send_tests(batch)
           worker.batch_done
           active << worker
           batch_sizes[worker.id] = batch.size
-          current_file = batch.first&.file || ""
+          current_file = batch.first&.file || ''
           progress.update(worker.id, done: 0, total: queue_totals[worker.id], file: current_file)
         end
       end
@@ -143,33 +165,38 @@ module Binpacker
       until active.empty?
         ready = active.find { |w| w.wait_for_batch }
         unless ready
-          active.reject! { |w| w.status == :crashed || w.status == :error }
+          active.reject! { |w| %i[crashed error].include?(w.status) }
           sleep 0.1
           next
         end
 
         begin
           all_passed &&= ready.success?
-          total_examples += ready.example_count
-          passed_examples += ready.passed_count
 
           worker_done[ready.id] += batch_sizes[ready.id]
+          # Worker#example_count accumulates across batches, so these
+          # are running totals per worker; the grand totals are summed
+          # once after the loop, never added per batch.
           worker_examples[ready.id] = ready.example_count
           worker_passed[ready.id] = ready.passed_count
 
           own_queue = queues[ready.id]
-          next_batch = drain_batch(own_queue)
+          next_batch = drain_batch(own_queue, floor)
 
           if next_batch.empty?
-            donor = queues.reject(&:empty?).max_by(&:size)
-            next_batch = drain_batch(donor) if donor
+            donor = queues.reject(&:empty?).max_by { |q| q.total_weight(@timings) }
+            if donor
+              next_batch = drain_batch(donor, floor)
+              queue_totals[ready.id] += next_batch.size
+              queue_totals[donor.worker_id] -= next_batch.size
+            end
           end
 
           if next_batch.any?
             ready.send_tests(next_batch)
             ready.batch_done
             batch_sizes[ready.id] = next_batch.size
-            current_file = next_batch.first&.file || ""
+            current_file = next_batch.first&.file || ''
             progress.update(ready.id, done: worker_done[ready.id], total: queue_totals[ready.id], file: current_file)
             progress.refresh
           else
@@ -177,11 +204,11 @@ module Binpacker
             all_timings.concat(ready.timings)
             active.delete(ready)
             worker_done[ready.id] = queue_totals[ready.id]
-            progress.update(ready.id, done: queue_totals[ready.id], total: queue_totals[ready.id], file: "done")
+            progress.update(ready.id, done: queue_totals[ready.id], total: queue_totals[ready.id], file: 'done')
             progress.refresh
           end
         rescue WorkerError => e
-          $stderr.puts "worker #{ready.id} error: #{e.message}"
+          warn "worker #{ready.id} error: #{e.message}"
           all_passed = false
           active.delete(ready)
         end
@@ -189,6 +216,9 @@ module Binpacker
 
       progress.finish
 
+      total_examples = worker_examples.sum
+      passed_examples = worker_passed.sum
+
       worker_stats = workers.map.with_index do |w, i|
         tw = w.timings.sum { |t| t[:time] }
         {
@@ -199,24 +229,62 @@ module Binpacker
         }
       end
       progress.summary(worker_stats)
+      write_report(worker_stats, all_timings)
 
       workers.each(&:cleanup)
       finalize(timing, all_timings, all_passed, total_examples, passed_examples, tests)
     end
 
-    def drain_batch(queue)
+    def write_report(worker_stats, all_timings)
+      return unless @report_path
+
+      Report.new(
+        profile: @config.profile,
+        algorithm: @config.scheduler['algorithm'],
+        predicted_loads: @predicted_loads,
+        worker_stats: worker_stats,
+        all_timings: all_timings,
+        timings: @timings
+      ).write(@report_path)
+    end
+
+    # Per-run batch-weight floor. Calibrated runs use the fixed
+    # seconds-scale MIN_BATCH_WEIGHT. On a cold start weights are KB, so
+    # that floor is meaningless; derive one that splits the total
+    # predicted weight into ~COLD_START_BATCHES_PER_WORKER batches per
+    # worker, falling back to MIN_BATCH_WEIGHT when there is nothing to
+    # divide (zero total weight or zero workers).
+    def min_batch_weight(timing, timings)
+      return MIN_BATCH_WEIGHT if timing.calibrated?
+
+      total = timings.values.sum(0.0)
+      workers = @config.worker_count
+      return MIN_BATCH_WEIGHT if total.zero? || workers.zero?
+
+      total / (workers * COLD_START_BATCHES_PER_WORKER)
+    end
+
+    # Drains the next batch off +queue+: enough tests to halve its
+    # remaining predicted weight, but never less than +floor+ (see
+    # #min_batch_weight) and never fewer than one test.
+    def drain_batch(queue, floor)
       return [] if queue.nil? || queue.empty?
+
+      target = [queue.total_weight(@timings) / 2.0, floor].max
       batch = []
-      BATCH_SIZE.times do
-        test = queue.pop
-        break unless test
+      weight = 0.0
+      while (batch.empty? || weight < target) && (test = queue.pop)
         batch << test
+        weight += @timings.fetch(test.key, Timing::DEFAULT_WEIGHT)
       end
       batch
     end
 
     def finalize(timing, all_timings, all_passed, total_examples, passed_examples, tests)
-      timing.append_all(all_timings) unless all_timings.empty?
+      unless all_timings.empty?
+        timing.append_all(all_timings)
+        timing.compact!
+      end
       empty_filter = minitest_empty_filter?(tests, total_examples)
       all_passed = false if empty_filter
 
@@ -230,7 +298,7 @@ module Binpacker
     end
 
     def minitest_empty_filter?(tests, total_examples)
-      return false unless @config.test_runner == "minitest"
+      return false unless @config.test_runner == 'minitest'
       return false unless tests.any?
       return false unless total_examples.zero?
 
@@ -240,8 +308,8 @@ module Binpacker
     def minitest_include_filter?
       @passthrough.any? do |arg|
         %w[--name --include -n -i].include?(arg) ||
-          arg.start_with?("--name=", "--include=") ||
-          (arg.start_with?("-n", "-i") && arg.length > 2)
+          arg.start_with?('--name=', '--include=') ||
+          (arg.start_with?('-n', '-i') && arg.length > 2)
       end
     end
   end