binpacker 0.1.0 → 0.3.0

data/lib/binpacker/skills.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Binpacker
+  # Registry for the gem-shipped, user-facing agent skills. Skills live in the
+  # gem's top-level skills/ directory, each as skills/<name>/SKILL.md.
+  # See docs/adr/0003-agent-driven-install-and-skills.md.
+  module Skills
+    ROOT = File.expand_path("../../skills", __dir__)
+
+    module_function
+
+    # [{ name:, path: }], sorted by name.
+    def list
+      Dir.glob(File.join(ROOT, "*", "SKILL.md")).sort.map do |file|
+        { name: File.basename(File.dirname(file)), path: file }
+      end
+    end
+
+    def exist?(name)
+      !path(name).nil?
+    end
+
+    # Absolute path of a skill's SKILL.md, or nil when unknown.
+    def path(name)
+      file = File.join(ROOT, name, "SKILL.md")
+      File.file?(file) ? file : nil
+    end
+
+    # The SKILL.md body for a skill.
+    def body(name)
+      file = path(name)
+      raise Error, "unknown skill: #{name}" unless file
+      File.read(file)
+    end
+  end
+end

data/lib/binpacker/test_discovery.rb

@@ -2,7 +2,6 @@
 
 module Binpacker
   Test = Struct.new(:file, :name, keyword_init: true) do
-    # Composite key used as timing lookup and identity.
     def key
       [file, name]
     end
@@ -15,7 +14,6 @@ module Binpacker
       @exclude = config.test_exclude
     end
 
-    # Returns Array<Test>. Concrete strategies subclass this.
     def enumerate
       raise NotImplementedError
     end
@@ -27,21 +25,77 @@ module Binpacker
         @exclude.any? { |ex| File.fnmatch?(ex, f) }
       }
     end
+
+    def add_project_load_paths
+      %w[lib test].each do |path|
+        expanded = File.expand_path(path)
+        $LOAD_PATH.unshift(expanded) if Dir.exist?(expanded) && !$LOAD_PATH.include?(expanded)
+      end
+    end
   end
 
   class RSpecDiscovery < TestDiscovery
     def enumerate
-      glob_files.map { |f| Test.new(file: f, name: f) }
+      if @config.respond_to?(:test_granularity) && @config.test_granularity == "example"
+        enumerate_examples
+      else
+        glob_files.map { |f| Test.new(file: f, name: f) }
+      end
+    end
+
+    private
+
+    def enumerate_examples
+      files = glob_files
+      return [] if files.empty?
+
+      examples = run_dry_run(files)
+      examples.map { |ex|
+        Test.new(
+          file: ex["file_path"],
+          name: ex["full_description"] || ex["description"]
+        )
+      }
+    end
+
+    def run_dry_run(files)
+      cmd = ["rspec", "--dry-run", "--format", "json", *files, { err: File::NULL }]
+      out = IO.popen(cmd) { |io| io.read }
+      JSON.parse(out)["examples"] || []
+    rescue JSON::ParserError
+      raise DiscoveryError, "failed to parse rspec --dry-run output"
     end
   end
 
   class MinitestDiscovery < TestDiscovery
-    # For Minitest, we can't easily enumerate test names without running them.
-    # Instead, treat each file as a single test unit, with the file path as the name.
     def enumerate
-      glob_files.map { |f|
-        Test.new(file: f, name: f)
-      }
+      begin
+        require "minitest"
+      rescue LoadError
+        raise DiscoveryError,
+          "minitest is not available. binpacker supports rspec and minitest; " \
+          "test-unit and other frameworks are not supported."
+      end
+      add_project_load_paths
+      def Minitest.autorun; end
+      Minitest.seed = 42
+
+      tests = []
+      glob_files.each do |file|
+        begin
+          klasses_before = Minitest::Runnable.runnables.dup
+          load File.expand_path(file)
+
+          (Minitest::Runnable.runnables - klasses_before).each do |klass|
+            klass.runnable_methods.each do |method_name|
+              tests << Test.new(file: file, name: "#{klass}##{method_name}")
+            end
+          end
+        rescue => e
+          $stderr.puts "minitest discovery: failed to load #{file}: #{e.message}"
+        end
+      end
+      tests
     end
   end
 end

data/lib/binpacker/timing.rb

@@ -44,6 +44,11 @@ module Binpacker
       Pathname(path).cleanpath.to_s.sub(/\A\.\//, "")
     end
 
+    # True when a measured Weight already exists for this Test.
+    def measured?(file:, name:)
+      load_raw.key?([normalize_path(file), name])
+    end
+
     def weight_for(file:, name:)
       measured = load_raw
       measured.fetch([file, name]) { filesize_weight(file) }

data/lib/binpacker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Binpacker
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/binpacker/worker.rb

@@ -6,13 +6,14 @@ module Binpacker
   class Worker
     attr_reader :id, :status, :example_count, :passed_count
 
-    def initialize(id, runner_class, passthrough: [])
+    def initialize(id, runner_class, passthrough: [], quiet: false)
       @id = id
       @runner_class = runner_class
       @passthrough = passthrough
+      @quiet = quiet
       @status = :created
       @timings = []
-      @exit_code = nil
+      @exit_code = 0
       @example_count = 0
       @passed_count = 0
     end
@@ -27,6 +28,7 @@ module Binpacker
       @pid = Process.spawn(
         RbConfig.ruby, worker_script,
         "--runner", @runner_class.runner_name,
+        "--id", @id.to_s,
         *passthrough_args,
         in: @stdin_r, out: @stdout_w, err: @stderr_w,
         close_others: true
@@ -35,7 +37,7 @@ module Binpacker
       @stdin_r.close; @stdout_w.close; @stderr_w.close
 
       @stderr_thread = Thread.new do
-        @stderr_r.each_line { |line| $stderr.write line }
+        @stderr_r.each_line { |line| $stderr.write line unless @quiet }
       end
 
       ready_line = read_line(timeout: 30)
@@ -59,13 +61,57 @@ module Binpacker
       @stdin_w.puts JSON.generate({ file: test.file, name: test.name })
     end
 
+    def batch_done
+      @stdin_w.puts JSON.generate({ type: "done" })
+    end
+
     def signal_done
       @stdin_w.puts JSON.generate({ type: "done" })
       @stdin_w.close
     end
 
+    def wait_for_batch
+      deadline = Time.now + 300
+      while Time.now < deadline
+        begin
+          line = read_line(timeout: 1)
+          return nil unless line
+          next unless line.strip.start_with?("{")
+          data = JSON.parse(line.strip)
+
+          if data["type"] == "timing"
+            @timings << { file: data["file"], name: data["name"], time: data["time"] }
+          elsif data["type"] == "batch_result"
+            @exit_code = data["passed"] ? 0 : 1
+            @example_count += data["total"] || 0
+            @passed_count += data["passed_count"] || 0
+            return {
+              timings: [],
+              exit_code: @exit_code,
+              examples: data["total"] || 0,
+              passed: data["passed_count"] || 0
+            }
+          elsif data["type"] == "result"
+            @exit_code = data["exit_code"]
+            @passed = data["passed"]
+            @example_count = data["total"] || 0
+            @passed_count = data["passed_count"] || 0
+            return {
+              timings: [],
+              exit_code: @exit_code,
+              examples: @example_count,
+              passed: @passed_count
+            }
+          end
+        rescue JSON::ParserError
+        end
+      end
+      nil
+    end
+
     def collect_results
       @status = :running
+
       @stdout_r.each_line do |line|
         data = JSON.parse(line.strip)
         case data["type"]
@@ -76,6 +122,13 @@ module Binpacker
           @passed = data["passed"]
           @example_count = data["total"] || 0
           @passed_count = data["passed_count"] || 0
+          break
+        when "batch_result"
+          @exit_code = data["passed"] ? 0 : 1
+          @passed = data["passed"]
+          @example_count = data["total"] || 0
+          @passed_count = data["passed_count"] || 0
+          break
         when "output"
           $stdout.write data["text"] if data["text"]
         end
@@ -109,7 +162,9 @@ module Binpacker
 
     def passthrough_args
       return [] if @passthrough.empty?
-      @passthrough.flat_map { |arg| ["--rspec-arg", arg] }
+
+      worker_arg = @runner_class.runner_name == "minitest" ? "--minitest-arg" : "--rspec-arg"
+      @passthrough.flat_map { |arg| [worker_arg, arg] }
     end
 
     def kill!

data/lib/binpacker.rb

@@ -9,7 +9,11 @@ require_relative "binpacker/scheduler"
 require_relative "binpacker/worker"
 require_relative "binpacker/test_runner"
 require_relative "binpacker/calibration"
+require_relative "binpacker/report"
+require_relative "binpacker/skills"
+require_relative "binpacker/project_state"
 require_relative "binpacker/orchestrator"
+require_relative "binpacker/progress"
 
 module Binpacker
   Error = Class.new(StandardError)

data/sig/binpacker/calibration.rbs

@@ -0,0 +1,6 @@
+module Binpacker
+  class Calibration
+    def initialize: (untyped) -> void
+    def run: (untyped, ?incremental: bool) -> Array[untyped]
+  end
+end

data/sig/binpacker/cli.rbs

@@ -0,0 +1,7 @@
+module Binpacker
+  class CLI
+    def self.start: (untyped) -> nil
+    def initialize: (untyped) -> void
+    def run: () -> nil
+  end
+end

data/sig/binpacker/config.rbs

@@ -0,0 +1,13 @@
+module Binpacker
+  class Config
+    def initialize: (?profile: untyped, ?config_path: untyped) -> void
+    def worker_count: () -> Integer
+    def test_runner: () -> String
+    def timing_file: () -> String
+    def test_pattern: () -> String
+    def test_exclude: () -> Array[String]
+    def test_granularity: () -> String
+    def report_file: () -> String?
+    def profile: () -> String
+  end
+end

data/sig/binpacker/orchestrator.rbs

@@ -0,0 +1,6 @@
+module Binpacker
+  class Orchestrator
+    def initialize: (untyped, ?passthrough: untyped, ?quiet: untyped, ?report_path: untyped) -> void
+    def run: () -> { passed: untyped, total: untyped, passed_count: untyped, timings: Array[untyped], empty_filter: untyped }
+  end
+end

data/sig/binpacker/progress.rbs

@@ -0,0 +1,5 @@
+module Binpacker
+  class ProgressDisplay
+    def initialize: (untyped, ?tty: untyped) -> void
+  end
+end

data/sig/binpacker/project_state.rbs

@@ -0,0 +1,16 @@
+module Binpacker
+  class ProjectState
+    CONFIG_FILE: String
+    DEFAULT_TIMING_FILE: String
+    SUPPORTED_FRAMEWORKS: Array[String]
+    TESTUNIT_HINT: Regexp
+
+    def config_present?: () -> bool
+    def timing_present?: () -> bool
+    def framework: () -> String?
+    def supported_framework?: () -> bool
+    def ci_wired?: () -> bool
+    def recommendation: () -> String
+    def to_h: () -> Hash[Symbol, untyped]
+  end
+end

data/sig/binpacker/report.rbs

@@ -0,0 +1,10 @@
+module Binpacker
+  class Report
+    SCHEMA: Integer
+    DRIFT_LIMIT: Integer
+
+    def initialize: (profile: untyped, algorithm: untyped, predicted_loads: untyped, worker_stats: untyped, all_timings: untyped, timings: untyped) -> void
+    def to_h: () -> Hash[Symbol, untyped]
+    def write: (untyped) -> untyped
+  end
+end

data/sig/binpacker/scheduler.rbs

@@ -0,0 +1,5 @@
+module Binpacker
+  class Scheduler
+    def self.for: (untyped) -> (Binpacker::LptScheduler | Binpacker::MultifitScheduler)
+  end
+end

data/sig/binpacker/skills.rbs

@@ -0,0 +1,10 @@
+module Binpacker
+  module Skills
+    ROOT: String
+
+    def self.list: () -> Array[Hash[Symbol, String]]
+    def self.exist?: (untyped) -> bool
+    def self.path: (untyped) -> String?
+    def self.body: (untyped) -> String
+  end
+end

data/sig/binpacker/test_discovery.rbs

@@ -0,0 +1,15 @@
+module Binpacker
+  def key: () -> [untyped, String]
+  class TestDiscovery
+    def initialize: (untyped) -> void
+    def enumerate: () -> Array[Binpacker::Test]
+  end
+  class RSpecDiscovery < TestDiscovery
+    def enumerate: () -> Array[Binpacker::Test]
+  end
+  class MinitestDiscovery < TestDiscovery
+    def enumerate: () -> Array[Binpacker::Test]
+  end
+  class Test
+  end
+end

data/sig/binpacker/test_runner.rbs

@@ -0,0 +1,13 @@
+module Binpacker
+  class TestRunner
+    def self.runner_name: () -> bot
+    def self.for: (untyped) -> (singleton(Binpacker::MinitestRunner) | singleton(Binpacker::RSpecRunner))
+  end
+  class RSpecRunner
+    def self.runner_name: () -> "rspec"
+    def self.discovery_command: (untyped) -> Array[untyped]
+  end
+  class MinitestRunner
+    def self.runner_name: () -> "minitest"
+  end
+end

data/sig/binpacker/timing.rbs

@@ -0,0 +1,11 @@
+module Binpacker
+  class Timing
+    def initialize: (untyped) -> void
+    def load_raw: () -> (Hash[[String, untyped], untyped] | {})
+    def load_per_file: () -> (Hash[String, untyped] | {})
+    def normalize_path: (untyped) -> String
+    def measured?: (file: untyped, name: untyped) -> bool
+    class Entry
+    end
+  end
+end

data/sig/binpacker/worker.rbs

@@ -0,0 +1,12 @@
+module Binpacker
+  class Worker
+    def initialize: (untyped, untyped, ?passthrough: untyped, ?quiet: untyped) -> void
+    def start: () -> Binpacker::Worker
+    def send_test: (untyped) -> nil
+    def batch_done: () -> nil
+    def wait_for_batch: () -> (nil | { timings: [], exit_code: 0 | 1, examples: untyped, passed: untyped } | { timings: [], exit_code: untyped, examples: untyped, passed: untyped })
+    def collect_results: () -> :finished
+    def timings: () -> Array[untyped]
+    def status: () -> (:crashed | :created | :error | :finished | :ready | :running)
+  end
+end

data/sig/binpacker/worker_queue.rbs

@@ -0,0 +1,6 @@
+module Binpacker
+  class WorkerQueue
+    def initialize: (untyped, ?untyped) -> void
+    def empty?: () -> (false | true)
+  end
+end

data/skills/binpacker-improve/SKILL.md

@@ -0,0 +1,52 @@
+---
+name: binpacker-improve
+description: Tune an already-configured binpacker project from real CI data — fetch recent run reports, analyze the gap between predicted and actual durations, and propose binpacker.yml adjustments. Use when binpacker is set up and running in CI and the user wants to reduce makespan, fix worker imbalance, or refresh stale timing data. Propose-only — never apply changes without confirmation.
+---
+
+# Binpacker Improve
+
+A recurring, **propose-only** workflow. It reads real CI data and proposes config changes; it never edits `binpacker.yml`, commits, or pushes without explicit user confirmation.
+
+Run `binpacker describe` first. If it recommends `binpacker-setup`, the project isn't configured yet — use that skill instead.
+
+## 1. Gather CI data
+
+Fetch recent run reports (see `docs/design/run-report-format.md` for the schema) from CI artifacts:
+
+```sh
+gh run list --workflow <ci-workflow> --limit 10
+gh run download <run-id> --name <report-artifact>
+```
+
+Collect several recent reports so you can see trends, not just one run. Each report holds `predicted_makespan`, `actual_makespan`, per-worker predicted/actual durations, `balance`, and the worst-drift Tests.
+
+## 2. Analyze
+
+Look for patterns across the reports:
+
+- **Persistent worker skew** — the same worker is consistently slowest → the partition or worker count is off.
+- **Predicted vs actual gap** — `actual_makespan` far above `predicted_makespan` → timing data is stale; the scheduler is planning against wrong Weights.
+- **Recurring high-drift files** — the same files top the `drift` list every run → their timing data is stale or they are outliers.
+- **Cache-miss frequency** — if CI logs show frequent timing-cache misses (e.g. after dependency bumps), the runtime cache isn't helping cold starts.
+
+## 3. Propose
+
+Present a concrete `binpacker.yml` diff and/or commands, with the evidence behind each change. Typical levers:
+
+- **`workers`** — raise/lower the `ci` worker count when actual makespan shows consistent under- or over-utilisation.
+- **`scheduler.algorithm`** — `lpt` ↔ `multifit`. Prefer `multifit` for tighter balance; only suggest `lpt` if `multifit` isn't helping.
+- **`scheduler.steal_enabled`** — enable when static balance is poor but runtime stealing would recover it.
+- **Re-calibrate** — when drift is high or timing is stale, recommend `binpacker calibrate --incremental` to refresh the drifting Tests.
+- **Commit a baseline** — when the cache misses often (dependency-bump churn) or timings have stabilised, recommend committing `binpacker.timings` as a durable baseline (cold-start floor). See `docs/adr/0005-two-tier-timing-persistence.md`.
+
+## 4. Confirm and apply
+
+- Show the diff. Explain the expected effect (e.g. "should cut max deviation from 6% toward <3%").
+- Apply only what the user approves. Committing a baseline or editing config both require confirmation.
+- If the user wants, open a PR with `gh`; otherwise leave the working tree changes for them to review.
+
+## Done when
+
+- The user has seen the predicted-vs-actual analysis with evidence.
+- A concrete, confirmed set of changes (or an explicit "no change needed") has been reached.
+- Any commit / PR was made only with confirmation.

data/skills/binpacker-setup/SKILL.md

@@ -0,0 +1,76 @@
+---
+name: binpacker-setup
+description: Bring a project from zero to a working parallel test run with binpacker — detect the framework, generate config, calibrate timing data, validate a parallel run, and wire CI. Use when binpacker is installed but not yet set up (no binpacker.yml, or no timing data), or when the user asks to set up / configure / adopt binpacker.
+---
+
+# Binpacker Setup
+
+A one-time workflow that takes a project from zero to a working parallel test run. Run `binpacker describe` first — if it recommends `binpacker-improve`, the project is already set up and you want that skill instead.
+
+Assess state as you go and **ask the user at each decision point** rather than assuming. Do not run destructive or outward-facing steps (committing, opening a PR) without explicit confirmation.
+
+## 1. Detect
+
+- Confirm binpacker is installed: `binpacker --version`.
+- Run `binpacker describe` to read current state (config present? timing data? framework? CI wired?).
+- Determine the test framework (rspec / minitest) and the test glob. If `binpacker describe` reports one, trust it; otherwise inspect the repo.
+
+## 2. Config
+
+If `binpacker.yml` is absent, create it with `binpacker init` (it auto-detects the framework and writes `default` + `ci` profiles). Then review the generated file with the user and adjust:
+
+- `workers` — `auto` locally; a fixed number (e.g. `4`) for the `ci` profile matching the CI runner.
+- `test_pattern` / `test_exclude` — match the project's real layout.
+- `scheduler.algorithm` — `multifit` (default) unless there's a reason to use `lpt`.
+- `scheduler.steal_enabled` — `true` to let idle workers steal at runtime.
+- `report_file` (on the `ci` profile) — e.g. `binpacker-report.json`, so CI always emits a run report.
+
+## 3. Calibrate
+
+Seed timing data so the scheduler has real Weights:
+
+```sh
+binpacker calibrate
+```
+
+If some timing data already exists, measure only the gaps:
+
+```sh
+binpacker calibrate --incremental
+```
+
+## 4. Validate
+
+Run the suite in parallel and confirm it is green and balanced:
+
+```sh
+binpacker run
+```
+
+Check the summary's `Balance: max deviation`. A large deviation on the first run is expected before timing data accumulates; note it and move on.
+
+## 5. Wire CI
+
+Add (or extend) a CI workflow so the suite runs under binpacker. The workflow should:
+
+- Install binpacker (via the project's dependency manager or `gem install binpacker`).
+- Restore and save `binpacker.timings` with `actions/cache` (key includes OS + a hash of the test files). This is the runtime timing layer.
+- Run `binpacker run --profile ci` (which emits the run report via `report_file`).
+- Upload the run report with `actions/upload-artifact` so `binpacker-improve` can read it later.
+
+Propose the workflow YAML and let the user review it before writing.
+
+## 6. Handoff
+
+Only after the user confirms:
+
+- Create a branch, commit `binpacker.yml`, the CI workflow, and (if the user wants a committed baseline) `binpacker.timings`.
+- Open a PR with `gh`, summarising what was set up.
+- After CI runs, point the user at `binpacker-improve` to tune from real CI data.
+
+## Done when
+
+- `binpacker run` passes locally across workers.
+- `binpacker.yml` reflects the project's framework, layout, and CI worker count.
+- CI runs `binpacker run` and uploads a run report.
+- The user has confirmed any commit / PR.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: binpacker
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - megurine
@@ -9,8 +9,9 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
-description: A test runner wrapper that manages worker processes and distributes tests
-  among them using LPT scheduling with optional work-stealing.
+description: A test runner wrapper that models test distribution as an identical-machines
+  scheduling problem and assigns tests to worker processes using LPT scheduling, with
+  optional work-stealing.
 executables:
 - binpacker
 extensions: []
@@ -24,13 +25,33 @@ files:
 - lib/binpacker/cli.rb
 - lib/binpacker/config.rb
 - lib/binpacker/orchestrator.rb
+- lib/binpacker/progress.rb
+- lib/binpacker/project_state.rb
+- lib/binpacker/report.rb
 - lib/binpacker/scheduler.rb
+- lib/binpacker/skills.rb
 - lib/binpacker/test_discovery.rb
 - lib/binpacker/test_runner.rb
 - lib/binpacker/timing.rb
 - lib/binpacker/version.rb
 - lib/binpacker/worker.rb
 - lib/binpacker/worker_queue.rb
+- sig/binpacker/calibration.rbs
+- sig/binpacker/cli.rbs
+- sig/binpacker/config.rbs
+- sig/binpacker/orchestrator.rbs
+- sig/binpacker/progress.rbs
+- sig/binpacker/project_state.rbs
+- sig/binpacker/report.rbs
+- sig/binpacker/scheduler.rbs
+- sig/binpacker/skills.rbs
+- sig/binpacker/test_discovery.rbs
+- sig/binpacker/test_runner.rbs
+- sig/binpacker/timing.rbs
+- sig/binpacker/worker.rbs
+- sig/binpacker/worker_queue.rbs
+- skills/binpacker-improve/SKILL.md
+- skills/binpacker-setup/SKILL.md
 homepage: https://github.com/rigortype/binpacker
 licenses:
 - MPL-2.0
@@ -53,6 +74,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.10
 specification_version: 4
-summary: Minimize CI test-suite makespan by solving an identical-machines scheduling
-  problem
+summary: Reduce CI test-suite makespan via identical-machines scheduling
 test_files: []