robot_lab-to 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.envrc +1 -0
- data/.github/workflows/deploy-github-pages.yml +52 -0
- data/.loki +21 -0
- data/.quality/reek_baseline.txt +17 -0
- data/.rubocop.yml +7 -0
- data/CHANGELOG.md +67 -0
- data/CLAUDE.md +74 -0
- data/COMMITS.md +27 -0
- data/README.md +352 -0
- data/Rakefile +132 -0
- data/bin/robot-to +14 -0
- data/docs/about/changelog.md +82 -0
- data/docs/assets/architecture.svg +110 -0
- data/docs/assets/images/robot_lab-to.jpg +0 -0
- data/docs/assets/iteration-loop.svg +97 -0
- data/docs/concepts/evals.md +254 -0
- data/docs/concepts/iteration-loop.md +128 -0
- data/docs/concepts/notes.md +91 -0
- data/docs/concepts/run-state.md +93 -0
- data/docs/concepts/stop-conditions.md +111 -0
- data/docs/concepts/verification.md +75 -0
- data/docs/configuration/cli.md +111 -0
- data/docs/configuration/index.md +93 -0
- data/docs/configuration/settings.md +270 -0
- data/docs/getting-started/anatomy-of-a-run.md +111 -0
- data/docs/getting-started/installation.md +78 -0
- data/docs/getting-started/quick-start.md +105 -0
- data/docs/index.md +96 -0
- data/docs/local-models/guardrails.md +117 -0
- data/docs/local-models/index.md +88 -0
- data/docs/local-models/ollama.md +122 -0
- data/docs/local-models/tools.md +92 -0
- data/docs/reference/architecture.md +139 -0
- data/examples/01_basic_usage/.gitignore +9 -0
- data/examples/01_basic_usage/README.md +173 -0
- data/examples/01_basic_usage/basic_usage.rb +386 -0
- data/examples/02_advanced_usage/.gitignore +8 -0
- data/examples/02_advanced_usage/README.md +117 -0
- data/examples/02_advanced_usage/advanced_usage.rb +308 -0
- data/examples/02_advanced_usage/quality_gate.rb +99 -0
- data/examples/03_scored/.gitignore +2 -0
- data/examples/03_scored/README.md +117 -0
- data/examples/03_scored/scored_run.rb +325 -0
- data/examples/04_prose/.gitignore +2 -0
- data/examples/04_prose/README.md +43 -0
- data/examples/04_prose/prompts_dir/grade_message.md +14 -0
- data/examples/04_prose/prompts_dir/judge.md +11 -0
- data/examples/04_prose/prompts_dir/outline_criteria.md +9 -0
- data/examples/04_prose/prompts_dir/outline_objective.md +11 -0
- data/examples/04_prose/prompts_dir/section_criteria.md +7 -0
- data/examples/04_prose/prompts_dir/sections_objective.md +14 -0
- data/examples/04_prose/prose_run.rb +302 -0
- data/lib/robot_lab/to/atomic_file.rb +69 -0
- data/lib/robot_lab/to/backoff.rb +46 -0
- data/lib/robot_lab/to/cli.rb +176 -0
- data/lib/robot_lab/to/commit_manager.rb +135 -0
- data/lib/robot_lab/to/config/defaults.yml +27 -0
- data/lib/robot_lab/to/config.rb +86 -0
- data/lib/robot_lab/to/decision.rb +40 -0
- data/lib/robot_lab/to/decision_manager.rb +193 -0
- data/lib/robot_lab/to/errors.rb +30 -0
- data/lib/robot_lab/to/evals/base.rb +23 -0
- data/lib/robot_lab/to/evals/code.rb +77 -0
- data/lib/robot_lab/to/evals/context.rb +17 -0
- data/lib/robot_lab/to/evals/factory.rb +84 -0
- data/lib/robot_lab/to/evals/null.rb +17 -0
- data/lib/robot_lab/to/evals/prose.rb +110 -0
- data/lib/robot_lab/to/evals/score.rb +22 -0
- data/lib/robot_lab/to/exit_summary.rb +75 -0
- data/lib/robot_lab/to/guards/checkpoint.rb +84 -0
- data/lib/robot_lab/to/guards/grader_lock.rb +62 -0
- data/lib/robot_lab/to/guards/path_resolution.rb +61 -0
- data/lib/robot_lab/to/guards/quality_monitor.rb +113 -0
- data/lib/robot_lab/to/guards/read_before_edit.rb +83 -0
- data/lib/robot_lab/to/guards/run_store.rb +31 -0
- data/lib/robot_lab/to/guards/write_guard.rb +71 -0
- data/lib/robot_lab/to/guards.rb +54 -0
- data/lib/robot_lab/to/iteration_result.rb +29 -0
- data/lib/robot_lab/to/jsonl_logger.rb +59 -0
- data/lib/robot_lab/to/notes_manager.rb +121 -0
- data/lib/robot_lab/to/orchestrator.rb +642 -0
- data/lib/robot_lab/to/prompt_builder.rb +208 -0
- data/lib/robot_lab/to/run.rb +109 -0
- data/lib/robot_lab/to/stop_conditions.rb +71 -0
- data/lib/robot_lab/to/tools/bash.rb +75 -0
- data/lib/robot_lab/to/tools/edit.rb +52 -0
- data/lib/robot_lab/to/tools/file_tool.rb +32 -0
- data/lib/robot_lab/to/tools/read.rb +57 -0
- data/lib/robot_lab/to/tools/request_decision.rb +66 -0
- data/lib/robot_lab/to/tools/submit_result.rb +50 -0
- data/lib/robot_lab/to/tools/write.rb +32 -0
- data/lib/robot_lab/to/verifier.rb +63 -0
- data/lib/robot_lab/to/version.rb +7 -0
- data/lib/robot_lab/to.rb +81 -0
- data/mkdocs.yml +145 -0
- metadata +175 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 27f85b41e0ea5e5a7da88c8e72363f6341670268863c31c6a94bc1bc4280c08b
|
|
4
|
+
data.tar.gz: 90605d38d5cc683a7af57c88eeef7142600b40e564e39304b6bf63f080fd3ee6
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: b2550b48a8e4826e7c7be883cd7ad6865971ad61aa7f07749abd70ff321099a15616a942fa8f04a1b7e8396e1cd8ceabb8e6f912c77023f2fa7d59ae395f3c12
|
|
7
|
+
data.tar.gz: dee01f98631dae8b4b1f90daed41de5f7478e24a0af377d2faaedeef03b7760bad0f1724a24a88912dc18eea0a204309e70d25050278c6ad7c5a3374878649a3
|
data/.envrc
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export RR=$(pwd)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
name: Deploy Documentation to GitHub Pages
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches:
|
|
5
|
+
- main
|
|
6
|
+
- develop
|
|
7
|
+
paths:
|
|
8
|
+
- "docs/**"
|
|
9
|
+
- "mkdocs.yml"
|
|
10
|
+
- ".github/workflows/deploy-github-pages.yml"
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
|
|
13
|
+
permissions:
|
|
14
|
+
contents: write
|
|
15
|
+
pages: write
|
|
16
|
+
id-token: write
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
deploy:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- name: Checkout code
|
|
23
|
+
uses: actions/checkout@v4
|
|
24
|
+
with:
|
|
25
|
+
fetch-depth: 0
|
|
26
|
+
|
|
27
|
+
- name: Setup Python
|
|
28
|
+
uses: actions/setup-python@v5
|
|
29
|
+
with:
|
|
30
|
+
python-version: 3.x
|
|
31
|
+
|
|
32
|
+
- name: Install dependencies
|
|
33
|
+
run: |
|
|
34
|
+
pip install mkdocs
|
|
35
|
+
pip install mkdocs-material
|
|
36
|
+
pip install mkdocs-macros-plugin
|
|
37
|
+
pip install mike
|
|
38
|
+
|
|
39
|
+
- name: Configure Git
|
|
40
|
+
run: |
|
|
41
|
+
git config --local user.email "action@github.com"
|
|
42
|
+
git config --local user.name "GitHub Action"
|
|
43
|
+
|
|
44
|
+
- name: Deploy to GitHub Pages
|
|
45
|
+
run: |
|
|
46
|
+
if [ "${{ github.ref }}" = "refs/heads/main" ]; then
|
|
47
|
+
echo "Deploying from main branch"
|
|
48
|
+
mkdocs gh-deploy --force --clean
|
|
49
|
+
else
|
|
50
|
+
echo "Deploying from develop branch"
|
|
51
|
+
mkdocs gh-deploy --force --clean
|
|
52
|
+
fi
|
data/.loki
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# robot_lab-to — autonomous overnight agent loop ("takeover") for RobotLab.
|
|
3
|
+
|
|
4
|
+
import_up "repo_dev.loki"
|
|
5
|
+
|
|
6
|
+
class Tasks
|
|
7
|
+
@@gem_name ||= "robot_lab-to".freeze
|
|
8
|
+
|
|
9
|
+
header "robot_lab-to v#{gem_version} — autonomous overnight loop harness"
|
|
10
|
+
|
|
11
|
+
desc "Remove all local robot-to/* branches left over from test runs"
|
|
12
|
+
def clean_up
|
|
13
|
+
sh "git checkout develop"
|
|
14
|
+
branches = `git branch --list 'robot-to/*'`.split.map(&:strip)
|
|
15
|
+
if branches.empty?
|
|
16
|
+
puts "No robot-to/* branches to remove."
|
|
17
|
+
else
|
|
18
|
+
branches.each { |b| sh "git branch -d #{b}" }
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
7 lib/robot_lab/to/cli.rb
|
|
2
|
+
2 lib/robot_lab/to/commit_manager.rb
|
|
3
|
+
19 lib/robot_lab/to/config.rb
|
|
4
|
+
9 lib/robot_lab/to/decision_manager.rb
|
|
5
|
+
2 lib/robot_lab/to/exit_summary.rb
|
|
6
|
+
1 lib/robot_lab/to/guards/checkpoint.rb
|
|
7
|
+
2 lib/robot_lab/to/guards/quality_monitor.rb
|
|
8
|
+
1 lib/robot_lab/to/guards/run_store.rb
|
|
9
|
+
3 lib/robot_lab/to/notes_manager.rb
|
|
10
|
+
30 lib/robot_lab/to/orchestrator.rb
|
|
11
|
+
1 lib/robot_lab/to/prompt_builder.rb
|
|
12
|
+
4 lib/robot_lab/to/run.rb
|
|
13
|
+
2 lib/robot_lab/to/stop_conditions.rb
|
|
14
|
+
3 lib/robot_lab/to/tools/bash.rb
|
|
15
|
+
3 lib/robot_lab/to/tools/edit.rb
|
|
16
|
+
3 lib/robot_lab/to/tools/request_decision.rb
|
|
17
|
+
1 lib/robot_lab/to/tools/submit_result.rb
|
data/.rubocop.yml
ADDED
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `robot_lab-to` are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
|
|
7
|
+
and [Conventional Commits](https://www.conventionalcommits.org/) (see `COMMITS.md`).
|
|
8
|
+
|
|
9
|
+
## [Unreleased]
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Human-in-the-loop decision files.** When the robot hits a choice it must not
|
|
14
|
+
make autonomously, it calls `request_decision` and the orchestrator writes a
|
|
15
|
+
decision file (situation, options, and its recommended "lean") under
|
|
16
|
+
`runs/<id>/decisions/`. A human resolves it out-of-band by editing the file
|
|
17
|
+
(`status: resolved` + `resolution:`); a later iteration reads the answer back.
|
|
18
|
+
New `Decision` value object, `DecisionManager`, and `RequestDecision` tool.
|
|
19
|
+
- **Blocking decisions gate the loop.** `--decision-mode wait` (default) polls
|
|
20
|
+
until the decision is resolved (`--decision-poll`, `--decision-timeout`);
|
|
21
|
+
`--decision-mode exit` stops the run with a `--resume` hint. Disable with
|
|
22
|
+
`--no-decisions`.
|
|
23
|
+
- **Resumable runs.** Run state is persisted to `run.json`; `robot-to --resume
|
|
24
|
+
<run_id>` (or `RobotLab::To.resume`) continues a stopped run on the same
|
|
25
|
+
branch — enabling cron/scheduled operation.
|
|
26
|
+
- **`decisions` subcommand.** `robot-to decisions [run_id]` lists pending and
|
|
27
|
+
resolved decisions with their file paths.
|
|
28
|
+
- **Local-model support.** Drive a local [Ollama](https://ollama.com) model
|
|
29
|
+
end-to-end with `--local-guards` and `--no-stream`.
|
|
30
|
+
- **Built-in workspace tools** (`read`, `write`, `edit`, `bash`) attached to the
|
|
31
|
+
per-iteration robot when `--local-guards` is set.
|
|
32
|
+
- **Small-model guardrails** — `write-guard`, `read-before-edit`, `checkpoint`,
|
|
33
|
+
and `quality-monitor` `RobotLab::Hook` policies that make tool use safe for
|
|
34
|
+
small local models.
|
|
35
|
+
- **`stream` setting** (`--no-stream`) — disable response streaming, required for
|
|
36
|
+
Ollama tool calls; tokens are then accounted from each iteration's result.
|
|
37
|
+
- **Documentation site** (`docs/`, `mkdocs.yml`) plus a GitHub Pages deploy
|
|
38
|
+
workflow.
|
|
39
|
+
|
|
40
|
+
### Fixed
|
|
41
|
+
|
|
42
|
+
- Tools were passed to `RobotLab.build` via `tools:` (a tool-name allowlist
|
|
43
|
+
filter) instead of `local_tools:` (the instance list), so the
|
|
44
|
+
`submit_iteration_result` tool was never attached to the robot. Now passed via
|
|
45
|
+
`local_tools:`, with a regression test asserting the built robot exposes the
|
|
46
|
+
tool.
|
|
47
|
+
|
|
48
|
+
## [0.1.0]
|
|
49
|
+
|
|
50
|
+
Initial release.
|
|
51
|
+
|
|
52
|
+
### Added
|
|
53
|
+
|
|
54
|
+
- Autonomous overnight loop (`Orchestrator`): a fresh robot per iteration, commit
|
|
55
|
+
on success, `git reset --hard` on failure.
|
|
56
|
+
- `SubmitResult` tool and `IterationResult` value object.
|
|
57
|
+
- Cross-iteration memory via `notes.md` (`NotesManager`).
|
|
58
|
+
- Stop conditions: `--max-iterations`, `--max-tokens`,
|
|
59
|
+
`--max-consecutive-failures`, and natural-language `--stop-when`.
|
|
60
|
+
- Independent verification gate (`--verify-command`, `Verifier`).
|
|
61
|
+
- Interruptible exponential `Backoff` for transient-error retries.
|
|
62
|
+
- JSONL event log (`JsonlLogger`) and post-run `ExitSummary`.
|
|
63
|
+
- `robot-to` CLI and `RobotLab::To.run` programmatic entry point.
|
|
64
|
+
- `Config` cascade (`myway_config`): defaults → user file → env → CLI.
|
|
65
|
+
|
|
66
|
+
[Unreleased]: https://github.com/MadBomber/robot_lab-to/compare/v0.1.0...HEAD
|
|
67
|
+
[0.1.0]: https://github.com/MadBomber/robot_lab-to/releases/tag/v0.1.0
|
data/CLAUDE.md
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
## Project: robot_lab-to
|
|
4
|
+
|
|
5
|
+
`robot_lab-to` ("takeover") is a robot_lab extension gem that runs a RobotLab Robot in an
|
|
6
|
+
autonomous loop toward a stated objective, committing one focused change per iteration.
|
|
7
|
+
|
|
8
|
+
## Commands
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
bundle install
|
|
12
|
+
bundle exec rake test # all tests
|
|
13
|
+
bundle exec rake test_verbose # verbose
|
|
14
|
+
bundle exec rake quality # tests + rubocop + flog
|
|
15
|
+
bin/console # IRB shell
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Architecture
|
|
19
|
+
|
|
20
|
+
- **`Orchestrator`** — main loop: setup → iterate → **eval-gated** commit/rollback → stop
|
|
21
|
+
- **`Evals`** — orchestrator-owned scorers (`Evals.build` factory + `register_eval` registry). `Base` returns a `Score(gate_ok, improved, met_target, value, detail, output)`. `Null` (default, unscored), `Code` (measured: verify floor + `--measure`/`--target`, composes `Verifier`), `Prose` (pairwise LLM judge vs. the parent commit + optional `--floor`). Commit requires `gate_ok && improved`; the run stops on `met_target`. `Base#protected_paths` feeds `GraderLock`.
|
|
22
|
+
- **`Guards::GraderLock`** — always-on hook (independent of `--local-guards`) that refuses write/edit/bash on the eval's grader artifacts (spec + `--protect-path`); wired in `Orchestrator#build_robot`.
|
|
23
|
+
- **`SubmitResultTool`** — `RobotLab::Tool` the robot must call to report its result
|
|
24
|
+
- **`RequestDecisionTool`** — `RobotLab::Tool` the robot calls to escalate a choice (async HITL)
|
|
25
|
+
- **`IterationResult`** — `Data.define` value object (success, summary, key_changes, key_learnings, should_fully_stop)
|
|
26
|
+
- **`Decision`** — `Data.define` snapshot of one decision file (status, blocking, question, recommendation, resolution)
|
|
27
|
+
- **`DecisionManager`** — read/write/query decision files; parallels `NotesManager`
|
|
28
|
+
- **`CommitManager`** — all git ops via `Open3.capture3` (no shell interpolation); `checkout_branch` for resume
|
|
29
|
+
- **`NotesManager`** — cross-iteration memory file (notes.md); orchestrator writes, robot reads
|
|
30
|
+
- **`StopConditions`** — max_iterations, max_tokens, consecutive_failures, stop_when, **stop_on_plateau** (N iterations without improvement — the primary terminator for prose)
|
|
31
|
+
- **`Backoff`** — exponential (60 × 2^n seconds) + fixed `sleep_seconds` for decision polling; interruptible via `interrupt!`
|
|
32
|
+
- **`PromptBuilder`** — builds per-iteration system prompt with objective, notes, resolved decisions, a **score-feedback section** (best score / target / plateau warning for scored runs), and conditional sections
|
|
33
|
+
- **`JsonlLogger`** — JSONL event log with 100-event pre-init buffer
|
|
34
|
+
- **`ExitSummary`** — post-run metrics table and next-step commands
|
|
35
|
+
- **`CLI`** — `OptionParser`-based, binary `robot-to`; also `--resume` and the `decisions` subcommand
|
|
36
|
+
- **`Config`** — `MywayConfig::Base`, file `~/.config/robot_lab/to.yml`, prefix `ROBOT_LAB_TO_*`
|
|
37
|
+
|
|
38
|
+
## Key Pattern: Per-Iteration Robot
|
|
39
|
+
|
|
40
|
+
A fresh `RobotLab::Tool` (`SubmitResult`, plus `RequestDecision` when decisions are
|
|
41
|
+
enabled) and `RobotLab::Robot` are created per iteration. After `robot.run()`
|
|
42
|
+
returns, the orchestrator reads `submit_tool.captured_result` (nil → treated as
|
|
43
|
+
failure) and persists any `decision_tool.captured_requests` as decision files.
|
|
44
|
+
|
|
45
|
+
## Async Human-in-the-Loop (decision files)
|
|
46
|
+
|
|
47
|
+
When the robot hits a choice it must not make alone it calls `request_decision`;
|
|
48
|
+
the orchestrator writes a decision file with YAML front matter + a human-readable
|
|
49
|
+
body. Status lifecycle: `pending` → `resolved` (human edits the file) → `closed`
|
|
50
|
+
(resolution injected into a committed iteration). A **blocking** decision gates
|
|
51
|
+
the loop via `handle_blocking_decisions`: `decision_mode: wait` polls until
|
|
52
|
+
resolved (`decision_wait_poll` / `decision_timeout`); `decision_mode: exit` stops
|
|
53
|
+
with a `--resume` hint. See `decision_files_plan.md` in the parent workspace.
|
|
54
|
+
(Note: the workspace's `hitl_plan.md` is a *separate*, synchronous terminal-gate
|
|
55
|
+
design — not this feature.)
|
|
56
|
+
|
|
57
|
+
## Resume / cron
|
|
58
|
+
|
|
59
|
+
Each iteration writes `run.json` (`Run#to_h`). `robot-to --resume <run_id>` /
|
|
60
|
+
`RobotLab::To.resume` reloads it (`Run.load`), checks out the branch, and
|
|
61
|
+
re-enters the loop — so an external scheduler can drive one commit per tick.
|
|
62
|
+
|
|
63
|
+
## Run State
|
|
64
|
+
|
|
65
|
+
All run state lives in `.robot_lab_to/runs/<run_id>/` (added to `.git/info/exclude`):
|
|
66
|
+
- `notes.md` — cross-iteration log
|
|
67
|
+
- `run.log` — JSONL event log
|
|
68
|
+
- `run.json` — serialized `Run` snapshot (enables `--resume`)
|
|
69
|
+
- `decisions/d-*.md` — decision files
|
|
70
|
+
|
|
71
|
+
## Testing
|
|
72
|
+
|
|
73
|
+
Minitest. Git-dependent tests use `Dir.mktmpdir` with real `git init`.
|
|
74
|
+
Coverage gate: 95% line / 75% branch (`rake quality` also runs rubocop + flog + flay).
|
data/COMMITS.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Commit Message Convention
|
|
2
|
+
|
|
3
|
+
This project follows Conventional Commits.
|
|
4
|
+
|
|
5
|
+
## Format
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
<type>(<scope>): <short summary>
|
|
9
|
+
|
|
10
|
+
[optional body]
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Types
|
|
14
|
+
|
|
15
|
+
- `feat` — new feature
|
|
16
|
+
- `fix` — bug fix
|
|
17
|
+
- `refactor` — code change that is neither a fix nor feature
|
|
18
|
+
- `test` — adding or updating tests
|
|
19
|
+
- `docs` — documentation only
|
|
20
|
+
- `chore` — build, deps, tooling
|
|
21
|
+
- `perf` — performance improvement
|
|
22
|
+
|
|
23
|
+
## Rules
|
|
24
|
+
|
|
25
|
+
- Summary line: imperative mood, lowercase, no trailing period, under 60 chars
|
|
26
|
+
- No co-author credits
|
|
27
|
+
- Reference GitHub issues as `(#123)` in the summary or body
|
data/README.md
ADDED
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<img src="docs/assets/images/robot_lab-to.jpg" alt="robot_lab-to" width="320"><br />
|
|
3
|
+
"Robot, takeover!"
|
|
4
|
+
</div>
|
|
5
|
+
|
|
6
|
+
# robot_lab-to
|
|
7
|
+
|
|
8
|
+
**Autonomous overnight agent loop for [RobotLab](https://github.com/MadBomber/robot_lab) — run a robot while you sleep.**
|
|
9
|
+
|
|
10
|
+
`robot_lab-to` ("takeover") runs a RobotLab `Robot` in an autonomous loop toward a
|
|
11
|
+
stated objective, committing **one focused change per iteration**. Each iteration
|
|
12
|
+
spins up a fresh robot, lets it work, and asks it to report a result. Good
|
|
13
|
+
iterations are committed; failures are rolled back. A running log (`notes.md`)
|
|
14
|
+
carries memory across iterations so the robot learns from what came before.
|
|
15
|
+
|
|
16
|
+
Start it before bed; review the branch in the morning.
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
robot-to "Increase test coverage of the parser to 90%" \
|
|
20
|
+
--max-iterations 20 \
|
|
21
|
+
--verify-command "bundle exec rake test"
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
> 📖 **Full documentation:** <https://madbomber.github.io/robot_lab-to>
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## How it works
|
|
29
|
+
|
|
30
|
+
Every iteration is an independent, sandboxed attempt at *one* improvement. The
|
|
31
|
+
orchestrator owns the git history and the stop conditions; the robot owns the
|
|
32
|
+
work.
|
|
33
|
+
|
|
34
|
+
1. **Build a fresh robot** with a per-iteration system prompt that includes the
|
|
35
|
+
objective and the accumulated notes.
|
|
36
|
+
2. **The robot works** and calls `submit_iteration_result` to report success or
|
|
37
|
+
failure.
|
|
38
|
+
3. **The orchestrator decides** — an independent `--verify-command` (if set) must
|
|
39
|
+
pass, then the change is committed. Otherwise the working tree is reset
|
|
40
|
+
(`git reset --hard`).
|
|
41
|
+
4. **Notes are updated** with what happened, becoming context for the next round.
|
|
42
|
+
5. **Stop conditions are checked** — iterations, tokens, consecutive failures, or
|
|
43
|
+
a natural-language `--stop-when` — and the loop continues or ends.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Scoring: evals
|
|
48
|
+
|
|
49
|
+
By default a "good" iteration is one the robot *reports* as success (and that
|
|
50
|
+
passes `--verify-command`, if set). That works for "don't break the build," but it
|
|
51
|
+
can't answer *"is this actually better?"* — and an agent will happily report
|
|
52
|
+
success on a change that made no real progress. An **eval** replaces the robot's
|
|
53
|
+
self-report with an orchestrator-owned, measurable judgement, and makes the loop
|
|
54
|
+
**descend toward a target** instead of merely not-breaking.
|
|
55
|
+
|
|
56
|
+
Two built-in evals cover the two kinds of product.
|
|
57
|
+
|
|
58
|
+
### Code — measured descent
|
|
59
|
+
|
|
60
|
+
For software, "better" is measurable. Give the loop a **measure** command that
|
|
61
|
+
prints a number (higher = better), and optionally a **target**:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
robot-to "Raise parser coverage to 90%" \
|
|
65
|
+
--verify "bundle exec rake test" \ # floor: must stay green (correctness)
|
|
66
|
+
--measure "bundle exec rake coverage" \ # descent signal (higher = better)
|
|
67
|
+
--target 90 # stop once the score reaches this
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
An iteration is committed only if it **improves** the measured score *and* the
|
|
71
|
+
verify floor still passes; a change that scores no better is rolled back, so the
|
|
72
|
+
branch descends monotonically and the run stops itself when the target is met.
|
|
73
|
+
Relax the monotonic gate with `--no-require-improvement`.
|
|
74
|
+
|
|
75
|
+
### Prose — pairwise judgement
|
|
76
|
+
|
|
77
|
+
For a document, an opinion piece, or a book there is no `rake coverage`. The
|
|
78
|
+
**prose** eval scores each draft with an LLM judge that compares it *pairwise*
|
|
79
|
+
against the last committed version — an LLM's absolute scores are too noisy to
|
|
80
|
+
descend, but "is B better than A?" is reliable:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
robot-to "Write an opinionated guide to the Viable Systems Model" \
|
|
84
|
+
--eval prose \
|
|
85
|
+
--spec outline.md \ # the spec the judge measures against
|
|
86
|
+
--floor "rake docs:lint" \ # optional mechanizable checks (links, TODOs)
|
|
87
|
+
--stop-on-plateau 3 # stop after 3 drafts with no improvement
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
A draft is committed only when the judge rules it **better** than its parent.
|
|
91
|
+
There is no absolute target, so the run ends on `--stop-on-plateau` (N drafts with
|
|
92
|
+
no improvement) or when you stop it.
|
|
93
|
+
|
|
94
|
+
### Guarding the grader
|
|
95
|
+
|
|
96
|
+
Whatever the eval, the robot must not be able to "win" by editing the criteria it
|
|
97
|
+
is scored against. The **spec** and any `--protect-path` files are locked for the
|
|
98
|
+
whole run — write/edit/bash attempts against them are refused. (The code test
|
|
99
|
+
suite is deliberately *not* locked; adding tests is legitimate work.)
|
|
100
|
+
|
|
101
|
+
### Custom evals
|
|
102
|
+
|
|
103
|
+
Point `--eval` at a strategy registered with `RobotLab::To.register_eval`, or in
|
|
104
|
+
Ruby pass any object responding to `#score(context)` (or a proc) as `eval:`. See
|
|
105
|
+
`RobotLab::To::Evals::Base`.
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## Human-in-the-loop: decision files
|
|
110
|
+
|
|
111
|
+
The loop runs unattended, so it can't stop and ask you a question the way a chat
|
|
112
|
+
agent can. Instead, when the robot hits a choice it **shouldn't make on its own**
|
|
113
|
+
— an irreversible change, a public API contract, a genuine ambiguity about intent
|
|
114
|
+
— it calls `request_decision` and the orchestrator writes a **decision file**:
|
|
115
|
+
the situation, the options, and the robot's own recommendation (its "lean").
|
|
116
|
+
|
|
117
|
+
A decision file is plain markdown with YAML front matter, under the run directory:
|
|
118
|
+
|
|
119
|
+
```markdown
|
|
120
|
+
---
|
|
121
|
+
id: d-20260701-143022-a1b2c3
|
|
122
|
+
status: pending # you change this to: resolved
|
|
123
|
+
blocking: true
|
|
124
|
+
resolution: # you fill this in
|
|
125
|
+
---
|
|
126
|
+
# Decision: Return 404 or 410 for deleted records?
|
|
127
|
+
|
|
128
|
+
## Options
|
|
129
|
+
1. 404 Not Found
|
|
130
|
+
2. 410 Gone
|
|
131
|
+
|
|
132
|
+
## Recommendation (robot's lean)
|
|
133
|
+
Option 2 (410 Gone) — the records are permanently deleted...
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
You resolve it out-of-band by editing the file: set `status: resolved` and fill
|
|
137
|
+
`resolution:`. The robot reads your answer on a later iteration and acts on it.
|
|
138
|
+
|
|
139
|
+
List what's waiting on you:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
robot-to decisions # pending decisions for the latest run
|
|
143
|
+
robot-to decisions <run_id> # ...for a specific run
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
**Blocking decisions** gate the loop until you answer, in one of two modes:
|
|
147
|
+
|
|
148
|
+
- `--decision-mode wait` (default) — the process stays alive and polls the
|
|
149
|
+
decision file until you resolve it (or `--decision-timeout` elapses).
|
|
150
|
+
- `--decision-mode exit` — the process commits its safe work and **stops**,
|
|
151
|
+
telling you how to resume. Ideal for cron: nothing runs until you decide.
|
|
152
|
+
|
|
153
|
+
### Resuming a run (and cron scheduling)
|
|
154
|
+
|
|
155
|
+
Every run persists its state to `run.json`, so a stopped run can be continued on
|
|
156
|
+
the same branch:
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
robot-to --resume <run_id>
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
This makes `robot_lab-to` schedulable — a cron entry that runs
|
|
163
|
+
`robot-to --resume <run_id>` each hour advances the work or re-pauses on the next
|
|
164
|
+
open decision, one commit at a time.
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## Installation
|
|
169
|
+
|
|
170
|
+
Requires **Ruby >= 3.2** and a git repository with at least one commit.
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
gem install robot_lab-to
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Or in a `Gemfile`:
|
|
177
|
+
|
|
178
|
+
```ruby
|
|
179
|
+
gem "robot_lab-to"
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
This installs the `robot-to` executable.
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Quick start
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
cd my-project
|
|
190
|
+
git commit --allow-empty -m "initial" # the loop needs a HEAD to branch from
|
|
191
|
+
|
|
192
|
+
robot-to "Add a CHANGELOG.md and document the public API in the README" \
|
|
193
|
+
--max-iterations 5 \
|
|
194
|
+
--provider anthropic \
|
|
195
|
+
--model claude-sonnet-4-6
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
The loop creates a `robot-to/<slug>-<timestamp>` branch and commits one focused
|
|
199
|
+
change per successful iteration. Review it in the morning:
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
git log --oneline robot-to/... # one commit per iteration
|
|
203
|
+
git diff main..HEAD # the full change
|
|
204
|
+
cat .robot_lab_to/runs/*/notes.md # the reasoning log
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### Programmatic use
|
|
208
|
+
|
|
209
|
+
```ruby
|
|
210
|
+
require "robot_lab"
|
|
211
|
+
require "robot_lab/to"
|
|
212
|
+
|
|
213
|
+
RobotLab::To.run(
|
|
214
|
+
"Add a CHANGELOG.md and document the public API",
|
|
215
|
+
provider: :anthropic,
|
|
216
|
+
model: "claude-sonnet-4-6",
|
|
217
|
+
max_iterations: 5,
|
|
218
|
+
verify_command: "bundle exec rake test"
|
|
219
|
+
)
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
---
|
|
223
|
+
|
|
224
|
+
## Key features
|
|
225
|
+
|
|
226
|
+
- **Bounded, reviewable history** — each iteration is a single commit; bad
|
|
227
|
+
attempts are reset before the next try.
|
|
228
|
+
- **Cross-iteration memory** — `notes.md` records summaries, changes, and
|
|
229
|
+
learnings so the robot doesn't repeat itself.
|
|
230
|
+
- **Independent verification** — the robot can't self-certify success; a real
|
|
231
|
+
command you choose (`--verify-command`) is the deciding authority.
|
|
232
|
+
- **It stops on its own** — `--max-iterations`, `--max-tokens`,
|
|
233
|
+
`--max-consecutive-failures`, and a natural-language `--stop-when`.
|
|
234
|
+
- **Asks before overstepping** — the robot escalates irreversible or ambiguous
|
|
235
|
+
choices as **decision files** you resolve out-of-band, instead of guessing.
|
|
236
|
+
- **Resumable & cron-friendly** — run state is persisted to `run.json`;
|
|
237
|
+
`robot-to --resume <run_id>` continues a stopped run, so an external scheduler
|
|
238
|
+
can drive it one commit per tick.
|
|
239
|
+
- **Runs on local models** — with `--local-guards` it ships built-in file tools
|
|
240
|
+
and small-model guardrails to drive a local Ollama model offline.
|
|
241
|
+
|
|
242
|
+
---
|
|
243
|
+
|
|
244
|
+
## Local models
|
|
245
|
+
|
|
246
|
+
`robot_lab-to` can drive a local model running offline against
|
|
247
|
+
[Ollama](https://ollama.com) — no API keys, no per-token cost.
|
|
248
|
+
|
|
249
|
+
```bash
|
|
250
|
+
ollama pull gpt-oss:20b
|
|
251
|
+
|
|
252
|
+
robot-to "Add a greet(name) method in greeter.rb" \
|
|
253
|
+
--provider openai \
|
|
254
|
+
--model gpt-oss:20b \
|
|
255
|
+
--local-guards \
|
|
256
|
+
--no-stream \
|
|
257
|
+
--max-iterations 5
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
`--local-guards` attaches built-in `read`/`write`/`edit`/`bash` tools plus
|
|
261
|
+
guardrail hooks (`write-guard`, `read-before-edit`, `checkpoint`,
|
|
262
|
+
`quality-monitor`) adapted from [`little-coder`](https://github.com/itayinbarr/little-coder)
|
|
263
|
+
that catch the mistakes small models reliably make. `--no-stream` is required —
|
|
264
|
+
Ollama suppresses tool calls when streaming, and `:openai` is used as the
|
|
265
|
+
provider against Ollama's OpenAI-compatible endpoint.
|
|
266
|
+
|
|
267
|
+
See the [Local Models guide](https://madbomber.github.io/robot_lab-to/local-models/)
|
|
268
|
+
for the full setup, including the RubyLLM configuration and model selection.
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
## Configuration
|
|
273
|
+
|
|
274
|
+
Settings cascade from lowest to highest precedence:
|
|
275
|
+
|
|
276
|
+
```
|
|
277
|
+
bundled defaults → ~/.config/robot_lab/to.yml → ROBOT_LAB_TO_* env → CLI flags
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
Run `robot-to --help` for all flags, or see the
|
|
281
|
+
[Configuration reference](https://madbomber.github.io/robot_lab-to/configuration/settings/).
|
|
282
|
+
|
|
283
|
+
### Environment variables
|
|
284
|
+
|
|
285
|
+
Every persistent setting can be set with a `ROBOT_LAB_TO_*` environment variable
|
|
286
|
+
(integers and booleans are coerced automatically). An env var overrides the
|
|
287
|
+
bundled default and the user config file, but is overridden by a CLI flag.
|
|
288
|
+
|
|
289
|
+
| Environment variable | Setting | Default |
|
|
290
|
+
|----------------------|---------|---------|
|
|
291
|
+
| `ROBOT_LAB_TO_PROVIDER` | LLM provider | `openai` |
|
|
292
|
+
| `ROBOT_LAB_TO_MODEL` | Model identifier | `gpt-5.5` |
|
|
293
|
+
| `ROBOT_LAB_TO_STREAM` | Stream responses (`--no-stream` to disable) | `true` |
|
|
294
|
+
| `ROBOT_LAB_TO_LOCAL_GUARDS` | Attach file tools + small-model guardrails | `false` |
|
|
295
|
+
| `ROBOT_LAB_TO_MAX_TOOL_ROUNDS` | Max tool-call rounds per iteration | `100` |
|
|
296
|
+
| `ROBOT_LAB_TO_MAX_CONSECUTIVE_FAILURES` | Abort after N consecutive failures | `3` |
|
|
297
|
+
| `ROBOT_LAB_TO_MAX_VERIFY_REPAIRS` | Repair-in-place attempts on verify failure | `2` |
|
|
298
|
+
| `ROBOT_LAB_TO_MAX_RETRIES` | Retries for transient (e.g. API) errors | `2` |
|
|
299
|
+
| `ROBOT_LAB_TO_MAX_SUBMIT_NUDGES` | Re-prompts when no result is submitted | `1` |
|
|
300
|
+
| `ROBOT_LAB_TO_VERIFY_TIMEOUT` | Timeout (seconds) for `--verify-command` | `600` |
|
|
301
|
+
| `ROBOT_LAB_TO_COMMIT_FORMAT` | `default` or `conventional` | `default` |
|
|
302
|
+
| `ROBOT_LAB_TO_RUN_DIR` | Directory for run state | `.robot_lab_to` |
|
|
303
|
+
| `ROBOT_LAB_TO_DECISIONS_ENABLED` | Offer the `request_decision` tool | `true` |
|
|
304
|
+
| `ROBOT_LAB_TO_DECISION_MODE` | On a blocking decision: `wait` or `exit` | `wait` |
|
|
305
|
+
| `ROBOT_LAB_TO_DECISION_WAIT_POLL` | Seconds between polls while waiting | `30` |
|
|
306
|
+
| `ROBOT_LAB_TO_DEBUG` | Keep verbose provider logging enabled | `false` |
|
|
307
|
+
|
|
308
|
+
```bash
|
|
309
|
+
export ROBOT_LAB_TO_MODEL="gpt-oss:20b"
|
|
310
|
+
export ROBOT_LAB_TO_LOCAL_GUARDS=true
|
|
311
|
+
export ROBOT_LAB_TO_STREAM=false
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
> **Per-run settings have no env var.** `--max-iterations`, `--max-tokens`,
|
|
315
|
+
> `--stop-when`, `--verify-command`, and `--decision-timeout` are intentionally
|
|
316
|
+
> CLI-only (they default to "unset / no limit") and must be passed on the command
|
|
317
|
+
> line or to `RobotLab::To.run`.
|
|
318
|
+
|
|
319
|
+
### Provider credentials
|
|
320
|
+
|
|
321
|
+
API keys are read by the underlying provider (via RobotLab / RubyLLM), **not** by
|
|
322
|
+
`robot_lab-to` itself — e.g. `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`. Local
|
|
323
|
+
[Ollama](https://ollama.com) models need no key; see the
|
|
324
|
+
[Local Models guide](https://madbomber.github.io/robot_lab-to/local-models/ollama/).
|
|
325
|
+
|
|
326
|
+
---
|
|
327
|
+
|
|
328
|
+
## Development
|
|
329
|
+
|
|
330
|
+
```bash
|
|
331
|
+
bundle install # install dependencies
|
|
332
|
+
bundle exec rake test # run the test suite (SimpleCov coverage)
|
|
333
|
+
bundle exec rubocop # lint
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
Documentation is built with [MkDocs Material](https://squidfunk.github.io/mkdocs-material/):
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
mkdocs serve # preview at http://127.0.0.1:8000
|
|
340
|
+
mkdocs build # render to site/
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
---
|
|
344
|
+
|
|
345
|
+
## Contributing
|
|
346
|
+
|
|
347
|
+
Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/);
|
|
348
|
+
see `COMMITS.md`. See `CHANGELOG.md` for release history.
|
|
349
|
+
|
|
350
|
+
## License
|
|
351
|
+
|
|
352
|
+
Released under the [MIT License](https://opensource.org/licenses/MIT).
|