robot_lab-to 0.2.7 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.envrc +6 -0
- data/Archspec.rb +34 -0
- data/CHANGELOG.md +4 -0
- data/CLAUDE.md +3 -2
- data/Rakefile +6 -108
- data/examples/.envrc +8 -0
- data/examples/01_basic_usage/README.md +14 -15
- data/examples/01_basic_usage/basic_usage.rb +20 -57
- data/examples/02_advanced_usage/README.md +10 -7
- data/examples/02_advanced_usage/advanced_usage.rb +23 -31
- data/examples/03_scored/scored_run.rb +19 -51
- data/examples/04_prose/README.md +15 -14
- data/examples/04_prose/prose_run.rb +22 -38
- data/examples/common.rb +114 -0
- data/lib/robot_lab/to/cli.rb +61 -34
- data/lib/robot_lab/to/commit_manager.rb +4 -0
- data/lib/robot_lab/to/config.rb +12 -0
- data/lib/robot_lab/to/decision_manager.rb +12 -1
- data/lib/robot_lab/to/exit_summary.rb +17 -16
- data/lib/robot_lab/to/guards/checkpoint.rb +6 -3
- data/lib/robot_lab/to/guards/quality_monitor.rb +4 -2
- data/lib/robot_lab/to/guards/run_store.rb +4 -2
- data/lib/robot_lab/to/notes_manager.rb +3 -0
- data/lib/robot_lab/to/orchestrator.rb +79 -32
- data/lib/robot_lab/to/prompt_builder.rb +2 -0
- data/lib/robot_lab/to/run.rb +8 -8
- data/lib/robot_lab/to/stop_conditions.rb +10 -6
- data/lib/robot_lab/to/tools/bash.rb +5 -0
- data/lib/robot_lab/to/tools/edit.rb +5 -0
- data/lib/robot_lab/to/tools/request_decision.rb +4 -0
- data/lib/robot_lab/to/tools/submit_result.rb +1 -0
- data/lib/robot_lab/to/version.rb +1 -1
- metadata +5 -2
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
# ---------------------------------------------------------------------------
|
|
27
27
|
# Run it
|
|
28
28
|
# ---------------------------------------------------------------------------
|
|
29
|
-
# # Local
|
|
30
|
-
#
|
|
29
|
+
# # Local LM Studio (default — no API key; common.rb starts the server and
|
|
30
|
+
# # loads the model for you if they aren't already running/loaded):
|
|
31
31
|
# bundle exec ruby examples/03_scored/scored_run.rb
|
|
32
32
|
#
|
|
33
33
|
# # A cloud model instead:
|
|
@@ -35,17 +35,15 @@
|
|
|
35
35
|
# ANTHROPIC_API_KEY=sk-... \
|
|
36
36
|
# bundle exec ruby examples/03_scored/scored_run.rb
|
|
37
37
|
#
|
|
38
|
-
# Configuration (all optional, via environment):
|
|
39
|
-
# RLTO_LOCAL true|false use a local
|
|
40
|
-
# RLTO_PROVIDER name LLM provider (default:
|
|
41
|
-
# RLTO_MODEL id model id (default: qwen3.
|
|
42
|
-
#
|
|
38
|
+
# Configuration (all optional, via environment; examples/.envrc sets these for you):
|
|
39
|
+
# RLTO_LOCAL true|false use a local LM Studio model (default true)
|
|
40
|
+
# RLTO_PROVIDER name LLM provider label (default: lms for local)
|
|
41
|
+
# RLTO_MODEL id model id (default: qwen/qwen3.8-27b for local)
|
|
42
|
+
# LMS_BASE_URL url LM Studio OpenAI-compatible endpoint (default localhost:1234/v1)
|
|
43
43
|
# ===========================================================================
|
|
44
44
|
|
|
45
45
|
require "fileutils"
|
|
46
|
-
require "logger"
|
|
47
46
|
require "open3"
|
|
48
|
-
require "net/http"
|
|
49
47
|
|
|
50
48
|
# Make the example runnable straight from the repo during development, with or
|
|
51
49
|
# without `bundle exec`. (When the gem is installed normally these paths don't
|
|
@@ -55,49 +53,24 @@ require "net/http"
|
|
|
55
53
|
File.expand_path("../../../robot_lab/lib", __dir__) # sibling robot_lab/lib
|
|
56
54
|
].each { |p| $LOAD_PATH.unshift(p) if Dir.exist?(p) }
|
|
57
55
|
|
|
58
|
-
require "ruby_llm"
|
|
59
56
|
require "robot_lab"
|
|
60
57
|
require "robot_lab/to"
|
|
58
|
+
require_relative "../common"
|
|
61
59
|
|
|
62
60
|
# --- configuration ---------------------------------------------------------
|
|
63
61
|
|
|
64
62
|
LOCAL = ENV.fetch("RLTO_LOCAL", "true") == "true"
|
|
65
|
-
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "
|
|
66
|
-
MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen3.
|
|
67
|
-
OLLAMA = ENV.fetch("OLLAMA_BASE", "http://localhost:11434/v1")
|
|
63
|
+
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "lms" : "anthropic").to_sym
|
|
64
|
+
MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen/qwen3.8-27b" : "claude-sonnet-4-6")
|
|
68
65
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
c.openai_api_base = OLLAMA
|
|
76
|
-
c.openai_api_key = "ollama"
|
|
77
|
-
c.request_timeout = 600
|
|
78
|
-
end
|
|
79
|
-
RubyLLM.logger.level = Logger::ERROR
|
|
80
|
-
RubyLLM.models.refresh!
|
|
81
|
-
rescue StandardError => e
|
|
82
|
-
warn "warning: could not refresh Ollama models (#{e.class}: #{e.message})"
|
|
83
|
-
end
|
|
66
|
+
# ruby_llm has no native "lms" adapter. "lms" is this example's friendly label for
|
|
67
|
+
# "a local LM Studio model"; setup (common.rb) resolves it to RubyLLM's :openai
|
|
68
|
+
# adapter pointed at LM Studio, starting the server and loading MODEL as needed.
|
|
69
|
+
# Everything passed to RobotLab uses the resolved provider; PROVIDER itself is
|
|
70
|
+
# kept only for display.
|
|
71
|
+
LLM_PROVIDER = setup(provider: PROVIDER, model: MODEL)
|
|
84
72
|
|
|
85
|
-
|
|
86
|
-
uri = URI.join(OLLAMA, "models")
|
|
87
|
-
Net::HTTP.start(uri.host, uri.port, open_timeout: 2, read_timeout: 2) { |h| h.get(uri.request_uri) }
|
|
88
|
-
rescue StandardError
|
|
89
|
-
abort <<~MSG
|
|
90
|
-
Cannot reach an Ollama server at #{OLLAMA}.
|
|
91
|
-
Start it and pull a tool-capable model first:
|
|
92
|
-
|
|
93
|
-
ollama serve &
|
|
94
|
-
ollama pull #{MODEL}
|
|
95
|
-
|
|
96
|
-
Or run against a cloud model:
|
|
97
|
-
RLTO_LOCAL=false RLTO_PROVIDER=anthropic RLTO_MODEL=claude-sonnet-4-6 \\
|
|
98
|
-
ANTHROPIC_API_KEY=sk-... ruby #{File.basename(__FILE__)}
|
|
99
|
-
MSG
|
|
100
|
-
end
|
|
73
|
+
TOTAL_TESTS = 9 # the seeded suite has 9 test methods — the target
|
|
101
74
|
|
|
102
75
|
# --- sandbox repository ----------------------------------------------------
|
|
103
76
|
|
|
@@ -290,15 +263,10 @@ OBJECTIVE = <<~OBJ.strip
|
|
|
290
263
|
Call submit_result each iteration describing what you improved.
|
|
291
264
|
OBJ
|
|
292
265
|
|
|
293
|
-
if LOCAL
|
|
294
|
-
preflight_local!
|
|
295
|
-
configure_local!
|
|
296
|
-
end
|
|
297
|
-
|
|
298
266
|
clean_slate!
|
|
299
267
|
sandbox = make_sandbox
|
|
300
268
|
puts "Project dir: #{sandbox}"
|
|
301
|
-
puts "Provider/model: #{PROVIDER}/#{MODEL} (#{LOCAL ? 'local
|
|
269
|
+
puts "Provider/model: #{PROVIDER}/#{MODEL} (#{LOCAL ? 'local LM Studio' : 'cloud'})"
|
|
302
270
|
puts "Eval: measured descent — score = passing tests, target #{TOTAL_TESTS}"
|
|
303
271
|
puts "Locked grader: score.rb, test/roman_numeral_test.rb"
|
|
304
272
|
puts
|
|
@@ -308,7 +276,7 @@ RobotLab.on(FeedbackHook)
|
|
|
308
276
|
Dir.chdir(sandbox) do
|
|
309
277
|
RobotLab::To.run(
|
|
310
278
|
OBJECTIVE,
|
|
311
|
-
provider:
|
|
279
|
+
provider: LLM_PROVIDER,
|
|
312
280
|
model: MODEL,
|
|
313
281
|
local_guards: LOCAL,
|
|
314
282
|
stream: !LOCAL,
|
data/examples/04_prose/README.md
CHANGED
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
Demo 03 scores **code** with a deterministic command (passing tests). Prose has no
|
|
4
4
|
`rake coverage`, so this demo uses the **`prose` eval** — a pairwise LLM judge — and
|
|
5
|
-
puts two
|
|
5
|
+
puts two roles on the same local model by default, each independently swappable:
|
|
6
6
|
|
|
7
7
|
| Role | Model (default) | Job |
|
|
8
8
|
|------|-----------------|-----|
|
|
9
|
-
| **Doer** | `qwen3.
|
|
10
|
-
| **Verifier** (judge) | `
|
|
9
|
+
| **Doer** | `qwen/qwen3.8-27b` | writes and improves `guide.md` |
|
|
10
|
+
| **Verifier** (judge) | `qwen/qwen3.8-27b` | compares each draft to the last committed one and rules it **better / worse / same** |
|
|
11
11
|
|
|
12
12
|
Only a draft the judge rules **better** is committed, so every commit is a genuine
|
|
13
13
|
improvement. There's no absolute target (LLM scores are too noisy to descend), so
|
|
@@ -18,25 +18,26 @@ cannot edit the criteria it's judged against.
|
|
|
18
18
|
## Run it
|
|
19
19
|
|
|
20
20
|
```bash
|
|
21
|
-
ollama pull qwen3.6:latest # doer
|
|
22
|
-
ollama pull gpt-oss:latest # verifier / judge
|
|
23
21
|
bundle exec ruby examples/04_prose/prose_run.rb
|
|
24
22
|
```
|
|
25
23
|
|
|
26
|
-
|
|
24
|
+
`common.rb` starts the LM Studio server and loads the doer + judge models for
|
|
25
|
+
you if they aren't already running/loaded. Override the models via env:
|
|
26
|
+
`RLTO_MODEL` (doer), `RLTO_JUDGE_MODEL` (verifier).
|
|
27
27
|
|
|
28
|
-
## Why
|
|
28
|
+
## Why the judge can be the same model as the doer
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
*
|
|
30
|
+
The **doer** is given the file-editing tools; the **judge** is given **no tools**
|
|
31
|
+
at all — it just reads two versions and replies `better`/`worse`/`same`. As a pure
|
|
32
|
+
text responder it doesn't need to be a stronger or different model to be reliable,
|
|
33
|
+
which is why both roles default to the same local `qwen/qwen3.8-27b`. This is still the
|
|
34
|
+
separation-of-duties payoff: the model that *decides* is called independently of
|
|
35
|
+
the model that *acts*, so swap in a stronger `RLTO_JUDGE_MODEL` if you want a
|
|
36
|
+
tougher judge without touching the doer.
|
|
36
37
|
|
|
37
38
|
## The mechanism
|
|
38
39
|
|
|
39
|
-
`RobotLab::To.run(..., eval: "prose", eval_judge_model: "
|
|
40
|
+
`RobotLab::To.run(..., eval: "prose", eval_judge_model: "qwen/qwen3.8-27b")` builds an
|
|
40
41
|
`Evals::Prose`. Each iteration, after the doer edits `guide.md`, the eval diffs the
|
|
41
42
|
working tree against the parent commit, shows both versions plus the spec to the
|
|
42
43
|
judge model, and maps its verdict onto `Score#improved`. Commit on *better*, roll
|
|
@@ -16,9 +16,10 @@
|
|
|
16
16
|
# section. The run ends when every outline section has a READY file.
|
|
17
17
|
# Finally the approved sections are assembled into guide.md.
|
|
18
18
|
#
|
|
19
|
-
# Two models, two roles
|
|
20
|
-
#
|
|
21
|
-
#
|
|
19
|
+
# Two models, two roles (the same local model by default, but independently
|
|
20
|
+
# configurable — pass --judge-model / RLTO_JUDGE_MODEL for a stronger judge):
|
|
21
|
+
# DOER (default qwen/qwen3.8-27b) — writes the outline and the sections
|
|
22
|
+
# VERIFIER (default qwen/qwen3.8-27b) — the judge; grades each artifact absolutely
|
|
22
23
|
#
|
|
23
24
|
# The judge writes its feedback to REVIEW.md (git-ignored in the sandbox); the
|
|
24
25
|
# doer is told to read REVIEW.md each iteration. write_guard is disabled for the
|
|
@@ -27,18 +28,17 @@
|
|
|
27
28
|
# ---------------------------------------------------------------------------
|
|
28
29
|
# Run it
|
|
29
30
|
# ---------------------------------------------------------------------------
|
|
30
|
-
# ollama pull qwen3.6:latest # doer
|
|
31
|
-
# ollama pull gpt-oss:latest # verifier / judge
|
|
32
31
|
# bundle exec ruby examples/04_prose/prose_run.rb
|
|
33
32
|
#
|
|
33
|
+
# common.rb starts the LM Studio server and loads the doer + judge models for
|
|
34
|
+
# you if they aren't already running/loaded.
|
|
35
|
+
#
|
|
34
36
|
# Env: RLTO_MODEL (doer), RLTO_JUDGE_MODEL (verifier), RLTO_TOPIC, RLTO_LOCAL,
|
|
35
|
-
# RLTO_PROVIDER,
|
|
37
|
+
# RLTO_PROVIDER, LMS_BASE_URL. (examples/.envrc sets these for you)
|
|
36
38
|
# ===========================================================================
|
|
37
39
|
|
|
38
40
|
require "fileutils"
|
|
39
|
-
require "logger"
|
|
40
41
|
require "open3"
|
|
41
|
-
require "net/http"
|
|
42
42
|
|
|
43
43
|
[
|
|
44
44
|
File.expand_path("../../lib", __dir__),
|
|
@@ -51,38 +51,27 @@ require "net/http"
|
|
|
51
51
|
PROMPTS_DIR = File.expand_path("prompts_dir", __dir__)
|
|
52
52
|
ENV["ROBOT_LAB_TEMPLATE_PATH"] = PROMPTS_DIR
|
|
53
53
|
|
|
54
|
-
require "ruby_llm"
|
|
55
54
|
require "robot_lab"
|
|
56
55
|
require "robot_lab/to"
|
|
56
|
+
require_relative "../common"
|
|
57
57
|
RobotLab.reload_config! if RobotLab.respond_to?(:reload_config!)
|
|
58
58
|
|
|
59
59
|
# --- configuration ---------------------------------------------------------
|
|
60
60
|
|
|
61
61
|
LOCAL = ENV.fetch("RLTO_LOCAL", "true") == "true"
|
|
62
|
-
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "
|
|
63
|
-
DOER_MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen3.
|
|
64
|
-
JUDGE_MODEL = ENV.fetch("RLTO_JUDGE_MODEL", LOCAL ? "
|
|
65
|
-
OLLAMA = ENV.fetch("OLLAMA_BASE", "http://localhost:11434/v1")
|
|
62
|
+
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "lms" : "anthropic").to_sym
|
|
63
|
+
DOER_MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen/qwen3.8-27b" : "claude-sonnet-4-6")
|
|
64
|
+
JUDGE_MODEL = ENV.fetch("RLTO_JUDGE_MODEL", LOCAL ? "qwen/qwen3.8-27b" : "claude-sonnet-4-6")
|
|
66
65
|
TOPIC = ENV.fetch("RLTO_TOPIC", "writing good Git commit messages")
|
|
67
66
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
rescue StandardError => e
|
|
77
|
-
warn "warning: could not refresh Ollama models (#{e.class}: #{e.message})"
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def preflight_local!
|
|
81
|
-
uri = URI.join(OLLAMA, "models")
|
|
82
|
-
Net::HTTP.start(uri.host, uri.port, open_timeout: 2, read_timeout: 2) { |h| h.get(uri.request_uri) }
|
|
83
|
-
rescue StandardError
|
|
84
|
-
abort "Cannot reach Ollama at #{OLLAMA}. Start it and pull #{DOER_MODEL} + #{JUDGE_MODEL}."
|
|
85
|
-
end
|
|
67
|
+
# ruby_llm has no native "lms" adapter. "lms" is this example's friendly label for
|
|
68
|
+
# "a local LM Studio model"; setup (common.rb) resolves it to RubyLLM's :openai
|
|
69
|
+
# adapter pointed at LM Studio, starting the server and loading each model as
|
|
70
|
+
# needed -- once for the doer, again for the judge (a no-op if they're the same
|
|
71
|
+
# model, or if it's already loaded). Everything passed to RobotLab uses the
|
|
72
|
+
# resolved provider; PROVIDER itself is kept only for display.
|
|
73
|
+
LLM_PROVIDER = setup(provider: PROVIDER, model: DOER_MODEL)
|
|
74
|
+
setup(provider: PROVIDER, model: JUDGE_MODEL)
|
|
86
75
|
|
|
87
76
|
# --- the judge (absolute grader) -------------------------------------------
|
|
88
77
|
|
|
@@ -90,7 +79,7 @@ end
|
|
|
90
79
|
# prompt is a robot_lab template (prompts_dir/judge.md).
|
|
91
80
|
# @return [Array(Boolean, String)] [ready?, feedback]
|
|
92
81
|
def grade(criteria, text)
|
|
93
|
-
judge = RobotLab.build(name: "judge", model: JUDGE_MODEL, provider:
|
|
82
|
+
judge = RobotLab.build(name: "judge", model: JUDGE_MODEL, provider: LLM_PROVIDER, template: :judge)
|
|
94
83
|
message = RobotLab.render_template(:grade_message, criteria: criteria, artifact: text)
|
|
95
84
|
reply = judge.run(message).last_text_content.to_s
|
|
96
85
|
ready = reply.match?(/\bREADY\b/i) && !reply.match?(/NEEDS_WORK/i)
|
|
@@ -255,11 +244,6 @@ end
|
|
|
255
244
|
|
|
256
245
|
# --- main ------------------------------------------------------------------
|
|
257
246
|
|
|
258
|
-
if LOCAL
|
|
259
|
-
preflight_local!
|
|
260
|
-
configure_local!
|
|
261
|
-
end
|
|
262
|
-
|
|
263
247
|
clean_slate!
|
|
264
248
|
sandbox = make_sandbox
|
|
265
249
|
puts "Project dir: #{sandbox}"
|
|
@@ -271,7 +255,7 @@ puts
|
|
|
271
255
|
RobotLab.on(FeedbackHook)
|
|
272
256
|
|
|
273
257
|
common = {
|
|
274
|
-
provider:
|
|
258
|
+
provider: LLM_PROVIDER, model: DOER_MODEL, local_guards: LOCAL, stream: !LOCAL,
|
|
275
259
|
run_dir: RUN_DIR, write_guard: false, require_improvement: false
|
|
276
260
|
}
|
|
277
261
|
|
data/examples/common.rb
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
#
|
|
3
|
+
# ===========================================================================
|
|
4
|
+
# common.rb — shared RubyLLM setup/teardown for the robot_lab-to example demos
|
|
5
|
+
# ===========================================================================
|
|
6
|
+
#
|
|
7
|
+
# Every example under examples/ requires this file (examples/.envrc supplies
|
|
8
|
+
# the RLTO_* / LMS_BASE_URL defaults) and calls `setup` before talking to
|
|
9
|
+
# RubyLLM -- once for a single-model example, or once per model role for an
|
|
10
|
+
# example with more than one (e.g. 04_prose's doer + judge).
|
|
11
|
+
#
|
|
12
|
+
# actual_provider = setup(provider: PROVIDER, model: MODEL)
|
|
13
|
+
# RobotLab::To.run(objective, provider: actual_provider, model: MODEL, ...)
|
|
14
|
+
#
|
|
15
|
+
# ruby_llm has no native "lms" adapter -- "lms" is this example suite's own
|
|
16
|
+
# label for "a local LM Studio model". When `provider` is "lms", setup:
|
|
17
|
+
# 1. points RubyLLM's :openai adapter at LMS_BASE_URL
|
|
18
|
+
# 2. starts the LM Studio server if it isn't already running
|
|
19
|
+
# 3. loads `model` into LM Studio if it isn't already loaded
|
|
20
|
+
# and returns :openai -- the RubyLLM-recognized provider to pass to RobotLab.
|
|
21
|
+
# Any other provider (a cloud one) passes straight through untouched; setup
|
|
22
|
+
# does nothing else for it (the example configures its own API key as before).
|
|
23
|
+
#
|
|
24
|
+
# teardown stops the LM Studio server, but ONLY if setup started it here -- a
|
|
25
|
+
# server you already had running (the LM Studio app, another example) is left
|
|
26
|
+
# alone. It runs automatically via an at_exit hook; examples never call it.
|
|
27
|
+
# ===========================================================================
|
|
28
|
+
|
|
29
|
+
require "json"
|
|
30
|
+
require "logger"
|
|
31
|
+
require "open3"
|
|
32
|
+
require "uri"
|
|
33
|
+
|
|
34
|
+
require "ruby_llm"
|
|
35
|
+
|
|
36
|
+
LMS_BASE_URL = ENV.fetch("LMS_BASE_URL", "http://localhost:1234/v1")
|
|
37
|
+
|
|
38
|
+
@lms_server_started_by_us = false
|
|
39
|
+
|
|
40
|
+
def setup(provider: ENV.fetch("RLTO_PROVIDER", "lms"), model: ENV.fetch("RLTO_MODEL", "qwen/qwen3.8-27b"))
|
|
41
|
+
provider = provider.to_sym
|
|
42
|
+
return provider unless provider == :lms
|
|
43
|
+
|
|
44
|
+
configure_lms!
|
|
45
|
+
ensure_lms_server_running!
|
|
46
|
+
ensure_lms_model_loaded!(model)
|
|
47
|
+
:openai
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def teardown
|
|
51
|
+
return unless @lms_server_started_by_us
|
|
52
|
+
|
|
53
|
+
puts "Stopping the LM Studio server (started for this run)…"
|
|
54
|
+
Open3.capture2e("lms", "server", "stop")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
at_exit { teardown }
|
|
58
|
+
|
|
59
|
+
# --- internals ---------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
# Route RubyLLM's :openai provider at LM Studio's OpenAI-compatible endpoint and
|
|
62
|
+
# refresh the registry so tool attachment works.
|
|
63
|
+
def configure_lms!
|
|
64
|
+
RubyLLM.configure do |c|
|
|
65
|
+
c.openai_api_base = LMS_BASE_URL
|
|
66
|
+
c.openai_api_key = "lm-studio" # ignored by LM Studio, but RubyLLM wants a value
|
|
67
|
+
c.request_timeout = 600
|
|
68
|
+
end
|
|
69
|
+
RubyLLM.logger.level = Logger::ERROR
|
|
70
|
+
RubyLLM.models.refresh!
|
|
71
|
+
rescue StandardError => e
|
|
72
|
+
warn "warning: could not refresh LM Studio models (#{e.class}: #{e.message})"
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def lms_server_running?
|
|
76
|
+
out, status = Open3.capture2("lms", "server", "status", "--json")
|
|
77
|
+
status.success? && JSON.parse(out)["running"] == true
|
|
78
|
+
rescue StandardError
|
|
79
|
+
false
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def ensure_lms_server_running!
|
|
83
|
+
return if lms_server_running?
|
|
84
|
+
|
|
85
|
+
puts "Starting the LM Studio server…"
|
|
86
|
+
port = URI.parse(LMS_BASE_URL).port.to_s
|
|
87
|
+
out, status = Open3.capture2e("lms", "server", "start", "--port", port)
|
|
88
|
+
abort "Could not start the LM Studio server:\n#{out}" unless status.success? && lms_server_running?
|
|
89
|
+
|
|
90
|
+
@lms_server_started_by_us = true
|
|
91
|
+
rescue Errno::ENOENT
|
|
92
|
+
abort "The `lms` CLI was not found. Install LM Studio (https://lmstudio.ai) and " \
|
|
93
|
+
"run `lms bootstrap` to put it on your PATH."
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def lms_model_loaded?(model)
|
|
97
|
+
out, status = Open3.capture2("lms", "ps", "--json")
|
|
98
|
+
return false unless status.success?
|
|
99
|
+
|
|
100
|
+
JSON.parse(out).any? { |m| m["modelKey"] == model || m["identifier"] == model }
|
|
101
|
+
rescue StandardError
|
|
102
|
+
false
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def ensure_lms_model_loaded!(model)
|
|
106
|
+
return if lms_model_loaded?(model)
|
|
107
|
+
|
|
108
|
+
puts "Loading #{model} into LM Studio (first run only)…"
|
|
109
|
+
out, status = Open3.capture2e("lms", "load", model, "-y")
|
|
110
|
+
abort "Could not load #{model} in LM Studio:\n#{out}" unless status.success?
|
|
111
|
+
rescue Errno::ENOENT
|
|
112
|
+
abort "The `lms` CLI was not found. Install LM Studio (https://lmstudio.ai) and " \
|
|
113
|
+
"run `lms bootstrap` to put it on your PATH."
|
|
114
|
+
end
|
data/lib/robot_lab/to/cli.rb
CHANGED
|
@@ -46,6 +46,17 @@ module RobotLab
|
|
|
46
46
|
"Seconds between polls while waiting on a decision (default: 30)", :decision_wait_poll]
|
|
47
47
|
].freeze
|
|
48
48
|
|
|
49
|
+
# Boolean flags with a fixed value when passed (no argument). Flags
|
|
50
|
+
# needing bespoke behavior (protect-path, require-improvement, help)
|
|
51
|
+
# stay in #add_flag_options.
|
|
52
|
+
BOOLEAN_FLAGS = [
|
|
53
|
+
["--no-decisions", "Disable the request_decision tool for this run", :decisions_enabled, false],
|
|
54
|
+
["--local-guards", "Add built-in file tools + small-model guardrails (for local models)", :local_guards, true],
|
|
55
|
+
["--no-stream", "Disable response streaming (required for local Ollama tool calls)", :stream, false],
|
|
56
|
+
["--debug", "Enable verbose JSONL logging to stderr", :debug, true],
|
|
57
|
+
["--version", "Print version and exit", :version, true]
|
|
58
|
+
].freeze
|
|
59
|
+
|
|
49
60
|
def self.run(argv = ARGV)
|
|
50
61
|
new.run(argv)
|
|
51
62
|
end
|
|
@@ -53,32 +64,33 @@ module RobotLab
|
|
|
53
64
|
def run(argv)
|
|
54
65
|
return run_decisions(argv[1..] || []) if argv.first == "decisions"
|
|
55
66
|
|
|
56
|
-
opts
|
|
57
|
-
parser = build_parser(opts)
|
|
58
|
-
args = parser.parse!(argv.dup)
|
|
67
|
+
opts, parser, args = parse_options(argv)
|
|
59
68
|
|
|
60
|
-
if opts[:version]
|
|
61
|
-
|
|
62
|
-
return
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
if opts[:resume]
|
|
66
|
-
return RobotLab::To.resume(opts[:resume], **opts.except(:version, :resume))
|
|
67
|
-
end
|
|
69
|
+
return puts("robot-to #{VERSION}") if opts[:version]
|
|
70
|
+
return RobotLab::To.resume(opts[:resume], **opts.except(:version, :resume)) if opts[:resume]
|
|
68
71
|
|
|
69
72
|
objective = args.first || read_stdin_objective
|
|
70
|
-
if objective.nil? || objective.strip.empty?
|
|
71
|
-
# $stderr.puts, not warn: warn is silenced when $VERBOSE is nil.
|
|
72
|
-
$stderr.puts "Error: objective required (pass as argument or via stdin)"
|
|
73
|
-
$stderr.puts parser
|
|
74
|
-
exit 1
|
|
75
|
-
end
|
|
73
|
+
abort_missing_objective!(parser) if objective.nil? || objective.strip.empty?
|
|
76
74
|
|
|
77
75
|
RobotLab::To.run(objective.strip, **opts.except(:version, :resume))
|
|
78
76
|
end
|
|
79
77
|
|
|
80
78
|
private
|
|
81
79
|
|
|
80
|
+
def parse_options(argv)
|
|
81
|
+
opts = {}
|
|
82
|
+
parser = build_parser(opts)
|
|
83
|
+
args = parser.parse!(argv.dup)
|
|
84
|
+
[opts, parser, args]
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def abort_missing_objective!(parser)
|
|
88
|
+
# $stderr.puts, not warn: warn is silenced when $VERBOSE is nil.
|
|
89
|
+
$stderr.puts "Error: objective required (pass as argument or via stdin)"
|
|
90
|
+
$stderr.puts parser
|
|
91
|
+
exit 1
|
|
92
|
+
end
|
|
93
|
+
|
|
82
94
|
# `robot-to decisions [run_id]` — list pending decisions and their file
|
|
83
95
|
# paths so a human knows what needs resolving before resuming.
|
|
84
96
|
def run_decisions(args)
|
|
@@ -96,17 +108,19 @@ module RobotLab
|
|
|
96
108
|
def print_decisions(run_id, manager)
|
|
97
109
|
pending = manager.pending
|
|
98
110
|
resolved = manager.resolved_open
|
|
99
|
-
|
|
100
|
-
puts ""
|
|
111
|
+
|
|
101
112
|
if pending.empty? && resolved.empty?
|
|
102
|
-
puts "
|
|
113
|
+
puts "Run #{run_id}\n\nNo open decisions."
|
|
103
114
|
return
|
|
104
115
|
end
|
|
116
|
+
|
|
117
|
+
puts "Run #{run_id}\n\n"
|
|
105
118
|
list_group("Pending (awaiting your answer)", pending)
|
|
106
119
|
list_group("Resolved (not yet consumed)", resolved)
|
|
107
|
-
puts
|
|
108
|
-
|
|
109
|
-
|
|
120
|
+
puts <<~MSG
|
|
121
|
+
Resolve a pending decision by editing its file: set `status: resolved`
|
|
122
|
+
and fill `resolution:`, then run `robot-to --resume #{run_id}`.
|
|
123
|
+
MSG
|
|
110
124
|
end
|
|
111
125
|
|
|
112
126
|
def list_group(title, decisions)
|
|
@@ -126,6 +140,7 @@ module RobotLab
|
|
|
126
140
|
.sort.map { |p| File.basename(p) }.last
|
|
127
141
|
end
|
|
128
142
|
|
|
143
|
+
# :reek:FeatureEnvy -- configuring the OptionParser instance being built.
|
|
129
144
|
def build_parser(opts)
|
|
130
145
|
OptionParser.new do |p|
|
|
131
146
|
p.banner = "Usage: robot-to [objective] [options]\n " \
|
|
@@ -136,30 +151,42 @@ module RobotLab
|
|
|
136
151
|
p.separator ""
|
|
137
152
|
p.separator "Options:"
|
|
138
153
|
|
|
139
|
-
|
|
140
|
-
p.on(flag, type, desc) { |v| opts[key] = v }
|
|
141
|
-
end
|
|
142
|
-
|
|
154
|
+
add_value_options(p, opts)
|
|
143
155
|
add_flag_options(p, opts)
|
|
144
156
|
end
|
|
145
157
|
end
|
|
146
158
|
|
|
159
|
+
# :reek:NestedIterators -- one .on registration per option, each with its
|
|
160
|
+
# own value-assignment callback; that's the OptionParser API shape.
|
|
161
|
+
def add_value_options(parser, opts)
|
|
162
|
+
VALUE_OPTIONS.each do |flag, type, desc, key|
|
|
163
|
+
parser.on(flag, type, desc) { |v| opts[key] = v }
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# :reek:FeatureEnvy -- registering flags on the OptionParser being built.
|
|
147
168
|
# Boolean and terminal flags (each has bespoke behavior).
|
|
148
169
|
def add_flag_options(parser, opts)
|
|
170
|
+
add_require_improvement_option(parser, opts)
|
|
171
|
+
add_protect_path_option(parser, opts)
|
|
172
|
+
BOOLEAN_FLAGS.each { |flag, desc, key, value| parser.on(flag, desc) { opts[key] = value } }
|
|
173
|
+
add_help_option(parser)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def add_require_improvement_option(parser, opts)
|
|
149
177
|
parser.on("--[no-]require-improvement",
|
|
150
178
|
"Roll back gate-passing iterations that don't improve (default: on)") do |v|
|
|
151
179
|
opts[:require_improvement] = v
|
|
152
180
|
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def add_protect_path_option(parser, opts)
|
|
153
184
|
parser.on("--protect-path GLOB", "Lock a grader file from robot edits (repeatable)") do |v|
|
|
154
185
|
(opts[:protect_paths] ||= []) << v
|
|
155
186
|
end
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
end
|
|
160
|
-
parser.on("--no-stream", "Disable response streaming (required for local Ollama tool calls)") { opts[:stream] = false }
|
|
161
|
-
parser.on("--debug", "Enable verbose JSONL logging to stderr") { opts[:debug] = true }
|
|
162
|
-
parser.on("--version", "Print version and exit") { opts[:version] = true }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def add_help_option(parser)
|
|
163
190
|
parser.on("-h", "--help", "Show this help") do
|
|
164
191
|
puts parser
|
|
165
192
|
exit
|
|
@@ -8,6 +8,9 @@ module RobotLab
|
|
|
8
8
|
#
|
|
9
9
|
# All subprocess calls use explicit argv arrays (no shell interpolation).
|
|
10
10
|
# GIT_TERMINAL_PROMPT=0 prevents credential prompts from hanging the loop.
|
|
11
|
+
# :reek:RepeatedConditional -- each `status.success?` belongs to a distinct
|
|
12
|
+
# Open3.capture3 call with its own failure handling; there is no shared
|
|
13
|
+
# condition to extract.
|
|
11
14
|
class CommitManager
|
|
12
15
|
GIT_ENV = { "GIT_TERMINAL_PROMPT" => "0" }.freeze
|
|
13
16
|
|
|
@@ -72,6 +75,7 @@ module RobotLab
|
|
|
72
75
|
parse_diff_stat(out)
|
|
73
76
|
end
|
|
74
77
|
|
|
78
|
+
# :reek:FeatureEnvy -- inherent to building/reading/appending a Pathname.
|
|
75
79
|
def add_to_local_exclude(entry)
|
|
76
80
|
exclude = Pathname.new(@work_dir).join(".git", "info", "exclude")
|
|
77
81
|
exclude.parent.mkpath
|
data/lib/robot_lab/to/config.rb
CHANGED
|
@@ -11,6 +11,15 @@ module RobotLab
|
|
|
11
11
|
# 2. User config file (~/.config/robot_lab/to.yml)
|
|
12
12
|
# 3. Environment variables (ROBOT_LAB_TO_*)
|
|
13
13
|
# 4. Constructor keyword arguments (CLI overrides)
|
|
14
|
+
# :reek:TooManyInstanceVariables -- a flat CLI-override bag for ~24
|
|
15
|
+
# independent settings; each ivar is genuinely one distinct setting.
|
|
16
|
+
# :reek:InstanceVariableAssumption -- the defaults.yml-backed ivars
|
|
17
|
+
# (provider, model, max_consecutive_failures, max_submit_nudges,
|
|
18
|
+
# max_verify_repairs, verify_timeout, run_dir, commit_format, local_guards,
|
|
19
|
+
# stream, debug, decisions_enabled, decision_mode, decision_wait_poll,
|
|
20
|
+
# decision_timeout) ARE assigned -- by `super()` (MywayConfig::Base /
|
|
21
|
+
# Anyway::Config), before any of this class's own code runs. Do NOT
|
|
22
|
+
# pre-nil them here: that overwrites the real loaded values with nil.
|
|
14
23
|
class Config < MywayConfig::Base
|
|
15
24
|
config_name :robot_lab_to
|
|
16
25
|
env_prefix :robot_lab_to
|
|
@@ -34,6 +43,9 @@ module RobotLab
|
|
|
34
43
|
|
|
35
44
|
def initialize(**overrides)
|
|
36
45
|
super()
|
|
46
|
+
# CLI-only, no YAML default (nil means "no limit / not set") -- unlike
|
|
47
|
+
# the defaults.yml-backed ivars above, super() never touches these.
|
|
48
|
+
@max_iterations = @max_tokens = @stop_when = @verify_command = nil
|
|
37
49
|
@eval = @eval_measure = @eval_target = nil
|
|
38
50
|
@require_improvement = @stop_on_plateau = nil
|
|
39
51
|
@eval_judge_model = @eval_spec = @eval_floor = nil
|
|
@@ -40,6 +40,11 @@ module RobotLab
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
# Persist a decision the robot raised. Returns the parsed Decision.
|
|
43
|
+
# :reek:BooleanParameter -- blocking is stored data (front matter), not a
|
|
44
|
+
# behavior switch.
|
|
45
|
+
# :reek:ControlParameter -- the ternary only coerces blocking to true/false.
|
|
46
|
+
# :reek:LongParameterList { max_params: 6 } -- one field per front-matter
|
|
47
|
+
# attribute; a hash would just move the same six names elsewhere.
|
|
43
48
|
def record(question:, situation: "", options: [], recommendation: "", blocking: false, iteration: 0)
|
|
44
49
|
id = generate_id
|
|
45
50
|
path = @dir.join("#{id}.md")
|
|
@@ -57,6 +62,7 @@ module RobotLab
|
|
|
57
62
|
def pending = all.select(&:pending?)
|
|
58
63
|
# resolved but not yet closed
|
|
59
64
|
def resolved_open = all.select(&:resolved?)
|
|
65
|
+
# :reek:FeatureEnvy -- filtering by a Decision's own predicates.
|
|
60
66
|
def blocking_pending = all.select { |d| d.pending? && d.blocking? }
|
|
61
67
|
def blocking_pending? = blocking_pending.any?
|
|
62
68
|
|
|
@@ -66,6 +72,7 @@ module RobotLab
|
|
|
66
72
|
# Mark a resolved decision closed: its resolution has been delivered to a
|
|
67
73
|
# robot and committed, so it should no longer be re-injected. Flips only
|
|
68
74
|
# the status line, preserving whatever the human wrote in the body.
|
|
75
|
+
# :reek:FeatureEnvy -- reading/rewriting the decision's own file.
|
|
69
76
|
def close(decision)
|
|
70
77
|
raw = File.read(decision.path)
|
|
71
78
|
AtomicFile.write(decision.path, flip_status(raw, "closed"))
|
|
@@ -82,8 +89,10 @@ module RobotLab
|
|
|
82
89
|
raw.sub(/^status:.*$/, "status: #{status}")
|
|
83
90
|
end
|
|
84
91
|
|
|
92
|
+
# :reek:LongParameterList { max_params: 7 } -- one field per front-matter
|
|
93
|
+
# attribute, mirroring #record.
|
|
85
94
|
def render_new(id:, question:, situation:, options:, recommendation:, blocking:, iteration:)
|
|
86
|
-
options_md = options.empty? ? "(none provided)\n" : options.each_with_index.map { |
|
|
95
|
+
options_md = options.empty? ? "(none provided)\n" : options.each_with_index.map { |opt, i| "#{i + 1}. #{opt}" }.join("\n") + "\n"
|
|
87
96
|
# Build the front matter with YAML.dump so question/recommendation are
|
|
88
97
|
# escaped correctly (they can contain colons, quotes, etc.).
|
|
89
98
|
front = {
|
|
@@ -116,6 +125,7 @@ module RobotLab
|
|
|
116
125
|
|
|
117
126
|
# Split a file into (front_matter_hash, body). Returns a Decision or nil
|
|
118
127
|
# when the file is unreadable.
|
|
128
|
+
# :reek:FeatureEnvy -- building a Decision from its own front matter hash.
|
|
119
129
|
def parse(path)
|
|
120
130
|
text = File.read(path)
|
|
121
131
|
fm, body = split_front_matter(text)
|
|
@@ -138,6 +148,7 @@ module RobotLab
|
|
|
138
148
|
nil
|
|
139
149
|
end
|
|
140
150
|
|
|
151
|
+
# :reek:FeatureEnvy -- slicing the raw file text into front matter/body.
|
|
141
152
|
def split_front_matter(text)
|
|
142
153
|
if text.start_with?("---\n") && (close_idx = text.index("\n---", 4))
|
|
143
154
|
raw_fm = text[4...close_idx]
|