robot_lab-to 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.envrc +6 -0
- data/Archspec.rb +34 -0
- data/CHANGELOG.md +4 -0
- data/CLAUDE.md +3 -2
- data/README.md +9 -7
- data/Rakefile +6 -108
- data/docs/concepts/stop-conditions.md +3 -3
- data/docs/configuration/cli.md +4 -5
- data/docs/configuration/index.md +2 -3
- data/docs/configuration/settings.md +7 -7
- data/docs/getting-started/installation.md +5 -5
- data/docs/index.md +3 -3
- data/docs/local-models/index.md +14 -17
- data/docs/local-models/lm-studio.md +92 -0
- data/docs/reference/architecture.md +4 -4
- data/examples/.envrc +8 -0
- data/examples/01_basic_usage/README.md +14 -15
- data/examples/01_basic_usage/basic_usage.rb +20 -57
- data/examples/02_advanced_usage/README.md +10 -7
- data/examples/02_advanced_usage/advanced_usage.rb +23 -31
- data/examples/03_scored/scored_run.rb +19 -51
- data/examples/04_prose/README.md +15 -14
- data/examples/04_prose/prose_run.rb +22 -38
- data/examples/common.rb +111 -0
- data/lib/robot_lab/to/cli.rb +61 -34
- data/lib/robot_lab/to/commit_manager.rb +4 -0
- data/lib/robot_lab/to/config.rb +12 -0
- data/lib/robot_lab/to/decision_manager.rb +12 -1
- data/lib/robot_lab/to/exit_summary.rb +17 -16
- data/lib/robot_lab/to/guards/checkpoint.rb +6 -3
- data/lib/robot_lab/to/guards/quality_monitor.rb +4 -2
- data/lib/robot_lab/to/guards/run_store.rb +4 -2
- data/lib/robot_lab/to/notes_manager.rb +3 -0
- data/lib/robot_lab/to/orchestrator.rb +85 -34
- data/lib/robot_lab/to/prompt_builder.rb +2 -0
- data/lib/robot_lab/to/run.rb +8 -8
- data/lib/robot_lab/to/stop_conditions.rb +10 -6
- data/lib/robot_lab/to/tools/bash.rb +7 -2
- data/lib/robot_lab/to/tools/edit.rb +9 -4
- data/lib/robot_lab/to/tools/read.rb +3 -3
- data/lib/robot_lab/to/tools/request_decision.rb +14 -10
- data/lib/robot_lab/to/tools/submit_result.rb +12 -11
- data/lib/robot_lab/to/tools/write.rb +2 -2
- data/lib/robot_lab/to/version.rb +1 -1
- data/lib/robot_lab/to.rb +22 -0
- data/mkdocs.yml +1 -1
- metadata +10 -7
- data/docs/local-models/ollama.md +0 -122
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
#
|
|
4
4
|
# Basic Usage would generally be as a CLI tool; however, its a library
|
|
5
|
-
# so that means you can build it
|
|
5
|
+
# so that means you can build it into an application program.
|
|
6
6
|
#
|
|
7
7
|
# ===========================================================================
|
|
8
8
|
# 01_basic_usage — drive robot_lab-to programmatically
|
|
@@ -32,8 +32,8 @@
|
|
|
32
32
|
# ---------------------------------------------------------------------------
|
|
33
33
|
# Run it
|
|
34
34
|
# ---------------------------------------------------------------------------
|
|
35
|
-
# # Local
|
|
36
|
-
#
|
|
35
|
+
# # Local LM Studio (default — no API key needed; common.rb starts the server
|
|
36
|
+
# # and loads the model for you if they aren't already running/loaded):
|
|
37
37
|
# bundle exec ruby examples/01_basic_usage/basic_usage.rb
|
|
38
38
|
#
|
|
39
39
|
# # A cloud model instead:
|
|
@@ -41,17 +41,15 @@
|
|
|
41
41
|
# ANTHROPIC_API_KEY=sk-... \
|
|
42
42
|
# bundle exec ruby examples/01_basic_usage/basic_usage.rb
|
|
43
43
|
#
|
|
44
|
-
# Configuration (all optional, via environment):
|
|
45
|
-
# RLTO_LOCAL true|false use a local
|
|
46
|
-
# RLTO_PROVIDER name LLM provider (default:
|
|
47
|
-
# RLTO_MODEL id model id (default:
|
|
48
|
-
#
|
|
44
|
+
# Configuration (all optional, via environment; examples/.envrc sets these for you):
|
|
45
|
+
# RLTO_LOCAL true|false use a local LM Studio model (default true)
|
|
46
|
+
# RLTO_PROVIDER name LLM provider label (default: lms for local)
|
|
47
|
+
# RLTO_MODEL id model id (default: qwen/qwen3.8-27b for local)
|
|
48
|
+
# LMS_BASE_URL url LM Studio OpenAI-compatible endpoint (default localhost:1234/v1)
|
|
49
49
|
# ===========================================================================
|
|
50
50
|
|
|
51
51
|
require "fileutils"
|
|
52
|
-
require "logger"
|
|
53
52
|
require "open3"
|
|
54
|
-
require "net/http"
|
|
55
53
|
|
|
56
54
|
# Make the example runnable straight from the repo during development, with or
|
|
57
55
|
# without `bundle exec`. (When the gem is installed normally, these paths simply
|
|
@@ -61,52 +59,22 @@ require "net/http"
|
|
|
61
59
|
File.expand_path("../../../robot_lab/lib", __dir__) # sibling robot_lab/lib
|
|
62
60
|
].each { |p| $LOAD_PATH.unshift(p) if Dir.exist?(p) }
|
|
63
61
|
|
|
64
|
-
require "ruby_llm"
|
|
65
62
|
require "robot_lab"
|
|
66
63
|
require "robot_lab/to"
|
|
64
|
+
require_relative "../common"
|
|
67
65
|
|
|
68
66
|
# --- configuration ---------------------------------------------------------
|
|
69
67
|
|
|
70
68
|
LOCAL = ENV.fetch("RLTO_LOCAL", "true") == "true"
|
|
71
|
-
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "
|
|
72
|
-
MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen3.
|
|
73
|
-
OLLAMA = ENV.fetch("OLLAMA_BASE", "http://localhost:11434/v1")
|
|
74
|
-
|
|
75
|
-
# For a local model we route RubyLLM's :openai provider at Ollama's
|
|
76
|
-
# OpenAI-compatible endpoint, refresh the registry so tool attachment works, and
|
|
77
|
-
# run non-streaming (Ollama suppresses tool calls when streaming). See the
|
|
78
|
-
# "Local Models" guide in the docs for why.
|
|
79
|
-
def configure_local!
|
|
80
|
-
RubyLLM.configure do |c|
|
|
81
|
-
c.openai_api_base = OLLAMA
|
|
82
|
-
c.openai_api_key = "ollama" # ignored by Ollama, but RubyLLM wants a value
|
|
83
|
-
c.request_timeout = 600
|
|
84
|
-
end
|
|
85
|
-
RubyLLM.logger.level = Logger::ERROR
|
|
86
|
-
RubyLLM.models.refresh!
|
|
87
|
-
rescue StandardError => e
|
|
88
|
-
warn "warning: could not refresh Ollama models (#{e.class}: #{e.message})"
|
|
89
|
-
end
|
|
69
|
+
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "lms" : "anthropic").to_sym
|
|
70
|
+
MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen/qwen3.8-27b" : "claude-sonnet-4-6")
|
|
90
71
|
|
|
91
|
-
#
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
rescue StandardError
|
|
98
|
-
abort <<~MSG
|
|
99
|
-
Cannot reach an Ollama server at #{OLLAMA}.
|
|
100
|
-
Start it and pull a tool-capable model first:
|
|
101
|
-
|
|
102
|
-
ollama serve &
|
|
103
|
-
ollama pull #{MODEL}
|
|
104
|
-
|
|
105
|
-
Or run against a cloud model:
|
|
106
|
-
RLTO_LOCAL=false RLTO_PROVIDER=anthropic RLTO_MODEL=claude-sonnet-4-6 \\
|
|
107
|
-
ANTHROPIC_API_KEY=sk-... ruby #{File.basename(__FILE__)}
|
|
108
|
-
MSG
|
|
109
|
-
end
|
|
72
|
+
# ruby_llm has no native "lms" adapter. "lms" is this example's friendly label for
|
|
73
|
+
# "a local LM Studio model"; setup (common.rb) resolves it to the :lms provider
|
|
74
|
+
# adapter pointed at LM Studio, starting the server and loading MODEL as needed.
|
|
75
|
+
# Everything passed to RobotLab uses the resolved provider; PROVIDER itself is
|
|
76
|
+
# kept only for display.
|
|
77
|
+
LLM_PROVIDER = setup(provider: PROVIDER, model: MODEL)
|
|
110
78
|
|
|
111
79
|
# --- sandbox repository ----------------------------------------------------
|
|
112
80
|
|
|
@@ -352,15 +320,10 @@ OBJECTIVE = <<~OBJ.strip
|
|
|
352
320
|
file. Call submit_result when the suite passes.
|
|
353
321
|
OBJ
|
|
354
322
|
|
|
355
|
-
if LOCAL
|
|
356
|
-
preflight_local!
|
|
357
|
-
configure_local!
|
|
358
|
-
end
|
|
359
|
-
|
|
360
323
|
clean_slate! # delete any project / .robot_lab_to left by a previous run
|
|
361
324
|
sandbox = make_sandbox
|
|
362
325
|
puts "Project dir: #{sandbox}"
|
|
363
|
-
puts "Provider/model: #{PROVIDER}/#{MODEL} (#{LOCAL ? 'local
|
|
326
|
+
puts "Provider/model: #{PROVIDER}/#{MODEL} (#{LOCAL ? 'local LM Studio' : 'cloud'})"
|
|
364
327
|
puts "Objective: implement lib/roman_numeral.rb to pass the test suite"
|
|
365
328
|
puts
|
|
366
329
|
|
|
@@ -370,10 +333,10 @@ RobotLab.on(FeedbackHook)
|
|
|
370
333
|
Dir.chdir(sandbox) do
|
|
371
334
|
RobotLab::To.run(
|
|
372
335
|
OBJECTIVE,
|
|
373
|
-
provider:
|
|
336
|
+
provider: LLM_PROVIDER,
|
|
374
337
|
model: MODEL,
|
|
375
338
|
local_guards: LOCAL, # built-in file tools + small-model guardrails
|
|
376
|
-
stream: !LOCAL, # local
|
|
339
|
+
stream: !LOCAL, # local LM Studio tool calls run non-streaming
|
|
377
340
|
max_iterations: 6, # a richer task needs room to iterate
|
|
378
341
|
run_dir: RUN_DIR, # keep logs + notes under this example directory
|
|
379
342
|
# The change only commits if the seeded test suite passes:
|
|
@@ -11,7 +11,7 @@ autonomously implements the plan behind a real **quality gate**.
|
|
|
11
11
|
│ (AskUser tool) │ │ acceptance │ spec │ passes AND the quality gate │
|
|
12
12
|
│ │ │ test suite │ │ is clean │
|
|
13
13
|
└───────────────────┘ └──────────────────┘ └───────────────────────────────┘
|
|
14
|
-
gpt-5.5 (cloud) · robot_lab Network qwen3.
|
|
14
|
+
gpt-5.5 (cloud) · robot_lab Network qwen/qwen3.8-27b (local LM Studio)
|
|
15
15
|
```
|
|
16
16
|
|
|
17
17
|
This is the natural progression from
|
|
@@ -25,11 +25,11 @@ the implementer) writes the spec, so the implementer still can't game it.
|
|
|
25
25
|
|-------|-------------------|-------|
|
|
26
26
|
| 1 — Ideate | `AskUser` tool, network task, templated robot | OpenAI `gpt-5.5` |
|
|
27
27
|
| 2 — Plan | sequential network `task … depends_on`, data hand-off, file tools | OpenAI `gpt-5.5` |
|
|
28
|
-
| 3 — Implement | `RobotLab::To.run` autonomous loop, verify gate, `stop_when`, guardrails | local
|
|
28
|
+
| 3 — Implement | `RobotLab::To.run` autonomous loop, verify gate, `stop_when`, guardrails | local LM Studio `qwen/qwen3.8-27b` |
|
|
29
29
|
|
|
30
30
|
The reasoning phases run on a capable cloud model; the implementation loop runs
|
|
31
31
|
fully local. Because both use RubyLLM's `:openai` provider but different endpoints
|
|
32
|
-
(`api.openai.com` vs
|
|
32
|
+
(`api.openai.com` vs LM Studio's `/v1`), and `openai_api_base` is global, the example
|
|
33
33
|
toggles it between the sequential phases.
|
|
34
34
|
|
|
35
35
|
## The quality gate
|
|
@@ -55,10 +55,12 @@ flog + flay) into an autonomous gate. Tune the thresholds with `FLOG_MAX` /
|
|
|
55
55
|
|
|
56
56
|
```bash
|
|
57
57
|
export OPENAI_API_KEY="sk-..." # ideation + planning (gpt-5.5)
|
|
58
|
-
ollama serve & # implementation (local)
|
|
59
|
-
ollama pull qwen3.6:latest
|
|
60
58
|
```
|
|
61
59
|
|
|
60
|
+
`common.rb` starts the LM Studio server and loads the build model itself if
|
|
61
|
+
they aren't already running/loaded, so there's nothing to start by hand for
|
|
62
|
+
the implementation phase.
|
|
63
|
+
|
|
62
64
|
## Run it
|
|
63
65
|
|
|
64
66
|
```bash
|
|
@@ -75,8 +77,9 @@ acceptance suite and the local model implement it.
|
|
|
75
77
|
|----------|---------|-------------|
|
|
76
78
|
| `RLTO_REASON_MODEL` | `gpt-5.5` | model for ideate + plan |
|
|
77
79
|
| `RLTO_REASON_PROVIDER` | `openai` | provider for ideate + plan |
|
|
78
|
-
| `
|
|
79
|
-
| `
|
|
80
|
+
| `RLTO_BUILD_PROVIDER` | `lms` (falls back to `RLTO_PROVIDER`) | provider label for implementation; `lms` resolves to `:openai` routed at `LMS_BASE_URL` |
|
|
81
|
+
| `RLTO_BUILD_MODEL` | `qwen/qwen3.8-27b` (falls back to `RLTO_MODEL`) | model for implementation |
|
|
82
|
+
| `LMS_BASE_URL` | `http://localhost:1234/v1` | LM Studio OpenAI-compatible endpoint |
|
|
80
83
|
| `FLOG_MAX` | `25` | per-method complexity ceiling |
|
|
81
84
|
| `FLAY_MAX` | `40` | duplication-mass ceiling |
|
|
82
85
|
|
|
@@ -18,28 +18,29 @@
|
|
|
18
18
|
# suite* into the project (test/). It does not implement anything. Its
|
|
19
19
|
# reply is a one-paragraph implementation objective.
|
|
20
20
|
#
|
|
21
|
-
# Phase 3 IMPLEMENT (robot_lab-to, local
|
|
21
|
+
# Phase 3 IMPLEMENT (robot_lab-to, local LM Studio qwen/qwen3.8-27b)
|
|
22
22
|
# robot_lab-to runs an autonomous loop that writes lib/ code until the
|
|
23
23
|
# Planner's acceptance suite passes AND a quality gate is clean. The
|
|
24
24
|
# verify command (quality_gate.rb) runs tests + rubocop + flog + flay, so
|
|
25
25
|
# the robot must earn each commit on correctness AND quality.
|
|
26
26
|
#
|
|
27
27
|
# Models (per your request): reasoning on OpenAI gpt-5.5, building on local
|
|
28
|
-
#
|
|
29
|
-
# endpoints (api.openai.com vs
|
|
30
|
-
# toggle it between the (sequential) phases.
|
|
28
|
+
# LM Studio qwen/qwen3.8-27b. Because the phases use different providers and different
|
|
29
|
+
# endpoints (api.openai.com vs LM Studio's /v1), and openai_api_base is global,
|
|
30
|
+
# we toggle it between the (sequential) phases.
|
|
31
31
|
#
|
|
32
32
|
# Everything stays under examples/02_advanced_usage/ (project/, .robot_lab_to/),
|
|
33
33
|
# both git-ignored and recreated on each run.
|
|
34
34
|
#
|
|
35
|
-
# Run it (from the gem root, with OPENAI_API_KEY set
|
|
35
|
+
# Run it (from the gem root, with OPENAI_API_KEY set; common.rb starts the LM
|
|
36
|
+
# Studio server and loads the build model for you if they aren't already
|
|
37
|
+
# running/loaded):
|
|
36
38
|
# bundle exec ruby examples/02_advanced_usage/advanced_usage.rb
|
|
37
39
|
# ===========================================================================
|
|
38
40
|
|
|
39
41
|
require "fileutils"
|
|
40
42
|
require "logger"
|
|
41
43
|
require "open3"
|
|
42
|
-
require "net/http"
|
|
43
44
|
|
|
44
45
|
# Make the example runnable straight from the repo, with or without bundler.
|
|
45
46
|
[
|
|
@@ -47,17 +48,19 @@ require "net/http"
|
|
|
47
48
|
File.expand_path("../../../robot_lab/lib", __dir__) # sibling robot_lab/lib
|
|
48
49
|
].each { |p| $LOAD_PATH.unshift(p) if Dir.exist?(p) }
|
|
49
50
|
|
|
50
|
-
require "ruby_llm"
|
|
51
51
|
require "robot_lab"
|
|
52
52
|
require "robot_lab/to"
|
|
53
|
+
require_relative "../common"
|
|
53
54
|
|
|
54
55
|
# --- configuration ---------------------------------------------------------
|
|
55
56
|
|
|
56
57
|
REASON_PROVIDER = ENV.fetch("RLTO_REASON_PROVIDER", "openai").to_sym
|
|
57
|
-
REASON_MODEL = ENV.fetch("RLTO_REASON_MODEL", "gpt-5.5")
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
58
|
+
REASON_MODEL = ENV.fetch("RLTO_REASON_MODEL", "gpt-5.5") # real OpenAI
|
|
59
|
+
# Falls back to the shared RLTO_PROVIDER / RLTO_MODEL (examples/.envrc sets these)
|
|
60
|
+
# so the build phase picks up the same local model as the other examples unless
|
|
61
|
+
# RLTO_BUILD_PROVIDER / RLTO_BUILD_MODEL override it specifically.
|
|
62
|
+
BUILD_PROVIDER = ENV.fetch("RLTO_BUILD_PROVIDER", ENV.fetch("RLTO_PROVIDER", "lms")).to_sym
|
|
63
|
+
BUILD_MODEL = ENV.fetch("RLTO_BUILD_MODEL", ENV.fetch("RLTO_MODEL", "qwen/qwen3.8-27b")) # local LM Studio
|
|
61
64
|
|
|
62
65
|
SANDBOX_DIR = File.expand_path("project", __dir__)
|
|
63
66
|
RUN_DIR = File.expand_path(".robot_lab_to", __dir__)
|
|
@@ -95,28 +98,17 @@ def use_real_openai!
|
|
|
95
98
|
RubyLLM.logger.level = Logger::ERROR # keep raw API traffic out of the feed
|
|
96
99
|
end
|
|
97
100
|
|
|
98
|
-
# Implementation phase
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
c.request_timeout = 600
|
|
104
|
-
end
|
|
105
|
-
RubyLLM.logger.level = Logger::ERROR
|
|
106
|
-
RubyLLM.models.refresh! # register local models so tool attachment works
|
|
107
|
-
rescue StandardError => e
|
|
108
|
-
warn "warning: could not refresh Ollama models (#{e.class}: #{e.message})"
|
|
101
|
+
# Implementation phase uses the :lms provider against the local LM Studio endpoint
|
|
102
|
+
# (or whatever BUILD_PROVIDER resolves to). common.rb's setup starts the LM Studio
|
|
103
|
+
# server and loads BUILD_MODEL as needed; the "lms" label resolves to the :lms provider.
|
|
104
|
+
def use_build_provider!
|
|
105
|
+
setup(provider: BUILD_PROVIDER, model: BUILD_MODEL)
|
|
109
106
|
end
|
|
110
107
|
|
|
111
108
|
# --- preflight -------------------------------------------------------------
|
|
112
109
|
|
|
113
110
|
def preflight!
|
|
114
111
|
abort "Set OPENAI_API_KEY (the ideation/planning phases use #{REASON_MODEL})." unless ENV["OPENAI_API_KEY"]
|
|
115
|
-
|
|
116
|
-
uri = URI.join(OLLAMA_BASE, "models")
|
|
117
|
-
Net::HTTP.start(uri.host, uri.port, open_timeout: 2, read_timeout: 2) { |h| h.get(uri.request_uri) }
|
|
118
|
-
rescue StandardError
|
|
119
|
-
abort "Cannot reach Ollama at #{OLLAMA_BASE}. Start it and `ollama pull #{BUILD_MODEL}`."
|
|
120
112
|
end
|
|
121
113
|
|
|
122
114
|
# --- sandbox ---------------------------------------------------------------
|
|
@@ -243,7 +235,7 @@ RobotLab::Narrator.enable! # live narration for every robot, across all phases
|
|
|
243
235
|
|
|
244
236
|
puts "Project dir: #{sandbox}"
|
|
245
237
|
puts "Reasoning: #{REASON_PROVIDER}/#{REASON_MODEL} (cloud) → ideate + plan"
|
|
246
|
-
puts "Building: #{BUILD_PROVIDER}/#{BUILD_MODEL} (local
|
|
238
|
+
puts "Building: #{BUILD_PROVIDER}/#{BUILD_MODEL} (local LM Studio) → implement"
|
|
247
239
|
puts
|
|
248
240
|
|
|
249
241
|
# -- Phases 1 & 2: ideate -> plan, as a robot_lab network --------------------
|
|
@@ -289,15 +281,15 @@ puts "Objective derived from the spec.\n\n"
|
|
|
289
281
|
|
|
290
282
|
# -- Phase 3: autonomous implementation with robot_lab-to --------------------
|
|
291
283
|
puts "── Phase 3: implementation (robot_lab-to, quality-gated) ──"
|
|
292
|
-
|
|
284
|
+
build_llm_provider = use_build_provider!
|
|
293
285
|
|
|
294
286
|
Dir.chdir(sandbox) do
|
|
295
287
|
RobotLab::To.run(
|
|
296
288
|
objective,
|
|
297
|
-
provider:
|
|
289
|
+
provider: build_llm_provider,
|
|
298
290
|
model: BUILD_MODEL,
|
|
299
291
|
local_guards: true, # built-in file tools + small-model guardrails
|
|
300
|
-
stream: false, #
|
|
292
|
+
stream: false, # local LM Studio tool calls run non-streaming
|
|
301
293
|
max_iterations: 8,
|
|
302
294
|
run_dir: RUN_DIR,
|
|
303
295
|
verify_command: "ruby quality_gate.rb", # tests + rubocop + flog + flay
|
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
# ---------------------------------------------------------------------------
|
|
27
27
|
# Run it
|
|
28
28
|
# ---------------------------------------------------------------------------
|
|
29
|
-
# # Local
|
|
30
|
-
#
|
|
29
|
+
# # Local LM Studio (default — no API key; common.rb starts the server and
|
|
30
|
+
# # loads the model for you if they aren't already running/loaded):
|
|
31
31
|
# bundle exec ruby examples/03_scored/scored_run.rb
|
|
32
32
|
#
|
|
33
33
|
# # A cloud model instead:
|
|
@@ -35,17 +35,15 @@
|
|
|
35
35
|
# ANTHROPIC_API_KEY=sk-... \
|
|
36
36
|
# bundle exec ruby examples/03_scored/scored_run.rb
|
|
37
37
|
#
|
|
38
|
-
# Configuration (all optional, via environment):
|
|
39
|
-
# RLTO_LOCAL true|false use a local
|
|
40
|
-
# RLTO_PROVIDER name LLM provider (default:
|
|
41
|
-
# RLTO_MODEL id model id (default: qwen3.
|
|
42
|
-
#
|
|
38
|
+
# Configuration (all optional, via environment; examples/.envrc sets these for you):
|
|
39
|
+
# RLTO_LOCAL true|false use a local LM Studio model (default true)
|
|
40
|
+
# RLTO_PROVIDER name LLM provider label (default: lms for local)
|
|
41
|
+
# RLTO_MODEL id model id (default: qwen/qwen3.8-27b for local)
|
|
42
|
+
# LMS_BASE_URL url LM Studio OpenAI-compatible endpoint (default localhost:1234/v1)
|
|
43
43
|
# ===========================================================================
|
|
44
44
|
|
|
45
45
|
require "fileutils"
|
|
46
|
-
require "logger"
|
|
47
46
|
require "open3"
|
|
48
|
-
require "net/http"
|
|
49
47
|
|
|
50
48
|
# Make the example runnable straight from the repo during development, with or
|
|
51
49
|
# without `bundle exec`. (When the gem is installed normally these paths don't
|
|
@@ -55,49 +53,24 @@ require "net/http"
|
|
|
55
53
|
File.expand_path("../../../robot_lab/lib", __dir__) # sibling robot_lab/lib
|
|
56
54
|
].each { |p| $LOAD_PATH.unshift(p) if Dir.exist?(p) }
|
|
57
55
|
|
|
58
|
-
require "ruby_llm"
|
|
59
56
|
require "robot_lab"
|
|
60
57
|
require "robot_lab/to"
|
|
58
|
+
require_relative "../common"
|
|
61
59
|
|
|
62
60
|
# --- configuration ---------------------------------------------------------
|
|
63
61
|
|
|
64
62
|
LOCAL = ENV.fetch("RLTO_LOCAL", "true") == "true"
|
|
65
|
-
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "
|
|
66
|
-
MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen3.
|
|
67
|
-
OLLAMA = ENV.fetch("OLLAMA_BASE", "http://localhost:11434/v1")
|
|
63
|
+
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "lms" : "anthropic").to_sym
|
|
64
|
+
MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen/qwen3.8-27b" : "claude-sonnet-4-6")
|
|
68
65
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
c.openai_api_base = OLLAMA
|
|
76
|
-
c.openai_api_key = "ollama"
|
|
77
|
-
c.request_timeout = 600
|
|
78
|
-
end
|
|
79
|
-
RubyLLM.logger.level = Logger::ERROR
|
|
80
|
-
RubyLLM.models.refresh!
|
|
81
|
-
rescue StandardError => e
|
|
82
|
-
warn "warning: could not refresh Ollama models (#{e.class}: #{e.message})"
|
|
83
|
-
end
|
|
66
|
+
# ruby_llm has no native "lms" adapter. "lms" is this example's friendly label for
|
|
67
|
+
# "a local LM Studio model"; setup (common.rb) resolves it to the :lms provider
|
|
68
|
+
# adapter pointed at LM Studio, starting the server and loading MODEL as needed.
|
|
69
|
+
# Everything passed to RobotLab uses the resolved provider; PROVIDER itself is
|
|
70
|
+
# kept only for display.
|
|
71
|
+
LLM_PROVIDER = setup(provider: PROVIDER, model: MODEL)
|
|
84
72
|
|
|
85
|
-
|
|
86
|
-
uri = URI.join(OLLAMA, "models")
|
|
87
|
-
Net::HTTP.start(uri.host, uri.port, open_timeout: 2, read_timeout: 2) { |h| h.get(uri.request_uri) }
|
|
88
|
-
rescue StandardError
|
|
89
|
-
abort <<~MSG
|
|
90
|
-
Cannot reach an Ollama server at #{OLLAMA}.
|
|
91
|
-
Start it and pull a tool-capable model first:
|
|
92
|
-
|
|
93
|
-
ollama serve &
|
|
94
|
-
ollama pull #{MODEL}
|
|
95
|
-
|
|
96
|
-
Or run against a cloud model:
|
|
97
|
-
RLTO_LOCAL=false RLTO_PROVIDER=anthropic RLTO_MODEL=claude-sonnet-4-6 \\
|
|
98
|
-
ANTHROPIC_API_KEY=sk-... ruby #{File.basename(__FILE__)}
|
|
99
|
-
MSG
|
|
100
|
-
end
|
|
73
|
+
TOTAL_TESTS = 9 # the seeded suite has 9 test methods — the target
|
|
101
74
|
|
|
102
75
|
# --- sandbox repository ----------------------------------------------------
|
|
103
76
|
|
|
@@ -290,15 +263,10 @@ OBJECTIVE = <<~OBJ.strip
|
|
|
290
263
|
Call submit_result each iteration describing what you improved.
|
|
291
264
|
OBJ
|
|
292
265
|
|
|
293
|
-
if LOCAL
|
|
294
|
-
preflight_local!
|
|
295
|
-
configure_local!
|
|
296
|
-
end
|
|
297
|
-
|
|
298
266
|
clean_slate!
|
|
299
267
|
sandbox = make_sandbox
|
|
300
268
|
puts "Project dir: #{sandbox}"
|
|
301
|
-
puts "Provider/model: #{PROVIDER}/#{MODEL} (#{LOCAL ? 'local
|
|
269
|
+
puts "Provider/model: #{PROVIDER}/#{MODEL} (#{LOCAL ? 'local LM Studio' : 'cloud'})"
|
|
302
270
|
puts "Eval: measured descent — score = passing tests, target #{TOTAL_TESTS}"
|
|
303
271
|
puts "Locked grader: score.rb, test/roman_numeral_test.rb"
|
|
304
272
|
puts
|
|
@@ -308,7 +276,7 @@ RobotLab.on(FeedbackHook)
|
|
|
308
276
|
Dir.chdir(sandbox) do
|
|
309
277
|
RobotLab::To.run(
|
|
310
278
|
OBJECTIVE,
|
|
311
|
-
provider:
|
|
279
|
+
provider: LLM_PROVIDER,
|
|
312
280
|
model: MODEL,
|
|
313
281
|
local_guards: LOCAL,
|
|
314
282
|
stream: !LOCAL,
|
data/examples/04_prose/README.md
CHANGED
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
Demo 03 scores **code** with a deterministic command (passing tests). Prose has no
|
|
4
4
|
`rake coverage`, so this demo uses the **`prose` eval** — a pairwise LLM judge — and
|
|
5
|
-
puts two
|
|
5
|
+
puts two roles on the same local model by default, each independently swappable:
|
|
6
6
|
|
|
7
7
|
| Role | Model (default) | Job |
|
|
8
8
|
|------|-----------------|-----|
|
|
9
|
-
| **Doer** | `qwen3.
|
|
10
|
-
| **Verifier** (judge) | `
|
|
9
|
+
| **Doer** | `qwen/qwen3.8-27b` | writes and improves `guide.md` |
|
|
10
|
+
| **Verifier** (judge) | `qwen/qwen3.8-27b` | compares each draft to the last committed one and rules it **better / worse / same** |
|
|
11
11
|
|
|
12
12
|
Only a draft the judge rules **better** is committed, so every commit is a genuine
|
|
13
13
|
improvement. There's no absolute target (LLM scores are too noisy to descend), so
|
|
@@ -18,25 +18,26 @@ cannot edit the criteria it's judged against.
|
|
|
18
18
|
## Run it
|
|
19
19
|
|
|
20
20
|
```bash
|
|
21
|
-
ollama pull qwen3.6:latest # doer
|
|
22
|
-
ollama pull gpt-oss:latest # verifier / judge
|
|
23
21
|
bundle exec ruby examples/04_prose/prose_run.rb
|
|
24
22
|
```
|
|
25
23
|
|
|
26
|
-
|
|
24
|
+
`common.rb` starts the LM Studio server and loads the doer + judge models for
|
|
25
|
+
you if they aren't already running/loaded. Override the models via env:
|
|
26
|
+
`RLTO_MODEL` (doer), `RLTO_JUDGE_MODEL` (verifier).
|
|
27
27
|
|
|
28
|
-
## Why
|
|
28
|
+
## Why the judge can be the same model as the doer
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
*
|
|
30
|
+
The **doer** is given the file-editing tools; the **judge** is given **no tools**
|
|
31
|
+
at all — it just reads two versions and replies `better`/`worse`/`same`. As a pure
|
|
32
|
+
text responder it doesn't need to be a stronger or different model to be reliable,
|
|
33
|
+
which is why both roles default to the same local `qwen/qwen3.8-27b`. This is still the
|
|
34
|
+
separation-of-duties payoff: the model that *decides* is called independently of
|
|
35
|
+
the model that *acts*, so swap in a stronger `RLTO_JUDGE_MODEL` if you want a
|
|
36
|
+
tougher judge without touching the doer.
|
|
36
37
|
|
|
37
38
|
## The mechanism
|
|
38
39
|
|
|
39
|
-
`RobotLab::To.run(..., eval: "prose", eval_judge_model: "
|
|
40
|
+
`RobotLab::To.run(..., eval: "prose", eval_judge_model: "qwen/qwen3.8-27b")` builds an
|
|
40
41
|
`Evals::Prose`. Each iteration, after the doer edits `guide.md`, the eval diffs the
|
|
41
42
|
working tree against the parent commit, shows both versions plus the spec to the
|
|
42
43
|
judge model, and maps its verdict onto `Score#improved`. Commit on *better*, roll
|
|
@@ -16,9 +16,10 @@
|
|
|
16
16
|
# section. The run ends when every outline section has a READY file.
|
|
17
17
|
# Finally the approved sections are assembled into guide.md.
|
|
18
18
|
#
|
|
19
|
-
# Two models, two roles
|
|
20
|
-
#
|
|
21
|
-
#
|
|
19
|
+
# Two models, two roles (the same local model by default, but independently
|
|
20
|
+
# configurable — pass --judge-model / RLTO_JUDGE_MODEL for a stronger judge):
|
|
21
|
+
# DOER (default qwen/qwen3.8-27b) — writes the outline and the sections
|
|
22
|
+
# VERIFIER (default qwen/qwen3.8-27b) — the judge; grades each artifact absolutely
|
|
22
23
|
#
|
|
23
24
|
# The judge writes its feedback to REVIEW.md (git-ignored in the sandbox); the
|
|
24
25
|
# doer is told to read REVIEW.md each iteration. write_guard is disabled for the
|
|
@@ -27,18 +28,17 @@
|
|
|
27
28
|
# ---------------------------------------------------------------------------
|
|
28
29
|
# Run it
|
|
29
30
|
# ---------------------------------------------------------------------------
|
|
30
|
-
# ollama pull qwen3.6:latest # doer
|
|
31
|
-
# ollama pull gpt-oss:latest # verifier / judge
|
|
32
31
|
# bundle exec ruby examples/04_prose/prose_run.rb
|
|
33
32
|
#
|
|
33
|
+
# common.rb starts the LM Studio server and loads the doer + judge models for
|
|
34
|
+
# you if they aren't already running/loaded.
|
|
35
|
+
#
|
|
34
36
|
# Env: RLTO_MODEL (doer), RLTO_JUDGE_MODEL (verifier), RLTO_TOPIC, RLTO_LOCAL,
|
|
35
|
-
# RLTO_PROVIDER,
|
|
37
|
+
# RLTO_PROVIDER, LMS_BASE_URL. (examples/.envrc sets these for you)
|
|
36
38
|
# ===========================================================================
|
|
37
39
|
|
|
38
40
|
require "fileutils"
|
|
39
|
-
require "logger"
|
|
40
41
|
require "open3"
|
|
41
|
-
require "net/http"
|
|
42
42
|
|
|
43
43
|
[
|
|
44
44
|
File.expand_path("../../lib", __dir__),
|
|
@@ -51,38 +51,27 @@ require "net/http"
|
|
|
51
51
|
PROMPTS_DIR = File.expand_path("prompts_dir", __dir__)
|
|
52
52
|
ENV["ROBOT_LAB_TEMPLATE_PATH"] = PROMPTS_DIR
|
|
53
53
|
|
|
54
|
-
require "ruby_llm"
|
|
55
54
|
require "robot_lab"
|
|
56
55
|
require "robot_lab/to"
|
|
56
|
+
require_relative "../common"
|
|
57
57
|
RobotLab.reload_config! if RobotLab.respond_to?(:reload_config!)
|
|
58
58
|
|
|
59
59
|
# --- configuration ---------------------------------------------------------
|
|
60
60
|
|
|
61
61
|
LOCAL = ENV.fetch("RLTO_LOCAL", "true") == "true"
|
|
62
|
-
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "
|
|
63
|
-
DOER_MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen3.
|
|
64
|
-
JUDGE_MODEL = ENV.fetch("RLTO_JUDGE_MODEL", LOCAL ? "
|
|
65
|
-
OLLAMA = ENV.fetch("OLLAMA_BASE", "http://localhost:11434/v1")
|
|
62
|
+
PROVIDER = ENV.fetch("RLTO_PROVIDER", LOCAL ? "lms" : "anthropic").to_sym
|
|
63
|
+
DOER_MODEL = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen/qwen3.8-27b" : "claude-sonnet-4-6")
|
|
64
|
+
JUDGE_MODEL = ENV.fetch("RLTO_JUDGE_MODEL", LOCAL ? "qwen/qwen3.8-27b" : "claude-sonnet-4-6")
|
|
66
65
|
TOPIC = ENV.fetch("RLTO_TOPIC", "writing good Git commit messages")
|
|
67
66
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
rescue StandardError => e
|
|
77
|
-
warn "warning: could not refresh Ollama models (#{e.class}: #{e.message})"
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def preflight_local!
|
|
81
|
-
uri = URI.join(OLLAMA, "models")
|
|
82
|
-
Net::HTTP.start(uri.host, uri.port, open_timeout: 2, read_timeout: 2) { |h| h.get(uri.request_uri) }
|
|
83
|
-
rescue StandardError
|
|
84
|
-
abort "Cannot reach Ollama at #{OLLAMA}. Start it and pull #{DOER_MODEL} + #{JUDGE_MODEL}."
|
|
85
|
-
end
|
|
67
|
+
# ruby_llm has no native "lms" adapter. "lms" is this example's friendly label for
|
|
68
|
+
# "a local LM Studio model"; setup (common.rb) resolves it to the :lms provider
|
|
69
|
+
# adapter pointed at LM Studio, starting the server and loading each model as
|
|
70
|
+
# needed -- once for the doer, again for the judge (a no-op if they're the same
|
|
71
|
+
# model, or if it's already loaded). Everything passed to RobotLab uses the
|
|
72
|
+
# resolved provider; PROVIDER itself is kept only for display.
|
|
73
|
+
LLM_PROVIDER = setup(provider: PROVIDER, model: DOER_MODEL)
|
|
74
|
+
setup(provider: PROVIDER, model: JUDGE_MODEL)
|
|
86
75
|
|
|
87
76
|
# --- the judge (absolute grader) -------------------------------------------
|
|
88
77
|
|
|
@@ -90,7 +79,7 @@ end
|
|
|
90
79
|
# prompt is a robot_lab template (prompts_dir/judge.md).
|
|
91
80
|
# @return [Array(Boolean, String)] [ready?, feedback]
|
|
92
81
|
def grade(criteria, text)
|
|
93
|
-
judge = RobotLab.build(name: "judge", model: JUDGE_MODEL, provider:
|
|
82
|
+
judge = RobotLab.build(name: "judge", model: JUDGE_MODEL, provider: LLM_PROVIDER, template: :judge)
|
|
94
83
|
message = RobotLab.render_template(:grade_message, criteria: criteria, artifact: text)
|
|
95
84
|
reply = judge.run(message).last_text_content.to_s
|
|
96
85
|
ready = reply.match?(/\bREADY\b/i) && !reply.match?(/NEEDS_WORK/i)
|
|
@@ -255,11 +244,6 @@ end
|
|
|
255
244
|
|
|
256
245
|
# --- main ------------------------------------------------------------------
|
|
257
246
|
|
|
258
|
-
if LOCAL
|
|
259
|
-
preflight_local!
|
|
260
|
-
configure_local!
|
|
261
|
-
end
|
|
262
|
-
|
|
263
247
|
clean_slate!
|
|
264
248
|
sandbox = make_sandbox
|
|
265
249
|
puts "Project dir: #{sandbox}"
|
|
@@ -271,7 +255,7 @@ puts
|
|
|
271
255
|
RobotLab.on(FeedbackHook)
|
|
272
256
|
|
|
273
257
|
common = {
|
|
274
|
-
provider:
|
|
258
|
+
provider: LLM_PROVIDER, model: DOER_MODEL, local_guards: LOCAL, stream: !LOCAL,
|
|
275
259
|
run_dir: RUN_DIR, write_guard: false, require_improvement: false
|
|
276
260
|
}
|
|
277
261
|
|