robot_lab-to 0.2.6

data/examples/04_prose/prose_run.rb

@@ -0,0 +1,302 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# ===========================================================================
+# 04_prose — two-phase authoring: outline (graded), then sections (graded)
+# ===========================================================================
+#
+# A single pairwise loop can't reliably build a structured document with a small
+# local model. This demo splits the work into two graded phases, each driven by a
+# CUSTOM eval that asks a *separate* judge model for an ABSOLUTE verdict:
+#
+#   Phase 1 — OUTLINE:  the doer writes outline.md; the judge grades it READY or
+#             NEEDS_WORK (with feedback). The run ends when the outline is READY.
+#   Phase 2 — SECTIONS: for the approved outline, the doer writes one section per
+#             iteration as its own file (sections/NN_*.md); the judge grades each
+#             section. The run ends when every outline section has a READY file.
+#   Finally the approved sections are assembled into guide.md.
+#
+# Two models, two roles:
+#   DOER     (default qwen3.6:latest) — writes the outline and the sections
+#   VERIFIER (default gpt-oss:latest) — the judge; grades each artifact absolutely
+#
+# The judge writes its feedback to REVIEW.md (git-ignored in the sandbox); the
+# doer is told to read REVIEW.md each iteration. write_guard is disabled for the
+# doer so it can freely rewrite the file it is refining.
+#
+# ---------------------------------------------------------------------------
+# Run it
+# ---------------------------------------------------------------------------
+#   ollama pull qwen3.6:latest      # doer
+#   ollama pull gpt-oss:latest      # verifier / judge
+#   bundle exec ruby examples/04_prose/prose_run.rb
+#
+# Env: RLTO_MODEL (doer), RLTO_JUDGE_MODEL (verifier), RLTO_TOPIC, RLTO_LOCAL,
+#      RLTO_PROVIDER, OLLAMA_BASE.
+# ===========================================================================
+
+require "fileutils"
+require "logger"
+require "open3"
+require "net/http"
+
+[
+  File.expand_path("../../lib", __dir__),
+  File.expand_path("../../../robot_lab/lib", __dir__)
+].each { |p| $LOAD_PATH.unshift(p) if Dir.exist?(p) }
+
+# Load robot_lab prompt templates (the judge's system prompt) from this demo's
+# prompts_dir via robot_lab's OWN config env var — no reaching into PromptManager.
+# Set before robot_lab loads its config so RobotLab.build(template:) resolves here.
+PROMPTS_DIR = File.expand_path("prompts_dir", __dir__)
+ENV["ROBOT_LAB_TEMPLATE_PATH"] = PROMPTS_DIR
+
+require "ruby_llm"
+require "robot_lab"
+require "robot_lab/to"
+RobotLab.reload_config! if RobotLab.respond_to?(:reload_config!)
+
+# --- configuration ---------------------------------------------------------
+
+LOCAL       = ENV.fetch("RLTO_LOCAL", "true") == "true"
+PROVIDER    = ENV.fetch("RLTO_PROVIDER", LOCAL ? "openai" : "anthropic").to_sym
+DOER_MODEL  = ENV.fetch("RLTO_MODEL", LOCAL ? "qwen3.6:latest" : "claude-sonnet-4-6")
+JUDGE_MODEL = ENV.fetch("RLTO_JUDGE_MODEL", LOCAL ? "gpt-oss:latest" : "claude-sonnet-4-6")
+OLLAMA      = ENV.fetch("OLLAMA_BASE", "http://localhost:11434/v1")
+TOPIC       = ENV.fetch("RLTO_TOPIC", "writing good Git commit messages")
+
+def configure_local!
+  RubyLLM.configure do |c|
+    c.openai_api_base = OLLAMA
+    c.openai_api_key  = "ollama"
+    c.request_timeout = 600
+  end
+  RubyLLM.logger.level = Logger::ERROR
+  RubyLLM.models.refresh!
+rescue StandardError => e
+  warn "warning: could not refresh Ollama models (#{e.class}: #{e.message})"
+end
+
+def preflight_local!
+  uri = URI.join(OLLAMA, "models")
+  Net::HTTP.start(uri.host, uri.port, open_timeout: 2, read_timeout: 2) { |h| h.get(uri.request_uri) }
+rescue StandardError
+  abort "Cannot reach Ollama at #{OLLAMA}. Start it and pull #{DOER_MODEL} + #{JUDGE_MODEL}."
+end
+
+# --- the judge (absolute grader) -------------------------------------------
+
+# Ask the judge model to grade `text` against `criteria`. The judge's system
+# prompt is a robot_lab template (prompts_dir/judge.md).
+# @return [Array(Boolean, String)] [ready?, feedback]
+def grade(criteria, text)
+  judge = RobotLab.build(name: "judge", model: JUDGE_MODEL, provider: PROVIDER, template: :judge)
+  message = RobotLab.render_template(:grade_message, criteria: criteria, artifact: text)
+  reply = judge.run(message).last_text_content.to_s
+  ready = reply.match?(/\bREADY\b/i) && !reply.match?(/NEEDS_WORK/i)
+  feedback = reply[/NEEDS_WORK:?\s*(.+)/im, 1].to_s.strip.slice(0, 400)
+  [ready, feedback.empty? ? reply.strip.slice(0, 400) : feedback]
+end
+
+# The judge's feedback for the doer's next iteration. Written by the eval
+# (orchestrator-side, so no WriteGuard); read by the doer. Git-ignored.
+def write_review(dir, text)
+  File.write(File.join(dir, "REVIEW.md"), "# Reviewer feedback (read me before revising)\n\n#{text}\n")
+rescue StandardError
+  nil
+end
+
+def pending_score(detail)
+  RobotLab::To::Evals::Score.new(gate_ok: true, improved: false, met_target: false,
+                                 value: nil, detail: detail, output: nil)
+end
+
+# --- Phase 1 eval: grade the outline ---------------------------------------
+
+class OutlineGrader < RobotLab::To::Evals::Base
+  def initialize(work_dir:)
+    super()
+    @work_dir = work_dir
+  end
+
+  def score(_context)
+    outline = File.read(File.join(@work_dir, "outline.md")) rescue ""
+    return pending_score("no outline yet") if outline.strip.empty?
+
+    ready, feedback = grade(RobotLab.render_template(:outline_criteria, topic: TOPIC), outline)
+    write_review(@work_dir, ready ? "The outline is approved. (Phase 2 will use it.)" : feedback)
+    RobotLab::To::Evals::Score.new(
+      gate_ok: true, improved: true, met_target: ready, value: nil,
+      detail: ready ? "outline READY" : "NEEDS_WORK: #{feedback}", output: feedback
+    )
+  end
+end
+
+# --- Phase 2 eval: grade each section, finish when all are written ----------
+
+class SectionGrader < RobotLab::To::Evals::Base
+  def initialize(work_dir:, total:)
+    super()
+    @work_dir = work_dir
+    @total    = total
+  end
+
+  def score(_context)
+    files = section_files
+    return pending_score("no sections yet") if files.empty?
+
+    latest = files.last
+    body   = File.read(latest) rescue ""
+    ready, feedback = grade(RobotLab.render_template(:section_criteria, topic: TOPIC), body)
+    done = files.size >= @total && ready
+    write_review(@work_dir, section_note(files.size, ready, feedback))
+    RobotLab::To::Evals::Score.new(
+      gate_ok: true, improved: true, met_target: done, value: nil,
+      detail: "sections #{files.size}/#{@total} latest=#{ready ? 'READY' : 'NEEDS_WORK'}",
+      output: feedback
+    )
+  end
+
+  private
+
+  def section_files
+    Dir.glob(File.join(@work_dir, "sections", "*.md")).sort
+  end
+
+  def section_note(count, ready, feedback)
+    return "Section #{count}/#{@total} needs work: #{feedback}" unless ready
+    return "Section #{count} looks good. Write the NEXT section (#{count + 1}/#{@total})." if count < @total
+
+    "All #{@total} sections are written and approved."
+  end
+end
+
+# --- live feedback (robot_lab hook) ----------------------------------------
+
+class FeedbackHook < RobotLab::Hook
+  class << self
+    def before_llm_generation(_ctx)
+      $stderr.puts "      🤔 thinking…"
+    rescue StandardError
+      nil
+    end
+
+    def before_tool_call(ctx)
+      args = ctx.tool_args || {}
+      target = tidy(args["path"] || args[:path] || args["command"] || args[:command])
+      $stderr.puts "      🔧 #{ctx.tool_name}: #{target}"
+    rescue StandardError
+      nil
+    end
+
+    def tidy(text)
+      text.to_s.lines.first.to_s.strip.gsub("#{Dir.pwd}/", "").slice(0, 90)
+    end
+  end
+end
+
+# --- sandbox ---------------------------------------------------------------
+
+def sh(*args, chdir:)
+  out, err, status = Open3.capture3(*args, chdir: chdir)
+  raise "command failed: #{args.join(' ')}\n#{err}" unless status.success?
+
+  out
+end
+
+SANDBOX_DIR = File.expand_path("project", __dir__)
+RUN_DIR     = File.expand_path(".robot_lab_to", __dir__)
+
+def clean_slate!
+  [SANDBOX_DIR, RUN_DIR].each do |path|
+    next unless File.exist?(path)
+
+    puts "Cleaning leftover: #{path}"
+    FileUtils.rm_rf(path)
+  end
+end
+
+def make_sandbox
+  FileUtils.mkdir_p(SANDBOX_DIR)
+  sh("git", "init", "-q", chdir: SANDBOX_DIR)
+  sh("git", "config", "user.email", "example@example.com", chdir: SANDBOX_DIR)
+  sh("git", "config", "user.name", "robot_lab-to example", chdir: SANDBOX_DIR)
+  File.write(File.join(SANDBOX_DIR, ".gitignore"), "REVIEW.md\n") # judge feedback, not part of the work
+  File.write(File.join(SANDBOX_DIR, "README.md"), "# Prose demo\n\nBuilt in two graded phases; see the guide.\n")
+  sh("git", "add", "-A", chdir: SANDBOX_DIR)
+  sh("git", "commit", "-qm", "initial: empty sandbox", chdir: SANDBOX_DIR)
+  SANDBOX_DIR
+end
+
+# Count the numbered items in the approved outline — the number of sections.
+def outline_section_count(dir)
+  File.read(File.join(dir, "outline.md")).scan(/^\s*\d+\.\s+\S/).size
+rescue StandardError
+  0
+end
+
+def assemble_guide(dir)
+  sections = Dir.glob(File.join(dir, "sections", "*.md")).sort
+  return if sections.empty?
+
+  body = sections.map { |f| File.read(f).strip }.join("\n\n")
+  File.write(File.join(dir, "guide.md"), "# An Opinionated Guide to #{TOPIC.capitalize}\n\n#{body}\n")
+  sh("git", "add", "-A", chdir: dir)
+  sh("git", "commit", "-qm", "assemble: guide.md from approved sections", chdir: dir)
+end
+
+# --- objectives ------------------------------------------------------------
+
+OUTLINE_OBJECTIVE = RobotLab.render_template(:outline_objective, topic: TOPIC).strip
+
+def sections_objective(total)
+  RobotLab.render_template(:sections_objective, total: total).strip
+end
+
+# --- main ------------------------------------------------------------------
+
+if LOCAL
+  preflight_local!
+  configure_local!
+end
+
+clean_slate!
+sandbox = make_sandbox
+puts "Project dir:     #{sandbox}"
+puts "Doer model:      #{DOER_MODEL}"
+puts "Verifier model:  #{JUDGE_MODEL}"
+puts "Topic:           #{TOPIC}"
+puts
+
+RobotLab.on(FeedbackHook)
+
+common = {
+  provider: PROVIDER, model: DOER_MODEL, local_guards: LOCAL, stream: !LOCAL,
+  run_dir: RUN_DIR, write_guard: false, require_improvement: false
+}
+
+Dir.chdir(sandbox) do
+  puts "=== Phase 1: outline (graded until READY) ==="
+  RobotLab::To.run(OUTLINE_OBJECTIVE, **common, max_iterations: 6,
+                   eval: OutlineGrader.new(work_dir: sandbox))
+
+  total = outline_section_count(sandbox)
+  total = 5 if total.zero? # fall back if the outline wasn't a clean numbered list
+  puts "\n=== Phase 2: #{total} sections (each graded) ==="
+  RobotLab::To.run(sections_objective(total), **common, max_iterations: total * 3 + 2,
+                   eval: SectionGrader.new(work_dir: sandbox, total: total))
+
+  assemble_guide(sandbox)
+end
+
+# --- report ----------------------------------------------------------------
+
+puts "\n=== Result ==="
+puts "Commits:"; puts sh("git", "log", "--oneline", chdir: sandbox)
+guide = File.join(sandbox, "guide.md")
+if File.exist?(guide)
+  puts "\nguide.md (#{File.read(guide).lines.size} lines):"
+  puts File.read(guide)
+else
+  puts "\n(guide.md not assembled — check outline.md and sections/ in #{sandbox})"
+end

data/lib/robot_lab/to/atomic_file.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module RobotLab
+  module To
+    # Crash-safe writes for run state (notes.md and friends).
+    #
+    # An overnight run rewrites its state every iteration; a crash or kill
+    # mid-write must not leave a truncated file (notes.md is re-injected into
+    # the next robot's prompt). The pattern: serialize writers with an flock,
+    # stage the full content in a sibling temp file, fsync it, atomically
+    # rename it over the target, then fsync the directory. A crash at any point
+    # leaves either the previous complete file or the new complete file --
+    # never a torn write.
+    module AtomicFile
+      class << self
+        # Atomically replace +path+ with +content+.
+        def write(path, content)
+          path = Pathname.new(path)
+          path.parent.mkpath
+          with_lock(path) { replace(path, content.to_s) }
+        end
+
+        # Atomically append +text+ to +path+ (read-modify-write under lock).
+        def append(path, text)
+          path = Pathname.new(path)
+          path.parent.mkpath
+          with_lock(path) do
+            existing = path.exist? ? path.read : +""
+            replace(path, existing + text.to_s)
+          end
+        end
+
+        private
+
+        def replace(path, content)
+          tmp = path.parent.join(".#{path.basename}.#{Process.pid}.tmp")
+          File.open(tmp, "w") do |f|
+            f.write(content)
+            f.flush
+            f.fsync
+          end
+          File.rename(tmp, path)
+          fsync_dir(path.parent)
+        ensure
+          # tmp is assigned on the first line, so it is always set here; on the
+          # success path the rename already moved it, so rm_f is a no-op.
+          FileUtils.rm_f(tmp)
+        end
+
+        def with_lock(path)
+          lock = path.parent.join(".#{path.basename}.lock")
+          File.open(lock, File::RDWR | File::CREAT, 0o600) do |f|
+            f.flock(File::LOCK_EX)
+            yield
+          end
+        end
+
+        def fsync_dir(dir)
+          File.open(dir, &:fsync)
+        rescue SystemCallError
+          # Some filesystems/platforms reject directory fsync; best-effort.
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/robot_lab/to/backoff.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module To
+    # Exponential backoff with interruptible sleep.
+    #
+    # Formula: 60 * 2^(consecutive_errors - 1) seconds
+    #   1 error →  60s
+    #   2 errors → 120s
+    #   3 errors → 240s
+    class Backoff
+      BASE_SECONDS = 60
+
+      def initialize
+        @interrupted = false
+      end
+
+      def sleep_for(consecutive_errors)
+        @interrupted = false
+        return if consecutive_errors <= 0
+
+        duration = (BASE_SECONDS * (2**(consecutive_errors - 1))).to_i
+        duration.times do
+          return if @interrupted
+
+          sleep(1)
+        end
+      end
+
+      # Sleep a fixed number of seconds, interruptibly (1s ticks). Used for
+      # polling a blocking decision file. Does NOT reset @interrupted: once a
+      # signal fires, subsequent waits return immediately so shutdown is prompt.
+      def sleep_seconds(seconds)
+        seconds.to_i.times do
+          return if @interrupted
+
+          sleep(1)
+        end
+      end
+
+      def interrupt!
+        @interrupted = true
+      end
+    end
+  end
+end

data/lib/robot_lab/to/cli.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module RobotLab
+  module To
+    # OptionParser-based CLI for robot-to.
+    class CLI
+      # Value options: [flag, type-or-choices, description, opts-key]. Each sets
+      # opts[key] = parsed value. Kept as data so build_parser stays simple.
+      VALUE_OPTIONS = [
+        ["--provider NAME", String, "LLM provider to use (default: openai)", :provider],
+        ["--model MODEL", String, "LLM model to use (default: gpt-5.5)", :model],
+        ["--max-iterations N", Integer, "Stop after N iterations (default: unlimited)", :max_iterations],
+        ["--max-tokens N", Integer, "Stop after N total tokens (default: unlimited)", :max_tokens],
+        ["--stop-when CONDITION", String,
+         "Natural-language condition; robot sets should_fully_stop when met", :stop_when],
+        ["--max-consecutive-failures N", Integer,
+         "Abort after N consecutive failures (default: 3)", :max_consecutive_failures],
+        ["--max-verify-repairs N", Integer,
+         "Repair-in-place attempts when --verify-command fails (default: 2)", :max_verify_repairs],
+        ["--verify-command CMD", String,
+         "Command that must pass before a successful iteration is committed", :verify_command],
+        ["--verify-timeout SECONDS", Integer, "Timeout for --verify-command (default: 600)", :verify_timeout],
+        ["--eval NAME", String, "Eval strategy: code | null (default: code when --verify-command set)", :eval],
+        ["--measure CMD", String,
+         "Command printing a numeric score; higher is better (Evals::Code)", :eval_measure],
+        ["--target FLOAT", Float, "Stop once the measured score reaches TARGET", :eval_target],
+        ["--spec PATH", String, "Spec/outline artifact the eval measures against (Evals::Prose)", :eval_spec],
+        ["--floor CMD", String,
+         "Mechanizable floor-check command for prose (links, coverage, AI-tells)", :eval_floor],
+        ["--judge-model MODEL", String,
+         "Model for the pairwise prose judge (default: --model)", :eval_judge_model],
+        ["--stop-on-plateau N", Integer,
+         "Stop after N iterations with no committed improvement", :stop_on_plateau],
+        ["--run-dir PATH", String, "Directory for run state (default: .robot_lab_to)", :run_dir],
+        ["--commit-format FORMAT", %w[default conventional],
+         "Commit message format: default or conventional", :commit_format],
+        ["--resume RUN_ID", String,
+         "Resume a paused run (loads objective + state from its run.json)", :resume],
+        ["--decision-mode MODE", %w[wait exit],
+         "On a blocking decision: wait (poll) or exit (stop for --resume). Default: wait", :decision_mode],
+        ["--decision-timeout SECONDS", Integer,
+         "Max seconds to wait on a blocking decision (default: no limit)", :decision_timeout],
+        ["--decision-poll SECONDS", Integer,
+         "Seconds between polls while waiting on a decision (default: 30)", :decision_wait_poll]
+      ].freeze
+
+      def self.run(argv = ARGV)
+        new.run(argv)
+      end
+
+      def run(argv)
+        return run_decisions(argv[1..] || []) if argv.first == "decisions"
+
+        opts   = {}
+        parser = build_parser(opts)
+        args   = parser.parse!(argv.dup)
+
+        if opts[:version]
+          puts "robot-to #{VERSION}"
+          return
+        end
+
+        if opts[:resume]
+          return RobotLab::To.resume(opts[:resume], **opts.except(:version, :resume))
+        end
+
+        objective = args.first || read_stdin_objective
+        if objective.nil? || objective.strip.empty?
+          # $stderr.puts, not warn: warn is silenced when $VERBOSE is nil.
+          $stderr.puts "Error: objective required (pass as argument or via stdin)"
+          $stderr.puts parser
+          exit 1
+        end
+
+        RobotLab::To.run(objective.strip, **opts.except(:version, :resume))
+      end
+
+      private
+
+      # `robot-to decisions [run_id]` — list pending decisions and their file
+      # paths so a human knows what needs resolving before resuming.
+      def run_decisions(args)
+        run_dir = Config.new.run_dir
+        run_id  = args.first || latest_run_id(run_dir)
+        if run_id.nil?
+          $stderr.puts "No runs found under #{run_dir}/runs"
+          exit 1
+        end
+
+        manager = DecisionManager.new(Pathname.new(run_dir).join("runs", run_id, "decisions"))
+        print_decisions(run_id, manager)
+      end
+
+      def print_decisions(run_id, manager)
+        pending  = manager.pending
+        resolved = manager.resolved_open
+        puts "Run #{run_id}"
+        puts ""
+        if pending.empty? && resolved.empty?
+          puts "No open decisions."
+          return
+        end
+        list_group("Pending (awaiting your answer)", pending)
+        list_group("Resolved (not yet consumed)", resolved)
+        puts ""
+        puts "Resolve a pending decision by editing its file: set `status: resolved`"
+        puts "and fill `resolution:`, then run `robot-to --resume #{run_id}`."
+      end
+
+      def list_group(title, decisions)
+        return if decisions.empty?
+
+        puts "#{title}:"
+        decisions.each do |d|
+          flag = d.blocking? ? " [BLOCKING]" : ""
+          puts "  - #{d.question}#{flag}"
+          puts "    #{d.path}"
+        end
+        puts ""
+      end
+
+      def latest_run_id(run_dir)
+        Dir.glob(Pathname.new(run_dir).join("runs", "*")).select { |p| File.directory?(p) }
+                                                         .sort.map { |p| File.basename(p) }.last
+      end
+
+      def build_parser(opts)
+        OptionParser.new do |p|
+          p.banner = "Usage: robot-to [objective] [options]\n       " \
+                     "robot-to --resume RUN_ID [options]\n       " \
+                     "robot-to decisions [RUN_ID]"
+          p.separator ""
+          p.separator "Runs a robot_lab Robot in an autonomous loop toward the given objective."
+          p.separator ""
+          p.separator "Options:"
+
+          VALUE_OPTIONS.each do |flag, type, desc, key|
+            p.on(flag, type, desc) { |v| opts[key] = v }
+          end
+
+          add_flag_options(p, opts)
+        end
+      end
+
+      # Boolean and terminal flags (each has bespoke behavior).
+      def add_flag_options(parser, opts)
+        parser.on("--[no-]require-improvement",
+                  "Roll back gate-passing iterations that don't improve (default: on)") do |v|
+          opts[:require_improvement] = v
+        end
+        parser.on("--protect-path GLOB", "Lock a grader file from robot edits (repeatable)") do |v|
+          (opts[:protect_paths] ||= []) << v
+        end
+        parser.on("--no-decisions", "Disable the request_decision tool for this run") { opts[:decisions_enabled] = false }
+        parser.on("--local-guards", "Add built-in file tools + small-model guardrails (for local models)") do
+          opts[:local_guards] = true
+        end
+        parser.on("--no-stream", "Disable response streaming (required for local Ollama tool calls)") { opts[:stream] = false }
+        parser.on("--debug", "Enable verbose JSONL logging to stderr") { opts[:debug] = true }
+        parser.on("--version", "Print version and exit") { opts[:version] = true }
+        parser.on("-h", "--help", "Show this help") do
+          puts parser
+          exit
+        end
+      end
+
+      def read_stdin_objective
+        return nil if $stdin.tty?
+
+        $stdin.read.strip.then { |s| s.empty? ? nil : s }
+      end
+    end
+  end
+end