robot_lab-to 0.2.6

data/lib/robot_lab/to/evals/factory.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module To
+    module Evals
+      # Wraps a bare callable (proc/lambda) as an Eval.
+      class ProcEval < Base
+        def initialize(callable)
+          super()
+          @callable = callable
+        end
+
+        def score(context)
+          @callable.call(context)
+        end
+      end
+
+      # Resolves the configured eval strategy into an object responding to
+      # #score(context). Precedence: an explicit instance (responds to #score) or
+      # proc, a registered symbol, then the named built-ins, then a default that
+      # matches legacy behavior (Code when a verify command is set, else Null).
+      #
+      # Phase 1 ships "code" and "null"; "prose", file paths, and "auto" arrive in
+      # later phases (they raise a clear error until then).
+      def self.build(config, git: CommitManager.new)
+        chosen = config.respond_to?(:eval) ? config.eval : nil
+        return chosen if chosen.respond_to?(:score)
+        return ProcEval.new(chosen) if chosen.is_a?(Proc)
+
+        from_name(chosen, config, git)
+      end
+
+      def self.from_name(chosen, config, git)
+        case chosen
+        when Symbol then from_registry(chosen, config)
+        when "code" then code(config)
+        when "null" then Null.new
+        when "prose" then prose(config, git)
+        when nil    then use_code?(config) ? code(config) : Null.new
+        else raise ArgumentError, unknown_strategy(chosen)
+        end
+      end
+
+      def self.prose(config, git)
+        Prose.new(objective_model: config.model, git: git,
+                  spec: config.eval_spec, floor: config.eval_floor,
+                  judge_model: config.eval_judge_model || config.model)
+      end
+
+      # Default to Code when any of its inputs are configured (a verify command,
+      # a measure command, or a target); otherwise Null preserves legacy behavior.
+      def self.use_code?(config)
+        !blank?(config.verify_command) || !config.eval_measure.nil? || !config.eval_target.nil?
+      end
+
+      def self.code(config)
+        Code.new(verify:  presence(config.verify_command),
+                 measure: presence(config.eval_measure),
+                 target:  config.eval_target,
+                 timeout: config.verify_timeout)
+      end
+
+      def self.from_registry(name, config)
+        block = RobotLab::To.evals[name]
+        raise ArgumentError, "no eval registered as #{name.inspect}" unless block
+
+        block.call(config)
+      end
+
+      def self.presence(value)
+        text = value.to_s.strip
+        text.empty? ? nil : text
+      end
+
+      def self.blank?(value) = presence(value).nil?
+
+      def self.unknown_strategy(chosen)
+        "unknown eval strategy: #{chosen.inspect} (available: code, null, " \
+          "a symbol registered via RobotLab::To.register_eval, " \
+          "an object responding to #score, or a proc)"
+      end
+    end
+  end
+end

data/lib/robot_lab/to/evals/null.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module To
+    module Evals
+      # The default Eval when nothing is configured: no gate, every reported
+      # success counts as progress, no measurable target. Preserves robot_lab-to's
+      # original "commit any claimed success; stop via --stop-when" behavior.
+      class Null < Base
+        def score(_context)
+          Score.new(gate_ok: true, improved: true, met_target: false,
+                    value: nil, detail: "no eval configured", output: nil)
+        end
+      end
+    end
+  end
+end

data/lib/robot_lab/to/evals/prose.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module To
+    module Evals
+      # Judged / pairwise eval for prose products (docs, opinion, a book).
+      #
+      #   spec        - optional spec/outline artifact the judge measures against
+      #   judge_model - model for the pairwise judge (defaults to the doer's model)
+      #   floor       - optional command for mechanizable checks (links, coverage)
+      #
+      # The descent signal is a pairwise verdict: the judge is asked whether the
+      # working tree is BETTER than the parent commit -- absolute scores from an
+      # LLM are too noisy to descend. There is no measurable target, so met_target
+      # is always false; runs end on --stop-on-plateau or a human.
+      class Prose < Base
+        JUDGE_SYSTEM = <<~PROMPT
+          You are a meticulous editor. You will see two versions of a document
+          (A = the previously accepted version, B = a new draft) plus the objective
+          and, if given, the spec they serve. Decide whether B is BETTER, WORSE, or
+          the SAME as A at satisfying them. Judge substance -- accuracy,
+          completeness, clarity, structure -- not length. Reply with exactly one
+          word: better, worse, or same.
+        PROMPT
+
+        def initialize(objective_model:, work_dir: Dir.pwd,
+                       git: CommitManager.new(work_dir: work_dir),
+                       judge_model: objective_model, spec: nil, floor: nil)
+          super()
+          @spec        = spec
+          @judge_model = judge_model
+          @floor       = floor
+          @work_dir    = work_dir
+          @git         = git
+        end
+
+        # The spec is the eval's own criteria and must be locked from the robot.
+        def protected_paths = Array(@spec)
+
+        def score(context)
+          gate, output = floor_result
+          verdict      = gate ? pairwise(context) : :worse
+          Score.new(gate_ok: gate, improved: verdict == :better, met_target: false,
+                    value: nil, detail: "floor=#{gate} judge=#{verdict}", output: output)
+        end
+
+        # Map a judge reply onto a verdict; defaults to :same when unclear so an
+        # ambiguous judgment does not count as improvement.
+        def parse_verdict(reply)
+          text = reply.to_s.downcase
+          return :better if text.match?(/\bbetter\b/)
+          return :worse  if text.match?(/\bworse\b/)
+
+          :same
+        end
+
+        private
+
+        def floor_result
+          return [true, nil] unless @floor
+
+          result = Verifier.new(@floor, work_dir: @work_dir).run
+          [result.passed?, result.output]
+        end
+
+        def pairwise(context)
+          files = @git.changed_vs_worktree(context.previous_ref)
+          return :same if files.empty?
+
+          old = files.map { |f| @git.show(context.previous_ref, f) }.join("\n\n")
+          new = files.map { |f| read_working(f) }.join("\n\n")
+          # A brand-new document (no prior version) is an improvement over nothing.
+          # Skip the judge: a pairwise "empty vs content" comparison is degenerate
+          # and models frequently (wrongly) call it "same".
+          return :better if old.strip.empty? && !new.strip.empty?
+
+          parse_verdict(ask_judge(context.objective, old, new))
+        end
+
+        def ask_judge(objective, old, new)
+          judge = RobotLab.build(name: "prose-judge", model: @judge_model, system_prompt: JUDGE_SYSTEM)
+          # robot.run returns a RobotResult; its text is #last_text_content, NOT
+          # #to_s (which is the object's inspection string).
+          judge.run(comparison(objective, old, new)).last_text_content.to_s
+        end
+
+        def comparison(objective, old_text, new_text)
+          spec_block = @spec && File.exist?(@spec) ? "SPEC:\n#{File.read(@spec)}\n\n" : ""
+          <<~MSG
+            OBJECTIVE: #{objective}
+
+            #{spec_block}VERSION A (accepted):
+            #{old_text}
+
+            VERSION B (new draft):
+            #{new_text}
+
+            Is B better, worse, or the same as A? Reply with one word.
+          MSG
+        end
+
+        def read_working(file)
+          File.read(File.join(@work_dir, file))
+        rescue SystemCallError
+          ""
+        end
+      end
+    end
+  end
+end

data/lib/robot_lab/to/evals/score.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module To
+    module Evals
+      # The verdict an Eval returns for one iteration.
+      #
+      #   gate_ok    - floor holds? (correctness) -- must be true to commit
+      #   improved   - better than the parent commit? (the descent signal)
+      #   met_target - objective reached? (orchestrator-owned stop)
+      #   value      - the score (higher = better); nil for pure-pairwise evals
+      #   detail     - one-line human summary for logs / progress
+      #   output     - diagnostic text (gate command output, judge rationale) used
+      #                for repair prompts and notes; nil when there is none
+      Score = Data.define(:gate_ok, :improved, :met_target, :value, :detail, :output) do
+        def gate_ok?    = gate_ok
+        def improved?   = improved
+        def met_target? = met_target
+      end
+    end
+  end
+end

data/lib/robot_lab/to/exit_summary.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module To
+    # Prints the post-run summary to stdout.
+    class ExitSummary
+      def initialize(run, config, abort_reason: nil)
+        @run          = run
+        @config       = config
+        @abort_reason = abort_reason
+      end
+
+      def print
+        stat       = @run.base_commit ? diff_stat : { insertions: 0, deletions: 0, files: 0 }
+        aborted    = !@abort_reason.nil?
+        good_iters = @run.commits
+        fail_iters = @run.iteration - good_iters
+
+        puts ""
+        print_header(aborted)
+        print_counters(good_iters, fail_iters, stat)
+        print_paths
+        print_next_steps
+        puts ""
+      end
+
+      private
+
+      def print_header(aborted)
+        if aborted
+          puts "robot-to stopped — #{@config.model} — #{@run.elapsed_human}"
+          puts "Reason: #{@abort_reason}"
+        else
+          puts "robot-to complete — #{@config.model} — #{@run.elapsed_human}"
+          puts "Branch: #{@run.branch}"
+        end
+      end
+
+      def print_counters(good_iters, fail_iters, stat)
+        puts ""
+        puts "Iterations:      #{@run.iteration} total  (#{good_iters} good / #{fail_iters} failed)"
+        puts "Tokens:          #{comma(@run.total_tokens)}  (#{comma(@run.input_tokens)} in / #{comma(@run.output_tokens)} out)"
+        puts format("%-16s %s", "Commits:", @run.commits.to_s)
+        puts "Changes:         +#{stat[:insertions]} / -#{stat[:deletions]} lines across #{stat[:files]} files"
+      end
+
+      def print_paths
+        puts ""
+        puts format("%-8s %s", "Notes:", @run.notes_path)
+        puts format("%-8s %s", "Log:", @run.log_path)
+      end
+
+      def print_next_steps
+        return unless @run.commits.positive?
+
+        puts ""
+        puts "Next steps:"
+        puts "  git log --oneline #{@run.base_commit}..HEAD"
+        puts "  git diff --stat #{@run.base_commit}..HEAD"
+        puts "  gh pr create"
+      end
+
+      def diff_stat
+        git = CommitManager.new
+        git.diff_stat(@run.base_commit)
+      rescue StandardError
+        { insertions: 0, deletions: 0, files: 0 }
+      end
+
+      def comma(n)
+        n.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse
+      end
+    end
+  end
+end

data/lib/robot_lab/to/guards/checkpoint.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+require_relative "path_resolution"
+require_relative "run_store"
+
+module RobotLab
+  module To
+    module Guards
+      # First-write-wins file snapshot before Write/Edit, ported from
+      # little-coder checkpoint. Finer-grained than robot_lab-to's git rollback:
+      # it survives WITHIN an iteration, so the model (or a future repair step)
+      # can revert a single botched edit without discarding the iteration's good
+      # work. Best-effort — never raises into the tool path.
+      #
+      # Backups land under <run_dir>/checkpoints/, falling back to a temp dir
+      # when no Run is supplied. Files that did not exist get a `.absent`
+      # sentinel. The snapshotted-set spans calls, so it lives in RunStore.
+      class Checkpoint < RobotLab::Hook
+        self.namespace = :checkpoint
+
+        KEY = :robot_lab_to_checkpoint
+        MUTATING_TOOLS = %w[write edit].freeze
+
+        class << self
+          def before_run(_ctx)
+            RunStore.reset(KEY, [])
+          end
+
+          def before_tool_call(ctx)
+            return unless MUTATING_TOOLS.include?(ctx.tool_name.to_s.downcase)
+
+            path = PathResolution.resolve(ctx.tool_args)
+            return unless path
+
+            snapshot(path, checkpoint_dir(ctx))
+          end
+
+          # Copy `path`'s current bytes into `dir`, once per run (first-write
+          # wins). Swallows all I/O errors — checkpointing must never break a
+          # tool call.
+          #
+          # @return [String, nil] the backup path written, or nil if skipped
+          def snapshot(path, dir)
+            tracked = RunStore.fetch(KEY, [])
+            return if tracked.include?(path)
+
+            tracked << path
+            FileUtils.mkdir_p(dir)
+            dest = File.join(dir, safe_name(path))
+            if File.exist?(path)
+              FileUtils.cp(path, dest)
+              dest
+            else
+              File.write("#{dest}.absent", "")
+              "#{dest}.absent"
+            end
+          rescue SystemCallError, IOError
+            nil
+          end
+
+          # @return [String] directory for this run's checkpoints (created)
+          def checkpoint_dir(ctx)
+            run  = ctx.local.run
+            base = run.respond_to?(:run_dir) ? run.run_dir : Dir.tmpdir
+            dir  = File.join(base.to_s, "checkpoints")
+            FileUtils.mkdir_p(dir)
+            dir
+          end
+
+          # Flatten an absolute path into a single safe filename.
+          #
+          # @param path [String]
+          # @return [String]
+          def safe_name(path)
+            flat = path.gsub(/[^A-Za-z0-9._-]/, "_")
+            flat.length > 200 ? flat[-200..] : flat
+          end
+        end
+      end
+    end
+  end
+end

data/lib/robot_lab/to/guards/grader_lock.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative "path_resolution"
+
+module RobotLab
+  module To
+    module Guards
+      # Refuses any attempt by the robot to modify its own grader artifacts (the
+      # eval's spec / floor script / --protect-path files), so it cannot "win" by
+      # rewriting the criteria it is scored against.
+      #
+      # Unlike the other guards, this one is installed unconditionally whenever
+      # protected paths exist -- not gated behind --local-guards -- because a
+      # frontier model can reward-hack too. The protected absolute paths are
+      # passed in via context: { paths: [...] }.
+      class GraderLock < RobotLab::Hook
+        self.namespace = :grader_lock
+
+        MUTATING_TOOLS = %w[write edit].freeze
+        BASH_TOOLS     = %w[bash].freeze
+
+        class << self
+          def around_tool_call(ctx)
+            paths = Array(ctx.local.paths)
+            return yield if paths.empty?
+
+            target = offending_path(ctx, paths)
+            return yield unless target
+
+            ctx.tool_result = refusal(target)
+          end
+
+          # The locked path a tool call targets, or nil if it touches none.
+          # Write/Edit are matched by resolved path; Bash by the command string
+          # referencing an absolute path or a locked basename (conservative).
+          def offending_path(ctx, paths)
+            name = ctx.tool_name.to_s.downcase
+            if MUTATING_TOOLS.include?(name)
+              resolved = PathResolution.resolve(ctx.tool_args)
+              resolved if resolved && paths.include?(resolved)
+            elsif BASH_TOOLS.include?(name)
+              command_hits(ctx.tool_args, paths)
+            end
+          end
+
+          def command_hits(args, paths)
+            command = (args["command"] || args[:command]).to_s
+            paths.find { |p| command.include?(p) || command.include?(File.basename(p)) }
+          end
+
+          def refusal(path)
+            <<~MSG
+              Refused -- #{path} is a locked grader artifact (the criteria you are
+              scored against). You must not create, edit, or delete it. Improve the
+              work being scored instead.
+            MSG
+          end
+        end
+      end
+    end
+  end
+end

data/lib/robot_lab/to/guards/path_resolution.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module To
+    module Guards
+      # Shared, pure path helpers used by the file guards. No hook state here so
+      # each function is trivially testable in isolation.
+      module PathResolution
+        module_function
+
+        # Keys a tool might use to carry the destination path.
+        PATH_KEYS = %w[path file_path].freeze
+
+        # Extract the raw path string from tool args, accepting either `path`
+        # (RubyLLM/MCP style) or `file_path`.
+        #
+        # @param args [Hash]
+        # @return [String, nil]
+        def raw_path(args)
+          key = PATH_KEYS.find { |k| args[k].is_a?(String) || args[k.to_sym].is_a?(String) }
+          return nil unless key
+
+          (args[key] || args[key.to_sym]).to_s
+        end
+
+        # Normalize a model-supplied path against cwd.
+        #
+        # Two deterministic rewrites (ported from little-coder write-guard):
+        #   1. "/<single-segment>" (e.g. "/foo.md") -> "<cwd>/foo.md" — small
+        #      models anchor at filesystem root when handed an "absolute path"
+        #      schema; a real system path always has an intermediate dir.
+        #   2. bare/relative path -> resolved against cwd.
+        # Any absolute path with an intermediate directory is left untouched.
+        #
+        # @param file_path [String]
+        # @param cwd [String]
+        # @return [String] the resolved absolute path
+        def normalize(file_path, cwd: Dir.pwd)
+          if file_path.match?(%r{\A/[^/]+\z})
+            File.join(cwd, file_path.delete_prefix("/"))
+          elsif !file_path.start_with?("/")
+            File.join(cwd, file_path)
+          else
+            file_path
+          end
+        end
+
+        # Resolve a tool's path argument to a concrete absolute path, or nil if
+        # the args carry no resolvable path.
+        #
+        # @param args [Hash]
+        # @param cwd [String]
+        # @return [String, nil]
+        def resolve(args, cwd: Dir.pwd)
+          raw = raw_path(args)
+          raw && normalize(raw, cwd: cwd)
+        end
+      end
+    end
+  end
+end

data/lib/robot_lab/to/guards/quality_monitor.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "run_store"
+
+module RobotLab
+  module To
+    module Guards
+      # Response-quality monitor, ported from little-coder quality-monitor.
+      # Detects the degenerate output modes that waste a small local model's
+      # token budget on an unattended overnight run:
+      #
+      #   repeated_tool_call  — exact name+args match with the previous call
+      #   empty_response      — final turn produced no text and no tool calls
+      #
+      # Loop detection (repeated_tool_call) is the highest-value guard: it is
+      # what stops a 9-35B model spinning on the same call until max_tokens. It
+      # runs in after_tool_call (RobotLab fires the llm_generation hook only once
+      # per run, so per-round comparison must key off tool calls), with the
+      # previous call held in the per-run RunStore.
+      #
+      # RobotLab gives no in-flight "steer" primitive, so after MAX_CONSECUTIVE
+      # repeats we raise QualityError. Orchestrator#execute_iteration already
+      # rescues iteration errors — it rolls back and records the reason in
+      # notes.md, so the next iteration's prompt carries the correction forward.
+      class QualityMonitor < RobotLab::Hook
+        self.namespace = :quality_monitor
+
+        KEY = :robot_lab_to_quality
+        MAX_CONSECUTIVE = 2
+
+        # Raised when the model exceeds the consecutive-repeat budget.
+        class QualityError < StandardError
+          attr_reader :reason
+
+          def initialize(reason)
+            @reason = reason
+            super("response quality failure: #{reason}")
+          end
+        end
+
+        class << self
+          def before_run(_ctx)
+            RunStore.reset(KEY, { previous: nil, repeats: 0 })
+          end
+
+          # Per-call loop detection. This is the load-bearing guard: it stops a
+          # small model spinning on the same tool call until the token budget is
+          # gone. Empty-response / no-progress handling is intentionally left to
+          # the orchestrator's submit-nudge → not_submitted path, which recovers
+          # gracefully instead of hard-erroring the iteration.
+          def after_tool_call(ctx)
+            call = { name: ctx.tool_name.to_s, input: ctx.tool_args }
+            state = RunStore.fetch(KEY, { previous: nil, repeats: 0 })
+
+            if repeated?([call], [state[:previous]].compact)
+              state[:repeats] += 1
+              raise QualityError, :repeated_tool_call if state[:repeats] > MAX_CONSECUTIVE
+            else
+              state[:repeats] = 0
+            end
+            state[:previous] = call
+          end
+
+          # ---- pure logic (testable in isolation) -----------------------------
+
+          # @param text [String]
+          # @param calls [Array<Hash>] each {name:, input:}
+          # @param previous [Array<Hash>] previous round's calls
+          # @param known [Array<String>] registered tool names (optional)
+          # @return [Symbol, String] :ok or a failure reason
+          def assess(text, calls, previous, known)
+            return :empty_response if text.to_s.strip.empty? && calls.empty?
+
+            calls.each do |c|
+              return :empty_tool_name if c[:name].to_s.empty?
+              return "unknown_tool:#{c[:name]}" if known.any? && !known.include?(c[:name])
+            end
+
+            return :repeated_tool_call if repeated?(calls, previous)
+
+            :ok
+          end
+
+          # True when any current call exactly matches any previous call.
+          def repeated?(calls, previous)
+            return false if calls.empty? || previous.empty?
+
+            calls.any? do |c|
+              previous.any? do |p|
+                c[:name] == p[:name] && JSON.generate(c[:input]) == JSON.generate(p[:input])
+              end
+            end
+          end
+
+          def correction_message(reason)
+            case reason.to_s
+            when "repeated_tool_call"
+              "You repeated the previous tool call verbatim — you appear to be " \
+              "stuck. Try a different approach or explain what is blocking you."
+            when "empty_response"
+              "Your response was empty. Respond with text or a tool call to make progress."
+            when /\Aunknown_tool:/
+              "Tool '#{reason.to_s.split(':', 2).last}' does not exist. Use only your registered tools."
+            else
+              "Response quality issue (#{reason}). Please correct and continue."
+            end
+          end
+        end
+      end
+    end
+  end
+end