active_harness 0.3.8 → 0.3.10

data/lib/generators/active_harness/install/templates/agents/files_agent.rb

@@ -0,0 +1,39 @@
+require_relative "../requests/files_request"
+require_relative "../tools/ls_tool"
+require_relative "../tools/read_file_tool"
+require_relative "../goals/answer_present_goal"
+require_relative "../concerns/agent_tracing"
+
+# A tool-calling agent: the model decides which tool to call, the agent validates
+# the call, runs the allowed tools and feeds the results back until an answer or a limit.
+#
+#   agent = FilesAgent.call(input: "What is in the workspace?")
+#   agent.output       # final answer
+#   agent.stop_reason  # :goal_reached | :max_turns | ...
+#   puts agent.trace
+class FilesAgent < ActiveHarness::Agent
+  include AgentTracing
+
+  # Which Request talks to the model each turn. It owns the model chain, retries and
+  # fallback — the agent never repeats a tool call just because a model failed.
+  # (Name more than one with `request :cheap, CheapRequest` and pick with `request_for`.)
+  request FilesRequest
+
+  # The only tools the model may call. Default deny: any other name is refused as
+  # "unknown tool" and never executed, however the model asks.
+  tools LsTool, ReadFileTool
+
+  # "The model stopped asking for tools" is not proof the job is done. A goal check
+  # judges the final answer and can send the agent back for another turn.
+  # Without `goal`, the first answer without tool calls ends the run.
+  goal AnswerPresentGoal
+
+  # Hard limits: a model that loops or wanders must not run (or spend) forever.
+  # Whichever trips first ends the run with a matching stop_reason.
+  max_turns 8        # at most 8 Request calls
+  max_cost  0.50     # USD summed over all turns; stops NEW steps, a running call may overshoot
+
+  # Stop early when the model repeats the same tool call with the same input
+  # several turns in a row (stop_reason: :repeating).
+  loop_guard
+end

data/lib/generators/active_harness/install/templates/concerns/agent_tracing.rb

@@ -0,0 +1,36 @@
+require_relative "tracing_log"
+
+# Basic tracing for any Agent: turns, tool calls (with refusals), the stop
+# reason and a final summary.
+#
+#   class FilesAgent < ActiveHarness::Agent
+#     include AgentTracing
+#   end
+module AgentTracing
+  def self.included(base)
+    base.include TracingLog
+
+    base.on(:before_turn) do |turn|
+      trace_log "turn #{turn.number}"
+    end
+
+    base.on(:after_tool) do |call, result|
+      if result.status == :ok
+        trace_log "  tool #{call.full_name} ok #{call.input.inspect}"
+      else
+        step = result.failed_step
+        trace_log "  tool #{call.full_name} #{result.status} at #{step.name}: #{step.message}#{" [#{step.detail}]" if step.detail}", level: :warn
+      end
+    end
+
+    base.on(:stopped) do |reason|
+      trace_log "stopped: #{reason}", level: :warn
+    end
+
+    base.on(:complete) do |result|
+      usage = result.usage
+      trace_log "#{status}/#{stop_reason} — #{turns.size} turns, #{tool_calls.size} tool calls (#{denied_calls.size} refused or failed), " \
+                "#{usage&.tokens&.total.to_i} tokens#{format(", $%.6f", usage.cost.total) if usage&.cost}, #{result.execution_time}s"
+    end
+  end
+end

data/lib/generators/active_harness/install/templates/concerns/pipeline_tracing.rb

@@ -0,0 +1,28 @@
+require_relative "tracing_log"
+
+# Basic tracing for any Pipeline: every step, a stop, completion.
+#
+#   class SupportPipeline < ActiveHarness::Pipeline
+#     include PipelineTracing
+#   end
+module PipelineTracing
+  def self.included(base)
+    base.include TracingLog
+
+    base.before(:step) do |step_name, _payload|
+      trace_log "→ :#{step_name}"
+    end
+
+    base.after(:step) do |step_name, result|
+      trace_log "✓ :#{step_name} (#{result.execution_time}s)"
+    end
+
+    base.callback(:stopped) do |step_name, _result|
+      trace_log "✗ stopped at :#{step_name}", level: :warn
+    end
+
+    base.callback(:complete) do |_last_result|
+      trace_log "✓ complete (#{execution_time}s)"
+    end
+  end
+end

data/lib/generators/active_harness/install/templates/concerns/request_tracing.rb

@@ -0,0 +1,44 @@
+require_relative "tracing_log"
+
+# Basic tracing for any Request: one line before the call, one after it
+# (model, time, tokens, cost), plus retries and total failure.
+#
+#   class SupportRequest < ActiveHarness::Request
+#     include RequestTracing
+#   end
+#
+# Add request-specific details by overriding #tracing_details:
+#
+#   def tracing_details(result)
+#     " spam=#{result.processed&.dig('spam')}"
+#   end
+module RequestTracing
+  def self.included(base)
+    base.include TracingLog
+
+    base.before(:call) do
+      trace_log "▶ call"
+    end
+
+    base.after(:call) do |result|
+      tokens = result.usage&.tokens&.total
+      cost   = result.usage&.cost&.total
+      trace_log "✓ #{result.model.name} #{result.execution_time}s" \
+                "#{" tokens=#{tokens}" if tokens}#{format(" cost=$%.6f", cost) if cost}" \
+                "#{tracing_details(result)}"
+    end
+
+    base.callback(:retry) do |entry, error|
+      trace_log "↺ retry #{entry&.dig(:model)} — #{error&.message}", level: :warn
+    end
+
+    base.callback(:failure) do |attempts|
+      trace_log "✗ all models failed (#{attempts.size} attempts)", level: :error
+    end
+  end
+
+  # Override to append request-specific details to the "done" line.
+  def tracing_details(_result)
+    ""
+  end
+end

data/lib/generators/active_harness/install/templates/concerns/tool_tracing.rb

@@ -0,0 +1,21 @@
+require_relative "tracing_log"
+
+# Basic tracing for any Tool: every step with its status. A refused or failed
+# step is logged with its internal `detail` — that text never reaches the model.
+#
+#   class LsTool < ActiveHarness::Tool
+#     include ToolTracing
+#   end
+module ToolTracing
+  def self.included(base)
+    base.include TracingLog
+
+    base.on(:after_step) do |step_name, step|
+      if step.status == :ok
+        trace_log "✓ #{step_name} (#{step.ms}ms)", level: :debug
+      else
+        trace_log "✗ #{step_name} #{step.status}: #{step.message}#{" [#{step.detail}]" if step.detail}", level: :warn
+      end
+    end
+  end
+end

data/lib/generators/active_harness/install/templates/concerns/tracing_log.rb

@@ -0,0 +1,18 @@
+# Shared by every *Tracing concern: ONE place that decides where trace lines go.
+# Today: Rails.logger. Swap this method to emit OpenTelemetry spans, StatsD
+# metrics or database rows, and every request/tribunal/pipeline/agent/tool that
+# includes a tracing concern follows — the examples themselves stay untouched.
+module TracingLog
+  private
+
+  def trace_log(message, level: :info)
+    logger = defined?(Rails) && Rails.logger ? Rails.logger : Logger.new($stdout)
+    logger.public_send(level, "[#{tracing_label}] #{message}")
+  end
+
+  # Override in a class to rename it in the logs.
+  def tracing_label
+    klass = self.class
+    klass.respond_to?(:full_name) ? klass.full_name : klass.name
+  end
+end

data/lib/generators/active_harness/install/templates/concerns/tribunal_tracing.rb

@@ -0,0 +1,28 @@
+require_relative "tracing_log"
+
+# Basic tracing for any Tribunal: start, each finished request, request errors, verdict.
+#
+#   class SupportGuardTribunal < ActiveHarness::Tribunal
+#     include TribunalTracing
+#   end
+module TribunalTracing
+  def self.included(base)
+    base.include TracingLog
+
+    base.on(:before_call) do
+      trace_log "▶ tribunal started"
+    end
+
+    base.on(:after_request) do |result, index|
+      trace_log "✓ request #{index + 1} done (#{result.execution_time}s) — #{result.processed}"
+    end
+
+    base.on(:request_error) do |name, error, _index|
+      trace_log "✗ #{name} failed — #{error&.message}", level: :warn
+    end
+
+    base.on(:after_verdict) do |verdict|
+      trace_log "verdict: #{verdict.inspect} (#{execution_time}s)", level: verdict ? :info : :warn
+    end
+  end
+end

data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb

@@ -12,7 +12,7 @@ class AiSupportController < ApplicationController
   # request.env / CSRF / anything else that relies on the real HTTP request.
   # ---------------------------------------------------------------------------
   def agent
-    result = SupportRequest.call(input: params.require(:input))
+    result = SupportRequest.call(input: params.require(:input)).result
 
     render json: {
       output: result.output,
@@ -30,7 +30,7 @@ class AiSupportController < ApplicationController
   # ---------------------------------------------------------------------------
   def request_memory
     memory = AppMemory.new(session_id: params.require(:session_id))
-    result = SupportRequest.call(input: params.require(:input), memory: memory)
+    result = SupportRequest.call(input: params.require(:input), memory: memory).result
 
     render json: {
       output: result.output,
@@ -74,6 +74,27 @@ class AiSupportController < ApplicationController
     end
   end
 
+  # ---------------------------------------------------------------------------
+  # POST /ai/files_agent
+  # body: { input: "What files are in storage and what is in the first one?" }
+  #
+  # Runs FilesAgent: the model calls os::ls / os::read_file (inside storage/ only)
+  # until it answers or a limit stops it. `trace` shows every turn and tool call.
+  # ---------------------------------------------------------------------------
+  def files_agent
+    agent = FilesAgent.call(input: params.require(:input))
+
+    render json: {
+      status:      agent.status,
+      stop_reason: agent.stop_reason,
+      output:      agent.output,
+      turns:       agent.turns.size,
+      tool_calls:  agent.tool_calls.size,
+      cost:        agent.usage&.cost&.total,
+      trace:       agent.trace
+    }
+  end
+
   # ---------------------------------------------------------------------------
   # GET /ai/request_stream?input=What+is+your+return+policy%3F
   #

data/lib/generators/active_harness/install/templates/goals/answer_present_goal.rb

@@ -0,0 +1,22 @@
+# A goal check judges the agent's final answer. Subclass ActiveHarness::GoalCheck and
+# override #call(candidate:, goal:, state:) to return a GoalCheckResult; plug it in with
+# `goal AnswerPresentGoal` in the agent. (The base class only raises a clear error if you
+# forget #call — any object with that method works too.)
+#
+#   candidate — the answer the model proposed
+#   goal      — what was asked (goal.task, goal.success_criteria)
+#   state     — the whole run (messages, events, tool calls) as evidence
+#
+# Prefer deterministic checks (tests, schemas, counts) to asking an LLM.
+class AnswerPresentGoal < ActiveHarness::GoalCheck
+  def call(candidate:, goal:, state:)
+    if candidate.to_s.strip.empty?
+      # :continue — the feedback is sent to the model and the agent takes another turn
+      # (at most `max_goal_retries`, default 2, then the run stops with :goal_failed).
+      # Other statuses: :achieved, :replan, :needs_input, :failed.
+      ActiveHarness::GoalCheckResult.new(status: :continue, feedback: "The answer is empty. Answer the question.")
+    else
+      ActiveHarness::GoalCheckResult.new(status: :achieved)
+    end
+  end
+end

data/lib/generators/active_harness/install/templates/pipelines/support_pipeline.rb

@@ -1,31 +1,21 @@
 require_relative "../tribunals/support_guard_tribunal"
 require_relative "../requests/support_request"
+require_relative "../concerns/pipeline_tracing"
 
 # Two-step pipeline: spam guard → answer.
 # Add more steps between them as needed.
 class SupportPipeline < ActiveHarness::Pipeline
-  # Step 1 — GUARD: reject spam before spending tokens on an answer
+  include PipelineTracing
+
+  # A GUARD step: cheap checks first, so no tokens are spent on an answer that
+  # should never be generated. A tribunal step only judges — it does not change
+  # the text that flows to the next step.
   step :spam_guard do
     use SupportGuardTribunal
+    # Stops the pipeline when true: pipeline.stopped? is set and later steps are skipped.
     stop_if ->(result) { result.processed["verdict"] == false }
   end
 
-  # Step 2 — RESPOND: generate the actual answer
+  # A TRANSFORM step: `step :name, SomeRequest` — its output becomes the input of the next step.
   step :respond, SupportRequest
-
-  before :step do |step_name, _payload|
-    puts "[pipeline] → :#{step_name}"
-  end
-
-  after :step do |step_name, _result|
-    puts "[pipeline] ✓ :#{step_name}"
-  end
-
-  callback :stopped do |step_name, _result|
-    puts "[pipeline] ✗ STOPPED at :#{step_name}"
-  end
-
-  callback :complete do |_last_result|
-    puts "[pipeline] ✓ complete"
-  end
 end

data/lib/generators/active_harness/install/templates/prompts/files_prompt.rb

@@ -0,0 +1,9 @@
+class FilesPrompt
+  def call
+    <<~PROMPT.strip
+      You are a file assistant for a workspace directory.
+      Use the tools to list directories and read files, then answer briefly.
+      If a tool is refused or fails, do not retry the same call: try another approach or tell the user what is not possible.
+    PROMPT
+  end
+end

data/lib/generators/active_harness/install/templates/prompts/support_prompt.rb

@@ -1,3 +1,6 @@
+# A prompt is a plain Ruby class with #call returning the system prompt text.
+# Before #call the Request injects @input, @context, @params, @memory and
+# @context_window, so a prompt can be built from them.
 class SupportPrompt
   def call
     "You are a concise and helpful assistant. Answer in 1-2 sentences."

data/lib/generators/active_harness/install/templates/requests/files_request.rb

@@ -0,0 +1,18 @@
+require_relative "../prompts/files_prompt"
+require_relative "../concerns/request_tracing"
+
+# The model behind FilesAgent. It must support tool calling.
+class FilesRequest < ActiveHarness::Request
+  include RequestTracing   # basic tracing lives in a concern — see app/ai/concerns/
+
+  # The system prompt: a class with #call, a String or a lambda. The agent adds the
+  # conversation and the tool schemas on top of it.
+  system_prompt FilesPrompt
+
+  # The model chain: `use` is tried first; on failure (after retries) the next
+  # `fallback` is tried. Not every provider supports tools — one that does not is skipped.
+  model do
+    use      provider: :openrouter, model: "openai/gpt-4.1-mini"
+    fallback provider: :openrouter, model: "google/gemini-2.0-flash-001"
+  end
+end

data/lib/generators/active_harness/install/templates/requests/support_guard_request.rb

@@ -1,11 +1,23 @@
 require_relative "../prompts/support_guard_prompt"
+require_relative "../concerns/request_tracing"
 
 class SupportGuardRequest < ActiveHarness::Request
+  include RequestTracing
+
   system_prompt SupportGuardPrompt
+
+  # Parse the answer as JSON into result.processed (a Hash). Needed explicitly: the gem
+  # cannot guess from the prompt that you asked for JSON. Also strips ```json fences.
   format :json
 
+  # A small, cheap model is enough for a yes/no classifier.
   model do
     use      provider: :openrouter, model: "meta-llama/llama-3.1-8b-instruct"
     fallback provider: :openrouter, model: "mistralai/mistral-nemo"
   end
+
+  # Request-specific details for the shared tracing line (see RequestTracing).
+  def tracing_details(result)
+    " spam=#{result.processed&.dig('spam')}"
+  end
 end

data/lib/generators/active_harness/install/templates/requests/support_request.rb

@@ -1,8 +1,14 @@
 require_relative "../prompts/support_prompt"
+require_relative "../concerns/request_tracing"
 
 class SupportRequest < ActiveHarness::Request
+  include RequestTracing   # basic tracing lives in a concern — see app/ai/concerns/
+
+  # The system prompt: a class with #call, a String or a lambda.
   system_prompt SupportPrompt
 
+  # The model chain: `use` is tried first; if it keeps failing (after its retries)
+  # the next `fallback` is tried. When all fail: ActiveHarness::Errors::AllModelsFailed.
   model do
     use      provider: :openrouter, model: "mistralai/mistral-nemo"
     fallback provider: :openrouter, model: "meta-llama/llama-3.1-8b-instruct"

data/lib/generators/active_harness/install/templates/tools/ls_tool.rb

@@ -0,0 +1,41 @@
+require_relative "../concerns/tool_tracing"
+
+# A safe directory listing: the model can only see inside ALLOWED_PATHS.
+class LsTool < ActiveHarness::Tool
+  include ActiveHarness::Tool::PathPolicy   # safe_path!: resolves symlinks and ".." and checks ALLOWED_PATHS
+  include ToolTracing                        # logs every step; see app/ai/concerns/
+
+  # `tool_scope` namespaces the tool (os::ls vs web::ls) and lets you allow or log a whole group;
+  # `tool_name` is the name the model sees. Both are explicit: renaming the class changes nothing for the model.
+  tool_scope :os
+  tool_name  "ls"
+
+  # What the model reads when it decides whether to call this tool — write it for the model.
+  description "List files in a directory inside the workspace"
+
+  # An argument the model must provide. `param` only DESCRIBES it to the model; the
+  # agent rejects missing or unknown arguments, and the checks below do the real policing.
+  param :path, "Directory to list (absolute path inside the workspace)"
+
+  # Fail closed: PathPolicy raises if a tool using it forgets to define this.
+  ALLOWED_PATHS = [Rails.root.join("storage").to_s].freeze
+
+  # Steps run in order BEFORE #perform. `transform: true` means the return value
+  # replaces the input for every later step and for #perform — so after this step the
+  # tool only ever sees a canonical path inside ALLOWED_PATHS.
+  step :resolve_path, transform: true do |input|
+    input.merge(path: safe_path!(input[:path]))
+  end
+
+  # A plain check: refuse with deny!. The text goes back to the model so it can adjust;
+  # pass `detail:` for anything that must stay in the logs only.
+  step :no_hidden_dirs do |input|
+    deny!("hidden dirs are off limits") if input[:path].split("/").any? { |segment| segment.start_with?(".") }
+  end
+
+  # The action itself — always last; its return value is what the model gets back.
+  # Cap the output: a huge result floods the model's context.
+  def perform(input)
+    Dir.children(input[:path]).first(200)
+  end
+end

data/lib/generators/active_harness/install/templates/tools/read_file_tool.rb

@@ -0,0 +1,44 @@
+require_relative "../concerns/tool_tracing"
+
+# Reads the first lines of a text file inside the workspace. Size and line caps
+# keep a single call from flooding the model's context.
+class ReadFileTool < ActiveHarness::Tool
+  include ActiveHarness::Tool::PathPolicy
+  include ToolTracing
+
+  tool_scope  :os                          # full name: os::read_file
+  tool_name   "read_file"
+  description "Read the first lines of a text file inside the workspace"
+
+  param :path,  "Absolute path of the file"
+  # Optional arguments: `required: false`; `type:` is :string by default (also :integer, :boolean, ...);
+  # `enum: [...]` limits the allowed values.
+  param :lines, "How many lines to read (default 50, max 100)", type: :integer, required: false
+
+  ALLOWED_PATHS = [Rails.root.join("storage").to_s].freeze
+  MAX_BYTES     = 200_000
+  MAX_LINES     = 100
+
+  # Canonicalize first, then check — validating the raw string would miss symlinks and "..".
+  step :resolve_path, transform: true do |input|
+    input.merge(path: safe_path!(input[:path]))
+  end
+
+  step :no_hidden_files do |input|
+    deny!("hidden files are off limits") if input[:path].split("/").any? { |segment| segment.start_with?(".") }
+  end
+
+  # `detail:` is for the logs only — the model just learns that the file is too large.
+  step :must_be_a_small_file do |input|
+    deny!("not a regular file") unless File.file?(input[:path])
+    deny!("file is too large", detail: "#{File.size(input[:path])} bytes") if File.size(input[:path]) > MAX_BYTES
+  end
+
+  # Anything raised here (other than deny!) becomes a generic "tool failed" for the
+  # model; the real error stays in the trace. If a write may have happened but you
+  # cannot tell (e.g. a timeout), call unknown_outcome!("...") instead of retrying.
+  def perform(input)
+    count = [(input[:lines] || 50).to_i, MAX_LINES].min
+    File.foreach(input[:path]).first(count).join
+  end
+end

data/lib/generators/active_harness/install/templates/tribunals/support_guard_tribunal.rb

@@ -1,10 +1,17 @@
 require_relative "../requests/support_guard_request"
+require_relative "../concerns/tribunal_tracing"
 
 # Runs SupportGuardRequest in parallel (single request here, extendable).
 # Verdict is true (safe) when no spam is detected.
 class SupportGuardTribunal < ActiveHarness::Tribunal
+  include TribunalTracing
+
+  # The requests that run IN PARALLEL on the same input. Several different checks —
+  # or the same check on different models — give a more reliable decision than one call.
   requests SupportGuardRequest
 
+  # Reduces all successful results to the verdict (true/false). Built-in shortcuts:
+  # `verdict :unanimous { ... }` and `verdict :majority { ... }`.
   process do |results|
     results.none? { |r| r.processed["spam"] == true }
   end

data/lib/generators/active_harness/pipeline/templates/pipeline.rb.tt

@@ -1,6 +1,11 @@
 class <%= class_name %>Pipeline < ActiveHarness::Pipeline
+  # A transform step: its output becomes the input of the next step.
+  # A guard step (`step :x do use SomeRequest; stop_if ->(r) { ... } end`) can stop the pipeline.
   step :respond, <%= class_name %>Request
 
+  # Hooks: `before :step` / `after :step` fire around every step, `callback :stopped`
+  # when a stop_if triggers, `callback :complete` after the last step.
+  # For reusable logging, move them into a concern (see app/ai/concerns/pipeline_tracing.rb).
   before :step do |step_name, _payload|
     puts "[pipeline] → :#{step_name}"
   end

data/lib/generators/active_harness/request/templates/request.rb.tt

@@ -1,6 +1,8 @@
 class <%= class_name %>Request < ActiveHarness::Request
+  # The system prompt: a class with #call, a String or a lambda.
   system_prompt <%= class_name %>Prompt
 
+  # `use` is tried first; on failure the next `fallback` is tried.
   model do
     use      provider: :openrouter, model: "mistralai/mistral-nemo"
     fallback provider: :openrouter, model: "meta-llama/llama-3.1-8b-instruct"

data/lib/generators/active_harness/tool/templates/tool.rb.tt

@@ -0,0 +1,24 @@
+class <%= class_name %>Tool < ActiveHarness::Tool
+  # Namespace + name the model sees: <%= file_name %> becomes app::<%= file_name %> (wire name app__<%= file_name %>).
+  # Both are explicit, so renaming this class never changes what the model calls. Use :os, :web, :ruby ... per area.
+  tool_scope :app
+  tool_name  "<%= file_name %>"
+
+  # Written for the model: when to call this tool.
+  description "Describe what this tool does for the model"
+
+  # An argument the model provides. It only DESCRIBES the argument — real policing is in the steps.
+  param :input, "What the model must pass"
+
+  # Checks run in order before #perform; refuse with deny!("safe text for the model")
+  # (add `detail:` for anything that must stay in the logs). `transform: true` lets a step
+  # return a new input for the later steps and #perform.
+  # step :allowed do |input|
+  #   deny!("not allowed") unless context[:user]
+  # end
+
+  # The action itself; its return value goes back to the model. Cap the size of the result.
+  def perform(input)
+    "result for #{input[:input]}"
+  end
+end

data/lib/generators/active_harness/tool/tool_generator.rb

@@ -0,0 +1,16 @@
+require "rails/generators"
+
+module ActiveHarness
+  module Generators
+    class ToolGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates an ActiveHarness tool in app/ai/tools/"
+
+      def create_tool
+        template "tool.rb.tt",
+                 "app/ai/tools/#{file_name}_tool.rb"
+      end
+    end
+  end
+end

data/lib/generators/active_harness/tribunal/templates/tribunal.rb.tt

@@ -1,6 +1,9 @@
 class <%= class_name %>Tribunal < ActiveHarness::Tribunal
+  # Requests that run in parallel on the same input.
   requests <%= class_name %>Request
 
+  # Reduces the successful results to the verdict (true/false).
+  # Shortcuts: `verdict :unanimous { ... }` / `verdict :majority { ... }`.
   process do |results|
     results.all? { |r| r.processed["result"] == true }
   end