turnkit 0.2.9 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7069b120432ec902d846961157f5635c946602a8298ed4471f09dde3e3e3e0d
-  data.tar.gz: '09a5d64ff294f89ebde99a6cf1d36dc8731c6cabbf06216d4e9b9551cbe88a1e'
+  metadata.gz: 268561a36c656098e1d23ea6de4c17616358ff931e05e1389e707a9e28fe458b
+  data.tar.gz: 8f6731d78fed5b3e3cc94d781c4f4e26accc4f8d05842b5c56eb58a6e7448907
 SHA512:
-  metadata.gz: de794838f5979194aa2469890848eb7cd60932d6f223e95d17be4d8912a6f2777afb55143f9776d7093be2072451c4a7ba0aa83ca8783c82a29375da56a11c90
-  data.tar.gz: c037fb4946a252ebf9bb2e0f99b76cca23d60f29275ce4e07a15f71232d4fdc0dce23337ad1b4b47bacd7df50ca7eedd3cf050c82167bcccb30debaa70cdfe22
+  metadata.gz: ae0a246b5937e586c808a25d28f051bafc54c2a922a52d89160eb3f5ef3bf7360b1d637cbb0c170d41eb74cd536638b6f9a1880275bd0ccd2fc8dcb4ac44db5c
+  data.tar.gz: 7ffebcfeadf51f193c7f2277a0842c2f56e00d9ff95d502915924f2a6d7e10744a0a710d1d2f5b1865182a9de21b2cce30edc3e94c16f49626912b93b1fc7063

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.2.10 - 2026-06-10
+
+- Add output audits and file-backed output policies for validating final run output.
+- Add per-tool execution limits and explicit budget errors.
+- Improve workflow event callbacks, model telemetry events, and compaction usage accounting.
+- Add an Amazon memo writer example and batched page reading in the workflow researcher example.
+
 ## 0.2.9 - 2026-06-08
 
 - Add `TurnKit::Workflow` for reusable single-orchestrator task runtimes with workflow skills, tools, guardrails, compaction, and run monitoring.

data/README.md

@@ -5,7 +5,7 @@
 [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE.md)
 
 Build durable Ruby and Rails agents with conversations, runs, workflows, tools,
-skills, sub-agents, and persistence.
+skills, output audits, sub-agents, and persistence.
 
 ## Installation
 
@@ -65,6 +65,11 @@ For runnable, API-key-free examples of the three core entry points, see
 - agent run: one bounded application task;
 - workflow: reusable task runner with skills, tools, and limits.
 
+For fuller workflow examples, see:
+
+- [`examples/workflow_researcher`](examples/workflow_researcher): source-grounded research with web tools, batch reads, per-tool limits, and deep monitoring;
+- [`examples/amazon_memo_writer`](examples/amazon_memo_writer): strict memo generation with research tools, a structured terminal submit tool, deterministic format checks, and an LLM output policy.
+
 ### Models
 
 Set a model:
@@ -208,6 +213,10 @@ workflow = TurnKit::Workflow.new(
   max_spend: 0.25,
   max_iterations: 12,
   max_tool_executions: 25,
+  max_tool_executions_by_name: {
+    web_search: 2,
+    read_web_page: 8
+  },
   compaction: {
     context_limit: 64_000,
     threshold: 0.75
@@ -253,6 +262,10 @@ Prompt caching and compaction solve different problems:
 - budgets (`max_spend`, `max_iterations`, `max_tool_executions`) keep autonomous
   loops bounded.
 
+Use `max_tool_executions_by_name` when a workflow needs different budgets for
+different tools. For example, allow many cheap reads but only one final submit
+tool, or cap web searches while allowing a batch page reader.
+
 Reach for separate agents and `sub_agents` only when the isolation is worth the
 extra model calls, such as different models, different tool permissions,
 parallel specialist review, or separate durable child conversations.
@@ -289,6 +302,71 @@ class SaveBrief < TurnKit::Tool
 end
 ```
 
+### Output audits and policies
+
+Use output audits for deterministic checks that should not depend on another
+model call: required headings, source counts, forbidden characters, JSON shape,
+or project-specific formatting rules.
+
+```ruby
+no_em_dash = ->(output) do
+  next unless output.include?("—")
+
+  { rule: "no_em_dash", message: "contains an em dash" }
+end
+
+numbered_lists_only = ->(output) do
+  lines = output.lines.each_with_index.filter_map do |line, index|
+    index + 1 if line.match?(/^\s*[-*]\s+/)
+  end
+
+  next if lines.empty?
+
+  {
+    rule: "numbered_lists_only",
+    message: "contains unordered list markers",
+    metadata: { lines: lines }
+  }
+end
+
+workflow = TurnKit::Workflow.new(
+  name: "memo_writer",
+  output_audit: [no_em_dash, numbered_lists_only],
+  output_audit_mode: :fail
+)
+```
+
+Run checks directly when you want to test a renderer or policy without calling a
+model:
+
+```ruby
+audit = TurnKit.audit_output(
+  "1. Recommendation\n- unordered item — fix this\n",
+  constraints: [no_em_dash, numbered_lists_only]
+)
+
+puts audit.clean?
+puts audit.messages
+```
+
+Use `output_policy` when a semantic judge is worth the extra model call. The
+policy can be a `.md`, `.markdown`, or `.txt` file path, a `TurnKit::OutputPolicy`,
+or any object that responds to `#call` or `#check`.
+
+```ruby
+workflow = TurnKit::Workflow.new(
+  name: "memo_writer",
+  output_policy: "app/ai/policies/amazon_memo.md",
+  output_policy_model: "gpt-4.1-mini",
+  output_policy_thinking: { effort: :low },
+  output_policy_mode: :report
+)
+```
+
+`output_policy_mode: :report` records violations while allowing the run to
+complete. `:fail` marks the run failed after recording the output and audit.
+Policy model usage and cost are counted on the parent run.
+
 ### Prompt Preview
 
 Preview a pending turn:
@@ -326,9 +404,7 @@ class SaveReport < TurnKit::Tool
   parameter :title, :string, required: true
   parameter :body, :string, required: true
 
-  def self.ends_turn? = true
-
-  def self.completion_message(result)
+  terminal! do |result|
     "Saved #{result.fetch("report_id")}."
   end
 
@@ -544,9 +620,12 @@ TurnKit.reconcile_stale!
 | `TurnKit.max_iterations` | Limit model loop iterations. |
 | `TurnKit.max_depth` | Limit sub-agent depth. |
 | `TurnKit.max_tool_executions` | Limit tool calls per turn. |
+| `TurnKit.max_tool_executions_by_name` | Limit specific tools independently. |
 | `TurnKit.timeout` | Limit turn runtime. |
 | `TurnKit.max_spend` | Limit estimated turn cost. |
 | `TurnKit.compaction` | Configure context compaction. |
+| `TurnKit.output_policy_model` | Default model for file-backed output policies. |
+| `TurnKit.output_policy_thinking` | Default thinking config for file-backed output policies. |
 | `TurnKit.on_event` | Subscribe to lifecycle events. |
 
 Set options globally:
@@ -555,6 +634,8 @@ Set options globally:
 TurnKit.default_model = "gpt-4.1-mini"
 TurnKit.max_spend = 0.25
 TurnKit.max_iterations = 25
+TurnKit.max_tool_executions_by_name = { web_search: 2 }
+TurnKit.output_policy_model = "gpt-4.1-mini"
 TurnKit.timeout = 300
 ```
 

data/lib/turnkit/agent.rb

@@ -3,13 +3,14 @@
 module TurnKit
   class Agent
     attr_reader :name, :description, :model, :instructions, :tools, :skills, :available_skills, :sub_agents
-    attr_reader :client, :store, :max_iterations, :timeout, :cost_limit, :max_depth, :max_tool_executions
+    attr_reader :client, :store, :max_iterations, :timeout, :cost_limit, :max_depth, :max_tool_executions, :max_tool_executions_by_name
     attr_reader :prompt_sections, :system_prompt, :prompt_mode, :thinking, :compaction, :output_schema, :on_event
+    attr_reader :output_audit, :output_audit_mode, :output_policy_model
 
     def initialize(name:, description: "", model: nil, instructions: "", tools: [], skills: [], available_skills: [], sub_agents: [],
       system_prompt: nil, prompt_sections: nil, prompt_mode: nil, client: nil, store: nil,
-      max_iterations: nil, timeout: nil, cost_limit: nil, max_depth: nil, max_tool_executions: nil, thinking: nil, compaction: nil,
-      output_schema: nil, on_event: nil)
+      max_iterations: nil, timeout: nil, cost_limit: nil, max_depth: nil, max_tool_executions: nil, max_tool_executions_by_name: nil, thinking: nil, compaction: nil,
+      output_schema: nil, output_audit: nil, output_audit_mode: nil, output_policy: nil, output_policy_mode: nil, output_policy_model: nil, output_policy_thinking: nil, on_event: nil)
       @name = name.to_s
       @description = description.to_s
       @model = model
@@ -28,9 +29,13 @@ module TurnKit
       @cost_limit = cost_limit
       @max_depth = max_depth
       @max_tool_executions = max_tool_executions
+      @max_tool_executions_by_name = max_tool_executions_by_name
       @thinking = self.class.normalize_thinking(thinking)
       @compaction = compaction
       @output_schema = output_schema
+      @output_policy_model = output_policy_model
+      @output_audit = normalize_output_policy_options(output_audit: output_audit, output_policy: output_policy, output_policy_model: output_policy_model, output_policy_thinking: output_policy_thinking)
+      @output_audit_mode = normalize_output_policy_mode(output_audit_mode: output_audit_mode, output_policy_mode: output_policy_mode)
       @on_event = on_event
       raise ArgumentError, "name is required" if @name.empty?
       validate_tools!
@@ -94,6 +99,18 @@ module TurnKit
       thinking
     end
 
+    def effective_output_audit
+      Array(output_audit).compact
+    end
+
+    def output_policy
+      output_audit
+    end
+
+    def output_policy_mode
+      output_audit_mode
+    end
+
     def effective_client
       client || TurnKit.client
     end
@@ -143,6 +160,7 @@ module TurnKit
         timeout: timeout || TurnKit.timeout,
         max_depth: max_depth || TurnKit.max_depth,
         max_tool_executions: max_tool_executions || TurnKit.max_tool_executions,
+        max_tool_executions_by_name: max_tool_executions_by_name || TurnKit.max_tool_executions_by_name,
         cost_limit: cost_limit || TurnKit.cost_limit,
         root_started_at: root_started_at
       )
@@ -170,6 +188,53 @@ module TurnKit
         effective_tools.each(&:validate_definition!)
       end
 
+      def normalize_output_policy_options(output_audit:, output_policy:, output_policy_model:, output_policy_thinking:)
+        raise ArgumentError, "use output_policy: or output_audit:, not both" if output_audit && output_policy
+
+        output_policy.nil? ? output_audit : normalize_output_policy(output_policy, model: output_policy_model, thinking: output_policy_thinking)
+      end
+
+      def normalize_output_policy(value, model: nil, thinking: nil)
+        case value
+        when nil
+          nil
+        when Array
+          value.map { |item| normalize_output_policy(item, model: model, thinking: thinking) }.compact
+        when String
+          output_policy_from_path(value, model: model, thinking: thinking)
+        when Pathname
+          output_policy_from_path(value.to_s, model: model, thinking: thinking)
+        else
+          return value if value.respond_to?(:call) || value.respond_to?(:check)
+
+          raise ArgumentError, "output_policy must be a policy file path, a #call/#check object, or an array of those"
+        end
+      end
+
+      def output_policy_from_path(path, model: nil, thinking: nil)
+        unless path.match?(/\.(md|markdown|txt)\z/i)
+          raise ArgumentError, "output_policy string must be a .md, .markdown, or .txt file path"
+        end
+
+        TurnKit::OutputPolicy.from_file(
+          path,
+          model: model || TurnKit.output_policy_model,
+          thinking: thinking || TurnKit.output_policy_thinking
+        )
+      end
+
+      def normalize_output_policy_mode(output_audit_mode:, output_policy_mode:)
+        if output_audit_mode && output_policy_mode && output_audit_mode.to_sym != output_policy_mode.to_sym
+          raise ArgumentError, "use output_policy_mode: or output_audit_mode:, not both"
+        end
+
+        value = output_policy_mode || output_audit_mode || :report
+        mode = value.to_sym
+        raise ArgumentError, "unknown output_policy_mode: #{value}" unless %i[report fail].include?(mode)
+
+        mode
+      end
+
       def task_message(task, input)
         text = task.to_s
         return text if input.nil?

data/lib/turnkit/budget.rb

@@ -2,32 +2,40 @@
 
 module TurnKit
   class Budget
-    attr_reader :root_started_at, :max_iterations, :timeout, :max_depth, :max_tool_executions, :cost_limit
+    attr_reader :root_started_at, :max_iterations, :timeout, :max_depth, :max_tool_executions, :max_tool_executions_by_name, :cost_limit
 
-    def initialize(max_iterations:, timeout:, max_depth:, max_tool_executions:, cost_limit: nil, root_started_at: Clock.now)
+    def initialize(max_iterations:, timeout:, max_depth:, max_tool_executions:, max_tool_executions_by_name: {}, cost_limit: nil, root_started_at: Clock.now)
       @root_started_at = root_started_at
       @max_iterations = max_iterations
       @timeout = timeout
       @max_depth = max_depth
       @max_tool_executions = max_tool_executions
+      @max_tool_executions_by_name = normalize_tool_limits(max_tool_executions_by_name)
       @cost_limit = cost_limit
       @iterations = 0
       @tool_executions = 0
+      @tool_executions_by_name = Hash.new(0)
       @cost = 0
       @mutex = Mutex.new
     end
 
     def count_iteration!
       @mutex.synchronize do
+        raise BudgetError, "maximum iterations reached" if max_iterations && @iterations >= max_iterations
+
         @iterations += 1
-        raise Error, "maximum iterations reached" if max_iterations && @iterations > max_iterations
       end
     end
 
-    def count_tool_execution!
+    def count_tool_execution!(name = nil)
       @mutex.synchronize do
+        key = name.to_s if name
+        limit = max_tool_executions_by_name[key] if key
+        raise BudgetError, "maximum tool executions reached" if max_tool_executions && @tool_executions >= max_tool_executions
+        raise BudgetError, "maximum executions reached for tool #{key}" if limit && @tool_executions_by_name[key] >= limit
+
         @tool_executions += 1
-        raise Error, "maximum tool executions reached" if max_tool_executions && @tool_executions > max_tool_executions
+        @tool_executions_by_name[key] += 1 if key
       end
     end
 
@@ -40,13 +48,20 @@ module TurnKit
 
       @mutex.synchronize do
         @cost += cost.to_f
-        raise Error, "cost limit reached" if @cost > cost_limit
+        raise BudgetError, "cost limit reached" if @cost > cost_limit
       end
     end
 
     def check!(depth:)
-      raise Error, "maximum sub-agent depth reached" if max_depth && depth > max_depth
-      raise Error, "turn timed out" if timeout && Clock.now >= root_started_at + timeout
+      raise BudgetError, "maximum sub-agent depth reached" if max_depth && depth > max_depth
+      raise BudgetError, "turn timed out" if timeout && Clock.now >= root_started_at + timeout
     end
+
+    private
+      def normalize_tool_limits(value)
+        value.to_h.transform_keys(&:to_s).transform_values do |limit|
+          limit.nil? ? nil : Integer(limit)
+        end
+      end
   end
 end

data/lib/turnkit/compaction.rb

@@ -117,6 +117,8 @@ module TurnKit
       return unless force || over_threshold?(messages, policy)
 
       compact!(turn.conversation, agent: turn.agent, turn: turn, focus: focus, auto: true, overrides: policy, force: true)
+    rescue BudgetError
+      raise
     rescue StandardError => error
       TurnKit.logger&.warn("TurnKit compaction failed: #{error.class}: #{error.message}")
       nil
@@ -144,12 +146,15 @@ module TurnKit
         target_tokens: summary_budget(selected_tokens, policy),
         fallback_model: turn&.model || conversation.model || agent.effective_model,
         conversation_id: conversation.id,
-        turn_id: turn&.id
+        turn_id: turn&.id,
+        turn: turn
       )
 
       append_summary(conversation, turn: turn, summary: summary, selected: selected, policy: policy, focus: focus, auto: auto, input_tokens: selected_tokens)
     rescue CompactionError
       raise
+    rescue BudgetError
+      raise
     rescue StandardError => error
       raise CompactionError, "#{error.class}: #{error.message}"
     end
@@ -350,18 +355,24 @@ module TurnKit
       index
     end
 
-    def generate_summary(agent:, policy:, messages:, previous_summary:, focus:, target_tokens:, fallback_model:, conversation_id:, turn_id:)
+    def generate_summary(agent:, policy:, messages:, previous_summary:, focus:, target_tokens:, fallback_model:, conversation_id:, turn_id:, turn: nil)
       client = policy["client"] || agent.effective_client
       model = policy["model"] || fallback_model
       safe_messages = messages.map { |message| sanitize_message(message, policy) }
       prompt = build_prompt(previous_summary: previous_summary, focus: focus, target_tokens: target_tokens)
-      result = client.chat(
+      attrs = {
         model: model,
         messages: MessageProjection.for(safe_messages) + [ { role: :user, content: prompt } ],
         tools: [],
         instructions: COMPACTION_SYSTEM_PROMPT,
         metadata: { compaction: true, conversation_id: conversation_id, turn_id: turn_id }
-      )
+      }
+      result = if turn
+        turn.internal_model_call(**attrs, purpose: "compaction", client: policy["client"])
+      else
+        client.validate!(model: model)
+        client.chat(**attrs)
+      end
       text = result.text.to_s.strip
       raise CompactionError, "compaction model returned an empty summary" if text.empty?
 

data/lib/turnkit/error.rb

@@ -2,6 +2,7 @@
 
 module TurnKit
   class Error < StandardError; end
+  class BudgetError < Error; end
   class ConfigError < Error; end
   class CompactionError < Error; end
   class ModelAccessError < ConfigError; end

data/lib/turnkit/output_audit.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module TurnKit
+  class OutputAudit
+    Violation = Struct.new(:rule, :message, :metadata, keyword_init: true) do
+      def to_h
+        { "rule" => rule.to_s, "message" => message.to_s, "metadata" => metadata || {} }
+      end
+    end
+
+    Result = Struct.new(:violations, keyword_init: true) do
+      def clean?
+        violations.empty?
+      end
+
+      def messages
+        violations.map(&:message)
+      end
+
+      def to_h
+        { "clean" => clean?, "violations" => violations.map(&:to_h) }
+      end
+    end
+
+    def self.check(output, constraints: [], context: {})
+      new(output, constraints: constraints, context: context).check
+    end
+
+    def initialize(output, constraints: [], context: {})
+      @output = output
+      @constraints = Array(constraints)
+      @context = context || {}
+    end
+
+    def check
+      Result.new(violations: constraints.flat_map { |constraint| normalize(check_constraint(constraint)) })
+    end
+
+    private
+      attr_reader :output, :constraints, :context
+
+      def check_constraint(constraint)
+        if constraint.respond_to?(:check)
+          call_with_optional_context(constraint.method(:check))
+        elsif constraint.respond_to?(:call)
+          callable = constraint.is_a?(Proc) ? constraint : constraint.method(:call)
+          call_with_optional_context(callable)
+        else
+          raise ArgumentError, "output constraints must respond to #call or #check"
+        end
+      end
+
+      def call_with_optional_context(method)
+        parameters = method.parameters
+        return method.call(output) unless parameters.any? { |kind, _| %i[key keyreq keyrest].include?(kind) }
+        return method.call(output, **context) if parameters.any? { |kind, _| kind == :keyrest }
+
+        accepted = parameters.filter_map { |kind, name| name if %i[key keyreq].include?(kind) }
+        method.call(output, **context.slice(*accepted))
+      end
+
+      def normalize(value)
+        case value
+        when nil, false, true
+          []
+        when Violation
+          [ value ]
+        when Result
+          value.violations
+        when String
+          [ Violation.new(rule: "output_constraint", message: value, metadata: {}) ]
+        when Hash
+          [ violation_from_hash(value) ]
+        else
+          if value.respond_to?(:to_ary)
+            value.to_ary.flat_map { |item| normalize(item) }
+          else
+            raise ArgumentError, "output constraint returned unsupported value: #{value.class}"
+          end
+        end
+      end
+
+      def violation_from_hash(value)
+        attrs = value.transform_keys(&:to_s)
+        Violation.new(
+          rule: attrs["rule"] || "output_constraint",
+          message: attrs["message"] || attrs["error"] || "output constraint failed",
+          metadata: attrs["metadata"] || attrs.reject { |key, _| %w[rule message error].include?(key) }
+        )
+      end
+  end
+end

data/lib/turnkit/output_policy.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module TurnKit
+  class OutputPolicy
+    DEFAULT_SCHEMA = {
+      type: "object",
+      properties: {
+        approved: { type: "boolean" },
+        violations: {
+          type: "array",
+          items: {
+            type: "object",
+            properties: {
+              rule: { type: "string" },
+              message: { type: "string" }
+            },
+            required: [ "rule", "message" ]
+          }
+        }
+      },
+      required: [ "approved", "violations" ]
+    }.freeze
+
+    attr_reader :name, :content, :model, :thinking, :client
+
+    def self.from_file(path, name: nil, **options)
+      new(name: name || File.basename(path, File.extname(path)), content: File.read(path), **options)
+    end
+
+    def initialize(content:, name: "output_policy", model: nil, thinking: nil, client: nil)
+      @name = name.to_s
+      @content = content.to_s
+      @model = model
+      @thinking = Agent.normalize_thinking(thinking)
+      @client = client
+      raise ArgumentError, "content is required" if @content.empty?
+    end
+
+    def call(output, run: nil, turn: nil)
+      model_name = model || turn&.model || run&.turn&.model || TurnKit.default_model
+      result = if turn
+        turn.internal_model_call(
+          model: model_name,
+          messages: audit_messages(output),
+          tools: [],
+          instructions: audit_instructions,
+          thinking: thinking,
+          output_schema: DEFAULT_SCHEMA,
+          metadata: { output_policy: name },
+          purpose: "output_policy",
+          client: client
+        )
+      else
+        audit_client = client || TurnKit.client
+        audit_client.validate!(model: model_name)
+        chat(audit_client, model: model_name, messages: audit_messages(output), tools: [], instructions: audit_instructions, thinking: thinking, output_schema: DEFAULT_SCHEMA, metadata: { output_policy: name })
+      end
+      data = result.output_data || parse_json(result.text)
+      return if data.fetch("approved", false)
+
+      Array(data["violations"]).map do |violation|
+        attrs = violation.transform_keys(&:to_s)
+        OutputAudit::Violation.new(
+          rule: attrs["rule"] || name,
+          message: attrs["message"] || "output policy failed",
+          metadata: attrs.reject { |key, _| %w[rule message].include?(key) }
+        )
+      end
+    end
+
+    private
+      def audit_instructions
+        <<~TEXT
+          You audit model outputs against the policy below.
+
+          Return only a JSON object matching this shape:
+          {"approved":true,"violations":[]}
+
+          Set approved to true only when the output satisfies the policy. For each violation, include a concise rule and message. Do not repair the output. Do not wrap the JSON in Markdown. Do not include commentary before or after the JSON.
+
+          Policy:
+          #{content}
+        TEXT
+      end
+
+      def audit_messages(output)
+        [ { role: :user, content: JSON.generate(output: output) } ]
+      end
+
+      def chat(client, **kwargs)
+        accepted = chat_keyword_names(client)
+        kwargs = kwargs.slice(*accepted) unless accepted.include?(:keyrest)
+        client.chat(**kwargs)
+      end
+
+      def chat_keyword_names(client)
+        client.method(:chat).parameters.filter_map do |kind, name|
+          return [ :keyrest ] if kind == :keyrest
+
+          name if %i[key keyreq].include?(kind)
+        end
+      end
+
+      def parse_json(value)
+        JSON.parse(extract_json(value.to_s))
+      rescue JSON::ParserError
+        { "approved" => false, "violations" => [ { "rule" => name, "message" => "output policy returned invalid JSON" } ] }
+      end
+
+      def extract_json(value)
+        text = value.strip
+        return text if text.start_with?("{") && text.end_with?("}")
+
+        fenced = text[/```(?:json)?\s*(\{.*?\})\s*```/m, 1]
+        return fenced if fenced
+
+        object = text[/\{.*\}/m]
+        object || text
+      end
+  end
+end

data/lib/turnkit/run.rb

@@ -14,6 +14,8 @@ module TurnKit
     def output = output_text
     def output_text = turn.output_text
     def output_data = turn.output_data
+    def output_audit = turn.output_audit
+    def output_audit_clean? = output_audit.nil? || output_audit.fetch("clean", false)
     def usage = Usage.from_records(turn_records)
     def cost = Cost.from_records(turn_records)
     def steps = turn_records.length

data/lib/turnkit/tool_runner.rb

@@ -23,10 +23,17 @@ module TurnKit
       attr_reader :turn
 
       def run(tool_call)
-        turn.budget.count_tool_execution!
-        tool = tool_for(tool_call.name)
         execution = ToolExecution.new(create_execution(tool_call))
 
+        begin
+          turn.budget.count_tool_execution!(tool_call.name)
+        rescue BudgetError => error
+          finish_error(execution, tool_call, error.message, details: { "class" => error.class.name, "budget_denied" => true })
+          raise
+        end
+
+        tool = tool_for(tool_call.name)
+
         unless tool
           return finish_error(execution, tool_call, "unknown tool: #{tool_call.name}")
         end
@@ -58,7 +65,7 @@ module TurnKit
       def finish_success(execution, tool_call, payload)
         attrs = turn.store.update_tool_execution(execution.id, "status" => "completed", "result" => payload, "completed_at" => Clock.now)
         append_result(execution, tool_call, payload)
-        turn.emit("tool_call.completed", id: tool_call.id, name: tool_call.name)
+        turn.emit("tool_call.completed", id: tool_call.id, name: tool_call.name, result_chars: payload.to_json.length)
         ToolExecution.new(attrs)
       end
 
@@ -66,7 +73,7 @@ module TurnKit
         error = { "message" => message.to_s, "details" => details }.compact
         attrs = turn.store.update_tool_execution(execution.id, "status" => "failed", "error" => error, "completed_at" => Clock.now)
         append_result(execution, tool_call, error)
-        turn.emit("tool_call.failed", id: tool_call.id, name: tool_call.name, error: error)
+        turn.emit("tool_call.failed", id: tool_call.id, name: tool_call.name, error: error, result_chars: error.to_json.length)
         ToolExecution.new(attrs)
       end
 

data/lib/turnkit/turn.rb

@@ -45,13 +45,13 @@ module TurnKit
         TurnKit::Compaction.maybe_compact!(self)
 
         request = model_request
-        emit("model.requested", model: request.model, tool_names: request.tool_names)
+        emit_model_requested("model.requested", request)
         result = call_client(request)
-        emit("model.completed", model: result.model || model, tool_call_count: result.tool_calls.length)
         result_cost = Cost.from_usage(result.usage, model: result.model || model)
 
-        budget.add_cost!(result_cost.total)
         add_usage!(result.usage, cost: result_cost)
+        emit_model_completed("model.completed", result, result_cost, model: model)
+        budget.add_cost!(result_cost.total)
         persist_assistant_message(result)
 
         if result.tool_calls?
@@ -62,8 +62,7 @@ module TurnKit
             break
           end
         else
-          update!(status: "completed", output_text: result.text, output_data: result.output_data, completed_at: Clock.now)
-          emit("turn.completed", status: status, output_text: result.text)
+          complete_with_output(result.text, output_data: result.output_data)
           break
         end
       end
@@ -96,6 +95,10 @@ module TurnKit
       @record["output_data"]
     end
 
+    def output_audit
+      (@record["options"] || {})["output_audit"]
+    end
+
     def usage
       Usage.from_h(@record["usage"] || {})
     end
@@ -125,6 +128,28 @@ module TurnKit
       emit_event(Event.new(type: type, turn_id: id, conversation_id: conversation.id, payload: payload))
     end
 
+    def internal_model_call(model:, messages:, instructions:, tools: [], thinking: nil, output_schema: nil, metadata: {}, purpose:, client: nil)
+      request = ModelRequest.new(
+        model: model,
+        messages: messages,
+        tools: tools,
+        instructions: instructions,
+        thinking: thinking,
+        output_schema: output_schema,
+        metadata: { purpose: purpose.to_s, turn_id: id, conversation_id: conversation.id }.merge(metadata || {})
+      )
+      model_client = client || agent.effective_client
+      model_client.validate!(model: request.model)
+
+      emit_model_requested("#{purpose}.model.requested", request)
+      result = call_client(request, client: model_client)
+      result_cost = Cost.from_usage(result.usage, model: result.model || request.model)
+      add_usage!(result.usage, cost: result_cost)
+      emit_model_completed("#{purpose}.model.completed", result, result_cost, model: request.model)
+      budget.add_cost!(result_cost.total)
+      result
+    end
+
     private
       def model_request
         prompt = SystemPrompt.new(agent: agent, turn: self, conversation: conversation, mode: prompt_mode || agent.effective_prompt_mode(turn: self))
@@ -148,7 +173,7 @@ module TurnKit
         )
       end
 
-      def call_client(request)
+      def call_client(request, client: agent.effective_client)
         kwargs = {
           model: request.model,
           messages: request.messages,
@@ -159,9 +184,9 @@ module TurnKit
           metadata: request.metadata,
           on_event: ->(event) { emit_event(event) }
         }
-        accepted = chat_keyword_names(agent.effective_client)
+        accepted = chat_keyword_names(client)
         kwargs = kwargs.slice(*accepted) unless accepted.include?(:keyrest)
-        agent.effective_client.chat(**kwargs)
+        client.chat(**kwargs)
       end
 
       def chat_keyword_names(client)
@@ -176,6 +201,26 @@ module TurnKit
         MessageProjection.for(TurnKit::Compaction.project(conversation.messages_for_turn(self)))
       end
 
+      def emit_model_requested(type, request)
+        emit(
+          type,
+          model: request.model,
+          tool_names: request.tool_names,
+          message_count: request.messages.length,
+          prompt: request.report
+        )
+      end
+
+      def emit_model_completed(type, result, cost, model: self.model)
+        emit(
+          type,
+          model: result.model || model,
+          tool_call_count: result.tool_calls.length,
+          usage: result.usage.to_h,
+          cost: cost.to_h
+        )
+      end
+
       def thinking_from_options
         options = (@record["options"] || {}).transform_keys(&:to_s)
         return Agent.normalize_thinking(options["thinking"]) if options.key?("thinking")
@@ -219,8 +264,40 @@ module TurnKit
         message = runner.completion_message(execution)
         assistant = conversation.append_message(role: "assistant", kind: "text", text: message, turn_id: id)
         emit("message.created", message_id: assistant.id, role: assistant.role, kind: assistant.kind)
-        update!(status: "completed", output_text: message, completed_at: Clock.now)
-        emit("turn.completed", status: status, output_text: message)
+        complete_with_output(message)
+      end
+
+      def complete_with_output(text, output_data: nil)
+        audit = audit_output(text, output_data: output_data)
+        attrs = { output_text: text, output_data: output_data, completed_at: Clock.now }
+        if audit && !audit.clean? && agent.output_audit_mode == :fail
+          attrs[:status] = "failed"
+          attrs[:error] = { "class" => "TurnKit::OutputAudit", "message" => audit.messages.join("; "), "output_audit" => audit.to_h }
+        else
+          attrs[:status] = "completed"
+        end
+        update!(attrs)
+        persist_output_audit(audit) if audit
+
+        if failed?
+          emit("turn.failed", error: @record["error"])
+        else
+          emit("turn.completed", status: status, output_text: text)
+        end
+      end
+
+      def audit_output(text, output_data: nil)
+        constraints = agent.effective_output_audit
+        return nil if constraints.empty?
+
+        output = output_data.nil? ? text : output_data
+        TurnKit.audit_output(output, constraints: constraints, context: { turn: self, output_text: text, output_data: output_data })
+      end
+
+      def persist_output_audit(audit)
+        options = (@record["options"] || {}).merge("output_audit" => audit.to_h)
+        update!(options: options)
+        emit("output_audit.completed", clean: audit.clean?, violation_count: audit.violations.length)
       end
 
       def add_usage!(usage, cost: nil)

data/lib/turnkit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TurnKit
-  VERSION = "0.2.9"
+  VERSION = "0.2.10"
 end

data/lib/turnkit/workflow.rb

@@ -6,7 +6,9 @@ module TurnKit
   class Workflow
     attr_reader :name, :description, :instructions, :tools, :skills, :available_skills
     attr_reader :model, :client, :store, :prompt_mode, :thinking, :compaction, :output_schema
-    attr_reader :max_iterations, :timeout, :cost_limit, :max_depth, :max_tool_executions
+    attr_reader :on_event
+    attr_reader :max_iterations, :timeout, :cost_limit, :max_depth, :max_tool_executions, :max_tool_executions_by_name
+    attr_reader :output_audit, :output_audit_mode, :output_policy, :output_policy_mode, :output_policy_model, :output_policy_thinking
 
     DEFAULT_INSTRUCTIONS = <<~TEXT.strip
       You are an autonomous task orchestrator. Navigate from the application
@@ -17,6 +19,10 @@ module TurnKit
       patterns. Iterate when work needs missing context, critique, revision, or
       verification.
 
+      When multiple independent items need the same kind of fetch or read, and
+      an available batch tool can handle them in one call, prefer the batch tool
+      over repeated one-item tool calls.
+
       Stop when the task is complete, when the available context and tools are
       sufficient for the best possible answer, or when further iteration would
       not materially improve the result. Respect runtime, cost, and iteration
@@ -25,9 +31,10 @@ module TurnKit
 
     def initialize(name: "workflow", description: "", instructions: nil,
       tools: [], skills: [], available_skills: [], model: nil, client: nil,
-      store: nil, prompt_mode: :task, thinking: nil, compaction: nil,
+      store: nil, prompt_mode: :task, thinking: nil, compaction: nil, on_event: nil,
       output_schema: nil, max_iterations: nil, timeout: nil, max_spend: nil,
-      cost_limit: nil, max_depth: nil, max_tool_executions: nil)
+      cost_limit: nil, max_depth: nil, max_tool_executions: nil, max_tool_executions_by_name: nil,
+      output_audit: nil, output_audit_mode: nil, output_policy: nil, output_policy_mode: nil, output_policy_model: nil, output_policy_thinking: nil)
 
       @name = name.to_s
       @description = description.to_s
@@ -41,14 +48,22 @@ module TurnKit
       @prompt_mode = prompt_mode
       @thinking = thinking
       @compaction = compaction
+      @on_event = on_event
       @output_schema = output_schema
       @max_iterations = max_iterations
       @timeout = timeout
       @cost_limit = cost_limit || max_spend
       @max_depth = max_depth
       @max_tool_executions = max_tool_executions
+      @max_tool_executions_by_name = max_tool_executions_by_name
+      @output_audit = output_audit
+      @output_audit_mode = output_audit_mode
+      @output_policy = output_policy
+      @output_policy_mode = output_policy_mode
+      @output_policy_model = output_policy_model
+      @output_policy_thinking = output_policy_thinking
       raise ArgumentError, "name is required" if @name.empty?
-      build_agent
+      @agent = build_agent
     end
 
     def run(prompt = nil, task: nil, input: nil, async: false, subject: nil, metadata: {},
@@ -57,7 +72,13 @@ module TurnKit
       task = task || prompt
       raise ArgumentError, "task is required" if task.to_s.empty?
 
-      build_agent(cost_limit: cost_limit || max_spend, **options).run(
+      runtime_agent = if options.empty? && cost_limit.nil? && max_spend.nil?
+        @agent
+      else
+        build_agent(cost_limit: cost_limit || max_spend, **options)
+      end
+
+      runtime_agent.run(
         task,
         input: input,
         async: async,
@@ -67,7 +88,7 @@ module TurnKit
     end
 
     def agent(**options)
-      build_agent(**options)
+      options.empty? ? @agent : build_agent(**options)
     end
 
     def max_spend
@@ -89,12 +110,20 @@ module TurnKit
           prompt_mode: prompt_mode,
           thinking: thinking,
           compaction: compaction,
+          on_event: on_event,
           output_schema: output_schema,
           max_iterations: max_iterations,
           timeout: timeout,
           cost_limit: cost_limit,
           max_depth: max_depth,
-          max_tool_executions: max_tool_executions
+          max_tool_executions: max_tool_executions,
+          max_tool_executions_by_name: max_tool_executions_by_name,
+          output_audit: output_audit,
+          output_audit_mode: output_audit_mode,
+          output_policy: output_policy,
+          output_policy_mode: output_policy_mode,
+          output_policy_model: output_policy_model,
+          output_policy_thinking: output_policy_thinking
         }
         attrs.merge!(overrides.compact)
         Agent.new(**attrs)

data/lib/turnkit.rb

@@ -5,6 +5,7 @@ require "digest"
 require "securerandom"
 require "time"
 require "date"
+require "pathname"
 
 require_relative "turnkit/version"
 require_relative "turnkit/error"
@@ -22,6 +23,8 @@ require_relative "turnkit/message"
 require_relative "turnkit/record"
 require_relative "turnkit/result"
 require_relative "turnkit/skill"
+require_relative "turnkit/output_audit"
+require_relative "turnkit/output_policy"
 require_relative "turnkit/prompt_data"
 require_relative "turnkit/prompt_context"
 require_relative "turnkit/prompt_contribution"
@@ -48,8 +51,10 @@ module TurnKit
   class << self
     attr_accessor :default_model, :client, :store, :logger
     attr_accessor :max_iterations, :timeout, :max_depth, :max_tool_executions
+    attr_accessor :max_tool_executions_by_name
     attr_accessor :cost_limit, :prompt_cache
     attr_accessor :compaction
+    attr_accessor :output_policy_model, :output_policy_thinking
     attr_accessor :cost_rates, :cost_calculator
     attr_accessor :prompt_sections, :prompt_behavior, :available_skills
     attr_accessor :prompt_data_max_chars, :context_contributors
@@ -66,6 +71,7 @@ module TurnKit
   self.timeout = 300
   self.max_depth = 3
   self.max_tool_executions = 100
+  self.max_tool_executions_by_name = {}
   self.prompt_cache = :auto
   self.compaction = true
   self.cost_rates = {}
@@ -76,6 +82,8 @@ module TurnKit
   self.system_prompt_contributors = []
   self.model_prompt_contributors = {}
   self.on_event = nil
+  self.output_policy_model = nil
+  self.output_policy_thinking = { effort: :low }
 
   def self.configure
     yield self
@@ -102,4 +110,8 @@ module TurnKit
       store.update_turn(turn.fetch("id"), "status" => "stale", "completed_at" => Clock.now)
     end
   end
+
+  def self.audit_output(output, constraints: [], context: {})
+    OutputAudit.check(output, constraints: constraints, context: context)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turnkit
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.2.10
 platform: ruby
 authors:
 - Sam Couch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-08 00:00:00.000000000 Z
+date: 2026-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_llm
@@ -61,6 +61,8 @@ files:
 - lib/turnkit/message.rb
 - lib/turnkit/message_projection.rb
 - lib/turnkit/model_request.rb
+- lib/turnkit/output_audit.rb
+- lib/turnkit/output_policy.rb
 - lib/turnkit/prompt_context.rb
 - lib/turnkit/prompt_contribution.rb
 - lib/turnkit/prompt_data.rb