phronomy 0.16.0 → 0.18.0

data/benchmark/baseline.json

@@ -4,6 +4,5 @@
   "tool_params_schema_definition": 19534379.159046534,
   "dispatch_parallel_10": 886.0,
   "cancellation_token_cancelled": 4335060.97443425,
-  "cancellation_token_raise_if_cancelled_noop": 3566903.189098373,
-  "trim_messages_2000": 2896552.0
-}
+  "cancellation_token_raise_if_cancelled_noop": 3566903.189098373
+}

data/benchmark/bench_agent_invoke.rb

@@ -3,11 +3,11 @@
 # bench_agent_invoke.rb — Agent#invoke framework overhead benchmark.
 #
 # Measures the per-invoke cost of the Phronomy::Agent::Base framework path
-# (context assembly, guardrail checks, before_completion hooks, response
-# handling) with a fully stubbed LLM.  No network calls are made.
+# (context assembly, filter checks, before_llm_input hooks, response handling)
+# with a fully stubbed LLM. No network calls are made.
 #
 # Scenarios:
-#   1. Minimal agent (no tools, no knowledge) — baseline framework overhead.
+#   1. Minimal agent (no tools, no persistent Knowledge) — baseline framework overhead.
 #   2. Tool-aware agent with a registered stub Tool.
 #   3. Agent#stream setup latency (first-chunk time with stubbed stream).
 
@@ -81,7 +81,7 @@ end
 bench_tool_class = Class.new(Phronomy::Agent::Base) do
   agent_definition id: "bench-tool", version: 1
   model "stub-model"
-  tools BenchNullTool
+  tools(BenchNullTool => nil)
 
   define_method(:build_chat) { |*| BenchStubChat.new(BENCH_RESP) }
 end

data/benchmark/bench_context_assembler.rb

@@ -1,54 +1,154 @@
 # frozen_string_literal: true
 
-# Benchmark: Context::Assembler#build
+# Benchmark: Manifest-first Context assembly and Default Context Policy.
 #
-# Tests context assembly performance for varying numbers of messages and
-# knowledge chunks. This path is exercised on every agent turn.
+# Usage:
+#   ruby benchmark/bench_context_assembler.rb
+#
+# Measures:
+#   1. ContextPolicies::Default selection cost for growing canonical candidate sets.
+#   2. ContextAssembler#build_initial end-to-end Manifest construction.
+#
+# No provider call is performed.
 
 require "benchmark"
 require_relative "../lib/phronomy"
 
-BenchAsmMessage = Struct.new(:content)
+module BenchContextAssembler
+  module_function
 
-def make_assembler(n_messages:, n_chunks:, with_budget: false)
-  budget = if with_budget
-    Phronomy::LlmContextWindow::TokenBudget.new(context_window: 4096, max_output_tokens: 512)
-  end
-  asm = Phronomy::LlmContextWindow::Assembler.new(budget: budget)
-  asm.add_instruction("You are a helpful assistant. Answer the user's question.")
-  n_chunks.times do |i|
-    asm.add_knowledge("Fact #{i}: The capital of country #{i} is City #{i}.", type: :entity, trusted: true)
+  def candidate(index)
+    category, role = if (index % 10).zero?
+      [:knowledge, :user]
+    elsif index.even?
+      [:assistant_message, :assistant]
+    else
+      [:external_message, :user]
+    end
+
+    Phronomy::Agent::ContextCandidate.new(
+      candidate_id: "candidate-#{index}",
+      source_kind: :journal,
+      category: category,
+      role: role,
+      content_ref: "content-#{index}",
+      record_id: "record-#{index}",
+      agent_id: "bench-agent",
+      execution_id: "execution-#{index / 4}",
+      llm_call_id: nil,
+      tool_call_id: nil,
+      sequence: index,
+      requirement: :optional,
+      priority: 0,
+      metadata: {
+        "estimated_tokens" => 8,
+        "source_sequence" => index
+      }
+    )
   end
-  msgs = Array.new(n_messages) { BenchAsmMessage.new("This is a conversation message.") }
-  asm.add_messages(msgs)
-  asm
-end
 
-BENCH_ASM_ITERATIONS = 1_000
+  def parts
+    {
+      unit_builder:
+        Phronomy::Agent::ContextParts::UnitBuilders::DependencyAwareUnitBuilder.new,
+      required_context_resolver:
+        Phronomy::Agent::ContextParts::Requirements::RequiredContextResolver.new,
+      recent_first_selector:
+        Phronomy::Agent::ContextParts::Selectors::RecentFirstSelector.new,
+      token_budget_packer:
+        Phronomy::Agent::ContextParts::Budget::TokenBudgetPacker.new
+    }.freeze
+  end
 
-puts "=== bench_context_assembler ==="
-Benchmark.bm(40) do |x|
-  x.report("build(10 msgs, 0 chunks)") do
-    BENCH_ASM_ITERATIONS.times { make_assembler(n_messages: 10, n_chunks: 0).build }
+  def request(candidate_count)
+    candidates = Array.new(candidate_count) { |i| candidate(i) }
+    Phronomy::Agent::ContextRequest.new(
+      agent_id: "bench-agent",
+      execution_id: "bench-execution",
+      call_sequence: 2,
+      call_mode: :complete,
+      candidates: candidates,
+      token_budget: Phronomy::LlmContextWindow::TokenBudget.new(
+        context_window: [candidate_count * 16, 4_096].max,
+        max_output_tokens: 512
+      ),
+      model_config: {},
+      previous_manifest: nil,
+      required_coverage: [],
+      parts: parts,
+      metadata: {"mandatory_token_estimate" => 32}
+    )
   end
 
-  x.report("build(100 msgs, 5 chunks)") do
-    BENCH_ASM_ITERATIONS.times { make_assembler(n_messages: 100, n_chunks: 5).build }
+  def assembler_fixture
+    persistence = Phronomy::Persistence::InMemory.new
+    agent_class = Class.new(Phronomy::Agent::Base) do
+      agent_definition id: "bench-manifest-context-assembler", version: 1
+      model "local-model"
+      context_window 16_384
+      max_output_tokens 1_024
+      instructions "Benchmark instruction"
+    end
+    agent = agent_class.new(
+      persistence: persistence,
+      knowledge: ["Persistent benchmark knowledge"]
+    )
+    root = agent.agent_root
+    input_ref = persistence.contents.put_text("benchmark input")
+    input_record = Phronomy::Agent::JournalRecord.new(
+      agent_id: agent.agent_id,
+      kind: :external_message,
+      channel: :external,
+      role: :user,
+      content_ref: input_ref,
+      context_generation: root.transcript_generation,
+      context_candidate: true
+    )
+    execution = Phronomy::Agent::AgentExecution.start(
+      agent_root: root,
+      input_record: input_record,
+      metadata: {
+        "current_input_ref" => input_ref,
+        "current_input_record_id" => input_record.record_id
+      }
+    ).with(
+      execution_revision: 0,
+      working_records: [input_record]
+    )
+
+    [
+      Phronomy::Agent::ContextAssembler.new(agent: agent, persistence: persistence),
+      root,
+      execution
+    ]
   end
+end
+
+puts "Manifest-first Context benchmark"
+puts "Ruby #{RUBY_VERSION} on #{RUBY_PLATFORM}"
+puts "=" * 72
 
-  x.report("build(1000 msgs, 10 chunks, no budget)") do
-    (BENCH_ASM_ITERATIONS / 10).times { make_assembler(n_messages: 1000, n_chunks: 10).build }
+policy = Phronomy::Agent::ContextPolicies::Default.new
+policy_requests = [10, 100, 1_000].to_h do |count|
+  [count, BenchContextAssembler.request(count)]
+end
+
+Benchmark.bm(46) do |x|
+  policy_requests.each do |count, request|
+    iterations = (count >= 1_000) ? 200 : 1_000
+    x.report("DefaultContextPolicy #{count} candidates x#{iterations}") do
+      iterations.times { policy.call(request) }
+    end
   end
 
-  x.report("build(1000 msgs, 10 chunks, budgeted)") do
-    (BENCH_ASM_ITERATIONS / 10).times do
-      # Assembler raises ContextLengthError when messages exceed the budget;
-      # callers (e.g. Agent::Base#build_context) are expected to pre-trim via
-      # trim_to_budget before calling build. The rescue here keeps the benchmark
-      # measuring build's fast path without triggering the error path.
-      make_assembler(n_messages: 1000, n_chunks: 10, with_budget: true).build
-    rescue Phronomy::ContextLengthError
-      # expected — budget exceeded
+  assembler, root, execution = BenchContextAssembler.assembler_fixture
+  x.report("ContextAssembler#build_initial x500") do
+    500.times do
+      assembler.build_initial(
+        input: "benchmark input",
+        agent_root: root,
+        execution: execution
+      )
     end
   end
 end

data/benchmark/bench_regression.rb

@@ -2,12 +2,19 @@
 
 # bench_regression.rb — Targeted regression benchmarks.
 #
-# Measures the five minimum regression targets defined in Issue #232:
+# Measures the six minimum regression targets:
 #   1. WorkflowContext#merge throughput
 #   2. Workflow.define (graph build) time
 #   3. Tool::Base#params_schema generation (10 params)
 #   4. Orchestrator#dispatch_parallel overhead (10 stub agents, no LLM)
-#   5. CancellationToken#cancelled? throughput (shared token, 8 threads)
+#   5. CancellationToken#cancelled? throughput under deliberate Thread contention
+#   6. CancellationToken#raise_if_cancelled! hot path
+#
+# Target 4 deliberately uses thread-free completion handles so the benchmark
+# measures EventLoop/FanOut coordination rather than fake per-child Thread.new
+# overhead. Target 5 intentionally uses Threads because cross-thread token access
+# is the behavior being measured there; it is not a Phronomy runtime execution
+# path.
 #
 # Results are stored in a global REGRESSION_RESULTS hash (keyed by metric name,
 # value = iterations per second) for use by run_all.rb baseline comparison.
@@ -89,10 +96,20 @@ end
 # ---------------------------------------------------------------------------
 stub_agent_class = Class.new(Phronomy::Agent::Base) do
   agent_definition id: "bench-stub", version: 1
-  define_method(:invoke) do |_input, messages: [], thread_id: nil, config: {}|
+
+  define_method(:invoke) do |_input, thread_id: nil, config: {}|
     {output: "stub", messages: []}
   end
-  define_method(:invoke_async) { |input, **_kw| Phronomy::Runtime.instance.spawn(name: "bench-stub") { invoke(input) } }
+
+  define_method(:invoke_async) do |input, **_kw|
+    task = Phronomy::Task.deferred(name: "bench-stub")
+    begin
+      task.complete(invoke(input))
+    rescue => error
+      task.fail(error)
+    end
+    task
+  end
 end
 
 orchestrator_class = Class.new(Phronomy::MultiAgent::Orchestrator)
@@ -108,8 +125,11 @@ t4 = Benchmark.measure("Orchestrator#dispatch_parallel (10 agents)") do
 end
 
 # ---------------------------------------------------------------------------
-# Target 5: CancellationToken#cancelled? throughput (8 threads)
+# Target 5: CancellationToken#cancelled? throughput (8 application Threads)
 # ---------------------------------------------------------------------------
+# Threads are intentional here: the benchmark specifically measures concurrent
+# access to one shared CancellationToken. They are not used to execute Phronomy
+# Agent/Workflow/Tool lifecycle work.
 CANCEL_TOKEN = Phronomy::Concurrency::CancellationToken.new
 CANCEL_ITERATIONS = 10_000
 
@@ -123,7 +143,7 @@ end
 # ---------------------------------------------------------------------------
 # Target 6: CancellationToken#raise_if_cancelled! hot path (no-op, single thread)
 # ---------------------------------------------------------------------------
-RAISE_TOKEN = Phronomy::Concurrency::CancellationToken.new  # not cancelled — no-op path
+RAISE_TOKEN = Phronomy::Concurrency::CancellationToken.new # not cancelled — no-op path
 RAISE_ITERATIONS = 200_000
 
 t6 = Benchmark.measure("CancellationToken#raise_if_cancelled! (no-op)") do

data/benchmark/bench_tool_schema.rb

@@ -1,16 +1,12 @@
 # frozen_string_literal: true
 
-# Benchmark: Tool::Base params_schema generation and static_knowledge_chunks cache
+# Benchmark: Tool::Base params_schema generation.
 #
-# Tool schema generation happens once per tool class (lazily memoised).
-# static_knowledge_chunks is cached at the class level; cache-hit overhead
-# should be negligible compared to cache-miss (which calls the knowledge source).
+# Tool schema generation happens once per tool class and is lazily memoized.
 
 require "benchmark"
 require_relative "../lib/phronomy"
 
-# --- Tool schema ---
-
 class BenchTool10Params < Phronomy::Agent::Context::Capability::Base
   description "A tool with 10 parameters for benchmarking purposes"
   param :param1, type: :string, desc: "First parameter"
@@ -29,7 +25,6 @@ class BenchTool10Params < Phronomy::Agent::Context::Capability::Base
   end
 end
 
-# Warm up memoisation
 BenchTool10Params.params_schema_definition
 
 BENCH_TOOL_ITERATIONS = 50_000
@@ -40,31 +35,3 @@ Benchmark.bm(35) do |x|
     BENCH_TOOL_ITERATIONS.times { BenchTool10Params.params_schema_definition }
   end
 end
-
-# --- static_knowledge_chunks cache ---
-
-class BenchKnowledgeSource < Phronomy::Agent::Context::Knowledge::Base
-  def fetch(query: nil)
-    [{content: "Cached knowledge fact.", type: :static}]
-  end
-
-  def static?
-    true
-  end
-end
-
-class BenchAgentWithKnowledge < Phronomy::Agent::Base
-  agent_definition id: "bench-knowledge", version: 1
-  model "gpt-4o-mini"
-  static_knowledge BenchKnowledgeSource.new
-end
-
-# Warm up cache
-BenchAgentWithKnowledge.static_knowledge_chunks
-
-puts "\n=== bench_static_knowledge_cache ==="
-Benchmark.bm(35) do |x|
-  x.report("static_knowledge_chunks (hit)") do
-    BENCH_TOOL_ITERATIONS.times { BenchAgentWithKnowledge.static_knowledge_chunks }
-  end
-end