phronomy 0.16.0 → 0.18.0

data/lib/phronomy/agent/context_assembler.rb

@@ -3,9 +3,12 @@
 module Phronomy
   module Agent
     class ContextAssembler
-      ASSEMBLY_POLICY_VERSION = 5
+      ASSEMBLY_POLICY_VERSION = 6
       SEGMENT_ORIGIN_METADATA_KEY = "phronomy_origin"
       BEFORE_LLM_INPUT_ORIGIN = "before_llm_input"
+      EARLY_CONTEXT_CATEGORIES = %i[
+        instruction structured_state knowledge memory summary
+      ].freeze
 
       def initialize(
         agent:,
@@ -45,14 +48,13 @@ module Phronomy
           model_config: model_cfg,
           tool_definitions: tool_set.definitions,
           tool_definitions_ref: nil,
-          prior_records: projection.transcript_records,
+          prior_records: projection.context_records,
           working_records: execution.working_records,
           excluded_record_ids: excluded,
           base_segments: base_segments,
           hook_candidates: hook_candidates,
           current_input_segment: segment(:current_input, :user, input_ref, :ask_argument),
-          mandatory_values: [system_text, current_input, tool_set.definitions] +
-            hook_candidates.map { |candidate| candidate.fetch(:content) }
+          mandatory_values: [system_text, current_input, tool_set.definitions]
         )
       end
 
@@ -81,14 +83,13 @@ module Phronomy
           model_config: model_cfg,
           tool_definitions: tool_definitions,
           tool_definitions_ref: base_manifest.tool_definitions_ref,
-          prior_records: projection.transcript_records,
+          prior_records: projection.context_records,
           working_records: execution.working_records,
           excluded_record_ids: [],
           base_segments: system_segments.map { |existing| segment_hash(existing) },
           hook_candidates: hook_candidates,
           current_input_segment: nil,
           mandatory_values: system_segments.map { |segment| fetch_content(segment.content_ref) } +
-            hook_candidates.map { |candidate| candidate.fetch(:content) } +
             [tool_definitions]
         )
       end
@@ -121,6 +122,14 @@ module Phronomy
           working_records: eligible_working,
           excluded_record_ids: excluded_record_ids
         )
+        candidates = merge_hook_candidates(
+          candidates,
+          hook_candidates,
+          agent_root: agent_root,
+          execution: execution,
+          call_sequence: call_sequence
+        )
+
         token_budget = TokenBudgetResolver.new(agent: @agent).resolve(model_config)
         mandatory_token_estimate = estimate_values(mandatory_values)
         parts = context_parts
@@ -144,12 +153,12 @@ module Phronomy
             "Derived Context persistence is not enabled in Context Policy phase 1-4"
         end
 
+        selected_candidates = validated.selected_candidates.sort_by do |candidate|
+          [candidate.sequence || 0, candidate.candidate_id]
+        end
         segments = Array(base_segments).dup
-        append_candidates(segments, hook_candidates, before_history: true)
-        validated.selected_candidates
-          .sort_by { |candidate| [candidate.sequence || 0, candidate.candidate_id] }
-          .each { |candidate| segments << segment_from_candidate(candidate) }
-        append_candidates(segments, hook_candidates, before_history: false)
+        append_selected_candidates(segments, selected_candidates, before_history: true)
+        append_selected_candidates(segments, selected_candidates, before_history: false)
         segments << current_input_segment if current_input_segment
 
         ContextParts::Validators::FinalBudgetValidator.new(
@@ -231,20 +240,55 @@ module Phronomy
           content = hash.fetch(:content) { hash.fetch("content") }
           category = (hash[:category] || hash["category"] || :knowledge).to_sym
           role = (hash[:role] || hash["role"] || default_role(category)).to_sym
-          {content: content.to_s, category: category, role: role}
+          metadata = hash[:metadata] || hash["metadata"] || {}
+          {
+            content: content.to_s,
+            category: category,
+            role: role,
+            metadata: metadata.to_h.transform_keys(&:to_s)
+          }
+        end
+      end
+
+      def merge_hook_candidates(candidates, hooks, agent_root:, execution:, call_sequence:)
+        next_sequence = Array(candidates).filter_map(&:sequence).max.to_i
+        generated = hooks.each_with_index.map do |hook, index|
+          content_ref = @persistence.contents.put_text(hook.fetch(:content))
+          metadata = hook.fetch(:metadata).merge(
+            SEGMENT_ORIGIN_METADATA_KEY => BEFORE_LLM_INPUT_ORIGIN,
+            "estimated_tokens" => Phronomy::LlmContextWindow::TokenEstimator.estimate(
+              fetch_content(content_ref)
+            ),
+            "source_kind" => "hook"
+          )
+          ContextCandidate.new(
+            candidate_id: "hook:#{execution.execution_id}:#{call_sequence}:#{index}",
+            source_kind: :hook,
+            category: hook.fetch(:category),
+            role: hook.fetch(:role),
+            content_ref: content_ref,
+            record_id: nil,
+            agent_id: agent_root.agent_id,
+            execution_id: execution.execution_id,
+            llm_call_id: nil,
+            tool_call_id: nil,
+            sequence: next_sequence + index + 1,
+            requirement: :optional,
+            priority: 0,
+            metadata: metadata
+          )
         end
+        (Array(candidates) + generated)
+          .sort_by { |candidate| [candidate.sequence || 0, candidate.candidate_id] }
+          .freeze
       end
 
-      def append_candidates(segments, candidates, before_history:)
+      def append_selected_candidates(segments, candidates, before_history:)
         candidates.each do |candidate|
-          early = %i[instruction structured_state knowledge memory summary].include?(candidate[:category])
+          early = EARLY_CONTEXT_CATEGORIES.include?(candidate.category)
           next unless early == before_history
-          segments << text_segment(
-            candidate[:category],
-            candidate[:role],
-            candidate[:content],
-            metadata: {SEGMENT_ORIGIN_METADATA_KEY => BEFORE_LLM_INPUT_ORIGIN}
-          )
+
+          segments << segment_from_candidate(candidate)
         end
       end
 
@@ -306,15 +350,7 @@ module Phronomy
       end
 
       def build_system_text(input)
-        instruction = @agent.send(:build_instructions, input)
-        knowledge = @agent.class.static_knowledge_chunks + @agent.send(:instance_knowledge_chunks)
-        parts = [instruction]
-        knowledge.each do |chunk|
-          parts << Phronomy::LlmContextWindow::Assembler.xml_tag(
-            chunk[:content], type: chunk[:type] || :static, trusted: true
-          )
-        end
-        parts.compact.join("\n\n")
+        @agent.send(:build_instructions, input)
       end
     end
   end

data/lib/phronomy/agent/context_parts/unit_builders/dependency_aware_unit_builder.rb

@@ -8,33 +8,29 @@ module Phronomy
       module UnitBuilders
         class DependencyAwareUnitBuilder
           def build(candidates)
-            ordered = Array(candidates).sort_by { |candidate| [candidate.sequence || 0, candidate.candidate_id] }
-            by_id = ordered.to_h { |candidate| [candidate.candidate_id, candidate] }
+            ordered = Array(candidates).sort_by do |candidate|
+              [candidate.sequence || 0, candidate.candidate_id]
+            end
 
-            canonical_assistants = ordered.select { |candidate| candidate.category == :assistant_message }
-            canonical_tool_messages = ordered.select { |candidate| candidate.category == :tool_message }
-              .group_by(&:tool_call_id)
+            canonical_assistants = ordered.select do |candidate|
+              candidate.category == :assistant_message
+            end
+            canonical_tool_messages = ordered.select do |candidate|
+              candidate.category == :tool_message
+            end.group_by(&:tool_call_id)
             assistant_by_tool_call_id = canonical_assistants.each_with_object({}) do |candidate, result|
               canonical_tool_call_ids(candidate).each do |tool_call_id|
                 if result.key?(tool_call_id)
-                  raise ArgumentError, "duplicate assistant Tool Call id in Context candidates: #{tool_call_id}"
+                  raise ArgumentError,
+                    "duplicate assistant Tool Call id in Context candidates: #{tool_call_id}"
                 end
                 result[tool_call_id] = candidate
               end
             end
 
-            legacy_tool_results = ordered.select { |candidate| candidate.category == :tool_result }
-              .group_by(&:tool_call_id)
-            legacy_calls_by_llm = ordered.select do |candidate|
-              candidate.category == :tool_call && candidate.llm_call_id
-            end.group_by(&:llm_call_id)
-            legacy_assistant_by_llm = ordered.select do |candidate|
-              candidate.llm_call_id && %i[llm_message tool_call].include?(candidate.category)
-            end.group_by(&:llm_call_id)
-
             claimed = Set.new
             units = []
-            ordered.each_with_index do |candidate, index|
+            ordered.each do |candidate|
               next if claimed.include?(candidate.candidate_id)
 
               group = canonical_tool_exchange(
@@ -42,23 +38,13 @@ module Phronomy
                 assistant_by_tool_call_id: assistant_by_tool_call_id,
                 tool_messages: canonical_tool_messages
               )
-              group ||= if legacy_runtime_tool_exchange?(candidate, legacy_calls_by_llm)
-                legacy_runtime_tool_exchange(
-                  candidate,
-                  assistant_by_llm: legacy_assistant_by_llm,
-                  calls_by_llm: legacy_calls_by_llm,
-                  tool_results: legacy_tool_results
-                )
-              else
-                legacy_import_tool_exchange(ordered, index, legacy_tool_results)
-              end
 
               if group && !group.empty?
                 group.each { |item| claimed << item.candidate_id }
                 units << build_unit(group, kind: :tool_exchange)
               else
                 claimed << candidate.candidate_id
-                units << build_unit([by_id.fetch(candidate.candidate_id)], kind: :message)
+                units << build_unit([candidate], kind: :message)
               end
             end
 
@@ -78,8 +64,12 @@ module Phronomy
             call_ids = canonical_tool_call_ids(assistant)
             return if call_ids.empty?
 
-            messages = call_ids.flat_map { |tool_call_id| tool_messages.fetch(tool_call_id, []) }
-            ([assistant] + messages).uniq.sort_by { |item| [item.sequence || 0, item.candidate_id] }
+            messages = call_ids.flat_map do |tool_call_id|
+              tool_messages.fetch(tool_call_id, [])
+            end
+            ([assistant] + messages).uniq.sort_by do |item|
+              [item.sequence || 0, item.candidate_id]
+            end
           end
 
           def canonical_tool_call_ids(candidate)
@@ -88,66 +78,6 @@ module Phronomy
               .map(&:to_s)
           end
 
-          def legacy_runtime_tool_exchange?(candidate, calls_by_llm)
-            candidate.llm_call_id &&
-              calls_by_llm.key?(candidate.llm_call_id) &&
-              %i[llm_message tool_call].include?(candidate.category)
-          end
-
-          def legacy_runtime_tool_exchange(candidate, assistant_by_llm:, calls_by_llm:, tool_results:)
-            llm_call_id = candidate.llm_call_id
-            calls = calls_by_llm.fetch(llm_call_id)
-            call_ids = calls.map(&:tool_call_id).compact
-            assistant = assistant_by_llm.fetch(llm_call_id, [])
-            results = call_ids.flat_map { |id| tool_results.fetch(id, []) }
-            (assistant + results).uniq.sort_by { |item| [item.sequence || 0, item.candidate_id] }
-          end
-
-          def legacy_import_tool_exchange(ordered, index, tool_results)
-            candidate = ordered.fetch(index)
-            return unless %i[llm_message tool_call].include?(candidate.category)
-            return if candidate.llm_call_id
-
-            group = []
-            cursor = index
-            if candidate.category == :llm_message
-              next_candidate = ordered[cursor + 1]
-              return unless next_candidate &&
-                next_candidate.category == :tool_call &&
-                contiguous_source?(candidate, next_candidate) &&
-                next_candidate.llm_call_id.nil?
-              group << candidate
-              cursor += 1
-            end
-
-            calls = []
-            while (item = ordered[cursor]) && item.category == :tool_call && item.llm_call_id.nil?
-              if calls.any? && !contiguous_source?(calls.last, item)
-                break
-              end
-              if group.any? && calls.empty? && !contiguous_source?(group.last, item)
-                break
-              end
-              calls << item
-              cursor += 1
-            end
-            return if calls.empty?
-
-            call_ids = calls.map(&:tool_call_id).compact
-            results = call_ids.flat_map { |id| tool_results.fetch(id, []) }
-            (group + calls + results).uniq.sort_by { |item| [item.sequence || 0, item.candidate_id] }
-          end
-
-          def contiguous_source?(left, right)
-            left_source = left.metadata["source_sequence"]
-            right_source = right.metadata["source_sequence"]
-            if left_source && right_source
-              Integer(right_source) == Integer(left_source) + 1
-            else
-              Integer(right.sequence) == Integer(left.sequence) + 1
-            end
-          end
-
           def build_unit(candidates, kind:)
             ids = candidates.map(&:candidate_id)
             sequences = candidates.map(&:sequence).compact

data/lib/phronomy/agent/context_plan_validator.rb

@@ -58,7 +58,6 @@ module Phronomy
       def validate_tool_dependencies!(all_candidates, selected_candidates)
         selected_ids = selected_candidates.to_h { |candidate| [candidate.candidate_id, true] }
         validate_canonical_tool_dependencies!(all_candidates, selected_ids)
-        validate_legacy_tool_dependencies!(all_candidates, selected_candidates, selected_ids)
       end
 
       def validate_canonical_tool_dependencies!(all_candidates, selected_ids)
@@ -113,38 +112,6 @@ module Phronomy
         end
       end
 
-      # Compatibility for Journal state written by the preceding stateful
-      # refactor, where assistant content, Tool Calls and Tool Results were
-      # flattened into separate Context candidates.
-      def validate_legacy_tool_dependencies!(all_candidates, selected_candidates, selected_ids)
-        all_calls = Array(all_candidates).select { |candidate| candidate.category == :tool_call }
-          .to_h { |candidate| [candidate.tool_call_id, candidate] }
-        all_results = Array(all_candidates).select { |candidate| candidate.category == :tool_result }
-          .group_by(&:tool_call_id)
-
-        selected_candidates.each do |candidate|
-          case candidate.category
-          when :tool_result
-            call = all_calls[candidate.tool_call_id]
-            unless call && selected_ids[call.candidate_id]
-              raise ArgumentError,
-                "ContextPlan selected orphan Tool Result: #{candidate.tool_call_id}"
-            end
-          when :tool_call
-            results = all_results.fetch(candidate.tool_call_id, [])
-            if results.empty?
-              raise ArgumentError,
-                "ContextPlan selected Tool Call without a Tool Result: #{candidate.tool_call_id}"
-            end
-            missing = results.reject { |result| selected_ids[result.candidate_id] }
-            unless missing.empty?
-              raise ArgumentError,
-                "ContextPlan split Tool Call/Result dependency: #{candidate.tool_call_id}"
-            end
-          end
-        end
-      end
-
       def canonical_tool_call_ids(candidate)
         Array(candidate.metadata["tool_call_ids"] || candidate.metadata[:tool_call_ids])
           .compact

data/lib/phronomy/agent/execution_coordinator.rb

@@ -19,7 +19,7 @@ module Phronomy
       )
         result_task = Phronomy::Task.deferred(name: "agent-#{@agent.agent_id}-#{mode}")
         runtime = Phronomy::Runtime.instance
-        preparation = runtime.blocking_io.submit do
+        preparation = runtime.offload.submit(on_full: :raise) do
           prepare(input, thread_id: thread_id, config: config)
         end
         preparation.on_complete do |prepared, error|
@@ -52,7 +52,7 @@ module Phronomy
           name: "agent-approval-resume:#{execution_id}"
         )
         runtime = Phronomy::Runtime.instance
-        preparation = runtime.blocking_io.submit do
+        preparation = runtime.offload.submit(on_full: :raise) do
           execution = @agent.persistence.executions.load(execution_id)
           unless execution.agent_id == @agent.agent_id && execution.status == :suspended
             raise ArgumentError,
@@ -86,7 +86,7 @@ module Phronomy
               config: config
             )
           rescue => start_error
-            commit = runtime.blocking_io.submit do
+            commit = runtime.offload.submit(on_full: :raise) do
               commit_failed(activation, activation.invocation, start_error)
             end
             commit.on_complete do |terminal, commit_error|
@@ -107,7 +107,7 @@ module Phronomy
 
       # Persists all Runtime events from the previous call, fixes the next
       # Canonical LLM Input Manifest, and materializes the next Runtime Projection.
-      # This method is executed on the blocking adapter pool, never on EventLoop.
+      # This method is executed on an OffloadPool worker, never on EventLoop.
       def prepare_next_llm_call(activation)
         snapshot = activation.runtime_snapshot
         manifest = manifest_ref = updated = root = nil
@@ -367,7 +367,6 @@ module Phronomy
         session = Agent::AgentInvocationSessionBuilder.build(
           agent: @agent,
           input: prepared.filtered_input,
-          messages: prepared.runtime_projection.messages,
           config: effective_config,
           approval_policy: approval_policy,
           approval_listener: approval_listener,
@@ -398,7 +397,7 @@ module Phronomy
 
       def finish(activation, result_task, invocation, error)
         runtime = Phronomy::Runtime.instance
-        operation = runtime.blocking_io.submit do
+        operation = runtime.offload.submit(on_full: :raise) do
           compute_terminal(activation, invocation, error)
         end
         operation.on_complete do |outcome, commit_error|
@@ -1070,7 +1069,7 @@ module Phronomy
       def dispatch_approval_listener(invocation, request)
         listener = invocation.approval_listener
         return unless listener
-        Phronomy::Runtime.instance.blocking_io.submit { listener.call(request) }
+        Phronomy::Runtime.instance.offload.submit(on_full: :raise) { listener.call(request) }
       end
 
       def json_value(value)

data/lib/phronomy/agent/journal_projection.rb

@@ -4,9 +4,11 @@ module Phronomy
   module Agent
     class JournalProjection
       MESSAGE_KINDS = %i[
-        external_message assistant_message tool_message
-        llm_message tool_call tool_result
+        external_message
+        assistant_message
+        tool_message
       ].freeze
+      KNOWLEDGE_RESET_KINDS = %i[knowledge_cleared context_reset].freeze
 
       def initialize(persistence:, agent_root:)
         @persistence = persistence
@@ -22,6 +24,16 @@ module Phronomy
         end
       end
 
+      # Returns all persistent records eligible for Context selection.
+      # Knowledge has an independent logical lifetime from the transcript:
+      # clear_transcript! does not remove Knowledge, while knowledge_cleared and
+      # context_reset invalidate earlier Knowledge records by Journal position.
+      def context_records
+        (transcript_records + active_knowledge_records)
+          .sort_by { |record| [record.sequence || 0, record.record_id] }
+          .freeze
+      end
+
       def llm_call_records
         records.select { |record| record.kind == :llm_call_recorded }
       end
@@ -29,6 +41,20 @@ module Phronomy
       def records
         @records ||= @persistence.journals.read(@agent_root.agent_id)
       end
+
+      private
+
+      def active_knowledge_records
+        reset_sequence = records.filter_map do |record|
+          record.sequence if KNOWLEDGE_RESET_KINDS.include?(record.kind)
+        end.max.to_i
+
+        records.select do |record|
+          record.context_candidate &&
+            record.kind == :knowledge &&
+            record.sequence.to_i > reset_sequence
+        end
+      end
     end
   end
 end

data/lib/phronomy/agent/ruby_llm_materializer.rb

@@ -112,102 +112,7 @@ module Phronomy
       end
 
       def materialize_message_segments(segments)
-        result = []
-        index = 0
-        while index < segments.length
-          segment = segments[index]
-          if legacy_assistant_segment?(segment)
-            grouped, consumed = legacy_assistant_group(segments, index)
-            result << materialize_legacy_assistant_group(grouped)
-            index += consumed
-          else
-            result << materialize_segment(segment)
-            index += 1
-          end
-        end
-        result
-      end
-
-      # New canonical records preserve one logical assistant/tool message per
-      # record. These legacy helpers remain only so already-persisted state from
-      # the preceding stateful refactor can still be read during migration.
-      def legacy_assistant_segment?(segment)
-        segment.role&.to_sym == :assistant &&
-          %i[llm_message tool_call].include?(segment.category.to_sym)
-      end
-
-      def legacy_assistant_group(segments, start_index)
-        first = segments.fetch(start_index)
-        llm_call_id = metadata_value(first, "llm_call_id")
-        return legacy_runtime_assistant_group(segments, start_index, llm_call_id) if llm_call_id
-
-        legacy_import_assistant_group(segments, start_index)
-      end
-
-      def legacy_runtime_assistant_group(segments, start_index, llm_call_id)
-        group = []
-        index = start_index
-        while (segment = segments[index]) && legacy_assistant_segment?(segment) &&
-            metadata_value(segment, "llm_call_id").to_s == llm_call_id.to_s
-          group << segment
-          index += 1
-        end
-        [group, group.length]
-      end
-
-      def legacy_import_assistant_group(segments, start_index)
-        first = segments.fetch(start_index)
-        group = [first]
-        index = start_index + 1
-        last = first
-
-        while (segment = segments[index]) && legacy_assistant_segment?(segment)
-          break unless segment.category.to_sym == :tool_call
-          break unless contiguous_source?(last, segment)
-          break if metadata_value(segment, "llm_call_id")
-
-          group << segment
-          last = segment
-          index += 1
-        end
-        [group, group.length]
-      end
-
-      def contiguous_source?(left, right)
-        left_sequence = metadata_value(left, "journal_sequence")
-        right_sequence = metadata_value(right, "journal_sequence")
-        return false unless left_sequence && right_sequence
-
-        Integer(right_sequence) == Integer(left_sequence) + 1
-      end
-
-      def materialize_legacy_assistant_group(segments)
-        content_segments = segments.select { |segment| segment.category.to_sym == :llm_message }
-        if content_segments.length > 1
-          raise ArgumentError, "ambiguous assistant message group contains multiple content records"
-        end
-
-        tool_calls = {}
-        segments.select { |segment| segment.category.to_sym == :tool_call }.each do |segment|
-          payload = fetch_json(segment.content_ref)
-          tool_call_id = payload.fetch("id").to_s
-          if tool_calls.key?(tool_call_id)
-            raise ArgumentError, "duplicate Tool Call in assistant message: #{tool_call_id}"
-          end
-          tool_calls[tool_call_id] = materialize_tool_call(payload)
-        end
-
-        content = if content_segments.empty?
-          ""
-        else
-          @persistence.contents.fetch_text(content_segments.first.content_ref)
-        end
-
-        RubyLLM::Message.new(
-          role: :assistant,
-          content: content,
-          tool_calls: tool_calls.empty? ? nil : tool_calls
-        )
+        segments.map { |segment| materialize_segment(segment) }
       end
 
       def materialize_segment(segment)
@@ -216,18 +121,8 @@ module Phronomy
           materialize_canonical_message(segment, expected_role: :assistant)
         when :tool_message
           materialize_canonical_message(segment, expected_role: :tool)
-        when :tool_call
-          materialize_legacy_assistant_group([segment])
         when :tool_result
-          unless segment.role&.to_sym == :tool
-            raise ArgumentError, "raw Tool result is not an LLM message"
-          end
-          RubyLLM::Message.new(
-            role: :tool,
-            content: @persistence.contents.fetch_text(segment.content_ref),
-            tool_call_id: segment.tool_call_id ||
-              segment.metadata["tool_call_id"] || segment.metadata[:tool_call_id]
-          )
+          raise ArgumentError, "raw Tool result is not an LLM message"
         else
           RubyLLM::Message.new(
             role: segment.role,
@@ -289,10 +184,6 @@ module Phronomy
           thought_signature: payload["thought_signature"]
         )
       end
-
-      def metadata_value(segment, name)
-        segment.metadata[name] || segment.metadata[name.to_sym]
-      end
     end
   end
 end