phronomy 0.22.0 → 0.23.0

data/lib/phronomy/agent/context_candidate_resolver.rb

@@ -43,7 +43,7 @@ module Phronomy
           "source_sequence" => record.sequence
         )
 
-        ContextCandidate.new(
+        Selection::Candidate.new(
           candidate_id: "record:#{record.record_id}",
           source_kind: source_kind,
           category: record.kind,
@@ -55,7 +55,7 @@ module Phronomy
           llm_call_id: record.llm_call_id,
           tool_call_id: tool_call_id,
           sequence: sequence,
-          requirement: :optional,
+          constraint: Selection::Constraint.selectable(origin: :context_policy),
           priority: (source_kind == :working) ? 100 : 0,
           metadata: metadata
         )

data/lib/phronomy/agent/context_plan.rb

@@ -3,21 +3,26 @@
 module Phronomy
   module Agent
     ContextPlan = Data.define(
-      :selected_unit_ids,
-      :derived_contents,
-      :selected_tool_ids,
-      :ordering_hints,
-      :policy_descriptor,
+      :instruction,
+      :knowledge,
+      :tools,
+      :conversation,
       :metadata
     ) do
-      def initialize(**values)
-        super(**values.merge(
-          selected_unit_ids: Array(values[:selected_unit_ids]).map(&:to_s).freeze,
-          derived_contents: Array(values[:derived_contents]).freeze,
-          selected_tool_ids: Array(values[:selected_tool_ids]).map(&:to_s).freeze,
-          ordering_hints: Immutable.copy(values[:ordering_hints] || {}),
-          metadata: Immutable.copy(values[:metadata] || {})
-        ))
+      def initialize(
+        instruction: [],
+        knowledge: [],
+        tools: [],
+        conversation: [],
+        metadata: {}
+      )
+        super(
+          instruction: Array(instruction).freeze,
+          knowledge: Array(knowledge).freeze,
+          tools: Array(tools).freeze,
+          conversation: Array(conversation).map { |group| Array(group).freeze }.freeze,
+          metadata: Immutable.copy(metadata || {})
+        )
         freeze
       end
     end

data/lib/phronomy/agent/context_plan_validator.rb

@@ -3,131 +3,289 @@
 module Phronomy
   module Agent
     class ContextPlanValidator
-      ValidatedContextPlan = Data.define(
-        :plan, :units, :selected_units, :selected_candidates
-      ) do
-        def initialize(**values)
-          super(**values.merge(
-            units: Array(values[:units]).freeze,
-            selected_units: Array(values[:selected_units]).freeze,
-            selected_candidates: Array(values[:selected_candidates]).freeze
-          ))
-          freeze
-        end
-      end
+      RESERVED_KIND_CATEGORIES = {
+        instruction: :instruction,
+        handoff_responsibility: :instruction,
+        knowledge: :knowledge,
+        memory: :knowledge,
+        summary: :knowledge,
+        structured_state: :knowledge,
+        external_message: :conversation,
+        assistant_message: :conversation,
+        tool_message: :conversation,
+        current_input: :conversation,
+        current_request: :conversation,
+        conversation: :conversation,
+        tool_result: :conversation
+      }.freeze
 
-      def validate!(request:, plan:)
+      def validate!(input:, plan:)
+        unless input.is_a?(ContextPolicyInput)
+          raise ArgumentError, "ContextPlanValidator expected ContextPolicyInput"
+        end
         unless plan.is_a?(ContextPlan)
           raise ArgumentError, "Context Policy returned #{plan.class}, expected #{ContextPlan}"
         end
 
-        units = request.parts.fetch(:unit_builder).build(request.candidates)
-        units = request.parts.fetch(:required_context_resolver).resolve(
-          request: request,
-          units: units
+        validate_flat_category!(
+          input: input.instruction,
+          output: plan.instruction,
+          item_class: ContextPolicyInput::InstructionItem,
+          label: :instruction,
+          generated_allowed: true
         )
-        unit_index = units.to_h { |unit| [unit.unit_id, unit] }
-        unknown = plan.selected_unit_ids.reject { |unit_id| unit_index.key?(unit_id) }
-        raise ArgumentError, "ContextPlan contains unknown units: #{unknown.inspect}" unless unknown.empty?
-
-        selected_units = plan.selected_unit_ids.map { |unit_id| unit_index.fetch(unit_id) }
-        required = units.reject { |unit| unit.requirement == :optional }.map(&:unit_id)
-        missing_required = required - plan.selected_unit_ids
-        unless missing_required.empty?
-          raise Phronomy::ContextBudgetExceededError,
-            "ContextPlan omitted required units: #{missing_required.inspect}"
-        end
-
-        candidate_index = request.candidates.to_h { |candidate| [candidate.candidate_id, candidate] }
-        selected_candidates = selected_units.flat_map(&:candidate_ids).uniq.map do |candidate_id|
-          candidate_index.fetch(candidate_id)
-        end
-        validate_tool_dependencies!(request.candidates, selected_candidates)
-        validate_derived_contents!(request, plan)
-
-        ValidatedContextPlan.new(
-          plan: plan,
-          units: units,
-          selected_units: selected_units,
-          selected_candidates: selected_candidates
+        validate_flat_category!(
+          input: input.knowledge,
+          output: plan.knowledge,
+          item_class: ContextPolicyInput::KnowledgeItem,
+          label: :knowledge,
+          generated_allowed: true
+        )
+        validate_flat_category!(
+          input: input.tools,
+          output: plan.tools,
+          item_class: ContextPolicyInput::ToolItem,
+          label: :tools,
+          generated_allowed: false
         )
+        validate_conversation!(input.conversation, plan.conversation)
+        validate_unique_ids!(plan)
+        plan
       end
 
       private
 
-      def validate_tool_dependencies!(all_candidates, selected_candidates)
-        selected_ids = selected_candidates.to_h { |candidate| [candidate.candidate_id, true] }
-        validate_canonical_tool_dependencies!(all_candidates, selected_ids)
-      end
+      def validate_flat_category!(input:, output:, item_class:, label:, generated_allowed:)
+        input_by_id = input.to_h { |item| [item.id, item] }
+        output.each do |item|
+          unless item.is_a?(item_class)
+            raise ArgumentError,
+              "ContextPlan #{label} contains #{item.class}; expected #{item_class}"
+          end
 
-      def validate_canonical_tool_dependencies!(all_candidates, selected_ids)
-        assistants = Array(all_candidates).select { |candidate| candidate.category == :assistant_message }
-        tool_messages = Array(all_candidates).select { |candidate| candidate.category == :tool_message }
-          .group_by(&:tool_call_id)
-        assistant_by_tool_call_id = {}
+          validate_semantic_kind!(item, label)
 
-        tool_messages.each do |tool_call_id, messages|
-          if messages.length > 1
-            raise ArgumentError, "duplicate Tool message in Context candidates: #{tool_call_id}"
+          source = input_by_id[item.id]
+          if source
+            unless source == item
+              raise ArgumentError, "ContextPlan modified input item #{item.id.inspect} in #{label}"
+            end
+          elsif !generated_allowed || item.provenance.origin != :policy_generated
+            raise ArgumentError, "ContextPlan contains unknown #{label} item: #{item.id.inspect}"
+          else
+            validate_policy_generated_metadata!(item)
           end
         end
 
-        assistants.each do |assistant|
-          canonical_tool_call_ids(assistant).each do |tool_call_id|
-            if assistant_by_tool_call_id.key?(tool_call_id)
-              raise ArgumentError,
-                "duplicate assistant Tool Call id in Context candidates: #{tool_call_id}"
+        missing = input.select(&:required?).reject do |required|
+          output.any? { |item| item.id == required.id }
+        end
+        return if missing.empty?
+
+        raise Phronomy::ContextBudgetExceededError,
+          "ContextPlan omitted required #{label} item(s): #{missing.map(&:id).inspect}"
+      end
+
+      def validate_conversation!(input_groups, output_groups)
+        input_by_group_ids = input_groups.to_h do |group|
+          [group.map(&:id), group]
+        end
+        selected_input_ids = {}
+
+        output_groups.each do |group|
+          unless group.is_a?(Array) && !group.empty?
+            raise ArgumentError, "ContextPlan conversation groups must be non-empty Arrays"
+          end
+          unless group.all? { |item| item.is_a?(ContextPolicyInput::ConversationItem) }
+            raise ArgumentError, "ContextPlan conversation group contains a non-ConversationItem"
+          end
+
+          group.each { |item| validate_semantic_kind!(item, :conversation) }
+
+          ids = group.map(&:id)
+          matching = input_by_group_ids[ids]
+          if matching
+            unless matching == group
+              raise ArgumentError, "ContextPlan modified an input conversation group: #{ids.inspect}"
             end
-            assistant_by_tool_call_id[tool_call_id] = assistant
+            ids.each { |id| selected_input_ids[id] = true }
+            next
+          end
+
+          overlaps = ids.any? do |id|
+            input_groups.any? { |input_group| input_group.any? { |item| item.id == id } }
           end
+          if overlaps
+            raise ArgumentError,
+              "ContextPlan split, merged, or reordered an input conversation group: #{ids.inspect}"
+          end
+          unless group.all? { |item| item.provenance.origin == :policy_generated }
+            raise ArgumentError, "ContextPlan contains unknown conversation item(s): #{ids.inspect}"
+          end
+          validate_generated_conversation_group!(group)
         end
 
-        assistants.each do |assistant|
-          next unless selected_ids[assistant.candidate_id]
+        missing_groups = input_groups.select do |group|
+          group.any?(&:required?) && group.none? { |item| selected_input_ids[item.id] }
+        end
+        return if missing_groups.empty?
+
+        raise Phronomy::ContextBudgetExceededError,
+          "ContextPlan omitted required conversation group(s): " \
+          "#{missing_groups.map { |group| group.map(&:id) }.inspect}"
+      end
+
+      def validate_semantic_kind!(item, semantic_category)
+        return unless item.respond_to?(:kind)
+
+        reserved_category = RESERVED_KIND_CATEGORIES[item.kind.to_sym]
+        return unless reserved_category && reserved_category != semantic_category
+
+        raise ArgumentError,
+          "ContextPlan #{semantic_category} item #{item.id.inspect} uses reserved " \
+          "#{item.kind.inspect} kind for #{reserved_category}"
+      end
+
+      def validate_policy_generated_metadata!(item)
+        conflicts = item.metadata.keys.map(&:to_s) & ContextPolicyInput::FRAMEWORK_METADATA_KEYS
+        return if conflicts.empty?
+
+        raise ArgumentError,
+          "Policy-generated Context item #{item.id.inspect} metadata uses " \
+          "Framework-reserved key(s): #{conflicts.sort.inspect}"
+      end
 
-          canonical_tool_call_ids(assistant).each do |tool_call_id|
-            messages = tool_messages.fetch(tool_call_id, [])
-            if messages.empty?
+      def validate_generated_conversation_group!(group)
+        group.each do |item|
+          validate_policy_generated_metadata!(item)
+          validate_generated_conversation_item!(item)
+        end
+
+        assistants = group.select { |item| item.kind == :assistant_message }
+        tools = group.select { |item| item.kind == :tool_message }
+        protocol_items = assistants + tools
+        if protocol_items.any? && protocol_items.length != group.length
+          raise ArgumentError,
+            "Policy-generated Tool exchange group may contain only assistant_message and tool_message items"
+        end
+
+        tool_by_id = tools.group_by(&:tool_call_id)
+        positions = group.each_with_index.to_h
+
+        tool_by_id.each do |tool_call_id, messages|
+          if tool_call_id.nil? || tool_call_id.empty? || messages.length != 1
+            raise ArgumentError,
+              "Policy-generated conversation group has invalid Tool message dependency"
+          end
+        end
+
+        assistant_by_call = {}
+        assistants.each do |assistant|
+          assistant.tool_call_ids.each do |tool_call_id|
+            if assistant_by_call.key?(tool_call_id)
               raise ArgumentError,
-                "ContextPlan selected assistant Tool Call without a Tool message: #{tool_call_id}"
+                "Policy-generated conversation group has duplicate assistant Tool Call: #{tool_call_id}"
             end
-            missing = messages.reject { |message| selected_ids[message.candidate_id] }
-            unless missing.empty?
+            assistant_by_call[tool_call_id] = assistant
+            tool = tool_by_id[tool_call_id]&.first
+            unless tool
               raise ArgumentError,
-                "ContextPlan split assistant/Tool message dependency: #{tool_call_id}"
+                "Policy-generated assistant Tool Call has no Tool message: #{tool_call_id}"
+            end
+            unless positions.fetch(tool) > positions.fetch(assistant)
+              raise ArgumentError,
+                "Policy-generated Tool message must follow its assistant Tool Call: #{tool_call_id}"
             end
           end
         end
 
-        tool_messages.each do |tool_call_id, messages|
-          messages.each do |message|
-            next unless selected_ids[message.candidate_id]
+        tools.each do |tool|
+          unless assistant_by_call.key?(tool.tool_call_id)
+            raise ArgumentError,
+              "Policy-generated conversation group contains orphan Tool message: #{tool.tool_call_id}"
+          end
+        end
+      end
 
-            assistant = assistant_by_tool_call_id[tool_call_id]
-            unless assistant && selected_ids[assistant.candidate_id]
+      def validate_generated_conversation_item!(item)
+        case item.kind
+        when :assistant_message
+          payload = validate_generated_canonical_message!(item, expected_role: :assistant)
+          raw_calls = payload["tool_calls"] || payload[:tool_calls]
+          unless raw_calls.nil? || raw_calls.is_a?(Array)
+            raise ArgumentError,
+              "Policy-generated assistant_message tool_calls must be an Array"
+          end
+          payload_call_ids = Array(raw_calls).map do |call|
+            unless call.is_a?(Hash)
               raise ArgumentError,
-                "ContextPlan selected orphan Tool message: #{tool_call_id}"
+                "Policy-generated assistant_message Tool Call must be a Hash"
             end
+            id = call["id"] || call[:id]
+            name = call["name"] || call[:name]
+            arguments = if call.key?("arguments")
+              call["arguments"]
+            else
+              call.fetch(:arguments, {})
+            end
+            if id.to_s.empty? || name.to_s.empty? || !arguments.is_a?(Hash)
+              raise ArgumentError,
+                "Policy-generated assistant_message has malformed Tool Call"
+            end
+            id.to_s
+          end
+          unless payload_call_ids == item.tool_call_ids
+            raise ArgumentError,
+              "Policy-generated assistant_message Tool Call IDs do not match ConversationItem"
+          end
+          if item.tool_call_id
+            raise ArgumentError,
+              "Policy-generated assistant_message must not set tool_call_id"
+          end
+        when :tool_message
+          payload = validate_generated_canonical_message!(item, expected_role: :tool)
+          payload_tool_call_id = payload["tool_call_id"] || payload[:tool_call_id]
+          if item.tool_call_id.to_s.empty? || payload_tool_call_id.to_s != item.tool_call_id
+            raise ArgumentError,
+              "Policy-generated tool_message tool_call_id does not match canonical content"
+          end
+          unless item.tool_call_ids.empty?
+            raise ArgumentError,
+              "Policy-generated tool_message must not set tool_call_ids"
+          end
+        when :tool_result
+          raise ArgumentError,
+            "Policy-generated conversation must not contain raw tool_result items"
+        else
+          if item.tool_call_id || !item.tool_call_ids.empty?
+            raise ArgumentError,
+              "Policy-generated non-Tool conversation item must not declare Tool Call IDs"
           end
         end
       end
 
-      def canonical_tool_call_ids(candidate)
-        Array(candidate.metadata["tool_call_ids"] || candidate.metadata[:tool_call_ids])
-          .compact
-          .map(&:to_s)
-      end
+      def validate_generated_canonical_message!(item, expected_role:)
+        unless item.content_format == :json && item.content.is_a?(Hash)
+          raise ArgumentError,
+            "Policy-generated #{item.kind} must use canonical JSON message content"
+        end
 
-      def validate_derived_contents!(request, plan)
-        known = request.candidates.to_h { |candidate| [candidate.candidate_id, true] }
-        plan.derived_contents.each do |derived|
-          unless derived.is_a?(DerivedContentSpec)
-            raise ArgumentError,
-              "ContextPlan derived content must be DerivedContentSpec, got #{derived.class}"
-          end
-          unknown = derived.coverage_candidate_ids.reject { |id| known[id] }
-          raise ArgumentError, "Derived content covers unknown candidates: #{unknown.inspect}" unless unknown.empty?
+        payload_role = item.content["role"] || item.content[:role]
+        unless item.role == expected_role && payload_role.to_s == expected_role.to_s
+          raise ArgumentError,
+            "Policy-generated #{item.kind} role does not match canonical content"
         end
+
+        item.content
+      end
+
+      def validate_unique_ids!(plan)
+        ids = plan.instruction.map(&:id) + plan.knowledge.map(&:id) + plan.tools.map(&:id) +
+          plan.conversation.flatten.map(&:id)
+        duplicates = ids.group_by(&:itself).select { |_id, values| values.length > 1 }.keys
+        return if duplicates.empty?
+
+        raise ArgumentError, "ContextPlan contains duplicate item IDs: #{duplicates.inspect}"
       end
     end
   end

data/lib/phronomy/agent/context_policies/default.rb

@@ -4,49 +4,138 @@ module Phronomy
   module Agent
     module ContextPolicies
       class Default < ContextPolicy
-        DESCRIPTOR = ContextPolicyDescriptor.new(
-          id: "default-recent-v1",
-          version: 1,
-          config: {}
-        )
+        CONVERSATION_SHARE = 0.60
 
-        def initialize(config = {})
-          @config = Immutable.copy(config || {})
+        def self.instance
+          @instance ||= new.freeze
         end
 
-        def descriptor
-          return DESCRIPTOR if @config.empty?
-          ContextPolicyDescriptor.new(id: DESCRIPTOR.id, version: DESCRIPTOR.version, config: @config)
-        end
+        def call(input)
+          unless input.is_a?(ContextPolicyInput)
+            raise ArgumentError, "Default ContextPolicy expected ContextPolicyInput"
+          end
 
-        def call(request)
-          units = request.parts.fetch(:unit_builder).build(request.candidates)
-          units = request.parts.fetch(:required_context_resolver).resolve(
-            request: request,
-            units: units
-          )
-          ordered = request.parts.fetch(:recent_first_selector).order(
-            request: request,
-            units: units
-          )
-          selected = request.parts.fetch(:token_budget_packer).pack(
-            request: request,
-            units: ordered
-          )
+          return all_context(input) unless input.token_budget
+
+          limit = input.token_budget.effective_input_limit
+          required_knowledge = input.knowledge.select(&:required?)
+          optional_knowledge = input.knowledge.reject(&:required?)
+          required_conversation = input.conversation.select { |group| group.any?(&:required?) }
+          optional_conversation = input.conversation.reject { |group| group.any?(&:required?) }
+
+          fixed_cost = item_cost(input.instruction) + item_cost(input.tools) +
+            item_cost(required_knowledge) + group_cost(required_conversation)
+          if fixed_cost > limit
+            raise Phronomy::ContextBudgetExceededError,
+              "Required Context (estimated #{fixed_cost} tokens) exceeds " \
+              "available input budget (#{limit} tokens)"
+          end
+
+          remaining = limit - fixed_cost
+          conversation_budget = (remaining * CONVERSATION_SHARE).floor
+          knowledge_budget = remaining - conversation_budget
 
-          ContextPlan.new(
-            selected_unit_ids: selected.map(&:unit_id),
-            derived_contents: [],
-            selected_tool_ids: [],
-            ordering_hints: {"history" => "chronological"},
-            policy_descriptor: descriptor,
+          selected_conversation, older_conversation, conversation_used =
+            select_recent_suffix(optional_conversation, conversation_budget)
+          selected_knowledge, remaining_knowledge, knowledge_used =
+            select_stable_fit(optional_knowledge, knowledge_budget)
+
+          reusable = remaining - conversation_used - knowledge_used
+          if reusable.positive? && !older_conversation.empty?
+            more_conversation, _, used =
+              select_recent_suffix(older_conversation, reusable)
+            selected_conversation = more_conversation + selected_conversation
+            reusable -= used
+          end
+          if reusable.positive? && !remaining_knowledge.empty?
+            more_knowledge, _remaining_knowledge, used =
+              select_stable_fit(remaining_knowledge, reusable)
+            selected_knowledge += more_knowledge
+            reusable -= used
+          end
+
+          conversation = (required_conversation + selected_conversation)
+            .uniq
+            .sort_by { |group| group_sequence(group) }
+          knowledge_ids = (required_knowledge + selected_knowledge).to_h { |item| [item.id, true] }
+          knowledge = input.knowledge.select { |item| knowledge_ids[item.id] }
+
+          plan(
+            instruction: input.instruction,
+            knowledge: knowledge,
+            tools: input.tools,
+            conversation: conversation,
             metadata: {
-              "candidate_count" => request.candidates.length,
-              "unit_count" => units.length,
-              "selected_unit_count" => selected.length
+              "default_policy" => true,
+              "estimated_fixed_tokens" => fixed_cost,
+              "estimated_unused_tokens" => reusable
             }
           )
         end
+
+        private
+
+        def all_context(input)
+          plan(
+            instruction: input.instruction,
+            knowledge: input.knowledge,
+            tools: input.tools,
+            conversation: input.conversation,
+            metadata: {"default_policy" => true}
+          )
+        end
+
+        def item_cost(items)
+          Array(items).sum { |item| Integer(item.estimated_tokens || 0) }
+        end
+
+        def group_cost(groups)
+          Array(groups).sum { |group| item_cost(group) }
+        end
+
+        def group_sequence(group)
+          group.filter_map(&:sequence).min || 0
+        end
+
+        def select_recent_suffix(groups, budget)
+          ordered = Array(groups).sort_by { |group| group_sequence(group) }
+          selected = []
+          remaining = Integer(budget)
+          stop_index = -1
+
+          (ordered.length - 1).downto(0) do |index|
+            cost = item_cost(ordered[index])
+            if cost > remaining
+              stop_index = index
+              break
+            end
+            selected.unshift(ordered[index])
+            remaining -= cost
+          end
+
+          older = if stop_index >= 0
+            ordered[0..stop_index]
+          else
+            []
+          end
+          [selected.freeze, older.freeze, Integer(budget) - remaining]
+        end
+
+        def select_stable_fit(items, budget)
+          remaining = Integer(budget)
+          selected = []
+          unselected = []
+          Array(items).each do |item|
+            cost = Integer(item.estimated_tokens || 0)
+            if cost <= remaining
+              selected << item
+              remaining -= cost
+            else
+              unselected << item
+            end
+          end
+          [selected.freeze, unselected.freeze, Integer(budget) - remaining]
+        end
       end
     end
   end