phronomy 0.15.1 → 0.17.0

data/lib/phronomy/agent/context_assembler.rb

@@ -0,0 +1,357 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    class ContextAssembler
+      ASSEMBLY_POLICY_VERSION = 6
+      SEGMENT_ORIGIN_METADATA_KEY = "phronomy_origin"
+      BEFORE_LLM_INPUT_ORIGIN = "before_llm_input"
+      EARLY_CONTEXT_CATEGORIES = %i[
+        instruction structured_state knowledge memory summary
+      ].freeze
+
+      def initialize(
+        agent:,
+        persistence:,
+        policy: ContextPolicies::Default.new,
+        candidate_resolver: nil
+      )
+        @agent = agent
+        @persistence = persistence
+        @policy = policy
+        @candidate_resolver = candidate_resolver || ContextCandidateResolver.new(
+          content_loader: method(:fetch_content)
+        )
+      end
+
+      def build_initial(input:, agent_root:, execution:, config: {}, patch: LLMInputPatch.empty)
+        projection = JournalProjection.new(persistence: @persistence, agent_root: agent_root)
+        model_cfg = effective_model_config(config, patch)
+        tool_set = ToolDefinitionSet.build(@agent)
+        system_text = build_system_text(input)
+        hook_candidates = normalize_candidates(patch.segment_candidates)
+        input_ref = execution.metadata.fetch("current_input_ref")
+        current_input = @persistence.contents.fetch_text(input_ref)
+        excluded = [execution.metadata["current_input_record_id"]].compact
+
+        base_segments = []
+        unless system_text.to_s.empty?
+          base_segments << text_segment(:instruction, :system, system_text)
+        end
+
+        assemble(
+          agent_root: agent_root,
+          execution: execution,
+          call_sequence: 1,
+          call_mode: :ask,
+          previous_manifest: nil,
+          model_config: model_cfg,
+          tool_definitions: tool_set.definitions,
+          tool_definitions_ref: nil,
+          prior_records: projection.context_records,
+          working_records: execution.working_records,
+          excluded_record_ids: excluded,
+          base_segments: base_segments,
+          hook_candidates: hook_candidates,
+          current_input_segment: segment(:current_input, :user, input_ref, :ask_argument),
+          mandatory_values: [system_text, current_input, tool_set.definitions]
+        )
+      end
+
+      def build_followup(
+        base_manifest:,
+        agent_root:,
+        execution:,
+        config: {},
+        patch: LLMInputPatch.empty
+      )
+        projection = JournalProjection.new(persistence: @persistence, agent_root: agent_root)
+        model_cfg = effective_model_config(config, patch)
+        hook_candidates = normalize_candidates(patch.segment_candidates)
+        system_segments = base_manifest.segments.select do |segment|
+          segment.role == :system && !before_llm_input_segment?(segment)
+        end
+        tool_definitions = base_manifest.tool_definitions_ref ?
+          fetch_json(base_manifest.tool_definitions_ref) : []
+
+        assemble(
+          agent_root: agent_root,
+          execution: execution,
+          call_sequence: execution.llm_calls.length + 1,
+          call_mode: :complete,
+          previous_manifest: base_manifest,
+          model_config: model_cfg,
+          tool_definitions: tool_definitions,
+          tool_definitions_ref: base_manifest.tool_definitions_ref,
+          prior_records: projection.context_records,
+          working_records: execution.working_records,
+          excluded_record_ids: [],
+          base_segments: system_segments.map { |existing| segment_hash(existing) },
+          hook_candidates: hook_candidates,
+          current_input_segment: nil,
+          mandatory_values: system_segments.map { |segment| fetch_content(segment.content_ref) } +
+            [tool_definitions]
+        )
+      end
+
+      private
+
+      def assemble(
+        agent_root:,
+        execution:,
+        call_sequence:,
+        call_mode:,
+        previous_manifest:,
+        model_config:,
+        tool_definitions:,
+        tool_definitions_ref:,
+        prior_records:,
+        working_records:,
+        excluded_record_ids:,
+        base_segments:,
+        hook_candidates:,
+        current_input_segment:,
+        mandatory_values:
+      )
+        generation = agent_root.transcript_generation
+        eligible_working = Array(working_records).select do |record|
+          record.context_candidate && record.context_generation == generation
+        end
+        candidates = @candidate_resolver.resolve(
+          prior_records: prior_records,
+          working_records: eligible_working,
+          excluded_record_ids: excluded_record_ids
+        )
+        candidates = merge_hook_candidates(
+          candidates,
+          hook_candidates,
+          agent_root: agent_root,
+          execution: execution,
+          call_sequence: call_sequence
+        )
+
+        token_budget = TokenBudgetResolver.new(agent: @agent).resolve(model_config)
+        mandatory_token_estimate = estimate_values(mandatory_values)
+        parts = context_parts
+        request = ContextRequest.new(
+          agent_id: agent_root.agent_id,
+          execution_id: execution.execution_id,
+          call_sequence: call_sequence,
+          call_mode: call_mode,
+          candidates: candidates,
+          token_budget: token_budget,
+          model_config: model_config,
+          previous_manifest: previous_manifest,
+          required_coverage: [],
+          parts: parts,
+          metadata: {"mandatory_token_estimate" => mandatory_token_estimate}
+        )
+        plan = @policy.call(request)
+        validated = ContextPlanValidator.new.validate!(request: request, plan: plan)
+        unless validated.plan.derived_contents.empty?
+          raise Phronomy::ConfigurationError,
+            "Derived Context persistence is not enabled in Context Policy phase 1-4"
+        end
+
+        selected_candidates = validated.selected_candidates.sort_by do |candidate|
+          [candidate.sequence || 0, candidate.candidate_id]
+        end
+        segments = Array(base_segments).dup
+        append_selected_candidates(segments, selected_candidates, before_history: true)
+        append_selected_candidates(segments, selected_candidates, before_history: false)
+        segments << current_input_segment if current_input_segment
+
+        ContextParts::Validators::FinalBudgetValidator.new(
+          content_loader: method(:fetch_content)
+        ).validate!(
+          token_budget: token_budget,
+          segments: segments,
+          extra_values: [tool_definitions]
+        )
+
+        store_manifest(
+          call_sequence: call_sequence,
+          call_mode: call_mode,
+          segments: segments,
+          model_config_ref: @persistence.contents.put_json(model_config),
+          tool_definitions_ref: tool_definitions_ref ||
+            @persistence.contents.put_json(tool_definitions)
+        )
+      end
+
+      def context_parts
+        {
+          unit_builder: ContextParts::UnitBuilders::DependencyAwareUnitBuilder.new,
+          required_context_resolver: ContextParts::Requirements::RequiredContextResolver.new,
+          recent_first_selector: ContextParts::Selectors::RecentFirstSelector.new,
+          token_budget_packer: ContextParts::Budget::TokenBudgetPacker.new
+        }.freeze
+      end
+
+      def estimate_values(values)
+        Array(values).sum do |value|
+          bytes = value.is_a?(String) ? value : Phronomy::CanonicalJSON.dump(value)
+          Phronomy::LlmContextWindow::TokenEstimator.estimate(bytes)
+        end
+      end
+
+      def store_manifest(call_sequence:, call_mode:, segments:, model_config_ref:, tool_definitions_ref:)
+        positioned = segments.each_with_index.map do |value, position|
+          LLMInputManifest::Segment.new(**value.merge(position: position))
+        end
+        manifest = LLMInputManifest.new(
+          call_sequence: call_sequence,
+          call_mode: call_mode,
+          segments: positioned,
+          model_config_ref: model_config_ref,
+          tool_definitions_ref: tool_definitions_ref,
+          assembly_policy_version: ASSEMBLY_POLICY_VERSION,
+          ruby_llm_version: defined?(RubyLLM::VERSION) ? RubyLLM::VERSION : nil,
+          adapter_name: Phronomy.configuration.llm_adapter.class.name
+        )
+        [manifest, @persistence.contents.put_json(manifest.to_h)]
+      end
+
+      def effective_model_config(config, patch)
+        apply_model_config_patch(model_config(config), patch.model_config_patch)
+      end
+
+      def model_config(config)
+        {
+          "model" => @agent.class.model&.to_s,
+          "provider" => @agent.class.provider&.to_s,
+          "temperature" => @agent.class.temperature,
+          "max_output_tokens" => @agent.class.max_output_tokens,
+          "context_window" => @agent.class.context_window,
+          "cache_instructions" => !!@agent.class.cache_instructions,
+          "parallel_tool_execution" => !!Phronomy.configuration.parallel_tool_execution,
+          "thread_id" => config[:thread_id]&.to_s
+        }.compact
+      end
+
+      def apply_model_config_patch(base, patch)
+        return base unless patch
+        base.merge(patch.to_h.transform_keys(&:to_s)).compact
+      end
+
+      def normalize_candidates(candidates)
+        Array(candidates).map do |candidate|
+          hash = candidate.to_h
+          content = hash.fetch(:content) { hash.fetch("content") }
+          category = (hash[:category] || hash["category"] || :knowledge).to_sym
+          role = (hash[:role] || hash["role"] || default_role(category)).to_sym
+          metadata = hash[:metadata] || hash["metadata"] || {}
+          {
+            content: content.to_s,
+            category: category,
+            role: role,
+            metadata: metadata.to_h.transform_keys(&:to_s)
+          }
+        end
+      end
+
+      def merge_hook_candidates(candidates, hooks, agent_root:, execution:, call_sequence:)
+        next_sequence = Array(candidates).filter_map(&:sequence).max.to_i
+        generated = hooks.each_with_index.map do |hook, index|
+          content_ref = @persistence.contents.put_text(hook.fetch(:content))
+          metadata = hook.fetch(:metadata).merge(
+            SEGMENT_ORIGIN_METADATA_KEY => BEFORE_LLM_INPUT_ORIGIN,
+            "estimated_tokens" => Phronomy::LlmContextWindow::TokenEstimator.estimate(
+              fetch_content(content_ref)
+            ),
+            "source_kind" => "hook"
+          )
+          ContextCandidate.new(
+            candidate_id: "hook:#{execution.execution_id}:#{call_sequence}:#{index}",
+            source_kind: :hook,
+            category: hook.fetch(:category),
+            role: hook.fetch(:role),
+            content_ref: content_ref,
+            record_id: nil,
+            agent_id: agent_root.agent_id,
+            execution_id: execution.execution_id,
+            llm_call_id: nil,
+            tool_call_id: nil,
+            sequence: next_sequence + index + 1,
+            requirement: :optional,
+            priority: 0,
+            metadata: metadata
+          )
+        end
+        (Array(candidates) + generated)
+          .sort_by { |candidate| [candidate.sequence || 0, candidate.candidate_id] }
+          .freeze
+      end
+
+      def append_selected_candidates(segments, candidates, before_history:)
+        candidates.each do |candidate|
+          early = EARLY_CONTEXT_CATEGORIES.include?(candidate.category)
+          next unless early == before_history
+
+          segments << segment_from_candidate(candidate)
+        end
+      end
+
+      def default_role(category)
+        (category == :instruction) ? :system : :user
+      end
+
+      def text_segment(category, role, content, metadata: {})
+        segment(
+          category, role, @persistence.contents.put_text(content.to_s), :chat_message,
+          metadata: metadata
+        )
+      end
+
+      def before_llm_input_segment?(segment)
+        segment.metadata[SEGMENT_ORIGIN_METADATA_KEY] == BEFORE_LLM_INPUT_ORIGIN
+      end
+
+      def segment_from_candidate(candidate)
+        metadata = candidate.metadata.reject do |key, _value|
+          %w[estimated_tokens source_kind source_sequence].include?(key.to_s)
+        end
+        metadata = metadata.merge(
+          "journal_record_id" => candidate.record_id,
+          "journal_sequence" => candidate.metadata["source_sequence"],
+          "llm_call_id" => candidate.llm_call_id
+        ).compact
+        segment(
+          candidate.category,
+          candidate.role,
+          candidate.content_ref,
+          :chat_message,
+          tool_call_id: candidate.tool_call_id,
+          metadata: metadata
+        )
+      end
+
+      def segment_hash(existing)
+        {
+          category: existing.category, role: existing.role,
+          content_ref: existing.content_ref, delivery: existing.delivery,
+          tool_call_id: existing.tool_call_id, metadata: existing.metadata
+        }
+      end
+
+      def segment(category, role, content_ref, delivery, tool_call_id: nil, metadata: {})
+        {category: category, role: role, content_ref: content_ref, delivery: delivery,
+         tool_call_id: tool_call_id, metadata: metadata}
+      end
+
+      def fetch_json(ref)
+        Phronomy::CanonicalJSON.load(@persistence.contents.fetch(ref))
+      end
+
+      def fetch_content(ref)
+        @persistence.contents.fetch_text(ref)
+      rescue
+        @persistence.contents.fetch(ref)
+      end
+
+      def build_system_text(input)
+        @agent.send(:build_instructions, input)
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context_candidate.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    ContextCandidate = Data.define(
+      :candidate_id,
+      :source_kind,
+      :category,
+      :role,
+      :content_ref,
+      :record_id,
+      :agent_id,
+      :execution_id,
+      :llm_call_id,
+      :tool_call_id,
+      :sequence,
+      :requirement,
+      :priority,
+      :metadata
+    ) do
+      def initialize(**values)
+        normalized = values.merge(
+          candidate_id: values.fetch(:candidate_id).to_s.freeze,
+          source_kind: values.fetch(:source_kind).to_sym,
+          category: values.fetch(:category).to_sym,
+          role: values[:role]&.to_sym,
+          content_ref: values.fetch(:content_ref).to_s.freeze,
+          record_id: values[:record_id]&.to_s&.freeze,
+          agent_id: values[:agent_id]&.to_s&.freeze,
+          execution_id: values[:execution_id]&.to_s&.freeze,
+          llm_call_id: values[:llm_call_id]&.to_s&.freeze,
+          tool_call_id: values[:tool_call_id]&.to_s&.freeze,
+          sequence: values[:sequence] && Integer(values[:sequence]),
+          requirement: (values[:requirement] || :optional).to_sym,
+          priority: Integer(values[:priority] || 0),
+          metadata: Immutable.copy(values[:metadata] || {})
+        )
+        unless %i[protocol_required declared_required optional].include?(normalized[:requirement])
+          raise ArgumentError, "unknown ContextCandidate requirement: #{normalized[:requirement].inspect}"
+        end
+
+        super(**normalized)
+        freeze
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context_candidate_resolver.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    class ContextCandidateResolver
+      def initialize(content_loader:)
+        @content_loader = content_loader
+      end
+
+      def resolve(prior_records:, working_records:, excluded_record_ids: [])
+        excluded = Array(excluded_record_ids).compact.map(&:to_s).to_h { |id| [id, true] }
+        prior = eligible(prior_records, excluded)
+        working = eligible(working_records, excluded)
+        next_sequence = prior.map(&:sequence).compact.max.to_i
+
+        candidates = []
+        prior.each do |record|
+          candidates << candidate_for(record, source_kind: :journal, sequence: record.sequence)
+        end
+        working.each do |record|
+          sequence = record.sequence || (next_sequence += 1)
+          candidates << candidate_for(record, source_kind: :working, sequence: sequence)
+        end
+        candidates.sort_by { |candidate| [candidate.sequence || 0, candidate.candidate_id] }.freeze
+      end
+
+      private
+
+      def eligible(records, excluded)
+        Array(records).select do |record|
+          record.context_candidate &&
+            record.content_ref &&
+            !excluded.key?(record.record_id.to_s)
+        end
+      end
+
+      def candidate_for(record, source_kind:, sequence:)
+        tool_call_id = record.metadata["tool_call_id"] || record.metadata[:tool_call_id]
+        bytes = @content_loader.call(record.content_ref)
+        metadata = record.metadata.merge(
+          "estimated_tokens" => Phronomy::LlmContextWindow::TokenEstimator.estimate(bytes),
+          "source_kind" => source_kind.to_s,
+          "source_sequence" => record.sequence
+        )
+
+        ContextCandidate.new(
+          candidate_id: "record:#{record.record_id}",
+          source_kind: source_kind,
+          category: record.kind,
+          role: record.role,
+          content_ref: record.content_ref,
+          record_id: record.record_id,
+          agent_id: record.agent_id,
+          execution_id: record.execution_id,
+          llm_call_id: record.llm_call_id,
+          tool_call_id: tool_call_id,
+          sequence: sequence,
+          requirement: :optional,
+          priority: (source_kind == :working) ? 100 : 0,
+          metadata: metadata
+        )
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context_importer.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    class ContextImporter
+      ImportedRecord = Data.define(
+        :kind, :channel, :role, :content, :content_format, :metadata
+      ) do
+        def initialize(**values)
+          super(**values.merge(
+            content: Immutable.copy(values[:content]),
+            metadata: Immutable.copy(values[:metadata] || {})
+          ))
+          freeze
+        end
+      end
+
+      ImportedContext = Data.define(:records) do
+        def initialize(records:)
+          super(records: records.freeze)
+          freeze
+        end
+      end
+
+      class << self
+        def import_messages(messages, system_message: :reject)
+          source_messages = Array(messages)
+          validate_protocol!(source_messages)
+          records = source_messages.map do |message|
+            import_message(message, system_message: system_message)
+          end
+          ImportedContext.new(records: records.freeze)
+        end
+
+        private
+
+        # Import validation is defined by the external-message contract, not by
+        # limitations of Phronomy's internal representation. Each supplied
+        # message is one explicit logical message and is journaled without
+        # splitting its assistant content from its Tool Calls.
+        def validate_protocol!(messages)
+          pending = {}
+          seen_tool_call_ids = {}
+
+          messages.each_with_index do |message, index|
+            role = read(message, :role).to_sym
+
+            if pending.any? && role != :tool
+              raise ArgumentError,
+                "message #{index} appears before Tool Results for: #{pending.keys.join(", ")}"
+            end
+
+            case role
+            when :assistant
+              tool_calls_for(message).each do |call|
+                payload = normalize(read_tool_call(call))
+                call_id = payload.fetch("id").to_s
+                raise ArgumentError, "tool_call requires id" if call_id.empty?
+                if seen_tool_call_ids[call_id]
+                  raise ArgumentError, "duplicate tool_call id: #{call_id}"
+                end
+                seen_tool_call_ids[call_id] = true
+                pending[call_id] = true
+              end
+            when :tool
+              call_id = read_optional(message, :tool_call_id).to_s
+              raise ArgumentError, "tool message requires tool_call_id" if call_id.empty?
+              unless pending.delete(call_id)
+                raise ArgumentError, "orphan or duplicate Tool Result: #{call_id}"
+              end
+            when :user, :system
+              # No Tool protocol state is introduced by these roles.
+            else
+              raise ArgumentError, "unsupported message role: #{role.inspect}"
+            end
+          end
+
+          return if pending.empty?
+
+          raise ArgumentError,
+            "imported history ends before Tool Results for: #{pending.keys.join(", ")}"
+        end
+
+        def import_message(message, system_message:)
+          role = read(message, :role).to_sym
+          case role
+          when :system
+            if system_message == :reject
+              raise ArgumentError,
+                "system messages must be imported as explicit instruction segments"
+            end
+            raise ArgumentError,
+              "unsupported system_message policy: #{system_message.inspect}"
+          when :user
+            text_record(:external_message, :external, :user, read(message, :content))
+          when :assistant
+            import_assistant(message)
+          when :tool
+            import_tool_message(message)
+          else
+            raise ArgumentError, "unsupported message role: #{role.inspect}"
+          end
+        end
+
+        def import_assistant(message)
+          calls = tool_calls_for(message).map { |call| normalize(read_tool_call(call)) }
+          content = read_optional(message, :content)
+          content = "" if content.nil? && !calls.empty?
+          if (content.nil? || (content.respond_to?(:empty?) && content.empty?)) && calls.empty?
+            raise ArgumentError,
+              "assistant message requires content or one or more tool_calls"
+          end
+
+          payload = {
+            "role" => "assistant",
+            "content" => normalize(content),
+            "tool_calls" => calls
+          }
+          model_id = read_optional(message, :model_id)
+          payload["model_id"] = model_id.to_s if model_id
+
+          ImportedRecord.new(
+            kind: :assistant_message,
+            channel: :llm,
+            role: :assistant,
+            content: payload,
+            content_format: :json,
+            metadata: assistant_metadata(calls)
+          )
+        end
+
+        def import_tool_message(message)
+          tool_call_id = read_optional(message, :tool_call_id).to_s
+          raise ArgumentError, "tool message requires tool_call_id" if tool_call_id.empty?
+
+          ImportedRecord.new(
+            kind: :tool_message,
+            channel: :tool,
+            role: :tool,
+            content: {
+              "role" => "tool",
+              "content" => String(read(message, :content)),
+              "tool_call_id" => tool_call_id
+            },
+            content_format: :json,
+            metadata: {"tool_call_id" => tool_call_id}
+          )
+        end
+
+        def assistant_metadata(calls)
+          {
+            "tool_call_ids" => calls.map { |call| call.fetch("id").to_s },
+            "tool_names" => calls.map { |call| call.fetch("name").to_s }
+          }
+        end
+
+        def tool_calls_for(message)
+          tool_calls = read_optional(message, :tool_calls)
+          Array(tool_calls.respond_to?(:values) ? tool_calls.values : tool_calls)
+        end
+
+        def text_record(kind, channel, role, content)
+          ImportedRecord.new(
+            kind: kind,
+            channel: channel,
+            role: role,
+            content: String(content),
+            content_format: :text,
+            metadata: {}
+          )
+        end
+
+        def read_tool_call(call)
+          return call.to_h if call.respond_to?(:to_h)
+
+          {
+            id: read(call, :id),
+            name: read(call, :name),
+            arguments: read_optional(call, :arguments) || {},
+            thought_signature: read_optional(call, :thought_signature)
+          }.compact
+        end
+
+        def read(value, name)
+          result = read_optional(value, name)
+          raise ArgumentError, "message is missing #{name}" if result.nil?
+
+          result
+        end
+
+        def read_optional(value, name)
+          return value.public_send(name) if value.respond_to?(name)
+          return value[name] if value.respond_to?(:key?) && value.key?(name)
+          return value[name.to_s] if value.respond_to?(:key?) && value.key?(name.to_s)
+
+          nil
+        end
+
+        def normalize(value)
+          case value
+          when Hash
+            value.to_h { |key, child| [key.to_s, normalize(child)] }
+          when Array
+            value.map { |child| normalize(child) }
+          when Symbol
+            value.to_s
+          when String, Integer, Float, TrueClass, FalseClass, NilClass
+            value
+          else
+            value.respond_to?(:to_h) ? normalize(value.to_h) :
+              raise(ArgumentError, "unsupported imported value: #{value.class}")
+          end
+        end
+      end
+    end
+  end
+end