anima-core 1.2.0 → 1.4.0

data/app/decorators/user_message_decorator.rb

@@ -3,22 +3,15 @@
 # Decorates user_message records for display in the TUI.
 # Basic mode returns role and content. Verbose mode adds a timestamp.
 # Debug mode adds token count (exact when counted, estimated when not).
-# Pending messages include `status: "pending"` so the TUI renders them
-# with a visual indicator (dimmed, clock icon).
 class UserMessageDecorator < MessageDecorator
-  # @return [Hash] structured user message data
-  #   `{role: :user, content: String}` or with `status: "pending"` when queued
+  # @return [Hash] structured user message data `{role: :user, content: String}`
   def render_basic
-    base = {role: :user, content: content}
-    base[:status] = "pending" if pending?
-    base
+    {role: :user, content: content}
   end
 
   # @return [Hash] structured user message with nanosecond timestamp
   def render_verbose
-    base = {role: :user, content: content, timestamp: timestamp}
-    base[:status] = "pending" if pending?
-    base
+    {role: :user, content: content, timestamp: timestamp}
   end
 
   # @return [Hash] verbose output plus token count for debugging
@@ -31,11 +24,4 @@ class UserMessageDecorator < MessageDecorator
   def render_brain
     "User: #{truncate_middle(content)}"
   end
-
-  private
-
-  # @return [Boolean] true when this message is queued but not yet sent to LLM
-  def pending?
-    payload["status"] == Message::PENDING_STATUS
-  end
 end

data/app/jobs/agent_request_job.rb

@@ -66,10 +66,10 @@ class AgentRequestJob < ApplicationJob
       agent_loop.run
     end
 
-    # Process any pending messages queued while we were busy.
+    # Process any pending messages that arrived after the last tool round.
     loop do
       promoted = session.promote_pending_messages!
-      break if promoted == 0
+      break if promoted[:texts].empty? && promoted[:pairs].empty?
       agent_loop.run
     end
 
@@ -155,18 +155,27 @@ class AgentRequestJob < ApplicationJob
 
   # Sets the session's processing flag atomically. Returns true if this
   # job claimed the lock, false if another job already holds it.
-  # Broadcasts the state change to the parent session's HUD.
+  # Broadcasts +session_state: llm_generating+ and the state change to
+  # the parent session's HUD.
   def claim_processing(session_id)
     claimed = Session.where(id: session_id, processing: false).update_all(processing: true) == 1
-    Session.find_by(id: session_id)&.broadcast_children_update_to_parent if claimed
+    if claimed
+      session = Session.find_by(id: session_id)
+      session&.broadcast_session_state("llm_generating")
+      session&.broadcast_children_update_to_parent
+    end
     claimed
   end
 
   # Clears the processing flag so the session can accept new jobs.
-  # Broadcasts the state change to the parent session's HUD.
+  # Broadcasts +session_state: idle+ to the session stream (replaces
+  # the old +processing_stopped+ action) and +children_updated+ to the
+  # parent session's HUD.
   def release_processing(session_id)
     Session.where(id: session_id).update_all(processing: false)
-    Session.find_by(id: session_id)&.broadcast_children_update_to_parent
+    session = Session.find_by(id: session_id)
+    session&.broadcast_session_state("idle")
+    session&.broadcast_children_update_to_parent
   end
 
   # Safety-net clearing of the interrupt flag.

data/app/jobs/passive_recall_job.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 # Runs passive recall after goal updates — searches message history for
-# context relevant to active goals and caches results on the session
-# for viewport injection.
+# context relevant to active goals and injects phantom tool_call/tool_response
+# pairs into the session's message stream.
 #
-# Idempotent: multiple enqueues for the same session safely overwrite
-# each other's results — last one wins.
+# Phantom pairs ride the conveyor belt like regular messages, getting
+# cached, evicted, and compressed by Mneme naturally.
 #
 # @example
 #   PassiveRecallJob.perform_later(session.id)
@@ -17,13 +17,8 @@ class PassiveRecallJob < ApplicationJob
   # @param session_id [Integer]
   def perform(session_id)
     session = Session.find(session_id)
-    results = Mneme::PassiveRecall.new(session).call
+    count = Mneme::PassiveRecall.new(session).call
 
-    if results.any?
-      session.update_column(:recalled_message_ids, results.map(&:message_id))
-      Mneme.logger.info("session=#{session_id} — passive recall found #{results.size} memories")
-    elsif session.recalled_message_ids.present?
-      session.update_column(:recalled_message_ids, [])
-    end
+    Mneme.logger.info("session=#{session_id} — passive recall injected #{count} phantom pairs") if count > 0
   end
 end

data/app/models/concerns/message/broadcasting.rb

@@ -68,6 +68,7 @@ module Message::Broadcasting
     mode = session.view_mode
     decorator = MessageDecorator.for(self)
     broadcast_payload = payload.merge("id" => id, "action" => action)
+    broadcast_payload["api_metrics"] = api_metrics if api_metrics.present?
 
     if decorator
       broadcast_payload["rendered"] = {mode => decorator.render(mode)}

data/app/models/goal.rb

@@ -25,6 +25,17 @@ class Goal < ApplicationRecord
   scope :completed, -> { where(status: "completed") }
   scope :root, -> { where(parent_goal_id: nil) }
 
+  # @!method self.not_evicted
+  #   Goals still visible in context (not yet evicted by the analytical brain).
+  #   @return [ActiveRecord::Relation]
+  scope :not_evicted, -> { where(evicted_at: nil) }
+
+  # @!method self.evictable
+  #   Completed goals not yet evicted — their phantom pairs remain in the
+  #   sliding window until Mneme compresses them during the eviction cycle.
+  #   @return [ActiveRecord::Relation]
+  scope :evictable, -> { completed.where(evicted_at: nil) }
+
   after_commit :broadcast_goals_update
   after_commit :schedule_passive_recall, on: [:create, :update]
 
@@ -37,6 +48,9 @@ class Goal < ApplicationRecord
   # @return [Boolean] true if this is a root goal (no parent)
   def root? = !parent_goal_id
 
+  # @return [Boolean] true if this goal has been evicted from display
+  def evicted? = evicted_at.present?
+
   # Cascades completion to all active sub-goals. Called when a root goal
   # is finished — remaining sub-items are implicitly resolved because
   # the semantic episode that spawned them has ended.

data/app/models/message.rb

@@ -28,9 +28,6 @@ class Message < ApplicationRecord
   CONTEXT_TYPES = %w[system_message user_message agent_message tool_call tool_response].freeze
   CONVERSATION_TYPES = %w[user_message agent_message system_message].freeze
   THINK_TOOL = "think"
-  SPAWN_TOOLS = %w[spawn_subagent spawn_specialist].freeze
-  PENDING_STATUS = "pending"
-
   # Message types that require a tool_use_id to pair call with response.
   TOOL_TYPES = %w[tool_call tool_response].freeze
 
@@ -39,6 +36,18 @@ class Message < ApplicationRecord
   # Heuristic: average bytes per token for English prose.
   BYTES_PER_TOKEN = 4
 
+  # Synthetic ID for system prompt entries in the TUI message store.
+  # Real message IDs are positive integers from the database, so 0
+  # is safe for deduplication without collision risk.
+  SYSTEM_PROMPT_ID = 0
+
+  # Estimates token count from a byte size using the {BYTES_PER_TOKEN} heuristic.
+  # @param bytesize [Integer] number of bytes
+  # @return [Integer] estimated token count (at least 1)
+  def self.estimate_token_count(bytesize)
+    [(bytesize / BYTES_PER_TOKEN.to_f).ceil, 1].max
+  end
+
   belongs_to :session
   has_many :pinned_messages, dependent: :destroy
 
@@ -60,28 +69,6 @@ class Message < ApplicationRecord
   #   @return [ActiveRecord::Relation]
   scope :context_messages, -> { where(message_type: CONTEXT_TYPES) }
 
-  # @!method self.pending
-  #   User messages queued during active agent processing, not yet sent to LLM.
-  #   @return [ActiveRecord::Relation]
-  scope :pending, -> { where(status: PENDING_STATUS) }
-
-  # @!method self.deliverable
-  #   Messages eligible for LLM context (excludes pending messages).
-  #   NULL status means delivered/processed — the only excluded value is "pending".
-  #   @return [ActiveRecord::Relation]
-  scope :deliverable, -> { where(status: nil) }
-
-  # @!method self.excluding_spawn_messages
-  #   Excludes spawn_subagent/spawn_specialist tool_call and tool_response messages.
-  #   Used when building parent context for sub-agents — spawn messages cause role
-  #   confusion because the sub-agent sees sibling spawn results and mistakes
-  #   itself for the parent.
-  #   @return [ActiveRecord::Relation]
-  scope :excluding_spawn_messages, -> {
-    where.not("message_type IN (?) AND json_extract(payload, '$.tool_name') IN (?)",
-      TOOL_TYPES, SPAWN_TOOLS)
-  }
-
   # Maps message_type to the Anthropic Messages API role.
   # @return [String] "user" or "assistant"
   def api_role
@@ -98,11 +85,6 @@ class Message < ApplicationRecord
     message_type.in?(CONTEXT_TYPES)
   end
 
-  # @return [Boolean] true if this is a pending message not yet sent to the LLM
-  def pending?
-    status == PENDING_STATUS
-  end
-
   # @return [Boolean] true if this is a conversation message (user/agent/system)
   #   or a think tool_call — the messages Mneme treats as "conversation" for boundary tracking
   def conversation_or_think?
@@ -121,7 +103,7 @@ class Message < ApplicationRecord
     else
       payload["content"].to_s
     end
-    [(text.bytesize / BYTES_PER_TOKEN.to_f).ceil, 1].max
+    self.class.estimate_token_count(text.bytesize)
   end
 
   private

data/app/models/pending_message.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+# A message waiting to enter a session's conversation history.
+# Pending messages live in their own table — they are NOT part of the
+# message stream and have no database ID that could interleave with
+# tool_call/tool_response pairs.
+#
+# Created when a message arrives while the session is processing.
+# Promoted to a real {Message} (delete + create in transaction) when
+# the current agent loop completes, giving the new message an ID that
+# naturally follows the tool batch.
+#
+# Each pending message knows its source (+source_type+, +source_name+)
+# and how to serialize itself for the LLM conversation via {#to_llm_messages}.
+# Non-user messages (sub-agent results, recalled skills, workflows, recall,
+# goal events) become synthetic tool_use/tool_result pairs so the LLM sees
+# "a tool I invoked returned a result" rather than "a user wrote me."
+#
+# @see Session#enqueue_user_message
+# @see Session#promote_pending_messages!
+class PendingMessage < ApplicationRecord
+  # Synthetic tool names used in tool_use/tool_result pairs injected into
+  # the parent LLM conversation when non-user messages are promoted.
+  # These tools don't exist in the agent's registry — the agent sees
+  # them as its own past actions (phantom tool calls).
+  SUBAGENT_TOOL = "subagent_message"
+  RECALL_SKILL_TOOL = "recall_skill"
+  RECALL_WORKFLOW_TOOL = "recall_workflow"
+  RECALL_MEMORY_TOOL = "recall_memory"
+  RECALL_GOAL_TOOL = "recall_goal"
+
+  # Source types that produce phantom tool_use/tool_result pairs on promotion.
+  # User messages produce plain text blocks instead.
+  PHANTOM_PAIR_TYPES = %w[subagent skill workflow recall goal].freeze
+
+  # Maps each phantom pair source type to its synthetic tool name.
+  PHANTOM_TOOL_NAMES = {
+    "subagent" => SUBAGENT_TOOL,
+    "skill" => RECALL_SKILL_TOOL,
+    "workflow" => RECALL_WORKFLOW_TOOL,
+    "recall" => RECALL_MEMORY_TOOL,
+    "goal" => RECALL_GOAL_TOOL
+  }.freeze
+
+  # Maps each phantom pair source type to a lambda building its tool input.
+  PHANTOM_TOOL_INPUTS = {
+    "subagent" => ->(name) { {from: name} },
+    "skill" => ->(name) { {skill: name} },
+    "workflow" => ->(name) { {workflow: name} },
+    "recall" => ->(name) { {message_id: name.to_i} },
+    "goal" => ->(name) { {goal_id: name.to_i} }
+  }.freeze
+
+  belongs_to :session
+
+  validates :content, presence: true
+  validates :source_type, inclusion: {in: %w[user subagent skill workflow recall goal]}
+  validates :source_name, presence: true, unless: :user?
+
+  after_create_commit :broadcast_created
+  after_destroy_commit :broadcast_removed
+
+  # @return [Boolean] true when this is a plain user message
+  def user?
+    source_type == "user"
+  end
+
+  # @return [Boolean] true when this message originated from a sub-agent
+  def subagent?
+    source_type == "subagent"
+  end
+
+  # @return [Boolean] true when this message carries recalled skill content
+  def skill?
+    source_type == "skill"
+  end
+
+  # @return [Boolean] true when this message carries recalled workflow content
+  def workflow?
+    source_type == "workflow"
+  end
+
+  # @return [Boolean] true when this message is an associative recall phantom pair
+  def recall?
+    source_type == "recall"
+  end
+
+  # @return [Boolean] true when this message carries a goal event
+  def goal?
+    source_type == "goal"
+  end
+
+  # @return [Boolean] true when promotion produces phantom tool_use/tool_result pairs
+  def phantom_pair?
+    source_type.in?(PHANTOM_PAIR_TYPES)
+  end
+
+  # Phantom tool name for DB persistence and LLM injection.
+  # Each phantom pair source type maps to a synthetic tool name.
+  #
+  # @return [String] phantom tool name
+  def phantom_tool_name
+    PHANTOM_TOOL_NAMES.fetch(source_type)
+  end
+
+  # Phantom tool input hash for DB persistence and LLM injection.
+  #
+  # @return [Hash] tool input hash
+  def phantom_tool_input
+    PHANTOM_TOOL_INPUTS.fetch(source_type).call(source_name)
+  end
+
+  # Content formatted for display and history persistence.
+  # Sub-agent messages include an attribution prefix. Skill/workflow
+  # messages include a recall label. User messages pass through unchanged.
+  #
+  # @return [String]
+  def display_content
+    case source_type
+    when "subagent"
+      format(Tools::ResponseTruncator::ATTRIBUTION_FORMAT, source_name, content)
+    when "skill"
+      "[recalled skill: #{source_name}]\n#{content}"
+    when "workflow"
+      "[recalled workflow: #{source_name}]\n#{content}"
+    when "goal"
+      "[goal #{source_name}]\n#{content}"
+    else
+      content
+    end
+  end
+
+  # Builds LLM message hashes for this pending message.
+  #
+  # Phantom pair types become synthetic tool_use/tool_result pairs so the
+  # LLM sees them as its own past invocations. User messages return plain
+  # content for injection as text blocks within the current tool_results turn.
+  #
+  # @return [Array<Hash>] synthetic tool pair for phantom pair types
+  # @return [String] raw content for user messages
+  def to_llm_messages
+    return content unless phantom_pair?
+
+    build_phantom_pair(phantom_tool_name, phantom_tool_input)
+  end
+
+  private
+
+  # Builds a phantom tool_use/tool_result message pair.
+  # Follows the same format for all non-user source types — the only
+  # difference is the tool name and input hash.
+  #
+  # Phantom pairs keep the system prompt stable for prompt caching (#395).
+  # Instead of injecting skills/workflows into the system prompt (which
+  # busts the cache on every change), they flow through the sliding window
+  # as messages the LLM "recalls" via phantom tool invocations.
+  #
+  # @param tool_name [String] phantom tool name (not in the agent's registry)
+  # @param input [Hash] tool input hash
+  # @return [Array<Hash>] two-element array: assistant tool_use + user tool_result
+  def build_phantom_pair(tool_name, input)
+    tool_use_id = "#{tool_name}_#{id}"
+    [
+      {role: "assistant", content: [
+        {type: "tool_use", id: tool_use_id, name: tool_name, input: input}
+      ]},
+      {role: "user", content: [
+        {type: "tool_result", tool_use_id: tool_use_id, content: content}
+      ]}
+    ]
+  end
+
+  # Broadcasts a pending message appearance so TUI clients render the
+  # dimmed indicator immediately.
+  def broadcast_created
+    ActionCable.server.broadcast("session_#{session_id}", {
+      "action" => "pending_message_created",
+      "pending_message_id" => id,
+      "content" => content
+    })
+  end
+
+  # Broadcasts pending message removal so TUI clients clear the entry.
+  # Fires on both promotion (normal flow) and recall (user edit).
+  def broadcast_removed
+    ActionCable.server.broadcast("session_#{session_id}", {
+      "action" => "pending_message_removed",
+      "pending_message_id" => id
+    })
+  end
+end

data/app/models/secret.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Encrypted key-value storage for runtime secrets (API tokens, credentials).
+# Replaces Rails encrypted credentials for secrets that must be readable
+# across forked Solid Queue workers without cache-busting hacks.
+#
+# Secrets are organized by namespace (e.g. +"anthropic"+, +"mcp"+) and key
+# (e.g. +"subscription_token"+). Values are encrypted at rest using Active
+# Record Encryption — only the +value+ column is encrypted; +namespace+ and
+# +key+ are plain text for queryability.
+#
+# @!attribute namespace
+#   @return [String] grouping key (e.g. "anthropic", "mcp")
+# @!attribute key
+#   @return [String] credential identifier within the namespace
+# @!attribute value
+#   @return [String] the secret value (encrypted at rest)
+class Secret < ApplicationRecord
+  encrypts :value
+
+  validates :namespace, presence: true
+  validates :key, presence: true
+  validates :value, presence: true
+  validates :key, uniqueness: {scope: :namespace}
+
+  # @!method self.for_namespace(ns)
+  #   @param ns [String] namespace to filter by
+  #   @return [ActiveRecord::Relation] secrets in the given namespace
+  scope :for_namespace, ->(ns) { where(namespace: ns) }
+
+  # Reads a single secret value.
+  #
+  # @param namespace [String] top-level grouping key
+  # @param key [String] credential key within the namespace
+  # @return [String, nil] decrypted value or nil if not found
+  def self.read(namespace, key)
+    find_by(namespace: namespace, key: key)&.value
+  end
+
+  # Writes one or more key-value pairs under a namespace.
+  # Each pair is upserted (insert or update). The entire batch is wrapped
+  # in a transaction so partial writes cannot occur.
+  #
+  # @param namespace [String] top-level grouping key
+  # @param pairs [Hash<String, String>] key-value pairs to store
+  # @return [void]
+  def self.write(namespace, pairs)
+    transaction do
+      pairs.each do |secret_key, secret_value|
+        record = find_or_initialize_by(namespace: namespace, key: secret_key)
+        record.update!(value: secret_value)
+      end
+    end
+  end
+
+  # Lists all keys under a namespace (not values).
+  #
+  # @param namespace [String] top-level grouping key
+  # @return [Array<String>] credential keys
+  def self.list(namespace)
+    for_namespace(namespace).pluck(:key)
+  end
+
+  # Removes a single key from a namespace.
+  #
+  # @param namespace [String] top-level grouping key
+  # @param key [String] credential key to remove
+  # @return [void]
+  def self.remove(namespace, key)
+    find_by(namespace: namespace, key: key)&.destroy!
+  end
+end