anima-core 1.1.3 → 1.3.0

data/app/models/concerns/{event → message}/broadcasting.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
-# Broadcasts Event records to connected WebSocket clients via ActionCable.
-# Follows the Turbo Streams pattern: events are broadcast on both create
+# Broadcasts Message records to connected WebSocket clients via ActionCable.
+# Follows the Turbo Streams pattern: messages are broadcast on both create
 # and update, with an action type so clients can distinguish append from
 # replace operations.
 #
-# Each broadcast includes the Event's database ID, enabling clients to
+# Each broadcast includes the Message's database ID, enabling clients to
 # maintain an ID-indexed store for efficient in-place updates (e.g. when
-# token counts arrive asynchronously from {CountEventTokensJob}).
+# token counts arrive asynchronously from {CountMessageTokensJob}).
 #
-# When a new event pushes old events out of the LLM's context window,
-# the broadcast includes `evicted_event_ids` so clients can remove
+# When a new message pushes old messages out of the LLM's context window,
+# the broadcast includes `evicted_message_ids` so clients can remove
 # phantom messages that the agent no longer knows about.
 #
 # @example Create broadcast payload
@@ -24,7 +24,7 @@
 #   {
 #     "type" => "agent_message", "content" => "...", ...,
 #     "id" => 99, "action" => "create",
-#     "evicted_event_ids" => [101, 102, 103]
+#     "evicted_message_ids" => [101, 102, 103]
 #   }
 #
 # @example Update broadcast payload (e.g. token count arrives)
@@ -33,7 +33,7 @@
 #     "id" => 42, "action" => "update",
 #     "rendered" => { "debug" => { "role" => "user", "content" => "hello", "tokens" => 15 } }
 #   }
-module Event::Broadcasting
+module Message::Broadcasting
   extend ActiveSupport::Concern
 
   ACTION_CREATE = "create"
@@ -47,26 +47,26 @@ module Event::Broadcasting
   private
 
   def broadcast_create
-    broadcast_event(action: ACTION_CREATE)
+    broadcast_message(action: ACTION_CREATE)
   end
 
   def broadcast_update
-    broadcast_event(action: ACTION_UPDATE)
+    broadcast_message(action: ACTION_UPDATE)
   end
 
-  # Decorates the event for the session's current view mode and broadcasts
+  # Decorates the message for the session's current view mode and broadcasts
   # the payload to the session's ActionCable stream. Includes viewport
   # eviction metadata so clients can remove messages the LLM has forgotten.
   #
-  # @param action [String] ACTION_CREATE or ACTION_UPDATE — tells clients how to handle the event
-  def broadcast_event(action:)
+  # @param action [String] ACTION_CREATE or ACTION_UPDATE — tells clients how to handle the message
+  def broadcast_message(action:)
     return unless session_id
 
     session = Session.find_by(id: session_id)
     return unless session
 
     mode = session.view_mode
-    decorator = EventDecorator.for(self)
+    decorator = MessageDecorator.for(self)
     broadcast_payload = payload.merge("id" => id, "action" => action)
 
     if decorator
@@ -74,11 +74,11 @@ module Event::Broadcasting
     end
 
     evicted_ids = session.recalculate_viewport!
-    broadcast_payload["evicted_event_ids"] = evicted_ids if evicted_ids.any?
+    broadcast_payload["evicted_message_ids"] = evicted_ids if evicted_ids.any?
 
     # The nil? branch fires on every broadcast until boundary initializes, but
     # schedule_mneme! returns early after setting the boundary — cost is one DB read + write.
-    session.schedule_mneme! if evicted_ids.any? || session.mneme_boundary_event_id.nil?
+    session.schedule_mneme! if evicted_ids.any? || session.mneme_boundary_message_id.nil?
 
     ActionCable.server.broadcast("session_#{session_id}", broadcast_payload)
   end

data/app/models/goal.rb

@@ -13,8 +13,8 @@ class Goal < ApplicationRecord
   belongs_to :session
   belongs_to :parent_goal, class_name: "Goal", optional: true
   has_many :sub_goals, -> { order(:created_at) }, class_name: "Goal", foreign_key: :parent_goal_id, dependent: :destroy
-  has_many :goal_pinned_events, dependent: :destroy
-  has_many :pinned_events, through: :goal_pinned_events
+  has_many :goal_pinned_messages, dependent: :destroy
+  has_many :pinned_messages, through: :goal_pinned_messages
 
   validates :description, presence: true
   validates :status, inclusion: {in: STATUSES}
@@ -25,6 +25,16 @@ class Goal < ApplicationRecord
   scope :completed, -> { where(status: "completed") }
   scope :root, -> { where(parent_goal_id: nil) }
 
+  # @!method self.not_evicted
+  #   Goals still visible in context (not yet evicted by the analytical brain).
+  #   @return [ActiveRecord::Relation]
+  scope :not_evicted, -> { where(evicted_at: nil) }
+
+  # @!method self.evictable
+  #   Completed goals pending eviction — visible to the brain for age-based review.
+  #   @return [ActiveRecord::Relation]
+  scope :evictable, -> { completed.where(evicted_at: nil) }
+
   after_commit :broadcast_goals_update
   after_commit :schedule_passive_recall, on: [:create, :update]
 
@@ -37,6 +47,9 @@ class Goal < ApplicationRecord
   # @return [Boolean] true if this is a root goal (no parent)
   def root? = !parent_goal_id
 
+  # @return [Boolean] true if this goal has been evicted from display
+  def evicted? = evicted_at.present?
+
   # Cascades completion to all active sub-goals. Called when a root goal
   # is finished — remaining sub-items are implicitly resolved because
   # the semantic episode that spawned them has ended.
@@ -52,14 +65,14 @@ class Goal < ApplicationRecord
     sub_goals.active.update_all(status: "completed", completed_at: now, updated_at: now)
   end
 
-  # Releases pinned events that have no remaining active Goal references
+  # Releases pinned messages that have no remaining active Goal references
   # anywhere in the session. Called after goal (and cascade) completion —
   # the orphaned scope checks all Goals, so pins shared with other active
   # Goals survive automatically via reference counting.
   #
   # @return [Integer] number of released pins
   def release_orphaned_pins!
-    orphaned = session.pinned_events.orphaned
+    orphaned = session.pinned_messages.orphaned
     orphaned.destroy_all.size
   end
 

data/app/models/goal_pinned_message.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Join record linking a {Goal} to a {PinnedMessage}. Many-to-many: one message
+# can be pinned to multiple Goals, and one Goal can reference multiple pins.
+# When the last Goal referencing a pin completes, the pin is released.
+class GoalPinnedMessage < ApplicationRecord
+  belongs_to :goal
+  belongs_to :pinned_message
+
+  validates :pinned_message_id, uniqueness: {scope: :goal_id}
+end

data/app/models/message.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+# A persisted record of what was said during a session — by whom and when.
+# Messages are the single source of truth for conversation history —
+# there is no separate chat log, only messages attached to a session.
+#
+# Not to be confused with {Events::Base} (transient bus signals).
+# Messages persist to SQLite; events flow through the bus and are gone.
+#
+# @!attribute message_type
+#   @return [String] one of {TYPES}: system_message, user_message,
+#     agent_message, tool_call, tool_response
+# @!attribute payload
+#   @return [Hash] message-specific data (content, tool_name, tool_input, etc.)
+# @!attribute timestamp
+#   @return [Integer] nanoseconds since epoch (Process::CLOCK_REALTIME)
+# @!attribute token_count
+#   @return [Integer] cached token count for this message's payload (0 until counted)
+# @!attribute tool_use_id
+#   @return [String] ID correlating tool_call and tool_response messages
+#     (Anthropic-assigned, or a SecureRandom.uuid fallback when the API returns nil;
+#     required for tool_call and tool_response messages)
+class Message < ApplicationRecord
+  include Message::Broadcasting
+
+  TYPES = %w[system_message user_message agent_message tool_call tool_response].freeze
+  LLM_TYPES = %w[user_message agent_message].freeze
+  CONTEXT_TYPES = %w[system_message user_message agent_message tool_call tool_response].freeze
+  CONVERSATION_TYPES = %w[user_message agent_message system_message].freeze
+  THINK_TOOL = "think"
+  SPAWN_TOOLS = %w[spawn_subagent spawn_specialist].freeze
+
+  # Message types that require a tool_use_id to pair call with response.
+  TOOL_TYPES = %w[tool_call tool_response].freeze
+
+  ROLE_MAP = {"user_message" => "user", "agent_message" => "assistant"}.freeze
+
+  # Heuristic: average bytes per token for English prose.
+  BYTES_PER_TOKEN = 4
+
+  # Synthetic ID for system prompt entries in the TUI message store.
+  # Real message IDs are positive integers from the database, so 0
+  # is safe for deduplication without collision risk.
+  SYSTEM_PROMPT_ID = 0
+
+  # Estimates token count from a byte size using the {BYTES_PER_TOKEN} heuristic.
+  # @param bytesize [Integer] number of bytes
+  # @return [Integer] estimated token count (at least 1)
+  def self.estimate_token_count(bytesize)
+    [(bytesize / BYTES_PER_TOKEN.to_f).ceil, 1].max
+  end
+
+  belongs_to :session
+  has_many :pinned_messages, dependent: :destroy
+
+  validates :message_type, presence: true, inclusion: {in: TYPES}
+  validates :payload, presence: true
+  validates :timestamp, presence: true
+  # Anthropic requires every tool_use to have a matching tool_result with the same ID
+  validates :tool_use_id, presence: true, if: -> { message_type.in?(TOOL_TYPES) }
+
+  after_create :schedule_token_count, if: :llm_message?
+
+  # @!method self.llm_messages
+  #   Messages that represent conversation turns sent to the LLM API.
+  #   @return [ActiveRecord::Relation]
+  scope :llm_messages, -> { where(message_type: LLM_TYPES) }
+
+  # @!method self.context_messages
+  #   Messages included in the LLM context window (conversation + tool interactions).
+  #   @return [ActiveRecord::Relation]
+  scope :context_messages, -> { where(message_type: CONTEXT_TYPES) }
+
+  # @!method self.excluding_spawn_messages
+  #   Excludes spawn_subagent/spawn_specialist tool_call and tool_response messages.
+  #   Used when building parent context for sub-agents — spawn messages cause role
+  #   confusion because the sub-agent sees sibling spawn results and mistakes
+  #   itself for the parent.
+  #   @return [ActiveRecord::Relation]
+  scope :excluding_spawn_messages, -> {
+    where.not("message_type IN (?) AND json_extract(payload, '$.tool_name') IN (?)",
+      TOOL_TYPES, SPAWN_TOOLS)
+  }
+
+  # Maps message_type to the Anthropic Messages API role.
+  # @return [String] "user" or "assistant"
+  def api_role
+    ROLE_MAP.fetch(message_type)
+  end
+
+  # @return [Boolean] true if this message represents an LLM conversation turn
+  def llm_message?
+    message_type.in?(LLM_TYPES)
+  end
+
+  # @return [Boolean] true if this message is part of the LLM context window
+  def context_message?
+    message_type.in?(CONTEXT_TYPES)
+  end
+
+  # @return [Boolean] true if this is a conversation message (user/agent/system)
+  #   or a think tool_call — the messages Mneme treats as "conversation" for boundary tracking
+  def conversation_or_think?
+    message_type.in?(CONVERSATION_TYPES) ||
+      (message_type == "tool_call" && payload["tool_name"] == THINK_TOOL)
+  end
+
+  # Heuristic token estimate: ~4 bytes per token for English prose.
+  # Tool messages are estimated from the full payload JSON since tool_input
+  # and tool metadata contribute to token count. Messages use content only.
+  #
+  # @return [Integer] estimated token count (at least 1)
+  def estimate_tokens
+    text = if message_type.in?(TOOL_TYPES)
+      payload.to_json
+    else
+      payload["content"].to_s
+    end
+    self.class.estimate_token_count(text.bytesize)
+  end
+
+  private
+
+  def schedule_token_count
+    CountMessageTokensJob.perform_later(id)
+  end
+end

data/app/models/pending_message.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# A user message waiting to enter a session's conversation history.
+# Pending messages live in their own table — they are NOT part of the
+# message stream and have no database ID that could interleave with
+# tool_call/tool_response pairs.
+#
+# Created when a user sends a message while the session is processing.
+# Promoted to a real {Message} (delete + create in transaction) when
+# the current agent loop completes, giving the new message an ID that
+# naturally follows the tool batch.
+#
+# @see Session#enqueue_user_message
+# @see Session#promote_pending_messages!
+class PendingMessage < ApplicationRecord
+  belongs_to :session
+
+  validates :content, presence: true
+
+  after_create_commit :broadcast_created
+  after_destroy_commit :broadcast_removed
+
+  private
+
+  # Broadcasts a pending message appearance so TUI clients render the
+  # dimmed indicator immediately.
+  def broadcast_created
+    ActionCable.server.broadcast("session_#{session_id}", {
+      "action" => "pending_message_created",
+      "pending_message_id" => id,
+      "content" => content
+    })
+  end
+
+  # Broadcasts pending message removal so TUI clients clear the entry.
+  # Fires on both promotion (normal flow) and recall (user edit).
+  def broadcast_removed
+    ActionCable.server.broadcast("session_#{session_id}", {
+      "action" => "pending_message_removed",
+      "pending_message_id" => id
+    })
+  end
+end

data/app/models/pinned_message.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# A conversation message pinned to one or more Goals by Mneme to protect it
+# from viewport eviction. Pinned messages appear in the Goals section of
+# the viewport, giving the main agent access to critical context that
+# would otherwise scroll out of the sliding window.
+#
+# Pinning is goal-scoped: when all Goals referencing a pin complete,
+# the pin is automatically released (reference-counted cleanup).
+#
+# @!attribute display_text
+#   @return [String] truncated message content (~200 chars) shown in the Goals section
+class PinnedMessage < ApplicationRecord
+  # Display text limit — enough to recognize content, cheap on tokens.
+  MAX_DISPLAY_TEXT_LENGTH = 200
+
+  belongs_to :message
+
+  has_many :goal_pinned_messages, dependent: :destroy
+  has_many :goals, through: :goal_pinned_messages
+
+  validates :display_text, presence: true, length: {maximum: MAX_DISPLAY_TEXT_LENGTH}
+  validates :message_id, uniqueness: true
+
+  # Pinned messages with no remaining active goals — safe to release.
+  #
+  # @return [ActiveRecord::Relation]
+  scope :orphaned, -> {
+    where.not(
+      "EXISTS (SELECT 1 FROM goal_pinned_messages gpm " \
+      "JOIN goals ON goals.id = gpm.goal_id " \
+      "WHERE gpm.pinned_message_id = pinned_messages.id " \
+      "AND goals.status = 'active')"
+    )
+  }
+
+  # @return [Integer] token cost estimate for viewport budget accounting
+  def token_cost
+    [(display_text.bytesize / Message::BYTES_PER_TOKEN.to_f).ceil, 1].max
+  end
+end

data/app/models/secret.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Encrypted key-value storage for runtime secrets (API tokens, credentials).
+# Replaces Rails encrypted credentials for secrets that must be readable
+# across forked Solid Queue workers without cache-busting hacks.
+#
+# Secrets are organized by namespace (e.g. +"anthropic"+, +"mcp"+) and key
+# (e.g. +"subscription_token"+). Values are encrypted at rest using Active
+# Record Encryption — only the +value+ column is encrypted; +namespace+ and
+# +key+ are plain text for queryability.
+#
+# @!attribute namespace
+#   @return [String] grouping key (e.g. "anthropic", "mcp")
+# @!attribute key
+#   @return [String] credential identifier within the namespace
+# @!attribute value
+#   @return [String] the secret value (encrypted at rest)
+class Secret < ApplicationRecord
+  encrypts :value
+
+  validates :namespace, presence: true
+  validates :key, presence: true
+  validates :value, presence: true
+  validates :key, uniqueness: {scope: :namespace}
+
+  # @!method self.for_namespace(ns)
+  #   @param ns [String] namespace to filter by
+  #   @return [ActiveRecord::Relation] secrets in the given namespace
+  scope :for_namespace, ->(ns) { where(namespace: ns) }
+
+  # Reads a single secret value.
+  #
+  # @param namespace [String] top-level grouping key
+  # @param key [String] credential key within the namespace
+  # @return [String, nil] decrypted value or nil if not found
+  def self.read(namespace, key)
+    find_by(namespace: namespace, key: key)&.value
+  end
+
+  # Writes one or more key-value pairs under a namespace.
+  # Each pair is upserted (insert or update). The entire batch is wrapped
+  # in a transaction so partial writes cannot occur.
+  #
+  # @param namespace [String] top-level grouping key
+  # @param pairs [Hash<String, String>] key-value pairs to store
+  # @return [void]
+  def self.write(namespace, pairs)
+    transaction do
+      pairs.each do |secret_key, secret_value|
+        record = find_or_initialize_by(namespace: namespace, key: secret_key)
+        record.update!(value: secret_value)
+      end
+    end
+  end
+
+  # Lists all keys under a namespace (not values).
+  #
+  # @param namespace [String] top-level grouping key
+  # @return [Array<String>] credential keys
+  def self.list(namespace)
+    for_namespace(namespace).pluck(:key)
+  end
+
+  # Removes a single key from a namespace.
+  #
+  # @param namespace [String] top-level grouping key
+  # @param key [String] credential key to remove
+  # @return [void]
+  def self.remove(namespace, key)
+    find_by(namespace: namespace, key: key)&.destroy!
+  end
+end