anima-core 1.3.0 → 1.5.0

data/app/jobs/tool_execution_job.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# Runs a single tool on behalf of the session and reports the outcome.
+#
+# Queued by {Events::Subscribers::LLMResponseHandler} when the LLM
+# returns a +tool_use+ block. The session is already in the +:executing+
+# state (transition owned by the response handler). This job:
+#
+# 1. Dispatches the tool via {Tools::Registry}.
+# 2. Truncates and formats the result.
+# 3. Emits {Events::ToolExecuted}.
+#
+# The job does not release the session or create the +tool_response+
+# PendingMessage — that's {Events::Subscribers::ToolResponseCreator}'s
+# job. Event emission is the final act that hands control off.
+class ToolExecutionJob < ApplicationJob
+  queue_as :default
+
+  discard_on ActiveRecord::RecordNotFound
+
+  # @param session_id [Integer]
+  # @param tool_use_id [String] Anthropic-assigned pairing ID
+  # @param tool_name [String]
+  # @param tool_input [Hash]
+  def perform(session_id, tool_use_id:, tool_name:, tool_input:)
+    session = Session.find(session_id)
+    # ShellSession.for_session returns the conversation's persistent shell
+    # — spawned on first use, reused on every subsequent tool call so the
+    # agent's cd's and exported env vars survive between calls. We do NOT
+    # finalize it here; the shell's lifetime is the Session's lifetime.
+    shell_session = ShellSession.for_session(session)
+    registry = Tools::Registry.build(session: session, shell_session: shell_session)
+
+    content, success = execute(registry, tool_name, tool_input, tool_use_id)
+
+    Events::Bus.emit(Events::ToolExecuted.new(
+      session_id: session_id,
+      tool_use_id: tool_use_id,
+      tool_name: tool_name,
+      content: content,
+      success: success
+    ))
+  rescue => error
+    # A missing {Events::ToolExecuted} would leave the session in +:executing+
+    # forever. Always emit a synthetic failure event so
+    # {Events::Subscribers::ToolResponseCreator} runs and releases the claim.
+    Rails.logger.error("ToolExecutionJob crashed: #{error.class}: #{error.message}")
+    Events::Bus.emit(Events::ToolExecuted.new(
+      session_id: session_id,
+      tool_use_id: tool_use_id,
+      tool_name: tool_name,
+      content: "#{error.class}: #{error.message}",
+      success: false
+    ))
+  end
+
+  private
+
+  # Always emits something executable back — a missing +tool_result+
+  # permanently corrupts the Anthropic conversation history.
+  def execute(registry, tool_name, tool_input, tool_use_id)
+    result = registry.execute(tool_name, tool_input, tool_use_id: tool_use_id)
+    result = ::ToolDecorator.call(tool_name, result)
+    content = format_result(result)
+    content = truncate(content, registry, tool_name)
+    [content, !result.is_a?(Hash) || !result.key?(:error)]
+  rescue => error
+    Rails.logger.error("Tool #{tool_name} raised #{error.class}: #{error.message}")
+    [format_result(error: "#{error.class}: #{error.message}"), false]
+  end
+
+  def format_result(result)
+    result.is_a?(Hash) ? result.to_json : result.to_s
+  end
+
+  def truncate(content, registry, tool_name)
+    threshold = registry.truncation_threshold(tool_name)
+    return content unless threshold
+
+    lines = ::Tools::ResponseTruncator::HEAD_LINES
+    ::Tools::ResponseTruncator.truncate(
+      content,
+      threshold: threshold,
+      reason: "#{tool_name} output displays first/last #{lines} lines"
+    )
+  end
+end

data/app/models/concerns/token_estimation.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# Shared token-count lifecycle for records that ride in the LLM context
+# window. Including models seed {#token_count} with a local heuristic on
+# create and schedule {CountTokensJob} to refine it with the real Anthropic
+# tokenizer count.
+#
+# Non-AR callers (TUI debug display, phantom-pair sizing, byte-cap
+# calculations) use {.estimate_token_count} and {BYTES_PER_TOKEN} as
+# module-level helpers without including the concern.
+#
+# Including models must implement +#tokenization_text+ returning the
+# string whose token count should be estimated and later refined.
+module TokenEstimation
+  extend ActiveSupport::Concern
+
+  # Heuristic: average bytes per token for English prose.
+  BYTES_PER_TOKEN = 4
+
+  # Estimates token count from a string using the {BYTES_PER_TOKEN} heuristic.
+  #
+  # @param text [String, nil]
+  # @return [Integer] estimated token count (0 for blank input)
+  def self.estimate_token_count(text)
+    (text.to_s.bytesize / BYTES_PER_TOKEN.to_f).ceil
+  end
+
+  included do
+    before_validation :set_estimated_token_count, on: :create
+    after_create :schedule_token_count
+  end
+
+  # Heuristic token estimate for this record's {#tokenization_text}.
+  #
+  # @return [Integer]
+  def estimate_tokens
+    TokenEstimation.estimate_token_count(tokenization_text)
+  end
+
+  private
+
+  # Seeds {#token_count} with a local estimate before the record is saved.
+  # Respects an explicit positive value passed by the caller (e.g. tests
+  # that want deterministic counts).
+  def set_estimated_token_count
+    return if token_count.to_i.positive?
+
+    self.token_count = estimate_tokens
+  end
+
+  def schedule_token_count
+    CountTokensJob.perform_later(self)
+  end
+end

data/app/models/goal.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-# A persistent objective tracked by the analytical brain during a session.
+# A persistent objective tracked by Melete during a session.
 # Goals form a two-level hierarchy: root goals represent high-level
 # objectives (semantic episodes), while sub-goals are TODO-style steps
 # rendered as checklist items in the agent's system prompt.
 #
-# The analytical brain creates and completes goals; the main agent sees
+# Melete creates and completes goals; the main agent sees
 # them in its context window but never manages them directly.
 class Goal < ApplicationRecord
   STATUSES = %w[active completed].freeze
@@ -26,17 +26,19 @@ class Goal < ApplicationRecord
   scope :root, -> { where(parent_goal_id: nil) }
 
   # @!method self.not_evicted
-  #   Goals still visible in context (not yet evicted by the analytical brain).
+  #   Goals still visible in context (not yet evicted by Melete).
   #   @return [ActiveRecord::Relation]
   scope :not_evicted, -> { where(evicted_at: nil) }
 
   # @!method self.evictable
-  #   Completed goals pending eviction — visible to the brain for age-based review.
+  #   Completed goals not yet evicted — their phantom pairs remain in the
+  #   sliding window until Mneme compresses them during the eviction cycle.
   #   @return [ActiveRecord::Relation]
   scope :evictable, -> { completed.where(evicted_at: nil) }
 
   after_commit :broadcast_goals_update
-  after_commit :schedule_passive_recall, on: [:create, :update]
+  after_commit :emit_goal_created, on: :create
+  after_commit :emit_goal_updated, on: :update, if: :saved_change_to_description?
 
   # @return [Boolean] true if this goal has been completed
   def completed? = status == "completed"
@@ -55,7 +57,7 @@ class Goal < ApplicationRecord
   # the semantic episode that spawned them has ended.
   #
   # Uses +update_all+ to avoid N per-record +after_commit+ broadcasts;
-  # the caller ({AnalyticalBrain::Tools::FinishGoal}) wraps the whole
+  # the caller ({Melete::Tools::FinishGoal}) wraps the whole
   # operation in a transaction so the root goal's single broadcast
   # includes the cascaded state.
   #
@@ -106,14 +108,24 @@ class Goal < ApplicationRecord
     errors.add(:parent_goal, "cannot nest deeper than two levels")
   end
 
-  # Triggers passive recall when goals change so relevant memories
-  # surface in the viewport automatically.
+  # Announces a freshly created goal so the active drain pipeline can
+  # decide whether Mneme should recall against the updated goal set.
+  # Only {MeleteEnrichmentJob}'s scoped listener observes these events;
+  # outside of a Melete run they are silently dropped.
   #
   # @return [void]
-  def schedule_passive_recall
-    return if session.sub_agent?
+  def emit_goal_created
+    Events::Bus.emit(Events::GoalCreated.new(session_id: session_id, goal_id: id))
+  end
 
-    PassiveRecallJob.perform_later(session_id)
+  # Announces a description-level change to an existing goal. Status-only
+  # updates (finish, cascade, mark_goal_completed) are filtered out by the
+  # +saved_change_to_description?+ guard on the callback — a completed goal
+  # carries no new search seed for Mneme.
+  #
+  # @return [void]
+  def emit_goal_updated
+    Events::Bus.emit(Events::GoalUpdated.new(session_id: session_id, goal_id: id))
   end
 
   # Broadcasts goal changes to all clients subscribed to this session.

data/app/models/message.rb

@@ -7,6 +7,10 @@
 # Not to be confused with {Events::Base} (transient bus signals).
 # Messages persist to SQLite; events flow through the bus and are gone.
 #
+# After commit, emits {Events::MessageCreated} and {Events::MessageUpdated}
+# lifecycle events so subscribers ({Events::Subscribers::MessageBroadcaster},
+# {Events::Subscribers::MnemeScheduler}) can react without coupling.
+#
 # @!attribute message_type
 #   @return [String] one of {TYPES}: system_message, user_message,
 #     agent_message, tool_call, tool_response
@@ -15,41 +19,30 @@
 # @!attribute timestamp
 #   @return [Integer] nanoseconds since epoch (Process::CLOCK_REALTIME)
 # @!attribute token_count
-#   @return [Integer] cached token count for this message's payload (0 until counted)
+#   @return [Integer] token count for this message's payload. Seeded with
+#     a local estimate on create and later refined by {CountTokensJob} using
+#     the real Anthropic tokenizer. Always positive — never zero or nil.
 # @!attribute tool_use_id
 #   @return [String] ID correlating tool_call and tool_response messages
 #     (Anthropic-assigned, or a SecureRandom.uuid fallback when the API returns nil;
 #     required for tool_call and tool_response messages)
 class Message < ApplicationRecord
-  include Message::Broadcasting
+  include TokenEstimation
 
   TYPES = %w[system_message user_message agent_message tool_call tool_response].freeze
   LLM_TYPES = %w[user_message agent_message].freeze
-  CONTEXT_TYPES = %w[system_message user_message agent_message tool_call tool_response].freeze
   CONVERSATION_TYPES = %w[user_message agent_message system_message].freeze
   THINK_TOOL = "think"
-  SPAWN_TOOLS = %w[spawn_subagent spawn_specialist].freeze
-
   # Message types that require a tool_use_id to pair call with response.
   TOOL_TYPES = %w[tool_call tool_response].freeze
 
   ROLE_MAP = {"user_message" => "user", "agent_message" => "assistant"}.freeze
 
-  # Heuristic: average bytes per token for English prose.
-  BYTES_PER_TOKEN = 4
-
   # Synthetic ID for system prompt entries in the TUI message store.
   # Real message IDs are positive integers from the database, so 0
   # is safe for deduplication without collision risk.
   SYSTEM_PROMPT_ID = 0
 
-  # Estimates token count from a byte size using the {BYTES_PER_TOKEN} heuristic.
-  # @param bytesize [Integer] number of bytes
-  # @return [Integer] estimated token count (at least 1)
-  def self.estimate_token_count(bytesize)
-    [(bytesize / BYTES_PER_TOKEN.to_f).ceil, 1].max
-  end
-
   belongs_to :session
   has_many :pinned_messages, dependent: :destroy
 
@@ -59,27 +52,22 @@ class Message < ApplicationRecord
   # Anthropic requires every tool_use to have a matching tool_result with the same ID
   validates :tool_use_id, presence: true, if: -> { message_type.in?(TOOL_TYPES) }
 
-  after_create :schedule_token_count, if: :llm_message?
+  after_create_commit :emit_created_event
+  after_update_commit :emit_updated_event
 
   # @!method self.llm_messages
   #   Messages that represent conversation turns sent to the LLM API.
   #   @return [ActiveRecord::Relation]
   scope :llm_messages, -> { where(message_type: LLM_TYPES) }
 
-  # @!method self.context_messages
-  #   Messages included in the LLM context window (conversation + tool interactions).
+  # @!method self.conversation_or_think
+  #   Conversation messages (user/agent/system) and think tool_calls —
+  #   the messages Mneme treats as boundary-eligible.
   #   @return [ActiveRecord::Relation]
-  scope :context_messages, -> { where(message_type: CONTEXT_TYPES) }
-
-  # @!method self.excluding_spawn_messages
-  #   Excludes spawn_subagent/spawn_specialist tool_call and tool_response messages.
-  #   Used when building parent context for sub-agents — spawn messages cause role
-  #   confusion because the sub-agent sees sibling spawn results and mistakes
-  #   itself for the parent.
-  #   @return [ActiveRecord::Relation]
-  scope :excluding_spawn_messages, -> {
-    where.not("message_type IN (?) AND json_extract(payload, '$.tool_name') IN (?)",
-      TOOL_TYPES, SPAWN_TOOLS)
+  scope :conversation_or_think, -> {
+    where(message_type: CONVERSATION_TYPES)
+      .or(where(message_type: "tool_call")
+        .where("json_extract(payload, '$.tool_name') = ?", THINK_TOOL))
   }
 
   # Maps message_type to the Anthropic Messages API role.
@@ -88,16 +76,6 @@ class Message < ApplicationRecord
     ROLE_MAP.fetch(message_type)
   end
 
-  # @return [Boolean] true if this message represents an LLM conversation turn
-  def llm_message?
-    message_type.in?(LLM_TYPES)
-  end
-
-  # @return [Boolean] true if this message is part of the LLM context window
-  def context_message?
-    message_type.in?(CONTEXT_TYPES)
-  end
-
   # @return [Boolean] true if this is a conversation message (user/agent/system)
   #   or a think tool_call — the messages Mneme treats as "conversation" for boundary tracking
   def conversation_or_think?
@@ -105,23 +83,43 @@ class Message < ApplicationRecord
       (message_type == "tool_call" && payload["tool_name"] == THINK_TOOL)
   end
 
-  # Heuristic token estimate: ~4 bytes per token for English prose.
-  # Tool messages are estimated from the full payload JSON since tool_input
-  # and tool metadata contribute to token count. Messages use content only.
+  # String fed to the token estimator and the remote tokenizer. Tool
+  # messages serialize the full payload as JSON so +tool_name+, +tool_input+,
+  # and +tool_use_id+ contribute to the count; conversation messages use
+  # the content field only.
   #
-  # @return [Integer] estimated token count (at least 1)
-  def estimate_tokens
-    text = if message_type.in?(TOOL_TYPES)
+  # @return [String]
+  def tokenization_text
+    if message_type.in?(TOOL_TYPES)
       payload.to_json
     else
       payload["content"].to_s
     end
-    self.class.estimate_token_count(text.bytesize)
+  end
+
+  # Draper hook: picks the concrete decorator subclass based on
+  # {#message_type}. Overrides {Draper::Decoratable#decorator_class},
+  # which would otherwise default to the abstract {MessageDecorator}
+  # base class. Called implicitly by +message.decorate+.
+  #
+  # @return [Class] a {MessageDecorator} subclass
+  def decorator_class
+    case message_type
+    when "user_message" then UserMessageDecorator
+    when "agent_message" then AgentMessageDecorator
+    when "system_message" then SystemMessageDecorator
+    when "tool_call" then ToolCallDecorator
+    when "tool_response" then ToolResponseDecorator
+    end
   end
 
   private
 
-  def schedule_token_count
-    CountMessageTokensJob.perform_later(id)
+  def emit_created_event
+    Events::Bus.emit(Events::MessageCreated.new(self))
+  end
+
+  def emit_updated_event
+    Events::Bus.emit(Events::MessageUpdated.new(self))
   end
 end