anima-core 1.3.0 → 1.5.0

data/app/decorators/pending_message_decorator.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# Base decorator for {PendingMessage} records, providing multi-resolution
+# rendering for the TUI ("basic" / "verbose" / "debug") and for the
+# enrichment subsystems Mneme and Melete.
+#
+# Each PM type has a dedicated subclass that mirrors the visual treatment
+# of its promoted-{Message} counterpart, with +status: "pending"+ added
+# so the TUI can render it dimmed.
+#
+# Subclasses must override {#render_basic}. Default delegations form a
+# two-step chain: +render_debug → render_verbose → render_basic+. A
+# subclass that only overrides +render_verbose+ inherits its
+# +render_debug+ for free. Melete and Mneme transcript modes return nil
+# by default — subclasses opt in by overriding {#render_melete} or
+# {#render_mneme}.
+#
+# Instantiate via +pending_message.decorate+ — {PendingMessage#decorator_class}
+# picks the concrete subclass based on +message_type+.
+class PendingMessageDecorator < ApplicationDecorator
+  delegate_all
+
+  RENDER_DISPATCH = {
+    "basic" => :render_basic,
+    "verbose" => :render_verbose,
+    "debug" => :render_debug,
+    "melete" => :render_melete,
+    "mneme" => :render_mneme
+  }.freeze
+  private_constant :RENDER_DISPATCH
+
+  # Dispatches to the render method for the given view mode.
+  #
+  # @param mode [String] one of "basic", "verbose", "debug", "melete", "mneme"
+  # @return [Hash, String, nil] structured TUI payload, transcript line, or nil to hide
+  # @raise [ArgumentError] if the mode is not supported
+  def render(mode)
+    method = RENDER_DISPATCH[mode]
+    raise ArgumentError, "Invalid view mode: #{mode.inspect}" unless method
+
+    public_send(method)
+  end
+
+  # @abstract Subclasses must implement to render the pending message for basic view mode.
+  # @return [Hash, nil] structured payload, or nil to hide
+  def render_basic
+    raise NotImplementedError, "#{self.class} must implement #render_basic"
+  end
+
+  # @return [Hash, nil] verbose payload (defaults to basic)
+  def render_verbose
+    render_basic
+  end
+
+  # @return [Hash, nil] debug payload (defaults to verbose)
+  def render_debug
+    render_verbose
+  end
+
+  # @return [String, nil] Melete transcript line, or nil to skip
+  def render_melete
+    nil
+  end
+
+  # @return [String, nil] Mneme transcript line, or nil to skip
+  def render_mneme
+    nil
+  end
+
+  protected
+
+  MIDDLE_TRUNCATION_MARKER = MessageDecorator::MIDDLE_TRUNCATION_MARKER
+
+  # Mirror of {MessageDecorator#truncate_middle} — duplicated here rather than
+  # inherited to keep the two decorator families independent.
+  def truncate_middle(text, max_chars: 500)
+    str = text.to_s
+    return str if str.length <= max_chars
+
+    keep = max_chars - MIDDLE_TRUNCATION_MARKER.length
+    head = keep / 2
+    tail = keep - head
+    "#{str[0, head]}#{MIDDLE_TRUNCATION_MARKER}#{str[-tail, tail]}"
+  end
+
+  # Mirror of {MessageDecorator#truncate_lines}.
+  def truncate_lines(text, max_lines:)
+    str = text.to_s
+    lines = str.split("\n")
+    return str unless lines.size > max_lines
+
+    lines.first(max_lines).push("...").join("\n")
+  end
+end

data/app/decorators/pending_subagent_decorator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Decorates a +subagent+ {PendingMessage} — a sub-agent's reply that
+# landed on the parent's mailbox via {SubagentMessageRouter}. Promotes
+# into a phantom +from_<nickname>+ tool_call/tool_response pair, but
+# while pending it surfaces as a labeled inbound delivery so the user
+# sees which sub-agent is talking to her.
+#
+# Hidden in basic (matches the promoted tool pair, which is hidden in
+# basic). Visible from verbose with a +[from <nickname>]+ badge.
+class PendingSubagentDecorator < PendingMessageDecorator
+  # @return [nil] sub-agent deliveries are hidden in basic mode
+  def render_basic
+    nil
+  end
+
+  # @return [Hash] dimmed sub-agent delivery payload
+  def render_verbose
+    {
+      role: :pending_subagent,
+      source: source_name,
+      content: truncate_lines(content, max_lines: 3),
+      status: "pending"
+    }
+  end
+
+  # @return [Hash] full sub-agent delivery payload
+  def render_debug
+    {
+      role: :pending_subagent,
+      source: source_name,
+      content: content,
+      status: "pending"
+    }
+  end
+
+  # @return [String] Melete transcript line
+  def render_melete
+    "Sub-agent #{source_name} (pending): #{truncate_middle(content)}"
+  end
+
+  # @return [String] Mneme transcript line
+  def render_mneme
+    "Sub-agent #{source_name} (pending): #{truncate_middle(content)}"
+  end
+end

data/app/decorators/pending_tool_response_decorator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Decorates a +tool_response+ {PendingMessage} — a tool result waiting in
+# the mailbox before the drain pairs it with its tool_call and feeds the
+# next LLM turn. Mirrors {ToolResponseDecorator}: hidden in basic
+# (aggregated by the tool counter), structured tool output in verbose,
+# full untruncated content in debug — all dimmed via
+# +status: "pending"+.
+class PendingToolResponseDecorator < PendingMessageDecorator
+  # @return [nil] tool responses are hidden in basic mode
+  def render_basic
+    nil
+  end
+
+  # @return [Hash] truncated tool response payload tagged as pending
+  def render_verbose
+    {
+      role: :tool_response,
+      tool: source_name,
+      content: truncate_lines(content, max_lines: 3),
+      # nil treated as success; only an explicit false flips the indicator
+      # — mirrors {ToolResponseDecorator}'s convention so legacy PMs
+      # without an explicit success column don't render as failures.
+      success: success != false,
+      tool_use_id: tool_use_id,
+      status: "pending"
+    }
+  end
+
+  # @return [Hash] full tool response payload tagged as pending
+  def render_debug
+    {
+      role: :tool_response,
+      tool: source_name,
+      content: content,
+      success: success != false,
+      tool_use_id: tool_use_id,
+      status: "pending"
+    }
+  end
+
+  # @return [String] Melete transcript line
+  def render_melete
+    "tool_response #{tool_use_id} (pending): #{truncate_middle(content)}"
+  end
+
+  # @return [String] Mneme transcript line
+  def render_mneme
+    "tool_response #{tool_use_id} (pending): #{truncate_middle(content)}"
+  end
+end

data/app/decorators/pending_user_message_decorator.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Decorates a +user_message+ {PendingMessage} — the user's input as it
+# sits in the mailbox between submission and promotion. Mirrors
+# {UserMessageDecorator}'s shape, with +status: "pending"+ added so the
+# TUI dims the entry.
+class PendingUserMessageDecorator < PendingMessageDecorator
+  # @return [Hash] dimmed user message payload
+  def render_basic
+    {role: :user, content: content, status: "pending"}
+  end
+
+  # @return [String] Melete transcript line
+  def render_melete
+    "User (pending): #{truncate_middle(content)}"
+  end
+
+  # @return [String] Mneme transcript line
+  def render_mneme
+    "User (pending): #{truncate_middle(content)}"
+  end
+end

data/app/decorators/system_message_decorator.rb

@@ -18,4 +18,9 @@ class SystemMessageDecorator < MessageDecorator
   def render_debug
     render_verbose
   end
+
+  # @return [String] transcript line for Mneme's eviction/context zones
+  def render_mneme
+    "message #{id} System: #{content}"
+  end
 end

data/app/decorators/tool_call_decorator.rb

@@ -50,8 +50,8 @@ class ToolCallDecorator < MessageDecorator
 
   # Think calls get full text — the agent's reasoning IS the signal.
   # Other tool calls show tool name + params (compact JSON).
-  # @return [String] transcript line for the analytical brain
-  def render_brain
+  # @return [String] transcript line for Melete
+  def render_melete
     if think?
       "Think: #{thoughts}"
     else
@@ -59,6 +59,17 @@ class ToolCallDecorator < MessageDecorator
     end
   end
 
+  # Think calls render as conversation. Regular tool calls return
+  # a +:tool_call+ marker for the counter accumulator.
+  # @return [String, Symbol] transcript line or counter marker
+  def render_mneme
+    if think?
+      "message #{id} Think: #{thoughts}"
+    else
+      :tool_call
+    end
+  end
+
   private
 
   def think?
@@ -105,10 +116,10 @@ class ToolCallDecorator < MessageDecorator
   # Formats write tool input with file path header and content body.
   # Content newlines are preserved so the TUI can render them as
   # separate lines, matching how read_file tool responses display file content.
-  # @param input [Hash] tool input hash with "file_path" and "content" keys
+  # @param input [Hash] tool input hash with "path" and "content" keys
   # @return [String] path + content with real newlines, or TOON-encoded hash when content is empty
   def format_write_content(input)
-    path = input.dig("file_path").to_s
+    path = input.dig("path").to_s
     content = input.dig("content").to_s
     return Toon.encode(input) if content.empty?
 
@@ -126,7 +137,7 @@ class ToolCallDecorator < MessageDecorator
     when "web_get"
       "GET #{input&.dig("url")}"
     when "read_file", "edit_file", "write_file"
-      input&.dig("file_path").to_s
+      input&.dig("path").to_s
     else
       truncate_lines(Toon.encode(input), max_lines: 2)
     end

data/app/decorators/tool_response_decorator.rb

@@ -46,10 +46,10 @@ class ToolResponseDecorator < MessageDecorator
     }.merge(token_info)
   end
 
-  # Think responses ("OK") are noise — excluded from the brain's transcript.
+  # Think responses ("OK") are noise — excluded from Melete's transcript.
   # Other tool responses are compressed to success/failure indicators only.
   # @return [String, nil] ✅ or ❌ indicator, nil for think responses
-  def render_brain
+  def render_melete
     return if think?
 
     (payload["success"] != false) ? "\u2705" : "\u274C"

data/app/decorators/user_message_decorator.rb

@@ -19,9 +19,14 @@ class UserMessageDecorator < MessageDecorator
     render_verbose.merge(token_info)
   end
 
-  # @return [String] user message for the analytical brain, middle-truncated
+  # @return [String] user message for Melete, middle-truncated
   #   if very long (preserves intent at start and conclusion at end)
-  def render_brain
+  def render_melete
     "User: #{truncate_middle(content)}"
   end
+
+  # @return [String] transcript line for Mneme's eviction/context zones
+  def render_mneme
+    "message #{id} User: #{content}"
+  end
 end

data/app/jobs/count_tokens_job.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Refines a record's +token_count+ with the real Anthropic tokenizer count,
+# replacing the local heuristic seeded during creation. Accepts any record
+# that includes {TokenEstimation} and implements +#tokenization_text+ —
+# Messages, Snapshots, and PinnedMessages share the same pipeline.
+class CountTokensJob < ApplicationJob
+  queue_as :default
+
+  retry_on Providers::Anthropic::Error, wait: :polynomially_longer, attempts: 3
+  discard_on ActiveRecord::RecordNotFound
+
+  # @param record [ActiveRecord::Base] any record responding to
+  #   +#tokenization_text+ and +token_count=+
+  def perform(record)
+    count = Providers::Anthropic.new.count_tokens(
+      model: Anima::Settings.model,
+      messages: [{role: "user", content: record.tokenization_text}]
+    )
+
+    record.update!(token_count: count)
+  end
+end

data/app/jobs/drain_job.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# Drains the PendingMessage mailbox into a single LLM round-trip.
+#
+# One invocation == one half-step of the event-driven agent loop:
+# 1. Claim the session via {Session#start_processing!} (atomic; bails if
+#    another drain already holds the session OR the in-flight tool round
+#    is incomplete — the AASM guard +tool_round_complete?+ handles both).
+# 2. Promote pending work into the conversation — every tool_response PM
+#    of the freshly completed round flushes in one transaction so the
+#    LLM sees a whole assistant turn paired with a whole user turn;
+#    background phantom pairs flush; one active FIFO message rides
+#    along. Promotion lives on {PendingMessage#promote!} — the job only
+#    decides what to pick and in which order.
+# 3. Make one LLM API call and emit {Events::LLMResponded}.
+#
+# On the happy path the job never releases the session — state
+# transitions after the emit belong to
+# {Events::Subscribers::LLMResponseHandler} (on text or tool dispatch).
+# {Events::Subscribers::ToolResponseCreator} no longer touches state;
+# the +executing → awaiting+ branch of +start_processing+ closes the
+# tool round and claims in one atomic, lock-protected step.
+#
+# The job DOES release its own claim when there is no responder to do
+# it: an empty mailbox (spurious kickoff) or an exception raised before
+# the LLM call succeeded. Those are lifecycle edges of the claim itself,
+# not hand-offs to responders.
+#
+# @see Events::Subscribers::DrainKickoff — enqueues this job
+# @see Events::LLMResponded — the event emitted on LLM completion
+class DrainJob < ApplicationJob
+  queue_as :default
+
+  # Transient provider errors retry inline within {#call_llm_and_emit}.
+  # A job-level +retry_on+ would be a no-op here: {PendingMessage#promote!}
+  # destroys the PM rows *before* the LLM call, so a retried job would find
+  # an empty mailbox and exit without ever re-issuing the request.
+
+  discard_on Providers::Anthropic::AuthenticationError do |job, error|
+    Events::Bus.emit(Events::AuthenticationRequired.new(
+      session_id: job.arguments.first,
+      content: error.message
+    ))
+  end
+
+  discard_on ActiveRecord::RecordNotFound
+
+  # @param session_id [Integer]
+  def perform(session_id)
+    @session = Session.find(session_id)
+    return unless @session.start_processing!
+
+    drained = drain_mailbox
+    return @session.response_complete! if drained.zero?
+
+    call_llm_and_emit
+  rescue Providers::Anthropic::AuthenticationError => error
+    release_after_failure(error) if @session
+    raise
+  rescue => error
+    release_after_failure(error) if @session
+    raise unless @active_pm&.bounce_back?
+  end
+
+  private
+
+  # Decides what the upcoming LLM round will carry and promotes those PMs.
+  #
+  # 1. All +tool_response+ PMs of the just-completed round — the AASM
+  #    guard guarantees they are all present when we arrive from
+  #    +:executing+; from +:idle+ this set is empty.
+  # 2. Pick one active FIFO message (user_message or subagent) to ride
+  #    along. If there are no tool_responses AND no active message, the
+  #    LLM call is a no-op: release the claim and do NOT flush background
+  #    PMs (they stay in the mailbox for the next turn).
+  # 3. Flush background phantom pairs so they sit above the active pick.
+  # 4. Promote the active pick.
+  #
+  # @return [Integer] count of PMs promoted this cycle (0 means "release
+  #   the claim without calling the LLM")
+  def drain_mailbox
+    tool_responses = @session.pending_messages.where(message_type: "tool_response").order(:created_at).to_a
+    @active_pm = @session.pending_messages.active.where.not(message_type: "tool_response").order(:created_at).first
+
+    promoted = tool_responses.size + (@active_pm ? 1 : 0)
+    return 0 if promoted.zero?
+
+    tool_responses.each(&:promote!)
+    @session.pending_messages.background.find_each(&:promote!)
+    @active_pm&.promote!
+
+    promoted
+  end
+
+  def call_llm_and_emit
+    prompt = @session.system_prompt
+    @session.broadcast_debug_context(system: prompt, tools: registry.schemas)
+
+    response = with_transient_retry do
+      client.provider.create_message(
+        model: client.model,
+        messages: @session.messages_for_llm,
+        max_tokens: client.max_tokens,
+        tools: registry.schemas,
+        include_metrics: true,
+        **(prompt ? {system: prompt} : {})
+      )
+    end
+
+    Events::Bus.emit(Events::LLMResponded.new(
+      session_id: @session.id,
+      response: response.to_h.stringify_keys,
+      api_metrics: response.api_metrics
+    ))
+  end
+
+  # Retries the LLM call in-place on transient provider errors. The
+  # polynomial-backoff formula mirrors ActiveJob's +:polynomially_longer+.
+  # On final exhaustion the subscriber-visible SystemMessage is emitted
+  # before the error re-raises into {#perform}'s rescue path for release.
+  TRANSIENT_RETRY_ATTEMPTS = 5
+
+  def with_transient_retry
+    tries = 0
+    begin
+      yield
+    rescue Providers::Anthropic::TransientError => error
+      tries += 1
+      if tries >= TRANSIENT_RETRY_ATTEMPTS
+        Events::Bus.emit(Events::SystemMessage.new(
+          content: "Failed after multiple retries: #{error.message}",
+          session_id: @session.id
+        ))
+        raise
+      end
+      sleep(transient_backoff(tries))
+      retry
+    end
+  end
+
+  def transient_backoff(attempt)
+    base = attempt**4
+    base + (rand * 0.15 * base)
+  end
+
+  def release_after_failure(error)
+    if @active_pm&.bounce_back?
+      @session.release_with_bounce_back(@active_pm, error)
+    elsif @session.may_response_complete?
+      @session.response_complete!
+    end
+  end
+
+  def client
+    @client ||= if @session.sub_agent?
+      LLM::Client.new(model: Anima::Settings.subagent_model)
+    else
+      LLM::Client.new
+    end
+  end
+
+  def registry
+    @registry ||= Tools::Registry.build(session: @session, shell_session: shell_session)
+  end
+
+  def shell_session
+    @shell_session ||= ShellSession.for_session(@session)
+  end
+end

data/app/jobs/melete_enrichment_job/goal_change_listener.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class MeleteEnrichmentJob < ApplicationJob
+  # Scoped subscriber that watches the event bus for goal mutation events
+  # (+anima.goal.created+, +anima.goal.updated+) belonging to one session,
+  # for the duration of a block.
+  #
+  # Returns +true+ from {.observe} when at least one matching event fired
+  # during the block, +false+ otherwise. The subscription is registered
+  # before the block runs and removed in an +ensure+ so it is cleaned up
+  # even if the block raises.
+  #
+  # @example
+  #   goal_changed = GoalChangeListener.observe(session_id: 42) do
+  #     Melete::Runner.new(session).call
+  #   end
+  class GoalChangeListener
+    EVENT_NAMES = [
+      "#{Events::Bus::NAMESPACE}.#{Events::GoalCreated::TYPE}",
+      "#{Events::Bus::NAMESPACE}.#{Events::GoalUpdated::TYPE}"
+    ].freeze
+
+    # @param session_id [Integer] only events whose payload session_id matches count
+    # @yield runs while the subscription is active
+    # @return [Boolean] whether a matching event fired during the block
+    def self.observe(session_id:, &block)
+      new(session_id).observe(&block)
+    end
+
+    def initialize(session_id)
+      @session_id = session_id
+      @triggered = false
+    end
+
+    def observe
+      Events::Bus.subscribe(self) do |event|
+        EVENT_NAMES.include?(event[:name]) &&
+          event[:payload][:session_id] == @session_id
+      end
+      yield
+      @triggered
+    ensure
+      Events::Bus.unsubscribe(self)
+    end
+
+    # Bus subscriber contract — flips the latch on any matching event.
+    # @api private
+    def emit(_event)
+      @triggered = true
+    end
+  end
+end

data/app/jobs/melete_enrichment_job.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# First stage of the drain pipeline: runs Melete to activate skills,
+# read workflows, and refine goals. Hands off to either Mneme (when goals
+# changed during this run) or directly to the drain loop.
+#
+# Triggered by {Events::Subscribers::MeleteKickoff} in response to
+# {Events::StartMelete}. Runs the existing synchronous {Melete::Runner}
+# — the event is only the entry/exit plumbing.
+#
+# A {GoalChangeListener} observes {Events::GoalCreated} and
+# {Events::GoalUpdated} for the duration of the runner call. When a goal
+# mutation is heard the job emits {Events::StartMneme} so Mneme recalls
+# against the fresh goal set; otherwise it emits {Events::StartProcessing}
+# and Mneme is skipped — there is no new search seed to recall against.
+#
+# Sub-agents skip Melete entirely (sub-agent nickname assignment is a
+# one-time step, not part of the recurring pipeline). With no runner
+# call, the listener never fires and the job falls through to
+# {Events::StartProcessing}.
+#
+# Exceptions from {Melete::Runner#call} propagate — no defensive rescue.
+# A crashed Melete leaves the session idle with the PM still in the
+# mailbox; the next PM create (or idle-wake re-route) retries the full
+# chain. Swallowing would surface a degraded response to the user without
+# the failure being visible anywhere (anti-pattern per the project's
+# "soft error paths" principle).
+class MeleteEnrichmentJob < ApplicationJob
+  queue_as :default
+
+  discard_on ActiveRecord::RecordNotFound
+
+  # @param session_id [Integer]
+  # @param pending_message_id [Integer, nil] the PM that kicked off the chain
+  def perform(session_id, pending_message_id: nil)
+    session = Session.find(session_id)
+
+    goal_changed = GoalChangeListener.observe(session_id: session_id) do
+      Melete::Runner.new(session).call unless session.sub_agent?
+    end
+
+    next_event_class = goal_changed ? Events::StartMneme : Events::StartProcessing
+    Events::Bus.emit(next_event_class.new(
+      session_id: session_id,
+      pending_message_id: pending_message_id
+    ))
+  end
+end

data/app/jobs/mneme_enrichment_job.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Second stage of the drain pipeline: runs Mneme's recall loop so any
+# older memory she judges useful lands in the mailbox as background
+# {PendingMessage}s, then hands off to the drain loop via
+# {Events::StartProcessing}.
+#
+# Triggered by {Events::Subscribers::MnemeKickoff} in response to
+# {Events::StartMneme}, which {MeleteEnrichmentJob} only emits when goals
+# changed during the preceding Melete run. Runs the phantom
+# {Mneme::RecallRunner} loop — the event is only the entry/exit plumbing.
+#
+# Mneme recall is *enrichment* — it adds recalled memories as background
+# phantom pairs but is never required for the primary pipeline to make
+# progress. If recall raises (bad FTS5 input, SQL glitch, …) the handoff
+# to the drain loop must still happen, otherwise the session's user
+# message is stranded in the mailbox with no retry trigger. Exceptions
+# are logged loudly so failures stay visible — they just don't gate the drain.
+class MnemeEnrichmentJob < ApplicationJob
+  queue_as :default
+
+  discard_on ActiveRecord::RecordNotFound
+
+  # @param session_id [Integer]
+  # @param pending_message_id [Integer, nil] the PM that kicked off the chain
+  def perform(session_id, pending_message_id: nil)
+    session = Session.find(session_id)
+
+    run_recall(session)
+
+    Events::Bus.emit(Events::StartProcessing.new(
+      session_id: session_id,
+      pending_message_id: pending_message_id
+    ))
+  end
+
+  private
+
+  def run_recall(session)
+    Mneme::RecallRunner.new(session).call
+  rescue => error
+    msg = "FAILED (recall) session=#{session.id}: #{error.class}: #{error.message}"
+    Rails.logger.error("Mneme #{msg}")
+    Mneme.logger.error("#{msg}\n#{error.backtrace&.first(10)&.join("\n")}")
+  end
+end