anima-core 1.0.2 → 1.1.1

data/app/decorators/web_get_tool_decorator.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "reverse_markdown"
+require "toon"
+
+# Transforms {Tools::WebGet} responses for LLM consumption by detecting
+# the Content-Type header and applying format-specific conversion.
+#
+# Content-Type maps to a method name via simple string normalization:
+#   "application/json" → {#application_json}
+#   "text/html"        → {#text_html}
+#   "text/plain"       → method_missing → passthrough
+#
+# Adding a new format = adding one method. Unknown types fall through
+# {#method_missing} and pass through unchanged.
+#
+# @example
+#   decorator = WebGetToolDecorator.new
+#   decorator.call(body: "<h1>Hi</h1>", content_type: "text/html")
+#   #=> "[Converted: HTML → Markdown]\n\n# Hi"
+class WebGetToolDecorator < ToolDecorator
+  # HTML elements that carry no useful content for an LLM.
+  NOISE_TAGS = %w[script style nav footer aside form noscript iframe
+    svg header menu menuitem].freeze
+
+  # @param result [Hash] `{body: String, content_type: String}`
+  # @return [String] LLM-optimized content with conversion metadata tag
+  def call(result)
+    return result.to_s unless result.is_a?(Hash) && result.key?(:body)
+
+    body = result[:body].to_s
+    content_type = result[:content_type] || "text/plain"
+    decorated = decorate(body, content_type: content_type)
+
+    assemble(**decorated)
+  end
+
+  # Dispatches to the format-specific method derived from Content-Type.
+  #
+  # @param body [String] raw response body
+  # @param content_type [String] HTTP Content-Type header value
+  # @return [Hash] `{text: String, meta: String|nil}`
+  def decorate(body, content_type:)
+    method_name = content_type.split(";").first.strip.tr("/", "_").tr("-", "_")
+    public_send(method_name, body)
+  end
+
+  # Compresses JSON using TOON (Token-Optimized Object Notation) for
+  # ~40% token savings on typical JSON arrays.
+  #
+  # @param body [String] JSON response body
+  # @return [Hash] `{text: String, meta: String}`
+  def application_json(body)
+    parsed = JSON.parse(body)
+    {text: Toon.encode(parsed), meta: "[Converted: JSON → TOON]"}
+  rescue JSON::ParserError
+    {text: body, meta: nil}
+  end
+
+  # Strips noise elements (scripts, styles, nav, ads) and converts
+  # semantic HTML to Markdown for clean LLM consumption.
+  #
+  # @param body [String] HTML response body
+  # @return [Hash] `{text: String, meta: String}`
+  def text_html(body)
+    markdown = html_to_markdown(body)
+    {text: markdown, meta: "[Converted: HTML → Markdown]"}
+  end
+
+  # Passthrough for unregistered content types.
+  #
+  # @return [Hash] `{text: String, meta: nil}`
+  def method_missing(_method_name, body, *)
+    {text: body, meta: nil}
+  end
+
+  def respond_to_missing?(*, **)
+    true
+  end
+
+  private
+
+  # Strips noise HTML elements then converts to Markdown.
+  #
+  # @param html [String] raw HTML
+  # @return [String] clean Markdown
+  def html_to_markdown(html)
+    doc = Nokogiri::HTML(html)
+    doc.css(NOISE_TAGS.join(", ")).remove
+    clean_html = doc.at("body")&.inner_html || doc.to_html
+    markdown = ReverseMarkdown.convert(clean_html, unknown_tags: :bypass, github_flavored: true)
+    collapse_whitespace(markdown)
+  end
+
+  # Collapses excessive blank lines down to a single blank line.
+  #
+  # @param text [String]
+  # @return [String]
+  def collapse_whitespace(text)
+    text.gsub(/\n{3,}/, "\n\n").strip
+  end
+end

data/app/jobs/agent_request_job.rb

@@ -3,19 +3,30 @@
 # Executes an LLM agent loop as a background job with retry logic
 # for transient failures (network errors, rate limits, server errors).
 #
-# Emits events via {Events::Bus} as it progresses, making results visible
-# to any subscriber (TUI, WebSocket clients). All retry and failure
-# notifications are emitted as {Events::SystemMessage} to avoid polluting
-# the LLM context window.
+# Supports two modes:
 #
-# @example Inline execution (TUI)
-#   AgentRequestJob.perform_now(session.id)
+# **Immediate Persist (event_id provided):** The user event was already
+# persisted and broadcast by the caller (e.g. {SessionChannel#speak}).
+# The job verifies LLM delivery — if the first API call fails, the
+# event is deleted and a {Events::BounceBack} is emitted so clients
+# can restore the text to the input field.
 #
-# @example Background execution (future Brain/TUI separation)
+# **Standard (no event_id):** Processes already-persisted events (e.g.
+# after pending message promotion). Uses ActiveJob retry/discard for
+# error handling.
+#
+# @example Immediate Persist — event already saved by SessionChannel
+#   AgentRequestJob.perform_later(session.id, event_id: 42)
+#
+# @example Standard — pending message processing
 #   AgentRequestJob.perform_later(session.id)
 class AgentRequestJob < ApplicationJob
   queue_as :default
 
+  # ActionCable action signaling clients to prompt for an API token.
+  AUTH_REQUIRED_ACTION = "authentication_required"
+
+  # Standard path only — immediate persist handles its own errors.
   retry_on Providers::Anthropic::TransientError,
     wait: :polynomially_longer, attempts: 5 do |job, error|
     Events::Bus.emit(Events::SystemMessage.new(
@@ -27,40 +38,41 @@ class AgentRequestJob < ApplicationJob
   discard_on ActiveRecord::RecordNotFound
   discard_on Providers::Anthropic::AuthenticationError do |job, error|
     session_id = job.arguments.first
-    # Persistent system message for the event log
     Events::Bus.emit(Events::SystemMessage.new(
       content: "Authentication failed: #{error.message}",
       session_id: session_id
     ))
-    # Transient signal to trigger TUI token setup popup (not persisted)
     ActionCable.server.broadcast(
       "session_#{session_id}",
-      {"action" => "authentication_required", "message" => error.message}
+      {"action" => AUTH_REQUIRED_ACTION, "message" => error.message}
     )
   end
 
   # @param session_id [Integer] ID of the session to process
-  def perform(session_id)
+  # @param event_id [Integer, nil] ID of a pre-persisted user event (triggers delivery verification)
+  def perform(session_id, event_id: nil)
     session = Session.find(session_id)
 
-    # Atomic: only one job processes a session at a time. If another job
-    # is already running, this one exits — the running job will pick up
-    # any pending messages after its current loop completes.
+    # Atomic: only one job processes a session at a time.
     return unless claim_processing(session_id)
 
-    # Run analytical brain BEFORE the main agent on user messages so
-    # activated skills are available for the current response.
     run_analytical_brain_blocking(session)
 
     agent_loop = AgentLoop.new(session: session)
-    loop do
+
+    if event_id
+      deliver_persisted_event(session, event_id, agent_loop)
+    else
       agent_loop.run
+    end
+
+    # Process any pending messages queued while we were busy.
+    loop do
       promoted = session.promote_pending_messages!
       break if promoted == 0
+      agent_loop.run
     end
 
-    # Non-blocking analytical brain run after agent completes —
-    # handles post-response updates (renaming, skill changes).
     session.schedule_analytical_brain!
   ensure
     release_processing(session_id)
@@ -70,6 +82,61 @@ class AgentRequestJob < ApplicationJob
 
   private
 
+  # Verifies LLM delivery for a pre-persisted user event.
+  #
+  # The event was already created and broadcast by the caller, so
+  # the user sees their message immediately. This method makes the
+  # first LLM API call — if it fails, the event is deleted and a
+  # {Events::BounceBack} notifies clients to remove the phantom
+  # message and restore the text to the input field. For
+  # {Providers::Anthropic::AuthenticationError}, an additional
+  # +authentication_required+ broadcast prompts the client to show
+  # the token entry dialog.
+  #
+  # Unlike the standard path (which uses +retry_on+ / +discard_on+),
+  # all errors here are caught and swallowed after emitting a
+  # BounceBack — the job completes normally so ActiveJob does not
+  # retry a message the user will re-send manually.
+  #
+  # After successful delivery, continues the agent loop (tool
+  # execution, subsequent API calls).
+  #
+  # @param session [Session] the conversation session
+  # @param event_id [Integer] database ID of the pre-persisted user event
+  # @param agent_loop [AgentLoop] agent loop instance (reused for continuation)
+  def deliver_persisted_event(session, event_id, agent_loop)
+    event = Event.find_by(id: event_id, session_id: session.id)
+    # Event may have been deleted between SessionChannel#speak and job
+    # execution (e.g. user recalled the message). Exit silently — there
+    # is nothing to deliver or bounce back.
+    return unless event
+
+    content = event.payload["content"]
+
+    begin
+      agent_loop.deliver!
+    rescue => error
+      event.destroy!
+      Events::Bus.emit(Events::BounceBack.new(
+        content: content,
+        error: error.message,
+        session_id: session.id,
+        event_id: event_id
+      ))
+      broadcast_auth_required(session.id, error) if error.is_a?(Providers::Anthropic::AuthenticationError)
+      return
+    end
+
+    agent_loop.run
+  end
+
+  def broadcast_auth_required(session_id, error)
+    ActionCable.server.broadcast(
+      "session_#{session_id}",
+      {"action" => AUTH_REQUIRED_ACTION, "message" => error.message}
+    )
+  end
+
   # Runs the analytical brain synchronously before the main agent loop.
   # Respects the blocking_on_user_message setting and session guards
   # (skips sub-agents and sessions with too few messages).
@@ -88,18 +155,21 @@ class AgentRequestJob < ApplicationJob
 
   # Sets the session's processing flag atomically. Returns true if this
   # job claimed the lock, false if another job already holds it.
+  # Broadcasts the state change to the parent session's HUD.
   def claim_processing(session_id)
-    Session.where(id: session_id, processing: false).update_all(processing: true) == 1
+    claimed = Session.where(id: session_id, processing: false).update_all(processing: true) == 1
+    Session.find_by(id: session_id)&.broadcast_children_update_to_parent if claimed
+    claimed
   end
 
   # Clears the processing flag so the session can accept new jobs.
+  # Broadcasts the state change to the parent session's HUD.
   def release_processing(session_id)
     Session.where(id: session_id).update_all(processing: false)
+    Session.find_by(id: session_id)&.broadcast_children_update_to_parent
   end
 
-  # Safety-net clearing of the interrupt flag. The primary clear happens in
-  # {LLM::Client#clear_interrupt!} after handling the interrupt; this ensures
-  # the flag is reset even if the job crashes before reaching that code path.
+  # Safety-net clearing of the interrupt flag.
   def clear_interrupt(session_id)
     Session.where(id: session_id, interrupt_requested: true).update_all(interrupt_requested: false)
   end

data/app/jobs/mneme_job.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Runs the Mneme memory department — a phantom LLM loop that observes
+# the main session and creates summaries of conversation context before
+# it evicts from the viewport.
+#
+# Triggered when the terminal event ({Session#mneme_boundary_event_id})
+# leaves the viewport, indicating that meaningful context is about to
+# be lost.
+#
+# After L1 snapshot creation, checks whether enough uncovered L1 snapshots
+# have accumulated to trigger Level 2 compression.
+#
+# @example
+#   MnemeJob.perform_later(session.id)
+class MnemeJob < ApplicationJob
+  queue_as :default
+
+  retry_on Providers::Anthropic::TransientError,
+    wait: :polynomially_longer, attempts: 3
+
+  discard_on ActiveRecord::RecordNotFound
+  discard_on Providers::Anthropic::AuthenticationError
+
+  # @param session_id [Integer] the main Session to summarize
+  def perform(session_id)
+    session = Session.find(session_id)
+    log.info("job started for session=#{session_id}")
+    Mneme::Runner.new(session).call
+    check_l2_compression(session)
+  rescue => error
+    log.error("FAILED session=#{session_id}: #{error.class}: #{error.message}")
+    raise
+  end
+
+  private
+
+  # Triggers L2 compression when enough uncovered L1 snapshots accumulate.
+  # Runs inline (same job) since L2 compression is a small, fast LLM call.
+  def check_l2_compression(session)
+    uncovered = session.snapshots.for_level(1).not_covered_by_l2.count
+    threshold = Anima::Settings.mneme_l2_snapshot_threshold
+
+    if uncovered >= threshold
+      log.info("session=#{session.id} — #{uncovered} uncovered L1 snapshots, triggering L2 compression")
+      Mneme::L2Runner.new(session).call
+    end
+  end
+
+  def log = Mneme.logger
+end

data/app/jobs/passive_recall_job.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Runs passive recall after goal updates — searches event history for
+# context relevant to active goals and caches results on the session
+# for viewport injection.
+#
+# Idempotent: multiple enqueues for the same session safely overwrite
+# each other's results — last one wins.
+#
+# @example
+#   PassiveRecallJob.perform_later(session.id)
+class PassiveRecallJob < ApplicationJob
+  queue_as :default
+
+  discard_on ActiveRecord::RecordNotFound
+
+  # @param session_id [Integer]
+  def perform(session_id)
+    session = Session.find(session_id)
+    results = Mneme::PassiveRecall.new(session).call
+
+    if results.any?
+      session.update_column(:recalled_event_ids, results.map(&:event_id))
+      Mneme.logger.info("session=#{session_id} — passive recall found #{results.size} memories")
+    elsif session.recalled_event_ids.present?
+      session.update_column(:recalled_event_ids, [])
+    end
+  end
+end

data/app/models/concerns/event/broadcasting.rb

@@ -76,6 +76,10 @@ module Event::Broadcasting
     evicted_ids = session.recalculate_viewport!
     broadcast_payload["evicted_event_ids"] = evicted_ids if evicted_ids.any?
 
+    # The nil? branch fires on every broadcast until boundary initializes, but
+    # schedule_mneme! returns early after setting the boundary — cost is one DB read + write.
+    session.schedule_mneme! if evicted_ids.any? || session.mneme_boundary_event_id.nil?
+
     ActionCable.server.broadcast("session_#{session_id}", broadcast_payload)
   end
 end

data/app/models/event.rb

@@ -21,6 +21,8 @@ class Event < ApplicationRecord
   TYPES = %w[system_message user_message agent_message tool_call tool_response].freeze
   LLM_TYPES = %w[user_message agent_message].freeze
   CONTEXT_TYPES = %w[system_message user_message agent_message tool_call tool_response].freeze
+  CONVERSATION_TYPES = %w[user_message agent_message system_message].freeze
+  THINK_TOOL = "think"
   PENDING_STATUS = "pending"
 
   ROLE_MAP = {"user_message" => "user", "agent_message" => "assistant"}.freeze
@@ -29,6 +31,7 @@ class Event < ApplicationRecord
   BYTES_PER_TOKEN = 4
 
   belongs_to :session
+  has_many :pinned_events, dependent: :destroy
 
   validates :event_type, presence: true, inclusion: {in: TYPES}
   validates :payload, presence: true
@@ -78,6 +81,13 @@ class Event < ApplicationRecord
     status == PENDING_STATUS
   end
 
+  # @return [Boolean] true if this is a conversation event (user/agent/system message)
+  #   or a think tool_call — the events Mneme treats as "conversation" for boundary tracking
+  def conversation_or_think?
+    event_type.in?(CONVERSATION_TYPES) ||
+      (event_type == "tool_call" && payload["tool_name"] == THINK_TOOL)
+  end
+
   # Heuristic token estimate: ~4 bytes per token for English prose.
   # Tool events are estimated from the full payload JSON since tool_input
   # and tool metadata contribute to token count. Messages use content only.

data/app/models/goal.rb

@@ -13,6 +13,8 @@ class Goal < ApplicationRecord
   belongs_to :session
   belongs_to :parent_goal, class_name: "Goal", optional: true
   has_many :sub_goals, -> { order(:created_at) }, class_name: "Goal", foreign_key: :parent_goal_id, dependent: :destroy
+  has_many :goal_pinned_events, dependent: :destroy
+  has_many :pinned_events, through: :goal_pinned_events
 
   validates :description, presence: true
   validates :status, inclusion: {in: STATUSES}
@@ -24,10 +26,14 @@ class Goal < ApplicationRecord
   scope :root, -> { where(parent_goal_id: nil) }
 
   after_commit :broadcast_goals_update
+  after_commit :schedule_passive_recall, on: [:create, :update]
 
   # @return [Boolean] true if this goal has been completed
   def completed? = status == "completed"
 
+  # @return [Boolean] true if this goal is still active
+  def active? = status == "active"
+
   # @return [Boolean] true if this is a root goal (no parent)
   def root? = !parent_goal_id
 
@@ -46,6 +52,17 @@ class Goal < ApplicationRecord
     sub_goals.active.update_all(status: "completed", completed_at: now, updated_at: now)
   end
 
+  # Releases pinned events that have no remaining active Goal references
+  # anywhere in the session. Called after goal (and cascade) completion —
+  # the orphaned scope checks all Goals, so pins shared with other active
+  # Goals survive automatically via reference counting.
+  #
+  # @return [Integer] number of released pins
+  def release_orphaned_pins!
+    orphaned = session.pinned_events.orphaned
+    orphaned.destroy_all.size
+  end
+
   # Serializes this goal for ActionCable broadcast and TUI display.
   # Includes nested sub-goals for root goals.
   #
@@ -76,6 +93,16 @@ class Goal < ApplicationRecord
     errors.add(:parent_goal, "cannot nest deeper than two levels")
   end
 
+  # Triggers passive recall when goals change so relevant memories
+  # surface in the viewport automatically.
+  #
+  # @return [void]
+  def schedule_passive_recall
+    return if session.sub_agent?
+
+    PassiveRecallJob.perform_later(session_id)
+  end
+
   # Broadcasts goal changes to all clients subscribed to this session.
   # Mirrors the Session#broadcast_active_skills_update pattern so the
   # TUI info panel updates reactively.

data/app/models/goal_pinned_event.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Join record linking a {Goal} to a {PinnedEvent}. Many-to-many: one event
+# can be pinned to multiple Goals, and one Goal can reference multiple pins.
+# When the last Goal referencing a pin completes, the pin is released.
+class GoalPinnedEvent < ApplicationRecord
+  belongs_to :goal
+  belongs_to :pinned_event
+
+  validates :pinned_event_id, uniqueness: {scope: :goal_id}
+end

data/app/models/pinned_event.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# A conversation event pinned to one or more Goals by Mneme to protect it
+# from viewport eviction. Pinned events appear in the Goals section of
+# the viewport, giving the main agent access to critical context that
+# would otherwise scroll out of the sliding window.
+#
+# Pinning is goal-scoped: when all Goals referencing a pin complete,
+# the pin is automatically released (reference-counted cleanup).
+#
+# @!attribute display_text
+#   @return [String] truncated event content (~200 chars) shown in the Goals section
+class PinnedEvent < ApplicationRecord
+  # Display text limit — enough to recognize content, cheap on tokens.
+  MAX_DISPLAY_TEXT_LENGTH = 200
+
+  belongs_to :event
+
+  has_many :goal_pinned_events, dependent: :destroy
+  has_many :goals, through: :goal_pinned_events
+
+  validates :display_text, presence: true, length: {maximum: MAX_DISPLAY_TEXT_LENGTH}
+  validates :event_id, uniqueness: true
+
+  # Pinned events with no remaining active goals — safe to release.
+  #
+  # @return [ActiveRecord::Relation]
+  scope :orphaned, -> {
+    where.not(
+      "EXISTS (SELECT 1 FROM goal_pinned_events gpe " \
+      "JOIN goals ON goals.id = gpe.goal_id " \
+      "WHERE gpe.pinned_event_id = pinned_events.id " \
+      "AND goals.status = 'active')"
+    )
+  }
+
+  # @return [Integer] token cost estimate for viewport budget accounting
+  def token_cost
+    [(display_text.bytesize / Event::BYTES_PER_TOKEN.to_f).ceil, 1].max
+  end
+end