anima-core 1.0.2 → 1.1.0

data/lib/mneme/passive_recall.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Mneme
+  # Passive recall — automatic memory surfacing triggered by Goal updates.
+  # When goals are created or updated, searches event history for related
+  # context and caches the results on the session for viewport injection.
+  #
+  # The agent never calls a tool; relevant memories appear automatically
+  # in the viewport between snapshots and the sliding window. This mirrors
+  # recognition memory in humans — context surfaces without conscious effort.
+  #
+  # @example Trigger after a goal update
+  #   Mneme::PassiveRecall.new(session).call
+  class PassiveRecall
+    # @param session [Session] the session whose goals drive recall
+    def initialize(session)
+      @session = session
+    end
+
+    # Searches event history using active goal descriptions as queries.
+    # Returns recall results suitable for viewport injection.
+    #
+    # @return [Array<Mneme::Search::Result>] deduplicated, relevance-sorted
+    def call
+      goals = @session.goals.active.root.includes(:sub_goals)
+      return [] if goals.empty?
+
+      search_terms = build_search_terms(goals)
+      return [] if search_terms.blank?
+
+      results = Mneme::Search.query(search_terms, limit: Anima::Settings.recall_max_results)
+
+      # Exclude events from the current session's viewport — no point recalling
+      # what the agent already sees.
+      viewport_ids = @session.viewport_event_ids.to_set
+      results.reject { |result| viewport_ids.include?(result.event_id) }
+    end
+
+    private
+
+    STOP_WORDS = Set.new(%w[
+      a an the is are was were be been being do does did
+      have has had in on at to for of and or but not with
+      this that it its by from as up out if about into
+      fix add create update remove implement check set get
+    ]).freeze
+
+    # Extracts meaningful keywords from active goals and joins with OR.
+    # Stop words and generic verbs are stripped — they're too common to
+    # produce useful recall results.
+    #
+    # @param goals [ActiveRecord::Relation<Goal>]
+    # @return [String] FTS5 OR-joined keywords
+    def build_search_terms(goals)
+      descriptions = goals.flat_map { |goal|
+        [goal.description] + goal.sub_goals.reject(&:completed?).map(&:description)
+      }
+
+      words = descriptions.join(" ")
+        .gsub(/[^a-zA-Z0-9\s-]/, "")
+        .downcase
+        .split
+        .uniq
+        .reject { |word| STOP_WORDS.include?(word) || word.length < 3 }
+
+      words.join(" OR ").truncate(500)
+    end
+  end
+end

data/lib/mneme/runner.rb

@@ -0,0 +1,254 @@
+# frozen_string_literal: true
+
+module Mneme
+  # Orchestrates the Mneme memory department — a phantom (non-persisted) LLM loop
+  # that observes a main session's compressed viewport and creates summaries of
+  # conversation context before it evicts from the viewport.
+  #
+  # Mneme is triggered when the terminal event (`mneme_boundary_event_id`) leaves
+  # the viewport. It receives a compressed viewport (no raw tool calls, zone
+  # delimiters present) and uses the `save_snapshot` tool to persist a summary.
+  #
+  # After completing, Mneme advances the terminal event to the boundary of what
+  # it just summarized, so the cycle repeats as more events accumulate.
+  #
+  # @example
+  #   Mneme::Runner.new(session).call
+  class Runner
+    TOOLS = [
+      Tools::SaveSnapshot,
+      Tools::AttachEventsToGoals,
+      Tools::EverythingOk
+    ].freeze
+
+    SYSTEM_PROMPT = <<~PROMPT
+      You are Mneme, the memory department of an AI agent named Anima.
+      Your job is to create concise summaries of conversation context that is
+      about to leave the agent's context window.
+
+      You MUST ONLY communicate through tool calls — NEVER output text.
+
+      ──────────────────────────────
+      WHAT YOU SEE
+      ──────────────────────────────
+      A compressed viewport with three zones:
+      - EVICTION ZONE: Events about to leave the viewport. Summarize these.
+      - MIDDLE ZONE: Events still visible but aging. Note key context.
+      - RECENT ZONE: Fresh events. Use for continuity with the summary.
+
+      Events are prefixed with `event N` (their database ID).
+      Tool calls are compressed to `[N tools called]` — the mechanical work
+      is not important, only the conversation flow.
+
+      ──────────────────────────────
+      YOUR TASK
+      ──────────────────────────────
+      1. Read the eviction zone carefully.
+      2. If it contains meaningful conversation (decisions, goals, context):
+         Call save_snapshot with a concise summary.
+      3. If any events in the eviction zone are too important to summarize
+         (exact user instructions, critical corrections, key decisions),
+         pin them to active goals with attach_events_to_goals.
+         Pinned events survive eviction intact — use this sparingly for
+         events where the exact wording matters.
+      4. If it contains only mechanical activity with no conversation:
+         Call everything_ok.
+
+      You may call BOTH save_snapshot AND attach_events_to_goals in one turn
+      when the zone has a mix of summarizable and pin-worthy events.
+
+      Write summaries that capture:
+      - What was discussed and decided
+      - Why decisions were made
+      - Active goals and their progress
+      - Key context the agent would need later
+
+      Do NOT include:
+      - Tool call details (which files were read, commands run)
+      - Mechanical execution steps
+      - Verbatim quotes (paraphrase instead)
+
+      Always finish with at least one tool call: save_snapshot, attach_events_to_goals,
+      or everything_ok. You may combine save_snapshot with attach_events_to_goals.
+    PROMPT
+
+    # @param session [Session] the main session to observe
+    # @param client [LLM::Client, nil] injectable LLM client (defaults to fast model)
+    def initialize(session, client: nil)
+      @session = session
+      @client = client || LLM::Client.new(
+        model: Anima::Settings.fast_model,
+        max_tokens: Anima::Settings.mneme_max_tokens,
+        logger: Mneme.logger
+      )
+    end
+
+    # Runs the Mneme loop: builds compressed viewport, calls LLM, executes
+    # snapshot tool, then advances the terminal event pointer.
+    #
+    # @return [String, nil] the LLM's final text response (discarded),
+    #   or nil if no context is available
+    def call
+      viewport = build_compressed_viewport
+      compressed_text = viewport.render
+      sid = @session.id
+
+      if compressed_text.empty?
+        log.debug("session=#{sid} — no events for Mneme, skipping")
+        return
+      end
+
+      messages = build_messages(compressed_text)
+      system = SYSTEM_PROMPT
+
+      log.info("session=#{sid} — running Mneme (#{viewport.events.size} events)")
+      log.debug("compressed viewport:\n#{compressed_text}")
+
+      result = @client.chat_with_tools(
+        messages,
+        registry: build_registry(viewport),
+        session_id: nil,
+        system: system
+      )
+
+      advance_boundary(viewport)
+      log.info("session=#{sid} — Mneme done: #{result.to_s.truncate(200)}")
+      result
+    end
+
+    private
+
+    # Builds the compressed viewport starting from the session's boundary event.
+    #
+    # @return [Mneme::CompressedViewport]
+    def build_compressed_viewport
+      token_budget = (Anima::Settings.token_budget * Anima::Settings.mneme_viewport_fraction).to_i
+
+      CompressedViewport.new(
+        @session,
+        token_budget: token_budget,
+        from_event_id: @session.mneme_boundary_event_id
+      )
+    end
+
+    # Frames the compressed viewport as a user message for the LLM.
+    #
+    # @param compressed_text [String] the rendered compressed viewport
+    # @return [Array<Hash>] single-element messages array
+    def build_messages(compressed_text)
+      goals_context = active_goals_section
+
+      content = <<~MSG.strip
+        Here is the compressed viewport of the main session:
+
+        #{compressed_text}
+        #{goals_context}
+        Review the eviction zone and decide whether to save a snapshot or signal everything_ok.
+      MSG
+
+      [{role: "user", content: content}]
+    end
+
+    # Builds the tool registry with session context for SaveSnapshot.
+    # Passes the event range from the viewport so the snapshot records
+    # which events it covers.
+    #
+    # @param viewport [Mneme::CompressedViewport]
+    # @return [Tools::Registry]
+    def build_registry(viewport)
+      viewport_events = viewport.events
+      registry = ::Tools::Registry.new(context: {
+        main_session: @session,
+        from_event_id: viewport_events.first&.id,
+        to_event_id: viewport_events.last&.id
+      })
+      TOOLS.each { |tool| registry.register(tool) }
+      registry
+    end
+
+    # Advances the terminal event pointer after Mneme completes.
+    # Runs unconditionally — even when the LLM called `everything_ok` (no snapshot
+    # needed), the zone was reviewed and should be advanced past. Without this,
+    # Mneme would re-examine the same mechanical-only content on every trigger.
+    #
+    # Sets it to the last conversation event in the viewport, ensuring
+    # the boundary is always a message/think event, never a tool_call/tool_response.
+    # Also updates the snapshot range pointers.
+    #
+    # @param viewport [Mneme::CompressedViewport]
+    def advance_boundary(viewport)
+      viewport_events = viewport.events
+      return if viewport_events.empty?
+
+      new_boundary = viewport_events.reverse_each.find { |event| conversation_or_think?(event) }
+      return unless new_boundary
+
+      boundary_id = new_boundary.id
+      updates = {mneme_boundary_event_id: boundary_id}
+
+      updates[:mneme_snapshot_first_event_id] = viewport_events.first.id if @session.mneme_snapshot_first_event_id.nil?
+      updates[:mneme_snapshot_last_event_id] = viewport_events.last.id
+
+      @session.update_columns(updates)
+      log.debug("session=#{@session.id} — boundary advanced to event #{boundary_id}")
+    end
+
+    # Delegates to {Event#conversation_or_think?} — single source of truth
+    # for which events Mneme treats as conversation boundaries.
+    #
+    # @return [Boolean]
+    def conversation_or_think?(event)
+      event.conversation_or_think?
+    end
+
+    # Builds the active goals section for Mneme's context so it knows
+    # what Goals exist, which events are already pinned, and can reference
+    # them when deciding what to pin or summarize.
+    #
+    # @return [String] formatted goals section, or empty string
+    def active_goals_section
+      root_goals = @session.goals.root.includes(:sub_goals).active.order(:created_at)
+      return "" if root_goals.empty?
+
+      lines = root_goals.map { |goal| format_goal_for_mneme(goal) }
+      pinned = format_existing_pins
+
+      section = "\n\n🎯 Active Goals\n#{lines.join("\n")}\n"
+      section += "\n📌 Already Pinned\n#{pinned}\n" if pinned
+      section
+    end
+
+    # Formats a goal with sub-goals for Mneme's context.
+    #
+    # @param goal [Goal] root goal with preloaded sub_goals
+    # @return [String]
+    def format_goal_for_mneme(goal)
+      parts = ["  ● #{goal.description} (id: #{goal.id})"]
+      goal.sub_goals.each do |sub|
+        checkbox = sub.completed? ? "[x]" : "[ ]"
+        parts << "    #{checkbox} #{sub.description} (id: #{sub.id})"
+      end
+      parts.join("\n")
+    end
+
+    # Lists already-pinned event IDs so Mneme avoids redundant pinning.
+    #
+    # @return [String, nil] formatted pin list, or nil when nothing is pinned
+    def format_existing_pins
+      pins = @session.pinned_events.includes(:goals).order(:event_id)
+      return nil if pins.empty?
+
+      pins.map { |pin| format_pin_for_mneme(pin) }.join("\n")
+    end
+
+    # @param pin [PinnedEvent] pin with preloaded goals
+    # @return [String] formatted pin line
+    def format_pin_for_mneme(pin)
+      goal_ids = pin.goals.map(&:id).join(", ")
+      "  event #{pin.event_id} → goals [#{goal_ids}]"
+    end
+
+    # @return [Logger]
+    def log = Mneme.logger
+  end
+end

data/lib/mneme/search.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Mneme
+  # Full-text search over event history using SQLite FTS5.
+  # Covers user messages, agent messages, and think events across all sessions.
+  #
+  # The interface is intentionally abstract — callers receive {Result} structs
+  # and never touch FTS5 directly. A future semantic search backend (embeddings,
+  # BM25 + re-ranking) can replace the implementation without changing callers.
+  #
+  # @example Search across all sessions
+  #   results = Mneme::Search.query("authentication flow")
+  #   results.each { |r| puts "event #{r.event_id}: #{r.snippet}" }
+  #
+  # @example Search within a single session
+  #   results = Mneme::Search.query("OAuth config", session_id: 42)
+  class Search
+    # A single search result with enough context for display and drill-down.
+    #
+    # @!attribute event_id [Integer] the event's database ID
+    # @!attribute session_id [Integer] the session owning this event
+    # @!attribute snippet [String] highlighted excerpt from the matching content
+    # @!attribute rank [Float] FTS5 relevance score (lower = more relevant)
+    # @!attribute event_type [String] one of Event::TYPES
+    Result = Struct.new(:event_id, :session_id, :snippet, :rank, :event_type, keyword_init: true)
+
+    # Searches event history for the given terms.
+    #
+    # @param terms [String] search query (FTS5 syntax: words, phrases, OR/AND/NOT)
+    # @param session_id [Integer, nil] scope to a specific session (nil = all sessions)
+    # @param limit [Integer] maximum results
+    # @return [Array<Result>] ranked by relevance (best first)
+    def self.query(terms, session_id: nil, limit: Anima::Settings.recall_max_results)
+      new(terms, session_id: session_id, limit: limit).call
+    end
+
+    def initialize(terms, session_id: nil, limit: 5)
+      @terms = sanitize_query(terms)
+      @session_id = session_id
+      @limit = limit
+    end
+
+    # @return [Array<Result>] ranked by relevance (best first)
+    def call
+      return [] if @terms.blank?
+
+      rows = execute_fts_query
+      rows.map { |row| build_result(row) }
+    end
+
+    private
+
+    # Executes the FTS5 MATCH query with optional session scoping.
+    # Joins back to events table for session_id and event_type.
+    #
+    # @return [Array<Hash>] raw database rows
+    def execute_fts_query
+      if @session_id
+        connection.select_all(scoped_sql, "Mneme::Search", [@terms, @session_id, @limit]).to_a
+      else
+        connection.select_all(global_sql, "Mneme::Search", [@terms, @limit]).to_a
+      end
+    end
+
+    # FTS5 query across all sessions.
+    # Contentless FTS5 can't use snippet() — extract content from events directly.
+    def global_sql
+      <<~SQL
+        SELECT
+          e.id AS event_id,
+          e.session_id,
+          e.event_type,
+          CASE
+            WHEN e.event_type IN ('user_message', 'agent_message', 'system_message')
+              THEN substr(json_extract(e.payload, '$.content'), 1, 300)
+            WHEN e.event_type = 'tool_call'
+              THEN substr(json_extract(e.payload, '$.tool_input.thoughts'), 1, 300)
+          END AS snippet,
+          rank
+        FROM events_fts
+        JOIN events e ON e.id = events_fts.rowid
+        WHERE events_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      SQL
+    end
+
+    # FTS5 query scoped to a specific session.
+    def scoped_sql
+      <<~SQL
+        SELECT
+          e.id AS event_id,
+          e.session_id,
+          e.event_type,
+          CASE
+            WHEN e.event_type IN ('user_message', 'agent_message', 'system_message')
+              THEN substr(json_extract(e.payload, '$.content'), 1, 300)
+            WHEN e.event_type = 'tool_call'
+              THEN substr(json_extract(e.payload, '$.tool_input.thoughts'), 1, 300)
+          END AS snippet,
+          rank
+        FROM events_fts
+        JOIN events e ON e.id = events_fts.rowid
+        WHERE events_fts MATCH ?
+          AND e.session_id = ?
+        ORDER BY rank
+        LIMIT ?
+      SQL
+    end
+
+    # Builds a Result from a raw database row.
+    #
+    # @param row [Hash]
+    # @return [Result]
+    def build_result(row)
+      Result.new(
+        event_id: row["event_id"],
+        session_id: row["session_id"],
+        snippet: row["snippet"],
+        rank: row["rank"],
+        event_type: row["event_type"]
+      )
+    end
+
+    # Sanitizes user input for FTS5 MATCH safety.
+    # Strips special FTS5 operators that could cause syntax errors,
+    # keeps only alphanumeric words and quoted phrases.
+    #
+    # @param raw [String]
+    # @return [String] safe FTS5 query
+    def sanitize_query(raw)
+      return "" unless raw
+
+      # Extract quoted phrases and individual words, drop FTS5 operators
+      tokens = raw.scan(/"[^"]+?"|\S+/).reject { |token| token.match?(/\A[*:^{}()]+\z/) }
+      tokens.filter_map { |token| sanitize_token(token) }.join(" ")
+    end
+
+    def sanitize_token(token)
+      return token if token.start_with?('"')
+
+      cleaned = token.gsub(/[^a-zA-Z0-9-]/, "")
+      cleaned.empty? ? nil : cleaned
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/mneme/tools/attach_events_to_goals.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Mneme
+  module Tools
+    # Pins critical events to active Goals so they survive viewport eviction.
+    # Mneme calls this when it sees important events (user instructions, key
+    # decisions, critical corrections) approaching the eviction zone.
+    #
+    # Events are pinned via a many-to-many join: one event can be attached
+    # to multiple Goals. When all referencing Goals complete, the pin is
+    # automatically released (reference-counted cleanup in {Goal#release_orphaned_pins!}).
+    class AttachEventsToGoals < ::Tools::Base
+      def self.tool_name = "attach_events_to_goals"
+
+      def self.description = "Pin critical events to active goals so they survive " \
+        "viewport eviction. Use this for events that are too important to lose — " \
+        "exact user instructions, key decisions, critical corrections. " \
+        "Events stay pinned until all attached goals complete."
+
+      def self.input_schema
+        {
+          type: "object",
+          properties: {
+            event_ids: {
+              type: "array",
+              items: {type: "integer"},
+              description: "Database IDs of events to pin (from `event N` prefixes in the viewport)"
+            },
+            goal_ids: {
+              type: "array",
+              items: {type: "integer"},
+              description: "IDs of active goals to attach the events to"
+            }
+          },
+          required: %w[event_ids goal_ids]
+        }
+      end
+
+      # @param main_session [Session] the session being observed
+      def initialize(main_session:, **)
+        @session = main_session
+      end
+
+      # @param input [Hash<String, Object>] with "event_ids" and "goal_ids"
+      # @return [String] confirmation with link count, or error description
+      def execute(input)
+        event_ids = Array(input["event_ids"]).map(&:to_i).uniq
+        goal_ids = Array(input["goal_ids"]).map(&:to_i).uniq
+
+        return "Error: event_ids cannot be empty" if event_ids.empty?
+        return "Error: goal_ids cannot be empty" if goal_ids.empty?
+
+        events = @session.events.where(id: event_ids)
+        goals = @session.goals.active.where(id: goal_ids)
+
+        missing_events = event_ids - events.pluck(:id)
+        inactive_goal_ids = goal_ids - goals.pluck(:id)
+
+        errors = []
+        errors << "Events not found: #{missing_events.join(", ")}" if missing_events.any?
+
+        if inactive_goal_ids.any?
+          completed_ids = @session.goals.completed.where(id: inactive_goal_ids).pluck(:id)
+          not_found_ids = inactive_goal_ids - completed_ids
+          errors << "Goals already completed: #{completed_ids.join(", ")}" if completed_ids.any?
+          errors << "Goals not found: #{not_found_ids.join(", ")}" if not_found_ids.any?
+        end
+
+        return "Error: #{errors.join("; ")}" if errors.any?
+
+        attached = attach(events, goals)
+        "Pinned #{attached} event-goal links"
+      end
+
+      private
+
+      def attach(events, goals)
+        events.sum do |event|
+          pinned = find_or_create_pinned_event(event)
+          link_to_goals(pinned, goals)
+        end
+      end
+
+      def link_to_goals(pinned, goals)
+        goals.each { |goal| GoalPinnedEvent.find_or_create_by!(goal: goal, pinned_event: pinned) }
+        goals.size
+      end
+
+      def find_or_create_pinned_event(event)
+        PinnedEvent.find_or_create_by!(event: event) do |pe|
+          pe.display_text = truncate_event_content(event)
+        end
+      end
+
+      def truncate_event_content(event)
+        content = event.payload&.dig("content").to_s.strip
+        content = "event #{event.id}" if content.empty?
+
+        if content.length > PinnedEvent::MAX_DISPLAY_TEXT_LENGTH
+          content[0, PinnedEvent::MAX_DISPLAY_TEXT_LENGTH - 1] + "…"
+        else
+          content
+        end
+      end
+    end
+  end
+end

data/lib/mneme/tools/everything_ok.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Mneme
+  module Tools
+    # Sentinel tool signaling that Mneme has reviewed the viewport and
+    # determined no snapshot is needed. Called when the conversation
+    # context doesn't contain enough meaningful content to summarize.
+    class EverythingOk < ::Tools::Base
+      def self.tool_name = "everything_ok"
+
+      def self.description = "Signal that no snapshot is needed. " \
+        "Call this when the eviction zone contains only mechanical " \
+        "activity (tool calls) with no meaningful conversation to summarize."
+
+      def self.input_schema
+        {type: "object", properties: {}, required: []}
+      end
+
+      def execute(_input)
+        "Acknowledged. No snapshot needed."
+      end
+    end
+  end
+end

data/lib/mneme/tools/save_snapshot.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Mneme
+  module Tools
+    # Saves a summary snapshot of conversation context that is about to
+    # leave the viewport. The snapshot captures the "gist" of what happened
+    # so the agent retains awareness of past context.
+    #
+    # The text field has a max_tokens limit for predictable sizing — each
+    # snapshot is a fixed-size tile, enabling calculation of how many fit
+    # at each compression level.
+    class SaveSnapshot < ::Tools::Base
+      def self.tool_name = "save_snapshot"
+
+      def self.description = "Save a summary of the conversation context " \
+        "that is about to leave the viewport. Write a concise summary " \
+        "capturing key decisions, topics discussed, and important context. " \
+        "Focus on WHAT was decided and WHY, not mechanical details."
+
+      def self.input_schema
+        {
+          type: "object",
+          properties: {
+            text: {
+              type: "string",
+              description: "The summary text. Be concise but preserve key decisions, " \
+                "goals discussed, and important context. Max #{Anima::Settings.mneme_max_tokens} tokens."
+            }
+          },
+          required: %w[text]
+        }
+      end
+
+      # @param main_session [Session] the session being observed
+      # @param from_event_id [Integer] first event ID covered by this snapshot
+      # @param to_event_id [Integer] last event ID covered by this snapshot
+      # @param level [Integer] compression level (1 = from events, 2 = from L1 snapshots)
+      def initialize(main_session:, from_event_id:, to_event_id:, level: 1, **)
+        @main_session = main_session
+        @from_event_id = from_event_id
+        @to_event_id = to_event_id
+        @level = level
+      end
+
+      def execute(input)
+        text = input["text"].to_s.strip
+        return "Error: Summary text cannot be blank" if text.empty?
+
+        snapshot = @main_session.snapshots.create!(
+          text: text,
+          from_event_id: @from_event_id,
+          to_event_id: @to_event_id,
+          level: @level,
+          token_count: estimate_tokens(text)
+        )
+
+        "Snapshot saved (id: #{snapshot.id}, events #{@from_event_id}..#{@to_event_id})"
+      end
+
+      private
+
+      # @return [Integer] estimated token count for the summary text
+      def estimate_tokens(text)
+        [(text.bytesize / Event::BYTES_PER_TOKEN.to_f).ceil, 1].max
+      end
+    end
+  end
+end