anima-core 1.4.0 → 1.5.0

data/lib/mneme/search.rb

@@ -1,19 +1,19 @@
 # frozen_string_literal: true
 
 module Mneme
-  # Full-text search over message history using SQLite FTS5.
-  # Covers user messages, agent messages, and think messages across all sessions.
+  # Full-text search over long-term memory — the message history outside
+  # the caller's current viewport. Covers user messages, agent messages,
+  # and think messages across every session Anima has ever held.
   #
   # The interface is intentionally abstract — callers receive {Result} structs
   # and never touch FTS5 directly. A future semantic search backend (embeddings,
   # BM25 + re-ranking) can replace the implementation without changing callers.
   #
-  # @example Search across all sessions
-  #   results = Mneme::Search.query("authentication flow")
-  #   results.each { |r| puts "message #{r.message_id}: #{r.snippet}" }
+  # @example Mneme's recall muse searching for the main session
+  #   Mneme::Search.query("authentication flow", caller_session: session)
   #
-  # @example Search within a single session
-  #   results = Mneme::Search.query("OAuth config", session_id: 42)
+  # @example Aoide searching actively from her own session
+  #   Mneme::Search.query("OAuth config", caller_session: session)
   class Search
     # A single search result with enough context for display and drill-down.
     #
@@ -24,19 +24,27 @@ module Mneme
     # @!attribute message_type [String] friendly label: human, anima, system, or thought
     Result = Struct.new(:message_id, :session_id, :snippet, :rank, :message_type, keyword_init: true)
 
-    # Searches message history for the given terms.
+    # Searches long-term memory for the given terms.
+    #
+    # Excludes messages currently in the caller's viewport so a `LIMIT`-bounded
+    # search never burns its slots returning things the caller already has in
+    # front of them. A caller with no established Mneme boundary yet (fresh
+    # main session, sub-agent) treats the whole session as "in viewport" — none
+    # of its own messages surface.
     #
     # @param terms [String] search query (FTS5 syntax: words, phrases, OR/AND/NOT)
-    # @param session_id [Integer, nil] scope to a specific session (nil = all sessions)
+    # @param caller_session [Session] the session doing the search — used to
+    #   exclude its own viewport from the results. Required; search always
+    #   happens from the perspective of a specific session.
     # @param limit [Integer] maximum results
     # @return [Array<Result>] ranked by relevance (best first)
-    def self.query(terms, session_id: nil, limit: Anima::Settings.recall_max_results)
-      new(terms, session_id: session_id, limit: limit).call
+    def self.query(terms, caller_session:, limit: Anima::Settings.recall_max_results)
+      new(terms, caller_session: caller_session, limit: limit).call
     end
 
-    def initialize(terms, session_id: nil, limit: 5)
+    def initialize(terms, caller_session:, limit: 5)
       @terms = sanitize_query(terms)
-      @session_id = session_id
+      @caller_session = caller_session
       @limit = limit
       @recency_decay = Anima::Settings.recall_recency_decay
     end
@@ -51,30 +59,29 @@ module Mneme
 
     private
 
-    # Executes the FTS5 MATCH query with optional session scoping.
-    # Joins back to messages table for session_id and message_type.
+    # Executes the FTS5 MATCH query with viewport exclusion for the caller.
     #
     # @return [Array<Hash>] raw database rows
     def execute_fts_query
-      sql = if @session_id
-        Arel.sql(scoped_sql, @recency_decay, @terms, @session_id, @limit)
-      else
-        Arel.sql(global_sql, @recency_decay, @terms, @limit)
-      end
-
-      connection.select_all(sql, "Mneme::Search").to_a
+      sql, binds = build_sql_and_binds
+      connection.select_all(Arel.sql(sql, *binds), "Mneme::Search").to_a
     end
 
-    # FTS5 query across all sessions.
-    # Contentless FTS5 can't use snippet() — extract content from messages directly.
-    #
-    # Ranking blends BM25 relevance with recency: rank is negative (more
-    # negative = better match), so dividing by a factor > 1 for older messages
-    # moves them closer to zero (less relevant). At decay 0.3, a one-year-old
-    # result needs ~30% better keyword relevance to beat an identical match
-    # from today.
-    def global_sql
-      <<~SQL
+    # Builds the FTS5 SQL. Viewport exclusion depends on whether the caller's
+    # session has a Mneme boundary:
+    # * boundary set → exclude caller's messages at or above it (they're visible).
+    # * boundary nil → exclude the caller's whole session (no eviction has
+    #   happened yet, so everything is visible).
+    # Other sessions are always unfiltered — their IDs and boundaries mean
+    # nothing to the caller's context.
+    def build_sql_and_binds
+      binds = [@recency_decay, @terms]
+
+      viewport_clause, viewport_binds = caller_viewport_exclusion
+      binds.concat(viewport_binds)
+      binds << @limit
+
+      sql = <<~SQL
         SELECT
           m.id AS message_id,
           m.session_id,
@@ -89,32 +96,22 @@ module Mneme
         FROM messages_fts
         JOIN messages m ON m.id = messages_fts.rowid
         WHERE messages_fts MATCH ?
+          AND #{viewport_clause}
         ORDER BY rank
         LIMIT ?
       SQL
+
+      [sql, binds]
     end
 
-    # FTS5 query scoped to a specific session.
-    def scoped_sql
-      <<~SQL
-        SELECT
-          m.id AS message_id,
-          m.session_id,
-          m.message_type,
-          CASE
-            WHEN m.message_type IN ('user_message', 'agent_message', 'system_message')
-              THEN substr(json_extract(m.payload, '$.content'), 1, 300)
-            WHEN m.message_type = 'tool_call'
-              THEN substr(json_extract(m.payload, '$.tool_input.thoughts'), 1, 300)
-          END AS snippet,
-          rank / (1.0 + ? * (julianday('now') - julianday(m.created_at)) / 365.0) AS rank
-        FROM messages_fts
-        JOIN messages m ON m.id = messages_fts.rowid
-        WHERE messages_fts MATCH ?
-          AND m.session_id = ?
-        ORDER BY rank
-        LIMIT ?
-      SQL
+    # Returns the SQL fragment + bind params that exclude the caller's viewport.
+    def caller_viewport_exclusion
+      boundary = @caller_session.mneme_boundary_message_id
+      if boundary
+        ["(m.session_id != ? OR m.id < ?)", [@caller_session.id, boundary]]
+      else
+        ["m.session_id != ?", [@caller_session.id]]
+      end
     end
 
     FRIENDLY_MESSAGE_TYPES = {
@@ -139,25 +136,70 @@ module Mneme
       )
     end
 
-    # Sanitizes user input for FTS5 MATCH safety.
-    # Strips special FTS5 operators that could cause syntax errors,
-    # keeps only alphanumeric words and quoted phrases.
+    # FTS5 logical-operator keywords callers may pass verbatim. Everything
+    # else is quote-wrapped, which is SQLite's recommended way to feed
+    # untrusted text to +MATCH+ — inside a quoted phrase the tokenizer
+    # treats +- : * ^ { } ( )+ and any future operator character as
+    # ordinary content, so hazards like +sub-agents+ (parsed as
+    # +sub NOT agents+ → +no such column: agents+) and +agents:foo+
+    # (parsed as a column filter) become literal phrases.
+    #
+    # Adding +NEAR+ or any new operator that a caller legitimately needs
+    # is a one-line change here; character-level blocklists would need to
+    # be re-audited against every FTS5 release.
+    #
+    # @see https://www.sqlite.org/fts5.html FTS5 query syntax
+    # @see https://www.mail-archive.com/sqlite-users@mailinglists.sqlite.org/msg118320.html
+    #   Dan Kennedy's canonical guidance on MATCH sanitization
+    FTS5_PASSTHROUGH_OPERATORS = Set.new(%w[AND OR NOT NEAR]).freeze
+    private_constant :FTS5_PASSTHROUGH_OPERATORS
+
+    # Sanitizes user input for FTS5 MATCH safety by quote-wrapping each
+    # token. Logical operators ({FTS5_PASSTHROUGH_OPERATORS}) pass through
+    # so callers that intentionally build +word1 OR word2+ queries still
+    # get boolean behavior.
+    #
+    # A query that collapses to operators only (e.g. user typed "and or
+    # not") has no operands and would trigger an FTS5 syntax error, so we
+    # return an empty string and let {#call} short-circuit via
+    # +@terms.blank?+.
     #
     # @param raw [String]
     # @return [String] safe FTS5 query
     def sanitize_query(raw)
       return "" unless raw
 
-      # Extract quoted phrases and individual words, drop FTS5 operators
-      tokens = raw.scan(/"[^"]+?"|\S+/).reject { |token| token.match?(/\A[*:^{}()]+\z/) }
-      tokens.filter_map { |token| sanitize_token(token) }.join(" ")
+      tokens = raw.scan(/"[^"]*"|\S+/).filter_map { |token| sanitize_token(token) }
+      return "" if tokens.all? { |t| FTS5_PASSTHROUGH_OPERATORS.include?(t) }
+
+      tokens.join(" ")
     end
 
+    # @param token [String] one whitespace-delimited chunk of user input
+    # @return [String, nil] nil when the token is empty after cleanup
     def sanitize_token(token)
-      return token if token.start_with?('"')
+      return token if FTS5_PASSTHROUGH_OPERATORS.include?(token)
+      return rewrap_phrase(token) if token.start_with?('"')
+
+      quote_as_phrase(token)
+    end
+
+    # Rebalances a user-supplied phrase so a stray or doubled quote can't
+    # leave the overall query syntactically broken.
+    #
+    # @param token [String] token starting with +"+
+    # @return [String, nil]
+    def rewrap_phrase(token)
+      inner = token.delete_prefix('"').delete_suffix('"').strip
+      inner.empty? ? nil : quote_as_phrase(inner)
+    end
+
+    # @param text [String] any token treated as literal content
+    # @return [String, nil]
+    def quote_as_phrase(text)
+      return nil if text.empty?
 
-      cleaned = token.gsub(/[^a-zA-Z0-9-]/, "")
-      cleaned.empty? ? nil : cleaned
+      %("#{text.gsub('"', '""')}")
     end
 
     def connection

data/lib/mneme/tools/nothing_to_surface.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Mneme
+  module Tools
+    # Finish-line tool for {Mneme::RecallRunner}. The muse calls this when
+    # she's done her work — whether she surfaced memories or decided
+    # nothing was worth carrying forward. Having a single finish line makes
+    # every recall run explicit: silence is intentional, not a timeout.
+    #
+    # Mirror of {EverythingOk} for the eviction runner.
+    class NothingToSurface < ::Tools::Base
+      def self.tool_name = "nothing_to_surface"
+
+      def self.description = "Finish the recall run. Call this when you're done — whether you surfaced memories or decided nothing was worth surfacing right now. Silence is a valid answer when older memory wouldn't help."
+
+      def self.input_schema
+        {type: "object", properties: {}, required: []}
+      end
+
+      def execute(_input)
+        "Acknowledged."
+      end
+    end
+  end
+end

data/lib/mneme/tools/save_snapshot.rb

@@ -20,7 +20,7 @@ module Mneme
           properties: {
             text: {
               type: "string",
-              maxLength: Anima::Settings.mneme_max_tokens * Message::BYTES_PER_TOKEN
+              maxLength: Anima::Settings.mneme_max_tokens * TokenEstimation::BYTES_PER_TOKEN
             }
           },
           required: %w[text]
@@ -46,19 +46,11 @@ module Mneme
           text: text,
           from_message_id: @from_message_id,
           to_message_id: @to_message_id,
-          level: @level,
-          token_count: estimate_tokens(text)
+          level: @level
         )
 
         "Snapshot saved (id: #{snapshot.id}, messages #{@from_message_id}..#{@to_message_id})"
       end
-
-      private
-
-      # @return [Integer] estimated token count for the summary text
-      def estimate_tokens(text)
-        [(text.bytesize / Message::BYTES_PER_TOKEN.to_f).ceil, 1].max
-      end
     end
   end
 end

data/lib/mneme/tools/surface_memory.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Mneme
+  module Tools
+    # Surfaces a past message into Aoide's next turn as a `from_mneme`
+    # phantom tool pair. Called by Mneme's recall loop when a search hit
+    # or a viewed message clears her relevance bar.
+    #
+    # The persisted {PendingMessage} carries the original +message_id+ in
+    # its +source_name+ (and through promotion ends up inside
+    # +tool_input.message_id+ of the phantom pair), so the same memory
+    # isn't re-surfaced on later cycles — Mneme::Search already excludes
+    # Aoide's viewport, and once a recall promotes it lives there.
+    #
+    # The muse explains +why+ she's surfacing this memory. The reason is
+    # logged but not shown to Aoide — keeping the surfaced content itself
+    # clean of meta-commentary.
+    class SurfaceMemory < ::Tools::Base
+      def self.tool_name = "surface_memory"
+
+      def self.description = "Surface a memory into Aoide's next turn. Use when a specific past message is genuinely useful for what she's working on now. Pass the message_id and a short reason — one sentence explaining why she needs this *now*."
+
+      def self.input_schema
+        {
+          type: "object",
+          properties: {
+            message_id: {type: "integer"},
+            why: {type: "string", description: "One-sentence justification — kept for logs, not shown to Aoide."}
+          },
+          required: %w[message_id why]
+        }
+      end
+
+      # @param main_session [Session] the session receiving the recall
+      def initialize(main_session:, **)
+        @main_session = main_session
+      end
+
+      def execute(input)
+        message_id = input["message_id"].to_i
+        why = input["why"].to_s.strip
+
+        message = Message.find_by(id: message_id)
+        return {error: "Message #{message_id} not found"} unless message
+        return {error: "Reason cannot be blank"} if why.empty?
+
+        content = render_snippet(message)
+
+        @main_session.pending_messages.create!(
+          content: content,
+          source_type: "recall",
+          source_name: message_id.to_s,
+          message_type: "from_mneme"
+        )
+
+        Mneme.logger.info("session=#{@main_session.id} — surfaced message #{message_id}: #{why}")
+
+        "Surfaced message #{message_id}."
+      end
+
+      private
+
+      # Formats the message as the text Aoide will read when the phantom
+      # pair promotes. Headed with origin metadata, bounded by the recall
+      # snippet-token budget so long messages don't blow out her viewport.
+      #
+      # @param message [Message]
+      # @return [String]
+      def render_snippet(message)
+        origin = message.session&.name.presence || "session ##{message.session_id}"
+        raw = extract_content(message)
+        max_chars = Anima::Settings.recall_max_snippet_tokens * TokenEstimation::BYTES_PER_TOKEN
+        "message #{message.id} (#{origin}): #{raw.truncate(max_chars)}"
+      end
+
+      def extract_content(message)
+        payload = message.payload
+        case message.message_type
+        when "user_message", "agent_message", "system_message"
+          payload["content"].to_s
+        when "tool_call"
+          payload.dig("tool_input", "thoughts").to_s
+        else
+          payload["content"].to_s
+        end
+      end
+    end
+  end
+end

data/lib/mneme.rb

@@ -1,12 +1,18 @@
 # frozen_string_literal: true
 
-# Mneme — the memory department. Watches for viewport eviction and creates
-# summaries before context is lost. Named after the Greek Titaness of memory.
+# Mneme — the muse of memory. Watches for viewport eviction and creates
+# summaries before context is lost. One of the Three Muses: she remembers
+# while Melete prepares and Aoide performs.
 #
-# Mneme is the third event bus department alongside Nous (main agent) and
-# the Analytical Brain. It operates as a phantom LLM loop: observes the
-# main session, creates snapshots, but leaves no trace of its own reasoning.
+# Operates as a phantom LLM loop: observes the main session, creates
+# snapshots, but leaves no trace of her own reasoning.
 module Mneme
+  # Estimated token overhead for a synthetic +tool_use+/+tool_result+
+  # pair — the wrapper JSON that phantom promotions emit around their
+  # content (tool name, input hash, ids, framing). Added to the content's
+  # token estimate when sizing phantom pairs in the viewport.
+  TOOL_PAIR_OVERHEAD_TOKENS = 50
+
   # Dev-only logger that writes to log/mneme.log.
   # In non-development environments returns a null logger so
   # call sites don't need conditionals.