anima-core 1.0.2 → 1.1.1

data/lib/mneme/compressed_viewport.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Mneme
+  # Builds a compressed viewport for Mneme's LLM context. Mneme sees
+  # conversation (user/agent messages and think events) but not mechanical
+  # execution (tool calls and responses). Tool calls are compressed to
+  # aggregate counters like `[4 tools called]`.
+  #
+  # The viewport is split into three zones separated by delimiters:
+  # - **Eviction zone** — events about to leave the viewport (upper third)
+  # - **Middle zone** — events in the middle of the viewport
+  # - **Recent zone** — the most recent events (lower third)
+  #
+  # Zone boundaries are calculated WITH tool call tokens (they affect
+  # position), then tool calls are removed and replaced with counters.
+  #
+  # @example
+  #   viewport = Mneme::CompressedViewport.new(session, token_budget: 60_000)
+  #   viewport.render  #=> "── EVICTION ZONE ──\nevent 42 User: ..."
+  class CompressedViewport
+    ZONE_DELIMITERS = {
+      eviction: "── EVICTION ZONE (upper third) ──",
+      middle: "── MIDDLE ZONE ──",
+      recent: "── RECENT ZONE (lower third) ──"
+    }.freeze
+
+    # @param session [Session] the session to build viewport for
+    # @param token_budget [Integer] total tokens available for Mneme's viewport
+    # @param from_event_id [Integer, nil] start from this event ID (inclusive);
+    #   when nil, uses the session's full viewport
+    def initialize(session, token_budget:, from_event_id: nil)
+      @session = session
+      @token_budget = token_budget
+      @from_event_id = from_event_id
+    end
+
+    # Renders the compressed viewport as a string ready for Mneme's LLM context.
+    #
+    # @return [String] compressed viewport with zone delimiters
+    def render
+      return "" if events.empty?
+
+      zones = split_into_zones(events)
+      render_zones(zones)
+    end
+
+    # @return [Array<Event>] the raw events selected for this viewport
+    def events
+      @events ||= fetch_events
+    end
+
+    private
+
+    # Fetches events within token budget, starting from from_event_id.
+    # Selects newest-first until budget exhausted, returns chronological.
+    # Caches per-event token costs in @event_costs for reuse by split_into_zones.
+    #
+    # @return [Array<Event>]
+    def fetch_events
+      scope = @session.events.context_events.deliverable
+
+      if @from_event_id
+        scope = scope.where("id >= ?", @from_event_id)
+      end
+
+      selected = []
+      @event_costs = {}
+      remaining = @token_budget
+
+      scope.reorder(id: :desc).each do |event|
+        cost = event_token_cost(event)
+        break if cost > remaining && selected.any?
+
+        selected << event
+        @event_costs[event.id] = cost
+        remaining -= cost
+      end
+
+      selected.reverse
+    end
+
+    # Splits events into three zones by token count.
+    # Zone boundaries are calculated including ALL events (tool calls count
+    # toward position), but zone assignment uses cumulative tokens.
+    #
+    # @return [Hash{Symbol => Array<Event>}] :eviction, :middle, :recent
+    def split_into_zones(events)
+      costs = events.map { |event| [event, @event_costs[event.id] || event_token_cost(event)] }
+      zone_size = costs.sum(&:last) / 3.0
+
+      result = {eviction: [], middle: [], recent: []}
+      cumulative = 0
+
+      costs.each do |event, cost|
+        cumulative += cost
+        result[zone_for_cumulative(cumulative, zone_size)] << event
+      end
+
+      result
+    end
+
+    # Renders zones with delimiters, compressing tool calls into counters.
+    #
+    # @param zones [Hash{Symbol => Array<Event>}]
+    # @return [String]
+    def render_zones(zones)
+      %i[eviction middle recent].flat_map { |name|
+        [ZONE_DELIMITERS[name], render_zone(zones[name])]
+      }.join("\n")
+    end
+
+    # Determines which zone an event belongs to based on cumulative token position.
+    #
+    # @param cumulative [Numeric] cumulative token count including this event
+    # @param zone_size [Float] token count per zone (total / 3)
+    # @return [Symbol] :eviction, :middle, or :recent
+    def zone_for_cumulative(cumulative, zone_size)
+      if cumulative <= zone_size
+        :eviction
+      elsif cumulative <= zone_size * 2
+        :middle
+      else
+        :recent
+      end
+    end
+
+    # Renders a single zone: conversation events as full text, consecutive
+    # tool calls/responses compressed into `[N tools called]` counters.
+    # tool_response events are intentionally silent — they affect zone boundaries
+    # via token cost but are not rendered; only tool_call events increment the counter.
+    #
+    # @param zone_events [Array<Event>]
+    # @return [String]
+    def render_zone(zone_events)
+      lines = []
+      tool_count = 0
+
+      zone_events.each do |event|
+        if conversation_event?(event) || think_event?(event)
+          lines << flush_tool_count(tool_count)
+          tool_count = 0
+          lines << render_event_line(event)
+        elsif event.event_type == "tool_call"
+          tool_count += 1
+        end
+      end
+
+      lines << flush_tool_count(tool_count)
+      lines.compact.join("\n")
+    end
+
+    # @return [Boolean] true if event is a user/agent/system message
+    def conversation_event?(event)
+      event.event_type.in?(Event::CONVERSATION_TYPES)
+    end
+
+    # Think events are tool_call events with tool_name == "think".
+    # They carry the agent's reasoning and are treated as conversation.
+    #
+    # @return [Boolean]
+    def think_event?(event)
+      event.event_type == "tool_call" && event.payload["tool_name"] == Event::THINK_TOOL
+    end
+
+    ROLE_LABELS = {
+      "user_message" => "User",
+      "agent_message" => "Assistant",
+      "system_message" => "System"
+    }.freeze
+
+    # Renders a single event as a transcript line.
+    #
+    # @param event [Event]
+    # @return [String]
+    def render_event_line(event)
+      prefix = "event #{event.id}"
+      data = event.payload
+      if think_event?(event)
+        "#{prefix} Think: #{data.dig("tool_input", "thoughts")}"
+      else
+        "#{prefix} #{ROLE_LABELS.fetch(event.event_type)}: #{data["content"]}"
+      end
+    end
+
+    # Returns a tool count string if any tools were called, nil otherwise.
+    #
+    # @param count [Integer] number of tool calls to flush
+    # @return [String, nil]
+    def flush_tool_count(count)
+      return if count == 0
+      "[#{count} #{(count == 1) ? "tool" : "tools"} called]"
+    end
+
+    # @return [Integer] token cost using cached count or heuristic
+    def event_token_cost(event)
+      cached = event.token_count
+      (cached > 0) ? cached : event.estimate_tokens
+    end
+  end
+end

data/lib/mneme/l2_runner.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Mneme
+  # Compresses multiple Level 1 snapshots into a single Level 2 snapshot.
+  # L2 snapshots capture days/weeks-scale context from hourly L1 summaries,
+  # preventing unbounded snapshot growth via recursive compression.
+  #
+  # Triggered from {MnemeJob} after an L1 snapshot is created, when enough
+  # uncovered L1 snapshots have accumulated (configurable via
+  # +mneme.l2_snapshot_threshold+ in config.toml).
+  #
+  # @example
+  #   Mneme::L2Runner.new(session).call
+  class L2Runner
+    TOOLS = [
+      Tools::SaveSnapshot,
+      Tools::EverythingOk
+    ].freeze
+
+    SYSTEM_PROMPT = <<~PROMPT
+      You are Mneme, the memory department of an AI agent named Anima.
+      Your job is to compress multiple conversation summaries into a single
+      higher-level summary.
+
+      You MUST ONLY communicate through tool calls — NEVER output text.
+
+      ──────────────────────────────
+      WHAT YOU SEE
+      ──────────────────────────────
+      Several Level 1 snapshots — hourly conversation summaries.
+      Each captures key decisions, goals discussed, and important context
+      from a portion of the conversation history.
+
+      ──────────────────────────────
+      YOUR TASK
+      ──────────────────────────────
+      Compress the snapshots into ONE Level 2 summary that captures the
+      essential arc across all of them. If the snapshots contain meaningful
+      content, call save_snapshot. If they are purely mechanical, call
+      everything_ok.
+
+      Preserve:
+      - Key decisions and their reasoning
+      - Goal progress across the time span
+      - Important context shifts or pivots
+      - Relationships and patterns across snapshots
+
+      Drop:
+      - Redundant details repeated across snapshots
+      - Mechanical execution details
+      - Interim decisions that were superseded by later ones
+
+      Always finish with exactly ONE tool call: either save_snapshot or everything_ok.
+    PROMPT
+
+    # @param session [Session] the main session whose L1 snapshots to compress
+    # @param client [LLM::Client, nil] injectable LLM client (defaults to fast model)
+    def initialize(session, client: nil)
+      @session = session
+      @client = client || LLM::Client.new(
+        model: Anima::Settings.fast_model,
+        max_tokens: Anima::Settings.mneme_max_tokens,
+        logger: Mneme.logger
+      )
+    end
+
+    # Compresses uncovered L1 snapshots into a single L2 snapshot.
+    # Returns early if not enough L1 snapshots have accumulated.
+    #
+    # @return [String, nil] LLM response text, or nil when skipped
+    def call
+      l1_snapshots = eligible_snapshots
+      threshold = Anima::Settings.mneme_l2_snapshot_threshold
+      sid = @session.id
+      snapshot_count = l1_snapshots.size
+
+      if snapshot_count < threshold
+        log.debug("session=#{sid} — only #{snapshot_count}/#{threshold} L1 snapshots, skipping L2")
+        return
+      end
+
+      messages = build_messages(l1_snapshots)
+      registry = build_registry(l1_snapshots)
+
+      log.info("session=#{sid} — running L2 compression (#{snapshot_count} L1 snapshots)")
+
+      result = @client.chat_with_tools(
+        messages,
+        registry: registry,
+        session_id: nil,
+        system: SYSTEM_PROMPT
+      )
+
+      log.info("session=#{sid} — L2 compression done: #{result.to_s.truncate(200)}")
+      result
+    end
+
+    private
+
+    # L1 snapshots that are not yet covered by any L2 snapshot.
+    #
+    # @return [Array<Snapshot>]
+    def eligible_snapshots
+      @session.snapshots.for_level(1).not_covered_by_l2.chronological.to_a
+    end
+
+    # Frames L1 snapshot texts as a user message for the LLM.
+    #
+    # @param snapshots [Array<Snapshot>]
+    # @return [Array<Hash>] single-element messages array
+    def build_messages(snapshots)
+      content = snapshots.map.with_index(1) { |snap, idx|
+        "--- Snapshot #{idx} (events #{snap.from_event_id}..#{snap.to_event_id}) ---\n#{snap.text}"
+      }.join("\n\n")
+
+      [{role: "user", content: "Compress these #{snapshots.size} Level 1 snapshots into a single Level 2 summary:\n\n#{content}"}]
+    end
+
+    # Builds the tool registry with L2 context for SaveSnapshot.
+    # The event range spans from the first L1's start to the last L1's end.
+    #
+    # @param snapshots [Array<Snapshot>]
+    # @return [Tools::Registry]
+    def build_registry(snapshots)
+      registry = ::Tools::Registry.new(context: {
+        main_session: @session,
+        from_event_id: snapshots.first.from_event_id,
+        to_event_id: snapshots.last.to_event_id,
+        level: 2
+      })
+      TOOLS.each { |tool| registry.register(tool) }
+      registry
+    end
+
+    # @return [Logger]
+    def log = Mneme.logger
+  end
+end

data/lib/mneme/passive_recall.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Mneme
+  # Passive recall — automatic memory surfacing triggered by Goal updates.
+  # When goals are created or updated, searches event history for related
+  # context and caches the results on the session for viewport injection.
+  #
+  # The agent never calls a tool; relevant memories appear automatically
+  # in the viewport between snapshots and the sliding window. This mirrors
+  # recognition memory in humans — context surfaces without conscious effort.
+  #
+  # @example Trigger after a goal update
+  #   Mneme::PassiveRecall.new(session).call
+  class PassiveRecall
+    # @param session [Session] the session whose goals drive recall
+    def initialize(session)
+      @session = session
+    end
+
+    # Searches event history using active goal descriptions as queries.
+    # Returns recall results suitable for viewport injection.
+    #
+    # @return [Array<Mneme::Search::Result>] deduplicated, relevance-sorted
+    def call
+      goals = @session.goals.active.root.includes(:sub_goals)
+      return [] if goals.empty?
+
+      search_terms = build_search_terms(goals)
+      return [] if search_terms.blank?
+
+      results = Mneme::Search.query(search_terms, limit: Anima::Settings.recall_max_results)
+
+      # Exclude events from the current session's viewport — no point recalling
+      # what the agent already sees.
+      viewport_ids = @session.viewport_event_ids.to_set
+      results.reject { |result| viewport_ids.include?(result.event_id) }
+    end
+
+    private
+
+    STOP_WORDS = Set.new(%w[
+      a an the is are was were be been being do does did
+      have has had in on at to for of and or but not with
+      this that it its by from as up out if about into
+      fix add create update remove implement check set get
+    ]).freeze
+
+    # Extracts meaningful keywords from active goals and joins with OR.
+    # Stop words and generic verbs are stripped — they're too common to
+    # produce useful recall results.
+    #
+    # @param goals [ActiveRecord::Relation<Goal>]
+    # @return [String] FTS5 OR-joined keywords
+    def build_search_terms(goals)
+      descriptions = goals.flat_map { |goal|
+        [goal.description] + goal.sub_goals.reject(&:completed?).map(&:description)
+      }
+
+      words = descriptions.join(" ")
+        .gsub(/[^a-zA-Z0-9\s-]/, "")
+        .downcase
+        .split
+        .uniq
+        .reject { |word| STOP_WORDS.include?(word) || word.length < 3 }
+
+      words.join(" OR ").truncate(500)
+    end
+  end
+end

data/lib/mneme/runner.rb

@@ -0,0 +1,254 @@
+# frozen_string_literal: true
+
+module Mneme
+  # Orchestrates the Mneme memory department — a phantom (non-persisted) LLM loop
+  # that observes a main session's compressed viewport and creates summaries of
+  # conversation context before it evicts from the viewport.
+  #
+  # Mneme is triggered when the terminal event (`mneme_boundary_event_id`) leaves
+  # the viewport. It receives a compressed viewport (no raw tool calls, zone
+  # delimiters present) and uses the `save_snapshot` tool to persist a summary.
+  #
+  # After completing, Mneme advances the terminal event to the boundary of what
+  # it just summarized, so the cycle repeats as more events accumulate.
+  #
+  # @example
+  #   Mneme::Runner.new(session).call
+  class Runner
+    TOOLS = [
+      Tools::SaveSnapshot,
+      Tools::AttachEventsToGoals,
+      Tools::EverythingOk
+    ].freeze
+
+    SYSTEM_PROMPT = <<~PROMPT
+      You are Mneme, the memory department of an AI agent named Anima.
+      Your job is to create concise summaries of conversation context that is
+      about to leave the agent's context window.
+
+      You MUST ONLY communicate through tool calls — NEVER output text.
+
+      ──────────────────────────────
+      WHAT YOU SEE
+      ──────────────────────────────
+      A compressed viewport with three zones:
+      - EVICTION ZONE: Events about to leave the viewport. Summarize these.
+      - MIDDLE ZONE: Events still visible but aging. Note key context.
+      - RECENT ZONE: Fresh events. Use for continuity with the summary.
+
+      Events are prefixed with `event N` (their database ID).
+      Tool calls are compressed to `[N tools called]` — the mechanical work
+      is not important, only the conversation flow.
+
+      ──────────────────────────────
+      YOUR TASK
+      ──────────────────────────────
+      1. Read the eviction zone carefully.
+      2. If it contains meaningful conversation (decisions, goals, context):
+         Call save_snapshot with a concise summary.
+      3. If any events in the eviction zone are too important to summarize
+         (exact user instructions, critical corrections, key decisions),
+         pin them to active goals with attach_events_to_goals.
+         Pinned events survive eviction intact — use this sparingly for
+         events where the exact wording matters.
+      4. If it contains only mechanical activity with no conversation:
+         Call everything_ok.
+
+      You may call BOTH save_snapshot AND attach_events_to_goals in one turn
+      when the zone has a mix of summarizable and pin-worthy events.
+
+      Write summaries that capture:
+      - What was discussed and decided
+      - Why decisions were made
+      - Active goals and their progress
+      - Key context the agent would need later
+
+      Do NOT include:
+      - Tool call details (which files were read, commands run)
+      - Mechanical execution steps
+      - Verbatim quotes (paraphrase instead)
+
+      Always finish with at least one tool call: save_snapshot, attach_events_to_goals,
+      or everything_ok. You may combine save_snapshot with attach_events_to_goals.
+    PROMPT
+
+    # @param session [Session] the main session to observe
+    # @param client [LLM::Client, nil] injectable LLM client (defaults to fast model)
+    def initialize(session, client: nil)
+      @session = session
+      @client = client || LLM::Client.new(
+        model: Anima::Settings.fast_model,
+        max_tokens: Anima::Settings.mneme_max_tokens,
+        logger: Mneme.logger
+      )
+    end
+
+    # Runs the Mneme loop: builds compressed viewport, calls LLM, executes
+    # snapshot tool, then advances the terminal event pointer.
+    #
+    # @return [String, nil] the LLM's final text response (discarded),
+    #   or nil if no context is available
+    def call
+      viewport = build_compressed_viewport
+      compressed_text = viewport.render
+      sid = @session.id
+
+      if compressed_text.empty?
+        log.debug("session=#{sid} — no events for Mneme, skipping")
+        return
+      end
+
+      messages = build_messages(compressed_text)
+      system = SYSTEM_PROMPT
+
+      log.info("session=#{sid} — running Mneme (#{viewport.events.size} events)")
+      log.debug("compressed viewport:\n#{compressed_text}")
+
+      result = @client.chat_with_tools(
+        messages,
+        registry: build_registry(viewport),
+        session_id: nil,
+        system: system
+      )
+
+      advance_boundary(viewport)
+      log.info("session=#{sid} — Mneme done: #{result.to_s.truncate(200)}")
+      result
+    end
+
+    private
+
+    # Builds the compressed viewport starting from the session's boundary event.
+    #
+    # @return [Mneme::CompressedViewport]
+    def build_compressed_viewport
+      token_budget = (Anima::Settings.token_budget * Anima::Settings.mneme_viewport_fraction).to_i
+
+      CompressedViewport.new(
+        @session,
+        token_budget: token_budget,
+        from_event_id: @session.mneme_boundary_event_id
+      )
+    end
+
+    # Frames the compressed viewport as a user message for the LLM.
+    #
+    # @param compressed_text [String] the rendered compressed viewport
+    # @return [Array<Hash>] single-element messages array
+    def build_messages(compressed_text)
+      goals_context = active_goals_section
+
+      content = <<~MSG.strip
+        Here is the compressed viewport of the main session:
+
+        #{compressed_text}
+        #{goals_context}
+        Review the eviction zone and decide whether to save a snapshot or signal everything_ok.
+      MSG
+
+      [{role: "user", content: content}]
+    end
+
+    # Builds the tool registry with session context for SaveSnapshot.
+    # Passes the event range from the viewport so the snapshot records
+    # which events it covers.
+    #
+    # @param viewport [Mneme::CompressedViewport]
+    # @return [Tools::Registry]
+    def build_registry(viewport)
+      viewport_events = viewport.events
+      registry = ::Tools::Registry.new(context: {
+        main_session: @session,
+        from_event_id: viewport_events.first&.id,
+        to_event_id: viewport_events.last&.id
+      })
+      TOOLS.each { |tool| registry.register(tool) }
+      registry
+    end
+
+    # Advances the terminal event pointer after Mneme completes.
+    # Runs unconditionally — even when the LLM called `everything_ok` (no snapshot
+    # needed), the zone was reviewed and should be advanced past. Without this,
+    # Mneme would re-examine the same mechanical-only content on every trigger.
+    #
+    # Sets it to the last conversation event in the viewport, ensuring
+    # the boundary is always a message/think event, never a tool_call/tool_response.
+    # Also updates the snapshot range pointers.
+    #
+    # @param viewport [Mneme::CompressedViewport]
+    def advance_boundary(viewport)
+      viewport_events = viewport.events
+      return if viewport_events.empty?
+
+      new_boundary = viewport_events.reverse_each.find { |event| conversation_or_think?(event) }
+      return unless new_boundary
+
+      boundary_id = new_boundary.id
+      updates = {mneme_boundary_event_id: boundary_id}
+
+      updates[:mneme_snapshot_first_event_id] = viewport_events.first.id if @session.mneme_snapshot_first_event_id.nil?
+      updates[:mneme_snapshot_last_event_id] = viewport_events.last.id
+
+      @session.update_columns(updates)
+      log.debug("session=#{@session.id} — boundary advanced to event #{boundary_id}")
+    end
+
+    # Delegates to {Event#conversation_or_think?} — single source of truth
+    # for which events Mneme treats as conversation boundaries.
+    #
+    # @return [Boolean]
+    def conversation_or_think?(event)
+      event.conversation_or_think?
+    end
+
+    # Builds the active goals section for Mneme's context so it knows
+    # what Goals exist, which events are already pinned, and can reference
+    # them when deciding what to pin or summarize.
+    #
+    # @return [String] formatted goals section, or empty string
+    def active_goals_section
+      root_goals = @session.goals.root.includes(:sub_goals).active.order(:created_at)
+      return "" if root_goals.empty?
+
+      lines = root_goals.map { |goal| format_goal_for_mneme(goal) }
+      pinned = format_existing_pins
+
+      section = "\n\n🎯 Active Goals\n#{lines.join("\n")}\n"
+      section += "\n📌 Already Pinned\n#{pinned}\n" if pinned
+      section
+    end
+
+    # Formats a goal with sub-goals for Mneme's context.
+    #
+    # @param goal [Goal] root goal with preloaded sub_goals
+    # @return [String]
+    def format_goal_for_mneme(goal)
+      parts = ["  ● #{goal.description} (id: #{goal.id})"]
+      goal.sub_goals.each do |sub|
+        checkbox = sub.completed? ? "[x]" : "[ ]"
+        parts << "    #{checkbox} #{sub.description} (id: #{sub.id})"
+      end
+      parts.join("\n")
+    end
+
+    # Lists already-pinned event IDs so Mneme avoids redundant pinning.
+    #
+    # @return [String, nil] formatted pin list, or nil when nothing is pinned
+    def format_existing_pins
+      pins = @session.pinned_events.includes(:goals).order(:event_id)
+      return nil if pins.empty?
+
+      pins.map { |pin| format_pin_for_mneme(pin) }.join("\n")
+    end
+
+    # @param pin [PinnedEvent] pin with preloaded goals
+    # @return [String] formatted pin line
+    def format_pin_for_mneme(pin)
+      goal_ids = pin.goals.map(&:id).join(", ")
+      "  event #{pin.event_id} → goals [#{goal_ids}]"
+    end
+
+    # @return [Logger]
+    def log = Mneme.logger
+  end
+end