@vellumai/assistant 0.5.15 → 0.6.0

package/src/memory/graph/conversation-graph-memory.ts

@@ -0,0 +1,630 @@
+// ---------------------------------------------------------------------------
+// Memory Graph — Conversation-level memory integration
+//
+// Replaces the old `prepareMemoryContext` from conversation-memory.ts.
+// Manages the InContextTracker lifecycle and dispatches to the correct
+// retrieval mode based on conversation state.
+// ---------------------------------------------------------------------------
+
+import { and, desc, eq, ne } from "drizzle-orm";
+
+import type { AssistantConfig } from "../../config/types.js";
+import { estimateTextTokens } from "../../context/token-estimator.js";
+import type { ServerMessage } from "../../daemon/message-protocol.js";
+import type {
+  ContentBlock,
+  ImageContent,
+  Message,
+} from "../../providers/types.js";
+import { getLogger } from "../../util/logger.js";
+import { getDb } from "../db.js";
+import { memorySummaries } from "../schema.js";
+import { conversations } from "../schema/conversations.js";
+import {
+  assembleContextBlock,
+  assembleInjectionBlock,
+  InContextTracker,
+  MAX_CONTEXT_LOAD_IMAGES,
+  MAX_PER_TURN_IMAGES,
+  MAX_REFRESH_IMAGES,
+  type ResolvedImage,
+  resolveInjectionImages,
+} from "./injection.js";
+import {
+  loadContextMemory,
+  REFRESH_INTERVAL_TURNS,
+  refreshContextMemory,
+  retrieveForTurn,
+} from "./retriever.js";
+
+const log = getLogger("graph-conversation-memory");
+
+const ESTIMATED_IMAGE_TOKENS = 1000;
+
+// ---------------------------------------------------------------------------
+// Per-conversation state
+// ---------------------------------------------------------------------------
+
+/**
+ * Manages memory graph state for a single conversation.
+ * Create one per Conversation instance. Persists across turns.
+ */
+export class ConversationGraphMemory {
+  readonly tracker = new InContextTracker();
+  private turnCount = 0;
+  private initialized = false;
+  private lastCompactedAt: number | null = null;
+  private needsReload = false;
+  private scopeId: string;
+  private conversationId: string;
+  private lastInjectedBlock: string | null = null;
+  private lastInjectedNodeIds: string[] = [];
+  private lastInjectedImages: Map<string, ResolvedImage> = new Map();
+
+  constructor(scopeId: string, conversationId: string) {
+    this.scopeId = scopeId;
+    this.conversationId = conversationId;
+  }
+
+  /**
+   * Fetch the most recent conversation summaries (excluding the current
+   * conversation, which won't have one yet at context-load time).
+   *
+   * Prioritizes user conversations (conversationType != "background"),
+   * allowing at most 1 background conversation summary so the retrieval
+   * signal is mostly from direct interactions.
+   *
+   * Returns up to 3 summary texts, most recent first.
+   */
+  private fetchRecentSummaries(): string[] {
+    try {
+      const db = getDb();
+      const baseWhere = and(
+        eq(memorySummaries.scope, "conversation"),
+        eq(memorySummaries.scopeId, this.scopeId),
+        ne(memorySummaries.scopeKey, this.conversationId),
+      );
+
+      // Fetch user conversations first (up to 3)
+      const userRows = db
+        .select({ summary: memorySummaries.summary })
+        .from(memorySummaries)
+        .innerJoin(
+          conversations,
+          eq(memorySummaries.scopeKey, conversations.id),
+        )
+        .where(and(baseWhere, ne(conversations.conversationType, "background")))
+        .orderBy(desc(memorySummaries.updatedAt))
+        .limit(3)
+        .all();
+
+      if (userRows.length >= 3) {
+        return userRows.map((r) => r.summary);
+      }
+
+      // Fill remaining slots with at most 1 background conversation
+      const remaining = Math.min(1, 3 - userRows.length);
+      const bgRows = db
+        .select({ summary: memorySummaries.summary })
+        .from(memorySummaries)
+        .innerJoin(
+          conversations,
+          eq(memorySummaries.scopeKey, conversations.id),
+        )
+        .where(and(baseWhere, eq(conversations.conversationType, "background")))
+        .orderBy(desc(memorySummaries.updatedAt))
+        .limit(remaining)
+        .all();
+
+      return [...userRows, ...bgRows].map((r) => r.summary);
+    } catch (err) {
+      log.warn({ err }, "Failed to fetch recent conversation summaries");
+      return [];
+    }
+  }
+
+  /**
+   * Notify that context compaction just happened.
+   * On the next turn, we'll re-run full context load.
+   */
+  onCompacted(compactedMessageCount: number): void {
+    // Evict everything — compaction summarized all prior turns.
+    // The tracker can't know exactly which turns were compacted,
+    // so we conservatively clear everything and reload.
+    this.tracker.evictCompactedTurns(this.tracker.getTurn());
+    this.needsReload = true;
+    this.lastCompactedAt = Date.now();
+    log.info(
+      { compactedMessageCount },
+      "Compaction detected — will reload context on next turn",
+    );
+  }
+
+  /**
+   * Re-inject the most recent memory block after context compaction.
+   * Synchronous — reuses the cached block from the last successful retrieval.
+   * Does NOT advance turn count or run new retrieval.
+   */
+  reinjectCachedMemory(messages: Message[]): {
+    runMessages: Message[];
+    injectedTokens: number;
+  } {
+    if (!this.lastInjectedBlock) {
+      return { runMessages: messages, injectedTokens: 0 };
+    }
+    // Re-track node IDs since onCompacted evicted them
+    this.tracker.add(this.lastInjectedNodeIds);
+    // Strip any existing <memory __injected> blocks from the last user message
+    // before re-injecting, so compaction sites don't end up with duplicates.
+    const cleaned = stripExistingMemoryInjections(messages);
+
+    const injectedTokens =
+      estimateTextTokens(this.lastInjectedBlock) +
+      this.lastInjectedImages.size * ESTIMATED_IMAGE_TOKENS;
+
+    if (this.lastInjectedImages.size > 0) {
+      return {
+        runMessages: injectMemoryBlock(
+          cleaned,
+          this.lastInjectedBlock,
+          this.lastInjectedImages,
+        ),
+        injectedTokens,
+      };
+    }
+
+    return {
+      runMessages: injectTextBlock(cleaned, this.lastInjectedBlock),
+      injectedTokens,
+    };
+  }
+
+  /**
+   * Main entry point — called on every turn before the LLM sees the messages.
+   *
+   * Dispatches to the appropriate retrieval mode:
+   * - Turn 1 (or after compaction): full context load
+   * - Every 5 turns: periodic refresh
+   * - Every other turn: per-turn injection
+   *
+   * Returns augmented messages with memory context prepended to the last
+   * user message, following the same injection pattern as the old system.
+   */
+  async prepareMemory(
+    messages: Message[],
+    config: AssistantConfig,
+    abortSignal: AbortSignal,
+    onEvent: (msg: ServerMessage) => void,
+  ): Promise<{
+    runMessages: Message[];
+    injectedTokens: number;
+    latencyMs: number;
+    mode: "context-load" | "refresh" | "per-turn" | "none";
+    /** The raw text content of the injected block (without XML wrapper), or null if nothing was injected. */
+    injectedBlockText: string | null;
+  }> {
+    this.turnCount++;
+    this.tracker.advanceTurn();
+
+    const noopResult = {
+      runMessages: messages,
+      injectedTokens: 0,
+      latencyMs: 0,
+      mode: "none" as const,
+      injectedBlockText: null as string | null,
+    };
+
+    // Gate: skip for empty/tool-result-only messages — unless we need to
+    // reload after compaction (needsReload) or haven't initialized yet.
+    const lastMessage = messages[messages.length - 1];
+    if (!lastMessage || lastMessage.role !== "user") return noopResult;
+    const hasUserContent = lastMessage.content.some(
+      (block) => block.type === "text" && block.text.trim().length > 0,
+    );
+    if (!hasUserContent && this.initialized && !this.needsReload)
+      return noopResult;
+
+    try {
+      // Decide which retrieval mode to use
+      if (!this.initialized || this.needsReload) {
+        const recentSummaries = this.fetchRecentSummaries();
+
+        // Extract the first user message as an additional retrieval signal
+        // so context-load biases toward what the user is asking about
+        const firstUserText = extractUserText(lastMessage);
+        if (firstUserText) {
+          recentSummaries.unshift(firstUserText);
+        }
+
+        return await this.runContextLoad(
+          messages,
+          config,
+          recentSummaries,
+          abortSignal,
+          onEvent,
+        );
+      }
+
+      if (this.turnCount % REFRESH_INTERVAL_TURNS === 0) {
+        return await this.runRefresh(messages, config, abortSignal);
+      }
+
+      return await this.runPerTurn(messages, config, abortSignal);
+    } catch (err) {
+      log.warn(
+        { err: err instanceof Error ? err.message : String(err) },
+        "Memory retrieval failed (non-fatal)",
+      );
+      return noopResult;
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+  // Retrieval modes
+  // ---------------------------------------------------------------------------
+
+  private async runContextLoad(
+    messages: Message[],
+    config: AssistantConfig,
+    recentSummaries: string[],
+    signal: AbortSignal,
+    onEvent: (msg: ServerMessage) => void,
+  ) {
+    const result = await loadContextMemory({
+      scopeId: this.scopeId,
+      recentSummaries,
+      config,
+      signal,
+    });
+
+    this.initialized = true;
+    this.needsReload = false;
+
+    if (result.nodes.length === 0) {
+      this.lastInjectedBlock = null;
+      this.lastInjectedNodeIds = [];
+      this.lastInjectedImages = new Map();
+      return {
+        runMessages: messages,
+        injectedTokens: 0,
+        latencyMs: result.latencyMs,
+        mode: "context-load" as const,
+        injectedBlockText: null,
+      };
+    }
+
+    // Track loaded nodes (including serendipity)
+    this.tracker.add(result.nodes.map((n) => n.node.id));
+    this.tracker.add(result.serendipityNodes.map((n) => n.node.id));
+
+    // Assemble context block
+    const contextBlock = assembleContextBlock(result.nodes, {
+      serendipityNodes: result.serendipityNodes,
+    });
+    if (!contextBlock) {
+      return {
+        runMessages: messages,
+        injectedTokens: 0,
+        latencyMs: result.latencyMs,
+        mode: "context-load" as const,
+        injectedBlockText: null,
+      };
+    }
+
+    // Resolve images from scored nodes
+    const images = await resolveInjectionImages(
+      [...result.nodes, ...result.serendipityNodes],
+      MAX_CONTEXT_LOAD_IMAGES,
+    );
+
+    const injectedTokens =
+      estimateTextTokens(contextBlock) + images.size * ESTIMATED_IMAGE_TOKENS;
+
+    onEvent({
+      type: "memory_status",
+      enabled: true,
+      degraded: false,
+    } as ServerMessage);
+
+    this.lastInjectedBlock = contextBlock;
+    this.lastInjectedNodeIds = [
+      ...result.nodes.map((n) => n.node.id),
+      ...result.serendipityNodes.map((n) => n.node.id),
+    ];
+    this.lastInjectedImages = images;
+
+    return {
+      runMessages: injectMemoryBlock(messages, contextBlock, images),
+      injectedTokens,
+      latencyMs: result.latencyMs,
+      mode: "context-load" as const,
+      injectedBlockText: contextBlock,
+    };
+  }
+
+  private async runRefresh(
+    messages: Message[],
+    config: AssistantConfig,
+    signal: AbortSignal,
+  ) {
+    // Build recent turns text from the last ~6 messages
+    const recentTurns = messages
+      .slice(-6)
+      .map((m) => {
+        const textBlocks = m.content.filter(
+          (b): b is Extract<typeof b, { type: "text" }> => b.type === "text",
+        );
+        if (textBlocks.length === 0) return "";
+        return `[${m.role}]: ${textBlocks.map((b) => b.text).join(" ")}`;
+      })
+      .filter((t) => t.length > 0)
+      .join("\n\n");
+
+    const result = await refreshContextMemory({
+      recentTurnsText: recentTurns,
+      scopeId: this.scopeId,
+      config,
+      tracker: this.tracker,
+      signal,
+    });
+
+    if (result.nodes.length === 0) {
+      this.lastInjectedBlock = null;
+      this.lastInjectedNodeIds = [];
+      this.lastInjectedImages = new Map();
+      return {
+        runMessages: messages,
+        injectedTokens: 0,
+        latencyMs: result.latencyMs,
+        mode: "refresh" as const,
+        injectedBlockText: null,
+      };
+    }
+
+    // Track new nodes
+    this.tracker.add(result.nodes.map((n) => n.node.id));
+
+    const injectionBlock = assembleInjectionBlock(result.nodes);
+    if (!injectionBlock) {
+      return {
+        runMessages: messages,
+        injectedTokens: 0,
+        latencyMs: result.latencyMs,
+        mode: "refresh" as const,
+        injectedBlockText: null,
+      };
+    }
+
+    // Resolve images from scored nodes
+    const images = await resolveInjectionImages(
+      result.nodes,
+      MAX_REFRESH_IMAGES,
+    );
+
+    this.lastInjectedBlock = injectionBlock;
+    this.lastInjectedNodeIds = result.nodes.map((n) => n.node.id);
+    this.lastInjectedImages = images;
+
+    return {
+      runMessages: injectMemoryBlock(messages, injectionBlock, images),
+      injectedTokens:
+        estimateTextTokens(injectionBlock) +
+        images.size * ESTIMATED_IMAGE_TOKENS,
+      latencyMs: result.latencyMs,
+      mode: "refresh" as const,
+      injectedBlockText: injectionBlock,
+    };
+  }
+
+  private async runPerTurn(
+    messages: Message[],
+    config: AssistantConfig,
+    signal: AbortSignal,
+  ) {
+    // Extract last assistant and user messages as text
+    let assistantLast = "";
+    let userLast = "";
+    let userLastBlocks: ContentBlock[] = [];
+
+    for (let i = messages.length - 1; i >= 0; i--) {
+      const msg = messages[i];
+      const text = msg.content
+        .filter(
+          (b): b is Extract<typeof b, { type: "text" }> => b.type === "text",
+        )
+        .map((b) => b.text)
+        .join(" ");
+
+      if (msg.role === "user") {
+        if (userLastBlocks.length === 0) {
+          userLastBlocks = msg.content;
+          userLast = text;
+        }
+      } else if (msg.role === "assistant" && !assistantLast) {
+        assistantLast = text;
+      }
+      if (userLastBlocks.length > 0 && assistantLast) break;
+    }
+
+    const result = await retrieveForTurn({
+      assistantLastMessage: assistantLast,
+      userLastMessage: userLast,
+      userLastMessageBlocks: userLastBlocks,
+      scopeId: this.scopeId,
+      config,
+      tracker: this.tracker,
+      signal,
+    });
+
+    if (result.nodes.length === 0) {
+      this.lastInjectedBlock = null;
+      this.lastInjectedNodeIds = [];
+      this.lastInjectedImages = new Map();
+      return {
+        runMessages: messages,
+        injectedTokens: 0,
+        latencyMs: result.latencyMs,
+        mode: "per-turn" as const,
+        injectedBlockText: null,
+      };
+    }
+
+    // Track new nodes
+    this.tracker.add(result.nodes.map((n) => n.node.id));
+
+    const injectionBlock = assembleInjectionBlock(result.nodes);
+    if (!injectionBlock) {
+      return {
+        runMessages: messages,
+        injectedTokens: 0,
+        latencyMs: result.latencyMs,
+        mode: "per-turn" as const,
+        injectedBlockText: null,
+      };
+    }
+
+    // Resolve images from scored nodes
+    const images = await resolveInjectionImages(
+      result.nodes,
+      MAX_PER_TURN_IMAGES,
+    );
+
+    this.lastInjectedBlock = injectionBlock;
+    this.lastInjectedNodeIds = result.nodes.map((n) => n.node.id);
+    this.lastInjectedImages = images;
+
+    return {
+      runMessages: injectMemoryBlock(messages, injectionBlock, images),
+      injectedTokens:
+        estimateTextTokens(injectionBlock) +
+        images.size * ESTIMATED_IMAGE_TOKENS,
+      latencyMs: result.latencyMs,
+      mode: "per-turn" as const,
+      injectedBlockText: injectionBlock,
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Injection helper — same pattern as old injectMemoryRecallAsUserBlock
+// ---------------------------------------------------------------------------
+
+/**
+ * Remove all memory-injected blocks from the last user message.
+ *
+ * `injectMemoryBlock` always prepends blocks in this order:
+ *   1. `<memory __injected>…</memory>` text block
+ *   2. For each image: `<memory_image>…</memory_image>` text + `image` block
+ *
+ * We strip all leading blocks that match this pattern so that
+ * `reinjectCachedMemory` is idempotent — no duplicate images after compaction.
+ */
+function stripExistingMemoryInjections(messages: Message[]): Message[] {
+  if (messages.length === 0) return messages;
+  const last = messages[messages.length - 1];
+  if (!last || last.role !== "user") return messages;
+
+  // Walk from the front and skip all memory-injected blocks.
+  // The injection prefix is always contiguous at the start of content.
+  let firstNonMemory = 0;
+  const content = last.content;
+  while (firstNonMemory < content.length) {
+    const block = content[firstNonMemory];
+    if (
+      block.type === "text" &&
+      block.text.startsWith("<memory __injected>\n")
+    ) {
+      firstNonMemory++;
+    } else if (
+      block.type === "text" &&
+      block.text.startsWith("<memory_image>")
+    ) {
+      firstNonMemory++;
+    } else if (block.type === "image") {
+      firstNonMemory++;
+    } else {
+      break;
+    }
+  }
+
+  // Nothing to strip
+  if (firstNonMemory === 0) return messages;
+
+  return [
+    ...messages.slice(0, -1),
+    { ...last, content: content.slice(firstNonMemory) },
+  ];
+}
+
+function injectTextBlock(messages: Message[], text: string): Message[] {
+  if (text.trim().length === 0) return messages;
+  if (messages.length === 0) return messages;
+  // Strip existing memory blocks from the last user message first to prevent
+  // duplicates when the message was loaded from DB with a persisted block.
+  const cleaned = stripExistingMemoryInjections(messages);
+  const userTail = cleaned[cleaned.length - 1];
+  if (!userTail || userTail.role !== "user") return messages;
+  return [
+    ...cleaned.slice(0, -1),
+    {
+      ...userTail,
+      content: [
+        {
+          type: "text" as const,
+          text: `<memory __injected>\n${text}\n</memory>`,
+        },
+        ...userTail.content,
+      ],
+    },
+  ];
+}
+
+function injectMemoryBlock(
+  messages: Message[],
+  text: string,
+  images: Map<string, ResolvedImage>,
+): Message[] {
+  if (text.trim().length === 0 && images.size === 0) return messages;
+  if (messages.length === 0) return messages;
+  // Strip existing memory blocks from the last user message first to prevent
+  // duplicates when the message was loaded from DB with a persisted block.
+  const cleaned = stripExistingMemoryInjections(messages);
+  const userTail = cleaned[cleaned.length - 1];
+  if (!userTail || userTail.role !== "user") return messages;
+
+  const blocks: ContentBlock[] = [
+    { type: "text" as const, text: `<memory __injected>\n${text}\n</memory>` },
+  ];
+
+  for (const [_nodeId, img] of images) {
+    blocks.push({
+      type: "text" as const,
+      text: `<memory_image>${img.description}</memory_image>`,
+    });
+    blocks.push({
+      type: "image" as const,
+      source: {
+        type: "base64" as const,
+        media_type: img.mediaType,
+        data: img.base64Data,
+      },
+    } as ImageContent);
+  }
+
+  return [
+    ...cleaned.slice(0, -1),
+    { ...userTail, content: [...blocks, ...userTail.content] },
+  ];
+}
+
+/** Extract text content from a user message. */
+function extractUserText(message: Message): string | null {
+  const texts = message.content
+    .filter((b): b is Extract<typeof b, { type: "text" }> => b.type === "text")
+    .map((b) => b.text.trim())
+    .filter((t) => t.length > 0);
+  if (texts.length === 0) return null;
+  const joined = texts.join(" ");
+  // Skip very short messages ("hi", "yes") — they produce vague embeddings
+  return joined.length > 10 ? joined : null;
+}