@vellumai/assistant 0.7.0 → 0.7.1

package/src/memory/conversation-crud.ts

@@ -174,6 +174,8 @@ export interface ConversationRow {
   contextSummary: string | null;
   contextCompactedMessageCount: number;
   contextCompactedAt: number | null;
+  slackContextCompactionWatermarkTs: string | null;
+  slackContextCompactionWatermarkAt: number | null;
   conversationType: string;
   source: string;
   memoryScopeId: string;
@@ -202,6 +204,8 @@ export const parseConversation = createRowMapper<
   contextSummary: "contextSummary",
   contextCompactedMessageCount: "contextCompactedMessageCount",
   contextCompactedAt: "contextCompactedAt",
+  slackContextCompactionWatermarkTs: "slackContextCompactionWatermarkTs",
+  slackContextCompactionWatermarkAt: "slackContextCompactionWatermarkAt",
   conversationType: "conversationType",
   source: "source",
   memoryScopeId: "memoryScopeId",
@@ -225,10 +229,7 @@ export interface MessageRow {
   metadata: string | null;
 }
 
-const parseMessage = createRowMapper<
-  typeof messages.$inferSelect,
-  MessageRow
->({
+const parseMessage = createRowMapper<typeof messages.$inferSelect, MessageRow>({
   id: "id",
   conversationId: "conversationId",
   role: "role",
@@ -295,6 +296,8 @@ export function createConversation(
     contextSummary: null as string | null,
     contextCompactedMessageCount: 0,
     contextCompactedAt: null as number | null,
+    slackContextCompactionWatermarkTs: null as string | null,
+    slackContextCompactionWatermarkAt: null as number | null,
     conversationType,
     source,
     memoryScopeId,
@@ -542,6 +545,12 @@ export function forkConversation(params: {
         contextCompactedAt: preserveSourceCompactionState
           ? sourceConversation.contextCompactedAt
           : null,
+        slackContextCompactionWatermarkTs: preserveSourceCompactionState
+          ? sourceConversation.slackContextCompactionWatermarkTs
+          : null,
+        slackContextCompactionWatermarkAt: preserveSourceCompactionState
+          ? sourceConversation.slackContextCompactionWatermarkAt
+          : null,
         inferenceProfile: sourceConversation.inferenceProfile,
       })
       .where(eq(conversations.id, fc.id))
@@ -1180,6 +1189,22 @@ export function updateConversationContextWindow(
     .run();
 }
 
+export function updateConversationSlackContextWatermark(
+  id: string,
+  watermarkTs: string,
+  compactedAt: number = Date.now(),
+): void {
+  const db = getDb();
+  db.update(conversations)
+    .set({
+      slackContextCompactionWatermarkTs: watermarkTs,
+      slackContextCompactionWatermarkAt: compactedAt,
+      updatedAt: Date.now(),
+    })
+    .where(eq(conversations.id, id))
+    .run();
+}
+
 export function archiveConversation(id: string): boolean {
   const conv = getConversation(id);
   if (!conv) return false;

package/src/memory/conversation-disk-view.ts

@@ -43,11 +43,7 @@ const log = getLogger("conversation-disk-view");
 // Directory helpers
 // ---------------------------------------------------------------------------
 
-export {
-  getConversationDirName,
-  getConversationDirPath,
-  getResolvedConversationDirPath,
-};
+export { getConversationDirName, getConversationDirPath };
 
 function ensureConversationDirPath(id: string, createdAtMs: number): string {
   const dirPath = getResolvedConversationDirPath(id, createdAtMs);

package/src/memory/conversation-starter-checkpoints.ts

@@ -0,0 +1,63 @@
+/**
+ * Shared checkpoint helpers for conversation starters.
+ *
+ * Used by both the job handler (generation) and the route handler (refresh decisions).
+ */
+
+import { eq } from "drizzle-orm";
+
+import { getDb } from "./db-connection.js";
+import { rawGet } from "./raw-query.js";
+import { memoryCheckpoints } from "./schema.js";
+
+// ── Checkpoint keys ──────────────────────────────────────────────
+
+export const CK_ITEM_COUNT = "conversation_starters:item_count_at_last_gen";
+export const CK_BATCH = "conversation_starters:generation_batch";
+export const CK_LAST_GEN_AT = "conversation_starters:last_gen_at";
+
+export function checkpointKey(base: string, scopeId: string): string {
+  return `${base}:${scopeId}`;
+}
+
+export function parseCheckpointInt(value: string | undefined): number | null {
+  if (value == null) return null;
+  const n = parseInt(value, 10);
+  return Number.isNaN(n) ? null : n;
+}
+
+export function getCheckpointValue(key: string): string | undefined {
+  return getDb()
+    .select({ value: memoryCheckpoints.value })
+    .from(memoryCheckpoints)
+    .where(eq(memoryCheckpoints.key, key))
+    .get()?.value;
+}
+
+// ── Writes ───────────────────────────────────────────────────────
+
+export function upsertCheckpoint(
+  key: string,
+  value: string,
+  now: number = Date.now(),
+): void {
+  getDb()
+    .insert(memoryCheckpoints)
+    .values({ key, value, updatedAt: now })
+    .onConflictDoUpdate({
+      target: memoryCheckpoints.key,
+      set: { value, updatedAt: now },
+    })
+    .run();
+}
+
+// ── Queries ──────────────────────────────────────────────────────
+
+export function countActiveMemoryNodes(scopeId: string): number {
+  return (
+    rawGet<{ c: number }>(
+      `SELECT COUNT(*) AS c FROM memory_graph_nodes WHERE fidelity != 'gone' AND scope_id = ?`,
+      scopeId,
+    )?.c ?? 0
+  );
+}

package/src/memory/db-connection.ts

@@ -1,3 +1,6 @@
+import { realpathSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join, resolve, sep } from "node:path";
 import { Database } from "bun:sqlite";
 
 import { drizzle } from "drizzle-orm/bun-sqlite";
@@ -9,8 +12,67 @@ export type DrizzleDb = ReturnType<typeof drizzle<typeof schema>>;
 
 let db: DrizzleDb | null = null;
 
+function canonicalizePathThroughExistingParent(path: string): string {
+  const resolvedPath = resolve(path);
+  const pendingSegments: string[] = [];
+  let currentPath = resolvedPath;
+
+  while (true) {
+    try {
+      return resolve(realpathSync(currentPath), ...pendingSegments.reverse());
+    } catch {
+      const parentPath = dirname(currentPath);
+      if (parentPath === currentPath) {
+        return resolvedPath;
+      }
+      pendingSegments.push(basename(currentPath));
+      currentPath = parentPath;
+    }
+  }
+}
+
+function assertTestDbIsIsolated(): void {
+  if (
+    process.env.NODE_ENV !== "test" ||
+    process.env.VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS === "1"
+  ) {
+    return;
+  }
+
+  const workspaceDir = process.env.VELLUM_WORKSPACE_DIR?.trim();
+  if (!workspaceDir) {
+    throw new Error(
+      [
+        "Refusing to open the assistant DB during tests without VELLUM_WORKSPACE_DIR.",
+        "Run assistant tests from the assistant package so the test preload can isolate state:",
+        "  cd assistant && bun test src/path/to/file.test.ts",
+      ].join("\n"),
+    );
+  }
+
+  const resolvedWorkspaceDir =
+    canonicalizePathThroughExistingParent(workspaceDir);
+  const realWorkspaceDir = canonicalizePathThroughExistingParent(
+    process.env.VELLUM_TEST_REAL_WORKSPACE_DIR?.trim() ||
+      join(homedir(), ".vellum", "workspace"),
+  );
+  if (
+    resolvedWorkspaceDir === realWorkspaceDir ||
+    resolvedWorkspaceDir.startsWith(realWorkspaceDir + sep)
+  ) {
+    throw new Error(
+      [
+        "Refusing to open the real assistant workspace DB during tests.",
+        `VELLUM_WORKSPACE_DIR resolved to ${resolvedWorkspaceDir}.`,
+        "Use a temp workspace for tests instead.",
+      ].join("\n"),
+    );
+  }
+}
+
 export function getDb(): DrizzleDb {
   if (!db) {
+    assertTestDbIsIsolated();
     ensureDataDir();
     const sqlite = new Database(getDbPath());
     sqlite.exec("PRAGMA journal_mode=WAL");

package/src/memory/db-init.ts

@@ -11,6 +11,7 @@ import { dirname, join } from "node:path";
 
 import { getLogger } from "../util/logger.js";
 import { ensureDataDir, getDbPath } from "../util/platform.js";
+import { backfillAppConversationIds } from "./app-store.js";
 import { getDb, getSqlite } from "./db-connection.js";
 import { migrateToolCreatedItems } from "./graph/bootstrap.js";
 import {
@@ -66,6 +67,7 @@ import {
   migrateConversationsLastMessageAt,
   migrateConversationsThreadTypeIndex,
   migrateCreateConversationGraphMemoryState,
+  migrateCreateDocumentConversations,
   migrateCreateMemoryGraphNodeEdits,
   migrateCreateMemoryGraphTables,
   migrateCreateMemoryRecallLogs,
@@ -105,9 +107,11 @@ import {
   migrateLlmRequestLogMessageId,
   migrateLlmRequestLogProvider,
   migrateLlmRequestLogsCreatedAtIndex,
+  migrateLlmUsageAttribution,
   migrateMemoryGraphImageRefs,
   migrateMemoryItemSupersession,
   migrateMemoryRecallLogsQueryContext,
+  migrateMemoryV2ActivationLogs,
   migrateMessagesConversationCreatedAtIndex,
   migrateMessagesFtsBackfill,
   migrateNormalizePhoneIdentities,
@@ -154,9 +158,11 @@ import {
   migrateScheduleWakeConversationId,
   migrateSchemaIndexesAndColumns,
   migrateScrubCorruptedImageAttachments,
+  migrateSlackCompactionWatermark,
   migrateStripIntegrationPrefixFromProviderKeys,
   migrateStripPlaceholderSentinelsFromMessages,
   migrateStripThinkingFromConsolidated,
+  migrateToolInvocationsMatchedRuleId,
   migrateUsageDashboardIndexes,
   migrateUsageLlmCallCount,
   migrateVoiceInviteColumns,
@@ -390,6 +396,14 @@ export function initializeDb(): void {
     migrate230AcpSessionHistory,
     migrate231RepairMemoryGraphEventDates,
     migrateActivationState,
+    migrateMemoryV2ActivationLogs,
+    migrateCreateDocumentConversations,
+    migrateLlmUsageAttribution,
+    migrateSlackCompactionWatermark,
+    migrateToolInvocationsMatchedRuleId,
+    function migrateBackfillAppConversationIds() {
+      backfillAppConversationIds();
+    },
   ];
 
   // Run each migration step, catching and logging individual failures so one

package/src/memory/embedding-backend.ts

@@ -11,8 +11,11 @@ import { GeminiEmbeddingBackend } from "./embedding-gemini.js";
 import { OllamaEmbeddingBackend } from "./embedding-ollama.js";
 import { OpenAIEmbeddingBackend } from "./embedding-openai.js";
 import {
+  type EmbeddingBackend,
   type EmbeddingInput,
   embeddingInputContentHash,
+  type EmbeddingProviderName,
+  type EmbeddingRequestOptions,
   type MultimodalEmbeddingInput,
   normalizeEmbeddingInput,
   type SparseEmbedding,
@@ -20,7 +23,6 @@ import {
 } from "./embedding-types.js";
 
 export type { EmbeddingInput, MultimodalEmbeddingInput, TextEmbeddingInput };
-export { embeddingInputContentHash, normalizeEmbeddingInput };
 
 const log = getLogger("memory-embeddings");
 
@@ -263,22 +265,6 @@ function geminiCacheExtras(config: AssistantConfig): string[] {
   return extras;
 }
 
-export type EmbeddingProviderName = "local" | "openai" | "gemini" | "ollama";
-
-export interface EmbeddingRequestOptions {
-  signal?: AbortSignal;
-}
-
-export interface EmbeddingBackend {
-  readonly provider: EmbeddingProviderName;
-  readonly model: string;
-  embed(
-    inputs: EmbeddingInput[],
-    options?: EmbeddingRequestOptions,
-  ): Promise<number[][]>;
-  dispose?(): void;
-}
-
 export interface EmbeddingBackendSelection {
   backend: EmbeddingBackend | null;
   reason: string | null;
@@ -637,10 +623,6 @@ export async function embedWithBackend(
   throw lastErr;
 }
 
-export function logMemoryEmbeddingWarning(err: unknown, context: string): void {
-  log.warn({ err }, `Memory embeddings failed (${context})`);
-}
-
 async function selectFallbackBackends(
   config: AssistantConfig,
   exclude: EmbeddingProviderName,

package/src/memory/embedding-gemini.ts

@@ -2,8 +2,6 @@ import type {
   EmbeddingBackend,
   EmbeddingInput,
   EmbeddingRequestOptions,
-} from "./embedding-backend.js";
-import type {
   EmbeddingTaskType,
   MultimodalEmbeddingInput,
 } from "./embedding-types.js";

package/src/memory/embedding-local.ts

@@ -14,13 +14,13 @@ import {
   getEmbedWorkerPidPath,
 } from "../util/platform.js";
 import { PromiseGuard } from "../util/promise-guard.js";
-import type {
-  EmbeddingBackend,
-  EmbeddingInput,
-  EmbeddingRequestOptions,
-} from "./embedding-backend.js";
 import { EmbeddingRuntimeManager } from "./embedding-runtime-manager.js";
-import { normalizeEmbeddingInput } from "./embedding-types.js";
+import {
+  type EmbeddingBackend,
+  type EmbeddingInput,
+  type EmbeddingRequestOptions,
+  normalizeEmbeddingInput,
+} from "./embedding-types.js";
 
 const log = getLogger("memory-embedding-local");
 

package/src/memory/embedding-ollama.ts

@@ -1,10 +1,10 @@
 import { getOllamaBaseUrlEnv } from "../config/env.js";
-import type {
-  EmbeddingBackend,
-  EmbeddingInput,
-  EmbeddingRequestOptions,
-} from "./embedding-backend.js";
-import { normalizeEmbeddingInput } from "./embedding-types.js";
+import {
+  type EmbeddingBackend,
+  type EmbeddingInput,
+  type EmbeddingRequestOptions,
+  normalizeEmbeddingInput,
+} from "./embedding-types.js";
 
 interface OllamaEmbeddingsResponse {
   data?: Array<{ embedding: number[] }>;

package/src/memory/embedding-openai.ts

@@ -1,11 +1,11 @@
 import OpenAI from "openai";
 
-import type {
-  EmbeddingBackend,
-  EmbeddingInput,
-  EmbeddingRequestOptions,
-} from "./embedding-backend.js";
-import { normalizeEmbeddingInput } from "./embedding-types.js";
+import {
+  type EmbeddingBackend,
+  type EmbeddingInput,
+  type EmbeddingRequestOptions,
+  normalizeEmbeddingInput,
+} from "./embedding-types.js";
 
 export class OpenAIEmbeddingBackend implements EmbeddingBackend {
   readonly provider = "openai" as const;

package/src/memory/embedding-types.ts

@@ -67,3 +67,24 @@ export function embeddingInputContentHash(input: EmbeddingInput): string {
   }
   return hash.digest("hex");
 }
+
+// ---------------------------------------------------------------------------
+// Backend interface types (extracted from embedding-backend.ts to break
+// circular imports between the factory and provider implementations)
+// ---------------------------------------------------------------------------
+
+export type EmbeddingProviderName = "local" | "openai" | "gemini" | "ollama";
+
+export interface EmbeddingRequestOptions {
+  signal?: AbortSignal;
+}
+
+export interface EmbeddingBackend {
+  readonly provider: EmbeddingProviderName;
+  readonly model: string;
+  embed(
+    inputs: EmbeddingInput[],
+    options?: EmbeddingRequestOptions,
+  ): Promise<number[][]>;
+  dispose?(): void;
+}

package/src/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts

@@ -140,10 +140,6 @@ beforeAll(() => {
   // Seed v2 layout with a single concept page so the real injection module
   // has something concrete to render. Generic placeholders only.
   mkdirSync(join(tmpWorkspace, "memory", "concepts"), { recursive: true });
-  writeFileSync(
-    join(tmpWorkspace, "memory", "edges.json"),
-    JSON.stringify({ version: 1, edges: [] }),
-  );
   writeFileSync(
     join(tmpWorkspace, "memory", "concepts", "alice-vscode.md"),
     `---\nedges: []\nref_files: []\n---\nAlice prefers VS Code as her editor.`,
@@ -337,7 +333,7 @@ describe("ConversationGraphMemory.prepareMemory — v2 routing (per-turn path)",
 
     expect(result.mode).toBe("per-turn");
     expect(result.injectedBlockText).not.toBeNull();
-    expect(result.injectedBlockText).toContain("<memory __injected>");
+    expect(result.injectedBlockText).toContain("<memory>");
     expect(result.injectedBlockText).toContain("### alice-vscode");
 
     // The leading content block on the user message is the v2 block.
@@ -346,7 +342,7 @@ describe("ConversationGraphMemory.prepareMemory — v2 routing (per-turn path)",
     const firstBlock = lastMsg?.content[0];
     expect(firstBlock?.type).toBe("text");
     if (firstBlock?.type !== "text") throw new Error("unexpected block type");
-    expect(firstBlock.text).toContain("<memory __injected>");
+    expect(firstBlock.text).toContain("<memory>");
   });
 
   test("flag on + config on with empty Qdrant hits → no v2 block, v1 fallback skipped", async () => {
@@ -388,7 +384,7 @@ describe("ConversationGraphMemory.prepareMemory — v2 routing (context-load pat
 
     expect(result.mode).toBe("context-load");
     expect(result.injectedBlockText).not.toBeNull();
-    expect(result.injectedBlockText).toContain("<memory __injected>");
+    expect(result.injectedBlockText).toContain("<memory>");
   });
 
   test("flag off → v2 not run on first turn either", async () => {

package/src/memory/graph/conversation-graph-memory.ts

@@ -235,7 +235,7 @@ export class ConversationGraphMemory {
     }
     // Re-track node IDs since onCompacted evicted them
     this.tracker.add(this.lastInjectedNodeIds);
-    // Strip any existing <memory __injected> blocks from the last user message
+    // Strip any existing <memory> blocks from the last user message
     // before re-injecting, so compaction sites don't end up with duplicates.
     const cleaned = stripExistingMemoryInjections(messages);
 
@@ -264,7 +264,7 @@ export class ConversationGraphMemory {
    * Re-register cached node IDs with the InContextTracker after compaction
    * WITHOUT modifying messages. Use this at post-agent-loop compaction sites
    * where the memory block already survives on the original user message
-   * (since `<memory __injected>` is not stripped by stripInjectionsForCompaction).
+   * (since `<memory>` is not stripped by stripInjectionsForCompaction).
    *
    * Calling reinjectCachedMemory at these sites would inject a duplicate
    * onto the last user message — which after tool calls is a tool_result,
@@ -403,11 +403,17 @@ export class ConversationGraphMemory {
     // assistantMessage is empty: context-load fires on turn 1 / post-
     // compaction, so there is no immediately-prior assistant turn to
     // weight the activation against.
+    //
+    // Use the raw user text (no >10-char filter) so even short greetings
+    // ("hi") get a fresh top-K activation dump on the first user message.
+    // The activation pipeline is robust to weak ANN signal — it still falls
+    // back to spreading + nowText to surface candidates.
+    const rawUserText = readRawUserText(messages[messages.length - 1]);
     const v2 = await this.maybeRouteV2Injection(
       messages,
       config,
       "context-load",
-      userQuery ?? "",
+      rawUserText ?? userQuery ?? "",
       "",
     );
     if (v2.routed) {
@@ -699,7 +705,7 @@ export class ConversationGraphMemory {
  * Count the leading content blocks on a user message that were added by
  * `injectMemoryBlock`. Memory-injected images use a 3-block pattern
  * (opening `<memory_image>` text + image + closing `</memory_image>` text),
- * followed by a `<memory __injected>…</memory>` text block. A legacy
+ * followed by a `<memory>…</memory>` text block (legacy `<memory __injected>` is also accepted). A legacy
  * 2-block image pattern (no closing tag) is also accepted for backward
  * compatibility. The injection prefix is always contiguous at the start,
  * so we stop at the first non-memory block.
@@ -712,7 +718,8 @@ export function countMemoryPrefixBlocks(content: ContentBlock[]): number {
     const block = content[firstNonMemory];
     if (
       block.type === "text" &&
-      block.text.startsWith("<memory __injected>\n")
+      (block.text.startsWith("<memory>\n") ||
+        block.text.startsWith("<memory __injected>\n"))
     ) {
       firstNonMemory++;
       prevWasMemoryImageMarker = false;
@@ -747,7 +754,7 @@ export function countMemoryPrefixBlocks(content: ContentBlock[]): number {
  *
  * `injectMemoryBlock` always prepends blocks in this order:
  *   1. For each image: `<memory_image __injected>…` text + `image` + `</memory_image>` text (3-block group)
- *   2. `<memory __injected>…</memory>` text block
+ *   2. `<memory>…</memory>` text block
  *
  * We strip all leading blocks that match this pattern so that
  * `reinjectCachedMemory` is idempotent — no duplicate images after compaction.
@@ -795,7 +802,7 @@ function injectTextBlock(messages: Message[], text: string): Message[] {
       content: [
         {
           type: "text" as const,
-          text: `<memory __injected>\n${text}\n</memory>`,
+          text: `<memory>\n${text}\n</memory>`,
         },
         ...userTail.content,
       ],
@@ -839,7 +846,7 @@ function injectMemoryBlock(
 
   blocks.push({
     type: "text" as const,
-    text: `<memory __injected>\n${text}\n</memory>`,
+    text: `<memory>\n${text}\n</memory>`,
   });
 
   return [
@@ -848,20 +855,35 @@ function injectMemoryBlock(
   ];
 }
 
-/** Extract text content from a user message. */
+/**
+ * Extract text content from a user message for v1's `loadContextMemory`,
+ * skipping very short messages because v1's path embeds a single dense
+ * vector and short queries produce vague results.
+ */
 function extractUserText(message: Message): string | null {
+  const joined = readRawUserText(message);
+  if (!joined) return null;
+  return joined.length > 10 ? joined : null;
+}
+
+/**
+ * Raw user-text reader (no length filter). The v2 activation pipeline can
+ * use even short queries because it spreads activation through the edge
+ * graph and combines user/assistant/now signals, so the ≤10-char guard
+ * v1 needs would unnecessarily starve v2 on short greetings.
+ */
+function readRawUserText(message: Message | undefined): string | null {
+  if (!message) return null;
   const texts = message.content
     .filter((b): b is Extract<typeof b, { type: "text" }> => b.type === "text")
     .map((b) => b.text.trim())
     .filter((t) => t.length > 0);
   if (texts.length === 0) return null;
-  const joined = texts.join(" ");
-  // Skip very short messages ("hi", "yes") — they produce vague embeddings
-  return joined.length > 10 ? joined : null;
+  return texts.join(" ");
 }
 
 /**
- * Prepend a pre-rendered `<memory __injected>` block (produced by
+ * Prepend a pre-rendered `<memory>` block (produced by
  * `injectMemoryV2Block`) to the last user message. Unlike v1's
  * {@link injectMemoryBlock}, the input here is already wrapped — we
  * just need to attach it as a leading text block. We still strip any

package/src/memory/graph/injection.test.ts

@@ -192,9 +192,9 @@ describe("assembleContextBlock", () => {
     expect(assembleContextBlock([])).toBe("");
   });
 
-  test("includes the main heading", () => {
+  test("omits the legacy `What I Remember Right Now` heading", () => {
     const result = assembleContextBlock([makeScored()]);
-    expect(result).toContain("## What I Remember Right Now");
+    expect(result).not.toContain("## What I Remember Right Now");
   });
 
   test("puts prospective nodes under Active Threads", () => {

package/src/memory/graph/injection.ts

@@ -381,7 +381,7 @@ export function assembleContextBlock(
   }
 
   if (parts.length === 0) return "";
-  return `## What I Remember Right Now\n\n${parts.join("\n\n")}`;
+  return parts.join("\n\n");
 }
 
 function buildSection(nodes: ScoredNode[], maxItems: number): string[] {