@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022248.91fa053

package/src/__tests__/job-handler-registration-guard.test.ts

@@ -0,0 +1,104 @@
+import { afterEach, describe, expect, it, spyOn } from "bun:test";
+
+import * as jobsWorker from "../persistence/jobs-worker.js";
+import {
+  getMemoryPersistenceHooks,
+  resetMemoryPersistenceHooksForTests,
+} from "../persistence/memory-lifecycle-hooks.js";
+import { registerDefaultPluginPersistenceHooks } from "../plugins/defaults/index.js";
+import { registerMemoryPluginJobHandlers } from "../plugins/defaults/memory/job-handler-registration.js";
+
+/**
+ * The exact memory-plugin job types. Locks the contribution against an
+ * accidental add/drop — the job-handler analog of the injector order guard.
+ */
+const MEMORY_JOB_TYPES = [
+  "embed_segment",
+  "embed_summary",
+  "backfill",
+  "rebuild_index",
+  "delete_qdrant_vectors",
+  "embed_media",
+  "embed_attachment",
+  "embed_graph_node",
+  "embed_pkb_file",
+  "graph_trigger_embed",
+  "graph_extract",
+  "graph_decay",
+  "graph_consolidate",
+  "graph_pattern_scan",
+  "graph_narrative_refine",
+  "graph_bootstrap",
+  "embed_concept_page",
+  "memory_v2_sweep",
+  "memory_v2_consolidate",
+  "memory_v2_migrate",
+  "memory_v2_reembed",
+  "memory_v2_activation_recompute",
+  "memory_v3_maintain",
+  "memory_retrospective",
+].sort();
+
+/**
+ * Job types the host owns (not a plugin): persistence cleanup, message-content
+ * lexical indexing, conversations, media, home, runtime. Registered alongside
+ * the memory handlers today; they will move to a memory-agnostic background
+ * schedule worker in a future rework.
+ */
+const NON_PLUGIN_JOB_TYPES = [
+  "prune_old_conversations",
+  "prune_old_llm_request_logs",
+  "prune_old_trace_events",
+  "prune_old_tool_invocations",
+  "index_message_lexical",
+  "purge_conversation_lexical",
+  "delete_message_lexical",
+  "backfill_lexical_index",
+  "build_conversation_summary",
+  "media_processing",
+  "conversation_analyze",
+  "generate_conversation_starters",
+].sort();
+
+function captureRegisteredTypes(register: () => void): string[] {
+  const captured: string[] = [];
+  const spy = spyOn(jobsWorker, "registerJobHandler").mockImplementation(
+    (type: string) => {
+      captured.push(type);
+    },
+  );
+  try {
+    register();
+  } finally {
+    spy.mockRestore();
+  }
+  return captured;
+}
+
+describe("job-handler registration", () => {
+  afterEach(() => {
+    resetMemoryPersistenceHooksForTests();
+  });
+
+  it("registers exactly the memory job types plus the non-plugin domain types", () => {
+    const captured = captureRegisteredTypes(registerMemoryPluginJobHandlers);
+    const expected = [...MEMORY_JOB_TYPES, ...NON_PLUGIN_JOB_TYPES].sort();
+    expect(captured.slice().sort()).toEqual(expected);
+    // Each type registered exactly once — no plugin/domain overlap, no drop.
+    expect(new Set(captured).size).toBe(captured.length);
+  });
+
+  it("the memory and domain type sets do not overlap", () => {
+    const memory = new Set(MEMORY_JOB_TYPES);
+    expect(NON_PLUGIN_JOB_TYPES.filter((t) => memory.has(t))).toEqual([]);
+  });
+
+  it("registerDefaultPluginPersistenceHooks wires the persistence-lifecycle seam (the standalone worker has no bootstrap)", () => {
+    resetMemoryPersistenceHooksForTests();
+    const before = getMemoryPersistenceHooks();
+    registerDefaultPluginPersistenceHooks();
+    // The seam must move off the no-op default — otherwise the standalone
+    // worker's fork-based retrospectives silently drop carried memory state.
+    expect(getMemoryPersistenceHooks()).not.toBe(before);
+  });
+});

package/src/__tests__/lexical-index-dual-write.test.ts

@@ -0,0 +1,426 @@
+// Integration coverage for the messages lexical-index write wiring:
+//   - a real persist (`addMessage`) enqueues one `index_message_lexical` job
+//     for the message, regardless of whether memory is enabled or disabled
+//     (message-content search is host infrastructure);
+//   - forking a conversation copies message rows WITHOUT routing through
+//     the persist path, so a fork enqueues ZERO `index_message_lexical`
+//     jobs (the fork-exclusion regression);
+//   - wiping a conversation enqueues one `purge_conversation_lexical` job.
+//
+// The heavy segment indexer (`indexMessageNow`) is stubbed to a no-op so these
+// tests exercise only the lexical enqueue seam, not the embedding/extraction
+// machinery. Memory config is real (default-enabled), flipped on disk for the
+// disabled case.
+
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { eq } from "drizzle-orm";
+
+import { makeMockLogger } from "./helpers/mock-logger.js";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeMockLogger(),
+}));
+
+// Stub the segment indexer so `onMessagePersisted` runs cheaply. The lexical
+// enqueue in the hook is a separate call and still fires. Other exports are
+// provided so any transitive importer of this module resolves. `throwFromIndex`
+// lets a test simulate a transient segment-indexing failure to prove the
+// lexical enqueue survives it.
+let throwFromIndex = false;
+mock.module("../plugins/defaults/memory/indexer.js", () => ({
+  MIN_SEGMENT_CHARS: 50,
+  indexMessageNow: async () => {
+    if (throwFromIndex) {
+      throw new Error("simulated segment-indexing failure");
+    }
+    return { indexedSegments: 0, enqueuedJobs: 0 };
+  },
+  enqueueBackfillJob: () => "",
+  enqueueRebuildIndexJob: () => "",
+}));
+
+import {
+  invalidateConfigCache,
+  loadRawConfig,
+  saveRawConfig,
+} from "../config/loader.js";
+import { consolidateAssistantMessages } from "../daemon/conversation-history.js";
+import {
+  addMessage,
+  createConversation,
+  deleteConversation,
+  deleteConversationGently,
+  deleteLastExchange,
+  deleteMessageById,
+  forkConversation,
+  getMessages,
+  updateMessageContent,
+  wipeConversation,
+} from "../persistence/conversation-crud.js";
+import { getMemoryDb } from "../persistence/db-connection.js";
+import { initializeDb } from "../persistence/db-init.js";
+import { enqueueLexicalIndexForMessage } from "../persistence/job-handlers/message-lexical.js";
+import type { MemoryJobType } from "../persistence/jobs-store.js";
+import {
+  getMemoryPersistenceHooks,
+  type MemoryPersistenceHooks,
+  registerMemoryPersistenceHooks,
+} from "../persistence/memory-lifecycle-hooks.js";
+import { memoryJobs } from "../persistence/schema/index.js";
+import { registerDefaultPluginPersistenceHooks } from "../plugins/defaults/index.js";
+
+await initializeDb();
+
+function countJobs(type: MemoryJobType, conversationId?: string): number {
+  const rows = getMemoryDb()!
+    .select({ payload: memoryJobs.payload })
+    .from(memoryJobs)
+    .where(eq(memoryJobs.type, type))
+    .all();
+  if (conversationId == null) {
+    return rows.length;
+  }
+  return rows.filter((r) => {
+    try {
+      return (
+        (JSON.parse(r.payload) as { conversationId?: string })
+          .conversationId === conversationId
+      );
+    } catch {
+      return false;
+    }
+  }).length;
+}
+
+function lexicalJobMessageIds(): string[] {
+  return getMemoryDb()!
+    .select({ payload: memoryJobs.payload })
+    .from(memoryJobs)
+    .where(eq(memoryJobs.type, "index_message_lexical"))
+    .all()
+    .map((r) => {
+      try {
+        return (
+          (JSON.parse(r.payload) as { messageId?: string }).messageId ?? ""
+        );
+      } catch {
+        return "";
+      }
+    });
+}
+
+function resetMemoryJobs(): void {
+  getMemoryDb()!.delete(memoryJobs).run();
+}
+
+function setMemoryEnabled(enabled: boolean): void {
+  const raw = loadRawConfig();
+  const memory =
+    raw.memory && typeof raw.memory === "object"
+      ? (raw.memory as Record<string, unknown>)
+      : {};
+  saveRawConfig({ ...raw, memory: { ...memory, enabled } });
+  invalidateConfigCache();
+}
+
+describe("messages lexical-index dual-write", () => {
+  beforeEach(() => {
+    resetMemoryJobs();
+    registerDefaultPluginPersistenceHooks();
+  });
+
+  afterEach(() => {
+    // Restore the default-enabled config so a disabled-case test cannot leak
+    // into later tests sharing this process.
+    setMemoryEnabled(true);
+    throwFromIndex = false;
+  });
+
+  test("addMessage enqueues one index_message_lexical job for the persisted message", async () => {
+    const conv = createConversation("Lexical persist thread");
+    const message = await addMessage(conv.id, "user", "hello lexical world");
+
+    const ids = lexicalJobMessageIds();
+    expect(ids).toContain(message.id);
+    expect(ids.filter((id) => id === message.id)).toHaveLength(1);
+  });
+
+  test("lexical job is still enqueued when segment indexing (indexMessageNow) throws", async () => {
+    // The sparse lexical job is independent of the dense embedding path, so a
+    // transient segment-indexing failure must not leave the message missing
+    // from the lexical index. `addMessage` catches the hook throw as non-fatal.
+    throwFromIndex = true;
+    const conv = createConversation("Segment failure thread");
+    const message = await addMessage(
+      conv.id,
+      "user",
+      "index me despite failure",
+    );
+
+    expect(lexicalJobMessageIds()).toContain(message.id);
+  });
+
+  test("addMessage still enqueues an index_message_lexical job when memory is disabled", async () => {
+    setMemoryEnabled(false);
+    const conv = createConversation("Disabled memory thread");
+    const message = await addMessage(
+      conv.id,
+      "user",
+      "lexically indexed regardless of memory state",
+    );
+
+    expect(lexicalJobMessageIds()).toContain(message.id);
+  });
+
+  test("enqueueLexicalIndexForMessage no-ops on an empty message id", () => {
+    enqueueLexicalIndexForMessage("");
+    expect(countJobs("index_message_lexical")).toBe(0);
+  });
+
+  test("forking a conversation enqueues ZERO index_message_lexical jobs", async () => {
+    const source = createConversation("Fork exclusion thread");
+    // Persist source messages through the real path so each enqueues a lexical
+    // job — this makes the "fork adds none" assertion meaningful.
+    await addMessage(source.id, "user", "Draft a launch plan");
+    await addMessage(source.id, "assistant", "Here is a first pass.");
+    await addMessage(source.id, "user", "Fork from here");
+
+    const beforeFork = countJobs("index_message_lexical");
+    expect(beforeFork).toBe(3);
+
+    // The fork copies every source row directly, bypassing onMessagePersisted.
+    const fork = forkConversation({ conversationId: source.id });
+    expect(fork.id).not.toBe(source.id);
+
+    // No new lexical index jobs were enqueued for the copied fork rows.
+    expect(countJobs("index_message_lexical")).toBe(beforeFork);
+    // ...and specifically none of the fork's own message ids were enqueued.
+    const enqueuedMessageIds = new Set(lexicalJobMessageIds());
+    for (const forkMessage of getMessages(fork.id)) {
+      expect(enqueuedMessageIds.has(forkMessage.id)).toBe(false);
+    }
+  });
+
+  test("wipeConversation enqueues a purge_conversation_lexical job for the conversation", async () => {
+    const conv = createConversation("Lexical wipe thread");
+    await addMessage(conv.id, "user", "index me then wipe me");
+
+    resetMemoryJobs();
+    wipeConversation(conv.id);
+
+    expect(countJobs("purge_conversation_lexical", conv.id)).toBe(1);
+  });
+
+  test("deleteConversation (direct, no route) enqueues a purge for the conversation", async () => {
+    // Retrospective startup cleanup calls deleteConversation directly, bypassing
+    // the HTTP route — the purge must still fire from the shared primitive.
+    const conv = createConversation("Direct delete thread");
+    await addMessage(conv.id, "user", "index me then delete me");
+
+    resetMemoryJobs();
+    deleteConversation(conv.id);
+
+    expect(countJobs("purge_conversation_lexical", conv.id)).toBe(1);
+  });
+
+  test("deleteConversationGently (retrospective GC) enqueues a purge for the conversation", async () => {
+    const conv = createConversation("Gentle delete thread");
+    await addMessage(conv.id, "user", "index me then gently delete me");
+
+    resetMemoryJobs();
+    await deleteConversationGently(conv.id);
+
+    expect(countJobs("purge_conversation_lexical", conv.id)).toBe(1);
+  });
+
+  test("deleteMessageById enqueues a delete_message_lexical job for the removed message", async () => {
+    const conv = createConversation("Single delete thread");
+    const message = await addMessage(conv.id, "user", "delete just me");
+
+    resetMemoryJobs();
+    deleteMessageById(message.id);
+
+    const ids = getMemoryDb()!
+      .select({ payload: memoryJobs.payload })
+      .from(memoryJobs)
+      .where(eq(memoryJobs.type, "delete_message_lexical"))
+      .all()
+      .map((r) => (JSON.parse(r.payload) as { messageId?: string }).messageId);
+    expect(ids).toEqual([message.id]);
+  });
+
+  test("deleteMessageById for a nonexistent message enqueues nothing", () => {
+    resetMemoryJobs();
+    deleteMessageById("does-not-exist");
+    expect(countJobs("delete_message_lexical")).toBe(0);
+  });
+
+  test("deleteLastExchange enqueues delete_message_lexical for every removed message", async () => {
+    // Undo bulk-deletes the last user turn + everything after it via a single
+    // `tx.delete(messages)`, bypassing `deleteMessageById`. It must still purge
+    // each removed message's lexical point.
+    const conv = createConversation("Undo thread");
+    await addMessage(conv.id, "user", "first user turn");
+    await addMessage(conv.id, "assistant", "first assistant reply");
+    const lastUser = await addMessage(conv.id, "user", "undo this turn");
+    const lastAssistant = await addMessage(
+      conv.id,
+      "assistant",
+      "and this reply",
+    );
+
+    resetMemoryJobs();
+    const removed = deleteLastExchange(conv.id);
+    expect(removed).toBe(2);
+
+    const deletedIds = getMemoryDb()!
+      .select({ payload: memoryJobs.payload })
+      .from(memoryJobs)
+      .where(eq(memoryJobs.type, "delete_message_lexical"))
+      .all()
+      .map((r) => (JSON.parse(r.payload) as { messageId?: string }).messageId);
+    expect(new Set(deletedIds)).toEqual(
+      new Set([lastUser.id, lastAssistant.id]),
+    );
+  });
+
+  test("updateMessageContent (CRUD primitive) does not enqueue a reindex on its own", async () => {
+    // The reindex is owned by the semantic seams (streaming finalize, edits,
+    // consolidation), NOT the low-level primitive — this keeps mid-stream
+    // partial flushes and tool-timing stamps from spamming reindex jobs.
+    const conv = createConversation("Raw update thread");
+    const message = await addMessage(conv.id, "user", "original text");
+
+    resetMemoryJobs();
+    updateMessageContent(
+      message.id,
+      JSON.stringify([{ type: "text", text: "edited text" }]),
+    );
+
+    expect(countJobs("index_message_lexical")).toBe(0);
+  });
+
+  test("consolidation reindexes the retained message and purges the merged-away rows", async () => {
+    const conv = createConversation("Consolidation thread");
+    const userMsg = await addMessage(conv.id, "user", "do a multi-step task");
+    const retained = await addMessage(
+      conv.id,
+      "assistant",
+      JSON.stringify([{ type: "text", text: "first assistant segment" }]),
+    );
+    const mergedAway = await addMessage(
+      conv.id,
+      "assistant",
+      JSON.stringify([{ type: "text", text: "second assistant segment" }]),
+    );
+
+    resetMemoryJobs();
+    const didConsolidate = consolidateAssistantMessages(conv.id, userMsg.id);
+    expect(didConsolidate).toBe(true);
+
+    // The retained (first) assistant row is reindexed with the merged content.
+    expect(lexicalJobMessageIds()).toContain(retained.id);
+    // The merged-away row's point is removed via delete_message_lexical.
+    const deletedIds = getMemoryDb()!
+      .select({ payload: memoryJobs.payload })
+      .from(memoryJobs)
+      .where(eq(memoryJobs.type, "delete_message_lexical"))
+      .all()
+      .map((r) => (JSON.parse(r.payload) as { messageId?: string }).messageId);
+    expect(deletedIds).toContain(mergedAway.id);
+    // The retained row is not itself scheduled for deletion.
+    expect(deletedIds).not.toContain(retained.id);
+  });
+});
+
+// The delete paths must route their cleanup through the persistence-hook seam
+// rather than importing memory internals directly, so persistence stays
+// decoupled from the plugin: single-message deletes fire `onMessagesDeleted`,
+// and whole-conversation deletes fire `onConversationDeleted` from the shared
+// `deleteConversation`/`deleteConversationGently` primitive (covering every
+// caller, not just the HTTP route).
+describe("delete paths route through the persistence-hook seam", () => {
+  const deletedBatches: string[][] = [];
+  const deletedConversations: string[] = [];
+  const spyHooks: MemoryPersistenceHooks = {
+    onMessagePersisted() {},
+    onConversationForked() {},
+    onConversationWiped() {
+      return 0;
+    },
+    onConversationDeleted(id) {
+      deletedConversations.push(id);
+    },
+    onMessagesDeleted(ids) {
+      deletedBatches.push(ids);
+    },
+    async onAllConversationsCleared() {},
+    onWorkerStartup() {},
+    countMemoryBufferLines() {
+      return 0;
+    },
+  };
+
+  beforeEach(() => {
+    resetMemoryJobs();
+    deletedBatches.length = 0;
+    deletedConversations.length = 0;
+    registerMemoryPersistenceHooks(spyHooks);
+  });
+
+  afterEach(() => {
+    // Restore the real memory hooks so later tests are unaffected.
+    registerDefaultPluginPersistenceHooks();
+  });
+
+  test("the spy hook is installed", () => {
+    expect(getMemoryPersistenceHooks()).toBe(spyHooks);
+  });
+
+  test("deleteConversation fires onConversationDeleted (covers all callers, not just the route)", async () => {
+    const conv = createConversation("Seam conversation delete");
+    await addMessage(conv.id, "user", "hello");
+
+    deletedConversations.length = 0;
+    deleteConversation(conv.id);
+
+    expect(deletedConversations).toEqual([conv.id]);
+  });
+
+  test("deleteConversationGently fires onConversationDeleted (the retrospective-GC path)", async () => {
+    const conv = createConversation("Seam gentle delete");
+    await addMessage(conv.id, "user", "hello");
+
+    deletedConversations.length = 0;
+    await deleteConversationGently(conv.id);
+
+    expect(deletedConversations).toEqual([conv.id]);
+  });
+
+  test("deleteMessageById fires onMessagesDeleted with the single id", async () => {
+    const conv = createConversation("Seam single delete");
+    const message = await addMessage(conv.id, "user", "delete me via seam");
+
+    deletedBatches.length = 0;
+    deleteMessageById(message.id);
+
+    expect(deletedBatches).toEqual([[message.id]]);
+  });
+
+  test("deleteLastExchange fires onMessagesDeleted with every removed id", async () => {
+    const conv = createConversation("Seam undo");
+    await addMessage(conv.id, "user", "first turn");
+    await addMessage(conv.id, "assistant", "first reply");
+    const lastUser = await addMessage(conv.id, "user", "undo this");
+    const lastAssistant = await addMessage(conv.id, "assistant", "and this");
+
+    deletedBatches.length = 0;
+    deleteLastExchange(conv.id);
+
+    expect(deletedBatches).toHaveLength(1);
+    expect(new Set(deletedBatches[0])).toEqual(
+      new Set([lastUser.id, lastAssistant.id]),
+    );
+  });
+});

package/src/__tests__/lifecycle-memory-v2-seed.test.ts

@@ -1,7 +1,7 @@
 /**
  * Tests for the memory-v2 skill seed gate and the v2 concept-page schema
  * rebuild gate, both invoked from the daemon startup path
- * (`assistant/src/daemon/memory-v2-startup.ts`).
+ * (`assistant/src/plugins/defaults/memory/v2/memory-v2-startup.ts`).
  *
  * The gates are exercised in isolation rather than mounting the full
  * lifecycle import graph. Coverage matrix:
@@ -142,7 +142,7 @@ const {
   maybeSeedMemoryV2Skills,
   maybeRebuildMemoryV2Concepts,
   rebuildBm25CorpusStatsAndReseedSkills,
-} = await import("../daemon/memory-v2-startup.js");
+} = await import("../plugins/defaults/memory/v2/memory-v2-startup.js");
 
 // ---------------------------------------------------------------------------
 // Test fixtures