@vellumai/assistant 0.10.8 → 0.10.9-dev.202607152225.ec6fdbf

package/src/plugins/defaults/memory/routes/memory-worker-routes.ts

@@ -0,0 +1,162 @@
+/**
+ * Memory worker control endpoints (start / stop / status).
+ *
+ * The memory jobs worker runs as a child process the daemon spawns at startup
+ * (so it shows up in `assistant ps` and is torn down on shutdown). These routes
+ * run inside the daemon so a worker spawned on demand is a direct child of the
+ * daemon too. `start` / `stop` manage the process lifecycle directly (respawn or
+ * SIGTERM); `status` reports its liveness plus the resolved embedding-backend
+ * status so a silent embedding-backend degradation — memory enqueues fine but
+ * never embeds — is observable. The `assistant memory worker` CLI is a thin IPC
+ * client over these routes.
+ */
+
+import { z } from "zod";
+
+import { getConfigReadOnly } from "../../../../config/loader.js";
+import { getMemoryBackendStatus } from "../../../../persistence/embeddings/embedding-backend.js";
+import { ACTOR_PRINCIPALS } from "../../../../runtime/auth/route-policy.js";
+import { InternalError } from "../../../../runtime/routes/errors.js";
+import type { RouteDefinition } from "../../../../runtime/routes/types.js";
+import { getLogger } from "../../../../util/logger.js";
+import { getMemoryWorkerPidPath } from "../../../../util/platform.js";
+import {
+  MemoryWorkerSpawnError,
+  probeMemoryWorker,
+  spawnMemoryWorkerProcess,
+  stopMemoryWorkerProcess,
+} from "../worker-control.js";
+
+const log = getLogger("memory-worker-routes");
+
+export const embeddingStatusSchema = z.object({
+  enabled: z.boolean(),
+  degraded: z.boolean(),
+  provider: z.enum(["local", "openai", "gemini", "ollama"]).nullable(),
+  model: z.string().nullable(),
+  reason: z.string().nullable(),
+});
+
+const startResponseSchema = z.object({
+  pid: z.number(),
+  alreadyRunning: z.boolean(),
+  pidPath: z.string(),
+});
+
+const stopResponseSchema = z.object({
+  workerWasRunning: z.boolean(),
+  pid: z.number().optional(),
+});
+
+const statusResponseSchema = z.object({
+  status: z.enum(["running", "not_running"]),
+  pid: z.number().optional(),
+  embedding: embeddingStatusSchema,
+});
+
+/**
+ * Spawn (or reuse) the memory worker process as a child of the daemon. The
+ * worker is always the sole drainer of the memory job queue, so a worker that
+ * comes up late is fine — `terminateOnTimeout` is not set.
+ */
+async function startMemoryWorker() {
+  let result: { pid: number; alreadyRunning: boolean };
+  try {
+    result = await spawnMemoryWorkerProcess({ detached: false });
+  } catch (err) {
+    const message =
+      err instanceof MemoryWorkerSpawnError || err instanceof Error
+        ? err.message
+        : String(err);
+    log.warn({ err }, "Failed to start memory worker process");
+    throw new InternalError(message);
+  }
+
+  return {
+    pid: result.pid,
+    alreadyRunning: result.alreadyRunning,
+    pidPath: getMemoryWorkerPidPath(),
+  };
+}
+
+/**
+ * SIGTERM the memory worker process if it is running. A worker that is not
+ * running is not an error.
+ */
+function stopMemoryWorker() {
+  let before: ReturnType<typeof stopMemoryWorkerProcess>;
+  try {
+    before = stopMemoryWorkerProcess();
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    log.warn({ err }, "Failed to signal memory worker process");
+    throw new InternalError(message);
+  }
+
+  return {
+    workerWasRunning: before.status === "running",
+    ...(before.pid != null ? { pid: before.pid } : {}),
+  };
+}
+
+/**
+ * Report the memory worker process liveness from its PID file, plus the
+ * resolved embedding backend (or the degraded state when none resolves).
+ */
+async function memoryWorkerStatus() {
+  const worker = probeMemoryWorker();
+  const embedding = await getMemoryBackendStatus(getConfigReadOnly());
+
+  return {
+    status: worker.status,
+    ...(worker.pid != null ? { pid: worker.pid } : {}),
+    embedding,
+  };
+}
+
+export const ROUTES: RouteDefinition[] = [
+  {
+    operationId: "memory_worker_start",
+    endpoint: "memory/worker/start",
+    method: "POST",
+    policy: {
+      requiredScopes: ["settings.write"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    handler: startMemoryWorker,
+    summary: "Start the memory worker",
+    description:
+      "Spawns (or reuses) the memory worker process as a child of the daemon.",
+    tags: ["system"],
+    responseBody: startResponseSchema,
+  },
+  {
+    operationId: "memory_worker_stop",
+    endpoint: "memory/worker/stop",
+    method: "POST",
+    policy: {
+      requiredScopes: ["settings.write"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    handler: stopMemoryWorker,
+    summary: "Stop the memory worker",
+    description: "SIGTERMs the memory worker process if it is running.",
+    tags: ["system"],
+    responseBody: stopResponseSchema,
+  },
+  {
+    operationId: "memory_worker_status",
+    endpoint: "memory/worker/status",
+    method: "GET",
+    policy: {
+      requiredScopes: ["settings.read"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    handler: memoryWorkerStatus,
+    summary: "Memory worker status",
+    description:
+      "Reports the memory worker process liveness and the embedding-backend status (including a degraded flag and reason when no backend resolves).",
+    tags: ["system"],
+    responseBody: statusResponseSchema,
+  },
+];

package/src/plugins/defaults/memory/startup.ts

@@ -207,13 +207,12 @@ export async function runMemoryStartup(config: AssistantConfig): Promise<void> {
     }
   }
 
-  // `startMemoryJobsWorker` starts the in-process supervisor (which owns the
-  // synchronous runner and stands down when an out-of-process worker is live)
-  // and spawns the out-of-process worker at boot when `memory.worker.enabled`
-  // is set. Shutdown stops whichever worker is actually running — see
-  // shutdown-handlers.ts. The job handlers were registered synchronously by the
-  // plugin's `init` hook before this function was kicked off, so the dispatch
-  // table is populated before the worker's first claim.
+  // `startMemoryJobsWorker` spawns the out-of-process memory worker as a child
+  // of the daemon; it is the sole drainer of the memory job queue. Shutdown
+  // stops it — see shutdown-handlers.ts and the memory plugin's `shutdown`
+  // hook. The job handlers were registered synchronously by the plugin's `init`
+  // hook before this function was kicked off, so the dispatch table is
+  // populated before the worker's first claim.
   log.info("Daemon startup: starting memory worker");
   startMemoryJobsWorker();
 

package/src/plugins/defaults/memory/task-memory-cleanup.ts

@@ -8,9 +8,8 @@ import { getLogger } from "./logging.js";
 const log = getLogger("task-memory-cleanup");
 
 /**
- * Check whether a conversation belongs to a failed task run or failed
- * schedule run. Derived from durable storage (task_runs / cron_runs)
- * so the check survives daemon restarts.
+ * Check whether a conversation belongs to a failed schedule run. Derived from
+ * durable storage (cron_runs) so the check survives daemon restarts.
  */
 export function isConversationFailed(conversationId: string): boolean {
   // For reused schedule conversations the same conversation_id appears in
@@ -22,29 +21,24 @@ export function isConversationFailed(conversationId: string): boolean {
   const row = rawGet<{ found: number }>(
     "taskMemory:isConversationFailed",
     `SELECT 1 AS found
-       FROM (
-         SELECT 1 FROM task_runs WHERE conversation_id = ? AND status = 'failed'
-         UNION ALL
-         SELECT 1 FROM cron_runs
-          WHERE conversation_id = ?
-            AND status = 'error'
-            AND id = (
-              SELECT id FROM cron_runs WHERE conversation_id = ?
-              ORDER BY created_at DESC LIMIT 1
-            )
-       )
+       FROM cron_runs
+      WHERE conversation_id = ?
+        AND status = 'error'
+        AND id = (
+          SELECT id FROM cron_runs WHERE conversation_id = ?
+          ORDER BY created_at DESC LIMIT 1
+        )
       LIMIT 1`,
     conversationId,
     conversationId,
-    conversationId,
   );
   return row != null;
 }
 
 /**
  * Invalidate assistant-inferred memory graph nodes sourced *exclusively* from
- * the given conversation. Called when a background task or schedule fails —
- * the assistant's optimistic claims are not trustworthy if the task didn't
+ * the given conversation. Called when a background schedule fails — the
+ * assistant's optimistic claims are not trustworthy if the run didn't
  * complete.
  *
  * Nodes that also have sources from other non-failed conversations are left
@@ -70,11 +64,6 @@ export function invalidateAssistantInferredItemsForConversation(
         AND NOT EXISTS (
           SELECT 1 FROM json_each(source_conversations) jc2
            WHERE jc2.value != ?
-             AND NOT EXISTS (
-               SELECT 1 FROM task_runs tr
-                WHERE tr.conversation_id = jc2.value
-                  AND tr.status = 'failed'
-             )
              AND NOT EXISTS (
                -- Check only the most recent cron_run for each conversation
                -- so reused conversations with historical errors but recent

package/src/plugins/defaults/memory/tools.test.ts

@@ -11,16 +11,9 @@ import {
   test,
 } from "bun:test";
 
+import { setConfig } from "../../../__tests__/helpers/set-config.js";
 import type { ToolContext } from "../../../tools/types.js";
 
-// This test exercises v1 PKB re-index enqueue. `config.memory.v2.enabled`
-// (default `true`) makes the enqueue path skipped — force it off so the
-// v1 PKB index path stays under test.
-mock.module("../../../config/loader.js", () => ({
-  getConfig: () => ({ memory: { v2: { enabled: false } } }),
-  loadConfig: () => ({ memory: { v2: { enabled: false } } }),
-}));
-
 let tmpWorkspace: string;
 let previousWorkspaceEnv: string | undefined;
 
@@ -68,6 +61,11 @@ beforeAll(() => {
   tmpWorkspace = mkdtempSync(join(tmpdir(), "remember-tool-test-"));
   previousWorkspaceEnv = process.env.VELLUM_WORKSPACE_DIR;
   process.env.VELLUM_WORKSPACE_DIR = tmpWorkspace;
+  // This test exercises the v1 PKB re-index enqueue. `memory.v2.enabled`
+  // (default true) skips the enqueue path — force it off so the v1 PKB index
+  // path stays under test. Seed into the just-overridden workspace so the real
+  // loader reads it.
+  setConfig("memory", { v2: { enabled: false } });
 });
 
 afterAll(() => {
@@ -372,3 +370,37 @@ describe("rememberTool.execute — batch (array) content", () => {
     expect(enqueueCalls).toHaveLength(0);
   });
 });
+
+describe("rememberTool definition — page-hint guidance gating", () => {
+  // The rest of this file exercises the v1 PKB path; restore its seed.
+  afterAll(() => {
+    setConfig("memory", { v2: { enabled: false } });
+  });
+
+  test("omits the [[slug]] hint guidance in v1/PKB mode", () => {
+    setConfig("memory", { v2: { enabled: false } });
+    expect(
+      rememberTool.input_schema.properties.content.description,
+    ).not.toContain("[[slug]]");
+  });
+
+  test("includes the [[slug]] hint guidance under wiki memory (v2)", () => {
+    setConfig("memory", { v2: { enabled: true } });
+    expect(rememberTool.input_schema.properties.content.description).toContain(
+      "[[slug]] wikilinks",
+    );
+    // The gate must survive JSON serialization — that's how the schema
+    // reaches the provider wire format.
+    expect(JSON.stringify(rememberTool.input_schema)).toContain(
+      "[[slug]] wikilinks",
+    );
+  });
+
+  test("re-resolves against config on each read, without re-registration", () => {
+    const schema = rememberTool.input_schema;
+    setConfig("memory", { v2: { enabled: true } });
+    expect(schema.properties.content.description).toContain("[[slug]]");
+    setConfig("memory", { v2: { enabled: false } });
+    expect(schema.properties.content.description).not.toContain("[[slug]]");
+  });
+});

package/src/plugins/defaults/memory/tools.ts

@@ -7,7 +7,7 @@
  * memory feature (`src/memory/*`).
  */
 
-import { getConfig } from "../../../config/loader.js";
+import { getConfig, getConfigReadOnly } from "../../../config/loader.js";
 import { RiskLevel } from "../../../permissions/types.js";
 import { resolveCapabilities } from "../../../runtime/capabilities.js";
 import type {
@@ -19,6 +19,7 @@ import { runAgenticRecall } from "./context-search/agent-runner.js";
 import type { RecallInput } from "./context-search/types.js";
 import { handleRemember, type RememberInput } from "./graph/tool-handlers.js";
 import {
+  buildRememberInputSchema,
   graphRecallDefinition,
   graphRememberDefinition,
 } from "./graph/tools.js";
@@ -31,7 +32,16 @@ export const rememberTool = {
   category: "memory",
   executionTarget: "sandbox",
   defaultRiskLevel: RiskLevel.Low,
-  input_schema: graphRememberDefinition.input_schema,
+  // The [[slug]] page-hint guidance applies only under the wiki memory model
+  // (v1/PKB has no pages for hints to reference). The thunk is re-resolved on
+  // every read of the content description — the registry's finalized tool
+  // shares this schema object by reference — so a runtime edit of
+  // `memory.v2.enabled` reaches the model on the next request, in lockstep
+  // with handleRemember re-reading config per call. Read-only accessor: a
+  // definition read must never create workspace directories.
+  input_schema: buildRememberInputSchema({
+    pageHints: () => getConfigReadOnly().memory.v2.enabled,
+  }),
 
   async execute(
     input: Record<string, unknown>,

package/src/plugins/defaults/memory/v2/__tests__/activation.test.ts

@@ -19,31 +19,12 @@
 
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
-import { makeMockLogger } from "../../../../../__tests__/helpers/mock-logger.js";
 import type { AssistantConfig } from "../../../../../config/types.js";
 
 // ---------------------------------------------------------------------------
 // Module-level mocks (registered before `await import("../activation.js")`).
 // ---------------------------------------------------------------------------
 
-mock.module("../../../../../util/logger.js", () => ({
-  getLogger: () => makeMockLogger(),
-}));
-
-const STUB_QDRANT_CONFIG = {
-  memory: {
-    qdrant: {
-      url: "http://127.0.0.1:6333",
-      vectorSize: 384,
-      onDisk: true,
-    },
-  },
-};
-mock.module("../../../../../config/loader.js", () => ({
-  getConfig: () => STUB_QDRANT_CONFIG,
-  loadConfig: () => STUB_QDRANT_CONFIG,
-}));
-
 const realQdrantClient =
   await import("../../../../../persistence/embeddings/qdrant-client.js");
 mock.module("../../../../../persistence/embeddings/qdrant-client.js", () => ({

package/src/plugins/defaults/memory/v2/__tests__/backfill-jobs.test.ts

@@ -29,16 +29,12 @@ import {
   test,
 } from "bun:test";
 
-import { makeMockLogger } from "../../../../../__tests__/helpers/mock-logger.js";
+import { setConfig } from "../../../../../__tests__/helpers/set-config.js";
 
 // ---------------------------------------------------------------------------
 // Module-level mocks (registered before importing the module under test).
 // ---------------------------------------------------------------------------
 
-mock.module("../../../../../util/logger.js", () => ({
-  getLogger: () => makeMockLogger(),
-}));
-
 // Migration runner — `migrate` job wraps this. The stub records call args
 // and lets each test choose the resolution shape (success, sentinel error).
 const migrationCalls: Array<{
@@ -86,42 +82,31 @@ mock.module("../migration.js", () => ({
 }));
 
 // `qdrant.ts#ensureConceptPageCollection` reads its vector size via
-// `getConfig()` (the runtime config singleton). Stub it with a fully-
-// specified memory.qdrant block so cross-test pollution from sibling test
-// files (which install their own loader mocks) cannot strip these fields.
-const STUB_RUNTIME_CONFIG = {
-  memory: {
-    qdrant: {
-      url: "http://127.0.0.1:6333",
-      vectorSize: 3,
-      onDisk: true,
-    },
-    v2: {
-      enabled: true,
-      d: 0.3,
-      c_user: 0.3,
-      c_assistant: 0.2,
-      c_now: 0.2,
-      k: 0.5,
-      hops: 2,
-      top_k: 20,
-      epsilon: 0.01,
-      dense_weight: 0.7,
-      sparse_weight: 0.3,
-      consolidation_interval_hours: 1,
-      max_page_chars: 5000,
-    },
+// `getConfig().memory.qdrant.vectorSize`, and the activation pipeline reads the
+// `memory.v2` tuning knobs. Seed the exact block below into the test workspace
+// (in `beforeAll`, after `VELLUM_WORKSPACE_DIR` is overridden).
+const SEEDED_MEMORY_CONFIG = {
+  qdrant: {
+    url: "http://127.0.0.1:6333",
+    vectorSize: 3,
+    onDisk: true,
+  },
+  v2: {
+    enabled: true,
+    d: 0.3,
+    c_user: 0.3,
+    c_assistant: 0.2,
+    c_now: 0.2,
+    k: 0.5,
+    hops: 2,
+    top_k: 20,
+    epsilon: 0.01,
+    dense_weight: 0.7,
+    sparse_weight: 0.3,
+    consolidation_interval_hours: 1,
+    max_page_chars: 5000,
   },
 };
-mock.module("../../../../../config/loader.js", () => ({
-  getConfig: () => STUB_RUNTIME_CONFIG,
-  getConfigReadOnly: () => STUB_RUNTIME_CONFIG,
-  loadConfig: () => STUB_RUNTIME_CONFIG,
-  loadRawConfig: () => ({}) as Record<string, unknown>,
-  saveRawConfig: () => {},
-  invalidateConfigCache: () => {},
-  applyNestedDefaults: () => STUB_RUNTIME_CONFIG,
-}));
 
 // Embedding backend — `activation` calls `embedWithBackend` and
 // `generateSparseEmbedding` to build the ANN candidate query. Stub both so
@@ -196,6 +181,7 @@ beforeAll(() => {
   mkdirSync(join(tmpWorkspace, "memory", ".v2-state"), { recursive: true });
   previousWorkspaceEnv = process.env.VELLUM_WORKSPACE_DIR;
   process.env.VELLUM_WORKSPACE_DIR = tmpWorkspace;
+  setConfig("memory", SEEDED_MEMORY_CONFIG);
 });
 
 afterAll(() => {
@@ -229,7 +215,7 @@ const {
 // reads its tunables from `config.memory.v2.*`. Hand the handler a config
 // shaped just enough to satisfy both paths — materializing the full default
 // config would otherwise pull in heavy schemas that don't add value here.
-const TEST_CONFIG = STUB_RUNTIME_CONFIG as Parameters<
+const TEST_CONFIG = { memory: SEEDED_MEMORY_CONFIG } as Parameters<
   typeof memoryV2ActivationRecomputeJob
 >[1];
 
@@ -378,6 +364,43 @@ describe("memoryV2ReembedJob", () => {
     expect(total).toBe(0);
   });
 
+  test("running the fan-out twice coalesces to one pending job per page", async () => {
+    // Stacked reembed runs (e.g. several consolidations completing before the
+    // embed lane drains) must not multiply the queue: the per-slug coalesce in
+    // the enqueue helper reuses the pending row.
+    await writePage(tmpWorkspace, {
+      slug: "alice",
+      frontmatter: { edges: [], ref_files: [], ref_urls: [] },
+      body: "Alice.\n",
+    });
+    await writePage(tmpWorkspace, {
+      slug: "bob",
+      frontmatter: { edges: [], ref_files: [], ref_urls: [] },
+      body: "Bob.\n",
+    });
+
+    const first = await memoryV2ReembedJob(
+      makeJob("memory_v2_reembed"),
+      TEST_CONFIG,
+    );
+    const second = await memoryV2ReembedJob(
+      makeJob("memory_v2_reembed"),
+      TEST_CONFIG,
+    );
+
+    // Both passes fan out over every page…
+    expect(first).toBe(2);
+    expect(second).toBe(2);
+
+    // …but the queue holds one pending row per slug, not one per pass.
+    const rows = getMemoryDb()!.select().from(memoryJobs).all();
+    if (rows.length > 0) {
+      expect(rows).toHaveLength(2);
+      const slugs = rows.map((row) => JSON.parse(row.payload).slug).sort();
+      expect(slugs).toEqual(["alice", "bob"]);
+    }
+  });
+
   test("does NOT enqueue reserved meta-file slugs", async () => {
     // The four prose meta files (essentials/threads/recent/buffer) live at
     // `memory/<name>.md` and are direct-injected into the system prompt via