@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/plugins/defaults/memory-v3-shadow/pool-select.test.ts

@@ -0,0 +1,146 @@
+/**
+ * Tests for `pool-select.ts` `selectPool` — focused on error SURFACING:
+ *   - a provider call that THROWS on every attempt surfaces the underlying
+ *     provider error (e.g. an upstream HTTP 4xx) in the thrown
+ *     `MemoryV3RetrievalUnavailableError` message, rather than the generic
+ *     "no usable selection" string that hid it;
+ *   - a 200 response carrying no usable `tool_use` still throws the generic
+ *     "no usable selection" message;
+ *   - happy paths preserved: explicit ids → selection, omitted ids → keepAll,
+ *     empty pool → [].
+ *
+ * `mock.module` is process-global and leaks into sibling files in a directory
+ * run, so the provider-send-message stub DELEGATES to the real implementation
+ * (keeping the real `extractToolUse`) unless this test is actively running
+ * (`selectMockActive`) — mirrors `prune.test.ts` / `ever-injected-store.test.ts`.
+ */
+
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type {
+  ContentBlock,
+  Provider,
+  ProviderResponse,
+} from "../../../providers/types.js";
+import type { MemoryRoutingTurn, Slug } from "./types.js";
+
+const realProviderSend = {
+  ...(await import("../../../providers/provider-send-message.js")),
+};
+
+let selectMockActive = false;
+let sendMessageImpl: (() => Promise<ProviderResponse>) | null = null;
+
+const mockProvider = {
+  name: "mock-memory-v3-selector",
+  async sendMessage(): Promise<ProviderResponse> {
+    if (!sendMessageImpl) throw new Error("sendMessageImpl not configured");
+    return sendMessageImpl();
+  },
+} as unknown as Provider;
+
+mock.module("../../../providers/provider-send-message.js", () => ({
+  ...realProviderSend,
+  getConfiguredProvider: (
+    ...args: Parameters<typeof realProviderSend.getConfiguredProvider>
+  ) =>
+    selectMockActive
+      ? Promise.resolve(mockProvider)
+      : realProviderSend.getConfiguredProvider(...args),
+}));
+
+const { MemoryV3RetrievalUnavailableError, selectPool } =
+  await import("./pool-select.js");
+
+function response(content: ContentBlock[]): ProviderResponse {
+  return {
+    content,
+    model: "mock",
+    usage: { inputTokens: 0, outputTokens: 0 },
+  } as ProviderResponse;
+}
+
+const turn: MemoryRoutingTurn = {
+  conversationId: "conv-1",
+  turnNumber: 0,
+  currentMessage: "echo something back",
+  recentContext: "",
+};
+
+const pool = {
+  stable: [],
+  finder: [{ slug: "page-a" as Slug, descriptor: "a descriptor" }],
+};
+
+describe("selectPool", () => {
+  beforeEach(() => {
+    selectMockActive = true;
+    sendMessageImpl = null;
+  });
+  afterAll(() => {
+    selectMockActive = false;
+  });
+
+  test("a provider throw surfaces the underlying error in the thrown message", async () => {
+    const upstream = "Together AI API error (400): 400 status code (no body)";
+    sendMessageImpl = async () => {
+      throw new Error(upstream);
+    };
+    let caught: unknown;
+    try {
+      await selectPool(pool, turn);
+    } catch (err) {
+      caught = err;
+    }
+    expect(caught).toBeInstanceOf(MemoryV3RetrievalUnavailableError);
+    expect((caught as Error).message).toContain(
+      "provider call failed after retries",
+    );
+    // The real upstream error is no longer hidden behind the generic message.
+    expect((caught as Error).message).toContain(upstream);
+  });
+
+  test("a 200 with no usable tool_use throws the generic message", async () => {
+    sendMessageImpl = async () =>
+      response([{ type: "text", text: "I cannot call the tool." }]);
+    let caught: unknown;
+    try {
+      await selectPool(pool, turn);
+    } catch (err) {
+      caught = err;
+    }
+    expect(caught).toBeInstanceOf(MemoryV3RetrievalUnavailableError);
+    expect((caught as Error).message).toBe(
+      "memory-v3 pool selector returned no usable selection after retries",
+    );
+  });
+
+  test("explicit ids select the matching candidates", async () => {
+    sendMessageImpl = async () =>
+      response([
+        {
+          type: "tool_use",
+          id: "call-1",
+          name: "select_pages",
+          input: { ids: [1] },
+        },
+      ]);
+    expect(await selectPool(pool, turn)).toEqual([
+      { slug: "page-a", pinned: false },
+    ]);
+  });
+
+  test("omitted ids keep all candidates (recall-safe)", async () => {
+    sendMessageImpl = async () =>
+      response([
+        { type: "tool_use", id: "call-1", name: "select_pages", input: {} },
+      ]);
+    expect(await selectPool(pool, turn)).toEqual([
+      { slug: "page-a", pinned: false },
+    ]);
+  });
+
+  test("an empty candidate pool returns no selections", async () => {
+    expect(await selectPool({ stable: [], finder: [] }, turn)).toEqual([]);
+  });
+});

package/src/plugins/defaults/memory-v3-shadow/pool-select.ts

@@ -50,6 +50,7 @@
 
 import { z } from "zod";
 
+import { loadPromptOverride } from "../../../memory/prompt-override.js";
 import { cachedTextBlock } from "../../../providers/cache-control.js";
 import {
   extractToolUse,
@@ -59,7 +60,9 @@ import type {
   ContentBlock,
   Message,
   ToolDefinition,
+  ToolUseContent,
 } from "../../../providers/types.js";
+import { redactLogString } from "../../../util/log-redact.js";
 import { getLogger } from "../../../util/logger.js";
 import { truncate } from "../../../util/truncate.js";
 import { retryForResult } from "./llm-retry.js";
@@ -118,9 +121,44 @@ export interface SelectorPool {
 
 /** Tool name forced via `tool_choice`. Shared constant so tests can match it. */
 const SELECT_PAGES_TOOL_NAME = "select_pages";
+const MEMORY_V3_SELECT_CALL_SITE = "memoryV3SelectL2" as const;
 
 /** Finder-line snippets are truncated to keep the dynamic tail compact. */
 const SNIPPET_MAX_CHARS = 300;
+const ERROR_MESSAGE_MAX_CHARS = 500;
+
+type PoolSelectorAttemptFailureReason =
+  | "provider_error"
+  | "missing_tool_use"
+  | "unexpected_tool_name"
+  | "schema_mismatch";
+
+interface PoolSelectorAttemptFailure {
+  attempt: number;
+  reason: PoolSelectorAttemptFailureReason;
+  callSite: typeof MEMORY_V3_SELECT_CALL_SITE;
+  providerName: string;
+  candidateCount: number;
+  stableCount: number;
+  finderCount: number;
+  response?: {
+    model: string;
+    actualProvider?: string;
+    stopReason: string;
+    requestModel?: string;
+    responseModel?: string;
+    contentBlockTypes: string[];
+    toolUseNames: string[];
+  };
+  error?: {
+    name: string;
+    message: string;
+    provider?: string;
+    statusCode?: number;
+  };
+  toolName?: string;
+  schemaIssues?: Array<{ path: string; code: string }>;
+}
 
 const SelectPagesSchema = z.object({
   // Optional: an omitted `ids` field is the recall-safe "keep everything"
@@ -164,11 +202,93 @@ A page can be relevant because of the current situation — the date or the live
 
 If the conversation is centrally ABOUT a page (rather than only peripherally relevant to it), mark that page as pinned. Call \`select_pages\` with the chosen IDs. Omit \`ids\` only as a recall-safe fallback when you cannot judge the pool (keeps every candidate); return \`[]\` when candidates are present but none are relevant.`;
 
+/**
+ * Resolve the selector system prompt: the file at `overridePath` when it is set
+ * and usable, otherwise the bundled {@link SYSTEM_PROMPT}. Path resolution and
+ * fallback follow the shared override loader (workspace-relative; a missing,
+ * empty, oversized, or unreadable file degrades to the bundled prompt with a
+ * warning). The selector prompt takes no placeholders — the candidate pool is
+ * the user message — so an override file is used verbatim.
+ */
+export function resolveSelectorPrompt(
+  overridePath: string | null,
+  workspaceDir: string,
+): string {
+  return (
+    loadPromptOverride({
+      overridePath,
+      workspaceDir,
+      log,
+      label: "memory-v3 selector prompt",
+    }) ?? SYSTEM_PROMPT
+  );
+}
+
 /** Collapse a descriptor to one line and cap its length for a finder line. */
 function renderSnippet(descriptor: string): string {
   return truncate(descriptor.replace(/\s+/g, " ").trim(), SNIPPET_MAX_CHARS);
 }
 
+function readStringField(value: unknown, key: string): string | undefined {
+  if (value === null || typeof value !== "object") return undefined;
+  const field = (value as Record<string, unknown>)[key];
+  return typeof field === "string" && field.length > 0 ? field : undefined;
+}
+
+function summarizeResponse(response: {
+  model: string;
+  actualProvider?: string;
+  stopReason: string;
+  rawRequest?: unknown;
+  rawResponse?: unknown;
+  content: ContentBlock[];
+}): NonNullable<PoolSelectorAttemptFailure["response"]> {
+  const requestModel = readStringField(response.rawRequest, "model");
+  const responseModel = readStringField(response.rawResponse, "model");
+  return {
+    model: response.model,
+    ...(response.actualProvider
+      ? { actualProvider: response.actualProvider }
+      : {}),
+    stopReason: response.stopReason,
+    ...(requestModel ? { requestModel } : {}),
+    ...(responseModel ? { responseModel } : {}),
+    contentBlockTypes: response.content.map((block) => block.type),
+    toolUseNames: response.content
+      .filter((block): block is ToolUseContent => block.type === "tool_use")
+      .map((block) => block.name),
+  };
+}
+
+function summarizeError(error: unknown): PoolSelectorAttemptFailure["error"] {
+  const record =
+    error !== null && typeof error === "object"
+      ? (error as Record<string, unknown>)
+      : {};
+  const name =
+    error instanceof Error
+      ? error.name
+      : typeof record.name === "string"
+        ? record.name
+        : "Error";
+  const rawMessage =
+    error instanceof Error
+      ? error.message
+      : typeof record.message === "string"
+        ? record.message
+        : String(error);
+  const provider =
+    typeof record.provider === "string" ? record.provider : undefined;
+  const statusCode =
+    typeof record.statusCode === "number" ? record.statusCode : undefined;
+  return {
+    name,
+    message: truncate(redactLogString(rawMessage), ERROR_MESSAGE_MAX_CHARS),
+    ...(provider ? { provider } : {}),
+    ...(statusCode !== undefined ? { statusCode } : {}),
+  };
+}
+
 /**
  * Render the stable prefix: each core/hot card prefixed with its pool number.
  * Pure concatenation of pre-rendered cards — byte-identical across turns for
@@ -218,10 +338,15 @@ function dedupeBySlug(
  * relevant" signal); an explicit `[]` keeps none; an infrastructure failure
  * (after a short re-prompt retry) keeps none, degrading to the deterministic
  * recall lanes the orchestrator unions in.
+ *
+ * `systemPrompt` is the selector's instruction scaffold; it defaults to the
+ * bundled {@link SYSTEM_PROMPT} and is overridable via `memory.v3.selectorPromptPath`
+ * (resolved by {@link resolveSelectorPrompt} at the call site).
  */
 export async function selectPool(
   pool: SelectorPool,
   turn: MemoryRoutingTurn,
+  systemPrompt: string = SYSTEM_PROMPT,
 ): Promise<SelectedPage[]> {
   // The concatenated numbering: ids 1…m are the stable-prefix cards, ids
   // m+1… are the finder lines.
@@ -234,10 +359,15 @@ export async function selectPool(
   const keepAll = (): SelectedPage[] =>
     dedupeBySlug(ordered.map((slug) => ({ slug, pinned: false })));
 
-  const provider = await getConfiguredProvider("memoryV3SelectL2");
+  const provider = await getConfiguredProvider(MEMORY_V3_SELECT_CALL_SITE);
   if (!provider) {
     log.warn(
-      { candidateCount: ordered.length },
+      {
+        callSite: MEMORY_V3_SELECT_CALL_SITE,
+        candidateCount: ordered.length,
+        stableCount: pool.stable.length,
+        finderCount: pool.finder.length,
+      },
       "pool selector provider unavailable — failing the turn rather than dropping memory",
     );
     throw new MemoryV3RetrievalUnavailableError(
@@ -264,35 +394,132 @@ export async function selectPool(
   content.push({ type: "text", text: tailParts.join("\n") });
 
   const userMsg: Message = { role: "user", content };
+  const failures: PoolSelectorAttemptFailure[] = [];
+  let attempt = 0;
+  const recordFailure = (
+    failure: Omit<
+      PoolSelectorAttemptFailure,
+      | "callSite"
+      | "providerName"
+      | "candidateCount"
+      | "stableCount"
+      | "finderCount"
+    >,
+  ): void => {
+    const diagnostic: PoolSelectorAttemptFailure = {
+      ...failure,
+      callSite: MEMORY_V3_SELECT_CALL_SITE,
+      providerName: provider.name,
+      candidateCount: ordered.length,
+      stableCount: pool.stable.length,
+      finderCount: pool.finder.length,
+    };
+    failures.push(diagnostic);
+    log.warn(diagnostic, "pool selector attempt failed");
+  };
 
   // One forced-tool call, retried a few times so a transient malformed response
   // (no usable tool_use, or tool input that fails the schema) re-prompts before
   // we give up. `null` from an attempt means "unusable, retry"; the provider
   // layer already backs off transient throws, so this loop adds no delay.
+  let lastError: unknown = null;
   const parsed = await retryForResult(async () => {
-    const response = await provider.sendMessage([userMsg], {
-      tools: [SELECT_PAGES_TOOL],
-      systemPrompt: SYSTEM_PROMPT,
-      config: {
-        callSite: "memoryV3SelectL2" as const,
-        tool_choice: { type: "tool" as const, name: SELECT_PAGES_TOOL_NAME },
-        // The last block of this one-shot message varies every turn; the
-        // provider's auto-applied turn-start breakpoint would land on it and
-        // pay cache_creation with no future hit. The stable-prefix block
-        // above carries its own breakpoint instead.
-        disableTurnStartCache: true,
-      },
-    });
+    attempt += 1;
+    let response: Awaited<ReturnType<typeof provider.sendMessage>>;
+    try {
+      response = await provider.sendMessage([userMsg], {
+        tools: [SELECT_PAGES_TOOL],
+        systemPrompt,
+        config: {
+          callSite: MEMORY_V3_SELECT_CALL_SITE,
+          tool_choice: { type: "tool" as const, name: SELECT_PAGES_TOOL_NAME },
+          // The last block of this one-shot message varies every turn; the
+          // provider's auto-applied turn-start breakpoint would land on it and
+          // pay cache_creation with no future hit. The stable-prefix block
+          // above carries its own breakpoint instead.
+          disableTurnStartCache: true,
+        },
+      });
+      lastError = null;
+    } catch (error) {
+      lastError = error;
+      recordFailure({
+        attempt,
+        reason: "provider_error",
+        error: summarizeError(error),
+      });
+      throw error;
+    }
     const toolBlock = extractToolUse(response);
-    if (!toolBlock || toolBlock.name !== SELECT_PAGES_TOOL_NAME) return null;
+    if (!toolBlock) {
+      recordFailure({
+        attempt,
+        reason: "missing_tool_use",
+        response: summarizeResponse(response),
+      });
+      return null;
+    }
+    if (toolBlock.name !== SELECT_PAGES_TOOL_NAME) {
+      recordFailure({
+        attempt,
+        reason: "unexpected_tool_name",
+        response: summarizeResponse(response),
+        toolName: toolBlock.name,
+      });
+      return null;
+    }
     const result = SelectPagesSchema.safeParse(toolBlock.input);
-    return result.success ? result.data : null;
+    if (!result.success) {
+      recordFailure({
+        attempt,
+        reason: "schema_mismatch",
+        response: summarizeResponse(response),
+        schemaIssues: result.error.issues.map((issue) => ({
+          path: issue.path.join("."),
+          code: issue.code,
+        })),
+      });
+      return null;
+    }
+    return result.data;
   });
 
   if (parsed === null) {
+    if (lastError !== null) {
+      // The selector's provider call threw on its final attempt (e.g. an
+      // upstream HTTP error). Surface the underlying error rather than
+      // reporting it as a model-output problem.
+      const detail =
+        lastError instanceof Error ? lastError.message : String(lastError);
+      const redactedDetail = truncate(
+        redactLogString(detail),
+        ERROR_MESSAGE_MAX_CHARS,
+      );
+      log.warn(
+        {
+          candidateCount: ordered.length,
+          stableCount: pool.stable.length,
+          finderCount: pool.finder.length,
+          callSite: MEMORY_V3_SELECT_CALL_SITE,
+          providerName: provider.name,
+          failures,
+        },
+        "pool selector provider call failed after retries — failing the turn rather than dropping memory",
+      );
+      throw new MemoryV3RetrievalUnavailableError(
+        `memory-v3 pool selector provider call failed after retries: ${redactedDetail}`,
+      );
+    }
     log.warn(
-      { candidateCount: ordered.length },
-      "pool selector could not obtain a selection after retries — failing the turn rather than dropping memory",
+      {
+        candidateCount: ordered.length,
+        stableCount: pool.stable.length,
+        finderCount: pool.finder.length,
+        callSite: MEMORY_V3_SELECT_CALL_SITE,
+        providerName: provider.name,
+        failures,
+      },
+      "pool selector returned no usable tool_use after retries — failing the turn rather than dropping memory",
     );
     throw new MemoryV3RetrievalUnavailableError(
       "memory-v3 pool selector returned no usable selection after retries",

package/src/plugins/defaults/memory-v3-shadow/shadow-plugin.ts

@@ -56,7 +56,10 @@ import { computeHotSet } from "./hot-set.js";
 import { computeLearnedEdgeGraph } from "./learned-edges.js";
 import type { OrchestrateResult } from "./orchestrate.js";
 import { orchestrate } from "./orchestrate.js";
-import { MemoryV3RetrievalUnavailableError } from "./pool-select.js";
+import {
+  MemoryV3RetrievalUnavailableError,
+  resolveSelectorPrompt,
+} from "./pool-select.js";
 import { ensureSectionCollection } from "./section-dense-store.js";
 import type { SectionNeedle } from "./section-needle.js";
 import { buildSectionNeedle } from "./section-needle.js";
@@ -576,6 +579,10 @@ export async function observeTurn(
       learnedGraph: lanes.learnedGraph,
       learnedPerSeed: v3.learnedEdges.perSeed,
       learnedCap: v3.learnedEdges.cap,
+      selectorPrompt: resolveSelectorPrompt(
+        v3.selectorPromptPath,
+        getWorkspaceDir(),
+      ),
     });
 
     // A zero-selection turn over a non-trivial pool is unusual enough to be

package/src/plugins/external-plugin-loader.ts

@@ -104,24 +104,28 @@ export interface LoadExternalPluginOptions {
  * Strip the npm scope from a package name. `@vellumai/simple-memory` →
  * `simple-memory`; an unscoped name passes through unchanged.
  */
-function stripScope(name: string): string {
+export function stripScope(name: string): string {
   const match = /^@[^/]+\/(.+)$/.exec(name);
   return match ? match[1]! : name;
 }
 
-function toToolNameSegment(value: string): string {
+export function toToolNameSegment(value: string): string {
   return value.replace(/[^a-zA-Z0-9]+/g, "_").replace(/^_+|_+$/g, "") || "tool";
 }
 
-function deriveToolName(toolFileBaseName: string): string {
+export function deriveToolName(toolFileBaseName: string): string {
   return toToolNameSegment(toolFileBaseName);
 }
 
 /**
  * Dynamic-import `absolutePath` and return its default export. Throws when
  * the module has no default export — callers attribute the error.
+ *
+ * Note: Bun caches dynamic `import()` by URL and does not bust on query
+ * string or hash changes, so hot-reload of hook *content* within the same
+ * process is limited. A process restart picks up all changes.
  */
-async function importDefault<T>(absolutePath: string): Promise<T> {
+export async function importDefault<T>(absolutePath: string): Promise<T> {
   const url = pathToFileURL(absolutePath).href;
   const mod = (await import(url)) as { default?: T };
   if (mod.default === undefined) {
@@ -132,7 +136,7 @@ async function importDefault<T>(absolutePath: string): Promise<T> {
   return mod.default;
 }
 
-interface SurfaceFile {
+export interface SurfaceFile {
   /** Basename without `.js`/`.ts` extension. */
   readonly name: string;
   /** Absolute path on disk. */
@@ -148,7 +152,7 @@ interface SurfaceFile {
  * Used to walk both `hooks/` and `tools/` — neither surface needs
  * subdirectory recursion today, so this stays flat on purpose.
  */
-function listSurfaceDir(dir: string): SurfaceFile[] {
+export function listSurfaceDir(dir: string): SurfaceFile[] {
   if (!existsSync(dir) || !statSync(dir).isDirectory()) return [];
   const entries = readdirSync(dir);
   const byBase = new Map<string, string>();
@@ -374,3 +378,40 @@ export async function loadExternalPlugin(
     );
   }
 }
+
+/**
+ * Parse a plugin's `package.json` manifest from disk. Returns the plugin
+ * name (scope-stripped) and version, or `undefined` when the file is
+ * missing, unparseable, or fails schema validation.
+ *
+ * Exported so the mtime cache can discover plugin identity without going
+ * through the full `buildExternalPlugin` path.
+ */
+export async function parsePluginManifest(
+  pluginDir: string,
+): Promise<{ name: string; version: string } | undefined> {
+  const pkgPath = join(pluginDir, "package.json");
+  let rawPkg: unknown;
+  try {
+    rawPkg = JSON.parse(await readFile(pkgPath, "utf8"));
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    log.error(
+      { err, pluginDir },
+      `package.json at ${pluginDir} could not be read or parsed: ${reason}`,
+    );
+    return undefined;
+  }
+  const parsed = PluginPackageJsonSchema.safeParse(rawPkg);
+  if (!parsed.success) {
+    log.error(
+      { err: parsed.error, pluginDir },
+      `package.json at ${pluginDir} failed schema validation: ${parsed.error.message}`,
+    );
+    return undefined;
+  }
+  const pkg: PluginPackageJson = parsed.data;
+  const name = stripScope(pkg.name);
+  const version = pkg.version && pkg.version.length > 0 ? pkg.version : "0.0.0";
+  return { name, version };
+}