@vellumai/assistant 0.11.10 → 0.11.11-staging.1

package/src/persistence/embeddings/embedding-types.ts

@@ -70,12 +70,36 @@ export function embeddingInputContentHash(input: EmbeddingInput): string {
   return hash.digest("hex");
 }
 
+/**
+ * Durable cache identity for an embedding input. Folds provider extras
+ * (Gemini task type / dimensions, custom endpoint URL / dimensions) into the
+ * bare content hash so a change that alters the vector for identical text is
+ * a cache miss. With no extras this is the bare {@link embeddingInputContentHash}.
+ */
+export function embeddingContentHashWithExtras(
+  input: EmbeddingInput,
+  extras: string[] = [],
+): string {
+  const base = embeddingInputContentHash(input);
+  if (extras.length === 0) {
+    return base;
+  }
+  return createHash("sha256")
+    .update(`${base}\0${extras.join("\0")}`)
+    .digest("hex");
+}
+
 // ---------------------------------------------------------------------------
 // Backend interface types (extracted from embedding-backend.ts to break
 // circular imports between the factory and provider implementations)
 // ---------------------------------------------------------------------------
 
-export type EmbeddingProviderName = "local" | "openai" | "gemini" | "ollama";
+export type EmbeddingProviderName =
+  | "local"
+  | "openai"
+  | "gemini"
+  | "ollama"
+  | "custom";
 
 export interface EmbeddingRequestOptions {
   signal?: AbortSignal;

package/src/persistence/job-utils.ts

@@ -7,13 +7,14 @@ import { BackendUnavailableError } from "../util/errors.js";
 import { getLogger } from "../util/logger.js";
 import { getMemoryDb } from "./db-connection.js";
 import {
+  durableEmbeddingCacheExtras,
   embedWithBackend,
   generateSparseEmbedding,
   getMemoryBackendStatus,
 } from "./embeddings/embedding-backend.js";
 import type { EmbeddingInput } from "./embeddings/embedding-types.js";
 import {
-  embeddingInputContentHash,
+  embeddingContentHashWithExtras,
   normalizeEmbeddingInput,
 } from "./embeddings/embedding-types.js";
 import { withQdrantBreaker } from "./embeddings/qdrant-circuit-breaker.js";
@@ -186,7 +187,10 @@ export async function embedAndUpsert(
     );
   }
 
-  const contentHash = embeddingInputContentHash(input);
+  const contentHash = embeddingContentHashWithExtras(
+    input,
+    durableEmbeddingCacheExtras(config, status.provider),
+  );
   let provider = status.provider;
   let model = status.model!;
   let vector: number[];

package/src/plugin-api/index.ts

@@ -164,7 +164,8 @@ export { publishEvent } from "./publish-event.js";
 // model id, a profile key, or a `ModelProfileInfo`; a bare string is resolved
 // as a model id first and then as a profile key. Profile resolution merges over
 // the workspace default and infers the provider for model-only profiles, then
-// looks up the model catalog's `supportsVision` flag (mix profiles are
+// looks up the model catalog's `supportsVision` flag, with a profile
+// `inputModalities.image` override winning when set (mix profiles are
 // vision-capable if any arm is). Returns false when nothing resolves.
 export { doesSupportVision } from "./vision-support.js";
 // Resolve a stored credential to its plaintext value — the same value

package/src/plugin-api/vision-support.test.ts

@@ -17,6 +17,12 @@ interface MockProfileEntry {
   model?: string;
   status?: string;
   mix?: Array<{ profile: string; weight: number }>;
+  inputModalities?: {
+    text?: { enabled?: boolean; supported?: boolean };
+    image?: { enabled?: boolean; supported?: boolean };
+    audio?: { enabled?: boolean; supported?: boolean };
+    video?: { enabled?: boolean; supported?: boolean };
+  };
 }
 
 let mockProfiles: Record<string, MockProfileEntry> = {};
@@ -103,6 +109,49 @@ describe("doesSupportVision", () => {
     expect(doesSupportVision(profile("unknown-model"))).toBe(false);
   });
 
+  test("an openai-compatible profile sends images when the override declares support", () => {
+    setMockConfig({
+      gateway: {
+        provider: "openai-compatible",
+        model: "qwen2.5-vl",
+        inputModalities: { image: { enabled: true, supported: true } },
+      },
+    });
+    expect(doesSupportVision(profile("gateway"))).toBe(true);
+  });
+
+  test("an openai-compatible profile without an override still fail-closes", () => {
+    setMockConfig({
+      gateway: {
+        provider: "openai-compatible",
+        model: "qwen2.5-vl",
+      },
+    });
+    expect(doesSupportVision(profile("gateway"))).toBe(false);
+  });
+
+  test("enabling image without declaring support keeps the catalog value", () => {
+    setMockConfig({
+      gateway: {
+        provider: "openai-compatible",
+        model: "qwen2.5-vl",
+        inputModalities: { image: { enabled: true } },
+      },
+    });
+    expect(doesSupportVision(profile("gateway"))).toBe(false);
+  });
+
+  test("a catalog vision model can be policy-disabled on the profile", () => {
+    setMockConfig({
+      "vision-profile": {
+        provider: "anthropic",
+        model: "claude-opus-4-6",
+        inputModalities: { image: { enabled: false } },
+      },
+    });
+    expect(doesSupportVision(profile("vision-profile"))).toBe(false);
+  });
+
   test("implies the provider from the catalog when profile only sets model", () => {
     setMockConfig({ "model-only": { model: "claude-opus-4-6" } });
     expect(doesSupportVision(profile("model-only"))).toBe(true);
@@ -144,6 +193,27 @@ describe("doesSupportVision", () => {
     expect(doesSupportVision(profile("mix-profile"))).toBe(true);
   });
 
+  test("mix profile returns true when an unknown arm declares image support", () => {
+    setMockConfig({
+      "mix-profile": {
+        mix: [
+          { profile: "text-arm", weight: 0.5 },
+          { profile: "gateway-arm", weight: 0.5 },
+        ],
+      },
+      "text-arm": {
+        provider: "fireworks",
+        model: "accounts/fireworks/models/glm-5p2",
+      },
+      "gateway-arm": {
+        provider: "openai-compatible",
+        model: "qwen2.5-vl",
+        inputModalities: { image: { enabled: true, supported: true } },
+      },
+    });
+    expect(doesSupportVision(profile("mix-profile"))).toBe(true);
+  });
+
   test("mix profile returns false when all arms are text-only", () => {
     setMockConfig({
       "mix-profile": {
@@ -315,4 +385,15 @@ describe("doesSupportVision with a bare string", () => {
     setMockConfig({});
     expect(doesSupportVision("some-unknown-string")).toBe(false);
   });
+
+  test("honors a free-text image override when looked up by model id", () => {
+    setMockConfig({
+      gateway: {
+        provider: "openai-compatible",
+        model: "qwen2.5-vl",
+        inputModalities: { image: { enabled: true, supported: true } },
+      },
+    });
+    expect(doesSupportVision("qwen2.5-vl")).toBe(true);
+  });
 });

package/src/plugin-api/vision-support.ts

@@ -22,6 +22,12 @@
  */
 
 import { resolveDefaultProfileForProvider } from "../config/default-profile-catalog.js";
+import {
+  catalogEntryFor,
+  type InputModalities,
+  modalitiesOf,
+  resolveModalityOverride,
+} from "../config/input-modalities.js";
 import { getConfig } from "../config/loader.js";
 import { resolveEntryProviderKind } from "../providers/connection-resolution.js";
 import { ROUTING_IDENTITY_PROVIDERS } from "../providers/inference/auth.js";
@@ -42,14 +48,52 @@ export function doesSupportVision(
   modelOrProfile: ModelProfileInfo | string,
 ): boolean {
   if (typeof modelOrProfile === "string") {
-    // Concrete model id first, then fall back to treating it as a profile key.
+    // Concrete catalog model id first, then a profile key, then any profile
+    // bound to that model id (so a free-text override is visible to callers
+    // that only have the provider-reported model, e.g. post-tool-use).
     return (
-      modelVision(modelOrProfile) ?? profileVision(modelOrProfile) ?? false
+      modelVision(modelOrProfile) ??
+      profileVision(modelOrProfile) ??
+      modelVisionFromBoundProfiles(modelOrProfile) ??
+      false
     );
   }
   return profileVision(modelOrProfile.key) ?? false;
 }
 
+/**
+ * Vision from a profile whose `model` field matches `model`. Used when the
+ * catalog does not know the id (openai-compatible / free-text). Any bound
+ * profile that declares image support wins so tool-result images are not
+ * captioned for a model the user marked as vision-capable.
+ */
+function modelVisionFromBoundProfiles(model: string): boolean | undefined {
+  const profiles = getConfig().llm.profiles ?? {};
+  let sawUnknown = false;
+  let sawFalse = false;
+  for (const entry of Object.values(profiles)) {
+    if (entry?.model !== model || entry.mix != null) {
+      continue;
+    }
+    const vision = resolveEntryVision(entry);
+    if (vision === true) {
+      return true;
+    }
+    if (vision === false) {
+      sawFalse = true;
+    } else {
+      sawUnknown = true;
+    }
+  }
+  if (sawUnknown) {
+    return undefined;
+  }
+  if (sawFalse) {
+    return false;
+  }
+  return undefined;
+}
+
 /**
  * Catalog vision flag for a concrete model id, or `undefined` when the catalog
  * doesn't know the model. The same model id carries the same capability under
@@ -110,13 +154,16 @@ function profileVision(profileKey: string): boolean | undefined {
 /**
  * Resolve whether a concrete (non-mix) profile entry supports vision from its
  * own `(provider, model)`, inferring the provider from the catalog when only
- * the model is set. Returns `undefined` when the effective `(provider, model)`
- * can't be determined or isn't in the catalog — an entry that omits its model
- * is not a usable resolution target, so it fails safe to "caption".
+ * the model is set. A profile `inputModalities.image` override wins over the
+ * catalog. Returns `undefined` when the effective `(provider, model)` can't
+ * be determined or isn't in the catalog and no override is set — an entry
+ * that omits its model is not a usable resolution target, so it fails safe
+ * to "caption".
  */
 function resolveEntryVision(entry: {
   provider?: string;
   model?: string;
+  inputModalities?: InputModalities | null;
 }): boolean | undefined {
   // Routing identities ("vellum"/"chatgpt") are not catalog providers; the
   // model's catalog owner is the capability source for them.
@@ -142,12 +189,12 @@ function resolveEntryVision(entry: {
     (typeof model === "string" ? getCatalogProviderForModel(model) : undefined);
 
   if (typeof effectiveProvider !== "string" || typeof model !== "string") {
-    return undefined;
+    return resolveModalityOverride(modalitiesOf(entry)?.image, undefined);
   }
 
-  const catalogProvider = PROVIDER_CATALOG.find(
-    (p) => p.id === effectiveProvider,
+  const catalogModel = catalogEntryFor(effectiveProvider, model);
+  return resolveModalityOverride(
+    modalitiesOf(entry)?.image,
+    catalogModel?.supportsVision,
   );
-  const catalogModel = catalogProvider?.models.find((m) => m.id === model);
-  return catalogModel?.supportsVision;
 }

package/src/plugins/defaults/memory/hooks/user-prompt-submit.ts

@@ -25,6 +25,10 @@
  * Registered first in the `user-prompt-submit` chain, so it heads the chain
  * ahead of history repair and title generation: those later hooks see the
  * fully assembled, memory-injected history.
+ *
+ * Memory-retrospective wakes skip this hook. Those forks already carry copied
+ * injection metadata on historical messages. The remember-pass does not need a
+ * fresh selector or router call, and that call is what this hook awaits.
  */
 
 import type {
@@ -252,6 +256,10 @@ const userPromptSubmitMemoryRetrieval: HookFunction<
   // threaded in, mirroring how `applyRuntimeInjections` self-resolves its
   // per-turn inputs.
   const conversation = findConversationOrSubagent(ctx.conversationId);
+  if (conversation?.currentCallSite === "memoryRetrospective") {
+    return;
+  }
+
   const config = getConfig();
   const abortSignal = conversation?.abortController?.signal;
   const isTrustedActor =

package/src/plugins/defaults/memory/jobs/__tests__/embed-concept-page.test.ts

@@ -67,6 +67,7 @@ mock.module(
       indices: [text.length % 100],
       values: [1],
     }),
+    durableEmbeddingCacheExtras: () => [],
     // Other exports from the real module — stubbed so adjacent imports
     // (e.g. via transitive `db.ts` → `indexer.ts`) don't crash on missing
     // names when the mock replaces the module wholesale.

package/src/plugins/defaults/memory/jobs/embed-concept-page.ts

@@ -23,10 +23,11 @@ import { and, eq } from "drizzle-orm";
 import type { AssistantConfig } from "../../../../config/types.js";
 import { getDb } from "../../../../persistence/db-connection.js";
 import {
+  durableEmbeddingCacheExtras,
   generateSparseEmbedding,
   getMemoryBackendStatus,
 } from "../../../../persistence/embeddings/embedding-backend.js";
-import { embeddingInputContentHash } from "../../../../persistence/embeddings/embedding-types.js";
+import { embeddingContentHashWithExtras } from "../../../../persistence/embeddings/embedding-types.js";
 import { withQdrantBreaker } from "../../../../persistence/embeddings/qdrant-circuit-breaker.js";
 import {
   asString,
@@ -131,7 +132,11 @@ export async function embedConceptPageJob(
   // and (optional) summary share the same provider/model — but each gets
   // its own cache row keyed by a distinct targetId so summary edits don't
   // invalidate the body cache and vice versa.
-  const bodyContentHash = embeddingInputContentHash({ type: "text", text });
+  const extras = durableEmbeddingCacheExtras(config, cacheProvider);
+  const bodyContentHash = embeddingContentHashWithExtras(
+    { type: "text", text },
+    extras,
+  );
   const bodyCache = mem
     ? readEmbeddingCache(mem, slug, cacheProvider, cacheModel, expectedDim)
     : null;
@@ -145,7 +150,10 @@ export async function embedConceptPageJob(
   const hasSummary = summaryText.length > 0;
   const summaryCacheId = `${slug}#summary`;
   const summaryContentHash = hasSummary
-    ? embeddingInputContentHash({ type: "text", text: summaryText })
+    ? embeddingContentHashWithExtras(
+        { type: "text", text: summaryText },
+        extras,
+      )
     : undefined;
   const summaryCache =
     hasSummary && mem

package/src/plugins/defaults/memory/src/__tests__/memory-worker-routes.test.ts

@@ -176,6 +176,26 @@ describe("memory_worker_status", () => {
     ).toBe(true);
   });
 
+  test("accepts custom as a documented embedding provider on the wire", async () => {
+    backendStatus = {
+      enabled: true,
+      degraded: false,
+      provider: "custom",
+      model: "text-embedding-3-small",
+      reason: null,
+    };
+
+    const res = await handler("memory_worker_status")();
+
+    expect(res).toMatchObject({
+      embedding: { provider: "custom" },
+    });
+    expect(
+      embeddingStatusSchema.safeParse((res as { embedding: unknown }).embedding)
+        .success,
+    ).toBe(true);
+  });
+
   test("surfaces a degraded embedding backend with a reason when none is configured", async () => {
     backendStatus = {
       enabled: true,

package/src/plugins/defaults/memory/src/memory-worker-routes.ts

@@ -32,7 +32,9 @@ const log = getLogger("memory-worker-routes");
 export const embeddingStatusSchema = z.object({
   enabled: z.boolean(),
   degraded: z.boolean(),
-  provider: z.enum(["local", "openai", "gemini", "ollama"]).nullable(),
+  provider: z
+    .enum(["local", "openai", "gemini", "ollama", "custom"])
+    .nullable(),
   model: z.string().nullable(),
   reason: z.string().nullable(),
 });

package/src/plugins/defaults/memory/substrate/__tests__/reembed-job.test.ts

@@ -54,6 +54,7 @@ mock.module(
       reason: null,
     }),
     selectedBackendSupportsMultimodal: async () => false,
+    durableEmbeddingCacheExtras: () => [],
   }),
 );
 

package/src/plugins/defaults/memory/v2/__tests__/backfill-jobs.test.ts

@@ -131,6 +131,7 @@ mock.module(
       reason: null,
     }),
     selectedBackendSupportsMultimodal: async () => false,
+    durableEmbeddingCacheExtras: () => [],
   }),
 );
 

package/src/plugins/defaults/memory/v3/__tests__/section-dense-store.test.ts

@@ -33,6 +33,8 @@ const embedState = {
   // the section cache treats it as a miss.
   geminiTaskType: undefined as string | undefined,
   geminiDimensions: undefined as number | undefined,
+  customBaseUrl: undefined as string | undefined,
+  customDimensions: undefined as number | undefined,
 };
 mock.module(
   "../../../../../persistence/embeddings/embedding-backend.js",
@@ -64,6 +66,42 @@ mock.module(
       }
       return extras;
     },
+    customCacheExtras: () => {
+      const extras: string[] = [];
+      if (embedState.customBaseUrl) {
+        extras.push(`url=${embedState.customBaseUrl}`);
+      }
+      if (embedState.customDimensions != null) {
+        extras.push(`dim=${embedState.customDimensions}`);
+      }
+      return extras;
+    },
+    durableEmbeddingCacheExtras: (
+      _config: unknown,
+      provider: string | null,
+    ) => {
+      if (provider === "gemini") {
+        const extras: string[] = [];
+        if (embedState.geminiTaskType) {
+          extras.push(`task=${embedState.geminiTaskType}`);
+        }
+        if (embedState.geminiDimensions != null) {
+          extras.push(`dim=${embedState.geminiDimensions}`);
+        }
+        return extras;
+      }
+      if (provider === "custom") {
+        const extras: string[] = [];
+        if (embedState.customBaseUrl) {
+          extras.push(`url=${embedState.customBaseUrl}`);
+        }
+        if (embedState.customDimensions != null) {
+          extras.push(`dim=${embedState.customDimensions}`);
+        }
+        return extras;
+      }
+      return [];
+    },
   }),
 );
 
@@ -320,6 +358,8 @@ function resetState(): void {
   embedState.embedModel = "test-model";
   embedState.geminiTaskType = undefined;
   embedState.geminiDimensions = undefined;
+  embedState.customBaseUrl = undefined;
+  embedState.customDimensions = undefined;
   cacheState.store.clear();
   cacheState.reads.length = 0;
   checkpointState.deletes.length = 0;
@@ -767,6 +807,48 @@ describe("memory v3 section-dense-store — embedding cache", () => {
     expect(embedState.calls).toHaveLength(1);
     expect(state.upsertCalls).toHaveLength(2);
   });
+
+  test("a custom endpoint change re-embeds unchanged text (cache miss)", async () => {
+    state.collectionExists = true;
+    embedState.statusProvider = "custom";
+    embedState.statusModel = "text-embedding-3-small";
+    embedState.embedProvider = "custom";
+    embedState.embedModel = "text-embedding-3-small";
+    embedState.customBaseUrl = "http://127.0.0.1:4000/v1";
+
+    await upsertSections(CONFIG, [
+      section("people/alice", 0, "alice lead text"),
+    ]);
+    expect(embedState.calls).toHaveLength(1);
+
+    embedState.customBaseUrl = "http://127.0.0.1:4001/v1";
+
+    await upsertSections(CONFIG, [
+      section("people/alice", 0, "alice lead text"),
+    ]);
+
+    expect(embedState.calls).toEqual([
+      ["alice lead text"],
+      ["alice lead text"],
+    ]);
+  });
+
+  test("an unchanged custom endpoint still serves from cache", async () => {
+    state.collectionExists = true;
+    embedState.statusProvider = "custom";
+    embedState.statusModel = "text-embedding-3-small";
+    embedState.embedProvider = "custom";
+    embedState.embedModel = "text-embedding-3-small";
+    embedState.customBaseUrl = "http://127.0.0.1:4000/v1";
+
+    const sections = [section("people/alice", 0, "alice lead text")];
+
+    await upsertSections(CONFIG, sections);
+    await upsertSections(CONFIG, sections);
+
+    expect(embedState.calls).toHaveLength(1);
+    expect(state.upsertCalls).toHaveLength(2);
+  });
 });
 
 describe("memory v3 section-dense-store — delete", () => {

package/src/plugins/defaults/memory/v3/section-dense-store.ts

@@ -19,8 +19,6 @@
 // shared Qdrant client; the read side (`dense.ts`) reuses that client to query
 // this collection.
 
-import { createHash } from "node:crypto";
-
 import { QdrantClient as QdrantRestClient } from "@qdrant/js-client-rest";
 import { v5 as uuidv5 } from "uuid";
 
@@ -31,14 +29,14 @@ import {
   setMemoryCheckpoint,
 } from "../../../../persistence/checkpoints.js";
 import {
-  geminiCacheExtras,
+  durableEmbeddingCacheExtras,
   getMemoryBackendStatus,
 } from "../../../../persistence/embeddings/embedding-backend.js";
 import {
   readEmbeddingCache,
   writeEmbeddingCache,
 } from "../../../../persistence/embeddings/embedding-cache.js";
-import { embeddingInputContentHash } from "../../../../persistence/embeddings/embedding-types.js";
+import { embeddingContentHashWithExtras } from "../../../../persistence/embeddings/embedding-types.js";
 import { embedWithBackend, resolveQdrantUrl } from "../embeddings.js";
 import { getLogger } from "../logging.js";
 import { memoryDbOrNull } from "../memory-db.js";
@@ -514,24 +512,6 @@ function sectionCacheId(article: string, ordinal: number): string {
   return `${article}#${ordinal}`;
 }
 
-/**
- * Content hash a section's cached vector is keyed by. Folds the provider's
- * embedding-option extras (Gemini task type / output dimensions) into the base
- * text hash, so changing an option that alters the vector for identical text is
- * a cache miss that re-embeds. With no extras the bare text hash is returned
- * unchanged, keeping existing rows valid for non-Gemini and default-Gemini
- * configs.
- */
-function sectionContentHash(text: string, extras: string[]): string {
-  const base = embeddingInputContentHash({ type: "text", text });
-  if (extras.length === 0) {
-    return base;
-  }
-  return createHash("sha256")
-    .update(`${base}\0${extras.join("\0")}`)
-    .digest("hex");
-}
-
 /**
  * Embed each section's `text` and upsert one point per section, keyed by a
  * deterministic `(article, ordinal)`-derived ID. Stable IDs mean re-upserting
@@ -641,11 +621,13 @@ async function embedSectionsCached(
   const status = await getMemoryBackendStatus(config);
   const mem = memoryDbOrNull("embedSectionsCached");
 
-  // Only Gemini's options change the vector for identical text, so fold the
-  // extras into the cache identity only when Gemini is the resolved provider;
-  // other backends keep the bare text hash. See {@link sectionContentHash}.
-  const extras = status.provider === "gemini" ? geminiCacheExtras(config) : [];
-  const hashes = sections.map((s) => sectionContentHash(s.text, extras));
+  // Fold provider extras that change the vector for identical text (Gemini
+  // task type / dimensions, custom endpoint URL / dimensions) into the
+  // durable cache identity. Other backends keep the bare text hash.
+  const extras = durableEmbeddingCacheExtras(config, status.provider);
+  const hashes = sections.map((s) =>
+    embeddingContentHashWithExtras({ type: "text", text: s.text }, extras),
+  );
 
   const result: Array<number[] | undefined> = new Array(sections.length);
   const missIndices: number[] = [];

package/src/providers/__tests__/provider-secret-catalog.test.ts

@@ -47,4 +47,8 @@ describe("API_KEY_PROVIDERS", () => {
     expect(API_KEY_PROVIDERS).toContain("tavily");
     expect(API_KEY_PROVIDERS).toContain("firecrawl");
   });
+
+  test("includes custom for OpenAI-compatible memory embeddings", () => {
+    expect(API_KEY_PROVIDERS).toContain("custom");
+  });
 });