@vellumai/assistant 0.4.44 → 0.4.45

package/src/media/managed-avatar-client.ts

@@ -1,3 +1,5 @@
+import { createHash } from "crypto";
+
 import { getPlatformBaseUrl } from "../config/env.js";
 import { getConfig } from "../config/loader.js";
 import { getSecureKey } from "../security/secure-keys.js";
@@ -7,8 +9,8 @@ import {
   AVATAR_MIME_ALLOWLIST,
   AVATAR_PROMPT_MAX_LENGTH,
   ManagedAvatarError,
-  type ManagedAvatarErrorResponse,
   type ManagedAvatarResponse,
+  VERTEX_IMAGE_DEFAULT_MODEL,
 } from "./avatar-types.js";
 
 const log = getLogger("managed-avatar-client");
@@ -30,7 +32,7 @@ export function isManagedAvailable(): boolean {
 
 export async function generateManagedAvatar(
   prompt: string,
-  options?: { correlationId?: string; idempotencyKey?: string },
+  options?: { correlationId?: string; idempotencyKey?: string; model?: string },
 ): Promise<ManagedAvatarResponse> {
   if (prompt.length > AVATAR_PROMPT_MAX_LENGTH) {
     throw new ManagedAvatarError({
@@ -68,7 +70,8 @@ export async function generateManagedAvatar(
     });
   }
 
-  const url = `${baseUrl}/v1/assistants/avatar/generate/`;
+  const model = options?.model ?? VERTEX_IMAGE_DEFAULT_MODEL;
+  const url = `${baseUrl}/v1/runtime-proxy/vertex/v1/models/${model}:predict`;
   const idempotencyKey = options?.idempotencyKey ?? crypto.randomUUID();
   const correlationId = options?.correlationId ?? crypto.randomUUID();
 
@@ -79,13 +82,23 @@ export async function generateManagedAvatar(
     "X-Correlation-Id": correlationId,
   };
 
-  log.debug({ url, correlationId }, "Requesting managed avatar generation");
+  log.debug(
+    { url, correlationId, model },
+    "Requesting managed avatar generation",
+  );
 
   let response: Response;
   try {
     response = await fetch(url, {
       method: "POST",
-      body: JSON.stringify({ prompt }),
+      body: JSON.stringify({
+        instances: [{ prompt }],
+        parameters: {
+          sampleCount: 1,
+          aspectRatio: "1:1",
+          outputOptions: { mimeType: "image/png" },
+        },
+      }),
       signal: AbortSignal.timeout(60_000),
       headers,
     });
@@ -105,64 +118,87 @@ export async function generateManagedAvatar(
   }
 
   if (!response.ok) {
-    let errorBody: ManagedAvatarErrorResponse;
+    let detail = `HTTP ${response.status}`;
     try {
-      errorBody = (await response.json()) as ManagedAvatarErrorResponse;
+      const text = await response.text();
+      if (text) detail = `HTTP ${response.status}: ${text}`;
     } catch {
-      throw new ManagedAvatarError({
-        code: "upstream_error",
-        subcode: "unparseable_response",
-        detail: `HTTP ${response.status}: unable to parse error response`,
-        retryable: response.status >= 500 || response.status === 429,
-        correlationId,
-        statusCode: response.status,
-      });
+      // ignore parse failures
     }
 
     throw new ManagedAvatarError({
-      code: errorBody.code ?? "upstream_error",
-      subcode: errorBody.subcode ?? "unknown",
-      detail: errorBody.detail ?? `HTTP ${response.status}`,
-      retryable:
-        errorBody.retryable ??
-        (response.status >= 500 || response.status === 429),
-      correlationId: errorBody.correlation_id ?? correlationId,
+      code: "upstream_error",
+      subcode: "http_error",
+      detail,
+      retryable: response.status >= 500 || response.status === 429,
+      correlationId,
       statusCode: response.status,
     });
   }
 
-  let body: ManagedAvatarResponse;
   try {
-    body = (await response.json()) as ManagedAvatarResponse;
+    const vertexBody = (await response.json()) as {
+      predictions: Array<{ bytesBase64Encoded: string; mimeType: string }>;
+    };
 
-    if (!AVATAR_MIME_ALLOWLIST.has(body.image.mime_type)) {
+    const prediction = vertexBody.predictions?.[0];
+    if (!prediction) {
+      throw new ManagedAvatarError({
+        code: "upstream_error",
+        subcode: "empty_predictions",
+        detail: "Vertex response contained no predictions",
+        retryable: false,
+        correlationId,
+        statusCode: 0,
+      });
+    }
+
+    const mimeType = prediction.mimeType;
+    const dataBase64 = prediction.bytesBase64Encoded;
+
+    if (!AVATAR_MIME_ALLOWLIST.has(mimeType)) {
       throw new ManagedAvatarError({
         code: "validation_error",
         subcode: "disallowed_mime_type",
-        detail: `Response MIME type "${body.image.mime_type}" is not in the allowlist`,
+        detail: `Response MIME type "${mimeType}" is not in the allowlist`,
         retryable: false,
         correlationId,
         statusCode: 0,
       });
     }
 
-    const b64 = body.image.data_base64;
-    const padding = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0;
-    const estimatedDecodedBytes = Math.ceil((b64.length * 3) / 4) - padding;
-    if (
-      estimatedDecodedBytes > AVATAR_MAX_DECODED_BYTES ||
-      body.image.bytes > AVATAR_MAX_DECODED_BYTES
-    ) {
+    const padding = dataBase64.endsWith("==")
+      ? 2
+      : dataBase64.endsWith("=")
+        ? 1
+        : 0;
+    const estimatedDecodedBytes =
+      Math.ceil((dataBase64.length * 3) / 4) - padding;
+    if (estimatedDecodedBytes > AVATAR_MAX_DECODED_BYTES) {
       throw new ManagedAvatarError({
         code: "validation_error",
         subcode: "oversized_image",
-        detail: `Response image size ${Math.max(estimatedDecodedBytes, body.image.bytes)} exceeds maximum of ${AVATAR_MAX_DECODED_BYTES} bytes`,
+        detail: `Response image size ${estimatedDecodedBytes} exceeds maximum of ${AVATAR_MAX_DECODED_BYTES} bytes`,
         retryable: false,
         correlationId,
         statusCode: 0,
       });
     }
 
+    const sha256 = createHash("sha256")
+      .update(Buffer.from(dataBase64, "base64"))
+      .digest("hex");
+
+    const body: ManagedAvatarResponse = {
+      image: {
+        mime_type: mimeType,
+        data_base64: dataBase64,
+        bytes: estimatedDecodedBytes,
+        sha256,
+      },
+      correlation_id: correlationId,
+    };
+
     log.debug(
       {
         correlationId,

package/src/memory/conversation-title-service.ts

@@ -30,13 +30,12 @@ export type TitleOrigin =
   | "voice_inbound"
   | "guardian_request"
   | "schedule"
-  | "reminder"
   | "task"
   | "watcher"
   | "subagent"
   | "sequence"
   | "heartbeat"
-  | "ipc"
+  | "local"
   | "task_submit"
   | "misc";
 

package/src/memory/db-init.ts

@@ -46,8 +46,10 @@ import {
   migrateContactsNotesColumn,
   migrateContactsRolePrincipal,
   migrateConversationsThreadTypeIndex,
+  migrateDropAccountsTable,
   migrateDropAssistantIdColumns,
   migrateDropLegacyMemberGuardianTables,
+  migrateDropRemindersTable,
   migrateDropUsageCompositeIndexes,
   migrateFkCascadeRebuilds,
   migrateGuardianActionFollowup,
@@ -64,10 +66,12 @@ import {
   migrateNormalizePhoneIdentities,
   migrateNotificationDeliveryThreadDecision,
   migrateReminderRoutingIntent,
+  migrateRemindersToSchedules,
   migrateRenameGuardianVerificationValues,
   migrateRenameVerificationSessionIdColumn,
   migrateRenameVerificationTable,
   migrateRenameVoiceToPhone,
+  migrateScheduleOneShotRouting,
   migrateSchemaIndexesAndColumns,
   migrateUsageDashboardIndexes,
   migrateVoiceInviteColumns,
@@ -324,6 +328,18 @@ export function initializeDb(): void {
   // 48. Rename stored "voice" channel values to "phone" across all channel text columns
   migrateRenameVoiceToPhone(database);
 
+  // 49. Drop the unused legacy accounts table after removing account_manage
+  migrateDropAccountsTable(database);
+
+  // 50. Extend cron_jobs table with one-shot and routing support
+  migrateScheduleOneShotRouting(database);
+
+  // 51. Migrate existing reminders into cron_jobs as one-shot schedules
+  migrateRemindersToSchedules(database);
+
+  // 52. Drop the legacy reminders table after data migration
+  migrateDropRemindersTable(database);
+
   validateMigrationState(database);
 
   if (process.env.BUN_TEST === "1") {

package/src/memory/embedding-backend.ts

@@ -6,6 +6,20 @@ import { getLogger } from "../util/logger.js";
 import { GeminiEmbeddingBackend } from "./embedding-gemini.js";
 import { OllamaEmbeddingBackend } from "./embedding-ollama.js";
 import { OpenAIEmbeddingBackend } from "./embedding-openai.js";
+import {
+  type EmbeddingInput,
+  embeddingInputContentHash,
+  type MultimodalEmbeddingInput,
+  normalizeEmbeddingInput,
+  type TextEmbeddingInput,
+} from "./embedding-types.js";
+
+export type {
+  EmbeddingInput,
+  MultimodalEmbeddingInput,
+  TextEmbeddingInput,
+};
+export { embeddingInputContentHash, normalizeEmbeddingInput };
 
 const log = getLogger("memory-embeddings");
 
@@ -34,12 +48,12 @@ class LazyLocalEmbeddingBackend implements EmbeddingBackend {
   }
 
   async embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
     const backend = await this.getDelegate();
     try {
-      return await backend.embed(texts, options);
+      return await backend.embed(inputs, options);
     } catch (err) {
       // The onnxruntime-node failure surfaces here during the first embed() call
       // (via LocalEmbeddingBackend.initialize()). Mark broken so auto mode stops
@@ -103,18 +117,26 @@ function estimateEntryBytes(key: string, vector: number[]): number {
   return key.length * 2 + vector.length * 8;
 }
 
-function vectorCacheKey(provider: string, model: string, text: string): string {
+function vectorCacheKey(
+  provider: string,
+  model: string,
+  input: EmbeddingInput,
+  extras?: string[],
+): string {
+  const contentHash = embeddingInputContentHash(input);
+  const suffix = extras && extras.length > 0 ? `\0${extras.join("\0")}` : "";
   return createHash("sha256")
-    .update(`${provider}\0${model}\0${text}`)
+    .update(`${provider}\0${model}\0${contentHash}${suffix}`)
     .digest("hex");
 }
 
 function getFromVectorCache(
   provider: string,
   model: string,
-  text: string,
+  input: EmbeddingInput,
+  extras?: string[],
 ): number[] | undefined {
-  const key = vectorCacheKey(provider, model, text);
+  const key = vectorCacheKey(provider, model, input, extras);
   const v = vectorCache.get(key);
   if (v !== undefined) {
     // LRU refresh: move to end of insertion order
@@ -127,10 +149,11 @@ function getFromVectorCache(
 function putInVectorCache(
   provider: string,
   model: string,
-  text: string,
+  input: EmbeddingInput,
   vector: number[],
+  extras?: string[],
 ): void {
-  const key = vectorCacheKey(provider, model, text);
+  const key = vectorCacheKey(provider, model, input, extras);
   // If replacing an existing entry, subtract its old cost first
   const existing = vectorCache.get(key);
   if (existing !== undefined) {
@@ -161,7 +184,10 @@ export function clearEmbeddingBackendCache(): void {
   localBackendBroken = false;
 }
 
-function cacheKey(provider: string, model: string): string {
+function cacheKey(provider: string, model: string, extras?: string[]): string {
+  if (extras && extras.length > 0) {
+    return `${provider}:${model}:${extras.join(":")}`;
+  }
   return `${provider}:${model}`;
 }
 
@@ -169,8 +195,9 @@ function getCachedOrCreate<T extends EmbeddingBackend>(
   provider: string,
   model: string,
   create: () => T,
+  extras?: string[],
 ): T {
-  const key = cacheKey(provider, model);
+  const key = cacheKey(provider, model, extras);
   const existing = backendCache.get(key);
   if (existing) return existing as T;
   const instance = create();
@@ -178,6 +205,17 @@ function getCachedOrCreate<T extends EmbeddingBackend>(
   return instance;
 }
 
+function geminiCacheExtras(config: AssistantConfig): string[] {
+  const extras: string[] = [];
+  if (config.memory.embeddings.geminiTaskType) {
+    extras.push(`task=${config.memory.embeddings.geminiTaskType}`);
+  }
+  if (config.memory.embeddings.geminiDimensions != null) {
+    extras.push(`dim=${config.memory.embeddings.geminiDimensions}`);
+  }
+  return extras;
+}
+
 export type EmbeddingProviderName = "local" | "openai" | "gemini" | "ollama";
 
 export interface EmbeddingRequestOptions {
@@ -188,7 +226,7 @@ export interface EmbeddingBackend {
   readonly provider: EmbeddingProviderName;
   readonly model: string;
   embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]>;
 }
@@ -272,7 +310,12 @@ export function selectEmbeddingBackend(
               new GeminiEmbeddingBackend(
                 config.apiKeys.gemini,
                 config.memory.embeddings.geminiModel,
+                {
+                  taskType: config.memory.embeddings.geminiTaskType,
+                  dimensions: config.memory.embeddings.geminiDimensions,
+                },
               ),
+            geminiCacheExtras(config),
           ),
           reason: null,
         };
@@ -336,7 +379,7 @@ export function getMemoryBackendStatus(config: AssistantConfig): {
 
 export async function embedWithBackend(
   config: AssistantConfig,
-  texts: string[],
+  inputs: EmbeddingInput[],
   options?: EmbeddingRequestOptions,
 ): Promise<{
   provider: EmbeddingProviderName;
@@ -354,15 +397,22 @@ export async function embedWithBackend(
   const { provider: primaryProvider, model: primaryModel } = selection.backend;
 
   // ── Build fallback backends list (needed for embed fallback) ──
+  // In auto mode, build a fallback chain from all configured backends
+  // (excluding the primary). This lets multimodal inputs fall through
+  // to Gemini even when the primary is local or openai.
   const fallbacks: EmbeddingBackend[] =
     config.memory.embeddings.provider === "auto" &&
-    selection.backend.provider === "local"
-      ? selectFallbackBackends(config, "local")
+    selection.backend.provider !== "gemini"
+      ? selectFallbackBackends(config, selection.backend.provider)
       : [];
 
+  // ── Compute provider-specific vector cache extras ───────────────
+  const vectorExtras =
+    primaryProvider === "gemini" ? geminiCacheExtras(config) : undefined;
+
   // ── In-memory cache check (primary provider only) ──────────────
-  const cached: (number[] | null)[] = texts.map((t) => {
-    const v = getFromVectorCache(primaryProvider, primaryModel, t);
+  const cached: (number[] | null)[] = inputs.map((input) => {
+    const v = getFromVectorCache(primaryProvider, primaryModel, input, vectorExtras);
     if (v && v.length === expectedDim) return v;
     return null;
   });
@@ -378,23 +428,33 @@ export async function embedWithBackend(
     };
   }
 
-  // ── Embed uncached texts ────────────────────────────────────────
+  // ── Embed uncached inputs ───────────────────────────────────────
   const backends: EmbeddingBackend[] = [selection.backend, ...fallbacks];
 
   let lastErr: unknown;
+  let anyBackendAttempted = false;
   for (const backend of backends) {
     const isPrimary = backend === selection.backend;
-    // For the primary backend, only embed uncached texts and merge with cached.
-    // For fallback backends, embed ALL texts since the cache was keyed to the primary.
-    const textsToEmbed = isPrimary
-      ? uncachedIndices.map((i) => texts[i])
-      : texts;
+    // For the primary backend, only embed uncached inputs and merge with cached.
+    // For fallback backends, embed ALL inputs since the cache was keyed to the primary.
+    const inputsToEmbed = isPrimary
+      ? uncachedIndices.map((i) => inputs[i])
+      : inputs;
+
+    // Skip text-only backends for multimodal inputs
+    const hasNonText = inputsToEmbed.some(
+      (i) => typeof i !== "string" && normalizeEmbeddingInput(i).type !== "text",
+    );
+    if (backend.provider !== "gemini" && hasNonText) {
+      continue;
+    }
 
     try {
-      const vectors = await backend.embed(textsToEmbed, options);
-      if (vectors.length !== textsToEmbed.length) {
+      anyBackendAttempted = true;
+      const vectors = await backend.embed(inputsToEmbed, options);
+      if (vectors.length !== inputsToEmbed.length) {
         throw new Error(
-          `Embedding backend returned ${vectors.length} vectors for ${textsToEmbed.length} texts`,
+          `Embedding backend returned ${vectors.length} vectors for ${inputsToEmbed.length} inputs`,
         );
       }
       for (const vec of vectors) {
@@ -406,12 +466,15 @@ export async function embedWithBackend(
       }
 
       // Populate cache with freshly embedded vectors
-      for (let i = 0; i < textsToEmbed.length; i++) {
+      const backendExtras =
+        backend.provider === "gemini" ? geminiCacheExtras(config) : undefined;
+      for (let i = 0; i < inputsToEmbed.length; i++) {
         putInVectorCache(
           backend.provider,
           backend.model,
-          textsToEmbed[i],
+          inputsToEmbed[i],
           vectors[i],
+          backendExtras,
         );
       }
 
@@ -437,6 +500,16 @@ export async function embedWithBackend(
       }
     }
   }
+  if (!anyBackendAttempted) {
+    const hasMultimodal = inputs.some(
+      (i) => typeof i !== "string" && normalizeEmbeddingInput(i).type !== "text",
+    );
+    if (hasMultimodal) {
+      throw new Error(
+        "No available embedding backend supports multimodal inputs. Gemini API key is required for image/audio/video embeddings.",
+      );
+    }
+  }
   throw lastErr;
 }
 
@@ -478,7 +551,12 @@ function selectFallbackBackends(
                 new GeminiEmbeddingBackend(
                   config.apiKeys.gemini,
                   config.memory.embeddings.geminiModel,
+                  {
+                    taskType: config.memory.embeddings.geminiTaskType,
+                    dimensions: config.memory.embeddings.geminiDimensions,
+                  },
                 ),
+              geminiCacheExtras(config),
             ),
           );
         }
@@ -505,6 +583,30 @@ function selectFallbackBackends(
   return backends;
 }
 
+/**
+ * Returns true when the embedding pipeline can handle multimodal inputs
+ * (images, audio, video). Today only Gemini supports multimodal.
+ *
+ * In auto mode, the primary backend is usually local or OpenAI, but
+ * embedWithBackend builds a fallback chain that includes Gemini when
+ * available. We check both the primary and fallback backends so that
+ * multimodal jobs are still enqueued when Gemini is reachable via fallback.
+ */
+export function selectedBackendSupportsMultimodal(
+  config: AssistantConfig,
+): boolean {
+  const { backend } = selectEmbeddingBackend(config);
+  if (!backend) return false;
+  if (backend.provider === "gemini") return true;
+
+  // In auto mode, check if Gemini is available as a fallback backend.
+  if (config.memory.embeddings.provider === "auto") {
+    const fallbacks = selectFallbackBackends(config, backend.provider);
+    return fallbacks.some((fb) => fb.provider === "gemini");
+  }
+  return false;
+}
+
 function isOllamaConfigured(config: AssistantConfig): boolean {
   return (
     config.provider === "ollama" ||