@vellumai/assistant 0.3.3 → 0.3.5

package/src/memory/embedding-backend.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import type { AssistantConfig } from '../config/types.js';
 import { getLogger } from '../util/logger.js';
 import { GeminiEmbeddingBackend } from './embedding-gemini.js';
@@ -10,9 +11,41 @@ const log = getLogger('memory-embeddings');
 /** Global cache of embedding backend instances, keyed by "provider:model". */
 const backendCache = new Map<string, EmbeddingBackend>();
 
-/** Clear cached embedding backends so new instances pick up fresh credentials. */
+// ── In-memory embedding vector cache ──────────────────────────────
+// LRU cache keyed by sha256(provider + model + text) → embedding vector.
+// Avoids redundant API calls / local compute for identical content.
+const VECTOR_CACHE_MAX_ENTRIES = 4096;
+const vectorCache = new Map<string, number[]>();
+
+function vectorCacheKey(provider: string, model: string, text: string): string {
+  return createHash('sha256').update(`${provider}\0${model}\0${text}`).digest('hex');
+}
+
+function getFromVectorCache(provider: string, model: string, text: string): number[] | undefined {
+  const key = vectorCacheKey(provider, model, text);
+  const v = vectorCache.get(key);
+  if (v !== undefined) {
+    // LRU refresh: move to end of insertion order
+    vectorCache.delete(key);
+    vectorCache.set(key, v);
+  }
+  return v;
+}
+
+function putInVectorCache(provider: string, model: string, text: string, vector: number[]): void {
+  const key = vectorCacheKey(provider, model, text);
+  vectorCache.delete(key);
+  if (vectorCache.size >= VECTOR_CACHE_MAX_ENTRIES) {
+    const oldest = vectorCache.keys().next().value;
+    if (oldest !== undefined) vectorCache.delete(oldest);
+  }
+  vectorCache.set(key, vector);
+}
+
+/** Clear cached embedding backends and the in-memory vector cache. */
 export function clearEmbeddingBackendCache(): void {
   backendCache.clear();
+  vectorCache.clear();
 }
 
 function cacheKey(provider: string, model: string): string {
@@ -153,22 +186,44 @@ export async function embedWithBackend(
     throw new Error(selection.reason ?? 'No memory embedding backend configured');
   }
 
-  // In auto mode, build a fallback list of backends to try
-  const backends: EmbeddingBackend[] = [selection.backend];
-  if (config.memory.embeddings.provider === 'auto' && selection.backend.provider === 'local') {
-    for (const fallback of selectFallbackBackends(config, 'local')) {
-      backends.push(fallback);
-    }
+  const expectedDim = config.memory.qdrant.vectorSize;
+  const { provider: primaryProvider, model: primaryModel } = selection.backend;
+
+  // ── Build fallback backends list (needed for embed fallback) ──
+  const fallbacks: EmbeddingBackend[] =
+    config.memory.embeddings.provider === 'auto' && selection.backend.provider === 'local'
+      ? selectFallbackBackends(config, 'local')
+      : [];
+
+  // ── In-memory cache check (primary provider only) ──────────────
+  const cached: (number[] | null)[] = texts.map(t => {
+    const v = getFromVectorCache(primaryProvider, primaryModel, t);
+    if (v && v.length === expectedDim) return v;
+    return null;
+  });
+  const uncachedIndices: number[] = [];
+  for (let i = 0; i < cached.length; i++) {
+    if (!cached[i]) uncachedIndices.push(i);
+  }
+  if (uncachedIndices.length === 0) {
+    return { provider: primaryProvider, model: primaryModel, vectors: cached as number[][] };
   }
 
+  // ── Embed uncached texts ────────────────────────────────────────
+  const backends: EmbeddingBackend[] = [selection.backend, ...fallbacks];
+
   let lastErr: unknown;
   for (const backend of backends) {
+    const isPrimary = backend === selection.backend;
+    // For the primary backend, only embed uncached texts and merge with cached.
+    // For fallback backends, embed ALL texts since the cache was keyed to the primary.
+    const textsToEmbed = isPrimary ? uncachedIndices.map(i => texts[i]) : texts;
+
     try {
-      const vectors = await backend.embed(texts, options);
-      if (vectors.length !== texts.length) {
-        throw new Error(`Embedding backend returned ${vectors.length} vectors for ${texts.length} texts`);
+      const vectors = await backend.embed(textsToEmbed, options);
+      if (vectors.length !== textsToEmbed.length) {
+        throw new Error(`Embedding backend returned ${vectors.length} vectors for ${textsToEmbed.length} texts`);
       }
-      const expectedDim = config.memory.qdrant.vectorSize;
       for (const vec of vectors) {
         if (vec.length !== expectedDim) {
           throw new Error(
@@ -176,6 +231,19 @@ export async function embedWithBackend(
           );
         }
       }
+
+      // Populate cache with freshly embedded vectors
+      for (let i = 0; i < textsToEmbed.length; i++) {
+        putInVectorCache(backend.provider, backend.model, textsToEmbed[i], vectors[i]);
+      }
+
+      if (isPrimary) {
+        const merged = [...cached] as number[][];
+        for (let i = 0; i < uncachedIndices.length; i++) {
+          merged[uncachedIndices[i]] = vectors[i];
+        }
+        return { provider: backend.provider, model: backend.model, vectors: merged };
+      }
       return { provider: backend.provider, model: backend.model, vectors };
     } catch (err) {
       lastErr = err;

package/src/memory/indexer.ts

@@ -7,6 +7,7 @@ import { getDb } from './db.js';
 import { enqueueMemoryJob, enqueueResolvePendingConflictsForMessageJob } from './jobs-store.js';
 import { extractTextFromStoredMessageContent } from './message-content.js';
 import { segmentText } from './segmenter.js';
+import { bumpMemoryVersion } from './recall-cache.js';
 import { memorySegments } from './schema.js';
 
 const log = getLogger('memory-indexer');
@@ -108,6 +109,7 @@ export function indexMessageNow(
     log.debug(`Skipped ${skippedEmbedJobs}/${segments.length} embed_segment jobs (content unchanged)`);
   }
 
+  bumpMemoryVersion();
   enqueueSummaryRollupJobsIfDue();
 
   const enqueuedJobs = (segments.length - skippedEmbedJobs) + (shouldExtract ? 2 : 1) + (shouldResolveConflicts ? 1 : 0);

package/src/memory/job-handlers/media-processing.ts

@@ -0,0 +1,100 @@
+import { getLogger } from '../../util/logger.js';
+import { asString } from '../job-utils.js';
+import { getMediaAssetById, updateMediaAssetStatus } from '../media-store.js';
+import type { MemoryJob } from '../jobs-store.js';
+import {
+  runPipeline,
+  type PipelineStageName,
+  type StageHandler,
+} from '../../config/bundled-skills/media-processing/services/processing-pipeline.js';
+import { extractKeyframesForAsset } from '../../config/bundled-skills/media-processing/tools/extract-keyframes.js';
+import { analyzeKeyframesForAsset } from '../../config/bundled-skills/media-processing/tools/analyze-keyframes.js';
+import { generateTimeline } from '../../config/bundled-skills/media-processing/services/timeline-service.js';
+import {
+  detectEvents,
+  type DetectionConfig,
+} from '../../config/bundled-skills/media-processing/services/event-detection-service.js';
+
+const log = getLogger('media-processing-job');
+
+const defaultDetectionConfig: DetectionConfig = {
+  eventType: 'scene_change',
+  rules: [
+    {
+      ruleType: 'segment_transition',
+      params: { field: 'segmentType' },
+      weight: 1.0,
+    },
+  ],
+};
+
+export async function mediaProcessingJob(job: MemoryJob): Promise<void> {
+  const mediaAssetId = asString(job.payload.mediaAssetId);
+  if (!mediaAssetId) {
+    log.warn({ jobId: job.id }, 'Missing mediaAssetId in job payload');
+    return;
+  }
+
+  const asset = getMediaAssetById(mediaAssetId);
+  if (!asset) {
+    log.warn({ jobId: job.id, mediaAssetId }, 'Media asset not found');
+    return;
+  }
+
+  if (asset.mediaType !== 'video') {
+    log.info(
+      { assetId: mediaAssetId, mediaType: asset.mediaType },
+      'Skipping media processing pipeline — only video assets are supported',
+    );
+    updateMediaAssetStatus(mediaAssetId, 'indexed');
+    return;
+  }
+
+  // Build detection config, allowing optional eventType override from payload
+  const eventType = asString(job.payload.eventType);
+  const detectionConfig: DetectionConfig = eventType
+    ? { ...defaultDetectionConfig, eventType }
+    : defaultDetectionConfig;
+
+  const handlers: Record<PipelineStageName, StageHandler> = {
+    keyframe_extraction: {
+      execute: (assetId, onProgress) =>
+        extractKeyframesForAsset(assetId, 1, onProgress),
+    },
+    vision_analysis: {
+      execute: (assetId, onProgress) =>
+        analyzeKeyframesForAsset(assetId, undefined, undefined, onProgress),
+    },
+    timeline_generation: {
+      execute: async (assetId, onProgress) => {
+        generateTimeline(assetId, { onProgress });
+      },
+    },
+    event_detection: {
+      execute: async (assetId, onProgress) => {
+        detectEvents(assetId, detectionConfig, { onProgress });
+      },
+    },
+  };
+
+  const result = await runPipeline(mediaAssetId, handlers, {
+    onProgress: (msg) => log.info({ mediaAssetId }, msg),
+  });
+
+  log.info(
+    {
+      mediaAssetId,
+      completedStages: result.completedStages,
+      failedStage: result.failedStage,
+      cancelled: result.cancelled,
+    },
+    'Media processing pipeline finished',
+  );
+
+  if (result.failedStage) {
+    throw new Error(`Media processing failed at stage ${result.failedStage}: ${result.failureReason}`);
+  }
+  if (result.cancelled) {
+    throw new Error(`Media processing cancelled for asset ${mediaAssetId}`);
+  }
+}

package/src/memory/job-utils.ts

@@ -1,6 +1,10 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { eq, and } from 'drizzle-orm';
 import { getLogger } from '../util/logger.js';
 import { embedWithBackend, getMemoryBackendStatus } from './embedding-backend.js';
+import { getDb } from './db.js';
 import { getQdrantClient } from './qdrant-client.js';
+import { memoryEmbeddings } from './schema.js';
 import type { AssistantConfig } from '../config/types.js';
 
 const log = getLogger('memory-jobs-worker');
@@ -111,9 +115,66 @@ export async function embedAndUpsert(
     );
   }
 
-  const embedded = await embedWithBackend(config, [text]);
-  const vector = embedded.vectors[0];
-  if (!vector) return;
+  const contentHash = createHash('sha256').update(text).digest('hex');
+  let provider = status.provider;
+  let model = status.model!;
+  let vector: number[];
+
+  // Check SQLite embedding cache for a matching content hash (primary provider only).
+  const db = getDb();
+  const expectedDim = config.memory.qdrant.vectorSize;
+  let cachedRow = db
+    .select({ vectorJson: memoryEmbeddings.vectorJson, dimensions: memoryEmbeddings.dimensions })
+    .from(memoryEmbeddings)
+    .where(
+      and(
+        eq(memoryEmbeddings.contentHash, contentHash),
+        eq(memoryEmbeddings.provider, provider),
+        eq(memoryEmbeddings.model, model),
+      ),
+    )
+    .get();
+  if (cachedRow && cachedRow.dimensions !== expectedDim) cachedRow = undefined;
+
+  if (cachedRow) {
+    vector = JSON.parse(cachedRow.vectorJson);
+  } else {
+    const embedded = await embedWithBackend(config, [text]);
+    vector = embedded.vectors[0];
+    if (!vector) return;
+    provider = embedded.provider;
+    model = embedded.model;
+  }
+
+  // Persist embedding in SQLite for cross-restart cache
+  const now = Date.now();
+  try {
+    db.insert(memoryEmbeddings)
+      .values({
+        id: randomUUID(),
+        targetType,
+        targetId,
+        provider,
+        model,
+        dimensions: vector.length,
+        vectorJson: JSON.stringify(vector),
+        contentHash,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .onConflictDoUpdate({
+        target: [memoryEmbeddings.targetType, memoryEmbeddings.targetId, memoryEmbeddings.provider, memoryEmbeddings.model],
+        set: {
+          vectorJson: JSON.stringify(vector),
+          dimensions: vector.length,
+          contentHash,
+          updatedAt: now,
+        },
+      })
+      .run();
+  } catch (err) {
+    log.warn({ err, targetType, targetId }, 'Failed to write embedding cache');
+  }
 
   let qdrant;
   try {
@@ -123,7 +184,6 @@ export async function embedAndUpsert(
   }
 
   try {
-    const now = Date.now();
     await qdrant.upsert(targetType, targetId, vector, {
       text,
       created_at: (extraPayload?.created_at as number) ?? now,

package/src/memory/jobs-store.ts

@@ -20,7 +20,8 @@ export type MemoryJobType =
   | 'build_conversation_summary'
   | 'backfill'
   | 'rebuild_index'
-  | 'delete_qdrant_vectors';
+  | 'delete_qdrant_vectors'
+  | 'media_processing';
 
 const EMBED_JOB_TYPES: MemoryJobType[] = ['embed_segment', 'embed_item', 'embed_summary'];
 

package/src/memory/jobs-worker.ts

@@ -18,6 +18,7 @@ import {
   retryDelayForAttempt,
   RETRY_MAX_ATTEMPTS,
 } from './job-utils.js';
+import { bumpMemoryVersion } from './recall-cache.js';
 
 // ── Per-job-type handlers ──────────────────────────────────────────
 
@@ -28,6 +29,7 @@ import { checkContradictionsJob, cleanupStaleSupersededItemsJob } from './job-ha
 import { buildConversationSummaryJob, buildGlobalSummaryJob } from './job-handlers/summarization.js';
 import { backfillJob, backfillEntityRelationsJob } from './job-handlers/backfill.js';
 import { rebuildIndexJob, deleteQdrantVectorsJob } from './job-handlers/index-maintenance.js';
+import { mediaProcessingJob } from './job-handlers/media-processing.js';
 
 // Re-export public utilities consumed by tests and other modules
 export { currentWeekWindow } from './job-utils.js';
@@ -121,9 +123,14 @@ export async function runMemoryJobsOnce(
           try {
             await processJob(job, config);
             completeMemoryJob(job.id);
+            bumpMemoryVersion();
             groupProcessed += 1;
           } catch (err) {
-            handleJobError(job, err);
+            try {
+              handleJobError(job, err);
+            } catch (handlerErr) {
+              log.error({ err: handlerErr, jobId: job.id, type: job.type }, 'handleJobError itself threw, job left in running status');
+            }
           }
         }
         return groupProcessed;
@@ -223,6 +230,9 @@ async function processJob(job: MemoryJob, config: AssistantConfig): Promise<void
     case 'delete_qdrant_vectors':
       await deleteQdrantVectorsJob(job);
       return;
+    case 'media_processing':
+      await mediaProcessingJob(job);
+      return;
     default:
       throw new Error(`Unknown memory job type: ${(job as { type: string }).type}`);
   }