@vellumai/assistant 0.3.3 → 0.3.5

package/src/memory/schema.ts

@@ -129,6 +129,7 @@ export const memoryEmbeddings = sqliteTable('memory_embeddings', {
   model: text('model').notNull(),
   dimensions: integer('dimensions').notNull(),
   vectorJson: text('vector_json').notNull(),
+  contentHash: text('content_hash'),
   createdAt: integer('created_at').notNull(),
   updatedAt: integer('updated_at').notNull(),
 });
@@ -679,3 +680,101 @@ export const channelGuardianRateLimits = sqliteTable('channel_guardian_rate_limi
   createdAt: integer('created_at').notNull(),
   updatedAt: integer('updated_at').notNull(),
 });
+
+// ── Media Assets ─────────────────────────────────────────────────────
+
+export const mediaAssets = sqliteTable('media_assets', {
+  id: text('id').primaryKey(),
+  title: text('title').notNull(),
+  filePath: text('file_path').notNull(),
+  mimeType: text('mime_type').notNull(),
+  durationSeconds: real('duration_seconds'),
+  fileHash: text('file_hash').notNull(),
+  status: text('status').notNull().default('registered'),       // registered | processing | indexed | failed
+  mediaType: text('media_type').notNull(),                      // video | audio | image
+  metadata: text('metadata'),                                   // JSON
+  createdAt: integer('created_at').notNull(),
+  updatedAt: integer('updated_at').notNull(),
+});
+
+export const processingStages = sqliteTable('processing_stages', {
+  id: text('id').primaryKey(),
+  assetId: text('asset_id').notNull()
+    .references(() => mediaAssets.id, { onDelete: 'cascade' }),
+  stage: text('stage').notNull(),
+  status: text('status').notNull().default('pending'),          // pending | running | completed | failed
+  progress: integer('progress').notNull().default(0),           // 0-100
+  lastError: text('last_error'),
+  startedAt: integer('started_at'),
+  completedAt: integer('completed_at'),
+});
+
+export const mediaKeyframes = sqliteTable('media_keyframes', {
+  id: text('id').primaryKey(),
+  assetId: text('asset_id').notNull()
+    .references(() => mediaAssets.id, { onDelete: 'cascade' }),
+  timestamp: real('timestamp').notNull(),
+  filePath: text('file_path').notNull(),
+  metadata: text('metadata'),                                   // JSON
+  createdAt: integer('created_at').notNull(),
+});
+
+export const mediaVisionOutputs = sqliteTable('media_vision_outputs', {
+  id: text('id').primaryKey(),
+  assetId: text('asset_id').notNull()
+    .references(() => mediaAssets.id, { onDelete: 'cascade' }),
+  keyframeId: text('keyframe_id').notNull()
+    .references(() => mediaKeyframes.id, { onDelete: 'cascade' }),
+  analysisType: text('analysis_type').notNull(),
+  output: text('output').notNull(),                             // JSON
+  confidence: real('confidence'),
+  createdAt: integer('created_at').notNull(),
+});
+
+export const mediaTimelines = sqliteTable('media_timelines', {
+  id: text('id').primaryKey(),
+  assetId: text('asset_id').notNull()
+    .references(() => mediaAssets.id, { onDelete: 'cascade' }),
+  startTime: real('start_time').notNull(),
+  endTime: real('end_time').notNull(),
+  segmentType: text('segment_type').notNull(),
+  attributes: text('attributes'),                               // JSON
+  confidence: real('confidence'),
+  createdAt: integer('created_at').notNull(),
+});
+
+export const mediaEvents = sqliteTable('media_events', {
+  id: text('id').primaryKey(),
+  assetId: text('asset_id').notNull()
+    .references(() => mediaAssets.id, { onDelete: 'cascade' }),
+  eventType: text('event_type').notNull(),
+  startTime: real('start_time').notNull(),
+  endTime: real('end_time').notNull(),
+  confidence: real('confidence').notNull(),
+  reasons: text('reasons').notNull(),                            // JSON array
+  metadata: text('metadata'),                                    // JSON
+  createdAt: integer('created_at').notNull(),
+});
+
+export const mediaTrackingProfiles = sqliteTable('media_tracking_profiles', {
+  id: text('id').primaryKey(),
+  assetId: text('asset_id').notNull()
+    .references(() => mediaAssets.id, { onDelete: 'cascade' }),
+  capabilities: text('capabilities').notNull(),                  // JSON: { [capName]: { enabled, tier } }
+  createdAt: integer('created_at').notNull(),
+});
+
+export const mediaEventFeedback = sqliteTable('media_event_feedback', {
+  id: text('id').primaryKey(),
+  assetId: text('asset_id').notNull()
+    .references(() => mediaAssets.id, { onDelete: 'cascade' }),
+  eventId: text('event_id').notNull()
+    .references(() => mediaEvents.id, { onDelete: 'cascade' }),
+  feedbackType: text('feedback_type').notNull(),                 // correct | incorrect | boundary_edit | missed
+  originalStartTime: real('original_start_time'),
+  originalEndTime: real('original_end_time'),
+  correctedStartTime: real('corrected_start_time'),
+  correctedEndTime: real('corrected_end_time'),
+  notes: text('notes'),
+  createdAt: integer('created_at').notNull(),
+});

package/src/memory/search/entity.ts

@@ -8,7 +8,7 @@ import {
   memoryItems,
   memoryItemSources,
 } from '../schema.js';
-import type { Candidate, CandidateSource, CandidateType, EntitySearchResult, MatchedEntityRow } from './types.js';
+import type { Candidate, CandidateSource, CandidateType, EntitySearchResult, MatchedEntityRow, TraversalOptions, TraversalResult, TraversalStep } from './types.js';
 import { computeRecencyScore } from './ranking.js';
 
 const log = getLogger('memory-retriever');
@@ -51,15 +51,16 @@ export function entitySearch(
   }
 
   const relationSeedEntityCount = seedEntityIds.length;
+
   const {
     neighborEntityIds,
     traversedEdgeCount: relationTraversedEdgeCount,
-  } = findNeighborEntities(
-    seedEntityIds,
-    relationConfig.maxEdges,
-    relationConfig.maxNeighborEntities,
-    relationConfig.maxDepth,
-  );
+    neighborDepths,
+  } = findNeighborEntities(seedEntityIds, {
+    maxEdges: relationConfig.maxEdges,
+    maxNeighborEntities: relationConfig.maxNeighborEntities,
+    maxDepth: relationConfig.maxDepth,
+  });
   const relationNeighborEntityCount = neighborEntityIds.length;
   const directItemIds = new Set(directCandidates.map((candidate) => candidate.id));
   const relationCandidates = getEntityLinkedItemCandidates(neighborEntityIds, {
@@ -70,12 +71,47 @@ export function entitySearch(
   });
   const relationExpandedItemCount = relationCandidates.length;
 
+  // Build candidate key → BFS depth map so ranking can apply distance-based decay
+  const candidateDepths = new Map<string, number>();
+  if (relationCandidates.length > 0 && neighborDepths.size > 0) {
+    const db = getDb();
+    const itemIds = relationCandidates.map((c) => c.id);
+    const links = db
+      .select({
+        memoryItemId: memoryItemEntities.memoryItemId,
+        entityId: memoryItemEntities.entityId,
+      })
+      .from(memoryItemEntities)
+      .where(inArray(memoryItemEntities.memoryItemId, itemIds))
+      .all();
+
+    // For each item, find the minimum depth among its linked neighbor entities
+    const itemDepthMap = new Map<string, number>();
+    for (const link of links) {
+      const depth = neighborDepths.get(link.entityId);
+      if (depth !== undefined) {
+        const existing = itemDepthMap.get(link.memoryItemId);
+        if (existing === undefined || depth < existing) {
+          itemDepthMap.set(link.memoryItemId, depth);
+        }
+      }
+    }
+
+    for (const candidate of relationCandidates) {
+      const depth = itemDepthMap.get(candidate.id);
+      if (depth !== undefined) {
+        candidateDepths.set(candidate.key, depth);
+      }
+    }
+  }
+
   return {
     candidates: [...directCandidates, ...relationCandidates],
     relationSeedEntityCount,
     relationTraversedEdgeCount,
     relationNeighborEntityCount,
     relationExpandedItemCount,
+    candidateDepths,
   };
 }
 
@@ -86,6 +122,7 @@ export function emptyEntitySearchResult(): EntitySearchResult {
     relationTraversedEdgeCount: 0,
     relationNeighborEntityCount: 0,
     relationExpandedItemCount: 0,
+    candidateDepths: new Map(),
   };
 }
 
@@ -152,18 +189,19 @@ export function findMatchedEntities(query: string, maxMatches: number): MatchedE
  */
 export function findNeighborEntities(
   seedEntityIds: string[],
-  maxEdges: number,
-  maxNeighborEntities: number,
-  maxDepth: number = 3,
-): { neighborEntityIds: string[]; traversedEdgeCount: number } {
+  opts: TraversalOptions,
+): TraversalResult {
+  const { maxEdges, maxNeighborEntities, maxDepth = 3, relationTypes, entityTypes, directed } = opts;
   if (seedEntityIds.length === 0 || maxEdges <= 0 || maxNeighborEntities <= 0 || maxDepth <= 0) {
-    return { neighborEntityIds: [], traversedEdgeCount: 0 };
+    return { neighborEntityIds: [], traversedEdgeCount: 0, neighborDepths: new Map() };
   }
 
   const db = getDb();
   const visited = new Set<string>(seedEntityIds);
   const neighbors: string[] = [];
+  const neighborDepths = new Map<string, number>();
   let totalEdgesTraversed = 0;
+  const filterByEntityType = entityTypes && entityTypes.length > 0;
 
   // BFS frontier starts with seed entities
   let frontier = [...seedEntityIds];
@@ -174,19 +212,91 @@ export function findNeighborEntities(
     const edgeBudget = maxEdges - totalEdgesTraversed;
     if (edgeBudget <= 0) break;
 
-    const rows = db
-      .select({
-        sourceEntityId: memoryEntityRelations.sourceEntityId,
-        targetEntityId: memoryEntityRelations.targetEntityId,
-      })
-      .from(memoryEntityRelations)
-      .where(or(
-        inArray(memoryEntityRelations.sourceEntityId, frontier),
-        inArray(memoryEntityRelations.targetEntityId, frontier),
-      ))
-      .orderBy(desc(memoryEntityRelations.lastSeenAt))
-      .limit(Math.max(1, edgeBudget))
-      .all();
+    let rows: Array<{ sourceEntityId: string; targetEntityId: string }>;
+
+    if (filterByEntityType) {
+      // When filtering by entity type, JOIN with memoryEntities on the neighbor
+      // side so non-matching edges are excluded at the SQL level and don't
+      // consume the edge budget.
+      const relationTypeCondition = relationTypes && relationTypes.length > 0
+        ? `AND r.relation IN (${relationTypes.map(() => '?').join(',')})`
+        : '';
+      const entityTypeFilter = `AND me.type IN (${entityTypes.map(() => '?').join(',')})`;
+      const frontierPlaceholders = frontier.map(() => '?').join(',');
+      const limit = Math.max(1, edgeBudget);
+
+      const raw = (db as unknown as { $client: { query: (q: string) => { all: (...params: unknown[]) => unknown[] } } }).$client;
+      const relationParams = relationTypes && relationTypes.length > 0 ? relationTypes : [];
+
+      if (directed) {
+        // GROUP BY deduplicates entity pairs that have multiple relation rows
+        const q1 = `
+          SELECT r.source_entity_id AS sourceEntityId, r.target_entity_id AS targetEntityId
+          FROM memory_entity_relations r
+          INNER JOIN memory_entities me ON me.id = r.target_entity_id
+          WHERE r.source_entity_id IN (${frontierPlaceholders})
+          ${relationTypeCondition} ${entityTypeFilter}
+          GROUP BY r.source_entity_id, r.target_entity_id
+          ORDER BY MAX(r.last_seen_at) DESC
+          LIMIT ?
+        `;
+        const params1 = [...frontier, ...relationParams, ...entityTypes, limit];
+        rows = raw.query(q1).all(...params1) as Array<{ sourceEntityId: string; targetEntityId: string }>;
+      } else {
+        // Combine both directions in a single query with global recency
+        // ordering so the edge budget isn't direction-biased.
+        const q = `
+          SELECT sourceEntityId, targetEntityId FROM (
+            SELECT r.source_entity_id AS sourceEntityId, r.target_entity_id AS targetEntityId, r.last_seen_at
+            FROM memory_entity_relations r
+            INNER JOIN memory_entities me ON me.id = r.target_entity_id
+            WHERE r.source_entity_id IN (${frontierPlaceholders})
+            ${relationTypeCondition} ${entityTypeFilter}
+            UNION ALL
+            SELECT r.source_entity_id AS sourceEntityId, r.target_entity_id AS targetEntityId, r.last_seen_at
+            FROM memory_entity_relations r
+            INNER JOIN memory_entities me ON me.id = r.source_entity_id
+            WHERE r.target_entity_id IN (${frontierPlaceholders})
+            ${relationTypeCondition} ${entityTypeFilter}
+          )
+          GROUP BY sourceEntityId, targetEntityId
+          ORDER BY MAX(last_seen_at) DESC
+          LIMIT ?
+        `;
+        const params = [
+          ...frontier,
+          ...relationParams,
+          ...entityTypes,
+          ...frontier,
+          ...relationParams,
+          ...entityTypes,
+          limit,
+        ];
+
+        rows = raw.query(q).all(...params) as Array<{ sourceEntityId: string; targetEntityId: string }>;
+      }
+    } else {
+      const frontierCondition = directed
+        ? inArray(memoryEntityRelations.sourceEntityId, frontier)
+        : or(
+            inArray(memoryEntityRelations.sourceEntityId, frontier),
+            inArray(memoryEntityRelations.targetEntityId, frontier),
+          );
+      const whereCondition = relationTypes && relationTypes.length > 0
+        ? and(frontierCondition, inArray(memoryEntityRelations.relation, relationTypes))
+        : frontierCondition;
+
+      rows = db
+        .select({
+          sourceEntityId: memoryEntityRelations.sourceEntityId,
+          targetEntityId: memoryEntityRelations.targetEntityId,
+        })
+        .from(memoryEntityRelations)
+        .where(whereCondition)
+        .orderBy(desc(memoryEntityRelations.lastSeenAt))
+        .limit(Math.max(1, edgeBudget))
+        .all();
+    }
 
     totalEdgesTraversed += rows.length;
 
@@ -194,16 +304,20 @@ export function findNeighborEntities(
     const frontierSet = new Set(frontier);
     for (const row of rows) {
       if (neighbors.length >= maxNeighborEntities) break;
+      // In directed mode, only follow source→target (frontier is always on source side)
       if (frontierSet.has(row.sourceEntityId) && !visited.has(row.targetEntityId)) {
         visited.add(row.targetEntityId);
         neighbors.push(row.targetEntityId);
         nextFrontier.push(row.targetEntityId);
+        neighborDepths.set(row.targetEntityId, depth + 1);
       }
+      if (directed) continue;
       if (neighbors.length >= maxNeighborEntities) break;
       if (frontierSet.has(row.targetEntityId) && !visited.has(row.sourceEntityId)) {
         visited.add(row.sourceEntityId);
         neighbors.push(row.sourceEntityId);
         nextFrontier.push(row.sourceEntityId);
+        neighborDepths.set(row.sourceEntityId, depth + 1);
       }
     }
 
@@ -213,6 +327,7 @@ export function findNeighborEntities(
   return {
     neighborEntityIds: neighbors.slice(0, maxNeighborEntities),
     traversedEdgeCount: totalEdgesTraversed,
+    neighborDepths,
   };
 }
 
@@ -296,3 +411,71 @@ export function getEntityLinkedItemCandidates(
     finalScore: 0,
   }));
 }
+
+/**
+ * Multi-step typed traversal: each step expands the frontier through
+ * edges matching the step's relation/entity type filters.
+ * Returns entity IDs reachable after all steps are applied in sequence.
+ */
+export function collectTypedNeighbors(
+  seedEntityIds: string[],
+  steps: TraversalStep[],
+  opts?: { maxResultsPerStep?: number; maxEdgesPerStep?: number },
+): string[] {
+  if (seedEntityIds.length === 0 || steps.length === 0) return [];
+
+  const maxResults = opts?.maxResultsPerStep ?? 20;
+  const maxEdges = opts?.maxEdgesPerStep ?? 40;
+
+  let currentSeeds = seedEntityIds;
+
+  for (const step of steps) {
+    if (currentSeeds.length === 0) break;
+
+    const result = findNeighborEntities(currentSeeds, {
+      maxEdges,
+      maxNeighborEntities: maxResults,
+      maxDepth: 1,
+      relationTypes: step.relationTypes,
+      entityTypes: step.entityTypes,
+      directed: true,
+    });
+
+    currentSeeds = result.neighborEntityIds;
+  }
+
+  return currentSeeds;
+}
+
+/**
+ * Find entities reachable from ALL given seeds via their respective
+ * typed traversal steps, then return the intersection.
+ */
+export function intersectReachable(
+  queries: Array<{
+    seedEntityIds: string[];
+    steps: TraversalStep[];
+  }>,
+  opts?: { maxResultsPerStep?: number; maxEdgesPerStep?: number },
+): string[] {
+  if (queries.length === 0) return [];
+
+  const resultSets: Set<string>[] = [];
+  for (const query of queries) {
+    const result = collectTypedNeighbors(
+      query.seedEntityIds,
+      query.steps,
+      opts,
+    );
+    resultSets.push(new Set(result));
+  }
+
+  if (resultSets.length === 0) return [];
+
+  // Intersect all sets: keep only entities present in ALL sets
+  const intersection = [...resultSets[0]].filter(id =>
+    resultSets.every(set => set.has(id)),
+  );
+
+  return intersection;
+}

package/src/memory/search/ranking.ts

@@ -53,6 +53,7 @@ export function mergeCandidates(
   entity: Candidate[] = [],
   freshnessConfig?: { enabled: boolean; maxAgeDays: Record<string, number>; staleDecay: number; reinforcementShieldDays: number },
   relationScoreMultiplier?: number,
+  candidateDepthMap?: Map<string, number>,
 ): Candidate[] {
   // Build effective weight map that reflects the actual scoring weight for
   // each source.  For entity_relation the static SOURCE_WEIGHTS entry is 1.0
@@ -126,7 +127,11 @@ export function mergeCandidates(
     const lastUsedAt = meta?.lastUsedAt ?? null;
     const freshnessWeight = computeFreshnessWeight(row, accessCount, lastUsedAt, freshnessConfig);
 
-    const sourceWeight = effectiveWeights[row.source] ?? 1.0;
+    let sourceWeight = effectiveWeights[row.source] ?? 1.0;
+    if (row.source === 'entity_relation' && candidateDepthMap && relationScoreMultiplier != null) {
+      const depth = candidateDepthMap.get(row.key) ?? 1;
+      sourceWeight = Math.pow(relationScoreMultiplier, depth);
+    }
     row.finalScore = rrfScore * (0.5 + 0.5 * effectiveImportance) * trustWeight * freshnessWeight * sourceWeight;
   }
 

package/src/memory/search/types.ts

@@ -94,6 +94,8 @@ export interface CollectedCandidates {
   relationNeighborEntityCount: number;
   relationExpandedItemCount: number;
   earlyTerminated: boolean;
+  /** True when semantic search was attempted but threw an error. */
+  semanticSearchFailed: boolean;
   merged: Candidate[];
 }
 
@@ -103,6 +105,7 @@ export interface EntitySearchResult {
   relationTraversedEdgeCount: number;
   relationNeighborEntityCount: number;
   relationExpandedItemCount: number;
+  candidateDepths?: Map<string, number>; // candidate key → BFS hop depth (1-based)
 }
 
 export interface MatchedEntityRow {
@@ -135,3 +138,26 @@ export interface MemorySearchResult {
     recency: number;
   };
 }
+
+import type { EntityRelationType, EntityType } from '../entity-extractor.js';
+
+export interface TraversalOptions {
+  maxEdges: number;
+  maxNeighborEntities: number;
+  maxDepth?: number; // default 3
+  relationTypes?: EntityRelationType[];
+  entityTypes?: EntityType[];
+  /** When true, only follow source→target edges (frontier must be on source side). */
+  directed?: boolean;
+}
+
+export interface TraversalResult {
+  neighborEntityIds: string[];
+  traversedEdgeCount: number;
+  neighborDepths: Map<string, number>; // entityId → depth (1-based)
+}
+
+export interface TraversalStep {
+  relationTypes?: EntityRelationType[];
+  entityTypes?: EntityType[];
+}

package/src/messaging/provider-types.ts

@@ -77,4 +77,6 @@ export interface SendOptions {
   subject?: string;
   /** For email: in-reply-to message ID */
   inReplyTo?: string;
+  /** Optional assistant scope for multi-assistant channels. */
+  assistantId?: string;
 }