@vellumai/assistant 0.3.4 → 0.3.5

package/src/daemon/session-agent-loop.ts

@@ -34,7 +34,7 @@ import {
   stripInjectedContext,
 } from './session-runtime-assembly.js';
 import { buildTemporalContext } from './date-context.js';
-import type { ActiveSurfaceContext, ChannelCapabilities } from './session-runtime-assembly.js';
+import type { ActiveSurfaceContext, ChannelCapabilities, GuardianRuntimeContext } from './session-runtime-assembly.js';
 import {
   cleanAssistantContent,
   drainDirectiveDisplayBuffer,
@@ -95,6 +95,7 @@ export interface AgentLoopSessionContext {
   workspaceTopLevelContext: string | null;
   workspaceTopLevelDirty: boolean;
   channelCapabilities?: ChannelCapabilities;
+  guardianContext?: GuardianRuntimeContext;
 
   readonly coreToolNames: Set<string>;
   allowedToolNames?: Set<string>;
@@ -296,6 +297,7 @@ export async function runAgentLoopImpl(
       activeSurface,
       workspaceTopLevelContext: ctx.workspaceTopLevelContext,
       channelCapabilities: ctx.channelCapabilities ?? null,
+      guardianContext: ctx.guardianContext ?? null,
       temporalContext,
     });
 
@@ -617,6 +619,7 @@ export async function runAgentLoopImpl(
           activeSurface,
           workspaceTopLevelContext: ctx.workspaceTopLevelContext,
           channelCapabilities: ctx.channelCapabilities ?? null,
+          guardianContext: ctx.guardianContext ?? null,
           temporalContext,
         });
         preRepairMessages = runMessages;
@@ -649,6 +652,7 @@ export async function runAgentLoopImpl(
             activeSurface,
             workspaceTopLevelContext: ctx.workspaceTopLevelContext,
             channelCapabilities: ctx.channelCapabilities ?? null,
+            guardianContext: ctx.guardianContext ?? null,
             temporalContext,
           });
           preRepairMessages = runMessages;

package/src/daemon/session-runtime-assembly.ts

@@ -25,6 +25,18 @@ export interface ChannelCapabilities {
   supportsVoiceInput: boolean;
 }
 
+/** Guardian identity/trust context for external chat channels. */
+export interface GuardianRuntimeContext {
+  sourceChannel: string;
+  actorRole: 'guardian' | 'non-guardian' | 'unverified_channel';
+  guardianChatId?: string;
+  guardianExternalUserId?: string;
+  requesterIdentifier?: string;
+  requesterExternalUserId?: string;
+  requesterChatId?: string;
+  denialReason?: 'no_binding' | 'no_identity';
+}
+
 /** Derive channel capabilities from a raw source channel identifier. */
 export function resolveChannelCapabilities(sourceChannel?: string | null): ChannelCapabilities {
   const channel = sourceChannel ?? 'dashboard';
@@ -264,6 +276,32 @@ export function injectChannelCapabilityContext(message: Message, caps: ChannelCa
   };
 }
 
+/**
+ * Prepend guardian trust/identity facts to the last user message so the
+ * model can reason about guardian status from deterministic runtime facts.
+ */
+export function injectGuardianContext(message: Message, ctx: GuardianRuntimeContext): Message {
+  const lines: string[] = ['<guardian_context>'];
+  lines.push(`source_channel: ${ctx.sourceChannel}`);
+  lines.push(`actor_role: ${ctx.actorRole}`);
+  lines.push(`guardian_external_user_id: ${ctx.guardianExternalUserId ?? 'unknown'}`);
+  lines.push(`guardian_chat_id: ${ctx.guardianChatId ?? 'unknown'}`);
+  lines.push(`requester_identifier: ${ctx.requesterIdentifier ?? 'unknown'}`);
+  lines.push(`requester_external_user_id: ${ctx.requesterExternalUserId ?? 'unknown'}`);
+  lines.push(`requester_chat_id: ${ctx.requesterChatId ?? 'unknown'}`);
+  lines.push(`denial_reason: ${ctx.denialReason ?? 'none'}`);
+  lines.push('</guardian_context>');
+
+  const block = lines.join('\n');
+  return {
+    ...message,
+    content: [
+      { type: 'text', text: block },
+      ...message.content,
+    ],
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Prefix-based stripping primitive
 // ---------------------------------------------------------------------------
@@ -298,6 +336,11 @@ export function stripChannelCapabilityContext(messages: Message[]): Message[] {
   return stripUserTextBlocksByPrefix(messages, ['<channel_capabilities>']);
 }
 
+/** Strip `<guardian_context>` blocks injected by `injectGuardianContext`. */
+export function stripGuardianContext(messages: Message[]): Message[] {
+  return stripUserTextBlocksByPrefix(messages, ['<guardian_context>']);
+}
+
 /**
  * Prepend workspace top-level directory context to a user message.
  */
@@ -358,6 +401,7 @@ export function stripActiveSurfaceContext(messages: Message[]): Message[] {
 /** Prefixes stripped by the pipeline (order doesn't matter — single pass). */
 const RUNTIME_INJECTION_PREFIXES = [
   '<channel_capabilities>',
+  '<guardian_context>',
   '<workspace_top_level>',
   TEMPORAL_INJECTED_PREFIX,
   '<active_workspace>',
@@ -398,6 +442,7 @@ export function applyRuntimeInjections(
     activeSurface?: ActiveSurfaceContext | null;
     workspaceTopLevelContext?: string | null;
     channelCapabilities?: ChannelCapabilities | null;
+    guardianContext?: GuardianRuntimeContext | null;
     temporalContext?: string | null;
   },
 ): Message[] {
@@ -433,6 +478,16 @@ export function applyRuntimeInjections(
     }
   }
 
+  if (options.guardianContext) {
+    const userTail = result[result.length - 1];
+    if (userTail && userTail.role === 'user') {
+      result = [
+        ...result.slice(0, -1),
+        injectGuardianContext(userTail, options.guardianContext),
+      ];
+    }
+  }
+
   // Temporal context is injected before workspace top-level so it
   // appears after workspace context in the final message content
   // (both are prepended, so later injections appear first).

package/src/daemon/session-tool-setup.ts

@@ -41,6 +41,7 @@ import { projectSkillTools, type SkillProjectionCache } from './session-skill-to
  */
 export interface ToolSetupContext extends SurfaceSessionContext {
   readonly conversationId: string;
+  assistantId?: string;
   currentRequestId?: string;
   workingDir: string;
   sandboxOverride?: boolean;
@@ -186,6 +187,7 @@ export function createToolExecutor(
       workingDir: ctx.workingDir,
       sessionId: ctx.conversationId,
       conversationId: ctx.conversationId,
+      assistantId: ctx.assistantId,
       requestId: ctx.currentRequestId,
       taskRunId: ctx.taskRunId,
       onOutput,

package/src/daemon/session.ts

@@ -38,7 +38,7 @@ import { getHookManager } from '../hooks/manager.js';
 import { ConflictGate } from './session-conflict-gate.js';
 import { MessageQueue } from './session-queue-manager.js';
 import type { QueueDrainReason } from './session-queue-manager.js';
-import type { ChannelCapabilities } from './session-runtime-assembly.js';
+import type { ChannelCapabilities, GuardianRuntimeContext } from './session-runtime-assembly.js';
 import type { AssistantAttachmentDraft } from './assistant-attachments.js';
 import {
   handleSurfaceAction as handleSurfaceActionImpl,
@@ -127,6 +127,8 @@ export class Session {
   /** @internal */ currentActiveSurfaceId?: string;
   /** @internal */ currentPage?: string;
   /** @internal */ channelCapabilities?: ChannelCapabilities;
+  /** @internal */ guardianContext?: GuardianRuntimeContext;
+  /** @internal */ assistantId?: string;
   /** @internal */ pendingSurfaceActions = new Map<string, { surfaceType: SurfaceType }>();
   /** @internal */ lastSurfaceAction = new Map<string, { actionId: string; data?: Record<string, unknown> }>();
   /** @internal */ surfaceState = new Map<string, { surfaceType: SurfaceType; data: SurfaceData }>();
@@ -327,6 +329,14 @@ export class Session {
     this.channelCapabilities = caps ?? undefined;
   }
 
+  setGuardianContext(ctx: GuardianRuntimeContext | null): void {
+    this.guardianContext = ctx ?? undefined;
+  }
+
+  setAssistantId(assistantId: string | null): void {
+    this.assistantId = assistantId ?? undefined;
+  }
+
   persistUserMessage(
     content: string,
     attachments: UserMessageAttachment[],

package/src/inbound/public-ingress-urls.ts

@@ -5,8 +5,8 @@
  *
  * The canonical public base URL is resolved through a two-level chain:
  *
- *   1. **User Settings** (`config.ingress.publicBaseUrl`) — set via the
- *      Settings UI or `config set ingress.publicBaseUrl`. This is the
+ *   1. **User Settings** (`config.ingress.publicBaseUrl`) — set via
+ *      the in-chat config flow, the Settings UI, or `config set ingress.publicBaseUrl`. This is the
  *      primary source of truth. When the assistant spawns or restarts
  *      the gateway, this value is forwarded as the `INGRESS_PUBLIC_BASE_URL`
  *      environment variable so both processes agree on the same URL.
@@ -51,7 +51,7 @@ function normalizeUrl(url: string): string {
 export function getPublicBaseUrl(config: IngressConfig): string {
   if (config.ingress?.enabled === false) {
     throw new Error(
-      'Public ingress is disabled. Enable it in Settings to use public-facing webhooks.',
+      'Public ingress is disabled. Ask the assistant to enable it, or update it from the Settings page.',
     );
   }
 

package/src/memory/channel-guardian-store.ts

@@ -142,6 +142,7 @@ export function createBinding(params: {
   guardianExternalUserId: string;
   guardianDeliveryChatId: string;
   verifiedVia?: string;
+  metadataJson?: string | null;
 }): GuardianBinding {
   const db = getDb();
   const now = Date.now();
@@ -156,7 +157,7 @@ export function createBinding(params: {
     status: 'active' as const,
     verifiedAt: now,
     verifiedVia: params.verifiedVia ?? 'challenge',
-    metadataJson: null,
+    metadataJson: params.metadataJson ?? null,
     createdAt: now,
     updatedAt: now,
   };

package/src/memory/db-init.ts

@@ -1015,5 +1015,149 @@ export function initializeDb(): void {
 
   database.run(/*sql*/ `CREATE UNIQUE INDEX IF NOT EXISTS idx_channel_guardian_rate_limits_actor ON channel_guardian_rate_limits(assistant_id, channel, actor_external_user_id, actor_chat_id)`);
 
+  // ── Media Assets ───────────────────────────────────────────────────
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS media_assets (
+      id TEXT PRIMARY KEY,
+      title TEXT NOT NULL,
+      file_path TEXT NOT NULL,
+      mime_type TEXT NOT NULL,
+      duration_seconds REAL,
+      file_hash TEXT NOT NULL,
+      status TEXT NOT NULL DEFAULT 'registered',
+      media_type TEXT NOT NULL,
+      metadata TEXT,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+
+  // Drop the old non-unique index so it can be recreated as UNIQUE (migration for existing databases)
+  database.run(/*sql*/ `DROP INDEX IF EXISTS idx_media_assets_file_hash`);
+  database.run(/*sql*/ `CREATE UNIQUE INDEX IF NOT EXISTS idx_media_assets_file_hash ON media_assets(file_hash)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_assets_status ON media_assets(status)`);
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS processing_stages (
+      id TEXT PRIMARY KEY,
+      asset_id TEXT NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
+      stage TEXT NOT NULL,
+      status TEXT NOT NULL DEFAULT 'pending',
+      progress INTEGER NOT NULL DEFAULT 0,
+      last_error TEXT,
+      started_at INTEGER,
+      completed_at INTEGER
+    )
+  `);
+
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_processing_stages_asset_id ON processing_stages(asset_id)`);
+
+  // ── Media Keyframes ─────────────────────────────────────────────────
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS media_keyframes (
+      id TEXT PRIMARY KEY,
+      asset_id TEXT NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
+      timestamp REAL NOT NULL,
+      file_path TEXT NOT NULL,
+      metadata TEXT,
+      created_at INTEGER NOT NULL
+    )
+  `);
+
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_keyframes_asset_id ON media_keyframes(asset_id)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_keyframes_asset_timestamp ON media_keyframes(asset_id, timestamp)`);
+
+  // ── Media Vision Outputs ────────────────────────────────────────────
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS media_vision_outputs (
+      id TEXT PRIMARY KEY,
+      asset_id TEXT NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
+      keyframe_id TEXT NOT NULL REFERENCES media_keyframes(id) ON DELETE CASCADE,
+      analysis_type TEXT NOT NULL,
+      output TEXT NOT NULL,
+      confidence REAL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_vision_outputs_asset_id ON media_vision_outputs(asset_id)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_vision_outputs_keyframe_id ON media_vision_outputs(keyframe_id)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_vision_outputs_asset_type ON media_vision_outputs(asset_id, analysis_type)`);
+
+  // ── Media Timelines ─────────────────────────────────────────────────
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS media_timelines (
+      id TEXT PRIMARY KEY,
+      asset_id TEXT NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
+      start_time REAL NOT NULL,
+      end_time REAL NOT NULL,
+      segment_type TEXT NOT NULL,
+      attributes TEXT,
+      confidence REAL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_timelines_asset_id ON media_timelines(asset_id)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_timelines_asset_time ON media_timelines(asset_id, start_time)`);
+
+  // ── Media Events ──────────────────────────────────────────────────
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS media_events (
+      id TEXT PRIMARY KEY,
+      asset_id TEXT NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
+      event_type TEXT NOT NULL,
+      start_time REAL NOT NULL,
+      end_time REAL NOT NULL,
+      confidence REAL NOT NULL,
+      reasons TEXT NOT NULL,
+      metadata TEXT,
+      created_at INTEGER NOT NULL
+    )
+  `);
+
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_events_asset_id ON media_events(asset_id)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_events_asset_type ON media_events(asset_id, event_type)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_events_confidence ON media_events(confidence DESC)`);
+
+  // ── Media Tracking Profiles ─────────────────────────────────────────
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS media_tracking_profiles (
+      id TEXT PRIMARY KEY,
+      asset_id TEXT NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
+      capabilities TEXT NOT NULL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_tracking_profiles_asset_id ON media_tracking_profiles(asset_id)`);
+
+  // ── Media Event Feedback ──────────────────────────────────────────
+
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS media_event_feedback (
+      id TEXT PRIMARY KEY,
+      asset_id TEXT NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
+      event_id TEXT NOT NULL REFERENCES media_events(id) ON DELETE CASCADE,
+      feedback_type TEXT NOT NULL,
+      original_start_time REAL,
+      original_end_time REAL,
+      corrected_start_time REAL,
+      corrected_end_time REAL,
+      notes TEXT,
+      created_at INTEGER NOT NULL
+    )
+  `);
+
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_event_feedback_asset_id ON media_event_feedback(asset_id)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_event_feedback_event_id ON media_event_feedback(event_id)`);
+  database.run(/*sql*/ `CREATE INDEX IF NOT EXISTS idx_media_event_feedback_type ON media_event_feedback(asset_id, feedback_type)`);
+
   migrateMemoryFtsBackfill(database);
 }

package/src/memory/job-handlers/media-processing.ts

@@ -0,0 +1,100 @@
+import { getLogger } from '../../util/logger.js';
+import { asString } from '../job-utils.js';
+import { getMediaAssetById, updateMediaAssetStatus } from '../media-store.js';
+import type { MemoryJob } from '../jobs-store.js';
+import {
+  runPipeline,
+  type PipelineStageName,
+  type StageHandler,
+} from '../../config/bundled-skills/media-processing/services/processing-pipeline.js';
+import { extractKeyframesForAsset } from '../../config/bundled-skills/media-processing/tools/extract-keyframes.js';
+import { analyzeKeyframesForAsset } from '../../config/bundled-skills/media-processing/tools/analyze-keyframes.js';
+import { generateTimeline } from '../../config/bundled-skills/media-processing/services/timeline-service.js';
+import {
+  detectEvents,
+  type DetectionConfig,
+} from '../../config/bundled-skills/media-processing/services/event-detection-service.js';
+
+const log = getLogger('media-processing-job');
+
+const defaultDetectionConfig: DetectionConfig = {
+  eventType: 'scene_change',
+  rules: [
+    {
+      ruleType: 'segment_transition',
+      params: { field: 'segmentType' },
+      weight: 1.0,
+    },
+  ],
+};
+
+export async function mediaProcessingJob(job: MemoryJob): Promise<void> {
+  const mediaAssetId = asString(job.payload.mediaAssetId);
+  if (!mediaAssetId) {
+    log.warn({ jobId: job.id }, 'Missing mediaAssetId in job payload');
+    return;
+  }
+
+  const asset = getMediaAssetById(mediaAssetId);
+  if (!asset) {
+    log.warn({ jobId: job.id, mediaAssetId }, 'Media asset not found');
+    return;
+  }
+
+  if (asset.mediaType !== 'video') {
+    log.info(
+      { assetId: mediaAssetId, mediaType: asset.mediaType },
+      'Skipping media processing pipeline — only video assets are supported',
+    );
+    updateMediaAssetStatus(mediaAssetId, 'indexed');
+    return;
+  }
+
+  // Build detection config, allowing optional eventType override from payload
+  const eventType = asString(job.payload.eventType);
+  const detectionConfig: DetectionConfig = eventType
+    ? { ...defaultDetectionConfig, eventType }
+    : defaultDetectionConfig;
+
+  const handlers: Record<PipelineStageName, StageHandler> = {
+    keyframe_extraction: {
+      execute: (assetId, onProgress) =>
+        extractKeyframesForAsset(assetId, 1, onProgress),
+    },
+    vision_analysis: {
+      execute: (assetId, onProgress) =>
+        analyzeKeyframesForAsset(assetId, undefined, undefined, onProgress),
+    },
+    timeline_generation: {
+      execute: async (assetId, onProgress) => {
+        generateTimeline(assetId, { onProgress });
+      },
+    },
+    event_detection: {
+      execute: async (assetId, onProgress) => {
+        detectEvents(assetId, detectionConfig, { onProgress });
+      },
+    },
+  };
+
+  const result = await runPipeline(mediaAssetId, handlers, {
+    onProgress: (msg) => log.info({ mediaAssetId }, msg),
+  });
+
+  log.info(
+    {
+      mediaAssetId,
+      completedStages: result.completedStages,
+      failedStage: result.failedStage,
+      cancelled: result.cancelled,
+    },
+    'Media processing pipeline finished',
+  );
+
+  if (result.failedStage) {
+    throw new Error(`Media processing failed at stage ${result.failedStage}: ${result.failureReason}`);
+  }
+  if (result.cancelled) {
+    throw new Error(`Media processing cancelled for asset ${mediaAssetId}`);
+  }
+}

package/src/memory/jobs-store.ts

@@ -20,7 +20,8 @@ export type MemoryJobType =
   | 'build_conversation_summary'
   | 'backfill'
   | 'rebuild_index'
-  | 'delete_qdrant_vectors';
+  | 'delete_qdrant_vectors'
+  | 'media_processing';
 
 const EMBED_JOB_TYPES: MemoryJobType[] = ['embed_segment', 'embed_item', 'embed_summary'];
 

package/src/memory/jobs-worker.ts

@@ -29,6 +29,7 @@ import { checkContradictionsJob, cleanupStaleSupersededItemsJob } from './job-ha
 import { buildConversationSummaryJob, buildGlobalSummaryJob } from './job-handlers/summarization.js';
 import { backfillJob, backfillEntityRelationsJob } from './job-handlers/backfill.js';
 import { rebuildIndexJob, deleteQdrantVectorsJob } from './job-handlers/index-maintenance.js';
+import { mediaProcessingJob } from './job-handlers/media-processing.js';
 
 // Re-export public utilities consumed by tests and other modules
 export { currentWeekWindow } from './job-utils.js';
@@ -229,6 +230,9 @@ async function processJob(job: MemoryJob, config: AssistantConfig): Promise<void
     case 'delete_qdrant_vectors':
       await deleteQdrantVectorsJob(job);
       return;
+    case 'media_processing':
+      await mediaProcessingJob(job);
+      return;
     default:
       throw new Error(`Unknown memory job type: ${(job as { type: string }).type}`);
   }