@framers/agentos 0.1.120 → 0.1.122

package/dist/core/video/types.d.ts

@@ -0,0 +1,441 @@
+/**
+ * @file types.ts
+ * Core type definitions for the video generation and analysis subsystem.
+ *
+ * These types are consumed by {@link IVideoGenerator}, {@link IVideoAnalyzer},
+ * and {@link FallbackVideoProxy} to provide a unified video pipeline across
+ * multiple provider backends (Runway, Pika, Kling, Luma, etc.).
+ */
+/** Well-known video provider identifiers. Extensible via `(string & {})`. */
+export type VideoProviderId = 'runway' | 'replicate' | 'fal' | (string & {});
+/** Output container format for generated videos. */
+export type VideoOutputFormat = 'mp4' | 'webm' | 'gif';
+/** Aspect ratio presets commonly supported by video generation APIs. */
+export type VideoAspectRatio = '1:1' | '16:9' | '9:16' | '4:3' | '3:4' | '21:9' | (string & {});
+/** Describes a video model exposed by a provider. */
+export interface VideoModelInfo {
+    /** Unique model identifier (e.g. `'gen-3-alpha'`). */
+    modelId: string;
+    /** Provider that hosts this model. */
+    providerId: string;
+    /** Human-readable display name. */
+    displayName?: string;
+    /** Short description of the model's capabilities. */
+    description?: string;
+    /** Maximum output duration in seconds. */
+    maxDurationSec?: number;
+    /** Supported generation capabilities. */
+    capabilities?: Array<'text-to-video' | 'image-to-video'>;
+}
+/** Aggregated usage / billing counters for a video generation session. */
+export interface VideoProviderUsage {
+    /** Number of videos generated in this session. */
+    totalVideos: number;
+    /** Total cost in USD, if the provider reports it. */
+    totalCostUSD?: number;
+    /** Total processing time in milliseconds. */
+    processingTimeMs?: number;
+}
+/**
+ * Request payload for text-to-video generation.
+ *
+ * Passed to {@link IVideoGenerator.generateVideo} by the high-level
+ * orchestration layer after normalising user input.
+ */
+export interface VideoGenerateRequest {
+    /** Model identifier to use for generation (e.g. `'gen-3-alpha'`). */
+    modelId?: string;
+    /** Text prompt describing the desired video content. */
+    prompt: string;
+    /** Negative prompt describing content to avoid. */
+    negativePrompt?: string;
+    /** Desired output duration in seconds. */
+    durationSec?: number;
+    /** Desired aspect ratio (e.g. `'16:9'`). */
+    aspectRatio?: VideoAspectRatio;
+    /** Desired output resolution (e.g. `'1280x720'`). */
+    resolution?: string;
+    /** Output container format. Defaults to `'mp4'`. */
+    outputFormat?: VideoOutputFormat;
+    /** Frames per second for the output video. */
+    fps?: number;
+    /** Seed for reproducible output. */
+    seed?: number;
+    /** Number of videos to generate. Defaults to `1`. */
+    n?: number;
+    /** Identifier of the requesting user (for billing / rate limiting). */
+    userId?: string;
+    /** Arbitrary provider-specific options. */
+    providerOptions?: Record<string, unknown>;
+}
+/**
+ * Request payload for image-to-video generation.
+ *
+ * Passed to {@link IVideoGenerator.imageToVideo} by the high-level
+ * orchestration layer. Requires a source image that serves as the first
+ * frame (or style reference) for the generated video.
+ */
+export interface ImageToVideoRequest {
+    /** Model identifier to use for generation. */
+    modelId?: string;
+    /** Source image as a raw `Buffer`. */
+    image: Buffer;
+    /** Text prompt describing the desired motion / narrative. */
+    prompt: string;
+    /** Negative prompt describing content to avoid. */
+    negativePrompt?: string;
+    /** Desired output duration in seconds. */
+    durationSec?: number;
+    /** Desired aspect ratio. */
+    aspectRatio?: VideoAspectRatio;
+    /** Output container format. Defaults to `'mp4'`. */
+    outputFormat?: VideoOutputFormat;
+    /** Frames per second for the output video. */
+    fps?: number;
+    /** Seed for reproducible output. */
+    seed?: number;
+    /** Identifier of the requesting user. */
+    userId?: string;
+    /** Arbitrary provider-specific options. */
+    providerOptions?: Record<string, unknown>;
+}
+/** A single generated video artifact. */
+export interface GeneratedVideo {
+    /** Public URL where the video can be downloaded. */
+    url?: string;
+    /** Base64-encoded video data. */
+    base64?: string;
+    /** MIME type of the video (e.g. `'video/mp4'`). */
+    mimeType?: string;
+    /** Duration of the generated video in seconds. */
+    durationSec?: number;
+    /** Width in pixels. */
+    width?: number;
+    /** Height in pixels. */
+    height?: number;
+    /** Thumbnail / poster image URL. */
+    thumbnailUrl?: string;
+    /** Provider-specific metadata (job ID, generation params, etc.). */
+    providerMetadata?: Record<string, unknown>;
+}
+/**
+ * Result envelope returned by {@link IVideoGenerator.generateVideo} and
+ * {@link IVideoGenerator.imageToVideo}.
+ */
+export interface VideoResult {
+    /** Unix timestamp (ms) when the result was created. */
+    created: number;
+    /** Model identifier that produced the result. */
+    modelId: string;
+    /** Provider identifier that produced the result. */
+    providerId: string;
+    /** The generated video(s). */
+    videos: GeneratedVideo[];
+    /** Usage / billing information, if available. */
+    usage?: VideoProviderUsage;
+}
+/**
+ * Request payload for video analysis / understanding.
+ *
+ * Passed to {@link IVideoAnalyzer.analyzeVideo}.
+ */
+export interface VideoAnalyzeRequest {
+    /** URL of the video to analyse. Mutually exclusive with `videoBuffer`. */
+    videoUrl?: string;
+    /** Raw video bytes. Mutually exclusive with `videoUrl`. */
+    videoBuffer?: Buffer;
+    /** Text prompt / question to guide the analysis. */
+    prompt?: string;
+    /** Model identifier to use for analysis. */
+    modelId?: string;
+    /** Maximum number of frames to sample for analysis. */
+    maxFrames?: number;
+    /** Arbitrary provider-specific options. */
+    providerOptions?: Record<string, unknown>;
+}
+/** Structured result from video analysis. */
+export interface VideoAnalysis {
+    /** Free-form textual description / answer from the analyser. */
+    description: string;
+    /** Detected scene segments with timestamps. */
+    scenes?: VideoScene[];
+    /** Detected objects / entities across the video. */
+    objects?: string[];
+    /** Detected on-screen or spoken text (OCR / ASR). */
+    text?: string[];
+    /** Overall duration of the analysed video in seconds. */
+    durationSec?: number;
+    /** Model that produced the analysis. */
+    modelId?: string;
+    /** Provider that produced the analysis. */
+    providerId?: string;
+    /** Provider-specific metadata. */
+    providerMetadata?: Record<string, unknown>;
+}
+/** A single scene / segment detected during video analysis. */
+export interface VideoScene {
+    /** Start time of the scene in seconds. */
+    startSec: number;
+    /** End time of the scene in seconds. */
+    endSec: number;
+    /** Human-readable description of what happens in this scene. */
+    description: string;
+}
+/**
+ * Output resolution for generated or analyzed video.
+ *
+ * Higher resolutions increase generation time and cost but produce
+ * sharper output. Not all providers support all resolutions — the
+ * adapter will fall back to the closest supported resolution.
+ */
+export type VideoResolution = '480p' | '720p' | '1080p';
+/**
+ * Typed progress event emitted during video generation.
+ *
+ * The generation lifecycle flows through these statuses in order:
+ * `queued` -> `processing` -> `downloading` -> `complete` (or `failed`
+ * at any point).
+ */
+export interface VideoProgressEvent {
+    /**
+     * Current status of the generation job.
+     *
+     * - `'queued'`      — Request accepted, waiting for GPU slot
+     * - `'processing'`  — Actively generating frames
+     * - `'downloading'` — Generation complete, downloading result
+     * - `'complete'`    — Fully done, result available
+     * - `'failed'`      — Terminal error, see {@link message}
+     */
+    status: 'queued' | 'processing' | 'downloading' | 'complete' | 'failed';
+    /**
+     * Estimated progress percentage (0-100).
+     * Not all providers report granular progress; may remain undefined
+     * until the final status transition.
+     */
+    progress?: number;
+    /**
+     * Estimated time remaining in milliseconds.
+     * Only available when the provider reports ETA information.
+     */
+    estimatedRemainingMs?: number;
+    /** Human-readable status message or error description. */
+    message?: string;
+}
+/**
+ * Progress event emitted during video analysis.
+ *
+ * The analysis lifecycle flows through these phases in order:
+ * `extracting-frames` -> `detecting-scenes` -> `describing` ->
+ * `transcribing` -> `summarizing`.
+ */
+export interface VideoAnalysisProgressEvent {
+    /**
+     * Current phase of the analysis pipeline.
+     *
+     * - `'extracting-frames'` — Decoding video and extracting frames
+     * - `'detecting-scenes'`  — Running scene boundary detection
+     * - `'describing'`        — Sending key frames to vision LLM
+     * - `'transcribing'`      — Running audio transcription via Whisper
+     * - `'summarizing'`       — Generating overall video summary
+     */
+    phase: 'extracting-frames' | 'detecting-scenes' | 'describing' | 'transcribing' | 'summarizing';
+    /**
+     * Estimated progress percentage (0-100) within the current phase.
+     * Not always available — depends on the phase and provider.
+     */
+    progress?: number;
+    /**
+     * 0-based index of the scene currently being processed.
+     * Only meaningful during the `'describing'` and `'transcribing'` phases.
+     */
+    currentScene?: number;
+    /** Human-readable status message for the current phase. */
+    message?: string;
+}
+/**
+ * A single scene detected within a video, with timestamps,
+ * description, and optional transcript.
+ *
+ * Scenes are contiguous segments of video bounded by visual
+ * discontinuities (hard cuts, dissolves, fades). The
+ * {@link SceneDetector} identifies boundaries, and a vision LLM
+ * describes the content of each scene.
+ *
+ * This is a richer version of the base {@link VideoScene} type that
+ * includes cut-type classification, confidence, transcript, and key
+ * frame data.
+ */
+export interface SceneDescription {
+    /** 0-based scene index within the video. */
+    index: number;
+    /** Start time of the scene in seconds from video start. */
+    startSec: number;
+    /** End time of the scene in seconds from video start. */
+    endSec: number;
+    /** Duration of the scene in seconds (`endSec - startSec`). */
+    durationSec: number;
+    /**
+     * Type of visual transition that marks the beginning of this scene.
+     *
+     * - `'hard-cut'`  — Abrupt frame-to-frame change
+     * - `'dissolve'`  — Cross-dissolve / superimposition transition
+     * - `'fade'`      — Fade from/to black or white
+     * - `'wipe'`      — Directional wipe transition
+     * - `'gradual'`   — Other gradual transition not fitting the above
+     * - `'start'`     — First scene in the video (no preceding transition)
+     */
+    cutType: 'hard-cut' | 'dissolve' | 'fade' | 'wipe' | 'gradual' | 'start';
+    /**
+     * Natural-language description of the scene content, generated
+     * by a vision LLM from the key frame.
+     */
+    description: string;
+    /**
+     * Transcript of speech/narration during this scene's time range.
+     * Only populated when audio transcription is enabled.
+     */
+    transcript?: string;
+    /**
+     * Base64-encoded key frame image (JPEG) representative of the scene.
+     * Typically the frame closest to the scene midpoint.
+     */
+    keyFrame?: string;
+    /**
+     * Confidence score (0-1) for the scene boundary detection.
+     * Higher values indicate a more definitive visual discontinuity.
+     */
+    confidence: number;
+}
+/**
+ * Detail level for scene descriptions produced during video analysis.
+ *
+ * - `'brief'`      — 1-2 sentences per scene, focusing on key action
+ * - `'detailed'`   — Full paragraph per scene with lighting, composition, mood
+ * - `'exhaustive'` — Multi-paragraph with frame-level details, objects, colors
+ */
+export type DescriptionDetail = 'brief' | 'detailed' | 'exhaustive';
+/**
+ * Rich video analysis request with scene detection, transcription,
+ * and RAG indexing support.
+ *
+ * Extends the simpler {@link VideoAnalyzeRequest} pattern with
+ * fine-grained control over scene detection thresholds, description
+ * detail, and optional RAG indexing of analysis results.
+ *
+ * @example
+ * ```typescript
+ * const request: VideoAnalyzeRequestRich = {
+ *   video: 'https://example.com/demo.mp4',
+ *   sceneThreshold: 0.3,
+ *   transcribeAudio: true,
+ *   descriptionDetail: 'detailed',
+ *   onProgress: (evt) => console.log(`${evt.phase}: ${evt.progress}%`),
+ * };
+ * ```
+ */
+export interface VideoAnalyzeRequestRich {
+    /**
+     * Video to analyze — either a URL string or a raw Buffer.
+     * When a URL is provided, the pipeline downloads the video to a
+     * temporary file before processing.
+     */
+    video: string | Buffer;
+    /**
+     * Optional analysis prompt or question that should guide the final answer.
+     * When omitted, the analyzer produces a general-purpose summary.
+     */
+    prompt?: string;
+    /**
+     * Threshold for scene change detection (0-1).
+     * Lower values detect more scene boundaries (more sensitive);
+     * higher values only detect dramatic cuts.
+     * @default 0.3
+     */
+    sceneThreshold?: number;
+    /**
+     * Whether to transcribe the audio track using Whisper.
+     * When enabled, each scene's transcript is populated and a
+     * full transcript is included in the analysis.
+     * @default true
+     */
+    transcribeAudio?: boolean;
+    /**
+     * How detailed scene descriptions should be.
+     * @default 'detailed'
+     */
+    descriptionDetail?: DescriptionDetail;
+    /**
+     * Maximum number of frames to sample from the extracted frame set.
+     * When the extracted frame count exceeds this value, frames are
+     * evenly downsampled before scene detection and description.
+     */
+    maxFrames?: number;
+    /**
+     * Maximum number of scenes to detect.
+     * Prevents runaway analysis on very long videos with many cuts.
+     * @default 100
+     */
+    maxScenes?: number;
+    /**
+     * Whether to index scene descriptions and transcripts into the
+     * RAG vector store for later retrieval.
+     * @default false
+     */
+    indexForRAG?: boolean;
+    /**
+     * Optional callback invoked as analysis progresses through phases.
+     * Called with a {@link VideoAnalysisProgressEvent} at each phase
+     * transition and when per-scene progress updates are available.
+     */
+    onProgress?: (event: VideoAnalysisProgressEvent) => void;
+}
+/**
+ * Rich video analysis result with full scene descriptions, summary,
+ * transcript, and optional RAG chunk references.
+ *
+ * This is a richer version of the base {@link VideoAnalysis} type that
+ * includes {@link SceneDescription} scenes (with cut types, confidence,
+ * key frames), a generated summary, and optional RAG indexing metadata.
+ *
+ * @example
+ * ```typescript
+ * const analysis: VideoAnalysisRich = await videoAnalyzer.analyze(request);
+ *
+ * console.log(`${analysis.sceneCount} scenes in ${analysis.durationSec}s`);
+ * for (const scene of analysis.scenes) {
+ *   console.log(`[${scene.startSec}s-${scene.endSec}s] ${scene.description}`);
+ * }
+ * ```
+ */
+export interface VideoAnalysisRich {
+    /** Total video duration in seconds. */
+    durationSec: number;
+    /** Number of scenes detected. */
+    sceneCount: number;
+    /** Ordered list of all detected scenes with rich descriptions. */
+    scenes: SceneDescription[];
+    /**
+     * Overall summary of the video content, generated by an LLM
+     * from the scene descriptions and transcript.
+     */
+    summary: string;
+    /**
+     * Full transcript of all audio in the video, when transcription
+     * was enabled. Concatenation of all scene transcripts with
+     * timestamp markers.
+     */
+    fullTranscript?: string;
+    /**
+     * IDs of RAG vector store chunks created from this analysis.
+     * Only populated when {@link VideoAnalyzeRequestRich.indexForRAG}
+     * was enabled.
+     */
+    ragChunkIds?: string[];
+    /**
+     * Additional metadata about the analyzed video.
+     * Provider-specific information that doesn't fit into the
+     * structured fields above.
+     */
+    metadata: Record<string, unknown>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/core/video/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/core/video/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,6EAA6E;AAC7E,MAAM,MAAM,eAAe,GACvB,QAAQ,GACR,WAAW,GACX,KAAK,GACL,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;AAElB,oDAAoD;AACpD,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC;AAEvD,wEAAwE;AACxE,MAAM,MAAM,gBAAgB,GACxB,KAAK,GACL,MAAM,GACN,MAAM,GACN,KAAK,GACL,KAAK,GACL,MAAM,GACN,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;AAMlB,qDAAqD;AACrD,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qDAAqD;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,yCAAyC;IACzC,YAAY,CAAC,EAAE,KAAK,CAAC,eAAe,GAAG,gBAAgB,CAAC,CAAC;CAC1D;AAED,0EAA0E;AAC1E,MAAM,WAAW,kBAAkB;IACjC,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6CAA6C;IAC7C,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAMD;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACnC,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4CAA4C;IAC5C,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC,8CAA8C;IAC9C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,uEAAuE;IACvE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,oDAAoD;IACpD,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC,8CAA8C;IAC9C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAMD,yCAAyC;AACzC,MAAM,WAAW,cAAc;IAC7B,oDAAoD;IACpD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,iDAAiD;IACjD,KAAK,CAAC,EAAE,kBAAkB,CAAC;CAC5B;AAMD;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,6CAA6C;AAC7C,MAAM,WAAW,aAAa;IAC5B,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qDAAqD;IACrD,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IACzB,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAMxD;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;;;OAQG;IACH,MAAM,EAAE,QAAQ,GAAG,YAAY,GAAG,aAAa,GAAG,UAAU,GAAG,QAAQ,CAAC;IAExE;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IACzC;;;;;;;;OAQG;IACH,KAAK,EACD,mBAAmB,GACnB,kBAAkB,GAClB,YAAY,GACZ,cAAc,GACd,aAAa,CAAC;IAElB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAMD;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IAEd,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;IAEjB,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;IAEf,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,EAAE,UAAU,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IAEzE;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,UAAU,GAAG,YAAY,CAAC;AAEpE;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;OAIG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IAEvB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;OAGG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IAEtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;CAC1D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;IAEpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IAEnB,kEAAkE;IAClE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAE3B;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC"}

package/dist/core/video/types.js

@@ -0,0 +1,10 @@
+/**
+ * @file types.ts
+ * Core type definitions for the video generation and analysis subsystem.
+ *
+ * These types are consumed by {@link IVideoGenerator}, {@link IVideoAnalyzer},
+ * and {@link FallbackVideoProxy} to provide a unified video pipeline across
+ * multiple provider backends (Runway, Pika, Kling, Luma, etc.).
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/video/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/video/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}

package/dist/core/vision/SceneDetector.d.ts

@@ -0,0 +1,180 @@
+/**
+ * @module core/vision/SceneDetector
+ *
+ * General-purpose visual change detection for video frame sequences.
+ *
+ * The SceneDetector analyses consecutive frames using configurable methods
+ * (histogram chi-squared distance, SSIM, or CLIP embeddings) to identify
+ * scene boundaries — the points where the visual content changes
+ * significantly enough to indicate a new scene.
+ *
+ * ## Detection methods
+ *
+ * | Method | Speed | Quality | Dependencies |
+ * |--------|-------|---------|-------------|
+ * | `histogram` | Fast | Good for hard cuts | None |
+ * | `ssim` | Medium | Better for gradual transitions | `sharp` (optional) |
+ * | `clip` | Slow | Best for semantic changes | CLIP provider |
+ *
+ * ## Usage
+ *
+ * ```typescript
+ * import { SceneDetector } from '../../core/vision';
+ * import type { Frame, SceneBoundary } from '../../core/vision';
+ *
+ * const detector = new SceneDetector({ hardCutThreshold: 0.3 });
+ *
+ * // Single-shot comparison
+ * const { changed, score } = detector.hasSceneChanged(frameA, frameB);
+ *
+ * // Streaming detection over an async frame sequence
+ * for await (const boundary of detector.detectScenes(frameStream)) {
+ *   console.log(`Scene ${boundary.index} at ${boundary.startTimeSec}s`);
+ * }
+ * ```
+ *
+ * @see {@link Frame} for the frame data structure.
+ * @see {@link SceneBoundary} for the boundary output type.
+ * @see {@link SceneDetectorConfig} for configuration options.
+ */
+import type { Frame, SceneBoundary, SceneDetectorConfig } from './types.js';
+/**
+ * Detects scene boundaries in video frame sequences using configurable
+ * visual change detection methods.
+ *
+ * The detector supports three detection methods that can be combined:
+ * - **histogram** — 768-bin RGB histogram chi-squared distance (fast, no deps)
+ * - **ssim** — Structural Similarity Index via `sharp` (falls back to histogram)
+ * - **clip** — CLIP embedding cosine distance (requires CLIP provider)
+ *
+ * Scene boundaries are classified by cut type (hard-cut, dissolve, fade,
+ * gradual) based on the magnitude of the visual change.
+ */
+export declare class SceneDetector {
+    /** Resolved configuration with defaults applied. */
+    private readonly config;
+    /**
+     * Create a new SceneDetector with the given configuration.
+     *
+     * Missing configuration values are filled with sensible defaults:
+     * - `methods`: `['histogram', 'ssim']`
+     * - `hardCutThreshold`: `0.3`
+     * - `gradualThreshold`: `0.15`
+     * - `minSceneDurationSec`: `1.0`
+     * - `clipProvider`: `'local'`
+     *
+     * @param config - Optional partial configuration to override defaults.
+     */
+    constructor(config?: SceneDetectorConfig);
+    /**
+     * Detect scene boundaries by streaming through an async iterable of frames.
+     *
+     * Yields a {@link SceneBoundary} each time the visual difference between
+     * consecutive frames exceeds the configured threshold. The first scene
+     * boundary is never yielded for the very first frame — it is tracked
+     * internally as the start of scene 0.
+     *
+     * Respects {@link SceneDetectorConfig.minSceneDurationSec}: if a scene
+     * change is detected too soon after the previous boundary, it is
+     * suppressed (the current scene is extended instead).
+     *
+     * At the end of the stream, the final scene boundary is yielded to close
+     * out the last scene.
+     *
+     * @param frames - Async iterable of decoded video frames in time order.
+     * @yields SceneBoundary for each detected scene transition.
+     *
+     * @example
+     * ```typescript
+     * const boundaries: SceneBoundary[] = [];
+     * for await (const b of detector.detectScenes(frameStream)) {
+     *   boundaries.push(b);
+     * }
+     * ```
+     */
+    detectScenes(frames: AsyncIterable<Frame>): AsyncGenerator<SceneBoundary>;
+    /**
+     * Compare two frames and determine if a scene change occurred.
+     *
+     * Returns whether the frames differ enough to constitute a scene change,
+     * the raw difference score, and optionally the cut type.
+     *
+     * @param frameA - Raw pixel buffer of the first frame (RGB, 3 bytes/pixel).
+     * @param frameB - Raw pixel buffer of the second frame (RGB, 3 bytes/pixel).
+     * @returns Object with `changed` boolean, `score` (0-1), and optional `type`.
+     *
+     * @example
+     * ```typescript
+     * const result = detector.hasSceneChanged(bufferA, bufferB);
+     * if (result.changed) {
+     *   console.log(`Scene change detected: ${result.type} (score: ${result.score})`);
+     * }
+     * ```
+     */
+    hasSceneChanged(frameA: Buffer, frameB: Buffer): {
+        changed: boolean;
+        score: number;
+        type?: string;
+    };
+    /**
+     * Compute a frame-difference score using the configured detection methods.
+     *
+     * When multiple methods are configured, the maximum score is used so that
+     * any strong signal can trigger a scene boundary.
+     */
+    private computeDiff;
+    /**
+     * Compute the histogram difference between two raw RGB buffers.
+     *
+     * Builds a 768-bin histogram (256 bins per R/G/B channel) for each
+     * buffer, then computes the chi-squared distance between the two
+     * histograms, normalized to the range [0, 1].
+     *
+     * A score of 0 means the histograms are identical (same color
+     * distribution). A score approaching 1 means they are maximally
+     * different.
+     *
+     * @param a - First frame's raw RGB pixel buffer.
+     * @param b - Second frame's raw RGB pixel buffer.
+     * @returns Normalized chi-squared distance in [0, 1].
+     */
+    histogramDiff(a: Buffer, b: Buffer): number;
+    /**
+     * Compute the Structural Similarity Index (SSIM) difference between
+     * two frames.
+     *
+     * Attempts to use `sharp` for proper SSIM computation. If `sharp` is
+     * not available, falls back to {@link histogramDiff}.
+     *
+     * @param a - First frame's raw RGB pixel buffer.
+     * @param b - Second frame's raw RGB pixel buffer.
+     * @returns Difference score in [0, 1] where 0 = identical, 1 = maximally different.
+     */
+    ssimDiff(a: Buffer, b: Buffer): Promise<number>;
+    /**
+     * Compute a 768-bin RGB histogram from a raw pixel buffer.
+     *
+     * The buffer is expected to contain RGB pixels (3 bytes per pixel,
+     * row-major order). The histogram has 256 bins for each of the three
+     * channels (R: bins 0-255, G: bins 256-511, B: bins 512-767).
+     *
+     * Values are normalized to sum to 1.0 across all bins.
+     *
+     * @param buf - Raw RGB pixel data.
+     * @returns Float32Array of 768 normalized histogram bins.
+     */
+    private computeHistogram;
+    /**
+     * Classify a gradual cut based on the difference score.
+     *
+     * This is a simplified heuristic classification:
+     * - Score > 0.25 — `'dissolve'` (cross-dissolve / superimposition)
+     * - Score > 0.20 — `'fade'` (fade from/to black or white)
+     * - Otherwise     — `'gradual'` (generic gradual transition)
+     *
+     * @param diff - The difference score from histogram or SSIM comparison.
+     * @returns The classified cut type string.
+     */
+    private classifyGradualCut;
+}
+//# sourceMappingURL=SceneDetector.d.ts.map

package/dist/core/vision/SceneDetector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SceneDetector.d.ts","sourceRoot":"","sources":["../../../src/core/vision/SceneDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAmB5E;;;;;;;;;;;GAWG;AACH,qBAAa,aAAa;IACxB,oDAAoD;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6F;IAEpH;;;;;;;;;;;OAWG;gBACS,MAAM,CAAC,EAAE,mBAAmB;IAYxC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACI,YAAY,CAAC,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,aAAa,CAAC;IAuEhF;;;;;;;;;;;;;;;;;OAiBG;IACH,eAAe,CACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GACb;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE;IAkBrD;;;;;OAKG;YACW,WAAW;IAmCzB;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IA2B3C;;;;;;;;;;OAUG;IACG,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAsBrD;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,gBAAgB;IA4BxB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,kBAAkB;CAK3B"}