@larkup/tool-video-audio 0.2.0

package/dist/url-importer.js

@@ -0,0 +1,291 @@
+import { spawn } from 'node:child_process';
+import { createWriteStream, promises as fs } from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { isIP } from 'node:net';
+import { lookup } from 'node:dns/promises';
+import { randomUUID } from 'node:crypto';
+export async function inspectMediaUrl(url) {
+    const parsed = validHttpUrl(url);
+    if (isYouTube(parsed)) {
+        const output = await runYtDlp([
+            '--dump-single-json',
+            '--simulate',
+            '--flat-playlist',
+            '--playlist-end',
+            '10',
+            url,
+        ]);
+        const data = JSON.parse(output);
+        const entries = data.entries?.slice(0, 10) ?? [];
+        return {
+            originalUrl: url,
+            title: data.title,
+            durationSecs: data.duration ?? entries.reduce((sum, entry) => sum + (entry.duration ?? 0), 0),
+            entryCount: Math.max(entries.length, 1),
+            mediaType: 'video',
+            isYouTube: true,
+        };
+    }
+    let response = await fetchPublic(url, { method: 'HEAD' });
+    if (response.status === 405 || response.status === 501) {
+        response = await fetchPublic(url, { headers: { Range: 'bytes=0-0' } });
+    }
+    if (!response.ok)
+        throw new Error(`Unable to inspect media URL (${response.status})`);
+    const headerMime = response.headers.get('content-type')?.split(';')[0];
+    const mimeType = mediaTypeFromMime(headerMime) === 'unknown'
+        ? mimeFromExtension(path.extname(parsed.pathname).slice(1))
+        : headerMime;
+    const rangeTotal = response.headers.get('content-range')?.match(/\/(\d+)$/)?.[1];
+    const inspection = {
+        originalUrl: url,
+        mimeType,
+        mediaType: mediaTypeFromMime(mimeType),
+        contentLength: Number(rangeTotal ?? response.headers.get('content-length')) || undefined,
+        durationSecs: Number(response.headers.get('content-duration')) || undefined,
+        entryCount: 1,
+        isYouTube: false,
+    };
+    await response.body?.cancel();
+    return inspection;
+}
+export async function importMediaUrl(url, options) {
+    const parsed = validHttpUrl(url);
+    await fs.mkdir(options.outputDir, { recursive: true });
+    if (isYouTube(parsed)) {
+        const template = path.join(options.outputDir, '%(title).120B [%(id)s].%(ext)s');
+        const print = '{"path":%(filepath)j,"title":%(title)j,"originalUrl":%(webpage_url)j,"ext":%(ext)j}';
+        const output = await runYtDlp([
+            '--no-progress',
+            '--playlist-end',
+            String(options.playlistMax ?? 10),
+            '--format',
+            'bestvideo[height<=480]+bestaudio/best[height<=480]/best',
+            '--merge-output-format',
+            'mp4',
+            '-o',
+            template,
+            '--print',
+            `after_move:${print}`,
+            url,
+        ]);
+        const imported = output
+            .split('\n')
+            .filter(Boolean)
+            .map((line) => {
+            const item = JSON.parse(line);
+            return {
+                path: item.path,
+                title: item.title,
+                originalUrl: item.originalUrl || url,
+                mimeType: mimeFromExtension(item.ext),
+                mediaType: mediaTypeFromMime(mimeFromExtension(item.ext)),
+            };
+        });
+        return Promise.all(imported.map(async (item) => ({
+            ...item,
+            sourceTranscript: await fetchYouTubeTranscript(item.originalUrl).catch(() => undefined),
+        })));
+    }
+    const response = await fetchPublic(url);
+    if (!response.ok || !response.body)
+        throw new Error(`Media download failed (${response.status})`);
+    const maxBytes = options.maxBytes ?? 500 * 1024 * 1024;
+    const declared = Number(response.headers.get('content-length'));
+    if (declared > maxBytes)
+        throw new Error(`Media exceeds download limit of ${maxBytes} bytes`);
+    const headerMime = response.headers.get('content-type')?.split(';')[0];
+    const rawName = path.basename(new URL(response.url).pathname) || 'download';
+    const name = rawName.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 180) || 'download';
+    const mimeType = mediaTypeFromMime(headerMime) === 'unknown'
+        ? mimeFromExtension(path.extname(name).slice(1))
+        : headerMime;
+    const outputPath = path.join(options.outputDir, `${randomUUID()}-${name}`);
+    let bytes = 0;
+    const body = Readable.fromWeb(response.body);
+    body.on('data', (chunk) => {
+        bytes += chunk.length;
+        if (bytes > maxBytes)
+            body.destroy(new Error(`Media exceeds download limit of ${maxBytes} bytes`));
+    });
+    try {
+        await pipeline(body, createWriteStream(outputPath, { flags: 'wx' }));
+    }
+    catch (error) {
+        await fs.rm(outputPath, { force: true });
+        throw error;
+    }
+    return [
+        {
+            path: outputPath,
+            originalUrl: url,
+            title: name,
+            mimeType,
+            mediaType: mediaTypeFromMime(mimeType),
+        },
+    ];
+}
+/**
+ * Read the video's own manual or automatic captions through yt-dlp. This is
+ * both faster and usually more accurate than transcribing compressed audio,
+ * especially for names and non-English speech.
+ */
+async function fetchYouTubeTranscript(url) {
+    const output = await runYtDlp(['--dump-single-json', '--skip-download', '--no-playlist', url]);
+    const data = JSON.parse(output);
+    const manualTrack = selectSubtitleTrack(data.subtitles, data.language);
+    const selected = manualTrack ?? selectSubtitleTrack(data.automatic_captions, data.language);
+    if (!selected)
+        return undefined;
+    const response = await fetch(selected.url);
+    if (!response.ok)
+        throw new Error(`YouTube captions download failed (${response.status})`);
+    const transcript = parseYouTubeJson3Transcript(await response.json(), data.duration ?? 0);
+    return transcript.chunks.length > 0
+        ? {
+            ...transcript,
+            language: selected.language,
+            origin: {
+                kind: manualTrack ? 'youtube-manual' : 'youtube-auto',
+                language: selected.language,
+            },
+        }
+        : undefined;
+}
+function selectSubtitleTrack(tracks, language) {
+    if (!tracks)
+        return undefined;
+    const languages = Object.keys(tracks).filter((key) => key !== 'live_chat');
+    const baseLanguage = language?.split('-')[0];
+    const preferredLanguages = [
+        language,
+        language ? `${language}-orig` : undefined,
+        baseLanguage,
+        baseLanguage ? `${baseLanguage}-orig` : undefined,
+        ...languages.filter((key) => key.endsWith('-orig')),
+        ...languages,
+    ].filter((key) => Boolean(key));
+    for (const key of [...new Set(preferredLanguages)]) {
+        const format = tracks[key]?.find((candidate) => candidate.ext === 'json3' && candidate.url);
+        if (format?.url)
+            return { url: format.url, language: key.replace(/-orig$/, '') };
+    }
+    return undefined;
+}
+/** Parse YouTube's timestamped json3 caption format into indexing chunks. */
+export function parseYouTubeJson3Transcript(data, durationSecs = 0, chunkDurationSecs = 30) {
+    const cues = (data.events ?? [])
+        .map((event) => ({
+        text: (event.segs ?? [])
+            .map((segment) => segment.utf8 ?? '')
+            .join('')
+            .replace(/\s+/g, ' ')
+            .trim(),
+        startSecs: Math.max(0, Number(event.tStartMs ?? 0) / 1_000),
+        endSecs: Math.max(0, Number(event.tStartMs ?? 0) / 1_000 + Number(event.dDurationMs ?? 0) / 1_000),
+    }))
+        .filter((cue) => cue.text);
+    const chunks = [];
+    for (const cue of cues) {
+        const current = chunks.at(-1);
+        if (!current || cue.startSecs - current.startSecs >= chunkDurationSecs) {
+            chunks.push({ ...cue });
+        }
+        else {
+            current.text = `${current.text} ${cue.text}`;
+            current.endSecs = Math.max(current.endSecs, cue.endSecs);
+        }
+    }
+    return {
+        fullText: chunks.map((chunk) => chunk.text).join(' '),
+        chunks,
+        durationSecs: Math.max(durationSecs, chunks.at(-1)?.endSecs ?? 0),
+    };
+}
+function validHttpUrl(value) {
+    const url = new URL(value);
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        throw new Error('Only http(s) media URLs are supported');
+    return url;
+}
+async function fetchPublic(url, init = {}) {
+    let current = validHttpUrl(url);
+    for (let redirects = 0; redirects <= 5; redirects++) {
+        await assertPublicHost(current.hostname);
+        const response = await fetch(current, { ...init, redirect: 'manual' });
+        if (![301, 302, 303, 307, 308].includes(response.status))
+            return response;
+        const location = response.headers.get('location');
+        if (!location)
+            throw new Error('Media URL redirected without a location');
+        current = validHttpUrl(new URL(location, current).toString());
+    }
+    throw new Error('Media URL redirected too many times');
+}
+async function assertPublicHost(hostname) {
+    const addresses = isIP(hostname)
+        ? [{ address: hostname }]
+        : await lookup(hostname, { all: true, verbatim: true });
+    if (!addresses.length || addresses.some(({ address }) => isPrivateAddress(address))) {
+        throw new Error('Private or local media URLs are not supported');
+    }
+}
+function isPrivateAddress(address) {
+    const normalized = address.toLowerCase();
+    if (normalized === '::1' || normalized === '::' || normalized.startsWith('fe80:'))
+        return true;
+    if (normalized.startsWith('fc') || normalized.startsWith('fd'))
+        return true;
+    const ipv4 = normalized.startsWith('::ffff:') ? normalized.slice(7) : normalized;
+    const parts = ipv4.split('.').map(Number);
+    if (parts.length !== 4 || parts.some(Number.isNaN))
+        return false;
+    const [a, b] = parts;
+    return (a === 0 ||
+        a === 10 ||
+        a === 127 ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 168) ||
+        a >= 224);
+}
+function isYouTube(url) {
+    return /(^|\.)youtube\.com$|(^|\.)youtu\.be$/.test(url.hostname);
+}
+function mediaTypeFromMime(mime) {
+    return mime?.startsWith('video/') ? 'video' : mime?.startsWith('audio/') ? 'audio' : 'unknown';
+}
+function mimeFromExtension(ext) {
+    return {
+        mp4: 'video/mp4',
+        webm: 'video/webm',
+        mkv: 'video/x-matroska',
+        mov: 'video/quicktime',
+        mp3: 'audio/mpeg',
+        m4a: 'audio/mp4',
+        wav: 'audio/wav',
+        ogg: 'audio/ogg',
+        flac: 'audio/flac',
+    }[ext.toLowerCase()];
+}
+function runYtDlp(args) {
+    return new Promise((resolve, reject) => {
+        const child = spawn('yt-dlp', args, { shell: false });
+        let stdout = '';
+        let stderr = '';
+        child.stdout.on('data', (data) => {
+            stdout += String(data);
+        });
+        child.stderr.on('data', (data) => {
+            stderr += String(data);
+        });
+        child.on('error', (error) => reject(error.code === 'ENOENT'
+            ? new Error('yt-dlp is required for YouTube URLs. Install it from https://github.com/yt-dlp/yt-dlp#installation')
+            : error));
+        child.on('close', (code) => code === 0
+            ? resolve(stdout.trim())
+            : reject(new Error(`yt-dlp failed (${code}): ${stderr.trim()}`)));
+    });
+}

package/dist/video-processor.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Video processing pipeline.
+ *
+ * Uses ffmpeg (via fluent-ffmpeg) to:
+ * 1. Extract audio track → send to audio-processor for transcription
+ * 2. Extract keyframes at configurable intervals
+ * 3. Generate thumbnails for each keyframe
+ *
+ * The caller (API route) handles Vision LLM captioning of frames.
+ */
+export interface VideoProcessResult {
+    /** Path to extracted audio file (WAV, 16kHz mono) */
+    audioPath?: string;
+    /** Extracted keyframe image paths with timestamps */
+    frames: {
+        path: string;
+        timestampSecs: number;
+    }[];
+    /** Video metadata */
+    meta: {
+        durationSecs: number;
+        width: number;
+        height: number;
+        codec: string;
+    };
+}
+export interface VideoProcessOptions {
+    /** Output directory for extracted files */
+    outputDir: string;
+    /** Extract one frame every N seconds (default: 10) */
+    frameIntervalSecs?: number;
+    /** Maximum number of frames to extract (default: 100) */
+    maxFrames?: number;
+    /** Scene detection sensitivity (default: 0.3; lower detects more changes) */
+    sceneThreshold?: number;
+    /** Maximum number of threads for FFmpeg to use */
+    threads?: number;
+    /** Whether to run parallel audio and frame extraction */
+    parallelExtraction?: boolean;
+    /** Skip audio extraction when an authoritative source transcript is available. */
+    skipAudioExtraction?: boolean;
+    /** Reports actual FFmpeg extraction progress as a value between 0 and 1. */
+    onProgress?: (progress: number) => void;
+}
+export interface VideoSamplingPlan {
+    /** Normalized video duration used to calculate the plan. */
+    durationSecs: number;
+    /** Hard ceiling for all extracted frames combined. */
+    maxFrames: number;
+    /** Cadence used for uniformly distributed coverage frames. */
+    periodicIntervalSecs: number;
+    /** Maximum number of uniformly distributed coverage frames. */
+    periodicFrameCount: number;
+    /** Maximum number of additional scene-change frames. */
+    sceneFrameCount: number;
+    /** Frames reserved for a denser look at the end of the recording. */
+    endingFrameCount: number;
+    /** Minimum time between selected scene changes. */
+    minimumSceneGapSecs: number;
+    /** Maximum periodic + scene + ending frames before de-duplication. */
+    estimatedFrameCount: number;
+}
+export interface EndingSamplingPlan {
+    startSecs: number;
+    intervalSecs: number;
+    frameCount: number;
+    timestamps: number[];
+}
+export interface TimedText {
+    text: string;
+    startSecs: number;
+    endSecs: number;
+}
+export interface MultimodalSegment extends TimedText {
+    transcript: string;
+    visualContext: string;
+    sequence: number;
+}
+/**
+ * Fuse speech and visual observations onto one timeline. Each searchable unit
+ * contains everything that happened in the same time window instead of an
+ * unrelated transcript or frame caption.
+ */
+export declare function buildMultimodalSegments(transcript: TimedText[], visuals: TimedText[], durationSecs: number, targetWindowSecs?: number): MultimodalSegment[];
+/**
+ * Create a bounded frame-sampling plan without treating maxFrames as a quota.
+ *
+ * Edited videos receive frequent anchors and scene-change coverage. As duration
+ * grows, the anchor cadence widens so multi-hour camera and screen recordings
+ * remain searchable without generating hundreds of near-identical frames.
+ */
+export declare function createVideoSamplingPlan(durationSecs: number, configuredInterval?: number, maxFrames?: number): VideoSamplingPlan;
+/** Densely sample the ending where final results and decisions commonly appear. */
+export declare function createEndingSamplingPlan(durationSecs: number, maxAdditionalFrames: number): EndingSamplingPlan;
+/**
+ * Process a video file: extract audio and keyframes.
+ */
+export declare function processVideo(videoPath: string, options: VideoProcessOptions): Promise<VideoProcessResult>;
+/** Extract scene-change frames in one ffmpeg process, with adaptive interval fallback. */
+export declare function extractSceneFrames(videoPath: string, options: {
+    outputDir: string;
+    maxFrames: number;
+    durationSecs?: number;
+    intervalSecs?: number;
+    sceneThreshold?: number;
+    threads?: number;
+    onProgress?: (progress: number) => void;
+}): Promise<{
+    path: string;
+    timestampSecs: number;
+}[]>;
+/**
+ * Extract keyframes from a video at regular intervals.
+ */
+export declare function extractFrames(videoPath: string, options: {
+    outputDir: string;
+    intervalSecs: number;
+    maxFrames: number;
+    durationSecs?: number;
+    /** Optional range start, used for denser ending/outcome evidence. */
+    startSecs?: number;
+    threads?: number;
+    onProgress?: (progress: number) => void;
+}): Promise<{
+    path: string;
+    timestampSecs: number;
+}[]>;