@larkup/tool-video-intelligence 0.2.0

package/dist/brief.d.ts

@@ -0,0 +1,2 @@
+import type { VideoIndexingBrief } from './contracts.js';
+export declare function createVideoIndexingBrief(input?: Partial<VideoIndexingBrief>): VideoIndexingBrief;

package/dist/brief.js

@@ -0,0 +1,37 @@
+const MODES = new Set(['fast', 'balanced', 'thorough']);
+function normalizeMode(value) {
+    return MODES.has(value) ? value : 'balanced';
+}
+export function createVideoIndexingBrief(input = {}) {
+    const contentType = input.contentType?.trim().slice(0, 120) || 'general';
+    const indexingMode = normalizeMode(input.indexingMode);
+    const retainSourceHours = Number.isFinite(input.retainSourceHours)
+        ? Math.max(0, Math.min(720, Math.floor(input.retainSourceHours)))
+        : 0;
+    return {
+        goal: input.goal?.trim().slice(0, 4_000) || undefined,
+        contentType,
+        knownEntities: uniqueStrings(input.knownEntities, 50),
+        expectedQuestions: uniqueStrings(input.expectedQuestions, 20),
+        language: input.language?.trim().slice(0, 32) || 'auto',
+        importantRanges: (input.importantRanges ?? [])
+            .filter((range) => Number.isFinite(range.startSecs) &&
+            Number.isFinite(range.endSecs) &&
+            range.startSecs >= 0 &&
+            range.endSecs > range.startSecs)
+            .slice(0, 20)
+            .map((range) => ({
+            startSecs: range.startSecs,
+            endSecs: range.endSecs,
+            note: range.note?.trim().slice(0, 500) || undefined,
+        })),
+        indexingMode,
+        processingAuthorityConfirmed: input.processingAuthorityConfirmed === true,
+        retainSourceHours,
+    };
+}
+function uniqueStrings(values, limit) {
+    return [...new Set((values ?? []).map((value) => value.trim()).filter(Boolean))]
+        .slice(0, limit)
+        .map((value) => value.slice(0, 500));
+}

package/dist/client.d.ts

@@ -0,0 +1,46 @@
+import type { SubmitVideoJobRequest, VideoIntelligenceClientContract, VideoJob, VideoRuntimeMode, VideoServiceEntitlement, VideoServiceUsage } from './contracts.js';
+export interface VideoIntelligenceClientOptions {
+    mode: VideoRuntimeMode;
+    endpoint?: string;
+    apiKey?: string;
+    fetch?: typeof globalThis.fetch;
+    timeoutMs?: number;
+}
+export declare class VideoIntelligenceClient implements VideoIntelligenceClientContract {
+    private readonly mode;
+    private readonly endpoint;
+    private readonly fetcher;
+    private readonly apiKey?;
+    private readonly timeoutMs;
+    constructor(options: VideoIntelligenceClientOptions);
+    health(): Promise<{
+        status: string;
+        version: string;
+        operators: Record<string, string>;
+    }>;
+    provisionDeviceAccess(installationId: string): Promise<{
+        apiKey: string;
+        entitlement: VideoServiceEntitlement;
+    }>;
+    upload(file: Blob, fileName: string): Promise<{
+        uploadId: string;
+    }>;
+    submitJob(request: SubmitVideoJobRequest): Promise<VideoJob>;
+    getJob(jobId: string): Promise<VideoJob>;
+    acknowledgeJobResult(jobId: string): Promise<{
+        status: string;
+    }>;
+    cancelJob(jobId: string): Promise<VideoJob>;
+    purgeJobData(jobId: string): Promise<void>;
+    /** Recovery path for an orphaned local asset. The service refuses ambiguity. */
+    cancelOnlyActiveJob(): Promise<{
+        status: string;
+        alreadyStopped?: boolean;
+    }>;
+    getUsage(): Promise<VideoServiceUsage>;
+    redeemAccessCode(code: string): Promise<{
+        apiKey: string;
+        entitlement: VideoServiceEntitlement;
+    }>;
+    private request;
+}

package/dist/client.js

@@ -0,0 +1,139 @@
+export class VideoIntelligenceClient {
+    mode;
+    endpoint;
+    fetcher;
+    apiKey;
+    timeoutMs;
+    constructor(options) {
+        this.mode = options.mode;
+        const defaultEndpoint = options.mode === 'local-docker' || options.mode === 'local-process'
+            ? 'http://127.0.0.1:8787'
+            : undefined;
+        if (!options.endpoint && !defaultEndpoint) {
+            throw new Error(`An endpoint is required for ${options.mode}.`);
+        }
+        this.endpoint = (options.endpoint ?? defaultEndpoint).replace(/\/$/, '');
+        this.fetcher = options.fetch ?? globalThis.fetch;
+        this.apiKey = options.apiKey;
+        this.timeoutMs = options.timeoutMs ?? 30_000;
+    }
+    async health() {
+        const health = await this.request('/v1/health');
+        const isManagedCloudHealth = health.runtime === 'managed-cloud' && typeof health.processingEnabled === 'boolean';
+        if (health.status !== 'ok' ||
+            typeof health.version !== 'string' ||
+            !health.version ||
+            ((!health.operators || typeof health.operators !== 'object') && !isManagedCloudHealth)) {
+            throw new Error('The endpoint did not identify itself as a Video Intelligence runtime.');
+        }
+        return {
+            status: health.status,
+            version: health.version,
+            operators: health.operators ?? {},
+        };
+    }
+    provisionDeviceAccess(installationId) {
+        return this.request('/v1/device-keys', {
+            method: 'POST',
+            body: JSON.stringify({ installationId }),
+            anonymous: true,
+        });
+    }
+    async upload(file, fileName) {
+        if (this.mode === 'managed-cloud') {
+            const initialized = await this.request('/v1/uploads', {
+                method: 'POST',
+                body: JSON.stringify({
+                    fileName,
+                    contentType: file.type || 'application/octet-stream',
+                    sizeBytes: file.size,
+                }),
+            });
+            const uploaded = await this.fetcher(initialized.uploadUrl, {
+                method: 'PUT',
+                headers: initialized.uploadHeaders,
+                body: file,
+            });
+            if (!uploaded.ok)
+                throw new Error(`Video upload returned HTTP ${uploaded.status}.`);
+            return { uploadId: initialized.uploadId };
+        }
+        const form = new FormData();
+        form.append('file', file, fileName);
+        return this.request('/v1/uploads', { method: 'POST', body: form });
+    }
+    submitJob(request) {
+        return this.request('/v1/jobs', { method: 'POST', body: JSON.stringify(request) });
+    }
+    async getJob(jobId) {
+        const job = await this.request(`/v1/jobs/${encodeURIComponent(jobId)}`);
+        if (job.status === 'completed' && !job.result && job.resultUrl) {
+            const response = await this.fetcher(job.resultUrl);
+            if (!response.ok)
+                throw new Error(`Video result returned HTTP ${response.status}.`);
+            job.result = (await response.json());
+        }
+        return job;
+    }
+    acknowledgeJobResult(jobId) {
+        return this.request(`/v1/jobs/${encodeURIComponent(jobId)}/result/ack`, {
+            method: 'POST',
+            body: JSON.stringify({}),
+        });
+    }
+    cancelJob(jobId) {
+        return this.request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: 'DELETE' });
+    }
+    async purgeJobData(jobId) {
+        await this.request(`/v1/jobs/${encodeURIComponent(jobId)}/data`, { method: 'DELETE' });
+    }
+    /** Recovery path for an orphaned local asset. The service refuses ambiguity. */
+    cancelOnlyActiveJob() {
+        return this.request('/v1/jobs/active', { method: 'DELETE' });
+    }
+    getUsage() {
+        return this.request('/v1/usage');
+    }
+    redeemAccessCode(code) {
+        return this.request('/v1/access-codes/redeem', {
+            method: 'POST',
+            body: JSON.stringify({ code }),
+            anonymous: true,
+        });
+    }
+    async request(path, init = {}) {
+        const headers = new Headers(init.headers);
+        if (!(init.body instanceof FormData))
+            headers.set('Content-Type', 'application/json');
+        if (this.apiKey && !init.anonymous)
+            headers.set('Authorization', `Bearer ${this.apiKey}`);
+        const canRetry = !init.method || init.method === 'GET';
+        for (let attempt = 0;; attempt += 1) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+            try {
+                const response = await this.fetcher(`${this.endpoint}${path}`, {
+                    ...init,
+                    headers,
+                    signal: controller.signal,
+                });
+                const body = (await response.json().catch(() => ({})));
+                const retryAfter = response.headers.get('Retry-After');
+                if (response.status === 429 && canRetry && attempt < 2 && retryAfter !== null) {
+                    const retryAfterSeconds = Number(retryAfter);
+                    await new Promise((resolve) => setTimeout(resolve, Number.isFinite(retryAfterSeconds)
+                        ? Math.max(0, Math.min(60, retryAfterSeconds)) * 1_000
+                        : 60_000));
+                    continue;
+                }
+                if (!response.ok) {
+                    throw new Error(body.detail ?? body.error ?? `Video service returned HTTP ${response.status}.`);
+                }
+                return body;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        }
+    }
+}

package/dist/contracts.d.ts

@@ -0,0 +1,331 @@
+/**
+ * 'local' is the user-facing/config-level mode (auto-detects Docker vs.
+ * native). 'local-docker' and 'local-process' remain the concrete kinds
+ * runtime.ts executes against once detection resolves 'local' to one of them.
+ */
+export type VideoRuntimeMode = 'local' | 'local-docker' | 'local-process' | 'managed-cloud' | 'custom-remote';
+export type LocalVideoRuntimeKind = 'local-docker' | 'local-process';
+export type VideoIndexingMode = 'fast' | 'balanced' | 'thorough';
+/** Optional free-form source description; it never selects a fixed pipeline. */
+export type VideoContentType = string;
+export type VideoJobStatus = 'queued' | 'running' | 'completed' | 'failed' | 'cancelled';
+export interface VideoIndexingBrief {
+    goal?: string;
+    contentType: VideoContentType;
+    knownEntities: string[];
+    expectedQuestions: string[];
+    language: string;
+    importantRanges: Array<{
+        startSecs: number;
+        endSecs: number;
+        note?: string;
+    }>;
+    indexingMode: VideoIndexingMode;
+    /** Confirms authority or another lawful basis; it is not necessarily GDPR consent. */
+    processingAuthorityConfirmed: boolean;
+    retainSourceHours: number;
+    /** The app supplies transcript evidence itself, so the cloud GPU skips speech decoding. */
+    skipTranscription?: boolean;
+    /** A bounded verification must retain its direct visual evidence source. */
+    requireSemanticVision?: boolean;
+    /** Reads the requested range as one chronology instead of independent clips. */
+    continuousSequence?: boolean;
+    /** Bounded visual budget chosen by the retrieval agent for this inspection. */
+    maxFrames?: number;
+}
+export interface VideoSource {
+    uploadId?: string;
+    path?: string;
+    objectKey?: string;
+    url?: string;
+    fileName?: string;
+    mimeType?: string;
+    /** Required by managed runtimes to reserve quota before a GPU worker starts. */
+    durationSecs?: number;
+}
+export interface VideoProviderModelCredential {
+    provider: string;
+    apiKey: string;
+    model: string;
+}
+/** User-owned providers required by the pipeline regardless of compute location. */
+export interface VideoJobModelConfiguration {
+    audio: VideoProviderModelCredential;
+    brain: VideoProviderModelCredential;
+    vision: VideoProviderModelCredential;
+}
+export interface SubmitVideoJobRequest {
+    source: VideoSource;
+    brief: VideoIndexingBrief;
+    /** Ephemeral BYOK bundle. Cloud control planes must never persist or log it. */
+    modelConfiguration?: VideoJobModelConfiguration;
+    idempotencyKey?: string;
+    webhookUrl?: string;
+}
+export interface VideoJobProgress {
+    stage: 'queued' | 'prepare' | 'probe' | 'decode' | 'transcribe' | 'ocr' | 'detect' | 'synthesize' | 'complete';
+    /** Progress across the whole job, monotonic and smoothed by the runtime. */
+    percent: number;
+    message: string;
+    /**
+     * Progress through `stage` alone, for a host that renders one bar per step.
+     * The runtime owns the mapping because only it knows how much of the job
+     * each stage represents; a host that derived this itself would need a copy
+     * of that budget and would silently break whenever the pipeline changed.
+     */
+    stagePercent?: number;
+    /** Monotonic worker update id; changing this proves the pipeline is alive. */
+    sequence?: number;
+    elapsedSeconds?: number;
+    /** Adaptive whole-worker ETA, recalibrated from measured throughput. */
+    estimatedRemainingSeconds?: number;
+    current?: number;
+    total?: number;
+    unit?: string;
+}
+export interface VideoUsageSummary {
+    sourceMinutes: number;
+    decodedFrames: number;
+    retainedFrames: number;
+    ocrFrames: number;
+    detectorFrames: number;
+    gpuSeconds: number;
+}
+export interface VideoJob {
+    id: string;
+    status: VideoJobStatus;
+    createdAt: string;
+    updatedAt: string;
+    progress: VideoJobProgress;
+    estimatedSourceMinutes: number;
+    result: VideoEvidenceBundle | null;
+    resultUrl?: string | null;
+    error: string | null;
+}
+export interface TranscriptEvidence {
+    startMs: number;
+    endMs: number;
+    text: string;
+    words: Array<{
+        startMs: number;
+        endMs: number;
+        text: string;
+        confidence: number;
+    }>;
+}
+export interface OcrEvidence {
+    text: string;
+    confidence: number;
+    box: number[][];
+}
+export interface TrackEvidence {
+    trackId: number;
+    classId: number;
+    label: string;
+    startMs: number;
+    endMs: number;
+    observations: number;
+    confidence: number;
+}
+export interface VisualObservation {
+    timeMs: number;
+    objects: Array<{
+        label: string;
+        classId: number;
+        trackId: number;
+        confidence: number;
+        box: [number, number, number, number];
+    }>;
+    ocr: OcrEvidence[];
+}
+/**
+ * Short on-screen text that stayed legible across several frames -- a title,
+ * a caption, a heading, a readout. It marks where a display existed and when
+ * it changed, so it is a source-navigation anchor rather than a fact.
+ */
+export interface RecurringOverlayText {
+    text: string;
+    firstSeenMs: number;
+    lastSeenMs: number;
+    observations: number;
+    timestampsMs: number[];
+    confidence: number;
+}
+export interface VideoEvidenceBundle {
+    schemaVersion: 1;
+    jobId: string;
+    durationMs: number;
+    video: {
+        width: number;
+        height: number;
+        fps: number;
+    };
+    brief: VideoIndexingBrief;
+    transcript: TranscriptEvidence[];
+    detectedLanguage?: string;
+    visualObservations: VisualObservation[];
+    tracks: TrackEvidence[];
+    recurringOverlayText?: RecurringOverlayText[];
+    entities: Array<{
+        name: string;
+        kind: 'object' | 'visible-text';
+        mentions: number;
+        timestampsMs?: number[];
+        confidence?: number;
+    }>;
+    coverage: {
+        requested: VideoIndexingMode;
+        sourceFrames?: number;
+        decodedFrames: number;
+        analyzedFrames: number;
+        heavyOperatorsDisabled: boolean;
+        priorityRanges?: Array<{
+            startSecs: number;
+            endSecs: number;
+            reason: string;
+        }>;
+        semanticClips?: number;
+    };
+    agentPlan?: {
+        mode: VideoIndexingMode;
+        summary: string;
+        extractionFocus: string[];
+        useTranscript: boolean;
+        useOcr: boolean;
+        useObjectDetection: boolean;
+        useSemanticVision: boolean;
+        useVideoEmbeddings: boolean;
+        useSceneCuts: boolean;
+        sampleIntervalSecs: number;
+        prioritySampleIntervalSecs: number;
+        clipWindowSecs: number;
+        framesPerClip: number;
+        priorityRanges: Array<{
+            startSecs: number;
+            endSecs: number;
+            reason: string;
+        }>;
+        estimatedSeconds: number;
+    };
+    agentDiagnostics?: {
+        attempted: boolean;
+        provider: string;
+        model: string;
+        requests: number;
+        latencyMs: number;
+        fallback: boolean;
+        error?: string | null;
+        promptTokens?: number;
+        completionTokens?: number;
+    };
+    processingDiagnostics?: {
+        estimatedTotalSeconds: number;
+        elapsedSeconds: number;
+        estimateErrorSeconds: number;
+    };
+    transcriptionDiagnostics?: {
+        provider?: string | null;
+        fallbackProvider?: string | null;
+        fallbackUsed?: boolean;
+        chunkCount?: number;
+        completedChunks?: number;
+        chunkErrors?: number;
+        error?: string | null;
+    };
+    knowledgeSummary?: {
+        overview: string;
+        participants: Array<{
+            name: string;
+            role: string;
+            evidence: Array<{
+                startMs: number;
+                endMs: number;
+            }>;
+        }>;
+        stateHistory: Array<{
+            startMs: number;
+            endMs: number;
+            state: string;
+            confidence: 'direct' | 'partial';
+        }>;
+        keyEvents: Array<{
+            startMs: number;
+            endMs: number;
+            event: string;
+            confidence: 'direct' | 'partial';
+        }>;
+        narrative?: Array<{
+            startMs: number;
+            endMs: number;
+            text: string;
+            confidence: 'direct' | 'partial';
+        }>;
+        context: Array<{
+            fact: string;
+            evidence: Array<{
+                startMs: number;
+                endMs: number;
+            }>;
+        }>;
+        sourceItems?: Array<{
+            kind: 'question' | 'heading' | 'slide-item' | 'board-item' | 'list-item';
+            channel: 'spoken' | 'visible';
+            text: string;
+            answer: string;
+            startMs: number;
+            endMs: number;
+        }>;
+        uncertainties: string[];
+    };
+    answeringGuide: {
+        goal?: string;
+        importantEntities: string[];
+        questionsToPrepareFor: string[];
+        extractionFocus?: string[];
+        instruction: string;
+    };
+}
+export interface VideoServiceEntitlement {
+    plan: string;
+    sourceMinutesPerMonth: number | null;
+    maxConcurrentJobs: number;
+}
+export interface VideoServiceUsage {
+    periodStart: string;
+    periodEnd: string;
+    sourceMinutesUsed: number;
+    sourceMinutesLimit: number | null;
+    activeJobs: number;
+    /** Exact active jobs allow a host to recover only its own orphaned asset. */
+    activeJobIds?: string[];
+    concurrentJobsLimit: number;
+}
+export interface VideoIntelligenceClientContract {
+    health(): Promise<{
+        status: string;
+        version: string;
+        operators: Record<string, string>;
+    }>;
+    /** Creates or rotates the opaque cloud credential for one local Larkup installation. */
+    provisionDeviceAccess(installationId: string): Promise<{
+        apiKey: string;
+        entitlement: VideoServiceEntitlement;
+    }>;
+    upload(file: Blob, fileName: string): Promise<{
+        uploadId: string;
+    }>;
+    submitJob(request: SubmitVideoJobRequest): Promise<VideoJob>;
+    getJob(jobId: string): Promise<VideoJob>;
+    /** Deletes the temporary encrypted result after the caller has validated it. */
+    acknowledgeJobResult(jobId: string): Promise<{
+        status: string;
+    }>;
+    cancelJob(jobId: string): Promise<VideoJob>;
+    /** Removes a local runtime's durable source/result cache for a deleted asset. */
+    purgeJobData(jobId: string): Promise<void>;
+    getUsage(): Promise<VideoServiceUsage>;
+    redeemAccessCode(code: string): Promise<{
+        apiKey: string;
+        entitlement: VideoServiceEntitlement;
+    }>;
+}

package/dist/contracts.js

@@ -0,0 +1 @@
+export {};

package/dist/index.d.ts

@@ -0,0 +1,87 @@
+import { type VideoIntelligenceAgentClient } from './agent.js';
+import type { LocalVideoRuntimeKind } from './contracts.js';
+export * from './brief.js';
+export * from './agent.js';
+export * from './client.js';
+export * from './contracts.js';
+export * from './runtime.js';
+export * from './ui.js';
+export declare const TOOL_META: {
+    readonly id: "video-intelligence";
+    readonly name: "Video Intelligence";
+    readonly version: "0.1.0";
+};
+export declare const TOOL_EXTENSION: {
+    id: "video-intelligence";
+    apiVersion: "1";
+    createClient: typeof createClientFromContext;
+    ensureRuntime(context: import("@larkup/marketplace/extension").ToolExtensionContext): Promise<void>;
+    restartRuntime(context: import("@larkup/marketplace/extension").ToolExtensionContext): Promise<void>;
+    /** Pulls the Docker image or installs uv + syncs Python deps, without starting anything. */
+    installRuntime(context: import("@larkup/marketplace/extension").ToolExtensionContext): Promise<void>;
+    stopRuntime(context: import("@larkup/marketplace/extension").ToolExtensionContext): Promise<void>;
+    verifyConfiguration(context: import("@larkup/marketplace/extension").ToolExtensionContext & {
+        verifyKey?: string;
+    }): Promise<void>;
+    removeRuntime(context: import("@larkup/marketplace/extension").ToolExtensionContext): Promise<void>;
+    /** Docker/native detection, system suitability, and AI-model availability for the Install alert. */
+    getHostCapabilities(context: import("@larkup/marketplace/extension").ToolExtensionContext): Promise<{
+        running: boolean;
+        gatewayKeyAvailable: boolean;
+        gatewayKeySource: string | null;
+        modelRequirement: {
+            configured: boolean;
+            message: string;
+            provider?: undefined;
+            model?: undefined;
+        } | {
+            configured: boolean;
+            provider: string | undefined;
+            model: string;
+            message: string;
+        };
+        docker: import("./runtime.js").DockerHostStatus;
+        native: import("./runtime.js").NativeHostStatus;
+        recommendedKind: LocalVideoRuntimeKind | null;
+        installed: boolean;
+        system: {
+            platform: NodeJS.Platform;
+            cpus: number;
+            totalMemGB: number;
+            freeMemGB: number;
+        };
+        acceleration: import("./runtime.js").LocalAcceleration;
+        suitability: {
+            level: "good" | "tight" | "unknown";
+            message: string;
+        };
+    }>;
+    provisionRuntime(context: import("@larkup/marketplace/extension").ToolExtensionContext): Promise<{
+        config: {
+            cloudInstallationId?: undefined;
+            cloudAccessKey?: undefined;
+        };
+        display?: undefined;
+    } | {
+        config: {
+            cloudInstallationId?: undefined;
+            cloudAccessKey?: undefined;
+        };
+        display: {
+            userId: string;
+        };
+    } | {
+        config: {
+            cloudInstallationId: string;
+            cloudAccessKey: string;
+        };
+        display: {
+            userId: string;
+        };
+    }>;
+};
+export default TOOL_EXTENSION;
+declare function createClientFromContext(context: {
+    config: Record<string, unknown>;
+    fetch?: typeof globalThis.fetch;
+}): VideoIntelligenceAgentClient;