@pellux/goodvibes-sdk 0.18.14 → 0.18.16

package/dist/_internal/platform/media/builtin-providers.js

@@ -0,0 +1,13 @@
+import { MediaProviderRegistry } from '@pellux/goodvibes-sdk/platform/media/provider-registry';
+import { createAnthropicImageUnderstandingProvider, createBuiltinImageUnderstandingProvider, createGeminiImageUnderstandingProvider, createLocalImageUnderstandingProvider, createOpenAIImageUnderstandingProvider, } from './builtin-image-understanding.js';
+import { builtinGenerationProviders } from '@pellux/goodvibes-sdk/platform/media/builtin-generation-providers';
+export function ensureBuiltinMediaProviders(registry, artifactStore, providerRegistry) {
+    registry.register(createOpenAIImageUnderstandingProvider(providerRegistry, artifactStore), { replace: true });
+    registry.register(createGeminiImageUnderstandingProvider(providerRegistry, artifactStore), { replace: true });
+    registry.register(createAnthropicImageUnderstandingProvider(providerRegistry, artifactStore), { replace: true });
+    registry.register(createLocalImageUnderstandingProvider(providerRegistry, artifactStore), { replace: true });
+    registry.register(createBuiltinImageUnderstandingProvider(providerRegistry, artifactStore), { replace: true });
+    for (const provider of builtinGenerationProviders()) {
+        registry.register(provider, { replace: true });
+    }
+}

package/dist/_internal/platform/media/index.d.ts

@@ -0,0 +1,4 @@
+export { MediaProviderRegistry, } from '@pellux/goodvibes-sdk/platform/media/provider-registry';
+export { ensureBuiltinMediaProviders } from './builtin-providers.js';
+export type { MediaAnalysisRequest, MediaAnalysisResult, MediaArtifact, MediaGenerationRequest, MediaGenerationResult, MediaProvider, MediaProviderCapability, MediaProviderDescriptor, MediaProviderState, MediaProviderStatus, MediaTransformRequest, MediaTransformResult, } from '@pellux/goodvibes-sdk/platform/media/provider-registry';
+//# sourceMappingURL=index.d.ts.map

package/dist/_internal/platform/media/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/_internal/platform/media/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,GACtB,MAAM,wDAAwD,CAAC;AAChE,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AACrE,YAAY,EACV,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,mBAAmB,EACnB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,wDAAwD,CAAC"}

package/dist/_internal/platform/media/index.js

@@ -0,0 +1,2 @@
+export { MediaProviderRegistry, } from '@pellux/goodvibes-sdk/platform/media/provider-registry';
+export { ensureBuiltinMediaProviders } from './builtin-providers.js';

package/dist/_internal/platform/multimodal/index.d.ts

@@ -0,0 +1,3 @@
+export { MultimodalService } from './service.js';
+export type { MultimodalAnalysisRequest, MultimodalAnalysisResult, MultimodalArtifactInput, MultimodalDetail, MultimodalKind, MultimodalPacket, MultimodalProviderDescriptor, MultimodalSegment, MultimodalServiceStatus, MultimodalWritebackResult, } from '@pellux/goodvibes-sdk/platform/multimodal/types';
+//# sourceMappingURL=index.d.ts.map

package/dist/_internal/platform/multimodal/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/_internal/platform/multimodal/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EACV,yBAAyB,EACzB,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,4BAA4B,EAC5B,iBAAiB,EACjB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,iDAAiD,CAAC"}

package/dist/_internal/platform/multimodal/index.js

@@ -0,0 +1 @@
+export { MultimodalService } from './service.js';

package/dist/_internal/platform/multimodal/service.d.ts

@@ -0,0 +1,30 @@
+import { ArtifactStore } from '@pellux/goodvibes-sdk/platform/artifacts/index';
+import { MediaProviderRegistry } from '../media/index.js';
+import { KnowledgeService } from '../knowledge/index.js';
+import { VoiceService } from '@pellux/goodvibes-sdk/platform/voice/index';
+import type { MultimodalAnalysisRequest, MultimodalAnalysisResult, MultimodalDetail, MultimodalPacket, MultimodalProviderDescriptor, MultimodalServiceStatus, MultimodalWritebackResult } from '@pellux/goodvibes-sdk/platform/multimodal/types';
+export declare class MultimodalService {
+    private readonly artifactStore;
+    private readonly mediaProviders;
+    private readonly voiceService;
+    private readonly knowledgeService;
+    constructor(artifactStore: ArtifactStore, mediaProviders: MediaProviderRegistry, voiceService: VoiceService, knowledgeService: KnowledgeService);
+    getStatus(): Promise<MultimodalServiceStatus>;
+    listProviders(): Promise<readonly MultimodalProviderDescriptor[]>;
+    analyze(request: MultimodalAnalysisRequest): Promise<MultimodalAnalysisResult>;
+    buildPacket(result: MultimodalAnalysisResult, detail?: MultimodalDetail, budgetLimit?: number): MultimodalPacket;
+    writeBackAnalysis(result: MultimodalAnalysisResult, input?: {
+        readonly sessionId?: string;
+        readonly title?: string;
+        readonly tags?: readonly string[];
+        readonly folderPath?: string;
+        readonly metadata?: Record<string, unknown>;
+    }): Promise<MultimodalWritebackResult>;
+    private resolveArtifactInput;
+    private resolveKind;
+    private analyzeImage;
+    private analyzeAudio;
+    private analyzeDocument;
+    private analyzeVideo;
+}
+//# sourceMappingURL=service.d.ts.map

package/dist/_internal/platform/multimodal/service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../../../src/_internal/platform/multimodal/service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAC;AAE/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAE1E,OAAO,KAAK,EACV,yBAAyB,EACzB,wBAAwB,EACxB,gBAAgB,EAEhB,gBAAgB,EAChB,4BAA4B,EAC5B,uBAAuB,EACvB,yBAAyB,EAC1B,MAAM,iDAAiD,CAAC;AAgFzD,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,gBAAgB;gBAHhB,aAAa,EAAE,aAAa,EAC5B,cAAc,EAAE,qBAAqB,EACrC,YAAY,EAAE,YAAY,EAC1B,gBAAgB,EAAE,gBAAgB;IAG/C,SAAS,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAU7C,aAAa,IAAI,OAAO,CAAC,SAAS,4BAA4B,EAAE,CAAC;IAoCjE,OAAO,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IAcpF,WAAW,CAAC,MAAM,EAAE,wBAAwB,EAAE,MAAM,GAAE,gBAA6B,EAAE,WAAW,SAAyB,GAAG,gBAAgB;IA4BtI,iBAAiB,CACrB,MAAM,EAAE,wBAAwB,EAChC,KAAK,GAAE;QACL,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;QAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACxC,GACL,OAAO,CAAC,yBAAyB,CAAC;YAoDvB,oBAAoB;IA0BlC,OAAO,CAAC,WAAW;YAaL,YAAY;YAwCZ,YAAY;YAyCZ,eAAe;YAgCf,YAAY;CAqF3B"}

package/dist/_internal/platform/multimodal/service.js

@@ -0,0 +1,443 @@
+import { randomUUID } from 'node:crypto';
+import { ArtifactStore } from '@pellux/goodvibes-sdk/platform/artifacts/index';
+import { MediaProviderRegistry } from '../media/index.js';
+import { extractKnowledgeArtifact } from '@pellux/goodvibes-sdk/platform/knowledge/extractors';
+import { KnowledgeService } from '../knowledge/index.js';
+import { VoiceService } from '@pellux/goodvibes-sdk/platform/voice/index';
+const PACKET_BUDGETS = {
+    compact: 280,
+    standard: 680,
+    detailed: 1_280,
+};
+function estimateTokens(...values) {
+    const total = values.reduce((sum, value) => sum + (value?.length ?? 0), 0);
+    return Math.max(1, Math.ceil(total / 4));
+}
+function compactText(value, maxLength = 240) {
+    const trimmed = value?.replace(/\s+/g, ' ').trim();
+    if (!trimmed)
+        return undefined;
+    if (trimmed.length <= maxLength)
+        return trimmed;
+    return `${trimmed.slice(0, maxLength - 1).trim()}…`;
+}
+function coerceStringList(value) {
+    if (Array.isArray(value)) {
+        return value
+            .filter((entry) => typeof entry === 'string' && entry.trim().length > 0)
+            .map((entry) => entry.trim());
+    }
+    if (typeof value === 'string' && value.trim().length > 0) {
+        return value.split(/[,\n]/).map((entry) => entry.trim()).filter(Boolean);
+    }
+    return [];
+}
+function topKeywords(input, limit = 8) {
+    const counts = new Map();
+    for (const token of input.toLowerCase().split(/[^a-z0-9_./:-]+/).map((entry) => entry.trim()).filter(Boolean)) {
+        if (token.length < 3)
+            continue;
+        if (/^(https?|www|the|and|for|with|from|this|that|into|over)$/.test(token))
+            continue;
+        counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+    return [...counts.entries()]
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .slice(0, Math.max(1, limit))
+        .map(([token]) => token);
+}
+function audioFormatFromMimeType(mimeType) {
+    const lower = mimeType.toLowerCase();
+    if (lower.includes('wav'))
+        return 'wav';
+    if (lower.includes('mpeg') || lower.includes('mp3'))
+        return 'mp3';
+    if (lower.includes('ogg'))
+        return 'ogg';
+    if (lower.includes('webm'))
+        return 'webm';
+    if (lower.includes('flac'))
+        return 'flac';
+    return 'wav';
+}
+function detailLimit(detail, compact, standard, detailed) {
+    switch (detail) {
+        case 'compact':
+            return compact;
+        case 'detailed':
+            return detailed;
+        default:
+            return standard;
+    }
+}
+function mergeUnique(...groups) {
+    const seen = new Set();
+    const result = [];
+    for (const group of groups) {
+        for (const entry of group ?? []) {
+            const trimmed = entry.trim();
+            if (!trimmed || seen.has(trimmed))
+                continue;
+            seen.add(trimmed);
+            result.push(trimmed);
+        }
+    }
+    return result;
+}
+export class MultimodalService {
+    artifactStore;
+    mediaProviders;
+    voiceService;
+    knowledgeService;
+    constructor(artifactStore, mediaProviders, voiceService, knowledgeService) {
+        this.artifactStore = artifactStore;
+        this.mediaProviders = mediaProviders;
+        this.voiceService = voiceService;
+        this.knowledgeService = knowledgeService;
+    }
+    async getStatus() {
+        const providers = await this.listProviders();
+        return {
+            enabled: providers.length > 0,
+            providerCount: providers.length,
+            providers,
+            note: 'Multimodal analysis routes images through media understanding, audio through speech-to-text, documents through TS extractors, and video through keyframe/transcript fusion. Generated markdown packets and write-back are optional.',
+        };
+    }
+    async listProviders() {
+        const media = (await this.mediaProviders.status())
+            .filter((provider) => provider.capabilities.includes('understand'))
+            .map((provider) => ({
+            id: provider.id,
+            label: provider.label,
+            transport: 'media',
+            capabilities: ['image.describe'],
+            configured: provider.configured,
+            metadata: provider.metadata,
+        }));
+        const voice = (await this.voiceService.getStatus(true)).providers
+            .filter((provider) => provider.capabilities.includes('stt'))
+            .map((provider) => ({
+            id: provider.id,
+            label: provider.label,
+            transport: provider.capabilities.includes('realtime') ? 'hybrid' : 'voice',
+            capabilities: provider.capabilities.includes('realtime')
+                ? ['audio.transcribe', 'audio.realtime']
+                : ['audio.transcribe'],
+            configured: provider.configured,
+            metadata: provider.metadata,
+        }));
+        const extractors = [
+            {
+                id: 'knowledge-extractors',
+                label: 'Built-in Knowledge Extractors',
+                transport: 'extractor',
+                capabilities: ['document.extract', 'video.keyframe-fusion'],
+                configured: true,
+                metadata: {},
+            },
+        ];
+        return [...media, ...voice, ...extractors].sort((a, b) => a.label.localeCompare(b.label) || a.id.localeCompare(b.id));
+    }
+    async analyze(request) {
+        const { descriptor, record, buffer } = await this.resolveArtifactInput(request);
+        switch (this.resolveKind(descriptor)) {
+            case 'image':
+                return this.analyzeImage(descriptor, request);
+            case 'audio':
+                return this.analyzeAudio(descriptor, buffer, request);
+            case 'video':
+                return this.analyzeVideo(descriptor, request);
+            default:
+                return this.analyzeDocument(record, buffer, request);
+        }
+    }
+    buildPacket(result, detail = 'standard', budgetLimit = PACKET_BUDGETS[detail]) {
+        const highlights = [
+            result.summary,
+            ...result.labels.slice(0, detailLimit(detail, 2, 4, 8)),
+            ...result.segments.map((segment) => compactText(segment.text, detailLimit(detail, 100, 180, 260))).filter((segment) => Boolean(segment)),
+        ].filter((entry) => Boolean(entry));
+        const lines = [
+            '## Multimodal Analysis',
+            `Kind: ${result.kind} | detail: ${detail} | artifact: ${result.artifact.id}`,
+        ];
+        if (result.summary)
+            lines.push(`Summary: ${result.summary}`);
+        if (result.labels.length > 0)
+            lines.push(`Labels: ${result.labels.join(', ')}`);
+        if (result.entities.length > 0)
+            lines.push(`Entities: ${result.entities.join(', ')}`);
+        if (result.text)
+            lines.push(`Text: ${compactText(result.text, detailLimit(detail, 180, 320, 520))}`);
+        for (const segment of result.segments.slice(0, detailLimit(detail, 2, 4, 8))) {
+            const segmentLabel = segment.title ? `${segment.kind}:${segment.title}` : segment.kind;
+            if (segment.text)
+                lines.push(`- ${segmentLabel}: ${compactText(segment.text, detailLimit(detail, 120, 220, 360))}`);
+        }
+        const rendered = lines.join('\n');
+        return {
+            detail,
+            budgetLimit,
+            estimatedTokens: estimateTokens(rendered),
+            rendered,
+            highlights: highlights.slice(0, detailLimit(detail, 4, 8, 16)),
+        };
+    }
+    async writeBackAnalysis(result, input = {}) {
+        const analysisArtifact = await this.artifactStore.create({
+            mimeType: 'application/json',
+            filename: `${result.artifact.filename ?? result.artifact.id}.analysis.json`,
+            text: `${JSON.stringify(result, null, 2)}\n`,
+            metadata: {
+                sourceArtifactId: result.artifact.id,
+                multimodalKind: result.kind,
+                providerIds: [...result.providerIds],
+                ...(input.metadata ?? {}),
+            },
+        });
+        const ingest = await this.knowledgeService.ingestArtifact({
+            artifactId: analysisArtifact.id,
+            title: input.title ?? `${result.artifact.filename ?? result.artifact.id} analysis`,
+            tags: mergeUnique(input.tags, result.labels, result.entities, [`multimodal:${result.kind}`]),
+            folderPath: input.folderPath,
+            sessionId: input.sessionId,
+            sourceType: 'document',
+            connectorId: 'multimodal-analysis',
+            metadata: {
+                sourceArtifactId: result.artifact.id,
+                multimodalKind: result.kind,
+                providerIds: [...result.providerIds],
+                ...(input.metadata ?? {}),
+            },
+        });
+        await this.knowledgeService.recordUsage({
+            targetKind: 'source',
+            targetId: ingest.source.id,
+            usageKind: 'multimodal-writeback',
+            sessionId: input.sessionId,
+            metadata: {
+                sourceArtifactId: result.artifact.id,
+                multimodalKind: result.kind,
+            },
+        });
+        return {
+            analysisArtifact,
+            knowledgeSourceId: ingest.source.id,
+            metadata: {
+                sourceArtifactId: result.artifact.id,
+                multimodalKind: result.kind,
+            },
+        };
+    }
+    async resolveArtifactInput(request) {
+        const artifactId = request.artifactId ?? request.artifact?.artifactId;
+        if (artifactId) {
+            const { record, buffer } = await this.artifactStore.readContent(artifactId);
+            return {
+                descriptor: this.artifactStore.get(artifactId),
+                record,
+                buffer,
+            };
+        }
+        const artifact = request.artifact;
+        if (!artifact)
+            throw new Error('Multimodal analysis requires artifactId or artifact input.');
+        const created = await this.artifactStore.create({
+            ...(artifact.dataBase64 ? { dataBase64: artifact.dataBase64 } : {}),
+            ...(artifact.uri ? { uri: artifact.uri, allowPrivateHosts: artifact.allowPrivateHosts } : {}),
+            ...(artifact.mimeType ? { mimeType: artifact.mimeType } : {}),
+            ...(artifact.filename ? { filename: artifact.filename } : {}),
+            metadata: artifact.metadata ?? {},
+        });
+        const { record, buffer } = await this.artifactStore.readContent(created.id);
+        return { descriptor: created, record, buffer };
+    }
+    resolveKind(descriptor) {
+        switch (descriptor.kind) {
+            case 'image':
+                return 'image';
+            case 'audio':
+                return 'audio';
+            case 'video':
+                return 'video';
+            default:
+                return 'document';
+        }
+    }
+    async analyzeImage(descriptor, request) {
+        const provider = this.mediaProviders.findProvider('understand', request.imageProviderId);
+        if (!provider?.analyze) {
+            throw new Error('No image-understanding provider is registered.');
+        }
+        const result = await provider.analyze({
+            artifact: {
+                artifactId: descriptor.id,
+                mimeType: descriptor.mimeType,
+                filename: descriptor.filename,
+                sizeBytes: descriptor.sizeBytes,
+                metadata: descriptor.metadata,
+            },
+            prompt: request.prompt,
+            modelId: request.modelId,
+            metadata: request.metadata,
+        });
+        const text = compactText(result.text, 2_000);
+        const summary = compactText(result.description ?? text, 320);
+        const segments = [
+            ...(text ? [{ kind: 'ocr', text, metadata: {} }] : []),
+            ...(summary ? [{ kind: 'summary', text: summary, metadata: {} }] : []),
+        ];
+        return {
+            id: `mm-${randomUUID().slice(0, 8)}`,
+            kind: 'image',
+            artifact: descriptor,
+            providerIds: [result.providerId],
+            summary,
+            ...(text ? { text } : {}),
+            labels: mergeUnique(result.labels),
+            entities: topKeywords([summary ?? '', text ?? '', ...result.labels ?? []].join(' '), 8),
+            segments,
+            metadata: result.metadata,
+        };
+    }
+    async analyzeAudio(descriptor, buffer, request) {
+        const transcript = await this.voiceService.transcribe(request.audioProviderId, {
+            audio: {
+                mimeType: descriptor.mimeType,
+                format: audioFormatFromMimeType(descriptor.mimeType),
+                dataBase64: buffer.toString('base64'),
+                metadata: descriptor.metadata,
+            },
+            language: request.language,
+            modelId: request.modelId,
+            prompt: request.prompt,
+            metadata: request.metadata,
+        });
+        const summary = compactText(transcript.text, 320);
+        const text = compactText(transcript.text, 4_000);
+        const segments = (transcript.segments ?? []).map((segment) => ({
+            kind: 'transcript',
+            text: segment.text,
+            startMs: segment.startMs,
+            endMs: segment.endMs,
+            confidence: segment.confidence,
+            metadata: {},
+        }));
+        return {
+            id: `mm-${randomUUID().slice(0, 8)}`,
+            kind: 'audio',
+            artifact: descriptor,
+            providerIds: [transcript.providerId],
+            summary,
+            ...(text ? { text } : {}),
+            labels: request.language ? [request.language] : transcript.language ? [transcript.language] : [],
+            entities: topKeywords(transcript.text, 10),
+            segments,
+            metadata: transcript.metadata,
+        };
+    }
+    async analyzeDocument(record, buffer, _request) {
+        const extraction = await extractKnowledgeArtifact(record, buffer);
+        const segments = extraction.sections.slice(0, 12).map((section) => ({
+            kind: 'section',
+            title: section,
+            text: section,
+            metadata: {},
+        }));
+        const text = compactText([extraction.summary ?? '', extraction.excerpt ?? ''].join('\n\n'), 4_000);
+        return {
+            id: `mm-${randomUUID().slice(0, 8)}`,
+            kind: 'document',
+            artifact: this.artifactStore.get(record.id),
+            providerIds: ['knowledge-extractors'],
+            summary: compactText(extraction.summary ?? extraction.excerpt, 320),
+            ...(text ? { text } : {}),
+            labels: mergeUnique(extraction.sections.slice(0, 8)),
+            entities: mergeUnique(topKeywords([extraction.title ?? '', extraction.summary ?? '', extraction.excerpt ?? ''].join(' '), 10)),
+            segments,
+            metadata: {
+                extractorId: extraction.extractorId,
+                format: extraction.format,
+                structure: extraction.structure,
+                extractionMetadata: extraction.metadata,
+            },
+        };
+    }
+    async analyzeVideo(descriptor, request) {
+        const requestMetadata = request.metadata ?? {};
+        const keyframeIds = coerceStringList(requestMetadata.keyframeArtifactIds ?? descriptor.metadata.keyframeArtifactIds);
+        const transcriptArtifactId = typeof requestMetadata.transcriptArtifactId === 'string'
+            ? requestMetadata.transcriptArtifactId
+            : typeof descriptor.metadata.transcriptArtifactId === 'string'
+                ? descriptor.metadata.transcriptArtifactId
+                : undefined;
+        const audioArtifactId = typeof requestMetadata.audioArtifactId === 'string'
+            ? requestMetadata.audioArtifactId
+            : typeof descriptor.metadata.audioArtifactId === 'string'
+                ? descriptor.metadata.audioArtifactId
+                : undefined;
+        const sceneSegments = [];
+        const labels = [];
+        const providerIds = new Set();
+        for (const [index, keyframeId] of keyframeIds.slice(0, 6).entries()) {
+            const frameDescriptor = this.artifactStore.get(keyframeId);
+            if (!frameDescriptor)
+                continue;
+            const frameAnalysis = await this.analyzeImage(frameDescriptor, {
+                imageProviderId: request.imageProviderId,
+                modelId: request.modelId,
+                prompt: request.prompt ? `${request.prompt}\nFocus on this video keyframe.` : 'Describe this video keyframe.',
+                metadata: request.metadata,
+            });
+            for (const providerId of frameAnalysis.providerIds)
+                providerIds.add(providerId);
+            labels.push(...frameAnalysis.labels);
+            sceneSegments.push({
+                kind: 'scene',
+                title: `Scene ${index + 1}`,
+                text: frameAnalysis.summary ?? frameAnalysis.text,
+                metadata: {
+                    frameArtifactId: frameDescriptor.id,
+                },
+            });
+        }
+        if (audioArtifactId || transcriptArtifactId) {
+            const audioDescriptor = this.artifactStore.get(audioArtifactId ?? transcriptArtifactId);
+            if (audioDescriptor) {
+                const audioResult = await this.analyzeAudio(audioDescriptor, (await this.artifactStore.readContent(audioArtifactId ?? transcriptArtifactId)).buffer, {
+                    audioProviderId: request.audioProviderId,
+                    modelId: request.modelId,
+                    prompt: request.prompt,
+                    language: request.language,
+                    metadata: request.metadata,
+                });
+                for (const providerId of audioResult.providerIds)
+                    providerIds.add(providerId);
+                labels.push(...audioResult.labels);
+                sceneSegments.push(...audioResult.segments.slice(0, 6));
+            }
+        }
+        const summary = compactText([
+            sceneSegments[0]?.text,
+            sceneSegments[1]?.text,
+            sceneSegments.find((segment) => segment.kind === 'transcript')?.text,
+        ].filter(Boolean).join(' '), 320) ?? 'Video metadata is available but no keyframes or transcripts were provided for deeper analysis.';
+        return {
+            id: `mm-${randomUUID().slice(0, 8)}`,
+            kind: 'video',
+            artifact: descriptor,
+            providerIds: [...providerIds].length > 0 ? [...providerIds] : ['knowledge-extractors'],
+            summary,
+            text: compactText(sceneSegments.map((segment) => segment.text).filter(Boolean).join('\n\n'), 4_000),
+            labels: mergeUnique(labels),
+            entities: topKeywords(sceneSegments.map((segment) => segment.text).filter(Boolean).join(' '), 10),
+            segments: sceneSegments,
+            metadata: {
+                keyframeArtifactIds: keyframeIds,
+                transcriptArtifactId,
+                audioArtifactId,
+            },
+        };
+    }
+}

package/dist/_internal/platform/panels/panel-manager.d.ts

@@ -0,0 +1,89 @@
+import type { Panel, PanelRegistration, PanelCategory } from './types.js';
+export interface Pane {
+    panels: Panel[];
+    activeIndex: number;
+}
+export interface WorkspaceTab {
+    readonly id: string;
+    readonly name: string;
+    readonly icon: string;
+    readonly pane: 'top' | 'bottom';
+    readonly active: boolean;
+    readonly focused: boolean;
+}
+export declare class PanelManager {
+    private registry;
+    private retainedPanels;
+    private _visible;
+    private _splitRatio;
+    private topPane;
+    private bottomPane;
+    private _focusedPane;
+    private _verticalSplitRatio;
+    private _bottomPaneVisible;
+    registerType(registration: PanelRegistration): void;
+    getRegisteredTypes(): PanelRegistration[];
+    getTypesByCategory(): Map<PanelCategory, PanelRegistration[]>;
+    prewarmRegistered(): void;
+    open(panelId: string, pane?: 'top' | 'bottom'): Panel;
+    close(panelId: string): void;
+    /**
+     * Move a panel to a specific pane. If panelId is omitted, moves the active
+     * panel from the currently focused pane.
+     */
+    moveToPane(dest: 'top' | 'bottom', panelId?: string): void;
+    /**
+     * Move a panel to the other pane. If panelId is omitted, moves the active
+     * panel from the currently focused pane.
+     */
+    moveToOtherPane(panelId?: string): void;
+    nextPanel(): void;
+    nextWorkspaceTab(): void;
+    prevWorkspaceTab(): void;
+    prevPanel(): void;
+    activateByIndex(index: number): void;
+    activateById(panelId: string): void;
+    focusPane(pane: 'top' | 'bottom'): void;
+    getFocusedPane(): 'top' | 'bottom';
+    /** Get the currently active (focused) panel, or null if none. */
+    getActivePanel(): Panel | null;
+    togglePaneFocus(): void;
+    toggleBottomPane(): void;
+    isBottomPaneVisible(): boolean;
+    getTopPane(): Readonly<Pane>;
+    getBottomPane(): Readonly<Pane>;
+    getOpen(): Panel[];
+    /**
+     * Returns all panels across both panes (top then bottom).
+     * Use this when you need to know if any panels exist at all.
+     */
+    getAllOpen(): Panel[];
+    getActive(): Panel | null;
+    getPanel(panelId: string): Panel | null;
+    getPaneOf(panelId: string): 'top' | 'bottom' | null;
+    getWorkspaceTabs(): WorkspaceTab[];
+    activateWorkspaceIndex(index: number): void;
+    toggle(): void;
+    show(): void;
+    hide(): void;
+    isVisible(): boolean;
+    getSplitRatio(): number;
+    setSplitRatio(ratio: number): void;
+    widenLeft(): void;
+    widenRight(): void;
+    getLeftWidth(totalWidth: number): number;
+    getRightWidth(totalWidth: number): number;
+    getVerticalSplitRatio(): number;
+    setVerticalSplitRatio(ratio: number): void;
+    destroyAll(): void;
+    private _getPane;
+    private _getFocusedPane;
+    private _findPaneOf;
+    private _moveBetweenPanes;
+    private _cycleWorkspaceTab;
+    private _obtainPanel;
+    private _getRegistration;
+    private _shouldRetain;
+    private _activateByIdInPane;
+}
+//# sourceMappingURL=panel-manager.d.ts.map

package/dist/_internal/platform/panels/panel-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"panel-manager.d.ts","sourceRoot":"","sources":["../../../../src/_internal/platform/panels/panel-manager.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAM1E,MAAM,WAAW,IAAI;IACnB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,KAAK,GAAG,QAAQ,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAMD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAA2B;IAC3C,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,WAAW,CAAe;IAGlC,OAAO,CAAC,OAAO,CAAwC;IACvD,OAAO,CAAC,UAAU,CAAwC;IAC1D,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,mBAAmB,CAAe;IAC1C,OAAO,CAAC,kBAAkB,CAAkB;IAM5C,YAAY,CAAC,YAAY,EAAE,iBAAiB,GAAG,IAAI;IASnD,kBAAkB,IAAI,iBAAiB,EAAE;IAIzC,kBAAkB,IAAI,GAAG,CAAC,aAAa,EAAE,iBAAiB,EAAE,CAAC;IAU7D,iBAAiB,IAAI,IAAI;IAazB,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,KAAK;IA+BrD,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAwC5B;;;OAGG;IACH,UAAU,CAAC,IAAI,EAAE,KAAK,GAAG,QAAQ,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;IAS1D;;;OAGG;IACH,eAAe,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;IAYvC,SAAS,IAAI,IAAI;IAUjB,gBAAgB,IAAI,IAAI;IAIxB,gBAAgB,IAAI,IAAI;IAIxB,SAAS,IAAI,IAAI;IAUjB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAWpC,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAUnC,SAAS,CAAC,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,IAAI;IAKvC,cAAc,IAAI,KAAK,GAAG,QAAQ;IAIlC,iEAAiE;IACjE,cAAc,IAAI,KAAK,GAAG,IAAI;IAK9B,eAAe,IAAI,IAAI;IASvB,gBAAgB,IAAI,IAAI;IA4BxB,mBAAmB,IAAI,OAAO;IAQ9B,UAAU,IAAI,QAAQ,CAAC,IAAI,CAAC;IAI5B,aAAa,IAAI,QAAQ,CAAC,IAAI,CAAC;IAQ/B,OAAO,IAAI,KAAK,EAAE;IAKlB;;;OAGG;IACH,UAAU,IAAI,KAAK,EAAE;IAIrB,SAAS,IAAI,KAAK,GAAG,IAAI;IAMzB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,GAAG,IAAI;IAMvC,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,IAAI;IAInD,gBAAgB,IAAI,YAAY,EAAE;IAqBlC,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAa3C,MAAM,IAAI,IAAI;IASd,IAAI,IAAI,IAAI;IAIZ,IAAI,IAAI,IAAI;IAIZ,SAAS,IAAI,OAAO;IAQpB,aAAa,IAAI,MAAM;IAIvB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAIlC,SAAS,IAAI,IAAI;IAIjB,UAAU,IAAI,IAAI;IAIlB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAIxC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAQzC,qBAAqB,IAAI,MAAM;IAI/B,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAQ1C,UAAU,IAAI,IAAI;IAiBlB,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,WAAW;IAMnB,OAAO,CAAC,iBAAiB;IAkCzB,OAAO,CAAC,kBAAkB;IAU1B,OAAO,CAAC,YAAY;IAapB,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,mBAAmB;CAW5B"}