@decido/shell 1.0.0

package/src/utils/ai/ai-media.ts

@@ -0,0 +1,292 @@
+// ═══════════════════════════════════════════════════════
+//  AI Media — Video, Audio, Image Generation
+// ═══════════════════════════════════════════════════════
+
+import { Type, Modality } from "@google/genai";
+import { getClient, getGlobalSettings, logger } from './ai-core';
+import { ScriptScene } from '@decido/canvas-core';
+
+// --- WAV HELPERS ---
+
+function addWavHeader(samples: Uint8Array, sampleRate: number = 24000, numChannels: number = 1): ArrayBuffer {
+    const buffer = new ArrayBuffer(44 + samples.length);
+    const view = new DataView(buffer);
+
+    writeString(view, 0, 'RIFF');
+    view.setUint32(4, 36 + samples.length, true);
+    writeString(view, 8, 'WAVE');
+    writeString(view, 12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, numChannels, true);
+    view.setUint32(24, sampleRate, true);
+    view.setUint32(28, sampleRate * numChannels * 2, true);
+    view.setUint16(32, numChannels * 2, true);
+    view.setUint16(34, 16, true);
+    writeString(view, 36, 'data');
+    view.setUint32(40, samples.length, true);
+
+    const dataView = new Uint8Array(buffer, 44);
+    dataView.set(samples);
+
+    return buffer;
+}
+
+function writeString(view: DataView, offset: number, string: string) {
+    for (let i = 0; i < string.length; i++) {
+        view.setUint8(offset + i, string.charCodeAt(i));
+    }
+}
+
+export function base64ToUint8Array(base64: string): Uint8Array {
+    const binaryString = atob(base64);
+    const len = binaryString.length;
+    const bytes = new Uint8Array(len);
+    for (let i = 0; i < len; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes;
+}
+
+export function arrayBufferToBase64(buffer: ArrayBuffer): string {
+    let binary = '';
+    const bytes = new Uint8Array(buffer);
+    const len = bytes.byteLength;
+    for (let i = 0; i < len; i++) {
+        binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
+}
+
+export const blobToBase64 = (blob: Blob): Promise<string> => {
+    return new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onloadend = () => {
+            const base64String = reader.result as string;
+            const base64Data = base64String.split(',')[1];
+            resolve(base64Data);
+        };
+        reader.onerror = reject;
+        reader.readAsDataURL(blob);
+    });
+};
+
+
+// --- VIDEO SCRIPT ---
+
+export const generateVideoScript = async (concept: string, style: string): Promise<ScriptScene[]> => {
+    const client = getClient('text');
+    if (!client) return [];
+    const settings = getGlobalSettings();
+
+    try {
+        const response = await client.models.generateContent({
+            model: settings.text.model || 'gemini-3-flash-preview',
+            contents: `Write a 4-scene video script for: "${concept}". Style: ${style}. 
+            
+            IMPORTANT RULES:
+            1. Detect the language of the concept (e.g., Spanish, French, English).
+            2. Write the 'narration' and 'visualPrompt' in that SAME language.
+            3. Return a JSON array.`,
+            config: {
+                responseMimeType: "application/json",
+                responseSchema: {
+                    type: Type.ARRAY,
+                    items: {
+                        type: Type.OBJECT,
+                        properties: {
+                            sceneNumber: { type: Type.INTEGER },
+                            narration: { type: Type.STRING },
+                            visualPrompt: { type: Type.STRING },
+                            cameraAngle: { type: Type.STRING },
+                            duration: { type: Type.NUMBER }
+                        },
+                        required: ["sceneNumber", "narration", "visualPrompt", "duration"]
+                    }
+                }
+            }
+        });
+
+        const text = response.text;
+        if (!text) return [];
+        const rawScenes = JSON.parse(text);
+
+        return rawScenes.map((s: any) => ({
+            ...s,
+            id: `scene-${Date.now()}-${Math.random()}`,
+            status: 'idle'
+        }));
+
+    } catch (error) {
+        logger.error("Script Gen Error:", error);
+        return [];
+    }
+};
+
+// --- IMAGE GENERATION ---
+
+export const generateKeyframe = async (prompt: string, aspectRatio: '16:9' | '9:16' | '1:1' = '16:9'): Promise<{ url: string | null; error?: string }> => {
+    const client = getClient('image');
+    if (!client) return { url: null, error: "API Key Config Missing" };
+    const settings = getGlobalSettings();
+    const selectedModel = settings.image.model || 'gemini-3-pro-image-preview';
+
+    const attemptGeneration = async (model: string, config: any) => {
+        const response = await client.models.generateContent({
+            model: model,
+            contents: { parts: [{ text: prompt }] },
+            config: config
+        });
+        for (const part of response.candidates?.[0]?.content?.parts || []) {
+            if (part.inlineData) {
+                return `data:image/png;base64,${part.inlineData.data}`;
+            }
+        }
+        return null;
+    };
+
+    try {
+        const url = await attemptGeneration(selectedModel, {
+            imageConfig: {
+                aspectRatio: aspectRatio,
+                imageSize: selectedModel.includes('pro') ? '1K' : undefined
+            }
+        });
+        return { url };
+
+    } catch (error: any) {
+        logger.warn(`${selectedModel} failed, checking fallback...`, error);
+
+        if (selectedModel.includes('pro') && (error.status === 429 || (error.message && error.message.includes('429')))) {
+            try {
+                const url = await attemptGeneration('gemini-2.5-flash-image', {
+                    imageConfig: { aspectRatio: aspectRatio }
+                });
+                if (url) return { url };
+            } catch (fallbackError: any) {
+                return { url: null, error: "Quota Exceeded (All Models). Try again later." };
+            }
+        }
+
+        return { url: null, error: error.message || "Image Generation Failed" };
+    }
+};
+
+// --- VIDEO GENERATION (VEO) ---
+
+export const generateVeoVideo = async (imageBase64: string, prompt: string, aspectRatio: '16:9' | '9:16' = '16:9'): Promise<{ url: string | null; error?: string }> => {
+    const client = getClient('video');
+    if (!client) return { url: null, error: "API Key Config Missing" };
+    const settings = getGlobalSettings();
+
+    // Check implementation specific window object if needed, skipping for shared package
+
+    try {
+        const base64Data = imageBase64.split(',')[1] || imageBase64;
+
+        let operation = await client.models.generateVideos({
+            model: settings.video.model || 'veo-3.1-generate-preview',
+            prompt: prompt,
+            image: {
+                imageBytes: base64Data,
+                mimeType: 'image/png'
+            },
+            config: {
+                numberOfVideos: 1,
+                resolution: '720p',
+                aspectRatio: aspectRatio
+            }
+        });
+
+        while (!operation.done) {
+            await new Promise(resolve => setTimeout(resolve, 5000));
+            operation = await client.operations.getVideosOperation({ operation: operation });
+        }
+
+        const videoUri = operation.response?.generatedVideos?.[0]?.video?.uri;
+
+        if (videoUri) {
+            const keyToUse = settings.video.apiKey || process.env.VITE_API_KEY;
+            return { url: `${videoUri}&key=${keyToUse}` };
+        }
+        return { url: null, error: "Video generation completed but returned no URI." };
+
+    } catch (error: any) {
+        logger.error("Veo Gen Error:", error);
+        return { url: null, error: error.message || "Video Generation Failed (Quota or Server Error)" };
+    }
+};
+
+// --- SPEECH GENERATION ---
+
+export const generateSpeech = async (text: string, voiceName: string = 'Kore'): Promise<string | null> => {
+    const client = getClient('audio');
+    if (!client) return null;
+    const settings = getGlobalSettings();
+
+    try {
+        const response = await client.models.generateContent({
+            model: settings.audio.model || "gemini-2.5-flash-preview-tts",
+            contents: [{ parts: [{ text }] }],
+            config: {
+                responseModalities: [Modality.AUDIO],
+                speechConfig: {
+                    voiceConfig: {
+                        prebuiltVoiceConfig: { voiceName }
+                    },
+                },
+            },
+        });
+
+        const base64PCM = response.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
+
+        if (base64PCM) {
+            const pcmData = base64ToUint8Array(base64PCM);
+            const wavBuffer = addWavHeader(pcmData, 24000, 1);
+            const wavBase64 = arrayBufferToBase64(wavBuffer);
+            return `data:audio/wav;base64,${wavBase64}`;
+        }
+        return null;
+    } catch (error) {
+        logger.error("TTS Error:", error);
+        return null;
+    }
+};
+
+export const analyzeAudio = async (audioBlob: Blob, task: 'TRANSCRIPTION' | 'SUMMARY'): Promise<string> => {
+    const client = getClient('text');
+    if (!client) return "API Key Missing";
+    const settings = getGlobalSettings();
+
+    try {
+        const base64Audio = await blobToBase64(audioBlob);
+
+        let prompt = "";
+        if (task === 'TRANSCRIPTION') {
+            prompt = "Transcribe the following audio accurately. Return only the text. Detect the language automatically.";
+        } else {
+            prompt = "Provide a concise summary of the key points discussed in this audio. Respond in the same language as the audio.";
+        }
+
+        const response = await client.models.generateContent({
+            model: settings.text.model || 'gemini-3-flash-preview',
+            contents: {
+                parts: [
+                    {
+                        inlineData: {
+                            mimeType: audioBlob.type,
+                            data: base64Audio
+                        }
+                    },
+                    { text: prompt }
+                ]
+            }
+        });
+
+        return response.text || "No result generated.";
+
+    } catch (error) {
+        logger.error("AI Audio Error:", error);
+        return "Failed to process audio.";
+    }
+};

package/src/utils/layoutGraph.ts

@@ -0,0 +1,67 @@
+import Dagre from '@dagrejs/dagre';
+import type { Node, Edge } from '@xyflow/react';
+
+interface LayoutOptions {
+    direction?: 'TB' | 'LR' | 'BT' | 'RL'; // Top-Bottom, Left-Right, etc.
+    nodeWidth?: number;
+    nodeHeight?: number;
+    rankSep?: number;  // Vertical gap between ranks
+    nodeSep?: number;  // Horizontal gap between nodes in same rank
+}
+
+/**
+ * Uses Dagre to compute a hierarchical auto-layout for React Flow nodes.
+ * Returns new nodes with updated positions. Does NOT mutate input.
+ */
+export function getLayoutedElements(
+    nodes: Node[],
+    edges: Edge[],
+    options: LayoutOptions = {}
+): { nodes: Node[]; edges: Edge[] } {
+    const {
+        direction = 'TB',
+        nodeWidth = 260,
+        nodeHeight = 180,
+        rankSep = 100,
+        nodeSep = 80,
+    } = options;
+
+    const g = new Dagre.graphlib.Graph().setDefaultEdgeLabel(() => ({}));
+
+    g.setGraph({
+        rankdir: direction,
+        ranksep: rankSep,
+        nodesep: nodeSep,
+        marginx: 50,
+        marginy: 50,
+    });
+
+    nodes.forEach((node) => {
+        // Use actual node dimensions if available, fallback to defaults
+        const w = (node.measured?.width) || nodeWidth;
+        const h = (node.measured?.height) || nodeHeight;
+        g.setNode(node.id, { width: w, height: h });
+    });
+
+    edges.forEach((edge) => {
+        g.setEdge(edge.source, edge.target);
+    });
+
+    Dagre.layout(g);
+
+    const layoutedNodes = nodes.map((node) => {
+        const dagNode = g.node(node.id);
+        const w = (node.measured?.width) || nodeWidth;
+        const h = (node.measured?.height) || nodeHeight;
+
+        return {
+            ...node,
+            position: {
+                x: dagNode.x - w / 2,
+                y: dagNode.y - h / 2,
+            },
+        };
+    });
+
+    return { nodes: layoutedNodes, edges };
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "compilerOptions": {
+        "strict": true,
+        "baseUrl": ".",
+        "paths": {
+            "~/*": ["./src/*"],
+            "@decido/contracts": ["../contracts/src"],
+            "@decido/plugin-engine": ["../plugin-engine/src"],
+            "@decido/shell-auth": ["../shell-auth/src"],
+            "@decido/shell-telemetry": ["../shell-telemetry/src"]
+        }
+    },
+    "include": [
+        "src"
+    ]
+}

package/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"root":["./src/agentplayer.tsx","./src/decidoplayer.tsx","./src/index.ts","./src/bridge/bridgeagent.ts","./src/components/decidoicon.tsx","./src/components/jsontreeeditor.tsx","./src/components/panelsplitter.tsx","./src/components/pluginerrorboundary.tsx","./src/components/safeliquidui.tsx","./src/components/transientlayer.tsx","./src/components/agent/agentchat.tsx","./src/components/chat-extensions/intentcatalogpanel.tsx","./src/components/chat-extensions/chatslashcommands.ts","./src/components/controls/creatorinputbar.tsx","./src/components/controls/ostoolbar.tsx","./src/components/controls/timelinetape.tsx","./src/components/debug/actiontimelinetab.tsx","./src/components/debug/cssinspectortab.tsx","./src/components/debug/exporttab.tsx","./src/components/debug/flowhealthtab.tsx","./src/components/debug/logstab.tsx","./src/components/debug/morphstacktab.tsx","./src/components/debug/networktab.tsx","./src/components/debug/performancetab.tsx","./src/components/debug/profilestab.tsx","./src/components/debug/replaytab.tsx","./src/components/debug/storestab.tsx","./src/components/debug/topologytab.tsx","./src/components/debug/debugconfig.tsx","./src/components/playground/debugpanel.tsx","./src/components/playground/headercentercontrols.tsx","./src/components/playground/keyframelistitem.tsx","./src/components/playground/playgroundappsidebar.tsx","./src/components/playground/playgroundbottomcontrols.tsx","./src/components/playground/playgroundcanvas.tsx","./src/components/playground/playgroundchat.tsx","./src/components/playground/playgrounderrorboundary.tsx","./src/components/playground/playgroundfloatinginput.tsx","./src/components/playground/playgroundheader.tsx","./src/components/playground/playgroundsidebar.tsx","./src/components/playground/playgroundterminal.tsx","./src/components/playground/suggestioncards.tsx","./src/components/playground/sidebardsdio.ts","./src/components/playground/demos/clinicaainode.tsx","./src/components/playground/demos/financeainode.tsx","./src/components/playground/demos/kiaacademynode.tsx","./src/components/playground/demos/kiabotnode.tsx","./src/components/playground/demos/kiacampaignnode.tsx","./src/components/playground/demos/kiacompliancenode.tsx","./src/components/playground/demos/kiacustomerjourneynode.tsx","./src/components/playground/demos/kiacybernode.tsx","./src/components/playground/demos/kiadashboardnode.tsx","./src/components/playground/demos/kiaembudooverviewnode.tsx","./src/components/playground/demos/kiaexecutivenode.tsx","./src/components/playground/demos/kiagamificationnode.tsx","./src/components/playground/demos/kiaintelligencehubnode.tsx","./src/components/playground/demos/kiainventorynode.tsx","./src/components/playground/demos/kialeadscoringnode.tsx","./src/components/playground/demos/kialivesimulationnode.tsx","./src/components/playground/demos/kiamultidealernode.tsx","./src/components/playground/demos/kianpsvoicenode.tsx","./src/components/playground/demos/kiaomnichannelnode.tsx","./src/components/playground/demos/kiapbibudgetnode.tsx","./src/components/playground/demos/kiapbiconversionnode.tsx","./src/components/playground/demos/kiapbifunnelnode.tsx","./src/components/playground/demos/kiapbiownershipnode.tsx","./src/components/playground/demos/kiapbipartnernode.tsx","./src/components/playground/demos/kiapbipreciosnode.tsx","./src/components/playground/demos/kiapbiruntnode.tsx","./src/components/playground/demos/kiapartnerscorenode.tsx","./src/components/playground/demos/kiapredictivenode.tsx","./src/components/playground/demos/kiashowroomnode.tsx","./src/components/playground/demos/kiastorenode.tsx","./src/components/playground/demos/kiasustainabilitynode.tsx","./src/components/playground/demos/kiausedvehiclesnode.tsx","./src/components/playground/demos/kiaworkshopnode.tsx","./src/components/playground/demos/smartcitynode.tsx","./src/components/playground/input-parts/aimodelselector.tsx","./src/components/playground/input-parts/inputactions.tsx","./src/components/playground/input-parts/inputtoolbar.tsx","./src/components/playground/input-parts/resourcelibrarypanel.tsx","./src/components/settings/settingspanel.tsx","./src/components/shell/appheader.tsx","./src/components/shell/appshell.tsx","./src/components/shell/artifactbar.tsx","./src/components/shell/bootscreen.tsx","./src/components/shell/centercomposite.tsx","./src/components/shell/codeeditorpanel.tsx","./src/components/shell/globaloverlays.tsx","./src/components/shell/layoutconfigurator.tsx","./src/components/shell/layoutgrid.tsx","./src/components/shell/morphshell.tsx","./src/components/shell/pluginviewer.tsx","./src/components/shell/shellnexuspreview.tsx","./src/components/shell/slotrenderer.tsx","./src/components/shell/tabbar.tsx","./src/components/shell/templatelibrary.tsx","./src/components/shell/layoutconstants.ts","./src/components/shell/morphstagemeta.ts","./src/components/shell/shells/builtinshells.tsx","./src/components/shell/shells/datawaychatshell.tsx","./src/components/shell/shells/tokenpreview.tsx","./src/components/shell/shells/bootshells.ts","./src/components/shells/creatorshell.tsx","./src/components/shells/decidoshell.tsx","./src/components/shells/experimentalchatshell.tsx","./src/components/shells/usercanvas.tsx","./src/components/studio/blueprintmanagerpanel.tsx","./src/components/studio/dependencytreepanel.tsx","./src/components/studio/nodepalette.tsx","./src/components/studio/nodepropertiespanel.tsx","./src/components/studio/reactfloweditor.tsx","./src/components/studio/timelineeditor.tsx","./src/components/studio/timelinekeyframecard.tsx","./src/components/studio/variablepanel.tsx","./src/components/studio/blueprint/blueprintcard.tsx","./src/components/studio/editor/canvascontextmenu.tsx","./src/components/studio/editor/editortoolbar.tsx","./src/components/studio/editor/stagecontentrenderer.tsx","./src/components/studio/editor/trackpropertyeditors.tsx","./src/components/studio/editor/treenodeitem.tsx","./src/components/studio/editor/edgestyles.ts","./src/components/studio/editor/editorkeyhandler.ts","./src/components/studio/editor/nodetyperegistry.ts","./src/components/studio/editor/palettecatalog.tsx","./src/components/studio/nodes/shell/interactionnodes.tsx","./src/components/studio/nodes/shell/layoutcontrolnodes.tsx","./src/components/studio/nodes/shell/registeractionnode.tsx","./src/components/studio/nodes/shell/registerbuttonnode.tsx","./src/components/studio/nodes/shell/registerpanelnode.tsx","./src/components/studio/nodes/shell/registersidebarnode.tsx","./src/components/studio/nodes/shell/registerstatusbarnode.tsx","./src/components/studio/nodes/shell/registertabnode.tsx","./src/components/studio/nodes/shell/registertopbarnode.tsx","./src/components/studio/nodes/shell/shellconfignode.tsx","./src/components/studio/nodes/shell/shellnodebase.tsx","./src/components/studio/nodes/shell/themenodes.tsx","./src/components/studio/nodes/shell/index.ts","./src/components/widgets/broadcastwidget.tsx","./src/components/widgets/marketplacewidget.tsx","./src/components/widgets/mcptoolswidget.tsx","./src/components/widgets/opsdashboard.tsx","./src/components/widgets/quickactionswidget.tsx","./src/components/widgets/usagewidget.tsx","./src/components/widgets/widgetrenderer.tsx","./src/components/widgets/widgetslotpanel.tsx","./src/config/iconregistry.ts","./src/contexts/networkprovider.tsx","./src/core/aidirector.ts","./src/core/eventbus.ts","./src/core/plugincontext.tsx","./src/hooks/morphbridge.ts","./src/hooks/useaimodelselector.ts","./src/hooks/useagentstream.ts","./src/hooks/useautoupdater.ts","./src/hooks/usebootsequence.ts","./src/hooks/useexportdsd.ts","./src/hooks/usefullscreen.ts","./src/hooks/usegeministream.ts","./src/hooks/useintentlens.ts","./src/hooks/usekeyboardshortcuts.ts","./src/hooks/useloggerbridge.ts","./src/hooks/usemcpclient.ts","./src/hooks/usenexusaideploy.ts","./src/hooks/useplaybackengine.ts","./src/hooks/useplaygroundcommander.ts","./src/hooks/usepluginengine.ts","./src/hooks/usescreenrecorder.ts","./src/hooks/useshellkeyboard.ts","./src/hooks/useshellshortcuts.ts","./src/hooks/usesoundeffects.ts","./src/hooks/usestudioconfig.ts","./src/hooks/usesystemboot.ts","./src/hooks/usesystemtelemetry.ts","./src/hooks/listeners/useuistatelistener.ts","./src/hooks/listeners/usewhatsapplistener.ts","./src/lib/debuglogger.ts","./src/lib/networkinterceptor.ts","./src/mocks/decido.tsx","./src/plugins/pluginapi.ts","./src/store/mcpstore.ts","./src/store/updaterstore.ts","./src/store/engine.ts","./src/store/index.ts","./src/store/layoutpresets.ts","./src/store/playgroundtypes.ts","./src/store/useactiontimelinestore.ts","./src/store/usedebugpanelstore.ts","./src/store/usedebugprofilestore.ts","./src/store/uselayoutstore.ts","./src/store/usemorphinstancestore.ts","./src/store/usemorphologystore.ts","./src/store/useplaygroundstore.ts","./src/store/useshellregistry.ts","./src/store/usesuggestionsstore.ts","./src/store/usethemestore.ts","./src/store/useuicomponentstore.ts","./src/types/decidostorydefinition.ts","./src/utils/layoutgraph.ts","./src/utils/ai/ai-architect.ts","./src/utils/ai/ai-code.ts","./src/utils/ai/ai-core.ts","./src/utils/ai/ai-media.ts"],"errors":true,"version":"5.9.3"}