@kapeta/local-cluster-service 0.65.0 → 0.67.0

package/src/storm/PageGenerator.ts

@@ -5,22 +5,27 @@
 
 import uuid from 'node-uuid';
 import { stormClient, UIPagePrompt } from './stormClient';
-import {
-    ReferenceClassification,
-    StormEvent,
-    StormEventPage,
-    StormEventReferenceClassification,
-    UIShell,
-} from './events';
+import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
-import { PromiseQueue } from './PromiseQueue';
+import { createFuture, Future, FuturePromise, PromiseQueue } from './PromiseQueue';
 import { hasPageOnDisk } from './page-utils';
 
+export interface ImagePrompt {
+    name: string;
+    description: string;
+    source: 'local' | 'cdn' | 'example';
+    title: string;
+    type: 'image' | 'css' | 'javascript' | 'html';
+    url: string;
+    content: string;
+}
+
 export class PageQueue extends EventEmitter {
     private readonly queue: PromiseQueue;
     private readonly systemId: string;
     private readonly references: Map<string, PageGenerator> = new Map();
     private uiShells: UIShell[] = [];
+    private theme = '';
 
     constructor(systemId: string, concurrency: number = 5) {
         super();
@@ -30,6 +35,7 @@ export class PageQueue extends EventEmitter {
 
     on(event: 'event', listener: (data: StormEvent) => void): this;
     on(event: 'page', listener: (data: StormEventPage) => void): this;
+    on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
 
     on(event: string, listener: (...args: any[]) => void): this {
         return super.on(event, listener);
@@ -37,6 +43,7 @@ export class PageQueue extends EventEmitter {
 
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
+    emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
     emit(eventName: string | symbol, ...args: any[]): boolean {
         return super.emit(eventName, ...args);
     }
@@ -45,24 +52,29 @@ export class PageQueue extends EventEmitter {
         this.uiShells.push(uiShell);
     }
 
+    public setUiTheme(theme: string) {
+        this.theme = theme;
+    }
+
     public addPrompt(
         initialPrompt: Omit<UIPagePrompt, 'shell_page'>,
         conversationId: string = uuid.v4(),
         overwrite: boolean = false
     ) {
         if (!overwrite && this.references.has(initialPrompt.path)) {
-            console.log('Ignoring duplicate prompt', initialPrompt.path);
+            //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
         }
 
         if (!overwrite && hasPageOnDisk(this.systemId, initialPrompt.method, initialPrompt.path)) {
-            console.log('Ignoring prompt with existing page', initialPrompt.path);
+            //console.log('Ignoring prompt with existing page', initialPrompt.path);
             return Promise.resolve();
         }
 
         const prompt: UIPagePrompt = {
             ...initialPrompt,
             shell_page: this.uiShells.find((shell) => shell.screens.includes(initialPrompt.name))?.content,
+            theme: this.theme,
         };
 
         const generator = new PageGenerator(prompt, conversationId);
@@ -73,9 +85,8 @@ export class PageQueue extends EventEmitter {
 
     private async addPageGenerator(generator: PageGenerator) {
         generator.on('event', (event: StormEvent) => this.emit('event', event));
-        generator.on('page_refs', ({ event, references }) => {
-            this.emit('page', event);
-            references.forEach((reference) => {
+        generator.on('page_refs', async ({ event, references }) => {
+            const promises = references.map(async (reference) => {
                 if (
                     reference.url.startsWith('#') ||
                     reference.url.startsWith('javascript:') ||
@@ -87,14 +98,17 @@ export class PageQueue extends EventEmitter {
 
                 switch (reference.type) {
                     case 'image':
-                        console.log('Ignoring image reference', reference);
+                        await this.addImagePrompt({
+                            ...reference,
+                            content: event.payload.content,
+                        });
                         break;
                     case 'css':
                     case 'javascript':
                         //console.log('Ignoring reference', reference);
                         break;
                     case 'html':
-                        console.log('Adding page generator for', reference);
+                        //console.log('Adding page generator for', reference);
                         const paths = Array.from(this.references.keys());
                         this.addPrompt({
                             name: reference.name,
@@ -110,10 +124,14 @@ export class PageQueue extends EventEmitter {
                                     : ''),
                             description: reference.description,
                             filename: '',
+                            theme: this.theme,
                         });
                         break;
                 }
             });
+
+            await Promise.allSettled(promises);
+            this.emit('page', event);
         });
         return this.queue.add(() => generator.generate());
     }
@@ -125,6 +143,31 @@ export class PageQueue extends EventEmitter {
     public wait() {
         return this.queue.wait();
     }
+
+    private async addImagePrompt(prompt: ImagePrompt) {
+        const result = await stormClient.createImage(
+            `Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim()
+        );
+
+        const futures: FuturePromise<void>[] = [];
+
+        result.on('data', async (event: StormEvent) => {
+            if (event.type === 'IMAGE') {
+                const future = createFuture();
+                futures.push(future);
+                this.emit('image', event, prompt, future);
+                setTimeout(() => {
+                    if (!future.isResolved()) {
+                        console.log('Image prompt timed out', prompt);
+                        future.reject(new Error('Image prompt timed out'));
+                    }
+                }, 30000);
+            }
+        });
+
+        await result.waitForDone();
+        await Promise.allSettled(futures.map((f) => f.promise));
+    }
 }
 
 export class PageGenerator extends EventEmitter {

package/src/storm/PromiseQueue.ts

@@ -4,6 +4,40 @@
  */
 export type Future<T> = () => Promise<T>;
 
+export type FuturePromise<T> = {
+    promise: Promise<T>;
+    resolve: (value: T) => void;
+    reject: (reason: any) => void;
+    isResolved: () => boolean;
+};
+
+export function createFuture<T = void>(): FuturePromise<T> {
+    let resolved = false;
+    let resolve: (value: T) => void = () => {
+        resolved = true;
+    };
+    let reject: (reason: any) => void = () => {
+        resolved = true;
+    };
+
+    const promise = new Promise<T>((res, rej) => {
+        resolve = (value: T) => {
+            resolved = true;
+            res(value);
+        };
+        reject = (reason: any) => {
+            resolved = true;
+            rej(reason);
+        };
+    });
+    return {
+        promise,
+        resolve,
+        reject,
+        isResolved: () => resolved,
+    };
+}
+
 type InternalFuture<T> = {
     execute: Future<T>;
     promise: Promise<T>;

package/src/storm/UIServer.ts

@@ -3,10 +3,11 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 import express, { Express, Request, Response } from 'express';
-import { readPageFromDisk } from './page-utils';
+import { getSystemBaseDir, readPageFromDisk } from './page-utils';
 import { clusterService } from '../clusterService';
 import { createServer, Server } from 'http';
 import { StormEventPage } from './events';
+import { join } from 'path';
 
 export class UIServer {
     private readonly systemId: string;
@@ -30,6 +31,9 @@ export class UIServer {
             );
         });
 
+        // Make it possible to serve static assets
+        app.use(express.static(join(getSystemBaseDir(this.systemId), 'public'), { fallthrough: true }));
+
         app.all('/*', (req: Request, res: Response) => {
             readPageFromDisk(this.systemId, req.params[0], req.method, res);
         });

package/src/storm/events.ts

@@ -311,6 +311,15 @@ export interface StormEventDone {
     created: number;
 }
 
+export interface StormImage {
+    type: 'IMAGE';
+    reason: string;
+    created: number;
+    payload: {
+        href: string;
+    };
+}
+
 export interface StormEventDefinitionChange {
     type: 'DEFINITION_CHANGE';
     reason: string;
@@ -494,4 +503,5 @@ export type StormEvent =
     | StormEventLandingPage
     | StormEventReferenceClassification
     | StormEventApiBase
-    | StormEventUIStarted;
+    | StormEventUIStarted
+    | StormImage;

package/src/storm/page-utils.ts

@@ -2,7 +2,8 @@
  * Copyright 2023 Kapeta Inc.
  * SPDX-License-Identifier: BUSL-1.1
  */
-import { StormEventPage } from './events';
+import { StormEventFileDone, StormEventPage, StormImage } from './events';
+
 import { Response } from 'express';
 import os from 'node:os';
 import Path from 'path';
@@ -10,6 +11,7 @@ import FS from 'fs-extra';
 import FSExtra from 'fs-extra';
 import { ConversationItem } from './stream';
 import exp from 'node:constants';
+import { ImagePrompt } from './PageGenerator';
 
 export const SystemIdHeader = 'System-Id';
 
@@ -21,7 +23,7 @@ function normalizePath(path: string) {
 }
 
 export async function writePageToDisk(systemId: string, event: StormEventPage) {
-    const baseDir = getBaseDir(systemId);
+    const baseDir = getSystemBaseDir(systemId);
     const filePath = getFilePath(event.payload.method);
     const path = Path.join(baseDir, normalizePath(event.payload.path), filePath);
     await FS.ensureDir(Path.dirname(path));
@@ -34,14 +36,46 @@ export async function writePageToDisk(systemId: string, event: StormEventPage) {
     };
 }
 
+export async function writeAssetToDisk(systemId: string, event: StormEventFileDone) {
+    const baseDir = getSystemBaseDir(systemId);
+    const path = Path.join(baseDir, 'public', event.payload.filename);
+    await FS.ensureDir(Path.dirname(path));
+    await FS.writeFile(path, event.payload.content);
+
+    return {
+        path,
+    };
+}
+
+export async function writeImageToDisk(systemId: string, event: StormImage, prompt: ImagePrompt) {
+    const baseDir = getSystemBaseDir(systemId);
+    const path = Path.join(baseDir, normalizePath(prompt.url));
+
+    const response = await fetch(event.payload.href);
+    if (!response.ok || !response.body) {
+        throw new Error(`Failed to fetch image: ${event.payload.href}`);
+    }
+
+    await FS.ensureDir(Path.dirname(path));
+
+    const buffer = await response.arrayBuffer();
+    await FS.writeFile(path, Buffer.from(buffer));
+
+    console.log(`Image written to disk: ${event.payload.href} > ${path}`);
+
+    return {
+        path,
+    };
+}
+
 export function hasPageOnDisk(systemId: string, method: string, path: string) {
-    const baseDir = getBaseDir(systemId);
+    const baseDir = getSystemBaseDir(systemId);
     const filePath = getFilePath(method);
     const fullPath = Path.join(baseDir, normalizePath(path), filePath);
     return FS.existsSync(fullPath);
 }
 
-function getBaseDir(systemId: string) {
+export function getSystemBaseDir(systemId: string) {
     return Path.join(os.tmpdir(), 'ai-systems', systemId);
 }
 
@@ -50,13 +84,24 @@ function getFilePath(method: string) {
 }
 
 export function resolveReadPath(systemId: string, path: string, method: string) {
-    const baseDir = getBaseDir(systemId);
+    const baseDir = getSystemBaseDir(systemId);
 
     path = normalizePath(path);
 
-    const filePath = getFilePath(method);
+    let fullPath = Path.join(baseDir, path);
+
+    //First check if there is a file at the exact path
+    try {
+        const stat = FS.statSync(fullPath);
+        if (stat && stat.isFile()) {
+            return fullPath;
+        }
+    } catch (e) {
+        // Ignore
+    }
 
-    const fullPath = Path.join(baseDir, path, filePath);
+    const filePath = getFilePath(method);
+    fullPath = Path.join(baseDir, path, filePath);
 
     if (FS.existsSync(fullPath)) {
         return fullPath;
@@ -108,7 +153,7 @@ export function readPageFromDisk(systemId: string, path: string, method: string,
 
     res.type(filePath.split('.').pop() as string);
 
-    const content = FS.readFileSync(filePath, 'utf8');
+    const content = FS.readFileSync(filePath);
     res.write(content);
     res.end();
 }

package/src/storm/routes.ts

@@ -23,7 +23,7 @@ import {
     UIPageVoteRequest,
     UIPageGetVoteRequest,
 } from './stormClient';
-import { Page, StormEvent, StormEventPage, StormEventPhaseType, UserJourneyScreen } from './events';
+import { Page, StormEvent, StormEventPage, StormEventPhaseType, StormImage, UserJourneyScreen } from './events';
 
 import {
     createPhaseEndEvent,
@@ -35,9 +35,17 @@ import {
 import { StormCodegen } from './codegen';
 import { assetManager } from '../assetManager';
 import uuid from 'node-uuid';
-import { readPageFromDisk, readPageFromDiskAsString, SystemIdHeader, writePageToDisk } from './page-utils';
+import {
+    readPageFromDisk,
+    readPageFromDiskAsString,
+    SystemIdHeader,
+    writeAssetToDisk,
+    writeImageToDisk,
+    writePageToDisk,
+} from './page-utils';
 import { UIServer } from './UIServer';
-import { PageQueue } from './PageGenerator';
+import { randomUUID } from 'crypto';
+import { ImagePrompt, PageQueue } from './PageGenerator';
 
 const UI_SERVERS: { [key: string]: UIServer } = {};
 const router = Router();
@@ -107,6 +115,21 @@ router.post('/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
             }
         });
 
+        queue.on('image', async (screenData, prompt, future) => {
+            if (!systemId) {
+                return;
+            }
+            try {
+                const promise = handleImageEvent(systemId, screenData, prompt);
+                promises.push(promise);
+                await promise;
+                future.resolve();
+            } catch (e) {
+                console.error('Failed to handle image event', e);
+                future.reject(e);
+            }
+        });
+
         await queue.addPrompt(aiRequest, conversationId, true);
 
         await queue.wait();
@@ -185,6 +208,8 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
                     title: landingPage.title,
                     filename: landingPage.filename,
                     storage_prefix: systemId + '_',
+                    // TODO: Add themes to this request type
+                    theme: '',
                 });
             } catch (e) {
                 console.error('Failed to process event', e);
@@ -202,6 +227,18 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
             pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
         });
 
+        pageQueue.on('image', async (screenData, prompt, future) => {
+            try {
+                const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
+                pageEventPromises.push(promise);
+                await promise;
+                future.resolve();
+            } catch (e) {
+                console.error('Failed to handle image event', e);
+                future.reject(e);
+            }
+        });
+
         pageQueue.on('event', (screenData: StormEvent) => {
             sendEvent(res, screenData);
         });
@@ -226,13 +263,13 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
 router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
     const handle = req.params.handle as string;
     try {
-        const conversationId = req.headers[ConversationIdHeader.toLowerCase()] as string | undefined;
+        const outerConversationId =
+            (req.headers[ConversationIdHeader.toLowerCase()] as string | undefined) || randomUUID();
 
         const aiRequest: BasePromptRequest = JSON.parse(req.stringBody ?? '{}');
 
         // Get user journeys
-        const userJourneysStream = await stormClient.createUIUserJourneys(aiRequest, conversationId);
-        const outerConversationId = userJourneysStream.getConversationId();
+        const userJourneysStream = await stormClient.createUIUserJourneys(aiRequest, outerConversationId);
 
         onRequestAborted(req, res, () => {
             userJourneysStream.abort();
@@ -271,11 +308,53 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
             sendError(error, res);
         });
 
+        let theme = '';
+        try {
+            const themeStream = await stormClient.createTheme(aiRequest, outerConversationId);
+            onRequestAborted(req, res, () => {
+                themeStream.abort();
+            });
+
+            themeStream.on('data', (evt) => {
+                sendEvent(res, evt);
+                if (evt.type === 'FILE_DONE') {
+                    theme = evt.payload.content;
+                    writeAssetToDisk(outerConversationId, evt).catch((err) => {
+                        sendEvent(res, {
+                            type: 'ERROR_INTERNAL',
+                            created: new Date().getTime(),
+                            payload: { error: err.message },
+                            reason: 'Failed to save theme',
+                        });
+                    });
+                }
+            });
+            themeStream.on('error', (error) => {
+                console.error(error);
+                sendEvent(res, {
+                    type: 'ERROR_INTERNAL',
+                    created: new Date().getTime(),
+                    payload: { error: error.message },
+                    reason: 'Failed to create theme',
+                });
+            });
+            await waitForStormStream(themeStream);
+        } catch (e: any) {
+            console.error('Failed to generate theme', e);
+            sendEvent(res, {
+                type: 'ERROR_INTERNAL',
+                created: new Date().getTime(),
+                payload: { error: e.message },
+                reason: 'Failed to create theme',
+            });
+        }
+
         await waitForStormStream(userJourneysStream);
 
         // Get the UI shells
         const shellsStream = await stormClient.createUIShells(
             {
+                theme: theme ? `// filename: theme.css\n${theme}` : undefined,
                 pages: Object.values(uniqueUserJourneyScreens).map((screen) => ({
                     name: screen.name,
                     title: screen.title,
@@ -285,7 +364,7 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
                     requirements: screen.requirements,
                 })),
             },
-            conversationId
+            outerConversationId
         );
 
         onRequestAborted(req, res, () => {
@@ -293,8 +372,10 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
         });
 
         const queue = new PageQueue(outerConversationId, 5);
+        queue.setUiTheme(theme);
+
         shellsStream.on('data', (data: StormEvent) => {
-            console.log('Processing shell event', data);
+            //console.log('Processing shell event', data);
             sendEvent(res, data);
 
             if (data.type !== 'UI_SHELL') {
@@ -341,6 +422,18 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
             pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
         });
 
+        queue.on('image', async (screenData, prompt, future) => {
+            try {
+                const promise = handleImageEvent(outerConversationId, screenData, prompt);
+                pageEventPromises.push(promise);
+                await promise;
+                future.resolve();
+            } catch (e) {
+                console.error('Failed to handle image event', e);
+                future.reject(e);
+            }
+        });
+
         queue.on('event', (screenData: StormEvent) => {
             sendEvent(res, screenData);
         });
@@ -356,6 +449,7 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
                     title: screen.title,
                     filename: screen.filename,
                     storage_prefix: outerConversationId + '_',
+                    theme,
                 })
             );
         }
@@ -712,4 +806,12 @@ async function sendPageEvent(mainConversationId: string, data: StormEventPage, r
     sendEvent(res, convertPageEvent(data, data.payload.conversationId, mainConversationId));
 }
 
+async function handleImageEvent(mainConversationId: string, data: StormImage, prompt: ImagePrompt) {
+    try {
+        await writeImageToDisk(mainConversationId, data, prompt);
+    } catch (err) {
+        console.error('Failed to write image to disk', err);
+    }
+}
+
 export default router;

package/src/storm/stormClient.ts

@@ -22,6 +22,7 @@ export const STORM_ID = 'storm';
 export const ConversationIdHeader = 'Conversation-Id';
 
 export interface UIShellsPrompt {
+    theme?: string;
     pages: {
         name: string;
         title: string;
@@ -42,6 +43,8 @@ export interface UIPagePrompt {
     description: string;
     storage_prefix: string;
     shell_page?: string;
+    // contents of theme.css
+    theme: string;
 }
 
 export interface UIPageSamplePrompt extends UIPagePrompt {
@@ -185,6 +188,13 @@ class StormClient {
         });
     }
 
+    public createTheme(prompt: BasePromptRequest, conversationId?: string) {
+        return this.send('/v2/ui/theme', {
+            prompt: prompt,
+            conversationId,
+        });
+    }
+
     public createUIShells(prompt: UIShellsPrompt, conversationId?: string) {
         return this.send('/v2/ui/shells', {
             prompt: JSON.stringify(prompt),
@@ -253,6 +263,13 @@ class StormClient {
         });
     }
 
+    public createImage(prompt: string, conversationId?: string) {
+        return this.send('/v2/ui/image', {
+            prompt,
+            conversationId,
+        });
+    }
+
     public createUIImplementation(prompt: StormUIImplementationPrompt, conversationId?: string) {
         return this.send('/v2/ui/merge', {
             prompt,