@kapeta/local-cluster-service 0.70.11 → 0.71.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [0.71.0](https://github.com/kapetacom/local-cluster-service/compare/v0.70.12...v0.71.0) (2024-09-16)
+
+
+### Features
+
+* Add two new phases ([d2805a8](https://github.com/kapetacom/local-cluster-service/commit/d2805a89d7e21b6f30674ae852ddc356954e7d79))
+
+## [0.70.12](https://github.com/kapetacom/local-cluster-service/compare/v0.70.11...v0.70.12) (2024-09-13)
+
+
+### Bug Fixes
+
+* wait for image resources on disk [CORE-3467] ([#249](https://github.com/kapetacom/local-cluster-service/issues/249)) ([934d9f4](https://github.com/kapetacom/local-cluster-service/commit/934d9f400ae917559a4ede08d3f3e5644b3b5053))
+
 ## [0.70.11](https://github.com/kapetacom/local-cluster-service/compare/v0.70.10...v0.70.11) (2024-09-12)
 
 

package/dist/cjs/src/storm/PageGenerator.js

@@ -242,13 +242,18 @@ class PageQueue extends node_events_1.EventEmitter {
         }
         this.images.set(prompt.url, prompt.description);
         const result = await stormClient_1.stormClient.createImage(`Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim());
-        //console.log('Adding image prompt', prompt);
+        let imageEvent = null;
         result.on('data', (event) => {
             if (event.type === 'IMAGE') {
-                this.emit('image', event, prompt);
+                imageEvent = event;
             }
         });
         await result.waitForDone();
+        if (!imageEvent) {
+            throw new Error('No image was generated');
+        }
+        await (0, page_utils_1.writeImageToDisk)(this.systemId, imageEvent, prompt);
+        this.emit('image', imageEvent, prompt);
     }
     async generate(prompt, conversationId) {
         const screenStream = await stormClient_1.stormClient.createUIPage(prompt, conversationId);

package/dist/cjs/src/storm/events.d.ts

@@ -271,6 +271,8 @@ export interface StormEventDefinitionChange {
     payload: StormDefinitions;
 }
 export declare enum StormEventPhaseType {
+    IMPLEMENT_APIS = "IMPLEMENT_APIS",// Implement APIs in the html pages
+    COMPOSE_SYSTEM_PROMPT = "COMPOSE_SYSTEM_PROMPT",// Compose system prompt for bottom-up approach
     META = "META",
     DEFINITIONS = "DEFINITIONS",
     IMPLEMENTATION = "IMPLEMENTATION",

package/dist/cjs/src/storm/events.js

@@ -11,6 +11,8 @@ var StormEventBlockStatusType;
 })(StormEventBlockStatusType || (exports.StormEventBlockStatusType = StormEventBlockStatusType = {}));
 var StormEventPhaseType;
 (function (StormEventPhaseType) {
+    StormEventPhaseType["IMPLEMENT_APIS"] = "IMPLEMENT_APIS";
+    StormEventPhaseType["COMPOSE_SYSTEM_PROMPT"] = "COMPOSE_SYSTEM_PROMPT";
     StormEventPhaseType["META"] = "META";
     StormEventPhaseType["DEFINITIONS"] = "DEFINITIONS";
     StormEventPhaseType["IMPLEMENTATION"] = "IMPLEMENTATION";

package/dist/cjs/src/storm/routes.js

@@ -72,6 +72,10 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     const srcDir = (0, page_utils_1.getSystemBaseDir)(systemId);
     const destDir = (0, page_utils_1.getSystemBaseImplDir)(systemId);
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+    res.set(stormClient_1.ConversationIdHeader, systemId);
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
     await (0, utils_1.copyDirectory)(srcDir, destDir, async (fileName, content) => {
         const result = await stormClient_1.stormClient.implementAPIClients({
             content: content,
@@ -79,8 +83,11 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
         });
         return result;
     });
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     const pages = (0, utils_1.readPages)(destDir);
     const prompt = await stormClient_1.stormClient.generatePrompt(pages);
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     req.query.systemId = systemId;
     const promptRequest = {
         prompt: prompt,
@@ -118,18 +125,6 @@ router.post('/ui/screen', async (req, res) => {
         });
         const promises = [];
         queue.on('page', (data) => (systemId ? sendPageEvent(systemId, data, res) : undefined));
-        queue.on('image', async (screenData, prompt) => {
-            if (!systemId) {
-                return;
-            }
-            try {
-                await handleImageEvent(systemId, screenData, prompt);
-            }
-            catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
         queue.on('error', (err) => {
             console.error('Failed to process page', err);
             sendError(err, res);
@@ -212,15 +207,6 @@ router.post('/:handle/ui/iterative', async (req, res) => {
             pageQueue.cancel();
         });
         pageQueue.on('page', (screenData) => sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        pageQueue.on('image', async (screenData, prompt) => {
-            try {
-                await handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
-            }
-            catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
         pageQueue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
@@ -374,15 +360,6 @@ router.post('/:handle/ui', async (req, res) => {
             queue.cancel();
         });
         queue.on('page', (screenData) => sendPageEvent(outerConversationId, screenData, res));
-        queue.on('image', async (screenData, prompt) => {
-            try {
-                await handleImageEvent(outerConversationId, screenData, prompt);
-            }
-            catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
         queue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
@@ -520,9 +497,13 @@ async function handleAll(req, res) {
         onRequestAborted(req, res, () => {
             metaStream.abort();
         });
-        res.set('Content-Type', 'application/x-ndjson');
-        res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
-        res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        // We check if the headers have been sent, because we might have already sent some data
+        // before this function is called
+        if (!res.headersSent) {
+            res.set('Content-Type', 'application/x-ndjson');
+            res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+            res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        }
         let currentPhase = events_1.StormEventPhaseType.META;
         // Helper to avoid sending the plan multiple times in a row
         const sendUpdatedPlan = lodash_1.default.debounce(sendDefinitions, 50, { maxWait: 200 });
@@ -722,12 +703,4 @@ async function sendPageEvent(mainConversationId, data, res) {
     }
     sendEvent(res, convertPageEvent(data, data.payload.conversationId, mainConversationId));
 }
-async function handleImageEvent(mainConversationId, data, prompt) {
-    try {
-        await (0, page_utils_1.writeImageToDisk)(mainConversationId, data, prompt);
-    }
-    catch (err) {
-        console.error('Failed to write image to disk', err);
-    }
-}
 exports.default = router;

package/dist/esm/src/storm/PageGenerator.js

@@ -242,13 +242,18 @@ class PageQueue extends node_events_1.EventEmitter {
         }
         this.images.set(prompt.url, prompt.description);
         const result = await stormClient_1.stormClient.createImage(`Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim());
-        //console.log('Adding image prompt', prompt);
+        let imageEvent = null;
         result.on('data', (event) => {
             if (event.type === 'IMAGE') {
-                this.emit('image', event, prompt);
+                imageEvent = event;
             }
         });
         await result.waitForDone();
+        if (!imageEvent) {
+            throw new Error('No image was generated');
+        }
+        await (0, page_utils_1.writeImageToDisk)(this.systemId, imageEvent, prompt);
+        this.emit('image', imageEvent, prompt);
     }
     async generate(prompt, conversationId) {
         const screenStream = await stormClient_1.stormClient.createUIPage(prompt, conversationId);

package/dist/esm/src/storm/events.d.ts

@@ -271,6 +271,8 @@ export interface StormEventDefinitionChange {
     payload: StormDefinitions;
 }
 export declare enum StormEventPhaseType {
+    IMPLEMENT_APIS = "IMPLEMENT_APIS",// Implement APIs in the html pages
+    COMPOSE_SYSTEM_PROMPT = "COMPOSE_SYSTEM_PROMPT",// Compose system prompt for bottom-up approach
     META = "META",
     DEFINITIONS = "DEFINITIONS",
     IMPLEMENTATION = "IMPLEMENTATION",

package/dist/esm/src/storm/events.js

@@ -11,6 +11,8 @@ var StormEventBlockStatusType;
 })(StormEventBlockStatusType || (exports.StormEventBlockStatusType = StormEventBlockStatusType = {}));
 var StormEventPhaseType;
 (function (StormEventPhaseType) {
+    StormEventPhaseType["IMPLEMENT_APIS"] = "IMPLEMENT_APIS";
+    StormEventPhaseType["COMPOSE_SYSTEM_PROMPT"] = "COMPOSE_SYSTEM_PROMPT";
     StormEventPhaseType["META"] = "META";
     StormEventPhaseType["DEFINITIONS"] = "DEFINITIONS";
     StormEventPhaseType["IMPLEMENTATION"] = "IMPLEMENTATION";

package/dist/esm/src/storm/routes.js

@@ -72,6 +72,10 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     const srcDir = (0, page_utils_1.getSystemBaseDir)(systemId);
     const destDir = (0, page_utils_1.getSystemBaseImplDir)(systemId);
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+    res.set(stormClient_1.ConversationIdHeader, systemId);
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
     await (0, utils_1.copyDirectory)(srcDir, destDir, async (fileName, content) => {
         const result = await stormClient_1.stormClient.implementAPIClients({
             content: content,
@@ -79,8 +83,11 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
         });
         return result;
     });
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     const pages = (0, utils_1.readPages)(destDir);
     const prompt = await stormClient_1.stormClient.generatePrompt(pages);
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     req.query.systemId = systemId;
     const promptRequest = {
         prompt: prompt,
@@ -118,18 +125,6 @@ router.post('/ui/screen', async (req, res) => {
         });
         const promises = [];
         queue.on('page', (data) => (systemId ? sendPageEvent(systemId, data, res) : undefined));
-        queue.on('image', async (screenData, prompt) => {
-            if (!systemId) {
-                return;
-            }
-            try {
-                await handleImageEvent(systemId, screenData, prompt);
-            }
-            catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
         queue.on('error', (err) => {
             console.error('Failed to process page', err);
             sendError(err, res);
@@ -212,15 +207,6 @@ router.post('/:handle/ui/iterative', async (req, res) => {
             pageQueue.cancel();
         });
         pageQueue.on('page', (screenData) => sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        pageQueue.on('image', async (screenData, prompt) => {
-            try {
-                await handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
-            }
-            catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
         pageQueue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
@@ -374,15 +360,6 @@ router.post('/:handle/ui', async (req, res) => {
             queue.cancel();
         });
         queue.on('page', (screenData) => sendPageEvent(outerConversationId, screenData, res));
-        queue.on('image', async (screenData, prompt) => {
-            try {
-                await handleImageEvent(outerConversationId, screenData, prompt);
-            }
-            catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
         queue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
@@ -520,9 +497,13 @@ async function handleAll(req, res) {
         onRequestAborted(req, res, () => {
             metaStream.abort();
         });
-        res.set('Content-Type', 'application/x-ndjson');
-        res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
-        res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        // We check if the headers have been sent, because we might have already sent some data
+        // before this function is called
+        if (!res.headersSent) {
+            res.set('Content-Type', 'application/x-ndjson');
+            res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+            res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        }
         let currentPhase = events_1.StormEventPhaseType.META;
         // Helper to avoid sending the plan multiple times in a row
         const sendUpdatedPlan = lodash_1.default.debounce(sendDefinitions, 50, { maxWait: 200 });
@@ -722,12 +703,4 @@ async function sendPageEvent(mainConversationId, data, res) {
     }
     sendEvent(res, convertPageEvent(data, data.payload.conversationId, mainConversationId));
 }
-async function handleImageEvent(mainConversationId, data, prompt) {
-    try {
-        await (0, page_utils_1.writeImageToDisk)(mainConversationId, data, prompt);
-    }
-    catch (err) {
-        console.error('Failed to write image to disk', err);
-    }
-}
 exports.default = router;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.70.11",
+    "version": "0.71.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {

package/src/storm/PageGenerator.ts

@@ -9,7 +9,7 @@ import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShel
 import { EventEmitter } from 'node:events';
 import PQueue from 'p-queue';
 
-import { hasPageOnDisk, normalizePath } from './page-utils';
+import { hasPageOnDisk, normalizePath, writeImageToDisk } from './page-utils';
 import * as mimetypes from 'mime-types';
 
 export interface ImagePrompt {
@@ -270,14 +270,20 @@ export class PageQueue extends EventEmitter {
             `Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim()
         );
 
-        //console.log('Adding image prompt', prompt);
+        let imageEvent: StormImage | null = null;
         result.on('data', (event: StormEvent) => {
             if (event.type === 'IMAGE') {
-                this.emit('image', event, prompt);
+                imageEvent = event;
             }
         });
 
         await result.waitForDone();
+        if (!imageEvent) {
+            throw new Error('No image was generated');
+        }
+
+        await writeImageToDisk(this.systemId, imageEvent, prompt);
+        this.emit('image', imageEvent, prompt);
     }
 
     public async generate(prompt: UIPagePrompt, conversationId: string) {

package/src/storm/events.ts

@@ -329,6 +329,8 @@ export interface StormEventDefinitionChange {
 }
 
 export enum StormEventPhaseType {
+    IMPLEMENT_APIS = 'IMPLEMENT_APIS', // Implement APIs in the html pages
+    COMPOSE_SYSTEM_PROMPT = 'COMPOSE_SYSTEM_PROMPT', // Compose system prompt for bottom-up approach
     META = 'META',
     DEFINITIONS = 'DEFINITIONS',
     IMPLEMENTATION = 'IMPLEMENTATION',

package/src/storm/routes.ts

@@ -41,7 +41,6 @@ import {
     resolveReadPath,
     SystemIdHeader,
     writeAssetToDisk,
-    writeImageToDisk,
     writePageToDisk,
 } from './page-utils';
 import { UIServer } from './UIServer';
@@ -106,6 +105,12 @@ router.post('/ui/create-system/:handle/:systemId', async (req: KapetaBodyRequest
     const srcDir = getSystemBaseDir(systemId);
     const destDir = getSystemBaseImplDir(systemId);
 
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', ConversationIdHeader);
+    res.set(ConversationIdHeader, systemId);
+
+    sendEvent(res, createPhaseStartEvent(StormEventPhaseType.IMPLEMENT_APIS));
+
     await copyDirectory(srcDir, destDir, async (fileName, content) => {
         const result = await stormClient.implementAPIClients({
             content: content,
@@ -114,9 +119,15 @@ router.post('/ui/create-system/:handle/:systemId', async (req: KapetaBodyRequest
         return result;
     });
 
+    sendEvent(res, createPhaseEndEvent(StormEventPhaseType.IMPLEMENT_APIS));
+
+    sendEvent(res, createPhaseStartEvent(StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
+
     const pages = readPages(destDir);
     const prompt = await stormClient.generatePrompt(pages);
 
+    sendEvent(res, createPhaseEndEvent(StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
+
     req.query.systemId = systemId;
     const promptRequest: BasePromptRequest = {
         prompt: prompt,
@@ -165,18 +176,6 @@ router.post('/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
 
         queue.on('page', (data) => (systemId ? sendPageEvent(systemId, data, res) : undefined));
 
-        queue.on('image', async (screenData, prompt) => {
-            if (!systemId) {
-                return;
-            }
-            try {
-                await handleImageEvent(systemId, screenData, prompt);
-            } catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
-
         queue.on('error', (err) => {
             console.error('Failed to process page', err);
             sendError(err as any, res);
@@ -275,15 +274,6 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
             sendPageEvent(landingPagesStream.getConversationId(), screenData, res)
         );
 
-        pageQueue.on('image', async (screenData, prompt) => {
-            try {
-                await handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
-            } catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
-
         pageQueue.on('event', (screenData: StormEvent) => {
             sendEvent(res, screenData);
         });
@@ -470,15 +460,6 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
 
         queue.on('page', (screenData: StormEventPage) => sendPageEvent(outerConversationId, screenData, res));
 
-        queue.on('image', async (screenData, prompt) => {
-            try {
-                await handleImageEvent(outerConversationId, screenData, prompt);
-            } catch (e) {
-                console.error('Failed to handle image event', e);
-                throw e;
-            }
-        });
-
         queue.on('event', (screenData: StormEvent) => {
             sendEvent(res, screenData);
         });
@@ -643,9 +624,13 @@ async function handleAll(req: KapetaBodyRequest, res: Response) {
             metaStream.abort();
         });
 
-        res.set('Content-Type', 'application/x-ndjson');
-        res.set('Access-Control-Expose-Headers', ConversationIdHeader);
-        res.set(ConversationIdHeader, metaStream.getConversationId());
+        // We check if the headers have been sent, because we might have already sent some data
+        // before this function is called
+        if (!res.headersSent) {
+            res.set('Content-Type', 'application/x-ndjson');
+            res.set('Access-Control-Expose-Headers', ConversationIdHeader);
+            res.set(ConversationIdHeader, metaStream.getConversationId());
+        }
 
         let currentPhase = StormEventPhaseType.META;
 
@@ -883,12 +868,4 @@ async function sendPageEvent(mainConversationId: string, data: StormEventPage, r
     sendEvent(res, convertPageEvent(data, data.payload.conversationId, mainConversationId));
 }
 
-async function handleImageEvent(mainConversationId: string, data: StormImage, prompt: ImagePrompt) {
-    try {
-        await writeImageToDisk(mainConversationId, data, prompt);
-    } catch (err) {
-        console.error('Failed to write image to disk', err);
-    }
-}
-
 export default router;