@kapeta/local-cluster-service 0.70.12 → 0.71.1

package/CHANGELOG.md

@@ -1,3 +1,18 @@
+## [0.71.1](https://github.com/kapetacom/local-cluster-service/compare/v0.71.0...v0.71.1) (2024-09-17)
+
+
+### Bug Fixes
+
+* emit all events except chunks from screen endpoints ([8e67b58](https://github.com/kapetacom/local-cluster-service/commit/8e67b58b6cc70c359556edfc61dd904254f9d3bc))
+* replace fetch retry handler library to avoid fetch failed ([59a4342](https://github.com/kapetacom/local-cluster-service/commit/59a4342203cd8efb706beba254284ce321fe313a))
+
+# [0.71.0](https://github.com/kapetacom/local-cluster-service/compare/v0.70.12...v0.71.0) (2024-09-16)
+
+
+### Features
+
+* Add two new phases ([d2805a8](https://github.com/kapetacom/local-cluster-service/commit/d2805a89d7e21b6f30674ae852ddc356954e7d79))
+
 ## [0.70.12](https://github.com/kapetacom/local-cluster-service/compare/v0.70.11...v0.70.12) (2024-09-13)
 
 

package/dist/cjs/src/storm/events.d.ts

@@ -271,6 +271,8 @@ export interface StormEventDefinitionChange {
     payload: StormDefinitions;
 }
 export declare enum StormEventPhaseType {
+    IMPLEMENT_APIS = "IMPLEMENT_APIS",// Implement APIs in the html pages
+    COMPOSE_SYSTEM_PROMPT = "COMPOSE_SYSTEM_PROMPT",// Compose system prompt for bottom-up approach
     META = "META",
     DEFINITIONS = "DEFINITIONS",
     IMPLEMENTATION = "IMPLEMENTATION",

package/dist/cjs/src/storm/events.js

@@ -11,6 +11,8 @@ var StormEventBlockStatusType;
 })(StormEventBlockStatusType || (exports.StormEventBlockStatusType = StormEventBlockStatusType = {}));
 var StormEventPhaseType;
 (function (StormEventPhaseType) {
+    StormEventPhaseType["IMPLEMENT_APIS"] = "IMPLEMENT_APIS";
+    StormEventPhaseType["COMPOSE_SYSTEM_PROMPT"] = "COMPOSE_SYSTEM_PROMPT";
     StormEventPhaseType["META"] = "META";
     StormEventPhaseType["DEFINITIONS"] = "DEFINITIONS";
     StormEventPhaseType["IMPLEMENTATION"] = "IMPLEMENTATION";

package/dist/cjs/src/storm/routes.js

@@ -72,6 +72,10 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     const srcDir = (0, page_utils_1.getSystemBaseDir)(systemId);
     const destDir = (0, page_utils_1.getSystemBaseImplDir)(systemId);
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+    res.set(stormClient_1.ConversationIdHeader, systemId);
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
     await (0, utils_1.copyDirectory)(srcDir, destDir, async (fileName, content) => {
         const result = await stormClient_1.stormClient.implementAPIClients({
             content: content,
@@ -79,8 +83,11 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
         });
         return result;
     });
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     const pages = (0, utils_1.readPages)(destDir);
     const prompt = await stormClient_1.stormClient.generatePrompt(pages);
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     req.query.systemId = systemId;
     const promptRequest = {
         prompt: prompt,
@@ -122,6 +129,12 @@ router.post('/ui/screen', async (req, res) => {
             console.error('Failed to process page', err);
             sendError(err, res);
         });
+        queue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
+        });
         await queue.addPrompt(aiRequest, conversationId, true);
         await queue.wait();
         await Promise.allSettled(promises);
@@ -200,8 +213,11 @@ router.post('/:handle/ui/iterative', async (req, res) => {
             pageQueue.cancel();
         });
         pageQueue.on('page', (screenData) => sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        pageQueue.on('event', (screenData) => {
-            sendEvent(res, screenData);
+        pageQueue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
         });
         pageQueue.on('error', (err) => {
             console.error('Failed to process page', err);
@@ -353,8 +369,11 @@ router.post('/:handle/ui', async (req, res) => {
             queue.cancel();
         });
         queue.on('page', (screenData) => sendPageEvent(outerConversationId, screenData, res));
-        queue.on('event', (screenData) => {
-            sendEvent(res, screenData);
+        queue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
         });
         queue.on('error', (err) => {
             console.error('Failed to process page', err);
@@ -410,10 +429,11 @@ router.post('/ui/edit', async (req, res) => {
                 return promise;
             }
         });
-        queue.on('event', (data) => {
-            if (data.type === 'FILE_START' || data.type === 'FILE_DONE' || data.type === 'FILE_STATE') {
-                sendEvent(res, data);
+        queue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
             }
+            sendEvent(res, event);
         });
         queue.on('error', (err) => {
             console.error('Failed to process page', err);
@@ -490,9 +510,13 @@ async function handleAll(req, res) {
         onRequestAborted(req, res, () => {
             metaStream.abort();
         });
-        res.set('Content-Type', 'application/x-ndjson');
-        res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
-        res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        // We check if the headers have been sent, because we might have already sent some data
+        // before this function is called
+        if (!res.headersSent) {
+            res.set('Content-Type', 'application/x-ndjson');
+            res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+            res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        }
         let currentPhase = events_1.StormEventPhaseType.META;
         // Helper to avoid sending the plan multiple times in a row
         const sendUpdatedPlan = lodash_1.default.debounce(sendDefinitions, 50, { maxWait: 200 });

package/dist/cjs/src/storm/stormClient.d.ts

@@ -1,3 +1,4 @@
+/// <reference types="node" />
 import { ConversationItem, ImplementAPIClientsRequest, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt, StormUIListPrompt } from './stream';
 import { Page, StormEventPageUrl } from './events';
 export declare const STORM_ID = "storm";
@@ -65,7 +66,7 @@ declare class StormClient {
     createUIShells(prompt: UIShellsPrompt, conversationId?: string): Promise<StormStream>;
     createUILandingPages(prompt: BasePromptRequest, conversationId?: string): Promise<StormStream>;
     createUIPage(prompt: UIPagePrompt, conversationId?: string, history?: ConversationItem[]): Promise<StormStream>;
-    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<import("undici").Response>;
+    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<Response>;
     getVoteUIPage(topic: string, conversationId: string, mainConversationId?: string): Promise<{
         vote: -1 | 0 | 1;
     }>;

package/dist/cjs/src/storm/stormClient.js

@@ -13,22 +13,8 @@ const utils_1 = require("../utils/utils");
 const promises_1 = __importDefault(require("node:readline/promises"));
 const node_stream_1 = require("node:stream");
 const stream_1 = require("./stream");
-const undici_1 = require("undici");
-// Will only retry on error codes and GET requests by default
-// See https://github.com/nodejs/undici/blob/990df2c7e37cbe5bb44fe2f576dddeaeb5916590/docs/docs/api/RetryAgent.md
-const retryAgent = new undici_1.RetryAgent(new undici_1.Agent(), {
-    methods: [
-        // Added methods ↓ (not idempotent), but we want to retry on POST:
-        'POST',
-        // defaults below
-        'GET',
-        'HEAD',
-        'OPTIONS',
-        'PUT',
-        'DELETE',
-        'TRACE',
-    ],
-});
+const fetch_retry_1 = __importDefault(require("fetch-retry"));
+const fetchWithRetries = (0, fetch_retry_1.default)(global.fetch, { retries: 5, retryDelay: 10 });
 exports.STORM_ID = 'storm';
 exports.ConversationIdHeader = 'Conversation-Id';
 class StormClient {
@@ -54,7 +40,6 @@ class StormClient {
             method: method,
             body: JSON.stringify(body),
             headers,
-            dispatcher: retryAgent,
         };
     }
     async send(path, body, method = 'POST') {
@@ -65,7 +50,7 @@ class StormClient {
         });
         const abort = new AbortController();
         options.signal = abort.signal;
-        const response = await (0, undici_1.fetch)(options.url, options);
+        const response = await fetchWithRetries(options.url, options);
         if (response.status !== 200) {
             throw new Error(`Got error response from ${options.url}: ${response.status}\nContent: ${await response.text()}`);
         }
@@ -138,19 +123,19 @@ class StormClient {
             prompt: JSON.stringify({ topic, vote, mainConversationId }),
             conversationId,
         });
-        return (0, undici_1.fetch)(options.url, options);
+        return fetch(options.url, options);
     }
     async getVoteUIPage(topic, conversationId, mainConversationId) {
         const options = await this.createOptions('/v2/ui/get-vote', 'POST', {
             prompt: JSON.stringify({ topic, mainConversationId }),
             conversationId,
         });
-        const response = await (0, undici_1.fetch)(options.url, options);
+        const response = await fetch(options.url, options);
         return response.json();
     }
     async implementAPIClients(prompt) {
         const u = `${this._baseUrl}/v2/ui/implement-api-clients`;
-        const response = await (0, undici_1.fetch)(u, {
+        const response = await fetch(u, {
             method: 'POST',
             body: JSON.stringify({
                 fileName: prompt.fileName,
@@ -162,7 +147,7 @@ class StormClient {
     }
     async generatePrompt(pages) {
         const u = `${this._baseUrl}/v2/ui/prompt`;
-        const response = await (0, undici_1.fetch)(u, {
+        const response = await fetch(u, {
             method: 'POST',
             body: JSON.stringify({
                 pages: pages,
@@ -235,7 +220,7 @@ class StormClient {
             prompt: '',
             conversationId: conversationId,
         });
-        const response = await (0, undici_1.fetch)(options.url, options);
+        const response = await fetch(options.url, options);
         return response.text();
     }
 }

package/dist/esm/src/storm/events.d.ts

@@ -271,6 +271,8 @@ export interface StormEventDefinitionChange {
     payload: StormDefinitions;
 }
 export declare enum StormEventPhaseType {
+    IMPLEMENT_APIS = "IMPLEMENT_APIS",// Implement APIs in the html pages
+    COMPOSE_SYSTEM_PROMPT = "COMPOSE_SYSTEM_PROMPT",// Compose system prompt for bottom-up approach
     META = "META",
     DEFINITIONS = "DEFINITIONS",
     IMPLEMENTATION = "IMPLEMENTATION",

package/dist/esm/src/storm/events.js

@@ -11,6 +11,8 @@ var StormEventBlockStatusType;
 })(StormEventBlockStatusType || (exports.StormEventBlockStatusType = StormEventBlockStatusType = {}));
 var StormEventPhaseType;
 (function (StormEventPhaseType) {
+    StormEventPhaseType["IMPLEMENT_APIS"] = "IMPLEMENT_APIS";
+    StormEventPhaseType["COMPOSE_SYSTEM_PROMPT"] = "COMPOSE_SYSTEM_PROMPT";
     StormEventPhaseType["META"] = "META";
     StormEventPhaseType["DEFINITIONS"] = "DEFINITIONS";
     StormEventPhaseType["IMPLEMENTATION"] = "IMPLEMENTATION";

package/dist/esm/src/storm/routes.js

@@ -72,6 +72,10 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     const srcDir = (0, page_utils_1.getSystemBaseDir)(systemId);
     const destDir = (0, page_utils_1.getSystemBaseImplDir)(systemId);
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+    res.set(stormClient_1.ConversationIdHeader, systemId);
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
     await (0, utils_1.copyDirectory)(srcDir, destDir, async (fileName, content) => {
         const result = await stormClient_1.stormClient.implementAPIClients({
             content: content,
@@ -79,8 +83,11 @@ router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
         });
         return result;
     });
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.IMPLEMENT_APIS));
+    sendEvent(res, (0, event_parser_1.createPhaseStartEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     const pages = (0, utils_1.readPages)(destDir);
     const prompt = await stormClient_1.stormClient.generatePrompt(pages);
+    sendEvent(res, (0, event_parser_1.createPhaseEndEvent)(events_1.StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
     req.query.systemId = systemId;
     const promptRequest = {
         prompt: prompt,
@@ -122,6 +129,12 @@ router.post('/ui/screen', async (req, res) => {
             console.error('Failed to process page', err);
             sendError(err, res);
         });
+        queue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
+        });
         await queue.addPrompt(aiRequest, conversationId, true);
         await queue.wait();
         await Promise.allSettled(promises);
@@ -200,8 +213,11 @@ router.post('/:handle/ui/iterative', async (req, res) => {
             pageQueue.cancel();
         });
         pageQueue.on('page', (screenData) => sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        pageQueue.on('event', (screenData) => {
-            sendEvent(res, screenData);
+        pageQueue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
         });
         pageQueue.on('error', (err) => {
             console.error('Failed to process page', err);
@@ -353,8 +369,11 @@ router.post('/:handle/ui', async (req, res) => {
             queue.cancel();
         });
         queue.on('page', (screenData) => sendPageEvent(outerConversationId, screenData, res));
-        queue.on('event', (screenData) => {
-            sendEvent(res, screenData);
+        queue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
         });
         queue.on('error', (err) => {
             console.error('Failed to process page', err);
@@ -410,10 +429,11 @@ router.post('/ui/edit', async (req, res) => {
                 return promise;
             }
         });
-        queue.on('event', (data) => {
-            if (data.type === 'FILE_START' || data.type === 'FILE_DONE' || data.type === 'FILE_STATE') {
-                sendEvent(res, data);
+        queue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
             }
+            sendEvent(res, event);
         });
         queue.on('error', (err) => {
             console.error('Failed to process page', err);
@@ -490,9 +510,13 @@ async function handleAll(req, res) {
         onRequestAborted(req, res, () => {
             metaStream.abort();
         });
-        res.set('Content-Type', 'application/x-ndjson');
-        res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
-        res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        // We check if the headers have been sent, because we might have already sent some data
+        // before this function is called
+        if (!res.headersSent) {
+            res.set('Content-Type', 'application/x-ndjson');
+            res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+            res.set(stormClient_1.ConversationIdHeader, metaStream.getConversationId());
+        }
         let currentPhase = events_1.StormEventPhaseType.META;
         // Helper to avoid sending the plan multiple times in a row
         const sendUpdatedPlan = lodash_1.default.debounce(sendDefinitions, 50, { maxWait: 200 });

package/dist/esm/src/storm/stormClient.d.ts

@@ -1,3 +1,4 @@
+/// <reference types="node" />
 import { ConversationItem, ImplementAPIClientsRequest, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt, StormUIListPrompt } from './stream';
 import { Page, StormEventPageUrl } from './events';
 export declare const STORM_ID = "storm";
@@ -65,7 +66,7 @@ declare class StormClient {
     createUIShells(prompt: UIShellsPrompt, conversationId?: string): Promise<StormStream>;
     createUILandingPages(prompt: BasePromptRequest, conversationId?: string): Promise<StormStream>;
     createUIPage(prompt: UIPagePrompt, conversationId?: string, history?: ConversationItem[]): Promise<StormStream>;
-    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<import("undici").Response>;
+    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<Response>;
     getVoteUIPage(topic: string, conversationId: string, mainConversationId?: string): Promise<{
         vote: -1 | 0 | 1;
     }>;

package/dist/esm/src/storm/stormClient.js

@@ -13,22 +13,8 @@ const utils_1 = require("../utils/utils");
 const promises_1 = __importDefault(require("node:readline/promises"));
 const node_stream_1 = require("node:stream");
 const stream_1 = require("./stream");
-const undici_1 = require("undici");
-// Will only retry on error codes and GET requests by default
-// See https://github.com/nodejs/undici/blob/990df2c7e37cbe5bb44fe2f576dddeaeb5916590/docs/docs/api/RetryAgent.md
-const retryAgent = new undici_1.RetryAgent(new undici_1.Agent(), {
-    methods: [
-        // Added methods ↓ (not idempotent), but we want to retry on POST:
-        'POST',
-        // defaults below
-        'GET',
-        'HEAD',
-        'OPTIONS',
-        'PUT',
-        'DELETE',
-        'TRACE',
-    ],
-});
+const fetch_retry_1 = __importDefault(require("fetch-retry"));
+const fetchWithRetries = (0, fetch_retry_1.default)(global.fetch, { retries: 5, retryDelay: 10 });
 exports.STORM_ID = 'storm';
 exports.ConversationIdHeader = 'Conversation-Id';
 class StormClient {
@@ -54,7 +40,6 @@ class StormClient {
             method: method,
             body: JSON.stringify(body),
             headers,
-            dispatcher: retryAgent,
         };
     }
     async send(path, body, method = 'POST') {
@@ -65,7 +50,7 @@ class StormClient {
         });
         const abort = new AbortController();
         options.signal = abort.signal;
-        const response = await (0, undici_1.fetch)(options.url, options);
+        const response = await fetchWithRetries(options.url, options);
         if (response.status !== 200) {
             throw new Error(`Got error response from ${options.url}: ${response.status}\nContent: ${await response.text()}`);
         }
@@ -138,19 +123,19 @@ class StormClient {
             prompt: JSON.stringify({ topic, vote, mainConversationId }),
             conversationId,
         });
-        return (0, undici_1.fetch)(options.url, options);
+        return fetch(options.url, options);
     }
     async getVoteUIPage(topic, conversationId, mainConversationId) {
         const options = await this.createOptions('/v2/ui/get-vote', 'POST', {
             prompt: JSON.stringify({ topic, mainConversationId }),
             conversationId,
         });
-        const response = await (0, undici_1.fetch)(options.url, options);
+        const response = await fetch(options.url, options);
         return response.json();
     }
     async implementAPIClients(prompt) {
         const u = `${this._baseUrl}/v2/ui/implement-api-clients`;
-        const response = await (0, undici_1.fetch)(u, {
+        const response = await fetch(u, {
             method: 'POST',
             body: JSON.stringify({
                 fileName: prompt.fileName,
@@ -162,7 +147,7 @@ class StormClient {
     }
     async generatePrompt(pages) {
         const u = `${this._baseUrl}/v2/ui/prompt`;
-        const response = await (0, undici_1.fetch)(u, {
+        const response = await fetch(u, {
             method: 'POST',
             body: JSON.stringify({
                 pages: pages,
@@ -235,7 +220,7 @@ class StormClient {
             prompt: '',
             conversationId: conversationId,
         });
-        const response = await (0, undici_1.fetch)(options.url, options);
+        const response = await fetch(options.url, options);
         return response.text();
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.70.12",
+    "version": "0.71.1",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -70,6 +70,7 @@
         "download-git-repo": "^3.0.2",
         "express": "4.17.1",
         "express-promise-router": "^4.1.1",
+        "fetch-retry": "^6.0.0",
         "fs-extra": "^11.1.0",
         "glob": "^7.1.6",
         "gunzip-maybe": "^1.4.2",
@@ -88,7 +89,6 @@
         "stream-json": "^1.8.0",
         "tar-stream": "^3.1.6",
         "typescript": "^5.1.6",
-        "undici": "^6.19.8",
         "uuid": "^9.0.1",
         "yaml": "^1.6.0"
     },

package/src/storm/PageGenerator.ts

@@ -259,7 +259,7 @@ export class PageQueue extends EventEmitter {
         // Add safeguard to avoid generating images for nonsense URLs
         // Sometimes we get entries for Base URLs that will then cause issues on the filesystem
         // Example: https://www.kapeta.com/images/
-        const mimeType = mimetypes.lookup(prompt.url) as string | false;
+        const mimeType = mimetypes.lookup(prompt.url);
         if (!mimeType || !mimeType.startsWith('image/')) {
             console.warn('Skipping image reference of type %s for url %s', mimeType, prompt.url);
             return;

package/src/storm/events.ts

@@ -329,6 +329,8 @@ export interface StormEventDefinitionChange {
 }
 
 export enum StormEventPhaseType {
+    IMPLEMENT_APIS = 'IMPLEMENT_APIS', // Implement APIs in the html pages
+    COMPOSE_SYSTEM_PROMPT = 'COMPOSE_SYSTEM_PROMPT', // Compose system prompt for bottom-up approach
     META = 'META',
     DEFINITIONS = 'DEFINITIONS',
     IMPLEMENTATION = 'IMPLEMENTATION',

package/src/storm/routes.ts

@@ -105,6 +105,12 @@ router.post('/ui/create-system/:handle/:systemId', async (req: KapetaBodyRequest
     const srcDir = getSystemBaseDir(systemId);
     const destDir = getSystemBaseImplDir(systemId);
 
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', ConversationIdHeader);
+    res.set(ConversationIdHeader, systemId);
+
+    sendEvent(res, createPhaseStartEvent(StormEventPhaseType.IMPLEMENT_APIS));
+
     await copyDirectory(srcDir, destDir, async (fileName, content) => {
         const result = await stormClient.implementAPIClients({
             content: content,
@@ -113,9 +119,15 @@ router.post('/ui/create-system/:handle/:systemId', async (req: KapetaBodyRequest
         return result;
     });
 
+    sendEvent(res, createPhaseEndEvent(StormEventPhaseType.IMPLEMENT_APIS));
+
+    sendEvent(res, createPhaseStartEvent(StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
+
     const pages = readPages(destDir);
     const prompt = await stormClient.generatePrompt(pages);
 
+    sendEvent(res, createPhaseEndEvent(StormEventPhaseType.COMPOSE_SYSTEM_PROMPT));
+
     req.query.systemId = systemId;
     const promptRequest: BasePromptRequest = {
         prompt: prompt,
@@ -169,6 +181,13 @@ router.post('/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
             sendError(err as any, res);
         });
 
+        queue.on('event', (event: StormEvent) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
+        });
+
         await queue.addPrompt(aiRequest, conversationId, true);
 
         await queue.wait();
@@ -262,8 +281,11 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
             sendPageEvent(landingPagesStream.getConversationId(), screenData, res)
         );
 
-        pageQueue.on('event', (screenData: StormEvent) => {
-            sendEvent(res, screenData);
+        pageQueue.on('event', (event: StormEvent) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
         });
 
         pageQueue.on('error', (err) => {
@@ -448,8 +470,11 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
 
         queue.on('page', (screenData: StormEventPage) => sendPageEvent(outerConversationId, screenData, res));
 
-        queue.on('event', (screenData: StormEvent) => {
-            sendEvent(res, screenData);
+        queue.on('event', (event: StormEvent) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
+            }
+            sendEvent(res, event);
         });
 
         queue.on('error', (err) => {
@@ -515,10 +540,11 @@ router.post('/ui/edit', async (req: KapetaBodyRequest, res: Response) => {
             }
         });
 
-        queue.on('event', (data) => {
-            if (data.type === 'FILE_START' || data.type === 'FILE_DONE' || data.type === 'FILE_STATE') {
-                sendEvent(res, data);
+        queue.on('event', (event) => {
+            if (event.type === 'FILE_CHUNK') {
+                return;
             }
+            sendEvent(res, event);
         });
 
         queue.on('error', (err) => {
@@ -612,9 +638,13 @@ async function handleAll(req: KapetaBodyRequest, res: Response) {
             metaStream.abort();
         });
 
-        res.set('Content-Type', 'application/x-ndjson');
-        res.set('Access-Control-Expose-Headers', ConversationIdHeader);
-        res.set(ConversationIdHeader, metaStream.getConversationId());
+        // We check if the headers have been sent, because we might have already sent some data
+        // before this function is called
+        if (!res.headersSent) {
+            res.set('Content-Type', 'application/x-ndjson');
+            res.set('Access-Control-Expose-Headers', ConversationIdHeader);
+            res.set(ConversationIdHeader, metaStream.getConversationId());
+        }
 
         let currentPhase = StormEventPhaseType.META;
 

package/src/storm/stormClient.ts

@@ -16,23 +16,8 @@ import {
     StormUIListPrompt,
 } from './stream';
 import { Page, StormEventPageUrl } from './events';
-import { fetch, RequestInit, Agent, RetryAgent } from 'undici';
-
-// Will only retry on error codes and GET requests by default
-// See https://github.com/nodejs/undici/blob/990df2c7e37cbe5bb44fe2f576dddeaeb5916590/docs/docs/api/RetryAgent.md
-const retryAgent = new RetryAgent(new Agent(), {
-    methods: [
-        // Added methods ↓ (not idempotent), but we want to retry on POST:
-        'POST',
-        // defaults below
-        'GET',
-        'HEAD',
-        'OPTIONS',
-        'PUT',
-        'DELETE',
-        'TRACE',
-    ],
-});
+import createFetch from 'fetch-retry';
+const fetchWithRetries = createFetch(global.fetch, { retries: 5, retryDelay: 10 });
 
 export const STORM_ID = 'storm';
 
@@ -129,7 +114,6 @@ class StormClient {
             method: method,
             body: JSON.stringify(body),
             headers,
-            dispatcher: retryAgent,
         };
     }
 
@@ -148,7 +132,7 @@ class StormClient {
         const abort = new AbortController();
         options.signal = abort.signal;
 
-        const response = await fetch(options.url, options);
+        const response = await fetchWithRetries(options.url, options);
 
         if (response.status !== 200) {
             throw new Error(