@kapeta/local-cluster-service 0.67.4 → 0.67.5

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [0.67.5](https://github.com/kapetacom/local-cluster-service/compare/v0.67.4...v0.67.5) (2024-09-02)
+
+
+### Bug Fixes
+
+* better tracking of async events [CORE-3409] ([5e8f8cd](https://github.com/kapetacom/local-cluster-service/commit/5e8f8cdbe7e714cf0dfb5b0e7cbb8b60f837c2f9))
+* dont generate individual pages when page matches path w/ params ([c3ca35c](https://github.com/kapetacom/local-cluster-service/commit/c3ca35ce56781ca90f3ca1070fee76aa3f5512ab))
+
 ## [0.67.4](https://github.com/kapetacom/local-cluster-service/compare/v0.67.3...v0.67.4) (2024-08-30)
 
 

package/dist/cjs/src/storm/PageGenerator.d.ts

@@ -6,7 +6,6 @@
 import { UIPagePrompt } from './stormClient';
 import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
-import { FuturePromise } from './PromiseQueue';
 export interface ImagePrompt {
     name: string;
     description: string;
@@ -21,6 +20,7 @@ type InitialPrompt = Omit<UIPagePrompt, 'shell_page'> & {
 };
 export declare class PageQueue extends EventEmitter {
     private readonly queue;
+    private readonly eventQueue;
     private readonly systemId;
     private readonly systemPrompt;
     private readonly references;
@@ -29,12 +29,12 @@ export declare class PageQueue extends EventEmitter {
     private uiShells;
     private theme;
     constructor(systemId: string, systemPrompt: string, concurrency?: number);
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(event: 'page', listener: (data: StormEventPage) => void): this;
-    on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
+    on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
+    on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
+    on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
-    emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
+    emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;
     addUiShell(uiShell: UIShell): void;
     setUiTheme(theme: string): void;
     private hasPrompt;
@@ -47,6 +47,7 @@ export declare class PageQueue extends EventEmitter {
     private addImagePrompt;
 }
 export declare class PageGenerator extends EventEmitter {
+    private readonly eventQueue;
     private readonly conversationId;
     readonly prompt: UIPagePrompt;
     constructor(prompt: UIPagePrompt, conversationId?: string);
@@ -54,13 +55,11 @@ export declare class PageGenerator extends EventEmitter {
     on(event: 'page_refs', listener: (data: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
-    }) => void): this;
+    }) => Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page_refs', event: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
     }): boolean;
     generate(): Promise<void>;
     private resolveReferences;

package/dist/cjs/src/storm/PageGenerator.js

@@ -15,6 +15,7 @@ const PromiseQueue_1 = require("./PromiseQueue");
 const page_utils_1 = require("./page-utils");
 class PageQueue extends node_events_1.EventEmitter {
     queue;
+    eventQueue;
     systemId;
     systemPrompt;
     references = new Map();
@@ -27,9 +28,12 @@ class PageQueue extends node_events_1.EventEmitter {
         this.systemId = systemId;
         this.systemPrompt = systemPrompt;
         this.queue = new PromiseQueue_1.PromiseQueue(concurrency);
+        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
     }
     on(event, listener) {
-        return super.on(event, listener);
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
     emit(eventName, ...args) {
         return super.emit(eventName, ...args);
@@ -56,6 +60,7 @@ class PageQueue extends node_events_1.EventEmitter {
             //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
         }
+        console.log('Generating page for', initialPrompt.method, initialPrompt.path);
         const prompt = {
             ...initialPrompt,
             shell_page: this.uiShells.find((shell) => shell.screens.includes(initialPrompt.name))?.content,
@@ -75,7 +80,7 @@ class PageQueue extends node_events_1.EventEmitter {
         return promptPrefix;
     }
     wrapPagePrompt(pagePath, prompt) {
-        let promptPrefix = this.getPrefix();
+        const promptPrefix = this.getPrefix();
         let promptPostfix = '';
         if (this.pages.size > 0) {
             promptPostfix = `\nThe following pages are already implemented:\n`;
@@ -96,10 +101,10 @@ class PageQueue extends node_events_1.EventEmitter {
     }
     async addPageGenerator(generator) {
         generator.on('event', (event) => this.emit('event', event));
-        generator.on('page_refs', async ({ event, references, future }) => {
+        generator.on('page_refs', async ({ event, references }) => {
             try {
                 const initialPrompts = [];
-                let promises = references.map(async (reference) => {
+                const resourcePromises = references.map(async (reference) => {
                     if (reference.url.startsWith('#') ||
                         reference.url.startsWith('javascript:') ||
                         reference.url.startsWith('http://') ||
@@ -135,8 +140,11 @@ class PageQueue extends node_events_1.EventEmitter {
                             break;
                     }
                 });
-                await Promise.allSettled(promises);
-                initialPrompts.forEach((prompt) => {
+                // Wait for resources to be generated
+                await Promise.allSettled(resourcePromises);
+                this.emit('page', event);
+                // Emit any new pages after the current page to increase responsiveness
+                const newPages = initialPrompts.map((prompt) => {
                     if (!this.hasPrompt(prompt.path)) {
                         this.emit('page', {
                             type: 'PAGE',
@@ -155,21 +163,24 @@ class PageQueue extends node_events_1.EventEmitter {
                             },
                         });
                     }
-                    this.addPrompt(prompt);
+                    return this.addPrompt(prompt);
                 });
-                this.emit('page', event);
+                await Promise.allSettled(newPages);
             }
-            finally {
-                future.resolve();
+            catch (e) {
+                console.error('Failed to process event', e);
+                throw e;
             }
         });
         return this.queue.add(() => generator.generate());
     }
     cancel() {
         this.queue.cancel();
+        this.eventQueue.cancel();
     }
-    wait() {
-        return this.queue.wait();
+    async wait() {
+        await this.eventQueue.wait();
+        await this.queue.wait();
     }
     async addImagePrompt(prompt) {
         if (this.images.has(prompt.url)) {
@@ -180,65 +191,54 @@ class PageQueue extends node_events_1.EventEmitter {
         const prefix = this.getPrefix();
         const result = await stormClient_1.stormClient.createImage(prefix + `Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim());
         //console.log('Adding image prompt', prompt);
-        const futures = [];
-        result.on('data', async (event) => {
+        result.on('data', (event) => {
             if (event.type === 'IMAGE') {
-                const future = (0, PromiseQueue_1.createFuture)();
-                futures.push(future);
-                this.emit('image', event, prompt, future);
-                setTimeout(() => {
-                    if (!future.isResolved()) {
-                        console.log('Image prompt timed out', prompt);
-                        future.reject(new Error('Image prompt timed out'));
-                    }
-                }, 30000);
+                this.emit('image', event, prompt);
             }
         });
         await result.waitForDone();
-        await Promise.allSettled(futures.map((f) => f.promise));
     }
 }
 exports.PageQueue = PageQueue;
 class PageGenerator extends node_events_1.EventEmitter {
+    eventQueue;
     conversationId;
     prompt;
     constructor(prompt, conversationId = node_uuid_1.default.v4()) {
         super();
         this.conversationId = conversationId;
         this.prompt = prompt;
+        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
     }
     on(event, listener) {
-        return super.on(event, listener);
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
     emit(eventName, ...args) {
         return super.emit(eventName, ...args);
     }
     async generate() {
-        return new Promise(async (resolve) => {
-            const promises = [];
-            const screenStream = await stormClient_1.stormClient.createUIPage(this.prompt, this.conversationId);
-            screenStream.on('data', (event) => {
-                if (event.type === 'PAGE') {
-                    event.payload.conversationId = this.conversationId;
-                    promises.push((async () => {
-                        const references = await this.resolveReferences(event.payload.content);
-                        //console.log('Resolved references for page', references, event.payload);
-                        const future = (0, PromiseQueue_1.createFuture)();
-                        this.emit('page_refs', {
-                            event,
-                            references,
-                            future,
-                        });
-                        await future.promise;
-                    })());
-                    return;
-                }
-                this.emit('event', event);
-            });
-            await screenStream.waitForDone();
-            await Promise.all(promises);
-            resolve();
+        const promises = [];
+        const screenStream = await stormClient_1.stormClient.createUIPage(this.prompt, this.conversationId);
+        screenStream.on('data', (event) => {
+            if (event.type === 'PAGE') {
+                event.payload.conversationId = this.conversationId;
+                promises.push((async () => {
+                    const references = await this.resolveReferences(event.payload.content);
+                    //console.log('Resolved references for page', references, event.payload);
+                    this.emit('page_refs', {
+                        event,
+                        references,
+                    });
+                })());
+                return;
+            }
+            this.emit('event', event);
         });
+        await screenStream.waitForDone();
+        await Promise.all(promises);
+        await this.eventQueue.wait();
     }
     async resolveReferences(content) {
         const referenceStream = await stormClient_1.stormClient.classifyUIReferences(content);

package/dist/cjs/src/storm/page-utils.js

@@ -12,7 +12,7 @@ function normalizePath(path) {
     return path
         .replace(/\?.*$/gi, '')
         .replace(/:[a-z][a-z_]*\b/gi, '*')
-        .replace(/\{[a-z]+}/gi, '*');
+        .replace(/\{[a-z-.]+}/gi, '*');
 }
 async function writePageToDisk(systemId, event) {
     const baseDir = getSystemBaseDir(systemId);
@@ -53,13 +53,8 @@ async function writeImageToDisk(systemId, event, prompt) {
 }
 exports.writeImageToDisk = writeImageToDisk;
 function hasPageOnDisk(systemId, method, path) {
-    if (!systemId || !method || !path) {
-        return false;
-    }
-    const baseDir = getSystemBaseDir(systemId);
-    const filePath = getFilePath(method);
-    const fullPath = path_1.default.join(baseDir, normalizePath(path), filePath);
-    return fs_extra_1.default.existsSync(fullPath);
+    const fullPath = resolveReadPath(systemId, method, path);
+    return !!fullPath && fs_extra_1.default.existsSync(fullPath);
 }
 exports.hasPageOnDisk = hasPageOnDisk;
 function getSystemBaseDir(systemId) {
@@ -90,7 +85,7 @@ function resolveReadPath(systemId, path, method) {
     }
     const parts = path.split(/\*/g);
     let currentPath = '';
-    for (let part in parts) {
+    for (let part of parts) {
         const thisPath = path_1.default.join(currentPath, part);
         const starPath = path_1.default.join(currentPath, '*');
         const thisPathDir = path_1.default.join(baseDir, thisPath);
@@ -103,7 +98,6 @@ function resolveReadPath(systemId, path, method) {
             currentPath = starPath;
             continue;
         }
-        console.log('Path not found', thisPathDir, starPathDir);
         // Path not found
         return null;
     }

package/dist/cjs/src/storm/routes.js

@@ -74,24 +74,17 @@ router.post('/ui/screen', async (req, res) => {
             queue.cancel();
         });
         const promises = [];
-        queue.on('page', (data) => {
-            if (systemId) {
-                promises.push(sendPageEvent(systemId, data, res));
-            }
-        });
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('page', (data) => (systemId ? sendPageEvent(systemId, data, res) : undefined));
+        queue.on('image', async (screenData, prompt) => {
             if (!systemId) {
                 return;
             }
             try {
-                const promise = handleImageEvent(systemId, screenData, prompt);
-                promises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(systemId, screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         await queue.addPrompt(aiRequest, conversationId, true);
@@ -186,19 +179,14 @@ router.post('/:handle/ui/iterative', async (req, res) => {
         onRequestAborted(req, res, () => {
             pageQueue.cancel();
         });
-        pageQueue.on('page', (screenData) => {
-            pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        });
-        pageQueue.on('image', async (screenData, prompt, future) => {
+        pageQueue.on('page', (screenData) => sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
+        pageQueue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         pageQueue.on('event', (screenData) => {
@@ -346,31 +334,25 @@ router.post('/:handle/ui', async (req, res) => {
             },
             created: Date.now(),
         });
-        const pagePromises = [];
         onRequestAborted(req, res, () => {
             queue.cancel();
         });
-        const pageEventPromises = [];
-        queue.on('page', (screenData) => {
-            pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
-        });
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('page', (screenData) => sendPageEvent(outerConversationId, screenData, res));
+        queue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(outerConversationId, screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(outerConversationId, screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         queue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
         for (const screen of Object.values(uniqueUserJourneyScreens)) {
-            pagePromises.push(queue.addPrompt({
+            queue
+                .addPrompt({
                 prompt: screen.requirements,
                 method: screen.method,
                 path: screen.path,
@@ -380,14 +362,15 @@ router.post('/:handle/ui', async (req, res) => {
                 filename: screen.filename,
                 storage_prefix: outerConversationId + '_',
                 theme,
-            }));
+            })
+                .catch((e) => {
+                console.error('Failed to generate page for screen %s', screen.name, e);
+            });
         }
         if (userJourneysStream.isAborted()) {
             return;
         }
         await queue.wait();
-        await Promise.allSettled(pagePromises);
-        await Promise.allSettled(pageEventPromises);
         sendDone(res);
     }
     catch (err) {

package/dist/esm/src/storm/PageGenerator.d.ts

@@ -6,7 +6,6 @@
 import { UIPagePrompt } from './stormClient';
 import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
-import { FuturePromise } from './PromiseQueue';
 export interface ImagePrompt {
     name: string;
     description: string;
@@ -21,6 +20,7 @@ type InitialPrompt = Omit<UIPagePrompt, 'shell_page'> & {
 };
 export declare class PageQueue extends EventEmitter {
     private readonly queue;
+    private readonly eventQueue;
     private readonly systemId;
     private readonly systemPrompt;
     private readonly references;
@@ -29,12 +29,12 @@ export declare class PageQueue extends EventEmitter {
     private uiShells;
     private theme;
     constructor(systemId: string, systemPrompt: string, concurrency?: number);
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(event: 'page', listener: (data: StormEventPage) => void): this;
-    on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
+    on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
+    on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
+    on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
-    emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
+    emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;
     addUiShell(uiShell: UIShell): void;
     setUiTheme(theme: string): void;
     private hasPrompt;
@@ -47,6 +47,7 @@ export declare class PageQueue extends EventEmitter {
     private addImagePrompt;
 }
 export declare class PageGenerator extends EventEmitter {
+    private readonly eventQueue;
     private readonly conversationId;
     readonly prompt: UIPagePrompt;
     constructor(prompt: UIPagePrompt, conversationId?: string);
@@ -54,13 +55,11 @@ export declare class PageGenerator extends EventEmitter {
     on(event: 'page_refs', listener: (data: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
-    }) => void): this;
+    }) => Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page_refs', event: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
     }): boolean;
     generate(): Promise<void>;
     private resolveReferences;

package/dist/esm/src/storm/PageGenerator.js

@@ -15,6 +15,7 @@ const PromiseQueue_1 = require("./PromiseQueue");
 const page_utils_1 = require("./page-utils");
 class PageQueue extends node_events_1.EventEmitter {
     queue;
+    eventQueue;
     systemId;
     systemPrompt;
     references = new Map();
@@ -27,9 +28,12 @@ class PageQueue extends node_events_1.EventEmitter {
         this.systemId = systemId;
         this.systemPrompt = systemPrompt;
         this.queue = new PromiseQueue_1.PromiseQueue(concurrency);
+        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
     }
     on(event, listener) {
-        return super.on(event, listener);
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
     emit(eventName, ...args) {
         return super.emit(eventName, ...args);
@@ -56,6 +60,7 @@ class PageQueue extends node_events_1.EventEmitter {
             //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
         }
+        console.log('Generating page for', initialPrompt.method, initialPrompt.path);
         const prompt = {
             ...initialPrompt,
             shell_page: this.uiShells.find((shell) => shell.screens.includes(initialPrompt.name))?.content,
@@ -75,7 +80,7 @@ class PageQueue extends node_events_1.EventEmitter {
         return promptPrefix;
     }
     wrapPagePrompt(pagePath, prompt) {
-        let promptPrefix = this.getPrefix();
+        const promptPrefix = this.getPrefix();
         let promptPostfix = '';
         if (this.pages.size > 0) {
             promptPostfix = `\nThe following pages are already implemented:\n`;
@@ -96,10 +101,10 @@ class PageQueue extends node_events_1.EventEmitter {
     }
     async addPageGenerator(generator) {
         generator.on('event', (event) => this.emit('event', event));
-        generator.on('page_refs', async ({ event, references, future }) => {
+        generator.on('page_refs', async ({ event, references }) => {
             try {
                 const initialPrompts = [];
-                let promises = references.map(async (reference) => {
+                const resourcePromises = references.map(async (reference) => {
                     if (reference.url.startsWith('#') ||
                         reference.url.startsWith('javascript:') ||
                         reference.url.startsWith('http://') ||
@@ -135,8 +140,11 @@ class PageQueue extends node_events_1.EventEmitter {
                             break;
                     }
                 });
-                await Promise.allSettled(promises);
-                initialPrompts.forEach((prompt) => {
+                // Wait for resources to be generated
+                await Promise.allSettled(resourcePromises);
+                this.emit('page', event);
+                // Emit any new pages after the current page to increase responsiveness
+                const newPages = initialPrompts.map((prompt) => {
                     if (!this.hasPrompt(prompt.path)) {
                         this.emit('page', {
                             type: 'PAGE',
@@ -155,21 +163,24 @@ class PageQueue extends node_events_1.EventEmitter {
                             },
                         });
                     }
-                    this.addPrompt(prompt);
+                    return this.addPrompt(prompt);
                 });
-                this.emit('page', event);
+                await Promise.allSettled(newPages);
             }
-            finally {
-                future.resolve();
+            catch (e) {
+                console.error('Failed to process event', e);
+                throw e;
             }
         });
         return this.queue.add(() => generator.generate());
     }
     cancel() {
         this.queue.cancel();
+        this.eventQueue.cancel();
     }
-    wait() {
-        return this.queue.wait();
+    async wait() {
+        await this.eventQueue.wait();
+        await this.queue.wait();
     }
     async addImagePrompt(prompt) {
         if (this.images.has(prompt.url)) {
@@ -180,65 +191,54 @@ class PageQueue extends node_events_1.EventEmitter {
         const prefix = this.getPrefix();
         const result = await stormClient_1.stormClient.createImage(prefix + `Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim());
         //console.log('Adding image prompt', prompt);
-        const futures = [];
-        result.on('data', async (event) => {
+        result.on('data', (event) => {
             if (event.type === 'IMAGE') {
-                const future = (0, PromiseQueue_1.createFuture)();
-                futures.push(future);
-                this.emit('image', event, prompt, future);
-                setTimeout(() => {
-                    if (!future.isResolved()) {
-                        console.log('Image prompt timed out', prompt);
-                        future.reject(new Error('Image prompt timed out'));
-                    }
-                }, 30000);
+                this.emit('image', event, prompt);
             }
         });
         await result.waitForDone();
-        await Promise.allSettled(futures.map((f) => f.promise));
     }
 }
 exports.PageQueue = PageQueue;
 class PageGenerator extends node_events_1.EventEmitter {
+    eventQueue;
     conversationId;
     prompt;
     constructor(prompt, conversationId = node_uuid_1.default.v4()) {
         super();
         this.conversationId = conversationId;
         this.prompt = prompt;
+        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
     }
     on(event, listener) {
-        return super.on(event, listener);
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
     emit(eventName, ...args) {
         return super.emit(eventName, ...args);
     }
     async generate() {
-        return new Promise(async (resolve) => {
-            const promises = [];
-            const screenStream = await stormClient_1.stormClient.createUIPage(this.prompt, this.conversationId);
-            screenStream.on('data', (event) => {
-                if (event.type === 'PAGE') {
-                    event.payload.conversationId = this.conversationId;
-                    promises.push((async () => {
-                        const references = await this.resolveReferences(event.payload.content);
-                        //console.log('Resolved references for page', references, event.payload);
-                        const future = (0, PromiseQueue_1.createFuture)();
-                        this.emit('page_refs', {
-                            event,
-                            references,
-                            future,
-                        });
-                        await future.promise;
-                    })());
-                    return;
-                }
-                this.emit('event', event);
-            });
-            await screenStream.waitForDone();
-            await Promise.all(promises);
-            resolve();
+        const promises = [];
+        const screenStream = await stormClient_1.stormClient.createUIPage(this.prompt, this.conversationId);
+        screenStream.on('data', (event) => {
+            if (event.type === 'PAGE') {
+                event.payload.conversationId = this.conversationId;
+                promises.push((async () => {
+                    const references = await this.resolveReferences(event.payload.content);
+                    //console.log('Resolved references for page', references, event.payload);
+                    this.emit('page_refs', {
+                        event,
+                        references,
+                    });
+                })());
+                return;
+            }
+            this.emit('event', event);
         });
+        await screenStream.waitForDone();
+        await Promise.all(promises);
+        await this.eventQueue.wait();
     }
     async resolveReferences(content) {
         const referenceStream = await stormClient_1.stormClient.classifyUIReferences(content);

package/dist/esm/src/storm/page-utils.js

@@ -12,7 +12,7 @@ function normalizePath(path) {
     return path
         .replace(/\?.*$/gi, '')
         .replace(/:[a-z][a-z_]*\b/gi, '*')
-        .replace(/\{[a-z]+}/gi, '*');
+        .replace(/\{[a-z-.]+}/gi, '*');
 }
 async function writePageToDisk(systemId, event) {
     const baseDir = getSystemBaseDir(systemId);
@@ -53,13 +53,8 @@ async function writeImageToDisk(systemId, event, prompt) {
 }
 exports.writeImageToDisk = writeImageToDisk;
 function hasPageOnDisk(systemId, method, path) {
-    if (!systemId || !method || !path) {
-        return false;
-    }
-    const baseDir = getSystemBaseDir(systemId);
-    const filePath = getFilePath(method);
-    const fullPath = path_1.default.join(baseDir, normalizePath(path), filePath);
-    return fs_extra_1.default.existsSync(fullPath);
+    const fullPath = resolveReadPath(systemId, method, path);
+    return !!fullPath && fs_extra_1.default.existsSync(fullPath);
 }
 exports.hasPageOnDisk = hasPageOnDisk;
 function getSystemBaseDir(systemId) {
@@ -90,7 +85,7 @@ function resolveReadPath(systemId, path, method) {
     }
     const parts = path.split(/\*/g);
     let currentPath = '';
-    for (let part in parts) {
+    for (let part of parts) {
         const thisPath = path_1.default.join(currentPath, part);
         const starPath = path_1.default.join(currentPath, '*');
         const thisPathDir = path_1.default.join(baseDir, thisPath);
@@ -103,7 +98,6 @@ function resolveReadPath(systemId, path, method) {
             currentPath = starPath;
             continue;
         }
-        console.log('Path not found', thisPathDir, starPathDir);
         // Path not found
         return null;
     }

package/dist/esm/src/storm/routes.js

@@ -74,24 +74,17 @@ router.post('/ui/screen', async (req, res) => {
             queue.cancel();
         });
         const promises = [];
-        queue.on('page', (data) => {
-            if (systemId) {
-                promises.push(sendPageEvent(systemId, data, res));
-            }
-        });
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('page', (data) => (systemId ? sendPageEvent(systemId, data, res) : undefined));
+        queue.on('image', async (screenData, prompt) => {
             if (!systemId) {
                 return;
             }
             try {
-                const promise = handleImageEvent(systemId, screenData, prompt);
-                promises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(systemId, screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         await queue.addPrompt(aiRequest, conversationId, true);
@@ -186,19 +179,14 @@ router.post('/:handle/ui/iterative', async (req, res) => {
         onRequestAborted(req, res, () => {
             pageQueue.cancel();
         });
-        pageQueue.on('page', (screenData) => {
-            pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        });
-        pageQueue.on('image', async (screenData, prompt, future) => {
+        pageQueue.on('page', (screenData) => sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
+        pageQueue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         pageQueue.on('event', (screenData) => {
@@ -346,31 +334,25 @@ router.post('/:handle/ui', async (req, res) => {
             },
             created: Date.now(),
         });
-        const pagePromises = [];
         onRequestAborted(req, res, () => {
             queue.cancel();
         });
-        const pageEventPromises = [];
-        queue.on('page', (screenData) => {
-            pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
-        });
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('page', (screenData) => sendPageEvent(outerConversationId, screenData, res));
+        queue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(outerConversationId, screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(outerConversationId, screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         queue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
         for (const screen of Object.values(uniqueUserJourneyScreens)) {
-            pagePromises.push(queue.addPrompt({
+            queue
+                .addPrompt({
                 prompt: screen.requirements,
                 method: screen.method,
                 path: screen.path,
@@ -380,14 +362,15 @@ router.post('/:handle/ui', async (req, res) => {
                 filename: screen.filename,
                 storage_prefix: outerConversationId + '_',
                 theme,
-            }));
+            })
+                .catch((e) => {
+                console.error('Failed to generate page for screen %s', screen.name, e);
+            });
         }
         if (userJourneysStream.isAborted()) {
             return;
         }
         await queue.wait();
-        await Promise.allSettled(pagePromises);
-        await Promise.allSettled(pageEventPromises);
         sendDone(res);
     }
     catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.67.4",
+    "version": "0.67.5",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {

package/src/storm/PageGenerator.ts

@@ -7,7 +7,7 @@ import uuid from 'node-uuid';
 import { stormClient, UIPagePrompt } from './stormClient';
 import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
-import { createFuture, Future, FuturePromise, PromiseQueue } from './PromiseQueue';
+import { PromiseQueue } from './PromiseQueue';
 import { hasPageOnDisk } from './page-utils';
 
 export interface ImagePrompt {
@@ -24,6 +24,7 @@ type InitialPrompt = Omit<UIPagePrompt, 'shell_page'> & { shellType?: 'public' |
 
 export class PageQueue extends EventEmitter {
     private readonly queue: PromiseQueue;
+    private readonly eventQueue: PromiseQueue;
     private readonly systemId: string;
     private readonly systemPrompt: string;
     private readonly references: Map<string, PageGenerator> = new Map();
@@ -37,19 +38,22 @@ export class PageQueue extends EventEmitter {
         this.systemId = systemId;
         this.systemPrompt = systemPrompt;
         this.queue = new PromiseQueue(concurrency);
+        this.eventQueue = new PromiseQueue(Number.MAX_VALUE);
     }
 
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(event: 'page', listener: (data: StormEventPage) => void): this;
-    on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
+    on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
+    on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
+    on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
 
-    on(event: string, listener: (...args: any[]) => void): this {
-        return super.on(event, listener);
+    on(event: string, listener: (...args: any[]) => void | Promise<void>): this {
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
 
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
-    emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
+    emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;
     emit(eventName: string | symbol, ...args: any[]): boolean {
         return super.emit(eventName, ...args);
     }
@@ -81,6 +85,7 @@ export class PageQueue extends EventEmitter {
             //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
         }
+        console.log('Generating page for', initialPrompt.method, initialPrompt.path);
 
         const prompt: UIPagePrompt = {
             ...initialPrompt,
@@ -104,7 +109,7 @@ export class PageQueue extends EventEmitter {
     }
 
     private wrapPagePrompt(pagePath: string, prompt: string): string {
-        let promptPrefix = this.getPrefix();
+        const promptPrefix = this.getPrefix();
         let promptPostfix = '';
 
         if (this.pages.size > 0) {
@@ -129,10 +134,10 @@ export class PageQueue extends EventEmitter {
 
     private async addPageGenerator(generator: PageGenerator) {
         generator.on('event', (event: StormEvent) => this.emit('event', event));
-        generator.on('page_refs', async ({ event, references, future }) => {
+        generator.on('page_refs', async ({ event, references }) => {
             try {
                 const initialPrompts: InitialPrompt[] = [];
-                let promises = references.map(async (reference) => {
+                const resourcePromises = references.map(async (reference) => {
                     if (
                         reference.url.startsWith('#') ||
                         reference.url.startsWith('javascript:') ||
@@ -174,8 +179,12 @@ export class PageQueue extends EventEmitter {
                     }
                 });
 
-                await Promise.allSettled(promises);
-                initialPrompts.forEach((prompt) => {
+                // Wait for resources to be generated
+                await Promise.allSettled(resourcePromises);
+                this.emit('page', event);
+
+                // Emit any new pages after the current page to increase responsiveness
+                const newPages = initialPrompts.map((prompt) => {
                     if (!this.hasPrompt(prompt.path)) {
                         this.emit('page', {
                             type: 'PAGE',
@@ -194,12 +203,12 @@ export class PageQueue extends EventEmitter {
                             },
                         });
                     }
-                    this.addPrompt(prompt);
+                    return this.addPrompt(prompt);
                 });
-
-                this.emit('page', event);
-            } finally {
-                future.resolve();
+                await Promise.allSettled(newPages);
+            } catch (e) {
+                console.error('Failed to process event', e);
+                throw e;
             }
         });
         return this.queue.add(() => generator.generate());
@@ -207,10 +216,12 @@ export class PageQueue extends EventEmitter {
 
     public cancel() {
         this.queue.cancel();
+        this.eventQueue.cancel();
     }
 
-    public wait() {
-        return this.queue.wait();
+    public async wait() {
+        await this.eventQueue.wait();
+        await this.queue.wait();
     }
 
     private async addImagePrompt(prompt: ImagePrompt) {
@@ -225,29 +236,18 @@ export class PageQueue extends EventEmitter {
         );
 
         //console.log('Adding image prompt', prompt);
-
-        const futures: FuturePromise<void>[] = [];
-
-        result.on('data', async (event: StormEvent) => {
+        result.on('data', (event: StormEvent) => {
             if (event.type === 'IMAGE') {
-                const future = createFuture();
-                futures.push(future);
-                this.emit('image', event, prompt, future);
-                setTimeout(() => {
-                    if (!future.isResolved()) {
-                        console.log('Image prompt timed out', prompt);
-                        future.reject(new Error('Image prompt timed out'));
-                    }
-                }, 30000);
+                this.emit('image', event, prompt);
             }
         });
 
         await result.waitForDone();
-        await Promise.allSettled(futures.map((f) => f.promise));
     }
 }
 
 export class PageGenerator extends EventEmitter {
+    private readonly eventQueue: PromiseQueue;
     private readonly conversationId: string;
     public readonly prompt: UIPagePrompt;
 
@@ -255,64 +255,54 @@ export class PageGenerator extends EventEmitter {
         super();
         this.conversationId = conversationId;
         this.prompt = prompt;
+        this.eventQueue = new PromiseQueue(Number.MAX_VALUE);
     }
 
     on(event: 'event', listener: (data: StormEvent) => void): this;
     on(
         event: 'page_refs',
-        listener: (data: {
-            event: StormEventPage;
-            references: ReferenceClassification[];
-            future: FuturePromise<void>;
-        }) => void
+        listener: (data: { event: StormEventPage; references: ReferenceClassification[] }) => Promise<void>
     ): this;
 
     on(event: string, listener: (...args: any[]) => void): this {
-        return super.on(event, listener);
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
 
     emit(type: 'event', event: StormEvent): boolean;
-    emit(
-        type: 'page_refs',
-        event: { event: StormEventPage; references: ReferenceClassification[]; future: FuturePromise<void> }
-    ): boolean;
+    emit(type: 'page_refs', event: { event: StormEventPage; references: ReferenceClassification[] }): boolean;
     emit(eventName: string | symbol, ...args: any[]): boolean {
         return super.emit(eventName, ...args);
     }
 
     public async generate() {
-        return new Promise<void>(async (resolve) => {
-            const promises: Promise<void>[] = [];
-            const screenStream = await stormClient.createUIPage(this.prompt, this.conversationId);
-
-            screenStream.on('data', (event: StormEvent) => {
-                if (event.type === 'PAGE') {
-                    event.payload.conversationId = this.conversationId;
-
-                    promises.push(
-                        (async () => {
-                            const references = await this.resolveReferences(event.payload.content);
-                            //console.log('Resolved references for page', references, event.payload);
-                            const future = createFuture();
-                            this.emit('page_refs', {
-                                event,
-                                references,
-                                future,
-                            });
-
-                            await future.promise;
-                        })()
-                    );
-                    return;
-                }
-
-                this.emit('event', event);
-            });
+        const promises: Promise<void>[] = [];
+        const screenStream = await stormClient.createUIPage(this.prompt, this.conversationId);
+
+        screenStream.on('data', (event: StormEvent) => {
+            if (event.type === 'PAGE') {
+                event.payload.conversationId = this.conversationId;
+
+                promises.push(
+                    (async () => {
+                        const references = await this.resolveReferences(event.payload.content);
+                        //console.log('Resolved references for page', references, event.payload);
+                        this.emit('page_refs', {
+                            event,
+                            references,
+                        });
+                    })()
+                );
+                return;
+            }
 
-            await screenStream.waitForDone();
-            await Promise.all(promises);
-            resolve();
+            this.emit('event', event);
         });
+
+        await screenStream.waitForDone();
+        await Promise.all(promises);
+        await this.eventQueue.wait();
     }
 
     private async resolveReferences(content: string) {

package/src/storm/page-utils.ts

@@ -19,7 +19,7 @@ function normalizePath(path: string) {
     return path
         .replace(/\?.*$/gi, '')
         .replace(/:[a-z][a-z_]*\b/gi, '*')
-        .replace(/\{[a-z]+}/gi, '*');
+        .replace(/\{[a-z-.]+}/gi, '*');
 }
 
 export async function writePageToDisk(systemId: string, event: StormEventPage) {
@@ -69,13 +69,8 @@ export async function writeImageToDisk(systemId: string, event: StormImage, prom
 }
 
 export function hasPageOnDisk(systemId: string, method: string, path: string) {
-    if (!systemId || !method || !path) {
-        return false;
-    }
-    const baseDir = getSystemBaseDir(systemId);
-    const filePath = getFilePath(method);
-    const fullPath = Path.join(baseDir, normalizePath(path), filePath);
-    return FS.existsSync(fullPath);
+    const fullPath = resolveReadPath(systemId, method, path);
+    return !!fullPath && FS.existsSync(fullPath);
 }
 
 export function getSystemBaseDir(systemId: string) {
@@ -114,7 +109,7 @@ export function resolveReadPath(systemId: string, path: string, method: string)
 
     let currentPath = '';
 
-    for (let part in parts) {
+    for (let part of parts) {
         const thisPath = Path.join(currentPath, part);
         const starPath = Path.join(currentPath, '*');
         const thisPathDir = Path.join(baseDir, thisPath);
@@ -130,7 +125,6 @@ export function resolveReadPath(systemId: string, path: string, method: string)
             continue;
         }
 
-        console.log('Path not found', thisPathDir, starPathDir);
         // Path not found
         return null;
     }

package/src/storm/routes.ts

@@ -102,24 +102,17 @@ router.post('/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
 
         const promises: Promise<void>[] = [];
 
-        queue.on('page', (data) => {
-            if (systemId) {
-                promises.push(sendPageEvent(systemId, data, res));
-            }
-        });
+        queue.on('page', (data) => (systemId ? sendPageEvent(systemId, data, res) : undefined));
 
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('image', async (screenData, prompt) => {
             if (!systemId) {
                 return;
             }
             try {
-                const promise = handleImageEvent(systemId, screenData, prompt);
-                promises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(systemId, screenData, prompt);
             } catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
 
@@ -229,19 +222,16 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
             pageQueue.cancel();
         });
 
-        pageQueue.on('page', (screenData: StormEventPage) => {
-            pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        });
+        pageQueue.on('page', (screenData: StormEventPage) =>
+            sendPageEvent(landingPagesStream.getConversationId(), screenData, res)
+        );
 
-        pageQueue.on('image', async (screenData, prompt, future) => {
+        pageQueue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
             } catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
 
@@ -420,25 +410,18 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
             created: Date.now(),
         });
 
-        const pagePromises: Promise<void>[] = [];
         onRequestAborted(req, res, () => {
             queue.cancel();
         });
 
-        const pageEventPromises: Promise<void>[] = [];
-        queue.on('page', (screenData: StormEventPage) => {
-            pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
-        });
+        queue.on('page', (screenData: StormEventPage) => sendPageEvent(outerConversationId, screenData, res));
 
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(outerConversationId, screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(outerConversationId, screenData, prompt);
             } catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
 
@@ -447,8 +430,8 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
         });
 
         for (const screen of Object.values(uniqueUserJourneyScreens)) {
-            pagePromises.push(
-                queue.addPrompt({
+            queue
+                .addPrompt({
                     prompt: screen.requirements,
                     method: screen.method,
                     path: screen.path,
@@ -459,7 +442,9 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
                     storage_prefix: outerConversationId + '_',
                     theme,
                 })
-            );
+                .catch((e) => {
+                    console.error('Failed to generate page for screen %s', screen.name, e);
+                });
         }
 
         if (userJourneysStream.isAborted()) {
@@ -467,8 +452,6 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
         }
 
         await queue.wait();
-        await Promise.allSettled(pagePromises);
-        await Promise.allSettled(pageEventPromises);
 
         sendDone(res);
     } catch (err) {