@kapeta/local-cluster-service 0.67.4 → 0.68.0

package/CHANGELOG.md

@@ -1,3 +1,18 @@
+# [0.68.0](https://github.com/kapetacom/local-cluster-service/compare/v0.67.5...v0.68.0) (2024-09-03)
+
+
+### Features
+
+* Add fallback page to ui server requests ([45bcf5d](https://github.com/kapetacom/local-cluster-service/commit/45bcf5d596ca359a9ddd63512b705ca840a555c8))
+
+## [0.67.5](https://github.com/kapetacom/local-cluster-service/compare/v0.67.4...v0.67.5) (2024-09-02)
+
+
+### Bug Fixes
+
+* better tracking of async events [CORE-3409] ([5e8f8cd](https://github.com/kapetacom/local-cluster-service/commit/5e8f8cdbe7e714cf0dfb5b0e7cbb8b60f837c2f9))
+* dont generate individual pages when page matches path w/ params ([c3ca35c](https://github.com/kapetacom/local-cluster-service/commit/c3ca35ce56781ca90f3ca1070fee76aa3f5512ab))
+
 ## [0.67.4](https://github.com/kapetacom/local-cluster-service/compare/v0.67.3...v0.67.4) (2024-08-30)
 
 

package/dist/cjs/src/storm/PageGenerator.d.ts

@@ -6,7 +6,6 @@
 import { UIPagePrompt } from './stormClient';
 import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
-import { FuturePromise } from './PromiseQueue';
 export interface ImagePrompt {
     name: string;
     description: string;
@@ -21,6 +20,7 @@ type InitialPrompt = Omit<UIPagePrompt, 'shell_page'> & {
 };
 export declare class PageQueue extends EventEmitter {
     private readonly queue;
+    private readonly eventQueue;
     private readonly systemId;
     private readonly systemPrompt;
     private readonly references;
@@ -29,12 +29,12 @@ export declare class PageQueue extends EventEmitter {
     private uiShells;
     private theme;
     constructor(systemId: string, systemPrompt: string, concurrency?: number);
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(event: 'page', listener: (data: StormEventPage) => void): this;
-    on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
+    on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
+    on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
+    on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
-    emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
+    emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;
     addUiShell(uiShell: UIShell): void;
     setUiTheme(theme: string): void;
     private hasPrompt;
@@ -47,6 +47,7 @@ export declare class PageQueue extends EventEmitter {
     private addImagePrompt;
 }
 export declare class PageGenerator extends EventEmitter {
+    private readonly eventQueue;
     private readonly conversationId;
     readonly prompt: UIPagePrompt;
     constructor(prompt: UIPagePrompt, conversationId?: string);
@@ -54,13 +55,11 @@ export declare class PageGenerator extends EventEmitter {
     on(event: 'page_refs', listener: (data: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
-    }) => void): this;
+    }) => Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page_refs', event: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
     }): boolean;
     generate(): Promise<void>;
     private resolveReferences;

package/dist/cjs/src/storm/PageGenerator.js

@@ -15,6 +15,7 @@ const PromiseQueue_1 = require("./PromiseQueue");
 const page_utils_1 = require("./page-utils");
 class PageQueue extends node_events_1.EventEmitter {
     queue;
+    eventQueue;
     systemId;
     systemPrompt;
     references = new Map();
@@ -27,9 +28,12 @@ class PageQueue extends node_events_1.EventEmitter {
         this.systemId = systemId;
         this.systemPrompt = systemPrompt;
         this.queue = new PromiseQueue_1.PromiseQueue(concurrency);
+        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
     }
     on(event, listener) {
-        return super.on(event, listener);
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
     emit(eventName, ...args) {
         return super.emit(eventName, ...args);
@@ -56,6 +60,7 @@ class PageQueue extends node_events_1.EventEmitter {
             //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
         }
+        console.log('Generating page for', initialPrompt.method, initialPrompt.path);
         const prompt = {
             ...initialPrompt,
             shell_page: this.uiShells.find((shell) => shell.screens.includes(initialPrompt.name))?.content,
@@ -75,7 +80,7 @@ class PageQueue extends node_events_1.EventEmitter {
         return promptPrefix;
     }
     wrapPagePrompt(pagePath, prompt) {
-        let promptPrefix = this.getPrefix();
+        const promptPrefix = this.getPrefix();
         let promptPostfix = '';
         if (this.pages.size > 0) {
             promptPostfix = `\nThe following pages are already implemented:\n`;
@@ -96,10 +101,10 @@ class PageQueue extends node_events_1.EventEmitter {
     }
     async addPageGenerator(generator) {
         generator.on('event', (event) => this.emit('event', event));
-        generator.on('page_refs', async ({ event, references, future }) => {
+        generator.on('page_refs', async ({ event, references }) => {
             try {
                 const initialPrompts = [];
-                let promises = references.map(async (reference) => {
+                const resourcePromises = references.map(async (reference) => {
                     if (reference.url.startsWith('#') ||
                         reference.url.startsWith('javascript:') ||
                         reference.url.startsWith('http://') ||
@@ -135,8 +140,11 @@ class PageQueue extends node_events_1.EventEmitter {
                             break;
                     }
                 });
-                await Promise.allSettled(promises);
-                initialPrompts.forEach((prompt) => {
+                // Wait for resources to be generated
+                await Promise.allSettled(resourcePromises);
+                this.emit('page', event);
+                // Emit any new pages after the current page to increase responsiveness
+                const newPages = initialPrompts.map((prompt) => {
                     if (!this.hasPrompt(prompt.path)) {
                         this.emit('page', {
                             type: 'PAGE',
@@ -155,21 +163,24 @@ class PageQueue extends node_events_1.EventEmitter {
                             },
                         });
                     }
-                    this.addPrompt(prompt);
+                    return this.addPrompt(prompt);
                 });
-                this.emit('page', event);
+                await Promise.allSettled(newPages);
             }
-            finally {
-                future.resolve();
+            catch (e) {
+                console.error('Failed to process event', e);
+                throw e;
             }
         });
         return this.queue.add(() => generator.generate());
     }
     cancel() {
         this.queue.cancel();
+        this.eventQueue.cancel();
     }
-    wait() {
-        return this.queue.wait();
+    async wait() {
+        await this.eventQueue.wait();
+        await this.queue.wait();
     }
     async addImagePrompt(prompt) {
         if (this.images.has(prompt.url)) {
@@ -180,65 +191,54 @@ class PageQueue extends node_events_1.EventEmitter {
         const prefix = this.getPrefix();
         const result = await stormClient_1.stormClient.createImage(prefix + `Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim());
         //console.log('Adding image prompt', prompt);
-        const futures = [];
-        result.on('data', async (event) => {
+        result.on('data', (event) => {
             if (event.type === 'IMAGE') {
-                const future = (0, PromiseQueue_1.createFuture)();
-                futures.push(future);
-                this.emit('image', event, prompt, future);
-                setTimeout(() => {
-                    if (!future.isResolved()) {
-                        console.log('Image prompt timed out', prompt);
-                        future.reject(new Error('Image prompt timed out'));
-                    }
-                }, 30000);
+                this.emit('image', event, prompt);
             }
         });
         await result.waitForDone();
-        await Promise.allSettled(futures.map((f) => f.promise));
     }
 }
 exports.PageQueue = PageQueue;
 class PageGenerator extends node_events_1.EventEmitter {
+    eventQueue;
     conversationId;
     prompt;
     constructor(prompt, conversationId = node_uuid_1.default.v4()) {
         super();
         this.conversationId = conversationId;
         this.prompt = prompt;
+        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
     }
     on(event, listener) {
-        return super.on(event, listener);
+        return super.on(event, (...args) => {
+            void this.eventQueue.add(async () => listener(...args));
+        });
     }
     emit(eventName, ...args) {
         return super.emit(eventName, ...args);
     }
     async generate() {
-        return new Promise(async (resolve) => {
-            const promises = [];
-            const screenStream = await stormClient_1.stormClient.createUIPage(this.prompt, this.conversationId);
-            screenStream.on('data', (event) => {
-                if (event.type === 'PAGE') {
-                    event.payload.conversationId = this.conversationId;
-                    promises.push((async () => {
-                        const references = await this.resolveReferences(event.payload.content);
-                        //console.log('Resolved references for page', references, event.payload);
-                        const future = (0, PromiseQueue_1.createFuture)();
-                        this.emit('page_refs', {
-                            event,
-                            references,
-                            future,
-                        });
-                        await future.promise;
-                    })());
-                    return;
-                }
-                this.emit('event', event);
-            });
-            await screenStream.waitForDone();
-            await Promise.all(promises);
-            resolve();
+        const promises = [];
+        const screenStream = await stormClient_1.stormClient.createUIPage(this.prompt, this.conversationId);
+        screenStream.on('data', (event) => {
+            if (event.type === 'PAGE') {
+                event.payload.conversationId = this.conversationId;
+                promises.push((async () => {
+                    const references = await this.resolveReferences(event.payload.content);
+                    //console.log('Resolved references for page', references, event.payload);
+                    this.emit('page_refs', {
+                        event,
+                        references,
+                    });
+                })());
+                return;
+            }
+            this.emit('event', event);
         });
+        await screenStream.waitForDone();
+        await Promise.all(promises);
+        await this.eventQueue.wait();
     }
     async resolveReferences(content) {
         const referenceStream = await stormClient_1.stormClient.classifyUIReferences(content);

package/dist/cjs/src/storm/page-utils.js

@@ -12,7 +12,7 @@ function normalizePath(path) {
     return path
         .replace(/\?.*$/gi, '')
         .replace(/:[a-z][a-z_]*\b/gi, '*')
-        .replace(/\{[a-z]+}/gi, '*');
+        .replace(/\{[a-z-.]+}/gi, '*');
 }
 async function writePageToDisk(systemId, event) {
     const baseDir = getSystemBaseDir(systemId);
@@ -53,13 +53,8 @@ async function writeImageToDisk(systemId, event, prompt) {
 }
 exports.writeImageToDisk = writeImageToDisk;
 function hasPageOnDisk(systemId, method, path) {
-    if (!systemId || !method || !path) {
-        return false;
-    }
-    const baseDir = getSystemBaseDir(systemId);
-    const filePath = getFilePath(method);
-    const fullPath = path_1.default.join(baseDir, normalizePath(path), filePath);
-    return fs_extra_1.default.existsSync(fullPath);
+    const fullPath = resolveReadPath(systemId, method, path);
+    return !!fullPath && fs_extra_1.default.existsSync(fullPath);
 }
 exports.hasPageOnDisk = hasPageOnDisk;
 function getSystemBaseDir(systemId) {
@@ -90,7 +85,7 @@ function resolveReadPath(systemId, path, method) {
     }
     const parts = path.split(/\*/g);
     let currentPath = '';
-    for (let part in parts) {
+    for (let part of parts) {
         const thisPath = path_1.default.join(currentPath, part);
         const starPath = path_1.default.join(currentPath, '*');
         const thisPathDir = path_1.default.join(baseDir, thisPath);
@@ -103,7 +98,6 @@ function resolveReadPath(systemId, path, method) {
             currentPath = starPath;
             continue;
         }
-        console.log('Path not found', thisPathDir, starPathDir);
         // Path not found
         return null;
     }
@@ -121,15 +115,101 @@ exports.readPageFromDiskAsString = readPageFromDiskAsString;
 function readPageFromDisk(systemId, path, method, res) {
     const filePath = resolveReadPath(systemId, path, method);
     if (!filePath || !fs_extra_1.default.existsSync(filePath)) {
-        res.status(404).send('Page not found');
+        if (method === 'HEAD') {
+            // For HEAD requests, only return the status and headers
+            res.status(202).set('Retry-After', '3').end();
+        }
+        else {
+            // For GET requests, return the fallback HTML with status 202
+            res.status(202).set('Retry-After', '3').send(getFallbackHtml(path, method));
+        }
         return;
     }
     res.type(filePath.split('.').pop());
     const content = fs_extra_1.default.readFileSync(filePath);
-    res.write(content);
-    res.end();
+    if (method === 'HEAD') {
+        // For HEAD requests, just end the response after setting headers
+        res.status(200).end();
+    }
+    else {
+        // For GET requests, return the full content
+        res.write(content);
+        res.end();
+    }
 }
 exports.readPageFromDisk = readPageFromDisk;
+function getFallbackHtml(path, method) {
+    return `
+        <!DOCTYPE html>
+        <html lang="en">
+
+        <head>
+            <meta charset="UTF-8">
+            <meta name="viewport" content="width=device-width, initial-scale=1.0">
+            <title>Page Not Ready</title>
+            <style>
+                @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@400;600&display=swap');
+
+                body {
+                    margin: 0;
+                    padding: 0;
+                    display: flex;
+                    align-items: center;
+                    justify-content: center;
+                    height: 100vh;
+                    font-family: 'Roboto', sans-serif;
+                    background: #1E1F20;
+                    color: white;
+                    text-align: center;
+                }
+
+                h1 {
+                    font-size: 2rem;
+                    font-weight: 600;
+                    margin-bottom: 1rem;
+                }
+
+                p {
+                    font-size: 1rem;
+                    font-weight: 400;
+                }
+            </style>
+        </head>
+
+        <body>
+            <div>
+                <h1>Page Not Ready</h1>
+                <p>Henrik is still working on this page. Please wait...</p>
+            </div>
+            <script>
+                const checkInterval = 3000;
+                function checkPageReady() {
+                    fetch('${path}', { method: 'HEAD' })
+                        .then(response => {
+                            if (response.status === 200) {
+                                // The page is ready, reload to fetch it
+                                window.location.reload(); 
+                            } else if (response.status === 202) {
+                                const retryAfter = response.headers.get('Retry-After');
+                                const retryInterval = retryAfter ? parseInt(retryAfter) * 1000 : 3000;
+                                setTimeout(checkPageReady, retryInterval);
+                            } else {
+                                // Handle other unexpected statuses
+                                setTimeout(checkPageReady, 3000);
+                            }
+                        })
+                        .catch(error => {
+                            console.error('Error checking page status:', error);
+                            setTimeout(checkPageReady, checkInterval);
+                        });
+                }
+                setTimeout(checkPageReady, checkInterval); 
+            </script>
+        </body>
+
+        </html>        
+    `;
+}
 function readConversationFromFile(filename) {
     if (!fs_extra_1.default.existsSync(filename)) {
         return [];

package/dist/cjs/src/storm/routes.js

@@ -74,24 +74,17 @@ router.post('/ui/screen', async (req, res) => {
             queue.cancel();
         });
         const promises = [];
-        queue.on('page', (data) => {
-            if (systemId) {
-                promises.push(sendPageEvent(systemId, data, res));
-            }
-        });
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('page', (data) => (systemId ? sendPageEvent(systemId, data, res) : undefined));
+        queue.on('image', async (screenData, prompt) => {
             if (!systemId) {
                 return;
             }
             try {
-                const promise = handleImageEvent(systemId, screenData, prompt);
-                promises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(systemId, screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         await queue.addPrompt(aiRequest, conversationId, true);
@@ -186,19 +179,14 @@ router.post('/:handle/ui/iterative', async (req, res) => {
         onRequestAborted(req, res, () => {
             pageQueue.cancel();
         });
-        pageQueue.on('page', (screenData) => {
-            pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
-        });
-        pageQueue.on('image', async (screenData, prompt, future) => {
+        pageQueue.on('page', (screenData) => sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
+        pageQueue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         pageQueue.on('event', (screenData) => {
@@ -346,31 +334,25 @@ router.post('/:handle/ui', async (req, res) => {
             },
             created: Date.now(),
         });
-        const pagePromises = [];
         onRequestAborted(req, res, () => {
             queue.cancel();
         });
-        const pageEventPromises = [];
-        queue.on('page', (screenData) => {
-            pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
-        });
-        queue.on('image', async (screenData, prompt, future) => {
+        queue.on('page', (screenData) => sendPageEvent(outerConversationId, screenData, res));
+        queue.on('image', async (screenData, prompt) => {
             try {
-                const promise = handleImageEvent(outerConversationId, screenData, prompt);
-                pageEventPromises.push(promise);
-                await promise;
-                future.resolve();
+                await handleImageEvent(outerConversationId, screenData, prompt);
             }
             catch (e) {
                 console.error('Failed to handle image event', e);
-                future.reject(e);
+                throw e;
             }
         });
         queue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
         for (const screen of Object.values(uniqueUserJourneyScreens)) {
-            pagePromises.push(queue.addPrompt({
+            queue
+                .addPrompt({
                 prompt: screen.requirements,
                 method: screen.method,
                 path: screen.path,
@@ -380,14 +362,15 @@ router.post('/:handle/ui', async (req, res) => {
                 filename: screen.filename,
                 storage_prefix: outerConversationId + '_',
                 theme,
-            }));
+            })
+                .catch((e) => {
+                console.error('Failed to generate page for screen %s', screen.name, e);
+            });
         }
         if (userJourneysStream.isAborted()) {
             return;
         }
         await queue.wait();
-        await Promise.allSettled(pagePromises);
-        await Promise.allSettled(pageEventPromises);
         sendDone(res);
     }
     catch (err) {

package/dist/esm/src/storm/PageGenerator.d.ts

@@ -6,7 +6,6 @@
 import { UIPagePrompt } from './stormClient';
 import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
-import { FuturePromise } from './PromiseQueue';
 export interface ImagePrompt {
     name: string;
     description: string;
@@ -21,6 +20,7 @@ type InitialPrompt = Omit<UIPagePrompt, 'shell_page'> & {
 };
 export declare class PageQueue extends EventEmitter {
     private readonly queue;
+    private readonly eventQueue;
     private readonly systemId;
     private readonly systemPrompt;
     private readonly references;
@@ -29,12 +29,12 @@ export declare class PageQueue extends EventEmitter {
     private uiShells;
     private theme;
     constructor(systemId: string, systemPrompt: string, concurrency?: number);
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(event: 'page', listener: (data: StormEventPage) => void): this;
-    on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
+    on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
+    on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
+    on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
-    emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
+    emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;
     addUiShell(uiShell: UIShell): void;
     setUiTheme(theme: string): void;
     private hasPrompt;
@@ -47,6 +47,7 @@ export declare class PageQueue extends EventEmitter {
     private addImagePrompt;
 }
 export declare class PageGenerator extends EventEmitter {
+    private readonly eventQueue;
     private readonly conversationId;
     readonly prompt: UIPagePrompt;
     constructor(prompt: UIPagePrompt, conversationId?: string);
@@ -54,13 +55,11 @@ export declare class PageGenerator extends EventEmitter {
     on(event: 'page_refs', listener: (data: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
-    }) => void): this;
+    }) => Promise<void>): this;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page_refs', event: {
         event: StormEventPage;
         references: ReferenceClassification[];
-        future: FuturePromise<void>;
     }): boolean;
     generate(): Promise<void>;
     private resolveReferences;