@kapeta/local-cluster-service 0.70.4 → 0.70.6

package/CHANGELOG.md

@@ -1,3 +1,19 @@
+## [0.70.6](https://github.com/kapetacom/local-cluster-service/compare/v0.70.5...v0.70.6) (2024-09-11)
+
+
+### Bug Fixes
+
+* disable sentry in dev mode ([9a7793f](https://github.com/kapetacom/local-cluster-service/commit/9a7793feaf81239d600d2b6650f126780f0ffdd5))
+* emit errors to res instead of into void ([d1302c1](https://github.com/kapetacom/local-cluster-service/commit/d1302c1b523a72de6596b619a44a2cde073f032f))
+* promise error handling and fetch retries ([d746e14](https://github.com/kapetacom/local-cluster-service/commit/d746e14c1edaaddd1a3aa5cb7bbf644dfde334b2))
+
+## [0.70.5](https://github.com/kapetacom/local-cluster-service/compare/v0.70.4...v0.70.5) (2024-09-11)
+
+
+### Bug Fixes
+
+* strip hash of URLs for refs ([#242](https://github.com/kapetacom/local-cluster-service/issues/242)) ([681a3fd](https://github.com/kapetacom/local-cluster-service/commit/681a3fdcfd837eb4d0d3411ee719e2dc68334522))
+
 ## [0.70.4](https://github.com/kapetacom/local-cluster-service/compare/v0.70.3...v0.70.4) (2024-09-10)
 
 

package/dist/cjs/index.js

@@ -64,7 +64,7 @@ const assetManager_1 = require("./src/assetManager");
 const instanceManager_1 = require("./src/instanceManager");
 Sentry.init({
     dsn: 'https://0b7cc946d82c591473d6f95fff5e210b@o4505820837249024.ingest.sentry.io/4506212692000768',
-    enabled: process.env.NODE_ENV !== 'development',
+    enabled: !!process.env.NODE_ENV && process.env.NODE_ENV !== 'development',
     // Performance Monitoring on every ~20th request
     tracesSampleRate: 0.05,
     // Set sampling rate for profiling - this is relative to tracesSampleRate

package/dist/cjs/src/storm/PageGenerator.d.ts

@@ -29,9 +29,11 @@ export declare class PageQueue extends EventEmitter {
     private uiShells;
     private theme;
     constructor(systemId: string, systemPrompt: string, concurrency?: number);
+    on(event: 'error', listener: (error: unknown) => void): this;
     on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
     on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
     on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
+    emit(type: 'error', error: unknown): boolean;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
     emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;

package/dist/cjs/src/storm/PageGenerator.js

@@ -32,7 +32,12 @@ class PageQueue extends node_events_1.EventEmitter {
     }
     on(event, listener) {
         return super.on(event, (...args) => {
-            void this.eventQueue.add(async () => listener(...args));
+            this.eventQueue
+                .add(async () => listener(...args))
+                // If the event queue fails, we want to emit an error
+                .catch((err) => {
+                this.emit('error', err);
+            });
         });
     }
     emit(eventName, ...args) {
@@ -121,6 +126,9 @@ class PageQueue extends node_events_1.EventEmitter {
                         await this.addImagePrompt({
                             ...reference,
                             content: event.payload.content,
+                        }).catch((err) => {
+                            console.error('Failed to generate image for reference', reference.name, err);
+                            this.emit('error', err);
                         });
                         break;
                     case 'css':
@@ -154,7 +162,7 @@ class PageQueue extends node_events_1.EventEmitter {
             await Promise.allSettled(resourcePromises);
             this.emit('page', event);
             // Emit any new pages after the current page to increase responsiveness
-            void initialPrompts.map((prompt) => {
+            initialPrompts.forEach((prompt) => {
                 if (!this.hasPrompt(prompt.path)) {
                     this.emit('page', {
                         type: 'PAGE',
@@ -173,7 +181,11 @@ class PageQueue extends node_events_1.EventEmitter {
                         },
                     });
                 }
-                return this.addPrompt(prompt);
+                // Trigger but don't wait for the "bonus" pages
+                this.addPrompt(prompt).catch((err) => {
+                    console.error('Failed to generate page reference', prompt.name, err);
+                    this.emit('error', err);
+                });
             });
         }
         catch (e) {
@@ -208,18 +220,21 @@ class PageQueue extends node_events_1.EventEmitter {
         await result.waitForDone();
     }
     async generate(prompt, conversationId) {
-        const promises = [];
         const screenStream = await stormClient_1.stormClient.createUIPage(prompt, conversationId);
+        let pageEvent = null;
         screenStream.on('data', (event) => {
             if (event.type === 'PAGE') {
                 event.payload.conversationId = conversationId;
-                promises.push(this.processPageEventWithReferences(event));
+                pageEvent = event;
                 return;
             }
             this.emit('event', event);
         });
         await screenStream.waitForDone();
-        await Promise.all(promises);
+        if (!pageEvent) {
+            throw new Error('No page was generated');
+        }
+        await this.processPageEventWithReferences(pageEvent);
     }
     async resolveReferences(content) {
         const referenceStream = await stormClient_1.stormClient.classifyUIReferences(content);

package/dist/cjs/src/storm/page-utils.js

@@ -10,9 +10,10 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 exports.SystemIdHeader = 'System-Id';
 function normalizePath(path) {
     return path
-        .replace(/\?.*$/gi, '')
-        .replace(/:[a-z][a-z_]*\b/gi, '*')
-        .replace(/\{[a-z-.]+}/gi, '*');
+        .replace(/#.*$/g, '') // Remove hash
+        .replace(/\?.*$/gi, '') // Remove query params
+        .replace(/:[a-z][a-z_]*\b/gi, '*') // Replace all params with *
+        .replace(/\{[a-z-.]+}/gi, '*'); // Replace all variables with *
 }
 exports.normalizePath = normalizePath;
 async function writePageToDisk(systemId, event) {

package/dist/cjs/src/storm/routes.js

@@ -124,6 +124,10 @@ router.post('/ui/screen', async (req, res) => {
                 throw e;
             }
         });
+        queue.on('error', (err) => {
+            console.error('Failed to process page', err);
+            sendError(err, res);
+        });
         await queue.addPrompt(aiRequest, conversationId, true);
         await queue.wait();
         await Promise.allSettled(promises);
@@ -214,6 +218,10 @@ router.post('/:handle/ui/iterative', async (req, res) => {
         pageQueue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
+        pageQueue.on('error', (err) => {
+            console.error('Failed to process page', err);
+            sendError(err, res);
+        });
         await waitForStormStream(landingPagesStream);
         await pageQueue.wait();
         await Promise.allSettled(pageEventPromises);

package/dist/cjs/src/storm/stormClient.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { ConversationItem, ImplementAPIClientsRequest, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt, StormUIListPrompt } from './stream';
 import { Page, StormEventPageUrl } from './events';
 export declare const STORM_ID = "storm";
@@ -66,7 +65,7 @@ declare class StormClient {
     createUIShells(prompt: UIShellsPrompt, conversationId?: string): Promise<StormStream>;
     createUILandingPages(prompt: BasePromptRequest, conversationId?: string): Promise<StormStream>;
     createUIPage(prompt: UIPagePrompt, conversationId?: string, history?: ConversationItem[]): Promise<StormStream>;
-    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<Response>;
+    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<import("undici").Response>;
     getVoteUIPage(topic: string, conversationId: string, mainConversationId?: string): Promise<{
         vote: -1 | 0 | 1;
     }>;

package/dist/cjs/src/storm/stormClient.js

@@ -13,6 +13,22 @@ const utils_1 = require("../utils/utils");
 const promises_1 = __importDefault(require("node:readline/promises"));
 const node_stream_1 = require("node:stream");
 const stream_1 = require("./stream");
+const undici_1 = require("undici");
+// Will only retry on error codes and GET requests by default
+// See https://github.com/nodejs/undici/blob/990df2c7e37cbe5bb44fe2f576dddeaeb5916590/docs/docs/api/RetryAgent.md
+const retryAgent = new undici_1.RetryAgent(new undici_1.Agent(), {
+    methods: [
+        // Added methods ↓ (not idempotent), but we want to retry on POST:
+        'POST',
+        // defaults below
+        'GET',
+        'HEAD',
+        'OPTIONS',
+        'PUT',
+        'DELETE',
+        'TRACE',
+    ],
+});
 exports.STORM_ID = 'storm';
 exports.ConversationIdHeader = 'Conversation-Id';
 class StormClient {
@@ -38,6 +54,7 @@ class StormClient {
             method: method,
             body: JSON.stringify(body),
             headers,
+            dispatcher: retryAgent,
         };
     }
     async send(path, body, method = 'POST') {
@@ -48,7 +65,7 @@ class StormClient {
         });
         const abort = new AbortController();
         options.signal = abort.signal;
-        const response = await fetch(options.url, options);
+        const response = await (0, undici_1.fetch)(options.url, options);
         if (response.status !== 200) {
             throw new Error(`Got error response from ${options.url}: ${response.status}\nContent: ${await response.text()}`);
         }
@@ -121,25 +138,19 @@ class StormClient {
             prompt: JSON.stringify({ topic, vote, mainConversationId }),
             conversationId,
         });
-        return fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        return (0, undici_1.fetch)(options.url, options);
     }
     async getVoteUIPage(topic, conversationId, mainConversationId) {
         const options = await this.createOptions('/v2/ui/get-vote', 'POST', {
             prompt: JSON.stringify({ topic, mainConversationId }),
             conversationId,
         });
-        const response = await fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        const response = await (0, undici_1.fetch)(options.url, options);
         return response.json();
     }
     async implementAPIClients(prompt) {
         const u = `${this._baseUrl}/v2/ui/implement-api-clients`;
-        const response = await fetch(u, {
+        const response = await (0, undici_1.fetch)(u, {
             method: 'POST',
             body: JSON.stringify({
                 fileName: prompt.fileName,
@@ -214,10 +225,7 @@ class StormClient {
             prompt: '',
             conversationId: conversationId,
         });
-        const response = await fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        const response = await (0, undici_1.fetch)(options.url, options);
         return response.text();
     }
 }

package/dist/esm/index.js

@@ -64,7 +64,7 @@ const assetManager_1 = require("./src/assetManager");
 const instanceManager_1 = require("./src/instanceManager");
 Sentry.init({
     dsn: 'https://0b7cc946d82c591473d6f95fff5e210b@o4505820837249024.ingest.sentry.io/4506212692000768',
-    enabled: process.env.NODE_ENV !== 'development',
+    enabled: !!process.env.NODE_ENV && process.env.NODE_ENV !== 'development',
     // Performance Monitoring on every ~20th request
     tracesSampleRate: 0.05,
     // Set sampling rate for profiling - this is relative to tracesSampleRate

package/dist/esm/src/storm/PageGenerator.d.ts

@@ -29,9 +29,11 @@ export declare class PageQueue extends EventEmitter {
     private uiShells;
     private theme;
     constructor(systemId: string, systemPrompt: string, concurrency?: number);
+    on(event: 'error', listener: (error: unknown) => void): this;
     on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
     on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
     on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
+    emit(type: 'error', error: unknown): boolean;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
     emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;

package/dist/esm/src/storm/PageGenerator.js

@@ -32,7 +32,12 @@ class PageQueue extends node_events_1.EventEmitter {
     }
     on(event, listener) {
         return super.on(event, (...args) => {
-            void this.eventQueue.add(async () => listener(...args));
+            this.eventQueue
+                .add(async () => listener(...args))
+                // If the event queue fails, we want to emit an error
+                .catch((err) => {
+                this.emit('error', err);
+            });
         });
     }
     emit(eventName, ...args) {
@@ -121,6 +126,9 @@ class PageQueue extends node_events_1.EventEmitter {
                         await this.addImagePrompt({
                             ...reference,
                             content: event.payload.content,
+                        }).catch((err) => {
+                            console.error('Failed to generate image for reference', reference.name, err);
+                            this.emit('error', err);
                         });
                         break;
                     case 'css':
@@ -154,7 +162,7 @@ class PageQueue extends node_events_1.EventEmitter {
             await Promise.allSettled(resourcePromises);
             this.emit('page', event);
             // Emit any new pages after the current page to increase responsiveness
-            void initialPrompts.map((prompt) => {
+            initialPrompts.forEach((prompt) => {
                 if (!this.hasPrompt(prompt.path)) {
                     this.emit('page', {
                         type: 'PAGE',
@@ -173,7 +181,11 @@ class PageQueue extends node_events_1.EventEmitter {
                         },
                     });
                 }
-                return this.addPrompt(prompt);
+                // Trigger but don't wait for the "bonus" pages
+                this.addPrompt(prompt).catch((err) => {
+                    console.error('Failed to generate page reference', prompt.name, err);
+                    this.emit('error', err);
+                });
             });
         }
         catch (e) {
@@ -208,18 +220,21 @@ class PageQueue extends node_events_1.EventEmitter {
         await result.waitForDone();
     }
     async generate(prompt, conversationId) {
-        const promises = [];
         const screenStream = await stormClient_1.stormClient.createUIPage(prompt, conversationId);
+        let pageEvent = null;
         screenStream.on('data', (event) => {
             if (event.type === 'PAGE') {
                 event.payload.conversationId = conversationId;
-                promises.push(this.processPageEventWithReferences(event));
+                pageEvent = event;
                 return;
             }
             this.emit('event', event);
         });
         await screenStream.waitForDone();
-        await Promise.all(promises);
+        if (!pageEvent) {
+            throw new Error('No page was generated');
+        }
+        await this.processPageEventWithReferences(pageEvent);
     }
     async resolveReferences(content) {
         const referenceStream = await stormClient_1.stormClient.classifyUIReferences(content);

package/dist/esm/src/storm/page-utils.js

@@ -10,9 +10,10 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 exports.SystemIdHeader = 'System-Id';
 function normalizePath(path) {
     return path
-        .replace(/\?.*$/gi, '')
-        .replace(/:[a-z][a-z_]*\b/gi, '*')
-        .replace(/\{[a-z-.]+}/gi, '*');
+        .replace(/#.*$/g, '') // Remove hash
+        .replace(/\?.*$/gi, '') // Remove query params
+        .replace(/:[a-z][a-z_]*\b/gi, '*') // Replace all params with *
+        .replace(/\{[a-z-.]+}/gi, '*'); // Replace all variables with *
 }
 exports.normalizePath = normalizePath;
 async function writePageToDisk(systemId, event) {

package/dist/esm/src/storm/routes.js

@@ -124,6 +124,10 @@ router.post('/ui/screen', async (req, res) => {
                 throw e;
             }
         });
+        queue.on('error', (err) => {
+            console.error('Failed to process page', err);
+            sendError(err, res);
+        });
         await queue.addPrompt(aiRequest, conversationId, true);
         await queue.wait();
         await Promise.allSettled(promises);
@@ -214,6 +218,10 @@ router.post('/:handle/ui/iterative', async (req, res) => {
         pageQueue.on('event', (screenData) => {
             sendEvent(res, screenData);
         });
+        pageQueue.on('error', (err) => {
+            console.error('Failed to process page', err);
+            sendError(err, res);
+        });
         await waitForStormStream(landingPagesStream);
         await pageQueue.wait();
         await Promise.allSettled(pageEventPromises);

package/dist/esm/src/storm/stormClient.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { ConversationItem, ImplementAPIClientsRequest, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt, StormUIListPrompt } from './stream';
 import { Page, StormEventPageUrl } from './events';
 export declare const STORM_ID = "storm";
@@ -66,7 +65,7 @@ declare class StormClient {
     createUIShells(prompt: UIShellsPrompt, conversationId?: string): Promise<StormStream>;
     createUILandingPages(prompt: BasePromptRequest, conversationId?: string): Promise<StormStream>;
     createUIPage(prompt: UIPagePrompt, conversationId?: string, history?: ConversationItem[]): Promise<StormStream>;
-    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<Response>;
+    voteUIPage(topic: string, conversationId: string, vote: -1 | 0 | 1, mainConversationId?: string): Promise<import("undici").Response>;
     getVoteUIPage(topic: string, conversationId: string, mainConversationId?: string): Promise<{
         vote: -1 | 0 | 1;
     }>;

package/dist/esm/src/storm/stormClient.js

@@ -13,6 +13,22 @@ const utils_1 = require("../utils/utils");
 const promises_1 = __importDefault(require("node:readline/promises"));
 const node_stream_1 = require("node:stream");
 const stream_1 = require("./stream");
+const undici_1 = require("undici");
+// Will only retry on error codes and GET requests by default
+// See https://github.com/nodejs/undici/blob/990df2c7e37cbe5bb44fe2f576dddeaeb5916590/docs/docs/api/RetryAgent.md
+const retryAgent = new undici_1.RetryAgent(new undici_1.Agent(), {
+    methods: [
+        // Added methods ↓ (not idempotent), but we want to retry on POST:
+        'POST',
+        // defaults below
+        'GET',
+        'HEAD',
+        'OPTIONS',
+        'PUT',
+        'DELETE',
+        'TRACE',
+    ],
+});
 exports.STORM_ID = 'storm';
 exports.ConversationIdHeader = 'Conversation-Id';
 class StormClient {
@@ -38,6 +54,7 @@ class StormClient {
             method: method,
             body: JSON.stringify(body),
             headers,
+            dispatcher: retryAgent,
         };
     }
     async send(path, body, method = 'POST') {
@@ -48,7 +65,7 @@ class StormClient {
         });
         const abort = new AbortController();
         options.signal = abort.signal;
-        const response = await fetch(options.url, options);
+        const response = await (0, undici_1.fetch)(options.url, options);
         if (response.status !== 200) {
             throw new Error(`Got error response from ${options.url}: ${response.status}\nContent: ${await response.text()}`);
         }
@@ -121,25 +138,19 @@ class StormClient {
             prompt: JSON.stringify({ topic, vote, mainConversationId }),
             conversationId,
         });
-        return fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        return (0, undici_1.fetch)(options.url, options);
     }
     async getVoteUIPage(topic, conversationId, mainConversationId) {
         const options = await this.createOptions('/v2/ui/get-vote', 'POST', {
             prompt: JSON.stringify({ topic, mainConversationId }),
             conversationId,
         });
-        const response = await fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        const response = await (0, undici_1.fetch)(options.url, options);
         return response.json();
     }
     async implementAPIClients(prompt) {
         const u = `${this._baseUrl}/v2/ui/implement-api-clients`;
-        const response = await fetch(u, {
+        const response = await (0, undici_1.fetch)(u, {
             method: 'POST',
             body: JSON.stringify({
                 fileName: prompt.fileName,
@@ -214,10 +225,7 @@ class StormClient {
             prompt: '',
             conversationId: conversationId,
         });
-        const response = await fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        const response = await (0, undici_1.fetch)(options.url, options);
         return response.text();
     }
 }

package/index.ts

@@ -39,7 +39,7 @@ import { instanceManager } from './src/instanceManager';
 
 Sentry.init({
     dsn: 'https://0b7cc946d82c591473d6f95fff5e210b@o4505820837249024.ingest.sentry.io/4506212692000768',
-    enabled: process.env.NODE_ENV !== 'development',
+    enabled: !!process.env.NODE_ENV && process.env.NODE_ENV !== 'development',
     // Performance Monitoring on every ~20th request
     tracesSampleRate: 0.05,
     // Set sampling rate for profiling - this is relative to tracesSampleRate

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.70.4",
+    "version": "0.70.6",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -87,6 +87,7 @@
         "stream-json": "^1.8.0",
         "tar-stream": "^3.1.6",
         "typescript": "^5.1.6",
+        "undici": "^6.19.8",
         "uuid": "^9.0.1",
         "yaml": "^1.6.0"
     },

package/src/storm/PageGenerator.ts

@@ -42,16 +42,23 @@ export class PageQueue extends EventEmitter {
         this.eventQueue = new PQueue({ concurrency: Number.MAX_VALUE });
     }
 
+    on(event: 'error', listener: (error: unknown) => void): this;
     on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
     on(event: 'page', listener: (data: StormEventPage) => void | Promise<void>): this;
     on(event: 'image', listener: (data: StormImage, source: ImagePrompt) => void | Promise<void>): this;
 
     on(event: string, listener: (...args: any[]) => void | Promise<void>): this {
         return super.on(event, (...args) => {
-            void this.eventQueue.add(async () => listener(...args));
+            this.eventQueue
+                .add(async () => listener(...args))
+                // If the event queue fails, we want to emit an error
+                .catch((err) => {
+                    this.emit('error', err);
+                });
         });
     }
 
+    emit(type: 'error', error: unknown): boolean;
     emit(type: 'event', event: StormEvent): boolean;
     emit(type: 'page', event: StormEventPage): boolean;
     emit(type: 'image', event: StormImage, source: ImagePrompt): boolean;
@@ -160,6 +167,9 @@ export class PageQueue extends EventEmitter {
                         await this.addImagePrompt({
                             ...reference,
                             content: event.payload.content,
+                        }).catch((err) => {
+                            console.error('Failed to generate image for reference', reference.name, err);
+                            this.emit('error', err);
                         });
                         break;
                     case 'css':
@@ -197,7 +207,7 @@ export class PageQueue extends EventEmitter {
             this.emit('page', event);
 
             // Emit any new pages after the current page to increase responsiveness
-            void initialPrompts.map((prompt) => {
+            initialPrompts.forEach((prompt) => {
                 if (!this.hasPrompt(prompt.path)) {
                     this.emit('page', {
                         type: 'PAGE',
@@ -216,7 +226,11 @@ export class PageQueue extends EventEmitter {
                         },
                     });
                 }
-                return this.addPrompt(prompt);
+                // Trigger but don't wait for the "bonus" pages
+                this.addPrompt(prompt).catch((err) => {
+                    console.error('Failed to generate page reference', prompt.name, err);
+                    this.emit('error', err);
+                });
             });
         } catch (e) {
             console.error('Failed to process event', e);
@@ -258,25 +272,25 @@ export class PageQueue extends EventEmitter {
     }
 
     public async generate(prompt: UIPagePrompt, conversationId: string) {
-        const promises: Promise<void>[] = [];
         const screenStream = await stormClient.createUIPage(prompt, conversationId);
-
+        let pageEvent: StormEventPage | null = null;
         screenStream.on('data', (event: StormEvent) => {
             if (event.type === 'PAGE') {
                 event.payload.conversationId = conversationId;
-
-                promises.push(this.processPageEventWithReferences(event));
+                pageEvent = event;
                 return;
             }
-
             this.emit('event', event);
         });
 
         await screenStream.waitForDone();
-        await Promise.all(promises);
+        if (!pageEvent) {
+            throw new Error('No page was generated');
+        }
+        await this.processPageEventWithReferences(pageEvent);
     }
 
-    private async resolveReferences(content: string) {
+    private async resolveReferences(content: string): Promise<ReferenceClassification[]> {
         const referenceStream = await stormClient.classifyUIReferences(content);
 
         const references: ReferenceClassification[] = [];

package/src/storm/page-utils.ts

@@ -17,9 +17,10 @@ export const SystemIdHeader = 'System-Id';
 
 export function normalizePath(path: string) {
     return path
-        .replace(/\?.*$/gi, '')
-        .replace(/:[a-z][a-z_]*\b/gi, '*')
-        .replace(/\{[a-z-.]+}/gi, '*');
+        .replace(/#.*$/g, '') // Remove hash
+        .replace(/\?.*$/gi, '') // Remove query params
+        .replace(/:[a-z][a-z_]*\b/gi, '*') // Replace all params with *
+        .replace(/\{[a-z-.]+}/gi, '*'); // Replace all variables with *
 }
 
 export async function writePageToDisk(systemId: string, event: StormEventPage) {

package/src/storm/routes.ts

@@ -168,6 +168,11 @@ router.post('/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
             }
         });
 
+        queue.on('error', (err) => {
+            console.error('Failed to process page', err);
+            sendError(err as any, res);
+        });
+
         await queue.addPrompt(aiRequest, conversationId, true);
 
         await queue.wait();
@@ -274,6 +279,11 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
             sendEvent(res, screenData);
         });
 
+        pageQueue.on('error', (err) => {
+            console.error('Failed to process page', err);
+            sendError(err as any, res);
+        });
+
         await waitForStormStream(landingPagesStream);
         await pageQueue.wait();
         await Promise.allSettled(pageEventPromises);

package/src/storm/stormClient.ts

@@ -16,6 +16,23 @@ import {
     StormUIListPrompt,
 } from './stream';
 import { Page, StormEventPageUrl } from './events';
+import { fetch, RequestInit, Agent, RetryAgent } from 'undici';
+
+// Will only retry on error codes and GET requests by default
+// See https://github.com/nodejs/undici/blob/990df2c7e37cbe5bb44fe2f576dddeaeb5916590/docs/docs/api/RetryAgent.md
+const retryAgent = new RetryAgent(new Agent(), {
+    methods: [
+        // Added methods ↓ (not idempotent), but we want to retry on POST:
+        'POST',
+        // defaults below
+        'GET',
+        'HEAD',
+        'OPTIONS',
+        'PUT',
+        'DELETE',
+        'TRACE',
+    ],
+});
 
 export const STORM_ID = 'storm';
 
@@ -112,6 +129,7 @@ class StormClient {
             method: method,
             body: JSON.stringify(body),
             headers,
+            dispatcher: retryAgent,
         };
     }
 
@@ -222,10 +240,7 @@ class StormClient {
             conversationId,
         });
 
-        return fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        return fetch(options.url, options);
     }
 
     public async getVoteUIPage(topic: string, conversationId: string, mainConversationId?: string) {
@@ -234,10 +249,7 @@ class StormClient {
             conversationId,
         });
 
-        const response = await fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        const response = await fetch(options.url, options);
 
         return response.json() as Promise<{ vote: -1 | 0 | 1 }>;
     }
@@ -331,10 +343,7 @@ class StormClient {
             conversationId: conversationId,
         });
 
-        const response = await fetch(options.url, {
-            method: options.method,
-            headers: options.headers,
-        });
+        const response = await fetch(options.url, options);
 
         return response.text();
     }