@kapeta/local-cluster-service 0.70.2 → 0.70.4

package/src/storm/PageGenerator.ts

@@ -7,8 +7,9 @@ import uuid from 'node-uuid';
 import { stormClient, UIPagePrompt } from './stormClient';
 import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
-import { PromiseQueue } from './PromiseQueue';
-import { hasPageOnDisk } from './page-utils';
+import PQueue from 'p-queue';
+
+import { hasPageOnDisk, normalizePath } from './page-utils';
 
 export interface ImagePrompt {
     name: string;
@@ -23,11 +24,11 @@ export interface ImagePrompt {
 type InitialPrompt = Omit<UIPagePrompt, 'shell_page'> & { shellType?: 'public' | 'admin' | 'user' };
 
 export class PageQueue extends EventEmitter {
-    private readonly queue: PromiseQueue;
-    private readonly eventQueue: PromiseQueue;
+    private readonly queue: PQueue;
+    private readonly eventQueue: PQueue;
     private readonly systemId: string;
     private readonly systemPrompt: string;
-    private readonly references: Map<string, PageGenerator> = new Map();
+    private readonly references: Map<string, boolean> = new Map();
     private readonly pages: Map<string, string> = new Map();
     private readonly images: Map<string, string> = new Map();
     private uiShells: UIShell[] = [];
@@ -37,8 +38,8 @@ export class PageQueue extends EventEmitter {
         super();
         this.systemId = systemId;
         this.systemPrompt = systemPrompt;
-        this.queue = new PromiseQueue(concurrency);
-        this.eventQueue = new PromiseQueue(Number.MAX_VALUE);
+        this.queue = new PQueue({ concurrency });
+        this.eventQueue = new PQueue({ concurrency: Number.MAX_VALUE });
     }
 
     on(event: 'event', listener: (data: StormEvent) => void | Promise<void>): this;
@@ -94,11 +95,10 @@ export class PageQueue extends EventEmitter {
             theme: this.theme,
         };
 
-        const generator = new PageGenerator(prompt, conversationId);
-        this.references.set(prompt.path, generator);
+        this.references.set(prompt.path, true);
         this.pages.set(prompt.path, prompt.description);
 
-        return this.addPageGenerator(generator);
+        return this.queue.add<void>(() => this.generate(prompt, conversationId));
     }
     private getPrefix(): string {
         let promptPrefix = '';
@@ -132,111 +132,110 @@ export class PageQueue extends EventEmitter {
         return promptPrefix + prompt + promptPostfix;
     }
 
-    private async addPageGenerator(generator: PageGenerator) {
-        generator.on('event', (event: StormEvent) => this.emit('event', event));
-        generator.on('page_refs', async ({ event, references }) => {
-            try {
-                const matchesExistingPages = (url: string) => {
-                    return [...this.pages.keys()].some((path) => new RegExp(path.replaceAll('/*', '/[^/]+')).test(url));
-                };
-                const initialPrompts: InitialPrompt[] = [];
-                const resourcePromises = references.map(async (reference) => {
-                    if (
-                        reference.url.startsWith('#') ||
-                        reference.url.startsWith('javascript:') ||
-                        reference.url.startsWith('http://') ||
-                        reference.url.startsWith('https://')
-                    ) {
-                        return;
-                    }
+    private async processPageEventWithReferences(event: StormEventPage) {
+        try {
+            console.log('Processing page event', event.payload.path);
+            const references = await this.resolveReferences(event.payload.content);
+            const matchesExistingPages = (url: string) => {
+                return [...this.pages.keys()].some((path) =>
+                    new RegExp(`^${path.replaceAll('/*', '/[^/]+')}$`).test(url)
+                );
+            };
+            const initialPrompts: InitialPrompt[] = [];
+            const resourcePromises = references.map(async (reference) => {
+                if (
+                    reference.url.startsWith('#') ||
+                    reference.url.startsWith('javascript:') ||
+                    reference.url.startsWith('http://') ||
+                    reference.url.startsWith('https://') ||
+                    reference.url.startsWith('data:') ||
+                    reference.url.startsWith('blob:') ||
+                    reference.url.startsWith('mailto:')
+                ) {
+                    return;
+                }
 
-                    switch (reference.type) {
-                        case 'image':
-                            await this.addImagePrompt({
-                                ...reference,
-                                content: event.payload.content,
-                            });
-                            break;
-                        case 'css':
-                        case 'javascript':
-                            //console.log('Ignoring reference', reference);
-                            break;
-                        case 'html':
-                            //console.log('Adding page generator for', reference);
-                            if (matchesExistingPages(reference.url)) {
-                                break;
-                            }
-                            this.pages.set(reference.url, reference.description);
-
-                            initialPrompts.push({
-                                name: reference.name,
-                                title: reference.title,
-                                path: reference.url,
-                                method: 'GET',
-                                storage_prefix: this.systemId + '_',
-                                prompt:
-                                    `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
-                                    `The page was referenced from this page: \n### PATH: ${event.payload.path}\n\`\`\`html\n${event.payload.content}\n\`\`\`\n`,
-                                description: reference.description,
-                                // Only used for matching
-                                filename: reference.name + '.ref.html',
-                                theme: this.theme,
-                            });
+                switch (reference.type) {
+                    case 'image':
+                        await this.addImagePrompt({
+                            ...reference,
+                            content: event.payload.content,
+                        });
+                        break;
+                    case 'css':
+                    case 'javascript':
+                        //console.log('Ignoring reference', reference);
+                        break;
+                    case 'html': {
+                        const normalizedPath = normalizePath(reference.url);
+                        if (matchesExistingPages(normalizedPath)) {
                             break;
-                    }
-                });
-
-                // Wait for resources to be generated
-                await Promise.allSettled(resourcePromises);
-                this.emit('page', event);
-
-                // Emit any new pages after the current page to increase responsiveness
-                const newPages = initialPrompts.map((prompt) => {
-                    if (!this.hasPrompt(prompt.path)) {
-                        this.emit('page', {
-                            type: 'PAGE',
-                            reason: 'reference',
-                            created: Date.now(),
-                            payload: {
-                                name: prompt.name,
-                                title: prompt.title,
-                                filename: prompt.filename,
-                                method: 'GET',
-                                path: prompt.path,
-                                prompt: prompt.description,
-                                conversationId: '',
-                                content: '',
-                                description: prompt.description,
-                            },
+                        }
+                        this.pages.set(normalizedPath, reference.description);
+
+                        initialPrompts.push({
+                            name: reference.name,
+                            title: reference.title,
+                            path: normalizedPath,
+                            method: 'GET',
+                            storage_prefix: this.systemId + '_',
+                            prompt:
+                                `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
+                                `The page was referenced from this page: \n### PATH: ${event.payload.path}\n\`\`\`html\n${event.payload.content}\n\`\`\`\n`,
+                            description: reference.description,
+                            // Only used for matching
+                            filename: reference.name + '.ref.html',
+                            theme: this.theme,
                         });
+                        break;
                     }
-                    return this.addPrompt(prompt);
-                });
-                await Promise.allSettled(newPages);
-            } catch (e) {
-                console.error('Failed to process event', e);
-                throw e;
-            }
-        });
-        return this.queue.add(() => generator.generate());
+                }
+            });
+
+            // Wait for resources to be generated
+            await Promise.allSettled(resourcePromises);
+            this.emit('page', event);
+
+            // Emit any new pages after the current page to increase responsiveness
+            void initialPrompts.map((prompt) => {
+                if (!this.hasPrompt(prompt.path)) {
+                    this.emit('page', {
+                        type: 'PAGE',
+                        reason: 'reference',
+                        created: Date.now(),
+                        payload: {
+                            name: prompt.name,
+                            title: prompt.title,
+                            filename: prompt.filename,
+                            method: 'GET',
+                            path: prompt.path,
+                            prompt: prompt.description,
+                            conversationId: '',
+                            content: '',
+                            description: prompt.description,
+                        },
+                    });
+                }
+                return this.addPrompt(prompt);
+            });
+        } catch (e) {
+            console.error('Failed to process event', e);
+            throw e;
+        }
     }
 
     public cancel() {
-        this.queue.cancel();
-        this.eventQueue.cancel();
+        this.queue.clear();
+        this.eventQueue.clear();
     }
 
     public async wait() {
-        while (!this.eventQueue.empty || !this.queue.empty) {
-            await this.eventQueue.wait();
-            await this.queue.wait();
+        while (this.eventQueue.size || this.eventQueue.pending || this.queue.size || this.queue.pending) {
+            await this.eventQueue.onIdle();
+            await this.queue.onIdle();
         }
     }
 
-    public get length() {
-        return this.queue.length + this.eventQueue.length;
-    }
-
     private async addImagePrompt(prompt: ImagePrompt) {
         if (this.images.has(prompt.url)) {
             //console.log('Ignoring duplicate image prompt', prompt);
@@ -257,56 +256,16 @@ export class PageQueue extends EventEmitter {
 
         await result.waitForDone();
     }
-}
-
-export class PageGenerator extends EventEmitter {
-    private readonly eventQueue: PromiseQueue;
-    private readonly conversationId: string;
-    public readonly prompt: UIPagePrompt;
-
-    constructor(prompt: UIPagePrompt, conversationId: string = uuid.v4()) {
-        super();
-        this.conversationId = conversationId;
-        this.prompt = prompt;
-        this.eventQueue = new PromiseQueue(Number.MAX_VALUE);
-    }
 
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(
-        event: 'page_refs',
-        listener: (data: { event: StormEventPage; references: ReferenceClassification[] }) => Promise<void>
-    ): this;
-
-    on(event: string, listener: (...args: any[]) => void): this {
-        return super.on(event, (...args) => {
-            void this.eventQueue.add(async () => listener(...args));
-        });
-    }
-
-    emit(type: 'event', event: StormEvent): boolean;
-    emit(type: 'page_refs', event: { event: StormEventPage; references: ReferenceClassification[] }): boolean;
-    emit(eventName: string | symbol, ...args: any[]): boolean {
-        return super.emit(eventName, ...args);
-    }
-
-    public async generate() {
+    public async generate(prompt: UIPagePrompt, conversationId: string) {
         const promises: Promise<void>[] = [];
-        const screenStream = await stormClient.createUIPage(this.prompt, this.conversationId);
+        const screenStream = await stormClient.createUIPage(prompt, conversationId);
 
         screenStream.on('data', (event: StormEvent) => {
             if (event.type === 'PAGE') {
-                event.payload.conversationId = this.conversationId;
-
-                promises.push(
-                    (async () => {
-                        const references = await this.resolveReferences(event.payload.content);
-                        //console.log('Resolved references for page', references, event.payload);
-                        this.emit('page_refs', {
-                            event,
-                            references,
-                        });
-                    })()
-                );
+                event.payload.conversationId = conversationId;
+
+                promises.push(this.processPageEventWithReferences(event));
                 return;
             }
 
@@ -315,7 +274,6 @@ export class PageGenerator extends EventEmitter {
 
         await screenStream.waitForDone();
         await Promise.all(promises);
-        await this.eventQueue.wait();
     }
 
     private async resolveReferences(content: string) {

package/src/storm/page-utils.ts

@@ -15,7 +15,7 @@ import { ImagePrompt } from './PageGenerator';
 
 export const SystemIdHeader = 'System-Id';
 
-function normalizePath(path: string) {
+export function normalizePath(path: string) {
     return path
         .replace(/\?.*$/gi, '')
         .replace(/:[a-z][a-z_]*\b/gi, '*')
@@ -78,7 +78,8 @@ export function getSystemBaseDir(systemId: string) {
 }
 
 function getFilePath(method: string) {
-    return Path.join(method.toLowerCase(), 'index.html');
+    // For HEAD requests, we assume we're serving looking for a GET resource
+    return Path.join(method === 'HEAD' ? 'get' : method.toLowerCase(), 'index.html');
 }
 
 export function resolveReadPath(systemId: string, path: string, method: string) {
@@ -105,7 +106,7 @@ export function resolveReadPath(systemId: string, path: string, method: string)
         return fullPath;
     }
 
-    const parts = path.split(/\*/g);
+    const parts = path.split('/');
 
     let currentPath = '';
 
@@ -214,7 +215,7 @@ function getFallbackHtml(path: string, method: string): string {
             <script>
                 const checkInterval = 3000;
                 function checkPageReady() {
-                    fetch('${path}', { method: 'HEAD' })
+                    fetch('/${path}', { method: 'HEAD' })
                         .then(response => {
                             if (response.status === 200) {
                                 // The page is ready, reload to fetch it

package/src/storm/routes.ts

@@ -47,8 +47,7 @@ import {
 import { UIServer } from './UIServer';
 import { randomUUID } from 'crypto';
 import { ImagePrompt, PageQueue } from './PageGenerator';
-import { createFuture } from './PromiseQueue';
-import { copyDirectory } from './utils';
+import { copyDirectory, createFuture } from './utils';
 
 const UI_SERVERS: { [key: string]: UIServer } = {};
 const router = Router();
@@ -250,9 +249,7 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
         UI_SERVERS[systemId] = new UIServer(systemId);
         await UI_SERVERS[systemId].start();
         waitForStormStream(landingPagesStream).then(() => {
-            if (!systemPrompt.isResolved()) {
-                systemPrompt.resolve(aiRequest.prompt);
-            }
+            systemPrompt.resolve(aiRequest.prompt);
         });
 
         const pageQueue = new PageQueue(systemId, await systemPrompt.promise, 5);

package/src/storm/utils.ts

@@ -5,7 +5,11 @@
 import FS from 'fs-extra';
 import Path from 'path';
 
-export async function copyDirectory(src: string, dest: string, modifyHtml: (fileName: string, content: string) => Promise<string>): Promise<void> {
+export async function copyDirectory(
+    src: string,
+    dest: string,
+    modifyHtml: (fileName: string, content: string) => Promise<string>
+): Promise<void> {
     await FS.promises.mkdir(dest, { recursive: true });
     const entries = await FS.promises.readdir(src, { withFileTypes: true });
 
@@ -26,3 +30,17 @@ export async function copyDirectory(src: string, dest: string, modifyHtml: (file
         }
     }
 }
+
+export function createFuture<T = void>() {
+    let resolve: (value: T | PromiseLike<T>) => void = () => {};
+    let reject: (reason?: any) => void = () => {};
+    const promise = new Promise<T>((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return {
+        promise,
+        resolve,
+        reject,
+    };
+}

package/dist/cjs/src/storm/PromiseQueue.d.ts

@@ -1,25 +0,0 @@
-/**
- * Copyright 2023 Kapeta Inc.
- * SPDX-License-Identifier: BUSL-1.1
- */
-export type Future<T> = () => Promise<T>;
-export type FuturePromise<T> = {
-    promise: Promise<T>;
-    resolve: (value: T) => void;
-    reject: (reason: any) => void;
-    isResolved: () => boolean;
-};
-export declare function createFuture<T = void>(): FuturePromise<T>;
-export declare class PromiseQueue {
-    private readonly queue;
-    private readonly active;
-    private readonly maxConcurrency;
-    constructor(maxConcurrency?: number);
-    private toInternal;
-    add<T>(future: Future<T>): Promise<T>;
-    private next;
-    cancel(): void;
-    wait(): Promise<void>;
-    get empty(): boolean;
-    get length(): number;
-}

package/dist/cjs/src/storm/PromiseQueue.js

@@ -1,97 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PromiseQueue = exports.createFuture = void 0;
-function createFuture() {
-    let resolved = false;
-    let resolve = () => {
-        resolved = true;
-    };
-    let reject = () => {
-        resolved = true;
-    };
-    const promise = new Promise((res, rej) => {
-        resolve = (value) => {
-            resolved = true;
-            res(value);
-        };
-        reject = (reason) => {
-            resolved = true;
-            rej(reason);
-        };
-    });
-    return {
-        promise,
-        resolve,
-        reject,
-        isResolved: () => resolved,
-    };
-}
-exports.createFuture = createFuture;
-class PromiseQueue {
-    queue = [];
-    active = [];
-    // Maximum number of concurrent promises
-    maxConcurrency;
-    constructor(maxConcurrency = 5) {
-        this.maxConcurrency = maxConcurrency;
-    }
-    toInternal(future) {
-        let resolve = () => { };
-        let reject = () => { };
-        const promise = new Promise((res, rej) => {
-            resolve = res;
-            reject = rej;
-        });
-        return {
-            execute: future,
-            promise,
-            resolve,
-            reject,
-        };
-    }
-    add(future) {
-        const internal = this.toInternal(future);
-        this.queue.push(internal);
-        this.next();
-        return internal.promise;
-    }
-    next() {
-        if (this.active.length >= this.maxConcurrency) {
-            return false;
-        }
-        if (this.queue.length === 0) {
-            return false;
-        }
-        const future = this.queue.shift();
-        if (!future) {
-            return false;
-        }
-        const promise = future
-            .execute()
-            .then(future.resolve)
-            .catch(future.reject)
-            .finally(() => {
-            this.active.splice(this.active.indexOf(promise), 1);
-            this.next();
-        });
-        this.active.push(promise);
-        return this.next();
-    }
-    cancel() {
-        this.queue.splice(0, this.queue.length);
-        this.active.splice(0, this.active.length);
-    }
-    async wait() {
-        while (this.queue.length > 0 || this.active.length > 0) {
-            await Promise.allSettled(this.active);
-            this.next();
-        }
-    }
-    get empty() {
-        return this.queue.length === 0 && this.active.length === 0;
-    }
-    get length() {
-        return this.queue.length + this.active.length;
-    }
-}
-exports.PromiseQueue = PromiseQueue;

package/dist/esm/src/storm/PromiseQueue.d.ts

@@ -1,25 +0,0 @@
-/**
- * Copyright 2023 Kapeta Inc.
- * SPDX-License-Identifier: BUSL-1.1
- */
-export type Future<T> = () => Promise<T>;
-export type FuturePromise<T> = {
-    promise: Promise<T>;
-    resolve: (value: T) => void;
-    reject: (reason: any) => void;
-    isResolved: () => boolean;
-};
-export declare function createFuture<T = void>(): FuturePromise<T>;
-export declare class PromiseQueue {
-    private readonly queue;
-    private readonly active;
-    private readonly maxConcurrency;
-    constructor(maxConcurrency?: number);
-    private toInternal;
-    add<T>(future: Future<T>): Promise<T>;
-    private next;
-    cancel(): void;
-    wait(): Promise<void>;
-    get empty(): boolean;
-    get length(): number;
-}

package/dist/esm/src/storm/PromiseQueue.js

@@ -1,97 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PromiseQueue = exports.createFuture = void 0;
-function createFuture() {
-    let resolved = false;
-    let resolve = () => {
-        resolved = true;
-    };
-    let reject = () => {
-        resolved = true;
-    };
-    const promise = new Promise((res, rej) => {
-        resolve = (value) => {
-            resolved = true;
-            res(value);
-        };
-        reject = (reason) => {
-            resolved = true;
-            rej(reason);
-        };
-    });
-    return {
-        promise,
-        resolve,
-        reject,
-        isResolved: () => resolved,
-    };
-}
-exports.createFuture = createFuture;
-class PromiseQueue {
-    queue = [];
-    active = [];
-    // Maximum number of concurrent promises
-    maxConcurrency;
-    constructor(maxConcurrency = 5) {
-        this.maxConcurrency = maxConcurrency;
-    }
-    toInternal(future) {
-        let resolve = () => { };
-        let reject = () => { };
-        const promise = new Promise((res, rej) => {
-            resolve = res;
-            reject = rej;
-        });
-        return {
-            execute: future,
-            promise,
-            resolve,
-            reject,
-        };
-    }
-    add(future) {
-        const internal = this.toInternal(future);
-        this.queue.push(internal);
-        this.next();
-        return internal.promise;
-    }
-    next() {
-        if (this.active.length >= this.maxConcurrency) {
-            return false;
-        }
-        if (this.queue.length === 0) {
-            return false;
-        }
-        const future = this.queue.shift();
-        if (!future) {
-            return false;
-        }
-        const promise = future
-            .execute()
-            .then(future.resolve)
-            .catch(future.reject)
-            .finally(() => {
-            this.active.splice(this.active.indexOf(promise), 1);
-            this.next();
-        });
-        this.active.push(promise);
-        return this.next();
-    }
-    cancel() {
-        this.queue.splice(0, this.queue.length);
-        this.active.splice(0, this.active.length);
-    }
-    async wait() {
-        while (this.queue.length > 0 || this.active.length > 0) {
-            await Promise.allSettled(this.active);
-            this.next();
-        }
-    }
-    get empty() {
-        return this.queue.length === 0 && this.active.length === 0;
-    }
-    get length() {
-        return this.queue.length + this.active.length;
-    }
-}
-exports.PromiseQueue = PromiseQueue;