@kapeta/local-cluster-service 0.70.2 → 0.70.4

package/CHANGELOG.md

@@ -1,3 +1,21 @@
+## [0.70.4](https://github.com/kapetacom/local-cluster-service/compare/v0.70.3...v0.70.4) (2024-09-10)
+
+
+### Bug Fixes
+
+* better dedupe of refs ([be5c8d2](https://github.com/kapetacom/local-cluster-service/commit/be5c8d2ee555b6e53a47043ce42ecec0f9a0f36c))
+* ignore more protocols in refs ([e44475b](https://github.com/kapetacom/local-cluster-service/commit/e44475b20409ed714b19d387f137600f9d461cd6))
+* migrate to p-queue package for tasks ([441c480](https://github.com/kapetacom/local-cluster-service/commit/441c480cc35d178a807fe57506b6cb54258e2518))
+* tweak page routing logic + WIP screen ([bd7cb0c](https://github.com/kapetacom/local-cluster-service/commit/bd7cb0c580c089a8e2d0a52253b1908142c39537))
+
+## [0.70.3](https://github.com/kapetacom/local-cluster-service/compare/v0.70.2...v0.70.3) (2024-09-10)
+
+
+### Bug Fixes
+
+* better dedupe of refs ([48f676e](https://github.com/kapetacom/local-cluster-service/commit/48f676e0f6780580c2b77cdb8e6f3336f6633e77))
+* match full pattern in lazy check ([5b5e9d7](https://github.com/kapetacom/local-cluster-service/commit/5b5e9d7cfaf3e6e9b3eacf378e90362e99433d86))
+
 ## [0.70.2](https://github.com/kapetacom/local-cluster-service/compare/v0.70.1...v0.70.2) (2024-09-09)
 
 

package/dist/cjs/src/storm/PageGenerator.d.ts

@@ -4,7 +4,7 @@
  */
 /// <reference types="node" />
 import { UIPagePrompt } from './stormClient';
-import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
+import { StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
 export interface ImagePrompt {
     name: string;
@@ -41,28 +41,11 @@ export declare class PageQueue extends EventEmitter {
     addPrompt(initialPrompt: InitialPrompt, conversationId?: string, overwrite?: boolean): Promise<void>;
     private getPrefix;
     private wrapPagePrompt;
-    private addPageGenerator;
+    private processPageEventWithReferences;
     cancel(): void;
     wait(): Promise<void>;
-    get length(): number;
     private addImagePrompt;
-}
-export declare class PageGenerator extends EventEmitter {
-    private readonly eventQueue;
-    private readonly conversationId;
-    readonly prompt: UIPagePrompt;
-    constructor(prompt: UIPagePrompt, conversationId?: string);
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(event: 'page_refs', listener: (data: {
-        event: StormEventPage;
-        references: ReferenceClassification[];
-    }) => Promise<void>): this;
-    emit(type: 'event', event: StormEvent): boolean;
-    emit(type: 'page_refs', event: {
-        event: StormEventPage;
-        references: ReferenceClassification[];
-    }): boolean;
-    generate(): Promise<void>;
+    generate(prompt: UIPagePrompt, conversationId: string): Promise<void>;
     private resolveReferences;
 }
 export {};

package/dist/cjs/src/storm/PageGenerator.js

@@ -7,11 +7,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PageGenerator = exports.PageQueue = void 0;
+exports.PageQueue = void 0;
 const node_uuid_1 = __importDefault(require("node-uuid"));
 const stormClient_1 = require("./stormClient");
 const node_events_1 = require("node:events");
-const PromiseQueue_1 = require("./PromiseQueue");
+const p_queue_1 = __importDefault(require("p-queue"));
 const page_utils_1 = require("./page-utils");
 class PageQueue extends node_events_1.EventEmitter {
     queue;
@@ -27,8 +27,8 @@ class PageQueue extends node_events_1.EventEmitter {
         super();
         this.systemId = systemId;
         this.systemPrompt = systemPrompt;
-        this.queue = new PromiseQueue_1.PromiseQueue(concurrency);
-        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
+        this.queue = new p_queue_1.default({ concurrency });
+        this.eventQueue = new p_queue_1.default({ concurrency: Number.MAX_VALUE });
     }
     on(event, listener) {
         return super.on(event, (...args) => {
@@ -67,10 +67,9 @@ class PageQueue extends node_events_1.EventEmitter {
             prompt: this.wrapPagePrompt(initialPrompt.path, initialPrompt.prompt),
             theme: this.theme,
         };
-        const generator = new PageGenerator(prompt, conversationId);
-        this.references.set(prompt.path, generator);
+        this.references.set(prompt.path, true);
         this.pages.set(prompt.path, prompt.description);
-        return this.addPageGenerator(generator);
+        return this.queue.add(() => this.generate(prompt, conversationId));
     }
     getPrefix() {
         let promptPrefix = '';
@@ -99,101 +98,99 @@ class PageQueue extends node_events_1.EventEmitter {
         }
         return promptPrefix + prompt + promptPostfix;
     }
-    async addPageGenerator(generator) {
-        generator.on('event', (event) => this.emit('event', event));
-        generator.on('page_refs', async ({ event, references }) => {
-            try {
-                const matchesExistingPages = (url) => {
-                    return [...this.pages.keys()].some((path) => new RegExp(path.replaceAll('/*', '/[^/]+')).test(url));
-                };
-                const initialPrompts = [];
-                const resourcePromises = references.map(async (reference) => {
-                    if (reference.url.startsWith('#') ||
-                        reference.url.startsWith('javascript:') ||
-                        reference.url.startsWith('http://') ||
-                        reference.url.startsWith('https://')) {
-                        return;
-                    }
-                    switch (reference.type) {
-                        case 'image':
-                            await this.addImagePrompt({
-                                ...reference,
-                                content: event.payload.content,
-                            });
-                            break;
-                        case 'css':
-                        case 'javascript':
-                            //console.log('Ignoring reference', reference);
-                            break;
-                        case 'html':
-                            //console.log('Adding page generator for', reference);
-                            if (matchesExistingPages(reference.url)) {
-                                break;
-                            }
-                            this.pages.set(reference.url, reference.description);
-                            initialPrompts.push({
-                                name: reference.name,
-                                title: reference.title,
-                                path: reference.url,
-                                method: 'GET',
-                                storage_prefix: this.systemId + '_',
-                                prompt: `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
-                                    `The page was referenced from this page: \n### PATH: ${event.payload.path}\n\`\`\`html\n${event.payload.content}\n\`\`\`\n`,
-                                description: reference.description,
-                                // Only used for matching
-                                filename: reference.name + '.ref.html',
-                                theme: this.theme,
-                            });
+    async processPageEventWithReferences(event) {
+        try {
+            console.log('Processing page event', event.payload.path);
+            const references = await this.resolveReferences(event.payload.content);
+            const matchesExistingPages = (url) => {
+                return [...this.pages.keys()].some((path) => new RegExp(`^${path.replaceAll('/*', '/[^/]+')}$`).test(url));
+            };
+            const initialPrompts = [];
+            const resourcePromises = references.map(async (reference) => {
+                if (reference.url.startsWith('#') ||
+                    reference.url.startsWith('javascript:') ||
+                    reference.url.startsWith('http://') ||
+                    reference.url.startsWith('https://') ||
+                    reference.url.startsWith('data:') ||
+                    reference.url.startsWith('blob:') ||
+                    reference.url.startsWith('mailto:')) {
+                    return;
+                }
+                switch (reference.type) {
+                    case 'image':
+                        await this.addImagePrompt({
+                            ...reference,
+                            content: event.payload.content,
+                        });
+                        break;
+                    case 'css':
+                    case 'javascript':
+                        //console.log('Ignoring reference', reference);
+                        break;
+                    case 'html': {
+                        const normalizedPath = (0, page_utils_1.normalizePath)(reference.url);
+                        if (matchesExistingPages(normalizedPath)) {
                             break;
-                    }
-                });
-                // Wait for resources to be generated
-                await Promise.allSettled(resourcePromises);
-                this.emit('page', event);
-                // Emit any new pages after the current page to increase responsiveness
-                const newPages = initialPrompts.map((prompt) => {
-                    if (!this.hasPrompt(prompt.path)) {
-                        this.emit('page', {
-                            type: 'PAGE',
-                            reason: 'reference',
-                            created: Date.now(),
-                            payload: {
-                                name: prompt.name,
-                                title: prompt.title,
-                                filename: prompt.filename,
-                                method: 'GET',
-                                path: prompt.path,
-                                prompt: prompt.description,
-                                conversationId: '',
-                                content: '',
-                                description: prompt.description,
-                            },
+                        }
+                        this.pages.set(normalizedPath, reference.description);
+                        initialPrompts.push({
+                            name: reference.name,
+                            title: reference.title,
+                            path: normalizedPath,
+                            method: 'GET',
+                            storage_prefix: this.systemId + '_',
+                            prompt: `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
+                                `The page was referenced from this page: \n### PATH: ${event.payload.path}\n\`\`\`html\n${event.payload.content}\n\`\`\`\n`,
+                            description: reference.description,
+                            // Only used for matching
+                            filename: reference.name + '.ref.html',
+                            theme: this.theme,
                         });
+                        break;
                     }
-                    return this.addPrompt(prompt);
-                });
-                await Promise.allSettled(newPages);
-            }
-            catch (e) {
-                console.error('Failed to process event', e);
-                throw e;
-            }
-        });
-        return this.queue.add(() => generator.generate());
+                }
+            });
+            // Wait for resources to be generated
+            await Promise.allSettled(resourcePromises);
+            this.emit('page', event);
+            // Emit any new pages after the current page to increase responsiveness
+            void initialPrompts.map((prompt) => {
+                if (!this.hasPrompt(prompt.path)) {
+                    this.emit('page', {
+                        type: 'PAGE',
+                        reason: 'reference',
+                        created: Date.now(),
+                        payload: {
+                            name: prompt.name,
+                            title: prompt.title,
+                            filename: prompt.filename,
+                            method: 'GET',
+                            path: prompt.path,
+                            prompt: prompt.description,
+                            conversationId: '',
+                            content: '',
+                            description: prompt.description,
+                        },
+                    });
+                }
+                return this.addPrompt(prompt);
+            });
+        }
+        catch (e) {
+            console.error('Failed to process event', e);
+            throw e;
+        }
     }
     cancel() {
-        this.queue.cancel();
-        this.eventQueue.cancel();
+        this.queue.clear();
+        this.eventQueue.clear();
     }
     async wait() {
-        while (!this.eventQueue.empty || !this.queue.empty) {
-            await this.eventQueue.wait();
-            await this.queue.wait();
+        while (this.eventQueue.size || this.eventQueue.pending || this.queue.size || this.queue.pending) {
+            await this.eventQueue.onIdle();
+            await this.queue.onIdle();
         }
     }
-    get length() {
-        return this.queue.length + this.eventQueue.length;
-    }
     async addImagePrompt(prompt) {
         if (this.images.has(prompt.url)) {
             //console.log('Ignoring duplicate image prompt', prompt);
@@ -210,47 +207,19 @@ class PageQueue extends node_events_1.EventEmitter {
         });
         await result.waitForDone();
     }
-}
-exports.PageQueue = PageQueue;
-class PageGenerator extends node_events_1.EventEmitter {
-    eventQueue;
-    conversationId;
-    prompt;
-    constructor(prompt, conversationId = node_uuid_1.default.v4()) {
-        super();
-        this.conversationId = conversationId;
-        this.prompt = prompt;
-        this.eventQueue = new PromiseQueue_1.PromiseQueue(Number.MAX_VALUE);
-    }
-    on(event, listener) {
-        return super.on(event, (...args) => {
-            void this.eventQueue.add(async () => listener(...args));
-        });
-    }
-    emit(eventName, ...args) {
-        return super.emit(eventName, ...args);
-    }
-    async generate() {
+    async generate(prompt, conversationId) {
         const promises = [];
-        const screenStream = await stormClient_1.stormClient.createUIPage(this.prompt, this.conversationId);
+        const screenStream = await stormClient_1.stormClient.createUIPage(prompt, conversationId);
         screenStream.on('data', (event) => {
             if (event.type === 'PAGE') {
-                event.payload.conversationId = this.conversationId;
-                promises.push((async () => {
-                    const references = await this.resolveReferences(event.payload.content);
-                    //console.log('Resolved references for page', references, event.payload);
-                    this.emit('page_refs', {
-                        event,
-                        references,
-                    });
-                })());
+                event.payload.conversationId = conversationId;
+                promises.push(this.processPageEventWithReferences(event));
                 return;
             }
             this.emit('event', event);
         });
         await screenStream.waitForDone();
         await Promise.all(promises);
-        await this.eventQueue.wait();
     }
     async resolveReferences(content) {
         const referenceStream = await stormClient_1.stormClient.classifyUIReferences(content);
@@ -266,4 +235,4 @@ class PageGenerator extends node_events_1.EventEmitter {
         return references;
     }
 }
-exports.PageGenerator = PageGenerator;
+exports.PageQueue = PageQueue;

package/dist/cjs/src/storm/page-utils.d.ts

@@ -7,6 +7,7 @@ import { Response } from 'express';
 import { ConversationItem } from './stream';
 import { ImagePrompt } from './PageGenerator';
 export declare const SystemIdHeader = "System-Id";
+export declare function normalizePath(path: string): string;
 export declare function writePageToDisk(systemId: string, event: StormEventPage): Promise<{
     path: string;
 }>;

package/dist/cjs/src/storm/page-utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeImageToDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.SystemIdHeader = void 0;
+exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeImageToDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.normalizePath = exports.SystemIdHeader = void 0;
 const node_os_1 = __importDefault(require("node:os"));
 const path_1 = __importDefault(require("path"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
@@ -14,6 +14,7 @@ function normalizePath(path) {
         .replace(/:[a-z][a-z_]*\b/gi, '*')
         .replace(/\{[a-z-.]+}/gi, '*');
 }
+exports.normalizePath = normalizePath;
 async function writePageToDisk(systemId, event) {
     const baseDir = getSystemBaseDir(systemId);
     const filePath = getFilePath(event.payload.method);
@@ -62,7 +63,8 @@ function getSystemBaseDir(systemId) {
 }
 exports.getSystemBaseDir = getSystemBaseDir;
 function getFilePath(method) {
-    return path_1.default.join(method.toLowerCase(), 'index.html');
+    // For HEAD requests, we assume we're serving looking for a GET resource
+    return path_1.default.join(method === 'HEAD' ? 'get' : method.toLowerCase(), 'index.html');
 }
 function resolveReadPath(systemId, path, method) {
     const baseDir = getSystemBaseDir(systemId);
@@ -83,7 +85,7 @@ function resolveReadPath(systemId, path, method) {
     if (fs_extra_1.default.existsSync(fullPath)) {
         return fullPath;
     }
-    const parts = path.split(/\*/g);
+    const parts = path.split('/');
     let currentPath = '';
     for (let part of parts) {
         const thisPath = path_1.default.join(currentPath, part);
@@ -184,7 +186,7 @@ function getFallbackHtml(path, method) {
             <script>
                 const checkInterval = 3000;
                 function checkPageReady() {
-                    fetch('${path}', { method: 'HEAD' })
+                    fetch('/${path}', { method: 'HEAD' })
                         .then(response => {
                             if (response.status === 200) {
                                 // The page is ready, reload to fetch it

package/dist/cjs/src/storm/routes.js

@@ -24,7 +24,6 @@ const page_utils_1 = require("./page-utils");
 const UIServer_1 = require("./UIServer");
 const crypto_1 = require("crypto");
 const PageGenerator_1 = require("./PageGenerator");
-const PromiseQueue_1 = require("./PromiseQueue");
 const utils_1 = require("./utils");
 const UI_SERVERS = {};
 const router = (0, express_promise_router_1.default)();
@@ -154,7 +153,7 @@ router.post('/:handle/ui/iterative', async (req, res) => {
         const promises = {};
         const pageEventPromises = [];
         const systemId = landingPagesStream.getConversationId();
-        const systemPrompt = (0, PromiseQueue_1.createFuture)();
+        const systemPrompt = (0, utils_1.createFuture)();
         if (aiRequest.skipImprovement) {
             systemPrompt.resolve(aiRequest.prompt);
         }
@@ -196,9 +195,7 @@ router.post('/:handle/ui/iterative', async (req, res) => {
         UI_SERVERS[systemId] = new UIServer_1.UIServer(systemId);
         await UI_SERVERS[systemId].start();
         waitForStormStream(landingPagesStream).then(() => {
-            if (!systemPrompt.isResolved()) {
-                systemPrompt.resolve(aiRequest.prompt);
-            }
+            systemPrompt.resolve(aiRequest.prompt);
         });
         const pageQueue = new PageGenerator_1.PageQueue(systemId, await systemPrompt.promise, 5);
         onRequestAborted(req, res, () => {

package/dist/cjs/src/storm/utils.d.ts

@@ -1 +1,6 @@
 export declare function copyDirectory(src: string, dest: string, modifyHtml: (fileName: string, content: string) => Promise<string>): Promise<void>;
+export declare function createFuture<T = void>(): {
+    promise: Promise<T>;
+    resolve: (value: T | PromiseLike<T>) => void;
+    reject: (reason?: any) => void;
+};

package/dist/cjs/src/storm/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.copyDirectory = void 0;
+exports.createFuture = exports.copyDirectory = void 0;
 /**
  * Copyright 2023 Kapeta Inc.
  * SPDX-License-Identifier: BUSL-1.1
@@ -29,3 +29,17 @@ async function copyDirectory(src, dest, modifyHtml) {
     }
 }
 exports.copyDirectory = copyDirectory;
+function createFuture() {
+    let resolve = () => { };
+    let reject = () => { };
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return {
+        promise,
+        resolve,
+        reject,
+    };
+}
+exports.createFuture = createFuture;

package/dist/esm/src/storm/PageGenerator.d.ts

@@ -4,7 +4,7 @@
  */
 /// <reference types="node" />
 import { UIPagePrompt } from './stormClient';
-import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
+import { StormEvent, StormEventPage, StormImage, UIShell } from './events';
 import { EventEmitter } from 'node:events';
 export interface ImagePrompt {
     name: string;
@@ -41,28 +41,11 @@ export declare class PageQueue extends EventEmitter {
     addPrompt(initialPrompt: InitialPrompt, conversationId?: string, overwrite?: boolean): Promise<void>;
     private getPrefix;
     private wrapPagePrompt;
-    private addPageGenerator;
+    private processPageEventWithReferences;
     cancel(): void;
     wait(): Promise<void>;
-    get length(): number;
     private addImagePrompt;
-}
-export declare class PageGenerator extends EventEmitter {
-    private readonly eventQueue;
-    private readonly conversationId;
-    readonly prompt: UIPagePrompt;
-    constructor(prompt: UIPagePrompt, conversationId?: string);
-    on(event: 'event', listener: (data: StormEvent) => void): this;
-    on(event: 'page_refs', listener: (data: {
-        event: StormEventPage;
-        references: ReferenceClassification[];
-    }) => Promise<void>): this;
-    emit(type: 'event', event: StormEvent): boolean;
-    emit(type: 'page_refs', event: {
-        event: StormEventPage;
-        references: ReferenceClassification[];
-    }): boolean;
-    generate(): Promise<void>;
+    generate(prompt: UIPagePrompt, conversationId: string): Promise<void>;
     private resolveReferences;
 }
 export {};