@kapeta/local-cluster-service 0.76.5 → 0.77.1

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.77.1](https://github.com/kapetacom/local-cluster-service/compare/v0.77.0...v0.77.1) (2024-10-04)
+
+
+### Bug Fixes
+
+* disable timeouts for createSimpleBackend call ([#270](https://github.com/kapetacom/local-cluster-service/issues/270)) ([9a6ba9c](https://github.com/kapetacom/local-cluster-service/commit/9a6ba9cb09ff99a3e31e93e33c027232ddeff8fb))
+
+# [0.77.0](https://github.com/kapetacom/local-cluster-service/compare/v0.76.5...v0.77.0) (2024-10-04)
+
+
+### Features
+
+* Enable page agent to distinguish beween global and local edits ([1e24653](https://github.com/kapetacom/local-cluster-service/commit/1e246536272f9a141acf2233eb96fdec2a30c709))
+
 ## [0.76.5](https://github.com/kapetacom/local-cluster-service/compare/v0.76.4...v0.76.5) (2024-10-03)
 
 

package/dist/cjs/src/storm/PageGenerator.d.ts

@@ -41,9 +41,11 @@ export declare class PageQueue extends EventEmitter {
     addUiShell(uiShell: UIShell): void;
     setUiTheme(theme: string): void;
     private hasPrompt;
-    addPrompt(initialPrompt: InitialPrompt, conversationId?: string, overwrite?: boolean): Promise<void>;
-    private getPrefix;
-    private wrapPagePrompt;
+    addPrompt(initialPrompt: InitialPrompt, conversationId?: string, overwrite?: boolean, globalEdit?: boolean): Promise<void>;
+    /**
+     * Get the existing pages
+     */
+    private getExistingPages;
     private processPageEventWithReferences;
     cancel(): void;
     wait(): Promise<void>;

package/dist/cjs/src/storm/PageGenerator.js

@@ -87,7 +87,9 @@ class PageQueue extends node_events_1.EventEmitter {
         }
         return false;
     }
-    addPrompt(initialPrompt, conversationId = node_uuid_1.default.v4(), overwrite = false) {
+    addPrompt(initialPrompt, conversationId = node_uuid_1.default.v4(), overwrite = false, 
+    // Set globalEdit to true when the same prompt is being sent to multiple pages
+    globalEdit = false) {
         if (!overwrite && this.hasPrompt(initialPrompt.path)) {
             //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
@@ -96,39 +98,26 @@ class PageQueue extends node_events_1.EventEmitter {
         const prompt = {
             ...initialPrompt,
             shell_page: this.uiShells.find((shell) => shell.screens.includes(initialPrompt.name))?.content,
-            prompt: this.wrapPagePrompt(initialPrompt.path, initialPrompt.prompt),
+            prompt: initialPrompt.prompt,
             theme: this.theme,
+            global_edit: globalEdit,
+            system_prompt: this.systemPrompt,
+            existing_pages: this.getExistingPages(initialPrompt.path),
+            existing_images: Object.entries(this.images).map(([path, description]) => ({ path, description })),
         };
         this.references.set(prompt.path, true);
         this.pages.set(prompt.path, prompt.description);
         return this.queue.add(() => this.generate(prompt, conversationId));
     }
-    getPrefix() {
-        let promptPrefix = '';
-        if (this.systemPrompt) {
-            promptPrefix = `For a system with this description: ${this.systemPrompt}\n`;
-        }
-        return promptPrefix;
-    }
-    wrapPagePrompt(pagePath, prompt) {
-        const promptPrefix = this.getPrefix();
-        let promptPostfix = '';
-        if (this.pages.size > 0) {
-            promptPostfix = `\nThe following pages are already implemented:\n`;
-            this.pages.forEach((description, path) => {
-                if (pagePath === path) {
-                    return;
-                }
-                promptPostfix += `- PAGE: '${path}' -> ${description}.\n`;
-            });
-        }
-        if (this.images.size > 0) {
-            promptPostfix += `\nThe following images already exist:\n`;
-            this.images.forEach((description, path) => {
-                promptPostfix += `- IMAGE: '${path}' -> ${description}.\n`;
-            });
-        }
-        return promptPrefix + prompt + promptPostfix;
+    /**
+     * Get the existing pages
+     */
+    getExistingPages(excludePath) {
+        return (Object.entries(this.pages)
+            // Possibly exclude one page. This is useful when we don't want to include the page
+            // we're currently editing.
+            .filter(([path]) => (excludePath ? path !== excludePath : true))
+            .map(([path, description]) => ({ path, description })));
     }
     async processPageEventWithReferences(event) {
         try {
@@ -176,9 +165,17 @@ class PageQueue extends node_events_1.EventEmitter {
                             path: normalizedPath,
                             method: 'GET',
                             storage_prefix: this.systemId + '_',
-                            prompt: `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
-                                `The page was referenced from this page: \n### PATH: ${event.payload.path}\n\`\`\`html\n${event.payload.content}\n\`\`\`\n`,
+                            prompt: `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}`,
                             description: reference.description,
+                            referenced_from: {
+                                path: event.payload.path,
+                                content: event.payload.content,
+                            },
+                            existing_pages: this.getExistingPages(),
+                            existing_images: Object.entries(this.images).map(([path, description]) => ({
+                                path,
+                                description,
+                            })),
                             // Only used for matching
                             filename: reference.name + '.ref.html',
                             theme: this.theme,

package/dist/cjs/src/storm/routes.js

@@ -226,6 +226,9 @@ router.delete('/ui/serve/:systemId', async (req, res) => {
     }
     res.status(200).json({ status: 'ok' });
 });
+/**
+ * Edit a single page
+ */
 router.post('/:handle/ui/screen', async (req, res) => {
     try {
         const handle = req.params.handle;
@@ -540,6 +543,9 @@ router.post('/:handle/ui', async (req, res) => {
         }
     }
 });
+/**
+ * Edit all pages
+ */
 router.post('/:handle/ui/edit', async (req, res) => {
     try {
         const handle = req.params.handle;
@@ -586,7 +592,8 @@ router.post('/:handle/ui/edit', async (req, res) => {
                     filename: page.filename,
                     prompt: aiRequest.prompt.prompt.prompt,
                     storage_prefix: storagePrefix,
-                }, page.conversationId, true);
+                }, page.conversationId, true, true // this is a global edit
+                );
             }
         }));
         await queue.wait();

package/dist/cjs/src/storm/stormClient.d.ts

@@ -28,6 +28,20 @@ export interface UIPagePrompt {
     storage_prefix: string;
     shell_page?: string;
     theme?: string;
+    global_edit?: boolean;
+    system_prompt?: string;
+    existing_pages?: {
+        path: string;
+        description: string;
+    }[];
+    existing_images?: {
+        path: string;
+        description: string;
+    }[];
+    referenced_from?: {
+        path: string;
+        content: string;
+    };
 }
 export interface UIPageSamplePrompt extends UIPagePrompt {
     variantId: string;

package/dist/cjs/src/storm/stormClient.js

@@ -14,6 +14,7 @@ const promises_1 = __importDefault(require("node:readline/promises"));
 const node_stream_1 = require("node:stream");
 const stream_1 = require("./stream");
 const fetch_retry_1 = __importDefault(require("fetch-retry"));
+const undici_1 = require("undici");
 const fetchWithRetries = (0, fetch_retry_1.default)(global.fetch, { retries: 5, retryDelay: 10 });
 exports.STORM_ID = 'storm';
 exports.ConversationIdHeader = 'Conversation-Id';
@@ -189,12 +190,18 @@ class StormClient {
         headers[exports.HandleHeader] = this._handle;
         headers[exports.ConversationIdHeader] = this._systemId;
         headers[exports.SystemIdHeader] = this._systemId;
-        const response = await fetch(u, {
+        // Fetch with undici to override the default timeouts
+        const response = await (0, undici_1.fetch)(u, {
             method: 'POST',
             body: JSON.stringify({
                 pages: input.pages,
             }),
             headers: headers,
+            dispatcher: new undici_1.Agent({
+                headersTimeout: 0,
+                keepAliveTimeout: 0,
+                bodyTimeout: 0,
+            }),
         });
         if (!response.ok) {
             throw new Error(`HTTP error! Status: ${response.status}`);

package/dist/esm/src/storm/PageGenerator.d.ts

@@ -41,9 +41,11 @@ export declare class PageQueue extends EventEmitter {
     addUiShell(uiShell: UIShell): void;
     setUiTheme(theme: string): void;
     private hasPrompt;
-    addPrompt(initialPrompt: InitialPrompt, conversationId?: string, overwrite?: boolean): Promise<void>;
-    private getPrefix;
-    private wrapPagePrompt;
+    addPrompt(initialPrompt: InitialPrompt, conversationId?: string, overwrite?: boolean, globalEdit?: boolean): Promise<void>;
+    /**
+     * Get the existing pages
+     */
+    private getExistingPages;
     private processPageEventWithReferences;
     cancel(): void;
     wait(): Promise<void>;

package/dist/esm/src/storm/PageGenerator.js

@@ -87,7 +87,9 @@ class PageQueue extends node_events_1.EventEmitter {
         }
         return false;
     }
-    addPrompt(initialPrompt, conversationId = node_uuid_1.default.v4(), overwrite = false) {
+    addPrompt(initialPrompt, conversationId = node_uuid_1.default.v4(), overwrite = false, 
+    // Set globalEdit to true when the same prompt is being sent to multiple pages
+    globalEdit = false) {
         if (!overwrite && this.hasPrompt(initialPrompt.path)) {
             //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
@@ -96,39 +98,26 @@ class PageQueue extends node_events_1.EventEmitter {
         const prompt = {
             ...initialPrompt,
             shell_page: this.uiShells.find((shell) => shell.screens.includes(initialPrompt.name))?.content,
-            prompt: this.wrapPagePrompt(initialPrompt.path, initialPrompt.prompt),
+            prompt: initialPrompt.prompt,
             theme: this.theme,
+            global_edit: globalEdit,
+            system_prompt: this.systemPrompt,
+            existing_pages: this.getExistingPages(initialPrompt.path),
+            existing_images: Object.entries(this.images).map(([path, description]) => ({ path, description })),
         };
         this.references.set(prompt.path, true);
         this.pages.set(prompt.path, prompt.description);
         return this.queue.add(() => this.generate(prompt, conversationId));
     }
-    getPrefix() {
-        let promptPrefix = '';
-        if (this.systemPrompt) {
-            promptPrefix = `For a system with this description: ${this.systemPrompt}\n`;
-        }
-        return promptPrefix;
-    }
-    wrapPagePrompt(pagePath, prompt) {
-        const promptPrefix = this.getPrefix();
-        let promptPostfix = '';
-        if (this.pages.size > 0) {
-            promptPostfix = `\nThe following pages are already implemented:\n`;
-            this.pages.forEach((description, path) => {
-                if (pagePath === path) {
-                    return;
-                }
-                promptPostfix += `- PAGE: '${path}' -> ${description}.\n`;
-            });
-        }
-        if (this.images.size > 0) {
-            promptPostfix += `\nThe following images already exist:\n`;
-            this.images.forEach((description, path) => {
-                promptPostfix += `- IMAGE: '${path}' -> ${description}.\n`;
-            });
-        }
-        return promptPrefix + prompt + promptPostfix;
+    /**
+     * Get the existing pages
+     */
+    getExistingPages(excludePath) {
+        return (Object.entries(this.pages)
+            // Possibly exclude one page. This is useful when we don't want to include the page
+            // we're currently editing.
+            .filter(([path]) => (excludePath ? path !== excludePath : true))
+            .map(([path, description]) => ({ path, description })));
     }
     async processPageEventWithReferences(event) {
         try {
@@ -176,9 +165,17 @@ class PageQueue extends node_events_1.EventEmitter {
                             path: normalizedPath,
                             method: 'GET',
                             storage_prefix: this.systemId + '_',
-                            prompt: `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
-                                `The page was referenced from this page: \n### PATH: ${event.payload.path}\n\`\`\`html\n${event.payload.content}\n\`\`\`\n`,
+                            prompt: `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}`,
                             description: reference.description,
+                            referenced_from: {
+                                path: event.payload.path,
+                                content: event.payload.content,
+                            },
+                            existing_pages: this.getExistingPages(),
+                            existing_images: Object.entries(this.images).map(([path, description]) => ({
+                                path,
+                                description,
+                            })),
                             // Only used for matching
                             filename: reference.name + '.ref.html',
                             theme: this.theme,

package/dist/esm/src/storm/routes.js

@@ -226,6 +226,9 @@ router.delete('/ui/serve/:systemId', async (req, res) => {
     }
     res.status(200).json({ status: 'ok' });
 });
+/**
+ * Edit a single page
+ */
 router.post('/:handle/ui/screen', async (req, res) => {
     try {
         const handle = req.params.handle;
@@ -540,6 +543,9 @@ router.post('/:handle/ui', async (req, res) => {
         }
     }
 });
+/**
+ * Edit all pages
+ */
 router.post('/:handle/ui/edit', async (req, res) => {
     try {
         const handle = req.params.handle;
@@ -586,7 +592,8 @@ router.post('/:handle/ui/edit', async (req, res) => {
                     filename: page.filename,
                     prompt: aiRequest.prompt.prompt.prompt,
                     storage_prefix: storagePrefix,
-                }, page.conversationId, true);
+                }, page.conversationId, true, true // this is a global edit
+                );
             }
         }));
         await queue.wait();

package/dist/esm/src/storm/stormClient.d.ts

@@ -28,6 +28,20 @@ export interface UIPagePrompt {
     storage_prefix: string;
     shell_page?: string;
     theme?: string;
+    global_edit?: boolean;
+    system_prompt?: string;
+    existing_pages?: {
+        path: string;
+        description: string;
+    }[];
+    existing_images?: {
+        path: string;
+        description: string;
+    }[];
+    referenced_from?: {
+        path: string;
+        content: string;
+    };
 }
 export interface UIPageSamplePrompt extends UIPagePrompt {
     variantId: string;

package/dist/esm/src/storm/stormClient.js

@@ -14,6 +14,7 @@ const promises_1 = __importDefault(require("node:readline/promises"));
 const node_stream_1 = require("node:stream");
 const stream_1 = require("./stream");
 const fetch_retry_1 = __importDefault(require("fetch-retry"));
+const undici_1 = require("undici");
 const fetchWithRetries = (0, fetch_retry_1.default)(global.fetch, { retries: 5, retryDelay: 10 });
 exports.STORM_ID = 'storm';
 exports.ConversationIdHeader = 'Conversation-Id';
@@ -189,12 +190,18 @@ class StormClient {
         headers[exports.HandleHeader] = this._handle;
         headers[exports.ConversationIdHeader] = this._systemId;
         headers[exports.SystemIdHeader] = this._systemId;
-        const response = await fetch(u, {
+        // Fetch with undici to override the default timeouts
+        const response = await (0, undici_1.fetch)(u, {
             method: 'POST',
             body: JSON.stringify({
                 pages: input.pages,
             }),
             headers: headers,
+            dispatcher: new undici_1.Agent({
+                headersTimeout: 0,
+                keepAliveTimeout: 0,
+                bodyTimeout: 0,
+            }),
         });
         if (!response.ok) {
             throw new Error(`HTTP error! Status: ${response.status}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.76.5",
+    "version": "0.77.1",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -90,6 +90,7 @@
         "tar": "^7.4.3",
         "tar-stream": "^3.1.6",
         "typescript": "^5.1.6",
+        "undici": "^6.19.8",
         "uuid": "^9.0.1",
         "yaml": "^1.6.0"
     },

package/src/storm/PageGenerator.ts

@@ -95,7 +95,13 @@ export class PageQueue extends EventEmitter {
         return false;
     }
 
-    public addPrompt(initialPrompt: InitialPrompt, conversationId: string = uuid.v4(), overwrite: boolean = false) {
+    public addPrompt(
+        initialPrompt: InitialPrompt,
+        conversationId: string = uuid.v4(),
+        overwrite: boolean = false,
+        // Set globalEdit to true when the same prompt is being sent to multiple pages
+        globalEdit: boolean = false
+    ) {
         if (!overwrite && this.hasPrompt(initialPrompt.path)) {
             //console.log('Ignoring duplicate prompt', initialPrompt.path);
             return Promise.resolve();
@@ -105,8 +111,12 @@ export class PageQueue extends EventEmitter {
         const prompt: UIPagePrompt = {
             ...initialPrompt,
             shell_page: this.uiShells.find((shell) => shell.screens.includes(initialPrompt.name))?.content,
-            prompt: this.wrapPagePrompt(initialPrompt.path, initialPrompt.prompt),
+            prompt: initialPrompt.prompt,
             theme: this.theme,
+            global_edit: globalEdit,
+            system_prompt: this.systemPrompt,
+            existing_pages: this.getExistingPages(initialPrompt.path),
+            existing_images: Object.entries(this.images).map(([path, description]) => ({ path, description })),
         };
 
         this.references.set(prompt.path, true);
@@ -114,36 +124,18 @@ export class PageQueue extends EventEmitter {
 
         return this.queue.add<void>(() => this.generate(prompt, conversationId));
     }
-    private getPrefix(): string {
-        let promptPrefix = '';
-        if (this.systemPrompt) {
-            promptPrefix = `For a system with this description: ${this.systemPrompt}\n`;
-        }
-        return promptPrefix;
-    }
-
-    private wrapPagePrompt(pagePath: string, prompt: string): string {
-        const promptPrefix = this.getPrefix();
-        let promptPostfix = '';
-
-        if (this.pages.size > 0) {
-            promptPostfix = `\nThe following pages are already implemented:\n`;
-            this.pages.forEach((description, path) => {
-                if (pagePath === path) {
-                    return;
-                }
-                promptPostfix += `- PAGE: '${path}' -> ${description}.\n`;
-            });
-        }
-
-        if (this.images.size > 0) {
-            promptPostfix += `\nThe following images already exist:\n`;
-            this.images.forEach((description, path) => {
-                promptPostfix += `- IMAGE: '${path}' -> ${description}.\n`;
-            });
-        }
 
-        return promptPrefix + prompt + promptPostfix;
+    /**
+     * Get the existing pages
+     */
+    private getExistingPages(excludePath?: string) {
+        return (
+            Object.entries(this.pages)
+                // Possibly exclude one page. This is useful when we don't want to include the page
+                // we're currently editing.
+                .filter(([path]) => (excludePath ? path !== excludePath : true))
+                .map(([path, description]) => ({ path, description }))
+        );
     }
 
     private async processPageEventWithReferences(event: StormEventPage) {
@@ -198,10 +190,17 @@ export class PageQueue extends EventEmitter {
                             path: normalizedPath,
                             method: 'GET',
                             storage_prefix: this.systemId + '_',
-                            prompt:
-                                `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
-                                `The page was referenced from this page: \n### PATH: ${event.payload.path}\n\`\`\`html\n${event.payload.content}\n\`\`\`\n`,
+                            prompt: `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}`,
                             description: reference.description,
+                            referenced_from: {
+                                path: event.payload.path,
+                                content: event.payload.content,
+                            },
+                            existing_pages: this.getExistingPages(),
+                            existing_images: Object.entries(this.images).map(([path, description]) => ({
+                                path,
+                                description,
+                            })),
                             // Only used for matching
                             filename: reference.name + '.ref.html',
                             theme: this.theme,

package/src/storm/routes.ts

@@ -293,6 +293,9 @@ router.delete('/ui/serve/:systemId', async (req: KapetaBodyRequest, res: Respons
     res.status(200).json({ status: 'ok' });
 });
 
+/**
+ * Edit a single page
+ */
 router.post('/:handle/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
     try {
         const handle = req.params.handle as string;
@@ -673,6 +676,9 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
     }
 });
 
+/**
+ * Edit all pages
+ */
 router.post('/:handle/ui/edit', async (req: KapetaBodyRequest, res: Response) => {
     try {
         const handle = req.params.handle as string;
@@ -732,7 +738,8 @@ router.post('/:handle/ui/edit', async (req: KapetaBodyRequest, res: Response) =>
                             storage_prefix: storagePrefix,
                         },
                         page.conversationId,
-                        true
+                        true,
+                        true // this is a global edit
                     );
                 }
             })

package/src/storm/stormClient.ts

@@ -19,6 +19,7 @@ import {
 } from './stream';
 import { Page, StormEventPageUrl } from './events';
 import createFetch from 'fetch-retry';
+import { Agent, fetch as undiciFetch } from 'undici';
 const fetchWithRetries = createFetch(global.fetch, { retries: 5, retryDelay: 10 });
 
 export const STORM_ID = 'storm';
@@ -51,6 +52,25 @@ export interface UIPagePrompt {
     shell_page?: string;
     // contents of theme.md
     theme?: string;
+    // whether this prompt is for a global edit
+    global_edit?: boolean;
+    // The prompt that was used to start the conversation (user prompt or improved prompt)
+    system_prompt?: string;
+    // The pages we have already created
+    existing_pages?: {
+        path: string;
+        description: string;
+    }[];
+    // The images we have already created
+    existing_images?: {
+        path: string;
+        description: string;
+    }[];
+    // The page that referenced this page
+    referenced_from?: {
+        path: string;
+        content: string;
+    };
 }
 
 export interface UIPageSamplePrompt extends UIPagePrompt {
@@ -299,12 +319,18 @@ export class StormClient {
         headers[ConversationIdHeader] = this._systemId;
         headers[SystemIdHeader] = this._systemId;
 
-        const response = await fetch(u, {
+        // Fetch with undici to override the default timeouts
+        const response = await undiciFetch(u, {
             method: 'POST',
             body: JSON.stringify({
                 pages: input.pages,
             }),
             headers: headers,
+            dispatcher: new Agent({
+                headersTimeout: 0,
+                keepAliveTimeout: 0,
+                bodyTimeout: 0,
+            }),
         });
         if (!response.ok) {
             throw new Error(`HTTP error! Status: ${response.status}`);