@kapeta/local-cluster-service 0.61.2 → 0.62.1

package/dist/esm/src/storm/stormClient.d.ts

@@ -23,6 +23,9 @@ export interface UIPagePrompt {
     storage_prefix: string;
     shell_page?: string;
 }
+export interface UIPageSamplePrompt extends UIPagePrompt {
+    variantId: string;
+}
 export interface UIPageEditPrompt {
     planDescription: string;
     blockDescription: string;
@@ -44,7 +47,9 @@ declare class StormClient {
     createUIPages(prompt: string, conversationId?: string): Promise<StormStream>;
     createUIUserJourneys(prompt: string, conversationId?: string): Promise<StormStream>;
     createUIShells(prompt: UIShellsPrompt, conversationId?: string): Promise<StormStream>;
-    createUIPage(prompt: UIPagePrompt, conversationId?: string): Promise<StormStream>;
+    createUILandingPages(prompt: string, conversationId?: string): Promise<StormStream>;
+    createUIPage(prompt: UIPagePrompt, conversationId?: string, history?: ConversationItem[]): Promise<StormStream>;
+    classifyUIReferences(prompt: string, conversationId?: string): Promise<StormStream>;
     editPages(prompt: UIPageEditPrompt, conversationId?: string): Promise<StormStream>;
     listScreens(prompt: StormUIListPrompt, conversationId?: string): Promise<StormStream>;
     createUIImplementation(prompt: StormUIImplementationPrompt, conversationId?: string): Promise<StormStream>;
@@ -53,6 +58,7 @@ declare class StormClient {
     createCodeFix(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
     createErrorDetails(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
     generateCode(prompt: StormFileImplementationPrompt, conversationId?: string): Promise<StormStream>;
+    deleteUIPageConversation(conversationId: string): Promise<string>;
 }
 export declare const stormClient: StormClient;
 export {};

package/dist/esm/src/storm/stormClient.js

@@ -95,13 +95,25 @@ class StormClient {
     createUIShells(prompt, conversationId) {
         return this.send('/v2/ui/shells', {
             prompt: JSON.stringify(prompt),
+        });
+    }
+    createUILandingPages(prompt, conversationId) {
+        return this.send('/v2/ui/landing-pages', {
+            prompt: prompt,
             conversationId,
         });
     }
-    createUIPage(prompt, conversationId) {
+    createUIPage(prompt, conversationId, history) {
         return this.send('/v2/ui/page', {
             prompt: prompt,
             conversationId,
+            history,
+        });
+    }
+    classifyUIReferences(prompt, conversationId) {
+        return this.send('/v2/ui/references', {
+            prompt: prompt,
+            conversationId,
         });
     }
     editPages(prompt, conversationId) {
@@ -152,5 +164,16 @@ class StormClient {
             conversationId: conversationId,
         });
     }
+    async deleteUIPageConversation(conversationId) {
+        const options = await this.createOptions('/v2/ui/page', 'DELETE', {
+            prompt: '',
+            conversationId: conversationId,
+        });
+        const response = await fetch(options.url, {
+            method: options.method,
+            headers: options.headers,
+        });
+        return response.text();
+    }
 }
 exports.stormClient = new StormClient();

package/dist/esm/src/storm/stream.d.ts

@@ -35,6 +35,7 @@ export interface ConversationItem {
 }
 export interface StormContextRequest<T = string> {
     conversationId?: string;
+    history?: ConversationItem[];
     prompt: T;
 }
 export interface StormCreateBlockRequest {

package/dist/esm/src/storm/stream.js

@@ -32,6 +32,7 @@ class StormStream extends node_events_1.EventEmitter {
         }
         catch (e) {
             this.emit('error', e);
+            console.warn('Failed to parse JSON line', e, line);
         }
     }
     end() {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.61.2",
+    "version": "0.62.1",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {

package/src/storm/PageGenerator.ts

@@ -0,0 +1,181 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import uuid from 'node-uuid';
+import { stormClient, UIPagePrompt } from './stormClient';
+import { ReferenceClassification, StormEvent, StormEventPage, StormEventReferenceClassification } from './events';
+import { EventEmitter } from 'node:events';
+import { PromiseQueue } from './PromiseQueue';
+
+export class PageQueue extends EventEmitter {
+    private readonly queue: PromiseQueue;
+    private readonly systemId: string;
+    private readonly references: Map<string, PageGenerator> = new Map();
+
+    constructor(systemId: string, concurrency: number = 5) {
+        super();
+        this.systemId = systemId;
+        this.queue = new PromiseQueue(concurrency);
+    }
+
+    on(event: 'event', listener: (data: StormEvent) => void): this;
+    on(event: 'page', listener: (data: StormEventPage) => void): this;
+
+    on(event: string, listener: (...args: any[]) => void): this {
+        return super.on(event, listener);
+    }
+
+    emit(type: 'event', event: StormEvent): boolean;
+    emit(type: 'page', event: StormEventPage): boolean;
+    emit(eventName: string | symbol, ...args: any[]): boolean {
+        return super.emit(eventName, ...args);
+    }
+
+    public addPrompt(initialPrompt: UIPagePrompt) {
+        if (this.references.has(initialPrompt.path)) {
+            console.log('Ignoring duplicate prompt', initialPrompt.path);
+            return Promise.resolve();
+        }
+        console.log('processing prompt', initialPrompt.path);
+        const generator = new PageGenerator(initialPrompt);
+        this.references.set(initialPrompt.path, generator);
+        return this.addPageGenerator(generator);
+    }
+
+    private async addPageGenerator(generator: PageGenerator) {
+        generator.on('event', (event: StormEvent) => this.emit('event', event));
+        generator.on('page_refs', ({ event, references }) => {
+            this.emit('page', event);
+            references.forEach((reference) => {
+                if (
+                    reference.url.startsWith('#') ||
+                    reference.url.startsWith('javascript:') ||
+                    reference.url.startsWith('http://') ||
+                    reference.url.startsWith('https://')
+                ) {
+                    return;
+                }
+
+                switch (reference.type) {
+                    case 'image':
+                        console.log('Ignoring image reference', reference);
+                        break;
+                    case 'css':
+                    case 'javascript':
+                        //console.log('Ignoring reference', reference);
+                        break;
+                    case 'html':
+                        console.log('Adding page generator for', reference);
+                        const paths = Array.from(this.references.keys());
+                        this.addPrompt({
+                            name: reference.name,
+                            title: reference.title,
+                            path: reference.url,
+                            method: 'GET',
+                            storage_prefix: this.systemId + '_',
+                            prompt:
+                                `Implement a page for ${reference.name} at ${reference.url} with the following description: ${reference.description}.\n` +
+                                `The page was referenced from this page: \`\`\`html\n${event.payload.content}\n\`\`\`\n` +
+                                (paths.length > 0
+                                    ? `\nThese paths are already implemented:\n- ${paths.join('\n - ')}\n\n`
+                                    : ''),
+                            description: reference.description,
+                            filename: '',
+                        });
+                        break;
+                }
+            });
+        });
+        return this.queue.add(() => generator.generate());
+    }
+
+    public cancel() {
+        this.queue.cancel();
+    }
+
+    public wait() {
+        return this.queue.wait();
+    }
+}
+
+export class PageGenerator extends EventEmitter {
+    private readonly conversationId: string;
+    private prompt: UIPagePrompt;
+
+    constructor(prompt: UIPagePrompt, conversationId: string = uuid.v4()) {
+        super();
+        this.conversationId = conversationId;
+        this.prompt = prompt;
+    }
+
+    on(event: 'event', listener: (data: StormEvent) => void): this;
+    on(
+        event: 'page_refs',
+        listener: (data: { event: StormEventPage; references: ReferenceClassification[] }) => void
+    ): this;
+
+    on(event: string, listener: (...args: any[]) => void): this {
+        return super.on(event, listener);
+    }
+
+    emit(type: 'event', event: StormEvent): boolean;
+    emit(type: 'page_refs', event: { event: StormEventPage; references: ReferenceClassification[] }): boolean;
+    emit(eventName: string | symbol, ...args: any[]): boolean {
+        return super.emit(eventName, ...args);
+    }
+
+    public async generate() {
+        return new Promise<void>(async (resolve) => {
+            const screenStream = await stormClient.createUIPage(this.prompt, this.conversationId);
+
+            const promises: Promise<void>[] = [];
+
+            screenStream.on('data', (event: StormEvent) => {
+                if (event.type === 'PAGE') {
+                    event.payload.conversationId = this.conversationId;
+
+                    promises.push(
+                        (async () => {
+                            const references = await this.resolveReferences(event.payload.content);
+                            //console.log('Resolved references for page', references, event.payload);
+                            this.emit('page_refs', {
+                                event,
+                                references,
+                            });
+                        })()
+                    );
+                    return;
+                }
+
+                this.emit('event', event);
+            });
+
+            screenStream.on('end', () => {
+                Promise.allSettled(promises).finally(resolve);
+            });
+
+            await screenStream.waitForDone();
+        });
+    }
+
+    private async resolveReferences(content: string) {
+        const referenceStream = await stormClient.classifyUIReferences(content);
+
+        const references: ReferenceClassification[] = [];
+
+        referenceStream.on('data', (referenceData: StormEvent) => {
+            if (referenceData.type !== 'REF_CLASSIFICATION') {
+                return;
+            }
+
+            //console.log('Processing reference classification', referenceData);
+            references.push(referenceData.payload);
+        });
+
+        await referenceStream.waitForDone();
+
+        return references;
+    }
+}

package/src/storm/event-parser.ts

@@ -357,6 +357,18 @@ export class StormEventParser {
                     this.blocks[evt.payload.blockName].models = [];
                 }
                 break;
+
+            case 'API_STREAM_CHUNK':
+            case 'API_STREAM_CHUNK_RESET':
+            case 'API_STREAM_DONE':
+            case 'API_STREAM_FAILED':
+            case 'API_STREAM_STATE':
+            case 'API_STREAM_START':
+                if ('blockName' in evt.payload) {
+                    evt.payload.blockRef = StormEventParser.toRef(handle, evt.payload.blockName).toNormalizedString();
+                    evt.payload.instanceId = StormEventParser.toInstanceIdFromRef(evt.payload.blockRef);
+                }
+                break;
         }
 
         return await this.toResult(handle);

package/src/storm/events.ts

@@ -263,6 +263,11 @@ export interface StormEventFileChunk extends StormEventFileBase {
     };
 }
 
+export interface StormEventApiBase {
+    type: 'API_STREAM_CHUNK' | 'API_STREAM_DONE' | 'API_STREAM_FAILED' | 'API_STREAM_STATE' | 'API_STREAM_START' | 'API_STREAM_CHUNK_RESET';
+    payload: StormEventFileBasePayload;
+}
+
 export interface StormEventBlockReady {
     type: 'BLOCK_READY';
     reason: string;
@@ -406,6 +411,40 @@ export interface StormEventPromptImprove {
     };
 }
 
+export interface LandingPage {
+    name: string;
+    title: string;
+    filename: string;
+    create_prompt: string;
+    path: string;
+    archetype: string;
+    requires_authentication: boolean;
+}
+
+// Event for defining a landing pages
+export interface StormEventLandingPage {
+    type: 'LANDING_PAGE';
+    reason: string;
+    created: number;
+    payload: LandingPage;
+}
+
+export interface ReferenceClassification {
+    name: string;
+    title: string;
+    url: string;
+    description: string;
+    type: 'image' | 'css' | 'javascript' | 'html';
+    source: 'local' | 'cdn' | 'example';
+}
+// Event for reference classification
+export interface StormEventReferenceClassification {
+    type: 'REF_CLASSIFICATION';
+    reason: string;
+    created: number;
+    payload: ReferenceClassification;
+}
+
 export type StormEvent =
     | StormEventCreateBlock
     | StormEventCreateConnection
@@ -435,4 +474,7 @@ export type StormEvent =
     | StormEventUIShell
     | StormEventPage
     | StormEventPageUrl
-    | StormEventPromptImprove;
+    | StormEventPromptImprove
+    | StormEventLandingPage
+    | StormEventReferenceClassification
+    | StormEventApiBase;

package/src/storm/page-utils.ts

@@ -7,18 +7,23 @@ import { Response } from 'express';
 import os from 'node:os';
 import Path from 'path';
 import FS from 'fs-extra';
+import FSExtra from 'fs-extra';
+import { ConversationItem } from './stream';
+import exp from 'node:constants';
 
 export const SystemIdHeader = 'System-Id';
 
+function normalizePath(path: string) {
+    return path
+        .replace(/\?.*$/gi, '')
+        .replace(/:[a-z][a-z_]*\b/gi, '*')
+        .replace(/\{[a-z]+}/gi, '*');
+}
+
 export async function writePageToDisk(systemId: string, event: StormEventPage) {
-    const path = Path.join(
-        os.tmpdir(),
-        'ai-systems',
-        systemId,
-        event.payload.path,
-        event.payload.method.toLowerCase(),
-        'index.html'
-    );
+    const baseDir = getBaseDir(systemId);
+    const filePath = getFilePath(event.payload.method);
+    const path = Path.join(baseDir, normalizePath(event.payload.path), filePath);
     await FS.ensureDir(Path.dirname(path));
     await FS.writeFile(path, event.payload.content);
 
@@ -29,9 +34,58 @@ export async function writePageToDisk(systemId: string, event: StormEventPage) {
     };
 }
 
+function getBaseDir(systemId: string) {
+    return Path.join(os.tmpdir(), 'ai-systems', systemId);
+}
+
+function getFilePath(method: string) {
+    return Path.join(method.toLowerCase(), 'index.html');
+}
+
+export function resolveReadPath(systemId: string, path: string, method: string) {
+    const baseDir = getBaseDir(systemId);
+
+    path = normalizePath(path);
+
+    const filePath = getFilePath(method);
+
+    const fullPath = Path.join(baseDir, path, filePath);
+
+    if (FS.existsSync(fullPath)) {
+        return fullPath;
+    }
+
+    const parts = path.split(/\*/g);
+
+    let currentPath = '';
+
+    for (let part in parts) {
+        const thisPath = Path.join(currentPath, part);
+        const starPath = Path.join(currentPath, '*');
+        const thisPathDir = Path.join(baseDir, thisPath);
+        const starPathDir = Path.join(baseDir, starPath);
+
+        if (FS.existsSync(thisPathDir)) {
+            currentPath = thisPath;
+            continue;
+        }
+
+        if (FS.existsSync(starPathDir)) {
+            currentPath = starPath;
+            continue;
+        }
+
+        console.log('Path not found', thisPathDir, starPathDir);
+        // Path not found
+        return null;
+    }
+
+    return Path.join(baseDir, currentPath, filePath);
+}
+
 export function readPageFromDiskAsString(systemId: string, path: string, method: string) {
-    const filePath = Path.join(os.tmpdir(), 'ai-systems', systemId, path, method.toLowerCase(), 'index.html');
-    if (!FS.existsSync(filePath)) {
+    const filePath = resolveReadPath(systemId, path, method);
+    if (!filePath || !FS.existsSync(filePath)) {
         return null;
     }
 
@@ -39,8 +93,8 @@ export function readPageFromDiskAsString(systemId: string, path: string, method:
 }
 
 export function readPageFromDisk(systemId: string, path: string, method: string, res: Response) {
-    const filePath = Path.join(os.tmpdir(), 'ai-systems', systemId, path, method.toLowerCase(), 'index.html');
-    if (!FS.existsSync(filePath)) {
+    const filePath = resolveReadPath(systemId, path, method);
+    if (!filePath || !FS.existsSync(filePath)) {
         res.status(404).send('Page not found');
         return;
     }
@@ -51,3 +105,25 @@ export function readPageFromDisk(systemId: string, path: string, method: string,
     res.write(content);
     res.end();
 }
+
+export interface Conversation {
+    messages: ConversationItem[];
+    variantId: string;
+    type: 'page';
+    filename: string;
+}
+
+export function readConversationFromFile(filename: string): Conversation[] {
+    if (!FS.existsSync(filename)) {
+        return [];
+    }
+    const content = FS.readFileSync(filename).toString();
+    if (!content.trim()) {
+        return [];
+    }
+    return content.split(/\n/g).map((line) => JSON.parse(line) as Conversation);
+}
+
+export function writeConversationToFile(filename: string, conversations: Conversation[]) {
+    FS.writeFileSync(filename, conversations.map((conversation) => JSON.stringify(conversation)).join('\n'));
+}