@kapeta/local-cluster-service 0.74.2 → 0.76.0

package/src/storm/codegen.ts

@@ -7,7 +7,6 @@ import { Definition } from '@kapeta/local-cluster-config';
 import {
     AIFileTypes,
     BlockCodeGenerator,
-    CodeGenerator,
     CodeWriter,
     GeneratedFile,
     GeneratedResult,
@@ -21,20 +20,18 @@ import { STORM_ID, StormClient } from './stormClient';
 import {
     StormEvent,
     StormEventBlockStatusType,
-    StormEventErrorDetailsFile,
     StormEventFileChunk,
     StormEventFileDone,
     StormEventFileLogical,
     StormEventScreen,
 } from './events';
 import { BlockDefinitionInfo, StormEventParser } from './event-parser';
-import { ConversationItem, StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
+import { StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
 import { KapetaURI, parseKapetaUri } from '@kapeta/nodejs-utils';
 import { writeFile } from 'fs/promises';
 import path from 'path';
 import Path, { join } from 'path';
 import os from 'node:os';
-import { readFileSync, writeFileSync, existsSync } from 'fs';
 import YAML from 'yaml';
 import { PREDEFINED_BLOCKS } from './predefined';
 import { Archetype } from './archetype';
@@ -46,12 +43,6 @@ type ImplementationGenerator<T = StormFileImplementationPrompt> = (
     conversationId?: string
 ) => Promise<StormStream>;
 
-interface ErrorClassification {
-    error: string;
-    lineNumber: number;
-    column: number;
-}
-
 const SIMULATED_DELAY = 1000;
 const ENABLE_SIMULATED_DELAY = false;
 class SimulatedFileDelay {
@@ -371,7 +362,7 @@ export class StormCodegen {
         });
         const uiEvents = [];
 
-        const stormClient = new StormClient(this.uiSystemId);
+        const stormClient = new StormClient(blockUri.handle, this.uiSystemId);
         // generate screens
         if (uiTemplates.length) {
             const screenStream = await stormClient.listScreens({
@@ -540,249 +531,6 @@ export class StormCodegen {
         } satisfies StormEvent);
     }
 
-    private async verifyAndFixCode(
-        blockUri: KapetaURI,
-        blockName: string,
-        codeGenerator: CodeGenerator,
-        basePath: string,
-        filesToBeFixed: StormFileInfo[],
-        allFiles: StormFileInfo[]
-    ) {
-        let attempts = 0;
-        let validCode = false;
-        for (let i = 0; i <= 3; i++) {
-            attempts++;
-            try {
-                console.log(`Validating the code in ${basePath} attempt #${attempts}`);
-                const result = await codeGenerator.validateForTarget(basePath);
-                if (result && result.valid) {
-                    validCode = true;
-                    break;
-                }
-
-                if (result && !result.valid) {
-                    console.debug('Validation error:', result);
-
-                    this.emitBlockStatus(blockUri, blockName, StormEventBlockStatusType.PLANNING_FIX);
-
-                    const errors = await this.classifyErrors(result.error, basePath);
-
-                    if (errors.size > 0) {
-                        this.emitBlockStatus(blockUri, blockName, StormEventBlockStatusType.FIXING);
-
-                        const promises = Array.from(errors.entries()).map(([filename, fileErrors]) => {
-                            if (filesToBeFixed.some((file) => file.filename === filename)) {
-                                return this.tryToFixFile(
-                                    blockUri,
-                                    blockName,
-                                    basePath,
-                                    filename,
-                                    fileErrors,
-                                    allFiles,
-                                    codeGenerator
-                                );
-                            }
-                        });
-
-                        await Promise.all(promises);
-                    }
-
-                    this.emitBlockStatus(blockUri, blockName, StormEventBlockStatusType.FIX_DONE);
-                }
-            } catch (e) {
-                console.error('Error:', e);
-            }
-        }
-
-        if (validCode) {
-            console.log(`Validation successful after ${attempts} attempts`);
-        } else {
-            console.error(`Validation failed for ${basePath} after ${attempts} attempts`);
-        }
-    }
-
-    private async tryToFixFile(
-        blockUri: KapetaURI,
-        blockName: string,
-        basePath: string,
-        filename: string,
-        fileErrors: ErrorClassification[],
-        allFiles: StormFileInfo[],
-        codeGenerator: CodeGenerator
-    ) {
-        console.log(`Processing ${filename}`);
-        const language = await codeGenerator.language();
-        const relevantFiles = allFiles.filter((file) => file.type != AIFileTypes.IGNORE);
-
-        for (let attempts = 1; attempts <= 5; attempts++) {
-            if (fileErrors.length == 0) {
-                console.log(`No more errors for ${filename}`);
-                return;
-            }
-
-            console.log(`Errors in ${filename} - requesting error details`);
-            const filesForContext = await this.getErrorDetailsForFile(
-                basePath,
-                filename,
-                fileErrors[0],
-                relevantFiles,
-                language
-            );
-            console.log(`Get error details for ${filename} requesting code fixes`);
-
-            const fix = this.createFixRequestForFile(
-                basePath,
-                filename,
-                fileErrors[0],
-                filesForContext,
-                relevantFiles,
-                language
-            );
-            const codeFixFile = await this.codeFix(blockUri, blockName, fix);
-            console.log(`Got fixed code for ${filename}`);
-            const filePath =
-                codeFixFile.filename.indexOf(basePath) > -1
-                    ? codeFixFile.filename
-                    : join(basePath, codeFixFile.filename);
-            const existing = readFileSync(filePath);
-            if (
-                existing.toString().replace(/(\r\n|\r|\n)+$/, '') == codeFixFile.content.replace(/(\r\n|\r|\n)+$/, '')
-            ) {
-                console.log(`${filename} not changed by gemini`);
-                continue;
-            }
-            writeFileSync(filePath, codeFixFile.content);
-
-            const result = await codeGenerator.validateForTarget(basePath);
-            if (result && result.valid) {
-                return;
-            }
-
-            const errors = await this.classifyErrors(result.error, basePath);
-            fileErrors = errors.get(filename) ?? [];
-        }
-    }
-
-    private async classifyErrors(errors: string, basePath: string): Promise<Map<string, ErrorClassification[]>> {
-        const stormClient = new StormClient(this.uiSystemId);
-        const errorStream = await stormClient.createErrorClassification(errors, []);
-        const fixes = new Map<string, ErrorClassification[]>();
-
-        this.out.on('aborted', () => {
-            errorStream.abort();
-        });
-
-        errorStream.on('data', (evt) => {
-            if (evt.type === 'ERROR_CLASSIFIER') {
-                const eventFileName = this.removePrefix(basePath + '/', evt.payload.filename);
-                const fix = {
-                    error: evt.payload.error,
-                    lineNumber: evt.payload.lineNumber,
-                    column: evt.payload.column,
-                };
-
-                let existingFixes = fixes.get(eventFileName);
-                if (existingFixes) {
-                    existingFixes.push(fix);
-                } else {
-                    fixes.set(eventFileName, [fix]);
-                }
-            }
-        });
-
-        await errorStream.waitForDone();
-
-        return fixes;
-    }
-
-    private async getErrorDetailsForFile(
-        basePath: string,
-        filename: string,
-        error: ErrorClassification,
-        allFiles: StormFileInfo[],
-        language: string
-    ): Promise<string[]> {
-        const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename); // to compensate when compiler returns absolute path
-        return new Promise<string[]>(async (resolve, reject) => {
-            const request = {
-                language: language,
-                sourceFile: {
-                    filename: filename,
-                    content: readFileSync(filePath, 'utf8'),
-                },
-                error: error,
-                projectFiles: allFiles.map((f) => f.filename),
-            };
-            const stormClient = new StormClient(this.uiSystemId);
-            const detailsStream = await stormClient.createErrorDetails(JSON.stringify(request), []);
-            detailsStream.on('data', (evt) => {
-                if (evt.type === 'ERROR_DETAILS') {
-                    resolve(evt.payload.files);
-                }
-                reject(new Error('Error details: Unexpected event [' + evt.type + ']'));
-            });
-            this.out.on('aborted', () => {
-                detailsStream.abort();
-                reject(new Error('aborted'));
-            });
-            detailsStream.on('error', (err) => {
-                reject(err);
-            });
-            await detailsStream.waitForDone();
-        });
-    }
-
-    private createFixRequestForFile(
-        basePath: string,
-        filename: string,
-        error: ErrorClassification,
-        filesForContext: string[],
-        allFiles: StormFileInfo[],
-        language: string
-    ): string {
-        const files = new Set(filesForContext);
-        files.add(filename);
-
-        const requestedFiles = Array.from(files).flatMap((file) => {
-            if (existsSync(file)) {
-                return file;
-            }
-
-            // file does not exist - look for similar
-            const candidateName = file.split('/').pop();
-            return allFiles.filter((file) => file.filename.split('/').pop() === candidateName).map((f) => f.filename);
-        });
-
-        const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename);
-        const content = readFileSync(filePath, 'utf8');
-        const affectedLine = this.getErrorLine(error, content);
-
-        const fixRequest = {
-            language: language,
-            filename: filename,
-            error: error.error,
-            affectedLine: affectedLine,
-            projectFiles: requestedFiles.map((filename) => {
-                const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename);
-                const content = readFileSync(filePath, 'utf8');
-                return { filename: filename, content: content };
-            }),
-        };
-
-        return JSON.stringify(fixRequest);
-    }
-
-    private getErrorLine(errorDetails: ErrorClassification, sourceCode: string): string {
-        const lines = sourceCode.split('\n');
-        const errorLine = lines[errorDetails.lineNumber - 1];
-
-        if (!errorLine) {
-            return 'Error: Line number out of range.';
-        }
-
-        return errorLine;
-    }
-
     removePrefix(prefix: string, str: string): string {
         if (str.startsWith(prefix)) {
             return str.slice(prefix.length);
@@ -790,49 +538,6 @@ export class StormCodegen {
         return str;
     }
 
-    /**
-     * Sends the code to the AI for a fix
-     */
-    private async codeFix(
-        blockUri: KapetaURI,
-        blockName: string,
-        fix: string,
-        history?: ConversationItem[]
-    ): Promise<StormEventErrorDetailsFile> {
-        return new Promise<StormEventErrorDetailsFile>(async (resolve, reject) => {
-            try {
-                const stormClient = new StormClient(this.uiSystemId);
-                const fixStream = await stormClient.createCodeFix(fix, history, this.conversationId);
-                let resolved = false;
-                fixStream.on('data', (evt) => {
-                    if (this.handleFileEvents(blockUri, blockName, evt)) {
-                        return;
-                    }
-                    this.handleFileDoneOutput(blockUri, blockName, evt);
-
-                    if (evt.type === 'CODE_FIX') {
-                        resolved = true;
-                        resolve(evt.payload);
-                    }
-                });
-                this.out.on('aborted', () => {
-                    fixStream.abort();
-                    reject(new Error('aborted'));
-                });
-                fixStream.on('error', (err) => {
-                    reject(err);
-                });
-                fixStream.on('end', () => {
-                    if (!resolved) {
-                        reject(new Error('Code fix never returned a valid event'));
-                    }
-                });
-            } catch (e) {
-                reject(e);
-            }
-        });
-    }
-
     /**
      * Emits the text-based files to the stream
      */

package/src/storm/routes.ts

@@ -152,6 +152,7 @@ router.post('/ui/conversations/:systemId/append', async (req: KapetaBodyRequest,
 
 router.post('/ui/create-system/:handle/:systemId', async (req: KapetaBodyRequest, res: Response) => {
     const systemId = req.params.systemId as string;
+    const handle = req.params.handle as string;
     const srcDir = getSystemBaseDir(systemId);
     const destDir = getSystemBaseImplDir(systemId);
 
@@ -162,7 +163,7 @@ router.post('/ui/create-system/:handle/:systemId', async (req: KapetaBodyRequest
     sendEvent(res, createPhaseStartEvent(StormEventPhaseType.IMPLEMENT_APIS));
 
     const pagesFromDisk = readFilesAndContent(srcDir);
-    const client = new StormClient(systemId)
+    const client = new StormClient(handle, systemId)
     const pagesWithImplementation = await client.replaceMockWithAPICall({
         pages: pagesFromDisk,
         systemId: systemId,
@@ -206,7 +207,7 @@ router.post('/ui/create-system-simple/:handle/:systemId', async (req: KapetaBody
     //res.set(ConversationIdHeader, systemId);
 
     //sendEvent(res, createPhaseStartEvent(StormEventPhaseType.IMPLEMENT_APIS));
-    const client = new StormClient(systemId);
+    const client = new StormClient(handle, systemId);
     try {
         const pagesFromDisk = readFilesAndContent(srcDir);
         const pagesWithImplementation = await client.replaceMockWithAPICall({
@@ -260,6 +261,23 @@ router.post('/ui/systems/:handle/:systemId/upload', async (req: KapetaBodyReques
     res.send({ ok: true });
 });
 
+router.put('/ui/systems/:handle/:systemId/thumbnail', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    await stormService.saveThumbnail(systemId, req.body as Buffer);
+    res.send({ ok: true });
+});
+
+router.get('/ui/systems/:handle/:systemId/thumbnail.png', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    const thumbnail = await stormService.getThumbnail(systemId);
+    if (thumbnail) {
+        res.set('Content-Type', 'image/png');
+        res.send(thumbnail);
+    } else {
+        res.status(404).send({ error: 'No thumbnail found' });
+    }
+});
+
 router.delete('/ui/serve/:systemId', async (req: KapetaBodyRequest, res: Response) => {
     const systemId = req.params.systemId as string | undefined;
     if (!systemId) {
@@ -289,7 +307,7 @@ router.post('/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
 
         const parentConversationId = systemId ?? '';
 
-        const queue = new PageQueue(parentConversationId, '', 5);
+        const queue = new PageQueue("", parentConversationId, '', 5);
         onRequestAborted(req, res, () => {
             queue.cancel();
         });
@@ -331,7 +349,7 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
         const conversationId = req.headers[ConversationIdHeader.toLowerCase()] as string | undefined;
 
         const aiRequest: BasePromptRequest = JSON.parse(req.stringBody ?? '{}');
-        const client = new StormClient(conversationId); //todo is this correct we are using the landing page getConversationId down below as well
+        const client = new StormClient(handle, conversationId); //todo is this correct we are using the landing page getConversationId down below as well
         const landingPagesStream = await client.createUILandingPages(aiRequest, conversationId);
 
         onRequestAborted(req, res, () => {
@@ -394,7 +412,7 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
             systemPrompt.resolve(aiRequest.prompt);
         });
 
-        const pageQueue = new PageQueue(systemId, await systemPrompt.promise, 5);
+        const pageQueue = new PageQueue(handle, systemId, await systemPrompt.promise, 5);
         onRequestAborted(req, res, () => {
             pageQueue.cancel();
         });
@@ -440,7 +458,7 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
             (req.headers[ConversationIdHeader.toLowerCase()] as string | undefined) || randomUUID();
 
         const aiRequest: BasePromptRequest = JSON.parse(req.stringBody ?? '{}');
-        const stormClient = new StormClient(outerConversationId);
+        const stormClient = new StormClient(handle, outerConversationId);
         // Get user journeys
         const userJourneysStream = await stormClient.createUIUserJourneys(aiRequest, outerConversationId);
 
@@ -555,7 +573,7 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
             shellsStream.abort();
         });
 
-        const queue = new PageQueue(outerConversationId, systemPrompt, 5);
+        const queue = new PageQueue(handle, outerConversationId, systemPrompt, 5);
         queue.setUiTheme(theme);
 
         shellsStream.on('data', (data: StormEvent) => {
@@ -662,7 +680,7 @@ router.post('/ui/edit', async (req: KapetaBodyRequest, res: Response) => {
         const aiRequest: StormContextRequest<UIPageEditRequest> = JSON.parse(req.stringBody ?? '{}');
         const storagePrefix = systemId ? systemId + '_' : 'mock_';
 
-        const queue = new PageQueue(systemId!, '', 5);
+        const queue = new PageQueue("",systemId!, '', 5);
 
         onRequestAborted(req, res, () => {
             queue.cancel();
@@ -736,7 +754,7 @@ router.post('/ui/vote', async (req: KapetaBodyRequest, res: Response) => {
     const aiRequest: UIPageVoteRequest = JSON.parse(req.stringBody ?? '{}');
     const { topic, vote, mainConversationId } = aiRequest;
     try {
-        const stormClient = new StormClient(mainConversationId);
+        const stormClient = new StormClient("", mainConversationId);
         await stormClient.voteUIPage(topic, conversationId, vote, mainConversationId);
     } catch (e: any) {
         res.status(500).send({ error: e.message });
@@ -748,7 +766,7 @@ router.post('/ui/get-vote', async (req: KapetaBodyRequest, res: Response) => {
     const aiRequest: UIPageGetVoteRequest = JSON.parse(req.stringBody ?? '{}');
     const { topic, mainConversationId } = aiRequest;
     try {
-        const stormClient = new StormClient(mainConversationId);
+        const stormClient = new StormClient("", mainConversationId);
         const vote = await stormClient.getVoteUIPage(topic, conversationId, mainConversationId);
         res.send({ vote });
     } catch (e: any) {
@@ -780,7 +798,7 @@ async function handleAll(req: KapetaBodyRequest, res: Response) {
         const conversationId = req.headers[ConversationIdHeader.toLowerCase()] as string | undefined;
 
         const aiRequest: BasePromptRequest = JSON.parse(req.stringBody ?? '{}');
-        const stormClient = new StormClient(systemId);
+        const stormClient = new StormClient(handle, systemId);
         const metaStream = await stormClient.createMetadata(aiRequest, conversationId);
 
         onRequestAborted(req, res, () => {

package/src/storm/stormClient.ts

@@ -25,6 +25,7 @@ export const STORM_ID = 'storm';
 
 export const ConversationIdHeader = 'Conversation-Id';
 export const SystemIdHeader = 'System-Id';
+export const HandleHeader = 'Handle';
 
 export interface UIShellsPrompt {
     theme?: string;
@@ -89,9 +90,11 @@ export interface BasePromptRequest {
 export class StormClient {
     private readonly _baseUrl: string;
     private readonly _systemId: string;
-    constructor(systemId?: string) {
+    private readonly _handle: string;
+    constructor(handle: string, systemId?: string) {
         this._baseUrl = getRemoteUrl('ai-service', 'https://ai.kapeta.com');
         this._systemId = systemId || "";
+        this._handle = handle;
     }
 
     private async createOptions(
@@ -115,6 +118,9 @@ export class StormClient {
         if (this._systemId) {
             headers[SystemIdHeader] = this._systemId
         }
+        if (this._handle) {
+            headers[HandleHeader] = this._handle
+        }
         return {
             url,
             method: method,
@@ -251,6 +257,7 @@ export class StormClient {
             body: JSON.stringify(prompt.pages),
             headers: {
                 'systemId': prompt.systemId,
+                'conversationId': prompt.systemId,
             },
         });
         return (await response.json()) as HTMLPage[];

package/src/stormService.ts

@@ -1,5 +1,5 @@
 import fs from 'fs/promises';
-import { glob } from 'glob';
+import { glob, Path } from 'glob';
 import { filesystemManager } from './filesystemManager';
 import path from 'path';
 import { existsSync } from 'fs';
@@ -16,6 +16,10 @@ export class StormService {
         return path.join(filesystemManager.getProjectRootFolder()!, 'ai-systems', conversationId, 'system.tar.gz');
     }
 
+    private getThumbnailFile(conversationId: string) {
+        return path.join(filesystemManager.getProjectRootFolder()!, 'ai-systems', conversationId, 'thumbnail.png');
+    }
+
     async listRemoteConversations() {
         // i.e. conversations from org / user on registry
         return [];
@@ -25,13 +29,24 @@ export class StormService {
         const systemsFolder = path.join(filesystemManager.getProjectRootFolder()!, 'ai-systems');
         const eventFiles = await glob('*/events.ndjson', {
             cwd: systemsFolder,
-            absolute: true,
+            stat: true,
+            withFileTypes: true,
         });
         // Returns list of UUIDs - probably want to make it more useful than that
-        const conversations: { id: string; description: string; title: string; url?: string }[] = [];
+        const conversations: {
+            id: string;
+            description: string;
+            title: string;
+            url?: string;
+            lastModified?: number;
+            createdAt?: number;
+            thumbnail?: string;
+        }[] = [];
+        // Sort by modification time, newest first
+        eventFiles.sort((a, b) => (b.mtimeMs || 0) - (a.mtimeMs || 0));
         for (const file of eventFiles) {
             try {
-                const nldContents = await fs.readFile(file as string, 'utf8');
+                const nldContents = await fs.readFile(file.fullpath(), 'utf8');
                 const events = nldContents.split('\n').map((e) => JSON.parse(e)) as {
                     // | { type: 'USER'; event: any } // IS stupid!
                     type: 'AI';
@@ -79,6 +94,9 @@ export class StormService {
                     description: initialPrompt,
                     title: title || 'New system',
                     url,
+                    lastModified: file.mtimeMs,
+                    createdAt: file.birthtimeMs,
+                    thumbnail: existsSync(this.getThumbnailFile(id)) ? `thumbnail.png?v=${file.mtimeMs}` : undefined,
                 });
             } catch (e) {
                 console.error('Failed to load conversation at %s', file, e);
@@ -131,14 +149,14 @@ export class StormService {
             },
             ['.']
         );
-        const stormClient = new StormClient(systemId);
+        const stormClient = new StormClient(handle, systemId);
         await stormClient.uploadSystem(handle, systemId, await fs.readFile(tarballFile));
     }
 
     async installProjectById(handle: string, systemId: string) {
         const tarballFile = this.getConversationTarball(systemId);
         const destDir = path.dirname(tarballFile);
-        const stormClient = new StormClient(systemId);
+        const stormClient = new StormClient(handle, systemId);
         const buffer = await stormClient.downloadSystem(handle, systemId);
         await fs.mkdir(destDir, { recursive: true });
         await fs.writeFile(tarballFile, buffer);
@@ -148,6 +166,20 @@ export class StormService {
         });
         await fs.unlink(tarballFile);
     }
+
+    async saveThumbnail(systemId: string, thumbnail: Buffer) {
+        const thumbnailFile = this.getThumbnailFile(systemId);
+        await fs.mkdir(path.dirname(thumbnailFile), { recursive: true });
+        await fs.writeFile(thumbnailFile, thumbnail);
+    }
+
+    async getThumbnail(systemId: string) {
+        const thumbnailFile = this.getThumbnailFile(systemId);
+        if (existsSync(thumbnailFile)) {
+            return fs.readFile(thumbnailFile);
+        }
+        return null;
+    }
 }
 
 export default new StormService();