@kapeta/local-cluster-service 0.52.2 → 0.53.0

package/src/storm/codegen.ts

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-import {Definition} from '@kapeta/local-cluster-config';
+import { Definition } from '@kapeta/local-cluster-config';
 import {
     AIFileTypes,
     BlockCodeGenerator,
@@ -13,27 +13,27 @@ import {
     GeneratedResult,
     MODE_CREATE_ONLY,
 } from '@kapeta/codegen';
-import {BlockDefinition} from '@kapeta/schemas';
-import {codeGeneratorManager} from '../codeGeneratorManager';
-import {STORM_ID, stormClient} from './stormClient';
+import { BlockDefinition } from '@kapeta/schemas';
+import { codeGeneratorManager } from '../codeGeneratorManager';
+import { STORM_ID, stormClient } from './stormClient';
 import {
     StormEvent,
+    StormEventBlockStatusType,
     StormEventErrorDetailsFile,
     StormEventFileChunk,
     StormEventFileDone,
-    StormEventFileLogical
+    StormEventFileLogical,
 } from './events';
-import {BlockDefinitionInfo, StormEventParser} from './event-parser';
-import {ConversationItem, StormFileImplementationPrompt, StormFileInfo, StormStream} from './stream';
-import {KapetaURI, parseKapetaUri} from '@kapeta/nodejs-utils';
-import {writeFile} from 'fs/promises';
+import { BlockDefinitionInfo, StormEventParser } from './event-parser';
+import { ConversationItem, StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
+import { KapetaURI, parseKapetaUri } from '@kapeta/nodejs-utils';
+import { writeFile } from 'fs/promises';
 import path from 'path';
-import Path, {join} from 'path';
+import Path, { join } from 'path';
 import os from 'node:os';
-import {readFileSync, writeFileSync} from 'fs';
-import YAML from "yaml";
-import * as fs from "node:fs";
-import {v4 as uuidv4} from 'uuid';
+import { readFileSync, writeFileSync } from 'fs';
+import YAML from 'yaml';
+import * as fs from 'node:fs';
 
 type ImplementationGenerator = (prompt: StormFileImplementationPrompt, conversationId?: string) => Promise<StormStream>;
 
@@ -44,7 +44,7 @@ interface ErrorClassification {
 }
 
 const SIMULATED_DELAY = 1000;
-
+const ENABLE_SIMULATED_DELAY = false;
 class SimulatedFileDelay {
     private readonly file: StormEventFileDone;
     public readonly stream;
@@ -166,43 +166,8 @@ export class StormCodegen {
                     },
                 });
                 break;
-            case 'FILE_START':
-            case 'FILE_CHUNK_RESET':
-                this.out.emit('data', {
-                    ...data,
-                    payload: {
-                        ...data.payload,
-                        blockName,
-                        blockRef,
-                        instanceId,
-                    },
-                });
-                break;
-            case 'FILE_CHUNK':
-                this.out.emit('data', {
-                    ...data,
-                    payload: {
-                        ...data.payload,
-                        blockName,
-                        blockRef,
-                        instanceId,
-                    },
-                });
-                break;
-            case 'FILE_STATE':
-                this.out.emit('data', {
-                    ...data,
-                    payload: {
-                        ...data.payload,
-                        blockName,
-                        blockRef,
-                        instanceId,
-                    },
-                });
-                break;
             case 'FILE_DONE':
                 return this.handleFileDoneOutput(blockUri, blockName, data);
-                break;
         }
     }
 
@@ -335,11 +300,13 @@ export class StormCodegen {
             (file) => ![AIFileTypes.SERVICE, AIFileTypes.WEB_SCREEN].includes(file.type)
         );
 
+        const blockUri = parseKapetaUri(block.uri);
+
         // Send the service and UI templates to the AI. These will be sent one-by-one in addition to the context files
         const serviceFiles: StormFileInfo[] = allFiles.filter((file) => file.type === AIFileTypes.SERVICE);
         if (serviceFiles.length > 0) {
             await this.processTemplates(
-                parseKapetaUri(block.uri),
+                blockUri,
                 block.aiName,
                 stormClient.createServiceImplementation.bind(stormClient),
                 serviceFiles,
@@ -371,7 +338,7 @@ export class StormCodegen {
             await writeFile(filePath, screenFile.payload.content);
         }
 
-        const screenFilesConverted = screenFiles.map(screenFile => {
+        const screenFilesConverted = screenFiles.map((screenFile) => {
             return {
                 filename: screenFile.payload.filename,
                 content: screenFile.payload.content,
@@ -381,12 +348,16 @@ export class StormCodegen {
             };
         });
         allFiles.push(...screenFilesConverted);
+        const blockRef = block.uri;
+
+        this.emitBlockStatus(blockUri, block.aiName, StormEventBlockStatusType.QA);
 
         const filesToBeFixed = serviceFiles.concat(contextFiles).concat(screenFilesConverted);
         const codeGenerator = new BlockCodeGenerator(block.content as BlockDefinition);
-        await this.verifyAndFixCode(codeGenerator, basePath, filesToBeFixed, allFiles);
 
-        const blockRef = block.uri;
+        this.emitBlockStatus(blockUri, block.aiName, StormEventBlockStatusType.BUILDING);
+        await this.verifyAndFixCode(blockUri, block.aiName, codeGenerator, basePath, filesToBeFixed, allFiles);
+
         this.out.emit('data', {
             type: 'BLOCK_READY',
             reason: 'Block ready',
@@ -400,7 +371,23 @@ export class StormCodegen {
         } satisfies StormEvent);
     }
 
+    private emitBlockStatus(blockUri: KapetaURI, blockName: string, status: StormEventBlockStatusType) {
+        this.out.emit('data', {
+            type: 'BLOCK_STATUS',
+            reason: status,
+            created: Date.now(),
+            payload: {
+                status,
+                blockName,
+                blockRef: blockUri.toNormalizedString(),
+                instanceId: StormEventParser.toInstanceIdFromRef(blockUri.toNormalizedString()),
+            },
+        } satisfies StormEvent);
+    }
+
     private async verifyAndFixCode(
+        blockUri: KapetaURI,
+        blockName: string,
         codeGenerator: CodeGenerator,
         basePath: string,
         filesToBeFixed: StormFileInfo[],
@@ -421,11 +408,30 @@ export class StormCodegen {
                 if (result && !result.valid) {
                     console.debug('Validation error:', result);
 
+                    this.emitBlockStatus(blockUri, blockName, StormEventBlockStatusType.PLANNING_FIX);
+
                     const errors = await this.classifyErrors(result.error, basePath);
-                    for (const [filename, fileErrors] of errors.entries()) {
-                        // todo: only try to fix file if it is part of filesToBeFixed
-                        await this.tryToFixFile(basePath, filename, fileErrors, allFiles, codeGenerator);
+
+                    if (errors.size > 0) {
+                        this.emitBlockStatus(blockUri, blockName, StormEventBlockStatusType.FIXING);
+
+                        const promises = Array.from(errors.entries()).map(([filename, fileErrors]) => {
+                            // todo: only try to fix file if it is part of filesToBeFixed
+                            return this.tryToFixFile(
+                                blockUri,
+                                blockName,
+                                basePath,
+                                filename,
+                                fileErrors,
+                                allFiles,
+                                codeGenerator
+                            );
+                        });
+
+                        await Promise.all(promises);
                     }
+
+                    this.emitBlockStatus(blockUri, blockName, StormEventBlockStatusType.FIX_DONE);
                 }
             } catch (e) {
                 console.error('Error:', e);
@@ -439,33 +445,57 @@ export class StormCodegen {
         }
     }
 
-    private async tryToFixFile(basePath: string, filename: string, fileErrors: ErrorClassification[], allFiles: StormFileInfo[], codeGenerator: CodeGenerator) {
+    private async tryToFixFile(
+        blockUri: KapetaURI,
+        blockName: string,
+        basePath: string,
+        filename: string,
+        fileErrors: ErrorClassification[],
+        allFiles: StormFileInfo[],
+        codeGenerator: CodeGenerator
+    ) {
         console.log(`Processing ${filename}`);
         const language = await codeGenerator.language();
-        const relevantFiles = allFiles.filter(file => file.type != AIFileTypes.IGNORE);
+        const relevantFiles = allFiles.filter((file) => file.type != AIFileTypes.IGNORE);
 
-        for (let attempts = 1;  attempts <= 5; attempts++) {
+        for (let attempts = 1; attempts <= 5; attempts++) {
             if (fileErrors.length == 0) {
                 console.log(`No more errors for ${filename}`);
                 return;
             }
 
             console.log(`Errors in ${filename} - requesting error details`);
-            const filesForContext = await this.getErrorDetailsForFile(basePath, filename, fileErrors[0], relevantFiles, language);
+            const filesForContext = await this.getErrorDetailsForFile(
+                basePath,
+                filename,
+                fileErrors[0],
+                relevantFiles,
+                language
+            );
             console.log(`Get error details for ${filename} requesting code fixes`);
 
-            const fix = this.createFixRequestForFile(basePath, filename, fileErrors[0], filesForContext, relevantFiles, language);
-            const codeFixFiles = await this.codeFix(fix);
+            const fix = this.createFixRequestForFile(
+                basePath,
+                filename,
+                fileErrors[0],
+                filesForContext,
+                relevantFiles,
+                language
+            );
+            const codeFixFile = await this.codeFix(blockUri, blockName, fix);
             console.log(`Got fixed code for ${filename}`);
-            for (const codeFixFile of codeFixFiles) {
-                const filePath = codeFixFile.filename.indexOf(basePath) > -1 ? codeFixFile.filename : join(basePath, codeFixFile.filename);
-                const existing = readFileSync(filePath);
-                if (existing.toString().replace(/(\r\n|\r|\n)+$/, '') == codeFixFile.content.replace(/(\r\n|\r|\n)+$/, '')) {
-                    console.log(`${filename} not changed by gemini`);
-                    continue;
-                }
-                writeFileSync(filePath, codeFixFile.content);
+            const filePath =
+                codeFixFile.filename.indexOf(basePath) > -1
+                    ? codeFixFile.filename
+                    : join(basePath, codeFixFile.filename);
+            const existing = readFileSync(filePath);
+            if (
+                existing.toString().replace(/(\r\n|\r|\n)+$/, '') == codeFixFile.content.replace(/(\r\n|\r|\n)+$/, '')
+            ) {
+                console.log(`${filename} not changed by gemini`);
+                continue;
             }
+            writeFileSync(filePath, codeFixFile.content);
 
             const result = await codeGenerator.validateForTarget(basePath);
             if (result && result.valid) {
@@ -488,7 +518,11 @@ export class StormCodegen {
         errorStream.on('data', (evt) => {
             if (evt.type === 'ERROR_CLASSIFIER') {
                 const eventFileName = this.removePrefix(basePath + '/', evt.payload.filename);
-                const fix = { error: evt.payload.error, lineNumber: evt.payload.lineNumber, column: evt.payload.column };
+                const fix = {
+                    error: evt.payload.error,
+                    lineNumber: evt.payload.lineNumber,
+                    column: evt.payload.column,
+                };
 
                 let existingFixes = fixes.get(eventFileName);
                 if (existingFixes) {
@@ -504,17 +538,23 @@ export class StormCodegen {
         return fixes;
     }
 
-    private async getErrorDetailsForFile(basePath: string, filename: string, error: ErrorClassification, allFiles: StormFileInfo[], language: string): Promise<string[]> {
+    private async getErrorDetailsForFile(
+        basePath: string,
+        filename: string,
+        error: ErrorClassification,
+        allFiles: StormFileInfo[],
+        language: string
+    ): Promise<string[]> {
         const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename); // to compensate when compiler returns absolute path
         return new Promise<string[]>(async (resolve, reject) => {
             const request = {
-                "language": language,
-                "sourceFile": {
-                    "filename": filename,
-                    "content": readFileSync(filePath, 'utf8')
+                language: language,
+                sourceFile: {
+                    filename: filename,
+                    content: readFileSync(filePath, 'utf8'),
                 },
-                "error": error,
-                "projectFiles": allFiles.map(f => f.filename)
+                error: error,
+                projectFiles: allFiles.map((f) => f.filename),
             };
 
             const detailsStream = await stormClient.createErrorDetails(JSON.stringify(request), []);
@@ -522,32 +562,38 @@ export class StormCodegen {
                 if (evt.type === 'ERROR_DETAILS') {
                     resolve(evt.payload.files);
                 }
+                reject(new Error('Error details: Unexpected event [' + evt.type + ']'));
             });
             this.out.on('aborted', () => {
                 detailsStream.abort();
+                reject(new Error('aborted'));
             });
             detailsStream.on('error', (err) => {
-                console.log("error", err);
                 reject(err);
             });
             await detailsStream.waitForDone();
         });
     }
 
-    private createFixRequestForFile(basePath: string, filename: string, error: ErrorClassification, filesForContext: string[], allFiles: StormFileInfo[], language: string): string {
+    private createFixRequestForFile(
+        basePath: string,
+        filename: string,
+        error: ErrorClassification,
+        filesForContext: string[],
+        allFiles: StormFileInfo[],
+        language: string
+    ): string {
         const files = new Set(filesForContext);
         files.add(filename);
 
-        const requestedFiles = Array.from(files).flatMap(file => {
+        const requestedFiles = Array.from(files).flatMap((file) => {
             if (fs.existsSync(file)) {
                 return file;
             }
 
             // file does not exist - look for similar
             const candidateName = file.split('/').pop();
-            return allFiles
-                .filter(file => file.filename.split('/').pop() === candidateName)
-                .map(f => f.filename);
+            return allFiles.filter((file) => file.filename.split('/').pop() === candidateName).map((f) => f.filename);
         });
 
         const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename);
@@ -555,31 +601,26 @@ export class StormCodegen {
         const affectedLine = this.getErrorLine(error, content);
 
         const fixRequest = {
-            "language": language,
-            "filename": filename,
-            "errors": error.error,
-            "affectedLine": affectedLine,
-            "projectFiles": requestedFiles.map(filename => {
+            language: language,
+            filename: filename,
+            error: error.error,
+            affectedLine: affectedLine,
+            projectFiles: requestedFiles.map((filename) => {
                 const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename);
                 const content = readFileSync(filePath, 'utf8');
                 return { filename: filename, content: content };
             }),
-            "conversationId": this.conversationId,
-            "sessionId": uuidv4()
         };
 
         return JSON.stringify(fixRequest);
     }
 
-    private getErrorLine(
-        errorDetails: ErrorClassification,
-        sourceCode: string
-    ): string {
+    private getErrorLine(errorDetails: ErrorClassification, sourceCode: string): string {
         const lines = sourceCode.split('\n');
         const errorLine = lines[errorDetails.lineNumber - 1];
 
         if (!errorLine) {
-            return "Error: Line number out of range.";
+            return 'Error: Line number out of range.';
         }
 
         return errorLine;
@@ -595,22 +636,41 @@ export class StormCodegen {
     /**
      * Sends the code to the AI for a fix
      */
-    private async codeFix(fix: string, history?: ConversationItem[]): Promise<StormEventErrorDetailsFile[]> {
-        return new Promise<StormEventErrorDetailsFile[]>(async (resolve, reject) => {
-            const fixStream = await stormClient.createCodeFix(fix, history);
-            fixStream.on('data', (evt) => {
-                if (evt.type === 'CODE_FIX') {
-                    resolve(evt.payload.content.files);
-                }
-            });
-            this.out.on('aborted', () => {
-                fixStream.abort();
-            });
-            fixStream.on('error', (err) => {
-                console.log("error", err);
-                reject(err);
-            });
-            await fixStream.waitForDone();
+    private async codeFix(
+        blockUri: KapetaURI,
+        blockName: string,
+        fix: string,
+        history?: ConversationItem[]
+    ): Promise<StormEventErrorDetailsFile> {
+        return new Promise<StormEventErrorDetailsFile>(async (resolve, reject) => {
+            try {
+                const fixStream = await stormClient.createCodeFix(fix, history, this.conversationId);
+                let resolved = false;
+                fixStream.on('data', (evt) => {
+                    if (this.handleFileEvents(blockUri, blockName, evt)) {
+                        return;
+                    }
+
+                    if (evt.type === 'CODE_FIX') {
+                        resolved = true;
+                        resolve(evt.payload);
+                    }
+                });
+                this.out.on('aborted', () => {
+                    fixStream.abort();
+                    reject(new Error('aborted'));
+                });
+                fixStream.on('error', (err) => {
+                    reject(err);
+                });
+                fixStream.on('end', () => {
+                    if (!resolved) {
+                        reject(new Error('Code fix never returned a valid event'));
+                    }
+                });
+            } catch (e) {
+                reject(e);
+            }
         });
     }
 
@@ -650,7 +710,13 @@ export class StormCodegen {
                     instanceId: StormEventParser.toInstanceIdFromRef(ref),
                 },
             };
-            return new SimulatedFileDelay(fileEvent, this.out).start();
+
+            if (ENABLE_SIMULATED_DELAY) {
+                // Simulate a delay when sending the file
+                return new SimulatedFileDelay(fileEvent, this.out).start();
+            } else {
+                this.out.emit('data', fileEvent);
+            }
         });
 
         return Promise.all(promises);

package/src/storm/event-parser.ts

@@ -431,9 +431,12 @@ export class StormEventParser {
                     }
 
                     const clientTypes = DSLDataTypeParser.parse(
-                        clientConsumerBlock.content.spec.entities.source!.value, {ignoreSemantics: true}
+                        clientConsumerBlock.content.spec.entities.source!.value,
+                        { ignoreSemantics: true }
                     );
-                    const apiTypes = DSLDataTypeParser.parse(apiProviderBlock.content.spec.entities?.source?.value, {ignoreSemantics: true});
+                    const apiTypes = DSLDataTypeParser.parse(apiProviderBlock.content.spec.entities?.source?.value, {
+                        ignoreSemantics: true,
+                    });
 
                     apiTypes.forEach((apiType) => {
                         if (clientTypes.some((clientType) => clientType.name === apiType.name)) {

package/src/storm/events.ts

@@ -139,9 +139,8 @@ export interface StormEventCodeFix {
     reason: string;
     created: number;
     payload: {
-        content: {
-            files: StormEventErrorDetailsFile[]
-        }
+        filename: string;
+        content: string;
     };
 }
 
@@ -162,7 +161,7 @@ export interface StormEventErrorDetails {
     reason: string;
     created: number;
     payload: {
-        files: string[]
+        files: string[];
     };
 }
 
@@ -252,6 +251,26 @@ export interface StormEventBlockReady {
     };
 }
 
+export enum StormEventBlockStatusType {
+    QA = 'QA',
+    FIXING = 'FIXING',
+    PLANNING_FIX = 'PLANNING_FIX',
+    FIX_DONE = 'FIX_DONE',
+    BUILDING = 'BUILDING',
+}
+
+export interface StormEventBlockStatus {
+    type: 'BLOCK_STATUS';
+    reason: string;
+    created: number;
+    payload: {
+        status: StormEventBlockStatusType;
+        blockName: string;
+        blockRef: string;
+        instanceId: string;
+    };
+}
+
 export interface StormEventDone {
     type: 'DONE';
     created: number;
@@ -268,6 +287,7 @@ export enum StormEventPhaseType {
     META = 'META',
     DEFINITIONS = 'DEFINITIONS',
     IMPLEMENTATION = 'IMPLEMENTATION',
+    QA = 'QA',
 }
 
 export interface StormEventPhases {
@@ -299,4 +319,5 @@ export type StormEvent =
     | StormEventCodeFix
     | StormEventErrorDetails
     | StormEventBlockReady
-    | StormEventPhases;
+    | StormEventPhases
+    | StormEventBlockStatus;

package/src/storm/routes.ts

@@ -215,6 +215,7 @@ function sendEvent(res: Response, evt: StormEvent) {
     if (res.closed) {
         return;
     }
+    console.log('Sending event', evt.type);
     res.write(JSON.stringify(evt) + '\n');
 }
 

package/src/storm/stormClient.ts

@@ -26,7 +26,11 @@ class StormClient {
         this._baseUrl = getRemoteUrl('ai-service', 'https://ai.kapeta.com');
     }
 
-    private async createOptions(path: string, method: string, body: StormContextRequest): Promise<RequestInit & { url: string }> {
+    private async createOptions(
+        path: string,
+        method: string,
+        body: StormContextRequest
+    ): Promise<RequestInit & { url: string }> {
         const url = `${this._baseUrl}${path}`;
         const headers: { [k: string]: string } = {
             'Content-Type': 'application/json',
@@ -92,7 +96,11 @@ class StormClient {
         });
 
         out.on('aborted', () => {
-            abort.abort();
+            try {
+                abort.abort();
+            } catch (e) {
+                console.warn('Error aborting stream', e);
+            }
         });
 
         return out;

package/src/storm/stream.ts

@@ -11,6 +11,7 @@ export class StormStream extends EventEmitter {
     private conversationId: string = '';
     private lines: string[] = [];
     private aborted: boolean = false;
+    private done: boolean = false;
 
     constructor(prompt: string = '', conversationId?: string | null) {
         super();
@@ -39,6 +40,7 @@ export class StormStream extends EventEmitter {
     }
 
     end() {
+        this.done = true;
         this.emit('end');
     }
 
@@ -59,6 +61,9 @@ export class StormStream extends EventEmitter {
     }
 
     waitForDone() {
+        if (this.done) {
+            return Promise.resolve();
+        }
         return new Promise<void>((resolve, reject) => {
             const errorHandler = (err: any) => {
                 this.removeListener('error', errorHandler);
@@ -70,8 +75,8 @@ export class StormStream extends EventEmitter {
                 this.removeListener('end', endHandler);
                 resolve();
             };
-            this.once('error', errorHandler);
-            this.once('end', endHandler);
+            this.on('error', errorHandler);
+            this.on('end', endHandler);
         });
     }
 
@@ -80,6 +85,7 @@ export class StormStream extends EventEmitter {
             return;
         }
         this.aborted = true;
+        this.done = true;
         this.emit('aborted');
     }
 }