@kapeta/local-cluster-service 0.52.0 → 0.52.2

package/src/storm/codegen.ts

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-import { Definition } from '@kapeta/local-cluster-config';
+import {Definition} from '@kapeta/local-cluster-config';
 import {
     AIFileTypes,
     BlockCodeGenerator,
@@ -13,22 +13,36 @@ import {
     GeneratedResult,
     MODE_CREATE_ONLY,
 } from '@kapeta/codegen';
-import { BlockDefinition } from '@kapeta/schemas';
-import { codeGeneratorManager } from '../codeGeneratorManager';
-import { STORM_ID, stormClient } from './stormClient';
-import { StormEvent, StormEventFileChunk, StormEventFileDone, StormEventFileLogical } from './events';
-import { BlockDefinitionInfo, StormEventParser } from './event-parser';
-import { StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
-import { KapetaURI, parseKapetaUri } from '@kapeta/nodejs-utils';
-import { writeFile } from 'fs/promises';
-import path, { join } from 'path';
+import {BlockDefinition} from '@kapeta/schemas';
+import {codeGeneratorManager} from '../codeGeneratorManager';
+import {STORM_ID, stormClient} from './stormClient';
+import {
+    StormEvent,
+    StormEventErrorDetailsFile,
+    StormEventFileChunk,
+    StormEventFileDone,
+    StormEventFileLogical
+} from './events';
+import {BlockDefinitionInfo, StormEventParser} from './event-parser';
+import {ConversationItem, StormFileImplementationPrompt, StormFileInfo, StormStream} from './stream';
+import {KapetaURI, parseKapetaUri} from '@kapeta/nodejs-utils';
+import {writeFile} from 'fs/promises';
+import path from 'path';
+import Path, {join} from 'path';
 import os from 'node:os';
-import { readFileSync, writeFileSync } from 'fs';
-import Path from 'path';
+import {readFileSync, writeFileSync} from 'fs';
 import YAML from "yaml";
+import * as fs from "node:fs";
+import {v4 as uuidv4} from 'uuid';
 
 type ImplementationGenerator = (prompt: StormFileImplementationPrompt, conversationId?: string) => Promise<StormStream>;
 
+interface ErrorClassification {
+    error: string;
+    lineNumber: number;
+    column: number;
+}
+
 const SIMULATED_DELAY = 1000;
 
 class SimulatedFileDelay {
@@ -321,7 +335,7 @@ export class StormCodegen {
             (file) => ![AIFileTypes.SERVICE, AIFileTypes.WEB_SCREEN].includes(file.type)
         );
 
-        // Send the service and UI templates to the AI. These will be send one-by-one in addition to the context files
+        // Send the service and UI templates to the AI. These will be sent one-by-one in addition to the context files
         const serviceFiles: StormFileInfo[] = allFiles.filter((file) => file.type === AIFileTypes.SERVICE);
         if (serviceFiles.length > 0) {
             await this.processTemplates(
@@ -366,8 +380,9 @@ export class StormCodegen {
                 type: AIFileTypes.WEB_SCREEN,
             };
         });
-        const filesToBeFixed = serviceFiles.concat(contextFiles).concat(screenFilesConverted);
         allFiles.push(...screenFilesConverted);
+
+        const filesToBeFixed = serviceFiles.concat(contextFiles).concat(screenFilesConverted);
         const codeGenerator = new BlockCodeGenerator(block.content as BlockDefinition);
         await this.verifyAndFixCode(codeGenerator, basePath, filesToBeFixed, allFiles);
 
@@ -389,7 +404,7 @@ export class StormCodegen {
         codeGenerator: CodeGenerator,
         basePath: string,
         filesToBeFixed: StormFileInfo[],
-        knownFiles: StormFileInfo[]
+        allFiles: StormFileInfo[]
     ) {
         let attempts = 0;
         let validCode = false;
@@ -405,46 +420,18 @@ export class StormCodegen {
 
                 if (result && !result.valid) {
                     console.debug('Validation error:', result);
-                    const errorStream = await stormClient.createErrorClassification(result.error, []);
-                    const fixes = new Map<string, Promise<string>>();
-
-                    this.out.on('aborted', () => {
-                        errorStream.abort();
-                    });
-
-                    errorStream.on('data', (evt) => {
-                        if (evt.type === 'ERROR_CLASSIFIER') {
-                            // find the file that caused the error
-                            // strip base path from event file name, if it exists sometimes the AI sends the full path
-                            const eventFileName = this.removePrefix(basePath + '/', evt.payload.filename);
-                            const file = filesToBeFixed.find((f) => f.filename === eventFileName);
-                            if (!file) {
-                                console.log(
-                                    `Could not find the file ${eventFileName} in the list of files to be fixed, Henrik might wanna create a new file for this fix`
-                                );
-                            }
-                            // read the content of the file
-                            const content = readFileSync(join(basePath, eventFileName), 'utf8');
-                            const fix = `${evt.payload.potentialFix}\n---\n${knownFiles
-                                .map((e) => e.filename)
-                                .join('\n')}\n---\n${content}`;
-                            //console.log(`trying to fix the code in ${eventFileName}`);
-                            //console.debug(`with the fix:\n${fix}`);
-                            const code = this.codeFix(fix);
-                            fixes.set(join(basePath, eventFileName), code);
-                        }
-                    });
-
-                    await errorStream.waitForDone();
-                    for (const [filename, codePromise] of fixes) {
-                        const code = await codePromise;
-                        writeFileSync(filename, code);
+
+                    const errors = await this.classifyErrors(result.error, basePath);
+                    for (const [filename, fileErrors] of errors.entries()) {
+                        // todo: only try to fix file if it is part of filesToBeFixed
+                        await this.tryToFixFile(basePath, filename, fileErrors, allFiles, codeGenerator);
                     }
                 }
             } catch (e) {
                 console.error('Error:', e);
             }
         }
+
         if (validCode) {
             console.log(`Validation successful after ${attempts} attempts`);
         } else {
@@ -452,6 +439,152 @@ export class StormCodegen {
         }
     }
 
+    private async tryToFixFile(basePath: string, filename: string, fileErrors: ErrorClassification[], allFiles: StormFileInfo[], codeGenerator: CodeGenerator) {
+        console.log(`Processing ${filename}`);
+        const language = await codeGenerator.language();
+        const relevantFiles = allFiles.filter(file => file.type != AIFileTypes.IGNORE);
+
+        for (let attempts = 1;  attempts <= 5; attempts++) {
+            if (fileErrors.length == 0) {
+                console.log(`No more errors for ${filename}`);
+                return;
+            }
+
+            console.log(`Errors in ${filename} - requesting error details`);
+            const filesForContext = await this.getErrorDetailsForFile(basePath, filename, fileErrors[0], relevantFiles, language);
+            console.log(`Get error details for ${filename} requesting code fixes`);
+
+            const fix = this.createFixRequestForFile(basePath, filename, fileErrors[0], filesForContext, relevantFiles, language);
+            const codeFixFiles = await this.codeFix(fix);
+            console.log(`Got fixed code for ${filename}`);
+            for (const codeFixFile of codeFixFiles) {
+                const filePath = codeFixFile.filename.indexOf(basePath) > -1 ? codeFixFile.filename : join(basePath, codeFixFile.filename);
+                const existing = readFileSync(filePath);
+                if (existing.toString().replace(/(\r\n|\r|\n)+$/, '') == codeFixFile.content.replace(/(\r\n|\r|\n)+$/, '')) {
+                    console.log(`${filename} not changed by gemini`);
+                    continue;
+                }
+                writeFileSync(filePath, codeFixFile.content);
+            }
+
+            const result = await codeGenerator.validateForTarget(basePath);
+            if (result && result.valid) {
+                return;
+            }
+
+            const errors = await this.classifyErrors(result.error, basePath);
+            fileErrors = errors.get(filename) ?? [];
+        }
+    }
+
+    private async classifyErrors(errors: string, basePath: string): Promise<Map<string, ErrorClassification[]>> {
+        const errorStream = await stormClient.createErrorClassification(errors, []);
+        const fixes = new Map<string, ErrorClassification[]>();
+
+        this.out.on('aborted', () => {
+            errorStream.abort();
+        });
+
+        errorStream.on('data', (evt) => {
+            if (evt.type === 'ERROR_CLASSIFIER') {
+                const eventFileName = this.removePrefix(basePath + '/', evt.payload.filename);
+                const fix = { error: evt.payload.error, lineNumber: evt.payload.lineNumber, column: evt.payload.column };
+
+                let existingFixes = fixes.get(eventFileName);
+                if (existingFixes) {
+                    existingFixes.push(fix);
+                } else {
+                    fixes.set(eventFileName, [fix]);
+                }
+            }
+        });
+
+        await errorStream.waitForDone();
+
+        return fixes;
+    }
+
+    private async getErrorDetailsForFile(basePath: string, filename: string, error: ErrorClassification, allFiles: StormFileInfo[], language: string): Promise<string[]> {
+        const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename); // to compensate when compiler returns absolute path
+        return new Promise<string[]>(async (resolve, reject) => {
+            const request = {
+                "language": language,
+                "sourceFile": {
+                    "filename": filename,
+                    "content": readFileSync(filePath, 'utf8')
+                },
+                "error": error,
+                "projectFiles": allFiles.map(f => f.filename)
+            };
+
+            const detailsStream = await stormClient.createErrorDetails(JSON.stringify(request), []);
+            detailsStream.on('data', (evt) => {
+                if (evt.type === 'ERROR_DETAILS') {
+                    resolve(evt.payload.files);
+                }
+            });
+            this.out.on('aborted', () => {
+                detailsStream.abort();
+            });
+            detailsStream.on('error', (err) => {
+                console.log("error", err);
+                reject(err);
+            });
+            await detailsStream.waitForDone();
+        });
+    }
+
+    private createFixRequestForFile(basePath: string, filename: string, error: ErrorClassification, filesForContext: string[], allFiles: StormFileInfo[], language: string): string {
+        const files = new Set(filesForContext);
+        files.add(filename);
+
+        const requestedFiles = Array.from(files).flatMap(file => {
+            if (fs.existsSync(file)) {
+                return file;
+            }
+
+            // file does not exist - look for similar
+            const candidateName = file.split('/').pop();
+            return allFiles
+                .filter(file => file.filename.split('/').pop() === candidateName)
+                .map(f => f.filename);
+        });
+
+        const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename);
+        const content = readFileSync(filePath, 'utf8');
+        const affectedLine = this.getErrorLine(error, content);
+
+        const fixRequest = {
+            "language": language,
+            "filename": filename,
+            "errors": error.error,
+            "affectedLine": affectedLine,
+            "projectFiles": requestedFiles.map(filename => {
+                const filePath = filename.indexOf(basePath) > -1 ? filename : join(basePath, filename);
+                const content = readFileSync(filePath, 'utf8');
+                return { filename: filename, content: content };
+            }),
+            "conversationId": this.conversationId,
+            "sessionId": uuidv4()
+        };
+
+        return JSON.stringify(fixRequest);
+    }
+
+    private getErrorLine(
+        errorDetails: ErrorClassification,
+        sourceCode: string
+    ): string {
+        const lines = sourceCode.split('\n');
+        const errorLine = lines[errorDetails.lineNumber - 1];
+
+        if (!errorLine) {
+            return "Error: Line number out of range.";
+        }
+
+        return errorLine;
+    }
+
     removePrefix(prefix: string, str: string): string {
         if (str.startsWith(prefix)) {
             return str.slice(prefix.length);
@@ -462,18 +595,19 @@ export class StormCodegen {
     /**
      * Sends the code to the AI for a fix
      */
-    private async codeFix(fix: string): Promise<string> {
-        return new Promise<string>(async (resolve, reject) => {
-            const fixStream = await stormClient.createCodeFix(fix, []);
+    private async codeFix(fix: string, history?: ConversationItem[]): Promise<StormEventErrorDetailsFile[]> {
+        return new Promise<StormEventErrorDetailsFile[]>(async (resolve, reject) => {
+            const fixStream = await stormClient.createCodeFix(fix, history);
             fixStream.on('data', (evt) => {
                 if (evt.type === 'CODE_FIX') {
-                    resolve(evt.payload.content);
+                    resolve(evt.payload.content.files);
                 }
             });
             this.out.on('aborted', () => {
                 fixStream.abort();
             });
             fixStream.on('error', (err) => {
+                console.log("error", err);
                 reject(err);
             });
             await fixStream.waitForDone();

package/src/storm/events.ts

@@ -139,14 +139,31 @@ export interface StormEventCodeFix {
     reason: string;
     created: number;
     payload: {
-        filename: string;
-        content: string;
+        content: {
+            files: StormEventErrorDetailsFile[]
+        }
     };
 }
+
 export interface StormEventErrorClassifierInfo {
     error: string;
     filename: string;
-    potentialFix: string;
+    lineNumber: number;
+    column: number;
+}
+
+export interface StormEventErrorDetailsFile {
+    filename: string;
+    content: string;
+}
+
+export interface StormEventErrorDetails {
+    type: 'ERROR_DETAILS';
+    reason: string;
+    created: number;
+    payload: {
+        files: string[]
+    };
 }
 
 export interface ScreenTemplate {
@@ -280,5 +297,6 @@ export type StormEvent =
     | StormEventDefinitionChange
     | StormEventErrorClassifier
     | StormEventCodeFix
+    | StormEventErrorDetails
     | StormEventBlockReady
     | StormEventPhases;

package/src/storm/routes.ts

@@ -156,7 +156,6 @@ router.post('/block/create', async (req: KapetaBodyRequest, res: Response) => {
         const [asset] = await assetManager.createAsset(ymlPath, createRequest.definition);
         res.send(asset);
     } catch (err: any) {
-        console.error('err');
         res.status(500).send({ error: err.message });
     }
 });
@@ -199,30 +198,16 @@ function sendError(err: Error, res: Response) {
     }
 }
 function waitForStormStream(result: StormStream) {
-    return new Promise<void>((resolve, reject) => {
-        result.on('error', (err) => {
-            reject(err);
-        });
-
-        result.on('end', () => {
-            resolve();
-        });
-    });
+    return result.waitForDone();
 }
 
 function streamStormPartialResponse(result: StormStream, res: Response) {
-    return new Promise<void>((resolve, reject) => {
+    return new Promise<void>((resolve) => {
         result.on('data', (data) => {
             sendEvent(res, data);
         });
 
-        result.on('error', (err) => {
-            reject(err);
-        });
-
-        result.on('end', () => {
-            resolve();
-        });
+        resolve(result.waitForDone());
     });
 }
 

package/src/storm/stormClient.ts

@@ -125,12 +125,20 @@ class StormClient {
             prompt,
         });
     }
+
     public createCodeFix(prompt: string, history?: ConversationItem[], conversationId?: string) {
         return this.send('/v2/code/fix', {
             conversationId: conversationId,
             prompt,
         });
     }
+
+    public createErrorDetails(prompt: string, history?: ConversationItem[], conversationId?: string) {
+        return this.send('/v2/code/errordetails', {
+            conversationId: conversationId,
+            prompt,
+        });
+    }
 }
 
 export const stormClient = new StormClient();

package/src/storm/stream.ts

@@ -60,13 +60,18 @@ export class StormStream extends EventEmitter {
 
     waitForDone() {
         return new Promise<void>((resolve, reject) => {
-            this.on('error', (err) => {
+            const errorHandler = (err: any) => {
+                this.removeListener('error', errorHandler);
+                this.removeListener('end', endHandler);
                 reject(err);
-            });
-
-            this.on('end', () => {
+            };
+            const endHandler = () => {
+                this.removeListener('error', errorHandler);
+                this.removeListener('end', endHandler);
                 resolve();
-            });
+            };
+            this.once('error', errorHandler);
+            this.once('end', endHandler);
         });
     }