@kapeta/local-cluster-service 0.54.2 → 0.54.4

package/CHANGELOG.md

@@ -1,3 +1,19 @@
+## [0.54.4](https://github.com/kapetacom/local-cluster-service/compare/v0.54.3...v0.54.4) (2024-06-20)
+
+
+### Bug Fixes
+
+* merge error ([4c29cab](https://github.com/kapetacom/local-cluster-service/commit/4c29cab33c311a3ad21913b974d92207968a15db))
+* minors ([c36e825](https://github.com/kapetacom/local-cluster-service/commit/c36e8258cf0d7a3fb0915080f64d524ffd925929))
+* processTemplate will possibly return a list of files for each file when enhancing services ([3f186a3](https://github.com/kapetacom/local-cluster-service/commit/3f186a3afbb6b527c933ff051d2659e07ab39bbd))
+
+## [0.54.3](https://github.com/kapetacom/local-cluster-service/compare/v0.54.2...v0.54.3) (2024-06-19)
+
+
+### Bug Fixes
+
+* emit file done events from codefixer ([#182](https://github.com/kapetacom/local-cluster-service/issues/182)) ([77649dc](https://github.com/kapetacom/local-cluster-service/commit/77649dc1d53a5053f1ff7ed54b8106a3bd410434))
+
 ## [0.54.2](https://github.com/kapetacom/local-cluster-service/compare/v0.54.1...v0.54.2) (2024-06-19)
 
 

package/dist/cjs/src/storm/codegen.js

@@ -117,16 +117,13 @@ class StormCodegen {
     getStream() {
         return this.out;
     }
-    handleTemplateFileOutput(blockUri, aiName, template, data) {
+    handleTemplateFileOutput(blockUri, aiName, data) {
         if (this.handleFileEvents(blockUri, aiName, data)) {
             return;
         }
         switch (data.type) {
             case 'FILE_DONE':
-                template.filename = data.payload.filename;
-                template.content = data.payload.content;
-                this.handleFileDoneOutput(blockUri, aiName, data);
-                break;
+                return this.handleFileDoneOutput(blockUri, aiName, data);
         }
     }
     handleUiOutput(blockUri, blockName, data) {
@@ -229,8 +226,10 @@ class StormCodegen {
         if (this.isAborted()) {
             return;
         }
+        const kapetaYaml = yaml_1.default.stringify(block.content);
+        const blockDefinition = block.content;
         // Generate the code for the block using the standard codegen templates
-        const generatedResult = await this.generateBlock(block.content);
+        const generatedResult = await this.generateBlock(blockDefinition);
         if (!generatedResult) {
             return;
         }
@@ -257,8 +256,8 @@ class StormCodegen {
         if (failedEvents.length > 0) {
             console.warn('Codegen encountered failed plan events. Plan might be invalid', failedEvents);
         }
+        // TODO: remove this when we have a better way to handle failed events
         // Skip failed events - they are not relevant to the materializer
-        // TODO: should the materializer be able to handle failed events?
         filteredEvents = filteredEvents.filter((event) => !event.type.endsWith('_FAILED') && !event.type.endsWith('ERROR_INTERNAL'));
         const screenEvents = [];
         const getScreenEventsFile = () => ({
@@ -321,7 +320,7 @@ class StormCodegen {
         if (this.isAborted()) {
             return;
         }
-        const basePath = this.getBasePath(block.content.metadata.name);
+        const basePath = this.getBasePath(blockDefinition.metadata.name);
         const screenFilesConverted = screenFiles.map((screenFile) => {
             return {
                 filename: screenFile.payload.filename,
@@ -342,7 +341,7 @@ class StormCodegen {
             };
             const stream = await stormClient_1.stormClient.generateCode(payload);
             stream.on('data', (evt) => {
-                this.handleTemplateFileOutput(blockUri, block.aiName, webRouter, evt);
+                this.handleTemplateFileOutput(blockUri, block.aiName, evt);
             });
             this.out.on('aborted', () => {
                 stream.abort();
@@ -352,36 +351,37 @@ class StormCodegen {
         // Gather the context files for implementation. These will be all be passed to the AI
         const contextFiles = relevantFiles.filter((file) => ![codegen_1.AIFileTypes.SERVICE, codegen_1.AIFileTypes.WEB_SCREEN, codegen_1.AIFileTypes.WEB_ROUTER].includes(file.type));
         // Send the service and UI templates to the AI. These will be sent one-by-one in addition to the context files
-        const serviceFiles = allFiles.filter((file) => file.type === codegen_1.AIFileTypes.SERVICE);
+        let serviceFiles = allFiles.filter((file) => file.type === codegen_1.AIFileTypes.SERVICE);
         if (serviceFiles.length > 0) {
-            await this.processTemplates(blockUri, block.aiName, stormClient_1.stormClient.createServiceImplementation.bind(stormClient_1.stormClient), serviceFiles, contextFiles);
+            serviceFiles = await this.processTemplates(blockUri, block.aiName, stormClient_1.stormClient.createServiceImplementation.bind(stormClient_1.stormClient), serviceFiles, contextFiles);
         }
         if (this.isAborted()) {
             return;
         }
+        for (const contextFile of contextFiles) {
+            const filePath = (0, path_2.join)(basePath, contextFile.filename);
+            await (0, promises_1.writeFile)(filePath, contextFile.content);
+        }
         for (const screenFile of screenFilesConverted) {
             const filePath = (0, path_2.join)(basePath, screenFile.filename);
             await (0, promises_1.writeFile)(filePath, screenFile.content);
         }
+        // this might decide to overwrite a file that is also a context file
         for (const serviceFile of serviceFiles) {
             const filePath = (0, path_2.join)(basePath, serviceFile.filename);
             await (0, promises_1.writeFile)(filePath, serviceFile.content);
         }
-        for (const serviceFile of contextFiles) {
-            const filePath = (0, path_2.join)(basePath, serviceFile.filename);
-            await (0, promises_1.writeFile)(filePath, serviceFile.content);
-        }
         // Write again after modifications
         for (const webRouterFile of webRouters) {
             const filePath = (0, path_2.join)(basePath, webRouterFile.filename);
             await (0, promises_1.writeFile)(filePath, webRouterFile.content);
         }
         const kapetaYmlPath = (0, path_2.join)(basePath, 'kapeta.yml');
-        await (0, promises_1.writeFile)(kapetaYmlPath, yaml_1.default.stringify(block.content));
+        await (0, promises_1.writeFile)(kapetaYmlPath, kapetaYaml);
         const blockRef = block.uri;
         this.emitBlockStatus(blockUri, block.aiName, events_1.StormEventBlockStatusType.QA);
         const filesToBeFixed = serviceFiles.concat(contextFiles).concat(screenFilesConverted);
-        const codeGenerator = new codegen_1.BlockCodeGenerator(block.content);
+        const codeGenerator = new codegen_1.BlockCodeGenerator(blockDefinition);
         this.emitBlockStatus(blockUri, block.aiName, events_1.StormEventBlockStatusType.BUILDING);
         await this.verifyAndFixCode(blockUri, block.aiName, codeGenerator, basePath, filesToBeFixed, allFiles);
         this.out.emit('data', {
@@ -587,6 +587,7 @@ class StormCodegen {
                     if (this.handleFileEvents(blockUri, blockName, evt)) {
                         return;
                     }
+                    this.handleFileDoneOutput(blockUri, blockName, evt);
                     if (evt.type === 'CODE_FIX') {
                         resolved = true;
                         resolve(evt.payload);
@@ -675,6 +676,7 @@ class StormCodegen {
      */
     async processTemplates(blockUri, aiName, generator, templates, contextFiles) {
         const promises = templates.map(async (templateFile) => {
+            let changedFiles = [];
             const stream = await generator({
                 context: contextFiles,
                 template: templateFile,
@@ -684,11 +686,28 @@ class StormCodegen {
                 stream.abort();
             });
             stream.on('data', (evt) => {
-                this.handleTemplateFileOutput(blockUri, aiName, templateFile, evt);
+                let changedFile = this.handleTemplateFileOutput(blockUri, aiName, evt);
+                if (changedFile) {
+                    changedFiles.push(...changedFiles, changedFile);
+                }
             });
             await stream.waitForDone();
+            return changedFiles;
         });
-        await Promise.all(promises);
+        const changedFiles = await Promise.all(promises);
+        let allFiles = templates.concat(contextFiles);
+        let result = [];
+        for (const changedFile of changedFiles.flat()) {
+            const find = allFiles.find((file) => file.filename === changedFile.payload.filename);
+            if (find) {
+                result.push({ type: find.type, filename: find.filename, mode: find.mode, permissions: find.permissions,
+                    content: changedFile.payload.content });
+            }
+            else {
+                console.warn("processTemplates: AI changed a file that wasn't in the input [" + changedFile.payload.filename + "]");
+            }
+        }
+        return result;
     }
     /**
      * Converts the generated files to a format that can be sent to the AI

package/dist/esm/src/storm/codegen.js

@@ -117,16 +117,13 @@ class StormCodegen {
     getStream() {
         return this.out;
     }
-    handleTemplateFileOutput(blockUri, aiName, template, data) {
+    handleTemplateFileOutput(blockUri, aiName, data) {
         if (this.handleFileEvents(blockUri, aiName, data)) {
             return;
         }
         switch (data.type) {
             case 'FILE_DONE':
-                template.filename = data.payload.filename;
-                template.content = data.payload.content;
-                this.handleFileDoneOutput(blockUri, aiName, data);
-                break;
+                return this.handleFileDoneOutput(blockUri, aiName, data);
         }
     }
     handleUiOutput(blockUri, blockName, data) {
@@ -229,8 +226,10 @@ class StormCodegen {
         if (this.isAborted()) {
             return;
         }
+        const kapetaYaml = yaml_1.default.stringify(block.content);
+        const blockDefinition = block.content;
         // Generate the code for the block using the standard codegen templates
-        const generatedResult = await this.generateBlock(block.content);
+        const generatedResult = await this.generateBlock(blockDefinition);
         if (!generatedResult) {
             return;
         }
@@ -257,8 +256,8 @@ class StormCodegen {
         if (failedEvents.length > 0) {
             console.warn('Codegen encountered failed plan events. Plan might be invalid', failedEvents);
         }
+        // TODO: remove this when we have a better way to handle failed events
         // Skip failed events - they are not relevant to the materializer
-        // TODO: should the materializer be able to handle failed events?
         filteredEvents = filteredEvents.filter((event) => !event.type.endsWith('_FAILED') && !event.type.endsWith('ERROR_INTERNAL'));
         const screenEvents = [];
         const getScreenEventsFile = () => ({
@@ -321,7 +320,7 @@ class StormCodegen {
         if (this.isAborted()) {
             return;
         }
-        const basePath = this.getBasePath(block.content.metadata.name);
+        const basePath = this.getBasePath(blockDefinition.metadata.name);
         const screenFilesConverted = screenFiles.map((screenFile) => {
             return {
                 filename: screenFile.payload.filename,
@@ -342,7 +341,7 @@ class StormCodegen {
             };
             const stream = await stormClient_1.stormClient.generateCode(payload);
             stream.on('data', (evt) => {
-                this.handleTemplateFileOutput(blockUri, block.aiName, webRouter, evt);
+                this.handleTemplateFileOutput(blockUri, block.aiName, evt);
             });
             this.out.on('aborted', () => {
                 stream.abort();
@@ -352,36 +351,37 @@ class StormCodegen {
         // Gather the context files for implementation. These will be all be passed to the AI
         const contextFiles = relevantFiles.filter((file) => ![codegen_1.AIFileTypes.SERVICE, codegen_1.AIFileTypes.WEB_SCREEN, codegen_1.AIFileTypes.WEB_ROUTER].includes(file.type));
         // Send the service and UI templates to the AI. These will be sent one-by-one in addition to the context files
-        const serviceFiles = allFiles.filter((file) => file.type === codegen_1.AIFileTypes.SERVICE);
+        let serviceFiles = allFiles.filter((file) => file.type === codegen_1.AIFileTypes.SERVICE);
         if (serviceFiles.length > 0) {
-            await this.processTemplates(blockUri, block.aiName, stormClient_1.stormClient.createServiceImplementation.bind(stormClient_1.stormClient), serviceFiles, contextFiles);
+            serviceFiles = await this.processTemplates(blockUri, block.aiName, stormClient_1.stormClient.createServiceImplementation.bind(stormClient_1.stormClient), serviceFiles, contextFiles);
         }
         if (this.isAborted()) {
             return;
         }
+        for (const contextFile of contextFiles) {
+            const filePath = (0, path_2.join)(basePath, contextFile.filename);
+            await (0, promises_1.writeFile)(filePath, contextFile.content);
+        }
         for (const screenFile of screenFilesConverted) {
             const filePath = (0, path_2.join)(basePath, screenFile.filename);
             await (0, promises_1.writeFile)(filePath, screenFile.content);
         }
+        // this might decide to overwrite a file that is also a context file
         for (const serviceFile of serviceFiles) {
             const filePath = (0, path_2.join)(basePath, serviceFile.filename);
             await (0, promises_1.writeFile)(filePath, serviceFile.content);
         }
-        for (const serviceFile of contextFiles) {
-            const filePath = (0, path_2.join)(basePath, serviceFile.filename);
-            await (0, promises_1.writeFile)(filePath, serviceFile.content);
-        }
         // Write again after modifications
         for (const webRouterFile of webRouters) {
             const filePath = (0, path_2.join)(basePath, webRouterFile.filename);
             await (0, promises_1.writeFile)(filePath, webRouterFile.content);
         }
         const kapetaYmlPath = (0, path_2.join)(basePath, 'kapeta.yml');
-        await (0, promises_1.writeFile)(kapetaYmlPath, yaml_1.default.stringify(block.content));
+        await (0, promises_1.writeFile)(kapetaYmlPath, kapetaYaml);
         const blockRef = block.uri;
         this.emitBlockStatus(blockUri, block.aiName, events_1.StormEventBlockStatusType.QA);
         const filesToBeFixed = serviceFiles.concat(contextFiles).concat(screenFilesConverted);
-        const codeGenerator = new codegen_1.BlockCodeGenerator(block.content);
+        const codeGenerator = new codegen_1.BlockCodeGenerator(blockDefinition);
         this.emitBlockStatus(blockUri, block.aiName, events_1.StormEventBlockStatusType.BUILDING);
         await this.verifyAndFixCode(blockUri, block.aiName, codeGenerator, basePath, filesToBeFixed, allFiles);
         this.out.emit('data', {
@@ -587,6 +587,7 @@ class StormCodegen {
                     if (this.handleFileEvents(blockUri, blockName, evt)) {
                         return;
                     }
+                    this.handleFileDoneOutput(blockUri, blockName, evt);
                     if (evt.type === 'CODE_FIX') {
                         resolved = true;
                         resolve(evt.payload);
@@ -675,6 +676,7 @@ class StormCodegen {
      */
     async processTemplates(blockUri, aiName, generator, templates, contextFiles) {
         const promises = templates.map(async (templateFile) => {
+            let changedFiles = [];
             const stream = await generator({
                 context: contextFiles,
                 template: templateFile,
@@ -684,11 +686,28 @@ class StormCodegen {
                 stream.abort();
             });
             stream.on('data', (evt) => {
-                this.handleTemplateFileOutput(blockUri, aiName, templateFile, evt);
+                let changedFile = this.handleTemplateFileOutput(blockUri, aiName, evt);
+                if (changedFile) {
+                    changedFiles.push(...changedFiles, changedFile);
+                }
             });
             await stream.waitForDone();
+            return changedFiles;
         });
-        await Promise.all(promises);
+        const changedFiles = await Promise.all(promises);
+        let allFiles = templates.concat(contextFiles);
+        let result = [];
+        for (const changedFile of changedFiles.flat()) {
+            const find = allFiles.find((file) => file.filename === changedFile.payload.filename);
+            if (find) {
+                result.push({ type: find.type, filename: find.filename, mode: find.mode, permissions: find.permissions,
+                    content: changedFile.payload.content });
+            }
+            else {
+                console.warn("processTemplates: AI changed a file that wasn't in the input [" + changedFile.payload.filename + "]");
+            }
+        }
+        return result;
     }
     /**
      * Converts the generated files to a format that can be sent to the AI

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.54.2",
+    "version": "0.54.4",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {

package/src/storm/codegen.ts

@@ -36,7 +36,6 @@ import Path, { join } from 'path';
 import os from 'node:os';
 import { readFileSync, writeFileSync, existsSync } from 'fs';
 import YAML from 'yaml';
-import assert from 'assert';
 
 type ImplementationGenerator<T = StormFileImplementationPrompt> = (
     prompt: T,
@@ -134,19 +133,15 @@ export class StormCodegen {
     private handleTemplateFileOutput(
         blockUri: KapetaURI,
         aiName: string,
-        template: StormFileInfo,
         data: StormEvent
-    ): void {
+    ): StormEventFileDone | undefined {
         if (this.handleFileEvents(blockUri, aiName, data)) {
             return;
         }
 
         switch (data.type) {
             case 'FILE_DONE':
-                template.filename = data.payload.filename;
-                template.content = data.payload.content;
-                this.handleFileDoneOutput(blockUri, aiName, data);
-                break;
+                return this.handleFileDoneOutput(blockUri, aiName, data);
         }
     }
 
@@ -227,7 +222,7 @@ export class StormCodegen {
         return false;
     }
 
-    private handleFileDoneOutput(blockUri: KapetaURI, aiName: string, data: StormEvent) {
+    private handleFileDoneOutput(blockUri: KapetaURI, aiName: string, data: StormEvent): StormEventFileDone | undefined {
         switch (data.type) {
             case 'FILE_DONE':
                 const ref = blockUri.toNormalizedString();
@@ -257,8 +252,12 @@ export class StormCodegen {
         if (this.isAborted()) {
             return;
         }
+
+        const kapetaYaml = YAML.stringify(block.content);
+        const blockDefinition = block.content;
+
         // Generate the code for the block using the standard codegen templates
-        const generatedResult = await this.generateBlock(block.content);
+        const generatedResult = await this.generateBlock(blockDefinition);
         if (!generatedResult) {
             return;
         }
@@ -295,8 +294,8 @@ export class StormCodegen {
         if (failedEvents.length > 0) {
             console.warn('Codegen encountered failed plan events. Plan might be invalid', failedEvents);
         }
+        // TODO: remove this when we have a better way to handle failed events
         // Skip failed events - they are not relevant to the materializer
-        // TODO: should the materializer be able to handle failed events?
         filteredEvents = filteredEvents.filter(
             (event) => !event.type.endsWith('_FAILED') && !event.type.endsWith('ERROR_INTERNAL')
         );
@@ -374,7 +373,7 @@ export class StormCodegen {
         if (this.isAborted()) {
             return;
         }
-        const basePath = this.getBasePath(block.content.metadata.name);
+        const basePath = this.getBasePath(blockDefinition.metadata.name);
 
         const screenFilesConverted = screenFiles.map((screenFile) => {
             return {
@@ -400,7 +399,7 @@ export class StormCodegen {
                 const stream = await stormClient.generateCode(payload);
 
                 stream.on('data', (evt) => {
-                    this.handleTemplateFileOutput(blockUri, block.aiName, webRouter, evt);
+                    this.handleTemplateFileOutput(blockUri, block.aiName, evt);
                 });
 
                 this.out.on('aborted', () => {
@@ -417,9 +416,9 @@ export class StormCodegen {
         );
 
         // Send the service and UI templates to the AI. These will be sent one-by-one in addition to the context files
-        const serviceFiles: StormFileInfo[] = allFiles.filter((file) => file.type === AIFileTypes.SERVICE);
+        let serviceFiles: StormFileInfo[] = allFiles.filter((file) => file.type === AIFileTypes.SERVICE);
         if (serviceFiles.length > 0) {
-            await this.processTemplates(
+            serviceFiles = await this.processTemplates(
                 blockUri,
                 block.aiName,
                 stormClient.createServiceImplementation.bind(stormClient),
@@ -432,21 +431,22 @@ export class StormCodegen {
             return;
         }
 
+        for (const contextFile of contextFiles) {
+            const filePath = join(basePath, contextFile.filename);
+            await writeFile(filePath, contextFile.content);
+        }
+
         for (const screenFile of screenFilesConverted) {
             const filePath = join(basePath, screenFile.filename);
             await writeFile(filePath, screenFile.content);
         }
 
+        // this might decide to overwrite a file that is also a context file
         for (const serviceFile of serviceFiles) {
             const filePath = join(basePath, serviceFile.filename);
             await writeFile(filePath, serviceFile.content);
         }
 
-        for (const serviceFile of contextFiles) {
-            const filePath = join(basePath, serviceFile.filename);
-            await writeFile(filePath, serviceFile.content);
-        }
-
         // Write again after modifications
         for (const webRouterFile of webRouters) {
             const filePath = join(basePath, webRouterFile.filename);
@@ -454,14 +454,14 @@ export class StormCodegen {
         }
 
         const kapetaYmlPath = join(basePath, 'kapeta.yml');
-        await writeFile(kapetaYmlPath, YAML.stringify(block.content));
+        await writeFile(kapetaYmlPath, kapetaYaml);
 
         const blockRef = block.uri;
 
         this.emitBlockStatus(blockUri, block.aiName, StormEventBlockStatusType.QA);
 
         const filesToBeFixed = serviceFiles.concat(contextFiles).concat(screenFilesConverted);
-        const codeGenerator = new BlockCodeGenerator(block.content);
+        const codeGenerator = new BlockCodeGenerator(blockDefinition);
 
         this.emitBlockStatus(blockUri, block.aiName, StormEventBlockStatusType.BUILDING);
         await this.verifyAndFixCode(blockUri, block.aiName, codeGenerator, basePath, filesToBeFixed, allFiles);
@@ -758,6 +758,7 @@ export class StormCodegen {
                     if (this.handleFileEvents(blockUri, blockName, evt)) {
                         return;
                     }
+                    this.handleFileDoneOutput(blockUri, blockName, evt);
 
                     if (evt.type === 'CODE_FIX') {
                         resolved = true;
@@ -863,8 +864,9 @@ export class StormCodegen {
         generator: ImplementationGenerator,
         templates: StormFileInfo[],
         contextFiles: StormFileInfo[]
-    ): Promise<void> {
+    ): Promise<StormFileInfo[]> {
         const promises = templates.map(async (templateFile) => {
+            let changedFiles: StormEventFileDone[] = [];
             const stream = await generator({
                 context: contextFiles,
                 template: templateFile,
@@ -876,13 +878,29 @@ export class StormCodegen {
             });
 
             stream.on('data', (evt) => {
-                this.handleTemplateFileOutput(blockUri, aiName, templateFile, evt);
+                let changedFile = this.handleTemplateFileOutput(blockUri, aiName, evt);
+                if (changedFile) {
+                    changedFiles.push(...changedFiles, changedFile);
+                }
             });
 
             await stream.waitForDone();
+            return changedFiles;
         });
 
-        await Promise.all(promises);
+        const changedFiles: StormEventFileDone[][] = await Promise.all(promises);
+        let allFiles = templates.concat(contextFiles);
+        let result: StormFileInfo[] = [];
+        for (const changedFile of changedFiles.flat()) {
+            const find = allFiles.find((file) => file.filename === changedFile.payload.filename);
+            if (find) {
+                result.push({ type: find.type, filename: find.filename, mode: find.mode, permissions: find.permissions,
+                    content: changedFile.payload.content });
+            } else {
+                console.warn("processTemplates: AI changed a file that wasn't in the input [" + changedFile.payload.filename + "]");
+            }
+        }
+        return result;
     }
 
     /**