@kapeta/local-cluster-service 0.48.4 → 0.49.0

package/.vscode/launch.json

@@ -6,7 +6,7 @@
             "request": "launch",
             "name": "Debug Node.js (npm run dev)",
             "runtimeExecutable": "npm",
-            "runtimeArgs": ["run-script", "dev"],
+            "runtimeArgs": ["run", "start:dev"],
             "restart": true,
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [0.49.0](https://github.com/kapetacom/local-cluster-service/compare/v0.48.5...v0.49.0) (2024-06-05)
+
+
+### Features
+
+* Implement ability to save ai generated plan ([#161](https://github.com/kapetacom/local-cluster-service/issues/161)) ([8223757](https://github.com/kapetacom/local-cluster-service/commit/8223757ff90ee6b5e1b7f171fc724d9877051df2))
+
+## [0.48.5](https://github.com/kapetacom/local-cluster-service/compare/v0.48.4...v0.48.5) (2024-06-04)
+
+
+### Bug Fixes
+
+* we should read the file previously written ([fe0b78e](https://github.com/kapetacom/local-cluster-service/commit/fe0b78efd5018be28308170a0c76b2619425a158))
+
 ## [0.48.4](https://github.com/kapetacom/local-cluster-service/compare/v0.48.3...v0.48.4) (2024-06-03)
 
 

package/dist/cjs/src/assetManager.d.ts

@@ -28,7 +28,7 @@ declare class AssetManager {
     getPlan(ref: string, noCache?: boolean): Promise<Plan>;
     getBlockInstance(systemId: string, instanceId: string): Promise<BlockInstance>;
     getAsset(ref: string, noCache?: boolean, autoFetch?: boolean): Promise<EnrichedAsset | undefined>;
-    createAsset(path: string, yaml: BlockDefinition, sourceOfChange?: SourceOfChange): Promise<EnrichedAsset[]>;
+    createAsset(path: string, yaml: BlockDefinition, sourceOfChange?: SourceOfChange, codegen?: boolean): Promise<EnrichedAsset[]>;
     updateAsset(ref: string, yaml: Definition, sourceOfChange?: SourceOfChange): Promise<void>;
     importFile(filePath: string): Promise<EnrichedAsset[]>;
     unregisterAsset(ref: string): Promise<void>;

package/dist/cjs/src/assetManager.js

@@ -142,7 +142,7 @@ class AssetManager {
         }
         return undefined;
     }
-    async createAsset(path, yaml, sourceOfChange = 'filesystem') {
+    async createAsset(path, yaml, sourceOfChange = 'filesystem', codegen = true) {
         if (await fs_extra_1.default.pathExists(path)) {
             throw new Error('File already exists: ' + path);
         }
@@ -159,9 +159,11 @@ class AssetManager {
             cacheManager_1.cacheManager.set(key, a, CACHE_TTL);
         });
         definitionsManager_1.definitionsManager.clearCache();
-        console.log(`Created asset at: ${path}`);
         const ref = `kapeta://${yaml.metadata.name}:local`;
-        await this.maybeGenerateCode(ref, path, yaml);
+        console.log(`Created asset ${ref} at: ${path}`);
+        if (codegen) {
+            await this.maybeGenerateCode(ref, path, yaml);
+        }
         return asset;
     }
     async updateAsset(ref, yaml, sourceOfChange = 'filesystem') {

package/dist/cjs/src/filesystem/routes.js

@@ -11,6 +11,7 @@ const express_promise_router_1 = __importDefault(require("express-promise-router
 const stringBody_1 = require("../middleware/stringBody");
 const filesystemManager_1 = require("../filesystemManager");
 const cors_1 = require("../middleware/cors");
+const fs_extra_1 = __importDefault(require("fs-extra"));
 let router = (0, express_promise_router_1.default)();
 router.use('/', cors_1.corsHandler);
 router.get('/root', (req, res) => {
@@ -81,6 +82,15 @@ router.get('/readfile', async (req, res) => {
         res.status(400).send({ error: '' + err });
     }
 });
+router.get('/exists', async (req, res) => {
+    let pathArg = req.query.path;
+    try {
+        res.send(await fs_extra_1.default.pathExists(pathArg));
+    }
+    catch (err) {
+        res.status(400).send({ error: '' + err });
+    }
+});
 router.put('/mkdir', async (req, res) => {
     let pathArg = req.query.path;
     try {

package/dist/cjs/src/storm/codegen.d.ts

@@ -11,7 +11,8 @@ export declare class StormCodegen {
     private readonly out;
     private readonly events;
     private readonly tmpDir;
-    constructor(userPrompt: string, blocks: BlockDefinitionInfo[], events: StormEvent[]);
+    private readonly conversationId;
+    constructor(conversationId: string, userPrompt: string, blocks: BlockDefinitionInfo[], events: StormEvent[]);
     process(): Promise<void>;
     getStream(): StormStream;
     private handleTemplateFileOutput;
@@ -22,6 +23,13 @@ export declare class StormCodegen {
      * Generates the code for a block and sends it to the AI
      */
     private processBlockCode;
+    private verifyAndFixCode;
+    removePrefix(prefix: string, str: string): string;
+    writeToFile(fileName: string, code: Promise<string>): Promise<void>;
+    /**
+     * Sends the code to the AI for a fix
+     */
+    private codeFix;
     /**
      * Emits the text-based files to the stream
      */

package/dist/cjs/src/storm/codegen.js

@@ -36,25 +36,31 @@ const codeGeneratorManager_1 = require("../codeGeneratorManager");
 const stormClient_1 = require("./stormClient");
 const event_parser_1 = require("./event-parser");
 const stream_1 = require("./stream");
+const nodejs_utils_1 = require("@kapeta/nodejs-utils");
 const promises_1 = require("fs/promises");
 const path_1 = __importStar(require("path"));
 const node_os_1 = __importDefault(require("node:os"));
+const fs_1 = require("fs");
+const path_2 = __importDefault(require("path"));
 class StormCodegen {
     userPrompt;
     blocks;
     out = new stream_1.StormStream();
     events;
     tmpDir;
-    constructor(userPrompt, blocks, events) {
+    conversationId;
+    constructor(conversationId, userPrompt, blocks, events) {
         this.userPrompt = userPrompt;
         this.blocks = blocks;
         this.events = events;
-        this.tmpDir = node_os_1.default.tmpdir();
+        this.tmpDir = path_2.default.join(node_os_1.default.tmpdir(), conversationId);
+        this.conversationId = conversationId;
     }
     async process() {
-        for (const block of this.blocks) {
-            await this.processBlockCode(block);
-        }
+        const promises = this.blocks.map((block) => {
+            return this.processBlockCode(block);
+        });
+        await Promise.all(promises);
         this.out.end();
     }
     getStream() {
@@ -118,7 +124,7 @@ class StormCodegen {
         }
         const allFiles = this.toStormFiles(generatedResult);
         // Send all the non-ai files to the stream
-        this.emitFiles(block.uri, block.aiName, allFiles);
+        this.emitFiles((0, nodejs_utils_1.parseKapetaUri)(block.uri), block.aiName, allFiles);
         const relevantFiles = allFiles.filter((file) => file.type !== codegen_1.AIFileTypes.IGNORE && file.type !== codegen_1.AIFileTypes.WEB_SCREEN);
         const uiTemplates = allFiles.filter((file) => file.type === codegen_1.AIFileTypes.WEB_SCREEN);
         if (uiTemplates.length > 0) {
@@ -130,7 +136,7 @@ class StormCodegen {
                 prompt: this.userPrompt,
             });
             uiStream.on('data', (evt) => {
-                this.handleUiOutput(block.uri, block.aiName, evt);
+                this.handleUiOutput((0, nodejs_utils_1.parseKapetaUri)(block.uri), block.aiName, evt);
             });
             await uiStream.waitForDone();
         }
@@ -139,7 +145,7 @@ class StormCodegen {
         // Send the service and UI templates to the AI. These will be send one-by-one in addition to the context files
         const serviceFiles = allFiles.filter((file) => file.type === codegen_1.AIFileTypes.SERVICE);
         if (serviceFiles.length > 0) {
-            await this.processTemplates(block.uri, block.aiName, stormClient_1.stormClient.createServiceImplementation.bind(stormClient_1.stormClient), serviceFiles, contextFiles);
+            await this.processTemplates((0, nodejs_utils_1.parseKapetaUri)(block.uri), block.aiName, stormClient_1.stormClient.createServiceImplementation.bind(stormClient_1.stormClient), serviceFiles, contextFiles);
         }
         const basePath = this.getBasePath(block.content.metadata.name);
         for (const serviceFile of serviceFiles) {
@@ -154,6 +160,103 @@ class StormCodegen {
             const filePath = (0, path_1.join)(basePath, uiFile.filename);
             await (0, promises_1.writeFile)(filePath, uiFile.content);
         }
+        const filesToBeFixed = serviceFiles.concat(contextFiles);
+        const codeGenerator = new codegen_1.BlockCodeGenerator(block.content);
+        await this.verifyAndFixCode(codeGenerator, basePath, filesToBeFixed, allFiles);
+        const blockRef = block.uri;
+        this.out.emit('data', {
+            type: 'BLOCK_READY',
+            reason: 'Block ready',
+            created: Date.now(),
+            payload: {
+                path: basePath,
+                blockName: block.aiName,
+                blockRef,
+                instanceId: event_parser_1.StormEventParser.toInstanceIdFromRef(blockRef),
+            },
+        });
+    }
+    async verifyAndFixCode(codeGenerator, basePath, filesToBeFixed, knownFiles) {
+        let attempts = 0;
+        let validCode = false;
+        for (let i = 0; i <= 3; i++) {
+            attempts++;
+            try {
+                console.log(`Validating the code in ${basePath} attempt #${attempts}`);
+                const result = await codeGenerator.validateForTarget(basePath);
+                if (result && result.valid) {
+                    validCode = true;
+                    break;
+                }
+                if (result && !result.valid) {
+                    console.debug('Validation error:', result);
+                    const errorStream = await stormClient_1.stormClient.createErrorClassification(result.error, []);
+                    const fixes = new Map();
+                    errorStream.on('data', (evt) => {
+                        if (evt.type === 'ERROR_CLASSIFIER') {
+                            // find the file that caused the error
+                            // strip base path from event file name, if it exists sometimes the AI sends the full path
+                            const eventFileName = this.removePrefix(basePath + '/', evt.payload.filename);
+                            const file = filesToBeFixed.find((f) => f.filename === eventFileName);
+                            if (!file) {
+                                console.log(`Could not find the file ${eventFileName} in the list of files to be fixed, Henrik might wanna create a new file for this fix`);
+                            }
+                            // read the content of the file
+                            const content = (0, fs_1.readFileSync)((0, path_1.join)(basePath, eventFileName), 'utf8');
+                            const fix = `${evt.payload.potentialFix}\n---\n${knownFiles
+                                .map((e) => e.filename)
+                                .join('\n')}\n---\n${content}`;
+                            console.log(`trying to fix the code in ${eventFileName}`);
+                            console.debug(`with the fix:\n${fix}`);
+                            const code = this.codeFix(fix);
+                            fixes.set((0, path_1.join)(basePath, eventFileName), code);
+                        }
+                    });
+                    await errorStream.waitForDone();
+                    for (const [filename, codePromise] of fixes) {
+                        const code = await codePromise;
+                        (0, fs_1.writeFileSync)(filename, code);
+                    }
+                }
+            }
+            catch (e) {
+                console.error('Error:', e);
+            }
+        }
+        if (validCode) {
+            console.log(`Validation successful after ${attempts} attempts`);
+        }
+        else {
+            console.error(`Validation failed for ${basePath} after ${attempts} attempts`);
+        }
+    }
+    removePrefix(prefix, str) {
+        if (str.startsWith(prefix)) {
+            return str.slice(prefix.length);
+        }
+        return str;
+    }
+    async writeToFile(fileName, code) {
+        console.log(`writing the fixed code to ${fileName}`);
+        const resolvedCode = await code;
+        (0, fs_1.writeFileSync)(fileName, resolvedCode);
+    }
+    /**
+     * Sends the code to the AI for a fix
+     */
+    async codeFix(fix) {
+        return new Promise(async (resolve, reject) => {
+            const fixStream = await stormClient_1.stormClient.createCodeFix(fix, []);
+            fixStream.on('data', (evt) => {
+                if (evt.type === 'CODE_FIX') {
+                    resolve(evt.payload.content);
+                }
+            });
+            fixStream.on('error', (err) => {
+                reject(err);
+            });
+            await fixStream.waitForDone();
+        });
     }
     /**
      * Emits the text-based files to the stream

package/dist/cjs/src/storm/event-parser.d.ts

@@ -6,7 +6,7 @@ import { StormEvent } from './events';
 import { BlockDefinition, Plan } from '@kapeta/schemas';
 import { KapetaURI } from '@kapeta/nodejs-utils';
 export interface BlockDefinitionInfo {
-    uri: KapetaURI;
+    uri: string;
     content: BlockDefinition;
     aiName: string;
 }

package/dist/cjs/src/storm/event-parser.js

@@ -315,6 +315,8 @@ class StormEventParser {
                 name: planRef.fullName,
                 title: this.planName,
                 description: this.planDescription,
+                structure: 'mono',
+                visibility: 'private',
             },
             spec: {
                 blocks,
@@ -331,7 +333,7 @@ class StormEventParser {
         Object.entries(this.blocks).forEach(([, blockInfo]) => {
             const blockRef = StormEventParser.toRef(handle, blockInfo.name);
             const blockDefinitionInfo = {
-                uri: blockRef,
+                uri: blockRef.toNormalizedString(),
                 aiName: blockInfo.name,
                 content: {
                     kind: this.toBlockKind(blockInfo.type),

package/dist/cjs/src/storm/events.d.ts

@@ -100,6 +100,26 @@ export interface StormEventError {
         error: string;
     };
 }
+export interface StormEventErrorClassifier {
+    type: 'ERROR_CLASSIFIER';
+    reason: string;
+    created: number;
+    payload: StormEventErrorClassifierInfo;
+}
+export interface StormEventCodeFix {
+    type: 'CODE_FIX';
+    reason: string;
+    created: number;
+    payload: {
+        filename: string;
+        content: string;
+    };
+}
+export interface StormEventErrorClassifierInfo {
+    error: string;
+    filename: string;
+    potentialFix: string;
+}
 export interface ScreenTemplate {
     name: string;
     template: string;
@@ -143,6 +163,17 @@ export interface StormEventFile {
         instanceId: string;
     };
 }
+export interface StormEventBlockReady {
+    type: 'BLOCK_READY';
+    reason: string;
+    created: number;
+    payload: {
+        path: string;
+        blockName: string;
+        blockRef: string;
+        instanceId: string;
+    };
+}
 export interface StormEventDone {
     type: 'DONE';
     created: number;
@@ -153,4 +184,4 @@ export interface StormEventDefinitionChange {
     created: number;
     payload: StormDefinitions;
 }
-export type StormEvent = StormEventCreateBlock | StormEventCreateConnection | StormEventCreatePlanProperties | StormEventInvalidResponse | StormEventPlanRetry | StormEventCreateDSL | StormEventCreateDSLResource | StormEventError | StormEventScreen | StormEventScreenCandidate | StormEventFile | StormEventDone | StormEventDefinitionChange;
+export type StormEvent = StormEventCreateBlock | StormEventCreateConnection | StormEventCreatePlanProperties | StormEventInvalidResponse | StormEventPlanRetry | StormEventCreateDSL | StormEventCreateDSLResource | StormEventError | StormEventScreen | StormEventScreenCandidate | StormEventFile | StormEventDone | StormEventDefinitionChange | StormEventErrorClassifier | StormEventCodeFix | StormEventBlockReady;

package/dist/cjs/src/storm/routes.js

@@ -8,11 +8,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const express_promise_router_1 = __importDefault(require("express-promise-router"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
 const cors_1 = require("../middleware/cors");
 const stringBody_1 = require("../middleware/stringBody");
 const stormClient_1 = require("./stormClient");
 const event_parser_1 = require("./event-parser");
 const codegen_1 = require("./codegen");
+const assetManager_1 = require("../assetManager");
+const path_1 = __importDefault(require("path"));
 const router = (0, express_promise_router_1.default)();
 router.use('/', cors_1.corsHandler);
 router.use('/', stringBody_1.stringBody);
@@ -47,7 +50,7 @@ router.post('/:handle/all', async (req, res) => {
         const result = eventParser.toResult(handle);
         sendDefinitions(res, result);
         if (!req.query.skipCodegen) {
-            const stormCodegen = new codegen_1.StormCodegen(aiRequest.prompt, result.blocks, eventParser.getEvents());
+            const stormCodegen = new codegen_1.StormCodegen(metaStream.getConversationId(), aiRequest.prompt, result.blocks, eventParser.getEvents());
             const codegenPromise = streamStormPartialResponse(stormCodegen.getStream(), res);
             await stormCodegen.process();
             await codegenPromise;
@@ -59,6 +62,28 @@ router.post('/:handle/all', async (req, res) => {
         res.end();
     }
 });
+router.post('/block/create', async (req, res) => {
+    const createRequest = JSON.parse(req.stringBody ?? '{}');
+    try {
+        const ymlPath = path_1.default.join(createRequest.newPath, 'kapeta.yml');
+        console.log('Creating block at', ymlPath);
+        const [asset] = await assetManager_1.assetManager.createAsset(ymlPath, createRequest.definition);
+        if (await fs_extra_1.default.pathExists(createRequest.tmpPath)) {
+            console.log('Moving block from', createRequest.tmpPath, 'to', createRequest.newPath);
+            await fs_extra_1.default.move(createRequest.tmpPath, createRequest.newPath, {
+                overwrite: true,
+            });
+            console.log('Updating asset', asset.ref);
+            res.send(await assetManager_1.assetManager.updateAsset(asset.ref, createRequest.definition));
+        }
+        else {
+            res.send(asset);
+        }
+    }
+    catch (err) {
+        res.status(500).send({ error: err.message });
+    }
+});
 function sendDefinitions(res, result) {
     sendEvent(res, {
         type: 'DEFINITION_CHANGE',

package/dist/cjs/src/storm/stormClient.d.ts

@@ -1,4 +1,4 @@
-import { StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt } from './stream';
+import { ConversationItem, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt } from './stream';
 export declare const STORM_ID = "storm";
 export declare const ConversationIdHeader = "Conversation-Id";
 declare class StormClient {
@@ -9,6 +9,8 @@ declare class StormClient {
     createMetadata(prompt: string, conversationId?: string): Promise<StormStream>;
     createUIImplementation(prompt: StormUIImplementationPrompt, conversationId?: string): Promise<StormStream>;
     createServiceImplementation(prompt: StormFileImplementationPrompt, conversationId?: string): Promise<StormStream>;
+    createErrorClassification(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
+    createCodeFix(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
 }
 export declare const stormClient: StormClient;
 export {};

package/dist/cjs/src/storm/stormClient.js

@@ -83,5 +83,17 @@ class StormClient {
             conversationId,
         });
     }
+    createErrorClassification(prompt, history, conversationId) {
+        return this.send('/v2/code/errorclassifier', {
+            conversationId: conversationId,
+            prompt,
+        });
+    }
+    createCodeFix(prompt, history, conversationId) {
+        return this.send('/v2/code/fix', {
+            conversationId: conversationId,
+            prompt,
+        });
+    }
 }
 exports.stormClient = new StormClient();

package/dist/cjs/src/storm/stream.d.ts

@@ -6,6 +6,7 @@
 import { EventEmitter } from 'node:events';
 import { StormEvent } from './events';
 import { AIFileTypes, GeneratedFile } from '@kapeta/codegen';
+import { BlockDefinition } from '@kapeta/schemas';
 export declare class StormStream extends EventEmitter {
     private conversationId;
     private lines;
@@ -29,6 +30,11 @@ export interface StormContextRequest<T = string> {
     conversationId?: string;
     prompt: T;
 }
+export interface StormCreateBlockRequest {
+    definition: BlockDefinition;
+    tmpPath: string;
+    newPath: string;
+}
 export interface StormFileInfo extends GeneratedFile {
     type: AIFileTypes;
 }

package/dist/esm/src/assetManager.d.ts

@@ -28,7 +28,7 @@ declare class AssetManager {
     getPlan(ref: string, noCache?: boolean): Promise<Plan>;
     getBlockInstance(systemId: string, instanceId: string): Promise<BlockInstance>;
     getAsset(ref: string, noCache?: boolean, autoFetch?: boolean): Promise<EnrichedAsset | undefined>;
-    createAsset(path: string, yaml: BlockDefinition, sourceOfChange?: SourceOfChange): Promise<EnrichedAsset[]>;
+    createAsset(path: string, yaml: BlockDefinition, sourceOfChange?: SourceOfChange, codegen?: boolean): Promise<EnrichedAsset[]>;
     updateAsset(ref: string, yaml: Definition, sourceOfChange?: SourceOfChange): Promise<void>;
     importFile(filePath: string): Promise<EnrichedAsset[]>;
     unregisterAsset(ref: string): Promise<void>;

package/dist/esm/src/assetManager.js

@@ -142,7 +142,7 @@ class AssetManager {
         }
         return undefined;
     }
-    async createAsset(path, yaml, sourceOfChange = 'filesystem') {
+    async createAsset(path, yaml, sourceOfChange = 'filesystem', codegen = true) {
         if (await fs_extra_1.default.pathExists(path)) {
             throw new Error('File already exists: ' + path);
         }
@@ -159,9 +159,11 @@ class AssetManager {
             cacheManager_1.cacheManager.set(key, a, CACHE_TTL);
         });
         definitionsManager_1.definitionsManager.clearCache();
-        console.log(`Created asset at: ${path}`);
         const ref = `kapeta://${yaml.metadata.name}:local`;
-        await this.maybeGenerateCode(ref, path, yaml);
+        console.log(`Created asset ${ref} at: ${path}`);
+        if (codegen) {
+            await this.maybeGenerateCode(ref, path, yaml);
+        }
         return asset;
     }
     async updateAsset(ref, yaml, sourceOfChange = 'filesystem') {

package/dist/esm/src/filesystem/routes.js

@@ -11,6 +11,7 @@ const express_promise_router_1 = __importDefault(require("express-promise-router
 const stringBody_1 = require("../middleware/stringBody");
 const filesystemManager_1 = require("../filesystemManager");
 const cors_1 = require("../middleware/cors");
+const fs_extra_1 = __importDefault(require("fs-extra"));
 let router = (0, express_promise_router_1.default)();
 router.use('/', cors_1.corsHandler);
 router.get('/root', (req, res) => {
@@ -81,6 +82,15 @@ router.get('/readfile', async (req, res) => {
         res.status(400).send({ error: '' + err });
     }
 });
+router.get('/exists', async (req, res) => {
+    let pathArg = req.query.path;
+    try {
+        res.send(await fs_extra_1.default.pathExists(pathArg));
+    }
+    catch (err) {
+        res.status(400).send({ error: '' + err });
+    }
+});
 router.put('/mkdir', async (req, res) => {
     let pathArg = req.query.path;
     try {

package/dist/esm/src/storm/codegen.d.ts

@@ -11,7 +11,8 @@ export declare class StormCodegen {
     private readonly out;
     private readonly events;
     private readonly tmpDir;
-    constructor(userPrompt: string, blocks: BlockDefinitionInfo[], events: StormEvent[]);
+    private readonly conversationId;
+    constructor(conversationId: string, userPrompt: string, blocks: BlockDefinitionInfo[], events: StormEvent[]);
     process(): Promise<void>;
     getStream(): StormStream;
     private handleTemplateFileOutput;
@@ -22,6 +23,13 @@ export declare class StormCodegen {
      * Generates the code for a block and sends it to the AI
      */
     private processBlockCode;
+    private verifyAndFixCode;
+    removePrefix(prefix: string, str: string): string;
+    writeToFile(fileName: string, code: Promise<string>): Promise<void>;
+    /**
+     * Sends the code to the AI for a fix
+     */
+    private codeFix;
     /**
      * Emits the text-based files to the stream
      */