@kapeta/local-cluster-service 0.48.5 → 0.50.0

package/src/filesystem/routes.ts

@@ -8,6 +8,7 @@ import { stringBody, StringBodyRequest } from '../middleware/stringBody';
 import { filesystemManager } from '../filesystemManager';
 import { corsHandler } from '../middleware/cors';
 import { NextFunction, Request, Response } from 'express';
+import FS from 'fs-extra';
 
 let router = Router();
 
@@ -99,6 +100,15 @@ router.get('/readfile', async (req: Request, res: Response) => {
     }
 });
 
+router.get('/exists', async (req: Request, res: Response) => {
+    let pathArg = req.query.path as string;
+    try {
+        res.send(await FS.pathExists(pathArg));
+    } catch (err) {
+        res.status(400).send({ error: '' + err });
+    }
+});
+
 router.put('/mkdir', async (req: Request, res: Response) => {
     let pathArg = req.query.path as string;
     try {

package/src/storm/codegen.ts

@@ -4,18 +4,26 @@
  */
 
 import { Definition } from '@kapeta/local-cluster-config';
-import { AIFileTypes, BlockCodeGenerator, CodeGenerator, CodeWriter, GeneratedFile, GeneratedResult } from '@kapeta/codegen';
+import {
+    AIFileTypes,
+    BlockCodeGenerator,
+    CodeGenerator,
+    CodeWriter,
+    GeneratedFile,
+    GeneratedResult,
+} from '@kapeta/codegen';
 import { BlockDefinition } from '@kapeta/schemas';
 import { codeGeneratorManager } from '../codeGeneratorManager';
 import { STORM_ID, stormClient } from './stormClient';
 import { StormEvent, StormEventFile } from './events';
 import { BlockDefinitionInfo, StormEventParser } from './event-parser';
-import { ConversationItem, StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
-import { KapetaURI } from '@kapeta/nodejs-utils';
+import { StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
+import { KapetaURI, parseKapetaUri } from '@kapeta/nodejs-utils';
 import { writeFile } from 'fs/promises';
 import path, { join } from 'path';
 import os from 'node:os';
 import { readFile, readFileSync, writeFileSync } from 'fs';
+import Path from 'path';
 
 type ImplementationGenerator = (prompt: StormFileImplementationPrompt, conversationId?: string) => Promise<StormStream>;
 
@@ -25,22 +33,28 @@ export class StormCodegen {
     private readonly out = new StormStream();
     private readonly events: StormEvent[];
     private readonly tmpDir: string;
+    private readonly conversationId: string;
 
-    constructor(userPrompt: string, blocks: BlockDefinitionInfo[], events: StormEvent[]) {
+    constructor(conversationId: string, userPrompt: string, blocks: BlockDefinitionInfo[], events: StormEvent[]) {
         this.userPrompt = userPrompt;
         this.blocks = blocks;
         this.events = events;
-        this.tmpDir = os.tmpdir();
+        this.tmpDir = Path.join(os.tmpdir(), conversationId);
+        this.conversationId = conversationId;
     }
 
     public async process() {
-        for (const block of this.blocks) {
-            await this.processBlockCode(block);
-        }
-
+        const promises = this.blocks.map((block) => {
+            return this.processBlockCode(block);
+        });
+        await Promise.all(promises);
         this.out.end();
     }
 
+    isAborted() {
+        return this.out.isAborted();
+    }
+
     public getStream() {
         return this.out;
     }
@@ -100,6 +114,9 @@ export class StormCodegen {
      * Generates the code for a block and sends it to the AI
      */
     private async processBlockCode(block: BlockDefinitionInfo) {
+        if (this.isAborted()) {
+            return;
+        }
         // Generate the code for the block using the standard codegen templates
         const generatedResult = await this.generateBlock(block.content);
         if (!generatedResult) {
@@ -109,7 +126,11 @@ export class StormCodegen {
         const allFiles = this.toStormFiles(generatedResult);
 
         // Send all the non-ai files to the stream
-        this.emitFiles(block.uri, block.aiName, allFiles);
+        this.emitFiles(parseKapetaUri(block.uri), block.aiName, allFiles);
+
+        if (this.isAborted()) {
+            return;
+        }
 
         const relevantFiles: StormFileInfo[] = allFiles.filter(
             (file) => file.type !== AIFileTypes.IGNORE && file.type !== AIFileTypes.WEB_SCREEN
@@ -125,12 +146,20 @@ export class StormCodegen {
             });
 
             uiStream.on('data', (evt) => {
-                this.handleUiOutput(block.uri, block.aiName, evt);
+                this.handleUiOutput(parseKapetaUri(block.uri), block.aiName, evt);
+            });
+
+            this.out.on('aborted', () => {
+                uiStream.abort();
             });
 
             await uiStream.waitForDone();
         }
 
+        if (this.isAborted()) {
+            return;
+        }
+
         // Gather the context files for implementation. These will be all be passed to the AI
         const contextFiles: StormFileInfo[] = relevantFiles.filter(
             (file) => ![AIFileTypes.SERVICE, AIFileTypes.WEB_SCREEN].includes(file.type)
@@ -140,7 +169,7 @@ export class StormCodegen {
         const serviceFiles: StormFileInfo[] = allFiles.filter((file) => file.type === AIFileTypes.SERVICE);
         if (serviceFiles.length > 0) {
             await this.processTemplates(
-                block.uri,
+                parseKapetaUri(block.uri),
                 block.aiName,
                 stormClient.createServiceImplementation.bind(stormClient),
                 serviceFiles,
@@ -150,6 +179,10 @@ export class StormCodegen {
 
         const basePath = this.getBasePath(block.content.metadata.name);
 
+        if (this.isAborted()) {
+            return;
+        }
+
         for (const serviceFile of serviceFiles) {
             const filePath = join(basePath, serviceFile.filename);
             await writeFile(filePath, serviceFile.content);
@@ -168,18 +201,36 @@ export class StormCodegen {
         const filesToBeFixed = serviceFiles.concat(contextFiles);
         const codeGenerator = new BlockCodeGenerator(block.content as BlockDefinition);
         await this.verifyAndFixCode(codeGenerator, basePath, filesToBeFixed, allFiles);
+
+        const blockRef = block.uri;
+        this.out.emit('data', {
+            type: 'BLOCK_READY',
+            reason: 'Block ready',
+            created: Date.now(),
+            payload: {
+                path: basePath,
+                blockName: block.aiName,
+                blockRef,
+                instanceId: StormEventParser.toInstanceIdFromRef(blockRef),
+            },
+        } satisfies StormEvent);
     }
 
-    private async verifyAndFixCode(codeGenerator: CodeGenerator, basePath: string, filesToBeFixed: StormFileInfo[], knownFiles: StormFileInfo[]) {
+    private async verifyAndFixCode(
+        codeGenerator: CodeGenerator,
+        basePath: string,
+        filesToBeFixed: StormFileInfo[],
+        knownFiles: StormFileInfo[]
+    ) {
         let attempts = 0;
         let validCode = false;
         for (let i = 0; i <= 3; i++) {
             attempts++;
             try {
-                console.log(`Validating the code in ${basePath} attempt #${attempts}`)
+                console.log(`Validating the code in ${basePath} attempt #${attempts}`);
                 const result = await codeGenerator.validateForTarget(basePath);
                 if (result && result.valid) {
-                     validCode = true;
+                    validCode = true;
                     break;
                 }
 
@@ -188,20 +239,28 @@ export class StormCodegen {
                     const errorStream = await stormClient.createErrorClassification(result.error, []);
                     const fixes = new Map<string, Promise<string>>();
 
+                    this.out.on('aborted', () => {
+                        errorStream.abort();
+                    });
+
                     errorStream.on('data', (evt) => {
                         if (evt.type === 'ERROR_CLASSIFIER') {
                             // find the file that caused the error
                             // strip base path from event file name, if it exists sometimes the AI sends the full path
-                            const eventFileName = this.removePrefix(basePath+'/', evt.payload.filename);
+                            const eventFileName = this.removePrefix(basePath + '/', evt.payload.filename);
                             const file = filesToBeFixed.find((f) => f.filename === eventFileName);
-                            if(!file) {
-                                console.log(`Could not find the file ${eventFileName} in the list of files to be fixed, Henrik might wanna create a new file for this fix`);
+                            if (!file) {
+                                console.log(
+                                    `Could not find the file ${eventFileName} in the list of files to be fixed, Henrik might wanna create a new file for this fix`
+                                );
                             }
                             // read the content of the file
                             const content = readFileSync(join(basePath, eventFileName), 'utf8');
-                            const fix = `${evt.payload.potentialFix}\n---\n${knownFiles.map(e => e.filename).join("\n")}\n---\n${content}`;
-                            console.log(`trying to fix the code in ${eventFileName}`);
-                            console.debug(`with the fix:\n${fix}`)
+                            const fix = `${evt.payload.potentialFix}\n---\n${knownFiles
+                                .map((e) => e.filename)
+                                .join('\n')}\n---\n${content}`;
+                            //console.log(`trying to fix the code in ${eventFileName}`);
+                            //console.debug(`with the fix:\n${fix}`);
                             const code = this.codeFix(fix);
                             fixes.set(join(basePath, eventFileName), code);
                         }
@@ -226,7 +285,7 @@ export class StormCodegen {
 
     removePrefix(prefix: string, str: string): string {
         if (str.startsWith(prefix)) {
-          return str.slice(prefix.length);
+            return str.slice(prefix.length);
         }
         return str;
     }
@@ -246,6 +305,9 @@ export class StormCodegen {
                     resolve(evt.payload.content);
                 }
             });
+            this.out.on('aborted', () => {
+                fixStream.abort();
+            });
             fixStream.on('error', (err) => {
                 reject(err);
             });
@@ -321,6 +383,10 @@ export class StormCodegen {
 
             const files: StormEventFile[] = [];
 
+            this.out.on('aborted', () => {
+                stream.abort();
+            });
+
             stream.on('data', (evt) => {
                 const file = this.handleTemplateFileOutput(blockUri, aiName, templateFile, evt);
                 if (file) {
@@ -379,6 +445,9 @@ export class StormCodegen {
      * Generates the code using codegen for a given block.
      */
     private async generateBlock(yamlContent: Definition) {
+        if (this.isAborted()) {
+            return;
+        }
         if (!yamlContent.spec.target?.kind) {
             //Not all block types have targets
             return;
@@ -395,4 +464,8 @@ export class StormCodegen {
         new CodeWriter(basePath).write(generatedResult);
         return generatedResult;
     }
+
+    abort() {
+        this.out.abort();
+    }
 }

package/src/storm/event-parser.ts

@@ -3,7 +3,15 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-import { StormBlockInfoFilled, StormBlockType, StormConnection, StormEvent, StormResourceType } from './events';
+import {
+    StormBlockInfoFilled,
+    StormBlockType,
+    StormConnection,
+    StormEvent,
+    StormEventPhases,
+    StormEventPhaseType,
+    StormResourceType,
+} from './events';
 import {
     BlockDefinition,
     BlockInstance,
@@ -29,7 +37,7 @@ import { v5 as uuid } from 'uuid';
 import { definitionsManager } from '../definitionsManager';
 
 export interface BlockDefinitionInfo {
-    uri: KapetaURI;
+    uri: string;
     content: BlockDefinition;
     aiName: string;
 }
@@ -87,6 +95,24 @@ function prettifyKaplang(source: string) {
     }
 }
 
+export function createPhaseStartEvent(type: StormEventPhaseType): StormEventPhases {
+    return createPhaseEvent(true, type);
+}
+
+export function createPhaseEndEvent(type: StormEventPhaseType): StormEventPhases {
+    return createPhaseEvent(false, type);
+}
+
+export function createPhaseEvent(start: boolean, type: StormEventPhaseType): StormEventPhases {
+    return {
+        type: start ? 'PHASE_START' : 'PHASE_END',
+        created: Date.now(),
+        payload: {
+            phaseType: type,
+        },
+    };
+}
+
 export async function resolveOptions(): Promise<StormOptions> {
     // Predefined types for now - TODO: Allow user to select / change
 
@@ -325,7 +351,7 @@ export class StormEventParser {
     }
 
     public toResult(handle: string): StormDefinitions {
-        const planRef = StormEventParser.toRef(handle, this.planName ?? 'undefined');
+        const planRef = StormEventParser.toRef(handle, this.planName || 'undefined');
         const blockDefinitions = this.toBlockDefinitions(handle);
         const refIdMap: { [key: string]: string } = {};
         const blocks = Object.entries(blockDefinitions).map(([ref, block]) => {
@@ -337,7 +363,7 @@ export class StormEventParser {
                 block: {
                     ref,
                 },
-                name: block.content.metadata.title ?? block.content.metadata.name,
+                name: block.content.metadata.title || block.content.metadata.name,
                 dimensions: {
                     left: 0,
                     top: 0,
@@ -454,6 +480,8 @@ export class StormEventParser {
                 name: planRef.fullName,
                 title: this.planName,
                 description: this.planDescription,
+                structure: 'mono',
+                visibility: 'private',
             },
             spec: {
                 blocks,
@@ -474,7 +502,7 @@ export class StormEventParser {
             const blockRef = StormEventParser.toRef(handle, blockInfo.name);
 
             const blockDefinitionInfo: BlockDefinitionInfo = {
-                uri: blockRef,
+                uri: blockRef.toNormalizedString(),
                 aiName: blockInfo.name,
                 content: {
                     kind: this.toBlockKind(blockInfo.type),

package/src/storm/events.ts

@@ -144,9 +144,9 @@ export interface StormEventCodeFix {
     };
 }
 export interface StormEventErrorClassifierInfo {
-    error: string,
-    filename: string,
-    potentialFix: string
+    error: string;
+    filename: string;
+    potentialFix: string;
 }
 
 export interface ScreenTemplate {
@@ -196,6 +196,18 @@ export interface StormEventFile {
     };
 }
 
+export interface StormEventBlockReady {
+    type: 'BLOCK_READY';
+    reason: string;
+    created: number;
+    payload: {
+        path: string;
+        blockName: string;
+        blockRef: string;
+        instanceId: string;
+    };
+}
+
 export interface StormEventDone {
     type: 'DONE';
     created: number;
@@ -208,6 +220,20 @@ export interface StormEventDefinitionChange {
     payload: StormDefinitions;
 }
 
+export enum StormEventPhaseType {
+    META = 'META',
+    DEFINITIONS = 'DEFINITIONS',
+    IMPLEMENTATION = 'IMPLEMENTATION',
+}
+
+export interface StormEventPhases {
+    type: 'PHASE_START' | 'PHASE_END';
+    created: number;
+    payload: {
+        phaseType: StormEventPhaseType;
+    };
+}
+
 export type StormEvent =
     | StormEventCreateBlock
     | StormEventCreateConnection
@@ -223,4 +249,6 @@ export type StormEvent =
     | StormEventDone
     | StormEventDefinitionChange
     | StormEventErrorClassifier
-    | StormEventCodeFix;
+    | StormEventCodeFix
+    | StormEventBlockReady
+    | StormEventPhases;

package/src/storm/routes.ts

@@ -4,15 +4,24 @@
  */
 
 import Router from 'express-promise-router';
+import FS from 'fs-extra';
 import { Response } from 'express';
 import { corsHandler } from '../middleware/cors';
 import { stringBody } from '../middleware/stringBody';
 import { KapetaBodyRequest } from '../types';
-import { StormContextRequest, StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
+import { StormContextRequest, StormCreateBlockRequest, StormStream } from './stream';
 import { ConversationIdHeader, stormClient } from './stormClient';
-import { StormEvent } from './events';
-import { resolveOptions, StormDefinitions, StormEventParser } from './event-parser';
+import { StormEvent, StormEventPhaseType } from './events';
+import {
+    createPhaseEndEvent,
+    createPhaseStartEvent,
+    resolveOptions,
+    StormDefinitions,
+    StormEventParser,
+} from './event-parser';
 import { StormCodegen } from './codegen';
+import { assetManager } from '../assetManager';
+import Path from 'path';
 
 const router = Router();
 
@@ -32,18 +41,48 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
         const aiRequest: StormContextRequest = JSON.parse(req.stringBody ?? '{}');
         const metaStream = await stormClient.createMetadata(aiRequest.prompt, conversationId);
 
+        onRequestAborted(req, res, () => {
+            metaStream.abort();
+        });
+
         res.set('Content-Type', 'application/x-ndjson');
         res.set('Access-Control-Expose-Headers', ConversationIdHeader);
         res.set(ConversationIdHeader, metaStream.getConversationId());
 
+        let currentPhase = StormEventPhaseType.META;
+
         metaStream.on('data', (data: StormEvent) => {
             const result = eventParser.processEvent(req.params.handle, data);
 
+            switch (data.type) {
+                case 'CREATE_API':
+                case 'CREATE_MODEL':
+                case 'CREATE_TYPE':
+                    if (currentPhase !== StormEventPhaseType.DEFINITIONS) {
+                        sendEvent(res, createPhaseEndEvent(StormEventPhaseType.META));
+                        currentPhase = StormEventPhaseType.DEFINITIONS;
+                        sendEvent(res, createPhaseStartEvent(StormEventPhaseType.DEFINITIONS));
+                    }
+                    break;
+            }
+
             sendEvent(res, data);
             sendDefinitions(res, result);
         });
 
-        await waitForStormStream(metaStream);
+        try {
+            sendEvent(res, createPhaseStartEvent(StormEventPhaseType.META));
+
+            await waitForStormStream(metaStream);
+        } finally {
+            if (!metaStream.isAborted()) {
+                sendEvent(res, createPhaseEndEvent(currentPhase));
+            }
+        }
+
+        if (metaStream.isAborted()) {
+            return;
+        }
 
         if (!eventParser.isValid()) {
             // We can't continue if the meta stream is invalid
@@ -59,22 +98,66 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
 
         const result = eventParser.toResult(handle);
 
+        if (metaStream.isAborted()) {
+            return;
+        }
+
         sendDefinitions(res, result);
 
         if (!req.query.skipCodegen) {
-            const stormCodegen = new StormCodegen(aiRequest.prompt, result.blocks, eventParser.getEvents());
-
-            const codegenPromise = streamStormPartialResponse(stormCodegen.getStream(), res);
-
-            await stormCodegen.process();
-
-            await codegenPromise;
+            try {
+                sendEvent(res, createPhaseStartEvent(StormEventPhaseType.IMPLEMENTATION));
+                const stormCodegen = new StormCodegen(
+                    metaStream.getConversationId(),
+                    aiRequest.prompt,
+                    result.blocks,
+                    eventParser.getEvents()
+                );
+
+                onRequestAborted(req, res, () => {
+                    stormCodegen.abort();
+                });
+
+                const codegenPromise = streamStormPartialResponse(stormCodegen.getStream(), res);
+
+                await stormCodegen.process();
+
+                await codegenPromise;
+            } finally {
+                if (!metaStream.isAborted()) {
+                    sendEvent(res, createPhaseEndEvent(StormEventPhaseType.IMPLEMENTATION));
+                }
+            }
         }
 
         sendDone(res);
     } catch (err: any) {
         sendError(err, res);
-        res.end();
+        if (!res.closed) {
+            res.end();
+        }
+    }
+});
+
+router.post('/block/create', async (req: KapetaBodyRequest, res: Response) => {
+    const createRequest: StormCreateBlockRequest = JSON.parse(req.stringBody ?? '{}');
+
+    try {
+        const ymlPath = Path.join(createRequest.newPath, 'kapeta.yml');
+
+        const [asset] = await assetManager.createAsset(ymlPath, createRequest.definition);
+
+        if (await FS.pathExists(createRequest.tmpPath)) {
+            await FS.move(createRequest.tmpPath, createRequest.newPath, {
+                overwrite: true,
+            });
+
+            res.send(await assetManager.updateAsset(asset.ref, createRequest.definition));
+        } else {
+            res.send(asset);
+        }
+    } catch (err: any) {
+        res.status(500).send({ error: err.message });
     }
 });
 
@@ -88,6 +171,9 @@ function sendDefinitions(res: Response, result: StormDefinitions) {
 }
 
 function sendDone(res: Response) {
+    if (res.closed) {
+        return;
+    }
     sendEvent(res, {
         type: 'DONE',
         created: Date.now(),
@@ -97,6 +183,9 @@ function sendDone(res: Response) {
 }
 
 function sendError(err: Error, res: Response) {
+    if (res.closed) {
+        return;
+    }
     console.error('Failed to send prompt', err);
     if (res.headersSent) {
         sendEvent(res, {
@@ -138,7 +227,19 @@ function streamStormPartialResponse(result: StormStream, res: Response) {
 }
 
 function sendEvent(res: Response, evt: StormEvent) {
+    if (res.closed) {
+        return;
+    }
     res.write(JSON.stringify(evt) + '\n');
 }
 
+function onRequestAborted(req: KapetaBodyRequest, res: Response, onAborted: () => void) {
+    req.on('close', () => {
+        onAborted();
+    });
+    res.on('close', () => {
+        onAborted();
+    });
+}
+
 export default router;

package/src/storm/stormClient.ts

@@ -13,6 +13,7 @@ import {
     StormStream,
     StormUIImplementationPrompt,
 } from './stream';
+import { getRawAsset } from 'node:sea';
 
 export const STORM_ID = 'storm';
 
@@ -60,6 +61,9 @@ class StormClient {
             conversationId: body.conversationId,
         });
 
+        const abort = new AbortController();
+        options.signal = abort.signal;
+
         const response = await fetch(options.url, options);
 
         if (response.status !== 200) {
@@ -69,7 +73,6 @@ class StormClient {
         }
 
         const conversationId = response.headers.get(ConversationIdHeader);
-        console.log('Received conversationId', conversationId);
 
         const out = new StormStream(stringPrompt, conversationId);
 
@@ -87,6 +90,10 @@ class StormClient {
             out.end();
         });
 
+        out.on('aborted', () => {
+            abort.abort();
+        });
+
         return out;
     }