@kapeta/local-cluster-service 0.45.0 → 0.47.0

package/dist/esm/src/storm/routes.js

@@ -21,17 +21,17 @@ router.post('/:handle/all', async (req, res) => {
     try {
         const stormOptions = await (0, event_parser_1.resolveOptions)();
         const eventParser = new event_parser_1.StormEventParser(stormOptions);
-        console.log('Got prompt', req.stringBody);
         const aiRequest = JSON.parse(req.stringBody ?? '{}');
         const metaStream = await stormClient_1.stormClient.createMetadata(aiRequest.prompt, aiRequest.history);
         res.set('Content-Type', 'application/x-ndjson');
         metaStream.on('data', (data) => {
-            eventParser.addEvent(data);
+            const result = eventParser.addEvent(req.params.handle, data);
+            sendDefinitions(res, result);
         });
         await streamStormPartialResponse(metaStream, res);
         if (!eventParser.isValid()) {
             // We can't continue if the meta stream is invalid
-            res.write({
+            sendEvent(res, {
                 type: 'ERROR_INTERNAL',
                 payload: { error: eventParser.getError() },
                 reason: 'Failed to generate system',
@@ -41,8 +41,8 @@ router.post('/:handle/all', async (req, res) => {
             return;
         }
         const result = eventParser.toResult(handle);
-        console.log('RESULT\n', JSON.stringify(result, null, 2));
-        const stormCodegen = new codegen_1.StormCodegen(aiRequest.prompt, result.blocks);
+        sendDefinitions(res, result);
+        const stormCodegen = new codegen_1.StormCodegen(aiRequest.prompt, result.blocks, eventParser.getEvents());
         const codegenStream = streamStormPartialResponse(stormCodegen.getStream(), res);
         await stormCodegen.process();
         await codegenStream;
@@ -52,42 +52,30 @@ router.post('/:handle/all', async (req, res) => {
         sendError(err, res);
     }
 });
-router.post('/metadata', async (req, res) => {
-    const aiRequest = JSON.parse(req.stringBody ?? '{}');
-    const result = await stormClient_1.stormClient.createMetadata(aiRequest.prompt, aiRequest.history);
-    await streamStormResponse(result, res);
-});
-router.post('/services/implement', async (req, res) => {
-    const aiRequest = JSON.parse(req.stringBody ?? '{}');
-    const result = await stormClient_1.stormClient.createServiceImplementation(aiRequest.prompt, aiRequest.history);
-    await streamStormResponse(result, res);
-});
-router.post('/ui/implement', async (req, res) => {
-    const aiRequest = JSON.parse(req.stringBody ?? '{}');
-    const result = await stormClient_1.stormClient.createUIImplementation(aiRequest.prompt, aiRequest.history);
-    await streamStormResponse(result, res);
-});
-async function streamStormResponse(result, res) {
-    res.set('Content-Type', 'application/x-ndjson');
-    await streamStormPartialResponse(result, res);
-    sendDone(res);
+function sendDefinitions(res, result) {
+    sendEvent(res, {
+        type: 'DEFINITION_CHANGE',
+        payload: result,
+        reason: 'Updates to definition',
+        created: Date.now(),
+    });
 }
 function sendDone(res) {
-    res.write(JSON.stringify({
+    sendEvent(res, {
         type: 'DONE',
         created: Date.now(),
-    }) + '\n');
+    });
     res.end();
 }
 function sendError(err, res) {
     console.error('Failed to send prompt', err);
     if (res.headersSent) {
-        res.write(JSON.stringify({
+        sendEvent(res, {
             type: 'ERROR_INTERNAL',
             created: Date.now(),
             payload: { error: err.message },
             reason: 'Failed while sending prompt',
-        }) + '\n');
+        });
     }
     else {
         res.status(400).send({ error: err.message });
@@ -96,7 +84,7 @@ function sendError(err, res) {
 function streamStormPartialResponse(result, res) {
     return new Promise((resolve, reject) => {
         result.on('data', (data) => {
-            res.write(JSON.stringify(data) + '\n');
+            sendEvent(res, data);
         });
         result.on('error', (err) => {
             reject(err);
@@ -106,4 +94,7 @@ function streamStormPartialResponse(result, res) {
         });
     });
 }
+function sendEvent(res, evt) {
+    res.write(JSON.stringify(evt) + '\n');
+}
 exports.default = router;

package/dist/esm/src/storm/stormClient.d.ts

@@ -1,4 +1,4 @@
-import { ConversationItem, StormFileImplementationPrompt, StormStream } from './stream';
+import { ConversationItem, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt } from './stream';
 export declare const STORM_ID = "storm";
 declare class StormClient {
     private readonly _baseUrl;
@@ -6,7 +6,7 @@ declare class StormClient {
     private createOptions;
     private send;
     createMetadata(prompt: string, history?: ConversationItem[]): Promise<StormStream>;
-    createUIImplementation(prompt: StormFileImplementationPrompt, history?: ConversationItem[]): Promise<StormStream>;
+    createUIImplementation(prompt: StormUIImplementationPrompt, history?: ConversationItem[]): Promise<StormStream>;
     createServiceImplementation(prompt: StormFileImplementationPrompt, history?: ConversationItem[]): Promise<StormStream>;
 }
 export declare const stormClient: StormClient;

package/dist/esm/src/storm/stormClient.js

@@ -53,12 +53,6 @@ class StormClient {
         jsonLStream.on('error', (error) => {
             out.emit('error', error);
         });
-        jsonLStream.on('pause', () => {
-            console.log('paused');
-        });
-        jsonLStream.on('resume', () => {
-            console.log('resumed');
-        });
         jsonLStream.on('close', () => {
             out.end();
         });
@@ -77,7 +71,6 @@ class StormClient {
         });
     }
     createServiceImplementation(prompt, history) {
-        console.log('SENDING SERVICE PROMPT', JSON.stringify(prompt, null, 2));
         return this.send('/v2/services/merge', {
             history: history ?? [],
             prompt,

package/dist/esm/src/storm/stream.d.ts

@@ -36,3 +36,10 @@ export interface StormFileImplementationPrompt {
     template: StormFileInfo;
     prompt: string;
 }
+export interface StormUIImplementationPrompt {
+    events: StormEvent[];
+    templates: StormFileInfo[];
+    context: StormFileInfo[];
+    blockName: string;
+    prompt: string;
+}

package/dist/esm/test/storm/event-parser.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+export {};

package/dist/esm/test/storm/event-parser.test.js

@@ -0,0 +1,169 @@
+"use strict";
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const event_parser_1 = require("../../src/storm/event-parser");
+const parserOptions = {
+    serviceKind: 'kapeta/block-service:local',
+    serviceLanguage: 'kapeta/language-target-nodejs-ts:local',
+    frontendKind: 'kapeta/block-type-frontend:local',
+    frontendLanguage: 'kapeta/language-target-react-ts:local',
+    cliKind: 'kapeta/block-type-cli:local',
+    cliLanguage: 'kapeta/language-target-nodejs-ts:local',
+    desktopKind: 'kapeta/block-type-desktop:local',
+    desktopLanguage: 'kapeta/language-target-electron-ts:local',
+    gatewayKind: 'kapeta/block-type-gateway:local',
+    mqKind: 'kapeta/block-type-mq:local',
+    exchangeKind: 'kapeta/resource-type-exchange:local',
+    queueKind: 'kapeta/resource-type-queue:local',
+    publisherKind: 'kapeta/resource-type-publisher:local',
+    subscriberKind: 'kapeta/resource-type-subscriber:local',
+    databaseKind: 'kapeta/block-type-database:local',
+    apiKind: 'kapeta/block-type-api:local',
+    clientKind: 'kapeta/block-type-client:local',
+    webPageKind: 'kapeta/block-type-web-page:local',
+    webFragmentKind: 'kapeta/block-type-web-fragment:local',
+    jwtProviderKind: 'kapeta/resource-type-jwt-provider:local',
+    jwtConsumerKind: 'kapeta/resource-type-jwt-consumer:local',
+    smtpKind: 'kapeta/resource-type-smtp:local',
+    externalApiKind: 'kapeta/resource-type-external-api:local',
+};
+const events = [
+    {
+        type: 'CREATE_PLAN_PROPERTIES',
+        created: Date.now(),
+        reason: 'create plan properties',
+        payload: {
+            name: 'my-plan',
+            description: 'my plan description',
+        },
+    },
+    {
+        type: 'CREATE_BLOCK',
+        reason: 'create backend',
+        created: Date.now(),
+        payload: {
+            name: 'service',
+            description: 'A service block',
+            type: 'BACKEND',
+            resources: [
+                {
+                    name: 'entities',
+                    type: 'DATABASE',
+                    description: 'A database resource',
+                },
+                {
+                    type: 'API',
+                    name: 'entities',
+                    description: 'An API resource',
+                },
+            ],
+        },
+    },
+    {
+        type: 'CREATE_BLOCK',
+        reason: 'create frontend',
+        created: Date.now(),
+        payload: {
+            name: 'ui',
+            description: 'A frontend block',
+            type: 'FRONTEND',
+            resources: [
+                {
+                    name: 'web',
+                    type: 'WEBPAGE',
+                    description: 'A web page',
+                },
+                {
+                    type: 'CLIENT',
+                    name: 'entities',
+                    description: 'Client for backend',
+                },
+            ],
+        },
+    },
+    {
+        type: 'CREATE_CONNECTION',
+        created: Date.now(),
+        reason: 'connect service to ui',
+        payload: {
+            fromComponent: 'service',
+            fromResource: 'entities',
+            fromResourceType: 'API',
+            toComponent: 'ui',
+            toResource: 'entities',
+            toResourceType: 'CLIENT',
+        },
+    },
+    {
+        type: 'CREATE_API',
+        reason: 'create api',
+        created: Date.now(),
+        payload: {
+            blockName: 'service',
+            content: `controller Entities('/entities') {
+                        @GET('/')
+                        list(): string[]
+                    }`,
+        },
+    },
+    {
+        type: 'CREATE_MODEL',
+        created: Date.now(),
+        reason: 'create model',
+        payload: {
+            blockName: 'service',
+            content: `type Entity {
+                        @Id
+                        id: string
+                        
+                        name: string
+                    }`,
+        },
+    },
+];
+describe('event-parser', () => {
+    it('it can parse events into a plan and blocks with proper layout', () => {
+        const parser = new event_parser_1.StormEventParser(parserOptions);
+        events.forEach((event) => parser.addEvent('kapeta', event));
+        const result = parser.toResult('kapeta');
+        expect(result.plan.metadata.name).toBe('kapeta/my-plan');
+        expect(result.plan.metadata.description).toBe('my plan description');
+        expect(result.blocks.length).toBe(2);
+        expect(result.blocks[0].content.metadata.name).toBe('kapeta/service');
+        expect(result.blocks[1].content.metadata.name).toBe('kapeta/ui');
+        const dbResource = result.blocks[0].content.spec.consumers?.[0];
+        const apiResource = result.blocks[0].content.spec.providers?.[0];
+        const clientResource = result.blocks[1].content.spec.consumers?.[0];
+        const pageResource = result.blocks[1].content.spec.providers?.[0];
+        expect(apiResource).toBeDefined();
+        expect(clientResource).toBeDefined();
+        expect(dbResource).toBeDefined();
+        expect(pageResource).toBeDefined();
+        expect(apiResource?.kind).toBe(parserOptions.apiKind);
+        expect(clientResource?.kind).toBe(parserOptions.clientKind);
+        expect(dbResource?.kind).toBe(parserOptions.databaseKind);
+        expect(pageResource?.kind).toBe(parserOptions.webPageKind);
+        expect(apiResource?.spec).toEqual(clientResource?.spec);
+        expect(dbResource?.spec.source.value).toContain('type Entity');
+        const serviceBlockInstance = result.plan.spec.blocks[0];
+        expect(serviceBlockInstance.name).toBe('service');
+        expect(serviceBlockInstance.dimensions.width).toBe(150);
+        expect(serviceBlockInstance.dimensions.height).toBe(200);
+        expect(serviceBlockInstance.dimensions.top).toBe(3);
+        expect(serviceBlockInstance.dimensions.left).toBe(6);
+        const uiBlockInstance = result.plan.spec.blocks[1];
+        expect(uiBlockInstance.name).toBe('ui');
+        expect(uiBlockInstance.dimensions.width).toBe(150);
+        expect(uiBlockInstance.dimensions.height).toBe(200);
+        expect(uiBlockInstance.dimensions.top).toBe(107);
+        expect(uiBlockInstance.dimensions.left).toBe(112);
+        expect(result.plan.spec.connections.length).toBe(1);
+        expect(result.plan.spec.connections[0].consumer.blockId).toBe(uiBlockInstance.id);
+        expect(result.plan.spec.connections[0].consumer.resourceName).toBe(clientResource?.metadata.name);
+        expect(result.plan.spec.connections[0].provider.blockId).toBe(serviceBlockInstance.id);
+        expect(result.plan.spec.connections[0].provider.resourceName).toBe(apiResource?.metadata.name);
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.45.0",
+    "version": "0.47.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -74,6 +74,8 @@
         "gunzip-maybe": "^1.4.2",
         "lodash": "^4.17.15",
         "md5": "2.2.1",
+        "ngraph.forcelayout": "^3.3.1",
+        "ngraph.graph": "^20.0.1",
         "node-cache": "^5.1.2",
         "node-uuid": "^1.4.8",
         "parse-data-uri": "^0.2.0",

package/src/storm/codegen.ts

@@ -8,7 +8,7 @@ import { AIFileTypes, BlockCodeGenerator, GeneratedFile, GeneratedResult } from
 import { BlockDefinition } from '@kapeta/schemas';
 import { codeGeneratorManager } from '../codeGeneratorManager';
 import { STORM_ID, stormClient } from './stormClient';
-import { ScreenTemplate, StormEvent, StormEventFile } from './events';
+import { ScreenTemplate, StormEvent, StormEventFile, StormEventScreen } from './events';
 import { BlockDefinitionInfo } from './event-parser';
 import { ConversationItem, StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
 import { KapetaURI } from '@kapeta/nodejs-utils';
@@ -22,15 +22,15 @@ export class StormCodegen {
     private readonly userPrompt: string;
     private readonly blocks: BlockDefinitionInfo[];
     private readonly out = new StormStream();
+    private readonly events: StormEvent[];
 
-    constructor(userPrompt: string, blocks: BlockDefinitionInfo[]) {
+    constructor(userPrompt: string, blocks: BlockDefinitionInfo[], events: StormEvent[]) {
         this.userPrompt = userPrompt;
         this.blocks = blocks;
+        this.events = events;
     }
 
     public async process() {
-        console.log('Processing blocks', this.blocks.length);
-
         for (const block of this.blocks) {
             await this.processBlockCode(block);
         }
@@ -42,13 +42,44 @@ export class StormCodegen {
         return this.out;
     }
 
-    private handleFileOutput(blockUri: KapetaURI, template: StormFileInfo, data: StormEvent) {
+    private handleTemplateFileOutput(blockUri: KapetaURI, template: StormFileInfo, data: StormEvent) {
         switch (data.type) {
             case 'FILE':
                 template.filename = data.payload.filename;
                 template.content = data.payload.content;
+                return this.handleFileOutput(blockUri, data);
+        }
+    }
+
+    private handleUiOutput(blockUri: KapetaURI, data: StormEvent) {
+        switch (data.type) {
+            case 'SCREEN':
+                this.out.emit('data', {
+                    type: 'SCREEN',
+                    reason: data.reason,
+                    created: Date.now(),
+                    payload: {
+                        ...data.payload,
+                        blockName: blockUri.toNormalizedString(),
+                    },
+                });
+            case 'FILE':
+                return this.handleFileOutput(blockUri, data);
+        }
+    }
+
+    private handleFileOutput(blockUri: KapetaURI, data: StormEvent) {
+        switch (data.type) {
+            case 'FILE':
                 this.emitFile(blockUri, data.payload.filename, data.payload.content, data.reason);
-                break;
+                return {
+                    type: 'FILE',
+                    created: Date.now(),
+                    payload: {
+                        filename: data.payload.filename,
+                        content: data.payload.content,
+                    },
+                } as StormEventFile;
         }
     }
 
@@ -57,9 +88,8 @@ export class StormCodegen {
      */
     private async processBlockCode(block: BlockDefinitionInfo) {
         // Generate the code for the block using the standard codegen templates
-        const generatedResult = await this.generateBlock(block.content, block.screens);
+        const generatedResult = await this.generateBlock(block.content);
         if (!generatedResult) {
-            console.warn('No generated result for block', block.uri);
             return;
         }
 
@@ -68,24 +98,41 @@ export class StormCodegen {
         // Send all the non-ai files to the stream
         this.emitFiles(block.uri, allFiles);
 
-        const implementFiles = [AIFileTypes.SERVICE, AIFileTypes.WEB_SCREEN];
+        const relevantFiles: StormFileInfo[] = allFiles.filter(
+            (file) => file.type !== AIFileTypes.IGNORE && file.type !== AIFileTypes.WEB_SCREEN
+        );
+        const uiTemplates: StormFileInfo[] = allFiles.filter((file) => file.type === AIFileTypes.WEB_SCREEN);
+        if (uiTemplates.length > 0) {
+            const uiStream = await stormClient.createUIImplementation({
+                events: this.events,
+                templates: uiTemplates,
+                context: relevantFiles,
+                blockName: block.aiName,
+                prompt: this.userPrompt,
+            });
+
+            uiStream.on('data', (evt) => {
+                this.handleUiOutput(block.uri, evt);
+            });
+
+            await uiStream.waitForDone();
+        }
 
-        // Gather the context files. These will be all be passed to the AI
-        const contextFiles: StormFileInfo[] = allFiles.filter(
-            (file) => file.type !== AIFileTypes.IGNORE && !implementFiles.includes(file.type)
+        // Gather the context files for implementation. These will be all be passed to the AI
+        const contextFiles: StormFileInfo[] = relevantFiles.filter(
+            (file) => ![AIFileTypes.SERVICE, AIFileTypes.WEB_SCREEN].includes(file.type)
         );
 
         // Send the service and UI templates to the AI. These will be send one-by-one in addition to the context files
         const serviceFiles: StormFileInfo[] = allFiles.filter((file) => file.type === AIFileTypes.SERVICE);
-        await this.processTemplates(
-            block.uri,
-            stormClient.createServiceImplementation.bind(stormClient),
-            serviceFiles,
-            contextFiles
-        );
-
-        //const uiTemplates: StormFileInfo[] = allFiles.filter((file) => file.type === 'ui');
-        //await this.processTemplates(stormClient.createUIImplementation, uiTemplates, contextFiles);
+        if (serviceFiles.length > 0) {
+            await this.processTemplates(
+                block.uri,
+                stormClient.createServiceImplementation.bind(stormClient),
+                serviceFiles,
+                contextFiles
+            );
+        }
     }
 
     /**
@@ -129,7 +176,7 @@ export class StormCodegen {
     /**
      * Sends the template to the AI and processes the response
      */
-    private processTemplates(
+    private async processTemplates(
         blockUri: KapetaURI,
         generator: ImplementationGenerator,
         templates: StormFileInfo[],
@@ -142,12 +189,22 @@ export class StormCodegen {
                 prompt: this.userPrompt,
             });
 
-            stream.on('data', (evt) => this.handleFileOutput(blockUri, templateFile, evt));
+            const files: StormEventFile[] = [];
 
-            return stream.waitForDone();
+            stream.on('data', (evt) => {
+                const file = this.handleTemplateFileOutput(blockUri, templateFile, evt);
+                if (file) {
+                    files.push(file);
+                }
+            });
+
+            await stream.waitForDone();
+            return files;
         });
 
-        return Promise.all(promises);
+        const fileChunks = await Promise.all(promises);
+
+        return fileChunks.flat();
     }
 
     /**
@@ -191,7 +248,7 @@ export class StormCodegen {
     /**
      * Generates the code using codegen for a given block.
      */
-    private async generateBlock(yamlContent: Definition, screens: ScreenTemplate[] | undefined) {
+    private async generateBlock(yamlContent: Definition) {
         if (!yamlContent.spec.target?.kind) {
             //Not all block types have targets
             return;
@@ -203,7 +260,6 @@ export class StormCodegen {
 
         const codeGenerator = new BlockCodeGenerator(yamlContent as BlockDefinition);
         codeGenerator.withOption('AIContext', STORM_ID);
-        codeGenerator.withOption('AIScreens', screens ?? []);
 
         return codeGenerator.generate();
     }