@kapeta/local-cluster-service 0.45.0 → 0.47.0

package/src/storm/event-parser.ts

@@ -3,14 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-import {
-    ScreenTemplate,
-    StormBlockInfoFilled,
-    StormBlockType,
-    StormConnection,
-    StormEvent,
-    StormResourceType,
-} from './events';
+import { StormBlockInfoFilled, StormBlockType, StormConnection, StormEvent, StormResourceType } from './events';
 import {
     BlockDefinition,
     BlockInstance,
@@ -24,14 +17,16 @@ import { KapetaURI, normalizeKapetaUri, parseKapetaUri } from '@kapeta/nodejs-ut
 import { KAPLANG_ID, KAPLANG_VERSION, RESTMethod } from '@kapeta/kaplang-core';
 import uuid from 'node-uuid';
 import { definitionsManager } from '../definitionsManager';
+import createGraph from 'ngraph.graph';
+import createLayout from 'ngraph.forcelayout';
 
 export interface BlockDefinitionInfo {
     uri: KapetaURI;
     content: BlockDefinition;
-    screens: ScreenTemplate[];
+    aiName: string;
 }
 
-export interface ParsedResult {
+export interface StormDefinitions {
     plan: Plan;
     blocks: BlockDefinitionInfo[];
 }
@@ -175,6 +170,8 @@ export async function resolveOptions(): Promise<StormOptions> {
     };
 }
 
+const LAYOUT_MARGIN = 50;
+
 export class StormEventParser {
     private events: StormEvent[] = [];
     private planName: string = '';
@@ -196,8 +193,7 @@ export class StormEventParser {
         this.connections = [];
     }
 
-    public addEvent(evt: StormEvent): void {
-        console.log('Processing storm event', evt);
+    public addEvent(handle:string, evt: StormEvent): StormDefinitions {
         this.events.push(evt);
         switch (evt.type) {
             case 'CREATE_PLAN_PROPERTIES':
@@ -210,7 +206,6 @@ export class StormEventParser {
                     apis: [],
                     models: [],
                     types: [],
-                    screens: [],
                 };
                 break;
             case 'PLAN_RETRY':
@@ -232,21 +227,14 @@ export class StormEventParser {
             case 'CREATE_CONNECTION':
                 this.connections.push(evt.payload);
                 break;
-            case 'SCREEN':
-                this.blocks[evt.payload.blockName].screens.push({
-                    name: evt.payload.name,
-                    description: evt.payload.description,
-                    url: evt.payload.url,
-                    template: evt.payload.template,
-                });
-                break;
 
             default:
             case 'SCREEN_CANDIDATE':
             case 'FILE':
-                console.warn('Unhandled event: %s', evt.type, evt);
                 break;
         }
+
+        return this.toResult(handle);
     }
 
     public getEvents(): StormEvent[] {
@@ -261,15 +249,57 @@ export class StormEventParser {
         return this.error;
     }
 
-    private applyLayoutToBlocks(result: ParsedResult): ParsedResult {
+    private applyLayoutToBlocks(result: StormDefinitions): StormDefinitions {
+        const graph = createGraph();
+        const blockInstances: { [key: string]: BlockInstance } = {};
+
+        result.plan.spec.blocks.forEach((block, index) => {
+            graph.addNode(block.id, block);
+            blockInstances[block.id] = block;
+        });
+
+        result.plan.spec.connections.forEach((connection) => {
+            graph.addLink(connection.provider.blockId, connection.consumer.blockId);
+        });
+
+        const layout = createLayout(graph, {
+            springLength: 150,
+            debug: true,
+            dimensions: 2,
+            gravity: 2,
+            springCoefficient: 0.0008,
+        });
+
+        for (let i = 0; i < 100; ++i) {
+            layout.step();
+        }
+
+        // Layout might place things in negative space. We move everything to positive space
+        const graphBox = layout.getGraphRect();
+        let yAdjust = 0;
+        let xAdjust = 0;
+        if (graphBox.y1 < 0) {
+            yAdjust = -graphBox.y1;
+        }
+        if (graphBox.x1 < 0) {
+            xAdjust = -graphBox.x1;
+        }
+
+        graph.forEachNode((node) => {
+            const position = layout.getNodePosition(node.id);
+            blockInstances[node.id].dimensions.left = LAYOUT_MARGIN + Math.round(position.x + xAdjust);
+            blockInstances[node.id].dimensions.top = LAYOUT_MARGIN + Math.round(position.y + yAdjust);
+        });
+
+        layout.dispose();
+
         return result;
     }
 
-    public toResult(handle: string): ParsedResult {
+    public toResult(handle: string): StormDefinitions {
         const planRef = this.toRef(handle, this.planName);
         const blockDefinitions = this.toBlockDefinitions(handle);
         const refIdMap: { [key: string]: string } = {};
-        const screens: { [key: string]: ScreenTemplate[] } = {};
         const blocks = Object.entries(blockDefinitions).map(([ref, block]) => {
             const id = uuid.v4();
             refIdMap[ref] = id;
@@ -282,23 +312,12 @@ export class StormEventParser {
                 dimensions: {
                     left: 0,
                     top: 0,
-                    width: 200,
+                    width: 150,
                     height: 200,
                 },
             } satisfies BlockInstance;
         });
 
-        Object.values(this.blocks).forEach((blockInfo) => {
-            const blockRef = this.toRef(handle, blockInfo.name);
-            const block = blockDefinitions[blockRef.toNormalizedString()];
-            if (!block) {
-                console.warn('Block not found: %s', blockInfo.name);
-                return;
-            }
-
-            screens[blockRef.fullName] = blockInfo.screens;
-        });
-
         // Copy API methods from API provider to CLIENT consumer
         this.connections
             .filter((connection) => connection.fromResourceType === 'API' && connection.toResourceType === 'CLIENT')
@@ -307,12 +326,12 @@ export class StormEventParser {
                 const clientConsumerRef = this.toRef(handle, apiConnection.toComponent);
                 const apiProviderBlock = blockDefinitions[apiProviderRef.toNormalizedString()];
                 if (!apiProviderBlock) {
-                    console.warn('API provider not found: %s', apiConnection.fromComponent);
+                    console.warn('API provider not found: %s', apiConnection.fromComponent, apiConnection);
                     return;
                 }
                 const clientConsumerBlock = blockDefinitions[clientConsumerRef.toNormalizedString()];
                 if (!clientConsumerBlock) {
-                    console.warn('Client consumer not found: %s', apiConnection.toComponent);
+                    console.warn('Client consumer not found: %s', apiConnection.toComponent, apiConnection);
                     return;
                 }
 
@@ -324,25 +343,27 @@ export class StormEventParser {
                     console.warn(
                         'API resource not found: %s on %s',
                         apiConnection.fromResource,
-                        apiProviderRef.toNormalizedString()
+                        apiProviderRef.toNormalizedString(),
+                        apiConnection
                     );
                     return;
                 }
 
                 const clientResource = clientConsumerBlock.content.spec.consumers?.find((clientResource) => {
                     if (clientResource.kind !== this.options.clientKind) {
-                        return;
-                    }
-                    if (clientResource.metadata.name !== apiConnection.toResource) {
-                        return;
+                        console.warn('Client resource kind mismatch: %s', clientResource.kind, this.options.clientKind);
+                        return false;
                     }
+
+                    return clientResource.metadata.name === apiConnection.toResource;
                 });
 
                 if (!clientResource) {
                     console.warn(
                         'Client resource not found: %s on %s',
                         apiConnection.toResource,
-                        clientConsumerRef.toNormalizedString()
+                        clientConsumerRef.toNormalizedString(),
+                        apiConnection
                     );
                     return;
                 }
@@ -407,6 +428,7 @@ export class StormEventParser {
 
             const blockDefinitionInfo: BlockDefinitionInfo = {
                 uri: blockRef,
+                aiName: blockInfo.name,
                 content: {
                     kind: this.toBlockKind(blockInfo.type),
                     metadata: {
@@ -428,7 +450,6 @@ export class StormEventParser {
                         consumers: [],
                     },
                 },
-                screens: blockInfo.screens,
             };
 
             const blockSpec = blockDefinitionInfo.content.spec;
@@ -524,7 +545,7 @@ export class StormEventParser {
                                 } satisfies SourceCode,
                             },
                         };
-                        blockSpec.providers!.push(dbResource);
+                        blockSpec.consumers!.push(dbResource);
                         break;
                     case 'JWTCONSUMER':
                     case 'WEBFRAGMENT':

package/src/storm/events.ts

@@ -2,6 +2,7 @@
  * Copyright 2023 Kapeta Inc.
  * SPDX-License-Identifier: BUSL-1.1
  */
+import {StormDefinitions} from "./event-parser";
 
 export type StormResourceType =
     | 'API'
@@ -35,7 +36,6 @@ export interface StormBlockInfoFilled extends StormBlockInfo {
     apis: string[];
     types: string[];
     models: string[];
-    screens: ScreenTemplate[];
 }
 
 export interface StormEventCreateBlock {
@@ -155,6 +155,13 @@ export interface StormEventDone {
     created: number;
 }
 
+export interface StormEventDefinitionChange {
+    type: 'DEFINITION_CHANGE';
+    reason: string;
+    created: number;
+    payload: StormDefinitions;
+}
+
 export type StormEvent =
     | StormEventCreateBlock
     | StormEventCreateConnection
@@ -166,4 +173,5 @@ export type StormEvent =
     | StormEventScreen
     | StormEventScreenCandidate
     | StormEventFile
-    | StormEventDone;
+    | StormEventDone
+    | StormEventDefinitionChange;

package/src/storm/routes.ts

@@ -4,15 +4,15 @@
  */
 
 import Router from 'express-promise-router';
-import { Response } from 'express';
-import { corsHandler } from '../middleware/cors';
-import { stringBody } from '../middleware/stringBody';
-import { KapetaBodyRequest } from '../types';
-import { StormContextRequest, StormFileImplementationPrompt, StormFileInfo, StormStream } from './stream';
-import { stormClient } from './stormClient';
-import { StormEvent } from './events';
-import { resolveOptions, StormEventParser } from './event-parser';
-import { StormCodegen } from './codegen';
+import {Response} from 'express';
+import {corsHandler} from '../middleware/cors';
+import {stringBody} from '../middleware/stringBody';
+import {KapetaBodyRequest} from '../types';
+import {StormContextRequest, StormFileImplementationPrompt, StormFileInfo, StormStream} from './stream';
+import {stormClient} from './stormClient';
+import {StormEvent} from './events';
+import {resolveOptions, StormDefinitions, StormEventParser} from './event-parser';
+import {StormCodegen} from './codegen';
 
 const router = Router();
 
@@ -27,34 +27,36 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
 
         const eventParser = new StormEventParser(stormOptions);
 
-        console.log('Got prompt', req.stringBody);
         const aiRequest: StormContextRequest = JSON.parse(req.stringBody ?? '{}');
         const metaStream = await stormClient.createMetadata(aiRequest.prompt, aiRequest.history);
 
         res.set('Content-Type', 'application/x-ndjson');
 
         metaStream.on('data', (data: StormEvent) => {
-            eventParser.addEvent(data);
+            const result = eventParser.addEvent(req.params.handle, data);
+
+            sendDefinitions(res, result);
         });
 
         await streamStormPartialResponse(metaStream, res);
 
         if (!eventParser.isValid()) {
             // We can't continue if the meta stream is invalid
-            res.write({
+            sendEvent(res, {
                 type: 'ERROR_INTERNAL',
-                payload: { error: eventParser.getError() },
+                payload: {error: eventParser.getError()},
                 reason: 'Failed to generate system',
                 created: Date.now(),
-            } satisfies StormEvent);
+            });
             res.end();
             return;
         }
+
         const result = eventParser.toResult(handle);
 
-        console.log('RESULT\n', JSON.stringify(result, null, 2));
+        sendDefinitions(res, result);
 
-        const stormCodegen = new StormCodegen(aiRequest.prompt, result.blocks);
+        const stormCodegen = new StormCodegen(aiRequest.prompt, result.blocks, eventParser.getEvents());
 
         const codegenStream = streamStormPartialResponse(stormCodegen.getStream(), res);
 
@@ -68,42 +70,20 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
     }
 });
 
-router.post('/metadata', async (req: KapetaBodyRequest, res: Response) => {
-    const aiRequest: StormContextRequest = JSON.parse(req.stringBody ?? '{}');
-    const result = await stormClient.createMetadata(aiRequest.prompt, aiRequest.history);
-
-    await streamStormResponse(result, res);
-});
-
-router.post('/services/implement', async (req: KapetaBodyRequest, res: Response) => {
-    const aiRequest: StormContextRequest<StormFileImplementationPrompt> = JSON.parse(req.stringBody ?? '{}');
-    const result = await stormClient.createServiceImplementation(aiRequest.prompt, aiRequest.history);
-
-    await streamStormResponse(result, res);
-});
-
-router.post('/ui/implement', async (req: KapetaBodyRequest, res: Response) => {
-    const aiRequest: StormContextRequest<StormFileImplementationPrompt> = JSON.parse(req.stringBody ?? '{}');
-    const result = await stormClient.createUIImplementation(aiRequest.prompt, aiRequest.history);
-
-    await streamStormResponse(result, res);
-});
-
-async function streamStormResponse(result: StormStream, res: Response) {
-    res.set('Content-Type', 'application/x-ndjson');
-
-    await streamStormPartialResponse(result, res);
-
-    sendDone(res);
+function sendDefinitions(res: Response, result: StormDefinitions) {
+    sendEvent(res, {
+        type: 'DEFINITION_CHANGE',
+        payload: result,
+        reason: 'Updates to definition',
+        created: Date.now(),
+    });
 }
 
 function sendDone(res: Response) {
-    res.write(
-        JSON.stringify({
-            type: 'DONE',
-            created: Date.now(),
-        } satisfies StormEvent) + '\n'
-    );
+    sendEvent(res, {
+        type: 'DONE',
+        created: Date.now(),
+    });
 
     res.end();
 }
@@ -111,23 +91,22 @@ function sendDone(res: Response) {
 function sendError(err: Error, res: Response) {
     console.error('Failed to send prompt', err);
     if (res.headersSent) {
-        res.write(
-            JSON.stringify({
-                type: 'ERROR_INTERNAL',
-                created: Date.now(),
-                payload: { error: err.message },
-                reason: 'Failed while sending prompt',
-            } satisfies StormEvent) + '\n'
-        );
+        sendEvent(res, {
+            type: 'ERROR_INTERNAL',
+            created: Date.now(),
+            payload: {error: err.message},
+            reason: 'Failed while sending prompt',
+        });
     } else {
-        res.status(400).send({ error: err.message });
+        res.status(400).send({error: err.message});
     }
 }
 
+
 function streamStormPartialResponse(result: StormStream, res: Response) {
     return new Promise<void>((resolve, reject) => {
         result.on('data', (data) => {
-            res.write(JSON.stringify(data) + '\n');
+            sendEvent(res, data);
         });
 
         result.on('error', (err) => {
@@ -140,4 +119,8 @@ function streamStormPartialResponse(result: StormStream, res: Response) {
     });
 }
 
+function sendEvent(res: Response, evt: StormEvent) {
+    res.write(JSON.stringify(evt) + '\n');
+}
+
 export default router;

package/src/storm/stormClient.ts

@@ -12,6 +12,7 @@ import {
     StormFileImplementationPrompt,
     StormFileInfo,
     StormStream,
+    StormUIImplementationPrompt,
 } from './stream';
 
 export const STORM_ID = 'storm';
@@ -73,14 +74,6 @@ class StormClient {
             out.emit('error', error);
         });
 
-        jsonLStream.on('pause', () => {
-            console.log('paused');
-        });
-
-        jsonLStream.on('resume', () => {
-            console.log('resumed');
-        });
-
         jsonLStream.on('close', () => {
             out.end();
         });
@@ -95,15 +88,14 @@ class StormClient {
         });
     }
 
-    public createUIImplementation(prompt: StormFileImplementationPrompt, history?: ConversationItem[]) {
-        return this.send<StormFileImplementationPrompt>('/v2/ui/merge', {
+    public createUIImplementation(prompt: StormUIImplementationPrompt, history?: ConversationItem[]) {
+        return this.send<StormUIImplementationPrompt>('/v2/ui/merge', {
             history: history ?? [],
             prompt,
         });
     }
 
     public createServiceImplementation(prompt: StormFileImplementationPrompt, history?: ConversationItem[]) {
-        console.log('SENDING SERVICE PROMPT', JSON.stringify(prompt, null, 2));
         return this.send<StormFileImplementationPrompt>('/v2/services/merge', {
             history: history ?? [],
             prompt,

package/src/storm/stream.ts

@@ -86,3 +86,11 @@ export interface StormFileImplementationPrompt {
     template: StormFileInfo;
     prompt: string;
 }
+
+export interface StormUIImplementationPrompt {
+    events: StormEvent[];
+    templates: StormFileInfo[];
+    context: StormFileInfo[];
+    blockName: string;
+    prompt: string;
+}

package/test/storm/event-parser.test.ts

@@ -0,0 +1,190 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { StormEventParser } from '../../src/storm/event-parser';
+import { StormEvent } from '../../src/storm/events';
+
+const parserOptions = {
+    serviceKind: 'kapeta/block-service:local',
+    serviceLanguage: 'kapeta/language-target-nodejs-ts:local',
+
+    frontendKind: 'kapeta/block-type-frontend:local',
+    frontendLanguage: 'kapeta/language-target-react-ts:local',
+
+    cliKind: 'kapeta/block-type-cli:local',
+    cliLanguage: 'kapeta/language-target-nodejs-ts:local',
+
+    desktopKind: 'kapeta/block-type-desktop:local',
+    desktopLanguage: 'kapeta/language-target-electron-ts:local',
+
+    gatewayKind: 'kapeta/block-type-gateway:local',
+
+    mqKind: 'kapeta/block-type-mq:local',
+    exchangeKind: 'kapeta/resource-type-exchange:local',
+    queueKind: 'kapeta/resource-type-queue:local',
+    publisherKind: 'kapeta/resource-type-publisher:local',
+    subscriberKind: 'kapeta/resource-type-subscriber:local',
+    databaseKind: 'kapeta/block-type-database:local',
+
+    apiKind: 'kapeta/block-type-api:local',
+    clientKind: 'kapeta/block-type-client:local',
+
+    webPageKind: 'kapeta/block-type-web-page:local',
+    webFragmentKind: 'kapeta/block-type-web-fragment:local',
+
+    jwtProviderKind: 'kapeta/resource-type-jwt-provider:local',
+    jwtConsumerKind: 'kapeta/resource-type-jwt-consumer:local',
+
+    smtpKind: 'kapeta/resource-type-smtp:local',
+    externalApiKind: 'kapeta/resource-type-external-api:local',
+};
+
+const events: StormEvent[] = [
+    {
+        type: 'CREATE_PLAN_PROPERTIES',
+        created: Date.now(),
+        reason: 'create plan properties',
+        payload: {
+            name: 'my-plan',
+            description: 'my plan description',
+        },
+    },
+    {
+        type: 'CREATE_BLOCK',
+        reason: 'create backend',
+        created: Date.now(),
+        payload: {
+            name: 'service',
+            description: 'A service block',
+            type: 'BACKEND',
+            resources: [
+                {
+                    name: 'entities',
+                    type: 'DATABASE',
+                    description: 'A database resource',
+                },
+                {
+                    type: 'API',
+                    name: 'entities',
+                    description: 'An API resource',
+                },
+            ],
+        },
+    },
+    {
+        type: 'CREATE_BLOCK',
+        reason: 'create frontend',
+        created: Date.now(),
+        payload: {
+            name: 'ui',
+            description: 'A frontend block',
+            type: 'FRONTEND',
+            resources: [
+                {
+                    name: 'web',
+                    type: 'WEBPAGE',
+                    description: 'A web page',
+                },
+                {
+                    type: 'CLIENT',
+                    name: 'entities',
+                    description: 'Client for backend',
+                },
+            ],
+        },
+    },
+    {
+        type: 'CREATE_CONNECTION',
+        created: Date.now(),
+        reason: 'connect service to ui',
+        payload: {
+            fromComponent: 'service',
+            fromResource: 'entities',
+            fromResourceType: 'API',
+            toComponent: 'ui',
+            toResource: 'entities',
+            toResourceType: 'CLIENT',
+        },
+    },
+    {
+        type: 'CREATE_API',
+        reason: 'create api',
+        created: Date.now(),
+        payload: {
+            blockName: 'service',
+            content: `controller Entities('/entities') {
+                        @GET('/')
+                        list(): string[]
+                    }`,
+        },
+    },
+    {
+        type: 'CREATE_MODEL',
+        created: Date.now(),
+        reason: 'create model',
+        payload: {
+            blockName: 'service',
+            content: `type Entity {
+                        @Id
+                        id: string
+                        
+                        name: string
+                    }`,
+        },
+    },
+];
+
+describe('event-parser', () => {
+    it('it can parse events into a plan and blocks with proper layout', () => {
+        const parser = new StormEventParser(parserOptions);
+        events.forEach((event) => parser.addEvent('kapeta', event));
+
+        const result = parser.toResult('kapeta');
+
+        expect(result.plan.metadata.name).toBe('kapeta/my-plan');
+        expect(result.plan.metadata.description).toBe('my plan description');
+        expect(result.blocks.length).toBe(2);
+        expect(result.blocks[0].content.metadata.name).toBe('kapeta/service');
+        expect(result.blocks[1].content.metadata.name).toBe('kapeta/ui');
+
+        const dbResource = result.blocks[0].content.spec.consumers?.[0];
+        const apiResource = result.blocks[0].content.spec.providers?.[0];
+        const clientResource = result.blocks[1].content.spec.consumers?.[0];
+        const pageResource = result.blocks[1].content.spec.providers?.[0];
+
+        expect(apiResource).toBeDefined();
+        expect(clientResource).toBeDefined();
+        expect(dbResource).toBeDefined();
+        expect(pageResource).toBeDefined();
+
+        expect(apiResource?.kind).toBe(parserOptions.apiKind);
+        expect(clientResource?.kind).toBe(parserOptions.clientKind);
+        expect(dbResource?.kind).toBe(parserOptions.databaseKind);
+        expect(pageResource?.kind).toBe(parserOptions.webPageKind);
+
+        expect(apiResource?.spec).toEqual(clientResource?.spec);
+        expect(dbResource?.spec.source.value).toContain('type Entity');
+
+        const serviceBlockInstance = result.plan.spec.blocks[0];
+        expect(serviceBlockInstance.name).toBe('service');
+        expect(serviceBlockInstance.dimensions.width).toBe(150);
+        expect(serviceBlockInstance.dimensions.height).toBe(200);
+        expect(serviceBlockInstance.dimensions.top).toBe(3);
+        expect(serviceBlockInstance.dimensions.left).toBe(6);
+
+        const uiBlockInstance = result.plan.spec.blocks[1];
+        expect(uiBlockInstance.name).toBe('ui');
+        expect(uiBlockInstance.dimensions.width).toBe(150);
+        expect(uiBlockInstance.dimensions.height).toBe(200);
+        expect(uiBlockInstance.dimensions.top).toBe(107);
+        expect(uiBlockInstance.dimensions.left).toBe(112);
+
+        expect(result.plan.spec.connections.length).toBe(1);
+        expect(result.plan.spec.connections[0].consumer.blockId).toBe(uiBlockInstance.id);
+        expect(result.plan.spec.connections[0].consumer.resourceName).toBe(clientResource?.metadata.name);
+        expect(result.plan.spec.connections[0].provider.blockId).toBe(serviceBlockInstance.id);
+        expect(result.plan.spec.connections[0].provider.resourceName).toBe(apiResource?.metadata.name);
+    });
+});