@kapeta/local-cluster-service 0.54.11 → 0.55.0

package/src/storm/event-parser.ts

@@ -36,11 +36,15 @@ import {
 } from '@kapeta/kaplang-core';
 import { v5 as uuid } from 'uuid';
 import { definitionsManager } from '../definitionsManager';
+import { PREDEFINED_BLOCKS } from './predefined';
+
+import _ from 'lodash';
 
 export interface BlockDefinitionInfo {
     uri: string;
     content: BlockDefinition;
     aiName: string;
+    archetype?: string;
 }
 
 export interface StormDefinitions {
@@ -268,7 +272,7 @@ export class StormEventParser {
     /**
      * Builds plan and block definitions - and enriches events with relevant refs and ids
      */
-    public processEvent(handle: string, evt: StormEvent): StormDefinitions {
+    public async processEvent(handle: string, evt: StormEvent): Promise<StormDefinitions> {
         let blockInfo;
         this.events.push(evt);
         switch (evt.type) {
@@ -338,7 +342,7 @@ export class StormEventParser {
                 break;
         }
 
-        return this.toResult(handle);
+        return await this.toResult(handle);
     }
 
     public getEvents(): StormEvent[] {
@@ -356,9 +360,9 @@ export class StormEventParser {
         return this.error;
     }
 
-    public toResult(handle: string): StormDefinitions {
+    public async toResult(handle: string): Promise<StormDefinitions> {
         const planRef = StormEventParser.toRef(handle, this.planName || 'undefined');
-        const blockDefinitions = this.toBlockDefinitions(handle);
+        const blockDefinitions = await this.toBlockDefinitions(handle);
         const refIdMap: { [key: string]: string } = {};
         const blocks = Object.entries(blockDefinitions).map(([ref, block]) => {
             // Create a deterministic uuid
@@ -504,163 +508,182 @@ export class StormEventParser {
         };
     }
 
-    public toBlockDefinitions(handle: string): { [key: string]: BlockDefinitionInfo } {
+    public async toBlockDefinitions(handle: string): Promise<{ [key: string]: BlockDefinitionInfo }> {
         const result: { [key: string]: BlockDefinitionInfo } = {};
 
-        Object.entries(this.blocks).forEach(([, blockInfo]) => {
+        for (const [, blockInfo] of Object.entries(this.blocks)) {
             const blockRef = StormEventParser.toRef(handle, blockInfo.name);
 
-            const blockDefinitionInfo: BlockDefinitionInfo = {
-                uri: blockRef.toNormalizedString(),
-                aiName: blockInfo.name,
-                content: {
-                    kind: this.toBlockKind(blockInfo.type),
-                    metadata: {
-                        title: blockInfo.name,
-                        name: blockRef.fullName,
-                        description: blockInfo.description,
-                    },
-                    spec: {
-                        entities: {
-                            types: [],
-                            source: {
-                                type: KAPLANG_ID,
-                                version: KAPLANG_VERSION,
-                                value: '',
-                            },
+            let blockDefinitionInfo: BlockDefinitionInfo;
+
+            if (blockInfo.archetype) {
+                blockDefinitionInfo = await this.resolveArchetypeBlockDefinition(blockRef, blockInfo);
+            } else {
+                blockDefinitionInfo = this.createBlockDefinitionInfo(blockRef, blockInfo, handle);
+            }
+
+            result[blockRef.toNormalizedString()] = blockDefinitionInfo;
+        }
+
+        return result;
+    }
+
+    private createBlockDefinitionInfo(
+        blockRef: KapetaURI,
+        blockInfo: StormBlockInfoFilled,
+        handle: string
+    ): BlockDefinitionInfo {
+        const blockDefinitionInfo: BlockDefinitionInfo = {
+            uri: blockRef.toNormalizedString(),
+            aiName: blockInfo.name,
+            content: {
+                kind: this.toBlockKind(blockInfo.type),
+                metadata: {
+                    title: blockInfo.name,
+                    name: blockRef.fullName,
+                    description: blockInfo.description,
+                },
+                spec: {
+                    entities: {
+                        types: [],
+                        source: {
+                            type: KAPLANG_ID,
+                            version: KAPLANG_VERSION,
+                            value: '',
                         },
-                        target: this.toBlockTarget(handle, blockInfo.type),
-                        providers: [],
-                        consumers: [],
                     },
+                    target: this.toBlockTarget(handle, blockInfo.type),
+                    providers: [],
+                    consumers: [],
                 },
-            };
+            },
+        };
 
-            const blockSpec = blockDefinitionInfo.content.spec;
+        const blockSpec = blockDefinitionInfo.content.spec;
 
-            const apiResources: { [key: string]: Resource | undefined } = {};
-            let dbResource: Resource | undefined = undefined;
+        const apiResources: { [key: string]: Resource | undefined } = {};
+        let dbResource: Resource | undefined = undefined;
 
-            (blockInfo.resources || []).forEach((resource) => {
-                const port = {
-                    type: this.toPortType(resource.type),
-                };
-                switch (resource.type) {
-                    case 'API': {
-                        const apiResource = {
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                                methods: {},
-                                source: {
-                                    type: KAPLANG_ID,
-                                    version: KAPLANG_VERSION,
-                                    value: '',
-                                } satisfies SourceCode,
-                            },
-                        };
-                        apiResources[resource.name] = apiResource;
-                        blockSpec.providers!.push(apiResource);
+        (blockInfo.resources || []).forEach((resource) => {
+            const port = {
+                type: this.toPortType(resource.type),
+            };
+            switch (resource.type) {
+                case 'API': {
+                    const apiResource = {
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                            methods: {},
+                            source: {
+                                type: KAPLANG_ID,
+                                version: KAPLANG_VERSION,
+                                value: '',
+                            } satisfies SourceCode,
+                        },
+                    };
+                    apiResources[resource.name] = apiResource;
+                    blockSpec.providers!.push(apiResource);
+                    break;
+                }
+                case 'CLIENT':
+                    blockSpec.consumers!.push({
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                            methods: {},
+                            source: {
+                                type: KAPLANG_ID,
+                                version: KAPLANG_VERSION,
+                                value: '',
+                            } satisfies SourceCode,
+                        },
+                    });
+                    break;
+                case 'EXTERNAL_API':
+                    break;
+                case 'EXCHANGE':
+                    break;
+                case 'PUBLISHER':
+                    break;
+                case 'QUEUE':
+                    break;
+                case 'SUBSCRIBER':
+                    break;
+                case 'JWTPROVIDER':
+                case 'WEBPAGE':
+                    blockSpec.providers!.push({
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                        },
+                    });
+                    break;
+                case 'DATABASE':
+                    if (dbResource) {
                         break;
                     }
-                    case 'CLIENT':
-                        blockSpec.consumers!.push({
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                                methods: {},
-                                source: {
-                                    type: KAPLANG_ID,
-                                    version: KAPLANG_VERSION,
-                                    value: '',
-                                } satisfies SourceCode,
-                            },
-                        });
-                        break;
-                    case 'EXTERNAL_API':
-                        break;
-                    case 'EXCHANGE':
-                        break;
-                    case 'PUBLISHER':
-                        break;
-                    case 'QUEUE':
-                        break;
-                    case 'SUBSCRIBER':
-                        break;
-                    case 'JWTPROVIDER':
-                    case 'WEBPAGE':
-                        blockSpec.providers!.push({
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                            },
-                        });
-                        break;
-                    case 'DATABASE':
-                        if (dbResource) {
-                            break;
-                        }
-                        dbResource = {
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                                models: [],
-                                source: {
-                                    type: KAPLANG_ID,
-                                    version: KAPLANG_VERSION,
-                                    value: '',
-                                } satisfies SourceCode,
-                            },
-                        };
-                        blockSpec.consumers!.push(dbResource);
-                        break;
-                    case 'JWTCONSUMER':
-                    case 'WEBFRAGMENT':
-                    case 'SMTPCLIENT':
-                        blockSpec.consumers!.push({
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                            },
-                        });
-                }
-            });
+                    dbResource = {
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                            models: [],
+                            source: {
+                                type: KAPLANG_ID,
+                                version: KAPLANG_VERSION,
+                                value: '',
+                            } satisfies SourceCode,
+                        },
+                    };
+                    blockSpec.consumers!.push(dbResource);
+                    break;
+                case 'JWTCONSUMER':
+                case 'WEBFRAGMENT':
+                case 'SMTPCLIENT':
+                    blockSpec.consumers!.push({
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                        },
+                    });
+            }
+        });
 
-            blockInfo.apis.forEach((api) => {
-                const dslApi = DSLAPIParser.parse(api, {
-                    ignoreSemantics: true,
-                }) as (DSLMethod | DSLController)[];
-
-                let exactMatch = false;
-                if (dslApi[0] && dslApi[0].type == DSLEntityType.CONTROLLER) {
-                    const name = dslApi[0].name.toLowerCase();
-                    const apiResourceName = Object.keys(apiResources).find((key) => key.indexOf(name) > -1);
-                    if (apiResourceName) {
-                        const exactResource = apiResources[apiResourceName];
-                        exactResource!.spec.source.value += api + '\n\n';
-                        exactMatch = true;
-                    }
+        blockInfo.apis.forEach((api) => {
+            const dslApi = DSLAPIParser.parse(api, {
+                ignoreSemantics: true,
+            }) as (DSLMethod | DSLController)[];
+
+            let exactMatch = false;
+            if (dslApi[0] && dslApi[0].type == DSLEntityType.CONTROLLER) {
+                const name = dslApi[0].name.toLowerCase();
+                const apiResourceName = Object.keys(apiResources).find((key) => key.indexOf(name) > -1);
+                if (apiResourceName) {
+                    const exactResource = apiResources[apiResourceName];
+                    exactResource!.spec.source.value += api + '\n\n';
+                    exactMatch = true;
                 }
+            }
 
                 if (!exactMatch) {
                     // if we couldn't place the given api on the exact resource we just park it on the first
@@ -675,20 +698,17 @@ export class StormEventParser {
                 }
             });
 
-            blockInfo.types.forEach((type) => {
-                blockSpec.entities!.source!.value += type + '\n';
-            });
-
-            if (dbResource) {
-                blockInfo.models.forEach((model) => {
-                    dbResource!.spec.source.value += model + '\n';
-                });
-            }
-
-            result[blockRef.toNormalizedString()] = blockDefinitionInfo;
+        blockInfo.types.forEach((type) => {
+            blockSpec.entities!.source!.value += type + '\n';
         });
 
-        return result;
+        if (dbResource) {
+            blockInfo.models.forEach((model) => {
+                dbResource!.spec.source.value += model + '\n';
+            });
+        }
+
+        return blockDefinitionInfo;
     }
 
     private toResourceKind(type: StormResourceType) {
@@ -861,4 +881,25 @@ export class StormEventParser {
         }
         return undefined;
     }
+
+    private async resolveArchetypeBlockDefinition(
+        blockRef: KapetaURI,
+        blockInfo: StormBlockInfoFilled
+    ): Promise<BlockDefinitionInfo> {
+        const predefinedBlock = PREDEFINED_BLOCKS.get(blockInfo.archetype!);
+        if (!predefinedBlock) {
+            throw new Error('Predefined block not found for archetype [' + blockInfo.archetype + ']');
+        }
+
+        const blockDefinition = await predefinedBlock.getBlockDefinition();
+        _.set(blockDefinition!, ['metadata', 'name'], blockRef.fullName);
+        _.set(blockDefinition!, ['metadata', 'title'], blockRef.name);
+
+        return {
+            uri: blockRef.toNormalizedString(),
+            aiName: blockInfo.name,
+            content: blockDefinition,
+            archetype: blockInfo.archetype,
+        } as BlockDefinitionInfo;
+    }
 }

package/src/storm/events.ts

@@ -32,6 +32,7 @@ export interface StormBlockInfo {
     }[];
     blockRef?: string;
     instanceId?: string;
+    archetype?: string;
 }
 
 export interface StormBlockInfoFilled extends StormBlockInfo {

package/src/storm/predefined.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { BlockDefinition } from '@kapeta/schemas';
+import * as yaml from 'js-yaml';
+
+export class PredefinedBlock {
+    constructor(
+        public archetype: string,
+        public kapetaYml: string,
+        public gitRepo: {
+            owner: string;
+            repo: string;
+            path: string;
+        }
+    ) {}
+
+    public getGitRepo(): { owner: string; repo: string; path: string } {
+        return this.gitRepo;
+    }
+
+    private async getKapetaYML(): Promise<string> {
+        const response = await fetch(this.kapetaYml);
+        if (!response.ok) {
+            throw new Error(`HTTP error! Status: ${response.status}`);
+        }
+        return await response.text();
+    }
+
+    public async getBlockDefinition(): Promise<BlockDefinition> {
+        const kapetaYml = await this.getKapetaYML();
+        return yaml.load(kapetaYml) as BlockDefinition;
+    }
+}
+
+const predefinedBlocks: PredefinedBlock[] = [
+    new PredefinedBlock(
+        'USER_SERVICE',
+        'https://raw.githubusercontent.com/kapetacom/everything/master/blocks/everything-user/kapeta.yml',
+        {
+            owner: 'kapetacom',
+            repo: 'everything',
+            path: 'blocks/everything-user',
+        }
+    ),
+];
+
+export const PREDEFINED_BLOCKS = new Map<string, PredefinedBlock>(
+    predefinedBlocks.map((block) => [block.archetype, block])
+);

package/src/storm/routes.ts

@@ -22,6 +22,7 @@ import {
 import { StormCodegen } from './codegen';
 import { assetManager } from '../assetManager';
 import Path from 'path';
+import _ from 'lodash';
 
 const router = Router();
 
@@ -29,7 +30,7 @@ router.use('/', corsHandler);
 router.use('/', stringBody);
 
 router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
-    const handle = req.params.handle;
+    const handle = req.params.handle as string;
 
     try {
         const stormOptions = await resolveOptions();
@@ -51,23 +52,29 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
 
         let currentPhase = StormEventPhaseType.META;
 
-        metaStream.on('data', (data: StormEvent) => {
-            const result = eventParser.processEvent(req.params.handle, data);
-
-            switch (data.type) {
-                case 'CREATE_API':
-                case 'CREATE_MODEL':
-                case 'CREATE_TYPE':
-                    if (currentPhase !== StormEventPhaseType.DEFINITIONS) {
-                        sendEvent(res, createPhaseEndEvent(StormEventPhaseType.META));
-                        currentPhase = StormEventPhaseType.DEFINITIONS;
-                        sendEvent(res, createPhaseStartEvent(StormEventPhaseType.DEFINITIONS));
-                    }
-                    break;
-            }
+        // Helper to avoid sending the plan multiple times in a row
+        const sendUpdatedPlan = _.debounce(sendDefinitions, 50, { maxWait: 200 });
+        metaStream.on('data', async (data: StormEvent) => {
+            try {
+                const result = await eventParser.processEvent(handle, data);
+
+                switch (data.type) {
+                    case 'CREATE_API':
+                    case 'CREATE_MODEL':
+                    case 'CREATE_TYPE':
+                        if (currentPhase !== StormEventPhaseType.DEFINITIONS) {
+                            sendEvent(res, createPhaseEndEvent(StormEventPhaseType.META));
+                            currentPhase = StormEventPhaseType.DEFINITIONS;
+                            sendEvent(res, createPhaseStartEvent(StormEventPhaseType.DEFINITIONS));
+                        }
+                        break;
+                }
 
-            sendEvent(res, data);
-            sendDefinitions(res, result);
+                sendEvent(res, data);
+                sendUpdatedPlan(res, result);
+            } catch (e) {
+                console.error('Failed to process event', e);
+            }
         });
 
         try {
@@ -96,7 +103,7 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
             return;
         }
 
-        const result = eventParser.toResult(handle);
+        const result = await eventParser.toResult(handle);
 
         if (metaStream.isAborted()) {
             return;

package/test/storm/codegen.test.ts

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import predefinedUserEvents from './predefined-user-events.json';
+import { StormEventParser } from '../../src/storm/event-parser';
+import { parserOptions } from './event-parser.test';
+import { StormEvent } from '../../src/storm/events';
+import { StormCodegen } from '../../src/storm/codegen';
+import uuid from 'node-uuid';
+import { StormStream } from '../../src/storm/stream';
+
+describe('codegen', () => {
+    it('predefined components', async () => {
+        const events = predefinedUserEvents as StormEvent[];
+        const parser = new StormEventParser(parserOptions);
+        for (const event of events) {
+            await parser.processEvent('kapeta', event);
+        }
+
+        const result = await parser.toResult('kapeta');
+
+        const conversationId = uuid.v4().toString();
+
+        let codegen = new StormCodegen(conversationId, '', result.blocks, parser.getEvents());
+        let codegenPromise = consumeStream(codegen.getStream());
+        await codegen.process();
+
+        const stormEvents = await codegenPromise;
+        expect(stormEvents[0].type).toBe('FILE_DONE');
+        expect(stormEvents[stormEvents.length - 1].type).toBe('BLOCK_READY');
+    });
+});
+
+async function consumeStream(stream: StormStream): Promise<StormEvent[]> {
+    const events: StormEvent[] = [];
+    return new Promise<StormEvent[]>((resolve) => {
+        stream.on('data', (data) => {
+            events.push(data);
+        });
+
+        stream.waitForDone();
+        resolve(events);
+    });
+}