@kapeta/local-cluster-service 0.54.12 → 0.55.0

package/src/storm/event-parser.ts

@@ -36,11 +36,15 @@ import {
 } from '@kapeta/kaplang-core';
 import { v5 as uuid } from 'uuid';
 import { definitionsManager } from '../definitionsManager';
+import { PREDEFINED_BLOCKS } from './predefined';
+
+import _ from 'lodash';
 
 export interface BlockDefinitionInfo {
     uri: string;
     content: BlockDefinition;
     aiName: string;
+    archetype?: string;
 }
 
 export interface StormDefinitions {
@@ -268,7 +272,7 @@ export class StormEventParser {
     /**
      * Builds plan and block definitions - and enriches events with relevant refs and ids
      */
-    public processEvent(handle: string, evt: StormEvent): StormDefinitions {
+    public async processEvent(handle: string, evt: StormEvent): Promise<StormDefinitions> {
         let blockInfo;
         this.events.push(evt);
         switch (evt.type) {
@@ -338,7 +342,7 @@ export class StormEventParser {
                 break;
         }
 
-        return this.toResult(handle);
+        return await this.toResult(handle);
     }
 
     public getEvents(): StormEvent[] {
@@ -356,9 +360,9 @@ export class StormEventParser {
         return this.error;
     }
 
-    public toResult(handle: string): StormDefinitions {
+    public async toResult(handle: string): Promise<StormDefinitions> {
         const planRef = StormEventParser.toRef(handle, this.planName || 'undefined');
-        const blockDefinitions = this.toBlockDefinitions(handle);
+        const blockDefinitions = await this.toBlockDefinitions(handle);
         const refIdMap: { [key: string]: string } = {};
         const blocks = Object.entries(blockDefinitions).map(([ref, block]) => {
             // Create a deterministic uuid
@@ -504,163 +508,182 @@ export class StormEventParser {
         };
     }
 
-    public toBlockDefinitions(handle: string): { [key: string]: BlockDefinitionInfo } {
+    public async toBlockDefinitions(handle: string): Promise<{ [key: string]: BlockDefinitionInfo }> {
         const result: { [key: string]: BlockDefinitionInfo } = {};
 
-        Object.entries(this.blocks).forEach(([, blockInfo]) => {
+        for (const [, blockInfo] of Object.entries(this.blocks)) {
             const blockRef = StormEventParser.toRef(handle, blockInfo.name);
 
-            const blockDefinitionInfo: BlockDefinitionInfo = {
-                uri: blockRef.toNormalizedString(),
-                aiName: blockInfo.name,
-                content: {
-                    kind: this.toBlockKind(blockInfo.type),
-                    metadata: {
-                        title: blockInfo.name,
-                        name: blockRef.fullName,
-                        description: blockInfo.description,
-                    },
-                    spec: {
-                        entities: {
-                            types: [],
-                            source: {
-                                type: KAPLANG_ID,
-                                version: KAPLANG_VERSION,
-                                value: '',
-                            },
+            let blockDefinitionInfo: BlockDefinitionInfo;
+
+            if (blockInfo.archetype) {
+                blockDefinitionInfo = await this.resolveArchetypeBlockDefinition(blockRef, blockInfo);
+            } else {
+                blockDefinitionInfo = this.createBlockDefinitionInfo(blockRef, blockInfo, handle);
+            }
+
+            result[blockRef.toNormalizedString()] = blockDefinitionInfo;
+        }
+
+        return result;
+    }
+
+    private createBlockDefinitionInfo(
+        blockRef: KapetaURI,
+        blockInfo: StormBlockInfoFilled,
+        handle: string
+    ): BlockDefinitionInfo {
+        const blockDefinitionInfo: BlockDefinitionInfo = {
+            uri: blockRef.toNormalizedString(),
+            aiName: blockInfo.name,
+            content: {
+                kind: this.toBlockKind(blockInfo.type),
+                metadata: {
+                    title: blockInfo.name,
+                    name: blockRef.fullName,
+                    description: blockInfo.description,
+                },
+                spec: {
+                    entities: {
+                        types: [],
+                        source: {
+                            type: KAPLANG_ID,
+                            version: KAPLANG_VERSION,
+                            value: '',
                         },
-                        target: this.toBlockTarget(handle, blockInfo.type),
-                        providers: [],
-                        consumers: [],
                     },
+                    target: this.toBlockTarget(handle, blockInfo.type),
+                    providers: [],
+                    consumers: [],
                 },
-            };
+            },
+        };
 
-            const blockSpec = blockDefinitionInfo.content.spec;
+        const blockSpec = blockDefinitionInfo.content.spec;
 
-            const apiResources: { [key: string]: Resource | undefined } = {};
-            let dbResource: Resource | undefined = undefined;
+        const apiResources: { [key: string]: Resource | undefined } = {};
+        let dbResource: Resource | undefined = undefined;
 
-            (blockInfo.resources || []).forEach((resource) => {
-                const port = {
-                    type: this.toPortType(resource.type),
-                };
-                switch (resource.type) {
-                    case 'API': {
-                        const apiResource = {
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                                methods: {},
-                                source: {
-                                    type: KAPLANG_ID,
-                                    version: KAPLANG_VERSION,
-                                    value: '',
-                                } satisfies SourceCode,
-                            },
-                        };
-                        apiResources[resource.name] = apiResource;
-                        blockSpec.providers!.push(apiResource);
+        (blockInfo.resources || []).forEach((resource) => {
+            const port = {
+                type: this.toPortType(resource.type),
+            };
+            switch (resource.type) {
+                case 'API': {
+                    const apiResource = {
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                            methods: {},
+                            source: {
+                                type: KAPLANG_ID,
+                                version: KAPLANG_VERSION,
+                                value: '',
+                            } satisfies SourceCode,
+                        },
+                    };
+                    apiResources[resource.name] = apiResource;
+                    blockSpec.providers!.push(apiResource);
+                    break;
+                }
+                case 'CLIENT':
+                    blockSpec.consumers!.push({
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                            methods: {},
+                            source: {
+                                type: KAPLANG_ID,
+                                version: KAPLANG_VERSION,
+                                value: '',
+                            } satisfies SourceCode,
+                        },
+                    });
+                    break;
+                case 'EXTERNAL_API':
+                    break;
+                case 'EXCHANGE':
+                    break;
+                case 'PUBLISHER':
+                    break;
+                case 'QUEUE':
+                    break;
+                case 'SUBSCRIBER':
+                    break;
+                case 'JWTPROVIDER':
+                case 'WEBPAGE':
+                    blockSpec.providers!.push({
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                        },
+                    });
+                    break;
+                case 'DATABASE':
+                    if (dbResource) {
                         break;
                     }
-                    case 'CLIENT':
-                        blockSpec.consumers!.push({
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                                methods: {},
-                                source: {
-                                    type: KAPLANG_ID,
-                                    version: KAPLANG_VERSION,
-                                    value: '',
-                                } satisfies SourceCode,
-                            },
-                        });
-                        break;
-                    case 'EXTERNAL_API':
-                        break;
-                    case 'EXCHANGE':
-                        break;
-                    case 'PUBLISHER':
-                        break;
-                    case 'QUEUE':
-                        break;
-                    case 'SUBSCRIBER':
-                        break;
-                    case 'JWTPROVIDER':
-                    case 'WEBPAGE':
-                        blockSpec.providers!.push({
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                            },
-                        });
-                        break;
-                    case 'DATABASE':
-                        if (dbResource) {
-                            break;
-                        }
-                        dbResource = {
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                                models: [],
-                                source: {
-                                    type: KAPLANG_ID,
-                                    version: KAPLANG_VERSION,
-                                    value: '',
-                                } satisfies SourceCode,
-                            },
-                        };
-                        blockSpec.consumers!.push(dbResource);
-                        break;
-                    case 'JWTCONSUMER':
-                    case 'WEBFRAGMENT':
-                    case 'SMTPCLIENT':
-                        blockSpec.consumers!.push({
-                            kind: this.toResourceKind(resource.type),
-                            metadata: {
-                                name: resource.name,
-                                description: resource.description,
-                            },
-                            spec: {
-                                port,
-                            },
-                        });
-                }
-            });
+                    dbResource = {
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                            models: [],
+                            source: {
+                                type: KAPLANG_ID,
+                                version: KAPLANG_VERSION,
+                                value: '',
+                            } satisfies SourceCode,
+                        },
+                    };
+                    blockSpec.consumers!.push(dbResource);
+                    break;
+                case 'JWTCONSUMER':
+                case 'WEBFRAGMENT':
+                case 'SMTPCLIENT':
+                    blockSpec.consumers!.push({
+                        kind: this.toResourceKind(resource.type),
+                        metadata: {
+                            name: resource.name,
+                            description: resource.description,
+                        },
+                        spec: {
+                            port,
+                        },
+                    });
+            }
+        });
 
-            blockInfo.apis.forEach((api) => {
-                const dslApi = DSLAPIParser.parse(api, {
-                    ignoreSemantics: true,
-                }) as (DSLMethod | DSLController)[];
-
-                let exactMatch = false;
-                if (dslApi[0] && dslApi[0].type == DSLEntityType.CONTROLLER) {
-                    const name = dslApi[0].name.toLowerCase();
-                    const apiResourceName = Object.keys(apiResources).find((key) => key.indexOf(name) > -1);
-                    if (apiResourceName) {
-                        const exactResource = apiResources[apiResourceName];
-                        exactResource!.spec.source.value += api + '\n\n';
-                        exactMatch = true;
-                    }
+        blockInfo.apis.forEach((api) => {
+            const dslApi = DSLAPIParser.parse(api, {
+                ignoreSemantics: true,
+            }) as (DSLMethod | DSLController)[];
+
+            let exactMatch = false;
+            if (dslApi[0] && dslApi[0].type == DSLEntityType.CONTROLLER) {
+                const name = dslApi[0].name.toLowerCase();
+                const apiResourceName = Object.keys(apiResources).find((key) => key.indexOf(name) > -1);
+                if (apiResourceName) {
+                    const exactResource = apiResources[apiResourceName];
+                    exactResource!.spec.source.value += api + '\n\n';
+                    exactMatch = true;
                 }
+            }
 
                 if (!exactMatch) {
                     // if we couldn't place the given api on the exact resource we just park it on the first
@@ -675,20 +698,17 @@ export class StormEventParser {
                 }
             });
 
-            blockInfo.types.forEach((type) => {
-                blockSpec.entities!.source!.value += type + '\n';
-            });
-
-            if (dbResource) {
-                blockInfo.models.forEach((model) => {
-                    dbResource!.spec.source.value += model + '\n';
-                });
-            }
-
-            result[blockRef.toNormalizedString()] = blockDefinitionInfo;
+        blockInfo.types.forEach((type) => {
+            blockSpec.entities!.source!.value += type + '\n';
         });
 
-        return result;
+        if (dbResource) {
+            blockInfo.models.forEach((model) => {
+                dbResource!.spec.source.value += model + '\n';
+            });
+        }
+
+        return blockDefinitionInfo;
     }
 
     private toResourceKind(type: StormResourceType) {
@@ -861,4 +881,25 @@ export class StormEventParser {
         }
         return undefined;
     }
+
+    private async resolveArchetypeBlockDefinition(
+        blockRef: KapetaURI,
+        blockInfo: StormBlockInfoFilled
+    ): Promise<BlockDefinitionInfo> {
+        const predefinedBlock = PREDEFINED_BLOCKS.get(blockInfo.archetype!);
+        if (!predefinedBlock) {
+            throw new Error('Predefined block not found for archetype [' + blockInfo.archetype + ']');
+        }
+
+        const blockDefinition = await predefinedBlock.getBlockDefinition();
+        _.set(blockDefinition!, ['metadata', 'name'], blockRef.fullName);
+        _.set(blockDefinition!, ['metadata', 'title'], blockRef.name);
+
+        return {
+            uri: blockRef.toNormalizedString(),
+            aiName: blockInfo.name,
+            content: blockDefinition,
+            archetype: blockInfo.archetype,
+        } as BlockDefinitionInfo;
+    }
 }

package/src/storm/events.ts

@@ -32,6 +32,7 @@ export interface StormBlockInfo {
     }[];
     blockRef?: string;
     instanceId?: string;
+    archetype?: string;
 }
 
 export interface StormBlockInfoFilled extends StormBlockInfo {

package/src/storm/predefined.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { BlockDefinition } from '@kapeta/schemas';
+import * as yaml from 'js-yaml';
+
+export class PredefinedBlock {
+    constructor(
+        public archetype: string,
+        public kapetaYml: string,
+        public gitRepo: {
+            owner: string;
+            repo: string;
+            path: string;
+        }
+    ) {}
+
+    public getGitRepo(): { owner: string; repo: string; path: string } {
+        return this.gitRepo;
+    }
+
+    private async getKapetaYML(): Promise<string> {
+        const response = await fetch(this.kapetaYml);
+        if (!response.ok) {
+            throw new Error(`HTTP error! Status: ${response.status}`);
+        }
+        return await response.text();
+    }
+
+    public async getBlockDefinition(): Promise<BlockDefinition> {
+        const kapetaYml = await this.getKapetaYML();
+        return yaml.load(kapetaYml) as BlockDefinition;
+    }
+}
+
+const predefinedBlocks: PredefinedBlock[] = [
+    new PredefinedBlock(
+        'USER_SERVICE',
+        'https://raw.githubusercontent.com/kapetacom/everything/master/blocks/everything-user/kapeta.yml',
+        {
+            owner: 'kapetacom',
+            repo: 'everything',
+            path: 'blocks/everything-user',
+        }
+    ),
+];
+
+export const PREDEFINED_BLOCKS = new Map<string, PredefinedBlock>(
+    predefinedBlocks.map((block) => [block.archetype, block])
+);

package/src/storm/routes.ts

@@ -54,9 +54,9 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
 
         // Helper to avoid sending the plan multiple times in a row
         const sendUpdatedPlan = _.debounce(sendDefinitions, 50, { maxWait: 200 });
-        metaStream.on('data', (data: StormEvent) => {
+        metaStream.on('data', async (data: StormEvent) => {
             try {
-                const result = eventParser.processEvent(handle, data);
+                const result = await eventParser.processEvent(handle, data);
 
                 switch (data.type) {
                     case 'CREATE_API':
@@ -103,7 +103,7 @@ router.post('/:handle/all', async (req: KapetaBodyRequest, res: Response) => {
             return;
         }
 
-        const result = eventParser.toResult(handle);
+        const result = await eventParser.toResult(handle);
 
         if (metaStream.isAborted()) {
             return;

package/test/storm/codegen.test.ts

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import predefinedUserEvents from './predefined-user-events.json';
+import { StormEventParser } from '../../src/storm/event-parser';
+import { parserOptions } from './event-parser.test';
+import { StormEvent } from '../../src/storm/events';
+import { StormCodegen } from '../../src/storm/codegen';
+import uuid from 'node-uuid';
+import { StormStream } from '../../src/storm/stream';
+
+describe('codegen', () => {
+    it('predefined components', async () => {
+        const events = predefinedUserEvents as StormEvent[];
+        const parser = new StormEventParser(parserOptions);
+        for (const event of events) {
+            await parser.processEvent('kapeta', event);
+        }
+
+        const result = await parser.toResult('kapeta');
+
+        const conversationId = uuid.v4().toString();
+
+        let codegen = new StormCodegen(conversationId, '', result.blocks, parser.getEvents());
+        let codegenPromise = consumeStream(codegen.getStream());
+        await codegen.process();
+
+        const stormEvents = await codegenPromise;
+        expect(stormEvents[0].type).toBe('FILE_DONE');
+        expect(stormEvents[stormEvents.length - 1].type).toBe('BLOCK_READY');
+    });
+});
+
+async function consumeStream(stream: StormStream): Promise<StormEvent[]> {
+    const events: StormEvent[] = [];
+    return new Promise<StormEvent[]>((resolve) => {
+        stream.on('data', (data) => {
+            events.push(data);
+        });
+
+        stream.waitForDone();
+        resolve(events);
+    });
+}

package/test/storm/event-parser.test.ts

@@ -6,8 +6,9 @@
 import { StormEventParser } from '../../src/storm/event-parser';
 import { StormEvent } from '../../src/storm/events';
 import simpleBlogEvents from './simple-blog-events.json';
+import predefinedUserEvents from './predefined-user-events.json';
 
-const parserOptions = {
+export const parserOptions = {
     serviceKind: 'kapeta/block-service:local',
     serviceLanguage: 'kapeta/language-target-nodejs-ts:local',
 
@@ -138,11 +139,13 @@ const events: StormEvent[] = [
 ];
 
 describe('event-parser', () => {
-    it('it can parse events into a plan and blocks with proper layout', () => {
+    it('it can parse events into a plan and blocks with proper layout', async () => {
         const parser = new StormEventParser(parserOptions);
-        events.forEach((event) => parser.processEvent('kapeta', event));
+        for (const event of events) {
+            await parser.processEvent('kapeta', event);
+        }
 
-        const result = parser.toResult('kapeta');
+        const result = await parser.toResult('kapeta');
 
         expect(result.plan.metadata.name).toBe('kapeta/my-plan');
         expect(result.plan.metadata.description).toBe('my plan description');
@@ -181,21 +184,40 @@ describe('event-parser', () => {
         expect(result.plan.spec.connections[0].provider.resourceName).toBe(apiResource?.metadata.name);
     });
 
-    it('it will split api into correct provider', () => {
+    it('it will split api into correct provider', async () => {
         const events = simpleBlogEvents as StormEvent[];
         const parser = new StormEventParser(parserOptions);
-        events.forEach((event) => parser.processEvent('kapeta', event));
+        for (const event of events) {
+            await parser.processEvent('kapeta', event);
+        }
 
-        const result = parser.toResult('kapeta');
+        const result = await parser.toResult('kapeta');
 
         const blogService = result.blocks.find((block) => block.aiName === 'blog-service');
         expect(blogService).toBeDefined();
         expect(blogService?.content).toBeDefined();
 
-        const apiProviders = blogService?.content?.spec?.providers?.filter((provider) => provider.kind === 'kapeta/block-type-api:local');
+        const apiProviders = blogService?.content?.spec?.providers?.filter(
+            (provider) => provider.kind === 'kapeta/block-type-api:local'
+        );
         expect(apiProviders).toBeDefined();
         expect(apiProviders!.length).toBe(2);
-        expect(apiProviders!["0"].spec.source.value).not.toBe('');
-        expect(apiProviders!["1"].spec.source.value).not.toBe('');
+        expect(apiProviders!['0'].spec.source.value).not.toBe('');
+        expect(apiProviders!['1'].spec.source.value).not.toBe('');
+    });
+
+    it('predefined components', async () => {
+        const events = predefinedUserEvents as StormEvent[];
+        const parser = new StormEventParser(parserOptions);
+        for (const event of events) {
+            await parser.processEvent('kapeta', event);
+        }
+
+        const result = await parser.toResult('kapeta');
+        expect(result.blocks.length).toBe(1);
+        expect(result.blocks[0].content.metadata.title).toBe('user-service');
+        expect(result.blocks[0].content.metadata.title).toBe('user-service');
+        expect(result.blocks[0].content.metadata.name).toBe('kapeta/user-service');
+        expect(result.blocks[0].archetype).toBeDefined();
     });
 });

package/test/storm/predefined-user-events.json

@@ -0,0 +1,13 @@
+[
+    {
+        "type": "CREATE_BLOCK",
+        "reason": "Handles user registration, authentication, and authorization.",
+        "payload": {
+            "name": "user-service",
+            "resources": [],
+            "type": "BACKEND",
+            "archetype": "USER_SERVICE"
+        },
+        "created": 1720597268038
+    }
+]