@kapeta/local-cluster-service 0.16.0 → 0.16.2

package/src/assetManager.ts

@@ -1,7 +1,6 @@
 import Path from 'node:path';
 import FS from 'fs-extra';
 import YAML from 'yaml';
-import NodeCache from 'node-cache';
 import { Definition, DefinitionInfo } from '@kapeta/local-cluster-config';
 import { codeGeneratorManager } from './codeGeneratorManager';
 import { ProgressListener } from './progressListener';
@@ -12,11 +11,12 @@ import { Actions } from '@kapeta/nodejs-registry-utils';
 import { definitionsManager } from './definitionsManager';
 import { normalizeKapetaUri } from './utils/utils';
 import { taskManager } from './taskManager';
+import { SourceOfChange } from './types';
+import { cacheManager } from './cacheManager';
 
-function clearAllCaches() {
-    definitionsManager.clearCache();
-    assetManager.clearCache();
-}
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour
+
+const toKey = (ref: string) => `assetManager:asset:${ref}`;
 
 export interface EnrichedAsset {
     ref: string;
@@ -59,18 +59,6 @@ function parseRef(ref: string) {
 }
 
 class AssetManager {
-    private cache: NodeCache;
-
-    constructor() {
-        this.cache = new NodeCache({
-            stdTTL: 60 * 60, // 1 hour
-        });
-    }
-
-    public clearCache() {
-        this.cache.flushAll();
-    }
-
     /**
      *
      * @param {string[]} [assetKinds]
@@ -113,9 +101,9 @@ class AssetManager {
         autoFetch: boolean = true
     ): Promise<EnrichedAsset | undefined> {
         ref = normalizeKapetaUri(ref);
-        const cacheKey = `getAsset:${ref}`;
-        if (!noCache && this.cache.has(cacheKey)) {
-            return this.cache.get(cacheKey);
+        const cacheKey = toKey(ref);
+        if (!noCache && cacheManager.has(cacheKey)) {
+            return cacheManager.get(cacheKey);
         }
         const uri = parseKapetaUri(ref);
         if (autoFetch) {
@@ -130,13 +118,17 @@ class AssetManager {
             throw new Error('Asset not found: ' + ref);
         }
         if (asset) {
-            this.cache.set(cacheKey, asset);
+            cacheManager.set(cacheKey, asset, CACHE_TTL);
         }
 
         return asset;
     }
 
-    async createAsset(path: string, yaml: BlockDefinition): Promise<EnrichedAsset[]> {
+    async createAsset(
+        path: string,
+        yaml: BlockDefinition,
+        sourceOfChange: SourceOfChange = 'filesystem'
+    ): Promise<EnrichedAsset[]> {
         if (await FS.pathExists(path)) {
             throw new Error('File already exists: ' + path);
         }
@@ -145,11 +137,17 @@ class AssetManager {
         if (!(await FS.pathExists(dirName))) {
             await FS.mkdirp(dirName);
         }
-
+        await repositoryManager.setSourceOfChangeFor(path, sourceOfChange);
         await FS.writeFile(path, YAML.stringify(yaml));
         const asset = await this.importFile(path);
+        asset.forEach((a) => {
+            const ref = normalizeKapetaUri(a.ref);
+            const key = toKey(ref);
+            cacheManager.set(key, a, CACHE_TTL);
+        });
 
-        clearAllCaches();
+        definitionsManager.clearCache();
+        console.log(`Created asset at: ${path}`);
 
         const ref = `kapeta://${yaml.metadata.name}:local`;
 
@@ -158,7 +156,8 @@ class AssetManager {
         return asset;
     }
 
-    async updateAsset(ref: string, yaml: BlockDefinition) {
+    async updateAsset(ref: string, yaml: BlockDefinition, sourceOfChange: SourceOfChange = 'filesystem') {
+        ref = normalizeKapetaUri(ref);
         const asset = await this.getAsset(ref, true, false);
         if (!asset) {
             throw new Error('Attempted to update unknown asset: ' + ref);
@@ -172,39 +171,14 @@ class AssetManager {
             throw new Error('Attempted to update corrupted asset: ' + ref);
         }
 
-        const path = asset.ymlPath;
-        try {
-            await repositoryManager.ignoreChangesFor(path);
-            await FS.writeFile(asset.ymlPath, YAML.stringify(yaml));
-            console.log('Wrote to ' + asset.ymlPath);
-            clearAllCaches();
-            this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
-        } finally {
-            //We need to wait a bit for the disk to settle before we can resume watching
-            setTimeout(async () => {
-                try {
-                    await repositoryManager.resumeChangedFor(path);
-                } catch (e) {}
-            }, 500);
-        }
-    }
+        await repositoryManager.setSourceOfChangeFor(asset.ymlPath, sourceOfChange);
+        await FS.writeFile(asset.ymlPath, YAML.stringify(yaml));
+        console.log(`Updated asset at: ${asset.ymlPath}`);
 
-    private maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
-        ref = normalizeKapetaUri(ref);
-        if (codeGeneratorManager.canGenerateCode(block)) {
-            const assetTitle = block.metadata.title ? block.metadata.title : parseKapetaUri(block.metadata.name).name;
-            taskManager.add(
-                `codegen:${ref}`,
-                async () => {
-                    await codeGeneratorManager.generate(ymlPath, block);
-                },
-                {
-                    name: `Generating code for ${assetTitle}`,
-                }
-            );
-            return true;
-        }
-        return false;
+        cacheManager.remove(toKey(ref));
+        definitionsManager.clearCache();
+
+        this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
     }
 
     async importFile(filePath: string) {
@@ -222,9 +196,15 @@ class AssetManager {
         await Actions.link(new ProgressListener(), Path.dirname(filePath));
 
         const version = 'local';
-        const refs = assetInfos.map((assetInfo) => `kapeta://${assetInfo.metadata.name}:${version}`);
+        const refs = assetInfos.map((assetInfo) =>
+            normalizeKapetaUri(`kapeta://${assetInfo.metadata.name}:${version}`)
+        );
+        refs.forEach((ref) => {
+            const key = toKey(ref);
+            cacheManager.remove(key);
+        });
 
-        clearAllCaches();
+        definitionsManager.clearCache();
 
         return this.getAssets().filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
     }
@@ -235,7 +215,9 @@ class AssetManager {
             throw new Error('Asset does not exists: ' + ref);
         }
 
-        clearAllCaches();
+        const key = toKey(ref);
+        cacheManager.remove(key);
+        definitionsManager.clearCache();
 
         await Actions.uninstall(new ProgressListener(), [asset.ref]);
     }
@@ -247,9 +229,30 @@ class AssetManager {
         }
         const uri = parseKapetaUri(ref);
         console.log('Installing %s', ref);
+        const key = toKey(ref);
+        cacheManager.remove(key);
+        definitionsManager.clearCache();
 
         return await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, false);
     }
+
+    private maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
+        ref = normalizeKapetaUri(ref);
+        if (codeGeneratorManager.canGenerateCode(block)) {
+            const assetTitle = block.metadata.title ? block.metadata.title : parseKapetaUri(block.metadata.name).name;
+            taskManager.add(
+                `codegen:${ref}`,
+                async () => {
+                    await codeGeneratorManager.generate(ymlPath, block);
+                },
+                {
+                    name: `Generating code for ${assetTitle}`,
+                }
+            );
+            return true;
+        }
+        return false;
+    }
 }
 
 export const assetManager = new AssetManager();

package/src/assets/routes.ts

@@ -70,7 +70,7 @@ router.post('/create', async (req: Request, res: Response) => {
     const content = parseBody(req);
 
     try {
-        const assets = await assetManager.createAsset(req.query.path as string, content);
+        const assets = await assetManager.createAsset(req.query.path as string, content, 'user');
 
         res.status(200).send(assets);
     } catch (err: any) {
@@ -91,7 +91,7 @@ router.put('/update', async (req: Request, res: Response) => {
     const content = parseBody(req);
 
     try {
-        await assetManager.updateAsset(req.query.ref as string, content);
+        await assetManager.updateAsset(req.query.ref as string, content, 'user');
 
         res.sendStatus(204);
     } catch (err: any) {

package/src/cacheManager.ts

@@ -0,0 +1,54 @@
+import NodeCache from 'node-cache';
+
+const DEFAULT_CACHE_TTL = 60 * 1000; // 1 min
+export interface CacheEntry<T = any> {
+    expires: number;
+    data: T;
+}
+export class CacheManager {
+    private cache: NodeCache = new NodeCache();
+
+    public flush() {
+        this.cache.flushAll();
+    }
+
+    public doCached<T>(key: string, getter: () => T, ttl = DEFAULT_CACHE_TTL) {
+        const data = this.cache.get<T>(key);
+        if (data !== undefined) {
+            return data;
+        }
+        const result = getter();
+        this.cache.set(key, result, ttl);
+        return result;
+    }
+
+    public get<T>(key: string): T | undefined {
+        return this.cache.get<T>(key);
+    }
+
+    public set<T>(key: string, data: T, ttl = DEFAULT_CACHE_TTL) {
+        this.cache.set(key, data, ttl);
+    }
+
+    public has(key: string): boolean {
+        return this.cache.has(key);
+    }
+
+    public remove(key: string) {
+        return this.cache.del(key);
+    }
+
+    public removePrefix(key: string) {
+        const keys = this.cache.keys();
+        for (const k of keys) {
+            if (k.startsWith(key)) {
+                this.remove(k);
+            }
+        }
+    }
+}
+
+export const cacheManager = new CacheManager();
+
+export const doCached = <T>(key: string, getter: () => T, ttl = DEFAULT_CACHE_TTL) =>
+    cacheManager.doCached(key, getter, ttl);

package/src/definitionsManager.ts

@@ -1,17 +1,9 @@
 import ClusterConfiguration, { DefinitionInfo } from '@kapeta/local-cluster-config';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
-
-const CACHE_TTL = 60 * 1000; // 1 min
-
-interface DefinitionCacheEntry {
-    expires: number;
-    definitions: DefinitionInfo[];
-}
+import { cacheManager, doCached } from './cacheManager';
 
 class DefinitionsManager {
-    private cache: { [key: string]: DefinitionCacheEntry } = {};
-
-    private getKey(kindFilter?: string | string[]) {
+    private getHash(kindFilter?: string | string[]) {
         if (kindFilter) {
             if (Array.isArray(kindFilter)) {
                 return kindFilter.join(',');
@@ -21,38 +13,22 @@ class DefinitionsManager {
         return 'none';
     }
 
-    public clearCache() {
-        this.cache = {};
-    }
-
-    private doCached(key: string, getter: () => DefinitionInfo[]) {
-        if (this.cache[key]) {
-            if (this.cache[key].expires > Date.now()) {
-                return this.cache[key].definitions;
-            }
-            delete this.cache[key];
-        }
-
-        this.cache[key] = {
-            expires: Date.now() + CACHE_TTL,
-            definitions: getter(),
-        };
-
-        return this.cache[key].definitions;
+    private getFullKey(kindFilter?: string | string[]) {
+        return `definitionsManager:${this.getHash(kindFilter)}`;
     }
 
-    public getDefinitions(kindFilter?: string | string[]) {
-        const key = this.getKey(kindFilter);
+    public getDefinitions(kindFilter?: string | string[]): DefinitionInfo[] {
+        const key = this.getFullKey(kindFilter);
 
-        return this.doCached(key, () => ClusterConfiguration.getDefinitions(kindFilter));
+        return doCached<DefinitionInfo[]>(key, () => ClusterConfiguration.getDefinitions(kindFilter));
     }
 
     public exists(ref: string) {
         return !!this.getDefinition(ref);
     }
 
-    public getProviderDefinitions() {
-        return this.doCached('providers', () => ClusterConfiguration.getProviderDefinitions());
+    public getProviderDefinitions(): DefinitionInfo[] {
+        return doCached<DefinitionInfo[]>('providers', () => ClusterConfiguration.getProviderDefinitions());
     }
 
     public getDefinition(ref: string) {
@@ -64,6 +40,10 @@ class DefinitionsManager {
             return parseKapetaUri(`${d.definition.metadata.name}:${d.version}`).id === uri.id;
         });
     }
+
+    public clearCache() {
+        cacheManager.removePrefix('definitionsManager:');
+    }
 }
 
 export const definitionsManager = new DefinitionsManager();

package/src/repositoryManager.ts

@@ -5,15 +5,10 @@ import { Actions, Config, RegistryService } from '@kapeta/nodejs-registry-utils'
 import { definitionsManager } from './definitionsManager';
 import { Task, taskManager } from './taskManager';
 import { normalizeKapetaUri } from './utils/utils';
-
 import { ProgressListener } from './progressListener';
 import { RepositoryWatcher } from './RepositoryWatcher';
-import { assetManager } from './assetManager';
-
-function clearAllCaches() {
-    definitionsManager.clearCache();
-    assetManager.clearCache();
-}
+import { SourceOfChange } from './types';
+import { cacheManager } from './cacheManager';
 
 const EVENT_DEFAULT_PROVIDERS_START = 'default-providers-start';
 const EVENT_DEFAULT_PROVIDERS_END = 'default-providers-end';
@@ -55,12 +50,16 @@ class RepositoryManager {
         return this.watcher.unwatch();
     }
 
-    ignoreChangesFor(file: string) {
-        return this.watcher.ignoreChangesFor(file);
+    /**
+     * Setting the source of change helps us know
+     * how to react to changes in the UI.
+     */
+    setSourceOfChangeFor(file: string, source: SourceOfChange) {
+        return this.watcher.setSourceOfChangeFor(file, source);
     }
 
-    resumeChangedFor(file: string) {
-        return this.watcher.resumeChangedFor(file);
+    clearSourceOfChangeFor(file: string) {
+        return this.watcher.clearSourceOfChangeFor(file);
     }
 
     public ensureDefaultProviders(): void {
@@ -93,7 +92,7 @@ class RepositoryManager {
                     console.error(`Failed to install asset: ${ref}`, e);
                     throw e;
                 }
-                clearAllCaches();
+                cacheManager.flush();
                 //console.log(`Asset installed: ${ref}`);
             };
         };

package/src/types.ts

@@ -1,4 +1,3 @@
-import EventEmitter from 'events';
 import express from 'express';
 import { Resource } from '@kapeta/schemas';
 import { StringBodyRequest } from './middleware/stringBody';
@@ -6,7 +5,8 @@ import { KapetaRequest } from './middleware/kapeta';
 
 export type StringMap = { [key: string]: string };
 export type AnyMap = { [key: string]: any };
-
+export type SourceOfChange = 'user' | 'filesystem';
+export type WatchEventName = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir';
 export type LogLevel = 'ERROR' | 'WARN' | 'INFO' | 'DEBUG' | 'TRACE' | 'FATAL';
 export type LogSource = 'stdout' | 'stderr';
 export type EnvironmentType = 'docker' | 'process';

package/src/utils/utils.ts

@@ -26,7 +26,7 @@ export function readYML(path: string) {
     try {
         return YAML.parse(rawYaml.toString());
     } catch (err) {
-        throw new Error('Failed to parse plan YAML: ' + err);
+        throw new Error(`Failed to parse plan YAML: ${err}`);
     }
 }