@kapeta/local-cluster-service 0.15.3 → 0.16.1

package/src/assetManager.ts

@@ -1,8 +1,6 @@
 import Path from 'node:path';
-import FS from 'node:fs';
-import FSExtra from 'fs-extra';
+import FS from 'fs-extra';
 import YAML from 'yaml';
-import NodeCache from 'node-cache';
 import { Definition, DefinitionInfo } from '@kapeta/local-cluster-config';
 import { codeGeneratorManager } from './codeGeneratorManager';
 import { ProgressListener } from './progressListener';
@@ -13,6 +11,10 @@ import { Actions } from '@kapeta/nodejs-registry-utils';
 import { definitionsManager } from './definitionsManager';
 import { normalizeKapetaUri } from './utils/utils';
 import { taskManager } from './taskManager';
+import { SourceOfChange } from './types';
+import { cacheManager } from './cacheManager';
+
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour
 
 export interface EnrichedAsset {
     ref: string;
@@ -55,18 +57,6 @@ function parseRef(ref: string) {
 }
 
 class AssetManager {
-    private cache: NodeCache;
-
-    constructor() {
-        this.cache = new NodeCache({
-            stdTTL: 60 * 60, // 1 hour
-        });
-    }
-
-    public clearCache() {
-        this.cache.flushAll();
-    }
-
     /**
      *
      * @param {string[]} [assetKinds]
@@ -110,8 +100,8 @@ class AssetManager {
     ): Promise<EnrichedAsset | undefined> {
         ref = normalizeKapetaUri(ref);
         const cacheKey = `getAsset:${ref}`;
-        if (!noCache && this.cache.has(cacheKey)) {
-            return this.cache.get(cacheKey);
+        if (!noCache && cacheManager.has(cacheKey)) {
+            return cacheManager.get(cacheKey);
         }
         const uri = parseKapetaUri(ref);
         if (autoFetch) {
@@ -126,30 +116,30 @@ class AssetManager {
             throw new Error('Asset not found: ' + ref);
         }
         if (asset) {
-            this.cache.set(cacheKey, asset);
+            cacheManager.set(cacheKey, asset, CACHE_TTL);
         }
 
         return asset;
     }
 
-    async createAsset(path: string, yaml: BlockDefinition): Promise<EnrichedAsset[]> {
-        if (FS.existsSync(path)) {
+    async createAsset(
+        path: string,
+        yaml: BlockDefinition,
+        sourceOfChange: SourceOfChange = 'filesystem'
+    ): Promise<EnrichedAsset[]> {
+        if (await FS.pathExists(path)) {
             throw new Error('File already exists: ' + path);
         }
 
         const dirName = Path.dirname(path);
-        if (!FS.existsSync(dirName)) {
-            FSExtra.mkdirpSync(dirName);
+        if (!(await FS.pathExists(dirName))) {
+            await FS.mkdirp(dirName);
         }
-
-        console.log('Wrote to ' + path);
-        FS.writeFileSync(path, YAML.stringify(yaml));
-
+        await repositoryManager.setSourceOfChangeFor(path, sourceOfChange);
+        await FS.writeFile(path, YAML.stringify(yaml));
         const asset = await this.importFile(path);
-        console.log('Imported');
 
-        this.cache.flushAll();
-        definitionsManager.clearCache();
+        cacheManager.flush();
 
         const ref = `kapeta://${yaml.metadata.name}:local`;
 
@@ -158,7 +148,7 @@ class AssetManager {
         return asset;
     }
 
-    async updateAsset(ref: string, yaml: BlockDefinition) {
+    async updateAsset(ref: string, yaml: BlockDefinition, sourceOfChange: SourceOfChange = 'filesystem') {
         const asset = await this.getAsset(ref, true, false);
         if (!asset) {
             throw new Error('Attempted to update unknown asset: ' + ref);
@@ -172,30 +162,13 @@ class AssetManager {
             throw new Error('Attempted to update corrupted asset: ' + ref);
         }
 
+        await repositoryManager.setSourceOfChangeFor(asset.ymlPath, sourceOfChange);
+        await FS.writeFile(asset.ymlPath, YAML.stringify(yaml));
         console.log('Wrote to ' + asset.ymlPath);
-        FS.writeFileSync(asset.ymlPath, YAML.stringify(yaml));
-        this.cache.flushAll();
-        definitionsManager.clearCache();
 
-        this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
-    }
+        cacheManager.flush();
 
-    private maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
-        ref = normalizeKapetaUri(ref);
-        if (codeGeneratorManager.canGenerateCode(block)) {
-            const assetTitle = block.metadata.title ? block.metadata.title : parseKapetaUri(block.metadata.name).name;
-            taskManager.add(
-                `codegen:${ref}`,
-                async () => {
-                    await codeGeneratorManager.generate(ymlPath, block);
-                },
-                {
-                    name: `Generating code for ${assetTitle}`,
-                }
-            );
-            return true;
-        }
-        return false;
+        this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
     }
 
     async importFile(filePath: string) {
@@ -203,17 +176,20 @@ class AssetManager {
             filePath = filePath.substring('file://'.length);
         }
 
-        if (!FS.existsSync(filePath)) {
+        if (!(await FS.pathExists(filePath))) {
             throw new Error('File not found: ' + filePath);
         }
+        const content = await FS.readFile(filePath);
 
-        const assetInfos = YAML.parseAllDocuments(FS.readFileSync(filePath).toString()).map((doc) => doc.toJSON());
+        const assetInfos = YAML.parseAllDocuments(content.toString()).map((doc) => doc.toJSON());
 
         await Actions.link(new ProgressListener(), Path.dirname(filePath));
 
         const version = 'local';
         const refs = assetInfos.map((assetInfo) => `kapeta://${assetInfo.metadata.name}:${version}`);
-        this.cache.flushAll();
+
+        cacheManager.flush();
+
         return this.getAssets().filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
     }
 
@@ -222,7 +198,9 @@ class AssetManager {
         if (!asset) {
             throw new Error('Asset does not exists: ' + ref);
         }
-        this.cache.flushAll();
+
+        cacheManager.flush();
+
         await Actions.uninstall(new ProgressListener(), [asset.ref]);
     }
 
@@ -236,6 +214,24 @@ class AssetManager {
 
         return await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, false);
     }
+
+    private maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
+        ref = normalizeKapetaUri(ref);
+        if (codeGeneratorManager.canGenerateCode(block)) {
+            const assetTitle = block.metadata.title ? block.metadata.title : parseKapetaUri(block.metadata.name).name;
+            taskManager.add(
+                `codegen:${ref}`,
+                async () => {
+                    await codeGeneratorManager.generate(ymlPath, block);
+                },
+                {
+                    name: `Generating code for ${assetTitle}`,
+                }
+            );
+            return true;
+        }
+        return false;
+    }
 }
 
 export const assetManager = new AssetManager();

package/src/assets/routes.ts

@@ -70,7 +70,7 @@ router.post('/create', async (req: Request, res: Response) => {
     const content = parseBody(req);
 
     try {
-        const assets = await assetManager.createAsset(req.query.path as string, content);
+        const assets = await assetManager.createAsset(req.query.path as string, content, 'user');
 
         res.status(200).send(assets);
     } catch (err: any) {
@@ -91,7 +91,7 @@ router.put('/update', async (req: Request, res: Response) => {
     const content = parseBody(req);
 
     try {
-        await assetManager.updateAsset(req.query.ref as string, content);
+        await assetManager.updateAsset(req.query.ref as string, content, 'user');
 
         res.sendStatus(204);
     } catch (err: any) {

package/src/cacheManager.ts

@@ -0,0 +1,45 @@
+import NodeCache from 'node-cache';
+
+const DEFAULT_CACHE_TTL = 60 * 1000; // 1 min
+export interface CacheEntry<T = any> {
+    expires: number;
+    data: T;
+}
+class CacheManager {
+    private cache: NodeCache = new NodeCache();
+
+    public flush() {
+        this.cache.flushAll();
+    }
+
+    public doCached<T>(key: string, getter: () => T, ttl = DEFAULT_CACHE_TTL) {
+        const data = this.cache.get<T>(key);
+        if (data !== undefined) {
+            return data;
+        }
+        const result = getter();
+        this.cache.set(key, result, ttl);
+        return result;
+    }
+
+    public get<T>(key: string): T | undefined {
+        return this.cache.get<T>(key);
+    }
+
+    public set<T>(key: string, data: T, ttl = DEFAULT_CACHE_TTL) {
+        this.cache.set(key, data, ttl);
+    }
+
+    public has(key: string): boolean {
+        return this.cache.has(key);
+    }
+
+    public remove(key: string) {
+        return this.cache.del(key);
+    }
+}
+
+export const cacheManager = new CacheManager();
+
+export const doCached = <T>(key: string, getter: () => T, ttl = DEFAULT_CACHE_TTL) =>
+    cacheManager.doCached(key, getter, ttl);

package/src/definitionsManager.ts

@@ -1,17 +1,9 @@
 import ClusterConfiguration, { DefinitionInfo } from '@kapeta/local-cluster-config';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
-
-const CACHE_TTL = 60 * 1000; // 1 min
-
-interface DefinitionCacheEntry {
-    expires: number;
-    definitions: DefinitionInfo[];
-}
+import { doCached } from './cacheManager';
 
 class DefinitionsManager {
-    private cache: { [key: string]: DefinitionCacheEntry } = {};
-
-    private getKey(kindFilter?: string | string[]) {
+    private getHash(kindFilter?: string | string[]) {
         if (kindFilter) {
             if (Array.isArray(kindFilter)) {
                 return kindFilter.join(',');
@@ -21,38 +13,22 @@ class DefinitionsManager {
         return 'none';
     }
 
-    public clearCache() {
-        this.cache = {};
-    }
-
-    private doCached(key: string, getter: () => DefinitionInfo[]) {
-        if (this.cache[key]) {
-            if (this.cache[key].expires > Date.now()) {
-                return this.cache[key].definitions;
-            }
-            delete this.cache[key];
-        }
-
-        this.cache[key] = {
-            expires: Date.now() + CACHE_TTL,
-            definitions: getter(),
-        };
-
-        return this.cache[key].definitions;
+    private getFullKey(kindFilter?: string | string[]) {
+        return `DefinitionsManager:${this.getHash(kindFilter)}`;
     }
 
-    public getDefinitions(kindFilter?: string | string[]) {
-        const key = this.getKey(kindFilter);
+    public getDefinitions(kindFilter?: string | string[]): DefinitionInfo[] {
+        const key = this.getFullKey(kindFilter);
 
-        return this.doCached(key, () => ClusterConfiguration.getDefinitions(kindFilter));
+        return doCached<DefinitionInfo[]>(key, () => ClusterConfiguration.getDefinitions(kindFilter));
     }
 
     public exists(ref: string) {
         return !!this.getDefinition(ref);
     }
 
-    public getProviderDefinitions() {
-        return this.doCached('providers', () => ClusterConfiguration.getProviderDefinitions());
+    public getProviderDefinitions(): DefinitionInfo[] {
+        return doCached<DefinitionInfo[]>('providers', () => ClusterConfiguration.getProviderDefinitions());
     }
 
     public getDefinition(ref: string) {

package/src/identities/routes.ts

@@ -6,7 +6,6 @@ import { Request, Response } from 'express';
 
 const router = Router();
 
-
 router.use('/', corsHandler);
 
 router.get('/current', async (req: Request, res: Response) => {

package/src/instanceManager.ts

@@ -74,8 +74,11 @@ export class InstanceManager {
         }
 
         const plan = planInfo.definition as Plan;
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
 
-        const instanceIds = plan.spec.blocks.map((block) => block.id);
+        const instanceIds = plan.spec?.blocks?.map((block) => block.id) || [];
 
         return this._instances.filter(
             (instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId)

package/src/repositoryManager.ts

@@ -1,18 +1,14 @@
-import FS from 'node:fs';
 import os from 'node:os';
-import Path from 'node:path';
-import watch from 'recursive-watch';
-import FSExtra from 'fs-extra';
-import ClusterConfiguration from '@kapeta/local-cluster-config';
-import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { socketManager } from './socketManager';
 import { Dependency } from '@kapeta/schemas';
 import { Actions, Config, RegistryService } from '@kapeta/nodejs-registry-utils';
 import { definitionsManager } from './definitionsManager';
 import { Task, taskManager } from './taskManager';
 import { normalizeKapetaUri } from './utils/utils';
-import { assetManager } from './assetManager';
 import { ProgressListener } from './progressListener';
+import { RepositoryWatcher } from './RepositoryWatcher';
+import { SourceOfChange } from './types';
+import { cacheManager } from './cacheManager';
 
 const EVENT_DEFAULT_PROVIDERS_START = 'default-providers-start';
 const EVENT_DEFAULT_PROVIDERS_END = 'default-providers-end';
@@ -35,101 +31,35 @@ const DEFAULT_PROVIDERS = [
 const INSTALL_ATTEMPTED: { [p: string]: boolean } = {};
 
 class RepositoryManager {
-    private changeEventsEnabled: boolean;
     private _registryService: RegistryService;
     private _cache: { [key: string]: boolean };
-    private watcher?: () => void;
+    private watcher: RepositoryWatcher;
 
     constructor() {
-        this.changeEventsEnabled = true;
-        this.listenForChanges();
         this._registryService = new RegistryService(Config.data.registry.url);
         this._cache = {};
-    }
-
-    setChangeEventsEnabled(enabled: boolean) {
-        this.changeEventsEnabled = enabled;
+        this.watcher = new RepositoryWatcher();
+        this.listenForChanges();
     }
 
     listenForChanges() {
-        const baseDir = ClusterConfiguration.getRepositoryBasedir();
-        if (!FS.existsSync(baseDir)) {
-            FSExtra.mkdirpSync(baseDir);
-        }
-
-        let allDefinitions = ClusterConfiguration.getDefinitions();
-
-        console.log('Watching local repository for provider changes: %s', baseDir);
-        try {
-            this.watcher = watch(baseDir, (filename: string) => {
-                if (!filename) {
-                    return;
-                }
-
-                const [handle, name, version] = filename.toString().split(/\//g);
-                if (!name || !version) {
-                    return;
-                }
-
-                if (!this.changeEventsEnabled) {
-                    return;
-                }
-
-                const ymlPath = Path.join(baseDir, handle, name, version, 'kapeta.yml');
-                const newDefinitions = ClusterConfiguration.getDefinitions();
-
-                const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
-                let currentDefinition = allDefinitions.find((d) => d.ymlPath === ymlPath);
-                const ymlExists = FS.existsSync(ymlPath);
-                let type;
-                if (ymlExists) {
-                    if (currentDefinition) {
-                        type = 'updated';
-                    } else if (newDefinition) {
-                        type = 'added';
-                        currentDefinition = newDefinition;
-                    } else {
-                        //Other definition was added / updated - ignore
-                        return;
-                    }
-                } else {
-                    if (currentDefinition) {
-                        const ref = parseKapetaUri(
-                            `${currentDefinition.definition.metadata.name}:${currentDefinition.version}`
-                        ).id;
-                        delete INSTALL_ATTEMPTED[ref];
-                        //Something was removed
-                        type = 'removed';
-                    } else {
-                        //Other definition was removed - ignore
-                        return;
-                    }
-                }
+        this.watcher.watch();
+    }
 
-                const payload = {
-                    type,
-                    definition: currentDefinition?.definition,
-                    asset: { handle, name, version },
-                };
+    async stopListening() {
+        return this.watcher.unwatch();
+    }
 
-                allDefinitions = newDefinitions;
-                socketManager.emit(`assets`, 'changed', payload);
-                definitionsManager.clearCache();
-            });
-        } catch (e) {
-            // Fallback to run without watch mode due to potential platform issues.
-            // https://nodejs.org/docs/latest/api/fs.html#caveats
-            console.log('Unable to watch for changes. Changes to assets will not update automatically.', e);
-            return;
-        }
+    /**
+     * Setting the source of change helps us know
+     * how to react to changes in the UI.
+     */
+    setSourceOfChangeFor(file: string, source: SourceOfChange) {
+        return this.watcher.setSourceOfChangeFor(file, source);
     }
 
-    stopListening() {
-        if (!this.watcher) {
-            return;
-        }
-        this.watcher();
-        this.watcher = undefined;
+    clearSourceOfChangeFor(file: string) {
+        return this.watcher.clearSourceOfChangeFor(file);
     }
 
     public ensureDefaultProviders(): void {
@@ -157,17 +87,12 @@ class RepositoryManager {
                 try {
                     //We change to a temp dir to avoid issues with the current working directory
                     process.chdir(os.tmpdir());
-                    //Disable change events while installing
-                    this.setChangeEventsEnabled(false);
                     await Actions.install(new ProgressListener(), [ref], {});
                 } catch (e) {
                     console.error(`Failed to install asset: ${ref}`, e);
                     throw e;
-                } finally {
-                    this.setChangeEventsEnabled(true);
                 }
-                definitionsManager.clearCache();
-                assetManager.clearCache();
+                cacheManager.flush();
                 //console.log(`Asset installed: ${ref}`);
             };
         };

package/src/types.ts

@@ -1,4 +1,3 @@
-import EventEmitter from 'events';
 import express from 'express';
 import { Resource } from '@kapeta/schemas';
 import { StringBodyRequest } from './middleware/stringBody';
@@ -6,7 +5,8 @@ import { KapetaRequest } from './middleware/kapeta';
 
 export type StringMap = { [key: string]: string };
 export type AnyMap = { [key: string]: any };
-
+export type SourceOfChange = 'user' | 'filesystem';
+export type WatchEventName = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir';
 export type LogLevel = 'ERROR' | 'WARN' | 'INFO' | 'DEBUG' | 'TRACE' | 'FATAL';
 export type LogSource = 'stdout' | 'stderr';
 export type EnvironmentType = 'docker' | 'process';

package/src/utils/utils.ts

@@ -26,7 +26,7 @@ export function readYML(path: string) {
     try {
         return YAML.parse(rawYaml.toString());
     } catch (err) {
-        throw new Error('Failed to parse plan YAML: ' + err);
+        throw new Error(`Failed to parse plan YAML: ${err}`);
     }
 }