@kapeta/local-cluster-service 0.16.8 → 0.18.0

package/src/repositoryManager.ts

@@ -58,19 +58,19 @@ class RepositoryManager {
         return this.watcher.clearSourceOfChangeFor(file);
     }
 
-    public ensureDefaultProviders(): void {
+    public async ensureDefaultProviders(): Promise<void> {
         socketManager.emitGlobal(EVENT_DEFAULT_PROVIDERS_START, { providers: DEFAULT_PROVIDERS });
-        const tasks = this._install(DEFAULT_PROVIDERS);
+        const tasks = await this.scheduleInstallation(DEFAULT_PROVIDERS);
         Promise.allSettled(tasks.map((t) => t.wait())).then(() => {
             socketManager.emitGlobal(EVENT_DEFAULT_PROVIDERS_END, {});
         });
     }
 
-    private _install(refs: string[]): Task[] {
+    private async scheduleInstallation(refs: string[]): Promise<Task[]> {
         //We make sure to only install one asset at a time - otherwise unexpected things might happen
         const createInstaller = (ref: string) => {
             return async () => {
-                if (definitionsManager.exists(ref)) {
+                if (await definitionsManager.exists(ref)) {
                     return;
                 }
                 //console.log(`Installing asset: ${ref}`);
@@ -97,7 +97,7 @@ class RepositoryManager {
             }
             ref = normalizeKapetaUri(ref);
 
-            if (definitionsManager.exists(ref)) {
+            if (await definitionsManager.exists(ref)) {
                 continue;
             }
 
@@ -126,10 +126,7 @@ class RepositoryManager {
             return;
         }
 
-        const definitions = definitionsManager.getDefinitions();
-        const installedAsset = definitions.find(
-            (d) => d.definition.metadata.name === fullName && d.version === version
-        );
+        const installedAsset = await definitionsManager.getDefinition(`${fullName}:${version}`);
 
         let assetVersion;
         try {
@@ -147,12 +144,12 @@ class RepositoryManager {
 
         let tasks: Task[] | undefined = undefined;
         if (!installedAsset) {
-            tasks = this._install([ref]);
+            tasks = await this.scheduleInstallation([ref]);
         } else {
             //Ensure dependencies are installed
             const refs = assetVersion.dependencies.map((dep: Dependency) => dep.name);
             if (refs.length > 0) {
-                tasks = this._install(refs);
+                tasks = await this.scheduleInstallation(refs);
             }
         }
 

package/src/socketManager.ts

@@ -37,10 +37,16 @@ export class SocketManager {
     }
 
     emit(context: string, type: string, payload: any) {
+        if (!this._io) {
+            return;
+        }
         this.io.to(context).emit(type, { context, payload });
     }
 
     emitGlobal(type: string, payload: any) {
+        if (!this._io) {
+            return;
+        }
         this.io.emit(type, payload);
     }
 

package/src/utils/BlockInstanceRunner.ts

@@ -25,8 +25,9 @@ const DOCKER_ENV_VARS = [
     `KAPETA_ENVIRONMENT_TYPE=docker`,
 ];
 
-function getProvider(uri: KapetaURI) {
-    return definitionsManager.getProviderDefinitions().find((provider) => {
+async function getProvider(uri: KapetaURI) {
+    const providers = await definitionsManager.getProviderDefinitions();
+    return providers.find((provider) => {
         const ref = `${provider.definition.metadata.name}:${provider.version}`;
         return parseKapetaUri(ref).id === uri.id;
     });
@@ -87,10 +88,7 @@ export class BlockInstanceRunner {
             blockUri.version = 'local';
         }
 
-        const assetVersion = definitionsManager.getDefinitions().find((definitions) => {
-            const ref = `${definitions.definition.metadata.name}:${definitions.version}`;
-            return parseKapetaUri(ref).id === blockUri.id;
-        });
+        const assetVersion = await definitionsManager.getDefinition(blockUri.id);
 
         if (!assetVersion) {
             throw new Error(`Block definition not found: ${blockUri.id}`);
@@ -98,7 +96,7 @@ export class BlockInstanceRunner {
 
         const kindUri = parseKapetaUri(assetVersion.definition.kind);
 
-        const providerVersion = getProvider(kindUri);
+        const providerVersion = await getProvider(kindUri);
 
         if (!providerVersion) {
             throw new Error(`Kind not found: ${kindUri.id}`);
@@ -150,7 +148,7 @@ export class BlockInstanceRunner {
 
         const kindUri = parseKapetaUri(assetVersion.definition.spec?.target?.kind);
 
-        const targetVersion = getProvider(kindUri);
+        const targetVersion = await getProvider(kindUri);
 
         if (!targetVersion) {
             throw new Error(`Target not found: ${kindUri.id}`);

package/src/utils/DefaultProviderInstaller.ts

@@ -0,0 +1,141 @@
+import Path from 'node:path';
+import OS from 'node:os';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import FS from 'fs-extra';
+import request from 'request';
+import { extract } from 'tar-stream';
+import gunzip from 'gunzip-maybe';
+import { filesystemManager } from '../filesystemManager';
+import { Actions } from '@kapeta/nodejs-registry-utils';
+import { ProgressListener } from '../progressListener';
+import { glob } from 'glob';
+
+const DEFAULT_PROVIDERS_URL = 'https://storage.googleapis.com/kapeta-production-cdn/archives/default-providers.tar.gz';
+const DEFAULT_PROJECT_HOME_DIR = 'KapetaProjects';
+
+const ARCHIVE_LOCAL_PREFIX = 'local';
+
+class DefaultProviderInstaller {
+    private readonly progressListener = new ProgressListener();
+
+    public async checkForDefault() {
+        const definitions = ClusterConfiguration.getDefinitions();
+        if (definitions.length < 1) {
+            console.log('Installing default providers');
+            try {
+                await this.install();
+            } catch (e) {
+                console.warn('Failed to install defaults', e);
+            }
+        }
+    }
+
+    private async install() {
+        await this.download();
+        await this.linkLocal();
+    }
+
+    private async linkLocal() {
+        const projectBase = await this.ensureDefaultProjectHome();
+        const folders = this.scanProjectBase(projectBase);
+        for (let folder of folders) {
+            console.log('Linking %s', folder);
+            await Actions.link(this.progressListener, folder);
+        }
+    }
+
+    private scanProjectBase(projectBase: string) {
+        const assetFiles = glob.sync('*/**/kapeta.yml', { cwd: projectBase });
+        return assetFiles.map((assetFile) => {
+            return Path.dirname(Path.join(projectBase, assetFile));
+        });
+    }
+
+    private async ensureDefaultProjectHome(): Promise<string> {
+        const defaultProjectHome = Path.join(OS.homedir(), DEFAULT_PROJECT_HOME_DIR);
+        let projectBase = filesystemManager.getProjectRootFolder();
+
+        if (!projectBase) {
+            filesystemManager.setProjectRootFolder(defaultProjectHome);
+            projectBase = defaultProjectHome;
+            if (!(await FS.pathExists(projectBase))) {
+                await FS.mkdirp(projectBase);
+            }
+        }
+        return projectBase;
+    }
+
+    private async download() {
+        const projectBase: string = await this.ensureDefaultProjectHome();
+        const repoBase: string = ClusterConfiguration.getRepositoryBasedir();
+
+        return new Promise<void>((resolve, reject) => {
+            const extractor = extract();
+            const dirCache = new Set<string>();
+            extractor.on('entry', async function (header, stream, next) {
+                if (header.type !== 'file') {
+                    stream.on('end', function () {
+                        next(); // ready for next entry
+                    });
+                    stream.resume(); // just auto drain the stream
+                    return;
+                }
+
+                // Local (editable) assets should be stored in the project folder
+                // - installed assets goes into the repository folder
+                const baseDir: string = header.name.startsWith(ARCHIVE_LOCAL_PREFIX) ? projectBase : repoBase;
+
+                const parts = header.name.split(/\//g);
+                parts.shift();
+                const filename = parts.join(Path.sep);
+
+                try {
+                    const dirname = Path.join(baseDir, Path.dirname(filename));
+                    if (!dirCache.has(dirname)) {
+                        let dirExists = false;
+                        try {
+                            await FS.stat(dirname);
+                            dirExists = true;
+                        } catch (e) {}
+                        if (!dirExists) {
+                            await FS.mkdirp(dirname);
+                        }
+                        dirCache.add(dirname);
+                    }
+                    const fileTarget = Path.join(baseDir, filename);
+                    stream.on('error', (err) => {
+                        reject(err);
+                    });
+                    stream.on('end', next);
+
+                    stream.pipe(
+                        FS.createWriteStream(fileTarget, {
+                            mode: header.mode,
+                        })
+                    );
+                } catch (e) {
+                    reject(e);
+                }
+            });
+
+            extractor.on('finish', function () {
+                // all entries done - lets finalize it
+                console.log('Default providers installed');
+                resolve();
+            });
+
+            extractor.on('error', function (err) {
+                reject(err);
+            });
+
+            console.log('Downloading default providers from %s', DEFAULT_PROVIDERS_URL);
+            const response = request(DEFAULT_PROVIDERS_URL);
+            response.on('error', function (err) {
+                reject(err);
+            });
+            response.pipe(gunzip()).pipe(extractor);
+        });
+    }
+}
+
+export const defaultProviderInstaller = new DefaultProviderInstaller();

package/tsconfig.json

@@ -1,9 +1,10 @@
 {
     "extends": "@tsconfig/node18/tsconfig.json",
     "compilerOptions": {
+        "module": "node16",
+        "moduleResolution": "node16",
         "outDir": "./dist",
-        "declaration": true,
-        "esModuleInterop": true
+        "declaration": true
     },
     "ts-node": {
         "files": true