@kapeta/local-cluster-service 0.0.0-96f91ef

package/src/utils/DefaultProviderInstaller.ts

@@ -0,0 +1,150 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Path from 'node:path';
+import OS from 'node:os';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import FS from 'fs-extra';
+import request from 'request';
+import { extract } from 'tar-stream';
+import gunzip from 'gunzip-maybe';
+import { filesystemManager } from '../filesystemManager';
+import { Actions } from '@kapeta/nodejs-registry-utils';
+import { ProgressListener } from '../progressListener';
+import { glob } from 'glob';
+
+const PROVIDERS_FILENAME =
+    ClusterConfiguration.getEnvironment() === 'staging'
+        ? 'default-providers-staging.tar.gz'
+        : 'default-providers.tar.gz';
+const DEFAULT_PROVIDERS_URL = `https://storage.googleapis.com/kapeta-production-cdn/archives/${PROVIDERS_FILENAME}`;
+const DEFAULT_PROJECT_HOME_DIR = 'KapetaProjects';
+
+const ARCHIVE_LOCAL_PREFIX = 'local';
+
+class DefaultProviderInstaller {
+    private readonly progressListener = new ProgressListener();
+
+    public async checkForDefault() {
+        const definitions = ClusterConfiguration.getDefinitions();
+        if (definitions.length < 1) {
+            console.log('Installing default providers');
+            try {
+                await this.install();
+            } catch (e) {
+                console.warn('Failed to install defaults', e);
+            }
+        }
+    }
+
+    private async install() {
+        await this.download();
+        await this.linkLocal();
+    }
+
+    private async linkLocal() {
+        const projectBase = await this.ensureDefaultProjectHome();
+        const folders = this.scanProjectBase(projectBase);
+        for (let folder of folders) {
+            console.log('Linking %s', folder);
+            await Actions.link(this.progressListener, folder);
+        }
+    }
+
+    private scanProjectBase(projectBase: string) {
+        const assetFiles = glob.sync('*/**/kapeta.yml', { cwd: projectBase });
+        return assetFiles.map((assetFile) => {
+            return Path.dirname(Path.join(projectBase, assetFile));
+        });
+    }
+
+    private async ensureDefaultProjectHome(): Promise<string> {
+        const defaultProjectHome = Path.join(OS.homedir(), DEFAULT_PROJECT_HOME_DIR);
+        let projectBase = filesystemManager.getProjectRootFolder();
+
+        if (!projectBase) {
+            filesystemManager.setProjectRootFolder(defaultProjectHome);
+            projectBase = defaultProjectHome;
+            if (!(await FS.pathExists(projectBase))) {
+                await FS.mkdirp(projectBase);
+            }
+        }
+        return projectBase;
+    }
+
+    private async download() {
+        const projectBase: string = await this.ensureDefaultProjectHome();
+        const repoBase: string = ClusterConfiguration.getRepositoryBasedir();
+
+        return new Promise<void>((resolve, reject) => {
+            const extractor = extract();
+            const dirCache = new Set<string>();
+            extractor.on('entry', async function (header, stream, next) {
+                if (header.type !== 'file') {
+                    stream.on('end', function () {
+                        next(); // ready for next entry
+                    });
+                    stream.resume(); // just auto drain the stream
+                    return;
+                }
+
+                // Local (editable) assets should be stored in the project folder
+                // - installed assets goes into the repository folder
+                const baseDir: string = header.name.startsWith(ARCHIVE_LOCAL_PREFIX) ? projectBase : repoBase;
+
+                const parts = header.name.split(/\//g);
+                parts.shift();
+                const filename = parts.join(Path.sep);
+
+                try {
+                    const dirname = Path.join(baseDir, Path.dirname(filename));
+                    if (!dirCache.has(dirname)) {
+                        let dirExists = false;
+                        try {
+                            await FS.stat(dirname);
+                            dirExists = true;
+                        } catch (e) {}
+                        if (!dirExists) {
+                            await FS.mkdirp(dirname);
+                        }
+                        dirCache.add(dirname);
+                    }
+                    const fileTarget = Path.join(baseDir, filename);
+                    stream.on('error', (err) => {
+                        reject(err);
+                    });
+                    stream.on('end', next);
+
+                    stream.pipe(
+                        FS.createWriteStream(fileTarget, {
+                            mode: header.mode,
+                        })
+                    );
+                } catch (e) {
+                    reject(e);
+                }
+            });
+
+            extractor.on('finish', function () {
+                // all entries done - lets finalize it
+                console.log('Default providers installed');
+                resolve();
+            });
+
+            extractor.on('error', function (err) {
+                reject(err);
+            });
+
+            console.log('Downloading default providers from %s', DEFAULT_PROVIDERS_URL);
+            const response = request(DEFAULT_PROVIDERS_URL);
+            response.on('error', function (err) {
+                reject(err);
+            });
+            response.pipe(gunzip()).pipe(extractor);
+        });
+    }
+}
+
+export const defaultProviderInstaller = new DefaultProviderInstaller();

package/src/utils/InternalConfigProvider.ts

@@ -0,0 +1,214 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+import {
+    BlockInstanceDetails,
+    ConfigProvider,
+    DefaultCredentials,
+    DefaultResourceOptions,
+    InstanceOperator,
+    ResourceInfo,
+} from '@kapeta/sdk-config';
+import { Definition, DefinitionInfo } from '@kapeta/local-cluster-config';
+import { normalizeKapetaUri } from '@kapeta/nodejs-utils';
+import { BlockDefinition, Plan } from '@kapeta/schemas';
+import { configManager } from '../configManager';
+import { AnyMap, EnvironmentType } from '../types';
+import _ from 'lodash';
+import { serviceManager } from '../serviceManager';
+import { operatorManager } from '../operatorManager';
+import { instanceManager } from '../instanceManager';
+import { definitionsManager } from '../definitionsManager';
+import { getBindAddressForEnvironment } from './utils';
+
+/**
+ * A configuration provider that does the same as the LocalConfigProvider
+ * but without calling the API of the local cluster service (since it's running in the same process)
+ */
+export class InternalConfigProvider implements ConfigProvider {
+    private readonly info: DefinitionInfo;
+    private readonly systemId: string;
+    private readonly instanceId: string;
+    private readonly config: AnyMap;
+    private readonly environment: EnvironmentType;
+
+    constructor(
+        systemId: string,
+        instanceId: string,
+        info: DefinitionInfo,
+        config: AnyMap,
+        environment: EnvironmentType = 'docker'
+    ) {
+        this.info = info;
+        this.systemId = normalizeKapetaUri(systemId);
+        this.instanceId = instanceId;
+        this.config = config;
+        this.environment = environment;
+    }
+
+    getBlockDefinition() {
+        return this.info.definition;
+    }
+    getBlockReference(): string {
+        return normalizeKapetaUri(this.info.definition.metadata.name + ':' + this.info.version);
+    }
+    getSystemId(): string {
+        return this.systemId;
+    }
+    getInstanceId(): string {
+        return this.instanceId;
+    }
+    getServerPort(portType?: string | undefined): Promise<string> {
+        return serviceManager.ensureServicePort(this.systemId, this.instanceId, portType);
+    }
+    async getServiceAddress(serviceName: string, portType: string): Promise<string | null> {
+        return serviceManager.getConsumerAddress(
+            this.systemId,
+            this.instanceId,
+            serviceName,
+            portType,
+            this.environment
+        );
+    }
+    getResourceInfo<Options = DefaultResourceOptions, Credentials = DefaultCredentials>(
+        resourceType: string,
+        portType: string,
+        resourceName: string
+    ): Promise<ResourceInfo<Options, Credentials> | null> {
+        return operatorManager.getConsumerResourceInfo(
+            this.systemId,
+            this.instanceId,
+            resourceType,
+            portType,
+            resourceName,
+            this.environment,
+            false
+        );
+    }
+    async getInstanceHost(instanceId: string): Promise<string | null> {
+        const instance = instanceManager.getInstance(this.systemId, instanceId);
+        return instance?.address ?? null;
+    }
+    async getServerHost(): Promise<string> {
+        return getBindAddressForEnvironment(this.environment);
+    }
+    getProviderId(): string {
+        return 'internal';
+    }
+    getOrDefault<T = any>(path: string, defaultValue: T): T {
+        return this.get(path) ?? defaultValue;
+    }
+    get<T = any>(path: string): T | undefined {
+        return _.get(this.config, path);
+    }
+
+    getInstanceOperator<Options = any, Credentials extends DefaultCredentials = DefaultCredentials>(
+        instanceId: string
+    ): Promise<InstanceOperator<Options, Credentials> | null> {
+        return instanceManager.getInstanceOperator(this.systemId, instanceId, this.environment, false);
+    }
+
+    public async getInstanceForConsumer<BlockType = BlockDefinition>(
+        resourceName: string
+    ): Promise<BlockInstanceDetails<BlockType> | null> {
+        const plan = await this.getPlan();
+        if (!plan) {
+            throw new Error('Could not find plan');
+        }
+        const instanceId = this.getInstanceId();
+        const connection = plan.spec.connections.find(
+            (connection) =>
+                connection.consumer.blockId === instanceId && connection.consumer.resourceName === resourceName
+        );
+
+        if (!connection) {
+            throw new Error(`Could not find connection for consumer ${resourceName}`);
+        }
+
+        const instance = plan.spec.blocks.find((b) => b.id === connection.provider.blockId);
+
+        if (!instance) {
+            throw new Error(`Could not find instance ${connection.provider.blockId} in plan`);
+        }
+
+        const block = await this.getBlock(instance.block.ref);
+
+        if (!block) {
+            throw new Error(`Could not find block ${instance.block.ref} in plan`);
+        }
+
+        return {
+            instanceId: connection.provider.blockId,
+            connections: [connection],
+            block: block as BlockType,
+        };
+    }
+
+    public async getInstancesForProvider<BlockType = BlockDefinition>(
+        resourceName: string
+    ): Promise<BlockInstanceDetails<BlockType>[]> {
+        const plan = await this.getPlan();
+        if (!plan) {
+            throw new Error('Could not find plan');
+        }
+        const instanceId = this.getInstanceId();
+
+        const blockDetails: { [key: string]: BlockInstanceDetails<BlockType> } = {};
+        const connections = plan.spec.connections.filter(
+            (connection) =>
+                connection.provider.blockId === instanceId && connection.provider.resourceName === resourceName
+        );
+
+        for (const connection of connections) {
+            const blockInstanceId = connection.consumer.blockId;
+            if (blockDetails[blockInstanceId]) {
+                blockDetails[blockInstanceId].connections.push(connection);
+                continue;
+            }
+
+            const instance = plan.spec.blocks.find((b) => b.id === blockInstanceId);
+            if (!instance) {
+                throw new Error(`Could not find instance ${blockInstanceId} in plan`);
+            }
+
+            const block = await this.getBlock(instance.block.ref);
+            if (!block) {
+                throw new Error(`Could not find block ${instance.block.ref} in plan`);
+            }
+
+            blockDetails[blockInstanceId] = {
+                instanceId: blockInstanceId,
+                connections: [connection],
+                block: block as BlockType,
+            };
+        }
+
+        return Object.values(blockDetails);
+    }
+
+    async getBlock(ref: any): Promise<Definition> {
+        const definition = await definitionsManager.getDefinition(ref);
+        if (!definition) {
+            throw new Error(`Could not find definition for ${ref}`);
+        }
+        return definition.definition;
+    }
+
+    async getPlan(): Promise<Plan> {
+        const definition = await definitionsManager.getDefinition(this.systemId);
+        if (!definition) {
+            throw new Error(`Could not find plan ${this.systemId}`);
+        }
+        return definition.definition as Plan;
+    }
+}
+
+export async function createInternalConfigProvider(
+    systemId: string,
+    instanceId: string,
+    info: DefinitionInfo
+): Promise<InternalConfigProvider> {
+    const config = await configManager.getConfigForBlockInstance(systemId, instanceId);
+    return new InternalConfigProvider(systemId, instanceId, info, config);
+}

package/src/utils/LogData.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { LogEntry, LogLevel, LogSource } from '../types';
+
+const MAX_LINES = 1000;
+export class LogData {
+    public static readonly MAX_LINES = MAX_LINES;
+    private readonly entries: LogEntry[] = [];
+    constructor() {}
+
+    /**
+     *
+     * @param {string} msg
+     * @param {string} [level]
+     * @param {string} [source]
+     */
+    addLog(msg: string, level: LogLevel = 'INFO', source: LogSource = 'stdout') {
+        while (this.entries.length > MAX_LINES) {
+            this.entries.shift();
+        }
+
+        if (!msg.endsWith('\n')) {
+            msg += '\n';
+        }
+
+        this.entries.push({
+            time: Date.now(),
+            message: msg,
+            level,
+            source,
+        });
+    }
+
+    /**
+     *
+     * @return {LogEntry[]}
+     */
+    getLogs() {
+        return this.entries;
+    }
+
+    toString() {
+        return this.getLogs()
+            .map((entry) => entry.message)
+            .join('\n');
+    }
+}

package/src/utils/commandLineUtils.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { spawn, hasApp } from '@kapeta/nodejs-process';
+import { Task, taskManager } from '../taskManager';
+
+export async function hasCLI() {
+    return hasApp('kap');
+}
+
+export async function ensureCLI() {
+    if (await hasCLI()) {
+        return null;
+    }
+
+    return taskManager.add(
+        `cli:install`,
+        (task: Task) => {
+            const process = spawn('npm', ['install', '-g', '@kapeta/kap'], {
+                shell: true,
+            });
+
+            process.process.stdout?.on('data', (data: any) => {
+                task.addLog(data.toString(), 'INFO');
+            });
+
+            process.process.stderr?.on('data', (data: any) => {
+                task.addLog(data.toString(), 'ERROR');
+            });
+
+            return process.wait();
+        },
+        {
+            name: `Installing Kapeta CLI`,
+        }
+    );
+}
+
+export function ensureCLICommands() {
+    console.log('Run `kap init` to ensure default commands are installed');
+    const process = spawn('kap', ['init'], { shell: true });
+    return process.wait();
+}

package/src/utils/pathTemplateParser.ts

@@ -0,0 +1,157 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { StringMap } from '../types';
+
+const TYPE_VARIABLE = 'variable';
+const TYPE_PATH = 'path';
+
+function normalizePath(path: string) {
+    if (!path.startsWith('/')) {
+        path = '/' + path;
+    }
+
+    if (path.includes('#')) {
+        path = path.split('#')[0];
+    }
+
+    if (path.includes('?')) {
+        path = path.split('?')[0];
+    }
+
+    return path;
+}
+
+/**
+ * A path template is a string that can be used to match a path and extract variables from it.
+ *
+ * E.g. /foo/{bar}/baz
+ *
+ * Would match /foo/123/baz and extract bar=123
+ *
+ * You can also specify a regex for the variable:
+ *  /foo/{bar:[0-9]+}/baz
+ *
+ */
+export class PathTemplate {
+    private _path: string;
+    private _parts: { type: 'variable' | 'path'; value: string; regex?: RegExp }[] = [];
+
+    constructor(pathTemplate: string) {
+        if (!pathTemplate.startsWith('/')) {
+            pathTemplate = '/' + pathTemplate;
+        }
+        this._path = pathTemplate;
+
+        const variableRegex = /{([^}]+)}/g;
+        let match,
+            offset = 0;
+        this._parts = [];
+        while ((match = variableRegex.exec(pathTemplate)) !== null) {
+            if (match.index > offset) {
+                this._parts.push({
+                    type: TYPE_PATH,
+                    value: pathTemplate.substring(offset, match.index),
+                });
+            }
+
+            let regex;
+            let value = match[1];
+            [value, regex] = value.split(/:/, 2);
+
+            if (regex) {
+                regex = new RegExp('^' + regex);
+            } else {
+                regex = /^[^\/]+/;
+            }
+
+            this._parts.push({
+                type: TYPE_VARIABLE,
+                value,
+                regex,
+            });
+            offset = match.index + match[0].length;
+        }
+
+        if (offset < pathTemplate.length) {
+            this._parts.push({
+                type: TYPE_PATH,
+                value: pathTemplate.substring(offset),
+            });
+        }
+    }
+
+    get path() {
+        return this._path;
+    }
+
+    matches(path: string) {
+        return this.parse(path) !== null;
+    }
+
+    parse(path: string) {
+        path = normalizePath(path);
+
+        const values: StringMap = {};
+
+        for (let i = 0; i < this._parts.length; i++) {
+            const part = this._parts[i];
+            switch (part.type) {
+                case TYPE_PATH:
+                    if (!path.startsWith(part.value)) {
+                        return null;
+                    }
+
+                    path = path.substring(part.value.length);
+                    break;
+                case TYPE_VARIABLE:
+                    if (!part.regex?.test(path)) {
+                        return null;
+                    }
+
+                    const newPath = path.replace(part.regex, '');
+                    const value = path.substr(0, path.length - newPath.length);
+                    values[part.value] = value;
+                    path = newPath;
+                    break;
+            }
+        }
+
+        if (path && path !== '/') {
+            //We did not match all of it
+            return null;
+        }
+
+        return values;
+    }
+
+    create(variables: StringMap) {
+        return this._parts
+            .map((part) => {
+                switch (part.type) {
+                    case TYPE_PATH:
+                        return part.value;
+                    case TYPE_VARIABLE:
+                        if (variables[part.value] === undefined || variables[part.value] === null) {
+                            return '';
+                        }
+
+                        return variables[part.value];
+                }
+            })
+            .join('');
+    }
+
+    toString() {
+        return 'tmpl: ' + this.path;
+    }
+}
+
+/**
+ * Parses a path into a RESTPath
+ */
+export function pathTemplateParser(path: string) {
+    return new PathTemplate(path);
+}