@kapeta/local-cluster-service 0.0.0-96f91ef

package/src/serviceManager.ts

@@ -0,0 +1,133 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import _ from 'lodash';
+import { clusterService } from './clusterService';
+import { storageService } from './storageService';
+import { DOCKER_HOST_INTERNAL, EnvironmentType } from './types';
+import { normalizeKapetaUri } from '@kapeta/nodejs-utils';
+import { resolvePortType } from './utils/BlockInstanceRunner';
+import { getRemoteHostForEnvironment } from './utils/utils';
+
+export const HTTP_PORT_TYPE = 'http';
+
+export const DEFAULT_PORT_TYPE = HTTP_PORT_TYPE;
+
+export const HTTP_PORTS = [HTTP_PORT_TYPE, 'web', 'rest'];
+
+class ServiceManager {
+    private _systems: any;
+
+    constructor() {
+        this._systems = storageService.get('services');
+        if (!this._systems) {
+            this._systems = {};
+        }
+
+        _.forEach(this._systems, (system) => {
+            _.forEach(system, (services) => {
+                _.forEach(services, (portInfo) => {
+                    clusterService.reservePort(portInfo.port);
+                });
+            });
+        });
+    }
+
+    _forLocal(port: string | number, path?: string, environmentType?: EnvironmentType) {
+        if (!path) {
+            path = '';
+        }
+        const hostname = getRemoteHostForEnvironment(environmentType);
+
+        if (path.startsWith('/')) {
+            path = path.substring(1);
+        }
+        return `http://${hostname}:${port}/${path}`;
+    }
+
+    _ensureSystem(systemId: string) {
+        systemId = normalizeKapetaUri(systemId);
+
+        if (!this._systems[systemId]) {
+            this._systems[systemId] = {};
+        }
+
+        return this._systems[systemId];
+    }
+
+    _ensureService(systemId: string, serviceId: string) {
+        const system = this._ensureSystem(systemId);
+
+        if (!system[serviceId]) {
+            system[serviceId] = {};
+        }
+
+        return system[serviceId];
+    }
+
+    async ensureServicePort(systemId: string, blockInstanceId: string, portType: string = DEFAULT_PORT_TYPE) {
+        systemId = normalizeKapetaUri(systemId);
+        if (!portType) {
+            portType = DEFAULT_PORT_TYPE;
+        }
+
+        portType = resolvePortType(portType);
+
+        const service = this._ensureService(systemId, blockInstanceId);
+
+        if (!service[portType]) {
+            const port = await clusterService.getNextAvailablePort();
+            service[portType] = { port };
+            this._save();
+        }
+
+        const portTypeSection = service[portType];
+
+        return portTypeSection.port;
+    }
+
+    _save() {
+        storageService.put('services', this._systems);
+    }
+
+    /**
+     * Gets the consumable address of a service block resource
+     *
+     * This returns a local proxy path to allow traffic inspection and control.
+     *
+     */
+    getConsumerAddress(
+        systemId: string,
+        consumerInstanceId: string,
+        consumerResourceName: string,
+        portType: string,
+        environmentType?: EnvironmentType
+    ): string {
+        systemId = normalizeKapetaUri(systemId);
+        const port = clusterService.getClusterServicePort();
+        const path = clusterService.getProxyPath(systemId, consumerInstanceId, consumerResourceName, portType);
+        return this._forLocal(port, path, environmentType);
+    }
+
+    /**
+     * Gets the direct address of a service block
+     *
+     * This returns the actual endpoint address of a service that we're talking to.
+     * For local services this address will be on localhost - for remote services it will
+     * be their remotely available address.
+     *
+     */
+    async getProviderAddress(systemId: string, providerInstanceId: string, portType: string): Promise<string> {
+        systemId = normalizeKapetaUri(systemId);
+        const port = await this.ensureServicePort(systemId, providerInstanceId, portType);
+        return this._forLocal(port);
+    }
+
+    getServices() {
+        return this._systems;
+    }
+}
+
+export const serviceManager = new ServiceManager();

package/src/socketManager.ts

@@ -0,0 +1,138 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import _ from 'lodash';
+import { Socket, Server } from 'socket.io';
+import { normalizeKapetaUri } from '@kapeta/nodejs-utils';
+import { LogEntry } from './types';
+import { containerManager } from './containerManager';
+export const EVENT_STATUS_CHANGED = 'status-changed';
+export const EVENT_INSTANCE_CREATED = 'instance-created';
+export const EVENT_INSTANCE_EXITED = 'instance-exited';
+export const EVENT_INSTANCE_LOG = 'instance-log';
+
+export const EVENT_SYSTEM_LOG = 'system-log';
+export const EVENT_LOG = 'log';
+
+export class SocketManager {
+    private _io: Server | null;
+    private readonly _sockets: Socket[];
+
+    constructor() {
+        this._io = null;
+        this._sockets = [];
+        return this;
+    }
+
+    setIo(io: Server) {
+        this._io = io;
+        this._bindIO();
+    }
+
+    isAlive() {
+        return !!this._io;
+    }
+
+    private get io() {
+        if (!this._io) {
+            throw new Error('Socket server not ready');
+        }
+        return this._io;
+    }
+
+    emit(context: string, type: string, payload: any) {
+        if (!this._io) {
+            return;
+        }
+        this.io.to(context).emit(type, { context, payload });
+    }
+
+    emitGlobal(type: string, payload: any) {
+        if (!this._io) {
+            return;
+        }
+        this.io.emit(type, payload);
+    }
+
+    emitSystemEvent(systemId: string, type: string, payload: any) {
+        systemId = normalizeKapetaUri(systemId);
+        try {
+            const contextId = `system-events/${encodeURIComponent(systemId)}`;
+            this.emit(contextId, type, payload);
+        } catch (e: any) {
+            console.warn('Failed to emit instance event: %s', e.message);
+        }
+    }
+
+    emitInstanceLog(systemId: string, instanceId: string, payload: LogEntry) {
+        systemId = normalizeKapetaUri(systemId);
+        try {
+            this.emit(
+                `instance-logs/${encodeURIComponent(systemId)}/${encodeURIComponent(instanceId)}`,
+                EVENT_INSTANCE_LOG,
+                payload
+            );
+        } catch (e: any) {
+            console.warn('Failed to emit instance event: %s', e.message);
+        }
+    }
+
+    emitSystemLog(systemId: string, payload: LogEntry) {
+        this.emitSystemEvent(systemId, EVENT_SYSTEM_LOG, payload);
+    }
+
+    emitGlobalLog(payload: LogEntry) {
+        this.emitGlobal(EVENT_LOG, payload);
+    }
+
+    private _bindIO() {
+        this.io.on('connection', (socket) => this._handleSocketCreated(socket));
+    }
+
+    private _handleSocketCreated(socket: Socket) {
+        this._bindSocket(socket);
+        this._sockets.push(socket);
+    }
+
+    private _bindSocket(socket: Socket) {
+        socket.on('disconnect', () => {
+            _.pull(this._sockets, socket);
+            socket.rooms.forEach((roomId) => {
+                this.handleLeaveRoom(roomId);
+            });
+        });
+        socket.on('join', (id) => {
+            socket.join(id);
+            this.handleJoinRoom(id);
+        });
+        socket.on('leave', (id) => {
+            socket.leave(id);
+            this.handleLeaveRoom(id);
+        });
+    }
+    private handleJoinRoom(id: string) {
+        if (id.startsWith('instance-logs/')) {
+            let [, systemId, instanceId] = id.split(/\//g);
+            systemId = decodeURIComponent(systemId);
+            instanceId = decodeURIComponent(instanceId);
+            console.log('Start listening for logs', systemId, instanceId);
+            containerManager
+                .ensureLogListening(systemId, instanceId, (log) => {
+                    this.emitInstanceLog(systemId, instanceId, log);
+                })
+                .catch((e) => {});
+        }
+    }
+    private handleLeaveRoom(id: string) {
+        if (id.startsWith('instance-logs/')) {
+            let [, systemId, instanceId] = id.split(/\//g);
+            systemId = decodeURIComponent(systemId);
+            instanceId = decodeURIComponent(instanceId);
+            console.log('Stop listening for logs', systemId, instanceId);
+            containerManager.stopLogListening(systemId, instanceId).catch((e) => {});
+        }
+    }
+}
+export const socketManager = new SocketManager();

package/src/storageService.ts

@@ -0,0 +1,97 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import _ from 'lodash';
+import FS from 'fs';
+import FSExtra from 'fs-extra';
+import YAML from 'yaml';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+
+/**
+ * Class that handles reading and writing from local configuration file.
+ */
+class StorageService {
+    private _data: { [key: string]: any };
+
+    constructor() {
+        this._data = this._readConfig();
+    }
+
+    getKapetaBasedir() {
+        return ClusterConfiguration.getKapetaBasedir();
+    }
+
+    _readConfig() {
+        return ClusterConfiguration.getClusterConfig();
+    }
+
+    _writeConfig() {
+        const configFile = ClusterConfiguration.getClusterConfigFile();
+
+        FSExtra.mkdirsSync(this.getKapetaBasedir());
+
+        FS.writeFileSync(configFile, YAML.stringify(this._data));
+    }
+
+    section<T = any>(section: string, defaultValue?: any): T {
+        if (!defaultValue) {
+            defaultValue = {};
+        }
+        if (!this._data[section]) {
+            this._data[section] = defaultValue;
+            this._writeConfig();
+        }
+
+        return this._data[section];
+    }
+
+    put(section: string, property: string | any, value?: any) {
+        if (!_.isString(property)) {
+            this._data[section] = property;
+            this._writeConfig();
+            return;
+        }
+
+        this.section(section)[property] = value;
+        this._writeConfig();
+    }
+
+    get<T = any>(section: string, property?: string, defaultValue?: T): T | undefined {
+        if (!property) {
+            return this.section(section);
+        }
+
+        if (!this.contains(section, property)) {
+            return defaultValue;
+        }
+
+        return this.section(section)[property];
+    }
+
+    contains(section: string, property: string) {
+        if (!this._data[section]) {
+            return false;
+        }
+
+        return this._data[section].hasOwnProperty(property);
+    }
+
+    ensure(section: string, property: string, value: any) {
+        if (this.contains(section, property)) {
+            return this.get(section, property);
+        }
+
+        let out = value;
+        if (typeof value === 'function') {
+            out = value();
+        }
+
+        this.put(section, property, out);
+
+        return out;
+    }
+}
+
+export const storageService = new StorageService();

package/src/taskManager.ts

@@ -0,0 +1,247 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+/**
+ * Class that handles processing background tasks.
+ */
+import { socketManager } from './socketManager';
+import { LogLevel } from './types';
+
+const EVENT_TASK_UPDATED = 'task-updated';
+const EVENT_TASK_ADDED = 'task-added';
+const EVENT_TASK_REMOVED = 'task-removed';
+const EVENT_TASK_LOG = 'task-log';
+
+export type TaskRunner<T> = (task: Task<T>) => Promise<T>;
+
+export enum TaskStatus {
+    PENDING = 'PENDING',
+    RUNNING = 'RUNNING',
+    COMPLETED = 'COMPLETED',
+    FAILED = 'FAILED',
+}
+
+interface Future<T = void> {
+    promise: Promise<T>;
+    resolve: (result: T) => void;
+    reject: (e: any) => void;
+}
+
+interface TaskMetadata {
+    name: string;
+    /**
+     * A unique prefix for the task. If defined only 1 task with this ID prefix will be executed at a time
+     */
+    group?: string;
+    progress?: number;
+
+    [key: string]: any;
+}
+
+interface TaskData<T = void> {
+    id: string;
+    status: TaskStatus;
+    errorMessage?: string;
+    metadata: TaskMetadata;
+    future: Future<T>;
+    run: TaskRunner<T>;
+}
+
+export class Task<T = void> implements TaskData<T> {
+    private data: TaskData<T>;
+
+    constructor(task: TaskData<T>) {
+        this.data = task;
+    }
+
+    get id() {
+        return this.data.id;
+    }
+
+    get status() {
+        return this.data.status;
+    }
+
+    get errorMessage() {
+        return this.data.errorMessage;
+    }
+
+    get metadata() {
+        return this.data.metadata;
+    }
+
+    get future() {
+        return this.data.future;
+    }
+
+    get run() {
+        return this.data.run;
+    }
+
+    set status(status: TaskStatus) {
+        this.data.status = status;
+    }
+
+    set errorMessage(errorMessage: string | undefined) {
+        this.data.errorMessage = errorMessage;
+    }
+
+    set metadata(metadata: TaskMetadata) {
+        this.data.metadata = metadata;
+    }
+
+    public emitUpdate() {
+        socketManager.emitGlobal(EVENT_TASK_UPDATED, this.toData());
+    }
+
+    public addLog(log: string, level: LogLevel = 'INFO') {
+        socketManager.emitGlobal(EVENT_TASK_LOG, {
+            id: this.id,
+            message: log,
+            level,
+            time: Date.now(),
+        });
+    }
+
+    async wait(): Promise<T> {
+        return this.future.promise;
+    }
+
+    toData() {
+        return { ...this.data };
+    }
+}
+
+function createFuture<T>(): Future<T> {
+    let resolve: (arg: T) => void = () => {};
+    let reject: () => void = () => {};
+    const promise = new Promise<T>((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+
+    // Ignore unhandled promise rejections
+    promise.catch(() => {});
+
+    return {
+        promise,
+        resolve,
+        reject,
+    };
+}
+
+class TaskManager {
+    private _tasks: Task<any>[] = [];
+
+    public add<T>(id: string, runner: TaskRunner<T>, metadata: TaskMetadata): Task<T> {
+        const existingTask = this.get(id);
+        if (existingTask) {
+            return existingTask;
+        }
+
+        const future = createFuture<T>();
+
+        const task = new Task<T>({
+            id,
+            status: TaskStatus.PENDING,
+            metadata,
+            future,
+            run: runner,
+        });
+
+        this._tasks.push(task);
+
+        socketManager.emitGlobal(EVENT_TASK_ADDED, task.toData());
+
+        this.invokeTask(task).catch((err) => {
+            console.warn(`Task ${task.id} failed`, err);
+        });
+
+        return task;
+    }
+
+    async waitFor(filter: (task: Task<any>) => boolean) {
+        const tasks = this._tasks.filter(filter);
+        while (tasks.length > 0) {
+            const task = tasks.shift();
+            if (!task) {
+                continue;
+            }
+            try {
+                await task.wait();
+            } catch (e) {
+                //Ignore
+            }
+        }
+    }
+
+    public get(taskId: string) {
+        return this._tasks.find((t) => t.id === taskId);
+    }
+
+    public exists(taskId: string) {
+        return !!this.get(taskId);
+    }
+
+    public remove(taskId: string) {
+        const task = this.get(taskId);
+        if (!task) {
+            return;
+        }
+
+        if (task.status === TaskStatus.RUNNING) {
+            throw new Error('Cannot remove a running task');
+        }
+
+        this._tasks = this._tasks.filter((t) => t.id !== taskId);
+        socketManager.emitGlobal(EVENT_TASK_REMOVED, task.toData());
+    }
+
+    public list(): TaskData[] {
+        return this._tasks.map((t) => t.toData());
+    }
+
+    private async invokeTask(task: Task<any>): Promise<void> {
+        if (task.metadata.group) {
+            const existingTaskInGroup = this._tasks.find(
+                (t) => t.id !== task.id && t.metadata.group === task.metadata.group && t.status === TaskStatus.RUNNING
+            );
+
+            if (existingTaskInGroup) {
+                return;
+            }
+        }
+
+        const startTime = Date.now();
+        try {
+            task.status = TaskStatus.RUNNING;
+            task.emitUpdate();
+            const result = await task.run(task);
+            task.status = TaskStatus.COMPLETED;
+            task.future.resolve(result);
+            task.emitUpdate();
+        } catch (e: any) {
+            console.warn(`Task ${task.id} failed while waiting for it to resolve`, e);
+            task.errorMessage = e.message;
+            task.status = TaskStatus.FAILED;
+            task.future.reject(e);
+            task.emitUpdate();
+        } finally {
+            this.remove(task.id);
+            console.log(`Task ${task.id} completed in ${Date.now() - startTime}ms`);
+        }
+
+        if (task.metadata.group) {
+            const nextTaskInGroup = this._tasks.find(
+                (t) => t.id !== task.id && t.metadata.group === task.metadata.group && t.status === TaskStatus.PENDING
+            );
+            if (nextTaskInGroup) {
+                return this.invokeTask(nextTaskInGroup);
+            }
+        }
+    }
+}
+
+export const taskManager = new TaskManager();

package/src/tasks/routes.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Router from 'express-promise-router';
+import { Request, Response } from 'express';
+
+import { corsHandler } from '../middleware/cors';
+import { taskManager } from '../taskManager';
+
+const router = Router();
+
+router.use('/', corsHandler);
+
+/**
+ * Get all current tasks
+ */
+router.get('/', (req: Request, res: Response) => {
+    res.send(taskManager.list());
+});
+
+router.get('/:taskId', (req: Request, res: Response) => {
+    const task = taskManager.get(req.params.taskId);
+    if (!task) {
+        res.status(404).send({ error: 'Task not found' });
+        return;
+    }
+
+    res.send(task.toData());
+});
+
+router.delete('/:taskId', (req: Request, res: Response) => {
+    try {
+        taskManager.remove(req.params.taskId);
+        res.send({ ok: true });
+    } catch (e: any) {
+        res.status(400).send({ error: e.message });
+        return;
+    }
+});
+
+export default router;

package/src/traffic/routes.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Router from 'express-promise-router';
+import { Request, Response } from 'express';
+import { networkManager } from '../networkManager';
+
+const router = Router();
+router.get('/:systemId/target/:connectionId/', (req: Request, res: Response) => {
+    res.send(networkManager.getTrafficForConnection(req.params.systemId, req.params.connectionId));
+});
+
+router.get('/:systemId/source/:blockInstanceId/', (req: Request, res: Response) => {
+    res.send(networkManager.getTrafficForSource(req.params.systemId, req.params.blockInstanceId));
+});
+
+router.get('/:systemId/target/:blockInstanceId/', (req: Request, res: Response) => {
+    res.send(networkManager.getTrafficForTarget(req.params.systemId, req.params.blockInstanceId));
+});
+
+export default router;