@nosana/node 1.0.0-rc

package/dist/NodeManager/provider/Provider.js

@@ -0,0 +1,457 @@
+import { createHash, getExposePorts, isOpExposed, } from '@nosana/sdk';
+import { configs } from '../configs/configs.js';
+import { NodeConfigsSingleton } from '../configs/NodeConfigs.js';
+import { ExposedPortHealthCheck } from './ExposedPortHealthCheck.js';
+import { s3HelperImage } from '../node/resource/definition/index.js';
+import { applyLoggingProxyToClass } from '../monitoring/proxy/loggingProxy.js';
+import { promiseTimeoutWrapper } from '../utils/timeoutPromiseWrapper.js';
+import { generateProxies, generateUrlSecretObject, } from '../utils/expose-util.js';
+import { ContainerStateManager } from './ContainerStateManager.js';
+export class Provider {
+    constructor(containerOrchestration, repository, resourceManager) {
+        this.containerOrchestration = containerOrchestration;
+        this.repository = repository;
+        this.resourceManager = resourceManager;
+        this.frpcImage = configs().frp.containerImage;
+        this.proxyStartupAbortController = undefined;
+        applyLoggingProxyToClass(this);
+    }
+    async stopReverseProxyApi(address) {
+        const frpc_name = `frpc-api-${address}`;
+        const networkName = `api-${address}`;
+        if (this.proxyStartupAbortController instanceof AbortController) {
+            this.proxyStartupAbortController.abort();
+        }
+        try {
+            // Check if the frpc container exists and stop/delete it
+            const doesFrpcExist = await this.containerOrchestration.doesContainerExist(frpc_name);
+            if (doesFrpcExist) {
+                await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
+            }
+            // check if network then delete
+            if (await this.containerOrchestration.hasNetwork(networkName)) {
+                await this.containerOrchestration.deleteNetwork(networkName);
+            }
+        }
+        catch (error) {
+            throw error;
+        }
+        return true;
+    }
+    // set up reverse proxy api for api handler
+    async setUpReverseProxyApi(address, port) {
+        try {
+            const nodeName = `nosana-node`;
+            const networkName = `api-${address}`;
+            const frpc_name = `frpc-api-${address}`;
+            await this.containerOrchestration.createNetwork(networkName);
+            const networks = {};
+            networks[networkName] = {};
+            this.proxyStartupAbortController = new AbortController();
+            await this.containerOrchestration.pullImage(this.frpcImage, undefined, this.proxyStartupAbortController);
+            this.resourceManager.images.setImage(this.frpcImage);
+            const doesFrpcExist = await this.containerOrchestration.doesContainerExist(frpc_name);
+            if (!doesFrpcExist) {
+                await this.containerOrchestration.runFlowContainer(this.frpcImage, {
+                    name: 'frpc-api-' + address,
+                    cmd: ['-c', '/etc/frp/frpc.toml'],
+                    networks,
+                    requires_network_mode: true,
+                    restart_policy: 'on-failure',
+                    env: {
+                        FRP_SERVER_ADDR: configs().frp.serverAddr,
+                        FRP_SERVER_PORT: configs().frp.serverPort.toString(),
+                        FRP_PROXIES: JSON.stringify([
+                            {
+                                name: 'API-' + address,
+                                localIp: nodeName,
+                                localPorts: port.toString(),
+                                customDomain: address + '.' + configs().frp.serverAddr,
+                            },
+                        ]),
+                    },
+                    volumes: [
+                        {
+                            name: `frpc-${address}-logs`,
+                            dest: '/data',
+                        },
+                    ],
+                });
+            }
+            else {
+                const hasFrpcExited = await this.containerOrchestration.isContainerExited(frpc_name);
+                if (hasFrpcExited) {
+                    await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
+                    await this.containerOrchestration.runFlowContainer(this.frpcImage, {
+                        name: 'frpc-api-' + address,
+                        cmd: ['-c', '/etc/frp/frpc.toml'],
+                        networks,
+                        requires_network_mode: true,
+                        restart_policy: 'on-failure',
+                        env: {
+                            FRP_SERVER_ADDR: configs().frp.serverAddr,
+                            FRP_SERVER_PORT: configs().frp.serverPort.toString(),
+                            FRP_PROXIES: JSON.stringify([
+                                {
+                                    name: 'API-' + address,
+                                    localIp: nodeName,
+                                    localPorts: port.toString(),
+                                    customDomain: address + '.' + configs().frp.serverAddr,
+                                },
+                            ]),
+                        },
+                        volumes: [
+                            {
+                                name: `frpc-${address}-logs`,
+                                dest: '/data',
+                            },
+                        ],
+                    });
+                }
+            }
+        }
+        catch (error) {
+            throw error;
+        }
+        this.proxyStartupAbortController = undefined;
+        return true;
+    }
+    async taskManagerContainerRunOperation(flow, op, controller, emitter) {
+        let stateManager;
+        let exposedPortHealthCheck;
+        if (controller.signal.aborted) {
+            emitter.emit('exit', { exitCode: 0 });
+            emitter.emit('end');
+            return;
+        }
+        try {
+            emitter.emit('start');
+            let frpcContainer;
+            let container = await this.containerOrchestration.getContainer(op.id);
+            const exist = await this.containerOrchestration.doesContainerExist(op.id);
+            const exited = await this.containerOrchestration.isContainerExited(op.id);
+            if (exist && !exited && container.id) {
+                stateManager = new ContainerStateManager(container, controller, emitter, op.args.restart_policy);
+                await stateManager.startMonitoring();
+            }
+            else {
+                await this.containerOrchestration.pullImage(op.args.image, op.args.authentication?.docker, controller);
+                if (!op.args.authentication?.docker) {
+                    this.resourceManager.images.setImage(op.args.image);
+                }
+                const index = getOpStateIndex(flow.jobDefinition.ops, op.id);
+                const name = flow.id + '-' + index;
+                const volumes = getVolumes(op.args, flow);
+                const gpu = getGpu(op.args, flow);
+                const entrypoint = getEntrypoint(op.args, flow);
+                const work_dir = getWorkingDir(op.args, flow);
+                const cmd = parseOpArgsCmd(op.args.cmd);
+                const env = {
+                    ...getGlobalEnv(flow),
+                    ...op.args.env,
+                    NOSANA_ID: flow.id,
+                };
+                const aliases = getAliases(op.args);
+                const networks = {};
+                let idMaps = new Map();
+                const ports = getExposePorts(op);
+                networks[name] = {};
+                await this.containerOrchestration.createNetwork(name, controller);
+                if (isOpExposed(op)) {
+                    await this.containerOrchestration.pullImage(this.frpcImage, undefined, controller);
+                    this.resourceManager.images.setImage(this.frpcImage);
+                    /**
+                     * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
+                     * deterministic url generation
+                     */
+                    const config = NodeConfigsSingleton.getInstance();
+                    const isLoadBalanced = !!flow.jobDefinition.deployment_id;
+                    const deploymentHash = isLoadBalanced
+                        ? // @ts-ignore
+                            config.options.isNodeRun
+                                ? flow.jobDefinition.deployment_id
+                                : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
+                        : undefined;
+                    const { proxies, idMap } = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
+                    idMaps = idMap;
+                    frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash));
+                    if (op.args.private) {
+                        this.repository.updateflowStateSecret(flow.id, {
+                            [flow.id]: generateUrlSecretObject(idMap, op.id),
+                            urlmode: 'private',
+                        });
+                        emitter.emit('flow:secrets-updated', {
+                            flowId: flow.id,
+                            opId: op.id,
+                        });
+                    }
+                    else {
+                        const newSecret = generateUrlSecretObject(idMap, op.id);
+                        const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
+                        const mergedSecrets = {
+                            ...currentSecrets,
+                            ...newSecret,
+                        };
+                        this.repository.updateflowStateSecret(flow.id, {
+                            [flow.id]: mergedSecrets,
+                            urlmode: 'public',
+                        });
+                        emitter.emit('flow:secrets-updated', {
+                            flowId: flow.id,
+                            opId: op.id,
+                        });
+                    }
+                }
+                if (op.args.resources) {
+                    await this.containerOrchestration.pullImage(s3HelperImage, undefined, controller);
+                    this.resourceManager.images.setImage(s3HelperImage);
+                    // transformCollections has already resolved any SpreadMarker objects by this point
+                    const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller);
+                    volumes.push(...resourceVolumes);
+                }
+                container = await this.containerOrchestration.runFlowContainer(op.args.image ?? flow.jobDefinition.global?.image, {
+                    name,
+                    cmd,
+                    env,
+                    networks,
+                    requires_network_mode: isOpExposed(op),
+                    gpu,
+                    entrypoint,
+                    work_dir,
+                    volumes,
+                    aliases,
+                    restart_policy: op.args.restart_policy,
+                });
+                emitter.emit('updateOpState', { providerId: container.id });
+                stateManager = new ContainerStateManager(container, controller, emitter, op.args.restart_policy);
+                await stateManager.startMonitoring();
+                if (isOpExposed(op)) {
+                    exposedPortHealthCheck = new ExposedPortHealthCheck(flow.id, frpcContainer, emitter, name);
+                    exposedPortHealthCheck.addExposedPortsMap(idMaps);
+                    exposedPortHealthCheck.startServiceExposedUrlHealthCheck();
+                }
+                else {
+                    emitter.emit('healthcheck:startup:success');
+                }
+                await stateManager.waitForExit();
+            }
+            const info = await promiseTimeoutWrapper(container.inspect({
+                abortSignal: controller.signal,
+            }), 360, controller);
+            emitter.emit('exit', { exitCode: info.State.ExitCode });
+        }
+        catch (error) {
+            emitter.emit('log', error, 'error');
+            emitter.emit('error', error);
+        }
+        // Clean up resources
+        stateManager?.stopMonitoring();
+        exposedPortHealthCheck?.stopAllHealthChecks();
+        exposedPortHealthCheck = undefined;
+        emitter.emit('end');
+    }
+    generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash) {
+        return {
+            name: 'frpc-' + name,
+            cmd: ['/entrypoint.sh'],
+            networks,
+            requires_network_mode: true,
+            env: {
+                FRP_SERVER_ADDR: configs().frp.serverAddr,
+                FRP_SERVER_PORT: configs().frp.serverPort.toString(),
+                NOSANA_ID: flow.id,
+                FRP_PROXIES: JSON.stringify(proxies),
+                DEPLOYMENT_ID: flow.jobDefinition.deployment_id ?? '',
+                JOB_ID: flow.id,
+                ...(isLoadBalanced && {
+                    FRP_LB_GROUP_KEY: deploymentHash,
+                }),
+            },
+        };
+    }
+    async taskManagerContainerStopRunOperation(flow, op, emitter) {
+        try {
+            const index = getOpStateIndex(flow.jobDefinition.ops, op.id);
+            const name = flow.id + '-' + index;
+            const opContainer = await this.containerOrchestration.getContainerByName(name);
+            const frpcContainer = await this.containerOrchestration.getContainerByName('frpc-' + name);
+            if (opContainer) {
+                const info = await this.containerOrchestration.stopAndDeleteContainer(opContainer.id);
+                if (info) {
+                    this.repository.updateOpStateExitCode(flow.id, index, info.State.ExitCode);
+                    this.repository.setOpStateDiagnosticsState(flow.id, index, {
+                        ...info.State,
+                        RestartCount: info.RestartCount,
+                    });
+                }
+            }
+            if (frpcContainer) {
+                await this.containerOrchestration.stopAndDeleteContainer(frpcContainer.id);
+            }
+            await this.containerOrchestration.deleteNetwork(name);
+            if (flow.jobDefinition.ops[index]
+                .args.authentication?.docker) {
+                await this.containerOrchestration.deleteImage(flow.jobDefinition.ops[index]
+                    .args.image);
+            }
+        }
+        catch (error) {
+            emitter?.emit('log', error, 'error');
+            throw error;
+        }
+    }
+    /**
+     * Runs a full container operation lifecycle:
+     * 1. Starts the container, emits lifecycle events, waits for completion or error.
+     * 2. Then stops and cleans up the container, network, and associated Docker resources.
+     *
+     * This function ties together both the **execution** and **cleanup** stages for a containerized operation.
+     *
+     * The steps are:
+     * - Emits 'start' → Pulls container image (if needed) → Creates volume/network (if needed)
+     * - Runs container with correct args/env/entrypoint
+     * - Follows logs and emits 'log' → Waits for completion → Emits 'exit' or 'error'
+     * - Regardless of outcome, cleans up resources (containers, networks, optional image)
+     *
+     * Emits:
+     * - 'start': when the operation begins
+     * - 'log': every time stdout/stderr produces output
+     * - 'updateOpState': to attach container ID
+     * - 'exit': when the operation finishes successfully or with failure
+     * - 'error': when something crashes
+     * - 'end': always, after cleanup
+     *
+     * @param flow - The job flow this operation is part of
+     * @param op - The specific container operation to run
+     * @param controller - AbortController to support cancellation
+     * @param emitter - EventEmitter to stream logs and events from the container lifecycle
+     */
+    async taskManagerContainerStartRunandStopOperation(flow, op, controller, emitter) {
+        try {
+            await this.taskManagerContainerRunOperation(flow, op, controller, emitter);
+        }
+        finally {
+            await this.taskManagerContainerStopRunOperation(flow, op, emitter);
+        }
+    }
+    async taskManagerVolumeStartRunandStopOperation(flow, op, controller, emitter) {
+        await this.taskManagerVolumeCreateOperation(flow, op, controller, emitter);
+        /**
+         * we don't put the volume stop here because it will delete the volume that
+         * might be used by other operations in the group, we will leave that for the
+         * final clean up.
+         * await this.taskManagerVolumeStopOperation(flow, op, emitter)
+         */
+    }
+    async taskManagerVolumeCreateOperation(flow, op, controller, emitter) {
+        if (controller.signal.aborted) {
+            emitter.emit('exit', { exitCode: 0 });
+            emitter.emit('end');
+            return;
+        }
+        try {
+            emitter.emit('start');
+            const name = flow.id + '-' + op.args.name;
+            const isVolume = await this.containerOrchestration.hasVolume(name);
+            if (!isVolume) {
+                const volume = await this.containerOrchestration.createVolume(name, controller);
+                emitter.emit('updateOpHost', { name });
+                emitter.emit('updateOpState', { providerId: volume.Status });
+            }
+            emitter.emit('healthcheck:startup:success');
+            emitter.emit('exit', { exitCode: 0 });
+        }
+        catch (error) {
+            emitter.emit('log', error, 'error');
+            emitter.emit('error', error);
+        }
+        emitter.emit('end');
+    }
+    async taskManagerVolumeStopOperation(flow, op, emitter) {
+        try {
+            const name = flow.id + '-' + op.args.name;
+            const isVolume = await this.containerOrchestration.hasVolume(name);
+            if (!isVolume) {
+                await this.containerOrchestration.deleteVolume(name);
+            }
+        }
+        catch (error) {
+            emitter?.emit('log', error, 'error');
+            throw error;
+        }
+    }
+    runTaskManagerOperation(flow, op, controller, emitter) {
+        switch (op.type) {
+            case 'container/run':
+                return this.taskManagerContainerStartRunandStopOperation(flow, op, controller, emitter);
+            case 'container/create-volume':
+                return this.taskManagerVolumeStartRunandStopOperation(flow, op, controller, emitter);
+            default:
+                throw new Error(`operation type '${op.type}' not supported`);
+        }
+    }
+    stopTaskManagerOperation(flow, op, emitter) {
+        switch (op.type) {
+            case 'container/run':
+                return this.taskManagerContainerStopRunOperation(flow, op, emitter);
+            case 'container/create-volume':
+                return this.taskManagerVolumeStopOperation(flow, op, emitter);
+            default:
+                throw new Error(`operation type '${op.type}' not supported`);
+        }
+    }
+}
+function getOpStateIndex(ops, opId) {
+    const index = ops.findIndex((op) => op.id === opId);
+    if (index === -1) {
+        throw new Error(`Operation not found for ID: ${opId}`);
+    }
+    return index;
+}
+export function parseOpArgsCmd(cmd) {
+    if (typeof cmd !== 'string')
+        return cmd;
+    return ['/bin/sh', '-c', cmd];
+}
+function getGlobalEnv(flow) {
+    return flow.jobDefinition.global && flow.jobDefinition.global.env
+        ? flow.jobDefinition.global.env
+        : {};
+}
+function getWorkingDir(arg, flow) {
+    return arg.work_dir ||
+        !flow.jobDefinition.global ||
+        !flow.jobDefinition.global.work_dir
+        ? arg.work_dir
+        : flow.jobDefinition.global.work_dir;
+}
+function getEntrypoint(arg, flow) {
+    return arg.entrypoint ||
+        !flow.jobDefinition.global ||
+        !flow.jobDefinition.global.entrypoint
+        ? arg.entrypoint
+        : flow.jobDefinition.global.entrypoint;
+}
+function getGpu(arg, flow) {
+    return (arg.gpu || (flow.jobDefinition.global && flow.jobDefinition.global.gpu));
+}
+function getVolumes(arg, flow) {
+    const volumes = [];
+    if (arg.volumes && arg.volumes.length > 0) {
+        for (let i = 0; i < arg.volumes.length; i++) {
+            const volume = arg.volumes[i];
+            volumes.push({
+                dest: volume.dest,
+                name: flow.id + '-' + volume.name,
+            });
+        }
+    }
+    return volumes;
+}
+function getAliases(args) {
+    if (!args.aliases)
+        return undefined;
+    if (typeof args.aliases === 'string')
+        return args.aliases.split(',');
+    return args.aliases;
+}

package/dist/NodeManager/provider/__tests__/Provider.spec.d.ts

@@ -0,0 +1 @@
+export {};