@nosana/node 1.0.0-rc

package/dist/NodeManager/provider/ExposedPortHealthCheck.js

@@ -0,0 +1,183 @@
+export class ExposedPortHealthCheck {
+    constructor(flowId, frpcContainer, jobEmitter, containerName, startupIntervalMs = 5000, continuousIntervalMs = 30000) {
+        this.exposedPortMap = new Map();
+        /**
+         * Tracks job previous state to show change in state of job mainly when going from failed to success.
+         */
+        this.healthStatus = new Map();
+        this.intervals = new Map();
+        this.flowId = flowId;
+        this.frpcContainer = frpcContainer;
+        this.jobEmitter = jobEmitter;
+        this.containerName = containerName;
+        this.startupIntervalMs = startupIntervalMs;
+        this.continuousIntervalMs = continuousIntervalMs;
+    }
+    addExposedPortsMap(exposedPortMap) {
+        this.exposedPortMap = exposedPortMap;
+    }
+    startServiceExposedUrlHealthCheck() {
+        for (const [id, exposedPort] of this.exposedPortMap) {
+            this.startStartupHealthCheck(id, exposedPort);
+        }
+    }
+    startStartupHealthCheck(id, exposedPort) {
+        let startupCheckInterval = null;
+        startupCheckInterval = setInterval(async () => {
+            const success = await this.checkPortHealth(exposedPort, true);
+            if (success == null) {
+                clearInterval(startupCheckInterval);
+                this.jobEmitter.emit('healthcheck:url:exposed', {
+                    id,
+                    flowId: this.flowId,
+                    port: exposedPort.port,
+                    service: exposedPort.type,
+                });
+                return;
+            }
+            if (success) {
+                clearInterval(startupCheckInterval);
+                this.jobEmitter.emit('healthcheck:startup:success', {
+                    id,
+                    flowId: this.flowId,
+                    port: exposedPort.port,
+                    service: exposedPort.type,
+                });
+                // **Start continuous health check only after startup success**
+                this.startContinuousHealthCheck(id, exposedPort);
+            }
+        }, this.startupIntervalMs);
+        // Store the interval so we can stop it later
+        this.intervals.set(id, startupCheckInterval);
+    }
+    startContinuousHealthCheck(id, exposedPort) {
+        let continuousCheckInterval = setInterval(async () => {
+            const healthStatus = await this.checkPortHealth(exposedPort, false);
+            if (healthStatus === null) {
+                clearInterval(continuousCheckInterval);
+                return; // Skip logging or emitting events if no check was done.
+            }
+            const previousState = this.healthStatus.get(id) || false;
+            if (healthStatus !== previousState) {
+                this.healthStatus.set(id, healthStatus);
+                if (!healthStatus) {
+                    this.jobEmitter.emit('healthcheck:continuous:failure', {
+                        id,
+                        flowId: this.flowId,
+                        port: exposedPort.port,
+                        service: exposedPort.type,
+                    });
+                }
+            }
+        }, this.continuousIntervalMs);
+        this.intervals.set(id, continuousCheckInterval);
+    }
+    stopHealthCheckForId(id) {
+        if (this.intervals.has(id)) {
+            clearInterval(this.intervals.get(id));
+            this.intervals.delete(id);
+            this.healthStatus.delete(id);
+            this.exposedPortMap.delete(id);
+        }
+    }
+    stopAllHealthChecks() {
+        this.intervals.forEach((interval, id) => {
+            clearInterval(interval);
+        });
+        this.intervals.clear();
+        this.healthStatus.clear();
+        this.exposedPortMap.clear();
+    }
+    async checkPortHealth(exposedPort, initialRun) {
+        if (!exposedPort.health_checks || exposedPort.health_checks.length === 0) {
+            return null; // Return null to indicate no check was done.
+        }
+        for (const healthCheck of exposedPort.health_checks) {
+            if (!initialRun && !healthCheck.continuous) {
+                return null;
+            }
+            if (healthCheck.type === 'http' && typeof exposedPort.port === 'number') {
+                const success = await this.runHttpHealthCheck(exposedPort.port, healthCheck);
+                if (!success)
+                    return false;
+            }
+            else if (healthCheck.type === 'websocket' &&
+                typeof exposedPort.port === 'number') {
+                const success = await this.runWebSocketHealthCheck(exposedPort.port, healthCheck);
+                if (!success)
+                    return false;
+            }
+        }
+        return true;
+    }
+    async runHttpHealthCheck(port, healthCheck) {
+        const url = `http://${this.containerName}:${port}${healthCheck.path}`;
+        const cmd = [
+            'curl',
+            '-s',
+            '-o',
+            '/dev/null',
+            '-w',
+            '%{http_code}',
+            '-X',
+            healthCheck.method,
+            url,
+        ];
+        if (healthCheck.headers) {
+            for (const [key, value] of Object.entries(healthCheck.headers)) {
+                cmd.push('-H', `${key}: ${value}`);
+            }
+        }
+        if (healthCheck.body && ['POST', 'PUT'].includes(healthCheck.method)) {
+            const bodyString = typeof healthCheck.body === 'string'
+                ? healthCheck.body
+                : JSON.stringify(healthCheck.body);
+            cmd.push('--data', bodyString);
+        }
+        try {
+            const output = await this.execCommand(cmd);
+            return (Buffer.from(output, 'utf-8')
+                .toString()
+                .replace(/[^\x20-\x7E]/g, '')
+                .trim() == healthCheck.expected_status.toString());
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    async runWebSocketHealthCheck(port, healthCheck) {
+        const cmd = ['curl', '--include', '--no-buffer', `ws://localhost:${port}`];
+        try {
+            const output = await this.execCommand(cmd);
+            return output.includes(healthCheck.expected_response);
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    async execCommand(cmd) {
+        try {
+            const exec = await this.frpcContainer.exec({
+                Cmd: cmd,
+                AttachStdout: true,
+                AttachStderr: true,
+            });
+            const stream = await exec.start({ hijack: true, stdin: true });
+            return new Promise((resolve, reject) => {
+                let output = '';
+                stream.on('data', (chunk) => {
+                    output += chunk.toString();
+                });
+                stream.on('end', () => {
+                    resolve(output);
+                });
+                stream.on('error', (err) => {
+                    reject(err);
+                });
+            });
+        }
+        catch (error) {
+            throw new Error(`Exec failed: ${error}`);
+        }
+    }
+}

package/dist/NodeManager/provider/Provider.d.ts

@@ -0,0 +1,70 @@
+/// <reference types="node" />
+import { Flow, Operation, OperationArgsMap, OperationType } from '@nosana/sdk';
+import EventEmitter from 'events';
+import { NodeRepository } from '../repository/NodeRepository.js';
+import { ResourceManager } from '../node/resource/resourceManager.js';
+import { ContainerOrchestrationInterface } from './containerOrchestration/interface.js';
+export declare class Provider {
+    containerOrchestration: ContainerOrchestrationInterface;
+    private repository;
+    private resourceManager;
+    private readonly frpcImage;
+    private proxyStartupAbortController;
+    constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository, resourceManager: ResourceManager);
+    stopReverseProxyApi(address: string): Promise<boolean>;
+    setUpReverseProxyApi(address: string, port: string): Promise<boolean>;
+    taskManagerContainerRunOperation(flow: Flow, op: Operation<'container/run'>, controller: AbortController, emitter: EventEmitter): Promise<void>;
+    generateFrpcContainerConfig(name: string, networks: {
+        [p: string]: {};
+    }, flow: Flow, proxies: any[], isLoadBalanced: boolean, deploymentHash: string | undefined): {
+        name: string;
+        cmd: string[];
+        networks: {
+            [p: string]: {};
+        };
+        requires_network_mode: boolean;
+        env: {
+            FRP_LB_GROUP_KEY?: string | undefined;
+            FRP_SERVER_ADDR: string;
+            FRP_SERVER_PORT: string;
+            NOSANA_ID: string;
+            FRP_PROXIES: string;
+            DEPLOYMENT_ID: string;
+            JOB_ID: string;
+        };
+    };
+    taskManagerContainerStopRunOperation(flow: Flow, op: Operation<'container/run'>, emitter?: EventEmitter): Promise<void>;
+    /**
+     * Runs a full container operation lifecycle:
+     * 1. Starts the container, emits lifecycle events, waits for completion or error.
+     * 2. Then stops and cleans up the container, network, and associated Docker resources.
+     *
+     * This function ties together both the **execution** and **cleanup** stages for a containerized operation.
+     *
+     * The steps are:
+     * - Emits 'start' → Pulls container image (if needed) → Creates volume/network (if needed)
+     * - Runs container with correct args/env/entrypoint
+     * - Follows logs and emits 'log' → Waits for completion → Emits 'exit' or 'error'
+     * - Regardless of outcome, cleans up resources (containers, networks, optional image)
+     *
+     * Emits:
+     * - 'start': when the operation begins
+     * - 'log': every time stdout/stderr produces output
+     * - 'updateOpState': to attach container ID
+     * - 'exit': when the operation finishes successfully or with failure
+     * - 'error': when something crashes
+     * - 'end': always, after cleanup
+     *
+     * @param flow - The job flow this operation is part of
+     * @param op - The specific container operation to run
+     * @param controller - AbortController to support cancellation
+     * @param emitter - EventEmitter to stream logs and events from the container lifecycle
+     */
+    taskManagerContainerStartRunandStopOperation(flow: Flow, op: Operation<'container/run'>, controller: AbortController, emitter: EventEmitter): Promise<void>;
+    taskManagerVolumeStartRunandStopOperation(flow: Flow, op: Operation<'container/create-volume'>, controller: AbortController, emitter: EventEmitter): Promise<void>;
+    taskManagerVolumeCreateOperation(flow: Flow, op: Operation<'container/create-volume'>, controller: AbortController, emitter: EventEmitter): Promise<void>;
+    taskManagerVolumeStopOperation(flow: Flow, op: Operation<'container/create-volume'>, emitter?: EventEmitter): Promise<void>;
+    runTaskManagerOperation(flow: Flow, op: Operation<OperationType>, controller: AbortController, emitter: EventEmitter): Promise<void>;
+    stopTaskManagerOperation(flow: Flow, op: Operation<OperationType>, emitter?: EventEmitter): Promise<void>;
+}
+export declare function parseOpArgsCmd(cmd: OperationArgsMap['container/run']['cmd']): string[] | undefined;