@nosana/node 1.1.41-rc → 1.1.43-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.41-rc",
+    "version": "1.1.43-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/node/task/TaskManager.js

@@ -307,6 +307,12 @@ export default class TaskManager {
                     });
                     // Queue each operation in the group for execution
                     for (const id of p.ops) {
+                        // Only an operation still pending has yet to run: one released by
+                        // the operation it depends on runs in that operation's group, which
+                        // can be earlier than this one.
+                        if (this.operationStatus.get(id) !== OperationProgressStatuses.PENDING) {
+                            continue;
+                        }
                         const dependencyContext = this.dependecyMap.get(id);
                         const dependencies = dependencyContext.dependencies || [];
                         const depsSatisfied = dependencies.length === 0 ||

package/dist/src/NodeManager/provider/Provider.js

@@ -131,151 +131,150 @@ export class Provider {
         try {
             emitter.emit('start');
             let frpcContainer;
-            let container = await this.containerOrchestration.getContainer(op.id);
-            const exist = await this.containerOrchestration.doesContainerExist(op.id);
-            const exited = await this.containerOrchestration.isContainerExited(op.id);
-            if (exist && !exited && container.id) {
-                stateManager = new ContainerStateManager(container, controller, emitter, op.args.restart_policy);
-                await stateManager.startMonitoring();
-                // Reused container is already running (no pull needed) — start the
-                // op-execution timeout clock here too.
-                emitter.emit('container:started');
-            }
-            else {
-                await this.containerOrchestration.pullImage(op.args.image, op.args.authentication?.docker, controller);
-                if (!op.args.authentication?.docker) {
-                    this.resourceManager.images.setImage(op.args.image);
-                }
-                const index = getOpStateIndex(flow.jobDefinition.ops, op.id);
-                const name = flow.id + '-' + index;
-                const volumes = getVolumes(op.args, flow);
-                const gpu = getGpu(op.args, flow);
-                const entrypoint = getEntrypoint(op.args, flow);
-                const work_dir = getWorkingDir(op.args, flow);
-                const cmd = parseOpArgsCmd(op.args.cmd);
-                const env = {
-                    ...getGlobalEnv(flow),
-                    ...op.args.env,
-                    NOSANA_ID: flow.id,
-                };
-                const aliases = getAliases(op.args);
-                const networks = {};
-                let idMaps = new Map();
-                const ports = getExposePorts(op);
-                networks[name] = {};
-                await this.containerOrchestration.createNetwork(name, controller);
-                if (isOpExposed(op)) {
-                    await this.containerOrchestration.pullImage(this.frpcImage, undefined, controller);
-                    this.resourceManager.images.setImage(this.frpcImage);
-                    /**
-                     * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
-                     * deterministic url generation
-                     */
-                    const config = NodeConfigsSingleton.getInstance();
-                    const isLoadBalanced = !!flow.jobDefinition.deployment_id;
-                    const deploymentHash = isLoadBalanced
-                        ? // @ts-ignore
-                            config.options.isNodeRun
-                                ? flow.jobDefinition.deployment_id
-                                : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
-                        : undefined;
-                    const { proxies, idMap } = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
-                    idMaps = idMap;
-                    frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash));
-                    if (op.args.private) {
-                        this.repository.updateflowStateSecret(flow.id, {
-                            [flow.id]: generateUrlSecretObject(idMap, op.id),
-                            urlmode: 'private',
-                        });
-                        emitter.emit('flow:secrets-updated', {
-                            flowId: flow.id,
-                            opId: op.id,
-                        });
-                    }
-                    else {
-                        const newSecret = generateUrlSecretObject(idMap, op.id);
-                        const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
-                        const mergedSecrets = {
-                            ...currentSecrets,
-                            ...newSecret,
-                        };
-                        this.repository.updateflowStateSecret(flow.id, {
-                            [flow.id]: mergedSecrets,
-                            urlmode: 'public',
-                        });
-                        emitter.emit('flow:secrets-updated', {
-                            flowId: flow.id,
-                            opId: op.id,
-                        });
-                    }
-                }
-                if (op.args.resources) {
-                    await this.containerOrchestration.pullImage(s3HelperImage, undefined, controller);
-                    this.resourceManager.images.setImage(s3HelperImage);
-                    // transformCollections has already resolved any SpreadMarker objects by this point
-                    const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index });
-                    volumes.push(...resourceVolumes);
+            // A resumed flow re-runs operations that may already have a container, and
+            // the name below is the only handle on it.
+            const index = getOpStateIndex(flow.jobDefinition.ops, op.id);
+            const name = flow.id + '-' + index;
+            // An operation runs the same way whether or not the flow it belongs to has
+            // already run, so whatever a previous run left behind under these names is
+            // removed rather than resumed from. The runtime refuses to create a
+            // container whose name is still taken, and all three are recreated below.
+            for (const leftover of [name, `frpc-${name}`, `bridge-${name}`]) {
+                if (await this.containerOrchestration.doesContainerExist(leftover)) {
+                    await this.containerOrchestration.stopAndDeleteContainer(leftover);
                 }
-                // The op only declares that it wants a TEE; the node's configured
-                // runtime decides how that is fulfilled.
-                const teeRuntime = op.args.trusted_execution_env
-                    ? this.containerOrchestration.teeRuntime
+            }
+            await this.containerOrchestration.pullImage(op.args.image, op.args.authentication?.docker, controller);
+            if (!op.args.authentication?.docker) {
+                this.resourceManager.images.setImage(op.args.image);
+            }
+            const volumes = getVolumes(op.args, flow);
+            const gpu = getGpu(op.args, flow);
+            const entrypoint = getEntrypoint(op.args, flow);
+            const work_dir = getWorkingDir(op.args, flow);
+            const cmd = parseOpArgsCmd(op.args.cmd);
+            const env = {
+                ...getGlobalEnv(flow),
+                ...op.args.env,
+                NOSANA_ID: flow.id,
+            };
+            const aliases = getAliases(op.args);
+            const networks = {};
+            let idMaps = new Map();
+            const ports = getExposePorts(op);
+            networks[name] = {};
+            await this.containerOrchestration.createNetwork(name, controller);
+            if (isOpExposed(op)) {
+                await this.containerOrchestration.pullImage(this.frpcImage, undefined, controller);
+                this.resourceManager.images.setImage(this.frpcImage);
+                /**
+                 * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
+                 * deterministic url generation
+                 */
+                const config = NodeConfigsSingleton.getInstance();
+                const isLoadBalanced = !!flow.jobDefinition.deployment_id;
+                const deploymentHash = isLoadBalanced
+                    ? // @ts-ignore
+                        config.options.isNodeRun
+                            ? flow.jobDefinition.deployment_id
+                            : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
                     : undefined;
-                const createContainerOptions = {
-                    name,
-                    cmd,
-                    env,
-                    networks,
-                    requires_network_mode: isOpExposed(op),
-                    gpu,
-                    entrypoint,
-                    work_dir,
-                    volumes,
-                    aliases,
-                    restart_policy: op.args.restart_policy,
-                    runtime: teeRuntime,
-                };
-                if (teeRuntime === 'SEV-SNP') {
-                    await this.containerOrchestration.pullImage("busybox");
-                    await this.resourceManager.images.setImage("busybox");
-                    const bridgeContainer = await this.containerOrchestration.runFlowContainer("busybox", {
-                        name: `bridge-${name}`,
-                        cmd: ['sleep', 'infinity'],
-                        aliases: createContainerOptions.aliases
+                const { proxies, idMap } = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
+                idMaps = idMap;
+                frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash));
+                if (op.args.private) {
+                    this.repository.updateflowStateSecret(flow.id, {
+                        [flow.id]: generateUrlSecretObject(idMap, op.id),
+                        urlmode: 'private',
+                    });
+                    emitter.emit('flow:secrets-updated', {
+                        flowId: flow.id,
+                        opId: op.id,
                     });
-                    delete createContainerOptions.aliases;
-                    createContainerOptions.bind_network_to_container = `container:bridge-${name}`;
-                    const bridgeInfo = await bridgeContainer.inspect();
-                    if (bridgeInfo.NetworkSettings?.Networks["NOSANA_GATEWAY"].IPAddress) {
-                        emitter.emit('setContainerInternalIp', bridgeInfo.NetworkSettings.Networks["NOSANA_GATEWAY"].IPAddress);
-                    }
-                }
-                container = await this.containerOrchestration.runFlowContainer(op.args.image ?? flow.jobDefinition.global?.image, createContainerOptions);
-                // The container is now running (image pull is already done). This is the
-                // point the op-execution timeout clock starts from, so pull time isn't
-                // counted against it.
-                emitter.emit('container:started');
-                const info = await container.inspect();
-                emitter.emit('updateOpState', { providerId: container.id });
-                if (teeRuntime !== 'SEV-SNP' && info.NetworkSettings?.Networks["NOSANA_GATEWAY"].IPAddress) {
-                    emitter.emit('setContainerInternalIp', info.NetworkSettings.Networks["NOSANA_GATEWAY"].IPAddress);
-                }
-                stateManager = new ContainerStateManager(container, controller, emitter, op.args.restart_policy);
-                await stateManager.startMonitoring();
-                if (isOpExposed(op)) {
-                    exposedPortHealthCheck = new ExposedPortHealthCheck(flow.id, frpcContainer, emitter, name);
-                    exposedPortHealthCheck.addExposedPortsMap(idMaps);
-                    exposedPortHealthCheck.startServiceExposedUrlHealthCheck();
                 }
                 else {
-                    emitter.emit('healthcheck:startup:success');
+                    const newSecret = generateUrlSecretObject(idMap, op.id);
+                    const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
+                    const mergedSecrets = {
+                        ...currentSecrets,
+                        ...newSecret,
+                    };
+                    this.repository.updateflowStateSecret(flow.id, {
+                        [flow.id]: mergedSecrets,
+                        urlmode: 'public',
+                    });
+                    emitter.emit('flow:secrets-updated', {
+                        flowId: flow.id,
+                        opId: op.id,
+                    });
+                }
+            }
+            if (op.args.resources) {
+                await this.containerOrchestration.pullImage(s3HelperImage, undefined, controller);
+                this.resourceManager.images.setImage(s3HelperImage);
+                // transformCollections has already resolved any SpreadMarker objects by this point
+                const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index });
+                volumes.push(...resourceVolumes);
+            }
+            // The op only declares that it wants a TEE; the node's configured
+            // runtime decides how that is fulfilled.
+            const teeRuntime = op.args.trusted_execution_env
+                ? this.containerOrchestration.teeRuntime
+                : undefined;
+            const createContainerOptions = {
+                name,
+                cmd,
+                env,
+                networks,
+                requires_network_mode: isOpExposed(op),
+                gpu,
+                entrypoint,
+                work_dir,
+                volumes,
+                aliases,
+                restart_policy: op.args.restart_policy,
+                runtime: teeRuntime,
+            };
+            if (teeRuntime === 'SEV-SNP') {
+                await this.containerOrchestration.pullImage("busybox");
+                await this.resourceManager.images.setImage("busybox");
+                const bridgeContainer = await this.containerOrchestration.runFlowContainer("busybox", {
+                    name: `bridge-${name}`,
+                    cmd: ['sleep', 'infinity'],
+                    aliases: createContainerOptions.aliases
+                });
+                delete createContainerOptions.aliases;
+                createContainerOptions.bind_network_to_container = `container:bridge-${name}`;
+                const bridgeInfo = await bridgeContainer.inspect();
+                if (bridgeInfo.NetworkSettings?.Networks["NOSANA_GATEWAY"].IPAddress) {
+                    emitter.emit('setContainerInternalIp', bridgeInfo.NetworkSettings.Networks["NOSANA_GATEWAY"].IPAddress);
                 }
-                await stateManager.waitForExit();
-                // `wait()` reporting the exit and the log stream finishing its final
-                // flush are independent; drain the stream before results are read off
-                // the logs so the last line (e.g. the op's result) isn't lost.
-                await stateManager.waitForLogsDrained();
             }
+            const container = await this.containerOrchestration.runFlowContainer(op.args.image ?? flow.jobDefinition.global?.image, createContainerOptions);
+            // The container is now running (image pull is already done). This is the
+            // point the op-execution timeout clock starts from, so pull time isn't
+            // counted against it.
+            emitter.emit('container:started');
+            const startedInfo = await container.inspect();
+            emitter.emit('updateOpState', { providerId: container.id });
+            if (teeRuntime !== 'SEV-SNP' && startedInfo.NetworkSettings?.Networks["NOSANA_GATEWAY"].IPAddress) {
+                emitter.emit('setContainerInternalIp', startedInfo.NetworkSettings.Networks["NOSANA_GATEWAY"].IPAddress);
+            }
+            stateManager = new ContainerStateManager(container, controller, emitter, op.args.restart_policy);
+            await stateManager.startMonitoring();
+            if (isOpExposed(op)) {
+                exposedPortHealthCheck = new ExposedPortHealthCheck(flow.id, frpcContainer, emitter, name);
+                exposedPortHealthCheck.addExposedPortsMap(idMaps);
+                exposedPortHealthCheck.startServiceExposedUrlHealthCheck();
+            }
+            else {
+                emitter.emit('healthcheck:startup:success');
+            }
+            await stateManager.waitForExit();
+            // `wait()` reporting the exit and the log stream finishing its final
+            // flush are independent; drain the stream before results are read off
+            // the logs so the last line (e.g. the op's result) isn't lost.
+            await stateManager.waitForLogsDrained();
             const info = await promiseTimeoutWrapper(container.inspect({
                 abortSignal: liveAbortSignal(controller.signal),
             }), 360, controller);

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.d.ts

@@ -32,7 +32,6 @@ export declare class DockerContainerOrchestration implements ContainerOrchestrat
     getRawVolume(name: string): Promise<Volume>;
     deleteVolume(name: string, controller?: AbortController): Promise<void>;
     healthy(): Promise<ReturnedStatus>;
-    getContainer(id: string): Promise<Container>;
     getProtocol(): string;
     setupContainerAbortListener(containerId: string, controller: AbortController): void;
     runContainer(args: ContainerCreateOptions, controller?: AbortController): Promise<Container>;

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js

@@ -193,10 +193,6 @@ export class DockerContainerOrchestration {
             return { status: false, error };
         }
     }
-    async getContainer(id) {
-        const container = this.docker.getContainer(id);
-        return container;
-    }
     getProtocol() {
         return this.protocol;
     }

package/dist/src/NodeManager/provider/containerOrchestration/interface.d.ts

@@ -43,7 +43,6 @@ export interface ContainerOrchestrationInterface {
     deleteVolume(name: string, controller?: AbortController): Promise<void>;
     getContainerByName(name: string): Promise<Container | undefined>;
     getContainersByName(names: string[]): Promise<Container[]>;
-    getContainer(id: string): Promise<Container>;
     runContainer(args: ContainerCreateOptions, controller?: AbortController): Promise<Container>;
     runFlowContainer(image: string, args: RunContainerArgs): Promise<Container>;
     stopContainer(id: string, controller?: AbortController): Promise<void>;

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.41-rc",
+  "version": "1.1.43-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.41-rc",
+      "version": "1.1.43-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.41-rc",
+  "version": "1.1.43-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"