@nosana/node 1.1.25 → 1.1.27-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.25",
+    "version": "1.1.27-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/node/api/ApiHandler.js

@@ -7,7 +7,7 @@ import { sleep } from '../../utils/utils.js';
 import { stateStreaming } from '../../monitoring/streaming/StateStreamer.js';
 import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
 import { verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware, verifyWSNodeOrJobOwnerSignatureMiddleware, } from './middlewares/index.js';
-import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, moveGroupOperationHandler, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, } from './routes/index.js';
+import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, } from './routes/index.js';
 import { NodeAlreadyActiveError } from '../../errors/NodeAlreadyActiveError.js';
 export class ApiHandler {
     constructor(sdk, repository, provider, port) {
@@ -147,7 +147,6 @@ export class ApiHandler {
         this.api.get('/job/:jobId/stats/stream', getJobStatsStreamRoute);
         // POST Routes
         this.api.post('/job/:jobId/job-definition', postJobDefinitionRoute);
-        this.api.post('/job/:jobId/group/:group/move', moveGroupOperationHandler);
         this.api.post('/job/:jobId/group/:group/operation/:opId/restart', restartOperationHandler);
         this.api.post('/job/:jobId/group/:group/restart', restartGroupOperationHandler);
         this.api.post('/job/:jobId/group/:group/operation/:opId/stop', stopOperationHandler);

package/dist/src/NodeManager/node/api/routes/post/job-operation-actions.d.ts

@@ -9,10 +9,6 @@ export declare function restartGroupOperationHandler(req: NodeAPIRequest<{
     jobId: string;
     group: string;
 }>, res: Response): Promise<Response<any, Record<string, any>>>;
-export declare function moveGroupOperationHandler(req: NodeAPIRequest<{
-    jobId: string;
-    group: string;
-}>, res: Response): Promise<Response<any, Record<string, any>>>;
 export declare function stopOperationHandler(req: NodeAPIRequest<{
     jobId: string;
     opId: string;

package/dist/src/NodeManager/node/api/routes/post/job-operation-actions.js

@@ -7,7 +7,7 @@ export async function restartOperationHandler(req, res) {
     }
     try {
         await task.restartTaskManagerOperation(group, opId);
-        return res.status(200).json({ message: 'Operation restarted' });
+        return res.status(202).json({ message: 'Operation restart initiated' });
     }
     catch (error) {
         return res
@@ -23,23 +23,7 @@ export async function restartGroupOperationHandler(req, res) {
     }
     try {
         await task.restartTaskManagerGroupOperations(group);
-        return res.status(200).json({ message: 'Group Operation restarted' });
-    }
-    catch (error) {
-        return res
-            .status(500)
-            .json({ error: 'Failed to restart group operations' });
-    }
-}
-export async function moveGroupOperationHandler(req, res) {
-    const { jobId, group } = req.params;
-    const task = TaskManagerRegistry.getInstance().get(jobId);
-    if (!task) {
-        return res.status(400).send('invalid job id');
-    }
-    try {
-        await task.moveTaskManagerGroupOperations(group);
-        return res.status(200).json({ message: 'Group Operation restarted' });
+        return res.status(202).json({ message: 'Group Operation restart initiated' });
     }
     catch (error) {
         return res

package/dist/src/NodeManager/node/task/TaskManager.d.ts

@@ -212,7 +212,6 @@ export default class TaskManager {
     stopTaskManagerOperation: (group: string, opId: string) => Promise<void>;
     stopTaskManagerGroupOperations: (group: string) => Promise<void>;
     restartTaskManagerGroupOperations: (group: string) => Promise<void>;
-    moveTaskManagerGroupOperations: (group: string) => Promise<void>;
     stopAllTaskManagerOperations: (reason: StopReason) => void;
     createOperationMap: () => Map<string, Operation<OperationType>>;
     createExecutionPlan: () => ExecutionContext[];
@@ -282,7 +281,8 @@ export default class TaskManager {
      * - Sets status to 'running' and updates the repository with start time.
      * - Iterates through execution groups and runs each op concurrently.
      * - Uses a dynamic while-loop to ensure all ops (including restarts) finish before advancing.
-     * - After all groups finish, updates the flow state to 'success' or keeps previous status.
+     * - After all groups finish, sets the final flow status: 'failed' if any op
+     *   failed, 'success' otherwise (including stopped/expired jobs).
      * - On any uncaught failure, marks the flow as 'failed' with end time.
      */
     start(): Promise<void>;

package/dist/src/NodeManager/node/task/TaskManager.js

@@ -13,7 +13,6 @@ import { createDependencyMap } from './executions/createDependencyMap.js';
 import { getCurrentGroup, getCurrentGroupStatus, getGroupStatus, getOperationsStatus, getOperationStatus, } from './operations/getOperationsInfos.js';
 import { addLog, getAllLogs, getLogsByGroup, getLogsByOp, subscribe, unsubscribe, } from './loggers/logManager.js';
 import { addStat, getAllStats, getStatsByOp, queryStats, getLatestStatPerOp, subscribeStats, unsubscribeStats, } from './loggers/statsManager.js';
-import { moveTaskManagerGroupOperations } from './operations/moveTaskManagerGroupOperation.js';
 import { setResult, setResults, setHost, setDefaults, setContainerIp, rehydrateEndpointsForOperation, getByPath, resolveLiteralsInString, interpolate, interpolateOperation, transformCollections, } from './globalStore/index.js';
 import { configs } from '../../configs/configs.js';
 import { getSDK } from '../../sdk/index.js';
@@ -148,8 +147,6 @@ export default class TaskManager {
             stopTaskManagerGroupOperations.bind(this);
         this.restartTaskManagerGroupOperations =
             restartTaskManagerGroupOperations.bind(this);
-        this.moveTaskManagerGroupOperations =
-            moveTaskManagerGroupOperations.bind(this);
         this.stopAllTaskManagerOperations = stopAllTaskManagerOperations.bind(this);
         this.createOperationMap = createOperationMap.bind(this);
         this.createExecutionPlan = createExecutionPlan.bind(this);
@@ -273,7 +270,8 @@ export default class TaskManager {
      * - Sets status to 'running' and updates the repository with start time.
      * - Iterates through execution groups and runs each op concurrently.
      * - Uses a dynamic while-loop to ensure all ops (including restarts) finish before advancing.
-     * - After all groups finish, updates the flow state to 'success' or keeps previous status.
+     * - After all groups finish, sets the final flow status: 'failed' if any op
+     *   failed, 'success' otherwise (including stopped/expired jobs).
      * - On any uncaught failure, marks the flow as 'failed' with end time.
      */
     async start() {
@@ -362,8 +360,14 @@ export default class TaskManager {
                         });
                     }
                 }
-                // If status is still 'running', mark the job as successful
-                this.status = this.status === 'running' ? 'success' : this.status;
+                // The final status only reflects the workload outcome: 'failed' if any
+                // operation failed, 'success' otherwise. How the job ended (stopped,
+                // expired) is tracked in the op statuses/diagnostics, so
+                // ops that were merely stopped don't fail the job.
+                const hasFailedOp = this.repository
+                    .getFlowState(this.job)
+                    .opStates.some((opState) => opState.status === Statuses.FAILED);
+                this.status = hasFailedOp ? Statuses.FAILED : Statuses.SUCCESS;
                 this.repository.updateflowState(this.job, {
                     status: this.status,
                     endTime: Date.now(),

package/dist/src/NodeManager/node/task/operations/restartTaskManagerOperation.js

@@ -33,6 +33,14 @@ export async function restartTaskManagerOperation(group, opId) {
     if (this.lockedOperations.has(opId)) {
         throw new Error(`OPERATION_${this.lockedOperations.get(opId)}`);
     }
+    /**
+     * Validate the operation exists up front, before we acquire the lock or kick
+     * off any async work. This lets the caller (HTTP handler) reject an invalid
+     * opId synchronously, and avoids leaving the operation permanently locked if
+     * the lookup were to fail later in the flow.
+     */
+    if (!this.opMap.has(opId))
+        throw new Error(`INVALID_OPID`);
     // Mark this op as "RESTARTING" so no one else touches it mid-process
     this.lockedOperations.set(opId, 'RESTARTING');
     this.operationStatus.set(opId, OperationProgressStatuses.RESTARTING);
@@ -77,41 +85,57 @@ export async function restartTaskManagerOperation(group, opId) {
     // This should trigger container teardown, logs finalization, and state updates
     controller?.abort('restart');
     /**
-     * If the operation had already started, wait for it to finish fully (even if it failed).
-     * This avoids stomping over any still-pending teardown work.
+     * The container teardown + relaunch below can take a while (graceful container
+     * stop, image checks, etc). We deliberately DO NOT await it here: this function
+     * is invoked from an HTTP handler whose response is tunneled through frp, and
+     * holding the response open for the whole teardown means frp has no live backend
+     * for that window and serves its "no backend" (503) page to the caller.
+     *
+     * Everything that guards against races — the lock, the RESTARTING status and the
+     * group-hold placeholder — has already been put in place synchronously above, so
+     * it is safe to let the heavy lifting run in the background and return now. The
+     * caller gets an immediate acknowledgement that the restart was initiated.
      */
-    const originalPromise = this.currentGroupOperationsPromises.get(opId);
-    if (originalPromise) {
+    void (async () => {
         try {
-            await originalPromise;
+            /**
+             * If the operation had already started, wait for it to finish fully (even if it
+             * failed). This avoids stomping over any still-pending teardown work.
+             */
+            const originalPromise = this.currentGroupOperationsPromises.get(opId);
+            if (originalPromise) {
+                try {
+                    await originalPromise;
+                }
+                catch {
+                    // We intentionally ignore any error here.
+                    // Our goal is just to make sure the cleanup is finished, not whether it succeeded.
+                }
+            }
+            /**
+             * Now that the original op has been stopped, we need to restart it.
+             * Get the latest flow data from storage — this could've been updated elsewhere
+             * in the meantime.
+             */
+            const flow = this.repository.getFlow(this.job);
+            /**
+             * Re-register the operation for execution and begin tracking it again.
+             * This effectively restarts the operation from scratch.
+             */
+            this.currentGroupOperationsPromises.set(opId, this.trackGroupOperationPromise(opId, this.setUpOperationFunc(flow, opId, [])));
         }
-        catch {
-            // We intentionally ignore any error here.
-            // Our goal is just to make sure the cleanup is finished, not whether it succeeded.
+        catch (error) {
+            emitter?.emit('log', `Failed to restart operation: ${error?.message ?? error}`, 'error');
         }
-    }
-    /**
-     * Now that the original op has been stopped, we need to restart it.
-     * Get the latest flow data from storage — this could’ve been updated elsewhere in the meantime.
-     */
-    const flow = this.repository.getFlow(this.job);
-    /**
-     * Look up the original operation details.
-     * This should always exist unless something went very wrong.
-     */
-    const op = this.opMap.get(opId);
-    if (!op)
-        throw new Error(`INVALID_OPID`);
-    /**
-     * Re-register the operation for execution and begin tracking it again.
-     * This effectively restarts the operation from scratch.
-     */
-    this.currentGroupOperationsPromises.set(opId, this.trackGroupOperationPromise(opId, this.setUpOperationFunc(flow, opId, [])));
-    /**
-     * Now that we've re-launched the op and it's being tracked,
-     * we can safely release the group hold so the group can eventually complete.
-     */
-    groupHold.releaseGroupOperationBlock();
-    // Finally, unlock the operation so other calls (like restart or stop) can act on it again
-    this.lockedOperations.delete(opId);
+        finally {
+            /**
+             * Release the group hold so the group can eventually complete, and unlock the
+             * operation so other calls (like restart or stop) can act on it again. This runs
+             * regardless of whether the relaunch succeeded, so we never leave the group held
+             * or the operation permanently locked.
+             */
+            groupHold.releaseGroupOperationBlock();
+            this.lockedOperations.delete(opId);
+        }
+    })();
 }

package/dist/src/NodeManager/node/task/operations/stopAllTaskManagerOperations.d.ts

@@ -7,15 +7,19 @@ import TaskManager, { StopReason } from '../TaskManager.js';
  * - Lifecycle shutdown for any running ops
  * - A consistent status applied to the flow and all ops
  *
- * The stop reason determines what status each op and the overall flow will be marked with.
+ * The stop reason determines what status each op will be marked with.
  *
- * | Reason   | Ops Status | Flow Status |
- * |----------|------------|-------------|
- * | expired  | success    | success     |
- * | stopped  | stopped    | stopped     |
- * | quit     | failed     | failed      |
- * | unknown  | failed     | failed      |
+ * | Reason   | Ops Status |
+ * |----------|------------|
+ * | expired  | success    |
+ * | stopped  | stopped    |
+ * | quit     | failed     |
+ * | unknown  | failed     |
  *
  * `getStatus()` is used internally to apply the correct status values.
+ *
+ * The flow status set here is transient: the final result status is derived
+ * from the op statuses when `start()` completes — 'failed' if any op failed,
+ * 'success' otherwise.
  */
 export declare function stopAllTaskManagerOperations(this: TaskManager, reason: StopReason): void;

package/dist/src/NodeManager/node/task/operations/stopAllTaskManagerOperations.js

@@ -7,16 +7,20 @@ import { OperationProgressStatuses, } from '../TaskManager.js';
  * - Lifecycle shutdown for any running ops
  * - A consistent status applied to the flow and all ops
  *
- * The stop reason determines what status each op and the overall flow will be marked with.
+ * The stop reason determines what status each op will be marked with.
  *
- * | Reason   | Ops Status | Flow Status |
- * |----------|------------|-------------|
- * | expired  | success    | success     |
- * | stopped  | stopped    | stopped     |
- * | quit     | failed     | failed      |
- * | unknown  | failed     | failed      |
+ * | Reason   | Ops Status |
+ * |----------|------------|
+ * | expired  | success    |
+ * | stopped  | stopped    |
+ * | quit     | failed     |
+ * | unknown  | failed     |
  *
  * `getStatus()` is used internally to apply the correct status values.
+ *
+ * The flow status set here is transient: the final result status is derived
+ * from the op statuses when `start()` completes — 'failed' if any op failed,
+ * 'success' otherwise.
  */
 export function stopAllTaskManagerOperations(reason) {
     /**

package/dist/src/NodeManager/node/task/operations/stopTaskManagerOperation.js

@@ -63,75 +63,89 @@ export async function stopTaskManagerOperation(group, opId) {
      */
     controller?.abort(StopReasons.STOPPED);
     /**
-     * Wait for the original promise tied to this operation to resolve.
-     * Even if it fails (e.g., due to abort), we silently swallow the error
-     * since the focus here is cleanup, not success.
+     * The container teardown below can take a while (graceful container stop, etc).
+     * We deliberately DO NOT await it here: this function is invoked from an HTTP
+     * handler whose response is tunneled through frp, and holding the response open
+     * for the whole teardown means frp has no live backend for that window and serves
+     * its "no backend" (503) page to the caller.
+     *
+     * The lock and the STOPPING status have already been set synchronously above, so
+     * it is safe to let the teardown — and the STOPPED-state finalization that depends
+     * on it — run in the background and return now. The caller gets an immediate
+     * acknowledgement that the stop was initiated.
      */
-    const originalPromise = this.currentGroupOperationsPromises.get(opId);
-    if (originalPromise) {
-        try {
-            await originalPromise;
+    void (async () => {
+        /**
+         * Wait for the original promise tied to this operation to resolve.
+         * Even if it fails (e.g., due to abort), we silently swallow the error
+         * since the focus here is cleanup, not success.
+         */
+        const originalPromise = this.currentGroupOperationsPromises.get(opId);
+        if (originalPromise) {
+            try {
+                await originalPromise;
+            }
+            catch {
+                // swallow error, we're only waiting for cleanup
+            }
         }
-        catch {
-            // swallow error, we're only waiting for cleanup
+        /**
+         * Ensure we clean up the operation's reference in the promise map
+         * to avoid holding stale references or leaking memory.
+         */
+        this.currentGroupOperationsPromises.delete(opId);
+        // Unlock the operation so it can be restarted or reused later
+        this.lockedOperations.delete(opId);
+        // Ensure final STOPPED state is reflected in memory and repository
+        this.operationStatus.set(opId, OperationProgressStatuses.STOPPED);
+        const index = this.getOpStateIndex(opId);
+        const opState = this.repository.getOpState(this.job, index);
+        const alreadyEnded = !!opState.endTime;
+        if (!alreadyEnded) {
+            this.repository.updateOpState(this.job, index, {
+                status: this.getStatus(StopReasons.STOPPED, 'ops'),
+                endTime: Date.now(),
+            });
         }
-    }
-    /**
-     * Ensure we clean up the operation's reference in the promise map
-     * to avoid holding stale references or leaking memory.
-     */
-    this.currentGroupOperationsPromises.delete(opId);
-    // Unlock the operation so it can be restarted or reused later
-    this.lockedOperations.delete(opId);
-    // Ensure final STOPPED state is reflected in memory and repository
-    this.operationStatus.set(opId, OperationProgressStatuses.STOPPED);
-    const index = this.getOpStateIndex(opId);
-    const opState = this.repository.getOpState(this.job, index);
-    const alreadyEnded = !!opState.endTime;
-    if (!alreadyEnded) {
-        this.repository.updateOpState(this.job, index, {
-            status: this.getStatus(StopReasons.STOPPED, 'ops'),
-            endTime: Date.now(),
+        this.events.emit('flow:updated', {
+            jobId: this.job,
+            opId,
+            type: 'status:stopped',
         });
-    }
-    this.events.emit('flow:updated', {
-        jobId: this.job,
-        opId,
-        type: 'status:stopped',
-    });
-    try {
-        const flow = this.repository.getFlow(this.job);
-        const secrets = flow?.state?.secrets;
-        let jobSecrets = {};
-        if (secrets &&
-            typeof secrets[this.job] === 'object' &&
-            secrets[this.job] !== 'private' &&
-            secrets[this.job] !== 'public' &&
-            secrets[this.job]) {
-            jobSecrets = secrets[this.job];
-        }
-        let updated = false;
-        const newJobSecrets = {};
-        for (const [exposeId, value] of Object.entries(jobSecrets)) {
-            const v = value;
-            if (v && typeof v === 'object' && v.opID === opId) {
-                newJobSecrets[exposeId] = { ...v, status: 'OFFLINE' };
-                updated = true;
+        try {
+            const flow = this.repository.getFlow(this.job);
+            const secrets = flow?.state?.secrets;
+            let jobSecrets = {};
+            if (secrets &&
+                typeof secrets[this.job] === 'object' &&
+                secrets[this.job] !== 'private' &&
+                secrets[this.job] !== 'public' &&
+                secrets[this.job]) {
+                jobSecrets = secrets[this.job];
             }
-            else {
-                newJobSecrets[exposeId] = v;
+            let updated = false;
+            const newJobSecrets = {};
+            for (const [exposeId, value] of Object.entries(jobSecrets)) {
+                const v = value;
+                if (v && typeof v === 'object' && v.opID === opId) {
+                    newJobSecrets[exposeId] = { ...v, status: 'OFFLINE' };
+                    updated = true;
+                }
+                else {
+                    newJobSecrets[exposeId] = v;
+                }
+            }
+            if (updated) {
+                this.repository.updateflowStateSecret(this.job, {
+                    [this.job]: newJobSecrets,
+                });
+                this.events.emit('flow:updated', {
+                    jobId: this.job,
+                    opId,
+                    type: 'flow:secrets-updated',
+                });
             }
         }
-        if (updated) {
-            this.repository.updateflowStateSecret(this.job, {
-                [this.job]: newJobSecrets,
-            });
-            this.events.emit('flow:updated', {
-                jobId: this.job,
-                opId,
-                type: 'flow:secrets-updated',
-            });
-        }
-    }
-    catch { }
+        catch { }
+    })();
 }

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.25",
+  "version": "1.1.27-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.25",
+      "version": "1.1.27-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.25",
+  "version": "1.1.27-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/node/task/operations/moveTaskManagerGroupOperation.d.ts

@@ -1,2 +0,0 @@
-import TaskManager from '../TaskManager.js';
-export declare function moveTaskManagerGroupOperations(this: TaskManager, group: string): Promise<void>;

package/dist/src/NodeManager/node/task/operations/moveTaskManagerGroupOperation.js

@@ -1,65 +0,0 @@
-export async function moveTaskManagerGroupOperations(group) {
-    /**
-     * Look up the group context to get its operation IDs.
-     * If the group doesn't exist in the execution plan, it's invalid.
-     */
-    const groupContext = this.executionPlan.find((ctx) => ctx.group === group);
-    if (!groupContext) {
-        throw new Error('GROUP_NOT_FOUND');
-    }
-    /**
-     * Ensure we're currently within the provided group context before proceeding.
-     * Prevents stopping inactive or completed groups.
-     */
-    if (this.currentGroup == group) {
-        throw new Error('GROUP_ACTIVE');
-    }
-    const currentGroupContext = this.executionPlan.find((ctx) => ctx.group === this.currentGroup);
-    if (!currentGroupContext) {
-        throw new Error('GROUP_NOT_FOUND');
-    }
-    /**
-     * The following block creates a "placeholder promise" that we temporarily insert into the group’s
-     * operation map. This ensures that even if all the original ops complete while we're restarting,
-     * the group will not advance to the next one until we explicitly release this block.
-     *
-     * This is CRITICAL to prevent race conditions where the restart is too late, and the system
-     * thinks the group has completed.
-     */
-    const groupHold = (() => {
-        let release;
-        const placeholder = new Promise((res) => (release = res));
-        const abortHandler = () => {
-            release();
-            this.mainAbortController.signal.removeEventListener('abort', abortHandler);
-        };
-        this.mainAbortController.signal.addEventListener('abort', abortHandler);
-        return {
-            groupHoldPromisePlaceHolder: placeholder,
-            releaseGroupOperationBlock: () => {
-                release();
-                this.mainAbortController.signal.removeEventListener('abort', abortHandler);
-            },
-        };
-    })();
-    // Use a special ID to register this placeholder in the group tracking map
-    // The op will still retain its original ID for restart purposes
-    const placeholderId = `${group}::restart-group`;
-    this.trackGroupOperationPromise(placeholderId, groupHold.groupHoldPromisePlaceHolder);
-    if (this.currentGroup) {
-        await this.stopTaskManagerGroupOperations(this.currentGroup);
-    }
-    /**
-     * update the current group to the group we about to start
-     */
-    this.currentGroup = group;
-    /**
-     * restart the new current group
-     */
-    await this.restartTaskManagerGroupOperations(group);
-    /**
-     * Now that we've re-launched the op and it's being tracked,
-     * we can safely release the group hold so the group can eventually complete.
-     */
-    groupHold.releaseGroupOperationBlock();
-}