@nosana/node 1.0.0-rc

package/dist/NodeManager/node/task/operations/runTaskManagerOperation.js

@@ -0,0 +1,488 @@
+import { EventEmitter } from 'events';
+import { Statuses, OperationProgressStatuses, StopReasons, } from '../TaskManager.js';
+import { finalizeEnvOnOperation } from '../globalStore/finalizeEnv.js';
+import { createResultsObject, extractResultFromLog, } from '../../utils/extractResultsFromLogs.js';
+import { stanatizeArrays } from '../globalStore/stanatizeArrays.js';
+import { logEmitter } from '../../../monitoring/proxy/loggingProxy.js';
+/**
+ * Executes a full lifecycle of a container-based operation using internal class state.
+ *
+ * This function is intended to be bound to a `TaskManager` instance using `.bind(this)`.
+ * It handles:
+ * - Setup
+ * - Log streaming
+ * - Abort controller registration
+ * - Container execution
+ * - Exit/error handling
+ * - Cleanup
+ *
+ * It uses internal state like `this.repository`, `this.provider`, `this.job`, etc.,
+ * and serves as the core execution driver for all operations.
+ *
+ * Keeps lifecycle doc comments in sync with original `setUpOperationFunc`.
+ */
+export async function runTaskManagerOperation(flow, op, dependent) {
+    /**
+     * Retrieves the index of the operation within the job definition's ops array.
+     *
+     * This index is used to look up and update the operation's state in the repository.
+     * It's necessary because the repository stores op states in an array, not a map,
+     * so index-based access is required for performance and mutation.
+     *
+     * The `getOpStateIndex` method is passed from the TaskManager to maintain context
+     * and ensure consistent lookup logic throughout the system.
+     */
+    const index = this.getOpStateIndex(op.id);
+    /**
+     * Creates a new EventEmitter instance that serves as the communication channel
+     * for the lifecycle of this operation.
+     * This emitter is used to handle logs, update operation state, and respond to various
+     * execution events like 'start', 'exit', 'error', and 'end'.
+     */
+    const emitter = new EventEmitter();
+    // register the emitter
+    this.registerAndRelayOpEmitter(op.id, emitter);
+    /**
+     * Initializes and registers an AbortController specific to this operation ID.
+     * This controller is used to handle cancellation of the operation.
+     * It is tied to the main task manager's AbortController, so a global stop
+     * will also trigger this one unless already aborted.
+     */
+    const abort = new AbortController();
+    const handleMainAbort = () => {
+        emitter.emit('log', 'Operation Stopping', 'info');
+        this.operationStatus.set(op.id, OperationProgressStatuses.STOPPING);
+        abort.abort(this.mainAbortController.signal.reason);
+        if ([StopReasons.EXPIRED, StopReasons.STOPPED, StopReasons.QUIT].includes(this.mainAbortController.signal.reason)) {
+            this.repository.setOpStateDiagnosticsReason(this.job, index, {
+                ...(this.mainAbortController.signal.reason === StopReasons.EXPIRED
+                    ? { jobExpired: true }
+                    : {}),
+                ...(this.mainAbortController.signal.reason === StopReasons.STOPPED
+                    ? { jobStopped: true }
+                    : {}),
+                ...(this.mainAbortController.signal.reason === StopReasons.QUIT
+                    ? { hostShutDown: true }
+                    : {}),
+                reason: `Operation stopped due to flow ${this.mainAbortController.signal.reason}`,
+            });
+        }
+    };
+    this.mainAbortController.signal.addEventListener('abort', handleMainAbort);
+    this.abortControllerMap.set(op.id, abort);
+    /**
+     * quit the operation if the main controller is already emmited
+     */
+    if (this.mainAbortController.signal.aborted) {
+        this.repository.updateOpState(this.job, index, {
+            exitCode: 0,
+            startTime: Date.now(),
+            endTime: Date.now(),
+            status: this.getStatus(this.mainAbortController.signal.reason, 'ops'),
+        });
+        return Promise.resolve();
+    }
+    /**
+     * Subscribes to 'log' events emitted during operation execution.
+     *
+     * These events are typically triggered by the operation’s execution engine (e.g., a container)
+     * to emit stdout/stderr logs. Every time a log is received, it's appended to the operation’s
+     * existing logs in the repository using `updateOpStateLogs`.
+     *
+     * This enables real-time or post-mortem log inspection, useful for debugging or UI display.
+     * Logs are stored in a buffer-like array, and later parsed during the `exit` or `error` phase
+     * to extract structured logs and results.
+     */
+    emitter.on('log', (log, logType, type, timestamp) => {
+        // add logs to the log manager
+        this.addlog({
+            opId: op.id,
+            group: this.currentGroup ?? '',
+            type: type,
+            message: log,
+            timestamp: Date.now(),
+        });
+        if (type == 'container') {
+            // FOR NOW Skip if it's an actual Error object
+            // TODO: Format errors and insert
+            if (log instanceof Error)
+                return;
+            this.repository.updateOpStateLogs(this.job, index, {
+                log,
+                type: logType || 'stdout',
+                timestamp: timestamp || Date.now().toString(),
+            });
+            this.repository.displayLog(log);
+        }
+    });
+    /**
+     * Subscribes to 'updateOpState' events that allow partial mutation of the operation's state.
+     *
+     * This is a flexible and general-purpose hook. It accepts a `body` payload which is a partial
+     * `OpState` object, which is merged into the existing operation state in the repository.
+     *
+     * Example use cases:
+     * - Updating metadata like progress percentage, internal flags, or status messages
+     * - Recording dynamic outputs or intermediate results during execution
+     *
+     * This gives the running operation a channel to self-report its own evolving state.
+     */
+    emitter.on('updateOpState', (body) => {
+        this.repository.updateOpState(this.job, index, body);
+    });
+    /**
+     * update the host name
+     */
+    emitter.on('updateOpHost', (body) => {
+        this.setHost(op.id, body.name);
+    });
+    /**
+     * Subscribes to the 'start' event, which marks the official beginning of an operation’s execution.
+     *
+     * When triggered, it updates the corresponding operation’s status to 'running' and records
+     * the `startTime` timestamp. This serves as the entry point for lifecycle tracking of this op.
+     *
+     * Important: This should be emitted **after** all setup is complete and just before the core
+     * execution logic begins (e.g., launching a container or invoking a function).
+     *
+     * This is crucial for analytics, timeout enforcement, and auditing purposes.
+     */
+    emitter.on('start', () => {
+        const opName = op.name || op.id;
+        this.addlog({
+            opId: op.id,
+            group: this.currentGroup ?? '',
+            type: 'info',
+            message: `Operation Started: ${opName}`,
+            timestamp: Date.now(),
+        });
+        this.operationStatus.set(op.id, OperationProgressStatuses.RUNNING);
+        this.repository.updateOpState(this.job, index, {
+            status: 'running',
+            startTime: Date.now(),
+            endTime: null,
+            exitCode: null,
+        });
+    });
+    /**
+     * Subscribes to the 'exit' event, which signals that the operation has finished executing.
+     *
+     * This event may fire under normal completion, error, or external cancellation (abort).
+     *
+     * What this handler does:
+     * - Determines whether the operation was aborted or completed naturally.
+     * - Uses the exit code and abort reason to infer the final operation status.
+     * - Converts raw log strings to a buffer, then extracts structured logs and results.
+     * - Updates the operation's state in the repository with logs, results, exit code, status, and end time.
+     */
+    emitter.on('exit', ({ exitCode }) => {
+        const opName = op.name || op.id;
+        this.addlog({
+            opId: op.id,
+            group: this.currentGroup ?? '',
+            type: 'info',
+            message: `Operation Completed: ${opName}`,
+            timestamp: Date.now(),
+        });
+        this.operationStatus.set(op.id, OperationProgressStatuses.FINISHED);
+        const wasAborted = abort.signal.aborted;
+        const reason = wasAborted ? abort.signal.reason : undefined;
+        const status = wasAborted
+            ? this.getStatus(reason, 'ops')
+            : exitCode === 0
+                ? Statuses.SUCCESS
+                : Statuses.FAILED;
+        const opState = this.repository.getOpState(this.job, index);
+        const results = op.results
+            ? createResultsObject(op.results)
+            : undefined;
+        if (results && op.results) {
+            for (const logObj of opState.logs) {
+                extractResultFromLog(results, logObj, op.results);
+            }
+        }
+        this.repository.updateOpState(this.job, index, {
+            logs: opState.logs,
+            results,
+            exitCode,
+            endTime: Date.now(),
+            status,
+        });
+        this.setResults(op.id, results ?? {});
+    });
+    /**
+     * Subscribes to the 'error' event, which is emitted when an unhandled exception or failure
+     * occurs during the operation's execution lifecycle.
+     *
+     * This could result from:
+     * - Internal execution issues (e.g., runtime crash, invalid state)
+     * - External interruptions (e.g., aborts triggered manually or by timeout)
+     *
+     * What this handler does:
+     * - Determines whether the operation was aborted or failed on its own.
+     * - Sets the final operation status accordingly.
+     * - Parses accumulated logs to extract clean output and structured results.
+     * - Updates the operation state with final logs, results, status, exit code, and end time.
+     *
+     * Notes:
+     * - `exitCode: 2` is used as a standard error indicator (non-zero, but distinct from process exit).
+     * - The actual `err` is not persisted currently, but could be added to logs in future.
+     */
+    emitter.on('error', (err) => {
+        const wasAborted = abort.signal.aborted;
+        const reason = wasAborted ? abort.signal.reason : undefined;
+        this.operationStatus.set(op.id, wasAborted
+            ? OperationProgressStatuses.STOPPED
+            : OperationProgressStatuses.FAILED);
+        const status = wasAborted
+            ? this.getStatus(reason, 'ops')
+            : Statuses.FAILED;
+        if (!wasAborted) {
+            emitter.emit('log', 'Operation Failed', 'info');
+            emitter.emit('log', err, 'error');
+        }
+        else {
+            emitter.emit('log', 'Operation Completed', 'info');
+        }
+        const opState = this.repository.getOpState(this.job, index);
+        const results = op.results
+            ? createResultsObject(op.results)
+            : undefined;
+        if (results && op.results) {
+            for (const logObj of opState.logs) {
+                extractResultFromLog(results, logObj, op.results);
+            }
+        }
+        // Format error message and add to opState.error array
+        if (!wasAborted) {
+            const message = ('message' in err && err.message) || String(err);
+            const event = ('eventType' in err && err.eventType) || 'operation-error';
+            let errorCode = undefined;
+            const httpCodeMatch = message.match(/HTTP code (\d{3})|status code (\d{3})/);
+            if (httpCodeMatch) {
+                errorCode = parseInt(httpCodeMatch[1] || httpCodeMatch[2]);
+            }
+            this.repository.updateOpStateError(this.job, index, {
+                event,
+                message,
+                ...(errorCode ? { code: errorCode } : {}),
+            });
+        }
+        this.repository.updateOpState(this.job, index, {
+            results,
+            logs: opState.logs,
+            exitCode: 2,
+            status,
+            endTime: Date.now(),
+        });
+        this.setResults(op.id, results ?? {});
+    });
+    /**
+     * what else can we clean up here?
+     */
+    emitter.on('end', async () => {
+        emitter.removeAllListeners();
+        this.abortControllerMap.delete(op.id);
+        this.mainAbortController.signal.removeEventListener('abort', handleMainAbort);
+    });
+    /**
+     * Subscribes to the 'healthcheck:startup:success' event, which is emitted when
+     * the container for a given operation has started up successfully and passed its
+     * initial health checks.
+     *
+     * Once this event fires, we treat the current operation as "ready", and begin triggering
+     * the **next operations** that are waiting for this one to start.
+     *
+     * What happens here:
+     * - For each of this operation's dependents (i.e., ops that declared `depends_on: [thisOp]`)
+     * - We re-check their dependents using `getDependentsFromPlan`, ensuring the correct downstream chain.
+     * - We then queue them for execution using `trackGroupOperationPromise` and `setUpOperationFunc`.
+     *
+     * This mechanism allows dynamic chaining of operations based on runtime readiness,
+     * ensuring ops start only after their declared dependencies are healthy and active.
+     */
+    emitter.on('healthcheck:startup:success', (payload) => {
+        emitter.emit('log', 'Operation StartUp Success', 'info');
+        const port = payload?.port;
+        if (typeof port === 'number' || typeof port === 'string') {
+            const portKey = String(port);
+            const stored_url = this.getByPath(op.id, `endpoint.${portKey}`);
+            if (!stored_url)
+                return;
+            const url = stored_url.startsWith('http')
+                ? stored_url
+                : `https://${stored_url}`;
+            logEmitter.emit('log', {
+                class: 'FlowHandler',
+                method: 'operationExposed',
+                arguments: [
+                    {
+                        port,
+                        opId: op.id,
+                    },
+                    true,
+                ],
+                timestamp: new Date().toISOString(),
+                type: 'return',
+                result: url,
+            });
+            try {
+                const flow = this.repository.getFlow(this.job);
+                const secrets = flow?.state?.secrets;
+                let jobSecrets = {};
+                if (secrets &&
+                    typeof secrets[this.job] === 'object' &&
+                    secrets[this.job] !== 'private' &&
+                    secrets[this.job] !== 'public' &&
+                    secrets[this.job]) {
+                    jobSecrets = secrets[this.job];
+                }
+                const targetHost = url.replace(/^https?:\/\//, '');
+                let updated = false;
+                const newJobSecrets = { ...jobSecrets };
+                for (const [exposeId, value] of Object.entries(jobSecrets)) {
+                    const v = value;
+                    if (v && typeof v === 'object' && v.url) {
+                        const host = v.url.replace(/^https?:\/\//, '');
+                        if (host === targetHost) {
+                            newJobSecrets[exposeId] = { ...v, status: 'ONLINE' };
+                            updated = true;
+                        }
+                    }
+                }
+                if (updated) {
+                    this.repository.updateflowStateSecret(this.job, {
+                        [this.job]: newJobSecrets,
+                    });
+                    emitter.emit('flow:secrets-updated', {
+                        flowId: this.job,
+                        opId: op.id,
+                    });
+                }
+            }
+            catch { }
+        }
+        // Loop through each operation that depends on this one
+        for (const id of dependent) {
+            const dependencyContext = this.dependecyMap.get(id);
+            // Schedule the dependent operation to start, now that its dependency is healthy
+            this.currentGroupOperationsPromises.set(id, this.trackGroupOperationPromise(id, this.setUpOperationFunc(flow, id, dependencyContext.dependents)));
+        }
+    });
+    emitter.on('healthcheck:continuous:failure', (payload) => {
+        const port = payload?.port;
+        if (typeof port !== 'number' && typeof port !== 'string')
+            return;
+        try {
+            const flow = this.repository.getFlow(this.job);
+            const secrets = flow?.state?.secrets;
+            let jobSecrets = {};
+            if (secrets &&
+                typeof secrets[this.job] === 'object' &&
+                secrets[this.job] !== 'private' &&
+                secrets[this.job] !== 'public' &&
+                secrets[this.job]) {
+                jobSecrets = secrets[this.job];
+            }
+            let updated = false;
+            const newJobSecrets = {};
+            for (const [exposeId, value] of Object.entries(jobSecrets)) {
+                const v = value;
+                if (v &&
+                    typeof v === 'object' &&
+                    v.opID === op.id &&
+                    String(v.port) === String(port)) {
+                    newJobSecrets[exposeId] = { ...v, status: 'OFFLINE' };
+                    updated = true;
+                }
+                else {
+                    newJobSecrets[exposeId] = v;
+                }
+            }
+            if (updated) {
+                this.repository.updateflowStateSecret(this.job, {
+                    [this.job]: newJobSecrets,
+                });
+                emitter.emit('flow:secrets-updated', {
+                    flowId: this.job,
+                    opId: op.id,
+                });
+            }
+        }
+        catch { }
+    });
+    emitter.on('healthcheck:url:exposed', () => {
+        emitter.emit('log', 'Operation Service URL exposed', 'info');
+    });
+    /**
+     * Hydrate an operation by resolving all literal references in its `args`.
+     *
+     * Interpolates strings like:
+     *   - %%ops.<opId>.results.someKey%%
+     *   - %%ops.<opId>.host%%
+     * against the TaskManager's global store, returning a new `Operation<T>` with
+     * the same `type`, `id`, and `results`, but with `args` fully substituted.
+     *
+     * Notes:
+     * - Only `args` are modified. `type`, `id`, and `results` are preserved.
+     * - `args.env` values are normalized to strings after interpolation.
+     * - Unresolved literals currently resolve to empty strings. If you prefer
+     *   strict behavior, update `resolveLiteralsInString` to throw on misses.
+     *
+     * @example
+     * // Before:
+     * // op.args.env.API = "http://%%ops.nginx-1.host%%:80"
+     * const interpolatedOp = this.interpolateOperation(op);
+     * // After:
+     * // interpolatedOp.args.env.API = "http://nginx-1:80"
+     */
+    let interpolatedOp = op;
+    try {
+        interpolatedOp = this.interpolateOperation(op);
+        /**
+         * now after interpolation we want to proceed to
+         * TransformCollections(interpolatedOp)
+         */
+        interpolatedOp = this.transformCollections(interpolatedOp);
+        interpolatedOp = finalizeEnvOnOperation(interpolatedOp);
+        interpolatedOp = stanatizeArrays(interpolatedOp);
+    }
+    catch (error) {
+        this.repository.updateOpStateLogs(this.job, index, {
+            type: 'nodeerr',
+            log: error.message,
+        });
+        this.repository.updateOpState(this.job, index, {
+            exitCode: 0,
+            startTime: Date.now(),
+            endTime: Date.now(),
+            status: Statuses.FAILED,
+        });
+        emitter.emit('log', 'Operation Stopping', 'info');
+        this.operationStatus.set(op.id, OperationProgressStatuses.STOPPING);
+        abort.abort(this.mainAbortController.signal.reason);
+        return Promise.resolve();
+    }
+    /**
+     * Executes the full lifecycle of a container-based operation:
+     *
+     * 1. **Start Phase**: Runs the container using the operation’s configuration,
+     *    emits all lifecycle events (`start`, `log`, `updateOpState`, `exit`, `error`, `end`),
+     *    and streams logs via the provided `EventEmitter`.
+     *
+     * 2. **Stop Phase**: Cleans up after the operation completes by stopping and removing
+     *    the container, its associated network, and optionally its Docker image
+     *    (if marked as authenticated/private).
+     *
+     * The underlying method blocks until the operation is fully completed or aborted,
+     * and ensures that no containers or networks are left dangling.
+     *
+     * This method is essential for:
+     * - Enforcing deterministic cleanup
+     * - Supporting clean restarts and re-runs
+     * - Ensuring resource isolation per operation
+     *
+     * @returns Promise<void> that resolves only after the operation has run and cleanup has completed
+     */
+    return this.provider.runTaskManagerOperation(flow, interpolatedOp, abort, emitter);
+}

package/dist/NodeManager/node/task/operations/stopAllTaskManagerOperations.d.ts

@@ -0,0 +1,21 @@
+import TaskManager, { StopReason } from '../TaskManager.js';
+/**
+ * Immediately stops the current task flow with the given reason.
+ *
+ * This triggers:
+ * - Flow-wide abort via `mainAbortController`
+ * - Lifecycle shutdown for any running ops
+ * - A consistent status applied to the flow and all ops
+ *
+ * The stop reason determines what status each op and the overall flow will be marked with.
+ *
+ * | Reason   | Ops Status | Flow Status |
+ * |----------|------------|-------------|
+ * | expired  | success    | success     |
+ * | stopped  | stopped    | stopped     |
+ * | quit     | failed     | failed      |
+ * | unknown  | failed     | failed      |
+ *
+ * `getStatus()` is used internally to apply the correct status values.
+ */
+export declare function stopAllTaskManagerOperations(this: TaskManager, reason: StopReason): void;

package/dist/NodeManager/node/task/operations/stopAllTaskManagerOperations.js

@@ -0,0 +1,54 @@
+import { OperationProgressStatuses, } from '../TaskManager.js';
+/**
+ * Immediately stops the current task flow with the given reason.
+ *
+ * This triggers:
+ * - Flow-wide abort via `mainAbortController`
+ * - Lifecycle shutdown for any running ops
+ * - A consistent status applied to the flow and all ops
+ *
+ * The stop reason determines what status each op and the overall flow will be marked with.
+ *
+ * | Reason   | Ops Status | Flow Status |
+ * |----------|------------|-------------|
+ * | expired  | success    | success     |
+ * | stopped  | stopped    | stopped     |
+ * | quit     | failed     | failed      |
+ * | unknown  | failed     | failed      |
+ *
+ * `getStatus()` is used internally to apply the correct status values.
+ */
+export function stopAllTaskManagerOperations(reason) {
+    /**
+     * Resolve the correct flow-level status from the stop reason.
+     * This ensures consistent results across all status tracking.
+     */
+    this.status = this.getStatus(reason, 'flow');
+    /**
+     * Abort the main controller signal, which in turn should cascade
+     * down to any op-level AbortControllers that are still active.
+     */
+    this.mainAbortController.abort(reason);
+    /**
+     * Mark any operations that have NOT started yet as stopped in the repository.
+     */
+    try {
+        const flow = this.repository.getFlow(this.job);
+        if (!flow)
+            return;
+        const now = Date.now();
+        for (const [opId] of this.opMap) {
+            const index = this.getOpStateIndex(opId);
+            const opState = flow.state.opStates[index];
+            if (opState?.startTime)
+                continue;
+            this.operationStatus.set(opId, OperationProgressStatuses.STOPPED);
+            this.repository.updateOpState(this.job, index, {
+                status: this.getStatus(reason, 'ops'),
+                endTime: now,
+                exitCode: 0,
+            });
+        }
+    }
+    catch { }
+}

package/dist/NodeManager/node/task/operations/stopTaskManagerGroupOperation.d.ts

@@ -0,0 +1,10 @@
+import TaskManager from '../TaskManager.js';
+/**
+ * Stops all operations currently running in the given group.
+ *
+ * This method:
+ * - Validates the group exists and is actively running
+ * - Iterates over all ops in the group and stops them in parallel
+ * - Waits for all stops to complete (or be caught if they fail)
+ */
+export declare function stopTaskManagerGroupOperations(this: TaskManager, group: string): Promise<void>;

package/dist/NodeManager/node/task/operations/stopTaskManagerGroupOperation.js

@@ -0,0 +1,43 @@
+import { OperationProgressStatuses } from '../TaskManager.js';
+/**
+ * Stops all operations currently running in the given group.
+ *
+ * This method:
+ * - Validates the group exists and is actively running
+ * - Iterates over all ops in the group and stops them in parallel
+ * - Waits for all stops to complete (or be caught if they fail)
+ */
+export async function stopTaskManagerGroupOperations(group) {
+    /**
+     * Look up the group context to get its operation IDs.
+     * If the group doesn't exist in the execution plan, it's invalid.
+     */
+    const groupContext = this.executionPlan.find((ctx) => ctx.group === group);
+    if (!groupContext) {
+        throw new Error('GROUP_NOT_FOUND');
+    }
+    /**
+     * Ensure we're currently within the provided group context before proceeding.
+     * Prevents stopping inactive or completed groups.
+     */
+    if (this.currentGroup !== group) {
+        throw new Error('GROUP_NOT_ACTIVE');
+    }
+    /**
+     * Stop each operation in parallel.
+     */
+    for (const id of groupContext.ops) {
+        this.operationStatus.set(id, OperationProgressStatuses.STOPPING);
+    }
+    // Emit once after all statuses are updated to avoid interleaving state reads
+    this.events?.emit('flow:updated', {
+        jobId: this.job,
+        group,
+        type: 'group:stopping',
+    });
+    const stopPromises = groupContext.ops.map((id) => this.stopTaskManagerOperation(group, id).catch((err) => {
+        console.warn(`Failed to stop operation ${id}:`, err.message);
+    }));
+    // Await all stop promises before resolving
+    await Promise.all(stopPromises);
+}

package/dist/NodeManager/node/task/operations/stopTaskManagerOperation.d.ts

@@ -0,0 +1,14 @@
+import TaskManager from '../TaskManager.js';
+/**
+ * Publicly exposed method to stop a running operation within the active group.
+ *
+ * This:
+ * - Validates the operation belongs to the current group
+ * - Prevents concurrent mutation using `lockedOperations`
+ * - Aborts the operation's container execution
+ * - Waits for full cleanup (exit/error + container teardown)
+ * - Ensures promise is resolved before continuing
+ *
+ * Usage: await taskManager.stopTaskOperation(groupId, opId)
+ */
+export declare function stopTaskManagerOperation(this: TaskManager, group: string, opId: string): Promise<void>;