@nosana/node 1.1.14-rc → 1.1.15-rc

package/dist/.env.dev

@@ -1,5 +1,4 @@
-BACKEND_URL=https://dashboard.k8s.dev.nos.ci/api
-BACKEND_AUTHORIZATION_ADDRESS=3R1YEz9crQR3tRbtSAuufjSsjQXMqtDr1D4f1YprMUSE
+HOST_MANAGER_BASE_URL=https://host-manager.k8s.dev.nos.ci
 EXPLORER_URL=https://devnet.nosana.io
 FRP_SERVER_ADDRESS=node.k8s.dev.nos.ci
 MIN_DISK_SPACE=20

package/dist/.env.prd

@@ -1,4 +1,3 @@
-BACKEND_URL=https://dashboard.k8s.prd.nos.ci/api
-BACKEND_AUTHORIZATION_ADDRESS=FEEw3nDocYSyrLT4HPjibjYuaNekakWNmasNvEx3nHKi
+HOST_MANAGER_BASE_URL=https://host-manager.k8s.prd.nos.ci
 EXPLORER_URL=https://explore.nosana.com
 FRP_SERVER_ADDRESS=node.k8s.prd.nos.ci

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.14-rc",
+    "version": "1.1.15-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"
@@ -12,7 +12,8 @@
         "dev:podman:start": "./podman-run.sh start",
         "dev:podman:run": "./podman-run.sh run",
         "build": "npm run prepare && tsc && cp .env ./dist/ && cp .env.prd ./dist/ && cp .env.dev ./dist/",
-        "generate": "openapi-typescript http://dashboard.k8s.prd.nos.ci/api/swagger/json -o ./src/NodeManager/client/schema.d.ts",
+        "generate": "npm run generate:host-manager",
+        "generate:host-manager": "openapi-typescript https://host-manager.k8s.dev.nos.ci/swagger/json -o ./src/NodeManager/clients/hostManager/schema.d.ts",
         "test": "APP_ENV=development DOCKER_CERT_PATH= vitest run",
         "ci:test": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage --reporter=junit --outputFile=./test-report.xml",
         "test:watch": "APP_ENV=development DOCKER_CERT_PATH= vitest",
@@ -71,6 +72,7 @@
     "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",
         "@inquirer/prompts": "^4.3.0",
+        "@nosana/ipfs": "^2.2.6",
         "@nosana/sdk": "^0.4.81",
         "@solana/web3.js": "^1.78.0",
         "bn.js": "^5.2.1",

package/dist/src/NodeManager/checkForMaintenance.d.ts

@@ -0,0 +1 @@
+export declare function checkForMaintenance(): Promise<void>;

package/dist/src/NodeManager/checkForMaintenance.js

@@ -0,0 +1,40 @@
+import chalk from "chalk";
+import { getMaintenanceStatus, waitForMaintenance, } from "./clients/maintenanceState.js";
+import { sleep } from "./utils/utils.js";
+import { validateCLIVersion } from "../version/index.js";
+const NPM_POLL_INTERVAL_S = 5 * 60;
+function displayUpcomingMaintenanceNotice(status) {
+    console.log("");
+    console.log(chalk.bgBlue.white.bold(" 🛠  SCHEDULED MAINTENANCE "));
+    console.log("");
+    if (status.startsAt) {
+        console.log(chalk.cyan(`• ${chalk.bold("Starts:")} ${status.startsAt}`));
+    }
+    if (status.expectedEndAt) {
+        console.log(chalk.cyan(`• ${chalk.bold("Expected end:")} ${status.expectedEndAt}`));
+    }
+    if (status.reason) {
+        console.log(chalk.cyan(`• ${chalk.bold("Reason:")} ${status.reason}`));
+    }
+    console.log("");
+}
+export async function checkForMaintenance() {
+    const status = await getMaintenanceStatus();
+    if (!status?.maintenance) {
+        if (status?.startsAt || status?.expectedEndAt || status?.reason) {
+            displayUpcomingMaintenanceNotice(status);
+        }
+        return;
+    }
+    // Re-poll npm during the maintenance window so a published fix triggers
+    // a wrapper restart instead of resuming on the stale version.
+    const maintenanceFinished = waitForMaintenance().catch(() => undefined);
+    let cleared = false;
+    maintenanceFinished.then(() => {
+        cleared = true;
+    });
+    while (!cleared) {
+        await validateCLIVersion();
+        await Promise.race([sleep(NPM_POLL_INTERVAL_S), maintenanceFinished]);
+    }
+}

package/dist/src/NodeManager/clients/hostManager/index.d.ts

@@ -0,0 +1,4 @@
+import { paths } from './schema.js';
+import type { AuthenticatedClient } from '../types.utils.js';
+export type HostManagerClient = AuthenticatedClient<paths>;
+export declare const hostManagerClientSelector: () => HostManagerClient;

package/dist/src/NodeManager/clients/hostManager/index.js

@@ -0,0 +1,25 @@
+import createClient from 'openapi-fetch';
+import { getSDK } from '../../sdk/index.js';
+import { configs } from '../../configs/configs.js';
+import { maintenanceMiddleware } from '../maintenanceMiddleware.js';
+let instance = undefined;
+export const hostManagerClientSelector = () => {
+    if (!instance) {
+        const sdk = getSDK();
+        const address = sdk.solana.provider.wallet.publicKey.toString();
+        instance = createClient({
+            baseUrl: configs().hostManagerBaseUrl,
+            headers: {
+                'Content-Type': 'application/json',
+            },
+        });
+        instance.use(maintenanceMiddleware);
+        instance.use({
+            async onRequest({ request }) {
+                const authHeader = await sdk.authorization.generate(address);
+                request.headers.set('Authorization', authHeader);
+            }
+        });
+    }
+    return instance;
+};

package/dist/src/NodeManager/clients/maintenanceMiddleware.d.ts

@@ -0,0 +1,2 @@
+import type { Middleware } from "openapi-fetch";
+export declare const maintenanceMiddleware: Middleware;

package/dist/src/NodeManager/clients/maintenanceMiddleware.js

@@ -0,0 +1,11 @@
+import { isMaintenanceResponse, waitForMaintenance, } from "./maintenanceState.js";
+export const maintenanceMiddleware = {
+    onResponse: async ({ response, request, options }) => {
+        let current = response;
+        while (await isMaintenanceResponse(current)) {
+            await waitForMaintenance();
+            current = await options.fetch(request.clone());
+        }
+        return current === response ? undefined : current;
+    },
+};

package/dist/src/NodeManager/clients/maintenanceState.d.ts

@@ -0,0 +1,10 @@
+export declare const MAINTENANCE_BODY = "under maintenance";
+export type MaintenanceResponse = {
+    maintenance: boolean;
+    startsAt?: string;
+    expectedEndAt?: string;
+    reason?: string;
+};
+export declare const isMaintenanceResponse: (response: Response) => Promise<boolean>;
+export declare const getMaintenanceStatus: () => Promise<MaintenanceResponse | null>;
+export declare const waitForMaintenance: () => Promise<void>;

package/dist/src/NodeManager/clients/maintenanceState.js

@@ -0,0 +1,66 @@
+import chalk from "chalk";
+import { configs } from "../configs/configs.js";
+import { consoleLogging } from "../monitoring/log/console/ConsoleLogger.js";
+import { sleep } from "../utils/utils.js";
+export const MAINTENANCE_BODY = "under maintenance";
+const MAINTENANCE_PATH = "/node-under-maintenance";
+const POLL_INTERVAL_S = 5;
+let maintenancePromise = null;
+const getMaintenanceUrl = () => {
+    return `${configs().hostManagerBaseUrl}${MAINTENANCE_PATH}`;
+};
+// Sibling endpoints signal maintenance via 503 + plain text body.
+export const isMaintenanceResponse = async (response) => {
+    if (response.status !== 503)
+        return false;
+    try {
+        const text = await response.clone().text();
+        return text.trim().toLowerCase() === MAINTENANCE_BODY;
+    }
+    catch {
+        return false;
+    }
+};
+export const getMaintenanceStatus = async () => {
+    try {
+        const response = await fetch(getMaintenanceUrl());
+        return (await response.json());
+    }
+    catch {
+        return null;
+    }
+};
+const formatSpinnerText = (body) => {
+    const parts = ["Node under maintenance"];
+    if (body?.reason)
+        parts.push(body.reason);
+    if (body?.expectedEndAt)
+        parts.push(`ETA ${body.expectedEndAt}`);
+    return chalk.yellow(parts.join(" — "));
+};
+export const waitForMaintenance = () => {
+    if (maintenancePromise)
+        return maintenancePromise;
+    const spinner = consoleLogging().spinner;
+    const previousText = spinner?.text;
+    if (spinner) {
+        spinner.text = formatSpinnerText(null);
+        if (!spinner.isSpinning)
+            spinner.start();
+    }
+    maintenancePromise = (async () => {
+        for (;;) {
+            const body = await getMaintenanceStatus();
+            if (body && !body.maintenance)
+                return;
+            if (spinner)
+                spinner.text = formatSpinnerText(body);
+            await sleep(POLL_INTERVAL_S);
+        }
+    })().finally(() => {
+        if (spinner && previousText !== undefined)
+            spinner.text = previousText;
+        maintenancePromise = null;
+    });
+    return maintenancePromise;
+};

package/dist/src/NodeManager/clients/types.utils.d.ts

@@ -0,0 +1,45 @@
+import createClient from "openapi-fetch";
+export type EnumValues<T> = T[keyof T];
+/**
+ * Utility type to remove header requirements from OpenAPI parameters
+ * This preserves the original structure but removes headers and makes undefined params optional
+ * We only modify the parameters, leaving responses and requestBody untouched
+ */
+type OmitHeaders<T> = T extends {
+    parameters: {
+        header?: any;
+        path?: infer Path;
+        query?: infer Query;
+        cookie?: infer Cookie;
+    };
+    responses?: infer Responses;
+    requestBody?: infer RequestBody;
+} ? {
+    parameters: {
+        header?: never;
+    } & (Path extends undefined ? {} : {
+        path: Path;
+    }) & (Query extends undefined ? {} : {
+        query: Query;
+    }) & (Cookie extends undefined ? {} : {
+        cookie: Cookie;
+    });
+} & (Responses extends undefined ? {} : {
+    responses: Responses;
+}) & (RequestBody extends undefined ? {} : {
+    requestBody: RequestBody;
+}) : T;
+/**
+ * Type that removes header requirements from all endpoints in the paths
+ * This should preserve response types and other properties
+ */
+export type AuthenticatedPaths<Paths extends Record<string, any>> = {
+    [P in keyof Paths]: {
+        [M in keyof Paths[P]]: OmitHeaders<Paths[P][M]>;
+    };
+};
+/**
+ * Explicit client type that should preserve all response typing
+ */
+export type AuthenticatedClient<Paths extends Record<string, any>> = ReturnType<typeof createClient<AuthenticatedPaths<Paths>>>;
+export {};

package/dist/src/NodeManager/configs/configs.d.ts

@@ -1,8 +1,6 @@
 export type configType = {
     isNodeRun: boolean;
-    backendUrl: string;
-    backendSolanaAddress: string;
-    backendAuthorizationAddress: string;
+    hostManagerBaseUrl: string;
     explorerUrl: string;
     signMessage: string;
     frp: {

package/dist/src/NodeManager/configs/configs.js

@@ -4,9 +4,7 @@ export const configs = (options) => {
     NodeConfigsSingleton.getInstance(options);
     return {
         isNodeRun: options?.isNodeRun,
-        backendUrl: loadConfigurationValue('BACKEND_URL'),
-        backendSolanaAddress: loadConfigurationValue('BACKEND_SOLANA_ADDRESS'),
-        backendAuthorizationAddress: loadConfigurationValue('BACKEND_AUTHORIZATION_ADDRESS'),
+        hostManagerBaseUrl: loadConfigurationValue('HOST_MANAGER_BASE_URL'),
         explorerUrl: loadConfigurationValue('EXPLORER_URL'),
         signMessage: loadConfigurationValue('SIGN_MESSAGE'),
         frp: {

package/dist/src/NodeManager/errors/NodeBannedError.d.ts

@@ -0,0 +1,3 @@
+export declare class NodeBannedError extends Error {
+    constructor();
+}

package/dist/src/NodeManager/errors/NodeBannedError.js

@@ -0,0 +1,6 @@
+export class NodeBannedError extends Error {
+    constructor() {
+        super('Node has been rejected (REJECTED status). Shutting down.');
+        this.name = 'NodeBannedError';
+    }
+}

package/dist/src/NodeManager/errors/NodeNotQualifiedError.d.ts

@@ -0,0 +1,3 @@
+export declare class NodeNotQualifiedError extends Error {
+    constructor();
+}

package/dist/src/NodeManager/errors/NodeNotQualifiedError.js

@@ -0,0 +1,6 @@
+export class NodeNotQualifiedError extends Error {
+    constructor() {
+        super('Node does not meet the minimum requirements for any market. Shutting down.');
+        this.name = 'NodeNotQualifiedError';
+    }
+}

package/dist/src/NodeManager/errors/NodeNotRegisteredError.d.ts

@@ -0,0 +1,3 @@
+export declare class NodeNotRegisteredError extends Error {
+    constructor();
+}

package/dist/src/NodeManager/errors/NodeNotRegisteredError.js

@@ -0,0 +1,6 @@
+export class NodeNotRegisteredError extends Error {
+    constructor() {
+        super('Node is not registered. Registration required before requesting a market.');
+        this.name = 'NodeNotRegisteredError';
+    }
+}

package/dist/src/NodeManager/index.js

@@ -6,11 +6,12 @@ import { log } from './monitoring/log/NodeLog.js';
 import { logStreaming } from './monitoring/streaming/LogStreamer.js';
 import { consoleLogging } from './monitoring/log/console/ConsoleLogger.js';
 import { validateCLIVersion } from '../version/index.js';
-import { configs } from './configs/configs.js';
+import { checkForMaintenance } from './checkForMaintenance.js';
 import { getSDK } from './sdk/index.js';
 import { ping } from './monitoring/ping/PingHandler.js';
 import { LogMonitoringRegistry } from './monitoring/LogMonitoringRegistry.js';
 import { checkWSLStatus } from './utils/wslCheck.js';
+import { HostManager } from './node/market/hostManager.js';
 export default class NodeManager {
     constructor(options) {
         this.exiting = false;
@@ -57,9 +58,6 @@ export default class NodeManager {
          * the node api, initializing everything that needs to be initialized
          */
         await this.node.start();
-        if (!this.node.isOnboarded) {
-            await this.node.register();
-        }
         /**
          * start the api of the node and register all the routes of the nodes,
          * we call this here in the init so the api survives restarts between jobs
@@ -78,28 +76,12 @@ export default class NodeManager {
     async start(marketArg) {
         this.exiting = false;
         if (this.inJobLoop) {
-            await validateCLIVersion();
+            await Promise.all([validateCLIVersion(), checkForMaintenance()]);
         }
         /**
          * maintenance
          */
         await this.node.maintenance();
-        /**
-         * specs
-         *
-         * this retrieves specs from the node to ensure it can run the jobs.
-         * It gets the GPUs, CPUs, and internet speed.
-         *
-         * if the specs fails restart the system
-         */
-        if (!(await this.node.specs())) {
-            /**
-             * start
-             *
-             * recursively start the the process again by calling the restart function
-             */
-            return await this.restart(marketArg);
-        }
         /**
          * WSL check
          * - Shows warning message for all WSL hosts
@@ -107,28 +89,19 @@ export default class NodeManager {
          */
         checkWSLStatus(this.node.getSystemEnvironment());
         /**
-         * grid
+         * market
          *
-         * if no market was supplied, we will register on the grid and get
-         * market and access key recommened for our PC based on specs result
+         * request a market to join from the host manager,
+         * we can pass a requested market or let the host manager recommend us one
          */
-        let market = marketArg;
-        if (!market || !market.length) {
-            market = await this.node.recommend();
-        }
-        /**
-         * setup
-         *
-         * this sets up everything need, downloads needed resources ...
-         */
-        await this.node.setup(market);
+        const market = await this.node.requestMarket(marketArg);
         /**
          * healthcheck
          *
          * this checks the health of the container tech,
          * the connectivity, and every other critical system.
          */
-        if (!(await this.node.healthcheck(market))) {
+        if (!(await this.node.healthcheck())) {
             /**
              * start
              *
@@ -136,6 +109,12 @@ export default class NodeManager {
              */
             return await this.restart(marketArg);
         }
+        /**
+         * setup
+         *
+         * this sets up everything need, downloads needed resources ...
+         */
+        await this.node.setup(market);
         /**
          * this variable was added to know when the node has gone past the setup/checks stages
          * and is now starting job work and queueing,
@@ -144,7 +123,7 @@ export default class NodeManager {
          */
         this.inJobLoop = true;
         if (!(await this.node.isApiActive())) {
-            throw new Error('Node API is detected offline');
+            throw new Error("Node API is detected offline");
         }
         /**
          * pending
@@ -160,7 +139,7 @@ export default class NodeManager {
              *
              * Enter the queue to wait for a job since we have no pending jobs.
              */
-            await this.node.queue(market);
+            await this.node.queue(market.address.toString());
         }
         /**
          *
@@ -268,14 +247,14 @@ export default class NodeManager {
             await this.stop();
             process.exit();
         };
-        process.on('SIGINT', exitHandler); // Handle Ctrl+C
-        process.on('SIGTERM', exitHandler); // Handle termination signals
+        process.on("SIGINT", exitHandler); // Handle Ctrl+C
+        process.on("SIGTERM", exitHandler); // Handle termination signals
         // log crashes
-        process.on('unhandledRejection', async (reason, p) => {
+        process.on("unhandledRejection", async (reason, p) => {
             try {
                 const e = reason;
                 await reportError({
-                    error_type: 'unhandledRejection',
+                    error_type: "unhandledRejection",
                     error_name: e.name,
                     error_message: e.message,
                     error_stack: e.stack ?? e.trace,
@@ -283,11 +262,11 @@ export default class NodeManager {
             }
             catch (_) { }
         });
-        process.on('uncaughtException', async (reason) => {
+        process.on("uncaughtException", async (reason) => {
             try {
                 const e = reason;
                 await reportError({
-                    error_type: 'uncaughtException',
+                    error_type: "uncaughtException",
                     error_name: e.name,
                     error_message: e.message,
                     error_stack: e.stack ?? e.trace,
@@ -299,19 +278,8 @@ export default class NodeManager {
 }
 const reportError = async (data) => {
     const nosana = getSDK();
-    const response = await fetch(`${configs().backendUrl}/errors/report`, {
-        method: 'POST',
-        headers: {
-            Authorization: await nosana.authorization.generate(configs().signMessage, {
-                includeTime: true,
-            }),
-            'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-            address: nosana.solana.provider.wallet.publicKey.toString(),
-            ...data,
-        }),
+    return HostManager.reportError({
+        address: nosana.solana.provider.wallet.publicKey.toString(),
+        ...data,
     });
-    const responseBody = await response.json();
-    return responseBody;
 };

package/dist/src/NodeManager/monitoring/log/NodeLog.d.ts

@@ -30,7 +30,6 @@ declare class NodeLog {
     private process;
     private handleRecommend;
     private handleResourceManager;
-    private handleSpecs;
     private handleRegister;
     private handleExpiryHandler;
     private handleContainerCheckHandler;

package/dist/src/NodeManager/monitoring/log/NodeLog.js

@@ -47,7 +47,17 @@ class NodeLog {
         }
     }
     process(data) {
-        if (data.class === 'TaskManager') {
+        if (data.class === 'BenchmarkTaskManager' && data.method === 'progress') {
+            this.addLog({
+                method: `${data.class}.${data.method}`,
+                job: this.job,
+                log: '',
+                timestamp: Date.now(),
+                type: 'update',
+                payload: data.payload,
+            });
+        }
+        if (data.class === 'TaskManager' || data.class === 'BenchmarkTaskManager') {
             if (data.method === 'start') {
                 if (data.type === 'call') {
                     this.addLog({
@@ -126,9 +136,6 @@ class NodeLog {
             (data.method === 'restartDelay' || data.method === 'delay')) {
             this.handleRestart(data);
         }
-        if (data.class === 'BasicNode' && data.method === 'specs') {
-            this.handleSpecs(data);
-        }
         if (data.class === 'BasicNode' && data.method === 'recommend') {
             this.handleRecommend(data);
         }
@@ -403,40 +410,6 @@ class NodeLog {
     //     });
     //   }
     // }
-    handleSpecs(data) {
-        if (data.type === 'call') {
-            this.addLog({
-                method: `${data.class}.${data.method}`,
-                job: this.job,
-                log: chalk.cyan('Node is checking specs'),
-                timestamp: Date.now(),
-                // type: 'info',
-                type: 'process',
-                pending: {
-                    isPending: true,
-                    expecting: `${data.class}.${data.method}`,
-                },
-            });
-        }
-        if (data.type === 'return') {
-            this.addLog({
-                method: `${data.class}.${data.method}`,
-                job: this.job,
-                log: chalk.green('Node specs check completed successfully'),
-                timestamp: Date.now(),
-                type: 'success',
-            });
-        }
-        if (data.type === 'error') {
-            this.addLog({
-                method: `${data.class}.${data.method}`,
-                job: this.job,
-                log: chalk.red('Node specs check failed'),
-                timestamp: Date.now(),
-                type: 'error',
-            });
-        }
-    }
     handleRegister(data) {
         if (data.type === 'call') {
             this.addLog({

package/dist/src/NodeManager/monitoring/log/console/ConsoleLogger.d.ts

@@ -11,6 +11,9 @@ export declare class ConsoleLogger implements LogObserver {
     private kill;
     private benchmarking;
     private taskManagerActive;
+    private taskManagerLabel;
+    private taskManagerStartMethod;
+    private benchmarkProgress;
     private running;
     private expiry;
     private taskManagerRunInterval;