@nosana/node 1.1.14-rc → 1.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.env.dev +1 -2
- package/dist/.env.prd +1 -2
- package/dist/package.json +4 -2
- package/dist/src/NodeManager/checkForMaintenance.d.ts +1 -0
- package/dist/src/NodeManager/checkForMaintenance.js +40 -0
- package/dist/src/NodeManager/clients/hostManager/index.d.ts +4 -0
- package/dist/src/NodeManager/clients/hostManager/index.js +25 -0
- package/dist/src/NodeManager/clients/maintenanceMiddleware.d.ts +2 -0
- package/dist/src/NodeManager/clients/maintenanceMiddleware.js +11 -0
- package/dist/src/NodeManager/clients/maintenanceState.d.ts +10 -0
- package/dist/src/NodeManager/clients/maintenanceState.js +66 -0
- package/dist/src/NodeManager/clients/types.utils.d.ts +45 -0
- package/dist/src/NodeManager/configs/configs.d.ts +1 -3
- package/dist/src/NodeManager/configs/configs.js +1 -3
- package/dist/src/NodeManager/errors/NodeBannedError.d.ts +3 -0
- package/dist/src/NodeManager/errors/NodeBannedError.js +6 -0
- package/dist/src/NodeManager/errors/NodeNotQualifiedError.d.ts +3 -0
- package/dist/src/NodeManager/errors/NodeNotQualifiedError.js +6 -0
- package/dist/src/NodeManager/errors/NodeNotRegisteredError.d.ts +3 -0
- package/dist/src/NodeManager/errors/NodeNotRegisteredError.js +6 -0
- package/dist/src/NodeManager/index.js +25 -57
- package/dist/src/NodeManager/monitoring/log/NodeLog.d.ts +0 -1
- package/dist/src/NodeManager/monitoring/log/NodeLog.js +11 -38
- package/dist/src/NodeManager/monitoring/log/console/ConsoleLogger.d.ts +3 -0
- package/dist/src/NodeManager/monitoring/log/console/ConsoleLogger.js +33 -5
- package/dist/src/NodeManager/monitoring/ping/PingHandler.d.ts +2 -4
- package/dist/src/NodeManager/monitoring/ping/PingHandler.js +31 -26
- package/dist/src/NodeManager/monitoring/proxy/loggingProxy.d.ts +2 -1
- package/dist/src/NodeManager/monitoring/proxy/loggingProxy.js +2 -3
- package/dist/src/NodeManager/node/Node.d.ts +4 -6
- package/dist/src/NodeManager/node/Node.js +16 -29
- package/dist/src/NodeManager/node/api/ApiHandler.js +2 -3
- package/dist/src/NodeManager/node/api/middlewares/index.d.ts +0 -1
- package/dist/src/NodeManager/node/api/middlewares/index.js +0 -1
- package/dist/src/NodeManager/node/api/routes/post/index.d.ts +0 -1
- package/dist/src/NodeManager/node/api/routes/post/index.js +0 -1
- package/dist/src/NodeManager/node/grid/gridHandler.d.ts +2 -8
- package/dist/src/NodeManager/node/grid/gridHandler.js +10 -143
- package/dist/src/NodeManager/node/health/healthHandler.d.ts +1 -1
- package/dist/src/NodeManager/node/health/healthHandler.js +13 -11
- package/dist/src/NodeManager/node/job/JobRegistry.js +5 -3
- package/dist/src/NodeManager/node/job/defination/strategy/IpfsJobDefinitionStrategy.d.ts +1 -3
- package/dist/src/NodeManager/node/job/defination/strategy/IpfsJobDefinitionStrategy.js +2 -4
- package/dist/src/NodeManager/node/job/jobExternalUtil.d.ts +2 -3
- package/dist/src/NodeManager/node/job/jobExternalUtil.js +4 -4
- package/dist/src/NodeManager/node/job/jobHandler.js +5 -3
- package/dist/src/NodeManager/node/key/keyHandler.d.ts +0 -1
- package/dist/src/NodeManager/node/key/keyHandler.js +1 -38
- package/dist/src/NodeManager/node/market/benchmark.d.ts +15 -0
- package/dist/src/NodeManager/node/market/benchmark.js +84 -0
- package/dist/src/NodeManager/node/market/hostManager.d.ts +114 -0
- package/dist/src/NodeManager/node/market/hostManager.js +112 -0
- package/dist/src/NodeManager/node/market/marketAccess.d.ts +8 -0
- package/dist/src/NodeManager/node/market/marketAccess.js +53 -0
- package/dist/src/NodeManager/node/market/marketHandler.d.ts +10 -3
- package/dist/src/NodeManager/node/market/marketHandler.js +88 -6
- package/dist/src/NodeManager/node/register/index.d.ts +2 -12
- package/dist/src/NodeManager/node/register/index.js +12 -56
- package/dist/src/NodeManager/node/resource/resourceManager.d.ts +2 -1
- package/dist/src/NodeManager/node/resource/resourceManager.js +4 -4
- package/dist/src/NodeManager/sdk/index.js +4 -14
- package/dist/src/NodeManager/utils/wslCheck.js +1 -1
- package/dist/src/cli/createNosanaCli.js +2 -0
- package/dist/src/ipfs/IpfsClient.d.ts +7 -0
- package/dist/src/ipfs/IpfsClient.js +17 -0
- package/npm-shrinkwrap.json +30 -2
- package/package.json +5 -3
- package/dist/src/NodeManager/client/index.d.ts +0 -4
- package/dist/src/NodeManager/client/index.js +0 -14
- package/dist/src/NodeManager/node/api/middlewares/rest/verifyBackendSignatureMiddleware.d.ts +0 -3
- package/dist/src/NodeManager/node/api/middlewares/rest/verifyBackendSignatureMiddleware.js +0 -20
- package/dist/src/NodeManager/node/api/routes/post/node-validation.d.ts +0 -4
- package/dist/src/NodeManager/node/api/routes/post/node-validation.js +0 -1787
- package/dist/src/NodeManager/node/specs/specsHandler.d.ts +0 -17
- package/dist/src/NodeManager/node/specs/specsHandler.js +0 -135
- package/dist/src/NodeManager/node/specs/type.d.ts +0 -15
- /package/dist/src/NodeManager/{node/specs/type.js → clients/types.utils.js} +0 -0
|
@@ -21,6 +21,9 @@ export class ConsoleLogger {
|
|
|
21
21
|
this.kill = false;
|
|
22
22
|
this.benchmarking = false;
|
|
23
23
|
this.taskManagerActive = false;
|
|
24
|
+
this.taskManagerLabel = ' RUNNING JOB ';
|
|
25
|
+
this.taskManagerStartMethod = 'TaskManager.start';
|
|
26
|
+
this.benchmarkProgress = { completed: 0, total: 0 };
|
|
24
27
|
this.running = false;
|
|
25
28
|
}
|
|
26
29
|
addObserver() {
|
|
@@ -30,7 +33,7 @@ export class ConsoleLogger {
|
|
|
30
33
|
return this.isNode;
|
|
31
34
|
}
|
|
32
35
|
update(log, isNode = true) {
|
|
33
|
-
const renderBase = `${chalk.bgGreenBright.black.bold(
|
|
36
|
+
const renderBase = `${chalk.bgGreenBright.black.bold(this.taskManagerLabel)} ${chalk.reset('')}`;
|
|
34
37
|
if (!this.kill && log.type == 'kill-process') {
|
|
35
38
|
if (this.pending) {
|
|
36
39
|
this.spinner.stop();
|
|
@@ -70,7 +73,9 @@ export class ConsoleLogger {
|
|
|
70
73
|
this.expiry = log.payload?.expiry;
|
|
71
74
|
}
|
|
72
75
|
// --- TaskManager quiet window ---
|
|
73
|
-
if (log.method === 'TaskManager.start'
|
|
76
|
+
if ((log.method === 'TaskManager.start' ||
|
|
77
|
+
log.method === 'BenchmarkTaskManager.start') &&
|
|
78
|
+
log.type === 'process') {
|
|
74
79
|
// Stop anything currently animating
|
|
75
80
|
if (this.pending)
|
|
76
81
|
this.spinner.stop();
|
|
@@ -78,11 +83,22 @@ export class ConsoleLogger {
|
|
|
78
83
|
this.progressBar.stop();
|
|
79
84
|
if (this.multiProgressBar)
|
|
80
85
|
this.multiProgressBar.stop();
|
|
86
|
+
this.taskManagerStartMethod = log.method;
|
|
87
|
+
this.taskManagerLabel =
|
|
88
|
+
log.method === 'BenchmarkTaskManager.start'
|
|
89
|
+
? ' COLLECTING METRICS '
|
|
90
|
+
: ' RUNNING JOB ';
|
|
91
|
+
const activeRenderBase = `${chalk.bgGreenBright.black.bold(this.taskManagerLabel)} ${chalk.reset('')}`;
|
|
81
92
|
this.taskManagerActive = true;
|
|
82
93
|
this.pending = true;
|
|
83
94
|
this.expiry = log.payload?.expiry;
|
|
84
95
|
const startTime = Date.now();
|
|
85
|
-
const
|
|
96
|
+
const isBenchmark = log.method === 'BenchmarkTaskManager.start';
|
|
97
|
+
if (isBenchmark)
|
|
98
|
+
this.benchmarkProgress = { completed: 0, total: 0 };
|
|
99
|
+
const renderBar = (duration = 0) => isBenchmark
|
|
100
|
+
? `${activeRenderBase} ${chalk.bgYellow.black.bold(` Ops: ${this.benchmarkProgress.completed}/${this.benchmarkProgress.total} `)} ${chalk.reset('')}`
|
|
101
|
+
: `${activeRenderBase} ${chalk.bgYellow.black.bold(` ⏱ Duration: ${formatTime(duration)} `)} ${chalk.reset('')} ${chalk.bgBlue.black.bold(` ⚡ Max Duration: ${formatTime(this.expiry ?? 0)} `)}`;
|
|
86
102
|
this.spinner = ora(renderBar(0)).start();
|
|
87
103
|
this.taskManagerRunInterval = setInterval(() => {
|
|
88
104
|
if (this.taskManagerActive) {
|
|
@@ -98,12 +114,24 @@ export class ConsoleLogger {
|
|
|
98
114
|
}
|
|
99
115
|
// While TaskManager is active, swallow all logs except the stop signal
|
|
100
116
|
if (this.taskManagerActive) {
|
|
101
|
-
if (log.method ===
|
|
102
|
-
this.
|
|
117
|
+
if (log.method === this.taskManagerStartMethod) {
|
|
118
|
+
const endingSuffix = this.taskManagerStartMethod === 'BenchmarkTaskManager.start'
|
|
119
|
+
? ' Completed'
|
|
120
|
+
: ` Ending ${log.job}...`;
|
|
121
|
+
this.spinner.succeed(renderBase + endingSuffix);
|
|
103
122
|
this.taskManagerActive = false;
|
|
104
123
|
this.pending = false;
|
|
105
124
|
return;
|
|
106
125
|
}
|
|
126
|
+
if (log.method === 'BenchmarkTaskManager.progress' &&
|
|
127
|
+
this.taskManagerStartMethod === 'BenchmarkTaskManager.start') {
|
|
128
|
+
const { completed, total } = log.payload ?? {};
|
|
129
|
+
if (typeof completed === 'number' && typeof total === 'number') {
|
|
130
|
+
this.benchmarkProgress = { completed, total };
|
|
131
|
+
this.spinner.text = `${renderBase} ${chalk.bgYellow.black.bold(` Ops: ${completed}/${total} `)} ${chalk.reset('')}`;
|
|
132
|
+
}
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
107
135
|
// Still inside TaskManager window: block other logs
|
|
108
136
|
return;
|
|
109
137
|
}
|
|
@@ -2,12 +2,10 @@ export declare const ping: () => Promise<PingHandler>;
|
|
|
2
2
|
export declare class PingHandler {
|
|
3
3
|
private timeoutId;
|
|
4
4
|
private intervalSeconds;
|
|
5
|
-
private
|
|
6
|
-
private address;
|
|
7
|
-
constructor();
|
|
5
|
+
private stopped;
|
|
8
6
|
start(): Promise<void>;
|
|
9
7
|
private scheduleNextPing;
|
|
10
8
|
private ping;
|
|
11
|
-
private
|
|
9
|
+
private getHeartbeatRetryAfterSeconds;
|
|
12
10
|
stop(): void;
|
|
13
11
|
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { configs } from '../../configs/configs.js';
|
|
1
|
+
import { HostManager } from '../../node/market/hostManager.js';
|
|
3
2
|
export const ping = (() => {
|
|
4
3
|
let instance = null;
|
|
5
4
|
return async () => {
|
|
@@ -14,18 +13,20 @@ export class PingHandler {
|
|
|
14
13
|
constructor() {
|
|
15
14
|
this.timeoutId = null;
|
|
16
15
|
this.intervalSeconds = 30;
|
|
17
|
-
this.
|
|
18
|
-
this.address = this.sdk.solana.provider.wallet.publicKey;
|
|
16
|
+
this.stopped = false;
|
|
19
17
|
}
|
|
20
18
|
async start() {
|
|
21
19
|
if (this.timeoutId)
|
|
22
20
|
return;
|
|
23
|
-
|
|
21
|
+
this.stopped = false;
|
|
24
22
|
await this.scheduleNextPing(Date.now());
|
|
25
23
|
}
|
|
26
24
|
async scheduleNextPing(expectedTime) {
|
|
27
|
-
await this.ping();
|
|
28
|
-
|
|
25
|
+
const { retryAfterSeconds } = await this.ping();
|
|
26
|
+
if (this.stopped)
|
|
27
|
+
return;
|
|
28
|
+
const intervalSeconds = retryAfterSeconds ?? this.intervalSeconds;
|
|
29
|
+
const nextExpected = expectedTime + intervalSeconds * 1000;
|
|
29
30
|
const delay = Math.max(nextExpected - Date.now(), 0);
|
|
30
31
|
this.timeoutId = setTimeout(() => {
|
|
31
32
|
this.scheduleNextPing(nextExpected);
|
|
@@ -33,33 +34,37 @@ export class PingHandler {
|
|
|
33
34
|
}
|
|
34
35
|
async ping() {
|
|
35
36
|
try {
|
|
36
|
-
const response = await
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
Authorization: `${this.address}:${await this.getAuthSignature()}`,
|
|
40
|
-
'Content-Type': 'application/json',
|
|
41
|
-
},
|
|
42
|
-
body: JSON.stringify({ ping: 'pong' }),
|
|
43
|
-
});
|
|
44
|
-
if (response.ok) {
|
|
45
|
-
const json = await response.json();
|
|
46
|
-
if (json.maxHeartbeatsPerDay) {
|
|
47
|
-
const newInterval = Math.floor((24 * 60 * 60) / json.maxHeartbeatsPerDay);
|
|
48
|
-
if (newInterval !== this.intervalSeconds) {
|
|
49
|
-
this.intervalSeconds = newInterval;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
37
|
+
const response = await HostManager.sendHeartbeat();
|
|
38
|
+
if (response.maxHeartbeatsPerDay) {
|
|
39
|
+
this.intervalSeconds = Math.floor((24 * 60 * 60) / response.maxHeartbeatsPerDay);
|
|
52
40
|
}
|
|
41
|
+
return {};
|
|
53
42
|
}
|
|
54
43
|
catch (err) {
|
|
44
|
+
if (err instanceof Error && err.message.includes('Node not found')) {
|
|
45
|
+
return {};
|
|
46
|
+
}
|
|
47
|
+
const retryAfterSeconds = this.getHeartbeatRetryAfterSeconds(err);
|
|
48
|
+
if (retryAfterSeconds !== null) {
|
|
49
|
+
return { retryAfterSeconds };
|
|
50
|
+
}
|
|
55
51
|
console.error('Ping failed:', err);
|
|
52
|
+
return {};
|
|
56
53
|
}
|
|
57
54
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
getHeartbeatRetryAfterSeconds(err) {
|
|
56
|
+
if (!(err instanceof Error))
|
|
57
|
+
return null;
|
|
58
|
+
const match = err.message.match(/next heartbeat in (\d+) seconds/i);
|
|
59
|
+
if (!match)
|
|
60
|
+
return null;
|
|
61
|
+
const retryAfterSeconds = Number.parseInt(match[1], 10);
|
|
62
|
+
return Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
|
|
63
|
+
? retryAfterSeconds
|
|
64
|
+
: null;
|
|
61
65
|
}
|
|
62
66
|
stop() {
|
|
67
|
+
this.stopped = true;
|
|
63
68
|
if (this.timeoutId) {
|
|
64
69
|
clearTimeout(this.timeoutId);
|
|
65
70
|
this.timeoutId = null;
|
|
@@ -9,7 +9,8 @@ export interface LogEntry {
|
|
|
9
9
|
type: 'call' | 'return' | 'error';
|
|
10
10
|
result?: any;
|
|
11
11
|
error?: any;
|
|
12
|
+
payload?: any;
|
|
12
13
|
}
|
|
13
14
|
export declare function applyLoggingProxyToClass(instance: any): void;
|
|
14
|
-
export declare function createLoggingProxy<T extends object>(target: T): T;
|
|
15
|
+
export declare function createLoggingProxy<T extends object>(target: T, classNameOverride?: string): T;
|
|
15
16
|
export declare function createFunctionLoggingProxy<T extends (...args: any[]) => any>(func: T, classLabel?: string): T;
|
|
@@ -17,7 +17,6 @@ export function applyLoggingProxyToClass(instance) {
|
|
|
17
17
|
'healthHandler',
|
|
18
18
|
'stakeHandler',
|
|
19
19
|
'expiryHandler',
|
|
20
|
-
'specsHandler',
|
|
21
20
|
'progressBarReporter',
|
|
22
21
|
'registerHandler',
|
|
23
22
|
];
|
|
@@ -28,8 +27,8 @@ export function applyLoggingProxyToClass(instance) {
|
|
|
28
27
|
}
|
|
29
28
|
});
|
|
30
29
|
}
|
|
31
|
-
export function createLoggingProxy(target) {
|
|
32
|
-
const className = Object.getPrototypeOf(target).constructor.name;
|
|
30
|
+
export function createLoggingProxy(target, classNameOverride) {
|
|
31
|
+
const className = classNameOverride ?? Object.getPrototypeOf(target).constructor.name;
|
|
33
32
|
const formatArguments = (args) => {
|
|
34
33
|
return args.map((arg) => {
|
|
35
34
|
if (typeof arg === 'function') {
|
|
@@ -1,12 +1,11 @@
|
|
|
1
|
+
import { Market } from '@nosana/sdk';
|
|
1
2
|
import { ApiHandler } from './api/ApiHandler.js';
|
|
2
3
|
export declare class BasicNode {
|
|
3
|
-
isOnboarded: boolean;
|
|
4
4
|
private apiHandler;
|
|
5
5
|
private balanceHandler;
|
|
6
6
|
private runHandler;
|
|
7
7
|
private marketHandler;
|
|
8
8
|
private jobHandler;
|
|
9
|
-
private specsHandler;
|
|
10
9
|
private keyHandler;
|
|
11
10
|
private healthHandler;
|
|
12
11
|
private expiryHandler;
|
|
@@ -22,9 +21,8 @@ export declare class BasicNode {
|
|
|
22
21
|
[key: string]: any;
|
|
23
22
|
});
|
|
24
23
|
register(): Promise<void>;
|
|
25
|
-
healthcheck(
|
|
26
|
-
|
|
27
|
-
recommend(): Promise<string>;
|
|
24
|
+
healthcheck(): Promise<boolean>;
|
|
25
|
+
requestMarket(market?: string): Promise<Market>;
|
|
28
26
|
api(): ApiHandler;
|
|
29
27
|
isApiActive(): Promise<boolean>;
|
|
30
28
|
node(): string;
|
|
@@ -32,7 +30,7 @@ export declare class BasicNode {
|
|
|
32
30
|
stop(): Promise<void>;
|
|
33
31
|
clean(): Promise<void>;
|
|
34
32
|
start(): Promise<void>;
|
|
35
|
-
setup(market:
|
|
33
|
+
setup(market: Market): Promise<void>;
|
|
36
34
|
run(): Promise<void>;
|
|
37
35
|
pending(): Promise<boolean>;
|
|
38
36
|
queue(market?: string): Promise<void>;
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import { getSDK } from '../sdk/index.js';
|
|
2
2
|
import { MarketHandler } from './market/marketHandler.js';
|
|
3
|
+
import { NodeNotRegisteredError } from '../errors/NodeNotRegisteredError.js';
|
|
3
4
|
import { RunHandler } from './run/runHandler.js';
|
|
4
5
|
import { JobHandler } from './job/jobHandler.js';
|
|
5
6
|
import { DB } from '../db/index.js';
|
|
6
7
|
import { Provider } from '../provider/Provider.js';
|
|
7
8
|
import { applyLoggingProxyToClass } from '../monitoring/proxy/loggingProxy.js';
|
|
8
|
-
import { isNodeOnboarded } from '../utils/utils.js';
|
|
9
9
|
import { ApiHandler } from './api/ApiHandler.js';
|
|
10
10
|
import { NodeRepository } from '../repository/NodeRepository.js';
|
|
11
|
-
import { SpecsHandler } from './specs/specsHandler.js';
|
|
12
11
|
import { HealthHandler } from './health/healthHandler.js';
|
|
13
12
|
import { KeyHandler } from './key/keyHandler.js';
|
|
14
13
|
import { ExpiryHandler } from './expiry/expiryHandler.js';
|
|
@@ -24,7 +23,6 @@ import { TaskManagerRegistry } from './task/TaskManagerRegistry.js';
|
|
|
24
23
|
import { StopReasons } from './task/TaskManager.js';
|
|
25
24
|
export class BasicNode {
|
|
26
25
|
constructor(options) {
|
|
27
|
-
this.isOnboarded = false;
|
|
28
26
|
this.exiting = false;
|
|
29
27
|
this.sdk = getSDK();
|
|
30
28
|
const db = new DB(options.config).db;
|
|
@@ -35,39 +33,35 @@ export class BasicNode {
|
|
|
35
33
|
this.apiHandler = new ApiHandler(this.sdk, this.repository, this.provider, options.port);
|
|
36
34
|
this.balanceHandler = new BalanceHandler(this.sdk);
|
|
37
35
|
this.gridHandler = new GridHandler(this.sdk, this.repository);
|
|
38
|
-
this.specsHandler = new SpecsHandler(this.provider, this.repository, this.sdk);
|
|
39
36
|
this.jobHandler = new JobHandler(this.sdk, this.provider, this.repository);
|
|
40
|
-
this.marketHandler = new MarketHandler(this.sdk);
|
|
37
|
+
this.marketHandler = new MarketHandler(this.sdk, this.provider, this.repository);
|
|
41
38
|
this.runHandler = new RunHandler(this.sdk);
|
|
42
39
|
this.keyHandler = new KeyHandler(this.sdk);
|
|
43
40
|
this.healthHandler = new HealthHandler(this.sdk, this.containerOrchestration, this.marketHandler, this.keyHandler);
|
|
44
|
-
this.registerHandler = new RegisterHandler(
|
|
41
|
+
this.registerHandler = new RegisterHandler();
|
|
45
42
|
this.expiryHandler = new ExpiryHandler(this.sdk);
|
|
46
43
|
applyLoggingProxyToClass(this);
|
|
47
44
|
}
|
|
48
45
|
async register() {
|
|
49
46
|
await this.registerHandler.register();
|
|
50
47
|
}
|
|
51
|
-
async healthcheck(
|
|
48
|
+
async healthcheck() {
|
|
52
49
|
/**
|
|
53
50
|
* run health check,
|
|
54
51
|
*/
|
|
55
|
-
return await this.healthHandler.run(
|
|
52
|
+
return await this.healthHandler.run();
|
|
56
53
|
}
|
|
57
|
-
async
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
* so we need to get a recommended one and return it
|
|
69
|
-
*/
|
|
70
|
-
return await this.gridHandler.recommend();
|
|
54
|
+
async requestMarket(market) {
|
|
55
|
+
try {
|
|
56
|
+
return await this.marketHandler.request(market);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
if (error instanceof NodeNotRegisteredError) {
|
|
60
|
+
await this.registerHandler.register();
|
|
61
|
+
return await this.marketHandler.request(market);
|
|
62
|
+
}
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
71
65
|
}
|
|
72
66
|
api() {
|
|
73
67
|
return this.apiHandler;
|
|
@@ -101,13 +95,6 @@ export class BasicNode {
|
|
|
101
95
|
nodeAbortControllerSelector().refresh();
|
|
102
96
|
}
|
|
103
97
|
async start() {
|
|
104
|
-
/**
|
|
105
|
-
* we query the grid to find out if the node has already been onboarded
|
|
106
|
-
* if it has been onboarded it might have been assigned/recommended a market
|
|
107
|
-
* if it has not been onboarded quit the process
|
|
108
|
-
*/
|
|
109
|
-
const nodeData = await this.gridHandler.getNodeStatus();
|
|
110
|
-
this.isOnboarded = isNodeOnboarded(nodeData.status);
|
|
111
98
|
if (await this.balanceHandler.balance()) {
|
|
112
99
|
await this.balanceHandler.check(true);
|
|
113
100
|
}
|
|
@@ -6,8 +6,8 @@ import { configs } from '../../configs/configs.js';
|
|
|
6
6
|
import { sleep } from '../../utils/utils.js';
|
|
7
7
|
import { stateStreaming } from '../../monitoring/streaming/StateStreamer.js';
|
|
8
8
|
import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
|
|
9
|
-
import {
|
|
10
|
-
import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute,
|
|
9
|
+
import { verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware, verifyWSNodeOrJobOwnerSignatureMiddleware, } from './middlewares/index.js';
|
|
10
|
+
import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, moveGroupOperationHandler, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, } from './routes/index.js';
|
|
11
11
|
import { NodeAlreadyActiveError } from '../../errors/NodeAlreadyActiveError.js';
|
|
12
12
|
export class ApiHandler {
|
|
13
13
|
constructor(sdk, repository, provider, port) {
|
|
@@ -146,7 +146,6 @@ export class ApiHandler {
|
|
|
146
146
|
this.api.get('/job/:jobId/stats', getJobStatsRoute);
|
|
147
147
|
this.api.get('/job/:jobId/stats/stream', getJobStatsStreamRoute);
|
|
148
148
|
// POST Routes
|
|
149
|
-
this.api.post('/node/validate', verifyBackendSignatureMiddleware, postNodeValidation);
|
|
150
149
|
this.api.post('/job/:jobId/job-definition', postJobDefinitionRoute);
|
|
151
150
|
this.api.post('/job/:jobId/group/:group/move', moveGroupOperationHandler);
|
|
152
151
|
this.api.post('/job/:jobId/group/:group/operation/:opId/restart', restartOperationHandler);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Client as SDK } from
|
|
2
|
-
import { NodeRepository } from
|
|
1
|
+
import { Client as SDK } from "@nosana/sdk";
|
|
2
|
+
import { NodeRepository } from "../../repository/NodeRepository.js";
|
|
3
3
|
export interface NodeData {
|
|
4
4
|
market?: string;
|
|
5
5
|
status: string;
|
|
@@ -9,11 +9,5 @@ export declare class GridHandler {
|
|
|
9
9
|
private repository;
|
|
10
10
|
private address;
|
|
11
11
|
constructor(sdk: SDK, repository: NodeRepository);
|
|
12
|
-
private getAuthSignature;
|
|
13
12
|
getNodeStatus(): Promise<NodeData>;
|
|
14
|
-
recommend(): Promise<any>;
|
|
15
|
-
private changeMarket;
|
|
16
|
-
private signAndSendTransaction;
|
|
17
|
-
private confirmTransaction;
|
|
18
|
-
private syncNodeAfterMint;
|
|
19
13
|
}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import { applyLoggingProxyToClass } from
|
|
2
|
-
import {
|
|
3
|
-
import { getRawTransaction } from '../../sdk/index.js';
|
|
4
|
-
import { sleep } from '../../utils/utils.js';
|
|
5
|
-
import { configs } from '../../configs/configs.js';
|
|
1
|
+
import { applyLoggingProxyToClass } from "../../monitoring/proxy/loggingProxy.js";
|
|
2
|
+
import { HostManager } from '../market/hostManager.js';
|
|
6
3
|
export class GridHandler {
|
|
7
4
|
constructor(sdk, repository) {
|
|
8
5
|
this.sdk = sdk;
|
|
@@ -10,159 +7,29 @@ export class GridHandler {
|
|
|
10
7
|
this.address = this.sdk.solana.provider.wallet.publicKey;
|
|
11
8
|
applyLoggingProxyToClass(this);
|
|
12
9
|
}
|
|
13
|
-
async getAuthSignature() {
|
|
14
|
-
const signature = (await this.sdk.solana.signMessage(configs().signMessage));
|
|
15
|
-
return Buffer.from(signature).toString('base64');
|
|
16
|
-
}
|
|
17
10
|
async getNodeStatus() {
|
|
18
11
|
try {
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
Authorization: `${this.address}:${await this.getAuthSignature()}`,
|
|
23
|
-
'Content-Type': 'application/json',
|
|
24
|
-
},
|
|
25
|
-
});
|
|
26
|
-
const data = await response.json();
|
|
27
|
-
if (!data || (data.name === 'Error' && data.message))
|
|
28
|
-
throw new Error(data.message);
|
|
12
|
+
const data = await HostManager.getNode(this.address.toString());
|
|
13
|
+
if (!data)
|
|
14
|
+
throw new Error('Node not found');
|
|
29
15
|
return {
|
|
30
16
|
status: data.status,
|
|
31
|
-
market: data.marketAddress,
|
|
17
|
+
market: data.marketAddress ?? undefined,
|
|
32
18
|
};
|
|
33
19
|
}
|
|
34
20
|
catch (error) {
|
|
35
21
|
if (error instanceof Error &&
|
|
36
|
-
error.message.includes(
|
|
37
|
-
throw new Error(
|
|
22
|
+
error.message.includes("Node not onboarded yet")) {
|
|
23
|
+
throw new Error("Node is still on the waitlist, wait until you are accepted.");
|
|
38
24
|
}
|
|
39
25
|
else if (error instanceof Error &&
|
|
40
|
-
!error.message.includes(
|
|
26
|
+
!error.message.includes("Node not found")) {
|
|
41
27
|
throw error;
|
|
42
28
|
}
|
|
43
29
|
return {
|
|
44
|
-
status:
|
|
30
|
+
status: "not-found",
|
|
45
31
|
market: undefined,
|
|
46
32
|
};
|
|
47
33
|
}
|
|
48
34
|
}
|
|
49
|
-
async recommend() {
|
|
50
|
-
const gpus = this.repository.getNodeInfo().gpus;
|
|
51
|
-
const signature = await this.getAuthSignature();
|
|
52
|
-
try {
|
|
53
|
-
const response = await fetch(`${configs().backendUrl}/nodes/${this.address}/check-market`, {
|
|
54
|
-
method: 'POST',
|
|
55
|
-
headers: {
|
|
56
|
-
Authorization: `${this.address}:${signature}`,
|
|
57
|
-
'Content-Type': 'application/json',
|
|
58
|
-
},
|
|
59
|
-
body: JSON.stringify({ gpus: JSON.stringify(gpus) }),
|
|
60
|
-
});
|
|
61
|
-
let data = await response.json();
|
|
62
|
-
if (!data) {
|
|
63
|
-
throw new Error('Something went wrong with recommending the market, please try again.');
|
|
64
|
-
}
|
|
65
|
-
if (data.needsChangeMarket) {
|
|
66
|
-
data = await this.changeMarket();
|
|
67
|
-
return data.newMarket;
|
|
68
|
-
}
|
|
69
|
-
if (data.address) {
|
|
70
|
-
return data.address;
|
|
71
|
-
}
|
|
72
|
-
if (data.message) {
|
|
73
|
-
throw new Error(data.message);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
throw error;
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
async changeMarket() {
|
|
81
|
-
try {
|
|
82
|
-
const response = await fetch(`${configs().backendUrl}/nodes/change-market`, {
|
|
83
|
-
method: 'POST',
|
|
84
|
-
headers: {
|
|
85
|
-
Authorization: `${this.address}:${await this.getAuthSignature()}`,
|
|
86
|
-
'Content-Type': 'application/json',
|
|
87
|
-
},
|
|
88
|
-
body: JSON.stringify({ address: this.address }),
|
|
89
|
-
});
|
|
90
|
-
const data = await response.json();
|
|
91
|
-
if (!data || data.name === 'Error')
|
|
92
|
-
throw new Error(data.message);
|
|
93
|
-
// Incase of blockheight exceeded error, retry 3 times
|
|
94
|
-
for (let i = 0; i < 3; i++) {
|
|
95
|
-
try {
|
|
96
|
-
const txnSignature = await this.signAndSendTransaction(data.tx);
|
|
97
|
-
await this.confirmTransaction(txnSignature);
|
|
98
|
-
break;
|
|
99
|
-
}
|
|
100
|
-
catch (error) {
|
|
101
|
-
if (i === 2 ||
|
|
102
|
-
!error?.message.includes('TransactionExpiredBlockheightExceededError')) {
|
|
103
|
-
throw error;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
// TODO: verify tx result with code below
|
|
108
|
-
// const result = await this.sdk.solana.connection?.getTransaction(
|
|
109
|
-
// txnSignature as string,
|
|
110
|
-
// { maxSupportedTransactionVersion: 0 },
|
|
111
|
-
// );
|
|
112
|
-
// // @ts-ignore
|
|
113
|
-
// if (result?.meta?.status.Err) {
|
|
114
|
-
// // @ts-ignore
|
|
115
|
-
// throw new Error(JSON.stringify(result?.meta?.status.Err));
|
|
116
|
-
// }
|
|
117
|
-
await sleep(30);
|
|
118
|
-
await this.syncNodeAfterMint();
|
|
119
|
-
return data;
|
|
120
|
-
}
|
|
121
|
-
catch (error) {
|
|
122
|
-
throw new Error('Something went wrong with minting your access key, please try again. ' +
|
|
123
|
-
error);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
async signAndSendTransaction(txData) {
|
|
127
|
-
const feePayer = (this.sdk.solana.provider?.wallet).payer;
|
|
128
|
-
const recoveredTransaction = await getRawTransaction(Uint8Array.from(Object.values(txData)));
|
|
129
|
-
if (recoveredTransaction instanceof VersionedTransaction) {
|
|
130
|
-
recoveredTransaction.sign([feePayer]);
|
|
131
|
-
}
|
|
132
|
-
else {
|
|
133
|
-
recoveredTransaction.partialSign(feePayer);
|
|
134
|
-
}
|
|
135
|
-
const txnSignature = await this.sdk.solana.connection?.sendRawTransaction(recoveredTransaction.serialize());
|
|
136
|
-
return txnSignature;
|
|
137
|
-
}
|
|
138
|
-
async confirmTransaction(txnSignature) {
|
|
139
|
-
const latestBlockHash = await this.sdk.solana.connection?.getLatestBlockhash();
|
|
140
|
-
if (latestBlockHash && txnSignature) {
|
|
141
|
-
const confirmStrategy = {
|
|
142
|
-
blockhash: latestBlockHash.blockhash,
|
|
143
|
-
lastValidBlockHeight: latestBlockHash.lastValidBlockHeight,
|
|
144
|
-
signature: txnSignature,
|
|
145
|
-
};
|
|
146
|
-
await this.sdk.solana.connection?.confirmTransaction(confirmStrategy);
|
|
147
|
-
}
|
|
148
|
-
else {
|
|
149
|
-
throw new Error('Could not confirm minting transaction');
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
async syncNodeAfterMint() {
|
|
153
|
-
try {
|
|
154
|
-
const response = await fetch(`${configs().backendUrl}/nodes/sync-node`, {
|
|
155
|
-
method: 'POST',
|
|
156
|
-
headers: {
|
|
157
|
-
Authorization: `${this.address}:${await this.getAuthSignature()}`,
|
|
158
|
-
'Content-Type': 'application/json',
|
|
159
|
-
},
|
|
160
|
-
body: JSON.stringify({ address: this.address }),
|
|
161
|
-
});
|
|
162
|
-
return response.json();
|
|
163
|
-
}
|
|
164
|
-
catch (error) {
|
|
165
|
-
throw error;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
35
|
}
|
|
@@ -11,6 +11,6 @@ export declare class HealthHandler {
|
|
|
11
11
|
private balanceHandler;
|
|
12
12
|
private stakeHandler;
|
|
13
13
|
constructor(sdk: SDK, containerOrchestration: ContainerOrchestrationInterface, marketHandler: MarketHandler, keyHandler: KeyHandler);
|
|
14
|
-
run(
|
|
14
|
+
run(): Promise<boolean>;
|
|
15
15
|
stake(market: Market): Promise<boolean>;
|
|
16
16
|
}
|
|
@@ -13,13 +13,13 @@ export class HealthHandler {
|
|
|
13
13
|
this.stakeHandler = new StakeHandler(this.sdk);
|
|
14
14
|
applyLoggingProxyToClass(this);
|
|
15
15
|
}
|
|
16
|
-
async run(
|
|
16
|
+
async run() {
|
|
17
17
|
/**
|
|
18
18
|
* market health check,
|
|
19
19
|
*
|
|
20
20
|
* check if the market supplied is valid and can be joined
|
|
21
21
|
*/
|
|
22
|
-
const marketAccount = await this.marketHandler.
|
|
22
|
+
const marketAccount = await this.marketHandler.getMarket();
|
|
23
23
|
/**
|
|
24
24
|
* balances (SOL) health check
|
|
25
25
|
*
|
|
@@ -29,15 +29,17 @@ export class HealthHandler {
|
|
|
29
29
|
if (await this.balanceHandler.balance()) {
|
|
30
30
|
await this.balanceHandler.check();
|
|
31
31
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
32
|
+
if (marketAccount) {
|
|
33
|
+
/**
|
|
34
|
+
* staking healtcheck, check if market can be joined or staking is needed
|
|
35
|
+
*/
|
|
36
|
+
await this.stake(marketAccount);
|
|
37
|
+
/**
|
|
38
|
+
* health check on market access key
|
|
39
|
+
*/
|
|
40
|
+
if (this.keyHandler.doesMarketNeedAccessKey(marketAccount)) {
|
|
41
|
+
await this.keyHandler.loadAccessKeyFromChain(marketAccount);
|
|
42
|
+
}
|
|
41
43
|
}
|
|
42
44
|
/**
|
|
43
45
|
* health check for provider (docker or podman) health
|