nebula-notebook 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -20
- package/dist/assets/{errorwidget-C4r2j2DQ.js → errorwidget-Wy5ajPiM.js} +1 -1
- package/dist/assets/{index-Day3QcNs.js → index-B_Bs5koo.js} +1 -1
- package/dist/assets/index-DK4nS74r.js +829 -0
- package/dist/assets/{index-D5w21_Z8.js → index-DeQN03nW.js} +1 -1
- package/dist/assets/index-DzjpmUak.css +32 -0
- package/dist/assets/{index-CsHoPQy-.js → index-g6MjT-VG.js} +1 -1
- package/dist/assets/{services-shim-D6p_A67v.js → services-shim-BirHVsih.js} +1 -1
- package/dist/index.html +2 -2
- package/node-server/dist/auth/auth-middleware.js +22 -4
- package/node-server/dist/auth/auth-service.js +37 -7
- package/node-server/dist/cluster/client-registration.d.ts +12 -1
- package/node-server/dist/cluster/client-registration.js +35 -2
- package/node-server/dist/cluster/kernel-proxy.js +19 -3
- package/node-server/dist/cluster/server-registry.d.ts +9 -0
- package/node-server/dist/cluster/server-registry.js +26 -0
- package/node-server/dist/discovery/conda-locations.d.ts +73 -0
- package/node-server/dist/discovery/conda-locations.js +427 -0
- package/node-server/dist/discovery/discovery-service.d.ts +73 -4
- package/node-server/dist/discovery/discovery-service.js +211 -127
- package/node-server/dist/discovery/types.d.ts +0 -1
- package/node-server/dist/discovery/types.js +1 -2
- package/node-server/dist/environment.d.ts +12 -0
- package/node-server/dist/environment.js +104 -0
- package/node-server/dist/fs/fs-service.d.ts +31 -1
- package/node-server/dist/fs/fs-service.js +160 -28
- package/node-server/dist/fs/types.d.ts +9 -0
- package/node-server/dist/fs/types.js +8 -0
- package/node-server/dist/index.js +71 -4
- package/node-server/dist/kernel/kernel-service.d.ts +17 -0
- package/node-server/dist/kernel/kernel-service.js +72 -4
- package/node-server/dist/kernel/kernelspec.d.ts +27 -0
- package/node-server/dist/kernel/kernelspec.js +75 -4
- package/node-server/dist/notebook/headless-handler.d.ts +10 -0
- package/node-server/dist/notebook/headless-handler.js +55 -5
- package/node-server/dist/notebook/undoRedoManager.d.ts +8 -0
- package/node-server/dist/notebook/undoRedoManager.js +39 -16
- package/node-server/dist/private-tmp.d.ts +14 -0
- package/node-server/dist/private-tmp.js +70 -0
- package/node-server/dist/routes/autocomplete.d.ts +19 -0
- package/node-server/dist/routes/autocomplete.js +294 -0
- package/node-server/dist/routes/fs.js +21 -4
- package/node-server/dist/routes/kernel.js +45 -2
- package/node-server/dist/routes/notebook.js +8 -2
- package/node-server/dist/routes/python.d.ts +1 -2
- package/node-server/dist/routes/python.js +139 -3
- package/node-server/dist/scheduler/allocation-service.d.ts +20 -0
- package/node-server/dist/scheduler/allocation-service.js +124 -8
- package/node-server/dist/scheduler/mock-scheduler.js +7 -3
- package/node-server/dist/scheduler/slurm-scheduler.js +21 -8
- package/node-server/dist/scheduler/types.d.ts +3 -2
- package/node-server/dist/terminal/agent-registry.d.ts +87 -0
- package/node-server/dist/terminal/agent-registry.js +246 -0
- package/node-server/dist/terminal/binding-store.d.ts +57 -0
- package/node-server/dist/terminal/binding-store.js +0 -0
- package/node-server/dist/terminal/pty-manager.d.ts +23 -6
- package/node-server/dist/terminal/pty-manager.js +130 -19
- package/node-server/dist/terminal/server.js +112 -21
- package/node-server/dist/terminal/types.d.ts +2 -0
- package/node-server/package.json +1 -0
- package/node_modules/nebula-autocomplete/README.md +145 -0
- package/node_modules/nebula-autocomplete/dist/backends/claude.d.ts +69 -0
- package/node_modules/nebula-autocomplete/dist/backends/claude.js +449 -0
- package/node_modules/nebula-autocomplete/dist/backends/codex.d.ts +45 -0
- package/node_modules/nebula-autocomplete/dist/backends/codex.js +108 -0
- package/node_modules/nebula-autocomplete/dist/client/fetcher.d.ts +16 -0
- package/node_modules/nebula-autocomplete/dist/client/fetcher.js +41 -0
- package/node_modules/nebula-autocomplete/dist/codemirror/ghostText.d.ts +43 -0
- package/node_modules/nebula-autocomplete/dist/codemirror/ghostText.js +175 -0
- package/node_modules/nebula-autocomplete/dist/core/engine.d.ts +17 -0
- package/node_modules/nebula-autocomplete/dist/core/engine.js +135 -0
- package/node_modules/nebula-autocomplete/dist/core/lru.d.ts +7 -0
- package/node_modules/nebula-autocomplete/dist/core/lru.js +26 -0
- package/node_modules/nebula-autocomplete/dist/core/prompt.d.ts +12 -0
- package/node_modules/nebula-autocomplete/dist/core/prompt.js +0 -0
- package/node_modules/nebula-autocomplete/dist/core/text.d.ts +53 -0
- package/node_modules/nebula-autocomplete/dist/core/text.js +169 -0
- package/node_modules/nebula-autocomplete/dist/index.d.ts +7 -0
- package/node_modules/nebula-autocomplete/dist/index.js +5 -0
- package/node_modules/nebula-autocomplete/dist/server/fastify.d.ts +40 -0
- package/node_modules/nebula-autocomplete/dist/server/fastify.js +52 -0
- package/node_modules/nebula-autocomplete/dist/transport.d.ts +55 -0
- package/node_modules/nebula-autocomplete/dist/transport.js +66 -0
- package/node_modules/nebula-autocomplete/dist/types.d.ts +103 -0
- package/node_modules/nebula-autocomplete/dist/types.js +1 -0
- package/node_modules/nebula-autocomplete/package.json +58 -0
- package/node_modules/nebula-autocomplete/server.d.ts +3 -0
- package/node_modules/nebula-autocomplete/server.js +3 -0
- package/package.json +17 -5
- package/scripts/postinstall.cjs +25 -5
- package/dist/assets/index-7-YBurka.js +0 -716
- package/dist/assets/index-CFBUnxSZ.css +0 -32
- package/node-server/dist/llm/index.d.ts +0 -5
- package/node-server/dist/llm/index.js +0 -21
- package/node-server/dist/llm/llm-service.d.ts +0 -77
- package/node-server/dist/llm/llm-service.js +0 -454
- package/node-server/dist/llm/types.d.ts +0 -40
- package/node-server/dist/llm/types.js +0 -15
- package/node-server/dist/routes/llm.d.ts +0 -8
- package/node-server/dist/routes/llm.js +0 -105
|
@@ -29,7 +29,27 @@ declare class AllocationService {
|
|
|
29
29
|
private allocations;
|
|
30
30
|
private pollTimer;
|
|
31
31
|
private enabled;
|
|
32
|
+
private lastPollAt;
|
|
33
|
+
private lostListenerRegistered;
|
|
32
34
|
init(scheduler: Scheduler, ctx: LaunchContext): void;
|
|
35
|
+
/** Poll immediately (debounced to 2s so error bursts don't hammer squeue). */
|
|
36
|
+
pollNow(): void;
|
|
37
|
+
private stateFile;
|
|
38
|
+
/**
|
|
39
|
+
* Allocations survive head-server restarts: persisted on every change,
|
|
40
|
+
* reloaded on init. A reloaded 'active' allocation is demoted to 'running'
|
|
41
|
+
* with its serverId cleared — the registry is empty after a restart, and
|
|
42
|
+
* the compute node's client-server re-registers itself (heartbeat -> 404
|
|
43
|
+
* -> re-register with its allocation token) within ~30s, at which point
|
|
44
|
+
* poll() re-correlates and promotes it back to 'active'. The SLURM job
|
|
45
|
+
* itself is re-checked by jobId on the next poll, so jobs that died while
|
|
46
|
+
* we were down are marked ended/failed instead of lingering.
|
|
47
|
+
*/
|
|
48
|
+
private persist;
|
|
49
|
+
private loadPersisted;
|
|
50
|
+
/** Pick the poll cadence from what we're actually waiting for. */
|
|
51
|
+
private nextPollDelay;
|
|
52
|
+
private scheduleNextPoll;
|
|
33
53
|
isEnabled(): boolean;
|
|
34
54
|
getScheduler(): Scheduler | null;
|
|
35
55
|
list(): Allocation[];
|
|
@@ -49,7 +49,13 @@ const fs = __importStar(require("fs"));
|
|
|
49
49
|
const path = __importStar(require("path"));
|
|
50
50
|
const job_template_1 = require("./job-template");
|
|
51
51
|
const server_registry_1 = require("../cluster/server-registry");
|
|
52
|
-
|
|
52
|
+
// Adaptive polling: fast only while a transition is imminent (job climbing
|
|
53
|
+
// the queue / server booting), slow once allocations are correlated and
|
|
54
|
+
// steady (we're only watching for job end), slowest when idle. Each squeue
|
|
55
|
+
// poll is a real scheduler hit on a shared login node — don't burn them.
|
|
56
|
+
const POLL_FAST_MS = 5_000; // pending/running/uncorrelated allocations
|
|
57
|
+
const POLL_STEADY_MS = 30_000; // all tracked allocations active + correlated
|
|
58
|
+
const POLL_IDLE_MS = 60_000; // nothing non-terminal to watch
|
|
53
59
|
const TERMINAL = ['ended', 'failed', 'cancelled'];
|
|
54
60
|
class AllocationService {
|
|
55
61
|
scheduler = null;
|
|
@@ -57,16 +63,116 @@ class AllocationService {
|
|
|
57
63
|
allocations = new Map();
|
|
58
64
|
pollTimer = null;
|
|
59
65
|
enabled = false;
|
|
66
|
+
lastPollAt = 0;
|
|
67
|
+
lostListenerRegistered = false;
|
|
60
68
|
init(scheduler, ctx) {
|
|
61
69
|
this.scheduler = scheduler;
|
|
62
70
|
this.ctx = ctx;
|
|
63
71
|
this.enabled = true;
|
|
64
72
|
fs.mkdirSync(ctx.stateDir, { recursive: true });
|
|
73
|
+
this.loadPersisted();
|
|
74
|
+
this.scheduleNextPoll(POLL_FAST_MS);
|
|
75
|
+
// React instantly when the cluster layer loses contact with a server
|
|
76
|
+
// that belongs to one of our allocations (kernel WS dropped, proxy
|
|
77
|
+
// request refused, heartbeat timeout) — verify against the scheduler
|
|
78
|
+
// NOW instead of waiting out the steady 30s cadence.
|
|
79
|
+
if (!this.lostListenerRegistered) {
|
|
80
|
+
this.lostListenerRegistered = true;
|
|
81
|
+
server_registry_1.serverRegistry.onServerLost((server) => {
|
|
82
|
+
const tracked = [...this.allocations.values()].some((a) => !TERMINAL.includes(a.state) &&
|
|
83
|
+
(a.serverId === server.id || (server.allocationToken && a.token === server.allocationToken)));
|
|
84
|
+
if (tracked) {
|
|
85
|
+
console.log(`[Scheduler] Lost contact with ${server.id} — checking its allocation now`);
|
|
86
|
+
this.pollNow();
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** Poll immediately (debounced to 2s so error bursts don't hammer squeue). */
|
|
92
|
+
pollNow() {
|
|
93
|
+
if (!this.enabled)
|
|
94
|
+
return;
|
|
95
|
+
if (Date.now() - this.lastPollAt < 2_000)
|
|
96
|
+
return;
|
|
97
|
+
this.scheduleNextPoll(0);
|
|
98
|
+
}
|
|
99
|
+
stateFile() {
|
|
100
|
+
return this.ctx ? path.join(this.ctx.stateDir, 'allocations.json') : null;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Allocations survive head-server restarts: persisted on every change,
|
|
104
|
+
* reloaded on init. A reloaded 'active' allocation is demoted to 'running'
|
|
105
|
+
* with its serverId cleared — the registry is empty after a restart, and
|
|
106
|
+
* the compute node's client-server re-registers itself (heartbeat -> 404
|
|
107
|
+
* -> re-register with its allocation token) within ~30s, at which point
|
|
108
|
+
* poll() re-correlates and promotes it back to 'active'. The SLURM job
|
|
109
|
+
* itself is re-checked by jobId on the next poll, so jobs that died while
|
|
110
|
+
* we were down are marked ended/failed instead of lingering.
|
|
111
|
+
*/
|
|
112
|
+
persist() {
|
|
113
|
+
const file = this.stateFile();
|
|
114
|
+
if (!file)
|
|
115
|
+
return;
|
|
116
|
+
try {
|
|
117
|
+
const all = [...this.allocations.values()];
|
|
118
|
+
// Cap history so the file can't grow unboundedly: all live ones,
|
|
119
|
+
// plus the 20 most recent terminal ones for the UI's history list.
|
|
120
|
+
const live = all.filter((a) => !TERMINAL.includes(a.state));
|
|
121
|
+
const done = all.filter((a) => TERMINAL.includes(a.state))
|
|
122
|
+
.sort((a, b) => b.createdAt - a.createdAt).slice(0, 20);
|
|
123
|
+
const tmp = `${file}.tmp`;
|
|
124
|
+
fs.writeFileSync(tmp, JSON.stringify([...live, ...done]));
|
|
125
|
+
fs.renameSync(tmp, file);
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
console.error('[Scheduler] failed to persist allocations:', err);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
loadPersisted() {
|
|
132
|
+
const file = this.stateFile();
|
|
133
|
+
if (!file || !fs.existsSync(file))
|
|
134
|
+
return;
|
|
135
|
+
try {
|
|
136
|
+
const list = JSON.parse(fs.readFileSync(file, 'utf-8'));
|
|
137
|
+
let revived = 0;
|
|
138
|
+
for (const alloc of list) {
|
|
139
|
+
if (!alloc?.id || this.allocations.has(alloc.id))
|
|
140
|
+
continue;
|
|
141
|
+
if (!TERMINAL.includes(alloc.state)) {
|
|
142
|
+
if (alloc.state === 'active')
|
|
143
|
+
alloc.state = 'running';
|
|
144
|
+
alloc.serverId = undefined; // fresh registry — re-correlate via token
|
|
145
|
+
revived++;
|
|
146
|
+
}
|
|
147
|
+
this.allocations.set(alloc.id, alloc);
|
|
148
|
+
}
|
|
149
|
+
if (revived)
|
|
150
|
+
console.log(`[Scheduler] Recovered ${revived} live allocation(s) from disk`);
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
console.error('[Scheduler] failed to load persisted allocations:', err);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** Pick the poll cadence from what we're actually waiting for. */
|
|
157
|
+
nextPollDelay() {
|
|
158
|
+
const live = [...this.allocations.values()].filter((a) => !TERMINAL.includes(a.state));
|
|
159
|
+
if (live.length === 0)
|
|
160
|
+
return POLL_IDLE_MS;
|
|
161
|
+
return live.some((a) => a.state !== 'active' || !a.serverId) ? POLL_FAST_MS : POLL_STEADY_MS;
|
|
162
|
+
}
|
|
163
|
+
scheduleNextPoll(delay) {
|
|
65
164
|
if (this.pollTimer)
|
|
66
|
-
|
|
67
|
-
this.pollTimer =
|
|
68
|
-
|
|
69
|
-
|
|
165
|
+
clearTimeout(this.pollTimer);
|
|
166
|
+
this.pollTimer = setTimeout(async () => {
|
|
167
|
+
try {
|
|
168
|
+
await this.poll();
|
|
169
|
+
}
|
|
170
|
+
catch (err) {
|
|
171
|
+
console.error('[Scheduler] poll error:', err);
|
|
172
|
+
}
|
|
173
|
+
if (this.enabled)
|
|
174
|
+
this.scheduleNextPoll(this.nextPollDelay());
|
|
175
|
+
}, delay);
|
|
70
176
|
}
|
|
71
177
|
isEnabled() {
|
|
72
178
|
return this.enabled;
|
|
@@ -92,9 +198,11 @@ class AllocationService {
|
|
|
92
198
|
const { jobId } = await this.scheduler.submit(scriptPath);
|
|
93
199
|
alloc.jobId = jobId;
|
|
94
200
|
this.allocations.set(id, alloc);
|
|
201
|
+
this.persist();
|
|
95
202
|
console.log(`[Scheduler] Allocation ${id} submitted as job ${jobId} (${spec.partition}${spec.qos ? '/' + spec.qos : ''})`);
|
|
96
|
-
//
|
|
97
|
-
|
|
203
|
+
// Poll now and drop back to the fast cadence — a create can land while
|
|
204
|
+
// the poller is in a slow idle/steady wait.
|
|
205
|
+
this.scheduleNextPoll(0);
|
|
98
206
|
return alloc;
|
|
99
207
|
}
|
|
100
208
|
async cancel(id) {
|
|
@@ -112,11 +220,14 @@ class AllocationService {
|
|
|
112
220
|
if (alloc.serverId)
|
|
113
221
|
server_registry_1.serverRegistry.unregister(alloc.serverId);
|
|
114
222
|
alloc.state = 'cancelled';
|
|
223
|
+
this.persist();
|
|
115
224
|
return true;
|
|
116
225
|
}
|
|
117
226
|
async poll() {
|
|
118
227
|
if (!this.scheduler)
|
|
119
228
|
return;
|
|
229
|
+
this.lastPollAt = Date.now();
|
|
230
|
+
let dirty = false;
|
|
120
231
|
for (const alloc of this.allocations.values()) {
|
|
121
232
|
if (TERMINAL.includes(alloc.state))
|
|
122
233
|
continue;
|
|
@@ -127,6 +238,7 @@ class AllocationService {
|
|
|
127
238
|
alloc.serverId = server.id;
|
|
128
239
|
alloc.state = 'active';
|
|
129
240
|
alloc.nodes = [server.host];
|
|
241
|
+
dirty = true;
|
|
130
242
|
if (!alloc.walltimeEndsAt) {
|
|
131
243
|
alloc.walltimeEndsAt = Date.now() + alloc.spec.walltimeMinutes * 60_000;
|
|
132
244
|
}
|
|
@@ -145,6 +257,7 @@ class AllocationService {
|
|
|
145
257
|
}
|
|
146
258
|
if (status.state === 'running' && alloc.state === 'pending') {
|
|
147
259
|
alloc.state = 'running';
|
|
260
|
+
dirty = true;
|
|
148
261
|
alloc.nodes = status.nodes.length ? status.nodes : alloc.nodes;
|
|
149
262
|
if (!alloc.walltimeEndsAt) {
|
|
150
263
|
alloc.walltimeEndsAt = Date.now() + alloc.spec.walltimeMinutes * 60_000;
|
|
@@ -153,15 +266,18 @@ class AllocationService {
|
|
|
153
266
|
else if (['completed', 'cancelled', 'failed'].includes(status.state)) {
|
|
154
267
|
alloc.state = status.state === 'failed' ? 'failed' : status.state === 'cancelled' ? 'cancelled' : 'ended';
|
|
155
268
|
alloc.reason = status.reason;
|
|
269
|
+
dirty = true;
|
|
156
270
|
if (alloc.serverId)
|
|
157
271
|
server_registry_1.serverRegistry.unregister(alloc.serverId);
|
|
158
272
|
console.log(`[Scheduler] Allocation ${alloc.id} ${alloc.state} (job ${alloc.jobId})`);
|
|
159
273
|
}
|
|
160
274
|
}
|
|
275
|
+
if (dirty)
|
|
276
|
+
this.persist();
|
|
161
277
|
}
|
|
162
278
|
shutdown() {
|
|
163
279
|
if (this.pollTimer) {
|
|
164
|
-
|
|
280
|
+
clearTimeout(this.pollTimer);
|
|
165
281
|
this.pollTimer = null;
|
|
166
282
|
}
|
|
167
283
|
}
|
|
@@ -32,10 +32,14 @@ function partitions() {
|
|
|
32
32
|
return [
|
|
33
33
|
p('cpu', 512, 236, '1-00:00:00', { pending: 5, running: 44 }),
|
|
34
34
|
p('cpu-long', 256, 14, '7-00:00:00', { pending: 71, running: 58 }),
|
|
35
|
-
|
|
36
|
-
p('gpu
|
|
35
|
+
// Heterogeneous GPU queue — two card models, shown as separate rows.
|
|
36
|
+
p('gpu', 128, 46, '1-00:00:00', { pending: 3, running: 12 }, [
|
|
37
|
+
{ type: 'nvidia_l40s', total: 16, idle: 7 },
|
|
38
|
+
{ type: 'nvidia_rtx_6000', total: 8, idle: 2 },
|
|
39
|
+
]),
|
|
40
|
+
p('gpu-a100', 96, 22, '1-00:00:00', { pending: 6, running: 9 }, [{ type: 'nvidia_a100_80gb', total: 8, idle: 3 }]),
|
|
37
41
|
p('bigmem', 192, 104, '2-00:00:00', { pending: 1, running: 6 }),
|
|
38
|
-
p('lab', 64, 28, '30-00:00:00', { pending: 0, running: 3 }, { type: 'nvidia_h100_80gb', total: 8, idle: 5 }),
|
|
42
|
+
p('lab', 64, 28, '30-00:00:00', { pending: 0, running: 3 }, [{ type: 'nvidia_h100_80gb', total: 8, idle: 5 }]),
|
|
39
43
|
];
|
|
40
44
|
}
|
|
41
45
|
function qoses() {
|
|
@@ -50,6 +50,10 @@ async function expandNodes(nodelist) {
|
|
|
50
50
|
const nl = (nodelist || '').trim();
|
|
51
51
|
if (!nl || nl === '(null)' || nl === 'None' || nl === 'n/a')
|
|
52
52
|
return [];
|
|
53
|
+
// Plain single hostname (no ranges/lists) — the common case for 1-node
|
|
54
|
+
// allocations. Don't shell out to scontrol just to echo it back.
|
|
55
|
+
if (!/[\[\],]/.test(nl))
|
|
56
|
+
return [nl];
|
|
53
57
|
try {
|
|
54
58
|
const { stdout } = await run('scontrol', ['show', 'hostnames', nl], 5_000);
|
|
55
59
|
return stdout.split('\n').map((s) => s.trim()).filter(Boolean);
|
|
@@ -65,7 +69,9 @@ class SlurmScheduler {
|
|
|
65
69
|
// Short-TTL caches + in-flight de-duplication so the dashboard's 15s poll, the
|
|
66
70
|
// allocation modal, and manual refresh don't each re-run the (potentially slow)
|
|
67
71
|
// sinfo/scontrol/squeue/sacctmgr queries against the scheduler.
|
|
68
|
-
|
|
72
|
+
// 20s: deliberately ABOVE the clients' 15s poll so steady-state polling
|
|
73
|
+
// alternates cache hit/miss instead of missing every time (10s never hit).
|
|
74
|
+
static LOAD_TTL_MS = 20_000;
|
|
69
75
|
static ASSOC_TTL_MS = 60_000;
|
|
70
76
|
loadCache = null;
|
|
71
77
|
loadInflight = null;
|
|
@@ -205,6 +211,7 @@ class SlurmScheduler {
|
|
|
205
211
|
// allocated (AllocTRES) `gres/gpu`, so we can report *idle* (available) GPUs
|
|
206
212
|
// rather than a per-node count. Generic — no site-specific node/gres names.
|
|
207
213
|
if (nodeRes.status === 'fulfilled') {
|
|
214
|
+
// Aggregate per (partition, GPU model) — heterogeneous queues mix cards.
|
|
208
215
|
const agg = new Map();
|
|
209
216
|
for (const line of nodeRes.value.stdout.split('\n')) {
|
|
210
217
|
if (!line.trim())
|
|
@@ -219,18 +226,24 @@ class SlurmScheduler {
|
|
|
219
226
|
const allocGpu = toInt((scontrolField(line, 'AllocTRES')?.match(/gres\/gpu=(\d+)/) || [])[1]);
|
|
220
227
|
const type = (scontrolField(line, 'Gres')?.match(/gpu:([^:(]+)/) || [])[1] || 'gpu';
|
|
221
228
|
for (const part of parts.split(',')) {
|
|
222
|
-
const
|
|
229
|
+
const byType = agg.get(part) ?? new Map();
|
|
230
|
+
const e = byType.get(type) || { total: 0, used: 0 };
|
|
223
231
|
e.total += cfgGpu;
|
|
224
232
|
e.used += allocGpu;
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
agg.set(part, e);
|
|
233
|
+
byType.set(type, e);
|
|
234
|
+
agg.set(part, byType);
|
|
228
235
|
}
|
|
229
236
|
}
|
|
230
|
-
for (const [part,
|
|
237
|
+
for (const [part, byType] of agg) {
|
|
231
238
|
const p = partitions.get(part);
|
|
232
|
-
if (p
|
|
233
|
-
|
|
239
|
+
if (!p)
|
|
240
|
+
continue;
|
|
241
|
+
const list = [...byType.entries()]
|
|
242
|
+
.filter(([, g]) => g.total > 0)
|
|
243
|
+
.map(([type, g]) => ({ type, total: g.total, idle: Math.max(0, g.total - g.used) }))
|
|
244
|
+
.sort((a, b) => b.total - a.total || a.type.localeCompare(b.type));
|
|
245
|
+
if (list.length)
|
|
246
|
+
p.gpus = list;
|
|
234
247
|
}
|
|
235
248
|
}
|
|
236
249
|
// Queue depth from squeue, aggregated per partition AND per QoS.
|
|
@@ -52,12 +52,13 @@ export interface PartitionLoad {
|
|
|
52
52
|
other: number;
|
|
53
53
|
total: number;
|
|
54
54
|
};
|
|
55
|
-
/** GPU capacity for the partition
|
|
55
|
+
/** GPU capacity for the partition, one entry per GPU model (heterogeneous
|
|
56
|
+
* queues carry several): total configured vs currently idle (available). */
|
|
56
57
|
gpus?: {
|
|
57
58
|
type: string;
|
|
58
59
|
total: number;
|
|
59
60
|
idle: number;
|
|
60
|
-
};
|
|
61
|
+
}[];
|
|
61
62
|
nodes: {
|
|
62
63
|
idle: number;
|
|
63
64
|
mixed: number;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent registry — the server-side ledger of agent sessions (Claude Code /
|
|
3
|
+
* Codex CLIs running inside terminal ptys).
|
|
4
|
+
*
|
|
5
|
+
* Agents are project-scoped (keyed by the working directory they were
|
|
6
|
+
* launched in) and decoupled from notebooks: switching notebooks never kills
|
|
7
|
+
* or switches an agent; the user does, through the agent manager. Records
|
|
8
|
+
* persist to disk so agents survive as *resumable* entities even when their
|
|
9
|
+
* pty dies (browser gone, server restart, user hibernates them): the CLI's
|
|
10
|
+
* own on-disk trajectory (`claude --resume <id>` / `codex resume`) can
|
|
11
|
+
* reconstruct the conversation — the registry just remembers what exists,
|
|
12
|
+
* where, and how to revive it. The client owns launch/revive command
|
|
13
|
+
* construction (it knows about remote-agent mode); the server owns truth
|
|
14
|
+
* about what is live.
|
|
15
|
+
*
|
|
16
|
+
* States: 'live' (pty exists) → 'hibernated' (pty gone, trajectory on disk).
|
|
17
|
+
* All records load as 'hibernated' on boot — ptys never survive a restart.
|
|
18
|
+
*/
|
|
19
|
+
export interface AgentRecord {
|
|
20
|
+
/** Terminal (pty) id the agent runs in — stable, derived from workdir. */
|
|
21
|
+
terminalId: string;
|
|
22
|
+
kind: 'claude' | 'codex';
|
|
23
|
+
/** Directory the agent was launched in — the project scope. */
|
|
24
|
+
workdir: string;
|
|
25
|
+
/** Where the CLI process runs: this server, or the user's machine over the reverse channel. */
|
|
26
|
+
location: 'server' | 'remote';
|
|
27
|
+
/** Claude --session-id (resume pointer). Codex resumes via its own picker. */
|
|
28
|
+
sessionId?: string;
|
|
29
|
+
/** Notebook that launched it (informational only — agents are not bound to notebooks). */
|
|
30
|
+
launchedFrom?: string;
|
|
31
|
+
/**
|
|
32
|
+
* Pinned workspace-mirror dir slug (`p-<hash>-<name>` under ~/.nebula/agent).
|
|
33
|
+
* Stored at launch so record-driven resumes keep finding the conversation
|
|
34
|
+
* even if the slug derivation ever changes — paths in records don't drift.
|
|
35
|
+
*/
|
|
36
|
+
mirrorSlug?: string;
|
|
37
|
+
state: 'live' | 'hibernated';
|
|
38
|
+
createdAt: number;
|
|
39
|
+
lastLaunchAt: number;
|
|
40
|
+
}
|
|
41
|
+
declare class AgentRegistry {
|
|
42
|
+
private records;
|
|
43
|
+
private exitUnsubs;
|
|
44
|
+
private loaded;
|
|
45
|
+
private ensureLoaded;
|
|
46
|
+
private persist;
|
|
47
|
+
/**
|
|
48
|
+
* Register (or re-register on revive) an agent launched in a terminal.
|
|
49
|
+
* MERGE semantics: fields the caller didn't supply NEVER overwrite stored
|
|
50
|
+
* truth — a resume relaunch that doesn't know the sessionId must not erase
|
|
51
|
+
* the registry's copy (that id is the conversation; losing it downgrades
|
|
52
|
+
* every later Continue to a picker).
|
|
53
|
+
*/
|
|
54
|
+
register(rec: Omit<AgentRecord, 'state' | 'createdAt' | 'lastLaunchAt'>): AgentRecord;
|
|
55
|
+
private static TUI_INIT_RE;
|
|
56
|
+
private static TUI_TEARDOWN_RE;
|
|
57
|
+
private dataUnsubs;
|
|
58
|
+
private graceTimers;
|
|
59
|
+
private watchLiveness;
|
|
60
|
+
/** Feed pty output through the liveness machine (public for tests). */
|
|
61
|
+
observeOutput(terminalId: string, chunk: string): void;
|
|
62
|
+
/** List all agents, reconciling 'live' against actual pty existence. */
|
|
63
|
+
list(): AgentRecord[];
|
|
64
|
+
/**
|
|
65
|
+
* list() plus process-table facts for every record whose pty exists:
|
|
66
|
+
* - `busy`: the pty's shell has a live child — SOMETHING is running there.
|
|
67
|
+
* Crucially reported on HIBERNATED records too: the TUI-stream liveness
|
|
68
|
+
* machine can mis-score (an agent resumed by hand never re-registers), and
|
|
69
|
+
* a client that trusts 'hibernated' then types a launch command straight
|
|
70
|
+
* into the running TUI's input box (lab report: a live codex politely
|
|
71
|
+
* declined to run the pasted ssh line). The process table is authoritative.
|
|
72
|
+
* - `idleShell` on live records: the inverse — 'live' but nothing running
|
|
73
|
+
* (a hung-then-dead ssh hop leaves exactly this; a dead transport emits
|
|
74
|
+
* no teardown, so only the process table can see it).
|
|
75
|
+
* Unknown checks claim nothing.
|
|
76
|
+
*/
|
|
77
|
+
listEnriched(): Promise<(AgentRecord & {
|
|
78
|
+
idleShell?: boolean;
|
|
79
|
+
busy?: boolean;
|
|
80
|
+
})[]>;
|
|
81
|
+
/** Hibernate: close the pty; the record (and on-disk trajectory) remain. */
|
|
82
|
+
hibernate(terminalId: string): boolean;
|
|
83
|
+
/** Forget the agent entirely (pty closed; registry record removed). */
|
|
84
|
+
remove(terminalId: string): boolean;
|
|
85
|
+
}
|
|
86
|
+
export declare const agentRegistry: AgentRegistry;
|
|
87
|
+
export {};
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Agent registry — the server-side ledger of agent sessions (Claude Code /
|
|
4
|
+
* Codex CLIs running inside terminal ptys).
|
|
5
|
+
*
|
|
6
|
+
* Agents are project-scoped (keyed by the working directory they were
|
|
7
|
+
* launched in) and decoupled from notebooks: switching notebooks never kills
|
|
8
|
+
* or switches an agent; the user does, through the agent manager. Records
|
|
9
|
+
* persist to disk so agents survive as *resumable* entities even when their
|
|
10
|
+
* pty dies (browser gone, server restart, user hibernates them): the CLI's
|
|
11
|
+
* own on-disk trajectory (`claude --resume <id>` / `codex resume`) can
|
|
12
|
+
* reconstruct the conversation — the registry just remembers what exists,
|
|
13
|
+
* where, and how to revive it. The client owns launch/revive command
|
|
14
|
+
* construction (it knows about remote-agent mode); the server owns truth
|
|
15
|
+
* about what is live.
|
|
16
|
+
*
|
|
17
|
+
* States: 'live' (pty exists) → 'hibernated' (pty gone, trajectory on disk).
|
|
18
|
+
* All records load as 'hibernated' on boot — ptys never survive a restart.
|
|
19
|
+
*/
|
|
20
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
21
|
+
if (k2 === undefined) k2 = k;
|
|
22
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
23
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
24
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
25
|
+
}
|
|
26
|
+
Object.defineProperty(o, k2, desc);
|
|
27
|
+
}) : (function(o, m, k, k2) {
|
|
28
|
+
if (k2 === undefined) k2 = k;
|
|
29
|
+
o[k2] = m[k];
|
|
30
|
+
}));
|
|
31
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
32
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
33
|
+
}) : function(o, v) {
|
|
34
|
+
o["default"] = v;
|
|
35
|
+
});
|
|
36
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
37
|
+
var ownKeys = function(o) {
|
|
38
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
39
|
+
var ar = [];
|
|
40
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
41
|
+
return ar;
|
|
42
|
+
};
|
|
43
|
+
return ownKeys(o);
|
|
44
|
+
};
|
|
45
|
+
return function (mod) {
|
|
46
|
+
if (mod && mod.__esModule) return mod;
|
|
47
|
+
var result = {};
|
|
48
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
49
|
+
__setModuleDefault(result, mod);
|
|
50
|
+
return result;
|
|
51
|
+
};
|
|
52
|
+
})();
|
|
53
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
54
|
+
exports.agentRegistry = void 0;
|
|
55
|
+
const fs = __importStar(require("fs"));
|
|
56
|
+
const path = __importStar(require("path"));
|
|
57
|
+
const os = __importStar(require("os"));
|
|
58
|
+
const pty_manager_1 = require("./pty-manager");
|
|
59
|
+
const STATE_FILE = process.env.NEBULA_AGENTS_FILE || path.join(os.homedir(), '.nebula', 'agents.json');
|
|
60
|
+
const MAX_RECORDS = 50; // oldest hibernated records beyond this are dropped
|
|
61
|
+
class AgentRegistry {
|
|
62
|
+
records = new Map();
|
|
63
|
+
exitUnsubs = new Map();
|
|
64
|
+
loaded = false;
|
|
65
|
+
ensureLoaded() {
|
|
66
|
+
if (this.loaded)
|
|
67
|
+
return;
|
|
68
|
+
this.loaded = true;
|
|
69
|
+
try {
|
|
70
|
+
if (fs.existsSync(STATE_FILE)) {
|
|
71
|
+
const list = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
|
72
|
+
for (const r of list) {
|
|
73
|
+
if (!r?.terminalId)
|
|
74
|
+
continue;
|
|
75
|
+
// Ptys never survive a server restart — everything reloads hibernated.
|
|
76
|
+
r.state = 'hibernated';
|
|
77
|
+
this.records.set(r.terminalId, r);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
console.error('[AgentRegistry] failed to load state:', err);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
persist() {
|
|
86
|
+
try {
|
|
87
|
+
const dir = path.dirname(STATE_FILE);
|
|
88
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
89
|
+
const all = [...this.records.values()].sort((a, b) => b.lastLaunchAt - a.lastLaunchAt);
|
|
90
|
+
const live = all.filter((r) => r.state === 'live');
|
|
91
|
+
const hibernated = all.filter((r) => r.state !== 'live').slice(0, MAX_RECORDS - live.length);
|
|
92
|
+
const tmp = `${STATE_FILE}.tmp`;
|
|
93
|
+
fs.writeFileSync(tmp, JSON.stringify([...live, ...hibernated], null, 2));
|
|
94
|
+
fs.renameSync(tmp, STATE_FILE);
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
console.error('[AgentRegistry] failed to persist state:', err);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Register (or re-register on revive) an agent launched in a terminal.
|
|
102
|
+
* MERGE semantics: fields the caller didn't supply NEVER overwrite stored
|
|
103
|
+
* truth — a resume relaunch that doesn't know the sessionId must not erase
|
|
104
|
+
* the registry's copy (that id is the conversation; losing it downgrades
|
|
105
|
+
* every later Continue to a picker).
|
|
106
|
+
*/
|
|
107
|
+
register(rec) {
|
|
108
|
+
this.ensureLoaded();
|
|
109
|
+
const existing = this.records.get(rec.terminalId);
|
|
110
|
+
const record = {
|
|
111
|
+
...existing,
|
|
112
|
+
...rec,
|
|
113
|
+
sessionId: rec.sessionId ?? existing?.sessionId,
|
|
114
|
+
mirrorSlug: rec.mirrorSlug ?? existing?.mirrorSlug,
|
|
115
|
+
launchedFrom: rec.launchedFrom ?? existing?.launchedFrom,
|
|
116
|
+
state: 'live',
|
|
117
|
+
createdAt: existing?.createdAt ?? Date.now(),
|
|
118
|
+
lastLaunchAt: Date.now(),
|
|
119
|
+
};
|
|
120
|
+
this.records.set(rec.terminalId, record);
|
|
121
|
+
// When the pty dies for any reason (user exits the CLI and closes the
|
|
122
|
+
// terminal, hibernate, server-side kill), the agent is not gone — its
|
|
123
|
+
// trajectory is on disk. Mark it hibernated, ready to revive.
|
|
124
|
+
this.exitUnsubs.get(rec.terminalId)?.();
|
|
125
|
+
const unsub = pty_manager_1.ptyManager.addExitListener(rec.terminalId, () => {
|
|
126
|
+
const r = this.records.get(rec.terminalId);
|
|
127
|
+
if (r && r.state === 'live') {
|
|
128
|
+
r.state = 'hibernated';
|
|
129
|
+
this.persist();
|
|
130
|
+
}
|
|
131
|
+
this.exitUnsubs.delete(rec.terminalId);
|
|
132
|
+
});
|
|
133
|
+
this.exitUnsubs.set(rec.terminalId, unsub);
|
|
134
|
+
// SERVER-SIDE liveness: the pty stream passes through this process, so the
|
|
135
|
+
// agent-alive state machine lives HERE, not in whichever browser happens to
|
|
136
|
+
// be attached. TUI teardown with no re-init within the grace window means
|
|
137
|
+
// the agent exited to the shell — the record flips to 'hibernated' even
|
|
138
|
+
// though the pty lives on (revive types the resume command into it).
|
|
139
|
+
this.watchLiveness(rec.terminalId);
|
|
140
|
+
this.persist();
|
|
141
|
+
return record;
|
|
142
|
+
}
|
|
143
|
+
static TUI_INIT_RE = /\x1b\[\?(1049|1004|1000|1002|1003)h/;
|
|
144
|
+
static TUI_TEARDOWN_RE = /\x1b\[\?(1049|1004|1000|1002|1003)l/;
|
|
145
|
+
dataUnsubs = new Map();
|
|
146
|
+
graceTimers = new Map();
|
|
147
|
+
watchLiveness(terminalId) {
|
|
148
|
+
this.dataUnsubs.get(terminalId)?.();
|
|
149
|
+
const unsub = pty_manager_1.ptyManager.addDataListener(terminalId, (chunk) => {
|
|
150
|
+
this.observeOutput(terminalId, chunk);
|
|
151
|
+
});
|
|
152
|
+
if (unsub)
|
|
153
|
+
this.dataUnsubs.set(terminalId, unsub);
|
|
154
|
+
}
|
|
155
|
+
/** Feed pty output through the liveness machine (public for tests). */
|
|
156
|
+
observeOutput(terminalId, chunk) {
|
|
157
|
+
const r = this.records.get(terminalId);
|
|
158
|
+
if (!r)
|
|
159
|
+
return;
|
|
160
|
+
if (AgentRegistry.TUI_INIT_RE.test(chunk)) {
|
|
161
|
+
const t = this.graceTimers.get(terminalId);
|
|
162
|
+
if (t) {
|
|
163
|
+
clearTimeout(t);
|
|
164
|
+
this.graceTimers.delete(terminalId);
|
|
165
|
+
}
|
|
166
|
+
if (r.state !== 'live') {
|
|
167
|
+
r.state = 'live';
|
|
168
|
+
r.lastLaunchAt = Date.now();
|
|
169
|
+
this.persist();
|
|
170
|
+
}
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
if (AgentRegistry.TUI_TEARDOWN_RE.test(chunk) && r.state === 'live' && !this.graceTimers.has(terminalId)) {
|
|
174
|
+
this.graceTimers.set(terminalId, setTimeout(() => {
|
|
175
|
+
this.graceTimers.delete(terminalId);
|
|
176
|
+
const rec = this.records.get(terminalId);
|
|
177
|
+
if (rec && rec.state === 'live') {
|
|
178
|
+
rec.state = 'hibernated'; // agent exited; pty may still be a plain shell
|
|
179
|
+
this.persist();
|
|
180
|
+
}
|
|
181
|
+
}, 2000));
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/** List all agents, reconciling 'live' against actual pty existence. */
|
|
185
|
+
list() {
|
|
186
|
+
this.ensureLoaded();
|
|
187
|
+
for (const r of this.records.values()) {
|
|
188
|
+
if (r.state === 'live' && !pty_manager_1.ptyManager.get(r.terminalId)) {
|
|
189
|
+
r.state = 'hibernated';
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return [...this.records.values()].sort((a, b) => b.lastLaunchAt - a.lastLaunchAt);
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* list() plus process-table facts for every record whose pty exists:
|
|
196
|
+
* - `busy`: the pty's shell has a live child — SOMETHING is running there.
|
|
197
|
+
* Crucially reported on HIBERNATED records too: the TUI-stream liveness
|
|
198
|
+
* machine can mis-score (an agent resumed by hand never re-registers), and
|
|
199
|
+
* a client that trusts 'hibernated' then types a launch command straight
|
|
200
|
+
* into the running TUI's input box (lab report: a live codex politely
|
|
201
|
+
* declined to run the pasted ssh line). The process table is authoritative.
|
|
202
|
+
* - `idleShell` on live records: the inverse — 'live' but nothing running
|
|
203
|
+
* (a hung-then-dead ssh hop leaves exactly this; a dead transport emits
|
|
204
|
+
* no teardown, so only the process table can see it).
|
|
205
|
+
* Unknown checks claim nothing.
|
|
206
|
+
*/
|
|
207
|
+
async listEnriched() {
|
|
208
|
+
return Promise.all(this.list().map(async (r) => {
|
|
209
|
+
if (!pty_manager_1.ptyManager.get(r.terminalId))
|
|
210
|
+
return r;
|
|
211
|
+
const hasChild = await pty_manager_1.ptyManager.hasLiveChild(r.terminalId);
|
|
212
|
+
if (hasChild === null)
|
|
213
|
+
return r;
|
|
214
|
+
return r.state === 'live'
|
|
215
|
+
? { ...r, idleShell: !hasChild, busy: hasChild }
|
|
216
|
+
: { ...r, busy: hasChild };
|
|
217
|
+
}));
|
|
218
|
+
}
|
|
219
|
+
/** Hibernate: close the pty; the record (and on-disk trajectory) remain. */
|
|
220
|
+
hibernate(terminalId) {
|
|
221
|
+
this.ensureLoaded();
|
|
222
|
+
const r = this.records.get(terminalId);
|
|
223
|
+
if (!r)
|
|
224
|
+
return false;
|
|
225
|
+
if (pty_manager_1.ptyManager.get(terminalId))
|
|
226
|
+
pty_manager_1.ptyManager.kill(terminalId); // exit listener marks hibernated
|
|
227
|
+
r.state = 'hibernated';
|
|
228
|
+
this.persist();
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
/** Forget the agent entirely (pty closed; registry record removed). */
|
|
232
|
+
remove(terminalId) {
|
|
233
|
+
this.ensureLoaded();
|
|
234
|
+
const r = this.records.get(terminalId);
|
|
235
|
+
if (!r)
|
|
236
|
+
return false;
|
|
237
|
+
this.exitUnsubs.get(terminalId)?.();
|
|
238
|
+
this.exitUnsubs.delete(terminalId);
|
|
239
|
+
if (pty_manager_1.ptyManager.get(terminalId))
|
|
240
|
+
pty_manager_1.ptyManager.kill(terminalId);
|
|
241
|
+
this.records.delete(terminalId);
|
|
242
|
+
this.persist();
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
exports.agentRegistry = new AgentRegistry();
|