nebula-notebook 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -16
- package/dist/assets/errorwidget-Wy5ajPiM.js +5 -0
- package/dist/assets/fa-brands-400-CEJbCg16.woff +0 -0
- package/dist/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
- package/dist/assets/fa-brands-400-DnkPfk3o.eot +0 -0
- package/dist/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
- package/dist/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
- package/dist/assets/fa-regular-400-BhTwtT8w.eot +0 -0
- package/dist/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
- package/dist/assets/fa-regular-400-DFnMcJPd.woff +0 -0
- package/dist/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
- package/dist/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
- package/dist/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
- package/dist/assets/fa-solid-900-B6Axprfb.eot +0 -0
- package/dist/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
- package/dist/assets/fa-solid-900-DOXgCApm.woff +0 -0
- package/dist/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
- package/dist/assets/index-B_Bs5koo.js +1 -0
- package/dist/assets/index-BtWv4MIT.css +7 -0
- package/dist/assets/index-DK4nS74r.js +829 -0
- package/dist/assets/index-DeQN03nW.js +81 -0
- package/dist/assets/index-DzjpmUak.css +32 -0
- package/dist/assets/index-g6MjT-VG.js +1 -0
- package/dist/assets/services-shim-BirHVsih.js +33 -0
- package/dist/assets/viewlist-uomDf7I7.js +1 -0
- package/dist/assets/widgets-X7J3NxEn.css +1 -0
- package/dist/index.html +2 -2
- package/node-server/dist/auth/auth-middleware.js +22 -4
- package/node-server/dist/auth/auth-service.js +37 -7
- package/node-server/dist/cluster/client-registration.d.ts +12 -1
- package/node-server/dist/cluster/client-registration.js +38 -2
- package/node-server/dist/cluster/kernel-proxy.js +43 -12
- package/node-server/dist/cluster/server-registry.d.ts +17 -0
- package/node-server/dist/cluster/server-registry.js +57 -7
- package/node-server/dist/discovery/conda-locations.d.ts +73 -0
- package/node-server/dist/discovery/conda-locations.js +427 -0
- package/node-server/dist/discovery/discovery-service.d.ts +73 -4
- package/node-server/dist/discovery/discovery-service.js +211 -127
- package/node-server/dist/discovery/types.d.ts +0 -1
- package/node-server/dist/discovery/types.js +1 -2
- package/node-server/dist/environment.d.ts +12 -0
- package/node-server/dist/environment.js +104 -0
- package/node-server/dist/fs/fs-service.d.ts +86 -8
- package/node-server/dist/fs/fs-service.js +640 -99
- package/node-server/dist/fs/notebook-formats/percent.d.ts +25 -0
- package/node-server/dist/fs/notebook-formats/percent.js +286 -0
- package/node-server/dist/fs/notebook-formats/qmd.d.ts +29 -0
- package/node-server/dist/fs/notebook-formats/qmd.js +307 -0
- package/node-server/dist/fs/notebook-formats/registry.d.ts +12 -0
- package/node-server/dist/fs/notebook-formats/registry.js +77 -0
- package/node-server/dist/fs/notebook-formats/types.d.ts +37 -0
- package/node-server/dist/fs/notebook-formats/types.js +13 -0
- package/node-server/dist/fs/types.d.ts +9 -0
- package/node-server/dist/fs/types.js +8 -0
- package/node-server/dist/idle-exit.d.ts +52 -0
- package/node-server/dist/idle-exit.js +83 -0
- package/node-server/dist/index.js +198 -8
- package/node-server/dist/kernel/kernel-service.d.ts +130 -2
- package/node-server/dist/kernel/kernel-service.js +834 -64
- package/node-server/dist/kernel/kernelspec.d.ts +27 -0
- package/node-server/dist/kernel/kernelspec.js +75 -4
- package/node-server/dist/notebook/headless-handler.d.ts +12 -0
- package/node-server/dist/notebook/headless-handler.js +125 -29
- package/node-server/dist/notebook/operation-router.js +7 -2
- package/node-server/dist/notebook/undoRedoManager.d.ts +12 -1
- package/node-server/dist/notebook/undoRedoManager.js +49 -18
- package/node-server/dist/output/display-data.js +2 -0
- package/node-server/dist/private-tmp.d.ts +14 -0
- package/node-server/dist/private-tmp.js +70 -0
- package/node-server/dist/routes/autocomplete.d.ts +19 -0
- package/node-server/dist/routes/autocomplete.js +294 -0
- package/node-server/dist/routes/cluster.js +2 -2
- package/node-server/dist/routes/compute.d.ts +8 -0
- package/node-server/dist/routes/compute.js +136 -0
- package/node-server/dist/routes/fs.js +23 -6
- package/node-server/dist/routes/kernel.js +156 -3
- package/node-server/dist/routes/notebook.js +37 -3
- package/node-server/dist/routes/python.d.ts +1 -2
- package/node-server/dist/routes/python.js +139 -3
- package/node-server/dist/scheduler/allocation-service.d.ts +63 -0
- package/node-server/dist/scheduler/allocation-service.js +285 -0
- package/node-server/dist/scheduler/job-template.d.ts +30 -0
- package/node-server/dist/scheduler/job-template.js +85 -0
- package/node-server/dist/scheduler/mock-scheduler.d.ts +30 -0
- package/node-server/dist/scheduler/mock-scheduler.js +125 -0
- package/node-server/dist/scheduler/slurm-scheduler.d.ts +31 -0
- package/node-server/dist/scheduler/slurm-scheduler.js +406 -0
- package/node-server/dist/scheduler/types.d.ts +118 -0
- package/node-server/dist/scheduler/types.js +8 -0
- package/node-server/dist/scheduler/util.d.ts +7 -0
- package/node-server/dist/scheduler/util.js +20 -0
- package/node-server/dist/terminal/agent-registry.d.ts +87 -0
- package/node-server/dist/terminal/agent-registry.js +246 -0
- package/node-server/dist/terminal/binding-store.d.ts +57 -0
- package/node-server/dist/terminal/binding-store.js +0 -0
- package/node-server/dist/terminal/pty-manager.d.ts +23 -6
- package/node-server/dist/terminal/pty-manager.js +138 -19
- package/node-server/dist/terminal/server.js +155 -23
- package/node-server/dist/terminal/types.d.ts +2 -0
- package/node-server/dist/update-check.d.ts +20 -0
- package/node-server/dist/update-check.js +114 -0
- package/node-server/package.json +2 -0
- package/node_modules/nebula-autocomplete/README.md +145 -0
- package/node_modules/nebula-autocomplete/dist/backends/claude.d.ts +69 -0
- package/node_modules/nebula-autocomplete/dist/backends/claude.js +449 -0
- package/node_modules/nebula-autocomplete/dist/backends/codex.d.ts +45 -0
- package/node_modules/nebula-autocomplete/dist/backends/codex.js +108 -0
- package/node_modules/nebula-autocomplete/dist/client/fetcher.d.ts +16 -0
- package/node_modules/nebula-autocomplete/dist/client/fetcher.js +41 -0
- package/node_modules/nebula-autocomplete/dist/codemirror/ghostText.d.ts +43 -0
- package/node_modules/nebula-autocomplete/dist/codemirror/ghostText.js +175 -0
- package/node_modules/nebula-autocomplete/dist/core/engine.d.ts +17 -0
- package/node_modules/nebula-autocomplete/dist/core/engine.js +135 -0
- package/node_modules/nebula-autocomplete/dist/core/lru.d.ts +7 -0
- package/node_modules/nebula-autocomplete/dist/core/lru.js +26 -0
- package/node_modules/nebula-autocomplete/dist/core/prompt.d.ts +12 -0
- package/node_modules/nebula-autocomplete/dist/core/prompt.js +0 -0
- package/node_modules/nebula-autocomplete/dist/core/text.d.ts +53 -0
- package/node_modules/nebula-autocomplete/dist/core/text.js +169 -0
- package/node_modules/nebula-autocomplete/dist/index.d.ts +7 -0
- package/node_modules/nebula-autocomplete/dist/index.js +5 -0
- package/node_modules/nebula-autocomplete/dist/server/fastify.d.ts +40 -0
- package/node_modules/nebula-autocomplete/dist/server/fastify.js +52 -0
- package/node_modules/nebula-autocomplete/dist/transport.d.ts +55 -0
- package/node_modules/nebula-autocomplete/dist/transport.js +66 -0
- package/node_modules/nebula-autocomplete/dist/types.d.ts +103 -0
- package/node_modules/nebula-autocomplete/dist/types.js +1 -0
- package/node_modules/nebula-autocomplete/package.json +58 -0
- package/node_modules/nebula-autocomplete/server.d.ts +3 -0
- package/node_modules/nebula-autocomplete/server.js +3 -0
- package/package.json +18 -5
- package/scripts/postinstall.cjs +25 -5
- package/dist/assets/index-BvrHu37J.js +0 -648
- package/dist/assets/index-Dfj_xsKU.css +0 -32
- package/node-server/dist/llm/index.d.ts +0 -5
- package/node-server/dist/llm/index.js +0 -21
- package/node-server/dist/llm/llm-service.d.ts +0 -77
- package/node-server/dist/llm/llm-service.js +0 -454
- package/node-server/dist/llm/types.d.ts +0 -40
- package/node-server/dist/llm/types.js +0 -15
- package/node-server/dist/routes/llm.d.ts +0 -8
- package/node-server/dist/routes/llm.js +0 -105
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* SLURM implementation of the Scheduler interface.
|
|
4
|
+
*
|
|
5
|
+
* Everything shells out to the standard SLURM client CLIs (sbatch, squeue,
|
|
6
|
+
* scancel, sinfo, sacctmgr, sacct). The Nebula main server runs on the login
|
|
7
|
+
* node where these are available; detection is capability-based.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.SlurmScheduler = void 0;
|
|
11
|
+
const child_process_1 = require("child_process");
|
|
12
|
+
const util_1 = require("util");
|
|
13
|
+
const util_2 = require("./util");
|
|
14
|
+
const execFileP = (0, util_1.promisify)(child_process_1.execFile);
|
|
15
|
+
async function run(cmd, args, timeoutMs = 15_000) {
|
|
16
|
+
const { stdout, stderr } = await execFileP(cmd, args, {
|
|
17
|
+
timeout: timeoutMs,
|
|
18
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
19
|
+
});
|
|
20
|
+
return { stdout: stdout.toString(), stderr: stderr.toString() };
|
|
21
|
+
}
|
|
22
|
+
function toInt(v) {
|
|
23
|
+
const n = parseInt((v ?? '').trim(), 10);
|
|
24
|
+
return Number.isFinite(n) ? n : 0;
|
|
25
|
+
}
|
|
26
|
+
/** Pull `Key=value` (value up to next space) out of a `scontrol -o` line. */
|
|
27
|
+
function scontrolField(line, key) {
|
|
28
|
+
const m = line.match(new RegExp(`(?:^|\\s)${key}=(\\S+)`));
|
|
29
|
+
return m ? m[1] : undefined;
|
|
30
|
+
}
|
|
31
|
+
/** Map a SLURM state code or name to our normalized JobState. */
|
|
32
|
+
function mapState(raw) {
|
|
33
|
+
const s = (raw || '').trim().toUpperCase();
|
|
34
|
+
if (s === 'PD' || s === 'PENDING' || s === 'CF' || s === 'CONFIGURING')
|
|
35
|
+
return 'pending';
|
|
36
|
+
if (s === 'R' || s === 'RUNNING')
|
|
37
|
+
return 'running';
|
|
38
|
+
if (s === 'CG' || s === 'COMPLETING')
|
|
39
|
+
return 'completing';
|
|
40
|
+
if (s === 'CD' || s === 'COMPLETED')
|
|
41
|
+
return 'completed';
|
|
42
|
+
if (s === 'CA' || s === 'CANCELLED' || s.startsWith('CANCELLED'))
|
|
43
|
+
return 'cancelled';
|
|
44
|
+
if (['F', 'FAILED', 'TO', 'TIMEOUT', 'NF', 'NODE_FAIL', 'OOM', 'OUT_OF_MEMORY', 'BF', 'BOOT_FAIL', 'DL', 'DEADLINE'].includes(s)) {
|
|
45
|
+
return 'failed';
|
|
46
|
+
}
|
|
47
|
+
return 'unknown';
|
|
48
|
+
}
|
|
49
|
+
async function expandNodes(nodelist) {
|
|
50
|
+
const nl = (nodelist || '').trim();
|
|
51
|
+
if (!nl || nl === '(null)' || nl === 'None' || nl === 'n/a')
|
|
52
|
+
return [];
|
|
53
|
+
// Plain single hostname (no ranges/lists) — the common case for 1-node
|
|
54
|
+
// allocations. Don't shell out to scontrol just to echo it back.
|
|
55
|
+
if (!/[\[\],]/.test(nl))
|
|
56
|
+
return [nl];
|
|
57
|
+
try {
|
|
58
|
+
const { stdout } = await run('scontrol', ['show', 'hostnames', nl], 5_000);
|
|
59
|
+
return stdout.split('\n').map((s) => s.trim()).filter(Boolean);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return [nl];
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
class SlurmScheduler {
|
|
66
|
+
name = 'slurm';
|
|
67
|
+
/** Cache of per-partition required-QoS (site job-submit filters change rarely). */
|
|
68
|
+
qosCache = new Map();
|
|
69
|
+
// Short-TTL caches + in-flight de-duplication so the dashboard's 15s poll, the
|
|
70
|
+
// allocation modal, and manual refresh don't each re-run the (potentially slow)
|
|
71
|
+
// sinfo/scontrol/squeue/sacctmgr queries against the scheduler.
|
|
72
|
+
// 20s: deliberately ABOVE the clients' 15s poll so steady-state polling
|
|
73
|
+
// alternates cache hit/miss instead of missing every time (10s never hit).
|
|
74
|
+
static LOAD_TTL_MS = 20_000;
|
|
75
|
+
static ASSOC_TTL_MS = 60_000;
|
|
76
|
+
loadCache = null;
|
|
77
|
+
loadInflight = null;
|
|
78
|
+
assocCache = new Map();
|
|
79
|
+
assocInflight = new Map();
|
|
80
|
+
async detect() {
|
|
81
|
+
try {
|
|
82
|
+
await run('sbatch', ['--version'], 5_000);
|
|
83
|
+
await run('squeue', ['--version'], 5_000);
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
async associations(user) {
|
|
91
|
+
const cached = this.assocCache.get(user);
|
|
92
|
+
if (cached && Date.now() - cached.at < SlurmScheduler.ASSOC_TTL_MS)
|
|
93
|
+
return cached.data;
|
|
94
|
+
const inflight = this.assocInflight.get(user);
|
|
95
|
+
if (inflight)
|
|
96
|
+
return inflight;
|
|
97
|
+
const p = this.associationsFresh(user)
|
|
98
|
+
.then((data) => { this.assocCache.set(user, { data, at: Date.now() }); return data; })
|
|
99
|
+
.finally(() => { this.assocInflight.delete(user); });
|
|
100
|
+
this.assocInflight.set(user, p);
|
|
101
|
+
return p;
|
|
102
|
+
}
|
|
103
|
+
async associationsFresh(user) {
|
|
104
|
+
let account;
|
|
105
|
+
const partitions = new Set();
|
|
106
|
+
const qoses = new Set();
|
|
107
|
+
let defaultQos;
|
|
108
|
+
try {
|
|
109
|
+
const { stdout } = await run('sacctmgr', [
|
|
110
|
+
'-nP', 'show', 'assoc', `user=${user}`,
|
|
111
|
+
'format=Account,Partition,QOS,DefaultQOS',
|
|
112
|
+
]);
|
|
113
|
+
for (const line of stdout.split('\n')) {
|
|
114
|
+
if (!line.trim())
|
|
115
|
+
continue;
|
|
116
|
+
const [acct, part, qos, defQos] = line.split('|');
|
|
117
|
+
if (acct && !account)
|
|
118
|
+
account = acct.trim();
|
|
119
|
+
if (part && part.trim())
|
|
120
|
+
partitions.add(part.trim());
|
|
121
|
+
if (qos)
|
|
122
|
+
qos.split(',').forEach((q) => q.trim() && qoses.add(q.trim()));
|
|
123
|
+
if (defQos && defQos.trim() && !defaultQos)
|
|
124
|
+
defaultQos = defQos.trim();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
/* sacctmgr may be unavailable / accounting disabled */
|
|
129
|
+
}
|
|
130
|
+
// Account-level associations often leave Partition blank (== not restricted).
|
|
131
|
+
// Fall back to the full partition list so the user can still pick one;
|
|
132
|
+
// an actually-disallowed choice is caught at submit / estimate time.
|
|
133
|
+
if (partitions.size === 0) {
|
|
134
|
+
try {
|
|
135
|
+
const { stdout } = await run('sinfo', ['-h', '-o', '%R']);
|
|
136
|
+
stdout.split('\n').forEach((p) => {
|
|
137
|
+
const t = p.trim();
|
|
138
|
+
if (t)
|
|
139
|
+
partitions.add(t);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
/* ignore */
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
account,
|
|
148
|
+
partitions: [...partitions],
|
|
149
|
+
qoses: [...qoses],
|
|
150
|
+
defaultQos,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
async load() {
|
|
154
|
+
if (this.loadCache && Date.now() - this.loadCache.at < SlurmScheduler.LOAD_TTL_MS) {
|
|
155
|
+
return this.loadCache.data;
|
|
156
|
+
}
|
|
157
|
+
if (this.loadInflight)
|
|
158
|
+
return this.loadInflight;
|
|
159
|
+
const p = this.loadFresh()
|
|
160
|
+
.then((data) => { this.loadCache = { data, at: Date.now() }; return data; })
|
|
161
|
+
.finally(() => { this.loadInflight = null; });
|
|
162
|
+
this.loadInflight = p;
|
|
163
|
+
return p;
|
|
164
|
+
}
|
|
165
|
+
async loadFresh() {
|
|
166
|
+
const partitions = new Map();
|
|
167
|
+
// Fire all four scheduler queries concurrently — the wall time is the slowest
|
|
168
|
+
// one, not the sum (scontrol/squeue dominate on big/busy clusters). Each is
|
|
169
|
+
// independent and optional; a failed query just leaves its slice of data empty.
|
|
170
|
+
const [sinfoRes, nodeRes, squeueRes, qosRes] = await Promise.allSettled([
|
|
171
|
+
run('sinfo', ['-h', '-o', '%R|%a|%l|%C|%D|%T']),
|
|
172
|
+
run('scontrol', ['show', 'node', '-o'], 20_000),
|
|
173
|
+
run('squeue', ['-h', '-r', '-o', '%P|%q|%t']),
|
|
174
|
+
run('sacctmgr', ['-nP', 'show', 'qos', 'format=Name,Priority,MaxWall,Preempt']),
|
|
175
|
+
]);
|
|
176
|
+
// sinfo emits one line per (partition, node-state) group; aggregate per partition.
|
|
177
|
+
if (sinfoRes.status === 'fulfilled') {
|
|
178
|
+
for (const line of sinfoRes.value.stdout.split('\n')) {
|
|
179
|
+
if (!line.trim())
|
|
180
|
+
continue;
|
|
181
|
+
const [name, avail, timeLimit, cpus, nodeCount, stateName] = line.split('|');
|
|
182
|
+
const [ca, ci, co, ct] = (cpus || '').split('/').map(toInt);
|
|
183
|
+
let p = partitions.get(name);
|
|
184
|
+
if (!p) {
|
|
185
|
+
p = {
|
|
186
|
+
name,
|
|
187
|
+
up: (avail || '').trim().toLowerCase() === 'up',
|
|
188
|
+
timeLimit: (timeLimit || '').trim(),
|
|
189
|
+
cpus: { alloc: 0, idle: 0, other: 0, total: 0 },
|
|
190
|
+
nodes: { idle: 0, mixed: 0, alloc: 0, down: 0, total: 0 },
|
|
191
|
+
jobs: { pending: 0, running: 0 },
|
|
192
|
+
};
|
|
193
|
+
partitions.set(name, p);
|
|
194
|
+
}
|
|
195
|
+
p.cpus.alloc += ca;
|
|
196
|
+
p.cpus.idle += ci;
|
|
197
|
+
p.cpus.other += co;
|
|
198
|
+
p.cpus.total += ct;
|
|
199
|
+
const nc = toInt(nodeCount);
|
|
200
|
+
p.nodes.total += nc;
|
|
201
|
+
const st = (stateName || '').trim().toLowerCase();
|
|
202
|
+
if (st.startsWith('idle'))
|
|
203
|
+
p.nodes.idle += nc;
|
|
204
|
+
else if (st.startsWith('mix') || st.startsWith('alloc'))
|
|
205
|
+
p.nodes.mixed += nc;
|
|
206
|
+
else if (st.startsWith('down') || st.startsWith('drain') || st.startsWith('fail'))
|
|
207
|
+
p.nodes.down += nc;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
// GPU capacity per partition, from per-node TRES: configured (CfgTRES) vs
|
|
211
|
+
// allocated (AllocTRES) `gres/gpu`, so we can report *idle* (available) GPUs
|
|
212
|
+
// rather than a per-node count. Generic — no site-specific node/gres names.
|
|
213
|
+
if (nodeRes.status === 'fulfilled') {
|
|
214
|
+
// Aggregate per (partition, GPU model) — heterogeneous queues mix cards.
|
|
215
|
+
const agg = new Map();
|
|
216
|
+
for (const line of nodeRes.value.stdout.split('\n')) {
|
|
217
|
+
if (!line.trim())
|
|
218
|
+
continue;
|
|
219
|
+
const cfg = scontrolField(line, 'CfgTRES') || '';
|
|
220
|
+
const cfgGpu = toInt((cfg.match(/gres\/gpu=(\d+)/) || [])[1]);
|
|
221
|
+
if (cfgGpu === 0)
|
|
222
|
+
continue; // node has no GPUs
|
|
223
|
+
const parts = scontrolField(line, 'Partitions');
|
|
224
|
+
if (!parts)
|
|
225
|
+
continue;
|
|
226
|
+
const allocGpu = toInt((scontrolField(line, 'AllocTRES')?.match(/gres\/gpu=(\d+)/) || [])[1]);
|
|
227
|
+
const type = (scontrolField(line, 'Gres')?.match(/gpu:([^:(]+)/) || [])[1] || 'gpu';
|
|
228
|
+
for (const part of parts.split(',')) {
|
|
229
|
+
const byType = agg.get(part) ?? new Map();
|
|
230
|
+
const e = byType.get(type) || { total: 0, used: 0 };
|
|
231
|
+
e.total += cfgGpu;
|
|
232
|
+
e.used += allocGpu;
|
|
233
|
+
byType.set(type, e);
|
|
234
|
+
agg.set(part, byType);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
for (const [part, byType] of agg) {
|
|
238
|
+
const p = partitions.get(part);
|
|
239
|
+
if (!p)
|
|
240
|
+
continue;
|
|
241
|
+
const list = [...byType.entries()]
|
|
242
|
+
.filter(([, g]) => g.total > 0)
|
|
243
|
+
.map(([type, g]) => ({ type, total: g.total, idle: Math.max(0, g.total - g.used) }))
|
|
244
|
+
.sort((a, b) => b.total - a.total || a.type.localeCompare(b.type));
|
|
245
|
+
if (list.length)
|
|
246
|
+
p.gpus = list;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
// Queue depth from squeue, aggregated per partition AND per QoS.
|
|
250
|
+
const qosUsage = new Map();
|
|
251
|
+
if (squeueRes.status === 'fulfilled') {
|
|
252
|
+
for (const line of squeueRes.value.stdout.split('\n')) {
|
|
253
|
+
if (!line.trim())
|
|
254
|
+
continue;
|
|
255
|
+
const [part, qos, state] = line.split('|');
|
|
256
|
+
const s = (state || '').trim().toUpperCase();
|
|
257
|
+
const p = partitions.get((part || '').trim());
|
|
258
|
+
if (p) {
|
|
259
|
+
if (s === 'PD')
|
|
260
|
+
p.jobs.pending += 1;
|
|
261
|
+
else if (s === 'R')
|
|
262
|
+
p.jobs.running += 1;
|
|
263
|
+
}
|
|
264
|
+
const qn = (qos || '').trim();
|
|
265
|
+
if (qn) {
|
|
266
|
+
const u = qosUsage.get(qn) || { running: 0, pending: 0 };
|
|
267
|
+
if (s === 'PD')
|
|
268
|
+
u.pending += 1;
|
|
269
|
+
else if (s === 'R')
|
|
270
|
+
u.running += 1;
|
|
271
|
+
qosUsage.set(qn, u);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
// QoS definitions + preemption graph. In SLURM a QoS's `Preempt` column lists
|
|
276
|
+
// the QoS's it can preempt; a QoS is therefore *preemptible* (can be
|
|
277
|
+
// interrupted) exactly when some other QoS lists it as a preempt target.
|
|
278
|
+
// This is derived entirely from the scheduler — no site-specific QoS names.
|
|
279
|
+
const qoses = [];
|
|
280
|
+
if (qosRes.status === 'fulfilled') {
|
|
281
|
+
const rows = [];
|
|
282
|
+
for (const line of qosRes.value.stdout.split('\n')) {
|
|
283
|
+
if (!line.trim())
|
|
284
|
+
continue;
|
|
285
|
+
const [name, priority, maxWall, preempt] = line.split('|');
|
|
286
|
+
rows.push({
|
|
287
|
+
name: (name || '').trim(),
|
|
288
|
+
priority: toInt(priority),
|
|
289
|
+
maxWall: maxWall && maxWall.trim() ? maxWall.trim() : undefined,
|
|
290
|
+
preempts: (preempt || '').split(',').map((s) => s.trim()).filter(Boolean),
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
const preemptedBy = new Set();
|
|
294
|
+
for (const r of rows)
|
|
295
|
+
for (const target of r.preempts)
|
|
296
|
+
preemptedBy.add(target);
|
|
297
|
+
for (const r of rows) {
|
|
298
|
+
qoses.push({
|
|
299
|
+
name: r.name,
|
|
300
|
+
priority: r.priority,
|
|
301
|
+
maxWall: r.maxWall,
|
|
302
|
+
preemptible: preemptedBy.has(r.name),
|
|
303
|
+
preempts: r.preempts,
|
|
304
|
+
jobs: qosUsage.get(r.name) || { running: 0, pending: 0 },
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return { partitions: [...partitions.values()], qoses, fetchedAt: Date.now() };
|
|
309
|
+
}
|
|
310
|
+
async allowedQos(partition) {
|
|
311
|
+
const cached = this.qosCache.get(partition);
|
|
312
|
+
if (cached && Date.now() - cached.at < 300_000)
|
|
313
|
+
return cached.allowed;
|
|
314
|
+
let allowed = null;
|
|
315
|
+
try {
|
|
316
|
+
// A no-QoS dry run: succeeds when the partition accepts any QoS. A site
|
|
317
|
+
// job-submit filter that requires an explicit QoS rejects it and names the
|
|
318
|
+
// acceptable set in the message ("... Allowed: a, b").
|
|
319
|
+
await run('sbatch', [
|
|
320
|
+
'--test-only',
|
|
321
|
+
`--partition=${partition}`,
|
|
322
|
+
'--cpus-per-task=1', '--mem=1G', '--time=00:10:00', '--wrap=true',
|
|
323
|
+
], 15_000);
|
|
324
|
+
allowed = null;
|
|
325
|
+
}
|
|
326
|
+
catch (e) {
|
|
327
|
+
const text = String(e?.stderr || e?.message || '');
|
|
328
|
+
const m = text.match(/requires explicit --qos\.?\s*Allowed:\s*([^\n]+)/i);
|
|
329
|
+
allowed = m ? m[1].split(',').map((s) => s.trim()).filter(Boolean) : null;
|
|
330
|
+
}
|
|
331
|
+
this.qosCache.set(partition, { allowed, at: Date.now() });
|
|
332
|
+
return allowed;
|
|
333
|
+
}
|
|
334
|
+
async estimateStart(spec) {
|
|
335
|
+
const args = [
|
|
336
|
+
'--test-only',
|
|
337
|
+
`--partition=${spec.partition}`,
|
|
338
|
+
`--cpus-per-task=${spec.cpus}`,
|
|
339
|
+
`--mem=${spec.memGb}G`,
|
|
340
|
+
`--time=${(0, util_2.formatWalltime)(spec.walltimeMinutes)}`,
|
|
341
|
+
];
|
|
342
|
+
if (spec.qos)
|
|
343
|
+
args.push(`--qos=${spec.qos}`);
|
|
344
|
+
if (spec.account)
|
|
345
|
+
args.push(`--account=${spec.account}`);
|
|
346
|
+
if (spec.gpus)
|
|
347
|
+
args.push(`--gres=gpu:${spec.gpuType ? `${spec.gpuType}:` : ''}${spec.gpus}`);
|
|
348
|
+
args.push('--wrap=true');
|
|
349
|
+
// `sbatch --test-only` writes "Job N to start at <time> ... on nodes <n>" to stderr.
|
|
350
|
+
try {
|
|
351
|
+
const { stdout, stderr } = await run('sbatch', args, 15_000);
|
|
352
|
+
const text = `${stderr}\n${stdout}`;
|
|
353
|
+
const m = text.match(/to start at (\S+)(?:.*?on nodes? (\S+))?/i);
|
|
354
|
+
if (m)
|
|
355
|
+
return { startsAt: m[1], nodes: m[2] ? [m[2]] : undefined };
|
|
356
|
+
return { reason: text.trim() || 'no estimate available' };
|
|
357
|
+
}
|
|
358
|
+
catch (e) {
|
|
359
|
+
const reason = (e?.stderr || e?.message || 'test-only failed').toString().trim();
|
|
360
|
+
return { reason };
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
async submit(scriptPath) {
|
|
364
|
+
const { stdout } = await run('sbatch', ['--parsable', scriptPath]);
|
|
365
|
+
const jobId = stdout.trim().split(';')[0].trim();
|
|
366
|
+
if (!/^\d+$/.test(jobId)) {
|
|
367
|
+
throw new Error(`Unexpected sbatch output: ${stdout.trim()}`);
|
|
368
|
+
}
|
|
369
|
+
return { jobId };
|
|
370
|
+
}
|
|
371
|
+
async query(jobId) {
|
|
372
|
+
// Active jobs: squeue.
|
|
373
|
+
try {
|
|
374
|
+
const { stdout } = await run('squeue', ['-h', '-j', jobId, '-o', '%T|%N|%r']);
|
|
375
|
+
const line = stdout.split('\n').find((l) => l.trim());
|
|
376
|
+
if (line) {
|
|
377
|
+
const [state, nodelist, reason] = line.split('|');
|
|
378
|
+
return {
|
|
379
|
+
state: mapState(state),
|
|
380
|
+
nodes: await expandNodes(nodelist),
|
|
381
|
+
reason: reason && reason.trim() && reason.trim() !== 'None' ? reason.trim() : undefined,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
catch {
|
|
386
|
+
/* fall through to sacct */
|
|
387
|
+
}
|
|
388
|
+
// Finished jobs: sacct.
|
|
389
|
+
try {
|
|
390
|
+
const { stdout } = await run('sacct', ['-nXP', '-j', jobId, '-o', 'State,NodeList']);
|
|
391
|
+
const line = stdout.split('\n').find((l) => l.trim());
|
|
392
|
+
if (line) {
|
|
393
|
+
const [state, nodelist] = line.split('|');
|
|
394
|
+
return { state: mapState(state), nodes: await expandNodes(nodelist) };
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
catch {
|
|
398
|
+
/* ignore */
|
|
399
|
+
}
|
|
400
|
+
return { state: 'unknown', nodes: [] };
|
|
401
|
+
}
|
|
402
|
+
async cancel(jobId) {
|
|
403
|
+
await run('scancel', [jobId]);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
exports.SlurmScheduler = SlurmScheduler;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scheduler abstraction — a thin interface over an HPC batch scheduler.
|
|
3
|
+
*
|
|
4
|
+
* SLURM is the first implementation; PBS/LSF can implement the same interface
|
|
5
|
+
* later without touching the allocation service or routes.
|
|
6
|
+
*/
|
|
7
|
+
export type JobState = 'pending' | 'running' | 'completing' | 'completed' | 'failed' | 'cancelled' | 'unknown';
|
|
8
|
+
/** The resource request for one compute allocation. */
|
|
9
|
+
export interface JobSpec {
|
|
10
|
+
partition: string;
|
|
11
|
+
qos?: string;
|
|
12
|
+
account?: string;
|
|
13
|
+
cpus: number;
|
|
14
|
+
memGb: number;
|
|
15
|
+
gpus?: number;
|
|
16
|
+
/** Specific GPU model to request (e.g. from the scheduler's gres names). Optional. */
|
|
17
|
+
gpuType?: string;
|
|
18
|
+
walltimeMinutes: number;
|
|
19
|
+
jobName: string;
|
|
20
|
+
/**
|
|
21
|
+
* Opt-in idle auto-release: the client server on the compute node exits its
|
|
22
|
+
* own process after this many minutes without kernel/terminal activity, so
|
|
23
|
+
* the batch job completes and the allocation ends naturally. Omitted = never.
|
|
24
|
+
*/
|
|
25
|
+
idleTimeoutMinutes?: number;
|
|
26
|
+
}
|
|
27
|
+
export interface JobStatus {
|
|
28
|
+
state: JobState;
|
|
29
|
+
nodes: string[];
|
|
30
|
+
reason?: string;
|
|
31
|
+
}
|
|
32
|
+
/** Result of a dry-run start-time estimate (SLURM `sbatch --test-only`). */
|
|
33
|
+
export interface StartEstimate {
|
|
34
|
+
startsAt?: string;
|
|
35
|
+
nodes?: string[];
|
|
36
|
+
reason?: string;
|
|
37
|
+
}
|
|
38
|
+
/** What a given user is allowed to submit to. */
|
|
39
|
+
export interface Associations {
|
|
40
|
+
account?: string;
|
|
41
|
+
partitions: string[];
|
|
42
|
+
qoses: string[];
|
|
43
|
+
defaultQos?: string;
|
|
44
|
+
}
|
|
45
|
+
export interface PartitionLoad {
|
|
46
|
+
name: string;
|
|
47
|
+
up: boolean;
|
|
48
|
+
timeLimit: string;
|
|
49
|
+
cpus: {
|
|
50
|
+
alloc: number;
|
|
51
|
+
idle: number;
|
|
52
|
+
other: number;
|
|
53
|
+
total: number;
|
|
54
|
+
};
|
|
55
|
+
/** GPU capacity for the partition, one entry per GPU model (heterogeneous
|
|
56
|
+
* queues carry several): total configured vs currently idle (available). */
|
|
57
|
+
gpus?: {
|
|
58
|
+
type: string;
|
|
59
|
+
total: number;
|
|
60
|
+
idle: number;
|
|
61
|
+
}[];
|
|
62
|
+
nodes: {
|
|
63
|
+
idle: number;
|
|
64
|
+
mixed: number;
|
|
65
|
+
alloc: number;
|
|
66
|
+
down: number;
|
|
67
|
+
total: number;
|
|
68
|
+
};
|
|
69
|
+
jobs: {
|
|
70
|
+
pending: number;
|
|
71
|
+
running: number;
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
export interface QosLoad {
|
|
75
|
+
name: string;
|
|
76
|
+
priority: number;
|
|
77
|
+
maxWall?: string;
|
|
78
|
+
/** True if some other QoS is configured to preempt this one (jobs may be interrupted). */
|
|
79
|
+
preemptible: boolean;
|
|
80
|
+
/** QoS names this QoS can preempt (empty for most). */
|
|
81
|
+
preempts: string[];
|
|
82
|
+
/** Live usage under this QoS across the cluster. */
|
|
83
|
+
jobs: {
|
|
84
|
+
running: number;
|
|
85
|
+
pending: number;
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
/** A point-in-time snapshot of cluster busyness for the launcher's load monitor. */
|
|
89
|
+
export interface QueueLoad {
|
|
90
|
+
partitions: PartitionLoad[];
|
|
91
|
+
qoses: QosLoad[];
|
|
92
|
+
fetchedAt: number;
|
|
93
|
+
}
|
|
94
|
+
export interface Scheduler {
|
|
95
|
+
readonly name: string;
|
|
96
|
+
/** Is this scheduler available on this host (are its CLIs present)? */
|
|
97
|
+
detect(): Promise<boolean>;
|
|
98
|
+
/** Partitions/QoS the user may submit to, and their default QoS. */
|
|
99
|
+
associations(user: string): Promise<Associations>;
|
|
100
|
+
/** Per-partition / per-QoS busyness snapshot. */
|
|
101
|
+
load(): Promise<QueueLoad>;
|
|
102
|
+
/**
|
|
103
|
+
* QoS names a partition will actually accept, or null when it accepts any
|
|
104
|
+
* (no explicit-QoS requirement). Lets the UI avoid offering a QoS the
|
|
105
|
+
* scheduler would reject. Discovered from the scheduler, not configured.
|
|
106
|
+
*/
|
|
107
|
+
allowedQos(partition: string): Promise<string[] | null>;
|
|
108
|
+
/** Dry-run estimated start time for a spec, without submitting. */
|
|
109
|
+
estimateStart(spec: JobSpec): Promise<StartEstimate>;
|
|
110
|
+
/** Submit a rendered job script; returns the scheduler job id. */
|
|
111
|
+
submit(scriptPath: string): Promise<{
|
|
112
|
+
jobId: string;
|
|
113
|
+
}>;
|
|
114
|
+
/** Current state of a submitted job. */
|
|
115
|
+
query(jobId: string): Promise<JobStatus>;
|
|
116
|
+
/** Cancel a submitted job. */
|
|
117
|
+
cancel(jobId: string): Promise<void>;
|
|
118
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Scheduler abstraction — a thin interface over an HPC batch scheduler.
|
|
4
|
+
*
|
|
5
|
+
* SLURM is the first implementation; PBS/LSF can implement the same interface
|
|
6
|
+
* later without touching the allocation service or routes.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Small shared helpers for the scheduler module.
|
|
3
|
+
*/
|
|
4
|
+
/** Format minutes as a SLURM walltime string (`D-HH:MM:SS` or `HH:MM:SS`). */
|
|
5
|
+
export declare function formatWalltime(minutes: number): string;
|
|
6
|
+
/** POSIX single-quote a string so it is safe to embed in a shell script. */
|
|
7
|
+
export declare function shellQuote(s: string): string;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Small shared helpers for the scheduler module.
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.formatWalltime = formatWalltime;
|
|
7
|
+
exports.shellQuote = shellQuote;
|
|
8
|
+
/** Format minutes as a SLURM walltime string (`D-HH:MM:SS` or `HH:MM:SS`). */
|
|
9
|
+
function formatWalltime(minutes) {
|
|
10
|
+
const total = Math.max(1, Math.floor(minutes));
|
|
11
|
+
const d = Math.floor(total / 1440);
|
|
12
|
+
const h = Math.floor((total % 1440) / 60);
|
|
13
|
+
const m = total % 60;
|
|
14
|
+
const hhmmss = `${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:00`;
|
|
15
|
+
return d > 0 ? `${d}-${hhmmss}` : hhmmss;
|
|
16
|
+
}
|
|
17
|
+
/** POSIX single-quote a string so it is safe to embed in a shell script. */
|
|
18
|
+
function shellQuote(s) {
|
|
19
|
+
return `'${String(s).replace(/'/g, `'\\''`)}'`;
|
|
20
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent registry — the server-side ledger of agent sessions (Claude Code /
|
|
3
|
+
* Codex CLIs running inside terminal ptys).
|
|
4
|
+
*
|
|
5
|
+
* Agents are project-scoped (keyed by the working directory they were
|
|
6
|
+
* launched in) and decoupled from notebooks: switching notebooks never kills
|
|
7
|
+
* or switches an agent; the user does, through the agent manager. Records
|
|
8
|
+
* persist to disk so agents survive as *resumable* entities even when their
|
|
9
|
+
* pty dies (browser gone, server restart, user hibernates them): the CLI's
|
|
10
|
+
* own on-disk trajectory (`claude --resume <id>` / `codex resume`) can
|
|
11
|
+
* reconstruct the conversation — the registry just remembers what exists,
|
|
12
|
+
* where, and how to revive it. The client owns launch/revive command
|
|
13
|
+
* construction (it knows about remote-agent mode); the server owns truth
|
|
14
|
+
* about what is live.
|
|
15
|
+
*
|
|
16
|
+
* States: 'live' (pty exists) → 'hibernated' (pty gone, trajectory on disk).
|
|
17
|
+
* All records load as 'hibernated' on boot — ptys never survive a restart.
|
|
18
|
+
*/
|
|
19
|
+
export interface AgentRecord {
|
|
20
|
+
/** Terminal (pty) id the agent runs in — stable, derived from workdir. */
|
|
21
|
+
terminalId: string;
|
|
22
|
+
kind: 'claude' | 'codex';
|
|
23
|
+
/** Directory the agent was launched in — the project scope. */
|
|
24
|
+
workdir: string;
|
|
25
|
+
/** Where the CLI process runs: this server, or the user's machine over the reverse channel. */
|
|
26
|
+
location: 'server' | 'remote';
|
|
27
|
+
/** Claude --session-id (resume pointer). Codex resumes via its own picker. */
|
|
28
|
+
sessionId?: string;
|
|
29
|
+
/** Notebook that launched it (informational only — agents are not bound to notebooks). */
|
|
30
|
+
launchedFrom?: string;
|
|
31
|
+
/**
|
|
32
|
+
* Pinned workspace-mirror dir slug (`p-<hash>-<name>` under ~/.nebula/agent).
|
|
33
|
+
* Stored at launch so record-driven resumes keep finding the conversation
|
|
34
|
+
* even if the slug derivation ever changes — paths in records don't drift.
|
|
35
|
+
*/
|
|
36
|
+
mirrorSlug?: string;
|
|
37
|
+
state: 'live' | 'hibernated';
|
|
38
|
+
createdAt: number;
|
|
39
|
+
lastLaunchAt: number;
|
|
40
|
+
}
|
|
41
|
+
declare class AgentRegistry {
|
|
42
|
+
private records;
|
|
43
|
+
private exitUnsubs;
|
|
44
|
+
private loaded;
|
|
45
|
+
private ensureLoaded;
|
|
46
|
+
private persist;
|
|
47
|
+
/**
|
|
48
|
+
* Register (or re-register on revive) an agent launched in a terminal.
|
|
49
|
+
* MERGE semantics: fields the caller didn't supply NEVER overwrite stored
|
|
50
|
+
* truth — a resume relaunch that doesn't know the sessionId must not erase
|
|
51
|
+
* the registry's copy (that id is the conversation; losing it downgrades
|
|
52
|
+
* every later Continue to a picker).
|
|
53
|
+
*/
|
|
54
|
+
register(rec: Omit<AgentRecord, 'state' | 'createdAt' | 'lastLaunchAt'>): AgentRecord;
|
|
55
|
+
private static TUI_INIT_RE;
|
|
56
|
+
private static TUI_TEARDOWN_RE;
|
|
57
|
+
private dataUnsubs;
|
|
58
|
+
private graceTimers;
|
|
59
|
+
private watchLiveness;
|
|
60
|
+
/** Feed pty output through the liveness machine (public for tests). */
|
|
61
|
+
observeOutput(terminalId: string, chunk: string): void;
|
|
62
|
+
/** List all agents, reconciling 'live' against actual pty existence. */
|
|
63
|
+
list(): AgentRecord[];
|
|
64
|
+
/**
|
|
65
|
+
* list() plus process-table facts for every record whose pty exists:
|
|
66
|
+
* - `busy`: the pty's shell has a live child — SOMETHING is running there.
|
|
67
|
+
* Crucially reported on HIBERNATED records too: the TUI-stream liveness
|
|
68
|
+
* machine can mis-score (an agent resumed by hand never re-registers), and
|
|
69
|
+
* a client that trusts 'hibernated' then types a launch command straight
|
|
70
|
+
* into the running TUI's input box (lab report: a live codex politely
|
|
71
|
+
* declined to run the pasted ssh line). The process table is authoritative.
|
|
72
|
+
* - `idleShell` on live records: the inverse — 'live' but nothing running
|
|
73
|
+
* (a hung-then-dead ssh hop leaves exactly this; a dead transport emits
|
|
74
|
+
* no teardown, so only the process table can see it).
|
|
75
|
+
* Unknown checks claim nothing.
|
|
76
|
+
*/
|
|
77
|
+
listEnriched(): Promise<(AgentRecord & {
|
|
78
|
+
idleShell?: boolean;
|
|
79
|
+
busy?: boolean;
|
|
80
|
+
})[]>;
|
|
81
|
+
/** Hibernate: close the pty; the record (and on-disk trajectory) remain. */
|
|
82
|
+
hibernate(terminalId: string): boolean;
|
|
83
|
+
/** Forget the agent entirely (pty closed; registry record removed). */
|
|
84
|
+
remove(terminalId: string): boolean;
|
|
85
|
+
}
|
|
86
|
+
export declare const agentRegistry: AgentRegistry;
|
|
87
|
+
export {};
|