@thermai/flightdeck-cli 0.1.0-pilot.5 → 0.1.0-pilot.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -17
- package/dist/index.js +1802 -45
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import { Command } from "commander";
|
|
5
|
-
import { readFileSync as
|
|
5
|
+
import { readFileSync as readFileSync5 } from "node:fs";
|
|
6
6
|
|
|
7
7
|
// src/submit-core.ts
|
|
8
8
|
import { existsSync as existsSync2, mkdtempSync, readFileSync as readFileSync2, rmSync } from "node:fs";
|
|
@@ -31,11 +31,42 @@ var CliError = class extends Error {
|
|
|
31
31
|
this.name = "CliError";
|
|
32
32
|
}
|
|
33
33
|
};
|
|
34
|
-
|
|
34
|
+
var NetworkError = class extends CliError {
|
|
35
|
+
constructor(message, options) {
|
|
36
|
+
super(message, options);
|
|
37
|
+
this.name = "NetworkError";
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
function suspensionMessage(status2, code) {
|
|
41
|
+
if (status2 !== 403) return void 0;
|
|
42
|
+
switch (code) {
|
|
43
|
+
case "account_suspended":
|
|
44
|
+
return "This account is suspended. Contact your organization administrator.";
|
|
45
|
+
case "membership_suspended":
|
|
46
|
+
return "Your membership in this organization is suspended. Contact your organization administrator.";
|
|
47
|
+
case "organization_suspended":
|
|
48
|
+
return "This organization is suspended. Contact your organization administrator or ThermAI support.";
|
|
49
|
+
default:
|
|
50
|
+
return void 0;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
async function suspensionError(response) {
|
|
54
|
+
if (response.status !== 403) return void 0;
|
|
55
|
+
try {
|
|
56
|
+
const body = await response.clone().json();
|
|
57
|
+
const code = body && typeof body === "object" && !Array.isArray(body) ? body.error : void 0;
|
|
58
|
+
if (suspensionMessage(response.status, code)) return httpError("Access denied", response.status, code);
|
|
59
|
+
} catch {
|
|
60
|
+
}
|
|
61
|
+
return void 0;
|
|
62
|
+
}
|
|
63
|
+
function httpError(action, status2, code) {
|
|
64
|
+
const suspension = suspensionMessage(status2, code);
|
|
65
|
+
if (suspension) return new CliError(`${action} (HTTP ${status2}). ${suspension}`);
|
|
35
66
|
const guidance = status2 === 401 ? " Check the account with flightdeck whoami; sign in again if the credential expired or was revoked." : status2 === 403 ? " Check the account permissions and requested credential scopes." : status2 === 429 ? " Wait briefly before retrying." : "";
|
|
36
67
|
return new CliError(`${action} (HTTP ${status2}).${guidance}`);
|
|
37
68
|
}
|
|
38
|
-
var ERROR_TYPES = /* @__PURE__ */ new Set(["Error", "TypeError", "RangeError", "SyntaxError", "CliError", "AbortError"]);
|
|
69
|
+
var ERROR_TYPES = /* @__PURE__ */ new Set(["Error", "TypeError", "RangeError", "SyntaxError", "CliError", "NetworkError", "AtcFatalError", "AbortError"]);
|
|
39
70
|
var ERROR_CODES = /* @__PURE__ */ new Set([
|
|
40
71
|
"EACCES",
|
|
41
72
|
"EPERM",
|
|
@@ -148,9 +179,53 @@ function saveConfig(config, path = CONFIG_PATH) {
|
|
|
148
179
|
}
|
|
149
180
|
}
|
|
150
181
|
function deleteConfig(path = CONFIG_PATH) {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
182
|
+
try {
|
|
183
|
+
unlinkSync(path);
|
|
184
|
+
return true;
|
|
185
|
+
} catch (error) {
|
|
186
|
+
if (error.code === "ENOENT") return false;
|
|
187
|
+
throw error;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
async function revokeCurrentCredential(config) {
|
|
191
|
+
const controller = new AbortController();
|
|
192
|
+
let timer;
|
|
193
|
+
const deadline = new Promise((resolve4) => {
|
|
194
|
+
timer = setTimeout(() => {
|
|
195
|
+
controller.abort();
|
|
196
|
+
resolve4("unconfirmed");
|
|
197
|
+
}, 3e3);
|
|
198
|
+
});
|
|
199
|
+
const revoke = async () => {
|
|
200
|
+
try {
|
|
201
|
+
const response = await fetch(endpoint(config.api_url, "/auth/credential/revoke"), {
|
|
202
|
+
method: "POST",
|
|
203
|
+
headers: { Authorization: `Bearer ${config.api_key}`, [ACCOUNTING_HEADER]: ACCOUNTING_CONTRACT },
|
|
204
|
+
credentials: "omit",
|
|
205
|
+
redirect: "error",
|
|
206
|
+
signal: controller.signal
|
|
207
|
+
});
|
|
208
|
+
if (response.status === 204) return "revoked";
|
|
209
|
+
if (response.status !== 200 && response.status !== 401) {
|
|
210
|
+
void response.body?.cancel().catch(() => {
|
|
211
|
+
});
|
|
212
|
+
return "unconfirmed";
|
|
213
|
+
}
|
|
214
|
+
const body = await response.json();
|
|
215
|
+
if (body && typeof body === "object") {
|
|
216
|
+
if (response.status === 200 && body.ok === true) return "revoked";
|
|
217
|
+
if (response.status === 401 && config.api_key.startsWith("fd_k_") && body.error === "unauthorized") return "invalid";
|
|
218
|
+
}
|
|
219
|
+
} catch {
|
|
220
|
+
}
|
|
221
|
+
return "unconfirmed";
|
|
222
|
+
};
|
|
223
|
+
try {
|
|
224
|
+
return await Promise.race([revoke(), deadline]);
|
|
225
|
+
} finally {
|
|
226
|
+
clearTimeout(timer);
|
|
227
|
+
controller.abort();
|
|
228
|
+
}
|
|
154
229
|
}
|
|
155
230
|
function requestBody(body, headers) {
|
|
156
231
|
if (body instanceof FormData) return body;
|
|
@@ -169,10 +244,10 @@ async function publicApiRequest(apiUrl, method, path, body) {
|
|
|
169
244
|
body: requestBody(body, headers)
|
|
170
245
|
});
|
|
171
246
|
}
|
|
172
|
-
async function apiRequest(method, path, body) {
|
|
173
|
-
return authenticatedRequest(loadConfig(), method, path, body);
|
|
247
|
+
async function apiRequest(method, path, body, options = {}) {
|
|
248
|
+
return authenticatedRequest(options.config ?? loadConfig(), method, path, body, "follow", options.signal);
|
|
174
249
|
}
|
|
175
|
-
async function authenticatedRequest(config, method, path, body, redirect = "follow") {
|
|
250
|
+
async function authenticatedRequest(config, method, path, body, redirect = "follow", signal) {
|
|
176
251
|
const headers = {
|
|
177
252
|
Authorization: `Bearer ${config.api_key}`,
|
|
178
253
|
[ACCOUNTING_HEADER]: ACCOUNTING_CONTRACT
|
|
@@ -186,8 +261,12 @@ async function authenticatedRequest(config, method, path, body, redirect = "foll
|
|
|
186
261
|
method,
|
|
187
262
|
headers,
|
|
188
263
|
body: requestBody(body, headers),
|
|
189
|
-
redirect
|
|
264
|
+
redirect,
|
|
265
|
+
signal
|
|
190
266
|
});
|
|
267
|
+
if (response.status === 403) {
|
|
268
|
+
throw await suspensionError(response) ?? httpError("Access denied", response.status);
|
|
269
|
+
}
|
|
191
270
|
if (path !== "/auth/credential/revoke" && (response.ok || response.status === 402)) assertAccountingContract(response);
|
|
192
271
|
if (response.status === 409) {
|
|
193
272
|
const body2 = await response.clone().json().catch(() => ({}));
|
|
@@ -229,7 +308,7 @@ async function request(url, options) {
|
|
|
229
308
|
try {
|
|
230
309
|
return await fetch(url, options);
|
|
231
310
|
} catch (cause) {
|
|
232
|
-
throw new
|
|
311
|
+
throw new NetworkError("Could not reach the Flightdeck API. Check your network connection and API URL before retrying.", { cause });
|
|
233
312
|
}
|
|
234
313
|
}
|
|
235
314
|
|
|
@@ -420,7 +499,7 @@ Track with: flightdeck status ${body.flight_id}`);
|
|
|
420
499
|
}
|
|
421
500
|
|
|
422
501
|
// src/commands/sbatch.ts
|
|
423
|
-
import { existsSync as
|
|
502
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4, rmSync as rmSync2 } from "node:fs";
|
|
424
503
|
import { dirname as dirname2, isAbsolute, normalize, relative, resolve as resolve2 } from "node:path";
|
|
425
504
|
|
|
426
505
|
// src/sbatch.ts
|
|
@@ -666,24 +745,24 @@ function parseGpuRequest(value, fromGres) {
|
|
|
666
745
|
if (tokens.length === 1) {
|
|
667
746
|
const onlyToken = tokens[0];
|
|
668
747
|
if (/^\d+$/.test(onlyToken)) {
|
|
669
|
-
const
|
|
670
|
-
if (!Number.isSafeInteger(
|
|
748
|
+
const count3 = Number(onlyToken);
|
|
749
|
+
if (!Number.isSafeInteger(count3) || count3 < 1 || count3 > MAX_GPU_COUNT) {
|
|
671
750
|
throw new Error(`GPU count must be between 1 and ${MAX_GPU_COUNT} in "${value}"`);
|
|
672
751
|
}
|
|
673
|
-
return { gpuCount:
|
|
752
|
+
return { gpuCount: count3 };
|
|
674
753
|
}
|
|
675
754
|
return { gpuType: onlyToken, gpuCount: 1 };
|
|
676
755
|
}
|
|
677
|
-
const [type,
|
|
678
|
-
if (!/^\d+$/.test(
|
|
756
|
+
const [type, count2] = tokens;
|
|
757
|
+
if (!/^\d+$/.test(count2) || !Number.isSafeInteger(Number(count2)) || Number(count2) < 1 || Number(count2) > MAX_GPU_COUNT) {
|
|
679
758
|
throw new Error(`GPU count must be between 1 and ${MAX_GPU_COUNT} in "${value}"`);
|
|
680
759
|
}
|
|
681
|
-
return { gpuType: type, gpuCount: Number(
|
|
760
|
+
return { gpuType: type, gpuCount: Number(count2) };
|
|
682
761
|
}
|
|
683
|
-
function parseSbatchScript(
|
|
762
|
+
function parseSbatchScript(text2) {
|
|
684
763
|
const result = { warnings: [], exportedEnv: {} };
|
|
685
764
|
const ignored = /* @__PURE__ */ new Set();
|
|
686
|
-
for (const rawLine of
|
|
765
|
+
for (const rawLine of text2.split("\n")) {
|
|
687
766
|
const line = rawLine.trim();
|
|
688
767
|
if (line === "") continue;
|
|
689
768
|
if (!line.startsWith("#")) break;
|
|
@@ -773,6 +852,59 @@ function parseSbatchScript(text) {
|
|
|
773
852
|
return result;
|
|
774
853
|
}
|
|
775
854
|
|
|
855
|
+
// src/batch-record.ts
|
|
856
|
+
import { chmodSync as chmodSync2, existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync3, renameSync as renameSync2, unlinkSync as unlinkSync2, writeFileSync as writeFileSync2 } from "node:fs";
|
|
857
|
+
import { homedir as homedir2 } from "node:os";
|
|
858
|
+
import { join as join3 } from "node:path";
|
|
859
|
+
var BATCHES_DIRECTORY = join3(homedir2(), ".flightdeck", "batches");
|
|
860
|
+
function isBatchJobId(value) {
|
|
861
|
+
return /^\d{1,32}$/.test(value);
|
|
862
|
+
}
|
|
863
|
+
function batchRecordPath(jobId, directory = BATCHES_DIRECTORY) {
|
|
864
|
+
if (!isBatchJobId(jobId)) throw new CliError(`Not an sbatch job ID: ${jobId}`);
|
|
865
|
+
return join3(directory, `${jobId}.json`);
|
|
866
|
+
}
|
|
867
|
+
function saveBatchRecord(record2, directory = BATCHES_DIRECTORY) {
|
|
868
|
+
const path = batchRecordPath(record2.job_id, directory);
|
|
869
|
+
mkdirSync2(directory, { recursive: true, mode: 448 });
|
|
870
|
+
chmodSync2(directory, 448);
|
|
871
|
+
const temporaryPath = `${path}.tmp-${process.pid}`;
|
|
872
|
+
try {
|
|
873
|
+
writeFileSync2(temporaryPath, `${JSON.stringify(record2, null, 2)}
|
|
874
|
+
`, { encoding: "utf8", mode: 384, flag: "wx" });
|
|
875
|
+
renameSync2(temporaryPath, path);
|
|
876
|
+
} finally {
|
|
877
|
+
if (existsSync3(temporaryPath)) unlinkSync2(temporaryPath);
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
function validFlight(value) {
|
|
881
|
+
if (!value || typeof value !== "object") return false;
|
|
882
|
+
const flight = value;
|
|
883
|
+
return typeof flight.flight_id === "string" && flight.flight_id.length > 0 && typeof flight.name === "string" && (flight.task_id === null || typeof flight.task_id === "number" && Number.isInteger(flight.task_id));
|
|
884
|
+
}
|
|
885
|
+
function validApiUrl(value) {
|
|
886
|
+
try {
|
|
887
|
+
normalizeApiUrl(value);
|
|
888
|
+
return true;
|
|
889
|
+
} catch {
|
|
890
|
+
return false;
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
function loadBatchRecord(jobId, directory = BATCHES_DIRECTORY) {
|
|
894
|
+
const path = batchRecordPath(jobId, directory);
|
|
895
|
+
if (!existsSync3(path)) return null;
|
|
896
|
+
let record2;
|
|
897
|
+
try {
|
|
898
|
+
record2 = JSON.parse(readFileSync3(path, "utf8"));
|
|
899
|
+
} catch {
|
|
900
|
+
throw new CliError(`The local record for sbatch job ${jobId} is unreadable: ${path}`);
|
|
901
|
+
}
|
|
902
|
+
if (record2?.version !== 1 || record2.job_id !== jobId || typeof record2.job_name !== "string" || typeof record2.api_url !== "string" || typeof record2.submitted_at !== "string" || !Number.isFinite(Date.parse(record2.submitted_at)) || !Array.isArray(record2.flights) || !record2.flights.every(validFlight) || !validApiUrl(record2.api_url)) {
|
|
903
|
+
throw new CliError(`The local record for sbatch job ${jobId} is not valid: ${path}`);
|
|
904
|
+
}
|
|
905
|
+
return record2;
|
|
906
|
+
}
|
|
907
|
+
|
|
776
908
|
// src/commands/sbatch.ts
|
|
777
909
|
var MAX_DURATION_S = 86400;
|
|
778
910
|
var WORKSPACE_ROOT = "/workspace";
|
|
@@ -903,12 +1035,49 @@ function buildSbatchPlans(args) {
|
|
|
903
1035
|
}
|
|
904
1036
|
}));
|
|
905
1037
|
}
|
|
1038
|
+
function batchRecorder(jobId, jobName, target = { apiUrl: readConfig()?.api_url }) {
|
|
1039
|
+
const apiUrl = target.apiUrl;
|
|
1040
|
+
const submittedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1041
|
+
let warned = false;
|
|
1042
|
+
const recorder = {
|
|
1043
|
+
saved: false,
|
|
1044
|
+
record(submitted) {
|
|
1045
|
+
if (!apiUrl) return;
|
|
1046
|
+
try {
|
|
1047
|
+
saveBatchRecord({
|
|
1048
|
+
version: 1,
|
|
1049
|
+
job_id: jobId,
|
|
1050
|
+
job_name: jobName,
|
|
1051
|
+
api_url: apiUrl,
|
|
1052
|
+
submitted_at: submittedAt,
|
|
1053
|
+
flights: submitted.map((flight) => ({ flight_id: flight.flightId, task_id: flight.taskId, name: flight.name }))
|
|
1054
|
+
}, target.directory);
|
|
1055
|
+
recorder.saved = true;
|
|
1056
|
+
} catch (error) {
|
|
1057
|
+
recorder.saved = false;
|
|
1058
|
+
if (!warned) {
|
|
1059
|
+
warned = true;
|
|
1060
|
+
console.error(` warning: could not record this batch for flightdeck atc (${error.code ?? "write failed"})`);
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
},
|
|
1064
|
+
/** A record missing the latest tasks would misrepresent the array; drop it. */
|
|
1065
|
+
finish() {
|
|
1066
|
+
if (recorder.saved || !apiUrl) return;
|
|
1067
|
+
try {
|
|
1068
|
+
rmSync2(batchRecordPath(jobId, target.directory), { force: true });
|
|
1069
|
+
} catch {
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
};
|
|
1073
|
+
return recorder;
|
|
1074
|
+
}
|
|
906
1075
|
async function sbatch(scriptPath, options) {
|
|
907
|
-
if (!
|
|
1076
|
+
if (!existsSync4(scriptPath)) {
|
|
908
1077
|
console.error(`Script not found: ${scriptPath}`);
|
|
909
1078
|
process.exit(1);
|
|
910
1079
|
}
|
|
911
|
-
const scriptText =
|
|
1080
|
+
const scriptText = readFileSync4(scriptPath, "utf-8");
|
|
912
1081
|
let directives;
|
|
913
1082
|
try {
|
|
914
1083
|
directives = parseSbatchScript(scriptText);
|
|
@@ -976,6 +1145,7 @@ Packaging files...`);
|
|
|
976
1145
|
`);
|
|
977
1146
|
const submitted = [];
|
|
978
1147
|
let failures = 0;
|
|
1148
|
+
const batch = batchRecorder(jobId, jobName);
|
|
979
1149
|
for (const task of tasks) {
|
|
980
1150
|
try {
|
|
981
1151
|
const result = await submitFlight(task.plan, bundle);
|
|
@@ -988,12 +1158,16 @@ Packaging files...`);
|
|
|
988
1158
|
} else {
|
|
989
1159
|
console.error(` ${task.plan.name}: ${err.message}`);
|
|
990
1160
|
}
|
|
1161
|
+
continue;
|
|
991
1162
|
}
|
|
1163
|
+
batch.record(submitted);
|
|
992
1164
|
}
|
|
1165
|
+
batch.finish();
|
|
993
1166
|
console.log(`
|
|
994
1167
|
${submitted.length} submitted, ${failures} failed.`);
|
|
995
1168
|
if (submitted.length > 0) {
|
|
996
1169
|
console.log(`Track with: flightdeck status <flight_id>`);
|
|
1170
|
+
if (batch.saved) console.log(`Watch live: flightdeck atc ${jobId}`);
|
|
997
1171
|
}
|
|
998
1172
|
if (failures > 0) process.exit(1);
|
|
999
1173
|
return { submitted, failures };
|
|
@@ -1084,6 +1258,1582 @@ async function cancel(flightId) {
|
|
|
1084
1258
|
console.log(`Cancelled ${flightId}`);
|
|
1085
1259
|
}
|
|
1086
1260
|
|
|
1261
|
+
// src/atc/input.ts
|
|
1262
|
+
var ESCAPE_SEQUENCES = [
|
|
1263
|
+
["\x1B[A", "up"],
|
|
1264
|
+
["\x1BOA", "up"],
|
|
1265
|
+
["\x1B[B", "down"],
|
|
1266
|
+
["\x1BOB", "down"],
|
|
1267
|
+
["\x1B[5~", "pageup"],
|
|
1268
|
+
["\x1B[6~", "pagedown"],
|
|
1269
|
+
["\x1B[H", "home"],
|
|
1270
|
+
["\x1BOH", "home"],
|
|
1271
|
+
["\x1B[1~", "home"],
|
|
1272
|
+
["\x1B[F", "end"],
|
|
1273
|
+
["\x1BOF", "end"],
|
|
1274
|
+
["\x1B[4~", "end"]
|
|
1275
|
+
];
|
|
1276
|
+
var SINGLE_CHARACTERS = {
|
|
1277
|
+
"\r": "enter",
|
|
1278
|
+
"\n": "enter",
|
|
1279
|
+
"": "interrupt",
|
|
1280
|
+
"\f": "redraw",
|
|
1281
|
+
r: "refresh",
|
|
1282
|
+
R: "refresh",
|
|
1283
|
+
q: "quit",
|
|
1284
|
+
Q: "quit",
|
|
1285
|
+
k: "up",
|
|
1286
|
+
j: "down",
|
|
1287
|
+
g: "home",
|
|
1288
|
+
G: "end",
|
|
1289
|
+
l: "logs",
|
|
1290
|
+
L: "logs",
|
|
1291
|
+
f: "follow",
|
|
1292
|
+
F: "follow",
|
|
1293
|
+
" ": "pagedown",
|
|
1294
|
+
b: "pageup"
|
|
1295
|
+
};
|
|
1296
|
+
var PASTE_START = "\x1B[200~";
|
|
1297
|
+
var PASTE_END = "\x1B[201~";
|
|
1298
|
+
var ESCAPE_TIMEOUT_MS = 50;
|
|
1299
|
+
var PASTE_TIMEOUT_MS = 1500;
|
|
1300
|
+
var COMPLETE_SEQUENCE = /^\x1b(\[[0-9;?]*[ -/]*[@-~]|O.)/;
|
|
1301
|
+
var INCOMPLETE_SEQUENCE = /^\x1b(\[[0-9;?]*[ -/]*|O)?$/;
|
|
1302
|
+
var KeyReader = class {
|
|
1303
|
+
pending = "";
|
|
1304
|
+
inPaste = false;
|
|
1305
|
+
/** How long to wait before `flushPending`, or null when nothing is held. */
|
|
1306
|
+
get pendingTimeoutMs() {
|
|
1307
|
+
if (this.inPaste) return PASTE_TIMEOUT_MS;
|
|
1308
|
+
return this.pending.length > 0 ? ESCAPE_TIMEOUT_MS : null;
|
|
1309
|
+
}
|
|
1310
|
+
feed(chunk) {
|
|
1311
|
+
let buffer = this.pending + chunk;
|
|
1312
|
+
this.pending = "";
|
|
1313
|
+
const keys = [];
|
|
1314
|
+
while (buffer.length > 0) {
|
|
1315
|
+
if (this.inPaste) {
|
|
1316
|
+
const end = buffer.indexOf(PASTE_END);
|
|
1317
|
+
const interrupt = buffer.indexOf("");
|
|
1318
|
+
if (interrupt >= 0 && (end < 0 || interrupt < end)) {
|
|
1319
|
+
this.inPaste = false;
|
|
1320
|
+
keys.push("interrupt");
|
|
1321
|
+
buffer = buffer.slice(interrupt + 1);
|
|
1322
|
+
continue;
|
|
1323
|
+
}
|
|
1324
|
+
if (end < 0) {
|
|
1325
|
+
let keep = Math.min(PASTE_END.length - 1, buffer.length);
|
|
1326
|
+
while (keep > 0 && !PASTE_END.startsWith(buffer.slice(-keep))) keep -= 1;
|
|
1327
|
+
this.pending = keep ? buffer.slice(-keep) : "";
|
|
1328
|
+
return keys;
|
|
1329
|
+
}
|
|
1330
|
+
this.inPaste = false;
|
|
1331
|
+
buffer = buffer.slice(end + PASTE_END.length);
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
if (buffer[0] !== "\x1B") {
|
|
1335
|
+
keys.push(SINGLE_CHARACTERS[buffer[0]] ?? "unknown");
|
|
1336
|
+
buffer = buffer.slice(1);
|
|
1337
|
+
continue;
|
|
1338
|
+
}
|
|
1339
|
+
if (buffer.startsWith(PASTE_START)) {
|
|
1340
|
+
this.inPaste = true;
|
|
1341
|
+
buffer = buffer.slice(PASTE_START.length);
|
|
1342
|
+
continue;
|
|
1343
|
+
}
|
|
1344
|
+
const known = ESCAPE_SEQUENCES.find(([sequence]) => buffer.startsWith(sequence));
|
|
1345
|
+
if (known) {
|
|
1346
|
+
keys.push(known[1]);
|
|
1347
|
+
buffer = buffer.slice(known[0].length);
|
|
1348
|
+
continue;
|
|
1349
|
+
}
|
|
1350
|
+
const complete = COMPLETE_SEQUENCE.exec(buffer);
|
|
1351
|
+
if (complete) {
|
|
1352
|
+
keys.push("unknown");
|
|
1353
|
+
buffer = buffer.slice(complete[0].length);
|
|
1354
|
+
continue;
|
|
1355
|
+
}
|
|
1356
|
+
if (INCOMPLETE_SEQUENCE.test(buffer)) {
|
|
1357
|
+
this.pending = buffer;
|
|
1358
|
+
return keys;
|
|
1359
|
+
}
|
|
1360
|
+
keys.push("unknown");
|
|
1361
|
+
buffer = buffer.slice(2);
|
|
1362
|
+
}
|
|
1363
|
+
return keys;
|
|
1364
|
+
}
|
|
1365
|
+
/**
|
|
1366
|
+
* The timeout passed: an unfinished paste is abandoned, and a held escape
|
|
1367
|
+
* prefix was a bare Escape (plus any junk after it).
|
|
1368
|
+
*/
|
|
1369
|
+
flushPending() {
|
|
1370
|
+
if (this.inPaste) {
|
|
1371
|
+
this.inPaste = false;
|
|
1372
|
+
this.pending = "";
|
|
1373
|
+
return [];
|
|
1374
|
+
}
|
|
1375
|
+
if (!this.pending) return [];
|
|
1376
|
+
const held = this.pending;
|
|
1377
|
+
this.pending = "";
|
|
1378
|
+
return held === "\x1B" ? ["escape"] : ["escape", ...this.feed(held.slice(1))];
|
|
1379
|
+
}
|
|
1380
|
+
};
|
|
1381
|
+
|
|
1382
|
+
// src/atc/model.ts
|
|
1383
|
+
var TERMINAL_STATUSES = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
|
|
1384
|
+
var AtcFatalError = class extends CliError {
|
|
1385
|
+
constructor(message, options) {
|
|
1386
|
+
super(message, options);
|
|
1387
|
+
this.name = "AtcFatalError";
|
|
1388
|
+
}
|
|
1389
|
+
};
|
|
1390
|
+
function isTerminal(status2) {
|
|
1391
|
+
return TERMINAL_STATUSES.has(status2);
|
|
1392
|
+
}
|
|
1393
|
+
function elapsedSeconds(flight, nowMs) {
|
|
1394
|
+
const startedIso = flight.started_at ?? flight.dispatched_at;
|
|
1395
|
+
if (!startedIso) return null;
|
|
1396
|
+
const endMs = flight.completed_at ? Date.parse(flight.completed_at) : nowMs;
|
|
1397
|
+
const seconds = Math.floor((endMs - Date.parse(startedIso)) / 1e3);
|
|
1398
|
+
return Number.isFinite(seconds) ? Math.max(0, seconds) : null;
|
|
1399
|
+
}
|
|
1400
|
+
function formatDuration(totalSeconds) {
|
|
1401
|
+
if (totalSeconds === null) return "\u2014";
|
|
1402
|
+
const hours = Math.floor(totalSeconds / 3600);
|
|
1403
|
+
const minutes = Math.floor(totalSeconds % 3600 / 60);
|
|
1404
|
+
const seconds = totalSeconds % 60;
|
|
1405
|
+
const pad = (value) => String(value).padStart(2, "0");
|
|
1406
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds)}` : `${pad(minutes)}:${pad(seconds)}`;
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
// src/atc/text.ts
|
|
1410
|
+
var COLOR_CODES = {
|
|
1411
|
+
red: 31,
|
|
1412
|
+
green: 32,
|
|
1413
|
+
yellow: 33,
|
|
1414
|
+
blue: 34,
|
|
1415
|
+
magenta: 35,
|
|
1416
|
+
cyan: 36,
|
|
1417
|
+
white: 37,
|
|
1418
|
+
gray: 90
|
|
1419
|
+
};
|
|
1420
|
+
var ZERO_WIDTH = /^[\p{Mn}\p{Me}\p{Cf}]$/u;
|
|
1421
|
+
var EMOJI_PRESENTATION = new RegExp("^\\p{Emoji_Presentation}$", "u");
|
|
1422
|
+
var PICTOGRAPHIC = new RegExp("\\p{Extended_Pictographic}", "u");
|
|
1423
|
+
var FLAG = new RegExp("^\\p{Regional_Indicator}{2}$", "u");
|
|
1424
|
+
var WIDE_RANGES = [
|
|
1425
|
+
[4352, 4447],
|
|
1426
|
+
[11904, 12350],
|
|
1427
|
+
[12353, 13311],
|
|
1428
|
+
[13312, 19903],
|
|
1429
|
+
[19968, 40959],
|
|
1430
|
+
[40960, 42191],
|
|
1431
|
+
[43360, 43391],
|
|
1432
|
+
[44032, 55203],
|
|
1433
|
+
[63744, 64255],
|
|
1434
|
+
[65040, 65049],
|
|
1435
|
+
[65072, 65135],
|
|
1436
|
+
[65280, 65376],
|
|
1437
|
+
[65504, 65510],
|
|
1438
|
+
[127744, 128591],
|
|
1439
|
+
[129280, 129535],
|
|
1440
|
+
[131072, 196605],
|
|
1441
|
+
[196608, 262141]
|
|
1442
|
+
];
|
|
1443
|
+
var graphemes = new Intl.Segmenter(void 0, { granularity: "grapheme" });
|
|
1444
|
+
function cellWidth(character) {
|
|
1445
|
+
if (ZERO_WIDTH.test(character)) return 0;
|
|
1446
|
+
const codePoint = character.codePointAt(0);
|
|
1447
|
+
if (EMOJI_PRESENTATION.test(character)) return 2;
|
|
1448
|
+
for (const [low, high] of WIDE_RANGES) {
|
|
1449
|
+
if (codePoint >= low && codePoint <= high) return 2;
|
|
1450
|
+
}
|
|
1451
|
+
return 1;
|
|
1452
|
+
}
|
|
1453
|
+
function clusterWidth(cluster) {
|
|
1454
|
+
const codePoints = [...cluster];
|
|
1455
|
+
if (codePoints.length > 1) {
|
|
1456
|
+
if (FLAG.test(cluster)) return 2;
|
|
1457
|
+
if (PICTOGRAPHIC.test(cluster) && (cluster.includes("\uFE0F") || cluster.includes("\u200D"))) return 2;
|
|
1458
|
+
}
|
|
1459
|
+
return cellWidth(codePoints[0]);
|
|
1460
|
+
}
|
|
1461
|
+
function sanitize(text2) {
|
|
1462
|
+
return text2.replace(/[--]/g, "").replace(/[\x00-\x1f\x7f-\x9f]/g, " ");
|
|
1463
|
+
}
|
|
1464
|
+
function* clustersOf(text2) {
|
|
1465
|
+
for (const { segment } of graphemes.segment(text2)) yield segment;
|
|
1466
|
+
}
|
|
1467
|
+
function textWidth(text2) {
|
|
1468
|
+
let total = 0;
|
|
1469
|
+
for (const cluster of clustersOf(sanitize(text2))) total += clusterWidth(cluster);
|
|
1470
|
+
return total;
|
|
1471
|
+
}
|
|
1472
|
+
function seg(text2, style) {
|
|
1473
|
+
return { text: text2, style };
|
|
1474
|
+
}
|
|
1475
|
+
function visibleLength(line) {
|
|
1476
|
+
return line.reduce((total, segment) => total + textWidth(segment.text), 0);
|
|
1477
|
+
}
|
|
1478
|
+
function sgr(style, colorEnabled) {
|
|
1479
|
+
if (!style) return "";
|
|
1480
|
+
const codes = [];
|
|
1481
|
+
if (style.bold) codes.push(1);
|
|
1482
|
+
if (style.dim) codes.push(2);
|
|
1483
|
+
if (style.inverse) codes.push(7);
|
|
1484
|
+
if (style.color && colorEnabled) codes.push(COLOR_CODES[style.color]);
|
|
1485
|
+
return codes.length ? `\x1B[${codes.join(";")}m` : "";
|
|
1486
|
+
}
|
|
1487
|
+
function takeCells(text2, width) {
|
|
1488
|
+
const scanLimit = width * 2 + 8;
|
|
1489
|
+
let scanned = text2.length > scanLimit ? text2.slice(0, scanLimit) : text2;
|
|
1490
|
+
if (scanned.length < text2.length && /[\ud800-\udbff]$/.test(scanned)) scanned = scanned.slice(0, -1);
|
|
1491
|
+
let visible = "";
|
|
1492
|
+
let used = 0;
|
|
1493
|
+
for (const cluster of clustersOf(sanitize(scanned))) {
|
|
1494
|
+
const clusterCells = clusterWidth(cluster);
|
|
1495
|
+
if (used + clusterCells > width) return { visible, used, consumedAll: false };
|
|
1496
|
+
visible += cluster;
|
|
1497
|
+
used += clusterCells;
|
|
1498
|
+
}
|
|
1499
|
+
return { visible, used, consumedAll: scanned.length === text2.length };
|
|
1500
|
+
}
|
|
1501
|
+
function encodeLine(line, width, colorEnabled, fill) {
|
|
1502
|
+
let remaining = width;
|
|
1503
|
+
let output = "";
|
|
1504
|
+
for (const segment of line) {
|
|
1505
|
+
if (remaining <= 0) break;
|
|
1506
|
+
const { visible, used, consumedAll } = takeCells(segment.text, remaining);
|
|
1507
|
+
remaining -= used;
|
|
1508
|
+
const open = sgr(segment.style, colorEnabled);
|
|
1509
|
+
output += open && visible ? `${open}${visible}\x1B[0m` : visible;
|
|
1510
|
+
if (!consumedAll) break;
|
|
1511
|
+
}
|
|
1512
|
+
if (remaining > 0) {
|
|
1513
|
+
const open = sgr(fill, colorEnabled);
|
|
1514
|
+
output += open ? `${open}${" ".repeat(remaining)}\x1B[0m` : " ".repeat(remaining);
|
|
1515
|
+
}
|
|
1516
|
+
return output;
|
|
1517
|
+
}
|
|
1518
|
+
function clipLine(line, width) {
|
|
1519
|
+
const clipped = [];
|
|
1520
|
+
let remaining = width;
|
|
1521
|
+
for (const segment of line) {
|
|
1522
|
+
if (remaining <= 0) break;
|
|
1523
|
+
const { visible, used, consumedAll } = takeCells(segment.text, remaining);
|
|
1524
|
+
clipped.push({ text: visible, style: segment.style });
|
|
1525
|
+
remaining -= used;
|
|
1526
|
+
if (!consumedAll) break;
|
|
1527
|
+
}
|
|
1528
|
+
return clipped;
|
|
1529
|
+
}
|
|
1530
|
+
function fixed(text2, width, align = "left") {
|
|
1531
|
+
const clean = sanitize(text2);
|
|
1532
|
+
const currentWidth = textWidth(clean);
|
|
1533
|
+
if (currentWidth > width) {
|
|
1534
|
+
const room = width <= 1 ? width : width - 1;
|
|
1535
|
+
const { visible, used } = takeCells(clean, room);
|
|
1536
|
+
return `${visible}${width <= 1 ? "" : "\u2026"}${" ".repeat(room - used)}`;
|
|
1537
|
+
}
|
|
1538
|
+
const padding = " ".repeat(width - currentWidth);
|
|
1539
|
+
return align === "left" ? clean + padding : padding + clean;
|
|
1540
|
+
}
|
|
1541
|
+
function highlight(line, style) {
|
|
1542
|
+
return line.map((segment) => ({ text: segment.text, style: { ...style, ...segment.style, inverse: style.inverse } }));
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
// src/atc/screen.ts
|
|
1546
|
+
var ENTER_ALTERNATE_SCREEN = "\x1B[?1049h\x1B[?25l\x1B[?7l\x1B[?2004h\x1B[2J";
|
|
1547
|
+
var LEAVE_ALTERNATE_SCREEN = "\x1B[0m\x1B[?2004l\x1B[?7h\x1B[?25h\x1B[?1049l";
|
|
1548
|
+
var BEGIN_FRAME = "\x1B[?2026h";
|
|
1549
|
+
var END_FRAME = "\x1B[?2026l";
|
|
1550
|
+
var Screen = class {
|
|
1551
|
+
constructor(output, colorEnabled) {
|
|
1552
|
+
this.output = output;
|
|
1553
|
+
this.colorEnabled = colorEnabled;
|
|
1554
|
+
}
|
|
1555
|
+
output;
|
|
1556
|
+
colorEnabled;
|
|
1557
|
+
previousRows = [];
|
|
1558
|
+
previousSize = "";
|
|
1559
|
+
active = false;
|
|
1560
|
+
get width() {
|
|
1561
|
+
return this.output.columns ?? 80;
|
|
1562
|
+
}
|
|
1563
|
+
get height() {
|
|
1564
|
+
return this.output.rows ?? 24;
|
|
1565
|
+
}
|
|
1566
|
+
enter() {
|
|
1567
|
+
if (this.active) return;
|
|
1568
|
+
this.active = true;
|
|
1569
|
+
this.output.write(ENTER_ALTERNATE_SCREEN);
|
|
1570
|
+
}
|
|
1571
|
+
leave() {
|
|
1572
|
+
if (!this.active) return;
|
|
1573
|
+
this.active = false;
|
|
1574
|
+
this.output.write(LEAVE_ALTERNATE_SCREEN);
|
|
1575
|
+
}
|
|
1576
|
+
/** Force a full repaint on the next draw (after a resize). */
|
|
1577
|
+
invalidate() {
|
|
1578
|
+
this.previousRows = [];
|
|
1579
|
+
}
|
|
1580
|
+
draw(lines) {
|
|
1581
|
+
const width = this.width;
|
|
1582
|
+
const height = this.height;
|
|
1583
|
+
const size = `${width}x${height}`;
|
|
1584
|
+
let frame = "";
|
|
1585
|
+
if (size !== this.previousSize) {
|
|
1586
|
+
frame += "\x1B[2J";
|
|
1587
|
+
this.previousRows = [];
|
|
1588
|
+
this.previousSize = size;
|
|
1589
|
+
}
|
|
1590
|
+
for (let row = 0; row < height; row += 1) {
|
|
1591
|
+
const encoded = encodeLine(lines[row] ?? [], width, this.colorEnabled);
|
|
1592
|
+
if (this.previousRows[row] === encoded) continue;
|
|
1593
|
+
this.previousRows[row] = encoded;
|
|
1594
|
+
frame += `\x1B[${row + 1};1H${encoded}`;
|
|
1595
|
+
}
|
|
1596
|
+
if (frame) this.output.write(BEGIN_FRAME + frame + END_FRAME);
|
|
1597
|
+
}
|
|
1598
|
+
};
|
|
1599
|
+
|
|
1600
|
+
// src/atc/state.ts
|
|
1601
|
+
var MAX_LOG_LINES = 5e3;
|
|
1602
|
+
var MAX_LOG_LINE_LENGTH = 64 * 1024;
|
|
1603
|
+
function initialState() {
|
|
1604
|
+
return {
|
|
1605
|
+
view: "board",
|
|
1606
|
+
logReturnView: "board",
|
|
1607
|
+
flights: [],
|
|
1608
|
+
selectedFlightId: null,
|
|
1609
|
+
events: null,
|
|
1610
|
+
logs: null,
|
|
1611
|
+
following: true,
|
|
1612
|
+
logTopLine: 0,
|
|
1613
|
+
flipStartedAtMs: /* @__PURE__ */ new Map(),
|
|
1614
|
+
lastUpdatedMs: null,
|
|
1615
|
+
lastError: null,
|
|
1616
|
+
selectionError: null,
|
|
1617
|
+
notice: null,
|
|
1618
|
+
quit: false
|
|
1619
|
+
};
|
|
1620
|
+
}
|
|
1621
|
+
function selectedFlight(state) {
|
|
1622
|
+
return state.flights.find((flight) => flight.flight_id === state.selectedFlightId);
|
|
1623
|
+
}
|
|
1624
|
+
function selectedIndex(state) {
|
|
1625
|
+
const index = state.flights.findIndex((flight) => flight.flight_id === state.selectedFlightId);
|
|
1626
|
+
return index < 0 ? 0 : index;
|
|
1627
|
+
}
|
|
1628
|
+
function mergeFlightRecord(state, record2, nowMs) {
|
|
1629
|
+
const previous = state.flights.find((flight) => flight.flight_id === record2.flight_id);
|
|
1630
|
+
if (!previous) return state;
|
|
1631
|
+
const flipStartedAtMs = previous.status !== record2.status ? new Map(state.flipStartedAtMs).set(record2.flight_id, nowMs) : state.flipStartedAtMs;
|
|
1632
|
+
return {
|
|
1633
|
+
...state,
|
|
1634
|
+
flipStartedAtMs,
|
|
1635
|
+
flights: state.flights.map((flight) => flight.flight_id === record2.flight_id ? record2 : flight)
|
|
1636
|
+
};
|
|
1637
|
+
}
|
|
1638
|
+
function applyFlights(state, incoming, nowMs) {
|
|
1639
|
+
const previousById = new Map(state.flights.map((flight) => [flight.flight_id, flight]));
|
|
1640
|
+
const previousStatusById = new Map(state.flights.map((flight) => [flight.flight_id, flight.status]));
|
|
1641
|
+
const flights = incoming.map((flight) => {
|
|
1642
|
+
const previous = previousById.get(flight.flight_id);
|
|
1643
|
+
const carry = flight.recovery === void 0 && previous?.recovery !== void 0 && previous.status === flight.status && !isTerminal(flight.status);
|
|
1644
|
+
return carry ? { ...flight, recovery: previous.recovery } : flight;
|
|
1645
|
+
});
|
|
1646
|
+
const flipStartedAtMs = new Map(state.flipStartedAtMs);
|
|
1647
|
+
for (const flight of flights) {
|
|
1648
|
+
const previousStatus = previousStatusById.get(flight.flight_id);
|
|
1649
|
+
if (previousStatus !== void 0 && previousStatus !== flight.status) flipStartedAtMs.set(flight.flight_id, nowMs);
|
|
1650
|
+
}
|
|
1651
|
+
const merged = { ...state, flights, flipStartedAtMs, lastUpdatedMs: nowMs, lastError: null };
|
|
1652
|
+
if (flights.some((flight) => flight.flight_id === state.selectedFlightId)) return merged;
|
|
1653
|
+
const vanished = state.view !== "board" ? state.flights.find((flight) => flight.flight_id === state.selectedFlightId) : void 0;
|
|
1654
|
+
return {
|
|
1655
|
+
...merged,
|
|
1656
|
+
selectedFlightId: flights[0]?.flight_id ?? null,
|
|
1657
|
+
view: vanished ? "board" : state.view,
|
|
1658
|
+
notice: vanished ? `${vanished.name} is no longer listed` : state.notice
|
|
1659
|
+
};
|
|
1660
|
+
}
|
|
1661
|
+
function capLineLength(content) {
|
|
1662
|
+
if (content.length <= MAX_LOG_LINE_LENGTH) return content;
|
|
1663
|
+
const capped = content.slice(0, MAX_LOG_LINE_LENGTH);
|
|
1664
|
+
return /[\ud800-\udbff]$/.test(capped) ? capped.slice(0, -1) : capped;
|
|
1665
|
+
}
|
|
1666
|
+
function resolveCarriageReturns(text2) {
|
|
1667
|
+
const body = text2.replace(/\r+$/, "");
|
|
1668
|
+
const pendingReturn = body.length !== text2.length;
|
|
1669
|
+
return { content: body.slice(body.lastIndexOf("\r") + 1), pendingReturn };
|
|
1670
|
+
}
|
|
1671
|
+
function splitLogChunks(existing, tail, chunks) {
|
|
1672
|
+
const lines = [...existing];
|
|
1673
|
+
let currentTail = tail;
|
|
1674
|
+
for (const chunk of chunks) {
|
|
1675
|
+
if (chunk.content === "") continue;
|
|
1676
|
+
const pieces = chunk.content.split("\n");
|
|
1677
|
+
pieces.forEach((piece, index) => {
|
|
1678
|
+
const isLastPiece = index === pieces.length - 1;
|
|
1679
|
+
if (isLastPiece && piece === "") {
|
|
1680
|
+
currentTail = "closed";
|
|
1681
|
+
return;
|
|
1682
|
+
}
|
|
1683
|
+
const previous = lines[lines.length - 1];
|
|
1684
|
+
const continues = index === 0 && currentTail !== "closed" && previous?.stream === chunk.stream;
|
|
1685
|
+
const prefix = continues ? previous.content + (currentTail === "carriage-return" ? "\r" : "") : "";
|
|
1686
|
+
const { content, pendingReturn } = resolveCarriageReturns(prefix + piece);
|
|
1687
|
+
const line = { stream: chunk.stream, content: capLineLength(content) };
|
|
1688
|
+
if (continues) lines[lines.length - 1] = line;
|
|
1689
|
+
else lines.push(line);
|
|
1690
|
+
currentTail = !isLastPiece ? "closed" : pendingReturn ? "carriage-return" : "open";
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
return { lines, tail: currentTail };
|
|
1694
|
+
}
|
|
1695
|
+
function appendLogs(state, flightId, chunks, cursor, complete, attemptSeq) {
|
|
1696
|
+
const existing = state.logs?.flightId === flightId ? state.logs : null;
|
|
1697
|
+
let priorLines = existing?.lines ?? [];
|
|
1698
|
+
let priorTail = existing?.tail ?? "closed";
|
|
1699
|
+
const hadAttempt = typeof existing?.attemptSeq === "number" || existing?.attemptSeq === null && existing.lines.length > 0;
|
|
1700
|
+
const newAttempt = hadAttempt && typeof attemptSeq === "number" && attemptSeq !== existing.attemptSeq;
|
|
1701
|
+
if (newAttempt) {
|
|
1702
|
+
priorLines = [...priorLines, { stream: "meta", content: `\u2500\u2500 dispatch attempt ${attemptSeq}: its output starts here \u2500\u2500` }];
|
|
1703
|
+
priorTail = "closed";
|
|
1704
|
+
}
|
|
1705
|
+
const split = splitLogChunks(priorLines, priorTail, chunks);
|
|
1706
|
+
const overflow = Math.max(0, split.lines.length - MAX_LOG_LINES);
|
|
1707
|
+
return {
|
|
1708
|
+
...state,
|
|
1709
|
+
logs: {
|
|
1710
|
+
flightId,
|
|
1711
|
+
lines: overflow ? split.lines.slice(overflow) : split.lines,
|
|
1712
|
+
tail: split.tail,
|
|
1713
|
+
attemptSeq: attemptSeq !== void 0 ? attemptSeq : existing?.attemptSeq,
|
|
1714
|
+
cursor,
|
|
1715
|
+
complete,
|
|
1716
|
+
droppedLineCount: (existing?.droppedLineCount ?? 0) + overflow
|
|
1717
|
+
},
|
|
1718
|
+
// Keep a paused view anchored on the same text when old lines are trimmed.
|
|
1719
|
+
logTopLine: Math.max(0, state.logTopLine - overflow)
|
|
1720
|
+
};
|
|
1721
|
+
}
|
|
1722
|
+
function restartLogs(state, flightId) {
|
|
1723
|
+
const marker = { stream: "meta", content: "\u2500\u2500 log position reset; output below restarts from the beginning \u2500\u2500" };
|
|
1724
|
+
const withMarker = appendLogs(state, flightId, [], "", false);
|
|
1725
|
+
return {
|
|
1726
|
+
...withMarker,
|
|
1727
|
+
logs: { ...withMarker.logs, lines: [...withMarker.logs.lines, marker], tail: "closed", cursor: void 0 }
|
|
1728
|
+
};
|
|
1729
|
+
}
|
|
1730
|
+
function moveSelection(state, delta) {
|
|
1731
|
+
if (state.flights.length === 0) return state;
|
|
1732
|
+
const index = Math.min(state.flights.length - 1, Math.max(0, selectedIndex(state) + delta));
|
|
1733
|
+
return { ...state, selectedFlightId: state.flights[index].flight_id };
|
|
1734
|
+
}
|
|
1735
|
+
function scrollLogs(state, delta, viewportRows) {
|
|
1736
|
+
const totalLines = state.logs?.lines.length ?? 0;
|
|
1737
|
+
if (totalLines <= viewportRows) return state;
|
|
1738
|
+
const lastTop = Math.max(0, totalLines - viewportRows);
|
|
1739
|
+
const currentTop = state.following ? lastTop : state.logTopLine;
|
|
1740
|
+
const nextTop = Math.min(lastTop, Math.max(0, currentTop + delta));
|
|
1741
|
+
if (delta > 0 && nextTop === lastTop) return { ...state, following: true, logTopLine: nextTop };
|
|
1742
|
+
return { ...state, following: false, logTopLine: nextTop };
|
|
1743
|
+
}
|
|
1744
|
+
function openLogs(state) {
|
|
1745
|
+
if (!state.selectedFlightId) return state;
|
|
1746
|
+
const sameFlight = state.logs?.flightId === state.selectedFlightId;
|
|
1747
|
+
return {
|
|
1748
|
+
...state,
|
|
1749
|
+
view: "logs",
|
|
1750
|
+
logReturnView: state.view === "logs" ? state.logReturnView : state.view,
|
|
1751
|
+
logs: sameFlight ? state.logs : null,
|
|
1752
|
+
following: true,
|
|
1753
|
+
logTopLine: 0
|
|
1754
|
+
};
|
|
1755
|
+
}
|
|
1756
|
+
function reduceKey(current, key, context) {
|
|
1757
|
+
const state = current.notice ? { ...current, notice: null } : current;
|
|
1758
|
+
if (key === "quit" || key === "interrupt") return { ...state, quit: true };
|
|
1759
|
+
if (state.view === "board") {
|
|
1760
|
+
switch (key) {
|
|
1761
|
+
case "up":
|
|
1762
|
+
return moveSelection(state, -1);
|
|
1763
|
+
case "down":
|
|
1764
|
+
return moveSelection(state, 1);
|
|
1765
|
+
case "pageup":
|
|
1766
|
+
return moveSelection(state, -context.boardViewportRows);
|
|
1767
|
+
case "pagedown":
|
|
1768
|
+
return moveSelection(state, context.boardViewportRows);
|
|
1769
|
+
case "home":
|
|
1770
|
+
return moveSelection(state, -state.flights.length);
|
|
1771
|
+
case "end":
|
|
1772
|
+
return moveSelection(state, state.flights.length);
|
|
1773
|
+
case "enter":
|
|
1774
|
+
return state.selectedFlightId ? { ...state, view: "detail" } : state;
|
|
1775
|
+
case "logs":
|
|
1776
|
+
return openLogs(state);
|
|
1777
|
+
default:
|
|
1778
|
+
return state;
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
if (state.view === "detail") {
|
|
1782
|
+
switch (key) {
|
|
1783
|
+
case "escape":
|
|
1784
|
+
return { ...state, view: "board" };
|
|
1785
|
+
case "logs":
|
|
1786
|
+
case "enter":
|
|
1787
|
+
return openLogs(state);
|
|
1788
|
+
// Walk through flights without leaving the detail view.
|
|
1789
|
+
case "up":
|
|
1790
|
+
return moveSelection(state, -1);
|
|
1791
|
+
case "down":
|
|
1792
|
+
return moveSelection(state, 1);
|
|
1793
|
+
default:
|
|
1794
|
+
return state;
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
switch (key) {
|
|
1798
|
+
case "escape":
|
|
1799
|
+
return { ...state, view: state.logReturnView };
|
|
1800
|
+
case "up":
|
|
1801
|
+
return scrollLogs(state, -1, context.logViewportRows);
|
|
1802
|
+
case "down":
|
|
1803
|
+
return scrollLogs(state, 1, context.logViewportRows);
|
|
1804
|
+
case "pageup":
|
|
1805
|
+
return scrollLogs(state, -context.logViewportRows, context.logViewportRows);
|
|
1806
|
+
case "pagedown":
|
|
1807
|
+
return scrollLogs(state, context.logViewportRows, context.logViewportRows);
|
|
1808
|
+
case "home":
|
|
1809
|
+
return { ...state, following: false, logTopLine: 0 };
|
|
1810
|
+
case "end":
|
|
1811
|
+
return { ...state, following: true };
|
|
1812
|
+
case "follow": {
|
|
1813
|
+
if (!state.following) return { ...state, following: true };
|
|
1814
|
+
const totalLines = state.logs?.lines.length ?? 0;
|
|
1815
|
+
return { ...state, following: false, logTopLine: Math.max(0, totalLines - context.logViewportRows) };
|
|
1816
|
+
}
|
|
1817
|
+
default:
|
|
1818
|
+
return state;
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
|
|
1822
|
+
// src/atc/view.ts
|
|
1823
|
+
var MIN_WIDTH = 60;
|
|
1824
|
+
var MIN_HEIGHT = 12;
|
|
1825
|
+
var FLIP_DURATION_MS = 700;
|
|
1826
|
+
var FRAME_MS = 100;
|
|
1827
|
+
var BOARD_CHROME_ROWS = 6;
|
|
1828
|
+
var LOG_CHROME_ROWS = 3;
|
|
1829
|
+
var SPINNERS = {
|
|
1830
|
+
dispatching: ["\u25D0", "\u25D3", "\u25D1", "\u25D2"],
|
|
1831
|
+
running: ["\u280B", "\u2819", "\u2839", "\u2838", "\u283C", "\u2834", "\u2826", "\u2827", "\u2807", "\u280F"],
|
|
1832
|
+
finalizing: ["\u2801", "\u2802", "\u2804", "\u2840", "\u2880", "\u2820", "\u2810", "\u2808"]
|
|
1833
|
+
};
|
|
1834
|
+
var STATUS_STYLE = {
|
|
1835
|
+
queued: { glyph: "\xB7", style: { color: "gray" } },
|
|
1836
|
+
dispatching: { glyph: "\u25D0", style: { color: "yellow" } },
|
|
1837
|
+
running: { glyph: "\u25CF", style: { color: "cyan", bold: true } },
|
|
1838
|
+
finalizing: { glyph: "\u25C6", style: { color: "blue" } },
|
|
1839
|
+
completed: { glyph: "\u2713", style: { color: "green" } },
|
|
1840
|
+
failed: { glyph: "\u2717", style: { color: "red", bold: true } },
|
|
1841
|
+
cancelled: { glyph: "\u2298", style: { color: "gray" } }
|
|
1842
|
+
};
|
|
1843
|
+
var UNKNOWN_STATUS = { glyph: "?", style: { color: "magenta" } };
|
|
1844
|
+
var SUMMARY_ORDER = ["completed", "failed", "cancelled", "finalizing", "running", "dispatching", "queued"];
|
|
1845
|
+
function statusStyle(status2) {
|
|
1846
|
+
return (STATUS_STYLE[status2] ?? UNKNOWN_STATUS).style;
|
|
1847
|
+
}
|
|
1848
|
+
function statusGlyph(status2, nowMs) {
|
|
1849
|
+
const frames = SPINNERS[status2];
|
|
1850
|
+
if (frames) return frames[Math.floor(nowMs / FRAME_MS) % frames.length];
|
|
1851
|
+
return (STATUS_STYLE[status2] ?? UNKNOWN_STATUS).glyph;
|
|
1852
|
+
}
|
|
1853
|
+
var FLAP_ALPHABET = "abcdefghijklmnopqrstuvwxyz";
|
|
1854
|
+
function splitFlap(target, width, flipStartedAtMs, nowMs) {
|
|
1855
|
+
const padded = fixed(target, width);
|
|
1856
|
+
if (flipStartedAtMs === void 0) return padded;
|
|
1857
|
+
const progress = (nowMs - flipStartedAtMs) / FLIP_DURATION_MS;
|
|
1858
|
+
if (progress >= 1 || progress < 0) return padded;
|
|
1859
|
+
const characters = [...padded];
|
|
1860
|
+
const settledCount = Math.floor(progress * characters.length);
|
|
1861
|
+
const frame = Math.floor(nowMs / 50);
|
|
1862
|
+
return characters.map((character, position) => {
|
|
1863
|
+
if (position < settledCount) return character;
|
|
1864
|
+
const scramble = (frame * 7 + position * 13 + (frame >> 2) * position) % FLAP_ALPHABET.length;
|
|
1865
|
+
return FLAP_ALPHABET[scramble];
|
|
1866
|
+
}).join("");
|
|
1867
|
+
}
|
|
1868
|
+
function budgetGauge(flight, nowMs, width) {
|
|
1869
|
+
const elapsed = elapsedSeconds(flight, nowMs);
|
|
1870
|
+
if (elapsed === null || !flight.max_duration_s) return [seg(fixed("", width))];
|
|
1871
|
+
const fraction = Math.min(1, elapsed / flight.max_duration_s);
|
|
1872
|
+
const filled = Math.round(fraction * width);
|
|
1873
|
+
const color = fraction >= 0.9 ? "red" : fraction >= 0.7 ? "yellow" : "green";
|
|
1874
|
+
const style = isTerminal(flight.status) ? { color: "gray" } : { color };
|
|
1875
|
+
return [seg("\u2588".repeat(filled), style), seg("\u2591".repeat(width - filled), { color: "gray", dim: true })];
|
|
1876
|
+
}
|
|
1877
|
+
function clockTime(nowMs) {
|
|
1878
|
+
if (!Number.isFinite(nowMs)) return "\u2014";
|
|
1879
|
+
const date = new Date(nowMs);
|
|
1880
|
+
const pad = (value) => String(value).padStart(2, "0");
|
|
1881
|
+
return `${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
|
|
1882
|
+
}
|
|
1883
|
+
function timestamp(iso) {
|
|
1884
|
+
return iso ? clockTime(Date.parse(iso)) : "\u2014";
|
|
1885
|
+
}
|
|
1886
|
+
function titleBar(left, context) {
|
|
1887
|
+
const right = ` ${clockTime(context.nowMs)} `;
|
|
1888
|
+
const leftText = fixed(` ${left} `, Math.max(1, context.width - textWidth(right) - 1)).trimEnd() + " ";
|
|
1889
|
+
const gap = Math.max(1, context.width - textWidth(leftText) - textWidth(right));
|
|
1890
|
+
const barStyle = { color: "cyan", bold: true, inverse: true };
|
|
1891
|
+
return [seg(leftText, barStyle), seg(" ".repeat(gap), barStyle), seg(right, barStyle)];
|
|
1892
|
+
}
|
|
1893
|
+
function rule(width) {
|
|
1894
|
+
return [seg("\u2500".repeat(width), { color: "gray", dim: true })];
|
|
1895
|
+
}
|
|
1896
|
+
function keyHints(hints) {
|
|
1897
|
+
const segments = [seg(" ")];
|
|
1898
|
+
for (const [key, action] of hints) {
|
|
1899
|
+
segments.push(seg(key, { color: "cyan", bold: true }), seg(` ${action} `, { color: "gray" }));
|
|
1900
|
+
}
|
|
1901
|
+
return segments;
|
|
1902
|
+
}
|
|
1903
|
+
function footer(state, context, hints) {
|
|
1904
|
+
const error = state.lastError ?? (state.view === "board" ? null : state.selectionError);
|
|
1905
|
+
if (error) return [seg(fixed(` \u26A0 ${error}`, context.width), { color: "red", bold: true })];
|
|
1906
|
+
let rightText;
|
|
1907
|
+
let rightStyle;
|
|
1908
|
+
if (state.notice) {
|
|
1909
|
+
rightText = ` ${state.notice} `;
|
|
1910
|
+
rightStyle = { color: "yellow", bold: true };
|
|
1911
|
+
} else if (state.lastUpdatedMs === null) {
|
|
1912
|
+
rightText = " connecting\u2026 ";
|
|
1913
|
+
rightStyle = { color: "yellow" };
|
|
1914
|
+
} else {
|
|
1915
|
+
const ageSeconds = Math.max(0, Math.floor((context.nowMs - state.lastUpdatedMs) / 1e3));
|
|
1916
|
+
rightText = ` updated ${ageSeconds}s ago `;
|
|
1917
|
+
rightStyle = { color: "gray" };
|
|
1918
|
+
}
|
|
1919
|
+
rightText = fixed(rightText, Math.min(textWidth(rightText), context.width));
|
|
1920
|
+
const hintWidth = context.width - textWidth(rightText);
|
|
1921
|
+
const left = clipLine(keyHints(hints), hintWidth);
|
|
1922
|
+
return [...left, seg(" ".repeat(Math.max(0, hintWidth - visibleLength(left)))), seg(rightText, rightStyle)];
|
|
1923
|
+
}
|
|
1924
|
+
function boardColumns(width) {
|
|
1925
|
+
const showGpu = width >= 72;
|
|
1926
|
+
const showCredits = width >= 80;
|
|
1927
|
+
const showBudget = width >= 96;
|
|
1928
|
+
const fixedColumns = [
|
|
1929
|
+
{ header: "", width: 2 },
|
|
1930
|
+
{ header: "FLIGHT", width: 13 },
|
|
1931
|
+
{ header: "NAME", width: 0 },
|
|
1932
|
+
...showGpu ? [{ header: "GPU", width: 10 }] : [],
|
|
1933
|
+
{ header: "STATUS", width: 15 },
|
|
1934
|
+
{ header: "ELAPSED", width: 9, align: "right" },
|
|
1935
|
+
...showBudget ? [{ header: " TIME BUDGET", width: 14 }] : [],
|
|
1936
|
+
...showCredits ? [{ header: "CREDITS", width: 8, align: "right" }] : []
|
|
1937
|
+
];
|
|
1938
|
+
const usedWidth = fixedColumns.reduce((total, column) => total + column.width + 1, 0);
|
|
1939
|
+
fixedColumns[2].width = Math.max(8, width - usedWidth - 1);
|
|
1940
|
+
return { columns: fixedColumns, showGpu, showBudget, showCredits };
|
|
1941
|
+
}
|
|
1942
|
+
function boardRow(flight, state, context, layout, isSelected) {
|
|
1943
|
+
const [cursorColumn, flightColumn, nameColumn] = layout.columns;
|
|
1944
|
+
const style = statusStyle(flight.status);
|
|
1945
|
+
const statusText = splitFlap(flight.status, 12, state.flipStartedAtMs.get(flight.flight_id), context.nowMs);
|
|
1946
|
+
const recovering = flight.recovery && flight.recovery.phase !== "finished";
|
|
1947
|
+
const line = [
|
|
1948
|
+
seg(fixed(isSelected ? "\u25B8" : "", cursorColumn.width + 1), { color: "cyan", bold: true }),
|
|
1949
|
+
seg(fixed(flight.flight_id, flightColumn.width) + " ", { color: "gray" }),
|
|
1950
|
+
seg(fixed(flight.name, nameColumn.width) + " ")
|
|
1951
|
+
];
|
|
1952
|
+
if (layout.showGpu) line.push(seg(fixed(`${flight.gpu_type ?? "any"} x${flight.gpu_count}`, 10) + " ", { color: "white" }));
|
|
1953
|
+
line.push(
|
|
1954
|
+
seg(`${statusGlyph(flight.status, context.nowMs)} `, style),
|
|
1955
|
+
seg(statusText + (recovering ? "\u27F3" : " ") + " ", recovering ? { ...style, color: "magenta" } : style),
|
|
1956
|
+
seg(fixed(formatDuration(elapsedSeconds(flight, context.nowMs)), 9, "right") + " ")
|
|
1957
|
+
);
|
|
1958
|
+
if (layout.showBudget) line.push(seg(" "), ...budgetGauge(flight, context.nowMs, 12), seg(" "));
|
|
1959
|
+
if (layout.showCredits) {
|
|
1960
|
+
const credits = flight.charge_credits;
|
|
1961
|
+
line.push(seg(fixed(credits == null ? "\u2014" : String(credits), 8, "right"), { color: "yellow" }));
|
|
1962
|
+
}
|
|
1963
|
+
if (!isSelected) return line;
|
|
1964
|
+
return highlight([...line, seg(" ".repeat(context.width))], { inverse: true });
|
|
1965
|
+
}
|
|
1966
|
+
function statusCounts(flights) {
|
|
1967
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1968
|
+
for (const flight of flights) counts.set(flight.status, (counts.get(flight.status) ?? 0) + 1);
|
|
1969
|
+
return counts;
|
|
1970
|
+
}
|
|
1971
|
+
function orderedStatuses(counts) {
|
|
1972
|
+
const unknown = [...counts.keys()].filter((status2) => !SUMMARY_ORDER.includes(status2)).sort();
|
|
1973
|
+
return [...SUMMARY_ORDER, ...unknown].filter((status2) => (counts.get(status2) ?? 0) > 0);
|
|
1974
|
+
}
|
|
1975
|
+
function progressBar(flights, width) {
|
|
1976
|
+
if (flights.length === 0 || width <= 0) return [seg("\u2591".repeat(Math.max(0, width)), { color: "gray", dim: true })];
|
|
1977
|
+
const counts = statusCounts(flights);
|
|
1978
|
+
const statuses = orderedStatuses(counts);
|
|
1979
|
+
const exact = statuses.map((status2) => (counts.get(status2) ?? 0) / flights.length * width);
|
|
1980
|
+
const widths = exact.map(Math.floor);
|
|
1981
|
+
let leftover = width - widths.reduce((total, value) => total + value, 0);
|
|
1982
|
+
const byRemainder = exact.map((value, index) => ({ index, remainder: value - Math.floor(value) })).sort((left, right) => right.remainder - left.remainder);
|
|
1983
|
+
for (const { index } of byRemainder) {
|
|
1984
|
+
if (leftover <= 0) break;
|
|
1985
|
+
widths[index] += 1;
|
|
1986
|
+
leftover -= 1;
|
|
1987
|
+
}
|
|
1988
|
+
const fillFor = (status2) => isTerminal(status2) ? "\u2588" : status2 === "queued" ? "\u2591" : "\u2593";
|
|
1989
|
+
return statuses.map((status2, index) => seg(fillFor(status2).repeat(widths[index]), statusStyle(status2)));
|
|
1990
|
+
}
|
|
1991
|
+
function summaryCounts(flights) {
|
|
1992
|
+
const counts = statusCounts(flights);
|
|
1993
|
+
const line = [seg(" ")];
|
|
1994
|
+
orderedStatuses(counts).forEach((status2, index) => {
|
|
1995
|
+
if (index > 0) line.push(seg(" \xB7 ", { color: "gray" }));
|
|
1996
|
+
line.push(seg(`${counts.get(status2)} ${status2}`, statusStyle(status2)));
|
|
1997
|
+
});
|
|
1998
|
+
if (line.length === 1) line.push(seg("no flights yet", { color: "gray" }));
|
|
1999
|
+
return line;
|
|
2000
|
+
}
|
|
2001
|
+
function boardViewportRows(height) {
|
|
2002
|
+
return Math.max(1, height - BOARD_CHROME_ROWS);
|
|
2003
|
+
}
|
|
2004
|
+
function renderBoard(state, context) {
|
|
2005
|
+
const layout = boardColumns(context.width);
|
|
2006
|
+
const bodyRows = boardViewportRows(context.height);
|
|
2007
|
+
const lines = [titleBar(`FLIGHTDECK ATC \u2500\u2500\u2500 ${context.title} \u2500\u2500\u2500 ${state.flights.length} flight${state.flights.length === 1 ? "" : "s"}`, context)];
|
|
2008
|
+
lines.push([seg(layout.columns.map((column) => fixed(column.header, column.width, column.align)).join(" "), { color: "gray", bold: true })]);
|
|
2009
|
+
const selected = selectedIndex(state);
|
|
2010
|
+
const top = Math.min(Math.max(0, selected - Math.floor(bodyRows / 2)), Math.max(0, state.flights.length - bodyRows));
|
|
2011
|
+
const visible = state.flights.slice(top, top + bodyRows);
|
|
2012
|
+
for (const [offset, flight] of visible.entries()) {
|
|
2013
|
+
lines.push(boardRow(flight, state, context, layout, top + offset === selected));
|
|
2014
|
+
}
|
|
2015
|
+
if (state.flights.length === 0) lines.push([seg(" waiting for flights\u2026", { color: "gray" })]);
|
|
2016
|
+
while (lines.length < bodyRows + 2) lines.push([]);
|
|
2017
|
+
const scrollNote = state.flights.length > bodyRows ? ` ${top + 1}\u2013${top + visible.length} of ${state.flights.length} ` : "";
|
|
2018
|
+
lines.push([...rule(context.width - textWidth(scrollNote)), seg(scrollNote, { color: "gray" })]);
|
|
2019
|
+
lines.push([seg(" "), ...progressBar(state.flights, context.width - 2)]);
|
|
2020
|
+
lines.push(summaryCounts(state.flights));
|
|
2021
|
+
lines.push(footer(state, context, [["\u2191\u2193", "select"], ["\u23CE", "detail"], ["l", "logs"], ["r", "refresh"], ["q", "quit"]]));
|
|
2022
|
+
return lines;
|
|
2023
|
+
}
|
|
2024
|
+
function field(label, value) {
|
|
2025
|
+
const valueSegments = typeof value === "string" ? [seg(value)] : value;
|
|
2026
|
+
return [seg(` ${fixed(label, 11)}`, { color: "gray" }), ...valueSegments];
|
|
2027
|
+
}
|
|
2028
|
+
function renderDetail(state, context) {
|
|
2029
|
+
const flight = selectedFlight(state);
|
|
2030
|
+
if (!flight) return renderBoard({ ...state, view: "board" }, context);
|
|
2031
|
+
const style = statusStyle(flight.status);
|
|
2032
|
+
const elapsed = elapsedSeconds(flight, context.nowMs);
|
|
2033
|
+
const lines = [titleBar(`FLIGHT ${selectedIndex(state) + 1}/${state.flights.length} \u2500\u2500\u2500 ${flight.name}`, context), []];
|
|
2034
|
+
const statusSegments = [
|
|
2035
|
+
seg(`${statusGlyph(flight.status, context.nowMs)} `, style),
|
|
2036
|
+
seg(splitFlap(flight.status, 12, state.flipStartedAtMs.get(flight.flight_id), context.nowMs).trimEnd(), { ...style, bold: true })
|
|
2037
|
+
];
|
|
2038
|
+
if (flight.recovery && flight.recovery.phase !== "finished") {
|
|
2039
|
+
statusSegments.push(seg(` \u27F3 GPU recovery: ${flight.recovery.phase}`, { color: "magenta" }));
|
|
2040
|
+
}
|
|
2041
|
+
lines.push(field("Flight", [seg(flight.flight_id, { color: "white", bold: true })]));
|
|
2042
|
+
lines.push(field("Name", flight.name));
|
|
2043
|
+
lines.push(field("Status", statusSegments));
|
|
2044
|
+
lines.push(field("GPU", `${flight.gpu_type ?? "any"} x${flight.gpu_count}${flight.provider ? ` via ${flight.provider}` : ""}`));
|
|
2045
|
+
lines.push(field("Timeline", [
|
|
2046
|
+
seg(`queued ${timestamp(flight.queued_at)}`),
|
|
2047
|
+
seg(" \u2192 ", { color: "gray" }),
|
|
2048
|
+
seg(`dispatched ${timestamp(flight.dispatched_at)}`),
|
|
2049
|
+
seg(" \u2192 ", { color: "gray" }),
|
|
2050
|
+
seg(`started ${timestamp(flight.started_at)}`),
|
|
2051
|
+
seg(" \u2192 ", { color: "gray" }),
|
|
2052
|
+
seg(`finished ${timestamp(flight.completed_at)}`)
|
|
2053
|
+
]));
|
|
2054
|
+
const gaugeWidth = Math.max(10, Math.min(40, context.width - 40));
|
|
2055
|
+
lines.push(field("Elapsed", [
|
|
2056
|
+
seg(fixed(formatDuration(elapsed), 10)),
|
|
2057
|
+
...flight.max_duration_s ? [
|
|
2058
|
+
...budgetGauge(flight, context.nowMs, gaugeWidth),
|
|
2059
|
+
seg(` of ${formatDuration(flight.max_duration_s)} max`, { color: "gray" })
|
|
2060
|
+
] : []
|
|
2061
|
+
]));
|
|
2062
|
+
if (flight.exit_code != null) lines.push(field("Exit code", [seg(String(flight.exit_code), { color: flight.exit_code === 0 ? "green" : "red", bold: true })]));
|
|
2063
|
+
if (flight.error_message) lines.push(field("Error", [seg(flight.error_message, { color: "red" })]));
|
|
2064
|
+
if (flight.charge_credits != null) lines.push(field("Credits", [seg(String(flight.charge_credits), { color: "yellow" })]));
|
|
2065
|
+
lines.push([]);
|
|
2066
|
+
lines.push([seg(" EVENTS ", { color: "cyan", bold: true }), ...rule(Math.max(0, context.width - 11))]);
|
|
2067
|
+
const eventRows = context.height - lines.length - 1;
|
|
2068
|
+
const events = state.events?.flightId === flight.flight_id ? state.events.items : null;
|
|
2069
|
+
if (events === null) {
|
|
2070
|
+
lines.push([seg(" loading\u2026", { color: "gray" })]);
|
|
2071
|
+
} else if (events.length === 0) {
|
|
2072
|
+
lines.push([seg(" no events recorded", { color: "gray" })]);
|
|
2073
|
+
} else {
|
|
2074
|
+
for (const event of events.slice(-Math.max(1, eventRows))) {
|
|
2075
|
+
lines.push([
|
|
2076
|
+
seg(` ${timestamp(event.created_at)} `, { color: "gray" }),
|
|
2077
|
+
seg(fixed(event.type, 26) + " ", { color: event.type.includes("fault") || event.type.includes("limit") ? "red" : "blue" }),
|
|
2078
|
+
seg(event.message ?? "")
|
|
2079
|
+
]);
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
while (lines.length < context.height - 1) lines.push([]);
|
|
2083
|
+
lines.length = context.height - 1;
|
|
2084
|
+
lines.push(footer(state, context, [["\u2191\u2193", "prev/next flight"], ["l", "logs"], ["r", "refresh"], ["esc", "board"], ["q", "quit"]]));
|
|
2085
|
+
return lines;
|
|
2086
|
+
}
|
|
2087
|
+
function logViewportRows(height) {
|
|
2088
|
+
return Math.max(1, height - LOG_CHROME_ROWS);
|
|
2089
|
+
}
|
|
2090
|
+
function renderLogs(state, context) {
|
|
2091
|
+
const flight = selectedFlight(state);
|
|
2092
|
+
const bodyRows = logViewportRows(context.height);
|
|
2093
|
+
const buffer = state.logs?.flightId === state.selectedFlightId ? state.logs : null;
|
|
2094
|
+
const allLines = buffer?.lines ?? [];
|
|
2095
|
+
const lastTop = Math.max(0, allLines.length - bodyRows);
|
|
2096
|
+
const top = state.following ? lastTop : Math.min(state.logTopLine, lastTop);
|
|
2097
|
+
const mode = [buffer?.complete ? "\u25A0 complete" : null, state.following ? buffer?.complete ? null : "\u25CF following" : "\u23F8 paused"].filter(Boolean).join(" \xB7 ");
|
|
2098
|
+
const lines = [titleBar(`LOGS \u2500\u2500\u2500 ${flight?.name ?? "?"} \u2500\u2500\u2500 ${flight?.flight_id ?? ""} \u2500\u2500\u2500 ${mode}`, context)];
|
|
2099
|
+
if (!buffer) {
|
|
2100
|
+
lines.push([seg(" loading\u2026", { color: "gray" })]);
|
|
2101
|
+
} else if (allLines.length === 0) {
|
|
2102
|
+
lines.push([seg(buffer.complete ? " (no output)" : " waiting for output\u2026", { color: "gray" })]);
|
|
2103
|
+
} else {
|
|
2104
|
+
for (const logLine of allLines.slice(top, top + bodyRows)) {
|
|
2105
|
+
const lineStyle = logLine.stream === "stderr" ? { color: "red" } : logLine.stream === "meta" ? { color: "cyan", dim: true } : void 0;
|
|
2106
|
+
lines.push([seg(logLine.content.replace(/\t/g, " "), lineStyle)]);
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
while (lines.length < bodyRows + 1) lines.push([]);
|
|
2110
|
+
const dropped = buffer?.droppedLineCount ? ` (+${buffer.droppedLineCount} earlier lines dropped)` : "";
|
|
2111
|
+
const position = allLines.length ? ` lines ${top + 1}\u2013${Math.min(allLines.length, top + bodyRows)} of ${allLines.length}${dropped} ` : "";
|
|
2112
|
+
lines.push([...rule(context.width - textWidth(position)), seg(position, { color: "gray" })]);
|
|
2113
|
+
lines.push(footer(state, context, [["\u2191\u2193", "scroll"], ["PgUp/PgDn", "page"], ["f", "follow"], ["esc", "back"], ["q", "quit"]]));
|
|
2114
|
+
return lines;
|
|
2115
|
+
}
|
|
2116
|
+
function render(state, context) {
|
|
2117
|
+
if (context.width < MIN_WIDTH || context.height < MIN_HEIGHT) {
|
|
2118
|
+
const message = `Terminal too small: ${context.width}\xD7${context.height}, need ${MIN_WIDTH}\xD7${MIN_HEIGHT}`;
|
|
2119
|
+
return [[seg(message, { color: "yellow" })]];
|
|
2120
|
+
}
|
|
2121
|
+
switch (state.view) {
|
|
2122
|
+
case "detail":
|
|
2123
|
+
return renderDetail(state, context);
|
|
2124
|
+
case "logs":
|
|
2125
|
+
return renderLogs(state, context);
|
|
2126
|
+
default:
|
|
2127
|
+
return renderBoard(state, context);
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
|
|
2131
|
+
// src/atc/app.ts
|
|
2132
|
+
var ANIMATION_INTERVAL_MS = 100;
|
|
2133
|
+
var MAX_LOG_PAGES_PER_REFRESH = 10;
|
|
2134
|
+
var MAX_BACKOFF_MS = 6e4;
|
|
2135
|
+
var SETTLED_REFRESH_MS = 15e3;
|
|
2136
|
+
var SIGNAL_EXIT_CODES = {
|
|
2137
|
+
SIGHUP: 129,
|
|
2138
|
+
SIGINT: 130,
|
|
2139
|
+
SIGQUIT: 131,
|
|
2140
|
+
SIGTERM: 143
|
|
2141
|
+
};
|
|
2142
|
+
var HANDLED_SIGNALS = Object.keys(SIGNAL_EXIT_CODES);
|
|
2143
|
+
function describeFailure(error) {
|
|
2144
|
+
return error instanceof CliError ? error.message : "update failed; retrying";
|
|
2145
|
+
}
|
|
2146
|
+
async function runAtc(source, terminal) {
|
|
2147
|
+
const now = terminal.now ?? Date.now;
|
|
2148
|
+
const screen = new Screen(terminal.output, terminal.colorEnabled);
|
|
2149
|
+
const keyReader = new KeyReader();
|
|
2150
|
+
let state = source.initialFlightId ? { ...initialState(), view: "detail", selectedFlightId: source.initialFlightId } : initialState();
|
|
2151
|
+
let stopped = false;
|
|
2152
|
+
let resolveRun = () => {
|
|
2153
|
+
};
|
|
2154
|
+
let rejectRun = () => {
|
|
2155
|
+
};
|
|
2156
|
+
const finished = new Promise((resolve4, reject) => {
|
|
2157
|
+
resolveRun = resolve4;
|
|
2158
|
+
rejectRun = reject;
|
|
2159
|
+
});
|
|
2160
|
+
const requests = new AbortController();
|
|
2161
|
+
let pollTimer;
|
|
2162
|
+
let selectionTimer;
|
|
2163
|
+
let animationTimer;
|
|
2164
|
+
let escapeTimer;
|
|
2165
|
+
let pollInFlight = false;
|
|
2166
|
+
let detailFetchInFlight = false;
|
|
2167
|
+
let consecutivePollFailures = 0;
|
|
2168
|
+
let consecutiveSelectionFailures = 0;
|
|
2169
|
+
const redraw = () => {
|
|
2170
|
+
if (stopped) return;
|
|
2171
|
+
try {
|
|
2172
|
+
screen.draw(render(state, { width: screen.width, height: screen.height, nowMs: now(), title: source.title }));
|
|
2173
|
+
} catch (error) {
|
|
2174
|
+
fail(error);
|
|
2175
|
+
}
|
|
2176
|
+
};
|
|
2177
|
+
const endIfFatal = (error) => {
|
|
2178
|
+
if (error instanceof AtcFatalError) fail(error);
|
|
2179
|
+
return stopped;
|
|
2180
|
+
};
|
|
2181
|
+
const refreshSelection = async () => {
|
|
2182
|
+
const flightId = state.selectedFlightId;
|
|
2183
|
+
const requestedView = state.view;
|
|
2184
|
+
if (stopped || detailFetchInFlight || !flightId || requestedView === "board") return;
|
|
2185
|
+
detailFetchInFlight = true;
|
|
2186
|
+
try {
|
|
2187
|
+
if (requestedView === "detail") {
|
|
2188
|
+
const [items, fullRecord] = await Promise.all([
|
|
2189
|
+
source.listEvents(flightId, requests.signal),
|
|
2190
|
+
source.readFlight?.(flightId, requests.signal)
|
|
2191
|
+
]);
|
|
2192
|
+
if (stopped || state.selectedFlightId !== flightId) return;
|
|
2193
|
+
state = { ...state, events: { flightId, items } };
|
|
2194
|
+
if (fullRecord) state = mergeFlightRecord(state, fullRecord, now());
|
|
2195
|
+
} else {
|
|
2196
|
+
for (let pageCount = 0; pageCount < MAX_LOG_PAGES_PER_REFRESH && !stopped; pageCount += 1) {
|
|
2197
|
+
const buffer = state.logs?.flightId === flightId ? state.logs : null;
|
|
2198
|
+
const requestedCursor = buffer?.cursor;
|
|
2199
|
+
const page = await source.readLogs(flightId, requestedCursor, requests.signal);
|
|
2200
|
+
const current = state.logs?.flightId === flightId ? state.logs : null;
|
|
2201
|
+
if (stopped || state.selectedFlightId !== flightId || current?.cursor !== requestedCursor) return;
|
|
2202
|
+
if (page.restart) {
|
|
2203
|
+
state = restartLogs(state, flightId);
|
|
2204
|
+
continue;
|
|
2205
|
+
}
|
|
2206
|
+
state = appendLogs(state, flightId, page.lines, page.nextCursor, page.complete, page.attemptSeq);
|
|
2207
|
+
if (!page.hasMore) break;
|
|
2208
|
+
redraw();
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
consecutiveSelectionFailures = 0;
|
|
2212
|
+
state = { ...state, selectionError: null };
|
|
2213
|
+
} catch (error) {
|
|
2214
|
+
if (!endIfFatal(error)) {
|
|
2215
|
+
consecutiveSelectionFailures += 1;
|
|
2216
|
+
state = { ...state, selectionError: describeFailure(error) };
|
|
2217
|
+
}
|
|
2218
|
+
} finally {
|
|
2219
|
+
detailFetchInFlight = false;
|
|
2220
|
+
redraw();
|
|
2221
|
+
}
|
|
2222
|
+
if (!stopped && (state.selectedFlightId !== flightId || state.view !== requestedView)) spawn(refreshSelection);
|
|
2223
|
+
};
|
|
2224
|
+
const poll = async () => {
|
|
2225
|
+
if (stopped || pollInFlight) return;
|
|
2226
|
+
pollInFlight = true;
|
|
2227
|
+
let delayMs = source.pollIntervalMs;
|
|
2228
|
+
try {
|
|
2229
|
+
const flights = await source.listFlights(requests.signal);
|
|
2230
|
+
if (stopped) return;
|
|
2231
|
+
consecutivePollFailures = 0;
|
|
2232
|
+
state = applyFlights(state, flights, now());
|
|
2233
|
+
} catch (error) {
|
|
2234
|
+
consecutivePollFailures += 1;
|
|
2235
|
+
delayMs = Math.min(MAX_BACKOFF_MS, source.pollIntervalMs * 2 ** consecutivePollFailures);
|
|
2236
|
+
if (!endIfFatal(error)) {
|
|
2237
|
+
const retrying = consecutivePollFailures > 1 ? ` (retrying in ${Math.round(delayMs / 1e3)}s)` : "";
|
|
2238
|
+
state = { ...state, lastError: describeFailure(error) + retrying };
|
|
2239
|
+
}
|
|
2240
|
+
} finally {
|
|
2241
|
+
pollInFlight = false;
|
|
2242
|
+
}
|
|
2243
|
+
redraw();
|
|
2244
|
+
if (!stopped) pollTimer = setTimeout(() => spawn(poll), delayMs);
|
|
2245
|
+
};
|
|
2246
|
+
const selectionDelayMs = () => {
|
|
2247
|
+
const base = source.selectionPollIntervalMs ?? source.pollIntervalMs;
|
|
2248
|
+
const flight = selectedFlight(state);
|
|
2249
|
+
const cadence = flight && isTerminal(flight.status) ? Math.max(base, SETTLED_REFRESH_MS) : base;
|
|
2250
|
+
return Math.min(MAX_BACKOFF_MS, cadence * 2 ** consecutiveSelectionFailures);
|
|
2251
|
+
};
|
|
2252
|
+
const runSelection = async () => {
|
|
2253
|
+
await refreshSelection();
|
|
2254
|
+
scheduleSelection();
|
|
2255
|
+
};
|
|
2256
|
+
function scheduleSelection(delayMs = selectionDelayMs()) {
|
|
2257
|
+
clearTimeout(selectionTimer);
|
|
2258
|
+
if (!stopped) selectionTimer = setTimeout(() => spawn(runSelection), delayMs);
|
|
2259
|
+
}
|
|
2260
|
+
function spawn(task) {
|
|
2261
|
+
task().catch(fail);
|
|
2262
|
+
}
|
|
2263
|
+
const applyKeys = (keys) => {
|
|
2264
|
+
const before = state;
|
|
2265
|
+
const context = { boardViewportRows: boardViewportRows(screen.height), logViewportRows: logViewportRows(screen.height) };
|
|
2266
|
+
let refreshRequested = false;
|
|
2267
|
+
for (const key of keys) {
|
|
2268
|
+
if (key === "redraw") screen.invalidate();
|
|
2269
|
+
if (key === "refresh") refreshRequested = true;
|
|
2270
|
+
state = reduceKey(state, key, context);
|
|
2271
|
+
}
|
|
2272
|
+
if (state.quit) {
|
|
2273
|
+
stop();
|
|
2274
|
+
return;
|
|
2275
|
+
}
|
|
2276
|
+
if (refreshRequested) {
|
|
2277
|
+
consecutivePollFailures = 0;
|
|
2278
|
+
consecutiveSelectionFailures = 0;
|
|
2279
|
+
clearTimeout(pollTimer);
|
|
2280
|
+
spawn(poll);
|
|
2281
|
+
spawn(runSelection);
|
|
2282
|
+
}
|
|
2283
|
+
if (state.view !== before.view || state.selectedFlightId !== before.selectedFlightId) {
|
|
2284
|
+
consecutiveSelectionFailures = 0;
|
|
2285
|
+
state = { ...state, selectionError: null };
|
|
2286
|
+
spawn(runSelection);
|
|
2287
|
+
}
|
|
2288
|
+
redraw();
|
|
2289
|
+
};
|
|
2290
|
+
const onKeys = (chunk) => {
|
|
2291
|
+
clearTimeout(escapeTimer);
|
|
2292
|
+
applyKeys(keyReader.feed(chunk));
|
|
2293
|
+
const timeoutMs = keyReader.pendingTimeoutMs;
|
|
2294
|
+
if (!stopped && timeoutMs !== null) {
|
|
2295
|
+
escapeTimer = setTimeout(() => applyKeys(keyReader.flushPending()), timeoutMs);
|
|
2296
|
+
}
|
|
2297
|
+
};
|
|
2298
|
+
const onResize = () => {
|
|
2299
|
+
screen.invalidate();
|
|
2300
|
+
redraw();
|
|
2301
|
+
};
|
|
2302
|
+
const restoreTerminal = () => {
|
|
2303
|
+
terminal.input.off("data", onKeys);
|
|
2304
|
+
terminal.output.off?.("resize", onResize);
|
|
2305
|
+
try {
|
|
2306
|
+
terminal.input.setRawMode?.(false);
|
|
2307
|
+
} catch {
|
|
2308
|
+
}
|
|
2309
|
+
terminal.input.pause();
|
|
2310
|
+
screen.leave();
|
|
2311
|
+
};
|
|
2312
|
+
const onProcessExit = () => restoreTerminal();
|
|
2313
|
+
const onSignal = (signal) => {
|
|
2314
|
+
process.exitCode = SIGNAL_EXIT_CODES[signal] ?? 1;
|
|
2315
|
+
stop();
|
|
2316
|
+
};
|
|
2317
|
+
function teardown() {
|
|
2318
|
+
if (stopped) return false;
|
|
2319
|
+
stopped = true;
|
|
2320
|
+
requests.abort();
|
|
2321
|
+
clearTimeout(pollTimer);
|
|
2322
|
+
clearTimeout(selectionTimer);
|
|
2323
|
+
clearTimeout(escapeTimer);
|
|
2324
|
+
clearInterval(animationTimer);
|
|
2325
|
+
process.off("exit", onProcessExit);
|
|
2326
|
+
for (const signal of HANDLED_SIGNALS) process.off(signal, onSignal);
|
|
2327
|
+
restoreTerminal();
|
|
2328
|
+
return true;
|
|
2329
|
+
}
|
|
2330
|
+
function stop() {
|
|
2331
|
+
if (teardown()) resolveRun();
|
|
2332
|
+
}
|
|
2333
|
+
function fail(error) {
|
|
2334
|
+
if (teardown()) rejectRun(error);
|
|
2335
|
+
}
|
|
2336
|
+
process.on("exit", onProcessExit);
|
|
2337
|
+
for (const signal of HANDLED_SIGNALS) process.on(signal, onSignal);
|
|
2338
|
+
terminal.input.setRawMode?.(true);
|
|
2339
|
+
terminal.input.setEncoding("utf8");
|
|
2340
|
+
terminal.input.on("data", onKeys);
|
|
2341
|
+
terminal.input.resume();
|
|
2342
|
+
terminal.output.on?.("resize", onResize);
|
|
2343
|
+
animationTimer = setInterval(redraw, ANIMATION_INTERVAL_MS);
|
|
2344
|
+
screen.enter();
|
|
2345
|
+
redraw();
|
|
2346
|
+
if (!stopped) {
|
|
2347
|
+
spawn(poll);
|
|
2348
|
+
spawn(runSelection);
|
|
2349
|
+
}
|
|
2350
|
+
await finished;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
// src/atc/demo-source.ts
|
|
2354
|
+
var MOLECULES = [
|
|
2355
|
+
"water",
|
|
2356
|
+
"methane",
|
|
2357
|
+
"ammonia",
|
|
2358
|
+
"ethanol",
|
|
2359
|
+
"benzene",
|
|
2360
|
+
"formaldehyde",
|
|
2361
|
+
"acetone",
|
|
2362
|
+
"caffeine",
|
|
2363
|
+
"glycine",
|
|
2364
|
+
"urea",
|
|
2365
|
+
"pyridine",
|
|
2366
|
+
"phenol",
|
|
2367
|
+
"toluene",
|
|
2368
|
+
"aspirin",
|
|
2369
|
+
"naphthalene",
|
|
2370
|
+
"acetic-acid",
|
|
2371
|
+
"imidazole",
|
|
2372
|
+
"furan",
|
|
2373
|
+
"thiophene",
|
|
2374
|
+
"indole",
|
|
2375
|
+
"serotonin"
|
|
2376
|
+
];
|
|
2377
|
+
var CONCURRENT_GPUS = 8;
|
|
2378
|
+
var MAX_DURATION_S2 = 150;
|
|
2379
|
+
var FAILING_TASK = 13;
|
|
2380
|
+
var RECOVERING_TASK = 6;
|
|
2381
|
+
var CANCELLED_TASK = 20;
|
|
2382
|
+
var CANCELLED_AT_S = 35;
|
|
2383
|
+
function seededRandom(seed) {
|
|
2384
|
+
let state = seed >>> 0;
|
|
2385
|
+
return () => {
|
|
2386
|
+
state = state + 1831565813 >>> 0;
|
|
2387
|
+
let mixed = state;
|
|
2388
|
+
mixed = Math.imul(mixed ^ mixed >>> 15, mixed | 1);
|
|
2389
|
+
mixed ^= mixed + Math.imul(mixed ^ mixed >>> 7, mixed | 61);
|
|
2390
|
+
return ((mixed ^ mixed >>> 14) >>> 0) / 4294967296;
|
|
2391
|
+
};
|
|
2392
|
+
}
|
|
2393
|
+
function between(random, low, high) {
|
|
2394
|
+
return low + random() * (high - low);
|
|
2395
|
+
}
|
|
2396
|
+
function buildLogs(plan, random) {
|
|
2397
|
+
const logs2 = [];
|
|
2398
|
+
const at = (offsetS, content, stream = "stdout") => logs2.push({ atS: plan.startAtS + offsetS, stream, content });
|
|
2399
|
+
at(0.2, `[molsim] task ${plan.taskIndex} of 21 \xB7 SLURM_ARRAY_TASK_ID=${plan.taskIndex}
|
|
2400
|
+
`);
|
|
2401
|
+
at(0.5, `[molsim] loading geometry inputs/${plan.molecule}.xyz
|
|
2402
|
+
`);
|
|
2403
|
+
at(0.9, `[molsim] method B3LYP/def2-TZVP \xB7 device cuda:0 (${plan.gpuType})
|
|
2404
|
+
`);
|
|
2405
|
+
at(1.2, "[molsim] building integrals ");
|
|
2406
|
+
for (let percent = 10; percent <= 100; percent += 30) at(1.2 + percent / 60, `\r[molsim] building integrals ${percent}%`);
|
|
2407
|
+
at(3.2, "\n");
|
|
2408
|
+
const runSeconds = plan.finalizeAtS - plan.startAtS;
|
|
2409
|
+
const iterations = Math.max(6, Math.floor((runSeconds - 4) / 1.6));
|
|
2410
|
+
let energy = -40 - random() * 400;
|
|
2411
|
+
let delta = 0.5 + random();
|
|
2412
|
+
for (let iteration = 1; iteration <= iterations; iteration += 1) {
|
|
2413
|
+
const offsetS = 3.6 + (iteration - 1) * ((runSeconds - 4.5) / iterations);
|
|
2414
|
+
if (plan.recoveryWindowS && plan.startAtS + offsetS >= plan.recoveryWindowS[0] && plan.startAtS + offsetS < plan.recoveryWindowS[1]) continue;
|
|
2415
|
+
energy -= delta;
|
|
2416
|
+
delta *= 0.35 + random() * 0.2;
|
|
2417
|
+
at(offsetS, `SCF iter ${String(iteration).padStart(3)} E = ${energy.toFixed(8)} Ha \u0394E = ${delta.toExponential(2)}
|
|
2418
|
+
`);
|
|
2419
|
+
if (iteration === 4 && random() < 0.4) at(offsetS + 0.1, "UserWarning: level shift applied to aid convergence\n", "stderr");
|
|
2420
|
+
}
|
|
2421
|
+
if (plan.recoveryWindowS) {
|
|
2422
|
+
const [faultS, resumeS] = plan.recoveryWindowS;
|
|
2423
|
+
logs2.push({ atS: faultS, stream: "stderr", content: "CUDA error: an illegal memory access was encountered (Xid 79)\n" });
|
|
2424
|
+
logs2.push({ atS: resumeS + 0.5, stream: "stdout", content: "[molsim] resumed on replacement GPU from checkpoint\n" });
|
|
2425
|
+
}
|
|
2426
|
+
if (plan.outcome === "failed") {
|
|
2427
|
+
logs2.push({ atS: plan.finalizeAtS - 0.6, stream: "stderr", content: 'Traceback (most recent call last):\n File "run.py", line 88, in <module>\n result = scf.kernel()\n' });
|
|
2428
|
+
logs2.push({ atS: plan.finalizeAtS - 0.4, stream: "stderr", content: "RuntimeError: SCF did not converge in 128 cycles\n" });
|
|
2429
|
+
} else if (plan.outcome === "completed") {
|
|
2430
|
+
logs2.push({ atS: plan.finalizeAtS - 0.4, stream: "stdout", content: `[molsim] converged: E(total) = ${energy.toFixed(8)} Ha
|
|
2431
|
+
` });
|
|
2432
|
+
logs2.push({ atS: plan.finalizeAtS + 0.3, stream: "stdout", content: `[molsim] wrote outputs/${plan.molecule}.json
|
|
2433
|
+
` });
|
|
2434
|
+
}
|
|
2435
|
+
return logs2.filter((entry) => entry.atS < plan.endAtS).sort((left, right) => left.atS - right.atS);
|
|
2436
|
+
}
|
|
2437
|
+
function buildEvents(plan) {
|
|
2438
|
+
const events = [{ atS: 0, type: "state_changed", message: "Flight queued" }];
|
|
2439
|
+
events.push({ atS: plan.dispatchAtS, type: "state_changed", message: `Dispatching to lambda (${plan.gpuType})` });
|
|
2440
|
+
events.push({ atS: plan.startAtS, type: "state_changed", message: "Instance booted; workload running" });
|
|
2441
|
+
if (plan.recoveryWindowS) {
|
|
2442
|
+
const [faultS, resumeS] = plan.recoveryWindowS;
|
|
2443
|
+
events.push({ atS: faultS, type: "gpu_recovery_started", message: "GPU fault detected (Xid 79); collecting partial outputs" });
|
|
2444
|
+
events.push({ atS: faultS + 2, type: "gpu_recovery_dispatch", message: "Replacement instance authorized (1 of 1)" });
|
|
2445
|
+
events.push({ atS: resumeS, type: "state_changed", message: "Replacement running" });
|
|
2446
|
+
}
|
|
2447
|
+
events.push({ atS: plan.finalizeAtS, type: "state_changed", message: "Collecting outputs" });
|
|
2448
|
+
if (plan.outcome === "cancelled") {
|
|
2449
|
+
const beforeCancel = events.filter((event) => event.atS < plan.endAtS);
|
|
2450
|
+
return [...beforeCancel, { atS: plan.endAtS, type: "state_changed", message: "Flight cancelled by user" }];
|
|
2451
|
+
}
|
|
2452
|
+
events.push({ atS: plan.endAtS, type: "state_changed", message: plan.outcome === "failed" ? "Workload exited with code 1" : "Flight completed" });
|
|
2453
|
+
return events;
|
|
2454
|
+
}
|
|
2455
|
+
function buildDemoPlan(seed = 21) {
|
|
2456
|
+
const random = seededRandom(seed);
|
|
2457
|
+
const gpuFreeAtS = Array.from({ length: CONCURRENT_GPUS }, () => 0);
|
|
2458
|
+
const plans = [];
|
|
2459
|
+
for (let taskIndex = 0; taskIndex < MOLECULES.length; taskIndex += 1) {
|
|
2460
|
+
const slot = gpuFreeAtS.indexOf(Math.min(...gpuFreeAtS));
|
|
2461
|
+
const dispatchAtS = Math.max(gpuFreeAtS[slot], 1 + taskIndex * 0.4);
|
|
2462
|
+
const startAtS = dispatchAtS + between(random, 3, 9);
|
|
2463
|
+
const recoveryWindowS = taskIndex === RECOVERING_TASK ? [startAtS + 14, startAtS + 24] : null;
|
|
2464
|
+
const finalizeAtS = startAtS + between(random, 18, 55) + (recoveryWindowS ? 10 : 0);
|
|
2465
|
+
const endAtS = finalizeAtS + between(random, 2, 4);
|
|
2466
|
+
let outcome = taskIndex === FAILING_TASK ? "failed" : "completed";
|
|
2467
|
+
let taskEndS = endAtS;
|
|
2468
|
+
if (taskIndex === CANCELLED_TASK) {
|
|
2469
|
+
outcome = "cancelled";
|
|
2470
|
+
taskEndS = CANCELLED_AT_S;
|
|
2471
|
+
}
|
|
2472
|
+
if (outcome !== "cancelled") gpuFreeAtS[slot] = endAtS;
|
|
2473
|
+
else if (taskEndS > dispatchAtS) gpuFreeAtS[slot] = taskEndS;
|
|
2474
|
+
const partial = {
|
|
2475
|
+
taskIndex,
|
|
2476
|
+
// Same shape as real IDs: "fl-" and 24 hex digits.
|
|
2477
|
+
flightId: `fl-${Math.floor(random() * 4294967295).toString(16).padStart(8, "0")}${String(taskIndex).padStart(16, "0")}`,
|
|
2478
|
+
name: `molsim[${taskIndex}]`,
|
|
2479
|
+
molecule: MOLECULES[taskIndex],
|
|
2480
|
+
gpuType: taskIndex % 5 === 4 ? "H100" : "A100",
|
|
2481
|
+
dispatchAtS,
|
|
2482
|
+
startAtS,
|
|
2483
|
+
finalizeAtS,
|
|
2484
|
+
endAtS: taskEndS,
|
|
2485
|
+
outcome,
|
|
2486
|
+
recoveryWindowS
|
|
2487
|
+
};
|
|
2488
|
+
const events = buildEvents(partial);
|
|
2489
|
+
plans.push({ ...partial, events, logs: buildLogs({ ...partial, events }, random) });
|
|
2490
|
+
}
|
|
2491
|
+
return plans;
|
|
2492
|
+
}
|
|
2493
|
+
function statusAt(plan, elapsedS) {
|
|
2494
|
+
if (elapsedS >= plan.endAtS) return plan.outcome;
|
|
2495
|
+
if (elapsedS < plan.dispatchAtS) return "queued";
|
|
2496
|
+
if (elapsedS < plan.startAtS) return "dispatching";
|
|
2497
|
+
if (elapsedS < plan.finalizeAtS) return "running";
|
|
2498
|
+
return "finalizing";
|
|
2499
|
+
}
|
|
2500
|
+
var DemoDataSource = class {
|
|
2501
|
+
constructor(now = Date.now, seed = 21) {
|
|
2502
|
+
this.now = now;
|
|
2503
|
+
this.plans = buildDemoPlan(seed);
|
|
2504
|
+
this.startedAtMs = now();
|
|
2505
|
+
}
|
|
2506
|
+
now;
|
|
2507
|
+
title = "DEMO \xB7 molsim array \xB7 21 tasks";
|
|
2508
|
+
pollIntervalMs = 500;
|
|
2509
|
+
plans;
|
|
2510
|
+
startedAtMs;
|
|
2511
|
+
elapsedS() {
|
|
2512
|
+
return (this.now() - this.startedAtMs) / 1e3;
|
|
2513
|
+
}
|
|
2514
|
+
iso(offsetS) {
|
|
2515
|
+
return new Date(this.startedAtMs + offsetS * 1e3).toISOString();
|
|
2516
|
+
}
|
|
2517
|
+
plan(flightId) {
|
|
2518
|
+
const plan = this.plans.find((candidate) => candidate.flightId === flightId);
|
|
2519
|
+
if (!plan) throw new Error(`Unknown demo flight ${flightId}`);
|
|
2520
|
+
return plan;
|
|
2521
|
+
}
|
|
2522
|
+
async listFlights() {
|
|
2523
|
+
const elapsedS = this.elapsedS();
|
|
2524
|
+
return this.plans.map((plan) => {
|
|
2525
|
+
const status2 = statusAt(plan, elapsedS);
|
|
2526
|
+
const reached = (offsetS) => elapsedS >= offsetS && offsetS < plan.endAtS ? this.iso(offsetS) : null;
|
|
2527
|
+
const finished = elapsedS >= plan.endAtS;
|
|
2528
|
+
const inRecovery = plan.recoveryWindowS !== null && elapsedS >= plan.recoveryWindowS[0] && elapsedS < plan.recoveryWindowS[1];
|
|
2529
|
+
return {
|
|
2530
|
+
flight_id: plan.flightId,
|
|
2531
|
+
name: plan.name,
|
|
2532
|
+
status: status2,
|
|
2533
|
+
gpu_type: plan.gpuType,
|
|
2534
|
+
gpu_count: 1,
|
|
2535
|
+
max_duration_s: MAX_DURATION_S2,
|
|
2536
|
+
queued_at: this.iso(0),
|
|
2537
|
+
dispatched_at: reached(plan.dispatchAtS),
|
|
2538
|
+
started_at: reached(plan.startAtS),
|
|
2539
|
+
completed_at: finished ? this.iso(plan.endAtS) : null,
|
|
2540
|
+
exit_code: finished && plan.outcome !== "cancelled" ? plan.outcome === "failed" ? 1 : 0 : null,
|
|
2541
|
+
error_message: finished && plan.outcome === "failed" ? "Workload exited with code 1" : null,
|
|
2542
|
+
charge_credits: finished && plan.startAtS < plan.endAtS ? Math.max(1, Math.ceil((plan.endAtS - plan.startAtS) / 20)) : null,
|
|
2543
|
+
provider: elapsedS >= plan.dispatchAtS && plan.dispatchAtS < plan.endAtS ? "lambda" : null,
|
|
2544
|
+
recovery: inRecovery ? { phase: elapsedS < plan.recoveryWindowS[0] + 2 ? "collecting" : "replacement" } : null
|
|
2545
|
+
};
|
|
2546
|
+
});
|
|
2547
|
+
}
|
|
2548
|
+
async listEvents(flightId) {
|
|
2549
|
+
const elapsedS = this.elapsedS();
|
|
2550
|
+
return this.plan(flightId).events.filter((event) => event.atS <= elapsedS).map((event, index) => ({ id: index + 1, type: event.type, message: event.message, created_at: this.iso(event.atS) }));
|
|
2551
|
+
}
|
|
2552
|
+
async readLogs(flightId, cursor) {
|
|
2553
|
+
const plan = this.plan(flightId);
|
|
2554
|
+
const elapsedS = this.elapsedS();
|
|
2555
|
+
const startIndex = cursor ? Number(cursor) : 0;
|
|
2556
|
+
const available = plan.logs.filter((entry) => entry.atS <= elapsedS);
|
|
2557
|
+
const lines = available.slice(startIndex).map(({ stream, content }) => ({ stream, content }));
|
|
2558
|
+
return {
|
|
2559
|
+
lines,
|
|
2560
|
+
nextCursor: String(available.length),
|
|
2561
|
+
complete: elapsedS >= plan.endAtS && available.length === plan.logs.length
|
|
2562
|
+
};
|
|
2563
|
+
}
|
|
2564
|
+
};
|
|
2565
|
+
|
|
2566
|
+
// src/atc/live-source.ts
|
|
2567
|
+
var RECENT_FLIGHT_LIMIT = 50;
|
|
2568
|
+
var SEARCH_FLIGHT_LIMIT = 200;
|
|
2569
|
+
var BATCH_WINDOW_MARGIN_MS = 5 * 6e4;
|
|
2570
|
+
var MAX_BATCH_PAGES = 10;
|
|
2571
|
+
var MAX_STRAGGLER_READS = 20;
|
|
2572
|
+
var MAX_CONSECUTIVE_STRAGGLER_FAILURES = 3;
|
|
2573
|
+
var REQUEST_TIMEOUT_MS = 15e3;
|
|
2574
|
+
function placeholder(member, status2) {
|
|
2575
|
+
return {
|
|
2576
|
+
flight_id: member.flight_id,
|
|
2577
|
+
name: member.name,
|
|
2578
|
+
status: status2,
|
|
2579
|
+
gpu_type: null,
|
|
2580
|
+
gpu_count: 1,
|
|
2581
|
+
max_duration_s: null,
|
|
2582
|
+
queued_at: null,
|
|
2583
|
+
dispatched_at: null,
|
|
2584
|
+
started_at: null,
|
|
2585
|
+
completed_at: null,
|
|
2586
|
+
exit_code: null,
|
|
2587
|
+
error_message: null
|
|
2588
|
+
};
|
|
2589
|
+
}
|
|
2590
|
+
function nameMatcher(pattern) {
|
|
2591
|
+
const source = [...pattern].map((character) => character === "*" ? ".*" : character === "?" ? "." : character.replace(/[.+^${}()|[\]\\]/g, "\\$&")).join("");
|
|
2592
|
+
const expression = new RegExp(`^${source}$`, "su");
|
|
2593
|
+
return (name) => expression.test(name);
|
|
2594
|
+
}
|
|
2595
|
+
var INVALID_RESPONSE = "Unexpected response from the Flightdeck API.";
|
|
2596
|
+
function record(value) {
|
|
2597
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new CliError(INVALID_RESPONSE);
|
|
2598
|
+
return value;
|
|
2599
|
+
}
|
|
2600
|
+
var text = (value) => typeof value === "string" ? value : null;
|
|
2601
|
+
var count = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
2602
|
+
function toFlight(value) {
|
|
2603
|
+
const raw = record(value);
|
|
2604
|
+
const flightId = text(raw.flight_id);
|
|
2605
|
+
if (!flightId) throw new CliError(INVALID_RESPONSE);
|
|
2606
|
+
const recoveryPhase = raw.recovery && typeof raw.recovery === "object" ? text(raw.recovery.phase) : null;
|
|
2607
|
+
return {
|
|
2608
|
+
flight_id: flightId,
|
|
2609
|
+
name: text(raw.name) ?? flightId,
|
|
2610
|
+
status: text(raw.status) ?? "unknown",
|
|
2611
|
+
gpu_type: text(raw.gpu_type),
|
|
2612
|
+
gpu_count: count(raw.gpu_count) ?? 1,
|
|
2613
|
+
max_duration_s: count(raw.max_duration_s),
|
|
2614
|
+
queued_at: text(raw.queued_at),
|
|
2615
|
+
dispatched_at: text(raw.dispatched_at),
|
|
2616
|
+
started_at: text(raw.started_at),
|
|
2617
|
+
completed_at: text(raw.completed_at),
|
|
2618
|
+
exit_code: count(raw.exit_code),
|
|
2619
|
+
error_message: text(raw.error_message),
|
|
2620
|
+
// Customers receive credits; platform admins receive monetary fields instead, which the board does not show.
|
|
2621
|
+
charge_credits: count(raw.charge_credits),
|
|
2622
|
+
provider: text(raw.provider),
|
|
2623
|
+
recovery: !("recovery" in raw) ? void 0 : recoveryPhase ? { phase: recoveryPhase } : null
|
|
2624
|
+
};
|
|
2625
|
+
}
|
|
2626
|
+
var LiveDataSource = class {
|
|
2627
|
+
constructor(target, config, request2 = apiRequest, timeoutMs = REQUEST_TIMEOUT_MS) {
|
|
2628
|
+
this.target = target;
|
|
2629
|
+
this.config = config;
|
|
2630
|
+
this.request = request2;
|
|
2631
|
+
this.timeoutMs = timeoutMs;
|
|
2632
|
+
if (target.kind === "flight") this.initialFlightId = target.flightId;
|
|
2633
|
+
}
|
|
2634
|
+
target;
|
|
2635
|
+
config;
|
|
2636
|
+
request;
|
|
2637
|
+
timeoutMs;
|
|
2638
|
+
pollIntervalMs = 5e3;
|
|
2639
|
+
selectionPollIntervalMs = 2e3;
|
|
2640
|
+
initialFlightId;
|
|
2641
|
+
accountLabel = null;
|
|
2642
|
+
/** Settled batch tasks never change, so they are not looked up again. */
|
|
2643
|
+
settledBatchFlights = /* @__PURE__ */ new Map();
|
|
2644
|
+
/** Last known record of every batch task, shown when a poll could not refresh it. */
|
|
2645
|
+
lastKnownBatchFlights = /* @__PURE__ */ new Map();
|
|
2646
|
+
/** When each straggler was last read singly, so the per-poll cap rotates through them. */
|
|
2647
|
+
stragglerReadAt = /* @__PURE__ */ new Map();
|
|
2648
|
+
get title() {
|
|
2649
|
+
const scope = this.target.kind === "recent" ? `recent flights (last ${RECENT_FLIGHT_LIMIT})` : this.target.kind === "flight" ? `flight ${this.target.flightId}` : this.target.kind === "batch" ? `sbatch ${this.target.record.job_id} \u2500\u2500\u2500 ${this.target.record.job_name}` : `flights named ${this.target.pattern} (last ${SEARCH_FLIGHT_LIMIT})`;
|
|
2650
|
+
return this.accountLabel ? `${scope} \u2500\u2500\u2500 ${this.accountLabel}` : scope;
|
|
2651
|
+
}
|
|
2652
|
+
/**
|
|
2653
|
+
* Run before the screen opens, so sign-in problems and unknown flights are
|
|
2654
|
+
* reported as ordinary CLI errors. Also learns the account name for the title.
|
|
2655
|
+
*/
|
|
2656
|
+
async preflight(signal) {
|
|
2657
|
+
try {
|
|
2658
|
+
const account = record(await (await this.get("/auth/me", "Unable to read account", signal)).json());
|
|
2659
|
+
this.accountLabel = text(account.name) ?? text(account.email);
|
|
2660
|
+
} catch {
|
|
2661
|
+
}
|
|
2662
|
+
await this.listFlights(signal);
|
|
2663
|
+
}
|
|
2664
|
+
async get(path, action, signal, allowed = []) {
|
|
2665
|
+
let response;
|
|
2666
|
+
try {
|
|
2667
|
+
response = await this.request("GET", path, void 0, {
|
|
2668
|
+
config: this.config,
|
|
2669
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(this.timeoutMs)])
|
|
2670
|
+
});
|
|
2671
|
+
} catch (error) {
|
|
2672
|
+
if (error instanceof NetworkError) throw error;
|
|
2673
|
+
if (error instanceof CliError) throw new AtcFatalError(error.message, { cause: error });
|
|
2674
|
+
throw error;
|
|
2675
|
+
}
|
|
2676
|
+
if (response.ok || allowed.includes(response.status)) return response;
|
|
2677
|
+
await response.body?.cancel();
|
|
2678
|
+
if (response.status === 401) throw new AtcFatalError(httpError(action, 401).message);
|
|
2679
|
+
throw httpError(action, response.status);
|
|
2680
|
+
}
|
|
2681
|
+
async listPage(query, signal) {
|
|
2682
|
+
const response = await this.get(`/flights?${query}`, "Unable to list flights", signal);
|
|
2683
|
+
const flights = record(await response.json()).flights;
|
|
2684
|
+
if (!Array.isArray(flights)) throw new CliError(INVALID_RESPONSE);
|
|
2685
|
+
return flights.map(toFlight);
|
|
2686
|
+
}
|
|
2687
|
+
async listFlights(signal) {
|
|
2688
|
+
switch (this.target.kind) {
|
|
2689
|
+
case "flight":
|
|
2690
|
+
return [await this.readFlight(this.target.flightId, signal)];
|
|
2691
|
+
case "recent":
|
|
2692
|
+
return this.listPage(`limit=${RECENT_FLIGHT_LIMIT}`, signal);
|
|
2693
|
+
case "name": {
|
|
2694
|
+
const matches = nameMatcher(this.target.pattern);
|
|
2695
|
+
return (await this.listPage(`limit=${SEARCH_FLIGHT_LIMIT}`, signal)).filter((flight) => matches(flight.name));
|
|
2696
|
+
}
|
|
2697
|
+
case "batch":
|
|
2698
|
+
return this.listBatch(this.target.record, signal);
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2701
|
+
/**
|
|
2702
|
+
* The list returns the newest matches first, so a large array's oldest tasks
|
|
2703
|
+
* fall past the first page. Walk back through the submission window with
|
|
2704
|
+
* `to=` until every unsettled task is found; settled tasks are remembered and
|
|
2705
|
+
* need no lookup. The first page is always read, even for a finished array,
|
|
2706
|
+
* so an expired login or lost connection still shows.
|
|
2707
|
+
*/
|
|
2708
|
+
async listBatch(batch, signal) {
|
|
2709
|
+
const from = new Date(Date.parse(batch.submitted_at) - BATCH_WINDOW_MARGIN_MS).toISOString();
|
|
2710
|
+
const wanted = new Set(batch.flights.map((member) => member.flight_id).filter((id) => !this.settledBatchFlights.has(id)));
|
|
2711
|
+
const found = /* @__PURE__ */ new Map();
|
|
2712
|
+
let to;
|
|
2713
|
+
for (let page = 0; page < MAX_BATCH_PAGES && (page === 0 || found.size < wanted.size); page += 1) {
|
|
2714
|
+
const window = `from=${encodeURIComponent(from)}${to ? `&to=${encodeURIComponent(to)}` : ""}`;
|
|
2715
|
+
const rows = await this.listPage(`${window}&limit=${SEARCH_FLIGHT_LIMIT}`, signal);
|
|
2716
|
+
for (const flight of rows) if (wanted.has(flight.flight_id)) found.set(flight.flight_id, flight);
|
|
2717
|
+
const oldest = rows[rows.length - 1]?.queued_at ?? void 0;
|
|
2718
|
+
if (rows.length < SEARCH_FLIGHT_LIMIT || !oldest || oldest === to) break;
|
|
2719
|
+
to = oldest;
|
|
2720
|
+
}
|
|
2721
|
+
const stragglers = [...wanted].filter((id) => !found.has(id)).sort((left, right) => (this.stragglerReadAt.get(left) ?? 0) - (this.stragglerReadAt.get(right) ?? 0)).slice(0, MAX_STRAGGLER_READS);
|
|
2722
|
+
let consecutiveFailures = 0;
|
|
2723
|
+
for (const flightId of stragglers) {
|
|
2724
|
+
const member = batch.flights.find((candidate) => candidate.flight_id === flightId);
|
|
2725
|
+
this.stragglerReadAt.set(flightId, Date.now());
|
|
2726
|
+
try {
|
|
2727
|
+
found.set(flightId, await this.readBatchMember(member, signal));
|
|
2728
|
+
consecutiveFailures = 0;
|
|
2729
|
+
} catch (error) {
|
|
2730
|
+
if (error instanceof AtcFatalError || error instanceof NetworkError || signal.aborted) throw error;
|
|
2731
|
+
if (++consecutiveFailures >= MAX_CONSECUTIVE_STRAGGLER_FAILURES) break;
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2734
|
+
return batch.flights.map((member) => {
|
|
2735
|
+
const flight = this.settledBatchFlights.get(member.flight_id) ?? found.get(member.flight_id) ?? this.lastKnownBatchFlights.get(member.flight_id) ?? placeholder(member, "unknown");
|
|
2736
|
+
if (isTerminal(flight.status) || flight.status === "missing") this.settledBatchFlights.set(member.flight_id, flight);
|
|
2737
|
+
this.lastKnownBatchFlights.set(member.flight_id, flight);
|
|
2738
|
+
return flight;
|
|
2739
|
+
});
|
|
2740
|
+
}
|
|
2741
|
+
async readBatchMember(member, signal) {
|
|
2742
|
+
const response = await this.get(`/flights/${encodeURIComponent(member.flight_id)}`, "Unable to read flight", signal, [404]);
|
|
2743
|
+
if (response.status !== 404) return toFlight(await response.json());
|
|
2744
|
+
await response.body?.cancel();
|
|
2745
|
+
return placeholder(member, "missing");
|
|
2746
|
+
}
|
|
2747
|
+
async readFlight(flightId, signal) {
|
|
2748
|
+
const response = await this.get(`/flights/${encodeURIComponent(flightId)}`, "Unable to read flight", signal, [404]);
|
|
2749
|
+
if (response.status === 404) {
|
|
2750
|
+
await response.body?.cancel();
|
|
2751
|
+
if (this.target.kind === "flight") throw new AtcFatalError(`Flight ${flightId} was not found for this account.`);
|
|
2752
|
+
throw httpError("Unable to read flight", 404);
|
|
2753
|
+
}
|
|
2754
|
+
return toFlight(await response.json());
|
|
2755
|
+
}
|
|
2756
|
+
async listEvents(flightId, signal) {
|
|
2757
|
+
const response = await this.get(`/flights/${encodeURIComponent(flightId)}/events`, "Unable to read flight events", signal);
|
|
2758
|
+
const events = record(await response.json()).events;
|
|
2759
|
+
if (!Array.isArray(events)) throw new CliError(INVALID_RESPONSE);
|
|
2760
|
+
return events.map((value) => {
|
|
2761
|
+
const raw = record(value);
|
|
2762
|
+
return { id: count(raw.id) ?? 0, type: text(raw.type) ?? "event", message: text(raw.message), created_at: text(raw.created_at) ?? "" };
|
|
2763
|
+
});
|
|
2764
|
+
}
|
|
2765
|
+
async readLogs(flightId, cursor, signal) {
|
|
2766
|
+
const query = cursor ? `?cursor=${encodeURIComponent(cursor)}` : "";
|
|
2767
|
+
const response = await this.get(`/flights/${encodeURIComponent(flightId)}/logs${query}`, "Unable to read logs", signal, [400]);
|
|
2768
|
+
if (response.status === 400) {
|
|
2769
|
+
const body2 = await response.json().catch(() => ({}));
|
|
2770
|
+
if (cursor && body2.error === "invalid_cursor") return { lines: [], nextCursor: "", complete: false, restart: true };
|
|
2771
|
+
throw httpError("Unable to read logs", 400);
|
|
2772
|
+
}
|
|
2773
|
+
const body = record(await response.json());
|
|
2774
|
+
if (typeof body.has_more !== "boolean") {
|
|
2775
|
+
throw new AtcFatalError("This backend does not support complete log pagination. Deploy the updated backend before using flightdeck atc.");
|
|
2776
|
+
}
|
|
2777
|
+
const nextCursor = text(body.next_cursor);
|
|
2778
|
+
if (!Array.isArray(body.logs) || typeof body.complete !== "boolean" || !nextCursor || body.has_more && body.complete) {
|
|
2779
|
+
throw new CliError(INVALID_RESPONSE);
|
|
2780
|
+
}
|
|
2781
|
+
if (body.has_more && nextCursor === cursor) throw new CliError("Log pagination cursor did not advance; retrying.");
|
|
2782
|
+
const lines = body.logs.map((value) => {
|
|
2783
|
+
const raw = record(value);
|
|
2784
|
+
const content = text(raw.content);
|
|
2785
|
+
if (content === null || typeof raw.stream !== "string") throw new CliError(INVALID_RESPONSE);
|
|
2786
|
+
return { stream: raw.stream, content };
|
|
2787
|
+
});
|
|
2788
|
+
const attemptSeq = body.attempt_seq === null ? null : count(body.attempt_seq) ?? void 0;
|
|
2789
|
+
return { lines, nextCursor, complete: body.complete, hasMore: body.has_more, attemptSeq };
|
|
2790
|
+
}
|
|
2791
|
+
};
|
|
2792
|
+
|
|
2793
|
+
// src/commands/atc.ts
|
|
2794
|
+
function batchTarget(jobId, config) {
|
|
2795
|
+
const record2 = loadBatchRecord(jobId);
|
|
2796
|
+
if (!record2) {
|
|
2797
|
+
throw new CliError(`No local record of sbatch job ${jobId}. \`flightdeck sbatch\` records arrays on the machine that submitted them (from this CLI version on). Try \`flightdeck atc --name '<job-name>*'\` instead.`);
|
|
2798
|
+
}
|
|
2799
|
+
if (normalizeApiUrl(record2.api_url) !== normalizeApiUrl(config.api_url)) {
|
|
2800
|
+
throw new CliError(`sbatch job ${jobId} was submitted to ${record2.api_url}, but this login uses ${config.api_url}.`);
|
|
2801
|
+
}
|
|
2802
|
+
return { kind: "batch", record: record2 };
|
|
2803
|
+
}
|
|
2804
|
+
function liveTarget(target, name, config) {
|
|
2805
|
+
if (name !== void 0) {
|
|
2806
|
+
if (target) throw new CliError("Use either a flight/job ID or --name, not both.");
|
|
2807
|
+
if (!name.trim()) throw new CliError("--name needs a pattern, such as 'molsim*'.");
|
|
2808
|
+
return { kind: "name", pattern: name };
|
|
2809
|
+
}
|
|
2810
|
+
if (!target) return { kind: "recent" };
|
|
2811
|
+
if (isBatchJobId(target)) return batchTarget(target, config);
|
|
2812
|
+
return { kind: "flight", flightId: target };
|
|
2813
|
+
}
|
|
2814
|
+
async function atc(target, options) {
|
|
2815
|
+
if (options.demo && (target || options.name !== void 0)) throw new CliError("--demo runs a simulated array; drop the flight/job ID or --name.");
|
|
2816
|
+
const config = options.demo ? null : loadConfig();
|
|
2817
|
+
const liveTargetChoice = config ? liveTarget(target, options.name, config) : null;
|
|
2818
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY || process.env.TERM === "dumb") {
|
|
2819
|
+
throw new CliError("flightdeck atc needs an interactive terminal. Use `flightdeck status` or `flightdeck logs` in scripts.");
|
|
2820
|
+
}
|
|
2821
|
+
let source;
|
|
2822
|
+
if (!config || !liveTargetChoice) {
|
|
2823
|
+
source = new DemoDataSource();
|
|
2824
|
+
} else {
|
|
2825
|
+
const liveSource = new LiveDataSource(liveTargetChoice, config);
|
|
2826
|
+
await liveSource.preflight(new AbortController().signal);
|
|
2827
|
+
source = liveSource;
|
|
2828
|
+
}
|
|
2829
|
+
await runAtc(source, {
|
|
2830
|
+
input: process.stdin,
|
|
2831
|
+
output: process.stdout,
|
|
2832
|
+
// https://no-color.org: any non-empty value disables color.
|
|
2833
|
+
colorEnabled: !process.env.NO_COLOR
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
|
|
1087
2837
|
// src/commands/outputs.ts
|
|
1088
2838
|
async function listOutputs(flightId) {
|
|
1089
2839
|
const response = await apiRequest("GET", `/flights/${flightId}/outputs`);
|
|
@@ -1097,7 +2847,7 @@ async function listOutputs(flightId) {
|
|
|
1097
2847
|
|
|
1098
2848
|
// src/commands/download.ts
|
|
1099
2849
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
1100
|
-
import { basename, join as
|
|
2850
|
+
import { basename, join as join4, resolve as resolve3 } from "node:path";
|
|
1101
2851
|
async function download(flightId, options) {
|
|
1102
2852
|
const manifestResponse = await apiRequest("GET", `/flights/${flightId}/outputs`);
|
|
1103
2853
|
if (!manifestResponse.ok) throw httpError("Unable to list outputs for download", manifestResponse.status);
|
|
@@ -1107,15 +2857,15 @@ async function download(flightId, options) {
|
|
|
1107
2857
|
for (const output of manifest.outputs ?? []) {
|
|
1108
2858
|
const response = await artifactRequest(flightId, output.artifact_id);
|
|
1109
2859
|
if (!response.ok) throw httpError("Unable to download output", response.status);
|
|
1110
|
-
const target =
|
|
2860
|
+
const target = join4(root, basename(output.filename));
|
|
1111
2861
|
await writeFile(target, Buffer.from(await response.arrayBuffer()), { flag: "wx" });
|
|
1112
2862
|
console.log(`Downloaded ${target}`);
|
|
1113
2863
|
}
|
|
1114
2864
|
}
|
|
1115
2865
|
|
|
1116
2866
|
// src/commands/slurm-test.ts
|
|
1117
|
-
import { mkdtempSync as mkdtempSync2, rmSync as
|
|
1118
|
-
import { join as
|
|
2867
|
+
import { mkdtempSync as mkdtempSync2, rmSync as rmSync3, writeFileSync as writeFileSync3 } from "node:fs";
|
|
2868
|
+
import { join as join5, relative as relative2 } from "node:path";
|
|
1119
2869
|
var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
|
|
1120
2870
|
var CONFORMANCE_SCRIPT = `#!/bin/bash
|
|
1121
2871
|
#SBATCH --job-name=slurm-conformance
|
|
@@ -1174,9 +2924,9 @@ async function verifyResults(flightId, taskId) {
|
|
|
1174
2924
|
async function slurmTest(options) {
|
|
1175
2925
|
const timeoutMs = positiveSeconds(options.timeout, "--timeout") * 1e3;
|
|
1176
2926
|
const pollIntervalMs = positiveSeconds(options.pollInterval, "--poll-interval") * 1e3;
|
|
1177
|
-
const tempDir = mkdtempSync2(
|
|
1178
|
-
const scriptPath =
|
|
1179
|
-
|
|
2927
|
+
const tempDir = mkdtempSync2(join5(process.cwd(), ".flightdeck-slurm-test-"));
|
|
2928
|
+
const scriptPath = join5(tempDir, "conformance.sbatch");
|
|
2929
|
+
writeFileSync3(scriptPath, CONFORMANCE_SCRIPT, { encoding: "utf8", mode: 448 });
|
|
1180
2930
|
let submitted;
|
|
1181
2931
|
try {
|
|
1182
2932
|
const result = await sbatch(relative2(process.cwd(), scriptPath), {
|
|
@@ -1184,7 +2934,7 @@ async function slurmTest(options) {
|
|
|
1184
2934
|
});
|
|
1185
2935
|
submitted = result.submitted;
|
|
1186
2936
|
} finally {
|
|
1187
|
-
|
|
2937
|
+
rmSync3(tempDir, { recursive: true, force: true });
|
|
1188
2938
|
}
|
|
1189
2939
|
if (submitted.length !== 2) throw new CliError(`expected 2 conformance flights, submitted ${submitted.length}`);
|
|
1190
2940
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -1221,6 +2971,8 @@ function sleep(milliseconds) {
|
|
|
1221
2971
|
return new Promise((resolve4) => setTimeout(resolve4, milliseconds));
|
|
1222
2972
|
}
|
|
1223
2973
|
async function loginFailure(response) {
|
|
2974
|
+
const suspended = await suspensionError(response);
|
|
2975
|
+
if (suspended) return suspended;
|
|
1224
2976
|
try {
|
|
1225
2977
|
const body = await response.json();
|
|
1226
2978
|
if (body.error === "device_authorization_denied") {
|
|
@@ -1275,23 +3027,27 @@ Verification code: ${authorization.user_code}`);
|
|
|
1275
3027
|
|
|
1276
3028
|
// src/commands/logout.ts
|
|
1277
3029
|
async function logout(options) {
|
|
1278
|
-
|
|
3030
|
+
const config = options.local ? null : readConfig();
|
|
3031
|
+
let removed;
|
|
3032
|
+
try {
|
|
3033
|
+
removed = deleteConfig();
|
|
3034
|
+
} catch (cause) {
|
|
3035
|
+
throw new CliError("Could not remove the local login. Local logout was not completed; check file permissions and retry flightdeck logout.", { cause });
|
|
3036
|
+
}
|
|
3037
|
+
if (!removed) {
|
|
1279
3038
|
console.log("No local FlightDeck login was found.");
|
|
1280
3039
|
return;
|
|
1281
3040
|
}
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
}
|
|
3041
|
+
console.log("Local login removed.");
|
|
3042
|
+
if (options.local) return;
|
|
3043
|
+
const result = config ? await revokeCurrentCredential(config) : "unconfirmed";
|
|
3044
|
+
if (result === "revoked") {
|
|
3045
|
+
console.log("Server credential revoked.");
|
|
3046
|
+
} else if (result === "invalid") {
|
|
3047
|
+
console.log("Server reports this credential is already invalid.");
|
|
3048
|
+
} else {
|
|
3049
|
+
console.error("Warning: Server revocation was not confirmed; other copies of this credential may remain valid. Use portal credential management or contact your organization admin.");
|
|
1292
3050
|
}
|
|
1293
|
-
deleteConfig();
|
|
1294
|
-
console.log(options.local ? "Local login removed." : "Logged out and credential revoked.");
|
|
1295
3051
|
}
|
|
1296
3052
|
|
|
1297
3053
|
// src/commands/whoami.ts
|
|
@@ -1310,9 +3066,9 @@ async function balance() {
|
|
|
1310
3066
|
|
|
1311
3067
|
// src/index.ts
|
|
1312
3068
|
var program = new Command();
|
|
1313
|
-
program.name("flightdeck").description("FlightDeck CLI \u2014 submit GPU compute jobs").option("--debug", "include safe error type/code and CLI frame diagnostics (no raw stack or credentials)").version(JSON.parse(
|
|
3069
|
+
program.name("flightdeck").description("FlightDeck CLI \u2014 submit GPU compute jobs").option("--debug", "include safe error type/code and CLI frame diagnostics (no raw stack or credentials)").version(JSON.parse(readFileSync5(new URL("../package.json", import.meta.url), "utf8")).version);
|
|
1314
3070
|
program.command("login").description("Log in through the FlightDeck portal").option("--api-url <url>", "FlightDeck API URL", DEFAULT_API_URL).option("--name <name>", "name for this CLI credential", "flightdeck-cli").action(login);
|
|
1315
|
-
program.command("logout").description("
|
|
3071
|
+
program.command("logout").description("Remove the local login and attempt server credential revocation").option("--local", "remove the local login without revoking the credential").action(logout);
|
|
1316
3072
|
program.command("whoami").description("Show the current FlightDeck account").action(whoami);
|
|
1317
3073
|
program.command("balance").description("Show the current credit balance").action(balance);
|
|
1318
3074
|
program.command("submit").description("Submit a job from local files").argument("<entrypoint>", 'command to run (e.g. "python train.py --epochs 50")').option("-n, --name <name>", "job name", "cli-job").option("-f, --files <glob...>", "files to include (default: current directory)").option("-r, --requirements <file>", "pip requirements file", "requirements.txt").option("--gpu <type>", "GPU type (e.g. A100, H100)").option("--gpu-count <n>", "number of GPUs", "1").option("--time <duration>", "max duration (e.g. 1h, 30m, 3600s)", "1h").option("--base-image <image>", "override base Docker image").option("--provider <id>", "pin the flight to one compute provider (default: operator's order)").option("-e, --env <vars...>", "environment variables (KEY=VALUE)").action(submit);
|
|
@@ -1323,6 +3079,7 @@ program.command("slurm-test").description("Run the provider-neutral Slurm confor
|
|
|
1323
3079
|
program.command("status").description("Check job status").argument("<flight_id>", "flight ID to check").action(status);
|
|
1324
3080
|
program.command("logs").description("Stream flight logs").argument("<flight_id>").option("-f, --follow", "follow until complete").action(logs);
|
|
1325
3081
|
program.command("cancel").description("Cancel a flight").argument("<flight_id>").action(cancel);
|
|
3082
|
+
program.command("atc").alias("tui").description("Live full-screen view of your flights, one flight, or an sbatch array (read-only)").argument("[flight_id|job_id]", "a flight, or an sbatch job ID submitted from this machine (default: recent flights)").option("--name <pattern>", "flights whose name matches, e.g. 'molsim*' (searches your last 200)").option("--demo", "run against a simulated 21-task array (no account needed)").action(atc);
|
|
1326
3083
|
program.command("outputs").description("List flight outputs").argument("<flight_id>").action(listOutputs);
|
|
1327
3084
|
program.command("download").description("Download all flight outputs").argument("<flight_id>").option("-o, --output-dir <dir>", "destination directory", ".").action(download);
|
|
1328
3085
|
try {
|