@ornncompute/cli 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -20
- package/package.json +1 -1
- package/src/cli.mjs +248 -40
package/README.md
CHANGED
|
@@ -56,10 +56,11 @@ ornn nodes keys list <node-id> [--json]
|
|
|
56
56
|
ornn node health <node-id> [--json]
|
|
57
57
|
ornn node diagnose <node-id> [--json]
|
|
58
58
|
ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]
|
|
59
|
-
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--
|
|
60
|
-
ornn fleet clean <
|
|
61
|
-
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--
|
|
59
|
+
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] [--json]
|
|
60
|
+
ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
|
|
61
|
+
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
|
|
62
62
|
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
63
|
+
ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]
|
|
63
64
|
ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
|
|
64
65
|
ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
|
|
65
66
|
ornn fleet deploy <fleet-id> --tenant <contact-email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
|
|
@@ -227,28 +228,56 @@ Fleet handoff is an ordered, Compute-backed workflow:
|
|
|
227
228
|
- Before first contact, independently verify each host key and add it to the
|
|
228
229
|
operator's SSH `known_hosts`; fleet SSH uses strict host-key checking and will
|
|
229
230
|
not trust a key first presented during cleanup.
|
|
230
|
-
- `fleet clean ... --dry-run` tries `ubuntu`, `admin`, then `ornn`,
|
|
231
|
-
host's hardware identity
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
231
|
+
- `fleet clean ... --dry-run` tries `ubuntu`, `admin`, then `ornn`, binds the
|
|
232
|
+
inventory to the host's hardware identity, records the plan in Compute, and
|
|
233
|
+
writes an editable mode-`0600` policy beside the local fleet manifest (or to
|
|
234
|
+
`--policy-out`). Inventory covers accounts and orphan homes, authorized SSH
|
|
235
|
+
keys, tenant/log/temp/platform paths, workload and listener processes, GPU
|
|
236
|
+
consumers, custom systemd services, Docker/Podman objects and bind-mount
|
|
237
|
+
dependencies, and every block drive. Each resource advertises its own valid
|
|
238
|
+
actions. Suggestions are never applied silently: unresolved choices and
|
|
239
|
+
conflicts are recorded as plan data, so edit the policy and re-run the same
|
|
240
|
+
fleet with `--policy`; planning is not terminalized just because a choice is
|
|
241
|
+
needed. `--source-user` and `--preserve-user` remain legacy user-only
|
|
242
|
+
shortcuts and cannot be mixed with a policy file.
|
|
243
|
+
- `fleet clean <fleet-id> ... --confirm-clean <plan-hash>` is shown only after
|
|
244
|
+
every choice is resolved. Execution rechecks hardware and resource identity,
|
|
245
|
+
runs an idempotent per-resource journal, preserves the management SSH key,
|
|
246
|
+
enforces key-only SSH, and returns exact deleted, preserved, exception, and
|
|
247
|
+
drive-assurance evidence. `ignore` requires a reason and produces
|
|
248
|
+
`clean_with_exceptions`; it is never reported as residue-free deletion.
|
|
249
|
+
Data-drive actions are selectable: `logical_clean`, `discard`, `sanitize`, or
|
|
250
|
+
`reformat`. The system drive and drives without a unique serial/WWN are
|
|
251
|
+
locked to keep/ignore for in-band cleanup. `sanitize` requires NVMe plus
|
|
252
|
+
controller-reported completion. Logical cleanup, discard, and reformat are
|
|
253
|
+
not certified physical-media erasure, and the system drive requires an
|
|
254
|
+
out-of-band reimage for that assurance.
|
|
255
|
+
If the local SSH command times out while a long device operation is still in
|
|
256
|
+
progress, the recorded run remains resumable; re-run the same
|
|
257
|
+
`--confirm-clean <plan-hash>` command instead of creating a different plan.
|
|
258
|
+
Plans created by a pre-v6 runner must be replanned with `--dry-run` before
|
|
259
|
+
approval. This keeps newly discovered resources bound to a fresh reviewed
|
|
260
|
+
hash instead of adapting a legacy account-only plan during execution.
|
|
261
|
+
|
|
262
|
+
In the generated policy, change only `action`, plus `reason` for `ignore` or
|
|
263
|
+
`replacement` for `replace`; keep each generated `identity_hash` intact. `keep`
|
|
264
|
+
leaves a resource running, `stop_keep` stops but retains it, `stop` terminates a
|
|
265
|
+
process, `delete` removes it, `restart` restarts a service, and `replace`
|
|
266
|
+
recreates a container from the supplied image/name/network/command/labels/mounts
|
|
267
|
+
specification. The generated `suggested_action` is review guidance, not an
|
|
268
|
+
automatic decision. Do not put secrets in replacement commands or labels; use
|
|
269
|
+
the platform secret manager or a reviewed secret mount.
|
|
270
|
+
|
|
247
271
|
- `fleet enroll <fleet-id>` refuses nodes without successful cleanup proof,
|
|
248
272
|
issues and revokes one cleanup-bound token per host, installs the agent,
|
|
249
273
|
assigns the IB island, verifies the same management account still has
|
|
250
274
|
key-based SSH and passwordless sudo, and records the resulting GPU node IDs
|
|
251
275
|
in Compute. Generic enrollment tokens retain the legacy enrollment sanitizer.
|
|
276
|
+
- `fleet delete <fleet-id> --confirm-delete <fleet-id>` hard-deletes only an
|
|
277
|
+
abandoned cleanup plan or an unassigned legacy enrollment record. It saves a
|
|
278
|
+
local audit manifest first and refuses any Commerce/tenant/deployment link,
|
|
279
|
+
active enrollment, assigned GPU node, or completed/running cleanup evidence.
|
|
280
|
+
It never deletes the linked GPU-node records.
|
|
252
281
|
- `reservations commerce create --tenant <email> --listing <listing-id>
|
|
253
282
|
--fleet <fleet-id>` creates and activates the tenant's Commerce reservation.
|
|
254
283
|
GPU count comes from the enrolled fleet; start, end, and hourly rate default
|
package/package.json
CHANGED
package/src/cli.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { createReadStream } from "node:fs";
|
|
4
|
-
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
5
5
|
import { createRequire } from "node:module";
|
|
6
6
|
import { isIP } from "node:net";
|
|
7
7
|
import { homedir, tmpdir } from "node:os";
|
|
8
|
-
import { basename, join } from "node:path";
|
|
8
|
+
import { basename, dirname, join } from "node:path";
|
|
9
9
|
|
|
10
10
|
import {
|
|
11
11
|
CliApiError,
|
|
@@ -132,11 +132,12 @@ Usage:
|
|
|
132
132
|
ornn tokens list [--json]
|
|
133
133
|
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
|
|
134
134
|
ornn tokens revoke <token-id> [--json]
|
|
135
|
-
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--
|
|
136
|
-
ornn fleet clean <
|
|
137
|
-
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--
|
|
135
|
+
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] [--json]
|
|
136
|
+
ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
|
|
137
|
+
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
|
|
138
138
|
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
139
139
|
ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
|
|
140
|
+
ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]
|
|
140
141
|
ornn reservations withdraw <reservation-id> [--json]
|
|
141
142
|
ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
|
|
142
143
|
ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
|
|
@@ -645,7 +646,12 @@ function validateHelpInvocation(command, args) {
|
|
|
645
646
|
"--ib-island",
|
|
646
647
|
"--identity-file",
|
|
647
648
|
"--confirm-clean",
|
|
649
|
+
"--confirm-delete",
|
|
648
650
|
"--confirm-takeover",
|
|
651
|
+
"--policy",
|
|
652
|
+
"--policy-out",
|
|
653
|
+
"--source-user",
|
|
654
|
+
"--preserve-user",
|
|
649
655
|
"--ssh-user",
|
|
650
656
|
"--ssh-port",
|
|
651
657
|
"--parallel",
|
|
@@ -675,7 +681,12 @@ function validateHelpInvocation(command, args) {
|
|
|
675
681
|
? null
|
|
676
682
|
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id>";
|
|
677
683
|
}
|
|
678
|
-
|
|
684
|
+
if (subcommand === "delete") {
|
|
685
|
+
return positionals.length === 1
|
|
686
|
+
? null
|
|
687
|
+
: "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id>";
|
|
688
|
+
}
|
|
689
|
+
return "Usage: ornn fleet clean|enroll|deploy|delete";
|
|
679
690
|
}
|
|
680
691
|
|
|
681
692
|
if (command === "nodes") {
|
|
@@ -2351,6 +2362,55 @@ function fleetAccountOptions(value, optionName) {
|
|
|
2351
2362
|
return [...new Set(normalized)].sort();
|
|
2352
2363
|
}
|
|
2353
2364
|
|
|
2365
|
+
async function readFleetCleanupPolicy(pathValue) {
|
|
2366
|
+
if (!optionProvided(pathValue)) return null;
|
|
2367
|
+
const path = expandUserPath(optionalStringOption(pathValue));
|
|
2368
|
+
let policy;
|
|
2369
|
+
try {
|
|
2370
|
+
policy = JSON.parse(await readFile(path, "utf8"));
|
|
2371
|
+
} catch (error) {
|
|
2372
|
+
throw new Error(`Could not read cleanup policy ${path}: ${fleetResultError(error)}`);
|
|
2373
|
+
}
|
|
2374
|
+
if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
|
|
2375
|
+
throw new Error("Cleanup policy must be a JSON object.");
|
|
2376
|
+
}
|
|
2377
|
+
if (policy.schema_version !== 1) {
|
|
2378
|
+
throw new Error("Cleanup policy schema_version must be 1.");
|
|
2379
|
+
}
|
|
2380
|
+
const direct =
|
|
2381
|
+
policy.resources && typeof policy.resources === "object" && !Array.isArray(policy.resources);
|
|
2382
|
+
const perNode = policy.nodes && typeof policy.nodes === "object" && !Array.isArray(policy.nodes);
|
|
2383
|
+
if (Boolean(direct) === Boolean(perNode)) {
|
|
2384
|
+
throw new Error("Cleanup policy must contain exactly one of resources or nodes.");
|
|
2385
|
+
}
|
|
2386
|
+
return { path, policy };
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
function cleanupPolicyForNode(loaded, node, nodeCount) {
|
|
2390
|
+
if (!loaded) return null;
|
|
2391
|
+
if (loaded.policy.resources) {
|
|
2392
|
+
if (nodeCount !== 1) {
|
|
2393
|
+
throw new Error(
|
|
2394
|
+
"A direct resources policy can only be used for one node; use nodes keyed by IP for a fleet.",
|
|
2395
|
+
);
|
|
2396
|
+
}
|
|
2397
|
+
return loaded.policy;
|
|
2398
|
+
}
|
|
2399
|
+
const policy = loaded.policy.nodes[node.ip_address] || loaded.policy.nodes[node.id];
|
|
2400
|
+
if (!policy) {
|
|
2401
|
+
throw new Error(`Cleanup policy has no entry for ${node.ip_address}.`);
|
|
2402
|
+
}
|
|
2403
|
+
if (
|
|
2404
|
+
policy.schema_version !== 1 ||
|
|
2405
|
+
!policy.resources ||
|
|
2406
|
+
typeof policy.resources !== "object" ||
|
|
2407
|
+
Array.isArray(policy.resources)
|
|
2408
|
+
) {
|
|
2409
|
+
throw new Error(`Cleanup policy entry for ${node.ip_address} is invalid.`);
|
|
2410
|
+
}
|
|
2411
|
+
return policy;
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2354
2414
|
async function fleet(args, context) {
|
|
2355
2415
|
const [subcommand, ...rest] = args;
|
|
2356
2416
|
if (subcommand === "clean") {
|
|
@@ -2362,12 +2422,75 @@ async function fleet(args, context) {
|
|
|
2362
2422
|
if (subcommand === "deploy") {
|
|
2363
2423
|
return await fleetDeploy(rest, context);
|
|
2364
2424
|
}
|
|
2365
|
-
|
|
2425
|
+
if (subcommand === "delete") {
|
|
2426
|
+
return await fleetDelete(rest, context);
|
|
2427
|
+
}
|
|
2428
|
+
throw new Error("Usage: ornn fleet clean|enroll|deploy|delete");
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2431
|
+
async function fleetDelete(args, context) {
|
|
2432
|
+
const usage = "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]";
|
|
2433
|
+
const { options, positionals } = parseOptions(args, {
|
|
2434
|
+
boolean: ["json"],
|
|
2435
|
+
value: ["confirm-delete"],
|
|
2436
|
+
});
|
|
2437
|
+
if (positionals.length !== 1) {
|
|
2438
|
+
throw new Error(usage);
|
|
2439
|
+
}
|
|
2440
|
+
const fleetId = positionals[0];
|
|
2441
|
+
if (!isUuid(fleetId)) {
|
|
2442
|
+
throw new Error("<fleet-id> must be a UUID.");
|
|
2443
|
+
}
|
|
2444
|
+
const confirmation = requiredOption(options.confirmDelete, "--confirm-delete");
|
|
2445
|
+
if (confirmation !== fleetId) {
|
|
2446
|
+
throw new Error("--confirm-delete must exactly match <fleet-id>.");
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
const preview = await operatorRequest({
|
|
2450
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2451
|
+
env: context.env,
|
|
2452
|
+
fetchImpl: context.fetchImpl,
|
|
2453
|
+
});
|
|
2454
|
+
if (String(preview?.id || "") !== fleetId) {
|
|
2455
|
+
throw new Error("Fleet lookup returned a different fleet id; deletion stopped.");
|
|
2456
|
+
}
|
|
2457
|
+
const manifestPath = await saveFleetManifest(
|
|
2458
|
+
{
|
|
2459
|
+
...preview,
|
|
2460
|
+
deletion: { requested_at: new Date().toISOString(), status: "requested" },
|
|
2461
|
+
},
|
|
2462
|
+
context.env,
|
|
2463
|
+
);
|
|
2464
|
+
const result = await operatorRequest({
|
|
2465
|
+
body: { confirm_fleet_id: confirmation },
|
|
2466
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2467
|
+
env: context.env,
|
|
2468
|
+
fetchImpl: context.fetchImpl,
|
|
2469
|
+
method: "DELETE",
|
|
2470
|
+
});
|
|
2471
|
+
const archived = {
|
|
2472
|
+
...(result?.fleet || preview),
|
|
2473
|
+
deletion: {
|
|
2474
|
+
deleted_at: result?.deleted_at || new Date().toISOString(),
|
|
2475
|
+
status: "deleted",
|
|
2476
|
+
},
|
|
2477
|
+
};
|
|
2478
|
+
await saveFleetManifest(archived, context.env);
|
|
2479
|
+
if (options.json) {
|
|
2480
|
+
writeJson(context.stdout, { ...result, audit_manifest: manifestPath });
|
|
2481
|
+
} else {
|
|
2482
|
+
context.stdout.write(`Deleted fleet ${fleetId}.\n`);
|
|
2483
|
+
context.stdout.write(
|
|
2484
|
+
`Released members: ${Array.isArray(archived.nodes) ? archived.nodes.length : 0}\n`,
|
|
2485
|
+
);
|
|
2486
|
+
context.stdout.write(`Audit manifest: ${manifestPath}\n`);
|
|
2487
|
+
}
|
|
2488
|
+
return 0;
|
|
2366
2489
|
}
|
|
2367
2490
|
|
|
2368
2491
|
async function fleetClean(args, context) {
|
|
2369
2492
|
const usage =
|
|
2370
|
-
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--
|
|
2493
|
+
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
|
|
2371
2494
|
const { options, positionals } = parseOptions(args, {
|
|
2372
2495
|
boolean: ["dry-run", "json"],
|
|
2373
2496
|
value: [
|
|
@@ -2375,6 +2498,8 @@ async function fleetClean(args, context) {
|
|
|
2375
2498
|
"ib-island",
|
|
2376
2499
|
"identity-file",
|
|
2377
2500
|
"confirm-clean",
|
|
2501
|
+
"policy",
|
|
2502
|
+
"policy-out",
|
|
2378
2503
|
"source-user",
|
|
2379
2504
|
"preserve-user",
|
|
2380
2505
|
"ssh-user",
|
|
@@ -2402,30 +2527,57 @@ async function fleetClean(args, context) {
|
|
|
2402
2527
|
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2403
2528
|
const sourceUsers = fleetAccountOptions(options.sourceUser, "--source-user");
|
|
2404
2529
|
const preserveUsers = fleetAccountOptions(options.preserveUser, "--preserve-user");
|
|
2530
|
+
const loadedPolicy = await readFleetCleanupPolicy(options.policy);
|
|
2531
|
+
const policyOut = optionProvided(options.policyOut)
|
|
2532
|
+
? expandUserPath(optionalStringOption(options.policyOut))
|
|
2533
|
+
: null;
|
|
2405
2534
|
const accountOverlap = sourceUsers.filter((user) => preserveUsers.includes(user));
|
|
2406
2535
|
if (accountOverlap.length) {
|
|
2407
2536
|
throw new Error(`Users cannot be both source and preserved: ${accountOverlap.join(", ")}.`);
|
|
2408
2537
|
}
|
|
2538
|
+
if (loadedPolicy && (sourceUsers.length || preserveUsers.length)) {
|
|
2539
|
+
throw new Error("Use either --policy or the legacy user shortcuts, not both.");
|
|
2540
|
+
}
|
|
2409
2541
|
const confirmHash = optionalStringOption(options.confirmClean);
|
|
2410
2542
|
if (confirmHash) {
|
|
2411
|
-
if (
|
|
2543
|
+
if (
|
|
2544
|
+
positionals.length !== 1 ||
|
|
2545
|
+
options.dryRun ||
|
|
2546
|
+
options.operator ||
|
|
2547
|
+
options.ibIsland ||
|
|
2548
|
+
loadedPolicy ||
|
|
2549
|
+
policyOut ||
|
|
2550
|
+
sourceUsers.length ||
|
|
2551
|
+
preserveUsers.length
|
|
2552
|
+
) {
|
|
2412
2553
|
throw new Error(usage);
|
|
2413
2554
|
}
|
|
2414
2555
|
if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
|
|
2415
2556
|
throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
|
|
2416
2557
|
}
|
|
2417
2558
|
const fleetId = positionals[0];
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2559
|
+
let approved;
|
|
2560
|
+
try {
|
|
2561
|
+
approved = await operatorRequest({
|
|
2562
|
+
body: {
|
|
2563
|
+
plan_hash: confirmHash,
|
|
2564
|
+
},
|
|
2565
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
|
|
2566
|
+
env: context.env,
|
|
2567
|
+
fetchImpl: context.fetchImpl,
|
|
2568
|
+
method: "POST",
|
|
2569
|
+
});
|
|
2570
|
+
} catch (error) {
|
|
2571
|
+
if (
|
|
2572
|
+
error instanceof CliApiError &&
|
|
2573
|
+
error.detail?.detail === "cleanup_plan_replan_required"
|
|
2574
|
+
) {
|
|
2575
|
+
throw new Error(
|
|
2576
|
+
`Fleet ${fleetId} has a legacy cleanup plan that the current runner cannot safely execute. Re-run the same fleet with --dry-run, review the generated resource policy, then approve the new plan hash.`,
|
|
2577
|
+
);
|
|
2578
|
+
}
|
|
2579
|
+
throw error;
|
|
2580
|
+
}
|
|
2429
2581
|
context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
|
|
2430
2582
|
const results = await mapLimit(approved.nodes, parallel, async (node) => {
|
|
2431
2583
|
let cleanupResult;
|
|
@@ -2452,7 +2604,7 @@ async function fleetClean(args, context) {
|
|
|
2452
2604
|
});
|
|
2453
2605
|
const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
|
|
2454
2606
|
let result;
|
|
2455
|
-
let
|
|
2607
|
+
let duringCanary = { failedChecks: 0, successfulChecks: 0 };
|
|
2456
2608
|
try {
|
|
2457
2609
|
result = await runFleetSshCapture({
|
|
2458
2610
|
context,
|
|
@@ -2465,7 +2617,10 @@ async function fleetClean(args, context) {
|
|
|
2465
2617
|
user: node.ssh_user,
|
|
2466
2618
|
});
|
|
2467
2619
|
} finally {
|
|
2468
|
-
|
|
2620
|
+
duringCanary = await stopCanary();
|
|
2621
|
+
}
|
|
2622
|
+
if (result.exitCode === 124) {
|
|
2623
|
+
throw new Error(`cleanup_execution_deferred: ${result.stderr}`);
|
|
2469
2624
|
}
|
|
2470
2625
|
const payload = fleetCleanupJson(result.stdout);
|
|
2471
2626
|
if (result.exitCode !== 0 || !payload.evidence) {
|
|
@@ -2485,7 +2640,8 @@ async function fleetClean(args, context) {
|
|
|
2485
2640
|
}
|
|
2486
2641
|
payload.evidence.management_ssh_canary = {
|
|
2487
2642
|
before: true,
|
|
2488
|
-
during_checks:
|
|
2643
|
+
during_checks: duringCanary.successfulChecks,
|
|
2644
|
+
during_failures: duringCanary.failedChecks,
|
|
2489
2645
|
after: true,
|
|
2490
2646
|
};
|
|
2491
2647
|
cleanupResult = {
|
|
@@ -2498,7 +2654,11 @@ async function fleetClean(args, context) {
|
|
|
2498
2654
|
cleanupResult = {
|
|
2499
2655
|
cleanup_run_id: node.cleanup_run_id,
|
|
2500
2656
|
succeeded: false,
|
|
2501
|
-
...(
|
|
2657
|
+
...(/cleanup_already_running|cleanup_execution_deferred|SSH command timed out/i.test(
|
|
2658
|
+
message,
|
|
2659
|
+
)
|
|
2660
|
+
? { deferred: true }
|
|
2661
|
+
: {}),
|
|
2502
2662
|
error: message,
|
|
2503
2663
|
};
|
|
2504
2664
|
}
|
|
@@ -2543,12 +2703,16 @@ async function fleetClean(args, context) {
|
|
|
2543
2703
|
env: context.env,
|
|
2544
2704
|
fetchImpl: context.fetchImpl,
|
|
2545
2705
|
});
|
|
2546
|
-
if (fleetRecord.status
|
|
2547
|
-
throw new Error("Only a failed fleet can be replanned
|
|
2706
|
+
if (!["failed", "cleaning"].includes(fleetRecord.status)) {
|
|
2707
|
+
throw new Error("Only a failed or still-planning fleet can be replanned.");
|
|
2548
2708
|
}
|
|
2549
|
-
planningNodes = fleetRecord.nodes.filter((node) =>
|
|
2709
|
+
planningNodes = fleetRecord.nodes.filter((node) =>
|
|
2710
|
+
fleetRecord.status === "failed"
|
|
2711
|
+
? node.status === "failed"
|
|
2712
|
+
: ["pending_clean", "clean_planned"].includes(node.status),
|
|
2713
|
+
);
|
|
2550
2714
|
if (!planningNodes.length) {
|
|
2551
|
-
throw new Error("This fleet has no
|
|
2715
|
+
throw new Error("This fleet has no cleanup members eligible for replanning.");
|
|
2552
2716
|
}
|
|
2553
2717
|
} else {
|
|
2554
2718
|
const ips = uniqueFleetIps(positionals);
|
|
@@ -2589,12 +2753,16 @@ async function fleetClean(args, context) {
|
|
|
2589
2753
|
...sourceUsers.map((user) => `--source-user ${shellQuote(user)}`),
|
|
2590
2754
|
...preserveUsers.map((user) => `--preserve-user ${shellQuote(user)}`),
|
|
2591
2755
|
].join(" ");
|
|
2756
|
+
const nodePolicy = cleanupPolicyForNode(loadedPolicy, node, planningNodes.length);
|
|
2757
|
+
const policyArgument = nodePolicy
|
|
2758
|
+
? ` --policy-b64 ${shellQuote(Buffer.from(JSON.stringify(nodePolicy), "utf8").toString("base64"))}`
|
|
2759
|
+
: "";
|
|
2592
2760
|
const result = await runFleetSshCapture({
|
|
2593
2761
|
context,
|
|
2594
2762
|
identityFile,
|
|
2595
2763
|
ip: node.ip_address,
|
|
2596
2764
|
port: sshPort,
|
|
2597
|
-
remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}`,
|
|
2765
|
+
remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}${policyArgument}`,
|
|
2598
2766
|
stdin: runner.script,
|
|
2599
2767
|
timeoutSeconds,
|
|
2600
2768
|
user: sshUser,
|
|
@@ -2604,6 +2772,9 @@ async function fleetClean(args, context) {
|
|
|
2604
2772
|
throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
|
|
2605
2773
|
}
|
|
2606
2774
|
return {
|
|
2775
|
+
ip: node.ip_address,
|
|
2776
|
+
observations: payload.observations || {},
|
|
2777
|
+
policyTemplate: payload.policy_template,
|
|
2607
2778
|
plan: {
|
|
2608
2779
|
fleet_node_id: node.id,
|
|
2609
2780
|
ssh_user: sshUser,
|
|
@@ -2622,6 +2793,23 @@ async function fleetClean(args, context) {
|
|
|
2622
2793
|
);
|
|
2623
2794
|
}
|
|
2624
2795
|
const plans = planned.map((result) => result.plan);
|
|
2796
|
+
const policyTemplate = {
|
|
2797
|
+
schema_version: 1,
|
|
2798
|
+
nodes: { ...(loadedPolicy?.policy?.nodes || {}) },
|
|
2799
|
+
};
|
|
2800
|
+
const observations = {};
|
|
2801
|
+
for (const result of [...planned].sort((left, right) => left.ip.localeCompare(right.ip))) {
|
|
2802
|
+
policyTemplate.nodes[result.ip] = result.policyTemplate;
|
|
2803
|
+
observations[result.ip] = result.observations;
|
|
2804
|
+
}
|
|
2805
|
+
const savedPolicyPath =
|
|
2806
|
+
policyOut || join(getFleetConfigDir(context.env), `${fleetRecord.id}.cleanup-policy.json`);
|
|
2807
|
+
await mkdir(dirname(savedPolicyPath), { recursive: true, mode: 0o700 });
|
|
2808
|
+
await writeFile(savedPolicyPath, `${JSON.stringify(policyTemplate, null, 2)}\n`, {
|
|
2809
|
+
encoding: "utf8",
|
|
2810
|
+
mode: 0o600,
|
|
2811
|
+
});
|
|
2812
|
+
await chmod(savedPolicyPath, 0o600);
|
|
2625
2813
|
recorded = await operatorRequest({
|
|
2626
2814
|
body: { nodes: plans },
|
|
2627
2815
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
|
|
@@ -2629,10 +2817,13 @@ async function fleetClean(args, context) {
|
|
|
2629
2817
|
fetchImpl: context.fetchImpl,
|
|
2630
2818
|
method: "POST",
|
|
2631
2819
|
});
|
|
2820
|
+
recorded.policy_template = policyTemplate;
|
|
2821
|
+
recorded.observations = observations;
|
|
2822
|
+
recorded.policy_file = savedPolicyPath;
|
|
2632
2823
|
} catch (error) {
|
|
2633
2824
|
const planningError = fleetResultError(error);
|
|
2634
2825
|
if (!createdFleet) {
|
|
2635
|
-
throw new Error(`${planningError} Fleet ${fleetRecord.id}
|
|
2826
|
+
throw new Error(`${planningError} Fleet ${fleetRecord.id} can be replanned after correction.`);
|
|
2636
2827
|
}
|
|
2637
2828
|
let failedFleet;
|
|
2638
2829
|
try {
|
|
@@ -2673,9 +2864,24 @@ async function fleetClean(args, context) {
|
|
|
2673
2864
|
}
|
|
2674
2865
|
}
|
|
2675
2866
|
context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
|
|
2676
|
-
context.stdout.write(
|
|
2677
|
-
|
|
2867
|
+
context.stdout.write(`Policy template: ${recorded.policy_file}\n`);
|
|
2868
|
+
const unresolved = recorded.fleet.nodes.reduce(
|
|
2869
|
+
(count, node) => count + (node.cleanup?.plan?.unresolved?.length || 0),
|
|
2870
|
+
0,
|
|
2871
|
+
);
|
|
2872
|
+
const conflicts = recorded.fleet.nodes.reduce(
|
|
2873
|
+
(count, node) => count + (node.cleanup?.plan?.conflicts?.length || 0),
|
|
2874
|
+
0,
|
|
2678
2875
|
);
|
|
2876
|
+
if (unresolved || conflicts) {
|
|
2877
|
+
context.stdout.write(
|
|
2878
|
+
`Decisions required: ${unresolved}; policy conflicts: ${conflicts}. Edit the policy template and re-run the dry-run for fleet ${fleetRecord.id}.\n`,
|
|
2879
|
+
);
|
|
2880
|
+
} else {
|
|
2881
|
+
context.stdout.write(
|
|
2882
|
+
`Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`,
|
|
2883
|
+
);
|
|
2884
|
+
}
|
|
2679
2885
|
}
|
|
2680
2886
|
return 0;
|
|
2681
2887
|
}
|
|
@@ -3431,12 +3637,11 @@ async function deriveFleetManagementPublicKey({ context, identityFile }) {
|
|
|
3431
3637
|
}
|
|
3432
3638
|
|
|
3433
3639
|
function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3434
|
-
let
|
|
3640
|
+
let failedChecks = 0;
|
|
3435
3641
|
let successfulChecks = 0;
|
|
3436
3642
|
let inFlight = Promise.resolve();
|
|
3437
3643
|
const check = () => {
|
|
3438
3644
|
inFlight = inFlight.then(async () => {
|
|
3439
|
-
if (failure) return;
|
|
3440
3645
|
const exitCode = await runFleetSsh({
|
|
3441
3646
|
context,
|
|
3442
3647
|
identityFile,
|
|
@@ -3447,7 +3652,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
|
|
|
3447
3652
|
user,
|
|
3448
3653
|
});
|
|
3449
3654
|
if (exitCode !== 0) {
|
|
3450
|
-
|
|
3655
|
+
failedChecks += 1;
|
|
3451
3656
|
} else {
|
|
3452
3657
|
successfulChecks += 1;
|
|
3453
3658
|
}
|
|
@@ -3459,8 +3664,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
|
|
|
3459
3664
|
return async () => {
|
|
3460
3665
|
clearInterval(timer);
|
|
3461
3666
|
await inFlight;
|
|
3462
|
-
|
|
3463
|
-
return successfulChecks;
|
|
3667
|
+
return { failedChecks, successfulChecks };
|
|
3464
3668
|
};
|
|
3465
3669
|
}
|
|
3466
3670
|
|
|
@@ -5940,14 +6144,17 @@ function inventoryAvailableGpuCount(record) {
|
|
|
5940
6144
|
|
|
5941
6145
|
function normalizeInventoryListing(record) {
|
|
5942
6146
|
const gpuCount = inventoryAvailableGpuCount(record);
|
|
6147
|
+
const facility = record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility";
|
|
6148
|
+
// Customer availability never exposes real supplier/operator identity.
|
|
6149
|
+
const operator = "Ornn Compute";
|
|
5943
6150
|
return {
|
|
5944
6151
|
id: record.id,
|
|
5945
6152
|
kind: "inventory",
|
|
5946
6153
|
source: "primary",
|
|
5947
6154
|
gpu_type: record.gpu_type ?? "GPU",
|
|
5948
6155
|
gpu_count: gpuCount || null,
|
|
5949
|
-
operator
|
|
5950
|
-
facility
|
|
6156
|
+
operator,
|
|
6157
|
+
facility,
|
|
5951
6158
|
location: record.location ?? record.region ?? null,
|
|
5952
6159
|
network: record.fabric_type ?? record.network_hardware ?? record.internet ?? null,
|
|
5953
6160
|
start_date: record.available_from ?? null,
|
|
@@ -5955,7 +6162,8 @@ function normalizeInventoryListing(record) {
|
|
|
5955
6162
|
price_per_gpu_hour: resolveBuyNowUsdPerGpuHour(record),
|
|
5956
6163
|
checkout_url: `/checkout?inventory=${encodeURIComponent(record.id)}`,
|
|
5957
6164
|
marketplace_url: `/marketplace/${encodeURIComponent(record.id)}`,
|
|
5958
|
-
inventory
|
|
6165
|
+
// Strip supplier identity before echoing the raw inventory row in --json.
|
|
6166
|
+
inventory: { ...record, site_operator: operator },
|
|
5959
6167
|
};
|
|
5960
6168
|
}
|
|
5961
6169
|
|