@ornncompute/cli 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +49 -20
  2. package/package.json +1 -1
  3. package/src/cli.mjs +248 -40
package/README.md CHANGED
@@ -56,10 +56,11 @@ ornn nodes keys list <node-id> [--json]
56
56
  ornn node health <node-id> [--json]
57
57
  ornn node diagnose <node-id> [--json]
58
58
  ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]
59
- ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] [--json]
60
- ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--json]
61
- ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]... [--json]
59
+ ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] [--json]
60
+ ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
61
+ ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
62
62
  ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
63
+ ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]
63
64
  ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
64
65
  ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
65
66
  ornn fleet deploy <fleet-id> --tenant <contact-email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
@@ -227,28 +228,56 @@ Fleet handoff is an ordered, Compute-backed workflow:
227
228
  - Before first contact, independently verify each host key and add it to the
228
229
  operator's SSH `known_hosts`; fleet SSH uses strict host-key checking and will
229
230
  not trust a key first presented during cleanup.
230
- - `fleet clean ... --dry-run` tries `ubuntu`, `admin`, then `ornn`, records each
231
- host's hardware identity and exact deletion plan, and prints one fleet plan
232
- hash. Repeat `--source-user` for Linux accounts owned by the departing tenant
233
- (including an explicitly reviewed orphaned `/home/<user>` directory) and
234
- `--preserve-user` for reviewed host accounts. Package/service-owned
235
- accounts are protected automatically; any other account stops planning for
236
- explicit review. Re-running with `--confirm-clean <plan-hash>` and the same
237
- account flags removes the approved tenant
238
- users, keys, containers/images/volumes/networks/build cache, workload paths,
239
- mounts, services, and tenant-scoped logs. Apply refuses plans with running
240
- containers, locks the management/root passwords, enforces key-only SSH, trims
241
- supported filesystems, preserves existing root and management keys and host
242
- logs, and stores per-node proof in Compute. A failed
243
- cleanup can be dry-run again with the same fleet ID; only failed members are
244
- replanned. Residual GPU
245
- consumers or tenant/platform listeners fail verification instead of being
246
- silently accepted.
231
+ - `fleet clean ... --dry-run` tries `ubuntu`, `admin`, then `ornn`, binds the
232
+ inventory to the host's hardware identity, records the plan in Compute, and
233
+ writes an editable mode-`0600` policy beside the local fleet manifest (or to
234
+ `--policy-out`). Inventory covers accounts and orphan homes, authorized SSH
235
+ keys, tenant/log/temp/platform paths, workload and listener processes, GPU
236
+ consumers, custom systemd services, Docker/Podman objects and bind-mount
237
+ dependencies, and every block drive. Each resource advertises its own valid
238
+ actions. Suggestions are never applied silently: unresolved choices and
239
+ conflicts are recorded as plan data, so edit the policy and re-run the same
240
+ fleet with `--policy`; planning is not terminalized just because a choice is
241
+ needed. `--source-user` and `--preserve-user` remain legacy user-only
242
+ shortcuts and cannot be mixed with a policy file.
243
+ - `fleet clean <fleet-id> ... --confirm-clean <plan-hash>` is shown only after
244
+ every choice is resolved. Execution rechecks hardware and resource identity,
245
+ runs an idempotent per-resource journal, preserves the management SSH key,
246
+ enforces key-only SSH, and returns exact deleted, preserved, exception, and
247
+ drive-assurance evidence. `ignore` requires a reason and produces
248
+ `clean_with_exceptions`; it is never reported as residue-free deletion.
249
+ Data-drive actions are selectable: `logical_clean`, `discard`, `sanitize`, or
250
+ `reformat`. The system drive and drives without a unique serial/WWN are
251
+ locked to keep/ignore for in-band cleanup. `sanitize` requires NVMe plus
252
+ controller-reported completion. Logical cleanup, discard, and reformat are
253
+ not certified physical-media erasure, and the system drive requires an
254
+ out-of-band reimage for that assurance.
255
+ If the local SSH command times out while a long device operation is still in
256
+ progress, the recorded run remains resumable; re-run the same
257
+ `--confirm-clean <plan-hash>` command instead of creating a different plan.
258
+ Plans created by a pre-v6 runner must be replanned with `--dry-run` before
259
+ approval. This keeps newly discovered resources bound to a fresh reviewed
260
+ hash instead of adapting a legacy account-only plan during execution.
261
+
262
+ In the generated policy, change only `action`, plus `reason` for `ignore` or
263
+ `replacement` for `replace`; keep each generated `identity_hash` intact. `keep`
264
+ leaves a resource running, `stop_keep` stops but retains it, `stop` terminates a
265
+ process, `delete` removes it, `restart` restarts a service, and `replace`
266
+ recreates a container from the supplied image/name/network/command/labels/mounts
267
+ specification. The generated `suggested_action` is review guidance, not an
268
+ automatic decision. Do not put secrets in replacement commands or labels; use
269
+ the platform secret manager or a reviewed secret mount.
270
+
247
271
  - `fleet enroll <fleet-id>` refuses nodes without successful cleanup proof,
248
272
  issues and revokes one cleanup-bound token per host, installs the agent,
249
273
  assigns the IB island, verifies the same management account still has
250
274
  key-based SSH and passwordless sudo, and records the resulting GPU node IDs
251
275
  in Compute. Generic enrollment tokens retain the legacy enrollment sanitizer.
276
+ - `fleet delete <fleet-id> --confirm-delete <fleet-id>` hard-deletes only an
277
+ abandoned cleanup plan or an unassigned legacy enrollment record. It saves a
278
+ local audit manifest first and refuses any Commerce/tenant/deployment link,
279
+ active enrollment, assigned GPU node, or completed/running cleanup evidence.
280
+ It never deletes the linked GPU-node records.
252
281
  - `reservations commerce create --tenant <email> --listing <listing-id>
253
282
  --fleet <fleet-id>` creates and activates the tenant's Commerce reservation.
254
283
  GPU count comes from the enrolled fleet; start, end, and hourly rate default
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ornncompute/cli",
3
- "version": "0.1.7",
3
+ "version": "0.1.8",
4
4
  "description": "Command-line interface for Ornn compute access workflows.",
5
5
  "type": "module",
6
6
  "bin": {
package/src/cli.mjs CHANGED
@@ -1,11 +1,11 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { randomUUID } from "node:crypto";
3
3
  import { createReadStream } from "node:fs";
4
- import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
4
+ import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
5
5
  import { createRequire } from "node:module";
6
6
  import { isIP } from "node:net";
7
7
  import { homedir, tmpdir } from "node:os";
8
- import { basename, join } from "node:path";
8
+ import { basename, dirname, join } from "node:path";
9
9
 
10
10
  import {
11
11
  CliApiError,
@@ -132,11 +132,12 @@ Usage:
132
132
  ornn tokens list [--json]
133
133
  ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
134
134
  ornn tokens revoke <token-id> [--json]
135
- ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] [--json]
136
- ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--json]
137
- ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]... [--json]
135
+ ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] [--json]
136
+ ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
137
+ ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
138
138
  ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
139
139
  ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
140
+ ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]
140
141
  ornn reservations withdraw <reservation-id> [--json]
141
142
  ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
142
143
  ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
@@ -645,7 +646,12 @@ function validateHelpInvocation(command, args) {
645
646
  "--ib-island",
646
647
  "--identity-file",
647
648
  "--confirm-clean",
649
+ "--confirm-delete",
648
650
  "--confirm-takeover",
651
+ "--policy",
652
+ "--policy-out",
653
+ "--source-user",
654
+ "--preserve-user",
649
655
  "--ssh-user",
650
656
  "--ssh-port",
651
657
  "--parallel",
@@ -675,7 +681,12 @@ function validateHelpInvocation(command, args) {
675
681
  ? null
676
682
  : "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id>";
677
683
  }
678
- return "Usage: ornn fleet clean|enroll|deploy";
684
+ if (subcommand === "delete") {
685
+ return positionals.length === 1
686
+ ? null
687
+ : "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id>";
688
+ }
689
+ return "Usage: ornn fleet clean|enroll|deploy|delete";
679
690
  }
680
691
 
681
692
  if (command === "nodes") {
@@ -2351,6 +2362,55 @@ function fleetAccountOptions(value, optionName) {
2351
2362
  return [...new Set(normalized)].sort();
2352
2363
  }
2353
2364
 
2365
+ async function readFleetCleanupPolicy(pathValue) {
2366
+ if (!optionProvided(pathValue)) return null;
2367
+ const path = expandUserPath(optionalStringOption(pathValue));
2368
+ let policy;
2369
+ try {
2370
+ policy = JSON.parse(await readFile(path, "utf8"));
2371
+ } catch (error) {
2372
+ throw new Error(`Could not read cleanup policy ${path}: ${fleetResultError(error)}`);
2373
+ }
2374
+ if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
2375
+ throw new Error("Cleanup policy must be a JSON object.");
2376
+ }
2377
+ if (policy.schema_version !== 1) {
2378
+ throw new Error("Cleanup policy schema_version must be 1.");
2379
+ }
2380
+ const direct =
2381
+ policy.resources && typeof policy.resources === "object" && !Array.isArray(policy.resources);
2382
+ const perNode = policy.nodes && typeof policy.nodes === "object" && !Array.isArray(policy.nodes);
2383
+ if (Boolean(direct) === Boolean(perNode)) {
2384
+ throw new Error("Cleanup policy must contain exactly one of resources or nodes.");
2385
+ }
2386
+ return { path, policy };
2387
+ }
2388
+
2389
+ function cleanupPolicyForNode(loaded, node, nodeCount) {
2390
+ if (!loaded) return null;
2391
+ if (loaded.policy.resources) {
2392
+ if (nodeCount !== 1) {
2393
+ throw new Error(
2394
+ "A direct resources policy can only be used for one node; use nodes keyed by IP for a fleet.",
2395
+ );
2396
+ }
2397
+ return loaded.policy;
2398
+ }
2399
+ const policy = loaded.policy.nodes[node.ip_address] || loaded.policy.nodes[node.id];
2400
+ if (!policy) {
2401
+ throw new Error(`Cleanup policy has no entry for ${node.ip_address}.`);
2402
+ }
2403
+ if (
2404
+ policy.schema_version !== 1 ||
2405
+ !policy.resources ||
2406
+ typeof policy.resources !== "object" ||
2407
+ Array.isArray(policy.resources)
2408
+ ) {
2409
+ throw new Error(`Cleanup policy entry for ${node.ip_address} is invalid.`);
2410
+ }
2411
+ return policy;
2412
+ }
2413
+
2354
2414
  async function fleet(args, context) {
2355
2415
  const [subcommand, ...rest] = args;
2356
2416
  if (subcommand === "clean") {
@@ -2362,12 +2422,75 @@ async function fleet(args, context) {
2362
2422
  if (subcommand === "deploy") {
2363
2423
  return await fleetDeploy(rest, context);
2364
2424
  }
2365
- throw new Error("Usage: ornn fleet clean|enroll|deploy");
2425
+ if (subcommand === "delete") {
2426
+ return await fleetDelete(rest, context);
2427
+ }
2428
+ throw new Error("Usage: ornn fleet clean|enroll|deploy|delete");
2429
+ }
2430
+
2431
+ async function fleetDelete(args, context) {
2432
+ const usage = "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]";
2433
+ const { options, positionals } = parseOptions(args, {
2434
+ boolean: ["json"],
2435
+ value: ["confirm-delete"],
2436
+ });
2437
+ if (positionals.length !== 1) {
2438
+ throw new Error(usage);
2439
+ }
2440
+ const fleetId = positionals[0];
2441
+ if (!isUuid(fleetId)) {
2442
+ throw new Error("<fleet-id> must be a UUID.");
2443
+ }
2444
+ const confirmation = requiredOption(options.confirmDelete, "--confirm-delete");
2445
+ if (confirmation !== fleetId) {
2446
+ throw new Error("--confirm-delete must exactly match <fleet-id>.");
2447
+ }
2448
+
2449
+ const preview = await operatorRequest({
2450
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
2451
+ env: context.env,
2452
+ fetchImpl: context.fetchImpl,
2453
+ });
2454
+ if (String(preview?.id || "") !== fleetId) {
2455
+ throw new Error("Fleet lookup returned a different fleet id; deletion stopped.");
2456
+ }
2457
+ const manifestPath = await saveFleetManifest(
2458
+ {
2459
+ ...preview,
2460
+ deletion: { requested_at: new Date().toISOString(), status: "requested" },
2461
+ },
2462
+ context.env,
2463
+ );
2464
+ const result = await operatorRequest({
2465
+ body: { confirm_fleet_id: confirmation },
2466
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
2467
+ env: context.env,
2468
+ fetchImpl: context.fetchImpl,
2469
+ method: "DELETE",
2470
+ });
2471
+ const archived = {
2472
+ ...(result?.fleet || preview),
2473
+ deletion: {
2474
+ deleted_at: result?.deleted_at || new Date().toISOString(),
2475
+ status: "deleted",
2476
+ },
2477
+ };
2478
+ await saveFleetManifest(archived, context.env);
2479
+ if (options.json) {
2480
+ writeJson(context.stdout, { ...result, audit_manifest: manifestPath });
2481
+ } else {
2482
+ context.stdout.write(`Deleted fleet ${fleetId}.\n`);
2483
+ context.stdout.write(
2484
+ `Released members: ${Array.isArray(archived.nodes) ? archived.nodes.length : 0}\n`,
2485
+ );
2486
+ context.stdout.write(`Audit manifest: ${manifestPath}\n`);
2487
+ }
2488
+ return 0;
2366
2489
  }
2367
2490
 
2368
2491
  async function fleetClean(args, context) {
2369
2492
  const usage =
2370
- "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]...";
2493
+ "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
2371
2494
  const { options, positionals } = parseOptions(args, {
2372
2495
  boolean: ["dry-run", "json"],
2373
2496
  value: [
@@ -2375,6 +2498,8 @@ async function fleetClean(args, context) {
2375
2498
  "ib-island",
2376
2499
  "identity-file",
2377
2500
  "confirm-clean",
2501
+ "policy",
2502
+ "policy-out",
2378
2503
  "source-user",
2379
2504
  "preserve-user",
2380
2505
  "ssh-user",
@@ -2402,30 +2527,57 @@ async function fleetClean(args, context) {
2402
2527
  : FLEET_DEFAULT_TIMEOUT_SECONDS;
2403
2528
  const sourceUsers = fleetAccountOptions(options.sourceUser, "--source-user");
2404
2529
  const preserveUsers = fleetAccountOptions(options.preserveUser, "--preserve-user");
2530
+ const loadedPolicy = await readFleetCleanupPolicy(options.policy);
2531
+ const policyOut = optionProvided(options.policyOut)
2532
+ ? expandUserPath(optionalStringOption(options.policyOut))
2533
+ : null;
2405
2534
  const accountOverlap = sourceUsers.filter((user) => preserveUsers.includes(user));
2406
2535
  if (accountOverlap.length) {
2407
2536
  throw new Error(`Users cannot be both source and preserved: ${accountOverlap.join(", ")}.`);
2408
2537
  }
2538
+ if (loadedPolicy && (sourceUsers.length || preserveUsers.length)) {
2539
+ throw new Error("Use either --policy or the legacy user shortcuts, not both.");
2540
+ }
2409
2541
  const confirmHash = optionalStringOption(options.confirmClean);
2410
2542
  if (confirmHash) {
2411
- if (positionals.length !== 1 || options.dryRun || options.operator || options.ibIsland) {
2543
+ if (
2544
+ positionals.length !== 1 ||
2545
+ options.dryRun ||
2546
+ options.operator ||
2547
+ options.ibIsland ||
2548
+ loadedPolicy ||
2549
+ policyOut ||
2550
+ sourceUsers.length ||
2551
+ preserveUsers.length
2552
+ ) {
2412
2553
  throw new Error(usage);
2413
2554
  }
2414
2555
  if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
2415
2556
  throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
2416
2557
  }
2417
2558
  const fleetId = positionals[0];
2418
- const approved = await operatorRequest({
2419
- body: {
2420
- plan_hash: confirmHash,
2421
- source_users: sourceUsers,
2422
- preserve_users: preserveUsers,
2423
- },
2424
- endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
2425
- env: context.env,
2426
- fetchImpl: context.fetchImpl,
2427
- method: "POST",
2428
- });
2559
+ let approved;
2560
+ try {
2561
+ approved = await operatorRequest({
2562
+ body: {
2563
+ plan_hash: confirmHash,
2564
+ },
2565
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
2566
+ env: context.env,
2567
+ fetchImpl: context.fetchImpl,
2568
+ method: "POST",
2569
+ });
2570
+ } catch (error) {
2571
+ if (
2572
+ error instanceof CliApiError &&
2573
+ error.detail?.detail === "cleanup_plan_replan_required"
2574
+ ) {
2575
+ throw new Error(
2576
+ `Fleet ${fleetId} has a legacy cleanup plan that the current runner cannot safely execute. Re-run the same fleet with --dry-run, review the generated resource policy, then approve the new plan hash.`,
2577
+ );
2578
+ }
2579
+ throw error;
2580
+ }
2429
2581
  context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
2430
2582
  const results = await mapLimit(approved.nodes, parallel, async (node) => {
2431
2583
  let cleanupResult;
@@ -2452,7 +2604,7 @@ async function fleetClean(args, context) {
2452
2604
  });
2453
2605
  const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
2454
2606
  let result;
2455
- let duringCanaryChecks = 0;
2607
+ let duringCanary = { failedChecks: 0, successfulChecks: 0 };
2456
2608
  try {
2457
2609
  result = await runFleetSshCapture({
2458
2610
  context,
@@ -2465,7 +2617,10 @@ async function fleetClean(args, context) {
2465
2617
  user: node.ssh_user,
2466
2618
  });
2467
2619
  } finally {
2468
- duringCanaryChecks = await stopCanary();
2620
+ duringCanary = await stopCanary();
2621
+ }
2622
+ if (result.exitCode === 124) {
2623
+ throw new Error(`cleanup_execution_deferred: ${result.stderr}`);
2469
2624
  }
2470
2625
  const payload = fleetCleanupJson(result.stdout);
2471
2626
  if (result.exitCode !== 0 || !payload.evidence) {
@@ -2485,7 +2640,8 @@ async function fleetClean(args, context) {
2485
2640
  }
2486
2641
  payload.evidence.management_ssh_canary = {
2487
2642
  before: true,
2488
- during_checks: duringCanaryChecks,
2643
+ during_checks: duringCanary.successfulChecks,
2644
+ during_failures: duringCanary.failedChecks,
2489
2645
  after: true,
2490
2646
  };
2491
2647
  cleanupResult = {
@@ -2498,7 +2654,11 @@ async function fleetClean(args, context) {
2498
2654
  cleanupResult = {
2499
2655
  cleanup_run_id: node.cleanup_run_id,
2500
2656
  succeeded: false,
2501
- ...(message.includes("cleanup_already_running") ? { deferred: true } : {}),
2657
+ ...(/cleanup_already_running|cleanup_execution_deferred|SSH command timed out/i.test(
2658
+ message,
2659
+ )
2660
+ ? { deferred: true }
2661
+ : {}),
2502
2662
  error: message,
2503
2663
  };
2504
2664
  }
@@ -2543,12 +2703,16 @@ async function fleetClean(args, context) {
2543
2703
  env: context.env,
2544
2704
  fetchImpl: context.fetchImpl,
2545
2705
  });
2546
- if (fleetRecord.status !== "failed") {
2547
- throw new Error("Only a failed fleet can be replanned; resume an approved cleanup with --confirm-clean.");
2706
+ if (!["failed", "cleaning"].includes(fleetRecord.status)) {
2707
+ throw new Error("Only a failed or still-planning fleet can be replanned.");
2548
2708
  }
2549
- planningNodes = fleetRecord.nodes.filter((node) => node.status === "failed");
2709
+ planningNodes = fleetRecord.nodes.filter((node) =>
2710
+ fleetRecord.status === "failed"
2711
+ ? node.status === "failed"
2712
+ : ["pending_clean", "clean_planned"].includes(node.status),
2713
+ );
2550
2714
  if (!planningNodes.length) {
2551
- throw new Error("This fleet has no failed cleanup members to replan.");
2715
+ throw new Error("This fleet has no cleanup members eligible for replanning.");
2552
2716
  }
2553
2717
  } else {
2554
2718
  const ips = uniqueFleetIps(positionals);
@@ -2589,12 +2753,16 @@ async function fleetClean(args, context) {
2589
2753
  ...sourceUsers.map((user) => `--source-user ${shellQuote(user)}`),
2590
2754
  ...preserveUsers.map((user) => `--preserve-user ${shellQuote(user)}`),
2591
2755
  ].join(" ");
2756
+ const nodePolicy = cleanupPolicyForNode(loadedPolicy, node, planningNodes.length);
2757
+ const policyArgument = nodePolicy
2758
+ ? ` --policy-b64 ${shellQuote(Buffer.from(JSON.stringify(nodePolicy), "utf8").toString("base64"))}`
2759
+ : "";
2592
2760
  const result = await runFleetSshCapture({
2593
2761
  context,
2594
2762
  identityFile,
2595
2763
  ip: node.ip_address,
2596
2764
  port: sshPort,
2597
- remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}`,
2765
+ remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}${policyArgument}`,
2598
2766
  stdin: runner.script,
2599
2767
  timeoutSeconds,
2600
2768
  user: sshUser,
@@ -2604,6 +2772,9 @@ async function fleetClean(args, context) {
2604
2772
  throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
2605
2773
  }
2606
2774
  return {
2775
+ ip: node.ip_address,
2776
+ observations: payload.observations || {},
2777
+ policyTemplate: payload.policy_template,
2607
2778
  plan: {
2608
2779
  fleet_node_id: node.id,
2609
2780
  ssh_user: sshUser,
@@ -2622,6 +2793,23 @@ async function fleetClean(args, context) {
2622
2793
  );
2623
2794
  }
2624
2795
  const plans = planned.map((result) => result.plan);
2796
+ const policyTemplate = {
2797
+ schema_version: 1,
2798
+ nodes: { ...(loadedPolicy?.policy?.nodes || {}) },
2799
+ };
2800
+ const observations = {};
2801
+ for (const result of [...planned].sort((left, right) => left.ip.localeCompare(right.ip))) {
2802
+ policyTemplate.nodes[result.ip] = result.policyTemplate;
2803
+ observations[result.ip] = result.observations;
2804
+ }
2805
+ const savedPolicyPath =
2806
+ policyOut || join(getFleetConfigDir(context.env), `${fleetRecord.id}.cleanup-policy.json`);
2807
+ await mkdir(dirname(savedPolicyPath), { recursive: true, mode: 0o700 });
2808
+ await writeFile(savedPolicyPath, `${JSON.stringify(policyTemplate, null, 2)}\n`, {
2809
+ encoding: "utf8",
2810
+ mode: 0o600,
2811
+ });
2812
+ await chmod(savedPolicyPath, 0o600);
2625
2813
  recorded = await operatorRequest({
2626
2814
  body: { nodes: plans },
2627
2815
  endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
@@ -2629,10 +2817,13 @@ async function fleetClean(args, context) {
2629
2817
  fetchImpl: context.fetchImpl,
2630
2818
  method: "POST",
2631
2819
  });
2820
+ recorded.policy_template = policyTemplate;
2821
+ recorded.observations = observations;
2822
+ recorded.policy_file = savedPolicyPath;
2632
2823
  } catch (error) {
2633
2824
  const planningError = fleetResultError(error);
2634
2825
  if (!createdFleet) {
2635
- throw new Error(`${planningError} Fleet ${fleetRecord.id} remains failed and can be replanned.`);
2826
+ throw new Error(`${planningError} Fleet ${fleetRecord.id} can be replanned after correction.`);
2636
2827
  }
2637
2828
  let failedFleet;
2638
2829
  try {
@@ -2673,9 +2864,24 @@ async function fleetClean(args, context) {
2673
2864
  }
2674
2865
  }
2675
2866
  context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
2676
- context.stdout.write(
2677
- `Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}${sourceUsers.map((user) => ` --source-user ${shellQuote(user)}`).join("")}${preserveUsers.map((user) => ` --preserve-user ${shellQuote(user)}`).join("")}\n`
2867
+ context.stdout.write(`Policy template: ${recorded.policy_file}\n`);
2868
+ const unresolved = recorded.fleet.nodes.reduce(
2869
+ (count, node) => count + (node.cleanup?.plan?.unresolved?.length || 0),
2870
+ 0,
2871
+ );
2872
+ const conflicts = recorded.fleet.nodes.reduce(
2873
+ (count, node) => count + (node.cleanup?.plan?.conflicts?.length || 0),
2874
+ 0,
2678
2875
  );
2876
+ if (unresolved || conflicts) {
2877
+ context.stdout.write(
2878
+ `Decisions required: ${unresolved}; policy conflicts: ${conflicts}. Edit the policy template and re-run the dry-run for fleet ${fleetRecord.id}.\n`,
2879
+ );
2880
+ } else {
2881
+ context.stdout.write(
2882
+ `Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`,
2883
+ );
2884
+ }
2679
2885
  }
2680
2886
  return 0;
2681
2887
  }
@@ -3431,12 +3637,11 @@ async function deriveFleetManagementPublicKey({ context, identityFile }) {
3431
3637
  }
3432
3638
 
3433
3639
  function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
3434
- let failure = null;
3640
+ let failedChecks = 0;
3435
3641
  let successfulChecks = 0;
3436
3642
  let inFlight = Promise.resolve();
3437
3643
  const check = () => {
3438
3644
  inFlight = inFlight.then(async () => {
3439
- if (failure) return;
3440
3645
  const exitCode = await runFleetSsh({
3441
3646
  context,
3442
3647
  identityFile,
@@ -3447,7 +3652,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
3447
3652
  user,
3448
3653
  });
3449
3654
  if (exitCode !== 0) {
3450
- failure = new Error("Management SSH canary failed during cleanup.");
3655
+ failedChecks += 1;
3451
3656
  } else {
3452
3657
  successfulChecks += 1;
3453
3658
  }
@@ -3459,8 +3664,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
3459
3664
  return async () => {
3460
3665
  clearInterval(timer);
3461
3666
  await inFlight;
3462
- if (failure) throw failure;
3463
- return successfulChecks;
3667
+ return { failedChecks, successfulChecks };
3464
3668
  };
3465
3669
  }
3466
3670
 
@@ -5940,14 +6144,17 @@ function inventoryAvailableGpuCount(record) {
5940
6144
 
5941
6145
  function normalizeInventoryListing(record) {
5942
6146
  const gpuCount = inventoryAvailableGpuCount(record);
6147
+ const facility = record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility";
6148
+ // Customer availability never exposes real supplier/operator identity.
6149
+ const operator = "Ornn Compute";
5943
6150
  return {
5944
6151
  id: record.id,
5945
6152
  kind: "inventory",
5946
6153
  source: "primary",
5947
6154
  gpu_type: record.gpu_type ?? "GPU",
5948
6155
  gpu_count: gpuCount || null,
5949
- operator: record.site_operator ?? record.operator ?? "Unknown operator",
5950
- facility: record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility",
6156
+ operator,
6157
+ facility,
5951
6158
  location: record.location ?? record.region ?? null,
5952
6159
  network: record.fabric_type ?? record.network_hardware ?? record.internet ?? null,
5953
6160
  start_date: record.available_from ?? null,
@@ -5955,7 +6162,8 @@ function normalizeInventoryListing(record) {
5955
6162
  price_per_gpu_hour: resolveBuyNowUsdPerGpuHour(record),
5956
6163
  checkout_url: `/checkout?inventory=${encodeURIComponent(record.id)}`,
5957
6164
  marketplace_url: `/marketplace/${encodeURIComponent(record.id)}`,
5958
- inventory: record,
6165
+ // Strip supplier identity before echoing the raw inventory row in --json.
6166
+ inventory: { ...record, site_operator: operator },
5959
6167
  };
5960
6168
  }
5961
6169