@ornncompute/cli 1.0.35 → 1.0.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -12
- package/package.json +1 -1
- package/src/admin-capacity-reservations.mjs +69 -139
- package/src/admin-node-lifecycle.mjs +48 -7
- package/src/admin-shell.mjs +101 -25
- package/src/catalog-shell.mjs +15 -20
- package/src/evidence-commands.mjs +30 -12
- package/src/kubernetes-commands.mjs +58 -16
- package/src/launch-mode-handlers.mjs +2 -2
- package/src/storage-commands.mjs +9 -1
- package/src/vm-commands.mjs +3 -3
- package/vendor/capabilities/capabilities/admin-capacity-reservations.js +36 -55
- package/vendor/capabilities/capabilities/admin.js +8 -6
- package/vendor/capabilities/capabilities/commerce-market.js +76 -101
- package/vendor/capabilities/capabilities/compute.js +4 -2
- package/vendor/capabilities/capabilities/listings.js +58 -38
- package/vendor/capabilities/capabilities/markets.js +7 -7
- package/vendor/capabilities/capabilities/native-listing.js +63 -0
- package/vendor/capabilities/capabilities/organization-invites.js +53 -0
- package/vendor/capabilities/capabilities/users.js +141 -31
- package/vendor/capabilities/cli-catalog.js +4 -2
- package/vendor/capabilities/cli-migrations.js +1 -2
- package/vendor/capabilities/software-catalog.js +52 -6
package/README.md
CHANGED
|
@@ -59,9 +59,8 @@ ornn resources
|
|
|
59
59
|
Advanced customer resources are registered under one noun and one verb:
|
|
60
60
|
|
|
61
61
|
```text
|
|
62
|
-
capacity-offers list | show |
|
|
62
|
+
capacity-offers list | show | terms | relist | delist
|
|
63
63
|
reservations list | show | invoices
|
|
64
|
-
fees show
|
|
65
64
|
spot books | book | trades | bid | orders | cancel
|
|
66
65
|
schedules list | show | buy
|
|
67
66
|
offers list | post | accept | cancel
|
|
@@ -82,12 +81,15 @@ filesystems deploy | show | wait | delete
|
|
|
82
81
|
`ssh-keys list` uses backend cursors; see [pagination](../../docs/cli.md#advanced-customer-resources).
|
|
83
82
|
|
|
84
83
|
Market commands call one gateway route each and print that route's JSON.
|
|
85
|
-
`capacity-offers` reads Commerce
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
price
|
|
84
|
+
`capacity-offers list|show|terms` reads native Commerce capacity
|
|
85
|
+
(`/commerce/v2/listings*`). List/show return parent fields with `capacityListingId`;
|
|
86
|
+
pass it as `--listing-id`. `terms` separately returns advertised financial choices
|
|
87
|
+
with `listingTermsId`, price, duration bounds, down payment and renewable flag.
|
|
88
|
+
List accepts limits of 1–100 and the owner cursor unchanged. Relisting uses
|
|
89
|
+
`POST /commerce/reservations/{id}/relist` with `nodeCount`, dates, optional node
|
|
90
|
+
selection and a `terms` array. Delisting uses the native parent’s `/delist` POST. `reservations` reads Commerce
|
|
89
91
|
reservations, schedule and spot alike. `spot`, `schedules`, and `offers` call
|
|
90
|
-
Markets (`/v1/markets`): place
|
|
92
|
+
Markets (`/v1/markets`): place 8-GPU spot bids (one full node each), buy a
|
|
91
93
|
secondary window at its asking price (`schedules buy`), post offers on a
|
|
92
94
|
primary listing's schedule, and accept or cancel offers on primaries you own.
|
|
93
95
|
Prices are integer micro-USD per GPU-hour (`1000000` is $1.00). Every write
|
|
@@ -99,17 +101,30 @@ ownership.
|
|
|
99
101
|
|
|
100
102
|
Only admins can create primary listings or reservations through
|
|
101
103
|
`ornn admin capacity-offers create` and `ornn admin reservations create`.
|
|
102
|
-
They post to the role-aware `POST /commerce/listings` and
|
|
104
|
+
They post to the role-aware `POST /commerce/v2/listings` and
|
|
103
105
|
`POST /commerce/reservations` with the admin's own session; Commerce enforces
|
|
104
106
|
the admin role. Both commands retain explicit commercial-term validation and
|
|
105
|
-
default-no human confirmation.
|
|
106
|
-
|
|
107
|
+
default-no human confirmation. Reservation creation takes `--customer UUID`,
|
|
108
|
+
`--listing-term UUID`, `--nodes COUNT`, `--gpus COUNT`, `--starts-at TIME`, and
|
|
109
|
+
`--transaction UUID`, plus optional `--ends-at TIME` and `--vm-disk UUID`.
|
|
110
|
+
Prices and down payments come from the selected listing term. Omit `--ends-at`
|
|
111
|
+
for hourly terms, and reuse the same transaction UUID and payload on retries.
|
|
112
|
+
Creation requires an eligible human terminal. Native capacity creation takes
|
|
113
|
+
`--gpu-slug`, `--capacity-node-count`, `--gpus-per-node`, `--facility-id`,
|
|
114
|
+
`--provisioning-type`, `--state`, `--terms` (a JSON array) and `--idempotency-key`,
|
|
115
|
+
plus optional `--organization-id`. Supply availability dates or `--days-to-rfs`.
|
|
116
|
+
Reuse the key and the same body on retries. Prices are integer micro-USD inside
|
|
117
|
+
terms, with explicit duration bounds, nullable deposit and renewable flag.
|
|
118
|
+
`ornn admin capacity-offers update` edits a selected term using `--listing-id`,
|
|
119
|
+
`--listing-terms-id`, the complete term fields and `--idempotency-key`. Financial
|
|
120
|
+
editing remains staff-only. The old customer `capacity-offers update` is removed.
|
|
107
121
|
|
|
108
122
|
All staff discovery and execution starts at `ornn admin`. A fresh staff preflight
|
|
109
|
-
runs before the catalog is shown. The
|
|
123
|
+
runs before the catalog is shown. The eight roots are:
|
|
110
124
|
|
|
111
125
|
```text
|
|
112
126
|
ornn admin users
|
|
127
|
+
ornn admin organizations
|
|
113
128
|
ornn admin suppliers
|
|
114
129
|
ornn admin facilities
|
|
115
130
|
ornn admin nodes
|
|
@@ -119,6 +134,18 @@ ornn admin reservations
|
|
|
119
134
|
```
|
|
120
135
|
|
|
121
136
|
Every `ornn admin` command, read or mutation, requires an admin session.
|
|
137
|
+
`ornn admin users list` and `ornn admin organizations list` search the
|
|
138
|
+
Authorization directory by person name, email, or company name, with bounded `--limit` and
|
|
139
|
+
`--offset` pagination. Users can be restricted by `--organization`; organization
|
|
140
|
+
results can be filtered by `--role`, `--approved`, or `--ids`.
|
|
141
|
+
|
|
142
|
+
`ornn admin organizations invite --organization UUID --email EMAIL` emails an
|
|
143
|
+
invitation immediately using the selected organization UUID from the directory.
|
|
144
|
+
It shares its schema and execution with MCP and Slack and uses the staff session
|
|
145
|
+
on `POST /v1/staff/invites`. The result includes the sign-in invitation path and
|
|
146
|
+
`email_sent`. If delivery is uncertain, the invitation remains valid; check
|
|
147
|
+
delivery or share the sign-in link before retrying. Use `--format json` for automation.
|
|
148
|
+
|
|
122
149
|
`ornn admin nodes --help` lists the authorized node commands. Node health
|
|
123
150
|
returns the node detail object; metrics preserve the owner `tier` and `windows`;
|
|
124
151
|
logs return `data.logs` and an opaque continuation cursor in `page.next_cursor`.
|
|
@@ -254,7 +281,7 @@ vNext is a clean cutover. Retired forms are never aliases: each fails before I/O
|
|
|
254
281
|
with one replacement and <https://docs.ornn.com/cli#migrating-to-vnext>.
|
|
255
282
|
|
|
256
283
|
| Retired form | Replacement |
|
|
257
|
-
|
|
|
284
|
+
| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------- |
|
|
258
285
|
| `ornn account`, `ornn whoami` | `ornn context` |
|
|
259
286
|
| `ornn catalog search`, `gpu-types`, `availability *`, `listing *`, `listings` read leaves | matching `ornn capacity-offers` leaf |
|
|
260
287
|
| `ornn listings create` | `ornn admin capacity-offers create` |
|
package/package.json
CHANGED
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
import { ADMIN_MUTATION_SAFETY, AdminMutationError, runAdminMutation } from "./admin-mutations.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { parseSemanticArguments } from "./interaction-runtime.mjs";
|
|
3
3
|
|
|
4
4
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
5
|
-
const DECIMAL = /^\d+(?:\.\d{1,6})?$/;
|
|
6
5
|
|
|
7
6
|
const USAGE = {
|
|
8
|
-
"listings.create":
|
|
9
|
-
"Usage: ornn admin capacity-offers create --gpu GPU --nodes COUNT --gpus-per-node COUNT --site SITE --starts-at TIME --ends-at TIME --price-per-gpu-hour USD [--supplier-price-per-gpu-hour USD] [--down-payment-cents CENTS] [--access-mode bare-metal|virtual-machine] [--yes]",
|
|
10
7
|
"reservations.create":
|
|
11
|
-
"Usage: ornn admin reservations create --
|
|
8
|
+
"Usage: ornn admin reservations create --customer ID --listing-term ID --nodes COUNT --gpus COUNT --starts-at TIME --transaction ID [--ends-at TIME] [--vm-disk ID] [--yes]",
|
|
12
9
|
};
|
|
13
|
-
const ACCESS_MODES = new Set(["bare-metal", "virtual-machine"]);
|
|
14
|
-
const STAFF_HARDWARE_CATALOG = "/catalog/hardware";
|
|
15
10
|
|
|
16
11
|
function parseOptions(argv, { values, positionals }, usage) {
|
|
17
12
|
const options = {};
|
|
@@ -81,88 +76,42 @@ function timestamp(options, name, usage) {
|
|
|
81
76
|
return new Date(parsed).toISOString();
|
|
82
77
|
}
|
|
83
78
|
|
|
84
|
-
|
|
79
|
+
// Preserve the microseconds Commerce uses to match a reservation to its accepted term.
|
|
80
|
+
function reservationTimestamp(options, name, usage) {
|
|
85
81
|
const value = required(options, name, usage);
|
|
86
|
-
|
|
82
|
+
const match = value.match(
|
|
83
|
+
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.(\d{1,6}))?(?:Z|[+-]\d{2}:\d{2})$/
|
|
84
|
+
);
|
|
85
|
+
if (!match) {
|
|
87
86
|
throw new AdminMutationError(
|
|
88
|
-
`--${name} must be
|
|
87
|
+
`--${name} must be an ISO timestamp with a timezone and at most 6 fractional digits.`,
|
|
89
88
|
{ field: name }
|
|
90
89
|
);
|
|
91
90
|
}
|
|
92
|
-
|
|
91
|
+
const normalized = timestamp(options, name, usage);
|
|
92
|
+
return match[1]?.length > 3 ? `${normalized.slice(0, 19)}.${match[1]}Z` : normalized;
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
function
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
const parsed = Number(value);
|
|
99
|
-
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
|
100
|
-
throw new AdminMutationError("--down-payment-cents must be a whole number of cents.", {
|
|
101
|
-
field: "down-payment-cents",
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
return parsed;
|
|
95
|
+
function timestampMicroseconds(value) {
|
|
96
|
+
const fraction = value.match(/\.(\d+)Z$/)?.[1] ?? "";
|
|
97
|
+
return BigInt(Date.parse(value)) * 1000n + BigInt(fraction.padEnd(6, "0").slice(3));
|
|
105
98
|
}
|
|
106
99
|
|
|
107
100
|
function assertWindow(start, end) {
|
|
108
|
-
if (
|
|
101
|
+
if (timestampMicroseconds(start) >= timestampMicroseconds(end)) {
|
|
109
102
|
throw new AdminMutationError("--ends-at must be later than --starts-at.", { field: "ends-at" });
|
|
110
103
|
}
|
|
111
104
|
}
|
|
112
105
|
|
|
113
106
|
function parseCapacityOffer(argv) {
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
argv,
|
|
117
|
-
{
|
|
118
|
-
positionals: 0,
|
|
119
|
-
values: [
|
|
120
|
-
"gpu",
|
|
121
|
-
"nodes",
|
|
122
|
-
"gpus-per-node",
|
|
123
|
-
"site",
|
|
124
|
-
"starts-at",
|
|
125
|
-
"ends-at",
|
|
126
|
-
"price-per-gpu-hour",
|
|
127
|
-
"supplier-price-per-gpu-hour",
|
|
128
|
-
"down-payment-cents",
|
|
129
|
-
"access-mode",
|
|
130
|
-
"idempotency-key",
|
|
131
|
-
],
|
|
132
|
-
},
|
|
133
|
-
usage
|
|
134
|
-
);
|
|
135
|
-
const startsAt = timestamp(options, "starts-at", usage);
|
|
136
|
-
const endsAt = timestamp(options, "ends-at", usage);
|
|
137
|
-
assertWindow(startsAt, endsAt);
|
|
138
|
-
const accessMode = options["access-mode"]?.trim();
|
|
139
|
-
if (accessMode !== undefined && !ACCESS_MODES.has(accessMode)) {
|
|
140
|
-
throw new AdminMutationError("--access-mode must be bare-metal or virtual-machine.", {
|
|
141
|
-
field: "access-mode",
|
|
142
|
-
});
|
|
143
|
-
}
|
|
144
|
-
const downPaymentCents = optionalCents(options);
|
|
107
|
+
const { $positionals, yes, ...input } = parseSemanticArguments(argv);
|
|
108
|
+
if ($positionals.length) throw new AdminMutationError("Use named options for listing writes.");
|
|
145
109
|
return {
|
|
146
|
-
input
|
|
147
|
-
gpu_type: required(options, "gpu", usage),
|
|
148
|
-
node_count: positiveInteger(options, "nodes", usage),
|
|
149
|
-
gpus_per_node: positiveInteger(options, "gpus-per-node", usage),
|
|
150
|
-
site_nickname: required(options, "site", usage),
|
|
151
|
-
available_start_at: startsAt,
|
|
152
|
-
available_end_at: endsAt,
|
|
153
|
-
buy_now_price_per_gpu_hour: Number(price(options, usage)),
|
|
154
|
-
...(options["supplier-price-per-gpu-hour"] !== undefined
|
|
155
|
-
? {
|
|
156
|
-
cost_per_gpu_hour: Number(price(options, usage, "supplier-price-per-gpu-hour")),
|
|
157
|
-
}
|
|
158
|
-
: {}),
|
|
159
|
-
...(downPaymentCents !== undefined ? { down_payment_cents: downPaymentCents } : {}),
|
|
160
|
-
...(accessMode ? { access_mode: accessMode } : {}),
|
|
161
|
-
},
|
|
110
|
+
input,
|
|
162
111
|
mutation: {
|
|
163
|
-
idempotencyKey:
|
|
112
|
+
idempotencyKey: input.idempotencyKey,
|
|
164
113
|
safety: ADMIN_MUTATION_SAFETY.eligibleHumanOnly,
|
|
165
|
-
yes:
|
|
114
|
+
yes: yes === true,
|
|
166
115
|
},
|
|
167
116
|
};
|
|
168
117
|
}
|
|
@@ -174,29 +123,33 @@ function parseReservationCreate(argv) {
|
|
|
174
123
|
{
|
|
175
124
|
positionals: 0,
|
|
176
125
|
values: [
|
|
177
|
-
"
|
|
178
|
-
"
|
|
126
|
+
"customer",
|
|
127
|
+
"listing-term",
|
|
128
|
+
"nodes",
|
|
129
|
+
"gpus",
|
|
179
130
|
"starts-at",
|
|
180
131
|
"ends-at",
|
|
181
|
-
"
|
|
182
|
-
"
|
|
132
|
+
"transaction",
|
|
133
|
+
"vm-disk",
|
|
183
134
|
"idempotency-key",
|
|
184
135
|
],
|
|
185
136
|
},
|
|
186
137
|
usage
|
|
187
138
|
);
|
|
188
|
-
const startsAt =
|
|
189
|
-
const endsAt =
|
|
190
|
-
|
|
191
|
-
|
|
139
|
+
const startsAt = reservationTimestamp(options, "starts-at", usage);
|
|
140
|
+
const endsAt =
|
|
141
|
+
options["ends-at"] === undefined ? null : reservationTimestamp(options, "ends-at", usage);
|
|
142
|
+
if (endsAt !== null) assertWindow(startsAt, endsAt);
|
|
192
143
|
return {
|
|
193
144
|
input: {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
145
|
+
customerId: uuid(options, "customer", usage),
|
|
146
|
+
listingTermId: uuid(options, "listing-term", usage),
|
|
147
|
+
reservedNodeCount: positiveInteger(options, "nodes", usage),
|
|
148
|
+
reservedGpuCount: positiveInteger(options, "gpus", usage),
|
|
149
|
+
startsAt,
|
|
150
|
+
endsAt,
|
|
151
|
+
transactionId: uuid(options, "transaction", usage),
|
|
152
|
+
...(options["vm-disk"] !== undefined ? { vmDiskId: uuid(options, "vm-disk", usage) } : {}),
|
|
200
153
|
},
|
|
201
154
|
mutation: {
|
|
202
155
|
idempotencyKey: options["idempotency-key"],
|
|
@@ -206,46 +159,16 @@ function parseReservationCreate(argv) {
|
|
|
206
159
|
};
|
|
207
160
|
}
|
|
208
161
|
|
|
209
|
-
function
|
|
210
|
-
if (value?.ok === false) {
|
|
211
|
-
throw new AdminMutationError("The capacity offer contains an unknown hardware catalog term.", {
|
|
212
|
-
code: value.error ?? "unknown_catalog_term",
|
|
213
|
-
});
|
|
214
|
-
}
|
|
215
|
-
if (!value?.payload || typeof value.payload !== "object") {
|
|
216
|
-
throw new AdminMutationError("Capacity-offer normalization returned no effective terms.", {
|
|
217
|
-
code: "admin_review_incomplete",
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
return value.payload;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
function usd(microUsd) {
|
|
224
|
-
return typeof microUsd === "number" ? (microUsd / 1_000_000).toFixed(6) : undefined;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
function capacityOfferReview(payload, input, gpuLabel) {
|
|
228
|
-
const listing = payload.listing ?? {};
|
|
162
|
+
function capacityOfferReview(body, editing) {
|
|
229
163
|
return {
|
|
230
|
-
title: "Create capacity offer",
|
|
231
|
-
fields: [
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
{ label: "Ends at", value: listing.endsAt ?? input.available_end_at },
|
|
239
|
-
{ label: "Price per GPU-hour (USD)", value: input.buy_now_price_per_gpu_hour },
|
|
240
|
-
{
|
|
241
|
-
label: "Supplier price per GPU-hour (USD)",
|
|
242
|
-
value: usd(listing.supplierPricePerGpuHourMicroUsd),
|
|
243
|
-
},
|
|
244
|
-
{ label: "Spread per GPU-hour (USD)", value: usd(listing.spreadPerGpuHourMicroUsd) },
|
|
245
|
-
{ label: "Down payment (cents)", value: listing.downPaymentCents },
|
|
246
|
-
],
|
|
247
|
-
consequence:
|
|
248
|
-
"Publishes a primary listing with these terms; attach its nodes before it reaches the market.",
|
|
164
|
+
title: editing ? "Edit listing term" : "Create capacity offer",
|
|
165
|
+
fields: Object.entries(body).map(([label, value]) => ({
|
|
166
|
+
label,
|
|
167
|
+
value: typeof value === "object" ? JSON.stringify(value) : value,
|
|
168
|
+
})),
|
|
169
|
+
consequence: editing
|
|
170
|
+
? "Updates the selected advertised financial term."
|
|
171
|
+
: "Creates native capacity and its advertised financial terms.",
|
|
249
172
|
};
|
|
250
173
|
}
|
|
251
174
|
|
|
@@ -253,28 +176,29 @@ function reservationCreateReview(input) {
|
|
|
253
176
|
return {
|
|
254
177
|
title: "Create capacity reservation",
|
|
255
178
|
fields: [
|
|
256
|
-
{ label: "
|
|
257
|
-
{ label: "
|
|
258
|
-
{ label: "
|
|
259
|
-
{ label: "
|
|
260
|
-
{ label: "
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
179
|
+
{ label: "Customer", value: input.customerId },
|
|
180
|
+
{ label: "Listing term", value: input.listingTermId },
|
|
181
|
+
{ label: "Reserved nodes", value: input.reservedNodeCount },
|
|
182
|
+
{ label: "Reserved GPUs", value: input.reservedGpuCount },
|
|
183
|
+
{ label: "Starts at", value: input.startsAt },
|
|
184
|
+
{ label: "Ends at", value: input.endsAt ?? "Open-ended" },
|
|
185
|
+
{ label: "Transaction", value: input.transactionId },
|
|
186
|
+
...(input.vmDiskId ? [{ label: "VM disk", value: input.vmDiskId }] : []),
|
|
264
187
|
],
|
|
265
|
-
consequence:
|
|
266
|
-
"Creates a schedule reservation on the listing's nodes with the reviewed dates and price.",
|
|
188
|
+
consequence: "Reserves the reviewed quantities and dates under the selected listing term.",
|
|
267
189
|
};
|
|
268
190
|
}
|
|
269
191
|
|
|
270
|
-
async function runCapacityOffer({
|
|
192
|
+
async function runCapacityOffer({ capability, execute, input, mutation, runtime }) {
|
|
271
193
|
return runAdminMutation({
|
|
272
194
|
...runtime,
|
|
273
195
|
...mutation,
|
|
274
196
|
prepare: async () => {
|
|
275
|
-
const payload =
|
|
276
|
-
|
|
277
|
-
|
|
197
|
+
const { payload } = await execute(input);
|
|
198
|
+
return {
|
|
199
|
+
review: capacityOfferReview(payload, capability.id === "listings.update"),
|
|
200
|
+
value: input,
|
|
201
|
+
};
|
|
278
202
|
},
|
|
279
203
|
execute: (value) => execute(value, { confirmed: true }),
|
|
280
204
|
});
|
|
@@ -291,7 +215,13 @@ async function runReservationCreate({ execute, input, mutation, runtime }) {
|
|
|
291
215
|
|
|
292
216
|
export const adminCapacityReservationHandlers = [
|
|
293
217
|
{
|
|
294
|
-
capabilityId: "
|
|
218
|
+
capabilityId: "listing.create",
|
|
219
|
+
parse: parseCapacityOffer,
|
|
220
|
+
run: runCapacityOffer,
|
|
221
|
+
safety: ADMIN_MUTATION_SAFETY.eligibleHumanOnly,
|
|
222
|
+
},
|
|
223
|
+
{
|
|
224
|
+
capabilityId: "listings.update",
|
|
295
225
|
parse: parseCapacityOffer,
|
|
296
226
|
run: runCapacityOffer,
|
|
297
227
|
safety: ADMIN_MUTATION_SAFETY.eligibleHumanOnly,
|
|
@@ -339,9 +339,7 @@ async function resolveLiveMachine(fetch, node) {
|
|
|
339
339
|
})
|
|
340
340
|
);
|
|
341
341
|
const refs = new Set([id, text(node?.id), text(node?.k8s_node_name)].filter(Boolean));
|
|
342
|
-
const matches = rows.filter((machine) =>
|
|
343
|
-
refs.has(text(machine?.node_id))
|
|
344
|
-
);
|
|
342
|
+
const matches = rows.filter((machine) => refs.has(text(machine?.node_id)));
|
|
345
343
|
if (matches.length > 1) {
|
|
346
344
|
throw mutationError(`Multiple live machines resolve to node ${id}; no reboot was submitted.`, {
|
|
347
345
|
code: "ambiguous_live_machine",
|
|
@@ -676,14 +674,39 @@ function goalSatisfied(command, node, input) {
|
|
|
676
674
|
if (command === "suspend-management") return isOffGrid(node);
|
|
677
675
|
if (command === "resume-management") return !isOffGrid(node);
|
|
678
676
|
if (command === "assign") return reservationId(node) === input.reservation_id;
|
|
679
|
-
if (
|
|
680
|
-
|
|
681
|
-
|
|
677
|
+
if (cleanupState(node) === "failed") return false;
|
|
678
|
+
return !restartInProgress(node);
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
function cleanupState(node) {
|
|
682
|
+
return text(node?.cleanup_state).toLowerCase();
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
function restartInProgress(node) {
|
|
686
|
+
if (["pending", "rebooting"].includes(cleanupState(node))) return true;
|
|
687
|
+
const readiness = node.health_readiness;
|
|
688
|
+
return !(
|
|
682
689
|
readiness?.ready === true ||
|
|
683
690
|
["ready", "running"].includes(text(readiness?.status).toLowerCase())
|
|
684
691
|
);
|
|
685
692
|
}
|
|
686
693
|
|
|
694
|
+
// Before the restart starts, a healthy node already looks "done", so reboot
|
|
695
|
+
// and hard-reset only count as complete after the restart has been observed.
|
|
696
|
+
function restartGoal(command, input) {
|
|
697
|
+
if (command !== "reboot" && command !== "hard-reset") {
|
|
698
|
+
return { isReady: (node) => goalSatisfied(command, node, input), isTerminal: () => false };
|
|
699
|
+
}
|
|
700
|
+
let restartSeen = false;
|
|
701
|
+
return {
|
|
702
|
+
isReady: (node) => {
|
|
703
|
+
if (node && restartInProgress(node)) restartSeen = true;
|
|
704
|
+
return restartSeen && goalSatisfied(command, node, input);
|
|
705
|
+
},
|
|
706
|
+
isTerminal: (node) => restartSeen && cleanupState(node) === "failed",
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
|
|
687
710
|
function waitResult(kind, command, id, accepted, lastState) {
|
|
688
711
|
const timeout = kind === "timeout";
|
|
689
712
|
return {
|
|
@@ -711,13 +734,15 @@ function waitResult(kind, command, id, accepted, lastState) {
|
|
|
711
734
|
async function waitForGoal({ command, fetch, input, nodeId: id, runtime, accepted }) {
|
|
712
735
|
let lastState = null;
|
|
713
736
|
let outcome;
|
|
737
|
+
const { isReady, isTerminal } = restartGoal(command, input);
|
|
714
738
|
try {
|
|
715
739
|
outcome = await waitForReadGoal(
|
|
716
740
|
{
|
|
717
741
|
command: `ornn admin nodes ${command}`,
|
|
718
742
|
initial: null,
|
|
719
743
|
intervalMs: POLL_INTERVAL_MS,
|
|
720
|
-
isReady
|
|
744
|
+
isReady,
|
|
745
|
+
isTerminal,
|
|
721
746
|
message: () => `Waiting for node ${id} to reach the requested state.`,
|
|
722
747
|
onObserve: (node) => {
|
|
723
748
|
lastState = node;
|
|
@@ -790,6 +815,22 @@ async function waitForGoal({ command, fetch, input, nodeId: id, runtime, accepte
|
|
|
790
815
|
errors: [],
|
|
791
816
|
};
|
|
792
817
|
}
|
|
818
|
+
if (outcome.kind === "terminal") {
|
|
819
|
+
return {
|
|
820
|
+
data: { ...accepted, node_id: id, state: safeNodeState(outcome.value) },
|
|
821
|
+
exitCode: 1,
|
|
822
|
+
outcome: { class: "failed", phase: "cleanup_failed" },
|
|
823
|
+
resume: resumeReference(id),
|
|
824
|
+
action: inspectionAction(id),
|
|
825
|
+
errors: [
|
|
826
|
+
{
|
|
827
|
+
code: "cleanup_failed",
|
|
828
|
+
message: "The node reported the operation as failed.",
|
|
829
|
+
retryable: false,
|
|
830
|
+
},
|
|
831
|
+
],
|
|
832
|
+
};
|
|
833
|
+
}
|
|
793
834
|
return waitResult(outcome.kind, command, id, accepted, outcome.value);
|
|
794
835
|
}
|
|
795
836
|
|