@cotal-ai/workspace 0.133.12 → 0.134.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/build/Release/darwin-arm64/build.json +1 -1
  2. package/build/Release/darwin-x64/build.json +1 -1
  3. package/build/Release/win32-arm64/build.json +1 -1
  4. package/build/Release/win32-x64/build.json +1 -1
  5. package/dist/control/actions.d.ts +141 -0
  6. package/dist/control/actions.d.ts.map +1 -0
  7. package/dist/control/actions.js +192 -0
  8. package/dist/control/actions.js.map +1 -0
  9. package/dist/control/dial-retry.d.ts +33 -0
  10. package/dist/control/dial-retry.d.ts.map +1 -0
  11. package/dist/control/dial-retry.js +70 -0
  12. package/dist/control/dial-retry.js.map +1 -0
  13. package/dist/control/index.d.ts +5 -0
  14. package/dist/control/index.d.ts.map +1 -0
  15. package/dist/control/index.js +5 -0
  16. package/dist/control/index.js.map +1 -0
  17. package/dist/control/manager.d.ts +291 -0
  18. package/dist/control/manager.d.ts.map +1 -0
  19. package/dist/control/manager.js +769 -0
  20. package/dist/control/manager.js.map +1 -0
  21. package/dist/control/seat.d.ts +168 -0
  22. package/dist/control/seat.d.ts.map +1 -0
  23. package/dist/control/seat.js +225 -0
  24. package/dist/control/seat.js.map +1 -0
  25. package/dist/launch.d.ts +94 -0
  26. package/dist/launch.d.ts.map +1 -0
  27. package/dist/launch.js +58 -0
  28. package/dist/launch.js.map +1 -0
  29. package/dist/maintenance.d.ts +7 -0
  30. package/dist/maintenance.d.ts.map +1 -1
  31. package/dist/maintenance.js +24 -0
  32. package/dist/maintenance.js.map +1 -1
  33. package/dist/output.d.ts +2 -0
  34. package/dist/output.d.ts.map +1 -1
  35. package/dist/output.js +5 -0
  36. package/dist/output.js.map +1 -1
  37. package/dist/view/index.d.ts +3 -0
  38. package/dist/view/index.d.ts.map +1 -0
  39. package/dist/view/index.js +3 -0
  40. package/dist/view/index.js.map +1 -0
  41. package/dist/view/mesh-view.d.ts +225 -0
  42. package/dist/view/mesh-view.d.ts.map +1 -0
  43. package/dist/view/mesh-view.js +550 -0
  44. package/dist/view/mesh-view.js.map +1 -0
  45. package/dist/view/observer.d.ts +26 -0
  46. package/dist/view/observer.d.ts.map +1 -0
  47. package/dist/view/observer.js +26 -0
  48. package/dist/view/observer.js.map +1 -0
  49. package/package.json +18 -4
@@ -0,0 +1,769 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { BASELINE_LIFECYCLE_ENDPOINT, DEV_OWNER, EP_UNANSWERED, EpEnvelopeError, GOAL_BEARING_COMMANDS, goalFollowRefusal, epProbeInstanceInterest, freezeExpectedSet, instancePinnedInstrumentCapabilities, invokeCommand, invokeRepairingSplit, issuedUserCaller, mintCreds, oneLine, parseEpSubject, respondedButUnbound, unansweredRequest, unansweredRail, registryReadFailed, noLiveInstances, undeclaredArg, undeclaredArgInstance, controlReplyFrom, controlReplyFromThrown, submitAndFollowGoal, scatterCommand, mintLifecycleUid, newIdentity, dialerFor, resolveService, standaloneConnectOpts, } from "@cotal-ai/core";
3
+ import { PermissionViolationError } from "@nats-io/transport-node";
4
+ import { jetstreamManager } from "@nats-io/jetstream";
5
+ import { c } from "../colors.js";
6
+ import { controlCaller } from "../control-target.js";
7
+ import { staleStoreHint } from "../maintenance.js";
8
+ import { CommandFailure, nativeEndpointRefusal } from "../output.js";
9
+ import { DIAL_TIMEOUT_MS, DialNotAdmitted, dialWithRetry, notAdmittedRefusal } from "./dial-retry.js";
10
+ /** Client-side request window for the manager's readiness-waiting launch ops (`start`, and the
11
+ * manifest `launch` — both funnel into the same startAgent readiness wait). #159 B1: the manager
12
+ * replies only on a REAL outcome — presence join, process exit, or its ~30s readiness backstop —
13
+ * so these requests must OUTLIVE that window, not the 5s op default. The tier rule forbids
14
+ * importing the manager's READINESS_TIMEOUT_MS here; the launch-parity smoke enforces the
15
+ * relation by test. */
16
+ export const START_TIMEOUT_MS = 40_000;
17
+ /** v0.3 ctl op → v0.4 typed command (P2 item 1, 1c.2b): the wire names the manager REGISTERS
18
+ * (manager-service-contract ROWS). `start` is creation (`spawn`), a NAMED `stop` is the one
19
+ * owner/any-mode terminal (`despawn`), the per-agent `status` read is `inspect`; the camelCase
20
+ * admin family maps to its kebab-case wire names. `targeted` marks the three commands whose
21
+ * `{name}` argument becomes a §13.2 target block (resolved to the agent's principal triple via
22
+ * the name-keyed `inspect` read — it rides the spawn capability arm, so resolution reach equals
23
+ * despawn/attach reach; the wire target is (owner, actor, lifecycleUid), never an alias). */
24
+ const EP_COMMANDS = {
25
+ managerStatus: { command: "status" },
26
+ start: { command: "spawn" },
27
+ resolveAgent: { command: "resolve-agent" },
28
+ resolveCwd: { command: "resolve-cwd" },
29
+ stop: { command: "despawn", targeted: true },
30
+ stopSelf: { command: "stop", self: true },
31
+ attach: { command: "attach", targeted: true },
32
+ input: { command: "input", targeted: true },
33
+ status: { command: "inspect" },
34
+ ps: { command: "ps" },
35
+ slots: { command: "slots" },
36
+ models: { command: "models" },
37
+ listPersonas: { command: "list-personas" },
38
+ launch: { command: "launch" },
39
+ purge: { command: "purge" },
40
+ resumePreserved: { command: "resume-preserved" },
41
+ commitResume: { command: "commit-resume" },
42
+ finalizeResume: { command: "finalize-resume" },
43
+ preparePreservation: { command: "prepare-preservation" },
44
+ commitPreservation: { command: "commit-preservation" },
45
+ abortPreservation: { command: "abort-preservation" },
46
+ transcriptReceive: { command: "transcript-receive" },
47
+ chatSessionOpen: { command: "chat-session-open" },
48
+ chatSessionRetire: { command: "chat-session-retire" },
49
+ runPs: { command: "run-ps" },
50
+ runStatus: { command: "run-status" },
51
+ };
52
+ /** The ep-rail control call — since 1d {@link askManager}'s ONLY path: one short-lived raw
53
+ * connection, a fresh `resolveService` (describe → §13.7 store fetch → digest-verified recompile
54
+ * — the generic item-5 caller, no hand-imported manager schemas), then the mapped command.
55
+ * Targeted ops resolve the alias to its CURRENT principal triple through the name-keyed
56
+ * `inspect` read first and ride mode `any` (admin instruments) or `owner` per {@link
57
+ * ControlReach}.
58
+ *
59
+ * `standaloneConnectOpts` handles static credentials, a user bearer plus sentinel, and bare open
60
+ * meshes. The selected dialer matters: a user mesh behind an HTTPS edge has a `wss://` URL, which
61
+ * the raw node transport refuses even though endpoint, send, and spawn rails can reach it.
62
+ * A connect the broker refuses or does not complete in time sends nothing, so it is tried again
63
+ * ({@link dialWithRetry}); when every attempt ends that way, {@link notAdmittedRefusal} is thrown. */
64
+ async function askManagerEp(client, space, server, op, args, auth, reach, timeoutMs, pin) {
65
+ if (pin?.instanceId !== undefined && auth.managerInstanceId !== undefined && pin.instanceId !== auth.managerInstanceId)
66
+ return { ok: false, error: "the manager pin differs from this credential's instance authority", code: "permission-denied" };
67
+ const instanceId = pin?.instanceId ?? auth.managerInstanceId;
68
+ const mapped = EP_COMMANDS[op];
69
+ if (!mapped)
70
+ return { ok: false, error: `unknown manager op "${op}" (no v0.4 command mapping)` };
71
+ // Each attempt is a new connection, so a user bearer's connection nonce is minted afresh with it.
72
+ let connectOpts = {};
73
+ const nc = await dialWithRetry(() => {
74
+ connectOpts = standaloneConnectOpts(auth.creds ? { creds: auth.creds, tls: auth.tls === true } : auth.bearer ? { bearer: auth.bearer, sentinelCreds: auth.sentinelCreds, tls: auth.tls === true } : { tls: auth.tls === true });
75
+ return dialerFor(server)({ servers: server, ...connectOpts, timeout: DIAL_TIMEOUT_MS, maxReconnectAttempts: 0 });
76
+ }).catch((e) => {
77
+ if (!(e instanceof DialNotAdmitted))
78
+ throw e;
79
+ throw notAdmittedRefusal(e, instanceId === undefined ? "this space's manager" : `manager instance ${instanceId}`, "nothing was sent to the manager", { space, ...(instanceId === undefined ? {} : { managerInstanceId: instanceId }) }, auth.bearer !== undefined);
80
+ });
81
+ let request;
82
+ let intent;
83
+ let observationCause;
84
+ let caller;
85
+ try {
86
+ // A logged-in user's manager view is issued at the callout (SPEC 13.15): read its generation
87
+ // back from this connection's accepted row; a missing row refuses, never a legacy-rail retry.
88
+ caller = auth.bearer !== undefined && auth.managerInstanceId !== undefined
89
+ ? await issuedUserCaller(nc, space, String(connectOpts.name), auth.epCaller)
90
+ : auth.epCaller;
91
+ // P2 item 3 `--on <instance>`: pin the resolve to the exact manager instance's `inst` route so a
92
+ // multi-manager space addresses the intended manager, never whichever wins the class anycast.
93
+ const followsGoal = GOAL_BEARING_COMMANDS.includes(mapped.command);
94
+ const resolveCommands = [...new Set([mapped.command, ...(mapped.targeted ? ["inspect"] : []), ...(followsGoal ? ["goal-result"] : [])])];
95
+ const service = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, { deadlineMs: 10_000, commands: resolveCommands, ...(followsGoal ? { optionalCommands: ["goal-result"] } : {}), ...(instanceId !== undefined ? { instanceId } : {}), ...(pin?.signal ? { signal: pin.signal } : {}) });
96
+ let target;
97
+ let sendArgs = args;
98
+ if (mapped.self)
99
+ target = { mode: "self" };
100
+ else if (mapped.targeted) {
101
+ const name = String(args?.name ?? "").trim();
102
+ if (!name)
103
+ return { ok: false, error: `${op} requires a name` };
104
+ // Alias -> CURRENT principal triple via the manager's name-keyed `inspect` read (§13.2: a
105
+ // target is a triple, never an alias) - the same resolution the connector uses. `inspect`
106
+ // rides the SPAWN capability arm as well as the instrument read set, so every caller class
107
+ // that can despawn/attach can also resolve its target (a `ps` SCAN here broker-drops exactly
108
+ // the spawn-scoped user bearers - the 1c.2b read narrowing - and hangs their stop/attach).
109
+ const info = await invokeRepairingSplit(nc, space, service, "inspect", { name }, { deadlineMs: 10_000 });
110
+ if (info.reply.ok !== true) {
111
+ const refused = controlReplyFrom(info.reply);
112
+ // Only the inspect read was published. Preserve its native error, but record the local
113
+ // proof that this targeted action was never submitted.
114
+ return { ...refused, preSubmitRefusal: true, managerInstanceId: info.responder.instanceId, managerEpoch: info.responder.epoch, error: `could not resolve "${name}": ${refused.error}` };
115
+ }
116
+ const row = info.reply.data;
117
+ if (pin?.expectedTarget && (row.id !== pin.expectedTarget.id || row.lifecycleUid !== pin.expectedTarget.lifecycleUid))
118
+ return { ok: false, code: "not-found", error: "the selected agent incarnation changed; refresh the agent list before attaching" };
119
+ // A STATIC row's `id` is the bare actor under the caller's own owner; a USER-mode row's `id`
120
+ // is the composite `owner.actor` principal key - split it (an embedded dot would break the
121
+ // target block's subject arity). Mode `any` spans owners (operator reach); mode `owner` pins
122
+ // the caller's own, so a foreign-owner target is broker-denied at publish.
123
+ const dot = row.id.indexOf(".");
124
+ const [tOwner, tActor] = dot > 0 ? [row.id.slice(0, dot), row.id.slice(dot + 1)] : [caller.owner, row.id];
125
+ // Target mode from the RESOLVED target owner: an own-domain target rides `owner` mode (pinned
126
+ // to the caller's own owner); a CROSS-owner target rides `any` mode - which the broker admits
127
+ // only for a caller holding the admin instrument rows (a static admin instrument, or a user
128
+ // bearer whose ledger `admin` scope the callout minted them into). `reach: "any"` forces
129
+ // any-mode for a static admin instrument even on its own domain. So a spawn-scoped caller's
130
+ // cross-owner despawn/attach is broker-denied at publish (no any-mode row), while an
131
+ // admin-scoped operator's is admitted and the manager's fresh ledger check governs.
132
+ const mode = reach === "any" || tOwner !== caller.owner ? "any" : "owner";
133
+ target = {
134
+ mode,
135
+ owner: tOwner,
136
+ actor: tActor,
137
+ lifecycleUid: row.lifecycleUid,
138
+ };
139
+ const { name: _dropped, ...rest } = args ?? {};
140
+ sendArgs = Object.keys(rest).length ? rest : undefined;
141
+ }
142
+ const blocked = followsGoal ? goalFollowRefusal(service) : undefined;
143
+ if (blocked)
144
+ return { ...controlReplyFrom(blocked.reply), nativeOutcome: "not-executed", managerInstanceId: service.responder.instanceId, managerEpoch: service.responder.epoch };
145
+ const id = randomUUID();
146
+ intent = { requestId: id, ...(followsGoal ? { requestedGoalId: id } : {}) };
147
+ if (pin?.journal)
148
+ await pin.journal.beforeSubmit(intent);
149
+ const invokeOpts = { ...(target ? { target } : {}), deadlineMs: timeoutMs ?? 10_000, id, ...(pin?.signal ? { signal: pin.signal } : {}) };
150
+ const submit = async (signal) => {
151
+ const answer = await invokeRepairingSplit(nc, space, service, mapped.command, sendArgs, { ...invokeOpts, ...(signal ? { signal } : {}) });
152
+ const data = answer.reply.data;
153
+ request = { requestId: answer.reply.id, managerInstanceId: answer.responder.instanceId, managerEpoch: answer.responder.epoch,
154
+ ...(answer.reply.ok && typeof data?.goalId === "string" ? { goalId: data.goalId } : {}),
155
+ ...(answer.reply.ok && typeof data?.fingerprint === "string" ? { fingerprint: data.fingerprint } : {}),
156
+ ...(answer.reply.ok && typeof data?.goalId === "string" && typeof data.name === "string" && typeof data.owner === "string" && typeof data.actor === "string" && typeof data.uid === "string"
157
+ ? { allocation: { name: data.name, owner: data.owner, actor: data.actor, lifecycleUid: data.uid } } : {}),
158
+ };
159
+ if (answer.reply.ok && request.allocation && pin?.journal)
160
+ await pin.journal.accepted(request);
161
+ return answer;
162
+ };
163
+ // P2 item 2 (2b): a goal-bearing command (spawn/launch) FOLLOWS its acceptance to the goal
164
+ // terminal, so `spawn --detach` still returns on the real outcome (join / exit / ~30s uncertain)
165
+ // exactly like the pre-action blocking reply — UX unchanged, no --no-wait.
166
+ const r = followsGoal
167
+ ? await submitAndFollowGoal(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, timeoutMs ?? START_TIMEOUT_MS, submit, {
168
+ reconcile: async (goalId, accepted, context) => {
169
+ // A class credential keeps its class route. An accepted instance ID grants no new rail;
170
+ // the native follower still verifies the canonical fact against the original caller/goal.
171
+ const current = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, context.caller, { ...(instanceId !== undefined ? { instanceId: accepted.responder.instanceId } : {}), commands: ["goal-result"], deadlineMs: context.deadlineMs, signal: context.signal });
172
+ const answer = await invokeRepairingSplit(nc, space, current, "goal-result", { goalId }, { deadlineMs: context.deadlineMs, signal: context.signal });
173
+ if (!answer.reply.ok) {
174
+ observationCause = { code: answer.reply.error?.code, outcome: answer.reply.error?.outcome };
175
+ throw new Error("the accepting manager did not provide a confirmed goal-result read");
176
+ }
177
+ return answer.reply.data;
178
+ },
179
+ })
180
+ : await submit();
181
+ if (r.reply.ok !== true)
182
+ return { ...controlReplyFrom(r.reply), answered: true, managerInstanceId: r.responder.instanceId, managerEpoch: r.responder.epoch, ...(observationCause ? { observationCause } : {}), ...(intent ? { intent } : {}), ...(request ? { request } : {}), ...(r.reply.error?.outcome ? { nativeOutcome: r.reply.error.outcome } : {}) };
183
+ // The ep `models` reply is normalized to `{catalogs}` — unwrap so call sites keep the ctl shape.
184
+ const data = mapped.command === "models" ? r.reply.data.catalogs : r.reply.data;
185
+ return { ok: true, managerInstanceId: r.responder.instanceId, managerEpoch: r.responder.epoch, ...(observationCause ? { observationCause } : {}), ...(intent ? { intent } : {}), ...(request ? { request } : {}), ...(data !== undefined ? { data } : {}) };
186
+ }
187
+ catch (e) {
188
+ // A key the manager's contract does not declare: ask the instance that refused it which release
189
+ // it runs, so the refusal can say which side is older.
190
+ const refusing = undeclaredArgInstance(e);
191
+ const managerVersion = refusing !== undefined && caller !== undefined ? await reportedManagerVersion(nc, space, caller, refusing, instanceId !== undefined) : undefined;
192
+ return { ...epRailFailure(client, e, instanceId === undefined ? pin : { instanceId }, managerVersion), ...(refusing !== undefined ? { managerInstanceId: refusing } : {}), ...(observationCause ? { observationCause } : {}), ...(intent ? { intent } : {}), ...(request ? { request, managerInstanceId: request.managerInstanceId, managerEpoch: request.managerEpoch } : {}), ...(e instanceof EpEnvelopeError && e.outcome ? { nativeOutcome: e.outcome } : {}) };
193
+ }
194
+ finally {
195
+ // Drain waits for in-flight NATS work. A dead broker never finishes that, and
196
+ // attach-auth-root hung here after spawn once nats-server exited.
197
+ await Promise.race([
198
+ nc.drain().catch(() => nc.close()),
199
+ new Promise((resolve) => setTimeout(resolve, 2_000).unref()),
200
+ ]);
201
+ try {
202
+ nc.close();
203
+ }
204
+ catch {
205
+ /* already closed */
206
+ }
207
+ }
208
+ }
209
+ /** Classify only a proved pre-effect refusal as refused; a missing effect outcome is not proof.
210
+ * Every message names the agent when the target carries one and ends with the manager's own
211
+ * reason, kept to one line. */
212
+ export function managerFailure(reply, operation, target) {
213
+ const native = { code: reply.code, outcome: reply.nativeOutcome, ...(reply.preSubmitRefusal ? { detail: { phase: "target-resolution", actionSubmitted: false } } : reply.request || reply.intent ? { detail: { request: reply.request ?? null, intent: reply.intent ?? null, observation: reply.observationCause ?? null } } : {}) };
214
+ const code = nativeEndpointRefusal(reply.code);
215
+ const agent = target?.agent;
216
+ const what = typeof agent === "string" ? `${operation} for agent "${oneLine(agent)}"` : operation;
217
+ const reason = reply.error ? `: ${oneLine(reply.error)}` : "";
218
+ if (reply.preSubmitRefusal && code)
219
+ return new CommandFailure("refused", code, reply.error ? oneLine(reply.error) : `${operation} was not submitted because its target could not be resolved`, { target, native, remedy: "run `cotal ps` to check the selected manager and agent name" });
220
+ if (reply.nativeOutcome === "not-executed" && code)
221
+ return new CommandFailure("refused", code, `the manager refused ${what} before execution${reason}`, { target, native, remedy: "nothing ran; fix what the manager reported, then run the command again" });
222
+ if (reply.nativeOutcome === "executed" && !reply.unanswered)
223
+ return new CommandFailure("failed", "OPERATION_FAILED", `the manager reported that ${what} failed${reason}`, { target, native });
224
+ return new CommandFailure("unknown", "OPERATION_UNKNOWN", `the manager did not confirm the outcome of ${what}; it was not submitted again${reason}`, { target, native, remedy: "inspect the recorded manager, request and target before retrying" });
225
+ }
226
+ /** A describe that drew no answer means the command after it was never sent, so it is refused, as
227
+ * `chat` and the run verbs refuse it, and not an unknown outcome. Undefined for any other failure. */
228
+ export function unansweredDescribeRefusal(reply, operation, target) {
229
+ if (!describeUnanswered(reply))
230
+ return undefined;
231
+ const agent = target?.agent;
232
+ const what = typeof agent === "string" ? `${operation} for agent "${oneLine(agent)}"` : operation;
233
+ return new CommandFailure("refused", "TARGET_UNAVAILABLE", `the manager registered for this space did not answer, so ${what} was not sent: ${oneLine(reply.error ?? "no reply")}`, { target, native: { code: reply.code }, remedy: "cotal status shows whether this space's manager answers; a stack manager that runs but does not answer is named there with its pid and the way to end it" });
234
+ }
235
+ /** Whether the describe before a command drew no answer, so the command itself was never sent. */
236
+ export function describeUnanswered(reply) {
237
+ return reply.details?.some((detail) => detail.kind === EP_UNANSWERED && detail.command === "describe") === true;
238
+ }
239
+ /** Render an ep-rail failure for the operator. Three outcomes, told apart by core's markers and never
240
+ * by the catalog code: a responder's own `ok:false` describe reply is rethrown under ITS code
241
+ * (`unavailable` included), and a store read after an answered describe raises the same code, so
242
+ * the code says nothing about whether anyone answered.
243
+ * - UNANSWERED ({@link unansweredRequest}: no responder, or the reply deadline elapsed). The
244
+ * reachability verdict "no manager reachable" is stated here and only here, and only unpinned:
245
+ * an unanswered PINNED call names the instance instead, since three managers may be answering
246
+ * while the one the operator typed is not there, and "no manager reachable" sends them to the
247
+ * broker for a typo. Measured on a live three-manager mesh during review. The verdict is also
248
+ * scoped to the RAIL ({@link unansweredRail}): on the versioned rail it names that rail and says what the
249
+ * silence does not establish, because SPEC 13.15 keeps the two rails disjoint at the broker.
250
+ * - a REGISTRY READ on this side failed ({@link registryReadFailed}: the scatter's freeze or its
251
+ * reconcile). The managers were not the failure and may all be up; a verdict on them here sent
252
+ * the operator to the managers for a broker read.
253
+ * - everything else answered, or failed on this side with its own cause, and is printed as is.
254
+ * Prepending a verdict made the headline contradict the body: a describe REFUSED BY THE BROKER
255
+ * read as an unreachable manager, which is precisely the misreading the refusal was reworded to
256
+ * stop.
257
+ * A failure that is not an {@link EpEnvelopeError} carries no answer provenance at all, so no verdict
258
+ * is stated for it either: its message stands alone. `managerVersion` is the version the refusing
259
+ * manager reported in its own `status`, for a key its contract does not declare. */
260
+ export function epRailFailure(client, e, pin, managerVersion) {
261
+ if (!(e instanceof EpEnvelopeError))
262
+ return { ok: false, unanswered: false, error: e instanceof Error ? e.message : String(e) };
263
+ return { ...controlReplyFromThrown(e, epRailMessage(client, e, pin, managerVersion)), unanswered: unansweredRequest(e) };
264
+ }
265
+ function epRailMessage(client, e, pin, managerVersion) {
266
+ const instanceId = pin?.instanceId;
267
+ const detail = `${e.code}: ${e.message}`;
268
+ if (unansweredRequest(e)) {
269
+ // The verdict is SCOPED TO THE RAIL the request rode (SPEC 13.15). The legacy and versioned rails are
270
+ // disjoint subject spaces at the broker and an endpoint is required to serve both, so a manager
271
+ // built before the versioned rail subscribes `ep` alone and an issued caller can never see it.
272
+ // "No manager reachable" is then a claim about the mesh drawn from silence on one half of it,
273
+ // and #1630 measured it against a manager that was up, idle on the roster and starting runs the
274
+ // whole time. On the legacy rail there is no other half, so the verdict stands as it was.
275
+ const rail = unansweredRail(e);
276
+ const versioned = rail !== undefined && rail !== "ep" ? rail : undefined;
277
+ // TWO CAUSES, BOTH NAMED. Silence on a versioned rail is consistent with no manager at all AND
278
+ // with one older than the rail, and this side cannot tell them apart: the registry records no
279
+ // package version. An earlier wording named only the skew, which sent an operator whose manager
280
+ // was simply down to go and check a version.
281
+ const skew = versioned === undefined ? "" :
282
+ ` The ${versioned} and legacy ep rails are disjoint at the broker (SPEC 13.15) and an endpoint must serve both, so this does not tell no manager running apart from one older than ${versioned}, which serves ep only and cannot answer here. Check whether a manager is running, and if it is, its version.`;
283
+ return instanceId !== undefined
284
+ ? `manager instance ${instanceId} did not answer${versioned === undefined ? "" : ` on the ${versioned} rail`} (${detail})${skew}`
285
+ : versioned === undefined
286
+ ? `no manager reachable on the ep rails (${detail})`
287
+ : `no manager answered on the ${versioned} rail (${detail})${skew}`;
288
+ }
289
+ if (registryReadFailed(e))
290
+ return `the manager registry could not be read: a broker read on this side, not the managers' silence, and they may all be up. Retry; if it persists, look at the broker's JetStream (${detail})`;
291
+ // A key the manager's contract does not declare means the manager runs a different release from
292
+ // the caller that built the args. Only the caller knows its own release and how it is updated, so
293
+ // it words the refusal.
294
+ const undeclared = undeclaredArg(e);
295
+ if (undeclared !== undefined)
296
+ return client.versionSkew(detail, undeclared, managerVersion);
297
+ // The unpinned class-queue split. Core says a call that addresses one instance does not split
298
+ // and stops there (a CLI flag name does not belong in a core error). The flag is named here only
299
+ // when the CALLER declared it has one (`pin` present) and did not pass it: an absent `pin` is a
300
+ // command with no `--on` at all, and telling it to type one is the same dead end one layer down.
301
+ // A marked `expired` is the other producer (a stale-epoch bind) and its remedy is re-resolving,
302
+ // so the flag is offered only for the split.
303
+ const unpinnedSplit = e.code === "failed-precondition" && respondedButUnbound(e) && pin !== undefined && instanceId === undefined;
304
+ return `${detail}${unpinnedSplit ? " Pin one manager instance with --on <instance> (the whole id, as `ps` prints it) to avoid the split." : ""}`;
305
+ }
306
+ /** Send one control command to the manager over the v0.4 service-endpoint rails and disconnect —
307
+ * since 1d the manager's ONLY control door (the `ctl` tiers are deleted). The target is already
308
+ * reachability- + auth-preflighted by {@link resolveControlTarget}. `reach` picks the operator
309
+ * mode for the two targeted ops (stop/attach): `owner` = the caller's own domain (the spawn
310
+ * capability's standing mint / a user bearer's own owner), `any` = the admin instrument's
311
+ * cross-agent reach. Three auth shapes reach the rails: a static instrument's caller triple, a
312
+ * user bearer's triple, or an OPEN mesh (no credential system — a bare connection under a
313
+ * synthesized DEV_OWNER triple, since the manager registered under DEV_OWNER and the broker
314
+ * enforces nothing). A raw `--creds` file from an older generation carries no ep rows and is
315
+ * refused loud (no silent ctl fallback exists anymore). */
316
+ export async function askManager(client, space, server, op, args, auth = {}, reach = "owner", timeoutMs, pin) {
317
+ const who = controlCaller(auth);
318
+ if ("refusal" in who)
319
+ return { ok: false, error: who.refusal };
320
+ return askManagerEp(client, space, server, op, args, { ...auth, epCaller: who.caller }, reach, timeoutMs, pin);
321
+ }
322
+ /** Open one short-lived control connection under whichever of the three auth shapes this command
323
+ * holds (static creds / user bearer + sentinel / bare open mesh) and hand it to `fn`. Extracted so
324
+ * the scatter's TWO connections cannot drift in their connect options: they differ in credential
325
+ * and in nothing else. The selected dialer matters because the raw node transport refuses a
326
+ * `wss://` mesh behind an HTTPS edge even when the other endpoint rails can reach it. */
327
+ export async function withControlConnection(server, auth, fn) {
328
+ const nc = await dialerFor(server)({
329
+ servers: server,
330
+ ...standaloneConnectOpts(auth.creds ? { creds: auth.creds, tls: auth.tls === true } : auth.bearer ? { bearer: auth.bearer, sentinelCreds: auth.sentinelCreds, tls: auth.tls === true } : { tls: auth.tls === true }),
331
+ maxReconnectAttempts: 0,
332
+ });
333
+ try {
334
+ return await fn(nc);
335
+ }
336
+ finally {
337
+ await nc.drain().catch(() => nc.close());
338
+ }
339
+ }
340
+ /** The instance a REFUSED SUBJECT names, or `undefined` when that subject is not one of this
341
+ * probe's requests at all.
342
+ *
343
+ * The id is read as a whole subject TOKEN, through the same §13.9 grammar parser that built the
344
+ * subject, and never matched inside the text. Lifecycle tokens are `[a-z0-9]{26,32}`, so one
345
+ * frozen id can be a PREFIX of another (26 chars and the same 26 plus a suffix are both legal),
346
+ * and a substring test then attributes one broker refusal to two instances: the operator is told
347
+ * the caller could not ask about a manager it never published for, and that instance's probe settles
348
+ * as `probe-refused` on evidence that belongs to a different rail. Parsing removes the class of
349
+ * mistake rather than one instance of it: the endpoint and the route mode must match too, so a
350
+ * violation on any other plane, endpoint, or route is not this probe's and is not attributed. */
351
+ function probedInstanceOf(subject, endpoint) {
352
+ const parsed = parseEpSubject(subject);
353
+ if (parsed === null || parsed.plane !== "request" || parsed.route !== "inst")
354
+ return undefined;
355
+ return parsed.endpoint === endpoint ? parsed.instanceId : undefined;
356
+ }
357
+ /** Watch a connection's status stream for the broker's own PERMISSION VIOLATIONS and attribute each
358
+ * to the instance whose rail it names.
359
+ *
360
+ * A refused publish is invisible where it matters. The violation is delivered on the CONNECTION,
361
+ * asynchronously, while the publish itself returns normally — so a liveness probe with no grant
362
+ * does not fail, it goes quiet, and quiet is exactly what a live-but-slow instance looks like. The
363
+ * whole probe would then read as `unknown`, the deadline would be paid in full, and the operator
364
+ * would be told the manager was slow when the truth was that the caller never asked. So the
365
+ * violation is caught, attributed to the instance its subject names, reported on stderr once, and
366
+ * used to settle that probe immediately instead of letting it expire.
367
+ *
368
+ * ATTRIBUTION IS STRUCTURAL, not textual. The client types a permission violation as
369
+ * `PermissionViolationError` and carries the refused SUBJECT as a field, so nothing here parses
370
+ * the broker's prose: the subject goes through {@link probedInstanceOf} and yields one exact id or
371
+ * none. NAMED RESIDUAL: a connection error the client does not type as a permission violation is
372
+ * not attributed, and that probe expires into `unknown` at its budget — the behaviour that existed
373
+ * before this watch, which is slower and never wrong. */
374
+ function watchProbeRefusals(nc, what, report) {
375
+ const ids = new Set(what.ids);
376
+ const settle = new Map();
377
+ const already = new Set();
378
+ const pending = new Map();
379
+ const refused = (id) => {
380
+ if (already.has(id))
381
+ return Promise.resolve();
382
+ let p = pending.get(id);
383
+ if (!p) {
384
+ p = new Promise((resolve) => settle.set(id, resolve));
385
+ pending.set(id, p);
386
+ }
387
+ return p;
388
+ };
389
+ void (async () => {
390
+ try {
391
+ for await (const s of nc.status()) {
392
+ if (s.type !== "error")
393
+ continue;
394
+ const err = s.error;
395
+ if (!(err instanceof PermissionViolationError))
396
+ continue;
397
+ const id = probedInstanceOf(err.subject, what.endpoint);
398
+ if (id === undefined || !ids.has(id) || already.has(id))
399
+ continue;
400
+ already.add(id);
401
+ report(`! the broker refused this command's liveness probe for manager instance ${id}: ${err.message}`);
402
+ settle.get(id)?.();
403
+ }
404
+ }
405
+ catch {
406
+ /* the connection closed; the scatter is over and there is nothing left to attribute */
407
+ }
408
+ })();
409
+ return { refused, wasRefused: (id) => already.has(id) };
410
+ }
411
+ /**
412
+ * THE §13.5 LIVENESS HOOK the caller hands to a scatter, and the record of what it established.
413
+ *
414
+ * It is built HERE and not in core because it is entirely a statement about this caller's
415
+ * CREDENTIAL. A probe publishes on an instance's own rail, and only the layer that minted the
416
+ * credential knows which instance rails it carries; core cannot know, and a core that guessed would
417
+ * publish requests the broker must refuse — invisibly, since a refused publish looks exactly like a
418
+ * slow instance from the caller's side.
419
+ *
420
+ * TWO RULES, and both are about not asking questions this credential cannot ask:
421
+ * - an id OUTSIDE the pinned set gets `unknown` WITHOUT PUBLISHING. The scatter re-freezes the
422
+ * class on its own connection, so it can name an instance that registered after the freeze this
423
+ * credential was minted against. That instance is NEW, not dead, and the right answer for it is
424
+ * the deadline — reached without sending a request that would be refused.
425
+ * - an id INSIDE the set that the broker refuses anyway settles IMMEDIATELY on the violation
426
+ * rather than expiring into `unknown` at the budget, and is reported by name.
427
+ *
428
+ * Only `gone` ever licenses anything downstream, so every mistake this can make is a slower and
429
+ * more truthful answer, never a faster wrong one.
430
+ */
431
+ export function pinnedLivenessProbe(nc, opts) {
432
+ const deadlineMs = opts.probeDeadlineMs ?? PROBE_DEADLINE_MS;
433
+ const report = opts.report ?? ((line) => console.error(c.dim(line)));
434
+ const refusals = watchProbeRefusals(nc, { endpoint: opts.endpoint, ids: [...opts.pinned] }, report);
435
+ const verdicts = new Map();
436
+ return {
437
+ probeLiveness: async (instanceId) => {
438
+ if (!opts.pinned.has(instanceId)) {
439
+ verdicts.set(instanceId, "not-probed");
440
+ return "unknown";
441
+ }
442
+ const verdict = await Promise.race([
443
+ epProbeInstanceInterest(nc, opts.space, opts.endpoint, instanceId, opts.caller, { deadlineMs }),
444
+ refusals.refused(instanceId).then(() => "unknown"),
445
+ ]);
446
+ verdicts.set(instanceId, verdict === "gone" ? "gone" : refusals.wasRefused(instanceId) ? "probe-refused" : "unknown");
447
+ return verdict;
448
+ },
449
+ // A slot with no recorded verdict was never handed to the hook at all, which is a different fact
450
+ // from "probed and silent" and prints as one.
451
+ livenessOf: (instanceId) => verdicts.get(instanceId) ?? (opts.pinned.has(instanceId) ? "unknown" : "not-probed"),
452
+ };
453
+ }
454
+ /** The tier a scatter re-mints at. Only ONE tier can scatter — the freeze read is a privileged row
455
+ * and the admin tier is denied it deliberately — so this is a constant rather than a parameter, and
456
+ * it is named here so the re-mint below can never silently widen: it reproduces the credential the
457
+ * caller already resolved with, pinned, never a higher one. */
458
+ const SCATTER_PROFILE = "control-caller-privileged";
459
+ /** How long a single liveness probe waits for the broker's no-responders answer. It is NOT a tuning
460
+ * knob and cannot make the command wrong: the only verdict that changes anything is `gone`, and
461
+ * giving up early yields `unknown`, which leaves the full gather deadline standing exactly as it
462
+ * did before the probe existed. Sized to be generous against a busy broker while still expiring
463
+ * well inside the gather. */
464
+ const PROBE_DEADLINE_MS = 5_000;
465
+ /** How long the host names a census shows may take, all together. They are display metadata, so an
466
+ * instance that has not answered by then is listed without one. */
467
+ const REPORTED_HOST_BUDGET_MS = 2_000;
468
+ const HOST_NAME = /^[A-Za-z0-9_](?:[A-Za-z0-9_.-]{0,251}[A-Za-z0-9_])?$/;
469
+ /** Ask each instance that answered the census for its own `status`, on its `inst` route at the
470
+ * epoch it answered with, where this credential's pinned rails reach it. One bound covers every
471
+ * ask and aborts what is still out. Only a well-formed host name from that exact instance is kept;
472
+ * anything else leaves the instance without one, and nothing here changes what the census found. */
473
+ async function askReportedHosts(nc, space, service, answered, instances, reachable) {
474
+ const abort = new AbortController();
475
+ const timer = setTimeout(() => abort.abort(), REPORTED_HOST_BUDGET_MS);
476
+ try {
477
+ await Promise.all(instances.filter((slot) => slot.error === undefined && reachable.has(slot.instanceId)).map(async (slot) => {
478
+ const epoch = answered.get(slot.instanceId)?.responder.epoch;
479
+ if (epoch === undefined)
480
+ return;
481
+ try {
482
+ const pinned = { ...service, responder: { instanceId: slot.instanceId, epoch }, pinnedInstanceId: slot.instanceId };
483
+ const answer = await invokeCommand(nc, space, pinned, "status", undefined, { deadlineMs: REPORTED_HOST_BUDGET_MS, signal: abort.signal });
484
+ const host = answer.reply.ok === true ? answer.reply.data?.host : undefined;
485
+ if (answer.responder.instanceId === slot.instanceId && typeof host === "string" && HOST_NAME.test(host))
486
+ slot.reportedHost = host;
487
+ }
488
+ catch { /* display metadata only: the instance is listed without a host */ }
489
+ }));
490
+ }
491
+ finally {
492
+ clearTimeout(timer);
493
+ }
494
+ }
495
+ const VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
496
+ /** The version manager instance `instanceId` gives in its own `status`, asked on the route the
497
+ * refused call rode: its `inst` rail when that call was pinned, else the class rail with only that
498
+ * instance's answer accepted. Only a well-formed version from that exact instance is kept; one that
499
+ * is not, or no answer inside the bound, leaves it unknown. */
500
+ async function reportedManagerVersion(nc, space, caller, instanceId, pinned) {
501
+ const signal = AbortSignal.timeout(REPORTED_HOST_BUDGET_MS);
502
+ try {
503
+ const service = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, { deadlineMs: REPORTED_HOST_BUDGET_MS, signal, commands: ["status"], ...(pinned ? { instanceId } : { classInstanceId: instanceId }) });
504
+ const answer = await invokeCommand(nc, space, service, "status", undefined, { deadlineMs: REPORTED_HOST_BUDGET_MS, signal });
505
+ const version = answer.reply.ok === true ? answer.reply.data?.version : undefined;
506
+ return answer.responder.instanceId === instanceId && typeof version === "string" && VERSION.test(version) ? version : undefined;
507
+ }
508
+ catch {
509
+ return undefined;
510
+ }
511
+ }
512
+ /**
513
+ * Whether the scatter's liveness probe rides a BARE connection, a freshly minted pinned instrument,
514
+ * or nothing at all. The sixth seed reach on the attach path, made explicit.
515
+ *
516
+ * Named and exported so it is GRADABLE, because it is the one site on this path that still answers
517
+ * "is this an open mesh?" from the absence of a credential. That inference is issue #1205 itself,
518
+ * and it is only tolerable here because of what this answer is allowed to DO: it selects whether a
519
+ * frozen instance's liveness is probed, so a wrong answer downgrades a row to `not-probed` and
520
+ * never refuses an attach. A caller that ever turns this into a refusal must change this function,
521
+ * which is the point of it having a name.
522
+ *
523
+ * bare — no credential and no bearer: an open mesh, whose connection can already publish.
524
+ * mint — a sealed mesh that holds its seed AND arrived on a credential: re-mint pinned.
525
+ * as-is — everything else (a user bearer, a raw off-registry cred, a sealed mesh with no seed):
526
+ * keep the credential it arrived with and probe nothing, which is the pre-existing
527
+ * behaviour and prints `not-probed` rather than degrading silently.
528
+ */
529
+ export function scatterProbeMaterial(auth, spaceAuth) {
530
+ if (!auth.creds && !auth.bearer)
531
+ return { kind: "bare" };
532
+ if (spaceAuth && auth.creds)
533
+ return { kind: "mint", auth: spaceAuth };
534
+ return { kind: "as-is" };
535
+ }
536
+ /** Commands whose closures one scatter run invokes: the scattered verb, plus status for host labels. */
537
+ export function scatterResolveCommands(command, reportHosts) {
538
+ return [...new Set([command, ...(reportHosts ? ["status"] : [])])];
539
+ }
540
+ /** The ep-rail CLASS SCATTER (P2 item 3, `cotal ps` default).
541
+ *
542
+ * TWO connections, because a connection's permissions are fixed at authentication and this command
543
+ * cannot know which instance rails it needs until it has read the registry:
544
+ *
545
+ * 0. FREEZE the live class from the records registry (the same §13.9 read the scatter itself
546
+ * does), then close. This yields the exact instance ids this invocation will address.
547
+ * 1. RE-MINT the one-shot instrument LOCALLY, pinned to exactly those ids, and run the
548
+ * unpinned {@link resolveService} + {@link scatterCommand} on it — now the §13.5 liveness probe
549
+ * can publish on each frozen instance's own rail and the gather can end as soon as every slot
550
+ * has either answered or been affirmed gone by the broker.
551
+ *
552
+ * THE COST IS THE MINT, NOT A ROUND TRIP. `mintCreds` is a local JWT signature against the space
553
+ * seed already in hand (measured at 22-45ms here); the extra connection is the only wire cost. The
554
+ * earlier attempt at this measured 3.2s because it re-ran the whole target resolve (registry read,
555
+ * preflight, stale-prune) to get a pinned credential, and shipped a regression to fix a hypothesis.
556
+ * This re-mints and nothing else.
557
+ *
558
+ * NO WILDCARD INSTANCE ROW is minted, which is the invariant the whole two-connection shape exists
559
+ * to preserve: the pin is a list of exact ids known before the mint, the same precondition `--on`
560
+ * satisfies with one id.
561
+ *
562
+ * WHERE THERE IS NO SEED THERE IS NO PROBE, and the deadline stands. A user-mode bearer and a raw
563
+ * off-registry cred cannot re-mint, so their frozen slots are `not-probed` and say so. This is not
564
+ * a silent degrade: it is the pre-existing behaviour, and the row prints which of the two it is.
565
+ */
566
+ async function askManagerScatterEp(client, space, server, op, auth, spaceAuth, timeoutMs, reportHosts = false) {
567
+ const mapped = EP_COMMANDS[op];
568
+ if (!mapped)
569
+ return { ok: false, error: `unknown manager op "${op}" (no v0.4 command mapping)` };
570
+ if (mapped.targeted)
571
+ return { ok: false, error: `${op} is targeted and cannot be scattered across instances` };
572
+ // ---- connection 0: freeze the class. A registry read that fails here fails the command with the
573
+ // registry's own verdict (never the managers'), which `epRailFailure` already words.
574
+ let pinnedIds;
575
+ try {
576
+ pinnedIds = await withControlConnection(server, auth, async (nc) => {
577
+ // `checkAPI: false` for the same reason `scatterCommand` uses it: the freeze rides the scoped
578
+ // §13.9 records rows, never an account-level JetStream probe.
579
+ const jsm = await jetstreamManager(nc, { checkAPI: false });
580
+ return (await freezeExpectedSet(jsm, space, BASELINE_LIFECYCLE_ENDPOINT)).map((f) => f.instanceId);
581
+ });
582
+ }
583
+ catch (e) {
584
+ const { error, code, unanswered } = epRailFailure(client, e);
585
+ return { ok: false, error: error ?? "error", ...(code === undefined ? {} : { code }), ...(unanswered ? { unanswered } : {}), ...(noLiveInstances(e) ? { noLiveManager: true } : {}) };
586
+ }
587
+ // ---- the LOCAL pinned re-mint, decided by {@link scatterProbeMaterial} rather than inline.
588
+ // This is the SIXTH place the attach path reaches for a seed (attach → pinForTarget → locateSeat
589
+ // → scatterManager → here), and it is named as such because its answer is still derived from the
590
+ // ABSENCE of a credential, which is the inference issue #1205 was about. It is safe here and it
591
+ // is NOT safe by accident: this site only decides whether to PROBE LIVENESS, so its worst outcome
592
+ // is a `not-probed` row, never a refusal to attach. It is extracted anyway so a future caller
593
+ // that starts making a REFUSAL out of this answer has to pass through a named decision that says
594
+ // so, instead of re-deriving openness from a nullable field for the second time in one path.
595
+ const material = scatterProbeMaterial(auth, spaceAuth);
596
+ let probeAuth = auth;
597
+ let caller = auth.epCaller;
598
+ let probed = material.kind === "bare" ? new Set(pinnedIds) : new Set();
599
+ if (material.kind === "mint") {
600
+ const identity = newIdentity();
601
+ const uid = mintLifecycleUid();
602
+ probeAuth = {
603
+ ...auth,
604
+ creds: await mintCreds(material.auth, identity, SCATTER_PROFILE, {
605
+ lifecycleUid: uid,
606
+ endpointCapabilities: instancePinnedInstrumentCapabilities("privileged", pinnedIds),
607
+ }),
608
+ };
609
+ caller = { owner: DEV_OWNER, actor: identity.id, uid };
610
+ probed = new Set(pinnedIds);
611
+ }
612
+ // ---- connection 1: resolve + scatter, with the probe closure this credential can actually back.
613
+ try {
614
+ return await withControlConnection(server, probeAuth, async (nc) => {
615
+ const { probeLiveness, livenessOf } = pinnedLivenessProbe(nc, { space, endpoint: BASELINE_LIFECYCLE_ENDPOINT, caller, pinned: probed });
616
+ const commands = scatterResolveCommands(mapped.command, reportHosts);
617
+ const service = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, { deadlineMs: 10_000, commands });
618
+ const result = await scatterCommand(nc, space, service, mapped.command, undefined, {
619
+ deadlineMs: timeoutMs ?? 8_000,
620
+ reconcileDeadlineMs: 3_000,
621
+ probeLiveness,
622
+ });
623
+ const instances = [];
624
+ for (const [instanceId, ar] of result.replies) {
625
+ if (ar.reply.ok === true)
626
+ instances.push({ instanceId, reachable: true, data: ar.reply.data });
627
+ else
628
+ instances.push({ instanceId, reachable: true, error: ar.reply.error?.message ?? ar.reply.error?.code ?? "error", ...(ar.reply.error?.code ? { code: ar.reply.error.code } : {}) });
629
+ }
630
+ // A frozen instance that never answered is UNREACHABLE — surfaced, never silently dropped
631
+ // (pin 3) — and now carries WHY it is silent, as far as this command could establish it.
632
+ for (const instanceId of result.missing)
633
+ instances.push({ instanceId, reachable: false, liveness: livenessOf(instanceId) });
634
+ if (reportHosts)
635
+ await askReportedHosts(nc, space, service, result.replies, instances, probed);
636
+ return { ok: true, instances };
637
+ });
638
+ }
639
+ catch (e) {
640
+ const { error, code, unanswered } = epRailFailure(client, e);
641
+ return { ok: false, error: error ?? "error", ...(code === undefined ? {} : { code }), ...(unanswered ? { unanswered } : {}) };
642
+ }
643
+ }
644
+ /** SCATTER one untargeted read (`ps`) across EVERY registered manager instance in the space and merge
645
+ * the attributed results — the `cotal ps` default in a multi-manager space (P2 item 3). Auth shapes
646
+ * match {@link askManager}: a minted instrument or user bearer rides its own caller triple; a raw
647
+ * pre-1c `--creds` file is refused loud; an OPEN mesh synthesizes a DEV_OWNER triple and connects
648
+ * bare (the broker enforces nothing, so the records freeze reads freely). `spaceAuth` is the
649
+ * resolved mesh's trust material from {@link resolveControlTarget}: present ⇒ the scatter re-mints
650
+ * its instrument pinned to the frozen instance ids and can probe their liveness; absent ⇒ it runs
651
+ * exactly as it did before and every silent slot is reported `not-probed`. */
652
+ export async function scatterManager(client, space, server, op, auth = {}, spaceAuth, timeoutMs, opts = {}) {
653
+ const who = controlCaller(auth);
654
+ if ("refusal" in who)
655
+ return { ok: false, error: who.refusal };
656
+ return askManagerScatterEp(client, space, server, op, { ...auth, epCaller: who.caller }, spaceAuth, timeoutMs, opts.reportHosts === true);
657
+ }
658
+ /** The refusal `ps` gives, raised before an unpinned operator read when the registry lists no live
659
+ * manager, rather than after the read's describe has waited out its whole deadline. Only the
660
+ * operator instrument holds the registry rows, so a caller passes a user's manager view or a pinned
661
+ * call straight to the read. Any other registry outcome, an unreadable registry included, leaves
662
+ * the read to decide. `purpose` completes "no manager is running for this space to …". Returns the
663
+ * live instance ids the registry listed, or undefined when it could not be read. */
664
+ export async function refuseWithoutLiveManager(space, server, auth, target, purpose) {
665
+ try {
666
+ return await liveManagerIds(space, server, auth);
667
+ }
668
+ catch (e) {
669
+ if (!(e instanceof EpEnvelopeError) || !noLiveInstances(e))
670
+ return undefined;
671
+ throw new CommandFailure("refused", "TARGET_UNAVAILABLE", `no manager is running for this space to ${purpose}`, { target, native: { code: e.code, detail: `${e.code}: ${e.message}` }, remedy: "start the stack's manager with bare `cotal up`; a stack started with --no-manager has none" });
672
+ }
673
+ }
674
+ /** The host one manager instance reports in its own `status`, the name `cotal ps` prints as its
675
+ * reported host, or undefined when it does not answer or names no well-formed host. Presence
676
+ * carries no instance id off a user-auth mesh, so this read is where a pinned manager's host comes
677
+ * from there. The view is the caller's own and is not kept.
678
+ *
679
+ * It takes the ordinary control deadline, NOT {@link REPORTED_HOST_BUDGET_MS}. That budget belongs
680
+ * to the census above, where a host name is display metadata and a slow instance is listed without
681
+ * one. Here the answer decides whether a launch stops to ask the operator a question, so a read
682
+ * that gives up early does not lose a column, it asks about a manager that is on this host.
683
+ * Measured: under heavy load the 2s budget expired and the question came back. */
684
+ export async function reportedManagerHost(client, space, server, auth, instanceId) {
685
+ let reply;
686
+ try {
687
+ reply = await askManager(client, space, server, "managerStatus", undefined, auth, "owner", undefined, { instanceId });
688
+ }
689
+ catch {
690
+ return undefined;
691
+ }
692
+ const host = reply.ok ? reply.data?.host : undefined;
693
+ return typeof host === "string" && HOST_NAME.test(host) ? host : undefined;
694
+ }
695
+ /** A pinned instance the space's registry does not list has nothing behind it to answer, so the
696
+ * command is refused before it waits out the reply deadline, naming the managers that are running.
697
+ * A registry this side cannot read leaves the request to settle it. */
698
+ export async function refuseUnregisteredManager(space, server, auth, on) {
699
+ let running;
700
+ try {
701
+ running = await liveManagerIds(space, server, auth);
702
+ }
703
+ catch (e) {
704
+ if (!(e instanceof EpEnvelopeError) || !noLiveInstances(e))
705
+ return;
706
+ running = [];
707
+ }
708
+ if (running.includes(on))
709
+ return;
710
+ const target = { space, requestedManagerInstance: on };
711
+ // With none running this is the unpinned refusal, which names the same way out.
712
+ if (running.length === 0)
713
+ throw new CommandFailure("refused", "TARGET_UNAVAILABLE", `no manager is running for this space, so --on ${on} names none. Nothing was sent`, { target, remedy: "start the stack's manager with bare `cotal up`; a stack started with --no-manager has none" });
714
+ const which = running.length === 1 ? `the one running is ${running[0]}` : `the ones running are ${running.join(", ")}`;
715
+ throw new CommandFailure("refused", "TARGET_NOT_FOUND", `--on ${on} names no manager running for space ${space}; ${which}. Nothing was sent`, {
716
+ target,
717
+ remedy: running.length === 1 ? `run it again with --on ${running[0]}, or without --on` : "run it again with --on naming one of them",
718
+ });
719
+ }
720
+ /** Whether the broker affirms that nothing is subscribed on manager `instanceId`'s own rail, so its
721
+ * registration outlived its host. The probe settles on the broker's no-responders answer, so a gone
722
+ * manager costs no reply deadline. A live manager never answers the probe, which then runs its whole
723
+ * budget, so a caller that also asks the manager itself passes `signal` and fires it once that
724
+ * answer arrives. Any other outcome, an aborted probe or one this credential cannot send included,
725
+ * is `false`. */
726
+ export async function managerRailGone(space, server, auth, instanceId, signal) {
727
+ const who = controlCaller(auth);
728
+ if ("refusal" in who || signal?.aborted)
729
+ return false;
730
+ const aborted = new Promise((resolve) => signal?.addEventListener("abort", () => resolve(false), { once: true }));
731
+ try {
732
+ return await withControlConnection(server, auth, (nc) => Promise.race([
733
+ epProbeInstanceInterest(nc, space, BASELINE_LIFECYCLE_ENDPOINT, instanceId, who.caller, { deadlineMs: PROBE_DEADLINE_MS }).then((verdict) => verdict === "gone"),
734
+ aborted,
735
+ ]));
736
+ }
737
+ catch {
738
+ return false;
739
+ }
740
+ }
741
+ /** The manager instances the space's registry lists as live, frozen as a scatter freezes them. An
742
+ * empty registry throws the registry's own refusal. */
743
+ export async function liveManagerIds(space, server, auth) {
744
+ return withControlConnection(server, auth, async (nc) => (await freezeExpectedSet(await jetstreamManager(nc, { checkAPI: false }), space, BASELINE_LIFECYCLE_ENDPOINT)).map((f) => f.instanceId));
745
+ }
746
+ /** The stable code a refused manager READ reports, by the endpoint's catalog code. A read has no
747
+ * effect, so an outcome the catalog does not classify is a failed read, never an unknown one. */
748
+ const READ_REFUSALS = {
749
+ unauthenticated: "AUTH_REQUIRED",
750
+ "permission-denied": "PERMISSION_DENIED",
751
+ "not-found": "TARGET_NOT_FOUND",
752
+ unavailable: "TARGET_UNAVAILABLE",
753
+ "deadline-exceeded": "TARGET_UNAVAILABLE",
754
+ unimplemented: "UNSUPPORTED",
755
+ "contract-mismatch": "UNSUPPORTED",
756
+ conflict: "CONFLICT",
757
+ };
758
+ /** A manager read that did not answer, as the classified failure a structured command reports: the
759
+ * same sentence, the endpoint's own code under `error.cause`, and the stale-store hint as its remedy
760
+ * when one applies. A request nobody answered carries `deadline-exceeded`, an unavailable target. */
761
+ export function readFailure(reply, target) {
762
+ const message = reply.error ?? "the manager refused the read";
763
+ const remedy = staleStoreHint(message);
764
+ const native = reply.code === undefined && reply.details === undefined ? undefined : { ...(reply.code === undefined ? {} : { code: reply.code }), ...(reply.details === undefined ? {} : { detail: reply.details }) };
765
+ const opts = { target, ...(remedy ? { remedy } : {}), ...(native ? { native } : {}) };
766
+ const code = reply.code === undefined ? undefined : READ_REFUSALS[reply.code];
767
+ return code ? new CommandFailure("refused", code, message, opts) : new CommandFailure("failed", "OPERATION_FAILED", message, opts);
768
+ }
769
+ //# sourceMappingURL=manager.js.map