@cotal-ai/workspace 0.133.12 → 0.134.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/Release/darwin-arm64/build.json +1 -1
- package/build/Release/darwin-x64/build.json +1 -1
- package/build/Release/win32-arm64/build.json +1 -1
- package/build/Release/win32-x64/build.json +1 -1
- package/dist/control/actions.d.ts +141 -0
- package/dist/control/actions.d.ts.map +1 -0
- package/dist/control/actions.js +192 -0
- package/dist/control/actions.js.map +1 -0
- package/dist/control/dial-retry.d.ts +33 -0
- package/dist/control/dial-retry.d.ts.map +1 -0
- package/dist/control/dial-retry.js +70 -0
- package/dist/control/dial-retry.js.map +1 -0
- package/dist/control/index.d.ts +5 -0
- package/dist/control/index.d.ts.map +1 -0
- package/dist/control/index.js +5 -0
- package/dist/control/index.js.map +1 -0
- package/dist/control/manager.d.ts +291 -0
- package/dist/control/manager.d.ts.map +1 -0
- package/dist/control/manager.js +769 -0
- package/dist/control/manager.js.map +1 -0
- package/dist/control/seat.d.ts +168 -0
- package/dist/control/seat.d.ts.map +1 -0
- package/dist/control/seat.js +225 -0
- package/dist/control/seat.js.map +1 -0
- package/dist/launch.d.ts +94 -0
- package/dist/launch.d.ts.map +1 -0
- package/dist/launch.js +58 -0
- package/dist/launch.js.map +1 -0
- package/dist/maintenance.d.ts +7 -0
- package/dist/maintenance.d.ts.map +1 -1
- package/dist/maintenance.js +24 -0
- package/dist/maintenance.js.map +1 -1
- package/dist/output.d.ts +2 -0
- package/dist/output.d.ts.map +1 -1
- package/dist/output.js +5 -0
- package/dist/output.js.map +1 -1
- package/dist/view/index.d.ts +3 -0
- package/dist/view/index.d.ts.map +1 -0
- package/dist/view/index.js +3 -0
- package/dist/view/index.js.map +1 -0
- package/dist/view/mesh-view.d.ts +225 -0
- package/dist/view/mesh-view.d.ts.map +1 -0
- package/dist/view/mesh-view.js +550 -0
- package/dist/view/mesh-view.js.map +1 -0
- package/dist/view/observer.d.ts +26 -0
- package/dist/view/observer.d.ts.map +1 -0
- package/dist/view/observer.js +26 -0
- package/dist/view/observer.js.map +1 -0
- package/package.json +18 -4
|
@@ -0,0 +1,769 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { BASELINE_LIFECYCLE_ENDPOINT, DEV_OWNER, EP_UNANSWERED, EpEnvelopeError, GOAL_BEARING_COMMANDS, goalFollowRefusal, epProbeInstanceInterest, freezeExpectedSet, instancePinnedInstrumentCapabilities, invokeCommand, invokeRepairingSplit, issuedUserCaller, mintCreds, oneLine, parseEpSubject, respondedButUnbound, unansweredRequest, unansweredRail, registryReadFailed, noLiveInstances, undeclaredArg, undeclaredArgInstance, controlReplyFrom, controlReplyFromThrown, submitAndFollowGoal, scatterCommand, mintLifecycleUid, newIdentity, dialerFor, resolveService, standaloneConnectOpts, } from "@cotal-ai/core";
|
|
3
|
+
import { PermissionViolationError } from "@nats-io/transport-node";
|
|
4
|
+
import { jetstreamManager } from "@nats-io/jetstream";
|
|
5
|
+
import { c } from "../colors.js";
|
|
6
|
+
import { controlCaller } from "../control-target.js";
|
|
7
|
+
import { staleStoreHint } from "../maintenance.js";
|
|
8
|
+
import { CommandFailure, nativeEndpointRefusal } from "../output.js";
|
|
9
|
+
import { DIAL_TIMEOUT_MS, DialNotAdmitted, dialWithRetry, notAdmittedRefusal } from "./dial-retry.js";
|
|
10
|
+
/** Client-side request window for the manager's readiness-waiting launch ops (`start`, and the
|
|
11
|
+
* manifest `launch` — both funnel into the same startAgent readiness wait). #159 B1: the manager
|
|
12
|
+
* replies only on a REAL outcome — presence join, process exit, or its ~30s readiness backstop —
|
|
13
|
+
* so these requests must OUTLIVE that window, not the 5s op default. The tier rule forbids
|
|
14
|
+
* importing the manager's READINESS_TIMEOUT_MS here; the launch-parity smoke enforces the
|
|
15
|
+
* relation by test. */
|
|
16
|
+
export const START_TIMEOUT_MS = 40_000;
|
|
17
|
+
/** v0.3 ctl op → v0.4 typed command (P2 item 1, 1c.2b): the wire names the manager REGISTERS
|
|
18
|
+
* (manager-service-contract ROWS). `start` is creation (`spawn`), a NAMED `stop` is the one
|
|
19
|
+
* owner/any-mode terminal (`despawn`), the per-agent `status` read is `inspect`; the camelCase
|
|
20
|
+
* admin family maps to its kebab-case wire names. `targeted` marks the three commands whose
|
|
21
|
+
* `{name}` argument becomes a §13.2 target block (resolved to the agent's principal triple via
|
|
22
|
+
* the name-keyed `inspect` read — it rides the spawn capability arm, so resolution reach equals
|
|
23
|
+
* despawn/attach reach; the wire target is (owner, actor, lifecycleUid), never an alias). */
|
|
24
|
+
const EP_COMMANDS = {
|
|
25
|
+
managerStatus: { command: "status" },
|
|
26
|
+
start: { command: "spawn" },
|
|
27
|
+
resolveAgent: { command: "resolve-agent" },
|
|
28
|
+
resolveCwd: { command: "resolve-cwd" },
|
|
29
|
+
stop: { command: "despawn", targeted: true },
|
|
30
|
+
stopSelf: { command: "stop", self: true },
|
|
31
|
+
attach: { command: "attach", targeted: true },
|
|
32
|
+
input: { command: "input", targeted: true },
|
|
33
|
+
status: { command: "inspect" },
|
|
34
|
+
ps: { command: "ps" },
|
|
35
|
+
slots: { command: "slots" },
|
|
36
|
+
models: { command: "models" },
|
|
37
|
+
listPersonas: { command: "list-personas" },
|
|
38
|
+
launch: { command: "launch" },
|
|
39
|
+
purge: { command: "purge" },
|
|
40
|
+
resumePreserved: { command: "resume-preserved" },
|
|
41
|
+
commitResume: { command: "commit-resume" },
|
|
42
|
+
finalizeResume: { command: "finalize-resume" },
|
|
43
|
+
preparePreservation: { command: "prepare-preservation" },
|
|
44
|
+
commitPreservation: { command: "commit-preservation" },
|
|
45
|
+
abortPreservation: { command: "abort-preservation" },
|
|
46
|
+
transcriptReceive: { command: "transcript-receive" },
|
|
47
|
+
chatSessionOpen: { command: "chat-session-open" },
|
|
48
|
+
chatSessionRetire: { command: "chat-session-retire" },
|
|
49
|
+
runPs: { command: "run-ps" },
|
|
50
|
+
runStatus: { command: "run-status" },
|
|
51
|
+
};
|
|
52
|
+
/** The ep-rail control call — since 1d {@link askManager}'s ONLY path: one short-lived raw
|
|
53
|
+
* connection, a fresh `resolveService` (describe → §13.7 store fetch → digest-verified recompile
|
|
54
|
+
* — the generic item-5 caller, no hand-imported manager schemas), then the mapped command.
|
|
55
|
+
* Targeted ops resolve the alias to its CURRENT principal triple through the name-keyed
|
|
56
|
+
* `inspect` read first and ride mode `any` (admin instruments) or `owner` per {@link
|
|
57
|
+
* ControlReach}.
|
|
58
|
+
*
|
|
59
|
+
* `standaloneConnectOpts` handles static credentials, a user bearer plus sentinel, and bare open
|
|
60
|
+
* meshes. The selected dialer matters: a user mesh behind an HTTPS edge has a `wss://` URL, which
|
|
61
|
+
* the raw node transport refuses even though endpoint, send, and spawn rails can reach it.
|
|
62
|
+
* A connect the broker refuses or does not complete in time sends nothing, so it is tried again
|
|
63
|
+
* ({@link dialWithRetry}); when every attempt ends that way, {@link notAdmittedRefusal} is thrown. */
|
|
64
|
+
async function askManagerEp(client, space, server, op, args, auth, reach, timeoutMs, pin) {
|
|
65
|
+
if (pin?.instanceId !== undefined && auth.managerInstanceId !== undefined && pin.instanceId !== auth.managerInstanceId)
|
|
66
|
+
return { ok: false, error: "the manager pin differs from this credential's instance authority", code: "permission-denied" };
|
|
67
|
+
const instanceId = pin?.instanceId ?? auth.managerInstanceId;
|
|
68
|
+
const mapped = EP_COMMANDS[op];
|
|
69
|
+
if (!mapped)
|
|
70
|
+
return { ok: false, error: `unknown manager op "${op}" (no v0.4 command mapping)` };
|
|
71
|
+
// Each attempt is a new connection, so a user bearer's connection nonce is minted afresh with it.
|
|
72
|
+
let connectOpts = {};
|
|
73
|
+
const nc = await dialWithRetry(() => {
|
|
74
|
+
connectOpts = standaloneConnectOpts(auth.creds ? { creds: auth.creds, tls: auth.tls === true } : auth.bearer ? { bearer: auth.bearer, sentinelCreds: auth.sentinelCreds, tls: auth.tls === true } : { tls: auth.tls === true });
|
|
75
|
+
return dialerFor(server)({ servers: server, ...connectOpts, timeout: DIAL_TIMEOUT_MS, maxReconnectAttempts: 0 });
|
|
76
|
+
}).catch((e) => {
|
|
77
|
+
if (!(e instanceof DialNotAdmitted))
|
|
78
|
+
throw e;
|
|
79
|
+
throw notAdmittedRefusal(e, instanceId === undefined ? "this space's manager" : `manager instance ${instanceId}`, "nothing was sent to the manager", { space, ...(instanceId === undefined ? {} : { managerInstanceId: instanceId }) }, auth.bearer !== undefined);
|
|
80
|
+
});
|
|
81
|
+
let request;
|
|
82
|
+
let intent;
|
|
83
|
+
let observationCause;
|
|
84
|
+
let caller;
|
|
85
|
+
try {
|
|
86
|
+
// A logged-in user's manager view is issued at the callout (SPEC 13.15): read its generation
|
|
87
|
+
// back from this connection's accepted row; a missing row refuses, never a legacy-rail retry.
|
|
88
|
+
caller = auth.bearer !== undefined && auth.managerInstanceId !== undefined
|
|
89
|
+
? await issuedUserCaller(nc, space, String(connectOpts.name), auth.epCaller)
|
|
90
|
+
: auth.epCaller;
|
|
91
|
+
// P2 item 3 `--on <instance>`: pin the resolve to the exact manager instance's `inst` route so a
|
|
92
|
+
// multi-manager space addresses the intended manager, never whichever wins the class anycast.
|
|
93
|
+
const followsGoal = GOAL_BEARING_COMMANDS.includes(mapped.command);
|
|
94
|
+
const resolveCommands = [...new Set([mapped.command, ...(mapped.targeted ? ["inspect"] : []), ...(followsGoal ? ["goal-result"] : [])])];
|
|
95
|
+
const service = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, { deadlineMs: 10_000, commands: resolveCommands, ...(followsGoal ? { optionalCommands: ["goal-result"] } : {}), ...(instanceId !== undefined ? { instanceId } : {}), ...(pin?.signal ? { signal: pin.signal } : {}) });
|
|
96
|
+
let target;
|
|
97
|
+
let sendArgs = args;
|
|
98
|
+
if (mapped.self)
|
|
99
|
+
target = { mode: "self" };
|
|
100
|
+
else if (mapped.targeted) {
|
|
101
|
+
const name = String(args?.name ?? "").trim();
|
|
102
|
+
if (!name)
|
|
103
|
+
return { ok: false, error: `${op} requires a name` };
|
|
104
|
+
// Alias -> CURRENT principal triple via the manager's name-keyed `inspect` read (§13.2: a
|
|
105
|
+
// target is a triple, never an alias) - the same resolution the connector uses. `inspect`
|
|
106
|
+
// rides the SPAWN capability arm as well as the instrument read set, so every caller class
|
|
107
|
+
// that can despawn/attach can also resolve its target (a `ps` SCAN here broker-drops exactly
|
|
108
|
+
// the spawn-scoped user bearers - the 1c.2b read narrowing - and hangs their stop/attach).
|
|
109
|
+
const info = await invokeRepairingSplit(nc, space, service, "inspect", { name }, { deadlineMs: 10_000 });
|
|
110
|
+
if (info.reply.ok !== true) {
|
|
111
|
+
const refused = controlReplyFrom(info.reply);
|
|
112
|
+
// Only the inspect read was published. Preserve its native error, but record the local
|
|
113
|
+
// proof that this targeted action was never submitted.
|
|
114
|
+
return { ...refused, preSubmitRefusal: true, managerInstanceId: info.responder.instanceId, managerEpoch: info.responder.epoch, error: `could not resolve "${name}": ${refused.error}` };
|
|
115
|
+
}
|
|
116
|
+
const row = info.reply.data;
|
|
117
|
+
if (pin?.expectedTarget && (row.id !== pin.expectedTarget.id || row.lifecycleUid !== pin.expectedTarget.lifecycleUid))
|
|
118
|
+
return { ok: false, code: "not-found", error: "the selected agent incarnation changed; refresh the agent list before attaching" };
|
|
119
|
+
// A STATIC row's `id` is the bare actor under the caller's own owner; a USER-mode row's `id`
|
|
120
|
+
// is the composite `owner.actor` principal key - split it (an embedded dot would break the
|
|
121
|
+
// target block's subject arity). Mode `any` spans owners (operator reach); mode `owner` pins
|
|
122
|
+
// the caller's own, so a foreign-owner target is broker-denied at publish.
|
|
123
|
+
const dot = row.id.indexOf(".");
|
|
124
|
+
const [tOwner, tActor] = dot > 0 ? [row.id.slice(0, dot), row.id.slice(dot + 1)] : [caller.owner, row.id];
|
|
125
|
+
// Target mode from the RESOLVED target owner: an own-domain target rides `owner` mode (pinned
|
|
126
|
+
// to the caller's own owner); a CROSS-owner target rides `any` mode - which the broker admits
|
|
127
|
+
// only for a caller holding the admin instrument rows (a static admin instrument, or a user
|
|
128
|
+
// bearer whose ledger `admin` scope the callout minted them into). `reach: "any"` forces
|
|
129
|
+
// any-mode for a static admin instrument even on its own domain. So a spawn-scoped caller's
|
|
130
|
+
// cross-owner despawn/attach is broker-denied at publish (no any-mode row), while an
|
|
131
|
+
// admin-scoped operator's is admitted and the manager's fresh ledger check governs.
|
|
132
|
+
const mode = reach === "any" || tOwner !== caller.owner ? "any" : "owner";
|
|
133
|
+
target = {
|
|
134
|
+
mode,
|
|
135
|
+
owner: tOwner,
|
|
136
|
+
actor: tActor,
|
|
137
|
+
lifecycleUid: row.lifecycleUid,
|
|
138
|
+
};
|
|
139
|
+
const { name: _dropped, ...rest } = args ?? {};
|
|
140
|
+
sendArgs = Object.keys(rest).length ? rest : undefined;
|
|
141
|
+
}
|
|
142
|
+
const blocked = followsGoal ? goalFollowRefusal(service) : undefined;
|
|
143
|
+
if (blocked)
|
|
144
|
+
return { ...controlReplyFrom(blocked.reply), nativeOutcome: "not-executed", managerInstanceId: service.responder.instanceId, managerEpoch: service.responder.epoch };
|
|
145
|
+
const id = randomUUID();
|
|
146
|
+
intent = { requestId: id, ...(followsGoal ? { requestedGoalId: id } : {}) };
|
|
147
|
+
if (pin?.journal)
|
|
148
|
+
await pin.journal.beforeSubmit(intent);
|
|
149
|
+
const invokeOpts = { ...(target ? { target } : {}), deadlineMs: timeoutMs ?? 10_000, id, ...(pin?.signal ? { signal: pin.signal } : {}) };
|
|
150
|
+
const submit = async (signal) => {
|
|
151
|
+
const answer = await invokeRepairingSplit(nc, space, service, mapped.command, sendArgs, { ...invokeOpts, ...(signal ? { signal } : {}) });
|
|
152
|
+
const data = answer.reply.data;
|
|
153
|
+
request = { requestId: answer.reply.id, managerInstanceId: answer.responder.instanceId, managerEpoch: answer.responder.epoch,
|
|
154
|
+
...(answer.reply.ok && typeof data?.goalId === "string" ? { goalId: data.goalId } : {}),
|
|
155
|
+
...(answer.reply.ok && typeof data?.fingerprint === "string" ? { fingerprint: data.fingerprint } : {}),
|
|
156
|
+
...(answer.reply.ok && typeof data?.goalId === "string" && typeof data.name === "string" && typeof data.owner === "string" && typeof data.actor === "string" && typeof data.uid === "string"
|
|
157
|
+
? { allocation: { name: data.name, owner: data.owner, actor: data.actor, lifecycleUid: data.uid } } : {}),
|
|
158
|
+
};
|
|
159
|
+
if (answer.reply.ok && request.allocation && pin?.journal)
|
|
160
|
+
await pin.journal.accepted(request);
|
|
161
|
+
return answer;
|
|
162
|
+
};
|
|
163
|
+
// P2 item 2 (2b): a goal-bearing command (spawn/launch) FOLLOWS its acceptance to the goal
|
|
164
|
+
// terminal, so `spawn --detach` still returns on the real outcome (join / exit / ~30s uncertain)
|
|
165
|
+
// exactly like the pre-action blocking reply — UX unchanged, no --no-wait.
|
|
166
|
+
const r = followsGoal
|
|
167
|
+
? await submitAndFollowGoal(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, timeoutMs ?? START_TIMEOUT_MS, submit, {
|
|
168
|
+
reconcile: async (goalId, accepted, context) => {
|
|
169
|
+
// A class credential keeps its class route. An accepted instance ID grants no new rail;
|
|
170
|
+
// the native follower still verifies the canonical fact against the original caller/goal.
|
|
171
|
+
const current = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, context.caller, { ...(instanceId !== undefined ? { instanceId: accepted.responder.instanceId } : {}), commands: ["goal-result"], deadlineMs: context.deadlineMs, signal: context.signal });
|
|
172
|
+
const answer = await invokeRepairingSplit(nc, space, current, "goal-result", { goalId }, { deadlineMs: context.deadlineMs, signal: context.signal });
|
|
173
|
+
if (!answer.reply.ok) {
|
|
174
|
+
observationCause = { code: answer.reply.error?.code, outcome: answer.reply.error?.outcome };
|
|
175
|
+
throw new Error("the accepting manager did not provide a confirmed goal-result read");
|
|
176
|
+
}
|
|
177
|
+
return answer.reply.data;
|
|
178
|
+
},
|
|
179
|
+
})
|
|
180
|
+
: await submit();
|
|
181
|
+
if (r.reply.ok !== true)
|
|
182
|
+
return { ...controlReplyFrom(r.reply), answered: true, managerInstanceId: r.responder.instanceId, managerEpoch: r.responder.epoch, ...(observationCause ? { observationCause } : {}), ...(intent ? { intent } : {}), ...(request ? { request } : {}), ...(r.reply.error?.outcome ? { nativeOutcome: r.reply.error.outcome } : {}) };
|
|
183
|
+
// The ep `models` reply is normalized to `{catalogs}` — unwrap so call sites keep the ctl shape.
|
|
184
|
+
const data = mapped.command === "models" ? r.reply.data.catalogs : r.reply.data;
|
|
185
|
+
return { ok: true, managerInstanceId: r.responder.instanceId, managerEpoch: r.responder.epoch, ...(observationCause ? { observationCause } : {}), ...(intent ? { intent } : {}), ...(request ? { request } : {}), ...(data !== undefined ? { data } : {}) };
|
|
186
|
+
}
|
|
187
|
+
catch (e) {
|
|
188
|
+
// A key the manager's contract does not declare: ask the instance that refused it which release
|
|
189
|
+
// it runs, so the refusal can say which side is older.
|
|
190
|
+
const refusing = undeclaredArgInstance(e);
|
|
191
|
+
const managerVersion = refusing !== undefined && caller !== undefined ? await reportedManagerVersion(nc, space, caller, refusing, instanceId !== undefined) : undefined;
|
|
192
|
+
return { ...epRailFailure(client, e, instanceId === undefined ? pin : { instanceId }, managerVersion), ...(refusing !== undefined ? { managerInstanceId: refusing } : {}), ...(observationCause ? { observationCause } : {}), ...(intent ? { intent } : {}), ...(request ? { request, managerInstanceId: request.managerInstanceId, managerEpoch: request.managerEpoch } : {}), ...(e instanceof EpEnvelopeError && e.outcome ? { nativeOutcome: e.outcome } : {}) };
|
|
193
|
+
}
|
|
194
|
+
finally {
|
|
195
|
+
// Drain waits for in-flight NATS work. A dead broker never finishes that, and
|
|
196
|
+
// attach-auth-root hung here after spawn once nats-server exited.
|
|
197
|
+
await Promise.race([
|
|
198
|
+
nc.drain().catch(() => nc.close()),
|
|
199
|
+
new Promise((resolve) => setTimeout(resolve, 2_000).unref()),
|
|
200
|
+
]);
|
|
201
|
+
try {
|
|
202
|
+
nc.close();
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
/* already closed */
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
/** Classify only a proved pre-effect refusal as refused; a missing effect outcome is not proof.
|
|
210
|
+
* Every message names the agent when the target carries one and ends with the manager's own
|
|
211
|
+
* reason, kept to one line. */
|
|
212
|
+
export function managerFailure(reply, operation, target) {
|
|
213
|
+
const native = { code: reply.code, outcome: reply.nativeOutcome, ...(reply.preSubmitRefusal ? { detail: { phase: "target-resolution", actionSubmitted: false } } : reply.request || reply.intent ? { detail: { request: reply.request ?? null, intent: reply.intent ?? null, observation: reply.observationCause ?? null } } : {}) };
|
|
214
|
+
const code = nativeEndpointRefusal(reply.code);
|
|
215
|
+
const agent = target?.agent;
|
|
216
|
+
const what = typeof agent === "string" ? `${operation} for agent "${oneLine(agent)}"` : operation;
|
|
217
|
+
const reason = reply.error ? `: ${oneLine(reply.error)}` : "";
|
|
218
|
+
if (reply.preSubmitRefusal && code)
|
|
219
|
+
return new CommandFailure("refused", code, reply.error ? oneLine(reply.error) : `${operation} was not submitted because its target could not be resolved`, { target, native, remedy: "run `cotal ps` to check the selected manager and agent name" });
|
|
220
|
+
if (reply.nativeOutcome === "not-executed" && code)
|
|
221
|
+
return new CommandFailure("refused", code, `the manager refused ${what} before execution${reason}`, { target, native, remedy: "nothing ran; fix what the manager reported, then run the command again" });
|
|
222
|
+
if (reply.nativeOutcome === "executed" && !reply.unanswered)
|
|
223
|
+
return new CommandFailure("failed", "OPERATION_FAILED", `the manager reported that ${what} failed${reason}`, { target, native });
|
|
224
|
+
return new CommandFailure("unknown", "OPERATION_UNKNOWN", `the manager did not confirm the outcome of ${what}; it was not submitted again${reason}`, { target, native, remedy: "inspect the recorded manager, request and target before retrying" });
|
|
225
|
+
}
|
|
226
|
+
/** A describe that drew no answer means the command after it was never sent, so it is refused, as
|
|
227
|
+
* `chat` and the run verbs refuse it, and not an unknown outcome. Undefined for any other failure. */
|
|
228
|
+
export function unansweredDescribeRefusal(reply, operation, target) {
|
|
229
|
+
if (!describeUnanswered(reply))
|
|
230
|
+
return undefined;
|
|
231
|
+
const agent = target?.agent;
|
|
232
|
+
const what = typeof agent === "string" ? `${operation} for agent "${oneLine(agent)}"` : operation;
|
|
233
|
+
return new CommandFailure("refused", "TARGET_UNAVAILABLE", `the manager registered for this space did not answer, so ${what} was not sent: ${oneLine(reply.error ?? "no reply")}`, { target, native: { code: reply.code }, remedy: "cotal status shows whether this space's manager answers; a stack manager that runs but does not answer is named there with its pid and the way to end it" });
|
|
234
|
+
}
|
|
235
|
+
/** Whether the describe before a command drew no answer, so the command itself was never sent. */
|
|
236
|
+
export function describeUnanswered(reply) {
|
|
237
|
+
return reply.details?.some((detail) => detail.kind === EP_UNANSWERED && detail.command === "describe") === true;
|
|
238
|
+
}
|
|
239
|
+
/** Render an ep-rail failure for the operator. Three outcomes, told apart by core's markers and never
|
|
240
|
+
* by the catalog code: a responder's own `ok:false` describe reply is rethrown under ITS code
|
|
241
|
+
* (`unavailable` included), and a store read after an answered describe raises the same code, so
|
|
242
|
+
* the code says nothing about whether anyone answered.
|
|
243
|
+
* - UNANSWERED ({@link unansweredRequest}: no responder, or the reply deadline elapsed). The
|
|
244
|
+
* reachability verdict "no manager reachable" is stated here and only here, and only unpinned:
|
|
245
|
+
* an unanswered PINNED call names the instance instead, since three managers may be answering
|
|
246
|
+
* while the one the operator typed is not there, and "no manager reachable" sends them to the
|
|
247
|
+
* broker for a typo. Measured on a live three-manager mesh during review. The verdict is also
|
|
248
|
+
* scoped to the RAIL ({@link unansweredRail}): on the versioned rail it names that rail and says what the
|
|
249
|
+
* silence does not establish, because SPEC 13.15 keeps the two rails disjoint at the broker.
|
|
250
|
+
* - a REGISTRY READ on this side failed ({@link registryReadFailed}: the scatter's freeze or its
|
|
251
|
+
* reconcile). The managers were not the failure and may all be up; a verdict on them here sent
|
|
252
|
+
* the operator to the managers for a broker read.
|
|
253
|
+
* - everything else answered, or failed on this side with its own cause, and is printed as is.
|
|
254
|
+
* Prepending a verdict made the headline contradict the body: a describe REFUSED BY THE BROKER
|
|
255
|
+
* read as an unreachable manager, which is precisely the misreading the refusal was reworded to
|
|
256
|
+
* stop.
|
|
257
|
+
* A failure that is not an {@link EpEnvelopeError} carries no answer provenance at all, so no verdict
|
|
258
|
+
* is stated for it either: its message stands alone. `managerVersion` is the version the refusing
|
|
259
|
+
* manager reported in its own `status`, for a key its contract does not declare. */
|
|
260
|
+
export function epRailFailure(client, e, pin, managerVersion) {
|
|
261
|
+
if (!(e instanceof EpEnvelopeError))
|
|
262
|
+
return { ok: false, unanswered: false, error: e instanceof Error ? e.message : String(e) };
|
|
263
|
+
return { ...controlReplyFromThrown(e, epRailMessage(client, e, pin, managerVersion)), unanswered: unansweredRequest(e) };
|
|
264
|
+
}
|
|
265
|
+
function epRailMessage(client, e, pin, managerVersion) {
|
|
266
|
+
const instanceId = pin?.instanceId;
|
|
267
|
+
const detail = `${e.code}: ${e.message}`;
|
|
268
|
+
if (unansweredRequest(e)) {
|
|
269
|
+
// The verdict is SCOPED TO THE RAIL the request rode (SPEC 13.15). The legacy and versioned rails are
|
|
270
|
+
// disjoint subject spaces at the broker and an endpoint is required to serve both, so a manager
|
|
271
|
+
// built before the versioned rail subscribes `ep` alone and an issued caller can never see it.
|
|
272
|
+
// "No manager reachable" is then a claim about the mesh drawn from silence on one half of it,
|
|
273
|
+
// and #1630 measured it against a manager that was up, idle on the roster and starting runs the
|
|
274
|
+
// whole time. On the legacy rail there is no other half, so the verdict stands as it was.
|
|
275
|
+
const rail = unansweredRail(e);
|
|
276
|
+
const versioned = rail !== undefined && rail !== "ep" ? rail : undefined;
|
|
277
|
+
// TWO CAUSES, BOTH NAMED. Silence on a versioned rail is consistent with no manager at all AND
|
|
278
|
+
// with one older than the rail, and this side cannot tell them apart: the registry records no
|
|
279
|
+
// package version. An earlier wording named only the skew, which sent an operator whose manager
|
|
280
|
+
// was simply down to go and check a version.
|
|
281
|
+
const skew = versioned === undefined ? "" :
|
|
282
|
+
` The ${versioned} and legacy ep rails are disjoint at the broker (SPEC 13.15) and an endpoint must serve both, so this does not tell no manager running apart from one older than ${versioned}, which serves ep only and cannot answer here. Check whether a manager is running, and if it is, its version.`;
|
|
283
|
+
return instanceId !== undefined
|
|
284
|
+
? `manager instance ${instanceId} did not answer${versioned === undefined ? "" : ` on the ${versioned} rail`} (${detail})${skew}`
|
|
285
|
+
: versioned === undefined
|
|
286
|
+
? `no manager reachable on the ep rails (${detail})`
|
|
287
|
+
: `no manager answered on the ${versioned} rail (${detail})${skew}`;
|
|
288
|
+
}
|
|
289
|
+
if (registryReadFailed(e))
|
|
290
|
+
return `the manager registry could not be read: a broker read on this side, not the managers' silence, and they may all be up. Retry; if it persists, look at the broker's JetStream (${detail})`;
|
|
291
|
+
// A key the manager's contract does not declare means the manager runs a different release from
|
|
292
|
+
// the caller that built the args. Only the caller knows its own release and how it is updated, so
|
|
293
|
+
// it words the refusal.
|
|
294
|
+
const undeclared = undeclaredArg(e);
|
|
295
|
+
if (undeclared !== undefined)
|
|
296
|
+
return client.versionSkew(detail, undeclared, managerVersion);
|
|
297
|
+
// The unpinned class-queue split. Core says a call that addresses one instance does not split
|
|
298
|
+
// and stops there (a CLI flag name does not belong in a core error). The flag is named here only
|
|
299
|
+
// when the CALLER declared it has one (`pin` present) and did not pass it: an absent `pin` is a
|
|
300
|
+
// command with no `--on` at all, and telling it to type one is the same dead end one layer down.
|
|
301
|
+
// A marked `expired` is the other producer (a stale-epoch bind) and its remedy is re-resolving,
|
|
302
|
+
// so the flag is offered only for the split.
|
|
303
|
+
const unpinnedSplit = e.code === "failed-precondition" && respondedButUnbound(e) && pin !== undefined && instanceId === undefined;
|
|
304
|
+
return `${detail}${unpinnedSplit ? " Pin one manager instance with --on <instance> (the whole id, as `ps` prints it) to avoid the split." : ""}`;
|
|
305
|
+
}
|
|
306
|
+
/** Send one control command to the manager over the v0.4 service-endpoint rails and disconnect —
|
|
307
|
+
* since 1d the manager's ONLY control door (the `ctl` tiers are deleted). The target is already
|
|
308
|
+
* reachability- + auth-preflighted by {@link resolveControlTarget}. `reach` picks the operator
|
|
309
|
+
* mode for the two targeted ops (stop/attach): `owner` = the caller's own domain (the spawn
|
|
310
|
+
* capability's standing mint / a user bearer's own owner), `any` = the admin instrument's
|
|
311
|
+
* cross-agent reach. Three auth shapes reach the rails: a static instrument's caller triple, a
|
|
312
|
+
* user bearer's triple, or an OPEN mesh (no credential system — a bare connection under a
|
|
313
|
+
* synthesized DEV_OWNER triple, since the manager registered under DEV_OWNER and the broker
|
|
314
|
+
* enforces nothing). A raw `--creds` file from an older generation carries no ep rows and is
|
|
315
|
+
* refused loud (no silent ctl fallback exists anymore). */
|
|
316
|
+
export async function askManager(client, space, server, op, args, auth = {}, reach = "owner", timeoutMs, pin) {
|
|
317
|
+
const who = controlCaller(auth);
|
|
318
|
+
if ("refusal" in who)
|
|
319
|
+
return { ok: false, error: who.refusal };
|
|
320
|
+
return askManagerEp(client, space, server, op, args, { ...auth, epCaller: who.caller }, reach, timeoutMs, pin);
|
|
321
|
+
}
|
|
322
|
+
/** Open one short-lived control connection under whichever of the three auth shapes this command
|
|
323
|
+
* holds (static creds / user bearer + sentinel / bare open mesh) and hand it to `fn`. Extracted so
|
|
324
|
+
* the scatter's TWO connections cannot drift in their connect options: they differ in credential
|
|
325
|
+
* and in nothing else. The selected dialer matters because the raw node transport refuses a
|
|
326
|
+
* `wss://` mesh behind an HTTPS edge even when the other endpoint rails can reach it. */
|
|
327
|
+
export async function withControlConnection(server, auth, fn) {
|
|
328
|
+
const nc = await dialerFor(server)({
|
|
329
|
+
servers: server,
|
|
330
|
+
...standaloneConnectOpts(auth.creds ? { creds: auth.creds, tls: auth.tls === true } : auth.bearer ? { bearer: auth.bearer, sentinelCreds: auth.sentinelCreds, tls: auth.tls === true } : { tls: auth.tls === true }),
|
|
331
|
+
maxReconnectAttempts: 0,
|
|
332
|
+
});
|
|
333
|
+
try {
|
|
334
|
+
return await fn(nc);
|
|
335
|
+
}
|
|
336
|
+
finally {
|
|
337
|
+
await nc.drain().catch(() => nc.close());
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
/** The instance a REFUSED SUBJECT names, or `undefined` when that subject is not one of this
|
|
341
|
+
* probe's requests at all.
|
|
342
|
+
*
|
|
343
|
+
* The id is read as a whole subject TOKEN, through the same §13.9 grammar parser that built the
|
|
344
|
+
* subject, and never matched inside the text. Lifecycle tokens are `[a-z0-9]{26,32}`, so one
|
|
345
|
+
* frozen id can be a PREFIX of another (26 chars and the same 26 plus a suffix are both legal),
|
|
346
|
+
* and a substring test then attributes one broker refusal to two instances: the operator is told
|
|
347
|
+
* the caller could not ask about a manager it never published for, and that instance's probe settles
|
|
348
|
+
* as `probe-refused` on evidence that belongs to a different rail. Parsing removes the class of
|
|
349
|
+
* mistake rather than one instance of it: the endpoint and the route mode must match too, so a
|
|
350
|
+
* violation on any other plane, endpoint, or route is not this probe's and is not attributed. */
|
|
351
|
+
function probedInstanceOf(subject, endpoint) {
|
|
352
|
+
const parsed = parseEpSubject(subject);
|
|
353
|
+
if (parsed === null || parsed.plane !== "request" || parsed.route !== "inst")
|
|
354
|
+
return undefined;
|
|
355
|
+
return parsed.endpoint === endpoint ? parsed.instanceId : undefined;
|
|
356
|
+
}
|
|
357
|
+
/** Watch a connection's status stream for the broker's own PERMISSION VIOLATIONS and attribute each
|
|
358
|
+
* to the instance whose rail it names.
|
|
359
|
+
*
|
|
360
|
+
* A refused publish is invisible where it matters. The violation is delivered on the CONNECTION,
|
|
361
|
+
* asynchronously, while the publish itself returns normally — so a liveness probe with no grant
|
|
362
|
+
* does not fail, it goes quiet, and quiet is exactly what a live-but-slow instance looks like. The
|
|
363
|
+
* whole probe would then read as `unknown`, the deadline would be paid in full, and the operator
|
|
364
|
+
* would be told the manager was slow when the truth was that the caller never asked. So the
|
|
365
|
+
* violation is caught, attributed to the instance its subject names, reported on stderr once, and
|
|
366
|
+
* used to settle that probe immediately instead of letting it expire.
|
|
367
|
+
*
|
|
368
|
+
* ATTRIBUTION IS STRUCTURAL, not textual. The client types a permission violation as
|
|
369
|
+
* `PermissionViolationError` and carries the refused SUBJECT as a field, so nothing here parses
|
|
370
|
+
* the broker's prose: the subject goes through {@link probedInstanceOf} and yields one exact id or
|
|
371
|
+
* none. NAMED RESIDUAL: a connection error the client does not type as a permission violation is
|
|
372
|
+
* not attributed, and that probe expires into `unknown` at its budget — the behaviour that existed
|
|
373
|
+
* before this watch, which is slower and never wrong. */
|
|
374
|
+
function watchProbeRefusals(nc, what, report) {
|
|
375
|
+
const ids = new Set(what.ids);
|
|
376
|
+
const settle = new Map();
|
|
377
|
+
const already = new Set();
|
|
378
|
+
const pending = new Map();
|
|
379
|
+
const refused = (id) => {
|
|
380
|
+
if (already.has(id))
|
|
381
|
+
return Promise.resolve();
|
|
382
|
+
let p = pending.get(id);
|
|
383
|
+
if (!p) {
|
|
384
|
+
p = new Promise((resolve) => settle.set(id, resolve));
|
|
385
|
+
pending.set(id, p);
|
|
386
|
+
}
|
|
387
|
+
return p;
|
|
388
|
+
};
|
|
389
|
+
void (async () => {
|
|
390
|
+
try {
|
|
391
|
+
for await (const s of nc.status()) {
|
|
392
|
+
if (s.type !== "error")
|
|
393
|
+
continue;
|
|
394
|
+
const err = s.error;
|
|
395
|
+
if (!(err instanceof PermissionViolationError))
|
|
396
|
+
continue;
|
|
397
|
+
const id = probedInstanceOf(err.subject, what.endpoint);
|
|
398
|
+
if (id === undefined || !ids.has(id) || already.has(id))
|
|
399
|
+
continue;
|
|
400
|
+
already.add(id);
|
|
401
|
+
report(`! the broker refused this command's liveness probe for manager instance ${id}: ${err.message}`);
|
|
402
|
+
settle.get(id)?.();
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
catch {
|
|
406
|
+
/* the connection closed; the scatter is over and there is nothing left to attribute */
|
|
407
|
+
}
|
|
408
|
+
})();
|
|
409
|
+
return { refused, wasRefused: (id) => already.has(id) };
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* THE §13.5 LIVENESS HOOK the caller hands to a scatter, and the record of what it established.
|
|
413
|
+
*
|
|
414
|
+
* It is built HERE and not in core because it is entirely a statement about this caller's
|
|
415
|
+
* CREDENTIAL. A probe publishes on an instance's own rail, and only the layer that minted the
|
|
416
|
+
* credential knows which instance rails it carries; core cannot know, and a core that guessed would
|
|
417
|
+
* publish requests the broker must refuse — invisibly, since a refused publish looks exactly like a
|
|
418
|
+
* slow instance from the caller's side.
|
|
419
|
+
*
|
|
420
|
+
* TWO RULES, and both are about not asking questions this credential cannot ask:
|
|
421
|
+
* - an id OUTSIDE the pinned set gets `unknown` WITHOUT PUBLISHING. The scatter re-freezes the
|
|
422
|
+
* class on its own connection, so it can name an instance that registered after the freeze this
|
|
423
|
+
* credential was minted against. That instance is NEW, not dead, and the right answer for it is
|
|
424
|
+
* the deadline — reached without sending a request that would be refused.
|
|
425
|
+
* - an id INSIDE the set that the broker refuses anyway settles IMMEDIATELY on the violation
|
|
426
|
+
* rather than expiring into `unknown` at the budget, and is reported by name.
|
|
427
|
+
*
|
|
428
|
+
* Only `gone` ever licenses anything downstream, so every mistake this can make is a slower and
|
|
429
|
+
* more truthful answer, never a faster wrong one.
|
|
430
|
+
*/
|
|
431
|
+
export function pinnedLivenessProbe(nc, opts) {
|
|
432
|
+
const deadlineMs = opts.probeDeadlineMs ?? PROBE_DEADLINE_MS;
|
|
433
|
+
const report = opts.report ?? ((line) => console.error(c.dim(line)));
|
|
434
|
+
const refusals = watchProbeRefusals(nc, { endpoint: opts.endpoint, ids: [...opts.pinned] }, report);
|
|
435
|
+
const verdicts = new Map();
|
|
436
|
+
return {
|
|
437
|
+
probeLiveness: async (instanceId) => {
|
|
438
|
+
if (!opts.pinned.has(instanceId)) {
|
|
439
|
+
verdicts.set(instanceId, "not-probed");
|
|
440
|
+
return "unknown";
|
|
441
|
+
}
|
|
442
|
+
const verdict = await Promise.race([
|
|
443
|
+
epProbeInstanceInterest(nc, opts.space, opts.endpoint, instanceId, opts.caller, { deadlineMs }),
|
|
444
|
+
refusals.refused(instanceId).then(() => "unknown"),
|
|
445
|
+
]);
|
|
446
|
+
verdicts.set(instanceId, verdict === "gone" ? "gone" : refusals.wasRefused(instanceId) ? "probe-refused" : "unknown");
|
|
447
|
+
return verdict;
|
|
448
|
+
},
|
|
449
|
+
// A slot with no recorded verdict was never handed to the hook at all, which is a different fact
|
|
450
|
+
// from "probed and silent" and prints as one.
|
|
451
|
+
livenessOf: (instanceId) => verdicts.get(instanceId) ?? (opts.pinned.has(instanceId) ? "unknown" : "not-probed"),
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
/** The tier a scatter re-mints at. Only ONE tier can scatter — the freeze read is a privileged row
|
|
455
|
+
* and the admin tier is denied it deliberately — so this is a constant rather than a parameter, and
|
|
456
|
+
* it is named here so the re-mint below can never silently widen: it reproduces the credential the
|
|
457
|
+
* caller already resolved with, pinned, never a higher one. */
|
|
458
|
+
const SCATTER_PROFILE = "control-caller-privileged";
|
|
459
|
+
/** How long a single liveness probe waits for the broker's no-responders answer. It is NOT a tuning
|
|
460
|
+
* knob and cannot make the command wrong: the only verdict that changes anything is `gone`, and
|
|
461
|
+
* giving up early yields `unknown`, which leaves the full gather deadline standing exactly as it
|
|
462
|
+
* did before the probe existed. Sized to be generous against a busy broker while still expiring
|
|
463
|
+
* well inside the gather. */
|
|
464
|
+
const PROBE_DEADLINE_MS = 5_000;
|
|
465
|
+
/** How long the host names a census shows may take, all together. They are display metadata, so an
|
|
466
|
+
* instance that has not answered by then is listed without one. */
|
|
467
|
+
const REPORTED_HOST_BUDGET_MS = 2_000;
|
|
468
|
+
const HOST_NAME = /^[A-Za-z0-9_](?:[A-Za-z0-9_.-]{0,251}[A-Za-z0-9_])?$/;
|
|
469
|
+
/** Ask each instance that answered the census for its own `status`, on its `inst` route at the
|
|
470
|
+
* epoch it answered with, where this credential's pinned rails reach it. One bound covers every
|
|
471
|
+
* ask and aborts what is still out. Only a well-formed host name from that exact instance is kept;
|
|
472
|
+
* anything else leaves the instance without one, and nothing here changes what the census found. */
|
|
473
|
+
async function askReportedHosts(nc, space, service, answered, instances, reachable) {
|
|
474
|
+
const abort = new AbortController();
|
|
475
|
+
const timer = setTimeout(() => abort.abort(), REPORTED_HOST_BUDGET_MS);
|
|
476
|
+
try {
|
|
477
|
+
await Promise.all(instances.filter((slot) => slot.error === undefined && reachable.has(slot.instanceId)).map(async (slot) => {
|
|
478
|
+
const epoch = answered.get(slot.instanceId)?.responder.epoch;
|
|
479
|
+
if (epoch === undefined)
|
|
480
|
+
return;
|
|
481
|
+
try {
|
|
482
|
+
const pinned = { ...service, responder: { instanceId: slot.instanceId, epoch }, pinnedInstanceId: slot.instanceId };
|
|
483
|
+
const answer = await invokeCommand(nc, space, pinned, "status", undefined, { deadlineMs: REPORTED_HOST_BUDGET_MS, signal: abort.signal });
|
|
484
|
+
const host = answer.reply.ok === true ? answer.reply.data?.host : undefined;
|
|
485
|
+
if (answer.responder.instanceId === slot.instanceId && typeof host === "string" && HOST_NAME.test(host))
|
|
486
|
+
slot.reportedHost = host;
|
|
487
|
+
}
|
|
488
|
+
catch { /* display metadata only: the instance is listed without a host */ }
|
|
489
|
+
}));
|
|
490
|
+
}
|
|
491
|
+
finally {
|
|
492
|
+
clearTimeout(timer);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
const VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
|
|
496
|
+
/** The version manager instance `instanceId` gives in its own `status`, asked on the route the
|
|
497
|
+
* refused call rode: its `inst` rail when that call was pinned, else the class rail with only that
|
|
498
|
+
* instance's answer accepted. Only a well-formed version from that exact instance is kept; one that
|
|
499
|
+
* is not, or no answer inside the bound, leaves it unknown. */
|
|
500
|
+
async function reportedManagerVersion(nc, space, caller, instanceId, pinned) {
|
|
501
|
+
const signal = AbortSignal.timeout(REPORTED_HOST_BUDGET_MS);
|
|
502
|
+
try {
|
|
503
|
+
const service = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, { deadlineMs: REPORTED_HOST_BUDGET_MS, signal, commands: ["status"], ...(pinned ? { instanceId } : { classInstanceId: instanceId }) });
|
|
504
|
+
const answer = await invokeCommand(nc, space, service, "status", undefined, { deadlineMs: REPORTED_HOST_BUDGET_MS, signal });
|
|
505
|
+
const version = answer.reply.ok === true ? answer.reply.data?.version : undefined;
|
|
506
|
+
return answer.responder.instanceId === instanceId && typeof version === "string" && VERSION.test(version) ? version : undefined;
|
|
507
|
+
}
|
|
508
|
+
catch {
|
|
509
|
+
return undefined;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
/**
|
|
513
|
+
* Whether the scatter's liveness probe rides a BARE connection, a freshly minted pinned instrument,
|
|
514
|
+
* or nothing at all. The sixth seed reach on the attach path, made explicit.
|
|
515
|
+
*
|
|
516
|
+
* Named and exported so it is GRADABLE, because it is the one site on this path that still answers
|
|
517
|
+
* "is this an open mesh?" from the absence of a credential. That inference is issue #1205 itself,
|
|
518
|
+
* and it is only tolerable here because of what this answer is allowed to DO: it selects whether a
|
|
519
|
+
* frozen instance's liveness is probed, so a wrong answer downgrades a row to `not-probed` and
|
|
520
|
+
* never refuses an attach. A caller that ever turns this into a refusal must change this function,
|
|
521
|
+
* which is the point of it having a name.
|
|
522
|
+
*
|
|
523
|
+
* bare — no credential and no bearer: an open mesh, whose connection can already publish.
|
|
524
|
+
* mint — a sealed mesh that holds its seed AND arrived on a credential: re-mint pinned.
|
|
525
|
+
* as-is — everything else (a user bearer, a raw off-registry cred, a sealed mesh with no seed):
|
|
526
|
+
* keep the credential it arrived with and probe nothing, which is the pre-existing
|
|
527
|
+
* behaviour and prints `not-probed` rather than degrading silently.
|
|
528
|
+
*/
|
|
529
|
+
export function scatterProbeMaterial(auth, spaceAuth) {
|
|
530
|
+
if (!auth.creds && !auth.bearer)
|
|
531
|
+
return { kind: "bare" };
|
|
532
|
+
if (spaceAuth && auth.creds)
|
|
533
|
+
return { kind: "mint", auth: spaceAuth };
|
|
534
|
+
return { kind: "as-is" };
|
|
535
|
+
}
|
|
536
|
+
/** Commands whose closures one scatter run invokes: the scattered verb, plus status for host labels. */
|
|
537
|
+
export function scatterResolveCommands(command, reportHosts) {
|
|
538
|
+
return [...new Set([command, ...(reportHosts ? ["status"] : [])])];
|
|
539
|
+
}
|
|
540
|
+
/** The ep-rail CLASS SCATTER (P2 item 3, `cotal ps` default).
|
|
541
|
+
*
|
|
542
|
+
* TWO connections, because a connection's permissions are fixed at authentication and this command
|
|
543
|
+
* cannot know which instance rails it needs until it has read the registry:
|
|
544
|
+
*
|
|
545
|
+
* 0. FREEZE the live class from the records registry (the same §13.9 read the scatter itself
|
|
546
|
+
* does), then close. This yields the exact instance ids this invocation will address.
|
|
547
|
+
* 1. RE-MINT the one-shot instrument LOCALLY, pinned to exactly those ids, and run the
|
|
548
|
+
* unpinned {@link resolveService} + {@link scatterCommand} on it — now the §13.5 liveness probe
|
|
549
|
+
* can publish on each frozen instance's own rail and the gather can end as soon as every slot
|
|
550
|
+
* has either answered or been affirmed gone by the broker.
|
|
551
|
+
*
|
|
552
|
+
* THE COST IS THE MINT, NOT A ROUND TRIP. `mintCreds` is a local JWT signature against the space
|
|
553
|
+
* seed already in hand (measured at 22-45ms here); the extra connection is the only wire cost. The
|
|
554
|
+
* earlier attempt at this measured 3.2s because it re-ran the whole target resolve (registry read,
|
|
555
|
+
* preflight, stale-prune) to get a pinned credential, and shipped a regression to fix a hypothesis.
|
|
556
|
+
* This re-mints and nothing else.
|
|
557
|
+
*
|
|
558
|
+
* NO WILDCARD INSTANCE ROW is minted, which is the invariant the whole two-connection shape exists
|
|
559
|
+
* to preserve: the pin is a list of exact ids known before the mint, the same precondition `--on`
|
|
560
|
+
* satisfies with one id.
|
|
561
|
+
*
|
|
562
|
+
* WHERE THERE IS NO SEED THERE IS NO PROBE, and the deadline stands. A user-mode bearer and a raw
|
|
563
|
+
* off-registry cred cannot re-mint, so their frozen slots are `not-probed` and say so. This is not
|
|
564
|
+
* a silent degrade: it is the pre-existing behaviour, and the row prints which of the two it is.
|
|
565
|
+
*/
|
|
566
|
+
async function askManagerScatterEp(client, space, server, op, auth, spaceAuth, timeoutMs, reportHosts = false) {
|
|
567
|
+
const mapped = EP_COMMANDS[op];
|
|
568
|
+
if (!mapped)
|
|
569
|
+
return { ok: false, error: `unknown manager op "${op}" (no v0.4 command mapping)` };
|
|
570
|
+
if (mapped.targeted)
|
|
571
|
+
return { ok: false, error: `${op} is targeted and cannot be scattered across instances` };
|
|
572
|
+
// ---- connection 0: freeze the class. A registry read that fails here fails the command with the
|
|
573
|
+
// registry's own verdict (never the managers'), which `epRailFailure` already words.
|
|
574
|
+
let pinnedIds;
|
|
575
|
+
try {
|
|
576
|
+
pinnedIds = await withControlConnection(server, auth, async (nc) => {
|
|
577
|
+
// `checkAPI: false` for the same reason `scatterCommand` uses it: the freeze rides the scoped
|
|
578
|
+
// §13.9 records rows, never an account-level JetStream probe.
|
|
579
|
+
const jsm = await jetstreamManager(nc, { checkAPI: false });
|
|
580
|
+
return (await freezeExpectedSet(jsm, space, BASELINE_LIFECYCLE_ENDPOINT)).map((f) => f.instanceId);
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
catch (e) {
|
|
584
|
+
const { error, code, unanswered } = epRailFailure(client, e);
|
|
585
|
+
return { ok: false, error: error ?? "error", ...(code === undefined ? {} : { code }), ...(unanswered ? { unanswered } : {}), ...(noLiveInstances(e) ? { noLiveManager: true } : {}) };
|
|
586
|
+
}
|
|
587
|
+
// ---- the LOCAL pinned re-mint, decided by {@link scatterProbeMaterial} rather than inline.
|
|
588
|
+
// This is the SIXTH place the attach path reaches for a seed (attach → pinForTarget → locateSeat
|
|
589
|
+
// → scatterManager → here), and it is named as such because its answer is still derived from the
|
|
590
|
+
// ABSENCE of a credential, which is the inference issue #1205 was about. It is safe here and it
|
|
591
|
+
// is NOT safe by accident: this site only decides whether to PROBE LIVENESS, so its worst outcome
|
|
592
|
+
// is a `not-probed` row, never a refusal to attach. It is extracted anyway so a future caller
|
|
593
|
+
// that starts making a REFUSAL out of this answer has to pass through a named decision that says
|
|
594
|
+
// so, instead of re-deriving openness from a nullable field for the second time in one path.
|
|
595
|
+
const material = scatterProbeMaterial(auth, spaceAuth);
|
|
596
|
+
let probeAuth = auth;
|
|
597
|
+
let caller = auth.epCaller;
|
|
598
|
+
let probed = material.kind === "bare" ? new Set(pinnedIds) : new Set();
|
|
599
|
+
if (material.kind === "mint") {
|
|
600
|
+
const identity = newIdentity();
|
|
601
|
+
const uid = mintLifecycleUid();
|
|
602
|
+
probeAuth = {
|
|
603
|
+
...auth,
|
|
604
|
+
creds: await mintCreds(material.auth, identity, SCATTER_PROFILE, {
|
|
605
|
+
lifecycleUid: uid,
|
|
606
|
+
endpointCapabilities: instancePinnedInstrumentCapabilities("privileged", pinnedIds),
|
|
607
|
+
}),
|
|
608
|
+
};
|
|
609
|
+
caller = { owner: DEV_OWNER, actor: identity.id, uid };
|
|
610
|
+
probed = new Set(pinnedIds);
|
|
611
|
+
}
|
|
612
|
+
// ---- connection 1: resolve + scatter, with the probe closure this credential can actually back.
|
|
613
|
+
try {
|
|
614
|
+
return await withControlConnection(server, probeAuth, async (nc) => {
|
|
615
|
+
const { probeLiveness, livenessOf } = pinnedLivenessProbe(nc, { space, endpoint: BASELINE_LIFECYCLE_ENDPOINT, caller, pinned: probed });
|
|
616
|
+
const commands = scatterResolveCommands(mapped.command, reportHosts);
|
|
617
|
+
const service = await resolveService(nc, space, BASELINE_LIFECYCLE_ENDPOINT, caller, { deadlineMs: 10_000, commands });
|
|
618
|
+
const result = await scatterCommand(nc, space, service, mapped.command, undefined, {
|
|
619
|
+
deadlineMs: timeoutMs ?? 8_000,
|
|
620
|
+
reconcileDeadlineMs: 3_000,
|
|
621
|
+
probeLiveness,
|
|
622
|
+
});
|
|
623
|
+
const instances = [];
|
|
624
|
+
for (const [instanceId, ar] of result.replies) {
|
|
625
|
+
if (ar.reply.ok === true)
|
|
626
|
+
instances.push({ instanceId, reachable: true, data: ar.reply.data });
|
|
627
|
+
else
|
|
628
|
+
instances.push({ instanceId, reachable: true, error: ar.reply.error?.message ?? ar.reply.error?.code ?? "error", ...(ar.reply.error?.code ? { code: ar.reply.error.code } : {}) });
|
|
629
|
+
}
|
|
630
|
+
// A frozen instance that never answered is UNREACHABLE — surfaced, never silently dropped
|
|
631
|
+
// (pin 3) — and now carries WHY it is silent, as far as this command could establish it.
|
|
632
|
+
for (const instanceId of result.missing)
|
|
633
|
+
instances.push({ instanceId, reachable: false, liveness: livenessOf(instanceId) });
|
|
634
|
+
if (reportHosts)
|
|
635
|
+
await askReportedHosts(nc, space, service, result.replies, instances, probed);
|
|
636
|
+
return { ok: true, instances };
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
catch (e) {
|
|
640
|
+
const { error, code, unanswered } = epRailFailure(client, e);
|
|
641
|
+
return { ok: false, error: error ?? "error", ...(code === undefined ? {} : { code }), ...(unanswered ? { unanswered } : {}) };
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
/** SCATTER one untargeted read (`ps`) across EVERY registered manager instance in the space and merge
|
|
645
|
+
* the attributed results — the `cotal ps` default in a multi-manager space (P2 item 3). Auth shapes
|
|
646
|
+
* match {@link askManager}: a minted instrument or user bearer rides its own caller triple; a raw
|
|
647
|
+
* pre-1c `--creds` file is refused loud; an OPEN mesh synthesizes a DEV_OWNER triple and connects
|
|
648
|
+
* bare (the broker enforces nothing, so the records freeze reads freely). `spaceAuth` is the
|
|
649
|
+
* resolved mesh's trust material from {@link resolveControlTarget}: present ⇒ the scatter re-mints
|
|
650
|
+
* its instrument pinned to the frozen instance ids and can probe their liveness; absent ⇒ it runs
|
|
651
|
+
* exactly as it did before and every silent slot is reported `not-probed`. */
|
|
652
|
+
export async function scatterManager(client, space, server, op, auth = {}, spaceAuth, timeoutMs, opts = {}) {
|
|
653
|
+
const who = controlCaller(auth);
|
|
654
|
+
if ("refusal" in who)
|
|
655
|
+
return { ok: false, error: who.refusal };
|
|
656
|
+
return askManagerScatterEp(client, space, server, op, { ...auth, epCaller: who.caller }, spaceAuth, timeoutMs, opts.reportHosts === true);
|
|
657
|
+
}
|
|
658
|
+
/** The refusal `ps` gives, raised before an unpinned operator read when the registry lists no live
|
|
659
|
+
* manager, rather than after the read's describe has waited out its whole deadline. Only the
|
|
660
|
+
* operator instrument holds the registry rows, so a caller passes a user's manager view or a pinned
|
|
661
|
+
* call straight to the read. Any other registry outcome, an unreadable registry included, leaves
|
|
662
|
+
* the read to decide. `purpose` completes "no manager is running for this space to …". Returns the
|
|
663
|
+
* live instance ids the registry listed, or undefined when it could not be read. */
|
|
664
|
+
export async function refuseWithoutLiveManager(space, server, auth, target, purpose) {
|
|
665
|
+
try {
|
|
666
|
+
return await liveManagerIds(space, server, auth);
|
|
667
|
+
}
|
|
668
|
+
catch (e) {
|
|
669
|
+
if (!(e instanceof EpEnvelopeError) || !noLiveInstances(e))
|
|
670
|
+
return undefined;
|
|
671
|
+
throw new CommandFailure("refused", "TARGET_UNAVAILABLE", `no manager is running for this space to ${purpose}`, { target, native: { code: e.code, detail: `${e.code}: ${e.message}` }, remedy: "start the stack's manager with bare `cotal up`; a stack started with --no-manager has none" });
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
/** The host one manager instance reports in its own `status`, the name `cotal ps` prints as its
|
|
675
|
+
* reported host, or undefined when it does not answer or names no well-formed host. Presence
|
|
676
|
+
* carries no instance id off a user-auth mesh, so this read is where a pinned manager's host comes
|
|
677
|
+
* from there. The view is the caller's own and is not kept.
|
|
678
|
+
*
|
|
679
|
+
* It takes the ordinary control deadline, NOT {@link REPORTED_HOST_BUDGET_MS}. That budget belongs
|
|
680
|
+
* to the census above, where a host name is display metadata and a slow instance is listed without
|
|
681
|
+
* one. Here the answer decides whether a launch stops to ask the operator a question, so a read
|
|
682
|
+
* that gives up early does not lose a column, it asks about a manager that is on this host.
|
|
683
|
+
* Measured: under heavy load the 2s budget expired and the question came back. */
|
|
684
|
+
export async function reportedManagerHost(client, space, server, auth, instanceId) {
|
|
685
|
+
let reply;
|
|
686
|
+
try {
|
|
687
|
+
reply = await askManager(client, space, server, "managerStatus", undefined, auth, "owner", undefined, { instanceId });
|
|
688
|
+
}
|
|
689
|
+
catch {
|
|
690
|
+
return undefined;
|
|
691
|
+
}
|
|
692
|
+
const host = reply.ok ? reply.data?.host : undefined;
|
|
693
|
+
return typeof host === "string" && HOST_NAME.test(host) ? host : undefined;
|
|
694
|
+
}
|
|
695
|
+
/** A pinned instance the space's registry does not list has nothing behind it to answer, so the
|
|
696
|
+
* command is refused before it waits out the reply deadline, naming the managers that are running.
|
|
697
|
+
* A registry this side cannot read leaves the request to settle it. */
|
|
698
|
+
export async function refuseUnregisteredManager(space, server, auth, on) {
|
|
699
|
+
let running;
|
|
700
|
+
try {
|
|
701
|
+
running = await liveManagerIds(space, server, auth);
|
|
702
|
+
}
|
|
703
|
+
catch (e) {
|
|
704
|
+
if (!(e instanceof EpEnvelopeError) || !noLiveInstances(e))
|
|
705
|
+
return;
|
|
706
|
+
running = [];
|
|
707
|
+
}
|
|
708
|
+
if (running.includes(on))
|
|
709
|
+
return;
|
|
710
|
+
const target = { space, requestedManagerInstance: on };
|
|
711
|
+
// With none running this is the unpinned refusal, which names the same way out.
|
|
712
|
+
if (running.length === 0)
|
|
713
|
+
throw new CommandFailure("refused", "TARGET_UNAVAILABLE", `no manager is running for this space, so --on ${on} names none. Nothing was sent`, { target, remedy: "start the stack's manager with bare `cotal up`; a stack started with --no-manager has none" });
|
|
714
|
+
const which = running.length === 1 ? `the one running is ${running[0]}` : `the ones running are ${running.join(", ")}`;
|
|
715
|
+
throw new CommandFailure("refused", "TARGET_NOT_FOUND", `--on ${on} names no manager running for space ${space}; ${which}. Nothing was sent`, {
|
|
716
|
+
target,
|
|
717
|
+
remedy: running.length === 1 ? `run it again with --on ${running[0]}, or without --on` : "run it again with --on naming one of them",
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
/** Whether the broker affirms that nothing is subscribed on manager `instanceId`'s own rail, so its
|
|
721
|
+
* registration outlived its host. The probe settles on the broker's no-responders answer, so a gone
|
|
722
|
+
* manager costs no reply deadline. A live manager never answers the probe, which then runs its whole
|
|
723
|
+
* budget, so a caller that also asks the manager itself passes `signal` and fires it once that
|
|
724
|
+
* answer arrives. Any other outcome, an aborted probe or one this credential cannot send included,
|
|
725
|
+
* is `false`. */
|
|
726
|
+
export async function managerRailGone(space, server, auth, instanceId, signal) {
|
|
727
|
+
const who = controlCaller(auth);
|
|
728
|
+
if ("refusal" in who || signal?.aborted)
|
|
729
|
+
return false;
|
|
730
|
+
const aborted = new Promise((resolve) => signal?.addEventListener("abort", () => resolve(false), { once: true }));
|
|
731
|
+
try {
|
|
732
|
+
return await withControlConnection(server, auth, (nc) => Promise.race([
|
|
733
|
+
epProbeInstanceInterest(nc, space, BASELINE_LIFECYCLE_ENDPOINT, instanceId, who.caller, { deadlineMs: PROBE_DEADLINE_MS }).then((verdict) => verdict === "gone"),
|
|
734
|
+
aborted,
|
|
735
|
+
]));
|
|
736
|
+
}
|
|
737
|
+
catch {
|
|
738
|
+
return false;
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
/** The manager instances the space's registry lists as live, frozen as a scatter freezes them. An
|
|
742
|
+
* empty registry throws the registry's own refusal. */
|
|
743
|
+
export async function liveManagerIds(space, server, auth) {
|
|
744
|
+
return withControlConnection(server, auth, async (nc) => (await freezeExpectedSet(await jetstreamManager(nc, { checkAPI: false }), space, BASELINE_LIFECYCLE_ENDPOINT)).map((f) => f.instanceId));
|
|
745
|
+
}
|
|
746
|
+
/** The stable code a refused manager READ reports, by the endpoint's catalog code. A read has no
|
|
747
|
+
* effect, so an outcome the catalog does not classify is a failed read, never an unknown one. */
|
|
748
|
+
const READ_REFUSALS = {
|
|
749
|
+
unauthenticated: "AUTH_REQUIRED",
|
|
750
|
+
"permission-denied": "PERMISSION_DENIED",
|
|
751
|
+
"not-found": "TARGET_NOT_FOUND",
|
|
752
|
+
unavailable: "TARGET_UNAVAILABLE",
|
|
753
|
+
"deadline-exceeded": "TARGET_UNAVAILABLE",
|
|
754
|
+
unimplemented: "UNSUPPORTED",
|
|
755
|
+
"contract-mismatch": "UNSUPPORTED",
|
|
756
|
+
conflict: "CONFLICT",
|
|
757
|
+
};
|
|
758
|
+
/** A manager read that did not answer, as the classified failure a structured command reports: the
|
|
759
|
+
* same sentence, the endpoint's own code under `error.cause`, and the stale-store hint as its remedy
|
|
760
|
+
* when one applies. A request nobody answered carries `deadline-exceeded`, an unavailable target. */
|
|
761
|
+
export function readFailure(reply, target) {
|
|
762
|
+
const message = reply.error ?? "the manager refused the read";
|
|
763
|
+
const remedy = staleStoreHint(message);
|
|
764
|
+
const native = reply.code === undefined && reply.details === undefined ? undefined : { ...(reply.code === undefined ? {} : { code: reply.code }), ...(reply.details === undefined ? {} : { detail: reply.details }) };
|
|
765
|
+
const opts = { target, ...(remedy ? { remedy } : {}), ...(native ? { native } : {}) };
|
|
766
|
+
const code = reply.code === undefined ? undefined : READ_REFUSALS[reply.code];
|
|
767
|
+
return code ? new CommandFailure("refused", code, message, opts) : new CommandFailure("failed", "OPERATION_FAILED", message, opts);
|
|
768
|
+
}
|
|
769
|
+
//# sourceMappingURL=manager.js.map
|