@jr2/cli 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/build.ts +4 -4
- package/src/caller-tokens.ts +108 -0
- package/src/cli.ts +33 -2
- package/src/client.ts +227 -23
- package/src/command-table.ts +269 -20
- package/src/commands/caller.ts +115 -0
- package/src/commands/completion.ts +236 -0
- package/src/commands/gates.ts +23 -0
- package/src/commands/hooks.ts +31 -0
- package/src/commands/init.ts +2 -3
- package/src/commands/logs.ts +107 -31
- package/src/commands/run.ts +77 -27
- package/src/commands/runs.ts +17 -78
- package/src/commands/send.ts +1 -1
- package/src/commands/status.ts +138 -7
- package/src/commands/steer.ts +47 -0
- package/src/commands/up.ts +440 -83
- package/src/commands/watch.ts +57 -0
- package/src/commands/whoami.ts +22 -0
- package/src/commands/workflows.ts +58 -0
- package/src/commands/workspace.ts +316 -0
- package/src/deploy.ts +409 -42
- package/src/instance-check.ts +57 -7
- package/src/instance.ts +16 -5
- package/src/kube.ts +37 -3
- package/src/landlock.ts +129 -0
- package/src/mount-wait.ts +82 -0
- package/src/output.ts +3 -0
- package/src/sse.ts +1 -1
- package/src/workspaces.ts +307 -0
package/src/commands/up.ts
CHANGED
|
@@ -65,6 +65,9 @@ import { homedir } from "node:os";
|
|
|
65
65
|
import { basename, join, relative } from "node:path";
|
|
66
66
|
import { fileURLToPath } from "node:url";
|
|
67
67
|
import {
|
|
68
|
+
callerTokenDigest,
|
|
69
|
+
callerTokenSecret,
|
|
70
|
+
storedValueSha256,
|
|
68
71
|
custodianBootstrap,
|
|
69
72
|
custodianRef,
|
|
70
73
|
custodianScript,
|
|
@@ -82,8 +85,6 @@ import {
|
|
|
82
85
|
doorCompat,
|
|
83
86
|
matchCredential,
|
|
84
87
|
packAssembly,
|
|
85
|
-
configMapDigest,
|
|
86
|
-
MOUNTED_CONFIGMAPS,
|
|
87
88
|
partsOf,
|
|
88
89
|
LABEL_EFFECTOR_POD,
|
|
89
90
|
LABEL_GIT_PUSH,
|
|
@@ -95,12 +96,12 @@ import {
|
|
|
95
96
|
type SandboxEffectorPlan,
|
|
96
97
|
secretPlacementsOf,
|
|
97
98
|
storedValueLength,
|
|
99
|
+
isStdioServer,
|
|
98
100
|
effectorKey,
|
|
99
101
|
targetText,
|
|
100
102
|
harnessTokenDigest,
|
|
101
103
|
formatCpu,
|
|
102
104
|
formatMemory,
|
|
103
|
-
sandboxToken,
|
|
104
105
|
type CarriedEffect,
|
|
105
106
|
type CarriedImage,
|
|
106
107
|
type CarriedRepo,
|
|
@@ -111,10 +112,13 @@ import {
|
|
|
111
112
|
PROVIDER_KEY_NAME,
|
|
112
113
|
type PlacementWalk,
|
|
113
114
|
type SandboxPlacement,
|
|
115
|
+
type StandingView,
|
|
114
116
|
workspaceHeldPod,
|
|
115
117
|
type JR2Config,
|
|
118
|
+
memoryBytes,
|
|
116
119
|
} from "@jr2/orchestrator/internal";
|
|
117
120
|
import { stageAssembly, stagedCopyOf } from "../assembly.ts";
|
|
121
|
+
import { awaitMounts, digestSecretsOf, mountedDigests, staleLine } from "../mount-wait.ts";
|
|
118
122
|
import { caFits, fingerprint, issueCa, issueLeaf, leafFits, trustBundles, type Pem } from "../held-pki.ts";
|
|
119
123
|
import {
|
|
120
124
|
assertEmulation,
|
|
@@ -142,6 +146,8 @@ import {
|
|
|
142
146
|
INSTANCE_HARNESS_PVC,
|
|
143
147
|
INSTANCE_HARNESS_SERVICE,
|
|
144
148
|
instanceHarnessObjects,
|
|
149
|
+
INSTANCE_HARNESS_CREDENTIALS,
|
|
150
|
+
podCredentials,
|
|
145
151
|
instanceEffectorObjects,
|
|
146
152
|
effectorDigest,
|
|
147
153
|
storedValueDigest,
|
|
@@ -160,6 +166,8 @@ import {
|
|
|
160
166
|
priorityClassNames,
|
|
161
167
|
priorityClassObjects,
|
|
162
168
|
REPO_CACHE,
|
|
169
|
+
STATE_PVC,
|
|
170
|
+
stateVolumePlan,
|
|
163
171
|
trustObject,
|
|
164
172
|
holdsAny,
|
|
165
173
|
LABEL_HELD_POD,
|
|
@@ -173,6 +181,7 @@ import {
|
|
|
173
181
|
CHECK_CONTAINER,
|
|
174
182
|
} from "../deploy.ts";
|
|
175
183
|
import { syncAgentsMd } from "../agents-md.ts";
|
|
184
|
+
import { planCallerTokens } from "../caller-tokens.ts";
|
|
176
185
|
import { resolveRoot } from "../instance.ts";
|
|
177
186
|
import { resolvePackage } from "../kit-version.ts";
|
|
178
187
|
import { effectorImagePaths, effectorImageRefusal, STOCK_HARNESS_IMAGE, type PushForm } from "../effector-image.ts";
|
|
@@ -189,10 +198,12 @@ import {
|
|
|
189
198
|
type PodEnd,
|
|
190
199
|
type RolloutTarget,
|
|
191
200
|
} from "../kube.ts";
|
|
201
|
+
import { awaitCacheAgent } from "../landlock.ts";
|
|
192
202
|
import { anyNodeHolds, buildNodes, sandboxNodes, type NodeObject } from "../nodes.ts";
|
|
193
203
|
import { activity, chooseOrBail, confirmOrBail, promptLine, readSecretInput, type Io } from "../output.ts";
|
|
194
204
|
import { kindCluster, sweepImages } from "../sweep.ts";
|
|
195
205
|
import { chartFilesIn, checkInstance } from "../instance-check.ts";
|
|
206
|
+
import { standingLine } from "./status.ts";
|
|
196
207
|
import { parseCommand } from "../command-table.ts";
|
|
197
208
|
|
|
198
209
|
/** How many times one converge rebuilds a host image a sweep took before it uses it (ADR-0039).
|
|
@@ -323,6 +334,69 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
323
334
|
activity(io, ` pick a different namespace (-n) or context (--context)`);
|
|
324
335
|
return 1;
|
|
325
336
|
}
|
|
337
|
+
// --- the state volume (ADR-0065): its size before anything is applied ---------------------------
|
|
338
|
+
// A claim never shrinks, and grows only where its StorageClass allows expansion: a size that the
|
|
339
|
+
// cluster would refuse is refused here, naming the class, before the namespace is touched. Both
|
|
340
|
+
// reads THROW on a failure (`listJson`, not `getJson`): a claim that reads as absent when the API
|
|
341
|
+
// could not be read would be applied over at another size. The claim is read even when the
|
|
342
|
+
// namespace read gave nothing: `getJson` reads a failure as absent, and a list in a namespace
|
|
343
|
+
// that is not there is empty, so only a real failure refuses.
|
|
344
|
+
type StateClaim = { spec?: { storageClassName?: string; resources?: { requests?: { storage?: string } } } };
|
|
345
|
+
let stateClaim: StateClaim | undefined;
|
|
346
|
+
try {
|
|
347
|
+
stateClaim = (
|
|
348
|
+
await kube.listJson<StateClaim>({
|
|
349
|
+
kind: "persistentvolumeclaim",
|
|
350
|
+
fieldSelector: `metadata.name=${STATE_PVC}`,
|
|
351
|
+
namespace,
|
|
352
|
+
...ctx,
|
|
353
|
+
})
|
|
354
|
+
)[0];
|
|
355
|
+
} catch (e) {
|
|
356
|
+
activity(
|
|
357
|
+
io,
|
|
358
|
+
`refusing: cannot read the state volume claim ${STATE_PVC} in namespace "${namespace}" (${(e as Error).message}). Nothing was applied.`,
|
|
359
|
+
);
|
|
360
|
+
return 1;
|
|
361
|
+
}
|
|
362
|
+
const claimSize = stateClaim?.spec?.resources?.requests?.storage;
|
|
363
|
+
const claimClass = stateClaim?.spec?.storageClassName;
|
|
364
|
+
const wantVolume = config.runs?.volume;
|
|
365
|
+
const grows = claimSize !== undefined && wantVolume !== undefined && memoryBytes(wantVolume) > memoryBytes(claimSize);
|
|
366
|
+
let claimClassObject: { allowVolumeExpansion?: boolean } | undefined;
|
|
367
|
+
if (grows && claimClass) {
|
|
368
|
+
try {
|
|
369
|
+
claimClassObject = (
|
|
370
|
+
await kube.listJson<{ allowVolumeExpansion?: boolean }>({
|
|
371
|
+
kind: "storageclass",
|
|
372
|
+
fieldSelector: `metadata.name=${claimClass}`,
|
|
373
|
+
...ctx,
|
|
374
|
+
})
|
|
375
|
+
)[0];
|
|
376
|
+
} catch (e) {
|
|
377
|
+
activity(
|
|
378
|
+
io,
|
|
379
|
+
`refusing: runs.volume is ${wantVolume}, larger than the state volume claim ${STATE_PVC} (${claimSize}), and jr2 cannot read StorageClass "${claimClass}" to see if it allows expansion (${(e as Error).message}). Nothing was applied.`,
|
|
380
|
+
);
|
|
381
|
+
return 1;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
const volumePlan = stateVolumePlan(
|
|
385
|
+
wantVolume,
|
|
386
|
+
claimSize === undefined
|
|
387
|
+
? undefined
|
|
388
|
+
: {
|
|
389
|
+
size: claimSize,
|
|
390
|
+
...(claimClass ? { storageClass: claimClass } : {}),
|
|
391
|
+
...(grows ? { expandable: claimClassObject?.allowVolumeExpansion === true } : {}),
|
|
392
|
+
},
|
|
393
|
+
);
|
|
394
|
+
if ("refused" in volumePlan) {
|
|
395
|
+
activity(io, `refusing: ${volumePlan.refused}. Nothing was applied.`);
|
|
396
|
+
return 1;
|
|
397
|
+
}
|
|
398
|
+
if (volumePlan.line) activity(io, volumePlan.line);
|
|
399
|
+
|
|
326
400
|
if (!ns || !owner) {
|
|
327
401
|
const what = ns ? `adopt existing namespace "${namespace}"` : `create namespace "${namespace}"`;
|
|
328
402
|
const ok =
|
|
@@ -714,12 +788,14 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
714
788
|
JR2_INSTANCE_TOKEN: instanceToken,
|
|
715
789
|
JR2_PUBLISH_TOKEN: publishToken,
|
|
716
790
|
JR2_SIGNING_KEY: signingKey,
|
|
717
|
-
// The Instance Harness Custodian's credential (ADR-0013/0031): a sandbox-style token signed
|
|
718
|
-
// for the placement's name — it may deliver only to Turns hosted THERE (tokens.ts), so the
|
|
719
|
-
// Instance token never enters that pod. Derived from the kept key, so re-runs converge to
|
|
720
|
-
// the same value; live Custodians keep verifying.
|
|
721
|
-
JR2_INSTANCE_HARNESS_TOKEN: sandboxToken(Buffer.from(signingKey, "base64"), INSTANCE_HARNESS_SERVICE),
|
|
722
791
|
};
|
|
792
|
+
// The named Caller tokens (ADR-0078): each declared name keeps the value its Secret holds, and a
|
|
793
|
+
// name with none gets one now. The Orchestrator gets only their digests. Read here and written
|
|
794
|
+
// with the instance objects below, so a converge that stops before the apply writes no value.
|
|
795
|
+
const callerTokens = await planCallerTokens(kube, Object.keys(config.callers?.tokens ?? {}), namespace, ctx);
|
|
796
|
+
// The Instance Harness Custodian's credentials are not here (ADR-0076): its control token and
|
|
797
|
+
// its secret Stand-ins ride its own Secret (`INSTANCE_HARNESS_CREDENTIALS`), whose Stand-ins its
|
|
798
|
+
// Harness container mounts — so it must never be this one.
|
|
723
799
|
// Git tokens (ADR-0051, ADR-0068): each entry's token, from `.env`, goes into that entry's own clone
|
|
724
800
|
// Secret (`jr2-git-<hash>`, Flux's key names), which the cache agent reads — never into this
|
|
725
801
|
// Secret, the Orchestrator's env. The Orchestrator names the Secret on a Repo and holds no git token.
|
|
@@ -770,6 +846,9 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
770
846
|
// Instance Effector — so a value edited in the user's Secret rolls them at the next `jr2 up`.
|
|
771
847
|
const referenced = new Map<string, Map<string, number> | undefined>();
|
|
772
848
|
const valueDigests = new Map<string, string>();
|
|
849
|
+
// The plain sha-256 of each stored value, for a `token` Hook's secret alone (ADR-0078): the door
|
|
850
|
+
// compares a delivery's token with it, so it cannot be keyed as `valueDigests` are.
|
|
851
|
+
const valueShas = new Map<string, string>();
|
|
773
852
|
for (const entry of config.secrets ?? []) {
|
|
774
853
|
const ref = entry.valueFrom?.secretKeyRef.name;
|
|
775
854
|
if (ref === undefined || referenced.has(ref)) continue;
|
|
@@ -785,6 +864,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
785
864
|
);
|
|
786
865
|
for (const [k, b64] of Object.entries(found?.data ?? {})) {
|
|
787
866
|
valueDigests.set(`${ref}/${k}`, storedValueDigest(Buffer.from(signingKey, "base64"), ref, k, b64));
|
|
867
|
+
valueShas.set(`${ref}/${k}`, storedValueSha256(b64));
|
|
788
868
|
}
|
|
789
869
|
}
|
|
790
870
|
const valueDigest = (secret: string, key: string) => valueDigests.get(`${secret}/${key}`);
|
|
@@ -800,6 +880,13 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
800
880
|
const placements = secretPlacementsOf(config, placementWalk, { secretKeys, valueLength });
|
|
801
881
|
for (const line of placements.report) activity(io, line);
|
|
802
882
|
for (const warning of placements.warnings) activity(io, `warning: ${warning}`);
|
|
883
|
+
// The `token` Hooks' digests (ADR-0078): the sha-256 of each value, which the Orchestrator mounts
|
|
884
|
+
// in `jr2-hook-digests`. The value itself never leaves this host or the user's Secret.
|
|
885
|
+
const hookDigests: Record<string, string> = {};
|
|
886
|
+
for (const t of placements.hookTokens) {
|
|
887
|
+
const digest = "value" in t ? callerTokenDigest(t.value) : valueShas.get(`${t.secret}/${t.key}`);
|
|
888
|
+
if (digest !== undefined) hookDigests[t.name] = digest;
|
|
889
|
+
}
|
|
803
890
|
const held = await resolveHeld(io, kube, {
|
|
804
891
|
pods: placements.held,
|
|
805
892
|
claimed: placements.claimed,
|
|
@@ -828,6 +915,17 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
828
915
|
`provider: ${custom.baseUrl} configured, but no carried Agent names a "${custom.id}/…" model — skipped`,
|
|
829
916
|
);
|
|
830
917
|
}
|
|
918
|
+
// Each Agent's allowlist of each remote MCP server it names (ADR-0079): the check pods list the
|
|
919
|
+
// servers' tools, and each list is held to these.
|
|
920
|
+
const mcpUses: McpUse[] = workflows.flatMap((w) =>
|
|
921
|
+
partsOf([w.machine]).mcpAgents.flatMap((a) =>
|
|
922
|
+
a.mcp.flatMap((e) => {
|
|
923
|
+
const server = config.mcp?.[e.server];
|
|
924
|
+
if (server === undefined || isStdioServer(server)) return [];
|
|
925
|
+
return [{ workflow: w.name, slot: a.slot, server: e.server, tools: e.tools ?? "*" }];
|
|
926
|
+
}),
|
|
927
|
+
),
|
|
928
|
+
);
|
|
831
929
|
await preflightProviders(io, kube, {
|
|
832
930
|
checkPods,
|
|
833
931
|
skip: values["skip-provider-preflight"] === true,
|
|
@@ -843,6 +941,9 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
843
941
|
trust: trustData(caPem, held.bundles),
|
|
844
942
|
harnessEnvData,
|
|
845
943
|
priorityClassName: priorityClassNames(config.priorityClasses).sandbox,
|
|
944
|
+
signingKey: Buffer.from(signingKey, "base64"),
|
|
945
|
+
...(config.mcp ? { mcp: config.mcp } : {}),
|
|
946
|
+
mcpUses,
|
|
846
947
|
});
|
|
847
948
|
|
|
848
949
|
// --- operator (per-cluster, shared): after the provider preflight ------------------------------
|
|
@@ -929,10 +1030,14 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
929
1030
|
const instanceList = instanceObjects({
|
|
930
1031
|
name,
|
|
931
1032
|
namespace,
|
|
1033
|
+
stateVolume: volumePlan.size,
|
|
932
1034
|
image: refs.orchestrator,
|
|
933
1035
|
secretData,
|
|
1036
|
+
callerTokens: callerTokens.values,
|
|
1037
|
+
hookDigests,
|
|
934
1038
|
harnessEnvData,
|
|
935
1039
|
harness: config.harness,
|
|
1040
|
+
...(config.mcp ? { mcp: config.mcp } : {}),
|
|
936
1041
|
held: held.objects,
|
|
937
1042
|
imageRefs: converged,
|
|
938
1043
|
repoCache: carried.composesSandbox ? { image: operatorImage!, placement: config.sandbox } : undefined,
|
|
@@ -943,6 +1048,20 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
943
1048
|
});
|
|
944
1049
|
activity(io, `orchestrator: applying (image ${refs.orchestrator})`);
|
|
945
1050
|
await kube.apply({ manifest: instanceList, warn: (line) => activity(io, line), ...ctx });
|
|
1051
|
+
// After the apply, so the line names a value that exists. It works only when the Orchestrator has
|
|
1052
|
+
// loaded its digest, which the publish below waits for (ADR-0078); `converged` says so.
|
|
1053
|
+
for (const n of callerTokens.made) {
|
|
1054
|
+
activity(
|
|
1055
|
+
io,
|
|
1056
|
+
`caller token "${n}": made — \`jr2 caller token ${n}\` prints it, and it works when \`jr2 up\` says converged`,
|
|
1057
|
+
);
|
|
1058
|
+
}
|
|
1059
|
+
// A name that the config no longer declares loses its Secret, so its value stops working
|
|
1060
|
+
// (ADR-0078). Its digest went with the digests Secret above.
|
|
1061
|
+
for (const n of callerTokens.undeclared) {
|
|
1062
|
+
await kube.deleteObject({ kind: "secret", name: callerTokenSecret(n), namespace, ...ctx });
|
|
1063
|
+
activity(io, `caller token "${n}": deleted — callers.tokens no longer declares it`);
|
|
1064
|
+
}
|
|
946
1065
|
// An entry whose clone Secret this converge did not write loses it: a dropped entry, a token
|
|
947
1066
|
// unset, or nothing left that clones.
|
|
948
1067
|
const cloneSecrets = new Set(gitClone.map((s) => s.name));
|
|
@@ -1015,14 +1134,17 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
1015
1134
|
// over from a Machine that dropped its `workspace()` is deleted with its RBAC: the layer
|
|
1016
1135
|
// converges toward the Machines like the Instance Harness does.
|
|
1017
1136
|
if (carried.composesSandbox) {
|
|
1137
|
+
// The agent is also the Landlock detector (ADR-0077): on a Sandbox node without Landlock it
|
|
1138
|
+
// exits at start with the reason, and the converge refuses, naming each such node.
|
|
1018
1139
|
activity(io, "repo cache: waiting for the cache agent on every Sandbox node");
|
|
1019
|
-
|
|
1140
|
+
const cacheTarget: RolloutTarget = {
|
|
1020
1141
|
kind: "daemonset",
|
|
1021
1142
|
name: REPO_CACHE,
|
|
1022
1143
|
namespace,
|
|
1023
1144
|
selector: `app=${REPO_CACHE}`,
|
|
1024
1145
|
...ctx,
|
|
1025
|
-
}
|
|
1146
|
+
};
|
|
1147
|
+
await awaitCacheAgent(kube, cacheTarget, () => awaitRollout(kube, cacheTarget));
|
|
1026
1148
|
await verifyRunningImage(io, kube, {
|
|
1027
1149
|
layer: "repo cache",
|
|
1028
1150
|
namespace,
|
|
@@ -1078,7 +1200,15 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
1078
1200
|
held: instance.manifest,
|
|
1079
1201
|
custodian: instance.custodian,
|
|
1080
1202
|
heldDigest: heldDigest(Buffer.from(signingKey, "base64"), held.objects, instance, valueDigest),
|
|
1203
|
+
// Its control token and secret Stand-ins (ADR-0076, ADR-0059): minted here, as the pod's
|
|
1204
|
+
// maker, from the kept key — so a re-run converges to the same values.
|
|
1205
|
+
credentials: podCredentials(Buffer.from(signingKey, "base64"), {
|
|
1206
|
+
placement: INSTANCE_HARNESS_SERVICE,
|
|
1207
|
+
heldPod: INSTANCE_HARNESS_HELD_POD,
|
|
1208
|
+
manifest: instance.manifest,
|
|
1209
|
+
}),
|
|
1081
1210
|
harness: config.harness,
|
|
1211
|
+
...(config.mcp ? { mcp: config.mcp } : {}),
|
|
1082
1212
|
caBundle: caPem !== undefined,
|
|
1083
1213
|
// The wire's gate (ADR-0058): the digest of the bearer the Orchestrator derives for this
|
|
1084
1214
|
// placement from the kept key — the same shape it stamps onto every Sandbox at provision.
|
|
@@ -1112,21 +1242,25 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
1112
1242
|
await kube.deleteObject({ kind: "persistentvolumeclaim", name: INSTANCE_HARNESS_PVC, namespace, ...ctx });
|
|
1113
1243
|
await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_SERVICE, namespace, ...ctx });
|
|
1114
1244
|
await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_PODS_SERVICE, namespace, ...ctx });
|
|
1115
|
-
// Its Custodian's files go with the pod they were
|
|
1245
|
+
// Its Custodian's files and its credentials go with the pod they were made for (ADR-0059,
|
|
1246
|
+
// ADR-0076).
|
|
1116
1247
|
await kube.deleteObject({ kind: "configmap", name: INSTANCE_CUSTODIAN_CONFIGMAP, namespace, ...ctx });
|
|
1248
|
+
await kube.deleteObject({ kind: "secret", name: INSTANCE_HARNESS_CREDENTIALS, namespace, ...ctx });
|
|
1117
1249
|
}
|
|
1118
1250
|
|
|
1119
|
-
// --- Instance Effector (ADR-0068): converged when an Effect is placed there
|
|
1120
|
-
//
|
|
1121
|
-
//
|
|
1122
|
-
//
|
|
1251
|
+
// --- Instance Effector (ADR-0068): converged when an Effect is placed there or a Hook needs it,
|
|
1252
|
+
// absent otherwise ---
|
|
1253
|
+
// The placement is read off the Machines and the Hooks: an `http` Effect runs there, a `mac`
|
|
1254
|
+
// Hook's check and a Hook kind's follow-up too. Its objects carry only the secrets those name,
|
|
1255
|
+
// and the placement's key, derived from the kept signing key — the key the Orchestrator derives
|
|
1256
|
+
// again to sign every call and verify every answer.
|
|
1123
1257
|
const effectorPlan = placements.instance;
|
|
1124
1258
|
if (effectorPlan) {
|
|
1125
1259
|
const names = effectorPlan.secrets.map((s) => `\`${s.name}\``).join(", ") || "none";
|
|
1260
|
+
const serves = [...effectorPlan.effects, ...effectorPlan.hooks.map((h) => `hook ${h}`)].join(", ");
|
|
1126
1261
|
activity(
|
|
1127
1262
|
io,
|
|
1128
|
-
`instance effector: converging (${
|
|
1129
|
-
`${effectorGraceSeconds(effectorPlan)}s)`,
|
|
1263
|
+
`instance effector: converging (${serves}; secrets ${names}; grace ${effectorGraceSeconds(effectorPlan)}s)`,
|
|
1130
1264
|
);
|
|
1131
1265
|
const signing = Buffer.from(signingKey, "base64");
|
|
1132
1266
|
const key = effectorKey(signing, "instance");
|
|
@@ -1161,7 +1295,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
1161
1295
|
...ctx,
|
|
1162
1296
|
});
|
|
1163
1297
|
} else {
|
|
1164
|
-
activity(io, "instance effector: none (no Effect is placed there)");
|
|
1298
|
+
activity(io, "instance effector: none (no Effect is placed there, and no Hook needs it)");
|
|
1165
1299
|
for (const kind of ["deployment", "service", "networkpolicy", "secret"]) {
|
|
1166
1300
|
await kube.deleteObject({ kind, name: INSTANCE_EFFECTOR, namespace, ...ctx });
|
|
1167
1301
|
}
|
|
@@ -1178,7 +1312,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
1178
1312
|
assembly,
|
|
1179
1313
|
token: publishToken,
|
|
1180
1314
|
force: values.force === true,
|
|
1181
|
-
mounts: mountedDigests(instanceList),
|
|
1315
|
+
mounts: mountedDigests((JSON.parse(instanceList) as { items: KubeObject[] }).items),
|
|
1182
1316
|
});
|
|
1183
1317
|
if (!published) return 1;
|
|
1184
1318
|
|
|
@@ -1232,21 +1366,12 @@ function mapRefs(map: Partial<ConvergedImages>): string[] {
|
|
|
1232
1366
|
}
|
|
1233
1367
|
|
|
1234
1368
|
/** What `GET /assembly` answers (ADR-0069), read loosely: this is the CLI's view of the door.
|
|
1235
|
-
* `mounts` is the digest of each
|
|
1236
|
-
type AssemblyView = {
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
const items = (JSON.parse(list) as { items: Array<KubeObject & { data?: Record<string, string> }> }).items;
|
|
1242
|
-
const out: Record<string, string> = {};
|
|
1243
|
-
for (const item of items) {
|
|
1244
|
-
if (item.kind === "ConfigMap" && item.metadata.name in MOUNTED_CONFIGMAPS) {
|
|
1245
|
-
out[item.metadata.name] = configMapDigest(item.data ?? {});
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1248
|
-
return out;
|
|
1249
|
-
}
|
|
1369
|
+
* `mounts` is the digest of each object the process reads through a mount (`MountsView`). */
|
|
1370
|
+
type AssemblyView = {
|
|
1371
|
+
current?: { hash?: string; kitVersion?: string } | null;
|
|
1372
|
+
mounts?: Record<string, string>;
|
|
1373
|
+
standing?: StandingView[];
|
|
1374
|
+
};
|
|
1250
1375
|
|
|
1251
1376
|
/**
|
|
1252
1377
|
* The key of each `file:` Sandbox Image context (ADR-0037, ADR-0049, ADR-0069): the digest of its
|
|
@@ -1280,23 +1405,18 @@ type PublishBody = {
|
|
|
1280
1405
|
error?: string;
|
|
1281
1406
|
report?: string[];
|
|
1282
1407
|
kitVersion?: unknown;
|
|
1408
|
+
/** Each standing Workflow's state after the publish (ADR-0017). */
|
|
1409
|
+
standing?: StandingView[];
|
|
1283
1410
|
};
|
|
1284
1411
|
|
|
1285
|
-
/**
|
|
1286
|
-
* How long a publish waits for the Orchestrator to read the ConfigMaps this converge wrote
|
|
1287
|
-
* (ADR-0038, ADR-0069). `jr2-images` and `jr2-held` are mounts, and the kubelet refreshes a mount
|
|
1288
|
-
* on its own period (about a minute by default). A publish rolls no pod, so until the refresh the
|
|
1289
|
-
* door's checks and the next provision read the previous data. The publish waits until the digests
|
|
1290
|
-
* `GET /assembly` reports are the ones this converge applied. Exported so a test can shorten it.
|
|
1291
|
-
*/
|
|
1292
|
-
export const MAP_PROPAGATION = { budgetMs: 150_000, stepMs: 5_000 };
|
|
1293
|
-
|
|
1294
1412
|
/**
|
|
1295
1413
|
* Publish the staged Assembly to the running Orchestrator (ADR-0069), over a port-forward of its
|
|
1296
1414
|
* Service, as `resolveTarget` reaches it. `GET /assembly` first: when the current Assembly is this
|
|
1297
1415
|
* one, at this Kit version, nothing is sent, so a steady-state converge publishes nothing. Then
|
|
1298
1416
|
* `PUT /assembly` with the publish token from the Instance Secret — never from the environment.
|
|
1299
|
-
* Before the PUT, it waits until the Orchestrator reads the mounted ConfigMaps
|
|
1417
|
+
* Before the PUT, it waits until the Orchestrator reads the mounted ConfigMaps and the digests
|
|
1418
|
+
* Secrets this converge wrote (`awaitMounts`). When nothing is sent, it still waits for the digests
|
|
1419
|
+
* Secrets, so that `converged` means a Caller token or Hook secret that this converge wrote works.
|
|
1300
1420
|
* `--force` sends the PUT even when the hashes agree; the door answers `unchanged` and replaces
|
|
1301
1421
|
* nothing, because the hash names the bytes it already holds.
|
|
1302
1422
|
*
|
|
@@ -1311,7 +1431,7 @@ async function publishAssembly(
|
|
|
1311
1431
|
assembly: { hash: string; kitVersion: string; workflows: string[]; tar: Buffer };
|
|
1312
1432
|
token: string;
|
|
1313
1433
|
force: boolean;
|
|
1314
|
-
/** The digest of each mounted
|
|
1434
|
+
/** The digest of each mounted object this converge applied, by name (`mountedDigests`). */
|
|
1315
1435
|
mounts: Record<string, string>;
|
|
1316
1436
|
},
|
|
1317
1437
|
): Promise<boolean> {
|
|
@@ -1330,34 +1450,32 @@ async function publishAssembly(
|
|
|
1330
1450
|
if (!seen.ok) throw new Error(`the Orchestrator answered GET /assembly with HTTP ${seen.status}`);
|
|
1331
1451
|
return (await seen.json()) as AssemblyView;
|
|
1332
1452
|
};
|
|
1333
|
-
|
|
1334
|
-
const current =
|
|
1335
|
-
|
|
1453
|
+
const first = await view();
|
|
1454
|
+
const current = first.current ?? null;
|
|
1455
|
+
const skip = !force && current?.hash === assembly.hash && current.kitVersion === assembly.kitVersion;
|
|
1456
|
+
// The mounts first (ADR-0069): the door checks a `file:` digest against `jr2-images`, and a
|
|
1457
|
+
// provision after the converge reads `jr2-held`. With no publish, only the digests Secrets
|
|
1458
|
+
// (ADR-0078): a token that this converge wrote works only when the Orchestrator holds its digest.
|
|
1459
|
+
const waited = await awaitMounts(io, {
|
|
1460
|
+
seen: first,
|
|
1461
|
+
view,
|
|
1462
|
+
want: skip ? digestSecretsOf(opts.mounts) : opts.mounts,
|
|
1463
|
+
prefix: "assembly",
|
|
1464
|
+
});
|
|
1465
|
+
if (waited.stale.length) {
|
|
1466
|
+
const tokens = Object.keys(digestSecretsOf(opts.mounts)).some((n) => waited.stale.includes(n));
|
|
1467
|
+
const consequence = [
|
|
1468
|
+
...(skip ? [] : ["nothing was published, and the previous Assembly is still current"]),
|
|
1469
|
+
...(tokens ? ["a Caller token or Hook secret that this converge wrote does not work yet"] : []),
|
|
1470
|
+
];
|
|
1471
|
+
activity(io, `refusing: ${staleLine(waited.stale)}; ${consequence.join("; ")}. Run \`jr2 up\` again`);
|
|
1472
|
+
return false;
|
|
1473
|
+
}
|
|
1474
|
+
if (skip) {
|
|
1336
1475
|
activity(io, `assembly: ${assembly.hash} (current — publish skipped)`);
|
|
1476
|
+
for (const s of Array.isArray(waited.seen.standing) ? waited.seen.standing : []) activity(io, standingLine(s));
|
|
1337
1477
|
return true;
|
|
1338
1478
|
}
|
|
1339
|
-
// The mounts first (ADR-0069): the door checks a `file:` digest against `jr2-images`, and a
|
|
1340
|
-
// provision after the converge reads `jr2-held`. A process that reports no mounts is not
|
|
1341
|
-
// deployed, and has nothing to wait for.
|
|
1342
|
-
const stale = (v: AssemblyView): string[] =>
|
|
1343
|
-
v.mounts ? Object.keys(opts.mounts).filter((name) => v.mounts![name] !== opts.mounts[name]) : [];
|
|
1344
|
-
const deadline = Date.now() + MAP_PROPAGATION.budgetMs;
|
|
1345
|
-
let said = false;
|
|
1346
|
-
for (let names = stale(seen); names.length; names = stale(seen)) {
|
|
1347
|
-
if (Date.now() >= deadline) {
|
|
1348
|
-
activity(
|
|
1349
|
-
io,
|
|
1350
|
-
`refusing: after ${Math.round(MAP_PROPAGATION.budgetMs / 1000)}s the Orchestrator still reads an older ` +
|
|
1351
|
-
`${names.join(", ")} from its mount — the kubelet has not refreshed it; nothing was published, and ` +
|
|
1352
|
-
`the previous Assembly is still current. Run \`jr2 up\` again`,
|
|
1353
|
-
);
|
|
1354
|
-
return false;
|
|
1355
|
-
}
|
|
1356
|
-
if (!said) activity(io, `assembly: waiting for the kubelet to deliver ${names.join(", ")} to the Orchestrator`);
|
|
1357
|
-
said = true;
|
|
1358
|
-
await new Promise((ok) => setTimeout(ok, MAP_PROPAGATION.stepMs));
|
|
1359
|
-
seen = await view();
|
|
1360
|
-
}
|
|
1361
1479
|
activity(
|
|
1362
1480
|
io,
|
|
1363
1481
|
`assembly: publishing ${assembly.hash} (${current?.hash ? `current is ${current.hash}` : "no current Assembly"})` +
|
|
@@ -1381,6 +1499,8 @@ async function publishAssembly(
|
|
|
1381
1499
|
`assembly: ${body.hash ?? assembly.hash} is current${body.unchanged ? " (unchanged)" : ""} — ` +
|
|
1382
1500
|
`workflows: ${names.join(", ") || "(none)"}`,
|
|
1383
1501
|
);
|
|
1502
|
+
// Each standing Workflow's state after the publish (ADR-0017), from the publish's answer.
|
|
1503
|
+
for (const st of Array.isArray(body.standing) ? body.standing : []) activity(io, standingLine(st));
|
|
1384
1504
|
return true;
|
|
1385
1505
|
}
|
|
1386
1506
|
const report = body.report ?? [];
|
|
@@ -1480,6 +1600,8 @@ async function resolveHeld(
|
|
|
1480
1600
|
models: opts.models,
|
|
1481
1601
|
});
|
|
1482
1602
|
for (const warning of warnings) activity(io, `warning: ${warning}`);
|
|
1603
|
+
const r14 = envFromHeldRefusal(config, manifest, opts.envFromKeys);
|
|
1604
|
+
if (r14) throw new Error(r14);
|
|
1483
1605
|
// Each pod's own: the bindings of the names it holds and their literal values. Its Custodian's
|
|
1484
1606
|
// files are rendered by whoever makes the pod — here for the Instance Harness alone; the
|
|
1485
1607
|
// Orchestrator at provision for a Sandbox, from the same manifest (ADR-0059).
|
|
@@ -1496,7 +1618,7 @@ async function resolveHeld(
|
|
|
1496
1618
|
const orchestrator = { host: `${ORCHESTRATOR_SERVICE}.${namespace}.svc`, port: ORCHESTRATOR_PORT };
|
|
1497
1619
|
const instance: CustodianFiles | undefined = instancePod && {
|
|
1498
1620
|
bootstrap: JSON.stringify(custodianBootstrap(instancePod.manifest, orchestrator)),
|
|
1499
|
-
script: custodianScript(instancePod.manifest),
|
|
1621
|
+
script: custodianScript(instancePod.manifest, { fetch: false }),
|
|
1500
1622
|
};
|
|
1501
1623
|
if (manifest.secrets.length === 0) return { objects: { pods, ...(instance && { instance }), leaves: {} } };
|
|
1502
1624
|
for (const s of manifest.secrets) {
|
|
@@ -1531,6 +1653,36 @@ async function resolveHeld(
|
|
|
1531
1653
|
};
|
|
1532
1654
|
}
|
|
1533
1655
|
|
|
1656
|
+
/**
|
|
1657
|
+
* R14 (ADR-0059): a held secret's name that is also a key of a Secret `harness.envFrom` loads. The
|
|
1658
|
+
* Stand-ins are files, not env (ADR-0076), so no Stand-in hides that key: the Secret's real value
|
|
1659
|
+
* would reach the Harness container's env under the held secret's own name, and the Harness uses a
|
|
1660
|
+
* held secret but never holds it. The key is compared as the kubelet writes it, `prefix` first.
|
|
1661
|
+
* The message names the Secret, the key and the held secret, and never a value. Exported for its
|
|
1662
|
+
* test.
|
|
1663
|
+
*/
|
|
1664
|
+
export function envFromHeldRefusal(
|
|
1665
|
+
config: JR2Config,
|
|
1666
|
+
manifest: HeldManifest,
|
|
1667
|
+
envFromKeys: ReadonlyMap<string, readonly string[]>,
|
|
1668
|
+
): string | undefined {
|
|
1669
|
+
const held = new Set(manifest.secrets.map((s) => s.name));
|
|
1670
|
+
for (const [i, ref] of (config.harness?.envFrom ?? []).entries()) {
|
|
1671
|
+
const secret = ref.secretRef?.name;
|
|
1672
|
+
if (secret === undefined) continue;
|
|
1673
|
+
for (const key of envFromKeys.get(`secret/${secret}`) ?? []) {
|
|
1674
|
+
const name = (ref.prefix ?? "") + key;
|
|
1675
|
+
if (!held.has(name)) continue;
|
|
1676
|
+
return (
|
|
1677
|
+
`harness.envFrom[${i}] loads Secret "${secret}", whose key "${key}" is the env var ${name} — and ${name} is ` +
|
|
1678
|
+
`a held secret. Its Stand-in is a file, so nothing hides that key, and its value would reach the ` +
|
|
1679
|
+
`Harness's env. Drop the key from Secret "${secret}", or rename the held secret (ADR-0059, R14)`
|
|
1680
|
+
);
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
return undefined;
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1534
1686
|
/** The Instance Harness's held objects and its Custodian's files — the walk always places it when
|
|
1535
1687
|
* it converges. */
|
|
1536
1688
|
function instanceHeld(held: HeldObjects): HeldPodObjects & { custodian: CustodianFiles } {
|
|
@@ -2297,11 +2449,29 @@ export type CheckPod = {
|
|
|
2297
2449
|
/** The bindings of the checked keys: each a part of a held pod's manifest. */
|
|
2298
2450
|
manifest: HeldManifest;
|
|
2299
2451
|
providers: CheckedProvider[];
|
|
2452
|
+
/** The remote MCP servers that an Agent of this class names, by name, sorted (ADR-0079): the
|
|
2453
|
+
* check lists each one's tools through the Custodian, with the Stand-in of its secret. */
|
|
2454
|
+
mcp: CheckedMcpServer[];
|
|
2300
2455
|
};
|
|
2301
2456
|
|
|
2302
|
-
/** One
|
|
2303
|
-
|
|
2304
|
-
|
|
2457
|
+
/** One remote MCP server a check pod lists (ADR-0079). */
|
|
2458
|
+
export type CheckedMcpServer = {
|
|
2459
|
+
server: string;
|
|
2460
|
+
url: string;
|
|
2461
|
+
/** Its secret, which the check pod holds as a pod of the class holds it; absent: no credential. */
|
|
2462
|
+
secret?: string;
|
|
2463
|
+
};
|
|
2464
|
+
|
|
2465
|
+
/** One Harness pod as the walk places it: its held pod key, its class, the models it runs, the
|
|
2466
|
+
* MCP servers its Agents name, and the bindings it holds (none: its HTTPS goes direct, with no
|
|
2467
|
+
* proxy). */
|
|
2468
|
+
type Seat = {
|
|
2469
|
+
pod: string;
|
|
2470
|
+
sandbox: boolean;
|
|
2471
|
+
models: readonly string[];
|
|
2472
|
+
servers: readonly string[];
|
|
2473
|
+
held: HeldManifest["secrets"];
|
|
2474
|
+
};
|
|
2305
2475
|
|
|
2306
2476
|
/**
|
|
2307
2477
|
* Which providers the preflight checks, and in which pod (ADR-0019, ADR-0059). Every carried model
|
|
@@ -2323,7 +2493,8 @@ export function checkPodsOf(config: JR2Config, walk: PlacementWalk, pods: readon
|
|
|
2323
2493
|
const seats: Seat[] = [
|
|
2324
2494
|
...walk.workspaces.map((ws) => {
|
|
2325
2495
|
const pod = workspaceHeldPod({ workflow: ws.workflow, route: ws.pod.route });
|
|
2326
|
-
|
|
2496
|
+
const servers = (ws.pod.mcp ?? []).map((m) => m.entry.server);
|
|
2497
|
+
return { pod, sandbox: true, models: ws.pod.models, servers, held: heldOf(pod) };
|
|
2327
2498
|
}),
|
|
2328
2499
|
...(walk.instanceHarness
|
|
2329
2500
|
? [
|
|
@@ -2331,6 +2502,7 @@ export function checkPodsOf(config: JR2Config, walk: PlacementWalk, pods: readon
|
|
|
2331
2502
|
pod: INSTANCE_HARNESS_HELD_POD,
|
|
2332
2503
|
sandbox: false,
|
|
2333
2504
|
models: walk.instanceHarness.models,
|
|
2505
|
+
servers: walk.instanceHarness.servers ?? [],
|
|
2334
2506
|
held: heldOf(INSTANCE_HARNESS_HELD_POD),
|
|
2335
2507
|
},
|
|
2336
2508
|
]
|
|
@@ -2370,6 +2542,7 @@ export function checkPodsOf(config: JR2Config, walk: PlacementWalk, pods: readon
|
|
|
2370
2542
|
// Stand-ins) is that of a pod of the class; the keys the class's other pods hold join it below.
|
|
2371
2543
|
manifest: { version: 1, secrets: [...group.seats[0]!.held] },
|
|
2372
2544
|
providers: [],
|
|
2545
|
+
mcp: [],
|
|
2373
2546
|
};
|
|
2374
2547
|
for (const model of models) {
|
|
2375
2548
|
const slash = model.indexOf("/");
|
|
@@ -2408,7 +2581,23 @@ export function checkPodsOf(config: JR2Config, walk: PlacementWalk, pods: readon
|
|
|
2408
2581
|
...(gateway ? { gateway } : {}),
|
|
2409
2582
|
});
|
|
2410
2583
|
}
|
|
2411
|
-
|
|
2584
|
+
// Each remote MCP server that an Agent of the class names (ADR-0079), with its secret's binding
|
|
2585
|
+
// from a pod of the class that holds it: the check reads the list as a Turn there reads it. A
|
|
2586
|
+
// stdio server is Workspace code, and the check pod runs none.
|
|
2587
|
+
const servers = [...new Set(group.seats.flatMap((s) => s.servers))].sort();
|
|
2588
|
+
for (const name of servers) {
|
|
2589
|
+
const declared = config.mcp?.[name];
|
|
2590
|
+
if (declared === undefined || isStdioServer(declared)) continue;
|
|
2591
|
+
const secret = declared.secret;
|
|
2592
|
+
if (secret !== undefined) {
|
|
2593
|
+
const binding = group.seats.map((s) => s.held.find((b) => b.name === secret)).find((b) => b !== undefined);
|
|
2594
|
+
if (binding && !checkPod.manifest.secrets.some((h) => h.name === secret)) {
|
|
2595
|
+
checkPod.manifest.secrets.push(binding);
|
|
2596
|
+
}
|
|
2597
|
+
}
|
|
2598
|
+
checkPod.mcp.push({ server: name, url: declared.url, ...(secret !== undefined ? { secret } : {}) });
|
|
2599
|
+
}
|
|
2600
|
+
if (checkPod.providers.length || checkPod.mcp.length) checkPods.push(checkPod);
|
|
2412
2601
|
}
|
|
2413
2602
|
return checkPods;
|
|
2414
2603
|
}
|
|
@@ -2422,6 +2611,9 @@ const CHECK_START_SECONDS = 300;
|
|
|
2422
2611
|
* `@jr2/harness`'s `preflight-line.ts`, which a test holds this to), so one slow model never uses
|
|
2423
2612
|
* the time of the others. Exported for that test. */
|
|
2424
2613
|
export const CHECK_MODEL_SECONDS = 360;
|
|
2614
|
+
/** Each MCP server's bound in the pod: its own budget in the check (`PREFLIGHT_MCP_BUDGET_SECONDS`
|
|
2615
|
+
* in `preflight-line.ts`, which a test holds this to). Exported for that test. */
|
|
2616
|
+
export const CHECK_MCP_SECONDS = 60;
|
|
2425
2617
|
const CHECK_RUN_MARGIN_SECONDS = 30;
|
|
2426
2618
|
/** A leftover a run left with no pod, older than this, is a dead run's, not a live one's. */
|
|
2427
2619
|
const CHECK_LEFTOVER_SECONDS = 3600;
|
|
@@ -2460,6 +2652,12 @@ async function preflightProviders(
|
|
|
2460
2652
|
trust: Record<string, string>;
|
|
2461
2653
|
harnessEnvData: Record<string, string>;
|
|
2462
2654
|
priorityClassName: string;
|
|
2655
|
+
/** The signing key, for each check pod's own Stand-ins (ADR-0019, ADR-0059). */
|
|
2656
|
+
signingKey: Buffer;
|
|
2657
|
+
/** The config's `mcp` block, for the check pod's harness config (ADR-0079). */
|
|
2658
|
+
mcp?: JR2Config["mcp"];
|
|
2659
|
+
/** Each Agent's allowlist of each remote server it names: what each listing is held to. */
|
|
2660
|
+
mcpUses: readonly McpUse[];
|
|
2463
2661
|
},
|
|
2464
2662
|
): Promise<void> {
|
|
2465
2663
|
const { namespace, ctx } = opts;
|
|
@@ -2469,10 +2667,12 @@ async function preflightProviders(
|
|
|
2469
2667
|
// Never silent: a converge that checked nothing must not read as one that checked. A model that
|
|
2470
2668
|
// two check pods would check is named once.
|
|
2471
2669
|
const models = [...new Set(opts.checkPods.flatMap((p) => p.providers.flatMap((v) => v.models)))];
|
|
2670
|
+
const servers = [...new Set(opts.checkPods.flatMap((p) => p.mcp.map((m) => m.server)))].sort();
|
|
2472
2671
|
activity(
|
|
2473
2672
|
io,
|
|
2474
2673
|
"warning: provider: preflight skipped (--skip-provider-preflight) — " +
|
|
2475
|
-
(models.length ? `these models were not checked: ${models.join(", ")}` : "no model was checked")
|
|
2674
|
+
(models.length ? `these models were not checked: ${models.join(", ")}` : "no model was checked") +
|
|
2675
|
+
(servers.length ? `; these MCP servers' tool lists were not checked: ${servers.join(", ")}` : ""),
|
|
2476
2676
|
);
|
|
2477
2677
|
return;
|
|
2478
2678
|
}
|
|
@@ -2480,9 +2680,12 @@ async function preflightProviders(
|
|
|
2480
2680
|
const run = randomBytes(4).toString("hex");
|
|
2481
2681
|
const orchestrator = { host: `${ORCHESTRATOR_SERVICE}.${namespace}.svc`, port: ORCHESTRATOR_PORT };
|
|
2482
2682
|
const faults: string[] = [];
|
|
2683
|
+
/** An MCP result already said: two check pods can list one server, and say it once. */
|
|
2684
|
+
const saidMcp = new Set<string>();
|
|
2483
2685
|
for (const [n, checkPod] of opts.checkPods.entries()) {
|
|
2484
2686
|
const object = `${PROVIDER_PREFLIGHT}-${run}-${n}`;
|
|
2485
2687
|
const models = checkPod.providers.flatMap((p) => p.models);
|
|
2688
|
+
const servers = checkPod.mcp.map((m) => m.server);
|
|
2486
2689
|
/** The models whose result is already said, while the pod ran, and what it was: a final log
|
|
2487
2690
|
* read that comes back empty must not turn a result already said into "not checked". */
|
|
2488
2691
|
const said = new Set<string>();
|
|
@@ -2501,12 +2704,27 @@ async function preflightProviders(
|
|
|
2501
2704
|
: `with no held key (${why}), direct, as a pod that holds no secret calls it`;
|
|
2502
2705
|
activity(io, `provider: checking ${p.models.join(", ")} on ${checkPod.where}, ${holds}`);
|
|
2503
2706
|
}
|
|
2707
|
+
for (const m of checkPod.mcp) {
|
|
2708
|
+
const holds = m.secret
|
|
2709
|
+
? `through a Custodian that holds ${m.secret}, with its Stand-in`
|
|
2710
|
+
: checkPod.manifest.secrets.length && m.url.startsWith("https:")
|
|
2711
|
+
? "with no credential, through the Custodian's tunnel"
|
|
2712
|
+
: "with no credential, direct";
|
|
2713
|
+
activity(io, `mcp: listing the tools of "${m.server}" at ${m.url} on ${checkPod.where}, ${holds}`);
|
|
2714
|
+
}
|
|
2504
2715
|
activity(
|
|
2505
2716
|
io,
|
|
2506
2717
|
`provider: the check pod has up to ${CHECK_START_SECONDS / 60} min to start, then up to ` +
|
|
2507
|
-
|
|
2718
|
+
(models.length
|
|
2719
|
+
? `${CHECK_MODEL_SECONDS / 60} min for ${models.length === 1 ? "its model" : `each of its ${models.length} models`}`
|
|
2720
|
+
: "") +
|
|
2721
|
+
(models.length && servers.length ? " and " : "") +
|
|
2722
|
+
(servers.length
|
|
2723
|
+
? `${CHECK_MCP_SECONDS / 60} min for ${servers.length === 1 ? "its MCP server" : `each of its ${servers.length} MCP servers`}`
|
|
2724
|
+
: ""),
|
|
2508
2725
|
);
|
|
2509
|
-
const runSeconds =
|
|
2726
|
+
const runSeconds =
|
|
2727
|
+
CHECK_MODEL_SECONDS * models.length + CHECK_MCP_SECONDS * servers.length + CHECK_RUN_MARGIN_SECONDS;
|
|
2510
2728
|
// Built before the apply: a fault in jr2 here is jr2's, never the cluster's refusal.
|
|
2511
2729
|
const manifest = checkPodObjects({
|
|
2512
2730
|
name: opts.name,
|
|
@@ -2521,11 +2739,20 @@ async function preflightProviders(
|
|
|
2521
2739
|
trust: opts.trust,
|
|
2522
2740
|
custodian: {
|
|
2523
2741
|
bootstrap: JSON.stringify(custodianBootstrap(checkPod.manifest, orchestrator)),
|
|
2524
|
-
script: custodianScript(checkPod.manifest),
|
|
2742
|
+
script: custodianScript(checkPod.manifest, { fetch: false }),
|
|
2525
2743
|
},
|
|
2526
|
-
|
|
2744
|
+
// Its own credentials (ADR-0019, ADR-0076): a control token the Orchestrator never signed,
|
|
2745
|
+
// and secret Stand-ins minted for this pod alone.
|
|
2746
|
+
credentials: podCredentials(opts.signingKey, {
|
|
2747
|
+
placement: object,
|
|
2748
|
+
heldPod: object,
|
|
2749
|
+
manifest: checkPod.manifest,
|
|
2750
|
+
controlToken: "unsigned",
|
|
2751
|
+
}),
|
|
2527
2752
|
harnessEnvData: opts.harnessEnvData,
|
|
2528
2753
|
models,
|
|
2754
|
+
servers,
|
|
2755
|
+
...(opts.mcp ? { mcp: opts.mcp } : {}),
|
|
2529
2756
|
...(checkPod.placement ? { placement: checkPod.placement } : {}),
|
|
2530
2757
|
priorityClassName: opts.priorityClassName,
|
|
2531
2758
|
...(opts.harness ? { harness: opts.harness } : {}),
|
|
@@ -2587,10 +2814,140 @@ async function preflightProviders(
|
|
|
2587
2814
|
faults.push(providerFault(p, model, checkPod.where, why, checkPod.manifest.secrets.length > 0));
|
|
2588
2815
|
}
|
|
2589
2816
|
}
|
|
2817
|
+
const listings = mcpCheckVerdicts(output);
|
|
2818
|
+
const startedMcp = mcpCheckStarts(output);
|
|
2819
|
+
for (const m of checkPod.mcp) {
|
|
2820
|
+
const verdict = listings.get(m.server);
|
|
2821
|
+
const lines = verdict
|
|
2822
|
+
? mcpVerdictLines(m, verdict, opts.mcpUses, checkPod.where)
|
|
2823
|
+
: {
|
|
2824
|
+
faults: [
|
|
2825
|
+
`mcp: "${m.server}" at ${m.url} was not checked on ${checkPod.where}: ` +
|
|
2826
|
+
(refused ??
|
|
2827
|
+
(startedMcp.has(m.server)
|
|
2828
|
+
? `the check pod stopped while it listed this server: ${podFault(end!, checkPod, opts)}`
|
|
2829
|
+
: `the check pod stopped before it reached this server: ${podFault(end!, checkPod, opts)}`)),
|
|
2830
|
+
],
|
|
2831
|
+
said: [],
|
|
2832
|
+
};
|
|
2833
|
+
for (const line of lines.said) {
|
|
2834
|
+
if (saidMcp.has(line)) continue;
|
|
2835
|
+
saidMcp.add(line);
|
|
2836
|
+
activity(io, line);
|
|
2837
|
+
}
|
|
2838
|
+
for (const fault of lines.faults) if (!faults.includes(fault)) faults.push(fault);
|
|
2839
|
+
}
|
|
2590
2840
|
}
|
|
2591
2841
|
if (faults.length) throw new Error(faults.join("\n"));
|
|
2592
2842
|
}
|
|
2593
2843
|
|
|
2844
|
+
/** One Agent's use of one remote MCP server, as the walk read it: the allowlist, or `"*"`. */
|
|
2845
|
+
export type McpUse = { workflow: string; slot: string; server: string; tools: readonly string[] | "*" };
|
|
2846
|
+
|
|
2847
|
+
/** One MCP server's result line (`preflight-line.ts` in `@jr2/harness`). */
|
|
2848
|
+
type McpCheckVerdict = { kind: "listed"; tools: string[] } | { kind: "refused" | "warn" | "fault"; reason: string };
|
|
2849
|
+
|
|
2850
|
+
/** The servers whose listing began (`start` lines). */
|
|
2851
|
+
export function mcpCheckStarts(output: string): Set<string> {
|
|
2852
|
+
const started = new Set<string>();
|
|
2853
|
+
for (const line of output.split("\n")) {
|
|
2854
|
+
const m = /^jr2\.preflight mcp=(\S+) start$/.exec(line.trim());
|
|
2855
|
+
if (m) started.add(m[1]!);
|
|
2856
|
+
}
|
|
2857
|
+
return started;
|
|
2858
|
+
}
|
|
2859
|
+
|
|
2860
|
+
/** Each MCP server's verdict, off the check container's output lines (`preflight-line.ts` in
|
|
2861
|
+
* `@jr2/harness`, held to this parser by a test). A list that is not JSON is the check's fault. */
|
|
2862
|
+
export function mcpCheckVerdicts(output: string): Map<string, McpCheckVerdict> {
|
|
2863
|
+
const verdicts = new Map<string, McpCheckVerdict>();
|
|
2864
|
+
for (const line of output.split("\n")) {
|
|
2865
|
+
const m = /^jr2\.preflight mcp=(\S+) (listed|refused|warn|fault)=(.*)$/.exec(line.trim());
|
|
2866
|
+
if (!m) continue;
|
|
2867
|
+
if (m[2] !== "listed") {
|
|
2868
|
+
verdicts.set(m[1]!, { kind: m[2] as "refused" | "warn" | "fault", reason: m[3]! });
|
|
2869
|
+
continue;
|
|
2870
|
+
}
|
|
2871
|
+
let tools: unknown;
|
|
2872
|
+
try {
|
|
2873
|
+
tools = JSON.parse(m[3]!);
|
|
2874
|
+
} catch {
|
|
2875
|
+
tools = undefined;
|
|
2876
|
+
}
|
|
2877
|
+
verdicts.set(
|
|
2878
|
+
m[1]!,
|
|
2879
|
+
Array.isArray(tools) && tools.every((t) => typeof t === "string")
|
|
2880
|
+
? { kind: "listed", tools: tools as string[] }
|
|
2881
|
+
: { kind: "fault", reason: `the check wrote a tool list that is not a JSON list of names — a jr2 defect` },
|
|
2882
|
+
);
|
|
2883
|
+
}
|
|
2884
|
+
return verdicts;
|
|
2885
|
+
}
|
|
2886
|
+
|
|
2887
|
+
/**
|
|
2888
|
+
* What one MCP server's listing comes to (ADR-0079): the lines said, and the faults. An allowlisted
|
|
2889
|
+
* name that the list does not have is refused, with what the server has; a server that refused the
|
|
2890
|
+
* request (401, 403) fails the converge, as a model key that the provider refuses does; a server
|
|
2891
|
+
* that did not answer is a warning. Pure, for its test.
|
|
2892
|
+
*/
|
|
2893
|
+
export function mcpVerdictLines(
|
|
2894
|
+
m: CheckedMcpServer,
|
|
2895
|
+
verdict: McpCheckVerdict,
|
|
2896
|
+
uses: readonly McpUse[],
|
|
2897
|
+
where: string,
|
|
2898
|
+
): { said: string[]; faults: string[] } {
|
|
2899
|
+
if (verdict.kind === "warn") {
|
|
2900
|
+
return {
|
|
2901
|
+
said: [
|
|
2902
|
+
`warning: mcp: "${m.server}" at ${m.url} did not answer from a pod on ${where} (${verdict.reason}) — its ` +
|
|
2903
|
+
"tool names are not checked; a Turn of an Agent that names it fails if it does not answer then (ADR-0079)",
|
|
2904
|
+
],
|
|
2905
|
+
faults: [],
|
|
2906
|
+
};
|
|
2907
|
+
}
|
|
2908
|
+
if (verdict.kind === "fault") {
|
|
2909
|
+
return { said: [], faults: [`mcp: "${m.server}" was not checked on ${where}: ${verdict.reason}`] };
|
|
2910
|
+
}
|
|
2911
|
+
if (verdict.kind === "refused") {
|
|
2912
|
+
return {
|
|
2913
|
+
said: [],
|
|
2914
|
+
faults: [
|
|
2915
|
+
`mcp: "${m.server}" at ${m.url} refused the request from a pod on ${where} (HTTP ${verdict.reason}) — ` +
|
|
2916
|
+
(m.secret
|
|
2917
|
+
? `a wrong or expired value of ${m.secret}, or a binding of it (hosts, headers, paths) that does not let ` +
|
|
2918
|
+
"its Stand-in through to the url"
|
|
2919
|
+
: "the server needs a credential: give the server a `secret`") +
|
|
2920
|
+
"; every call of an Agent that names it would fail (ADR-0079)",
|
|
2921
|
+
],
|
|
2922
|
+
};
|
|
2923
|
+
}
|
|
2924
|
+
if (verdict.kind !== "listed") return { said: [], faults: [] };
|
|
2925
|
+
const listed = verdict.tools;
|
|
2926
|
+
const missing = new Map<string, string[]>();
|
|
2927
|
+
for (const use of uses) {
|
|
2928
|
+
if (use.server !== m.server || use.tools === "*") continue;
|
|
2929
|
+
for (const tool of use.tools) {
|
|
2930
|
+
if (listed.includes(tool)) continue;
|
|
2931
|
+
const by = `workflow "${use.workflow}": Agent "${use.slot}"`;
|
|
2932
|
+
missing.set(tool, [...new Set([...(missing.get(tool) ?? []), by])]);
|
|
2933
|
+
}
|
|
2934
|
+
}
|
|
2935
|
+
if (missing.size === 0) {
|
|
2936
|
+
return {
|
|
2937
|
+
said: [`mcp: "${m.server}" lists ${listed.length} tool(s); every allowlisted name is there`],
|
|
2938
|
+
faults: [],
|
|
2939
|
+
};
|
|
2940
|
+
}
|
|
2941
|
+
const has = listed.length ? listed.join(", ") : "no tool";
|
|
2942
|
+
return {
|
|
2943
|
+
said: [],
|
|
2944
|
+
faults: [...missing].map(
|
|
2945
|
+
([tool, by]) =>
|
|
2946
|
+
`mcp: ${by.join(", ")} allowlists the tool "${tool}", which "${m.server}" does not list — it lists: ${has} (ADR-0079)`,
|
|
2947
|
+
),
|
|
2948
|
+
};
|
|
2949
|
+
}
|
|
2950
|
+
|
|
2594
2951
|
/** How long the first apply in a new namespace waits for its `default` ServiceAccount: the
|
|
2595
2952
|
* controller makes it a moment after the namespace, and until then a pod is refused. */
|
|
2596
2953
|
const FIRST_CONTACT_TRIES = 10;
|
|
@@ -2666,7 +3023,7 @@ async function removeCheckLeftovers(
|
|
|
2666
3023
|
if (!(since(finished ?? p.metadata.creationTimestamp) > CHECK_ENDED_MARGIN_SECONDS)) continue;
|
|
2667
3024
|
await removeCheckPod(io, kube, p.metadata.name, namespace, ctx);
|
|
2668
3025
|
}
|
|
2669
|
-
for (const kind of ["configmap", "secret"]) {
|
|
3026
|
+
for (const kind of ["configmap", "secret", "networkpolicy"]) {
|
|
2670
3027
|
for (const o of await kube.listJson<CheckObject>({ kind, selector, namespace, ...ctx })) {
|
|
2671
3028
|
const owner = o.metadata.name.replace(/-(ca|tls|env)$/, "");
|
|
2672
3029
|
if (named.has(owner) || !(since(o.metadata.creationTimestamp) > CHECK_LEFTOVER_SECONDS)) continue;
|
|
@@ -2799,7 +3156,7 @@ export function podFault(
|
|
|
2799
3156
|
return `the check pod ended ${end.phase}${said}, and jr2 up could not read its log — run \`jr2 up\` again`;
|
|
2800
3157
|
}
|
|
2801
3158
|
const tail = end.output.trim().split("\n").filter(Boolean).slice(-5).join("; ");
|
|
2802
|
-
return `the check pod ended ${end.phase} before it checked this
|
|
3159
|
+
return `the check pod ended ${end.phase} before it checked this one${said} — it said: ${tail}`;
|
|
2803
3160
|
}
|
|
2804
3161
|
|
|
2805
3162
|
/** One model's preflight failure: the model, the host, and the likely causes. */
|