@jr2/cli 0.2.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/jr2.js +3 -3
- package/manifests/operator.yaml +266 -24
- package/package.json +6 -4
- package/src/assembly.ts +432 -0
- package/src/build.ts +137 -300
- package/src/cli.ts +6 -2
- package/src/client.ts +126 -33
- package/src/commands/down.ts +2 -1
- package/src/commands/init.ts +9 -2
- package/src/commands/kit.ts +24 -12
- package/src/commands/logs.ts +9 -0
- package/src/commands/run.ts +58 -4
- package/src/commands/runs.ts +96 -4
- package/src/commands/send.ts +20 -7
- package/src/commands/status.ts +90 -14
- package/src/commands/up.ts +1174 -183
- package/src/commands/version.ts +18 -13
- package/src/deploy.ts +782 -111
- package/src/effector-image.ts +133 -0
- package/src/held-pki.ts +306 -0
- package/src/index.ts +4 -0
- package/src/kube.ts +115 -9
- package/src/nodes.ts +27 -2
- package/src/sweep.ts +27 -13
package/src/commands/up.ts
CHANGED
|
@@ -1,15 +1,23 @@
|
|
|
1
1
|
// `jr2 up [--yes] [--force] [-n <ns>] [--context <ctx>]` (ADR-0019): idempotently converge the target
|
|
2
2
|
// namespace to this instance — every layer, loudly narrated, safe to re-run. Layers in order:
|
|
3
|
-
// typecheck → the Machine walk →
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
3
|
+
// typecheck → the Machine walk → staging the Assembly (ADR-0069) → the Kubernetes floor (ADR-0037)
|
|
4
|
+
// → ownership → image resolution → operator → Kit images (the Harness, the Orchestrator) → Sandbox
|
|
5
|
+
// Images → Secret (+ preflight of referenced Secrets) → apply + rollout → the cache agent
|
|
6
|
+
// (ADR-0051: a DaemonSet converged when a registered Machine composes a Sandbox, deleted when none
|
|
7
|
+
// does) → Instance Harness (ADR-0031: converged by convention when a carried definition declares
|
|
8
|
+
// `workspace: "none"`, deleted when none does) → Instance Effector → publishing the Assembly to the
|
|
9
|
+
// running Orchestrator (ADR-0069) → a report of live workspaces still on an older image.
|
|
10
|
+
//
|
|
11
|
+
// The Assembly is staged EARLY and published LAST (ADR-0069). Early, so its refusals (a second copy
|
|
12
|
+
// of a Kit package, a Kit version mismatch, a tree that names its staging path) come before any
|
|
13
|
+
// cluster spend. Last, because the door's own checks are true only once the cluster work is done:
|
|
14
|
+
// the image map holds every `file:` digest, the Secrets exist, and the Orchestrator runs the Kit
|
|
15
|
+
// version the Assembly names. A Machine edit therefore rolls nothing: it is a new Assembly, not a
|
|
16
|
+
// new pod template. The Orchestrator creates a Repo resource per
|
|
17
|
+
// Repo the walk found bound, at boot and at each publish, and the cache agent clones it onto a node on first need
|
|
18
|
+
// (ADR-0051); a configured custom provider is preflighted from inside the cluster. A bound ssh url
|
|
19
|
+
// asks where its deploy key comes from (ADR-0047), and a converge that GENERATED one ends by saying
|
|
20
|
+
// so — the key is dead until a human registers it.
|
|
13
21
|
//
|
|
14
22
|
// The typecheck is FIRST and is a gate (ADR-0050): a Machine names its Agents, its composed
|
|
15
23
|
// Machines, and its Repo Slots by string, and since ADR-0049 those strings are typed, so a wrong
|
|
@@ -24,11 +32,12 @@
|
|
|
24
32
|
// `--platform` passed explicitly on every build. The cluster's schedulable nodes choose that set
|
|
25
33
|
// (`platforms` in the config overrides absolutely), so the daemon default and
|
|
26
34
|
// `DOCKER_DEFAULT_PLATFORM` steer nothing any more. In a kit CHECKOUT the built set includes the
|
|
27
|
-
// Harness,
|
|
35
|
+
// Harness, the operator and the Orchestrator (ADR-0069); installed from npm those sources do not resolve and the published
|
|
28
36
|
// `<kitversion>` refs are used with no docker at all (they are multi-arch manifest lists, so they
|
|
29
37
|
// take no suffix and never had this hole). There is no `images` config block and no env escape
|
|
30
38
|
// hatch — nobody gets to point a real cluster at a hand-picked Harness (ADR-0027's no-eject-hatch,
|
|
31
|
-
// enforced rather than stated).
|
|
39
|
+
// enforced rather than stated). An installed Instance with no `workspace()` therefore needs no
|
|
40
|
+
// docker at all: it builds nothing, and its Workflows ride the Assembly (ADR-0069).
|
|
32
41
|
// The converged name→ref map is stamped on the Orchestrator Deployment and diffed on the next run,
|
|
33
42
|
// so a steady-state converge spends directory walks and no docker. When that record is silent (a
|
|
34
43
|
// fresh namespace), the host daemon's labeled listing answers the BUILD question instead
|
|
@@ -45,43 +54,80 @@
|
|
|
45
54
|
// labeled objects found → converge silently (it's home); nothing → confirm first-time setup
|
|
46
55
|
// (`--yes` for CI); objects labeled as a DIFFERENT instance → refuse.
|
|
47
56
|
|
|
48
|
-
import { createHash, randomBytes } from "node:crypto";
|
|
57
|
+
import { createHash, createHmac, randomBytes } from "node:crypto";
|
|
49
58
|
import { open, readdir, readFile } from "node:fs/promises";
|
|
50
59
|
import { homedir } from "node:os";
|
|
51
|
-
import { basename, join } from "node:path";
|
|
60
|
+
import { basename, join, relative } from "node:path";
|
|
61
|
+
import { fileURLToPath } from "node:url";
|
|
52
62
|
import { parseArgs } from "node:util";
|
|
53
63
|
import {
|
|
54
|
-
|
|
64
|
+
custodianBootstrap,
|
|
65
|
+
custodianRef,
|
|
66
|
+
custodianScript,
|
|
67
|
+
CUSTODIAN_PROGRAM,
|
|
55
68
|
defaultImageContext,
|
|
69
|
+
heldSecretsOf,
|
|
70
|
+
HELD_CA_SECRET,
|
|
71
|
+
HELD_TLS_SECRET,
|
|
72
|
+
INSTANCE_CUSTODIAN_CONFIGMAP,
|
|
73
|
+
INSTANCE_HARNESS_HELD_POD,
|
|
74
|
+
ORCHESTRATOR_PORT,
|
|
56
75
|
imageContextDigest,
|
|
57
76
|
isSshUrl,
|
|
58
77
|
loadConfig,
|
|
78
|
+
loadCharts,
|
|
59
79
|
loadWorkflows,
|
|
80
|
+
chartFileRefusals,
|
|
81
|
+
machineBodies,
|
|
60
82
|
matchCredential,
|
|
83
|
+
openPartRefusals,
|
|
84
|
+
packAssembly,
|
|
85
|
+
configMapDigest,
|
|
86
|
+
MOUNTED_CONFIGMAPS,
|
|
61
87
|
partsOf,
|
|
88
|
+
placementWalkOf,
|
|
89
|
+
refuseEffectSecrets,
|
|
90
|
+
LABEL_EFFECTOR_POD,
|
|
91
|
+
LABEL_GIT_PUSH,
|
|
92
|
+
LABEL_GIT_CLONE,
|
|
93
|
+
gitCloneSecretsOf,
|
|
94
|
+
gitPushPlansOf,
|
|
95
|
+
repoIdentity,
|
|
96
|
+
type GitPushManifest,
|
|
97
|
+
type SandboxEffectorPlan,
|
|
98
|
+
secretPlacementsOf,
|
|
99
|
+
storedValueLength,
|
|
100
|
+
effectorKey,
|
|
101
|
+
targetText,
|
|
62
102
|
harnessTokenDigest,
|
|
103
|
+
formatCpu,
|
|
104
|
+
formatMemory,
|
|
105
|
+
resolveSize,
|
|
63
106
|
sandboxToken,
|
|
64
|
-
|
|
107
|
+
seatLine,
|
|
108
|
+
serializeMachine,
|
|
109
|
+
splitSize,
|
|
65
110
|
type CarriedImage,
|
|
66
111
|
type CarriedRepo,
|
|
112
|
+
type HeldManifest,
|
|
113
|
+
type HeldPodPlan,
|
|
67
114
|
type ImageRefs,
|
|
68
115
|
type JR2Config,
|
|
116
|
+
type ResolvedSize,
|
|
69
117
|
} from "@jr2/orchestrator";
|
|
118
|
+
import { stageAssembly, stagedCopyOf } from "../assembly.ts";
|
|
119
|
+
import { caFits, fingerprint, issueCa, issueLeaf, leafFits, trustBundles, type Pem } from "../held-pki.ts";
|
|
70
120
|
import {
|
|
71
121
|
assertEmulation,
|
|
72
122
|
buildSandboxImage,
|
|
73
123
|
choosePlatforms,
|
|
74
124
|
detectKitCheckout,
|
|
75
|
-
instanceImageLabels,
|
|
76
125
|
kitImageBuild,
|
|
77
126
|
kitImageRefs,
|
|
78
127
|
normalizeRef,
|
|
79
|
-
platformSuffix,
|
|
80
128
|
pnpmDockerBuild,
|
|
81
129
|
publishedKitRefs,
|
|
82
130
|
sandboxImageTag,
|
|
83
|
-
stageInstanceBundle,
|
|
84
|
-
INSTANCE_DOCKERFILE,
|
|
85
131
|
KIT_IMAGE_HOME,
|
|
86
132
|
type BuildPort,
|
|
87
133
|
type KitImageName,
|
|
@@ -90,29 +136,50 @@ import {
|
|
|
90
136
|
import {
|
|
91
137
|
ANNOTATION_IMAGES,
|
|
92
138
|
compareVersions,
|
|
139
|
+
INSTANCE_HARNESS_DRAIN_SECONDS,
|
|
140
|
+
INSTANCE_HARNESS_PODS_SERVICE,
|
|
141
|
+
INSTANCE_HARNESS_PVC,
|
|
93
142
|
INSTANCE_HARNESS_SERVICE,
|
|
94
143
|
instanceHarnessObjects,
|
|
144
|
+
instanceEffectorObjects,
|
|
145
|
+
effectorDigest,
|
|
146
|
+
storedValueDigest,
|
|
147
|
+
type ValueDigest,
|
|
148
|
+
effectorGraceSeconds,
|
|
149
|
+
INSTANCE_EFFECTOR,
|
|
95
150
|
instanceObjects,
|
|
96
151
|
KIT_VERSION,
|
|
97
|
-
LABEL_HASH,
|
|
98
152
|
LABEL_INSTANCE,
|
|
99
153
|
LABEL_VERSION,
|
|
100
154
|
OPERATOR_DEPLOYMENT,
|
|
101
155
|
OPERATOR_NAMESPACE,
|
|
102
156
|
OPERATOR_SELECTOR,
|
|
103
157
|
operatorManifest,
|
|
158
|
+
priorityClassNames,
|
|
159
|
+
priorityClassObjects,
|
|
104
160
|
REPO_CACHE,
|
|
161
|
+
trustObject,
|
|
162
|
+
holdsAny,
|
|
163
|
+
LABEL_HELD_POD,
|
|
164
|
+
type HeldObjects,
|
|
165
|
+
type CustodianFiles,
|
|
166
|
+
type HeldPodObjects,
|
|
105
167
|
} from "../deploy.ts";
|
|
106
168
|
import { resolveRoot } from "../instance.ts";
|
|
169
|
+
import { effectorImagePaths, effectorImageRefusal, STOCK_HARNESS_IMAGE, type PushForm } from "../effector-image.ts";
|
|
107
170
|
import {
|
|
171
|
+
kubeFloorRefusal,
|
|
172
|
+
podResourcesProbe,
|
|
108
173
|
kubectlAdmin,
|
|
174
|
+
kubectlKube,
|
|
109
175
|
rolloutFailure,
|
|
176
|
+
KUBE_FLOOR,
|
|
110
177
|
ORCHESTRATOR_SERVICE,
|
|
111
178
|
type KubeAdmin,
|
|
112
179
|
type KubeObject,
|
|
113
180
|
type RolloutTarget,
|
|
114
181
|
} from "../kube.ts";
|
|
115
|
-
import { buildNodes, sandboxNodes, type NodeObject } from "../nodes.ts";
|
|
182
|
+
import { anyNodeHolds, buildNodes, sandboxNodes, type NodeObject } from "../nodes.ts";
|
|
116
183
|
import { activity, chooseOrBail, confirmOrBail, promptLine, readSecretInput, type Io } from "../output.ts";
|
|
117
184
|
import { kindCluster, sweepImages } from "../sweep.ts";
|
|
118
185
|
import { tscTypecheck } from "../typecheck.ts";
|
|
@@ -174,42 +241,233 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
174
241
|
// The Machine is named by where it sits under the registered root — the slot chain a
|
|
175
242
|
// `customize()` walks — and the fix line nests the same way, so it pastes into
|
|
176
243
|
// `workflows/<name>.ts` with the imported Machine and the url filled in.
|
|
177
|
-
//
|
|
178
|
-
//
|
|
179
|
-
//
|
|
244
|
+
// Every Open part is refused in ONE listing, in walk order within each kind: a composer fixing a
|
|
245
|
+
// packaged Machine fixes all of them in the same edit, and a converge that refused the Repo Slot,
|
|
246
|
+
// then the Agent one run later, would cost them two rounds to learn one thing. The listing is the
|
|
247
|
+
// door's own (ADR-0069): the Orchestrator refuses a publish with the same lines, so the host walk
|
|
248
|
+
// and the door's walk cannot drift.
|
|
249
|
+
const open = openPartRefusals(workflows);
|
|
250
|
+
if (open.length) {
|
|
251
|
+
for (const line of open) activity(io, `refusing: ${line}`);
|
|
252
|
+
return 1;
|
|
253
|
+
}
|
|
254
|
+
// Each Chart file is admitted HERE by the door's own check (ADR-0070), so a Chart file the
|
|
255
|
+
// Orchestrator would refuse is refused at the same command as `tsc`'s errors, before any
|
|
256
|
+
// cluster work, with the lines the publish would answer.
|
|
257
|
+
const chartRefusals = chartFileRefusals(
|
|
258
|
+
await loadCharts(root, (line) => activity(io, `warning: ${line}`)),
|
|
259
|
+
workflows,
|
|
260
|
+
);
|
|
261
|
+
if (chartRefusals.length) {
|
|
262
|
+
for (const line of chartRefusals) activity(io, `refusing: ${line}`);
|
|
263
|
+
return 1;
|
|
264
|
+
}
|
|
265
|
+
// What each registered Machine runs outside itself (ADR-0067, ADR-0068): every Effect slot, with
|
|
266
|
+
// the plain data it carries — a command's argv, its timeout, the Repo Slot it runs in; an http
|
|
267
|
+
// Effect's method, its fixed target and the secrets it names — read, never run.
|
|
180
268
|
for (const w of workflows) {
|
|
181
|
-
const
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
if (path === undefined) {
|
|
269
|
+
const effects = partsOf([w.machine]).effects;
|
|
270
|
+
if (!effects.length) continue;
|
|
271
|
+
activity(io, `effects: workflow "${w.name}" — what it runs outside itself (ADR-0067)`);
|
|
272
|
+
for (const e of effects) {
|
|
273
|
+
const where = e.path?.length ? ` (on ${e.path.map((k) => `"${k}"`).join(" → ")})` : "";
|
|
274
|
+
const slot = e.slot !== undefined ? `, in Repo Slot "${e.slot}"` : "";
|
|
275
|
+
if (e.kind === "http") {
|
|
276
|
+
const names = e.secrets.length ? `, secrets ${e.secrets.map((n) => `"${n}"`).join(", ")}` : "";
|
|
277
|
+
// Its body decides where it runs (ADR-0067): a Worktree file needs its Workspace's Sandbox Effector.
|
|
278
|
+
const where2 = e.file
|
|
279
|
+
? `in the Sandbox Effector, a Worktree file as its body${slot}`
|
|
280
|
+
: "in the Instance Effector";
|
|
194
281
|
activity(
|
|
195
282
|
io,
|
|
196
|
-
`
|
|
197
|
-
|
|
198
|
-
`bind it there (ADR-0049, ${adr})`,
|
|
283
|
+
` ${e.name}${where}: http ${e.method} ${e.target ? targetText({ target: e.target }) : "<open>"} — ${e.timeoutMs / 1000}s, ` +
|
|
284
|
+
`${where2}${names}${e.idempotent ? ", idempotent" : ""}`,
|
|
199
285
|
);
|
|
200
|
-
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
if (e.kind === "push") {
|
|
289
|
+
const opts = (["pin", "sign", "squash", "force"] as const).filter((k) => e[k]);
|
|
290
|
+
activity(
|
|
291
|
+
io,
|
|
292
|
+
` ${e.name}${where}: push${opts.length ? ` (${opts.join(", ")})` : ""} — ${e.timeoutMs / 1000}s, ` +
|
|
293
|
+
`in the Sandbox Effector${slot}`,
|
|
294
|
+
);
|
|
295
|
+
continue;
|
|
201
296
|
}
|
|
202
|
-
|
|
297
|
+
if (!("argv" in e)) continue;
|
|
298
|
+
const names = e.secrets?.length ? `, secrets ${e.secrets.map((n) => `"${n}"`).join(", ")}` : "";
|
|
299
|
+
const seat = e.secrets?.length ? `the Sandbox Effector${e.pin ? ", pinned" : ""}` : "the Harness";
|
|
203
300
|
activity(
|
|
204
301
|
io,
|
|
205
|
-
`
|
|
206
|
-
`registered: export const machine = ${customizeLine("<import>", path, slot, part)} (${adr})`,
|
|
302
|
+
` ${e.name}${where}: ${e.kind} ${JSON.stringify(e.argv)} — ${e.timeoutMs / 1000}s, in ${seat}${slot}${names}`,
|
|
207
303
|
);
|
|
208
|
-
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
// An Effect that names a secret the `secrets` list does not hold, and a literal Effect secret too
|
|
307
|
+
// short to scrub (ADR-0068), are refused HERE, off the walk and the config alone — before anything is
|
|
308
|
+
// built or applied (ADR-0050, ADR-0068). A secret read from the user's Secret is placed below,
|
|
309
|
+
// once the cluster is reached.
|
|
310
|
+
// What names each secret, read off the walk (ADR-0059, ADR-0068): every Effect, every Workspace's
|
|
311
|
+
// pod — its `workspace()`'s `secrets` and the models of the Agents whose Turns run there — and the
|
|
312
|
+
// models of the `"none"` Agents the Instance Harness hosts.
|
|
313
|
+
// An Open body that allows "none" needs the Instance Harness, holding a key for each model it
|
|
314
|
+
// allows (ADR-0070), as a "none" Agent does. The door's walk says both, so `jr2 up` and the
|
|
315
|
+
// publish door place the same set.
|
|
316
|
+
const placementWalk = placementWalkOf(workflows);
|
|
317
|
+
refuseEffectSecrets(config, placementWalk);
|
|
318
|
+
// Who may send what, per state (ADR-0015): the audience rule's answer for each registered
|
|
319
|
+
// Machine and every Machine it composes, as its doc carries it — so an author reads who holds a
|
|
320
|
+
// pick before a run finds out. Then what the build found legal and suspect: a warning, never a
|
|
321
|
+
// refusal (the refusals threw when the Machine loaded).
|
|
322
|
+
for (const w of workflows) {
|
|
323
|
+
const bodies = machineBodies(serializeMachine(w.name, w.machine));
|
|
324
|
+
const seats = bodies.flatMap((body) => body.seats.map(seatLine));
|
|
325
|
+
if (seats.length) {
|
|
326
|
+
activity(io, `seats: workflow "${w.name}" — who may send what, per state (ADR-0015)`);
|
|
327
|
+
for (const line of seats) activity(io, ` ${line}`);
|
|
328
|
+
}
|
|
329
|
+
for (const body of bodies) {
|
|
330
|
+
for (const warning of body.warnings) activity(io, `warning: workflow "${w.name}": ${warning} (ADR-0015)`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
// The Size (ADR-0060): every `workspace()` a registered Machine composes, resolved down its chain
|
|
334
|
+
// (the Machine's, then `sandbox.resources`, then the kit's) and split inside the pod, exactly as
|
|
335
|
+
// the provision will. A split that leaves the Harness below its floor is refused HERE, naming the
|
|
336
|
+
// Machine and the number to change, rather than at the first provision of a run. The resolved
|
|
337
|
+
// Sizes are kept for the Sandbox-node warning below.
|
|
338
|
+
const sizes: Array<{ workflow: string; size: ResolvedSize }> = [];
|
|
339
|
+
for (const w of workflows) {
|
|
340
|
+
for (const { path, resources, user } of partsOf([w.machine]).sizes) {
|
|
341
|
+
try {
|
|
342
|
+
const size = resolveSize(resources, config.sandbox?.resources);
|
|
343
|
+
splitSize(size, user);
|
|
344
|
+
sizes.push({ workflow: w.name, size });
|
|
345
|
+
} catch (e) {
|
|
346
|
+
const where =
|
|
347
|
+
path !== undefined && path.length
|
|
348
|
+
? ` (on the Machine composed as ${path.map((k) => `"${k}"`).join(" → ")})`
|
|
349
|
+
: "";
|
|
350
|
+
const fix =
|
|
351
|
+
path === undefined
|
|
352
|
+
? ""
|
|
353
|
+
: `; a composer retunes it where the Machine is registered: export const machine = ${sizeLine(path)}`;
|
|
354
|
+
activity(io, `refusing: workflow "${w.name}"${where}: ${(e as Error).message}${fix}`);
|
|
355
|
+
return 1;
|
|
356
|
+
}
|
|
209
357
|
}
|
|
210
358
|
}
|
|
211
359
|
const agentNames = [...new Set(agents.map((a) => a.name))].join(", ") || "(none)";
|
|
212
360
|
activity(io, `agents: ${agentNames} (carried by ${workflows.length} workflow machine(s))`);
|
|
361
|
+
// Each Chart wrapper (ADR-0070): what its Open body lends a Chart, how many models a Chart's own
|
|
362
|
+
// Agents may name (each gets its key placed below), and the Chart files the Assembly carries for
|
|
363
|
+
// it. The walk reads the Open body's declaration, so the keys and the Instance Harness a Chart
|
|
364
|
+
// could need are placed before any run.
|
|
365
|
+
const chartFiles = await chartFilesIn(root);
|
|
366
|
+
for (const w of workflows) {
|
|
367
|
+
const bodies = partsOf([w.machine]).openBodies.filter((b) => b.path !== undefined && b.path.length === 0);
|
|
368
|
+
if (!bodies.length) continue;
|
|
369
|
+
const lends = [
|
|
370
|
+
...new Set(
|
|
371
|
+
bodies.flatMap(({ declaration: { lends } }) => [
|
|
372
|
+
...(lends.agents ?? []),
|
|
373
|
+
...(lends.effects ?? []),
|
|
374
|
+
...(lends.machines ?? []),
|
|
375
|
+
...(lends.guards ?? []),
|
|
376
|
+
...(lends.actions ?? []),
|
|
377
|
+
...(lends.delays ?? []),
|
|
378
|
+
]),
|
|
379
|
+
),
|
|
380
|
+
];
|
|
381
|
+
const models = new Set(bodies.flatMap((b) => b.declaration.allows.models));
|
|
382
|
+
const files = chartFiles.filter((n) => n.startsWith(`${w.name}/`)).length;
|
|
383
|
+
activity(
|
|
384
|
+
io,
|
|
385
|
+
`chart wrapper "${w.name}": lends ${lends.join(", ") || "nothing"}; allows ${models.size} model(s); ${files} Chart file(s)`,
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// --- the Assembly (ADR-0069): staged here, published last ---------------------------------------
|
|
390
|
+
// Staged BEFORE any cluster spend, so its refusals cost nothing: a second copy of a Kit package
|
|
391
|
+
// (ADR-0054's silent-empty-Menus failure), a Kit version other than this CLI's (ADR-0056), a tree
|
|
392
|
+
// that names its own staging path (ADR-0038). Packed at once and the scratch tree removed: the
|
|
393
|
+
// archive is what the door receives, and no later layer reads the tree.
|
|
394
|
+
const build = io.build ?? pnpmDockerBuild;
|
|
395
|
+
activity(io, "assembly: staging");
|
|
396
|
+
const staged = await stageAssembly(build, root);
|
|
397
|
+
let assembly: { hash: string; kitVersion: string; workflows: string[]; tar: Buffer };
|
|
398
|
+
// The scaffolded `images/default` is a PATH CONVENTION, not discovery (ADR-0049/0050): exactly
|
|
399
|
+
// one path is checked, and it is keyed `default` rather than by digest, because that reserved
|
|
400
|
+
// key IS the middle leg of the resolution chain a `workspace()` that names no image lands on.
|
|
401
|
+
const fallback = await defaultImageContext(root);
|
|
402
|
+
// Every `file:` context is keyed by the digest of its STAGED copy (ADR-0037, ADR-0069): the
|
|
403
|
+
// bytes the door checks against the image map and a provision resolves, which the Instance
|
|
404
|
+
// folder's own copy need not equal (staging drops `node_modules`, `.git`, `.env`, `.npmrc` at
|
|
405
|
+
// every depth). The build below still reads the folder; only the key comes from here. A context
|
|
406
|
+
// the Assembly does not carry is refused now, before any cluster spend.
|
|
407
|
+
let contextDigests: Map<string, string>;
|
|
408
|
+
// The Chart files the staged Assembly carries (ADR-0070): Workflows the door registers too.
|
|
409
|
+
let stagedCharts: string[] = [];
|
|
410
|
+
try {
|
|
411
|
+
assembly = {
|
|
412
|
+
hash: staged.hash,
|
|
413
|
+
kitVersion: staged.kitVersion,
|
|
414
|
+
workflows: staged.workflows,
|
|
415
|
+
tar: await packAssembly(staged.dir),
|
|
416
|
+
};
|
|
417
|
+
contextDigests = await stagedContextDigests(staged.dir, root, [
|
|
418
|
+
...(fallback ? [fallback] : []),
|
|
419
|
+
...contexts.map((image: CarriedImage) => image.dir),
|
|
420
|
+
]);
|
|
421
|
+
stagedCharts = await chartFilesIn(staged.dir);
|
|
422
|
+
} finally {
|
|
423
|
+
await staged.dispose();
|
|
424
|
+
}
|
|
425
|
+
// The walk above loaded the Instance folder; the door loads the Assembly (ADR-0069). The two must
|
|
426
|
+
// hold the same Workflows, or a Workflow the walk built and placed for is absent at the door, and
|
|
427
|
+
// `jr2 run` names it unknown after a converge that said it succeeded.
|
|
428
|
+
// A Chart file is a Workflow too (ADR-0070), `<wrapper>/<name>`, at `workflows/<wrapper>/<name>.chart.json`.
|
|
429
|
+
const walked = [...workflows.map((w) => w.name), ...chartFiles];
|
|
430
|
+
const inAssembly = [...assembly.workflows, ...stagedCharts];
|
|
431
|
+
const absent = walked.filter((n) => !inAssembly.includes(n));
|
|
432
|
+
const extra = inAssembly.filter((n) => !walked.includes(n));
|
|
433
|
+
const fileOf = (n: string) => (n.includes("/") ? `workflows/${n}.chart.json` : `workflows/${n}.ts`);
|
|
434
|
+
if (absent.length || extra.length) {
|
|
435
|
+
activity(
|
|
436
|
+
io,
|
|
437
|
+
`refusing: the staged Assembly does not hold the Workflows this folder holds (ADR-0069)` +
|
|
438
|
+
(absent.length ? `; absent from the Assembly: ${absent.map(fileOf).join(", ")}` : "") +
|
|
439
|
+
(extra.length ? `; only in the Assembly: ${extra.map(fileOf).join(", ")}` : "") +
|
|
440
|
+
` — the Assembly holds what the Instance's install keeps; check the \`files:\` list in package.json`,
|
|
441
|
+
);
|
|
442
|
+
return 1;
|
|
443
|
+
}
|
|
444
|
+
activity(
|
|
445
|
+
io,
|
|
446
|
+
`assembly: ${assembly.hash} (kit v${assembly.kitVersion}; ${assembly.workflows.length} workflow(s): ` +
|
|
447
|
+
`${assembly.workflows.join(", ") || "(none)"})`,
|
|
448
|
+
);
|
|
449
|
+
|
|
450
|
+
// --- the Kubernetes floor (ADR-0037) ----------------------------------------------------------
|
|
451
|
+
// Every Sandbox mounts jr2's runtime as an `image` volume, and there is no fallback below the
|
|
452
|
+
// version that turns them on — so a cluster under it is refused HERE, naming what it runs and the
|
|
453
|
+
// floor (ADR-0046), rather than converging and leaving every Sandbox Pending on a volume type its
|
|
454
|
+
// kubelet does not know. Before ownership, so first contact never asks to deploy somewhere that
|
|
455
|
+
// cannot host it. Unconditional: the floor is the Instance's, not one Machine's. The container
|
|
456
|
+
// runtime owes image volumes too (containerd ≥ 2.1, CRI-O ≥ 1.31); that is not cheaply visible
|
|
457
|
+
// from here, so an older one fails at the first provision, in the pod's own events.
|
|
458
|
+
let serverVersion: string;
|
|
459
|
+
try {
|
|
460
|
+
serverVersion = await kube.serverVersion(ctx);
|
|
461
|
+
} catch (e) {
|
|
462
|
+
activity(io, `refusing: cannot reach the API server of ${context} (${(e as Error).message}). Nothing was applied.`);
|
|
463
|
+
return 1;
|
|
464
|
+
}
|
|
465
|
+
const belowFloor = kubeFloorRefusal(serverVersion);
|
|
466
|
+
if (belowFloor) {
|
|
467
|
+
activity(io, `refusing: ${context} ${belowFloor}. Nothing was applied.`);
|
|
468
|
+
return 1;
|
|
469
|
+
}
|
|
470
|
+
activity(io, `kubernetes: ${serverVersion} (floor ${KUBE_FLOOR.major}.${KUBE_FLOOR.minor}, ADR-0037)`);
|
|
213
471
|
|
|
214
472
|
// --- ownership: the cluster is the record (ADR-0019) -------------------------------------------
|
|
215
473
|
const ns = await kube.getJson({ kind: "namespace", name: namespace, ...ctx });
|
|
@@ -238,8 +496,23 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
238
496
|
...ctx,
|
|
239
497
|
});
|
|
240
498
|
|
|
499
|
+
// --- pod-level resources (ADR-0060): the whole Size is one pod-level ceiling (KEP-2837) ------
|
|
500
|
+
// A server with the PodLevelResources gate off drops `spec.resources` silently, and every
|
|
501
|
+
// Sandbox runs Burstable. A server-side dry-run shows it; it needs the namespace, so it runs here.
|
|
502
|
+
try {
|
|
503
|
+
const admitted = await kube.dryRunCreate({ manifest: JSON.stringify(podResourcesProbe(namespace)), ...ctx });
|
|
504
|
+
if (admitted.spec?.resources === undefined) {
|
|
505
|
+
activity(
|
|
506
|
+
io,
|
|
507
|
+
`refusing: ${context} (Kubernetes ${serverVersion}) drops pod-level resources — the PodLevelResources feature gate is off, and a Workspace's Size cannot hold (ADR-0060). Enable the gate on the API server and kubelets. The namespace was created; nothing else was applied.`,
|
|
508
|
+
);
|
|
509
|
+
return 1;
|
|
510
|
+
}
|
|
511
|
+
} catch (e) {
|
|
512
|
+
activity(io, `warning: could not confirm pod-level resources on ${context}: ${(e as Error).message}`);
|
|
513
|
+
}
|
|
514
|
+
|
|
241
515
|
// --- image resolution (ADR-0038): decide every ref BEFORE any layer spends a build -----------
|
|
242
|
-
const build = io.build ?? pnpmDockerBuild;
|
|
243
516
|
const registry = config.registry;
|
|
244
517
|
const cluster = kindCluster(context);
|
|
245
518
|
|
|
@@ -295,6 +568,22 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
295
568
|
"`sandbox: { nodeSelector, tolerations }` in jr2.config.ts (raw pod-spec shapes, ADR-0052).",
|
|
296
569
|
);
|
|
297
570
|
}
|
|
571
|
+
// A Size no Sandbox node could hold today WARNS (ADR-0060, the ADR-0052 stance): the node set
|
|
572
|
+
// moves, and an autoscaler may bring a bigger node for the Pending pod. A node that reports no
|
|
573
|
+
// allocatable is an unknown, never a no. One line per distinct Size, naming its workflows.
|
|
574
|
+
const unheld = new Map<string, string[]>();
|
|
575
|
+
for (const { workflow, size } of sizes) {
|
|
576
|
+
if (candidates.length === 0 || anyNodeHolds(candidates, size) !== false) continue;
|
|
577
|
+
const key = `memory ${formatMemory(size.memory.bytes)}, cpu ${formatCpu(size.cpu.millis)}`;
|
|
578
|
+
unheld.set(key, [...new Set([...(unheld.get(key) ?? []), workflow])]);
|
|
579
|
+
}
|
|
580
|
+
for (const [key, names] of unheld) {
|
|
581
|
+
activity(
|
|
582
|
+
io,
|
|
583
|
+
`warning: no Sandbox node right now has the allocatable for a Size of ${key} ` +
|
|
584
|
+
`(workflow ${names.map((n) => `"${n}"`).join(", ")}) — its Sandbox stays Pending until one does`,
|
|
585
|
+
);
|
|
586
|
+
}
|
|
298
587
|
}
|
|
299
588
|
// A multi-platform build is `docker buildx build --push`: it delivers by pushing, and there is
|
|
300
589
|
// nowhere to push without a registry. By CONSTRUCTION this never fires — a mixed-arch cluster is
|
|
@@ -323,19 +612,29 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
323
612
|
activity(
|
|
324
613
|
io,
|
|
325
614
|
kitRoot
|
|
326
|
-
? `images: kit checkout at ${kitRoot} — building the Harness,
|
|
327
|
-
: `images: installed kit — the published v${KIT_VERSION} Harness,
|
|
615
|
+
? `images: kit checkout at ${kitRoot} — building the Harness, operator and Orchestrator from source`
|
|
616
|
+
: `images: installed kit — the published v${KIT_VERSION} Harness, operator and Orchestrator, pulled from ` +
|
|
328
617
|
`${config.kitRegistry ? `${config.kitRegistry} (kitRegistry)` : KIT_IMAGE_HOME}`,
|
|
329
618
|
);
|
|
619
|
+
// The Custodian's image is deployed and never built, in either world (ADR-0038, ADR-0059): the
|
|
620
|
+
// pinned upstream Envoy, re-homed onto `kitRegistry` like a Kit image when one is set.
|
|
621
|
+
const custodianImage = custodianRef(config.kitRegistry);
|
|
622
|
+
activity(io, `custodian image: ${custodianImage} (pinned, deployed — never built)`);
|
|
330
623
|
|
|
331
|
-
// Read the Orchestrator Deployment ONCE: it carries
|
|
332
|
-
//
|
|
624
|
+
// Read the Orchestrator Deployment ONCE: it carries the convergence record — the previous
|
|
625
|
+
// name→ref map (an annotation, ADR-0038), the Orchestrator's own Kit ref included (ADR-0069).
|
|
333
626
|
const orch = await kube.getJson({ kind: "deployment", name: ORCHESTRATOR_SERVICE, namespace, ...ctx });
|
|
334
627
|
const previous = previousImages(orch);
|
|
335
|
-
const converged: ConvergedImages = {
|
|
628
|
+
const converged: ConvergedImages = {
|
|
629
|
+
harness: refs.harness,
|
|
630
|
+
custodian: custodianImage,
|
|
631
|
+
orchestrator: refs.orchestrator,
|
|
632
|
+
sandbox: {},
|
|
633
|
+
sandboxUser: {},
|
|
634
|
+
};
|
|
336
635
|
|
|
337
|
-
// ONE transport branch for every image (ADR-0038), so
|
|
338
|
-
// drift into
|
|
636
|
+
// ONE transport branch for every image (ADR-0038), so Kit images and Sandbox Images cannot
|
|
637
|
+
// drift into two delivery stories.
|
|
339
638
|
const deliver = async (tag: string): Promise<void> => {
|
|
340
639
|
// A multi-platform build already pushed (ADR-0045): `docker buildx build --push` IS the
|
|
341
640
|
// delivery, and pushing again would push a tag the daemon never held. Sound at every call site
|
|
@@ -419,6 +718,17 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
419
718
|
return ref;
|
|
420
719
|
};
|
|
421
720
|
|
|
721
|
+
// --- priority (per-cluster, shared, ADR-0060) --------------------------------------------------
|
|
722
|
+
// The two PriorityClasses, beside the CRDs and before anything that names them: a pod whose class
|
|
723
|
+
// does not exist is refused at admission. Unconditional on `operator.manage` — the Instance's own
|
|
724
|
+
// pods name them too — and skipped per key the Instance's `priorityClasses` names.
|
|
725
|
+
const classes = priorityClassObjects(config.priorityClasses);
|
|
726
|
+
for (const manifest of classes) await kube.apply({ manifest, ...ctx });
|
|
727
|
+
const named = Object.entries(config.priorityClasses ?? {}).filter(([, v]) => v !== undefined);
|
|
728
|
+
if (named.length) {
|
|
729
|
+
activity(io, `priority: the Instance's own — ${named.map(([tier, cls]) => `${tier} "${cls}"`).join(", ")}`);
|
|
730
|
+
}
|
|
731
|
+
|
|
422
732
|
// --- operator (per-cluster, shared) ------------------------------------------------------------
|
|
423
733
|
// The operator IMAGE is resolved whenever anything this converge deploys runs it: the operator
|
|
424
734
|
// layer itself, or the data plane's cache agent, which is the same binary (`/manager repo-cache`,
|
|
@@ -445,7 +755,10 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
445
755
|
activity(io, `operator: leaving v${deployed} (newer than this kit's v${KIT_VERSION} — never downgraded)`);
|
|
446
756
|
} else {
|
|
447
757
|
activity(io, `operator: applying v${KIT_VERSION} (${image})${deployed ? ` over v${deployed}` : ""}`);
|
|
448
|
-
await kube.apply({
|
|
758
|
+
await kube.apply({
|
|
759
|
+
manifest: await operatorManifest(image, priorityClassNames(config.priorityClasses).control),
|
|
760
|
+
...ctx,
|
|
761
|
+
});
|
|
449
762
|
await kube.label({
|
|
450
763
|
kind: "deployment",
|
|
451
764
|
name: OPERATOR_DEPLOYMENT,
|
|
@@ -474,60 +787,20 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
474
787
|
}
|
|
475
788
|
|
|
476
789
|
// --- the kit's own runtime images (ADR-0038) ---------------------------------------------------
|
|
477
|
-
// Built here rather than lazily beside
|
|
478
|
-
// (ADR-0037):
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
// so a kit edit re-images future pods and re-tags nothing of the user's.
|
|
790
|
+
// Built here rather than lazily beside its consumers. The Harness is the injection source
|
|
791
|
+
// (ADR-0037): every Sandbox mounts this exact ref as an `image` volume, its `/opt/jr2` read-only
|
|
792
|
+
// at `/opt/jr2` in the Sandbox Image. It is not a hash input for a Sandbox Image — the runtime rides
|
|
793
|
+
// the pod's volume, so a kit edit re-images future pods and re-tags nothing of the user's.
|
|
482
794
|
await ensureKitImage("harness");
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
// Stage first, THEN decide: the hash is over the materialized bundle — the actual image inputs,
|
|
487
|
-
// kit included — so the tag is a content address rather than a guess about what changed.
|
|
488
|
-
const staged = await stageInstanceBundle(build, root);
|
|
489
|
-
// The content address covers (inputs × platform set), ADR-0045 — so the platform rides the HASH
|
|
490
|
-
// this layer records, not just the tag composed from it. That is what keeps the staleness key and
|
|
491
|
-
// the tag one fact: a converge whose platform set moved must not read the recorded hash as fresh,
|
|
492
|
-
// skip the build, and then apply a tag nothing ever delivered.
|
|
493
|
-
const hash = `${staged.hash}${platformSuffix(platforms)}`;
|
|
494
|
-
const tag = registry ? `${registry}/jr2-instance-${name}:${hash}` : `jr2-instance-${name}:${hash}`;
|
|
495
|
-
try {
|
|
496
|
-
if (orch?.metadata.labels?.[LABEL_HASH] === hash && values.force !== true) {
|
|
497
|
-
// Safe to skip only because the rolled-out pod is verified against `tag` below: this decides
|
|
498
|
-
// whether to spend a docker build, not whether the cluster is already correct.
|
|
499
|
-
activity(io, `image: ${tag} (fresh — build skipped; --force to rebuild anyway)`);
|
|
500
|
-
} else {
|
|
501
|
-
assertDeliverable();
|
|
502
|
-
if (await hostBuilt(tag)) {
|
|
503
|
-
activity(io, `image: ${tag} (host-built — build skipped, delivering; --force to rebuild)`);
|
|
504
|
-
await deliver(tag);
|
|
505
|
-
} else {
|
|
506
|
-
activity(io, `image: building ${tag}${values.force === true ? " (--force)" : ""}`);
|
|
507
|
-
await ensureEmulation();
|
|
508
|
-
await build.build({
|
|
509
|
-
tag,
|
|
510
|
-
platforms,
|
|
511
|
-
context: staged.dir,
|
|
512
|
-
dockerfileContent: INSTANCE_DOCKERFILE,
|
|
513
|
-
labels: instanceImageLabels(name),
|
|
514
|
-
});
|
|
515
|
-
await deliver(tag);
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
} finally {
|
|
519
|
-
await staged.dispose();
|
|
520
|
-
}
|
|
795
|
+
// The Orchestrator is a Kit image too (ADR-0069): the engine and its closure, nothing of this
|
|
796
|
+
// Instance. A Machine edit moves the Assembly, never this ref, so it never rolls the pod.
|
|
797
|
+
await ensureKitImage("orchestrator");
|
|
521
798
|
|
|
522
799
|
// --- Sandbox Images (ADR-0037/0049): the contexts the Machines carry, plus `images/default` -----
|
|
523
800
|
// Gated on the data-plane switch (ADR-0012/0031/0051): an instance none of whose Machines
|
|
524
801
|
// composes a Sandbox has no Sandboxes, so it must not pay a docker build for an image it can
|
|
525
802
|
// never use. Said out loud, because a silent skip of a folder you just wrote reads as a bug.
|
|
526
803
|
if (carried.composesSandbox) {
|
|
527
|
-
// The scaffolded `images/default` is a PATH CONVENTION, not discovery (ADR-0049/0050): exactly
|
|
528
|
-
// one path is checked, and it is keyed `default` rather than by digest, because that reserved
|
|
529
|
-
// key IS the middle leg of the resolution chain a `workspace()` that names no image lands on.
|
|
530
|
-
const fallback = await defaultImageContext(root);
|
|
531
804
|
const built: Array<{ dir: string; name: string; reserved?: "default" }> = [
|
|
532
805
|
...(fallback ? [{ dir: fallback, name: "default", reserved: "default" as const }] : []),
|
|
533
806
|
...contexts.map((image: CarriedImage) => ({ dir: image.dir, name: image.name })),
|
|
@@ -536,20 +809,13 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
536
809
|
activity(io, "sandbox images: none carried (a workspace() names one, or scaffold images/default)");
|
|
537
810
|
}
|
|
538
811
|
for (const image of built) {
|
|
539
|
-
//
|
|
540
|
-
// context
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
`the Sandbox Image context ${image.dir} cannot be read (${err.code ?? "read failed"}) — a ` +
|
|
544
|
-
'`workspace()` names its image with `import.meta.resolve("./<dir>")`, so that directory must exist ' +
|
|
545
|
-
"beside the Machine's own module (ADR-0037/0049).",
|
|
546
|
-
);
|
|
547
|
-
});
|
|
548
|
-
// Every carried context is keyed by that digest, which is how the baked Orchestrator finds
|
|
549
|
-
// this ref again from the `file:` URL its own copy of the module carries — no path table on
|
|
550
|
-
// either side (ADR-0049). The TAG carries the digest either way, so a Machine that names
|
|
812
|
+
// The digest of the context's staged copy, taken above with the Assembly: every carried
|
|
813
|
+
// context is keyed by it, which is how the Orchestrator finds this ref again from the
|
|
814
|
+
// `file:` URL the Assembly's copy of the module carries — no path table on either side
|
|
815
|
+
// (ADR-0049, ADR-0069). The TAG carries the digest either way, so a Machine that names
|
|
551
816
|
// `images/default` by URL lands on the same tag as the reserved key: two records, one build
|
|
552
817
|
// (the host-held check below spends nothing the second time).
|
|
818
|
+
const digest = contextDigests.get(image.dir)!;
|
|
553
819
|
const key = image.reserved ?? digest;
|
|
554
820
|
const ref = sandboxImageTag(name, image.name, digest, { platforms, registry });
|
|
555
821
|
// A record is only worth skipping on when it is COMPLETE: the ref AND the seat the image
|
|
@@ -558,7 +824,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
558
824
|
// as stale here and the image is re-inspected (the host skip below then spends no build).
|
|
559
825
|
const remembered = previous?.sandboxUser?.[key];
|
|
560
826
|
if (previous?.sandbox?.[key] === ref && remembered !== undefined && values.force !== true) {
|
|
561
|
-
// Unlike
|
|
827
|
+
// Unlike a Kit image there is no rollout to verify a Sandbox Image against, so a
|
|
562
828
|
// recorded-but-absent ref only shows up as ImagePullBackOff at the next provision —
|
|
563
829
|
// `--force` rebuilds and re-delivers it. The seat is carried forward from the same record:
|
|
564
830
|
// the tag is a content address, so the image the record names declares what it declared.
|
|
@@ -609,75 +875,205 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
609
875
|
const keep = (key: string, mint: () => string): string =>
|
|
610
876
|
secret?.data?.[key] ? Buffer.from(secret.data[key], "base64").toString("utf8") : mint();
|
|
611
877
|
const instanceToken = keep("JR2_INSTANCE_TOKEN", () => randomBytes(24).toString("hex"));
|
|
878
|
+
// The publish token (ADR-0069): publishing an Assembly imports code into the process that holds
|
|
879
|
+
// the ServiceAccount, so it costs what deploying an image cost — cluster write access. It lives
|
|
880
|
+
// here beside the Instance token, and the publish below reads it from THIS Secret alone: no env
|
|
881
|
+
// var, flag or `--url` path can supply it, so an ingress-exposed Instance exposes no publish door.
|
|
882
|
+
const publishToken = keep("JR2_PUBLISH_TOKEN", () => randomBytes(24).toString("hex"));
|
|
612
883
|
const signingKey = keep("JR2_SIGNING_KEY", () => randomBytes(32).toString("base64"));
|
|
613
884
|
const secretData: Record<string, string> = {
|
|
614
885
|
JR2_INSTANCE_TOKEN: instanceToken,
|
|
886
|
+
JR2_PUBLISH_TOKEN: publishToken,
|
|
615
887
|
JR2_SIGNING_KEY: signingKey,
|
|
616
|
-
// The Instance Harness
|
|
888
|
+
// The Instance Harness Custodian's credential (ADR-0013/0031): a sandbox-style token signed
|
|
617
889
|
// for the placement's name — it may deliver only to Turns hosted THERE (tokens.ts), so the
|
|
618
890
|
// Instance token never enters that pod. Derived from the kept key, so re-runs converge to
|
|
619
|
-
// the same value; live
|
|
891
|
+
// the same value; live Custodians keep verifying.
|
|
620
892
|
JR2_INSTANCE_HARNESS_TOKEN: sandboxToken(Buffer.from(signingKey, "base64"), INSTANCE_HARNESS_SERVICE),
|
|
621
893
|
};
|
|
622
|
-
// Git tokens (ADR-
|
|
623
|
-
//
|
|
624
|
-
// Orchestrator
|
|
625
|
-
//
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
}
|
|
894
|
+
// Git tokens (ADR-0051, ADR-0068): each entry's token, from `.env`, goes into that entry's own clone
|
|
895
|
+
// Secret (`jr2-git-<hash>`, Flux's key names), which the cache agent reads — never into this
|
|
896
|
+
// Secret, the Orchestrator's env. The Orchestrator names the Secret on a Repo and holds no git token.
|
|
897
|
+
// Only when a registered Machine composes a Sandbox: nothing else clones.
|
|
898
|
+
const gitClone = carried.composesSandbox ? gitCloneSecretsOf(config.git?.credentials ?? [], (n) => io.env[n]) : [];
|
|
899
|
+
for (const s of gitClone) activity(io, `git.credentials "${s.match}": clone Secret ${s.name}`);
|
|
629
900
|
|
|
630
901
|
// The HARNESS containers' env — a separate Secret (ADR-0013): Agent code executes where these
|
|
631
902
|
// land, so the Instance token/signing key above must be unreachable from it. Values declared in
|
|
632
|
-
// config (usually read off process.env/.env) materialize here (ADR-0019)
|
|
633
|
-
//
|
|
903
|
+
// config (usually read off process.env/.env) materialize here (ADR-0019). The provider key does
|
|
904
|
+
// NOT: it is a held secret (ADR-0059), and the Agent can read everything in this Secret.
|
|
634
905
|
const harnessEnvData: Record<string, string> = {};
|
|
635
906
|
for (const v of config.harness?.env ?? []) if (v.value !== undefined) harnessEnvData[v.name] = v.value;
|
|
636
|
-
if (config.harness?.provider?.apiKey) harnessEnvData.JR2_PROVIDER_API_KEY = config.harness.provider.apiKey;
|
|
637
907
|
|
|
638
908
|
// Preflight referenced-but-unmanaged Secrets: turn the CreateContainerConfigError hang into an
|
|
639
|
-
// immediate, named error (ADR-0019). Sealed/External Secrets ride this seam untouched.
|
|
909
|
+
// immediate, named error (ADR-0019). Sealed/External Secrets ride this seam untouched. The read
|
|
910
|
+
// keeps each one's key NAMES (never a value), which the held-secret rules check for a model key.
|
|
911
|
+
const envFromKeys = new Map<string, string[]>();
|
|
640
912
|
for (const ref of config.harness?.envFrom ?? []) {
|
|
641
|
-
const refName = ref.secretRef?.name;
|
|
913
|
+
const refName = ref.secretRef?.name ?? ref.configMapRef?.name;
|
|
642
914
|
if (!refName) continue;
|
|
643
|
-
|
|
915
|
+
const kind = ref.secretRef ? "secret" : "configmap";
|
|
916
|
+
const found = await kube.getJson<KubeObject & { data?: Record<string, string> }>({
|
|
917
|
+
kind,
|
|
918
|
+
name: refName,
|
|
919
|
+
namespace,
|
|
920
|
+
...ctx,
|
|
921
|
+
});
|
|
922
|
+
if (!found && kind === "secret") {
|
|
644
923
|
throw new Error(
|
|
645
924
|
`harness.envFrom references Secret "${refName}", which does not exist in namespace "${namespace}" — ` +
|
|
646
925
|
`create it first: kubectl -n ${namespace} create secret generic ${refName} --from-literal=KEY=...`,
|
|
647
926
|
);
|
|
648
927
|
}
|
|
928
|
+
envFromKeys.set(`${kind}/${refName}`, Object.keys(found?.data ?? {}));
|
|
649
929
|
}
|
|
650
930
|
|
|
651
931
|
// --- private-CA bundle (ADR-0020): read HERE, host-side — the in-cluster config eval never
|
|
652
932
|
// touches the file (the path may not exist there); consumers get the ConfigMap.
|
|
653
933
|
const caPem = await readCaBundle(root, config);
|
|
654
934
|
|
|
935
|
+
// --- secrets (ADR-0059, ADR-0068): resolved HERE, host-side, where `.env` exists --------------
|
|
936
|
+
// R10: a Secret an entry references must exist with its key. Read once per Secret — each key's
|
|
937
|
+
// NAME, the LENGTH of its value, and a keyed digest of it, never the value — for both
|
|
938
|
+
// placements. The length is ADR-0068's: an Effect secret shorter than 8 characters is refused
|
|
939
|
+
// wherever its value is kept (ADR-0068). The digest goes into the template digest of each
|
|
940
|
+
// long-lived pod that reads the value once at start — the Instance Harness's Custodian and the
|
|
941
|
+
// Instance Effector — so a value edited in the user's Secret rolls them at the next `jr2 up`.
|
|
942
|
+
const referenced = new Map<string, Map<string, number> | undefined>();
|
|
943
|
+
const valueDigests = new Map<string, string>();
|
|
944
|
+
for (const entry of config.secrets ?? []) {
|
|
945
|
+
const ref = entry.valueFrom?.secretKeyRef.name;
|
|
946
|
+
if (ref === undefined || referenced.has(ref)) continue;
|
|
947
|
+
const found = await kube.getJson<KubeObject & { data?: Record<string, string> }>({
|
|
948
|
+
kind: "secret",
|
|
949
|
+
name: ref,
|
|
950
|
+
namespace,
|
|
951
|
+
...ctx,
|
|
952
|
+
});
|
|
953
|
+
referenced.set(
|
|
954
|
+
ref,
|
|
955
|
+
found ? new Map(Object.entries(found.data ?? {}).map(([k, b64]) => [k, storedValueLength(b64)])) : undefined,
|
|
956
|
+
);
|
|
957
|
+
for (const [k, b64] of Object.entries(found?.data ?? {})) {
|
|
958
|
+
valueDigests.set(`${ref}/${k}`, storedValueDigest(Buffer.from(signingKey, "base64"), ref, k, b64));
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
const valueDigest = (secret: string, key: string) => valueDigests.get(`${secret}/${key}`);
|
|
962
|
+
const secretKeys = (secret: string) => {
|
|
963
|
+
const keys = referenced.get(secret);
|
|
964
|
+
return keys && [...keys.keys()];
|
|
965
|
+
};
|
|
966
|
+
const valueLength = (secret: string, key: string) => referenced.get(secret)?.get(key);
|
|
967
|
+
// Where each secret goes (ADR-0059, ADR-0068): read off the list and what names it, never
|
|
968
|
+
// declared — each Harness pod gets the Held secrets something in it names, the Instance Effector
|
|
969
|
+
// what its Effects name. The walk's refusals ran above; a `valueFrom` the user's Secrets do not
|
|
970
|
+
// hold is refused here, before the instance objects are applied.
|
|
971
|
+
const placements = secretPlacementsOf(config, placementWalk, { secretKeys, valueLength });
|
|
972
|
+
for (const line of placements.report) activity(io, line);
|
|
973
|
+
for (const warning of placements.warnings) activity(io, `warning: ${warning}`);
|
|
974
|
+
const held = await resolveHeld(io, kube, {
|
|
975
|
+
pods: placements.held,
|
|
976
|
+
claimed: placements.claimed,
|
|
977
|
+
config,
|
|
978
|
+
name,
|
|
979
|
+
namespace,
|
|
980
|
+
ctx,
|
|
981
|
+
envFromKeys,
|
|
982
|
+
secretKeys,
|
|
983
|
+
// Every bound Agent's model, and every model an Open body allows a Chart's own Agents (ADR-0070).
|
|
984
|
+
models: carried.models,
|
|
985
|
+
caBundle: caPem,
|
|
986
|
+
});
|
|
987
|
+
|
|
655
988
|
// --- git over ssh: where the key comes from (ADR-0019, ADR-0047) -------------------------------
|
|
656
989
|
// Runs BEFORE the apply, because a missing key source is a converge that must not start, and the
|
|
657
990
|
// notice it hands back is owed to the END of the converge — a generated key is dead until a human
|
|
658
991
|
// registers it, and by then this line has scrolled away.
|
|
659
992
|
const gitSsh = await ensureGitSsh(io, kube, config, carried.repos, namespace, ctx, values.yes === true);
|
|
660
993
|
|
|
994
|
+
// --- what each Sandbox Effector's image owes its pushes (ADR-0068) ------------------------
|
|
995
|
+
// Before anything is applied: an image that lacks a push form's tools, or the passwd entry ssh
|
|
996
|
+
// needs, is refused here with the fix, not at the first push in a pod.
|
|
997
|
+
await checkEffectorImages(build, config, placements.sandbox, converged, platforms, contextDigests);
|
|
998
|
+
|
|
999
|
+
// --- push credentials (ADR-0051): one Secret per git.credentials entry, never per pod ----
|
|
1000
|
+
// After the deploy keys exist (a generated one included), before anything else is applied: a
|
|
1001
|
+
// static Binding whose entry cannot push is refused here.
|
|
1002
|
+
const gitPush = await resolveGitPush(io, kube, config, placements.sandbox, namespace, ctx);
|
|
1003
|
+
|
|
661
1004
|
// --- provider preflight (ADR-0019): probe the endpoint FROM INSIDE the cluster ----------------
|
|
662
|
-
await preflightProvider(io, kube, config,
|
|
1005
|
+
await preflightProvider(io, kube, config, carried.models, namespace, ctx, caPem);
|
|
663
1006
|
|
|
664
1007
|
// --- apply + rollout ---------------------------------------------------------------------------
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
1008
|
+
// The trust bundles first, and server-side: every pod that mounts them must find them, and they
|
|
1009
|
+
// carry every root certificate — past what client-side apply's annotation can hold.
|
|
1010
|
+
const trust = trustObject({ name, namespace, caBundle: caPem, bundles: held.bundles });
|
|
1011
|
+
if (trust) await kube.apply({ manifest: trust, serverSide: true, ...ctx });
|
|
1012
|
+
activity(io, `orchestrator: applying (image ${refs.orchestrator})`);
|
|
1013
|
+
const instanceList = instanceObjects({
|
|
1014
|
+
name,
|
|
1015
|
+
namespace,
|
|
1016
|
+
image: refs.orchestrator,
|
|
1017
|
+
secretData,
|
|
1018
|
+
harnessEnvData,
|
|
1019
|
+
harness: config.harness,
|
|
1020
|
+
held: held.objects,
|
|
1021
|
+
imageRefs: converged,
|
|
1022
|
+
repoCache: carried.composesSandbox ? { image: operatorImage!, placement: config.sandbox } : undefined,
|
|
1023
|
+
...(config.priorityClasses ? { priorityClasses: config.priorityClasses } : {}),
|
|
1024
|
+
sandboxEffectors: placements.sandbox,
|
|
1025
|
+
gitPush,
|
|
1026
|
+
gitClone,
|
|
680
1027
|
});
|
|
1028
|
+
await kube.apply({ manifest: instanceList, ...ctx });
|
|
1029
|
+
// An entry whose clone Secret this converge did not write loses it: a dropped entry, a token
|
|
1030
|
+
// unset, or nothing left that clones.
|
|
1031
|
+
const cloneSecrets = new Set(gitClone.map((s) => s.name));
|
|
1032
|
+
for (const secret of await kube.listJson<KubeObject>({
|
|
1033
|
+
kind: "secret",
|
|
1034
|
+
selector: LABEL_GIT_CLONE,
|
|
1035
|
+
namespace,
|
|
1036
|
+
...ctx,
|
|
1037
|
+
})) {
|
|
1038
|
+
if (cloneSecrets.has(secret.metadata.name)) continue;
|
|
1039
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1040
|
+
}
|
|
1041
|
+
// An entry whose push Secret this converge did not write loses it: a dropped entry, a token
|
|
1042
|
+
// unset, or no push left anywhere.
|
|
1043
|
+
const pushSecrets = new Set(gitPush.secrets.map((s) => s.name));
|
|
1044
|
+
for (const secret of await kube.listJson<KubeObject>({
|
|
1045
|
+
kind: "secret",
|
|
1046
|
+
selector: LABEL_GIT_PUSH,
|
|
1047
|
+
namespace,
|
|
1048
|
+
...ctx,
|
|
1049
|
+
})) {
|
|
1050
|
+
if (pushSecrets.has(secret.metadata.name)) continue;
|
|
1051
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1052
|
+
}
|
|
1053
|
+
// A Workspace whose body no longer places an Effect in its Sandbox Effector loses its Secret;
|
|
1054
|
+
// its manifest went with the ConfigMap's data above, so the next provision adds no container.
|
|
1055
|
+
const effectorPods = new Set(placements.sandbox.map((p) => p.pod));
|
|
1056
|
+
for (const secret of await kube.listJson<KubeObject>({
|
|
1057
|
+
kind: "secret",
|
|
1058
|
+
selector: LABEL_EFFECTOR_POD,
|
|
1059
|
+
namespace,
|
|
1060
|
+
...ctx,
|
|
1061
|
+
})) {
|
|
1062
|
+
if (effectorPods.has(secret.metadata.labels?.[LABEL_EFFECTOR_POD] ?? "")) continue;
|
|
1063
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1064
|
+
}
|
|
1065
|
+
// A pod's literal values go when it holds none any more, or when the pod is gone; with no secret
|
|
1066
|
+
// held anywhere, the leaves go too. The CA stays (`jr2-held-ca`), so a later converge that holds
|
|
1067
|
+
// one again issues leaves under the same CA.
|
|
1068
|
+
const literalPods = new Set(held.objects.pods.filter((p) => Object.keys(p.values).length).map((p) => p.pod));
|
|
1069
|
+
const stale = await kube.listJson<KubeObject>({ kind: "secret", selector: LABEL_HELD_POD, namespace, ...ctx });
|
|
1070
|
+
for (const secret of stale) {
|
|
1071
|
+
if (literalPods.has(secret.metadata.labels?.[LABEL_HELD_POD] ?? "")) continue;
|
|
1072
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1073
|
+
}
|
|
1074
|
+
if (!holdsAny(held.objects)) {
|
|
1075
|
+
await kube.deleteObject({ kind: "secret", name: HELD_TLS_SECRET, namespace, ...ctx });
|
|
1076
|
+
}
|
|
681
1077
|
activity(io, "orchestrator: waiting for rollout");
|
|
682
1078
|
await awaitRollout(kube, {
|
|
683
1079
|
name: ORCHESTRATOR_SERVICE,
|
|
@@ -689,7 +1085,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
689
1085
|
layer: "orchestrator",
|
|
690
1086
|
namespace,
|
|
691
1087
|
selector: `app=${ORCHESTRATOR_SERVICE}`,
|
|
692
|
-
image:
|
|
1088
|
+
image: refs.orchestrator,
|
|
693
1089
|
...ctx,
|
|
694
1090
|
});
|
|
695
1091
|
|
|
@@ -719,8 +1115,8 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
719
1115
|
});
|
|
720
1116
|
} else {
|
|
721
1117
|
// The Repo resources are NOT deleted here: eviction is reachability plus age (ADR-0051), and
|
|
722
|
-
// the
|
|
723
|
-
// of them (server.ts) and `jr2 gc` takes them at the TTL. The bare clones under
|
|
1118
|
+
// the Assembly this converge publishes binds nothing, so the Orchestrator's Repo pass at that
|
|
1119
|
+
// publish unlabels every one of them (server.ts, ADR-0069) and `jr2 gc` takes them at the TTL. The bare clones under
|
|
724
1120
|
// `/var/lib/jr2/<ns>/repos` outlive the agent that would evict them — inert, like the ones
|
|
725
1121
|
// `jr2 down` leaves (down.ts), and swept by the agent's own walk if a `workspace()` ever
|
|
726
1122
|
// returns here.
|
|
@@ -733,34 +1129,54 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
733
1129
|
// The scan is the Machine walk above, and it is DEFINITION-level (the line ADR-0018 drew:
|
|
734
1130
|
// workflow internals are not statically recoverable) — a carried-but-never-invoked `"none"`
|
|
735
1131
|
// Agent over-deploys, erring toward "the convention works when you need it". No `"none"`
|
|
736
|
-
// definitions → nothing, and a stale
|
|
737
|
-
// deleted: the layer converges toward the Machines like every other layer
|
|
738
|
-
// config.
|
|
1132
|
+
// definitions → nothing, and a stale StatefulSet from a definition that dropped its `"none"` is
|
|
1133
|
+
// deleted with its conversations: the layer converges toward the Machines like every other layer
|
|
1134
|
+
// converges toward the config.
|
|
739
1135
|
const menuOnly = [...new Set(agents.filter((a) => a.definition.workspace === "none").map((a) => a.name))];
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
1136
|
+
// A Chart wrapper whose Open body allows "none" needs it too: a Chart's own Agent may be Menu-only (ADR-0070).
|
|
1137
|
+
const noneWrappers = workflows
|
|
1138
|
+
.filter((w) => partsOf([w.machine]).openBodies.some((b) => b.declaration.allows.workspace.includes("none")))
|
|
1139
|
+
.map((w) => w.name);
|
|
1140
|
+
if (carried.instanceHarness !== undefined) {
|
|
1141
|
+
const why = [
|
|
1142
|
+
...(menuOnly.length
|
|
1143
|
+
? [`${menuOnly.join(", ")} declare${menuOnly.length === 1 ? "s" : ""} workspace: "none"`]
|
|
1144
|
+
: []),
|
|
1145
|
+
...(noneWrappers.length
|
|
1146
|
+
? [
|
|
1147
|
+
`chart wrapper${noneWrappers.length === 1 ? "" : "s"} ${noneWrappers.map((n) => `"${n}"`).join(", ")} allow${noneWrappers.length === 1 ? "s" : ""} workspace: "none"`,
|
|
1148
|
+
]
|
|
1149
|
+
: []),
|
|
1150
|
+
];
|
|
1151
|
+
activity(io, `instance harness: converging (${why.join("; ")})`);
|
|
745
1152
|
const harnessImage = refs.harness;
|
|
1153
|
+
const instance = instanceHeld(held.objects);
|
|
746
1154
|
await kube.apply({
|
|
747
1155
|
manifest: instanceHarnessObjects({
|
|
748
1156
|
name,
|
|
749
1157
|
namespace,
|
|
750
1158
|
harnessImage,
|
|
751
|
-
|
|
1159
|
+
custodianImage,
|
|
1160
|
+
held: instance.manifest,
|
|
1161
|
+
custodian: instance.custodian,
|
|
1162
|
+
heldDigest: heldDigest(Buffer.from(signingKey, "base64"), held.objects, instance, valueDigest),
|
|
752
1163
|
harness: config.harness,
|
|
753
1164
|
caBundle: caPem !== undefined,
|
|
754
1165
|
// The wire's gate (ADR-0058): the digest of the bearer the Orchestrator derives for this
|
|
755
1166
|
// placement from the kept key — the same shape it stamps onto every Sandbox at provision.
|
|
756
1167
|
bearerSha256: harnessTokenDigest(Buffer.from(signingKey, "base64"), INSTANCE_HARNESS_SERVICE),
|
|
1168
|
+
...(config.priorityClasses ? { priorityClasses: config.priorityClasses } : {}),
|
|
757
1169
|
}),
|
|
758
1170
|
...ctx,
|
|
759
1171
|
});
|
|
760
1172
|
await awaitRollout(kube, {
|
|
1173
|
+
kind: "statefulset",
|
|
761
1174
|
name: INSTANCE_HARNESS_SERVICE,
|
|
762
1175
|
namespace,
|
|
763
1176
|
selector: `app=${INSTANCE_HARNESS_SERVICE}`,
|
|
1177
|
+
// A rollout waits out the old pod's drain (ADR-0031): up to a Turn's worst case, then the
|
|
1178
|
+
// new pod's start.
|
|
1179
|
+
timeoutSeconds: INSTANCE_HARNESS_DRAIN_SECONDS + 180,
|
|
764
1180
|
...ctx,
|
|
765
1181
|
});
|
|
766
1182
|
await verifyRunningImage(io, kube, {
|
|
@@ -772,12 +1188,83 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
772
1188
|
...ctx,
|
|
773
1189
|
});
|
|
774
1190
|
} else {
|
|
775
|
-
await kube.deleteObject({ kind: "
|
|
1191
|
+
await kube.deleteObject({ kind: "statefulset", name: INSTANCE_HARNESS_SERVICE, namespace, ...ctx });
|
|
1192
|
+
// The claim's retention policy deletes it with the StatefulSet; deleted here too, so the
|
|
1193
|
+
// conversations go even from a StatefulSet the controller never got to.
|
|
1194
|
+
await kube.deleteObject({ kind: "persistentvolumeclaim", name: INSTANCE_HARNESS_PVC, namespace, ...ctx });
|
|
776
1195
|
await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_SERVICE, namespace, ...ctx });
|
|
1196
|
+
await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_PODS_SERVICE, namespace, ...ctx });
|
|
1197
|
+
// Its Custodian's files go with the pod they were rendered for (ADR-0059).
|
|
1198
|
+
await kube.deleteObject({ kind: "configmap", name: INSTANCE_CUSTODIAN_CONFIGMAP, namespace, ...ctx });
|
|
777
1199
|
}
|
|
778
1200
|
|
|
1201
|
+
// --- Instance Effector (ADR-0068): converged when an Effect is placed there, absent otherwise ---
|
|
1202
|
+
// The placement is read off the Machines: an `http` Effect runs there. Its objects carry only the
|
|
1203
|
+
// secrets those Effects name, and the placement's key, derived from the kept signing key — the
|
|
1204
|
+
// key the Orchestrator derives again to sign every call and verify every answer.
|
|
1205
|
+
const effectorPlan = placements.instance;
|
|
1206
|
+
if (effectorPlan) {
|
|
1207
|
+
const names = effectorPlan.secrets.map((s) => `\`${s.name}\``).join(", ") || "none";
|
|
1208
|
+
activity(
|
|
1209
|
+
io,
|
|
1210
|
+
`instance effector: converging (${effectorPlan.effects.join(", ")}; secrets ${names}; grace ` +
|
|
1211
|
+
`${effectorGraceSeconds(effectorPlan)}s)`,
|
|
1212
|
+
);
|
|
1213
|
+
const signing = Buffer.from(signingKey, "base64");
|
|
1214
|
+
const key = effectorKey(signing, "instance");
|
|
1215
|
+
await kube.apply({
|
|
1216
|
+
manifest: instanceEffectorObjects({
|
|
1217
|
+
name,
|
|
1218
|
+
namespace,
|
|
1219
|
+
harnessImage: refs.harness,
|
|
1220
|
+
plan: effectorPlan,
|
|
1221
|
+
key,
|
|
1222
|
+
digest: effectorDigest(signing, effectorPlan, key, valueDigest),
|
|
1223
|
+
caBundle: caPem !== undefined,
|
|
1224
|
+
...(config.priorityClasses ? { priorityClasses: config.priorityClasses } : {}),
|
|
1225
|
+
}),
|
|
1226
|
+
...ctx,
|
|
1227
|
+
});
|
|
1228
|
+
await awaitRollout(kube, {
|
|
1229
|
+
name: INSTANCE_EFFECTOR,
|
|
1230
|
+
namespace,
|
|
1231
|
+
selector: `app=${INSTANCE_EFFECTOR}`,
|
|
1232
|
+
// A rollout waits out the old replica's drain: the longest Effect, and its read.
|
|
1233
|
+
timeoutSeconds: effectorGraceSeconds(effectorPlan) + 180,
|
|
1234
|
+
...ctx,
|
|
1235
|
+
});
|
|
1236
|
+
await verifyRunningImage(io, kube, {
|
|
1237
|
+
layer: "instance effector",
|
|
1238
|
+
namespace,
|
|
1239
|
+
selector: `app=${INSTANCE_EFFECTOR}`,
|
|
1240
|
+
image: refs.harness,
|
|
1241
|
+
container: "effector",
|
|
1242
|
+
...ctx,
|
|
1243
|
+
});
|
|
1244
|
+
} else {
|
|
1245
|
+
activity(io, "instance effector: none (no Effect is placed there)");
|
|
1246
|
+
for (const kind of ["deployment", "service", "networkpolicy", "secret"]) {
|
|
1247
|
+
await kube.deleteObject({ kind, name: INSTANCE_EFFECTOR, namespace, ...ctx });
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
// --- the Assembly (ADR-0069): published last of the layers, before the sweep -------------------
|
|
1252
|
+
// After every layer, because the door's checks are true only once the cluster work is done: the
|
|
1253
|
+
// image map names every `file:` digest, the Secrets exist, and the rollout above put the Kit
|
|
1254
|
+
// version the Assembly names in front of the door. A refusal leaves the previous Assembly
|
|
1255
|
+
// current, and the converge fails before the sweep, so nothing it still needs is collected.
|
|
1256
|
+
const published = await publishAssembly(io, {
|
|
1257
|
+
namespace,
|
|
1258
|
+
ctx,
|
|
1259
|
+
assembly,
|
|
1260
|
+
token: publishToken,
|
|
1261
|
+
force: values.force === true,
|
|
1262
|
+
mounts: mountedDigests(instanceList),
|
|
1263
|
+
});
|
|
1264
|
+
if (!published) return 1;
|
|
1265
|
+
|
|
779
1266
|
await reportOlderWorkspaces(io, kube, namespace, ctx, converged);
|
|
780
|
-
await sweepAfterConverge(io, { build, kube, context, ctx, converged,
|
|
1267
|
+
await sweepAfterConverge(io, { build, kube, context, ctx, converged, previous });
|
|
781
1268
|
noteDeferred(io, carried.repos);
|
|
782
1269
|
activity(io, `converged — \`jr2 run <workflow>\` when ready`);
|
|
783
1270
|
// LAST, after the success line (ADR-0047): the converge succeeded and the Repos still cannot be
|
|
@@ -787,8 +1274,8 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
787
1274
|
}
|
|
788
1275
|
|
|
789
1276
|
/**
|
|
790
|
-
* Every image ref one converge resolved. The Orchestrator reads `harness`/`
|
|
791
|
-
* the mounted map (`ImageRefs`); `operator`
|
|
1277
|
+
* Every image ref one converge resolved. The Orchestrator reads `harness`/`custodian`/`sandbox` from
|
|
1278
|
+
* the mounted map (`ImageRefs`); `operator` and `orchestrator` ride the same JSON because the record `jr2 up` diffs
|
|
792
1279
|
* must cover every image it builds, and one map is what keeps the record it diffs and the map pods
|
|
793
1280
|
* read from ever disagreeing (ADR-0038).
|
|
794
1281
|
*
|
|
@@ -802,7 +1289,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
802
1289
|
* never inspected, never in this map — so it runs as whatever its own `USER` says, and one that
|
|
803
1290
|
* would run as root fails the Harness container's `runAsNonRoot` at provision.
|
|
804
1291
|
*/
|
|
805
|
-
type ConvergedImages = ImageRefs & { operator?: string; sandboxUser: Record<string, string> };
|
|
1292
|
+
type ConvergedImages = ImageRefs & { operator?: string; orchestrator?: string; sandboxUser: Record<string, string> };
|
|
806
1293
|
|
|
807
1294
|
/** The map the LAST converge recorded, off the Orchestrator Deployment's annotation. Anything
|
|
808
1295
|
* unreadable (absent, hand-edited, a foreign shape) reads as "no record", which costs a rebuild —
|
|
@@ -820,11 +1307,177 @@ function previousImages(orch?: KubeObject): Partial<ConvergedImages> | undefined
|
|
|
820
1307
|
|
|
821
1308
|
/** Every ref one image map names, flattened — the kit's own plus each Sandbox Image. */
|
|
822
1309
|
function mapRefs(map: Partial<ConvergedImages>): string[] {
|
|
823
|
-
return [map.harness, map.
|
|
1310
|
+
return [map.harness, map.custodian, map.operator, map.orchestrator, ...Object.values(map.sandbox ?? {})].filter(
|
|
824
1311
|
(ref): ref is string => typeof ref === "string",
|
|
825
1312
|
);
|
|
826
1313
|
}
|
|
827
1314
|
|
|
1315
|
+
/** What `GET /assembly` answers (ADR-0069), read loosely: this is the CLI's view of the door.
|
|
1316
|
+
* `mounts` is the digest of each ConfigMap the process reads through a mount. */
|
|
1317
|
+
type AssemblyView = { current?: { hash?: string; kitVersion?: string } | null; mounts?: Record<string, string> };
|
|
1318
|
+
|
|
1319
|
+
/** The digest of each mounted ConfigMap's data in the List this converge applied (ADR-0069), by
|
|
1320
|
+
* name: what the Orchestrator must read from its mounts before the publish. */
|
|
1321
|
+
function mountedDigests(list: string): Record<string, string> {
|
|
1322
|
+
const items = (JSON.parse(list) as { items: Array<KubeObject & { data?: Record<string, string> }> }).items;
|
|
1323
|
+
const out: Record<string, string> = {};
|
|
1324
|
+
for (const item of items) {
|
|
1325
|
+
if (item.kind === "ConfigMap" && item.metadata.name in MOUNTED_CONFIGMAPS) {
|
|
1326
|
+
out[item.metadata.name] = configMapDigest(item.data ?? {});
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
return out;
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
/**
|
|
1333
|
+
* The key of each `file:` Sandbox Image context (ADR-0037, ADR-0049, ADR-0069): the digest of its
|
|
1334
|
+
* copy in the staged Assembly at `stagedDir`, by the host directory the walk named. A context the
|
|
1335
|
+
* Assembly does not carry is refused by name: a `file:` URL that names no folder, or a folder the
|
|
1336
|
+
* Instance's install leaves out, is checkable HERE and nowhere later, since the pod only ever
|
|
1337
|
+
* resolves a digest.
|
|
1338
|
+
*/
|
|
1339
|
+
async function stagedContextDigests(stagedDir: string, root: string, dirs: string[]): Promise<Map<string, string>> {
|
|
1340
|
+
const digests = new Map<string, string>();
|
|
1341
|
+
for (const dir of new Set(dirs)) {
|
|
1342
|
+
const copy = await stagedCopyOf(stagedDir, root, dir);
|
|
1343
|
+
const shown = relative(root, dir).startsWith("..") ? dir : relative(root, dir);
|
|
1344
|
+
if (copy === undefined) {
|
|
1345
|
+
throw new Error(
|
|
1346
|
+
`the Sandbox Image context ${shown} is not in the staged Assembly — a \`workspace()\` names its image with ` +
|
|
1347
|
+
'`import.meta.resolve("./<dir>")`, so that directory must exist beside the Machine\'s own module and ' +
|
|
1348
|
+
"travel with it: check that it exists, and that the package's `files:` list keeps it (ADR-0037/0049, ADR-0069)",
|
|
1349
|
+
);
|
|
1350
|
+
}
|
|
1351
|
+
digests.set(dir, await imageContextDigest(copy));
|
|
1352
|
+
}
|
|
1353
|
+
return digests;
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
/** What `PUT /assembly` answers, read loosely: the 200 body, or a refusal with its report. */
|
|
1357
|
+
type PublishBody = {
|
|
1358
|
+
hash?: string;
|
|
1359
|
+
workflows?: string[];
|
|
1360
|
+
unchanged?: boolean;
|
|
1361
|
+
error?: string;
|
|
1362
|
+
report?: string[];
|
|
1363
|
+
kitVersion?: unknown;
|
|
1364
|
+
};
|
|
1365
|
+
|
|
1366
|
+
/**
|
|
1367
|
+
* How long a publish waits for the Orchestrator to read the ConfigMaps this converge wrote
|
|
1368
|
+
* (ADR-0038, ADR-0069). `jr2-images` and `jr2-held` are mounts, and the kubelet refreshes a mount
|
|
1369
|
+
* on its own period (about a minute by default). A publish rolls no pod, so until the refresh the
|
|
1370
|
+
* door's checks and the next provision read the previous data. The publish waits until the digests
|
|
1371
|
+
* `GET /assembly` reports are the ones this converge applied. Exported so a test can shorten it.
|
|
1372
|
+
*/
|
|
1373
|
+
export const MAP_PROPAGATION = { budgetMs: 150_000, stepMs: 5_000 };
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* Publish the staged Assembly to the running Orchestrator (ADR-0069), over a port-forward of its
|
|
1377
|
+
* Service, as `resolveTarget` reaches it. `GET /assembly` first: when the current Assembly is this
|
|
1378
|
+
* one, at this Kit version, nothing is sent, so a steady-state converge publishes nothing. Then
|
|
1379
|
+
* `PUT /assembly` with the publish token from the Instance Secret — never from the environment.
|
|
1380
|
+
* Before the PUT, it waits until the Orchestrator reads the mounted ConfigMaps this converge wrote.
|
|
1381
|
+
* `--force` sends the PUT even when the hashes agree; the door answers `unchanged` and replaces
|
|
1382
|
+
* nothing, because the hash names the bytes it already holds.
|
|
1383
|
+
*
|
|
1384
|
+
* Returns false after it has said why, when the door refused: the previous Assembly is still
|
|
1385
|
+
* current, and the converge fails.
|
|
1386
|
+
*/
|
|
1387
|
+
async function publishAssembly(
|
|
1388
|
+
io: Io,
|
|
1389
|
+
opts: {
|
|
1390
|
+
namespace: string;
|
|
1391
|
+
ctx: { context?: string };
|
|
1392
|
+
assembly: { hash: string; kitVersion: string; workflows: string[]; tar: Buffer };
|
|
1393
|
+
token: string;
|
|
1394
|
+
force: boolean;
|
|
1395
|
+
/** The digest of each mounted ConfigMap this converge applied, by name. */
|
|
1396
|
+
mounts: Record<string, string>;
|
|
1397
|
+
},
|
|
1398
|
+
): Promise<boolean> {
|
|
1399
|
+
const { namespace, ctx, assembly, token, force } = opts;
|
|
1400
|
+
const fetchImpl = io.fetch ?? globalThis.fetch;
|
|
1401
|
+
const fwd = await (io.kube ?? kubectlKube).portForward({
|
|
1402
|
+
namespace,
|
|
1403
|
+
service: ORCHESTRATOR_SERVICE,
|
|
1404
|
+
port: ORCHESTRATOR_PORT,
|
|
1405
|
+
...ctx,
|
|
1406
|
+
});
|
|
1407
|
+
try {
|
|
1408
|
+
const base = fwd.url.replace(/\/+$/, "");
|
|
1409
|
+
const view = async (): Promise<AssemblyView> => {
|
|
1410
|
+
const seen = await fetchImpl(`${base}/assembly`);
|
|
1411
|
+
if (!seen.ok) throw new Error(`the Orchestrator answered GET /assembly with HTTP ${seen.status}`);
|
|
1412
|
+
return (await seen.json()) as AssemblyView;
|
|
1413
|
+
};
|
|
1414
|
+
let seen = await view();
|
|
1415
|
+
const current = seen.current ?? null;
|
|
1416
|
+
if (!force && current?.hash === assembly.hash && current.kitVersion === assembly.kitVersion) {
|
|
1417
|
+
activity(io, `assembly: ${assembly.hash} (current — publish skipped)`);
|
|
1418
|
+
return true;
|
|
1419
|
+
}
|
|
1420
|
+
// The mounts first (ADR-0069): the door checks a `file:` digest against `jr2-images`, and a
|
|
1421
|
+
// provision after the converge reads `jr2-held`. A process that reports no mounts is not
|
|
1422
|
+
// deployed, and has nothing to wait for.
|
|
1423
|
+
const stale = (v: AssemblyView): string[] =>
|
|
1424
|
+
v.mounts ? Object.keys(opts.mounts).filter((name) => v.mounts![name] !== opts.mounts[name]) : [];
|
|
1425
|
+
const deadline = Date.now() + MAP_PROPAGATION.budgetMs;
|
|
1426
|
+
let said = false;
|
|
1427
|
+
for (let names = stale(seen); names.length; names = stale(seen)) {
|
|
1428
|
+
if (Date.now() >= deadline) {
|
|
1429
|
+
activity(
|
|
1430
|
+
io,
|
|
1431
|
+
`refusing: after ${Math.round(MAP_PROPAGATION.budgetMs / 1000)}s the Orchestrator still reads an older ` +
|
|
1432
|
+
`${names.join(", ")} from its mount — the kubelet has not refreshed it; nothing was published, and ` +
|
|
1433
|
+
`the previous Assembly is still current. Run \`jr2 up\` again`,
|
|
1434
|
+
);
|
|
1435
|
+
return false;
|
|
1436
|
+
}
|
|
1437
|
+
if (!said) activity(io, `assembly: waiting for the kubelet to deliver ${names.join(", ")} to the Orchestrator`);
|
|
1438
|
+
said = true;
|
|
1439
|
+
await new Promise((ok) => setTimeout(ok, MAP_PROPAGATION.stepMs));
|
|
1440
|
+
seen = await view();
|
|
1441
|
+
}
|
|
1442
|
+
activity(
|
|
1443
|
+
io,
|
|
1444
|
+
`assembly: publishing ${assembly.hash} (${current?.hash ? `current is ${current.hash}` : "no current Assembly"})` +
|
|
1445
|
+
(force ? " (--force)" : ""),
|
|
1446
|
+
);
|
|
1447
|
+
const res = await fetchImpl(`${base}/assembly`, {
|
|
1448
|
+
method: "PUT",
|
|
1449
|
+
headers: {
|
|
1450
|
+
authorization: `Bearer ${token}`,
|
|
1451
|
+
"content-type": "application/x-tar",
|
|
1452
|
+
"jr2-kit-version": assembly.kitVersion,
|
|
1453
|
+
"jr2-assembly-hash": assembly.hash,
|
|
1454
|
+
},
|
|
1455
|
+
body: new Uint8Array(assembly.tar),
|
|
1456
|
+
});
|
|
1457
|
+
const body = (await res.json().catch(() => ({}))) as PublishBody;
|
|
1458
|
+
if (res.ok) {
|
|
1459
|
+
const names = body.workflows ?? assembly.workflows;
|
|
1460
|
+
activity(
|
|
1461
|
+
io,
|
|
1462
|
+
`assembly: ${body.hash ?? assembly.hash} is current${body.unchanged ? " (unchanged)" : ""} — ` +
|
|
1463
|
+
`workflows: ${names.join(", ") || "(none)"}`,
|
|
1464
|
+
);
|
|
1465
|
+
return true;
|
|
1466
|
+
}
|
|
1467
|
+
const report = body.report ?? [];
|
|
1468
|
+
activity(
|
|
1469
|
+
io,
|
|
1470
|
+
`refusing: the Orchestrator refused Assembly ${assembly.hash} (HTTP ${res.status}): ` +
|
|
1471
|
+
`${body.error ?? "no reason given"}` +
|
|
1472
|
+
(res.status >= 500 ? "" : " — the previous Assembly is still current"),
|
|
1473
|
+
);
|
|
1474
|
+
for (const line of report) activity(io, ` ${line}`);
|
|
1475
|
+
return false;
|
|
1476
|
+
} finally {
|
|
1477
|
+
fwd.close();
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
|
|
828
1481
|
/**
|
|
829
1482
|
* Collect (ADR-0039) — and only here, at the end of a converge that fully succeeded: the annotation
|
|
830
1483
|
* is applied and every rollout is verified, which is the moment the root set moved, and moving the
|
|
@@ -850,11 +1503,10 @@ async function sweepAfterConverge(
|
|
|
850
1503
|
context: string;
|
|
851
1504
|
ctx: { context?: string };
|
|
852
1505
|
converged: ConvergedImages;
|
|
853
|
-
instanceImage: string;
|
|
854
1506
|
previous?: Partial<ConvergedImages>;
|
|
855
1507
|
},
|
|
856
1508
|
): Promise<void> {
|
|
857
|
-
const { build, kube, context, ctx, converged,
|
|
1509
|
+
const { build, kube, context, ctx, converged, previous } = opts;
|
|
858
1510
|
try {
|
|
859
1511
|
await sweepImages({
|
|
860
1512
|
io,
|
|
@@ -862,9 +1514,9 @@ async function sweepAfterConverge(
|
|
|
862
1514
|
kube,
|
|
863
1515
|
context,
|
|
864
1516
|
ctx,
|
|
865
|
-
// The
|
|
866
|
-
//
|
|
867
|
-
extraKeep:
|
|
1517
|
+
// The map carries every ref this converge resolved, the Orchestrator's own Kit ref included
|
|
1518
|
+
// (ADR-0069), so it is the whole of what this converge adds to the keep set.
|
|
1519
|
+
extraKeep: mapRefs(converged),
|
|
868
1520
|
grace: previous ? mapRefs(previous) : [],
|
|
869
1521
|
});
|
|
870
1522
|
} catch (err) {
|
|
@@ -872,6 +1524,131 @@ async function sweepAfterConverge(
|
|
|
872
1524
|
}
|
|
873
1525
|
}
|
|
874
1526
|
|
|
1527
|
+
/**
|
|
1528
|
+
* Held secrets, resolved (ADR-0059): the rules (`heldSecretsOf`, over the names some pod claims, fed
|
|
1529
|
+
* what only the cluster knows), each pod's own manifest, values and Custodian config, the
|
|
1530
|
+
* per-Instance CA (kept, or minted on first need), one leaf per bound target (reused while it still
|
|
1531
|
+
* fits), and the trust bundles. Narrated one line per secret — its name, its hosts, its source — and
|
|
1532
|
+
* never a value.
|
|
1533
|
+
*/
|
|
1534
|
+
async function resolveHeld(
|
|
1535
|
+
io: Io,
|
|
1536
|
+
kube: KubeAdmin,
|
|
1537
|
+
opts: {
|
|
1538
|
+
/** Each Harness pod and the names it holds (`secretPlacementsOf`). */
|
|
1539
|
+
pods: readonly HeldPodPlan[];
|
|
1540
|
+
claimed: ReadonlySet<string>;
|
|
1541
|
+
config: JR2Config;
|
|
1542
|
+
name: string;
|
|
1543
|
+
namespace: string;
|
|
1544
|
+
ctx: { context?: string };
|
|
1545
|
+
envFromKeys: Map<string, string[]>;
|
|
1546
|
+
/** R10's read of the user's Secrets: key NAMES, never values. */
|
|
1547
|
+
secretKeys: (secret: string) => readonly string[] | undefined;
|
|
1548
|
+
models: string[];
|
|
1549
|
+
/** The user's `caBundle` (ADR-0020): trusted upstream, and by the Harness beside the jr2 CA. */
|
|
1550
|
+
caBundle?: string;
|
|
1551
|
+
},
|
|
1552
|
+
): Promise<{ objects: HeldObjects; bundles?: { extra: string; bundle: string; upstream: string } }> {
|
|
1553
|
+
const { config, namespace, ctx } = opts;
|
|
1554
|
+
type Data = KubeObject & { data?: Record<string, string> };
|
|
1555
|
+
const decode = (data: Record<string, string> | undefined, key: string): string | undefined =>
|
|
1556
|
+
data?.[key] === undefined ? undefined : Buffer.from(data[key], "base64").toString("utf8");
|
|
1557
|
+
|
|
1558
|
+
const { manifest, values, warnings } = heldSecretsOf(config, opts.claimed, {
|
|
1559
|
+
envFromKeys: opts.envFromKeys,
|
|
1560
|
+
secretKeys: opts.secretKeys,
|
|
1561
|
+
models: opts.models,
|
|
1562
|
+
});
|
|
1563
|
+
for (const warning of warnings) activity(io, `warning: ${warning}`);
|
|
1564
|
+
// Each pod's own: the bindings of the names it holds and their literal values. Its Custodian's
|
|
1565
|
+
// files are rendered by whoever makes the pod — here for the Instance Harness alone; the
|
|
1566
|
+
// Orchestrator at provision for a Sandbox, from the same manifest (ADR-0059).
|
|
1567
|
+
const pods: HeldPodObjects[] = opts.pods.map((plan) => {
|
|
1568
|
+
const own: HeldManifest = { version: 1, secrets: manifest.secrets.filter((b) => plan.names.includes(b.name)) };
|
|
1569
|
+
const literals = Object.fromEntries(
|
|
1570
|
+
own.secrets.filter((b) => b.source.kind === "literal").map((b) => [b.name, values[b.name] as string]),
|
|
1571
|
+
);
|
|
1572
|
+
return { pod: plan.pod, manifest: own, values: literals };
|
|
1573
|
+
});
|
|
1574
|
+
// The Instance Harness, when the walk placed one: the one pod `jr2 up` makes, so the one whose
|
|
1575
|
+
// Custodian files it renders.
|
|
1576
|
+
const instancePod = pods.find((p) => p.pod === INSTANCE_HARNESS_HELD_POD);
|
|
1577
|
+
const orchestrator = { host: `${ORCHESTRATOR_SERVICE}.${namespace}.svc`, port: ORCHESTRATOR_PORT };
|
|
1578
|
+
const instance: CustodianFiles | undefined = instancePod && {
|
|
1579
|
+
bootstrap: JSON.stringify(custodianBootstrap(instancePod.manifest, orchestrator)),
|
|
1580
|
+
script: custodianScript(instancePod.manifest),
|
|
1581
|
+
};
|
|
1582
|
+
if (manifest.secrets.length === 0) return { objects: { pods, ...(instance && { instance }), leaves: {} } };
|
|
1583
|
+
for (const s of manifest.secrets) {
|
|
1584
|
+
const where = s.hosts.map((h) => (h.port === 443 ? h.host : `${h.host}:${h.port}`)).join(", ");
|
|
1585
|
+
const source = s.source.kind === "literal" ? "literal" : `secret ${s.source.secret}/${s.source.key}`;
|
|
1586
|
+
activity(io, `held secret ${s.name} → ${where} (${source})`);
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
// The CA: kept whenever the cluster holds a usable one — a live pod's Harness trusts it from its
|
|
1590
|
+
// start, so a leaf a restarted Custodian reads must still chain to it. A human rotates it by
|
|
1591
|
+
// deleting the Secret.
|
|
1592
|
+
const stored = await kube.getJson<Data>({ kind: "secret", name: HELD_CA_SECRET, namespace, ...ctx });
|
|
1593
|
+
const kept = { cert: decode(stored?.data, "ca.crt") ?? "", key: decode(stored?.data, "ca.key") ?? "" };
|
|
1594
|
+
const ca: Pem = stored && caFits(kept) ? kept : issueCa(opts.name);
|
|
1595
|
+
if (ca !== kept) activity(io, `held secrets: minted the Instance's CA into ${HELD_CA_SECRET}`);
|
|
1596
|
+
|
|
1597
|
+
const tls = await kube.getJson<Data>({ kind: "secret", name: HELD_TLS_SECRET, namespace, ...ctx });
|
|
1598
|
+
const leaves: Record<string, Pem> = {};
|
|
1599
|
+
for (const s of manifest.secrets) {
|
|
1600
|
+
for (const h of s.hosts) {
|
|
1601
|
+
if (leaves[h.leaf]) continue;
|
|
1602
|
+
const existing = {
|
|
1603
|
+
cert: decode(tls?.data, `${h.leaf}.crt`) ?? "",
|
|
1604
|
+
key: decode(tls?.data, `${h.leaf}.key`) ?? "",
|
|
1605
|
+
};
|
|
1606
|
+
leaves[h.leaf] = leafFits(existing, ca, h.host) ? existing : issueLeaf(ca, h.host);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
return {
|
|
1610
|
+
objects: { pods, ...(instance && { instance }), ca, leaves },
|
|
1611
|
+
bundles: trustBundles(ca.cert, opts.caBundle),
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
/** The Instance Harness's held objects and its Custodian's files — the walk always places it when
|
|
1616
|
+
* it converges. */
|
|
1617
|
+
function instanceHeld(held: HeldObjects): HeldPodObjects & { custodian: CustodianFiles } {
|
|
1618
|
+
const pod = held.pods.find((p) => p.pod === INSTANCE_HARNESS_HELD_POD);
|
|
1619
|
+
if (!pod || !held.instance)
|
|
1620
|
+
throw new Error("jr2 up placed no held secrets for the Instance Harness it converges — a jr2 defect");
|
|
1621
|
+
return { ...pod, custodian: held.instance };
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
/** The Instance Harness's `jr2.dev/held-digest` (ADR-0059): an HMAC over its pod's held-secret
|
|
1625
|
+
* inputs — a literal's value, a `valueFrom`'s reference and the digest of its value — keyed with
|
|
1626
|
+
* the signing key so the annotation reveals nothing about a value. Its Custodian reads each value
|
|
1627
|
+
* once at start, so a value edited in the user's Secret must roll it — and so must an edit to its
|
|
1628
|
+
* two rendered files or the program they load. `program` is the kit's program; it is an input
|
|
1629
|
+
* only so that its test can change it. Exported for its test alone. */
|
|
1630
|
+
export function heldDigest(
|
|
1631
|
+
key: Buffer,
|
|
1632
|
+
all: HeldObjects,
|
|
1633
|
+
held: HeldPodObjects & { custodian: CustodianFiles },
|
|
1634
|
+
valueDigest: ValueDigest,
|
|
1635
|
+
program: string = CUSTODIAN_PROGRAM,
|
|
1636
|
+
): string {
|
|
1637
|
+
const mac = createHmac("sha256", key);
|
|
1638
|
+
mac.update(JSON.stringify(held.manifest));
|
|
1639
|
+
const stems = new Set(held.manifest.secrets.flatMap((s) => s.hosts.map((h) => h.leaf)));
|
|
1640
|
+
for (const stem of [...stems].sort()) mac.update(`${stem}:${fingerprint(all.leaves[stem]!.cert)}`);
|
|
1641
|
+
for (const name of Object.keys(held.values).sort()) mac.update(`${name}=${held.values[name]}`);
|
|
1642
|
+
for (const s of held.manifest.secrets) {
|
|
1643
|
+
if (s.source.kind === "secret") mac.update(`${s.name}<-${valueDigest(s.source.secret, s.source.key) ?? ""}`);
|
|
1644
|
+
}
|
|
1645
|
+
// Its Custodian's files, and the program they load: Envoy reads all three once at start.
|
|
1646
|
+
mac.update(held.custodian.bootstrap);
|
|
1647
|
+
mac.update(held.custodian.script);
|
|
1648
|
+
mac.update(program);
|
|
1649
|
+
return mac.digest("base64url").slice(0, 32);
|
|
1650
|
+
}
|
|
1651
|
+
|
|
875
1652
|
/** The one delivery failure that is a configuration error rather than a docker one (ADR-0019). */
|
|
876
1653
|
function undeliverable(context: string): Error {
|
|
877
1654
|
return new Error(
|
|
@@ -926,9 +1703,15 @@ async function reportOlderWorkspaces(
|
|
|
926
1703
|
* evidence gathering that hid inside the kubectl port could never be exercised without a cluster.
|
|
927
1704
|
*/
|
|
928
1705
|
async function awaitRollout(kube: KubeAdmin, target: RolloutTarget): Promise<void> {
|
|
929
|
-
const { kind, name, namespace, context } = target;
|
|
1706
|
+
const { kind, name, namespace, context, timeoutSeconds } = target;
|
|
930
1707
|
try {
|
|
931
|
-
await kube.waitRollout({
|
|
1708
|
+
await kube.waitRollout({
|
|
1709
|
+
...(kind ? { kind } : {}),
|
|
1710
|
+
name,
|
|
1711
|
+
namespace,
|
|
1712
|
+
...(context ? { context } : {}),
|
|
1713
|
+
...(timeoutSeconds ? { timeoutSeconds } : {}),
|
|
1714
|
+
});
|
|
932
1715
|
} catch (err) {
|
|
933
1716
|
throw await rolloutFailure(kube, err, target);
|
|
934
1717
|
}
|
|
@@ -941,7 +1724,7 @@ async function awaitRollout(kube: KubeAdmin, target: RolloutTarget): Promise<voi
|
|
|
941
1724
|
* template never moved (stale image, no rollout) printed `converged` while running old code.
|
|
942
1725
|
*
|
|
943
1726
|
* Tag equality is the whole check, which is only sound because the tag IS the content address (see
|
|
944
|
-
* `
|
|
1727
|
+
* `kitImageRefs`): on the `kind load` path a pod's `imageID` is containerd's manifest digest
|
|
945
1728
|
* under a rewritten `import-<date>` repo, comparable to nothing the host holds.
|
|
946
1729
|
*/
|
|
947
1730
|
async function verifyRunningImage(
|
|
@@ -955,7 +1738,7 @@ async function verifyRunningImage(
|
|
|
955
1738
|
// thing serving — the question is what the cluster runs now, not what it is done running.
|
|
956
1739
|
const live = pods.filter((p) => !p.metadata.deletionTimestamp);
|
|
957
1740
|
// `container` narrows a multi-container pod to the one this layer's image claim is about
|
|
958
|
-
// (the Instance Harness pod also carries the
|
|
1741
|
+
// (the Instance Harness pod also carries the Custodian, which is not this check's business).
|
|
959
1742
|
const stale = live.filter((p) =>
|
|
960
1743
|
p.spec.containers.some((c) => (container === undefined || c.name === container) && c.image !== image),
|
|
961
1744
|
);
|
|
@@ -984,6 +1767,192 @@ type PodObject = {
|
|
|
984
1767
|
* does. A supplied key produces none of this — it is registered already. */
|
|
985
1768
|
type GitSshNotice = { secret: string; publicKey: string; urls: string[] };
|
|
986
1769
|
|
|
1770
|
+
/** What `jr2 up` writes for the push credentials (ADR-0051): each entry's Secret, its data base64, and
|
|
1771
|
+
* the manifest the provision reads. */
|
|
1772
|
+
export type GitPushObjects = {
|
|
1773
|
+
secrets: Array<{ name: string; data: Record<string, string> }>;
|
|
1774
|
+
manifest: GitPushManifest;
|
|
1775
|
+
};
|
|
1776
|
+
|
|
1777
|
+
/**
|
|
1778
|
+
* Each `git.credentials` entry's push Secret (ADR-0051), when any Workspace pushes: the
|
|
1779
|
+
* entry's token from `.env` (Flux's `username`/`password`), and the keys of the Secrets its
|
|
1780
|
+
* `sshKey`, `push.sshKey` and `sign` name, copied as data — base64 to base64, never decoded — so
|
|
1781
|
+
* one Secret holds what the entry pushes and signs with, and the provision names it on the Sandbox
|
|
1782
|
+
* of a push whose url the entry matches. An entry with nothing it can push with is not written,
|
|
1783
|
+
* and says why. A push on a slot the Machine binds to an https url whose entry's token is unset is
|
|
1784
|
+
* refused, with its fix. A key's Secret that is absent, or a signing key's, is a warning, not a
|
|
1785
|
+
* refusal: the Secret may be written after `jr2 up` (ADR-0047's prompt, a scenario, a human), and
|
|
1786
|
+
* the entry's Secret is written without that key, so the provision refuses the push that needs it
|
|
1787
|
+
* until the Secret exists and `jr2 up` runs again.
|
|
1788
|
+
*/
|
|
1789
|
+
async function resolveGitPush(
|
|
1790
|
+
io: Io,
|
|
1791
|
+
kube: KubeAdmin,
|
|
1792
|
+
config: JR2Config,
|
|
1793
|
+
sandbox: readonly SandboxEffectorPlan[],
|
|
1794
|
+
namespace: string,
|
|
1795
|
+
ctx: { context?: string },
|
|
1796
|
+
): Promise<GitPushObjects> {
|
|
1797
|
+
const none: GitPushObjects = { secrets: [], manifest: { version: 1, entries: [] } };
|
|
1798
|
+
if (!sandbox.some((p) => p.pushes.length > 0)) return none;
|
|
1799
|
+
const { plans, skipped } = gitPushPlansOf(config.git?.credentials ?? [], (n) => io.env[n]);
|
|
1800
|
+
const read = new Map<string, Record<string, string> | undefined>();
|
|
1801
|
+
const dataOf = async (secret: string) => {
|
|
1802
|
+
if (!read.has(secret)) {
|
|
1803
|
+
const found = await kube.getJson<KubeObject & { data?: Record<string, string> }>({
|
|
1804
|
+
kind: "secret",
|
|
1805
|
+
name: secret,
|
|
1806
|
+
namespace,
|
|
1807
|
+
...ctx,
|
|
1808
|
+
});
|
|
1809
|
+
read.set(secret, found ? (found.data ?? {}) : undefined);
|
|
1810
|
+
}
|
|
1811
|
+
return read.get(secret);
|
|
1812
|
+
};
|
|
1813
|
+
const out: GitPushObjects = { secrets: [], manifest: { version: 1, entries: [] } };
|
|
1814
|
+
for (const plan of plans) {
|
|
1815
|
+
const data: Record<string, string> = {};
|
|
1816
|
+
let sshMissing: string | undefined;
|
|
1817
|
+
let signMissing: string | undefined;
|
|
1818
|
+
for (const source of plan.sources) {
|
|
1819
|
+
if ("literal" in source) {
|
|
1820
|
+
data[source.key] = Buffer.from(source.literal).toString("base64");
|
|
1821
|
+
continue;
|
|
1822
|
+
}
|
|
1823
|
+
const held = await dataOf(source.from.secret);
|
|
1824
|
+
const value = held?.[source.from.key];
|
|
1825
|
+
if (value !== undefined) {
|
|
1826
|
+
data[source.key] = value;
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1829
|
+
if (source.optional) continue;
|
|
1830
|
+
const why =
|
|
1831
|
+
held === undefined
|
|
1832
|
+
? `the Secret "${source.from.secret}" does not exist in namespace ${namespace}`
|
|
1833
|
+
: `the Secret "${source.from.secret}" holds no key ${source.from.key}`;
|
|
1834
|
+
if (source.key.startsWith("sign.")) signMissing ??= why;
|
|
1835
|
+
else sshMissing ??= why;
|
|
1836
|
+
// Not a refusal: the Secret may be written later (a scenario writes it, a human creates it).
|
|
1837
|
+
// A push under the entry that needs it is refused at provision until `jr2 up` runs again.
|
|
1838
|
+
activity(
|
|
1839
|
+
io,
|
|
1840
|
+
`warning: git.credentials "${plan.entry.match}": ${why} — a push that needs it is refused at provision ` +
|
|
1841
|
+
"until it exists and jr2 up runs again (ADR-0051)",
|
|
1842
|
+
);
|
|
1843
|
+
}
|
|
1844
|
+
const entry = { ...plan.entry };
|
|
1845
|
+
if (sshMissing !== undefined) {
|
|
1846
|
+
entry.push = entry.push.filter((k) => k !== "ssh");
|
|
1847
|
+
for (const k of ["identity", "identity.pub", "known_hosts"]) delete data[k];
|
|
1848
|
+
}
|
|
1849
|
+
if (signMissing !== undefined) {
|
|
1850
|
+
delete entry.sign;
|
|
1851
|
+
for (const k of ["sign.identity", "sign.private.asc"]) delete data[k];
|
|
1852
|
+
const json = JSON.parse(Buffer.from(data["entry.json"]!, "base64").toString("utf8")) as Record<string, unknown>;
|
|
1853
|
+
delete json.sign;
|
|
1854
|
+
data["entry.json"] = Buffer.from(JSON.stringify({ ...json, push: entry.push })).toString("base64");
|
|
1855
|
+
} else if (sshMissing !== undefined) {
|
|
1856
|
+
const json = JSON.parse(Buffer.from(data["entry.json"]!, "base64").toString("utf8")) as Record<string, unknown>;
|
|
1857
|
+
data["entry.json"] = Buffer.from(JSON.stringify({ ...json, push: entry.push })).toString("base64");
|
|
1858
|
+
}
|
|
1859
|
+
if (entry.push.length === 0) continue;
|
|
1860
|
+
out.secrets.push({ name: entry.secret, data });
|
|
1861
|
+
out.manifest.entries.push(entry);
|
|
1862
|
+
activity(
|
|
1863
|
+
io,
|
|
1864
|
+
`git.credentials "${entry.match}": push Secret ${entry.secret} — pushes over ` +
|
|
1865
|
+
`${entry.push.map((k) => (k === "ssh" ? "ssh" : "https")).join(" and ")}` +
|
|
1866
|
+
`${entry.sign ? `; signs with ${entry.sign}` : ""}`,
|
|
1867
|
+
);
|
|
1868
|
+
}
|
|
1869
|
+
for (const [match, why] of skipped) activity(io, `git.credentials "${match}": ${why}`);
|
|
1870
|
+
// A slot the Machine binds to an https url: its token's value is known now. An unset one is
|
|
1871
|
+
// refused, with the fix, as the clone's token is.
|
|
1872
|
+
for (const plan of sandbox) {
|
|
1873
|
+
for (const push of plan.pushes) {
|
|
1874
|
+
if (push.url === undefined || push.match === undefined) continue;
|
|
1875
|
+
if (repoIdentity(push.url).scheme === "ssh") continue;
|
|
1876
|
+
const placed = out.manifest.entries.find((e) => e.match === push.match);
|
|
1877
|
+
const why = skipped.get(push.match);
|
|
1878
|
+
if (!placed?.push.includes("token") && why !== undefined) {
|
|
1879
|
+
throw new Error(
|
|
1880
|
+
`${plan.label}: the push of Repo Slot "${push.slot}" (${push.url}): git.credentials entry ` +
|
|
1881
|
+
`"${push.match}" cannot push — ${why} (ADR-0051)`,
|
|
1882
|
+
);
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
return out;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
/**
|
|
1890
|
+
* What each Workspace's Sandbox Effector image owes its pushes (ADR-0068), read off the image
|
|
1891
|
+
* this converge built — its effector image, else its Sandbox Image — by the push forms `jr2 up` can
|
|
1892
|
+
* see: a slot the Machine binds (its url's scheme, its entry's `sign`), never one the run or a
|
|
1893
|
+
* `customize()` binds, whose url is the provision's. The stock Harness image — the fallback when the
|
|
1894
|
+
* Workspace names no image and there is no `images/default` — is the kit's, so what it holds is
|
|
1895
|
+
* known, not read (STOCK_HARNESS_IMAGE). An author's image `jr2 up` did not build (a registry ref)
|
|
1896
|
+
* is not read, as its `USER` is not.
|
|
1897
|
+
*/
|
|
1898
|
+
async function checkEffectorImages(
|
|
1899
|
+
build: BuildPort,
|
|
1900
|
+
config: JR2Config,
|
|
1901
|
+
sandbox: readonly SandboxEffectorPlan[],
|
|
1902
|
+
converged: ConvergedImages,
|
|
1903
|
+
platforms: readonly string[],
|
|
1904
|
+
/** Each `file:` context's key in the map: the digest of its staged copy (ADR-0069). */
|
|
1905
|
+
contextDigests: ReadonlyMap<string, string>,
|
|
1906
|
+
): Promise<void> {
|
|
1907
|
+
const list = config.git?.credentials ?? [];
|
|
1908
|
+
for (const plan of sandbox) {
|
|
1909
|
+
const forms: Array<{ form: PushForm; push: string }> = [];
|
|
1910
|
+
for (const push of plan.pushes) {
|
|
1911
|
+
if (push.url === undefined || push.match === undefined) continue;
|
|
1912
|
+
const what = `the push of Repo Slot "${push.slot}" (${push.url})`;
|
|
1913
|
+
if (isSshUrl(push.url)) forms.push({ form: "ssh-transport", push: what });
|
|
1914
|
+
const sign = push.sign ? list.find((e) => e.match === push.match)?.sign : undefined;
|
|
1915
|
+
if (sign) forms.push({ form: "gpgKey" in sign ? "gpg-sign" : "ssh-sign", push: what });
|
|
1916
|
+
}
|
|
1917
|
+
if (forms.length === 0) continue;
|
|
1918
|
+
const image = plan.image ?? plan.sandboxImage;
|
|
1919
|
+
let key = "default";
|
|
1920
|
+
if (image !== undefined) {
|
|
1921
|
+
if (!image.startsWith("file:")) continue;
|
|
1922
|
+
// A Workspace's image is a context the walk carried, so it was keyed with the Assembly.
|
|
1923
|
+
const digest = contextDigests.get(fileURLToPath(image));
|
|
1924
|
+
if (digest === undefined) continue;
|
|
1925
|
+
key = digest;
|
|
1926
|
+
}
|
|
1927
|
+
const ref = converged.sandbox[key];
|
|
1928
|
+
if (ref === undefined) {
|
|
1929
|
+
if (image !== undefined) continue;
|
|
1930
|
+
// No image named and no images/default: the Effector runs in the stock Harness image.
|
|
1931
|
+
const refusal = effectorImageRefusal({
|
|
1932
|
+
label: plan.label,
|
|
1933
|
+
ref: converged.harness,
|
|
1934
|
+
user: STOCK_HARNESS_IMAGE.user,
|
|
1935
|
+
forms,
|
|
1936
|
+
present: new Set(STOCK_HARNESS_IMAGE.present),
|
|
1937
|
+
passwd: STOCK_HARNESS_IMAGE.passwd,
|
|
1938
|
+
stock: true,
|
|
1939
|
+
});
|
|
1940
|
+
if (refusal) throw new Error(refusal);
|
|
1941
|
+
continue;
|
|
1942
|
+
}
|
|
1943
|
+
const files = await build.imageFiles(ref, platforms, effectorImagePaths(forms.map((f) => f.form)));
|
|
1944
|
+
const refusal = effectorImageRefusal({
|
|
1945
|
+
label: plan.label,
|
|
1946
|
+
ref,
|
|
1947
|
+
user: converged.sandboxUser[key] ?? "",
|
|
1948
|
+
forms,
|
|
1949
|
+
present: new Set(files.present),
|
|
1950
|
+
passwd: files.passwd,
|
|
1951
|
+
});
|
|
1952
|
+
if (refusal) throw new Error(refusal);
|
|
1953
|
+
}
|
|
1954
|
+
}
|
|
1955
|
+
|
|
987
1956
|
/** Where the git ssh key comes from — the three sources ADR-0047 offers, as a pick. */
|
|
988
1957
|
type GitSshSource = { kind: "generate" } | { kind: "file"; path: string } | { kind: "paste" };
|
|
989
1958
|
|
|
@@ -1312,7 +2281,7 @@ async function preflightProvider(
|
|
|
1312
2281
|
io: Io,
|
|
1313
2282
|
kube: KubeAdmin,
|
|
1314
2283
|
config: JR2Config,
|
|
1315
|
-
|
|
2284
|
+
carriedModels: readonly string[],
|
|
1316
2285
|
namespace: string,
|
|
1317
2286
|
ctx: { context?: string },
|
|
1318
2287
|
caPem?: string,
|
|
@@ -1327,14 +2296,8 @@ async function preflightProvider(
|
|
|
1327
2296
|
// An Open model is nobody's model yet (ADR-0054), so there is nothing to probe for it. The walk
|
|
1328
2297
|
// already refused the converge above, so this filter is the type's, not the flow's.
|
|
1329
2298
|
const prefix = `${provider.id}/`;
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
agents
|
|
1333
|
-
.map((a) => a.definition.model)
|
|
1334
|
-
.filter((m): m is string => typeof m === "string" && m.startsWith(prefix))
|
|
1335
|
-
.map((m) => m.slice(prefix.length)),
|
|
1336
|
-
),
|
|
1337
|
-
];
|
|
2299
|
+
// An Open body's allowed models are probed too (ADR-0070): a Chart's own Agent may name one.
|
|
2300
|
+
const models = [...new Set(carriedModels.filter((m) => m.startsWith(prefix)).map((m) => m.slice(prefix.length)))];
|
|
1338
2301
|
if (models.length === 0) {
|
|
1339
2302
|
activity(
|
|
1340
2303
|
io,
|
|
@@ -1403,8 +2366,36 @@ function noteDeferred(io: Io, repos: CarriedRepo[]): void {
|
|
|
1403
2366
|
if (repos.length) {
|
|
1404
2367
|
activity(
|
|
1405
2368
|
io,
|
|
1406
|
-
`repos: ${repos.length} bound Repo(s) — the Orchestrator creates their Repo resources at boot; the cache ` +
|
|
2369
|
+
`repos: ${repos.length} bound Repo(s) — the Orchestrator creates their Repo resources at boot and at each publish (ADR-0069); the cache ` +
|
|
1407
2370
|
"agent clones on first need (`jr2 status` reports sync state)",
|
|
1408
2371
|
);
|
|
1409
2372
|
}
|
|
1410
2373
|
}
|
|
2374
|
+
|
|
2375
|
+
/**
|
|
2376
|
+
* The `customize` line that retunes a composed Machine's Size (ADR-0060), nested through `actors`
|
|
2377
|
+
* the way `customize()` accepts — the Size twin of `customizeLine`, so the refusal pastes.
|
|
2378
|
+
*/
|
|
2379
|
+
function sizeLine(path: readonly string[]): string {
|
|
2380
|
+
const binds = '{ resources: { limits: { memory: "<memory>" } } }';
|
|
2381
|
+
return `customize(<import>, ${path.reduceRight((parts, key) => `{ actors: { ${key}: ${parts} } }`, binds)})`;
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
/**
|
|
2385
|
+
* The Chart files under an Assembly root (ADR-0070), as Workflow names `<wrapper>/<name>`: each
|
|
2386
|
+
* `workflows/<wrapper>/<name>.chart.json`, one level below the flat `workflows/*.ts`, in a directory
|
|
2387
|
+
* named after its wrapper. The layout the Orchestrator's discovery reads (charts.ts); a directory
|
|
2388
|
+
* whose name starts with `_` or `.` is not a wrapper's.
|
|
2389
|
+
*/
|
|
2390
|
+
async function chartFilesIn(root: string): Promise<string[]> {
|
|
2391
|
+
const out: string[] = [];
|
|
2392
|
+
const dir = join(root, "workflows");
|
|
2393
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
2394
|
+
for (const entry of entries) {
|
|
2395
|
+
if (!entry.isDirectory() || entry.name.startsWith("_") || entry.name.startsWith(".")) continue;
|
|
2396
|
+
const files = await readdir(join(dir, entry.name)).catch(() => [] as string[]);
|
|
2397
|
+
for (const f of files)
|
|
2398
|
+
if (f.endsWith(".chart.json")) out.push(`${entry.name}/${f.slice(0, -".chart.json".length)}`);
|
|
2399
|
+
}
|
|
2400
|
+
return out.sort();
|
|
2401
|
+
}
|