@jr2/cli 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/jr2.js +3 -3
- package/manifests/operator.yaml +265 -23
- package/package.json +5 -5
- package/src/assembly.ts +432 -0
- package/src/build.ts +125 -287
- package/src/cli.ts +6 -2
- package/src/client.ts +126 -33
- package/src/commands/down.ts +2 -1
- package/src/commands/init.ts +9 -2
- package/src/commands/kit.ts +3 -3
- package/src/commands/logs.ts +9 -0
- package/src/commands/run.ts +58 -4
- package/src/commands/runs.ts +96 -4
- package/src/commands/send.ts +20 -7
- package/src/commands/status.ts +90 -14
- package/src/commands/up.ts +1058 -207
- package/src/commands/version.ts +17 -12
- package/src/deploy.ts +657 -78
- package/src/effector-image.ts +133 -0
- package/src/kube.ts +107 -5
- package/src/nodes.ts +27 -2
- package/src/sweep.ts +24 -10
package/src/commands/up.ts
CHANGED
|
@@ -1,15 +1,23 @@
|
|
|
1
1
|
// `jr2 up [--yes] [--force] [-n <ns>] [--context <ctx>]` (ADR-0019): idempotently converge the target
|
|
2
2
|
// namespace to this instance — every layer, loudly narrated, safe to re-run. Layers in order:
|
|
3
|
-
// typecheck → the Machine walk →
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
3
|
+
// typecheck → the Machine walk → staging the Assembly (ADR-0069) → the Kubernetes floor (ADR-0037)
|
|
4
|
+
// → ownership → image resolution → operator → Kit images (the Harness, the Orchestrator) → Sandbox
|
|
5
|
+
// Images → Secret (+ preflight of referenced Secrets) → apply + rollout → the cache agent
|
|
6
|
+
// (ADR-0051: a DaemonSet converged when a registered Machine composes a Sandbox, deleted when none
|
|
7
|
+
// does) → Instance Harness (ADR-0031: converged by convention when a carried definition declares
|
|
8
|
+
// `workspace: "none"`, deleted when none does) → Instance Effector → publishing the Assembly to the
|
|
9
|
+
// running Orchestrator (ADR-0069) → a report of live workspaces still on an older image.
|
|
10
|
+
//
|
|
11
|
+
// The Assembly is staged EARLY and published LAST (ADR-0069). Early, so its refusals (a second copy
|
|
12
|
+
// of a Kit package, a Kit version mismatch, a tree that names its staging path) come before any
|
|
13
|
+
// cluster spend. Last, because the door's own checks are true only once the cluster work is done:
|
|
14
|
+
// the image map holds every `file:` digest, the Secrets exist, and the Orchestrator runs the Kit
|
|
15
|
+
// version the Assembly names. A Machine edit therefore rolls nothing: it is a new Assembly, not a
|
|
16
|
+
// new pod template. The Orchestrator creates a Repo resource per
|
|
17
|
+
// Repo the walk found bound, at boot and at each publish, and the cache agent clones it onto a node on first need
|
|
18
|
+
// (ADR-0051); a configured custom provider is preflighted from inside the cluster. A bound ssh url
|
|
19
|
+
// asks where its deploy key comes from (ADR-0047), and a converge that GENERATED one ends by saying
|
|
20
|
+
// so — the key is dead until a human registers it.
|
|
13
21
|
//
|
|
14
22
|
// The typecheck is FIRST and is a gate (ADR-0050): a Machine names its Agents, its composed
|
|
15
23
|
// Machines, and its Repo Slots by string, and since ADR-0049 those strings are typed, so a wrong
|
|
@@ -24,11 +32,12 @@
|
|
|
24
32
|
// `--platform` passed explicitly on every build. The cluster's schedulable nodes choose that set
|
|
25
33
|
// (`platforms` in the config overrides absolutely), so the daemon default and
|
|
26
34
|
// `DOCKER_DEFAULT_PLATFORM` steer nothing any more. In a kit CHECKOUT the built set includes the
|
|
27
|
-
// Harness and the
|
|
35
|
+
// Harness, the operator and the Orchestrator (ADR-0069); installed from npm those sources do not resolve and the published
|
|
28
36
|
// `<kitversion>` refs are used with no docker at all (they are multi-arch manifest lists, so they
|
|
29
37
|
// take no suffix and never had this hole). There is no `images` config block and no env escape
|
|
30
38
|
// hatch — nobody gets to point a real cluster at a hand-picked Harness (ADR-0027's no-eject-hatch,
|
|
31
|
-
// enforced rather than stated).
|
|
39
|
+
// enforced rather than stated). An installed Instance with no `workspace()` therefore needs no
|
|
40
|
+
// docker at all: it builds nothing, and its Workflows ride the Assembly (ADR-0069).
|
|
32
41
|
// The converged name→ref map is stamped on the Orchestrator Deployment and diffed on the next run,
|
|
33
42
|
// so a steady-state converge spends directory walks and no docker. When that record is silent (a
|
|
34
43
|
// fresh namespace), the host daemon's labeled listing answers the BUILD question instead
|
|
@@ -48,50 +57,77 @@
|
|
|
48
57
|
import { createHash, createHmac, randomBytes } from "node:crypto";
|
|
49
58
|
import { open, readdir, readFile } from "node:fs/promises";
|
|
50
59
|
import { homedir } from "node:os";
|
|
51
|
-
import { basename, join } from "node:path";
|
|
60
|
+
import { basename, join, relative } from "node:path";
|
|
61
|
+
import { fileURLToPath } from "node:url";
|
|
52
62
|
import { parseArgs } from "node:util";
|
|
53
63
|
import {
|
|
54
64
|
custodianBootstrap,
|
|
55
65
|
custodianRef,
|
|
56
66
|
custodianScript,
|
|
57
|
-
|
|
67
|
+
CUSTODIAN_PROGRAM,
|
|
58
68
|
defaultImageContext,
|
|
59
69
|
heldSecretsOf,
|
|
60
70
|
HELD_CA_SECRET,
|
|
61
|
-
HELD_SECRETS_SECRET,
|
|
62
71
|
HELD_TLS_SECRET,
|
|
72
|
+
INSTANCE_CUSTODIAN_CONFIGMAP,
|
|
73
|
+
INSTANCE_HARNESS_HELD_POD,
|
|
63
74
|
ORCHESTRATOR_PORT,
|
|
64
75
|
imageContextDigest,
|
|
65
76
|
isSshUrl,
|
|
66
77
|
loadConfig,
|
|
78
|
+
loadCharts,
|
|
67
79
|
loadWorkflows,
|
|
80
|
+
chartFileRefusals,
|
|
81
|
+
machineBodies,
|
|
68
82
|
matchCredential,
|
|
83
|
+
openPartRefusals,
|
|
84
|
+
packAssembly,
|
|
85
|
+
configMapDigest,
|
|
86
|
+
MOUNTED_CONFIGMAPS,
|
|
69
87
|
partsOf,
|
|
88
|
+
placementWalkOf,
|
|
89
|
+
refuseEffectSecrets,
|
|
90
|
+
LABEL_EFFECTOR_POD,
|
|
91
|
+
LABEL_GIT_PUSH,
|
|
92
|
+
LABEL_GIT_CLONE,
|
|
93
|
+
gitCloneSecretsOf,
|
|
94
|
+
gitPushPlansOf,
|
|
95
|
+
repoIdentity,
|
|
96
|
+
type GitPushManifest,
|
|
97
|
+
type SandboxEffectorPlan,
|
|
98
|
+
secretPlacementsOf,
|
|
99
|
+
storedValueLength,
|
|
100
|
+
effectorKey,
|
|
101
|
+
targetText,
|
|
70
102
|
harnessTokenDigest,
|
|
103
|
+
formatCpu,
|
|
104
|
+
formatMemory,
|
|
105
|
+
resolveSize,
|
|
71
106
|
sandboxToken,
|
|
72
|
-
|
|
107
|
+
seatLine,
|
|
108
|
+
serializeMachine,
|
|
109
|
+
splitSize,
|
|
73
110
|
type CarriedImage,
|
|
74
111
|
type CarriedRepo,
|
|
75
112
|
type HeldManifest,
|
|
113
|
+
type HeldPodPlan,
|
|
76
114
|
type ImageRefs,
|
|
77
115
|
type JR2Config,
|
|
116
|
+
type ResolvedSize,
|
|
78
117
|
} from "@jr2/orchestrator";
|
|
118
|
+
import { stageAssembly, stagedCopyOf } from "../assembly.ts";
|
|
79
119
|
import { caFits, fingerprint, issueCa, issueLeaf, leafFits, trustBundles, type Pem } from "../held-pki.ts";
|
|
80
120
|
import {
|
|
81
121
|
assertEmulation,
|
|
82
122
|
buildSandboxImage,
|
|
83
123
|
choosePlatforms,
|
|
84
124
|
detectKitCheckout,
|
|
85
|
-
instanceImageLabels,
|
|
86
125
|
kitImageBuild,
|
|
87
126
|
kitImageRefs,
|
|
88
127
|
normalizeRef,
|
|
89
|
-
platformSuffix,
|
|
90
128
|
pnpmDockerBuild,
|
|
91
129
|
publishedKitRefs,
|
|
92
130
|
sandboxImageTag,
|
|
93
|
-
stageInstanceBundle,
|
|
94
|
-
INSTANCE_DOCKERFILE,
|
|
95
131
|
KIT_IMAGE_HOME,
|
|
96
132
|
type BuildPort,
|
|
97
133
|
type KitImageName,
|
|
@@ -100,31 +136,50 @@ import {
|
|
|
100
136
|
import {
|
|
101
137
|
ANNOTATION_IMAGES,
|
|
102
138
|
compareVersions,
|
|
139
|
+
INSTANCE_HARNESS_DRAIN_SECONDS,
|
|
140
|
+
INSTANCE_HARNESS_PODS_SERVICE,
|
|
141
|
+
INSTANCE_HARNESS_PVC,
|
|
103
142
|
INSTANCE_HARNESS_SERVICE,
|
|
104
143
|
instanceHarnessObjects,
|
|
144
|
+
instanceEffectorObjects,
|
|
145
|
+
effectorDigest,
|
|
146
|
+
storedValueDigest,
|
|
147
|
+
type ValueDigest,
|
|
148
|
+
effectorGraceSeconds,
|
|
149
|
+
INSTANCE_EFFECTOR,
|
|
105
150
|
instanceObjects,
|
|
106
151
|
KIT_VERSION,
|
|
107
|
-
LABEL_HASH,
|
|
108
152
|
LABEL_INSTANCE,
|
|
109
153
|
LABEL_VERSION,
|
|
110
154
|
OPERATOR_DEPLOYMENT,
|
|
111
155
|
OPERATOR_NAMESPACE,
|
|
112
156
|
OPERATOR_SELECTOR,
|
|
113
157
|
operatorManifest,
|
|
158
|
+
priorityClassNames,
|
|
159
|
+
priorityClassObjects,
|
|
114
160
|
REPO_CACHE,
|
|
115
161
|
trustObject,
|
|
162
|
+
holdsAny,
|
|
163
|
+
LABEL_HELD_POD,
|
|
116
164
|
type HeldObjects,
|
|
165
|
+
type CustodianFiles,
|
|
166
|
+
type HeldPodObjects,
|
|
117
167
|
} from "../deploy.ts";
|
|
118
168
|
import { resolveRoot } from "../instance.ts";
|
|
169
|
+
import { effectorImagePaths, effectorImageRefusal, STOCK_HARNESS_IMAGE, type PushForm } from "../effector-image.ts";
|
|
119
170
|
import {
|
|
171
|
+
kubeFloorRefusal,
|
|
172
|
+
podResourcesProbe,
|
|
120
173
|
kubectlAdmin,
|
|
174
|
+
kubectlKube,
|
|
121
175
|
rolloutFailure,
|
|
176
|
+
KUBE_FLOOR,
|
|
122
177
|
ORCHESTRATOR_SERVICE,
|
|
123
178
|
type KubeAdmin,
|
|
124
179
|
type KubeObject,
|
|
125
180
|
type RolloutTarget,
|
|
126
181
|
} from "../kube.ts";
|
|
127
|
-
import { buildNodes, sandboxNodes, type NodeObject } from "../nodes.ts";
|
|
182
|
+
import { anyNodeHolds, buildNodes, sandboxNodes, type NodeObject } from "../nodes.ts";
|
|
128
183
|
import { activity, chooseOrBail, confirmOrBail, promptLine, readSecretInput, type Io } from "../output.ts";
|
|
129
184
|
import { kindCluster, sweepImages } from "../sweep.ts";
|
|
130
185
|
import { tscTypecheck } from "../typecheck.ts";
|
|
@@ -186,42 +241,233 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
186
241
|
// The Machine is named by where it sits under the registered root — the slot chain a
|
|
187
242
|
// `customize()` walks — and the fix line nests the same way, so it pastes into
|
|
188
243
|
// `workflows/<name>.ts` with the imported Machine and the url filled in.
|
|
189
|
-
//
|
|
190
|
-
//
|
|
191
|
-
//
|
|
244
|
+
// Every Open part is refused in ONE listing, in walk order within each kind: a composer fixing a
|
|
245
|
+
// packaged Machine fixes all of them in the same edit, and a converge that refused the Repo Slot,
|
|
246
|
+
// then the Agent one run later, would cost them two rounds to learn one thing. The listing is the
|
|
247
|
+
// door's own (ADR-0069): the Orchestrator refuses a publish with the same lines, so the host walk
|
|
248
|
+
// and the door's walk cannot drift.
|
|
249
|
+
const open = openPartRefusals(workflows);
|
|
250
|
+
if (open.length) {
|
|
251
|
+
for (const line of open) activity(io, `refusing: ${line}`);
|
|
252
|
+
return 1;
|
|
253
|
+
}
|
|
254
|
+
// Each Chart file is admitted HERE by the door's own check (ADR-0070), so a Chart file the
|
|
255
|
+
// Orchestrator would refuse is refused at the same command as `tsc`'s errors, before any
|
|
256
|
+
// cluster work, with the lines the publish would answer.
|
|
257
|
+
const chartRefusals = chartFileRefusals(
|
|
258
|
+
await loadCharts(root, (line) => activity(io, `warning: ${line}`)),
|
|
259
|
+
workflows,
|
|
260
|
+
);
|
|
261
|
+
if (chartRefusals.length) {
|
|
262
|
+
for (const line of chartRefusals) activity(io, `refusing: ${line}`);
|
|
263
|
+
return 1;
|
|
264
|
+
}
|
|
265
|
+
// What each registered Machine runs outside itself (ADR-0067, ADR-0068): every Effect slot, with
|
|
266
|
+
// the plain data it carries — a command's argv, its timeout, the Repo Slot it runs in; an http
|
|
267
|
+
// Effect's method, its fixed target and the secrets it names — read, never run.
|
|
192
268
|
for (const w of workflows) {
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
if (path === undefined) {
|
|
269
|
+
const effects = partsOf([w.machine]).effects;
|
|
270
|
+
if (!effects.length) continue;
|
|
271
|
+
activity(io, `effects: workflow "${w.name}" — what it runs outside itself (ADR-0067)`);
|
|
272
|
+
for (const e of effects) {
|
|
273
|
+
const where = e.path?.length ? ` (on ${e.path.map((k) => `"${k}"`).join(" → ")})` : "";
|
|
274
|
+
const slot = e.slot !== undefined ? `, in Repo Slot "${e.slot}"` : "";
|
|
275
|
+
if (e.kind === "http") {
|
|
276
|
+
const names = e.secrets.length ? `, secrets ${e.secrets.map((n) => `"${n}"`).join(", ")}` : "";
|
|
277
|
+
// Its body decides where it runs (ADR-0067): a Worktree file needs its Workspace's Sandbox Effector.
|
|
278
|
+
const where2 = e.file
|
|
279
|
+
? `in the Sandbox Effector, a Worktree file as its body${slot}`
|
|
280
|
+
: "in the Instance Effector";
|
|
206
281
|
activity(
|
|
207
282
|
io,
|
|
208
|
-
`
|
|
209
|
-
|
|
210
|
-
`bind it there (ADR-0049, ${adr})`,
|
|
283
|
+
` ${e.name}${where}: http ${e.method} ${e.target ? targetText({ target: e.target }) : "<open>"} — ${e.timeoutMs / 1000}s, ` +
|
|
284
|
+
`${where2}${names}${e.idempotent ? ", idempotent" : ""}`,
|
|
211
285
|
);
|
|
212
|
-
|
|
286
|
+
continue;
|
|
213
287
|
}
|
|
214
|
-
|
|
288
|
+
if (e.kind === "push") {
|
|
289
|
+
const opts = (["pin", "sign", "squash", "force"] as const).filter((k) => e[k]);
|
|
290
|
+
activity(
|
|
291
|
+
io,
|
|
292
|
+
` ${e.name}${where}: push${opts.length ? ` (${opts.join(", ")})` : ""} — ${e.timeoutMs / 1000}s, ` +
|
|
293
|
+
`in the Sandbox Effector${slot}`,
|
|
294
|
+
);
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
if (!("argv" in e)) continue;
|
|
298
|
+
const names = e.secrets?.length ? `, secrets ${e.secrets.map((n) => `"${n}"`).join(", ")}` : "";
|
|
299
|
+
const seat = e.secrets?.length ? `the Sandbox Effector${e.pin ? ", pinned" : ""}` : "the Harness";
|
|
215
300
|
activity(
|
|
216
301
|
io,
|
|
217
|
-
`
|
|
218
|
-
`registered: export const machine = ${customizeLine("<import>", path, slot, part)} (${adr})`,
|
|
302
|
+
` ${e.name}${where}: ${e.kind} ${JSON.stringify(e.argv)} — ${e.timeoutMs / 1000}s, in ${seat}${slot}${names}`,
|
|
219
303
|
);
|
|
220
|
-
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
// An Effect that names a secret the `secrets` list does not hold, and a literal Effect secret too
|
|
307
|
+
// short to scrub (ADR-0068), are refused HERE, off the walk and the config alone — before anything is
|
|
308
|
+
// built or applied (ADR-0050, ADR-0068). A secret read from the user's Secret is placed below,
|
|
309
|
+
// once the cluster is reached.
|
|
310
|
+
// What names each secret, read off the walk (ADR-0059, ADR-0068): every Effect, every Workspace's
|
|
311
|
+
// pod — its `workspace()`'s `secrets` and the models of the Agents whose Turns run there — and the
|
|
312
|
+
// models of the `"none"` Agents the Instance Harness hosts.
|
|
313
|
+
// An Open body that allows "none" needs the Instance Harness, holding a key for each model it
|
|
314
|
+
// allows (ADR-0070), as a "none" Agent does. The door's walk says both, so `jr2 up` and the
|
|
315
|
+
// publish door place the same set.
|
|
316
|
+
const placementWalk = placementWalkOf(workflows);
|
|
317
|
+
refuseEffectSecrets(config, placementWalk);
|
|
318
|
+
// Who may send what, per state (ADR-0015): the audience rule's answer for each registered
|
|
319
|
+
// Machine and every Machine it composes, as its doc carries it — so an author reads who holds a
|
|
320
|
+
// pick before a run finds out. Then what the build found legal and suspect: a warning, never a
|
|
321
|
+
// refusal (the refusals threw when the Machine loaded).
|
|
322
|
+
for (const w of workflows) {
|
|
323
|
+
const bodies = machineBodies(serializeMachine(w.name, w.machine));
|
|
324
|
+
const seats = bodies.flatMap((body) => body.seats.map(seatLine));
|
|
325
|
+
if (seats.length) {
|
|
326
|
+
activity(io, `seats: workflow "${w.name}" — who may send what, per state (ADR-0015)`);
|
|
327
|
+
for (const line of seats) activity(io, ` ${line}`);
|
|
328
|
+
}
|
|
329
|
+
for (const body of bodies) {
|
|
330
|
+
for (const warning of body.warnings) activity(io, `warning: workflow "${w.name}": ${warning} (ADR-0015)`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
// The Size (ADR-0060): every `workspace()` a registered Machine composes, resolved down its chain
|
|
334
|
+
// (the Machine's, then `sandbox.resources`, then the kit's) and split inside the pod, exactly as
|
|
335
|
+
// the provision will. A split that leaves the Harness below its floor is refused HERE, naming the
|
|
336
|
+
// Machine and the number to change, rather than at the first provision of a run. The resolved
|
|
337
|
+
// Sizes are kept for the Sandbox-node warning below.
|
|
338
|
+
const sizes: Array<{ workflow: string; size: ResolvedSize }> = [];
|
|
339
|
+
for (const w of workflows) {
|
|
340
|
+
for (const { path, resources, user } of partsOf([w.machine]).sizes) {
|
|
341
|
+
try {
|
|
342
|
+
const size = resolveSize(resources, config.sandbox?.resources);
|
|
343
|
+
splitSize(size, user);
|
|
344
|
+
sizes.push({ workflow: w.name, size });
|
|
345
|
+
} catch (e) {
|
|
346
|
+
const where =
|
|
347
|
+
path !== undefined && path.length
|
|
348
|
+
? ` (on the Machine composed as ${path.map((k) => `"${k}"`).join(" → ")})`
|
|
349
|
+
: "";
|
|
350
|
+
const fix =
|
|
351
|
+
path === undefined
|
|
352
|
+
? ""
|
|
353
|
+
: `; a composer retunes it where the Machine is registered: export const machine = ${sizeLine(path)}`;
|
|
354
|
+
activity(io, `refusing: workflow "${w.name}"${where}: ${(e as Error).message}${fix}`);
|
|
355
|
+
return 1;
|
|
356
|
+
}
|
|
221
357
|
}
|
|
222
358
|
}
|
|
223
359
|
const agentNames = [...new Set(agents.map((a) => a.name))].join(", ") || "(none)";
|
|
224
360
|
activity(io, `agents: ${agentNames} (carried by ${workflows.length} workflow machine(s))`);
|
|
361
|
+
// Each Chart wrapper (ADR-0070): what its Open body lends a Chart, how many models a Chart's own
|
|
362
|
+
// Agents may name (each gets its key placed below), and the Chart files the Assembly carries for
|
|
363
|
+
// it. The walk reads the Open body's declaration, so the keys and the Instance Harness a Chart
|
|
364
|
+
// could need are placed before any run.
|
|
365
|
+
const chartFiles = await chartFilesIn(root);
|
|
366
|
+
for (const w of workflows) {
|
|
367
|
+
const bodies = partsOf([w.machine]).openBodies.filter((b) => b.path !== undefined && b.path.length === 0);
|
|
368
|
+
if (!bodies.length) continue;
|
|
369
|
+
const lends = [
|
|
370
|
+
...new Set(
|
|
371
|
+
bodies.flatMap(({ declaration: { lends } }) => [
|
|
372
|
+
...(lends.agents ?? []),
|
|
373
|
+
...(lends.effects ?? []),
|
|
374
|
+
...(lends.machines ?? []),
|
|
375
|
+
...(lends.guards ?? []),
|
|
376
|
+
...(lends.actions ?? []),
|
|
377
|
+
...(lends.delays ?? []),
|
|
378
|
+
]),
|
|
379
|
+
),
|
|
380
|
+
];
|
|
381
|
+
const models = new Set(bodies.flatMap((b) => b.declaration.allows.models));
|
|
382
|
+
const files = chartFiles.filter((n) => n.startsWith(`${w.name}/`)).length;
|
|
383
|
+
activity(
|
|
384
|
+
io,
|
|
385
|
+
`chart wrapper "${w.name}": lends ${lends.join(", ") || "nothing"}; allows ${models.size} model(s); ${files} Chart file(s)`,
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// --- the Assembly (ADR-0069): staged here, published last ---------------------------------------
|
|
390
|
+
// Staged BEFORE any cluster spend, so its refusals cost nothing: a second copy of a Kit package
|
|
391
|
+
// (ADR-0054's silent-empty-Menus failure), a Kit version other than this CLI's (ADR-0056), a tree
|
|
392
|
+
// that names its own staging path (ADR-0038). Packed at once and the scratch tree removed: the
|
|
393
|
+
// archive is what the door receives, and no later layer reads the tree.
|
|
394
|
+
const build = io.build ?? pnpmDockerBuild;
|
|
395
|
+
activity(io, "assembly: staging");
|
|
396
|
+
const staged = await stageAssembly(build, root);
|
|
397
|
+
let assembly: { hash: string; kitVersion: string; workflows: string[]; tar: Buffer };
|
|
398
|
+
// The scaffolded `images/default` is a PATH CONVENTION, not discovery (ADR-0049/0050): exactly
|
|
399
|
+
// one path is checked, and it is keyed `default` rather than by digest, because that reserved
|
|
400
|
+
// key IS the middle leg of the resolution chain a `workspace()` that names no image lands on.
|
|
401
|
+
const fallback = await defaultImageContext(root);
|
|
402
|
+
// Every `file:` context is keyed by the digest of its STAGED copy (ADR-0037, ADR-0069): the
|
|
403
|
+
// bytes the door checks against the image map and a provision resolves, which the Instance
|
|
404
|
+
// folder's own copy need not equal (staging drops `node_modules`, `.git`, `.env`, `.npmrc` at
|
|
405
|
+
// every depth). The build below still reads the folder; only the key comes from here. A context
|
|
406
|
+
// the Assembly does not carry is refused now, before any cluster spend.
|
|
407
|
+
let contextDigests: Map<string, string>;
|
|
408
|
+
// The Chart files the staged Assembly carries (ADR-0070): Workflows the door registers too.
|
|
409
|
+
let stagedCharts: string[] = [];
|
|
410
|
+
try {
|
|
411
|
+
assembly = {
|
|
412
|
+
hash: staged.hash,
|
|
413
|
+
kitVersion: staged.kitVersion,
|
|
414
|
+
workflows: staged.workflows,
|
|
415
|
+
tar: await packAssembly(staged.dir),
|
|
416
|
+
};
|
|
417
|
+
contextDigests = await stagedContextDigests(staged.dir, root, [
|
|
418
|
+
...(fallback ? [fallback] : []),
|
|
419
|
+
...contexts.map((image: CarriedImage) => image.dir),
|
|
420
|
+
]);
|
|
421
|
+
stagedCharts = await chartFilesIn(staged.dir);
|
|
422
|
+
} finally {
|
|
423
|
+
await staged.dispose();
|
|
424
|
+
}
|
|
425
|
+
// The walk above loaded the Instance folder; the door loads the Assembly (ADR-0069). The two must
|
|
426
|
+
// hold the same Workflows, or a Workflow the walk built and placed for is absent at the door, and
|
|
427
|
+
// `jr2 run` names it unknown after a converge that said it succeeded.
|
|
428
|
+
// A Chart file is a Workflow too (ADR-0070), `<wrapper>/<name>`, at `workflows/<wrapper>/<name>.chart.json`.
|
|
429
|
+
const walked = [...workflows.map((w) => w.name), ...chartFiles];
|
|
430
|
+
const inAssembly = [...assembly.workflows, ...stagedCharts];
|
|
431
|
+
const absent = walked.filter((n) => !inAssembly.includes(n));
|
|
432
|
+
const extra = inAssembly.filter((n) => !walked.includes(n));
|
|
433
|
+
const fileOf = (n: string) => (n.includes("/") ? `workflows/${n}.chart.json` : `workflows/${n}.ts`);
|
|
434
|
+
if (absent.length || extra.length) {
|
|
435
|
+
activity(
|
|
436
|
+
io,
|
|
437
|
+
`refusing: the staged Assembly does not hold the Workflows this folder holds (ADR-0069)` +
|
|
438
|
+
(absent.length ? `; absent from the Assembly: ${absent.map(fileOf).join(", ")}` : "") +
|
|
439
|
+
(extra.length ? `; only in the Assembly: ${extra.map(fileOf).join(", ")}` : "") +
|
|
440
|
+
` — the Assembly holds what the Instance's install keeps; check the \`files:\` list in package.json`,
|
|
441
|
+
);
|
|
442
|
+
return 1;
|
|
443
|
+
}
|
|
444
|
+
activity(
|
|
445
|
+
io,
|
|
446
|
+
`assembly: ${assembly.hash} (kit v${assembly.kitVersion}; ${assembly.workflows.length} workflow(s): ` +
|
|
447
|
+
`${assembly.workflows.join(", ") || "(none)"})`,
|
|
448
|
+
);
|
|
449
|
+
|
|
450
|
+
// --- the Kubernetes floor (ADR-0037) ----------------------------------------------------------
|
|
451
|
+
// Every Sandbox mounts jr2's runtime as an `image` volume, and there is no fallback below the
|
|
452
|
+
// version that turns them on — so a cluster under it is refused HERE, naming what it runs and the
|
|
453
|
+
// floor (ADR-0046), rather than converging and leaving every Sandbox Pending on a volume type its
|
|
454
|
+
// kubelet does not know. Before ownership, so first contact never asks to deploy somewhere that
|
|
455
|
+
// cannot host it. Unconditional: the floor is the Instance's, not one Machine's. The container
|
|
456
|
+
// runtime owes image volumes too (containerd ≥ 2.1, CRI-O ≥ 1.31); that is not cheaply visible
|
|
457
|
+
// from here, so an older one fails at the first provision, in the pod's own events.
|
|
458
|
+
let serverVersion: string;
|
|
459
|
+
try {
|
|
460
|
+
serverVersion = await kube.serverVersion(ctx);
|
|
461
|
+
} catch (e) {
|
|
462
|
+
activity(io, `refusing: cannot reach the API server of ${context} (${(e as Error).message}). Nothing was applied.`);
|
|
463
|
+
return 1;
|
|
464
|
+
}
|
|
465
|
+
const belowFloor = kubeFloorRefusal(serverVersion);
|
|
466
|
+
if (belowFloor) {
|
|
467
|
+
activity(io, `refusing: ${context} ${belowFloor}. Nothing was applied.`);
|
|
468
|
+
return 1;
|
|
469
|
+
}
|
|
470
|
+
activity(io, `kubernetes: ${serverVersion} (floor ${KUBE_FLOOR.major}.${KUBE_FLOOR.minor}, ADR-0037)`);
|
|
225
471
|
|
|
226
472
|
// --- ownership: the cluster is the record (ADR-0019) -------------------------------------------
|
|
227
473
|
const ns = await kube.getJson({ kind: "namespace", name: namespace, ...ctx });
|
|
@@ -250,8 +496,23 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
250
496
|
...ctx,
|
|
251
497
|
});
|
|
252
498
|
|
|
499
|
+
// --- pod-level resources (ADR-0060): the whole Size is one pod-level ceiling (KEP-2837) ------
|
|
500
|
+
// A server with the PodLevelResources gate off drops `spec.resources` silently, and every
|
|
501
|
+
// Sandbox runs Burstable. A server-side dry-run shows it; it needs the namespace, so it runs here.
|
|
502
|
+
try {
|
|
503
|
+
const admitted = await kube.dryRunCreate({ manifest: JSON.stringify(podResourcesProbe(namespace)), ...ctx });
|
|
504
|
+
if (admitted.spec?.resources === undefined) {
|
|
505
|
+
activity(
|
|
506
|
+
io,
|
|
507
|
+
`refusing: ${context} (Kubernetes ${serverVersion}) drops pod-level resources — the PodLevelResources feature gate is off, and a Workspace's Size cannot hold (ADR-0060). Enable the gate on the API server and kubelets. The namespace was created; nothing else was applied.`,
|
|
508
|
+
);
|
|
509
|
+
return 1;
|
|
510
|
+
}
|
|
511
|
+
} catch (e) {
|
|
512
|
+
activity(io, `warning: could not confirm pod-level resources on ${context}: ${(e as Error).message}`);
|
|
513
|
+
}
|
|
514
|
+
|
|
253
515
|
// --- image resolution (ADR-0038): decide every ref BEFORE any layer spends a build -----------
|
|
254
|
-
const build = io.build ?? pnpmDockerBuild;
|
|
255
516
|
const registry = config.registry;
|
|
256
517
|
const cluster = kindCluster(context);
|
|
257
518
|
|
|
@@ -307,6 +568,22 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
307
568
|
"`sandbox: { nodeSelector, tolerations }` in jr2.config.ts (raw pod-spec shapes, ADR-0052).",
|
|
308
569
|
);
|
|
309
570
|
}
|
|
571
|
+
// A Size no Sandbox node could hold today WARNS (ADR-0060, the ADR-0052 stance): the node set
|
|
572
|
+
// moves, and an autoscaler may bring a bigger node for the Pending pod. A node that reports no
|
|
573
|
+
// allocatable is an unknown, never a no. One line per distinct Size, naming its workflows.
|
|
574
|
+
const unheld = new Map<string, string[]>();
|
|
575
|
+
for (const { workflow, size } of sizes) {
|
|
576
|
+
if (candidates.length === 0 || anyNodeHolds(candidates, size) !== false) continue;
|
|
577
|
+
const key = `memory ${formatMemory(size.memory.bytes)}, cpu ${formatCpu(size.cpu.millis)}`;
|
|
578
|
+
unheld.set(key, [...new Set([...(unheld.get(key) ?? []), workflow])]);
|
|
579
|
+
}
|
|
580
|
+
for (const [key, names] of unheld) {
|
|
581
|
+
activity(
|
|
582
|
+
io,
|
|
583
|
+
`warning: no Sandbox node right now has the allocatable for a Size of ${key} ` +
|
|
584
|
+
`(workflow ${names.map((n) => `"${n}"`).join(", ")}) — its Sandbox stays Pending until one does`,
|
|
585
|
+
);
|
|
586
|
+
}
|
|
310
587
|
}
|
|
311
588
|
// A multi-platform build is `docker buildx build --push`: it delivers by pushing, and there is
|
|
312
589
|
// nowhere to push without a registry. By CONSTRUCTION this never fires — a mixed-arch cluster is
|
|
@@ -335,8 +612,8 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
335
612
|
activity(
|
|
336
613
|
io,
|
|
337
614
|
kitRoot
|
|
338
|
-
? `images: kit checkout at ${kitRoot} — building the Harness and
|
|
339
|
-
: `images: installed kit — the published v${KIT_VERSION} Harness and
|
|
615
|
+
? `images: kit checkout at ${kitRoot} — building the Harness, operator and Orchestrator from source`
|
|
616
|
+
: `images: installed kit — the published v${KIT_VERSION} Harness, operator and Orchestrator, pulled from ` +
|
|
340
617
|
`${config.kitRegistry ? `${config.kitRegistry} (kitRegistry)` : KIT_IMAGE_HOME}`,
|
|
341
618
|
);
|
|
342
619
|
// The Custodian's image is deployed and never built, in either world (ADR-0038, ADR-0059): the
|
|
@@ -344,14 +621,20 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
344
621
|
const custodianImage = custodianRef(config.kitRegistry);
|
|
345
622
|
activity(io, `custodian image: ${custodianImage} (pinned, deployed — never built)`);
|
|
346
623
|
|
|
347
|
-
// Read the Orchestrator Deployment ONCE: it carries
|
|
348
|
-
//
|
|
624
|
+
// Read the Orchestrator Deployment ONCE: it carries the convergence record — the previous
|
|
625
|
+
// name→ref map (an annotation, ADR-0038), the Orchestrator's own Kit ref included (ADR-0069).
|
|
349
626
|
const orch = await kube.getJson({ kind: "deployment", name: ORCHESTRATOR_SERVICE, namespace, ...ctx });
|
|
350
627
|
const previous = previousImages(orch);
|
|
351
|
-
const converged: ConvergedImages = {
|
|
628
|
+
const converged: ConvergedImages = {
|
|
629
|
+
harness: refs.harness,
|
|
630
|
+
custodian: custodianImage,
|
|
631
|
+
orchestrator: refs.orchestrator,
|
|
632
|
+
sandbox: {},
|
|
633
|
+
sandboxUser: {},
|
|
634
|
+
};
|
|
352
635
|
|
|
353
|
-
// ONE transport branch for every image (ADR-0038), so
|
|
354
|
-
// drift into
|
|
636
|
+
// ONE transport branch for every image (ADR-0038), so Kit images and Sandbox Images cannot
|
|
637
|
+
// drift into two delivery stories.
|
|
355
638
|
const deliver = async (tag: string): Promise<void> => {
|
|
356
639
|
// A multi-platform build already pushed (ADR-0045): `docker buildx build --push` IS the
|
|
357
640
|
// delivery, and pushing again would push a tag the daemon never held. Sound at every call site
|
|
@@ -435,6 +718,17 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
435
718
|
return ref;
|
|
436
719
|
};
|
|
437
720
|
|
|
721
|
+
// --- priority (per-cluster, shared, ADR-0060) --------------------------------------------------
|
|
722
|
+
// The two PriorityClasses, beside the CRDs and before anything that names them: a pod whose class
|
|
723
|
+
// does not exist is refused at admission. Unconditional on `operator.manage` — the Instance's own
|
|
724
|
+
// pods name them too — and skipped per key the Instance's `priorityClasses` names.
|
|
725
|
+
const classes = priorityClassObjects(config.priorityClasses);
|
|
726
|
+
for (const manifest of classes) await kube.apply({ manifest, ...ctx });
|
|
727
|
+
const named = Object.entries(config.priorityClasses ?? {}).filter(([, v]) => v !== undefined);
|
|
728
|
+
if (named.length) {
|
|
729
|
+
activity(io, `priority: the Instance's own — ${named.map(([tier, cls]) => `${tier} "${cls}"`).join(", ")}`);
|
|
730
|
+
}
|
|
731
|
+
|
|
438
732
|
// --- operator (per-cluster, shared) ------------------------------------------------------------
|
|
439
733
|
// The operator IMAGE is resolved whenever anything this converge deploys runs it: the operator
|
|
440
734
|
// layer itself, or the data plane's cache agent, which is the same binary (`/manager repo-cache`,
|
|
@@ -461,7 +755,10 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
461
755
|
activity(io, `operator: leaving v${deployed} (newer than this kit's v${KIT_VERSION} — never downgraded)`);
|
|
462
756
|
} else {
|
|
463
757
|
activity(io, `operator: applying v${KIT_VERSION} (${image})${deployed ? ` over v${deployed}` : ""}`);
|
|
464
|
-
await kube.apply({
|
|
758
|
+
await kube.apply({
|
|
759
|
+
manifest: await operatorManifest(image, priorityClassNames(config.priorityClasses).control),
|
|
760
|
+
...ctx,
|
|
761
|
+
});
|
|
465
762
|
await kube.label({
|
|
466
763
|
kind: "deployment",
|
|
467
764
|
name: OPERATOR_DEPLOYMENT,
|
|
@@ -491,57 +788,19 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
491
788
|
|
|
492
789
|
// --- the kit's own runtime images (ADR-0038) ---------------------------------------------------
|
|
493
790
|
// Built here rather than lazily beside its consumers. The Harness is the injection source
|
|
494
|
-
// (ADR-0037):
|
|
495
|
-
//
|
|
791
|
+
// (ADR-0037): every Sandbox mounts this exact ref as an `image` volume, its `/opt/jr2` read-only
|
|
792
|
+
// at `/opt/jr2` in the Sandbox Image. It is not a hash input for a Sandbox Image — the runtime rides
|
|
496
793
|
// the pod's volume, so a kit edit re-images future pods and re-tags nothing of the user's.
|
|
497
794
|
await ensureKitImage("harness");
|
|
498
|
-
|
|
499
|
-
//
|
|
500
|
-
|
|
501
|
-
// kit included — so the tag is a content address rather than a guess about what changed.
|
|
502
|
-
const staged = await stageInstanceBundle(build, root);
|
|
503
|
-
// The content address covers (inputs × platform set), ADR-0045 — so the platform rides the HASH
|
|
504
|
-
// this layer records, not just the tag composed from it. That is what keeps the staleness key and
|
|
505
|
-
// the tag one fact: a converge whose platform set moved must not read the recorded hash as fresh,
|
|
506
|
-
// skip the build, and then apply a tag nothing ever delivered.
|
|
507
|
-
const hash = `${staged.hash}${platformSuffix(platforms)}`;
|
|
508
|
-
const tag = registry ? `${registry}/jr2-instance-${name}:${hash}` : `jr2-instance-${name}:${hash}`;
|
|
509
|
-
try {
|
|
510
|
-
if (orch?.metadata.labels?.[LABEL_HASH] === hash && values.force !== true) {
|
|
511
|
-
// Safe to skip only because the rolled-out pod is verified against `tag` below: this decides
|
|
512
|
-
// whether to spend a docker build, not whether the cluster is already correct.
|
|
513
|
-
activity(io, `image: ${tag} (fresh — build skipped; --force to rebuild anyway)`);
|
|
514
|
-
} else {
|
|
515
|
-
assertDeliverable();
|
|
516
|
-
if (await hostBuilt(tag)) {
|
|
517
|
-
activity(io, `image: ${tag} (host-built — build skipped, delivering; --force to rebuild)`);
|
|
518
|
-
await deliver(tag);
|
|
519
|
-
} else {
|
|
520
|
-
activity(io, `image: building ${tag}${values.force === true ? " (--force)" : ""}`);
|
|
521
|
-
await ensureEmulation();
|
|
522
|
-
await build.build({
|
|
523
|
-
tag,
|
|
524
|
-
platforms,
|
|
525
|
-
context: staged.dir,
|
|
526
|
-
dockerfileContent: INSTANCE_DOCKERFILE,
|
|
527
|
-
labels: instanceImageLabels(name),
|
|
528
|
-
});
|
|
529
|
-
await deliver(tag);
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
} finally {
|
|
533
|
-
await staged.dispose();
|
|
534
|
-
}
|
|
795
|
+
// The Orchestrator is a Kit image too (ADR-0069): the engine and its closure, nothing of this
|
|
796
|
+
// Instance. A Machine edit moves the Assembly, never this ref, so it never rolls the pod.
|
|
797
|
+
await ensureKitImage("orchestrator");
|
|
535
798
|
|
|
536
799
|
// --- Sandbox Images (ADR-0037/0049): the contexts the Machines carry, plus `images/default` -----
|
|
537
800
|
// Gated on the data-plane switch (ADR-0012/0031/0051): an instance none of whose Machines
|
|
538
801
|
// composes a Sandbox has no Sandboxes, so it must not pay a docker build for an image it can
|
|
539
802
|
// never use. Said out loud, because a silent skip of a folder you just wrote reads as a bug.
|
|
540
803
|
if (carried.composesSandbox) {
|
|
541
|
-
// The scaffolded `images/default` is a PATH CONVENTION, not discovery (ADR-0049/0050): exactly
|
|
542
|
-
// one path is checked, and it is keyed `default` rather than by digest, because that reserved
|
|
543
|
-
// key IS the middle leg of the resolution chain a `workspace()` that names no image lands on.
|
|
544
|
-
const fallback = await defaultImageContext(root);
|
|
545
804
|
const built: Array<{ dir: string; name: string; reserved?: "default" }> = [
|
|
546
805
|
...(fallback ? [{ dir: fallback, name: "default", reserved: "default" as const }] : []),
|
|
547
806
|
...contexts.map((image: CarriedImage) => ({ dir: image.dir, name: image.name })),
|
|
@@ -550,20 +809,13 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
550
809
|
activity(io, "sandbox images: none carried (a workspace() names one, or scaffold images/default)");
|
|
551
810
|
}
|
|
552
811
|
for (const image of built) {
|
|
553
|
-
//
|
|
554
|
-
// context
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
`the Sandbox Image context ${image.dir} cannot be read (${err.code ?? "read failed"}) — a ` +
|
|
558
|
-
'`workspace()` names its image with `import.meta.resolve("./<dir>")`, so that directory must exist ' +
|
|
559
|
-
"beside the Machine's own module (ADR-0037/0049).",
|
|
560
|
-
);
|
|
561
|
-
});
|
|
562
|
-
// Every carried context is keyed by that digest, which is how the baked Orchestrator finds
|
|
563
|
-
// this ref again from the `file:` URL its own copy of the module carries — no path table on
|
|
564
|
-
// either side (ADR-0049). The TAG carries the digest either way, so a Machine that names
|
|
812
|
+
// The digest of the context's staged copy, taken above with the Assembly: every carried
|
|
813
|
+
// context is keyed by it, which is how the Orchestrator finds this ref again from the
|
|
814
|
+
// `file:` URL the Assembly's copy of the module carries — no path table on either side
|
|
815
|
+
// (ADR-0049, ADR-0069). The TAG carries the digest either way, so a Machine that names
|
|
565
816
|
// `images/default` by URL lands on the same tag as the reserved key: two records, one build
|
|
566
817
|
// (the host-held check below spends nothing the second time).
|
|
818
|
+
const digest = contextDigests.get(image.dir)!;
|
|
567
819
|
const key = image.reserved ?? digest;
|
|
568
820
|
const ref = sandboxImageTag(name, image.name, digest, { platforms, registry });
|
|
569
821
|
// A record is only worth skipping on when it is COMPLETE: the ref AND the seat the image
|
|
@@ -572,7 +824,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
572
824
|
// as stale here and the image is re-inspected (the host skip below then spends no build).
|
|
573
825
|
const remembered = previous?.sandboxUser?.[key];
|
|
574
826
|
if (previous?.sandbox?.[key] === ref && remembered !== undefined && values.force !== true) {
|
|
575
|
-
// Unlike
|
|
827
|
+
// Unlike a Kit image there is no rollout to verify a Sandbox Image against, so a
|
|
576
828
|
// recorded-but-absent ref only shows up as ImagePullBackOff at the next provision —
|
|
577
829
|
// `--force` rebuilds and re-delivers it. The seat is carried forward from the same record:
|
|
578
830
|
// the tag is a content address, so the image the record names declares what it declared.
|
|
@@ -623,9 +875,15 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
623
875
|
const keep = (key: string, mint: () => string): string =>
|
|
624
876
|
secret?.data?.[key] ? Buffer.from(secret.data[key], "base64").toString("utf8") : mint();
|
|
625
877
|
const instanceToken = keep("JR2_INSTANCE_TOKEN", () => randomBytes(24).toString("hex"));
|
|
878
|
+
// The publish token (ADR-0069): publishing an Assembly imports code into the process that holds
|
|
879
|
+
// the ServiceAccount, so it costs what deploying an image cost — cluster write access. It lives
|
|
880
|
+
// here beside the Instance token, and the publish below reads it from THIS Secret alone: no env
|
|
881
|
+
// var, flag or `--url` path can supply it, so an ingress-exposed Instance exposes no publish door.
|
|
882
|
+
const publishToken = keep("JR2_PUBLISH_TOKEN", () => randomBytes(24).toString("hex"));
|
|
626
883
|
const signingKey = keep("JR2_SIGNING_KEY", () => randomBytes(32).toString("base64"));
|
|
627
884
|
const secretData: Record<string, string> = {
|
|
628
885
|
JR2_INSTANCE_TOKEN: instanceToken,
|
|
886
|
+
JR2_PUBLISH_TOKEN: publishToken,
|
|
629
887
|
JR2_SIGNING_KEY: signingKey,
|
|
630
888
|
// The Instance Harness Custodian's credential (ADR-0013/0031): a sandbox-style token signed
|
|
631
889
|
// for the placement's name — it may deliver only to Turns hosted THERE (tokens.ts), so the
|
|
@@ -633,13 +891,12 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
633
891
|
// the same value; live Custodians keep verifying.
|
|
634
892
|
JR2_INSTANCE_HARNESS_TOKEN: sandboxToken(Buffer.from(signingKey, "base64"), INSTANCE_HARNESS_SERVICE),
|
|
635
893
|
};
|
|
636
|
-
// Git tokens (ADR-
|
|
637
|
-
//
|
|
638
|
-
// Orchestrator
|
|
639
|
-
//
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
}
|
|
894
|
+
// Git tokens (ADR-0051, ADR-0068): each entry's token, from `.env`, goes into that entry's own clone
|
|
895
|
+
// Secret (`jr2-git-<hash>`, Flux's key names), which the cache agent reads — never into this
|
|
896
|
+
// Secret, the Orchestrator's env. The Orchestrator names the Secret on a Repo and holds no git token.
|
|
897
|
+
// Only when a registered Machine composes a Sandbox: nothing else clones.
|
|
898
|
+
const gitClone = carried.composesSandbox ? gitCloneSecretsOf(config.git?.credentials ?? [], (n) => io.env[n]) : [];
|
|
899
|
+
for (const s of gitClone) activity(io, `git.credentials "${s.match}": clone Secret ${s.name}`);
|
|
643
900
|
|
|
644
901
|
// The HARNESS containers' env — a separate Secret (ADR-0013): Agent code executes where these
|
|
645
902
|
// land, so the Instance token/signing key above must be unreachable from it. Values declared in
|
|
@@ -675,14 +932,56 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
675
932
|
// touches the file (the path may not exist there); consumers get the ConfigMap.
|
|
676
933
|
const caPem = await readCaBundle(root, config);
|
|
677
934
|
|
|
678
|
-
// ---
|
|
935
|
+
// --- secrets (ADR-0059, ADR-0068): resolved HERE, host-side, where `.env` exists --------------
|
|
936
|
+
// R10: a Secret an entry references must exist with its key. Read once per Secret — each key's
|
|
937
|
+
// NAME, the LENGTH of its value, and a keyed digest of it, never the value — for both
|
|
938
|
+
// placements. The length is ADR-0068's: an Effect secret shorter than 8 characters is refused
|
|
939
|
+
// wherever its value is kept (ADR-0068). The digest goes into the template digest of each
|
|
940
|
+
// long-lived pod that reads the value once at start — the Instance Harness's Custodian and the
|
|
941
|
+
// Instance Effector — so a value edited in the user's Secret rolls them at the next `jr2 up`.
|
|
942
|
+
const referenced = new Map<string, Map<string, number> | undefined>();
|
|
943
|
+
const valueDigests = new Map<string, string>();
|
|
944
|
+
for (const entry of config.secrets ?? []) {
|
|
945
|
+
const ref = entry.valueFrom?.secretKeyRef.name;
|
|
946
|
+
if (ref === undefined || referenced.has(ref)) continue;
|
|
947
|
+
const found = await kube.getJson<KubeObject & { data?: Record<string, string> }>({
|
|
948
|
+
kind: "secret",
|
|
949
|
+
name: ref,
|
|
950
|
+
namespace,
|
|
951
|
+
...ctx,
|
|
952
|
+
});
|
|
953
|
+
referenced.set(
|
|
954
|
+
ref,
|
|
955
|
+
found ? new Map(Object.entries(found.data ?? {}).map(([k, b64]) => [k, storedValueLength(b64)])) : undefined,
|
|
956
|
+
);
|
|
957
|
+
for (const [k, b64] of Object.entries(found?.data ?? {})) {
|
|
958
|
+
valueDigests.set(`${ref}/${k}`, storedValueDigest(Buffer.from(signingKey, "base64"), ref, k, b64));
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
const valueDigest = (secret: string, key: string) => valueDigests.get(`${secret}/${key}`);
|
|
962
|
+
const secretKeys = (secret: string) => {
|
|
963
|
+
const keys = referenced.get(secret);
|
|
964
|
+
return keys && [...keys.keys()];
|
|
965
|
+
};
|
|
966
|
+
const valueLength = (secret: string, key: string) => referenced.get(secret)?.get(key);
|
|
967
|
+
// Where each secret goes (ADR-0059, ADR-0068): read off the list and what names it, never
|
|
968
|
+
// declared — each Harness pod gets the Held secrets something in it names, the Instance Effector
|
|
969
|
+
// what its Effects name. The walk's refusals ran above; a `valueFrom` the user's Secrets do not
|
|
970
|
+
// hold is refused here, before the instance objects are applied.
|
|
971
|
+
const placements = secretPlacementsOf(config, placementWalk, { secretKeys, valueLength });
|
|
972
|
+
for (const line of placements.report) activity(io, line);
|
|
973
|
+
for (const warning of placements.warnings) activity(io, `warning: ${warning}`);
|
|
679
974
|
const held = await resolveHeld(io, kube, {
|
|
975
|
+
pods: placements.held,
|
|
976
|
+
claimed: placements.claimed,
|
|
680
977
|
config,
|
|
681
978
|
name,
|
|
682
979
|
namespace,
|
|
683
980
|
ctx,
|
|
684
981
|
envFromKeys,
|
|
685
|
-
|
|
982
|
+
secretKeys,
|
|
983
|
+
// Every bound Agent's model, and every model an Open body allows a Chart's own Agents (ADR-0070).
|
|
984
|
+
models: carried.models,
|
|
686
985
|
caBundle: caPem,
|
|
687
986
|
});
|
|
688
987
|
|
|
@@ -692,38 +991,88 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
692
991
|
// registers it, and by then this line has scrolled away.
|
|
693
992
|
const gitSsh = await ensureGitSsh(io, kube, config, carried.repos, namespace, ctx, values.yes === true);
|
|
694
993
|
|
|
994
|
+
// --- what each Sandbox Effector's image owes its pushes (ADR-0068) ------------------------
|
|
995
|
+
// Before anything is applied: an image that lacks a push form's tools, or the passwd entry ssh
|
|
996
|
+
// needs, is refused here with the fix, not at the first push in a pod.
|
|
997
|
+
await checkEffectorImages(build, config, placements.sandbox, converged, platforms, contextDigests);
|
|
998
|
+
|
|
999
|
+
// --- push credentials (ADR-0051): one Secret per git.credentials entry, never per pod ----
|
|
1000
|
+
// After the deploy keys exist (a generated one included), before anything else is applied: a
|
|
1001
|
+
// static Binding whose entry cannot push is refused here.
|
|
1002
|
+
const gitPush = await resolveGitPush(io, kube, config, placements.sandbox, namespace, ctx);
|
|
1003
|
+
|
|
695
1004
|
// --- provider preflight (ADR-0019): probe the endpoint FROM INSIDE the cluster ----------------
|
|
696
|
-
await preflightProvider(io, kube, config,
|
|
1005
|
+
await preflightProvider(io, kube, config, carried.models, namespace, ctx, caPem);
|
|
697
1006
|
|
|
698
1007
|
// --- apply + rollout ---------------------------------------------------------------------------
|
|
699
1008
|
// The trust bundles first, and server-side: every pod that mounts them must find them, and they
|
|
700
1009
|
// carry every root certificate — past what client-side apply's annotation can hold.
|
|
701
1010
|
const trust = trustObject({ name, namespace, caBundle: caPem, bundles: held.bundles });
|
|
702
1011
|
if (trust) await kube.apply({ manifest: trust, serverSide: true, ...ctx });
|
|
703
|
-
activity(io, `orchestrator: applying (image ${
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
1012
|
+
activity(io, `orchestrator: applying (image ${refs.orchestrator})`);
|
|
1013
|
+
const instanceList = instanceObjects({
|
|
1014
|
+
name,
|
|
1015
|
+
namespace,
|
|
1016
|
+
image: refs.orchestrator,
|
|
1017
|
+
secretData,
|
|
1018
|
+
harnessEnvData,
|
|
1019
|
+
harness: config.harness,
|
|
1020
|
+
held: held.objects,
|
|
1021
|
+
imageRefs: converged,
|
|
1022
|
+
repoCache: carried.composesSandbox ? { image: operatorImage!, placement: config.sandbox } : undefined,
|
|
1023
|
+
...(config.priorityClasses ? { priorityClasses: config.priorityClasses } : {}),
|
|
1024
|
+
sandboxEffectors: placements.sandbox,
|
|
1025
|
+
gitPush,
|
|
1026
|
+
gitClone,
|
|
718
1027
|
});
|
|
719
|
-
|
|
720
|
-
//
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
1028
|
+
await kube.apply({ manifest: instanceList, ...ctx });
|
|
1029
|
+
// An entry whose clone Secret this converge did not write loses it: a dropped entry, a token
|
|
1030
|
+
// unset, or nothing left that clones.
|
|
1031
|
+
const cloneSecrets = new Set(gitClone.map((s) => s.name));
|
|
1032
|
+
for (const secret of await kube.listJson<KubeObject>({
|
|
1033
|
+
kind: "secret",
|
|
1034
|
+
selector: LABEL_GIT_CLONE,
|
|
1035
|
+
namespace,
|
|
1036
|
+
...ctx,
|
|
1037
|
+
})) {
|
|
1038
|
+
if (cloneSecrets.has(secret.metadata.name)) continue;
|
|
1039
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1040
|
+
}
|
|
1041
|
+
// An entry whose push Secret this converge did not write loses it: a dropped entry, a token
|
|
1042
|
+
// unset, or no push left anywhere.
|
|
1043
|
+
const pushSecrets = new Set(gitPush.secrets.map((s) => s.name));
|
|
1044
|
+
for (const secret of await kube.listJson<KubeObject>({
|
|
1045
|
+
kind: "secret",
|
|
1046
|
+
selector: LABEL_GIT_PUSH,
|
|
1047
|
+
namespace,
|
|
1048
|
+
...ctx,
|
|
1049
|
+
})) {
|
|
1050
|
+
if (pushSecrets.has(secret.metadata.name)) continue;
|
|
1051
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1052
|
+
}
|
|
1053
|
+
// A Workspace whose body no longer places an Effect in its Sandbox Effector loses its Secret;
|
|
1054
|
+
// its manifest went with the ConfigMap's data above, so the next provision adds no container.
|
|
1055
|
+
const effectorPods = new Set(placements.sandbox.map((p) => p.pod));
|
|
1056
|
+
for (const secret of await kube.listJson<KubeObject>({
|
|
1057
|
+
kind: "secret",
|
|
1058
|
+
selector: LABEL_EFFECTOR_POD,
|
|
1059
|
+
namespace,
|
|
1060
|
+
...ctx,
|
|
1061
|
+
})) {
|
|
1062
|
+
if (effectorPods.has(secret.metadata.labels?.[LABEL_EFFECTOR_POD] ?? "")) continue;
|
|
1063
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1064
|
+
}
|
|
1065
|
+
// A pod's literal values go when it holds none any more, or when the pod is gone; with no secret
|
|
1066
|
+
// held anywhere, the leaves go too. The CA stays (`jr2-held-ca`), so a later converge that holds
|
|
1067
|
+
// one again issues leaves under the same CA.
|
|
1068
|
+
const literalPods = new Set(held.objects.pods.filter((p) => Object.keys(p.values).length).map((p) => p.pod));
|
|
1069
|
+
const stale = await kube.listJson<KubeObject>({ kind: "secret", selector: LABEL_HELD_POD, namespace, ...ctx });
|
|
1070
|
+
for (const secret of stale) {
|
|
1071
|
+
if (literalPods.has(secret.metadata.labels?.[LABEL_HELD_POD] ?? "")) continue;
|
|
1072
|
+
await kube.deleteObject({ kind: "secret", name: secret.metadata.name, namespace, ...ctx });
|
|
1073
|
+
}
|
|
1074
|
+
if (!holdsAny(held.objects)) {
|
|
1075
|
+
await kube.deleteObject({ kind: "secret", name: HELD_TLS_SECRET, namespace, ...ctx });
|
|
727
1076
|
}
|
|
728
1077
|
activity(io, "orchestrator: waiting for rollout");
|
|
729
1078
|
await awaitRollout(kube, {
|
|
@@ -736,7 +1085,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
736
1085
|
layer: "orchestrator",
|
|
737
1086
|
namespace,
|
|
738
1087
|
selector: `app=${ORCHESTRATOR_SERVICE}`,
|
|
739
|
-
image:
|
|
1088
|
+
image: refs.orchestrator,
|
|
740
1089
|
...ctx,
|
|
741
1090
|
});
|
|
742
1091
|
|
|
@@ -766,8 +1115,8 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
766
1115
|
});
|
|
767
1116
|
} else {
|
|
768
1117
|
// The Repo resources are NOT deleted here: eviction is reachability plus age (ADR-0051), and
|
|
769
|
-
// the
|
|
770
|
-
// of them (server.ts) and `jr2 gc` takes them at the TTL. The bare clones under
|
|
1118
|
+
// the Assembly this converge publishes binds nothing, so the Orchestrator's Repo pass at that
|
|
1119
|
+
// publish unlabels every one of them (server.ts, ADR-0069) and `jr2 gc` takes them at the TTL. The bare clones under
|
|
771
1120
|
// `/var/lib/jr2/<ns>/repos` outlive the agent that would evict them — inert, like the ones
|
|
772
1121
|
// `jr2 down` leaves (down.ts), and swept by the agent's own walk if a `workspace()` ever
|
|
773
1122
|
// returns here.
|
|
@@ -780,36 +1129,54 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
780
1129
|
// The scan is the Machine walk above, and it is DEFINITION-level (the line ADR-0018 drew:
|
|
781
1130
|
// workflow internals are not statically recoverable) — a carried-but-never-invoked `"none"`
|
|
782
1131
|
// Agent over-deploys, erring toward "the convention works when you need it". No `"none"`
|
|
783
|
-
// definitions → nothing, and a stale
|
|
784
|
-
// deleted: the layer converges toward the Machines like every other layer
|
|
785
|
-
// config.
|
|
1132
|
+
// definitions → nothing, and a stale StatefulSet from a definition that dropped its `"none"` is
|
|
1133
|
+
// deleted with its conversations: the layer converges toward the Machines like every other layer
|
|
1134
|
+
// converges toward the config.
|
|
786
1135
|
const menuOnly = [...new Set(agents.filter((a) => a.definition.workspace === "none").map((a) => a.name))];
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
1136
|
+
// A Chart wrapper whose Open body allows "none" needs it too: a Chart's own Agent may be Menu-only (ADR-0070).
|
|
1137
|
+
const noneWrappers = workflows
|
|
1138
|
+
.filter((w) => partsOf([w.machine]).openBodies.some((b) => b.declaration.allows.workspace.includes("none")))
|
|
1139
|
+
.map((w) => w.name);
|
|
1140
|
+
if (carried.instanceHarness !== undefined) {
|
|
1141
|
+
const why = [
|
|
1142
|
+
...(menuOnly.length
|
|
1143
|
+
? [`${menuOnly.join(", ")} declare${menuOnly.length === 1 ? "s" : ""} workspace: "none"`]
|
|
1144
|
+
: []),
|
|
1145
|
+
...(noneWrappers.length
|
|
1146
|
+
? [
|
|
1147
|
+
`chart wrapper${noneWrappers.length === 1 ? "" : "s"} ${noneWrappers.map((n) => `"${n}"`).join(", ")} allow${noneWrappers.length === 1 ? "s" : ""} workspace: "none"`,
|
|
1148
|
+
]
|
|
1149
|
+
: []),
|
|
1150
|
+
];
|
|
1151
|
+
activity(io, `instance harness: converging (${why.join("; ")})`);
|
|
792
1152
|
const harnessImage = refs.harness;
|
|
1153
|
+
const instance = instanceHeld(held.objects);
|
|
793
1154
|
await kube.apply({
|
|
794
1155
|
manifest: instanceHarnessObjects({
|
|
795
1156
|
name,
|
|
796
1157
|
namespace,
|
|
797
1158
|
harnessImage,
|
|
798
1159
|
custodianImage,
|
|
799
|
-
held:
|
|
800
|
-
|
|
1160
|
+
held: instance.manifest,
|
|
1161
|
+
custodian: instance.custodian,
|
|
1162
|
+
heldDigest: heldDigest(Buffer.from(signingKey, "base64"), held.objects, instance, valueDigest),
|
|
801
1163
|
harness: config.harness,
|
|
802
1164
|
caBundle: caPem !== undefined,
|
|
803
1165
|
// The wire's gate (ADR-0058): the digest of the bearer the Orchestrator derives for this
|
|
804
1166
|
// placement from the kept key — the same shape it stamps onto every Sandbox at provision.
|
|
805
1167
|
bearerSha256: harnessTokenDigest(Buffer.from(signingKey, "base64"), INSTANCE_HARNESS_SERVICE),
|
|
1168
|
+
...(config.priorityClasses ? { priorityClasses: config.priorityClasses } : {}),
|
|
806
1169
|
}),
|
|
807
1170
|
...ctx,
|
|
808
1171
|
});
|
|
809
1172
|
await awaitRollout(kube, {
|
|
1173
|
+
kind: "statefulset",
|
|
810
1174
|
name: INSTANCE_HARNESS_SERVICE,
|
|
811
1175
|
namespace,
|
|
812
1176
|
selector: `app=${INSTANCE_HARNESS_SERVICE}`,
|
|
1177
|
+
// A rollout waits out the old pod's drain (ADR-0031): up to a Turn's worst case, then the
|
|
1178
|
+
// new pod's start.
|
|
1179
|
+
timeoutSeconds: INSTANCE_HARNESS_DRAIN_SECONDS + 180,
|
|
813
1180
|
...ctx,
|
|
814
1181
|
});
|
|
815
1182
|
await verifyRunningImage(io, kube, {
|
|
@@ -821,12 +1188,83 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
821
1188
|
...ctx,
|
|
822
1189
|
});
|
|
823
1190
|
} else {
|
|
824
|
-
await kube.deleteObject({ kind: "
|
|
1191
|
+
await kube.deleteObject({ kind: "statefulset", name: INSTANCE_HARNESS_SERVICE, namespace, ...ctx });
|
|
1192
|
+
// The claim's retention policy deletes it with the StatefulSet; deleted here too, so the
|
|
1193
|
+
// conversations go even from a StatefulSet the controller never got to.
|
|
1194
|
+
await kube.deleteObject({ kind: "persistentvolumeclaim", name: INSTANCE_HARNESS_PVC, namespace, ...ctx });
|
|
825
1195
|
await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_SERVICE, namespace, ...ctx });
|
|
1196
|
+
await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_PODS_SERVICE, namespace, ...ctx });
|
|
1197
|
+
// Its Custodian's files go with the pod they were rendered for (ADR-0059).
|
|
1198
|
+
await kube.deleteObject({ kind: "configmap", name: INSTANCE_CUSTODIAN_CONFIGMAP, namespace, ...ctx });
|
|
826
1199
|
}
|
|
827
1200
|
|
|
1201
|
+
// --- Instance Effector (ADR-0068): converged when an Effect is placed there, absent otherwise ---
|
|
1202
|
+
// The placement is read off the Machines: an `http` Effect runs there. Its objects carry only the
|
|
1203
|
+
// secrets those Effects name, and the placement's key, derived from the kept signing key — the
|
|
1204
|
+
// key the Orchestrator derives again to sign every call and verify every answer.
|
|
1205
|
+
const effectorPlan = placements.instance;
|
|
1206
|
+
if (effectorPlan) {
|
|
1207
|
+
const names = effectorPlan.secrets.map((s) => `\`${s.name}\``).join(", ") || "none";
|
|
1208
|
+
activity(
|
|
1209
|
+
io,
|
|
1210
|
+
`instance effector: converging (${effectorPlan.effects.join(", ")}; secrets ${names}; grace ` +
|
|
1211
|
+
`${effectorGraceSeconds(effectorPlan)}s)`,
|
|
1212
|
+
);
|
|
1213
|
+
const signing = Buffer.from(signingKey, "base64");
|
|
1214
|
+
const key = effectorKey(signing, "instance");
|
|
1215
|
+
await kube.apply({
|
|
1216
|
+
manifest: instanceEffectorObjects({
|
|
1217
|
+
name,
|
|
1218
|
+
namespace,
|
|
1219
|
+
harnessImage: refs.harness,
|
|
1220
|
+
plan: effectorPlan,
|
|
1221
|
+
key,
|
|
1222
|
+
digest: effectorDigest(signing, effectorPlan, key, valueDigest),
|
|
1223
|
+
caBundle: caPem !== undefined,
|
|
1224
|
+
...(config.priorityClasses ? { priorityClasses: config.priorityClasses } : {}),
|
|
1225
|
+
}),
|
|
1226
|
+
...ctx,
|
|
1227
|
+
});
|
|
1228
|
+
await awaitRollout(kube, {
|
|
1229
|
+
name: INSTANCE_EFFECTOR,
|
|
1230
|
+
namespace,
|
|
1231
|
+
selector: `app=${INSTANCE_EFFECTOR}`,
|
|
1232
|
+
// A rollout waits out the old replica's drain: the longest Effect, and its read.
|
|
1233
|
+
timeoutSeconds: effectorGraceSeconds(effectorPlan) + 180,
|
|
1234
|
+
...ctx,
|
|
1235
|
+
});
|
|
1236
|
+
await verifyRunningImage(io, kube, {
|
|
1237
|
+
layer: "instance effector",
|
|
1238
|
+
namespace,
|
|
1239
|
+
selector: `app=${INSTANCE_EFFECTOR}`,
|
|
1240
|
+
image: refs.harness,
|
|
1241
|
+
container: "effector",
|
|
1242
|
+
...ctx,
|
|
1243
|
+
});
|
|
1244
|
+
} else {
|
|
1245
|
+
activity(io, "instance effector: none (no Effect is placed there)");
|
|
1246
|
+
for (const kind of ["deployment", "service", "networkpolicy", "secret"]) {
|
|
1247
|
+
await kube.deleteObject({ kind, name: INSTANCE_EFFECTOR, namespace, ...ctx });
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
// --- the Assembly (ADR-0069): published last of the layers, before the sweep -------------------
|
|
1252
|
+
// After every layer, because the door's checks are true only once the cluster work is done: the
|
|
1253
|
+
// image map names every `file:` digest, the Secrets exist, and the rollout above put the Kit
|
|
1254
|
+
// version the Assembly names in front of the door. A refusal leaves the previous Assembly
|
|
1255
|
+
// current, and the converge fails before the sweep, so nothing it still needs is collected.
|
|
1256
|
+
const published = await publishAssembly(io, {
|
|
1257
|
+
namespace,
|
|
1258
|
+
ctx,
|
|
1259
|
+
assembly,
|
|
1260
|
+
token: publishToken,
|
|
1261
|
+
force: values.force === true,
|
|
1262
|
+
mounts: mountedDigests(instanceList),
|
|
1263
|
+
});
|
|
1264
|
+
if (!published) return 1;
|
|
1265
|
+
|
|
828
1266
|
await reportOlderWorkspaces(io, kube, namespace, ctx, converged);
|
|
829
|
-
await sweepAfterConverge(io, { build, kube, context, ctx, converged,
|
|
1267
|
+
await sweepAfterConverge(io, { build, kube, context, ctx, converged, previous });
|
|
830
1268
|
noteDeferred(io, carried.repos);
|
|
831
1269
|
activity(io, `converged — \`jr2 run <workflow>\` when ready`);
|
|
832
1270
|
// LAST, after the success line (ADR-0047): the converge succeeded and the Repos still cannot be
|
|
@@ -837,7 +1275,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
837
1275
|
|
|
838
1276
|
/**
|
|
839
1277
|
* Every image ref one converge resolved. The Orchestrator reads `harness`/`custodian`/`sandbox` from
|
|
840
|
-
* the mounted map (`ImageRefs`); `operator`
|
|
1278
|
+
* the mounted map (`ImageRefs`); `operator` and `orchestrator` ride the same JSON because the record `jr2 up` diffs
|
|
841
1279
|
* must cover every image it builds, and one map is what keeps the record it diffs and the map pods
|
|
842
1280
|
* read from ever disagreeing (ADR-0038).
|
|
843
1281
|
*
|
|
@@ -851,7 +1289,7 @@ export async function up(args: string[], io: Io): Promise<number> {
|
|
|
851
1289
|
* never inspected, never in this map — so it runs as whatever its own `USER` says, and one that
|
|
852
1290
|
* would run as root fails the Harness container's `runAsNonRoot` at provision.
|
|
853
1291
|
*/
|
|
854
|
-
type ConvergedImages = ImageRefs & { operator?: string; sandboxUser: Record<string, string> };
|
|
1292
|
+
type ConvergedImages = ImageRefs & { operator?: string; orchestrator?: string; sandboxUser: Record<string, string> };
|
|
855
1293
|
|
|
856
1294
|
/** The map the LAST converge recorded, off the Orchestrator Deployment's annotation. Anything
|
|
857
1295
|
* unreadable (absent, hand-edited, a foreign shape) reads as "no record", which costs a rebuild —
|
|
@@ -869,11 +1307,177 @@ function previousImages(orch?: KubeObject): Partial<ConvergedImages> | undefined
|
|
|
869
1307
|
|
|
870
1308
|
/** Every ref one image map names, flattened — the kit's own plus each Sandbox Image. */
|
|
871
1309
|
function mapRefs(map: Partial<ConvergedImages>): string[] {
|
|
872
|
-
return [map.harness, map.custodian, map.operator, ...Object.values(map.sandbox ?? {})].filter(
|
|
1310
|
+
return [map.harness, map.custodian, map.operator, map.orchestrator, ...Object.values(map.sandbox ?? {})].filter(
|
|
873
1311
|
(ref): ref is string => typeof ref === "string",
|
|
874
1312
|
);
|
|
875
1313
|
}
|
|
876
1314
|
|
|
1315
|
+
/** What `GET /assembly` answers (ADR-0069), read loosely: this is the CLI's view of the door.
|
|
1316
|
+
* `mounts` is the digest of each ConfigMap the process reads through a mount. */
|
|
1317
|
+
type AssemblyView = { current?: { hash?: string; kitVersion?: string } | null; mounts?: Record<string, string> };
|
|
1318
|
+
|
|
1319
|
+
/** The digest of each mounted ConfigMap's data in the List this converge applied (ADR-0069), by
|
|
1320
|
+
* name: what the Orchestrator must read from its mounts before the publish. */
|
|
1321
|
+
function mountedDigests(list: string): Record<string, string> {
|
|
1322
|
+
const items = (JSON.parse(list) as { items: Array<KubeObject & { data?: Record<string, string> }> }).items;
|
|
1323
|
+
const out: Record<string, string> = {};
|
|
1324
|
+
for (const item of items) {
|
|
1325
|
+
if (item.kind === "ConfigMap" && item.metadata.name in MOUNTED_CONFIGMAPS) {
|
|
1326
|
+
out[item.metadata.name] = configMapDigest(item.data ?? {});
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
return out;
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
/**
|
|
1333
|
+
* The key of each `file:` Sandbox Image context (ADR-0037, ADR-0049, ADR-0069): the digest of its
|
|
1334
|
+
* copy in the staged Assembly at `stagedDir`, by the host directory the walk named. A context the
|
|
1335
|
+
* Assembly does not carry is refused by name: a `file:` URL that names no folder, or a folder the
|
|
1336
|
+
* Instance's install leaves out, is checkable HERE and nowhere later, since the pod only ever
|
|
1337
|
+
* resolves a digest.
|
|
1338
|
+
*/
|
|
1339
|
+
async function stagedContextDigests(stagedDir: string, root: string, dirs: string[]): Promise<Map<string, string>> {
|
|
1340
|
+
const digests = new Map<string, string>();
|
|
1341
|
+
for (const dir of new Set(dirs)) {
|
|
1342
|
+
const copy = await stagedCopyOf(stagedDir, root, dir);
|
|
1343
|
+
const shown = relative(root, dir).startsWith("..") ? dir : relative(root, dir);
|
|
1344
|
+
if (copy === undefined) {
|
|
1345
|
+
throw new Error(
|
|
1346
|
+
`the Sandbox Image context ${shown} is not in the staged Assembly — a \`workspace()\` names its image with ` +
|
|
1347
|
+
'`import.meta.resolve("./<dir>")`, so that directory must exist beside the Machine\'s own module and ' +
|
|
1348
|
+
"travel with it: check that it exists, and that the package's `files:` list keeps it (ADR-0037/0049, ADR-0069)",
|
|
1349
|
+
);
|
|
1350
|
+
}
|
|
1351
|
+
digests.set(dir, await imageContextDigest(copy));
|
|
1352
|
+
}
|
|
1353
|
+
return digests;
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
/** What `PUT /assembly` answers, read loosely: the 200 body, or a refusal with its report. */
|
|
1357
|
+
type PublishBody = {
|
|
1358
|
+
hash?: string;
|
|
1359
|
+
workflows?: string[];
|
|
1360
|
+
unchanged?: boolean;
|
|
1361
|
+
error?: string;
|
|
1362
|
+
report?: string[];
|
|
1363
|
+
kitVersion?: unknown;
|
|
1364
|
+
};
|
|
1365
|
+
|
|
1366
|
+
/**
|
|
1367
|
+
* How long a publish waits for the Orchestrator to read the ConfigMaps this converge wrote
|
|
1368
|
+
* (ADR-0038, ADR-0069). `jr2-images` and `jr2-held` are mounts, and the kubelet refreshes a mount
|
|
1369
|
+
* on its own period (about a minute by default). A publish rolls no pod, so until the refresh the
|
|
1370
|
+
* door's checks and the next provision read the previous data. The publish waits until the digests
|
|
1371
|
+
* `GET /assembly` reports are the ones this converge applied. Exported so a test can shorten it.
|
|
1372
|
+
*/
|
|
1373
|
+
export const MAP_PROPAGATION = { budgetMs: 150_000, stepMs: 5_000 };
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* Publish the staged Assembly to the running Orchestrator (ADR-0069), over a port-forward of its
|
|
1377
|
+
* Service, as `resolveTarget` reaches it. `GET /assembly` first: when the current Assembly is this
|
|
1378
|
+
* one, at this Kit version, nothing is sent, so a steady-state converge publishes nothing. Then
|
|
1379
|
+
* `PUT /assembly` with the publish token from the Instance Secret — never from the environment.
|
|
1380
|
+
* Before the PUT, it waits until the Orchestrator reads the mounted ConfigMaps this converge wrote.
|
|
1381
|
+
* `--force` sends the PUT even when the hashes agree; the door answers `unchanged` and replaces
|
|
1382
|
+
* nothing, because the hash names the bytes it already holds.
|
|
1383
|
+
*
|
|
1384
|
+
* Returns false after it has said why, when the door refused: the previous Assembly is still
|
|
1385
|
+
* current, and the converge fails.
|
|
1386
|
+
*/
|
|
1387
|
+
async function publishAssembly(
|
|
1388
|
+
io: Io,
|
|
1389
|
+
opts: {
|
|
1390
|
+
namespace: string;
|
|
1391
|
+
ctx: { context?: string };
|
|
1392
|
+
assembly: { hash: string; kitVersion: string; workflows: string[]; tar: Buffer };
|
|
1393
|
+
token: string;
|
|
1394
|
+
force: boolean;
|
|
1395
|
+
/** The digest of each mounted ConfigMap this converge applied, by name. */
|
|
1396
|
+
mounts: Record<string, string>;
|
|
1397
|
+
},
|
|
1398
|
+
): Promise<boolean> {
|
|
1399
|
+
const { namespace, ctx, assembly, token, force } = opts;
|
|
1400
|
+
const fetchImpl = io.fetch ?? globalThis.fetch;
|
|
1401
|
+
const fwd = await (io.kube ?? kubectlKube).portForward({
|
|
1402
|
+
namespace,
|
|
1403
|
+
service: ORCHESTRATOR_SERVICE,
|
|
1404
|
+
port: ORCHESTRATOR_PORT,
|
|
1405
|
+
...ctx,
|
|
1406
|
+
});
|
|
1407
|
+
try {
|
|
1408
|
+
const base = fwd.url.replace(/\/+$/, "");
|
|
1409
|
+
const view = async (): Promise<AssemblyView> => {
|
|
1410
|
+
const seen = await fetchImpl(`${base}/assembly`);
|
|
1411
|
+
if (!seen.ok) throw new Error(`the Orchestrator answered GET /assembly with HTTP ${seen.status}`);
|
|
1412
|
+
return (await seen.json()) as AssemblyView;
|
|
1413
|
+
};
|
|
1414
|
+
let seen = await view();
|
|
1415
|
+
const current = seen.current ?? null;
|
|
1416
|
+
if (!force && current?.hash === assembly.hash && current.kitVersion === assembly.kitVersion) {
|
|
1417
|
+
activity(io, `assembly: ${assembly.hash} (current — publish skipped)`);
|
|
1418
|
+
return true;
|
|
1419
|
+
}
|
|
1420
|
+
// The mounts first (ADR-0069): the door checks a `file:` digest against `jr2-images`, and a
|
|
1421
|
+
// provision after the converge reads `jr2-held`. A process that reports no mounts is not
|
|
1422
|
+
// deployed, and has nothing to wait for.
|
|
1423
|
+
const stale = (v: AssemblyView): string[] =>
|
|
1424
|
+
v.mounts ? Object.keys(opts.mounts).filter((name) => v.mounts![name] !== opts.mounts[name]) : [];
|
|
1425
|
+
const deadline = Date.now() + MAP_PROPAGATION.budgetMs;
|
|
1426
|
+
let said = false;
|
|
1427
|
+
for (let names = stale(seen); names.length; names = stale(seen)) {
|
|
1428
|
+
if (Date.now() >= deadline) {
|
|
1429
|
+
activity(
|
|
1430
|
+
io,
|
|
1431
|
+
`refusing: after ${Math.round(MAP_PROPAGATION.budgetMs / 1000)}s the Orchestrator still reads an older ` +
|
|
1432
|
+
`${names.join(", ")} from its mount — the kubelet has not refreshed it; nothing was published, and ` +
|
|
1433
|
+
`the previous Assembly is still current. Run \`jr2 up\` again`,
|
|
1434
|
+
);
|
|
1435
|
+
return false;
|
|
1436
|
+
}
|
|
1437
|
+
if (!said) activity(io, `assembly: waiting for the kubelet to deliver ${names.join(", ")} to the Orchestrator`);
|
|
1438
|
+
said = true;
|
|
1439
|
+
await new Promise((ok) => setTimeout(ok, MAP_PROPAGATION.stepMs));
|
|
1440
|
+
seen = await view();
|
|
1441
|
+
}
|
|
1442
|
+
activity(
|
|
1443
|
+
io,
|
|
1444
|
+
`assembly: publishing ${assembly.hash} (${current?.hash ? `current is ${current.hash}` : "no current Assembly"})` +
|
|
1445
|
+
(force ? " (--force)" : ""),
|
|
1446
|
+
);
|
|
1447
|
+
const res = await fetchImpl(`${base}/assembly`, {
|
|
1448
|
+
method: "PUT",
|
|
1449
|
+
headers: {
|
|
1450
|
+
authorization: `Bearer ${token}`,
|
|
1451
|
+
"content-type": "application/x-tar",
|
|
1452
|
+
"jr2-kit-version": assembly.kitVersion,
|
|
1453
|
+
"jr2-assembly-hash": assembly.hash,
|
|
1454
|
+
},
|
|
1455
|
+
body: new Uint8Array(assembly.tar),
|
|
1456
|
+
});
|
|
1457
|
+
const body = (await res.json().catch(() => ({}))) as PublishBody;
|
|
1458
|
+
if (res.ok) {
|
|
1459
|
+
const names = body.workflows ?? assembly.workflows;
|
|
1460
|
+
activity(
|
|
1461
|
+
io,
|
|
1462
|
+
`assembly: ${body.hash ?? assembly.hash} is current${body.unchanged ? " (unchanged)" : ""} — ` +
|
|
1463
|
+
`workflows: ${names.join(", ") || "(none)"}`,
|
|
1464
|
+
);
|
|
1465
|
+
return true;
|
|
1466
|
+
}
|
|
1467
|
+
const report = body.report ?? [];
|
|
1468
|
+
activity(
|
|
1469
|
+
io,
|
|
1470
|
+
`refusing: the Orchestrator refused Assembly ${assembly.hash} (HTTP ${res.status}): ` +
|
|
1471
|
+
`${body.error ?? "no reason given"}` +
|
|
1472
|
+
(res.status >= 500 ? "" : " — the previous Assembly is still current"),
|
|
1473
|
+
);
|
|
1474
|
+
for (const line of report) activity(io, ` ${line}`);
|
|
1475
|
+
return false;
|
|
1476
|
+
} finally {
|
|
1477
|
+
fwd.close();
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
|
|
877
1481
|
/**
|
|
878
1482
|
* Collect (ADR-0039) — and only here, at the end of a converge that fully succeeded: the annotation
|
|
879
1483
|
* is applied and every rollout is verified, which is the moment the root set moved, and moving the
|
|
@@ -899,11 +1503,10 @@ async function sweepAfterConverge(
|
|
|
899
1503
|
context: string;
|
|
900
1504
|
ctx: { context?: string };
|
|
901
1505
|
converged: ConvergedImages;
|
|
902
|
-
instanceImage: string;
|
|
903
1506
|
previous?: Partial<ConvergedImages>;
|
|
904
1507
|
},
|
|
905
1508
|
): Promise<void> {
|
|
906
|
-
const { build, kube, context, ctx, converged,
|
|
1509
|
+
const { build, kube, context, ctx, converged, previous } = opts;
|
|
907
1510
|
try {
|
|
908
1511
|
await sweepImages({
|
|
909
1512
|
io,
|
|
@@ -911,9 +1514,9 @@ async function sweepAfterConverge(
|
|
|
911
1514
|
kube,
|
|
912
1515
|
context,
|
|
913
1516
|
ctx,
|
|
914
|
-
// The
|
|
915
|
-
//
|
|
916
|
-
extraKeep:
|
|
1517
|
+
// The map carries every ref this converge resolved, the Orchestrator's own Kit ref included
|
|
1518
|
+
// (ADR-0069), so it is the whole of what this converge adds to the keep set.
|
|
1519
|
+
extraKeep: mapRefs(converged),
|
|
917
1520
|
grace: previous ? mapRefs(previous) : [],
|
|
918
1521
|
});
|
|
919
1522
|
} catch (err) {
|
|
@@ -922,20 +1525,26 @@ async function sweepAfterConverge(
|
|
|
922
1525
|
}
|
|
923
1526
|
|
|
924
1527
|
/**
|
|
925
|
-
* Held secrets, resolved (ADR-0059): the rules (`heldSecretsOf`,
|
|
926
|
-
* the
|
|
927
|
-
*
|
|
928
|
-
* — its name, its hosts, its source — and
|
|
1528
|
+
* Held secrets, resolved (ADR-0059): the rules (`heldSecretsOf`, over the names some pod claims, fed
|
|
1529
|
+
* what only the cluster knows), each pod's own manifest, values and Custodian config, the
|
|
1530
|
+
* per-Instance CA (kept, or minted on first need), one leaf per bound target (reused while it still
|
|
1531
|
+
* fits), and the trust bundles. Narrated one line per secret — its name, its hosts, its source — and
|
|
1532
|
+
* never a value.
|
|
929
1533
|
*/
|
|
930
1534
|
async function resolveHeld(
|
|
931
1535
|
io: Io,
|
|
932
1536
|
kube: KubeAdmin,
|
|
933
1537
|
opts: {
|
|
1538
|
+
/** Each Harness pod and the names it holds (`secretPlacementsOf`). */
|
|
1539
|
+
pods: readonly HeldPodPlan[];
|
|
1540
|
+
claimed: ReadonlySet<string>;
|
|
934
1541
|
config: JR2Config;
|
|
935
1542
|
name: string;
|
|
936
1543
|
namespace: string;
|
|
937
1544
|
ctx: { context?: string };
|
|
938
1545
|
envFromKeys: Map<string, string[]>;
|
|
1546
|
+
/** R10's read of the user's Secrets: key NAMES, never values. */
|
|
1547
|
+
secretKeys: (secret: string) => readonly string[] | undefined;
|
|
939
1548
|
models: string[];
|
|
940
1549
|
/** The user's `caBundle` (ADR-0020): trusted upstream, and by the Harness beside the jr2 CA. */
|
|
941
1550
|
caBundle?: string;
|
|
@@ -946,28 +1555,31 @@ async function resolveHeld(
|
|
|
946
1555
|
const decode = (data: Record<string, string> | undefined, key: string): string | undefined =>
|
|
947
1556
|
data?.[key] === undefined ? undefined : Buffer.from(data[key], "base64").toString("utf8");
|
|
948
1557
|
|
|
949
|
-
|
|
950
|
-
const referenced = new Map<string, string[] | undefined>();
|
|
951
|
-
for (const s of config.harness?.heldSecrets ?? []) {
|
|
952
|
-
const ref = s.valueFrom?.secretKeyRef.name;
|
|
953
|
-
if (ref === undefined || referenced.has(ref)) continue;
|
|
954
|
-
const found = await kube.getJson<Data>({ kind: "secret", name: ref, namespace, ...ctx });
|
|
955
|
-
referenced.set(ref, found ? Object.keys(found.data ?? {}) : undefined);
|
|
956
|
-
}
|
|
957
|
-
const { manifest, values, warnings } = heldSecretsOf(config, {
|
|
1558
|
+
const { manifest, values, warnings } = heldSecretsOf(config, opts.claimed, {
|
|
958
1559
|
envFromKeys: opts.envFromKeys,
|
|
959
|
-
secretKeys:
|
|
1560
|
+
secretKeys: opts.secretKeys,
|
|
960
1561
|
models: opts.models,
|
|
961
1562
|
});
|
|
962
1563
|
for (const warning of warnings) activity(io, `warning: ${warning}`);
|
|
1564
|
+
// Each pod's own: the bindings of the names it holds and their literal values. Its Custodian's
|
|
1565
|
+
// files are rendered by whoever makes the pod — here for the Instance Harness alone; the
|
|
1566
|
+
// Orchestrator at provision for a Sandbox, from the same manifest (ADR-0059).
|
|
1567
|
+
const pods: HeldPodObjects[] = opts.pods.map((plan) => {
|
|
1568
|
+
const own: HeldManifest = { version: 1, secrets: manifest.secrets.filter((b) => plan.names.includes(b.name)) };
|
|
1569
|
+
const literals = Object.fromEntries(
|
|
1570
|
+
own.secrets.filter((b) => b.source.kind === "literal").map((b) => [b.name, values[b.name] as string]),
|
|
1571
|
+
);
|
|
1572
|
+
return { pod: plan.pod, manifest: own, values: literals };
|
|
1573
|
+
});
|
|
1574
|
+
// The Instance Harness, when the walk placed one: the one pod `jr2 up` makes, so the one whose
|
|
1575
|
+
// Custodian files it renders.
|
|
1576
|
+
const instancePod = pods.find((p) => p.pod === INSTANCE_HARNESS_HELD_POD);
|
|
963
1577
|
const orchestrator = { host: `${ORCHESTRATOR_SERVICE}.${namespace}.svc`, port: ORCHESTRATOR_PORT };
|
|
964
|
-
const
|
|
965
|
-
bootstrap: JSON.stringify(custodianBootstrap(manifest, orchestrator)),
|
|
966
|
-
script: custodianScript(manifest),
|
|
1578
|
+
const instance: CustodianFiles | undefined = instancePod && {
|
|
1579
|
+
bootstrap: JSON.stringify(custodianBootstrap(instancePod.manifest, orchestrator)),
|
|
1580
|
+
script: custodianScript(instancePod.manifest),
|
|
967
1581
|
};
|
|
968
|
-
if (manifest.secrets.length === 0) {
|
|
969
|
-
return { objects: { manifest, values, leaves: {}, ...rendered } };
|
|
970
|
-
}
|
|
1582
|
+
if (manifest.secrets.length === 0) return { objects: { pods, ...(instance && { instance }), leaves: {} } };
|
|
971
1583
|
for (const s of manifest.secrets) {
|
|
972
1584
|
const where = s.hosts.map((h) => (h.port === 443 ? h.host : `${h.host}:${h.port}`)).join(", ");
|
|
973
1585
|
const source = s.source.kind === "literal" ? "literal" : `secret ${s.source.secret}/${s.source.key}`;
|
|
@@ -995,20 +1607,45 @@ async function resolveHeld(
|
|
|
995
1607
|
}
|
|
996
1608
|
}
|
|
997
1609
|
return {
|
|
998
|
-
objects: {
|
|
1610
|
+
objects: { pods, ...(instance && { instance }), ca, leaves },
|
|
999
1611
|
bundles: trustBundles(ca.cert, opts.caBundle),
|
|
1000
1612
|
};
|
|
1001
1613
|
}
|
|
1002
1614
|
|
|
1003
|
-
/** The Instance Harness's
|
|
1004
|
-
*
|
|
1005
|
-
function
|
|
1615
|
+
/** The Instance Harness's held objects and its Custodian's files — the walk always places it when
|
|
1616
|
+
* it converges. */
|
|
1617
|
+
function instanceHeld(held: HeldObjects): HeldPodObjects & { custodian: CustodianFiles } {
|
|
1618
|
+
const pod = held.pods.find((p) => p.pod === INSTANCE_HARNESS_HELD_POD);
|
|
1619
|
+
if (!pod || !held.instance)
|
|
1620
|
+
throw new Error("jr2 up placed no held secrets for the Instance Harness it converges — a jr2 defect");
|
|
1621
|
+
return { ...pod, custodian: held.instance };
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
/** The Instance Harness's `jr2.dev/held-digest` (ADR-0059): an HMAC over its pod's held-secret
|
|
1625
|
+
* inputs — a literal's value, a `valueFrom`'s reference and the digest of its value — keyed with
|
|
1626
|
+
* the signing key so the annotation reveals nothing about a value. Its Custodian reads each value
|
|
1627
|
+
* once at start, so a value edited in the user's Secret must roll it — and so must an edit to its
|
|
1628
|
+
* two rendered files or the program they load. `program` is the kit's program; it is an input
|
|
1629
|
+
* only so that its test can change it. Exported for its test alone. */
|
|
1630
|
+
export function heldDigest(
|
|
1631
|
+
key: Buffer,
|
|
1632
|
+
all: HeldObjects,
|
|
1633
|
+
held: HeldPodObjects & { custodian: CustodianFiles },
|
|
1634
|
+
valueDigest: ValueDigest,
|
|
1635
|
+
program: string = CUSTODIAN_PROGRAM,
|
|
1636
|
+
): string {
|
|
1006
1637
|
const mac = createHmac("sha256", key);
|
|
1007
1638
|
mac.update(JSON.stringify(held.manifest));
|
|
1008
|
-
|
|
1639
|
+
const stems = new Set(held.manifest.secrets.flatMap((s) => s.hosts.map((h) => h.leaf)));
|
|
1640
|
+
for (const stem of [...stems].sort()) mac.update(`${stem}:${fingerprint(all.leaves[stem]!.cert)}`);
|
|
1009
1641
|
for (const name of Object.keys(held.values).sort()) mac.update(`${name}=${held.values[name]}`);
|
|
1010
|
-
|
|
1011
|
-
|
|
1642
|
+
for (const s of held.manifest.secrets) {
|
|
1643
|
+
if (s.source.kind === "secret") mac.update(`${s.name}<-${valueDigest(s.source.secret, s.source.key) ?? ""}`);
|
|
1644
|
+
}
|
|
1645
|
+
// Its Custodian's files, and the program they load: Envoy reads all three once at start.
|
|
1646
|
+
mac.update(held.custodian.bootstrap);
|
|
1647
|
+
mac.update(held.custodian.script);
|
|
1648
|
+
mac.update(program);
|
|
1012
1649
|
return mac.digest("base64url").slice(0, 32);
|
|
1013
1650
|
}
|
|
1014
1651
|
|
|
@@ -1066,9 +1703,15 @@ async function reportOlderWorkspaces(
|
|
|
1066
1703
|
* evidence gathering that hid inside the kubectl port could never be exercised without a cluster.
|
|
1067
1704
|
*/
|
|
1068
1705
|
async function awaitRollout(kube: KubeAdmin, target: RolloutTarget): Promise<void> {
|
|
1069
|
-
const { kind, name, namespace, context } = target;
|
|
1706
|
+
const { kind, name, namespace, context, timeoutSeconds } = target;
|
|
1070
1707
|
try {
|
|
1071
|
-
await kube.waitRollout({
|
|
1708
|
+
await kube.waitRollout({
|
|
1709
|
+
...(kind ? { kind } : {}),
|
|
1710
|
+
name,
|
|
1711
|
+
namespace,
|
|
1712
|
+
...(context ? { context } : {}),
|
|
1713
|
+
...(timeoutSeconds ? { timeoutSeconds } : {}),
|
|
1714
|
+
});
|
|
1072
1715
|
} catch (err) {
|
|
1073
1716
|
throw await rolloutFailure(kube, err, target);
|
|
1074
1717
|
}
|
|
@@ -1081,7 +1724,7 @@ async function awaitRollout(kube: KubeAdmin, target: RolloutTarget): Promise<voi
|
|
|
1081
1724
|
* template never moved (stale image, no rollout) printed `converged` while running old code.
|
|
1082
1725
|
*
|
|
1083
1726
|
* Tag equality is the whole check, which is only sound because the tag IS the content address (see
|
|
1084
|
-
* `
|
|
1727
|
+
* `kitImageRefs`): on the `kind load` path a pod's `imageID` is containerd's manifest digest
|
|
1085
1728
|
* under a rewritten `import-<date>` repo, comparable to nothing the host holds.
|
|
1086
1729
|
*/
|
|
1087
1730
|
async function verifyRunningImage(
|
|
@@ -1124,6 +1767,192 @@ type PodObject = {
|
|
|
1124
1767
|
* does. A supplied key produces none of this — it is registered already. */
|
|
1125
1768
|
type GitSshNotice = { secret: string; publicKey: string; urls: string[] };
|
|
1126
1769
|
|
|
1770
|
+
/** What `jr2 up` writes for the push credentials (ADR-0051): each entry's Secret, its data base64, and
|
|
1771
|
+
* the manifest the provision reads. */
|
|
1772
|
+
export type GitPushObjects = {
|
|
1773
|
+
secrets: Array<{ name: string; data: Record<string, string> }>;
|
|
1774
|
+
manifest: GitPushManifest;
|
|
1775
|
+
};
|
|
1776
|
+
|
|
1777
|
+
/**
|
|
1778
|
+
* Each `git.credentials` entry's push Secret (ADR-0051), when any Workspace pushes: the
|
|
1779
|
+
* entry's token from `.env` (Flux's `username`/`password`), and the keys of the Secrets its
|
|
1780
|
+
* `sshKey`, `push.sshKey` and `sign` name, copied as data — base64 to base64, never decoded — so
|
|
1781
|
+
* one Secret holds what the entry pushes and signs with, and the provision names it on the Sandbox
|
|
1782
|
+
* of a push whose url the entry matches. An entry with nothing it can push with is not written,
|
|
1783
|
+
* and says why. A push on a slot the Machine binds to an https url whose entry's token is unset is
|
|
1784
|
+
* refused, with its fix. A key's Secret that is absent, or a signing key's, is a warning, not a
|
|
1785
|
+
* refusal: the Secret may be written after `jr2 up` (ADR-0047's prompt, a scenario, a human), and
|
|
1786
|
+
* the entry's Secret is written without that key, so the provision refuses the push that needs it
|
|
1787
|
+
* until the Secret exists and `jr2 up` runs again.
|
|
1788
|
+
*/
|
|
1789
|
+
async function resolveGitPush(
|
|
1790
|
+
io: Io,
|
|
1791
|
+
kube: KubeAdmin,
|
|
1792
|
+
config: JR2Config,
|
|
1793
|
+
sandbox: readonly SandboxEffectorPlan[],
|
|
1794
|
+
namespace: string,
|
|
1795
|
+
ctx: { context?: string },
|
|
1796
|
+
): Promise<GitPushObjects> {
|
|
1797
|
+
const none: GitPushObjects = { secrets: [], manifest: { version: 1, entries: [] } };
|
|
1798
|
+
if (!sandbox.some((p) => p.pushes.length > 0)) return none;
|
|
1799
|
+
const { plans, skipped } = gitPushPlansOf(config.git?.credentials ?? [], (n) => io.env[n]);
|
|
1800
|
+
const read = new Map<string, Record<string, string> | undefined>();
|
|
1801
|
+
const dataOf = async (secret: string) => {
|
|
1802
|
+
if (!read.has(secret)) {
|
|
1803
|
+
const found = await kube.getJson<KubeObject & { data?: Record<string, string> }>({
|
|
1804
|
+
kind: "secret",
|
|
1805
|
+
name: secret,
|
|
1806
|
+
namespace,
|
|
1807
|
+
...ctx,
|
|
1808
|
+
});
|
|
1809
|
+
read.set(secret, found ? (found.data ?? {}) : undefined);
|
|
1810
|
+
}
|
|
1811
|
+
return read.get(secret);
|
|
1812
|
+
};
|
|
1813
|
+
const out: GitPushObjects = { secrets: [], manifest: { version: 1, entries: [] } };
|
|
1814
|
+
for (const plan of plans) {
|
|
1815
|
+
const data: Record<string, string> = {};
|
|
1816
|
+
let sshMissing: string | undefined;
|
|
1817
|
+
let signMissing: string | undefined;
|
|
1818
|
+
for (const source of plan.sources) {
|
|
1819
|
+
if ("literal" in source) {
|
|
1820
|
+
data[source.key] = Buffer.from(source.literal).toString("base64");
|
|
1821
|
+
continue;
|
|
1822
|
+
}
|
|
1823
|
+
const held = await dataOf(source.from.secret);
|
|
1824
|
+
const value = held?.[source.from.key];
|
|
1825
|
+
if (value !== undefined) {
|
|
1826
|
+
data[source.key] = value;
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1829
|
+
if (source.optional) continue;
|
|
1830
|
+
const why =
|
|
1831
|
+
held === undefined
|
|
1832
|
+
? `the Secret "${source.from.secret}" does not exist in namespace ${namespace}`
|
|
1833
|
+
: `the Secret "${source.from.secret}" holds no key ${source.from.key}`;
|
|
1834
|
+
if (source.key.startsWith("sign.")) signMissing ??= why;
|
|
1835
|
+
else sshMissing ??= why;
|
|
1836
|
+
// Not a refusal: the Secret may be written later (a scenario writes it, a human creates it).
|
|
1837
|
+
// A push under the entry that needs it is refused at provision until `jr2 up` runs again.
|
|
1838
|
+
activity(
|
|
1839
|
+
io,
|
|
1840
|
+
`warning: git.credentials "${plan.entry.match}": ${why} — a push that needs it is refused at provision ` +
|
|
1841
|
+
"until it exists and jr2 up runs again (ADR-0051)",
|
|
1842
|
+
);
|
|
1843
|
+
}
|
|
1844
|
+
const entry = { ...plan.entry };
|
|
1845
|
+
if (sshMissing !== undefined) {
|
|
1846
|
+
entry.push = entry.push.filter((k) => k !== "ssh");
|
|
1847
|
+
for (const k of ["identity", "identity.pub", "known_hosts"]) delete data[k];
|
|
1848
|
+
}
|
|
1849
|
+
if (signMissing !== undefined) {
|
|
1850
|
+
delete entry.sign;
|
|
1851
|
+
for (const k of ["sign.identity", "sign.private.asc"]) delete data[k];
|
|
1852
|
+
const json = JSON.parse(Buffer.from(data["entry.json"]!, "base64").toString("utf8")) as Record<string, unknown>;
|
|
1853
|
+
delete json.sign;
|
|
1854
|
+
data["entry.json"] = Buffer.from(JSON.stringify({ ...json, push: entry.push })).toString("base64");
|
|
1855
|
+
} else if (sshMissing !== undefined) {
|
|
1856
|
+
const json = JSON.parse(Buffer.from(data["entry.json"]!, "base64").toString("utf8")) as Record<string, unknown>;
|
|
1857
|
+
data["entry.json"] = Buffer.from(JSON.stringify({ ...json, push: entry.push })).toString("base64");
|
|
1858
|
+
}
|
|
1859
|
+
if (entry.push.length === 0) continue;
|
|
1860
|
+
out.secrets.push({ name: entry.secret, data });
|
|
1861
|
+
out.manifest.entries.push(entry);
|
|
1862
|
+
activity(
|
|
1863
|
+
io,
|
|
1864
|
+
`git.credentials "${entry.match}": push Secret ${entry.secret} — pushes over ` +
|
|
1865
|
+
`${entry.push.map((k) => (k === "ssh" ? "ssh" : "https")).join(" and ")}` +
|
|
1866
|
+
`${entry.sign ? `; signs with ${entry.sign}` : ""}`,
|
|
1867
|
+
);
|
|
1868
|
+
}
|
|
1869
|
+
for (const [match, why] of skipped) activity(io, `git.credentials "${match}": ${why}`);
|
|
1870
|
+
// A slot the Machine binds to an https url: its token's value is known now. An unset one is
|
|
1871
|
+
// refused, with the fix, as the clone's token is.
|
|
1872
|
+
for (const plan of sandbox) {
|
|
1873
|
+
for (const push of plan.pushes) {
|
|
1874
|
+
if (push.url === undefined || push.match === undefined) continue;
|
|
1875
|
+
if (repoIdentity(push.url).scheme === "ssh") continue;
|
|
1876
|
+
const placed = out.manifest.entries.find((e) => e.match === push.match);
|
|
1877
|
+
const why = skipped.get(push.match);
|
|
1878
|
+
if (!placed?.push.includes("token") && why !== undefined) {
|
|
1879
|
+
throw new Error(
|
|
1880
|
+
`${plan.label}: the push of Repo Slot "${push.slot}" (${push.url}): git.credentials entry ` +
|
|
1881
|
+
`"${push.match}" cannot push — ${why} (ADR-0051)`,
|
|
1882
|
+
);
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
return out;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
/**
|
|
1890
|
+
* What each Workspace's Sandbox Effector image owes its pushes (ADR-0068), read off the image
|
|
1891
|
+
* this converge built — its effector image, else its Sandbox Image — by the push forms `jr2 up` can
|
|
1892
|
+
* see: a slot the Machine binds (its url's scheme, its entry's `sign`), never one the run or a
|
|
1893
|
+
* `customize()` binds, whose url is the provision's. The stock Harness image — the fallback when the
|
|
1894
|
+
* Workspace names no image and there is no `images/default` — is the kit's, so what it holds is
|
|
1895
|
+
* known, not read (STOCK_HARNESS_IMAGE). An author's image `jr2 up` did not build (a registry ref)
|
|
1896
|
+
* is not read, as its `USER` is not.
|
|
1897
|
+
*/
|
|
1898
|
+
async function checkEffectorImages(
|
|
1899
|
+
build: BuildPort,
|
|
1900
|
+
config: JR2Config,
|
|
1901
|
+
sandbox: readonly SandboxEffectorPlan[],
|
|
1902
|
+
converged: ConvergedImages,
|
|
1903
|
+
platforms: readonly string[],
|
|
1904
|
+
/** Each `file:` context's key in the map: the digest of its staged copy (ADR-0069). */
|
|
1905
|
+
contextDigests: ReadonlyMap<string, string>,
|
|
1906
|
+
): Promise<void> {
|
|
1907
|
+
const list = config.git?.credentials ?? [];
|
|
1908
|
+
for (const plan of sandbox) {
|
|
1909
|
+
const forms: Array<{ form: PushForm; push: string }> = [];
|
|
1910
|
+
for (const push of plan.pushes) {
|
|
1911
|
+
if (push.url === undefined || push.match === undefined) continue;
|
|
1912
|
+
const what = `the push of Repo Slot "${push.slot}" (${push.url})`;
|
|
1913
|
+
if (isSshUrl(push.url)) forms.push({ form: "ssh-transport", push: what });
|
|
1914
|
+
const sign = push.sign ? list.find((e) => e.match === push.match)?.sign : undefined;
|
|
1915
|
+
if (sign) forms.push({ form: "gpgKey" in sign ? "gpg-sign" : "ssh-sign", push: what });
|
|
1916
|
+
}
|
|
1917
|
+
if (forms.length === 0) continue;
|
|
1918
|
+
const image = plan.image ?? plan.sandboxImage;
|
|
1919
|
+
let key = "default";
|
|
1920
|
+
if (image !== undefined) {
|
|
1921
|
+
if (!image.startsWith("file:")) continue;
|
|
1922
|
+
// A Workspace's image is a context the walk carried, so it was keyed with the Assembly.
|
|
1923
|
+
const digest = contextDigests.get(fileURLToPath(image));
|
|
1924
|
+
if (digest === undefined) continue;
|
|
1925
|
+
key = digest;
|
|
1926
|
+
}
|
|
1927
|
+
const ref = converged.sandbox[key];
|
|
1928
|
+
if (ref === undefined) {
|
|
1929
|
+
if (image !== undefined) continue;
|
|
1930
|
+
// No image named and no images/default: the Effector runs in the stock Harness image.
|
|
1931
|
+
const refusal = effectorImageRefusal({
|
|
1932
|
+
label: plan.label,
|
|
1933
|
+
ref: converged.harness,
|
|
1934
|
+
user: STOCK_HARNESS_IMAGE.user,
|
|
1935
|
+
forms,
|
|
1936
|
+
present: new Set(STOCK_HARNESS_IMAGE.present),
|
|
1937
|
+
passwd: STOCK_HARNESS_IMAGE.passwd,
|
|
1938
|
+
stock: true,
|
|
1939
|
+
});
|
|
1940
|
+
if (refusal) throw new Error(refusal);
|
|
1941
|
+
continue;
|
|
1942
|
+
}
|
|
1943
|
+
const files = await build.imageFiles(ref, platforms, effectorImagePaths(forms.map((f) => f.form)));
|
|
1944
|
+
const refusal = effectorImageRefusal({
|
|
1945
|
+
label: plan.label,
|
|
1946
|
+
ref,
|
|
1947
|
+
user: converged.sandboxUser[key] ?? "",
|
|
1948
|
+
forms,
|
|
1949
|
+
present: new Set(files.present),
|
|
1950
|
+
passwd: files.passwd,
|
|
1951
|
+
});
|
|
1952
|
+
if (refusal) throw new Error(refusal);
|
|
1953
|
+
}
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1127
1956
|
/** Where the git ssh key comes from — the three sources ADR-0047 offers, as a pick. */
|
|
1128
1957
|
type GitSshSource = { kind: "generate" } | { kind: "file"; path: string } | { kind: "paste" };
|
|
1129
1958
|
|
|
@@ -1452,7 +2281,7 @@ async function preflightProvider(
|
|
|
1452
2281
|
io: Io,
|
|
1453
2282
|
kube: KubeAdmin,
|
|
1454
2283
|
config: JR2Config,
|
|
1455
|
-
|
|
2284
|
+
carriedModels: readonly string[],
|
|
1456
2285
|
namespace: string,
|
|
1457
2286
|
ctx: { context?: string },
|
|
1458
2287
|
caPem?: string,
|
|
@@ -1467,14 +2296,8 @@ async function preflightProvider(
|
|
|
1467
2296
|
// An Open model is nobody's model yet (ADR-0054), so there is nothing to probe for it. The walk
|
|
1468
2297
|
// already refused the converge above, so this filter is the type's, not the flow's.
|
|
1469
2298
|
const prefix = `${provider.id}/`;
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
agents
|
|
1473
|
-
.map((a) => a.definition.model)
|
|
1474
|
-
.filter((m): m is string => typeof m === "string" && m.startsWith(prefix))
|
|
1475
|
-
.map((m) => m.slice(prefix.length)),
|
|
1476
|
-
),
|
|
1477
|
-
];
|
|
2299
|
+
// An Open body's allowed models are probed too (ADR-0070): a Chart's own Agent may name one.
|
|
2300
|
+
const models = [...new Set(carriedModels.filter((m) => m.startsWith(prefix)).map((m) => m.slice(prefix.length)))];
|
|
1478
2301
|
if (models.length === 0) {
|
|
1479
2302
|
activity(
|
|
1480
2303
|
io,
|
|
@@ -1543,8 +2366,36 @@ function noteDeferred(io: Io, repos: CarriedRepo[]): void {
|
|
|
1543
2366
|
if (repos.length) {
|
|
1544
2367
|
activity(
|
|
1545
2368
|
io,
|
|
1546
|
-
`repos: ${repos.length} bound Repo(s) — the Orchestrator creates their Repo resources at boot; the cache ` +
|
|
2369
|
+
`repos: ${repos.length} bound Repo(s) — the Orchestrator creates their Repo resources at boot and at each publish (ADR-0069); the cache ` +
|
|
1547
2370
|
"agent clones on first need (`jr2 status` reports sync state)",
|
|
1548
2371
|
);
|
|
1549
2372
|
}
|
|
1550
2373
|
}
|
|
2374
|
+
|
|
2375
|
+
/**
|
|
2376
|
+
* The `customize` line that retunes a composed Machine's Size (ADR-0060), nested through `actors`
|
|
2377
|
+
* the way `customize()` accepts — the Size twin of `customizeLine`, so the refusal pastes.
|
|
2378
|
+
*/
|
|
2379
|
+
function sizeLine(path: readonly string[]): string {
|
|
2380
|
+
const binds = '{ resources: { limits: { memory: "<memory>" } } }';
|
|
2381
|
+
return `customize(<import>, ${path.reduceRight((parts, key) => `{ actors: { ${key}: ${parts} } }`, binds)})`;
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
/**
|
|
2385
|
+
* The Chart files under an Assembly root (ADR-0070), as Workflow names `<wrapper>/<name>`: each
|
|
2386
|
+
* `workflows/<wrapper>/<name>.chart.json`, one level below the flat `workflows/*.ts`, in a directory
|
|
2387
|
+
* named after its wrapper. The layout the Orchestrator's discovery reads (charts.ts); a directory
|
|
2388
|
+
* whose name starts with `_` or `.` is not a wrapper's.
|
|
2389
|
+
*/
|
|
2390
|
+
async function chartFilesIn(root: string): Promise<string[]> {
|
|
2391
|
+
const out: string[] = [];
|
|
2392
|
+
const dir = join(root, "workflows");
|
|
2393
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
2394
|
+
for (const entry of entries) {
|
|
2395
|
+
if (!entry.isDirectory() || entry.name.startsWith("_") || entry.name.startsWith(".")) continue;
|
|
2396
|
+
const files = await readdir(join(dir, entry.name)).catch(() => [] as string[]);
|
|
2397
|
+
for (const f of files)
|
|
2398
|
+
if (f.endsWith(".chart.json")) out.push(`${entry.name}/${f.slice(0, -".chart.json".length)}`);
|
|
2399
|
+
}
|
|
2400
|
+
return out.sort();
|
|
2401
|
+
}
|