@jr2/cli 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/deploy.ts CHANGED
@@ -3,20 +3,31 @@
3
3
  // namespace are constants (namespace is the identity); every object carries the instance label so
4
4
  // ownership is derivable from the cluster — there is no local target state.
5
5
 
6
+ import { createHmac } from "node:crypto";
6
7
  import { readFile } from "node:fs/promises";
7
8
  import { fileURLToPath } from "node:url";
8
9
  import {
9
10
  CA_CONFIGMAP,
11
+ CA_MOUNT,
12
+ EFFECTOR_PORT,
13
+ INSTANCE_EFFECTOR,
14
+ CONVERSATIONS_DIR,
15
+ CONVERSATIONS_DIR_ENV,
16
+ CONVERSATIONS_VOLUME,
10
17
  custodianComposition,
11
18
  CUSTODIAN_BOOTSTRAP_KEY,
19
+ CUSTODIAN_PROGRAM,
20
+ CUSTODIAN_PROGRAM_KEY,
12
21
  CUSTODIAN_SCRIPT_KEY,
22
+ INSTANCE_CUSTODIAN_CONFIGMAP,
13
23
  GIT_SSH_SECRET,
14
24
  HELD_CA_SECRET,
15
25
  HELD_CONFIGMAP,
16
- HELD_KEY,
17
26
  HELD_MOUNT,
18
- HELD_SECRETS_SECRET,
19
27
  HELD_TLS_SECRET,
28
+ heldManifestKey,
29
+ heldSecretsSecret,
30
+ INSTANCE_HARNESS_HELD_POD,
20
31
  HARNESS_ENV_SECRET,
21
32
  IMAGES_CONFIGMAP,
22
33
  IMAGES_KEY,
@@ -27,18 +38,35 @@ import {
27
38
  INSTANCE_HARNESS_SERVICE,
28
39
  INSTANCE_SECRET,
29
40
  KIT_VERSION,
41
+ NO_DISRUPT_ANNOTATIONS,
30
42
  ORCHESTRATOR_PORT,
31
43
  ORCHESTRATOR_SERVICE,
32
44
  REPO_CACHE,
45
+ PRIORITY_CLASS_CONTROL,
46
+ PRIORITY_CLASS_SANDBOX,
33
47
  REPO_CACHE_HOSTPATH,
48
+ SERVICE_ACCOUNT_CA,
34
49
  STATE_PVC,
50
+ type EffectorPlan,
51
+ type SandboxEffectorPlan,
52
+ effectorManifestKey,
53
+ LABEL_EFFECTOR_POD,
54
+ sandboxEffectorObjectsOf,
55
+ GIT_PUSH_MANIFEST_KEY,
56
+ LABEL_GIT_PUSH,
57
+ GIT_CLONE_MANIFEST_KEY,
58
+ LABEL_GIT_CLONE,
59
+ type GitCloneSecret,
60
+ type GitPushManifest,
35
61
  type HarnessConfig,
36
62
  type HeldManifest,
63
+ type PriorityClasses,
37
64
  type SandboxPlacement,
38
65
  } from "@jr2/orchestrator";
39
66
  import type { Pem } from "./held-pki.ts";
40
67
 
41
68
  export {
69
+ INSTANCE_EFFECTOR,
42
70
  GIT_SSH_SECRET,
43
71
  HARNESS_CONFIGMAP,
44
72
  HARNESS_ENV_SECRET,
@@ -50,7 +78,6 @@ export {
50
78
 
51
79
  export const LABEL_INSTANCE = "jr2.dev/instance";
52
80
  export const LABEL_VERSION = "jr2.dev/version";
53
- export const LABEL_HASH = "jr2.dev/content-hash";
54
81
 
55
82
  /** The converged name→ref image map, stamped on the Orchestrator Deployment's OWN metadata so the
56
83
  * next `jr2 up` can diff it and spend no docker on what has not moved (ADR-0038). An ANNOTATION, not
@@ -66,6 +93,20 @@ export const ANNOTATION_HELD_DIGEST = "jr2.dev/held-digest";
66
93
  * same reason: a group every container is granted, which the Custodian's values are readable by. */
67
94
  const INSTANCE_HARNESS_FS_GROUP = 2000;
68
95
 
96
+ /** How long the Instance Harness pod is given to stop (ADR-0031's drain): on SIGTERM it stops
97
+ * admitting and finishes the Turns it holds, so the grace is a Turn's worst case. `jr2 up` waits
98
+ * out a rollout that long, and no longer. */
99
+ export const INSTANCE_HARNESS_DRAIN_SECONDS = 600;
100
+
101
+ /** The Instance Harness's conversations volume (ADR-0031): the StatefulSet's one claim, and the
102
+ * PVC ordinal 0 gets from it — `<claim>-<statefulset>-<ordinal>`. */
103
+ export const INSTANCE_HARNESS_PVC = `${CONVERSATIONS_VOLUME}-${INSTANCE_HARNESS_SERVICE}-0`;
104
+
105
+ /** The Instance Harness StatefulSet's governing Service (ADR-0031): headless, and dialed by no one
106
+ * today — it is what gives each ordinal the stable name N > 1 would pin a conversation to. The
107
+ * Orchestrator dials the ClusterIP Service `INSTANCE_HARNESS_SERVICE`, never this one. */
108
+ export const INSTANCE_HARNESS_PODS_SERVICE = `${INSTANCE_HARNESS_SERVICE}-pods`;
109
+
69
110
  /** The two ingress NetworkPolicies (ADR-0058) — see `harnessIngressPolicies`. */
70
111
  export const SANDBOX_INGRESS_POLICY = "jr2-sandbox-ingress";
71
112
  export const INSTANCE_HARNESS_INGRESS_POLICY = "jr2-instance-harness-ingress";
@@ -77,13 +118,21 @@ export const OPERATOR_DEPLOYMENT = "jr2-controller-manager";
77
118
  export const OPERATOR_SELECTOR = "control-plane=controller-manager";
78
119
 
79
120
  /** The rendered operator install manifest shipped inside this package (`just operator-manifest`
80
- * regenerates it from operator/config). The manager image ref is substituted at apply time. */
81
- export async function operatorManifest(image: string): Promise<string> {
121
+ * regenerates it from operator/config). The manager image ref is substituted at apply time, and so
122
+ * is its PriorityClass (ADR-0060): the manifest names `jr2-control`, and an Instance that names its
123
+ * own control class gets no pod that names one `jr2 up` did not create. */
124
+ export async function operatorManifest(image: string, controlClass = PRIORITY_CLASS_CONTROL): Promise<string> {
82
125
  const raw = await readFile(fileURLToPath(new URL("../manifests/operator.yaml", import.meta.url)), "utf8");
83
126
  if (!raw.includes("image: controller:latest")) {
84
127
  throw new Error("packaged operator.yaml has no `image: controller:latest` placeholder — regenerate it");
85
128
  }
86
- return raw.replace("image: controller:latest", `image: ${image}`);
129
+ const classLine = `priorityClassName: ${PRIORITY_CLASS_CONTROL}`;
130
+ if (!raw.includes(classLine)) {
131
+ throw new Error(`packaged operator.yaml has no \`${classLine}\` — regenerate it`);
132
+ }
133
+ return raw
134
+ .replace("image: controller:latest", `image: ${image}`)
135
+ .replace(classLine, `priorityClassName: ${controlClass}`);
87
136
  }
88
137
 
89
138
  /** Compare dotted versions: negative when a < b, 0 when equal, positive when a > b. */
@@ -100,30 +149,129 @@ export function compareVersions(a: string, b: string): number {
100
149
  type KubeManifest = Record<string, unknown>;
101
150
 
102
151
  /**
103
- * What `jr2 up` resolved about held secrets (ADR-0059), ready to become objects. Values appear in
104
- * two places only: `values` (the literals, for the `jr2-held-secrets` Secret) and the CA's and the
105
- * leaves' keys — each lands in a Secret no Harness container mounts.
152
+ * jr2's own pods are kit-sized, with no config key (ADR-0060): memory request = limit, a CPU request
153
+ * and NO CPU limit — a provisioning burst needs CPU, and no Agent Substrate move applies to them.
154
+ * The Repo cache agent is the one exception to request = limit: `git index-pack` spikes, and its OOM
155
+ * costs only a refetch. The operator's own size rides its manifest (operator/config).
106
156
  */
107
- export type HeldObjects = {
157
+ export const KIT_POD_SIZES = {
158
+ orchestrator: { requests: { cpu: "500m", memory: "1Gi" }, limits: { memory: "1Gi" } },
159
+ /** It holds the conversations its live runs hold, and frees the rest on the live set (ADR-0031). */
160
+ instanceHarness: { requests: { cpu: "250m", memory: "1Gi" }, limits: { memory: "1Gi" } },
161
+ /** The Instance Harness pod's Custodian: the same 64Mi/50m slice a Sandbox's takes, kit-sized. */
162
+ instanceCustodian: { requests: { cpu: "50m", memory: "64Mi" }, limits: { memory: "64Mi" } },
163
+ repoCache: { requests: { cpu: "50m", memory: "128Mi" }, limits: { memory: "1Gi" } },
164
+ /** The Instance Effector (ADR-0068): one http call at a time per Effect, its text body bounded. */
165
+ instanceEffector: { requests: { cpu: "50m", memory: "128Mi" }, limits: { memory: "128Mi" } },
166
+ } as const;
167
+
168
+ /** A fresh copy, so no manifest shares an object with another (or with the constant). */
169
+ const sized = (r: { requests: Record<string, string>; limits: Record<string, string> }) => ({
170
+ requests: { ...r.requests },
171
+ limits: { ...r.limits },
172
+ });
173
+
174
+ /** The PriorityClass each of jr2's two tiers runs under (ADR-0060): the Instance's own when
175
+ * `priorityClasses` names one, else the one `jr2 up` creates. */
176
+ export function priorityClassNames(classes: PriorityClasses | undefined): { control: string; sandbox: string } {
177
+ return {
178
+ control: classes?.control ?? PRIORITY_CLASS_CONTROL,
179
+ sandbox: classes?.sandbox ?? PRIORITY_CLASS_SANDBOX,
180
+ };
181
+ }
182
+
183
+ /**
184
+ * The PriorityClasses `jr2 up` creates, cluster-scoped, beside the CRDs (ADR-0060) — one manifest
185
+ * each, and none for a key `priorityClasses` names, since the cluster then has its own scheme.
186
+ * `jr2-control` preempts lower priorities: the Orchestrator, the operator, the Repo cache agent and the
187
+ * Instance Effector are small and fixed, and the first three are needed by every Sandbox.
188
+ * `jr2-sandbox` never preempts: a waiting Sandbox evicts nobody, and an ordinary pod (priority 0)
189
+ * cannot preempt a live one. Both stay far below the
190
+ * system classes (2000000000+). Shared by every Instance on the cluster, like the operator, so they
191
+ * carry no instance label; a class's value and policy are immutable, and these never change.
192
+ */
193
+ export function priorityClassObjects(classes: PriorityClasses | undefined): string[] {
194
+ const managed = { "app.kubernetes.io/managed-by": "jr2" };
195
+ const pc = (name: string, value: number, preemptionPolicy: string, description: string): string =>
196
+ JSON.stringify({
197
+ apiVersion: "scheduling.k8s.io/v1",
198
+ kind: "PriorityClass",
199
+ metadata: { name, labels: managed },
200
+ value,
201
+ preemptionPolicy,
202
+ globalDefault: false,
203
+ description,
204
+ });
205
+ return [
206
+ ...(classes?.control === undefined
207
+ ? [
208
+ pc(
209
+ PRIORITY_CLASS_CONTROL,
210
+ 100000,
211
+ "PreemptLowerPriority",
212
+ "jr2 control: the Orchestrator, the operator and the Repo cache agent (ADR-0060)",
213
+ ),
214
+ ]
215
+ : []),
216
+ ...(classes?.sandbox === undefined
217
+ ? [
218
+ pc(
219
+ PRIORITY_CLASS_SANDBOX,
220
+ 1000,
221
+ "Never",
222
+ "jr2 Sandboxes and the Instance Harness: never preempted by ordinary pods, never preempting (ADR-0060)",
223
+ ),
224
+ ]
225
+ : []),
226
+ ];
227
+ }
228
+
229
+ /**
230
+ * One Harness pod's held objects (ADR-0059): what `jr2 up` placed in it, ready to become objects.
231
+ * A Workspace's, or the Instance Harness's.
232
+ */
233
+ export type HeldPodObjects = {
234
+ /** The pod key (`INSTANCE_HARNESS_HELD_POD`, `workspaceHeldPod`). */
235
+ pod: string;
108
236
  manifest: HeldManifest;
109
- /** The literal values, by name — never in the manifest, never in a ConfigMap. */
237
+ /** Its literal values, by name — for its `jr2-held-secrets-<pod>` Secret; never in the manifest,
238
+ * never in a ConfigMap. */
110
239
  values: Record<string, string>;
240
+ };
241
+
242
+ /** A Custodian's two rendered files: Envoy's bootstrap (JSON) and the `HELD` table script. */
243
+ export type CustodianFiles = { bootstrap: string; script: string };
244
+
245
+ /**
246
+ * What `jr2 up` resolved about held secrets (ADR-0059), ready to become objects. Values appear in
247
+ * two places only: each pod's `values` (the literals, for its own `jr2-held-secrets-<pod>` Secret)
248
+ * and the CA's and the leaves' keys — each lands in a Secret no Harness container mounts.
249
+ */
250
+ export type HeldObjects = {
251
+ /** Every Harness pod the walk found, each with what its Custodian holds. */
252
+ pods: HeldPodObjects[];
253
+ /** The Instance Harness's Custodian files, when the walk placed it — the one pod `jr2 up` makes,
254
+ * so the one it renders for; a Sandbox's are rendered by the Orchestrator at provision
255
+ * (`kubeSandbox`). */
256
+ instance?: CustodianFiles;
111
257
  /** The per-Instance CA — kept across converges; present whenever a secret is held. */
112
258
  ca?: Pem;
113
- /** One leaf per bound target, by file stem (`leafStem`). */
259
+ /** One leaf per bound target of any pod, by file stem (`leafStem`). */
114
260
  leaves: Record<string, Pem>;
115
- /** The Custodian's rendered bootstrap (JSON) and script. */
116
- bootstrap: string;
117
- script: string;
118
261
  };
119
262
 
263
+ /** Does any pod hold a secret? */
264
+ export const holdsAny = (held: HeldObjects): boolean => held.pods.some((p) => p.manifest.secrets.length > 0);
265
+
120
266
  /** Everything `jr2 up` converges inside the instance's namespace, as one apply-able List. */
121
267
  export function instanceObjects(opts: {
122
268
  name: string;
123
269
  namespace: string;
270
+ /** The Orchestrator Kit image's ref (ADR-0069). The Deployment carries no content hash: the
271
+ * Assembly's hash is an Orchestrator fact, read from `GET /assembly`. */
124
272
  image: string;
125
- hash: string;
126
- /** Secret data (token, signing key, harness env literals) — written stringData, kube encodes. */
273
+ /** Secret data (the Instance token, the publish token, the signing key) — written stringData,
274
+ * kube encodes. The Orchestrator reads it through `envFrom`. */
127
275
  secretData: Record<string, string>;
128
276
  /** The HARNESS containers' literal `harness.env` values — a SEPARATE Secret from `secretData` by
129
277
  * doctrine (ADR-0013): Agent code executes where this lands, so the Instance token and signing
@@ -145,8 +293,20 @@ export function instanceObjects(opts: {
145
293
  * resolved operator ref — the same binary is the cache agent (`/manager repo-cache`). Absent, no
146
294
  * DaemonSet and none of its RBAC is applied; `up` deletes a stale one. */
147
295
  repoCache?: { image: string; placement?: SandboxPlacement };
296
+ /** `priorityClasses` from jr2.config.ts (ADR-0060): the classes to name instead of jr2's own. */
297
+ priorityClasses?: PriorityClasses;
298
+ /** Each Workspace's Sandbox Effector (ADR-0068): its manifest rides `jr2-held`, its
299
+ * `jr2-effector-<pod>` Secret holds its literals and `effector.json`. */
300
+ sandboxEffectors?: readonly SandboxEffectorPlan[];
301
+ /** Each `git.credentials` entry's push Secret and the manifest the provision reads (ADR-0051). */
302
+ gitPush?: { secrets: ReadonlyArray<{ name: string; data: Record<string, string> }>; manifest: GitPushManifest };
303
+ /** Each `git.credentials` entry's clone Secret (ADR-0068): the token the cache agent spends. The
304
+ * Orchestrator names it from `git-clone.json` and never holds it. */
305
+ gitClone?: readonly GitCloneSecret[];
148
306
  }): string {
149
307
  const labels = { [LABEL_INSTANCE]: opts.name, "app.kubernetes.io/managed-by": "jr2" };
308
+ const priority = priorityClassNames(opts.priorityClasses);
309
+ const effectors = (opts.sandboxEffectors ?? []).map((plan) => ({ plan, ...sandboxEffectorObjectsOf(plan) }));
150
310
  const meta = (name: string, extra: Record<string, string> = {}): KubeManifest => ({
151
311
  name,
152
312
  namespace: opts.namespace,
@@ -167,21 +327,27 @@ export function instanceObjects(opts: {
167
327
  { apiVersion: "v1", kind: "ServiceAccount", metadata: meta(ORCHESTRATOR_SA) },
168
328
  {
169
329
  // The orchestrator drives Sandbox CRs (+ their token Secrets) in its own namespace
170
- // (ADR-0012/0013) and creates the Repo CRs the cache agent reconciles (ADR-0051); pod
171
- // exec/port-forward are the attach path (ADR-0004).
330
+ // (ADR-0012/0013), watches them (ADR-0063), and creates the Repo CRs the cache agent
331
+ // reconciles (ADR-0051). Nothing on Pods: it never reads one — the operator publishes the pod
332
+ // facts on the Sandbox's status — and the attach is the Harness's own `POST /attach`, so no
333
+ // `pods/exec` stream crosses the API server (ADR-0063).
172
334
  apiVersion: "rbac.authorization.k8s.io/v1",
173
335
  kind: "Role",
174
336
  metadata: meta(ORCHESTRATOR_SA),
175
337
  rules: [
176
338
  { apiGroups: ["core.jr2.dev"], resources: ["sandboxes", "repos"], verbs: ["*"] },
177
- // patch/update: the token Secret is `kubectl apply`d idempotently and later ownerRef-patched.
178
- {
179
- apiGroups: [""],
180
- resources: ["secrets"],
181
- verbs: ["get", "list", "create", "delete", "patch", "update"],
182
- },
183
- { apiGroups: [""], resources: ["pods", "pods/log"], verbs: ["get", "list", "watch"] },
184
- { apiGroups: [""], resources: ["pods/exec", "pods/portforward"], verbs: ["create"] },
339
+ // Secrets: create and patch, nothing else (ADR-0068). The Orchestrator server-side
340
+ // applies its own per-Sandbox `<sandbox>-token` and `<sandbox>-effector`, owned by the CR,
341
+ // and holds no git credential and no Effect secret (ADR-0068): no get, no list, so it cannot read or enumerate the Effect
342
+ // secrets, the clone and push Secrets, or a user's Secret through the API. RBAC cannot
343
+ // pin `create` to a name, and `resourceNames` has no prefix form.
344
+ { apiGroups: [""], resources: ["secrets"], verbs: ["create", "patch"] },
345
+ // A Sandbox's Custodian files, rendered at provision and server-side applied (ADR-0059);
346
+ // owned by the CR, so the garbage collector deletes them with the pod.
347
+ { apiGroups: [""], resources: ["configmaps"], verbs: ["create", "patch"] },
348
+ // A Sandbox Effector's headless Service, written at provision beside its key (ADR-0068): how
349
+ // this process reaches the pod's own address. Owned by the CR, like the ConfigMap.
350
+ { apiGroups: [""], resources: ["services"], verbs: ["create", "patch"] },
185
351
  ],
186
352
  },
187
353
  {
@@ -235,20 +401,77 @@ export function instanceObjects(opts: {
235
401
  metadata: meta(IMAGES_CONFIGMAP),
236
402
  data: { [IMAGES_KEY]: imagesJson },
237
403
  },
238
- // Held secrets (ADR-0059). `held.json` is what every composition reads — the Orchestrator per
239
- // provision from the mount below, `jr2 up` for the Instance Harness — beside the Custodian's
240
- // bootstrap and script, which Envoy reads at start. Names, hosts, headers: never a value.
404
+ // Held secrets (ADR-0059): what `jr2 up` DECLARES. A pod's `held-<pod>.json` is what its
405
+ // composition reads — the Orchestrator per provision from the mount below, `jr2 up` for the
406
+ // Instance Harness — and the Custodian's program is here once. Envoy's per-pod files are
407
+ // rendered by whoever makes the pod, never collected here: a shared object stops at 1 MiB.
408
+ // Names, hosts, headers: never a value.
241
409
  {
242
410
  apiVersion: "v1",
243
411
  kind: "ConfigMap",
244
412
  metadata: meta(HELD_CONFIGMAP),
245
413
  data: {
246
- [HELD_KEY]: JSON.stringify(opts.held.manifest, null, 2),
247
- [CUSTODIAN_BOOTSTRAP_KEY]: opts.held.bootstrap,
248
- [CUSTODIAN_SCRIPT_KEY]: opts.held.script,
414
+ ...Object.fromEntries(opts.held.pods.map((p) => [heldManifestKey(p.pod), JSON.stringify(p.manifest, null, 2)])),
415
+ [CUSTODIAN_PROGRAM_KEY]: CUSTODIAN_PROGRAM,
416
+ // Each Sandbox Effector's manifest (ADR-0068): the sources of its secrets mount, by name.
417
+ // The provision adds the container only for a pod whose manifest is here.
418
+ ...Object.fromEntries(
419
+ effectors.map((e) => [effectorManifestKey(e.plan.pod), JSON.stringify(e.manifest, null, 2)]),
420
+ ),
421
+ // The push manifest (ADR-0051): which entries have a push Secret, and what each can spend.
422
+ // Names only; the provision names the matched entry's Secret on the Sandbox CR.
423
+ ...(opts.gitPush?.manifest.entries.length
424
+ ? { [GIT_PUSH_MANIFEST_KEY]: JSON.stringify(opts.gitPush.manifest, null, 2) }
425
+ : {}),
426
+ // The clone manifest (ADR-0068): which entries' clone Secrets exist, by name. The Orchestrator
427
+ // names one as a Repo's `secretRef` only when it is listed here.
428
+ ...(opts.gitClone?.length
429
+ ? {
430
+ [GIT_CLONE_MANIFEST_KEY]: JSON.stringify(
431
+ { version: 1, secrets: opts.gitClone.map((s) => s.name) },
432
+ null,
433
+ 2,
434
+ ),
435
+ }
436
+ : {}),
249
437
  },
250
438
  },
439
+ // Each entry's clone Secret (ADR-0068), in Flux's key names: the operator's cache agent reads the
440
+ // one a Repo's `secretRef` names. The Orchestrator neither writes nor reads it.
441
+ ...(opts.gitClone ?? []).map(
442
+ (s): KubeManifest => ({
443
+ apiVersion: "v1",
444
+ kind: "Secret",
445
+ metadata: meta(s.name, { [LABEL_GIT_CLONE]: "true" }),
446
+ type: "Opaque",
447
+ stringData: s.data,
448
+ }),
449
+ ),
450
+ // Each entry's push Secret (ADR-0051): what it pushes and signs with, in Flux's key names where
451
+ // Flux has them. The operator projects one into the Sandbox Effector of a Sandbox whose push
452
+ // the entry's match covers; nothing else mounts it.
453
+ ...(opts.gitPush?.secrets ?? []).map(
454
+ (s): KubeManifest => ({
455
+ apiVersion: "v1",
456
+ kind: "Secret",
457
+ metadata: meta(s.name, { [LABEL_GIT_PUSH]: "true" }),
458
+ type: "Opaque",
459
+ data: s.data,
460
+ }),
461
+ ),
251
462
  ...heldSecretObjects(opts.held, meta),
463
+ // Each Sandbox Effector's own Secret (ADR-0068): the literals its Effects name and
464
+ // `effector.json`. A `valueFrom` stays in the user's Secret, projected by reference at
465
+ // provision.
466
+ ...effectors.map(
467
+ (e): KubeManifest => ({
468
+ apiVersion: "v1",
469
+ kind: "Secret",
470
+ metadata: meta(e.secret.name, { [LABEL_EFFECTOR_POD]: e.plan.pod }),
471
+ type: "Opaque",
472
+ stringData: e.secret.stringData,
473
+ }),
474
+ ),
252
475
  {
253
476
  apiVersion: "v1",
254
477
  kind: "Secret",
@@ -270,7 +493,7 @@ export function instanceObjects(opts: {
270
493
  apiVersion: "apps/v1",
271
494
  kind: "Deployment",
272
495
  metadata: {
273
- ...meta(ORCHESTRATOR_SERVICE, { [LABEL_HASH]: opts.hash, [LABEL_VERSION]: KIT_VERSION }),
496
+ ...meta(ORCHESTRATOR_SERVICE, { [LABEL_VERSION]: KIT_VERSION }),
274
497
  // The converged image map, on the DEPLOYMENT'S OWN metadata and never on
275
498
  // `spec.template.metadata` (ADR-0038). On the pod template it would be part of the pod
276
499
  // spec, so every re-resolved ref would roll the Orchestrator — the exact cost the
@@ -279,13 +502,22 @@ export function instanceObjects(opts: {
279
502
  annotations: { [ANNOTATION_IMAGES]: imagesJson },
280
503
  },
281
504
  spec: {
282
- replicas: 1, // single WRITER (CONTEXT.md): the snapshot store brooks no split-brain
505
+ replicas: 1, // single WRITER (GLOSSARY.md): the snapshot store brooks no split-brain
283
506
  strategy: { type: "Recreate" }, // two writers may never overlap on the PVC
284
507
  selector: { matchLabels: { app: ORCHESTRATOR_SERVICE } },
285
508
  template: {
286
- metadata: { labels: { ...labels, app: ORCHESTRATOR_SERVICE } },
509
+ metadata: {
510
+ labels: { ...labels, app: ORCHESTRATOR_SERVICE },
511
+ // The Instance Secret reaches the process through `envFrom`, which a running pod reads
512
+ // once. A minted replacement for a lost Secret (a new publish token above all,
513
+ // ADR-0069) must roll the pod, or every publish after it is refused with 401.
514
+ annotations: { [ANNOTATION_SECRET_DIGEST]: instanceSecretDigest(opts.secretData) },
515
+ },
287
516
  spec: {
288
517
  serviceAccountName: ORCHESTRATOR_SA,
518
+ // Control tier (ADR-0060): small, fixed, needed by every Sandbox. No disruption opt-out:
519
+ // a restarted Orchestrator re-attaches (ADR-0007).
520
+ priorityClassName: priority.control,
289
521
  containers: [
290
522
  {
291
523
  name: "orchestrator",
@@ -296,23 +528,26 @@ export function instanceObjects(opts: {
296
528
  env: [
297
529
  // The entrypoint derives its own Service DNS + Sandbox namespace from these.
298
530
  { name: "JR2_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
299
- // What `/healthz` reports as this instance's identity. The same content address
300
- // the image tag carries (ADR-0019), in-process so a CLI can ask over HTTP
301
- // instead of needing kube access to read the Deployment's labels.
302
- { name: "JR2_CONTENT_HASH", value: opts.hash },
531
+ // The API server's trust (ADR-0063): the Orchestrator's own Kubernetes client is
532
+ // built-in `fetch`, which trusts the ServiceAccount's CA through this and nothing
533
+ // else — no TLS code, no dependency.
534
+ { name: "NODE_EXTRA_CA_CERTS", value: SERVICE_ACCOUNT_CA },
303
535
  ],
304
536
  // The Orchestrator creates Repo resources and never clones (ADR-0051) — the cache
305
537
  // agent on each node does, reading the credential Secret a Repo's `secretRef`
306
- // names — so nothing of git's is mounted here: state and the image map only.
538
+ // names, which `jr2 up` wrote (ADR-0068) — so nothing of git's is mounted here and no
539
+ // git token is in this container's env: state, the image map and the manifests.
307
540
  volumeMounts: [
308
541
  { name: "state", mountPath: "/instance/.jr2" },
309
542
  // The image map, read per provision (ADR-0038). A mount, so `jr2 up` rewriting
310
543
  // it costs one kubelet propagation window instead of a rollout.
311
544
  { name: "images", mountPath: IMAGES_MOUNT, readOnly: true },
312
- // What the Custodian holds, read per provision the same way (ADR-0059): names
313
- // and hosts, which this process could not derive — its `.env` values are absent.
545
+ // What each Workspace's Custodian holds, read per provision the same way
546
+ // (ADR-0059): names and hosts, which this process could not derive — its `.env`
547
+ // values are absent.
314
548
  { name: "held", mountPath: HELD_MOUNT, readOnly: true },
315
549
  ],
550
+ resources: sized(KIT_POD_SIZES.orchestrator),
316
551
  // The period, not the boot, is what `up`'s rollout wait measures. Measured: the
317
552
  // container answers `/healthz` 1.1s after it starts, and the default 10s period
318
553
  // billed that as 11.0s — one probe fired before the server was up, then a whole
@@ -321,7 +556,7 @@ export function instanceObjects(opts: {
321
556
  // `failureThreshold` is then raised to hold the tolerance the default period gave,
322
557
  // because ONE knob sets two unrelated things: how fast a boot is noticed, and how
323
558
  // long a running pod may stall before the kubelet takes it out of service. The
324
- // Orchestrator is `replicas: 1` (CONTEXT.md: single writer), so its Service has
559
+ // Orchestrator is `replicas: 1` (GLOSSARY.md: single writer), so its Service has
325
560
  // exactly one endpoint and losing it is an outage, not a failover — and its store
326
561
  // writes are synchronous, so a GC pause or a node busy with a docker build can
327
562
  // silence `/healthz` for seconds. 15 × 2s keeps the 30s the default 3 × 10s gave.
@@ -331,12 +566,23 @@ export function instanceObjects(opts: {
331
566
  periodSeconds: 2,
332
567
  failureThreshold: 15,
333
568
  },
569
+ // Liveness restarts, which readiness never does (ADR-0008). A restart of this
570
+ // `replicas: 1` writer is an outage plus a restore, so it fires only on a process
571
+ // that has answered nothing for five minutes (30 × 10s): silent that long is dead,
572
+ // not stalled, and without it the pod sits NotReady until a human deletes it. A
573
+ // process that EXITS needs no probe — the restart policy already has it. The same
574
+ // five minutes cover a boot's restore, so there is no initial delay.
575
+ livenessProbe: {
576
+ httpGet: { path: "/healthz", port: ORCHESTRATOR_PORT },
577
+ periodSeconds: 10,
578
+ failureThreshold: 30,
579
+ },
334
580
  },
335
581
  ],
336
582
  volumes: [
337
583
  { name: "state", persistentVolumeClaim: { claimName: STATE_PVC } },
338
584
  { name: "images", configMap: { name: IMAGES_CONFIGMAP } },
339
- { name: "held", configMap: { name: HELD_CONFIGMAP, items: [{ key: HELD_KEY, path: HELD_KEY }] } },
585
+ { name: "held", configMap: { name: HELD_CONFIGMAP } },
340
586
  ],
341
587
  },
342
588
  },
@@ -352,7 +598,15 @@ export function instanceObjects(opts: {
352
598
  },
353
599
  },
354
600
  ...harnessIngressPolicies(opts.name, meta),
355
- ...(opts.repoCache ? repoCacheObjects({ ...opts.repoCache, namespace: opts.namespace, labels, meta }) : []),
601
+ ...(opts.repoCache
602
+ ? repoCacheObjects({
603
+ ...opts.repoCache,
604
+ namespace: opts.namespace,
605
+ labels,
606
+ meta,
607
+ priorityClassName: priority.control,
608
+ })
609
+ : []),
356
610
  ];
357
611
 
358
612
  return JSON.stringify({ apiVersion: "v1", kind: "List", items });
@@ -425,8 +679,10 @@ function repoCacheObjects(opts: {
425
679
  namespace: string;
426
680
  labels: Record<string, string>;
427
681
  meta: (name: string, extra?: Record<string, string>) => KubeManifest;
682
+ /** The control tier's class (ADR-0060): every Sandbox on the node needs this agent. */
683
+ priorityClassName: string;
428
684
  }): KubeManifest[] {
429
- const { image, namespace, labels, meta, placement } = opts;
685
+ const { image, namespace, labels, meta, placement, priorityClassName } = opts;
430
686
  return [
431
687
  { apiVersion: "v1", kind: "ServiceAccount", metadata: meta(REPO_CACHE) },
432
688
  {
@@ -463,6 +719,7 @@ function repoCacheObjects(opts: {
463
719
  spec: {
464
720
  serviceAccountName: REPO_CACHE,
465
721
  automountServiceAccountToken: true,
722
+ priorityClassName,
466
723
  // One agent per Sandbox node (ADR-0052): the pod's own placement, verbatim. A node no
467
724
  // Sandbox can reach gets no agent — a cache there is a clone nobody reads, and an
468
725
  // affinity term the scheduler cannot honor. (The DaemonSet controller still adds its
@@ -487,7 +744,8 @@ function repoCacheObjects(opts: {
487
744
  { name: "home", mountPath: REPO_CACHE_HOME },
488
745
  { name: "tmp", mountPath: "/tmp" },
489
746
  ],
490
- resources: { requests: { cpu: "20m", memory: "64Mi" } },
747
+ // Kit-sized (ADR-0060): the one request below its limit — `git index-pack` spikes.
748
+ resources: sized(KIT_POD_SIZES.repoCache),
491
749
  securityContext: {
492
750
  runAsUser: 0,
493
751
  runAsGroup: 0,
@@ -513,20 +771,27 @@ function repoCacheObjects(opts: {
513
771
  ];
514
772
  }
515
773
 
774
+ /** The label on each pod's `jr2-held-secrets-<pod>` Secret: its pod, so `jr2 up` finds and deletes
775
+ * the one a pod no longer needs. */
776
+ export const LABEL_HELD_POD = "jr2.dev/held-pod";
777
+
516
778
  /**
517
779
  * The held-secret Secrets (ADR-0059), present only while a secret is held: the CA (which NO pod
518
- * mounts — `jr2 up` alone reads it back), the literal values and the leaves (the Custodian's alone).
519
- * `stringData` as written: no trailing newline is added to a value.
780
+ * mounts — `jr2 up` alone reads it back), each pod's literal values (its Custodian's alone) and the
781
+ * leaves (the Custodians'). `stringData` as written: no trailing newline is added to a value.
520
782
  */
521
783
  function heldSecretObjects(held: HeldObjects, meta: (name: string) => KubeManifest): KubeManifest[] {
522
- if (held.manifest.secrets.length === 0 || !held.ca) return [];
523
- const secret = (name: string, stringData: Record<string, string>): KubeManifest => ({
524
- apiVersion: "v1",
525
- kind: "Secret",
526
- metadata: meta(name),
527
- type: "Opaque",
528
- stringData,
529
- });
784
+ if (!holdsAny(held) || !held.ca) return [];
785
+ const secret = (name: string, stringData: Record<string, string>, pod?: string): KubeManifest => {
786
+ const m = meta(name);
787
+ return {
788
+ apiVersion: "v1",
789
+ kind: "Secret",
790
+ metadata: pod === undefined ? m : { ...m, labels: { ...(m.labels as object), [LABEL_HELD_POD]: pod } },
791
+ type: "Opaque",
792
+ stringData,
793
+ };
794
+ };
530
795
  const leaves: Record<string, string> = {};
531
796
  for (const [stem, pem] of Object.entries(held.leaves)) {
532
797
  leaves[`${stem}.crt`] = pem.cert;
@@ -534,7 +799,9 @@ function heldSecretObjects(held: HeldObjects, meta: (name: string) => KubeManife
534
799
  }
535
800
  return [
536
801
  secret(HELD_CA_SECRET, { "ca.crt": held.ca.cert, "ca.key": held.ca.key }),
537
- ...(Object.keys(held.values).length ? [secret(HELD_SECRETS_SECRET, held.values)] : []),
802
+ ...held.pods.flatMap((p) =>
803
+ Object.keys(p.values).length ? [secret(heldSecretsSecret(p.pod), p.values, p.pod)] : [],
804
+ ),
538
805
  secret(HELD_TLS_SECRET, leaves),
539
806
  ];
540
807
  }
@@ -570,7 +837,7 @@ export function trustObject(opts: {
570
837
  }
571
838
 
572
839
  /**
573
- * The Instance Harness (ADR-0031): the per-instance Harness Deployment + Service `jr2 up`
840
+ * The Instance Harness (ADR-0031): the per-instance Harness StatefulSet + Service `jr2 up`
574
841
  * converges whenever an Agent a registered Machine CARRIES declares `workspace: "none"`
575
842
  * (ADR-0049's walk) — the placement for every Menu-only Agent's Turn, regardless of any enclosing
576
843
  * Workspace. The one Harness shape, minus the Workspace: the stock Harness image plus the Custodian,
@@ -590,15 +857,18 @@ export function instanceHarnessObjects(opts: {
590
857
  *
591
858
  * The accepted asymmetry: the Instance Harness names its images HERE, in the pod template, while
592
859
  * a Sandbox's refs travel through the `jr2-images` ConfigMap. Both are right for what they are —
593
- * this Deployment is supposed to roll when its image moves; the Orchestrator is not. */
860
+ * this StatefulSet is supposed to roll when its image moves; the Orchestrator is not. */
594
861
  harnessImage: string;
595
862
  /** The resolved Custodian ref: the Harness's one route to its Menu, kept even though a `"none"`
596
863
  * Agent cannot execute code — forking the path for one pod buys a divergence ADR-0031 declines. */
597
864
  custodianImage: string;
598
- /** What this converge resolved about held secrets (ADR-0059) — the same manifest the
599
- * Orchestrator composes every Sandbox's Custodian from. */
865
+ /** What this converge placed in the Instance Harness (ADR-0059): its own manifest — the model
866
+ * keys its `"none"` Agents' providers read, and nothing else. */
600
867
  held: HeldManifest;
601
- /** An HMAC over every held-secret input (`held.json`, the leaves, the literal values, the
868
+ /** Its Custodian's two files, rendered from `held` — applied here, with the pod they are for
869
+ * (`INSTANCE_CUSTODIAN_CONFIGMAP`, ADR-0059). */
870
+ custodian: CustodianFiles;
871
+ /** An HMAC over this pod's held-secret inputs (its manifest, its leaves, its literal values, its
602
872
  * Custodian's config), keyed with the signing key so it reveals nothing — on the pod template, so
603
873
  * a change to any of them rolls this pod (ADR-0059). */
604
874
  heldDigest: string;
@@ -610,6 +880,8 @@ export function instanceHarnessObjects(opts: {
610
880
  * every Sandbox Harness container gets, so this pod can verify the Orchestrator and mint
611
881
  * nothing. */
612
882
  bearerSha256: string;
883
+ /** `priorityClasses` from jr2.config.ts (ADR-0060): the classes to name instead of jr2's own. */
884
+ priorityClasses?: PriorityClasses;
613
885
  }): string {
614
886
  const labels = { [LABEL_INSTANCE]: opts.name, "app.kubernetes.io/managed-by": "jr2" };
615
887
  const meta = (): KubeManifest => ({
@@ -629,12 +901,15 @@ export function instanceHarnessObjects(opts: {
629
901
 
630
902
  // The Custodian (ADR-0013, ADR-0059): the placement's Sandbox token — signed for this
631
903
  // placement's name (`up.ts` mints it into the instance Secret), so it speaks only for the Turns
632
- // hosted HERE (tokens.ts) — plus every held secret. Mounted into the Custodian alone; the Instance
633
- // token itself never enters this pod.
904
+ // hosted HERE (tokens.ts) — plus the held secrets placed here. Mounted into the Custodian alone;
905
+ // the Instance token itself never enters this pod.
634
906
  const custodian = custodianComposition(opts.held, {
635
907
  image: opts.custodianImage,
908
+ pod: INSTANCE_HARNESS_HELD_POD,
909
+ config: INSTANCE_CUSTODIAN_CONFIGMAP,
636
910
  token: { secret: INSTANCE_SECRET, key: "JR2_INSTANCE_HARNESS_TOKEN" },
637
911
  caBundle: opts.caBundle === true,
912
+ resources: sized(KIT_POD_SIZES.instanceCustodian),
638
913
  });
639
914
 
640
915
  const harnessContainer = {
@@ -657,47 +932,90 @@ export function instanceHarnessObjects(opts: {
657
932
  { name: "JR2_MENU_ONLY", value: "1" },
658
933
  ...(opts.harness?.env ?? []).filter((v) => v.valueFrom !== undefined),
659
934
  ...custodian.harnessEnv,
935
+ // Where the conversations persist (ADR-0031): the claim below. Mechanism, as on a Sandbox.
936
+ { name: CONVERSATIONS_DIR_ENV, value: CONVERSATIONS_DIR },
660
937
  // The wire's gate (ADR-0058), last — as on a Sandbox — so no `harness.env` entry chooses it.
661
938
  { name: "JR2_HARNESS_TOKEN_SHA256", value: opts.bearerSha256 },
662
939
  ],
663
940
  envFrom: [{ secretRef: { name: HARNESS_ENV_SECRET } }, ...(opts.harness?.envFrom ?? [])],
664
- ...(custodian.harnessMounts.length ? { volumeMounts: custodian.harnessMounts } : {}),
941
+ // The Harness container's alone: the Custodian has no business with an Agent's history.
942
+ volumeMounts: [{ name: CONVERSATIONS_VOLUME, mountPath: CONVERSATIONS_DIR }, ...custodian.harnessMounts],
665
943
  // The operator probes a Sandbox's Harness the same way: serving = the socket accepts.
666
944
  // Period and threshold as reasoned on the Orchestrator above: the default 10s period is the
667
945
  // rollout wait rather than the boot, and the threshold then has to carry the stall tolerance
668
946
  // the period used to supply. This pod is single-replica too.
947
+ // The Harness's own route, not the port (ADR-0063): a socket probe would pass for whatever
948
+ // listens there. A draining Harness still answers it (ADR-0031).
669
949
  readinessProbe: {
670
- tcpSocket: { port: INSTANCE_HARNESS_PORT },
950
+ httpGet: { path: "/healthz", port: INSTANCE_HARNESS_PORT },
671
951
  initialDelaySeconds: 1,
672
952
  periodSeconds: 2,
673
953
  failureThreshold: 15,
674
954
  },
955
+ resources: sized(KIT_POD_SIZES.instanceHarness),
675
956
  securityContext: hardenedContainerSecurityContext(),
676
957
  };
677
958
 
678
959
  const items: KubeManifest[] = [
960
+ // The one pod `jr2 up` makes, so the one whose Custodian files it renders: its bootstrap and
961
+ // its HELD table, from its own manifest, beside the program its volume projects from `jr2-held`.
962
+ {
963
+ apiVersion: "v1",
964
+ kind: "ConfigMap",
965
+ metadata: { ...meta(), name: INSTANCE_CUSTODIAN_CONFIGMAP },
966
+ data: {
967
+ [CUSTODIAN_BOOTSTRAP_KEY]: opts.custodian.bootstrap,
968
+ [CUSTODIAN_SCRIPT_KEY]: opts.custodian.script,
969
+ },
970
+ },
679
971
  {
680
972
  apiVersion: "apps/v1",
681
- kind: "Deployment",
973
+ kind: "StatefulSet",
682
974
  metadata: { ...meta(), labels: { ...labels, [LABEL_VERSION]: KIT_VERSION } },
683
975
  spec: {
684
- // ONE replica, Recreate: a conversation is an Instance ID on one Harness PROCESS
685
- // (ADR-0031) — two pods behind this Service would route one conversation to two servers,
686
- // the exact amnesia definition-wins placement exists to prevent. A restart loses the
687
- // conversations (live-only, ADR-0023); the Deployment restores the endpoint, not the
688
- // history.
976
+ // A StatefulSet of ONE (ADR-0031): the value is the stable volume — a restart or a rollout
977
+ // brings the same conversations back — and the stable name is what N > 1 would pin a
978
+ // conversation to. Never more than one pod behind this Service for one conversation, and
979
+ // never an HPA: a scale-down would kill conversations. At N = 1 a rollout has no surge,
980
+ // so admissions are refused while the new pod starts, which the Orchestrator absorbs
981
+ // (ADR-0042).
689
982
  replicas: 1,
690
- strategy: { type: "Recreate" },
983
+ serviceName: INSTANCE_HARNESS_PODS_SERVICE,
691
984
  selector: { matchLabels: { app: INSTANCE_HARNESS_SERVICE } },
985
+ // The conversations go with the StatefulSet, never with a scale-down: an instance whose
986
+ // Machines dropped every `"none"` Agent keeps no history for them.
987
+ persistentVolumeClaimRetentionPolicy: { whenDeleted: "Delete", whenScaled: "Retain" },
988
+ volumeClaimTemplates: [
989
+ {
990
+ metadata: { name: CONVERSATIONS_VOLUME, labels },
991
+ spec: {
992
+ accessModes: ["ReadWriteOnce"],
993
+ resources: { requests: { storage: "1Gi" } },
994
+ },
995
+ },
996
+ ],
692
997
  template: {
693
998
  metadata: {
694
999
  labels: { ...labels, app: INSTANCE_HARNESS_SERVICE },
695
1000
  // A held-secret edit rolls this pod; a live Sandbox keeps what its Custodian read at
696
- // start (ADR-0059, the ADR-0037 stance).
697
- annotations: { [ANNOTATION_HELD_DIGEST]: opts.heldDigest },
1001
+ // start (ADR-0059, the ADR-0037 stance). A move costs a drain, not the conversations
1002
+ // (the volume keeps them), yet no autoscaler may choose one (ADR-0060) — a drain still
1003
+ // can.
1004
+ annotations: { [ANNOTATION_HELD_DIGEST]: opts.heldDigest, ...NO_DISRUPT_ANNOTATIONS },
698
1005
  },
699
1006
  spec: {
700
- containers: [harnessContainer, custodian.custodianContainer],
1007
+ // The drain (ADR-0031): on SIGTERM the Harness stops admitting and finishes the Turns
1008
+ // it holds, so the grace is a Turn's worst case — a deploy loses no Turn.
1009
+ terminationGracePeriodSeconds: INSTANCE_HARNESS_DRAIN_SECONDS,
1010
+ // The Sandbox tier's class (ADR-0060): it holds live conversations, as a Sandbox holds
1011
+ // `/work`, and it may wait rather than preempt.
1012
+ priorityClassName: priorityClassNames(opts.priorityClasses).sandbox,
1013
+ // The Custodian is a native sidecar, so it outlives the drain (ADR-0031): the kubelet
1014
+ // stops a pod's sidecars only after its main containers exit, and the draining Harness
1015
+ // needs its Custodian to the end — its one route to the Menu and to the model. As a
1016
+ // plain container Envoy took SIGTERM with the Harness and quit at once.
1017
+ initContainers: [{ ...custodian.custodianContainer, restartPolicy: "Always" }],
1018
+ containers: [harnessContainer],
701
1019
  // The operator's isolation baseline (sandbox_controller.go), mirrored: same Harness
702
1020
  // image, same "never reach the Kubernetes API" north star — JR2_MENU_ONLY makes code
703
1021
  // execution here unlikely, not unimaginable. No `shareProcessNamespace`: the Harness
@@ -719,6 +1037,22 @@ export function instanceHarnessObjects(opts: {
719
1037
  kind: "Service",
720
1038
  metadata: meta(),
721
1039
  spec: {
1040
+ // The address the Agent actor dials: a ClusterIP Service, deliberately not the headless
1041
+ // one below. A draining pod is terminating, so DNS stops naming it, but kube-proxy still
1042
+ // routes a ClusterIP to a terminating pod that serves while no other is ready — and at
1043
+ // N = 1 none is until it exits. So during the drain reads, waits and aborts keep reaching
1044
+ // it (ADR-0031), and a refused admission meets its 503, not a name that resolves to nothing.
1045
+ selector: { app: INSTANCE_HARNESS_SERVICE },
1046
+ ports: [{ port: INSTANCE_HARNESS_PORT, targetPort: INSTANCE_HARNESS_PORT }],
1047
+ },
1048
+ },
1049
+ {
1050
+ apiVersion: "v1",
1051
+ kind: "Service",
1052
+ metadata: { ...meta(), name: INSTANCE_HARNESS_PODS_SERVICE },
1053
+ spec: {
1054
+ // Headless — the StatefulSet's governing Service (`serviceName`), for the per-ordinal names.
1055
+ clusterIP: "None",
722
1056
  selector: { app: INSTANCE_HARNESS_SERVICE },
723
1057
  ports: [{ port: INSTANCE_HARNESS_PORT, targetPort: INSTANCE_HARNESS_PORT }],
724
1058
  },
@@ -727,3 +1061,248 @@ export function instanceHarnessObjects(opts: {
727
1061
 
728
1062
  return JSON.stringify({ apiVersion: "v1", kind: "List", items });
729
1063
  }
1064
+
1065
+ /** The Orchestrator's pod template digest of the Instance Secret: it rolls the pod when a value
1066
+ * the process reads through `envFrom` changes (ADR-0069). */
1067
+ export const ANNOTATION_SECRET_DIGEST = "jr2.dev/secret-digest";
1068
+
1069
+ /** A keyed digest of the Instance Secret's data, in key order. Keyed with the signing key it
1070
+ * holds, so the annotation reveals nothing about a token. */
1071
+ export function instanceSecretDigest(secretData: Record<string, string>): string {
1072
+ const mac = createHmac("sha256", Buffer.from(secretData.JR2_SIGNING_KEY ?? "", "base64"));
1073
+ mac.update("jr2-instance-secret\n");
1074
+ for (const key of Object.keys(secretData).sort()) {
1075
+ const value = Buffer.from(secretData[key]!, "utf8");
1076
+ mac.update(`${key}\n${value.length}:`).update(value);
1077
+ }
1078
+ return mac.digest("base64url");
1079
+ }
1080
+
1081
+ /** The Instance Effector's pod template digest (ADR-0068): an HMAC over every secret input, so an
1082
+ * edit to one rolls it, as `jr2.dev/held-digest` rolls the Instance Harness (ADR-0059). */
1083
+ export const ANNOTATION_EFFECTOR_DIGEST = "jr2.dev/effector-digest";
1084
+
1085
+ /** The Instance Effector pod's `fsGroup`: its Secret files are readable by that group, and the pod
1086
+ * has one container. The group a Sandbox's and the Instance Harness's pods use (ADR-0005). */
1087
+ const INSTANCE_EFFECTOR_FS_GROUP = 2000;
1088
+
1089
+ /** Where the Effector reads what it holds (ADR-0068): its secrets, one file each; its key and
1090
+ * its config. Its scratch is `$HOME`. */
1091
+ const EFFECTOR_SECRETS_DIR = "/run/jr2/secrets";
1092
+ const EFFECTOR_OWN_DIR = "/run/jr2/effector";
1093
+ const EFFECTOR_SCRATCH = "/run/jr2/scratch";
1094
+
1095
+ /** How much more than the longest Effect timeout a stopping replica is given: room for the
1096
+ * Orchestrator's next poll to read the outcome it finished. */
1097
+ export const EFFECTOR_READ_MARGIN_SECONDS = 30;
1098
+
1099
+ /** The Secret's key of one literal Effect secret: prefixed, so no secret's name is `key` or
1100
+ * `effector.json`. */
1101
+ const effectorSecretKey = (name: string): string => `secret.${name}`;
1102
+
1103
+ /** The Instance Effector's grace period: the longest timeout among the Effects placed there, plus
1104
+ * the read margin. */
1105
+ export const effectorGraceSeconds = (plan: EffectorPlan): number =>
1106
+ Math.ceil(plan.longestTimeoutMs / 1000) + EFFECTOR_READ_MARGIN_SECONDS;
1107
+
1108
+ /**
1109
+ * The Instance Effector (ADR-0068): the objects `jr2 up` converges when a registered Machine
1110
+ * places an Effect there — an `http` Effect — and deletes otherwise.
1111
+ *
1112
+ * - A Deployment, not a StatefulSet: a rollout must run the new pod beside the old one (surge), so
1113
+ * the old one finishes an Effect in flight while the new one takes new Effects. The Kit's
1114
+ * `jr2-harness` image with `/opt/jr2/bin/effector` as its command; hardened, non-root, a read-only
1115
+ * root filesystem, an emptyDir scratch as `$HOME`. Its grace period is the longest timeout among
1116
+ * the Effects placed there, plus a margin for the outcome to be read.
1117
+ * - A headless Service that publishes addresses that are not ready: "ask every replica" is its
1118
+ * name's addresses, a stopping replica included.
1119
+ * - A NetworkPolicy of `jr2-sandbox-ingress`'s form: ingress from this Instance's Orchestrator only.
1120
+ * No egress rule: the pod holds real values and calls the hosts its Effects name.
1121
+ * - The Secret `jr2-instance-effector`: the literal Effect secrets placed here, the placement's key,
1122
+ * and `effector.json` (the targets). A secret from the user's own Secret is projected from there
1123
+ * by reference, never copied.
1124
+ */
1125
+ export function instanceEffectorObjects(opts: {
1126
+ name: string;
1127
+ namespace: string;
1128
+ /** The resolved `jr2-harness` ref — it carries the binary (ADR-0037). */
1129
+ harnessImage: string;
1130
+ plan: EffectorPlan;
1131
+ /** `effectorKey(signingKey, "instance")`. */
1132
+ key: string;
1133
+ /** The HMAC over every secret input (`effectorDigest`). */
1134
+ digest: string;
1135
+ /** The instance ships a private-CA bundle (ADR-0020): it is mounted and trusted. */
1136
+ caBundle?: boolean;
1137
+ priorityClasses?: PriorityClasses;
1138
+ }): string {
1139
+ const labels = { [LABEL_INSTANCE]: opts.name, "app.kubernetes.io/managed-by": "jr2" };
1140
+ const meta = (): KubeManifest => ({ name: INSTANCE_EFFECTOR, namespace: opts.namespace, labels });
1141
+ const grace = effectorGraceSeconds(opts.plan);
1142
+
1143
+ const literal = opts.plan.secrets.filter((s) => s.source.kind === "literal");
1144
+ const sources: KubeManifest[] = [
1145
+ ...(literal.length
1146
+ ? [
1147
+ {
1148
+ secret: {
1149
+ name: INSTANCE_EFFECTOR,
1150
+ items: literal.map((s) => ({ key: effectorSecretKey(s.name), path: s.name })),
1151
+ },
1152
+ },
1153
+ ]
1154
+ : []),
1155
+ ...opts.plan.secrets.flatMap((s) =>
1156
+ s.source.kind === "secret"
1157
+ ? [{ secret: { name: s.source.secret, items: [{ key: s.source.key, path: s.name }] } }]
1158
+ : [],
1159
+ ),
1160
+ ];
1161
+ const config = { version: 1, placement: opts.plan.placement, targets: opts.plan.targets };
1162
+
1163
+ const container = {
1164
+ name: "effector",
1165
+ image: opts.harnessImage,
1166
+ imagePullPolicy: "IfNotPresent",
1167
+ command: ["/opt/jr2/bin/effector"],
1168
+ ports: [{ containerPort: EFFECTOR_PORT }],
1169
+ env: [
1170
+ { name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" } } },
1171
+ { name: "POD_IP", valueFrom: { fieldRef: { fieldPath: "status.podIP" } } },
1172
+ { name: "JR2_EFFECTOR_PORT", value: String(EFFECTOR_PORT) },
1173
+ { name: "JR2_EFFECTOR_KEY", value: `${EFFECTOR_OWN_DIR}/key` },
1174
+ { name: "JR2_EFFECTOR_CONFIG", value: `${EFFECTOR_OWN_DIR}/effector.json` },
1175
+ { name: "JR2_EFFECTOR_SECRETS", value: EFFECTOR_SECRETS_DIR },
1176
+ // The drain ends a few seconds before the kubelet's kill, so the replica exits on its own.
1177
+ { name: "JR2_EFFECTOR_DRAIN_SECONDS", value: String(Math.max(grace - 5, 1)) },
1178
+ { name: "HOME", value: EFFECTOR_SCRATCH },
1179
+ ...(opts.caBundle ? [{ name: "JR2_CA_FILE", value: `${CA_MOUNT}/ca.crt` }] : []),
1180
+ ],
1181
+ volumeMounts: [
1182
+ { name: "secrets", mountPath: EFFECTOR_SECRETS_DIR, readOnly: true },
1183
+ { name: "effector", mountPath: EFFECTOR_OWN_DIR, readOnly: true },
1184
+ { name: "scratch", mountPath: EFFECTOR_SCRATCH },
1185
+ ...(opts.caBundle ? [{ name: "ca", mountPath: CA_MOUNT, readOnly: true }] : []),
1186
+ ],
1187
+ readinessProbe: { httpGet: { path: "/healthz", port: EFFECTOR_PORT }, periodSeconds: 2, failureThreshold: 3 },
1188
+ resources: sized(KIT_POD_SIZES.instanceEffector),
1189
+ securityContext: {
1190
+ runAsNonRoot: true,
1191
+ allowPrivilegeEscalation: false,
1192
+ readOnlyRootFilesystem: true,
1193
+ capabilities: { drop: ["ALL"] },
1194
+ seccompProfile: { type: "RuntimeDefault" },
1195
+ },
1196
+ };
1197
+
1198
+ const items: KubeManifest[] = [
1199
+ {
1200
+ apiVersion: "v1",
1201
+ kind: "Secret",
1202
+ metadata: meta(),
1203
+ type: "Opaque",
1204
+ stringData: {
1205
+ key: opts.key,
1206
+ "effector.json": JSON.stringify(config, null, 2),
1207
+ ...Object.fromEntries(literal.map((s) => [effectorSecretKey(s.name), opts.plan.values[s.name] ?? ""])),
1208
+ },
1209
+ },
1210
+ {
1211
+ apiVersion: "apps/v1",
1212
+ kind: "Deployment",
1213
+ metadata: { ...meta(), labels: { ...labels, [LABEL_VERSION]: KIT_VERSION } },
1214
+ spec: {
1215
+ replicas: 1,
1216
+ selector: { matchLabels: { app: INSTANCE_EFFECTOR } },
1217
+ // Surge, never a gap: the new replica starts before the old one stops.
1218
+ strategy: { type: "RollingUpdate", rollingUpdate: { maxSurge: 1, maxUnavailable: 0 } },
1219
+ template: {
1220
+ metadata: {
1221
+ labels: { ...labels, app: INSTANCE_EFFECTOR },
1222
+ annotations: { [ANNOTATION_EFFECTOR_DIGEST]: opts.digest },
1223
+ },
1224
+ spec: {
1225
+ terminationGracePeriodSeconds: grace,
1226
+ priorityClassName: priorityClassNames(opts.priorityClasses).control,
1227
+ automountServiceAccountToken: false,
1228
+ enableServiceLinks: false,
1229
+ securityContext: {
1230
+ runAsNonRoot: true,
1231
+ seccompProfile: { type: "RuntimeDefault" },
1232
+ fsGroup: INSTANCE_EFFECTOR_FS_GROUP,
1233
+ },
1234
+ containers: [container],
1235
+ volumes: [
1236
+ { name: "secrets", projected: { defaultMode: 0o400, sources } },
1237
+ {
1238
+ name: "effector",
1239
+ secret: {
1240
+ secretName: INSTANCE_EFFECTOR,
1241
+ defaultMode: 0o400,
1242
+ items: [
1243
+ { key: "key", path: "key" },
1244
+ { key: "effector.json", path: "effector.json" },
1245
+ ],
1246
+ },
1247
+ },
1248
+ { name: "scratch", emptyDir: {} },
1249
+ ...(opts.caBundle ? [{ name: "ca", configMap: { name: CA_CONFIGMAP } }] : []),
1250
+ ],
1251
+ },
1252
+ },
1253
+ },
1254
+ },
1255
+ {
1256
+ apiVersion: "v1",
1257
+ kind: "Service",
1258
+ metadata: meta(),
1259
+ spec: {
1260
+ clusterIP: "None",
1261
+ // A stopping replica still answers for the Effects it finishes: its address stays listed.
1262
+ publishNotReadyAddresses: true,
1263
+ selector: { app: INSTANCE_EFFECTOR },
1264
+ ports: [{ port: EFFECTOR_PORT, targetPort: EFFECTOR_PORT }],
1265
+ },
1266
+ },
1267
+ {
1268
+ apiVersion: "networking.k8s.io/v1",
1269
+ kind: "NetworkPolicy",
1270
+ metadata: meta(),
1271
+ spec: {
1272
+ podSelector: { matchLabels: { app: INSTANCE_EFFECTOR } },
1273
+ policyTypes: ["Ingress"],
1274
+ ingress: [
1275
+ { from: [{ podSelector: { matchLabels: { app: ORCHESTRATOR_SERVICE, [LABEL_INSTANCE]: opts.name } } }] },
1276
+ ],
1277
+ },
1278
+ },
1279
+ ];
1280
+ return JSON.stringify({ apiVersion: "v1", kind: "List", items });
1281
+ }
1282
+
1283
+ /** A keyed digest of one key's value in the user's Secret, as the API returns it (base64): what a
1284
+ * template digest holds for a `valueFrom`, so an edit of the value rolls the pod and the annotation
1285
+ * reveals nothing about it. `jr2 up` keeps the digest, never the value. */
1286
+ export function storedValueDigest(signingKey: Buffer, secret: string, key: string, base64: string): string {
1287
+ return createHmac("sha256", signingKey)
1288
+ .update(`jr2-secret-value\n${secret}\n${key}\n`)
1289
+ .update(Buffer.from(base64, "base64"))
1290
+ .digest("base64url");
1291
+ }
1292
+
1293
+ /** The digest of each `valueFrom` value, by the user's Secret and key; undefined when not read. */
1294
+ export type ValueDigest = (secret: string, key: string) => string | undefined;
1295
+
1296
+ /** The Instance Effector's template digest: an HMAC over every secret input — a literal's value, a
1297
+ * `valueFrom`'s reference and the digest of its value — keyed with the signing key so the
1298
+ * annotation reveals nothing about a value. */
1299
+ export function effectorDigest(signingKey: Buffer, plan: EffectorPlan, key: string, valueDigest: ValueDigest): string {
1300
+ const mac = createHmac("sha256", signingKey);
1301
+ mac.update(JSON.stringify({ secrets: plan.secrets, targets: plan.targets, longest: plan.longestTimeoutMs }));
1302
+ for (const name of Object.keys(plan.values).sort()) mac.update(`${name}=${plan.values[name]}`);
1303
+ for (const s of plan.secrets) {
1304
+ if (s.source.kind === "secret") mac.update(`${s.name}<-${valueDigest(s.source.secret, s.source.key) ?? ""}`);
1305
+ }
1306
+ mac.update(key);
1307
+ return mac.digest("base64url").slice(0, 32);
1308
+ }