@jr2/orchestrator 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -4
- package/src/actor.ts +12 -7
- package/src/agent.ts +2 -2
- package/src/config.ts +119 -14
- package/src/custodian.lua +312 -0
- package/src/custodian.ts +899 -0
- package/src/harness-client.ts +30 -10
- package/src/held-secrets.ts +587 -0
- package/src/http.ts +25 -19
- package/src/images.ts +11 -10
- package/src/index.ts +6 -2
- package/src/instance.ts +13 -10
- package/src/model-keys.ts +92 -0
- package/src/names.ts +30 -6
- package/src/registration.ts +11 -3
- package/src/repo-fetch.ts +1 -1
- package/src/run-host.ts +34 -21
- package/src/sandbox-kubectl.ts +82 -110
- package/src/server.ts +5 -17
- package/src/tokens.ts +40 -9
- package/src/workspace.ts +2 -2
package/src/sandbox-kubectl.ts
CHANGED
|
@@ -40,21 +40,22 @@
|
|
|
40
40
|
// initContainer preflight the USER'S image + that volume → ADR-0037's probe, the thing that
|
|
41
41
|
// proves a registry ref, whose first appearance is this provision
|
|
42
42
|
// container harness the Sandbox Image, command overridden, /work + /opt/jr2 mounted
|
|
43
|
-
// container
|
|
43
|
+
// container custodian jr2-composed, the pod's only credential holder (below)
|
|
44
44
|
// container user optional, the image's own entrypoint, the checkouts (/work, plus
|
|
45
45
|
// /repos and /opt/jr2 read-only) and NOTHING else
|
|
46
46
|
//
|
|
47
|
-
// This is also where the
|
|
48
|
-
// ADR-0001 made `Sidecars` generic container fragments it schedules WITHOUT
|
|
49
|
-
//
|
|
50
|
-
// builds is the pod's asymmetry:
|
|
47
|
+
// This is also where the CUSTODIAN is composed (ADR-0013, ADR-0059). The operator needs no change
|
|
48
|
+
// to carry it: ADR-0001 made `Sidecars` generic container fragments it schedules WITHOUT
|
|
49
|
+
// understanding, so the Custodian is exactly that — a container with an image, its mounts, and a
|
|
50
|
+
// readiness probe the pod's Ready waits on. What this module builds is the pod's asymmetry:
|
|
51
51
|
//
|
|
52
|
-
// harness container
|
|
53
|
-
//
|
|
52
|
+
// harness container JR2_CUSTODIAN_URL + the Stand-in of every credential (no credential)
|
|
53
|
+
// custodian container the Sandbox token and every held secret, as files (the credentials)
|
|
54
54
|
//
|
|
55
55
|
// The Agent has code execution in the first and none in the second. The token is minted here — a
|
|
56
56
|
// signed Sandbox name (see tokens.ts), so re-provisioning after a restart yields the SAME token and
|
|
57
|
-
// the Secret re-applies as a no-op.
|
|
57
|
+
// the Secret re-applies as a no-op. What else the Custodian holds is `jr2 up`'s resolution, read
|
|
58
|
+
// per provision from the `jr2-held` mount (custodian.ts builds both placements' pods alike).
|
|
58
59
|
|
|
59
60
|
import { execFile } from "node:child_process";
|
|
60
61
|
import { join } from "node:path";
|
|
@@ -65,19 +66,18 @@ import {
|
|
|
65
66
|
type HarnessEnvVar,
|
|
66
67
|
type SandboxPlacement,
|
|
67
68
|
} from "./config.ts";
|
|
69
|
+
import { custodianComposition, type CustodianComposition } from "./custodian.ts";
|
|
70
|
+
import { readHeldManifest } from "./held-secrets.ts";
|
|
68
71
|
import { readImageRefs, resolveSandboxImage, resolveUserImage, type ImageRefs } from "./images.ts";
|
|
69
|
-
import {
|
|
72
|
+
import { HELD_KEY, HELD_MOUNT, IMAGES_KEY, IMAGES_MOUNT, REPOS_MOUNT } from "./names.ts";
|
|
70
73
|
import { repoIdentity } from "./repo-identity.ts";
|
|
71
74
|
import type { RepoResources } from "./repos.ts";
|
|
72
|
-
import { sandboxToken } from "./tokens.ts";
|
|
75
|
+
import { harnessTokenDigest, sandboxToken } from "./tokens.ts";
|
|
73
76
|
import type { ProvisionedRepo, SandboxPort, WorkspaceSpec } from "./workspace.ts";
|
|
74
77
|
|
|
75
78
|
/** Run one kubectl invocation to completion. `input` is piped to stdin (`apply -f -`). */
|
|
76
79
|
export type KubectlExec = (args: string[], opts?: { input?: string }) => Promise<{ stdout: string; stderr: string }>;
|
|
77
80
|
|
|
78
|
-
/** Where the Harness container sees the instance's CA bundle (ADR-0020). */
|
|
79
|
-
const CA_MOUNT = "/etc/jr2/ca";
|
|
80
|
-
|
|
81
81
|
/** Where jr2's runtime lands in every container that gets it (ADR-0037). `/opt/jr2` and not `/app`
|
|
82
82
|
* because a stranger's base may already use `/app`, and one layout must serve both the stock
|
|
83
83
|
* Harness image and an arbitrary Sandbox Image. It is a PUBLISHED surface: `bin/` beside `lib/`
|
|
@@ -97,10 +97,6 @@ const HARNESS_COMMAND = [`${RUNTIME_MOUNT}/bin/node`, `${RUNTIME_MOUNT}/src/main
|
|
|
97
97
|
* checkouts must hold this volume — which is why the User Container mounts it (ADR-0005). */
|
|
98
98
|
const UPLOAD_PACK = `${RUNTIME_MOUNT}/bin/jr2-upload-pack`;
|
|
99
99
|
|
|
100
|
-
/** The Adapter's port on the pod's loopback. The program defaults to this address too, so the
|
|
101
|
-
* fetch url names it only when the composition moved it (attachScript). */
|
|
102
|
-
const DEFAULT_ADAPTER_PORT = 8081;
|
|
103
|
-
|
|
104
100
|
/** ADR-0005's default work group. Convention, not config: the pod's `fsGroup` is granted to every
|
|
105
101
|
* container as a supplemental group, so the Harness writes `/work` whatever the number and no
|
|
106
102
|
* image's `/etc/group` needs to know it. The one override is the spec's `workGroup`. */
|
|
@@ -263,6 +259,10 @@ export type KubectlSandboxOptions = {
|
|
|
263
259
|
* off it at invoke time, handed to `provision()` as a string — and resolved against this map at
|
|
264
260
|
* provision. The per-run spec never names one (ADR-0051). */
|
|
265
261
|
imagesPath?: string;
|
|
262
|
+
/** The held-secret manifest `jr2 up` resolved (ADR-0059) — read per provision like the image map,
|
|
263
|
+
* so a held-secret edit reaches future Sandboxes without rolling this process. Default: the
|
|
264
|
+
* `jr2-held` ConfigMap's mount. */
|
|
265
|
+
heldPath?: string;
|
|
266
266
|
/** Extra env for the HARNESS container (`harness.env`) — merged ahead of the
|
|
267
267
|
* mechanism-owned vars, which win on collision. */
|
|
268
268
|
env?: HarnessEnvVar[];
|
|
@@ -273,20 +273,11 @@ export type KubectlSandboxOptions = {
|
|
|
273
273
|
* `sandbox.tolerations`, written on the CR verbatim and copied onto the pod by the operator, which
|
|
274
274
|
* merges nothing with them. Absent → wherever an ordinary pod lands. */
|
|
275
275
|
placement?: SandboxPlacement;
|
|
276
|
-
/** The instance ships a private-CA bundle (ADR-0020):
|
|
277
|
-
*
|
|
278
|
-
* HTTP to the Orchestrator's Service (the same asymmetry as env/envFrom above). */
|
|
276
|
+
/** The instance ships a private-CA bundle (ADR-0020): the HARNESS container trusts it, and the
|
|
277
|
+
* Custodian verifies a bound host with it (custodian.ts). */
|
|
279
278
|
caBundle?: boolean;
|
|
280
|
-
/**
|
|
281
|
-
* Where the Adapter reaches the Orchestrator FROM INSIDE THE CLUSTER — the orchestrator's own
|
|
282
|
-
* Service DNS (derived from JR2_NAMESPACE by the entrypoint). The Agent is never told it.
|
|
283
|
-
* A thunk is still accepted for callers that resolve their address late.
|
|
284
|
-
*/
|
|
285
|
-
orchestratorUrl?: string | (() => string | undefined);
|
|
286
279
|
/** The key Sandbox tokens are signed with — from the instance Secret (ADR-0013/0019). */
|
|
287
280
|
signingKey?: Buffer;
|
|
288
|
-
/** The Adapter's port on the pod's loopback. Default 8081. */
|
|
289
|
-
adapterPort?: number;
|
|
290
281
|
/** Kube namespace for Sandbox CRs. Default `default`. */
|
|
291
282
|
namespace?: string;
|
|
292
283
|
/** kubectl `--context` override. Default: the current context (ADR-0009). */
|
|
@@ -339,6 +330,7 @@ export type KubectlSandboxOptions = {
|
|
|
339
330
|
export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
340
331
|
const ns = opts.namespace ?? "default";
|
|
341
332
|
const imagesPath = opts.imagesPath ?? join(IMAGES_MOUNT, IMAGES_KEY);
|
|
333
|
+
const heldPath = opts.heldPath ?? join(HELD_MOUNT, HELD_KEY);
|
|
342
334
|
const workRoot = opts.workRoot ?? "/work";
|
|
343
335
|
const readyTimeoutMs = opts.readyTimeoutMs ?? 120_000;
|
|
344
336
|
const repoTimeoutMs = opts.repoTimeoutMs ?? 25 * 60_000;
|
|
@@ -346,32 +338,12 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
346
338
|
const exec = opts.exec ?? defaultKubectlExec;
|
|
347
339
|
const credentials = opts.credentials ?? [];
|
|
348
340
|
|
|
349
|
-
const adapterPort = opts.adapterPort ?? DEFAULT_ADAPTER_PORT;
|
|
350
341
|
const leaseIntervalMs = opts.leaseIntervalMs ?? 5 * 60_000;
|
|
351
342
|
const base = ["--namespace", ns, ...(opts.context ? ["--context", opts.context] : [])];
|
|
352
343
|
|
|
353
|
-
/** The Sandbox's token Secret —
|
|
344
|
+
/** The Sandbox's token Secret — mounted into the Custodian container, and nothing else in the pod. */
|
|
354
345
|
const secretName = (name: string) => `${name}-token`;
|
|
355
346
|
|
|
356
|
-
/** Resolved at provision time (see the option's doc): the Orchestrator's in-cluster address. */
|
|
357
|
-
const orchestratorUrl = (): string | undefined =>
|
|
358
|
-
typeof opts.orchestratorUrl === "function" ? opts.orchestratorUrl() : opts.orchestratorUrl;
|
|
359
|
-
|
|
360
|
-
/** The Adapter, as the operator sees it: an opaque container fragment (ADR-0001). */
|
|
361
|
-
const adapterSidecar = (name: string, refs: ImageRefs) => ({
|
|
362
|
-
name: "adapter",
|
|
363
|
-
image: refs.adapter,
|
|
364
|
-
env: [
|
|
365
|
-
{ name: "JR2_ORCHESTRATOR_URL", value: orchestratorUrl() },
|
|
366
|
-
{ name: "JR2_SANDBOX", value: name },
|
|
367
|
-
{ name: "JR2_ADAPTER_PORT", value: String(adapterPort) },
|
|
368
|
-
],
|
|
369
|
-
// The credential, and the reason this is a separate container: `local()` tools give the Agent
|
|
370
|
-
// code execution in the HARNESS container, so anything mounted there is the Agent's. Here, it
|
|
371
|
-
// is out of reach — different container, no shared process namespace.
|
|
372
|
-
envFrom: [{ secretRef: { name: secretName(name) } }],
|
|
373
|
-
});
|
|
374
|
-
|
|
375
347
|
/**
|
|
376
348
|
* The User Container (ADR-0005): the opt-in third seat, composed only when the wrapper's static
|
|
377
349
|
* `user` option names an image (ADR-0049). The ZERO-CONTRACT seat — jr2 injects nothing, probes
|
|
@@ -395,8 +367,8 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
395
367
|
* Agent, with no credential of their own. Nothing else follows it in: `ext::` names the program
|
|
396
368
|
* by absolute path, so this seat still gets no env, no command, and no probe. The repo volumes
|
|
397
369
|
* are the operator's — it defines `repo-<key>` for every key the CR names — so this seat mounts
|
|
398
|
-
* them by name. The
|
|
399
|
-
* container holding the pod's
|
|
370
|
+
* them by name. The Custodian is deliberately not given any of the three: it reads no worktree, and it is the
|
|
371
|
+
* container holding the pod's credentials, so it gets the narrowest mount set that works.
|
|
400
372
|
* It also carries no `securityContext`, which the operator reads as the exemption — root is
|
|
401
373
|
* ALLOWED here, because hardening a seat whose identity is "what jr2 does not own" is an opinion,
|
|
402
374
|
* and the standard managed-access shape (a root sshd that setuids sessions down) must run
|
|
@@ -413,25 +385,31 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
413
385
|
});
|
|
414
386
|
|
|
415
387
|
/** The pod's sidecar list (ADR-0001: opaque fragments the operator schedules verbatim). The
|
|
416
|
-
*
|
|
417
|
-
*
|
|
418
|
-
*
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
adapterSidecar(name, refs),
|
|
388
|
+
* Custodian is ALWAYS here: a Sandbox without one is a pod that comes up Ready and then parks its
|
|
389
|
+
* Machine forever on a Menu it cannot read (ADR-0013). The User Container joins it only when the
|
|
390
|
+
* spec named one. */
|
|
391
|
+
const sidecarsFor = async (custodian: CustodianComposition, refs: ImageRefs, keys: string[], user?: string) => [
|
|
392
|
+
custodian.custodianContainer,
|
|
422
393
|
...(user !== undefined ? [await userSidecar(refs, user, keys)] : []),
|
|
423
394
|
];
|
|
424
395
|
|
|
425
396
|
// The Harness container's env: the instance's passthrough (`harness.env` — e.g. model
|
|
426
|
-
// config) first, then the mechanism-owned vars (the
|
|
427
|
-
// win on collision. Note the asymmetry stands
|
|
428
|
-
//
|
|
429
|
-
//
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
]
|
|
397
|
+
// config) first, then the mechanism-owned vars (the Custodian's address and every Stand-in, the
|
|
398
|
+
// proxy and the trust bundles — custodian.ts), which win on collision. Note the asymmetry stands
|
|
399
|
+
// (ADR-0013): user env/envFrom land on the HARNESS container only — never on the Custodian, whose
|
|
400
|
+
// mounts are composed here and carry the pod's credentials.
|
|
401
|
+
//
|
|
402
|
+
// The gate rides LAST (ADR-0058): the digest of the bearer the Orchestrator derives for THIS
|
|
403
|
+
// Sandbox. A digest because the Agent reads this env; last because a `harness.env` entry of the
|
|
404
|
+
// same name must not be able to choose the Harness's credential.
|
|
405
|
+
const harnessEnv = (name: string, custodian: CustodianComposition): HarnessEnvVar[] => {
|
|
406
|
+
if (!opts.signingKey) throw new Error("kubectlSandbox: a Harness needs a signingKey to check its bearer");
|
|
407
|
+
return [
|
|
408
|
+
...(opts.env ?? []),
|
|
409
|
+
...custodian.harnessEnv,
|
|
410
|
+
{ name: "JR2_HARNESS_TOKEN_SHA256", value: harnessTokenDigest(opts.signingKey, name) },
|
|
411
|
+
];
|
|
412
|
+
};
|
|
435
413
|
|
|
436
414
|
/**
|
|
437
415
|
* The two init steps, in order (ADR-0037). They are plain container fragments the operator
|
|
@@ -523,10 +501,11 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
523
501
|
req: { name: string; runId: string; workflow: string; image?: string; user?: string; workGroup?: number },
|
|
524
502
|
refs: ImageRefs,
|
|
525
503
|
repos: FencedRepo[],
|
|
504
|
+
custodian: CustodianComposition,
|
|
526
505
|
) => {
|
|
527
506
|
const seat = await seatFor(refs, req.image);
|
|
528
507
|
const sidecars = await sidecarsFor(
|
|
529
|
-
|
|
508
|
+
custodian,
|
|
530
509
|
refs,
|
|
531
510
|
repos.map((r) => r.key),
|
|
532
511
|
req.user,
|
|
@@ -569,14 +548,13 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
569
548
|
fsGroup: req.workGroup ?? DEFAULT_WORK_GROUP,
|
|
570
549
|
// Ordered, and before any container starts: populate `/opt/jr2`, then prove the image on it.
|
|
571
550
|
initContainers: initContainersFor(refs, seat),
|
|
572
|
-
// Never empty
|
|
573
|
-
//
|
|
574
|
-
//
|
|
575
|
-
|
|
576
|
-
env: [...seat.env, ...harnessEnv()],
|
|
551
|
+
// Never empty: the Custodian's address is unconditional. The seat's own vars (the fallback
|
|
552
|
+
// `HOME`) come FIRST, so the instance's `harness.env` can still override them the way it
|
|
553
|
+
// overrides anything the image set.
|
|
554
|
+
env: [...seat.env, ...harnessEnv(req.name, custodian)],
|
|
577
555
|
...(opts.envFrom?.length ? { envFrom: opts.envFrom } : {}),
|
|
578
|
-
// What the AGENT gets: an address on its own loopback
|
|
579
|
-
//
|
|
556
|
+
// What the AGENT gets: an address on its own loopback and a Stand-in for every credential,
|
|
557
|
+
// and no credential anywhere. The Custodian beside it holds them (ADR-0013, ADR-0059).
|
|
580
558
|
sidecars,
|
|
581
559
|
// The Repos this Sandbox attaches, by cache key (ADR-0051). The operator does the rest: a
|
|
582
560
|
// `repo-<key>` volume per entry — the node's cache, hostPath, read-only in the primary
|
|
@@ -599,18 +577,19 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
599
577
|
{ name: "runtime", emptyDir: {} },
|
|
600
578
|
// Only for ADR-0037's fallback seat: uid 1000 on a stranger's base has no home at all.
|
|
601
579
|
...seat.homeVolume,
|
|
602
|
-
|
|
580
|
+
// The Custodian's (its values, leaves and config), and the trust ConfigMap (ADR-0020).
|
|
581
|
+
...custodian.volumes,
|
|
603
582
|
],
|
|
604
|
-
// CR-level volumeMounts land on the HARNESS container only (the operator's contract)
|
|
605
|
-
//
|
|
606
|
-
//
|
|
583
|
+
// CR-level volumeMounts land on the HARNESS container only (the operator's contract): the
|
|
584
|
+
// trust bundles, and none of the Custodian's volumes. The Repo caches are not listed: the
|
|
585
|
+
// operator mounts each `repo-<key>` into this container itself.
|
|
607
586
|
volumeMounts: [
|
|
608
587
|
{ name: "work", mountPath: workRoot },
|
|
609
588
|
// Read-only: nothing writes under `/opt/jr2` at runtime, and the Agent has code execution
|
|
610
589
|
// in this container — leaving its own runtime writable would let a turn edit it.
|
|
611
590
|
{ name: "runtime", mountPath: RUNTIME_MOUNT, readOnly: true },
|
|
612
591
|
...seat.homeMount,
|
|
613
|
-
...
|
|
592
|
+
...custodian.harnessMounts,
|
|
614
593
|
],
|
|
615
594
|
},
|
|
616
595
|
};
|
|
@@ -668,23 +647,13 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
668
647
|
|
|
669
648
|
/**
|
|
670
649
|
* Mint this Sandbox's token into a Secret, BEFORE the CR exists — the operator creates the pod
|
|
671
|
-
* the moment it sees the CR, and a pod whose
|
|
672
|
-
*
|
|
673
|
-
*
|
|
674
|
-
*
|
|
650
|
+
* the moment it sees the CR, and a pod whose volume names an absent Secret never starts.
|
|
651
|
+
* Idempotent by construction: the token is the Sandbox's name, signed (tokens.ts), so a
|
|
652
|
+
* re-provision after an orchestrator restart re-applies the SAME value, and the Custodian that has
|
|
653
|
+
* been holding it all along stays valid.
|
|
675
654
|
*/
|
|
676
655
|
const applyTokenSecret = async (name: string): Promise<void> => {
|
|
677
|
-
if (!opts.signingKey) throw new Error("kubectlSandbox:
|
|
678
|
-
// Fail the provision rather than ship an Adapter that cannot reach the Orchestrator. A mute
|
|
679
|
-
// Adapter is the worst possible outcome: the pod comes up Ready, the Agent is admitted, its
|
|
680
|
-
// tool call dies on `localhost`, and the Machine simply parks forever — a hang with no error.
|
|
681
|
-
if (!orchestratorUrl()) {
|
|
682
|
-
throw new Error(
|
|
683
|
-
"kubectlSandbox: the Adapter has no route to the Orchestrator (no orchestratorUrl — deployed " +
|
|
684
|
-
"instances derive Service DNS from JR2_NAMESPACE). An Agent with no Adapter cannot drive its " +
|
|
685
|
-
"Machine at all (ADR-0013).",
|
|
686
|
-
);
|
|
687
|
-
}
|
|
656
|
+
if (!opts.signingKey) throw new Error("kubectlSandbox: a Custodian needs a signingKey to mint its Sandbox token");
|
|
688
657
|
const secret = {
|
|
689
658
|
apiVersion: "v1",
|
|
690
659
|
kind: "Secret",
|
|
@@ -775,7 +744,17 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
775
744
|
// reaches future Sandboxes without rolling the Orchestrator. Reading before the Secret apply
|
|
776
745
|
// also means an unknown image name costs nothing: no Secret, no CR, nothing to clean up.
|
|
777
746
|
const refs = await readImageRefs(imagesPath);
|
|
778
|
-
|
|
747
|
+
// The same stance for what the Custodian holds (ADR-0059): `jr2 up` resolved it host-side,
|
|
748
|
+
// where `.env` exists, and this process reads the result — never its own config, whose
|
|
749
|
+
// `.env` values are absent in-cluster.
|
|
750
|
+
const manifest = await readHeldManifest(heldPath);
|
|
751
|
+
const custodian = custodianComposition(manifest, {
|
|
752
|
+
image: refs.custodian,
|
|
753
|
+
token: { secret: secretName(req.name), key: "JR2_SANDBOX_TOKEN" },
|
|
754
|
+
sandbox: req.name,
|
|
755
|
+
caBundle: opts.caBundle === true,
|
|
756
|
+
});
|
|
757
|
+
const cr = await crFor(req, refs, repos, custodian);
|
|
779
758
|
|
|
780
759
|
// The Repo resources, BEFORE the CR names them (ADR-0051): a bound one already exists from
|
|
781
760
|
// the boot and only its eviction clock moves — this run's spelling never rewrites the spec
|
|
@@ -786,7 +765,7 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
786
765
|
// a Sandbox whose Repo could not be recorded.
|
|
787
766
|
for (const repo of repos) await opts.repos.ensure(repo);
|
|
788
767
|
|
|
789
|
-
await applyTokenSecret(req.name); // before the CR: the pod's
|
|
768
|
+
await applyTokenSecret(req.name); // before the CR: the pod's Custodian mounts it at start
|
|
790
769
|
await exec(["apply", ...base, "-f", "-"], { input: JSON.stringify(cr) });
|
|
791
770
|
|
|
792
771
|
const applied = Date.now();
|
|
@@ -860,12 +839,7 @@ export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
|
|
|
860
839
|
},
|
|
861
840
|
|
|
862
841
|
async attach(req) {
|
|
863
|
-
const { script, repos, review } = attachScript(req.spec, req.repos, {
|
|
864
|
-
reposMount: REPOS_MOUNT,
|
|
865
|
-
workRoot,
|
|
866
|
-
// The fetch url names the Adapter only when it is somewhere unexpected (ADR-0053).
|
|
867
|
-
...(adapterPort !== DEFAULT_ADAPTER_PORT ? { adapterUrl: `http://127.0.0.1:${adapterPort}` } : {}),
|
|
868
|
-
});
|
|
842
|
+
const { script, repos, review } = attachScript(req.spec, req.repos, { reposMount: REPOS_MOUNT, workRoot });
|
|
869
843
|
// `-c harness` is unchanged and still correct after ADR-0037: the primary container runs the
|
|
870
844
|
// Sandbox Image, so `git` here is the git the user chose. Never `-c user` — that seat is
|
|
871
845
|
// zero-contract, may hold no git at all, and jr2 commands nothing in it (ADR-0005).
|
|
@@ -994,7 +968,7 @@ function staleSlots(
|
|
|
994
968
|
export function attachScript(
|
|
995
969
|
spec: WorkspaceSpec,
|
|
996
970
|
repos: Array<{ slot: string; url: string; ref?: string }>,
|
|
997
|
-
paths: { reposMount: string; workRoot: string
|
|
971
|
+
paths: { reposMount: string; workRoot: string },
|
|
998
972
|
): { script: string; repos: Record<string, string>; review?: Record<string, string> } {
|
|
999
973
|
const worktrees: Record<string, string> = {};
|
|
1000
974
|
const review: Record<string, string> = {};
|
|
@@ -1040,7 +1014,7 @@ export function attachScript(
|
|
|
1040
1014
|
// (ADR-0051). Push still succeeds only with a caller-supplied credential (a forwarded agent
|
|
1041
1015
|
// in the User Container); the pod itself holds none. `--` keeps the url an operand, never an
|
|
1042
1016
|
// option.
|
|
1043
|
-
`git -C ${sq(dflt)} remote set-url origin -- ${sq(fetchUrl(identity
|
|
1017
|
+
`git -C ${sq(dflt)} remote set-url origin -- ${sq(fetchUrl(identity))}`,
|
|
1044
1018
|
`git -C ${sq(dflt)} remote set-url --push origin -- ${sq(repo.url)}`,
|
|
1045
1019
|
// `ext` is on git's own "known scary" list, so its built-in default is `never` and the url
|
|
1046
1020
|
// above would die with `fatal: transport 'ext' not allowed` before the program ever ran.
|
|
@@ -1078,13 +1052,11 @@ export function attachScript(
|
|
|
1078
1052
|
|
|
1079
1053
|
/**
|
|
1080
1054
|
* `origin`'s fetch url for one Repo (ADR-0053): the `ext::` transport, the program's absolute path
|
|
1081
|
-
* on the runtime volume, the service git asks for, and the Repo's identity.
|
|
1082
|
-
*
|
|
1083
|
-
* the program reads `$JR2_ADAPTER_URL` and then falls back to that same address, so spelling it out
|
|
1084
|
-
* unconditionally would put a number in every `git remote -v` that says nothing.
|
|
1055
|
+
* on the runtime volume, the service git asks for, and the Repo's identity. No address: the
|
|
1056
|
+
* program asks the Custodian at its one loopback address, which every Harness pod has (ADR-0059).
|
|
1085
1057
|
*/
|
|
1086
|
-
function fetchUrl(identity: string
|
|
1087
|
-
return `ext::${UPLOAD_PACK} %S ${extArg(identity)}
|
|
1058
|
+
function fetchUrl(identity: string): string {
|
|
1059
|
+
return `ext::${UPLOAD_PACK} %S ${extArg(identity)}`;
|
|
1088
1060
|
}
|
|
1089
1061
|
|
|
1090
1062
|
/**
|
package/src/server.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// JR2_SIGNING_KEY base64 key Sandbox tokens are signed with; from the Secret so live Sandboxes
|
|
12
12
|
// survive a pod restart. Absent → minted into `<dir>/.jr2/secret` (dev-grade).
|
|
13
13
|
// JR2_NAMESPACE the pod's own namespace (Deployment fieldRef) — presence = "deployed":
|
|
14
|
-
// Sandboxes are driven in it, and the
|
|
14
|
+
// Sandboxes are driven in it, and the Custodians' route home is Service DNS.
|
|
15
15
|
// <git.credentials[].token>
|
|
16
16
|
// each token env var the config names rides the instance Secret (ADR-0051):
|
|
17
17
|
// the Orchestrator materializes it into the Repo's credential Secret.
|
|
@@ -25,7 +25,6 @@
|
|
|
25
25
|
// previous deploy bound — an Instance that drops its last `workspace()` leaves `jr2 gc` able to
|
|
26
26
|
// collect what it stopped using.
|
|
27
27
|
|
|
28
|
-
import { createHash } from "node:crypto";
|
|
29
28
|
import { join } from "node:path";
|
|
30
29
|
import { loadConfig } from "./config.ts";
|
|
31
30
|
import { loadWorkflows, startInstance, type RunningInstance } from "./instance.ts";
|
|
@@ -37,7 +36,6 @@ import {
|
|
|
37
36
|
IMAGES_MOUNT,
|
|
38
37
|
INSTANCE_HARNESS_PORT,
|
|
39
38
|
INSTANCE_HARNESS_SERVICE,
|
|
40
|
-
ORCHESTRATOR_SERVICE,
|
|
41
39
|
} from "./names.ts";
|
|
42
40
|
import { partsOf, type CarriedRepo } from "./parts.ts";
|
|
43
41
|
import { kubectlRepoFetches, type RepoFetches } from "./repo-fetch.ts";
|
|
@@ -66,14 +64,12 @@ export async function serverMain(opts: ServerMainOptions): Promise<RunningInstan
|
|
|
66
64
|
const signingKey =
|
|
67
65
|
env.JR2_SIGNING_KEY !== undefined ? Buffer.from(env.JR2_SIGNING_KEY, "base64") : await loadSigningKey(opts.dir);
|
|
68
66
|
|
|
69
|
-
// Deployed (JR2_NAMESPACE set):
|
|
70
|
-
// by nature, which is what lets live Sandboxes outlive orchestrator restarts (ADR-0013).
|
|
67
|
+
// Deployed (JR2_NAMESPACE set): Custodians dial the orchestrator at its own Service DNS — stable
|
|
68
|
+
// by nature, which is what lets live Sandboxes outlive orchestrator restarts (ADR-0013). `jr2 up`
|
|
69
|
+
// writes that address into the Custodian's bootstrap; nothing here passes it on.
|
|
71
70
|
const namespace = env.JR2_NAMESPACE;
|
|
72
71
|
|
|
73
|
-
//
|
|
74
|
-
// sha-256 as its echo gate (ADR-0023, below), so the token must exist before the first
|
|
75
|
-
// provision. From the instance Secret when deployed; per-boot for a host-booted fixture,
|
|
76
|
-
// exactly as before.
|
|
72
|
+
// From the instance Secret when deployed; per-boot for a host-booted fixture.
|
|
77
73
|
const instanceToken = env.JR2_INSTANCE_TOKEN ?? mintInstanceToken();
|
|
78
74
|
|
|
79
75
|
// The data-plane switch (ADR-0012/0031/0051): a registered Machine COMPOSES a Sandbox — read off
|
|
@@ -124,13 +120,6 @@ export async function serverMain(opts: ServerMainOptions): Promise<RunningInstan
|
|
|
124
120
|
name: "JR2_HARNESS_JSON",
|
|
125
121
|
valueFrom: { configMapKeyRef: { name: HARNESS_CONFIGMAP, key: HARNESS_CONFIG_KEY } },
|
|
126
122
|
},
|
|
127
|
-
// The echo gate (ADR-0023): the Harness verifies echo bearers against this sha-256. The
|
|
128
|
-
// digest, never the token — the Agent executes code in the Harness container, and a
|
|
129
|
-
// digest inverts to nothing (the Instance token itself never enters a Sandbox, ADR-0013).
|
|
130
|
-
{
|
|
131
|
-
name: "JR2_ECHO_TOKEN_SHA256",
|
|
132
|
-
value: createHash("sha256").update(instanceToken).digest("base64url"),
|
|
133
|
-
},
|
|
134
123
|
...(config?.harness?.env ?? []).filter((v) => v.valueFrom !== undefined),
|
|
135
124
|
],
|
|
136
125
|
envFrom: [{ secretRef: { name: HARNESS_ENV_SECRET } }, ...(config?.harness?.envFrom ?? [])],
|
|
@@ -139,7 +128,6 @@ export async function serverMain(opts: ServerMainOptions): Promise<RunningInstan
|
|
|
139
128
|
caBundle: config?.harness?.caBundle !== undefined,
|
|
140
129
|
// Which nodes are Sandbox nodes (ADR-0052) — the CR carries it, the operator reads no config.
|
|
141
130
|
...(config?.sandbox ? { placement: config.sandbox } : {}),
|
|
142
|
-
orchestratorUrl: `http://${ORCHESTRATOR_SERVICE}.${namespace}.svc:${port}`,
|
|
143
131
|
signingKey,
|
|
144
132
|
namespace,
|
|
145
133
|
...(opts.exec ? { exec: opts.exec } : {}),
|
package/src/tokens.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Bearer tokens (ADR-0013), because the Sandbox boundary is otherwise theater. The Harness
|
|
2
2
|
// container shares the pod's network namespace, so an Agent can reach the Orchestrator directly;
|
|
3
|
-
// a per-pod NetworkPolicy cannot distinguish its packets from the
|
|
3
|
+
// a per-pod NetworkPolicy cannot distinguish its packets from the Custodian's. Only authentication
|
|
4
4
|
// closes this, and only the token bounds what a caller may DO (as opposed to where it may talk).
|
|
5
5
|
//
|
|
6
6
|
// Two principals, and the asymmetry between them is the whole design:
|
|
@@ -10,24 +10,32 @@
|
|
|
10
10
|
// announced once on stdout (fixtures capture it). An Agent never has it — it
|
|
11
11
|
// never enters a Sandbox.
|
|
12
12
|
//
|
|
13
|
-
// Sandbox token the
|
|
13
|
+
// Sandbox token the Custodian's credential. Authorizes exactly: deliver to `kind: "agent"`
|
|
14
14
|
// registrations whose Sandbox is THIS one. Never a Gate (a compromised Agent
|
|
15
15
|
// must not approve its own review), never another Sandbox (`coding.ts`'s iids
|
|
16
16
|
// are derivable and feature ids are readable from the Work Source, so a merely
|
|
17
17
|
// run-scoped token would let one feature's coder inject a verdict into another
|
|
18
|
-
// feature's reviewer). Delivered as a Secret
|
|
19
|
-
//
|
|
18
|
+
// feature's reviewer). Delivered as a Secret mounted into the Custodian
|
|
19
|
+
// container ALONE, which the Agent cannot read; the Harness holds its
|
|
20
|
+
// Stand-in (ADR-0059).
|
|
20
21
|
// The signed name is a POD hosting Turns, not a Sandbox CR per se: the Instance
|
|
21
|
-
// Harness's
|
|
22
|
+
// Harness's Custodian holds one signed for that placement's Service name
|
|
22
23
|
// (ADR-0031), scoping it to the Menu-only registrations placed there.
|
|
23
24
|
//
|
|
25
|
+
// And one bearer the other way (ADR-0058), which is no principal here because this server never
|
|
26
|
+
// checks it:
|
|
27
|
+
//
|
|
28
|
+
// Harness bearer what the Orchestrator presents to the Harness in one pod — admit, stream,
|
|
29
|
+
// abort, echo. Derived per placement (`harnessToken`); the Harness holds only
|
|
30
|
+
// its sha-256 (`harnessTokenDigest`), so it verifies and mints nothing.
|
|
31
|
+
//
|
|
24
32
|
// The Sandbox token is a SIGNED NAME, not a random string in a table: `<sandbox>.<hmac(key, name)>`,
|
|
25
33
|
// verified by recomputing. Three things fall out that a token table would have to work for — the
|
|
26
34
|
// Orchestrator holds no per-Sandbox state, `provision` stays idempotent (re-minting yields the same
|
|
27
35
|
// token, so a re-applied Secret is a no-op), and a token minted before a restart still verifies
|
|
28
|
-
// after one, which is exactly what ADR-0012's re-attach promises the still-running
|
|
36
|
+
// after one, which is exactly what ADR-0012's re-attach promises the still-running Custodian.
|
|
29
37
|
|
|
30
|
-
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
|
|
38
|
+
import { createHash, createHmac, randomBytes, timingSafeEqual } from "node:crypto";
|
|
31
39
|
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
32
40
|
import { join } from "node:path";
|
|
33
41
|
|
|
@@ -35,7 +43,7 @@ import { join } from "node:path";
|
|
|
35
43
|
export type Principal =
|
|
36
44
|
/** The instance's own operator: the CLI, a human, a webhook translator holding the token. */
|
|
37
45
|
| { kind: "instance" }
|
|
38
|
-
/** One Sandbox's
|
|
46
|
+
/** One Sandbox's Custodian, speaking for the Agent in that pod — and for no one else. */
|
|
39
47
|
| { kind: "sandbox"; sandbox: string };
|
|
40
48
|
|
|
41
49
|
/** Resolve a bearer token to a principal; undefined = not a token we minted (→ 401). */
|
|
@@ -46,6 +54,29 @@ export function sandboxToken(key: Buffer, sandbox: string): string {
|
|
|
46
54
|
return `${sandbox}.${sign(key, sandbox)}`;
|
|
47
55
|
}
|
|
48
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The Harness bearer for one placement (ADR-0058): what the Orchestrator presents on every Harness
|
|
59
|
+
* wire call — admit, stream, abort, echo — to the Harness in the pod `placement` names (a
|
|
60
|
+
* Sandbox, or `jr2-instance-harness`). The same signed-name idea as the Sandbox token, turned the
|
|
61
|
+
* other way: here the Orchestrator is the CALLER. Deterministic for the same reason — a restarted
|
|
62
|
+
* Orchestrator re-derives the bearer a live Harness already checks against.
|
|
63
|
+
*
|
|
64
|
+
* Domain-separated from `sandboxToken` by the `harness:` prefix, which no DNS-label name can
|
|
65
|
+
* produce, so no Sandbox token ever doubles as a Harness bearer or the reverse.
|
|
66
|
+
*/
|
|
67
|
+
export function harnessToken(key: Buffer, placement: string): string {
|
|
68
|
+
return sign(key, `harness:${placement}`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* What the Harness in `placement` holds to CHECK its bearer: the sha-256 of it, never the bearer
|
|
73
|
+
* (ADR-0058). The Agent executes code in that container, and a digest inverts to nothing — so the
|
|
74
|
+
* Harness can verify its own bearer and can mint none, for itself or for any other pod.
|
|
75
|
+
*/
|
|
76
|
+
export function harnessTokenDigest(key: Buffer, placement: string): string {
|
|
77
|
+
return createHash("sha256").update(harnessToken(key, placement)).digest("base64url");
|
|
78
|
+
}
|
|
79
|
+
|
|
49
80
|
/** Mint the per-boot Instance token (opaque and random — it names nothing, it just IS the trust). */
|
|
50
81
|
export function mintInstanceToken(): string {
|
|
51
82
|
return randomBytes(32).toString("base64url");
|
|
@@ -55,7 +86,7 @@ export function mintInstanceToken(): string {
|
|
|
55
86
|
* The instance's HMAC signing key, at `<dir>/.jr2/secret` (0600), created on first use.
|
|
56
87
|
*
|
|
57
88
|
* It MUST outlive the process: an Orchestrator restart leaves live Sandboxes running (ADR-0012 re-attach),
|
|
58
|
-
* and their
|
|
89
|
+
* and their Custodians still hold tokens minted by the process that died. A fresh key would reject
|
|
59
90
|
* every one of them — the Agent would silently lose its only route to its Machine.
|
|
60
91
|
*/
|
|
61
92
|
export async function loadSigningKey(dir: string): Promise<Buffer> {
|
package/src/workspace.ts
CHANGED
|
@@ -688,7 +688,7 @@ function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any
|
|
|
688
688
|
const wrapperRef = self._parent;
|
|
689
689
|
if (!wrapperRef) return;
|
|
690
690
|
const disposeHandles = registerAmbientHandles(wrapperRef, input.handles);
|
|
691
|
-
const detachEcho = runBindingOf(system as AnyActorSystem).echo?.(input.handles.endpoint);
|
|
691
|
+
const detachEcho = runBindingOf(system as AnyActorSystem).echo?.(input.handles.endpoint, input.handles.sandbox);
|
|
692
692
|
return () => {
|
|
693
693
|
detachEcho?.();
|
|
694
694
|
disposeHandles();
|
|
@@ -805,7 +805,7 @@ function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any
|
|
|
805
805
|
endpoint: ctx.endpoint!,
|
|
806
806
|
// Derived, not remembered: the same function every port operation names the CR
|
|
807
807
|
// with, so the Sandbox the registrar publishes — and the Agent actor records on its
|
|
808
|
-
// registration — is the one the
|
|
808
|
+
// registration — is the one the Custodian's token is scoped to, by construction
|
|
809
809
|
// (ADR-0013).
|
|
810
810
|
sandbox: workspaceName(runBindingOf(system as AnyActorSystem).runId, ctx.wsId),
|
|
811
811
|
repos: out.repos,
|