@jr2/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +18 -0
- package/bin/jr2.js +38 -0
- package/manifests/operator.yaml +6669 -0
- package/package.json +56 -0
- package/src/build.ts +1551 -0
- package/src/cli.ts +110 -0
- package/src/client.ts +219 -0
- package/src/commands/down.ts +104 -0
- package/src/commands/gc.ts +73 -0
- package/src/commands/init.ts +238 -0
- package/src/commands/kit.ts +141 -0
- package/src/commands/logs.ts +50 -0
- package/src/commands/run.ts +64 -0
- package/src/commands/runs.ts +19 -0
- package/src/commands/send.ts +83 -0
- package/src/commands/status.ts +90 -0
- package/src/commands/up.ts +1402 -0
- package/src/deploy.ts +592 -0
- package/src/env.ts +68 -0
- package/src/index.ts +11 -0
- package/src/instance.ts +105 -0
- package/src/kube.ts +809 -0
- package/src/nodes.ts +74 -0
- package/src/output.ts +211 -0
- package/src/repo-sweep.ts +134 -0
- package/src/run-id.ts +85 -0
- package/src/sse.ts +41 -0
- package/src/sweep.ts +232 -0
- package/src/typecheck.ts +75 -0
package/src/deploy.ts
ADDED
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
// What `jr2 up` deploys (ADR-0019): pure manifest builders + the small decision helpers, kept free
|
|
2
|
+
// of subprocesses so the converge logic is unit-testable. Object names inside the instance's
|
|
3
|
+
// namespace are constants (namespace is the identity); every object carries the instance label so
|
|
4
|
+
// ownership is derivable from the cluster — there is no local target state.
|
|
5
|
+
|
|
6
|
+
import { readFile } from "node:fs/promises";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
8
|
+
import {
|
|
9
|
+
CA_CONFIGMAP,
|
|
10
|
+
GIT_SSH_SECRET,
|
|
11
|
+
HARNESS_ENV_SECRET,
|
|
12
|
+
IMAGES_CONFIGMAP,
|
|
13
|
+
IMAGES_KEY,
|
|
14
|
+
IMAGES_MOUNT,
|
|
15
|
+
HARNESS_CONFIGMAP,
|
|
16
|
+
HARNESS_CONFIG_KEY,
|
|
17
|
+
INSTANCE_HARNESS_PORT,
|
|
18
|
+
INSTANCE_HARNESS_SERVICE,
|
|
19
|
+
INSTANCE_SECRET,
|
|
20
|
+
KIT_VERSION,
|
|
21
|
+
ORCHESTRATOR_PORT,
|
|
22
|
+
ORCHESTRATOR_SERVICE,
|
|
23
|
+
REPO_CACHE,
|
|
24
|
+
REPO_CACHE_HOSTPATH,
|
|
25
|
+
STATE_PVC,
|
|
26
|
+
type HarnessConfig,
|
|
27
|
+
type SandboxPlacement,
|
|
28
|
+
} from "@jr2/orchestrator";
|
|
29
|
+
|
|
30
|
+
export {
|
|
31
|
+
GIT_SSH_SECRET,
|
|
32
|
+
HARNESS_CONFIGMAP,
|
|
33
|
+
HARNESS_ENV_SECRET,
|
|
34
|
+
INSTANCE_HARNESS_SERVICE,
|
|
35
|
+
KIT_VERSION,
|
|
36
|
+
REPO_CACHE,
|
|
37
|
+
STATE_PVC,
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const LABEL_INSTANCE = "jr2.dev/instance";
|
|
41
|
+
export const LABEL_VERSION = "jr2.dev/version";
|
|
42
|
+
export const LABEL_HASH = "jr2.dev/content-hash";
|
|
43
|
+
|
|
44
|
+
/** The converged name→ref image map, stamped on the Orchestrator Deployment's OWN metadata so the
|
|
45
|
+
* next `jr2 up` can diff it and spend no docker on what has not moved (ADR-0038). An ANNOTATION, not
|
|
46
|
+
* a label: a serialized map blows past the 63-character label-value limit immediately. */
|
|
47
|
+
export const ANNOTATION_IMAGES = "jr2.dev/images";
|
|
48
|
+
|
|
49
|
+
export const ORCHESTRATOR_SA = "jr2-orchestrator";
|
|
50
|
+
|
|
51
|
+
/** The operator's install location — per-cluster, shared by every instance (ADR-0019). */
|
|
52
|
+
export const OPERATOR_NAMESPACE = "jr2-system";
|
|
53
|
+
export const OPERATOR_DEPLOYMENT = "jr2-controller-manager";
|
|
54
|
+
/** The operator Deployment's pod selector, as rendered into `manifests/operator.yaml`. */
|
|
55
|
+
export const OPERATOR_SELECTOR = "control-plane=controller-manager";
|
|
56
|
+
|
|
57
|
+
/** The rendered operator install manifest shipped inside this package (`just operator-manifest`
|
|
58
|
+
* regenerates it from operator/config). The manager image ref is substituted at apply time. */
|
|
59
|
+
export async function operatorManifest(image: string): Promise<string> {
|
|
60
|
+
const raw = await readFile(fileURLToPath(new URL("../manifests/operator.yaml", import.meta.url)), "utf8");
|
|
61
|
+
if (!raw.includes("image: controller:latest")) {
|
|
62
|
+
throw new Error("packaged operator.yaml has no `image: controller:latest` placeholder — regenerate it");
|
|
63
|
+
}
|
|
64
|
+
return raw.replace("image: controller:latest", `image: ${image}`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Compare dotted versions: negative when a < b, 0 when equal, positive when a > b. */
|
|
68
|
+
export function compareVersions(a: string, b: string): number {
|
|
69
|
+
const pa = a.split(/[.-]/).map((s) => Number(s) || 0);
|
|
70
|
+
const pb = b.split(/[.-]/).map((s) => Number(s) || 0);
|
|
71
|
+
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
|
72
|
+
const d = (pa[i] ?? 0) - (pb[i] ?? 0);
|
|
73
|
+
if (d !== 0) return d;
|
|
74
|
+
}
|
|
75
|
+
return 0;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
type KubeManifest = Record<string, unknown>;
|
|
79
|
+
|
|
80
|
+
/** Everything `jr2 up` converges inside the instance's namespace, as one apply-able List. */
|
|
81
|
+
export function instanceObjects(opts: {
|
|
82
|
+
name: string;
|
|
83
|
+
namespace: string;
|
|
84
|
+
image: string;
|
|
85
|
+
hash: string;
|
|
86
|
+
/** Secret data (token, signing key, harness env literals) — written stringData, kube encodes. */
|
|
87
|
+
secretData: Record<string, string>;
|
|
88
|
+
/** The HARNESS containers' env values (creds, provider key) — a SEPARATE Secret from
|
|
89
|
+
* `secretData` by doctrine (ADR-0013): Agent code executes where this lands, so the Instance
|
|
90
|
+
* token and signing key must never share a Secret with it. */
|
|
91
|
+
harnessEnvData: Record<string, string>;
|
|
92
|
+
harness?: HarnessConfig;
|
|
93
|
+
/** The private-CA PEM bundle (`harness.caBundle` file contents, read by `up` — ADR-0020). */
|
|
94
|
+
caBundle?: string;
|
|
95
|
+
/** Every image ref THIS converge resolved (ADR-0037/0038/0049): `{ harness, adapter, operator?,
|
|
96
|
+
* sandbox: { <key>: ref }, sandboxUser: { <key>: user } }`, where a key is a build context's
|
|
97
|
+
* content digest or the reserved `default`. It lands twice, deliberately as one JSON so the
|
|
98
|
+
* record `up` diffs and the map pods read can never disagree: as the `jr2-images` ConfigMap the
|
|
99
|
+
* Orchestrator reads per provision, and as an annotation on the Deployment's own metadata.
|
|
100
|
+
* `sandboxUser` rides along because a provision cannot inspect an image and the pod's uid-1000
|
|
101
|
+
* fallback turns on whether the image declares a `USER` (up.ts, ADR-0037). */
|
|
102
|
+
imageRefs: Record<string, unknown>;
|
|
103
|
+
/** The data plane (ADR-0051): present iff a registered Machine composes a Sandbox, carrying the
|
|
104
|
+
* resolved operator ref — the same binary is the cache agent (`/manager repo-cache`). Absent, no
|
|
105
|
+
* DaemonSet and none of its RBAC is applied; `up` deletes a stale one. */
|
|
106
|
+
repoCache?: { image: string; placement?: SandboxPlacement };
|
|
107
|
+
}): string {
|
|
108
|
+
const labels = { [LABEL_INSTANCE]: opts.name, "app.kubernetes.io/managed-by": "jr2" };
|
|
109
|
+
const meta = (name: string, extra: Record<string, string> = {}): KubeManifest => ({
|
|
110
|
+
name,
|
|
111
|
+
namespace: opts.namespace,
|
|
112
|
+
labels: { ...labels, ...extra },
|
|
113
|
+
});
|
|
114
|
+
const imagesJson = JSON.stringify(opts.imageRefs, null, 2);
|
|
115
|
+
|
|
116
|
+
const items: KubeManifest[] = [
|
|
117
|
+
{
|
|
118
|
+
apiVersion: "v1",
|
|
119
|
+
kind: "PersistentVolumeClaim",
|
|
120
|
+
metadata: meta(STATE_PVC),
|
|
121
|
+
spec: {
|
|
122
|
+
accessModes: ["ReadWriteOnce"],
|
|
123
|
+
resources: { requests: { storage: "1Gi" } },
|
|
124
|
+
},
|
|
125
|
+
},
|
|
126
|
+
{ apiVersion: "v1", kind: "ServiceAccount", metadata: meta(ORCHESTRATOR_SA) },
|
|
127
|
+
{
|
|
128
|
+
// The orchestrator drives Sandbox CRs (+ their token Secrets) in its own namespace
|
|
129
|
+
// (ADR-0012/0013) and creates the Repo CRs the cache agent reconciles (ADR-0051); pod
|
|
130
|
+
// exec/port-forward are the attach path (ADR-0004).
|
|
131
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
132
|
+
kind: "Role",
|
|
133
|
+
metadata: meta(ORCHESTRATOR_SA),
|
|
134
|
+
rules: [
|
|
135
|
+
{ apiGroups: ["core.jr2.dev"], resources: ["sandboxes", "repos"], verbs: ["*"] },
|
|
136
|
+
// patch/update: the token Secret is `kubectl apply`d idempotently and later ownerRef-patched.
|
|
137
|
+
{
|
|
138
|
+
apiGroups: [""],
|
|
139
|
+
resources: ["secrets"],
|
|
140
|
+
verbs: ["get", "list", "create", "delete", "patch", "update"],
|
|
141
|
+
},
|
|
142
|
+
{ apiGroups: [""], resources: ["pods", "pods/log"], verbs: ["get", "list", "watch"] },
|
|
143
|
+
{ apiGroups: [""], resources: ["pods/exec", "pods/portforward"], verbs: ["create"] },
|
|
144
|
+
],
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
148
|
+
kind: "RoleBinding",
|
|
149
|
+
metadata: meta(ORCHESTRATOR_SA),
|
|
150
|
+
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: ORCHESTRATOR_SA },
|
|
151
|
+
subjects: [{ kind: "ServiceAccount", name: ORCHESTRATOR_SA, namespace: opts.namespace }],
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
// What this instance can REACH (ADR-0018), consumed by every Harness container at pod boot.
|
|
155
|
+
// Deployment fact, so it is config (ADR-0050) — and it carries NO Agents: a Machine carries
|
|
156
|
+
// its own and the definition rides each admission (ADR-0049), so a provider edit is the only
|
|
157
|
+
// thing this ConfigMap + a pod restart still delivers.
|
|
158
|
+
apiVersion: "v1",
|
|
159
|
+
kind: "ConfigMap",
|
|
160
|
+
metadata: meta(HARNESS_CONFIGMAP),
|
|
161
|
+
data: {
|
|
162
|
+
[HARNESS_CONFIG_KEY]: JSON.stringify(
|
|
163
|
+
{
|
|
164
|
+
// apiKey is deliberately dropped: it materializes into the Secret as
|
|
165
|
+
// JR2_PROVIDER_API_KEY (`up`), and the Harness reads it from env — a ConfigMap is not
|
|
166
|
+
// a place for a credential.
|
|
167
|
+
provider: opts.harness?.provider
|
|
168
|
+
? {
|
|
169
|
+
id: opts.harness.provider.id,
|
|
170
|
+
api: opts.harness.provider.api,
|
|
171
|
+
baseUrl: opts.harness.provider.baseUrl,
|
|
172
|
+
// Token limits are model properties, not credentials — they ride the ConfigMap
|
|
173
|
+
// so the Harness can register the provider with them.
|
|
174
|
+
contextWindow: opts.harness.provider.contextWindow,
|
|
175
|
+
maxTokens: opts.harness.provider.maxTokens,
|
|
176
|
+
models: opts.harness.provider.models,
|
|
177
|
+
}
|
|
178
|
+
: undefined,
|
|
179
|
+
},
|
|
180
|
+
null,
|
|
181
|
+
2,
|
|
182
|
+
),
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
// The resolved image map (ADR-0037/0038): what a Sandbox is made of, read PER PROVISION
|
|
187
|
+
// from the mount below. A ConfigMap and not Deployment env, because env is a pod-template
|
|
188
|
+
// change: adding a CLI to a Sandbox Dockerfile would roll the Orchestrator and put every
|
|
189
|
+
// live run through snapshot restore (ADR-0007) for a change affecting only FUTURE Sandboxes.
|
|
190
|
+
// Mounted by name, so a content update propagates in place and nothing rolls.
|
|
191
|
+
apiVersion: "v1",
|
|
192
|
+
kind: "ConfigMap",
|
|
193
|
+
metadata: meta(IMAGES_CONFIGMAP),
|
|
194
|
+
data: { [IMAGES_KEY]: imagesJson },
|
|
195
|
+
},
|
|
196
|
+
// The private-CA bundle (ADR-0020) — a ConfigMap, not a Secret: CA certs are public data.
|
|
197
|
+
// kubectlSandbox mounts it into the Harness container and points NODE_EXTRA_CA_CERTS at it.
|
|
198
|
+
...(opts.caBundle
|
|
199
|
+
? [
|
|
200
|
+
{
|
|
201
|
+
apiVersion: "v1",
|
|
202
|
+
kind: "ConfigMap",
|
|
203
|
+
metadata: meta(CA_CONFIGMAP),
|
|
204
|
+
data: { "ca.crt": opts.caBundle },
|
|
205
|
+
},
|
|
206
|
+
]
|
|
207
|
+
: []),
|
|
208
|
+
{
|
|
209
|
+
apiVersion: "v1",
|
|
210
|
+
kind: "Secret",
|
|
211
|
+
metadata: meta(INSTANCE_SECRET),
|
|
212
|
+
type: "Opaque",
|
|
213
|
+
stringData: opts.secretData,
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
// What the HARNESS containers envFrom (see `harnessEnvData` above): Agent creds, never the
|
|
217
|
+
// Instance token. Always applied (possibly empty) so the Sandbox spec can reference it
|
|
218
|
+
// unconditionally.
|
|
219
|
+
apiVersion: "v1",
|
|
220
|
+
kind: "Secret",
|
|
221
|
+
metadata: meta(HARNESS_ENV_SECRET),
|
|
222
|
+
type: "Opaque",
|
|
223
|
+
stringData: opts.harnessEnvData,
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
apiVersion: "apps/v1",
|
|
227
|
+
kind: "Deployment",
|
|
228
|
+
metadata: {
|
|
229
|
+
...meta(ORCHESTRATOR_SERVICE, { [LABEL_HASH]: opts.hash, [LABEL_VERSION]: KIT_VERSION }),
|
|
230
|
+
// The converged image map, on the DEPLOYMENT'S OWN metadata and never on
|
|
231
|
+
// `spec.template.metadata` (ADR-0038). On the pod template it would be part of the pod
|
|
232
|
+
// spec, so every re-resolved ref would roll the Orchestrator — the exact cost the
|
|
233
|
+
// ConfigMap exists to avoid. Here it is a record `jr2 up` reads back and diffs, which is
|
|
234
|
+
// what makes a steady-state converge spend a directory walk and no docker at all.
|
|
235
|
+
annotations: { [ANNOTATION_IMAGES]: imagesJson },
|
|
236
|
+
},
|
|
237
|
+
spec: {
|
|
238
|
+
replicas: 1, // single WRITER (CONTEXT.md): the snapshot store brooks no split-brain
|
|
239
|
+
strategy: { type: "Recreate" }, // two writers may never overlap on the PVC
|
|
240
|
+
selector: { matchLabels: { app: ORCHESTRATOR_SERVICE } },
|
|
241
|
+
template: {
|
|
242
|
+
metadata: { labels: { ...labels, app: ORCHESTRATOR_SERVICE } },
|
|
243
|
+
spec: {
|
|
244
|
+
serviceAccountName: ORCHESTRATOR_SA,
|
|
245
|
+
containers: [
|
|
246
|
+
{
|
|
247
|
+
name: "orchestrator",
|
|
248
|
+
image: opts.image,
|
|
249
|
+
imagePullPolicy: "IfNotPresent",
|
|
250
|
+
ports: [{ containerPort: ORCHESTRATOR_PORT }],
|
|
251
|
+
envFrom: [{ secretRef: { name: INSTANCE_SECRET } }],
|
|
252
|
+
env: [
|
|
253
|
+
// The entrypoint derives its own Service DNS + Sandbox namespace from these.
|
|
254
|
+
{ name: "JR2_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
|
|
255
|
+
// What `/healthz` reports as this instance's identity. The same content address
|
|
256
|
+
// the image tag carries (ADR-0019), in-process so a CLI can ask over HTTP
|
|
257
|
+
// instead of needing kube access to read the Deployment's labels.
|
|
258
|
+
{ name: "JR2_CONTENT_HASH", value: opts.hash },
|
|
259
|
+
],
|
|
260
|
+
// The Orchestrator creates Repo resources and never clones (ADR-0051) — the cache
|
|
261
|
+
// agent on each node does, reading the credential Secret a Repo's `secretRef`
|
|
262
|
+
// names — so nothing of git's is mounted here: state and the image map only.
|
|
263
|
+
volumeMounts: [
|
|
264
|
+
{ name: "state", mountPath: "/instance/.jr2" },
|
|
265
|
+
// The image map, read per provision (ADR-0038). A mount, so `jr2 up` rewriting
|
|
266
|
+
// it costs one kubelet propagation window instead of a rollout.
|
|
267
|
+
{ name: "images", mountPath: IMAGES_MOUNT, readOnly: true },
|
|
268
|
+
],
|
|
269
|
+
// The period, not the boot, is what `up`'s rollout wait measures. Measured: the
|
|
270
|
+
// container answers `/healthz` 1.1s after it starts, and the default 10s period
|
|
271
|
+
// billed that as 11.0s — one probe fired before the server was up, then a whole
|
|
272
|
+
// missed period. 2s quantizes a ~1s boot at ~1s.
|
|
273
|
+
//
|
|
274
|
+
// `failureThreshold` is then raised to hold the tolerance the default period gave,
|
|
275
|
+
// because ONE knob sets two unrelated things: how fast a boot is noticed, and how
|
|
276
|
+
// long a running pod may stall before the kubelet takes it out of service. The
|
|
277
|
+
// Orchestrator is `replicas: 1` (CONTEXT.md: single writer), so its Service has
|
|
278
|
+
// exactly one endpoint and losing it is an outage, not a failover — and its store
|
|
279
|
+
// writes are synchronous, so a GC pause or a node busy with a docker build can
|
|
280
|
+
// silence `/healthz` for seconds. 15 × 2s keeps the 30s the default 3 × 10s gave.
|
|
281
|
+
readinessProbe: {
|
|
282
|
+
httpGet: { path: "/healthz", port: ORCHESTRATOR_PORT },
|
|
283
|
+
initialDelaySeconds: 1,
|
|
284
|
+
periodSeconds: 2,
|
|
285
|
+
failureThreshold: 15,
|
|
286
|
+
},
|
|
287
|
+
},
|
|
288
|
+
],
|
|
289
|
+
volumes: [
|
|
290
|
+
{ name: "state", persistentVolumeClaim: { claimName: STATE_PVC } },
|
|
291
|
+
{ name: "images", configMap: { name: IMAGES_CONFIGMAP } },
|
|
292
|
+
],
|
|
293
|
+
},
|
|
294
|
+
},
|
|
295
|
+
},
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
apiVersion: "v1",
|
|
299
|
+
kind: "Service",
|
|
300
|
+
metadata: meta(ORCHESTRATOR_SERVICE),
|
|
301
|
+
spec: {
|
|
302
|
+
selector: { app: ORCHESTRATOR_SERVICE },
|
|
303
|
+
ports: [{ port: ORCHESTRATOR_PORT, targetPort: ORCHESTRATOR_PORT }],
|
|
304
|
+
},
|
|
305
|
+
},
|
|
306
|
+
...(opts.repoCache ? repoCacheObjects({ ...opts.repoCache, namespace: opts.namespace, labels, meta }) : []),
|
|
307
|
+
];
|
|
308
|
+
|
|
309
|
+
return JSON.stringify({ apiVersion: "v1", kind: "List", items });
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** Where the cache agent's pod sees the node's directory: `--cache-dir`'s default. */
|
|
313
|
+
const REPO_CACHE_MOUNT = "/cache";
|
|
314
|
+
/** The agent's `$HOME` — an emptyDir, where it writes a deploy key for the life of the pod. */
|
|
315
|
+
const REPO_CACHE_HOME = "/home/jr2";
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* The data plane's node half (ADR-0051, ADR-0004): the cache agent as a DaemonSet, one pod per Sandbox node,
|
|
319
|
+
* each the one writer of `/var/lib/jr2/<namespace>/repos` on its node — the hostPath the operator
|
|
320
|
+
* mounts one leaf of, read-only, into every Sandbox there that names the key. It runs the operator
|
|
321
|
+
* image (`/manager repo-cache`), so the kit's operator ref is resolved even when the operator layer
|
|
322
|
+
* itself is unmanaged.
|
|
323
|
+
*
|
|
324
|
+
* The seat is ROOT, and deliberately so: the kubelet creates a `DirectoryOrCreate` hostPath owned by
|
|
325
|
+
* root, and a Sandbox's mount of the leaf may exist before the agent has written anything there.
|
|
326
|
+
* Everything else is hardened as the operator's baseline is — no capabilities, no escalation, a
|
|
327
|
+
* read-only root filesystem (the two writable places are the emptyDirs below), the default seccomp
|
|
328
|
+
* profile. It carries the Sandbox pod's own `nodeSelector` and `tolerations` and nothing wider
|
|
329
|
+
* (ADR-0052), so an agent lands on exactly the nodes a Sandbox can be placed on. The ServiceAccount token IS mounted: the agent is a client of the Repo resources and
|
|
330
|
+
* of the pods on its node (never of Sandboxes — demand is a pod's mount), unlike a Sandbox, whose
|
|
331
|
+
* north star is never reaching the API.
|
|
332
|
+
*/
|
|
333
|
+
function repoCacheObjects(opts: {
|
|
334
|
+
image: string;
|
|
335
|
+
/** The Instance's Sandbox node predicate (ADR-0052): the same `nodeSelector` and `tolerations`
|
|
336
|
+
* every Sandbox pod carries, so the agent runs on exactly the Sandbox nodes. */
|
|
337
|
+
placement?: SandboxPlacement;
|
|
338
|
+
namespace: string;
|
|
339
|
+
labels: Record<string, string>;
|
|
340
|
+
meta: (name: string, extra?: Record<string, string>) => KubeManifest;
|
|
341
|
+
}): KubeManifest[] {
|
|
342
|
+
const { image, namespace, labels, meta, placement } = opts;
|
|
343
|
+
return [
|
|
344
|
+
{ apiVersion: "v1", kind: "ServiceAccount", metadata: meta(REPO_CACHE) },
|
|
345
|
+
{
|
|
346
|
+
// What one agent writes on the API is its own node's entry in each Repo's status; it reads
|
|
347
|
+
// the Repos, the pods on its node that mount their caches (demand and eviction are a pod's
|
|
348
|
+
// mount, not a Sandbox resource — ADR-0051), and the credential Secret a Repo's `secretRef`
|
|
349
|
+
// names — nothing it could create or delete.
|
|
350
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
351
|
+
kind: "Role",
|
|
352
|
+
metadata: meta(REPO_CACHE),
|
|
353
|
+
rules: [
|
|
354
|
+
{ apiGroups: ["core.jr2.dev"], resources: ["repos"], verbs: ["get", "list", "watch"] },
|
|
355
|
+
{ apiGroups: ["core.jr2.dev"], resources: ["repos/status"], verbs: ["get", "patch", "update"] },
|
|
356
|
+
{ apiGroups: [""], resources: ["pods"], verbs: ["get", "list", "watch"] },
|
|
357
|
+
{ apiGroups: [""], resources: ["secrets"], verbs: ["get"] },
|
|
358
|
+
],
|
|
359
|
+
},
|
|
360
|
+
{
|
|
361
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
362
|
+
kind: "RoleBinding",
|
|
363
|
+
metadata: meta(REPO_CACHE),
|
|
364
|
+
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: REPO_CACHE },
|
|
365
|
+
subjects: [{ kind: "ServiceAccount", name: REPO_CACHE, namespace }],
|
|
366
|
+
},
|
|
367
|
+
{
|
|
368
|
+
apiVersion: "apps/v1",
|
|
369
|
+
kind: "DaemonSet",
|
|
370
|
+
metadata: meta(REPO_CACHE, { [LABEL_VERSION]: KIT_VERSION }),
|
|
371
|
+
spec: {
|
|
372
|
+
selector: { matchLabels: { app: REPO_CACHE } },
|
|
373
|
+
updateStrategy: { type: "RollingUpdate" },
|
|
374
|
+
template: {
|
|
375
|
+
metadata: { labels: { ...labels, app: REPO_CACHE } },
|
|
376
|
+
spec: {
|
|
377
|
+
serviceAccountName: REPO_CACHE,
|
|
378
|
+
automountServiceAccountToken: true,
|
|
379
|
+
// One agent per Sandbox node (ADR-0052): the pod's own placement, verbatim. A node no
|
|
380
|
+
// Sandbox can reach gets no agent — a cache there is a clone nobody reads, and an
|
|
381
|
+
// affinity term the scheduler cannot honor. (The DaemonSet controller still adds its
|
|
382
|
+
// own toleration for `unschedulable`, so a cordoned Sandbox node keeps its agent.)
|
|
383
|
+
...(placement?.nodeSelector ? { nodeSelector: placement.nodeSelector } : {}),
|
|
384
|
+
...(placement?.tolerations?.length ? { tolerations: placement.tolerations } : {}),
|
|
385
|
+
containers: [
|
|
386
|
+
{
|
|
387
|
+
name: "agent",
|
|
388
|
+
image,
|
|
389
|
+
imagePullPolicy: "IfNotPresent",
|
|
390
|
+
command: ["/manager", "repo-cache"],
|
|
391
|
+
env: [
|
|
392
|
+
// The downward API names the node this pod is the writer for, and the namespace
|
|
393
|
+
// whose Repos and pods it watches (the agent's `--node` / `--namespace`).
|
|
394
|
+
{ name: "NODE_NAME", valueFrom: { fieldRef: { fieldPath: "spec.nodeName" } } },
|
|
395
|
+
{ name: "JR2_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
|
|
396
|
+
{ name: "HOME", value: REPO_CACHE_HOME },
|
|
397
|
+
],
|
|
398
|
+
volumeMounts: [
|
|
399
|
+
{ name: "cache", mountPath: REPO_CACHE_MOUNT },
|
|
400
|
+
{ name: "home", mountPath: REPO_CACHE_HOME },
|
|
401
|
+
{ name: "tmp", mountPath: "/tmp" },
|
|
402
|
+
],
|
|
403
|
+
resources: { requests: { cpu: "20m", memory: "64Mi" } },
|
|
404
|
+
securityContext: {
|
|
405
|
+
runAsUser: 0,
|
|
406
|
+
runAsGroup: 0,
|
|
407
|
+
allowPrivilegeEscalation: false,
|
|
408
|
+
capabilities: { drop: ["ALL"] },
|
|
409
|
+
readOnlyRootFilesystem: true,
|
|
410
|
+
seccompProfile: { type: "RuntimeDefault" },
|
|
411
|
+
},
|
|
412
|
+
},
|
|
413
|
+
],
|
|
414
|
+
volumes: [
|
|
415
|
+
{
|
|
416
|
+
name: "cache",
|
|
417
|
+
hostPath: { path: `${REPO_CACHE_HOSTPATH}/${namespace}/repos`, type: "DirectoryOrCreate" },
|
|
418
|
+
},
|
|
419
|
+
{ name: "home", emptyDir: {} },
|
|
420
|
+
{ name: "tmp", emptyDir: {} },
|
|
421
|
+
],
|
|
422
|
+
},
|
|
423
|
+
},
|
|
424
|
+
},
|
|
425
|
+
},
|
|
426
|
+
];
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/** Where the Instance Harness's Harness container sees the CA bundle — the same path
|
|
430
|
+
* `kubectlSandbox` mounts it at in a Sandbox pod (ADR-0020). */
|
|
431
|
+
const CA_MOUNT = "/etc/jr2/ca";
|
|
432
|
+
|
|
433
|
+
/** The Adapter's port on the pod's loopback — the same default the Sandbox pod uses. */
|
|
434
|
+
const ADAPTER_PORT = 8081;
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* The Instance Harness (ADR-0031): the per-instance Harness Deployment + Service `jr2 up`
|
|
438
|
+
* converges whenever an Agent a registered Machine CARRIES declares `workspace: "none"`
|
|
439
|
+
* (ADR-0049's walk) — the placement for every Menu-only Agent's Turn, regardless of any enclosing
|
|
440
|
+
* Workspace. The one Harness shape, minus the Workspace: the stock Harness image plus the Adapter
|
|
441
|
+
* sidecar, the same harness-config ConfigMap and env/envFrom/CA wiring a Sandbox's Harness
|
|
442
|
+
* container gets — and NO `/work` volume, no attach step. It runs the STOCK image permanently: `workspace: "none"` withholds the whole
|
|
443
|
+
* Working toolset (ADR-0028), so there are no tools to carry and no Sandbox Image to resolve
|
|
444
|
+
* (ADR-0037). No config key names, sizes, addresses, or enables it: the Machine walk is the
|
|
445
|
+
* entire surface.
|
|
446
|
+
*/
|
|
447
|
+
export function instanceHarnessObjects(opts: {
|
|
448
|
+
name: string;
|
|
449
|
+
namespace: string;
|
|
450
|
+
/** The RESOLVED stock Harness ref (ADR-0018/0031/0038) — a content-addressed tag in a kit
|
|
451
|
+
* checkout, the published `<kitversion>` tag installed. Not an override seat: `images.harness`
|
|
452
|
+
* is gone, and the only Harness this instance can run is the one `jr2 up` resolved.
|
|
453
|
+
*
|
|
454
|
+
* The accepted asymmetry: the Instance Harness names its images HERE, in the pod template, while
|
|
455
|
+
* a Sandbox's refs travel through the `jr2-images` ConfigMap. Both are right for what they are —
|
|
456
|
+
* this Deployment is supposed to roll when its image moves; the Orchestrator is not. */
|
|
457
|
+
harnessImage: string;
|
|
458
|
+
/** The resolved Adapter ref: the Harness's one menu-delivery path, kept even though a `"none"`
|
|
459
|
+
* Agent cannot execute code — forking the path for one pod buys a divergence ADR-0031 declines. */
|
|
460
|
+
adapterImage: string;
|
|
461
|
+
harness?: HarnessConfig;
|
|
462
|
+
/** The instance ships a private-CA bundle (ADR-0020): mount `jr2-ca` into the Harness container. */
|
|
463
|
+
caBundle?: boolean;
|
|
464
|
+
/** The Instance token's sha-256 — the Harness's echo gate (ADR-0023). The digest, never the
|
|
465
|
+
* token: the same env every Sandbox Harness container gets, kept here so the one-Harness-shape
|
|
466
|
+
* claim stays whole even though nothing narrates to the Instance Harness today. */
|
|
467
|
+
echoTokenSha256?: string;
|
|
468
|
+
}): string {
|
|
469
|
+
const labels = { [LABEL_INSTANCE]: opts.name, "app.kubernetes.io/managed-by": "jr2" };
|
|
470
|
+
const meta = (): KubeManifest => ({
|
|
471
|
+
name: INSTANCE_HARNESS_SERVICE,
|
|
472
|
+
namespace: opts.namespace,
|
|
473
|
+
labels,
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
// The per-container half of the operator's baseline (hardenedContainerSecurityContext there):
|
|
477
|
+
// drop every capability, forbid escalation, non-root under the default seccomp profile.
|
|
478
|
+
const hardenedContainerSecurityContext = () => ({
|
|
479
|
+
runAsNonRoot: true,
|
|
480
|
+
allowPrivilegeEscalation: false,
|
|
481
|
+
capabilities: { drop: ["ALL"] },
|
|
482
|
+
seccompProfile: { type: "RuntimeDefault" },
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
const harnessContainer = {
|
|
486
|
+
name: "harness",
|
|
487
|
+
image: opts.harnessImage,
|
|
488
|
+
imagePullPolicy: "IfNotPresent",
|
|
489
|
+
ports: [{ containerPort: INSTANCE_HARNESS_PORT }],
|
|
490
|
+
// The same asymmetry the Sandbox pod builds (ADR-0013/0020): the mounted harness config and
|
|
491
|
+
// the instance's valueFrom entries ride `env` (literal values live in the jr2-harness-env
|
|
492
|
+
// Secret), the CA trust lands here and nowhere else, and no credential ever does.
|
|
493
|
+
env: [
|
|
494
|
+
{
|
|
495
|
+
name: "JR2_HARNESS_JSON",
|
|
496
|
+
valueFrom: { configMapKeyRef: { name: HARNESS_CONFIGMAP, key: HARNESS_CONFIG_KEY } },
|
|
497
|
+
},
|
|
498
|
+
// The placement gate (ADR-0031): every admission carries its own definition (ADR-0049) and
|
|
499
|
+
// the wire is unauthenticated in-cluster, so the Harness itself refuses any admission whose
|
|
500
|
+
// definition declares Workspace access — Menu-only Agents alone run here, which is what
|
|
501
|
+
// makes "no code execution in this pod" true rather than asserted.
|
|
502
|
+
{ name: "JR2_MENU_ONLY", value: "1" },
|
|
503
|
+
...(opts.echoTokenSha256 ? [{ name: "JR2_ECHO_TOKEN_SHA256", value: opts.echoTokenSha256 }] : []),
|
|
504
|
+
...(opts.harness?.env ?? []).filter((v) => v.valueFrom !== undefined),
|
|
505
|
+
{ name: "JR2_ADAPTER_URL", value: `http://127.0.0.1:${ADAPTER_PORT}` },
|
|
506
|
+
...(opts.caBundle ? [{ name: "NODE_EXTRA_CA_CERTS", value: `${CA_MOUNT}/ca.crt` }] : []),
|
|
507
|
+
],
|
|
508
|
+
envFrom: [{ secretRef: { name: HARNESS_ENV_SECRET } }, ...(opts.harness?.envFrom ?? [])],
|
|
509
|
+
...(opts.caBundle ? { volumeMounts: [{ name: "ca", mountPath: CA_MOUNT, readOnly: true }] } : {}),
|
|
510
|
+
// The operator probes a Sandbox's Harness the same way: serving = the socket accepts.
|
|
511
|
+
// Period and threshold as reasoned on the Orchestrator above: the default 10s period is the
|
|
512
|
+
// rollout wait rather than the boot, and the threshold then has to carry the stall tolerance
|
|
513
|
+
// the period used to supply. This pod is single-replica too.
|
|
514
|
+
readinessProbe: {
|
|
515
|
+
tcpSocket: { port: INSTANCE_HARNESS_PORT },
|
|
516
|
+
initialDelaySeconds: 1,
|
|
517
|
+
periodSeconds: 2,
|
|
518
|
+
failureThreshold: 15,
|
|
519
|
+
},
|
|
520
|
+
securityContext: hardenedContainerSecurityContext(),
|
|
521
|
+
};
|
|
522
|
+
|
|
523
|
+
const adapterContainer = {
|
|
524
|
+
name: "adapter",
|
|
525
|
+
image: opts.adapterImage,
|
|
526
|
+
imagePullPolicy: "IfNotPresent",
|
|
527
|
+
securityContext: hardenedContainerSecurityContext(),
|
|
528
|
+
env: [
|
|
529
|
+
{
|
|
530
|
+
name: "JR2_ORCHESTRATOR_URL",
|
|
531
|
+
value: `http://${ORCHESTRATOR_SERVICE}.${opts.namespace}.svc:${ORCHESTRATOR_PORT}`,
|
|
532
|
+
},
|
|
533
|
+
{ name: "JR2_ADAPTER_PORT", value: String(ADAPTER_PORT) },
|
|
534
|
+
// The Adapter's bearer env, carrying a sandbox-style token SIGNED FOR THIS PLACEMENT's
|
|
535
|
+
// name (`up.ts` mints it into the instance Secret): ADR-0013's delivery doctrine, extended
|
|
536
|
+
// to the second placement — the token speaks only for registrations that record the
|
|
537
|
+
// Instance Harness as the pod hosting their Turn (tokens.ts), never a Workspace's, and the
|
|
538
|
+
// Instance token itself never enters this pod. The credential lives in this container,
|
|
539
|
+
// where no Agent can read it — and `JR2_MENU_ONLY` above is what keeps that true: only
|
|
540
|
+
// Menu-only Agents run here, so nothing in this pod executes code (ADR-0031's
|
|
541
|
+
// defense-in-depth bonus).
|
|
542
|
+
{
|
|
543
|
+
name: "JR2_SANDBOX_TOKEN",
|
|
544
|
+
valueFrom: { secretKeyRef: { name: INSTANCE_SECRET, key: "JR2_INSTANCE_HARNESS_TOKEN" } },
|
|
545
|
+
},
|
|
546
|
+
],
|
|
547
|
+
};
|
|
548
|
+
|
|
549
|
+
const items: KubeManifest[] = [
|
|
550
|
+
{
|
|
551
|
+
apiVersion: "apps/v1",
|
|
552
|
+
kind: "Deployment",
|
|
553
|
+
metadata: { ...meta(), labels: { ...labels, [LABEL_VERSION]: KIT_VERSION } },
|
|
554
|
+
spec: {
|
|
555
|
+
// ONE replica, Recreate: a conversation is an Instance ID on one Harness PROCESS
|
|
556
|
+
// (ADR-0031) — two pods behind this Service would route one conversation to two servers,
|
|
557
|
+
// the exact amnesia definition-wins placement exists to prevent. A restart loses the
|
|
558
|
+
// conversations (live-only, ADR-0023); the Deployment restores the endpoint, not the
|
|
559
|
+
// history.
|
|
560
|
+
replicas: 1,
|
|
561
|
+
strategy: { type: "Recreate" },
|
|
562
|
+
selector: { matchLabels: { app: INSTANCE_HARNESS_SERVICE } },
|
|
563
|
+
template: {
|
|
564
|
+
metadata: { labels: { ...labels, app: INSTANCE_HARNESS_SERVICE } },
|
|
565
|
+
spec: {
|
|
566
|
+
containers: [harnessContainer, adapterContainer],
|
|
567
|
+
// The operator's isolation baseline (sandbox_controller.go), mirrored: same Harness
|
|
568
|
+
// image, same "never reach the Kubernetes API" north star — JR2_MENU_ONLY makes code
|
|
569
|
+
// execution here unlikely, not unimaginable.
|
|
570
|
+
automountServiceAccountToken: false,
|
|
571
|
+
securityContext: {
|
|
572
|
+
runAsNonRoot: true,
|
|
573
|
+
seccompProfile: { type: "RuntimeDefault" },
|
|
574
|
+
},
|
|
575
|
+
...(opts.caBundle ? { volumes: [{ name: "ca", configMap: { name: CA_CONFIGMAP } }] } : {}),
|
|
576
|
+
},
|
|
577
|
+
},
|
|
578
|
+
},
|
|
579
|
+
},
|
|
580
|
+
{
|
|
581
|
+
apiVersion: "v1",
|
|
582
|
+
kind: "Service",
|
|
583
|
+
metadata: meta(),
|
|
584
|
+
spec: {
|
|
585
|
+
selector: { app: INSTANCE_HARNESS_SERVICE },
|
|
586
|
+
ports: [{ port: INSTANCE_HARNESS_PORT, targetPort: INSTANCE_HARNESS_PORT }],
|
|
587
|
+
},
|
|
588
|
+
},
|
|
589
|
+
];
|
|
590
|
+
|
|
591
|
+
return JSON.stringify({ apiVersion: "v1", kind: "List", items });
|
|
592
|
+
}
|
package/src/env.ts
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// `.env` at the instance root (ADR-0019). Deployment-varying values — a vLLM `baseUrl`, a model
|
|
2
|
+
// specifier, provider keys — must NOT be hardcoded in `jr2.config.ts`; the config reads them from
|
|
3
|
+
// `process.env`, and the uncommitted `.env` beside it is where they live. This is the loader that
|
|
4
|
+
// makes that literal, rather than a `set -a; . ./.env; set +a` ritual the user has to remember (and
|
|
5
|
+
// whose omission fails SILENTLY: an unset var just makes `harness.provider` undefined).
|
|
6
|
+
//
|
|
7
|
+
// Discovery mirrors `resolveRoot` — walk up from cwd to the folder holding `jr2.config.ts`, read the
|
|
8
|
+
// `.env` beside it — so `jr2` works from any subdirectory of an instance. Outside an instance
|
|
9
|
+
// (`jr2 init`) or with no file: nothing happens, never an error.
|
|
10
|
+
//
|
|
11
|
+
// Precedence: the REAL environment always wins. `VLLM_BASE_URL=… jr2 up` and an exported shell var
|
|
12
|
+
// both override the file, so `.env` is the default layer, not an override one.
|
|
13
|
+
//
|
|
14
|
+
// Applied keys (never values) are announced on stderr, for the same reason `resolveTarget` prints
|
|
15
|
+
// its target: ambient context that changes what a command does stays visible.
|
|
16
|
+
|
|
17
|
+
import { readFileSync } from "node:fs";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { findRoot } from "./instance.ts";
|
|
20
|
+
import { activity, type Io } from "./output.ts";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Load the instance's `.env` into `io.env`, without clobbering what is already set.
|
|
24
|
+
*
|
|
25
|
+
* Mutates `io.env` in place ON PURPOSE: in the real bin that object IS `process.env`, and instance
|
|
26
|
+
* config modules read `process.env` directly, so a copy would never reach them. Must therefore run
|
|
27
|
+
* before anything imports `jr2.config.ts` — `main` calls it first thing.
|
|
28
|
+
*/
|
|
29
|
+
export function loadDotenv(io: Io): void {
|
|
30
|
+
const root = findRoot(io.cwd);
|
|
31
|
+
if (!root) return;
|
|
32
|
+
|
|
33
|
+
let text: string;
|
|
34
|
+
try {
|
|
35
|
+
text = readFileSync(join(root, ".env"), "utf8");
|
|
36
|
+
} catch {
|
|
37
|
+
return; // no .env is the normal case, not a failure
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const applied: string[] = [];
|
|
41
|
+
for (const [key, value] of Object.entries(parseDotenv(text))) {
|
|
42
|
+
if (io.env[key] !== undefined) continue; // the real environment wins
|
|
43
|
+
io.env[key] = value;
|
|
44
|
+
applied.push(key);
|
|
45
|
+
}
|
|
46
|
+
if (applied.length > 0) activity(io, `→ .env: ${applied.join(", ")}`);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// One assignment per match: an optional `export `, the key, then a single-quoted (literal),
|
|
50
|
+
// double-quoted (escapes honored), or bare value. Quoted forms may span lines — deploy keys are a
|
|
51
|
+
// plausible thing to park here. A bare value ends at ` #`, so trailing comments don't leak in.
|
|
52
|
+
const ASSIGNMENT =
|
|
53
|
+
/^[ \t]*(?:export[ \t]+)?([A-Za-z_][A-Za-z0-9_]*)[ \t]*=[ \t]*(?:'([^']*)'|"((?:[^"\\]|\\.)*)"|([^#\r\n]*?))[ \t]*(?:#[^\r\n]*)?$/gm;
|
|
54
|
+
|
|
55
|
+
/** Parse `.env` text into a plain record. Malformed lines are ignored, not fatal. */
|
|
56
|
+
export function parseDotenv(text: string): Record<string, string> {
|
|
57
|
+
const out: Record<string, string> = {};
|
|
58
|
+
for (const [, key, single, double, bare] of text.replace(/^/, "").matchAll(ASSIGNMENT)) {
|
|
59
|
+
if (!key) continue;
|
|
60
|
+
out[key] = single ?? (double !== undefined ? unescape(double) : (bare ?? ""));
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Double-quoted values honor the usual escapes; everything else passes through verbatim. */
|
|
66
|
+
function unescape(value: string): string {
|
|
67
|
+
return value.replace(/\\([nrt"'\\])/g, (_, ch: string) => ({ n: "\n", r: "\r", t: "\t" })[ch] ?? ch);
|
|
68
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Library entry for `@jr2/cli`: the dispatch + the HTTP client surface, for programmatic callers and
|
|
2
|
+
// tests. The `jr2` binary itself is `bin/jr2.js`.
|
|
3
|
+
|
|
4
|
+
export { main } from "./cli.ts";
|
|
5
|
+
export { JR2Client } from "./client.ts";
|
|
6
|
+
export type { RunStatus, RunFeedEvent, RunEvent, FetchLike } from "./client.ts";
|
|
7
|
+
export type { Io } from "./output.ts";
|
|
8
|
+
export { resolveRoot, resolveTarget } from "./instance.ts";
|
|
9
|
+
export type { Target, TargetOptions } from "./instance.ts";
|
|
10
|
+
export { kubectlKube } from "./kube.ts";
|
|
11
|
+
export type { KubePort } from "./kube.ts";
|