@workos/quickstudy 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +270 -0
  3. package/examples/harbor-notes/README.md +40 -0
  4. package/examples/harbor-notes/evals/create-note/EVAL.ts +14 -0
  5. package/examples/harbor-notes/evals/create-note/PROMPT.md +9 -0
  6. package/examples/harbor-notes/evals/create-note/local/README.txt +1 -0
  7. package/examples/harbor-notes/experiments/scripted.ts +6 -0
  8. package/examples/harbor-notes/package.json +6 -0
  9. package/examples/harbor-notes/quickstudy.identity.json +1 -0
  10. package/examples/harbor-notes/runtime.ts +48 -0
  11. package/examples/harbor-notes/semantic-example.ts +21 -0
  12. package/images/agent-runtime/Dockerfile +58 -0
  13. package/images/egress-proxy/Dockerfile +28 -0
  14. package/images/mcp-proxy/Dockerfile +30 -0
  15. package/package.json +53 -0
  16. package/src/adapters/claude.ts +107 -0
  17. package/src/adapters/codex.ts +107 -0
  18. package/src/adapters/echo.ts +57 -0
  19. package/src/adapters/parse.ts +117 -0
  20. package/src/adapters/types.ts +152 -0
  21. package/src/build-info.generated.ts +12 -0
  22. package/src/cli.ts +787 -0
  23. package/src/completeness.ts +104 -0
  24. package/src/diagnose/excerpt.ts +106 -0
  25. package/src/diagnose/prompt.ts +175 -0
  26. package/src/diagnose/render.ts +55 -0
  27. package/src/diagnose/run.ts +290 -0
  28. package/src/diagnose/select.ts +110 -0
  29. package/src/diagnose/types.ts +88 -0
  30. package/src/evals/discovery.ts +173 -0
  31. package/src/evals/prompt.ts +190 -0
  32. package/src/evals/result.ts +10 -0
  33. package/src/evals/types.ts +115 -0
  34. package/src/execution-policy.ts +71 -0
  35. package/src/experiments/discovery.ts +76 -0
  36. package/src/experiments/groups.ts +119 -0
  37. package/src/experiments/types.ts +116 -0
  38. package/src/export-types.ts +127 -0
  39. package/src/export.ts +381 -0
  40. package/src/hash.ts +74 -0
  41. package/src/identity-diff.ts +30 -0
  42. package/src/ids.ts +30 -0
  43. package/src/index.ts +58 -0
  44. package/src/isolation/docker.ts +639 -0
  45. package/src/isolation/image-contexts.generated.ts +927 -0
  46. package/src/isolation/images.ts +138 -0
  47. package/src/isolation/mcp-proxy/server.ts +260 -0
  48. package/src/isolation/mcp.ts +144 -0
  49. package/src/isolation/proxy/allowlist.ts +148 -0
  50. package/src/isolation/proxy/server.ts +382 -0
  51. package/src/llm.ts +132 -0
  52. package/src/manifest.ts +228 -0
  53. package/src/model-identity.ts +12 -0
  54. package/src/plan.ts +55 -0
  55. package/src/probe.ts +426 -0
  56. package/src/report/pass-at-k.ts +76 -0
  57. package/src/report/report.ts +731 -0
  58. package/src/runner/context.ts +96 -0
  59. package/src/runner/deadline.ts +37 -0
  60. package/src/runner/execute.ts +992 -0
  61. package/src/runner/run-lock.ts +32 -0
  62. package/src/runner/scheduler.ts +62 -0
  63. package/src/runner/score-worker.ts +107 -0
  64. package/src/runner/scorer-worker.ts +61 -0
  65. package/src/runtime/types.ts +89 -0
  66. package/src/secrets.ts +151 -0
  67. package/src/semantic.ts +185 -0
  68. package/src/serve.ts +52 -0
  69. package/src/source-identity.ts +76 -0
  70. package/src/store/artifacts.ts +146 -0
  71. package/src/store/db.ts +318 -0
  72. package/src/store/schema.ts +39 -0
  73. package/src/surface-usage.ts +297 -0
  74. package/src/ui-bundle.generated.ts +12 -0
  75. package/ui/dist/index.html +32 -0
@@ -0,0 +1,639 @@
1
+ /**
2
+ * Container lifecycle for one attempt, wrapping the Docker CLI via
3
+ * `Bun.spawn` — no daemon SDK dependency, so any Docker-compatible daemon
4
+ * works (Colima, OrbStack, Docker Desktop, Linux).
5
+ *
6
+ * The isolation contract, made concrete:
7
+ * - fresh filesystem per attempt (a new container from the image);
8
+ * - the ONLY host mount is the per-attempt results directory at /results
9
+ * ("write-only" in the sense that the harness never reads host state
10
+ * through it — it only collects outputs);
11
+ * - the fixture is delivered via `docker cp`, never a mount;
12
+ * - secrets arrive as environment variables at container create (via a
13
+ * transient env-file so values never appear in host process listings);
14
+ * - every container carries a `quickstudy.attempt=<id>` label so teardown
15
+ * is verifiable: `docker ps -aq --filter label=quickstudy.attempt` must
16
+ * be empty after a sweep;
17
+ * - teardown is `docker rm -f` — kill and remove in one step.
18
+ */
19
+
20
+ import { mkdir, mkdtemp, rm, writeFile, cp, readdir, lstat, readlink, realpath } from "node:fs/promises";
21
+ import { tmpdir } from "node:os";
22
+ import { join, resolve } from "node:path";
23
+ import { sanitizeServerName, type McpUpstream } from "./mcp-proxy/server.ts";
24
+ import { EXCLUDED_WORKSPACE_DIRS } from "../store/artifacts.ts";
25
+
26
+ /** The label key every attempt container carries. */
27
+ export const ATTEMPT_LABEL = "quickstudy.attempt";
28
+ /**
29
+ * The label key carried by per-run resources: the internal egress network
30
+ * and the egress-proxy sidecar container. Same teardown-accounting pattern
31
+ * as ATTEMPT_LABEL — `quickstudy clean` sweeps by it.
32
+ */
33
+ export const RUN_LABEL = "quickstudy.run";
34
+ /** Where the fixture lands and the agent works, inside the container. */
35
+ export const CONTAINER_WORKSPACE = "/workspace";
36
+ /** The sole host mount, inside the container. */
37
+ export const CONTAINER_RESULTS = "/results";
38
+
39
+ /** A docker CLI invocation failed. */
40
+ export class DockerError extends Error {
41
+ readonly args: string[];
42
+ readonly exitCode: number;
43
+ readonly stderr: string;
44
+
45
+ constructor(args: string[], exitCode: number, stderr: string) {
46
+ super(`docker ${args.join(" ")} exited ${exitCode}: ${stderr.trim()}`);
47
+ this.name = "DockerError";
48
+ this.args = args;
49
+ this.exitCode = exitCode;
50
+ this.stderr = stderr;
51
+ }
52
+ }
53
+
54
+ export interface ExecResult {
55
+ exitCode: number;
56
+ stdout: string;
57
+ stderr: string;
58
+ /** True when the command was killed by its wall-clock timeout. */
59
+ timedOut: boolean;
60
+ }
61
+
62
+ /** Run `docker <args>`, capturing output. Never throws for non-zero exits. */
63
+ async function runDocker(
64
+ args: string[],
65
+ opts: { timeoutMs?: number; stdin?: string } = {},
66
+ ): Promise<ExecResult> {
67
+ const proc = Bun.spawn(["docker", ...args], {
68
+ stdin: opts.stdin === undefined ? "ignore" : Buffer.from(opts.stdin),
69
+ stdout: "pipe",
70
+ stderr: "pipe",
71
+ });
72
+ let timedOut = false;
73
+ let timer: ReturnType<typeof setTimeout> | undefined;
74
+ {
75
+ timer = setTimeout(() => {
76
+ timedOut = true;
77
+ proc.kill("SIGKILL");
78
+ }, opts.timeoutMs ?? 30_000);
79
+ }
80
+ const [stdout, stderr, exitCode] = await Promise.all([
81
+ new Response(proc.stdout).text(),
82
+ new Response(proc.stderr).text(),
83
+ proc.exited,
84
+ ]);
85
+ if (timer !== undefined) clearTimeout(timer);
86
+ return { exitCode, stdout, stderr, timedOut };
87
+ }
88
+
89
+ /** Run `docker <args>` and throw a DockerError on non-zero exit. */
90
+ async function mustDocker(args: string[], opts: { timeoutMs?: number; stdin?: string } = {}): Promise<ExecResult> {
91
+ const result = await runDocker(args, opts);
92
+ if (result.exitCode !== 0) throw new DockerError(args, result.exitCode, result.stderr);
93
+ return result;
94
+ }
95
+
96
+ /** Preflight: is a Docker daemon reachable at all? */
97
+ export async function dockerDaemonReachable(): Promise<boolean> {
98
+ const result = await runDocker(["version", "--format", "{{.Server.Version}}"], { timeoutMs: 10_000 });
99
+ return result.exitCode === 0;
100
+ }
101
+
102
+ /** Preflight: does the image exist locally? */
103
+ export async function imageExists(image: string): Promise<boolean> {
104
+ const result = await runDocker(["image", "inspect", image]);
105
+ return result.exitCode === 0;
106
+ }
107
+
108
+ export interface DockerImageIdentity {
109
+ tag: string;
110
+ id: string;
111
+ repoDigests: string[];
112
+ }
113
+
114
+ /** Resolve a mutable image tag to the content identity Docker will execute. */
115
+ export async function inspectImageIdentity(image: string): Promise<DockerImageIdentity> {
116
+ const result = await mustDocker(["image", "inspect", image, "--format", "{{json .}}"]);
117
+ const parsed = JSON.parse(result.stdout) as { Id?: string; RepoDigests?: string[] };
118
+ if (!parsed.Id) throw new Error(`docker image inspect returned no Id for "${image}"`);
119
+ return { tag: image, id: parsed.Id, repoDigests: [...(parsed.RepoDigests ?? [])].sort() };
120
+ }
121
+
122
+ /** Run a harmless version probe in an image during preflight. */
123
+ export async function inspectImageCommand(image: string, command: readonly string[]): Promise<string> {
124
+ if (command.length === 0) throw new Error("image version command must not be empty");
125
+ const [entrypoint, ...args] = command;
126
+ const result = await mustDocker(["run", "--rm", "--entrypoint", entrypoint as string, image, ...args]);
127
+ const version = `${result.stdout}\n${result.stderr}`.trim();
128
+ if (version === "") throw new Error(`version command ${command.join(" ")} returned no output in image "${image}"`);
129
+ return version;
130
+ }
131
+
132
+ /** All containers (running or exited) carrying the attempt label. Full ids. */
133
+ export async function listAttemptContainers(): Promise<string[]> {
134
+ const result = await mustDocker(["ps", "-aq", "--no-trunc", "--filter", `label=${ATTEMPT_LABEL}`]);
135
+ return result.stdout.split("\n").filter((line) => line !== "");
136
+ }
137
+
138
+ /**
139
+ * Force-remove every `quickstudy.attempt`-labeled container — the
140
+ * `quickstudy clean` sweep, also run at sweep start so an earlier hard-killed
141
+ * harness never leaks containers into this run's teardown check.
142
+ */
143
+ export async function removeAttemptContainers(): Promise<number> {
144
+ const ids = await listAttemptContainers();
145
+ if (ids.length === 0) return 0;
146
+ await mustDocker(["rm", "-f", ...ids]);
147
+ return ids.length;
148
+ }
149
+
150
+ export interface CreateAttemptContainerOptions {
151
+ image: string;
152
+ /** Provider keys + per-attempt runtime credentials. Values never hit argv. */
153
+ env: Record<string, string>;
154
+ /** Host directory mounted at /results — the sole mount. */
155
+ resultsDir: string;
156
+ /** Recorded as the `quickstudy.attempt` label value. */
157
+ attemptId: string;
158
+ /** Host fixture directory, delivered via `docker cp` (never a mount). */
159
+ fixtureDir: string;
160
+ /**
161
+ * Docker network to attach INSTEAD of the default bridge. The egress-proxy
162
+ * run mode passes the run's internal network here: no default route out —
163
+ * the proxy sidecar is the container's only path to the world.
164
+ */
165
+ network?: string;
166
+ }
167
+
168
+ export interface AttemptContainer {
169
+ id: string;
170
+ /** Run a command in /workspace inside the container. Never throws for non-zero exits. */
171
+ execInWorkspace(cmd: string[], opts?: { timeoutMs?: number }): Promise<ExecResult>;
172
+ /** Write a file inside the container (parent directories created). */
173
+ writeFile(path: string, content: string): Promise<void>;
174
+ /** Copy /workspace to a host directory, excluding node_modules/vendor/.venv. */
175
+ exportWorkspace(toDir: string): Promise<void>;
176
+ /** `docker rm -f` — kill and remove. Idempotent. */
177
+ teardown(): Promise<void>;
178
+ }
179
+
180
+ /**
181
+ * Create and start one attempt container: labeled, env-injected, with the
182
+ * per-attempt results directory as its only mount, then `docker cp` the
183
+ * fixture into /workspace. On any failure the partial container is removed
184
+ * before the error propagates — a failed create never leaks a container.
185
+ */
186
+ export async function createAttemptContainer(opts: CreateAttemptContainerOptions): Promise<AttemptContainer> {
187
+ // Secrets travel via a transient env-file, not argv (argv is visible in
188
+ // host process listings while `docker run` executes).
189
+ const envDir = await mkdtemp(join(tmpdir(), "quickstudy-env-"));
190
+ const envFile = join(envDir, "attempt.env");
191
+ const lines = Object.entries(opts.env).map(([key, value]) => {
192
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) throw new Error(`invalid environment variable name "${key}"`);
193
+ if (value.includes("\n")) throw new Error(`environment variable "${key}" contains a newline`);
194
+ return `${key}=${value}`;
195
+ });
196
+ await writeFile(envFile, lines.join("\n") + (lines.length > 0 ? "\n" : ""), { mode: 0o600 });
197
+
198
+ let id: string;
199
+ try {
200
+ const created = await mustDocker([
201
+ "run",
202
+ "-d",
203
+ "--label",
204
+ `${ATTEMPT_LABEL}=${opts.attemptId}`,
205
+ "--env-file",
206
+ envFile,
207
+ "-v",
208
+ `${resolve(opts.resultsDir)}:${CONTAINER_RESULTS}`,
209
+ "-w",
210
+ CONTAINER_WORKSPACE,
211
+ ...(opts.network !== undefined ? ["--network", opts.network] : []),
212
+ opts.image,
213
+ "sleep",
214
+ "infinity",
215
+ ]);
216
+ id = created.stdout.trim();
217
+ } catch (err) {
218
+ // `docker run` can fail AFTER creating the container (start failure) —
219
+ // sweep anything already labeled for this attempt so a failed create
220
+ // never orphans a container.
221
+ const leftover = await runDocker(["ps", "-aq", "--no-trunc", "--filter", `label=${ATTEMPT_LABEL}=${opts.attemptId}`]);
222
+ const ids = leftover.stdout.split("\n").filter((line) => line !== "");
223
+ if (ids.length > 0) await runDocker(["rm", "-f", ...ids]);
224
+ throw err;
225
+ } finally {
226
+ await rm(envDir, { recursive: true, force: true });
227
+ }
228
+
229
+ const container: AttemptContainer = {
230
+ id,
231
+ execInWorkspace: async (cmd, execOpts = {}) => {
232
+ const result = await runDocker(["exec", "-w", CONTAINER_WORKSPACE, id, ...cmd], execOpts);
233
+ // Killing only the Docker client leaves the exec process alive in the
234
+ // daemon. Stop the whole attempt on timeout, including descendants.
235
+ if (result.timedOut) await runDocker(["kill", id], { timeoutMs: 10_000 });
236
+ return result;
237
+ },
238
+ writeFile: async (path, content) => {
239
+ // `$1` positional keeps the target path out of shell interpolation.
240
+ await mustDocker(["exec", "-i", id, "sh", "-c", 'mkdir -p "$(dirname "$1")" && cat > "$1"', "sh", path], {
241
+ stdin: content,
242
+ });
243
+ },
244
+ exportWorkspace: async (toDir) => {
245
+ // Filter in the container, then stream tar directly to an empty host
246
+ // scratch directory. Dependency contents never cross the Docker boundary.
247
+ const scratch = await mkdtemp(join(tmpdir(), "quickstudy-export-"));
248
+ let producer: ReturnType<typeof Bun.spawn> | undefined;
249
+ let consumer: ReturnType<typeof Bun.spawn> | undefined;
250
+ let timer: ReturnType<typeof setTimeout> | undefined;
251
+ try {
252
+ producer = Bun.spawn(["docker", "exec", id, "tar", ...EXCLUDED_WORKSPACE_DIRS.map((name) => `--exclude=${name}`),
253
+ "--exclude=.env", "--exclude=.env.*", "-C", CONTAINER_WORKSPACE, "-cf", "-", "."], { stdout: "pipe", stderr: "pipe" });
254
+ consumer = Bun.spawn(["tar", "-xf", "-", "-C", scratch], { stdin: producer.stdout, stdout: "ignore", stderr: "pipe" });
255
+ timer = setTimeout(() => { producer?.kill("SIGKILL"); consumer?.kill("SIGKILL"); }, 60_000);
256
+ const [produced, consumed, producerError, consumerError] = await Promise.all([
257
+ producer.exited, consumer.exited, new Response(producer.stderr as ReadableStream).text(), new Response(consumer.stderr as ReadableStream).text(),
258
+ ]);
259
+ if (produced !== 0 || consumed !== 0) throw new Error(`workspace transfer failed: ${producerError} ${consumerError}`);
260
+ const canonicalScratch = await realpath(scratch);
261
+ const cleanLinks = async (dir: string): Promise<void> => {
262
+ for (const name of await readdir(dir)) {
263
+ const path = join(dir, name); const stat = await lstat(path);
264
+ if (stat.isSymbolicLink()) {
265
+ const target = await readlink(path);
266
+ const destination = await realpath(path).catch(() => resolve(dir, target).replace(scratch, canonicalScratch));
267
+ if (target.startsWith("/") || !destination.startsWith(`${canonicalScratch}/`)) await rm(path);
268
+ } else if (stat.isDirectory()) await cleanLinks(path);
269
+ }
270
+ };
271
+ await cleanLinks(scratch);
272
+ await mkdir(toDir, { recursive: true });
273
+ await cp(scratch, toDir, { recursive: true, verbatimSymlinks: true });
274
+ } finally {
275
+ if (timer) clearTimeout(timer);
276
+ producer?.kill(); consumer?.kill();
277
+ await rm(scratch, { recursive: true, force: true });
278
+ }
279
+ },
280
+ teardown: async () => {
281
+ const result = await runDocker(["rm", "-f", id]);
282
+ // Idempotent: an already-removed container is success, not an error.
283
+ if (result.exitCode !== 0 && !/no such container/i.test(result.stderr)) {
284
+ throw new DockerError(["rm", "-f", id], result.exitCode, result.stderr);
285
+ }
286
+ },
287
+ };
288
+
289
+ try {
290
+ // Fixture delivery: docker cp, never a mount — the agent can never write
291
+ // through to host fixture sources, and the mount count stays at one.
292
+ await mustDocker(["cp", `${opts.fixtureDir}/.`, `${id}:${CONTAINER_WORKSPACE}`]);
293
+ } catch (err) {
294
+ await container.teardown();
295
+ throw err;
296
+ }
297
+
298
+ return container;
299
+ }
300
+
301
+ /** `docker inspect` a container's mounts — the isolation criterion's probe. */
302
+ export async function inspectMounts(id: string): Promise<Array<{ destination: string; source: string }>> {
303
+ const result = await mustDocker(["inspect", "--format", "{{json .Mounts}}", id]);
304
+ const mounts = JSON.parse(result.stdout.trim()) as Array<{ Destination: string; Source: string }>;
305
+ return mounts.map((m) => ({ destination: m.Destination, source: m.Source }));
306
+ }
307
+
308
+ /** `docker inspect` a container's label value for the attempt label. */
309
+ export async function inspectAttemptLabel(id: string): Promise<string> {
310
+ const result = await mustDocker(["inspect", "--format", `{{index .Config.Labels "${ATTEMPT_LABEL}"}}`, id]);
311
+ return result.stdout.trim();
312
+ }
313
+
314
+ /** A container's IP address on one specific network. */
315
+ export async function inspectContainerIp(id: string, network: string): Promise<string> {
316
+ const result = await mustDocker([
317
+ "inspect",
318
+ "--format",
319
+ `{{(index .NetworkSettings.Networks "${network}").IPAddress}}`,
320
+ id,
321
+ ]);
322
+ const ip = result.stdout.trim();
323
+ if (ip === "" || ip === "<no value>") {
324
+ throw new DockerError(["inspect", id], 0, `container ${id.slice(0, 12)} has no address on network "${network}"`);
325
+ }
326
+ return ip;
327
+ }
328
+
329
+ // ---------------------------------------------------------------------------
330
+ // Egress-proxy run mode: per-run internal network + proxy sidecar.
331
+ // ---------------------------------------------------------------------------
332
+
333
+ /** The per-run internal network's name. */
334
+ export function runNetworkName(runId: string): string {
335
+ return `quickstudy-run-${runId.toLowerCase()}`;
336
+ }
337
+
338
+ /**
339
+ * Create the run's internal network: `--internal` means NO default route out
340
+ * — containers on it can reach each other (the proxy sidecar) and nothing
341
+ * else. Labeled for the same verifiable-teardown accounting as containers.
342
+ */
343
+ export async function createRunNetwork(runId: string): Promise<string> {
344
+ const name = runNetworkName(runId);
345
+ await mustDocker(["network", "create", "--internal", "--label", `${RUN_LABEL}=${runId}`, name]);
346
+ return name;
347
+ }
348
+
349
+ /** Remove one run network. Idempotent (an already-removed network is success). */
350
+ export async function removeRunNetwork(name: string): Promise<void> {
351
+ const result = await runDocker(["network", "rm", name]);
352
+ if (result.exitCode !== 0 && !/no such network|not found/i.test(result.stderr)) {
353
+ throw new DockerError(["network", "rm", name], result.exitCode, result.stderr);
354
+ }
355
+ }
356
+
357
+ /** All `quickstudy.run`-labeled networks (leak accounting / clean). */
358
+ export async function listRunNetworks(): Promise<string[]> {
359
+ const result = await mustDocker(["network", "ls", "-q", "--filter", `label=${RUN_LABEL}`, "--format", "{{.Name}}"]);
360
+ return result.stdout.split("\n").filter((line) => line !== "");
361
+ }
362
+
363
+ /** All `quickstudy.run`-labeled containers (egress-proxy sidecars). Full ids. */
364
+ export async function listRunContainers(): Promise<string[]> {
365
+ const result = await mustDocker(["ps", "-aq", "--no-trunc", "--filter", `label=${RUN_LABEL}`]);
366
+ return result.stdout.split("\n").filter((line) => line !== "");
367
+ }
368
+
369
+ /**
370
+ * The `quickstudy clean` extension: remove leaked sidecar containers first
371
+ * (a network with attached containers cannot be removed), then the networks.
372
+ * Returns counts for the operator message.
373
+ */
374
+ export async function removeRunResources(): Promise<{ containers: number; networks: number }> {
375
+ const containers = await listRunContainers();
376
+ if (containers.length > 0) await mustDocker(["rm", "-f", ...containers]);
377
+ const networks = await listRunNetworks();
378
+ for (const network of networks) await removeRunNetwork(network);
379
+ return { containers: containers.length, networks: networks.length };
380
+ }
381
+
382
+ /** How long the sidecar may take to report `{"type":"listening"}`. */
383
+ const SIDECAR_START_TIMEOUT_MS = 30_000;
384
+
385
+ export interface StartEgressSidecarOptions {
386
+ runId: string;
387
+ /** The run's internal network (createRunNetwork). */
388
+ networkName: string;
389
+ /** Composed, validated allowlist (isolation/proxy/allowlist.ts). */
390
+ allowlist: readonly string[];
391
+ /** Proxy image tag. The caller passes EGRESS_PROXY_IMAGE. */
392
+ image: string;
393
+ /** Listen port inside the sidecar. Default 3128. */
394
+ port?: number;
395
+ }
396
+
397
+ export interface EgressSidecar {
398
+ id: string;
399
+ networkName: string;
400
+ /** The sidecar's address on the internal network. Updated on restart. */
401
+ ip: string;
402
+ port: number;
403
+ /**
404
+ * Proxy env for attempt containers. Both cases: vendor CLIs disagree on
405
+ * which spelling they read. NO_PROXY keeps loopback traffic (dev servers,
406
+ * graders' probes) away from the proxy.
407
+ */
408
+ proxyEnv(): Record<string, string>;
409
+ /** Health probe: is the sidecar process still running? */
410
+ isRunning(): Promise<boolean>;
411
+ /** Restart the sidecar in place and re-resolve its internal-network IP. */
412
+ restart(): Promise<void>;
413
+ /** Full decision log (JSON lines) since sidecar start. */
414
+ logs(): Promise<string>;
415
+ /** `docker rm -f`. Idempotent. */
416
+ teardown(): Promise<void>;
417
+ }
418
+
419
+ function countListeningLines(logs: string): number {
420
+ return logs.split("\n").filter((line) => line.includes('"type":"listening"')).length;
421
+ }
422
+
423
+ /**
424
+ * Poll `docker logs` until the proxy's listening line appears at least
425
+ * `minCount` times — `docker logs` persists across restarts, so a restart
426
+ * waits for one MORE listening line than it saw before.
427
+ */
428
+ async function awaitSidecarListening(id: string, minCount = 1, label = "egress-proxy"): Promise<void> {
429
+ const deadline = Date.now() + SIDECAR_START_TIMEOUT_MS;
430
+ for (;;) {
431
+ const logs = await runDocker(["logs", id]);
432
+ if (countListeningLines(logs.stdout) >= minCount) return;
433
+ const state = await runDocker(["inspect", "--format", "{{.State.Running}}", id]);
434
+ if (state.exitCode === 0 && state.stdout.trim() !== "true") {
435
+ throw new DockerError(
436
+ ["logs", id],
437
+ 1,
438
+ `${label} sidecar exited during startup:\n${logs.stdout}${logs.stderr}`,
439
+ );
440
+ }
441
+ if (Date.now() > deadline) {
442
+ throw new DockerError(["logs", id], 1, `${label} sidecar never reported listening:\n${logs.stdout}${logs.stderr}`);
443
+ }
444
+ await new Promise((resolve) => setTimeout(resolve, 250));
445
+ }
446
+ }
447
+
448
+ /**
449
+ * Start the run's egress-proxy sidecar: created ON the internal network (its
450
+ * face toward the attempt containers — starting on a user-defined network
451
+ * also wires Docker's embedded DNS into the container, which forwards to
452
+ * upstream resolvers for public names), then connected to the default
453
+ * bridge, whose gateway becomes its only route OUT (the internal network
454
+ * has none). On any start failure the partial container is removed — a
455
+ * failed start never leaks a sidecar.
456
+ */
457
+ export async function startEgressSidecar(opts: StartEgressSidecarOptions): Promise<EgressSidecar> {
458
+ const port = opts.port ?? 3128;
459
+ const created = await mustDocker([
460
+ "run",
461
+ "-d",
462
+ "--label",
463
+ `${RUN_LABEL}=${opts.runId}`,
464
+ "--network",
465
+ opts.networkName,
466
+ "-e",
467
+ `QUICKSTUDY_EGRESS_ALLOWLIST=${opts.allowlist.join(",")}`,
468
+ "-e",
469
+ `QUICKSTUDY_EGRESS_PORT=${port}`,
470
+ opts.image,
471
+ ]);
472
+ const id = created.stdout.trim();
473
+
474
+ try {
475
+ await mustDocker(["network", "connect", "bridge", id]);
476
+ await awaitSidecarListening(id);
477
+ } catch (err) {
478
+ await runDocker(["rm", "-f", id]);
479
+ throw err;
480
+ }
481
+
482
+ const sidecar: EgressSidecar = {
483
+ id,
484
+ networkName: opts.networkName,
485
+ ip: await inspectContainerIp(id, opts.networkName),
486
+ port,
487
+ proxyEnv() {
488
+ const url = `http://${this.ip}:${this.port}`;
489
+ const noProxy = "localhost,127.0.0.1";
490
+ return {
491
+ HTTP_PROXY: url,
492
+ HTTPS_PROXY: url,
493
+ http_proxy: url,
494
+ https_proxy: url,
495
+ NO_PROXY: noProxy,
496
+ no_proxy: noProxy,
497
+ };
498
+ },
499
+ async isRunning() {
500
+ const state = await runDocker(["inspect", "--format", "{{.State.Running}}", id]);
501
+ return state.exitCode === 0 && state.stdout.trim() === "true";
502
+ },
503
+ async restart() {
504
+ const seenBefore = countListeningLines(await this.logs());
505
+ await mustDocker(["restart", id]);
506
+ await awaitSidecarListening(id, seenBefore + 1);
507
+ this.ip = await inspectContainerIp(id, opts.networkName);
508
+ },
509
+ async logs() {
510
+ const result = await mustDocker(["logs", id]);
511
+ return result.stdout;
512
+ },
513
+ async teardown() {
514
+ const result = await runDocker(["rm", "-f", id]);
515
+ if (result.exitCode !== 0 && !/no such container/i.test(result.stderr)) {
516
+ throw new DockerError(["rm", "-f", id], result.exitCode, result.stderr);
517
+ }
518
+ },
519
+ };
520
+ return sidecar;
521
+ }
522
+
523
+ export interface StartMcpSidecarOptions {
524
+ runId: string;
525
+ /**
526
+ * The run's internal network (egress mode). Omitted = default bridge —
527
+ * attempt containers reach the sidecar by IP either way.
528
+ */
529
+ networkName?: string;
530
+ /** Proxy image tag. The caller passes MCP_PROXY_IMAGE. */
531
+ image: string;
532
+ /** Upstream config (no secrets — safe for `docker inspect`). */
533
+ upstreams: Record<string, McpUpstream>;
534
+ /** Initial refresh tokens by server name. Env-file only, never argv. */
535
+ refreshTokens: Record<string, string>;
536
+ /** Listen port inside the sidecar. Default 8914. */
537
+ port?: number;
538
+ }
539
+
540
+ export interface McpSidecar {
541
+ id: string;
542
+ /** The sidecar's address as attempt containers see it. */
543
+ ip: string;
544
+ port: number;
545
+ /**
546
+ * Current (possibly rotated) refresh token per server, read from the
547
+ * sidecar's token-state files. Called at teardown — rotation makes the
548
+ * on-disk token stale the moment the sidecar refreshes once.
549
+ */
550
+ readRotatedRefreshTokens(): Promise<Record<string, string | undefined>>;
551
+ isRunning(): Promise<boolean>;
552
+ teardown(): Promise<void>;
553
+ }
554
+
555
+ /**
556
+ * Start the run's MCP token-injecting sidecar. Same topology as the egress
557
+ * sidecar under `--egress-proxy` (internal network + bridge for its own way
558
+ * out); on plain runs it sits on the default bridge. Secrets travel via a
559
+ * transient 0600 env-file, mirroring createAttemptContainer.
560
+ */
561
+ export async function startMcpSidecar(opts: StartMcpSidecarOptions): Promise<McpSidecar> {
562
+ const port = opts.port ?? 8914;
563
+ const envDir = await mkdtemp(join(tmpdir(), "quickstudy-mcp-env-"));
564
+ const envFile = join(envDir, "sidecar.env");
565
+ const env: Record<string, string> = {
566
+ QUICKSTUDY_MCP_UPSTREAMS: JSON.stringify(opts.upstreams),
567
+ QUICKSTUDY_MCP_PORT: String(port),
568
+ };
569
+ for (const [name, token] of Object.entries(opts.refreshTokens)) {
570
+ if (token.includes("\n")) throw new Error(`mcp refresh token for "${name}" contains a newline`);
571
+ env[`QUICKSTUDY_MCP_REFRESH_TOKEN_${sanitizeServerName(name)}`] = token;
572
+ }
573
+ await writeFile(
574
+ envFile,
575
+ Object.entries(env)
576
+ .map(([key, value]) => `${key}=${value}`)
577
+ .join("\n") + "\n",
578
+ { mode: 0o600 },
579
+ );
580
+
581
+ let id: string;
582
+ try {
583
+ const created = await mustDocker([
584
+ "run",
585
+ "-d",
586
+ "--label",
587
+ `${RUN_LABEL}=${opts.runId}`,
588
+ "--env-file",
589
+ envFile,
590
+ ...(opts.networkName !== undefined ? ["--network", opts.networkName] : []),
591
+ opts.image,
592
+ ]);
593
+ id = created.stdout.trim();
594
+ } finally {
595
+ await rm(envDir, { recursive: true, force: true });
596
+ }
597
+
598
+ try {
599
+ if (opts.networkName !== undefined) await mustDocker(["network", "connect", "bridge", id]);
600
+ await awaitSidecarListening(id, 1, "mcp-proxy");
601
+ } catch (err) {
602
+ await runDocker(["rm", "-f", id]);
603
+ throw err;
604
+ }
605
+
606
+ return {
607
+ id,
608
+ ip: await inspectContainerIp(id, opts.networkName ?? "bridge"),
609
+ port,
610
+ async readRotatedRefreshTokens() {
611
+ const tokens: Record<string, string | undefined> = {};
612
+ for (const name of Object.keys(opts.upstreams)) {
613
+ const result = await runDocker(["exec", id, "cat", `/run/quickstudy-mcp/${sanitizeServerName(name)}.refresh-token`]);
614
+ const value = result.stdout.trim();
615
+ tokens[name] = result.exitCode === 0 && value !== "" ? value : undefined;
616
+ }
617
+ return tokens;
618
+ },
619
+ async isRunning() {
620
+ const state = await runDocker(["inspect", "--format", "{{.State.Running}}", id]);
621
+ return state.exitCode === 0 && state.stdout.trim() === "true";
622
+ },
623
+ async teardown() {
624
+ const result = await runDocker(["rm", "-f", id]);
625
+ if (result.exitCode !== 0 && !/no such container/i.test(result.stderr)) {
626
+ throw new DockerError(["rm", "-f", id], result.exitCode, result.stderr);
627
+ }
628
+ },
629
+ };
630
+ }
631
+
632
+ /** Attach shared infrastructure to an attempt network, returning its local address. */
633
+ export async function connectContainerNetwork(id: string, network: string): Promise<string> {
634
+ await mustDocker(["network", "connect", network, id]);
635
+ return inspectContainerIp(id, network);
636
+ }
637
+ export async function disconnectContainerNetwork(id: string, network: string): Promise<void> {
638
+ await mustDocker(["network", "disconnect", network, id]);
639
+ }