@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/CONTRIBUTING.md +170 -0
  2. package/LICENSE +63 -136
  3. package/README.md +83 -66
  4. package/docs/README.md +44 -0
  5. package/docs/acknowledgements.md +14 -0
  6. package/docs/architecture.md +146 -0
  7. package/docs/assets/README.md +27 -0
  8. package/docs/assets/sevro-logo.png +0 -0
  9. package/docs/assets/sevro-symbol.svg +1 -0
  10. package/docs/assets/sevro-wordmark.svg +1 -0
  11. package/docs/builtin-graders-v1.md +41 -13
  12. package/docs/codex-host.md +184 -52
  13. package/docs/contracts.md +19 -0
  14. package/docs/creating-cases.md +139 -0
  15. package/docs/development-cli.md +190 -30
  16. package/docs/documentation-quality.md +61 -0
  17. package/docs/evaluation-workflow.md +110 -0
  18. package/docs/evidence/code-mode-controls-71.md +64 -0
  19. package/docs/evidence/code-mode-programs-85.md +64 -0
  20. package/docs/evidence/codex-state-73/README.md +204 -0
  21. package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
  22. package/docs/evidence/codex-state-73/results.json +661 -0
  23. package/docs/evidence/local-http-74.md +104 -0
  24. package/docs/evidence/runtime-startup-30/README.md +185 -0
  25. package/docs/evidence/runtime-startup-30/after.json +319 -0
  26. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
  27. package/docs/evidence/runtime-startup-30/before.json +403 -0
  28. package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
  29. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
  30. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
  31. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
  32. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
  33. package/docs/evidence/skill-reads-70/results.json +823 -0
  34. package/docs/evidence.md +21 -0
  35. package/docs/extension-protocol-v1.md +50 -16
  36. package/docs/fixture-cleanup-validation.md +3 -0
  37. package/docs/getting-started.md +102 -0
  38. package/docs/guide-evaluation.md +89 -0
  39. package/docs/guide-verification.json +239 -0
  40. package/docs/identity-v1.md +35 -1
  41. package/docs/installing.md +88 -0
  42. package/docs/licenses/BUSL-1.1-0.1.0-rc.2.txt +159 -0
  43. package/docs/licensing.md +42 -0
  44. package/docs/native-controls.md +76 -5
  45. package/docs/native-hosts.md +228 -0
  46. package/docs/reading-results.md +141 -0
  47. package/docs/recipes.md +489 -0
  48. package/docs/releases.md +81 -40
  49. package/docs/report-v1.md +29 -1
  50. package/docs/results-v1.md +180 -0
  51. package/docs/run-settings-v1.md +147 -0
  52. package/docs/running-evaluations.md +31 -0
  53. package/docs/runtime-v1.md +614 -0
  54. package/docs/specs/repository-guide.md +58 -0
  55. package/docs/troubleshooting.md +30 -0
  56. package/docs/typescript-quality.md +234 -0
  57. package/docs/usability-study.md +114 -0
  58. package/examples/basic/host.ts +2 -2
  59. package/examples/recipes/candidate.ts +26 -0
  60. package/examples/recipes/cases/coding-change.json +21 -0
  61. package/examples/recipes/cases/host-tools.json +21 -0
  62. package/examples/recipes/cases/mise.json +21 -0
  63. package/examples/recipes/cases/node.json +21 -0
  64. package/examples/recipes/cases/python-uv.json +21 -0
  65. package/examples/recipes/cases/semantic.json +17 -0
  66. package/examples/recipes/coding-change/check.ts +23 -0
  67. package/examples/recipes/extension-artifacts.ts +65 -0
  68. package/examples/recipes/extension-methods.ts +174 -0
  69. package/examples/recipes/extension.ts +37 -0
  70. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  71. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  72. package/examples/recipes/repository-skill/README.md +3 -0
  73. package/examples/recipes/run-inputs.ts +188 -0
  74. package/examples/recipes/run-options.ts +131 -0
  75. package/examples/recipes/run.ts +63 -0
  76. package/examples/recipes/runtime/empty.json +1 -0
  77. package/examples/recipes/runtime/goals.json +1 -0
  78. package/examples/recipes/runtime/mise.json +6 -0
  79. package/examples/recipes/runtime/path.json +1 -0
  80. package/examples/recipes/runtime/plugins.json +4 -0
  81. package/examples/recipes/runtime/python-uv.json +10 -0
  82. package/examples/recipes/semantic-judge.ts +29 -0
  83. package/examples/templates/coding/adapter.ts +17 -0
  84. package/examples/templates/coding/case.json +15 -0
  85. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  86. package/examples/templates/repository-skill/extension-data.ts +64 -0
  87. package/examples/templates/repository-skill/extension.ts +31 -0
  88. package/examples/templates/response/adapter.ts +12 -0
  89. package/examples/templates/response/case.json +13 -0
  90. package/package.json +39 -6
  91. package/schemas/report-v1.schema.json +9 -0
  92. package/schemas/run-evidence-v1.schema.json +2 -1
  93. package/schemas/runtime-v1.schema.json +193 -0
  94. package/src/advisory.ts +10 -5
  95. package/src/candidate-artifacts.ts +208 -0
  96. package/src/candidate-codex-state.ts +185 -0
  97. package/src/case-validation.ts +105 -0
  98. package/src/cli/cancellation.ts +20 -0
  99. package/src/cli/command.ts +81 -0
  100. package/src/cli/completion.ts +15 -0
  101. package/src/cli/demo.ts +59 -0
  102. package/src/cli/evaluation.ts +55 -0
  103. package/src/cli/extension.ts +79 -0
  104. package/src/cli/help.ts +222 -0
  105. package/src/cli/hosts.ts +59 -0
  106. package/src/cli/init-settings.ts +30 -0
  107. package/src/cli/init.ts +111 -0
  108. package/src/cli/invocation.ts +627 -0
  109. package/src/cli/loading.ts +151 -0
  110. package/src/cli/preparation.ts +163 -0
  111. package/src/cli/presentation.ts +91 -0
  112. package/src/cli/progress.ts +6 -0
  113. package/src/cli/run-manifest.ts +133 -0
  114. package/src/cli/run-options.ts +101 -0
  115. package/src/cli/run-overrides.ts +83 -0
  116. package/src/cli/settings.ts +160 -0
  117. package/src/cli/summary.ts +43 -0
  118. package/src/cli.ts +13 -843
  119. package/src/contract-types.ts +94 -0
  120. package/src/engine.ts +19 -2373
  121. package/src/evaluation/advisory.ts +150 -0
  122. package/src/evaluation/check-plan.ts +510 -0
  123. package/src/evaluation/collection.ts +88 -0
  124. package/src/evaluation/context.ts +10 -0
  125. package/src/evaluation/grading-host.ts +39 -0
  126. package/src/evaluation/grading.ts +258 -0
  127. package/src/evaluation/invocation-plan.ts +548 -0
  128. package/src/evaluation/plan.ts +736 -0
  129. package/src/evaluation/progress.ts +19 -0
  130. package/src/evaluation/resources.ts +241 -0
  131. package/src/evaluation/run.ts +206 -0
  132. package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +3 -1
  133. package/src/evaluation/semantic.ts +312 -0
  134. package/src/evaluation/trial.ts +866 -0
  135. package/src/evaluation/types.ts +207 -0
  136. package/src/evidence/host-results.ts +92 -0
  137. package/src/evidence/retained-artifacts.ts +18 -0
  138. package/src/evidence/trial-writer.ts +203 -0
  139. package/src/evidence/usage.ts +17 -0
  140. package/src/extension-client.ts +228 -148
  141. package/src/extension-session.ts +326 -195
  142. package/src/fixtures/advisory.ts +338 -0
  143. package/src/fixtures/bin.ts +57 -0
  144. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  145. package/src/fixtures/create.ts +187 -0
  146. package/src/fixtures/generated.ts +300 -0
  147. package/src/fixtures/git-hooks.ts +24 -0
  148. package/src/fixtures/preparation.ts +270 -0
  149. package/src/{repository-fixture.ts → fixtures/repository.ts} +207 -85
  150. package/src/fixtures/setup.ts +224 -0
  151. package/src/fixtures/tools.ts +88 -0
  152. package/src/fixtures/workspace-state.ts +212 -0
  153. package/src/generated/cli-result.cjs +2 -0
  154. package/src/generated/cli-result.d.cts +3 -0
  155. package/src/generated/extension.cjs +2 -0
  156. package/src/generated/extension.d.cts +3 -0
  157. package/src/generated/report.cjs +2 -0
  158. package/src/generated/report.d.cts +3 -0
  159. package/src/generated/run-evidence.cjs +2 -0
  160. package/src/generated/run-evidence.d.cts +3 -0
  161. package/src/generated/runtime.cjs +2 -0
  162. package/src/generated/runtime.d.cts +3 -0
  163. package/src/graders/artifact.ts +56 -23
  164. package/src/graders/git-head.ts +79 -75
  165. package/src/graders/output.ts +222 -140
  166. package/src/graders/semantic.ts +102 -43
  167. package/src/graders/shell.ts +379 -207
  168. package/src/host-diagnostic-redaction.ts +52 -0
  169. package/src/host-diagnostics.ts +319 -0
  170. package/src/hosts/claude-continuation.ts +63 -18
  171. package/src/hosts/claude-credential.ts +43 -62
  172. package/src/hosts/claude-events.ts +86 -51
  173. package/src/hosts/claude-keychain.ts +59 -0
  174. package/src/hosts/claude-native-goal.ts +57 -33
  175. package/src/hosts/claude-process.ts +111 -0
  176. package/src/hosts/claude-repository-invocation.ts +208 -117
  177. package/src/hosts/claude-settings.ts +240 -21
  178. package/src/hosts/claude-skill-activations.ts +166 -0
  179. package/src/hosts/claude-state.ts +403 -0
  180. package/src/hosts/claude-tool-calls.ts +171 -106
  181. package/src/hosts/claude.ts +561 -451
  182. package/src/hosts/codex-app-server.ts +326 -178
  183. package/src/hosts/codex-configuration.ts +44 -18
  184. package/src/hosts/codex-events.ts +97 -60
  185. package/src/hosts/codex-exec-controls.ts +516 -0
  186. package/src/hosts/codex-invocation.ts +70 -0
  187. package/src/hosts/codex-marketplace.ts +283 -0
  188. package/src/hosts/codex-native-calls.ts +546 -500
  189. package/src/hosts/codex-process.ts +202 -0
  190. package/src/hosts/codex-profile.ts +257 -28
  191. package/src/hosts/codex-session-files.ts +106 -0
  192. package/src/hosts/codex-state.ts +492 -0
  193. package/src/hosts/codex.ts +783 -832
  194. package/src/hosts/isolation-common.ts +94 -0
  195. package/src/hosts/isolation-roots.ts +54 -27
  196. package/src/hosts/isolation.ts +10 -0
  197. package/src/hosts/linux-sandbox.ts +159 -0
  198. package/src/hosts/mac-sandbox.ts +187 -61
  199. package/src/hosts/native-controls.ts +152 -62
  200. package/src/hosts/process-lifecycle.ts +37 -0
  201. package/src/hosts/protected-worktrees.ts +51 -19
  202. package/src/hosts/skill-hook-catalog.ts +131 -0
  203. package/src/hosts/skill-hook-handler.ts +374 -0
  204. package/src/hosts/skill-hooks.ts +580 -0
  205. package/src/hosts/workspace-fingerprint.ts +55 -35
  206. package/src/identity.ts +102 -48
  207. package/src/instrumentation.ts +109 -48
  208. package/src/native-transcript-state.ts +131 -0
  209. package/src/native-transcript-view.ts +90 -0
  210. package/src/native-transcripts.ts +286 -0
  211. package/src/process-lifecycle.ts +150 -0
  212. package/src/provenance.ts +161 -81
  213. package/src/prune.ts +215 -0
  214. package/src/report.ts +172 -140
  215. package/src/resolved-case.ts +33 -0
  216. package/src/results-lock.ts +29 -0
  217. package/src/results.ts +133 -86
  218. package/src/run-owner.ts +154 -63
  219. package/src/runtime/config.ts +456 -0
  220. package/src/runtime/discovery-apple.ts +90 -0
  221. package/src/runtime/discovery-providers.ts +63 -0
  222. package/src/runtime/discovery.ts +70 -0
  223. package/src/runtime/hook-processes.ts +97 -0
  224. package/src/runtime/hooks.ts +476 -0
  225. package/src/runtime/paths.ts +313 -0
  226. package/src/runtime/query.ts +85 -0
  227. package/src/runtime/seeds.ts +134 -0
  228. package/src/runtime/shared-cache.ts +143 -0
  229. package/src/runtime/state.ts +143 -0
  230. package/src/schema-types.ts +198 -0
  231. package/src/schema.ts +15 -19
  232. package/src/transcript-retention.ts +146 -0
  233. package/src/value-guards.ts +20 -0
  234. package/src/advisory-fixture.ts +0 -282
  235. package/src/fixture-bin.ts +0 -87
  236. package/src/fixture-setup.ts +0 -163
  237. package/src/generated-fixture.ts +0 -220
  238. package/src/git-hooks.ts +0 -63
  239. package/src/hosts/claude-nested-skills.ts +0 -218
  240. package/src/hosts/codex-command-output.ts +0 -399
  241. package/src/hosts/codex-executor-command.ts +0 -195
  242. package/src/hosts/codex-skill-reads.ts +0 -391
  243. package/src/preparation.ts +0 -211
@@ -0,0 +1,94 @@
1
+ import { realpath } from "node:fs/promises";
2
+ import { isAbsolute, relative, sep } from "node:path";
3
+ import { insideRuntimeRoot } from "../runtime/paths";
4
+ import { workspaceBoundaryException } from "../fixtures/workspace-state";
5
+ import type { CandidateArtifactRoots } from "../candidate-artifacts";
6
+
7
+ export class HostIsolationError extends Error {
8
+ constructor(message: string) {
9
+ super(message);
10
+ this.name = "HostIsolationError";
11
+ }
12
+ }
13
+
14
+ export interface IsolationOptions {
15
+ argv: string[];
16
+ workspace: string;
17
+ protectedRoots: string[];
18
+ protectedRootsCanonical?: boolean;
19
+ privateStateRoot: string;
20
+ denyNetwork?: boolean;
21
+ localHttp?: boolean;
22
+ readOnlyRoots?: string[];
23
+ transcriptReadRoots?: string[];
24
+ candidateArtifactRoots?: CandidateArtifactRoots;
25
+ hookReadRoots?: string[];
26
+ writableRuntimeRoot?: string;
27
+ writableCacheRoots?: string[];
28
+ }
29
+
30
+ export interface IsolatedCommand {
31
+ argv: string[];
32
+ release(): Promise<void>;
33
+ }
34
+
35
+ export function inside(root: string, path: string): boolean {
36
+ const child = relative(root, path);
37
+ return (
38
+ child === "" ||
39
+ (child !== ".." && !child.startsWith(`..${sep}`) && !child.startsWith(sep))
40
+ );
41
+ }
42
+
43
+ export async function commandRuntimeRoot(
44
+ stateRoot: string,
45
+ path: string | undefined,
46
+ ): Promise<string | undefined> {
47
+ if (!path) return undefined;
48
+ const root = await realpath(path);
49
+ if (root === stateRoot || !insideRuntimeRoot(stateRoot, root))
50
+ throw new HostIsolationError(
51
+ "command runtime must be contained in private state",
52
+ );
53
+ return root;
54
+ }
55
+
56
+ export function requireIsolationPaths(options: IsolationOptions): void {
57
+ if (!options.argv.length || options.argv.some((part) => !part))
58
+ throw new HostIsolationError("isolated command must be nonempty");
59
+ if (!options.protectedRoots.length)
60
+ throw new HostIsolationError("isolated command needs protected roots");
61
+ if (!absoluteIsolationPaths(options))
62
+ throw new HostIsolationError("isolation paths must be absolute");
63
+ }
64
+
65
+ function absoluteIsolationPaths(options: IsolationOptions): boolean {
66
+ return (
67
+ isAbsolute(options.workspace) &&
68
+ isAbsolute(options.privateStateRoot) &&
69
+ (options.writableCacheRoots ?? []).every(isAbsolute) &&
70
+ options.protectedRoots.every(isAbsolute)
71
+ );
72
+ }
73
+
74
+ export function requireDisjointState(
75
+ workspace: string,
76
+ roots: string[],
77
+ stateRoot: string,
78
+ ): void {
79
+ if (
80
+ roots.some(
81
+ (root) =>
82
+ inside(root, workspace) &&
83
+ !workspaceBoundaryException(root, workspace).length,
84
+ ) ||
85
+ inside(stateRoot, workspace)
86
+ )
87
+ throw new HostIsolationError(
88
+ "protected root includes the candidate workspace",
89
+ );
90
+ if (inside(workspace, stateRoot))
91
+ throw new HostIsolationError(
92
+ "sandbox state is inside the candidate workspace",
93
+ );
94
+ }
@@ -1,21 +1,28 @@
1
- import { readdir, realpath } from "node:fs/promises";
2
- import { homedir, tmpdir, userInfo } from "node:os";
3
- import { isAbsolute, join, resolve } from "node:path";
1
+ import { isMissingFile } from "../fixtures/tools";
2
+ import { realpath } from "node:fs/promises";
3
+ import { homedir, userInfo } from "node:os";
4
+ import { isAbsolute, resolve } from "node:path";
4
5
  import { protectedWorktrees } from "./protected-worktrees";
6
+ import { nativeStateParent } from "../native-transcript-state";
7
+ import { workspaceStateParent } from "../fixtures/workspace-state";
5
8
 
6
9
  /** Include source, results, host configuration, and every active peer fixture. */
7
- export async function evaluationProtectedRoots(options: {
10
+ interface ProtectedRootsOptions {
8
11
  workspace: string;
9
12
  projectRoot: string;
10
13
  resultsRoot: string;
11
14
  additionalRoots: string[];
12
- }): Promise<string[]> {
13
- const workspace = await realpath(options.workspace);
15
+ ownedStateRoot?: string;
16
+ }
17
+
18
+ async function protectedRootCandidates(options: ProtectedRootsOptions) {
14
19
  const optionalConfigRoots = [
15
20
  process.env.CODEX_HOME,
16
21
  process.env.CLAUDE_CONFIG_DIR,
17
22
  ].filter((value): value is string => Boolean(value));
18
23
  const candidates = [
24
+ await nativeStateParent(),
25
+ await workspaceStateParent(),
19
26
  resolve(import.meta.dir, "../.."),
20
27
  options.projectRoot,
21
28
  options.resultsRoot,
@@ -26,29 +33,29 @@ export async function evaluationProtectedRoots(options: {
26
33
  ];
27
34
  if (candidates.some((root) => !isAbsolute(root)))
28
35
  throw new Error("evaluation protected roots must be absolute");
29
- const peers = (await readdir(tmpdir(), { withFileTypes: true }))
30
- .filter(
31
- (entry) => entry.isDirectory() && entry.name.startsWith("sevro-case-"),
32
- )
33
- .map((entry) => join(tmpdir(), entry.name));
36
+ return { candidates, optionalConfigRoots };
37
+ }
38
+
39
+ /** Include source, results, host configuration, and every active peer fixture. */
40
+ export async function evaluationProtectedRoots(
41
+ options: ProtectedRootsOptions,
42
+ ): Promise<string[]> {
43
+ const workspace = await realpath(options.workspace);
44
+ const { candidates, optionalConfigRoots } =
45
+ await protectedRootCandidates(options);
46
+ const namespace = await workspaceStateParent();
47
+ const nativeNamespace = await nativeStateParent();
34
48
  const roots: string[] = [];
35
- for (const candidate of [...candidates, ...peers]) {
36
- let canonical: string;
37
- try {
38
- canonical = await realpath(candidate);
39
- } catch (error) {
40
- if (
41
- (peers.includes(candidate) ||
42
- optionalConfigRoots.includes(candidate)) &&
43
- (error as NodeJS.ErrnoException).code === "ENOENT"
44
- )
45
- continue;
46
- throw new Error("evaluation protected root is unreadable");
47
- }
48
- if (canonical !== workspace) {
49
+ for (const candidate of candidates) {
50
+ const canonical = await canonicalProtectedRoot(candidate, [
51
+ ...optionalConfigRoots,
52
+ ]);
53
+ if (!includedProtectedRoot(canonical, workspace, options.ownedStateRoot))
54
+ continue;
55
+ {
49
56
  roots.push(canonical);
50
- // Peers are denied directly; their Git preparation may still be in progress.
51
- if (!peers.includes(candidate))
57
+ // Private namespaces are boundaries, not Git source trees to traverse.
58
+ if (canonical !== namespace && canonical !== nativeNamespace)
52
59
  roots.push(
53
60
  ...(await protectedWorktrees(canonical)).filter(
54
61
  (root) => root !== workspace,
@@ -58,3 +65,23 @@ export async function evaluationProtectedRoots(options: {
58
65
  }
59
66
  return [...new Set(roots)];
60
67
  }
68
+
69
+ function includedProtectedRoot(
70
+ root: string | null,
71
+ workspace: string,
72
+ state: string | undefined,
73
+ ): root is string {
74
+ return root !== null && root !== workspace && root !== state;
75
+ }
76
+
77
+ async function canonicalProtectedRoot(
78
+ candidate: string,
79
+ optionalRoots: string[],
80
+ ): Promise<string | null> {
81
+ try {
82
+ return await realpath(candidate);
83
+ } catch (cause) {
84
+ if (optionalRoots.includes(candidate) && isMissingFile(cause)) return null;
85
+ throw new Error("evaluation protected root is unreadable", { cause });
86
+ }
87
+ }
@@ -0,0 +1,10 @@
1
+ import { HostIsolationError, type IsolationOptions } from "./isolation-common";
2
+ import { prepareMacSandboxCommand } from "./mac-sandbox";
3
+ import { prepareLinuxSandboxCommand } from "./linux-sandbox";
4
+
5
+ /** Select the native filesystem boundary for evaluator-owned commands. */
6
+ export function prepareIsolatedCommand(options: IsolationOptions) {
7
+ if (process.platform === "darwin") return prepareMacSandboxCommand(options);
8
+ if (process.platform === "linux") return prepareLinuxSandboxCommand(options);
9
+ throw new HostIsolationError("host isolation is unavailable");
10
+ }
@@ -0,0 +1,159 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { existsSync } from "node:fs";
3
+ import { mkdir, realpath, rm, stat, writeFile } from "node:fs/promises";
4
+ import { resolve } from "node:path";
5
+ import {
6
+ requireNativeTranscriptViews,
7
+ requireNativeHookRoots,
8
+ } from "../native-transcript-state";
9
+ import { requireRuntimeReadRoots } from "../runtime/paths";
10
+ import { requireWritableCacheRoots } from "../runtime/shared-cache";
11
+ import { requireCandidateArtifactRoots } from "../candidate-artifacts";
12
+ import {
13
+ commandRuntimeRoot,
14
+ HostIsolationError,
15
+ requireDisjointState,
16
+ requireIsolationPaths,
17
+ type IsolatedCommand,
18
+ type IsolationOptions,
19
+ } from "./isolation-common";
20
+
21
+ const BWRAP = "/usr/bin/bwrap";
22
+
23
+ async function protectedKind(path: string): Promise<"file" | "directory"> {
24
+ try {
25
+ return (await stat(path)).isDirectory() ? "directory" : "file";
26
+ } catch (error) {
27
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return "directory";
28
+ throw error;
29
+ }
30
+ }
31
+
32
+ function requireLinuxSandbox(): void {
33
+ if (process.platform !== "linux" || !existsSync(BWRAP))
34
+ throw new HostIsolationError("Linux bubblewrap isolation is unavailable");
35
+ }
36
+
37
+ function networkArgs(denyNetwork: boolean): string[] {
38
+ return denyNetwork ? ["--unshare-net"] : [];
39
+ }
40
+
41
+ function mountProtectedRoot(
42
+ path: string,
43
+ kind: "file" | "directory" | undefined,
44
+ blockedFile: string,
45
+ ): string[] {
46
+ return kind === "directory"
47
+ ? ["--tmpfs", path, "--chmod", "0111", path]
48
+ : ["--ro-bind", blockedFile, path];
49
+ }
50
+
51
+ /** Mask protected inputs before launching an evaluator-owned command. */
52
+ export async function prepareLinuxSandboxCommand(
53
+ options: IsolationOptions,
54
+ ): Promise<IsolatedCommand> {
55
+ requireLinuxSandbox();
56
+ await requireNativeTranscriptViews(options.transcriptReadRoots);
57
+ await requireCandidateArtifactRoots(options.candidateArtifactRoots);
58
+ await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
59
+ requireIsolationPaths(options);
60
+ const workspace = await realpath(options.workspace);
61
+ const roots = options.protectedRootsCanonical
62
+ ? options.protectedRoots
63
+ : await Promise.all(options.protectedRoots.map((path) => realpath(path)));
64
+ await mkdir(options.privateStateRoot, { recursive: true, mode: 0o700 });
65
+ const stateRoot = await realpath(options.privateStateRoot);
66
+ const ownedRoot = await commandRuntimeRoot(
67
+ stateRoot,
68
+ options.writableRuntimeRoot,
69
+ );
70
+ const readRoots = options.readOnlyRoots ?? [];
71
+ const writableCacheRoots = await cacheMountRoots(options, [
72
+ ...roots,
73
+ stateRoot,
74
+ ]);
75
+ requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
76
+ requireDisjointState(workspace, roots, stateRoot);
77
+ const blockedFile = resolve(stateRoot, `blocked-${randomUUID()}`);
78
+ await writeFile(blockedFile, "", { flag: "wx", mode: 0o000 });
79
+ try {
80
+ const protectedRoots = [...new Set([...roots, stateRoot])].sort(
81
+ (left, right) => left.length - right.length,
82
+ );
83
+ const kinds = await Promise.all(protectedRoots.map(protectedKind));
84
+ const masks = protectedRoots.flatMap((path, index) =>
85
+ mountProtectedRoot(path, kinds[index], blockedFile),
86
+ );
87
+ return {
88
+ argv: [
89
+ BWRAP,
90
+ "--die-with-parent",
91
+ "--unshare-user",
92
+ "--unshare-pid",
93
+ ...networkArgs(Boolean(options.denyNetwork)),
94
+ "--ro-bind",
95
+ "/",
96
+ "/",
97
+ "--dev-bind",
98
+ "/dev",
99
+ "/dev",
100
+ "--proc",
101
+ "/proc",
102
+ "--bind",
103
+ "/tmp",
104
+ "/tmp",
105
+ ...masks,
106
+ "--bind",
107
+ workspace,
108
+ workspace,
109
+ ...readRoots.flatMap((path) => ["--ro-bind", path, path]),
110
+ ...transcriptMounts(options),
111
+ ...candidateArtifactMounts(options),
112
+ ...hookMounts(options),
113
+ ...(ownedRoot ? ["--bind", ownedRoot, ownedRoot] : []),
114
+ ...writableCacheRoots.flatMap((path) => ["--bind", path, path]),
115
+ "--chdir",
116
+ workspace,
117
+ "--",
118
+ ...options.argv,
119
+ ],
120
+ release: async () => {
121
+ await rm(blockedFile, { force: true });
122
+ },
123
+ };
124
+ } catch (error) {
125
+ await rm(blockedFile, { force: true });
126
+ throw error;
127
+ }
128
+ }
129
+
130
+ function cacheMountRoots(options: IsolationOptions, protectedRoots: string[]) {
131
+ return requireWritableCacheRoots(
132
+ options.writableCacheRoots ?? [],
133
+ protectedRoots,
134
+ );
135
+ }
136
+
137
+ function transcriptMounts(options: IsolationOptions): string[] {
138
+ return (options.transcriptReadRoots ?? []).flatMap((path) => [
139
+ "--ro-bind",
140
+ path,
141
+ path,
142
+ ]);
143
+ }
144
+
145
+ function candidateArtifactMounts(options: IsolationOptions): string[] {
146
+ return Object.values(options.candidateArtifactRoots ?? {}).flatMap((path) => [
147
+ "--ro-bind",
148
+ path,
149
+ path,
150
+ ]);
151
+ }
152
+
153
+ function hookMounts(options: IsolationOptions): string[] {
154
+ return (options.hookReadRoots ?? []).flatMap((path) => [
155
+ "--ro-bind",
156
+ path,
157
+ path,
158
+ ]);
159
+ }
@@ -1,103 +1,216 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { existsSync } from "node:fs";
3
3
  import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
4
- import { isAbsolute, relative, resolve, sep } from "node:path";
4
+ import { dirname, resolve } from "node:path";
5
+ import {
6
+ requireNativeTranscriptViews,
7
+ requireNativeHookRoots,
8
+ } from "../native-transcript-state";
9
+ import { requireCandidateArtifactRoots } from "../candidate-artifacts";
10
+ import { workspaceBoundaryException } from "../fixtures/workspace-state";
11
+ import { requireWritableCacheRoots } from "../runtime/shared-cache";
12
+ import {
13
+ insideRuntimeRoot,
14
+ isRuntimeHomeRoot,
15
+ requireRuntimeReadRoots,
16
+ } from "../runtime/paths";
17
+ import {
18
+ commandRuntimeRoot,
19
+ HostIsolationError,
20
+ requireDisjointState,
21
+ requireIsolationPaths,
22
+ type IsolatedCommand,
23
+ type IsolationOptions,
24
+ } from "./isolation-common";
5
25
 
6
- const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
7
-
8
- export class HostIsolationError extends Error {
9
- constructor(message: string) {
10
- super(message);
11
- this.name = "HostIsolationError";
12
- }
13
- }
26
+ export { HostIsolationError } from "./isolation-common";
27
+ export type { IsolatedCommand } from "./isolation-common";
14
28
 
15
- function inside(root: string, path: string): boolean {
16
- const child = relative(root, path);
17
- return (
18
- child === "" ||
19
- (child !== ".." && !child.startsWith(`..${sep}`) && !child.startsWith(sep))
20
- );
21
- }
29
+ const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
22
30
 
23
31
  function quoted(path: string): string {
24
- if (/[\x00-\x1f\x7f]/.test(path))
32
+ if (
33
+ Array.from(path).some(
34
+ (character) =>
35
+ character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
36
+ )
37
+ )
25
38
  throw new HostIsolationError("sandbox path contains a control character");
26
39
  return path.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
27
40
  }
28
41
 
29
- /** Deny both observation and mutation of every protected canonical root. */
42
+ /** Protect canonical roots while preserving explicitly granted access. */
30
43
  export function macSandboxProfile(
31
44
  deniedRoots: string[],
32
45
  denyNetwork = false,
46
+ readOnlyRoots: string[] = [],
47
+ writableRuntimeRoot?: string,
48
+ transcriptReadRoots: string[] = [],
49
+ workspace?: string,
50
+ localHttp?: boolean,
51
+ writableCacheRoots?: string[],
33
52
  ): string {
34
- if (!deniedRoots.length || deniedRoots.some((path) => !path.startsWith("/")))
35
- throw new HostIsolationError("sandbox roots must be absolute and nonempty");
53
+ requireProfileRoots(deniedRoots);
36
54
  return [
37
55
  "(version 1)",
38
56
  "(allow default)",
39
57
  ...(denyNetwork ? ["(deny network*)"] : []),
58
+ ...localHttpRules(localHttp),
40
59
  ...[...new Set(deniedRoots)]
41
60
  .sort()
42
61
  .flatMap((path) => [
43
- `(deny file-read* (subpath "${quoted(path)}"))`,
44
- `(deny file-write* (subpath "${quoted(path)}"))`,
62
+ `(deny file-read* ${workspaceReadFilter(path, readExceptions(path, readOnlyRoots, writableRuntimeRoot, transcriptReadRoots, workspace, writableCacheRoots), workspace)})`,
63
+ `(deny file-write* ${rootFilter(path, [...writableExceptions(path, writableRuntimeRoot, writableCacheRoots), ...workspaceBoundaryException(path, workspace)])})`,
45
64
  ]),
65
+ ...deniedRoots.flatMap((path) =>
66
+ workspaceBoundaryException(path, workspace).map(
67
+ () => `(deny file-read-data (literal "${quoted(path)}"))`,
68
+ ),
69
+ ),
70
+ ...[...readOnlyRoots, ...transcriptReadRoots].map(
71
+ (path) => `(deny file-write* (subpath "${quoted(path)}"))`,
72
+ ),
73
+ ...deniedRoots.flatMap((root) =>
74
+ ancestorMetadataRules(
75
+ root,
76
+ readExceptions(
77
+ root,
78
+ readOnlyRoots,
79
+ writableRuntimeRoot,
80
+ transcriptReadRoots,
81
+ workspace,
82
+ writableCacheRoots,
83
+ ),
84
+ ),
85
+ ),
46
86
  "",
47
87
  ].join("\n");
48
88
  }
49
89
 
50
- export interface IsolatedCommand {
51
- argv: string[];
52
- release(): Promise<void>;
90
+ function localHttpRules(enabled: boolean | undefined): string[] {
91
+ return enabled
92
+ ? [
93
+ '(allow network-bind (local ip "*:*"))',
94
+ '(allow network-inbound (local ip "localhost:*"))',
95
+ '(allow network-outbound (remote ip "localhost:*"))',
96
+ ]
97
+ : [];
98
+ }
99
+
100
+ function ancestorMetadataRules(root: string, readRoots: string[]): string[] {
101
+ const ancestors = new Set<string>();
102
+ for (const readRoot of readRoots) {
103
+ for (
104
+ let path = dirname(readRoot);
105
+ insideRuntimeRoot(root, path);
106
+ path = dirname(path)
107
+ ) {
108
+ ancestors.add(path);
109
+ if (path === resolve(root)) break;
110
+ }
111
+ }
112
+ return [...ancestors]
113
+ .sort()
114
+ .map((path) => `(allow file-read-metadata (literal "${quoted(path)}"))`);
115
+ }
116
+
117
+ function requireProfileRoots(roots: string[]): void {
118
+ if (!roots.length || roots.some((path) => !path.startsWith("/")))
119
+ throw new HostIsolationError("sandbox roots must be absolute and nonempty");
120
+ }
121
+
122
+ function readExceptions(
123
+ root: string,
124
+ readRoots: string[],
125
+ writableRuntimeRoot: string | undefined,
126
+ transcriptReadRoots: string[],
127
+ workspace?: string,
128
+ writableCacheRoots: string[] = [],
129
+ ): string[] {
130
+ const exceptions = isRuntimeHomeRoot(root)
131
+ ? readRoots.filter((path) => insideRuntimeRoot(root, path))
132
+ : [];
133
+ return [
134
+ ...transcriptReadRoots.filter((path) => insideRuntimeRoot(root, path)),
135
+ ...exceptions,
136
+ ...writableExceptions(root, writableRuntimeRoot, writableCacheRoots),
137
+ ...workspaceBoundaryException(root, workspace),
138
+ ];
139
+ }
140
+
141
+ function writableExceptions(
142
+ root: string,
143
+ runtime: string | undefined,
144
+ caches: string[] = [],
145
+ ): string[] {
146
+ return [
147
+ ...ownedException(root, runtime),
148
+ ...caches.flatMap((cache) => ownedException(root, cache)),
149
+ ];
150
+ }
151
+
152
+ function rootFilter(root: string, exceptions: string[]): string {
153
+ const base = `(subpath "${quoted(root)}")`;
154
+ if (!exceptions.length) return base;
155
+ return `(require-all ${base} ${exceptions.map((path) => `(require-not (subpath "${quoted(path)}"))`).join(" ")})`;
156
+ }
157
+
158
+ function workspaceReadFilter(
159
+ root: string,
160
+ exceptions: string[],
161
+ workspace?: string,
162
+ ): string {
163
+ const filter = rootFilter(root, exceptions);
164
+ return workspaceBoundaryException(root, workspace).length
165
+ ? `(require-all ${filter} (require-not (literal "${quoted(root)}")))`
166
+ : filter;
167
+ }
168
+
169
+ function ownedException(root: string, ownedRoot: string | undefined): string[] {
170
+ if (ownedRoot && insideRuntimeRoot(root, ownedRoot)) return [ownedRoot];
171
+ return [];
53
172
  }
54
173
 
55
174
  /** Build the outer boundary before launching an agent or candidate-controlled code. */
56
- export async function prepareMacSandboxCommand(options: {
57
- argv: string[];
58
- workspace: string;
59
- protectedRoots: string[];
60
- protectedRootsCanonical?: boolean;
61
- privateStateRoot: string;
62
- denyNetwork?: boolean;
63
- }): Promise<IsolatedCommand> {
64
- if (process.platform !== "darwin" || !existsSync(SANDBOX_EXEC))
65
- throw new HostIsolationError("macOS sandbox-exec isolation is unavailable");
66
- if (!options.argv.length || options.argv.some((part) => !part))
67
- throw new HostIsolationError("isolated command must be nonempty");
68
- if (!options.protectedRoots.length)
69
- throw new HostIsolationError("isolated command needs protected roots");
70
- if (
71
- !isAbsolute(options.workspace) ||
72
- !isAbsolute(options.privateStateRoot) ||
73
- options.protectedRoots.some((path) => !isAbsolute(path))
74
- )
75
- throw new HostIsolationError("isolation paths must be absolute");
175
+ export async function prepareMacSandboxCommand(
176
+ options: IsolationOptions,
177
+ ): Promise<IsolatedCommand> {
178
+ requireSandboxAvailable();
179
+ await requireNativeTranscriptViews(options.transcriptReadRoots);
180
+ await requireCandidateArtifactRoots(options.candidateArtifactRoots);
181
+ await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
182
+ requireIsolationPaths(options);
76
183
  const workspace = await realpath(options.workspace);
77
184
  const roots = options.protectedRootsCanonical
78
185
  ? options.protectedRoots
79
186
  : await Promise.all(options.protectedRoots.map((path) => realpath(path)));
80
187
  await mkdir(options.privateStateRoot, { recursive: true, mode: 0o700 });
81
188
  const stateRoot = await realpath(options.privateStateRoot);
82
- if (
83
- roots.some((root) => inside(root, workspace)) ||
84
- inside(stateRoot, workspace)
85
- )
86
- throw new HostIsolationError(
87
- "protected root includes the candidate workspace",
88
- );
89
- if (inside(workspace, stateRoot))
90
- throw new HostIsolationError(
91
- "sandbox state is inside the candidate workspace",
92
- );
189
+ const ownedRoot = await commandRuntimeRoot(
190
+ stateRoot,
191
+ options.writableRuntimeRoot,
192
+ );
193
+ const readRoots = options.readOnlyRoots ?? [];
194
+ const writableCacheRoots = await requireWritableCacheRoots(
195
+ options.writableCacheRoots ?? [],
196
+ [...roots, stateRoot],
197
+ );
198
+ requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
199
+ requireDisjointState(workspace, roots, stateRoot);
93
200
  const profilePath = resolve(stateRoot, `host-${randomUUID()}.sb`);
94
201
  await writeFile(
95
202
  profilePath,
96
- macSandboxProfile([...roots, stateRoot], options.denyNetwork),
97
- {
98
- flag: "wx",
99
- mode: 0o600,
100
- },
203
+ macSandboxProfile(
204
+ [...roots, stateRoot],
205
+ options.denyNetwork,
206
+ readRoots,
207
+ ownedRoot,
208
+ curatedReadRoots(options),
209
+ workspace,
210
+ options.localHttp,
211
+ writableCacheRoots,
212
+ ),
213
+ { flag: "wx", mode: 0o600 },
101
214
  );
102
215
  return {
103
216
  argv: [SANDBOX_EXEC, "-f", profilePath, ...options.argv],
@@ -106,3 +219,16 @@ export async function prepareMacSandboxCommand(options: {
106
219
  },
107
220
  };
108
221
  }
222
+
223
+ function curatedReadRoots(options: IsolationOptions): string[] {
224
+ return [
225
+ ...(options.transcriptReadRoots ?? []),
226
+ ...(options.hookReadRoots ?? []),
227
+ ...Object.values(options.candidateArtifactRoots ?? {}),
228
+ ];
229
+ }
230
+
231
+ function requireSandboxAvailable(): void {
232
+ if (process.platform !== "darwin" || !existsSync(SANDBOX_EXEC))
233
+ throw new HostIsolationError("macOS sandbox-exec isolation is unavailable");
234
+ }