@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +170 -0
- package/LICENSE +63 -136
- package/README.md +83 -66
- package/docs/README.md +44 -0
- package/docs/acknowledgements.md +14 -0
- package/docs/architecture.md +146 -0
- package/docs/assets/README.md +27 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +1 -0
- package/docs/assets/sevro-wordmark.svg +1 -0
- package/docs/builtin-graders-v1.md +41 -13
- package/docs/codex-host.md +184 -52
- package/docs/contracts.md +19 -0
- package/docs/creating-cases.md +139 -0
- package/docs/development-cli.md +190 -30
- package/docs/documentation-quality.md +61 -0
- package/docs/evaluation-workflow.md +110 -0
- package/docs/evidence/code-mode-controls-71.md +64 -0
- package/docs/evidence/code-mode-programs-85.md +64 -0
- package/docs/evidence/codex-state-73/README.md +204 -0
- package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
- package/docs/evidence/codex-state-73/results.json +661 -0
- package/docs/evidence/local-http-74.md +104 -0
- package/docs/evidence/runtime-startup-30/README.md +185 -0
- package/docs/evidence/runtime-startup-30/after.json +319 -0
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
- package/docs/evidence/runtime-startup-30/before.json +403 -0
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
- package/docs/evidence/skill-reads-70/results.json +823 -0
- package/docs/evidence.md +21 -0
- package/docs/extension-protocol-v1.md +50 -16
- package/docs/fixture-cleanup-validation.md +3 -0
- package/docs/getting-started.md +102 -0
- package/docs/guide-evaluation.md +89 -0
- package/docs/guide-verification.json +239 -0
- package/docs/identity-v1.md +35 -1
- package/docs/installing.md +88 -0
- package/docs/licenses/BUSL-1.1-0.1.0-rc.2.txt +159 -0
- package/docs/licensing.md +42 -0
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +228 -0
- package/docs/reading-results.md +141 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +81 -40
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +180 -0
- package/docs/run-settings-v1.md +147 -0
- package/docs/running-evaluations.md +31 -0
- package/docs/runtime-v1.md +614 -0
- package/docs/specs/repository-guide.md +58 -0
- package/docs/troubleshooting.md +30 -0
- package/docs/typescript-quality.md +234 -0
- package/docs/usability-study.md +114 -0
- package/examples/basic/host.ts +2 -2
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +39 -6
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +193 -0
- package/src/advisory.ts +10 -5
- package/src/candidate-artifacts.ts +208 -0
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +105 -0
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +81 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +59 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +222 -0
- package/src/cli/hosts.ts +59 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/cli/invocation.ts +627 -0
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +163 -0
- package/src/cli/presentation.ts +91 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +133 -0
- package/src/cli/run-options.ts +101 -0
- package/src/cli/run-overrides.ts +83 -0
- package/src/cli/settings.ts +160 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -843
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +19 -2373
- package/src/evaluation/advisory.ts +150 -0
- package/src/evaluation/check-plan.ts +510 -0
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/evaluation/grading-host.ts +39 -0
- package/src/evaluation/grading.ts +258 -0
- package/src/evaluation/invocation-plan.ts +548 -0
- package/src/evaluation/plan.ts +736 -0
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +3 -1
- package/src/evaluation/semantic.ts +312 -0
- package/src/evaluation/trial.ts +866 -0
- package/src/evaluation/types.ts +207 -0
- package/src/evidence/host-results.ts +92 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +228 -148
- package/src/extension-session.ts +326 -195
- package/src/fixtures/advisory.ts +338 -0
- package/src/fixtures/bin.ts +57 -0
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +187 -0
- package/src/fixtures/generated.ts +300 -0
- package/src/fixtures/git-hooks.ts +24 -0
- package/src/fixtures/preparation.ts +270 -0
- package/src/{repository-fixture.ts → fixtures/repository.ts} +207 -85
- package/src/fixtures/setup.ts +224 -0
- package/src/fixtures/tools.ts +88 -0
- package/src/fixtures/workspace-state.ts +212 -0
- package/src/generated/cli-result.cjs +2 -0
- package/src/generated/cli-result.d.cts +3 -0
- package/src/generated/extension.cjs +2 -0
- package/src/generated/extension.d.cts +3 -0
- package/src/generated/report.cjs +2 -0
- package/src/generated/report.d.cts +3 -0
- package/src/generated/run-evidence.cjs +2 -0
- package/src/generated/run-evidence.d.cts +3 -0
- package/src/generated/runtime.cjs +2 -0
- package/src/generated/runtime.d.cts +3 -0
- package/src/graders/artifact.ts +56 -23
- package/src/graders/git-head.ts +79 -75
- package/src/graders/output.ts +222 -140
- package/src/graders/semantic.ts +102 -43
- package/src/graders/shell.ts +379 -207
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +319 -0
- package/src/hosts/claude-continuation.ts +63 -18
- package/src/hosts/claude-credential.ts +43 -62
- package/src/hosts/claude-events.ts +86 -51
- package/src/hosts/claude-keychain.ts +59 -0
- package/src/hosts/claude-native-goal.ts +57 -33
- package/src/hosts/claude-process.ts +111 -0
- package/src/hosts/claude-repository-invocation.ts +208 -117
- package/src/hosts/claude-settings.ts +240 -21
- package/src/hosts/claude-skill-activations.ts +166 -0
- package/src/hosts/claude-state.ts +403 -0
- package/src/hosts/claude-tool-calls.ts +171 -106
- package/src/hosts/claude.ts +561 -451
- package/src/hosts/codex-app-server.ts +326 -178
- package/src/hosts/codex-configuration.ts +44 -18
- package/src/hosts/codex-events.ts +97 -60
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-invocation.ts +70 -0
- package/src/hosts/codex-marketplace.ts +283 -0
- package/src/hosts/codex-native-calls.ts +546 -500
- package/src/hosts/codex-process.ts +202 -0
- package/src/hosts/codex-profile.ts +257 -28
- package/src/hosts/codex-session-files.ts +106 -0
- package/src/hosts/codex-state.ts +492 -0
- package/src/hosts/codex.ts +783 -832
- package/src/hosts/isolation-common.ts +94 -0
- package/src/hosts/isolation-roots.ts +54 -27
- package/src/hosts/isolation.ts +10 -0
- package/src/hosts/linux-sandbox.ts +159 -0
- package/src/hosts/mac-sandbox.ts +187 -61
- package/src/hosts/native-controls.ts +152 -62
- package/src/hosts/process-lifecycle.ts +37 -0
- package/src/hosts/protected-worktrees.ts +51 -19
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +580 -0
- package/src/hosts/workspace-fingerprint.ts +55 -35
- package/src/identity.ts +102 -48
- package/src/instrumentation.ts +109 -48
- package/src/native-transcript-state.ts +131 -0
- package/src/native-transcript-view.ts +90 -0
- package/src/native-transcripts.ts +286 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +161 -81
- package/src/prune.ts +215 -0
- package/src/report.ts +172 -140
- package/src/resolved-case.ts +33 -0
- package/src/results-lock.ts +29 -0
- package/src/results.ts +133 -86
- package/src/run-owner.ts +154 -63
- package/src/runtime/config.ts +456 -0
- package/src/runtime/discovery-apple.ts +90 -0
- package/src/runtime/discovery-providers.ts +63 -0
- package/src/runtime/discovery.ts +70 -0
- package/src/runtime/hook-processes.ts +97 -0
- package/src/runtime/hooks.ts +476 -0
- package/src/runtime/paths.ts +313 -0
- package/src/runtime/query.ts +85 -0
- package/src/runtime/seeds.ts +134 -0
- package/src/runtime/shared-cache.ts +143 -0
- package/src/runtime/state.ts +143 -0
- package/src/schema-types.ts +198 -0
- package/src/schema.ts +15 -19
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +20 -0
- package/src/advisory-fixture.ts +0 -282
- package/src/fixture-bin.ts +0 -87
- package/src/fixture-setup.ts +0 -163
- package/src/generated-fixture.ts +0 -220
- package/src/git-hooks.ts +0 -63
- package/src/hosts/claude-nested-skills.ts +0 -218
- package/src/hosts/codex-command-output.ts +0 -399
- package/src/hosts/codex-executor-command.ts +0 -195
- package/src/hosts/codex-skill-reads.ts +0 -391
- package/src/preparation.ts +0 -211
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { realpath } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, relative, sep } from "node:path";
|
|
3
|
+
import { insideRuntimeRoot } from "../runtime/paths";
|
|
4
|
+
import { workspaceBoundaryException } from "../fixtures/workspace-state";
|
|
5
|
+
import type { CandidateArtifactRoots } from "../candidate-artifacts";
|
|
6
|
+
|
|
7
|
+
export class HostIsolationError extends Error {
|
|
8
|
+
constructor(message: string) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "HostIsolationError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface IsolationOptions {
|
|
15
|
+
argv: string[];
|
|
16
|
+
workspace: string;
|
|
17
|
+
protectedRoots: string[];
|
|
18
|
+
protectedRootsCanonical?: boolean;
|
|
19
|
+
privateStateRoot: string;
|
|
20
|
+
denyNetwork?: boolean;
|
|
21
|
+
localHttp?: boolean;
|
|
22
|
+
readOnlyRoots?: string[];
|
|
23
|
+
transcriptReadRoots?: string[];
|
|
24
|
+
candidateArtifactRoots?: CandidateArtifactRoots;
|
|
25
|
+
hookReadRoots?: string[];
|
|
26
|
+
writableRuntimeRoot?: string;
|
|
27
|
+
writableCacheRoots?: string[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface IsolatedCommand {
|
|
31
|
+
argv: string[];
|
|
32
|
+
release(): Promise<void>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function inside(root: string, path: string): boolean {
|
|
36
|
+
const child = relative(root, path);
|
|
37
|
+
return (
|
|
38
|
+
child === "" ||
|
|
39
|
+
(child !== ".." && !child.startsWith(`..${sep}`) && !child.startsWith(sep))
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export async function commandRuntimeRoot(
|
|
44
|
+
stateRoot: string,
|
|
45
|
+
path: string | undefined,
|
|
46
|
+
): Promise<string | undefined> {
|
|
47
|
+
if (!path) return undefined;
|
|
48
|
+
const root = await realpath(path);
|
|
49
|
+
if (root === stateRoot || !insideRuntimeRoot(stateRoot, root))
|
|
50
|
+
throw new HostIsolationError(
|
|
51
|
+
"command runtime must be contained in private state",
|
|
52
|
+
);
|
|
53
|
+
return root;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function requireIsolationPaths(options: IsolationOptions): void {
|
|
57
|
+
if (!options.argv.length || options.argv.some((part) => !part))
|
|
58
|
+
throw new HostIsolationError("isolated command must be nonempty");
|
|
59
|
+
if (!options.protectedRoots.length)
|
|
60
|
+
throw new HostIsolationError("isolated command needs protected roots");
|
|
61
|
+
if (!absoluteIsolationPaths(options))
|
|
62
|
+
throw new HostIsolationError("isolation paths must be absolute");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function absoluteIsolationPaths(options: IsolationOptions): boolean {
|
|
66
|
+
return (
|
|
67
|
+
isAbsolute(options.workspace) &&
|
|
68
|
+
isAbsolute(options.privateStateRoot) &&
|
|
69
|
+
(options.writableCacheRoots ?? []).every(isAbsolute) &&
|
|
70
|
+
options.protectedRoots.every(isAbsolute)
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function requireDisjointState(
|
|
75
|
+
workspace: string,
|
|
76
|
+
roots: string[],
|
|
77
|
+
stateRoot: string,
|
|
78
|
+
): void {
|
|
79
|
+
if (
|
|
80
|
+
roots.some(
|
|
81
|
+
(root) =>
|
|
82
|
+
inside(root, workspace) &&
|
|
83
|
+
!workspaceBoundaryException(root, workspace).length,
|
|
84
|
+
) ||
|
|
85
|
+
inside(stateRoot, workspace)
|
|
86
|
+
)
|
|
87
|
+
throw new HostIsolationError(
|
|
88
|
+
"protected root includes the candidate workspace",
|
|
89
|
+
);
|
|
90
|
+
if (inside(workspace, stateRoot))
|
|
91
|
+
throw new HostIsolationError(
|
|
92
|
+
"sandbox state is inside the candidate workspace",
|
|
93
|
+
);
|
|
94
|
+
}
|
|
@@ -1,21 +1,28 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { isMissingFile } from "../fixtures/tools";
|
|
2
|
+
import { realpath } from "node:fs/promises";
|
|
3
|
+
import { homedir, userInfo } from "node:os";
|
|
4
|
+
import { isAbsolute, resolve } from "node:path";
|
|
4
5
|
import { protectedWorktrees } from "./protected-worktrees";
|
|
6
|
+
import { nativeStateParent } from "../native-transcript-state";
|
|
7
|
+
import { workspaceStateParent } from "../fixtures/workspace-state";
|
|
5
8
|
|
|
6
9
|
/** Include source, results, host configuration, and every active peer fixture. */
|
|
7
|
-
|
|
10
|
+
interface ProtectedRootsOptions {
|
|
8
11
|
workspace: string;
|
|
9
12
|
projectRoot: string;
|
|
10
13
|
resultsRoot: string;
|
|
11
14
|
additionalRoots: string[];
|
|
12
|
-
|
|
13
|
-
|
|
15
|
+
ownedStateRoot?: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async function protectedRootCandidates(options: ProtectedRootsOptions) {
|
|
14
19
|
const optionalConfigRoots = [
|
|
15
20
|
process.env.CODEX_HOME,
|
|
16
21
|
process.env.CLAUDE_CONFIG_DIR,
|
|
17
22
|
].filter((value): value is string => Boolean(value));
|
|
18
23
|
const candidates = [
|
|
24
|
+
await nativeStateParent(),
|
|
25
|
+
await workspaceStateParent(),
|
|
19
26
|
resolve(import.meta.dir, "../.."),
|
|
20
27
|
options.projectRoot,
|
|
21
28
|
options.resultsRoot,
|
|
@@ -26,29 +33,29 @@ export async function evaluationProtectedRoots(options: {
|
|
|
26
33
|
];
|
|
27
34
|
if (candidates.some((root) => !isAbsolute(root)))
|
|
28
35
|
throw new Error("evaluation protected roots must be absolute");
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
36
|
+
return { candidates, optionalConfigRoots };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Include source, results, host configuration, and every active peer fixture. */
|
|
40
|
+
export async function evaluationProtectedRoots(
|
|
41
|
+
options: ProtectedRootsOptions,
|
|
42
|
+
): Promise<string[]> {
|
|
43
|
+
const workspace = await realpath(options.workspace);
|
|
44
|
+
const { candidates, optionalConfigRoots } =
|
|
45
|
+
await protectedRootCandidates(options);
|
|
46
|
+
const namespace = await workspaceStateParent();
|
|
47
|
+
const nativeNamespace = await nativeStateParent();
|
|
34
48
|
const roots: string[] = [];
|
|
35
|
-
for (const candidate of
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
optionalConfigRoots.includes(candidate)) &&
|
|
43
|
-
(error as NodeJS.ErrnoException).code === "ENOENT"
|
|
44
|
-
)
|
|
45
|
-
continue;
|
|
46
|
-
throw new Error("evaluation protected root is unreadable");
|
|
47
|
-
}
|
|
48
|
-
if (canonical !== workspace) {
|
|
49
|
+
for (const candidate of candidates) {
|
|
50
|
+
const canonical = await canonicalProtectedRoot(candidate, [
|
|
51
|
+
...optionalConfigRoots,
|
|
52
|
+
]);
|
|
53
|
+
if (!includedProtectedRoot(canonical, workspace, options.ownedStateRoot))
|
|
54
|
+
continue;
|
|
55
|
+
{
|
|
49
56
|
roots.push(canonical);
|
|
50
|
-
//
|
|
51
|
-
if (
|
|
57
|
+
// Private namespaces are boundaries, not Git source trees to traverse.
|
|
58
|
+
if (canonical !== namespace && canonical !== nativeNamespace)
|
|
52
59
|
roots.push(
|
|
53
60
|
...(await protectedWorktrees(canonical)).filter(
|
|
54
61
|
(root) => root !== workspace,
|
|
@@ -58,3 +65,23 @@ export async function evaluationProtectedRoots(options: {
|
|
|
58
65
|
}
|
|
59
66
|
return [...new Set(roots)];
|
|
60
67
|
}
|
|
68
|
+
|
|
69
|
+
function includedProtectedRoot(
|
|
70
|
+
root: string | null,
|
|
71
|
+
workspace: string,
|
|
72
|
+
state: string | undefined,
|
|
73
|
+
): root is string {
|
|
74
|
+
return root !== null && root !== workspace && root !== state;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function canonicalProtectedRoot(
|
|
78
|
+
candidate: string,
|
|
79
|
+
optionalRoots: string[],
|
|
80
|
+
): Promise<string | null> {
|
|
81
|
+
try {
|
|
82
|
+
return await realpath(candidate);
|
|
83
|
+
} catch (cause) {
|
|
84
|
+
if (optionalRoots.includes(candidate) && isMissingFile(cause)) return null;
|
|
85
|
+
throw new Error("evaluation protected root is unreadable", { cause });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { HostIsolationError, type IsolationOptions } from "./isolation-common";
|
|
2
|
+
import { prepareMacSandboxCommand } from "./mac-sandbox";
|
|
3
|
+
import { prepareLinuxSandboxCommand } from "./linux-sandbox";
|
|
4
|
+
|
|
5
|
+
/** Select the native filesystem boundary for evaluator-owned commands. */
|
|
6
|
+
export function prepareIsolatedCommand(options: IsolationOptions) {
|
|
7
|
+
if (process.platform === "darwin") return prepareMacSandboxCommand(options);
|
|
8
|
+
if (process.platform === "linux") return prepareLinuxSandboxCommand(options);
|
|
9
|
+
throw new HostIsolationError("host isolation is unavailable");
|
|
10
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { mkdir, realpath, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { resolve } from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
requireNativeTranscriptViews,
|
|
7
|
+
requireNativeHookRoots,
|
|
8
|
+
} from "../native-transcript-state";
|
|
9
|
+
import { requireRuntimeReadRoots } from "../runtime/paths";
|
|
10
|
+
import { requireWritableCacheRoots } from "../runtime/shared-cache";
|
|
11
|
+
import { requireCandidateArtifactRoots } from "../candidate-artifacts";
|
|
12
|
+
import {
|
|
13
|
+
commandRuntimeRoot,
|
|
14
|
+
HostIsolationError,
|
|
15
|
+
requireDisjointState,
|
|
16
|
+
requireIsolationPaths,
|
|
17
|
+
type IsolatedCommand,
|
|
18
|
+
type IsolationOptions,
|
|
19
|
+
} from "./isolation-common";
|
|
20
|
+
|
|
21
|
+
const BWRAP = "/usr/bin/bwrap";
|
|
22
|
+
|
|
23
|
+
async function protectedKind(path: string): Promise<"file" | "directory"> {
|
|
24
|
+
try {
|
|
25
|
+
return (await stat(path)).isDirectory() ? "directory" : "file";
|
|
26
|
+
} catch (error) {
|
|
27
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return "directory";
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function requireLinuxSandbox(): void {
|
|
33
|
+
if (process.platform !== "linux" || !existsSync(BWRAP))
|
|
34
|
+
throw new HostIsolationError("Linux bubblewrap isolation is unavailable");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function networkArgs(denyNetwork: boolean): string[] {
|
|
38
|
+
return denyNetwork ? ["--unshare-net"] : [];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function mountProtectedRoot(
|
|
42
|
+
path: string,
|
|
43
|
+
kind: "file" | "directory" | undefined,
|
|
44
|
+
blockedFile: string,
|
|
45
|
+
): string[] {
|
|
46
|
+
return kind === "directory"
|
|
47
|
+
? ["--tmpfs", path, "--chmod", "0111", path]
|
|
48
|
+
: ["--ro-bind", blockedFile, path];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Mask protected inputs before launching an evaluator-owned command. */
|
|
52
|
+
export async function prepareLinuxSandboxCommand(
|
|
53
|
+
options: IsolationOptions,
|
|
54
|
+
): Promise<IsolatedCommand> {
|
|
55
|
+
requireLinuxSandbox();
|
|
56
|
+
await requireNativeTranscriptViews(options.transcriptReadRoots);
|
|
57
|
+
await requireCandidateArtifactRoots(options.candidateArtifactRoots);
|
|
58
|
+
await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
|
|
59
|
+
requireIsolationPaths(options);
|
|
60
|
+
const workspace = await realpath(options.workspace);
|
|
61
|
+
const roots = options.protectedRootsCanonical
|
|
62
|
+
? options.protectedRoots
|
|
63
|
+
: await Promise.all(options.protectedRoots.map((path) => realpath(path)));
|
|
64
|
+
await mkdir(options.privateStateRoot, { recursive: true, mode: 0o700 });
|
|
65
|
+
const stateRoot = await realpath(options.privateStateRoot);
|
|
66
|
+
const ownedRoot = await commandRuntimeRoot(
|
|
67
|
+
stateRoot,
|
|
68
|
+
options.writableRuntimeRoot,
|
|
69
|
+
);
|
|
70
|
+
const readRoots = options.readOnlyRoots ?? [];
|
|
71
|
+
const writableCacheRoots = await cacheMountRoots(options, [
|
|
72
|
+
...roots,
|
|
73
|
+
stateRoot,
|
|
74
|
+
]);
|
|
75
|
+
requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
|
|
76
|
+
requireDisjointState(workspace, roots, stateRoot);
|
|
77
|
+
const blockedFile = resolve(stateRoot, `blocked-${randomUUID()}`);
|
|
78
|
+
await writeFile(blockedFile, "", { flag: "wx", mode: 0o000 });
|
|
79
|
+
try {
|
|
80
|
+
const protectedRoots = [...new Set([...roots, stateRoot])].sort(
|
|
81
|
+
(left, right) => left.length - right.length,
|
|
82
|
+
);
|
|
83
|
+
const kinds = await Promise.all(protectedRoots.map(protectedKind));
|
|
84
|
+
const masks = protectedRoots.flatMap((path, index) =>
|
|
85
|
+
mountProtectedRoot(path, kinds[index], blockedFile),
|
|
86
|
+
);
|
|
87
|
+
return {
|
|
88
|
+
argv: [
|
|
89
|
+
BWRAP,
|
|
90
|
+
"--die-with-parent",
|
|
91
|
+
"--unshare-user",
|
|
92
|
+
"--unshare-pid",
|
|
93
|
+
...networkArgs(Boolean(options.denyNetwork)),
|
|
94
|
+
"--ro-bind",
|
|
95
|
+
"/",
|
|
96
|
+
"/",
|
|
97
|
+
"--dev-bind",
|
|
98
|
+
"/dev",
|
|
99
|
+
"/dev",
|
|
100
|
+
"--proc",
|
|
101
|
+
"/proc",
|
|
102
|
+
"--bind",
|
|
103
|
+
"/tmp",
|
|
104
|
+
"/tmp",
|
|
105
|
+
...masks,
|
|
106
|
+
"--bind",
|
|
107
|
+
workspace,
|
|
108
|
+
workspace,
|
|
109
|
+
...readRoots.flatMap((path) => ["--ro-bind", path, path]),
|
|
110
|
+
...transcriptMounts(options),
|
|
111
|
+
...candidateArtifactMounts(options),
|
|
112
|
+
...hookMounts(options),
|
|
113
|
+
...(ownedRoot ? ["--bind", ownedRoot, ownedRoot] : []),
|
|
114
|
+
...writableCacheRoots.flatMap((path) => ["--bind", path, path]),
|
|
115
|
+
"--chdir",
|
|
116
|
+
workspace,
|
|
117
|
+
"--",
|
|
118
|
+
...options.argv,
|
|
119
|
+
],
|
|
120
|
+
release: async () => {
|
|
121
|
+
await rm(blockedFile, { force: true });
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
} catch (error) {
|
|
125
|
+
await rm(blockedFile, { force: true });
|
|
126
|
+
throw error;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function cacheMountRoots(options: IsolationOptions, protectedRoots: string[]) {
|
|
131
|
+
return requireWritableCacheRoots(
|
|
132
|
+
options.writableCacheRoots ?? [],
|
|
133
|
+
protectedRoots,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function transcriptMounts(options: IsolationOptions): string[] {
|
|
138
|
+
return (options.transcriptReadRoots ?? []).flatMap((path) => [
|
|
139
|
+
"--ro-bind",
|
|
140
|
+
path,
|
|
141
|
+
path,
|
|
142
|
+
]);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function candidateArtifactMounts(options: IsolationOptions): string[] {
|
|
146
|
+
return Object.values(options.candidateArtifactRoots ?? {}).flatMap((path) => [
|
|
147
|
+
"--ro-bind",
|
|
148
|
+
path,
|
|
149
|
+
path,
|
|
150
|
+
]);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function hookMounts(options: IsolationOptions): string[] {
|
|
154
|
+
return (options.hookReadRoots ?? []).flatMap((path) => [
|
|
155
|
+
"--ro-bind",
|
|
156
|
+
path,
|
|
157
|
+
path,
|
|
158
|
+
]);
|
|
159
|
+
}
|
package/src/hosts/mac-sandbox.ts
CHANGED
|
@@ -1,103 +1,216 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
3
|
import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
|
|
4
|
-
import {
|
|
4
|
+
import { dirname, resolve } from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
requireNativeTranscriptViews,
|
|
7
|
+
requireNativeHookRoots,
|
|
8
|
+
} from "../native-transcript-state";
|
|
9
|
+
import { requireCandidateArtifactRoots } from "../candidate-artifacts";
|
|
10
|
+
import { workspaceBoundaryException } from "../fixtures/workspace-state";
|
|
11
|
+
import { requireWritableCacheRoots } from "../runtime/shared-cache";
|
|
12
|
+
import {
|
|
13
|
+
insideRuntimeRoot,
|
|
14
|
+
isRuntimeHomeRoot,
|
|
15
|
+
requireRuntimeReadRoots,
|
|
16
|
+
} from "../runtime/paths";
|
|
17
|
+
import {
|
|
18
|
+
commandRuntimeRoot,
|
|
19
|
+
HostIsolationError,
|
|
20
|
+
requireDisjointState,
|
|
21
|
+
requireIsolationPaths,
|
|
22
|
+
type IsolatedCommand,
|
|
23
|
+
type IsolationOptions,
|
|
24
|
+
} from "./isolation-common";
|
|
5
25
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
export class HostIsolationError extends Error {
|
|
9
|
-
constructor(message: string) {
|
|
10
|
-
super(message);
|
|
11
|
-
this.name = "HostIsolationError";
|
|
12
|
-
}
|
|
13
|
-
}
|
|
26
|
+
export { HostIsolationError } from "./isolation-common";
|
|
27
|
+
export type { IsolatedCommand } from "./isolation-common";
|
|
14
28
|
|
|
15
|
-
|
|
16
|
-
const child = relative(root, path);
|
|
17
|
-
return (
|
|
18
|
-
child === "" ||
|
|
19
|
-
(child !== ".." && !child.startsWith(`..${sep}`) && !child.startsWith(sep))
|
|
20
|
-
);
|
|
21
|
-
}
|
|
29
|
+
const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
|
|
22
30
|
|
|
23
31
|
function quoted(path: string): string {
|
|
24
|
-
if (
|
|
32
|
+
if (
|
|
33
|
+
Array.from(path).some(
|
|
34
|
+
(character) =>
|
|
35
|
+
character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
|
|
36
|
+
)
|
|
37
|
+
)
|
|
25
38
|
throw new HostIsolationError("sandbox path contains a control character");
|
|
26
39
|
return path.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
|
|
27
40
|
}
|
|
28
41
|
|
|
29
|
-
/**
|
|
42
|
+
/** Protect canonical roots while preserving explicitly granted access. */
|
|
30
43
|
export function macSandboxProfile(
|
|
31
44
|
deniedRoots: string[],
|
|
32
45
|
denyNetwork = false,
|
|
46
|
+
readOnlyRoots: string[] = [],
|
|
47
|
+
writableRuntimeRoot?: string,
|
|
48
|
+
transcriptReadRoots: string[] = [],
|
|
49
|
+
workspace?: string,
|
|
50
|
+
localHttp?: boolean,
|
|
51
|
+
writableCacheRoots?: string[],
|
|
33
52
|
): string {
|
|
34
|
-
|
|
35
|
-
throw new HostIsolationError("sandbox roots must be absolute and nonempty");
|
|
53
|
+
requireProfileRoots(deniedRoots);
|
|
36
54
|
return [
|
|
37
55
|
"(version 1)",
|
|
38
56
|
"(allow default)",
|
|
39
57
|
...(denyNetwork ? ["(deny network*)"] : []),
|
|
58
|
+
...localHttpRules(localHttp),
|
|
40
59
|
...[...new Set(deniedRoots)]
|
|
41
60
|
.sort()
|
|
42
61
|
.flatMap((path) => [
|
|
43
|
-
`(deny file-read*
|
|
44
|
-
`(deny file-write*
|
|
62
|
+
`(deny file-read* ${workspaceReadFilter(path, readExceptions(path, readOnlyRoots, writableRuntimeRoot, transcriptReadRoots, workspace, writableCacheRoots), workspace)})`,
|
|
63
|
+
`(deny file-write* ${rootFilter(path, [...writableExceptions(path, writableRuntimeRoot, writableCacheRoots), ...workspaceBoundaryException(path, workspace)])})`,
|
|
45
64
|
]),
|
|
65
|
+
...deniedRoots.flatMap((path) =>
|
|
66
|
+
workspaceBoundaryException(path, workspace).map(
|
|
67
|
+
() => `(deny file-read-data (literal "${quoted(path)}"))`,
|
|
68
|
+
),
|
|
69
|
+
),
|
|
70
|
+
...[...readOnlyRoots, ...transcriptReadRoots].map(
|
|
71
|
+
(path) => `(deny file-write* (subpath "${quoted(path)}"))`,
|
|
72
|
+
),
|
|
73
|
+
...deniedRoots.flatMap((root) =>
|
|
74
|
+
ancestorMetadataRules(
|
|
75
|
+
root,
|
|
76
|
+
readExceptions(
|
|
77
|
+
root,
|
|
78
|
+
readOnlyRoots,
|
|
79
|
+
writableRuntimeRoot,
|
|
80
|
+
transcriptReadRoots,
|
|
81
|
+
workspace,
|
|
82
|
+
writableCacheRoots,
|
|
83
|
+
),
|
|
84
|
+
),
|
|
85
|
+
),
|
|
46
86
|
"",
|
|
47
87
|
].join("\n");
|
|
48
88
|
}
|
|
49
89
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
90
|
+
function localHttpRules(enabled: boolean | undefined): string[] {
|
|
91
|
+
return enabled
|
|
92
|
+
? [
|
|
93
|
+
'(allow network-bind (local ip "*:*"))',
|
|
94
|
+
'(allow network-inbound (local ip "localhost:*"))',
|
|
95
|
+
'(allow network-outbound (remote ip "localhost:*"))',
|
|
96
|
+
]
|
|
97
|
+
: [];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function ancestorMetadataRules(root: string, readRoots: string[]): string[] {
|
|
101
|
+
const ancestors = new Set<string>();
|
|
102
|
+
for (const readRoot of readRoots) {
|
|
103
|
+
for (
|
|
104
|
+
let path = dirname(readRoot);
|
|
105
|
+
insideRuntimeRoot(root, path);
|
|
106
|
+
path = dirname(path)
|
|
107
|
+
) {
|
|
108
|
+
ancestors.add(path);
|
|
109
|
+
if (path === resolve(root)) break;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return [...ancestors]
|
|
113
|
+
.sort()
|
|
114
|
+
.map((path) => `(allow file-read-metadata (literal "${quoted(path)}"))`);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function requireProfileRoots(roots: string[]): void {
|
|
118
|
+
if (!roots.length || roots.some((path) => !path.startsWith("/")))
|
|
119
|
+
throw new HostIsolationError("sandbox roots must be absolute and nonempty");
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function readExceptions(
|
|
123
|
+
root: string,
|
|
124
|
+
readRoots: string[],
|
|
125
|
+
writableRuntimeRoot: string | undefined,
|
|
126
|
+
transcriptReadRoots: string[],
|
|
127
|
+
workspace?: string,
|
|
128
|
+
writableCacheRoots: string[] = [],
|
|
129
|
+
): string[] {
|
|
130
|
+
const exceptions = isRuntimeHomeRoot(root)
|
|
131
|
+
? readRoots.filter((path) => insideRuntimeRoot(root, path))
|
|
132
|
+
: [];
|
|
133
|
+
return [
|
|
134
|
+
...transcriptReadRoots.filter((path) => insideRuntimeRoot(root, path)),
|
|
135
|
+
...exceptions,
|
|
136
|
+
...writableExceptions(root, writableRuntimeRoot, writableCacheRoots),
|
|
137
|
+
...workspaceBoundaryException(root, workspace),
|
|
138
|
+
];
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function writableExceptions(
|
|
142
|
+
root: string,
|
|
143
|
+
runtime: string | undefined,
|
|
144
|
+
caches: string[] = [],
|
|
145
|
+
): string[] {
|
|
146
|
+
return [
|
|
147
|
+
...ownedException(root, runtime),
|
|
148
|
+
...caches.flatMap((cache) => ownedException(root, cache)),
|
|
149
|
+
];
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function rootFilter(root: string, exceptions: string[]): string {
|
|
153
|
+
const base = `(subpath "${quoted(root)}")`;
|
|
154
|
+
if (!exceptions.length) return base;
|
|
155
|
+
return `(require-all ${base} ${exceptions.map((path) => `(require-not (subpath "${quoted(path)}"))`).join(" ")})`;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function workspaceReadFilter(
|
|
159
|
+
root: string,
|
|
160
|
+
exceptions: string[],
|
|
161
|
+
workspace?: string,
|
|
162
|
+
): string {
|
|
163
|
+
const filter = rootFilter(root, exceptions);
|
|
164
|
+
return workspaceBoundaryException(root, workspace).length
|
|
165
|
+
? `(require-all ${filter} (require-not (literal "${quoted(root)}")))`
|
|
166
|
+
: filter;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function ownedException(root: string, ownedRoot: string | undefined): string[] {
|
|
170
|
+
if (ownedRoot && insideRuntimeRoot(root, ownedRoot)) return [ownedRoot];
|
|
171
|
+
return [];
|
|
53
172
|
}
|
|
54
173
|
|
|
55
174
|
/** Build the outer boundary before launching an agent or candidate-controlled code. */
|
|
56
|
-
export async function prepareMacSandboxCommand(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (process.platform !== "darwin" || !existsSync(SANDBOX_EXEC))
|
|
65
|
-
throw new HostIsolationError("macOS sandbox-exec isolation is unavailable");
|
|
66
|
-
if (!options.argv.length || options.argv.some((part) => !part))
|
|
67
|
-
throw new HostIsolationError("isolated command must be nonempty");
|
|
68
|
-
if (!options.protectedRoots.length)
|
|
69
|
-
throw new HostIsolationError("isolated command needs protected roots");
|
|
70
|
-
if (
|
|
71
|
-
!isAbsolute(options.workspace) ||
|
|
72
|
-
!isAbsolute(options.privateStateRoot) ||
|
|
73
|
-
options.protectedRoots.some((path) => !isAbsolute(path))
|
|
74
|
-
)
|
|
75
|
-
throw new HostIsolationError("isolation paths must be absolute");
|
|
175
|
+
export async function prepareMacSandboxCommand(
|
|
176
|
+
options: IsolationOptions,
|
|
177
|
+
): Promise<IsolatedCommand> {
|
|
178
|
+
requireSandboxAvailable();
|
|
179
|
+
await requireNativeTranscriptViews(options.transcriptReadRoots);
|
|
180
|
+
await requireCandidateArtifactRoots(options.candidateArtifactRoots);
|
|
181
|
+
await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
|
|
182
|
+
requireIsolationPaths(options);
|
|
76
183
|
const workspace = await realpath(options.workspace);
|
|
77
184
|
const roots = options.protectedRootsCanonical
|
|
78
185
|
? options.protectedRoots
|
|
79
186
|
: await Promise.all(options.protectedRoots.map((path) => realpath(path)));
|
|
80
187
|
await mkdir(options.privateStateRoot, { recursive: true, mode: 0o700 });
|
|
81
188
|
const stateRoot = await realpath(options.privateStateRoot);
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
189
|
+
const ownedRoot = await commandRuntimeRoot(
|
|
190
|
+
stateRoot,
|
|
191
|
+
options.writableRuntimeRoot,
|
|
192
|
+
);
|
|
193
|
+
const readRoots = options.readOnlyRoots ?? [];
|
|
194
|
+
const writableCacheRoots = await requireWritableCacheRoots(
|
|
195
|
+
options.writableCacheRoots ?? [],
|
|
196
|
+
[...roots, stateRoot],
|
|
197
|
+
);
|
|
198
|
+
requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
|
|
199
|
+
requireDisjointState(workspace, roots, stateRoot);
|
|
93
200
|
const profilePath = resolve(stateRoot, `host-${randomUUID()}.sb`);
|
|
94
201
|
await writeFile(
|
|
95
202
|
profilePath,
|
|
96
|
-
macSandboxProfile(
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
203
|
+
macSandboxProfile(
|
|
204
|
+
[...roots, stateRoot],
|
|
205
|
+
options.denyNetwork,
|
|
206
|
+
readRoots,
|
|
207
|
+
ownedRoot,
|
|
208
|
+
curatedReadRoots(options),
|
|
209
|
+
workspace,
|
|
210
|
+
options.localHttp,
|
|
211
|
+
writableCacheRoots,
|
|
212
|
+
),
|
|
213
|
+
{ flag: "wx", mode: 0o600 },
|
|
101
214
|
);
|
|
102
215
|
return {
|
|
103
216
|
argv: [SANDBOX_EXEC, "-f", profilePath, ...options.argv],
|
|
@@ -106,3 +219,16 @@ export async function prepareMacSandboxCommand(options: {
|
|
|
106
219
|
},
|
|
107
220
|
};
|
|
108
221
|
}
|
|
222
|
+
|
|
223
|
+
function curatedReadRoots(options: IsolationOptions): string[] {
|
|
224
|
+
return [
|
|
225
|
+
...(options.transcriptReadRoots ?? []),
|
|
226
|
+
...(options.hookReadRoots ?? []),
|
|
227
|
+
...Object.values(options.candidateArtifactRoots ?? {}),
|
|
228
|
+
];
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function requireSandboxAvailable(): void {
|
|
232
|
+
if (process.platform !== "darwin" || !existsSync(SANDBOX_EXEC))
|
|
233
|
+
throw new HostIsolationError("macOS sandbox-exec isolation is unavailable");
|
|
234
|
+
}
|