@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +22 -6
- package/README.md +5 -1
- package/docs/README.md +19 -15
- package/docs/architecture.md +138 -17
- package/docs/assets/README.md +5 -4
- package/docs/builtin-graders-v1.md +17 -3
- package/docs/codex-host.md +155 -49
- package/docs/contracts.md +2 -1
- package/docs/creating-cases.md +9 -2
- package/docs/development-cli.md +187 -19
- package/docs/documentation-quality.md +12 -2
- package/docs/evaluation-workflow.md +110 -0
- package/docs/extension-protocol-v1.md +26 -3
- package/docs/getting-started.md +23 -2
- package/docs/guide-evaluation.md +33 -1
- package/docs/identity-v1.md +25 -2
- package/docs/installing.md +11 -11
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +196 -7
- package/docs/reading-results.md +102 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +31 -12
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +141 -4
- package/docs/run-settings-v1.md +147 -0
- package/docs/runtime-v1.md +307 -18
- package/docs/troubleshooting.md +12 -12
- package/docs/typescript-quality.md +89 -26
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +11 -5
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +63 -7
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +206 -0
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +73 -50
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +87 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +60 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +198 -0
- package/src/cli/hosts.ts +55 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +154 -0
- package/src/cli/presentation.ts +93 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +128 -0
- package/src/cli/run-options.ts +403 -0
- package/src/cli/run-overrides.ts +69 -0
- package/src/cli/settings.ts +154 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -591
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +13 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +235 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +133 -115
- package/src/evaluation/trial.ts +800 -0
- package/src/evaluation/types.ts +207 -0
- package/src/evidence/host-results.ts +85 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +34 -96
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +15 -76
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +183 -0
- package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
- package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
- package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/fixtures/workspace-state.ts +212 -0
- package/src/generated/cli-result.d.cts +2 -1
- package/src/generated/extension.d.cts +2 -1
- package/src/generated/report.cjs +1 -1
- package/src/generated/report.d.cts +2 -1
- package/src/generated/run-evidence.cjs +1 -1
- package/src/generated/run-evidence.d.cts +2 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/generated/runtime.d.cts +2 -1
- package/src/git.ts +93 -0
- package/src/graders/git-head.ts +29 -84
- package/src/graders/shell.ts +57 -68
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +331 -0
- package/src/hosts/claude-continuation.ts +57 -40
- package/src/hosts/claude-credential.ts +160 -63
- package/src/hosts/claude-events.ts +34 -18
- package/src/hosts/claude-keychain.ts +78 -21
- package/src/hosts/claude-login-recovery.ts +85 -0
- package/src/hosts/claude-login-refresh.ts +178 -0
- package/src/hosts/claude-login-store.ts +162 -0
- package/src/hosts/claude-login.ts +107 -0
- package/src/hosts/claude-native-goal.ts +15 -12
- package/src/hosts/claude-process.ts +10 -50
- package/src/hosts/claude-refresh-lock.ts +81 -0
- package/src/hosts/claude-repository-invocation.ts +9 -34
- package/src/hosts/claude-settings.ts +91 -14
- package/src/hosts/claude-skill-activations.ts +169 -0
- package/src/hosts/claude-state.ts +46 -16
- package/src/hosts/claude-tool-calls.ts +13 -10
- package/src/hosts/claude.ts +122 -73
- package/src/hosts/codex-app-server.ts +78 -52
- package/src/hosts/codex-configuration.ts +5 -10
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-marketplace.ts +12 -10
- package/src/hosts/codex-native-calls.ts +15 -122
- package/src/hosts/codex-process.ts +10 -54
- package/src/hosts/codex-profile.ts +50 -21
- package/src/hosts/codex-session-files.ts +1 -9
- package/src/hosts/codex-state.ts +141 -19
- package/src/hosts/codex.ts +221 -179
- package/src/hosts/isolation-common.ts +15 -11
- package/src/hosts/isolation-roots.ts +13 -20
- package/src/hosts/linux-sandbox.ts +27 -3
- package/src/hosts/mac-sandbox.ts +111 -18
- package/src/hosts/native-controls.ts +72 -22
- package/src/hosts/native-request.ts +55 -0
- package/src/hosts/process-lifecycle.ts +221 -45
- package/src/hosts/protected-worktrees.ts +1 -1
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +577 -0
- package/src/instrumentation.ts +7 -8
- package/src/native-transcript-state.ts +22 -3
- package/src/native-transcript-view.ts +52 -13
- package/src/native-transcripts.ts +34 -15
- package/src/paths.ts +22 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +27 -7
- package/src/prune.ts +300 -0
- package/src/report.ts +63 -36
- package/src/resolved-case.ts +1 -1
- package/src/results-lock.ts +42 -0
- package/src/results.ts +14 -18
- package/src/run-owner.ts +4 -4
- package/src/{runtime-config.ts → runtime/config.ts} +182 -44
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +33 -22
- package/src/{runtime-paths.ts → runtime/paths.ts} +6 -18
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/runtime/shared-cache.ts +130 -0
- package/src/{runtime-state.ts → runtime/state.ts} +28 -11
- package/src/schema-types.ts +73 -161
- package/src/shell.ts +4 -0
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +22 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +0 -1
- package/docs/assets/sevro-wordmark.svg +0 -1
- package/docs/cache-ownership-validation.md +0 -49
- package/docs/claude-continuation-validation.md +0 -47
- package/docs/concurrency-validation.md +0 -86
- package/docs/configuration-root-validation.md +0 -70
- package/docs/engine-slice.md +0 -114
- package/docs/evidence/runtime-startup-30/README.md +0 -185
- package/docs/evidence/runtime-startup-30/after.json +0 -319
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
- package/docs/evidence/runtime-startup-30/before.json +0 -403
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
- package/docs/evidence.md +0 -17
- package/docs/fixture-cleanup-validation.md +0 -70
- package/docs/guide-verification.json +0 -239
- package/src/evaluation-context.ts +0 -257
- package/src/evaluation-fixture.ts +0 -313
- package/src/evaluation-run.ts +0 -198
- package/src/evaluation-trial.ts +0 -946
- package/src/evaluation-types.ts +0 -149
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -510
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -609
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { realpath } from "node:fs/promises";
|
|
2
|
-
import { isAbsolute
|
|
3
|
-
import {
|
|
2
|
+
import { isAbsolute } from "node:path";
|
|
3
|
+
import { insideRoot as inside } from "../paths";
|
|
4
|
+
export { insideRoot as inside } from "../paths";
|
|
5
|
+
import { insideRuntimeRoot } from "../runtime/paths";
|
|
6
|
+
import { workspaceBoundaryException } from "../fixtures/workspace-state";
|
|
7
|
+
import type { CandidateArtifactRoots } from "../candidate-artifacts";
|
|
4
8
|
|
|
5
9
|
export class HostIsolationError extends Error {
|
|
6
10
|
constructor(message: string) {
|
|
@@ -16,10 +20,13 @@ export interface IsolationOptions {
|
|
|
16
20
|
protectedRootsCanonical?: boolean;
|
|
17
21
|
privateStateRoot: string;
|
|
18
22
|
denyNetwork?: boolean;
|
|
23
|
+
localHttp?: boolean;
|
|
19
24
|
readOnlyRoots?: string[];
|
|
20
25
|
transcriptReadRoots?: string[];
|
|
26
|
+
candidateArtifactRoots?: CandidateArtifactRoots;
|
|
21
27
|
hookReadRoots?: string[];
|
|
22
28
|
writableRuntimeRoot?: string;
|
|
29
|
+
writableCacheRoots?: string[];
|
|
23
30
|
}
|
|
24
31
|
|
|
25
32
|
export interface IsolatedCommand {
|
|
@@ -27,14 +34,6 @@ export interface IsolatedCommand {
|
|
|
27
34
|
release(): Promise<void>;
|
|
28
35
|
}
|
|
29
36
|
|
|
30
|
-
export function inside(root: string, path: string): boolean {
|
|
31
|
-
const child = relative(root, path);
|
|
32
|
-
return (
|
|
33
|
-
child === "" ||
|
|
34
|
-
(child !== ".." && !child.startsWith(`..${sep}`) && !child.startsWith(sep))
|
|
35
|
-
);
|
|
36
|
-
}
|
|
37
|
-
|
|
38
37
|
export async function commandRuntimeRoot(
|
|
39
38
|
stateRoot: string,
|
|
40
39
|
path: string | undefined,
|
|
@@ -61,6 +60,7 @@ function absoluteIsolationPaths(options: IsolationOptions): boolean {
|
|
|
61
60
|
return (
|
|
62
61
|
isAbsolute(options.workspace) &&
|
|
63
62
|
isAbsolute(options.privateStateRoot) &&
|
|
63
|
+
(options.writableCacheRoots ?? []).every(isAbsolute) &&
|
|
64
64
|
options.protectedRoots.every(isAbsolute)
|
|
65
65
|
);
|
|
66
66
|
}
|
|
@@ -71,7 +71,11 @@ export function requireDisjointState(
|
|
|
71
71
|
stateRoot: string,
|
|
72
72
|
): void {
|
|
73
73
|
if (
|
|
74
|
-
roots.some(
|
|
74
|
+
roots.some(
|
|
75
|
+
(root) =>
|
|
76
|
+
inside(root, workspace) &&
|
|
77
|
+
!workspaceBoundaryException(root, workspace).length,
|
|
78
|
+
) ||
|
|
75
79
|
inside(stateRoot, workspace)
|
|
76
80
|
)
|
|
77
81
|
throw new HostIsolationError(
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { isMissingFile } from "../
|
|
2
|
-
import {
|
|
3
|
-
import { homedir,
|
|
4
|
-
import { isAbsolute,
|
|
1
|
+
import { isMissingFile } from "../fixtures/tools";
|
|
2
|
+
import { realpath } from "node:fs/promises";
|
|
3
|
+
import { homedir, userInfo } from "node:os";
|
|
4
|
+
import { isAbsolute, resolve } from "node:path";
|
|
5
5
|
import { protectedWorktrees } from "./protected-worktrees";
|
|
6
6
|
import { nativeStateParent } from "../native-transcript-state";
|
|
7
|
+
import { workspaceStateParent } from "../fixtures/workspace-state";
|
|
7
8
|
|
|
8
9
|
/** Include source, results, host configuration, and every active peer fixture. */
|
|
9
10
|
interface ProtectedRootsOptions {
|
|
@@ -21,6 +22,7 @@ async function protectedRootCandidates(options: ProtectedRootsOptions) {
|
|
|
21
22
|
].filter((value): value is string => Boolean(value));
|
|
22
23
|
const candidates = [
|
|
23
24
|
await nativeStateParent(),
|
|
25
|
+
await workspaceStateParent(),
|
|
24
26
|
resolve(import.meta.dir, "../.."),
|
|
25
27
|
options.projectRoot,
|
|
26
28
|
options.resultsRoot,
|
|
@@ -31,17 +33,7 @@ async function protectedRootCandidates(options: ProtectedRootsOptions) {
|
|
|
31
33
|
];
|
|
32
34
|
if (candidates.some((root) => !isAbsolute(root)))
|
|
33
35
|
throw new Error("evaluation protected roots must be absolute");
|
|
34
|
-
|
|
35
|
-
.filter(
|
|
36
|
-
(entry) =>
|
|
37
|
-
entry.isDirectory() &&
|
|
38
|
-
/^(sevro-case-|sevro-codex-state-|sevro-claude-state-|sevro-transcript-view-)/.test(
|
|
39
|
-
entry.name,
|
|
40
|
-
),
|
|
41
|
-
)
|
|
42
|
-
.map((entry) => join(tmpdir(), entry.name))
|
|
43
|
-
.filter((path) => path !== options.ownedStateRoot);
|
|
44
|
-
return { candidates, peers, optionalConfigRoots };
|
|
36
|
+
return { candidates, optionalConfigRoots };
|
|
45
37
|
}
|
|
46
38
|
|
|
47
39
|
/** Include source, results, host configuration, and every active peer fixture. */
|
|
@@ -49,20 +41,21 @@ export async function evaluationProtectedRoots(
|
|
|
49
41
|
options: ProtectedRootsOptions,
|
|
50
42
|
): Promise<string[]> {
|
|
51
43
|
const workspace = await realpath(options.workspace);
|
|
52
|
-
const { candidates,
|
|
44
|
+
const { candidates, optionalConfigRoots } =
|
|
53
45
|
await protectedRootCandidates(options);
|
|
46
|
+
const namespace = await workspaceStateParent();
|
|
47
|
+
const nativeNamespace = await nativeStateParent();
|
|
54
48
|
const roots: string[] = [];
|
|
55
|
-
for (const candidate of
|
|
49
|
+
for (const candidate of candidates) {
|
|
56
50
|
const canonical = await canonicalProtectedRoot(candidate, [
|
|
57
|
-
...peers,
|
|
58
51
|
...optionalConfigRoots,
|
|
59
52
|
]);
|
|
60
53
|
if (!includedProtectedRoot(canonical, workspace, options.ownedStateRoot))
|
|
61
54
|
continue;
|
|
62
55
|
{
|
|
63
56
|
roots.push(canonical);
|
|
64
|
-
//
|
|
65
|
-
if (
|
|
57
|
+
// Private namespaces are boundaries, not Git source trees to traverse.
|
|
58
|
+
if (canonical !== namespace && canonical !== nativeNamespace)
|
|
66
59
|
roots.push(
|
|
67
60
|
...(await protectedWorktrees(canonical)).filter(
|
|
68
61
|
(root) => root !== workspace,
|
|
@@ -6,7 +6,9 @@ import {
|
|
|
6
6
|
requireNativeTranscriptViews,
|
|
7
7
|
requireNativeHookRoots,
|
|
8
8
|
} from "../native-transcript-state";
|
|
9
|
-
import { requireRuntimeReadRoots } from "../runtime
|
|
9
|
+
import { requireRuntimeReadRoots } from "../runtime/paths";
|
|
10
|
+
import { requireWritableCacheRoots } from "../runtime/shared-cache";
|
|
11
|
+
import { requireCandidateArtifactRoots } from "../candidate-artifacts";
|
|
10
12
|
import {
|
|
11
13
|
commandRuntimeRoot,
|
|
12
14
|
HostIsolationError,
|
|
@@ -52,7 +54,8 @@ export async function prepareLinuxSandboxCommand(
|
|
|
52
54
|
): Promise<IsolatedCommand> {
|
|
53
55
|
requireLinuxSandbox();
|
|
54
56
|
await requireNativeTranscriptViews(options.transcriptReadRoots);
|
|
55
|
-
await
|
|
57
|
+
await requireCandidateArtifactRoots(options.candidateArtifactRoots);
|
|
58
|
+
await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
|
|
56
59
|
requireIsolationPaths(options);
|
|
57
60
|
const workspace = await realpath(options.workspace);
|
|
58
61
|
const roots = options.protectedRootsCanonical
|
|
@@ -65,6 +68,10 @@ export async function prepareLinuxSandboxCommand(
|
|
|
65
68
|
options.writableRuntimeRoot,
|
|
66
69
|
);
|
|
67
70
|
const readRoots = options.readOnlyRoots ?? [];
|
|
71
|
+
const writableCacheRoots = await cacheMountRoots(options, [
|
|
72
|
+
...roots,
|
|
73
|
+
stateRoot,
|
|
74
|
+
]);
|
|
68
75
|
requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
|
|
69
76
|
requireDisjointState(workspace, roots, stateRoot);
|
|
70
77
|
const blockedFile = resolve(stateRoot, `blocked-${randomUUID()}`);
|
|
@@ -95,14 +102,16 @@ export async function prepareLinuxSandboxCommand(
|
|
|
95
102
|
"--bind",
|
|
96
103
|
"/tmp",
|
|
97
104
|
"/tmp",
|
|
105
|
+
...masks,
|
|
98
106
|
"--bind",
|
|
99
107
|
workspace,
|
|
100
108
|
workspace,
|
|
101
|
-
...masks,
|
|
102
109
|
...readRoots.flatMap((path) => ["--ro-bind", path, path]),
|
|
103
110
|
...transcriptMounts(options),
|
|
111
|
+
...candidateArtifactMounts(options),
|
|
104
112
|
...hookMounts(options),
|
|
105
113
|
...(ownedRoot ? ["--bind", ownedRoot, ownedRoot] : []),
|
|
114
|
+
...writableCacheRoots.flatMap((path) => ["--bind", path, path]),
|
|
106
115
|
"--chdir",
|
|
107
116
|
workspace,
|
|
108
117
|
"--",
|
|
@@ -118,6 +127,13 @@ export async function prepareLinuxSandboxCommand(
|
|
|
118
127
|
}
|
|
119
128
|
}
|
|
120
129
|
|
|
130
|
+
function cacheMountRoots(options: IsolationOptions, protectedRoots: string[]) {
|
|
131
|
+
return requireWritableCacheRoots(
|
|
132
|
+
options.writableCacheRoots ?? [],
|
|
133
|
+
protectedRoots,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
121
137
|
function transcriptMounts(options: IsolationOptions): string[] {
|
|
122
138
|
return (options.transcriptReadRoots ?? []).flatMap((path) => [
|
|
123
139
|
"--ro-bind",
|
|
@@ -126,6 +142,14 @@ function transcriptMounts(options: IsolationOptions): string[] {
|
|
|
126
142
|
]);
|
|
127
143
|
}
|
|
128
144
|
|
|
145
|
+
function candidateArtifactMounts(options: IsolationOptions): string[] {
|
|
146
|
+
return Object.values(options.candidateArtifactRoots ?? {}).flatMap((path) => [
|
|
147
|
+
"--ro-bind",
|
|
148
|
+
path,
|
|
149
|
+
path,
|
|
150
|
+
]);
|
|
151
|
+
}
|
|
152
|
+
|
|
129
153
|
function hookMounts(options: IsolationOptions): string[] {
|
|
130
154
|
return (options.hookReadRoots ?? []).flatMap((path) => [
|
|
131
155
|
"--ro-bind",
|
package/src/hosts/mac-sandbox.ts
CHANGED
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { containsControl } from "../value-guards";
|
|
2
3
|
import { existsSync } from "node:fs";
|
|
3
4
|
import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
|
|
4
|
-
import { resolve } from "node:path";
|
|
5
|
-
import {
|
|
5
|
+
import { dirname, resolve } from "node:path";
|
|
6
|
+
import {
|
|
7
|
+
requireNativeTranscriptViews,
|
|
8
|
+
requireNativeHookRoots,
|
|
9
|
+
} from "../native-transcript-state";
|
|
10
|
+
import { requireCandidateArtifactRoots } from "../candidate-artifacts";
|
|
11
|
+
import { workspaceBoundaryException } from "../fixtures/workspace-state";
|
|
12
|
+
import { requireWritableCacheRoots } from "../runtime/shared-cache";
|
|
6
13
|
import {
|
|
7
14
|
insideRuntimeRoot,
|
|
8
15
|
isRuntimeHomeRoot,
|
|
9
16
|
requireRuntimeReadRoots,
|
|
10
|
-
} from "../runtime
|
|
17
|
+
} from "../runtime/paths";
|
|
11
18
|
import {
|
|
12
19
|
commandRuntimeRoot,
|
|
13
20
|
HostIsolationError,
|
|
@@ -23,61 +30,119 @@ export type { IsolatedCommand } from "./isolation-common";
|
|
|
23
30
|
const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
|
|
24
31
|
|
|
25
32
|
function quoted(path: string): string {
|
|
26
|
-
if (
|
|
27
|
-
Array.from(path).some(
|
|
28
|
-
(character) =>
|
|
29
|
-
character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
|
|
30
|
-
)
|
|
31
|
-
)
|
|
33
|
+
if (containsControl(path))
|
|
32
34
|
throw new HostIsolationError("sandbox path contains a control character");
|
|
33
35
|
return path.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
|
|
34
36
|
}
|
|
35
37
|
|
|
36
|
-
/**
|
|
38
|
+
/** Protect canonical roots while preserving explicitly granted access. */
|
|
37
39
|
export function macSandboxProfile(
|
|
38
40
|
deniedRoots: string[],
|
|
39
41
|
denyNetwork = false,
|
|
40
42
|
readOnlyRoots: string[] = [],
|
|
41
43
|
writableRuntimeRoot?: string,
|
|
42
44
|
transcriptReadRoots: string[] = [],
|
|
45
|
+
workspace?: string,
|
|
46
|
+
localHttp?: boolean,
|
|
47
|
+
writableCacheRoots?: string[],
|
|
43
48
|
): string {
|
|
44
49
|
requireProfileRoots(deniedRoots);
|
|
45
50
|
return [
|
|
46
51
|
"(version 1)",
|
|
47
52
|
"(allow default)",
|
|
48
53
|
...(denyNetwork ? ["(deny network*)"] : []),
|
|
54
|
+
...localHttpRules(localHttp),
|
|
49
55
|
...[...new Set(deniedRoots)]
|
|
50
56
|
.sort()
|
|
51
57
|
.flatMap((path) => [
|
|
52
|
-
`(deny file-read* ${
|
|
53
|
-
`(deny file-write* ${rootFilter(path,
|
|
58
|
+
`(deny file-read* ${workspaceReadFilter(path, readExceptions(path, readOnlyRoots, writableRuntimeRoot, transcriptReadRoots, workspace, writableCacheRoots), workspace)})`,
|
|
59
|
+
`(deny file-write* ${rootFilter(path, [...writableExceptions(path, writableRuntimeRoot, writableCacheRoots), ...workspaceBoundaryException(path, workspace)])})`,
|
|
54
60
|
]),
|
|
61
|
+
...deniedRoots.flatMap((path) =>
|
|
62
|
+
workspaceBoundaryException(path, workspace).map(
|
|
63
|
+
() => `(deny file-read-data (literal "${quoted(path)}"))`,
|
|
64
|
+
),
|
|
65
|
+
),
|
|
55
66
|
...[...readOnlyRoots, ...transcriptReadRoots].map(
|
|
56
67
|
(path) => `(deny file-write* (subpath "${quoted(path)}"))`,
|
|
57
68
|
),
|
|
69
|
+
...deniedRoots.flatMap((root) =>
|
|
70
|
+
ancestorMetadataRules(
|
|
71
|
+
root,
|
|
72
|
+
readExceptions(
|
|
73
|
+
root,
|
|
74
|
+
readOnlyRoots,
|
|
75
|
+
writableRuntimeRoot,
|
|
76
|
+
transcriptReadRoots,
|
|
77
|
+
workspace,
|
|
78
|
+
writableCacheRoots,
|
|
79
|
+
),
|
|
80
|
+
),
|
|
81
|
+
),
|
|
58
82
|
"",
|
|
59
83
|
].join("\n");
|
|
60
84
|
}
|
|
61
85
|
|
|
86
|
+
function localHttpRules(enabled: boolean | undefined): string[] {
|
|
87
|
+
return enabled
|
|
88
|
+
? [
|
|
89
|
+
'(allow network-bind (local ip "*:*"))',
|
|
90
|
+
'(allow network-inbound (local ip "localhost:*"))',
|
|
91
|
+
'(allow network-outbound (remote ip "localhost:*"))',
|
|
92
|
+
]
|
|
93
|
+
: [];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function ancestorMetadataRules(root: string, readRoots: string[]): string[] {
|
|
97
|
+
const ancestors = new Set<string>();
|
|
98
|
+
for (const readRoot of readRoots) {
|
|
99
|
+
for (
|
|
100
|
+
let path = dirname(readRoot);
|
|
101
|
+
insideRuntimeRoot(root, path);
|
|
102
|
+
path = dirname(path)
|
|
103
|
+
) {
|
|
104
|
+
ancestors.add(path);
|
|
105
|
+
if (path === resolve(root)) break;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return [...ancestors]
|
|
109
|
+
.sort()
|
|
110
|
+
.map((path) => `(allow file-read-metadata (literal "${quoted(path)}"))`);
|
|
111
|
+
}
|
|
112
|
+
|
|
62
113
|
function requireProfileRoots(roots: string[]): void {
|
|
63
114
|
if (!roots.length || roots.some((path) => !path.startsWith("/")))
|
|
64
115
|
throw new HostIsolationError("sandbox roots must be absolute and nonempty");
|
|
65
116
|
}
|
|
66
117
|
|
|
67
|
-
function
|
|
118
|
+
function readExceptions(
|
|
68
119
|
root: string,
|
|
69
120
|
readRoots: string[],
|
|
70
121
|
writableRuntimeRoot: string | undefined,
|
|
71
122
|
transcriptReadRoots: string[],
|
|
72
|
-
|
|
123
|
+
workspace?: string,
|
|
124
|
+
writableCacheRoots: string[] = [],
|
|
125
|
+
): string[] {
|
|
73
126
|
const exceptions = isRuntimeHomeRoot(root)
|
|
74
127
|
? readRoots.filter((path) => insideRuntimeRoot(root, path))
|
|
75
128
|
: [];
|
|
76
|
-
return
|
|
129
|
+
return [
|
|
77
130
|
...transcriptReadRoots.filter((path) => insideRuntimeRoot(root, path)),
|
|
78
131
|
...exceptions,
|
|
79
|
-
...
|
|
80
|
-
|
|
132
|
+
...writableExceptions(root, writableRuntimeRoot, writableCacheRoots),
|
|
133
|
+
...workspaceBoundaryException(root, workspace),
|
|
134
|
+
];
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function writableExceptions(
|
|
138
|
+
root: string,
|
|
139
|
+
runtime: string | undefined,
|
|
140
|
+
caches: string[] = [],
|
|
141
|
+
): string[] {
|
|
142
|
+
return [
|
|
143
|
+
...ownedException(root, runtime),
|
|
144
|
+
...caches.flatMap((cache) => ownedException(root, cache)),
|
|
145
|
+
];
|
|
81
146
|
}
|
|
82
147
|
|
|
83
148
|
function rootFilter(root: string, exceptions: string[]): string {
|
|
@@ -86,6 +151,17 @@ function rootFilter(root: string, exceptions: string[]): string {
|
|
|
86
151
|
return `(require-all ${base} ${exceptions.map((path) => `(require-not (subpath "${quoted(path)}"))`).join(" ")})`;
|
|
87
152
|
}
|
|
88
153
|
|
|
154
|
+
function workspaceReadFilter(
|
|
155
|
+
root: string,
|
|
156
|
+
exceptions: string[],
|
|
157
|
+
workspace?: string,
|
|
158
|
+
): string {
|
|
159
|
+
const filter = rootFilter(root, exceptions);
|
|
160
|
+
return workspaceBoundaryException(root, workspace).length
|
|
161
|
+
? `(require-all ${filter} (require-not (literal "${quoted(root)}")))`
|
|
162
|
+
: filter;
|
|
163
|
+
}
|
|
164
|
+
|
|
89
165
|
function ownedException(root: string, ownedRoot: string | undefined): string[] {
|
|
90
166
|
if (ownedRoot && insideRuntimeRoot(root, ownedRoot)) return [ownedRoot];
|
|
91
167
|
return [];
|
|
@@ -97,6 +173,8 @@ export async function prepareMacSandboxCommand(
|
|
|
97
173
|
): Promise<IsolatedCommand> {
|
|
98
174
|
requireSandboxAvailable();
|
|
99
175
|
await requireNativeTranscriptViews(options.transcriptReadRoots);
|
|
176
|
+
await requireCandidateArtifactRoots(options.candidateArtifactRoots);
|
|
177
|
+
await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
|
|
100
178
|
requireIsolationPaths(options);
|
|
101
179
|
const workspace = await realpath(options.workspace);
|
|
102
180
|
const roots = options.protectedRootsCanonical
|
|
@@ -109,6 +187,10 @@ export async function prepareMacSandboxCommand(
|
|
|
109
187
|
options.writableRuntimeRoot,
|
|
110
188
|
);
|
|
111
189
|
const readRoots = options.readOnlyRoots ?? [];
|
|
190
|
+
const writableCacheRoots = await requireWritableCacheRoots(
|
|
191
|
+
options.writableCacheRoots ?? [],
|
|
192
|
+
[...roots, stateRoot],
|
|
193
|
+
);
|
|
112
194
|
requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
|
|
113
195
|
requireDisjointState(workspace, roots, stateRoot);
|
|
114
196
|
const profilePath = resolve(stateRoot, `host-${randomUUID()}.sb`);
|
|
@@ -119,7 +201,10 @@ export async function prepareMacSandboxCommand(
|
|
|
119
201
|
options.denyNetwork,
|
|
120
202
|
readRoots,
|
|
121
203
|
ownedRoot,
|
|
122
|
-
options
|
|
204
|
+
curatedReadRoots(options),
|
|
205
|
+
workspace,
|
|
206
|
+
options.localHttp,
|
|
207
|
+
writableCacheRoots,
|
|
123
208
|
),
|
|
124
209
|
{ flag: "wx", mode: 0o600 },
|
|
125
210
|
);
|
|
@@ -131,6 +216,14 @@ export async function prepareMacSandboxCommand(
|
|
|
131
216
|
};
|
|
132
217
|
}
|
|
133
218
|
|
|
219
|
+
function curatedReadRoots(options: IsolationOptions): string[] {
|
|
220
|
+
return [
|
|
221
|
+
...(options.transcriptReadRoots ?? []),
|
|
222
|
+
...(options.hookReadRoots ?? []),
|
|
223
|
+
...Object.values(options.candidateArtifactRoots ?? {}),
|
|
224
|
+
];
|
|
225
|
+
}
|
|
226
|
+
|
|
134
227
|
function requireSandboxAvailable(): void {
|
|
135
228
|
if (process.platform !== "darwin" || !existsSync(SANDBOX_EXEC))
|
|
136
229
|
throw new HostIsolationError("macOS sandbox-exec isolation is unavailable");
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { isRecord, isUnknownArray } from "../value-guards";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
summarizeClaudeEvents,
|
|
4
|
+
parseClaudeEvents,
|
|
5
|
+
type ClaudeEvents,
|
|
6
|
+
} from "./claude-events";
|
|
3
7
|
import type { codexNativeCallObservation } from "./codex-native-calls";
|
|
8
|
+
import type { ExecCallReceipt } from "./codex-exec-controls";
|
|
4
9
|
|
|
5
10
|
const MAX_CALLS = 128;
|
|
6
11
|
type Call = { ordinal: number; namespace: string; name: string };
|
|
@@ -13,7 +18,14 @@ export interface NativeControlObservation {
|
|
|
13
18
|
method: "native_control_calls";
|
|
14
19
|
calls: Call[];
|
|
15
20
|
acceptedAgentCount: number | null;
|
|
21
|
+
directAcceptedAgentCount?: number;
|
|
22
|
+
coverage?: {
|
|
23
|
+
nativeCalls: NativeControlObservation["completeness"];
|
|
24
|
+
execInspection: "complete" | "partial";
|
|
25
|
+
acceptedAgents: "direct_native_calls";
|
|
26
|
+
};
|
|
16
27
|
submittedExecCalls: number | null;
|
|
28
|
+
execCalls?: ExecCallReceipt[];
|
|
17
29
|
truncated: boolean | null;
|
|
18
30
|
};
|
|
19
31
|
}
|
|
@@ -30,10 +42,11 @@ function label(value: unknown, max: number): value is string {
|
|
|
30
42
|
export function codexNativeControls(
|
|
31
43
|
observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
|
|
32
44
|
): NativeControlObservation {
|
|
33
|
-
const complete =
|
|
45
|
+
const complete =
|
|
46
|
+
observation.completeness === "complete" && inspectedExecCalls(observation);
|
|
34
47
|
return {
|
|
35
48
|
id: "sevro.host.native-controls",
|
|
36
|
-
completeness: observation.completeness,
|
|
49
|
+
completeness: controlCompleteness(observation.completeness, complete),
|
|
37
50
|
data: {
|
|
38
51
|
method: "native_control_calls",
|
|
39
52
|
calls: observation.data.toolCalls.map(({ ordinal, namespace, name }) => ({
|
|
@@ -44,22 +57,59 @@ export function codexNativeControls(
|
|
|
44
57
|
acceptedAgentCount: complete
|
|
45
58
|
? observation.data.acceptedSpawns.length
|
|
46
59
|
: null,
|
|
47
|
-
|
|
48
|
-
|
|
60
|
+
directAcceptedAgentCount: observation.data.acceptedSpawns.length,
|
|
61
|
+
coverage: {
|
|
62
|
+
nativeCalls: observation.completeness,
|
|
63
|
+
execInspection: inspectedExecCalls(observation)
|
|
64
|
+
? "complete"
|
|
65
|
+
: "partial",
|
|
66
|
+
acceptedAgents: "direct_native_calls",
|
|
67
|
+
},
|
|
68
|
+
...nativeCounts(observation),
|
|
69
|
+
...execEvidence(observation.data.execCalls),
|
|
49
70
|
},
|
|
50
71
|
};
|
|
51
72
|
}
|
|
73
|
+
function nativeCounts(
|
|
74
|
+
observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
|
|
75
|
+
) {
|
|
76
|
+
const complete = observation.completeness === "complete";
|
|
77
|
+
return {
|
|
78
|
+
submittedExecCalls: complete ? observation.data.submittedExecCalls : null,
|
|
79
|
+
truncated: complete ? false : null,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function controlCompleteness(
|
|
83
|
+
source: NativeControlObservation["completeness"],
|
|
84
|
+
complete: boolean,
|
|
85
|
+
): NativeControlObservation["completeness"] {
|
|
86
|
+
return source === "complete" && !complete ? "partial" : source;
|
|
87
|
+
}
|
|
88
|
+
function execEvidence(receipts: ExecCallReceipt[] | undefined) {
|
|
89
|
+
return receipts ? { execCalls: receipts } : {};
|
|
90
|
+
}
|
|
52
91
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
92
|
+
function inspectedExecCalls(
|
|
93
|
+
observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
|
|
94
|
+
): boolean {
|
|
95
|
+
const receipts = observation.data.execCalls ?? [];
|
|
96
|
+
const submissions = observation.data.toolCalls.filter(
|
|
97
|
+
(call) => call.name === "exec",
|
|
98
|
+
);
|
|
99
|
+
if (
|
|
100
|
+
submissions.length !== observation.data.submittedExecCalls ||
|
|
101
|
+
receipts.length !== submissions.length
|
|
102
|
+
)
|
|
103
|
+
return false;
|
|
104
|
+
return receipts.every(
|
|
105
|
+
(receipt, index) =>
|
|
106
|
+
receipt.completeness === "complete" &&
|
|
107
|
+
receipt.ordinal === submissions[index]?.ordinal,
|
|
108
|
+
);
|
|
61
109
|
}
|
|
62
110
|
|
|
111
|
+
const MALFORMED = Symbol("malformed native control event");
|
|
112
|
+
|
|
63
113
|
function assistantBlocks(event: unknown): unknown[] | null | typeof MALFORMED {
|
|
64
114
|
if (event === MALFORMED) return MALFORMED;
|
|
65
115
|
if (!assistantEvent(event)) return null;
|
|
@@ -87,15 +137,14 @@ class ClaudeNativeControls {
|
|
|
87
137
|
malformed = false;
|
|
88
138
|
truncated = false;
|
|
89
139
|
|
|
90
|
-
parse(stream:
|
|
91
|
-
for (const line of stream.
|
|
92
|
-
if (!line.
|
|
93
|
-
if (!this.observeLine(line)) break;
|
|
140
|
+
parse(stream: ClaudeEvents): void {
|
|
141
|
+
for (const line of stream.lines) {
|
|
142
|
+
if (!this.observeEvent(line.parsed ? line.value : MALFORMED)) break;
|
|
94
143
|
}
|
|
95
144
|
}
|
|
96
145
|
|
|
97
|
-
private
|
|
98
|
-
const blocks = assistantBlocks(
|
|
146
|
+
private observeEvent(event: unknown): boolean {
|
|
147
|
+
const blocks = assistantBlocks(event);
|
|
99
148
|
if (blocks === null) return true;
|
|
100
149
|
if (blocks === MALFORMED) {
|
|
101
150
|
this.malformed = true;
|
|
@@ -133,12 +182,13 @@ class ClaudeNativeControls {
|
|
|
133
182
|
|
|
134
183
|
/** Observe all native labels without retaining arguments or interpreting policy. */
|
|
135
184
|
export function claudeNativeControls(
|
|
136
|
-
stream: string,
|
|
185
|
+
stream: string | ClaudeEvents,
|
|
137
186
|
exitCode: number,
|
|
138
187
|
): NativeControlObservation {
|
|
139
|
-
const
|
|
188
|
+
const events = parseClaudeEvents(stream);
|
|
189
|
+
const summary = summarizeClaudeEvents(events, exitCode);
|
|
140
190
|
const controls = new ClaudeNativeControls();
|
|
141
|
-
controls.parse(
|
|
191
|
+
controls.parse(events);
|
|
142
192
|
return {
|
|
143
193
|
id: "sevro.host.native-controls",
|
|
144
194
|
completeness:
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import type { HostAdapter } from "../engine";
|
|
3
|
+
import {
|
|
4
|
+
requireCandidateRuntime,
|
|
5
|
+
requireCandidateArtifactRoots,
|
|
6
|
+
} from "../candidate-artifacts";
|
|
7
|
+
import { requireNativeTranscriptView } from "../native-transcript-state";
|
|
8
|
+
import { isOptionalNonblankString } from "../value-guards";
|
|
9
|
+
|
|
10
|
+
export const MAX_HOST_TIMEOUT_MS = 30 * 60_000;
|
|
11
|
+
|
|
12
|
+
export function validHostTimeout(timeoutMs: number): boolean {
|
|
13
|
+
return (
|
|
14
|
+
Number.isSafeInteger(timeoutMs) &&
|
|
15
|
+
timeoutMs >= 1 &&
|
|
16
|
+
timeoutMs <= MAX_HOST_TIMEOUT_MS
|
|
17
|
+
);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Common admission checks run before native state or processes are allocated. */
|
|
21
|
+
export async function validateNativeHostRequest(
|
|
22
|
+
request: Parameters<HostAdapter["run"]>[0],
|
|
23
|
+
host: "Codex" | "Claude",
|
|
24
|
+
): Promise<void> {
|
|
25
|
+
requireFollowUp(request.followUpPrompt, host);
|
|
26
|
+
await requireNativeTranscriptView(request.candidateTranscriptRoot);
|
|
27
|
+
await requireCandidateRuntime(
|
|
28
|
+
request.candidateRuntimeRoot,
|
|
29
|
+
request.runtimeRole,
|
|
30
|
+
);
|
|
31
|
+
await requireCandidateArtifactRoots(request.candidateArtifactRoots);
|
|
32
|
+
requireFixtureBin(request);
|
|
33
|
+
requirePassiveRequest(request, host);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function requireFollowUp(prompt: string | undefined, host: string): void {
|
|
37
|
+
if (!isOptionalNonblankString(prompt))
|
|
38
|
+
throw new Error(`${host} follow-up prompt must be nonempty`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function requireFixtureBin(request: Parameters<HostAdapter["run"]>[0]): void {
|
|
42
|
+
if (
|
|
43
|
+
request.fixtureBinDir !== undefined &&
|
|
44
|
+
request.fixtureBinDir !== join(request.workspace, ".git", "fixture-bin")
|
|
45
|
+
)
|
|
46
|
+
throw new Error("fixture binary path is outside the workspace");
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function requirePassiveRequest(
|
|
50
|
+
request: Parameters<HostAdapter["run"]>[0],
|
|
51
|
+
host: string,
|
|
52
|
+
): void {
|
|
53
|
+
if (request.instrumentation?.length || request.condition !== "passive")
|
|
54
|
+
throw new Error(`${host} enforcement instrumentation is unavailable`);
|
|
55
|
+
}
|