@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +22 -6
- package/README.md +5 -1
- package/docs/README.md +19 -15
- package/docs/architecture.md +138 -17
- package/docs/assets/README.md +5 -4
- package/docs/builtin-graders-v1.md +9 -2
- package/docs/codex-host.md +155 -57
- package/docs/contracts.md +2 -1
- package/docs/creating-cases.md +9 -2
- package/docs/development-cli.md +187 -19
- package/docs/documentation-quality.md +12 -2
- package/docs/evaluation-workflow.md +110 -0
- package/docs/extension-protocol-v1.md +26 -3
- package/docs/getting-started.md +23 -2
- package/docs/guide-evaluation.md +33 -1
- package/docs/identity-v1.md +25 -2
- package/docs/installing.md +11 -11
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +196 -7
- package/docs/reading-results.md +102 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +31 -12
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +122 -4
- package/docs/run-settings-v1.md +147 -0
- package/docs/runtime-v1.md +255 -22
- package/docs/troubleshooting.md +12 -12
- package/docs/typescript-quality.md +89 -26
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +11 -5
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +43 -7
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +42 -11
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +73 -50
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +87 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +60 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +198 -0
- package/src/cli/hosts.ts +55 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +154 -0
- package/src/cli/presentation.ts +93 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +128 -0
- package/src/cli/run-options.ts +403 -0
- package/src/cli/run-overrides.ts +69 -0
- package/src/cli/settings.ts +154 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -591
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +9 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +235 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
- package/src/evaluation/trial.ts +800 -0
- package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
- package/src/evidence/host-results.ts +85 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +34 -96
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +183 -0
- package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
- package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
- package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
- package/src/generated/cli-result.d.cts +2 -1
- package/src/generated/extension.d.cts +2 -1
- package/src/generated/report.cjs +1 -1
- package/src/generated/report.d.cts +2 -1
- package/src/generated/run-evidence.cjs +1 -1
- package/src/generated/run-evidence.d.cts +2 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/generated/runtime.d.cts +2 -1
- package/src/git.ts +93 -0
- package/src/graders/git-head.ts +29 -84
- package/src/graders/shell.ts +50 -68
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +331 -0
- package/src/hosts/claude-continuation.ts +57 -40
- package/src/hosts/claude-credential.ts +160 -63
- package/src/hosts/claude-events.ts +34 -18
- package/src/hosts/claude-keychain.ts +78 -21
- package/src/hosts/claude-login-recovery.ts +85 -0
- package/src/hosts/claude-login-refresh.ts +178 -0
- package/src/hosts/claude-login-store.ts +162 -0
- package/src/hosts/claude-login.ts +107 -0
- package/src/hosts/claude-native-goal.ts +15 -12
- package/src/hosts/claude-process.ts +10 -50
- package/src/hosts/claude-refresh-lock.ts +81 -0
- package/src/hosts/claude-repository-invocation.ts +9 -34
- package/src/hosts/claude-settings.ts +46 -9
- package/src/hosts/claude-skill-activations.ts +169 -0
- package/src/hosts/claude-state.ts +28 -15
- package/src/hosts/claude-tool-calls.ts +13 -10
- package/src/hosts/claude.ts +121 -81
- package/src/hosts/codex-app-server.ts +78 -52
- package/src/hosts/codex-configuration.ts +5 -10
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-marketplace.ts +12 -10
- package/src/hosts/codex-native-calls.ts +15 -122
- package/src/hosts/codex-process.ts +10 -183
- package/src/hosts/codex-profile.ts +36 -21
- package/src/hosts/codex-session-files.ts +1 -9
- package/src/hosts/codex-state.ts +134 -18
- package/src/hosts/codex.ts +210 -183
- package/src/hosts/isolation-common.ts +8 -11
- package/src/hosts/isolation-roots.ts +2 -2
- package/src/hosts/linux-sandbox.ts +15 -2
- package/src/hosts/mac-sandbox.ts +87 -18
- package/src/hosts/native-controls.ts +72 -22
- package/src/hosts/native-request.ts +55 -0
- package/src/hosts/process-lifecycle.ts +216 -48
- package/src/hosts/protected-worktrees.ts +1 -1
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +577 -0
- package/src/instrumentation.ts +7 -8
- package/src/native-transcript-state.ts +22 -3
- package/src/native-transcript-view.ts +52 -13
- package/src/native-transcripts.ts +26 -11
- package/src/paths.ts +22 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +27 -7
- package/src/prune.ts +300 -0
- package/src/report.ts +63 -36
- package/src/resolved-case.ts +1 -1
- package/src/results-lock.ts +42 -0
- package/src/results.ts +14 -18
- package/src/{runtime-config.ts → runtime/config.ts} +160 -46
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +32 -22
- package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/runtime/shared-cache.ts +130 -0
- package/src/{runtime-state.ts → runtime/state.ts} +28 -11
- package/src/schema-types.ts +73 -161
- package/src/shell.ts +4 -0
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +22 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +0 -1
- package/docs/assets/sevro-wordmark.svg +0 -1
- package/docs/cache-ownership-validation.md +0 -49
- package/docs/claude-continuation-validation.md +0 -47
- package/docs/concurrency-validation.md +0 -86
- package/docs/configuration-root-validation.md +0 -70
- package/docs/engine-slice.md +0 -114
- package/docs/evidence/runtime-startup-30/README.md +0 -185
- package/docs/evidence/runtime-startup-30/after.json +0 -319
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
- package/docs/evidence/runtime-startup-30/before.json +0 -403
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
- package/docs/evidence.md +0 -17
- package/docs/fixture-cleanup-validation.md +0 -73
- package/docs/guide-verification.json +0 -239
- package/src/evaluation-context.ts +0 -259
- package/src/evaluation-fixture.ts +0 -302
- package/src/evaluation-run.ts +0 -200
- package/src/evaluation-trial.ts +0 -1016
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -510
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -783
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { realpath } from "node:fs/promises";
|
|
2
|
-
import { isAbsolute
|
|
3
|
-
import {
|
|
4
|
-
|
|
2
|
+
import { isAbsolute } from "node:path";
|
|
3
|
+
import { insideRoot as inside } from "../paths";
|
|
4
|
+
export { insideRoot as inside } from "../paths";
|
|
5
|
+
import { insideRuntimeRoot } from "../runtime/paths";
|
|
6
|
+
import { workspaceBoundaryException } from "../fixtures/workspace-state";
|
|
5
7
|
import type { CandidateArtifactRoots } from "../candidate-artifacts";
|
|
6
8
|
|
|
7
9
|
export class HostIsolationError extends Error {
|
|
@@ -18,11 +20,13 @@ export interface IsolationOptions {
|
|
|
18
20
|
protectedRootsCanonical?: boolean;
|
|
19
21
|
privateStateRoot: string;
|
|
20
22
|
denyNetwork?: boolean;
|
|
23
|
+
localHttp?: boolean;
|
|
21
24
|
readOnlyRoots?: string[];
|
|
22
25
|
transcriptReadRoots?: string[];
|
|
23
26
|
candidateArtifactRoots?: CandidateArtifactRoots;
|
|
24
27
|
hookReadRoots?: string[];
|
|
25
28
|
writableRuntimeRoot?: string;
|
|
29
|
+
writableCacheRoots?: string[];
|
|
26
30
|
}
|
|
27
31
|
|
|
28
32
|
export interface IsolatedCommand {
|
|
@@ -30,14 +34,6 @@ export interface IsolatedCommand {
|
|
|
30
34
|
release(): Promise<void>;
|
|
31
35
|
}
|
|
32
36
|
|
|
33
|
-
export function inside(root: string, path: string): boolean {
|
|
34
|
-
const child = relative(root, path);
|
|
35
|
-
return (
|
|
36
|
-
child === "" ||
|
|
37
|
-
(child !== ".." && !child.startsWith(`..${sep}`) && !child.startsWith(sep))
|
|
38
|
-
);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
37
|
export async function commandRuntimeRoot(
|
|
42
38
|
stateRoot: string,
|
|
43
39
|
path: string | undefined,
|
|
@@ -64,6 +60,7 @@ function absoluteIsolationPaths(options: IsolationOptions): boolean {
|
|
|
64
60
|
return (
|
|
65
61
|
isAbsolute(options.workspace) &&
|
|
66
62
|
isAbsolute(options.privateStateRoot) &&
|
|
63
|
+
(options.writableCacheRoots ?? []).every(isAbsolute) &&
|
|
67
64
|
options.protectedRoots.every(isAbsolute)
|
|
68
65
|
);
|
|
69
66
|
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { isMissingFile } from "../
|
|
1
|
+
import { isMissingFile } from "../fixtures/tools";
|
|
2
2
|
import { realpath } from "node:fs/promises";
|
|
3
3
|
import { homedir, userInfo } from "node:os";
|
|
4
4
|
import { isAbsolute, resolve } from "node:path";
|
|
5
5
|
import { protectedWorktrees } from "./protected-worktrees";
|
|
6
6
|
import { nativeStateParent } from "../native-transcript-state";
|
|
7
|
-
import { workspaceStateParent } from "../workspace-state";
|
|
7
|
+
import { workspaceStateParent } from "../fixtures/workspace-state";
|
|
8
8
|
|
|
9
9
|
/** Include source, results, host configuration, and every active peer fixture. */
|
|
10
10
|
interface ProtectedRootsOptions {
|
|
@@ -6,7 +6,8 @@ import {
|
|
|
6
6
|
requireNativeTranscriptViews,
|
|
7
7
|
requireNativeHookRoots,
|
|
8
8
|
} from "../native-transcript-state";
|
|
9
|
-
import { requireRuntimeReadRoots } from "../runtime
|
|
9
|
+
import { requireRuntimeReadRoots } from "../runtime/paths";
|
|
10
|
+
import { requireWritableCacheRoots } from "../runtime/shared-cache";
|
|
10
11
|
import { requireCandidateArtifactRoots } from "../candidate-artifacts";
|
|
11
12
|
import {
|
|
12
13
|
commandRuntimeRoot,
|
|
@@ -54,7 +55,7 @@ export async function prepareLinuxSandboxCommand(
|
|
|
54
55
|
requireLinuxSandbox();
|
|
55
56
|
await requireNativeTranscriptViews(options.transcriptReadRoots);
|
|
56
57
|
await requireCandidateArtifactRoots(options.candidateArtifactRoots);
|
|
57
|
-
await requireNativeHookRoots(options.hookReadRoots);
|
|
58
|
+
await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
|
|
58
59
|
requireIsolationPaths(options);
|
|
59
60
|
const workspace = await realpath(options.workspace);
|
|
60
61
|
const roots = options.protectedRootsCanonical
|
|
@@ -67,6 +68,10 @@ export async function prepareLinuxSandboxCommand(
|
|
|
67
68
|
options.writableRuntimeRoot,
|
|
68
69
|
);
|
|
69
70
|
const readRoots = options.readOnlyRoots ?? [];
|
|
71
|
+
const writableCacheRoots = await cacheMountRoots(options, [
|
|
72
|
+
...roots,
|
|
73
|
+
stateRoot,
|
|
74
|
+
]);
|
|
70
75
|
requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
|
|
71
76
|
requireDisjointState(workspace, roots, stateRoot);
|
|
72
77
|
const blockedFile = resolve(stateRoot, `blocked-${randomUUID()}`);
|
|
@@ -106,6 +111,7 @@ export async function prepareLinuxSandboxCommand(
|
|
|
106
111
|
...candidateArtifactMounts(options),
|
|
107
112
|
...hookMounts(options),
|
|
108
113
|
...(ownedRoot ? ["--bind", ownedRoot, ownedRoot] : []),
|
|
114
|
+
...writableCacheRoots.flatMap((path) => ["--bind", path, path]),
|
|
109
115
|
"--chdir",
|
|
110
116
|
workspace,
|
|
111
117
|
"--",
|
|
@@ -121,6 +127,13 @@ export async function prepareLinuxSandboxCommand(
|
|
|
121
127
|
}
|
|
122
128
|
}
|
|
123
129
|
|
|
130
|
+
function cacheMountRoots(options: IsolationOptions, protectedRoots: string[]) {
|
|
131
|
+
return requireWritableCacheRoots(
|
|
132
|
+
options.writableCacheRoots ?? [],
|
|
133
|
+
protectedRoots,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
124
137
|
function transcriptMounts(options: IsolationOptions): string[] {
|
|
125
138
|
return (options.transcriptReadRoots ?? []).flatMap((path) => [
|
|
126
139
|
"--ro-bind",
|
package/src/hosts/mac-sandbox.ts
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { containsControl } from "../value-guards";
|
|
2
3
|
import { existsSync } from "node:fs";
|
|
3
4
|
import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
|
|
4
|
-
import { resolve } from "node:path";
|
|
5
|
-
import {
|
|
5
|
+
import { dirname, resolve } from "node:path";
|
|
6
|
+
import {
|
|
7
|
+
requireNativeTranscriptViews,
|
|
8
|
+
requireNativeHookRoots,
|
|
9
|
+
} from "../native-transcript-state";
|
|
6
10
|
import { requireCandidateArtifactRoots } from "../candidate-artifacts";
|
|
7
|
-
import { workspaceBoundaryException } from "../workspace-state";
|
|
11
|
+
import { workspaceBoundaryException } from "../fixtures/workspace-state";
|
|
12
|
+
import { requireWritableCacheRoots } from "../runtime/shared-cache";
|
|
8
13
|
import {
|
|
9
14
|
insideRuntimeRoot,
|
|
10
15
|
isRuntimeHomeRoot,
|
|
11
16
|
requireRuntimeReadRoots,
|
|
12
|
-
} from "../runtime
|
|
17
|
+
} from "../runtime/paths";
|
|
13
18
|
import {
|
|
14
19
|
commandRuntimeRoot,
|
|
15
20
|
HostIsolationError,
|
|
@@ -25,17 +30,12 @@ export type { IsolatedCommand } from "./isolation-common";
|
|
|
25
30
|
const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
|
|
26
31
|
|
|
27
32
|
function quoted(path: string): string {
|
|
28
|
-
if (
|
|
29
|
-
Array.from(path).some(
|
|
30
|
-
(character) =>
|
|
31
|
-
character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
|
|
32
|
-
)
|
|
33
|
-
)
|
|
33
|
+
if (containsControl(path))
|
|
34
34
|
throw new HostIsolationError("sandbox path contains a control character");
|
|
35
35
|
return path.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
/**
|
|
38
|
+
/** Protect canonical roots while preserving explicitly granted access. */
|
|
39
39
|
export function macSandboxProfile(
|
|
40
40
|
deniedRoots: string[],
|
|
41
41
|
denyNetwork = false,
|
|
@@ -43,17 +43,20 @@ export function macSandboxProfile(
|
|
|
43
43
|
writableRuntimeRoot?: string,
|
|
44
44
|
transcriptReadRoots: string[] = [],
|
|
45
45
|
workspace?: string,
|
|
46
|
+
localHttp?: boolean,
|
|
47
|
+
writableCacheRoots?: string[],
|
|
46
48
|
): string {
|
|
47
49
|
requireProfileRoots(deniedRoots);
|
|
48
50
|
return [
|
|
49
51
|
"(version 1)",
|
|
50
52
|
"(allow default)",
|
|
51
53
|
...(denyNetwork ? ["(deny network*)"] : []),
|
|
54
|
+
...localHttpRules(localHttp),
|
|
52
55
|
...[...new Set(deniedRoots)]
|
|
53
56
|
.sort()
|
|
54
57
|
.flatMap((path) => [
|
|
55
|
-
`(deny file-read* ${workspaceReadFilter(path,
|
|
56
|
-
`(deny file-write* ${rootFilter(path, [...
|
|
58
|
+
`(deny file-read* ${workspaceReadFilter(path, readExceptions(path, readOnlyRoots, writableRuntimeRoot, transcriptReadRoots, workspace, writableCacheRoots), workspace)})`,
|
|
59
|
+
`(deny file-write* ${rootFilter(path, [...writableExceptions(path, writableRuntimeRoot, writableCacheRoots), ...workspaceBoundaryException(path, workspace)])})`,
|
|
57
60
|
]),
|
|
58
61
|
...deniedRoots.flatMap((path) =>
|
|
59
62
|
workspaceBoundaryException(path, workspace).map(
|
|
@@ -63,10 +66,50 @@ export function macSandboxProfile(
|
|
|
63
66
|
...[...readOnlyRoots, ...transcriptReadRoots].map(
|
|
64
67
|
(path) => `(deny file-write* (subpath "${quoted(path)}"))`,
|
|
65
68
|
),
|
|
69
|
+
...deniedRoots.flatMap((root) =>
|
|
70
|
+
ancestorMetadataRules(
|
|
71
|
+
root,
|
|
72
|
+
readExceptions(
|
|
73
|
+
root,
|
|
74
|
+
readOnlyRoots,
|
|
75
|
+
writableRuntimeRoot,
|
|
76
|
+
transcriptReadRoots,
|
|
77
|
+
workspace,
|
|
78
|
+
writableCacheRoots,
|
|
79
|
+
),
|
|
80
|
+
),
|
|
81
|
+
),
|
|
66
82
|
"",
|
|
67
83
|
].join("\n");
|
|
68
84
|
}
|
|
69
85
|
|
|
86
|
+
function localHttpRules(enabled: boolean | undefined): string[] {
|
|
87
|
+
return enabled
|
|
88
|
+
? [
|
|
89
|
+
'(allow network-bind (local ip "*:*"))',
|
|
90
|
+
'(allow network-inbound (local ip "localhost:*"))',
|
|
91
|
+
'(allow network-outbound (remote ip "localhost:*"))',
|
|
92
|
+
]
|
|
93
|
+
: [];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function ancestorMetadataRules(root: string, readRoots: string[]): string[] {
|
|
97
|
+
const ancestors = new Set<string>();
|
|
98
|
+
for (const readRoot of readRoots) {
|
|
99
|
+
for (
|
|
100
|
+
let path = dirname(readRoot);
|
|
101
|
+
insideRuntimeRoot(root, path);
|
|
102
|
+
path = dirname(path)
|
|
103
|
+
) {
|
|
104
|
+
ancestors.add(path);
|
|
105
|
+
if (path === resolve(root)) break;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return [...ancestors]
|
|
109
|
+
.sort()
|
|
110
|
+
.map((path) => `(allow file-read-metadata (literal "${quoted(path)}"))`);
|
|
111
|
+
}
|
|
112
|
+
|
|
70
113
|
function requireProfileRoots(roots: string[]): void {
|
|
71
114
|
if (!roots.length || roots.some((path) => !path.startsWith("/")))
|
|
72
115
|
throw new HostIsolationError("sandbox roots must be absolute and nonempty");
|
|
@@ -77,6 +120,8 @@ function readExceptions(
|
|
|
77
120
|
readRoots: string[],
|
|
78
121
|
writableRuntimeRoot: string | undefined,
|
|
79
122
|
transcriptReadRoots: string[],
|
|
123
|
+
workspace?: string,
|
|
124
|
+
writableCacheRoots: string[] = [],
|
|
80
125
|
): string[] {
|
|
81
126
|
const exceptions = isRuntimeHomeRoot(root)
|
|
82
127
|
? readRoots.filter((path) => insideRuntimeRoot(root, path))
|
|
@@ -84,7 +129,19 @@ function readExceptions(
|
|
|
84
129
|
return [
|
|
85
130
|
...transcriptReadRoots.filter((path) => insideRuntimeRoot(root, path)),
|
|
86
131
|
...exceptions,
|
|
87
|
-
...
|
|
132
|
+
...writableExceptions(root, writableRuntimeRoot, writableCacheRoots),
|
|
133
|
+
...workspaceBoundaryException(root, workspace),
|
|
134
|
+
];
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function writableExceptions(
|
|
138
|
+
root: string,
|
|
139
|
+
runtime: string | undefined,
|
|
140
|
+
caches: string[] = [],
|
|
141
|
+
): string[] {
|
|
142
|
+
return [
|
|
143
|
+
...ownedException(root, runtime),
|
|
144
|
+
...caches.flatMap((cache) => ownedException(root, cache)),
|
|
88
145
|
];
|
|
89
146
|
}
|
|
90
147
|
|
|
@@ -117,6 +174,7 @@ export async function prepareMacSandboxCommand(
|
|
|
117
174
|
requireSandboxAvailable();
|
|
118
175
|
await requireNativeTranscriptViews(options.transcriptReadRoots);
|
|
119
176
|
await requireCandidateArtifactRoots(options.candidateArtifactRoots);
|
|
177
|
+
await requireNativeHookRoots(options.hookReadRoots, options.privateStateRoot);
|
|
120
178
|
requireIsolationPaths(options);
|
|
121
179
|
const workspace = await realpath(options.workspace);
|
|
122
180
|
const roots = options.protectedRootsCanonical
|
|
@@ -129,6 +187,10 @@ export async function prepareMacSandboxCommand(
|
|
|
129
187
|
options.writableRuntimeRoot,
|
|
130
188
|
);
|
|
131
189
|
const readRoots = options.readOnlyRoots ?? [];
|
|
190
|
+
const writableCacheRoots = await requireWritableCacheRoots(
|
|
191
|
+
options.writableCacheRoots ?? [],
|
|
192
|
+
[...roots, stateRoot],
|
|
193
|
+
);
|
|
132
194
|
requireRuntimeReadRoots(readRoots, [...roots, stateRoot]);
|
|
133
195
|
requireDisjointState(workspace, roots, stateRoot);
|
|
134
196
|
const profilePath = resolve(stateRoot, `host-${randomUUID()}.sb`);
|
|
@@ -139,11 +201,10 @@ export async function prepareMacSandboxCommand(
|
|
|
139
201
|
options.denyNetwork,
|
|
140
202
|
readRoots,
|
|
141
203
|
ownedRoot,
|
|
142
|
-
|
|
143
|
-
...(options.transcriptReadRoots ?? []),
|
|
144
|
-
...Object.values(options.candidateArtifactRoots ?? {}),
|
|
145
|
-
],
|
|
204
|
+
curatedReadRoots(options),
|
|
146
205
|
workspace,
|
|
206
|
+
options.localHttp,
|
|
207
|
+
writableCacheRoots,
|
|
147
208
|
),
|
|
148
209
|
{ flag: "wx", mode: 0o600 },
|
|
149
210
|
);
|
|
@@ -155,6 +216,14 @@ export async function prepareMacSandboxCommand(
|
|
|
155
216
|
};
|
|
156
217
|
}
|
|
157
218
|
|
|
219
|
+
function curatedReadRoots(options: IsolationOptions): string[] {
|
|
220
|
+
return [
|
|
221
|
+
...(options.transcriptReadRoots ?? []),
|
|
222
|
+
...(options.hookReadRoots ?? []),
|
|
223
|
+
...Object.values(options.candidateArtifactRoots ?? {}),
|
|
224
|
+
];
|
|
225
|
+
}
|
|
226
|
+
|
|
158
227
|
function requireSandboxAvailable(): void {
|
|
159
228
|
if (process.platform !== "darwin" || !existsSync(SANDBOX_EXEC))
|
|
160
229
|
throw new HostIsolationError("macOS sandbox-exec isolation is unavailable");
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { isRecord, isUnknownArray } from "../value-guards";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
summarizeClaudeEvents,
|
|
4
|
+
parseClaudeEvents,
|
|
5
|
+
type ClaudeEvents,
|
|
6
|
+
} from "./claude-events";
|
|
3
7
|
import type { codexNativeCallObservation } from "./codex-native-calls";
|
|
8
|
+
import type { ExecCallReceipt } from "./codex-exec-controls";
|
|
4
9
|
|
|
5
10
|
const MAX_CALLS = 128;
|
|
6
11
|
type Call = { ordinal: number; namespace: string; name: string };
|
|
@@ -13,7 +18,14 @@ export interface NativeControlObservation {
|
|
|
13
18
|
method: "native_control_calls";
|
|
14
19
|
calls: Call[];
|
|
15
20
|
acceptedAgentCount: number | null;
|
|
21
|
+
directAcceptedAgentCount?: number;
|
|
22
|
+
coverage?: {
|
|
23
|
+
nativeCalls: NativeControlObservation["completeness"];
|
|
24
|
+
execInspection: "complete" | "partial";
|
|
25
|
+
acceptedAgents: "direct_native_calls";
|
|
26
|
+
};
|
|
16
27
|
submittedExecCalls: number | null;
|
|
28
|
+
execCalls?: ExecCallReceipt[];
|
|
17
29
|
truncated: boolean | null;
|
|
18
30
|
};
|
|
19
31
|
}
|
|
@@ -30,10 +42,11 @@ function label(value: unknown, max: number): value is string {
|
|
|
30
42
|
export function codexNativeControls(
|
|
31
43
|
observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
|
|
32
44
|
): NativeControlObservation {
|
|
33
|
-
const complete =
|
|
45
|
+
const complete =
|
|
46
|
+
observation.completeness === "complete" && inspectedExecCalls(observation);
|
|
34
47
|
return {
|
|
35
48
|
id: "sevro.host.native-controls",
|
|
36
|
-
completeness: observation.completeness,
|
|
49
|
+
completeness: controlCompleteness(observation.completeness, complete),
|
|
37
50
|
data: {
|
|
38
51
|
method: "native_control_calls",
|
|
39
52
|
calls: observation.data.toolCalls.map(({ ordinal, namespace, name }) => ({
|
|
@@ -44,22 +57,59 @@ export function codexNativeControls(
|
|
|
44
57
|
acceptedAgentCount: complete
|
|
45
58
|
? observation.data.acceptedSpawns.length
|
|
46
59
|
: null,
|
|
47
|
-
|
|
48
|
-
|
|
60
|
+
directAcceptedAgentCount: observation.data.acceptedSpawns.length,
|
|
61
|
+
coverage: {
|
|
62
|
+
nativeCalls: observation.completeness,
|
|
63
|
+
execInspection: inspectedExecCalls(observation)
|
|
64
|
+
? "complete"
|
|
65
|
+
: "partial",
|
|
66
|
+
acceptedAgents: "direct_native_calls",
|
|
67
|
+
},
|
|
68
|
+
...nativeCounts(observation),
|
|
69
|
+
...execEvidence(observation.data.execCalls),
|
|
49
70
|
},
|
|
50
71
|
};
|
|
51
72
|
}
|
|
73
|
+
function nativeCounts(
|
|
74
|
+
observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
|
|
75
|
+
) {
|
|
76
|
+
const complete = observation.completeness === "complete";
|
|
77
|
+
return {
|
|
78
|
+
submittedExecCalls: complete ? observation.data.submittedExecCalls : null,
|
|
79
|
+
truncated: complete ? false : null,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function controlCompleteness(
|
|
83
|
+
source: NativeControlObservation["completeness"],
|
|
84
|
+
complete: boolean,
|
|
85
|
+
): NativeControlObservation["completeness"] {
|
|
86
|
+
return source === "complete" && !complete ? "partial" : source;
|
|
87
|
+
}
|
|
88
|
+
function execEvidence(receipts: ExecCallReceipt[] | undefined) {
|
|
89
|
+
return receipts ? { execCalls: receipts } : {};
|
|
90
|
+
}
|
|
52
91
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
92
|
+
function inspectedExecCalls(
|
|
93
|
+
observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
|
|
94
|
+
): boolean {
|
|
95
|
+
const receipts = observation.data.execCalls ?? [];
|
|
96
|
+
const submissions = observation.data.toolCalls.filter(
|
|
97
|
+
(call) => call.name === "exec",
|
|
98
|
+
);
|
|
99
|
+
if (
|
|
100
|
+
submissions.length !== observation.data.submittedExecCalls ||
|
|
101
|
+
receipts.length !== submissions.length
|
|
102
|
+
)
|
|
103
|
+
return false;
|
|
104
|
+
return receipts.every(
|
|
105
|
+
(receipt, index) =>
|
|
106
|
+
receipt.completeness === "complete" &&
|
|
107
|
+
receipt.ordinal === submissions[index]?.ordinal,
|
|
108
|
+
);
|
|
61
109
|
}
|
|
62
110
|
|
|
111
|
+
const MALFORMED = Symbol("malformed native control event");
|
|
112
|
+
|
|
63
113
|
function assistantBlocks(event: unknown): unknown[] | null | typeof MALFORMED {
|
|
64
114
|
if (event === MALFORMED) return MALFORMED;
|
|
65
115
|
if (!assistantEvent(event)) return null;
|
|
@@ -87,15 +137,14 @@ class ClaudeNativeControls {
|
|
|
87
137
|
malformed = false;
|
|
88
138
|
truncated = false;
|
|
89
139
|
|
|
90
|
-
parse(stream:
|
|
91
|
-
for (const line of stream.
|
|
92
|
-
if (!line.
|
|
93
|
-
if (!this.observeLine(line)) break;
|
|
140
|
+
parse(stream: ClaudeEvents): void {
|
|
141
|
+
for (const line of stream.lines) {
|
|
142
|
+
if (!this.observeEvent(line.parsed ? line.value : MALFORMED)) break;
|
|
94
143
|
}
|
|
95
144
|
}
|
|
96
145
|
|
|
97
|
-
private
|
|
98
|
-
const blocks = assistantBlocks(
|
|
146
|
+
private observeEvent(event: unknown): boolean {
|
|
147
|
+
const blocks = assistantBlocks(event);
|
|
99
148
|
if (blocks === null) return true;
|
|
100
149
|
if (blocks === MALFORMED) {
|
|
101
150
|
this.malformed = true;
|
|
@@ -133,12 +182,13 @@ class ClaudeNativeControls {
|
|
|
133
182
|
|
|
134
183
|
/** Observe all native labels without retaining arguments or interpreting policy. */
|
|
135
184
|
export function claudeNativeControls(
|
|
136
|
-
stream: string,
|
|
185
|
+
stream: string | ClaudeEvents,
|
|
137
186
|
exitCode: number,
|
|
138
187
|
): NativeControlObservation {
|
|
139
|
-
const
|
|
188
|
+
const events = parseClaudeEvents(stream);
|
|
189
|
+
const summary = summarizeClaudeEvents(events, exitCode);
|
|
140
190
|
const controls = new ClaudeNativeControls();
|
|
141
|
-
controls.parse(
|
|
191
|
+
controls.parse(events);
|
|
142
192
|
return {
|
|
143
193
|
id: "sevro.host.native-controls",
|
|
144
194
|
completeness:
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import type { HostAdapter } from "../engine";
|
|
3
|
+
import {
|
|
4
|
+
requireCandidateRuntime,
|
|
5
|
+
requireCandidateArtifactRoots,
|
|
6
|
+
} from "../candidate-artifacts";
|
|
7
|
+
import { requireNativeTranscriptView } from "../native-transcript-state";
|
|
8
|
+
import { isOptionalNonblankString } from "../value-guards";
|
|
9
|
+
|
|
10
|
+
export const MAX_HOST_TIMEOUT_MS = 30 * 60_000;
|
|
11
|
+
|
|
12
|
+
export function validHostTimeout(timeoutMs: number): boolean {
|
|
13
|
+
return (
|
|
14
|
+
Number.isSafeInteger(timeoutMs) &&
|
|
15
|
+
timeoutMs >= 1 &&
|
|
16
|
+
timeoutMs <= MAX_HOST_TIMEOUT_MS
|
|
17
|
+
);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Common admission checks run before native state or processes are allocated. */
|
|
21
|
+
export async function validateNativeHostRequest(
|
|
22
|
+
request: Parameters<HostAdapter["run"]>[0],
|
|
23
|
+
host: "Codex" | "Claude",
|
|
24
|
+
): Promise<void> {
|
|
25
|
+
requireFollowUp(request.followUpPrompt, host);
|
|
26
|
+
await requireNativeTranscriptView(request.candidateTranscriptRoot);
|
|
27
|
+
await requireCandidateRuntime(
|
|
28
|
+
request.candidateRuntimeRoot,
|
|
29
|
+
request.runtimeRole,
|
|
30
|
+
);
|
|
31
|
+
await requireCandidateArtifactRoots(request.candidateArtifactRoots);
|
|
32
|
+
requireFixtureBin(request);
|
|
33
|
+
requirePassiveRequest(request, host);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function requireFollowUp(prompt: string | undefined, host: string): void {
|
|
37
|
+
if (!isOptionalNonblankString(prompt))
|
|
38
|
+
throw new Error(`${host} follow-up prompt must be nonempty`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function requireFixtureBin(request: Parameters<HostAdapter["run"]>[0]): void {
|
|
42
|
+
if (
|
|
43
|
+
request.fixtureBinDir !== undefined &&
|
|
44
|
+
request.fixtureBinDir !== join(request.workspace, ".git", "fixture-bin")
|
|
45
|
+
)
|
|
46
|
+
throw new Error("fixture binary path is outside the workspace");
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function requirePassiveRequest(
|
|
50
|
+
request: Parameters<HostAdapter["run"]>[0],
|
|
51
|
+
host: string,
|
|
52
|
+
): void {
|
|
53
|
+
if (request.instrumentation?.length || request.condition !== "passive")
|
|
54
|
+
throw new Error(`${host} enforcement instrumentation is unavailable`);
|
|
55
|
+
}
|