@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +170 -0
- package/LICENSE +63 -136
- package/README.md +83 -66
- package/docs/README.md +44 -0
- package/docs/acknowledgements.md +14 -0
- package/docs/architecture.md +146 -0
- package/docs/assets/README.md +27 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +1 -0
- package/docs/assets/sevro-wordmark.svg +1 -0
- package/docs/builtin-graders-v1.md +41 -13
- package/docs/codex-host.md +184 -52
- package/docs/contracts.md +19 -0
- package/docs/creating-cases.md +139 -0
- package/docs/development-cli.md +190 -30
- package/docs/documentation-quality.md +61 -0
- package/docs/evaluation-workflow.md +110 -0
- package/docs/evidence/code-mode-controls-71.md +64 -0
- package/docs/evidence/code-mode-programs-85.md +64 -0
- package/docs/evidence/codex-state-73/README.md +204 -0
- package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
- package/docs/evidence/codex-state-73/results.json +661 -0
- package/docs/evidence/local-http-74.md +104 -0
- package/docs/evidence/runtime-startup-30/README.md +185 -0
- package/docs/evidence/runtime-startup-30/after.json +319 -0
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
- package/docs/evidence/runtime-startup-30/before.json +403 -0
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
- package/docs/evidence/skill-reads-70/results.json +823 -0
- package/docs/evidence.md +21 -0
- package/docs/extension-protocol-v1.md +50 -16
- package/docs/fixture-cleanup-validation.md +3 -0
- package/docs/getting-started.md +102 -0
- package/docs/guide-evaluation.md +89 -0
- package/docs/guide-verification.json +239 -0
- package/docs/identity-v1.md +35 -1
- package/docs/installing.md +88 -0
- package/docs/licenses/BUSL-1.1-0.1.0-rc.2.txt +159 -0
- package/docs/licensing.md +42 -0
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +228 -0
- package/docs/reading-results.md +141 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +81 -40
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +180 -0
- package/docs/run-settings-v1.md +147 -0
- package/docs/running-evaluations.md +31 -0
- package/docs/runtime-v1.md +614 -0
- package/docs/specs/repository-guide.md +58 -0
- package/docs/troubleshooting.md +30 -0
- package/docs/typescript-quality.md +234 -0
- package/docs/usability-study.md +114 -0
- package/examples/basic/host.ts +2 -2
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +39 -6
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +193 -0
- package/src/advisory.ts +10 -5
- package/src/candidate-artifacts.ts +208 -0
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +105 -0
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +81 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +59 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +222 -0
- package/src/cli/hosts.ts +59 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/cli/invocation.ts +627 -0
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +163 -0
- package/src/cli/presentation.ts +91 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +133 -0
- package/src/cli/run-options.ts +101 -0
- package/src/cli/run-overrides.ts +83 -0
- package/src/cli/settings.ts +160 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -843
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +19 -2373
- package/src/evaluation/advisory.ts +150 -0
- package/src/evaluation/check-plan.ts +510 -0
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/evaluation/grading-host.ts +39 -0
- package/src/evaluation/grading.ts +258 -0
- package/src/evaluation/invocation-plan.ts +548 -0
- package/src/evaluation/plan.ts +736 -0
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +3 -1
- package/src/evaluation/semantic.ts +312 -0
- package/src/evaluation/trial.ts +866 -0
- package/src/evaluation/types.ts +207 -0
- package/src/evidence/host-results.ts +92 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +228 -148
- package/src/extension-session.ts +326 -195
- package/src/fixtures/advisory.ts +338 -0
- package/src/fixtures/bin.ts +57 -0
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +187 -0
- package/src/fixtures/generated.ts +300 -0
- package/src/fixtures/git-hooks.ts +24 -0
- package/src/fixtures/preparation.ts +270 -0
- package/src/{repository-fixture.ts → fixtures/repository.ts} +207 -85
- package/src/fixtures/setup.ts +224 -0
- package/src/fixtures/tools.ts +88 -0
- package/src/fixtures/workspace-state.ts +212 -0
- package/src/generated/cli-result.cjs +2 -0
- package/src/generated/cli-result.d.cts +3 -0
- package/src/generated/extension.cjs +2 -0
- package/src/generated/extension.d.cts +3 -0
- package/src/generated/report.cjs +2 -0
- package/src/generated/report.d.cts +3 -0
- package/src/generated/run-evidence.cjs +2 -0
- package/src/generated/run-evidence.d.cts +3 -0
- package/src/generated/runtime.cjs +2 -0
- package/src/generated/runtime.d.cts +3 -0
- package/src/graders/artifact.ts +56 -23
- package/src/graders/git-head.ts +79 -75
- package/src/graders/output.ts +222 -140
- package/src/graders/semantic.ts +102 -43
- package/src/graders/shell.ts +379 -207
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +319 -0
- package/src/hosts/claude-continuation.ts +63 -18
- package/src/hosts/claude-credential.ts +43 -62
- package/src/hosts/claude-events.ts +86 -51
- package/src/hosts/claude-keychain.ts +59 -0
- package/src/hosts/claude-native-goal.ts +57 -33
- package/src/hosts/claude-process.ts +111 -0
- package/src/hosts/claude-repository-invocation.ts +208 -117
- package/src/hosts/claude-settings.ts +240 -21
- package/src/hosts/claude-skill-activations.ts +166 -0
- package/src/hosts/claude-state.ts +403 -0
- package/src/hosts/claude-tool-calls.ts +171 -106
- package/src/hosts/claude.ts +561 -451
- package/src/hosts/codex-app-server.ts +326 -178
- package/src/hosts/codex-configuration.ts +44 -18
- package/src/hosts/codex-events.ts +97 -60
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-invocation.ts +70 -0
- package/src/hosts/codex-marketplace.ts +283 -0
- package/src/hosts/codex-native-calls.ts +546 -500
- package/src/hosts/codex-process.ts +202 -0
- package/src/hosts/codex-profile.ts +257 -28
- package/src/hosts/codex-session-files.ts +106 -0
- package/src/hosts/codex-state.ts +492 -0
- package/src/hosts/codex.ts +783 -832
- package/src/hosts/isolation-common.ts +94 -0
- package/src/hosts/isolation-roots.ts +54 -27
- package/src/hosts/isolation.ts +10 -0
- package/src/hosts/linux-sandbox.ts +159 -0
- package/src/hosts/mac-sandbox.ts +187 -61
- package/src/hosts/native-controls.ts +152 -62
- package/src/hosts/process-lifecycle.ts +37 -0
- package/src/hosts/protected-worktrees.ts +51 -19
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +580 -0
- package/src/hosts/workspace-fingerprint.ts +55 -35
- package/src/identity.ts +102 -48
- package/src/instrumentation.ts +109 -48
- package/src/native-transcript-state.ts +131 -0
- package/src/native-transcript-view.ts +90 -0
- package/src/native-transcripts.ts +286 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +161 -81
- package/src/prune.ts +215 -0
- package/src/report.ts +172 -140
- package/src/resolved-case.ts +33 -0
- package/src/results-lock.ts +29 -0
- package/src/results.ts +133 -86
- package/src/run-owner.ts +154 -63
- package/src/runtime/config.ts +456 -0
- package/src/runtime/discovery-apple.ts +90 -0
- package/src/runtime/discovery-providers.ts +63 -0
- package/src/runtime/discovery.ts +70 -0
- package/src/runtime/hook-processes.ts +97 -0
- package/src/runtime/hooks.ts +476 -0
- package/src/runtime/paths.ts +313 -0
- package/src/runtime/query.ts +85 -0
- package/src/runtime/seeds.ts +134 -0
- package/src/runtime/shared-cache.ts +143 -0
- package/src/runtime/state.ts +143 -0
- package/src/schema-types.ts +198 -0
- package/src/schema.ts +15 -19
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +20 -0
- package/src/advisory-fixture.ts +0 -282
- package/src/fixture-bin.ts +0 -87
- package/src/fixture-setup.ts +0 -163
- package/src/generated-fixture.ts +0 -220
- package/src/git-hooks.ts +0 -63
- package/src/hosts/claude-nested-skills.ts +0 -218
- package/src/hosts/codex-command-output.ts +0 -399
- package/src/hosts/codex-executor-command.ts +0 -195
- package/src/hosts/codex-skill-reads.ts +0 -391
- package/src/preparation.ts +0 -211
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { isRecord, isStringArray } from "../value-guards";
|
|
2
|
+
import { isMissingFile } from "./tools";
|
|
1
3
|
import { lstat, mkdir, realpath, stat, writeFile } from "node:fs/promises";
|
|
2
4
|
import { isAbsolute, join, relative, sep } from "node:path";
|
|
3
5
|
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { canonicalJson } from "
|
|
5
|
-
import { prepareFixtureBin } from "./
|
|
6
|
+
import { canonicalJson } from "../identity";
|
|
7
|
+
import { prepareFixtureBin } from "./bin";
|
|
6
8
|
import { prepareGitHooks } from "./git-hooks";
|
|
7
9
|
import type { PreparationSources } from "./preparation";
|
|
8
10
|
import { fixtureParts } from "./preparation";
|
|
@@ -21,53 +23,110 @@ export interface RepositoryFixture {
|
|
|
21
23
|
bin?: Record<string, string>;
|
|
22
24
|
}
|
|
23
25
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
function repositoryRecord(value: unknown): Record<string, unknown> {
|
|
27
|
+
if (!isRecord(value)) throw new Error("invalid repository fixture");
|
|
28
|
+
return value;
|
|
29
|
+
}
|
|
30
|
+
function requireRepositoryIdentity(
|
|
31
|
+
fixture: Record<string, unknown>,
|
|
32
|
+
): asserts fixture is Record<string, unknown> & {
|
|
33
|
+
kind: "repository";
|
|
34
|
+
sourceRef: string;
|
|
35
|
+
} {
|
|
36
|
+
const allowed = [
|
|
37
|
+
"kind",
|
|
38
|
+
"sourceRef",
|
|
39
|
+
"files",
|
|
40
|
+
"staged",
|
|
41
|
+
"commitFiles",
|
|
42
|
+
"hooks",
|
|
43
|
+
"bin",
|
|
44
|
+
];
|
|
29
45
|
if (
|
|
30
46
|
fixture.kind !== "repository" ||
|
|
31
|
-
Object.keys(fixture).some(
|
|
32
|
-
(key) =>
|
|
33
|
-
![
|
|
34
|
-
"kind",
|
|
35
|
-
"sourceRef",
|
|
36
|
-
"files",
|
|
37
|
-
"staged",
|
|
38
|
-
"commitFiles",
|
|
39
|
-
"hooks",
|
|
40
|
-
"bin",
|
|
41
|
-
].includes(key),
|
|
42
|
-
) ||
|
|
47
|
+
Object.keys(fixture).some((key) => !allowed.includes(key)) ||
|
|
43
48
|
typeof fixture.sourceRef !== "string" ||
|
|
44
|
-
!fixture.sourceRef
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
49
|
+
!fixture.sourceRef
|
|
50
|
+
)
|
|
51
|
+
throw new Error("invalid repository fixture");
|
|
52
|
+
}
|
|
53
|
+
function optionalRepositoryFiles(
|
|
54
|
+
value: unknown,
|
|
55
|
+
): value is Record<string, string> | undefined {
|
|
56
|
+
return (
|
|
57
|
+
value === undefined ||
|
|
58
|
+
(isRecord(value) &&
|
|
59
|
+
Object.values(value).every((item) => typeof item === "string"))
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
function optionalRepositoryStaging(
|
|
63
|
+
value: unknown,
|
|
64
|
+
): value is string[] | undefined {
|
|
65
|
+
return value === undefined || isStringArray(value);
|
|
66
|
+
}
|
|
67
|
+
function optionalRepositoryCommit(
|
|
68
|
+
value: unknown,
|
|
69
|
+
): value is boolean | undefined {
|
|
70
|
+
return value === undefined || typeof value === "boolean";
|
|
71
|
+
}
|
|
72
|
+
function requireRepositoryOptions(
|
|
73
|
+
fixture: Record<string, unknown>,
|
|
74
|
+
): asserts fixture is Record<string, unknown> & {
|
|
75
|
+
files?: Record<string, string>;
|
|
76
|
+
staged?: string[];
|
|
77
|
+
commitFiles?: boolean;
|
|
78
|
+
} {
|
|
79
|
+
if (
|
|
80
|
+
!optionalRepositoryFiles(fixture.files) ||
|
|
81
|
+
!optionalRepositoryStaging(fixture.staged) ||
|
|
82
|
+
!optionalRepositoryCommit(fixture.commitFiles)
|
|
57
83
|
)
|
|
58
84
|
throw new Error("invalid repository fixture");
|
|
85
|
+
}
|
|
86
|
+
function requireRepositoryTools(
|
|
87
|
+
fixture: Record<string, unknown>,
|
|
88
|
+
): asserts fixture is Record<string, unknown> & {
|
|
89
|
+
hooks?: Record<string, string>;
|
|
90
|
+
bin?: Record<string, string>;
|
|
91
|
+
} {
|
|
59
92
|
prepareGitHooks(fixture.hooks);
|
|
60
93
|
prepareFixtureBin(fixture.bin);
|
|
61
|
-
|
|
62
|
-
|
|
94
|
+
}
|
|
95
|
+
function repositoryFixtureData(value: unknown): RepositoryFixture {
|
|
96
|
+
const fixture = repositoryRecord(value);
|
|
97
|
+
requireRepositoryIdentity(fixture);
|
|
98
|
+
requireRepositoryOptions(fixture);
|
|
99
|
+
requireRepositoryTools(fixture);
|
|
100
|
+
return fixture;
|
|
101
|
+
}
|
|
102
|
+
function requireOverlayPath(path: string): void {
|
|
103
|
+
if (fixtureParts(path).some((part) => part.toLowerCase() === ".git"))
|
|
104
|
+
throw new Error("repository fixture cannot write repository metadata");
|
|
105
|
+
}
|
|
106
|
+
function overlayPathsCollide(left: string, right: string): boolean {
|
|
107
|
+
return (
|
|
108
|
+
left === right ||
|
|
109
|
+
left.startsWith(`${right}/`) ||
|
|
110
|
+
right.startsWith(`${left}/`)
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
function requireDistinctOverlayPaths(paths: string[]): void {
|
|
114
|
+
const normalized = paths.map((path) => path.toLowerCase());
|
|
115
|
+
for (const [index, path] of normalized.entries())
|
|
116
|
+
for (const other of normalized.slice(0, index))
|
|
117
|
+
if (overlayPathsCollide(path, other))
|
|
118
|
+
throw new Error("repository fixture overlay paths collide");
|
|
119
|
+
}
|
|
120
|
+
function requireBoundedOverlay(
|
|
121
|
+
files: Record<string, string>,
|
|
122
|
+
staged: string[],
|
|
123
|
+
): void {
|
|
63
124
|
const paths = Object.keys(files);
|
|
64
125
|
if (paths.length > MAX_OVERLAY_FILES)
|
|
65
126
|
throw new Error("repository fixture exceeds the file limit");
|
|
66
127
|
let bytes = 0;
|
|
67
128
|
for (const [path, content] of Object.entries(files)) {
|
|
68
|
-
|
|
69
|
-
if (parts.some((part) => part.toLowerCase() === ".git"))
|
|
70
|
-
throw new Error("repository fixture cannot write repository metadata");
|
|
129
|
+
requireOverlayPath(path);
|
|
71
130
|
bytes +=
|
|
72
131
|
Buffer.byteLength(path, "utf8") + Buffer.byteLength(content, "utf8");
|
|
73
132
|
}
|
|
@@ -77,24 +136,30 @@ export function prepareRepositoryFixture(value: unknown): RepositoryFixture {
|
|
|
77
136
|
);
|
|
78
137
|
if (bytes > MAX_OVERLAY_BYTES)
|
|
79
138
|
throw new Error("repository fixture exceeds the size limit");
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
)
|
|
88
|
-
throw new Error("repository fixture overlay paths collide");
|
|
89
|
-
}
|
|
90
|
-
}
|
|
139
|
+
requireDistinctOverlayPaths(paths);
|
|
140
|
+
}
|
|
141
|
+
function requireRepositoryStaging(
|
|
142
|
+
fixture: RepositoryFixture,
|
|
143
|
+
files: Record<string, string>,
|
|
144
|
+
staged: string[],
|
|
145
|
+
): void {
|
|
91
146
|
if (
|
|
92
147
|
new Set(staged.map((path) => path.toLowerCase())).size !== staged.length ||
|
|
93
148
|
staged.some((path) => !Object.hasOwn(files, path)) ||
|
|
94
|
-
(fixture.commitFiles === true &&
|
|
149
|
+
(fixture.commitFiles === true && Object.keys(files).length === 0)
|
|
95
150
|
)
|
|
96
151
|
throw new Error("invalid repository fixture staging");
|
|
97
|
-
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Validate repository source identity and bounded working-tree changes. */
|
|
155
|
+
export function prepareRepositoryFixture(value: unknown): RepositoryFixture {
|
|
156
|
+
const fixture = repositoryFixtureData(value);
|
|
157
|
+
const files = fixture.files ?? {},
|
|
158
|
+
staged = fixture.staged ?? [];
|
|
159
|
+
requireBoundedOverlay(files, staged);
|
|
160
|
+
requireRepositoryStaging(fixture, files, staged);
|
|
161
|
+
const clone: unknown = JSON.parse(canonicalJson(fixture));
|
|
162
|
+
return repositoryFixtureData(clone);
|
|
98
163
|
}
|
|
99
164
|
|
|
100
165
|
export interface RepositorySource {
|
|
@@ -137,7 +202,7 @@ async function git(args: string[], cwd: string): Promise<string> {
|
|
|
137
202
|
},
|
|
138
203
|
});
|
|
139
204
|
try {
|
|
140
|
-
const [stdout,
|
|
205
|
+
const [stdout, , code] = await Promise.all([
|
|
141
206
|
boundedText(proc.stdout),
|
|
142
207
|
boundedText(proc.stderr),
|
|
143
208
|
proc.exited,
|
|
@@ -156,38 +221,78 @@ export async function resolveRepositorySource(
|
|
|
156
221
|
sourceRef: string,
|
|
157
222
|
sources?: PreparationSources,
|
|
158
223
|
): Promise<RepositorySource> {
|
|
224
|
+
const selected = repositorySourceSelection(sourceRef, sources);
|
|
225
|
+
const { root, path } = await canonicalRepositorySource(selected);
|
|
226
|
+
requireContainedRepository(root, path);
|
|
227
|
+
return { path, revision: await repositoryCommit(path) };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function requireRepositorySources(
|
|
231
|
+
sources: PreparationSources | undefined,
|
|
232
|
+
): asserts sources is PreparationSources {
|
|
159
233
|
if (!sources || !isAbsolute(sources.root))
|
|
160
234
|
throw new Error("repository source root is required and must be absolute");
|
|
235
|
+
}
|
|
236
|
+
function repositorySourceSelection(
|
|
237
|
+
sourceRef: string,
|
|
238
|
+
sources: PreparationSources | undefined,
|
|
239
|
+
) {
|
|
240
|
+
requireRepositorySources(sources);
|
|
161
241
|
if (!Object.hasOwn(sources.refs, sourceRef))
|
|
162
242
|
throw new Error("repository source reference is not declared");
|
|
163
243
|
const url = sources.refs[sourceRef];
|
|
164
244
|
if (typeof url !== "string" || !url.startsWith("file:///"))
|
|
165
245
|
throw new Error("repository source reference must be a file URL");
|
|
166
|
-
|
|
167
|
-
|
|
246
|
+
return { root: sources.root, url };
|
|
247
|
+
}
|
|
248
|
+
async function canonicalRepositorySource(selected: {
|
|
249
|
+
root: string;
|
|
250
|
+
url: string;
|
|
251
|
+
}) {
|
|
168
252
|
try {
|
|
169
|
-
[root, path] = await Promise.all([
|
|
170
|
-
realpath(
|
|
171
|
-
realpath(fileURLToPath(url)),
|
|
253
|
+
const [root, path] = await Promise.all([
|
|
254
|
+
realpath(selected.root),
|
|
255
|
+
realpath(fileURLToPath(selected.url)),
|
|
172
256
|
]);
|
|
173
257
|
if (!(await stat(path)).isDirectory()) throw new Error();
|
|
174
|
-
|
|
175
|
-
|
|
258
|
+
return { root, path };
|
|
259
|
+
} catch (cause) {
|
|
260
|
+
throw new Error("repository source is unreadable", { cause });
|
|
176
261
|
}
|
|
262
|
+
}
|
|
263
|
+
function requireContainedRepository(root: string, path: string): void {
|
|
177
264
|
const child = relative(root, path);
|
|
178
265
|
if (child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child))
|
|
179
266
|
throw new Error("repository source escapes its declared root");
|
|
267
|
+
}
|
|
268
|
+
async function repositoryCommit(path: string): Promise<string> {
|
|
180
269
|
const top = await git(["rev-parse", "--show-toplevel"], path);
|
|
181
270
|
if ((await realpath(top)) !== path)
|
|
182
271
|
throw new Error("repository source must name the repository root");
|
|
183
272
|
const revision = await git(["rev-parse", "--verify", "HEAD^{commit}"], path);
|
|
184
273
|
if (!/^[a-f0-9]{40,64}$/.test(revision))
|
|
185
274
|
throw new Error("repository source has no valid commit");
|
|
186
|
-
if (
|
|
275
|
+
if (
|
|
276
|
+
await git(
|
|
277
|
+
[
|
|
278
|
+
"status",
|
|
279
|
+
"--porcelain=v1",
|
|
280
|
+
"--untracked-files=all",
|
|
281
|
+
"--ignore-submodules=none",
|
|
282
|
+
],
|
|
283
|
+
path,
|
|
284
|
+
)
|
|
285
|
+
)
|
|
187
286
|
throw new Error("repository source has uncommitted changes");
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
287
|
+
return revision;
|
|
288
|
+
}
|
|
289
|
+
function requireRepositorySnapshot(
|
|
290
|
+
current: RepositorySource,
|
|
291
|
+
expected: RepositorySource,
|
|
292
|
+
message: string,
|
|
293
|
+
): void {
|
|
294
|
+
if (current.path !== expected.path || current.revision !== expected.revision)
|
|
295
|
+
throw new Error(message);
|
|
191
296
|
}
|
|
192
297
|
|
|
193
298
|
/** Clone a fixed commit without hardlinks or a remote back to the source. */
|
|
@@ -198,14 +303,18 @@ export async function cloneRepositorySource(
|
|
|
198
303
|
workspace: string,
|
|
199
304
|
): Promise<void> {
|
|
200
305
|
const current = await resolveRepositorySource(sourceRef, sources);
|
|
201
|
-
|
|
202
|
-
|
|
306
|
+
requireRepositorySnapshot(
|
|
307
|
+
current,
|
|
308
|
+
expected,
|
|
309
|
+
"repository source changed during run",
|
|
310
|
+
);
|
|
203
311
|
await git(
|
|
204
312
|
[
|
|
205
313
|
"clone",
|
|
206
314
|
"--local",
|
|
207
315
|
"--no-hardlinks",
|
|
208
316
|
"--no-checkout",
|
|
317
|
+
"--no-recurse-submodules",
|
|
209
318
|
"--quiet",
|
|
210
319
|
current.path,
|
|
211
320
|
workspace,
|
|
@@ -220,8 +329,11 @@ export async function cloneRepositorySource(
|
|
|
220
329
|
.filter(Boolean))
|
|
221
330
|
await git(["remote", "remove", remote], workspace);
|
|
222
331
|
const after = await resolveRepositorySource(sourceRef, sources);
|
|
223
|
-
|
|
224
|
-
|
|
332
|
+
requireRepositorySnapshot(
|
|
333
|
+
after,
|
|
334
|
+
expected,
|
|
335
|
+
"repository source changed during clone",
|
|
336
|
+
);
|
|
225
337
|
}
|
|
226
338
|
|
|
227
339
|
async function safeOverlayTarget(
|
|
@@ -232,30 +344,31 @@ async function safeOverlayTarget(
|
|
|
232
344
|
let parent = workspace;
|
|
233
345
|
for (const part of parts.slice(0, -1)) {
|
|
234
346
|
parent = join(parent, part);
|
|
235
|
-
|
|
236
|
-
const entry = await lstat(parent);
|
|
237
|
-
if (!entry.isDirectory() || entry.isSymbolicLink())
|
|
238
|
-
throw new Error("repository fixture overlay traverses a non-directory");
|
|
239
|
-
} catch (error) {
|
|
240
|
-
if (!(
|
|
241
|
-
error instanceof Error &&
|
|
242
|
-
"code" in error &&
|
|
243
|
-
error.code === "ENOENT"
|
|
244
|
-
))
|
|
245
|
-
throw error;
|
|
246
|
-
await mkdir(parent, { mode: 0o700 });
|
|
247
|
-
}
|
|
347
|
+
await overlayParent(parent);
|
|
248
348
|
}
|
|
249
349
|
const target = join(workspace, ...parts);
|
|
350
|
+
await requireOverlayTarget(target);
|
|
351
|
+
return target;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
async function overlayParent(path: string): Promise<void> {
|
|
250
355
|
try {
|
|
251
|
-
const entry = await lstat(
|
|
356
|
+
const entry = await lstat(path);
|
|
357
|
+
if (!entry.isDirectory() || entry.isSymbolicLink())
|
|
358
|
+
throw new Error("repository fixture overlay traverses a non-directory");
|
|
359
|
+
} catch (error) {
|
|
360
|
+
if (!isMissingFile(error)) throw error;
|
|
361
|
+
await mkdir(path, { mode: 0o700 });
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
async function requireOverlayTarget(path: string): Promise<void> {
|
|
365
|
+
try {
|
|
366
|
+
const entry = await lstat(path);
|
|
252
367
|
if (!entry.isFile() || entry.isSymbolicLink())
|
|
253
368
|
throw new Error("repository fixture overlay targets a non-file");
|
|
254
369
|
} catch (error) {
|
|
255
|
-
if (!(error
|
|
256
|
-
throw error;
|
|
370
|
+
if (!isMissingFile(error)) throw error;
|
|
257
371
|
}
|
|
258
|
-
return target;
|
|
259
372
|
}
|
|
260
373
|
|
|
261
374
|
/** Apply declared files after cloning, optionally committing or staging them. */
|
|
@@ -269,11 +382,20 @@ export async function applyRepositoryOverlay(
|
|
|
269
382
|
flag: "w",
|
|
270
383
|
mode: 0o600,
|
|
271
384
|
});
|
|
272
|
-
|
|
273
|
-
|
|
385
|
+
await stageRepositoryOverlay(fixture, workspace, Object.keys(files));
|
|
386
|
+
}
|
|
387
|
+
function disabledHooksArgs(): string[] {
|
|
388
|
+
return [
|
|
274
389
|
"-c",
|
|
275
390
|
`core.hooksPath=${process.platform === "win32" ? "NUL" : "/dev/null"}`,
|
|
276
391
|
];
|
|
392
|
+
}
|
|
393
|
+
async function stageRepositoryOverlay(
|
|
394
|
+
fixture: RepositoryFixture,
|
|
395
|
+
workspace: string,
|
|
396
|
+
paths: string[],
|
|
397
|
+
): Promise<void> {
|
|
398
|
+
const noHooks = disabledHooksArgs();
|
|
277
399
|
if (fixture.commitFiles) {
|
|
278
400
|
await git([...noHooks, "add", "--", ...paths], workspace);
|
|
279
401
|
await git(
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { mkdir, mkdtemp, rm } from "node:fs/promises";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { delimiter, isAbsolute, join } from "node:path";
|
|
4
|
+
import { isRecord } from "../value-guards";
|
|
5
|
+
import { requireCandidateRuntime } from "../candidate-artifacts";
|
|
6
|
+
import {
|
|
7
|
+
ProcessDeadline,
|
|
8
|
+
terminateOwnedProcessGroup,
|
|
9
|
+
} from "../process-lifecycle";
|
|
10
|
+
|
|
11
|
+
const MAX_COMMAND_BYTES = 64 * 1024;
|
|
12
|
+
const MAX_ENV_BYTES = 16 * 1024;
|
|
13
|
+
const TIMEOUT_MS = 120_000;
|
|
14
|
+
const RESERVED_ENV = new Set([
|
|
15
|
+
"PATH",
|
|
16
|
+
"HOME",
|
|
17
|
+
"TMPDIR",
|
|
18
|
+
"GIT_CONFIG_GLOBAL",
|
|
19
|
+
"GIT_CONFIG_NOSYSTEM",
|
|
20
|
+
"GIT_TERMINAL_PROMPT",
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
export interface FixtureSetupDeclaration {
|
|
24
|
+
command: string[];
|
|
25
|
+
environment?: Record<string, string>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface FixtureSetup extends FixtureSetupDeclaration {
|
|
29
|
+
environment: Record<string, string>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function setupRecord(value: unknown): Record<string, unknown> {
|
|
33
|
+
if (!isRecord(value)) throw new Error("invalid fixture setup");
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function validArgument(value: unknown): value is string {
|
|
38
|
+
return typeof value === "string" && value.length > 0 && !value.includes("\0");
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function argumentList(value: unknown): value is string[] {
|
|
42
|
+
return (
|
|
43
|
+
Array.isArray(value) &&
|
|
44
|
+
value.length > 0 &&
|
|
45
|
+
value.length <= 16 &&
|
|
46
|
+
value.every(validArgument)
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function setupCommand(value: unknown): string[] {
|
|
51
|
+
if (!argumentList(value)) throw new Error("invalid fixture setup command");
|
|
52
|
+
const executable = value[0];
|
|
53
|
+
if (
|
|
54
|
+
!absoluteExecutable(executable) ||
|
|
55
|
+
Buffer.byteLength(value.join("\0"), "utf8") > MAX_COMMAND_BYTES
|
|
56
|
+
)
|
|
57
|
+
throw new Error("invalid fixture setup command");
|
|
58
|
+
return value;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function absoluteExecutable(value: string | undefined): boolean {
|
|
62
|
+
return value !== undefined && isAbsolute(value);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function validEnvironmentEntry([name, value]: [string, unknown]): boolean {
|
|
66
|
+
return (
|
|
67
|
+
/^[A-Z][A-Z0-9_]*$/.test(name) &&
|
|
68
|
+
!RESERVED_ENV.has(name) &&
|
|
69
|
+
validEnvironmentValue(value)
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function validEnvironmentValue(value: unknown): value is string {
|
|
74
|
+
return (
|
|
75
|
+
typeof value === "string" &&
|
|
76
|
+
!value.includes("\0") &&
|
|
77
|
+
!/\{\{(?!sevro\.(?:project|workspace)\}\})/.test(value)
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function environmentRecord(value: unknown): Record<string, unknown> {
|
|
82
|
+
if (!isRecord(value)) throw new Error("invalid fixture setup environment");
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function setupEnvironment(value: unknown): Record<string, string> {
|
|
87
|
+
const environment = environmentRecord(value);
|
|
88
|
+
const entries = Object.entries(environment);
|
|
89
|
+
if (
|
|
90
|
+
entries.length > 32 ||
|
|
91
|
+
entries.some((entry) => !validEnvironmentEntry(entry)) ||
|
|
92
|
+
Buffer.byteLength(JSON.stringify(environment), "utf8") > MAX_ENV_BYTES
|
|
93
|
+
)
|
|
94
|
+
throw new Error("invalid fixture setup environment");
|
|
95
|
+
return environment as Record<string, string>;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Validate a trusted extension's bounded fixture setup command. */
|
|
99
|
+
export function prepareFixtureSetup(value: unknown): FixtureSetup | null {
|
|
100
|
+
if (value === undefined) return null;
|
|
101
|
+
const input = setupRecord(value);
|
|
102
|
+
if (
|
|
103
|
+
Object.keys(input).some((key) => key !== "command" && key !== "environment")
|
|
104
|
+
)
|
|
105
|
+
throw new Error("unsupported fixture setup field");
|
|
106
|
+
const command = setupCommand(input.command);
|
|
107
|
+
const environment = setupEnvironment(input.environment ?? {});
|
|
108
|
+
return { command, environment };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
interface FixtureSetupOptions {
|
|
112
|
+
workspace: string;
|
|
113
|
+
projectRoot: string;
|
|
114
|
+
fixtureBinDir?: string;
|
|
115
|
+
signal?: AbortSignal;
|
|
116
|
+
candidateRuntimeRoot?: string;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function setupPath(directory: string | undefined): string {
|
|
120
|
+
const inherited = process.env.PATH ?? "/usr/bin:/bin";
|
|
121
|
+
return directory ? `${directory}${delimiter}${inherited}` : inherited;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function executionEnvironment(
|
|
125
|
+
setup: FixtureSetup,
|
|
126
|
+
options: FixtureSetupOptions,
|
|
127
|
+
home: string,
|
|
128
|
+
) {
|
|
129
|
+
const environment = Object.fromEntries(
|
|
130
|
+
Object.entries(setup.environment).map(([name, value]) => [
|
|
131
|
+
name,
|
|
132
|
+
value
|
|
133
|
+
.replaceAll("{{sevro.project}}", options.projectRoot)
|
|
134
|
+
.replaceAll("{{sevro.workspace}}", options.workspace),
|
|
135
|
+
]),
|
|
136
|
+
);
|
|
137
|
+
return {
|
|
138
|
+
PATH: setupPath(options.fixtureBinDir),
|
|
139
|
+
HOME: options.candidateRuntimeRoot
|
|
140
|
+
? join(options.candidateRuntimeRoot, "home")
|
|
141
|
+
: home,
|
|
142
|
+
TMPDIR: home,
|
|
143
|
+
LANG: process.env.LANG ?? "C",
|
|
144
|
+
GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : "/dev/null",
|
|
145
|
+
GIT_CONFIG_NOSYSTEM: "1",
|
|
146
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
147
|
+
GIT_AUTHOR_NAME: "Sevro Fixture",
|
|
148
|
+
GIT_AUTHOR_EMAIL: "fixture@sevro.invalid",
|
|
149
|
+
GIT_COMMITTER_NAME: "Sevro Fixture",
|
|
150
|
+
GIT_COMMITTER_EMAIL: "fixture@sevro.invalid",
|
|
151
|
+
GIT_AUTHOR_DATE: "2000-01-01T00:00:00+00:00",
|
|
152
|
+
GIT_COMMITTER_DATE: "2000-01-01T00:00:00+00:00",
|
|
153
|
+
...environment,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
class FixtureSetupExecution {
|
|
158
|
+
private proc: Bun.Subprocess | undefined;
|
|
159
|
+
private readonly deadline: ProcessDeadline;
|
|
160
|
+
constructor(
|
|
161
|
+
private readonly setup: FixtureSetup,
|
|
162
|
+
private readonly options: FixtureSetupOptions,
|
|
163
|
+
private readonly home: string,
|
|
164
|
+
) {
|
|
165
|
+
this.deadline = new ProcessDeadline({
|
|
166
|
+
timeoutMs: TIMEOUT_MS,
|
|
167
|
+
signal: options.signal,
|
|
168
|
+
interruption: (reason) =>
|
|
169
|
+
new Error(
|
|
170
|
+
reason === "timeout"
|
|
171
|
+
? "fixture setup timed out"
|
|
172
|
+
: "fixture setup cancelled",
|
|
173
|
+
),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async run(): Promise<void> {
|
|
178
|
+
this.proc = Bun.spawn(this.setup.command, {
|
|
179
|
+
cwd: this.options.workspace,
|
|
180
|
+
env: executionEnvironment(this.setup, this.options, this.home),
|
|
181
|
+
detached: process.platform !== "win32",
|
|
182
|
+
stdin: "ignore",
|
|
183
|
+
stdout: "ignore",
|
|
184
|
+
stderr: "ignore",
|
|
185
|
+
});
|
|
186
|
+
const code = await this.deadline.wait(this.proc.exited);
|
|
187
|
+
if (code !== 0) throw new Error(`fixture setup failed (${code})`);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async stop(): Promise<void> {
|
|
191
|
+
if (!this.proc) return;
|
|
192
|
+
terminateOwnedProcessGroup(this.proc);
|
|
193
|
+
await this.proc.exited;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async dispose(): Promise<void> {
|
|
197
|
+
this.deadline.dispose();
|
|
198
|
+
await rm(this.home, { recursive: true, force: true });
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Run setup after fixture history and before evaluator artifacts are mounted. */
|
|
203
|
+
export async function runFixtureSetup(
|
|
204
|
+
setup: FixtureSetup,
|
|
205
|
+
options: FixtureSetupOptions,
|
|
206
|
+
): Promise<void> {
|
|
207
|
+
if (options.signal?.aborted) throw new Error("fixture setup cancelled");
|
|
208
|
+
await requireCandidateRuntime(options.candidateRuntimeRoot);
|
|
209
|
+
if (options.candidateRuntimeRoot)
|
|
210
|
+
await mkdir(join(options.candidateRuntimeRoot, "home"), {
|
|
211
|
+
recursive: true,
|
|
212
|
+
mode: 0o700,
|
|
213
|
+
});
|
|
214
|
+
const home = await mkdtemp(join(tmpdir(), "sevro-setup-"));
|
|
215
|
+
const execution = new FixtureSetupExecution(setup, options, home);
|
|
216
|
+
try {
|
|
217
|
+
await execution.run();
|
|
218
|
+
} catch (error) {
|
|
219
|
+
await execution.stop();
|
|
220
|
+
throw error;
|
|
221
|
+
} finally {
|
|
222
|
+
await execution.dispose();
|
|
223
|
+
}
|
|
224
|
+
}
|