@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/CONTRIBUTING.md +170 -0
  2. package/LICENSE +63 -136
  3. package/README.md +83 -66
  4. package/docs/README.md +44 -0
  5. package/docs/acknowledgements.md +14 -0
  6. package/docs/architecture.md +146 -0
  7. package/docs/assets/README.md +27 -0
  8. package/docs/assets/sevro-logo.png +0 -0
  9. package/docs/assets/sevro-symbol.svg +1 -0
  10. package/docs/assets/sevro-wordmark.svg +1 -0
  11. package/docs/builtin-graders-v1.md +41 -13
  12. package/docs/codex-host.md +184 -52
  13. package/docs/contracts.md +19 -0
  14. package/docs/creating-cases.md +139 -0
  15. package/docs/development-cli.md +190 -30
  16. package/docs/documentation-quality.md +61 -0
  17. package/docs/evaluation-workflow.md +110 -0
  18. package/docs/evidence/code-mode-controls-71.md +64 -0
  19. package/docs/evidence/code-mode-programs-85.md +64 -0
  20. package/docs/evidence/codex-state-73/README.md +204 -0
  21. package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
  22. package/docs/evidence/codex-state-73/results.json +661 -0
  23. package/docs/evidence/local-http-74.md +104 -0
  24. package/docs/evidence/runtime-startup-30/README.md +185 -0
  25. package/docs/evidence/runtime-startup-30/after.json +319 -0
  26. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
  27. package/docs/evidence/runtime-startup-30/before.json +403 -0
  28. package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
  29. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
  30. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
  31. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
  32. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
  33. package/docs/evidence/skill-reads-70/results.json +823 -0
  34. package/docs/evidence.md +21 -0
  35. package/docs/extension-protocol-v1.md +50 -16
  36. package/docs/fixture-cleanup-validation.md +3 -0
  37. package/docs/getting-started.md +102 -0
  38. package/docs/guide-evaluation.md +89 -0
  39. package/docs/guide-verification.json +239 -0
  40. package/docs/identity-v1.md +35 -1
  41. package/docs/installing.md +88 -0
  42. package/docs/licenses/BUSL-1.1-0.1.0-rc.2.txt +159 -0
  43. package/docs/licensing.md +42 -0
  44. package/docs/native-controls.md +76 -5
  45. package/docs/native-hosts.md +228 -0
  46. package/docs/reading-results.md +141 -0
  47. package/docs/recipes.md +489 -0
  48. package/docs/releases.md +81 -40
  49. package/docs/report-v1.md +29 -1
  50. package/docs/results-v1.md +180 -0
  51. package/docs/run-settings-v1.md +147 -0
  52. package/docs/running-evaluations.md +31 -0
  53. package/docs/runtime-v1.md +614 -0
  54. package/docs/specs/repository-guide.md +58 -0
  55. package/docs/troubleshooting.md +30 -0
  56. package/docs/typescript-quality.md +234 -0
  57. package/docs/usability-study.md +114 -0
  58. package/examples/basic/host.ts +2 -2
  59. package/examples/recipes/candidate.ts +26 -0
  60. package/examples/recipes/cases/coding-change.json +21 -0
  61. package/examples/recipes/cases/host-tools.json +21 -0
  62. package/examples/recipes/cases/mise.json +21 -0
  63. package/examples/recipes/cases/node.json +21 -0
  64. package/examples/recipes/cases/python-uv.json +21 -0
  65. package/examples/recipes/cases/semantic.json +17 -0
  66. package/examples/recipes/coding-change/check.ts +23 -0
  67. package/examples/recipes/extension-artifacts.ts +65 -0
  68. package/examples/recipes/extension-methods.ts +174 -0
  69. package/examples/recipes/extension.ts +37 -0
  70. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  71. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  72. package/examples/recipes/repository-skill/README.md +3 -0
  73. package/examples/recipes/run-inputs.ts +188 -0
  74. package/examples/recipes/run-options.ts +131 -0
  75. package/examples/recipes/run.ts +63 -0
  76. package/examples/recipes/runtime/empty.json +1 -0
  77. package/examples/recipes/runtime/goals.json +1 -0
  78. package/examples/recipes/runtime/mise.json +6 -0
  79. package/examples/recipes/runtime/path.json +1 -0
  80. package/examples/recipes/runtime/plugins.json +4 -0
  81. package/examples/recipes/runtime/python-uv.json +10 -0
  82. package/examples/recipes/semantic-judge.ts +29 -0
  83. package/examples/templates/coding/adapter.ts +17 -0
  84. package/examples/templates/coding/case.json +15 -0
  85. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  86. package/examples/templates/repository-skill/extension-data.ts +64 -0
  87. package/examples/templates/repository-skill/extension.ts +31 -0
  88. package/examples/templates/response/adapter.ts +12 -0
  89. package/examples/templates/response/case.json +13 -0
  90. package/package.json +39 -6
  91. package/schemas/report-v1.schema.json +9 -0
  92. package/schemas/run-evidence-v1.schema.json +2 -1
  93. package/schemas/runtime-v1.schema.json +193 -0
  94. package/src/advisory.ts +10 -5
  95. package/src/candidate-artifacts.ts +208 -0
  96. package/src/candidate-codex-state.ts +185 -0
  97. package/src/case-validation.ts +105 -0
  98. package/src/cli/cancellation.ts +20 -0
  99. package/src/cli/command.ts +81 -0
  100. package/src/cli/completion.ts +15 -0
  101. package/src/cli/demo.ts +59 -0
  102. package/src/cli/evaluation.ts +55 -0
  103. package/src/cli/extension.ts +79 -0
  104. package/src/cli/help.ts +222 -0
  105. package/src/cli/hosts.ts +59 -0
  106. package/src/cli/init-settings.ts +30 -0
  107. package/src/cli/init.ts +111 -0
  108. package/src/cli/invocation.ts +627 -0
  109. package/src/cli/loading.ts +151 -0
  110. package/src/cli/preparation.ts +163 -0
  111. package/src/cli/presentation.ts +91 -0
  112. package/src/cli/progress.ts +6 -0
  113. package/src/cli/run-manifest.ts +133 -0
  114. package/src/cli/run-options.ts +101 -0
  115. package/src/cli/run-overrides.ts +83 -0
  116. package/src/cli/settings.ts +160 -0
  117. package/src/cli/summary.ts +43 -0
  118. package/src/cli.ts +13 -843
  119. package/src/contract-types.ts +94 -0
  120. package/src/engine.ts +19 -2373
  121. package/src/evaluation/advisory.ts +150 -0
  122. package/src/evaluation/check-plan.ts +510 -0
  123. package/src/evaluation/collection.ts +88 -0
  124. package/src/evaluation/context.ts +10 -0
  125. package/src/evaluation/grading-host.ts +39 -0
  126. package/src/evaluation/grading.ts +258 -0
  127. package/src/evaluation/invocation-plan.ts +548 -0
  128. package/src/evaluation/plan.ts +736 -0
  129. package/src/evaluation/progress.ts +19 -0
  130. package/src/evaluation/resources.ts +241 -0
  131. package/src/evaluation/run.ts +206 -0
  132. package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +3 -1
  133. package/src/evaluation/semantic.ts +312 -0
  134. package/src/evaluation/trial.ts +866 -0
  135. package/src/evaluation/types.ts +207 -0
  136. package/src/evidence/host-results.ts +92 -0
  137. package/src/evidence/retained-artifacts.ts +18 -0
  138. package/src/evidence/trial-writer.ts +203 -0
  139. package/src/evidence/usage.ts +17 -0
  140. package/src/extension-client.ts +228 -148
  141. package/src/extension-session.ts +326 -195
  142. package/src/fixtures/advisory.ts +338 -0
  143. package/src/fixtures/bin.ts +57 -0
  144. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  145. package/src/fixtures/create.ts +187 -0
  146. package/src/fixtures/generated.ts +300 -0
  147. package/src/fixtures/git-hooks.ts +24 -0
  148. package/src/fixtures/preparation.ts +270 -0
  149. package/src/{repository-fixture.ts → fixtures/repository.ts} +207 -85
  150. package/src/fixtures/setup.ts +224 -0
  151. package/src/fixtures/tools.ts +88 -0
  152. package/src/fixtures/workspace-state.ts +212 -0
  153. package/src/generated/cli-result.cjs +2 -0
  154. package/src/generated/cli-result.d.cts +3 -0
  155. package/src/generated/extension.cjs +2 -0
  156. package/src/generated/extension.d.cts +3 -0
  157. package/src/generated/report.cjs +2 -0
  158. package/src/generated/report.d.cts +3 -0
  159. package/src/generated/run-evidence.cjs +2 -0
  160. package/src/generated/run-evidence.d.cts +3 -0
  161. package/src/generated/runtime.cjs +2 -0
  162. package/src/generated/runtime.d.cts +3 -0
  163. package/src/graders/artifact.ts +56 -23
  164. package/src/graders/git-head.ts +79 -75
  165. package/src/graders/output.ts +222 -140
  166. package/src/graders/semantic.ts +102 -43
  167. package/src/graders/shell.ts +379 -207
  168. package/src/host-diagnostic-redaction.ts +52 -0
  169. package/src/host-diagnostics.ts +319 -0
  170. package/src/hosts/claude-continuation.ts +63 -18
  171. package/src/hosts/claude-credential.ts +43 -62
  172. package/src/hosts/claude-events.ts +86 -51
  173. package/src/hosts/claude-keychain.ts +59 -0
  174. package/src/hosts/claude-native-goal.ts +57 -33
  175. package/src/hosts/claude-process.ts +111 -0
  176. package/src/hosts/claude-repository-invocation.ts +208 -117
  177. package/src/hosts/claude-settings.ts +240 -21
  178. package/src/hosts/claude-skill-activations.ts +166 -0
  179. package/src/hosts/claude-state.ts +403 -0
  180. package/src/hosts/claude-tool-calls.ts +171 -106
  181. package/src/hosts/claude.ts +561 -451
  182. package/src/hosts/codex-app-server.ts +326 -178
  183. package/src/hosts/codex-configuration.ts +44 -18
  184. package/src/hosts/codex-events.ts +97 -60
  185. package/src/hosts/codex-exec-controls.ts +516 -0
  186. package/src/hosts/codex-invocation.ts +70 -0
  187. package/src/hosts/codex-marketplace.ts +283 -0
  188. package/src/hosts/codex-native-calls.ts +546 -500
  189. package/src/hosts/codex-process.ts +202 -0
  190. package/src/hosts/codex-profile.ts +257 -28
  191. package/src/hosts/codex-session-files.ts +106 -0
  192. package/src/hosts/codex-state.ts +492 -0
  193. package/src/hosts/codex.ts +783 -832
  194. package/src/hosts/isolation-common.ts +94 -0
  195. package/src/hosts/isolation-roots.ts +54 -27
  196. package/src/hosts/isolation.ts +10 -0
  197. package/src/hosts/linux-sandbox.ts +159 -0
  198. package/src/hosts/mac-sandbox.ts +187 -61
  199. package/src/hosts/native-controls.ts +152 -62
  200. package/src/hosts/process-lifecycle.ts +37 -0
  201. package/src/hosts/protected-worktrees.ts +51 -19
  202. package/src/hosts/skill-hook-catalog.ts +131 -0
  203. package/src/hosts/skill-hook-handler.ts +374 -0
  204. package/src/hosts/skill-hooks.ts +580 -0
  205. package/src/hosts/workspace-fingerprint.ts +55 -35
  206. package/src/identity.ts +102 -48
  207. package/src/instrumentation.ts +109 -48
  208. package/src/native-transcript-state.ts +131 -0
  209. package/src/native-transcript-view.ts +90 -0
  210. package/src/native-transcripts.ts +286 -0
  211. package/src/process-lifecycle.ts +150 -0
  212. package/src/provenance.ts +161 -81
  213. package/src/prune.ts +215 -0
  214. package/src/report.ts +172 -140
  215. package/src/resolved-case.ts +33 -0
  216. package/src/results-lock.ts +29 -0
  217. package/src/results.ts +133 -86
  218. package/src/run-owner.ts +154 -63
  219. package/src/runtime/config.ts +456 -0
  220. package/src/runtime/discovery-apple.ts +90 -0
  221. package/src/runtime/discovery-providers.ts +63 -0
  222. package/src/runtime/discovery.ts +70 -0
  223. package/src/runtime/hook-processes.ts +97 -0
  224. package/src/runtime/hooks.ts +476 -0
  225. package/src/runtime/paths.ts +313 -0
  226. package/src/runtime/query.ts +85 -0
  227. package/src/runtime/seeds.ts +134 -0
  228. package/src/runtime/shared-cache.ts +143 -0
  229. package/src/runtime/state.ts +143 -0
  230. package/src/schema-types.ts +198 -0
  231. package/src/schema.ts +15 -19
  232. package/src/transcript-retention.ts +146 -0
  233. package/src/value-guards.ts +20 -0
  234. package/src/advisory-fixture.ts +0 -282
  235. package/src/fixture-bin.ts +0 -87
  236. package/src/fixture-setup.ts +0 -163
  237. package/src/generated-fixture.ts +0 -220
  238. package/src/git-hooks.ts +0 -63
  239. package/src/hosts/claude-nested-skills.ts +0 -218
  240. package/src/hosts/codex-command-output.ts +0 -399
  241. package/src/hosts/codex-executor-command.ts +0 -195
  242. package/src/hosts/codex-skill-reads.ts +0 -391
  243. package/src/preparation.ts +0 -211
@@ -1,8 +1,10 @@
1
+ import { isRecord, isStringArray } from "../value-guards";
2
+ import { isMissingFile } from "./tools";
1
3
  import { lstat, mkdir, realpath, stat, writeFile } from "node:fs/promises";
2
4
  import { isAbsolute, join, relative, sep } from "node:path";
3
5
  import { fileURLToPath } from "node:url";
4
- import { canonicalJson } from "./identity";
5
- import { prepareFixtureBin } from "./fixture-bin";
6
+ import { canonicalJson } from "../identity";
7
+ import { prepareFixtureBin } from "./bin";
6
8
  import { prepareGitHooks } from "./git-hooks";
7
9
  import type { PreparationSources } from "./preparation";
8
10
  import { fixtureParts } from "./preparation";
@@ -21,53 +23,110 @@ export interface RepositoryFixture {
21
23
  bin?: Record<string, string>;
22
24
  }
23
25
 
24
- /** Validate repository source identity and bounded working-tree changes. */
25
- export function prepareRepositoryFixture(value: unknown): RepositoryFixture {
26
- if (!value || typeof value !== "object" || Array.isArray(value))
27
- throw new Error("invalid repository fixture");
28
- const fixture = value as Record<string, unknown>;
26
+ function repositoryRecord(value: unknown): Record<string, unknown> {
27
+ if (!isRecord(value)) throw new Error("invalid repository fixture");
28
+ return value;
29
+ }
30
+ function requireRepositoryIdentity(
31
+ fixture: Record<string, unknown>,
32
+ ): asserts fixture is Record<string, unknown> & {
33
+ kind: "repository";
34
+ sourceRef: string;
35
+ } {
36
+ const allowed = [
37
+ "kind",
38
+ "sourceRef",
39
+ "files",
40
+ "staged",
41
+ "commitFiles",
42
+ "hooks",
43
+ "bin",
44
+ ];
29
45
  if (
30
46
  fixture.kind !== "repository" ||
31
- Object.keys(fixture).some(
32
- (key) =>
33
- ![
34
- "kind",
35
- "sourceRef",
36
- "files",
37
- "staged",
38
- "commitFiles",
39
- "hooks",
40
- "bin",
41
- ].includes(key),
42
- ) ||
47
+ Object.keys(fixture).some((key) => !allowed.includes(key)) ||
43
48
  typeof fixture.sourceRef !== "string" ||
44
- !fixture.sourceRef ||
45
- (fixture.files !== undefined &&
46
- (!fixture.files ||
47
- typeof fixture.files !== "object" ||
48
- Array.isArray(fixture.files) ||
49
- Object.values(fixture.files).some(
50
- (item) => typeof item !== "string",
51
- ))) ||
52
- (fixture.staged !== undefined &&
53
- (!Array.isArray(fixture.staged) ||
54
- fixture.staged.some((item) => typeof item !== "string"))) ||
55
- (fixture.commitFiles !== undefined &&
56
- typeof fixture.commitFiles !== "boolean")
49
+ !fixture.sourceRef
50
+ )
51
+ throw new Error("invalid repository fixture");
52
+ }
53
+ function optionalRepositoryFiles(
54
+ value: unknown,
55
+ ): value is Record<string, string> | undefined {
56
+ return (
57
+ value === undefined ||
58
+ (isRecord(value) &&
59
+ Object.values(value).every((item) => typeof item === "string"))
60
+ );
61
+ }
62
+ function optionalRepositoryStaging(
63
+ value: unknown,
64
+ ): value is string[] | undefined {
65
+ return value === undefined || isStringArray(value);
66
+ }
67
+ function optionalRepositoryCommit(
68
+ value: unknown,
69
+ ): value is boolean | undefined {
70
+ return value === undefined || typeof value === "boolean";
71
+ }
72
+ function requireRepositoryOptions(
73
+ fixture: Record<string, unknown>,
74
+ ): asserts fixture is Record<string, unknown> & {
75
+ files?: Record<string, string>;
76
+ staged?: string[];
77
+ commitFiles?: boolean;
78
+ } {
79
+ if (
80
+ !optionalRepositoryFiles(fixture.files) ||
81
+ !optionalRepositoryStaging(fixture.staged) ||
82
+ !optionalRepositoryCommit(fixture.commitFiles)
57
83
  )
58
84
  throw new Error("invalid repository fixture");
85
+ }
86
+ function requireRepositoryTools(
87
+ fixture: Record<string, unknown>,
88
+ ): asserts fixture is Record<string, unknown> & {
89
+ hooks?: Record<string, string>;
90
+ bin?: Record<string, string>;
91
+ } {
59
92
  prepareGitHooks(fixture.hooks);
60
93
  prepareFixtureBin(fixture.bin);
61
- const files = (fixture.files ?? {}) as Record<string, string>;
62
- const staged = (fixture.staged ?? []) as string[];
94
+ }
95
+ function repositoryFixtureData(value: unknown): RepositoryFixture {
96
+ const fixture = repositoryRecord(value);
97
+ requireRepositoryIdentity(fixture);
98
+ requireRepositoryOptions(fixture);
99
+ requireRepositoryTools(fixture);
100
+ return fixture;
101
+ }
102
+ function requireOverlayPath(path: string): void {
103
+ if (fixtureParts(path).some((part) => part.toLowerCase() === ".git"))
104
+ throw new Error("repository fixture cannot write repository metadata");
105
+ }
106
+ function overlayPathsCollide(left: string, right: string): boolean {
107
+ return (
108
+ left === right ||
109
+ left.startsWith(`${right}/`) ||
110
+ right.startsWith(`${left}/`)
111
+ );
112
+ }
113
+ function requireDistinctOverlayPaths(paths: string[]): void {
114
+ const normalized = paths.map((path) => path.toLowerCase());
115
+ for (const [index, path] of normalized.entries())
116
+ for (const other of normalized.slice(0, index))
117
+ if (overlayPathsCollide(path, other))
118
+ throw new Error("repository fixture overlay paths collide");
119
+ }
120
+ function requireBoundedOverlay(
121
+ files: Record<string, string>,
122
+ staged: string[],
123
+ ): void {
63
124
  const paths = Object.keys(files);
64
125
  if (paths.length > MAX_OVERLAY_FILES)
65
126
  throw new Error("repository fixture exceeds the file limit");
66
127
  let bytes = 0;
67
128
  for (const [path, content] of Object.entries(files)) {
68
- const parts = fixtureParts(path);
69
- if (parts.some((part) => part.toLowerCase() === ".git"))
70
- throw new Error("repository fixture cannot write repository metadata");
129
+ requireOverlayPath(path);
71
130
  bytes +=
72
131
  Buffer.byteLength(path, "utf8") + Buffer.byteLength(content, "utf8");
73
132
  }
@@ -77,24 +136,30 @@ export function prepareRepositoryFixture(value: unknown): RepositoryFixture {
77
136
  );
78
137
  if (bytes > MAX_OVERLAY_BYTES)
79
138
  throw new Error("repository fixture exceeds the size limit");
80
- const normalized = paths.map((path) => path.toLowerCase());
81
- for (let index = 0; index < normalized.length; index++) {
82
- for (let other = 0; other < index; other++) {
83
- if (
84
- normalized[index] === normalized[other] ||
85
- normalized[index]!.startsWith(`${normalized[other]}/`) ||
86
- normalized[other]!.startsWith(`${normalized[index]}/`)
87
- )
88
- throw new Error("repository fixture overlay paths collide");
89
- }
90
- }
139
+ requireDistinctOverlayPaths(paths);
140
+ }
141
+ function requireRepositoryStaging(
142
+ fixture: RepositoryFixture,
143
+ files: Record<string, string>,
144
+ staged: string[],
145
+ ): void {
91
146
  if (
92
147
  new Set(staged.map((path) => path.toLowerCase())).size !== staged.length ||
93
148
  staged.some((path) => !Object.hasOwn(files, path)) ||
94
- (fixture.commitFiles === true && paths.length === 0)
149
+ (fixture.commitFiles === true && Object.keys(files).length === 0)
95
150
  )
96
151
  throw new Error("invalid repository fixture staging");
97
- return JSON.parse(canonicalJson(fixture)) as RepositoryFixture;
152
+ }
153
+
154
+ /** Validate repository source identity and bounded working-tree changes. */
155
+ export function prepareRepositoryFixture(value: unknown): RepositoryFixture {
156
+ const fixture = repositoryFixtureData(value);
157
+ const files = fixture.files ?? {},
158
+ staged = fixture.staged ?? [];
159
+ requireBoundedOverlay(files, staged);
160
+ requireRepositoryStaging(fixture, files, staged);
161
+ const clone: unknown = JSON.parse(canonicalJson(fixture));
162
+ return repositoryFixtureData(clone);
98
163
  }
99
164
 
100
165
  export interface RepositorySource {
@@ -137,7 +202,7 @@ async function git(args: string[], cwd: string): Promise<string> {
137
202
  },
138
203
  });
139
204
  try {
140
- const [stdout, _stderr, code] = await Promise.all([
205
+ const [stdout, , code] = await Promise.all([
141
206
  boundedText(proc.stdout),
142
207
  boundedText(proc.stderr),
143
208
  proc.exited,
@@ -156,38 +221,78 @@ export async function resolveRepositorySource(
156
221
  sourceRef: string,
157
222
  sources?: PreparationSources,
158
223
  ): Promise<RepositorySource> {
224
+ const selected = repositorySourceSelection(sourceRef, sources);
225
+ const { root, path } = await canonicalRepositorySource(selected);
226
+ requireContainedRepository(root, path);
227
+ return { path, revision: await repositoryCommit(path) };
228
+ }
229
+
230
+ function requireRepositorySources(
231
+ sources: PreparationSources | undefined,
232
+ ): asserts sources is PreparationSources {
159
233
  if (!sources || !isAbsolute(sources.root))
160
234
  throw new Error("repository source root is required and must be absolute");
235
+ }
236
+ function repositorySourceSelection(
237
+ sourceRef: string,
238
+ sources: PreparationSources | undefined,
239
+ ) {
240
+ requireRepositorySources(sources);
161
241
  if (!Object.hasOwn(sources.refs, sourceRef))
162
242
  throw new Error("repository source reference is not declared");
163
243
  const url = sources.refs[sourceRef];
164
244
  if (typeof url !== "string" || !url.startsWith("file:///"))
165
245
  throw new Error("repository source reference must be a file URL");
166
- let root: string;
167
- let path: string;
246
+ return { root: sources.root, url };
247
+ }
248
+ async function canonicalRepositorySource(selected: {
249
+ root: string;
250
+ url: string;
251
+ }) {
168
252
  try {
169
- [root, path] = await Promise.all([
170
- realpath(sources.root),
171
- realpath(fileURLToPath(url)),
253
+ const [root, path] = await Promise.all([
254
+ realpath(selected.root),
255
+ realpath(fileURLToPath(selected.url)),
172
256
  ]);
173
257
  if (!(await stat(path)).isDirectory()) throw new Error();
174
- } catch {
175
- throw new Error("repository source is unreadable");
258
+ return { root, path };
259
+ } catch (cause) {
260
+ throw new Error("repository source is unreadable", { cause });
176
261
  }
262
+ }
263
+ function requireContainedRepository(root: string, path: string): void {
177
264
  const child = relative(root, path);
178
265
  if (child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child))
179
266
  throw new Error("repository source escapes its declared root");
267
+ }
268
+ async function repositoryCommit(path: string): Promise<string> {
180
269
  const top = await git(["rev-parse", "--show-toplevel"], path);
181
270
  if ((await realpath(top)) !== path)
182
271
  throw new Error("repository source must name the repository root");
183
272
  const revision = await git(["rev-parse", "--verify", "HEAD^{commit}"], path);
184
273
  if (!/^[a-f0-9]{40,64}$/.test(revision))
185
274
  throw new Error("repository source has no valid commit");
186
- if (await git(["status", "--porcelain=v1", "--untracked-files=all"], path))
275
+ if (
276
+ await git(
277
+ [
278
+ "status",
279
+ "--porcelain=v1",
280
+ "--untracked-files=all",
281
+ "--ignore-submodules=none",
282
+ ],
283
+ path,
284
+ )
285
+ )
187
286
  throw new Error("repository source has uncommitted changes");
188
- if (await git(["submodule", "status"], path))
189
- throw new Error("repository submodules are not supported");
190
- return { path, revision };
287
+ return revision;
288
+ }
289
+ function requireRepositorySnapshot(
290
+ current: RepositorySource,
291
+ expected: RepositorySource,
292
+ message: string,
293
+ ): void {
294
+ if (current.path !== expected.path || current.revision !== expected.revision)
295
+ throw new Error(message);
191
296
  }
192
297
 
193
298
  /** Clone a fixed commit without hardlinks or a remote back to the source. */
@@ -198,14 +303,18 @@ export async function cloneRepositorySource(
198
303
  workspace: string,
199
304
  ): Promise<void> {
200
305
  const current = await resolveRepositorySource(sourceRef, sources);
201
- if (current.path !== expected.path || current.revision !== expected.revision)
202
- throw new Error("repository source changed during run");
306
+ requireRepositorySnapshot(
307
+ current,
308
+ expected,
309
+ "repository source changed during run",
310
+ );
203
311
  await git(
204
312
  [
205
313
  "clone",
206
314
  "--local",
207
315
  "--no-hardlinks",
208
316
  "--no-checkout",
317
+ "--no-recurse-submodules",
209
318
  "--quiet",
210
319
  current.path,
211
320
  workspace,
@@ -220,8 +329,11 @@ export async function cloneRepositorySource(
220
329
  .filter(Boolean))
221
330
  await git(["remote", "remove", remote], workspace);
222
331
  const after = await resolveRepositorySource(sourceRef, sources);
223
- if (after.path !== expected.path || after.revision !== expected.revision)
224
- throw new Error("repository source changed during clone");
332
+ requireRepositorySnapshot(
333
+ after,
334
+ expected,
335
+ "repository source changed during clone",
336
+ );
225
337
  }
226
338
 
227
339
  async function safeOverlayTarget(
@@ -232,30 +344,31 @@ async function safeOverlayTarget(
232
344
  let parent = workspace;
233
345
  for (const part of parts.slice(0, -1)) {
234
346
  parent = join(parent, part);
235
- try {
236
- const entry = await lstat(parent);
237
- if (!entry.isDirectory() || entry.isSymbolicLink())
238
- throw new Error("repository fixture overlay traverses a non-directory");
239
- } catch (error) {
240
- if (!(
241
- error instanceof Error &&
242
- "code" in error &&
243
- error.code === "ENOENT"
244
- ))
245
- throw error;
246
- await mkdir(parent, { mode: 0o700 });
247
- }
347
+ await overlayParent(parent);
248
348
  }
249
349
  const target = join(workspace, ...parts);
350
+ await requireOverlayTarget(target);
351
+ return target;
352
+ }
353
+
354
+ async function overlayParent(path: string): Promise<void> {
250
355
  try {
251
- const entry = await lstat(target);
356
+ const entry = await lstat(path);
357
+ if (!entry.isDirectory() || entry.isSymbolicLink())
358
+ throw new Error("repository fixture overlay traverses a non-directory");
359
+ } catch (error) {
360
+ if (!isMissingFile(error)) throw error;
361
+ await mkdir(path, { mode: 0o700 });
362
+ }
363
+ }
364
+ async function requireOverlayTarget(path: string): Promise<void> {
365
+ try {
366
+ const entry = await lstat(path);
252
367
  if (!entry.isFile() || entry.isSymbolicLink())
253
368
  throw new Error("repository fixture overlay targets a non-file");
254
369
  } catch (error) {
255
- if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
256
- throw error;
370
+ if (!isMissingFile(error)) throw error;
257
371
  }
258
- return target;
259
372
  }
260
373
 
261
374
  /** Apply declared files after cloning, optionally committing or staging them. */
@@ -269,11 +382,20 @@ export async function applyRepositoryOverlay(
269
382
  flag: "w",
270
383
  mode: 0o600,
271
384
  });
272
- const paths = Object.keys(files);
273
- const noHooks = [
385
+ await stageRepositoryOverlay(fixture, workspace, Object.keys(files));
386
+ }
387
+ function disabledHooksArgs(): string[] {
388
+ return [
274
389
  "-c",
275
390
  `core.hooksPath=${process.platform === "win32" ? "NUL" : "/dev/null"}`,
276
391
  ];
392
+ }
393
+ async function stageRepositoryOverlay(
394
+ fixture: RepositoryFixture,
395
+ workspace: string,
396
+ paths: string[],
397
+ ): Promise<void> {
398
+ const noHooks = disabledHooksArgs();
277
399
  if (fixture.commitFiles) {
278
400
  await git([...noHooks, "add", "--", ...paths], workspace);
279
401
  await git(
@@ -0,0 +1,224 @@
1
+ import { mkdir, mkdtemp, rm } from "node:fs/promises";
2
+ import { tmpdir } from "node:os";
3
+ import { delimiter, isAbsolute, join } from "node:path";
4
+ import { isRecord } from "../value-guards";
5
+ import { requireCandidateRuntime } from "../candidate-artifacts";
6
+ import {
7
+ ProcessDeadline,
8
+ terminateOwnedProcessGroup,
9
+ } from "../process-lifecycle";
10
+
11
+ const MAX_COMMAND_BYTES = 64 * 1024;
12
+ const MAX_ENV_BYTES = 16 * 1024;
13
+ const TIMEOUT_MS = 120_000;
14
+ const RESERVED_ENV = new Set([
15
+ "PATH",
16
+ "HOME",
17
+ "TMPDIR",
18
+ "GIT_CONFIG_GLOBAL",
19
+ "GIT_CONFIG_NOSYSTEM",
20
+ "GIT_TERMINAL_PROMPT",
21
+ ]);
22
+
23
+ export interface FixtureSetupDeclaration {
24
+ command: string[];
25
+ environment?: Record<string, string>;
26
+ }
27
+
28
+ export interface FixtureSetup extends FixtureSetupDeclaration {
29
+ environment: Record<string, string>;
30
+ }
31
+
32
+ function setupRecord(value: unknown): Record<string, unknown> {
33
+ if (!isRecord(value)) throw new Error("invalid fixture setup");
34
+ return value;
35
+ }
36
+
37
+ function validArgument(value: unknown): value is string {
38
+ return typeof value === "string" && value.length > 0 && !value.includes("\0");
39
+ }
40
+
41
+ function argumentList(value: unknown): value is string[] {
42
+ return (
43
+ Array.isArray(value) &&
44
+ value.length > 0 &&
45
+ value.length <= 16 &&
46
+ value.every(validArgument)
47
+ );
48
+ }
49
+
50
+ function setupCommand(value: unknown): string[] {
51
+ if (!argumentList(value)) throw new Error("invalid fixture setup command");
52
+ const executable = value[0];
53
+ if (
54
+ !absoluteExecutable(executable) ||
55
+ Buffer.byteLength(value.join("\0"), "utf8") > MAX_COMMAND_BYTES
56
+ )
57
+ throw new Error("invalid fixture setup command");
58
+ return value;
59
+ }
60
+
61
+ function absoluteExecutable(value: string | undefined): boolean {
62
+ return value !== undefined && isAbsolute(value);
63
+ }
64
+
65
+ function validEnvironmentEntry([name, value]: [string, unknown]): boolean {
66
+ return (
67
+ /^[A-Z][A-Z0-9_]*$/.test(name) &&
68
+ !RESERVED_ENV.has(name) &&
69
+ validEnvironmentValue(value)
70
+ );
71
+ }
72
+
73
+ function validEnvironmentValue(value: unknown): value is string {
74
+ return (
75
+ typeof value === "string" &&
76
+ !value.includes("\0") &&
77
+ !/\{\{(?!sevro\.(?:project|workspace)\}\})/.test(value)
78
+ );
79
+ }
80
+
81
+ function environmentRecord(value: unknown): Record<string, unknown> {
82
+ if (!isRecord(value)) throw new Error("invalid fixture setup environment");
83
+ return value;
84
+ }
85
+
86
+ function setupEnvironment(value: unknown): Record<string, string> {
87
+ const environment = environmentRecord(value);
88
+ const entries = Object.entries(environment);
89
+ if (
90
+ entries.length > 32 ||
91
+ entries.some((entry) => !validEnvironmentEntry(entry)) ||
92
+ Buffer.byteLength(JSON.stringify(environment), "utf8") > MAX_ENV_BYTES
93
+ )
94
+ throw new Error("invalid fixture setup environment");
95
+ return environment as Record<string, string>;
96
+ }
97
+
98
+ /** Validate a trusted extension's bounded fixture setup command. */
99
+ export function prepareFixtureSetup(value: unknown): FixtureSetup | null {
100
+ if (value === undefined) return null;
101
+ const input = setupRecord(value);
102
+ if (
103
+ Object.keys(input).some((key) => key !== "command" && key !== "environment")
104
+ )
105
+ throw new Error("unsupported fixture setup field");
106
+ const command = setupCommand(input.command);
107
+ const environment = setupEnvironment(input.environment ?? {});
108
+ return { command, environment };
109
+ }
110
+
111
+ interface FixtureSetupOptions {
112
+ workspace: string;
113
+ projectRoot: string;
114
+ fixtureBinDir?: string;
115
+ signal?: AbortSignal;
116
+ candidateRuntimeRoot?: string;
117
+ }
118
+
119
+ function setupPath(directory: string | undefined): string {
120
+ const inherited = process.env.PATH ?? "/usr/bin:/bin";
121
+ return directory ? `${directory}${delimiter}${inherited}` : inherited;
122
+ }
123
+
124
+ function executionEnvironment(
125
+ setup: FixtureSetup,
126
+ options: FixtureSetupOptions,
127
+ home: string,
128
+ ) {
129
+ const environment = Object.fromEntries(
130
+ Object.entries(setup.environment).map(([name, value]) => [
131
+ name,
132
+ value
133
+ .replaceAll("{{sevro.project}}", options.projectRoot)
134
+ .replaceAll("{{sevro.workspace}}", options.workspace),
135
+ ]),
136
+ );
137
+ return {
138
+ PATH: setupPath(options.fixtureBinDir),
139
+ HOME: options.candidateRuntimeRoot
140
+ ? join(options.candidateRuntimeRoot, "home")
141
+ : home,
142
+ TMPDIR: home,
143
+ LANG: process.env.LANG ?? "C",
144
+ GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : "/dev/null",
145
+ GIT_CONFIG_NOSYSTEM: "1",
146
+ GIT_TERMINAL_PROMPT: "0",
147
+ GIT_AUTHOR_NAME: "Sevro Fixture",
148
+ GIT_AUTHOR_EMAIL: "fixture@sevro.invalid",
149
+ GIT_COMMITTER_NAME: "Sevro Fixture",
150
+ GIT_COMMITTER_EMAIL: "fixture@sevro.invalid",
151
+ GIT_AUTHOR_DATE: "2000-01-01T00:00:00+00:00",
152
+ GIT_COMMITTER_DATE: "2000-01-01T00:00:00+00:00",
153
+ ...environment,
154
+ };
155
+ }
156
+
157
+ class FixtureSetupExecution {
158
+ private proc: Bun.Subprocess | undefined;
159
+ private readonly deadline: ProcessDeadline;
160
+ constructor(
161
+ private readonly setup: FixtureSetup,
162
+ private readonly options: FixtureSetupOptions,
163
+ private readonly home: string,
164
+ ) {
165
+ this.deadline = new ProcessDeadline({
166
+ timeoutMs: TIMEOUT_MS,
167
+ signal: options.signal,
168
+ interruption: (reason) =>
169
+ new Error(
170
+ reason === "timeout"
171
+ ? "fixture setup timed out"
172
+ : "fixture setup cancelled",
173
+ ),
174
+ });
175
+ }
176
+
177
+ async run(): Promise<void> {
178
+ this.proc = Bun.spawn(this.setup.command, {
179
+ cwd: this.options.workspace,
180
+ env: executionEnvironment(this.setup, this.options, this.home),
181
+ detached: process.platform !== "win32",
182
+ stdin: "ignore",
183
+ stdout: "ignore",
184
+ stderr: "ignore",
185
+ });
186
+ const code = await this.deadline.wait(this.proc.exited);
187
+ if (code !== 0) throw new Error(`fixture setup failed (${code})`);
188
+ }
189
+
190
+ async stop(): Promise<void> {
191
+ if (!this.proc) return;
192
+ terminateOwnedProcessGroup(this.proc);
193
+ await this.proc.exited;
194
+ }
195
+
196
+ async dispose(): Promise<void> {
197
+ this.deadline.dispose();
198
+ await rm(this.home, { recursive: true, force: true });
199
+ }
200
+ }
201
+
202
+ /** Run setup after fixture history and before evaluator artifacts are mounted. */
203
+ export async function runFixtureSetup(
204
+ setup: FixtureSetup,
205
+ options: FixtureSetupOptions,
206
+ ): Promise<void> {
207
+ if (options.signal?.aborted) throw new Error("fixture setup cancelled");
208
+ await requireCandidateRuntime(options.candidateRuntimeRoot);
209
+ if (options.candidateRuntimeRoot)
210
+ await mkdir(join(options.candidateRuntimeRoot, "home"), {
211
+ recursive: true,
212
+ mode: 0o700,
213
+ });
214
+ const home = await mkdtemp(join(tmpdir(), "sevro-setup-"));
215
+ const execution = new FixtureSetupExecution(setup, options, home);
216
+ try {
217
+ await execution.run();
218
+ } catch (error) {
219
+ await execution.stop();
220
+ throw error;
221
+ } finally {
222
+ await execution.dispose();
223
+ }
224
+ }