@bjoernrochel/sevro 0.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/LICENSE +159 -0
  2. package/README.md +71 -0
  3. package/docs/advisory-fixture.md +23 -0
  4. package/docs/builtin-graders-v1.md +77 -0
  5. package/docs/cache-ownership-validation.md +49 -0
  6. package/docs/claude-continuation-validation.md +47 -0
  7. package/docs/codex-host.md +134 -0
  8. package/docs/concurrency-validation.md +86 -0
  9. package/docs/configuration-root-validation.md +70 -0
  10. package/docs/development-cli.md +327 -0
  11. package/docs/engine-slice.md +114 -0
  12. package/docs/extension-protocol-v1.md +292 -0
  13. package/docs/fixture-cleanup-validation.md +70 -0
  14. package/docs/identity-v1.md +65 -0
  15. package/docs/native-controls.md +29 -0
  16. package/docs/releases.md +88 -0
  17. package/docs/report-v1.md +32 -0
  18. package/docs/results-v1.md +118 -0
  19. package/examples/basic/graded.json +13 -0
  20. package/examples/basic/host.ts +8 -0
  21. package/examples/basic/prompt-only.json +7 -0
  22. package/package.json +45 -0
  23. package/schemas/cli-result-v1.schema.json +184 -0
  24. package/schemas/extension-v1.schema.json +659 -0
  25. package/schemas/report-v1.schema.json +164 -0
  26. package/schemas/run-evidence-v1.schema.json +510 -0
  27. package/src/advisory-fixture.ts +282 -0
  28. package/src/advisory.ts +105 -0
  29. package/src/cli.ts +837 -0
  30. package/src/engine.ts +2341 -0
  31. package/src/extension-client.ts +257 -0
  32. package/src/extension-session.ts +335 -0
  33. package/src/fixture-bin.ts +87 -0
  34. package/src/fixture-cleanup.ts +38 -0
  35. package/src/fixture-setup.ts +163 -0
  36. package/src/generated-fixture.ts +220 -0
  37. package/src/git-hooks.ts +63 -0
  38. package/src/graders/git-head.ts +176 -0
  39. package/src/graders/output.ts +359 -0
  40. package/src/graders/semantic.ts +120 -0
  41. package/src/graders/shell.ts +277 -0
  42. package/src/hosts/claude-continuation.ts +94 -0
  43. package/src/hosts/claude-credential.ts +73 -0
  44. package/src/hosts/claude-events.ts +128 -0
  45. package/src/hosts/claude-nested-skills.ts +218 -0
  46. package/src/hosts/claude-repository-invocation.ts +239 -0
  47. package/src/hosts/claude-settings.ts +50 -0
  48. package/src/hosts/claude-tool-calls.ts +159 -0
  49. package/src/hosts/claude.ts +511 -0
  50. package/src/hosts/codex-configuration.ts +85 -0
  51. package/src/hosts/codex-events.ts +102 -0
  52. package/src/hosts/codex-native-calls.ts +702 -0
  53. package/src/hosts/codex-profile.ts +78 -0
  54. package/src/hosts/codex-skill-reads.ts +345 -0
  55. package/src/hosts/codex.ts +769 -0
  56. package/src/hosts/isolation-roots.ts +60 -0
  57. package/src/hosts/mac-sandbox.ts +108 -0
  58. package/src/hosts/native-controls.ts +124 -0
  59. package/src/hosts/protected-worktrees.ts +56 -0
  60. package/src/hosts/workspace-fingerprint.ts +55 -0
  61. package/src/identity.ts +156 -0
  62. package/src/instrumentation.ts +106 -0
  63. package/src/preparation.ts +211 -0
  64. package/src/provenance.ts +244 -0
  65. package/src/report.ts +351 -0
  66. package/src/repository-fixture.ts +286 -0
  67. package/src/results.ts +160 -0
  68. package/src/run-owner.ts +276 -0
  69. package/src/schema.ts +28 -0
  70. package/src/storage.ts +23 -0
  71. package/src/trial-scheduler.ts +27 -0
@@ -0,0 +1,78 @@
1
+ import { isAbsolute } from "node:path";
2
+
3
+ function toml(value: string): string {
4
+ if (/[\x00-\x1f\x7f]/.test(value))
5
+ throw new Error("Codex profile value contains a control character");
6
+ return JSON.stringify(value);
7
+ }
8
+
9
+ /** Runner-owned Codex command policy; the Codex parent keeps access to auth. */
10
+ export function codexPermissionProfile(options: {
11
+ id: string;
12
+ workspace: string;
13
+ commandHome: string;
14
+ commandTemp: string;
15
+ executableReadRoots: string[];
16
+ pluginReadRoot?: string;
17
+ protectedRoots: string[];
18
+ }): string {
19
+ if (!/^[a-z][a-z0-9_]*$/.test(options.id))
20
+ throw new Error("invalid Codex permission profile ID");
21
+ if (
22
+ !isAbsolute(options.workspace) ||
23
+ !isAbsolute(options.commandHome) ||
24
+ !isAbsolute(options.commandTemp) ||
25
+ !options.executableReadRoots.length ||
26
+ options.executableReadRoots.some(
27
+ (path) => !isAbsolute(path) || path === "/",
28
+ ) ||
29
+ (options.pluginReadRoot !== undefined &&
30
+ (!isAbsolute(options.pluginReadRoot) ||
31
+ options.pluginReadRoot === "/")) ||
32
+ !options.protectedRoots.length ||
33
+ options.protectedRoots.some((root) => !isAbsolute(root))
34
+ )
35
+ throw new Error("Codex profile paths must be absolute and protected");
36
+ const name = options.id;
37
+ return [
38
+ `default_permissions = ${toml(name)}`,
39
+ 'approval_policy = "never"',
40
+ "allow_login_shell = false",
41
+ "",
42
+ "[shell_environment_policy]",
43
+ 'inherit = "none"',
44
+ "ignore_default_excludes = false",
45
+ "",
46
+ "[shell_environment_policy.set]",
47
+ 'PATH = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"',
48
+ `HOME = ${toml(options.commandHome)}`,
49
+ `TMPDIR = ${toml(options.commandTemp)}`,
50
+ "",
51
+ `[permissions.${name}]`,
52
+ 'extends = ":workspace"',
53
+ "",
54
+ `[permissions.${name}.filesystem]`,
55
+ '":root" = "deny"',
56
+ '":minimal" = "read"',
57
+ '":tmpdir" = "deny"',
58
+ '":slash_tmp" = "deny"',
59
+ ...[...new Set(options.protectedRoots)]
60
+ .sort()
61
+ .map((root) => `${toml(root)} = "deny"`),
62
+ ...[...new Set(options.executableReadRoots)]
63
+ .sort()
64
+ .map((path) => `${toml(path)} = "read"`),
65
+ ...(options.pluginReadRoot
66
+ ? [`${toml(options.pluginReadRoot)} = "read"`]
67
+ : []),
68
+ `${toml(options.commandHome)} = "write"`,
69
+ `${toml(options.commandTemp)} = "write"`,
70
+ "",
71
+ `[permissions.${name}.filesystem.\":workspace_roots\"]`,
72
+ '"." = "write"',
73
+ "",
74
+ `[permissions.${name}.network]`,
75
+ "enabled = false",
76
+ "",
77
+ ].join("\n");
78
+ }
@@ -0,0 +1,345 @@
1
+ import { readFile, realpath } from "node:fs/promises";
2
+ import { isAbsolute, relative, resolve, sep } from "node:path";
3
+
4
+ const MAX_SKILL_BYTES = 1024 * 1024;
5
+ const MAX_NATIVE_READ_ATTEMPTS = 64;
6
+ const SKILL_PATH = /^\.agents\/skills\/([A-Za-z0-9._-]+)\/SKILL\.md$/;
7
+ const PLUGIN_SKILL_PATH = /^skills\/([A-Za-z0-9._-]+)\/SKILL\.md$/;
8
+
9
+ interface CodexItem {
10
+ id?: unknown;
11
+ type?: unknown;
12
+ command?: unknown;
13
+ status?: unknown;
14
+ exit_code?: unknown;
15
+ aggregated_output?: unknown;
16
+ }
17
+
18
+ interface CodexEvent {
19
+ type?: unknown;
20
+ item?: CodexItem;
21
+ }
22
+
23
+ function shellPayload(command: string): string {
24
+ let payload = command;
25
+ for (let depth = 0; depth < 2; depth++) {
26
+ const wrapper = payload.match(
27
+ /^(?:\/bin\/(?:ba|z)?sh\s+-l?c|lean-ctx\s+-c)\s+(["'])([\s\S]*)\1$/,
28
+ );
29
+ if (!wrapper) break;
30
+ payload = wrapper[2]!;
31
+ }
32
+ return payload;
33
+ }
34
+
35
+ interface DirectRead {
36
+ path: string;
37
+ firstLine?: number;
38
+ lastLine?: number;
39
+ }
40
+
41
+ function directRead(command: string): DirectRead | null {
42
+ const payload = shellPayload(command).trim();
43
+ const path = "(?:\"([^\"]+)\"|'([^']+)'|([^\\s'\"`$;&|<>]+))";
44
+ const cat = payload.match(
45
+ new RegExp(`^(?:/bin/)?cat\\s+(?:--\\s+)?${path}$`),
46
+ );
47
+ if (cat) return { path: cat[1] ?? cat[2] ?? cat[3]! };
48
+ const sed = payload.match(
49
+ new RegExp(`^(?:/bin/)?sed\\s+-n\\s+['"]?(\\d+),(\\d+)p['"]?\\s+${path}$`),
50
+ );
51
+ if (!sed) return null;
52
+ const firstLine = Number(sed[1]);
53
+ const lastLine = Number(sed[2]);
54
+ if (
55
+ !Number.isSafeInteger(firstLine) ||
56
+ firstLine < 1 ||
57
+ !Number.isSafeInteger(lastLine) ||
58
+ lastLine < firstLine
59
+ )
60
+ return null;
61
+ return { path: sed[3] ?? sed[4] ?? sed[5]!, firstLine, lastLine };
62
+ }
63
+
64
+ function within(root: string, path: string): boolean {
65
+ const child = relative(root, path);
66
+ return child !== ".." && !child.startsWith(`..${sep}`) && !isAbsolute(child);
67
+ }
68
+
69
+ function readRange(body: string, read: DirectRead): [number, number] {
70
+ if (read.firstLine === undefined || read.lastLine === undefined)
71
+ return [0, body.length];
72
+ const lines = body.match(/[^\n]*\n|[^\n]+$/g) ?? [];
73
+ const start = lines.slice(0, read.firstLine - 1).join("").length;
74
+ const end = lines.slice(0, read.lastLine).join("").length;
75
+ return [start, end];
76
+ }
77
+
78
+ async function verifiedSkillRead(
79
+ item: CodexItem,
80
+ workspace: string,
81
+ pluginRoots: string[],
82
+ commandCwd = workspace,
83
+ ): Promise<{
84
+ skill: string;
85
+ path: string;
86
+ range: [number, number];
87
+ bodyLength: number;
88
+ } | null> {
89
+ if (typeof item.command !== "string") return null;
90
+ const read = directRead(item.command);
91
+ if (!read || typeof item.aggregated_output !== "string") return null;
92
+ const resolved = resolve(commandCwd, read.path);
93
+ const actual = await realpath(resolved).catch(() => null);
94
+ if (!actual) return null;
95
+ const mountedMatch = relative(workspace, actual)
96
+ .split(sep)
97
+ .join("/")
98
+ .match(SKILL_PATH);
99
+ const pluginMatch = pluginRoots
100
+ .map((root) =>
101
+ relative(root, actual).split(sep).join("/").match(PLUGIN_SKILL_PATH),
102
+ )
103
+ .find((match) => match !== null);
104
+ const match = mountedMatch ?? pluginMatch;
105
+ if (!match) return null;
106
+ const body = await readFile(actual).catch(() => null);
107
+ if (!body || body.byteLength > MAX_SKILL_BYTES) return null;
108
+ const text = body.toString("utf8");
109
+ const skill = match[1]!;
110
+ if (!text.startsWith("---\n") || !text.includes(`\nname: ${skill}\n`))
111
+ return null;
112
+ const range = readRange(text, read);
113
+ if (range[0] === range[1]) return null;
114
+ if (!item.aggregated_output.includes(text.slice(...range))) return null;
115
+ return { skill, path: actual, range, bodyLength: text.length };
116
+ }
117
+
118
+ function fullCoverage(
119
+ ranges: Array<[number, number]>,
120
+ length: number,
121
+ ): boolean {
122
+ const ordered = ranges.sort((left, right) => left[0] - right[0]);
123
+ let covered = 0;
124
+ for (const [start, end] of ordered) {
125
+ if (start > covered) return false;
126
+ covered = Math.max(covered, end);
127
+ }
128
+ return covered === length;
129
+ }
130
+
131
+ interface NativeEntry {
132
+ ordinal: number;
133
+ payload: Record<string, unknown>;
134
+ }
135
+
136
+ export interface NativeReadDiagnostic {
137
+ completeness: "complete" | "partial";
138
+ observedSkills: string[];
139
+ completedReads: Array<{ skill: string; ordinal: number }>;
140
+ commandExecutions: number;
141
+ readAttempts: number;
142
+ truncated: boolean;
143
+ }
144
+
145
+ function nativeCommandText(value: unknown): string | null {
146
+ if (typeof value === "string") return value;
147
+ if (!Array.isArray(value) || !value.every((part) => typeof part === "string"))
148
+ return null;
149
+ if (value.length >= 3 && (value[1] === "-lc" || value[1] === "-c"))
150
+ return value.slice(2).join(" ");
151
+ return value.join(" ");
152
+ }
153
+
154
+ /** Verify native skill-file reads without retaining commands or output. */
155
+ export async function codexNativeReadDiagnostic(
156
+ entries: NativeEntry[],
157
+ workspace: string,
158
+ installedPluginRoots: string[] = [],
159
+ ): Promise<NativeReadDiagnostic> {
160
+ const canonicalWorkspace = await realpath(workspace);
161
+ const pluginRoots = await Promise.all(
162
+ installedPluginRoots.map((root) => realpath(root)),
163
+ );
164
+ const attempted = new Map<
165
+ string,
166
+ { ranges: Array<[number, number]>; bodyLength: number }
167
+ >();
168
+ const observedSkills: string[] = [];
169
+ const completedReads: Array<{ skill: string; ordinal: number }> = [];
170
+ let commandExecutions = 0;
171
+ let readAttempts = 0;
172
+ let partial = false;
173
+ let truncated = false;
174
+ for (const { ordinal, payload } of entries) {
175
+ const item = payload.item;
176
+ if (
177
+ payload.type !== "item_completed" ||
178
+ !item ||
179
+ typeof item !== "object" ||
180
+ Array.isArray(item) ||
181
+ (item as CodexItem).type !== "CommandExecution"
182
+ )
183
+ continue;
184
+ const commandItem = item as CodexItem & { cwd?: unknown };
185
+ commandExecutions++;
186
+ const command = nativeCommandText(commandItem.command);
187
+ if (!command) {
188
+ partial = true;
189
+ continue;
190
+ }
191
+ if (!command.includes("SKILL.md")) continue;
192
+ readAttempts++;
193
+ if (readAttempts > MAX_NATIVE_READ_ATTEMPTS) {
194
+ readAttempts = MAX_NATIVE_READ_ATTEMPTS;
195
+ truncated = true;
196
+ partial = true;
197
+ break;
198
+ }
199
+ if (
200
+ typeof commandItem.exit_code !== "number" ||
201
+ commandItem.status !== "completed"
202
+ ) {
203
+ partial = true;
204
+ continue;
205
+ }
206
+ const cwd =
207
+ commandItem.cwd === undefined
208
+ ? canonicalWorkspace
209
+ : typeof commandItem.cwd === "string" && isAbsolute(commandItem.cwd)
210
+ ? await realpath(commandItem.cwd).catch(() => null)
211
+ : null;
212
+ if (!cwd) {
213
+ partial = true;
214
+ continue;
215
+ }
216
+ const read = await verifiedSkillRead(
217
+ { ...commandItem, command },
218
+ canonicalWorkspace,
219
+ pluginRoots,
220
+ cwd,
221
+ );
222
+ if (!read) {
223
+ partial = true;
224
+ continue;
225
+ }
226
+ const prior = attempted.get(read.path);
227
+ const coverage =
228
+ prior && !fullCoverage(prior.ranges, prior.bodyLength)
229
+ ? prior
230
+ : {
231
+ ranges: [],
232
+ bodyLength: read.bodyLength,
233
+ };
234
+ coverage.ranges.push(read.range);
235
+ attempted.set(read.path, coverage);
236
+ if (fullCoverage(coverage.ranges, coverage.bodyLength)) {
237
+ completedReads.push({ skill: read.skill, ordinal });
238
+ if (!observedSkills.includes(read.skill)) observedSkills.push(read.skill);
239
+ }
240
+ }
241
+ return {
242
+ completeness:
243
+ !partial &&
244
+ [...attempted.values()].every(({ ranges, bodyLength }) =>
245
+ fullCoverage(ranges, bodyLength),
246
+ )
247
+ ? "complete"
248
+ : "partial",
249
+ observedSkills,
250
+ completedReads,
251
+ commandExecutions,
252
+ readAttempts,
253
+ truncated,
254
+ };
255
+ }
256
+
257
+ /** Retain only ordered mounted-skill names, never commands or skill bodies. */
258
+ export async function codexSkillReadObservation(
259
+ stream: string,
260
+ workspace: string,
261
+ installedPluginRoots: string[] = [],
262
+ ): Promise<{
263
+ id: "sevro.codex.skill-reads";
264
+ completeness: "complete" | "partial";
265
+ data: {
266
+ method: "skill_file_read_probe";
267
+ primarySkill: string | null;
268
+ observedSkills: string[];
269
+ };
270
+ }> {
271
+ const observedSkills: string[] = [];
272
+ const attempted = new Map<
273
+ string,
274
+ { ranges: Array<[number, number]>; bodyLength: number }
275
+ >();
276
+ const pending = new Set<string>();
277
+ let partial = false;
278
+ let completedTurn = false;
279
+ const canonicalWorkspace = await realpath(workspace);
280
+ const pluginRoots = await Promise.all(
281
+ installedPluginRoots.map((root) => realpath(root)),
282
+ );
283
+ for (const line of stream.split("\n")) {
284
+ if (!line.trim()) continue;
285
+ const event = JSON.parse(line) as CodexEvent;
286
+ const item = event.item;
287
+ if (event.type === "turn.completed") completedTurn = true;
288
+ if (event.type === "turn.failed") partial = true;
289
+ if (!item || item.type !== "command_execution") continue;
290
+ if (event.type === "item.started") {
291
+ if (
292
+ typeof item.command === "string" &&
293
+ item.command.includes("SKILL.md")
294
+ ) {
295
+ if (typeof item.id === "string") pending.add(item.id);
296
+ else partial = true;
297
+ }
298
+ continue;
299
+ }
300
+ if (event.type !== "item.completed") continue;
301
+ if (typeof item.id === "string") pending.delete(item.id);
302
+ if (typeof item.command !== "string") {
303
+ partial = true;
304
+ continue;
305
+ }
306
+ if (!item.command.includes("SKILL.md")) continue;
307
+ if (typeof item.exit_code !== "number" || item.status !== "completed") {
308
+ partial = true;
309
+ continue;
310
+ }
311
+ const read = await verifiedSkillRead(item, canonicalWorkspace, pluginRoots);
312
+ if (!read) {
313
+ partial = true;
314
+ continue;
315
+ }
316
+ const coverage = attempted.get(read.path) ?? {
317
+ ranges: [],
318
+ bodyLength: read.bodyLength,
319
+ };
320
+ coverage.ranges.push(read.range);
321
+ attempted.set(read.path, coverage);
322
+ if (
323
+ fullCoverage(coverage.ranges, coverage.bodyLength) &&
324
+ !observedSkills.includes(read.skill)
325
+ )
326
+ observedSkills.push(read.skill);
327
+ }
328
+ return {
329
+ id: "sevro.codex.skill-reads",
330
+ completeness:
331
+ completedTurn &&
332
+ !partial &&
333
+ pending.size === 0 &&
334
+ [...attempted.values()].every(({ ranges, bodyLength }) =>
335
+ fullCoverage(ranges, bodyLength),
336
+ )
337
+ ? "complete"
338
+ : "partial",
339
+ data: {
340
+ method: "skill_file_read_probe",
341
+ primarySkill: observedSkills[0] ?? null,
342
+ observedSkills,
343
+ },
344
+ };
345
+ }