@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +17 -3
  7. package/docs/codex-host.md +155 -49
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +141 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +307 -18
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +63 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +206 -0
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +13 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +133 -115
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/evaluation/types.ts +207 -0
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +15 -76
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/fixtures/workspace-state.ts +212 -0
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +57 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +91 -14
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +46 -16
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +122 -73
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -54
  158. package/src/hosts/codex-profile.ts +50 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +141 -19
  161. package/src/hosts/codex.ts +221 -179
  162. package/src/hosts/isolation-common.ts +15 -11
  163. package/src/hosts/isolation-roots.ts +13 -20
  164. package/src/hosts/linux-sandbox.ts +27 -3
  165. package/src/hosts/mac-sandbox.ts +111 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +221 -45
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +34 -15
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/run-owner.ts +4 -4
  186. package/src/{runtime-config.ts → runtime/config.ts} +182 -44
  187. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  188. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  189. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  190. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  191. package/src/{runtime-hooks.ts → runtime/hooks.ts} +33 -22
  192. package/src/{runtime-paths.ts → runtime/paths.ts} +6 -18
  193. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  194. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  195. package/src/runtime/shared-cache.ts +130 -0
  196. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  197. package/src/schema-types.ts +73 -161
  198. package/src/shell.ts +4 -0
  199. package/src/transcript-retention.ts +146 -0
  200. package/src/value-guards.ts +22 -0
  201. package/docs/assets/sevro-logo.png +0 -0
  202. package/docs/assets/sevro-symbol.svg +0 -1
  203. package/docs/assets/sevro-wordmark.svg +0 -1
  204. package/docs/cache-ownership-validation.md +0 -49
  205. package/docs/claude-continuation-validation.md +0 -47
  206. package/docs/concurrency-validation.md +0 -86
  207. package/docs/configuration-root-validation.md +0 -70
  208. package/docs/engine-slice.md +0 -114
  209. package/docs/evidence/runtime-startup-30/README.md +0 -185
  210. package/docs/evidence/runtime-startup-30/after.json +0 -319
  211. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  212. package/docs/evidence/runtime-startup-30/before.json +0 -403
  213. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  214. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  216. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  217. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  218. package/docs/evidence.md +0 -17
  219. package/docs/fixture-cleanup-validation.md +0 -70
  220. package/docs/guide-verification.json +0 -239
  221. package/src/evaluation-context.ts +0 -257
  222. package/src/evaluation-fixture.ts +0 -313
  223. package/src/evaluation-run.ts +0 -198
  224. package/src/evaluation-trial.ts +0 -946
  225. package/src/evaluation-types.ts +0 -149
  226. package/src/hosts/claude-nested-skills.ts +0 -272
  227. package/src/hosts/codex-command-output.ts +0 -510
  228. package/src/hosts/codex-executor-command.ts +0 -291
  229. package/src/hosts/codex-skill-reads.ts +0 -609
  230. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
@@ -1,291 +0,0 @@
1
- import type ts from "typescript";
2
- import { createRequire } from "node:module";
3
-
4
- const require = createRequire(import.meta.url);
5
- let parser: typeof ts | undefined;
6
-
7
- function typescript(): typeof ts {
8
- return (parser ??= require("typescript") as typeof ts);
9
- }
10
-
11
- export interface LiteralExecutorCommand {
12
- command: string;
13
- cwd?: string;
14
- shell?: string;
15
- }
16
-
17
- function primitive(node: ts.Expression): string | number | boolean | undefined {
18
- const ts = typescript();
19
- if (ts.isStringLiteralLike(node)) return node.text;
20
- if (ts.isNumericLiteral(node)) return Number(node.text);
21
- if (node.kind === ts.SyntaxKind.TrueKeyword) return true;
22
- if (node.kind === ts.SyntaxKind.FalseKeyword) return false;
23
- return undefined;
24
- }
25
-
26
- function literalOptions(
27
- node: ts.Expression,
28
- ): Map<string, string | number | boolean> | undefined {
29
- const ts = typescript();
30
- if (!ts.isObjectLiteralExpression(node)) return undefined;
31
- const values = new Map<string, string | number | boolean>();
32
- const allowed = new Set([
33
- "cmd",
34
- "workdir",
35
- "shell",
36
- "yield_time_ms",
37
- "max_output_tokens",
38
- "login",
39
- "tty",
40
- ]);
41
- for (const property of node.properties) {
42
- const option = literalOption(property);
43
- if (!option || !validOptionName(option.name, allowed, values))
44
- return undefined;
45
- values.set(option.name, option.value);
46
- }
47
- return values;
48
- }
49
-
50
- function literalOption(
51
- property: ts.ObjectLiteralElementLike,
52
- ): { name: string; value: string | number | boolean } | undefined {
53
- const ts = typescript();
54
- if (!ts.isPropertyAssignment(property)) return undefined;
55
- const name = propertyName(property.name);
56
- const value = primitive(property.initializer);
57
- if (!name || value === undefined) return undefined;
58
- return { name, value };
59
- }
60
-
61
- function validOptionName(
62
- name: string,
63
- allowed: Set<string>,
64
- values: Map<string, unknown>,
65
- ): boolean {
66
- return allowed.has(name) && !values.has(name);
67
- }
68
-
69
- function propertyName(name: ts.PropertyName): string | undefined {
70
- const ts = typescript();
71
- return ts.isIdentifier(name) || ts.isStringLiteral(name)
72
- ? name.text
73
- : undefined;
74
- }
75
-
76
- function optionalString(value: unknown): value is string | undefined {
77
- return value === undefined || typeof value === "string";
78
- }
79
-
80
- function commandOptions(
81
- node: ts.Expression,
82
- ): LiteralExecutorCommand | undefined {
83
- const values = literalOptions(node);
84
- if (!values) return undefined;
85
- return resolvedCommandOptions(
86
- values.get("cmd"),
87
- values.get("workdir"),
88
- values.get("shell"),
89
- );
90
- }
91
-
92
- function nonemptyCommand(command: unknown): command is string {
93
- return typeof command === "string" && command.length > 0;
94
- }
95
-
96
- function resolvedCommandOptions(
97
- command: unknown,
98
- cwd: unknown,
99
- shell: unknown,
100
- ): LiteralExecutorCommand | undefined {
101
- if (
102
- !nonemptyCommand(command) ||
103
- !optionalString(cwd) ||
104
- !optionalString(shell)
105
- )
106
- return undefined;
107
- return { command, ...optionalCommandLocations(cwd, shell) };
108
- }
109
-
110
- function optionalCommandLocations(
111
- cwd: string | undefined,
112
- shell: string | undefined,
113
- ) {
114
- return {
115
- ...(cwd === undefined ? {} : { cwd }),
116
- ...(shell === undefined ? {} : { shell }),
117
- };
118
- }
119
-
120
- function commandDeclaration(statement: ts.Statement):
121
- | {
122
- name: string;
123
- command: LiteralExecutorCommand;
124
- }
125
- | undefined {
126
- const declaration = singleConstDeclaration(statement);
127
- if (!declaration) return undefined;
128
- const name = literalVariableName(declaration.name);
129
- if (name === undefined || !declaration.initializer) return undefined;
130
- const command = awaitedCommand(declaration.initializer);
131
- return command ? { name, command } : undefined;
132
- }
133
-
134
- function singleConstDeclaration(
135
- statement: ts.Statement,
136
- ): ts.VariableDeclaration | undefined {
137
- const ts = typescript();
138
- if (
139
- !ts.isVariableStatement(statement) ||
140
- !(statement.declarationList.flags & ts.NodeFlags.Const) ||
141
- statement.declarationList.declarations.length !== 1
142
- )
143
- return undefined;
144
- return statement.declarationList.declarations[0];
145
- }
146
-
147
- function literalVariableName(name: ts.BindingName): string | undefined {
148
- const ts = typescript();
149
- if (!ts.isIdentifier(name) || ["tools", "text", "store"].includes(name.text))
150
- return undefined;
151
- return name.text;
152
- }
153
-
154
- function commandAccess(expression: ts.Expression): boolean {
155
- const ts = typescript();
156
- return (
157
- ts.isPropertyAccessExpression(expression) &&
158
- ts.isIdentifier(expression.expression) &&
159
- expression.expression.text === "tools" &&
160
- expression.name.text === "exec_command"
161
- );
162
- }
163
-
164
- function executorCall(
165
- expression: ts.Expression,
166
- ): ts.CallExpression | undefined {
167
- const ts = typescript();
168
- if (
169
- !ts.isCallExpression(expression) ||
170
- expression.arguments.length !== 1 ||
171
- !commandAccess(expression.expression)
172
- )
173
- return undefined;
174
- return expression;
175
- }
176
-
177
- function awaitedCommand(
178
- expression: ts.Expression,
179
- ): LiteralExecutorCommand | undefined {
180
- const ts = typescript();
181
- if (!ts.isAwaitExpression(expression)) return undefined;
182
- const call = executorCall(expression.expression);
183
- if (!call) return undefined;
184
- const [argument] = call.arguments;
185
- return argument ? commandOptions(argument) : undefined;
186
- }
187
-
188
- function resultProperty(
189
- node: ts.Expression,
190
- name: string,
191
- field: string,
192
- ): boolean {
193
- const ts = typescript();
194
- return (
195
- ts.isPropertyAccessExpression(node) &&
196
- ts.isIdentifier(node.expression) &&
197
- node.expression.text === name &&
198
- node.name.text === field
199
- );
200
- }
201
-
202
- function outputStatement(
203
- statement: ts.Statement,
204
- name: string,
205
- ): string | undefined {
206
- const ts = typescript();
207
- if (
208
- !ts.isExpressionStatement(statement) ||
209
- !ts.isCallExpression(statement.expression) ||
210
- !ts.isIdentifier(statement.expression.expression)
211
- )
212
- return undefined;
213
- return directOutput(statement.expression, name);
214
- }
215
-
216
- function directOutput(
217
- call: ts.CallExpression,
218
- name: string,
219
- ): string | undefined {
220
- const operation = (call.expression as ts.Identifier).text;
221
- if (operation === "text" && textOutput(call, name)) return "text";
222
- if (operation === "store" && storeOutput(call, name)) return "store";
223
- return undefined;
224
- }
225
-
226
- function textOutput(call: ts.CallExpression, name: string): boolean {
227
- const [argument] = call.arguments;
228
- return (
229
- call.arguments.length === 1 &&
230
- argument !== undefined &&
231
- resultProperty(argument, name, "output")
232
- );
233
- }
234
-
235
- function storeOutput(call: ts.CallExpression, name: string): boolean {
236
- const ts = typescript();
237
- const [key, value] = call.arguments;
238
- if (call.arguments.length !== 2 || key === undefined || value === undefined)
239
- return false;
240
- return (
241
- ts.isStringLiteralLike(key) && resultProperty(value, name, "session_id")
242
- );
243
- }
244
-
245
- function validExecutorSource(source: ts.SourceFile): boolean {
246
- const diagnostics = (
247
- source as ts.SourceFile & { parseDiagnostics?: readonly ts.Diagnostic[] }
248
- ).parseDiagnostics;
249
- return (
250
- diagnostics !== undefined &&
251
- diagnostics.length === 0 &&
252
- source.statements.length >= 2 &&
253
- source.statements.length <= 3
254
- );
255
- }
256
-
257
- function validExecutorOutputs(outputs: (string | undefined)[]): boolean {
258
- return (
259
- !outputs.some((output) => output === undefined) &&
260
- outputs.filter((output) => output === "text").length === 1 &&
261
- outputs.filter((output) => output === "store").length <= 1
262
- );
263
- }
264
-
265
- /** Recognize only one unconditional awaited command whose output is returned directly. */
266
- export function literalExecutorCommand(
267
- code: unknown,
268
- ): LiteralExecutorCommand | undefined {
269
- if (typeof code !== "string") return undefined;
270
- const ts = typescript();
271
- const source = ts.createSourceFile(
272
- "native-exec.js",
273
- code,
274
- ts.ScriptTarget.Latest,
275
- false,
276
- ts.ScriptKind.JS,
277
- );
278
- if (!validExecutorSource(source)) return undefined;
279
- const declaration = leadingCommandDeclaration(source);
280
- if (!declaration) return undefined;
281
- const outputs = source.statements
282
- .slice(1)
283
- .map((statement) => outputStatement(statement, declaration.name));
284
- if (!validExecutorOutputs(outputs)) return undefined;
285
- return declaration.command;
286
- }
287
-
288
- function leadingCommandDeclaration(source: ts.SourceFile) {
289
- const [statement] = source.statements;
290
- return statement ? commandDeclaration(statement) : undefined;
291
- }