@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +9 -2
  7. package/docs/codex-host.md +155 -57
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +122 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +255 -22
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +43 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +42 -11
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +9 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +50 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +46 -9
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +28 -15
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +121 -81
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -183
  158. package/src/hosts/codex-profile.ts +36 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +134 -18
  161. package/src/hosts/codex.ts +210 -183
  162. package/src/hosts/isolation-common.ts +8 -11
  163. package/src/hosts/isolation-roots.ts +2 -2
  164. package/src/hosts/linux-sandbox.ts +15 -2
  165. package/src/hosts/mac-sandbox.ts +87 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +216 -48
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +26 -11
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/{runtime-config.ts → runtime/config.ts} +160 -46
  186. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  187. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  188. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  189. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  190. package/src/{runtime-hooks.ts → runtime/hooks.ts} +32 -22
  191. package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
  192. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  193. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  194. package/src/runtime/shared-cache.ts +130 -0
  195. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  196. package/src/schema-types.ts +73 -161
  197. package/src/shell.ts +4 -0
  198. package/src/transcript-retention.ts +146 -0
  199. package/src/value-guards.ts +22 -0
  200. package/docs/assets/sevro-logo.png +0 -0
  201. package/docs/assets/sevro-symbol.svg +0 -1
  202. package/docs/assets/sevro-wordmark.svg +0 -1
  203. package/docs/cache-ownership-validation.md +0 -49
  204. package/docs/claude-continuation-validation.md +0 -47
  205. package/docs/concurrency-validation.md +0 -86
  206. package/docs/configuration-root-validation.md +0 -70
  207. package/docs/engine-slice.md +0 -114
  208. package/docs/evidence/runtime-startup-30/README.md +0 -185
  209. package/docs/evidence/runtime-startup-30/after.json +0 -319
  210. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  211. package/docs/evidence/runtime-startup-30/before.json +0 -403
  212. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  213. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  214. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  216. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  217. package/docs/evidence.md +0 -17
  218. package/docs/fixture-cleanup-validation.md +0 -73
  219. package/docs/guide-verification.json +0 -239
  220. package/src/evaluation-context.ts +0 -259
  221. package/src/evaluation-fixture.ts +0 -302
  222. package/src/evaluation-run.ts +0 -200
  223. package/src/evaluation-trial.ts +0 -1016
  224. package/src/hosts/claude-nested-skills.ts +0 -272
  225. package/src/hosts/codex-command-output.ts +0 -510
  226. package/src/hosts/codex-executor-command.ts +0 -291
  227. package/src/hosts/codex-skill-reads.ts +0 -783
  228. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
@@ -1,291 +0,0 @@
1
- import type ts from "typescript";
2
- import { createRequire } from "node:module";
3
-
4
- const require = createRequire(import.meta.url);
5
- let parser: typeof ts | undefined;
6
-
7
- function typescript(): typeof ts {
8
- return (parser ??= require("typescript") as typeof ts);
9
- }
10
-
11
- export interface LiteralExecutorCommand {
12
- command: string;
13
- cwd?: string;
14
- shell?: string;
15
- }
16
-
17
- function primitive(node: ts.Expression): string | number | boolean | undefined {
18
- const ts = typescript();
19
- if (ts.isStringLiteralLike(node)) return node.text;
20
- if (ts.isNumericLiteral(node)) return Number(node.text);
21
- if (node.kind === ts.SyntaxKind.TrueKeyword) return true;
22
- if (node.kind === ts.SyntaxKind.FalseKeyword) return false;
23
- return undefined;
24
- }
25
-
26
- function literalOptions(
27
- node: ts.Expression,
28
- ): Map<string, string | number | boolean> | undefined {
29
- const ts = typescript();
30
- if (!ts.isObjectLiteralExpression(node)) return undefined;
31
- const values = new Map<string, string | number | boolean>();
32
- const allowed = new Set([
33
- "cmd",
34
- "workdir",
35
- "shell",
36
- "yield_time_ms",
37
- "max_output_tokens",
38
- "login",
39
- "tty",
40
- ]);
41
- for (const property of node.properties) {
42
- const option = literalOption(property);
43
- if (!option || !validOptionName(option.name, allowed, values))
44
- return undefined;
45
- values.set(option.name, option.value);
46
- }
47
- return values;
48
- }
49
-
50
- function literalOption(
51
- property: ts.ObjectLiteralElementLike,
52
- ): { name: string; value: string | number | boolean } | undefined {
53
- const ts = typescript();
54
- if (!ts.isPropertyAssignment(property)) return undefined;
55
- const name = propertyName(property.name);
56
- const value = primitive(property.initializer);
57
- if (!name || value === undefined) return undefined;
58
- return { name, value };
59
- }
60
-
61
- function validOptionName(
62
- name: string,
63
- allowed: Set<string>,
64
- values: Map<string, unknown>,
65
- ): boolean {
66
- return allowed.has(name) && !values.has(name);
67
- }
68
-
69
- function propertyName(name: ts.PropertyName): string | undefined {
70
- const ts = typescript();
71
- return ts.isIdentifier(name) || ts.isStringLiteral(name)
72
- ? name.text
73
- : undefined;
74
- }
75
-
76
- function optionalString(value: unknown): value is string | undefined {
77
- return value === undefined || typeof value === "string";
78
- }
79
-
80
- function commandOptions(
81
- node: ts.Expression,
82
- ): LiteralExecutorCommand | undefined {
83
- const values = literalOptions(node);
84
- if (!values) return undefined;
85
- return resolvedCommandOptions(
86
- values.get("cmd"),
87
- values.get("workdir"),
88
- values.get("shell"),
89
- );
90
- }
91
-
92
- function nonemptyCommand(command: unknown): command is string {
93
- return typeof command === "string" && command.length > 0;
94
- }
95
-
96
- function resolvedCommandOptions(
97
- command: unknown,
98
- cwd: unknown,
99
- shell: unknown,
100
- ): LiteralExecutorCommand | undefined {
101
- if (
102
- !nonemptyCommand(command) ||
103
- !optionalString(cwd) ||
104
- !optionalString(shell)
105
- )
106
- return undefined;
107
- return { command, ...optionalCommandLocations(cwd, shell) };
108
- }
109
-
110
- function optionalCommandLocations(
111
- cwd: string | undefined,
112
- shell: string | undefined,
113
- ) {
114
- return {
115
- ...(cwd === undefined ? {} : { cwd }),
116
- ...(shell === undefined ? {} : { shell }),
117
- };
118
- }
119
-
120
- function commandDeclaration(statement: ts.Statement):
121
- | {
122
- name: string;
123
- command: LiteralExecutorCommand;
124
- }
125
- | undefined {
126
- const declaration = singleConstDeclaration(statement);
127
- if (!declaration) return undefined;
128
- const name = literalVariableName(declaration.name);
129
- if (name === undefined || !declaration.initializer) return undefined;
130
- const command = awaitedCommand(declaration.initializer);
131
- return command ? { name, command } : undefined;
132
- }
133
-
134
- function singleConstDeclaration(
135
- statement: ts.Statement,
136
- ): ts.VariableDeclaration | undefined {
137
- const ts = typescript();
138
- if (
139
- !ts.isVariableStatement(statement) ||
140
- !(statement.declarationList.flags & ts.NodeFlags.Const) ||
141
- statement.declarationList.declarations.length !== 1
142
- )
143
- return undefined;
144
- return statement.declarationList.declarations[0];
145
- }
146
-
147
- function literalVariableName(name: ts.BindingName): string | undefined {
148
- const ts = typescript();
149
- if (!ts.isIdentifier(name) || ["tools", "text", "store"].includes(name.text))
150
- return undefined;
151
- return name.text;
152
- }
153
-
154
- function commandAccess(expression: ts.Expression): boolean {
155
- const ts = typescript();
156
- return (
157
- ts.isPropertyAccessExpression(expression) &&
158
- ts.isIdentifier(expression.expression) &&
159
- expression.expression.text === "tools" &&
160
- expression.name.text === "exec_command"
161
- );
162
- }
163
-
164
- function executorCall(
165
- expression: ts.Expression,
166
- ): ts.CallExpression | undefined {
167
- const ts = typescript();
168
- if (
169
- !ts.isCallExpression(expression) ||
170
- expression.arguments.length !== 1 ||
171
- !commandAccess(expression.expression)
172
- )
173
- return undefined;
174
- return expression;
175
- }
176
-
177
- function awaitedCommand(
178
- expression: ts.Expression,
179
- ): LiteralExecutorCommand | undefined {
180
- const ts = typescript();
181
- if (!ts.isAwaitExpression(expression)) return undefined;
182
- const call = executorCall(expression.expression);
183
- if (!call) return undefined;
184
- const [argument] = call.arguments;
185
- return argument ? commandOptions(argument) : undefined;
186
- }
187
-
188
- function resultProperty(
189
- node: ts.Expression,
190
- name: string,
191
- field: string,
192
- ): boolean {
193
- const ts = typescript();
194
- return (
195
- ts.isPropertyAccessExpression(node) &&
196
- ts.isIdentifier(node.expression) &&
197
- node.expression.text === name &&
198
- node.name.text === field
199
- );
200
- }
201
-
202
- function outputStatement(
203
- statement: ts.Statement,
204
- name: string,
205
- ): string | undefined {
206
- const ts = typescript();
207
- if (
208
- !ts.isExpressionStatement(statement) ||
209
- !ts.isCallExpression(statement.expression) ||
210
- !ts.isIdentifier(statement.expression.expression)
211
- )
212
- return undefined;
213
- return directOutput(statement.expression, name);
214
- }
215
-
216
- function directOutput(
217
- call: ts.CallExpression,
218
- name: string,
219
- ): string | undefined {
220
- const operation = (call.expression as ts.Identifier).text;
221
- if (operation === "text" && textOutput(call, name)) return "text";
222
- if (operation === "store" && storeOutput(call, name)) return "store";
223
- return undefined;
224
- }
225
-
226
- function textOutput(call: ts.CallExpression, name: string): boolean {
227
- const [argument] = call.arguments;
228
- return (
229
- call.arguments.length === 1 &&
230
- argument !== undefined &&
231
- resultProperty(argument, name, "output")
232
- );
233
- }
234
-
235
- function storeOutput(call: ts.CallExpression, name: string): boolean {
236
- const ts = typescript();
237
- const [key, value] = call.arguments;
238
- if (call.arguments.length !== 2 || key === undefined || value === undefined)
239
- return false;
240
- return (
241
- ts.isStringLiteralLike(key) && resultProperty(value, name, "session_id")
242
- );
243
- }
244
-
245
- function validExecutorSource(source: ts.SourceFile): boolean {
246
- const diagnostics = (
247
- source as ts.SourceFile & { parseDiagnostics?: readonly ts.Diagnostic[] }
248
- ).parseDiagnostics;
249
- return (
250
- diagnostics !== undefined &&
251
- diagnostics.length === 0 &&
252
- source.statements.length >= 2 &&
253
- source.statements.length <= 3
254
- );
255
- }
256
-
257
- function validExecutorOutputs(outputs: (string | undefined)[]): boolean {
258
- return (
259
- !outputs.some((output) => output === undefined) &&
260
- outputs.filter((output) => output === "text").length === 1 &&
261
- outputs.filter((output) => output === "store").length <= 1
262
- );
263
- }
264
-
265
- /** Recognize only one unconditional awaited command whose output is returned directly. */
266
- export function literalExecutorCommand(
267
- code: unknown,
268
- ): LiteralExecutorCommand | undefined {
269
- if (typeof code !== "string") return undefined;
270
- const ts = typescript();
271
- const source = ts.createSourceFile(
272
- "native-exec.js",
273
- code,
274
- ts.ScriptTarget.Latest,
275
- false,
276
- ts.ScriptKind.JS,
277
- );
278
- if (!validExecutorSource(source)) return undefined;
279
- const declaration = leadingCommandDeclaration(source);
280
- if (!declaration) return undefined;
281
- const outputs = source.statements
282
- .slice(1)
283
- .map((statement) => outputStatement(statement, declaration.name));
284
- if (!validExecutorOutputs(outputs)) return undefined;
285
- return declaration.command;
286
- }
287
-
288
- function leadingCommandDeclaration(source: ts.SourceFile) {
289
- const [statement] = source.statements;
290
- return statement ? commandDeclaration(statement) : undefined;
291
- }