@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +9 -2
  7. package/docs/codex-host.md +155 -57
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +122 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +255 -22
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +43 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +42 -11
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +9 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +50 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +46 -9
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +28 -15
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +121 -81
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -183
  158. package/src/hosts/codex-profile.ts +36 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +134 -18
  161. package/src/hosts/codex.ts +210 -183
  162. package/src/hosts/isolation-common.ts +8 -11
  163. package/src/hosts/isolation-roots.ts +2 -2
  164. package/src/hosts/linux-sandbox.ts +15 -2
  165. package/src/hosts/mac-sandbox.ts +87 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +216 -48
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +26 -11
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/{runtime-config.ts → runtime/config.ts} +160 -46
  186. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  187. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  188. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  189. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  190. package/src/{runtime-hooks.ts → runtime/hooks.ts} +32 -22
  191. package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
  192. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  193. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  194. package/src/runtime/shared-cache.ts +130 -0
  195. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  196. package/src/schema-types.ts +73 -161
  197. package/src/shell.ts +4 -0
  198. package/src/transcript-retention.ts +146 -0
  199. package/src/value-guards.ts +22 -0
  200. package/docs/assets/sevro-logo.png +0 -0
  201. package/docs/assets/sevro-symbol.svg +0 -1
  202. package/docs/assets/sevro-wordmark.svg +0 -1
  203. package/docs/cache-ownership-validation.md +0 -49
  204. package/docs/claude-continuation-validation.md +0 -47
  205. package/docs/concurrency-validation.md +0 -86
  206. package/docs/configuration-root-validation.md +0 -70
  207. package/docs/engine-slice.md +0 -114
  208. package/docs/evidence/runtime-startup-30/README.md +0 -185
  209. package/docs/evidence/runtime-startup-30/after.json +0 -319
  210. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  211. package/docs/evidence/runtime-startup-30/before.json +0 -403
  212. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  213. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  214. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  216. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  217. package/docs/evidence.md +0 -17
  218. package/docs/fixture-cleanup-validation.md +0 -73
  219. package/docs/guide-verification.json +0 -239
  220. package/src/evaluation-context.ts +0 -259
  221. package/src/evaluation-fixture.ts +0 -302
  222. package/src/evaluation-run.ts +0 -200
  223. package/src/evaluation-trial.ts +0 -1016
  224. package/src/hosts/claude-nested-skills.ts +0 -272
  225. package/src/hosts/codex-command-output.ts +0 -510
  226. package/src/hosts/codex-executor-command.ts +0 -291
  227. package/src/hosts/codex-skill-reads.ts +0 -783
  228. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
@@ -1,783 +0,0 @@
1
- import { isRecord, isUnknownArray } from "../value-guards";
2
- import type { RecoveredCommandOutput } from "./codex-command-output";
3
- import { nativeCommandOutputs } from "./codex-command-output";
4
- import { readFile, realpath } from "node:fs/promises";
5
- import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
6
-
7
- const MAX_SKILL_BYTES = 1024 * 1024;
8
- const MAX_NATIVE_READ_ATTEMPTS = 64;
9
- const SKILL_PATH = /^\.agents\/skills\/([A-Za-z0-9._-]+)\/SKILL\.md$/;
10
- const PLUGIN_SKILL_PATH = /^skills\/([A-Za-z0-9._-]+)\/SKILL\.md$/;
11
-
12
- interface CodexItem {
13
- id?: unknown;
14
- type?: unknown;
15
- command?: unknown;
16
- status?: unknown;
17
- exit_code?: unknown;
18
- aggregated_output?: unknown;
19
- }
20
-
21
- interface CodexEvent {
22
- type?: unknown;
23
- item?: CodexItem;
24
- }
25
-
26
- function shellPayload(command: string): string {
27
- let payload = command;
28
- for (let depth = 0; depth < 2; depth++) {
29
- const wrapper = payload.match(
30
- /^(?:\/bin\/(?:ba|z)?sh\s+-l?c|lean-ctx\s+-c)\s+(["'])([\s\S]*)\1$/,
31
- );
32
- if (!wrapper) break;
33
- payload = wrapper.slice(2, 3).join("");
34
- }
35
- return payload;
36
- }
37
-
38
- interface DirectRead {
39
- path: string;
40
- firstLine?: number;
41
- lastLine?: number;
42
- }
43
-
44
- function capturedReadPath(match: RegExpMatchArray, first: number): string {
45
- return match
46
- .slice(first, first + 3)
47
- .filter(Boolean)
48
- .join("");
49
- }
50
- function validReadLines(firstLine: number, lastLine: number): boolean {
51
- return (
52
- Number.isSafeInteger(firstLine) &&
53
- firstLine >= 1 &&
54
- Number.isSafeInteger(lastLine) &&
55
- lastLine >= firstLine
56
- );
57
- }
58
- function sedRead(payload: string, path: string): DirectRead | null {
59
- const sed = payload.match(
60
- new RegExp(`^(?:/bin/)?sed\\s+-n\\s+['"]?(\\d+),(\\d+)p['"]?\\s+${path}$`),
61
- );
62
- if (!sed) return null;
63
- const firstLine = Number(sed[1]),
64
- lastLine = Number(sed[2]);
65
- if (!validReadLines(firstLine, lastLine)) return null;
66
- return { path: capturedReadPath(sed, 3), firstLine, lastLine };
67
- }
68
- function directRead(command: string): DirectRead | null {
69
- const payload = shellPayload(command).trim();
70
- const path = "(?:\"([^\"]+)\"|'([^']+)'|([^\\s'\"`$;&|<>]+))";
71
- return catRead(payload, path) ?? sedRead(payload, path);
72
- }
73
-
74
- function catRead(payload: string, path: string): DirectRead | null {
75
- const cat = payload.match(
76
- new RegExp(`^(?:/bin/)?cat\\s+(?:--\\s+)?(${path}(?:\\s+${path})*)$`),
77
- );
78
- if (!cat) return null;
79
- const paths = [
80
- ...cat.slice(1, 2).join("").matchAll(new RegExp(path, "g")),
81
- ].map((match) => capturedReadPath(match, 1));
82
- const skills = paths.filter((value) => /(?:^|\/)SKILL\.md$/.test(value));
83
- if (skills.length !== 1 || paths.length > MAX_NATIVE_READ_ATTEMPTS)
84
- return null;
85
- return { path: skills.join("") };
86
- }
87
-
88
- function readRange(body: string, read: DirectRead): [number, number] {
89
- if (read.firstLine === undefined || read.lastLine === undefined)
90
- return [0, body.length];
91
- const lines = body.match(/[^\n]*\n|[^\n]+$/g) ?? [];
92
- const start = lines.slice(0, read.firstLine - 1).join("").length;
93
- const end = lines.slice(0, read.lastLine).join("").length;
94
- return [start, end];
95
- }
96
-
97
- type VerifiedRead = {
98
- skill: string;
99
- path: string;
100
- range: [number, number];
101
- bodyLength: number;
102
- };
103
- function readDeclaration(
104
- item: CodexItem,
105
- ): { read: DirectRead; output: string } | null {
106
- if (typeof item.command !== "string") return null;
107
- const read = directRead(item.command);
108
- if (!read || typeof item.aggregated_output !== "string") return null;
109
- return { read, output: item.aggregated_output };
110
- }
111
- async function verifiedSkillRead(
112
- item: CodexItem,
113
- workspace: string,
114
- pluginRoots: string[],
115
- commandCwd = workspace,
116
- ): Promise<VerifiedRead | null> {
117
- const declaration = readDeclaration(item);
118
- if (!declaration) return null;
119
- return resolvedSkillRead(declaration, workspace, pluginRoots, commandCwd);
120
- }
121
- async function mountedSkillReadAttempt(
122
- command: string,
123
- workspace: string,
124
- pluginRoots: string[],
125
- commandCwd = workspace,
126
- ): Promise<boolean> {
127
- for (const path of readAttemptPaths(command)) {
128
- if (await mountedReadPath(path, commandCwd, workspace, pluginRoots))
129
- return true;
130
- }
131
- return false;
132
- }
133
- async function mountedReadPath(
134
- path: string,
135
- commandCwd: string,
136
- workspace: string,
137
- pluginRoots: string[],
138
- ): Promise<boolean> {
139
- const absolute = resolve(commandCwd, path);
140
- const actual = await realpath(absolute).catch(() => null);
141
- return (
142
- actual !== null && mountedSkillName(actual, workspace, pluginRoots) !== null
143
- );
144
- }
145
- function readAttemptPaths(command: string): string[] {
146
- const payload = shellPayload(command).trim();
147
- const path = "(?:\"([^\"]+)\"|'([^']+)'|([^\\s'\"`$;&|<>]+))";
148
- const cat = payload.match(
149
- new RegExp(`^(?:/bin/)?cat\\s+(?:--\\s+)?(${path}(?:\\s+${path})*)$`),
150
- );
151
- if (cat) {
152
- return [...cat.slice(1, 2).join("").matchAll(new RegExp(path, "g"))]
153
- .map((match) => capturedReadPath(match, 1))
154
- .filter((value) => /(?:^|\/)SKILL\.md$/.test(value));
155
- }
156
- const sed = payload.match(
157
- new RegExp(`^(?:/bin/)?sed\\s+-n\\s+[^\\s]+\\s+${path}$`),
158
- );
159
- if (sed) {
160
- const value = capturedReadPath(sed, 2);
161
- if (/(?:^|\/)SKILL\.md$/.test(value)) return [value];
162
- }
163
- return [];
164
- }
165
- async function exposesMountedSkillFrontmatter(
166
- output: string,
167
- workspace: string,
168
- pluginRoots: string[],
169
- ): Promise<boolean> {
170
- const skills = new Set(
171
- [...output.matchAll(/(?:^|\n)name: ([A-Za-z0-9._-]+)\r?$/gm)]
172
- .map((match) => match[1])
173
- .filter((skill): skill is string => skill !== undefined),
174
- );
175
- for (const skill of skills) {
176
- const paths = [
177
- resolve(workspace, ".agents", "skills", skill, "SKILL.md"),
178
- ...pluginRoots.map((root) => resolve(root, "skills", skill, "SKILL.md")),
179
- ];
180
- for (const path of paths)
181
- if (
182
- await verifiedMountedFrontmatterPath(
183
- path,
184
- skill,
185
- workspace,
186
- pluginRoots,
187
- )
188
- )
189
- return true;
190
- }
191
- return false;
192
- }
193
- async function verifiedMountedFrontmatterPath(
194
- path: string,
195
- skill: string,
196
- workspace: string,
197
- pluginRoots: string[],
198
- ): Promise<boolean> {
199
- const actual = await realpath(path).catch(() => null);
200
- if (actual && mountedSkillName(actual, workspace, pluginRoots) === skill)
201
- if ((await verifiedSkillBody(actual, skill)) !== null) return true;
202
- const directory = await realpath(dirname(path)).catch(() => null);
203
- return (
204
- directory !== null &&
205
- mountedSkillDirectoryName(directory, workspace, pluginRoots) === skill
206
- );
207
- }
208
- function mountedSkillDirectoryName(
209
- directory: string,
210
- workspace: string,
211
- pluginRoots: string[],
212
- ): string | null {
213
- const mounted = relative(workspace, directory)
214
- .split(sep)
215
- .join("/")
216
- .match(/^\.agents\/skills\/([A-Za-z0-9._-]+)$/);
217
- const plugin = pluginRoots
218
- .map((root) =>
219
- relative(root, directory)
220
- .split(sep)
221
- .join("/")
222
- .match(/^skills\/([A-Za-z0-9._-]+)$/),
223
- )
224
- .find((match) => match !== null);
225
- return (mounted ?? plugin)?.[1] ?? null;
226
- }
227
- function mountedSkillName(
228
- actual: string,
229
- workspace: string,
230
- pluginRoots: string[],
231
- ): string | null {
232
- const mounted = relative(workspace, actual)
233
- .split(sep)
234
- .join("/")
235
- .match(SKILL_PATH);
236
- const plugin = pluginRoots
237
- .map((root) =>
238
- relative(root, actual).split(sep).join("/").match(PLUGIN_SKILL_PATH),
239
- )
240
- .find((match) => match !== null);
241
- const match = mounted ?? plugin;
242
- return match ? match.slice(1, 2).join("") : null;
243
- }
244
- async function verifiedSkillBody(
245
- actual: string,
246
- skill: string,
247
- ): Promise<string | null> {
248
- const body = await readFile(actual).catch(() => null);
249
- if (!body || body.byteLength > MAX_SKILL_BYTES) return null;
250
- const text = body.toString("utf8");
251
- if (!text.startsWith("---\n") || !text.includes(`\nname: ${skill}\n`))
252
- return null;
253
- return text;
254
- }
255
- async function resolvedSkillRead(
256
- declaration: { read: DirectRead; output: string },
257
- workspace: string,
258
- pluginRoots: string[],
259
- commandCwd: string,
260
- ): Promise<VerifiedRead | null> {
261
- const actual = await realpath(
262
- resolve(commandCwd, declaration.read.path),
263
- ).catch(() => null);
264
- if (!actual) return null;
265
- const skill = mountedSkillName(actual, workspace, pluginRoots);
266
- if (skill === null) return null;
267
- const text = await verifiedSkillBody(actual, skill);
268
- if (text === null) return null;
269
- return skillReadReceipt(actual, skill, declaration, text);
270
- }
271
- function skillReadReceipt(
272
- actual: string,
273
- skill: string,
274
- declaration: { read: DirectRead; output: string },
275
- text: string,
276
- ): VerifiedRead | null {
277
- const range = readRange(text, declaration.read);
278
- if (range[0] === range[1]) return null;
279
- if (!declaration.output.includes(text.slice(...range))) return null;
280
- return { skill, path: actual, range, bodyLength: text.length };
281
- }
282
-
283
- function fullCoverage(
284
- ranges: Array<[number, number]>,
285
- length: number,
286
- ): boolean {
287
- const ordered = ranges.sort((left, right) => left[0] - right[0]);
288
- let covered = 0;
289
- for (const [start, end] of ordered) {
290
- if (start > covered) return false;
291
- covered = Math.max(covered, end);
292
- }
293
- return covered === length;
294
- }
295
-
296
- interface NativeEntry {
297
- ordinal: number;
298
- payload: Record<string, unknown>;
299
- }
300
-
301
- export interface NativeReadDiagnostic {
302
- completeness: "complete" | "partial";
303
- observedSkills: string[];
304
- completedReads: Array<{ skill: string; ordinal: number }>;
305
- recoverySources: Array<{
306
- ordinal: number;
307
- nativeOutput: boolean;
308
- yieldedChunks: number;
309
- completedCall: boolean;
310
- literalCommandCall: boolean;
311
- }>;
312
- commandExecutions: number;
313
- readAttempts: number;
314
- truncated: boolean;
315
- }
316
-
317
- function nativeShellArguments(value: string[]): boolean {
318
- return value.length >= 3 && (value[1] === "-lc" || value[1] === "-c");
319
- }
320
- function nativeCommandText(value: unknown): string | null {
321
- if (typeof value === "string") return value;
322
- if (
323
- !isUnknownArray(value) ||
324
- !value.every((part): part is string => typeof part === "string")
325
- )
326
- return null;
327
- return nativeShellArguments(value)
328
- ? value.slice(2).join(" ")
329
- : value.join(" ");
330
- }
331
-
332
- type Coverage = { ranges: Array<[number, number]>; bodyLength: number };
333
- interface CoverageState {
334
- attempted: Map<string, Coverage>;
335
- observedSkills: string[];
336
- }
337
- interface NativeReadState extends CoverageState {
338
- completedReads: NativeReadDiagnostic["completedReads"];
339
- recoverySources: NativeReadDiagnostic["recoverySources"];
340
- commandExecutions: number;
341
- readAttempts: number;
342
- partial: boolean;
343
- truncated: boolean;
344
- }
345
- interface ReadContext {
346
- workspace: string;
347
- pluginRoots: string[];
348
- recovered: Map<number, RecoveredCommandOutput>;
349
- }
350
- function nativeReadState(): NativeReadState {
351
- return {
352
- attempted: new Map(),
353
- observedSkills: [],
354
- completedReads: [],
355
- recoverySources: [],
356
- commandExecutions: 0,
357
- readAttempts: 0,
358
- partial: false,
359
- truncated: false,
360
- };
361
- }
362
-
363
- /** Verify native skill-file reads without retaining commands or output. */
364
- export async function codexNativeReadDiagnostic(
365
- entries: NativeEntry[],
366
- workspace: string,
367
- installedPluginRoots: string[] = [],
368
- ): Promise<NativeReadDiagnostic> {
369
- const canonicalWorkspace = await realpath(workspace);
370
- const pluginRoots = await Promise.all(
371
- installedPluginRoots.map((root) => realpath(root)),
372
- );
373
- const context: ReadContext = {
374
- workspace: canonicalWorkspace,
375
- pluginRoots,
376
- recovered: nativeCommandOutputs(entries),
377
- };
378
- const state = nativeReadState();
379
- for (const entry of entries)
380
- if (!(await nativeReadEntry(entry, context, state))) break;
381
- return nativeReadResult(state);
382
- }
383
-
384
- function nativeCommandItem(
385
- payload: Record<string, unknown>,
386
- ): (CodexItem & { cwd?: unknown }) | null {
387
- if (
388
- payload.type !== "item_completed" ||
389
- !isRecord(payload.item) ||
390
- payload.item.type !== "CommandExecution"
391
- )
392
- return null;
393
- return payload.item;
394
- }
395
-
396
- async function nativeReadEntry(
397
- { ordinal, payload }: NativeEntry,
398
- context: ReadContext,
399
- state: NativeReadState,
400
- ): Promise<boolean> {
401
- const item = nativeCommandItem(payload);
402
- if (!item) return true;
403
- state.commandExecutions++;
404
- const command = nativeCommandText(item.command);
405
- if (!command) {
406
- if (
407
- await exposesMountedSkillFrontmatter(
408
- combinedNativeOutput(item, context.recovered.get(ordinal)),
409
- context.workspace,
410
- context.pluginRoots,
411
- )
412
- ) {
413
- if (!countNativeRead(state)) return false;
414
- const recovered = context.recovered.get(ordinal);
415
- state.recoverySources.push({
416
- ordinal,
417
- nativeOutput: hasNativeOutput(item),
418
- ...recoveryFlags(recovered),
419
- });
420
- state.partial = true;
421
- }
422
- return true;
423
- }
424
- return processNativeRead(item, command, ordinal, context, state);
425
- }
426
-
427
- function countNativeRead(state: NativeReadState): boolean {
428
- state.readAttempts++;
429
- if (state.readAttempts <= MAX_NATIVE_READ_ATTEMPTS) return true;
430
- state.readAttempts = MAX_NATIVE_READ_ATTEMPTS;
431
- state.truncated = true;
432
- state.partial = true;
433
- return false;
434
- }
435
-
436
- async function processNativeRead(
437
- item: CodexItem & { cwd?: unknown },
438
- command: string,
439
- ordinal: number,
440
- context: ReadContext,
441
- state: NativeReadState,
442
- ): Promise<boolean> {
443
- const attempted = await nativeSkillReadAttempt(item, command, context);
444
- const exposed = await exposesMountedSkillFrontmatter(
445
- combinedNativeOutput(item, context.recovered.get(ordinal)),
446
- context.workspace,
447
- context.pluginRoots,
448
- );
449
- if (!attempted && !exposed) return true;
450
- if (!countNativeRead(state)) return false;
451
- const recovered = context.recovered.get(ordinal);
452
- state.recoverySources.push({
453
- ordinal,
454
- nativeOutput: hasNativeOutput(item),
455
- ...recoveryFlags(recovered),
456
- });
457
- if (!attempted) {
458
- state.partial = true;
459
- return true;
460
- }
461
- await completeNativeRead(item, command, ordinal, context, state);
462
- return true;
463
- }
464
- async function nativeSkillReadAttempt(
465
- item: CodexItem & { cwd?: unknown },
466
- command: string,
467
- context: ReadContext,
468
- ): Promise<boolean> {
469
- const cwd = await nativeReadCwd(item.cwd, context.workspace);
470
- if (cwd === null) return readAttemptPaths(command).length > 0;
471
- return mountedSkillReadAttempt(
472
- command,
473
- context.workspace,
474
- context.pluginRoots,
475
- cwd,
476
- );
477
- }
478
- function hasNativeOutput(item: CodexItem): boolean {
479
- return (
480
- typeof item.aggregated_output === "string" &&
481
- item.aggregated_output.length > 0
482
- );
483
- }
484
- function recoveryFlags(recovered: RecoveredCommandOutput | undefined) {
485
- return {
486
- yieldedChunks: recovered?.chunks ?? 0,
487
- completedCall: recovered?.completedCall === true,
488
- literalCommandCall: recovered?.literalCommandCall === true,
489
- };
490
- }
491
- function successfulReadCommand(item: CodexItem): boolean {
492
- return item.exit_code === 0 && item.status === "completed";
493
- }
494
- async function nativeReadCwd(
495
- cwd: unknown,
496
- workspace: string,
497
- ): Promise<string | null> {
498
- if (cwd === undefined) return workspace;
499
- if (typeof cwd !== "string" || !isAbsolute(cwd)) return null;
500
- return realpath(cwd).catch(() => null);
501
- }
502
- function nativeOutput(item: CodexItem): string {
503
- return typeof item.aggregated_output === "string"
504
- ? item.aggregated_output
505
- : "";
506
- }
507
- function combinedNativeOutput(
508
- item: CodexItem,
509
- recovered: RecoveredCommandOutput | undefined,
510
- ): string {
511
- return (recovered?.output ?? "") + nativeOutput(item);
512
- }
513
- async function completeNativeRead(
514
- item: CodexItem & { cwd?: unknown },
515
- command: string,
516
- ordinal: number,
517
- context: ReadContext,
518
- state: NativeReadState,
519
- ): Promise<void> {
520
- if (!successfulReadCommand(item)) {
521
- state.partial = true;
522
- return;
523
- }
524
- const cwd = await nativeReadCwd(item.cwd, context.workspace);
525
- if (!cwd) {
526
- state.partial = true;
527
- return;
528
- }
529
- const read = await verifiedSkillRead(
530
- {
531
- ...item,
532
- command,
533
- aggregated_output: combinedNativeOutput(
534
- item,
535
- context.recovered.get(ordinal),
536
- ),
537
- },
538
- context.workspace,
539
- context.pluginRoots,
540
- cwd,
541
- );
542
- if (!read) {
543
- state.partial = true;
544
- return;
545
- }
546
- retainNativeRead(state, read, ordinal);
547
- }
548
- function continuingNativeCoverage(
549
- state: NativeReadState,
550
- read: VerifiedRead,
551
- ): Coverage {
552
- const prior = state.attempted.get(read.path);
553
- return prior && !fullCoverage(prior.ranges, prior.bodyLength)
554
- ? prior
555
- : { ranges: [], bodyLength: read.bodyLength };
556
- }
557
- function appendReadCoverage(
558
- state: CoverageState,
559
- read: VerifiedRead,
560
- coverage: Coverage,
561
- ): boolean {
562
- coverage.ranges.push(read.range);
563
- state.attempted.set(read.path, coverage);
564
- return fullCoverage(coverage.ranges, coverage.bodyLength);
565
- }
566
- function retainNativeRead(
567
- state: NativeReadState,
568
- read: VerifiedRead,
569
- ordinal: number,
570
- ): void {
571
- const coverage = continuingNativeCoverage(state, read);
572
- if (appendReadCoverage(state, read, coverage)) {
573
- state.completedReads.push({ skill: read.skill, ordinal });
574
- if (!state.observedSkills.includes(read.skill))
575
- state.observedSkills.push(read.skill);
576
- }
577
- }
578
- function completeCoverage(state: CoverageState): boolean {
579
- return [...state.attempted.values()].every(({ ranges, bodyLength }) =>
580
- fullCoverage(ranges, bodyLength),
581
- );
582
- }
583
- function nativeReadResult(state: NativeReadState): NativeReadDiagnostic {
584
- return {
585
- completeness:
586
- !state.partial && completeCoverage(state) ? "complete" : "partial",
587
- observedSkills: state.observedSkills,
588
- completedReads: state.completedReads,
589
- recoverySources: state.recoverySources,
590
- commandExecutions: state.commandExecutions,
591
- readAttempts: state.readAttempts,
592
- truncated: state.truncated,
593
- };
594
- }
595
-
596
- interface StreamReadState extends CoverageState {
597
- pending: Set<string>;
598
- partial: boolean;
599
- completedTurn: boolean;
600
- }
601
- type RecoveredRead = { command: string; output: string };
602
- function streamReadState(): StreamReadState {
603
- return {
604
- attempted: new Map(),
605
- observedSkills: [],
606
- pending: new Set(),
607
- partial: false,
608
- completedTurn: false,
609
- };
610
- }
611
- function readEvent(line: string): CodexEvent {
612
- const value: unknown = JSON.parse(line);
613
- if (!isRecord(value)) throw new Error("Invalid Codex skill-read event");
614
- return {
615
- type: value.type,
616
- item: isRecord(value.item) ? value.item : undefined,
617
- };
618
- }
619
- function recoveredReadItem(
620
- item: CodexItem | undefined,
621
- recoveredReads: Map<string, RecoveredRead>,
622
- ): CodexItem | undefined {
623
- if (typeof item?.id !== "string") return item;
624
- return correlatedReadItem(item, recoveredReads.get(item.id));
625
- }
626
- function correlatedReadItem(
627
- item: CodexItem,
628
- recovered: RecoveredRead | undefined,
629
- ): CodexItem {
630
- if (
631
- !recovered ||
632
- typeof item.command !== "string" ||
633
- shellPayload(item.command) !== shellPayload(recovered.command)
634
- )
635
- return item;
636
- return { ...item, aggregated_output: recovered.output + nativeOutput(item) };
637
- }
638
- function observeStreamTurn(event: CodexEvent, state: StreamReadState): void {
639
- if (event.type === "turn.completed") state.completedTurn = true;
640
- if (event.type === "turn.failed") state.partial = true;
641
- }
642
- async function streamReadEvent(
643
- event: CodexEvent,
644
- item: CodexItem | undefined,
645
- context: Pick<ReadContext, "workspace" | "pluginRoots">,
646
- state: StreamReadState,
647
- ): Promise<void> {
648
- observeStreamTurn(event, state);
649
- if (!item || item.type !== "command_execution") return;
650
- if (event.type === "item.started") {
651
- await pendingStreamRead(item, context, state);
652
- return;
653
- }
654
- if (event.type !== "item.completed") return;
655
- await completeStreamRead(item, context, state);
656
- }
657
- async function pendingStreamRead(
658
- item: CodexItem,
659
- context: Pick<ReadContext, "workspace" | "pluginRoots">,
660
- state: StreamReadState,
661
- ): Promise<void> {
662
- if (
663
- typeof item.command !== "string" ||
664
- !(await mountedSkillReadAttempt(
665
- item.command,
666
- context.workspace,
667
- context.pluginRoots,
668
- ))
669
- )
670
- return;
671
- if (typeof item.id === "string") state.pending.add(item.id);
672
- else state.partial = true;
673
- }
674
- async function eligibleCompletedRead(
675
- item: CodexItem,
676
- context: Pick<ReadContext, "workspace" | "pluginRoots">,
677
- state: StreamReadState,
678
- ): Promise<boolean> {
679
- const wasPending =
680
- typeof item.id === "string" && state.pending.delete(item.id);
681
- if (await completedMountedRead(item, context, state)) return true;
682
- if (wasPending) {
683
- state.partial = true;
684
- return false;
685
- }
686
- state.partial ||= await exposesMountedSkillFrontmatter(
687
- nativeOutput(item),
688
- context.workspace,
689
- context.pluginRoots,
690
- );
691
- return false;
692
- }
693
- async function completedMountedRead(
694
- item: CodexItem,
695
- context: Pick<ReadContext, "workspace" | "pluginRoots">,
696
- state: StreamReadState,
697
- ): Promise<boolean> {
698
- if (typeof item.command !== "string") return false;
699
- const attempted = await mountedSkillReadAttempt(
700
- item.command,
701
- context.workspace,
702
- context.pluginRoots,
703
- );
704
- if (!attempted) return false;
705
- if (!successfulReadCommand(item)) state.partial = true;
706
- return successfulReadCommand(item);
707
- }
708
- async function completeStreamRead(
709
- item: CodexItem,
710
- context: Pick<ReadContext, "workspace" | "pluginRoots">,
711
- state: StreamReadState,
712
- ): Promise<void> {
713
- if (!(await eligibleCompletedRead(item, context, state))) return;
714
- const read = await verifiedSkillRead(
715
- item,
716
- context.workspace,
717
- context.pluginRoots,
718
- );
719
- if (!read) {
720
- state.partial = true;
721
- return;
722
- }
723
- const coverage = state.attempted.get(read.path) ?? {
724
- ranges: [],
725
- bodyLength: read.bodyLength,
726
- };
727
- retainStreamRead(state, read, coverage);
728
- }
729
-
730
- function retainStreamRead(
731
- state: StreamReadState,
732
- read: VerifiedRead,
733
- coverage: Coverage,
734
- ): void {
735
- if (
736
- appendReadCoverage(state, read, coverage) &&
737
- !state.observedSkills.includes(read.skill)
738
- )
739
- state.observedSkills.push(read.skill);
740
- }
741
- function streamReadCompleteness(
742
- state: StreamReadState,
743
- ): "complete" | "partial" {
744
- return state.completedTurn &&
745
- !state.partial &&
746
- state.pending.size === 0 &&
747
- completeCoverage(state)
748
- ? "complete"
749
- : "partial";
750
- }
751
- function streamReadResult(state: StreamReadState) {
752
- return {
753
- id: "sevro.codex.skill-reads" as const,
754
- completeness: streamReadCompleteness(state),
755
- data: {
756
- method: "skill_file_read_probe",
757
- primarySkill: state.observedSkills[0] ?? null,
758
- observedSkills: state.observedSkills,
759
- },
760
- };
761
- }
762
-
763
- /** Retain only ordered mounted-skill names, never commands or skill bodies. */
764
- export async function codexSkillReadObservation(
765
- stream: string,
766
- workspace: string,
767
- installedPluginRoots: string[] = [],
768
- recoveredReads: Map<string, RecoveredRead> = new Map(),
769
- ) {
770
- const state = streamReadState();
771
- const canonicalWorkspace = await realpath(workspace);
772
- const pluginRoots = await Promise.all(
773
- installedPluginRoots.map((root) => realpath(root)),
774
- );
775
- const context = { workspace: canonicalWorkspace, pluginRoots };
776
- for (const line of stream.split("\n")) {
777
- if (!line.trim()) continue;
778
- const event = readEvent(line);
779
- const item = recoveredReadItem(event.item, recoveredReads);
780
- await streamReadEvent(event, item, context, state);
781
- }
782
- return streamReadResult(state);
783
- }