@wix/pathgrade 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. package/bin/pathgrade.js +2 -0
  2. package/dist/affected/anchor.d.ts +18 -0
  3. package/dist/affected/anchor.js +42 -0
  4. package/dist/affected/config.d.ts +25 -0
  5. package/dist/affected/config.js +98 -0
  6. package/dist/affected/format.d.ts +45 -0
  7. package/dist/affected/format.js +87 -0
  8. package/dist/affected/git.d.ts +33 -0
  9. package/dist/affected/git.js +68 -0
  10. package/dist/affected/glob.d.ts +17 -0
  11. package/dist/affected/glob.js +25 -0
  12. package/dist/affected/meta.d.ts +26 -0
  13. package/dist/affected/meta.js +158 -0
  14. package/dist/affected/select.d.ts +30 -0
  15. package/dist/affected/select.js +101 -0
  16. package/dist/affected/sidecar.d.ts +32 -0
  17. package/dist/affected/sidecar.js +86 -0
  18. package/dist/affected/types.d.ts +43 -0
  19. package/dist/affected/types.js +7 -0
  20. package/dist/agents/claude/ask-user-answer-store.d.ts +28 -0
  21. package/dist/agents/claude/ask-user-answer-store.js +29 -0
  22. package/dist/agents/claude/ask-user-bridge.d.ts +56 -0
  23. package/dist/agents/claude/ask-user-bridge.js +162 -0
  24. package/dist/agents/claude/sdk-message-projector.d.ts +43 -0
  25. package/dist/agents/claude/sdk-message-projector.js +208 -0
  26. package/dist/agents/claude/sdk-options.d.ts +52 -0
  27. package/dist/agents/claude/sdk-options.js +79 -0
  28. package/dist/agents/claude.d.ts +56 -0
  29. package/dist/agents/claude.js +143 -0
  30. package/dist/agents/codex-app-server/agent.d.ts +35 -0
  31. package/dist/agents/codex-app-server/agent.js +388 -0
  32. package/dist/agents/codex-app-server/fixtures/app-server-client.d.ts +19 -0
  33. package/dist/agents/codex-app-server/fixtures/app-server-client.js +104 -0
  34. package/dist/agents/codex-app-server/fixtures/run-gate.d.ts +25 -0
  35. package/dist/agents/codex-app-server/fixtures/run-gate.js +48 -0
  36. package/dist/agents/codex-app-server/protocol/ClientRequest.d.ts +27 -0
  37. package/dist/agents/codex-app-server/protocol/ClientRequest.js +7 -0
  38. package/dist/agents/codex-app-server/protocol/DynamicToolCallParams.d.ts +9 -0
  39. package/dist/agents/codex-app-server/protocol/DynamicToolCallParams.js +4 -0
  40. package/dist/agents/codex-app-server/protocol/GrantedPermissionProfile.d.ts +6 -0
  41. package/dist/agents/codex-app-server/protocol/GrantedPermissionProfile.js +7 -0
  42. package/dist/agents/codex-app-server/protocol/McpElicitationRequestParams.d.ts +23 -0
  43. package/dist/agents/codex-app-server/protocol/McpElicitationRequestParams.js +8 -0
  44. package/dist/agents/codex-app-server/protocol/Op.d.ts +11 -0
  45. package/dist/agents/codex-app-server/protocol/Op.js +5 -0
  46. package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.d.ts +10 -0
  47. package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.js +6 -0
  48. package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.d.ts +15 -0
  49. package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.js +4 -0
  50. package/dist/agents/codex-app-server/protocol/SandboxMode.d.ts +1 -0
  51. package/dist/agents/codex-app-server/protocol/SandboxMode.js +4 -0
  52. package/dist/agents/codex-app-server/protocol/ServerRequest.d.ts +42 -0
  53. package/dist/agents/codex-app-server/protocol/ServerRequest.js +7 -0
  54. package/dist/agents/codex-app-server/protocol/ThreadStartParams.d.ts +45 -0
  55. package/dist/agents/codex-app-server/protocol/ThreadStartParams.js +10 -0
  56. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.d.ts +6 -0
  57. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.js +4 -0
  58. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputOption.d.ts +7 -0
  59. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputOption.js +4 -0
  60. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputParams.d.ts +10 -0
  61. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputParams.js +4 -0
  62. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.d.ts +12 -0
  63. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.js +4 -0
  64. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputResponse.d.ts +9 -0
  65. package/dist/agents/codex-app-server/protocol/ToolRequestUserInputResponse.js +4 -0
  66. package/dist/agents/codex-app-server/protocol/TurnCompletedNotification.d.ts +5 -0
  67. package/dist/agents/codex-app-server/protocol/TurnCompletedNotification.js +4 -0
  68. package/dist/agents/codex-app-server/protocol/index.d.ts +16 -0
  69. package/dist/agents/codex-app-server/protocol/index.js +11 -0
  70. package/dist/agents/codex-app-server/transport.d.ts +84 -0
  71. package/dist/agents/codex-app-server/transport.js +246 -0
  72. package/dist/agents/codex-app-server/wire-translators.d.ts +19 -0
  73. package/dist/agents/codex-app-server/wire-translators.js +52 -0
  74. package/dist/agents/codex.d.ts +7 -0
  75. package/dist/agents/codex.js +164 -0
  76. package/dist/agents/cursor.d.ts +39 -0
  77. package/dist/agents/cursor.js +233 -0
  78. package/dist/agents/registry.d.ts +17 -0
  79. package/dist/agents/registry.js +27 -0
  80. package/dist/agents/transcript-agent.d.ts +14 -0
  81. package/dist/agents/transcript-agent.js +59 -0
  82. package/dist/analytics/engine.d.ts +18 -0
  83. package/dist/analytics/engine.js +66 -0
  84. package/dist/commands/affected.d.ts +31 -0
  85. package/dist/commands/affected.js +150 -0
  86. package/dist/commands/analyze.d.ts +21 -0
  87. package/dist/commands/analyze.js +167 -0
  88. package/dist/commands/init.d.ts +3 -0
  89. package/dist/commands/init.js +242 -0
  90. package/dist/commands/preview-reactions.d.ts +10 -0
  91. package/dist/commands/preview-reactions.js +88 -0
  92. package/dist/commands/preview.d.ts +5 -0
  93. package/dist/commands/preview.js +17 -0
  94. package/dist/commands/report.d.ts +39 -0
  95. package/dist/commands/report.js +108 -0
  96. package/dist/commands/run-args.d.ts +22 -0
  97. package/dist/commands/run-args.js +69 -0
  98. package/dist/commands/run-changed.d.ts +26 -0
  99. package/dist/commands/run-changed.js +138 -0
  100. package/dist/commands/validate.d.ts +13 -0
  101. package/dist/commands/validate.js +233 -0
  102. package/dist/core/mcp-mock.d.ts +3 -0
  103. package/dist/core/mcp-mock.js +23 -0
  104. package/dist/core/mcp-mock.types.d.ts +15 -0
  105. package/dist/core/mcp-mock.types.js +1 -0
  106. package/dist/core/skills.d.ts +15 -0
  107. package/dist/core/skills.js +76 -0
  108. package/dist/evals/discovery.d.ts +7 -0
  109. package/dist/evals/discovery.js +69 -0
  110. package/dist/mcp-mock-server.d.ts +1 -0
  111. package/dist/mcp-mock-server.js +116 -0
  112. package/dist/pathgrade.d.ts +10 -0
  113. package/dist/pathgrade.js +243 -0
  114. package/dist/plugin/index.d.ts +9 -0
  115. package/dist/plugin/index.js +81 -0
  116. package/dist/plugin/lifecycle.d.ts +45 -0
  117. package/dist/plugin/lifecycle.js +146 -0
  118. package/dist/plugin/reporter.d.ts +36 -0
  119. package/dist/plugin/reporter.js +275 -0
  120. package/dist/plugin/setup.d.ts +1 -0
  121. package/dist/plugin/setup.js +3 -0
  122. package/dist/providers/copy-filter.d.ts +18 -0
  123. package/dist/providers/copy-filter.js +66 -0
  124. package/dist/providers/credentials.d.ts +36 -0
  125. package/dist/providers/credentials.js +202 -0
  126. package/dist/providers/mcp-config.d.ts +34 -0
  127. package/dist/providers/mcp-config.js +55 -0
  128. package/dist/providers/sandbox-exec.d.ts +8 -0
  129. package/dist/providers/sandbox-exec.js +82 -0
  130. package/dist/providers/sandbox.d.ts +22 -0
  131. package/dist/providers/sandbox.js +158 -0
  132. package/dist/providers/sandboxed-claude-spawn.d.ts +24 -0
  133. package/dist/providers/sandboxed-claude-spawn.js +57 -0
  134. package/dist/providers/workspace.d.ts +15 -0
  135. package/dist/providers/workspace.js +73 -0
  136. package/dist/reporters/browser.d.ts +2 -0
  137. package/dist/reporters/browser.js +85 -0
  138. package/dist/reporters/cli.d.ts +2 -0
  139. package/dist/reporters/cli.js +131 -0
  140. package/dist/reporters/diagnostics.d.ts +46 -0
  141. package/dist/reporters/diagnostics.js +147 -0
  142. package/dist/reporters/github-comment.d.ts +67 -0
  143. package/dist/reporters/github-comment.js +252 -0
  144. package/dist/reporters/loader.d.ts +8 -0
  145. package/dist/reporters/loader.js +36 -0
  146. package/dist/reporters/results-path.d.ts +2 -0
  147. package/dist/reporters/results-path.js +10 -0
  148. package/dist/reporters/verbose-emitter.d.ts +69 -0
  149. package/dist/reporters/verbose-emitter.js +103 -0
  150. package/dist/sdk/agent-crash.d.ts +19 -0
  151. package/dist/sdk/agent-crash.js +20 -0
  152. package/dist/sdk/agent-resolution.d.ts +12 -0
  153. package/dist/sdk/agent-resolution.js +21 -0
  154. package/dist/sdk/agent-result-log.d.ts +19 -0
  155. package/dist/sdk/agent-result-log.js +45 -0
  156. package/dist/sdk/agent.d.ts +4 -0
  157. package/dist/sdk/agent.js +354 -0
  158. package/dist/sdk/ask-bus/bus.d.ts +21 -0
  159. package/dist/sdk/ask-bus/bus.js +129 -0
  160. package/dist/sdk/ask-bus/handler.d.ts +31 -0
  161. package/dist/sdk/ask-bus/handler.js +244 -0
  162. package/dist/sdk/ask-bus/parsers.d.ts +9 -0
  163. package/dist/sdk/ask-bus/parsers.js +30 -0
  164. package/dist/sdk/ask-bus/projection.d.ts +26 -0
  165. package/dist/sdk/ask-bus/projection.js +45 -0
  166. package/dist/sdk/ask-bus/types.d.ts +69 -0
  167. package/dist/sdk/ask-bus/types.js +1 -0
  168. package/dist/sdk/chat.d.ts +29 -0
  169. package/dist/sdk/chat.js +93 -0
  170. package/dist/sdk/conversation-window.d.ts +14 -0
  171. package/dist/sdk/conversation-window.js +38 -0
  172. package/dist/sdk/converse.d.ts +42 -0
  173. package/dist/sdk/converse.js +329 -0
  174. package/dist/sdk/eval-runtime.d.ts +10 -0
  175. package/dist/sdk/eval-runtime.js +17 -0
  176. package/dist/sdk/evaluate.d.ts +13 -0
  177. package/dist/sdk/evaluate.js +325 -0
  178. package/dist/sdk/index.d.ts +31 -0
  179. package/dist/sdk/index.js +20 -0
  180. package/dist/sdk/judge-pipeline.d.ts +7 -0
  181. package/dist/sdk/judge-pipeline.js +178 -0
  182. package/dist/sdk/judge-prompt-builder.d.ts +7 -0
  183. package/dist/sdk/judge-prompt-builder.js +76 -0
  184. package/dist/sdk/judge-tool-runner.d.ts +19 -0
  185. package/dist/sdk/judge-tool-runner.js +51 -0
  186. package/dist/sdk/judge-tool-session.d.ts +47 -0
  187. package/dist/sdk/judge-tool-session.js +213 -0
  188. package/dist/sdk/judge-tools.d.ts +29 -0
  189. package/dist/sdk/judge-tools.js +283 -0
  190. package/dist/sdk/managed-session.d.ts +50 -0
  191. package/dist/sdk/managed-session.js +110 -0
  192. package/dist/sdk/persona.d.ts +5 -0
  193. package/dist/sdk/persona.js +33 -0
  194. package/dist/sdk/reaction-loader.d.ts +4 -0
  195. package/dist/sdk/reaction-loader.js +119 -0
  196. package/dist/sdk/reaction-preview.d.ts +6 -0
  197. package/dist/sdk/reaction-preview.js +94 -0
  198. package/dist/sdk/run-scorer.d.ts +10 -0
  199. package/dist/sdk/run-scorer.js +124 -0
  200. package/dist/sdk/runtime-policy.d.ts +21 -0
  201. package/dist/sdk/runtime-policy.js +89 -0
  202. package/dist/sdk/scorer-utils.d.ts +6 -0
  203. package/dist/sdk/scorer-utils.js +39 -0
  204. package/dist/sdk/scorers.d.ts +64 -0
  205. package/dist/sdk/scorers.js +80 -0
  206. package/dist/sdk/snapshots.d.ts +41 -0
  207. package/dist/sdk/snapshots.js +148 -0
  208. package/dist/sdk/types.d.ts +416 -0
  209. package/dist/sdk/types.js +17 -0
  210. package/dist/sdk/visible-turn.d.ts +4 -0
  211. package/dist/sdk/visible-turn.js +23 -0
  212. package/dist/tool-events.d.ts +45 -0
  213. package/dist/tool-events.js +167 -0
  214. package/dist/types.d.ts +372 -0
  215. package/dist/types.js +37 -0
  216. package/dist/utils/cli.d.ts +55 -0
  217. package/dist/utils/cli.js +131 -0
  218. package/dist/utils/env.d.ts +8 -0
  219. package/dist/utils/env.js +26 -0
  220. package/dist/utils/llm-mocks.d.ts +43 -0
  221. package/dist/utils/llm-mocks.js +108 -0
  222. package/dist/utils/llm-providers/anthropic.d.ts +2 -0
  223. package/dist/utils/llm-providers/anthropic.js +132 -0
  224. package/dist/utils/llm-providers/cli.d.ts +20 -0
  225. package/dist/utils/llm-providers/cli.js +162 -0
  226. package/dist/utils/llm-providers/openai.d.ts +2 -0
  227. package/dist/utils/llm-providers/openai.js +56 -0
  228. package/dist/utils/llm-types.d.ts +107 -0
  229. package/dist/utils/llm-types.js +1 -0
  230. package/dist/utils/llm.d.ts +36 -0
  231. package/dist/utils/llm.js +209 -0
  232. package/dist/utils/shutdown.d.ts +15 -0
  233. package/dist/utils/shutdown.js +41 -0
  234. package/dist/utils/timeout.d.ts +8 -0
  235. package/dist/utils/timeout.js +32 -0
  236. package/dist/viewer.html +1178 -0
  237. package/package.json +88 -0
  238. package/templates/eval.ts.template +28 -0
@@ -0,0 +1,372 @@
1
+ import type { DiagnosticsReport } from './reporters/diagnostics.js';
2
+ import type { RuntimePolicyDescriptor } from './sdk/runtime-policy.js';
3
+ import type { LLMPort } from './utils/llm-types.js';
4
+ export interface CommandResult {
5
+ stdout: string;
6
+ stderr: string;
7
+ exitCode: number;
8
+ timedOut?: boolean;
9
+ killed?: boolean;
10
+ }
11
+ export interface TurnCommand extends CommandResult {
12
+ command: string;
13
+ }
14
+ export type ConversationReplySource = 'opener' | 'reaction' | 'persona_llm';
15
+ export interface ConversationTurn {
16
+ turn_number: number;
17
+ user_message: string;
18
+ user_message_source: ConversationReplySource;
19
+ raw_agent_output: string;
20
+ assistant_message: string;
21
+ duration_ms: number;
22
+ commands: TurnCommand[];
23
+ turn_status: 'completed' | 'error' | 'timeout';
24
+ step_scorer_results?: ScorerResult[];
25
+ tool_events?: import('./tool-events.js').ToolEvent[];
26
+ }
27
+ export type ConversationCompletionReason = 'max_turns' | 'signal' | 'done_phrase' | 'done_when' | 'timeout' | 'no_replies' | 'error';
28
+ export interface ScorerResult {
29
+ scorer_type: string;
30
+ score: number;
31
+ weight: number;
32
+ details: string;
33
+ status?: import('./sdk/types.js').ScorerStatus;
34
+ }
35
+ /**
36
+ * JSON contract for deterministic scorer stdout.
37
+ * The scorer script must print a JSON object matching this shape to stdout.
38
+ */
39
+ export interface ScorerOutput {
40
+ /** Score between 0.0 and 1.0 (required). Clamped by pathgrade. */
41
+ score: number;
42
+ /** Human-readable summary (optional). */
43
+ details?: string;
44
+ /** Per-check breakdown, rendered as checkmarks in reports (optional). */
45
+ checks?: ScorerCheck[];
46
+ }
47
+ /** Individual check result within a ScorerOutput. */
48
+ export interface ScorerCheck {
49
+ name: string;
50
+ passed: boolean;
51
+ message?: string;
52
+ }
53
+ export interface JudgeToolCallLogData {
54
+ /** Tool name (readFile | listDir | grep | getToolEvents). */
55
+ name: string;
56
+ /** Arguments the LLM passed to the tool. */
57
+ input: unknown;
58
+ /** True if the tool returned content; false if it errored. */
59
+ ok: boolean;
60
+ /** Bytes of content returned to the LLM (0 on error). */
61
+ bytes: number;
62
+ /** Error message when ok is false. */
63
+ errorMessage?: string;
64
+ /** The owning judge scorer's name, for per-judge reporting. */
65
+ judge_name: string;
66
+ }
67
+ export interface LogEntry {
68
+ type: 'agent_start' | 'command' | 'agent_result' | 'scorer' | 'reward' | 'user_reply' | 'step_scorer' | 'tool_event' | 'conversation_end' | 'judge_tool_call' | 'ask_batch';
69
+ timestamp: string;
70
+ instruction?: string;
71
+ command?: string;
72
+ stdout?: string;
73
+ stderr?: string;
74
+ exitCode?: number;
75
+ output?: string;
76
+ assistant_message?: string;
77
+ assistant_message_source?: VisibleAssistantMessageSource | 'blocked_prompt';
78
+ raw_assistant_message?: string;
79
+ value?: number;
80
+ scorer_result?: ScorerResult;
81
+ turn_number?: number;
82
+ reply_source?: ConversationReplySource;
83
+ step_scorer_key?: string;
84
+ tool_event?: import('./tool-events.js').ToolEvent;
85
+ judge_tool_call?: JudgeToolCallLogData;
86
+ /**
87
+ * @deprecated Read-only compatibility for snapshots predating the
88
+ * blocked-prompt synthesis deletion. New writers never emit any of the
89
+ * `blocked_prompt_*` / `synthetic_blocked_prompt` fields, but
90
+ * `loadRunSnapshot` still parses them so historical run-snapshots load.
91
+ * Do not set these fields from new code paths.
92
+ */
93
+ synthetic_blocked_prompt?: boolean;
94
+ /** @deprecated See `synthetic_blocked_prompt`. */
95
+ blocked_prompt_source_turn?: number;
96
+ /** @deprecated See `synthetic_blocked_prompt`. */
97
+ blocked_prompt_index?: number;
98
+ /** @deprecated See `synthetic_blocked_prompt`. */
99
+ blocked_prompt_count?: number;
100
+ /** @deprecated See `synthetic_blocked_prompt`. */
101
+ blocked_prompt_source_tool?: string;
102
+ /** @deprecated See `synthetic_blocked_prompt`. */
103
+ blocked_prompt_tool_use_id?: string;
104
+ runtime_policies_applied?: RuntimePolicyDescriptor[];
105
+ /**
106
+ * Per-turn agent cost in USD when the upstream provider reports it (the
107
+ * Claude Agent SDK populates `total_cost_usd` on result messages).
108
+ * Written by `buildModelAgentResultLogEntry` only when the turn result
109
+ * carries a `costUsd` value; absent for agents that do not expose cost.
110
+ */
111
+ cost_usd?: number;
112
+ completion_reason?: string;
113
+ completion_detail?: string;
114
+ turn_timings?: Array<{
115
+ turn: number;
116
+ durationMs: number;
117
+ }>;
118
+ duration_ms?: number;
119
+ output_lines?: number;
120
+ output_chars?: number;
121
+ turn_details?: Array<{
122
+ turn: number;
123
+ durationMs: number;
124
+ outputLines: number;
125
+ outputChars: number;
126
+ }>;
127
+ reactions_fired?: Array<{
128
+ turn: number;
129
+ reactionIndex: number;
130
+ pattern: string;
131
+ reply: string;
132
+ }>;
133
+ /** Upstream batch identifier (ToolRequestUserInput itemId when available; synthesized for adapters without one). */
134
+ batch_id?: string;
135
+ /** Agent/adapter that produced the batch. */
136
+ source?: import('./sdk/ask-bus/types.js').AskSource;
137
+ /** Lifecycle tag; determines whether answers flow back and when. */
138
+ lifecycle?: import('./sdk/ask-bus/types.js').AskLifecycle;
139
+ /** Upstream tool identifier (`AskUserQuestion` | `request_user_input` | `AskQuestion`). */
140
+ source_tool?: string;
141
+ /** Upstream tool-use correlation id when available. */
142
+ tool_use_id?: string;
143
+ /** Number of questions in the batch. */
144
+ question_count?: number;
145
+ /** Present for 'live' batches; absent for 'post-hoc'. */
146
+ resolved?: boolean;
147
+ }
148
+ export interface TrialResult {
149
+ trial_id: number;
150
+ name?: string;
151
+ reward: number;
152
+ scorer_results: ScorerResult[];
153
+ duration_ms: number;
154
+ n_commands: number;
155
+ input_tokens: number;
156
+ output_tokens: number;
157
+ conversation_input_tokens?: number;
158
+ conversation_output_tokens?: number;
159
+ /**
160
+ * Sum of agent-turn `costUsd` values accumulated during the
161
+ * conversation, attributed at evaluation time. Today only the Claude
162
+ * SDK driver populates per-turn cost, so this field is set on Claude
163
+ * trials and absent on Codex / Cursor trials.
164
+ */
165
+ conversation_cost_usd?: number;
166
+ /**
167
+ * Sum of all known cost components (conversation + judge + …).
168
+ * Conservative — emitted ONLY when every included component has a
169
+ * known cost. Today judge LLM providers do not expose cost, so this
170
+ * field is never emitted; `conversation_cost_usd` is the only
171
+ * guaranteed cost surface until that changes.
172
+ */
173
+ total_cost_usd?: number;
174
+ session_log: LogEntry[];
175
+ skills_used?: string[];
176
+ diagnostics?: DiagnosticsReport;
177
+ conversation?: {
178
+ turns: ConversationTurn[];
179
+ total_turns: number;
180
+ completion_reason: ConversationCompletionReason;
181
+ timeout_triggered_at_turn?: number;
182
+ };
183
+ }
184
+ export interface EvalReport {
185
+ task: string;
186
+ pass_rate: number;
187
+ pass_at_k: number;
188
+ pass_pow_k: number;
189
+ trials: TrialResult[];
190
+ skills_used: string[];
191
+ }
192
+ /**
193
+ * TrialResult with `session_log` and `conversation` stripped. These fields
194
+ * live only in the per-group trace files; the consolidated results.json keeps
195
+ * the rest so consumers (preview, `pathgrade report`) can compute summaries
196
+ * without loading trace data.
197
+ */
198
+ export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversation'>;
199
+ /**
200
+ * Per-group entry in the consolidated `.pathgrade/results.json` report.
201
+ */
202
+ export type PathgradeGroupReport = Omit<EvalReport, 'trials'> & {
203
+ trials: StrippedTrialResult[];
204
+ /** Relative path from `.pathgrade/` to the trace file for this group. */
205
+ trace_file: string;
206
+ };
207
+ /**
208
+ * Selection metadata merged into `PathgradeReport.selection` when
209
+ * `pathgrade run --changed` produced the run. Wire shape uses `snake_case`
210
+ * and matches the `.pathgrade/selection.json` sidecar written by the CLI.
211
+ *
212
+ * Reasons mirror `SelectionReason` from `src/affected/types.ts` (single
213
+ * source of truth — new variants added there propagate here automatically).
214
+ */
215
+ export interface PathgradeSelectionReport {
216
+ base_ref: string;
217
+ changed_files_count: number;
218
+ global_match?: string;
219
+ selected: string[];
220
+ skipped: Array<{
221
+ file: string;
222
+ reason: 'no-matching-deps';
223
+ }>;
224
+ }
225
+ /**
226
+ * Shape of `.pathgrade/results.json` — the consolidated consolidated pathgrade
227
+ * run report. `version` is pinned to 1 so the `pathgrade report` command and
228
+ * external consumers can gate on schema revisions.
229
+ */
230
+ export interface PathgradeReport {
231
+ version: 1;
232
+ timestamp: string;
233
+ /** `ci.threshold` from the plugin config, if configured. */
234
+ threshold?: number;
235
+ /** Weighted average of every individual trial score across all groups. */
236
+ overall_pass_rate: number;
237
+ /**
238
+ * Threshold check result. When `threshold` is set: `'pass'` iff
239
+ * `overall_pass_rate >= threshold`. Otherwise: `'pass'` iff every trial
240
+ * in every group passed its vitest test.
241
+ */
242
+ status: 'pass' | 'fail';
243
+ groups: PathgradeGroupReport[];
244
+ /**
245
+ * Present when `pathgrade run --changed` produced the run. Absent on
246
+ * plain `pathgrade run`. Backward compatible — older consumers ignore
247
+ * this field.
248
+ */
249
+ selection?: PathgradeSelectionReport;
250
+ }
251
+ export interface TrialPaths {
252
+ root: string;
253
+ workspace: string;
254
+ home: string;
255
+ xdg?: string;
256
+ xdgState?: string;
257
+ xdgCache?: string;
258
+ tmp: string;
259
+ }
260
+ export interface TrialRuntime {
261
+ handle: string;
262
+ workspacePath: string;
263
+ env: Record<string, string>;
264
+ paths?: TrialPaths;
265
+ }
266
+ export type EnvironmentHandle = string | TrialRuntime;
267
+ export type AgentCommandRunner = (cmd: string) => Promise<CommandResult>;
268
+ export interface AgentTurnInput {
269
+ message: string;
270
+ continueSession?: boolean;
271
+ }
272
+ export type VisibleAssistantMessageSource = 'assistant_message';
273
+ export interface AgentTurnResult {
274
+ rawOutput: string;
275
+ assistantMessage: string;
276
+ visibleAssistantMessage: string;
277
+ visibleAssistantMessageSource: VisibleAssistantMessageSource;
278
+ exitCode: number;
279
+ traceOutput?: string;
280
+ timedOut?: boolean;
281
+ toolEvents: import('./tool-events.js').ToolEvent[];
282
+ runtimePoliciesApplied?: RuntimePolicyDescriptor[];
283
+ inputTokens?: number;
284
+ outputTokens?: number;
285
+ /**
286
+ * Optional cache-token breakdown sourced from the Claude Agent SDK's
287
+ * `cache_creation_input_tokens` field. Additive only — `inputTokens`
288
+ * still includes cache-creation volume, matching pathgrade's existing
289
+ * convention.
290
+ */
291
+ cacheCreationInputTokens?: number;
292
+ /**
293
+ * Optional cache-token breakdown sourced from the Claude Agent SDK's
294
+ * `cache_read_input_tokens` field. Additive only — `inputTokens` still
295
+ * includes cache-read volume.
296
+ */
297
+ cacheReadInputTokens?: number;
298
+ /**
299
+ * Optional turn cost in USD reported by providers that expose exact
300
+ * pricing. The Claude Agent SDK populates this from the result
301
+ * message's `total_cost_usd`. Other drivers (Codex, Cursor) leave it
302
+ * undefined until their providers expose comparable metadata.
303
+ */
304
+ costUsd?: number;
305
+ /**
306
+ * Typed error subtype set when the turn ended in error; absent on success.
307
+ * Lets eval consumers triage failures correctly without regex on the
308
+ * result text.
309
+ *
310
+ * The `error_*` values are SDK-reported result subtypes (`SDKResultError`
311
+ * per `sdk.d.ts`). `'bus_rejection'` is driver-synthesized: when the live
312
+ * ask-user bridge captures a bus error (timeout, missing subscriber,
313
+ * subscriber throw), the Claude SDK driver constructs an error
314
+ * `AgentTurnResult` with this subtype rather than throwing, so the
315
+ * partial-turn observability pipeline (`ask_batch`, `model_agent_result`,
316
+ * turn timings/details) captures the rejected turn before the runner
317
+ * propagates the error. The naming distinction (no `error_` prefix)
318
+ * marks the layer of origin: SDK vs driver.
319
+ */
320
+ errorSubtype?: 'error_during_execution' | 'error_max_turns' | 'error_max_budget_usd' | 'error_max_structured_output_retries' | 'bus_rejection';
321
+ /**
322
+ * Populated when the agent subprocess died mid-turn under a stateful
323
+ * transport (Codex `app-server`). Consumers should translate this into
324
+ * `ConversationResult.crashDiagnostic` via `AgentCrashError`.
325
+ */
326
+ crashInfo?: {
327
+ pid?: number;
328
+ signal?: NodeJS.Signals | string | null;
329
+ exitCode?: number | null;
330
+ };
331
+ }
332
+ export interface AgentSession {
333
+ start(input: AgentTurnInput): Promise<AgentTurnResult>;
334
+ reply(input: AgentTurnInput): Promise<AgentTurnResult>;
335
+ /**
336
+ * Optional explicit teardown hook. Drivers that hold OS resources
337
+ * (subprocess, stdio pipes, long-lived promises) implement this so session
338
+ * wrappers can release them in a `finally` block. Must be idempotent and
339
+ * safe to call before any turn was ever run.
340
+ */
341
+ dispose?(): Promise<void>;
342
+ }
343
+ export declare function getWorkspacePath(handle: EnvironmentHandle): string;
344
+ export declare function getRuntimeHandle(handle: EnvironmentHandle): string;
345
+ export declare function getRuntimeEnv(handle: EnvironmentHandle): Record<string, string>;
346
+ export interface AgentSessionOptions {
347
+ mcpConfigPath?: string;
348
+ model?: string;
349
+ conversationWindow?: import('./sdk/types.js').ConversationWindowConfig | false;
350
+ runtimePolicies?: RuntimePolicyDescriptor[];
351
+ /** LLM port for conversation window summarization. */
352
+ llm?: LLMPort;
353
+ /**
354
+ * Codex transport. Claude/Cursor ignore this. When `'app-server'`, the
355
+ * Codex driver swaps to the reliable ask-user channel. Resolved by
356
+ * `createManagedSession`; option/env plumbing that feeds it lands in slice #7.
357
+ */
358
+ transport?: import('./sdk/types.js').AgentTransport;
359
+ /**
360
+ * Per-conversation bus through which adapters emit ask_user question
361
+ * batches and subscribers respond. Constructed by `createManagedSession`.
362
+ * Drivers emitting `lifecycle: 'live'` batches (Codex app-server) MUST
363
+ * enforce presence via `requireAskBusForLiveBatches`; adapters emitting
364
+ * only `lifecycle: 'post-hoc'` no-op when absent.
365
+ */
366
+ askBus?: import('./sdk/ask-bus/types.js').AskBus;
367
+ }
368
+ export declare abstract class BaseAgent {
369
+ createSession(runtime: EnvironmentHandle, runCommand: AgentCommandRunner, options?: AgentSessionOptions): Promise<AgentSession>;
370
+ run(_instruction: string, _workspacePath: string, _runCommand: AgentCommandRunner): Promise<string>;
371
+ }
372
+ export declare function createAgentSession(agent: BaseAgent, runtime: EnvironmentHandle, runCommand: AgentCommandRunner, options?: AgentSessionOptions): Promise<AgentSession>;
package/dist/types.js ADDED
@@ -0,0 +1,37 @@
1
+ export function getWorkspacePath(handle) {
2
+ return typeof handle === 'string' ? handle : handle.workspacePath;
3
+ }
4
+ export function getRuntimeHandle(handle) {
5
+ return typeof handle === 'string' ? handle : handle.handle;
6
+ }
7
+ export function getRuntimeEnv(handle) {
8
+ return typeof handle === 'string' ? {} : handle.env;
9
+ }
10
+ export class BaseAgent {
11
+ async createSession(runtime, runCommand, options) {
12
+ // Default: wrap run() into a session for simple agents
13
+ const runTurn = async (message) => {
14
+ const rawOutput = await this.run(message, getWorkspacePath(runtime), runCommand);
15
+ return {
16
+ rawOutput,
17
+ assistantMessage: rawOutput,
18
+ visibleAssistantMessage: rawOutput,
19
+ visibleAssistantMessageSource: 'assistant_message',
20
+ exitCode: 0,
21
+ traceOutput: rawOutput,
22
+ toolEvents: [],
23
+ runtimePoliciesApplied: [],
24
+ };
25
+ };
26
+ return {
27
+ start: async ({ message }) => runTurn(message),
28
+ reply: async ({ message }) => runTurn(message),
29
+ };
30
+ }
31
+ run(_instruction, _workspacePath, _runCommand) {
32
+ throw new Error('Agent must implement createSession() or run()');
33
+ }
34
+ }
35
+ export async function createAgentSession(agent, runtime, runCommand, options) {
36
+ return agent.createSession(runtime, runCommand, options);
37
+ }
@@ -0,0 +1,55 @@
1
+ /**
2
+ * CLI formatting utilities.
3
+ *
4
+ * Uses ANSI codes that work on both light and dark terminals.
5
+ * Respects NO_COLOR env var (https://no-color.org/).
6
+ */
7
+ export declare const fmt: {
8
+ bold: (s: string) => string;
9
+ dim: (s: string) => string;
10
+ green: (s: string) => string;
11
+ red: (s: string) => string;
12
+ cyan: (s: string) => string;
13
+ pass: (s: string) => string;
14
+ fail: (s: string) => string;
15
+ };
16
+ /** Print a section header with a rule line */
17
+ export declare function header(title: string, width?: number): void;
18
+ /** Print a labeled key-value pair */
19
+ export declare function kv(label: string, value: string, indent?: number): void;
20
+ /** Print a trial result row */
21
+ export declare function trialRow(trialId: number, total: number, reward: number, duration: string, commands: number, scorers: {
22
+ type: string;
23
+ score: number;
24
+ }[]): void;
25
+ /** Print the results summary block */
26
+ export declare function resultsSummary(passRate: number, passAtK: number, passPowK: number, trials: number, preset?: string): void;
27
+ /** Print a validation result */
28
+ export declare function validationResult(passed: boolean, reward: number, scorers: {
29
+ type: string;
30
+ score: number;
31
+ details: string;
32
+ }[]): void;
33
+ /**
34
+ * In-place spinner that shows current phase + elapsed time on one line.
35
+ *
36
+ * Usage:
37
+ * const s = new Spinner('1/3', 'building image');
38
+ * s.update('running agent');
39
+ * s.update('grading');
40
+ * s.stop('PASS 0.85 120.3s 2 cmds');
41
+ */
42
+ export declare class Spinner {
43
+ private frames;
44
+ private frameIdx;
45
+ private interval;
46
+ private prefix;
47
+ private phase;
48
+ private startTime;
49
+ constructor(label: string, initialPhase: string);
50
+ private render;
51
+ /** Update the spinner's phase text */
52
+ update(phase: string): void;
53
+ /** Stop the spinner and print the final line (replaces the spinner) */
54
+ stop(finalLine: string): void;
55
+ }
@@ -0,0 +1,131 @@
1
+ /**
2
+ * CLI formatting utilities.
3
+ *
4
+ * Uses ANSI codes that work on both light and dark terminals.
5
+ * Respects NO_COLOR env var (https://no-color.org/).
6
+ */
7
+ const NO_COLOR = !!process.env.NO_COLOR;
8
+ const code = (n) => NO_COLOR ? '' : `\x1b[${n}m`;
9
+ const reset = code('0');
10
+ const bold = code('1');
11
+ const dim = code('2');
12
+ const green = code('32');
13
+ const red = code('31');
14
+ const cyan = code('36');
15
+ export const fmt = {
16
+ bold: (s) => `${bold}${s}${reset}`,
17
+ dim: (s) => `${dim}${s}${reset}`,
18
+ green: (s) => `${green}${s}${reset}`,
19
+ red: (s) => `${red}${s}${reset}`,
20
+ cyan: (s) => `${cyan}${s}${reset}`,
21
+ pass: (s) => `${bold}${green}${s}${reset}`,
22
+ fail: (s) => `${bold}${red}${s}${reset}`,
23
+ };
24
+ /** Print a section header with a rule line */
25
+ export function header(title, width = 60) {
26
+ const rule = '─'.repeat(Math.max(0, width - title.length - 3));
27
+ console.log(`\n${fmt.bold(`── ${title} `)}${fmt.dim(rule)}`);
28
+ }
29
+ /** Print a labeled key-value pair */
30
+ export function kv(label, value, indent = 2) {
31
+ const pad = ' '.repeat(indent);
32
+ const labelPad = label.padEnd(12);
33
+ console.log(`${pad}${fmt.dim(labelPad)}${value}`);
34
+ }
35
+ /** Print a trial result row */
36
+ export function trialRow(trialId, total, reward, duration, commands, scorers) {
37
+ const pad = ' ';
38
+ const status = reward >= 0.5 ? fmt.pass('PASS') : fmt.fail('FAIL');
39
+ const rewardStr = reward.toFixed(2);
40
+ const trialLabel = `${trialId}/${total}`.padEnd(6);
41
+ const scorerStr = scorers.map(g => {
42
+ const name = g.type === 'deterministic' ? 'deterministic' : 'llm_rubric';
43
+ const scoreStr = g.score.toFixed(2);
44
+ const color = g.score >= 0.5 ? fmt.green(scoreStr) : fmt.red(scoreStr);
45
+ return `${fmt.dim(name)} ${color}`;
46
+ }).join(' ');
47
+ console.log(`${pad} ${fmt.dim(trialLabel)} ${status} ${fmt.bold(rewardStr)} ${fmt.dim(duration.padEnd(7))} ${fmt.dim(commands + ' cmds')} ${scorerStr}`);
48
+ }
49
+ /** Print the results summary block */
50
+ export function resultsSummary(passRate, passAtK, passPowK, trials, preset) {
51
+ const presetLabel = preset === 'smoke' ? ' (smoke test)'
52
+ : preset === 'reliable' ? ' (reliable)'
53
+ : preset === 'regression' ? ' (regression)'
54
+ : '';
55
+ header(`Results${presetLabel}`);
56
+ const fmtPct = (v) => `${(v * 100).toFixed(1)}%`.padStart(7);
57
+ const marker = (key) => preset === key ? fmt.cyan(' ◂') : '';
58
+ console.log(` Pass Rate ${fmt.bold(fmtPct(passRate))}${marker('reliable')}`);
59
+ console.log(` pass@${trials} ${fmtPct(passAtK)}${marker('smoke')}`);
60
+ console.log(` pass^${trials} ${fmtPct(passPowK)}${marker('regression')}`);
61
+ console.log();
62
+ }
63
+ /** Print a validation result */
64
+ export function validationResult(passed, reward, scorers) {
65
+ for (const g of scorers) {
66
+ const scoreStr = g.score.toFixed(2);
67
+ const color = g.score >= 0.5 ? fmt.green(scoreStr) : fmt.red(scoreStr);
68
+ console.log(` ${fmt.dim(g.type.padEnd(16))} ${color} ${fmt.dim(g.details.substring(0, 60))}`);
69
+ }
70
+ console.log();
71
+ if (passed) {
72
+ console.log(` ${fmt.pass('PASSED')} reward ${fmt.bold(reward.toFixed(2))}`);
73
+ }
74
+ else {
75
+ console.log(` ${fmt.fail('FAILED')} reward ${fmt.bold(reward.toFixed(2))}`);
76
+ }
77
+ console.log();
78
+ }
79
+ /**
80
+ * In-place spinner that shows current phase + elapsed time on one line.
81
+ *
82
+ * Usage:
83
+ * const s = new Spinner('1/3', 'building image');
84
+ * s.update('running agent');
85
+ * s.update('grading');
86
+ * s.stop('PASS 0.85 120.3s 2 cmds');
87
+ */
88
+ export class Spinner {
89
+ frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
90
+ frameIdx = 0;
91
+ interval = null;
92
+ prefix;
93
+ phase;
94
+ startTime;
95
+ constructor(label, initialPhase) {
96
+ this.prefix = ` ${fmt.dim(label.padEnd(6))}`;
97
+ this.phase = initialPhase;
98
+ this.startTime = Date.now();
99
+ if (process.stdout.isTTY && !NO_COLOR) {
100
+ this.interval = setInterval(() => this.render(), 80);
101
+ this.render();
102
+ }
103
+ else {
104
+ // Non-TTY: just print once
105
+ process.stdout.write(`${this.prefix} ${fmt.dim(this.phase)}\n`);
106
+ }
107
+ }
108
+ render() {
109
+ const frame = this.frames[this.frameIdx % this.frames.length];
110
+ this.frameIdx++;
111
+ const elapsed = ((Date.now() - this.startTime) / 1000).toFixed(0);
112
+ const line = `${this.prefix} ${fmt.cyan(frame)} ${fmt.dim(this.phase)} ${fmt.dim(elapsed + 's')}`;
113
+ process.stdout.write(`\r\x1b[K${line}`);
114
+ }
115
+ /** Update the spinner's phase text */
116
+ update(phase) {
117
+ this.phase = phase;
118
+ if (!this.interval) {
119
+ // Non-TTY: print each phase
120
+ process.stdout.write(`${this.prefix} ${fmt.dim(this.phase)}\n`);
121
+ }
122
+ }
123
+ /** Stop the spinner and print the final line (replaces the spinner) */
124
+ stop(finalLine) {
125
+ if (this.interval) {
126
+ clearInterval(this.interval);
127
+ this.interval = null;
128
+ }
129
+ process.stdout.write(`\r\x1b[K${this.prefix} ${finalLine}\n`);
130
+ }
131
+ }
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Shared .env file parser.
3
+ */
4
+ /**
5
+ * Parse a .env file content into key-value pairs.
6
+ * Supports: KEY=VALUE, KEY="VALUE", KEY='VALUE', comments (#), blank lines.
7
+ */
8
+ export declare function parseEnvFile(content: string): Record<string, string>;
@@ -0,0 +1,26 @@
1
+ /**
2
+ * Shared .env file parser.
3
+ */
4
+ /**
5
+ * Parse a .env file content into key-value pairs.
6
+ * Supports: KEY=VALUE, KEY="VALUE", KEY='VALUE', comments (#), blank lines.
7
+ */
8
+ export function parseEnvFile(content) {
9
+ const env = {};
10
+ for (const line of content.split('\n')) {
11
+ const trimmed = line.trim();
12
+ if (!trimmed || trimmed.startsWith('#'))
13
+ continue;
14
+ const eqIdx = trimmed.indexOf('=');
15
+ if (eqIdx === -1)
16
+ continue;
17
+ const key = trimmed.substring(0, eqIdx).trim();
18
+ let value = trimmed.substring(eqIdx + 1).trim();
19
+ if ((value.startsWith('"') && value.endsWith('"')) ||
20
+ (value.startsWith("'") && value.endsWith("'"))) {
21
+ value = value.slice(1, -1);
22
+ }
23
+ env[key] = value;
24
+ }
25
+ return env;
26
+ }
@@ -0,0 +1,43 @@
1
+ import type { LLMPort, LLMCallOptions, ToolUseMessage, ToolUseBlock, CallWithToolsOptions } from './llm-types.js';
2
+ export type MockResponse = string | {
3
+ text: string;
4
+ inputTokens?: number;
5
+ outputTokens?: number;
6
+ provider?: 'anthropic' | 'openai' | 'cli';
7
+ model?: string;
8
+ } | {
9
+ kind: 'tool_use';
10
+ blocks: ToolUseBlock[];
11
+ text?: string;
12
+ inputTokens?: number;
13
+ outputTokens?: number;
14
+ } | {
15
+ kind: 'final';
16
+ text: string;
17
+ inputTokens?: number;
18
+ outputTokens?: number;
19
+ } | {
20
+ throws: Error;
21
+ };
22
+ export interface CreateMockLLMOptions {
23
+ responses?: MockResponse[];
24
+ /** Returned whenever the queue is empty. Overrides the default "out of responses" error. */
25
+ defaultResponse?: MockResponse;
26
+ /** When set, replaces the queue for `.call()`. Useful for dynamic / side-effecting tests. */
27
+ respond?: (prompt: string, opts?: LLMCallOptions) => MockResponse | Promise<MockResponse>;
28
+ }
29
+ export interface MockCallRecord {
30
+ prompt: string;
31
+ opts?: LLMCallOptions;
32
+ }
33
+ export interface MockToolCallRecord {
34
+ messages: ToolUseMessage[];
35
+ opts: CallWithToolsOptions;
36
+ }
37
+ export interface MockLLM extends LLMPort {
38
+ queueResponse(r: MockResponse): void;
39
+ clearCalls(): void;
40
+ readonly calls: readonly MockCallRecord[];
41
+ readonly toolCalls: readonly MockToolCallRecord[];
42
+ }
43
+ export declare function createMockLLM(opts?: CreateMockLLMOptions): MockLLM;