@wix/pathgrade 0.37.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/README.md +116 -32
  2. package/dist/adapter-kit/index.d.ts +17 -0
  3. package/dist/adapter-kit/index.js +14 -0
  4. package/dist/adapters/jest/index.d.ts +3 -0
  5. package/dist/adapters/jest/index.js +3 -0
  6. package/dist/adapters/jest/invocation-adapter.d.ts +15 -0
  7. package/dist/adapters/jest/invocation-adapter.js +115 -0
  8. package/dist/adapters/jest/lifecycle.d.ts +21 -0
  9. package/dist/adapters/jest/lifecycle.js +62 -0
  10. package/dist/adapters/jest/metadata.d.ts +6 -0
  11. package/dist/adapters/jest/metadata.js +35 -0
  12. package/dist/adapters/jest/reporter.cjs +10 -0
  13. package/dist/adapters/jest/reporter.d.ts +5 -0
  14. package/dist/adapters/jest/reporter.js +67 -0
  15. package/dist/adapters/jest/results.d.ts +31 -0
  16. package/dist/adapters/jest/results.js +144 -0
  17. package/dist/adapters/jest/runner-adapter.d.ts +9 -0
  18. package/dist/adapters/jest/runner-adapter.js +49 -0
  19. package/dist/adapters/jest/setup.d.ts +1 -0
  20. package/dist/adapters/jest/setup.js +10 -0
  21. package/dist/adapters/node-test/index.d.ts +8 -0
  22. package/dist/adapters/node-test/index.js +106 -0
  23. package/dist/adapters/node-test/invocation-adapter.d.ts +5 -0
  24. package/dist/adapters/node-test/invocation-adapter.js +47 -0
  25. package/dist/adapters/node-test/runner-adapter.d.ts +2 -0
  26. package/dist/adapters/node-test/runner-adapter.js +116 -0
  27. package/dist/adapters/vitest/index.d.ts +9 -0
  28. package/dist/adapters/vitest/index.js +82 -0
  29. package/dist/adapters/vitest/lifecycle.d.ts +24 -0
  30. package/dist/adapters/vitest/lifecycle.js +41 -0
  31. package/dist/adapters/vitest/reporter.d.ts +12 -0
  32. package/dist/adapters/vitest/reporter.js +75 -0
  33. package/dist/adapters/vitest/setup.d.ts +1 -0
  34. package/dist/adapters/vitest/setup.js +3 -0
  35. package/dist/agents/claude/sdk-options.js +6 -0
  36. package/dist/agents/codex-app-server/agent.js +5 -1
  37. package/dist/agents/codex-app-server/protocol/index.js +3 -3
  38. package/dist/agents/codex-app-server/turn-completion.d.ts +1 -0
  39. package/dist/agents/codex-app-server/turn-completion.js +9 -0
  40. package/dist/commands/affected.js +22 -15
  41. package/dist/commands/run-args.d.ts +4 -3
  42. package/dist/commands/run-args.js +12 -6
  43. package/dist/commands/run-changed.d.ts +4 -4
  44. package/dist/commands/run-changed.js +45 -58
  45. package/dist/config/pathgrade.d.ts +48 -0
  46. package/dist/config/pathgrade.js +204 -0
  47. package/dist/pathgrade.d.ts +2 -2
  48. package/dist/pathgrade.js +35 -34
  49. package/dist/plugin/index.d.ts +2 -9
  50. package/dist/plugin/index.js +1 -81
  51. package/dist/plugin/lifecycle.d.ts +1 -45
  52. package/dist/plugin/lifecycle.js +1 -146
  53. package/dist/plugin/reporter.d.ts +1 -37
  54. package/dist/plugin/reporter.js +1 -280
  55. package/dist/plugin/setup.d.ts +1 -1
  56. package/dist/plugin/setup.js +1 -3
  57. package/dist/providers/copy-filter.d.ts +5 -0
  58. package/dist/providers/copy-filter.js +9 -0
  59. package/dist/providers/credentials.js +1 -1
  60. package/dist/providers/sandbox-lifecycle.d.ts +22 -0
  61. package/dist/providers/sandbox-lifecycle.js +133 -0
  62. package/dist/providers/sandbox.js +4 -3
  63. package/dist/providers/workspace.js +4 -14
  64. package/dist/reporters/diagnostics.d.ts +2 -40
  65. package/dist/reporters/diagnostics.js +1 -80
  66. package/dist/reporters/report-summary.d.ts +6 -0
  67. package/dist/reporters/report-summary.js +29 -0
  68. package/dist/reporting/artifacts.d.ts +2 -0
  69. package/dist/reporting/artifacts.js +20 -0
  70. package/dist/reporting/core.d.ts +2 -0
  71. package/dist/reporting/core.js +173 -0
  72. package/dist/reporting/types.d.ts +57 -0
  73. package/dist/reporting/types.js +1 -0
  74. package/dist/reporting/vitest-edge.d.ts +1 -0
  75. package/dist/reporting/vitest-edge.js +1 -0
  76. package/dist/runners/adapter-loader.d.ts +17 -0
  77. package/dist/runners/adapter-loader.js +67 -0
  78. package/dist/runners/adapter.d.ts +65 -0
  79. package/dist/runners/adapter.js +1 -0
  80. package/dist/runners/invocation.d.ts +10 -0
  81. package/dist/runners/invocation.js +1 -0
  82. package/dist/runners/lifecycle-hooks.d.ts +2 -0
  83. package/dist/runners/lifecycle-hooks.js +18 -0
  84. package/dist/runners/model-builders.d.ts +4 -0
  85. package/dist/runners/model-builders.js +67 -0
  86. package/dist/runners/model-validation.d.ts +12 -0
  87. package/dist/runners/model-validation.js +213 -0
  88. package/dist/runners/model.d.ts +117 -0
  89. package/dist/runners/model.js +1 -0
  90. package/dist/runners/orchestrator.d.ts +30 -0
  91. package/dist/runners/orchestrator.js +74 -0
  92. package/dist/runners/report-projection.d.ts +6 -0
  93. package/dist/runners/report-projection.js +62 -0
  94. package/dist/runners/selection.d.ts +6 -0
  95. package/dist/runners/selection.js +10 -0
  96. package/dist/runners/vitest-adapter.d.ts +8 -0
  97. package/dist/runners/vitest-adapter.js +108 -0
  98. package/dist/runners/vitest-invocation.d.ts +10 -0
  99. package/dist/runners/vitest-invocation.js +44 -0
  100. package/dist/runners/vitest-lifecycle.d.ts +43 -0
  101. package/dist/runners/vitest-lifecycle.js +150 -0
  102. package/dist/sdk/agent.js +15 -16
  103. package/dist/sdk/case-context.d.ts +20 -0
  104. package/dist/sdk/case-context.js +31 -0
  105. package/dist/sdk/diagnostics.d.ts +40 -0
  106. package/dist/sdk/diagnostics.js +80 -0
  107. package/dist/sdk/eval-runtime.d.ts +4 -0
  108. package/dist/sdk/eval-runtime.js +11 -1
  109. package/dist/sdk/evaluate.js +2 -1
  110. package/dist/sdk/index.d.ts +2 -0
  111. package/dist/sdk/index.js +1 -0
  112. package/dist/sdk/lifecycle.d.ts +40 -0
  113. package/dist/sdk/lifecycle.js +170 -0
  114. package/dist/sdk/result-capture.d.ts +18 -0
  115. package/dist/sdk/result-capture.js +49 -0
  116. package/dist/sdk/types.d.ts +1 -1
  117. package/dist/types.d.ts +2 -2
  118. package/dist/utils/llm-providers/anthropic.js +9 -2
  119. package/package.json +56 -4
package/README.md CHANGED
@@ -1,10 +1,10 @@
1
1
  # Pathgrade
2
2
 
3
- **Evaluate AI coding agents with Vitest.** Write evals as normal `.eval.ts` files, run Claude Code, Codex, or Cursor in isolated sandboxes, and score the result with deterministic checks, rubric judges, and tool-usage assertions.
3
+ **Evaluate AI coding agents with Vitest or Jest.** Write evals as normal `.eval.ts` files, run Claude Code, Codex, or Cursor in isolated sandboxes, and score the result with deterministic checks, rubric judges, and tool-usage assertions.
4
4
 
5
5
  ## Why Pathgrade?
6
6
 
7
- - Write evals in plain TypeScript with Vitest
7
+ - Write evals in plain TypeScript with Vitest or Jest
8
8
  - Run each trial in an isolated workspace and HOME directory
9
9
  - Seed trials from fixtures, real skills, or mocked MCP servers
10
10
  - Score both final artifacts and the workflow that produced them
@@ -13,12 +13,18 @@
13
13
 
14
14
  ## Quick Start
15
15
 
16
- **Prerequisites**: Node.js 20.11+, Vitest 4+, and at least one of [Claude Code](https://docs.anthropic.com/en/docs/claude-code), [Codex CLI](https://github.com/openai/codex), or the `cursor-agent` CLI
16
+ **Prerequisites**: Node.js 20.11+, Vitest 4+ or Jest 30+, and at least one configured agent runtime. Claude uses the bundled `@anthropic-ai/claude-agent-sdk` binary by default; Codex requires the `codex` CLI; Cursor requires the `cursor-agent` CLI.
17
17
 
18
18
  ```bash
19
19
  yarn add -D @wix/pathgrade
20
20
  ```
21
21
 
22
+ For Jest projects, install Jest too:
23
+
24
+ ```bash
25
+ yarn add -D @wix/pathgrade jest
26
+ ```
27
+
22
28
  ### Authentication
23
29
 
24
30
  By default, Pathgrade tries to reuse the agent CLI's native auth before falling back to explicit environment variables.
@@ -27,9 +33,9 @@ By default, Pathgrade tries to reuse the agent CLI's native auth before falling
27
33
  - macOS: reuses Claude Code OAuth from Keychain
28
34
  - other platforms: forwards `ANTHROPIC_API_KEY` when present
29
35
  - **Codex**
30
- - reuses cached `~/.codex/auth.json` when available
31
- - or forwards `OPENAI_API_KEY`
32
- - or runs `codex login --with-api-key` inside the sandbox when an API key is present but no cached login exists
36
+ - forwards `OPENAI_API_KEY` when present
37
+ - or runs `codex login --with-api-key` inside the sandbox when an API key is present
38
+ - `codex exec` can reuse cached `~/.codex/auth.json` when no key is available; `app-server` may reject cached ChatGPT-token refreshes, so prefer `OPENAI_API_KEY` for the default transport
33
39
  - **Cursor**
34
40
  - forwards `CURSOR_API_KEY` when set
35
41
  - macOS: reuses `cursor-agent login` OAuth tokens from the login Keychain
@@ -52,7 +58,7 @@ const agent = await createAgent({
52
58
 
53
59
  Codex supports two transports and Pathgrade defaults to `app-server`:
54
60
 
55
- - `app-server` (default) — uses `codex app-server` and keeps native thread state. Required for `AskUserReaction` handshakes (`request_user_input` reaches the model). Requires `OPENAI_API_KEY`; cached `~/.codex/auth.json` is not supported under this transport.
61
+ - `app-server` (default) — uses `codex app-server` and keeps native thread state. Required for `AskUserReaction` handshakes (`request_user_input` reaches the model). Prefer `OPENAI_API_KEY`; cached ChatGPT auth can fail if the app-server asks Pathgrade to refresh tokens.
56
62
  - `exec` — uses `codex exec` and re-injects the transcript every turn. Kept for stateless CI matrices that don't need the handshake.
57
63
 
58
64
  Precedence: `createAgent({ transport })` > `PATHGRADE_CODEX_TRANSPORT` env > default (`app-server`). An invalid env value throws at `createAgent` time.
@@ -68,17 +74,17 @@ If `transport: 'exec'` is resolved and any `AskUserReaction` is present in `Conv
68
74
 
69
75
  Migrating from `exec` to `app-server`:
70
76
 
71
- - Export `OPENAI_API_KEY`, or set `transport: 'exec'` / `PATHGRADE_CODEX_TRANSPORT=exec` to stay on the old transport.
77
+ - Export `OPENAI_API_KEY`, or set `transport: 'exec'` / `PATHGRADE_CODEX_TRANSPORT=exec` to stay on the old transport and its cached-auth behavior.
72
78
  - The `noninteractive-user-question` runtime policy no longer attaches under `app-server`. Snapshots that captured model output influenced by that policy text may need re-recording.
73
79
  - `MAX_TURN_RETRIES` does not apply under `app-server` — a crashed turn ends the conversation with `completionReason: 'agent_crashed'`.
74
80
 
75
81
  ### Plugin Setup
76
82
 
77
- Create a `vitest.config.ts` with the Pathgrade plugin:
83
+ Create a `vitest.config.ts` with the built-in Vitest adapter plugin:
78
84
 
79
85
  ```typescript
80
86
  import { defineConfig } from 'vitest/config';
81
- import { pathgrade } from '@wix/pathgrade/plugin';
87
+ import { pathgrade } from '@wix/pathgrade/adapters/vitest';
82
88
 
83
89
  export default defineConfig({
84
90
  plugins: [pathgrade({ timeout: 120 })],
@@ -87,6 +93,24 @@ export default defineConfig({
87
93
 
88
94
  The plugin registers the setup hooks Pathgrade needs, wires in the reporter, and automatically cleans up agent workspaces after each test.
89
95
 
96
+ For Pathgrade CLI behavior, use `pathgrade.config.ts`:
97
+
98
+ ```typescript
99
+ export default {
100
+ runner: {
101
+ adapter: 'vitest', // default; use 'jest' for Jest projects
102
+ args: [],
103
+ },
104
+ evals: {
105
+ include: ['**/*.eval.ts'],
106
+ exclude: ['**/fixtures/**'],
107
+ },
108
+ affected: {
109
+ global: ['package.json', 'yarn.lock'],
110
+ },
111
+ };
112
+ ```
113
+
90
114
  ### First Eval
91
115
 
92
116
  Write an eval file such as `hello.eval.ts`:
@@ -122,7 +146,7 @@ Run your evals:
122
146
  npx pathgrade run
123
147
  ```
124
148
 
125
- `pathgrade run` is the recommended wrapper: it loads `.env`, warns when no auth is configured, and adds Pathgrade-specific flags such as `--changed`, `--diagnostics`, and `--verbose`. Plain `npx vitest run` works too if you do not need those extras.
149
+ `pathgrade run` is the recommended wrapper: it loads `.env`, warns when no auth is configured, and adds Pathgrade-specific flags such as `--changed`, `--diagnostics`, and `--verbose`. Plain `npx vitest run` or direct Jest runs work too if you configure the Pathgrade adapter hooks yourself.
126
150
 
127
151
  ## Core Concepts
128
152
 
@@ -193,7 +217,7 @@ judge('spec-structure', {
193
217
  });
194
218
  ```
195
219
 
196
- Tool-using judges currently require the Anthropic HTTP provider (`ANTHROPIC_API_KEY`); other providers produce a clean `provider_not_supported` error. See the [User Guide](packages/pathgrade/docs/USER_GUIDE.md#tool-using-judges--judge-tools-) for the full tool list, failure codes, and the migration recipe from `input`-helper probes.
220
+ Tool-using judges currently require the Anthropic HTTP provider (`ANTHROPIC_API_KEY`); other providers produce a clean `provider_not_supported` error. See the [User Guide](docs/USER_GUIDE.md#tool-using-judges) for the full tool list, failure codes, and the migration recipe from `input`-helper probes.
197
221
 
198
222
  ### `toolUsage()` - Tool event matching
199
223
 
@@ -207,7 +231,7 @@ toolUsage('expected-workflow', [
207
231
 
208
232
  ## Conversations
209
233
 
210
- Pathgrade currently supports three agent backends: `claude`, `codex`, and `cursor`. Set the backend per test via `createAgent({ agent: 'claude' })` or globally via `PATHGRADE_AGENT`.
234
+ Pathgrade currently supports three agent backends: `claude`, `codex`, and `cursor`. Set the backend per test via `createAgent({ agent: 'claude' })`, or omit `agent` and use `PATHGRADE_AGENT` as the process-wide fallback.
211
235
 
212
236
  ### `agent.prompt()` - One shot
213
237
 
@@ -274,11 +298,11 @@ Pathgrade exposes a few useful features that are easy to miss from the basic exa
274
298
  - `conversationWindow` on agents and personas keeps long transcripts bounded with summarization instead of sending the full conversation every turn.
275
299
  - `copyIgnore` and `DEFAULT_COPY_IGNORE` let you control what gets copied into the sandbox when seeding from large fixtures or skill directories.
276
300
 
277
- See [sdk-showcase](packages/pathgrade/examples/sdk-showcase/) for a single example suite that demonstrates these APIs together.
301
+ See [sdk-showcase](examples/sdk-showcase/) for a single example suite that demonstrates these APIs together.
278
302
 
279
303
  ## MCP Mock Servers
280
304
 
281
- Simulate MCP tools when testing Claude-driven evals:
305
+ Simulate MCP tools when testing Claude, Codex app-server, or Cursor evals:
282
306
 
283
307
  ```typescript
284
308
  import { mockMcpServer } from '@wix/pathgrade/mcp-mock';
@@ -299,7 +323,7 @@ const agent = await createAgent({ agent: 'claude', mcpMock: mock });
299
323
  ## CLI
300
324
 
301
325
  ```bash
302
- pathgrade run [--changed] [--diagnostics] [--verbose] [-- vitest-args]
326
+ pathgrade run [--changed] [--since=<ref>] [--changed-files=<path>] [--adapter=<name|path>] [--diagnostics] [--verbose] [--quiet] [-- runner-args]
303
327
  pathgrade init [--force]
304
328
  pathgrade validate <file.eval.ts>
305
329
  pathgrade validate --affected
@@ -312,36 +336,74 @@ pathgrade report [--results-path=<path>] [--no-comment] [--comment-id=<id>]
312
336
 
313
337
  Useful details:
314
338
 
315
- - `pathgrade run --changed` computes affected evals first, writes selection metadata to `.pathgrade/selection.json`, and only then launches Vitest.
339
+ - `pathgrade run --changed` computes affected evals first, writes selection metadata to `.pathgrade/selection.json`, and only then launches the selected runner adapter. Vitest is the default adapter; Jest is selected with `runner.adapter: 'jest'` or `--adapter=jest`.
316
340
  - `pathgrade preview browser` starts a local viewer on `http://localhost:3847`.
317
341
  - `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate.
318
342
  - `pathgrade validate --affected` is a strict mode for CI: every discovered eval must either live under a `SKILL.md` anchor or export valid `__pathgradeMeta`.
319
343
 
320
344
  Run `pathgrade --help` for the full help text.
321
345
 
322
- ## Plugin Options
346
+ ## Configuration
347
+
348
+ ```typescript
349
+ // pathgrade.config.ts
350
+ export default {
351
+ runner: {
352
+ adapter: 'vitest',
353
+ args: [],
354
+ },
355
+ evals: {
356
+ include: ['**/*.eval.ts'], // default
357
+ exclude: ['**/fixtures/**'], // replaces the default exclude list if set
358
+ },
359
+ affected: {
360
+ global: ['package.json', 'yarn.lock'],
361
+ },
362
+ ci: { threshold: 0.8 },
363
+ };
364
+ ```
365
+
366
+ Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. Built-in adapters currently include `vitest`, `jest`, and the narrow `node-test` proof adapter.
367
+
368
+ Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
369
+
370
+ - `vitest`, `jest`, `node-test`: built-in adapters
371
+ - `demo`: package `@wix/pathgrade-adapter-demo`, resolved from the project
372
+ - `@scope/pathgrade-adapter-demo` or another specifier containing `/`: package specifier, resolved from the project
373
+ - `./local-adapter.mjs` or `/abs/local-adapter.mjs`: local adapter module
374
+
375
+ External adapter modules used by `pathgrade run` must export `createPathgradeInvocationAdapter({ config })`, returning a `RunnerInvocationAdapter`. Modules used by lower-level orchestration can also export `createPathgradeAdapter()`, returning a `RunnerAdapter` with `discover`, `invoke`, and `collectNormalizedRunSnapshot`.
376
+
377
+ Vitest runner behavior still belongs in `vitest.config.ts`:
323
378
 
324
379
  ```typescript
325
- import { pathgrade } from '@wix/pathgrade/plugin';
380
+ import { pathgrade } from '@wix/pathgrade/adapters/vitest';
326
381
 
327
382
  pathgrade({
328
- include: ['**/*.eval.ts'], // default: ['**/*.eval.ts']
329
383
  timeout: 300, // seconds, default: 300
330
384
  reporter: 'cli', // 'cli' | 'browser' | 'json'
331
385
  diagnostics: false, // print full diagnostics for passing evals too
332
386
  verbose: false, // stream live per-turn events to stderr while evals run
333
- ci: { threshold: 0.8 }, // fail when the mean test score drops below threshold
334
- affected: {
335
- global: ['package.json', 'yarn.lock'],
336
- },
337
387
  });
338
388
  ```
339
389
 
390
+ Jest runner behavior still belongs in `jest.config.*`. Pathgrade injects only the setup and reporter it needs when you use `pathgrade run --adapter=jest`; transforms, test environment, module resolution, and ESM/TypeScript support remain your Jest config's job.
391
+
392
+ For direct Jest runs, configure the same entry points explicitly:
393
+
394
+ ```js
395
+ // jest.config.mjs
396
+ export default {
397
+ setupFilesAfterEnv: ['@wix/pathgrade/adapters/jest/setup'],
398
+ reporters: ['default', '@wix/pathgrade/adapters/jest/reporter'],
399
+ };
400
+ ```
401
+
340
402
  Notes:
341
403
 
342
- - `exclude` is also supported. If you set it, it replaces the default exclude list instead of merging with it.
404
+ - Legacy `@wix/pathgrade/plugin` and `@wix/pathgrade/plugin/vitest` imports remain as compatibility fallbacks, but new config should use `@wix/pathgrade/adapters/vitest` and `pathgrade.config.*`.
343
405
  - `reporter: 'browser'` writes results JSON and opens the viewer automatically after the run.
344
- - `affected.global` is a repo-level "rerun everything" escape hatch for `pathgrade affected` and `pathgrade run --changed`.
406
+ - SDK-only consumers can import `@wix/pathgrade` without installing Vitest or Jest. Adapter users need their selected runner available.
345
407
 
346
408
  ## Environment Variables
347
409
 
@@ -359,6 +421,28 @@ Notes:
359
421
  | `PATHGRADE_DIAGNOSTICS` | `1` prints full diagnostics for passing evals too |
360
422
  | `NO_COLOR` | Disable ANSI colors |
361
423
 
424
+ ### Experimental `node:test` Adapter
425
+
426
+ `node-test` is a small proof adapter for validating the runner boundary without Vitest. It is intentionally narrow: eval files import the `test` wrapper from `@wix/pathgrade/adapters/node-test`, run through Node's built-in test runner, and still produce the normal `.pathgrade/results.json` and trace artifacts.
427
+
428
+ ```ts
429
+ import { test } from '@wix/pathgrade/adapters/node-test';
430
+ import { createAgent, evaluate, check } from '@wix/pathgrade';
431
+
432
+ test('minimal node proof', async () => {
433
+ const agent = await createAgent({ workspace: process.cwd() });
434
+ await evaluate(agent, [check('passes', () => true)]);
435
+ });
436
+ ```
437
+
438
+ Run it with:
439
+
440
+ ```bash
441
+ pathgrade run --adapter=node-test
442
+ ```
443
+
444
+ This proof validates Pathgrade's adapter, lifecycle, result capture, and reporting boundaries. It does not imply Mocha, Playwright, or runnerless CLI support.
445
+
362
446
  `pathgrade run` loads `.env` from the working directory automatically.
363
447
 
364
448
  ## CI / GitHub Actions
@@ -401,17 +485,17 @@ jobs:
401
485
 
402
486
  - `fetch-depth: 0` is required for `--changed`; shallow clones break merge-base resolution.
403
487
  - Evals under a `SKILL.md` are tracked automatically; use `__pathgradeMeta` for cross-skill or non-standard dependencies.
404
- - Set `ci: { threshold: 0.8 }` in the plugin config to fail the run when the mean test score drops below your threshold.
488
+ - Set `ci: { threshold: 0.8 }` in `pathgrade.config.ts` to fail the run when the mean test score drops below your threshold.
405
489
 
406
- See the [User Guide - CI Integration](packages/pathgrade/docs/USER_GUIDE.md#ci-integration) for the full reference.
490
+ See the [User Guide - CI Integration](docs/USER_GUIDE.md#ci-integration) for the full reference.
407
491
 
408
492
  ## Links
409
493
 
410
- - [User Guide](packages/pathgrade/docs/USER_GUIDE.md) - full API reference and usage patterns
494
+ - [User Guide](docs/USER_GUIDE.md) - full API reference and usage patterns
411
495
  - Examples:
412
- - [start-chat](packages/pathgrade/examples/start-chat/) - multi-turn conversation
413
- - [sdk-showcase](packages/pathgrade/examples/sdk-showcase/) - advanced SDK features in one suite
414
- - [tool-judge-demo](packages/pathgrade/examples/tool-judge-demo/) - `judge({ tools })` reading workspace artifacts
496
+ - [start-chat](examples/start-chat/) - multi-turn conversation
497
+ - [sdk-showcase](examples/sdk-showcase/) - advanced SDK features in one suite
498
+ - [tool-judge-demo](examples/tool-judge-demo/) - `judge({ tools })` reading workspace artifacts
415
499
 
416
500
  ## Note on AI provider dependencies
417
501
 
@@ -0,0 +1,17 @@
1
+ export { runnerAdapterContractVersion, type AdapterCaseContext, type AdapterDiagnostic, type AdapterDiscoveryInput, type AdapterDiscoveryResult, type AdapterEvalUnit, type AdapterInvocationInput, type AdapterLifecycleHooks, type AdapterRunHandle, type AdapterRunStatus, type EvalResultEvent, type RunnerAdapter, } from '../runners/adapter.js';
2
+ export type { RunnerInvocationAdapter, RunnerInvocationInput, } from '../runners/invocation.js';
3
+ export type { AttemptOutcome, AttemptRecord, AssertionRecord, Diagnostic, EvalUnitRecord, EvaluationRecord, GroupingHint, NativeReference, NormalizedRunModel, NormalizedRunSnapshot, RunCaseRecord, RunCaseState, RunRecord, RunStatus, ScoringPolicy, SnapshotCompleteness, } from '../runners/model.js';
4
+ export { buildNormalizedRunSnapshotFromReportGroups } from '../runners/model-builders.js';
5
+ export { validateNormalizedRunSnapshot } from '../runners/model-validation.js';
6
+ export { projectNormalizedRunSnapshotToReportInput } from '../runners/report-projection.js';
7
+ export { runWithAdapter, type PathgradeRunOptions, type AdapterReporterMode } from '../runners/orchestrator.js';
8
+ export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
9
+ export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
10
+ export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, type PathgradeConfig, type ResolvedPathgradeConfig, } from '../config/pathgrade.js';
11
+ export { readSidecar } from '../affected/sidecar.js';
12
+ export { getPathgradeDir } from '../reporters/results-path.js';
13
+ export { printReportSummary } from '../reporters/report-summary.js';
14
+ export { fmt } from '../utils/cli.js';
15
+ export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, type CaseContext, type CaseContextProvider, type CaseContextProviderHandle, type CaseContextScope, type CurrentCaseContext, } from '../sdk/case-context.js';
16
+ export { subscribeToEvalResults, type EvalResultObserver, type ResultObserverHandle, type ResultObserverOptions, type ResultObserverOwner, } from '../sdk/result-capture.js';
17
+ export type { Agent, PathgradeTestMeta, RecordedEvalResult, } from '../sdk/types.js';
@@ -0,0 +1,14 @@
1
+ export { runnerAdapterContractVersion, } from '../runners/adapter.js';
2
+ export { buildNormalizedRunSnapshotFromReportGroups } from '../runners/model-builders.js';
3
+ export { validateNormalizedRunSnapshot } from '../runners/model-validation.js';
4
+ export { projectNormalizedRunSnapshotToReportInput } from '../runners/report-projection.js';
5
+ export { runWithAdapter } from '../runners/orchestrator.js';
6
+ export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
7
+ export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
8
+ export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, } from '../config/pathgrade.js';
9
+ export { readSidecar } from '../affected/sidecar.js';
10
+ export { getPathgradeDir } from '../reporters/results-path.js';
11
+ export { printReportSummary } from '../reporters/report-summary.js';
12
+ export { fmt } from '../utils/cli.js';
13
+ export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, } from '../sdk/case-context.js';
14
+ export { subscribeToEvalResults, } from '../sdk/result-capture.js';
@@ -0,0 +1,3 @@
1
+ export { createJestAdapter, createPathgradeAdapter } from './runner-adapter.js';
2
+ export { createJestInvocationAdapter, createPathgradeInvocationAdapter } from './invocation-adapter.js';
3
+ export { default as PathgradeJestReporter } from './reporter.js';
@@ -0,0 +1,3 @@
1
+ export { createJestAdapter, createPathgradeAdapter } from './runner-adapter.js';
2
+ export { createJestInvocationAdapter, createPathgradeInvocationAdapter } from './invocation-adapter.js';
3
+ export { default as PathgradeJestReporter } from './reporter.js';
@@ -0,0 +1,15 @@
1
+ import { type ResolvedPathgradeConfig, type RunnerInvocationAdapter } from '@wix/pathgrade/adapter-kit';
2
+ export interface SpawnJestRequest {
3
+ argv: string[];
4
+ env: NodeJS.ProcessEnv;
5
+ cwd: string;
6
+ }
7
+ export type SpawnJest = (req: SpawnJestRequest) => Promise<number> | number;
8
+ export declare function createJestInvocationAdapter(input: {
9
+ config: ResolvedPathgradeConfig;
10
+ spawnJest?: SpawnJest;
11
+ }): RunnerInvocationAdapter;
12
+ export declare function createPathgradeInvocationAdapter(input: {
13
+ config: ResolvedPathgradeConfig;
14
+ }): RunnerInvocationAdapter;
15
+ export declare function resolveLocalJestBinPath(cwd: string): string;
@@ -0,0 +1,115 @@
1
+ import * as fs from 'node:fs';
2
+ import * as path from 'node:path';
3
+ import { spawn } from 'node:child_process';
4
+ import { createRequire } from 'node:module';
5
+ import { getPathgradeDir, printReportSummary, readSidecar, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
6
+ import { removeJestMetadata, withJestMetadataEnv } from './metadata.js';
7
+ import { createJestAdapter } from './runner-adapter.js';
8
+ export function createJestInvocationAdapter(input) {
9
+ const spawnJest = input.spawnJest ?? defaultSpawnJest;
10
+ return {
11
+ name: 'jest',
12
+ async run(runInput) {
13
+ const selection = await readSidecar(runInput.cwd, msg => {
14
+ process.stderr.write(`[pathgrade] ${msg}\n`);
15
+ }) ?? undefined;
16
+ const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
17
+ ? runInput.selectedFiles
18
+ : input.config.evals.include;
19
+ const exclude = runInput.selectedFiles ? [] : input.config.evals.exclude;
20
+ const discovered = await createJestAdapter().discover({
21
+ cwd: runInput.cwd,
22
+ include,
23
+ exclude,
24
+ selection,
25
+ });
26
+ if (discovered.units.length === 0) {
27
+ return await runWithAdapter({
28
+ adapter: createJestAdapter({ results: { testResults: [] } }),
29
+ options: {
30
+ cwd: runInput.cwd,
31
+ discovery: {
32
+ cwd: runInput.cwd,
33
+ include,
34
+ exclude,
35
+ selection,
36
+ },
37
+ runnerArgs: [],
38
+ env: runInput.env,
39
+ artifactRoot: getPathgradeDir(runInput.cwd),
40
+ reporterMode: input.config.reporter ?? 'cli',
41
+ threshold: input.config.ci.threshold,
42
+ selection,
43
+ printSummary: summaries => {
44
+ printReportSummary(summaries, {
45
+ forceVerbose: input.config.diagnostics
46
+ || runInput.env.PATHGRADE_DIAGNOSTICS === '1',
47
+ });
48
+ },
49
+ log: () => {
50
+ process.stdout.write(`\n Results written to ${getPathgradeDir(runInput.cwd)}\n\n`);
51
+ },
52
+ onThresholdFailure: ({ overallPassRate, threshold }) => {
53
+ process.stdout.write(`\n CI THRESHOLD FAILED avg score ${overallPassRate.toFixed(3)} < threshold ${threshold}\n\n`);
54
+ },
55
+ },
56
+ });
57
+ }
58
+ const env = withJestMetadataEnv(runInput.env, runInput.cwd);
59
+ removeJestMetadata(env.PATHGRADE_JEST_METADATA_PATH);
60
+ return await spawnJest({
61
+ cwd: runInput.cwd,
62
+ env,
63
+ argv: [
64
+ ...discovered.units.map(unit => unit.sourceRef).filter((sourceRef) => typeof sourceRef === 'string'),
65
+ '--setupFilesAfterEnv',
66
+ resolveAdapterFile('setup'),
67
+ '--reporters',
68
+ 'default',
69
+ '--reporters',
70
+ resolveReporterFile(),
71
+ ...runInput.runnerArgs,
72
+ ],
73
+ });
74
+ },
75
+ };
76
+ }
77
+ export function createPathgradeInvocationAdapter(input) {
78
+ return createJestInvocationAdapter(input);
79
+ }
80
+ function resolveAdapterFile(name) {
81
+ return path.resolve(import.meta.dirname, `${name}.js`);
82
+ }
83
+ function resolveReporterFile() {
84
+ return path.resolve(import.meta.dirname, 'reporter.cjs');
85
+ }
86
+ async function defaultSpawnJest(req) {
87
+ const jestBinPath = resolveLocalJestBinPath(req.cwd);
88
+ return await new Promise(resolve => {
89
+ const child = spawn(process.execPath, [jestBinPath, ...req.argv], {
90
+ stdio: 'inherit',
91
+ env: req.env,
92
+ cwd: req.cwd,
93
+ });
94
+ child.on('close', code => resolve(code ?? 0));
95
+ child.on('error', () => resolve(1));
96
+ });
97
+ }
98
+ export function resolveLocalJestBinPath(cwd) {
99
+ let packageJsonPath;
100
+ try {
101
+ const req = createRequire(path.join(cwd, 'package.json'));
102
+ packageJsonPath = req.resolve('jest/package.json');
103
+ }
104
+ catch {
105
+ throw new Error('pathgrade: Jest adapter requires Jest to be installed in the project. Install it with `npm install --save-dev jest` or the equivalent for your package manager.');
106
+ }
107
+ const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
108
+ const bin = typeof packageJson.bin === 'string'
109
+ ? packageJson.bin
110
+ : packageJson.bin?.jest;
111
+ if (!bin) {
112
+ throw new Error(`pathgrade: could not find the Jest executable declared by ${packageJsonPath}.`);
113
+ }
114
+ return path.resolve(path.dirname(packageJsonPath), bin);
115
+ }
@@ -0,0 +1,21 @@
1
+ import { type CaseContext, type CaseContextProviderHandle, type ResultObserverHandle, type PathgradeTestMeta, type AdapterLifecycleHooks, type EvalResultEvent } from '@wix/pathgrade/adapter-kit';
2
+ export type JestHook = (fn: () => void | Promise<void>) => void;
3
+ export interface JestState {
4
+ testPath?: string;
5
+ currentTestName?: string;
6
+ }
7
+ export interface JestLifecycleInstallInput {
8
+ beforeEach: JestHook;
9
+ afterEach: JestHook;
10
+ afterAll?: JestHook;
11
+ getState: () => JestState;
12
+ lifecycle?: AdapterLifecycleHooks;
13
+ subscribeToResults?: (callback: (event: EvalResultEvent) => void) => ResultObserverHandle;
14
+ installCaseContextProvider?: (provider: () => CaseContext | null) => CaseContextProviderHandle;
15
+ }
16
+ export interface JestLifecycleInstallHandle {
17
+ restore(): void;
18
+ }
19
+ export declare function installJestLifecycle(input: JestLifecycleInstallInput): JestLifecycleInstallHandle;
20
+ export declare function getJestLifecycleMetadata(): Map<string, PathgradeTestMeta[]>;
21
+ export declare function resetJestLifecycleMetadata(): void;
@@ -0,0 +1,62 @@
1
+ import { installCaseContextProvider, subscribeToEvalResults, createRunnerLifecycleHooks, } from '@wix/pathgrade/adapter-kit';
2
+ import { appendJestMetadata } from './metadata.js';
3
+ import { jestCaseId } from './results.js';
4
+ const metadataByCaseId = new Map();
5
+ export function installJestLifecycle(input) {
6
+ const lifecycle = input.lifecycle ?? createRunnerLifecycleHooks();
7
+ const occurrenceCounts = new Map();
8
+ let activeCase = null;
9
+ let restored = false;
10
+ const resultCapture = (input.subscribeToResults ?? defaultSubscribeToResults)(event => lifecycle.onResult(event));
11
+ const contextProvider = (input.installCaseContextProvider ?? installCaseContextProvider)(() => activeCase);
12
+ metadataByCaseId.clear();
13
+ input.beforeEach(() => {
14
+ const state = input.getState();
15
+ const filePath = state.testPath ?? 'jest';
16
+ const caseName = state.currentTestName ?? 'Jest test';
17
+ const caseId = jestCaseId({
18
+ filePath,
19
+ fullName: caseName,
20
+ occurrenceCounts,
21
+ });
22
+ activeCase = {
23
+ caseId,
24
+ caseName,
25
+ filePath,
26
+ scope: 'runner-case',
27
+ };
28
+ });
29
+ input.afterEach(async () => {
30
+ const current = activeCase;
31
+ if (!current)
32
+ return;
33
+ const metadata = await lifecycle.flushCase(current.caseId);
34
+ if (metadata.length > 0) {
35
+ metadataByCaseId.set(current.caseId, metadata);
36
+ appendJestMetadata(current.caseId, metadata);
37
+ }
38
+ activeCase = null;
39
+ });
40
+ input.afterAll?.(async () => {
41
+ await lifecycle.cleanupRun();
42
+ restore();
43
+ });
44
+ return { restore };
45
+ function restore() {
46
+ if (restored)
47
+ return;
48
+ restored = true;
49
+ activeCase = null;
50
+ resultCapture.unsubscribe();
51
+ contextProvider.restore();
52
+ }
53
+ }
54
+ export function getJestLifecycleMetadata() {
55
+ return new Map(metadataByCaseId);
56
+ }
57
+ export function resetJestLifecycleMetadata() {
58
+ metadataByCaseId.clear();
59
+ }
60
+ function defaultSubscribeToResults(callback) {
61
+ return subscribeToEvalResults(({ result, agent }) => callback({ result, agent }), { owner: 'adapter', key: 'jest-lifecycle' });
62
+ }
@@ -0,0 +1,6 @@
1
+ import type { PathgradeTestMeta } from '@wix/pathgrade/adapter-kit';
2
+ export declare function resolveJestMetadataPath(cwd?: string): string;
3
+ export declare function withJestMetadataEnv(env: NodeJS.ProcessEnv, cwd: string): NodeJS.ProcessEnv;
4
+ export declare function appendJestMetadata(caseId: string, metadata: PathgradeTestMeta[]): void;
5
+ export declare function readJestMetadata(metadataPath?: string): Map<string, PathgradeTestMeta[]>;
6
+ export declare function removeJestMetadata(metadataPath?: string): void;
@@ -0,0 +1,35 @@
1
+ import * as fs from 'node:fs';
2
+ import * as path from 'node:path';
3
+ const METADATA_ENV = 'PATHGRADE_JEST_METADATA_PATH';
4
+ const FALLBACK_METADATA_FILE = 'jest-metadata.jsonl';
5
+ export function resolveJestMetadataPath(cwd = process.cwd()) {
6
+ return process.env[METADATA_ENV] ?? path.join(cwd, '.pathgrade', FALLBACK_METADATA_FILE);
7
+ }
8
+ export function withJestMetadataEnv(env, cwd) {
9
+ return {
10
+ ...env,
11
+ [METADATA_ENV]: env[METADATA_ENV] ?? path.join(cwd, '.pathgrade', `${process.pid}-${Date.now()}-${FALLBACK_METADATA_FILE}`),
12
+ };
13
+ }
14
+ export function appendJestMetadata(caseId, metadata) {
15
+ if (metadata.length === 0)
16
+ return;
17
+ const metadataPath = resolveJestMetadataPath();
18
+ fs.mkdirSync(path.dirname(metadataPath), { recursive: true });
19
+ fs.appendFileSync(metadataPath, `${JSON.stringify({ caseId, metadata })}\n`, 'utf8');
20
+ }
21
+ export function readJestMetadata(metadataPath = resolveJestMetadataPath()) {
22
+ if (!fs.existsSync(metadataPath))
23
+ return new Map();
24
+ const metadataByCaseId = new Map();
25
+ for (const line of fs.readFileSync(metadataPath, 'utf8').split('\n')) {
26
+ if (line.trim().length === 0)
27
+ continue;
28
+ const entry = JSON.parse(line);
29
+ metadataByCaseId.set(entry.caseId, entry.metadata);
30
+ }
31
+ return metadataByCaseId;
32
+ }
33
+ export function removeJestMetadata(metadataPath = resolveJestMetadataPath()) {
34
+ fs.rmSync(metadataPath, { force: true });
35
+ }
@@ -0,0 +1,10 @@
1
+ class PathgradeJestReporterBridge {
2
+ async onRunComplete(...args) {
3
+ const mod = await import('./reporter.js');
4
+ const Reporter = mod.default;
5
+ const reporter = new Reporter();
6
+ return await reporter.onRunComplete(...args);
7
+ }
8
+ }
9
+
10
+ module.exports = PathgradeJestReporterBridge;
@@ -0,0 +1,5 @@
1
+ import type { JestAggregatedResult } from './results.js';
2
+ export default class PathgradeJestReporter {
3
+ onRunComplete(_contexts: unknown, results: JestAggregatedResult): Promise<void>;
4
+ private openBrowserViewer;
5
+ }