@wix/pathgrade 0.37.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -32
- package/dist/adapter-kit/index.d.ts +17 -0
- package/dist/adapter-kit/index.js +14 -0
- package/dist/adapters/jest/index.d.ts +3 -0
- package/dist/adapters/jest/index.js +3 -0
- package/dist/adapters/jest/invocation-adapter.d.ts +15 -0
- package/dist/adapters/jest/invocation-adapter.js +115 -0
- package/dist/adapters/jest/lifecycle.d.ts +21 -0
- package/dist/adapters/jest/lifecycle.js +62 -0
- package/dist/adapters/jest/metadata.d.ts +6 -0
- package/dist/adapters/jest/metadata.js +35 -0
- package/dist/adapters/jest/reporter.cjs +10 -0
- package/dist/adapters/jest/reporter.d.ts +5 -0
- package/dist/adapters/jest/reporter.js +67 -0
- package/dist/adapters/jest/results.d.ts +31 -0
- package/dist/adapters/jest/results.js +144 -0
- package/dist/adapters/jest/runner-adapter.d.ts +9 -0
- package/dist/adapters/jest/runner-adapter.js +49 -0
- package/dist/adapters/jest/setup.d.ts +1 -0
- package/dist/adapters/jest/setup.js +10 -0
- package/dist/adapters/node-test/index.d.ts +8 -0
- package/dist/adapters/node-test/index.js +106 -0
- package/dist/adapters/node-test/invocation-adapter.d.ts +5 -0
- package/dist/adapters/node-test/invocation-adapter.js +47 -0
- package/dist/adapters/node-test/runner-adapter.d.ts +2 -0
- package/dist/adapters/node-test/runner-adapter.js +116 -0
- package/dist/adapters/vitest/index.d.ts +9 -0
- package/dist/adapters/vitest/index.js +82 -0
- package/dist/adapters/vitest/lifecycle.d.ts +24 -0
- package/dist/adapters/vitest/lifecycle.js +41 -0
- package/dist/adapters/vitest/reporter.d.ts +12 -0
- package/dist/adapters/vitest/reporter.js +75 -0
- package/dist/adapters/vitest/setup.d.ts +1 -0
- package/dist/adapters/vitest/setup.js +3 -0
- package/dist/agents/codex-app-server/agent.js +5 -1
- package/dist/agents/codex-app-server/protocol/index.js +3 -3
- package/dist/agents/codex-app-server/turn-completion.d.ts +1 -0
- package/dist/agents/codex-app-server/turn-completion.js +9 -0
- package/dist/commands/affected.js +22 -15
- package/dist/commands/run-args.d.ts +4 -3
- package/dist/commands/run-args.js +12 -6
- package/dist/commands/run-changed.d.ts +4 -4
- package/dist/commands/run-changed.js +45 -58
- package/dist/config/pathgrade.d.ts +48 -0
- package/dist/config/pathgrade.js +204 -0
- package/dist/pathgrade.d.ts +2 -2
- package/dist/pathgrade.js +35 -34
- package/dist/plugin/index.d.ts +2 -9
- package/dist/plugin/index.js +1 -81
- package/dist/plugin/lifecycle.d.ts +1 -45
- package/dist/plugin/lifecycle.js +1 -146
- package/dist/plugin/reporter.d.ts +1 -37
- package/dist/plugin/reporter.js +1 -280
- package/dist/plugin/setup.d.ts +1 -1
- package/dist/plugin/setup.js +1 -3
- package/dist/providers/copy-filter.d.ts +5 -0
- package/dist/providers/copy-filter.js +9 -0
- package/dist/providers/sandbox.js +2 -2
- package/dist/providers/workspace.d.ts +6 -0
- package/dist/providers/workspace.js +34 -14
- package/dist/reporters/diagnostics.d.ts +2 -40
- package/dist/reporters/diagnostics.js +1 -80
- package/dist/reporters/report-summary.d.ts +6 -0
- package/dist/reporters/report-summary.js +29 -0
- package/dist/reporting/artifacts.d.ts +2 -0
- package/dist/reporting/artifacts.js +20 -0
- package/dist/reporting/core.d.ts +2 -0
- package/dist/reporting/core.js +173 -0
- package/dist/reporting/types.d.ts +57 -0
- package/dist/reporting/types.js +1 -0
- package/dist/reporting/vitest-edge.d.ts +1 -0
- package/dist/reporting/vitest-edge.js +1 -0
- package/dist/runners/adapter-loader.d.ts +17 -0
- package/dist/runners/adapter-loader.js +67 -0
- package/dist/runners/adapter.d.ts +65 -0
- package/dist/runners/adapter.js +1 -0
- package/dist/runners/invocation.d.ts +10 -0
- package/dist/runners/invocation.js +1 -0
- package/dist/runners/lifecycle-hooks.d.ts +2 -0
- package/dist/runners/lifecycle-hooks.js +18 -0
- package/dist/runners/model-builders.d.ts +4 -0
- package/dist/runners/model-builders.js +67 -0
- package/dist/runners/model-validation.d.ts +12 -0
- package/dist/runners/model-validation.js +213 -0
- package/dist/runners/model.d.ts +117 -0
- package/dist/runners/model.js +1 -0
- package/dist/runners/orchestrator.d.ts +30 -0
- package/dist/runners/orchestrator.js +74 -0
- package/dist/runners/report-projection.d.ts +6 -0
- package/dist/runners/report-projection.js +62 -0
- package/dist/runners/selection.d.ts +6 -0
- package/dist/runners/selection.js +10 -0
- package/dist/runners/vitest-adapter.d.ts +8 -0
- package/dist/runners/vitest-adapter.js +108 -0
- package/dist/runners/vitest-invocation.d.ts +10 -0
- package/dist/runners/vitest-invocation.js +44 -0
- package/dist/runners/vitest-lifecycle.d.ts +43 -0
- package/dist/runners/vitest-lifecycle.js +150 -0
- package/dist/sdk/agent.js +15 -16
- package/dist/sdk/case-context.d.ts +20 -0
- package/dist/sdk/case-context.js +31 -0
- package/dist/sdk/diagnostics.d.ts +40 -0
- package/dist/sdk/diagnostics.js +80 -0
- package/dist/sdk/eval-runtime.d.ts +4 -0
- package/dist/sdk/eval-runtime.js +11 -1
- package/dist/sdk/evaluate.js +2 -1
- package/dist/sdk/index.d.ts +2 -0
- package/dist/sdk/index.js +1 -0
- package/dist/sdk/lifecycle.d.ts +40 -0
- package/dist/sdk/lifecycle.js +170 -0
- package/dist/sdk/result-capture.d.ts +18 -0
- package/dist/sdk/result-capture.js +49 -0
- package/dist/sdk/types.d.ts +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/utils/llm-providers/anthropic.js +9 -2
- package/package.json +56 -4
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Pathgrade
|
|
2
2
|
|
|
3
|
-
**Evaluate AI coding agents with Vitest.** Write evals as normal `.eval.ts` files, run Claude Code, Codex, or Cursor in isolated sandboxes, and score the result with deterministic checks, rubric judges, and tool-usage assertions.
|
|
3
|
+
**Evaluate AI coding agents with Vitest or Jest.** Write evals as normal `.eval.ts` files, run Claude Code, Codex, or Cursor in isolated sandboxes, and score the result with deterministic checks, rubric judges, and tool-usage assertions.
|
|
4
4
|
|
|
5
5
|
## Why Pathgrade?
|
|
6
6
|
|
|
7
|
-
- Write evals in plain TypeScript with Vitest
|
|
7
|
+
- Write evals in plain TypeScript with Vitest or Jest
|
|
8
8
|
- Run each trial in an isolated workspace and HOME directory
|
|
9
9
|
- Seed trials from fixtures, real skills, or mocked MCP servers
|
|
10
10
|
- Score both final artifacts and the workflow that produced them
|
|
@@ -13,12 +13,18 @@
|
|
|
13
13
|
|
|
14
14
|
## Quick Start
|
|
15
15
|
|
|
16
|
-
**Prerequisites**: Node.js 20.11+, Vitest 4+, and at least one
|
|
16
|
+
**Prerequisites**: Node.js 20.11+, Vitest 4+ or Jest 30+, and at least one configured agent runtime. Claude uses the bundled `@anthropic-ai/claude-agent-sdk` binary by default; Codex requires the `codex` CLI; Cursor requires the `cursor-agent` CLI.
|
|
17
17
|
|
|
18
18
|
```bash
|
|
19
19
|
yarn add -D @wix/pathgrade
|
|
20
20
|
```
|
|
21
21
|
|
|
22
|
+
For Jest projects, install Jest too:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
yarn add -D @wix/pathgrade jest
|
|
26
|
+
```
|
|
27
|
+
|
|
22
28
|
### Authentication
|
|
23
29
|
|
|
24
30
|
By default, Pathgrade tries to reuse the agent CLI's native auth before falling back to explicit environment variables.
|
|
@@ -27,9 +33,9 @@ By default, Pathgrade tries to reuse the agent CLI's native auth before falling
|
|
|
27
33
|
- macOS: reuses Claude Code OAuth from Keychain
|
|
28
34
|
- other platforms: forwards `ANTHROPIC_API_KEY` when present
|
|
29
35
|
- **Codex**
|
|
30
|
-
-
|
|
31
|
-
- or
|
|
32
|
-
-
|
|
36
|
+
- forwards `OPENAI_API_KEY` when present
|
|
37
|
+
- or runs `codex login --with-api-key` inside the sandbox when an API key is present
|
|
38
|
+
- `codex exec` can reuse cached `~/.codex/auth.json` when no key is available; `app-server` may reject cached ChatGPT-token refreshes, so prefer `OPENAI_API_KEY` for the default transport
|
|
33
39
|
- **Cursor**
|
|
34
40
|
- forwards `CURSOR_API_KEY` when set
|
|
35
41
|
- macOS: reuses `cursor-agent login` OAuth tokens from the login Keychain
|
|
@@ -52,7 +58,7 @@ const agent = await createAgent({
|
|
|
52
58
|
|
|
53
59
|
Codex supports two transports and Pathgrade defaults to `app-server`:
|
|
54
60
|
|
|
55
|
-
- `app-server` (default) — uses `codex app-server` and keeps native thread state. Required for `AskUserReaction` handshakes (`request_user_input` reaches the model).
|
|
61
|
+
- `app-server` (default) — uses `codex app-server` and keeps native thread state. Required for `AskUserReaction` handshakes (`request_user_input` reaches the model). Prefer `OPENAI_API_KEY`; cached ChatGPT auth can fail if the app-server asks Pathgrade to refresh tokens.
|
|
56
62
|
- `exec` — uses `codex exec` and re-injects the transcript every turn. Kept for stateless CI matrices that don't need the handshake.
|
|
57
63
|
|
|
58
64
|
Precedence: `createAgent({ transport })` > `PATHGRADE_CODEX_TRANSPORT` env > default (`app-server`). An invalid env value throws at `createAgent` time.
|
|
@@ -68,17 +74,17 @@ If `transport: 'exec'` is resolved and any `AskUserReaction` is present in `Conv
|
|
|
68
74
|
|
|
69
75
|
Migrating from `exec` to `app-server`:
|
|
70
76
|
|
|
71
|
-
- Export `OPENAI_API_KEY`, or set `transport: 'exec'` / `PATHGRADE_CODEX_TRANSPORT=exec` to stay on the old transport.
|
|
77
|
+
- Export `OPENAI_API_KEY`, or set `transport: 'exec'` / `PATHGRADE_CODEX_TRANSPORT=exec` to stay on the old transport and its cached-auth behavior.
|
|
72
78
|
- The `noninteractive-user-question` runtime policy no longer attaches under `app-server`. Snapshots that captured model output influenced by that policy text may need re-recording.
|
|
73
79
|
- `MAX_TURN_RETRIES` does not apply under `app-server` — a crashed turn ends the conversation with `completionReason: 'agent_crashed'`.
|
|
74
80
|
|
|
75
81
|
### Plugin Setup
|
|
76
82
|
|
|
77
|
-
Create a `vitest.config.ts` with the
|
|
83
|
+
Create a `vitest.config.ts` with the built-in Vitest adapter plugin:
|
|
78
84
|
|
|
79
85
|
```typescript
|
|
80
86
|
import { defineConfig } from 'vitest/config';
|
|
81
|
-
import { pathgrade } from '@wix/pathgrade/
|
|
87
|
+
import { pathgrade } from '@wix/pathgrade/adapters/vitest';
|
|
82
88
|
|
|
83
89
|
export default defineConfig({
|
|
84
90
|
plugins: [pathgrade({ timeout: 120 })],
|
|
@@ -87,6 +93,24 @@ export default defineConfig({
|
|
|
87
93
|
|
|
88
94
|
The plugin registers the setup hooks Pathgrade needs, wires in the reporter, and automatically cleans up agent workspaces after each test.
|
|
89
95
|
|
|
96
|
+
For Pathgrade CLI behavior, use `pathgrade.config.ts`:
|
|
97
|
+
|
|
98
|
+
```typescript
|
|
99
|
+
export default {
|
|
100
|
+
runner: {
|
|
101
|
+
adapter: 'vitest', // default; use 'jest' for Jest projects
|
|
102
|
+
args: [],
|
|
103
|
+
},
|
|
104
|
+
evals: {
|
|
105
|
+
include: ['**/*.eval.ts'],
|
|
106
|
+
exclude: ['**/fixtures/**'],
|
|
107
|
+
},
|
|
108
|
+
affected: {
|
|
109
|
+
global: ['package.json', 'yarn.lock'],
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
```
|
|
113
|
+
|
|
90
114
|
### First Eval
|
|
91
115
|
|
|
92
116
|
Write an eval file such as `hello.eval.ts`:
|
|
@@ -122,7 +146,7 @@ Run your evals:
|
|
|
122
146
|
npx pathgrade run
|
|
123
147
|
```
|
|
124
148
|
|
|
125
|
-
`pathgrade run` is the recommended wrapper: it loads `.env`, warns when no auth is configured, and adds Pathgrade-specific flags such as `--changed`, `--diagnostics`, and `--verbose`. Plain `npx vitest run`
|
|
149
|
+
`pathgrade run` is the recommended wrapper: it loads `.env`, warns when no auth is configured, and adds Pathgrade-specific flags such as `--changed`, `--diagnostics`, and `--verbose`. Plain `npx vitest run` or direct Jest runs work too if you configure the Pathgrade adapter hooks yourself.
|
|
126
150
|
|
|
127
151
|
## Core Concepts
|
|
128
152
|
|
|
@@ -193,7 +217,7 @@ judge('spec-structure', {
|
|
|
193
217
|
});
|
|
194
218
|
```
|
|
195
219
|
|
|
196
|
-
Tool-using judges currently require the Anthropic HTTP provider (`ANTHROPIC_API_KEY`); other providers produce a clean `provider_not_supported` error. See the [User Guide](
|
|
220
|
+
Tool-using judges currently require the Anthropic HTTP provider (`ANTHROPIC_API_KEY`); other providers produce a clean `provider_not_supported` error. See the [User Guide](docs/USER_GUIDE.md#tool-using-judges) for the full tool list, failure codes, and the migration recipe from `input`-helper probes.
|
|
197
221
|
|
|
198
222
|
### `toolUsage()` - Tool event matching
|
|
199
223
|
|
|
@@ -207,7 +231,7 @@ toolUsage('expected-workflow', [
|
|
|
207
231
|
|
|
208
232
|
## Conversations
|
|
209
233
|
|
|
210
|
-
Pathgrade currently supports three agent backends: `claude`, `codex`, and `cursor`. Set the backend per test via `createAgent({ agent: 'claude' })
|
|
234
|
+
Pathgrade currently supports three agent backends: `claude`, `codex`, and `cursor`. Set the backend per test via `createAgent({ agent: 'claude' })`, or omit `agent` and use `PATHGRADE_AGENT` as the process-wide fallback.
|
|
211
235
|
|
|
212
236
|
### `agent.prompt()` - One shot
|
|
213
237
|
|
|
@@ -274,11 +298,11 @@ Pathgrade exposes a few useful features that are easy to miss from the basic exa
|
|
|
274
298
|
- `conversationWindow` on agents and personas keeps long transcripts bounded with summarization instead of sending the full conversation every turn.
|
|
275
299
|
- `copyIgnore` and `DEFAULT_COPY_IGNORE` let you control what gets copied into the sandbox when seeding from large fixtures or skill directories.
|
|
276
300
|
|
|
277
|
-
See [sdk-showcase](
|
|
301
|
+
See [sdk-showcase](examples/sdk-showcase/) for a single example suite that demonstrates these APIs together.
|
|
278
302
|
|
|
279
303
|
## MCP Mock Servers
|
|
280
304
|
|
|
281
|
-
Simulate MCP tools when testing Claude-
|
|
305
|
+
Simulate MCP tools when testing Claude, Codex app-server, or Cursor evals:
|
|
282
306
|
|
|
283
307
|
```typescript
|
|
284
308
|
import { mockMcpServer } from '@wix/pathgrade/mcp-mock';
|
|
@@ -299,7 +323,7 @@ const agent = await createAgent({ agent: 'claude', mcpMock: mock });
|
|
|
299
323
|
## CLI
|
|
300
324
|
|
|
301
325
|
```bash
|
|
302
|
-
pathgrade run [--changed] [--diagnostics] [--verbose] [--
|
|
326
|
+
pathgrade run [--changed] [--since=<ref>] [--changed-files=<path>] [--adapter=<name|path>] [--diagnostics] [--verbose] [--quiet] [-- runner-args]
|
|
303
327
|
pathgrade init [--force]
|
|
304
328
|
pathgrade validate <file.eval.ts>
|
|
305
329
|
pathgrade validate --affected
|
|
@@ -312,36 +336,74 @@ pathgrade report [--results-path=<path>] [--no-comment] [--comment-id=<id>]
|
|
|
312
336
|
|
|
313
337
|
Useful details:
|
|
314
338
|
|
|
315
|
-
- `pathgrade run --changed` computes affected evals first, writes selection metadata to `.pathgrade/selection.json`, and only then launches Vitest.
|
|
339
|
+
- `pathgrade run --changed` computes affected evals first, writes selection metadata to `.pathgrade/selection.json`, and only then launches the selected runner adapter. Vitest is the default adapter; Jest is selected with `runner.adapter: 'jest'` or `--adapter=jest`.
|
|
316
340
|
- `pathgrade preview browser` starts a local viewer on `http://localhost:3847`.
|
|
317
341
|
- `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate.
|
|
318
342
|
- `pathgrade validate --affected` is a strict mode for CI: every discovered eval must either live under a `SKILL.md` anchor or export valid `__pathgradeMeta`.
|
|
319
343
|
|
|
320
344
|
Run `pathgrade --help` for the full help text.
|
|
321
345
|
|
|
322
|
-
##
|
|
346
|
+
## Configuration
|
|
347
|
+
|
|
348
|
+
```typescript
|
|
349
|
+
// pathgrade.config.ts
|
|
350
|
+
export default {
|
|
351
|
+
runner: {
|
|
352
|
+
adapter: 'vitest',
|
|
353
|
+
args: [],
|
|
354
|
+
},
|
|
355
|
+
evals: {
|
|
356
|
+
include: ['**/*.eval.ts'], // default
|
|
357
|
+
exclude: ['**/fixtures/**'], // replaces the default exclude list if set
|
|
358
|
+
},
|
|
359
|
+
affected: {
|
|
360
|
+
global: ['package.json', 'yarn.lock'],
|
|
361
|
+
},
|
|
362
|
+
ci: { threshold: 0.8 },
|
|
363
|
+
};
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. Built-in adapters currently include `vitest`, `jest`, and the narrow `node-test` proof adapter.
|
|
367
|
+
|
|
368
|
+
Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
|
|
369
|
+
|
|
370
|
+
- `vitest`, `jest`, `node-test`: built-in adapters
|
|
371
|
+
- `demo`: package `@wix/pathgrade-adapter-demo`, resolved from the project
|
|
372
|
+
- `@scope/pathgrade-adapter-demo` or another specifier containing `/`: package specifier, resolved from the project
|
|
373
|
+
- `./local-adapter.mjs` or `/abs/local-adapter.mjs`: local adapter module
|
|
374
|
+
|
|
375
|
+
External adapter modules used by `pathgrade run` must export `createPathgradeInvocationAdapter({ config })`, returning a `RunnerInvocationAdapter`. Modules used by lower-level orchestration can also export `createPathgradeAdapter()`, returning a `RunnerAdapter` with `discover`, `invoke`, and `collectNormalizedRunSnapshot`.
|
|
376
|
+
|
|
377
|
+
Vitest runner behavior still belongs in `vitest.config.ts`:
|
|
323
378
|
|
|
324
379
|
```typescript
|
|
325
|
-
import { pathgrade } from '@wix/pathgrade/
|
|
380
|
+
import { pathgrade } from '@wix/pathgrade/adapters/vitest';
|
|
326
381
|
|
|
327
382
|
pathgrade({
|
|
328
|
-
include: ['**/*.eval.ts'], // default: ['**/*.eval.ts']
|
|
329
383
|
timeout: 300, // seconds, default: 300
|
|
330
384
|
reporter: 'cli', // 'cli' | 'browser' | 'json'
|
|
331
385
|
diagnostics: false, // print full diagnostics for passing evals too
|
|
332
386
|
verbose: false, // stream live per-turn events to stderr while evals run
|
|
333
|
-
ci: { threshold: 0.8 }, // fail when the mean test score drops below threshold
|
|
334
|
-
affected: {
|
|
335
|
-
global: ['package.json', 'yarn.lock'],
|
|
336
|
-
},
|
|
337
387
|
});
|
|
338
388
|
```
|
|
339
389
|
|
|
390
|
+
Jest runner behavior still belongs in `jest.config.*`. Pathgrade injects only the setup and reporter it needs when you use `pathgrade run --adapter=jest`; transforms, test environment, module resolution, and ESM/TypeScript support remain your Jest config's job.
|
|
391
|
+
|
|
392
|
+
For direct Jest runs, configure the same entry points explicitly:
|
|
393
|
+
|
|
394
|
+
```js
|
|
395
|
+
// jest.config.mjs
|
|
396
|
+
export default {
|
|
397
|
+
setupFilesAfterEnv: ['@wix/pathgrade/adapters/jest/setup'],
|
|
398
|
+
reporters: ['default', '@wix/pathgrade/adapters/jest/reporter'],
|
|
399
|
+
};
|
|
400
|
+
```
|
|
401
|
+
|
|
340
402
|
Notes:
|
|
341
403
|
|
|
342
|
-
- `
|
|
404
|
+
- Legacy `@wix/pathgrade/plugin` and `@wix/pathgrade/plugin/vitest` imports remain as compatibility fallbacks, but new config should use `@wix/pathgrade/adapters/vitest` and `pathgrade.config.*`.
|
|
343
405
|
- `reporter: 'browser'` writes results JSON and opens the viewer automatically after the run.
|
|
344
|
-
-
|
|
406
|
+
- SDK-only consumers can import `@wix/pathgrade` without installing Vitest or Jest. Adapter users need their selected runner available.
|
|
345
407
|
|
|
346
408
|
## Environment Variables
|
|
347
409
|
|
|
@@ -359,6 +421,28 @@ Notes:
|
|
|
359
421
|
| `PATHGRADE_DIAGNOSTICS` | `1` prints full diagnostics for passing evals too |
|
|
360
422
|
| `NO_COLOR` | Disable ANSI colors |
|
|
361
423
|
|
|
424
|
+
### Experimental `node:test` Adapter
|
|
425
|
+
|
|
426
|
+
`node-test` is a small proof adapter for validating the runner boundary without Vitest. It is intentionally narrow: eval files import the `test` wrapper from `@wix/pathgrade/adapters/node-test`, run through Node's built-in test runner, and still produce the normal `.pathgrade/results.json` and trace artifacts.
|
|
427
|
+
|
|
428
|
+
```ts
|
|
429
|
+
import { test } from '@wix/pathgrade/adapters/node-test';
|
|
430
|
+
import { createAgent, evaluate, check } from '@wix/pathgrade';
|
|
431
|
+
|
|
432
|
+
test('minimal node proof', async () => {
|
|
433
|
+
const agent = await createAgent({ workspace: process.cwd() });
|
|
434
|
+
await evaluate(agent, [check('passes', () => true)]);
|
|
435
|
+
});
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
Run it with:
|
|
439
|
+
|
|
440
|
+
```bash
|
|
441
|
+
pathgrade run --adapter=node-test
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
This proof validates Pathgrade's adapter, lifecycle, result capture, and reporting boundaries. It does not imply Mocha, Playwright, or runnerless CLI support.
|
|
445
|
+
|
|
362
446
|
`pathgrade run` loads `.env` from the working directory automatically.
|
|
363
447
|
|
|
364
448
|
## CI / GitHub Actions
|
|
@@ -401,17 +485,17 @@ jobs:
|
|
|
401
485
|
|
|
402
486
|
- `fetch-depth: 0` is required for `--changed`; shallow clones break merge-base resolution.
|
|
403
487
|
- Evals under a `SKILL.md` are tracked automatically; use `__pathgradeMeta` for cross-skill or non-standard dependencies.
|
|
404
|
-
- Set `ci: { threshold: 0.8 }` in
|
|
488
|
+
- Set `ci: { threshold: 0.8 }` in `pathgrade.config.ts` to fail the run when the mean test score drops below your threshold.
|
|
405
489
|
|
|
406
|
-
See the [User Guide - CI Integration](
|
|
490
|
+
See the [User Guide - CI Integration](docs/USER_GUIDE.md#ci-integration) for the full reference.
|
|
407
491
|
|
|
408
492
|
## Links
|
|
409
493
|
|
|
410
|
-
- [User Guide](
|
|
494
|
+
- [User Guide](docs/USER_GUIDE.md) - full API reference and usage patterns
|
|
411
495
|
- Examples:
|
|
412
|
-
- [start-chat](
|
|
413
|
-
- [sdk-showcase](
|
|
414
|
-
- [tool-judge-demo](
|
|
496
|
+
- [start-chat](examples/start-chat/) - multi-turn conversation
|
|
497
|
+
- [sdk-showcase](examples/sdk-showcase/) - advanced SDK features in one suite
|
|
498
|
+
- [tool-judge-demo](examples/tool-judge-demo/) - `judge({ tools })` reading workspace artifacts
|
|
415
499
|
|
|
416
500
|
## Note on AI provider dependencies
|
|
417
501
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export { runnerAdapterContractVersion, type AdapterCaseContext, type AdapterDiagnostic, type AdapterDiscoveryInput, type AdapterDiscoveryResult, type AdapterEvalUnit, type AdapterInvocationInput, type AdapterLifecycleHooks, type AdapterRunHandle, type AdapterRunStatus, type EvalResultEvent, type RunnerAdapter, } from '../runners/adapter.js';
|
|
2
|
+
export type { RunnerInvocationAdapter, RunnerInvocationInput, } from '../runners/invocation.js';
|
|
3
|
+
export type { AttemptOutcome, AttemptRecord, AssertionRecord, Diagnostic, EvalUnitRecord, EvaluationRecord, GroupingHint, NativeReference, NormalizedRunModel, NormalizedRunSnapshot, RunCaseRecord, RunCaseState, RunRecord, RunStatus, ScoringPolicy, SnapshotCompleteness, } from '../runners/model.js';
|
|
4
|
+
export { buildNormalizedRunSnapshotFromReportGroups } from '../runners/model-builders.js';
|
|
5
|
+
export { validateNormalizedRunSnapshot } from '../runners/model-validation.js';
|
|
6
|
+
export { projectNormalizedRunSnapshotToReportInput } from '../runners/report-projection.js';
|
|
7
|
+
export { runWithAdapter, type PathgradeRunOptions, type AdapterReporterMode } from '../runners/orchestrator.js';
|
|
8
|
+
export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
|
|
9
|
+
export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
|
|
10
|
+
export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, type PathgradeConfig, type ResolvedPathgradeConfig, } from '../config/pathgrade.js';
|
|
11
|
+
export { readSidecar } from '../affected/sidecar.js';
|
|
12
|
+
export { getPathgradeDir } from '../reporters/results-path.js';
|
|
13
|
+
export { printReportSummary } from '../reporters/report-summary.js';
|
|
14
|
+
export { fmt } from '../utils/cli.js';
|
|
15
|
+
export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, type CaseContext, type CaseContextProvider, type CaseContextProviderHandle, type CaseContextScope, type CurrentCaseContext, } from '../sdk/case-context.js';
|
|
16
|
+
export { subscribeToEvalResults, type EvalResultObserver, type ResultObserverHandle, type ResultObserverOptions, type ResultObserverOwner, } from '../sdk/result-capture.js';
|
|
17
|
+
export type { Agent, PathgradeTestMeta, RecordedEvalResult, } from '../sdk/types.js';
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export { runnerAdapterContractVersion, } from '../runners/adapter.js';
|
|
2
|
+
export { buildNormalizedRunSnapshotFromReportGroups } from '../runners/model-builders.js';
|
|
3
|
+
export { validateNormalizedRunSnapshot } from '../runners/model-validation.js';
|
|
4
|
+
export { projectNormalizedRunSnapshotToReportInput } from '../runners/report-projection.js';
|
|
5
|
+
export { runWithAdapter } from '../runners/orchestrator.js';
|
|
6
|
+
export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
|
|
7
|
+
export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
|
|
8
|
+
export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, } from '../config/pathgrade.js';
|
|
9
|
+
export { readSidecar } from '../affected/sidecar.js';
|
|
10
|
+
export { getPathgradeDir } from '../reporters/results-path.js';
|
|
11
|
+
export { printReportSummary } from '../reporters/report-summary.js';
|
|
12
|
+
export { fmt } from '../utils/cli.js';
|
|
13
|
+
export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, } from '../sdk/case-context.js';
|
|
14
|
+
export { subscribeToEvalResults, } from '../sdk/result-capture.js';
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type ResolvedPathgradeConfig, type RunnerInvocationAdapter } from '@wix/pathgrade/adapter-kit';
|
|
2
|
+
export interface SpawnJestRequest {
|
|
3
|
+
argv: string[];
|
|
4
|
+
env: NodeJS.ProcessEnv;
|
|
5
|
+
cwd: string;
|
|
6
|
+
}
|
|
7
|
+
export type SpawnJest = (req: SpawnJestRequest) => Promise<number> | number;
|
|
8
|
+
export declare function createJestInvocationAdapter(input: {
|
|
9
|
+
config: ResolvedPathgradeConfig;
|
|
10
|
+
spawnJest?: SpawnJest;
|
|
11
|
+
}): RunnerInvocationAdapter;
|
|
12
|
+
export declare function createPathgradeInvocationAdapter(input: {
|
|
13
|
+
config: ResolvedPathgradeConfig;
|
|
14
|
+
}): RunnerInvocationAdapter;
|
|
15
|
+
export declare function resolveLocalJestBinPath(cwd: string): string;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { spawn } from 'node:child_process';
|
|
4
|
+
import { createRequire } from 'node:module';
|
|
5
|
+
import { getPathgradeDir, printReportSummary, readSidecar, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
|
|
6
|
+
import { removeJestMetadata, withJestMetadataEnv } from './metadata.js';
|
|
7
|
+
import { createJestAdapter } from './runner-adapter.js';
|
|
8
|
+
export function createJestInvocationAdapter(input) {
|
|
9
|
+
const spawnJest = input.spawnJest ?? defaultSpawnJest;
|
|
10
|
+
return {
|
|
11
|
+
name: 'jest',
|
|
12
|
+
async run(runInput) {
|
|
13
|
+
const selection = await readSidecar(runInput.cwd, msg => {
|
|
14
|
+
process.stderr.write(`[pathgrade] ${msg}\n`);
|
|
15
|
+
}) ?? undefined;
|
|
16
|
+
const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
|
|
17
|
+
? runInput.selectedFiles
|
|
18
|
+
: input.config.evals.include;
|
|
19
|
+
const exclude = runInput.selectedFiles ? [] : input.config.evals.exclude;
|
|
20
|
+
const discovered = await createJestAdapter().discover({
|
|
21
|
+
cwd: runInput.cwd,
|
|
22
|
+
include,
|
|
23
|
+
exclude,
|
|
24
|
+
selection,
|
|
25
|
+
});
|
|
26
|
+
if (discovered.units.length === 0) {
|
|
27
|
+
return await runWithAdapter({
|
|
28
|
+
adapter: createJestAdapter({ results: { testResults: [] } }),
|
|
29
|
+
options: {
|
|
30
|
+
cwd: runInput.cwd,
|
|
31
|
+
discovery: {
|
|
32
|
+
cwd: runInput.cwd,
|
|
33
|
+
include,
|
|
34
|
+
exclude,
|
|
35
|
+
selection,
|
|
36
|
+
},
|
|
37
|
+
runnerArgs: [],
|
|
38
|
+
env: runInput.env,
|
|
39
|
+
artifactRoot: getPathgradeDir(runInput.cwd),
|
|
40
|
+
reporterMode: input.config.reporter ?? 'cli',
|
|
41
|
+
threshold: input.config.ci.threshold,
|
|
42
|
+
selection,
|
|
43
|
+
printSummary: summaries => {
|
|
44
|
+
printReportSummary(summaries, {
|
|
45
|
+
forceVerbose: input.config.diagnostics
|
|
46
|
+
|| runInput.env.PATHGRADE_DIAGNOSTICS === '1',
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
log: () => {
|
|
50
|
+
process.stdout.write(`\n Results written to ${getPathgradeDir(runInput.cwd)}\n\n`);
|
|
51
|
+
},
|
|
52
|
+
onThresholdFailure: ({ overallPassRate, threshold }) => {
|
|
53
|
+
process.stdout.write(`\n CI THRESHOLD FAILED avg score ${overallPassRate.toFixed(3)} < threshold ${threshold}\n\n`);
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
const env = withJestMetadataEnv(runInput.env, runInput.cwd);
|
|
59
|
+
removeJestMetadata(env.PATHGRADE_JEST_METADATA_PATH);
|
|
60
|
+
return await spawnJest({
|
|
61
|
+
cwd: runInput.cwd,
|
|
62
|
+
env,
|
|
63
|
+
argv: [
|
|
64
|
+
...discovered.units.map(unit => unit.sourceRef).filter((sourceRef) => typeof sourceRef === 'string'),
|
|
65
|
+
'--setupFilesAfterEnv',
|
|
66
|
+
resolveAdapterFile('setup'),
|
|
67
|
+
'--reporters',
|
|
68
|
+
'default',
|
|
69
|
+
'--reporters',
|
|
70
|
+
resolveReporterFile(),
|
|
71
|
+
...runInput.runnerArgs,
|
|
72
|
+
],
|
|
73
|
+
});
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
export function createPathgradeInvocationAdapter(input) {
|
|
78
|
+
return createJestInvocationAdapter(input);
|
|
79
|
+
}
|
|
80
|
+
function resolveAdapterFile(name) {
|
|
81
|
+
return path.resolve(import.meta.dirname, `${name}.js`);
|
|
82
|
+
}
|
|
83
|
+
function resolveReporterFile() {
|
|
84
|
+
return path.resolve(import.meta.dirname, 'reporter.cjs');
|
|
85
|
+
}
|
|
86
|
+
async function defaultSpawnJest(req) {
|
|
87
|
+
const jestBinPath = resolveLocalJestBinPath(req.cwd);
|
|
88
|
+
return await new Promise(resolve => {
|
|
89
|
+
const child = spawn(process.execPath, [jestBinPath, ...req.argv], {
|
|
90
|
+
stdio: 'inherit',
|
|
91
|
+
env: req.env,
|
|
92
|
+
cwd: req.cwd,
|
|
93
|
+
});
|
|
94
|
+
child.on('close', code => resolve(code ?? 0));
|
|
95
|
+
child.on('error', () => resolve(1));
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
export function resolveLocalJestBinPath(cwd) {
|
|
99
|
+
let packageJsonPath;
|
|
100
|
+
try {
|
|
101
|
+
const req = createRequire(path.join(cwd, 'package.json'));
|
|
102
|
+
packageJsonPath = req.resolve('jest/package.json');
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
throw new Error('pathgrade: Jest adapter requires Jest to be installed in the project. Install it with `npm install --save-dev jest` or the equivalent for your package manager.');
|
|
106
|
+
}
|
|
107
|
+
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
|
|
108
|
+
const bin = typeof packageJson.bin === 'string'
|
|
109
|
+
? packageJson.bin
|
|
110
|
+
: packageJson.bin?.jest;
|
|
111
|
+
if (!bin) {
|
|
112
|
+
throw new Error(`pathgrade: could not find the Jest executable declared by ${packageJsonPath}.`);
|
|
113
|
+
}
|
|
114
|
+
return path.resolve(path.dirname(packageJsonPath), bin);
|
|
115
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type CaseContext, type CaseContextProviderHandle, type ResultObserverHandle, type PathgradeTestMeta, type AdapterLifecycleHooks, type EvalResultEvent } from '@wix/pathgrade/adapter-kit';
|
|
2
|
+
export type JestHook = (fn: () => void | Promise<void>) => void;
|
|
3
|
+
export interface JestState {
|
|
4
|
+
testPath?: string;
|
|
5
|
+
currentTestName?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface JestLifecycleInstallInput {
|
|
8
|
+
beforeEach: JestHook;
|
|
9
|
+
afterEach: JestHook;
|
|
10
|
+
afterAll?: JestHook;
|
|
11
|
+
getState: () => JestState;
|
|
12
|
+
lifecycle?: AdapterLifecycleHooks;
|
|
13
|
+
subscribeToResults?: (callback: (event: EvalResultEvent) => void) => ResultObserverHandle;
|
|
14
|
+
installCaseContextProvider?: (provider: () => CaseContext | null) => CaseContextProviderHandle;
|
|
15
|
+
}
|
|
16
|
+
export interface JestLifecycleInstallHandle {
|
|
17
|
+
restore(): void;
|
|
18
|
+
}
|
|
19
|
+
export declare function installJestLifecycle(input: JestLifecycleInstallInput): JestLifecycleInstallHandle;
|
|
20
|
+
export declare function getJestLifecycleMetadata(): Map<string, PathgradeTestMeta[]>;
|
|
21
|
+
export declare function resetJestLifecycleMetadata(): void;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { installCaseContextProvider, subscribeToEvalResults, createRunnerLifecycleHooks, } from '@wix/pathgrade/adapter-kit';
|
|
2
|
+
import { appendJestMetadata } from './metadata.js';
|
|
3
|
+
import { jestCaseId } from './results.js';
|
|
4
|
+
const metadataByCaseId = new Map();
|
|
5
|
+
export function installJestLifecycle(input) {
|
|
6
|
+
const lifecycle = input.lifecycle ?? createRunnerLifecycleHooks();
|
|
7
|
+
const occurrenceCounts = new Map();
|
|
8
|
+
let activeCase = null;
|
|
9
|
+
let restored = false;
|
|
10
|
+
const resultCapture = (input.subscribeToResults ?? defaultSubscribeToResults)(event => lifecycle.onResult(event));
|
|
11
|
+
const contextProvider = (input.installCaseContextProvider ?? installCaseContextProvider)(() => activeCase);
|
|
12
|
+
metadataByCaseId.clear();
|
|
13
|
+
input.beforeEach(() => {
|
|
14
|
+
const state = input.getState();
|
|
15
|
+
const filePath = state.testPath ?? 'jest';
|
|
16
|
+
const caseName = state.currentTestName ?? 'Jest test';
|
|
17
|
+
const caseId = jestCaseId({
|
|
18
|
+
filePath,
|
|
19
|
+
fullName: caseName,
|
|
20
|
+
occurrenceCounts,
|
|
21
|
+
});
|
|
22
|
+
activeCase = {
|
|
23
|
+
caseId,
|
|
24
|
+
caseName,
|
|
25
|
+
filePath,
|
|
26
|
+
scope: 'runner-case',
|
|
27
|
+
};
|
|
28
|
+
});
|
|
29
|
+
input.afterEach(async () => {
|
|
30
|
+
const current = activeCase;
|
|
31
|
+
if (!current)
|
|
32
|
+
return;
|
|
33
|
+
const metadata = await lifecycle.flushCase(current.caseId);
|
|
34
|
+
if (metadata.length > 0) {
|
|
35
|
+
metadataByCaseId.set(current.caseId, metadata);
|
|
36
|
+
appendJestMetadata(current.caseId, metadata);
|
|
37
|
+
}
|
|
38
|
+
activeCase = null;
|
|
39
|
+
});
|
|
40
|
+
input.afterAll?.(async () => {
|
|
41
|
+
await lifecycle.cleanupRun();
|
|
42
|
+
restore();
|
|
43
|
+
});
|
|
44
|
+
return { restore };
|
|
45
|
+
function restore() {
|
|
46
|
+
if (restored)
|
|
47
|
+
return;
|
|
48
|
+
restored = true;
|
|
49
|
+
activeCase = null;
|
|
50
|
+
resultCapture.unsubscribe();
|
|
51
|
+
contextProvider.restore();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
export function getJestLifecycleMetadata() {
|
|
55
|
+
return new Map(metadataByCaseId);
|
|
56
|
+
}
|
|
57
|
+
export function resetJestLifecycleMetadata() {
|
|
58
|
+
metadataByCaseId.clear();
|
|
59
|
+
}
|
|
60
|
+
function defaultSubscribeToResults(callback) {
|
|
61
|
+
return subscribeToEvalResults(({ result, agent }) => callback({ result, agent }), { owner: 'adapter', key: 'jest-lifecycle' });
|
|
62
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { PathgradeTestMeta } from '@wix/pathgrade/adapter-kit';
|
|
2
|
+
export declare function resolveJestMetadataPath(cwd?: string): string;
|
|
3
|
+
export declare function withJestMetadataEnv(env: NodeJS.ProcessEnv, cwd: string): NodeJS.ProcessEnv;
|
|
4
|
+
export declare function appendJestMetadata(caseId: string, metadata: PathgradeTestMeta[]): void;
|
|
5
|
+
export declare function readJestMetadata(metadataPath?: string): Map<string, PathgradeTestMeta[]>;
|
|
6
|
+
export declare function removeJestMetadata(metadataPath?: string): void;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
const METADATA_ENV = 'PATHGRADE_JEST_METADATA_PATH';
|
|
4
|
+
const FALLBACK_METADATA_FILE = 'jest-metadata.jsonl';
|
|
5
|
+
export function resolveJestMetadataPath(cwd = process.cwd()) {
|
|
6
|
+
return process.env[METADATA_ENV] ?? path.join(cwd, '.pathgrade', FALLBACK_METADATA_FILE);
|
|
7
|
+
}
|
|
8
|
+
export function withJestMetadataEnv(env, cwd) {
|
|
9
|
+
return {
|
|
10
|
+
...env,
|
|
11
|
+
[METADATA_ENV]: env[METADATA_ENV] ?? path.join(cwd, '.pathgrade', `${process.pid}-${Date.now()}-${FALLBACK_METADATA_FILE}`),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export function appendJestMetadata(caseId, metadata) {
|
|
15
|
+
if (metadata.length === 0)
|
|
16
|
+
return;
|
|
17
|
+
const metadataPath = resolveJestMetadataPath();
|
|
18
|
+
fs.mkdirSync(path.dirname(metadataPath), { recursive: true });
|
|
19
|
+
fs.appendFileSync(metadataPath, `${JSON.stringify({ caseId, metadata })}\n`, 'utf8');
|
|
20
|
+
}
|
|
21
|
+
export function readJestMetadata(metadataPath = resolveJestMetadataPath()) {
|
|
22
|
+
if (!fs.existsSync(metadataPath))
|
|
23
|
+
return new Map();
|
|
24
|
+
const metadataByCaseId = new Map();
|
|
25
|
+
for (const line of fs.readFileSync(metadataPath, 'utf8').split('\n')) {
|
|
26
|
+
if (line.trim().length === 0)
|
|
27
|
+
continue;
|
|
28
|
+
const entry = JSON.parse(line);
|
|
29
|
+
metadataByCaseId.set(entry.caseId, entry.metadata);
|
|
30
|
+
}
|
|
31
|
+
return metadataByCaseId;
|
|
32
|
+
}
|
|
33
|
+
export function removeJestMetadata(metadataPath = resolveJestMetadataPath()) {
|
|
34
|
+
fs.rmSync(metadataPath, { force: true });
|
|
35
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
class PathgradeJestReporterBridge {
|
|
2
|
+
async onRunComplete(...args) {
|
|
3
|
+
const mod = await import('./reporter.js');
|
|
4
|
+
const Reporter = mod.default;
|
|
5
|
+
const reporter = new Reporter();
|
|
6
|
+
return await reporter.onRunComplete(...args);
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
module.exports = PathgradeJestReporterBridge;
|