@post-print/agent-test 2.0.10 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/sdk/cli.d.ts +10 -0
- package/dist/sdk/cli.d.ts.map +1 -1
- package/dist/sdk/cli.js +27 -2
- package/dist/sdk/cli.js.map +1 -1
- package/dist/sdk/expect.d.ts +6 -4
- package/dist/sdk/expect.d.ts.map +1 -1
- package/dist/sdk/expect.js +90 -26
- package/dist/sdk/expect.js.map +1 -1
- package/dist/sdk/judge.d.ts +1 -0
- package/dist/sdk/judge.d.ts.map +1 -1
- package/dist/sdk/judge.js +9 -5
- package/dist/sdk/judge.js.map +1 -1
- package/dist/sdk/run-command.d.ts +19 -0
- package/dist/sdk/run-command.d.ts.map +1 -0
- package/dist/sdk/run-command.js +33 -0
- package/dist/sdk/run-command.js.map +1 -0
- package/dist/sdk/runtime.d.ts +3 -0
- package/dist/sdk/runtime.d.ts.map +1 -1
- package/dist/sdk/runtime.js +11 -8
- package/dist/sdk/runtime.js.map +1 -1
- package/dist/sdk/stored-value.d.ts.map +1 -1
- package/dist/sdk/stored-value.js +11 -0
- package/dist/sdk/stored-value.js.map +1 -1
- package/dist/sdk/workspace.d.ts +5 -0
- package/dist/sdk/workspace.d.ts.map +1 -1
- package/dist/sdk/workspace.js +26 -1
- package/dist/sdk/workspace.js.map +1 -1
- package/dist/viewer/client-entry.js +21 -20
- package/dist/viewer/main.js +16 -4
- package/dist/viewer/main.js.map +1 -1
- package/dist/viewer/presentation.d.ts +20 -2
- package/dist/viewer/presentation.d.ts.map +1 -1
- package/dist/viewer/presentation.js +64 -2
- package/dist/viewer/presentation.js.map +1 -1
- package/dist/viewer/run-inbox.d.ts +7 -0
- package/dist/viewer/run-inbox.d.ts.map +1 -0
- package/dist/viewer/run-inbox.js +335 -0
- package/dist/viewer/run-inbox.js.map +1 -0
- package/dist/viewer/styles.d.ts.map +1 -1
- package/dist/viewer/styles.js +5 -0
- package/dist/viewer/styles.js.map +1 -1
- package/dist/viewer/stylex.css +230 -2
- package/dist/viewer/tool-images.d.ts +13 -0
- package/dist/viewer/tool-images.d.ts.map +1 -0
- package/dist/viewer/tool-images.js +77 -0
- package/dist/viewer/tool-images.js.map +1 -0
- package/dist/viewer/v2-client.d.ts.map +1 -1
- package/dist/viewer/v2-client.js +300 -172
- package/dist/viewer/v2-client.js.map +1 -1
- package/dist/viewer/viewer-queries.d.ts +26 -0
- package/dist/viewer/viewer-queries.d.ts.map +1 -0
- package/dist/viewer/viewer-queries.js +31 -0
- package/dist/viewer/viewer-queries.js.map +1 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# @post-print/agent-test
|
|
2
2
|
|
|
3
3
|
<!-- source-of-truth: named agent and judge factories for TypeScript tests -->
|
|
4
|
-
<!-- doc-meta: owner=eng | last-reviewed=2026-10-
|
|
4
|
+
<!-- doc-meta: owner=eng | last-reviewed=2026-10-09 -->
|
|
5
5
|
<!-- review-deps: paths=packages/test/src/*.ts,packages/test/src/**/*.ts,packages/test/package.json -->
|
|
6
6
|
|
|
7
7
|
```ts
|
|
@@ -9,22 +9,23 @@ import { describe, expect, z } from "@post-print/agent-test";
|
|
|
9
9
|
|
|
10
10
|
const test = describe("project answers", ({ agent, judge }) => ({
|
|
11
11
|
coder: agent({ skills: ["./skills/research"] }),
|
|
12
|
-
|
|
13
|
-
prompt: "
|
|
14
|
-
schema: z.object({
|
|
12
|
+
grounding: judge({
|
|
13
|
+
prompt: "Does the answer explain where in the project the owner is recorded?",
|
|
14
|
+
schema: z.object({ grounded: z.boolean(), reason: z.string() }),
|
|
15
15
|
}),
|
|
16
16
|
}));
|
|
17
17
|
|
|
18
|
-
test("answers correctly", async ({ coder,
|
|
19
|
-
const run = await coder.run({ prompt: "Who owns the project?" });
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
test("answers correctly", async ({ coder, grounding }) => {
|
|
19
|
+
const run = await coder.run({ prompt: "Who owns the project, and where is that recorded?" });
|
|
20
|
+
expect(run.output).toContain("Mina");
|
|
21
|
+
const evaluation = await grounding.run({ input: { answer: run.output } });
|
|
22
|
+
expect(evaluation.output.grounded).toBe(true);
|
|
22
23
|
});
|
|
23
24
|
```
|
|
24
25
|
|
|
25
26
|
Set default `agent`, `judge`, and `workspace` directly in `defineConfig`. Harness definitions select providers, models, and auth. Factories only declare resources; each test receives its own executable handles. Every run starts fresh; use `run.continue` for the same task. Compare with ordinary JavaScript and `expect`. Use `expect(actual, "Readable criterion")` to give a derived viewer criterion a concise label, or declare explicit test criteria when they should override derived assertions.
|
|
26
27
|
|
|
27
|
-
Judge input is explicitly selected JSON data. Output is schema-validated and typed. The runner records artifacts and usage, handles cancellation, and owns cleanup. Built-in timeouts are recorded as terminal infrastructure failures, including the configured deadline and partial trace. A new recorded run also recovers dead-owner summaries without requiring viewer navigation. Global skills default to false. No browser is required for CLI execution.
|
|
28
|
+
Use assertions for anything code can decide, `runCommand` to verify the final workspace yourself, and a judge only for semantic questions. Judge input is explicitly selected JSON data. Output is schema-validated and typed. The runner records artifacts and usage, handles cancellation, and owns cleanup. Built-in timeouts are recorded as terminal infrastructure failures, including the configured deadline and partial trace. A new recorded run also recovers dead-owner summaries without requiring viewer navigation. Global skills default to false. A run or judge that names a path outside its workspace fails with `WorkspaceEscapeError`; see [isolation](../../docs/isolation.md#isolation-boundary). No browser is required for CLI execution.
|
|
28
29
|
|
|
29
30
|
See [the SDK guide](../../docs/sdk-v2.md), [tour](../../agent-suites/tour/tour.spec.ts), and [getting started](../../docs/getting-started.md). The old `test.use`, `compare`, `defineJudge`, and run-owned judge API are removed.
|
|
30
31
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
export { WorkspaceEscapeError } from "@post-print/agent-harness";
|
|
1
2
|
export { z } from "zod/v4";
|
|
2
3
|
export { expect } from "./sdk/expect.js";
|
|
3
4
|
export { statistics } from "./sdk/metrics.js";
|
|
4
5
|
export { type ExecutionProgress, reportProgress } from "./sdk/progress.js";
|
|
6
|
+
export { type CommandResult, type RunCommandOptions, runCommand } from "./sdk/run-command.js";
|
|
5
7
|
export { type AgentTestConfig, defineConfig, describe, type SuiteTest } from "./sdk/test.js";
|
|
6
8
|
export type { AgentFixture, AgentSettings, Evaluation, JsonValue, JudgeFixture, JudgeSettings, Run, RunOptions, RunUsage, Statistics, WorkspaceSetup, } from "./sdk/types.js";
|
|
7
9
|
export type { Workspace } from "./sdk/workspace.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,iBAAiB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EAAE,KAAK,eAAe,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAC7F,YAAY,EACX,YAAY,EACZ,aAAa,EACb,UAAU,EACV,SAAS,EACT,YAAY,EACZ,aAAa,EACb,GAAG,EACH,UAAU,EACV,QAAQ,EACR,UAAU,EACV,cAAc,GACd,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,iBAAiB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,iBAAiB,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC9F,OAAO,EAAE,KAAK,eAAe,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAC7F,YAAY,EACX,YAAY,EACZ,aAAa,EACb,UAAU,EACV,SAAS,EACT,YAAY,EACZ,aAAa,EACb,GAAG,EACH,UAAU,EACV,QAAQ,EACR,UAAU,EACV,cAAc,GACd,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
export { WorkspaceEscapeError } from "@post-print/agent-harness";
|
|
1
2
|
export { z } from "zod/v4";
|
|
2
3
|
export { expect } from "./sdk/expect.js";
|
|
3
4
|
export { statistics } from "./sdk/metrics.js";
|
|
4
5
|
export { reportProgress } from "./sdk/progress.js";
|
|
6
|
+
export { runCommand } from "./sdk/run-command.js";
|
|
5
7
|
export { defineConfig, describe } from "./sdk/test.js";
|
|
6
8
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAA0B,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EAAwB,YAAY,EAAE,QAAQ,EAAkB,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAA0B,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EAA8C,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC9F,OAAO,EAAwB,YAAY,EAAE,QAAQ,EAAkB,MAAM,eAAe,CAAC"}
|
package/dist/sdk/cli.d.ts
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
1
|
export { playwrightCli } from "./playwright-cli.js";
|
|
2
2
|
export declare function runSdkCli(args: string[]): Promise<number>;
|
|
3
|
+
/**
|
|
4
|
+
* Closing lines after a recorded run. A terminal user gets the command; a non-interactive
|
|
5
|
+
* caller, usually a coding agent, is also told to ask before starting the viewer.
|
|
6
|
+
*/
|
|
7
|
+
export declare function viewerHint(input: {
|
|
8
|
+
config: string;
|
|
9
|
+
executionId: string;
|
|
10
|
+
interactive: boolean;
|
|
11
|
+
ci: boolean;
|
|
12
|
+
}): string[];
|
|
3
13
|
//# sourceMappingURL=cli.d.ts.map
|
package/dist/sdk/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/sdk/cli.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAwCpD,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/sdk/cli.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAwCpD,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwB/D;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,EAAE,EAAE,OAAO,CAAC;CACZ,GAAG,MAAM,EAAE,CAaX"}
|
package/dist/sdk/cli.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { resolve } from "node:path";
|
|
1
|
+
import { relative, resolve } from "node:path";
|
|
2
2
|
import { startRecordedExecution } from "./execution-runner.js";
|
|
3
3
|
export { playwrightCli } from "./playwright-cli.js";
|
|
4
4
|
function runnerArgument(args, index) {
|
|
@@ -46,10 +46,35 @@ export async function runSdkCli(args) {
|
|
|
46
46
|
const cancel = () => execution.cancel();
|
|
47
47
|
process.once("SIGINT", cancel);
|
|
48
48
|
try {
|
|
49
|
-
|
|
49
|
+
const code = await execution.completed;
|
|
50
|
+
if (!passThrough.includes("--list"))
|
|
51
|
+
for (const line of viewerHint({
|
|
52
|
+
config,
|
|
53
|
+
executionId: execution.id,
|
|
54
|
+
interactive: process.stdout.isTTY === true,
|
|
55
|
+
ci: process.env.CI === "true",
|
|
56
|
+
}))
|
|
57
|
+
process.stdout.write(`${line}\n`);
|
|
58
|
+
return code;
|
|
50
59
|
}
|
|
51
60
|
finally {
|
|
52
61
|
process.removeListener("SIGINT", cancel);
|
|
53
62
|
}
|
|
54
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Closing lines after a recorded run. A terminal user gets the command; a non-interactive
|
|
66
|
+
* caller, usually a coding agent, is also told to ask before starting the viewer.
|
|
67
|
+
*/
|
|
68
|
+
export function viewerHint(input) {
|
|
69
|
+
if (input.ci)
|
|
70
|
+
return [];
|
|
71
|
+
const configPath = relative(process.cwd(), input.config);
|
|
72
|
+
const command = configPath === "agent-test.config.ts"
|
|
73
|
+
? "agent-test viewer"
|
|
74
|
+
: `agent-test viewer --config ${configPath}`;
|
|
75
|
+
const lines = [`View run ${input.executionId.slice(0, 8)}: ${command}`];
|
|
76
|
+
if (!input.interactive)
|
|
77
|
+
lines.push("Agent: ask the user whether to open the results viewer; start it only if they agree.");
|
|
78
|
+
return lines;
|
|
79
|
+
}
|
|
55
80
|
//# sourceMappingURL=cli.js.map
|
package/dist/sdk/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/sdk/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/sdk/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAMpD,SAAS,cAAc,CAAC,IAAc,EAAE,KAAa;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,IAAI;QACzC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,sBAAsB,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;IAC9F,IAAI,KAAK,KAAK,YAAY;QACzB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;IAChF,IAAI,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC;QAClC,OAAO;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM;YAClD,IAAI,EAAE,KAAK,GAAG,CAAC;SACf,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;AACjD,CAAC;AACD,SAAS,eAAe,CAAC,IAAc;IACtC,IAAI,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC7C,IAAI,QAAQ,GAAG,MAAM,CAAC;IACtB,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,GAAI,CAAC;QAC3C,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7C,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;QACtB,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;YACvB,KAAK,QAAQ;gBACZ,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACjC,MAAM;YACP,KAAK,UAAU;gBACd,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC;gBAC1B,MAAM;YACP,KAAK,MAAM;gBACV,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AAC1C,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAc;IAC7C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,WAAW,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC;QAC9C,MAAM;QACN,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,iBAAiB;QAC3B,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC;KACrD,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC;QACvC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAClC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC;gBAC7B,MAAM;gBACN,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;gBAC1C,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM;aAC7B,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC;IACb,CAAC;YAAS,CAAC;QACV,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,KAK1B;IACA,IAAI,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,OAAO,GACZ,UAAU,KAAK,sBAAsB;QACpC,CAAC,CAAC,mBAAmB;QACrB,CAAC,CAAC,8BAA8B,UAAU,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,YAAY,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,KAAK,CAAC,WAAW;QACrB,KAAK,CAAC,IAAI,CACT,sFAAsF,CACtF,CAAC;IACH,OAAO,KAAK,CAAC;AACd,CAAC"}
|
package/dist/sdk/expect.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import type { Run } from "./types.js";
|
|
2
|
+
interface ExpectedCommand {
|
|
3
|
+
command: string | RegExp;
|
|
4
|
+
exitCode?: number;
|
|
5
|
+
}
|
|
2
6
|
export declare const expect: import("@playwright/test").Expect<{
|
|
3
|
-
toHaveExecutedCommand(run: Run, expected: {
|
|
4
|
-
command: string | RegExp;
|
|
5
|
-
exitCode?: number;
|
|
6
|
-
}): {
|
|
7
|
+
toHaveExecutedCommand(run: Run, expected: ExpectedCommand): {
|
|
7
8
|
pass: boolean;
|
|
8
9
|
message: () => string;
|
|
9
10
|
};
|
|
@@ -28,4 +29,5 @@ export declare const expect: import("@playwright/test").Expect<{
|
|
|
28
29
|
message: () => string;
|
|
29
30
|
};
|
|
30
31
|
}>;
|
|
32
|
+
export {};
|
|
31
33
|
//# sourceMappingURL=expect.d.ts.map
|
package/dist/sdk/expect.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../src/sdk/expect.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../src/sdk/expect.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAiHtC,UAAU,eAAe;IACxB,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAgHD,eAAO,MAAM,MAAM;+BAnGS,GAAG,YAAY,eAAe;;;;0BAoBnC,GAAG,YAAY,MAAM,GAAG,MAAM;;;;kCAOtB,GAAG,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE;;;;wBAU5C,GAAG,QAAQ,MAAM;;;;4BAmBb,GAAG,QAAQ,MAAM;;;;4BAOjB,GAAG,QAAQ,MAAM;;;;EAyCxC,CAAC"}
|
package/dist/sdk/expect.js
CHANGED
|
@@ -1,58 +1,130 @@
|
|
|
1
|
-
import { basename, resolve } from "node:path";
|
|
1
|
+
import { basename, relative, resolve, sep } from "node:path";
|
|
2
2
|
import { expect as base } from "@playwright/test";
|
|
3
|
+
import { shellSegments } from "@post-print/agent-harness";
|
|
3
4
|
import { consumeCriterionOperations } from "./criterion-provenance.js";
|
|
4
5
|
const FILE_PROTOCOL = /^file:\/\//;
|
|
5
6
|
const COMMAND_TOOL = /shell|bash|terminal|exec|command/i;
|
|
6
7
|
const READ_TOOL = /read/i;
|
|
7
|
-
const
|
|
8
|
-
const
|
|
9
|
-
const
|
|
8
|
+
const SEARCH_TOOL = /^(grep|glob)$/i;
|
|
9
|
+
const GLOB_CHARS = /[*?[]/;
|
|
10
|
+
const PATH_ARG_KEYS = ["path", "file_path", "filePath", "target_file", "uri", "pattern"];
|
|
10
11
|
const FILE_READING_COMMANDS = new Set([
|
|
11
12
|
"awk",
|
|
13
|
+
"bat",
|
|
12
14
|
"cat",
|
|
15
|
+
"cut",
|
|
13
16
|
"grep",
|
|
14
17
|
"head",
|
|
15
18
|
"less",
|
|
16
19
|
"more",
|
|
20
|
+
"nl",
|
|
17
21
|
"rg",
|
|
18
22
|
"sed",
|
|
19
23
|
"tail",
|
|
24
|
+
"xxd",
|
|
20
25
|
]);
|
|
21
26
|
const matches = (actual, expected) => typeof expected === "string"
|
|
22
27
|
? actual === expected
|
|
23
28
|
: new RegExp(expected.source, expected.flags.replace("g", "").replace("y", "")).test(actual);
|
|
29
|
+
function workspacePath(run, value) {
|
|
30
|
+
return resolve(run.workspace.root, value.replace(FILE_PROTOCOL, ""));
|
|
31
|
+
}
|
|
24
32
|
function samePath(run, actual, expected) {
|
|
25
|
-
return (
|
|
26
|
-
|
|
33
|
+
return workspacePath(run, actual) === workspacePath(run, expected);
|
|
34
|
+
}
|
|
35
|
+
/** True when `actual` names `expected` itself or a directory that contains it. */
|
|
36
|
+
function coversPath(run, actual, expected) {
|
|
37
|
+
const rel = relative(workspacePath(run, actual), workspacePath(run, expected));
|
|
38
|
+
return rel === "" || !(rel === ".." || rel.startsWith(`..${sep}`) || resolve(rel) === rel);
|
|
39
|
+
}
|
|
40
|
+
function globRegExp(glob) {
|
|
41
|
+
const source = glob
|
|
42
|
+
.split("**")
|
|
43
|
+
.map((part) => part
|
|
44
|
+
.replace(/[.+^${}()|\\]/g, "\\$&")
|
|
45
|
+
.replaceAll("*", "[^/]*")
|
|
46
|
+
.replaceAll("?", "[^/]"))
|
|
47
|
+
.join(".*");
|
|
48
|
+
return new RegExp(`^${source}$`);
|
|
49
|
+
}
|
|
50
|
+
function globMatches(run, glob, expected) {
|
|
51
|
+
return globRegExp(workspacePath(run, glob)).test(workspacePath(run, expected));
|
|
52
|
+
}
|
|
53
|
+
/** A token that may expose `expected`: the path, an enclosing directory, or a matching glob. */
|
|
54
|
+
function mayExpose(run, token, expected) {
|
|
55
|
+
return GLOB_CHARS.test(token)
|
|
56
|
+
? globMatches(run, token, expected)
|
|
57
|
+
: coversPath(run, token, expected);
|
|
58
|
+
}
|
|
59
|
+
function commandText(call) {
|
|
60
|
+
return String(call.args?.command ?? call.args?.cmd ?? "");
|
|
27
61
|
}
|
|
28
62
|
function requireTools(run) {
|
|
29
63
|
if (!run.capabilities.toolCalls)
|
|
30
64
|
throw new Error("Tool-call observation unavailable for this adapter");
|
|
31
65
|
}
|
|
32
66
|
function shellReadsPath(run, command, path) {
|
|
33
|
-
|
|
34
|
-
return payload.split(SHELL_SEGMENTS).some((segment) => {
|
|
35
|
-
const tokens = segment.trim().split(SHELL_SEPARATORS).filter(Boolean);
|
|
67
|
+
return shellSegments(command).some(({ tokens }) => {
|
|
36
68
|
const executable = tokens[0];
|
|
37
69
|
return Boolean(executable &&
|
|
38
70
|
FILE_READING_COMMANDS.has(basename(executable)) &&
|
|
39
71
|
tokens.slice(1).some((token) => samePath(run, token, path)));
|
|
40
72
|
});
|
|
41
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* The shell exit code belongs to a segment only when it runs last and every earlier
|
|
76
|
+
* segment is joined by `&&` or `;`. Pipes and `||` hide or replace its status.
|
|
77
|
+
*/
|
|
78
|
+
function ownsExitCode(segments, index) {
|
|
79
|
+
if (index !== segments.length - 1)
|
|
80
|
+
return false;
|
|
81
|
+
return segments.slice(0, index).every(({ next }) => next === "&&" || next === ";");
|
|
82
|
+
}
|
|
83
|
+
function segmentText(segment) {
|
|
84
|
+
return segment.tokens.join(" ");
|
|
85
|
+
}
|
|
86
|
+
function commandMatches(call, expected) {
|
|
87
|
+
const raw = commandText(call);
|
|
88
|
+
const segments = shellSegments(raw);
|
|
89
|
+
const index = segments.findIndex((segment) => matches(segmentText(segment), expected));
|
|
90
|
+
if (index >= 0)
|
|
91
|
+
return { call, ownsExitCode: ownsExitCode(segments, index) };
|
|
92
|
+
return matches(raw, expected) ? { call, ownsExitCode: segments.length <= 1 } : undefined;
|
|
93
|
+
}
|
|
94
|
+
function accessesPath(run, call, path) {
|
|
95
|
+
const args = call.args ?? {};
|
|
96
|
+
const named = PATH_ARG_KEYS.map((key) => args[key]).filter((value) => typeof value === "string");
|
|
97
|
+
if (SEARCH_TOOL.test(call.name) && !args.path)
|
|
98
|
+
return true;
|
|
99
|
+
if (named.some((value) => mayExpose(run, value, path)))
|
|
100
|
+
return true;
|
|
101
|
+
if (!COMMAND_TOOL.test(call.name))
|
|
102
|
+
return false;
|
|
103
|
+
return shellSegments(commandText(call)).some(({ tokens }) => tokens.slice(1).some((token) => !token.startsWith("-") && mayExpose(run, token, path)));
|
|
104
|
+
}
|
|
42
105
|
const outcome = (pass, description) => ({ pass, message: () => description });
|
|
106
|
+
function executedCommandMessage(expected, calls, unattributed) {
|
|
107
|
+
const exit = expected.exitCode === undefined ? "" : ` with exit code ${expected.exitCode}`;
|
|
108
|
+
const note = unattributed
|
|
109
|
+
? `\n${unattributed} matching command(s) ran in a pipeline, after ||, or before another command, so the recorded exit code is not theirs.`
|
|
110
|
+
: "";
|
|
111
|
+
return `Expected command ${String(expected.command)}${exit}${note}\nObserved: ${JSON.stringify(calls)}`;
|
|
112
|
+
}
|
|
43
113
|
const extendedExpect = base.extend({
|
|
44
114
|
toHaveExecutedCommand(run, expected) {
|
|
45
115
|
requireTools(run);
|
|
46
|
-
|
|
116
|
+
const wantsExit = expected.exitCode !== undefined;
|
|
117
|
+
if (wantsExit && !run.capabilities.commandExitCodes)
|
|
47
118
|
throw new Error("Command exit-code evidence unavailable for this adapter");
|
|
48
119
|
const calls = run.toolCalls.filter((call) => COMMAND_TOOL.test(call.name));
|
|
49
|
-
const matching = calls
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
120
|
+
const matching = calls
|
|
121
|
+
.map((call) => commandMatches(call, expected.command))
|
|
122
|
+
.filter((match) => match !== undefined);
|
|
123
|
+
const attributable = wantsExit ? matching.filter((match) => match.ownsExitCode) : matching;
|
|
124
|
+
const found = attributable.some((match) => !wantsExit || match.call.exitCode === expected.exitCode);
|
|
125
|
+
if (!found && wantsExit && attributable.some((match) => match.call.exitCode === undefined))
|
|
54
126
|
throw new Error("Command exit-code evidence unavailable");
|
|
55
|
-
return outcome(found,
|
|
127
|
+
return outcome(found, executedCommandMessage(expected, calls, matching.length - attributable.length));
|
|
56
128
|
},
|
|
57
129
|
toHaveCalledTool(run, expected) {
|
|
58
130
|
requireTools(run);
|
|
@@ -71,12 +143,7 @@ const extendedExpect = base.extend({
|
|
|
71
143
|
throw new Error("Successful file-read evidence unavailable for this adapter");
|
|
72
144
|
const readCalls = run.toolCalls.filter((call) => READ_TOOL.test(call.name) &&
|
|
73
145
|
Object.values(call.args ?? {}).some((value) => typeof value === "string" && samePath(run, value, path)));
|
|
74
|
-
const shellCalls = run.toolCalls.filter((call) =>
|
|
75
|
-
if (!COMMAND_TOOL.test(call.name))
|
|
76
|
-
return false;
|
|
77
|
-
const command = String(call.args?.command ?? call.args?.cmd ?? "");
|
|
78
|
-
return shellReadsPath(run, command, path);
|
|
79
|
-
});
|
|
146
|
+
const shellCalls = run.toolCalls.filter((call) => COMMAND_TOOL.test(call.name) && shellReadsPath(run, commandText(call), path));
|
|
80
147
|
const calls = [...readCalls, ...shellCalls];
|
|
81
148
|
const found = calls.some((call) => call.succeeded === true && call.result !== undefined);
|
|
82
149
|
if (!found && calls.some((call) => call.result === undefined || call.succeeded === undefined))
|
|
@@ -85,10 +152,7 @@ const extendedExpect = base.extend({
|
|
|
85
152
|
},
|
|
86
153
|
toHaveAccessedPath(run, path) {
|
|
87
154
|
requireTools(run);
|
|
88
|
-
return outcome(run.toolCalls.some((call) =>
|
|
89
|
-
(samePath(run, value, path) ||
|
|
90
|
-
(COMMAND_TOOL.test(call.name) &&
|
|
91
|
-
value.split(SHELL_SEPARATORS).some((token) => samePath(run, token, path)))))), `Expected attempted access to ${path}`);
|
|
155
|
+
return outcome(run.toolCalls.some((call) => accessesPath(run, call, path)), `Expected possible access to ${path} (exact path, enclosing directory, matching glob, or an unscoped search)`);
|
|
92
156
|
},
|
|
93
157
|
toHaveModifiedPath(run, path) {
|
|
94
158
|
return outcome(run.workspace.changedPaths.includes(path), `Expected resulting modification to ${path}; changed: ${run.workspace.changedPaths.join(", ")}`);
|
package/dist/sdk/expect.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expect.js","sourceRoot":"","sources":["../../src/sdk/expect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"expect.js","sourceRoot":"","sources":["../../src/sdk/expect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,MAAM,IAAI,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAqB,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AAGvE,MAAM,aAAa,GAAG,YAAY,CAAC;AACnC,MAAM,YAAY,GAAG,mCAAmC,CAAC;AACzD,MAAM,SAAS,GAAG,OAAO,CAAC;AAC1B,MAAM,WAAW,GAAG,gBAAgB,CAAC;AACrC,MAAM,UAAU,GAAG,OAAO,CAAC;AAC3B,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;AACzF,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC;IACrC,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,KAAK;CACL,CAAC,CAAC;AAGH,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,QAAyB,EAAE,EAAE,CAC7D,OAAO,QAAQ,KAAK,QAAQ;IAC3B,CAAC,CAAC,MAAM,KAAK,QAAQ;IACrB,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/F,SAAS,aAAa,CAAC,GAAQ,EAAE,KAAa;IAC7C,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC;AACD,SAAS,QAAQ,CAAC,GAAQ,EAAE,MAAc,EAAE,QAAgB;IAC3D,OAAO,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACpE,CAAC;AACD,kFAAkF;AAClF,SAAS,UAAU,CAAC,GAAQ,EAAE,MAAc,EAAE,QAAgB;IAC7D,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC/E,OAAO,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC;AAC5F,CAAC;AACD,SAAS,UAAU,CAAC,IAAY;IAC/B,MAAM,MAAM,GAAG,IAAI;SACjB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACb,IAAI;SACF,OAAO,CAAC,gBAAgB,EAAE,MAAM,CAAC;SACjC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC;SACxB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CACzB;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,OAAO,IAAI,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;AAClC,CAAC;AACD,SAAS,WAAW,CAAC,GAAQ,EAAE,IAAY,EAAE,QAAgB;IAC5D,OAAO,UAAU,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;AAChF,CAAC;AACD,gGAAgG;AAChG,SAAS,SAAS,CAAC,GAAQ,EAAE,KAAa,EAAE,QAAgB;IAC3D,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC5B,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC;QACnC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;AACrC,CAAC;AACD,SAAS,WAAW,CAAC,IAAc;IAClC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;AAC3D,CAAC;AACD,SAAS,YAAY,CAAC,GAAQ;IAC7B,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS;QAC9B,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACxE,CAAC;AACD,SAAS,cAAc,CAAC,GAAQ,EAAE,OAAe,EAAE,IAAY;IAC9D,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,OAAO,OAAO,CACb,UAAU;YACT,qBAAqB,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC/C,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAC5D,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC;AACD;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAwB,EAAE,KAAa;IAC5D,IAAI,KAAK,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC;AACpF,CAAC;AACD,SAAS,WAAW,CAAC,OAAqB;IACzC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAKD,SAAS,cAAc,CAAC,IAAc,EAAE,QAAyB;IAChE,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvF,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;IAC7E,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1F,CAAC;AACD,SAAS,YAAY,CAAC,GAAQ,EAAE,IAAc,EAAE,IAAY;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CACzD,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CACrD,CAAC;IACF,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAC3D,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAC3D,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CACtF,CAAC;AACH,CAAC;AACD,MAAM,OAAO,GAAG,CAAC,IAAa,EAAE,WAAmB,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;AAK/F,SAAS,sBAAsB,CAC9B,QAAyB,EACzB,KAAiB,EACjB,YAAoB;IAEpB,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAC3F,MAAM,IAAI,GAAG,YAAY;QACxB,CAAC,CAAC,KAAK,YAAY,uHAAuH;QAC1I,CAAC,CAAC,EAAE,CAAC;IACN,OAAO,oBAAoB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,IAAI,GAAG,IAAI,eAAe,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;AACzG,CAAC;AACD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,qBAAqB,CAAC,GAAQ,EAAE,QAAyB;QACxD,YAAY,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC;QAClD,IAAI,SAAS,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,gBAAgB;YAClD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC5E,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,KAAK;aACpB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;aACrD,MAAM,CAAC,CAAC,KAAK,EAAyB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAChE,MAAM,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC3F,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAClE,CAAC;QACF,IAAI,CAAC,KAAK,IAAI,SAAS,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC;YACzF,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC3D,OAAO,OAAO,CACb,KAAK,EACL,sBAAsB,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAC9E,CAAC;IACH,CAAC;IACD,gBAAgB,CAAC,GAAQ,EAAE,QAAyB;QACnD,YAAY,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,OAAO,CACb,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAC1D,iBAAiB,MAAM,CAAC,QAAQ,CAAC,cAAc,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClG,CAAC;IACH,CAAC;IACD,wBAAwB,CAAC,GAAQ,EAAE,QAA6B;QAC/D,YAAY,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,SAAS;YAC/B,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAAE,KAAK,EAAE,CAAC;QAC7E,OAAO,OAAO,CACb,KAAK,KAAK,QAAQ,CAAC,MAAM,EACzB,0BAA0B,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAC5D,CAAC;IACH,CAAC;IACD,cAAc,CAAC,GAAQ,EAAE,IAAY;QACpC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS;YAC9B,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC/E,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CACrC,CAAC,IAAI,EAAE,EAAE,CACR,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACzB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAClE,CACF,CAAC;QACF,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CACtC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CACtF,CAAC;QACF,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,UAAU,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QACzF,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC;YAC5F,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,EAAE,CAAC,CAAC;QACvE,OAAO,OAAO,CAAC,KAAK,EAAE,+BAA+B,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,kBAAkB,CAAC,GAAQ,EAAE,IAAY;QACxC,YAAY,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,OAAO,CACb,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,EAC3D,+BAA+B,IAAI,0EAA0E,CAC7G,CAAC;IACH,CAAC;IACD,kBAAkB,CAAC,GAAQ,EAAE,IAAY;QACxC,OAAO,OAAO,CACb,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EACzC,sCAAsC,IAAI,cAAc,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/F,CAAC;IACH,CAAC;CACD,CAAC,CAAC;AAEH,SAAS,qBAAqB,CAAC,MAAe,EAAE,SAAkB;IACjE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,0BAA0B,EAAE,CAAC,CAAC;IACrD,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrD,IACC,OAAO,SAAS,KAAK,QAAQ;QAC7B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAE3E,OAAO;IACR,KAAK,MAAM,KAAK,IAAI,MAAM;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,gBAAgB,IAAI,CAAC,SAAS,CAAC;YAC9B,KAAK;YACL,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,EAAE,SAAS,EAAE;SACpB,CAAC,IAAI,CACN,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACxC,OAAO,OAAO,CACb,KAAK;QACJ,OAAO,KAAK,KAAK,QAAQ;QACzB,IAAI,IAAI,KAAK;QACb,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ;QAC5B,OAAO,IAAI,KAAK,CACjB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,cAAc,EAAE;IAC/C,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa;QACnC,qBAAqB,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IACtD,CAAC;CACD,CAAC,CAAC"}
|
package/dist/sdk/judge.d.ts
CHANGED
package/dist/sdk/judge.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/sdk/judge.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,eAAe,
|
|
1
|
+
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/sdk/judge.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,eAAe,EAKpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAG3B,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMvE,wBAAgB,eAAe,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAEzF;AACD,wBAAgB,cAAc,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM,CAevD;AACD,MAAM,WAAW,iBAAiB,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO;IACrD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,eAAe,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAC3B,KAAK,EAAE,SAAS,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,SAAS,CAAC;QAAC,UAAU,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,OAAO,CAAA;SAAE,CAAA;KAAE,KAAK,IAAI,CAAC;CACjG;AAiBD,wBAAsB,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EACjD,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAC3B,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAiDlC"}
|
package/dist/sdk/judge.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { mkdir,
|
|
1
|
+
import { mkdir, realpath } from "node:fs/promises";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { createAgentSession, } from "@post-print/agent-harness";
|
|
3
|
+
import { assertInsideWorkspace, createAgentSession, createEmptySealedWorkspace, } from "@post-print/agent-harness";
|
|
4
4
|
import { z } from "zod/v4";
|
|
5
5
|
import { validateSkills } from "./definitions.js";
|
|
6
6
|
import { recordedFailure } from "./failure.js";
|
|
@@ -39,8 +39,10 @@ async function evaluationPrompt(request, workspace) {
|
|
|
39
39
|
export async function evaluate(request) {
|
|
40
40
|
request.signal.throwIfAborted();
|
|
41
41
|
const id = request.id ?? crypto.randomUUID(), directory = join(request.outputDir, id);
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
await mkdir(directory, { recursive: true });
|
|
43
|
+
// The reviewer works outside the run artifacts and the caller checkout.
|
|
44
|
+
const sealed = await createEmptySealedWorkspace();
|
|
45
|
+
const workspace = await realpath(sealed.path);
|
|
44
46
|
let session;
|
|
45
47
|
try {
|
|
46
48
|
const prepared = await evaluationPrompt(request, workspace);
|
|
@@ -50,12 +52,14 @@ export async function evaluate(request) {
|
|
|
50
52
|
workspace,
|
|
51
53
|
signal: request.signal,
|
|
52
54
|
readOnly: true,
|
|
55
|
+
protectedPaths: request.protectedPaths,
|
|
53
56
|
});
|
|
54
57
|
request.onStart?.({
|
|
55
58
|
input: prepared.input,
|
|
56
59
|
evaluation: { prompt: request.settings.prompt, schema: prepared.schema },
|
|
57
60
|
});
|
|
58
61
|
const trace = await session.run(prepared.prompt);
|
|
62
|
+
assertInsideWorkspace(trace, workspace);
|
|
59
63
|
const response = trace.messages
|
|
60
64
|
.filter((message) => message.role === "assistant")
|
|
61
65
|
.map((message) => message.content)
|
|
@@ -80,7 +84,7 @@ export async function evaluate(request) {
|
|
|
80
84
|
await session?.close();
|
|
81
85
|
}
|
|
82
86
|
finally {
|
|
83
|
-
await
|
|
87
|
+
await sealed.cleanup();
|
|
84
88
|
}
|
|
85
89
|
}
|
|
86
90
|
}
|
package/dist/sdk/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/sdk/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/sdk/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAEN,qBAAqB,EACrB,kBAAkB,EAClB,0BAA0B,GAE1B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEtE,MAAM,UAAU,GAAG,kBAAkB,CAAC;AACtC,MAAM,WAAW,GAAG,SAAS,CAAC;AAC9B,MAAM,UAAU,eAAe,CAAsB,IAAY,EAAE,MAAS;IAC3E,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACxF,CAAC;AACD,MAAM,UAAU,cAAc,CAAC,KAAgB;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAClD,IACC,KAAK,KAAK,SAAS;YACnB,OAAO,KAAK,KAAK,UAAU;YAC3B,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,KAAK,KAAK,QAAQ;YAEzB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC5D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC7D,OAAO,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IACH,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC;AACb,CAAC;AAaD,KAAK,UAAU,gBAAgB,CAC9B,OAA6B,EAC7B,SAAiB;IAEjB,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,WAAW,CACzB,+JAA+J,OAAO,CAAC,QAAQ,CAAC,MAAM,qBAAqB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,KAAK,EAAE,EACrP,OAAO,CACP,CAAC;IACF,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,OAAO;QACtC,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC1E,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC9D,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC7B,OAA6B;IAE7B,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,EAC3C,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACzC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,wEAAwE;IACxE,MAAM,MAAM,GAAG,MAAM,0BAA0B,EAAE,CAAC;IAClD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,OAAmC,CAAC;IACxC,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC3D,OAAO,GAAG,MAAM,kBAAkB,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,UAAU;YACzB,SAAS;YACT,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,OAAO,CAAC,cAAc;SACtC,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,UAAU,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;SACxE,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjD,qBAAqB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ;aAC7B,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC;aACjD,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;aACjC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG;YACd,EAAE;YACF,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC1D,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;YAC5B,QAAQ,EAAE,SAAS;SACnB,CAAC;QACF,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC;QACvE,MAAM,KAAK,CAAC;IACb,CAAC;YAAS,CAAC;QACV,IAAI,CAAC;YACJ,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;QACxB,CAAC;gBAAS,CAAC;YACV,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACxB,CAAC;IACF,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface CommandResult {
|
|
2
|
+
exitCode: number;
|
|
3
|
+
stdout: string;
|
|
4
|
+
stderr: string;
|
|
5
|
+
}
|
|
6
|
+
export interface RunCommandOptions {
|
|
7
|
+
/** Milliseconds before the command is killed and the call rejects. Default 120000. */
|
|
8
|
+
timeoutMs?: number;
|
|
9
|
+
signal?: AbortSignal;
|
|
10
|
+
env?: NodeJS.ProcessEnv;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Run one program with an argument list (never a shell string) in `directory`.
|
|
14
|
+
* Tests use it to verify a run's final workspace copy themselves rather than trusting
|
|
15
|
+
* commands the agent reported. A nonzero exit resolves; a timeout, abort, or missing
|
|
16
|
+
* program rejects.
|
|
17
|
+
*/
|
|
18
|
+
export declare function runCommand(directory: string, argv: readonly [string, ...string[]], options?: RunCommandOptions): Promise<CommandResult>;
|
|
19
|
+
//# sourceMappingURL=run-command.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-command.d.ts","sourceRoot":"","sources":["../../src/sdk/run-command.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CACf;AACD,MAAM,WAAW,iBAAiB;IACjC,sFAAsF;IACtF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACxB;AASD;;;;;GAKG;AACH,wBAAgB,UAAU,CACzB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,EACpC,OAAO,GAAE,iBAAsB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAyBxB"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
3
|
+
const MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
|
|
4
|
+
function failureExitCode(error) {
|
|
5
|
+
return typeof error.code === "number" ? error.code : undefined;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Run one program with an argument list (never a shell string) in `directory`.
|
|
9
|
+
* Tests use it to verify a run's final workspace copy themselves rather than trusting
|
|
10
|
+
* commands the agent reported. A nonzero exit resolves; a timeout, abort, or missing
|
|
11
|
+
* program rejects.
|
|
12
|
+
*/
|
|
13
|
+
export function runCommand(directory, argv, options = {}) {
|
|
14
|
+
const [program, ...args] = argv;
|
|
15
|
+
const timeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
16
|
+
return new Promise((resolve, reject) => {
|
|
17
|
+
execFile(program, args, {
|
|
18
|
+
cwd: directory,
|
|
19
|
+
timeout,
|
|
20
|
+
signal: options.signal,
|
|
21
|
+
env: options.env,
|
|
22
|
+
maxBuffer: MAX_OUTPUT_BYTES,
|
|
23
|
+
}, (error, stdout, stderr) => {
|
|
24
|
+
if (!error)
|
|
25
|
+
return resolve({ exitCode: 0, stdout, stderr });
|
|
26
|
+
const exitCode = failureExitCode(error);
|
|
27
|
+
if (exitCode === undefined || error.killed)
|
|
28
|
+
return reject(new Error(`Command failed to complete: ${argv.join(" ")}: ${error.message}`));
|
|
29
|
+
resolve({ exitCode, stdout, stderr });
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=run-command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-command.js","sourceRoot":"","sources":["../../src/sdk/run-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAclE,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1C,SAAS,eAAe,CAAC,KAAoB;IAC5C,OAAO,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAChE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CACzB,SAAiB,EACjB,IAAoC,EACpC,OAAO,GAAsB,EAAE;IAE/B,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAChC,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IACxD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,QAAQ,CACP,OAAO,EACP,IAAI,EACJ;YACC,GAAG,EAAE,SAAS;YACd,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,gBAAgB;SAC3B,EACD,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YACzB,IAAI,CAAC,KAAK;gBAAE,OAAO,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5D,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM;gBACzC,OAAO,MAAM,CACZ,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAC5E,CAAC;YACH,OAAO,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACvC,CAAC,CACD,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/sdk/runtime.d.ts
CHANGED
|
@@ -21,9 +21,12 @@ export declare class TestRuntime {
|
|
|
21
21
|
private readonly controller;
|
|
22
22
|
private readonly snapshotStorage;
|
|
23
23
|
private readonly sourceEvidenceStorage;
|
|
24
|
+
private protectedPaths?;
|
|
24
25
|
private nextInvocationIndex;
|
|
25
26
|
private closed;
|
|
26
27
|
constructor(options: RuntimeOptions);
|
|
28
|
+
/** Paths every session in this test is denied: the checkout holding the suite. */
|
|
29
|
+
private checkoutPaths;
|
|
27
30
|
private get signal();
|
|
28
31
|
private track;
|
|
29
32
|
agent(name: string, settings: AgentSettings, preparation?: readonly WorkspaceSetup[]): AgentFixture;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/sdk/runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,eAAe,EAKpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAW3B,OAAO,KAAK,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,aAAa,EAEb,cAAc,EACd,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/sdk/runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,eAAe,EAKpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAW3B,OAAO,KAAK,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,aAAa,EAEb,cAAc,EACd,MAAM,YAAY,CAAC;AAapB,MAAM,WAAW,cAAc;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;CAC3E;AAuDD,qBAAa,WAAW;IASX,QAAQ,CAAC,OAAO,EAAE,cAAc;IAR5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgE;IAC1F,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA+B;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAoB;IACpD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAA0B;IAChE,OAAO,CAAC,cAAc,CAAC,CAAoB;IAC3C,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,MAAM,CAAS;IACvB,YAAqB,OAAO,EAAE,cAAc,EAAI;IAChD,kFAAkF;IAClF,OAAO,CAAC,aAAa;IAIrB,OAAO,KAAK,MAAM,GAEjB;IACD,OAAO,CAAC,KAAK;IAUb,KAAK,CACJ,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,aAAa,EACvB,WAAW,GAAE,SAAS,cAAc,EAAO,GACzC,YAAY,CAad;IACD,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAkD9F;YACa,KAAK;IA2CnB,OAAO,CAAC,YAAY;YAgBN,GAAG;IA+CjB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,eAAe;YAIT,cAAc;IAiBtB,KAAK,kBAuBV;CACD"}
|