@post-print/agent-test 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -0
- package/dist/cli.d.ts +24 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +58 -8
- package/dist/cli.js.map +1 -1
- package/dist/debug-bundle.d.ts +66 -0
- package/dist/debug-bundle.d.ts.map +1 -0
- package/dist/debug-bundle.js +241 -0
- package/dist/debug-bundle.js.map +1 -0
- package/dist/expect.d.ts +9 -0
- package/dist/expect.d.ts.map +1 -1
- package/dist/expect.js +125 -46
- package/dist/expect.js.map +1 -1
- package/dist/failures.d.ts +4 -0
- package/dist/failures.d.ts.map +1 -0
- package/dist/failures.js +7 -0
- package/dist/failures.js.map +1 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/live-isolation.d.ts +7 -9
- package/dist/live-isolation.d.ts.map +1 -1
- package/dist/live-isolation.js +37 -10
- package/dist/live-isolation.js.map +1 -1
- package/dist/load-suite.d.ts.map +1 -1
- package/dist/load-suite.js +18 -7
- package/dist/load-suite.js.map +1 -1
- package/dist/mcp-config.d.ts +5 -0
- package/dist/mcp-config.d.ts.map +1 -0
- package/dist/mcp-config.js +75 -0
- package/dist/mcp-config.js.map +1 -0
- package/dist/record-trace.d.ts +8 -0
- package/dist/record-trace.d.ts.map +1 -1
- package/dist/record-trace.js +23 -4
- package/dist/record-trace.js.map +1 -1
- package/dist/run-suite.d.ts +6 -0
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +193 -61
- package/dist/run-suite.js.map +1 -1
- package/dist/theme.d.ts +9 -1
- package/dist/theme.d.ts.map +1 -1
- package/dist/theme.js +30 -6
- package/dist/theme.js.map +1 -1
- package/dist/types.d.ts +30 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/load-suite.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { isMcpServersMap } from "./mcp-config.js";
|
|
2
3
|
export async function loadSuiteFile(path) {
|
|
3
4
|
const raw = await readFile(path, "utf8");
|
|
4
5
|
const parsed = JSON.parse(raw);
|
|
@@ -12,18 +13,28 @@ function isScenario(value) {
|
|
|
12
13
|
return false;
|
|
13
14
|
}
|
|
14
15
|
const scenario = value;
|
|
15
|
-
|
|
16
|
-
typeof scenario.prompt
|
|
17
|
-
typeof scenario.rubric
|
|
18
|
-
scenario.rubric
|
|
16
|
+
if (typeof scenario.name !== "string" ||
|
|
17
|
+
typeof scenario.prompt !== "string" ||
|
|
18
|
+
typeof scenario.rubric !== "object" ||
|
|
19
|
+
scenario.rubric === null) {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
if (scenario.mcpServers !== undefined && !isMcpServersMap(scenario.mcpServers)) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
return true;
|
|
19
26
|
}
|
|
20
27
|
function isSuiteFile(value) {
|
|
21
28
|
if (typeof value !== "object" || value === null) {
|
|
22
29
|
return false;
|
|
23
30
|
}
|
|
24
31
|
const suite = value;
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
32
|
+
if (typeof suite.name !== "string" || !Array.isArray(suite.scenarios)) {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
if (suite.defaults?.mcpServers !== undefined && !isMcpServersMap(suite.defaults.mcpServers)) {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
return suite.scenarios.every(isScenario);
|
|
28
39
|
}
|
|
29
40
|
//# sourceMappingURL=load-suite.js.map
|
package/dist/load-suite.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"load-suite.js","sourceRoot":"","sources":["../src/load-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"load-suite.js","sourceRoot":"","sources":["../src/load-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACjD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,KAAsB,CAAC;IACxC,IACC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ;QACjC,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ;QACnC,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ;QACnC,QAAQ,CAAC,MAAM,KAAK,IAAI,EACvB,CAAC;QACF,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAChF,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACjD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAuB,CAAC;IACtC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QACvE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,EAAE,UAAU,KAAK,SAAS,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7F,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { McpServerConfig } from "@post-print/agent-harness";
|
|
2
|
+
/** Validate a suite/scenario mcpServers map (shape only; env expansion happens at run time). */
|
|
3
|
+
export declare function isMcpServersMap(value: unknown): value is Record<string, McpServerConfig>;
|
|
4
|
+
export declare function isMcpServerConfig(value: unknown): value is McpServerConfig;
|
|
5
|
+
//# sourceMappingURL=mcp-config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mcp-config.d.ts","sourceRoot":"","sources":["../src/mcp-config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAEjE,gGAAgG;AAChG,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAaxF;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,eAAe,CAiC1E"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/** Validate a suite/scenario mcpServers map (shape only; env expansion happens at run time). */
|
|
2
|
+
export function isMcpServersMap(value) {
|
|
3
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
4
|
+
return false;
|
|
5
|
+
}
|
|
6
|
+
for (const [name, config] of Object.entries(value)) {
|
|
7
|
+
if (typeof name !== "string" || name.length === 0) {
|
|
8
|
+
return false;
|
|
9
|
+
}
|
|
10
|
+
if (!isMcpServerConfig(config)) {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
export function isMcpServerConfig(value) {
|
|
17
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
const config = value;
|
|
21
|
+
if (typeof config.command === "string" && config.command.length > 0) {
|
|
22
|
+
if (config.type !== undefined && config.type !== "stdio") {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
if (config.args !== undefined && !isStringArray(config.args)) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (config.env !== undefined && !isStringRecord(config.env)) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
if (config.cwd !== undefined && typeof config.cwd !== "string") {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
if (typeof config.url === "string" && config.url.length > 0) {
|
|
37
|
+
if (config.type !== undefined && config.type !== "http" && config.type !== "sse") {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
if (config.headers !== undefined && !isStringRecord(config.headers)) {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
if (config.auth !== undefined && !isMcpAuth(config.auth)) {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
function isStringArray(value) {
|
|
51
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
52
|
+
}
|
|
53
|
+
function isStringRecord(value) {
|
|
54
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
return Object.values(value).every((item) => typeof item === "string");
|
|
58
|
+
}
|
|
59
|
+
function isMcpAuth(value) {
|
|
60
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
const auth = value;
|
|
64
|
+
if (typeof auth.CLIENT_ID !== "string") {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
if (auth.CLIENT_SECRET !== undefined && typeof auth.CLIENT_SECRET !== "string") {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
if (auth.scopes !== undefined && !isStringArray(auth.scopes)) {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=mcp-config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mcp-config.js","sourceRoot":"","sources":["../src/mcp-config.ts"],"names":[],"mappings":"AAEA,gGAAgG;AAChG,MAAM,UAAU,eAAe,CAAC,KAAc;IAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrE,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1D,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7D,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YAChE,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAClF,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACpC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAChF,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9D,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC"}
|
package/dist/record-trace.d.ts
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import type { AgentTrace } from "@post-print/agent-harness";
|
|
2
2
|
import type { AssertionFailure } from "./types.js";
|
|
3
3
|
export declare const LIVE_STAGING_DIR_NAME = "agent-spec";
|
|
4
|
+
/** Override the sessions parent (default: `$TMPDIR/agent-spec`). Used by `--debug-dir`. */
|
|
5
|
+
export declare function setLiveStagingRootOverride(root: string | undefined): void;
|
|
6
|
+
export declare function getLiveStagingRootOverride(): string | undefined;
|
|
7
|
+
/**
|
|
8
|
+
* Stable filesystem basename for a scenario. Readable slug + short hash so
|
|
9
|
+
* distinct names that normalize alike (e.g. "Foo Bar" / "foo-bar") never collide.
|
|
10
|
+
*/
|
|
11
|
+
export declare function scenarioArtifactSlug(name: string): string;
|
|
4
12
|
export declare function getLiveStagingRoot(): string;
|
|
5
13
|
export declare function getLiveStagingSessionRoot(sessionId: string): string;
|
|
6
14
|
/** Staging JSON path for a live scenario trace (matches recordTrace layout). */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"record-trace.d.ts","sourceRoot":"","sources":["../src/record-trace.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"record-trace.d.ts","sourceRoot":"","sources":["../src/record-trace.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAG5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,eAAO,MAAM,qBAAqB,eAAe,CAAC;AAIlD,2FAA2F;AAC3F,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAEzE;AAED,wBAAgB,0BAA0B,IAAI,MAAM,GAAG,SAAS,CAE/D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASzD;AAED,wBAAgB,kBAAkB,IAAI,MAAM,CAK3C;AAED,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEnE;AAMD,gFAAgF;AAChF,wBAAgB,mBAAmB,CAClC,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GAClB,MAAM,CAMR;AAED,gGAAgG;AAChG,wBAAgB,wBAAwB,CACvC,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GAClB,MAAM,CAMR;AAED,8FAA8F;AAC9F,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGvE;AAED,wFAAwF;AACxF,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAQpF;AAED,wFAAwF;AACxF,wBAAgB,oBAAoB,CACnC,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GAClB,MAAM,CAMR;AAED,MAAM,WAAW,yBAAyB;IACzC,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,0BAA0B,IAAI,MAAM,CAEnD;AAED,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,CAAC;AAEtD,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,iBAAiB,CAAC;CACxB;AAED,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,6FAA6F;AAC7F,wBAAgB,oBAAoB,CACnC,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,cAAc,EAAE,OAAO,EACvB,OAAO,EAAE,2BAA2B,GAClC,qBAAqB,GAAG,SAAS,CAmBnC;AAED,0EAA0E;AAC1E,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAkBxF;AAED,mFAAmF;AACnF,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAGxE;AAED,8FAA8F;AAC9F,wBAAsB,kBAAkB,CACvC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,yBAAyB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAGf;AAED,0FAA0F;AAC1F,wBAAsB,iBAAiB,CACtC,IAAI,EAAE,MAAM,GACV,OAAO,CAAC,yBAAyB,GAAG,SAAS,CAAC,CAMhD;AAED,6DAA6D;AAC7D,wBAAsB,qBAAqB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9E;AAED,mFAAmF;AACnF,wBAAsB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuBrF"}
|
package/dist/record-trace.js
CHANGED
|
@@ -1,23 +1,41 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
2
3
|
import { tmpdir } from "node:os";
|
|
3
4
|
import { dirname, join, resolve } from "node:path";
|
|
4
5
|
import { enrichTrace } from "@post-print/agent-harness";
|
|
5
6
|
export const LIVE_STAGING_DIR_NAME = "agent-spec";
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
let liveStagingRootOverride;
|
|
8
|
+
/** Override the sessions parent (default: `$TMPDIR/agent-spec`). Used by `--debug-dir`. */
|
|
9
|
+
export function setLiveStagingRootOverride(root) {
|
|
10
|
+
liveStagingRootOverride = root?.trim() ? root : undefined;
|
|
11
|
+
}
|
|
12
|
+
export function getLiveStagingRootOverride() {
|
|
13
|
+
return liveStagingRootOverride;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Stable filesystem basename for a scenario. Readable slug + short hash so
|
|
17
|
+
* distinct names that normalize alike (e.g. "Foo Bar" / "foo-bar") never collide.
|
|
18
|
+
*/
|
|
19
|
+
export function scenarioArtifactSlug(name) {
|
|
20
|
+
const base = name
|
|
8
21
|
.toLowerCase()
|
|
9
22
|
.replace(/[^a-z0-9]+/g, "-")
|
|
10
23
|
.replace(/^-+|-+$/g, "")
|
|
11
|
-
.slice(0,
|
|
24
|
+
.slice(0, 40) || "scenario";
|
|
25
|
+
const hash = createHash("sha256").update(name).digest("hex").slice(0, 8);
|
|
26
|
+
return `${base}-${hash}`;
|
|
12
27
|
}
|
|
13
28
|
export function getLiveStagingRoot() {
|
|
29
|
+
if (liveStagingRootOverride) {
|
|
30
|
+
return join(liveStagingRootOverride, "sessions");
|
|
31
|
+
}
|
|
14
32
|
return join(tmpdir(), LIVE_STAGING_DIR_NAME, "sessions");
|
|
15
33
|
}
|
|
16
34
|
export function getLiveStagingSessionRoot(sessionId) {
|
|
17
35
|
return join(getLiveStagingRoot(), sessionId);
|
|
18
36
|
}
|
|
19
37
|
function stagingScenarioBasename(scenarioName) {
|
|
20
|
-
return
|
|
38
|
+
return scenarioArtifactSlug(scenarioName);
|
|
21
39
|
}
|
|
22
40
|
/** Staging JSON path for a live scenario trace (matches recordTrace layout). */
|
|
23
41
|
export function getStagingTracePath(stagingSessionId, suiteName, scenarioName) {
|
|
@@ -81,6 +99,7 @@ export async function recordTrace(outputPath, trace) {
|
|
|
81
99
|
artifacts: enriched.artifacts,
|
|
82
100
|
routing: enriched.routing,
|
|
83
101
|
skillsInvoked: enriched.skillsInvoked,
|
|
102
|
+
assistantTextBeforeTools: enriched.assistantTextBeforeTools,
|
|
84
103
|
judgeVerdicts: enriched.judgeVerdicts,
|
|
85
104
|
};
|
|
86
105
|
await mkdir(dirname(outputPath), { recursive: true });
|
package/dist/record-trace.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"record-trace.js","sourceRoot":"","sources":["../src/record-trace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGnD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAIxD,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAY,CAAC;AAElD,SAAS,
|
|
1
|
+
{"version":3,"file":"record-trace.js","sourceRoot":"","sources":["../src/record-trace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGnD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAIxD,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAY,CAAC;AAElD,IAAI,uBAA2C,CAAC;AAEhD,2FAA2F;AAC3F,MAAM,UAAU,0BAA0B,CAAC,IAAwB;IAClE,uBAAuB,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,0BAA0B;IACzC,OAAO,uBAAuB,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAChD,MAAM,IAAI,GACT,IAAI;SACF,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC;IAC9B,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,OAAO,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,kBAAkB;IACjC,IAAI,uBAAuB,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,qBAAqB,EAAE,UAAU,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,SAAiB;IAC1D,OAAO,IAAI,CAAC,kBAAkB,EAAE,EAAE,SAAS,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,uBAAuB,CAAC,YAAoB;IACpD,OAAO,oBAAoB,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,mBAAmB,CAClC,gBAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,OAAO,IAAI,CACV,yBAAyB,CAAC,gBAAgB,CAAC,EAC3C,SAAS,EACT,GAAG,uBAAuB,CAAC,YAAY,CAAC,OAAO,CAC/C,CAAC;AACH,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,wBAAwB,CACvC,gBAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,OAAO,IAAI,CACV,yBAAyB,CAAC,gBAAgB,CAAC,EAC3C,SAAS,EACT,GAAG,uBAAuB,CAAC,YAAY,CAAC,cAAc,CACtD,CAAC;AACH,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY;IACvD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY;IACtD,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,oBAAoB,CACnC,gBAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,OAAO,IAAI,CACV,yBAAyB,CAAC,gBAAgB,CAAC,EAC3C,SAAS,EACT,GAAG,uBAAuB,CAAC,YAAY,CAAC,cAAc,CACtD,CAAC;AACH,CAAC;AAQD,MAAM,UAAU,0BAA0B;IACzC,OAAO,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AACvC,CAAC;AAcD,6FAA6F;AAC7F,MAAM,UAAU,oBAAoB,CACnC,SAAiB,EACjB,YAAoB,EACpB,WAA+B,EAC/B,cAAuB,EACvB,OAAoC;IAEpC,IAAI,cAAc,EAAE,CAAC;QACpB,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO;YACN,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC;YAC5C,IAAI,EAAE,SAAS;SACf,CAAC;IACH,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC/B,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,OAAO;QACN,IAAI,EAAE,mBAAmB,CAAC,OAAO,CAAC,gBAAgB,EAAE,SAAS,EAAE,YAAY,CAAC;QAC5E,IAAI,EAAE,SAAS;KACf,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB,EAAE,KAAiB;IACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG;QACf,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,wBAAwB,EAAE,QAAQ,CAAC,wBAAwB;QAC3D,aAAa,EAAE,QAAQ,CAAC,aAAa;KACrC,CAAC;IAEF,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7E,OAAO,UAAU,CAAC;AACnB,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAe,CAAC;IACnE,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,IAAY,EACZ,MAAiC;IAEjC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AACvE,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,IAAY;IAEZ,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAA8B,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,WAAmB;IAC9D,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,QAAgB;IACjE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACJ,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;QAC5E,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC;YAC1B,MAAM,EAAE,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC"}
|
package/dist/run-suite.d.ts
CHANGED
|
@@ -27,6 +27,10 @@ export interface RunSuiteOptions {
|
|
|
27
27
|
timeoutMs?: number;
|
|
28
28
|
/** Allow AskQuestion-style tools in live runs (default false). */
|
|
29
29
|
allowUserInput?: boolean;
|
|
30
|
+
/** Evidence-rich failures + on-disk debug bundles. */
|
|
31
|
+
debug?: boolean;
|
|
32
|
+
/** Override staging sessions parent (from --debug-dir). */
|
|
33
|
+
debugDir?: string;
|
|
30
34
|
}
|
|
31
35
|
/** Live-only mode hint from rubric — not part of the user scenario prompt. */
|
|
32
36
|
export declare function outputContractForRubric(rubric: ScenarioRubric): RoutingContract | undefined;
|
|
@@ -45,6 +49,8 @@ export declare function runAllSuites(options: {
|
|
|
45
49
|
keepRecordings?: boolean;
|
|
46
50
|
timeoutMs?: number;
|
|
47
51
|
allowUserInput?: boolean;
|
|
52
|
+
debug?: boolean;
|
|
53
|
+
debugDir?: string;
|
|
48
54
|
}): Promise<SuiteRunReport[]>;
|
|
49
55
|
export { discoverSuites } from "./discover-suites.js";
|
|
50
56
|
//# sourceMappingURL=run-suite.d.ts.map
|
package/dist/run-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-suite.d.ts","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-suite.d.ts","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACX,SAAS,EAIT,eAAe,EAEf,MAAM,2BAA2B,CAAC;AAgDnC,OAAO,KAAK,EAMX,cAAc,EACd,cAAc,EACd,MAAM,YAAY,CAAC;AA8BpB,0FAA0F;AAC1F,wBAAgB,sBAAsB,IAAI,OAAO,CAEhD;AAED,oFAAoF;AACpF,wBAAgB,uBAAuB,IAAI,IAAI,CA6B9C;AASD,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mDAAmD;IACnD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2FAA2F;IAC3F,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,yEAAyE;IACzE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,8EAA8E;IAC9E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,sDAAsD;IACtD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,8EAA8E;AAC9E,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,cAAc,GAAG,eAAe,GAAG,SAAS,CAQ3F;AA4LD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAahF;AAgiBD,wBAAsB,YAAY,CAAC,OAAO,EAAE;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAiC5B;AAED,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC"}
|