@post-print/agent-test 1.1.9 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -66
- package/dist/cli.d.ts +2 -64
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +26 -691
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +1 -1
- package/dist/compare-scenario.d.ts.map +1 -1
- package/dist/compare-scenario.js +72 -75
- package/dist/compare-scenario.js.map +1 -1
- package/dist/failures.d.ts +5 -1
- package/dist/failures.d.ts.map +1 -1
- package/dist/failures.js +1 -1
- package/dist/failures.js.map +1 -1
- package/dist/host-login.d.ts +1 -26
- package/dist/host-login.d.ts.map +1 -1
- package/dist/host-login.js +1 -92
- package/dist/host-login.js.map +1 -1
- package/dist/index.d.ts +6 -28
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -26
- package/dist/index.js.map +1 -1
- package/dist/scenario-story.d.ts.map +1 -1
- package/dist/scenario-story.js +57 -39
- package/dist/scenario-story.js.map +1 -1
- package/dist/sdk/cli.d.ts +3 -0
- package/dist/sdk/cli.d.ts.map +1 -0
- package/dist/sdk/cli.js +20 -0
- package/dist/sdk/cli.js.map +1 -0
- package/dist/sdk/definitions.d.ts +22 -0
- package/dist/sdk/definitions.d.ts.map +1 -0
- package/dist/sdk/definitions.js +58 -0
- package/dist/sdk/definitions.js.map +1 -0
- package/dist/sdk/expect.d.ts +31 -0
- package/dist/sdk/expect.d.ts.map +1 -0
- package/dist/sdk/expect.js +68 -0
- package/dist/sdk/expect.js.map +1 -0
- package/dist/sdk/judge.d.ts +16 -0
- package/dist/sdk/judge.d.ts.map +1 -0
- package/dist/sdk/judge.js +82 -0
- package/dist/sdk/judge.js.map +1 -0
- package/dist/sdk/metrics.d.ts +3 -0
- package/dist/sdk/metrics.d.ts.map +1 -0
- package/dist/sdk/metrics.js +24 -0
- package/dist/sdk/metrics.js.map +1 -0
- package/dist/sdk/reporter.d.ts +12 -0
- package/dist/sdk/reporter.d.ts.map +1 -0
- package/dist/sdk/reporter.js +58 -0
- package/dist/sdk/reporter.js.map +1 -0
- package/dist/sdk/runtime.d.ts +33 -0
- package/dist/sdk/runtime.d.ts.map +1 -0
- package/dist/sdk/runtime.js +188 -0
- package/dist/sdk/runtime.js.map +1 -0
- package/dist/sdk/test.d.ts +27 -0
- package/dist/sdk/test.d.ts.map +1 -0
- package/dist/sdk/test.js +92 -0
- package/dist/sdk/test.js.map +1 -0
- package/dist/sdk/types.d.ts +75 -0
- package/dist/sdk/types.d.ts.map +1 -0
- package/dist/sdk/types.js +2 -0
- package/dist/sdk/types.js.map +1 -0
- package/dist/sdk/usage.d.ts +10 -0
- package/dist/sdk/usage.d.ts.map +1 -0
- package/dist/sdk/usage.js +13 -0
- package/dist/sdk/usage.js.map +1 -0
- package/dist/sdk/viewer-results.d.ts +43 -0
- package/dist/sdk/viewer-results.d.ts.map +1 -0
- package/dist/sdk/viewer-results.js +126 -0
- package/dist/sdk/viewer-results.js.map +1 -0
- package/dist/sdk/viewer.d.ts +11 -0
- package/dist/sdk/viewer.d.ts.map +1 -0
- package/dist/sdk/viewer.js +160 -0
- package/dist/sdk/viewer.js.map +1 -0
- package/dist/sdk/workspace.d.ts +33 -0
- package/dist/sdk/workspace.d.ts.map +1 -0
- package/dist/sdk/workspace.js +100 -0
- package/dist/sdk/workspace.js.map +1 -0
- package/dist/types.d.ts +4 -40
- package/dist/types.d.ts.map +1 -1
- package/dist/viewer/catalog.d.ts +2 -8
- package/dist/viewer/catalog.d.ts.map +1 -1
- package/dist/viewer/catalog.js +23 -128
- package/dist/viewer/catalog.js.map +1 -1
- package/dist/viewer/client.d.ts.map +1 -1
- package/dist/viewer/client.js +12 -12
- package/dist/viewer/client.js.map +1 -1
- package/dist/viewer/run-controller.d.ts +4 -2
- package/dist/viewer/run-controller.d.ts.map +1 -1
- package/dist/viewer/run-controller.js +297 -248
- package/dist/viewer/run-controller.js.map +1 -1
- package/dist/viewer/server.d.ts +4 -3
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +149 -112
- package/dist/viewer/server.js.map +1 -1
- package/dist/viewer/styles.d.ts +2 -0
- package/dist/viewer/styles.d.ts.map +1 -0
- package/dist/viewer/styles.js +444 -0
- package/dist/viewer/styles.js.map +1 -0
- package/dist/worker-pool.d.ts +5 -1
- package/dist/worker-pool.d.ts.map +1 -1
- package/dist/worker-pool.js +1 -1
- package/dist/worker-pool.js.map +1 -1
- package/package.json +7 -7
- package/dist/check.d.ts +0 -38
- package/dist/check.d.ts.map +0 -1
- package/dist/check.js +0 -169
- package/dist/check.js.map +0 -1
- package/dist/command-allowlist.d.ts +0 -9
- package/dist/command-allowlist.d.ts.map +0 -1
- package/dist/command-allowlist.js +0 -29
- package/dist/command-allowlist.js.map +0 -1
- package/dist/debug-bundle.d.ts +0 -81
- package/dist/debug-bundle.d.ts.map +0 -1
- package/dist/debug-bundle.js +0 -392
- package/dist/debug-bundle.js.map +0 -1
- package/dist/discover-suites.d.ts +0 -2
- package/dist/discover-suites.d.ts.map +0 -1
- package/dist/discover-suites.js +0 -19
- package/dist/discover-suites.js.map +0 -1
- package/dist/doctor.d.ts +0 -24
- package/dist/doctor.d.ts.map +0 -1
- package/dist/doctor.js +0 -272
- package/dist/doctor.js.map +0 -1
- package/dist/expect.d.ts +0 -59
- package/dist/expect.d.ts.map +0 -1
- package/dist/expect.js +0 -518
- package/dist/expect.js.map +0 -1
- package/dist/host-log.d.ts +0 -17
- package/dist/host-log.d.ts.map +0 -1
- package/dist/host-log.js +0 -154
- package/dist/host-log.js.map +0 -1
- package/dist/hosts.d.ts +0 -22
- package/dist/hosts.d.ts.map +0 -1
- package/dist/hosts.js +0 -84
- package/dist/hosts.js.map +0 -1
- package/dist/html-report.d.ts +0 -18
- package/dist/html-report.d.ts.map +0 -1
- package/dist/html-report.js +0 -1276
- package/dist/html-report.js.map +0 -1
- package/dist/live-isolation.d.ts +0 -82
- package/dist/live-isolation.d.ts.map +0 -1
- package/dist/live-isolation.js +0 -399
- package/dist/live-isolation.js.map +0 -1
- package/dist/live-timeout.d.ts +0 -11
- package/dist/live-timeout.d.ts.map +0 -1
- package/dist/live-timeout.js +0 -29
- package/dist/live-timeout.js.map +0 -1
- package/dist/load-adapters.d.ts +0 -13
- package/dist/load-adapters.d.ts.map +0 -1
- package/dist/load-adapters.js +0 -76
- package/dist/load-adapters.js.map +0 -1
- package/dist/load-suite.d.ts +0 -22
- package/dist/load-suite.d.ts.map +0 -1
- package/dist/load-suite.js +0 -154
- package/dist/load-suite.js.map +0 -1
- package/dist/mcp-config.d.ts +0 -13
- package/dist/mcp-config.d.ts.map +0 -1
- package/dist/mcp-config.js +0 -121
- package/dist/mcp-config.js.map +0 -1
- package/dist/preflight.d.ts +0 -3
- package/dist/preflight.d.ts.map +0 -1
- package/dist/preflight.js +0 -17
- package/dist/preflight.js.map +0 -1
- package/dist/progress.d.ts +0 -24
- package/dist/progress.d.ts.map +0 -1
- package/dist/progress.js +0 -111
- package/dist/progress.js.map +0 -1
- package/dist/record-trace.d.ts +0 -64
- package/dist/record-trace.d.ts.map +0 -1
- package/dist/record-trace.js +0 -124
- package/dist/record-trace.js.map +0 -1
- package/dist/report-preview-server.d.ts +0 -2
- package/dist/report-preview-server.d.ts.map +0 -1
- package/dist/report-preview-server.js +0 -24
- package/dist/report-preview-server.js.map +0 -1
- package/dist/report-preview.d.ts +0 -35
- package/dist/report-preview.d.ts.map +0 -1
- package/dist/report-preview.js +0 -205
- package/dist/report-preview.js.map +0 -1
- package/dist/run-suite.d.ts +0 -139
- package/dist/run-suite.d.ts.map +0 -1
- package/dist/run-suite.js +0 -1688
- package/dist/run-suite.js.map +0 -1
- package/dist/scenario-finalizer.d.ts +0 -27
- package/dist/scenario-finalizer.d.ts.map +0 -1
- package/dist/scenario-finalizer.js +0 -39
- package/dist/scenario-finalizer.js.map +0 -1
- package/dist/scenario-retry.d.ts +0 -15
- package/dist/scenario-retry.d.ts.map +0 -1
- package/dist/scenario-retry.js +0 -52
- package/dist/scenario-retry.js.map +0 -1
- package/dist/scenario-seed.d.ts +0 -13
- package/dist/scenario-seed.d.ts.map +0 -1
- package/dist/scenario-seed.js +0 -72
- package/dist/scenario-seed.js.map +0 -1
- package/dist/suite-summary.d.ts +0 -16
- package/dist/suite-summary.d.ts.map +0 -1
- package/dist/suite-summary.js +0 -218
- package/dist/suite-summary.js.map +0 -1
- package/dist/theme.d.ts +0 -96
- package/dist/theme.d.ts.map +0 -1
- package/dist/theme.js +0 -446
- package/dist/theme.js.map +0 -1
- package/dist/validate-seeds.d.ts +0 -20
- package/dist/validate-seeds.d.ts.map +0 -1
- package/dist/validate-seeds.js +0 -72
- package/dist/validate-seeds.js.map +0 -1
- package/dist/validate-suite.d.ts +0 -24
- package/dist/validate-suite.d.ts.map +0 -1
- package/dist/validate-suite.js +0 -542
- package/dist/validate-suite.js.map +0 -1
- package/dist/viewer/cli.d.ts +0 -17
- package/dist/viewer/cli.d.ts.map +0 -1
- package/dist/viewer/cli.js +0 -38
- package/dist/viewer/cli.js.map +0 -1
- package/dist/viewer/compare-metrics.d.ts +0 -30
- package/dist/viewer/compare-metrics.d.ts.map +0 -1
- package/dist/viewer/compare-metrics.js +0 -101
- package/dist/viewer/compare-metrics.js.map +0 -1
- package/dist/viewer/context-files.d.ts +0 -8
- package/dist/viewer/context-files.d.ts.map +0 -1
- package/dist/viewer/context-files.js +0 -57
- package/dist/viewer/context-files.js.map +0 -1
- package/dist/viewer/emit.d.ts +0 -7
- package/dist/viewer/emit.d.ts.map +0 -1
- package/dist/viewer/emit.js +0 -25
- package/dist/viewer/emit.js.map +0 -1
- package/dist/viewer/live-runner.d.ts +0 -27
- package/dist/viewer/live-runner.d.ts.map +0 -1
- package/dist/viewer/live-runner.js +0 -166
- package/dist/viewer/live-runner.js.map +0 -1
- package/dist/viewer/session-server.d.ts +0 -2
- package/dist/viewer/session-server.d.ts.map +0 -1
- package/dist/viewer/session-server.js +0 -71
- package/dist/viewer/session-server.js.map +0 -1
- package/dist/warnings.d.ts +0 -3
- package/dist/warnings.d.ts.map +0 -1
- package/dist/warnings.js +0 -13
- package/dist/warnings.js.map +0 -1
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { AgentCapabilities, AgentDefinition, AgentOptions, AgentTrace } from "@post-print/agent-harness";
|
|
2
|
+
import type { z } from "zod/v4";
|
|
3
|
+
import type { StartingContext, Workspace, WorkspaceSnapshot } from "./workspace.js";
|
|
4
|
+
export interface AgentSettings extends AgentOptions {
|
|
5
|
+
agent?: AgentDefinition;
|
|
6
|
+
workspace?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface RunOptions extends Omit<AgentSettings, "agent"> {
|
|
9
|
+
prompt: string;
|
|
10
|
+
}
|
|
11
|
+
export interface JudgeSettings<S extends z.ZodType = z.ZodType> extends Omit<AgentSettings, "workspace"> {
|
|
12
|
+
prompt: string;
|
|
13
|
+
schema: S;
|
|
14
|
+
}
|
|
15
|
+
export type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
16
|
+
[key: string]: JsonValue;
|
|
17
|
+
};
|
|
18
|
+
export interface RunUsage {
|
|
19
|
+
tokens: {
|
|
20
|
+
input?: number;
|
|
21
|
+
output?: number;
|
|
22
|
+
total?: number;
|
|
23
|
+
cacheRead?: number;
|
|
24
|
+
cacheWrite?: number;
|
|
25
|
+
reasoning?: number;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export interface Run {
|
|
29
|
+
id: string;
|
|
30
|
+
name: string;
|
|
31
|
+
prompt: string;
|
|
32
|
+
output: string;
|
|
33
|
+
trace: AgentTrace;
|
|
34
|
+
conversation: AgentTrace;
|
|
35
|
+
toolCalls: AgentTrace["toolCalls"];
|
|
36
|
+
durationMs: number;
|
|
37
|
+
usage: RunUsage;
|
|
38
|
+
capabilities: AgentCapabilities;
|
|
39
|
+
startingContext: StartingContext;
|
|
40
|
+
artifact: string;
|
|
41
|
+
workspace: {
|
|
42
|
+
root: string;
|
|
43
|
+
initial: WorkspaceSnapshot;
|
|
44
|
+
final: WorkspaceSnapshot;
|
|
45
|
+
changedPaths: string[];
|
|
46
|
+
};
|
|
47
|
+
continue(options: {
|
|
48
|
+
prompt: string;
|
|
49
|
+
}): Promise<Run>;
|
|
50
|
+
}
|
|
51
|
+
export interface Evaluation<T = unknown> {
|
|
52
|
+
id: string;
|
|
53
|
+
name: string;
|
|
54
|
+
output: T;
|
|
55
|
+
usage: RunUsage;
|
|
56
|
+
artifact: string;
|
|
57
|
+
}
|
|
58
|
+
export type WorkspaceSetup = (workspace: Workspace) => void | Promise<void>;
|
|
59
|
+
export interface AgentFixture {
|
|
60
|
+
setup(prepare: WorkspaceSetup): AgentFixture;
|
|
61
|
+
run(options: RunOptions): Promise<Run>;
|
|
62
|
+
}
|
|
63
|
+
export interface JudgeFixture<T> {
|
|
64
|
+
run(options: {
|
|
65
|
+
input: JsonValue;
|
|
66
|
+
}): Promise<Evaluation<T>>;
|
|
67
|
+
}
|
|
68
|
+
export interface Statistics {
|
|
69
|
+
available: boolean;
|
|
70
|
+
count: number;
|
|
71
|
+
mean: number;
|
|
72
|
+
min: number;
|
|
73
|
+
max: number;
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/sdk/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,UAAU,EACV,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACpF,MAAM,WAAW,aAAc,SAAQ,YAAY;IAClD,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AACD,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC;IAC/D,MAAM,EAAE,MAAM,CAAC;CACf;AACD,MAAM,WAAW,aAAa,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAC7D,SAAQ,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,CAAC,CAAC;CACV;AACD,MAAM,MAAM,SAAS,GAClB,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAChC,MAAM,WAAW,QAAQ;IACxB,MAAM,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACF;AACD,MAAM,WAAW,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,UAAU,CAAC;IAClB,YAAY,EAAE,UAAU,CAAC;IACzB,SAAS,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,QAAQ,CAAC;IAChB,YAAY,EAAE,iBAAiB,CAAC;IAChC,eAAe,EAAE,eAAe,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,iBAAiB,CAAC;QAC3B,KAAK,EAAE,iBAAiB,CAAC;QACzB,YAAY,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;CACpD;AACD,MAAM,WAAW,UAAU,CAAC,CAAC,GAAG,OAAO;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,CAAC,CAAC;IACV,KAAK,EAAE,QAAQ,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CACjB;AACD,MAAM,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAC5E,MAAM,WAAW,YAAY;IAC5B,KAAK,CAAC,OAAO,EAAE,cAAc,GAAG,YAAY,CAAC;IAC7C,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;CACvC;AACD,MAAM,WAAW,YAAY,CAAC,CAAC;IAC9B,GAAG,CAAC,OAAO,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;CAC3D;AACD,MAAM,WAAW,UAAU;IAC1B,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACZ"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/sdk/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RunUsage } from "./types.js";
|
|
2
|
+
export declare function runUsage(usage?: {
|
|
3
|
+
inputTokens?: number;
|
|
4
|
+
outputTokens?: number;
|
|
5
|
+
totalTokens?: number;
|
|
6
|
+
cacheReadTokens?: number;
|
|
7
|
+
cacheWriteTokens?: number;
|
|
8
|
+
reasoningTokens?: number;
|
|
9
|
+
}): RunUsage;
|
|
10
|
+
//# sourceMappingURL=usage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"usage.d.ts","sourceRoot":"","sources":["../../src/sdk/usage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,wBAAgB,QAAQ,CAAC,KAAK,CAAC,EAAE;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB,GAAG,QAAQ,CAWX"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export function runUsage(usage) {
|
|
2
|
+
return {
|
|
3
|
+
tokens: {
|
|
4
|
+
input: usage?.inputTokens,
|
|
5
|
+
output: usage?.outputTokens,
|
|
6
|
+
total: usage?.totalTokens,
|
|
7
|
+
cacheRead: usage?.cacheReadTokens,
|
|
8
|
+
cacheWrite: usage?.cacheWriteTokens,
|
|
9
|
+
reasoning: usage?.reasoningTokens,
|
|
10
|
+
},
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=usage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"usage.js","sourceRoot":"","sources":["../../src/sdk/usage.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,QAAQ,CAAC,KAOxB;IACA,OAAO;QACN,MAAM,EAAE;YACP,KAAK,EAAE,KAAK,EAAE,WAAW;YACzB,MAAM,EAAE,KAAK,EAAE,YAAY;YAC3B,KAAK,EAAE,KAAK,EAAE,WAAW;YACzB,SAAS,EAAE,KAAK,EAAE,eAAe;YACjC,UAAU,EAAE,KAAK,EAAE,gBAAgB;YACnC,SAAS,EAAE,KAAK,EAAE,eAAe;SACjC;KACD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { ViewerJob } from "../viewer/catalog.js";
|
|
2
|
+
import type { ViewerEvent } from "../viewer/events.js";
|
|
3
|
+
export interface TestEntry {
|
|
4
|
+
id: string;
|
|
5
|
+
title: string;
|
|
6
|
+
listFile: string;
|
|
7
|
+
file: string;
|
|
8
|
+
project: string;
|
|
9
|
+
}
|
|
10
|
+
interface RunEvent {
|
|
11
|
+
type: string;
|
|
12
|
+
runId: string;
|
|
13
|
+
value: unknown;
|
|
14
|
+
}
|
|
15
|
+
export interface WireEvent {
|
|
16
|
+
type: string;
|
|
17
|
+
tests?: TestEntry[];
|
|
18
|
+
event?: RunEvent;
|
|
19
|
+
passed?: boolean;
|
|
20
|
+
skipped?: boolean;
|
|
21
|
+
durationMs?: number;
|
|
22
|
+
errors?: string[];
|
|
23
|
+
message?: string;
|
|
24
|
+
}
|
|
25
|
+
/** One test result owns its captured runs, checks, and independent grades. */
|
|
26
|
+
export declare class ViewerTestResult {
|
|
27
|
+
private readonly emit;
|
|
28
|
+
private readonly runs;
|
|
29
|
+
private readonly evaluations;
|
|
30
|
+
private result?;
|
|
31
|
+
private readonly envelope;
|
|
32
|
+
constructor(job: ViewerJob, emit: (event: ViewerEvent) => void);
|
|
33
|
+
start(): void;
|
|
34
|
+
receive(wire: WireEvent): void;
|
|
35
|
+
private capture;
|
|
36
|
+
private agentEvent;
|
|
37
|
+
private section;
|
|
38
|
+
private completedResult;
|
|
39
|
+
finish(code: number, cancelled: boolean): void;
|
|
40
|
+
private missingResult;
|
|
41
|
+
}
|
|
42
|
+
export {};
|
|
43
|
+
//# sourceMappingURL=viewer-results.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"viewer-results.d.ts","sourceRoot":"","sources":["../../src/sdk/viewer-results.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CAChB;AACD,UAAU,QAAQ;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;CACf;AACD,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AASD,8EAA8E;AAC9E,qBAAa,gBAAgB;IAO3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IANtB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0B;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,MAAM,CAAC,CAAiB;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAC1B,YACC,GAAG,EAAE,SAAS,EACG,IAAI,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,EAGnD;IACD,KAAK,SAEJ;IACD,OAAO,CAAC,IAAI,EAAE,SAAS,QAKtB;IACD,OAAO,CAAC,OAAO;IAoBf,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,eAAe;IAgCvB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,QAWtC;IACD,OAAO,CAAC,aAAa;CAgBrB"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
function combinedTrace(runs) {
|
|
2
|
+
return {
|
|
3
|
+
messages: runs.flatMap((run) => run.trace.messages),
|
|
4
|
+
toolCalls: runs.flatMap((run) => run.toolCalls),
|
|
5
|
+
shellCommands: runs.flatMap((run) => run.trace.shellCommands),
|
|
6
|
+
artifacts: {},
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
/** One test result owns its captured runs, checks, and independent grades. */
|
|
10
|
+
export class ViewerTestResult {
|
|
11
|
+
emit;
|
|
12
|
+
runs = new Map();
|
|
13
|
+
evaluations = [];
|
|
14
|
+
result;
|
|
15
|
+
envelope;
|
|
16
|
+
constructor(job, emit) {
|
|
17
|
+
this.emit = emit;
|
|
18
|
+
this.envelope = { suite: job.suite, scenario: job.scenario, host: job.host };
|
|
19
|
+
}
|
|
20
|
+
start() {
|
|
21
|
+
this.emit({ ...this.envelope, type: "cell_started" });
|
|
22
|
+
}
|
|
23
|
+
receive(wire) {
|
|
24
|
+
if (wire.event)
|
|
25
|
+
this.capture(wire.event);
|
|
26
|
+
if (wire.type === "error")
|
|
27
|
+
this.emit({ ...this.envelope, type: "status", text: wire.message ?? "Runner error" });
|
|
28
|
+
if (wire.type === "result")
|
|
29
|
+
this.result = this.completedResult(wire);
|
|
30
|
+
}
|
|
31
|
+
capture(event) {
|
|
32
|
+
switch (event.type) {
|
|
33
|
+
case "start":
|
|
34
|
+
this.emit({
|
|
35
|
+
...this.envelope,
|
|
36
|
+
type: "prompt",
|
|
37
|
+
text: event.value.prompt,
|
|
38
|
+
});
|
|
39
|
+
break;
|
|
40
|
+
case "agent":
|
|
41
|
+
this.agentEvent(event.value);
|
|
42
|
+
break;
|
|
43
|
+
case "complete":
|
|
44
|
+
this.runs.set(event.runId, event.value);
|
|
45
|
+
break;
|
|
46
|
+
case "evaluation":
|
|
47
|
+
this.evaluations.push(event.value);
|
|
48
|
+
break;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
agentEvent(raw) {
|
|
52
|
+
const value = raw;
|
|
53
|
+
if (value.type === "text")
|
|
54
|
+
this.emit({ ...this.envelope, type: "text", text: value.text ?? "" });
|
|
55
|
+
if (value.type === "tool")
|
|
56
|
+
this.emit({ ...this.envelope, type: "tool", name: value.name ?? "tool", args: value.args });
|
|
57
|
+
}
|
|
58
|
+
section(run) {
|
|
59
|
+
return {
|
|
60
|
+
title: run.name,
|
|
61
|
+
checks: [],
|
|
62
|
+
notes: [run.output, `Agent tokens: ${run.usage.tokens.total ?? "unavailable"}`],
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
completedResult(wire) {
|
|
66
|
+
const all = [...this.runs.values()];
|
|
67
|
+
const errors = wire.errors?.length ? wire.errors : ["Test did not pass"];
|
|
68
|
+
return {
|
|
69
|
+
authoring: "typescript",
|
|
70
|
+
suite: this.envelope.suite,
|
|
71
|
+
scenario: this.envelope.scenario,
|
|
72
|
+
passed: wire.passed === true,
|
|
73
|
+
skipped: wire.skipped,
|
|
74
|
+
durationMs: wire.durationMs ?? 0,
|
|
75
|
+
trace: combinedTrace(all),
|
|
76
|
+
failures: wire.passed
|
|
77
|
+
? []
|
|
78
|
+
: errors.map((message) => ({ matcher: "test", message, category: "rubric_miss" })),
|
|
79
|
+
story: {
|
|
80
|
+
criteria: [],
|
|
81
|
+
result: [],
|
|
82
|
+
verdict: wire.errors ?? [],
|
|
83
|
+
sections: [
|
|
84
|
+
...all.map((run) => this.section(run)),
|
|
85
|
+
...this.evaluations.map((value) => ({
|
|
86
|
+
title: value.name,
|
|
87
|
+
checks: [],
|
|
88
|
+
notes: [
|
|
89
|
+
JSON.stringify(value.output),
|
|
90
|
+
`Judge tokens: ${value.usage.tokens.total ?? "unavailable"}`,
|
|
91
|
+
],
|
|
92
|
+
})),
|
|
93
|
+
],
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
finish(code, cancelled) {
|
|
98
|
+
const result = this.result ?? this.missingResult(code, cancelled);
|
|
99
|
+
this.emit({ ...this.envelope, type: "scenario_result", result });
|
|
100
|
+
this.emit({
|
|
101
|
+
...this.envelope,
|
|
102
|
+
type: "cell_finished",
|
|
103
|
+
passed: result.passed,
|
|
104
|
+
skipped: result.skipped,
|
|
105
|
+
durationMs: result.durationMs,
|
|
106
|
+
failures: result.failures,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
missingResult(code, cancelled) {
|
|
110
|
+
return {
|
|
111
|
+
authoring: "typescript",
|
|
112
|
+
suite: this.envelope.suite,
|
|
113
|
+
scenario: this.envelope.scenario,
|
|
114
|
+
passed: false,
|
|
115
|
+
durationMs: 0,
|
|
116
|
+
failures: [
|
|
117
|
+
{
|
|
118
|
+
matcher: "runner",
|
|
119
|
+
message: cancelled ? "Cancelled" : `Runner exited without a result (${code})`,
|
|
120
|
+
category: "agent_runtime",
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=viewer-results.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"viewer-results.js","sourceRoot":"","sources":["../../src/sdk/viewer-results.ts"],"names":[],"mappings":"AA4BA,SAAS,aAAa,CAAC,IAAW;IACjC,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;QACnD,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;QAC/C,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC;QAC7D,SAAS,EAAE,EAAE;KACb,CAAC;AACH,CAAC;AACD,8EAA8E;AAC9E,MAAM,OAAO,gBAAgB;IAOV,IAAI;IANL,IAAI,GAAG,IAAI,GAAG,EAAe,CAAC;IAC9B,WAAW,GAAiB,EAAE,CAAC;IACxC,MAAM,CAAkB;IACf,QAAQ,CAAC;IAC1B,YACC,GAAc,EACG,IAAkC;oBAAlC,IAAI;QAErB,IAAI,CAAC,QAAQ,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IACD,KAAK;QACJ,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,CAAC,IAAe;QACtB,IAAI,IAAI,CAAC,KAAK;YAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;YACxB,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,IAAI,cAAc,EAAE,CAAC,CAAC;QACvF,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;YAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IACO,OAAO,CAAC,KAAe;QAC9B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,OAAO;gBACX,IAAI,CAAC,IAAI,CAAC;oBACT,GAAG,IAAI,CAAC,QAAQ;oBAChB,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAG,KAAK,CAAC,KAA4B,CAAC,MAAM;iBAChD,CAAC,CAAC;gBACH,MAAM;YACP,KAAK,OAAO;gBACX,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC7B,MAAM;YACP,KAAK,UAAU;gBACd,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAY,CAAC,CAAC;gBAC/C,MAAM;YACP,KAAK,YAAY;gBAChB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,KAAmB,CAAC,CAAC;gBACjD,MAAM;QACR,CAAC;IACF,CAAC;IACO,UAAU,CAAC,GAAY;QAC9B,MAAM,KAAK,GAAG,GAKb,CAAC;QACF,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;YACxB,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;YACxB,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9F,CAAC;IACO,OAAO,CAAC,GAAQ;QACvB,OAAO;YACN,KAAK,EAAE,GAAG,CAAC,IAAI;YACf,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,iBAAiB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;SAC/E,CAAC;IACH,CAAC;IACO,eAAe,CAAC,IAAe;QACtC,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QACzE,OAAO;YACN,SAAS,EAAE,YAAY;YACvB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAChC,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,IAAI;YAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC;YAChC,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,MAAM;gBACpB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;YACnF,KAAK,EAAE;gBACN,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBAC1B,QAAQ,EAAE;oBACT,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;wBACnC,KAAK,EAAE,KAAK,CAAC,IAAI;wBACjB,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE;4BACN,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;4BAC5B,iBAAiB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,aAAa,EAAE;yBAC5D;qBACD,CAAC,CAAC;iBACH;aACD;SACD,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAY,EAAE,SAAkB;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,IAAI,CAAC;YACT,GAAG,IAAI,CAAC,QAAQ;YAChB,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SACzB,CAAC,CAAC;IACJ,CAAC;IACO,aAAa,CAAC,IAAY,EAAE,SAAkB;QACrD,OAAO;YACN,SAAS,EAAE,YAAY;YACvB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAChC,MAAM,EAAE,KAAK;YACb,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE;gBACT;oBACC,OAAO,EAAE,QAAQ;oBACjB,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,mCAAmC,IAAI,GAAG;oBAC7E,QAAQ,EAAE,eAAe;iBACzB;aACD;SACD,CAAC;IACH,CAAC;CACD"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ViewerCatalog } from "../viewer/catalog.js";
|
|
2
|
+
import type { ViewerRunner } from "../viewer/run-controller.js";
|
|
3
|
+
import { type TestEntry } from "./viewer-results.js";
|
|
4
|
+
export declare function loadSdkCatalog(config: string): Promise<{
|
|
5
|
+
catalog: ViewerCatalog;
|
|
6
|
+
tests: TestEntry[];
|
|
7
|
+
}>;
|
|
8
|
+
export declare function sdkViewerRunner(config: string, tests: TestEntry[]): ViewerRunner;
|
|
9
|
+
export declare function startSdkViewer(config?: string, port?: number): Promise<number>;
|
|
10
|
+
export declare function runSdkViewerCli(args: string[]): Promise<number>;
|
|
11
|
+
//# sourceMappingURL=viewer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"viewer.d.ts","sourceRoot":"","sources":["../../src/sdk/viewer.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,sBAAsB,CAAC;AAErE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAGhE,OAAO,EAAE,KAAK,SAAS,EAAoC,MAAM,qBAAqB,CAAC;AA+DvF,wBAAsB,cAAc,CACnC,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,OAAO,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,SAAS,EAAE,CAAA;CAAE,CAAC,CAkCzD;AAiBD,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,YAAY,CAgBhF;AACD,wBAAsB,cAAc,CACnC,MAAM,SAAkC,EACxC,IAAI,SAAI,GACN,OAAO,CAAC,MAAM,CAAC,CAmBjB;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CASrE"}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { mkdir } from "node:fs/promises";
|
|
3
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { parseArgs } from "node:util";
|
|
6
|
+
import { listenViewer } from "../viewer/server.js";
|
|
7
|
+
import { playwrightCli } from "./cli.js";
|
|
8
|
+
import { ViewerTestResult } from "./viewer-results.js";
|
|
9
|
+
function parseEvent(line, receive) {
|
|
10
|
+
let event;
|
|
11
|
+
try {
|
|
12
|
+
event = JSON.parse(line);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
receive(event);
|
|
18
|
+
}
|
|
19
|
+
async function runPlaywright(config, args, observer) {
|
|
20
|
+
const { receive, signal } = observer;
|
|
21
|
+
return new Promise((resolveExit, reject) => {
|
|
22
|
+
const child = spawn(process.execPath, [
|
|
23
|
+
playwrightCli(),
|
|
24
|
+
"test",
|
|
25
|
+
"-c",
|
|
26
|
+
config,
|
|
27
|
+
"--reporter",
|
|
28
|
+
fileURLToPath(new URL("./reporter.js", import.meta.url)),
|
|
29
|
+
...args,
|
|
30
|
+
], {
|
|
31
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
32
|
+
env: { ...process.env, AGENT_TEST_VIEWER_EVENTS: "1" },
|
|
33
|
+
});
|
|
34
|
+
let buffer = "", stderr = "";
|
|
35
|
+
child.stdout.setEncoding("utf8");
|
|
36
|
+
child.stdout.on("data", (chunk) => {
|
|
37
|
+
const lines = (buffer + chunk).split("\n");
|
|
38
|
+
buffer = lines.pop() ?? "";
|
|
39
|
+
for (const line of lines) {
|
|
40
|
+
try {
|
|
41
|
+
parseEvent(line, receive);
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
child.kill("SIGTERM");
|
|
45
|
+
reject(error);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
child.stderr.on("data", (chunk) => {
|
|
50
|
+
stderr = (stderr + chunk).slice(-8192);
|
|
51
|
+
});
|
|
52
|
+
const abort = () => child.kill("SIGINT");
|
|
53
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
54
|
+
if (signal?.aborted)
|
|
55
|
+
abort();
|
|
56
|
+
child.on("error", reject);
|
|
57
|
+
child.on("close", (code) => {
|
|
58
|
+
signal?.removeEventListener("abort", abort);
|
|
59
|
+
if (code && !signal?.aborted && stderr)
|
|
60
|
+
receive({ type: "error", message: stderr });
|
|
61
|
+
resolveExit(code ?? 1);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
export async function loadSdkCatalog(config) {
|
|
66
|
+
let tests = [];
|
|
67
|
+
const errors = [];
|
|
68
|
+
const code = await runPlaywright(config, ["--list"], {
|
|
69
|
+
receive: (event) => {
|
|
70
|
+
if (event.tests)
|
|
71
|
+
tests = event.tests;
|
|
72
|
+
if (event.message)
|
|
73
|
+
errors.push(event.message);
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
if (code)
|
|
77
|
+
throw new Error(`Test discovery failed: ${errors.join("\n")}`);
|
|
78
|
+
const catalog = {
|
|
79
|
+
suitesDir: config,
|
|
80
|
+
defaultSelectedHosts: [...new Set(tests.map((test) => test.project))],
|
|
81
|
+
suites: [],
|
|
82
|
+
};
|
|
83
|
+
for (const entry of tests) {
|
|
84
|
+
let suite = catalog.suites.find((suite) => suite.name === relative(dirname(config), entry.file));
|
|
85
|
+
if (!suite) {
|
|
86
|
+
suite = { name: relative(dirname(config), entry.file), hosts: [], scenarios: [] };
|
|
87
|
+
catalog.suites.push(suite);
|
|
88
|
+
}
|
|
89
|
+
if (!suite.hosts.includes(entry.project))
|
|
90
|
+
suite.hosts.push(entry.project);
|
|
91
|
+
if (!suite.scenarios.some((scenario) => scenario.name === entry.title))
|
|
92
|
+
suite.scenarios.push({
|
|
93
|
+
name: entry.title,
|
|
94
|
+
authoring: "typescript",
|
|
95
|
+
prompt: "Named agents and evaluations run when this test executes.",
|
|
96
|
+
rubric: {},
|
|
97
|
+
contextMode: "host-native",
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
return { catalog, tests };
|
|
101
|
+
}
|
|
102
|
+
const REGEX_META = /[.*+?^${}()|[\]\\]/g;
|
|
103
|
+
async function executeViewerJob(config, entry, job) {
|
|
104
|
+
const outputDir = join(dirname(config), "test-results", "agent-viewer", crypto.randomUUID());
|
|
105
|
+
await mkdir(outputDir, { recursive: true });
|
|
106
|
+
const title = entry.title.split(" › ").join(" ").replace(REGEX_META, "\\$&");
|
|
107
|
+
const args = [entry.file, "--grep", `${title}$`, "--workers", "1", "--output", outputDir];
|
|
108
|
+
if (entry.project !== "default")
|
|
109
|
+
args.push("--project", entry.project);
|
|
110
|
+
return runPlaywright(config, args, {
|
|
111
|
+
receive: (wire) => job.result.receive(wire),
|
|
112
|
+
signal: job.signal,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
export function sdkViewerRunner(config, tests) {
|
|
116
|
+
return {
|
|
117
|
+
async runJob(job, emit, signal) {
|
|
118
|
+
const entry = tests.find((test) => relative(dirname(config), test.file) === job.suite &&
|
|
119
|
+
test.title === job.scenario &&
|
|
120
|
+
test.project === job.host);
|
|
121
|
+
if (!entry)
|
|
122
|
+
throw new Error("Test is no longer in the discovered catalog");
|
|
123
|
+
const result = new ViewerTestResult(job, emit);
|
|
124
|
+
result.start();
|
|
125
|
+
const code = await executeViewerJob(config, entry, { result, signal });
|
|
126
|
+
result.finish(code, signal.aborted);
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
export async function startSdkViewer(config = resolve("agent-test.config.ts"), port = 0) {
|
|
131
|
+
const { catalog, tests } = await loadSdkCatalog(config);
|
|
132
|
+
const handle = await listenViewer({
|
|
133
|
+
catalog,
|
|
134
|
+
runner: sdkViewerRunner(config, tests),
|
|
135
|
+
cwd: process.cwd(),
|
|
136
|
+
suitesDir: config,
|
|
137
|
+
workers: 1,
|
|
138
|
+
port,
|
|
139
|
+
});
|
|
140
|
+
console.log(`Agent test viewer: ${handle.url}`);
|
|
141
|
+
await new Promise((resolveClose) => {
|
|
142
|
+
const stop = () => {
|
|
143
|
+
void handle.close().then(resolveClose);
|
|
144
|
+
};
|
|
145
|
+
process.once("SIGINT", stop);
|
|
146
|
+
process.once("SIGTERM", stop);
|
|
147
|
+
});
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
|
150
|
+
export async function runSdkViewerCli(args) {
|
|
151
|
+
const { values } = parseArgs({
|
|
152
|
+
args,
|
|
153
|
+
options: { config: { type: "string", short: "c" }, port: { type: "string" } },
|
|
154
|
+
});
|
|
155
|
+
const port = Number(values.port ?? "0");
|
|
156
|
+
if (!Number.isInteger(port) || port < 0 || port > 65535)
|
|
157
|
+
throw new Error("Port must be an integer from 0 through 65535");
|
|
158
|
+
return startSdkViewer(resolve(values.config ?? "agent-test.config.ts"), port);
|
|
159
|
+
}
|
|
160
|
+
//# sourceMappingURL=viewer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"viewer.js","sourceRoot":"","sources":["../../src/sdk/viewer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAkB,gBAAgB,EAAkB,MAAM,qBAAqB,CAAC;AAEvF,SAAS,UAAU,CAAC,IAAY,EAAE,OAAmC;IACpE,IAAI,KAAgB,CAAC;IACrB,IAAI,CAAC;QACJ,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO;IACR,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAC;AAChB,CAAC;AACD,KAAK,UAAU,aAAa,CAC3B,MAAc,EACd,IAAc,EACd,QAAuE;IAEvE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;IACrC,OAAO,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,KAAK,GAAG,KAAK,CAClB,OAAO,CAAC,QAAQ,EAChB;YACC,aAAa,EAAE;YACf,MAAM;YACN,IAAI;YACJ,MAAM;YACN,YAAY;YACZ,aAAa,CAAC,IAAI,GAAG,CAAC,eAAe,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;YACxD,GAAG,IAAI;SACP,EACD;YACC,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACjC,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,wBAAwB,EAAE,GAAG,EAAE;SACtD,CACD,CAAC;QACF,IAAI,MAAM,GAAG,EAAE,EACd,MAAM,GAAG,EAAE,CAAC;QACb,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,KAAK,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,CAAC;oBACJ,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC3B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACtB,MAAM,CAAC,KAAK,CAAC,CAAC;gBACf,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,MAAM,EAAE,OAAO;YAAE,KAAK,EAAE,CAAC;QAC7B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YAC1B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC5C,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,MAAM;gBAAE,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YACpF,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,MAAc;IAEd,IAAI,KAAK,GAAgB,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE;QACpD,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAClB,IAAI,KAAK,CAAC,KAAK;gBAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YACrC,IAAI,KAAK,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC;KACD,CAAC,CAAC;IACH,IAAI,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzE,MAAM,OAAO,GAAkB;QAC9B,SAAS,EAAE,MAAM;QACjB,oBAAoB,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,EAAE,EAAE;KACV,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC3B,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAC/D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;YAClF,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC;YACrE,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC;gBACpB,IAAI,EAAE,KAAK,CAAC,KAAK;gBACjB,SAAS,EAAE,YAAY;gBACvB,MAAM,EAAE,2DAA2D;gBACnE,MAAM,EAAE,EAAE;gBACV,WAAW,EAAE,aAAa;aAC1B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AACD,MAAM,UAAU,GAAG,qBAAqB,CAAC;AACzC,KAAK,UAAU,gBAAgB,CAC9B,MAAc,EACd,KAAgB,EAChB,GAAsD;IAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC7F,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,KAAK,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAC1F,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACvE,OAAO,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE;QAClC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;QAC3C,MAAM,EAAE,GAAG,CAAC,MAAM;KAClB,CAAC,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,KAAkB;IACjE,OAAO;QACN,KAAK,CAAC,MAAM,CAAC,GAAc,EAAE,IAAkC,EAAE,MAAmB;YACnF,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CACvB,CAAC,IAAI,EAAE,EAAE,CACR,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK;gBAClD,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,QAAQ;gBAC3B,IAAI,CAAC,OAAO,KAAK,GAAG,CAAC,IAAI,CAC1B,CAAC;YACF,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YACvE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;KACD,CAAC;AACH,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC,EACxC,IAAI,GAAG,CAAC;IAER,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QACjC,OAAO;QACP,MAAM,EAAE,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC;QACtC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;QAClB,SAAS,EAAE,MAAM;QACjB,OAAO,EAAE,CAAC;QACV,IAAI;KACJ,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,OAAO,CAAO,CAAC,YAAY,EAAE,EAAE;QACxC,MAAM,IAAI,GAAG,GAAG,EAAE;YACjB,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,CAAC;AACV,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAc;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC5B,IAAI;QACJ,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;KAC7E,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK;QACtD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjE,OAAO,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,sBAAsB,CAAC,EAAE,IAAI,CAAC,CAAC;AAC/E,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type AgentDefinition, type SealedWorkspace } from "@post-print/agent-harness";
|
|
2
|
+
export interface Workspace {
|
|
3
|
+
path: string;
|
|
4
|
+
readFile(path: string): Promise<string>;
|
|
5
|
+
}
|
|
6
|
+
export interface WorkspaceSnapshot {
|
|
7
|
+
path: string;
|
|
8
|
+
files: Record<string, {
|
|
9
|
+
sha256: string;
|
|
10
|
+
bytes: number;
|
|
11
|
+
}>;
|
|
12
|
+
}
|
|
13
|
+
export declare function under(root: string, path: string): string;
|
|
14
|
+
export declare function copyTree(source: string, target: string): Promise<void>;
|
|
15
|
+
export declare function snapshot(source: string, target: string): Promise<WorkspaceSnapshot>;
|
|
16
|
+
export declare function changedPaths(before: WorkspaceSnapshot, after: WorkspaceSnapshot): string[];
|
|
17
|
+
export declare function prepareWorkspace(baseDir: string, source: string): Promise<SealedWorkspace & Workspace>;
|
|
18
|
+
export interface StartingContext {
|
|
19
|
+
instructions: string[];
|
|
20
|
+
files: {
|
|
21
|
+
path: string;
|
|
22
|
+
content: string;
|
|
23
|
+
}[];
|
|
24
|
+
skills: {
|
|
25
|
+
source: string;
|
|
26
|
+
destination: string;
|
|
27
|
+
}[];
|
|
28
|
+
includeGlobalSkills: boolean;
|
|
29
|
+
}
|
|
30
|
+
export declare function prepareAgent(agent: AgentDefinition, baseDir: string, workspace: string): Promise<StartingContext>;
|
|
31
|
+
export declare function withContext(prompt: string, context: StartingContext): string;
|
|
32
|
+
export declare function writeJson(path: string, value: unknown): Promise<void>;
|
|
33
|
+
//# sourceMappingURL=workspace.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workspace.d.ts","sourceRoot":"","sources":["../../src/sdk/workspace.ts"],"names":[],"mappings":"AAGA,OAAO,EACN,KAAK,eAAe,EAEpB,KAAK,eAAe,EACpB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxC;AACD,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACzD;AACD,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMxD;AACD,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAW5E;AACD,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBzF;AACD,wBAAgB,YAAY,CAAC,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,GAAG,MAAM,EAAE,CAI1F;AACD,wBAAsB,gBAAgB,CACrC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CActC;AACD,MAAM,WAAW,eAAe;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC3C,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAClD,mBAAmB,EAAE,OAAO,CAAC;CAC7B;AACD,wBAAsB,YAAY,CACjC,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC,CAyB1B;AACD,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAG5E;AACD,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,iBAG3D"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { cp, mkdir, readdir, readFile, realpath, writeFile } from "node:fs/promises";
|
|
3
|
+
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { createSealedWorkspace, } from "@post-print/agent-harness";
|
|
5
|
+
export function under(root, path) {
|
|
6
|
+
const target = resolve(root, path);
|
|
7
|
+
const rel = relative(resolve(root), target);
|
|
8
|
+
if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel))
|
|
9
|
+
throw new Error(`Path escapes workspace: ${path}`);
|
|
10
|
+
return target;
|
|
11
|
+
}
|
|
12
|
+
export async function copyTree(source, target) {
|
|
13
|
+
await mkdir(target, { recursive: true });
|
|
14
|
+
for (const item of await readdir(source, { withFileTypes: true })) {
|
|
15
|
+
if ([".git", "node_modules"].includes(item.name))
|
|
16
|
+
continue;
|
|
17
|
+
if (item.isSymbolicLink())
|
|
18
|
+
throw new Error(`Workspace snapshots do not support symlinks: ${join(source, item.name)}`);
|
|
19
|
+
const from = join(source, item.name), to = join(target, item.name);
|
|
20
|
+
if (item.isDirectory())
|
|
21
|
+
await copyTree(from, to);
|
|
22
|
+
else if (item.isFile())
|
|
23
|
+
await cp(from, to);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export async function snapshot(source, target) {
|
|
27
|
+
await copyTree(source, target);
|
|
28
|
+
const files = {};
|
|
29
|
+
async function walk(path) {
|
|
30
|
+
for (const entry of await readdir(path, { withFileTypes: true })) {
|
|
31
|
+
const full = join(path, entry.name);
|
|
32
|
+
if (entry.isDirectory())
|
|
33
|
+
await walk(full);
|
|
34
|
+
else {
|
|
35
|
+
const bytes = await readFile(full);
|
|
36
|
+
files[relative(target, full).split(sep).join("/")] = {
|
|
37
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
38
|
+
bytes: bytes.length,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
await walk(target);
|
|
44
|
+
return { path: target, files };
|
|
45
|
+
}
|
|
46
|
+
export function changedPaths(before, after) {
|
|
47
|
+
return [...new Set([...Object.keys(before.files), ...Object.keys(after.files)])]
|
|
48
|
+
.filter((path) => before.files[path]?.sha256 !== after.files[path]?.sha256)
|
|
49
|
+
.sort();
|
|
50
|
+
}
|
|
51
|
+
export async function prepareWorkspace(baseDir, source) {
|
|
52
|
+
const sealed = await createSealedWorkspace({
|
|
53
|
+
callerCwd: baseDir,
|
|
54
|
+
workspace: source,
|
|
55
|
+
overlayPaths: [],
|
|
56
|
+
});
|
|
57
|
+
return {
|
|
58
|
+
...sealed,
|
|
59
|
+
path: await realpath(sealed.path),
|
|
60
|
+
async readFile(path) {
|
|
61
|
+
const actual = await realpath(under(sealed.path, path));
|
|
62
|
+
return readFile(under(await realpath(sealed.path), actual), "utf8");
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export async function prepareAgent(agent, baseDir, workspace) {
|
|
67
|
+
const context = {
|
|
68
|
+
instructions: [...(agent.options.context?.instructions ?? [])],
|
|
69
|
+
files: [],
|
|
70
|
+
skills: [],
|
|
71
|
+
includeGlobalSkills: agent.options.includeGlobalSkills === true,
|
|
72
|
+
};
|
|
73
|
+
for (const file of agent.options.context?.files ?? [])
|
|
74
|
+
context.files.push({ path: file, content: await readFile(resolve(baseDir, file), "utf8") });
|
|
75
|
+
for (const skill of agent.options.skills ?? []) {
|
|
76
|
+
const source = resolve(baseDir, skill);
|
|
77
|
+
await readFile(join(source, "SKILL.md"), "utf8");
|
|
78
|
+
const root = agent.host === "claude"
|
|
79
|
+
? ".claude/skills"
|
|
80
|
+
: agent.host === "cursor"
|
|
81
|
+
? ".cursor/skills"
|
|
82
|
+
: ".agents/skills";
|
|
83
|
+
const destination = join(root, basename(source));
|
|
84
|
+
const target = under(workspace, destination);
|
|
85
|
+
await mkdir(dirname(target), { recursive: true });
|
|
86
|
+
await copyTree(source, target);
|
|
87
|
+
context.skills.push({ source: skill, destination });
|
|
88
|
+
}
|
|
89
|
+
return context;
|
|
90
|
+
}
|
|
91
|
+
export function withContext(prompt, context) {
|
|
92
|
+
if (!context.instructions.length && !context.files.length)
|
|
93
|
+
return prompt;
|
|
94
|
+
return `Session instructions:\n${context.instructions.join("\n")}\n\nStarting context:\n${JSON.stringify(context.files)}\n\nUser request:\n${prompt}`;
|
|
95
|
+
}
|
|
96
|
+
export async function writeJson(path, value) {
|
|
97
|
+
await mkdir(dirname(path), { recursive: true });
|
|
98
|
+
await writeFile(path, JSON.stringify(value, null, 2));
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=workspace.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workspace.js","sourceRoot":"","sources":["../../src/sdk/workspace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrF,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACxF,OAAO,EAEN,qBAAqB,GAErB,MAAM,2BAA2B,CAAC;AAUnC,MAAM,UAAU,KAAK,CAAC,IAAY,EAAE,IAAY;IAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IAC5C,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC;AACf,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc,EAAE,MAAc;IAC5D,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACnE,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC3D,IAAI,IAAI,CAAC,cAAc,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5F,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,EACnC,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,IAAI,CAAC,WAAW,EAAE;YAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;aAC5C,IAAI,IAAI,CAAC,MAAM,EAAE;YAAE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC5C,CAAC;AACF,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc,EAAE,MAAc;IAC5D,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,MAAM,KAAK,GAA+B,EAAE,CAAC;IAC7C,KAAK,UAAU,IAAI,CAAC,IAAY;QAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;iBACrC,CAAC;gBACL,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACnC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG;oBACpD,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBACxD,KAAK,EAAE,KAAK,CAAC,MAAM;iBACnB,CAAC;YACH,CAAC;QACF,CAAC;IACF,CAAC;IACD,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAChC,CAAC;AACD,MAAM,UAAU,YAAY,CAAC,MAAyB,EAAE,KAAwB;IAC/E,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAC9E,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;SAC1E,IAAI,EAAE,CAAC;AACV,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAe,EACf,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC;QAC1C,SAAS,EAAE,OAAO;QAClB,SAAS,EAAE,MAAM;QACjB,YAAY,EAAE,EAAE;KAChB,CAAC,CAAC;IACH,OAAO;QACN,GAAG,MAAM;QACT,IAAI,EAAE,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,CAAC,QAAQ,CAAC,IAAI;YAClB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YACxD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QACrE,CAAC;KACD,CAAC;AACH,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,KAAsB,EACtB,OAAe,EACf,SAAiB;IAEjB,MAAM,OAAO,GAAoB;QAChC,YAAY,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;QAC9D,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,mBAAmB,KAAK,IAAI;KAC/D,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QACpD,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7F,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,IAAI,GACT,KAAK,CAAC,IAAI,KAAK,QAAQ;YACtB,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ;gBACxB,CAAC,CAAC,gBAAgB;gBAClB,CAAC,CAAC,gBAAgB,CAAC;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC7C,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AACD,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,OAAwB;IACnE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC;IACzE,OAAO,0BAA0B,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,EAAE,CAAC;AACvJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,KAAc;IAC3D,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC"}
|