@aether-agent/evals 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/Agent.d.ts +2 -2
- package/dist/Agent.d.ts.map +1 -1
- package/dist/DockerAgent.d.ts +2 -2
- package/dist/DockerAgent.d.ts.map +1 -1
- package/dist/DockerAgent.js +6 -6
- package/dist/DockerAgent.js.map +1 -1
- package/dist/FakeAgent.d.ts +4 -4
- package/dist/FakeAgent.d.ts.map +1 -1
- package/dist/FakeAgent.js +34 -22
- package/dist/FakeAgent.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/judge.d.ts +6 -6
- package/dist/judge.d.ts.map +1 -1
- package/dist/judge.js +29 -18
- package/dist/judge.js.map +1 -1
- package/dist/transcript.d.ts +8 -6
- package/dist/transcript.d.ts.map +1 -1
- package/dist/transcript.js +29 -25
- package/dist/transcript.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -51,7 +51,10 @@ test("agent edits notes.txt", async () => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
const trace = await Transcript.fromStream(agent.run(task));
|
|
54
|
-
expect(trace.
|
|
54
|
+
expect(trace.events.at(-1)).toMatchObject({
|
|
55
|
+
category: "turn",
|
|
56
|
+
event: { type: "ended" },
|
|
57
|
+
});
|
|
55
58
|
|
|
56
59
|
// Assert against the retained workspace and the recorded tool calls.
|
|
57
60
|
expect(await readFile(join(workspace.path, "notes.txt"), "utf8")).toBe(
|
|
@@ -102,7 +105,7 @@ test("agent makes a maintainer-quality edit", async () => {
|
|
|
102
105
|
instructions: "Grade strictly using only the provided transcript and files.",
|
|
103
106
|
task: "Change the first line of notes.txt from alpha to beta",
|
|
104
107
|
context: {
|
|
105
|
-
transcript: trace.
|
|
108
|
+
transcript: trace.events,
|
|
106
109
|
files: {
|
|
107
110
|
"notes.txt": await readFile(join(workspace.path, "notes.txt"), "utf8"),
|
|
108
111
|
},
|
package/dist/Agent.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AgentEvent } from "@aether-agent/sdk";
|
|
2
2
|
import type { Task } from "./task.js";
|
|
3
3
|
export interface Agent {
|
|
4
|
-
run(task: Task): AsyncIterable<
|
|
4
|
+
run(task: Task): AsyncIterable<AgentEvent>;
|
|
5
5
|
}
|
|
6
6
|
//# sourceMappingURL=Agent.d.ts.map
|
package/dist/Agent.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Agent.d.ts","sourceRoot":"","sources":["../src/Agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"Agent.d.ts","sourceRoot":"","sources":["../src/Agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEtC,MAAM,WAAW,KAAK;IACpB,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;CAC5C"}
|
package/dist/DockerAgent.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AgentEvent } from "@aether-agent/sdk";
|
|
2
2
|
import { Container } from "./containers/container.js";
|
|
3
3
|
import type { Agent } from "./Agent.js";
|
|
4
4
|
import type { Task } from "./task.js";
|
|
@@ -18,7 +18,7 @@ export declare class DockerAgent implements Agent {
|
|
|
18
18
|
constructor(options: DockerAgentOptions);
|
|
19
19
|
withEnvVar(key: string, value: string): DockerAgent;
|
|
20
20
|
withEnvVars(env: Record<string, string>): DockerAgent;
|
|
21
|
-
run(task: Task): AsyncIterable<
|
|
21
|
+
run(task: Task): AsyncIterable<AgentEvent>;
|
|
22
22
|
}
|
|
23
23
|
export declare function defaultEvalEnvVars(hostEnv?: Record<string, string | undefined>): Record<string, string>;
|
|
24
24
|
//# sourceMappingURL=DockerAgent.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DockerAgent.d.ts","sourceRoot":"","sources":["../src/DockerAgent.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"DockerAgent.d.ts","sourceRoot":"","sources":["../src/DockerAgent.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAEtD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGtC,eAAO,MAAM,qBAAqB,kBAAkB,CAAC;AACrD,eAAO,MAAM,2BAA2B,4BAA4B,CAAC;AACrE,eAAO,MAAM,8BAA8B,+BAA+B,CAAC;AAC3E,eAAO,MAAM,mBAAmB,oBAAoB,CAAC;AAWrD,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED,qBAAa,WAAY,YAAW,KAAK;IACvC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAEzB,OAAO,EAAE,kBAAkB;IAevC,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW;IAQnD,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,WAAW;IAQ9C,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC;CAiClD;AAED,wBAAgB,kBAAkB,CAChC,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAc,GACvD,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAexB"}
|
package/dist/DockerAgent.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { env as processEnv } from "node:process";
|
|
2
2
|
import { AetherEvalError } from "./errors.js";
|
|
3
|
-
import {
|
|
3
|
+
import { isTerminalEvent } from "./transcript.js";
|
|
4
4
|
export const CONTAINER_AETHER_HOME = "/root/.aether";
|
|
5
5
|
export const AETHER_EVAL_TASK_PROMPT_ENV = "AETHER_EVAL_TASK_PROMPT";
|
|
6
6
|
export const AETHER_EVAL_WORKSPACE_ROOT_ENV = "AETHER_EVAL_WORKSPACE_ROOT";
|
|
@@ -63,14 +63,14 @@ export class DockerAgent {
|
|
|
63
63
|
const trimmed = line.trim();
|
|
64
64
|
if (!trimmed)
|
|
65
65
|
continue;
|
|
66
|
-
const message =
|
|
67
|
-
finished =
|
|
66
|
+
const message = parseAgentEvent(trimmed);
|
|
67
|
+
finished = isTerminalEvent(message);
|
|
68
68
|
yield message;
|
|
69
69
|
if (finished)
|
|
70
70
|
break;
|
|
71
71
|
}
|
|
72
72
|
if (!finished) {
|
|
73
|
-
throw new AetherEvalError("command_exit_without_terminal", `agent command exited without emitting a terminal
|
|
73
|
+
throw new AetherEvalError("command_exit_without_terminal", `agent command exited without emitting a terminal AgentEvent.\nstderr:\n${stderr}`);
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
}
|
|
@@ -90,12 +90,12 @@ export function defaultEvalEnvVars(hostEnv = processEnv) {
|
|
|
90
90
|
env.AETHER_HOME = CONTAINER_AETHER_HOME;
|
|
91
91
|
return env;
|
|
92
92
|
}
|
|
93
|
-
function
|
|
93
|
+
function parseAgentEvent(line) {
|
|
94
94
|
try {
|
|
95
95
|
return JSON.parse(line);
|
|
96
96
|
}
|
|
97
97
|
catch (err) {
|
|
98
|
-
throw new AetherEvalError("
|
|
98
|
+
throw new AetherEvalError("agent_event_json_line", `agent emitted an invalid AgentEvent JSON line: ${line}`, err);
|
|
99
99
|
}
|
|
100
100
|
}
|
|
101
101
|
//# sourceMappingURL=DockerAgent.js.map
|
package/dist/DockerAgent.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DockerAgent.js","sourceRoot":"","sources":["../src/DockerAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG9C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"DockerAgent.js","sourceRoot":"","sources":["../src/DockerAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG9C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,CAAC,MAAM,qBAAqB,GAAG,eAAe,CAAC;AACrD,MAAM,CAAC,MAAM,2BAA2B,GAAG,yBAAyB,CAAC;AACrE,MAAM,CAAC,MAAM,8BAA8B,GAAG,4BAA4B,CAAC;AAC3E,MAAM,CAAC,MAAM,mBAAmB,GAAG,iBAAiB,CAAC;AACrD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,mBAAmB;IACnB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,gBAAgB;IAChB,oBAAoB;IACpB,aAAa;CACd,CAAC,CAAC;AAQH,MAAM,OAAO,WAAW;IACb,SAAS,CAAY;IACrB,OAAO,CAAW;IAClB,GAAG,CAAyB;IAErC,YAAY,OAA2B;QACrC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,eAAe,CACvB,qBAAqB,EACrB,uCAAuC,CACxC,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,GAAG;YACT,GAAG,kBAAkB,CAAC,UAAU,CAAC;YACjC,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;SACvB,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,GAAW,EAAE,KAAa;QACnC,OAAO,IAAI,WAAW,CAAC;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE;SACnC,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,GAA2B;QACrC,OAAO,IAAI,WAAW,CAAC;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,CAAC,GAAG,CAAC,IAAU;QACnB,MAAM,GAAG,GAAG;YACV,GAAG,IAAI,CAAC,GAAG;YACX,CAAC,8BAA8B,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa;YAC9D,CAAC,mBAAmB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG;YACzC,CAAC,2BAA2B,CAAC,EAAE,IAAI,CAAC,MAAM;SAC3C,CAAC;QAEF,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;YACpD,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG;YACvB,GAAG;YACH,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC;YAClB,CAAC;SACF,CAAC,EAAE,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;YACzC,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,OAAO,CAAC;YACd,IAAI,QAAQ;gBAAE,MAAM;QACtB,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,eAAe,CACvB,+BAA+B,EAC/B,0EAA0E,MAAM,EAAE,CACnF,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,kBAAkB,CAChC,UAA8C,UAAU;IAExD,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAClC,IAAI,GAAG,KAAK,aAAa;YAAE,SAAS;QACpC,IACE,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC;YAC1B,GAAG,KAAK,aAAa;YACrB,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EACzB,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,GAAG,CAAC,WAAW,GAAG,qBAAqB,CAAC;IACxC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAC;IACxC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,eAAe,CACvB,uBAAuB,EACvB,kDAAkD,IAAI,EAAE,EACxD,GAAG,CACJ,CAAC;IACJ,CAAC;AACH,CAAC"}
|
package/dist/FakeAgent.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AgentEvent } from "@aether-agent/sdk";
|
|
2
2
|
import type { Agent } from "./Agent.js";
|
|
3
3
|
import type { Task } from "./task.js";
|
|
4
4
|
import type { Workspace } from "./workspace.js";
|
|
@@ -7,15 +7,15 @@ import type { Workspace } from "./workspace.js";
|
|
|
7
7
|
* and optionally writes files into the workspace, mirroring how a real agent is observed.
|
|
8
8
|
*/
|
|
9
9
|
export declare class FakeAgent implements Agent {
|
|
10
|
-
readonly
|
|
10
|
+
readonly events: AgentEvent[];
|
|
11
11
|
readonly fileWrites: ReadonlyArray<readonly [string, string]>;
|
|
12
12
|
readonly workspace: Workspace | undefined;
|
|
13
|
-
constructor(
|
|
13
|
+
constructor(events: AgentEvent[], fileWrites?: ReadonlyArray<readonly [string, string]>, workspace?: Workspace | undefined);
|
|
14
14
|
static success(): FakeAgent;
|
|
15
15
|
static withToolCall(toolName: string, result: string): FakeAgent;
|
|
16
16
|
static writesFile(path: string, contents: string): FakeAgent;
|
|
17
17
|
withFileWrite(path: string, contents: string): FakeAgent;
|
|
18
18
|
withWorkspace(workspace: Workspace): FakeAgent;
|
|
19
|
-
run(_task: Task): AsyncIterable<
|
|
19
|
+
run(_task: Task): AsyncIterable<AgentEvent>;
|
|
20
20
|
}
|
|
21
21
|
//# sourceMappingURL=FakeAgent.d.ts.map
|
package/dist/FakeAgent.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FakeAgent.d.ts","sourceRoot":"","sources":["../src/FakeAgent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"FakeAgent.d.ts","sourceRoot":"","sources":["../src/FakeAgent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD;;;GAGG;AACH,qBAAa,SAAU,YAAW,KAAK;IAEnC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE;IAC7B,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS;gBAFhC,MAAM,EAAE,UAAU,EAAE,EACpB,UAAU,GAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM,EACzD,SAAS,GAAE,SAAS,GAAG,SAAqB;IAGvD,MAAM,CAAC,OAAO,IAAI,SAAS;IAe3B,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS;IA2BhE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS;IAI5D,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS;IAQxD,aAAa,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS;IAIvC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC;CAkBnD"}
|
package/dist/FakeAgent.js
CHANGED
|
@@ -1,56 +1,68 @@
|
|
|
1
1
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
3
|
import { AetherEvalError } from "./errors.js";
|
|
4
|
+
import { turnEnded } from "./transcript.js";
|
|
4
5
|
/**
|
|
5
6
|
* In-memory {@link Agent} for testing eval harnesses without Docker. Streams a scripted transcript
|
|
6
7
|
* and optionally writes files into the workspace, mirroring how a real agent is observed.
|
|
7
8
|
*/
|
|
8
9
|
export class FakeAgent {
|
|
9
|
-
|
|
10
|
+
events;
|
|
10
11
|
fileWrites;
|
|
11
12
|
workspace;
|
|
12
|
-
constructor(
|
|
13
|
-
this.
|
|
13
|
+
constructor(events, fileWrites = [], workspace = undefined) {
|
|
14
|
+
this.events = events;
|
|
14
15
|
this.fileWrites = fileWrites;
|
|
15
16
|
this.workspace = workspace;
|
|
16
17
|
}
|
|
17
18
|
static success() {
|
|
18
19
|
return new FakeAgent([
|
|
19
20
|
{
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
21
|
+
category: "message",
|
|
22
|
+
event: {
|
|
23
|
+
type: "text",
|
|
24
|
+
message_id: "fake_1",
|
|
25
|
+
chunk: "Task completed successfully",
|
|
26
|
+
is_complete: true,
|
|
27
|
+
},
|
|
25
28
|
},
|
|
26
|
-
|
|
29
|
+
turnEnded(),
|
|
27
30
|
]);
|
|
28
31
|
}
|
|
29
32
|
static withToolCall(toolName, result) {
|
|
30
33
|
return new FakeAgent([
|
|
31
34
|
{
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
+
category: "tool",
|
|
36
|
+
event: {
|
|
37
|
+
type: "result",
|
|
38
|
+
result: {
|
|
39
|
+
id: "fake_call_1",
|
|
40
|
+
name: toolName,
|
|
41
|
+
arguments: "{}",
|
|
42
|
+
result,
|
|
43
|
+
},
|
|
44
|
+
},
|
|
35
45
|
},
|
|
36
46
|
{
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
category: "message",
|
|
48
|
+
event: {
|
|
49
|
+
type: "text",
|
|
50
|
+
message_id: "fake_2",
|
|
51
|
+
chunk: "Task completed using tools",
|
|
52
|
+
is_complete: true,
|
|
53
|
+
},
|
|
42
54
|
},
|
|
43
|
-
|
|
55
|
+
turnEnded(),
|
|
44
56
|
]);
|
|
45
57
|
}
|
|
46
58
|
static writesFile(path, contents) {
|
|
47
59
|
return FakeAgent.success().withFileWrite(path, contents);
|
|
48
60
|
}
|
|
49
61
|
withFileWrite(path, contents) {
|
|
50
|
-
return new FakeAgent(this.
|
|
62
|
+
return new FakeAgent(this.events, [...this.fileWrites, [path, contents]], this.workspace);
|
|
51
63
|
}
|
|
52
64
|
withWorkspace(workspace) {
|
|
53
|
-
return new FakeAgent(this.
|
|
65
|
+
return new FakeAgent(this.events, this.fileWrites, workspace);
|
|
54
66
|
}
|
|
55
67
|
async *run(_task) {
|
|
56
68
|
if (this.fileWrites.length > 0 && !this.workspace) {
|
|
@@ -61,8 +73,8 @@ export class FakeAgent {
|
|
|
61
73
|
await mkdir(dirname(target), { recursive: true });
|
|
62
74
|
await writeFile(target, contents);
|
|
63
75
|
}
|
|
64
|
-
for (const
|
|
65
|
-
yield
|
|
76
|
+
for (const event of this.events) {
|
|
77
|
+
yield event;
|
|
66
78
|
}
|
|
67
79
|
}
|
|
68
80
|
}
|
package/dist/FakeAgent.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FakeAgent.js","sourceRoot":"","sources":["../src/FakeAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"FakeAgent.js","sourceRoot":"","sources":["../src/FakeAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;GAGG;AACH,MAAM,OAAO,SAAS;IAET;IACA;IACA;IAHX,YACW,MAAoB,EACpB,aAAuD,EAAE,EACzD,YAAmC,SAAS;QAF5C,WAAM,GAAN,MAAM,CAAc;QACpB,eAAU,GAAV,UAAU,CAA+C;QACzD,cAAS,GAAT,SAAS,CAAmC;IACpD,CAAC;IAEJ,MAAM,CAAC,OAAO;QACZ,OAAO,IAAI,SAAS,CAAC;YACnB;gBACE,QAAQ,EAAE,SAAS;gBACnB,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,QAAQ;oBACpB,KAAK,EAAE,6BAA6B;oBACpC,WAAW,EAAE,IAAI;iBAClB;aACF;YACD,SAAS,EAAE;SACZ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,YAAY,CAAC,QAAgB,EAAE,MAAc;QAClD,OAAO,IAAI,SAAS,CAAC;YACnB;gBACE,QAAQ,EAAE,MAAM;gBAChB,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE;wBACN,EAAE,EAAE,aAAa;wBACjB,IAAI,EAAE,QAAQ;wBACd,SAAS,EAAE,IAAI;wBACf,MAAM;qBACP;iBACF;aACF;YACD;gBACE,QAAQ,EAAE,SAAS;gBACnB,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,QAAQ;oBACpB,KAAK,EAAE,4BAA4B;oBACnC,WAAW,EAAE,IAAI;iBAClB;aACF;YACD,SAAS,EAAE;SACZ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,IAAY,EAAE,QAAgB;QAC9C,OAAO,SAAS,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC3D,CAAC;IAED,aAAa,CAAC,IAAY,EAAE,QAAgB;QAC1C,OAAO,IAAI,SAAS,CAClB,IAAI,CAAC,MAAM,EACX,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EACtC,IAAI,CAAC,SAAS,CACf,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,SAAoB;QAChC,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,CAAC,GAAG,CAAC,KAAW;QACpB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAClD,MAAM,IAAI,eAAe,CACvB,qBAAqB,EACrB,iDAAiD,CAClD,CAAC;QACJ,CAAC;QAED,KAAK,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAU,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
package/dist/errors.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type AetherEvalErrorCode = "configuration_error" | "command_exit_without_terminal" | "
|
|
1
|
+
export type AetherEvalErrorCode = "configuration_error" | "command_exit_without_terminal" | "agent_event_json_line" | "invalid_image_reference";
|
|
2
2
|
export declare class AetherEvalError extends Error {
|
|
3
3
|
readonly code: AetherEvalErrorCode;
|
|
4
4
|
readonly cause?: unknown;
|
package/dist/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,mBAAmB,GAC3B,qBAAqB,GACrB,+BAA+B,GAC/B,
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,mBAAmB,GAC3B,qBAAqB,GACrB,+BAA+B,GAC/B,uBAAuB,GACvB,yBAAyB,CAAC;AAE9B,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,SAAkB,KAAK,CAAC,EAAE,OAAO,CAAC;gBAEtB,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAMxE"}
|
package/dist/index.d.ts
CHANGED
|
@@ -13,8 +13,8 @@ export { generate } from "./generate.js";
|
|
|
13
13
|
export type { GenerateJsonOptions, GenerateOptions, GenerateResult, ReasoningEffort, } from "./generate.js";
|
|
14
14
|
export { judge, JudgeCriterionResponseSchema, JudgeResponseSchema, } from "./judge.js";
|
|
15
15
|
export type { Judge, JudgeContext, JudgeCriterionResponse, JudgeCriterionSpec, JudgeCriterionSummary, JudgeInput, JudgeRubricResponse, JudgeSummary, } from "./judge.js";
|
|
16
|
-
export { Transcript, TranscriptError, ToolCall } from "./transcript.js";
|
|
16
|
+
export { isTerminalEvent, Transcript, TranscriptError, ToolCall, turnEnded, } from "./transcript.js";
|
|
17
17
|
export { createGitBundle, Workspace } from "./workspace.js";
|
|
18
18
|
export type { GitBundleSpec, GitRepoSpec, RetainedWorkspaceInfo, WorkspaceSource, } from "./workspace.js";
|
|
19
|
-
export type {
|
|
19
|
+
export type { AgentEvent, ContextEvent, ContextUsage, FileDiff, LlmCallOutcome, LlmCallPurpose, MessageEvent, ModelEvent, PlanMeta, PlanMetaEntry, PlanMetaStatus, StopReason, TokenUsage, ToolCallError, ToolCallRequest, ToolCallResult, ToolDisplayMeta, ToolEvent, ToolResultMeta, TurnEvent, TurnOutcome, } from "@aether-agent/sdk";
|
|
20
20
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC3E,YAAY,EACV,SAAS,EACT,oBAAoB,EACpB,yBAAyB,EACzB,UAAU,EACV,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,YAAY,EACV,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,eAAe,GAChB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,EACL,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,KAAK,EACL,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,EACrB,UAAU,EACV,mBAAmB,EACnB,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC3E,YAAY,EACV,SAAS,EACT,oBAAoB,EACpB,yBAAyB,EACzB,UAAU,EACV,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,YAAY,EACV,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,eAAe,GAChB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,EACL,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,KAAK,EACL,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,EACrB,UAAU,EACV,mBAAmB,EACnB,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACR,SAAS,GACV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC5D,YAAY,EACV,aAAa,EACb,WAAW,EACX,qBAAqB,EACrB,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,QAAQ,EACR,cAAc,EACd,cAAc,EACd,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,aAAa,EACb,cAAc,EACd,UAAU,EACV,UAAU,EACV,aAAa,EACb,eAAe,EACf,cAAc,EACd,eAAe,EACf,SAAS,EACT,cAAc,EACd,SAAS,EACT,WAAW,GACZ,MAAM,mBAAmB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,6 @@ export { Task } from "./task.js";
|
|
|
6
6
|
export { diffStatsFromDiff } from "./diff.js";
|
|
7
7
|
export { generate } from "./generate.js";
|
|
8
8
|
export { judge, JudgeCriterionResponseSchema, JudgeResponseSchema, } from "./judge.js";
|
|
9
|
-
export { Transcript, TranscriptError, ToolCall } from "./transcript.js";
|
|
9
|
+
export { isTerminalEvent, Transcript, TranscriptError, ToolCall, turnEnded, } from "./transcript.js";
|
|
10
10
|
export { createGitBundle, Workspace } from "./workspace.js";
|
|
11
11
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAS3E,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAOzC,OAAO,EACL,KAAK,EACL,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAWpB,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAS3E,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAOzC,OAAO,EACL,KAAK,EACL,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAWpB,OAAO,EACL,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACR,SAAS,GACV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC"}
|
package/dist/judge.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import type {
|
|
2
|
+
import type { AgentEvent, JudgeCriterionResponse, JudgeCriterionSpec, JudgeCriterionSummary, JudgeRubricResponse, JudgeSummary } from "@aether-agent/sdk";
|
|
3
3
|
export type { JudgeCriterionResponse, JudgeCriterionSpec, JudgeCriterionSummary, JudgeRubricResponse, JudgeSummary, };
|
|
4
4
|
export interface JudgeInput {
|
|
5
5
|
/** High-level grading instructions. */
|
|
@@ -13,7 +13,7 @@ export interface JudgeInput {
|
|
|
13
13
|
}
|
|
14
14
|
export interface JudgeContext {
|
|
15
15
|
/** The agent transcript, e.g. messages collected from `Agent.run` with `Transcript.fromStream`. */
|
|
16
|
-
transcript?:
|
|
16
|
+
transcript?: AgentEvent[];
|
|
17
17
|
/** A workspace diff to grade against. */
|
|
18
18
|
diff?: string;
|
|
19
19
|
/** Final file contents to include, keyed by path. */
|
|
@@ -40,8 +40,8 @@ export declare const JudgeResponseSchema: z.ZodObject<{
|
|
|
40
40
|
}, z.core.$strip>;
|
|
41
41
|
/** Compose a judge prompt and deterministic rubric summarizer from structured context. */
|
|
42
42
|
export declare function judge(input: JudgeInput): Judge;
|
|
43
|
-
/** Render a transcript of streamed `
|
|
44
|
-
export declare function formatTranscript(
|
|
45
|
-
/** Format a single agent
|
|
46
|
-
export declare function
|
|
43
|
+
/** Render a transcript of streamed `AgentEvent`s as readable lines for a judge prompt. */
|
|
44
|
+
export declare function formatTranscript(events: AgentEvent[]): string;
|
|
45
|
+
/** Format a single agent event into a readable line for a judge prompt. */
|
|
46
|
+
export declare function eventToString(event: AgentEvent): string;
|
|
47
47
|
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EACV,
|
|
1
|
+
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EACV,UAAU,EACV,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,EACb,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EACV,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,GACb,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,uCAAuC;IACvC,YAAY,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,wEAAwE;IACxE,QAAQ,EAAE,kBAAkB,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,mGAAmG;IACnG,UAAU,CAAC,EAAE,UAAU,EAAE,CAAC;IAC1B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,mBAAmB,CAAC;IACnC,QAAQ,EAAE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;IACzC,SAAS,CAAC,QAAQ,EAAE,mBAAmB,GAAG,YAAY,CAAC;CACxD;AAED,eAAO,MAAM,4BAA4B;;;;iBAIvC,CAAC;AAEH,eAAO,MAAM,mBAAmB;;;;;;;iBAG9B,CAAC;AAEH,0FAA0F;AAC1F,wBAAgB,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,CA4D9C;AAED,0FAA0F;AAC1F,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,CAiC7D;AAED,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAwBvD"}
|
package/dist/judge.js
CHANGED
|
@@ -48,8 +48,8 @@ export function judge(input) {
|
|
|
48
48
|
summarize: (response) => summarizeJudgeResponse(criteria, response),
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
|
-
/** Render a transcript of streamed `
|
|
52
|
-
export function formatTranscript(
|
|
51
|
+
/** Render a transcript of streamed `AgentEvent`s as readable lines for a judge prompt. */
|
|
52
|
+
export function formatTranscript(events) {
|
|
53
53
|
const lines = [];
|
|
54
54
|
const buffers = new Map();
|
|
55
55
|
const flush = (id) => {
|
|
@@ -58,8 +58,10 @@ export function formatTranscript(messages) {
|
|
|
58
58
|
lines.push(`[${buffer.kind}] ${buffer.text}`);
|
|
59
59
|
buffers.delete(id);
|
|
60
60
|
};
|
|
61
|
-
for (const
|
|
62
|
-
if (
|
|
61
|
+
for (const event of events) {
|
|
62
|
+
if (event.category === "message" &&
|
|
63
|
+
(event.event.type === "text" || event.event.type === "thought")) {
|
|
64
|
+
const message = event.event;
|
|
63
65
|
const kind = message.type === "text" ? "agent" : "thinking";
|
|
64
66
|
const buffer = buffers.get(message.message_id) ?? { kind, text: "" };
|
|
65
67
|
buffer.text += message.chunk;
|
|
@@ -68,7 +70,7 @@ export function formatTranscript(messages) {
|
|
|
68
70
|
flush(message.message_id);
|
|
69
71
|
}
|
|
70
72
|
else {
|
|
71
|
-
const formatted =
|
|
73
|
+
const formatted = eventToString(event);
|
|
72
74
|
if (formatted)
|
|
73
75
|
lines.push(formatted);
|
|
74
76
|
}
|
|
@@ -77,20 +79,29 @@ export function formatTranscript(messages) {
|
|
|
77
79
|
flush(id);
|
|
78
80
|
return lines.join("\n");
|
|
79
81
|
}
|
|
80
|
-
/** Format a single agent
|
|
81
|
-
export function
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
82
|
+
/** Format a single agent event into a readable line for a judge prompt. */
|
|
83
|
+
export function eventToString(event) {
|
|
84
|
+
if (event.category === "tool") {
|
|
85
|
+
switch (event.event.type) {
|
|
86
|
+
case "call":
|
|
87
|
+
return `[tool-call] ${event.event.request.name} arguments=${event.event.request.arguments}`.trimEnd();
|
|
88
|
+
case "result":
|
|
89
|
+
return `[tool-result] ${event.event.result.name}: ${event.event.result.result}`;
|
|
90
|
+
case "error":
|
|
91
|
+
return `[tool-error] ${event.event.error.name}: ${event.event.error.error}`;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (event.category === "turn" && event.event.type === "ended") {
|
|
95
|
+
switch (event.event.outcome.status) {
|
|
96
|
+
case "failed":
|
|
97
|
+
return `[error] ${event.event.outcome.error}`;
|
|
98
|
+
case "cancelled":
|
|
99
|
+
return "[cancelled]";
|
|
100
|
+
case "completed":
|
|
101
|
+
return "[done]";
|
|
102
|
+
}
|
|
93
103
|
}
|
|
104
|
+
return "";
|
|
94
105
|
}
|
|
95
106
|
function normalizeCriteria(criteria) {
|
|
96
107
|
if (criteria.length === 0) {
|
package/dist/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AA6CnD,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IACnD,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC;IAC/C,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;CAC3B,CAAC,CAAC;AAEH,0FAA0F;AAC1F,MAAM,UAAU,KAAK,CAAC,KAAiB;IACrC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAa;QACzB,iBAAiB;QACjB,GAAG,KAAK,CAAC,YAAY,EAAE;QAEvB,SAAS;QACT,0DAA0D,KAAK,CAAC,IAAI,SAAS;KAC9E,CAAC;IAEF,IAAI,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QACtC,QAAQ,CAAC,IAAI,CACX,qBAAqB,EACrB,0DAA0D,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,CACpH,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CACX,aAAa,EACb,2DAA2D,KAAK,CAAC,OAAO,CAAC,IAAI,SAAS,CACvF,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,CACzE;YACE,QAAQ;YACR,SAAS,IAAI,SAAS;YACtB,aAAa,OAAO,aAAa;YACjC,SAAS;SACV,CAAC,IAAI,CAAC,EAAE,CAAC,CACX,CAAC;QAEF,QAAQ,CAAC,IAAI,CACX,kBAAkB,EAClB,kCAAkC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAC9D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CACzB,CAAC,SAAS,EAAE,EAAE,CACZ,SAAS,SAAS,CAAC,EAAE,iBAAiB,SAAS,CAAC,QAAQ,eAAe,SAAS,CAAC,MAAM,kBAAkB,SAAS,CAAC,SAAS,oBAAoB,SAAS,CAAC,WAAW,EAAE,CAC1K,CAAC;IACF,QAAQ,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACzE,QAAQ,CAAC,IAAI,CACX;QACE,+EAA+E;QAC/E,oDAAoD;QACpD,uDAAuD;QACvD,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;KAC7D,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,MAAM,EAAE,mBAAmB;QAC3B,QAAQ;QACR,SAAS,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC;KACpE,CAAC;AACJ,CAAC;AAED,
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AA6CnD,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IACnD,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC;IAC/C,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;CAC3B,CAAC,CAAC;AAEH,0FAA0F;AAC1F,MAAM,UAAU,KAAK,CAAC,KAAiB;IACrC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAa;QACzB,iBAAiB;QACjB,GAAG,KAAK,CAAC,YAAY,EAAE;QAEvB,SAAS;QACT,0DAA0D,KAAK,CAAC,IAAI,SAAS;KAC9E,CAAC;IAEF,IAAI,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QACtC,QAAQ,CAAC,IAAI,CACX,qBAAqB,EACrB,0DAA0D,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,CACpH,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CACX,aAAa,EACb,2DAA2D,KAAK,CAAC,OAAO,CAAC,IAAI,SAAS,CACvF,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,CACzE;YACE,QAAQ;YACR,SAAS,IAAI,SAAS;YACtB,aAAa,OAAO,aAAa;YACjC,SAAS;SACV,CAAC,IAAI,CAAC,EAAE,CAAC,CACX,CAAC;QAEF,QAAQ,CAAC,IAAI,CACX,kBAAkB,EAClB,kCAAkC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAC9D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CACzB,CAAC,SAAS,EAAE,EAAE,CACZ,SAAS,SAAS,CAAC,EAAE,iBAAiB,SAAS,CAAC,QAAQ,eAAe,SAAS,CAAC,MAAM,kBAAkB,SAAS,CAAC,SAAS,oBAAoB,SAAS,CAAC,WAAW,EAAE,CAC1K,CAAC;IACF,QAAQ,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACzE,QAAQ,CAAC,IAAI,CACX;QACE,+EAA+E;QAC/E,oDAAoD;QACpD,uDAAuD;QACvD,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;KAC7D,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,MAAM,EAAE,mBAAmB;QAC3B,QAAQ;QACR,SAAS,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC;KACpE,CAAC;AACJ,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,gBAAgB,CAAC,MAAoB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,IAAI,GAAG,EAGpB,CAAC;IAEJ,MAAM,KAAK,GAAG,CAAC,EAAU,EAAQ,EAAE;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE;YAC9B,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrB,CAAC,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IACE,KAAK,CAAC,QAAQ,KAAK,SAAS;YAC5B,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,EAC/D,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC;YAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;YAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACxC,IAAI,OAAO,CAAC,WAAW;gBAAE,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAAE,KAAK,CAAC,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,KAAiB;IAC7C,IAAI,KAAK,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAC9B,QAAQ,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACzB,KAAK,MAAM;gBACT,OAAO,eAAe,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,cAAc,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,CAAC;YACxG,KAAK,QAAQ;gBACX,OAAO,iBAAiB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAClF,KAAK,OAAO;gBACV,OAAO,gBAAgB,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAChF,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,KAAK,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC9D,QAAQ,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACnC,KAAK,QAAQ;gBACX,OAAO,WAAW,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAChD,KAAK,WAAW;gBACd,OAAO,aAAa,CAAC;YACvB,KAAK,WAAW;gBACd,OAAO,QAAQ,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,iBAAiB,CACxB,QAA8B;IAE9B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,iBAAiB,CAAC,kCAAkC,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAChC,MAAM,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,EAAE;YAAE,MAAM,iBAAiB,CAAC,sCAAsC,CAAC,CAAC;QACzE,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,iBAAiB,CAAC,kCAAkC,EAAE,IAAI,CAAC,CAAC;QACpE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEZ,MAAM,UAAU,GAAG;YACjB,EAAE;YACF,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ,IAAI,IAAI;YACpC,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,CAAC;YAC7B,SAAS,EAAE,SAAS,CAAC,SAAS,IAAI,CAAC;SACpC,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YACnC,MAAM,iBAAiB,CACrB,qBAAqB,EAAE,kCAAkC,CAC1D,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClE,MAAM,iBAAiB,CACrB,qBAAqB,EAAE,kCAAkC,CAC1D,CAAC;QACJ,CAAC;QAED,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC;YACtC,UAAU,CAAC,SAAS,GAAG,CAAC;YACxB,UAAU,CAAC,SAAS,GAAG,CAAC,EACxB,CAAC;YACD,MAAM,iBAAiB,CACrB,qBAAqB,EAAE,0CAA0C,CAClE,CAAC;QACJ,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAAwC,EACxC,QAA6B;IAE7B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkC,CAAC;IAC5D,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC1C,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,eAAe,CACnB,qCAAqC,SAAS,CAAC,EAAE,IAAI,CACtD,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,SAAS,GAA4B,EAAE,CAAC;IAC9C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,cAAc,GAAG,KAAK,CAAC;IAE3B,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;QACjC,MAAM,iBAAiB,GAAG,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,MAAM,eAAe,CAAC,gCAAgC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;QAC1E,CAAC;QACD,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE/B,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,IAAI,SAAS,CAAC,SAAS,CAAC;QAC9D,cAAc,KAAK,SAAS,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC;QACjD,aAAa,IAAI,iBAAiB,CAAC,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QAC5D,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC;QAChC,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,KAAK,EAAE,iBAAiB,CAAC,KAAK;YAC9B,MAAM;YACN,MAAM,EAAE,iBAAiB,CAAC,MAAM;SACjC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAA2B,CAAC;IACtE,IAAI,SAAS;QACX,MAAM,eAAe,CAAC,gCAAgC,SAAS,IAAI,CAAC,CAAC;IAEvE,aAAa,IAAI,WAAW,CAAC;IAC7B,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;IACjD,MAAM,MAAM,GAAG,cAAc;QAC3B,CAAC,CAAC,kBAAkB,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,kCAAkC,QAAQ,CAAC,cAAc,EAAE;QACvG,CAAC,CAAC,kBAAkB,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,QAAQ,CAAC,cAAc,EAAE,CAAC;IAE/F,OAAO,EAAE,MAAM,EAAE,CAAC,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACxC,OAAO,IAAI,cAAc,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO,IAAI,cAAc,CAAC,yBAAyB,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}
|
package/dist/transcript.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AgentEvent, ContextUsage, TurnOutcome } from "@aether-agent/sdk";
|
|
2
2
|
export declare class ToolCall {
|
|
3
3
|
readonly name: string;
|
|
4
4
|
readonly arguments: string;
|
|
@@ -10,15 +10,17 @@ export declare class TranscriptError extends Error {
|
|
|
10
10
|
constructor(transcript: Transcript, cause: unknown);
|
|
11
11
|
}
|
|
12
12
|
export declare class Transcript {
|
|
13
|
-
readonly
|
|
14
|
-
constructor(
|
|
15
|
-
static fromStream(stream: AsyncIterable<
|
|
16
|
-
add(
|
|
13
|
+
readonly events: AgentEvent[];
|
|
14
|
+
constructor(events?: AgentEvent[]);
|
|
15
|
+
static fromStream(stream: AsyncIterable<AgentEvent>): Promise<Transcript>;
|
|
16
|
+
add(event: AgentEvent): void;
|
|
17
17
|
allToolCalls(): ToolCall[];
|
|
18
18
|
toolCalls(name: string): ToolCall[];
|
|
19
19
|
toolCalled(name: string): boolean;
|
|
20
20
|
toolCallCount(name: string): number;
|
|
21
21
|
usage(): ContextUsage;
|
|
22
22
|
}
|
|
23
|
-
export declare function
|
|
23
|
+
export declare function isTerminalEvent(event: AgentEvent): boolean;
|
|
24
|
+
/** Build the terminal turn event, defaulting to a completed turn. */
|
|
25
|
+
export declare function turnEnded(outcome?: TurnOutcome): AgentEvent;
|
|
24
26
|
//# sourceMappingURL=transcript.d.ts.map
|
package/dist/transcript.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transcript.d.ts","sourceRoot":"","sources":["../src/transcript.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"transcript.d.ts","sourceRoot":"","sources":["../src/transcript.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE/E,qBAAa,QAAQ;IAIjB,QAAQ,CAAC,IAAI,EAAE,MAAM;IAHvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;gBAGhB,IAAI,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM;IAKxB,aAAa,IAAI,OAAO;CAGzB;AAED,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;gBAEpB,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO;CAKnD;AAED,qBAAa,UAAU;IACrB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;gBAElB,MAAM,GAAE,UAAU,EAAO;WAIxB,UAAU,CACrB,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,GAChC,OAAO,CAAC,UAAU,CAAC;IAYtB,GAAG,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAI5B,YAAY,IAAI,QAAQ,EAAE;IAI1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE;IAInC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIjC,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAInC,KAAK,IAAI,YAAY;CAGtB;AAiBD,wBAAgB,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAE1D;AAED,qEAAqE;AACrE,wBAAgB,SAAS,CACvB,OAAO,GAAE,WAAqC,GAC7C,UAAU,CAEZ"}
|
package/dist/transcript.js
CHANGED
|
@@ -18,15 +18,15 @@ export class TranscriptError extends Error {
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
export class Transcript {
|
|
21
|
-
|
|
22
|
-
constructor(
|
|
23
|
-
this.
|
|
21
|
+
events;
|
|
22
|
+
constructor(events = []) {
|
|
23
|
+
this.events = [...events];
|
|
24
24
|
}
|
|
25
25
|
static async fromStream(stream) {
|
|
26
26
|
const transcript = new Transcript();
|
|
27
27
|
try {
|
|
28
|
-
for await (const
|
|
29
|
-
transcript.add(
|
|
28
|
+
for await (const event of stream) {
|
|
29
|
+
transcript.add(event);
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
catch (err) {
|
|
@@ -34,11 +34,11 @@ export class Transcript {
|
|
|
34
34
|
}
|
|
35
35
|
return transcript;
|
|
36
36
|
}
|
|
37
|
-
add(
|
|
38
|
-
this.
|
|
37
|
+
add(event) {
|
|
38
|
+
this.events.push(event);
|
|
39
39
|
}
|
|
40
40
|
allToolCalls() {
|
|
41
|
-
return extractToolCalls(this.
|
|
41
|
+
return extractToolCalls(this.events);
|
|
42
42
|
}
|
|
43
43
|
toolCalls(name) {
|
|
44
44
|
return this.allToolCalls().filter((call) => call.name === name);
|
|
@@ -50,11 +50,10 @@ export class Transcript {
|
|
|
50
50
|
return this.toolCalls(name).length;
|
|
51
51
|
}
|
|
52
52
|
usage() {
|
|
53
|
-
return summarizeUsage(this.
|
|
53
|
+
return summarizeUsage(this.events);
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
const ZERO_USAGE = {
|
|
57
|
-
type: "context_usage",
|
|
58
57
|
input_tokens: 0,
|
|
59
58
|
output_tokens: 0,
|
|
60
59
|
cache_read_tokens: null,
|
|
@@ -68,28 +67,33 @@ const ZERO_USAGE = {
|
|
|
68
67
|
total_cache_creation_tokens: 0,
|
|
69
68
|
total_reasoning_tokens: 0,
|
|
70
69
|
};
|
|
71
|
-
export function
|
|
72
|
-
return
|
|
73
|
-
message.type === "error" ||
|
|
74
|
-
message.type === "cancelled");
|
|
70
|
+
export function isTerminalEvent(event) {
|
|
71
|
+
return event.category === "turn" && event.event.type === "ended";
|
|
75
72
|
}
|
|
76
|
-
|
|
73
|
+
/** Build the terminal turn event, defaulting to a completed turn. */
|
|
74
|
+
export function turnEnded(outcome = { status: "completed" }) {
|
|
75
|
+
return { category: "turn", event: { type: "ended", outcome } };
|
|
76
|
+
}
|
|
77
|
+
function extractToolCalls(events) {
|
|
77
78
|
const calls = [];
|
|
78
|
-
for (const
|
|
79
|
-
if (
|
|
80
|
-
|
|
79
|
+
for (const event of events) {
|
|
80
|
+
if (event.category !== "tool")
|
|
81
|
+
continue;
|
|
82
|
+
const tool = event.event;
|
|
83
|
+
if (tool.type === "result") {
|
|
84
|
+
calls.push(new ToolCall(tool.result.name, tool.result.arguments));
|
|
81
85
|
}
|
|
82
|
-
else if (
|
|
83
|
-
calls.push(new ToolCall(
|
|
86
|
+
else if (tool.type === "error") {
|
|
87
|
+
calls.push(new ToolCall(tool.error.name, tool.error.arguments ?? ""));
|
|
84
88
|
}
|
|
85
89
|
}
|
|
86
90
|
return calls;
|
|
87
91
|
}
|
|
88
|
-
function summarizeUsage(
|
|
89
|
-
for (let i =
|
|
90
|
-
const
|
|
91
|
-
if (
|
|
92
|
-
return
|
|
92
|
+
function summarizeUsage(events) {
|
|
93
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
94
|
+
const event = events[i];
|
|
95
|
+
if (event?.category === "context" && event.event.type === "usage_updated")
|
|
96
|
+
return event.event.usage;
|
|
93
97
|
}
|
|
94
98
|
return { ...ZERO_USAGE };
|
|
95
99
|
}
|
package/dist/transcript.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transcript.js","sourceRoot":"","sources":["../src/transcript.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,QAAQ;IAIR;IAHF,SAAS,CAAS;IAE3B,YACW,IAAY,EACrB,cAAsB;QADb,SAAI,GAAJ,IAAI,CAAQ;QAGrB,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC;IAClC,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,UAAU,CAAa;IAEhC,YAAY,UAAsB,EAAE,KAAc;QAChD,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,UAAU;IACZ,
|
|
1
|
+
{"version":3,"file":"transcript.js","sourceRoot":"","sources":["../src/transcript.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,QAAQ;IAIR;IAHF,SAAS,CAAS;IAE3B,YACW,IAAY,EACrB,cAAsB;QADb,SAAI,GAAJ,IAAI,CAAQ;QAGrB,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC;IAClC,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,UAAU,CAAa;IAEhC,YAAY,UAAsB,EAAE,KAAc;QAChD,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,UAAU;IACZ,MAAM,CAAe;IAE9B,YAAY,SAAuB,EAAE;QACnC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,MAAiC;QAEjC,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,eAAe,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,GAAG,CAAC,KAAiB;QACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,YAAY;QACV,OAAO,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,aAAa,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,KAAK;QACH,OAAO,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAM,UAAU,GAAiB;IAC/B,YAAY,EAAE,CAAC;IACf,aAAa,EAAE,CAAC;IAChB,iBAAiB,EAAE,IAAI;IACvB,qBAAqB,EAAE,IAAI;IAC3B,gBAAgB,EAAE,IAAI;IACtB,WAAW,EAAE,IAAI;IACjB,aAAa,EAAE,IAAI;IACnB,kBAAkB,EAAE,CAAC;IACrB,mBAAmB,EAAE,CAAC;IACtB,uBAAuB,EAAE,CAAC;IAC1B,2BAA2B,EAAE,CAAC;IAC9B,sBAAsB,EAAE,CAAC;CAC1B,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,OAAO,KAAK,CAAC,QAAQ,KAAK,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;AACnE,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,SAAS,CACvB,UAAuB,EAAE,MAAM,EAAE,WAAW,EAAE;IAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;AACjE,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAoB;IAC5C,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,QAAQ,KAAK,MAAM;YAAE,SAAS;QACxC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,MAAoB;IAC1C,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,KAAK,EAAE,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,eAAe;YACvE,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;IAC7B,CAAC;IACD,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC;AAC3B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aether-agent/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Evaluation harness for the Aether agent SDK",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -19,9 +19,9 @@
|
|
|
19
19
|
"README.md"
|
|
20
20
|
],
|
|
21
21
|
"dependencies": {
|
|
22
|
+
"@aether-agent/sdk": "^0.4.1",
|
|
22
23
|
"testcontainers": "^12.0.3",
|
|
23
|
-
"zod": "^4.4.3"
|
|
24
|
-
"@aether-agent/sdk": "^0.3.4"
|
|
24
|
+
"zod": "^4.4.3"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
27
|
"@types/dockerode": "^3.3.31",
|