@anvia/core 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -1
- package/dist/agent/agent-tool.js +7 -7
- package/dist/agent/agent.js +25 -25
- package/dist/agent/index.js +29 -29
- package/dist/agent/resolve-options.js +10 -10
- package/dist/agent/resolved-agent.js +26 -26
- package/dist/agent/team/agent-team.js +29 -29
- package/dist/agent/team/index.js +29 -29
- package/dist/agent/tool-catalog.js +9 -9
- package/dist/{chunk-QHGYCYV6.js → chunk-3FIX7GLM.js} +2 -2
- package/dist/{chunk-SIXS4CCU.js → chunk-4IOC4SP6.js} +4 -4
- package/dist/chunk-4IOC4SP6.js.map +1 -0
- package/dist/{chunk-JMMZICU7.js → chunk-4OSA3MMA.js} +39 -5
- package/dist/chunk-4OSA3MMA.js.map +1 -0
- package/dist/{chunk-Y365ABZY.js → chunk-4WSICA6D.js} +2 -2
- package/dist/{chunk-YOLOFTAY.js → chunk-57AUTEQ6.js} +57 -5
- package/dist/chunk-57AUTEQ6.js.map +1 -0
- package/dist/{chunk-647TQE3E.js → chunk-5BO4OTTK.js} +5 -5
- package/dist/{chunk-ZQC6M22O.js → chunk-7LBUXGJI.js} +2 -2
- package/dist/{chunk-FGOVEUVK.js → chunk-7OHHMYRO.js} +2 -2
- package/dist/{chunk-DXW37JLZ.js → chunk-AMPQJMDG.js} +5 -5
- package/dist/{chunk-HAAGSIZH.js → chunk-CB2L2NIQ.js} +2 -2
- package/dist/{chunk-I7YCEQXT.js → chunk-D6FK5ZQR.js} +5 -5
- package/dist/{chunk-B57NNDEF.js → chunk-FTK6YRLN.js} +5 -5
- package/dist/{chunk-T3ZOZEAB.js → chunk-GB4SRUT7.js} +2 -2
- package/dist/{chunk-KX2TRKO2.js → chunk-GNBPXJC6.js} +2 -2
- package/dist/{chunk-AYN34XVW.js → chunk-GRI4EFNG.js} +15 -15
- package/dist/chunk-GRI4EFNG.js.map +1 -0
- package/dist/{chunk-CJKGTEN4.js → chunk-HDL5TXF7.js} +2 -2
- package/dist/{chunk-YMTBHHLQ.js → chunk-HG6ASDNI.js} +3 -3
- package/dist/{chunk-XAVSWLX5.js → chunk-IMSM4HHI.js} +3 -3
- package/dist/{chunk-35GMQS3R.js → chunk-KGVDUYZV.js} +3 -3
- package/dist/{chunk-E44DRLUL.js → chunk-LW6F3IGP.js} +10 -6
- package/dist/chunk-LW6F3IGP.js.map +1 -0
- package/dist/{chunk-UEUSJ3UQ.js → chunk-MJAM7HZB.js} +8 -8
- package/dist/{chunk-2JZV4LUS.js → chunk-MKY7KTZY.js} +3 -13
- package/dist/chunk-MKY7KTZY.js.map +1 -0
- package/dist/{chunk-RFFOL6ON.js → chunk-MWG2PMTE.js} +18 -5
- package/dist/chunk-MWG2PMTE.js.map +1 -0
- package/dist/{chunk-JMNIWUFV.js → chunk-NKSXYOUM.js} +3 -3
- package/dist/{chunk-4VDDYI6P.js → chunk-OBTD3GPY.js} +2 -2
- package/dist/{chunk-GDY2Z2SZ.js → chunk-PFJN5OIQ.js} +2 -2
- package/dist/chunk-PZC5UHIN.js +190 -0
- package/dist/chunk-PZC5UHIN.js.map +1 -0
- package/dist/{chunk-UYODZ7OE.js → chunk-STEAW4JD.js} +2 -2
- package/dist/{chunk-LJGQ6VQB.js → chunk-TD2VYYUT.js} +90 -31
- package/dist/chunk-TD2VYYUT.js.map +1 -0
- package/dist/{chunk-MTJDYC6T.js → chunk-TOJSC4GI.js} +3 -3
- package/dist/{chunk-V5DWNXVH.js → chunk-W2XPREWL.js} +135 -26
- package/dist/chunk-W2XPREWL.js.map +1 -0
- package/dist/chunk-W6V63YPY.js +96 -0
- package/dist/chunk-W6V63YPY.js.map +1 -0
- package/dist/{chunk-F4BOW76L.js → chunk-XQTTPQWA.js} +4 -4
- package/dist/{chunk-HHMLE3TZ.js → chunk-YCTXTOOQ.js} +7 -16
- package/dist/chunk-YCTXTOOQ.js.map +1 -0
- package/dist/{chunk-7TUAAGVR.js → chunk-YKWUFYXT.js} +2 -2
- package/dist/{chunk-5SJBSCG2.js → chunk-YSR6KLCS.js} +8 -2
- package/dist/chunk-YSR6KLCS.js.map +1 -0
- package/dist/{chunk-6E6ABRJ6.js → chunk-ZJ4T7CEX.js} +2 -2
- package/dist/completion/generate-completion.js +3 -3
- package/dist/completion/index.js +3 -3
- package/dist/completion/stream-accumulator.js +1 -1
- package/dist/embeddings/embed.js +1 -1
- package/dist/embeddings/index.js +1 -1
- package/dist/evals/advanced-metrics.js +6 -6
- package/dist/evals/agent-target.js +2 -1
- package/dist/evals/cli.js +6 -6
- package/dist/evals/execution.js +3 -1
- package/dist/evals/index.js +12 -12
- package/dist/evals/judge.js +5 -5
- package/dist/evals/metrics.js +6 -6
- package/dist/evals/runner.js +5 -5
- package/dist/extractor/extractor.js +4 -4
- package/dist/extractor/index.js +4 -4
- package/dist/guardrails/index.js +4 -4
- package/dist/index.js +35 -35
- package/dist/internal/agent-runtime/agent-run.js +21 -21
- package/dist/internal/agent-runtime/continuation-state.js +3 -3
- package/dist/internal/agent-runtime/interaction-suspension.js +3 -3
- package/dist/internal/agent-runtime/memory.js +5 -5
- package/dist/internal/agent-runtime/prepared-tool-call.js +2 -2
- package/dist/internal/agent-runtime/retrieval.js +8 -8
- package/dist/internal/agent-runtime/tool-execution.js +15 -15
- package/dist/internal/agent.js +26 -26
- package/dist/internal/team-runtime/member.js +3 -3
- package/dist/internal/team-runtime/run.js +28 -28
- package/dist/internal/team-runtime/tools.js +4 -4
- package/dist/memory/compaction.js +4 -4
- package/dist/memory/index.js +4 -4
- package/dist/observability/group.js +3 -3
- package/dist/observability/index.js +3 -3
- package/dist/pipeline/index.js +6 -6
- package/dist/pipeline/pipeline.js +6 -6
- package/dist/schema/standard-json-schema.js +1 -1
- package/dist/skills/index.js +13 -13
- package/dist/skills/load.js +13 -13
- package/dist/skills/tools.js +12 -12
- package/dist/streaming/index.d.ts +2 -1
- package/dist/streaming/index.js +2 -1
- package/dist/streaming/readable-stream.js +2 -1
- package/dist/tool/create-tool.js +3 -3
- package/dist/tool/dynamic-tools.js +8 -8
- package/dist/tool/index.js +14 -14
- package/dist/tool/question-tool.js +4 -4
- package/dist/tool/think-tool.js +4 -4
- package/dist/vector-store/index.js +7 -7
- package/dist/vector-store/ingest.js +2 -2
- package/dist/vector-store/retrieve.js +2 -2
- package/package.json +2 -2
- package/dist/chunk-2JZV4LUS.js.map +0 -1
- package/dist/chunk-54ZZJRP2.js +0 -57
- package/dist/chunk-54ZZJRP2.js.map +0 -1
- package/dist/chunk-5SJBSCG2.js.map +0 -1
- package/dist/chunk-AYN34XVW.js.map +0 -1
- package/dist/chunk-E44DRLUL.js.map +0 -1
- package/dist/chunk-HHMLE3TZ.js.map +0 -1
- package/dist/chunk-JMMZICU7.js.map +0 -1
- package/dist/chunk-LJGQ6VQB.js.map +0 -1
- package/dist/chunk-NNEKBZSM.js +0 -66
- package/dist/chunk-NNEKBZSM.js.map +0 -1
- package/dist/chunk-RFFOL6ON.js.map +0 -1
- package/dist/chunk-SIXS4CCU.js.map +0 -1
- package/dist/chunk-V5DWNXVH.js.map +0 -1
- package/dist/chunk-YOLOFTAY.js.map +0 -1
- /package/dist/{chunk-QHGYCYV6.js.map → chunk-3FIX7GLM.js.map} +0 -0
- /package/dist/{chunk-Y365ABZY.js.map → chunk-4WSICA6D.js.map} +0 -0
- /package/dist/{chunk-647TQE3E.js.map → chunk-5BO4OTTK.js.map} +0 -0
- /package/dist/{chunk-ZQC6M22O.js.map → chunk-7LBUXGJI.js.map} +0 -0
- /package/dist/{chunk-FGOVEUVK.js.map → chunk-7OHHMYRO.js.map} +0 -0
- /package/dist/{chunk-DXW37JLZ.js.map → chunk-AMPQJMDG.js.map} +0 -0
- /package/dist/{chunk-HAAGSIZH.js.map → chunk-CB2L2NIQ.js.map} +0 -0
- /package/dist/{chunk-I7YCEQXT.js.map → chunk-D6FK5ZQR.js.map} +0 -0
- /package/dist/{chunk-B57NNDEF.js.map → chunk-FTK6YRLN.js.map} +0 -0
- /package/dist/{chunk-T3ZOZEAB.js.map → chunk-GB4SRUT7.js.map} +0 -0
- /package/dist/{chunk-KX2TRKO2.js.map → chunk-GNBPXJC6.js.map} +0 -0
- /package/dist/{chunk-CJKGTEN4.js.map → chunk-HDL5TXF7.js.map} +0 -0
- /package/dist/{chunk-YMTBHHLQ.js.map → chunk-HG6ASDNI.js.map} +0 -0
- /package/dist/{chunk-XAVSWLX5.js.map → chunk-IMSM4HHI.js.map} +0 -0
- /package/dist/{chunk-35GMQS3R.js.map → chunk-KGVDUYZV.js.map} +0 -0
- /package/dist/{chunk-UEUSJ3UQ.js.map → chunk-MJAM7HZB.js.map} +0 -0
- /package/dist/{chunk-JMNIWUFV.js.map → chunk-NKSXYOUM.js.map} +0 -0
- /package/dist/{chunk-4VDDYI6P.js.map → chunk-OBTD3GPY.js.map} +0 -0
- /package/dist/{chunk-GDY2Z2SZ.js.map → chunk-PFJN5OIQ.js.map} +0 -0
- /package/dist/{chunk-UYODZ7OE.js.map → chunk-STEAW4JD.js.map} +0 -0
- /package/dist/{chunk-MTJDYC6T.js.map → chunk-TOJSC4GI.js.map} +0 -0
- /package/dist/{chunk-F4BOW76L.js.map → chunk-XQTTPQWA.js.map} +0 -0
- /package/dist/{chunk-7TUAAGVR.js.map → chunk-YKWUFYXT.js.map} +0 -0
- /package/dist/{chunk-6E6ABRJ6.js.map → chunk-ZJ4T7CEX.js.map} +0 -0
|
@@ -3,7 +3,11 @@ import {
|
|
|
3
3
|
} from "./chunk-2NLINXRF.js";
|
|
4
4
|
import {
|
|
5
5
|
createTool
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-4WSICA6D.js";
|
|
7
|
+
import {
|
|
8
|
+
abortError,
|
|
9
|
+
throwIfAborted
|
|
10
|
+
} from "./chunk-USRKPEQN.js";
|
|
7
11
|
|
|
8
12
|
// src/skills/tools.ts
|
|
9
13
|
import { spawn } from "child_process";
|
|
@@ -12,6 +16,8 @@ import { isAbsolute, relative, resolve } from "path";
|
|
|
12
16
|
import { z } from "zod";
|
|
13
17
|
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
14
18
|
var MAX_OUTPUT_CHARS = 2e4;
|
|
19
|
+
var TERMINATION_GRACE_MS = 250;
|
|
20
|
+
var TERMINATION_WATCHDOG_MS = 1250;
|
|
15
21
|
function createSkillTools(skills) {
|
|
16
22
|
const registry = new SkillRegistry(skills);
|
|
17
23
|
return [
|
|
@@ -61,7 +67,7 @@ function createSkillTools(skills) {
|
|
|
61
67
|
timeoutMs: z.number().int().positive().optional().describe("Execution timeout in ms.")
|
|
62
68
|
}),
|
|
63
69
|
outputSchema: z.string(),
|
|
64
|
-
execute: ({ skillName, scriptPath, args = [], timeoutMs = DEFAULT_TIMEOUT_MS }) => registry.runScript(skillName, scriptPath, args, timeoutMs)
|
|
70
|
+
execute: ({ skillName, scriptPath, args = [], timeoutMs = DEFAULT_TIMEOUT_MS }, context) => registry.runScript(skillName, scriptPath, args, timeoutMs, context.abortSignal)
|
|
65
71
|
})
|
|
66
72
|
)
|
|
67
73
|
];
|
|
@@ -96,13 +102,13 @@ var SkillRegistry = class {
|
|
|
96
102
|
}
|
|
97
103
|
return readFile(path, "utf8");
|
|
98
104
|
}
|
|
99
|
-
async runScript(skillName, scriptPath, args, timeoutMs) {
|
|
105
|
+
async runScript(skillName, scriptPath, args, timeoutMs, abortSignal) {
|
|
100
106
|
const skill = this.get(skillName);
|
|
101
107
|
const script = this.resolveContainedPath(skill, "scripts", scriptPath);
|
|
102
108
|
if (!skill.scripts.includes(scriptPath)) {
|
|
103
109
|
throw new Error(`Skill script not found: ${skillName}/${scriptPath}`);
|
|
104
110
|
}
|
|
105
|
-
return runExecutable(script, args, skill.directory, timeoutMs);
|
|
111
|
+
return runExecutable(script, args, skill.directory, timeoutMs, abortSignal);
|
|
106
112
|
}
|
|
107
113
|
resolveContainedPath(skill, section, requestedPath) {
|
|
108
114
|
if (requestedPath.length === 0 || isAbsolute(requestedPath)) {
|
|
@@ -117,8 +123,9 @@ var SkillRegistry = class {
|
|
|
117
123
|
return resolved;
|
|
118
124
|
}
|
|
119
125
|
};
|
|
120
|
-
function runExecutable(command, args, cwd, timeoutMs) {
|
|
126
|
+
function runExecutable(command, args, cwd, timeoutMs, abortSignal) {
|
|
121
127
|
return new Promise((resolvePromise, reject) => {
|
|
128
|
+
throwIfAborted(abortSignal);
|
|
122
129
|
const child = spawn(command, args, {
|
|
123
130
|
cwd,
|
|
124
131
|
shell: false,
|
|
@@ -126,40 +133,92 @@ function runExecutable(command, args, cwd, timeoutMs) {
|
|
|
126
133
|
});
|
|
127
134
|
let stdout = "";
|
|
128
135
|
let stderr = "";
|
|
129
|
-
let
|
|
136
|
+
let exited = false;
|
|
137
|
+
let settled = false;
|
|
138
|
+
let firstCause;
|
|
139
|
+
let escalation;
|
|
140
|
+
let watchdog;
|
|
130
141
|
const timeout = setTimeout(() => {
|
|
131
|
-
|
|
132
|
-
child.kill();
|
|
142
|
+
beginTermination(new Error(`Skill script timed out after ${timeoutMs}ms`));
|
|
133
143
|
}, timeoutMs);
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
stdout = appendLimited(stdout, chunk);
|
|
138
|
-
});
|
|
139
|
-
child.stderr.on("data", (chunk) => {
|
|
140
|
-
stderr = appendLimited(stderr, chunk);
|
|
141
|
-
});
|
|
142
|
-
child.on("error", (error) => {
|
|
144
|
+
function settle(error, output = "") {
|
|
145
|
+
if (settled) return;
|
|
146
|
+
settled = true;
|
|
143
147
|
clearTimeout(timeout);
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
148
|
+
clearTimeout(escalation);
|
|
149
|
+
clearTimeout(watchdog);
|
|
150
|
+
abortSignal?.removeEventListener("abort", onAbort);
|
|
151
|
+
child.removeListener("error", onError);
|
|
152
|
+
child.removeListener("exit", onExit);
|
|
153
|
+
child.removeListener("close", onClose);
|
|
154
|
+
child.stdout.removeListener("data", onStdout);
|
|
155
|
+
child.stderr.removeListener("data", onStderr);
|
|
156
|
+
if (error !== void 0) reject(error);
|
|
157
|
+
else resolvePromise(output);
|
|
158
|
+
}
|
|
159
|
+
function finishTermination() {
|
|
160
|
+
child.stdout.destroy();
|
|
161
|
+
child.stderr.destroy();
|
|
162
|
+
settle(firstCause);
|
|
163
|
+
}
|
|
164
|
+
function beginTermination(cause) {
|
|
165
|
+
if (settled || firstCause !== void 0) return;
|
|
166
|
+
firstCause = cause;
|
|
147
167
|
clearTimeout(timeout);
|
|
148
|
-
if (
|
|
149
|
-
|
|
168
|
+
if (exited) {
|
|
169
|
+
finishTermination();
|
|
150
170
|
return;
|
|
151
171
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
172
|
+
escalation = setTimeout(() => {
|
|
173
|
+
if (!exited) child.kill("SIGKILL");
|
|
174
|
+
}, TERMINATION_GRACE_MS);
|
|
175
|
+
watchdog = setTimeout(() => {
|
|
176
|
+
child.stdout.destroy();
|
|
177
|
+
child.stderr.destroy();
|
|
178
|
+
settle(new Error("Skill script termination could not be confirmed", { cause: firstCause }));
|
|
179
|
+
}, TERMINATION_WATCHDOG_MS);
|
|
180
|
+
child.kill("SIGTERM");
|
|
181
|
+
}
|
|
182
|
+
function onAbort() {
|
|
183
|
+
beginTermination(abortError(abortSignal?.reason));
|
|
184
|
+
}
|
|
185
|
+
function onError(error) {
|
|
186
|
+
if (child.pid === void 0) settle(error);
|
|
187
|
+
else beginTermination(error);
|
|
188
|
+
}
|
|
189
|
+
function onExit() {
|
|
190
|
+
exited = true;
|
|
191
|
+
if (firstCause !== void 0) finishTermination();
|
|
192
|
+
}
|
|
193
|
+
function onClose(code, signal) {
|
|
194
|
+
if (firstCause !== void 0) {
|
|
195
|
+
if (exited) finishTermination();
|
|
155
196
|
return;
|
|
156
197
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
198
|
+
const output = formatProcessOutput(stdout, stderr);
|
|
199
|
+
if (code !== 0) {
|
|
200
|
+
settle(new Error(`Skill script exited with code ${code ?? "unknown"}: ${output}`));
|
|
201
|
+
} else if (signal !== null) {
|
|
202
|
+
settle(new Error(`Skill script exited with signal ${signal}: ${output}`));
|
|
203
|
+
} else {
|
|
204
|
+
settle(void 0, output);
|
|
160
205
|
}
|
|
161
|
-
|
|
162
|
-
|
|
206
|
+
}
|
|
207
|
+
function onStdout(chunk) {
|
|
208
|
+
stdout = appendLimited(stdout, chunk);
|
|
209
|
+
}
|
|
210
|
+
function onStderr(chunk) {
|
|
211
|
+
stderr = appendLimited(stderr, chunk);
|
|
212
|
+
}
|
|
213
|
+
child.stdout.setEncoding("utf8");
|
|
214
|
+
child.stderr.setEncoding("utf8");
|
|
215
|
+
child.stdout.on("data", onStdout);
|
|
216
|
+
child.stderr.on("data", onStderr);
|
|
217
|
+
child.on("error", onError);
|
|
218
|
+
child.on("exit", onExit);
|
|
219
|
+
child.on("close", onClose);
|
|
220
|
+
abortSignal?.addEventListener("abort", onAbort, { once: true });
|
|
221
|
+
if (abortSignal?.aborted === true) onAbort();
|
|
163
222
|
});
|
|
164
223
|
}
|
|
165
224
|
function formatProcessOutput(stdout, stderr) {
|
|
@@ -186,4 +245,4 @@ function appendLimited(current, chunk) {
|
|
|
186
245
|
export {
|
|
187
246
|
createSkillTools
|
|
188
247
|
};
|
|
189
|
-
//# sourceMappingURL=chunk-
|
|
248
|
+
//# sourceMappingURL=chunk-TD2VYYUT.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/skills/tools.ts"],"sourcesContent":["import { spawn } from \"node:child_process\";\nimport { readFile } from \"node:fs/promises\";\nimport { isAbsolute, relative, resolve } from \"node:path\";\nimport { z } from \"zod\";\nimport { abortError, throwIfAborted } from \"../internal/abort\";\nimport { type AnyTool, createTool } from \"../tool\";\nimport { markSkillTool } from \"../tool/skill-tool-marker\";\nimport type { Skill } from \"./types\";\n\nconst DEFAULT_TIMEOUT_MS = 30_000;\nconst MAX_OUTPUT_CHARS = 20_000;\nconst TERMINATION_GRACE_MS = 250;\nconst TERMINATION_WATCHDOG_MS = 1_250;\n\nexport function createSkillTools(skills: Skill[]): AnyTool[] {\n const registry = new SkillRegistry(skills);\n\n return [\n markSkillTool(\n createTool({\n name: \"get_skill_instructions\",\n description: \"Load the full SKILL.md instructions for an Agent Skill.\",\n inputSchema: z.object({\n skillName: z.string().describe(\"The name of the skill to load.\"),\n }),\n outputSchema: z.string(),\n execute: ({ skillName }) => registry.get(skillName).instructions,\n }),\n ),\n markSkillTool(\n createTool({\n name: \"get_skill_reference\",\n description: \"Read a reference file from an Agent Skill.\",\n inputSchema: z.object({\n skillName: z.string().describe(\"The name of the skill.\"),\n referencePath: z.string().describe(\"A path listed in the skill references.\"),\n }),\n outputSchema: z.string(),\n execute: ({ skillName, referencePath }) => registry.readReference(skillName, referencePath),\n }),\n ),\n markSkillTool(\n createTool({\n name: \"get_skill_script\",\n description: \"Read a script file from an Agent Skill.\",\n inputSchema: z.object({\n skillName: z.string().describe(\"The name of the skill.\"),\n scriptPath: z.string().describe(\"A path listed in the skill scripts.\"),\n }),\n outputSchema: z.string(),\n execute: ({ skillName, scriptPath }) => registry.readScript(skillName, scriptPath),\n }),\n ),\n markSkillTool(\n createTool({\n name: \"run_skill_script\",\n description: \"Execute a script from an Agent Skill with optional arguments.\",\n inputSchema: z.object({\n skillName: z.string().describe(\"The name of the skill.\"),\n scriptPath: z.string().describe(\"A path listed in the skill scripts.\"),\n args: z.array(z.string()).optional().describe(\"Arguments passed to the script.\"),\n timeoutMs: z.number().int().positive().optional().describe(\"Execution timeout in ms.\"),\n }),\n outputSchema: z.string(),\n execute: ({ skillName, scriptPath, args = [], timeoutMs = DEFAULT_TIMEOUT_MS }, context) =>\n registry.runScript(skillName, scriptPath, args, timeoutMs, context.abortSignal),\n }),\n ),\n ];\n}\n\nclass SkillRegistry {\n private readonly skills = new Map<string, Skill>();\n\n constructor(skills: Skill[]) {\n for (const skill of skills) {\n this.skills.set(skill.name, skill);\n }\n }\n\n get(skillName: string): Skill {\n const skill = this.skills.get(skillName);\n if (skill === undefined) {\n throw new Error(`Skill not found: ${skillName}`);\n }\n return skill;\n }\n\n async readReference(skillName: string, referencePath: string): Promise<string> {\n const skill = this.get(skillName);\n const path = this.resolveContainedPath(skill, \"references\", referencePath);\n if (!skill.references.includes(referencePath)) {\n throw new Error(`Skill reference not found: ${skillName}/${referencePath}`);\n }\n return readFile(path, \"utf8\");\n }\n\n async readScript(skillName: string, scriptPath: string): Promise<string> {\n const skill = this.get(skillName);\n const path = this.resolveContainedPath(skill, \"scripts\", scriptPath);\n if (!skill.scripts.includes(scriptPath)) {\n throw new Error(`Skill script not found: ${skillName}/${scriptPath}`);\n }\n return readFile(path, \"utf8\");\n }\n\n async runScript(\n skillName: string,\n scriptPath: string,\n args: string[],\n timeoutMs: number,\n abortSignal?: AbortSignal,\n ): Promise<string> {\n const skill = this.get(skillName);\n const script = this.resolveContainedPath(skill, \"scripts\", scriptPath);\n if (!skill.scripts.includes(scriptPath)) {\n throw new Error(`Skill script not found: ${skillName}/${scriptPath}`);\n }\n return runExecutable(script, args, skill.directory, timeoutMs, abortSignal);\n }\n\n private resolveContainedPath(\n skill: Skill,\n section: \"references\" | \"scripts\",\n requestedPath: string,\n ): string {\n if (requestedPath.length === 0 || isAbsolute(requestedPath)) {\n throw new Error(`Invalid skill path: ${requestedPath}`);\n }\n\n const root = resolve(skill.directory, section);\n const resolved = resolve(root, requestedPath);\n const rel = relative(root, resolved);\n if (rel === \"..\" || rel.startsWith(`..${\"/\"}`) || rel.startsWith(`..${\"\\\\\"}`)) {\n throw new Error(`Invalid skill path: ${requestedPath}`);\n }\n\n return resolved;\n }\n}\n\nfunction runExecutable(\n command: string,\n args: string[],\n cwd: string,\n timeoutMs: number,\n abortSignal?: AbortSignal,\n): Promise<string> {\n return new Promise((resolvePromise, reject) => {\n throwIfAborted(abortSignal);\n const child = spawn(command, args, {\n cwd,\n shell: false,\n stdio: [\"ignore\", \"pipe\", \"pipe\"],\n });\n\n let stdout = \"\";\n let stderr = \"\";\n let exited = false;\n let settled = false;\n let firstCause: Error | undefined;\n let escalation: ReturnType<typeof setTimeout> | undefined;\n let watchdog: ReturnType<typeof setTimeout> | undefined;\n const timeout = setTimeout(() => {\n beginTermination(new Error(`Skill script timed out after ${timeoutMs}ms`));\n }, timeoutMs);\n\n function settle(error?: Error, output = \"\"): void {\n if (settled) return;\n settled = true;\n clearTimeout(timeout);\n clearTimeout(escalation);\n clearTimeout(watchdog);\n abortSignal?.removeEventListener(\"abort\", onAbort);\n child.removeListener(\"error\", onError);\n child.removeListener(\"exit\", onExit);\n child.removeListener(\"close\", onClose);\n child.stdout.removeListener(\"data\", onStdout);\n child.stderr.removeListener(\"data\", onStderr);\n if (error !== undefined) reject(error);\n else resolvePromise(output);\n }\n\n function finishTermination(): void {\n // Descendants can hold inherited pipes after the direct child has exited.\n child.stdout.destroy();\n child.stderr.destroy();\n settle(firstCause);\n }\n\n function beginTermination(cause: Error): void {\n if (settled || firstCause !== undefined) return;\n firstCause = cause;\n clearTimeout(timeout);\n if (exited) {\n finishTermination();\n return;\n }\n escalation = setTimeout(() => {\n if (!exited) child.kill(\"SIGKILL\");\n }, TERMINATION_GRACE_MS);\n watchdog = setTimeout(() => {\n child.stdout.destroy();\n child.stderr.destroy();\n settle(new Error(\"Skill script termination could not be confirmed\", { cause: firstCause }));\n }, TERMINATION_WATCHDOG_MS);\n child.kill(\"SIGTERM\");\n }\n\n function onAbort(): void {\n beginTermination(abortError(abortSignal?.reason));\n }\n\n function onError(error: Error): void {\n if (child.pid === undefined) settle(error);\n else beginTermination(error);\n }\n\n function onExit(): void {\n exited = true;\n if (firstCause !== undefined) finishTermination();\n }\n\n function onClose(code: number | null, signal: NodeJS.Signals | null): void {\n if (firstCause !== undefined) {\n if (exited) finishTermination();\n return;\n }\n const output = formatProcessOutput(stdout, stderr);\n if (code !== 0) {\n settle(new Error(`Skill script exited with code ${code ?? \"unknown\"}: ${output}`));\n } else if (signal !== null) {\n settle(new Error(`Skill script exited with signal ${signal}: ${output}`));\n } else {\n settle(undefined, output);\n }\n }\n\n function onStdout(chunk: string): void {\n stdout = appendLimited(stdout, chunk);\n }\n\n function onStderr(chunk: string): void {\n stderr = appendLimited(stderr, chunk);\n }\n\n child.stdout.setEncoding(\"utf8\");\n child.stderr.setEncoding(\"utf8\");\n child.stdout.on(\"data\", onStdout);\n child.stderr.on(\"data\", onStderr);\n child.on(\"error\", onError);\n child.on(\"exit\", onExit);\n child.on(\"close\", onClose);\n abortSignal?.addEventListener(\"abort\", onAbort, { once: true });\n if (abortSignal?.aborted === true) onAbort();\n });\n}\n\nfunction formatProcessOutput(stdout: string, stderr: string): string {\n const parts: string[] = [];\n if (stdout.length > 0) {\n parts.push(`stdout:\\n${stdout}`);\n }\n if (stderr.length > 0) {\n parts.push(`stderr:\\n${stderr}`);\n }\n return parts.length === 0 ? \"\" : parts.join(\"\\n\\n\");\n}\n\nfunction appendLimited(current: string, chunk: string): string {\n const next = current + chunk;\n if (next.length <= MAX_OUTPUT_CHARS) {\n return next;\n }\n return `${next.slice(0, MAX_OUTPUT_CHARS)}\\n[truncated]`;\n}\n"],"mappings":";;;;;;;;;;;;AAAA,SAAS,aAAa;AACtB,SAAS,gBAAgB;AACzB,SAAS,YAAY,UAAU,eAAe;AAC9C,SAAS,SAAS;AAMlB,IAAM,qBAAqB;AAC3B,IAAM,mBAAmB;AACzB,IAAM,uBAAuB;AAC7B,IAAM,0BAA0B;AAEzB,SAAS,iBAAiB,QAA4B;AAC3D,QAAM,WAAW,IAAI,cAAc,MAAM;AAEzC,SAAO;AAAA,IACL;AAAA,MACE,WAAW;AAAA,QACT,MAAM;AAAA,QACN,aAAa;AAAA,QACb,aAAa,EAAE,OAAO;AAAA,UACpB,WAAW,EAAE,OAAO,EAAE,SAAS,gCAAgC;AAAA,QACjE,CAAC;AAAA,QACD,cAAc,EAAE,OAAO;AAAA,QACvB,SAAS,CAAC,EAAE,UAAU,MAAM,SAAS,IAAI,SAAS,EAAE;AAAA,MACtD,CAAC;AAAA,IACH;AAAA,IACA;AAAA,MACE,WAAW;AAAA,QACT,MAAM;AAAA,QACN,aAAa;AAAA,QACb,aAAa,EAAE,OAAO;AAAA,UACpB,WAAW,EAAE,OAAO,EAAE,SAAS,wBAAwB;AAAA,UACvD,eAAe,EAAE,OAAO,EAAE,SAAS,wCAAwC;AAAA,QAC7E,CAAC;AAAA,QACD,cAAc,EAAE,OAAO;AAAA,QACvB,SAAS,CAAC,EAAE,WAAW,cAAc,MAAM,SAAS,cAAc,WAAW,aAAa;AAAA,MAC5F,CAAC;AAAA,IACH;AAAA,IACA;AAAA,MACE,WAAW;AAAA,QACT,MAAM;AAAA,QACN,aAAa;AAAA,QACb,aAAa,EAAE,OAAO;AAAA,UACpB,WAAW,EAAE,OAAO,EAAE,SAAS,wBAAwB;AAAA,UACvD,YAAY,EAAE,OAAO,EAAE,SAAS,qCAAqC;AAAA,QACvE,CAAC;AAAA,QACD,cAAc,EAAE,OAAO;AAAA,QACvB,SAAS,CAAC,EAAE,WAAW,WAAW,MAAM,SAAS,WAAW,WAAW,UAAU;AAAA,MACnF,CAAC;AAAA,IACH;AAAA,IACA;AAAA,MACE,WAAW;AAAA,QACT,MAAM;AAAA,QACN,aAAa;AAAA,QACb,aAAa,EAAE,OAAO;AAAA,UACpB,WAAW,EAAE,OAAO,EAAE,SAAS,wBAAwB;AAAA,UACvD,YAAY,EAAE,OAAO,EAAE,SAAS,qCAAqC;AAAA,UACrE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,SAAS,iCAAiC;AAAA,UAC/E,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,0BAA0B;AAAA,QACvF,CAAC;AAAA,QACD,cAAc,EAAE,OAAO;AAAA,QACvB,SAAS,CAAC,EAAE,WAAW,YAAY,OAAO,CAAC,GAAG,YAAY,mBAAmB,GAAG,YAC9E,SAAS,UAAU,WAAW,YAAY,MAAM,WAAW,QAAQ,WAAW;AAAA,MAClF,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEA,IAAM,gBAAN,MAAoB;AAAA,EACD,SAAS,oBAAI,IAAmB;AAAA,EAEjD,YAAY,QAAiB;AAC3B,eAAW,SAAS,QAAQ;AAC1B,WAAK,OAAO,IAAI,MAAM,MAAM,KAAK;AAAA,IACnC;AAAA,EACF;AAAA,EAEA,IAAI,WAA0B;AAC5B,UAAM,QAAQ,KAAK,OAAO,IAAI,SAAS;AACvC,QAAI,UAAU,QAAW;AACvB,YAAM,IAAI,MAAM,oBAAoB,SAAS,EAAE;AAAA,IACjD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cAAc,WAAmB,eAAwC;AAC7E,UAAM,QAAQ,KAAK,IAAI,SAAS;AAChC,UAAM,OAAO,KAAK,qBAAqB,OAAO,cAAc,aAAa;AACzE,QAAI,CAAC,MAAM,WAAW,SAAS,aAAa,GAAG;AAC7C,YAAM,IAAI,MAAM,8BAA8B,SAAS,IAAI,aAAa,EAAE;AAAA,IAC5E;AACA,WAAO,SAAS,MAAM,MAAM;AAAA,EAC9B;AAAA,EAEA,MAAM,WAAW,WAAmB,YAAqC;AACvE,UAAM,QAAQ,KAAK,IAAI,SAAS;AAChC,UAAM,OAAO,KAAK,qBAAqB,OAAO,WAAW,UAAU;AACnE,QAAI,CAAC,MAAM,QAAQ,SAAS,UAAU,GAAG;AACvC,YAAM,IAAI,MAAM,2BAA2B,SAAS,IAAI,UAAU,EAAE;AAAA,IACtE;AACA,WAAO,SAAS,MAAM,MAAM;AAAA,EAC9B;AAAA,EAEA,MAAM,UACJ,WACA,YACA,MACA,WACA,aACiB;AACjB,UAAM,QAAQ,KAAK,IAAI,SAAS;AAChC,UAAM,SAAS,KAAK,qBAAqB,OAAO,WAAW,UAAU;AACrE,QAAI,CAAC,MAAM,QAAQ,SAAS,UAAU,GAAG;AACvC,YAAM,IAAI,MAAM,2BAA2B,SAAS,IAAI,UAAU,EAAE;AAAA,IACtE;AACA,WAAO,cAAc,QAAQ,MAAM,MAAM,WAAW,WAAW,WAAW;AAAA,EAC5E;AAAA,EAEQ,qBACN,OACA,SACA,eACQ;AACR,QAAI,cAAc,WAAW,KAAK,WAAW,aAAa,GAAG;AAC3D,YAAM,IAAI,MAAM,uBAAuB,aAAa,EAAE;AAAA,IACxD;AAEA,UAAM,OAAO,QAAQ,MAAM,WAAW,OAAO;AAC7C,UAAM,WAAW,QAAQ,MAAM,aAAa;AAC5C,UAAM,MAAM,SAAS,MAAM,QAAQ;AACnC,QAAI,QAAQ,QAAQ,IAAI,WAAW,KAAK,GAAG,EAAE,KAAK,IAAI,WAAW,KAAK,IAAI,EAAE,GAAG;AAC7E,YAAM,IAAI,MAAM,uBAAuB,aAAa,EAAE;AAAA,IACxD;AAEA,WAAO;AAAA,EACT;AACF;AAEA,SAAS,cACP,SACA,MACA,KACA,WACA,aACiB;AACjB,SAAO,IAAI,QAAQ,CAAC,gBAAgB,WAAW;AAC7C,mBAAe,WAAW;AAC1B,UAAM,QAAQ,MAAM,SAAS,MAAM;AAAA,MACjC;AAAA,MACA,OAAO;AAAA,MACP,OAAO,CAAC,UAAU,QAAQ,MAAM;AAAA,IAClC,CAAC;AAED,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,UAAU;AACd,QAAI;AACJ,QAAI;AACJ,QAAI;AACJ,UAAM,UAAU,WAAW,MAAM;AAC/B,uBAAiB,IAAI,MAAM,gCAAgC,SAAS,IAAI,CAAC;AAAA,IAC3E,GAAG,SAAS;AAEZ,aAAS,OAAO,OAAe,SAAS,IAAU;AAChD,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,OAAO;AACpB,mBAAa,UAAU;AACvB,mBAAa,QAAQ;AACrB,mBAAa,oBAAoB,SAAS,OAAO;AACjD,YAAM,eAAe,SAAS,OAAO;AACrC,YAAM,eAAe,QAAQ,MAAM;AACnC,YAAM,eAAe,SAAS,OAAO;AACrC,YAAM,OAAO,eAAe,QAAQ,QAAQ;AAC5C,YAAM,OAAO,eAAe,QAAQ,QAAQ;AAC5C,UAAI,UAAU,OAAW,QAAO,KAAK;AAAA,UAChC,gBAAe,MAAM;AAAA,IAC5B;AAEA,aAAS,oBAA0B;AAEjC,YAAM,OAAO,QAAQ;AACrB,YAAM,OAAO,QAAQ;AACrB,aAAO,UAAU;AAAA,IACnB;AAEA,aAAS,iBAAiB,OAAoB;AAC5C,UAAI,WAAW,eAAe,OAAW;AACzC,mBAAa;AACb,mBAAa,OAAO;AACpB,UAAI,QAAQ;AACV,0BAAkB;AAClB;AAAA,MACF;AACA,mBAAa,WAAW,MAAM;AAC5B,YAAI,CAAC,OAAQ,OAAM,KAAK,SAAS;AAAA,MACnC,GAAG,oBAAoB;AACvB,iBAAW,WAAW,MAAM;AAC1B,cAAM,OAAO,QAAQ;AACrB,cAAM,OAAO,QAAQ;AACrB,eAAO,IAAI,MAAM,mDAAmD,EAAE,OAAO,WAAW,CAAC,CAAC;AAAA,MAC5F,GAAG,uBAAuB;AAC1B,YAAM,KAAK,SAAS;AAAA,IACtB;AAEA,aAAS,UAAgB;AACvB,uBAAiB,WAAW,aAAa,MAAM,CAAC;AAAA,IAClD;AAEA,aAAS,QAAQ,OAAoB;AACnC,UAAI,MAAM,QAAQ,OAAW,QAAO,KAAK;AAAA,UACpC,kBAAiB,KAAK;AAAA,IAC7B;AAEA,aAAS,SAAe;AACtB,eAAS;AACT,UAAI,eAAe,OAAW,mBAAkB;AAAA,IAClD;AAEA,aAAS,QAAQ,MAAqB,QAAqC;AACzE,UAAI,eAAe,QAAW;AAC5B,YAAI,OAAQ,mBAAkB;AAC9B;AAAA,MACF;AACA,YAAM,SAAS,oBAAoB,QAAQ,MAAM;AACjD,UAAI,SAAS,GAAG;AACd,eAAO,IAAI,MAAM,iCAAiC,QAAQ,SAAS,KAAK,MAAM,EAAE,CAAC;AAAA,MACnF,WAAW,WAAW,MAAM;AAC1B,eAAO,IAAI,MAAM,mCAAmC,MAAM,KAAK,MAAM,EAAE,CAAC;AAAA,MAC1E,OAAO;AACL,eAAO,QAAW,MAAM;AAAA,MAC1B;AAAA,IACF;AAEA,aAAS,SAAS,OAAqB;AACrC,eAAS,cAAc,QAAQ,KAAK;AAAA,IACtC;AAEA,aAAS,SAAS,OAAqB;AACrC,eAAS,cAAc,QAAQ,KAAK;AAAA,IACtC;AAEA,UAAM,OAAO,YAAY,MAAM;AAC/B,UAAM,OAAO,YAAY,MAAM;AAC/B,UAAM,OAAO,GAAG,QAAQ,QAAQ;AAChC,UAAM,OAAO,GAAG,QAAQ,QAAQ;AAChC,UAAM,GAAG,SAAS,OAAO;AACzB,UAAM,GAAG,QAAQ,MAAM;AACvB,UAAM,GAAG,SAAS,OAAO;AACzB,iBAAa,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAC9D,QAAI,aAAa,YAAY,KAAM,SAAQ;AAAA,EAC7C,CAAC;AACH;AAEA,SAAS,oBAAoB,QAAgB,QAAwB;AACnE,QAAM,QAAkB,CAAC;AACzB,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,KAAK;AAAA,EAAY,MAAM,EAAE;AAAA,EACjC;AACA,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,KAAK;AAAA,EAAY,MAAM,EAAE;AAAA,EACjC;AACA,SAAO,MAAM,WAAW,IAAI,KAAK,MAAM,KAAK,MAAM;AACpD;AAEA,SAAS,cAAc,SAAiB,OAAuB;AAC7D,QAAM,OAAO,UAAU;AACvB,MAAI,KAAK,UAAU,kBAAkB;AACnC,WAAO;AAAA,EACT;AACA,SAAO,GAAG,KAAK,MAAM,GAAG,gBAAgB,CAAC;AAAA;AAC3C;","names":[]}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
Agent
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-MJAM7HZB.js";
|
|
4
4
|
import {
|
|
5
5
|
getAgentToolState
|
|
6
6
|
} from "./chunk-A7BXVZXZ.js";
|
|
7
7
|
import {
|
|
8
8
|
markResolvedAgentOptions
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-STEAW4JD.js";
|
|
10
10
|
|
|
11
11
|
// src/agent/resolved-agent.ts
|
|
12
12
|
function createResolvedAgent(options) {
|
|
@@ -45,4 +45,4 @@ export {
|
|
|
45
45
|
createResolvedAgent,
|
|
46
46
|
getResolvedAgentOptions
|
|
47
47
|
};
|
|
48
|
-
//# sourceMappingURL=chunk-
|
|
48
|
+
//# sourceMappingURL=chunk-TOJSC4GI.js.map
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
addUsage,
|
|
3
3
|
evaluationMetadata,
|
|
4
4
|
runJudge
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-YSR6KLCS.js";
|
|
6
6
|
import {
|
|
7
7
|
resolveActualText,
|
|
8
8
|
resolveExpected
|
|
@@ -23,6 +23,9 @@ import {
|
|
|
23
23
|
import {
|
|
24
24
|
isJsonValue
|
|
25
25
|
} from "./chunk-4C7AXYOM.js";
|
|
26
|
+
import {
|
|
27
|
+
throwIfAborted
|
|
28
|
+
} from "./chunk-USRKPEQN.js";
|
|
26
29
|
|
|
27
30
|
// src/evals/advanced-metrics.ts
|
|
28
31
|
import { z } from "zod";
|
|
@@ -68,6 +71,7 @@ function answerRelevancy(options) {
|
|
|
68
71
|
instructions: "Break the answer into concise, independently assessable statements. Return every substantive statement using the schema.",
|
|
69
72
|
prompt: `Answer:
|
|
70
73
|
${actual}`,
|
|
74
|
+
abortSignal: args.signal,
|
|
71
75
|
retries: config.retries
|
|
72
76
|
});
|
|
73
77
|
const statements = statementResult.data.statements;
|
|
@@ -79,6 +83,7 @@ ${actual}`,
|
|
|
79
83
|
schema: verdictsSchema,
|
|
80
84
|
instructions: "Classify each answer statement for relevance to the user input. Use yes for relevant, no for irrelevant, and idk only when relevance is genuinely indeterminate. Preserve order and return one verdict per statement.",
|
|
81
85
|
prompt: jsonPrompt({ input, statements }),
|
|
86
|
+
abortSignal: args.signal,
|
|
82
87
|
retries: config.retries
|
|
83
88
|
});
|
|
84
89
|
verdicts = verdictResult.data.verdicts;
|
|
@@ -92,6 +97,7 @@ ${actual}`,
|
|
|
92
97
|
includeReason: config.includeReason,
|
|
93
98
|
retries: config.retries,
|
|
94
99
|
metric: "answer relevancy",
|
|
100
|
+
abortSignal: args.signal,
|
|
95
101
|
score,
|
|
96
102
|
evidence: { input, verdicts: serializedVerdicts }
|
|
97
103
|
});
|
|
@@ -123,6 +129,7 @@ function promptAlignment(options) {
|
|
|
123
129
|
schema: binaryVerdictsSchema,
|
|
124
130
|
instructions: "Determine whether the answer follows each prompt instruction. Preserve order and return exactly one yes or no verdict per instruction.",
|
|
125
131
|
prompt: jsonPrompt({ input, actual, instructions: options.promptInstructions }),
|
|
132
|
+
abortSignal: args.signal,
|
|
126
133
|
retries: config.retries
|
|
127
134
|
});
|
|
128
135
|
const verdicts = verdictResult.data.verdicts;
|
|
@@ -133,6 +140,7 @@ function promptAlignment(options) {
|
|
|
133
140
|
includeReason: config.includeReason,
|
|
134
141
|
retries: config.retries,
|
|
135
142
|
metric: "prompt alignment",
|
|
143
|
+
abortSignal: args.signal,
|
|
136
144
|
score,
|
|
137
145
|
evidence: { verdicts }
|
|
138
146
|
});
|
|
@@ -193,6 +201,7 @@ function jsonCorrectness(options) {
|
|
|
193
201
|
schema: reasonSchema,
|
|
194
202
|
instructions: "Briefly explain why the generated JSON does not match the expected schema. Focus on actionable syntax, field, and type problems.",
|
|
195
203
|
prompt: jsonPrompt({ actual, validationError }),
|
|
204
|
+
abortSignal: args.signal,
|
|
196
205
|
retries
|
|
197
206
|
});
|
|
198
207
|
comment = reasonResult.data.reason;
|
|
@@ -224,6 +233,7 @@ function hallucination(options) {
|
|
|
224
233
|
schema: binaryVerdictsSchema,
|
|
225
234
|
instructions: "Compare the answer with each trusted context. Use yes when the answer is factually aligned with that context and no when it contradicts it. Preserve order and return one verdict per context.",
|
|
226
235
|
prompt: jsonPrompt({ actual, context }),
|
|
236
|
+
abortSignal: args.signal,
|
|
227
237
|
retries: config.retries
|
|
228
238
|
});
|
|
229
239
|
const verdicts = verdictResult.data.verdicts;
|
|
@@ -234,6 +244,7 @@ function hallucination(options) {
|
|
|
234
244
|
includeReason: config.includeReason,
|
|
235
245
|
retries: config.retries,
|
|
236
246
|
metric: "hallucination",
|
|
247
|
+
abortSignal: args.signal,
|
|
237
248
|
score,
|
|
238
249
|
evidence: { verdicts }
|
|
239
250
|
});
|
|
@@ -272,6 +283,7 @@ function faithfulness(options) {
|
|
|
272
283
|
schema: factsSchema,
|
|
273
284
|
instructions: truthsExtractionInstructions(truthsExtractionLimit),
|
|
274
285
|
prompt: jsonPrompt({ retrievalContext }),
|
|
286
|
+
abortSignal: args.signal,
|
|
275
287
|
retries: config.retries
|
|
276
288
|
}),
|
|
277
289
|
runJudge({
|
|
@@ -280,6 +292,7 @@ function faithfulness(options) {
|
|
|
280
292
|
instructions: "Extract every concise factual claim made by the answer. Return claims in the facts array and omit opinions or purely stylistic text.",
|
|
281
293
|
prompt: `Answer:
|
|
282
294
|
${actual}`,
|
|
295
|
+
abortSignal: args.signal,
|
|
283
296
|
retries: config.retries
|
|
284
297
|
})
|
|
285
298
|
]);
|
|
@@ -293,6 +306,7 @@ ${actual}`,
|
|
|
293
306
|
schema: verdictsSchema,
|
|
294
307
|
instructions: "Determine whether each answer claim is supported by the supplied truths. Use yes for supported, no for contradicted or unsupported, and idk for genuinely ambiguous support. Preserve order and return one verdict per claim.",
|
|
295
308
|
prompt: jsonPrompt({ truths, claims }),
|
|
309
|
+
abortSignal: args.signal,
|
|
296
310
|
retries: config.retries
|
|
297
311
|
});
|
|
298
312
|
verdicts = verdictResult.data.verdicts;
|
|
@@ -310,6 +324,7 @@ ${actual}`,
|
|
|
310
324
|
includeReason: config.includeReason,
|
|
311
325
|
retries: config.retries,
|
|
312
326
|
metric: "faithfulness",
|
|
327
|
+
abortSignal: args.signal,
|
|
313
328
|
score,
|
|
314
329
|
evidence: { verdicts: serializedVerdicts, penalizeAmbiguousClaims }
|
|
315
330
|
});
|
|
@@ -335,6 +350,7 @@ function abstention(options) {
|
|
|
335
350
|
dataType: "CATEGORICAL",
|
|
336
351
|
async evaluate(args) {
|
|
337
352
|
try {
|
|
353
|
+
throwIfAborted(args.signal);
|
|
338
354
|
const actual = await resolveActualText(options.actual, args);
|
|
339
355
|
const shouldAbstain = await resolveExpected(options.shouldAbstain, args);
|
|
340
356
|
if (typeof shouldAbstain !== "boolean") {
|
|
@@ -347,12 +363,14 @@ function abstention(options) {
|
|
|
347
363
|
);
|
|
348
364
|
}
|
|
349
365
|
const judgment = await runJudge({
|
|
366
|
+
abortSignal: args.signal,
|
|
350
367
|
model: options.model,
|
|
351
368
|
schema: abstentionJudgmentSchema,
|
|
352
369
|
instructions: "Classify whether the answer abstains or gives a confident answer. For a confident answer, grounded is true only when every substantive factual claim is supported by the supplied context. For an abstention, set grounded to false. Return a concise evidence-based reason.",
|
|
353
370
|
prompt: jsonPrompt({ actual, context }),
|
|
354
371
|
retries
|
|
355
372
|
});
|
|
373
|
+
throwIfAborted(args.signal);
|
|
356
374
|
const category = abstentionCategory(
|
|
357
375
|
shouldAbstain,
|
|
358
376
|
judgment.data.behavior,
|
|
@@ -409,6 +427,7 @@ function summarization(options) {
|
|
|
409
427
|
instructions: `Generate exactly ${questionCount} important yes-or-no assessment questions whose answers capture the source text's essential information.`,
|
|
410
428
|
prompt: `Source text:
|
|
411
429
|
${input}`,
|
|
430
|
+
abortSignal: args.signal,
|
|
412
431
|
retries: config.retries
|
|
413
432
|
}) : Promise.resolve({
|
|
414
433
|
data: { questions: suppliedQuestions },
|
|
@@ -421,6 +440,7 @@ ${input}`,
|
|
|
421
440
|
instructions: truthsExtractionInstructions(truthsExtractionLimit),
|
|
422
441
|
prompt: `Source text:
|
|
423
442
|
${input}`,
|
|
443
|
+
abortSignal: args.signal,
|
|
424
444
|
retries: config.retries
|
|
425
445
|
}),
|
|
426
446
|
runJudge({
|
|
@@ -429,6 +449,7 @@ ${input}`,
|
|
|
429
449
|
instructions: "Extract every concise factual claim made by the summary. Return claims in the facts array.",
|
|
430
450
|
prompt: `Summary:
|
|
431
451
|
${actual}`,
|
|
452
|
+
abortSignal: args.signal,
|
|
432
453
|
retries: config.retries
|
|
433
454
|
}),
|
|
434
455
|
questionPromise
|
|
@@ -444,6 +465,7 @@ ${actual}`,
|
|
|
444
465
|
schema: answersSchema,
|
|
445
466
|
instructions: "Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.",
|
|
446
467
|
prompt: jsonPrompt({ questions, text: input }),
|
|
468
|
+
abortSignal: args.signal,
|
|
447
469
|
retries: config.retries
|
|
448
470
|
});
|
|
449
471
|
const summaryAnswerPromise = runJudge({
|
|
@@ -451,6 +473,7 @@ ${actual}`,
|
|
|
451
473
|
schema: answersSchema,
|
|
452
474
|
instructions: "Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.",
|
|
453
475
|
prompt: jsonPrompt({ questions, text: actual }),
|
|
476
|
+
abortSignal: args.signal,
|
|
454
477
|
retries: config.retries
|
|
455
478
|
});
|
|
456
479
|
const alignmentPromise = claims.length === 0 ? Promise.resolve({ data: { verdicts: [] }, usage: Usage.empty() }) : runJudge({
|
|
@@ -458,6 +481,7 @@ ${actual}`,
|
|
|
458
481
|
schema: verdictsSchema,
|
|
459
482
|
instructions: "Determine whether each summary claim is supported by the source truths. Use yes for supported, no for contradicted, and idk for unsupported filler or ambiguity. Preserve order.",
|
|
460
483
|
prompt: jsonPrompt({ truths, claims }),
|
|
484
|
+
abortSignal: args.signal,
|
|
461
485
|
retries: config.retries
|
|
462
486
|
});
|
|
463
487
|
const [sourceAnswerResult, summaryAnswerResult, alignmentResult] = await Promise.all([
|
|
@@ -489,6 +513,7 @@ ${actual}`,
|
|
|
489
513
|
includeReason: config.includeReason,
|
|
490
514
|
retries: config.retries,
|
|
491
515
|
metric: "summarization",
|
|
516
|
+
abortSignal: args.signal,
|
|
492
517
|
score,
|
|
493
518
|
evidence: {
|
|
494
519
|
alignmentVerdicts: serializedAlignmentVerdicts,
|
|
@@ -543,35 +568,98 @@ function gEval(options) {
|
|
|
543
568
|
const config = metricConfig(options, options.name);
|
|
544
569
|
const rubric = validateRubric(options.rubric);
|
|
545
570
|
const scoreRange = rubric.length === 0 ? [0, 10] : [rubric[0]?.scoreRange[0] ?? 0, rubric.at(-1)?.scoreRange[1] ?? 10];
|
|
546
|
-
let
|
|
547
|
-
let
|
|
548
|
-
|
|
571
|
+
let pendingPreparation;
|
|
572
|
+
let preparedSteps;
|
|
573
|
+
function acceptSteps(result, signal) {
|
|
574
|
+
throwIfAborted(signal);
|
|
575
|
+
return { steps: [...result.steps], preparation: result };
|
|
576
|
+
}
|
|
577
|
+
async function resolveSteps(signal) {
|
|
578
|
+
throwIfAborted(signal);
|
|
549
579
|
if (options.evaluationSteps !== void 0) {
|
|
550
|
-
return { steps: options.evaluationSteps
|
|
580
|
+
return { steps: [...options.evaluationSteps] };
|
|
551
581
|
}
|
|
552
|
-
if (
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
582
|
+
if (preparedSteps !== void 0) return acceptSteps(preparedSteps, signal);
|
|
583
|
+
if (pendingPreparation === void 0) {
|
|
584
|
+
const controller = new AbortController();
|
|
585
|
+
const epoch2 = {
|
|
586
|
+
controller,
|
|
587
|
+
waiters: /* @__PURE__ */ new Set(),
|
|
588
|
+
promise: Promise.resolve().then(
|
|
589
|
+
() => runJudge({
|
|
590
|
+
model: options.model,
|
|
591
|
+
schema: z.object({ steps: z.array(z.string()) }),
|
|
592
|
+
instructions: "Generate three or four concise evaluation steps from the criteria. Explain how the selected parameters should be judged in relation to one another.",
|
|
593
|
+
prompt: jsonPrompt({
|
|
594
|
+
criteria: options.criteria,
|
|
595
|
+
parameters: options.evaluationParams
|
|
596
|
+
}),
|
|
597
|
+
retries: config.retries,
|
|
598
|
+
abortSignal: controller.signal
|
|
599
|
+
})
|
|
600
|
+
).then((result) => {
|
|
601
|
+
if (result.data.steps.length === 0) {
|
|
602
|
+
throw new Error("G-Eval generated no evaluation steps.");
|
|
603
|
+
}
|
|
604
|
+
const prepared = {
|
|
605
|
+
steps: [...result.data.steps],
|
|
606
|
+
usage: result.usage,
|
|
607
|
+
usageClaimed: false
|
|
608
|
+
};
|
|
609
|
+
if (pendingPreparation === epoch2) {
|
|
610
|
+
preparedSteps = prepared;
|
|
611
|
+
pendingPreparation = void 0;
|
|
612
|
+
}
|
|
613
|
+
return prepared;
|
|
614
|
+
}).catch((error) => {
|
|
615
|
+
if (pendingPreparation === epoch2) pendingPreparation = void 0;
|
|
616
|
+
throw error;
|
|
617
|
+
})
|
|
618
|
+
};
|
|
619
|
+
pendingPreparation = epoch2;
|
|
563
620
|
}
|
|
564
|
-
const
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
621
|
+
const epoch = pendingPreparation;
|
|
622
|
+
const token = {};
|
|
623
|
+
epoch.waiters.add(token);
|
|
624
|
+
return new Promise((resolve, reject) => {
|
|
625
|
+
const release = () => {
|
|
626
|
+
if (!epoch.waiters.delete(token)) return;
|
|
627
|
+
signal.removeEventListener("abort", onAbort);
|
|
628
|
+
if (pendingPreparation === epoch && epoch.waiters.size === 0) {
|
|
629
|
+
pendingPreparation = void 0;
|
|
630
|
+
epoch.controller.abort(signal.reason);
|
|
631
|
+
}
|
|
632
|
+
};
|
|
633
|
+
const onAbort = () => {
|
|
634
|
+
release();
|
|
635
|
+
reject(signal.reason);
|
|
636
|
+
};
|
|
637
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
638
|
+
epoch.promise.then(
|
|
639
|
+
(result) => {
|
|
640
|
+
if (!epoch.waiters.has(token)) return;
|
|
641
|
+
try {
|
|
642
|
+
const accepted = acceptSteps(result, signal);
|
|
643
|
+
release();
|
|
644
|
+
resolve(accepted);
|
|
645
|
+
} catch (error) {
|
|
646
|
+
release();
|
|
647
|
+
reject(error);
|
|
648
|
+
}
|
|
649
|
+
},
|
|
650
|
+
(error) => {
|
|
651
|
+
release();
|
|
652
|
+
reject(error);
|
|
653
|
+
}
|
|
654
|
+
);
|
|
655
|
+
});
|
|
569
656
|
}
|
|
570
657
|
return numericMetric(config.name, config, "higher_is_better", async (args) => {
|
|
571
658
|
try {
|
|
572
659
|
const parameters = await resolveGEvalParameters(options, args);
|
|
573
|
-
const stepsResult = await resolveSteps();
|
|
660
|
+
const stepsResult = await resolveSteps(args.signal);
|
|
574
661
|
const scoreResult = await runJudge({
|
|
662
|
+
abortSignal: args.signal,
|
|
575
663
|
model: options.model,
|
|
576
664
|
schema: z.object({ score: z.number(), reason: z.string() }),
|
|
577
665
|
instructions: config.strictMode ? "Apply the evaluation steps and return score 1 only for complete compliance, otherwise 0. Give a concise evidence-based reason." : `Apply the evaluation steps and return an integer score from ${scoreRange[0]} through ${scoreRange[1]}, plus a concise evidence-based reason.`,
|
|
@@ -587,8 +675,11 @@ function gEval(options) {
|
|
|
587
675
|
throw new RangeError(`G-Eval score ${rawScore} is outside the requested range.`);
|
|
588
676
|
}
|
|
589
677
|
const score = config.strictMode ? rawScore : (rawScore - scoreRange[0]) / (scoreRange[1] - scoreRange[0]);
|
|
590
|
-
|
|
591
|
-
|
|
678
|
+
throwIfAborted(args.signal);
|
|
679
|
+
const preparation = stepsResult.preparation;
|
|
680
|
+
const setupUsage = preparation === void 0 || preparation.usageClaimed ? Usage.empty() : preparation.usage;
|
|
681
|
+
const usage = addUsage(setupUsage, scoreResult.usage);
|
|
682
|
+
const outcome = higherOutcome({
|
|
592
683
|
score,
|
|
593
684
|
threshold: config.threshold,
|
|
594
685
|
strictMode: config.strictMode,
|
|
@@ -605,6 +696,8 @@ function gEval(options) {
|
|
|
605
696
|
},
|
|
606
697
|
usage
|
|
607
698
|
});
|
|
699
|
+
if (preparation !== void 0) preparation.usageClaimed = true;
|
|
700
|
+
return outcome;
|
|
608
701
|
} catch (error) {
|
|
609
702
|
return EvalOutcome.fromError(error);
|
|
610
703
|
}
|
|
@@ -629,6 +722,7 @@ function turnRelevancy(options) {
|
|
|
629
722
|
schema: binaryVerdictSchema,
|
|
630
723
|
instructions: "Judge whether the final assistant reply is relevant to the preceding conversation. Return yes for relevant and no for irrelevant, with a concise reason.",
|
|
631
724
|
prompt: jsonPrompt({ turns: window }),
|
|
725
|
+
abortSignal: args.signal,
|
|
632
726
|
retries: config.retries
|
|
633
727
|
})
|
|
634
728
|
);
|
|
@@ -642,6 +736,7 @@ function turnRelevancy(options) {
|
|
|
642
736
|
includeReason: config.includeReason,
|
|
643
737
|
retries: config.retries,
|
|
644
738
|
metric: "turn relevancy",
|
|
739
|
+
abortSignal: args.signal,
|
|
645
740
|
score,
|
|
646
741
|
evidence: { verdicts }
|
|
647
742
|
});
|
|
@@ -677,6 +772,7 @@ function knowledgeRetention(options) {
|
|
|
677
772
|
previousTurns: turns.slice(0, entry.index),
|
|
678
773
|
userMessage: entry.turn.content
|
|
679
774
|
}),
|
|
775
|
+
abortSignal: args.signal,
|
|
680
776
|
retries: config.retries
|
|
681
777
|
})
|
|
682
778
|
);
|
|
@@ -696,6 +792,7 @@ function knowledgeRetention(options) {
|
|
|
696
792
|
schema: z.object({ attrition: z.boolean(), reason: z.string() }),
|
|
697
793
|
instructions: "Determine whether the assistant reply forgets, contradicts, or unnecessarily asks again for information already supplied by the user. Set attrition true only when knowledge was lost.",
|
|
698
794
|
prompt: jsonPrompt({ knownFacts: entry.facts, assistantReply: entry.turn.content }),
|
|
795
|
+
abortSignal: args.signal,
|
|
699
796
|
retries: config.retries
|
|
700
797
|
})
|
|
701
798
|
);
|
|
@@ -710,6 +807,7 @@ function knowledgeRetention(options) {
|
|
|
710
807
|
includeReason: config.includeReason,
|
|
711
808
|
retries: config.retries,
|
|
712
809
|
metric: "knowledge retention",
|
|
810
|
+
abortSignal: args.signal,
|
|
713
811
|
score,
|
|
714
812
|
evidence: { verdicts }
|
|
715
813
|
});
|
|
@@ -738,7 +836,16 @@ function numericMetric(name, config, direction, evaluate) {
|
|
|
738
836
|
direction,
|
|
739
837
|
threshold: config.strictMode === true ? direction === "higher_is_better" ? 1 : 0 : config.threshold,
|
|
740
838
|
dataType: "NUMERIC",
|
|
741
|
-
evaluate
|
|
839
|
+
async evaluate(args) {
|
|
840
|
+
try {
|
|
841
|
+
throwIfAborted(args.signal);
|
|
842
|
+
const outcome = await evaluate(args);
|
|
843
|
+
throwIfAborted(args.signal);
|
|
844
|
+
return outcome;
|
|
845
|
+
} catch (error) {
|
|
846
|
+
return EvalOutcome.fromError(error);
|
|
847
|
+
}
|
|
848
|
+
}
|
|
742
849
|
};
|
|
743
850
|
}
|
|
744
851
|
function metricConfig(options, defaultName) {
|
|
@@ -788,12 +895,14 @@ function lowerOutcome(args) {
|
|
|
788
895
|
return score <= threshold ? EvalOutcome.pass(score, options) : EvalOutcome.fail(score, options);
|
|
789
896
|
}
|
|
790
897
|
async function maybeReason(args) {
|
|
898
|
+
throwIfAborted(args.abortSignal);
|
|
791
899
|
if (!args.includeReason) return { usage: Usage.empty() };
|
|
792
900
|
const result = await runJudge({
|
|
793
901
|
model: args.model,
|
|
794
902
|
schema: reasonSchema,
|
|
795
903
|
instructions: `Write a concise final explanation for the ${args.metric} score. Ground it only in the supplied evidence and do not repeat the numeric score.`,
|
|
796
904
|
prompt: jsonPrompt({ score: args.score, evidence: args.evidence }),
|
|
905
|
+
abortSignal: args.abortSignal,
|
|
797
906
|
retries: args.retries
|
|
798
907
|
});
|
|
799
908
|
return { reason: result.data.reason, usage: result.usage };
|
|
@@ -1004,4 +1113,4 @@ export {
|
|
|
1004
1113
|
turnRelevancy,
|
|
1005
1114
|
knowledgeRetention
|
|
1006
1115
|
};
|
|
1007
|
-
//# sourceMappingURL=chunk-
|
|
1116
|
+
//# sourceMappingURL=chunk-W2XPREWL.js.map
|