@providerkit/agents 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/LICENSE-APACHE-2.0 +202 -0
- package/README.md +2328 -2
- package/THIRD_PARTY_NOTICES.md +521 -0
- package/dist/a2a/auth.d.ts +10 -0
- package/dist/a2a/auth.d.ts.map +1 -0
- package/dist/a2a/auth.js +52 -0
- package/dist/a2a/auth.js.map +1 -0
- package/dist/a2a/client.d.ts +74 -0
- package/dist/a2a/client.d.ts.map +1 -0
- package/dist/a2a/client.js +601 -0
- package/dist/a2a/client.js.map +1 -0
- package/dist/a2a/index.d.ts +22 -0
- package/dist/a2a/index.d.ts.map +1 -0
- package/dist/a2a/index.js +11 -0
- package/dist/a2a/index.js.map +1 -0
- package/dist/a2a/memory-runs.d.ts +62 -0
- package/dist/a2a/memory-runs.d.ts.map +1 -0
- package/dist/a2a/memory-runs.js +349 -0
- package/dist/a2a/memory-runs.js.map +1 -0
- package/dist/a2a/policy.d.ts +15 -0
- package/dist/a2a/policy.d.ts.map +1 -0
- package/dist/a2a/policy.js +114 -0
- package/dist/a2a/policy.js.map +1 -0
- package/dist/a2a/push.d.ts +34 -0
- package/dist/a2a/push.d.ts.map +1 -0
- package/dist/a2a/push.js +180 -0
- package/dist/a2a/push.js.map +1 -0
- package/dist/a2a/remote.d.ts +155 -0
- package/dist/a2a/remote.d.ts.map +1 -0
- package/dist/a2a/remote.js +1196 -0
- package/dist/a2a/remote.js.map +1 -0
- package/dist/a2a/server.d.ts +110 -0
- package/dist/a2a/server.d.ts.map +1 -0
- package/dist/a2a/server.js +1139 -0
- package/dist/a2a/server.js.map +1 -0
- package/dist/a2a/signing.d.ts +21 -0
- package/dist/a2a/signing.d.ts.map +1 -0
- package/dist/a2a/signing.js +305 -0
- package/dist/a2a/signing.js.map +1 -0
- package/dist/a2a/tasks.d.ts +88 -0
- package/dist/a2a/tasks.d.ts.map +1 -0
- package/dist/a2a/tasks.js +170 -0
- package/dist/a2a/tasks.js.map +1 -0
- package/dist/a2a/wire.d.ts +267 -0
- package/dist/a2a/wire.d.ts.map +1 -0
- package/dist/a2a/wire.js +1202 -0
- package/dist/a2a/wire.js.map +1 -0
- package/dist/background-types.d.ts +76 -0
- package/dist/background-types.d.ts.map +1 -0
- package/dist/background-types.js +2 -0
- package/dist/background-types.js.map +1 -0
- package/dist/child.d.ts +25 -0
- package/dist/child.d.ts.map +1 -0
- package/dist/child.js +129 -0
- package/dist/child.js.map +1 -0
- package/dist/code-driver.d.ts +40 -0
- package/dist/code-driver.d.ts.map +1 -0
- package/dist/code-driver.js +2 -0
- package/dist/code-driver.js.map +1 -0
- package/dist/code-mode.d.ts +3 -0
- package/dist/code-mode.d.ts.map +1 -0
- package/dist/code-mode.js +2 -0
- package/dist/code-mode.js.map +1 -0
- package/dist/code-script.d.ts +3 -0
- package/dist/code-script.d.ts.map +1 -0
- package/dist/code-script.js +149 -0
- package/dist/code-script.js.map +1 -0
- package/dist/code.d.ts +75 -0
- package/dist/code.d.ts.map +1 -0
- package/dist/code.js +326 -0
- package/dist/code.js.map +1 -0
- package/dist/compaction.d.ts +15 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +168 -0
- package/dist/compaction.js.map +1 -0
- package/dist/delegation.d.ts +135 -0
- package/dist/delegation.d.ts.map +1 -0
- package/dist/delegation.js +401 -0
- package/dist/delegation.js.map +1 -0
- package/dist/evals/driver.d.ts +10 -0
- package/dist/evals/driver.d.ts.map +1 -0
- package/dist/evals/driver.js +192 -0
- package/dist/evals/driver.js.map +1 -0
- package/dist/evals/journal.d.ts +26 -0
- package/dist/evals/journal.d.ts.map +1 -0
- package/dist/evals/journal.js +483 -0
- package/dist/evals/journal.js.map +1 -0
- package/dist/evals/judge.d.ts +31 -0
- package/dist/evals/judge.d.ts.map +1 -0
- package/dist/evals/judge.js +142 -0
- package/dist/evals/judge.js.map +1 -0
- package/dist/evals/report.d.ts +38 -0
- package/dist/evals/report.d.ts.map +1 -0
- package/dist/evals/report.js +241 -0
- package/dist/evals/report.js.map +1 -0
- package/dist/evals/runner.d.ts +24 -0
- package/dist/evals/runner.d.ts.map +1 -0
- package/dist/evals/runner.js +655 -0
- package/dist/evals/runner.js.map +1 -0
- package/dist/evals/scorers.d.ts +36 -0
- package/dist/evals/scorers.d.ts.map +1 -0
- package/dist/evals/scorers.js +259 -0
- package/dist/evals/scorers.js.map +1 -0
- package/dist/evals/simulation.d.ts +7 -0
- package/dist/evals/simulation.d.ts.map +1 -0
- package/dist/evals/simulation.js +38 -0
- package/dist/evals/simulation.js.map +1 -0
- package/dist/evals/testing.d.ts +36 -0
- package/dist/evals/testing.d.ts.map +1 -0
- package/dist/evals/testing.js +87 -0
- package/dist/evals/testing.js.map +1 -0
- package/dist/evals/trajectory.d.ts +11 -0
- package/dist/evals/trajectory.d.ts.map +1 -0
- package/dist/evals/trajectory.js +180 -0
- package/dist/evals/trajectory.js.map +1 -0
- package/dist/evals/types.d.ts +431 -0
- package/dist/evals/types.d.ts.map +1 -0
- package/dist/evals/types.js +4 -0
- package/dist/evals/types.js.map +1 -0
- package/dist/evals.d.ts +14 -0
- package/dist/evals.d.ts.map +1 -0
- package/dist/evals.js +9 -0
- package/dist/evals.js.map +1 -0
- package/dist/extensions.d.ts +33 -0
- package/dist/extensions.d.ts.map +1 -0
- package/dist/extensions.js +188 -0
- package/dist/extensions.js.map +1 -0
- package/dist/handoff.d.ts +58 -0
- package/dist/handoff.d.ts.map +1 -0
- package/dist/handoff.js +160 -0
- package/dist/handoff.js.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/input.d.ts +42 -0
- package/dist/input.d.ts.map +1 -0
- package/dist/input.js +266 -0
- package/dist/input.js.map +1 -0
- package/dist/mcp.d.ts +77 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +505 -0
- package/dist/mcp.js.map +1 -0
- package/dist/memory-store.d.ts +11 -0
- package/dist/memory-store.d.ts.map +1 -0
- package/dist/memory-store.js +272 -0
- package/dist/memory-store.js.map +1 -0
- package/dist/memory.d.ts +212 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +613 -0
- package/dist/memory.js.map +1 -0
- package/dist/otel.d.ts +74 -0
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +427 -0
- package/dist/otel.js.map +1 -0
- package/dist/progress.d.ts +17 -0
- package/dist/progress.d.ts.map +1 -0
- package/dist/progress.js +193 -0
- package/dist/progress.js.map +1 -0
- package/dist/quickjs.d.ts +19 -0
- package/dist/quickjs.d.ts.map +1 -0
- package/dist/quickjs.js +519 -0
- package/dist/quickjs.js.map +1 -0
- package/dist/recovery.d.ts +10 -0
- package/dist/recovery.d.ts.map +1 -0
- package/dist/recovery.js +62 -0
- package/dist/recovery.js.map +1 -0
- package/dist/request.d.ts +88 -0
- package/dist/request.d.ts.map +1 -0
- package/dist/request.js +422 -0
- package/dist/request.js.map +1 -0
- package/dist/run.d.ts +20 -0
- package/dist/run.d.ts.map +1 -0
- package/dist/run.js +2407 -0
- package/dist/run.js.map +1 -0
- package/dist/runs.d.ts +110 -0
- package/dist/runs.d.ts.map +1 -0
- package/dist/runs.js +438 -0
- package/dist/runs.js.map +1 -0
- package/dist/schema-to-ts.d.ts +4 -0
- package/dist/schema-to-ts.d.ts.map +1 -0
- package/dist/schema-to-ts.js +191 -0
- package/dist/schema-to-ts.js.map +1 -0
- package/dist/session.d.ts +23 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +137 -0
- package/dist/session.js.map +1 -0
- package/dist/snapshot.d.ts +75 -0
- package/dist/snapshot.d.ts.map +1 -0
- package/dist/snapshot.js +1607 -0
- package/dist/snapshot.js.map +1 -0
- package/dist/steering.d.ts +75 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/steering.js +214 -0
- package/dist/steering.js.map +1 -0
- package/dist/subagents.d.ts +141 -0
- package/dist/subagents.d.ts.map +1 -0
- package/dist/subagents.js +746 -0
- package/dist/subagents.js.map +1 -0
- package/dist/tool-retry.d.ts +12 -0
- package/dist/tool-retry.d.ts.map +1 -0
- package/dist/tool-retry.js +53 -0
- package/dist/tool-retry.js.map +1 -0
- package/dist/tree.d.ts +98 -0
- package/dist/tree.d.ts.map +1 -0
- package/dist/tree.js +320 -0
- package/dist/tree.js.map +1 -0
- package/dist/types.d.ts +1007 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/workflow-snapshot.d.ts +8 -0
- package/dist/workflow-snapshot.d.ts.map +1 -0
- package/dist/workflow-snapshot.js +276 -0
- package/dist/workflow-snapshot.js.map +1 -0
- package/dist/workflow-types.d.ts +171 -0
- package/dist/workflow-types.d.ts.map +1 -0
- package/dist/workflow-types.js +2 -0
- package/dist/workflow-types.js.map +1 -0
- package/dist/workflow.d.ts +19 -0
- package/dist/workflow.d.ts.map +1 -0
- package/dist/workflow.js +1053 -0
- package/dist/workflow.js.map +1 -0
- package/examples/a2a-remote.ts +85 -0
- package/examples/a2a-server.ts +100 -0
- package/examples/background.ts +59 -0
- package/examples/basic.ts +44 -0
- package/examples/code-mode.ts +76 -0
- package/examples/conversation.ts +57 -0
- package/examples/evals.ts +79 -0
- package/examples/guardrails.ts +56 -0
- package/examples/handoff.ts +79 -0
- package/examples/memory.ts +44 -0
- package/examples/otel.ts +99 -0
- package/examples/specialists.ts +99 -0
- package/examples/steering.ts +69 -0
- package/examples/structured-handoff.ts +92 -0
- package/examples/subagents.ts +127 -0
- package/examples/workflow.ts +74 -0
- package/package.json +114 -4
- package/src/a2a/auth.ts +62 -0
- package/src/a2a/client.ts +733 -0
- package/src/a2a/index.ts +74 -0
- package/src/a2a/memory-runs.ts +448 -0
- package/src/a2a/policy.ts +148 -0
- package/src/a2a/push.ts +251 -0
- package/src/a2a/remote.ts +1652 -0
- package/src/a2a/server.ts +1390 -0
- package/src/a2a/signing.ts +348 -0
- package/src/a2a/tasks.ts +260 -0
- package/src/a2a/wire.ts +1443 -0
- package/src/background-types.ts +86 -0
- package/src/child.ts +181 -0
- package/src/code-driver.ts +48 -0
- package/src/code-mode.ts +12 -0
- package/src/code-script.ts +149 -0
- package/src/code.ts +424 -0
- package/src/compaction.ts +212 -0
- package/src/delegation.ts +694 -0
- package/src/evals/driver.ts +222 -0
- package/src/evals/journal.ts +605 -0
- package/src/evals/judge.ts +188 -0
- package/src/evals/report.ts +295 -0
- package/src/evals/runner.ts +826 -0
- package/src/evals/scorers.ts +339 -0
- package/src/evals/simulation.ts +46 -0
- package/src/evals/testing.ts +126 -0
- package/src/evals/trajectory.ts +223 -0
- package/src/evals/types.ts +463 -0
- package/src/evals.ts +72 -0
- package/src/extensions.ts +247 -0
- package/src/handoff.ts +214 -0
- package/src/index.ts +62 -0
- package/src/input.ts +319 -0
- package/src/mcp.ts +690 -0
- package/src/memory-store.ts +366 -0
- package/src/memory.ts +893 -0
- package/src/otel.ts +568 -0
- package/src/progress.ts +214 -0
- package/src/quickjs.ts +524 -0
- package/src/recovery.ts +90 -0
- package/src/request.ts +513 -0
- package/src/run.ts +3199 -0
- package/src/runs.ts +696 -0
- package/src/schema-to-ts.ts +194 -0
- package/src/session.ts +178 -0
- package/src/snapshot.ts +2099 -0
- package/src/steering.ts +329 -0
- package/src/subagents.ts +1084 -0
- package/src/tool-retry.ts +86 -0
- package/src/tree.ts +366 -0
- package/src/types.ts +944 -0
- package/src/workflow-snapshot.ts +366 -0
- package/src/workflow-types.ts +213 -0
- package/src/workflow.ts +1316 -0
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
import { addUsage, costUsd, EMPTY_USAGE, HOSTED_TOOL_TYPES, parseToolArgs, } from "@providerkit/core";
|
|
2
|
+
import { RESULT_TOOL_NAME, } from "../index.js";
|
|
3
|
+
import { allModels } from "../compaction.js";
|
|
4
|
+
import { inputOf, readInput } from "../input.js";
|
|
5
|
+
import { agentSnapshot, copyData, isId, isRecord, isUsage, jsonArguments, readLimits, readSnapshot, } from "../snapshot.js";
|
|
6
|
+
import { snapshots } from "../tree.js";
|
|
7
|
+
import { checkExpectation } from "./trajectory.js";
|
|
8
|
+
/** Uses the SDK's structural and journal checks, never resume or an output validator. A stored
|
|
9
|
+
* journal is data, not permission to run a tool. Unknown output contracts stay unknown JSON. */
|
|
10
|
+
export function readJournal(raw) {
|
|
11
|
+
const value = isRecord(raw) && "snapshot" in raw ? raw.snapshot : raw;
|
|
12
|
+
if (!isRecord(value) || value.kind !== "agent" || !isId(value.agentId) || !isId(value.version))
|
|
13
|
+
throw new TypeError("Provide a format-3 agent run journal or an SDK agent outcome.");
|
|
14
|
+
const limits = readLimits(value.limits);
|
|
15
|
+
const definition = value.definition;
|
|
16
|
+
const checkedOutput = isRecord(definition) &&
|
|
17
|
+
Array.isArray(definition.enabledTools) &&
|
|
18
|
+
definition.enabledTools.includes(RESULT_TOOL_NAME);
|
|
19
|
+
const root = readSnapshot(value, value.agentId, value.version, limits, checkedOutput ? RESULT_TOOL_NAME : undefined);
|
|
20
|
+
// readSnapshot checks the mixed tree's shape and every workflow's operations. Agent
|
|
21
|
+
// descendants need their own operation checks, including those below workflow steps.
|
|
22
|
+
for (const node of snapshots(root)) {
|
|
23
|
+
if (node.kind !== "agent")
|
|
24
|
+
continue;
|
|
25
|
+
if (node !== root)
|
|
26
|
+
readSnapshot(node, node.agentId, node.version, node.limits, node.definition.enabledTools.includes(RESULT_TOOL_NAME) ? RESULT_TOOL_NAME : undefined);
|
|
27
|
+
for (const model of allModels(node)) {
|
|
28
|
+
checkUsage(model.usage);
|
|
29
|
+
for (const attempt of model.interrupted)
|
|
30
|
+
checkUsage(attempt.usage);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return root;
|
|
34
|
+
}
|
|
35
|
+
function checkUsage(value) {
|
|
36
|
+
if (!isUsage(value) ||
|
|
37
|
+
(value.hostedCalls !== undefined &&
|
|
38
|
+
(!isRecord(value.hostedCalls) ||
|
|
39
|
+
Object.entries(value.hostedCalls).some(([key, count]) => !HOSTED_TOOL_TYPES.some((type) => type === key) ||
|
|
40
|
+
typeof count !== "number" ||
|
|
41
|
+
!Number.isFinite(count) ||
|
|
42
|
+
count < 0))))
|
|
43
|
+
throw new TypeError("The journal has invalid token or hosted-call usage.");
|
|
44
|
+
}
|
|
45
|
+
export function emptyUsage() {
|
|
46
|
+
return {
|
|
47
|
+
usage: { ...EMPTY_USAGE },
|
|
48
|
+
knownCostUsd: 0,
|
|
49
|
+
costUsd: 0,
|
|
50
|
+
unknownCharges: 0,
|
|
51
|
+
unpricedCalls: 0,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export function combineUsage(values) {
|
|
55
|
+
const result = emptyUsage();
|
|
56
|
+
for (const value of values) {
|
|
57
|
+
result.usage = addUsage(result.usage, value.usage);
|
|
58
|
+
result.knownCostUsd += value.knownCostUsd;
|
|
59
|
+
result.unknownCharges += value.unknownCharges;
|
|
60
|
+
result.unpricedCalls += value.unpricedCalls;
|
|
61
|
+
}
|
|
62
|
+
result.costUsd = result.unknownCharges || result.unpricedCalls ? null : result.knownCostUsd;
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
export function usageOf(root, price) {
|
|
66
|
+
const totals = emptyUsage();
|
|
67
|
+
const seen = new Set();
|
|
68
|
+
for (const node of snapshots(root)) {
|
|
69
|
+
if (node.kind === "remote") {
|
|
70
|
+
// The peer supplied no local model bill. Each admitted send/cancel remains unknown.
|
|
71
|
+
totals.unknownCharges += node.remote.sends.length + (node.remote.cancel ? 1 : 0);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (node.kind !== "agent")
|
|
75
|
+
continue;
|
|
76
|
+
for (const model of allModels(node)) {
|
|
77
|
+
if (seen.has(model.id))
|
|
78
|
+
continue;
|
|
79
|
+
seen.add(model.id);
|
|
80
|
+
const ref = {
|
|
81
|
+
agentId: node.agentId,
|
|
82
|
+
...(model.status === "completed" ? model.turn.source : undefined),
|
|
83
|
+
};
|
|
84
|
+
for (const attempt of [
|
|
85
|
+
...model.interrupted.map((attempt) => ({ ...attempt, billing: "unknown" })),
|
|
86
|
+
model,
|
|
87
|
+
]) {
|
|
88
|
+
checkUsage(attempt.usage);
|
|
89
|
+
totals.usage = addUsage(totals.usage, attempt.usage);
|
|
90
|
+
if (attempt.billing === "unknown")
|
|
91
|
+
totals.unknownCharges++;
|
|
92
|
+
const measured = attempt.usage;
|
|
93
|
+
if (measured.reportedCostUsd !== undefined) {
|
|
94
|
+
totals.knownCostUsd += measured.reportedCostUsd;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
// A reserved operation with no usage has not given us a priceable measurement yet.
|
|
98
|
+
const hasMeasurement = measured.inputTokens + measured.outputTokens > 0 ||
|
|
99
|
+
Object.values(measured.hostedCalls ?? {}).some((n) => n > 0);
|
|
100
|
+
if (!hasMeasurement && model.status === "running" && root.state.status === "running")
|
|
101
|
+
continue;
|
|
102
|
+
// An interrupted attempt can have used a different fallback route. The replacement
|
|
103
|
+
// turn's source is not evidence of which model served the lost attempt.
|
|
104
|
+
const rate = price?.(attempt === model ? ref : { agentId: node.agentId });
|
|
105
|
+
if (rate &&
|
|
106
|
+
[
|
|
107
|
+
rate.input,
|
|
108
|
+
rate.output,
|
|
109
|
+
rate.cacheRead,
|
|
110
|
+
rate.cacheWrite ?? rate.input,
|
|
111
|
+
...Object.values(rate.hosted ?? {}),
|
|
112
|
+
].some((n) => !Number.isFinite(n) || n < 0))
|
|
113
|
+
throw new TypeError("Prices must be finite, non-negative USD rates.");
|
|
114
|
+
const missingHostedRate = HOSTED_TOOL_TYPES.some((type) => (measured.hostedCalls?.[type] ?? 0) > 0 && rate?.hosted?.[type] === undefined);
|
|
115
|
+
const settled = attempt !== model || model.status === "completed" || root.state.status !== "running";
|
|
116
|
+
// A running receipt has no source yet. Do not reject a model-specific caller price
|
|
117
|
+
// before the completed turn identifies the model. The unknown charge still counts.
|
|
118
|
+
if ((!rate || missingHostedRate) && settled)
|
|
119
|
+
totals.unpricedCalls++;
|
|
120
|
+
if (rate)
|
|
121
|
+
totals.knownCostUsd += costUsd(measured, rate);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
totals.costUsd = totals.unknownCharges || totals.unpricedCalls ? null : totals.knownCostUsd;
|
|
126
|
+
return totals;
|
|
127
|
+
}
|
|
128
|
+
function childTrajectory({ key, snapshot, }) {
|
|
129
|
+
return {
|
|
130
|
+
key,
|
|
131
|
+
...(snapshot.kind === "workflow"
|
|
132
|
+
? { kind: "workflow", workflowId: snapshot.workflowId }
|
|
133
|
+
: snapshot.kind === "remote"
|
|
134
|
+
? { kind: "remote", agentId: snapshot.agentId, version: snapshot.version }
|
|
135
|
+
: { kind: "agent", agentId: snapshot.agentId }),
|
|
136
|
+
runId: snapshot.runId,
|
|
137
|
+
trajectory: journalTrajectory(snapshot),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
function remoteAttempts(snapshot) {
|
|
141
|
+
return [
|
|
142
|
+
...snapshot.remote.sends.map((send) => ({ ...send, action: "send" })),
|
|
143
|
+
...(snapshot.remote.cancel ? [{ ...snapshot.remote.cancel, action: "cancel" }] : []),
|
|
144
|
+
].map((receipt) => ({
|
|
145
|
+
kind: "remote",
|
|
146
|
+
action: receipt.action,
|
|
147
|
+
agentId: snapshot.agentId,
|
|
148
|
+
version: snapshot.version,
|
|
149
|
+
runId: snapshot.runId,
|
|
150
|
+
operationId: receipt.id,
|
|
151
|
+
attemptId: receipt.attemptId,
|
|
152
|
+
status: receipt.status,
|
|
153
|
+
billing: "unknown",
|
|
154
|
+
}));
|
|
155
|
+
}
|
|
156
|
+
function approvedInput(input, asks = []) {
|
|
157
|
+
let effective = input;
|
|
158
|
+
for (const ask of asks) {
|
|
159
|
+
if (ask.decision?.type === "approve")
|
|
160
|
+
effective = ask.decision.arguments ?? input;
|
|
161
|
+
}
|
|
162
|
+
return effective;
|
|
163
|
+
}
|
|
164
|
+
// Admission reserves budget for a pending ask; evals count only journaled attempt receipts.
|
|
165
|
+
function toolAttempts(tool) {
|
|
166
|
+
return tool.interrupted.length + tool.retries.length + (tool.status === "pending" ? 0 : 1);
|
|
167
|
+
}
|
|
168
|
+
/** Admission order comes from the journal, never completion time. Workflow steps retain their
|
|
169
|
+
* own kind and input; only agent nodes contribute model/tool calls. Hosted charges enter usageOf. */
|
|
170
|
+
export function journalTrajectory(root) {
|
|
171
|
+
const result = {
|
|
172
|
+
steps: [],
|
|
173
|
+
flat: [],
|
|
174
|
+
modelCalls: root.kind === "agent"
|
|
175
|
+
? allModels(root).reduce((count, model) => count + 1 + model.interrupted.length, 0)
|
|
176
|
+
: 0,
|
|
177
|
+
toolCalls: 0,
|
|
178
|
+
};
|
|
179
|
+
if (root.kind === "workflow") {
|
|
180
|
+
result.steps = root.steps.map((step) => ({
|
|
181
|
+
kind: step.kind,
|
|
182
|
+
name: step.key,
|
|
183
|
+
workflowId: root.workflowId,
|
|
184
|
+
runId: root.runId,
|
|
185
|
+
operationId: step.id,
|
|
186
|
+
input: copyData(approvedInput(step.definition.input, step.asks)),
|
|
187
|
+
...(step.status === "completed" ? { output: copyData(step.result) } : {}),
|
|
188
|
+
status: step.status,
|
|
189
|
+
attempts: step.kind === "step"
|
|
190
|
+
? step.attempts.filter((attempt) => attempt.status !== "not_run").length
|
|
191
|
+
: step.kind === "agent" || step.kind === "remote" || step.kind === "workflow"
|
|
192
|
+
? step.children.length
|
|
193
|
+
: 1,
|
|
194
|
+
asks: copyData(step.asks ?? []),
|
|
195
|
+
children: step.children.map(childTrajectory),
|
|
196
|
+
}));
|
|
197
|
+
}
|
|
198
|
+
else if (root.kind === "remote") {
|
|
199
|
+
result.steps = remoteAttempts(root).map((receipt, index) => ({
|
|
200
|
+
...receipt,
|
|
201
|
+
kind: "remote-operation",
|
|
202
|
+
name: `remote_${receipt.action}`,
|
|
203
|
+
attempts: 1,
|
|
204
|
+
...(receipt.action === "send" && {
|
|
205
|
+
input: copyData(root.remote.sends[index]?.message.parts),
|
|
206
|
+
}),
|
|
207
|
+
asks: copyData(root.remote.sends[index]?.asks ?? []),
|
|
208
|
+
children: [],
|
|
209
|
+
}));
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
for (const [index, model] of root.models.entries()) {
|
|
213
|
+
if (model.status !== "completed")
|
|
214
|
+
continue;
|
|
215
|
+
for (const [position, call] of (model.turn.message.toolCalls ?? []).entries()) {
|
|
216
|
+
const execution = model.tools[position];
|
|
217
|
+
if (!execution || call.name === RESULT_TOOL_NAME)
|
|
218
|
+
continue;
|
|
219
|
+
const attempts = toolAttempts(execution);
|
|
220
|
+
if (!attempts && !execution.calls?.length)
|
|
221
|
+
continue;
|
|
222
|
+
const journal = root.delegations.find((entry) => entry.operationId === execution.id);
|
|
223
|
+
const code = Object.hasOwn(root.definition.codeTools ?? {}, call.name);
|
|
224
|
+
const args = approvedInput(code ? (jsonArguments(call.arguments) ?? {}) : parseToolArgs(call.arguments), execution.asks);
|
|
225
|
+
result.steps.push({
|
|
226
|
+
kind: code ? "code" : (journal?.kind ?? "tool"),
|
|
227
|
+
name: call.name,
|
|
228
|
+
arguments: copyData(args),
|
|
229
|
+
status: execution.status === "completed"
|
|
230
|
+
? "completed"
|
|
231
|
+
: execution.status === "pending"
|
|
232
|
+
? "interrupted"
|
|
233
|
+
: "running",
|
|
234
|
+
attempts,
|
|
235
|
+
...(execution.status === "completed" ? { result: copyData(execution.result) } : {}),
|
|
236
|
+
asks: copyData(execution.asks ?? []),
|
|
237
|
+
agentId: root.agentId,
|
|
238
|
+
runId: root.runId,
|
|
239
|
+
operationId: execution.id,
|
|
240
|
+
turn: index + 1,
|
|
241
|
+
children: (journal?.children ?? []).map(childTrajectory),
|
|
242
|
+
});
|
|
243
|
+
result.toolCalls += attempts;
|
|
244
|
+
for (const inner of execution.calls ?? []) {
|
|
245
|
+
const attempts = toolAttempts(inner);
|
|
246
|
+
result.steps.push({
|
|
247
|
+
kind: "tool",
|
|
248
|
+
name: inner.name,
|
|
249
|
+
arguments: copyData(approvedInput(inner.arguments, inner.asks)),
|
|
250
|
+
status: inner.status,
|
|
251
|
+
attempts,
|
|
252
|
+
...(inner.status === "completed" && { result: copyData(inner.result) }),
|
|
253
|
+
asks: copyData(inner.asks ?? []),
|
|
254
|
+
agentId: root.agentId,
|
|
255
|
+
runId: root.runId,
|
|
256
|
+
operationId: inner.id,
|
|
257
|
+
parentOperationId: execution.id,
|
|
258
|
+
turn: index + 1,
|
|
259
|
+
children: [],
|
|
260
|
+
});
|
|
261
|
+
result.toolCalls += attempts;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
for (const step of result.steps) {
|
|
267
|
+
result.flat.push(step);
|
|
268
|
+
for (const child of step.children) {
|
|
269
|
+
result.flat.push(...child.trajectory.flat);
|
|
270
|
+
result.modelCalls += child.trajectory.modelCalls;
|
|
271
|
+
result.toolCalls += child.trajectory.toolCalls;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
return result;
|
|
275
|
+
}
|
|
276
|
+
export function trajectoryOf(raw) {
|
|
277
|
+
return journalTrajectory(readJournal(raw));
|
|
278
|
+
}
|
|
279
|
+
export function evalRun(root, price, durationMs) {
|
|
280
|
+
// The last admitted handoff owns the answer; delegated specialists do not.
|
|
281
|
+
let active = root;
|
|
282
|
+
while (true) {
|
|
283
|
+
const child = active.delegations.find((entry) => entry.kind === "handoff" && entry.children.length > 0)?.children[0];
|
|
284
|
+
if (!child)
|
|
285
|
+
break;
|
|
286
|
+
active = agentSnapshot(child.snapshot);
|
|
287
|
+
}
|
|
288
|
+
const last = active.models.at(-1);
|
|
289
|
+
return {
|
|
290
|
+
runId: root.runId,
|
|
291
|
+
agentId: root.agentId,
|
|
292
|
+
version: root.version,
|
|
293
|
+
input: inputOf(root),
|
|
294
|
+
status: root.state.status,
|
|
295
|
+
...(root.state.status === "completed" ? { value: copyData(root.state.candidate) } : {}),
|
|
296
|
+
text: last?.status === "completed" ? last.turn.message.content : "",
|
|
297
|
+
snapshot: root,
|
|
298
|
+
activeAgent: { id: active.agentId, version: active.version },
|
|
299
|
+
trajectory: journalTrajectory(root),
|
|
300
|
+
durationMs,
|
|
301
|
+
...usageOf(root, price),
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
export function runResult(snapshot, usage, role, includePrivateData = false, turn, durationMs) {
|
|
305
|
+
const run = evalRun(snapshot, undefined, durationMs);
|
|
306
|
+
const agents = [...snapshots(snapshot)].filter((node) => node.kind === "agent");
|
|
307
|
+
return {
|
|
308
|
+
runId: snapshot.runId,
|
|
309
|
+
attemptId: snapshot.attemptId,
|
|
310
|
+
agent: { id: snapshot.agentId, version: snapshot.version },
|
|
311
|
+
activeAgent: run.activeAgent,
|
|
312
|
+
role,
|
|
313
|
+
turn,
|
|
314
|
+
status: snapshot.state.status,
|
|
315
|
+
modelCalls: run.trajectory.modelCalls,
|
|
316
|
+
toolCalls: run.trajectory.toolCalls,
|
|
317
|
+
durationMs,
|
|
318
|
+
...usage,
|
|
319
|
+
modelAttempts: agents.flatMap((node) => allModels(node).flatMap((model) => [
|
|
320
|
+
...model.interrupted.map((attempt) => ({
|
|
321
|
+
operationId: model.id,
|
|
322
|
+
attemptId: attempt.attemptId,
|
|
323
|
+
status: "interrupted",
|
|
324
|
+
billing: "unknown",
|
|
325
|
+
usage: copyData(attempt.usage),
|
|
326
|
+
})),
|
|
327
|
+
{
|
|
328
|
+
operationId: model.id,
|
|
329
|
+
attemptId: model.attemptId,
|
|
330
|
+
status: model.status,
|
|
331
|
+
billing: model.billing,
|
|
332
|
+
usage: copyData(model.usage),
|
|
333
|
+
},
|
|
334
|
+
])),
|
|
335
|
+
toolAttempts: agents.flatMap((node) => node.models.flatMap((model) => model.status !== "completed"
|
|
336
|
+
? []
|
|
337
|
+
: model.tools.flatMap((outer) => [
|
|
338
|
+
{ tool: outer, parentOperationId: undefined },
|
|
339
|
+
...(outer.calls ?? []).map((tool) => ({ tool, parentOperationId: outer.id })),
|
|
340
|
+
].flatMap(({ tool, parentOperationId }) => [
|
|
341
|
+
...tool.interrupted.map((attempt) => ({
|
|
342
|
+
operationId: tool.id,
|
|
343
|
+
...(parentOperationId && { parentOperationId }),
|
|
344
|
+
attemptId: attempt.attemptId,
|
|
345
|
+
status: "interrupted",
|
|
346
|
+
})),
|
|
347
|
+
...tool.retries.map((attempt) => ({
|
|
348
|
+
operationId: tool.id,
|
|
349
|
+
...(parentOperationId && { parentOperationId }),
|
|
350
|
+
attemptId: attempt.attemptId,
|
|
351
|
+
status: "completed",
|
|
352
|
+
ok: false,
|
|
353
|
+
})),
|
|
354
|
+
...(tool.status === "pending"
|
|
355
|
+
? []
|
|
356
|
+
: [
|
|
357
|
+
{
|
|
358
|
+
operationId: tool.id,
|
|
359
|
+
...(parentOperationId && { parentOperationId }),
|
|
360
|
+
attemptId: tool.attemptId,
|
|
361
|
+
status: tool.status === "completed"
|
|
362
|
+
? "completed"
|
|
363
|
+
: "running",
|
|
364
|
+
...(tool.status === "completed" ? { ok: tool.result.ok } : {}),
|
|
365
|
+
},
|
|
366
|
+
]),
|
|
367
|
+
])))),
|
|
368
|
+
remoteAttempts: [...snapshots(snapshot)].flatMap((node) => node.kind === "remote" ? remoteAttempts(node) : []),
|
|
369
|
+
...(includePrivateData ? { snapshot: copyData(snapshot) } : {}),
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
/** Strip executable scorers before freezing context or validating stored case data. */
|
|
373
|
+
export function caseData(item) {
|
|
374
|
+
const { scorers: _scorers, turns, ...data } = item;
|
|
375
|
+
return {
|
|
376
|
+
...data,
|
|
377
|
+
...(turns ? { turns: turns.map(({ scorers: _turnScorers, ...turn }) => turn) } : {}),
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
export function readCases(raw, validateExpected) {
|
|
381
|
+
if (!Array.isArray(raw))
|
|
382
|
+
throw new TypeError("Cases must be an array.");
|
|
383
|
+
const ids = new Set();
|
|
384
|
+
function expectations(value) {
|
|
385
|
+
const status = value.expectStatus;
|
|
386
|
+
if (status !== undefined &&
|
|
387
|
+
status !== "completed" &&
|
|
388
|
+
status !== "failed" &&
|
|
389
|
+
status !== "limited" &&
|
|
390
|
+
status !== "suspended" &&
|
|
391
|
+
status !== "cancelled")
|
|
392
|
+
throw new TypeError("Invalid expected run status.");
|
|
393
|
+
if (value.trajectory !== undefined)
|
|
394
|
+
checkExpectation(value.trajectory);
|
|
395
|
+
return {
|
|
396
|
+
...(Object.hasOwn(value, "expected")
|
|
397
|
+
? {
|
|
398
|
+
expected: validateExpected
|
|
399
|
+
? validateExpected(value.expected)
|
|
400
|
+
: copyData(value.expected),
|
|
401
|
+
}
|
|
402
|
+
: {}),
|
|
403
|
+
...(status ? { expectStatus: status } : {}),
|
|
404
|
+
...(value.trajectory === undefined ? {} : { trajectory: value.trajectory }),
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
return raw.map((rawValue) => {
|
|
408
|
+
const value = copyData(rawValue);
|
|
409
|
+
if (!isRecord(value) ||
|
|
410
|
+
!isId(value.id) ||
|
|
411
|
+
ids.has(value.id) ||
|
|
412
|
+
Object.keys(value).some((key) => !["id", "input", "turns", "user", "expected", "expectStatus", "trajectory"].includes(key)) ||
|
|
413
|
+
[value.input, value.turns, value.user].filter((entry) => entry !== undefined).length !== 1)
|
|
414
|
+
throw new TypeError("Each case needs a unique id and exactly one of input, turns and user. Drivers and code are not case data.");
|
|
415
|
+
ids.add(value.id);
|
|
416
|
+
const base = { id: value.id, ...expectations(value) };
|
|
417
|
+
if (value.turns !== undefined) {
|
|
418
|
+
if (!Array.isArray(value.turns) || !value.turns.length)
|
|
419
|
+
throw new TypeError("A conversation case needs at least one turn.");
|
|
420
|
+
const turns = value.turns.map((turn) => {
|
|
421
|
+
if (!isRecord(turn) ||
|
|
422
|
+
Object.keys(turn).some((key) => !["input", "expected", "expectStatus", "trajectory"].includes(key)) ||
|
|
423
|
+
isRecord(turn.input))
|
|
424
|
+
throw new TypeError("Each session turn needs a new user input and only supported data fields.");
|
|
425
|
+
return { input: readInput(turn.input, "caller").input, ...expectations(turn) };
|
|
426
|
+
});
|
|
427
|
+
return { ...base, turns };
|
|
428
|
+
}
|
|
429
|
+
if (value.user !== undefined) {
|
|
430
|
+
const user = value.user;
|
|
431
|
+
if (!isRecord(user) ||
|
|
432
|
+
Object.keys(user).some((key) => !["opening", "plan", "persona", "maxTurns"].includes(key)) ||
|
|
433
|
+
!isId(user.plan) ||
|
|
434
|
+
(user.persona !== undefined && !isId(user.persona)) ||
|
|
435
|
+
isRecord(user.opening) ||
|
|
436
|
+
(user.maxTurns !== undefined &&
|
|
437
|
+
(typeof user.maxTurns !== "number" ||
|
|
438
|
+
!Number.isSafeInteger(user.maxTurns) ||
|
|
439
|
+
user.maxTurns < 1 ||
|
|
440
|
+
user.maxTurns > 100)))
|
|
441
|
+
throw new TypeError("A simulated user needs a plan, an opening and a turn limit from 1 to 100.");
|
|
442
|
+
return {
|
|
443
|
+
...base,
|
|
444
|
+
user: {
|
|
445
|
+
opening: readInput(user.opening, "caller").input,
|
|
446
|
+
plan: user.plan,
|
|
447
|
+
...(user.persona === undefined ? {} : { persona: user.persona }),
|
|
448
|
+
...(user.maxTurns === undefined ? {} : { maxTurns: user.maxTurns }),
|
|
449
|
+
},
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
const input = readInput(value.input, "caller");
|
|
453
|
+
return {
|
|
454
|
+
...base,
|
|
455
|
+
input: input.history.length
|
|
456
|
+
? { messages: [...input.history, { role: "user", content: input.input }] }
|
|
457
|
+
: input.input,
|
|
458
|
+
};
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
/** Explicitly copies private input, output and tool arguments. Nothing is executed or approved.
|
|
462
|
+
* Supply a sandboxed agent when you later run this case; stored results are not tool mocks. */
|
|
463
|
+
export function caseFromRun(raw, options) {
|
|
464
|
+
if (!isId(options.id))
|
|
465
|
+
throw new TypeError("Give the golden case a non-empty id.");
|
|
466
|
+
const snapshot = readJournal(raw);
|
|
467
|
+
if (snapshot.state.status !== "completed")
|
|
468
|
+
throw new TypeError("Only a completed run can become a golden case. No interrupted work is resumed.");
|
|
469
|
+
if ([...snapshots(snapshot)].some((node) => node.kind === "agent" && node.inputs.length > 0))
|
|
470
|
+
throw new TypeError("This run received messages while working. Record its input timing in the host instead of replaying it as one case.");
|
|
471
|
+
return {
|
|
472
|
+
id: options.id,
|
|
473
|
+
input: inputOf(snapshot),
|
|
474
|
+
expected: copyData(snapshot.state.candidate),
|
|
475
|
+
trajectory: {
|
|
476
|
+
match: "exact",
|
|
477
|
+
calls: journalTrajectory(snapshot)
|
|
478
|
+
.steps.filter((step) => step.result?.ok)
|
|
479
|
+
.map((step) => ({ name: step.name, arguments: step.arguments, argumentsMatch: "exact" })),
|
|
480
|
+
},
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
//# sourceMappingURL=journal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"journal.js","sourceRoot":"","sources":["../../src/evals/journal.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,aAAa,GAEd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,gBAAgB,GAKjB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EACL,aAAa,EACb,QAAQ,EACR,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,aAAa,EACb,UAAU,EACV,YAAY,GACb,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAanD;gGACgG;AAChG,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;IACtE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAC5F,MAAM,IAAI,SAAS,CAAC,+DAA+D,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACpC,MAAM,aAAa,GACjB,QAAQ,CAAC,UAAU,CAAC;QACpB,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;QACtC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,YAAY,CACvB,KAAK,EACL,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,OAAO,EACb,MAAM,EACN,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAC7C,CAAC;IACF,oFAAoF;IACpF,qFAAqF;IACrF,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QACpC,IAAI,IAAI,KAAK,IAAI;YACf,YAAY,CACV,IAAI,EACJ,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CACvF,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACxB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,WAAW;gBAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB;IACnC,IACE,CAAC,OAAO,CAAC,KAAK,CAAC;QACf,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS;YAC9B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;gBAC3B,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC;oBAC/C,OAAO,KAAK,KAAK,QAAQ;oBACzB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;oBACvB,KAAK,GAAG,CAAC,CACZ,CAAC,CAAC;QAEP,MAAM,IAAI,SAAS,CAAC,qDAAqD,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO;QACL,KAAK,EAAE,EAAE,GAAG,WAAW,EAAE;QACzB,YAAY,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAA2B;IACtD,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,CAAC;QAC1C,MAAM,CAAC,cAAc,IAAI,KAAK,CAAC,cAAc,CAAC;QAC9C,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,CAAC;IAC9C,CAAC;IACD,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAC5F,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAc,EAAE,KAAiB;IACvD,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,oFAAoF;YACpF,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACjF,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QACpC,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,SAAS;YACjC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnB,MAAM,GAAG,GAAa;gBACpB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;aAClE,CAAC;YACF,KAAK,MAAM,OAAO,IAAI;gBACpB,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC3E,KAAK;aACN,EAAE,CAAC;gBACF,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;gBACrD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;oBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;gBAC/B,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;oBAC3C,MAAM,CAAC,YAAY,IAAI,QAAQ,CAAC,eAAe,CAAC;oBAChD,SAAS;gBACX,CAAC;gBACD,mFAAmF;gBACnF,MAAM,cAAc,GAClB,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,YAAY,GAAG,CAAC;oBAChD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/D,IAAI,CAAC,cAAc,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;oBAClF,SAAS;gBACX,mFAAmF;gBACnF,wEAAwE;gBACxE,MAAM,IAAI,GAAG,KAAK,EAAE,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1E,IACE,IAAI;oBACJ;wBACE,IAAI,CAAC,KAAK;wBACV,IAAI,CAAC,MAAM;wBACX,IAAI,CAAC,SAAS;wBACd,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK;wBAC7B,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;qBACpC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAE3C,MAAM,IAAI,SAAS,CAAC,gDAAgD,CAAC,CAAC;gBACxE,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAC9C,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,SAAS,CACxF,CAAC;gBACF,MAAM,OAAO,GACX,OAAO,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;gBACvF,mFAAmF;gBACnF,mFAAmF;gBACnF,IAAI,CAAC,CAAC,IAAI,IAAI,iBAAiB,CAAC,IAAI,OAAO;oBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;gBACpE,IAAI,IAAI;oBAAE,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAC5F,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,EACvB,GAAG,EACH,QAAQ,GAIT;IACC,OAAO;QACL,GAAG;QACH,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU;YAC9B,CAAC,CAAC,EAAE,IAAI,EAAE,UAAmB,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE;YAChE,CAAC,CAAC,QAAQ,CAAC,IAAI,KAAK,QAAQ;gBAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,QAAiB,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE;gBACnF,CAAC,CAAC,EAAE,IAAI,EAAE,OAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5D,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,iBAAiB,CAAC,QAAQ,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,QAAwB;IAC9C,OAAO;QACL,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,MAAe,EAAE,CAAC,CAAC;QAC9E,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9F,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClB,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,WAAW,EAAE,OAAO,CAAC,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,SAAS;KACnB,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,aAAa,CAAI,KAAQ,EAAE,IAAI,GAAyB,EAAE;IACjE,IAAI,SAAS,GAAgC,KAAK,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,EAAE,IAAI,KAAK,SAAS;YAAE,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,KAAK,CAAC;IACpF,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4FAA4F;AAC5F,SAAS,YAAY,CAAC,IAAmB;IACvC,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7F,CAAC;AAED;qGACqG;AACrG,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,MAAM,MAAM,GAAe;QACzB,KAAK,EAAE,EAAE;QACT,IAAI,EAAE,EAAE;QACR,UAAU,EACR,IAAI,CAAC,IAAI,KAAK,OAAO;YACnB,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;YACnF,CAAC,CAAC,CAAC;QACP,SAAS,EAAE,CAAC;KACb,CAAC;IACF,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,GAAG;YACd,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,EAAE;YACpB,KAAK,EAAE,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAChE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EACN,IAAI,CAAC,IAAI,KAAK,MAAM;gBAClB,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;gBACxE,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU;oBAC3E,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM;oBACtB,CAAC,CAAC,CAAC;YACT,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC;SAC7C,CAAC,CAAC,CAAC;IACN,CAAC;SAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3D,GAAG,OAAO;YACV,IAAI,EAAE,kBAAkB;YACxB,IAAI,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;YAChC,QAAQ,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI;gBAC/B,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;aACzD,CAAC;YACF,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;YACpD,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC,CAAC;IACN,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW;gBAAE,SAAS;YAC3C,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACxC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB;oBAAE,SAAS;gBAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM;oBAAE,SAAS;gBACpD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,CAAC,CAAC;gBACrF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,IAAI,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,aAAa,CACxB,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,EAC5E,SAAS,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;oBAChB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,IAAI,MAAM,CAAC;oBAC/C,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC;oBACzB,MAAM,EACJ,SAAS,CAAC,MAAM,KAAK,WAAW;wBAC9B,CAAC,CAAC,WAAW;wBACb,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,SAAS;4BAC9B,CAAC,CAAC,aAAa;4BACf,CAAC,CAAC,SAAS;oBACjB,QAAQ;oBACR,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnF,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;oBACpC,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,IAAI,EAAE,KAAK,GAAG,CAAC;oBACf,QAAQ,EAAE,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC;iBACzD,CAAC,CAAC;gBACH,MAAM,CAAC,SAAS,IAAI,QAAQ,CAAC;gBAC7B,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;oBAC1C,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;oBACrC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;wBAChB,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS,EAAE,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC/D,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ;wBACR,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;wBACvE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;wBAChC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,WAAW,EAAE,KAAK,CAAC,EAAE;wBACrB,iBAAiB,EAAE,SAAS,CAAC,EAAE;wBAC/B,IAAI,EAAE,KAAK,GAAG,CAAC;wBACf,QAAQ,EAAE,EAAE;qBACb,CAAC,CAAC;oBACH,MAAM,CAAC,SAAS,IAAI,QAAQ,CAAC;gBAC/B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;YACjD,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAY;IACvC,OAAO,iBAAiB,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAiB,EAAE,KAAiB,EAAE,UAAmB;IAC/E,2EAA2E;IAC3E,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CACnC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CACjE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;QACf,IAAI,CAAC,KAAK;YAAE,MAAM;QAClB,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;QACzB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,EAAE,IAAI,EAAE,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QACnE,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE;QAC5D,UAAU,EAAE,iBAAiB,CAAC,IAAI,CAAC;QACnC,UAAU;QACV,GAAG,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,QAAqB,EACrB,KAAgB,EAChB,IAA2B,EAC3B,kBAAkB,GAAG,KAAK,EAC1B,IAAa,EACb,UAAmB;IAEnB,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAChF,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,KAAK,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE;QAC1D,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,IAAI;QACJ,IAAI;QACJ,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;QAC7B,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,UAAU;QACrC,SAAS,EAAE,GAAG,CAAC,UAAU,CAAC,SAAS;QACnC,UAAU;QACV,GAAG,KAAK;QACR,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YACjC,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACrC,WAAW,EAAE,KAAK,CAAC,EAAE;gBACrB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,aAAsB;gBAC9B,OAAO,EAAE,SAAkB;gBAC3B,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;aAC/B,CAAC,CAAC;YACH;gBACE,WAAW,EAAE,KAAK,CAAC,EAAE;gBACrB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;aAC7B;SACF,CAAC,CACH;QACD,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACpC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAC5B,KAAK,CAAC,MAAM,KAAK,WAAW;YAC1B,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAC5B;gBACE,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE;gBAC7C,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;aAC9E,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC;gBACzC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBACpC,WAAW,EAAE,IAAI,CAAC,EAAE;oBACpB,GAAG,CAAC,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;oBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,MAAM,EAAE,aAAsB;iBAC/B,CAAC,CAAC;gBACH,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBAChC,WAAW,EAAE,IAAI,CAAC,EAAE;oBACpB,GAAG,CAAC,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;oBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,MAAM,EAAE,WAAoB;oBAC5B,EAAE,EAAE,KAAK;iBACV,CAAC,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;oBAC3B,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC;wBACE;4BACE,WAAW,EAAE,IAAI,CAAC,EAAE;4BACpB,GAAG,CAAC,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;4BAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;4BACzB,MAAM,EACJ,IAAI,CAAC,MAAM,KAAK,WAAW;gCACzB,CAAC,CAAE,WAAqB;gCACxB,CAAC,CAAE,SAAmB;4BAC1B,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;yBAC/D;qBACF,CAAC;aACP,CAAC,CACH,CACN,CACF;QACD,cAAc,EAAE,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACxD,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CACnD;QACD,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChE,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,QAAQ,CAAI,IAAiB;IAC3C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IACnD,OAAO;QACL,GAAG,IAAI;QACP,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrF,CAAC;AACJ,CAAC;AAMD,MAAM,UAAU,SAAS,CACvB,GAAY,EACZ,gBAA8C;IAE9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;IACxE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,SAAS,YAAY,CACnB,KAA8B;QAE9B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC;QAClC,IACE,MAAM,KAAK,SAAS;YACpB,MAAM,KAAK,WAAW;YACtB,MAAM,KAAK,QAAQ;YACnB,MAAM,KAAK,SAAS;YACpB,MAAM,KAAK,WAAW;YACtB,MAAM,KAAK,WAAW;YAEtB,MAAM,IAAI,SAAS,CAAC,8BAA8B,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS;YAAE,gBAAgB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACvE,OAAO;YACL,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC;gBAClC,CAAC,CAAC;oBACE,QAAQ,EAAE,gBAAgB;wBACxB,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;wBAClC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;iBAC7B;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;SAC5E,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAY,EAAE;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACjC,IACE,CAAC,QAAQ,CAAC,KAAK,CAAC;YAChB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACf,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CACrB,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC5F;YACD,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC;YAE1F,MAAM,IAAI,SAAS,CACjB,2GAA2G,CAC5G,CAAC;QACJ,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM;gBACpD,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAC;YACtE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAY,EAAE;gBAC/C,IACE,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACf,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CACpB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC5E;oBACD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;oBAEpB,MAAM,IAAI,SAAS,CACjB,0EAA0E,CAC3E,CAAC;gBACJ,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,KAAK,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YACjF,CAAC,CAAC,CAAC;YACH,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACxB,IACE,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CACpB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CACnE;gBACD,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACnD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;gBACtB,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;oBAC1B,CAAC,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ;wBAChC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;wBACpC,IAAI,CAAC,QAAQ,GAAG,CAAC;wBACjB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;gBAEzB,MAAM,IAAI,SAAS,CACjB,2EAA2E,CAC5E,CAAC;YACJ,OAAO;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE;oBACJ,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,KAAK;oBAChD,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;oBAChE,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;iBACpE;aACF,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC/C,OAAO;YACL,GAAG,IAAI;YACP,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;gBACzB,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE;gBAC1E,CAAC,CAAC,KAAK,CAAC,KAAK;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;+FAC+F;AAC/F,MAAM,UAAU,WAAW,CAAC,GAAY,EAAE,OAAuB;IAC/D,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACnF,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW;QACvC,MAAM,IAAI,SAAS,CACjB,gFAAgF,CACjF,CAAC;IACJ,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1F,MAAM,IAAI,SAAS,CACjB,oHAAoH,CACrH,CAAC;IACJ,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC;QACxB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC;QAC5C,UAAU,EAAE;YACV,KAAK,EAAE,OAAO;YACd,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC;iBAC/B,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;iBACvC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;SAC5F;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Provider } from "@providerkit/core";
|
|
2
|
+
import { type ModelSettings } from "../index.ts";
|
|
3
|
+
import type { Scorer } from "./types.ts";
|
|
4
|
+
export interface JudgeOptions {
|
|
5
|
+
provider: Provider;
|
|
6
|
+
rubric: string;
|
|
7
|
+
/** Text/JSON only. Output is always sent. Input and operation data are opt-in; expected defaults on. */
|
|
8
|
+
include?: {
|
|
9
|
+
input?: boolean;
|
|
10
|
+
expected?: boolean;
|
|
11
|
+
trajectory?: boolean;
|
|
12
|
+
};
|
|
13
|
+
scale?: {
|
|
14
|
+
min: number;
|
|
15
|
+
max: number;
|
|
16
|
+
steps: readonly string[];
|
|
17
|
+
};
|
|
18
|
+
threshold?: number;
|
|
19
|
+
samples?: number;
|
|
20
|
+
modelSettings?: Omit<ModelSettings, "toolChoice" | "hostedTools" | "maxAttempts">;
|
|
21
|
+
name?: string;
|
|
22
|
+
gate?: boolean;
|
|
23
|
+
}
|
|
24
|
+
export declare function section(tag: string, value: unknown): string;
|
|
25
|
+
/** Costs money only when explicitly evaluated with allowJudge and the supplied provider.
|
|
26
|
+
* A judge is a fallible measurement, not proof that an output is safe or correct. */
|
|
27
|
+
export declare function llmJudge(options: JudgeOptions): Scorer;
|
|
28
|
+
/** A measured model opinion about recorded evidence, not proof of truth. Sends tool results
|
|
29
|
+
* to the explicit judge provider only with allowJudge. No usable results means no score. */
|
|
30
|
+
export declare function grounded(options: Omit<JudgeOptions, "rubric" | "include">): Scorer;
|
|
31
|
+
//# sourceMappingURL=judge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAiC,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AAGhF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,wGAAwG;IACxG,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACxE,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAC;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,YAAY,GAAG,aAAa,GAAG,aAAa,CAAC,CAAC;IAClF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAQD,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAG3D;AAED;qFACqF;AACrF,wBAAgB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAmItD;AAED;4FAC4F;AAC5F,wBAAgB,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,GAAG,SAAS,CAAC,GAAG,MAAM,CAgBlF"}
|