@actuarial-ts/agents 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/advisor.d.ts +2 -1
- package/dist/advisor.d.ts.map +1 -1
- package/dist/advisor.js +3 -1
- package/dist/advisor.js.map +1 -1
- package/dist/divergence.d.ts.map +1 -1
- package/dist/divergence.js +6 -1
- package/dist/divergence.js.map +1 -1
- package/dist/judgment.d.ts +26 -0
- package/dist/judgment.d.ts.map +1 -1
- package/dist/judgment.js +21 -0
- package/dist/judgment.js.map +1 -1
- package/dist/mcp.d.ts +44 -9
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +57 -40
- package/dist/mcp.js.map +1 -1
- package/dist/promotion.d.ts +26 -0
- package/dist/promotion.d.ts.map +1 -1
- package/dist/promotion.js +49 -1
- package/dist/promotion.js.map +1 -1
- package/dist/remote.d.ts.map +1 -1
- package/dist/remote.js +11 -1
- package/dist/remote.js.map +1 -1
- package/dist/tools.d.ts +87 -6
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +175 -15
- package/dist/tools.js.map +1 -1
- package/package.json +7 -5
- package/src/advisor.ts +141 -0
- package/src/divergence.ts +642 -0
- package/src/errors.ts +62 -0
- package/src/evals.ts +162 -0
- package/src/index.ts +9 -0
- package/src/judgment.ts +458 -0
- package/src/mcp.ts +274 -0
- package/src/promotion.ts +1240 -0
- package/src/remote.ts +371 -0
- package/src/tools.ts +561 -0
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Divergence explainer (interchange spec rev 2.1, Section 9 item 3): the
|
|
3
|
+
* agent invoked ONLY when the deterministic referee returns verdict
|
|
4
|
+
* "disagree". It never re-judges the comparison - the referee's verdict is
|
|
5
|
+
* final - it produces a structured HYPOTHESIS about why two engines diverged
|
|
6
|
+
* ("engine b requested sigma_interpolation=log-linear; the mack1993-vw
|
|
7
|
+
* profile requires mack; expected signature: SE-concentrated deviations -
|
|
8
|
+
* observed"), for the reviewing actuary to act on.
|
|
9
|
+
*
|
|
10
|
+
* Architecture decisions (each is load-bearing):
|
|
11
|
+
*
|
|
12
|
+
* - EVIDENCE ASSEMBLY IS PURE AND DETERMINISTIC. assembleDivergenceEvidence
|
|
13
|
+
* is a pure function of (CrosscheckReportDoc, MethodResultDoc a,
|
|
14
|
+
* MethodResultDoc b) plus the interchange profile registry - no clock, no
|
|
15
|
+
* randomness, no I/O. The convention map travels AS DATA: the alignment
|
|
16
|
+
* requirements are imported from @actuarial-ts/interchange's
|
|
17
|
+
* CONVENTION_PROFILES (the executable form of docs/interop/
|
|
18
|
+
* convention-map.md), never re-read from a file. Identical inputs yield
|
|
19
|
+
* identical evidence and an identical prompt, so the fixture tests can
|
|
20
|
+
* assert exactly what any model would be shown.
|
|
21
|
+
*
|
|
22
|
+
* - ONLY-ON-DISAGREE IS STRUCTURAL. A report whose verdict is not
|
|
23
|
+
* "disagree" throws AgentsError("VERDICT_NOT_DISAGREE") at assembly time;
|
|
24
|
+
* there is no flag to loosen. Agreement needs no explaining, and running
|
|
25
|
+
* an explainer on an agree/verified-by-value report would manufacture
|
|
26
|
+
* doubt about a comparison the referee already settled.
|
|
27
|
+
*
|
|
28
|
+
* - VIOLATIONS SURFACE FIRST. The requirement-by-requirement alignment
|
|
29
|
+
* check orders findings violated > unverifiable > satisfied, so the most
|
|
30
|
+
* probable root cause (a profile requirement the engine visibly did not
|
|
31
|
+
* run) is the first thing both the model and the human read.
|
|
32
|
+
*
|
|
33
|
+
* - READ-ONLY BY CONSTRUCTION. The explainer gets exactly one tool, and it
|
|
34
|
+
* only returns the already-assembled evidence (carried on the
|
|
35
|
+
* RequestContext under DIVERGENCE_EVIDENCE_CONTEXT_KEY); there is no
|
|
36
|
+
* action tool to reach for. The prompt also embeds the full evidence, so
|
|
37
|
+
* a single generate call suffices and the tool exists for follow-up
|
|
38
|
+
* turns, not as a required hop.
|
|
39
|
+
*
|
|
40
|
+
* HOUSE GOTCHA honored: no literal backtick characters in instruction
|
|
41
|
+
* content (a backtick inside a template literal once broke server boot).
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import { Agent } from "@mastra/core/agent";
|
|
45
|
+
import { RequestContext } from "@mastra/core/request-context";
|
|
46
|
+
import { ReservingError } from "@actuarial-ts/core";
|
|
47
|
+
import {
|
|
48
|
+
CONVENTION_PROFILES,
|
|
49
|
+
crosscheckReportDocSchema,
|
|
50
|
+
methodResultDocSchema,
|
|
51
|
+
type ConventionProfile,
|
|
52
|
+
type CrosscheckReportDoc,
|
|
53
|
+
type EngineAlignment,
|
|
54
|
+
type MethodResultDoc,
|
|
55
|
+
} from "@actuarial-ts/interchange";
|
|
56
|
+
import { z } from "zod";
|
|
57
|
+
import { AgentsError } from "./errors.js";
|
|
58
|
+
import { defineActuarialTool, toolRegistry } from "./tools.js";
|
|
59
|
+
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
// Evidence shapes
|
|
62
|
+
|
|
63
|
+
/** One profile requirement checked against what an engine actually ran. */
|
|
64
|
+
export interface AlignmentFinding {
|
|
65
|
+
/** Which side of the crosscheck this finding is about. */
|
|
66
|
+
engine: "a" | "b";
|
|
67
|
+
engineName: string;
|
|
68
|
+
parameter: string;
|
|
69
|
+
/** The value the convention profile requires for this parameter. */
|
|
70
|
+
required: unknown;
|
|
71
|
+
/** What the engine requested; undefined = the engine recorded no such parameter. */
|
|
72
|
+
requested: unknown;
|
|
73
|
+
/** What the engine says it actually ran when it deviated; undefined = as requested. */
|
|
74
|
+
effective: unknown;
|
|
75
|
+
/**
|
|
76
|
+
* violated = the engine's effective-else-requested value differs from the
|
|
77
|
+
* requirement; satisfied = it matches; unverifiable = the requirement is
|
|
78
|
+
* prose (not a literal pin) or the engine recorded no value to compare.
|
|
79
|
+
*/
|
|
80
|
+
status: "violated" | "satisfied" | "unverifiable";
|
|
81
|
+
detail: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** One engine's parameter story: what it claimed, requested, and ran. */
|
|
85
|
+
export interface EngineParameterEvidence {
|
|
86
|
+
name: string;
|
|
87
|
+
version: string;
|
|
88
|
+
conventionProfile: string | null;
|
|
89
|
+
method: string;
|
|
90
|
+
requested: Record<string, unknown>;
|
|
91
|
+
/** null = the engine deviated nowhere (no effectiveParameters recorded). */
|
|
92
|
+
effective: Record<string, unknown> | null;
|
|
93
|
+
/** The profile's alignment requirements for this engine (entry point,
|
|
94
|
+
* pinned parameters, trap notes), or null when the profile does not know
|
|
95
|
+
* the engine. This IS the convention map, as data. */
|
|
96
|
+
profileAlignment: EngineAlignment | null;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Where the disagreement concentrates, measured against the applied tolerance. */
|
|
100
|
+
export interface DeviationSignature {
|
|
101
|
+
tolerance: { central: number; standardError: number | null };
|
|
102
|
+
/** Max relative deviation across per-origin and total ultimates/unpaid. */
|
|
103
|
+
maxCentral: number;
|
|
104
|
+
/** Max relative SE deviation; null when no SE cell was compared. */
|
|
105
|
+
maxStandardError: number | null;
|
|
106
|
+
centralExceedsTolerance: boolean;
|
|
107
|
+
standardErrorExceedsTolerance: boolean;
|
|
108
|
+
/** central | standard-error | mixed: which metric family breaches tolerance. */
|
|
109
|
+
concentration: "central" | "standard-error" | "mixed" | "none";
|
|
110
|
+
totals: { ultimate: number | null; unpaid: number | null; standardError: number | null };
|
|
111
|
+
/** The largest per-origin deviations, ranked descending (top 5). */
|
|
112
|
+
worstOrigins: {
|
|
113
|
+
origin: string;
|
|
114
|
+
metric: "ultimate" | "unpaid" | "standardError";
|
|
115
|
+
deviation: number;
|
|
116
|
+
}[];
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Everything the explainer (model or human) needs to hypothesize a cause. */
|
|
120
|
+
export interface DivergenceEvidence {
|
|
121
|
+
verdict: "disagree";
|
|
122
|
+
profile: {
|
|
123
|
+
/** The convention profile both results claim, or null when none stated. */
|
|
124
|
+
id: string | null;
|
|
125
|
+
/** false = the claimed profile is not in this package's registry. */
|
|
126
|
+
known: boolean;
|
|
127
|
+
description: string | null;
|
|
128
|
+
tolerance: { central: number; standardError: number | null } | null;
|
|
129
|
+
};
|
|
130
|
+
/** Requirement-by-requirement check, VIOLATIONS FIRST (then unverifiable,
|
|
131
|
+
* then satisfied). */
|
|
132
|
+
alignmentFindings: AlignmentFinding[];
|
|
133
|
+
engines: { a: EngineParameterEvidence; b: EngineParameterEvidence };
|
|
134
|
+
deviationSignature: DeviationSignature;
|
|
135
|
+
warnings: { report: string[]; engineA: string[]; engineB: string[] };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export interface AssembleDivergenceEvidenceOptions {
|
|
139
|
+
report: CrosscheckReportDoc;
|
|
140
|
+
a: MethodResultDoc;
|
|
141
|
+
b: MethodResultDoc;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// ---------------------------------------------------------------------------
|
|
145
|
+
// Evidence assembly (pure)
|
|
146
|
+
|
|
147
|
+
function stableJson(value: unknown): string {
|
|
148
|
+
return value === undefined ? "(absent)" : JSON.stringify(value);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function sameValue(x: unknown, y: unknown): boolean {
|
|
152
|
+
return x === y || JSON.stringify(x) === JSON.stringify(y);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function validateDoc<T>(
|
|
156
|
+
label: string,
|
|
157
|
+
doc: unknown,
|
|
158
|
+
schema: { safeParse(input: unknown): { success: boolean; data?: T; error?: z.ZodError } },
|
|
159
|
+
): T {
|
|
160
|
+
const parsed = schema.safeParse(doc);
|
|
161
|
+
if (!parsed.success) {
|
|
162
|
+
throw new ReservingError(
|
|
163
|
+
"BAD_INTERCHANGE",
|
|
164
|
+
`assembleDivergenceEvidence input "${label}" is malformed: ${parsed
|
|
165
|
+
.error!.issues.map((i) => `${i.path.join(".") || "$"}: ${i.message}`)
|
|
166
|
+
.join("; ")}`,
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
return parsed.data as T;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function engineMatches(
|
|
173
|
+
doc: MethodResultDoc,
|
|
174
|
+
stamp: { name: string; version: string },
|
|
175
|
+
): boolean {
|
|
176
|
+
return doc.result.engine.name === stamp.name && doc.result.engine.version === stamp.version;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const STATUS_RANK: Record<AlignmentFinding["status"], number> = {
|
|
180
|
+
violated: 0,
|
|
181
|
+
unverifiable: 1,
|
|
182
|
+
satisfied: 2,
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
function alignmentFindingsFor(
|
|
186
|
+
side: "a" | "b",
|
|
187
|
+
doc: MethodResultDoc,
|
|
188
|
+
profile: ConventionProfile | null,
|
|
189
|
+
claimedProfileId: string | null,
|
|
190
|
+
): AlignmentFinding[] {
|
|
191
|
+
const engineName = doc.result.engine.name;
|
|
192
|
+
if (profile === null) {
|
|
193
|
+
return [
|
|
194
|
+
{
|
|
195
|
+
engine: side,
|
|
196
|
+
engineName,
|
|
197
|
+
parameter: "*",
|
|
198
|
+
required: undefined,
|
|
199
|
+
requested: undefined,
|
|
200
|
+
effective: undefined,
|
|
201
|
+
status: "unverifiable",
|
|
202
|
+
detail:
|
|
203
|
+
claimedProfileId === null
|
|
204
|
+
? `engine ${side} (${engineName}): no convention profile is claimed on either result, so ` +
|
|
205
|
+
"no alignment requirements exist to check against"
|
|
206
|
+
: `engine ${side} (${engineName}): claimed profile "${claimedProfileId}" is not in the ` +
|
|
207
|
+
"interchange profile registry; its requirements cannot be checked",
|
|
208
|
+
},
|
|
209
|
+
];
|
|
210
|
+
}
|
|
211
|
+
const alignment =
|
|
212
|
+
(profile.alignment as Record<string, EngineAlignment | undefined>)[engineName] ?? null;
|
|
213
|
+
if (alignment === null) {
|
|
214
|
+
return [
|
|
215
|
+
{
|
|
216
|
+
engine: side,
|
|
217
|
+
engineName,
|
|
218
|
+
parameter: "*",
|
|
219
|
+
required: undefined,
|
|
220
|
+
requested: undefined,
|
|
221
|
+
effective: undefined,
|
|
222
|
+
status: "unverifiable",
|
|
223
|
+
detail:
|
|
224
|
+
`engine ${side} (${engineName}): profile "${profile.id}" has no alignment entry for ` +
|
|
225
|
+
"this engine name; its requirements cannot be checked",
|
|
226
|
+
},
|
|
227
|
+
];
|
|
228
|
+
}
|
|
229
|
+
const requestedParams = doc.result.parameters;
|
|
230
|
+
const effectiveParams = doc.result.effectiveParameters;
|
|
231
|
+
const findings: AlignmentFinding[] = [];
|
|
232
|
+
for (const [parameter, required] of Object.entries(alignment.parameters)) {
|
|
233
|
+
const requested = requestedParams[parameter];
|
|
234
|
+
const effective = effectiveParams?.[parameter];
|
|
235
|
+
const ran = effective !== undefined ? effective : requested;
|
|
236
|
+
const base: Omit<AlignmentFinding, "status" | "detail"> = {
|
|
237
|
+
engine: side,
|
|
238
|
+
engineName,
|
|
239
|
+
parameter,
|
|
240
|
+
required,
|
|
241
|
+
requested,
|
|
242
|
+
effective,
|
|
243
|
+
};
|
|
244
|
+
if (ran === undefined) {
|
|
245
|
+
findings.push({
|
|
246
|
+
...base,
|
|
247
|
+
status: "unverifiable",
|
|
248
|
+
detail:
|
|
249
|
+
`engine ${side} (${engineName}) parameter "${parameter}": profile "${profile.id}" requires ` +
|
|
250
|
+
`${stableJson(required)}, but the result records no such parameter (a prose requirement or ` +
|
|
251
|
+
"an unrecorded setting); not mechanically checkable",
|
|
252
|
+
});
|
|
253
|
+
} else if (sameValue(required, ran)) {
|
|
254
|
+
findings.push({
|
|
255
|
+
...base,
|
|
256
|
+
status: "satisfied",
|
|
257
|
+
detail:
|
|
258
|
+
`engine ${side} (${engineName}) parameter "${parameter}": profile "${profile.id}" requires ` +
|
|
259
|
+
`${stableJson(required)}; the engine ran ${stableJson(ran)}`,
|
|
260
|
+
});
|
|
261
|
+
} else {
|
|
262
|
+
const deviationNote =
|
|
263
|
+
effective !== undefined && !sameValue(effective, requested)
|
|
264
|
+
? ` (requested ${stableJson(requested)}, effective ${stableJson(effective)})`
|
|
265
|
+
: "";
|
|
266
|
+
findings.push({
|
|
267
|
+
...base,
|
|
268
|
+
status: "violated",
|
|
269
|
+
detail:
|
|
270
|
+
`VIOLATED: profile "${profile.id}" requires ${parameter}=${stableJson(required)} for ` +
|
|
271
|
+
`${engineName}; engine ${side} ran ${parameter}=${stableJson(ran)}${deviationNote}`,
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return findings;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function deviationSignatureOf(report: CrosscheckReportDoc): DeviationSignature {
|
|
279
|
+
const body = report.report;
|
|
280
|
+
const tolerance = {
|
|
281
|
+
central: body.tolerance.central,
|
|
282
|
+
standardError: body.tolerance.standardError,
|
|
283
|
+
};
|
|
284
|
+
let maxCentral = 0;
|
|
285
|
+
let maxSe: number | null = null;
|
|
286
|
+
const ranked: DeviationSignature["worstOrigins"] = [];
|
|
287
|
+
for (const row of body.deviations.perOrigin) {
|
|
288
|
+
for (const metric of ["ultimate", "unpaid"] as const) {
|
|
289
|
+
const deviation = row[metric];
|
|
290
|
+
if (deviation === null) continue;
|
|
291
|
+
maxCentral = Math.max(maxCentral, deviation);
|
|
292
|
+
ranked.push({ origin: row.origin, metric, deviation });
|
|
293
|
+
}
|
|
294
|
+
if (row.standardError !== null) {
|
|
295
|
+
maxSe = Math.max(maxSe ?? 0, row.standardError);
|
|
296
|
+
ranked.push({ origin: row.origin, metric: "standardError", deviation: row.standardError });
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const totals = body.deviations.totals;
|
|
300
|
+
for (const metric of ["ultimate", "unpaid"] as const) {
|
|
301
|
+
const deviation = totals[metric];
|
|
302
|
+
if (deviation !== null) maxCentral = Math.max(maxCentral, deviation);
|
|
303
|
+
}
|
|
304
|
+
if (totals.standardError !== null) maxSe = Math.max(maxSe ?? 0, totals.standardError);
|
|
305
|
+
|
|
306
|
+
const centralExceeds = maxCentral > tolerance.central;
|
|
307
|
+
const seExceeds =
|
|
308
|
+
tolerance.standardError !== null && maxSe !== null && maxSe > tolerance.standardError;
|
|
309
|
+
const concentration: DeviationSignature["concentration"] =
|
|
310
|
+
centralExceeds && seExceeds
|
|
311
|
+
? "mixed"
|
|
312
|
+
: centralExceeds
|
|
313
|
+
? "central"
|
|
314
|
+
: seExceeds
|
|
315
|
+
? "standard-error"
|
|
316
|
+
: "none";
|
|
317
|
+
ranked.sort((x, y) => y.deviation - x.deviation);
|
|
318
|
+
return {
|
|
319
|
+
tolerance,
|
|
320
|
+
maxCentral,
|
|
321
|
+
maxStandardError: maxSe,
|
|
322
|
+
centralExceedsTolerance: centralExceeds,
|
|
323
|
+
standardErrorExceedsTolerance: seExceeds,
|
|
324
|
+
concentration,
|
|
325
|
+
totals: {
|
|
326
|
+
ultimate: totals.ultimate,
|
|
327
|
+
unpaid: totals.unpaid,
|
|
328
|
+
standardError: totals.standardError,
|
|
329
|
+
},
|
|
330
|
+
worstOrigins: ranked.slice(0, 5),
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* Assembles the divergence evidence for a disagree crosscheck: pure and
|
|
336
|
+
* deterministic (see the module doc). Throws:
|
|
337
|
+
* - AgentsError("VERDICT_NOT_DISAGREE") when the report's verdict is not
|
|
338
|
+
* "disagree" - the only-on-disagree rule is structural, not advisory;
|
|
339
|
+
* - AgentsError("DIVERGENCE_INPUT_MISMATCH") when the supplied result docs
|
|
340
|
+
* do not match the report's engine stamps (wrong docs, or a/b swapped);
|
|
341
|
+
* - ReservingError("BAD_INTERCHANGE") when any input document is malformed.
|
|
342
|
+
*/
|
|
343
|
+
export function assembleDivergenceEvidence(
|
|
344
|
+
options: AssembleDivergenceEvidenceOptions,
|
|
345
|
+
): DivergenceEvidence {
|
|
346
|
+
const report = validateDoc<CrosscheckReportDoc>(
|
|
347
|
+
"report",
|
|
348
|
+
options.report,
|
|
349
|
+
crosscheckReportDocSchema,
|
|
350
|
+
);
|
|
351
|
+
const a = validateDoc<MethodResultDoc>("a", options.a, methodResultDocSchema);
|
|
352
|
+
const b = validateDoc<MethodResultDoc>("b", options.b, methodResultDocSchema);
|
|
353
|
+
const body = report.report;
|
|
354
|
+
|
|
355
|
+
if (body.verdict !== "disagree") {
|
|
356
|
+
throw new AgentsError(
|
|
357
|
+
"VERDICT_NOT_DISAGREE",
|
|
358
|
+
`The divergence explainer is invoked ONLY on a "disagree" crosscheck verdict (spec 9 item 3); ` +
|
|
359
|
+
`this report's verdict is "${body.verdict}" - there is no divergence to explain`,
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
if (!engineMatches(a, body.engines.a) || !engineMatches(b, body.engines.b)) {
|
|
363
|
+
const swapped = engineMatches(a, body.engines.b) && engineMatches(b, body.engines.a);
|
|
364
|
+
throw new AgentsError(
|
|
365
|
+
"DIVERGENCE_INPUT_MISMATCH",
|
|
366
|
+
swapped
|
|
367
|
+
? "The supplied result docs are SWAPPED relative to the report: doc a matches the report's " +
|
|
368
|
+
"engine b and vice versa; pass them in the report's a/b order"
|
|
369
|
+
: `The supplied result docs do not match the report's engine stamps: report compared ` +
|
|
370
|
+
`a=${body.engines.a.name}@${body.engines.a.version} vs ` +
|
|
371
|
+
`b=${body.engines.b.name}@${body.engines.b.version}, got ` +
|
|
372
|
+
`a=${a.result.engine.name}@${a.result.engine.version} and ` +
|
|
373
|
+
`b=${b.result.engine.name}@${b.result.engine.version}`,
|
|
374
|
+
);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const claimedProfileId =
|
|
378
|
+
a.result.engine.conventionProfile ?? b.result.engine.conventionProfile ?? null;
|
|
379
|
+
const profile = claimedProfileId !== null ? (CONVENTION_PROFILES[claimedProfileId] ?? null) : null;
|
|
380
|
+
|
|
381
|
+
const engineEvidenceOf = (doc: MethodResultDoc): EngineParameterEvidence => ({
|
|
382
|
+
name: doc.result.engine.name,
|
|
383
|
+
version: doc.result.engine.version,
|
|
384
|
+
conventionProfile: doc.result.engine.conventionProfile ?? null,
|
|
385
|
+
method: doc.result.method,
|
|
386
|
+
requested: doc.result.parameters,
|
|
387
|
+
effective: doc.result.effectiveParameters ?? null,
|
|
388
|
+
profileAlignment:
|
|
389
|
+
profile === null
|
|
390
|
+
? null
|
|
391
|
+
: ((profile.alignment as Record<string, EngineAlignment | undefined>)[
|
|
392
|
+
doc.result.engine.name
|
|
393
|
+
] ?? null),
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
const findings = [
|
|
397
|
+
...alignmentFindingsFor("a", a, profile, claimedProfileId),
|
|
398
|
+
...alignmentFindingsFor("b", b, profile, claimedProfileId),
|
|
399
|
+
].sort((x, y) => STATUS_RANK[x.status] - STATUS_RANK[y.status]);
|
|
400
|
+
|
|
401
|
+
return {
|
|
402
|
+
verdict: "disagree",
|
|
403
|
+
profile: {
|
|
404
|
+
id: claimedProfileId,
|
|
405
|
+
known: profile !== null,
|
|
406
|
+
description: profile?.description ?? null,
|
|
407
|
+
tolerance: profile !== null ? { ...profile.tolerance } : null,
|
|
408
|
+
},
|
|
409
|
+
alignmentFindings: findings,
|
|
410
|
+
engines: { a: engineEvidenceOf(a), b: engineEvidenceOf(b) },
|
|
411
|
+
deviationSignature: deviationSignatureOf(report),
|
|
412
|
+
warnings: {
|
|
413
|
+
report: [...body.warnings],
|
|
414
|
+
engineA: [...(a.result.warnings ?? [])],
|
|
415
|
+
engineB: [...(b.result.warnings ?? [])],
|
|
416
|
+
},
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// ---------------------------------------------------------------------------
|
|
421
|
+
// Structured hypothesis
|
|
422
|
+
|
|
423
|
+
/** The explainer's structured output (spec 9 item 3). */
|
|
424
|
+
export const divergenceHypothesisSchema = z.object({
|
|
425
|
+
/** The single most probable root cause, stated as a testable claim. */
|
|
426
|
+
suspectedCause: z.string().min(1),
|
|
427
|
+
/** The exact misaligned parameter/flag name (e.g. "sigma_interpolation"),
|
|
428
|
+
* or null when no specific flag is implicated. */
|
|
429
|
+
misalignedFlag: z.string().min(1).nullable(),
|
|
430
|
+
/** The deviation signature the suspected cause WOULD produce. */
|
|
431
|
+
expectedSignature: z.string().min(1),
|
|
432
|
+
/** The deviation signature the evidence actually shows. */
|
|
433
|
+
observedSignature: z.string().min(1),
|
|
434
|
+
/** What the operator should do next to confirm or fix. */
|
|
435
|
+
recommendation: z.string().min(1),
|
|
436
|
+
});
|
|
437
|
+
|
|
438
|
+
export type DivergenceHypothesis = z.infer<typeof divergenceHypothesisSchema>;
|
|
439
|
+
|
|
440
|
+
// ---------------------------------------------------------------------------
|
|
441
|
+
// Instructions + prompt assembly (pure, deterministic)
|
|
442
|
+
|
|
443
|
+
/** RequestContext key the evidence tool reads (set by explainDivergence). */
|
|
444
|
+
export const DIVERGENCE_EVIDENCE_CONTEXT_KEY = "divergenceEvidence";
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* The explainer's instruction template. A constant, not a builder: the
|
|
448
|
+
* explainer has exactly one job and no host-specific domain sections. No
|
|
449
|
+
* backtick characters (house gotcha).
|
|
450
|
+
*/
|
|
451
|
+
export const DIVERGENCE_EXPLAINER_INSTRUCTIONS = [
|
|
452
|
+
"You are a cross-engine divergence diagnostician inside an actuarial reserving toolchain. A deterministic referee compared the same computation run by two independent engines and returned the verdict \"disagree\". Your job is to produce a structured HYPOTHESIS about the root cause - you never re-litigate the verdict, and you never change any state.",
|
|
453
|
+
"## Working rules",
|
|
454
|
+
[
|
|
455
|
+
"1. Every claim you make must come from the supplied divergence evidence (in the user message, and available again via the get_divergence_evidence tool). Never invent parameters, deviations, or profile requirements.",
|
|
456
|
+
"2. The alignment findings are ordered violations first. A VIOLATED finding - a convention-profile requirement the engine visibly did not run - is the strongest root-cause candidate; check whether its expected deviation signature matches the observed one before settling on it.",
|
|
457
|
+
"3. The deviation signature tells you WHERE the disagreement lives: central estimates (ultimates/unpaid), standard errors, or both. A sigma/variance-convention misalignment concentrates in standard errors while central estimates agree; a factor/selection misalignment moves central estimates.",
|
|
458
|
+
"4. In misalignedFlag, name the exact parameter as it appears in the evidence (for example sigma_interpolation), or use null when no specific flag is implicated.",
|
|
459
|
+
"5. Be direct and technical; the reader is a credentialed actuary. State the hypothesis, the expected vs observed signature, and one concrete next step.",
|
|
460
|
+
].join("\n"),
|
|
461
|
+
].join("\n\n");
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Deterministic user-prompt assembly: a readable summary (violations first,
|
|
465
|
+
* then the deviation signature) followed by the full evidence as JSON.
|
|
466
|
+
* Identical evidence yields a byte-identical prompt, so tests can pin
|
|
467
|
+
* exactly what any model is shown.
|
|
468
|
+
*/
|
|
469
|
+
export function assembleDivergencePrompt(evidence: DivergenceEvidence): string {
|
|
470
|
+
const sig = evidence.deviationSignature;
|
|
471
|
+
const profileLine =
|
|
472
|
+
evidence.profile.id === null
|
|
473
|
+
? "Convention profile: none claimed."
|
|
474
|
+
: `Convention profile: "${evidence.profile.id}"${evidence.profile.known ? "" : " (NOT in the registry)"} - ${
|
|
475
|
+
evidence.profile.description ?? "no description"
|
|
476
|
+
}`;
|
|
477
|
+
const findingLines = evidence.alignmentFindings.map(
|
|
478
|
+
(f) => `- [${f.status.toUpperCase()}] ${f.detail}`,
|
|
479
|
+
);
|
|
480
|
+
const seLine =
|
|
481
|
+
sig.maxStandardError === null
|
|
482
|
+
? "standard errors: not compared"
|
|
483
|
+
: `max standard-error deviation ${sig.maxStandardError} (tolerance ${
|
|
484
|
+
sig.tolerance.standardError ?? "out of scope"
|
|
485
|
+
}; ${sig.standardErrorExceedsTolerance ? "EXCEEDED" : "within"})`;
|
|
486
|
+
const sections = [
|
|
487
|
+
"A crosscheck referee compared two engines running the same computation and returned verdict DISAGREE.",
|
|
488
|
+
`Engine a: ${evidence.engines.a.name}@${evidence.engines.a.version} (method ${evidence.engines.a.method}). ` +
|
|
489
|
+
`Engine b: ${evidence.engines.b.name}@${evidence.engines.b.version} (method ${evidence.engines.b.method}).`,
|
|
490
|
+
profileLine,
|
|
491
|
+
"Alignment findings (violations first):\n" + findingLines.join("\n"),
|
|
492
|
+
`Deviation signature: concentration ${sig.concentration}; max central deviation ${sig.maxCentral} ` +
|
|
493
|
+
`(tolerance ${sig.tolerance.central}; ${sig.centralExceedsTolerance ? "EXCEEDED" : "within"}); ${seLine}.`,
|
|
494
|
+
"Full assembled evidence (JSON):\n" + JSON.stringify(evidence, null, 2),
|
|
495
|
+
"Produce the structured hypothesis: suspectedCause, misalignedFlag (the exact parameter name, or null), expectedSignature, observedSignature, recommendation.",
|
|
496
|
+
];
|
|
497
|
+
return sections.join("\n\n");
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// ---------------------------------------------------------------------------
|
|
501
|
+
// The read-only evidence tool
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* The explainer's single tool: returns the already-assembled evidence from
|
|
505
|
+
* the request context. Read-only by construction - it can only restate what
|
|
506
|
+
* explainDivergence assembled; there is nothing to mutate.
|
|
507
|
+
*/
|
|
508
|
+
export function createDivergenceEvidenceTool() {
|
|
509
|
+
return defineActuarialTool({
|
|
510
|
+
id: "get_divergence_evidence",
|
|
511
|
+
description:
|
|
512
|
+
"Returns the assembled divergence evidence for the disagree crosscheck under review: " +
|
|
513
|
+
"convention-profile alignment requirements, each engine's requested vs effective parameters, " +
|
|
514
|
+
"requirement-by-requirement findings (violations first), the deviation signature, and all " +
|
|
515
|
+
"warnings. Read-only.",
|
|
516
|
+
kind: "read",
|
|
517
|
+
inputSchema: z.object({}),
|
|
518
|
+
// Tenant-free BY DESIGN: this tool restates divergence evidence the host
|
|
519
|
+
// already assembled and injected into the request context via
|
|
520
|
+
// explainDivergence. There is no per-tenant data path to protect — the
|
|
521
|
+
// evidence is scoped by the host before this tool can see it.
|
|
522
|
+
tenant: "none",
|
|
523
|
+
execute: async (_input, _tenant, context) => {
|
|
524
|
+
const evidence = context.requestContext?.get(DIVERGENCE_EVIDENCE_CONTEXT_KEY);
|
|
525
|
+
if (evidence === undefined) {
|
|
526
|
+
throw new AgentsError(
|
|
527
|
+
"NO_DIVERGENCE_EVIDENCE",
|
|
528
|
+
`No assembled divergence evidence in the request context (key "${DIVERGENCE_EVIDENCE_CONTEXT_KEY}"); ` +
|
|
529
|
+
"drive this agent through explainDivergence, which assembles and injects it",
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
return { success: true as const, evidence: evidence as DivergenceEvidence };
|
|
533
|
+
},
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// ---------------------------------------------------------------------------
|
|
538
|
+
// Agent factory + driver
|
|
539
|
+
|
|
540
|
+
/** Config slices lifted from the installed Agent constructor so the factory tracks the host's Mastra version. */
|
|
541
|
+
type AgentCtorConfig = ConstructorParameters<typeof Agent>[0];
|
|
542
|
+
|
|
543
|
+
export interface CreateDivergenceExplainerOptions {
|
|
544
|
+
/** Defaults to "divergence-explainer". */
|
|
545
|
+
id?: string;
|
|
546
|
+
/** Defaults to "Divergence Explainer". */
|
|
547
|
+
name?: string;
|
|
548
|
+
description?: string;
|
|
549
|
+
/** The language model (same type the host's Agent constructor takes). */
|
|
550
|
+
model: AgentCtorConfig["model"];
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* Builds the divergence-explainer Agent: the constant instruction template
|
|
555
|
+
* plus the single read-only evidence tool. Drive it with explainDivergence,
|
|
556
|
+
* which assembles the evidence, injects it into the request context, and
|
|
557
|
+
* runs the one structured-output generate call.
|
|
558
|
+
*/
|
|
559
|
+
export function createDivergenceExplainer(options: CreateDivergenceExplainerOptions): Agent {
|
|
560
|
+
return new Agent({
|
|
561
|
+
id: options.id ?? "divergence-explainer",
|
|
562
|
+
name: options.name ?? "Divergence Explainer",
|
|
563
|
+
description:
|
|
564
|
+
options.description ??
|
|
565
|
+
"Cross-engine divergence diagnostician: on a disagree crosscheck verdict, reads the assembled " +
|
|
566
|
+
"evidence (profile requirements, requested vs effective parameters, deviation signature) and " +
|
|
567
|
+
"produces a structured root-cause hypothesis. Read-only; never re-judges the referee.",
|
|
568
|
+
instructions: DIVERGENCE_EXPLAINER_INSTRUCTIONS,
|
|
569
|
+
model: options.model,
|
|
570
|
+
tools: toolRegistry([createDivergenceEvidenceTool()]).tools,
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* The structural slice of an agent explainDivergence needs: generate with a
|
|
576
|
+
* structured-output option resolving to { object }. Structural on purpose
|
|
577
|
+
* (mirrors evals.ts' ToolStreamingAgent): package tests substitute a canned
|
|
578
|
+
* stub with no LLM, and hosts pass any compatible Mastra Agent.
|
|
579
|
+
*/
|
|
580
|
+
export interface StructuredGeneratingAgent {
|
|
581
|
+
generate(
|
|
582
|
+
messages: Array<{ role: "user"; content: string }>,
|
|
583
|
+
options?: Record<string, unknown>,
|
|
584
|
+
): Promise<{ object?: unknown }>;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
/** The request-context slice explainDivergence needs (Mastra's RequestContext satisfies it). */
|
|
588
|
+
export interface DivergenceRequestContext {
|
|
589
|
+
get(key: string): unknown;
|
|
590
|
+
set(key: string, value: unknown): void;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
export interface ExplainDivergenceOptions {
|
|
594
|
+
explainer: StructuredGeneratingAgent;
|
|
595
|
+
report: CrosscheckReportDoc;
|
|
596
|
+
a: MethodResultDoc;
|
|
597
|
+
b: MethodResultDoc;
|
|
598
|
+
/**
|
|
599
|
+
* Host request context (the evidence is set on it under
|
|
600
|
+
* DIVERGENCE_EVIDENCE_CONTEXT_KEY); a fresh RequestContext is created when
|
|
601
|
+
* omitted. The tenant seam is untouched either way - this key carries
|
|
602
|
+
* evidence, never identity.
|
|
603
|
+
*/
|
|
604
|
+
requestContext?: DivergenceRequestContext;
|
|
605
|
+
/** Max agent steps for the single generate call. Default 4 (evidence is
|
|
606
|
+
* in the prompt; the tool is an optional re-read, not a required hop). */
|
|
607
|
+
maxSteps?: number;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
export interface DivergenceExplanation {
|
|
611
|
+
hypothesis: DivergenceHypothesis;
|
|
612
|
+
evidence: DivergenceEvidence;
|
|
613
|
+
/** The exact prompt the model was shown (byte-deterministic per evidence). */
|
|
614
|
+
prompt: string;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Drives one structured-output generate call: assembles the evidence
|
|
619
|
+
* (throwing on a non-disagree verdict), injects it into the request context
|
|
620
|
+
* for the evidence tool, prompts the explainer, and zod-validates the
|
|
621
|
+
* model's hypothesis before returning it - the boundary is validated in
|
|
622
|
+
* both directions.
|
|
623
|
+
*/
|
|
624
|
+
export async function explainDivergence(
|
|
625
|
+
options: ExplainDivergenceOptions,
|
|
626
|
+
): Promise<DivergenceExplanation> {
|
|
627
|
+
const evidence = assembleDivergenceEvidence({
|
|
628
|
+
report: options.report,
|
|
629
|
+
a: options.a,
|
|
630
|
+
b: options.b,
|
|
631
|
+
});
|
|
632
|
+
const requestContext = options.requestContext ?? new RequestContext();
|
|
633
|
+
requestContext.set(DIVERGENCE_EVIDENCE_CONTEXT_KEY, evidence);
|
|
634
|
+
const prompt = assembleDivergencePrompt(evidence);
|
|
635
|
+
const result = await options.explainer.generate([{ role: "user", content: prompt }], {
|
|
636
|
+
structuredOutput: { schema: divergenceHypothesisSchema },
|
|
637
|
+
requestContext,
|
|
638
|
+
maxSteps: options.maxSteps ?? 4,
|
|
639
|
+
});
|
|
640
|
+
const hypothesis = divergenceHypothesisSchema.parse(result.object);
|
|
641
|
+
return { hypothesis, evidence, prompt };
|
|
642
|
+
}
|