@adonis-agora/agent 0.30.2 → 0.33.0

package/dist/src/evals/scorers/answer-relevancy.scorer.d.ts

@@ -0,0 +1,28 @@
+import type { ModelProvider } from '../../spi/model-provider.js';
+import type { ScorableRun, ScoreResult, Scorer, ScorerKind } from '../types.js';
+/** {@link AnswerRelevancyScorer}'s stable name. */
+export declare const ANSWER_RELEVANCY_SCORER = "answer-relevancy";
+export interface AnswerRelevancyOptions {
+    /** The judge. Deliberately a separate {@link ModelProvider} — nothing forces it to be the model under test. */
+    model: ModelProvider;
+    /** Override the grading instructions (e.g. to add domain rules the judge must apply). */
+    systemPrompt?: string;
+}
+/**
+ * LLM-as-judge: does the answer address the question that was asked?
+ *
+ * The expensive family, and the reason this module is offline-first — grading a turn costs a whole
+ * model call, so it happens over stored runs on your schedule, not inside the turn where it would
+ * double the latency and the bill of every message a user sends.
+ *
+ * `null` when there is no question or no answer to compare: a judge asked to grade an empty string
+ * will invent a number, and `run-completion` has already scored that run 0.
+ */
+export declare class AnswerRelevancyScorer implements Scorer {
+    private readonly options;
+    readonly name = "answer-relevancy";
+    readonly kind: ScorerKind;
+    constructor(options: AnswerRelevancyOptions);
+    score(run: ScorableRun): Promise<ScoreResult | null>;
+}
+//# sourceMappingURL=answer-relevancy.scorer.d.ts.map

package/dist/src/evals/scorers/answer-relevancy.scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer-relevancy.scorer.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/answer-relevancy.scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEjE,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEhF,mDAAmD;AACnD,eAAO,MAAM,uBAAuB,qBAAqB,CAAC;AAW1D,MAAM,WAAW,sBAAsB;IACrC,+GAA+G;IAC/G,KAAK,EAAE,aAAa,CAAC;IACrB,yFAAyF;IACzF,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;GASG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAItC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAHpC,QAAQ,CAAC,IAAI,sBAA2B;IACxC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAW;IAEpC,YAA6B,OAAO,EAAE,sBAAsB,EAAI;IAE1D,KAAK,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAWzD;CACF"}

package/dist/src/evals/scorers/answer-relevancy.scorer.js

@@ -0,0 +1,42 @@
+import { discardingSink, MAX_JUDGE_SCORE, parseJudgeVerdict } from '../judge.js';
+/** {@link AnswerRelevancyScorer}'s stable name. */
+export const ANSWER_RELEVANCY_SCORER = 'answer-relevancy';
+const DEFAULT_JUDGE_PROMPT = [
+    'You grade an AI assistant answer for RELEVANCY to the question it was given.',
+    'Judge only whether the answer addresses what was asked — not style, length, or whether you',
+    'personally agree with it. An answer that is fluent but answers a different question scores low.',
+    '',
+    `Reply in exactly this shape, with no other text: "SCORE: <0-${MAX_JUDGE_SCORE}>" on the first`,
+    'line and "REASON: <one sentence>" on the second.',
+].join('\n');
+/**
+ * LLM-as-judge: does the answer address the question that was asked?
+ *
+ * The expensive family, and the reason this module is offline-first — grading a turn costs a whole
+ * model call, so it happens over stored runs on your schedule, not inside the turn where it would
+ * double the latency and the bill of every message a user sends.
+ *
+ * `null` when there is no question or no answer to compare: a judge asked to grade an empty string
+ * will invent a number, and `run-completion` has already scored that run 0.
+ */
+export class AnswerRelevancyScorer {
+    options;
+    name = ANSWER_RELEVANCY_SCORER;
+    kind = 'model';
+    constructor(options) {
+        this.options = options;
+    }
+    async score(run) {
+        if (run.input.trim() === '' || run.output.trim() === '') {
+            return null;
+        }
+        const result = await this.options.model.runTurn({
+            system: this.options.systemPrompt ?? DEFAULT_JUDGE_PROMPT,
+            messages: [{ role: 'user', content: `QUESTION:\n${run.input}\n\nANSWER:\n${run.output}` }],
+            tools: [],
+            sink: discardingSink(),
+        });
+        return parseJudgeVerdict(result.text);
+    }
+}
+//# sourceMappingURL=answer-relevancy.scorer.js.map

package/dist/src/evals/scorers/answer-relevancy.scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer-relevancy.scorer.js","sourceRoot":"","sources":["../../../../src/evals/scorers/answer-relevancy.scorer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGjF,mDAAmD;AACnD,MAAM,CAAC,MAAM,uBAAuB,GAAG,kBAAkB,CAAC;AAE1D,MAAM,oBAAoB,GAAG;IAC3B,8EAA8E;IAC9E,4FAA4F;IAC5F,iGAAiG;IACjG,EAAE;IACF,+DAA+D,eAAe,iBAAiB;IAC/F,kDAAkD;CACnD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AASb;;;;;;;;;GASG;AACH,MAAM,OAAO,qBAAqB;IAIH,OAAO;IAH3B,IAAI,GAAG,uBAAuB,CAAC;IAC/B,IAAI,GAAe,OAAO,CAAC;IAEpC,YAA6B,OAA+B;uBAA/B,OAAO;IAA2B,CAAC;IAEhE,KAAK,CAAC,KAAK,CAAC,GAAgB;QAC1B,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACxD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC;YAC9C,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,oBAAoB;YACzD,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,GAAG,CAAC,KAAK,gBAAgB,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1F,KAAK,EAAE,EAAE;YACT,IAAI,EAAE,cAAc,EAAE;SACvB,CAAC,CAAC;QACH,OAAO,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;CACF"}

package/dist/src/evals/scorers/approval-outcome.scorer.d.ts

@@ -0,0 +1,22 @@
+import type { ScorableRun, ScoreResult, Scorer, ScorerKind } from '../types.js';
+/** {@link ApprovalOutcomeScorer}'s stable name. */
+export declare const APPROVAL_OUTCOME_SCORER = "approval-outcome";
+/**
+ * Every HITL rejection is a negative quality label a human produced for free.
+ *
+ * The library already stops an `action` tool and asks a person "should this run?". That answer is
+ * recorded on the tool call, and it is the only ground truth in the whole system that nobody had to
+ * be paid to collect: an operator who clicks reject has told you the agent proposed the wrong thing,
+ * in the words of the domain, at the moment it mattered. This scorer reads that back as the fraction
+ * of a run's decided actions a human said yes to.
+ *
+ * `null` — not 1 — for a run that proposed no action, or whose actions are all still
+ * `pending_approval`. Most runs are read-only and carry no human verdict at all; counting those as
+ * perfect would bury the runs that do carry one under an average of ~1.
+ */
+export declare class ApprovalOutcomeScorer implements Scorer {
+    readonly name = "approval-outcome";
+    readonly kind: ScorerKind;
+    score(run: ScorableRun): Promise<ScoreResult | null>;
+}
+//# sourceMappingURL=approval-outcome.scorer.d.ts.map

package/dist/src/evals/scorers/approval-outcome.scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"approval-outcome.scorer.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/approval-outcome.scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAoB,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAElG,mDAAmD;AACnD,eAAO,MAAM,uBAAuB,qBAAqB,CAAC;AAW1D;;;;;;;;;;;;GAYG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,QAAQ,CAAC,IAAI,sBAA2B;IACxC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAU;IAE7B,KAAK,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CA2BzD;CACF"}

package/dist/src/evals/scorers/approval-outcome.scorer.js

@@ -0,0 +1,60 @@
+/** {@link ApprovalOutcomeScorer}'s stable name. */
+export const APPROVAL_OUTCOME_SCORER = 'approval-outcome';
+/**
+ * The statuses an `action` tool call can only reach AFTER a human approved it. `failed` belongs
+ * here: the approval happened and the tool then blew up on its own, which is a tool problem, not a
+ * verdict on whether the agent should have proposed it. (`run-completion` is what charges for that
+ * failure.) `pending_approval` is the undecided state, and `auto_executed` never applies — an action
+ * never auto-executes.
+ */
+const APPROVED_STATUSES = new Set(['executed', 'failed']);
+/**
+ * Every HITL rejection is a negative quality label a human produced for free.
+ *
+ * The library already stops an `action` tool and asks a person "should this run?". That answer is
+ * recorded on the tool call, and it is the only ground truth in the whole system that nobody had to
+ * be paid to collect: an operator who clicks reject has told you the agent proposed the wrong thing,
+ * in the words of the domain, at the moment it mattered. This scorer reads that back as the fraction
+ * of a run's decided actions a human said yes to.
+ *
+ * `null` — not 1 — for a run that proposed no action, or whose actions are all still
+ * `pending_approval`. Most runs are read-only and carry no human verdict at all; counting those as
+ * perfect would bury the runs that do carry one under an average of ~1.
+ */
+export class ApprovalOutcomeScorer {
+    name = APPROVAL_OUTCOME_SCORER;
+    kind = 'rule';
+    async score(run) {
+        const actions = run.toolCalls.filter((call) => call.toolType === 'action');
+        const approved = actions.filter((call) => APPROVED_STATUSES.has(call.status));
+        const rejected = actions.filter((call) => call.status === 'rejected');
+        const decided = approved.length + rejected.length;
+        if (decided === 0) {
+            return null;
+        }
+        const metadata = {
+            proposed: actions.length,
+            approved: approved.length,
+            rejected: rejected.length,
+            pending: actions.length - decided,
+            rejectedTools: toolNames(rejected),
+        };
+        if (rejected.length === 0) {
+            return {
+                score: 1,
+                reason: `a human approved all ${decided} decided action${decided === 1 ? '' : 's'}`,
+                metadata,
+            };
+        }
+        return {
+            score: approved.length / decided,
+            reason: `a human approved ${approved.length} of ${decided} decided actions (rejected: ${toolNames(rejected).join(', ')})`,
+            metadata,
+        };
+    }
+}
+/** The distinct tool names in a set of calls, sorted — a stable reason string and metadata value. */
+function toolNames(calls) {
+    return [...new Set(calls.map((call) => call.toolName))].sort();
+}
+//# sourceMappingURL=approval-outcome.scorer.js.map

package/dist/src/evals/scorers/approval-outcome.scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"approval-outcome.scorer.js","sourceRoot":"","sources":["../../../../src/evals/scorers/approval-outcome.scorer.ts"],"names":[],"mappings":"AAEA,mDAAmD;AACnD,MAAM,CAAC,MAAM,uBAAuB,GAAG,kBAAkB,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,qBAAqB;IACvB,IAAI,GAAG,uBAAuB,CAAC;IAC/B,IAAI,GAAe,MAAM,CAAC;IAEnC,KAAK,CAAC,KAAK,CAAC,GAAgB;QAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9E,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAClD,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,QAAQ,GAAG;YACf,QAAQ,EAAE,OAAO,CAAC,MAAM;YACxB,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,OAAO;YACjC,aAAa,EAAE,SAAS,CAAC,QAAQ,CAAC;SACnC,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,wBAAwB,OAAO,kBAAkB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE;gBACnF,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,OAAO;YAChC,MAAM,EAAE,oBAAoB,QAAQ,CAAC,MAAM,OAAO,OAAO,+BAA+B,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACzH,QAAQ;SACT,CAAC;IACJ,CAAC;CACF;AAED,qGAAqG;AACrG,SAAS,SAAS,CAAC,KAAyB;IAC1C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACjE,CAAC"}

package/dist/src/evals/scorers/approval-risk.scorer.d.ts

@@ -0,0 +1,28 @@
+import { type ToolApprovalPrior } from '../approval-prior.js';
+import type { ScorableRun, ScoreResult, Scorer, ScorerKind } from '../types.js';
+/** {@link ApprovalRiskScorer}'s stable name. */
+export declare const APPROVAL_RISK_SCORER = "approval-risk";
+/**
+ * How likely a human is to reject what this run proposed — BEFORE a human looks at it.
+ *
+ * {@link import('./approval-outcome.scorer.js').ApprovalOutcomeScorer} reads verdicts that already
+ * exist; this turns the same corpus into a prediction, so a pending approvals inbox can be drained
+ * riskiest-first and a run that is about to be rejected can be caught while it is still cheap. It is
+ * the statistical family, not the rule-based one: the number is a smoothed frequency estimate over
+ * past decisions, and its `metadata` carries the sample it rests on so a 1-of-1 rejection is never
+ * mistaken for certainty.
+ *
+ * A run scores as its WORST proposed action, not the average of them. A turn that proposes one tool
+ * humans reject nine times in ten is a risky turn no matter how many safe reads it made alongside.
+ *
+ * `null` for a run that proposed no action — the same "not applicable, not perfect" rule as the
+ * outcome scorer.
+ */
+export declare class ApprovalRiskScorer implements Scorer {
+    private readonly prior;
+    readonly name = "approval-risk";
+    readonly kind: ScorerKind;
+    constructor(prior: ToolApprovalPrior);
+    score(run: ScorableRun): Promise<ScoreResult | null>;
+}
+//# sourceMappingURL=approval-risk.scorer.d.ts.map

package/dist/src/evals/scorers/approval-risk.scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"approval-risk.scorer.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/approval-risk.scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEhF,gDAAgD;AAChD,eAAO,MAAM,oBAAoB,kBAAkB,CAAC;AAUpD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAInC,OAAO,CAAC,QAAQ,CAAC,KAAK;IAHlC,QAAQ,CAAC,IAAI,mBAAwB;IACrC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAiB;IAE1C,YAA6B,KAAK,EAAE,iBAAiB,EAAI;IAEnD,KAAK,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CA4BzD;CACF"}

package/dist/src/evals/scorers/approval-risk.scorer.js

@@ -0,0 +1,54 @@
+import { approvalPosterior } from '../approval-prior.js';
+/** {@link ApprovalRiskScorer}'s stable name. */
+export const APPROVAL_RISK_SCORER = 'approval-risk';
+/**
+ * How likely a human is to reject what this run proposed — BEFORE a human looks at it.
+ *
+ * {@link import('./approval-outcome.scorer.js').ApprovalOutcomeScorer} reads verdicts that already
+ * exist; this turns the same corpus into a prediction, so a pending approvals inbox can be drained
+ * riskiest-first and a run that is about to be rejected can be caught while it is still cheap. It is
+ * the statistical family, not the rule-based one: the number is a smoothed frequency estimate over
+ * past decisions, and its `metadata` carries the sample it rests on so a 1-of-1 rejection is never
+ * mistaken for certainty.
+ *
+ * A run scores as its WORST proposed action, not the average of them. A turn that proposes one tool
+ * humans reject nine times in ten is a risky turn no matter how many safe reads it made alongside.
+ *
+ * `null` for a run that proposed no action — the same "not applicable, not perfect" rule as the
+ * outcome scorer.
+ */
+export class ApprovalRiskScorer {
+    prior;
+    name = APPROVAL_RISK_SCORER;
+    kind = 'statistical';
+    constructor(prior) {
+        this.prior = prior;
+    }
+    async score(run) {
+        const actions = run.toolCalls.filter((call) => call.toolType === 'action');
+        if (actions.length === 0) {
+            return null;
+        }
+        const proposed = [...new Set(actions.map((call) => call.toolName))]
+            .sort()
+            .map((toolName) => {
+            const counts = this.prior.get(toolName);
+            return {
+                toolName,
+                approved: counts?.approved ?? 0,
+                rejected: counts?.rejected ?? 0,
+                pApprove: approvalPosterior(counts),
+            };
+        });
+        const riskiest = proposed.reduce((worst, candidate) => candidate.pApprove < worst.pApprove ? candidate : worst);
+        const decided = riskiest.approved + riskiest.rejected;
+        return {
+            score: riskiest.pApprove,
+            reason: decided === 0
+                ? `no human has ever decided on ${riskiest.toolName}, the riskiest of ${proposed.length} proposed action(s)`
+                : `the riskiest proposed action is ${riskiest.toolName} — humans approved ${riskiest.approved} of its ${decided} past calls`,
+            metadata: { riskiestTool: riskiest.toolName, proposed },
+        };
+    }
+}
+//# sourceMappingURL=approval-risk.scorer.js.map

package/dist/src/evals/scorers/approval-risk.scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"approval-risk.scorer.js","sourceRoot":"","sources":["../../../../src/evals/scorers/approval-risk.scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAA0B,MAAM,sBAAsB,CAAC;AAGjF,gDAAgD;AAChD,MAAM,CAAC,MAAM,oBAAoB,GAAG,eAAe,CAAC;AAUpD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,kBAAkB;IAIA,KAAK;IAHzB,IAAI,GAAG,oBAAoB,CAAC;IAC5B,IAAI,GAAe,aAAa,CAAC;IAE1C,YAA6B,KAAwB;qBAAxB,KAAK;IAAsB,CAAC;IAEzD,KAAK,CAAC,KAAK,CAAC,GAAgB;QAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;QAC3E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,QAAQ,GAAqB,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;aAClF,IAAI,EAAE;aACN,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAChB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACxC,OAAO;gBACL,QAAQ;gBACR,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,CAAC;gBAC/B,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,CAAC;gBAC/B,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;QACL,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CACpD,SAAS,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CACxD,CAAC;QACF,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACtD,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,QAAQ;YACxB,MAAM,EACJ,OAAO,KAAK,CAAC;gBACX,CAAC,CAAC,gCAAgC,QAAQ,CAAC,QAAQ,qBAAqB,QAAQ,CAAC,MAAM,qBAAqB;gBAC5G,CAAC,CAAC,mCAAmC,QAAQ,CAAC,QAAQ,sBAAsB,QAAQ,CAAC,QAAQ,WAAW,OAAO,aAAa;YAChI,QAAQ,EAAE,EAAE,YAAY,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE;SACxD,CAAC;IACJ,CAAC;CACF"}

package/dist/src/evals/scorers/run-completion.scorer.d.ts

@@ -0,0 +1,18 @@
+import type { ScorableRun, ScoreResult, Scorer, ScorerKind } from '../types.js';
+/** {@link RunCompletionScorer}'s stable name, for a `where.scorer` filter or a resumed backfill. */
+export declare const RUN_COMPLETION_SCORER = "run-completion";
+/**
+ * Did the turn actually deliver? Deterministic, free, and the baseline every other scorer is read
+ * against.
+ *
+ * It is NOT the governance success rate. That counts a run as a success the moment it settles
+ * `completed`, which a turn that answered nothing after three failed tool calls also does. This asks
+ * the narrower question the ledger cannot: was there an answer, and was the agent whole while it
+ * wrote one.
+ */
+export declare class RunCompletionScorer implements Scorer {
+    readonly name = "run-completion";
+    readonly kind: ScorerKind;
+    score(run: ScorableRun): Promise<ScoreResult | null>;
+}
+//# sourceMappingURL=run-completion.scorer.d.ts.map

package/dist/src/evals/scorers/run-completion.scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-completion.scorer.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/run-completion.scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEhF,oGAAoG;AACpG,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAStD;;;;;;;;GAQG;AACH,qBAAa,mBAAoB,YAAW,MAAM;IAChD,QAAQ,CAAC,IAAI,oBAAyB;IACtC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAU;IAE7B,KAAK,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAiCzD;CACF"}

package/dist/src/evals/scorers/run-completion.scorer.js

@@ -0,0 +1,55 @@
+/** {@link RunCompletionScorer}'s stable name, for a `where.scorer` filter or a resumed backfill. */
+export const RUN_COMPLETION_SCORER = 'run-completion';
+/**
+ * An answer assembled from a partially-broken toolset. Kept distinguishable from BOTH a clean answer
+ * (1) and no answer at all (0): the user did get a reply, and it was written without whatever the
+ * failed tool was going to say.
+ */
+const PARTIAL_TOOLING_SCORE = 0.5;
+/**
+ * Did the turn actually deliver? Deterministic, free, and the baseline every other scorer is read
+ * against.
+ *
+ * It is NOT the governance success rate. That counts a run as a success the moment it settles
+ * `completed`, which a turn that answered nothing after three failed tool calls also does. This asks
+ * the narrower question the ledger cannot: was there an answer, and was the agent whole while it
+ * wrote one.
+ */
+export class RunCompletionScorer {
+    name = RUN_COMPLETION_SCORER;
+    kind = 'rule';
+    async score(run) {
+        const failedCalls = run.toolCalls.filter((call) => call.status === 'failed');
+        const metadata = {
+            status: run.status,
+            outputChars: run.output.trim().length,
+            toolCalls: run.toolCalls.length,
+            failedToolCalls: failedCalls.length,
+        };
+        // A run still in flight has no outcome to judge — scoring it would freeze a verdict the run is
+        // about to contradict.
+        if (run.status === 'running') {
+            return null;
+        }
+        if (run.status === 'failed' || run.status === 'cancelled') {
+            return {
+                score: 0,
+                reason: `the run ${run.status} (${run.error ?? 'unclassified'})`,
+                metadata,
+            };
+        }
+        if (run.output.trim() === '') {
+            return { score: 0, reason: 'the run completed without answering anything', metadata };
+        }
+        if (failedCalls.length > 0) {
+            const names = [...new Set(failedCalls.map((call) => call.toolName))].sort().join(', ');
+            return {
+                score: PARTIAL_TOOLING_SCORE,
+                reason: `answered, but ${failedCalls.length} of ${run.toolCalls.length} tool calls failed (${names})`,
+                metadata,
+            };
+        }
+        return { score: 1, reason: 'the run completed and answered with every tool intact', metadata };
+    }
+}
+//# sourceMappingURL=run-completion.scorer.js.map

package/dist/src/evals/scorers/run-completion.scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-completion.scorer.js","sourceRoot":"","sources":["../../../../src/evals/scorers/run-completion.scorer.ts"],"names":[],"mappings":"AAEA,oGAAoG;AACpG,MAAM,CAAC,MAAM,qBAAqB,GAAG,gBAAgB,CAAC;AAEtD;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC;;;;;;;;GAQG;AACH,MAAM,OAAO,mBAAmB;IACrB,IAAI,GAAG,qBAAqB,CAAC;IAC7B,IAAI,GAAe,MAAM,CAAC;IAEnC,KAAK,CAAC,KAAK,CAAC,GAAgB;QAC1B,MAAM,WAAW,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG;YACf,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM;YACrC,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;YAC/B,eAAe,EAAE,WAAW,CAAC,MAAM;SACpC,CAAC;QAEF,+FAA+F;QAC/F,uBAAuB;QACvB,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC1D,OAAO;gBACL,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,WAAW,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,KAAK,IAAI,cAAc,GAAG;gBAChE,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC7B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,8CAA8C,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvF,OAAO;gBACL,KAAK,EAAE,qBAAqB;gBAC5B,MAAM,EAAE,iBAAiB,WAAW,CAAC,MAAM,OAAO,GAAG,CAAC,SAAS,CAAC,MAAM,uBAAuB,KAAK,GAAG;gBACrG,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,uDAAuD,EAAE,QAAQ,EAAE,CAAC;IACjG,CAAC;CACF"}

package/dist/src/evals/summarize.d.ts

@@ -0,0 +1,44 @@
+import type { RunScore } from './score-store.js';
+import type { ScorerKind } from './types.js';
+/**
+ * The pure aggregation core over persisted scores, mirroring the governance read-model: an adapter's
+ * only job is fetching rows, and every surface that reports a mean score runs the SAME arithmetic
+ * here, so a dashboard and a CI gate can never disagree about whether quality moved.
+ */
+/** Rollup for one scorer over a set of scores. */
+export interface ScorerSummaryRow {
+    scorer: string;
+    kind: ScorerKind;
+    /** How many runs this scorer actually scored. A mean over 3 samples is not a metric — read this first. */
+    samples: number;
+    meanScore: number;
+    minScore: number;
+    maxScore: number;
+}
+/** Rollup for one agent under one scorer. */
+export interface AgentScoreRow {
+    /** `'(default)'` when the scored run named no agent, mirroring the governance breakdown. */
+    agentName: string;
+    scorer: string;
+    samples: number;
+    meanScore: number;
+}
+/** One point on the daily quality trend, per scorer. */
+export interface ScoreTrendPoint {
+    day: string;
+    scorer: string;
+    samples: number;
+    meanScore: number;
+}
+/** Rollup per scorer, worst mean first — the ordering that puts the problem at the top. */
+export declare function summarizeByScorer(rows: RunScore[]): ScorerSummaryRow[];
+/** Rollup per (agent, scorer) pair, worst mean first — which persona drags which signal down. */
+export declare function summarizeByAgent(rows: RunScore[]): AgentScoreRow[];
+/** Daily mean per scorer, ascending by day then scorer — the "is it getting better" series. */
+export declare function bucketScoreTrend(rows: RunScore[]): ScoreTrendPoint[];
+/**
+ * The lowest-scoring runs, worst first, capped at `limit` — the triage queue. Every scorer carries a
+ * `reason`, so this is the one view that answers "show me what went wrong" without a second read.
+ */
+export declare function worstScoredRuns(rows: RunScore[], limit: number): RunScore[];
+//# sourceMappingURL=summarize.d.ts.map

package/dist/src/evals/summarize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"summarize.d.ts","sourceRoot":"","sources":["../../../src/evals/summarize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C;;;;GAIG;AAEH,kDAAkD;AAClD,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;IACjB,0GAA0G;IAC1G,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,6CAA6C;AAC7C,MAAM,WAAW,aAAa;IAC5B,4FAA4F;IAC5F,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAcD,2FAA2F;AAC3F,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,gBAAgB,EAAE,CAsBtE;AAED,iGAAiG;AACjG,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,aAAa,EAAE,CAyBlE;AAED,+FAA+F;AAC/F,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,eAAe,EAAE,CAqBpE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE,CAI3E"}

package/dist/src/evals/summarize.js

@@ -0,0 +1,87 @@
+/** Bucket key for a score whose run named no agent (mirrors the governance run breakdown). */
+const DEFAULT_AGENT_BUCKET = '(default)';
+/** Mean of a bucket. Every caller below builds its buckets by pushing, so none is ever empty. */
+function mean(values) {
+    let total = 0;
+    for (const value of values) {
+        total += value;
+    }
+    return total / values.length;
+}
+/** Rollup per scorer, worst mean first — the ordering that puts the problem at the top. */
+export function summarizeByScorer(rows) {
+    const byScorer = new Map();
+    for (const row of rows) {
+        const bucket = byScorer.get(row.scorer) ?? { kind: row.kind, scores: [] };
+        bucket.scores.push(row.score);
+        byScorer.set(row.scorer, bucket);
+    }
+    const result = [];
+    for (const [scorer, bucket] of byScorer) {
+        result.push({
+            scorer,
+            kind: bucket.kind,
+            samples: bucket.scores.length,
+            meanScore: mean(bucket.scores),
+            minScore: Math.min(...bucket.scores),
+            maxScore: Math.max(...bucket.scores),
+        });
+    }
+    result.sort((left, right) => left.meanScore - right.meanScore || left.scorer.localeCompare(right.scorer));
+    return result;
+}
+/** Rollup per (agent, scorer) pair, worst mean first — which persona drags which signal down. */
+export function summarizeByAgent(rows) {
+    const byPair = new Map();
+    for (const row of rows) {
+        const agentName = row.agentName ?? DEFAULT_AGENT_BUCKET;
+        const key = `${agentName} ${row.scorer}`;
+        const bucket = byPair.get(key) ?? { agentName, scorer: row.scorer, scores: [] };
+        bucket.scores.push(row.score);
+        byPair.set(key, bucket);
+    }
+    const result = [];
+    for (const bucket of byPair.values()) {
+        result.push({
+            agentName: bucket.agentName,
+            scorer: bucket.scorer,
+            samples: bucket.scores.length,
+            meanScore: mean(bucket.scores),
+        });
+    }
+    result.sort((left, right) => left.meanScore - right.meanScore ||
+        left.agentName.localeCompare(right.agentName) ||
+        left.scorer.localeCompare(right.scorer));
+    return result;
+}
+/** Daily mean per scorer, ascending by day then scorer — the "is it getting better" series. */
+export function bucketScoreTrend(rows) {
+    const byDay = new Map();
+    for (const row of rows) {
+        const key = `${row.day} ${row.scorer}`;
+        const bucket = byDay.get(key) ?? { day: row.day, scorer: row.scorer, scores: [] };
+        bucket.scores.push(row.score);
+        byDay.set(key, bucket);
+    }
+    const result = [];
+    for (const bucket of byDay.values()) {
+        result.push({
+            day: bucket.day,
+            scorer: bucket.scorer,
+            samples: bucket.scores.length,
+            meanScore: mean(bucket.scores),
+        });
+    }
+    result.sort((left, right) => left.day.localeCompare(right.day) || left.scorer.localeCompare(right.scorer));
+    return result;
+}
+/**
+ * The lowest-scoring runs, worst first, capped at `limit` — the triage queue. Every scorer carries a
+ * `reason`, so this is the one view that answers "show me what went wrong" without a second read.
+ */
+export function worstScoredRuns(rows, limit) {
+    return [...rows]
+        .sort((left, right) => left.score - right.score || left.runId.localeCompare(right.runId))
+        .slice(0, limit);
+}
+//# sourceMappingURL=summarize.js.map

package/dist/src/evals/summarize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"summarize.js","sourceRoot":"","sources":["../../../src/evals/summarize.ts"],"names":[],"mappings":"AAqCA,8FAA8F;AAC9F,MAAM,oBAAoB,GAAG,WAAW,CAAC;AAEzC,iGAAiG;AACjG,SAAS,IAAI,CAAC,MAAgB;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,IAAI,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;AAC/B,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,iBAAiB,CAAC,IAAgB;IAChD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkD,CAAC;IAC3E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC;YACV,MAAM;YACN,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC7B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YAC9B,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YACpC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CACT,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAC7F,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,gBAAgB,CAAC,IAAgB;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmE,CAAC;IAC1F,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,oBAAoB,CAAC;QACxD,MAAM,GAAG,GAAG,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC;YACV,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC7B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CACT,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS;QAChC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC;QAC7C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAC1C,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,gBAAgB,CAAC,IAAgB;IAC/C,MAAM,KAAK,GAAG,IAAI,GAAG,EAA6D,CAAC;IACnF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC7B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CACT,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAC9F,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,IAAgB,EAAE,KAAa;IAC7D,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;SACxF,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrB,CAAC"}

package/dist/src/evals/types.d.ts

@@ -0,0 +1,81 @@
+/**
+ * The evaluation SPI: a {@link Scorer} turns ONE stored run into a number plus the sentence that
+ * justifies it. Everything else here (the batch runner, the score store, the summaries) is plumbing
+ * around that one call.
+ *
+ * Scoring is offline-first. A {@link ScorableRun} is assembled from what the agent already persisted
+ * — the run row, its transcript, its tool calls and their HITL outcomes — so a scorer never sits
+ * inside a turn, never adds latency to one, and can be re-run over history the day a new scorer is
+ * written.
+ */
+/** One tool call the run made, reduced to what a scorer can judge. */
+export interface ScorableToolCall {
+    toolCallId: string;
+    toolName: string;
+    /** `'read'` (auto-executed) or `'action'` — only an `action` carries a human decision. */
+    toolType: string;
+    /** A {@link import('../types.js').ToolCallStatus} as the store recorded it. */
+    status: string;
+    /** Wall time of the execution; `null` for a call that never executed. */
+    executionMs: number | null;
+    /** The failure text for a `failed` call; `null` otherwise. */
+    error: string | null;
+}
+/** One stored run, flattened into the unit a scorer judges. */
+export interface ScorableRun {
+    runId: string;
+    threadId: string;
+    actorRef: string;
+    agentName: string | null;
+    /** `'running'` | `'completed'` | `'failed'` | `'cancelled'`, as the store recorded it. */
+    status: string;
+    /** The user text that opened the turn; `''` when the run holds none. */
+    input: string;
+    /** The assistant text the turn produced; `''` when the run answered nothing. */
+    output: string;
+    toolCalls: ScorableToolCall[];
+    durationMs: number | null;
+    /**
+     * The run's failure message as the read-model carries it; `null` for a run that did not fail. The
+     * read-model records prose here rather than a stable slug, so a scorer quotes it instead of
+     * branching on it.
+     */
+    error: string | null;
+    /** ISO timestamp the run started — the day a score is bucketed under. */
+    startedAt: string;
+}
+/**
+ * Which family a scorer belongs to, kept on the persisted score so a summary can separate a free
+ * deterministic signal from one that cost money to produce.
+ *
+ * - `rule`        — deterministic, derived from the run alone (no corpus, no model).
+ * - `statistical` — an estimate over a corpus of past runs (frequencies, priors).
+ * - `model`       — LLM-as-judge: another model call, therefore the expensive family.
+ */
+export type ScorerKind = 'rule' | 'statistical' | 'model';
+/** One scorer's verdict on one run. */
+export interface ScoreResult {
+    /** Normalized to `0..1`, where 1 is good. A scorer that cannot honour that range is a bug. */
+    score: number;
+    /** Why THAT number, in one sentence — the thing an operator reads before the number. */
+    reason: string;
+    /** Scorer-specific detail (counts, per-tool breakdowns) persisted alongside the score. */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * A quality signal over a stored run.
+ *
+ * `score` returns `null` for a run the scorer has nothing to say about — an approval scorer facing a
+ * run that proposed no action, a judge facing an empty answer. That is deliberately NOT a 0: a score
+ * of 0 says "bad", `null` says "not applicable", and averaging the two together is how a quality
+ * metric silently stops meaning anything.
+ */
+export interface Scorer {
+    /** Stable identifier, persisted on every score row and used to resume a backfill. */
+    name: string;
+    kind: ScorerKind;
+    score(run: ScorableRun): Promise<ScoreResult | null>;
+}
+/** Clamp a computed score into the `0..1` range {@link ScoreResult.score} promises. */
+export declare function clampScore(value: number): number;
+//# sourceMappingURL=types.d.ts.map

package/dist/src/evals/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/evals/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,sEAAsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,0FAA0F;IAC1F,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,8DAA8D;IAC9D,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,+DAA+D;AAC/D,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,0FAA0F;IAC1F,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,KAAK,EAAE,MAAM,CAAC;IACd,gFAAgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B;;;;OAIG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,OAAO,CAAC;AAE1D,uCAAuC;AACvC,MAAM,WAAW,WAAW;IAC1B,8FAA8F;IAC9F,KAAK,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAC;IACf,0FAA0F;IAC1F,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,MAAM;IACrB,qFAAqF;IACrF,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;CACtD;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhD"}

package/dist/src/evals/types.js

@@ -0,0 +1,15 @@
+/**
+ * The evaluation SPI: a {@link Scorer} turns ONE stored run into a number plus the sentence that
+ * justifies it. Everything else here (the batch runner, the score store, the summaries) is plumbing
+ * around that one call.
+ *
+ * Scoring is offline-first. A {@link ScorableRun} is assembled from what the agent already persisted
+ * — the run row, its transcript, its tool calls and their HITL outcomes — so a scorer never sits
+ * inside a turn, never adds latency to one, and can be re-run over history the day a new scorer is
+ * written.
+ */
+/** Clamp a computed score into the `0..1` range {@link ScoreResult.score} promises. */
+export function clampScore(value) {
+    return Math.min(1, Math.max(0, value));
+}
+//# sourceMappingURL=types.js.map

package/dist/src/evals/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/evals/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA2EH,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC"}