@adonis-agora/agent 0.30.2 → 0.33.0

package/dist/src/evals/governance-sample-source.js

@@ -0,0 +1,93 @@
+/**
+ * A {@link RunSampleSource} over what the agent already persisted, read through
+ * {@link AgentGovernanceQueries} alone. Works against any adapter that implements it (Lucid, the
+ * in-memory one) — this module adds no tables of its own on the read side.
+ *
+ * The prompt and the answer come off `runDetail`, whose messages are the ones stamped with this
+ * run's own `run_id`. There is no transcript join and no time-window heuristic: a run that wrote no
+ * message has empty text, and no other run's answer can ever be handed to it.
+ *
+ * One seam remains, and it is the read-model's rather than this class's: a run whose messages were
+ * written before message-level run stamping reads as empty text. A rule-based scorer over tool
+ * outcomes still works on it; a judge over the answer has nothing to read.
+ */
+export class GovernanceRunSampleSource {
+    queries;
+    constructor(queries) {
+        this.queries = queries;
+    }
+    async listRuns(query) {
+        const filter = {
+            ...(query.actorRef !== undefined ? { actor: query.actorRef } : {}),
+            ...(query.agentName !== undefined ? { agent: query.agentName } : {}),
+            ...(query.status !== undefined ? { status: query.status } : {}),
+            ...(query.fromDay !== undefined ? { from: query.fromDay } : {}),
+            ...(query.toDay !== undefined ? { to: query.toDay } : {}),
+        };
+        // The read-model clamps a page to 200, so a batch bigger than that has to walk the cursor rather
+        // than ask for the whole thing and quietly get a fifth of it.
+        const runs = [];
+        let cursor = null;
+        while (runs.length < query.limit) {
+            const page = await this.queries.listRuns({
+                ...filter,
+                first: query.limit - runs.length,
+                ...(cursor !== null ? { after: cursor } : {}),
+            });
+            for (const row of page.items) {
+                const detail = await this.queries.runDetail(row.runId);
+                if (detail !== null) {
+                    runs.push(toScorableRun(detail));
+                }
+            }
+            cursor = page.nextCursor;
+            if (cursor === null || page.items.length === 0) {
+                break;
+            }
+        }
+        return runs;
+    }
+    async getRun(runId) {
+        const detail = await this.queries.runDetail(runId);
+        return detail === null ? null : toScorableRun(detail);
+    }
+}
+/**
+ * Flatten a run's trace into the unit a scorer judges. A turn can persist several assistant messages
+ * (one per model step); the LAST is the answer and the earlier ones are the steps that called tools,
+ * so both sides reduce by taking the last of their role.
+ */
+function toScorableRun(detail) {
+    const { run } = detail;
+    let input = '';
+    let output = '';
+    for (const message of detail.messages) {
+        if (message.role === 'user') {
+            input = message.content;
+        }
+        else if (message.role === 'assistant') {
+            output = message.content;
+        }
+    }
+    return {
+        runId: run.runId,
+        threadId: run.threadId,
+        actorRef: run.actorRef,
+        agentName: run.agentName,
+        status: run.status,
+        input,
+        output,
+        toolCalls: detail.toolCalls.map((call) => ({
+            toolCallId: call.toolCallId,
+            toolName: call.toolName,
+            toolType: call.toolType,
+            status: call.status,
+            executionMs: call.executionMs,
+            error: call.error,
+        })),
+        durationMs: run.durationMs,
+        error: run.error,
+        startedAt: run.startedAt,
+    };
+}
+//# sourceMappingURL=governance-sample-source.js.map

package/dist/src/evals/governance-sample-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"governance-sample-source.js","sourceRoot":"","sources":["../../../src/evals/governance-sample-source.ts"],"names":[],"mappings":"AAQA;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,yBAAyB;IACP,OAAO;IAApC,YAA6B,OAA+B;uBAA/B,OAAO;IAA2B,CAAC;IAEhE,KAAK,CAAC,QAAQ,CAAC,KAAqB;QAClC,MAAM,MAAM,GAAmB;YAC7B,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,GAAG,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1D,CAAC;QACF,iGAAiG;QACjG,8DAA8D;QAC9D,MAAM,IAAI,GAAkB,EAAE,CAAC;QAC/B,IAAI,MAAM,GAAkB,IAAI,CAAC;QACjC,OAAO,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACvC,GAAG,MAAM;gBACT,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM;gBAChC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9C,CAAC,CAAC;YACH,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACvD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;oBACpB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;YACzB,IAAI,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/C,MAAM;YACR,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,MAAiB;IACtC,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;IACvB,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5B,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;QAC1B,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACxC,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK;QACL,MAAM;QACN,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,CAC7B,CAAC,IAAI,EAAoB,EAAE,CAAC,CAAC;YAC3B,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CACH;QACD,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,SAAS,EAAE,GAAG,CAAC,SAAS;KACzB,CAAC;AACJ,CAAC"}

package/dist/src/evals/in-memory-score-store.d.ts

@@ -0,0 +1,16 @@
+import type { RunScore, ScoreStore, ScoreWhere } from './score-store.js';
+/**
+ * A fully in-memory {@link ScoreStore} for unit tests and the offline demo.
+ *
+ * Append-only: `recordScores` never replaces a previous verdict, so re-scoring the same run after a
+ * scorer changed leaves both rows and the history stays auditable. Callers that want one row per
+ * `(runId, scorer)` skip the already-scored runs via {@link InMemoryScoreStore.scoredRunIds}, which
+ * is what `runEvaluation` does by default.
+ */
+export declare class InMemoryScoreStore implements ScoreStore {
+    private readonly scores;
+    recordScores(scores: RunScore[]): Promise<void>;
+    scoredRunIds(scorer: string, runIds: string[]): Promise<string[]>;
+    listScores(where?: ScoreWhere): Promise<RunScore[]>;
+}
+//# sourceMappingURL=in-memory-score-store.d.ts.map

package/dist/src/evals/in-memory-score-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"in-memory-score-store.d.ts","sourceRoot":"","sources":["../../../src/evals/in-memory-score-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAqBzE;;;;;;;GAOG;AACH,qBAAa,kBAAmB,YAAW,UAAU;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IAEnC,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIpD;IAEK,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAStE;IAEK,UAAU,CAAC,KAAK,GAAE,UAAe,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAc5D;CACF"}

package/dist/src/evals/in-memory-score-store.js

@@ -0,0 +1,57 @@
+/**
+ * Whether a `YYYY-MM-DD` day falls within an inclusive range whose bounds are each optional — an
+ * absent bound imposes no constraint on that side. Lexicographic comparison is valid because the
+ * days are zero-padded ISO dates.
+ */
+function dayWithinBounds(day, fromDay, toDay) {
+    if (fromDay !== undefined && day < fromDay) {
+        return false;
+    }
+    if (toDay !== undefined && day > toDay) {
+        return false;
+    }
+    return true;
+}
+/**
+ * A fully in-memory {@link ScoreStore} for unit tests and the offline demo.
+ *
+ * Append-only: `recordScores` never replaces a previous verdict, so re-scoring the same run after a
+ * scorer changed leaves both rows and the history stays auditable. Callers that want one row per
+ * `(runId, scorer)` skip the already-scored runs via {@link InMemoryScoreStore.scoredRunIds}, which
+ * is what `runEvaluation` does by default.
+ */
+export class InMemoryScoreStore {
+    scores = [];
+    async recordScores(scores) {
+        for (const score of scores) {
+            this.scores.push({ ...score });
+        }
+    }
+    async scoredRunIds(scorer, runIds) {
+        const wanted = new Set(runIds);
+        const seen = new Set();
+        for (const row of this.scores) {
+            if (row.scorer === scorer && wanted.has(row.runId)) {
+                seen.add(row.runId);
+            }
+        }
+        return [...seen];
+    }
+    async listScores(where = {}) {
+        return this.scores
+            .filter((row) => {
+            if (where.scorer !== undefined && row.scorer !== where.scorer)
+                return false;
+            if (where.agentName !== undefined && row.agentName !== where.agentName)
+                return false;
+            if (where.threadId !== undefined && row.threadId !== where.threadId)
+                return false;
+            if (where.runId !== undefined && row.runId !== where.runId)
+                return false;
+            return dayWithinBounds(row.day, where.fromDay, where.toDay);
+        })
+            .sort((left, right) => right.scoredAt.localeCompare(left.scoredAt) || right.runId.localeCompare(left.runId))
+            .map((row) => ({ ...row }));
+    }
+}
+//# sourceMappingURL=in-memory-score-store.js.map

package/dist/src/evals/in-memory-score-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"in-memory-score-store.js","sourceRoot":"","sources":["../../../src/evals/in-memory-score-store.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,SAAS,eAAe,CACtB,GAAW,EACX,OAA2B,EAC3B,KAAyB;IAEzB,IAAI,OAAO,KAAK,SAAS,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;QAC3C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,KAAK,SAAS,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,kBAAkB;IACZ,MAAM,GAAe,EAAE,CAAC;IAEzC,KAAK,CAAC,YAAY,CAAC,MAAkB;QACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,MAAgB;QACjD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAK,GAAe,EAAE;QACrC,OAAO,IAAI,CAAC,MAAM;aACf,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YACd,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAC;YAC5E,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;gBAAE,OAAO,KAAK,CAAC;YACrF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAClF,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAC;YACzE,OAAO,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9D,CAAC,CAAC;aACD,IAAI,CACH,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CACvF;aACA,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;IAChC,CAAC;CACF"}

package/dist/src/evals/index.d.ts

@@ -0,0 +1,15 @@
+export { type ApprovalPriorQuery, approvalPosterior, buildApprovalPrior, loadApprovalPrior, priorFromRuns, type ToolApprovalCounts, type ToolApprovalDecision, type ToolApprovalPrior, } from './approval-prior.js';
+export { type EvaluationOptions, type EvaluationSummary, runEvaluation, type ScorerFailure, toRunScore, } from './evaluate.js';
+export { GovernanceRunSampleSource } from './governance-sample-source.js';
+export { InMemoryScoreStore } from './in-memory-score-store.js';
+export { discardingSink, JudgeVerdictError, MAX_JUDGE_SCORE, parseJudgeVerdict, } from './judge.js';
+export { attachLiveScoring, type LiveScoring, type LiveScoringOptions } from './live-scoring.js';
+export { type RunSampleQuery, type RunSampleSource, StaticRunSampleSource, } from './sample-source.js';
+export type { RunScore, ScoreStore, ScoreWhere } from './score-store.js';
+export { ANSWER_RELEVANCY_SCORER, type AnswerRelevancyOptions, AnswerRelevancyScorer, } from './scorers/answer-relevancy.scorer.js';
+export { APPROVAL_OUTCOME_SCORER, ApprovalOutcomeScorer, } from './scorers/approval-outcome.scorer.js';
+export { APPROVAL_RISK_SCORER, ApprovalRiskScorer } from './scorers/approval-risk.scorer.js';
+export { RUN_COMPLETION_SCORER, RunCompletionScorer } from './scorers/run-completion.scorer.js';
+export { type AgentScoreRow, bucketScoreTrend, type ScorerSummaryRow, type ScoreTrendPoint, summarizeByAgent, summarizeByScorer, worstScoredRuns, } from './summarize.js';
+export { clampScore, type ScorableRun, type ScorableToolCall, type ScoreResult, type Scorer, type ScorerKind, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/evals/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EACvB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,aAAa,EACb,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,aAAa,EACb,KAAK,aAAa,EAClB,UAAU,GACX,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,iBAAiB,EAAE,KAAK,WAAW,EAAE,KAAK,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACjG,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EACL,uBAAuB,EACvB,KAAK,sBAAsB,EAC3B,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAC7F,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AAChG,OAAO,EACL,KAAK,aAAa,EAClB,gBAAgB,EAChB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,UAAU,EACV,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,WAAW,EAChB,KAAK,MAAM,EACX,KAAK,UAAU,GAChB,MAAM,YAAY,CAAC"}

package/dist/src/evals/index.js

@@ -0,0 +1,14 @@
+export { approvalPosterior, buildApprovalPrior, loadApprovalPrior, priorFromRuns, } from './approval-prior.js';
+export { runEvaluation, toRunScore, } from './evaluate.js';
+export { GovernanceRunSampleSource } from './governance-sample-source.js';
+export { InMemoryScoreStore } from './in-memory-score-store.js';
+export { discardingSink, JudgeVerdictError, MAX_JUDGE_SCORE, parseJudgeVerdict, } from './judge.js';
+export { attachLiveScoring } from './live-scoring.js';
+export { StaticRunSampleSource, } from './sample-source.js';
+export { ANSWER_RELEVANCY_SCORER, AnswerRelevancyScorer, } from './scorers/answer-relevancy.scorer.js';
+export { APPROVAL_OUTCOME_SCORER, ApprovalOutcomeScorer, } from './scorers/approval-outcome.scorer.js';
+export { APPROVAL_RISK_SCORER, ApprovalRiskScorer } from './scorers/approval-risk.scorer.js';
+export { RUN_COMPLETION_SCORER, RunCompletionScorer } from './scorers/run-completion.scorer.js';
+export { bucketScoreTrend, summarizeByAgent, summarizeByScorer, worstScoredRuns, } from './summarize.js';
+export { clampScore, } from './types.js';
+//# sourceMappingURL=index.js.map

package/dist/src/evals/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,aAAa,GAId,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAGL,aAAa,EAEb,UAAU,GACX,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,iBAAiB,EAA6C,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAGL,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,uBAAuB,EAEvB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAC7F,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AAChG,OAAO,EAEL,gBAAgB,EAGhB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,UAAU,GAMX,MAAM,YAAY,CAAC"}

package/dist/src/evals/judge.d.ts

@@ -0,0 +1,30 @@
+import type { SinkWriter } from '../spi/token-stream-sink.js';
+import type { ScoreResult } from './types.js';
+/** The toolkit a model-graded (LLM-as-judge) scorer needs: a sink to nowhere and a verdict parser. */
+/**
+ * A {@link SinkWriter} that drops every token.
+ *
+ * {@link import('../spi/model-provider.js').ModelProvider.runTurn} streams deltas as it generates,
+ * but a judge's verdict is read from the assembled `text` it returns and nobody is watching it
+ * arrive — a judge call has no run, so there is no live stream to join.
+ */
+export declare function discardingSink(): SinkWriter;
+/**
+ * The upper end of the scale a judge is asked for. A 0–5 integer band is easier for a model to hold
+ * consistently than a raw 0–1 float, so the prompt asks for that and the parser normalizes.
+ */
+export declare const MAX_JUDGE_SCORE = 5;
+/** A judge reply that could not be read as a verdict. */
+export declare class JudgeVerdictError extends Error {
+    constructor(message: string);
+}
+/**
+ * Read a `SCORE: n` / `REASON: …` reply into a {@link ScoreResult}, normalizing the score to `0..1`.
+ *
+ * A missing or out-of-band score THROWS rather than scoring 0: a judge that did not answer is a
+ * broken evaluation, and recording it as a bad run would put the blame on the agent. The batch
+ * runner catches it as a per-run scorer failure. A missing reason is tolerated — the number is the
+ * verdict, the sentence is the courtesy.
+ */
+export declare function parseJudgeVerdict(text: string): ScoreResult;
+//# sourceMappingURL=judge.d.ts.map

package/dist/src/evals/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../src/evals/judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,sGAAsG;AAEtG;;;;;;GAMG;AACH,wBAAgB,cAAc,IAAI,UAAU,CAK3C;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC,yDAAyD;AACzD,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAO,EAAE,MAAM,EAG1B;CACF;AAKD;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAiB3D"}

package/dist/src/evals/judge.js

@@ -0,0 +1,53 @@
+/** The toolkit a model-graded (LLM-as-judge) scorer needs: a sink to nowhere and a verdict parser. */
+/**
+ * A {@link SinkWriter} that drops every token.
+ *
+ * {@link import('../spi/model-provider.js').ModelProvider.runTurn} streams deltas as it generates,
+ * but a judge's verdict is read from the assembled `text` it returns and nobody is watching it
+ * arrive — a judge call has no run, so there is no live stream to join.
+ */
+export function discardingSink() {
+    return {
+        write: () => { },
+        end: () => { },
+    };
+}
+/**
+ * The upper end of the scale a judge is asked for. A 0–5 integer band is easier for a model to hold
+ * consistently than a raw 0–1 float, so the prompt asks for that and the parser normalizes.
+ */
+export const MAX_JUDGE_SCORE = 5;
+/** A judge reply that could not be read as a verdict. */
+export class JudgeVerdictError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'JudgeVerdictError';
+    }
+}
+const SCORE_PATTERN = /SCORE:\s*(\d+(?:\.\d+)?)/i;
+const REASON_PATTERN = /REASON:\s*([\s\S]+)/i;
+/**
+ * Read a `SCORE: n` / `REASON: …` reply into a {@link ScoreResult}, normalizing the score to `0..1`.
+ *
+ * A missing or out-of-band score THROWS rather than scoring 0: a judge that did not answer is a
+ * broken evaluation, and recording it as a bad run would put the blame on the agent. The batch
+ * runner catches it as a per-run scorer failure. A missing reason is tolerated — the number is the
+ * verdict, the sentence is the courtesy.
+ */
+export function parseJudgeVerdict(text) {
+    const scoreMatch = SCORE_PATTERN.exec(text);
+    const raw = scoreMatch?.[1];
+    if (raw === undefined) {
+        throw new JudgeVerdictError(`judge reply carried no "SCORE:" line: ${text.slice(0, 200)}`);
+    }
+    const rawScore = Number(raw);
+    if (!Number.isFinite(rawScore) || rawScore > MAX_JUDGE_SCORE) {
+        throw new JudgeVerdictError(`judge scored ${raw}, outside the 0-${MAX_JUDGE_SCORE} scale it was asked for`);
+    }
+    return {
+        score: rawScore / MAX_JUDGE_SCORE,
+        reason: REASON_PATTERN.exec(text)?.[1]?.trim() ?? 'the judge gave no reason',
+        metadata: { rawScore, maxScore: MAX_JUDGE_SCORE },
+    };
+}
+//# sourceMappingURL=judge.js.map

package/dist/src/evals/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../../src/evals/judge.ts"],"names":[],"mappings":"AAGA,sGAAsG;AAEtG;;;;;;GAMG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO;QACL,KAAK,EAAE,GAAG,EAAE,GAAE,CAAC;QACf,GAAG,EAAE,GAAG,EAAE,GAAE,CAAC;KACd,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AAEjC,yDAAyD;AACzD,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED,MAAM,aAAa,GAAG,2BAA2B,CAAC;AAClD,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAE9C;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,iBAAiB,CAAC,yCAAyC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,eAAe,EAAE,CAAC;QAC7D,MAAM,IAAI,iBAAiB,CACzB,gBAAgB,GAAG,mBAAmB,eAAe,yBAAyB,CAC/E,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,QAAQ,GAAG,eAAe;QACjC,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,0BAA0B;QAC5E,QAAQ,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE;KAClD,CAAC;AACJ,CAAC"}

package/dist/src/evals/live-scoring.d.ts

@@ -0,0 +1,42 @@
+import type { RunSampleSource } from './sample-source.js';
+import type { ScoreStore } from './score-store.js';
+import type { Scorer } from './types.js';
+export interface LiveScoringOptions {
+    source: RunSampleSource;
+    scorers: Scorer[];
+    store: ScoreStore;
+    /**
+     * Fraction of finished runs to score, `0..1`. Default `1`. Turn it down for a scorer that costs
+     * money per run — a trend needs a representative sample, not every row.
+     */
+    sampleRate?: number;
+    /** Sampler; injectable so a test can pin the decision. */
+    random?: () => number;
+    now?: () => Date;
+    /** Told about anything that went wrong. Default: swallowed — a score is never worth a log storm. */
+    onError?: (error: unknown) => void;
+}
+/** The handle {@link attachLiveScoring} returns. */
+export interface LiveScoring {
+    /** Detach the subscription. In-flight scoring already started still settles. */
+    dispose(): void;
+    /** Resolves once every scoring started so far has settled — what a test awaits. */
+    settled(): Promise<void>;
+}
+/**
+ * Score runs as they finish, by listening to `agora:agent:run.finished`.
+ *
+ * Opt-in: nothing calls this for you. And deliberately NOT inline in the turn — a scorer wired into
+ * the agent loop would add its latency to every message a user sends, and a model-graded one would
+ * double the cost of the product. This runs AFTER the run has already finished and reported so, on a
+ * diagnostics subscriber, in a detached promise. There is no code path from a scorer back into the
+ * turn: the run has settled and its stream has closed before the first scorer is called, and every
+ * failure below is caught and handed to `onError`. A scorer cannot fail a turn because by the time
+ * it runs there is no turn left to fail.
+ *
+ * It still costs real work per run, so the offline {@link import('./evaluate.js').runEvaluation}
+ * batch remains the default path — reach for this when you want a dashboard that moves within the
+ * hour, and prefer the `rule`/`statistical` scorers here.
+ */
+export declare function attachLiveScoring(options: LiveScoringOptions): LiveScoring;
+//# sourceMappingURL=live-scoring.d.ts.map

package/dist/src/evals/live-scoring.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"live-scoring.d.ts","sourceRoot":"","sources":["../../../src/evals/live-scoring.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAY,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AA4BzC,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,eAAe,CAAC;IACxB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,UAAU,CAAC;IAClB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,oGAAoG;IACpG,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CACpC;AAED,oDAAoD;AACpD,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,OAAO,IAAI,IAAI,CAAC;IAChB,mFAAmF;IACnF,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,WAAW,CA8B1E"}

package/dist/src/evals/live-scoring.js

@@ -0,0 +1,103 @@
+import diagnostics_channel from 'node:diagnostics_channel';
+import { toRunScore } from './evaluate.js';
+/**
+ * The channel the agent loop publishes a settled run on. Subscribed over Node's builtin
+ * `node:diagnostics_channel`, exactly as the media-ingestion bridge does — but the loop only PUBLISHES
+ * when `@adonis-agora/diagnostics` is installed to fill the emit slot, so live scoring is silently
+ * inert without it. The offline batch has no such dependency.
+ */
+const RUN_FINISHED_CHANNEL = 'agora:agent:run.finished';
+/** Pull `.payload` off a diagnostics-channel message without an unguarded cast. */
+function envelopePayload(message) {
+    if (typeof message === 'object' && message !== null && 'payload' in message) {
+        return message.payload;
+    }
+    return undefined;
+}
+/** The `runId` a `run.finished` envelope carries, or `undefined` for anything else. */
+function finishedRunId(message) {
+    const payload = envelopePayload(message);
+    if (typeof payload !== 'object' || payload === null || !('runId' in payload)) {
+        return undefined;
+    }
+    const runId = payload.runId;
+    return typeof runId === 'string' ? runId : undefined;
+}
+/**
+ * Score runs as they finish, by listening to `agora:agent:run.finished`.
+ *
+ * Opt-in: nothing calls this for you. And deliberately NOT inline in the turn — a scorer wired into
+ * the agent loop would add its latency to every message a user sends, and a model-graded one would
+ * double the cost of the product. This runs AFTER the run has already finished and reported so, on a
+ * diagnostics subscriber, in a detached promise. There is no code path from a scorer back into the
+ * turn: the run has settled and its stream has closed before the first scorer is called, and every
+ * failure below is caught and handed to `onError`. A scorer cannot fail a turn because by the time
+ * it runs there is no turn left to fail.
+ *
+ * It still costs real work per run, so the offline {@link import('./evaluate.js').runEvaluation}
+ * batch remains the default path — reach for this when you want a dashboard that moves within the
+ * hour, and prefer the `rule`/`statistical` scorers here.
+ */
+export function attachLiveScoring(options) {
+    const now = options.now ?? (() => new Date());
+    const random = options.random ?? Math.random;
+    const sampleRate = options.sampleRate ?? 1;
+    const channel = diagnostics_channel.channel(RUN_FINISHED_CHANNEL);
+    const inFlight = new Set();
+    const onMessage = (message) => {
+        // The whole handler is guarded: this runs on the publisher's stack, so anything that escapes
+        // here surfaces inside whoever emitted the event.
+        try {
+            const runId = finishedRunId(message);
+            if (runId === undefined || (sampleRate < 1 && random() >= sampleRate)) {
+                return;
+            }
+            const task = scoreFinishedRun({ runId, options, now }).catch(options.onError ?? (() => { }));
+            inFlight.add(task);
+            void task.finally(() => inFlight.delete(task));
+        }
+        catch (error) {
+            options.onError?.(error);
+        }
+    };
+    channel.subscribe(onMessage);
+    return {
+        dispose: () => channel.unsubscribe(onMessage),
+        settled: async () => {
+            await Promise.all([...inFlight]);
+        },
+    };
+}
+/** Load the finished run and persist whatever the scorers had to say about it. */
+async function scoreFinishedRun(args) {
+    const { runId, options, now } = args;
+    const run = await options.source.getRun(runId);
+    if (run === null) {
+        return;
+    }
+    const scores = [];
+    for (const scorer of options.scorers) {
+        try {
+            const result = await scorer.score(run);
+            if (result !== null) {
+                scores.push(toRunScore({ run, scorer, result, scoredAt: now() }));
+            }
+        }
+        catch (error) {
+            options.onError?.(error);
+        }
+    }
+    if (scores.length > 0) {
+        await recordQuietly({ store: options.store, scores, onError: options.onError });
+    }
+}
+/** A store that rejects must not take the whole subscriber down with it. */
+async function recordQuietly(args) {
+    try {
+        await args.store.recordScores(args.scores);
+    }
+    catch (error) {
+        args.onError?.(error);
+    }
+}
+//# sourceMappingURL=live-scoring.js.map

package/dist/src/evals/live-scoring.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"live-scoring.js","sourceRoot":"","sources":["../../../src/evals/live-scoring.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAK3C;;;;;GAKG;AACH,MAAM,oBAAoB,GAAG,0BAA0B,CAAC;AAExD,mFAAmF;AACnF,SAAS,eAAe,CAAC,OAAgB;IACvC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;QAC5E,OAAQ,OAAgC,CAAC,OAAO,CAAC;IACnD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,uFAAuF;AACvF,SAAS,aAAa,CAAC,OAAgB;IACrC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC;QAC7E,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,KAAK,GAAI,OAA8B,CAAC,KAAK,CAAC;IACpD,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC;AA0BD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAA2B;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;IAE1C,MAAM,SAAS,GAAG,CAAC,OAAgB,EAAQ,EAAE;QAC3C,6FAA6F;QAC7F,kDAAkD;QAClD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,MAAM,EAAE,IAAI,UAAU,CAAC,EAAE,CAAC;gBACtE,OAAO;YACT,CAAC;YACD,MAAM,IAAI,GAAG,gBAAgB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;YAC5F,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnB,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC;IAEF,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC7B,OAAO;QACL,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC;QAC7C,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;QACnC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,kFAAkF;AAClF,KAAK,UAAU,gBAAgB,CAAC,IAI/B;IACC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACrC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,aAAa,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,4EAA4E;AAC5E,KAAK,UAAU,aAAa,CAAC,IAI5B;IACC,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;AACH,CAAC"}

package/dist/src/evals/sample-source.d.ts

@@ -0,0 +1,40 @@
+import type { RunStatus } from '../spi/governance-queries.js';
+import type { ScorableRun } from './types.js';
+/**
+ * Which stored runs to score. `limit` caps the batch; every other field is an optional filter, and
+ * each one maps to a filter the governance read-model actually has
+ * ({@link import('../spi/governance-queries.js').ListRunsFilter}) — a filter this package applied
+ * after the page was already clamped would return fewer runs than `limit` asked for and look like a
+ * quiet corpus.
+ */
+export interface RunSampleQuery {
+    /** Max runs returned, newest first. */
+    limit: number;
+    /** Inclusive UTC day bounds on the run's start, `YYYY-MM-DD`. */
+    fromDay?: string;
+    toDay?: string;
+    agentName?: string;
+    status?: RunStatus;
+    /** Exact `actorRef` — score one person's runs, or one tenant's service account. */
+    actorRef?: string;
+}
+/**
+ * Where {@link ScorableRun}s come from. The evaluation runner reads through this and nothing else,
+ * so a host can point it at the live agent tables, at a frozen regression set checked into a repo,
+ * or at a hand-written fixture in a test — the scorers never learn which.
+ */
+export interface RunSampleSource {
+    listRuns(query: RunSampleQuery): Promise<ScorableRun[]>;
+    getRun(runId: string): Promise<ScorableRun | null>;
+}
+/**
+ * A {@link RunSampleSource} over a fixed list of runs — the regression-set / fixture source. Also
+ * what a test uses instead of standing up a store.
+ */
+export declare class StaticRunSampleSource implements RunSampleSource {
+    private readonly runs;
+    constructor(runs: ScorableRun[]);
+    listRuns(query: RunSampleQuery): Promise<ScorableRun[]>;
+    getRun(runId: string): Promise<ScorableRun | null>;
+}
+//# sourceMappingURL=sample-source.d.ts.map

package/dist/src/evals/sample-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample-source.d.ts","sourceRoot":"","sources":["../../../src/evals/sample-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,mFAAmF;IACnF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACxD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;CACpD;AAED;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,eAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,IAAI;IAAjC,YAA6B,IAAI,EAAE,WAAW,EAAE,EAAI;IAE9C,QAAQ,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAgB5D;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAEvD;CACF"}

package/dist/src/evals/sample-source.js

@@ -0,0 +1,33 @@
+/**
+ * A {@link RunSampleSource} over a fixed list of runs — the regression-set / fixture source. Also
+ * what a test uses instead of standing up a store.
+ */
+export class StaticRunSampleSource {
+    runs;
+    constructor(runs) {
+        this.runs = runs;
+    }
+    async listRuns(query) {
+        return this.runs
+            .filter((run) => {
+            if (query.agentName !== undefined && run.agentName !== query.agentName)
+                return false;
+            if (query.status !== undefined && run.status !== query.status)
+                return false;
+            if (query.actorRef !== undefined && run.actorRef !== query.actorRef)
+                return false;
+            const day = run.startedAt.slice(0, 10);
+            if (query.fromDay !== undefined && day < query.fromDay)
+                return false;
+            if (query.toDay !== undefined && day > query.toDay)
+                return false;
+            return true;
+        })
+            .sort((left, right) => right.startedAt.localeCompare(left.startedAt) || right.runId.localeCompare(left.runId))
+            .slice(0, query.limit);
+    }
+    async getRun(runId) {
+        return this.runs.find((run) => run.runId === runId) ?? null;
+    }
+}
+//# sourceMappingURL=sample-source.js.map

package/dist/src/evals/sample-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample-source.js","sourceRoot":"","sources":["../../../src/evals/sample-source.ts"],"names":[],"mappings":"AAgCA;;;GAGG;AACH,MAAM,OAAO,qBAAqB;IACH,IAAI;IAAjC,YAA6B,IAAmB;oBAAnB,IAAI;IAAkB,CAAC;IAEpD,KAAK,CAAC,QAAQ,CAAC,KAAqB;QAClC,OAAO,IAAI,CAAC,IAAI;aACb,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YACd,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;gBAAE,OAAO,KAAK,CAAC;YACrF,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAC;YAC5E,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAClF,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvC,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,GAAG,GAAG,KAAK,CAAC,OAAO;gBAAE,OAAO,KAAK,CAAC;YACrE,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,GAAG,KAAK,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAC;YACjE,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;aACD,IAAI,CACH,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CACzF;aACA,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC;IAC9D,CAAC;CACF"}

package/dist/src/evals/score-store.d.ts

@@ -0,0 +1,54 @@
+import type { ScorerKind } from './types.js';
+/**
+ * Persistence for scores, separate from {@link import('../spi/agent-store.js').AgentStore} the same
+ * way the governance read-model is: the agent's own tables are the INPUT to scoring and must not
+ * grow a column every time someone writes a scorer. A host binds whichever adapter it likes;
+ * {@link import('./in-memory-score-store.js').InMemoryScoreStore} covers tests and the offline demo.
+ */
+/** One scorer's verdict on one run, as persisted. */
+export interface RunScore {
+    runId: string;
+    threadId: string;
+    agentName: string | null;
+    /** {@link import('./types.js').Scorer.name}. */
+    scorer: string;
+    kind: ScorerKind;
+    /** `0..1`, 1 is good. */
+    score: number;
+    reason: string;
+    metadata?: Record<string, unknown>;
+    /**
+     * The `YYYY-MM-DD` UTC day the SCORED RUN started — not the day the batch ran. A trend must move
+     * when the agent's quality moves, not when someone re-ran a backfill over last month.
+     */
+    day: string;
+    /** ISO timestamp the score was produced. */
+    scoredAt: string;
+}
+/** Filters for {@link ScoreStore.listScores}; an absent field imposes no constraint. */
+export interface ScoreWhere {
+    scorer?: string;
+    agentName?: string;
+    threadId?: string;
+    runId?: string;
+    /** Inclusive UTC day bounds on {@link RunScore.day}, `YYYY-MM-DD`. */
+    fromDay?: string;
+    toDay?: string;
+}
+export interface ScoreStore {
+    /**
+     * Append scores. Idempotency is the caller's job via {@link ScoreStore.scoredRunIds} — an adapter
+     * that upserts on `(runId, scorer)` and one that appends both satisfy this SPI, so a consumer must
+     * not read "the score" of a run without deciding which it wants.
+     */
+    recordScores(scores: RunScore[]): Promise<void>;
+    /**
+     * Which of `runIds` already carry a score from `scorer`. The resume seam: a backfill interrupted
+     * halfway is restarted by skipping these, so re-running it costs nothing and — for a model-graded
+     * scorer — bills nothing.
+     */
+    scoredRunIds(scorer: string, runIds: string[]): Promise<string[]>;
+    /** Every score matching `where`, newest-scored first. The feed the summaries aggregate. */
+    listScores(where: ScoreWhere): Promise<RunScore[]>;
+}
+//# sourceMappingURL=score-store.d.ts.map

package/dist/src/evals/score-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score-store.d.ts","sourceRoot":"","sources":["../../../src/evals/score-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C;;;;;GAKG;AAEH,qDAAqD;AACrD,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;IACjB,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wFAAwF;AACxF,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD;;;;OAIG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClE,2FAA2F;IAC3F,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CACpD"}

package/dist/src/evals/score-store.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=score-store.js.map

package/dist/src/evals/score-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score-store.js","sourceRoot":"","sources":["../../../src/evals/score-store.ts"],"names":[],"mappings":""}