@adonis-agora/agent 0.30.2 → 0.33.0

package/dist/src/elicitation.js

@@ -0,0 +1,315 @@
+/**
+ * Asking the USER a structured question, and waiting for the answer.
+ *
+ * `awaitApproval` collects a yes/no about work already proposed; this collects the scope BEFORE the
+ * work. Two surfaces produce it — a configured intake (`AgentLoopDeps.intake`) and the model-callable
+ * `ask` tool (`AgentLoopDeps.ask`) — and they deliberately produce the SAME {@link
+ * ElicitationRequest}, persist through the same tool-call row, and resume through the same
+ * `tool:<runId>:<callId>` signal. A consumer cannot tell which one asked, and should not have to.
+ */
+/**
+ * Thrown when a reply is the wrong shape for the wait it was delivered to: answers to a question
+ * set, addressed to a tool call that is waiting on an approve/reject.
+ *
+ * Refused rather than reduced, because there is no honest reduction in that direction. A yes/no can
+ * be read as "confirmed the pre-picked answers" (see {@link normalizeElicitationReply}); a set of
+ * answers says nothing at all about whether the proposed work should go ahead, and reading its
+ * absent `approved` as `false` would record a rejection the person never made.
+ */
+export class HumanReplyMismatchError extends Error {
+    runId;
+    toolCallId;
+    constructor(runId, toolCallId) {
+        super(`Tool call "${toolCallId}" on run "${runId}" is waiting for an approve/reject, not for answers`);
+        this.runId = runId;
+        this.toolCallId = toolCallId;
+        this.name = 'HumanReplyMismatchError';
+    }
+}
+/**
+ * Is this a human's yes/no {@link Decision}, rather than answers to a question set? Structural
+ * rather than by class: both shapes are plain JSON off an HTTP body by the time they arrive, and
+ * `approved` is the only field a `Decision` carries.
+ */
+export function isHumanDecision(reply) {
+    return (typeof reply === 'object' &&
+        reply !== null &&
+        typeof reply.approved === 'boolean');
+}
+function optionValues(question) {
+    return new Set(question.options.map((option) => option.value));
+}
+/**
+ * Read whatever the human channel delivered as an {@link ElicitationReply}.
+ *
+ * A question set is persisted as an `action` tool call in `pending_approval` — that is what puts it
+ * in the approvals inbox a deployment already has, instead of needing one of its own. The cost of
+ * that choice is that the thing which comes back may be a {@link Decision} someone pressed
+ * Approve/Reject on rather than a set of answers, and a `Decision` carries no `answers` at all.
+ *
+ * Approve means every question keeps its own pre-picked `defaults`, which is exactly what "just
+ * submit" already means on this surface; Reject is the same declining-to-answer a skip is. Neither
+ * reading is a guess — a yes/no channel cannot say more than that, and saying it here is what lets
+ * one inbox settle both kinds of pending work.
+ *
+ * Returns the reply UNCHANGED when it already carries answers, so the common path allocates nothing
+ * and a caller can identity-compare.
+ */
+export function normalizeElicitationReply(reply) {
+    if (typeof reply !== 'object' || reply === null) {
+        return { answers: {} };
+    }
+    const candidate = reply;
+    if (typeof candidate.answers === 'object' && candidate.answers !== null) {
+        return reply;
+    }
+    const answeredByRef = candidate.answeredByRef ?? candidate.executedByRef;
+    return {
+        answers: {},
+        ...(candidate.approved === false || candidate.skipped === true ? { skipped: true } : {}),
+        ...(answeredByRef !== undefined ? { answeredByRef } : {}),
+    };
+}
+/**
+ * Settle a reply against the request it answers: fill every unanswered question from its own
+ * `defaults`, drop submitted values that aren't on offer, and collapse a single-choice question to
+ * one value.
+ *
+ * PURE, and deliberately so. Both of its inputs are already journaled by the time the loop calls it
+ * — the request came from module config or from an `llm:<i>` checkpoint, the reply from the signal
+ * checkpoint — so every process replaying the turn reaches the same values without a checkpoint of
+ * its own. Resolving defaults in the HTTP layer instead would put them behind a store read that a
+ * replay would have to repeat.
+ */
+export function resolveElicitation(args) {
+    const { request } = args;
+    const reply = normalizeElicitationReply(args.reply);
+    const answers = {};
+    const defaulted = [];
+    for (const question of request.questions) {
+        const submitted = reply.skipped === true ? undefined : reply.answers[question.id];
+        if (submitted === undefined) {
+            answers[question.id] = [...(question.defaults ?? [])];
+            defaulted.push(question.id);
+            continue;
+        }
+        const allowed = optionValues(question);
+        const valid = question.allowFreeText === true ? submitted : submitted.filter((value) => allowed.has(value));
+        answers[question.id] = question.multiple === true ? valid : valid.slice(0, 1);
+    }
+    return { answers, skipped: reply.skipped === true, defaulted };
+}
+/** {@link resolveElicitation} plus its human-readable rendering. Pure, for the same reason. */
+export function settleElicitation(args) {
+    const outcome = resolveElicitation(args);
+    return { ...outcome, summary: renderElicitationAnswers({ request: args.request, outcome }) };
+}
+/**
+ * The answers as the model reads them: the question's own prompt against the chosen options' LABELS,
+ * not their opaque `value`s — a model shown `{"scope":["b"]}` has been told nothing.
+ */
+export function renderElicitationAnswers(args) {
+    const { request, outcome } = args;
+    const lines = request.questions.map((question) => {
+        const chosen = outcome.answers[question.id] ?? [];
+        const labels = chosen.map((value) => question.options.find((option) => option.value === value)?.label ?? value);
+        return `${question.prompt} → ${labels.length > 0 ? labels.join(', ') : '(no answer)'}`;
+    });
+    const preface = outcome.skipped
+        ? 'The user declined to answer and asked you to proceed on these assumptions:'
+        : 'The user answered:';
+    return `${preface}\n${lines.join('\n')}`;
+}
+/** The reserved tool name the model calls to ask the user something. */
+export const ASK_TOOL_NAME = 'ask';
+/** How many questions one `ask` may carry. A form the user has to scroll is a form they skip. */
+export const MAX_ASK_QUESTIONS = 5;
+function issue(path, message) {
+    return { message, path };
+}
+function parseOption(raw, path, issues) {
+    if (typeof raw !== 'object' || raw === null) {
+        issues.push(issue(path, 'must be an object'));
+        return undefined;
+    }
+    const candidate = raw;
+    if (typeof candidate.value !== 'string' || candidate.value.length === 0) {
+        issues.push(issue([...path, 'value'], 'must be a non-empty string'));
+        return undefined;
+    }
+    if (typeof candidate.label !== 'string' || candidate.label.length === 0) {
+        issues.push(issue([...path, 'label'], 'must be a non-empty string'));
+        return undefined;
+    }
+    return {
+        value: candidate.value,
+        label: candidate.label,
+        ...(typeof candidate.hotkey === 'string' ? { hotkey: candidate.hotkey } : {}),
+    };
+}
+function parseQuestion(raw, path, issues) {
+    if (typeof raw !== 'object' || raw === null) {
+        issues.push(issue(path, 'must be an object'));
+        return undefined;
+    }
+    const candidate = raw;
+    if (typeof candidate.id !== 'string' || candidate.id.length === 0) {
+        issues.push(issue([...path, 'id'], 'must be a non-empty string'));
+        return undefined;
+    }
+    if (typeof candidate.prompt !== 'string' || candidate.prompt.length === 0) {
+        issues.push(issue([...path, 'prompt'], 'must be a non-empty string'));
+        return undefined;
+    }
+    if (!Array.isArray(candidate.options) || candidate.options.length === 0) {
+        issues.push(issue([...path, 'options'], 'must be a non-empty array'));
+        return undefined;
+    }
+    const options = [];
+    for (const [index, rawOption] of candidate.options.entries()) {
+        const option = parseOption(rawOption, [...path, 'options', index], issues);
+        if (option !== undefined) {
+            options.push(option);
+        }
+    }
+    // The one rule that carries the design: a question with no pre-picked answer is a question the
+    // user has to stop and think about, and this surface exists to avoid that.
+    if (!Array.isArray(candidate.defaults) || candidate.defaults.length === 0) {
+        issues.push(issue([...path, 'defaults'], 'must pre-pick at least one option — say what you would choose so the user can just confirm'));
+        return undefined;
+    }
+    const offered = new Set(options.map((option) => option.value));
+    const defaults = candidate.defaults.filter((value) => typeof value === 'string' && offered.has(value));
+    if (defaults.length === 0) {
+        issues.push(issue([...path, 'defaults'], "must name values that appear in this question's options"));
+        return undefined;
+    }
+    return {
+        id: candidate.id,
+        prompt: candidate.prompt,
+        options,
+        defaults,
+        ...(candidate.multiple === true ? { multiple: true } : {}),
+        ...(candidate.allowFreeText === true ? { allowFreeText: true } : {}),
+    };
+}
+const ASK_JSON_SCHEMA = {
+    type: 'object',
+    additionalProperties: false,
+    properties: {
+        preamble: {
+            type: 'string',
+            description: 'One sentence shown above the form, e.g. "Three questions before I start. I have pre-picked what I would choose, so confirming is enough."',
+        },
+        questions: {
+            type: 'array',
+            minItems: 1,
+            maxItems: MAX_ASK_QUESTIONS,
+            items: {
+                type: 'object',
+                additionalProperties: false,
+                required: ['id', 'prompt', 'options', 'defaults'],
+                properties: {
+                    id: {
+                        type: 'string',
+                        description: 'Unique within this call; answers come back under it.',
+                    },
+                    prompt: { type: 'string' },
+                    multiple: { type: 'boolean', description: 'Allow more than one option to be chosen.' },
+                    allowFreeText: {
+                        type: 'boolean',
+                        description: 'Accept an answer that is not an option.',
+                    },
+                    defaults: {
+                        type: 'array',
+                        minItems: 1,
+                        items: { type: 'string' },
+                        description: 'REQUIRED. The option values you would pick yourself, so the user can confirm rather than decide.',
+                    },
+                    options: {
+                        type: 'array',
+                        minItems: 2,
+                        items: {
+                            type: 'object',
+                            additionalProperties: false,
+                            required: ['value', 'label'],
+                            properties: {
+                                value: { type: 'string' },
+                                label: { type: 'string' },
+                                hotkey: { type: 'string', maxLength: 1 },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+    required: ['questions'],
+};
+/**
+ * The `ask` tool's input schema, hand-written rather than borrowed from a validation library: core
+ * depends on no validator, and the schema has to carry a JSON Schema a provider can constrain
+ * generation against. It publishes one through the Standard JSON Schema extension
+ * (`~standard.jsonSchema.input`), which is the path the AI SDK adapter already recognises for
+ * Valibot / ArkType / Zod 4.
+ */
+export const askInputSchema = {
+    '~standard': {
+        version: 1,
+        vendor: 'adonis-agora-agent',
+        validate: (value) => {
+            const issues = [];
+            if (typeof value !== 'object' || value === null) {
+                return { issues: [issue([], 'must be an object')] };
+            }
+            const candidate = value;
+            if (!Array.isArray(candidate.questions) || candidate.questions.length === 0) {
+                return { issues: [issue(['questions'], 'must be a non-empty array')] };
+            }
+            if (candidate.questions.length > MAX_ASK_QUESTIONS) {
+                return {
+                    issues: [issue(['questions'], `must hold at most ${MAX_ASK_QUESTIONS} questions`)],
+                };
+            }
+            const questions = [];
+            for (const [index, raw] of candidate.questions.entries()) {
+                const question = parseQuestion(raw, ['questions', index], issues);
+                if (question !== undefined) {
+                    questions.push(question);
+                }
+            }
+            if (issues.length > 0) {
+                return { issues };
+            }
+            return {
+                value: {
+                    questions,
+                    ...(typeof candidate.preamble === 'string' ? { preamble: candidate.preamble } : {}),
+                },
+            };
+        },
+        jsonSchema: { input: () => ASK_JSON_SCHEMA },
+    },
+};
+/**
+ * What the model is told the `ask` tool is for. Written to discourage the two failure modes that
+ * make a clarifying question worse than a guess: asking about something the conversation already
+ * settled, and asking without saying what you would have done.
+ */
+export const ASK_TOOL_DESCRIPTION = 'Ask the user to settle the scope of the work before you do it. Use it when a reasonable person would produce a materially different result depending on the answer — not to confirm something the conversation already says. Every question must pre-pick the answer you would choose, so the user can confirm instead of deciding. The user may decline, in which case you proceed on those pre-picked answers.';
+/**
+ * The `ask` tool as the model sees it. NOT a `ToolSpec` and never registered: `ask` has no handler,
+ * because the loop settles it against a human instead of invoking anything. Keeping it out of the
+ * `ToolRegistry` is also what keeps the kind decision off a process-local lookup — see
+ * `claimToolCall`.
+ */
+export function askToolDefinition() {
+    return {
+        name: ASK_TOOL_NAME,
+        kind: 'ask',
+        description: ASK_TOOL_DESCRIPTION,
+        inputSchema: askInputSchema,
+    };
+}
+export const DEFAULT_INTAKE_PREAMBLE = 'A few questions before I start. I have pre-picked what I would choose, so confirming is enough.';
+//# sourceMappingURL=elicitation.js.map

package/dist/src/elicitation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"elicitation.js","sourceRoot":"","sources":["../../src/elicitation.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAaH;;;;;;;;GAQG;AACH,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAE9B,KAAK;IACL,UAAU;IAF5B,YACkB,KAAa,EACb,UAAkB;QAElC,KAAK,CACH,cAAc,UAAU,aAAa,KAAK,qDAAqD,CAChG,CAAC;qBALc,KAAK;0BACL,UAAU;QAK1B,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,QAAQ,KAAK,SAAS,CAC3D,CAAC;AACJ,CAAC;AA6ED,SAAS,YAAY,CAAC,QAA6B;IACjD,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,yBAAyB,CAAC,KAAiB;IACzD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACzB,CAAC;IACD,MAAM,SAAS,GAAG,KAAsD,CAAC;IACzE,IAAI,OAAO,SAAS,CAAC,OAAO,KAAK,QAAQ,IAAI,SAAS,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACxE,OAAO,KAAyB,CAAC;IACnC,CAAC;IACD,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,aAAa,CAAC;IACzE,OAAO;QACL,OAAO,EAAE,EAAE;QACX,GAAG,CAAC,SAAS,CAAC,QAAQ,KAAK,KAAK,IAAI,SAAS,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAGlC;IACC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACzB,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAClF,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,KAAK,GACT,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAChG,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,SAAS,EAAE,CAAC;AACjE,CAAC;AAYD,+FAA+F;AAC/F,MAAM,UAAU,iBAAiB,CAAC,IAGjC;IACC,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,wBAAwB,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAGxC;IACC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAClC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CACvB,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,EAAE,KAAK,IAAI,KAAK,CACrF,CAAC;QACF,OAAO,GAAG,QAAQ,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;QAC7B,CAAC,CAAC,4EAA4E;QAC9E,CAAC,CAAC,oBAAoB,CAAC;IACzB,OAAO,GAAG,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,CAAC;AAQnC,iGAAiG;AACjG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC,SAAS,KAAK,CAAC,IAAyB,EAAE,OAAe;IACvD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,WAAW,CAClB,GAAY,EACZ,IAAyB,EACzB,MAAgC;IAEhC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC9C,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,SAAS,GAAG,GAAiC,CAAC;IACpD,IAAI,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,EAAE,4BAA4B,CAAC,CAAC,CAAC;QACrE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,EAAE,4BAA4B,CAAC,CAAC,CAAC;QACrE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO;QACL,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,GAAY,EACZ,IAAyB,EACzB,MAAgC;IAEhC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC9C,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,SAAS,GAAG,GAAmC,CAAC;IACtD,IAAI,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,IAAI,SAAS,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,EAAE,4BAA4B,CAAC,CAAC,CAAC;QAClE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC,EAAE,4BAA4B,CAAC,CAAC,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,SAAS,CAAC,EAAE,2BAA2B,CAAC,CAAC,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAwB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC,GAAG,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;QAC3E,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,+FAA+F;IAC/F,2EAA2E;IAC3E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,MAAM,CAAC,IAAI,CACT,KAAK,CACH,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,EACrB,4FAA4F,CAC7F,CACF,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,CACxC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAC5E,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CACT,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,EAAE,yDAAyD,CAAC,CACxF,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,OAAO;QACP,QAAQ;QACR,GAAG,CAAC,SAAS,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,SAAS,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,IAAI,EAAE,QAAQ;IACd,oBAAoB,EAAE,KAAK;IAC3B,UAAU,EAAE;QACV,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,2IAA2I;SAC9I;QACD,SAAS,EAAE;YACT,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,oBAAoB,EAAE,KAAK;gBAC3B,QAAQ,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC;gBACjD,UAAU,EAAE;oBACV,EAAE,EAAE;wBACF,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sDAAsD;qBACpE;oBACD,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBAC1B,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,0CAA0C,EAAE;oBACtF,aAAa,EAAE;wBACb,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,yCAAyC;qBACvD;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,OAAO;wBACb,QAAQ,EAAE,CAAC;wBACX,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;wBACzB,WAAW,EACT,kGAAkG;qBACrG;oBACD,OAAO,EAAE;wBACP,IAAI,EAAE,OAAO;wBACb,QAAQ,EAAE,CAAC;wBACX,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,oBAAoB,EAAE,KAAK;4BAC3B,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC;4BAC5B,UAAU,EAAE;gCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE;6BACzC;yBACF;qBACF;iBACF;aACF;SACF;KACF;IACD,QAAQ,EAAE,CAAC,WAAW,CAAC;CACf,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,cAAc,GAA4C;IACrE,WAAW,EAAE;QACX,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,oBAAoB;QAC5B,QAAQ,EAAE,CAAC,KAAc,EAAE,EAAE;YAC3B,MAAM,MAAM,GAA6B,EAAE,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,OAAO,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC,EAAE,CAAC;YACtD,CAAC;YACD,MAAM,SAAS,GAAG,KAA8B,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5E,OAAO,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,EAAE,2BAA2B,CAAC,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,IAAI,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACnD,OAAO;oBACL,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,EAAE,qBAAqB,iBAAiB,YAAY,CAAC,CAAC;iBACnF,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAA0B,EAAE,CAAC;YAC5C,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;gBACzD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;gBAClE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,OAAO,EAAE,MAAM,EAAE,CAAC;YACpB,CAAC;YACD,OAAO;gBACL,KAAK,EAAE;oBACL,SAAS;oBACT,GAAG,CAAC,OAAO,SAAS,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACpF;aACF,CAAC;QACJ,CAAC;QACD,UAAU,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE;KAC7C;CACyC,CAAC;AAE7C;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAC/B,kZAAkZ,CAAC;AAErZ;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO;QACL,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,KAAK;QACX,WAAW,EAAE,oBAAoB;QACjC,WAAW,EAAE,cAAc;KAC5B,CAAC;AACJ,CAAC;AAgBD,MAAM,CAAC,MAAM,uBAAuB,GAClC,iGAAiG,CAAC"}

package/dist/src/evals/approval-prior.d.ts

@@ -0,0 +1,58 @@
+import type { AgentGovernanceQueries } from '../spi/governance-queries.js';
+import type { ScorableRun } from './types.js';
+/**
+ * The corpus behind {@link import('./scorers/approval-risk.scorer.js').ApprovalRiskScorer}: how
+ * often humans have approved each `action` tool, across the HITL decisions in view.
+ */
+/** One recorded human decision on an `action` tool call. */
+export interface ToolApprovalDecision {
+    toolName: string;
+    /** The tool call's final status, as the store recorded it. */
+    status: string;
+}
+/** How a single tool has fared with humans. */
+export interface ToolApprovalCounts {
+    approved: number;
+    rejected: number;
+}
+/** Approval counts per tool name. */
+export type ToolApprovalPrior = ReadonlyMap<string, ToolApprovalCounts>;
+/** Fold recorded decisions into per-tool approval counts. Undecided statuses are ignored. */
+export declare function buildApprovalPrior(decisions: Iterable<ToolApprovalDecision>): Map<string, ToolApprovalCounts>;
+/**
+ * The prior over the runs a batch already loaded — the cheapest way to get one, and free of extra
+ * reads. {@link import('./governance-sample-source.js').GovernanceRunSampleSource} returns each run
+ * with its tool calls already attached, so the whole sample's HITL history is in hand before the
+ * first scorer runs. It is a prior over the corpus being scored, which is the honest reading of a
+ * batch that also IS the corpus; reach for {@link loadApprovalPrior} when the prior should come from
+ * a wider slice of history than the batch covers.
+ */
+export declare function priorFromRuns(runs: Iterable<ScorableRun>): Map<string, ToolApprovalCounts>;
+/** How much decision history {@link loadApprovalPrior} reads. */
+export interface ApprovalPriorQuery {
+    /**
+     * How many of the most recent tool calls to read. The read-model's activity feed is not filtered
+     * by tool type, so this bounds calls of EVERY kind and a deployment whose traffic is mostly reads
+     * needs a generous number to see its actions. Hard cap on the work, by design: building a prior
+     * must never turn into an unbounded scan.
+     */
+    limit?: number;
+    /** Inclusive UTC day bounds on when the call was made, `YYYY-MM-DD`. */
+    fromDay?: string;
+    toDay?: string;
+}
+/**
+ * Build the prior from the store's own HITL history, off the governance activity feed. Offline by
+ * construction: it reads decisions humans already made, so a batch can score a whole backlog against
+ * the taste of the people who have been running the system.
+ */
+export declare function loadApprovalPrior(args: {
+    queries: AgentGovernanceQueries;
+    query?: ApprovalPriorQuery;
+}): Promise<Map<string, ToolApprovalCounts>>;
+/**
+ * The smoothed probability that a human approves this tool the next time it is proposed. `0.5` for a
+ * tool nobody has ever decided on, which is what "we have no evidence" should look like.
+ */
+export declare function approvalPosterior(counts: ToolApprovalCounts | undefined): number;
+//# sourceMappingURL=approval-prior.d.ts.map

package/dist/src/evals/approval-prior.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"approval-prior.d.ts","sourceRoot":"","sources":["../../../src/evals/approval-prior.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;GAGG;AAEH,4DAA4D;AAC5D,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;AAexE,6FAA6F;AAC7F,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GACxC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAgBjC;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAU1F;AAED,iEAAiE;AACjE,MAAM,WAAW,kBAAkB;IACjC;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE;IAC5C,OAAO,EAAE,sBAAsB,CAAC;IAChC,KAAK,CAAC,EAAE,kBAAkB,CAAC;CAC5B,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAc3C;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,SAAS,GAAG,MAAM,CAIhF"}

package/dist/src/evals/approval-prior.js

@@ -0,0 +1,82 @@
+/** See the approval-outcome scorer — an approved action can still end `failed`. */
+const APPROVED_STATUSES = new Set(['executed', 'failed']);
+/**
+ * Pseudo-counts of a Beta(1,1) prior — one imagined approval and one imagined rejection per tool.
+ *
+ * Without it a tool rejected once out of one call scores 0.0 and a tool approved once out of one
+ * scores 1.0, and the whole ranking is decided by whichever tool happened to be used first. With it,
+ * an unseen tool sits at exactly 0.5 ("no evidence either way") and evidence moves it as it
+ * accumulates — 1-of-1 rejected reads 0.33, 1-of-6 reads 0.25, 0-of-50 reads 0.02.
+ */
+const PRIOR_PSEUDO_COUNT = 1;
+/** Fold recorded decisions into per-tool approval counts. Undecided statuses are ignored. */
+export function buildApprovalPrior(decisions) {
+    const prior = new Map();
+    for (const decision of decisions) {
+        const approved = APPROVED_STATUSES.has(decision.status);
+        if (!approved && decision.status !== 'rejected') {
+            continue;
+        }
+        const counts = prior.get(decision.toolName) ?? { approved: 0, rejected: 0 };
+        if (approved) {
+            counts.approved += 1;
+        }
+        else {
+            counts.rejected += 1;
+        }
+        prior.set(decision.toolName, counts);
+    }
+    return prior;
+}
+/**
+ * The prior over the runs a batch already loaded — the cheapest way to get one, and free of extra
+ * reads. {@link import('./governance-sample-source.js').GovernanceRunSampleSource} returns each run
+ * with its tool calls already attached, so the whole sample's HITL history is in hand before the
+ * first scorer runs. It is a prior over the corpus being scored, which is the honest reading of a
+ * batch that also IS the corpus; reach for {@link loadApprovalPrior} when the prior should come from
+ * a wider slice of history than the batch covers.
+ */
+export function priorFromRuns(runs) {
+    const decisions = [];
+    for (const run of runs) {
+        for (const call of run.toolCalls) {
+            if (call.toolType === 'action') {
+                decisions.push({ toolName: call.toolName, status: call.status });
+            }
+        }
+    }
+    return buildApprovalPrior(decisions);
+}
+const DEFAULT_LIMIT = 2_000;
+/**
+ * Build the prior from the store's own HITL history, off the governance activity feed. Offline by
+ * construction: it reads decisions humans already made, so a batch can score a whole backlog against
+ * the taste of the people who have been running the system.
+ */
+export async function loadApprovalPrior(args) {
+    const { queries, query = {} } = args;
+    const rows = await queries.recentToolCalls(query.limit ?? DEFAULT_LIMIT);
+    const decisions = [];
+    for (const row of rows) {
+        if (row.toolType !== 'action') {
+            continue;
+        }
+        const day = row.createdAt.slice(0, 10);
+        if (query.fromDay !== undefined && day < query.fromDay)
+            continue;
+        if (query.toDay !== undefined && day > query.toDay)
+            continue;
+        decisions.push({ toolName: row.toolName, status: row.status });
+    }
+    return buildApprovalPrior(decisions);
+}
+/**
+ * The smoothed probability that a human approves this tool the next time it is proposed. `0.5` for a
+ * tool nobody has ever decided on, which is what "we have no evidence" should look like.
+ */
+export function approvalPosterior(counts) {
+    const approved = counts?.approved ?? 0;
+    const rejected = counts?.rejected ?? 0;
+    return (approved + PRIOR_PSEUDO_COUNT) / (approved + rejected + PRIOR_PSEUDO_COUNT * 2);
+}
+//# sourceMappingURL=approval-prior.js.map

package/dist/src/evals/approval-prior.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"approval-prior.js","sourceRoot":"","sources":["../../../src/evals/approval-prior.ts"],"names":[],"mappings":"AAwBA,mFAAmF;AACnF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D;;;;;;;GAOG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,6FAA6F;AAC7F,MAAM,UAAU,kBAAkB,CAChC,SAAyC;IAEzC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA8B,CAAC;IACpD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAChD,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC5E,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,IAA2B;IACvD,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,kBAAkB,CAAC,SAAS,CAAC,CAAC;AACvC,CAAC;AAgBD,MAAM,aAAa,GAAG,KAAK,CAAC;AAE5B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAGvC;IACC,MAAM,EAAE,OAAO,EAAE,KAAK,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,KAAK,IAAI,aAAa,CAAC,CAAC;IACzE,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,GAAG,GAAG,KAAK,CAAC,OAAO;YAAE,SAAS;QACjE,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,GAAG,KAAK,CAAC,KAAK;YAAE,SAAS;QAC7D,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,kBAAkB,CAAC,SAAS,CAAC,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAsC;IACtE,MAAM,QAAQ,GAAG,MAAM,EAAE,QAAQ,IAAI,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,MAAM,EAAE,QAAQ,IAAI,CAAC,CAAC;IACvC,OAAO,CAAC,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,GAAG,QAAQ,GAAG,kBAAkB,GAAG,CAAC,CAAC,CAAC;AAC1F,CAAC"}

package/dist/src/evals/evaluate.d.ts

@@ -0,0 +1,51 @@
+import type { RunSampleQuery, RunSampleSource } from './sample-source.js';
+import type { RunScore, ScoreStore } from './score-store.js';
+import type { ScorableRun, ScoreResult, Scorer } from './types.js';
+/** One scorer blowing up on one run. Collected, never thrown — see {@link runEvaluation}. */
+export interface ScorerFailure {
+    runId: string;
+    scorer: string;
+    message: string;
+}
+/** What a batch did. `runsRead - runsScored` is the runs no scorer had anything to say about. */
+export interface EvaluationSummary {
+    /** Runs the source returned for the query. */
+    runsRead: number;
+    /** Distinct runs that produced at least one score. */
+    runsScored: number;
+    /** `(run, scorer)` pairs skipped because that scorer had already covered that run. */
+    pairsSkipped: number;
+    scoresRecorded: number;
+    failures: ScorerFailure[];
+}
+export interface EvaluationOptions {
+    source: RunSampleSource;
+    scorers: Scorer[];
+    store: ScoreStore;
+    query: RunSampleQuery;
+    /**
+     * Score runs a scorer has already covered. Default `false`, which makes a batch resumable: an
+     * interrupted backfill restarted with the same query costs nothing for the part it finished — and
+     * for a model-graded scorer, bills nothing.
+     */
+    rescore?: boolean;
+    /** Clock for {@link RunScore.scoredAt}; injectable so a test can pin it. */
+    now?: () => Date;
+}
+/**
+ * Score a batch of stored runs and persist the results — the offline path, and the one to reach for.
+ *
+ * A scorer that throws is recorded as a {@link ScorerFailure} and the batch carries on. One
+ * malformed run, one judge that returned prose instead of a verdict, one adapter row a scorer did
+ * not expect: none of those should cost you the other 499 runs of a backfill, and a failure that
+ * silently became a score of 0 would be worse still — it would read as the agent's fault.
+ */
+export declare function runEvaluation(options: EvaluationOptions): Promise<EvaluationSummary>;
+/** Assemble the persisted row: the scorer's verdict plus the provenance a summary groups by. */
+export declare function toRunScore(args: {
+    run: ScorableRun;
+    scorer: Scorer;
+    result: ScoreResult;
+    scoredAt: Date;
+}): RunScore;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/src/evals/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../../src/evals/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEnE,6FAA6F;AAC7F,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iGAAiG;AACjG,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,eAAe,CAAC;IACxB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,UAAU,CAAC;IAClB,KAAK,EAAE,cAAc,CAAC;IACtB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,4EAA4E;IAC5E,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAuD1F;AAED,gGAAgG;AAChG,wBAAgB,UAAU,CAAC,IAAI,EAAE;IAC/B,GAAG,EAAE,WAAW,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,IAAI,CAAC;CAChB,GAAG,QAAQ,CAeX"}

package/dist/src/evals/evaluate.js

@@ -0,0 +1,76 @@
+/**
+ * Score a batch of stored runs and persist the results — the offline path, and the one to reach for.
+ *
+ * A scorer that throws is recorded as a {@link ScorerFailure} and the batch carries on. One
+ * malformed run, one judge that returned prose instead of a verdict, one adapter row a scorer did
+ * not expect: none of those should cost you the other 499 runs of a backfill, and a failure that
+ * silently became a score of 0 would be worse still — it would read as the agent's fault.
+ */
+export async function runEvaluation(options) {
+    const now = options.now ?? (() => new Date());
+    const runs = await options.source.listRuns(options.query);
+    const runIds = runs.map((run) => run.runId);
+    const alreadyScored = new Map();
+    if (options.rescore !== true) {
+        for (const scorer of options.scorers) {
+            alreadyScored.set(scorer.name, new Set(await options.store.scoredRunIds(scorer.name, runIds)));
+        }
+    }
+    const scores = [];
+    const failures = [];
+    const scoredRuns = new Set();
+    let pairsSkipped = 0;
+    for (const run of runs) {
+        for (const scorer of options.scorers) {
+            if (alreadyScored.get(scorer.name)?.has(run.runId) === true) {
+                pairsSkipped += 1;
+                continue;
+            }
+            let result;
+            try {
+                result = await scorer.score(run);
+            }
+            catch (error) {
+                failures.push({
+                    runId: run.runId,
+                    scorer: scorer.name,
+                    message: error instanceof Error ? error.message : String(error),
+                });
+                continue;
+            }
+            if (result === null) {
+                continue;
+            }
+            scores.push(toRunScore({ run, scorer, result, scoredAt: now() }));
+            scoredRuns.add(run.runId);
+        }
+    }
+    if (scores.length > 0) {
+        await options.store.recordScores(scores);
+    }
+    return {
+        runsRead: runs.length,
+        runsScored: scoredRuns.size,
+        pairsSkipped,
+        scoresRecorded: scores.length,
+        failures,
+    };
+}
+/** Assemble the persisted row: the scorer's verdict plus the provenance a summary groups by. */
+export function toRunScore(args) {
+    const { run, scorer, result, scoredAt } = args;
+    return {
+        runId: run.runId,
+        threadId: run.threadId,
+        agentName: run.agentName,
+        scorer: scorer.name,
+        kind: scorer.kind,
+        score: result.score,
+        reason: result.reason,
+        ...(result.metadata !== undefined ? { metadata: result.metadata } : {}),
+        // The RUN's day, not the batch's — see RunScore.day.
+        day: run.startedAt.slice(0, 10),
+        scoredAt: scoredAt.toISOString(),
+    };
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/src/evals/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../../src/evals/evaluate.ts"],"names":[],"mappings":"AAsCA;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA0B;IAC5D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAE5C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IACrD,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,aAAa,CAAC,GAAG,CACf,MAAM,CAAC,IAAI,EACX,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5D,YAAY,IAAI,CAAC,CAAC;gBAClB,SAAS;YACX,CAAC;YACD,IAAI,MAA0B,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,MAAM,CAAC,IAAI;oBACnB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAChE,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;YAClE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,UAAU,EAAE,UAAU,CAAC,IAAI;QAC3B,YAAY;QACZ,cAAc,EAAE,MAAM,CAAC,MAAM;QAC7B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,UAAU,CAAC,IAK1B;IACC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC/C,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,qDAAqD;QACrD,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAC/B,QAAQ,EAAE,QAAQ,CAAC,WAAW,EAAE;KACjC,CAAC;AACJ,CAAC"}

package/dist/src/evals/governance-sample-source.d.ts

@@ -0,0 +1,23 @@
+import type { AgentGovernanceQueries } from '../spi/governance-queries.js';
+import type { RunSampleQuery, RunSampleSource } from './sample-source.js';
+import type { ScorableRun } from './types.js';
+/**
+ * A {@link RunSampleSource} over what the agent already persisted, read through
+ * {@link AgentGovernanceQueries} alone. Works against any adapter that implements it (Lucid, the
+ * in-memory one) — this module adds no tables of its own on the read side.
+ *
+ * The prompt and the answer come off `runDetail`, whose messages are the ones stamped with this
+ * run's own `run_id`. There is no transcript join and no time-window heuristic: a run that wrote no
+ * message has empty text, and no other run's answer can ever be handed to it.
+ *
+ * One seam remains, and it is the read-model's rather than this class's: a run whose messages were
+ * written before message-level run stamping reads as empty text. A rule-based scorer over tool
+ * outcomes still works on it; a judge over the answer has nothing to read.
+ */
+export declare class GovernanceRunSampleSource implements RunSampleSource {
+    private readonly queries;
+    constructor(queries: AgentGovernanceQueries);
+    listRuns(query: RunSampleQuery): Promise<ScorableRun[]>;
+    getRun(runId: string): Promise<ScorableRun | null>;
+}
+//# sourceMappingURL=governance-sample-source.d.ts.map

package/dist/src/evals/governance-sample-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"governance-sample-source.d.ts","sourceRoot":"","sources":["../../../src/evals/governance-sample-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,sBAAsB,EAGvB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAoB,MAAM,YAAY,CAAC;AAEhE;;;;;;;;;;;;GAYG;AACH,qBAAa,yBAA0B,YAAW,eAAe;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO;IAApC,YAA6B,OAAO,EAAE,sBAAsB,EAAI;IAE1D,QAAQ,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CA8B5D;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAGvD;CACF"}