@kindgi/api 0.1.2 → 0.1.4-rc.0

package/dist/judged-dispatcher.js

@@ -0,0 +1,297 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+import { itemChanges, matchJudged, outputItems, scoreItems, } from './judged-items.js';
+export const DEFAULT_COMPARISON = {
+    baseline: 'recorded',
+    reads: 'recorded',
+    repetitions: 1,
+    k: 10,
+};
+/** Cases read per page. */
+const CASE_PAGE = 100;
+function validateComparison(suite, target, comparison) {
+    if (target.version === undefined) {
+        return {
+            kind: 'err',
+            message: 'agentId' in target
+                ? '`agentRef.version` is required: the candidate is one version of the agent.'
+                : '`flowRef.version` is required: the candidate is one version of the flow.',
+        };
+    }
+    if (suite.spec.caseCount === 0) {
+        return { kind: 'err', message: 'The test set has no cases.' };
+    }
+    const c = comparison ?? DEFAULT_COMPARISON;
+    if (c.baseline !== 'recorded') {
+        return { kind: 'err', message: "Only `baseline: 'recorded'` runs today." };
+    }
+    return { kind: 'ok' };
+}
+export function createJudgedDispatcher(options) {
+    return {
+        kind: 'judged',
+        validate: validateComparison,
+        async dispatch(ctx) {
+            const comparison = ctx.comparison ?? DEFAULT_COMPARISON;
+            const all = await allCases(options.cases, ctx);
+            if (ctx.dryRun) {
+                return { result: { dryRun: true, cases: all.length, comparison } };
+            }
+            const results = [];
+            const models = new Map();
+            for (const judgedCase of all) {
+                if (ctx.abortSignal.aborted)
+                    break;
+                const result = await runCase(ctx, comparison, judgedCase, models);
+                results.push(result);
+                ctx.onProgress(result);
+            }
+            const summary = summarize(ctx, comparison, all, results, [...models.values()]);
+            return {
+                result: { summary, perCase: results },
+                ...(ctx.abortSignal.aborted && { error: 'cancelled' }),
+            };
+        },
+    };
+}
+async function allCases(store, ctx) {
+    const out = [];
+    let cursor;
+    for (;;) {
+        const page = await store.listCases({
+            tenantId: ctx.tenantId,
+            suiteId: ctx.suite.id,
+            version: ctx.suite.version,
+            limit: CASE_PAGE,
+            ...(cursor !== undefined && { cursor }),
+        });
+        out.push(...page.data);
+        if (!page.hasMore || page.nextCursor === undefined)
+            return out;
+        cursor = page.nextCursor;
+    }
+}
+/** One case's repetitions as they come back. */
+class CaseTally {
+    judgedCase;
+    comparison;
+    agentTurn;
+    runIds = [];
+    candidate = [];
+    first;
+    error;
+    stopped;
+    diverged = false;
+    refusedWrites = 0;
+    approvalSkipped = false;
+    constructor(judgedCase, comparison, agentTurn) {
+        this.judgedCase = judgedCase;
+        this.comparison = comparison;
+        this.agentTurn = agentTurn;
+    }
+    add(outcome) {
+        if (outcome.runId !== undefined)
+            this.runIds.push(outcome.runId);
+        if (outcome.stopped !== undefined) {
+            this.stopped ??= outcome.stopped;
+            this.refusedWrites = Math.max(this.refusedWrites, refusedCount(outcome));
+            this.first ??= { changes: { kept: [], dropped: [], new: [] }, ...replayOf(outcome) };
+            return 'stopped';
+        }
+        if (outcome.error !== undefined) {
+            this.error ??= outcome.error;
+            return 'error';
+        }
+        this.ran(outcome);
+        return 'ran';
+    }
+    ran(outcome) {
+        const { judgedCase, comparison } = this;
+        const matched = matchJudged(outputItems(outcome.output, this.agentTurn), judgedCase.items, judgedCase.output);
+        this.candidate.push(scoreItems(matched, comparison.k));
+        const tools = outcome.replay?.tools ?? [];
+        this.refusedWrites = Math.max(this.refusedWrites, refusedCount(outcome));
+        if (comparison.reads === 'recorded' && tools.some((t) => t.source === 'live')) {
+            this.diverged = true;
+        }
+        if (outcome.replay?.approval === 'skipped')
+            this.approvalSkipped = true;
+        this.first ??= { changes: itemChanges(matched, judgedCase.items), ...replayOf(outcome) };
+    }
+    result(baseline) {
+        const none = this.candidate.length === 0;
+        return {
+            caseId: this.judgedCase.caseId,
+            runIds: this.runIds,
+            baseline,
+            candidate: this.candidate,
+            ...(this.first !== undefined && { changes: this.first.changes }),
+            ...(this.first?.replay !== undefined && { tools: this.first.replay.tools }),
+            diverged: this.diverged,
+            refusedWrites: this.refusedWrites,
+            noContext: this.judgedCase.context === undefined,
+            approvalSkipped: this.approvalSkipped,
+            // A case none of whose repetitions ran stopped (at a refused write) or errored.
+            ...(none && this.stopped !== undefined && { stopped: this.stopped }),
+            ...(none && this.stopped === undefined && this.error !== undefined && { error: this.error }),
+        };
+    }
+}
+async function runCase(ctx, comparison, judgedCase, models) {
+    // An agent turn's items are its answer and typed result; a flow run's, its whole output.
+    const agentTurn = judgedCase.subject.kind === 'agent';
+    const baseline = scoreItems(matchJudged(outputItems(judgedCase.output, agentTurn), judgedCase.items, judgedCase.output), comparison.k);
+    const tally = new CaseTally(judgedCase, comparison, agentTurn);
+    for (let rep = 0; rep < comparison.repetitions; rep++) {
+        const outcome = await invokeCase(ctx, judgedCase);
+        if (tally.add(outcome) === 'ran')
+            countModel(models, outcome);
+    }
+    return tally.result(baseline);
+}
+function refusedCount(outcome) {
+    return (outcome.replay?.tools ?? []).filter((t) => t.source === 'refused').length;
+}
+function replayOf(outcome) {
+    return outcome.replay !== undefined ? { replay: outcome.replay } : {};
+}
+async function invokeCase(ctx, judgedCase) {
+    try {
+        return await ctx.subject.invoke({
+            tenantId: ctx.tenantId,
+            target: ctx.target,
+            // An agent turn's input is its user message.
+            input: judgedCase.input,
+            dryRun: false,
+            abortSignal: ctx.abortSignal,
+            ...(ctx.projectId !== undefined && { projectId: ctx.projectId }),
+            replay: {
+                of: judgedCase.caseId,
+                evalRunId: ctx.runId,
+            },
+            ...(judgedCase.subject.kind === 'agent' && { history: judgedCase.context?.history ?? [] }),
+        });
+    }
+    catch (cause) {
+        return { error: cause instanceof Error ? cause.message : String(cause) };
+    }
+}
+function countModel(models, outcome) {
+    if (outcome.provider === undefined)
+        return;
+    const key = `${outcome.provider.id}\u0000${outcome.provider.model}`;
+    const kept = models.get(key) ?? {
+        providerId: outcome.provider.id,
+        model: outcome.provider.model,
+        runs: 0,
+    };
+    kept.runs += 1;
+    models.set(key, kept);
+}
+const yesShare = (s) => ({ yes: s.yesWeight, total: s.totalWeight });
+const precisionAtK = (s) => ({ yes: s.topK.yesWeight, total: s.topK.totalWeight });
+/** Coverage's ratio is judged items over items; its evidence weight is the judged weight. */
+const coverage = (s) => ({ yes: s.judgedItems, total: s.items });
+/** A metric over cases: Σ numerator / Σ denominator, with its evidence. */
+function pooled(scores, side, weightOf) {
+    let yes = 0;
+    let total = 0;
+    let n = 0;
+    let weight = 0;
+    for (const s of scores) {
+        const { yes: y, total: t } = side(s);
+        if (t <= 0)
+            continue;
+        yes += y;
+        total += t;
+        n += 1;
+        weight += weightOf(s);
+    }
+    return { value: total > 0 ? yes / total : null, n, weight };
+}
+function metric(results, repetitions, side, weightOf, extra = {}) {
+    // Only cases the candidate ran: errored and stopped cases leave both sides.
+    const scored = results.filter((r) => r.candidate.length > 0);
+    const baseline = pooled(scored.map((r) => r.baseline), side, weightOf);
+    // Each repetition is a full pass over the cases: its value, then the spread across them.
+    const perRep = Array.from({ length: repetitions }, (_, rep) => pooled(scored.flatMap((r) => (r.candidate[rep] !== undefined ? [r.candidate[rep]] : [])), side, weightOf));
+    const values = perRep.map((p) => p.value).filter((v) => v !== null);
+    const candidate = values.length > 0 ? values.reduce((a, b) => a + b, 0) / values.length : null;
+    const cases = new Set();
+    scored.forEach((r, i) => {
+        if (r.candidate.some((s) => side(s).total > 0))
+            cases.add(i);
+    });
+    const weight = perRep.reduce((a, p) => a + p.weight, 0) / Math.max(1, perRep.length);
+    return {
+        baseline: baseline.value,
+        candidate,
+        delta: baseline.value !== null && candidate !== null ? candidate - baseline.value : null,
+        n: cases.size,
+        weight,
+        baselineN: baseline.n,
+        baselineWeight: baseline.weight,
+        direction: 'higher',
+        ...(extra.k !== undefined && { k: extra.k }),
+        ...(repetitions > 1 &&
+            values.length > 1 && { spread: Math.max(...values) - Math.min(...values) }),
+    };
+}
+function summarize(ctx, comparison, cases, results, models) {
+    const errors = results.filter((r) => r.error !== undefined).length;
+    const stopped = results.filter((r) => r.stopped !== undefined).length;
+    const versions = new Map();
+    for (const c of cases) {
+        const key = `${c.subject.kind}:${c.subject.id}@${c.subject.version}`;
+        const kept = versions.get(key) ?? {
+            id: c.subject.id,
+            flow: c.subject.kind === 'flow',
+            version: c.subject.version,
+            cases: 0,
+        };
+        kept.cases += 1;
+        versions.set(key, kept);
+    }
+    const specProject = ctx.suite.spec.projectId;
+    const projectId = typeof specProject === 'string'
+        ? specProject
+        : ctx.projectId;
+    const judgedWeight = (s) => s.totalWeight;
+    return {
+        evalRunId: ctx.runId,
+        status: results.length > 0 && errors === 0
+            ? 'completed'
+            : errors < results.length
+                ? 'partial'
+                : 'failed',
+        completedAt: new Date().toISOString(),
+        suite: { id: ctx.suite.id, version: ctx.suite.version },
+        candidate: candidateOf(ctx.target),
+        baseline: {
+            kind: 'recorded',
+            versions: [...versions.values()].map((v) => v.flow
+                ? { flowId: v.id, version: v.version, cases: v.cases }
+                : { agentId: v.id, version: v.version, cases: v.cases }),
+        },
+        scope: { ...(projectId !== undefined && { projectId }) },
+        cases: results.length,
+        diverged: results.filter((r) => r.diverged).length,
+        refusedWrites: results.reduce((a, r) => a + r.refusedWrites, 0),
+        errors,
+        stopped,
+        reads: comparison.reads,
+        sampling: { models },
+        repetitions: comparison.repetitions,
+        metrics: {
+            weightedYesShare: metric(results, comparison.repetitions, yesShare, judgedWeight),
+            judgedCoverage: metric(results, comparison.repetitions, coverage, judgedWeight),
+            weightedPrecisionAtK: metric(results, comparison.repetitions, precisionAtK, (s) => s.topK.totalWeight, { k: comparison.k }),
+        },
+    };
+}
+function candidateOf(target) {
+    return 'agentId' in target
+        ? { kind: 'agent', agentId: target.agentId, version: target.version ?? '' }
+        : { kind: 'flow', flowId: target.flowId, version: target.version ?? '' };
+}
+//# sourceMappingURL=judged-dispatcher.js.map

package/dist/judged-dispatcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judged-dispatcher.js","sourceRoot":"","sources":["../src/judged-dispatcher.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AA2BjC,OAAO,EAGL,WAAW,EACX,WAAW,EACX,WAAW,EACX,UAAU,GACX,MAAM,mBAAmB,CAAC;AAE3B,MAAM,CAAC,MAAM,kBAAkB,GAAmB;IAChD,QAAQ,EAAE,UAAU;IACpB,KAAK,EAAE,UAAU;IACjB,WAAW,EAAE,CAAC;IACd,CAAC,EAAE,EAAE;CACN,CAAC;AA2GF,2BAA2B;AAC3B,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,SAAS,kBAAkB,CACzB,KAA2D,EAC3D,MAA0B,EAC1B,UAAsC;IAEtC,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO;YACL,IAAI,EAAE,KAAK;YACX,OAAO,EACL,SAAS,IAAI,MAAM;gBACjB,CAAC,CAAC,4EAA4E;gBAC9E,CAAC,CAAC,0EAA0E;SACjF,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;IAChE,CAAC;IACD,MAAM,CAAC,GAAG,UAAU,IAAI,kBAAkB,CAAC;IAC3C,IAAI,CAAC,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,yCAAyC,EAAE,CAAC;IAC7E,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAAgC;IACrE,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,kBAAkB;QAC5B,KAAK,CAAC,QAAQ,CAAC,GAAG;YAChB,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;YACxD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC/C,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACf,OAAO,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;YACrE,CAAC;YACD,MAAM,OAAO,GAAuB,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAA+D,CAAC;YACtF,KAAK,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC;gBAC7B,IAAI,GAAG,CAAC,WAAW,CAAC,OAAO;oBAAE,MAAM;gBACnC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,GAAG,CAAC,UAAU,CAAC,MAAsD,CAAC,CAAC;YACzE,CAAC;YACD,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC/E,OAAO;gBACL,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE;gBACrC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;aACvD,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,KAA2B,EAC3B,GAAoB;IAEpB,MAAM,GAAG,GAAqB,EAAE,CAAC;IACjC,IAAI,MAAkE,CAAC;IACvE,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE;YACrB,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO;YAC1B,KAAK,EAAE,SAAS;YAChB,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;SACxC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAC/D,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,gDAAgD;AAChD,MAAM,SAAS;IAWM;IACA;IACA;IAZV,MAAM,GAAa,EAAE,CAAC;IACtB,SAAS,GAAkB,EAAE,CAAC;IACvC,KAAK,CAAkE;IACvE,KAAK,CAAqB;IAC1B,OAAO,CAA8B;IACrC,QAAQ,GAAG,KAAK,CAAC;IACjB,aAAa,GAAG,CAAC,CAAC;IAClB,eAAe,GAAG,KAAK,CAAC;IAExB,YACmB,UAA0B,EAC1B,UAA0B,EAC1B,SAAkB;QAFlB,eAAU,GAAV,UAAU,CAAgB;QAC1B,eAAU,GAAV,UAAU,CAAgB;QAC1B,cAAS,GAAT,SAAS,CAAS;IAClC,CAAC;IAEJ,GAAG,CAAC,OAAoC;QACtC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAA0B,CAAC,CAAC;QACtF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,KAAK,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACrF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC;YAC7B,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,GAAG,CAAC,OAAoC;QAC9C,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;QACxC,MAAM,OAAO,GAAG,WAAW,CACzB,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAC3C,UAAU,CAAC,KAAK,EAChB,UAAU,CAAC,MAAM,CAClB,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACzE,IAAI,UAAU,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;YAC9E,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,QAAQ,KAAK,SAAS;YAAE,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QACxE,IAAI,CAAC,KAAK,KAAK,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,CAAC,QAAqB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;QACzC,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;YAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ;YACR,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAChE,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC3E,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,KAAK,SAAS;YAChD,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,gFAAgF;YAChF,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;YACpE,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;SAC7F,CAAC;IACJ,CAAC;CACF;AAED,KAAK,UAAU,OAAO,CACpB,GAAoB,EACpB,UAA0B,EAC1B,UAA0B,EAC1B,MAAwE;IAExE,yFAAyF;IACzF,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC;IACtD,MAAM,QAAQ,GAAG,UAAU,CACzB,WAAW,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,EAC3F,UAAU,CAAC,CAAC,CACb,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAC/D,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,UAAU,CAAC,WAAW,EAAE,GAAG,EAAE,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK;YAAE,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,OAAoC;IACxD,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;AACpF,CAAC;AAED,SAAS,QAAQ,CAAC,OAAoC;IACpD,OAAO,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,GAAoB,EACpB,UAA0B;IAE1B,IAAI,CAAC;QACH,OAAO,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,6CAA6C;YAC7C,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,KAAK;YACb,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,GAAG,CAAC,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC;YAChE,MAAM,EAAE;gBACN,EAAE,EAAE,UAAU,CAAC,MAA0B;gBACzC,SAAS,EAAE,GAAG,CAAC,KAA0B;aAC1C;YACD,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,KAAK,OAAO,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;SAC3F,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,MAAwE,EACxE,OAAoC;IAEpC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO;IAC3C,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACpE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI;QAC9B,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE;QAC/B,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK;QAC7B,IAAI,EAAE,CAAC;KACR,CAAC;IACF,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACf,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACxB,CAAC;AAID,MAAM,QAAQ,GAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAC3E,MAAM,YAAY,GAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;AACzF,6FAA6F;AAC7F,MAAM,QAAQ,GAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;AAEvE,2EAA2E;AAC3E,SAAS,MAAM,CACb,MAA8B,EAC9B,IAAU,EACV,QAAoC;IAEpC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACrB,GAAG,IAAI,CAAC,CAAC;QACT,KAAK,IAAI,CAAC,CAAC;QACX,CAAC,IAAI,CAAC,CAAC;QACP,MAAM,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,MAAM,CACb,OAAoC,EACpC,WAAmB,EACnB,IAAU,EACV,QAAoC,EACpC,QAAiC,EAAE;IAEnC,4EAA4E;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,MAAM,CACrB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC7B,IAAI,EACJ,QAAQ,CACT,CAAC;IACF,yFAAyF;IACzF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAC5D,MAAM,CACJ,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EACjF,IAAI,EACJ,QAAQ,CACT,CACF,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACjF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACrF,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,KAAK;QACxB,SAAS;QACT,KAAK,EAAE,QAAQ,CAAC,KAAK,KAAK,IAAI,IAAI,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QACxF,CAAC,EAAE,KAAK,CAAC,IAAI;QACb,MAAM;QACN,SAAS,EAAE,QAAQ,CAAC,CAAC;QACrB,cAAc,EAAE,QAAQ,CAAC,MAAM;QAC/B,SAAS,EAAE,QAAQ;QACnB,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,SAAS,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QAC5C,GAAG,CAAC,WAAW,GAAG,CAAC;YACjB,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAChB,GAAoB,EACpB,UAA0B,EAC1B,KAAgC,EAChC,OAAoC,EACpC,MAAsE;IAEtE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAyE,CAAC;IAClG,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACrE,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI;YAChC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE;YAChB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;YAC/B,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;YAC1B,KAAK,EAAE,CAAC;SACT,CAAC;QACF,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAChB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;IAC7C,MAAM,SAAS,GACb,OAAO,WAAW,KAAK,QAAQ;QAC7B,CAAC,CAAC,WAAW;QACb,CAAC,CAAE,GAAG,CAAC,SAA2C,CAAC;IACvD,MAAM,YAAY,GAAG,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;IACvD,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,KAA0B;QACzC,MAAM,EACJ,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,KAAK,CAAC;YAChC,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;gBACvB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,QAAQ;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,KAAK,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE;QACvD,SAAS,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;QAClC,QAAQ,EAAE;YACR,IAAI,EAAE,UAAU;YAChB,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAClC,CAAC,CAAC,EAAmB,EAAE,CACrB,CAAC,CAAC,IAAI;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE;gBACtD,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAC5D;SACF;QACD,KAAK,EAAE,EAAE,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE;QACxD,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM;QAClD,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;QAC/D,MAAM;QACN,OAAO;QACP,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,QAAQ,EAAE,EAAE,MAAM,EAAE;QACpB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,OAAO,EAAE;YACP,gBAAgB,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,WAAW,EAAE,QAAQ,EAAE,YAAY,CAAC;YACjF,cAAc,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,WAAW,EAAE,QAAQ,EAAE,YAAY,CAAC;YAC/E,oBAAoB,EAAE,MAAM,CAC1B,OAAO,EACP,UAAU,CAAC,WAAW,EACtB,YAAY,EACZ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EACzB,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,CACpB;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAA0B;IAC7C,OAAO,SAAS,IAAI,MAAM;QACxB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAA4B,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE;QAChG,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAA2B,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;AAClG,CAAC"}

package/dist/judged-items.d.ts

@@ -0,0 +1,86 @@
+/**
+ * The items of a run's output that people judge, and how a new output is
+ * scored against the judgments of an old one. Pure, with no dependencies,
+ * so a console can derive items the same way.
+ *
+ * Items:
+ *   - an agent turn's answer (its last agent message) is `answer`;
+ *   - a flow run's whole output is `output`;
+ *   - each element of a list in the output (an agent's typed result sits
+ *     under `output`) is an item, keyed by its own `id` or `key` when it
+ *     has one, else by where it sits (a JSON Pointer), with its rank.
+ *
+ * A judgment carries over to a new output's item when it's the same item:
+ * an item keyed by its own id is the same item when the key matches; any
+ * other (the answer, an element keyed by its place) only when its content
+ * is the same too. So a changed answer is a new item, with no judgments.
+ */
+export interface OutputItem {
+    /** The key judgments use for this item. */
+    readonly key: string;
+    /** Where it sits in the output (JSON Pointer). */
+    readonly pointer: string;
+    /** Its place in its list (0 = first). */
+    readonly rank?: number;
+    /** `true` when the key is the item's own id, so it identifies the item by itself. */
+    readonly ownKey: boolean;
+    readonly value: unknown;
+}
+/** The items of a run's output (`agentTurn`: an agent turn's result). */
+export declare function outputItems(output: unknown, agentTurn: boolean): readonly OutputItem[];
+/** The value at a JSON Pointer in `doc`, or `undefined` when there's none. */
+export declare function valueAt(doc: unknown, pointer: string): unknown;
+/** What people said about one judged item (as a judged test-set case keeps it). */
+export interface ItemJudgments {
+    readonly key: string;
+    readonly pointer?: string;
+    readonly rank?: number;
+    readonly yesWeight: number;
+    readonly totalWeight: number;
+}
+/** A new output's item and the judged item it is the same as, if any. */
+export interface MatchedItem {
+    readonly item: OutputItem;
+    readonly judged?: ItemJudgments;
+}
+/**
+ * Match a new output's items to the items judged in a past output
+ * (`judgedOutput`, where the judged items' pointers point).
+ */
+export declare function matchJudged(items: readonly OutputItem[], judged: readonly ItemJudgments[], judgedOutput: unknown): readonly MatchedItem[];
+/** An output scored against judgments: the sums the metrics are made of. */
+export interface OutputScore {
+    /** Σ yesWeight and Σ totalWeight over the output's judged items. */
+    readonly yesWeight: number;
+    readonly totalWeight: number;
+    readonly items: number;
+    readonly judgedItems: number;
+    /** The same sums over the judged items among the first `k` ranked items. */
+    readonly topK: {
+        readonly yesWeight: number;
+        readonly totalWeight: number;
+    };
+}
+export declare function scoreItems(matched: readonly MatchedItem[], k: number): OutputScore;
+/** How a new output's items moved against the judged ones. */
+export interface ItemChanges {
+    /** Judged items the new output still has, with their rank before and now. */
+    readonly kept: readonly {
+        readonly key: string;
+        readonly rankBefore?: number;
+        readonly rank?: number;
+    }[];
+    /** Judged items the new output no longer has. */
+    readonly dropped: readonly {
+        readonly key: string;
+        readonly rankBefore?: number;
+    }[];
+    /** The new output's items no one judged (for experts to judge). */
+    readonly new: readonly {
+        readonly key: string;
+        readonly pointer: string;
+        readonly rank?: number;
+    }[];
+}
+export declare function itemChanges(matched: readonly MatchedItem[], judged: readonly ItemJudgments[]): ItemChanges;
+//# sourceMappingURL=judged-items.d.ts.map

package/dist/judged-items.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judged-items.d.ts","sourceRoot":"","sources":["../src/judged-items.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,WAAW,UAAU;IACzB,2CAA2C;IAC3C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAkED,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAetF;AAED,8EAA8E;AAC9E,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAe9D;AAgBD,mFAAmF;AACnF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;CACjC;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,MAAM,EAAE,SAAS,aAAa,EAAE,EAChC,YAAY,EAAE,OAAO,GACpB,SAAS,WAAW,EAAE,CAWxB;AAED,4EAA4E;AAC5E,MAAM,WAAW,WAAW;IAC1B,oEAAoE;IACpE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE;QAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7E;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,WAAW,CAqBlF;AAED,8DAA8D;AAC9D,MAAM,WAAW,WAAW;IAC1B,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,SAAS;QACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;KACxB,EAAE,CAAC;IACJ,iDAAiD;IACjD,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACpF,mEAAmE;IACnE,QAAQ,CAAC,GAAG,EAAE,SAAS;QACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;KACxB,EAAE,CAAC;CACL;AAED,wBAAgB,WAAW,CACzB,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,MAAM,EAAE,SAAS,aAAa,EAAE,GAC/B,WAAW,CAwBb"}

package/dist/judged-items.js

@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+const MAX_DEPTH = 3;
+const MAX_LIST = 100;
+function obj(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+function escapeToken(token) {
+    return token.replace(/~/g, '~0').replace(/\//g, '~1');
+}
+function unescapeToken(token) {
+    return token.replace(/~1/g, '/').replace(/~0/g, '~');
+}
+/** An element's own id, if it has a usable one. */
+function ownId(element) {
+    const o = obj(element);
+    for (const field of ['id', 'key']) {
+        const v = o?.[field];
+        if (typeof v === 'string' && v.length > 0)
+            return v;
+        if (typeof v === 'number')
+            return String(v);
+    }
+    return undefined;
+}
+function listItems(value, pointer, depth) {
+    if (depth > MAX_DEPTH)
+        return [];
+    if (Array.isArray(value)) {
+        if (value.length === 0 || value.length > MAX_LIST)
+            return [];
+        const seen = new Set();
+        return value.map((element, i) => {
+            const id = ownId(element);
+            // Duplicate ids within one list fall back to the element's place.
+            const own = id !== undefined && !seen.has(id);
+            const key = own ? id : `${pointer}/${i}`;
+            seen.add(key);
+            return { key, pointer: `${pointer}/${i}`, rank: i, ownKey: own, value: element };
+        });
+    }
+    const o = obj(value);
+    if (o === undefined)
+        return [];
+    return Object.entries(o).flatMap(([field, child]) => listItems(child, `${pointer}/${escapeToken(field)}`, depth + 1));
+}
+function turnAnswer(output) {
+    const appended = Array.isArray(output.appended) ? output.appended : [];
+    for (let i = appended.length - 1; i >= 0; i--) {
+        const m = obj(appended[i]);
+        if (m?.role === 'agent' && m.content !== undefined && !hasToolCalls(m.content)) {
+            return { key: 'answer', pointer: `/appended/${i}/content`, ownKey: false, value: m.content };
+        }
+    }
+    return undefined;
+}
+function hasToolCalls(content) {
+    return Array.isArray(obj(content)?.toolCalls);
+}
+/** The items of a run's output (`agentTurn`: an agent turn's result). */
+export function outputItems(output, agentTurn) {
+    const o = obj(output);
+    if (agentTurn) {
+        if (o === undefined)
+            return [];
+        const answer = turnAnswer(o);
+        return [
+            ...(answer !== undefined ? [answer] : []),
+            ...(o.output !== undefined ? listItems(o.output, '/output', 0) : []),
+        ];
+    }
+    if (output === undefined || output === null)
+        return [];
+    return [
+        { key: 'output', pointer: '', ownKey: false, value: output },
+        ...listItems(output, '', 0),
+    ];
+}
+/** The value at a JSON Pointer in `doc`, or `undefined` when there's none. */
+export function valueAt(doc, pointer) {
+    if (pointer === '')
+        return doc;
+    if (!pointer.startsWith('/'))
+        return undefined;
+    let current = doc;
+    for (const raw of pointer.slice(1).split('/')) {
+        const token = unescapeToken(raw);
+        if (Array.isArray(current)) {
+            const i = Number(token);
+            current = Number.isInteger(i) ? current[i] : undefined;
+        }
+        else {
+            current = obj(current)?.[token];
+        }
+        if (current === undefined)
+            return undefined;
+    }
+    return current;
+}
+/** JSON with sorted keys, so equal values compare equal. */
+function canonical(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonical).join(',')}]`;
+    const o = obj(value);
+    if (o !== undefined) {
+        return `{${Object.keys(o)
+            .sort()
+            .filter((k) => o[k] !== undefined)
+            .map((k) => `${JSON.stringify(k)}:${canonical(o[k])}`)
+            .join(',')}}`;
+    }
+    return JSON.stringify(value) ?? 'null';
+}
+/**
+ * Match a new output's items to the items judged in a past output
+ * (`judgedOutput`, where the judged items' pointers point).
+ */
+export function matchJudged(items, judged, judgedOutput) {
+    const byKey = new Map(judged.map((j) => [j.key, j]));
+    return items.map((item) => {
+        const j = byKey.get(item.key);
+        if (j === undefined)
+            return { item };
+        if (item.ownKey)
+            return { item, judged: j };
+        const before = j.pointer === undefined ? undefined : valueAt(judgedOutput, j.pointer);
+        return before !== undefined && canonical(before) === canonical(item.value)
+            ? { item, judged: j }
+            : { item };
+    });
+}
+export function scoreItems(matched, k) {
+    let yesWeight = 0;
+    let totalWeight = 0;
+    let judgedItems = 0;
+    for (const m of matched) {
+        if (m.judged === undefined)
+            continue;
+        judgedItems += 1;
+        yesWeight += m.judged.yesWeight;
+        totalWeight += m.judged.totalWeight;
+    }
+    const top = matched
+        .filter((m) => m.item.rank !== undefined)
+        .sort((a, b) => (a.item.rank ?? 0) - (b.item.rank ?? 0))
+        .slice(0, k);
+    const topK = { yesWeight: 0, totalWeight: 0 };
+    for (const m of top) {
+        if (m.judged === undefined)
+            continue;
+        topK.yesWeight += m.judged.yesWeight;
+        topK.totalWeight += m.judged.totalWeight;
+    }
+    return { yesWeight, totalWeight, items: matched.length, judgedItems, topK };
+}
+export function itemChanges(matched, judged) {
+    const keptKeys = new Set();
+    const kept = [];
+    const fresh = [];
+    for (const m of matched) {
+        if (m.judged !== undefined) {
+            keptKeys.add(m.judged.key);
+            kept.push({
+                key: m.judged.key,
+                ...(m.judged.rank !== undefined && { rankBefore: m.judged.rank }),
+                ...(m.item.rank !== undefined && { rank: m.item.rank }),
+            });
+        }
+        else {
+            fresh.push({
+                key: m.item.key,
+                pointer: m.item.pointer,
+                ...(m.item.rank !== undefined && { rank: m.item.rank }),
+            });
+        }
+    }
+    const dropped = judged
+        .filter((j) => !keptKeys.has(j.key))
+        .map((j) => ({ key: j.key, ...(j.rank !== undefined && { rankBefore: j.rank }) }));
+    return { kept, dropped, new: fresh };
+}
+//# sourceMappingURL=judged-items.js.map

package/dist/judged-items.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judged-items.js","sourceRoot":"","sources":["../src/judged-items.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAgCjC,MAAM,SAAS,GAAG,CAAC,CAAC;AACpB,MAAM,QAAQ,GAAG,GAAG,CAAC;AAErB,SAAS,GAAG,CAAC,KAAc;IACzB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACzE,CAAC,CAAE,KAAiC;QACpC,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AACvD,CAAC;AAED,mDAAmD;AACnD,SAAS,KAAK,CAAC,OAAgB;IAC7B,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;QACpD,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,SAAS,CAAC,KAAc,EAAE,OAAe,EAAE,KAAa;IAC/D,IAAI,KAAK,GAAG,SAAS;QAAE,OAAO,EAAE,CAAC;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;YAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;YAC1B,kEAAkE;YAClE,MAAM,GAAG,GAAG,EAAE,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QACnF,CAAC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;IACrB,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAC/B,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAClD,SAAS,CAAC,KAAK,EAAE,GAAG,OAAO,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAChE,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,MAA+B;IACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/E,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAC/F,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,OAAgB;IACpC,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC;AAChD,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,WAAW,CAAC,MAAe,EAAE,SAAkB;IAC7D,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,OAAO;YACL,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACvD,OAAO;QACL,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE;QAC5D,GAAG,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,OAAO,CAAC,GAAY,EAAE,OAAe;IACnD,IAAI,OAAO,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC;IAC/B,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/C,IAAI,OAAO,GAAY,GAAG,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YACxB,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,4DAA4D;AAC5D,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACvE,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;IACrB,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;QACpB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;aACtB,IAAI,EAAE;aACN,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC;aACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC;AACzC,CAAC;AAiBD;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAA4B,EAC5B,MAAgC,EAChC,YAAqB;IAErB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QACtF,OAAO,MAAM,KAAK,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC;YACxE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YACrB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAaD,MAAM,UAAU,UAAU,CAAC,OAA+B,EAAE,CAAS;IACnE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAAE,SAAS;QACrC,WAAW,IAAI,CAAC,CAAC;QACjB,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;QAChC,WAAW,IAAI,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC;IACtC,CAAC;IACD,MAAM,GAAG,GAAG,OAAO;SAChB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC;SACxC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;SACvD,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACf,MAAM,IAAI,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAAE,SAAS;QACrC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;QACrC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC;IAC3C,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AAC9E,CAAC;AAoBD,MAAM,UAAU,WAAW,CACzB,OAA+B,EAC/B,MAAgC;IAEhC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,IAAI,GAAkC,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAiC,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC3B,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG;gBACjB,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACjE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC;gBACT,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG;gBACf,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO;gBACvB,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,MAAM;SACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;SACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACvC,CAAC"}