@kindgi/api 0.0.0-bootstrap.0 → 0.1.1

package/dist/eval-run-dispatcher.js

@@ -0,0 +1,368 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+import { randomUUID } from 'node:crypto';
+import { EVAL_RUN_STATUSES } from './eval-run-binding.js';
+/**
+ * Reference dispatcher for `EvalKind === 'accuracy'`. Reads
+ * `spec.cases` from the registered suite; for each case, invokes the
+ * subject with `case.input`, grades `(caseInput, actualOutput,
+ * case.expected ?? case.expectedOutput)` through the injected
+ * `EvalGrader`, and aggregates `{ passCount, totalCount, meanScore,
+ * perCase[] }`.
+ *
+ * A dry-run echoes the plan (case count + first-case preview) without
+ * invoking the subject — proves the same envelope is available on
+ * `dryRun: true` requests. Callers use it as a sanity check before
+ * paying for the real run.
+ */
+export function createAccuracyDispatcher(options) {
+    return {
+        kind: 'accuracy',
+        validate(suite) {
+            const cases = extractCases(suite.spec);
+            if (cases === null) {
+                return { kind: 'err', message: 'accuracy suite.spec.cases must be an array' };
+            }
+            if (cases.length === 0) {
+                return { kind: 'err', message: 'accuracy suite.spec.cases must not be empty' };
+            }
+            return { kind: 'ok' };
+        },
+        async dispatch(ctx) {
+            const cases = extractCases(ctx.suite.spec) ?? [];
+            if (ctx.dryRun) {
+                return {
+                    result: {
+                        passCount: 0,
+                        totalCount: cases.length,
+                        meanScore: 0,
+                        perCase: [],
+                        dryRun: true,
+                        preview: cases.slice(0, 1).map((c, i) => ({
+                            caseIndex: i,
+                            input: c.input,
+                            expected: extractExpected(c),
+                        })),
+                    },
+                };
+            }
+            const perCase = [];
+            let passCount = 0;
+            let scoreSum = 0;
+            for (let i = 0; i < cases.length; i++) {
+                if (ctx.abortSignal.aborted) {
+                    return {
+                        result: {
+                            passCount,
+                            totalCount: cases.length,
+                            meanScore: perCase.length > 0 ? scoreSum / perCase.length : 0,
+                            perCase,
+                            cancelled: true,
+                        },
+                        error: 'cancelled',
+                    };
+                }
+                const c = cases[i];
+                if (c === undefined)
+                    continue;
+                const expected = extractExpected(c);
+                const invocation = await ctx.subject.invoke({
+                    tenantId: ctx.tenantId,
+                    target: ctx.target,
+                    input: c.input,
+                    dryRun: ctx.dryRun,
+                    abortSignal: ctx.abortSignal,
+                });
+                if (invocation.error !== undefined) {
+                    const entry = {
+                        caseIndex: i,
+                        ...(c.id !== undefined && { caseId: c.id }),
+                        pass: false,
+                        score: 0,
+                        error: invocation.error,
+                    };
+                    perCase.push(entry);
+                    ctx.onProgress(entry);
+                    continue;
+                }
+                const grade = await options.grader.grade({
+                    caseInput: c.input,
+                    actualOutput: invocation.output,
+                    expected,
+                });
+                if (grade.pass)
+                    passCount += 1;
+                scoreSum += grade.score;
+                const entry = {
+                    caseIndex: i,
+                    ...(c.id !== undefined && { caseId: c.id }),
+                    pass: grade.pass,
+                    score: grade.score,
+                    ...(grade.reason !== undefined && { reason: grade.reason }),
+                    ...(grade.metadata !== undefined && { judgeMetadata: grade.metadata }),
+                };
+                perCase.push(entry);
+                ctx.onProgress(entry);
+            }
+            return {
+                result: {
+                    passCount,
+                    totalCount: cases.length,
+                    meanScore: cases.length > 0 ? scoreSum / cases.length : 0,
+                    perCase,
+                },
+            };
+        },
+    };
+}
+function extractCases(spec) {
+    const cases = spec.cases;
+    if (!Array.isArray(cases))
+        return null;
+    const out = [];
+    for (const c of cases) {
+        if (c === null || typeof c !== 'object')
+            return null;
+        const rec = c;
+        if (!('input' in rec))
+            return null;
+        out.push(rec);
+    }
+    return out;
+}
+function extractExpected(c) {
+    if (c.expected !== undefined)
+        return c.expected;
+    return c.expectedOutput;
+}
+/**
+ * Reference in-process `EvalRunBinding`. Persists eval-run rows in a
+ * `Map<runId, EvalRun>`; suitable for tests and dev deployments.
+ * Production deployments swap in a durable binding that starts real
+ * runs (`runGraph`) — the surface is identical, only the persistence +
+ * execution substrate changes.
+ *
+ * A run starts as `running`, resolves per its dispatcher's outcome,
+ * and can be cancelled via `AbortController.abort()` on the wrapped
+ * controller. Cancellation is best-effort — a dispatcher that has
+ * already returned cannot be undone; the row transitions to
+ * `cancelled` and any partial `perCase` results are preserved.
+ */
+export function createInProcessEvalRunBinding(options) {
+    const now = options.now ?? (() => new Date());
+    const runs = new Map();
+    const controllers = new Map();
+    const setRun = (r) => {
+        runs.set(r.runId, r);
+    };
+    return {
+        async start(input) {
+            const suite = await options.suiteRegistry.get({
+                tenantId: input.tenantId,
+                suiteId: input.suiteId,
+            });
+            if (suite === null) {
+                return { kind: 'suite-not-found', suiteId: input.suiteId };
+            }
+            const dispatcher = options.dispatchers[suite.kind];
+            if (dispatcher === undefined) {
+                return { kind: 'dispatcher-not-registered', evalKind: suite.kind };
+            }
+            const target = input.agentRef ?? input.flowRef;
+            if (target === undefined) {
+                return {
+                    kind: 'dispatcher-input-invalid',
+                    message: 'agentRef or flowRef required',
+                };
+            }
+            if (dispatcher.validate !== undefined) {
+                const v = dispatcher.validate(suite, target);
+                if (v.kind === 'err') {
+                    return { kind: 'dispatcher-input-invalid', message: v.message };
+                }
+            }
+            const runId = randomUUID();
+            const startedAt = now().toISOString();
+            const record = {
+                runId,
+                tenantId: input.tenantId,
+                suiteId: suite.id,
+                suiteVersion: suite.version,
+                kind: suite.kind,
+                ...(input.agentRef !== undefined && { agentRef: input.agentRef }),
+                ...(input.flowRef !== undefined && { flowRef: input.flowRef }),
+                status: 'running',
+                dryRun: input.dryRun === true,
+                startedAt,
+                ...(input.correlationId !== undefined && { correlationId: input.correlationId }),
+            };
+            setRun(record);
+            const controller = new AbortController();
+            controllers.set(runId, controller);
+            // Fire-and-forget the dispatch — completion updates the row.
+            // Errors thrown by the dispatcher itself land the run in
+            // `failed` (bug in the dispatcher, not a case-level failure).
+            void (async () => {
+                let outcome;
+                try {
+                    outcome = await dispatcher.dispatch({
+                        tenantId: input.tenantId,
+                        runId,
+                        suite,
+                        target,
+                        dryRun: input.dryRun === true,
+                        abortSignal: controller.signal,
+                        subject: options.subject,
+                        onProgress: (perCaseEntry) => {
+                            const current = runs.get(runId);
+                            if (current === undefined)
+                                return;
+                            const currentResult = (current.result ?? {});
+                            const perCase = Array.isArray(currentResult.perCase)
+                                ? [...currentResult.perCase, perCaseEntry]
+                                : [perCaseEntry];
+                            setRun({
+                                ...current,
+                                result: { ...currentResult, perCase },
+                            });
+                        },
+                    });
+                }
+                catch (cause) {
+                    const message = cause instanceof Error ? cause.message : String(cause);
+                    const current = runs.get(runId);
+                    if (current === undefined)
+                        return;
+                    setRun({
+                        ...current,
+                        status: 'failed',
+                        completedAt: now().toISOString(),
+                        error: `Dispatcher threw: ${message}`,
+                    });
+                    return;
+                }
+                const current = runs.get(runId);
+                if (current === undefined)
+                    return;
+                if (current.status === 'cancelled') {
+                    // Cancel path already set the terminal row; preserve any
+                    // partial result the dispatcher produced.
+                    setRun({
+                        ...current,
+                        result: outcome.result,
+                        ...(current.completedAt === undefined && {
+                            completedAt: now().toISOString(),
+                        }),
+                    });
+                    return;
+                }
+                const status = outcome.error === undefined
+                    ? 'completed'
+                    : outcome.error === 'cancelled'
+                        ? 'cancelled'
+                        : 'failed';
+                setRun({
+                    ...current,
+                    status,
+                    completedAt: now().toISOString(),
+                    result: outcome.result,
+                    ...(outcome.error !== undefined &&
+                        outcome.error !== 'cancelled' && {
+                        error: outcome.error,
+                    }),
+                });
+            })();
+            return { kind: 'ok', runId };
+        },
+        async get(input) {
+            const record = runs.get(input.runId);
+            if (record === undefined)
+                return null;
+            if (record.tenantId !== input.tenantId) {
+                return null;
+            }
+            return record;
+        },
+        async list(input) {
+            const filter = input.filter ?? {};
+            const all = [];
+            for (const r of runs.values()) {
+                if (r.tenantId !== input.tenantId)
+                    continue;
+                if (filter.suiteId !== undefined && r.suiteId !== filter.suiteId)
+                    continue;
+                if (filter.status !== undefined && r.status !== filter.status)
+                    continue;
+                if (filter.agentId !== undefined) {
+                    if (r.agentRef === undefined)
+                        continue;
+                    if (r.agentRef.agentId !== filter.agentId) {
+                        continue;
+                    }
+                }
+                if (filter.flowId !== undefined) {
+                    if (r.flowRef === undefined)
+                        continue;
+                    if (r.flowRef.flowId !== filter.flowId) {
+                        continue;
+                    }
+                }
+                if (filter.from !== undefined) {
+                    if (r.startedAt < filter.from)
+                        continue;
+                }
+                if (filter.to !== undefined) {
+                    if (r.startedAt > filter.to)
+                        continue;
+                }
+                all.push(r);
+            }
+            all.sort((a, b) => {
+                const at = a.startedAt;
+                const bt = b.startedAt;
+                if (at !== bt)
+                    return at < bt ? 1 : -1;
+                return a.runId < b.runId ? 1 : -1;
+            });
+            const cursor = input.cursor;
+            const startAt = cursor === undefined
+                ? 0
+                : Math.max(0, all.findIndex((r) => r.runId === cursor) + 1);
+            const slice = all.slice(startAt, startAt + input.limit);
+            const hasMore = startAt + slice.length < all.length;
+            const last = slice[slice.length - 1];
+            return {
+                data: slice,
+                ...(hasMore &&
+                    last !== undefined && {
+                    nextCursor: last.runId,
+                }),
+            };
+        },
+        async cancel(input) {
+            const record = runs.get(input.runId);
+            if (record === undefined)
+                return { kind: 'not-found' };
+            if (record.tenantId !== input.tenantId) {
+                return { kind: 'not-found' };
+            }
+            if (isTerminal(record.status)) {
+                return { kind: 'already-terminal', status: record.status };
+            }
+            const controller = controllers.get(input.runId);
+            controller?.abort();
+            setRun({
+                ...record,
+                status: 'cancelled',
+                completedAt: now().toISOString(),
+            });
+            return { kind: 'ok' };
+        },
+    };
+}
+function isTerminal(status) {
+    return status === 'completed' || status === 'failed' || status === 'cancelled';
+}
+// re-export for callers that only import the dispatcher module.
+export { EVAL_RUN_STATUSES };
+//# sourceMappingURL=eval-run-dispatcher.js.map

package/dist/eval-run-dispatcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-run-dispatcher.js","sourceRoot":"","sources":["../src/eval-run-dispatcher.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAEjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAkBzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAwG1D;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAkC;IACzE,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,QAAQ,CAAC,KAAK;YACZ,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAC;YAChF,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,6CAA6C,EAAE,CAAC;YACjF,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,GAAG;YAChB,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACjD,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACf,OAAO;oBACL,MAAM,EAAE;wBACN,SAAS,EAAE,CAAC;wBACZ,UAAU,EAAE,KAAK,CAAC,MAAM;wBACxB,SAAS,EAAE,CAAC;wBACZ,OAAO,EAAE,EAAE;wBACX,MAAM,EAAE,IAAI;wBACZ,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;4BACxC,SAAS,EAAE,CAAC;4BACZ,KAAK,EAAE,CAAC,CAAC,KAAK;4BACd,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC;yBAC7B,CAAC,CAAC;qBACJ;iBACF,CAAC;YACJ,CAAC;YACD,MAAM,OAAO,GAA8B,EAAE,CAAC;YAC9C,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,IAAI,GAAG,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;oBAC5B,OAAO;wBACL,MAAM,EAAE;4BACN,SAAS;4BACT,UAAU,EAAE,KAAK,CAAC,MAAM;4BACxB,SAAS,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;4BAC7D,OAAO;4BACP,SAAS,EAAE,IAAI;yBAChB;wBACD,KAAK,EAAE,WAAW;qBACnB,CAAC;gBACJ,CAAC;gBACD,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACnB,IAAI,CAAC,KAAK,SAAS;oBAAE,SAAS;gBAC9B,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;oBAC1C,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,WAAW,EAAE,GAAG,CAAC,WAAW;iBAC7B,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;oBACnC,MAAM,KAAK,GAAG;wBACZ,SAAS,EAAE,CAAC;wBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;wBAC3C,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE,CAAC;wBACR,KAAK,EAAE,UAAU,CAAC,KAAK;qBACxB,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;oBACtB,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;oBACvC,SAAS,EAAE,CAAC,CAAC,KAAK;oBAClB,YAAY,EAAE,UAAU,CAAC,MAAM;oBAC/B,QAAQ;iBACT,CAAC,CAAC;gBACH,IAAI,KAAK,CAAC,IAAI;oBAAE,SAAS,IAAI,CAAC,CAAC;gBAC/B,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC;gBACxB,MAAM,KAAK,GAA4B;oBACrC,SAAS,EAAE,CAAC;oBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;oBAC3C,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;oBAC3D,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;iBACvE,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACpB,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;YACD,OAAO;gBACL,MAAM,EAAE;oBACN,SAAS;oBACT,UAAU,EAAE,KAAK,CAAC,MAAM;oBACxB,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBACzD,OAAO;iBACR;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,IAAuC;IAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACrD,MAAM,GAAG,GAAG,CAA4B,CAAC;QACzC,IAAI,CAAC,CAAC,OAAO,IAAI,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,GAAG,CAAC,IAAI,CAAC,GAA0B,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,eAAe,CAAC,CAAW;IAClC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChD,OAAO,CAAC,CAAC,cAAc,CAAC;AAC1B,CAAC;AAeD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,6BAA6B,CAC3C,OAAuC;IAEvC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAS,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA2B,CAAC;IAEvD,MAAM,MAAM,GAAG,CAAC,CAAU,EAAQ,EAAE;QAClC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAA0B,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,CAAC,KAAK,CAAC,KAAwB;YAClC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC;gBAC5C,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC,CAAC;YACH,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;YAC7D,CAAC;YACD,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,OAAO,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;YACrE,CAAC;YACD,MAAM,MAAM,GAAmC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC;YAC/E,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,OAAO;oBACL,IAAI,EAAE,0BAA0B;oBAChC,OAAO,EAAE,8BAA8B;iBACxC,CAAC;YACJ,CAAC;YACD,IAAI,UAAU,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC7C,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACrB,OAAO,EAAE,IAAI,EAAE,0BAA0B,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAClE,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,UAAU,EAAsB,CAAC;YAC/C,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,WAAW,EAA0B,CAAC;YAC9D,MAAM,MAAM,GAAY;gBACtB,KAAK;gBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,YAAY,EAAE,KAAK,CAAC,OAAO;gBAC3B,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACjE,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9D,MAAM,EAAE,SAA0B;gBAClC,MAAM,EAAE,KAAK,CAAC,MAAM,KAAK,IAAI;gBAC7B,SAAS;gBACT,GAAG,CAAC,KAAK,CAAC,aAAa,KAAK,SAAS,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC;aACjF,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,CAAC;YACf,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,WAAW,CAAC,GAAG,CAAC,KAA0B,EAAE,UAAU,CAAC,CAAC;YAExD,6DAA6D;YAC7D,yDAAyD;YACzD,8DAA8D;YAC9D,KAAK,CAAC,KAAK,IAAmB,EAAE;gBAC9B,IAAI,OAAuB,CAAC;gBAC5B,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC;wBAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,KAAK;wBACL,KAAK;wBACL,MAAM;wBACN,MAAM,EAAE,KAAK,CAAC,MAAM,KAAK,IAAI;wBAC7B,WAAW,EAAE,UAAU,CAAC,MAAM;wBAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,UAAU,EAAE,CAAC,YAA+C,EAAE,EAAE;4BAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAA0B,CAAC,CAAC;4BACrD,IAAI,OAAO,KAAK,SAAS;gCAAE,OAAO;4BAClC,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAA4B,CAAC;4BACxE,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC;gCAClD,CAAC,CAAC,CAAC,GAAI,aAAa,CAAC,OAAqB,EAAE,YAAY,CAAC;gCACzD,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;4BACnB,MAAM,CAAC;gCACL,GAAG,OAAO;gCACV,MAAM,EAAE,EAAE,GAAG,aAAa,EAAE,OAAO,EAAuC;6BAC3E,CAAC,CAAC;wBACL,CAAC;qBACF,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACvE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAA0B,CAAC,CAAC;oBACrD,IAAI,OAAO,KAAK,SAAS;wBAAE,OAAO;oBAClC,MAAM,CAAC;wBACL,GAAG,OAAO;wBACV,MAAM,EAAE,QAAQ;wBAChB,WAAW,EAAE,GAAG,EAAE,CAAC,WAAW,EAA0B;wBACxD,KAAK,EAAE,qBAAqB,OAAO,EAAE;qBACtC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAA0B,CAAC,CAAC;gBACrD,IAAI,OAAO,KAAK,SAAS;oBAAE,OAAO;gBAClC,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBACnC,yDAAyD;oBACzD,0CAA0C;oBAC1C,MAAM,CAAC;wBACL,GAAG,OAAO;wBACV,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI;4BACvC,WAAW,EAAE,GAAG,EAAE,CAAC,WAAW,EAA0B;yBACzD,CAAC;qBACH,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,MAAM,GACV,OAAO,CAAC,KAAK,KAAK,SAAS;oBACzB,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,KAAK,KAAK,WAAW;wBAC7B,CAAC,CAAC,WAAW;wBACb,CAAC,CAAC,QAAQ,CAAC;gBACjB,MAAM,CAAC;oBACL,GAAG,OAAO;oBACV,MAAM;oBACN,WAAW,EAAE,GAAG,EAAE,CAAC,WAAW,EAA0B;oBACxD,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS;wBAC7B,OAAO,CAAC,KAAK,KAAK,WAAW,IAAI;wBAC/B,KAAK,EAAE,OAAO,CAAC,KAAK;qBACrB,CAAC;iBACL,CAAC,CAAC;YACL,CAAC,CAAC,EAAE,CAAC;YAEL,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC/B,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,KAAsB;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAA0B,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YACtC,IAAK,MAAM,CAAC,QAA8B,KAAM,KAAK,CAAC,QAA8B,EAAE,CAAC;gBACrF,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,KAAuB;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;YAClC,MAAM,GAAG,GAAc,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC9B,IAAK,CAAC,CAAC,QAA8B,KAAM,KAAK,CAAC,QAA8B;oBAAE,SAAS;gBAC1F,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO;oBAAE,SAAS;gBAC3E,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;oBAAE,SAAS;gBACxE,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;oBACjC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;wBAAE,SAAS;oBACvC,IAAK,CAAC,CAAC,QAAQ,CAAC,OAA6B,KAAM,MAAM,CAAC,OAA6B,EAAE,CAAC;wBACxF,SAAS;oBACX,CAAC;gBACH,CAAC;gBACD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;oBAChC,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;wBAAE,SAAS;oBACtC,IAAK,CAAC,CAAC,OAAO,CAAC,MAA4B,KAAM,MAAM,CAAC,MAA4B,EAAE,CAAC;wBACrF,SAAS;oBACX,CAAC;gBACH,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC9B,IAAK,CAAC,CAAC,SAA+B,GAAI,MAAM,CAAC,IAA0B;wBAAE,SAAS;gBACxF,CAAC;gBACD,IAAI,MAAM,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;oBAC5B,IAAK,CAAC,CAAC,SAA+B,GAAI,MAAM,CAAC,EAAwB;wBAAE,SAAS;gBACtF,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAChB,MAAM,EAAE,GAAG,CAAC,CAAC,SAA8B,CAAC;gBAC5C,MAAM,EAAE,GAAG,CAAC,CAAC,SAA8B,CAAC;gBAC5C,IAAI,EAAE,KAAK,EAAE;oBAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvC,OAAQ,CAAC,CAAC,KAA2B,GAAI,CAAC,CAAC,KAA2B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClF,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,KAAK,CAAC,MAAuC,CAAC;YAC7D,MAAM,OAAO,GACX,MAAM,KAAK,SAAS;gBAClB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,KAA2B,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACvF,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;YACpD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACrC,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,GAAG,CAAC,OAAO;oBACT,IAAI,KAAK,SAAS,IAAI;oBACpB,UAAU,EAAE,IAAI,CAAC,KAAkD;iBACpE,CAAC;aACL,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,KAAyB;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAA0B,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;YACvD,IAAK,MAAM,CAAC,QAA8B,KAAM,KAAK,CAAC,QAA8B,EAAE,CAAC;gBACrF,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;YAC/B,CAAC;YACD,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7D,CAAC;YACD,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,KAA0B,CAAC,CAAC;YACrE,UAAU,EAAE,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,WAAW;gBACnB,WAAW,EAAE,GAAG,EAAE,CAAC,WAAW,EAA0B;aACzD,CAAC,CAAC;YACH,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACxB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,MAAqB;IACvC,OAAO,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,WAAW,CAAC;AACjF,CAAC;AAED,gEAAgE;AAChE,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/eval-suite-binding.d.ts

@@ -0,0 +1,227 @@
+import type { TupleEnqueueHook } from '@kindgi/authz';
+import type { Scope } from '@kindgi/platform';
+import type { Cursor, ProjectId, TenantId } from '@kindgi/types';
+/**
+ * Caller-plugged surface for the evaluation-harness suite catalog —
+ * part of the admin control plane. Registry-only: publish + read; run
+ * execution + per-kind grader dispatch are out of scope (see
+ * `EvalRunBinding`). Graders — an eval-judge adapter, the HITL bridge,
+ * a sandboxed handler — consume suite definitions from this store at
+ * their own boundary.
+ *
+ * Mirrors `PolicyRegistryBinding` 1:1 — same versioned-CRUD shape,
+ * same tenant-scoped inputs, same opaque cursors, same "unregister a
+ * specific version" semantics.
+ *
+ * The wire shape is intentionally kind-neutral: `evalKind` selects the
+ * shape of `spec`. The route validates the closed set of `kind` values
+ * and the required top-level fields; deeper `spec` shape is the runtime
+ * consumer's responsibility (each kind's grader knows its own contract).
+ *
+ * Cursors are opaque — the binding chooses its encoding (index-based,
+ * `id@version`, ISO timestamps). The API layer only validates that a
+ * cursor round-trips as a string; it never inspects the payload.
+ */
+export interface EvalSuiteRegistryBinding {
+    /**
+     * Cursor-paginated list of eval suites (latest version per id, sorted
+     * by suite id ascending). Optional filters:
+     * - `evalKind` narrows to suites of a single kind (exact match).
+     * - `nameFilter` is a prefix match on the suite id — dotted
+     *   namespaces are the natural filter shape (`acme.accuracy`).
+     */
+    list(input: EvalSuiteListInput): Promise<EvalSuitePage>;
+    /**
+     * Latest version of the given suite id, or `null` if unknown. The
+     * route surfaces `null` as `404 eval-suite-not-found`.
+     */
+    get(input: EvalSuiteGetInput): Promise<EvalSuite | null>;
+    /**
+     * Specific `(suiteId, version)` lookup, or `null` if unknown.
+     * Returns tombstoned versions too (provenance paths).
+     */
+    getVersion(input: EvalSuiteGetVersionInput): Promise<EvalSuite | null>;
+    /**
+     * Head-row existence probe. Lets GET distinguish 410 gone from
+     * 404 not-found.
+     */
+    headExists(input: EvalSuiteGetInput): Promise<boolean>;
+    /**
+     * Cursor-paginated list of versions for a specific suite id. Sort
+     * order is binding-defined (for example ascending semver). Returns an empty page (no error) when the id
+     * is unknown — the route flips that to `404` via a prior `get`.
+     */
+    listVersions(input: EvalSuiteListVersionsInput): Promise<EvalSuitePage>;
+    /**
+     * Publish a validated eval-suite definition. The API route validates
+     * the wire shape (id / tenantId / semver version / kind / spec object)
+     * before calling — the binding receives a well-formed `EvalSuite`.
+     * Bindings MAY reject with `already-registered` when the same
+     * `(suiteId, version)` is re-published; the route maps that to `409`.
+     */
+    publish(input: EvalSuitePublishInput): Promise<EvalSuitePublishOutcome>;
+    /**
+     * Soft-tombstone a specific `(suiteId, version)`. When this leaves
+     * NO active versions, the suite has no latest version — derived
+     * "retired" state (GET returns 410 gone). Publish reactivates.
+     */
+    unregister(input: EvalSuiteUnregisterInput): Promise<EvalSuiteUnregisterOutcome>;
+    /**
+     * Un-tombstone a specific `(suiteId, version)`. Recomputes the
+     * suite's latest version. Restores bytes verbatim (semver hygiene).
+     * Idempotent.
+     */
+    reinstateVersion(input: EvalSuiteReinstateVersionInput): Promise<EvalSuiteReinstateVersionOutcome>;
+}
+/**
+ * Closed set of eval kinds the framework knows about. Extended
+ * additively — new kinds require a spec + validator update in tandem so
+ * the registry never accepts a kind no runtime consumer honors.
+ *
+ * - `accuracy` — metric-based grading against ground truth (input →
+ *   expectedOutput pairs; grader is an eval-judge adapter).
+ * - `pairwise` — A/B comparisons between two agent/flow versions.
+ * - `regression` — compare against a stored baseline run.
+ * - `human-review` — delegates rubric grading to HITL reviewers.
+ * - `benchmark` — reference to a standardized suite (name + version).
+ * - `custom` — caller-supplied evaluator handler.
+ */
+export declare const EVAL_KINDS: readonly ["accuracy", "pairwise", "regression", "human-review", "benchmark", "custom"];
+export type EvalKind = (typeof EVAL_KINDS)[number];
+/**
+ * Wire shape for a single evaluation suite. `spec` is a JSON object
+ * whose shape is dictated by `kind` — the registry treats it as opaque
+ * JSON so new kinds can extend the catalog without touching this file.
+ * Runtime consumers (the eval-judge adapter for `accuracy` / `pairwise`
+ * / `regression`, the HITL bridge for `human-review`, the sandbox
+ * handler for `custom`) deserialize `spec` against their own contract.
+ *
+ * Case data lives IN `spec` (not stored separately) — small enough at
+ * registry scale.
+ */
+export interface EvalSuite {
+    readonly id: string;
+    readonly tenantId: TenantId;
+    readonly version: string;
+    readonly kind: EvalKind;
+    readonly description?: string;
+    readonly spec: Readonly<Record<string, unknown>>;
+}
+export interface EvalSuiteListInput {
+    readonly tenantId: TenantId;
+    readonly limit: number;
+    readonly cursor?: Cursor;
+    /** Exact match on `EvalSuite.kind`. Undefined = no filter. */
+    readonly evalKind?: EvalKind;
+    /** Prefix match on `EvalSuite.id`. Undefined = no filter. */
+    readonly nameFilter?: string;
+    /**
+     * Narrow the list to a specific scope. Absent = no scope narrow
+     * (return every row in the tenant the caller can see — admin/audit
+     * default).
+     *
+     * Content-scoped semantics (this binding):
+     * - `{ kind: 'project', projectId }` — rows in that project.
+     * - `{ kind: 'org', orgId }` — rows in every project belonging to
+     *   that org.
+     * - `{ kind: 'tenant', tenantId }` — every row in the tenant.
+     *
+     * Content rows always belong to a project, so `inherit` has no
+     * effect here.
+     */
+    readonly scope?: Scope;
+    /**
+     * `false` = literal-at-this-scope only (admin/audit view).
+     * `true` (default) = inheritance walk (user-facing view).
+     * No-op for content-scoped bindings (rows only exist at
+     * project-level — there is no upward hierarchy to walk). Kept for
+     * uniformity: scope-aware bindings share one filter shape across the
+     * SDK and OpenAPI schemas.
+     */
+    readonly inherit?: boolean;
+}
+export interface EvalSuiteGetInput {
+    readonly tenantId: TenantId;
+    readonly suiteId: string;
+}
+export interface EvalSuiteGetVersionInput {
+    readonly tenantId: TenantId;
+    readonly suiteId: string;
+    readonly version: string;
+}
+export interface EvalSuiteListVersionsInput {
+    readonly tenantId: TenantId;
+    readonly suiteId: string;
+    readonly limit: number;
+    readonly cursor?: Cursor;
+}
+export interface EvalSuitePublishInput {
+    readonly tenantId: TenantId;
+    /**
+     * Project the eval suite is published into. REQUIRED. Callers that
+     * don't naturally know a projectId
+     * resolve the tenant's Default via
+     * `projectBinding.getDefault(tenantId)` at THEIR layer and thread
+     * the resolved id here — the storage adapter never silently falls
+     * back to Default.
+     */
+    readonly projectId: ProjectId;
+    readonly suite: EvalSuite;
+    /**
+     * REQUIRED. Called inside the binding's write tx after row insert,
+     * receiving the business `suiteId`. Returns tuples for the same tx.
+     * See `AgentPublishInput.enqueueTuples` for the design rationale.
+     */
+    readonly enqueueTuples: TupleEnqueueHook;
+}
+export interface EvalSuiteUnregisterInput {
+    readonly tenantId: TenantId;
+    readonly suiteId: string;
+    readonly version: string;
+}
+export interface EvalSuiteReinstateVersionInput {
+    readonly tenantId: TenantId;
+    readonly suiteId: string;
+    readonly version: string;
+}
+export interface EvalSuitePage {
+    readonly data: readonly EvalSuite[];
+    readonly nextCursor?: Cursor;
+}
+export type EvalSuitePublishOutcome = {
+    readonly kind: 'ok';
+    readonly suiteId: string;
+    readonly version: string;
+} | {
+    readonly kind: 'already-registered';
+    readonly suiteId: string;
+    readonly version: string;
+} | {
+    /**
+     * The caller-supplied `projectId` does not resolve to a real
+     * project row in this tenant. Distinct from
+     * `already-registered` so the route + deployment loop can
+     * surface a clean `400 bad-input` (per the route's error
+     * contract) rather than the misleading `already-registered`
+     * fallback. Implementations detect a missing project at publish
+     * time (a SQL store typically from its foreign-key violation).
+     */
+    readonly kind: 'project-not-found';
+    readonly suiteId: string;
+    readonly version: string;
+    readonly projectId: ProjectId;
+};
+export type EvalSuiteUnregisterOutcome = {
+    readonly unregistered: boolean;
+};
+export type EvalSuiteReinstateVersionOutcome = {
+    readonly kind: 'ok';
+    readonly suiteId: string;
+    readonly version: string;
+    readonly wasTombstoned: boolean;
+} | {
+    readonly kind: 'not-found';
+    readonly suiteId: string;
+    readonly version: string;
+};
+//# sourceMappingURL=eval-suite-binding.d.ts.map

package/dist/eval-suite-binding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-suite-binding.d.ts","sourceRoot":"","sources":["../src/eval-suite-binding.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEjE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,wBAAwB;IACvC;;;;;;OAMG;IACH,IAAI,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACxD;;;OAGG;IACH,GAAG,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACzD;;;OAGG;IACH,UAAU,CAAC,KAAK,EAAE,wBAAwB,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACvE;;;OAGG;IACH,UAAU,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD;;;;OAIG;IACH,YAAY,CAAC,KAAK,EAAE,0BAA0B,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACxE;;;;;;OAMG;IACH,OAAO,CAAC,KAAK,EAAE,qBAAqB,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACxE;;;;OAIG;IACH,UAAU,CAAC,KAAK,EAAE,wBAAwB,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;IACjF;;;;OAIG;IACH,gBAAgB,CACd,KAAK,EAAE,8BAA8B,GACpC,OAAO,CAAC,gCAAgC,CAAC,CAAC;CAC9C;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,UAAU,wFAOb,CAAC;AACX,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB;;;;;;;OAOG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B;;;;;;;OAOG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,aAAa,EAAE,gBAAgB,CAAC;CAC1C;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,SAAS,SAAS,EAAE,CAAC;IACpC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,MAAM,uBAAuB,GAC/B;IACE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,GACD;IACE,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,GACD;IACE;;;;;;;;OAQG;IACH,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;CAC/B,CAAC;AAEN,MAAM,MAAM,0BAA0B,GAAG;IACvC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,gCAAgC,GACxC;IACE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;CACjC,GACD;IACE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC"}

package/dist/eval-suite-binding.js

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+/**
+ * Closed set of eval kinds the framework knows about. Extended
+ * additively — new kinds require a spec + validator update in tandem so
+ * the registry never accepts a kind no runtime consumer honors.
+ *
+ * - `accuracy` — metric-based grading against ground truth (input →
+ *   expectedOutput pairs; grader is an eval-judge adapter).
+ * - `pairwise` — A/B comparisons between two agent/flow versions.
+ * - `regression` — compare against a stored baseline run.
+ * - `human-review` — delegates rubric grading to HITL reviewers.
+ * - `benchmark` — reference to a standardized suite (name + version).
+ * - `custom` — caller-supplied evaluator handler.
+ */
+export const EVAL_KINDS = [
+    'accuracy',
+    'pairwise',
+    'regression',
+    'human-review',
+    'benchmark',
+    'custom',
+];
+//# sourceMappingURL=eval-suite-binding.js.map

package/dist/eval-suite-binding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-suite-binding.js","sourceRoot":"","sources":["../src/eval-suite-binding.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAiFjC;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,UAAU;IACV,UAAU;IACV,YAAY;IACZ,cAAc;IACd,WAAW;IACX,QAAQ;CACA,CAAC"}