@calliopelabs/cli 3.2.0 → 3.3.0

package/dist/judgment/cli.js

@@ -0,0 +1,141 @@
+/** `calliope judge`: typed judgments from the terminal, for scripts and CI. */
+import { parseArgs } from 'node:util';
+import * as fs from 'node:fs';
+import { parseModelFlags } from '../preferences/flags.js';
+import { resolvePreferences } from '../preferences/resolve.js';
+import { evaluate } from './evaluate.js';
+import { runJudgePolicy } from './policy.js';
+import { JudgmentError } from './types.js';
+export const JUDGE_USAGE = `calliope judge --request <file|-> [--provider <name>|typesafe] [--model <id>] [--json]
+calliope judge --questions <file> (--state <text> | --state-file <file|->) [--provider <name>] [--model <id>] [--json]
+calliope judge --policy <rules.json> [--provider <name>] [--model <id>] [--json]   pre-tool policy engine, tool call on stdin`;
+/** Remove `--provider typesafe` / `--provider=typesafe` before chat-provider parsing. */
+function stripTypesafeFlag(args) {
+    const kept = [];
+    let typesafe = false;
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === '--') {
+            kept.push(...args.slice(index));
+            break;
+        }
+        if (arg === '--provider=typesafe' || (arg === '--provider' && args[index + 1] === 'typesafe')) {
+            if (typesafe)
+                throw new Error('Duplicate --provider option');
+            typesafe = true;
+            if (arg === '--provider')
+                index++;
+            continue;
+        }
+        kept.push(arg);
+    }
+    return { typesafe, args: kept };
+}
+function readSource(path, stdin) {
+    return path === '-' ? stdin() : fs.readFileSync(path, 'utf8');
+}
+function parseJson(label, text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new JudgmentError('invalid-request', `${label} is not valid JSON.`);
+    }
+}
+/** Build the request from flags; the state may be raw text or a JSON document. */
+export function loadRequest(values, stdin) {
+    const sources = [values.request, values.questions].filter(Boolean).length;
+    if (sources !== 1)
+        throw new JudgmentError('invalid-request', 'Pass exactly one of --request or --questions.');
+    if (values.request) {
+        if (values.state !== undefined || values['state-file'] !== undefined)
+            throw new JudgmentError('invalid-request', '--request already carries the state.');
+        return parseJson('Request', readSource(values.request, stdin));
+    }
+    const stateFlags = [values.state, values['state-file']].filter(v => v !== undefined).length;
+    if (stateFlags !== 1)
+        throw new JudgmentError('invalid-request', 'Pass exactly one of --state or --state-file with --questions.');
+    const raw = values.state !== undefined ? values.state : readSource(values['state-file'], stdin);
+    let state = raw;
+    if (values['state-file'] !== undefined && /\.json$/i.test(values['state-file']))
+        state = parseJson('State file', raw);
+    return { state, questions: parseJson('Questions', readSource(values.questions, stdin)) };
+}
+function formatAnswer(id, answer) {
+    const dist = (p, legend) => Object.entries(p).map(([key, value]) => `${legend?.[key] ?? key} ${value.toFixed(2)}`).join(' · ');
+    if (answer.type === 'noul')
+        return `${id}\tnoul\t${answer.noul.toFixed(2)}`;
+    if (answer.type === 'choice')
+        return `${id}\tchoice\t${answer.choice}\t(confidence ${answer.confidence.toFixed(2)})\t${dist(answer.probabilities)}`;
+    return `${id}\tscore\t${answer.score.toFixed(2)}\t(confidence ${answer.confidence.toFixed(2)})\t${dist(answer.probabilities, answer.legend)}`;
+}
+export function formatJudgment(response) {
+    const lines = [`${response.provider}/${response.model}  in ${response.usage.input_tokens} out ${response.usage.output_tokens}`];
+    for (const [id, answer] of Object.entries(response.answers))
+        lines.push(formatAnswer(id, answer));
+    return lines.join('\n') + '\n';
+}
+export async function runJudge(rawArgs, options = {}) {
+    const write = options.write ?? ((text) => { process.stdout.write(text); });
+    const stdin = options.stdin ?? (() => fs.readFileSync(0, 'utf8'));
+    let json = false;
+    const fail = (failure, exitCode) => {
+        if (json)
+            write(JSON.stringify({ version: 1, type: 'judgment', ...failure }) + '\n');
+        else
+            write(`${failure.message}\n${failure.error === 'invalid-arguments' ? `\n${JUDGE_USAGE}\n` : ''}`);
+        return exitCode;
+    };
+    // `typesafe` is a judgment engine, not a chat provider, so it never reaches
+    // preference validation; every other --provider/--model flag resolves normally.
+    let native, preference, rest;
+    try {
+        native = stripTypesafeFlag(rawArgs);
+        ({ preference, args: rest } = parseModelFlags(native.args));
+    }
+    catch (error) {
+        return fail({ error: 'invalid-arguments', message: error instanceof Error ? error.message : String(error) }, 2);
+    }
+    if (native.typesafe && preference.provider !== undefined)
+        return fail({ error: 'invalid-arguments', message: 'Duplicate --provider option' }, 2);
+    let parsed;
+    try {
+        parsed = parseArgs({ args: rest, allowPositionals: false, options: {
+                json: { type: 'boolean' }, request: { type: 'string' }, questions: { type: 'string' }, state: { type: 'string' }, 'state-file': { type: 'string' }, policy: { type: 'string' },
+            } });
+    }
+    catch (error) {
+        return fail({ error: 'invalid-arguments', message: error instanceof Error ? error.message : String(error) }, 2);
+    }
+    json = !!parsed.values.json;
+    let provider, model;
+    if (native.typesafe) {
+        provider = 'typesafe';
+        model = preference.model ?? undefined;
+    }
+    else {
+        const resolved = resolvePreferences(options.cwd ?? process.cwd(), { turn: preference });
+        provider = resolved.provider;
+        model = resolved.model;
+    }
+    // The policy engine owns its own exit codes and fails closed on every error.
+    if (parsed.values.policy !== undefined) {
+        if (['request', 'questions', 'state', 'state-file'].some(flag => parsed.values[flag] !== undefined))
+            return fail({ error: 'invalid-arguments', message: '--policy takes the pending tool call on stdin and its questions from the rules file.' }, 2);
+        return runJudgePolicy(String(parsed.values.policy), { provider, model, signal: options.signal, json, stdin, write, ...(options.writeErr ? { writeErr: options.writeErr } : {}) });
+    }
+    try {
+        const request = loadRequest(parsed.values, stdin);
+        const response = await evaluate(request, { provider, model, signal: options.signal });
+        write(json ? JSON.stringify({ version: 1, type: 'judgment', ...response }) + '\n' : formatJudgment(response));
+        return 0;
+    }
+    catch (error) {
+        if (options.signal?.aborted)
+            return fail({ error: 'cancelled', message: 'Judgment cancelled.' }, 130);
+        if (error instanceof JudgmentError)
+            return fail({ error: error.code, message: error.message }, error.code === 'invalid-request' ? 2 : 1);
+        return fail({ error: 'unavailable', message: error instanceof Error ? error.message : String(error) }, 1);
+    }
+}
+//# sourceMappingURL=cli.js.map

package/dist/judgment/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/judgment/cli.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,aAAa,EAA2D,MAAM,YAAY,CAAC;AAEpG,MAAM,CAAC,MAAM,WAAW,GAAG;;8HAEmG,CAAC;AAa/H,yFAAyF;AACzF,SAAS,iBAAiB,CAAC,IAAc;IACvC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;QACzB,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,MAAM;QAAC,CAAC;QAC7D,IAAI,GAAG,KAAK,qBAAqB,IAAI,CAAC,GAAG,KAAK,YAAY,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU,CAAC,EAAE,CAAC;YAC9F,IAAI,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7D,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,GAAG,KAAK,YAAY;gBAAE,KAAK,EAAE,CAAC;YAClC,SAAS;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,KAAmB;IACnD,OAAO,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAY;IAC5C,IAAI,CAAC;QAAC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;IAChC,MAAM,CAAC;QAAC,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,GAAG,KAAK,qBAAqB,CAAC,CAAC;IAAC,CAAC;AACtF,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,WAAW,CAAC,MAAuF,EAAE,KAAmB;IACtI,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,OAAO,KAAK,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,+CAA+C,CAAC,CAAC;IAC/G,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,SAAS;YAAE,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,sCAAsC,CAAC,CAAC;QACzJ,OAAO,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,UAAU,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAC5F,IAAI,UAAU,KAAK,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,+DAA+D,CAAC,CAAC;IAClI,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAE,EAAE,KAAK,CAAC,CAAC;IACjG,IAAI,KAAK,GAAY,GAAG,CAAC;IACzB,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAAE,KAAK,GAAG,SAAS,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACtH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,SAAU,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,YAAY,CAAC,EAAU,EAAE,MAAc;IAC9C,MAAM,IAAI,GAAG,CAAC,CAAyB,EAAE,MAA+B,EAAE,EAAE,CAC1E,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrG,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,EAAE,WAAW,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5E,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,GAAG,EAAE,aAAa,MAAM,CAAC,MAAM,iBAAiB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IACpJ,OAAO,GAAG,EAAE,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;AAChJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAA0B;IACvD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,QAAQ,QAAQ,CAAC,KAAK,CAAC,YAAY,QAAQ,QAAQ,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;IAChI,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAClG,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAiB,EAAE,OAAO,GAAwB,EAAE;IACjF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,MAAM,IAAI,GAAG,CAAC,OAAqB,EAAE,QAAgB,EAAU,EAAE;QAC/D,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;;YAChF,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,KAAK,KAAK,mBAAmB,CAAC,CAAC,CAAC,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvG,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IACF,4EAA4E;IAC5E,gFAAgF;IAChF,IAAI,MAA4C,EAAE,UAA4D,EAAE,IAAc,CAAC;IAC/H,IAAI,CAAC;QAAC,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAAC,CAAC;IACzG,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAAC,CAAC;IAClI,IAAI,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,6BAA6B,EAAE,EAAE,CAAC,CAAC,CAAC;IACjJ,IAAI,MAAiG,CAAC;IACtG,IAAI,CAAC;QACH,MAAM,GAAG,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;gBACjE,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC/K,EAAE,CAAC,CAAC;IACP,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAAC,CAAC;IACpI,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAE5B,IAAI,QAAwB,EAAE,KAAyB,CAAC;IACxD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAAC,QAAQ,GAAG,UAAU,CAAC;QAAC,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,SAAS,CAAC;IAAC,CAAC;SACjF,CAAC;QACJ,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QACxF,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;IACvD,CAAC;IAED,6EAA6E;IAC7E,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC;YACjG,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,sFAAsF,EAAE,EAAE,CAAC,CAAC,CAAC;QAClJ,OAAO,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACpL,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,MAA2C,EAAE,KAAK,CAAC,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9G,OAAO,CAAC,CAAC;IACX,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,qBAAqB,EAAE,EAAE,GAAG,CAAC,CAAC;QACtG,IAAI,KAAK,YAAY,aAAa;YAAE,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,IAAI,KAAK,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzI,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5G,CAAC;AACH,CAAC"}

package/dist/judgment/evaluate.d.ts

@@ -0,0 +1,47 @@
+import { type ChatOptions } from '../providers/index.js';
+import { type Message } from '../types.js';
+import { type Answer, type JudgmentEngine, type JudgmentRequest, type JudgmentResponse, type Question } from './types.js';
+/**
+ * The prompted engine's `chat()` call admits the same bounded-execution
+ * controls any other in-runtime caller uses: a reviewed output limit and,
+ * when the caller supplies one, an `attemptBudget` that reserves and settles
+ * the request against its ledger. Neither has an effect on the native
+ * TypeSafe engine, which never calls `chat()`; `evaluate()` rejects them for
+ * `provider: 'typesafe'` rather than silently admitting an unaccounted call.
+ */
+export interface EvaluateOptions extends Pick<ChatOptions, 'maxOutputTokens' | 'bounded' | 'attemptBudget'> {
+    provider?: JudgmentEngine;
+    model?: string;
+    signal?: AbortSignal;
+    /** Injected for tests; defaults to global fetch. */
+    fetch?: typeof fetch;
+}
+export declare const TYPESAFE_DEFAULT_MODEL = "jev-latest";
+/** Reject malformed input before any network call; returns a typed copy. */
+export declare function validateRequest(input: unknown): JudgmentRequest;
+export declare function buildMessages(request: JudgmentRequest): {
+    messages: Message[];
+    ids: string[];
+};
+/** Grammar schema for backends that constrain output (Ollama `format`); others ignore it. */
+export declare function buildOutputSchema(request: JudgmentRequest): Record<string, unknown>;
+/** Accept a bare object, or one wrapped in fences/prose; reject anything else. */
+export declare function parseModelOutput(text: string): Record<string, unknown>;
+/** Clamp, fill missing outcomes with 0 and renormalize; all-zero is a model failure. */
+export declare function normalizeDistribution(id: string, keys: string[], raw: unknown): Record<string, number>;
+/**
+ * How far the leading outcome sits above chance: (max - 1/n) / (1 - 1/n).
+ * 1 when one outcome holds all mass, 0 when the distribution is uniform. This
+ * tracks the confidence TypeSafe reports for the same distributions closely,
+ * though their exact statistic is unpublished.
+ */
+export declare function confidenceOf(probabilities: Record<string, number>): number;
+/** Derive the typed answer from a normalized distribution. */
+export declare function toAnswer(question: Question, probabilities: Record<string, number>): Answer;
+export declare function typesafeCredentials(): {
+    apiKey?: string;
+    baseUrl: string;
+};
+/** Evaluate every question in one request; answers come back under the caller's ids. */
+export declare function evaluate(input: unknown, options?: EvaluateOptions): Promise<JudgmentResponse>;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/judgment/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/judgment/evaluate.ts"],"names":[],"mappings":"AASA,OAAO,EAAwB,KAAK,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAE/E,OAAO,EAAoC,KAAK,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAiB,KAAK,MAAM,EAAE,KAAK,cAAc,EAAE,KAAK,eAAe,EAAE,KAAK,gBAAgB,EAAE,KAAK,QAAQ,EAAsB,MAAM,YAAY,CAAC;AAE7J;;;;;;;GAOG;AACH,MAAM,WAAW,eAAgB,SAAQ,IAAI,CAAC,WAAW,EAAE,iBAAiB,GAAG,SAAS,GAAG,eAAe,CAAC;IACzG,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oDAAoD;IACpD,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED,eAAO,MAAM,sBAAsB,eAAe,CAAC;AA6DnD,4EAA4E;AAC5E,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe,CAc/D;AAmCD,wBAAgB,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG;IAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAA;CAAE,CAK9F;AAED,6FAA6F;AAC7F,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAYnF;AAED,kFAAkF;AAClF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAQtE;AAED,wFAAwF;AACxF,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAStG;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAK1E;AAQD,8DAA8D;AAC9D,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAU1F;AAiCD,wBAAgB,mBAAmB,IAAI;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAM1E;AAuFD,wFAAwF;AACxF,wBAAsB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAUvG"}

package/dist/judgment/evaluate.js

@@ -0,0 +1,369 @@
+/**
+ * Judgment evaluation over two engines:
+ *  - any chat backend: the model is prompted for a probability distribution per
+ *    question and the answer types are derived here (prompted engine);
+ *  - `typesafe`: the native TypeSafe endpoint, whose System One models return
+ *    calibrated distributions directly.
+ * Both return the same JudgmentResponse contract.
+ */
+import * as config from '../config.js';
+import { chat, selectProvider } from '../providers/index.js';
+import { throwIfCancelled } from '../cancellation.js';
+import { DEFAULT_MODELS } from '../types.js';
+import { JudgmentError } from './types.js';
+export const TYPESAFE_DEFAULT_MODEL = 'jev-latest';
+const TYPESAFE_DEFAULT_BASE_URL = 'https://api.typesafe.ai/v1';
+const MAX_QUESTIONS = 64;
+const MAX_OPTIONS = 32;
+const ID_PATTERN = /^[A-Za-z0-9_.-]{1,64}$/;
+// ---------------------------------------------------------------------------
+// Validation
+function invalid(message) { throw new JudgmentError('invalid-request', message); }
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function validateInstructions(id, value) {
+    if (typeof value === 'string') {
+        if (!value.trim())
+            invalid(`Question "${id}" has empty instructions.`);
+        return;
+    }
+    if (isRecord(value) || Array.isArray(value))
+        return;
+    invalid(`Question "${id}" needs string, object or array instructions.`);
+}
+function validateQuestion(id, value) {
+    if (!isRecord(value))
+        invalid(`Question "${id}" must be an object.`);
+    validateInstructions(id, value.instructions);
+    switch (value.type) {
+        case 'noul': {
+            const criteria = value.criteria;
+            if (criteria !== undefined) {
+                if (!isRecord(criteria))
+                    invalid(`Question "${id}": noul criteria must be an object with optional "true"/"false" strings.`);
+                for (const key of Object.keys(criteria)) {
+                    if (key !== 'true' && key !== 'false')
+                        invalid(`Question "${id}": noul criteria only accept "true" and "false" keys.`);
+                    if (typeof criteria[key] !== 'string')
+                        invalid(`Question "${id}": noul criteria "${key}" must be a string.`);
+                }
+            }
+            return value;
+        }
+        case 'choice': {
+            const criteria = value.criteria;
+            if (!isRecord(criteria))
+                invalid(`Question "${id}": choice criteria must map each option to a description or null.`);
+            const options = Object.keys(criteria);
+            if (options.length < 2)
+                invalid(`Question "${id}": choice needs at least two options.`);
+            if (options.length > MAX_OPTIONS)
+                invalid(`Question "${id}": choice allows at most ${MAX_OPTIONS} options.`);
+            for (const option of options) {
+                if (!option.trim())
+                    invalid(`Question "${id}": choice options must be non-empty strings.`);
+                const description = criteria[option];
+                if (description !== null && typeof description !== 'string')
+                    invalid(`Question "${id}": option "${option}" must describe with a string or null.`);
+            }
+            return value;
+        }
+        case 'score': {
+            const criteria = value.criteria;
+            if (!Array.isArray(criteria) || criteria.length < 2)
+                invalid(`Question "${id}": score criteria must be an ordered array of at least two level descriptions.`);
+            if (criteria.length > MAX_OPTIONS)
+                invalid(`Question "${id}": score allows at most ${MAX_OPTIONS} levels.`);
+            for (const level of criteria)
+                if (typeof level !== 'string' || !level.trim())
+                    invalid(`Question "${id}": every score level must be a non-empty string.`);
+            return value;
+        }
+        default:
+            invalid(`Question "${id}" has unknown type ${JSON.stringify(value.type)}; expected noul, choice or score.`);
+    }
+}
+/** Reject malformed input before any network call; returns a typed copy. */
+export function validateRequest(input) {
+    if (!isRecord(input))
+        invalid('Request must be an object with "state" and "questions".');
+    if (input.state === undefined || input.state === null)
+        invalid('Request "state" is required.');
+    if (typeof input.state === 'string' && !input.state.trim())
+        invalid('Request "state" must not be empty.');
+    if (!isRecord(input.questions))
+        invalid('Request "questions" must be a map of question id to question.');
+    const ids = Object.keys(input.questions);
+    if (!ids.length)
+        invalid('Request needs at least one question.');
+    if (ids.length > MAX_QUESTIONS)
+        invalid(`Request allows at most ${MAX_QUESTIONS} questions.`);
+    const questions = {};
+    for (const id of ids) {
+        if (!ID_PATTERN.test(id))
+            invalid(`Question id ${JSON.stringify(id)} must match ${ID_PATTERN}.`);
+        questions[id] = validateQuestion(id, input.questions[id]);
+    }
+    return { state: input.state, questions };
+}
+// ---------------------------------------------------------------------------
+// Prompted engine
+/** The keys the model must assign probability to, in order. */
+function outcomeKeys(question) {
+    if (question.type === 'noul')
+        return ['yes', 'no'];
+    if (question.type === 'choice')
+        return Object.keys(question.criteria);
+    return question.criteria.map((_, index) => String(index));
+}
+/** Question ids are replaced by positional ids so they never reach the model. */
+function positionalQuestions(questions) {
+    const ids = Object.keys(questions), prompt = {};
+    ids.forEach((id, index) => {
+        const question = questions[id];
+        const entry = { type: question.type, instructions: question.instructions };
+        if (question.type === 'noul') {
+            entry.outcomes = { yes: question.criteria?.true ?? 'The answer is yes', no: question.criteria?.false ?? 'The answer is no' };
+        }
+        else if (question.type === 'choice') {
+            entry.outcomes = Object.fromEntries(Object.entries(question.criteria).map(([option, description]) => [option, description ?? option]));
+        }
+        else {
+            entry.outcomes = Object.fromEntries(question.criteria.map((level, i) => [String(i), level]));
+            entry.note = 'Outcomes are ordered levels from lowest to highest.';
+        }
+        prompt[`q${index + 1}`] = entry;
+    });
+    return { ids, prompt };
+}
+const SYSTEM_PROMPT = `You are a calibrated judgment engine. You do not converse, explain or reason aloud.
+You receive STATE (text or JSON) and QUESTIONS. For each question, assign a probability to every listed outcome key, reflecting how likely that outcome is the correct judgment given only the STATE. Probabilities for one question must sum to 1. Use the full range: near-certain judgments get near 0 or 1, genuinely ambiguous ones spread probability.
+Reply with exactly one JSON object and nothing else: {"q1": {"<outcome>": <probability>, ...}, "q2": {...}}. Include every question id and every outcome key. No prose, no code fences.`;
+export function buildMessages(request) {
+    const { ids, prompt } = positionalQuestions(request.questions);
+    const state = typeof request.state === 'string' ? request.state : JSON.stringify(request.state, null, 2);
+    const user = `STATE:\n${state}\n\nQUESTIONS:\n${JSON.stringify(prompt, null, 2)}`;
+    return { messages: [{ role: 'system', content: SYSTEM_PROMPT }, { role: 'user', content: user }], ids };
+}
+/** Grammar schema for backends that constrain output (Ollama `format`); others ignore it. */
+export function buildOutputSchema(request) {
+    const properties = {};
+    Object.values(request.questions).forEach((question, index) => {
+        const keys = outcomeKeys(question);
+        properties[`q${index + 1}`] = {
+            type: 'object',
+            properties: Object.fromEntries(keys.map(key => [key, { type: 'number' }])),
+            required: keys,
+            additionalProperties: false,
+        };
+    });
+    return { type: 'object', properties, required: Object.keys(properties), additionalProperties: false };
+}
+/** Accept a bare object, or one wrapped in fences/prose; reject anything else. */
+export function parseModelOutput(text) {
+    const start = text.indexOf('{'), end = text.lastIndexOf('}');
+    if (start < 0 || end <= start)
+        throw new JudgmentError('model-output', 'The model returned no JSON object.');
+    let parsed;
+    try {
+        parsed = JSON.parse(text.slice(start, end + 1));
+    }
+    catch {
+        throw new JudgmentError('model-output', 'The model returned malformed JSON.');
+    }
+    if (!isRecord(parsed))
+        throw new JudgmentError('model-output', 'The model returned JSON that is not an object.');
+    return parsed;
+}
+/** Clamp, fill missing outcomes with 0 and renormalize; all-zero is a model failure. */
+export function normalizeDistribution(id, keys, raw) {
+    if (!isRecord(raw))
+        throw new JudgmentError('model-output', `No distribution returned for question "${id}".`);
+    const values = keys.map(key => {
+        const value = raw[key];
+        return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0;
+    });
+    const total = values.reduce((sum, value) => sum + value, 0);
+    if (total <= 0)
+        throw new JudgmentError('model-output', `Question "${id}" received no usable probabilities.`);
+    return Object.fromEntries(keys.map((key, index) => [key, values[index] / total]));
+}
+/**
+ * How far the leading outcome sits above chance: (max - 1/n) / (1 - 1/n).
+ * 1 when one outcome holds all mass, 0 when the distribution is uniform. This
+ * tracks the confidence TypeSafe reports for the same distributions closely,
+ * though their exact statistic is unpublished.
+ */
+export function confidenceOf(probabilities) {
+    const values = Object.values(probabilities), n = values.length;
+    if (n < 2)
+        return 1;
+    const chance = 1 / n;
+    return round(Math.max(0, Math.min(1, (Math.max(...values) - chance) / (1 - chance))));
+}
+function round(value) { return Math.round(value * 1e4) / 1e4; }
+function roundAll(probabilities) {
+    return Object.fromEntries(Object.entries(probabilities).map(([key, value]) => [key, round(value)]));
+}
+/** Derive the typed answer from a normalized distribution. */
+export function toAnswer(question, probabilities) {
+    if (question.type === 'noul')
+        return { type: 'noul', noul: round(probabilities.yes ?? 0) };
+    const confidence = confidenceOf(probabilities);
+    if (question.type === 'choice') {
+        const choice = Object.entries(probabilities).reduce((best, entry) => (entry[1] > best[1] ? entry : best))[0];
+        return { type: 'choice', choice, probabilities: roundAll(probabilities), confidence };
+    }
+    const score = Object.entries(probabilities).reduce((sum, [level, p]) => sum + Number(level) * p, 0);
+    const legend = Object.fromEntries(question.criteria.map((level, index) => [String(index), level]));
+    return { type: 'score', score: round(score), legend, probabilities: roundAll(probabilities), confidence };
+}
+async function evaluatePrompted(request, provider, model, options) {
+    const { messages, ids } = buildMessages(request);
+    // Mirror chat()'s own resolution so the response names the backend that answered.
+    const actualProvider = selectProvider(provider), actualModel = model || DEFAULT_MODELS[actualProvider];
+    const chatOptions = {
+        signal: options.signal,
+        format: buildOutputSchema(request),
+        selectionMode: provider === 'auto' ? 'auto' : 'explicit',
+        maxOutputTokens: options.maxOutputTokens,
+        bounded: options.bounded,
+        attemptBudget: options.attemptBudget,
+    };
+    const response = await chat(provider, messages, [], model, undefined, undefined, chatOptions);
+    if (response.finishReason === 'error' || response.errorCode === 'refusal')
+        throw new JudgmentError('model-output', 'The model did not complete the judgment request.');
+    const parsed = parseModelOutput(response.content);
+    const answers = {};
+    ids.forEach((id, index) => {
+        const question = request.questions[id];
+        answers[id] = toAnswer(question, normalizeDistribution(id, outcomeKeys(question), parsed[`q${index + 1}`]));
+    });
+    return {
+        provider: actualProvider,
+        model: actualModel,
+        answers,
+        usage: { input_tokens: response.usage?.inputTokens ?? 0, output_tokens: response.usage?.outputTokens ?? 0 },
+    };
+}
+// ---------------------------------------------------------------------------
+// Native TypeSafe engine
+export function typesafeCredentials() {
+    const stored = config.getProviderCred('typesafe');
+    return {
+        apiKey: process.env.TYPESAFE_API_KEY || stored.apiKey,
+        baseUrl: (process.env.TYPESAFE_BASE_URL || stored.baseUrl || TYPESAFE_DEFAULT_BASE_URL).replace(/\/+$/, ''),
+    };
+}
+function expectNumber(id, value, field) {
+    if (typeof value !== 'number' || !Number.isFinite(value))
+        throw new JudgmentError('model-output', `TypeSafe answer "${id}" is missing numeric "${field}".`);
+    return value;
+}
+function expectDistribution(id, value) {
+    if (!isRecord(value))
+        throw new JudgmentError('model-output', `TypeSafe answer "${id}" is missing "probabilities".`);
+    return Object.fromEntries(Object.entries(value).map(([key, p]) => [key, expectNumber(id, p, `probabilities.${key}`)]));
+}
+/** Keep the native answer, validating shape against the question that asked for it. */
+function nativeAnswer(id, question, raw) {
+    if (!isRecord(raw) || raw.type !== question.type)
+        throw new JudgmentError('model-output', `TypeSafe answer "${id}" does not match its ${question.type} question.`);
+    if (question.type === 'noul')
+        return { type: 'noul', noul: expectNumber(id, raw.noul, 'noul') };
+    if (question.type === 'choice') {
+        if (typeof raw.choice !== 'string' || !(raw.choice in question.criteria))
+            throw new JudgmentError('model-output', `TypeSafe answer "${id}" chose an option outside the criteria.`);
+        return { type: 'choice', choice: raw.choice, probabilities: expectDistribution(id, raw.probabilities), confidence: expectNumber(id, raw.confidence, 'confidence') };
+    }
+    const legend = isRecord(raw.legend) ? Object.fromEntries(Object.entries(raw.legend).map(([k, v]) => [k, String(v)])) : Object.fromEntries(question.criteria.map((level, i) => [String(i), level]));
+    return { type: 'score', score: expectNumber(id, raw.score, 'score'), legend, probabilities: expectDistribution(id, raw.probabilities), confidence: expectNumber(id, raw.confidence, 'confidence') };
+}
+const TYPESAFE_MAX_ATTEMPTS = 4;
+const TYPESAFE_MAX_DELAY_MS = 30000;
+/** Honor `retry-after` when present, else exponential backoff from 1s. */
+function retryDelayMs(response, attempt) {
+    const header = Number(response.headers.get('retry-after'));
+    const delay = Number.isFinite(header) && header > 0 ? header * 1000 : 1000 * 2 ** (attempt - 1);
+    return Math.min(delay, TYPESAFE_MAX_DELAY_MS);
+}
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => { signal?.removeEventListener('abort', abort); resolve(); }, ms);
+        const abort = () => { clearTimeout(timer); reject(signal?.reason ?? new Error('Cancelled')); };
+        if (signal?.aborted)
+            abort();
+        else
+            signal?.addEventListener('abort', abort, { once: true });
+    });
+}
+async function evaluateTypesafe(request, model, options) {
+    const { apiKey, baseUrl } = typesafeCredentials();
+    if (!apiKey)
+        throw new JudgmentError('unavailable', 'TypeSafe needs TYPESAFE_API_KEY (or a stored typesafe credential).');
+    const doFetch = options.fetch ?? fetch;
+    const payload = JSON.stringify({ state: request.state, model, questions: request.questions });
+    let response;
+    // Rate limits (429) and overload (529) are documented as retry-with-backoff.
+    for (let attempt = 1;; attempt++) {
+        throwIfCancelled(options.signal);
+        try {
+            response = await doFetch(`${baseUrl}/systemone`, {
+                method: 'POST',
+                headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },
+                body: payload,
+                signal: options.signal,
+            });
+        }
+        catch (error) {
+            throwIfCancelled(options.signal);
+            throw new JudgmentError('unavailable', `TypeSafe request failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if ((response.status !== 429 && response.status !== 529) || attempt >= TYPESAFE_MAX_ATTEMPTS)
+            break;
+        await sleep(retryDelayMs(response, attempt), options.signal);
+    }
+    if (!response.ok) {
+        const code = response.status === 422 ? 'invalid-request' : 'unavailable';
+        let detail = '';
+        try {
+            detail = (await response.text()).slice(0, 300);
+        }
+        catch { /* body is optional */ }
+        throw new JudgmentError(code, `TypeSafe returned HTTP ${response.status}${detail ? `: ${detail}` : ''}`);
+    }
+    let body;
+    try {
+        body = await response.json();
+    }
+    catch {
+        throw new JudgmentError('model-output', 'TypeSafe returned a non-JSON body.');
+    }
+    if (!isRecord(body) || !isRecord(body.answers))
+        throw new JudgmentError('model-output', 'TypeSafe response is missing "answers".');
+    const answers = {};
+    for (const [id, question] of Object.entries(request.questions))
+        answers[id] = nativeAnswer(id, question, body.answers[id]);
+    const usage = isRecord(body.usage) ? body.usage : {};
+    return {
+        provider: 'typesafe',
+        model: typeof body.model === 'string' ? body.model : model,
+        answers,
+        usage: { input_tokens: typeof usage.input_tokens === 'number' ? usage.input_tokens : 0, output_tokens: typeof usage.output_tokens === 'number' ? usage.output_tokens : 0 },
+    };
+}
+// ---------------------------------------------------------------------------
+/** Evaluate every question in one request; answers come back under the caller's ids. */
+export async function evaluate(input, options = {}) {
+    const request = validateRequest(input);
+    throwIfCancelled(options.signal);
+    const provider = options.provider ?? 'auto';
+    if (provider === 'typesafe') {
+        if (options.attemptBudget || options.bounded || options.maxOutputTokens !== undefined)
+            invalid('Bounded execution controls (maxOutputTokens, bounded, attemptBudget) have no effect on the typesafe engine, which never calls chat().');
+        return evaluateTypesafe(request, options.model ?? TYPESAFE_DEFAULT_MODEL, options);
+    }
+    return evaluatePrompted(request, provider, options.model, options);
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/judgment/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/judgment/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,MAAM,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,cAAc,EAAoB,MAAM,uBAAuB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAkC,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAoH,MAAM,YAAY,CAAC;AAkB7J,MAAM,CAAC,MAAM,sBAAsB,GAAG,YAAY,CAAC;AACnD,MAAM,yBAAyB,GAAG,4BAA4B,CAAC;AAC/D,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,UAAU,GAAG,wBAAwB,CAAC;AAE5C,8EAA8E;AAC9E,aAAa;AAEb,SAAS,OAAO,CAAC,OAAe,IAAW,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjG,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,oBAAoB,CAAC,EAAU,EAAE,KAAc;IACtD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,aAAa,EAAE,2BAA2B,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IAClH,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO;IACpD,OAAO,CAAC,aAAa,EAAE,+CAA+C,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAU,EAAE,KAAc;IAClD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,aAAa,EAAE,sBAAsB,CAAC,CAAC;IACrE,oBAAoB,CAAC,EAAE,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC7C,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,MAAM,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;YAChC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAAE,OAAO,CAAC,aAAa,EAAE,0EAA0E,CAAC,CAAC;gBAC5H,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACxC,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO;wBAAE,OAAO,CAAC,aAAa,EAAE,uDAAuD,CAAC,CAAC;oBACvH,IAAI,OAAO,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ;wBAAE,OAAO,CAAC,aAAa,EAAE,qBAAqB,GAAG,qBAAqB,CAAC,CAAC;gBAC/G,CAAC;YACH,CAAC;YACD,OAAO,KAA4B,CAAC;QACtC,CAAC;QACD,KAAK,QAAQ,EAAE,CAAC;YACd,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO,CAAC,aAAa,EAAE,mEAAmE,CAAC,CAAC;YACrH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,aAAa,EAAE,uCAAuC,CAAC,CAAC;YACxF,IAAI,OAAO,CAAC,MAAM,GAAG,WAAW;gBAAE,OAAO,CAAC,aAAa,EAAE,4BAA4B,WAAW,WAAW,CAAC,CAAC;YAC7G,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;oBAAE,OAAO,CAAC,aAAa,EAAE,8CAA8C,CAAC,CAAC;gBAC3F,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,WAAW,KAAK,IAAI,IAAI,OAAO,WAAW,KAAK,QAAQ;oBAAE,OAAO,CAAC,aAAa,EAAE,cAAc,MAAM,wCAAwC,CAAC,CAAC;YACpJ,CAAC;YACD,OAAO,KAA4B,CAAC;QACtC,CAAC;QACD,KAAK,OAAO,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,aAAa,EAAE,gFAAgF,CAAC,CAAC;YAC9J,IAAI,QAAQ,CAAC,MAAM,GAAG,WAAW;gBAAE,OAAO,CAAC,aAAa,EAAE,2BAA2B,WAAW,UAAU,CAAC,CAAC;YAC5G,KAAK,MAAM,KAAK,IAAI,QAAQ;gBAAE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;oBAAE,OAAO,CAAC,aAAa,EAAE,kDAAkD,CAAC,CAAC;YACzJ,OAAO,KAA4B,CAAC;QACtC,CAAC;QACD;YACE,OAAO,CAAC,aAAa,EAAE,sBAAsB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAChH,CAAC;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,yDAAyD,CAAC,CAAC;IACzF,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAC/F,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;QAAE,OAAO,CAAC,oCAAoC,CAAC,CAAC;IAC1G,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,+DAA+D,CAAC,CAAC;IACzG,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,CAAC,GAAG,CAAC,MAAM;QAAE,OAAO,CAAC,sCAAsC,CAAC,CAAC;IACjE,IAAI,GAAG,CAAC,MAAM,GAAG,aAAa;QAAE,OAAO,CAAC,0BAA0B,aAAa,aAAa,CAAC,CAAC;IAC9F,MAAM,SAAS,GAA6B,EAAE,CAAC;IAC/C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,eAAe,UAAU,GAAG,CAAC,CAAC;QACjG,SAAS,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,EAAE,EAAE,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;AAC3C,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAElB,+DAA+D;AAC/D,SAAS,WAAW,CAAC,QAAkB;IACrC,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACnD,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,OAAO,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,iFAAiF;AACjF,SAAS,mBAAmB,CAAC,SAAmC;IAC9D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,GAA4B,EAAE,CAAC;IACzE,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAE,CAAC;QAChC,MAAM,KAAK,GAA4B,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC;QACpG,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,KAAK,CAAC,QAAQ,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,QAAQ,EAAE,IAAI,IAAI,mBAAmB,EAAE,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,IAAI,kBAAkB,EAAE,CAAC;QAC/H,CAAC;aAAM,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QACzI,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7F,KAAK,CAAC,IAAI,GAAG,qDAAqD,CAAC;QACrE,CAAC;QACD,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,aAAa,GAAG;;wLAEkK,CAAC;AAEzL,MAAM,UAAU,aAAa,CAAC,OAAwB;IACpD,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,mBAAmB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACzG,MAAM,IAAI,GAAG,WAAW,KAAK,mBAAmB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;IAClF,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;AAC1G,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,iBAAiB,CAAC,OAAwB;IACxD,MAAM,UAAU,GAA4B,EAAE,CAAC;IAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACnC,UAAU,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,GAAG;YAC5B,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC1E,QAAQ,EAAE,IAAI;YACd,oBAAoB,EAAE,KAAK;SAC5B,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;AACxG,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC7D,IAAI,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,oCAAoC,CAAC,CAAC;IAC7G,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAAC,CAAC;IACxD,MAAM,CAAC;QAAC,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,oCAAoC,CAAC,CAAC;IAAC,CAAC;IACxF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,gDAAgD,CAAC,CAAC;IACjH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,qBAAqB,CAAC,EAAU,EAAE,IAAc,EAAE,GAAY;IAC5E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,0CAA0C,EAAE,IAAI,CAAC,CAAC;IAC9G,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5D,IAAI,KAAK,IAAI,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,aAAa,EAAE,qCAAqC,CAAC,CAAC;IAC9G,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,aAAqC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/D,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,IAAY,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;AAE/E,SAAS,QAAQ,CAAC,aAAqC;IACrD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACtG,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,QAAQ,CAAC,QAAkB,EAAE,aAAqC;IAChF,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC;IAC3F,MAAM,UAAU,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;IAC/C,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7G,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,UAAU,EAAE,CAAC;IACxF,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpG,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACnG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,UAAU,EAAE,CAAC;AAC5G,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,OAAwB,EAAE,QAAqB,EAAE,KAAyB,EAAE,OAAwB;IAClI,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACjD,kFAAkF;IAClF,MAAM,cAAc,GAAG,cAAc,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,KAAK,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACvG,MAAM,WAAW,GAAgB;QAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,iBAAiB,CAAC,OAAO,CAAC;QAClC,aAAa,EAAE,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU;QACxD,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,aAAa,EAAE,OAAO,CAAC,aAAa;KACrC,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAC9F,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,kDAAkD,CAAC,CAAC;IACvK,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,EAAE,CAAE,CAAC;QACxC,OAAO,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,qBAAqB,CAAC,EAAE,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9G,CAAC,CAAC,CAAC;IACH,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,KAAK,EAAE,WAAW;QAClB,OAAO;QACP,KAAK,EAAE,EAAE,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,EAAE;KAC5G,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AAEzB,MAAM,UAAU,mBAAmB;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IAClD,OAAO;QACL,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM;QACrD,OAAO,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC,OAAO,IAAI,yBAAyB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;KAC5G,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,EAAU,EAAE,KAAc,EAAE,KAAa;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,oBAAoB,EAAE,yBAAyB,KAAK,IAAI,CAAC,CAAC;IAC5J,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,kBAAkB,CAAC,EAAU,EAAE,KAAc;IACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,oBAAoB,EAAE,+BAA+B,CAAC,CAAC;IACrH,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,EAAE,iBAAiB,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACzH,CAAC;AAED,uFAAuF;AACvF,SAAS,YAAY,CAAC,EAAU,EAAE,QAAkB,EAAE,GAAY;IAChE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,oBAAoB,EAAE,wBAAwB,QAAQ,CAAC,IAAI,YAAY,CAAC,CAAC;IACnK,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;IAChG,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC;YAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,oBAAoB,EAAE,yCAAyC,CAAC,CAAC;QACnL,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,kBAAkB,CAAC,EAAE,EAAE,GAAG,CAAC,aAAa,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,EAAE,CAAC;IACtK,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAE,QAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACtN,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,kBAAkB,CAAC,EAAE,EAAE,GAAG,CAAC,aAAa,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,EAAE,CAAC;AACtM,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAEpC,0EAA0E;AAC1E,SAAS,YAAY,CAAC,QAAkB,EAAE,OAAe;IACvD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IAChG,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,MAAoB;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChG,MAAM,KAAK,GAAG,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/F,IAAI,MAAM,EAAE,OAAO;YAAE,KAAK,EAAE,CAAC;;YAAM,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,OAAwB,EAAE,KAAa,EAAE,OAAwB;IAC/F,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,mBAAmB,EAAE,CAAC;IAClD,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,aAAa,CAAC,aAAa,EAAE,oEAAoE,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9F,IAAI,QAAkB,CAAC;IACvB,6EAA6E;IAC7E,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;QAClC,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,OAAO,YAAY,EAAE;gBAC/C,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAClF,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,IAAI,aAAa,CAAC,aAAa,EAAE,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/H,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,OAAO,IAAI,qBAAqB;YAAE,MAAM;QACpG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC;QACzE,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC;YAAC,MAAM,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;QACxF,MAAM,IAAI,aAAa,CAAC,IAAI,EAAE,0BAA0B,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QAAC,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,oCAAoC,CAAC,CAAC;IAAC,CAAC;IAC9H,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,aAAa,CAAC,cAAc,EAAE,yCAAyC,CAAC,CAAC;IACnI,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3H,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACrD,OAAO;QACL,QAAQ,EAAE,UAAU;QACpB,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK;QAC1D,OAAO;QACP,KAAK,EAAE,EAAE,YAAY,EAAE,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE;KAC3K,CAAC;AACJ,CAAC;AAED,8EAA8E;AAE9E,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAc,EAAE,OAAO,GAAoB,EAAE;IAC1E,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACvC,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC;IAC5C,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC5B,IAAI,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS;YACnF,OAAO,CAAC,uIAAuI,CAAC,CAAC;QACnJ,OAAO,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,IAAI,sBAAsB,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AACrE,CAAC"}

package/dist/judgment/index.d.ts

@@ -0,0 +1,8 @@
+export { evaluate, validateRequest, confidenceOf, TYPESAFE_DEFAULT_MODEL } from './evaluate.js';
+export type { EvaluateOptions } from './evaluate.js';
+export { runJudge, formatJudgment, JUDGE_USAGE } from './cli.js';
+export { runJudgePolicy, validatePolicyRules, decidePolicy } from './policy.js';
+export type { DenyRule, PolicyRules, PolicyVerdict } from './policy.js';
+export { JudgmentError } from './types.js';
+export type { Answer, ChoiceAnswer, ChoiceQuestion, Instructions, JudgmentEngine, JudgmentErrorCode, JudgmentRequest, JudgmentResponse, NoulAnswer, NoulQuestion, Question, ScoreAnswer, ScoreQuestion } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/judgment/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/judgment/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAChG,YAAY,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChF,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,iBAAiB,EAAE,eAAe,EAAE,gBAAgB,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}

package/dist/judgment/index.js

@@ -0,0 +1,5 @@
+export { evaluate, validateRequest, confidenceOf, TYPESAFE_DEFAULT_MODEL } from './evaluate.js';
+export { runJudge, formatJudgment, JUDGE_USAGE } from './cli.js';
+export { runJudgePolicy, validatePolicyRules, decidePolicy } from './policy.js';
+export { JudgmentError } from './types.js';
+//# sourceMappingURL=index.js.map

package/dist/judgment/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/judgment/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEhG,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhF,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}