@dsh-enhanced/assistant-evaluation 0.1.24 → 0.1.30

package/lib/benchmark/holdout.js

@@ -0,0 +1,391 @@
+import { createHash } from 'node:crypto';
+import { acceptanceCanonicalJson } from '@dsh-enhanced/task-acceptance-contract';
+import { HoldoutEvidenceStore, } from './holdout-evidence.js';
+import { holdoutAuthorityKeyId, holdoutEnvelopeDigest, holdoutInputBytes, parseSignedHoldoutFinish, parseSignedHoldoutInput, parseSignedHoldoutManifest, parseSignedHoldoutVerdict, } from './holdout-protocol.js';
+import { runBenchmark } from './runner.js';
+import { BenchmarkError, benchmarkAssert, benchmarkHash, benchmarkObject, benchmarkPlanDigest, benchmarkSchedule, benchmarkSnapshot, benchmarkVersionKeys, parseBenchmarkMetrics, parseBenchmarkPlan, } from './schema.js';
+import { BenchmarkStore } from './store.js';
+const DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024;
+const DEFAULT_DELEGATE_CLOSE_TIMEOUT_MS = 10_000;
+const DEFAULT_PROVIDER_TIMEOUT_MS = 10_000;
+const MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
+const MAX_CLOSE_TIMEOUT_MS = 300_000;
+const CONTENT_TYPE = /^[A-Za-z0-9][A-Za-z0-9!#$&^_.+/-]{0,126}(?:; charset=utf-8)?$/u;
+function same(left, right) {
+    return acceptanceCanonicalJson(left) === acceptanceCanonicalJson(right);
+}
+function exactData(value, fields, label) {
+    benchmarkAssert(value !== null && typeof value === 'object' && !Array.isArray(value)
+        && Object.getPrototypeOf(value) === Object.prototype, `${label} must be a plain object`);
+    const record = value;
+    const keys = Reflect.ownKeys(record);
+    benchmarkAssert(keys.length === fields.length
+        && keys.every(key => typeof key === 'string' && fields.includes(key)), `${label} has unexpected fields`);
+    for (const key of keys) {
+        const descriptor = Object.getOwnPropertyDescriptor(record, key);
+        benchmarkAssert(descriptor.enumerable && Object.hasOwn(descriptor, 'value'), `${label} contains an unsafe property`);
+    }
+    return record;
+}
+function boundedInteger(value, fallback, maximum, label) {
+    const selected = value ?? fallback;
+    benchmarkAssert(typeof selected === 'number' && Number.isSafeInteger(selected)
+        && selected >= 1 && selected <= maximum, `invalid ${label}`);
+    return selected;
+}
+function assertContentType(value, label) {
+    benchmarkAssert(typeof value === 'string' && CONTENT_TYPE.test(value), `invalid ${label}`);
+}
+function validateDatasetPin(value, plan) {
+    const pin = exactData(value, ['id', 'version', 'digest'], 'holdout dataset pin');
+    benchmarkHash(pin.digest);
+    benchmarkAssert(typeof pin.id === 'string' && typeof pin.version === 'string'
+        && same(pin, { id: plan.dataset.id, version: plan.dataset.version, digest: plan.dataset.digest }), 'holdout plan differs from the operator-pinned dataset');
+    return Object.freeze({ id: pin.id, version: pin.version, digest: pin.digest });
+}
+function validateManifest(value, plan, pinnedPublicKey) {
+    const manifest = parseSignedHoldoutManifest(value, pinnedPublicKey);
+    benchmarkAssert(same(manifest.dataset, plan.dataset) && same(manifest.cases, plan.cases), 'signed holdout manifest differs from the operator-pinned plan');
+    return manifest;
+}
+function validateBinding(value) {
+    benchmarkAssert(value !== null && typeof value === 'object'
+        && typeof value.execute === 'function'
+        && typeof value.close === 'function', 'invalid holdout delegate binding');
+    return value;
+}
+function validateDelegateResult(value, request, maxOutputBytes) {
+    const raw = exactData(value, ['output', 'versions', 'metrics', 'executionEvidenceDigest', 'quiescent'], 'holdout delegate result');
+    const output = exactData(raw.output, ['contentType', 'bytes'], 'holdout delegate output');
+    assertContentType(output.contentType, 'holdout output content type');
+    benchmarkAssert(output.bytes instanceof Uint8Array && Object.getPrototypeOf(output.bytes) === Uint8Array.prototype, 'holdout output bytes must be a Uint8Array');
+    const rawOutputBytes = output.bytes;
+    benchmarkAssert(rawOutputBytes.byteLength <= maxOutputBytes, 'holdout output exceeds its byte limit');
+    const outputBytes = Uint8Array.from(rawOutputBytes);
+    const versions = benchmarkObject(benchmarkSnapshot(raw.versions), benchmarkVersionKeys);
+    benchmarkAssert(same(versions, request.variant.versions), 'holdout delegate loaded different versions');
+    const metrics = parseBenchmarkMetrics(raw.metrics);
+    benchmarkAssert(metrics.latencyMs === null && metrics.inputTokens !== null && metrics.outputTokens !== null
+        && metrics.toolCalls !== null && (request.budget.costUsdMicros === null || metrics.costUsdMicros !== null), 'holdout delegate metrics are incomplete or contain adapter latency');
+    benchmarkAssert(metrics.inputTokens <= request.budget.inputTokens && metrics.outputTokens <= request.budget.outputTokens
+        && metrics.toolCalls <= request.budget.toolCalls
+        && (request.budget.costUsdMicros === null || metrics.costUsdMicros <= request.budget.costUsdMicros), 'holdout delegate exceeded a Host budget');
+    benchmarkHash(raw.executionEvidenceDigest);
+    benchmarkAssert(raw.quiescent === true, 'holdout delegate did not prove quiescence');
+    return Object.freeze({
+        outputContentType: output.contentType, outputBytes, rawOutputBytes, metrics,
+        executionEvidenceDigest: raw.executionEvidenceDigest,
+    });
+}
+async function within(promise, milliseconds, message = 'holdout delegate cleanup was not confirmed') {
+    let timer;
+    try {
+        return await Promise.race([promise, new Promise((_resolve, reject) => {
+                timer = setTimeout(() => reject(new BenchmarkError(message)), milliseconds);
+                timer.unref();
+            })]);
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+    }
+}
+async function withinAbort(promise, signal, milliseconds, message) {
+    if (signal?.aborted)
+        throw new BenchmarkError(message);
+    let removeAbort;
+    try {
+        const aborted = signal === undefined ? new Promise(() => { }) : new Promise((_resolve, reject) => {
+            const abort = () => reject(new BenchmarkError(message));
+            signal.addEventListener('abort', abort, { once: true });
+            removeAbort = () => signal.removeEventListener('abort', abort);
+        });
+        return await within(Promise.race([promise, aborted]), milliseconds, message);
+    }
+    finally {
+        removeAbort?.();
+    }
+}
+/** Build the verifier used by a HoldoutEvidenceStore from the same operator-pinned key. */
+export function createHoldoutEvidenceVerifier(pinnedPublicKey) {
+    holdoutAuthorityKeyId(pinnedPublicKey);
+    return Object.freeze({
+        verifyManifest: (envelope) => parseSignedHoldoutManifest(envelope, pinnedPublicKey),
+        verifyVerdict: (input) => parseSignedHoldoutVerdict(input.envelope, input.manifest, input.planDigest, input.cell, input.outputDigest, pinnedPublicKey),
+        verifyFinish: (input) => parseSignedHoldoutFinish(input.envelope, input.manifest, input.planDigest, input.cells, input.verdictEnvelopeDigests, pinnedPublicKey),
+    });
+}
+function verifyPinnedRun(evidence, plan, results, pinnedPublicKey) {
+    const verified = evidence.verifyRun(plan, results);
+    const manifest = validateManifest(verified.manifest.envelope, plan, pinnedPublicKey);
+    const planDigest = benchmarkPlanDigest(plan);
+    const cells = benchmarkSchedule(plan);
+    benchmarkAssert(verified.verdicts.length === cells.length
+        && verified.verdicts.every(item => item.manifestEvidenceDigest === verified.completion.manifestEvidenceDigest)
+        && verified.finish.manifestEvidenceDigest === verified.completion.manifestEvidenceDigest, 'holdout evidence combines different signed manifests');
+    const verdictEnvelopeDigests = verified.verdicts.map((item, index) => {
+        parseSignedHoldoutVerdict(item.envelope, manifest, planDigest, cells[index], item.host.outputDigest, pinnedPublicKey);
+        return holdoutEnvelopeDigest(item.envelope);
+    });
+    parseSignedHoldoutFinish(verified.finish.envelope, manifest, planDigest, cells, verdictEnvelopeDigests, pinnedPublicKey);
+    return verified;
+}
+/**
+ * Run an authority-signed holdout without exposing private input to the journal or evidence store.
+ * Recovery deliberately never replays a completed prefix: the authority transport owns its ordered
+ * session state, and the v1 journal has no durable provider resume token.
+ */
+export async function runIndependentHoldout(options) {
+    benchmarkAssert(options !== null && typeof options === 'object'
+        && options.store instanceof BenchmarkStore && options.evidence instanceof HoldoutEvidenceStore
+        && typeof options.openProvider === 'function' && typeof options.openDelegate === 'function', 'invalid independent holdout options');
+    const plan = parseBenchmarkPlan(options.plan);
+    benchmarkAssert(plan.dataset.split === 'holdout', 'independent holdout requires a holdout dataset');
+    const expectedDataset = validateDatasetPin(options.expectedDataset, plan);
+    benchmarkAssert(typeof options.pinnedPublicKey === 'string', 'invalid holdout public key');
+    holdoutAuthorityKeyId(options.pinnedPublicKey);
+    const maxOutputBytes = boundedInteger(options.maxOutputBytes, DEFAULT_MAX_OUTPUT_BYTES, MAX_OUTPUT_BYTES, 'holdout output byte limit');
+    const delegateCloseTimeoutMs = boundedInteger(options.delegateCloseTimeoutMs, DEFAULT_DELEGATE_CLOSE_TIMEOUT_MS, MAX_CLOSE_TIMEOUT_MS, 'holdout delegate close timeout');
+    const providerTimeoutMs = boundedInteger(options.providerTimeoutMs, DEFAULT_PROVIDER_TIMEOUT_MS, MAX_CLOSE_TIMEOUT_MS, 'holdout provider timeout');
+    const cells = benchmarkSchedule(plan);
+    const planDigest = benchmarkPlanDigest(plan);
+    options.store.create(plan);
+    const status = options.store.status(plan.id);
+    benchmarkAssert(status.runningCell === null, 'holdout journal has an unresolved running intent');
+    const recorded = status.results;
+    if (recorded.some(result => result.status === 'unknown'))
+        return Object.freeze({ results: recorded });
+    const existingCompletion = options.evidence.readPlanCompletion(plan);
+    if (recorded.length === cells.length) {
+        benchmarkAssert(existingCompletion !== undefined, 'completed holdout journal has no signed finish marker');
+        const verified = verifyPinnedRun(options.evidence, plan, recorded, options.pinnedPublicKey);
+        return Object.freeze({ results: recorded, completion: verified.completion });
+    }
+    benchmarkAssert(recorded.length === 0 && existingCompletion === undefined, 'partial holdout recovery requires a durable authority resume protocol');
+    benchmarkAssert(!options.signal?.aborted, 'holdout run was aborted before provider startup');
+    let provider;
+    let providerOpening;
+    let providerClose;
+    let delegate;
+    let delegateOpening;
+    let delegateClose;
+    let delegateExecution;
+    let resourcesClosing = false;
+    let resourcesClose;
+    const sensitiveInputs = new Set();
+    const closeProvider = (opened) => providerClose ??= within(Promise.resolve().then(() => opened.close()), providerTimeoutMs, 'holdout provider cleanup was not confirmed');
+    const requestProvider = (operation, value, signal) => {
+        benchmarkAssert(provider !== undefined && !resourcesClosing && !signal?.aborted, 'holdout provider cannot request after cancellation');
+        return withinAbort(Promise.resolve().then(() => provider.request(operation, value, signal)), signal, providerTimeoutMs, `holdout provider ${operation} was not confirmed`);
+    };
+    const getDelegate = async (signal) => {
+        benchmarkAssert(!resourcesClosing && !signal.aborted, 'holdout delegate cannot start after cancellation');
+        delegateOpening ??= Promise.resolve(options.openDelegate(signal)).then(validateBinding);
+        const opened = await delegateOpening;
+        delegate = opened;
+        if (resourcesClosing || signal.aborted) {
+            delegateClose ??= within(Promise.resolve().then(() => opened.close()), delegateCloseTimeoutMs);
+            await delegateClose;
+        }
+        benchmarkAssert(!resourcesClosing && !signal.aborted, 'holdout delegate opened after cancellation');
+        return opened;
+    };
+    const closeResources = (awaitExecution = true) => resourcesClose ??= (async () => {
+        resourcesClosing = true;
+        for (const bytes of sensitiveInputs)
+            bytes.fill(0);
+        const execution = delegateExecution;
+        const openingSettlement = (async () => {
+            let opened = delegate;
+            if (opened === undefined && delegateOpening !== undefined)
+                opened = await within(delegateOpening, delegateCloseTimeoutMs);
+            return opened;
+        })();
+        const delegateCloseSettlement = openingSettlement.then(opened => {
+            if (opened === undefined)
+                return;
+            delegateClose ??= within(Promise.resolve().then(() => opened.close()), delegateCloseTimeoutMs);
+            return delegateClose;
+        });
+        const executionSettlement = awaitExecution && execution !== undefined
+            ? within(execution.then(() => undefined, () => undefined), delegateCloseTimeoutMs)
+            : Promise.resolve();
+        const providerSettlement = provider === undefined ? Promise.resolve() : closeProvider(provider);
+        const settlements = await Promise.allSettled([delegateCloseSettlement, executionSettlement, providerSettlement]);
+        const failed = settlements.find((result) => result.status === 'rejected');
+        if (failed)
+            throw failed.reason;
+    })();
+    try {
+        providerOpening = Promise.resolve().then(() => options.openProvider(options.signal));
+        // A startup loser can resolve after the owning Fiber has finished. It must be closed without
+        // touching the journal or evidence stores, and its cleanup rejection must remain observed.
+        void providerOpening.then(opened => {
+            if (resourcesClosing || options.signal?.aborted)
+                void closeProvider(opened).catch(() => undefined);
+        }, () => undefined);
+        const openedProvider = await withinAbort(providerOpening, options.signal, providerTimeoutMs, 'holdout provider startup was not confirmed');
+        benchmarkAssert(!resourcesClosing && !options.signal?.aborted, 'holdout provider opened after cancellation');
+        provider = openedProvider;
+        benchmarkAssert(provider !== null && typeof provider === 'object'
+            && typeof provider.request === 'function' && typeof provider.close === 'function', 'invalid holdout provider binding');
+        const manifest = validateManifest(await requestProvider('manifest', { expectedDataset }, options.signal), plan, options.pinnedPublicKey);
+        options.evidence.writeManifest({ plan, manifest });
+        const verdictEvidenceDigests = [];
+        const verdictEnvelopeDigests = [];
+        const failureSnapshots = new Map();
+        let nextCell = 0;
+        const executor = Object.freeze({
+            async execute(request) {
+                const expectedCell = cells[nextCell];
+                benchmarkAssert(expectedCell !== undefined && same(request.cell, expectedCell) && request.planId === plan.id
+                    && same(request.dataset, plan.dataset) && same(request.budget, plan.budget), 'holdout runner request order or identity drifted');
+                benchmarkAssert(!request.signal.aborted && !resourcesClosing, 'holdout cell was cancelled before input');
+                const inputEnvelope = parseSignedHoldoutInput(await requestProvider('input', { plan, cell: request.cell }, request.signal), manifest, planDigest, request.cell, options.pinnedPublicKey);
+                const inputBytes = holdoutInputBytes(inputEnvelope);
+                sensitiveInputs.add(inputBytes);
+                let rawOutputBytes;
+                let outputBytes;
+                try {
+                    const binding = await getDelegate(request.signal);
+                    const execution = Promise.resolve().then(() => binding.execute(Object.freeze({
+                        planId: plan.id, dataset: plan.dataset, cell: request.cell, task: request.task, variant: request.variant, budget: request.budget,
+                        input: Object.freeze({ contentType: inputEnvelope.contentType, bytes: inputBytes }), signal: request.signal,
+                    })));
+                    delegateExecution = execution;
+                    const delegateResult = await execution;
+                    delegateExecution = undefined;
+                    if (delegateResult !== null && typeof delegateResult === 'object') {
+                        const candidate = Object.getOwnPropertyDescriptor(delegateResult, 'output')?.value;
+                        if (candidate !== null && typeof candidate === 'object') {
+                            const bytes = Object.getOwnPropertyDescriptor(candidate, 'bytes')?.value;
+                            if (bytes instanceof Uint8Array)
+                                rawOutputBytes = bytes;
+                        }
+                    }
+                    const validated = validateDelegateResult(delegateResult, request, maxOutputBytes);
+                    rawOutputBytes = validated.rawOutputBytes;
+                    outputBytes = validated.outputBytes;
+                    benchmarkAssert(!request.signal.aborted && !resourcesClosing, 'holdout cell was cancelled after delegate settlement');
+                    const outputDigest = createHash('sha256').update(outputBytes).digest('hex');
+                    const verdict = parseSignedHoldoutVerdict(await requestProvider('verdict', {
+                        planDigest, cell: request.cell, inputEnvelopeDigest: holdoutEnvelopeDigest(inputEnvelope),
+                        output: {
+                            contentType: validated.outputContentType,
+                            outputBase64url: Buffer.from(outputBytes.buffer, outputBytes.byteOffset, outputBytes.byteLength).toString('base64url'),
+                            outputDigest,
+                        },
+                    }, request.signal), manifest, planDigest, request.cell, outputDigest, options.pinnedPublicKey);
+                    benchmarkAssert(!request.signal.aborted && !resourcesClosing, 'holdout cell was cancelled after authority verdict');
+                    const saved = options.evidence.writeVerdict({
+                        plan, manifest, cell: request.cell, outputDigest,
+                        host: { versions: request.variant.versions, hostMetrics: validated.metrics, quiescent: true,
+                            executionEvidenceDigest: validated.executionEvidenceDigest, outputDigest },
+                        envelope: verdict,
+                    });
+                    const observation = Object.freeze({
+                        versions: request.variant.versions, inputDigest: inputEnvelope.inputDigest, acceptanceDigest: verdict.acceptanceDigest,
+                        verdict: verdict.verdict, metrics: validated.metrics, evidenceDigest: saved.digest, quiescent: true,
+                    });
+                    if (verdict.verdict === 'unknown') {
+                        failureSnapshots.set(request.cell.id, observation);
+                        throw new BenchmarkError('holdout authority returned an unknown verdict');
+                    }
+                    verdictEvidenceDigests.push(saved.digest);
+                    verdictEnvelopeDigests.push(holdoutEnvelopeDigest(verdict));
+                    nextCell++;
+                    if (nextCell === cells.length) {
+                        const finish = parseSignedHoldoutFinish(await requestProvider('finish', {
+                            planDigest, cells, verdictEnvelopeDigests,
+                        }, request.signal), manifest, planDigest, cells, verdictEnvelopeDigests, options.pinnedPublicKey);
+                        benchmarkAssert(finish.complete && !request.signal.aborted, 'holdout authority did not finalize the complete plan');
+                        // This is the current execution promise; waiting for it here would await ourselves.
+                        await closeResources(false);
+                        benchmarkAssert(!request.signal.aborted, 'holdout cell was cancelled while resources were closing');
+                        options.evidence.writeFinish({ plan, manifest, verdictEvidenceDigests, envelope: finish });
+                        benchmarkAssert(!request.signal.aborted, 'holdout cell was cancelled while finish evidence was published');
+                    }
+                    return observation;
+                }
+                finally {
+                    delegateExecution = undefined;
+                    outputBytes?.fill(0);
+                    rawOutputBytes?.fill(0);
+                    inputBytes.fill(0);
+                    sensitiveInputs.delete(inputBytes);
+                }
+            },
+            failure(request) {
+                return failureSnapshots.get(request.cell.id);
+            },
+        });
+        const results = await runBenchmark(options.store, plan, executor, options.signal);
+        if (results.length === cells.length && results.every(result => result.status === 'completed')) {
+            const verified = verifyPinnedRun(options.evidence, plan, results, options.pinnedPublicKey);
+            return Object.freeze({ results, completion: verified.completion });
+        }
+        return Object.freeze({ results });
+    }
+    finally {
+        await closeResources();
+    }
+}
+/**
+ * Fiber-owned Host entrypoint. Store descriptors and the authority/delegate session belong to the
+ * current Cordis Fiber; unloading aborts admission, awaits the run's bounded cleanup, then closes
+ * both stores. Calling the plain SDK entrypoint remains explicit and never mounts a plugin.
+ */
+export function runIndependentHoldoutInContext(ctx, options) {
+    benchmarkAssert(ctx !== null && typeof ctx === 'object' && typeof ctx.effect === 'function', 'invalid Cordis holdout context');
+    let task;
+    let closeOwned;
+    const disposeEffect = ctx.effect(() => {
+        const lifecycle = new AbortController();
+        const store = new BenchmarkStore(options.databasePath);
+        let evidence;
+        try {
+            evidence = new HoldoutEvidenceStore({ root: options.evidenceRoot, verifier: createHoldoutEvidenceVerifier(options.pinnedPublicKey) });
+        }
+        catch (error) {
+            store.close();
+            throw error;
+        }
+        const signal = options.signal === undefined ? lifecycle.signal : AbortSignal.any([lifecycle.signal, options.signal]);
+        task = runIndependentHoldout({ ...options, store, evidence, signal });
+        let closePromise;
+        closeOwned = () => closePromise ??= (async () => {
+            lifecycle.abort();
+            try {
+                await task;
+            }
+            catch { /* The caller observes the original run error. */ }
+            let firstError;
+            try {
+                evidence.close();
+            }
+            catch (error) {
+                firstError = error;
+            }
+            try {
+                store.close();
+            }
+            catch (error) {
+                firstError ??= error;
+            }
+            if (firstError !== undefined)
+                throw firstError;
+        })();
+        return closeOwned;
+    }, 'independent holdout run');
+    benchmarkAssert(task !== undefined && closeOwned !== undefined, 'Cordis did not synchronously register the holdout effect');
+    const running = task;
+    const close = closeOwned;
+    return running.finally(async () => {
+        await close();
+        await disposeEffect();
+    });
+}
+export { HoldoutEvidenceError, HoldoutEvidenceStore } from './holdout-evidence.js';
+export { HoldoutProviderError, openHoldoutProvider } from './holdout-provider.js';
+export { HOLDOUT_PROTOCOL_V1, holdoutAuthorityKeyId, holdoutEnvelopeDigest, holdoutInputBytes, holdoutUnsignedCanonicalJson, holdoutVerdictsDigest, parseSignedHoldoutFinish, parseSignedHoldoutInput, parseSignedHoldoutManifest, parseSignedHoldoutVerdict, verifyHoldoutEnvelopeSignature, } from './holdout-protocol.js';
+//# sourceMappingURL=holdout.js.map

package/lib/benchmark/holdout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"holdout.js","sourceRoot":"","sources":["../../src/benchmark/holdout.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExC,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAChF,OAAO,EACL,oBAAoB,GACrB,MAAM,uBAAuB,CAAA;AAE9B,OAAO,EACL,qBAAqB,EAAE,qBAAqB,EAAE,iBAAiB,EAC/D,wBAAwB,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,yBAAyB,GAEzG,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,YAAY,EAA0D,MAAM,aAAa,CAAA;AAClG,OAAO,EACL,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EACvG,iBAAiB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,kBAAkB,GACnF,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAM3C,MAAM,wBAAwB,GAAG,IAAI,GAAG,IAAI,CAAA;AAC5C,MAAM,iCAAiC,GAAG,MAAM,CAAA;AAChD,MAAM,2BAA2B,GAAG,MAAM,CAAA;AAC1C,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AACxC,MAAM,oBAAoB,GAAG,OAAO,CAAA;AACpC,MAAM,YAAY,GAAG,gEAAgE,CAAA;AA4DrF,SAAS,IAAI,CAAC,IAAa,EAAE,KAAc;IACzC,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,CAAC,CAAA;AACzE,CAAC;AAED,SAAS,SAAS,CAAC,KAAc,EAAE,MAAyB,EAAE,KAAa;IACzE,eAAe,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;WAC/E,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,KAAK,yBAAyB,CAAC,CAAA;IAC1F,MAAM,MAAM,GAAG,KAAgC,CAAA;IAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACpC,eAAe,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;WACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,wBAAwB,CAAC,CAAA;IAC1G,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,MAAM,CAAC,wBAAwB,CAAC,MAAM,EAAE,GAAG,CAAE,CAAA;QAChE,eAAe,CAAC,UAAU,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,GAAG,KAAK,8BAA8B,CAAC,CAAA;IACtH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,QAAgB,EAAE,OAAe,EAAE,KAAa;IACtF,MAAM,QAAQ,GAAG,KAAK,IAAI,QAAQ,CAAA;IAClC,eAAe,CAAC,OAAO,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC;WACzE,QAAQ,IAAI,CAAC,IAAI,QAAQ,IAAI,OAAO,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;IAC9D,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,KAAa;IACtD,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AAC5F,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,IAA6B;IACvE,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,qBAAqB,CAAC,CAAA;IAChF,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,eAAe,CAAC,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;WACxE,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EACnG,uDAAuD,CAAC,CAAA;IACxD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;AAChF,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,IAA6B,EAAE,eAAuB;IAC9F,MAAM,QAAQ,GAAG,0BAA0B,CAAC,KAAK,EAAE,eAAe,CAAC,CAAA;IACnE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EACtF,+DAA+D,CAAC,CAAA;IAClE,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,eAAe,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;WACtD,OAAQ,KAAgC,CAAC,OAAO,KAAK,UAAU;WAC/D,OAAQ,KAAgC,CAAC,KAAK,KAAK,UAAU,EAAE,kCAAkC,CAAC,CAAA;IACvG,OAAO,KAA+B,CAAA;AACxC,CAAC;AAUD,SAAS,sBAAsB,CAC7B,KAAc,EAAE,OAAkC,EAAE,cAAsB;IAE1E,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,yBAAyB,EAAE,WAAW,CAAC,EAAE,yBAAyB,CAAC,CAAA;IAClI,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,yBAAyB,CAAC,CAAA;IACzF,iBAAiB,CAAC,MAAM,CAAC,WAAW,EAAE,6BAA6B,CAAC,CAAA;IACpE,eAAe,CAAC,MAAM,CAAC,KAAK,YAAY,UAAU,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,SAAS,EAChH,2CAA2C,CAAC,CAAA;IAC9C,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAA;IACnC,eAAe,CAAC,cAAc,CAAC,UAAU,IAAI,cAAc,EAAE,uCAAuC,CAAC,CAAA;IACrG,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACvF,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,4CAA4C,CAAC,CAAA;IACvG,MAAM,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAClD,eAAe,CAAC,OAAO,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,YAAY,KAAK,IAAI;WACtG,OAAO,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,CAAC,aAAa,KAAK,IAAI,CAAC,EAC5G,oEAAoE,CAAC,CAAA;IACrE,eAAe,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,YAAY;WACnH,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS;WAC7C,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,CAAC,aAAc,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EACtG,yCAAyC,CAAC,CAAA;IAC1C,aAAa,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;IAC1C,eAAe,CAAC,GAAG,CAAC,SAAS,KAAK,IAAI,EAAE,2CAA2C,CAAC,CAAA;IACpF,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,iBAAiB,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,OAAO;QAC3E,uBAAuB,EAAE,GAAG,CAAC,uBAAuB;KACrD,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CAAI,OAAmB,EAAE,YAAoB,EAAE,OAAO,GAAG,4CAA4C;IACxH,IAAI,KAAgD,CAAA;IACpD,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;gBAC1E,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAA;gBAC3E,KAAK,CAAC,KAAK,EAAE,CAAA;YACf,CAAC,CAAC,CAAC,CAAC,CAAA;IACN,CAAC;YAAS,CAAC;QACT,IAAI,KAAK,KAAK,SAAS;YAAE,YAAY,CAAC,KAAK,CAAC,CAAA;IAC9C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAI,OAAmB,EAAE,MAA+B,EAAE,YAAoB,EAAE,OAAe;IACvH,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,cAAc,CAAC,OAAO,CAAC,CAAA;IACtD,IAAI,WAAqC,CAAA;IACzC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,OAAO,CAAQ,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAC5G,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC,CAAA;YACvD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YACvD,WAAW,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAChE,CAAC,CAAC,CAAA;QACF,OAAO,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,CAAA;IAC9E,CAAC;YAAS,CAAC;QACT,WAAW,EAAE,EAAE,CAAA;IACjB,CAAC;AACH,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,6BAA6B,CAAC,eAAuB;IACnE,qBAAqB,CAAC,eAAe,CAAC,CAAA;IACtC,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,cAAc,EAAE,CAAC,QAAiB,EAAE,EAAE,CAAC,0BAA0B,CAAC,QAAQ,EAAE,eAAe,CAAC;QAC5F,aAAa,EAAE,CAAC,KAA8D,EAAE,EAAE,CAAC,yBAAyB,CAC1G,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,EAAE,eAAe,CAClG;QACD,YAAY,EAAE,CAAC,KAA6D,EAAE,EAAE,CAAC,wBAAwB,CACvG,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,sBAAsB,EAAE,eAAe,CAC7G;KACF,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,QAA8B,EAAE,IAA6B,EAAE,OAAmC,EAAE,eAAuB;IAE3H,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAClD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,eAAe,CAAC,CAAA;IACpF,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;IAC5C,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAA;IACrC,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;WACpD,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,sBAAsB,KAAK,QAAQ,CAAC,UAAU,CAAC,sBAAsB,CAAC;WAC3G,QAAQ,CAAC,MAAM,CAAC,sBAAsB,KAAK,QAAQ,CAAC,UAAU,CAAC,sBAAsB,EAC1F,sDAAsD,CAAC,CAAA;IACvD,MAAM,sBAAsB,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACnE,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAE,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;QACtH,OAAO,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;IACF,wBAAwB,CACtB,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,sBAAsB,EAAE,eAAe,CAC/F,CAAA;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAkC;IAC5E,eAAe,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ;WAC1D,OAAO,CAAC,KAAK,YAAY,cAAc,IAAI,OAAO,CAAC,QAAQ,YAAY,oBAAoB;WAC3F,OAAO,OAAO,CAAC,YAAY,KAAK,UAAU,IAAI,OAAO,OAAO,CAAC,YAAY,KAAK,UAAU,EAC7F,qCAAqC,CAAC,CAAA;IACtC,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAC7C,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,gDAAgD,CAAC,CAAA;IACnG,MAAM,eAAe,GAAG,kBAAkB,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,CAAA;IACzE,eAAe,CAAC,OAAO,OAAO,CAAC,eAAe,KAAK,QAAQ,EAAE,4BAA4B,CAAC,CAAA;IAC1F,qBAAqB,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IAC9C,MAAM,cAAc,GAAG,cAAc,CAAC,OAAO,CAAC,cAAc,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,2BAA2B,CAAC,CAAA;IACtI,MAAM,sBAAsB,GAAG,cAAc,CAC3C,OAAO,CAAC,sBAAsB,EAAE,iCAAiC,EAAE,oBAAoB,EAAE,gCAAgC,CAC1H,CAAA;IACD,MAAM,iBAAiB,GAAG,cAAc,CACtC,OAAO,CAAC,iBAAiB,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,0BAA0B,CACzG,CAAA;IACD,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;IAE5C,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC5C,eAAe,CAAC,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,kDAAkD,CAAC,CAAA;IAChG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAA;IAC/B,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAA;IACrG,MAAM,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAA;IACpE,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACrC,eAAe,CAAC,kBAAkB,KAAK,SAAS,EAAE,uDAAuD,CAAC,CAAA;QAC1G,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;QAC3F,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9E,CAAC;IACD,eAAe,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,KAAK,SAAS,EACvE,uEAAuE,CAAC,CAAA;IAC1E,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,iDAAiD,CAAC,CAAA;IAE5F,IAAI,QAA8C,CAAA;IAClD,IAAI,eAA8D,CAAA;IAClE,IAAI,aAAwC,CAAA;IAC5C,IAAI,QAA4C,CAAA;IAChD,IAAI,eAA4D,CAAA;IAChE,IAAI,aAAwC,CAAA;IAC5C,IAAI,iBAA+C,CAAA;IACnD,IAAI,gBAAgB,GAAG,KAAK,CAAA;IAC5B,IAAI,cAAyC,CAAA;IAC7C,MAAM,eAAe,GAAG,IAAI,GAAG,EAAc,CAAA;IAE7C,MAAM,aAAa,GAAG,CAAC,MAAgC,EAAiB,EAAE,CAAC,aAAa,KAAK,MAAM,CACjG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,4CAA4C,CAC9G,CAAA;IAED,MAAM,eAAe,GAAG,CAAC,SAA6D,EAAE,KAAc,EAAE,MAAoB,EAAoB,EAAE;QAChJ,eAAe,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,oDAAoD,CAAC,CAAA;QACtI,OAAO,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAS,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,iBAAiB,EACrH,oBAAoB,SAAS,oBAAoB,CAAC,CAAA;IACtD,CAAC,CAAA;IAED,MAAM,WAAW,GAAG,KAAK,EAAE,MAAmB,EAAmC,EAAE;QACjF,eAAe,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,kDAAkD,CAAC,CAAA;QACzG,eAAe,KAAK,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QACvF,MAAM,MAAM,GAAG,MAAM,eAAe,CAAA;QACpC,QAAQ,GAAG,MAAM,CAAA;QACjB,IAAI,gBAAgB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACvC,aAAa,KAAK,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,sBAAsB,CAAC,CAAA;YAC9F,MAAM,aAAa,CAAA;QACrB,CAAC;QACD,eAAe,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,4CAA4C,CAAC,CAAA;QACnG,OAAO,MAAM,CAAA;IACf,CAAC,CAAA;IAED,MAAM,cAAc,GAAG,CAAC,cAAc,GAAG,IAAI,EAAiB,EAAE,CAAC,cAAc,KAAK,CAAC,KAAK,IAAI,EAAE;QAC9F,gBAAgB,GAAG,IAAI,CAAA;QACvB,KAAK,MAAM,KAAK,IAAI,eAAe;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAClD,MAAM,SAAS,GAAG,iBAAiB,CAAA;QACnC,MAAM,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACpC,IAAI,MAAM,GAAG,QAAQ,CAAA;YACrB,IAAI,MAAM,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS;gBAAE,MAAM,GAAG,MAAM,MAAM,CAAC,eAAe,EAAE,sBAAsB,CAAC,CAAA;YACzH,OAAO,MAAM,CAAA;QACf,CAAC,CAAC,EAAE,CAAA;QACJ,MAAM,uBAAuB,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAC9D,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAM;YAChC,aAAa,KAAK,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,sBAAsB,CAAC,CAAA;YAC9F,OAAO,aAAa,CAAA;QACtB,CAAC,CAAC,CAAA;QACF,MAAM,mBAAmB,GAAG,cAAc,IAAI,SAAS,KAAK,SAAS;YACnE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,sBAAsB,CAAC;YAClF,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACrB,MAAM,kBAAkB,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;QAC/F,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,uBAAuB,EAAE,mBAAmB,EAAE,kBAAkB,CAAC,CAAC,CAAA;QAChH,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,EAAmC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;QAC1G,IAAI,MAAM;YAAE,MAAM,MAAM,CAAC,MAAM,CAAA;IACjC,CAAC,CAAC,EAAE,CAAA;IAEJ,IAAI,CAAC;QACH,eAAe,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;QACpF,6FAA6F;QAC7F,2FAA2F;QAC3F,KAAK,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YACjC,IAAI,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,KAAK,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QACpG,CAAC,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QACnB,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,EAAE,iBAAiB,EAAE,4CAA4C,CAAC,CAAA;QAC1I,eAAe,CAAC,CAAC,gBAAgB,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,4CAA4C,CAAC,CAAA;QAC5G,QAAQ,GAAG,cAAc,CAAA;QACzB,eAAe,CAAC,QAAQ,KAAK,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ;eAC5D,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,IAAI,OAAO,QAAQ,CAAC,KAAK,KAAK,UAAU,EAAE,kCAAkC,CAAC,CAAA;QACxH,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,eAAe,CAAC,UAAU,EAAE,EAAE,eAAe,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;QACxI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;QAElD,MAAM,sBAAsB,GAAa,EAAE,CAAA;QAC3C,MAAM,sBAAsB,GAAa,EAAE,CAAA;QAC3C,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAgC,CAAA;QAChE,IAAI,QAAQ,GAAG,CAAC,CAAA;QAEhB,MAAM,QAAQ,GAAsB,MAAM,CAAC,MAAM,CAAC;YAChD,KAAK,CAAC,OAAO,CAAC,OAAkC;gBAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAA;gBACpC,eAAe,CAAC,YAAY,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE;uBACvG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,kDAAkD,CAAC,CAAA;gBAClI,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,gBAAgB,EAAE,yCAAyC,CAAC,CAAA;gBACxG,MAAM,aAAa,GAAG,uBAAuB,CAC3C,MAAM,eAAe,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAC5E,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,eAAe,CAC5D,CAAA;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,aAAa,CAAC,CAAA;gBACnD,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;gBAC/B,IAAI,cAAsC,CAAA;gBAC1C,IAAI,WAAmC,CAAA;gBACvC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;oBACjD,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;wBAC3E,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM;wBAChI,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM;qBAC5G,CAAC,CAAC,CAAC,CAAA;oBACJ,iBAAiB,GAAG,SAAS,CAAA;oBAC7B,MAAM,cAAc,GAAG,MAAM,SAAS,CAAA;oBACtC,iBAAiB,GAAG,SAAS,CAAA;oBAC7B,IAAI,cAAc,KAAK,IAAI,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE,CAAC;wBAClE,MAAM,SAAS,GAAG,MAAM,CAAC,wBAAwB,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,KAAK,CAAA;wBAClF,IAAI,SAAS,KAAK,IAAI,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;4BACxD,MAAM,KAAK,GAAG,MAAM,CAAC,wBAAwB,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,KAAK,CAAA;4BACxE,IAAI,KAAK,YAAY,UAAU;gCAAE,cAAc,GAAG,KAAK,CAAA;wBACzD,CAAC;oBACH,CAAC;oBACD,MAAM,SAAS,GAAG,sBAAsB,CAAC,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;oBACjF,cAAc,GAAG,SAAS,CAAC,cAAc,CAAA;oBACzC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAA;oBACnC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,gBAAgB,EAAE,sDAAsD,CAAC,CAAA;oBACrH,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;oBAC3E,MAAM,OAAO,GAAG,yBAAyB,CAAC,MAAM,eAAe,CAAC,SAAS,EAAE;wBACzE,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,mBAAmB,EAAE,qBAAqB,CAAC,aAAa,CAAC;wBACzF,MAAM,EAAE;4BACN,WAAW,EAAE,SAAS,CAAC,iBAAiB;4BACxC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;4BACtH,YAAY;yBACb;qBACF,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;oBAC9F,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,gBAAgB,EAAE,oDAAoD,CAAC,CAAA;oBACnH,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAC1C,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,YAAY;wBAChD,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI;4BACzF,uBAAuB,EAAE,SAAS,CAAC,uBAAuB,EAAE,YAAY,EAAE;wBAC5E,QAAQ,EAAE,OAAO;qBAClB,CAAC,CAAA;oBACF,MAAM,WAAW,GAAyB,MAAM,CAAC,MAAM,CAAC;wBACtD,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,aAAa,CAAC,WAAW,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;wBACtH,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI;qBACpG,CAAC,CAAA;oBACF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;wBAClC,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;wBAClD,MAAM,IAAI,cAAc,CAAC,+CAA+C,CAAC,CAAA;oBAC3E,CAAC;oBACD,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;oBACzC,sBAAsB,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAA;oBAC3D,QAAQ,EAAE,CAAA;oBACV,IAAI,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;wBAC9B,MAAM,MAAM,GAAG,wBAAwB,CAAC,MAAM,eAAe,CAAC,QAAQ,EAAE;4BACtE,UAAU,EAAE,KAAK,EAAE,sBAAsB;yBAC1C,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,sBAAsB,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;wBACjG,eAAe,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,sDAAsD,CAAC,CAAA;wBACnH,oFAAoF;wBACpF,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;wBAC3B,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,yDAAyD,CAAC,CAAA;wBACnG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;wBAC1F,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,gEAAgE,CAAC,CAAA;oBAC5G,CAAC;oBACD,OAAO,WAAW,CAAA;gBACpB,CAAC;wBAAS,CAAC;oBACT,iBAAiB,GAAG,SAAS,CAAA;oBAC7B,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;oBACpB,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;oBACvB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;oBAClB,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;gBACpC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,OAAkC;gBACxC,OAAO,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC9C,CAAC;SACF,CAAC,CAAA;QAEF,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;QACjF,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,EAAE,CAAC;YAC9F,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;YAC1F,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;QACpE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAA;IACnC,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,EAAE,CAAA;IACxB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAC5C,GAAY,EAAE,OAAyC;IAEvD,eAAe,CAAC,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,UAAU,EAAE,gCAAgC,CAAC,CAAA;IAC9H,IAAI,IAA6D,CAAA;IACjE,IAAI,UAA6C,CAAA;IACjD,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE;QACpC,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAA;QACvC,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;QACtD,IAAI,QAA8B,CAAA;QAClC,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,YAAY,EAAE,QAAQ,EAAE,6BAA6B,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;QACvI,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,MAAM,KAAK,CAAA;QACb,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;QACpH,IAAI,GAAG,qBAAqB,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;QACrE,IAAI,YAAuC,CAAA;QAC3C,UAAU,GAAG,GAAG,EAAE,CAAC,YAAY,KAAK,CAAC,KAAK,IAAI,EAAE;YAC9C,SAAS,CAAC,KAAK,EAAE,CAAA;YACjB,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAA;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,iDAAiD,CAAC,CAAC;YAC9E,IAAI,UAAmB,CAAA;YACvB,IAAI,CAAC;gBAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;YAAC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,UAAU,GAAG,KAAK,CAAA;YAAC,CAAC;YAC7D,IAAI,CAAC;gBAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAAC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,UAAU,KAAK,KAAK,CAAA;YAAC,CAAC;YAC5D,IAAI,UAAU,KAAK,SAAS;gBAAE,MAAM,UAAU,CAAA;QAChD,CAAC,CAAC,EAAE,CAAA;QACJ,OAAO,UAAU,CAAA;IACnB,CAAC,EAAE,yBAAyB,CAAC,CAAA;IAC7B,eAAe,CAAC,IAAI,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,0DAA0D,CAAC,CAAA;IAC3H,MAAM,OAAO,GAAG,IAAI,CAAA;IACpB,MAAM,KAAK,GAAG,UAAU,CAAA;IACxB,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;QAChC,MAAM,KAAK,EAAE,CAAA;QACb,MAAM,aAAa,EAAE,CAAA;IACvB,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAA;AAMlF,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAIjF,OAAO,EACL,mBAAmB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,4BAA4B,EAClH,qBAAqB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,0BAA0B,EACpG,yBAAyB,EAAE,8BAA8B,GAC1D,MAAM,uBAAuB,CAAA"}

package/lib/benchmark/index.d.ts

@@ -0,0 +1,9 @@
+export * from './types.js';
+export { BenchmarkError, benchmarkPlanDigest, benchmarkSchedule, parseBenchmarkPlan } from './schema.js';
+export { BenchmarkStore } from './store.js';
+export { benchmarkReport } from './report.js';
+export { runBenchmark } from './runner.js';
+export type { BenchmarkExecutor, BenchmarkExecutionRequest } from './runner.js';
+export { developmentCorpus, developmentDataset, developmentCases, developmentPrompt, judgeDevelopmentResponse } from './corpus.js';
+export type { DevelopmentTask, BenchmarkJson } from './corpus.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/benchmark/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/benchmark/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACxG,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,YAAY,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAA;AAC/E,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAA;AAClI,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA"}

package/lib/benchmark/index.js

@@ -0,0 +1,7 @@
+export * from './types.js';
+export { BenchmarkError, benchmarkPlanDigest, benchmarkSchedule, parseBenchmarkPlan } from './schema.js';
+export { BenchmarkStore } from './store.js';
+export { benchmarkReport } from './report.js';
+export { runBenchmark } from './runner.js';
+export { developmentCorpus, developmentDataset, developmentCases, developmentPrompt, judgeDevelopmentResponse } from './corpus.js';
+//# sourceMappingURL=index.js.map

package/lib/benchmark/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/benchmark/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACxG,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAE1C,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAA"}

package/lib/benchmark/memory-corpus.d.ts

@@ -0,0 +1,55 @@
+import type { BenchmarkCase, BenchmarkPlan, BenchmarkVerdict, BenchmarkDomain } from './types.js';
+export type MemoryBenchmarkJson = null | boolean | number | string | readonly MemoryBenchmarkJson[] | {
+    readonly [key: string]: MemoryBenchmarkJson;
+};
+export type MemoryVisibility = 'visible' | 'other-owner' | 'other-workspace' | 'sensitive' | 'expired' | 'removed';
+export interface MemoryDevelopmentTask {
+    readonly id: string;
+    readonly domain: BenchmarkDomain;
+    readonly objective: string;
+    readonly sources: readonly {
+        readonly id: string;
+        readonly text: string;
+    }[];
+    readonly memories: readonly {
+        readonly id: string;
+        readonly visibility: MemoryVisibility;
+        readonly entry: {
+            readonly kind: 'experience';
+            readonly content: string;
+            readonly sensitivity: 'private' | 'sensitive';
+            readonly trust: 'user-confirmed';
+            readonly confidence: number;
+            readonly provenance: {
+                readonly source: string;
+                readonly observedAt: number;
+                readonly uri?: string;
+            };
+            readonly knowledge?: {
+                readonly applicability?: readonly string[];
+                readonly counterexamples?: readonly string[];
+                readonly claim?: {
+                    readonly key: string;
+                    readonly value: string;
+                };
+            };
+        };
+    }[];
+    readonly snapshotLimit: number;
+    readonly acceptance: {
+        readonly answer: MemoryBenchmarkJson;
+        readonly citations: readonly string[];
+    };
+}
+export declare const memoryDevelopmentCorpus: readonly MemoryDevelopmentTask[];
+export declare const memoryDevelopmentCorpusV2: readonly MemoryDevelopmentTask[];
+export declare const memoryDevelopmentDataset: BenchmarkPlan['dataset'];
+export declare const memoryDevelopmentDatasetV2: BenchmarkPlan['dataset'];
+export declare function memoryDevelopmentCases(version?: '1' | '2'): readonly BenchmarkCase[];
+export declare function memoryDevelopmentTask(caseId: string, version?: '1' | '2'): Readonly<MemoryDevelopmentTask>;
+export declare function memoryDevelopmentPrompt(caseId: string, version?: '1' | '2'): string;
+export declare function judgeMemoryDevelopmentResponse(caseId: string, output: string, version?: '1' | '2'): {
+    verdict: BenchmarkVerdict;
+    evidenceDigest: string;
+};
+//# sourceMappingURL=memory-corpus.d.ts.map

package/lib/benchmark/memory-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-corpus.d.ts","sourceRoot":"","sources":["../../src/benchmark/memory-corpus.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEjG,MAAM,MAAM,mBAAmB,GAAG,IAAI,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,mBAAmB,EAAE,GAAG;IAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAAA;CAAE,CAAA;AACrJ,MAAM,MAAM,gBAAgB,GAAG,SAAS,GAAG,aAAa,GAAG,iBAAiB,GAAG,WAAW,GAAG,SAAS,GAAG,SAAS,CAAA;AAElH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;IAC3E,QAAQ,CAAC,QAAQ,EAAE,SAAS;QAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAA;QACrC,QAAQ,CAAC,KAAK,EAAE;YACd,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;YAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;YACxB,QAAQ,CAAC,WAAW,EAAE,SAAS,GAAG,WAAW,CAAA;YAC7C,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAA;YAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;YAC3B,QAAQ,CAAC,UAAU,EAAE;gBAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;gBAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;aAAE,CAAA;YACpG,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;gBAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;gBAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;oBAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;oBAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;iBAAE,CAAA;aAAE,CAAA;SACrL,CAAA;KACF,EAAE,CAAA;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,UAAU,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAA;CACrG;AAsFD,eAAO,MAAM,uBAAuB,EAAE,SAAS,qBAAqB,EAA4G,CAAA;AAQhL,eAAO,MAAM,yBAAyB,EAAE,SAAS,qBAAqB,EAIlE,CAAA;AAKJ,eAAO,MAAM,wBAAwB,EAAE,aAAa,CAAC,SAAS,CAE5D,CAAA;AAEF,eAAO,MAAM,0BAA0B,EAAE,aAAa,CAAC,SAAS,CAE9D,CAAA;AAEF,wBAAgB,sBAAsB,CAAC,OAAO,GAAE,GAAG,GAAG,GAAS,GAAG,SAAS,aAAa,EAAE,CAEzF;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,GAAG,GAAG,GAAS,GAAG,QAAQ,CAAC,qBAAqB,CAAC,CAI/G;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,GAAG,GAAG,GAAS,GAAG,MAAM,CAIxF;AAaD,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,GAAG,GAAG,GAAS,GAAG;IAAE,OAAO,EAAE,gBAAgB,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAO9J"}