niceeval 0.12.2-canary.31 → 0.13.0

package/dist/analysis/bindings.cjs

@@ -0,0 +1,617 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.builtinDomainViewBindings = exports.sandboxHistoryDomainBinding = exports.sourcesDomainBinding = exports.sourceNavigationDomainBinding = exports.fileChangesDomainBinding = exports.attemptObservabilityDomainBinding = exports.attemptEvidenceDomainBinding = exports.publishedAnalysisInputBindings = exports.attemptObservabilityFamily = void 0;
+const effect_1 = require("effect");
+const verdict_ts_1 = require("../eval/record/verdict.cjs");
+const catalog_ts_1 = require("../record/family/catalog.cjs");
+const file_changes_ts_1 = require("./file-changes.cjs");
+function fixedFamilyBinding(input) {
+    return Object.freeze({ ...input });
+}
+const assertionsFamily = fixedFamilyBinding({
+    owner: "attempt",
+    descriptor: catalog_ts_1.assertionsRecordFamily,
+    read: (reader, owner) => reader.readAssertions(owner),
+});
+/** @internal Cost projection owns this sealed, attempt-scoped Usage seam. */
+exports.attemptObservabilityFamily = fixedFamilyBinding({
+    owner: "attempt",
+    descriptor: catalog_ts_1.attemptObservabilityRecordFamily,
+    read: (reader, owner) => reader.readAttemptObservability(owner),
+});
+const fileChangesFamily = fixedFamilyBinding({
+    owner: "attempt",
+    descriptor: catalog_ts_1.fileChangesRecordFamily,
+    read: (reader, owner) => reader.readFileChanges(owner),
+});
+const sourceNavigationFamily = fixedFamilyBinding({
+    owner: "attempt",
+    descriptor: catalog_ts_1.sourceNavigationRecordFamily,
+    read: (reader, owner) => reader.readSourceNavigation(owner),
+});
+const originSourcesFamily = fixedFamilyBinding({
+    owner: "origin-run",
+    descriptor: catalog_ts_1.sourcesRecordFamily,
+    read: (reader, owner) => reader.readSources(owner),
+});
+/** The complete published input catalog; ids are semantic, never schema ids. */
+exports.publishedAnalysisInputBindings = Object.freeze({
+    attemptPassed: Object.freeze({
+        id: "niceeval.attempt-passed",
+        family: assertionsFamily,
+        project: ({ core, payload }) => {
+            const gateFailed = payload.entries.some((entry) => entry.result.gate === "failed");
+            const gateUnavailable = payload.entries.some((entry) => entry.result.gate === "unavailable");
+            if (core.outcome !== "completed") {
+                return Object.freeze({
+                    state: "missing",
+                    message: "the Attempt did not complete with a pass/fail assertion result",
+                });
+            }
+            if (gateUnavailable) {
+                return Object.freeze({
+                    state: "missing",
+                    message: "a gating Assertion result is unavailable",
+                });
+            }
+            return Object.freeze({ state: "value", value: !gateFailed });
+        },
+    }),
+    attemptLatencyMs: Object.freeze({
+        id: "niceeval.attempt-latency-ms",
+        family: exports.attemptObservabilityFamily,
+        project: ({ payload }) => {
+            const timing = payload.timing;
+            if (timing.collection.state !== "complete") {
+                return collectionProjection(timing.collection, "timing", "timing collection is incomplete");
+            }
+            const intervals = timing.intervals.filter((interval) => interval.phase === "eval.run");
+            if (intervals.length === 0) {
+                return Object.freeze({
+                    state: "missing",
+                    message: "no eval.run interval was recorded",
+                });
+            }
+            return Object.freeze({
+                state: "value",
+                value: intervals.reduce((total, interval) => total + interval.durationMs, 0),
+            });
+        },
+    }),
+    /**
+     * The final token reading is `inputTokens + outputTokens`. It keeps that
+     * exact, non-overlapping pair from the fixed Usage family: cache buckets
+     * are separately accounted input and reasoning is already included in the
+     * output bucket, so neither belongs in this total.
+     */
+    attemptTokens: Object.freeze({
+        id: "niceeval.attempt-input-output-tokens",
+        family: exports.attemptObservabilityFamily,
+        project: ({ payload }) => {
+            const usage = payload.usage;
+            if (usage.collection.state !== "complete") {
+                return collectionProjection(usage.collection, "usage", "usage collection is incomplete");
+            }
+            let input = 0;
+            let output = 0;
+            let hasInput = false;
+            let hasOutput = false;
+            for (const observation of usage.observations) {
+                if (observation.kind !== "token-bucket")
+                    continue;
+                if (observation.bucket === "input") {
+                    input += observation.tokens;
+                    hasInput = true;
+                }
+                else if (observation.bucket === "output") {
+                    output += observation.tokens;
+                    hasOutput = true;
+                }
+            }
+            if (!hasInput || !hasOutput) {
+                return Object.freeze({
+                    state: "missing",
+                    message: "recorded usage does not contain both input and output token buckets",
+                });
+            }
+            const total = input + output;
+            if (!Number.isSafeInteger(total)) {
+                return Object.freeze({
+                    state: "failed",
+                    message: "recorded input plus output tokens exceed the safe integer range",
+                });
+            }
+            return Object.freeze({ state: "value", value: total });
+        },
+    }),
+    attemptToolFailure: Object.freeze({
+        id: "niceeval.attempt-tool-failure",
+        family: exports.attemptObservabilityFamily,
+        project: ({ payload }) => {
+            const commands = payload.commands;
+            const failed = commands.commands.some((command) => command.result.outcome.kind !== "exited" || command.result.outcome.exitCode !== 0);
+            if (failed)
+                return Object.freeze({ state: "value", value: true });
+            if (commands.collection.state === "complete") {
+                return Object.freeze({ state: "value", value: false });
+            }
+            return collectionProjection(commands.collection, "command", "command collection is incomplete");
+        },
+    }),
+});
+exports.attemptEvidenceDomainBinding = Object.freeze({
+    id: "niceeval.domain.attempt-evidence",
+    kind: "attempt-evidence",
+    family: assertionsFamily,
+    project: ({ core, payload, blobs }) => closeAssertions(core, payload, blobs),
+});
+exports.attemptObservabilityDomainBinding = Object.freeze({
+    id: "niceeval.domain.attempt-observability",
+    kind: "attempt-observability",
+    family: exports.attemptObservabilityFamily,
+    project: ({ payload, blobs }) => closeObservability(payload, blobs, false),
+});
+exports.fileChangesDomainBinding = Object.freeze({
+    id: "niceeval.domain.file-changes",
+    kind: "file-changes",
+    family: fileChangesFamily,
+    project: ({ payload, blobs }) => closeFileChanges(payload, blobs),
+});
+exports.sourceNavigationDomainBinding = Object.freeze({
+    id: "niceeval.domain.source-navigation",
+    kind: "source-navigation",
+    family: sourceNavigationFamily,
+    project: ({ payload }) => closeSourceNavigation(payload),
+});
+exports.sourcesDomainBinding = Object.freeze({
+    id: "niceeval.domain.sources",
+    kind: "sources",
+    family: originSourcesFamily,
+    project: ({ payload, blobs }) => closeSources(payload, blobs),
+});
+exports.sandboxHistoryDomainBinding = Object.freeze({
+    id: "niceeval.domain.sandbox-history",
+    kind: "sandbox-history",
+    family: exports.attemptObservabilityFamily,
+    project: ({ payload, blobs }) => closeObservability(payload, blobs, true),
+});
+exports.builtinDomainViewBindings = Object.freeze({
+    "attempt-evidence": exports.attemptEvidenceDomainBinding,
+    "attempt-observability": exports.attemptObservabilityDomainBinding,
+    "file-changes": exports.fileChangesDomainBinding,
+    "source-navigation": exports.sourceNavigationDomainBinding,
+    sources: exports.sourcesDomainBinding,
+    "sandbox-history": exports.sandboxHistoryDomainBinding,
+});
+function closeAssertions(core, payload, _blobs) {
+    return Object.freeze({
+        outcome: core.outcome,
+        verdict: (0, verdict_ts_1.foldRecordedAttemptVerdict)({ outcome: core.outcome, assertions: payload }),
+        entries: Object.freeze(payload.entries.map((entry) => Object.freeze({
+            entryId: entry.entryId,
+            display: closeJson(entry.display),
+            criterion: closeJson(entry.criterion),
+            result: closeJson(entry.result),
+            coverage: closeJson(entry.coverage),
+            limitations: closeJson(entry.limitations),
+            subject: closeAssertionMaterial(entry.subject),
+            evidence: Object.freeze(entry.evidence.map(closeAssertionMaterial)),
+        }))),
+        sourceSites: Object.freeze(payload.sourceSites.map(closeJson)),
+    });
+}
+function closeSourceNavigation(payload) {
+    return Object.freeze({
+        collection: Object.freeze({
+            state: payload.collection.state,
+            limitations: Object.freeze(payload.collection.limitations.map((limitation) => Object.freeze({
+                code: limitation.code,
+                target: limitation.target,
+                omittedAtLeast: limitation.omittedAtLeast,
+            }))),
+        }),
+        rows: Object.freeze(payload.rows.map((row) => Object.freeze({
+            turnId: row.turnId,
+            sourceOrder: row.sourceOrder,
+            source: row.source.state === "mapped"
+                ? Object.freeze({
+                    state: "mapped",
+                    sourceItemId: row.source.sourceItemId,
+                    sha256: row.source.sha256,
+                    start: Object.freeze({ line: row.source.start.line, column: row.source.start.column }),
+                    end: Object.freeze({ line: row.source.end.line, column: row.source.end.column }),
+                })
+                : Object.freeze({ state: "unmapped", reason: row.source.reason }),
+            timing: row.timing.state === "linked"
+                ? Object.freeze({ state: "linked", intervalId: row.timing.intervalId })
+                : Object.freeze({ state: "unavailable", reason: row.timing.reason }),
+        }))),
+    });
+}
+function collectionProjection(collection, target, fallback) {
+    if (collection.limitations.some((limitation) => isObservationLimitation(limitation, "unsupported-input", target))) {
+        return Object.freeze({ state: "unsupported", message: `${target} is unsupported by the recorded collector` });
+    }
+    if (collection.limitations.some((limitation) => isObservationLimitation(limitation, "capture-failed", target) || isObservationLimitation(limitation, "capture-interrupted", target))) {
+        return Object.freeze({ state: "failed", message: `${target} capture failed` });
+    }
+    return Object.freeze({ state: "missing", message: fallback });
+}
+function isObservationLimitation(value, code, target) {
+    return typeof value === "object" && value !== null
+        && value.code === code
+        && value.target === target;
+}
+function closeObservability(payload, blobs, sandboxOnly) {
+    const isSandboxPhase = (phase) => phase === "sandbox.prepare" || phase === "sandbox.command";
+    const commands = payload.commands.commands
+        .filter((command) => !sandboxOnly || isSandboxPhase(command.manifest.phase))
+        .map((command) => closeCommand(command, blobs));
+    const timing = payload.timing.intervals
+        .filter((interval) => !sandboxOnly || isSandboxPhase(interval.phase))
+        .map(closeTimingInterval);
+    const diagnostics = payload.diagnostics.diagnostics
+        .filter((diagnostic) => !sandboxOnly || isSandboxPhase(diagnostic.phase))
+        .map(closeDiagnostic);
+    return Object.freeze({
+        conversation: sandboxOnly
+            ? Object.freeze({
+                collection: closeTraceCollection(payload.conversation.collection),
+                turns: Object.freeze([]),
+                items: Object.freeze([]),
+            })
+            : closeConversation(payload.conversation),
+        commands: Object.freeze({
+            collection: closeTraceCollection(payload.commands.collection),
+            entries: Object.freeze(commands),
+        }),
+        usage: sandboxOnly
+            ? Object.freeze({
+                collection: closeTraceCollection(payload.usage.collection),
+                observations: Object.freeze([]),
+            })
+            : closeUsage(payload.usage),
+        timing: Object.freeze({
+            collection: closeTraceCollection(payload.timing.collection),
+            intervals: Object.freeze(timing),
+        }),
+        diagnostics: Object.freeze({
+            collection: closeTraceCollection(payload.diagnostics.collection),
+            diagnostics: Object.freeze(diagnostics),
+        }),
+    });
+}
+function closeConversation(value) {
+    return Object.freeze({
+        collection: closeTraceCollection(value.collection),
+        turns: Object.freeze(value.turns.map((turn) => Object.freeze({
+            turnId: turn.turnId,
+            sequence: turn.sequence,
+            outcome: turn.outcome,
+        }))),
+        items: Object.freeze(value.items.map(closeConversationItem)),
+    });
+}
+function closeConversationItem(value) {
+    const base = {
+        itemId: value.itemId,
+        turnId: value.turnId,
+        sequence: value.sequence,
+    };
+    switch (value.kind) {
+        case "message":
+            return Object.freeze({ ...base, kind: value.kind, role: value.role, text: value.text });
+        case "tool-call":
+            return Object.freeze({ ...base, kind: value.kind, callId: value.callId, tool: value.tool, inputSummary: value.inputSummary });
+        case "tool-result":
+            return Object.freeze({ ...base, kind: value.kind, callId: value.callId, outcome: value.outcome, outputSummary: value.outputSummary });
+        case "thinking-summary":
+        case "compaction":
+        case "context-injection":
+            return Object.freeze({ ...base, kind: value.kind, summary: value.summary });
+        case "subagent":
+            return Object.freeze({ ...base, kind: value.kind, state: value.state, label: value.label, summary: value.summary });
+        case "input-request":
+            return Object.freeze({
+                ...base,
+                kind: value.kind,
+                state: value.state,
+                promptSummary: value.promptSummary,
+                responseSummary: value.responseSummary,
+            });
+        case "skill-load":
+        case "conversation-error":
+            return Object.freeze({ ...base, kind: value.kind, code: value.code, summary: value.summary });
+    }
+}
+function closeCommand(value, blobs) {
+    return Object.freeze({
+        commandId: value.commandId,
+        manifest: Object.freeze({
+            phase: value.manifest.phase,
+            invocation: closeCommandInvocation(value.manifest.invocation),
+            workingDirectory: closeWorkingDirectory(value.manifest.workingDirectory),
+        }),
+        result: Object.freeze({
+            outcome: closeCommandOutcome(value.result.outcome),
+            stdout: closeCommandStream(value.result.stdout, blobs),
+            stderr: closeCommandStream(value.result.stderr, blobs),
+        }),
+    });
+}
+function closeCommandInvocation(value) {
+    return value.kind === "argv"
+        ? Object.freeze({ kind: "argv", executable: value.executable, arguments: Object.freeze([...value.arguments]) })
+        : Object.freeze({ kind: "shell", command: value.command });
+}
+function closeWorkingDirectory(value) {
+    if (value.kind === "project-relative")
+        return Object.freeze({ kind: value.kind, path: value.path });
+    return Object.freeze({ kind: value.kind });
+}
+function closeCommandOutcome(value) {
+    if (value.kind === "exited")
+        return Object.freeze({ kind: value.kind, exitCode: value.exitCode });
+    return Object.freeze({ kind: value.kind, reason: value.reason });
+}
+function closeCommandStream(value, blobs) {
+    if (value.storage.kind === "inline") {
+        return Object.freeze({
+            kind: "inline",
+            text: value.storage.text,
+            retainedBytes: value.retainedBytes,
+            totalSafeUtf8Bytes: value.totalSafeUtf8Bytes,
+        });
+    }
+    return Object.freeze({
+        kind: "blob",
+        retainedBytes: value.retainedBytes,
+        totalSafeUtf8Bytes: value.totalSafeUtf8Bytes,
+        content: closeBlob(value.storage.ref, blobs),
+    });
+}
+function closeUsage(value) {
+    return Object.freeze({
+        collection: closeTraceCollection(value.collection),
+        observations: Object.freeze(value.observations.map((observation) => {
+            switch (observation.kind) {
+                case "token-bucket":
+                    return Object.freeze({
+                        provider: observation.provider,
+                        kind: observation.kind,
+                        bucket: observation.bucket,
+                        tokens: observation.tokens,
+                    });
+                case "request":
+                    return Object.freeze({
+                        provider: observation.provider,
+                        kind: observation.kind,
+                        requestKind: observation.requestKind,
+                    });
+                case "provider-cost":
+                    return Object.freeze({
+                        provider: observation.provider,
+                        kind: observation.kind,
+                        amount: observation.amount,
+                        currency: observation.currency,
+                    });
+            }
+        })),
+    });
+}
+function closeTimingInterval(value) {
+    return Object.freeze({
+        intervalId: value.intervalId,
+        phase: value.phase,
+        label: value.label,
+        startOffsetMs: value.startOffsetMs,
+        durationMs: value.durationMs,
+        parentIntervalId: value.parentIntervalId,
+        outcome: value.outcome,
+    });
+}
+function closeDiagnostic(value) {
+    return Object.freeze({
+        diagnosticId: value.diagnosticId,
+        kind: value.kind,
+        code: value.code,
+        phase: value.phase,
+        summary: value.summary,
+        causes: Object.freeze(value.causes.map((cause) => Object.freeze({ code: cause.code, summary: cause.summary }))),
+        redaction: value.redaction.state === "none"
+            ? Object.freeze({ state: "none" })
+            : Object.freeze({ state: "applied", replacements: value.redaction.replacements }),
+        sourceFrame: value.sourceFrame === null
+            ? null
+            : Object.freeze({
+                sourceItemId: value.sourceFrame.sourceItemId,
+                sha256: value.sourceFrame.sha256,
+                start: Object.freeze({ ...value.sourceFrame.start }),
+                end: Object.freeze({ ...value.sourceFrame.end }),
+            }),
+    });
+}
+function closeTraceCollection(value) {
+    return Object.freeze({ state: value.state, limitations: Object.freeze(value.limitations.map(closeJson)) });
+}
+function closeFileChanges(payload, blobs) {
+    return (0, file_changes_ts_1.projectFileChangesDomainDetail)({
+        attribution: Object.freeze({
+            kind: payload.attribution.kind,
+            policy: Object.freeze({
+                defaultPolicy: payload.attribution.policy.defaultPolicy,
+                include: Object.freeze([...payload.attribution.policy.include]),
+                ignore: Object.freeze([...payload.attribution.policy.ignore]),
+            }),
+        }),
+        collection: closeFileChangesCollection(payload.collection),
+        windows: Object.freeze(payload.windows.map((window) => Object.freeze({
+            windowId: window.windowId,
+            sequence: window.sequence,
+            changes: Object.freeze(window.changes.map((change) => Object.freeze({
+                changeId: change.changeId,
+                path: change.path,
+                kind: change.kind,
+                before: closeFileChangesEndpoint(change.before, blobs),
+                after: closeFileChangesEndpoint(change.after, blobs),
+            }))),
+        }))),
+        structuralPartial: hasStructuralFileChangesPartial(payload.collection),
+    });
+}
+function closeFileChangesCollection(value) {
+    return Object.freeze({
+        state: value.state,
+        limitations: Object.freeze(value.limitations.map(closeFileChangesLimitation)),
+    });
+}
+/**
+ * The snapshot conditional type intentionally erases some Schema union
+ * details. Re-establish the already-validated durable shape while closing it,
+ * rather than letting an unknown limitation escape the DomainView.
+ */
+function closeFileChangesLimitation(limitation) {
+    if (typeof limitation !== "object" || limitation === null || Array.isArray(limitation)) {
+        throw new TypeError("a File Changes collection limitation must be an object");
+    }
+    const value = limitation;
+    switch (value.code) {
+        case "capture-failed":
+        case "capture-interrupted":
+            if (!isCaptureLimitationStage(value.stage)
+                || !(typeof value.atWindowId === "string" || value.atWindowId === null)) {
+                throw new TypeError("a File Changes capture limitation has an invalid durable shape");
+            }
+            return Object.freeze({
+                code: value.code,
+                stage: value.stage,
+                atWindowId: value.atWindowId,
+            });
+        case "collection-cap-reached":
+            if (!isFileChangesCapTarget(value.target)
+                || !isPositiveSafeInteger(value.omittedAtLeast)
+                || !(typeof value.atWindowId === "string" || value.atWindowId === null)) {
+                throw new TypeError("a File Changes cap limitation has an invalid durable shape");
+            }
+            return Object.freeze({
+                code: "collection-cap-reached",
+                target: value.target,
+                omittedAtLeast: value.omittedAtLeast,
+                atWindowId: value.atWindowId,
+            });
+        case "unsupported-input":
+            if (value.target !== "endpoint-metadata" || !isPositiveSafeInteger(value.omittedAtLeast)) {
+                throw new TypeError("a File Changes unsupported-input limitation has an invalid durable shape");
+            }
+            return Object.freeze({
+                code: "unsupported-input",
+                target: "endpoint-metadata",
+                omittedAtLeast: value.omittedAtLeast,
+            });
+        default:
+            throw new TypeError("a File Changes collection limitation has an unknown code");
+    }
+}
+function closeFileChangesEndpoint(value, blobs) {
+    if (value.state === "absent")
+        return Object.freeze({ state: "absent" });
+    const revision = value.revision;
+    switch (revision.kind) {
+        case "text":
+            return Object.freeze({
+                state: "present",
+                revision: Object.freeze({
+                    kind: "text",
+                    sha256: revision.sha256,
+                    byteLength: revision.byteLength,
+                    content: revision.content.state === "available"
+                        ? Object.freeze({ state: "available", content: closeBlob(revision.content.ref, blobs) })
+                        : Object.freeze({ state: "omitted", reason: "collection-cap" }),
+                }),
+            });
+        case "elided":
+            return Object.freeze({
+                state: "present",
+                revision: Object.freeze({
+                    kind: "elided",
+                    reason: revision.reason,
+                    byteLength: revision.byteLength,
+                }),
+            });
+        case "unavailable":
+            return Object.freeze({
+                state: "present",
+                revision: Object.freeze({ kind: "unavailable", reason: revision.reason }),
+            });
+    }
+}
+function hasStructuralFileChangesPartial(collection) {
+    return collection.state === "partial" && collection.limitations
+        .map(closeFileChangesLimitation)
+        .some((limitation) => {
+        switch (limitation.code) {
+            case "capture-failed":
+            case "capture-interrupted":
+            case "unsupported-input":
+                return true;
+            case "collection-cap-reached":
+                return limitation.target === "window" || limitation.target === "change" || limitation.target === "json-byte";
+        }
+    });
+}
+function isCaptureLimitationStage(value) {
+    return value === "checkpoint" || value === "export" || value === "finalizer-export" || value === "normalize";
+}
+function isFileChangesCapTarget(value) {
+    return value === "window" || value === "change" || value === "content-blob" || value === "content-byte" || value === "json-byte";
+}
+function isPositiveSafeInteger(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value > 0;
+}
+function closeSources(payload, blobs) {
+    return Object.freeze({
+        items: Object.freeze(payload.items.map((item) => Object.freeze({
+            sourceItemId: item.sourceItemId,
+            path: item.path,
+            sha256: item.sha256,
+            content: closeBlob(item.content, blobs),
+        }))),
+    });
+}
+function closeAssertionMaterial(value) {
+    if (value.kind === "snapshot") {
+        return Object.freeze({ kind: "snapshot", value: closeJson(value.value) });
+    }
+    const blob = value;
+    return Object.freeze({ kind: "blob", encoding: blob.encoding, byteLength: blob.byteLength, preview: blob.preview });
+}
+function closeBlob(ref, blobs) {
+    const bytes = blobs.bytes(ref);
+    if (effect_1.Either.isLeft(bytes))
+        return Object.freeze({ state: "unavailable" });
+    try {
+        return Object.freeze({
+            state: "available",
+            text: new TextDecoder("utf-8", { fatal: true }).decode(bytes.right),
+        });
+    }
+    catch {
+        return Object.freeze({ state: "binary" });
+    }
+}
+function closeJson(value) {
+    if (value === null || typeof value === "string" || typeof value === "boolean")
+        return value;
+    if (typeof value === "number")
+        return Number.isFinite(value) ? value : null;
+    if (Array.isArray(value))
+        return Object.freeze(value.map(closeJson));
+    if (typeof value === "object" && value !== null) {
+        const closed = {};
+        for (const [key, item] of Object.entries(value))
+            closed[key] = closeJson(item);
+        return Object.freeze(closed);
+    }
+    return null;
+}
+//# sourceMappingURL=bindings.cjs.map