niceeval 0.12.1 → 0.12.2-canary.31

package/dist/runner/reporters/braintrust.cjs

@@ -0,0 +1,302 @@
+"use strict";
+const __niceevalDynamicImport = (specifier) => import(specifier);
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Braintrust = Braintrust;
+exports.toBraintrustEvent = toBraintrustEvent;
+const sink_ts_1 = require("../feedback/sink.cjs");
+const NOT_PROJECTED_ASSESSMENT = Object.freeze({
+    state: "not-projected",
+    reason: "reporter-contract-does-not-provide-record-projections",
+});
+/**
+ * 创建 Braintrust 报告器。挂在 `defineConfig({ reporters })` 上观测整次运行,
+ * 或挂在单个 eval 的 `reporters` 上只观测它(同一实例被多个 eval 引用时共享一个实验)。
+ */
+function Braintrust(config = {}) {
+    let sdk;
+    let experiment;
+    return {
+        async onInvocationStart(evals) {
+            sdk = await loadBraintrustSdk();
+            experiment = await sdk.init({
+                project: config.projectId ? undefined : (config.project ?? "niceeval"),
+                projectId: config.projectId,
+                experiment: config.experiment,
+                baseExperiment: config.baseExperiment,
+                baseExperimentId: config.baseExperimentId,
+                update: config.update,
+                apiKey: config.apiKey,
+                // 不再写顶层单一 agent:一次 Invocation 可能横跨多个 (agent, model, flags) 配置,
+                // 启动时还没有任何结果,写一个必然只代表其中一份配置的值就是撒谎(见
+                // docs/runner.md「Reporter 与运行器事件」)。每行自己的 agent 身份仍在
+                // toBraintrustEvent() 的 metadata.agent 里,逐 attempt 精确;跨行的 agent 集合可以
+                // 从 Braintrust 自己按 metadata.agent 分组得到,不需要实验级再存一份。
+                metadata: {
+                    evals: evals.map((e) => e.id),
+                    ...config.metadata,
+                },
+                // 不设为全局 current experiment,避免污染用户代码里的 braintrust 全局态。
+                setCurrent: false,
+            });
+        },
+        onEvalComplete(result) {
+            experiment?.log(toBraintrustEvent(result, NOT_PROJECTED_ASSESSMENT));
+        },
+        async onInvocationComplete() {
+            if (!experiment)
+                return;
+            try {
+                await sdk?.flush();
+                const summary = await experiment.summarize();
+                if (summary.experimentUrl) {
+                    (0, sink_ts_1.reportActivity)(`Braintrust experiment: ${summary.experimentUrl}`);
+                }
+            }
+            finally {
+                experiment = undefined;
+                sdk = undefined;
+            }
+        },
+    };
+}
+/**
+ * EvalResult identity/transport data plus an optional post-Record assessment
+ * become one Braintrust row. Assertions, Verdict, and Score only enter through
+ * that assessment's public Attachment projections; this function never reads
+ * a historical result graph.
+ */
+function toBraintrustEvent(result, assessment = NOT_PROJECTED_ASSESSMENT) {
+    const projectedVerdict = assessment.state === "projected"
+        ? availableProjectionValue(assessment.verdict)
+        : undefined;
+    const verdict = projectedVerdict ?? result.verdict;
+    const metrics = {};
+    if (result.startedAt) {
+        const start = Date.parse(result.startedAt) / 1000;
+        if (Number.isFinite(start)) {
+            metrics.start = start;
+            metrics.end = start + result.durationMs / 1000;
+        }
+    }
+    if (result.usage) {
+        const { inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens, reasoningTokens, requests } = result.usage;
+        if (inputTokens !== undefined)
+            metrics.prompt_tokens = inputTokens;
+        if (outputTokens !== undefined)
+            metrics.completion_tokens = outputTokens;
+        if (inputTokens !== undefined && outputTokens !== undefined)
+            metrics.tokens = inputTokens + outputTokens;
+        if (cacheReadTokens !== undefined)
+            metrics.cache_read_tokens = cacheReadTokens;
+        if (cacheCreationTokens !== undefined)
+            metrics.cache_creation_tokens = cacheCreationTokens;
+        if (reasoningTokens !== undefined)
+            metrics.reasoning_tokens = reasoningTokens;
+        if (requests !== undefined)
+            metrics.requests = requests;
+    }
+    if (result.estimatedCostUSD !== undefined)
+        metrics.estimated_cost_usd = result.estimatedCostUSD;
+    const metadata = {
+        eval: result.id,
+        agent: result.agent,
+        attempt: result.attempt,
+        verdict,
+        status: verdict,
+        evaluationAlgorithm: result.evaluationAlgorithm,
+        evaluationKind: result.evaluationKind,
+        assessmentProjection: assessment.state,
+    };
+    const scores = {};
+    if (assessment.state === "projected") {
+        metadata.assertions = projectionMetadata(assessment.assertions, (assertions) => ({ entries: assertionMetadata(assertions.entries) }));
+        metadata.verdictProjection = projectionMetadata(assessment.verdict, (value) => ({ value }));
+        if (assessment.score !== undefined) {
+            metadata.score = projectionMetadata(assessment.score, scoreMetadata);
+            const score = availableProjectionValue(assessment.score);
+            if (score?.state === "complete")
+                scores["niceeval:score"] = score.earned;
+        }
+    }
+    else {
+        metadata.assessmentProjectionReason = assessment.reason;
+    }
+    if (result.model !== undefined)
+        metadata.model = result.model;
+    if (result.experimentId !== undefined)
+        metadata.experiment = result.experimentId;
+    if (result.experiment?.flags && Object.keys(result.experiment.flags).length > 0) {
+        metadata.flags = result.experiment.flags;
+    }
+    if (result.skipReason !== undefined)
+        metadata.skipReason = result.skipReason;
+    // 一次运行内 (experiment, eval, agent, model, attempt) 唯一;Braintrust 按 id 合并重复行。
+    const id = [result.experimentId ?? "", result.id, result.agent, result.model ?? "", `a${result.attempt}`].join("|");
+    return {
+        id,
+        input: result.description ?? result.id,
+        output: lastAssistantText(result.events),
+        error: result.error,
+        metadata,
+        metrics,
+        ...(Object.keys(scores).length > 0 ? { scores } : {}),
+    };
+}
+function availableProjectionValue(entry) {
+    return entry.state === "attachment-result" && entry.attachment.state === "available"
+        ? entry.attachment.value
+        : undefined;
+}
+function projectionMetadata(entry, available) {
+    if (entry.state !== "attachment-result") {
+        return { state: entry.state };
+    }
+    switch (entry.attachment.state) {
+        case "available":
+            return { state: "available", ...available(entry.attachment.value) };
+        case "unavailable":
+            return { state: "unavailable" };
+        case "migration-required":
+            return {
+                state: "migration-required",
+                from: entry.attachment.from,
+                to: entry.attachment.to,
+                command: entry.attachment.command,
+            };
+        case "migration-unavailable":
+            return {
+                state: "migration-unavailable",
+                from: entry.attachment.from,
+                to: entry.attachment.to,
+                reason: entry.attachment.reason,
+            };
+        case "unsupported":
+            return { state: "unsupported", schemaId: entry.attachment.schemaId };
+        case "invalid":
+            return {
+                state: "invalid",
+                issues: entry.attachment.issues.map((issue) => ({
+                    code: issue.code,
+                    path: [...issue.path],
+                })),
+            };
+    }
+}
+function assertionMetadata(entries) {
+    const metadata = [];
+    for (const entry of entries) {
+        const display = entry.entry.display;
+        const result = entry.entry.result;
+        const item = {
+            entryId: entry.entry.entryId,
+            groupPath: [...display.groupPath],
+            outcome: entry.state === "available" ? result.state : entry.state,
+            coverage: coverageMetadata(entry.entry.coverage),
+            limitations: entry.entry.limitations.map(limitationMetadata),
+            scoreContribution: scoreContributionMetadata(result.score),
+        };
+        if (display.label !== undefined)
+            item.label = display.label;
+        if (display.key !== undefined)
+            item.key = display.key;
+        const reason = assertionReason(entry);
+        if (reason !== undefined)
+            item.reason = reason;
+        metadata.push(item);
+    }
+    return metadata;
+}
+function assertionReason(entry) {
+    if (entry.state !== "available")
+        return entry.reason;
+    switch (entry.entry.result.state) {
+        case "matched":
+            return undefined;
+        case "mismatched":
+        case "unavailable":
+        case "errored":
+        case "not-applicable":
+            return entry.entry.result.reason;
+    }
+}
+function coverageMetadata(coverage) {
+    const metadata = { state: coverage.state };
+    if (coverage.state !== "complete")
+        metadata.reason = coverage.reason;
+    return metadata;
+}
+function limitationMetadata(limitation) {
+    switch (limitation.kind) {
+        case "redacted":
+            return { kind: limitation.kind, fieldCount: limitation.fieldCount };
+        case "sampled":
+            return {
+                kind: limitation.kind,
+                captured: limitation.captured,
+                ...(limitation.knownTotal === undefined ? {} : { knownTotal: limitation.knownTotal }),
+            };
+        case "truncated":
+            return { kind: limitation.kind, omittedBytes: limitation.omittedBytes };
+        case "provider-limited":
+            return { kind: limitation.kind };
+    }
+}
+function scoreContributionMetadata(contribution) {
+    switch (contribution.state) {
+        case "not-scored":
+            return { state: contribution.state };
+        case "earned":
+            return {
+                state: contribution.state,
+                points: contribution.points,
+                earned: contribution.earned,
+            };
+        case "unavailable":
+            return {
+                state: contribution.state,
+                points: contribution.points,
+                reason: contribution.reason,
+            };
+    }
+}
+function scoreMetadata(score) {
+    switch (score.state) {
+        case "complete":
+            return { state: score.state, earned: score.earned, comparable: score.comparable };
+        case "partial":
+            return {
+                state: score.state,
+                earned: score.earned,
+                reasons: [...score.reasons],
+                comparable: score.comparable,
+            };
+        case "unavailable":
+            return {
+                state: score.state,
+                reasons: [...score.reasons],
+                comparable: score.comparable,
+            };
+    }
+}
+/** agent 的最终回复文本(事件流里最后一条 assistant message);没有就不填。 */
+function lastAssistantText(events) {
+    if (!events)
+        return undefined;
+    for (let i = events.length - 1; i >= 0; i--) {
+        const e = events[i];
+        if (e.type === "message" && e.role === "assistant")
+            return e.text;
+    }
+    return undefined;
+}
+/** 动态 import 说明:specifier 放变量里,`braintrust` 不进编译期依赖,没装也能 typecheck。 */
+const BRAINTRUST_PACKAGE = "braintrust";
+async function loadBraintrustSdk() {
+    try {
+        return (await __niceevalDynamicImport(BRAINTRUST_PACKAGE));
+    }
+    catch {
+        throw new Error("The 'braintrust' package is required for the Braintrust reporter but was not found. Install it with: npm install braintrust (and set BRAINTRUST_API_KEY).");
+    }
+}
+//# sourceMappingURL=braintrust.cjs.map

package/dist/runner/reporters/braintrust.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"braintrust.js","sourceRoot":"","sources":["braintrust.ts"],"names":[],"mappings":";;AAqFA,gCA8CC;AAOD,8CA4EC;AA7MD,iDAAqD;AAoErD,MAAM,wBAAwB,GAAyB,MAAM,CAAC,MAAM,CAAC;IACjE,KAAK,EAAE,eAAwB;IAC/B,MAAM,EAAE,uDAAgE;CAC3E,CAAC,CAAC;AACH;;;GAGG;AACH,SAAgB,UAAU,CAAC,SAA2B,EAAE;IACpD,IAAI,GAA8B,CAAC;IACnC,IAAI,UAA4C,CAAC;IACjD,OAAO;QACH,KAAK,CAAC,iBAAiB,CAAC,KAAK;YACzB,GAAG,GAAG,MAAM,iBAAiB,EAAE,CAAC;YAChC,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC;gBACxB,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,UAAU,CAAC;gBACtE,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,+DAA+D;gBAC/D,oCAAoC;gBACpC,oDAAoD;gBACpD,qEAAqE;gBACrE,mDAAmD;gBACnD,QAAQ,EAAE;oBACN,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,MAAM,CAAC,QAAQ;iBACrB;gBACD,sDAAsD;gBACtD,UAAU,EAAE,KAAK;aACpB,CAAC,CAAC;QACP,CAAC;QACD,cAAc,CAAC,MAAM;YACjB,UAAU,EAAE,GAAG,CAAC,iBAAiB,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC,CAAC;QACzE,CAAC;QACD,KAAK,CAAC,oBAAoB;YACtB,IAAI,CAAC,UAAU;gBACX,OAAO;YACX,IAAI,CAAC;gBACD,MAAM,GAAG,EAAE,KAAK,EAAE,CAAC;gBACnB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;gBAC7C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;oBACxB,IAAA,wBAAc,EAAC,0BAA0B,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;gBACtE,CAAC;YACL,CAAC;oBACO,CAAC;gBACL,UAAU,GAAG,SAAS,CAAC;gBACvB,GAAG,GAAG,SAAS,CAAC;YACpB,CAAC;QACL,CAAC;KACJ,CAAC;AACN,CAAC;AACD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,MAAkB,EAAE,aAAmC,wBAAwB;IAC7G,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,KAAK,WAAW;QACrD,CAAC,CAAC,wBAAwB,CAAC,UAAU,CAAC,OAAO,CAAC;QAC9C,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,OAAO,GAAG,gBAAgB,IAAI,MAAM,CAAC,OAAO,CAAC;IACnD,MAAM,OAAO,GAAsC,EAAE,CAAC;IACtD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;QAClD,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;YACtB,OAAO,CAAC,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;QACnD,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,eAAe,EAAE,mBAAmB,EAAE,eAAe,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC;QACpH,IAAI,WAAW,KAAK,SAAS;YACzB,OAAO,CAAC,aAAa,GAAG,WAAW,CAAC;QACxC,IAAI,YAAY,KAAK,SAAS;YAC1B,OAAO,CAAC,iBAAiB,GAAG,YAAY,CAAC;QAC7C,IAAI,WAAW,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS;YACvD,OAAO,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC;QAChD,IAAI,eAAe,KAAK,SAAS;YAC7B,OAAO,CAAC,iBAAiB,GAAG,eAAe,CAAC;QAChD,IAAI,mBAAmB,KAAK,SAAS;YACjC,OAAO,CAAC,qBAAqB,GAAG,mBAAmB,CAAC;QACxD,IAAI,eAAe,KAAK,SAAS;YAC7B,OAAO,CAAC,gBAAgB,GAAG,eAAe,CAAC;QAC/C,IAAI,QAAQ,KAAK,SAAS;YACtB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACpC,CAAC;IACD,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS;QACrC,OAAO,CAAC,kBAAkB,GAAG,MAAM,CAAC,gBAAgB,CAAC;IACzD,MAAM,QAAQ,GAAyC;QACnD,IAAI,EAAE,MAAM,CAAC,EAAE;QACf,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,OAAO;QACP,MAAM,EAAE,OAAO;QACf,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;QAC/C,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,oBAAoB,EAAE,UAAU,CAAC,KAAK;KACzC,CAAC;IACF,MAAM,MAAM,GAAsC,EAAE,CAAC;IACrD,IAAI,UAAU,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QACnC,QAAQ,CAAC,UAAU,GAAG,kBAAkB,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACtI,QAAQ,CAAC,iBAAiB,GAAG,kBAAkB,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAC5F,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,QAAQ,CAAC,KAAK,GAAG,kBAAkB,CAAC,UAAU,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,wBAAwB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,KAAK,EAAE,KAAK,KAAK,UAAU;gBAC3B,MAAM,CAAC,gBAAgB,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QAChD,CAAC;IACL,CAAC;SACI,CAAC;QACF,QAAQ,CAAC,0BAA0B,GAAG,UAAU,CAAC,MAAM,CAAC;IAC5D,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;QAC1B,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAClC,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS;QACjC,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC;IAC9C,IAAI,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9E,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;IAC7C,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS;QAC/B,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IAC5C,4EAA4E;IAC5E,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpH,OAAO;QACH,EAAE;QACF,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,EAAE;QACtC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC;QACxC,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ;QACR,OAAO;QACP,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxD,CAAC;AACN,CAAC;AACD,SAAS,wBAAwB,CAAQ,KAAuC;IAC5E,OAAO,KAAK,CAAC,KAAK,KAAK,mBAAmB,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,KAAK,WAAW;QAChF,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK;QACxB,CAAC,CAAC,SAAS,CAAC;AACpB,CAAC;AACD,SAAS,kBAAkB,CAAQ,KAAuC,EAAE,SAAiE;IACzI,IAAI,KAAK,CAAC,KAAK,KAAK,mBAAmB,EAAE,CAAC;QACtC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IACD,QAAQ,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC7B,KAAK,WAAW;YACZ,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,KAAK,aAAa;YACd,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC;QACpC,KAAK,oBAAoB;YACrB,OAAO;gBACH,KAAK,EAAE,oBAAoB;gBAC3B,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI;gBAC3B,EAAE,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE;gBACvB,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO;aACpC,CAAC;QACN,KAAK,uBAAuB;YACxB,OAAO;gBACH,KAAK,EAAE,uBAAuB;gBAC9B,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI;gBAC3B,EAAE,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE;gBACvB,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM;aAClC,CAAC;QACN,KAAK,aAAa;YACd,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QACzE,KAAK,SAAS;YACV,OAAO;gBACH,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;iBACxB,CAAC,CAAC;aACN,CAAC;IACV,CAAC;AACL,CAAC;AACD,SAAS,iBAAiB,CAAC,OAAiD;IACxE,MAAM,QAAQ,GAAgB,EAAE,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,IAAI,GAAyC;YAC/C,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO;YAC5B,SAAS,EAAE,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC;YACjC,OAAO,EAAE,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK;YACjE,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC;YAChD,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,kBAAkB,CAAC;YAC5D,iBAAiB,EAAE,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC;SAC7D,CAAC;QACF,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;YAC3B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS;YACzB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM,KAAK,SAAS;YACpB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC;AACD,SAAS,eAAe,CAAC,KAAoC;IACzD,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW;QAC3B,OAAO,KAAK,CAAC,MAAM,CAAC;IACxB,QAAQ,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,SAAS;YACV,OAAO,SAAS,CAAC;QACrB,KAAK,YAAY,CAAC;QAClB,KAAK,aAAa,CAAC;QACnB,KAAK,SAAS,CAAC;QACf,KAAK,gBAAgB;YACjB,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACzC,CAAC;AACL,CAAC;AACD,SAAS,gBAAgB,CAAC,QAA6B;IACnD,MAAM,QAAQ,GAAyC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;IACjF,IAAI,QAAQ,CAAC,KAAK,KAAK,UAAU;QAC7B,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IACtC,OAAO,QAAQ,CAAC;AACpB,CAAC;AACD,SAAS,kBAAkB,CAAC,UAAiC;IACzD,QAAQ,UAAU,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,UAAU;YACX,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC;QACxE,KAAK,SAAS;YACV,OAAO;gBACH,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,GAAG,CAAC,UAAU,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC;aACxF,CAAC;QACN,KAAK,WAAW;YACZ,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,UAAU,CAAC,YAAY,EAAE,CAAC;QAC5E,KAAK,kBAAkB;YACnB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;IACzC,CAAC;AACL,CAAC;AACD,SAAS,yBAAyB,CAAC,YAAiC;IAChE,QAAQ,YAAY,CAAC,KAAK,EAAE,CAAC;QACzB,KAAK,YAAY;YACb,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,EAAE,CAAC;QACzC,KAAK,QAAQ;YACT,OAAO;gBACH,KAAK,EAAE,YAAY,CAAC,KAAK;gBACzB,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,MAAM,EAAE,YAAY,CAAC,MAAM;aAC9B,CAAC;QACN,KAAK,aAAa;YACd,OAAO;gBACH,KAAK,EAAE,YAAY,CAAC,KAAK;gBACzB,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,MAAM,EAAE,YAAY,CAAC,MAAM;aAC9B,CAAC;IACV,CAAC;AACL,CAAC;AACD,SAAS,aAAa,CAAC,KAAwB;IAC3C,QAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;QAClB,KAAK,UAAU;YACX,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QACtF,KAAK,SAAS;YACV,OAAO;gBACH,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;gBAC3B,UAAU,EAAE,KAAK,CAAC,UAAU;aAC/B,CAAC;QACN,KAAK,aAAa;YACd,OAAO;gBACH,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;gBAC3B,UAAU,EAAE,KAAK,CAAC,UAAU;aAC/B,CAAC;IACV,CAAC;AACL,CAAC;AACD,uDAAuD;AACvD,SAAS,iBAAiB,CAAC,MAA4B;IACnD,IAAI,CAAC,MAAM;QACP,OAAO,SAAS,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW;YAC9C,OAAO,CAAC,CAAC,IAAI,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,CAAC;AACrB,CAAC;AACD,uEAAuE;AACvE,MAAM,kBAAkB,GAAG,YAAY,CAAC;AACxC,KAAK,UAAU,iBAAiB;IAC5B,IAAI,CAAC;QACD,OAAO,CAAC,MAAM,uBAAuB,CAAC,kBAAkB,CAAC,CAA6B,CAAC;IAC3F,CAAC;IACD,MAAM,CAAC;QACH,MAAM,IAAI,KAAK,CAAC,2JAA2J,CAAC,CAAC;IACjL,CAAC;AACL,CAAC","sourcesContent":["// Braintrust 报告器:把一次 niceeval 运行作为一个 Braintrust experiment 上报,\n// 每个 attempt 一行(scores = 断言,metrics = 时长/token/成本),跨提交比较与团队共享。\n// `braintrust` 是可选依赖:动态 import,装了才用得上,没装在 onInvocationStart 报错\n// (reporter 错误按框架约定只记 diagnostic,不会让运行崩)。\nimport type { EvalResult, JsonValue, Reporter } from \"../../types.ts\";\nimport type { AssertionCoverageV1, AssertionEntryReadV1, AssertionLimitationV1, AssertionsProjectionV1, ScoreContributionV1, } from \"../../assertions/record/model.ts\";\nimport type { ScoreProjectionV1 } from \"../../eval/record/score.ts\";\nimport type { AttemptSlotProjectedEntry } from \"../../projection/index.ts\";\nimport type { Verdict } from \"../../shared/types.ts\";\nimport { reportActivity } from \"../feedback/sink.ts\";\nexport interface BraintrustConfig {\n    /** Braintrust 项目名;省略时用 \"niceeval\"。 */\n    project?: string;\n    /** Braintrust 项目 id;与 project 给一个即可。 */\n    projectId?: string;\n    /** 实验名;省略时由 Braintrust 自动命名。 */\n    experiment?: string;\n    /** 作为对比基线(diff base)的既有实验名。 */\n    baseExperiment?: string;\n    /** 作为对比基线(diff base)的既有实验 id。 */\n    baseExperimentId?: string;\n    /** true = 更新同名既有实验,而不是新建一个。 */\n    update?: boolean;\n    /** 实验级附加 metadata;与 niceeval 自动写入的字段合并,同名以这里为准。 */\n    metadata?: globalThis.Record<string, JsonValue>;\n    /** API key;省略时 SDK 读 BRAINTRUST_API_KEY 环境变量。 */\n    apiKey?: string;\n}\n/**\n * Braintrust SDK 的最小类型面。本地声明是为了不把 `braintrust` 变成编译期依赖;\n * 动态 import 后按这个形状断言。\n */\ninterface BraintrustSdk {\n    // SDK 里 init 同步返回(登录与建实验都是惰性的);统一 await,同步/异步都兼容。\n    init(options: {\n        project?: string;\n        projectId?: string;\n        experiment?: string;\n        baseExperiment?: string;\n        baseExperimentId?: string;\n        update?: boolean;\n        apiKey?: string;\n        metadata?: globalThis.Record<string, JsonValue>;\n        setCurrent?: boolean;\n    }): BraintrustExperiment | Promise<BraintrustExperiment>;\n    flush(): Promise<void>;\n}\ninterface BraintrustExperiment {\n    log(event: BraintrustLogEvent): void;\n    summarize(): Promise<{\n        experimentUrl?: string;\n    }>;\n}\nexport interface BraintrustLogEvent {\n    id?: string;\n    input?: string;\n    output?: string;\n    error?: EvalResult[\"error\"];\n    scores?: globalThis.Record<string, number>;\n    metadata?: globalThis.Record<string, JsonValue>;\n    metrics?: globalThis.Record<string, number>;\n}\n/**\n * A post-Record Projection calculation supplies this optional assessment. The\n * live Reporter callback currently has no RecordProjection input, so its\n * absence is represented explicitly rather than as a made-up Attachment read.\n */\nexport type BraintrustAssessment = {\n    readonly state: \"projected\";\n    /** Exact Report-style public projections, including every data state. */\n    readonly assertions: AttemptSlotProjectedEntry<AssertionsProjectionV1<unknown>>;\n    readonly verdict: AttemptSlotProjectedEntry<Verdict>;\n    readonly score?: AttemptSlotProjectedEntry<ScoreProjectionV1>;\n} | {\n    readonly state: \"not-projected\";\n    readonly reason: \"reporter-contract-does-not-provide-record-projections\";\n};\nconst NOT_PROJECTED_ASSESSMENT: BraintrustAssessment = Object.freeze({\n    state: \"not-projected\" as const,\n    reason: \"reporter-contract-does-not-provide-record-projections\" as const,\n});\n/**\n * 创建 Braintrust 报告器。挂在 `defineConfig({ reporters })` 上观测整次运行,\n * 或挂在单个 eval 的 `reporters` 上只观测它(同一实例被多个 eval 引用时共享一个实验)。\n */\nexport function Braintrust(config: BraintrustConfig = {}): Reporter {\n    let sdk: BraintrustSdk | undefined;\n    let experiment: BraintrustExperiment | undefined;\n    return {\n        async onInvocationStart(evals) {\n            sdk = await loadBraintrustSdk();\n            experiment = await sdk.init({\n                project: config.projectId ? undefined : (config.project ?? \"niceeval\"),\n                projectId: config.projectId,\n                experiment: config.experiment,\n                baseExperiment: config.baseExperiment,\n                baseExperimentId: config.baseExperimentId,\n                update: config.update,\n                apiKey: config.apiKey,\n                // 不再写顶层单一 agent:一次 Invocation 可能横跨多个 (agent, model, flags) 配置,\n                // 启动时还没有任何结果,写一个必然只代表其中一份配置的值就是撒谎(见\n                // docs/runner.md「Reporter 与运行器事件」)。每行自己的 agent 身份仍在\n                // toBraintrustEvent() 的 metadata.agent 里,逐 attempt 精确;跨行的 agent 集合可以\n                // 从 Braintrust 自己按 metadata.agent 分组得到,不需要实验级再存一份。\n                metadata: {\n                    evals: evals.map((e) => e.id),\n                    ...config.metadata,\n                },\n                // 不设为全局 current experiment,避免污染用户代码里的 braintrust 全局态。\n                setCurrent: false,\n            });\n        },\n        onEvalComplete(result) {\n            experiment?.log(toBraintrustEvent(result, NOT_PROJECTED_ASSESSMENT));\n        },\n        async onInvocationComplete() {\n            if (!experiment)\n                return;\n            try {\n                await sdk?.flush();\n                const summary = await experiment.summarize();\n                if (summary.experimentUrl) {\n                    reportActivity(`Braintrust experiment: ${summary.experimentUrl}`);\n                }\n            }\n            finally {\n                experiment = undefined;\n                sdk = undefined;\n            }\n        },\n    };\n}\n/**\n * EvalResult identity/transport data plus an optional post-Record assessment\n * become one Braintrust row. Assertions, Verdict, and Score only enter through\n * that assessment's public Attachment projections; this function never reads\n * a historical result graph.\n */\nexport function toBraintrustEvent(result: EvalResult, assessment: BraintrustAssessment = NOT_PROJECTED_ASSESSMENT): BraintrustLogEvent {\n    const projectedVerdict = assessment.state === \"projected\"\n        ? availableProjectionValue(assessment.verdict)\n        : undefined;\n    const verdict = projectedVerdict ?? result.verdict;\n    const metrics: globalThis.Record<string, number> = {};\n    if (result.startedAt) {\n        const start = Date.parse(result.startedAt) / 1000;\n        if (Number.isFinite(start)) {\n            metrics.start = start;\n            metrics.end = start + result.durationMs / 1000;\n        }\n    }\n    if (result.usage) {\n        const { inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens, reasoningTokens, requests } = result.usage;\n        if (inputTokens !== undefined)\n            metrics.prompt_tokens = inputTokens;\n        if (outputTokens !== undefined)\n            metrics.completion_tokens = outputTokens;\n        if (inputTokens !== undefined && outputTokens !== undefined)\n            metrics.tokens = inputTokens + outputTokens;\n        if (cacheReadTokens !== undefined)\n            metrics.cache_read_tokens = cacheReadTokens;\n        if (cacheCreationTokens !== undefined)\n            metrics.cache_creation_tokens = cacheCreationTokens;\n        if (reasoningTokens !== undefined)\n            metrics.reasoning_tokens = reasoningTokens;\n        if (requests !== undefined)\n            metrics.requests = requests;\n    }\n    if (result.estimatedCostUSD !== undefined)\n        metrics.estimated_cost_usd = result.estimatedCostUSD;\n    const metadata: globalThis.Record<string, JsonValue> = {\n        eval: result.id,\n        agent: result.agent,\n        attempt: result.attempt,\n        verdict,\n        status: verdict,\n        evaluationAlgorithm: result.evaluationAlgorithm,\n        evaluationKind: result.evaluationKind,\n        assessmentProjection: assessment.state,\n    };\n    const scores: globalThis.Record<string, number> = {};\n    if (assessment.state === \"projected\") {\n        metadata.assertions = projectionMetadata(assessment.assertions, (assertions) => ({ entries: assertionMetadata(assertions.entries) }));\n        metadata.verdictProjection = projectionMetadata(assessment.verdict, (value) => ({ value }));\n        if (assessment.score !== undefined) {\n            metadata.score = projectionMetadata(assessment.score, scoreMetadata);\n            const score = availableProjectionValue(assessment.score);\n            if (score?.state === \"complete\")\n                scores[\"niceeval:score\"] = score.earned;\n        }\n    }\n    else {\n        metadata.assessmentProjectionReason = assessment.reason;\n    }\n    if (result.model !== undefined)\n        metadata.model = result.model;\n    if (result.experimentId !== undefined)\n        metadata.experiment = result.experimentId;\n    if (result.experiment?.flags && Object.keys(result.experiment.flags).length > 0) {\n        metadata.flags = result.experiment.flags;\n    }\n    if (result.skipReason !== undefined)\n        metadata.skipReason = result.skipReason;\n    // 一次运行内 (experiment, eval, agent, model, attempt) 唯一;Braintrust 按 id 合并重复行。\n    const id = [result.experimentId ?? \"\", result.id, result.agent, result.model ?? \"\", `a${result.attempt}`].join(\"|\");\n    return {\n        id,\n        input: result.description ?? result.id,\n        output: lastAssistantText(result.events),\n        error: result.error,\n        metadata,\n        metrics,\n        ...(Object.keys(scores).length > 0 ? { scores } : {}),\n    };\n}\nfunction availableProjectionValue<Value>(entry: AttemptSlotProjectedEntry<Value>): Value | undefined {\n    return entry.state === \"attachment-result\" && entry.attachment.state === \"available\"\n        ? entry.attachment.value\n        : undefined;\n}\nfunction projectionMetadata<Value>(entry: AttemptSlotProjectedEntry<Value>, available: (value: Value) => globalThis.Record<string, JsonValue>): JsonValue {\n    if (entry.state !== \"attachment-result\") {\n        return { state: entry.state };\n    }\n    switch (entry.attachment.state) {\n        case \"available\":\n            return { state: \"available\", ...available(entry.attachment.value) };\n        case \"unavailable\":\n            return { state: \"unavailable\" };\n        case \"migration-required\":\n            return {\n                state: \"migration-required\",\n                from: entry.attachment.from,\n                to: entry.attachment.to,\n                command: entry.attachment.command,\n            };\n        case \"migration-unavailable\":\n            return {\n                state: \"migration-unavailable\",\n                from: entry.attachment.from,\n                to: entry.attachment.to,\n                reason: entry.attachment.reason,\n            };\n        case \"unsupported\":\n            return { state: \"unsupported\", schemaId: entry.attachment.schemaId };\n        case \"invalid\":\n            return {\n                state: \"invalid\",\n                issues: entry.attachment.issues.map((issue) => ({\n                    code: issue.code,\n                    path: [...issue.path],\n                })),\n            };\n    }\n}\nfunction assertionMetadata(entries: readonly AssertionEntryReadV1<unknown>[]): JsonValue {\n    const metadata: JsonValue[] = [];\n    for (const entry of entries) {\n        const display = entry.entry.display;\n        const result = entry.entry.result;\n        const item: globalThis.Record<string, JsonValue> = {\n            entryId: entry.entry.entryId,\n            groupPath: [...display.groupPath],\n            outcome: entry.state === \"available\" ? result.state : entry.state,\n            coverage: coverageMetadata(entry.entry.coverage),\n            limitations: entry.entry.limitations.map(limitationMetadata),\n            scoreContribution: scoreContributionMetadata(result.score),\n        };\n        if (display.label !== undefined)\n            item.label = display.label;\n        if (display.key !== undefined)\n            item.key = display.key;\n        const reason = assertionReason(entry);\n        if (reason !== undefined)\n            item.reason = reason;\n        metadata.push(item);\n    }\n    return metadata;\n}\nfunction assertionReason(entry: AssertionEntryReadV1<unknown>): string | undefined {\n    if (entry.state !== \"available\")\n        return entry.reason;\n    switch (entry.entry.result.state) {\n        case \"matched\":\n            return undefined;\n        case \"mismatched\":\n        case \"unavailable\":\n        case \"errored\":\n        case \"not-applicable\":\n            return entry.entry.result.reason;\n    }\n}\nfunction coverageMetadata(coverage: AssertionCoverageV1): globalThis.Record<string, JsonValue> {\n    const metadata: globalThis.Record<string, JsonValue> = { state: coverage.state };\n    if (coverage.state !== \"complete\")\n        metadata.reason = coverage.reason;\n    return metadata;\n}\nfunction limitationMetadata(limitation: AssertionLimitationV1): globalThis.Record<string, JsonValue> {\n    switch (limitation.kind) {\n        case \"redacted\":\n            return { kind: limitation.kind, fieldCount: limitation.fieldCount };\n        case \"sampled\":\n            return {\n                kind: limitation.kind,\n                captured: limitation.captured,\n                ...(limitation.knownTotal === undefined ? {} : { knownTotal: limitation.knownTotal }),\n            };\n        case \"truncated\":\n            return { kind: limitation.kind, omittedBytes: limitation.omittedBytes };\n        case \"provider-limited\":\n            return { kind: limitation.kind };\n    }\n}\nfunction scoreContributionMetadata(contribution: ScoreContributionV1): globalThis.Record<string, JsonValue> {\n    switch (contribution.state) {\n        case \"not-scored\":\n            return { state: contribution.state };\n        case \"earned\":\n            return {\n                state: contribution.state,\n                points: contribution.points,\n                earned: contribution.earned,\n            };\n        case \"unavailable\":\n            return {\n                state: contribution.state,\n                points: contribution.points,\n                reason: contribution.reason,\n            };\n    }\n}\nfunction scoreMetadata(score: ScoreProjectionV1): globalThis.Record<string, JsonValue> {\n    switch (score.state) {\n        case \"complete\":\n            return { state: score.state, earned: score.earned, comparable: score.comparable };\n        case \"partial\":\n            return {\n                state: score.state,\n                earned: score.earned,\n                reasons: [...score.reasons],\n                comparable: score.comparable,\n            };\n        case \"unavailable\":\n            return {\n                state: score.state,\n                reasons: [...score.reasons],\n                comparable: score.comparable,\n            };\n    }\n}\n/** agent 的最终回复文本(事件流里最后一条 assistant message);没有就不填。 */\nfunction lastAssistantText(events: EvalResult[\"events\"]): string | undefined {\n    if (!events)\n        return undefined;\n    for (let i = events.length - 1; i >= 0; i--) {\n        const e = events[i];\n        if (e.type === \"message\" && e.role === \"assistant\")\n            return e.text;\n    }\n    return undefined;\n}\n/** 动态 import 说明:specifier 放变量里,`braintrust` 不进编译期依赖,没装也能 typecheck。 */\nconst BRAINTRUST_PACKAGE = \"braintrust\";\nasync function loadBraintrustSdk(): Promise<BraintrustSdk> {\n    try {\n        return (await __niceevalDynamicImport(BRAINTRUST_PACKAGE)) as unknown as BraintrustSdk;\n    }\n    catch {\n        throw new Error(\"The 'braintrust' package is required for the Braintrust reporter but was not found. Install it with: npm install braintrust (and set BRAINTRUST_API_KEY).\");\n    }\n}\n"]}

package/dist/runner/reporters/braintrust.d.cts

@@ -0,0 +1,59 @@
+import type { EvalResult, JsonValue, Reporter } from "../../types.cjs";
+import type { AssertionsProjectionV1 } from "../../assertions/record/model.cjs";
+import type { ScoreProjectionV1 } from "../../eval/record/score.cjs";
+import type { AttemptSlotProjectedEntry } from "../../projection/index.cjs";
+import type { Verdict } from "../../shared/types.cjs";
+export interface BraintrustConfig {
+    /** Braintrust 项目名;省略时用 "niceeval"。 */
+    project?: string;
+    /** Braintrust 项目 id;与 project 给一个即可。 */
+    projectId?: string;
+    /** 实验名;省略时由 Braintrust 自动命名。 */
+    experiment?: string;
+    /** 作为对比基线(diff base)的既有实验名。 */
+    baseExperiment?: string;
+    /** 作为对比基线(diff base)的既有实验 id。 */
+    baseExperimentId?: string;
+    /** true = 更新同名既有实验,而不是新建一个。 */
+    update?: boolean;
+    /** 实验级附加 metadata;与 niceeval 自动写入的字段合并,同名以这里为准。 */
+    metadata?: globalThis.Record<string, JsonValue>;
+    /** API key;省略时 SDK 读 BRAINTRUST_API_KEY 环境变量。 */
+    apiKey?: string;
+}
+export interface BraintrustLogEvent {
+    id?: string;
+    input?: string;
+    output?: string;
+    error?: EvalResult["error"];
+    scores?: globalThis.Record<string, number>;
+    metadata?: globalThis.Record<string, JsonValue>;
+    metrics?: globalThis.Record<string, number>;
+}
+/**
+ * A post-Record Projection calculation supplies this optional assessment. The
+ * live Reporter callback currently has no RecordProjection input, so its
+ * absence is represented explicitly rather than as a made-up Attachment read.
+ */
+export type BraintrustAssessment = {
+    readonly state: "projected";
+    /** Exact Report-style public projections, including every data state. */
+    readonly assertions: AttemptSlotProjectedEntry<AssertionsProjectionV1<unknown>>;
+    readonly verdict: AttemptSlotProjectedEntry<Verdict>;
+    readonly score?: AttemptSlotProjectedEntry<ScoreProjectionV1>;
+} | {
+    readonly state: "not-projected";
+    readonly reason: "reporter-contract-does-not-provide-record-projections";
+};
+/**
+ * 创建 Braintrust 报告器。挂在 `defineConfig({ reporters })` 上观测整次运行,
+ * 或挂在单个 eval 的 `reporters` 上只观测它(同一实例被多个 eval 引用时共享一个实验)。
+ */
+export declare function Braintrust(config?: BraintrustConfig): Reporter;
+/**
+ * EvalResult identity/transport data plus an optional post-Record assessment
+ * become one Braintrust row. Assertions, Verdict, and Score only enter through
+ * that assessment's public Attachment projections; this function never reads
+ * a historical result graph.
+ */
+export declare function toBraintrustEvent(result: EvalResult, assessment?: BraintrustAssessment): BraintrustLogEvent;

package/dist/runner/reporters/braintrust.d.mts

@@ -0,0 +1,59 @@
+import type { EvalResult, JsonValue, Reporter } from "../../types.mjs";
+import type { AssertionsProjectionV1 } from "../../assertions/record/model.mjs";
+import type { ScoreProjectionV1 } from "../../eval/record/score.mjs";
+import type { AttemptSlotProjectedEntry } from "../../projection/index.mjs";
+import type { Verdict } from "../../shared/types.mjs";
+export interface BraintrustConfig {
+    /** Braintrust 项目名;省略时用 "niceeval"。 */
+    project?: string;
+    /** Braintrust 项目 id;与 project 给一个即可。 */
+    projectId?: string;
+    /** 实验名;省略时由 Braintrust 自动命名。 */
+    experiment?: string;
+    /** 作为对比基线(diff base)的既有实验名。 */
+    baseExperiment?: string;
+    /** 作为对比基线(diff base)的既有实验 id。 */
+    baseExperimentId?: string;
+    /** true = 更新同名既有实验,而不是新建一个。 */
+    update?: boolean;
+    /** 实验级附加 metadata;与 niceeval 自动写入的字段合并,同名以这里为准。 */
+    metadata?: globalThis.Record<string, JsonValue>;
+    /** API key;省略时 SDK 读 BRAINTRUST_API_KEY 环境变量。 */
+    apiKey?: string;
+}
+export interface BraintrustLogEvent {
+    id?: string;
+    input?: string;
+    output?: string;
+    error?: EvalResult["error"];
+    scores?: globalThis.Record<string, number>;
+    metadata?: globalThis.Record<string, JsonValue>;
+    metrics?: globalThis.Record<string, number>;
+}
+/**
+ * A post-Record Projection calculation supplies this optional assessment. The
+ * live Reporter callback currently has no RecordProjection input, so its
+ * absence is represented explicitly rather than as a made-up Attachment read.
+ */
+export type BraintrustAssessment = {
+    readonly state: "projected";
+    /** Exact Report-style public projections, including every data state. */
+    readonly assertions: AttemptSlotProjectedEntry<AssertionsProjectionV1<unknown>>;
+    readonly verdict: AttemptSlotProjectedEntry<Verdict>;
+    readonly score?: AttemptSlotProjectedEntry<ScoreProjectionV1>;
+} | {
+    readonly state: "not-projected";
+    readonly reason: "reporter-contract-does-not-provide-record-projections";
+};
+/**
+ * 创建 Braintrust 报告器。挂在 `defineConfig({ reporters })` 上观测整次运行,
+ * 或挂在单个 eval 的 `reporters` 上只观测它(同一实例被多个 eval 引用时共享一个实验)。
+ */
+export declare function Braintrust(config?: BraintrustConfig): Reporter;
+/**
+ * EvalResult identity/transport data plus an optional post-Record assessment
+ * become one Braintrust row. Assertions, Verdict, and Score only enter through
+ * that assessment's public Attachment projections; this function never reads
+ * a historical result graph.
+ */
+export declare function toBraintrustEvent(result: EvalResult, assessment?: BraintrustAssessment): BraintrustLogEvent;

package/dist/runner/reporters/braintrust.d.ts

@@ -0,0 +1,59 @@
+import type { EvalResult, JsonValue, Reporter } from "../../types.js";
+import type { AssertionsProjectionV1 } from "../../assertions/record/model.js";
+import type { ScoreProjectionV1 } from "../../eval/record/score.js";
+import type { AttemptSlotProjectedEntry } from "../../projection/index.js";
+import type { Verdict } from "../../shared/types.js";
+export interface BraintrustConfig {
+    /** Braintrust 项目名;省略时用 "niceeval"。 */
+    project?: string;
+    /** Braintrust 项目 id;与 project 给一个即可。 */
+    projectId?: string;
+    /** 实验名;省略时由 Braintrust 自动命名。 */
+    experiment?: string;
+    /** 作为对比基线(diff base)的既有实验名。 */
+    baseExperiment?: string;
+    /** 作为对比基线(diff base)的既有实验 id。 */
+    baseExperimentId?: string;
+    /** true = 更新同名既有实验,而不是新建一个。 */
+    update?: boolean;
+    /** 实验级附加 metadata;与 niceeval 自动写入的字段合并,同名以这里为准。 */
+    metadata?: globalThis.Record<string, JsonValue>;
+    /** API key;省略时 SDK 读 BRAINTRUST_API_KEY 环境变量。 */
+    apiKey?: string;
+}
+export interface BraintrustLogEvent {
+    id?: string;
+    input?: string;
+    output?: string;
+    error?: EvalResult["error"];
+    scores?: globalThis.Record<string, number>;
+    metadata?: globalThis.Record<string, JsonValue>;
+    metrics?: globalThis.Record<string, number>;
+}
+/**
+ * A post-Record Projection calculation supplies this optional assessment. The
+ * live Reporter callback currently has no RecordProjection input, so its
+ * absence is represented explicitly rather than as a made-up Attachment read.
+ */
+export type BraintrustAssessment = {
+    readonly state: "projected";
+    /** Exact Report-style public projections, including every data state. */
+    readonly assertions: AttemptSlotProjectedEntry<AssertionsProjectionV1<unknown>>;
+    readonly verdict: AttemptSlotProjectedEntry<Verdict>;
+    readonly score?: AttemptSlotProjectedEntry<ScoreProjectionV1>;
+} | {
+    readonly state: "not-projected";
+    readonly reason: "reporter-contract-does-not-provide-record-projections";
+};
+/**
+ * 创建 Braintrust 报告器。挂在 `defineConfig({ reporters })` 上观测整次运行,
+ * 或挂在单个 eval 的 `reporters` 上只观测它(同一实例被多个 eval 引用时共享一个实验)。
+ */
+export declare function Braintrust(config?: BraintrustConfig): Reporter;
+/**
+ * EvalResult identity/transport data plus an optional post-Record assessment
+ * become one Braintrust row. Assertions, Verdict, and Score only enter through
+ * that assessment's public Attachment projections; this function never reads
+ * a historical result graph.
+ */
+export declare function toBraintrustEvent(result: EvalResult, assessment?: BraintrustAssessment): BraintrustLogEvent;

package/dist/runner/reporters/braintrust.js

@@ -0,0 +1,8 @@
+import { createRequire as __niceevalCreateRequire } from "node:module";
+const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
+const __niceevalCanonical = __niceevalRequire("./braintrust.cjs");
+const __niceevalExport0 = __niceevalCanonical["Braintrust"];
+export { __niceevalExport0 as Braintrust };
+const __niceevalExport1 = __niceevalCanonical["toBraintrustEvent"];
+export { __niceevalExport1 as toBraintrustEvent };
+//# sourceMappingURL=braintrust.js.map

package/dist/runner/reporters/braintrust.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"braintrust.js","sources":[],"names":[],"mappings":""}

package/dist/runner/reporters/index.cjs

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Braintrust = exports.JUnit = exports.Json = void 0;
+// 内置报告器:Json / JUnit / Braintrust。
+// 其它第三方实验跟踪平台也走同一条 Reporter 通道。
+// 终端反馈不是 Reporter——三种 --output profile 见 src/runner/feedback/。
+var json_ts_1 = require("./json.cjs");
+Object.defineProperty(exports, "Json", { enumerable: true, get: function () { return json_ts_1.Json; } });
+Object.defineProperty(exports, "JUnit", { enumerable: true, get: function () { return json_ts_1.JUnit; } });
+var braintrust_ts_1 = require("./braintrust.cjs");
+Object.defineProperty(exports, "Braintrust", { enumerable: true, get: function () { return braintrust_ts_1.Braintrust; } });
+//# sourceMappingURL=index.cjs.map

package/dist/runner/reporters/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":";;;AAAA,mCAAmC;AACnC,gCAAgC;AAChC,+DAA+D;AAC/D,qCAAwC;AAA/B,+FAAA,IAAI,OAAA;AAAE,gGAAA,KAAK,OAAA;AACpB,iDAAoE;AAA3D,2GAAA,UAAU,OAAA","sourcesContent":["// 内置报告器:Json / JUnit / Braintrust。\n// 其它第三方实验跟踪平台也走同一条 Reporter 通道。\n// 终端反馈不是 Reporter——三种 --output profile 见 src/runner/feedback/。\nexport { Json, JUnit } from \"./json.ts\";\nexport { Braintrust, type BraintrustConfig } from \"./braintrust.ts\";\n"]}

package/dist/runner/reporters/index.d.cts

@@ -0,0 +1,2 @@
+export { Json, JUnit } from "./json.cjs";
+export { Braintrust, type BraintrustConfig } from "./braintrust.cjs";

package/dist/runner/reporters/index.d.mts

@@ -0,0 +1,2 @@
+export { Json, JUnit } from "./json.mjs";
+export { Braintrust, type BraintrustConfig } from "./braintrust.mjs";

package/dist/runner/reporters/index.d.ts

@@ -0,0 +1,2 @@
+export { Json, JUnit } from "./json.js";
+export { Braintrust, type BraintrustConfig } from "./braintrust.js";

package/dist/runner/reporters/index.js

@@ -0,0 +1,10 @@
+import { createRequire as __niceevalCreateRequire } from "node:module";
+const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
+const __niceevalCanonical = __niceevalRequire("./index.cjs");
+const __niceevalExport0 = __niceevalCanonical["Braintrust"];
+export { __niceevalExport0 as Braintrust };
+const __niceevalExport1 = __niceevalCanonical["JUnit"];
+export { __niceevalExport1 as JUnit };
+const __niceevalExport2 = __niceevalCanonical["Json"];
+export { __niceevalExport2 as Json };
+//# sourceMappingURL=index.js.map

package/dist/runner/reporters/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sources":[],"names":[],"mappings":""}

package/dist/runner/reporters/index.mjs

@@ -0,0 +1,10 @@
+import { createRequire as __niceevalCreateRequire } from "node:module";
+const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
+const __niceevalCanonical = __niceevalRequire("./index.cjs");
+const __niceevalExport0 = __niceevalCanonical["Braintrust"];
+export { __niceevalExport0 as Braintrust };
+const __niceevalExport1 = __niceevalCanonical["JUnit"];
+export { __niceevalExport1 as JUnit };
+const __niceevalExport2 = __niceevalCanonical["Json"];
+export { __niceevalExport2 as Json };
+//# sourceMappingURL=index.mjs.map

package/dist/runner/reporters/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sources":[],"names":[],"mappings":""}