openmerit 0.4.1 → 0.4.2

package/dist/connectors/langfuse/index.js

@@ -1,20 +1,18 @@
-// Langfuse eval connector: reads labelled datasets and experiment results, and records the
-// evaluations Scout runs as Langfuse experiments, on Langfuse Cloud or a self-hosted instance.
+// Langfuse connector, read-only, for Langfuse Cloud or a self-hosted instance. It reads:
+// - per-item pass/fail of experiments the application's own eval recorded (experiment items and
+//   their scores), which is how Scout measures a model without writing anything itself;
+// - dataset item IDs (never their contents), to know which items an eval run must cover;
+// - aggregate production evidence: generation counts, cost and latency by model, score averages.
+// It never creates datasets, experiments, traces or scores, and it never reads inputs, outputs or comments.
 //
-// Read paths: /api/public/projects, /api/public/v2/datasets, /api/public/dataset-items,
-// /api/public/experiments, /api/public/experiment-items (fields=scores), /api/public/v3/scores
-// (trace-level scores that experiment items omit) and /api/public/v2/observations (cost).
-// Write paths: /api/public/v2/datasets and /api/public/dataset-items (ensureDataset), OTLP
-// /api/public/otel/v1/traces with langfuse.experiment.* attributes, and /api/public/scores.
-// The deprecated dataset-run-items API is not used.
+// The only non-GET request is the Viewer check: a deliberately invalid dataset creation ({} has no
+// name). A Viewer key gets 403; any other answer means the key can write, so setup refuses it.
 import { createTransport, LangfuseError } from "./client.js";
-import { asText, attribute, exportRequest, hexId, span } from "./otlp.js";
 export { connectionFromEnv, LangfuseError, normalizeBaseUrl } from "./client.js";
-/** Version written into recorded traces' scope. */
-export const CONNECTOR_VERSION = "0.1.0";
 const DAY_MS = 86_400_000;
 const isObject = (value) => !!value && typeof value === "object" && !Array.isArray(value);
 const str = (value) => (typeof value === "string" ? value : null);
+const num = (value) => (typeof value === "number" && Number.isFinite(value) ? value : null);
 export function evaluatePass(rule, value) {
     if (value === undefined)
         return false;
@@ -44,24 +42,11 @@ function scoreValue(score) {
         return value;
     return undefined;
 }
-const iso = (since, now) => (since === undefined ? new Date(now - 30 * DAY_MS).toISOString() : new Date(since).toISOString());
-function stable(value) {
-    if (Array.isArray(value))
-        return `[${value.map(stable).join(",")}]`;
-    if (isObject(value))
-        return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stable(value[key])}`).join(",")}}`;
-    return JSON.stringify(value) ?? "null";
-}
-/** Trace ID of one recorded item: the first 32 hex characters of SHA-256 over its experiment and item. */
-export const recordedTraceId = (experimentId, itemId) => hexId(`openmerit:experiment:${experimentId}:item:${itemId}`, 32);
 export function createLangfuseConnector(connection, options = {}) {
-    const http = createTransport(connection, options);
-    const environment = options.environment ?? "openmerit";
-    if (!/^(?!langfuse)[a-z0-9_-]{1,40}$/.test(environment))
-        throw new LangfuseError("The Langfuse environment must be lowercase letters, digits, - or _, and must not start with 'langfuse'.");
-    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
-    let project;
-    const get = async (path, query, allowNotFound = false) => (await http.request("GET", path, { ...(query ? { query } : {}), allowNotFound })).data;
+    const transport = createTransport(connection, options);
+    const sleep = options.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
+    const get = async (path, query) => (await transport.request("GET", path, query ? { query } : {})).data;
+    const window = (days) => ({ from: new Date(Date.now() - days * DAY_MS).toISOString(), to: new Date().toISOString() });
     async function* cursorPages(path, query) {
         const seen = new Set();
         let cursor;
@@ -77,52 +62,14 @@ export function createLangfuseConnector(connection, options = {}) {
                 seen.add(cursor);
         } while (cursor);
     }
-    async function* numberedPages(path, query) {
-        for (let page = 1;; page += 1) {
-            const body = await get(path, { ...query, page, limit: 100 });
-            if (!body || !Array.isArray(body.data))
-                throw new LangfuseError(`Langfuse GET ${path} returned no data array.`);
-            yield body.data;
-            const total = typeof body.meta?.totalPages === "number" ? body.meta.totalPages : 0;
-            if (page >= total || body.data.length === 0)
-                return;
-        }
-    }
-    function getProject() {
-        project ??= (async () => {
-            const body = await get("/api/public/projects");
-            const first = Array.isArray(body?.data) ? body.data[0] : undefined;
-            if (!isObject(first) || !str(first.id))
-                throw new LangfuseError("Langfuse returned no project for this key pair. Use a project-scoped API key.");
-            return { id: String(first.id), name: str(first.name) ?? "" };
-        })();
-        project.catch(() => (project = undefined));
-        return project;
-    }
-    async function getDataset(name) {
-        const body = await get(`/api/public/v2/datasets/${encodeURIComponent(name)}`, undefined, true);
-        if (!body || !str(body.id))
-            throw new LangfuseError(`Langfuse has no dataset named "${name}" in this project.`, 404);
-        return { id: String(body.id), name: str(body.name) ?? name };
-    }
-    async function listExperiments(query = {}) {
-        const datasetId = query.datasetId ?? (query.datasetName ? (await getDataset(query.datasetName)).id : undefined);
-        const experiments = [];
-        for await (const page of cursorPages("/api/public/experiments", { fields: "core,metadata", limit: 100, fromStartTime: iso(query.since, Date.now()), datasetId, name: query.names?.join(","), id: query.ids?.join(",") })) {
-            for (const raw of page) {
-                if (!isObject(raw) || !str(raw.id))
-                    continue;
-                const metadata = isObject(raw.metadata) ? raw.metadata : {};
-                const models = {};
-                for (const [key, value] of Object.entries(metadata))
-                    if (key.startsWith("model_") && typeof value === "string")
-                        models[key.slice("model_".length)] = value;
-                experiments.push({ id: String(raw.id), name: str(raw.name) ?? "", datasetId: str(raw.datasetId), itemCount: typeof raw.itemCount === "number" ? raw.itemCount : null, startTime: str(raw.startTime), metadata, models });
-            }
-        }
-        return experiments;
-    }
-    /** Latest score per name for each trace, from /api/public/v3/scores (trace- and observation-level). */
+    const metrics = async (query) => {
+        const data = await get("/api/public/v2/metrics", { query: JSON.stringify(query) });
+        if (!data || !Array.isArray(data.data))
+            throw new LangfuseError("Langfuse returned invalid aggregate metrics.");
+        return data.data.filter(isObject);
+    };
+    const generation = { column: "type", operator: "=", value: "GENERATION", type: "string" };
+    /** Latest score of one name per trace, from /api/public/v3/scores (trace- and observation-level). */
     async function scoresByTrace(traceIds, name, since) {
         const latest = new Map();
         for (let i = 0; i < traceIds.length; i += 40) {
@@ -143,40 +90,19 @@ export function createLangfuseConnector(connection, options = {}) {
         }
         return new Map([...latest].map(([traceId, { value }]) => [traceId, value]));
     }
-    async function costsByTrace(traceIds, since) {
-        const costs = new Map();
-        for (let i = 0; i < traceIds.length; i += 40) {
-            const filter = JSON.stringify([
-                { type: "stringOptions", column: "traceId", operator: "any of", value: traceIds.slice(i, i + 40) },
-                { type: "string", column: "type", operator: "=", value: "GENERATION" },
-                { type: "datetime", column: "startTime", operator: ">=", value: since },
-            ]);
-            for await (const page of cursorPages("/api/public/v2/observations", { fields: "core,usage", limit: 100, filter })) {
-                for (const raw of page) {
-                    if (!isObject(raw) || !str(raw.traceId) || typeof raw.totalCost !== "number" || !Number.isFinite(raw.totalCost))
-                        continue;
-                    costs.set(String(raw.traceId), (costs.get(String(raw.traceId)) ?? 0) + raw.totalCost);
-                }
-            }
-        }
-        return costs;
-    }
     async function experimentResults(query) {
-        if (!query.experimentId === !query.experimentName)
-            throw new LangfuseError("Pass exactly one of experimentId or experimentName.");
+        if (!query.experimentName)
+            throw new LangfuseError("Pass the experiment name to read.");
         if (!query.pass?.score)
             throw new LangfuseError("Pass a pass rule naming the score that decides each item.");
-        const since = iso(query.since, Date.now());
         const latest = new Map();
         const experimentIds = new Set();
-        let experimentName = query.experimentName ?? "";
         let datasetId = null;
-        for await (const page of cursorPages("/api/public/experiment-items", { fields: "core,dataset,scores", limit: 100, scoreLimit: 50, fromStartTime: since, experimentId: query.experimentId, experimentName: query.experimentName })) {
+        for await (const page of cursorPages("/api/public/experiment-items", { fields: "core,scores", limit: 100, scoreLimit: 50, fromStartTime: query.since, experimentName: query.experimentName })) {
             for (const raw of page) {
                 if (!isObject(raw) || !str(raw.experimentItemId) || !str(raw.traceId))
                     continue;
                 experimentIds.add(str(raw.experimentId) ?? "");
-                experimentName = str(raw.experimentName) ?? experimentName;
                 datasetId = str(raw.experimentDatasetId) ?? datasetId;
                 const itemId = String(raw.experimentItemId);
                 const at = str(raw.startTime) ?? "";
@@ -187,108 +113,81 @@ export function createLangfuseConnector(connection, options = {}) {
             }
         }
         if (experimentIds.size > 1)
-            throw new LangfuseError(`Experiment name "${query.experimentName}" matches ${experimentIds.size} experiments (${[...experimentIds].join(", ")}). Query by experimentId.`);
+            throw new LangfuseError(`Experiment name "${query.experimentName}" matches ${experimentIds.size} experiments; the eval must use a fresh name per run.`);
         const wanted = query.itemIds ? new Set(query.itemIds) : null;
         const items = [];
         for (const [itemId, { raw }] of latest) {
             if (wanted && !wanted.has(itemId))
                 continue;
-            const scores = {};
+            let best;
             for (const score of Array.isArray(raw.scores) ? raw.scores : []) {
-                if (!isObject(score) || !str(score.name))
+                if (!isObject(score) || score.name !== query.pass.score)
                     continue;
-                const value = scoreValue(score);
-                const at = str(score.timestamp) ?? "";
-                const old = scores[String(score.name)];
-                if (value !== undefined && (!old || at > old.at))
-                    scores[String(score.name)] = { at, value };
+                const value = scoreValue(score), at = str(score.timestamp) ?? "";
+                if (value !== undefined && (!best || at > best.at))
+                    best = { at, value };
             }
-            items.push({ itemId, traceId: String(raw.traceId), passed: false, missingScore: false, scores: Object.fromEntries(Object.entries(scores).map(([name, { value }]) => [name, value])), costUsd: null });
+            items.push({ itemId, traceId: String(raw.traceId), passed: false, missingScore: false, ...(best ? { score: best.value } : {}) });
         }
         // Experiment items list observation-level scores; scores posted on the trace come from the scores API.
-        const unscored = items.filter((item) => item.scores[query.pass.score] === undefined);
+        const unscored = items.filter(item => item.score === undefined);
         if (unscored.length) {
-            const found = await scoresByTrace(unscored.map((item) => item.traceId), query.pass.score, since);
+            const found = await scoresByTrace(unscored.map(item => item.traceId), query.pass.score, query.since);
             for (const item of unscored) {
                 const value = found.get(item.traceId);
                 if (value !== undefined)
-                    item.scores[query.pass.score] = value;
+                    item.score = value;
             }
         }
-        for (const item of items) {
-            const value = item.scores[query.pass.score];
-            item.missingScore = value === undefined;
-            item.passed = evaluatePass(query.pass, value);
-        }
-        let costUsd = null;
-        if (query.includeCost && items.length) {
-            const costs = await costsByTrace(items.map((item) => item.traceId), since);
-            for (const item of items)
-                item.costUsd = costs.get(item.traceId) ?? null;
-            costUsd = costs.size ? [...costs.values()].reduce((sum, cost) => sum + cost, 0) : null;
-        }
-        items.sort((a, b) => a.itemId.localeCompare(b.itemId));
+        const results = items.map(({ score, ...item }) => ({ ...item, missingScore: score === undefined, passed: evaluatePass(query.pass, score) }));
+        results.sort((a, b) => a.itemId.localeCompare(b.itemId));
         return {
-            experimentId: query.experimentId ?? [...experimentIds][0] ?? "",
-            experimentName,
-            datasetId,
-            items,
-            passes: items.filter((item) => item.passed).length,
-            total: items.length,
-            missing: query.itemIds ? query.itemIds.filter((id) => !latest.has(id)) : [],
-            missingScores: items.filter((item) => item.missingScore).map((item) => item.itemId),
-            costUsd,
+            experimentName: query.experimentName, experimentId: [...experimentIds][0] ?? "", datasetId, items: results,
+            passes: results.filter(item => item.passed).length, total: results.length,
+            missing: query.itemIds ? query.itemIds.filter(id => !latest.has(id)) : [],
+            missingScores: results.filter(item => item.missingScore).map(item => item.itemId),
         };
     }
-    async function experimentUrl(datasetId, experimentId) {
-        const { id } = await getProject();
-        return `${http.baseUrl}/project/${encodeURIComponent(id)}/datasets/${encodeURIComponent(datasetId)}/runs/${encodeURIComponent(experimentId)}`;
-    }
     return {
-        baseUrl: http.baseUrl,
+        baseUrl: transport.baseUrl,
         async check() {
-            const { id, name } = await getProject();
-            return { baseUrl: http.baseUrl, projectId: id, projectName: name };
+            const data = await get("/api/public/projects");
+            const project = data?.data?.[0];
+            if (!project?.id)
+                throw new LangfuseError("Langfuse returned no project for this key pair.");
+            return { baseUrl: transport.baseUrl, projectId: project.id, projectName: project.name ?? project.id };
         },
-        async listDatasets() {
-            const datasets = [];
-            for await (const page of numberedPages("/api/public/v2/datasets", {}))
-                for (const raw of page)
-                    if (isObject(raw) && str(raw.id) && str(raw.name))
-                        datasets.push({ id: String(raw.id), name: String(raw.name) });
-            return datasets;
-        },
-        getDataset,
-        async getDatasetItems(datasetName) {
-            const items = [];
-            for await (const page of numberedPages("/api/public/dataset-items", { datasetName })) {
-                for (const raw of page) {
-                    if (!isObject(raw) || !str(raw.id) || raw.status === "ARCHIVED")
-                        continue;
-                    items.push({ id: String(raw.id), input: raw.input ?? null, expectedOutput: raw.expectedOutput ?? null, metadata: isObject(raw.metadata) ? raw.metadata : {} });
-                }
+        async verifyViewer() {
+            let status;
+            try {
+                status = (await transport.request("POST", "/api/public/v2/datasets", { body: {} })).status;
             }
-            return items;
+            catch (error) {
+                if (!(error instanceof LangfuseError) || error.status === null)
+                    throw error;
+                status = error.status;
+            }
+            if (status !== 403)
+                throw new LangfuseError("This key pair can write to Langfuse. Create a separate Viewer key in Langfuse Project Settings → API Keys; OpenMerit only reads and keeps your tracing keys untouched.");
         },
-        async ensureDataset(input) {
-            if (!input.name?.trim())
-                throw new LangfuseError("A dataset needs a name.");
-            const existing = await http.request("GET", `/api/public/v2/datasets/${encodeURIComponent(input.name)}`, { allowNotFound: true });
-            const dataset = existing.data ?? (await http.request("POST", "/api/public/v2/datasets", { body: { name: input.name, description: input.description, metadata: input.metadata } })).data;
-            if (!isObject(dataset) || !str(dataset.id))
-                throw new LangfuseError(`Langfuse returned no ID for dataset "${input.name}".`);
-            const itemIds = [];
-            for (const item of input.items) {
-                const id = item.id ?? `om-${hexId(`${input.name}\n${stable(item.input)}`, 16)}`;
-                // Upserts by ID. Langfuse item IDs are unique per project, so reuse an ID in one dataset only.
-                await http.request("POST", "/api/public/dataset-items", { body: { datasetName: input.name, id, input: item.input, expectedOutput: item.expectedOutput, metadata: item.metadata } });
-                itemIds.push(id);
+        async datasetItemIds(name) {
+            const ids = [];
+            for (let page = 1;; page += 1) {
+                const body = await get("/api/public/dataset-items", { datasetName: name, page, limit: 100 });
+                if (!body || !Array.isArray(body.data))
+                    throw new LangfuseError("Langfuse GET /api/public/dataset-items returned no data array.");
+                for (const raw of body.data)
+                    if (isObject(raw) && str(raw.id) && raw.status !== "ARCHIVED")
+                        ids.push(String(raw.id));
+                if (page >= (num(body.meta?.totalPages) ?? 0) || body.data.length === 0)
+                    break;
             }
-            return { datasetId: String(dataset.id), itemIds };
+            if (!ids.length)
+                throw new LangfuseError(`Langfuse dataset "${name}" has no active items (or does not exist in this project).`);
+            return ids;
         },
-        listExperiments,
         experimentResults,
-        async waitForExperiment(query, { expectItems, timeoutMs = 600_000, intervalMs = 15_000 }) {
+        async waitForExperiment(query, { expectItems, timeoutMs = 300_000, intervalMs = 15_000 }) {
             const deadline = Date.now() + timeoutMs;
             for (;;) {
                 const result = await experimentResults(query);
@@ -296,83 +195,33 @@ export function createLangfuseConnector(connection, options = {}) {
                 if (scored >= expectItems)
                     return result;
                 if (Date.now() + intervalMs > deadline) {
-                    throw new LangfuseError(`Langfuse shows ${scored}/${expectItems} scored items for experiment ${query.experimentId ?? query.experimentName} after ${Math.round(timeoutMs / 1000)} s (missing items: ${result.missing.join(", ") || "none"}; unscored: ${result.missingScores.join(", ") || "none"}).`);
+                    throw new LangfuseError(`Langfuse shows ${scored}/${expectItems} items scored with "${query.pass.score}" in experiment "${query.experimentName}" after ${Math.round(timeoutMs / 1000)} s (missing items: ${result.missing.slice(0, 10).join(", ") || "none"}; unscored: ${result.missingScores.slice(0, 10).join(", ") || "none"}). The eval must record every requested item under OPENMERIT_EXPERIMENT.`);
                 }
                 await sleep(intervalMs);
             }
         },
-        async recordExperimentItems(recording) {
-            const { datasetId, experimentId, experimentName } = recording;
-            if (![datasetId, experimentId, experimentName].every((value) => typeof value === "string" && value.trim()))
-                throw new LangfuseError("A recording needs datasetId, experimentId and experimentName.");
-            if (!recording.items.length)
-                throw new LangfuseError("A recording needs at least one item.");
-            const traceName = recording.traceName ?? "openmerit-eval";
-            const traceIds = {};
-            const spans = [];
-            const scores = [];
-            for (const item of recording.items) {
-                if (traceIds[item.itemId])
-                    throw new LangfuseError(`Item ${item.itemId} appears twice in one recording.`);
-                const traceId = recordedTraceId(experimentId, item.itemId);
-                const rootId = hexId(`${traceId}:root`, 16);
-                traceIds[item.itemId] = traceId;
-                const common = [
-                    attribute("langfuse.trace.name", traceName),
-                    attribute("langfuse.trace.tags", ["openmerit", `experiment:${experimentId}`]),
-                    attribute("langfuse.environment", environment),
-                    attribute("langfuse.release", `openmerit-langfuse-connector@${CONNECTOR_VERSION}`),
-                    attribute("langfuse.trace.metadata.model", item.model),
-                    ...Object.entries(item.metadata ?? {}).map(([key, value]) => attribute(`langfuse.trace.metadata.${key}`, String(value))),
-                ];
-                spans.push(span({
-                    traceId, spanId: rootId, name: traceName, startMs: item.startMs, endMs: item.endMs, ...(item.error ? { error: item.error } : {}),
-                    attributes: [
-                        ...common,
-                        attribute("langfuse.observation.type", "span"),
-                        attribute("langfuse.observation.input", asText(item.input)),
-                        attribute("langfuse.observation.output", asText(item.output ?? null)),
-                        ...(item.error ? [attribute("langfuse.observation.level", "ERROR"), attribute("langfuse.observation.status_message", item.error.slice(0, 500))] : []),
-                        attribute("langfuse.experiment.id", experimentId),
-                        attribute("langfuse.experiment.name", experimentName),
-                        attribute("langfuse.experiment.dataset.id", datasetId),
-                        attribute("langfuse.experiment.item.id", item.itemId),
-                        attribute("langfuse.experiment.item.root_observation_id", rootId),
-                        ...Object.entries(recording.metadata ?? {}).map(([key, value]) => attribute(`langfuse.experiment.metadata.${key}`, String(value))),
-                    ],
-                }), span({
-                    traceId, spanId: hexId(`${traceId}:generation`, 16), parentSpanId: rootId, name: `${traceName}-model`, startMs: item.startMs, endMs: item.endMs, ...(item.error ? { error: item.error } : {}),
-                    attributes: [
-                        ...common,
-                        attribute("langfuse.observation.type", "generation"),
-                        attribute("langfuse.observation.model.name", item.model),
-                        ...(item.usage ? [attribute("langfuse.observation.usage_details", JSON.stringify(item.usage))] : []),
-                        ...(item.costUsd !== undefined ? [attribute("langfuse.observation.cost_details", JSON.stringify({ total: item.costUsd }))] : []),
-                    ],
-                }));
-                for (const score of item.scores) {
-                    const dataType = score.dataType ?? (typeof score.value === "boolean" ? "BOOLEAN" : typeof score.value === "number" ? "NUMERIC" : "CATEGORICAL");
-                    const value = typeof score.value === "boolean" ? (score.value ? 1 : 0) : score.value;
-                    if (dataType === "CATEGORICAL" ? typeof value !== "string" : typeof value !== "number" || !Number.isFinite(value))
-                        throw new LangfuseError(`Score ${score.name} on item ${item.itemId} has a ${typeof score.value} value for data type ${dataType}.`);
-                    // A deterministic ID makes a retried recording update the score instead of adding a second one.
-                    scores.push({ id: hexId(`openmerit:score:${traceId}:${score.name}`, 32), traceId, observationId: rootId, name: score.name, value, dataType, environment, ...(score.comment ? { comment: score.comment.slice(0, 1000) } : {}), metadata: { source: "openmerit", experiment: experimentId } });
-                }
-            }
-            for (let i = 0; i < spans.length; i += 200) {
-                const response = await http.request("POST", "/api/public/otel/v1/traces", { body: exportRequest(spans.slice(i, i + 200), CONNECTOR_VERSION), headers: { "x-langfuse-ingestion-version": "4" } });
-                const rejected = Number(response.data?.partialSuccess?.rejectedSpans ?? 0);
-                if (rejected > 0)
-                    throw new LangfuseError(`Langfuse rejected ${rejected} spans: ${String(response.data?.partialSuccess?.errorMessage ?? "").slice(0, 300)}`);
-            }
-            for (let i = 0; i < scores.length; i += 100) {
-                const response = await http.request("POST", "/api/public/scores", { body: scores.slice(i, i + 100) });
-                // 207 reports rejected scores with a 2xx status; Langfuse asks clients not to retry the batch.
-                if (response.status === 207)
-                    throw new LangfuseError(`Langfuse rejected some scores (HTTP 207): ${JSON.stringify(response.data?.errors ?? response.data).slice(0, 300)}`, 207);
-            }
-            return { experimentId, traceIds, url: await experimentUrl(datasetId, experimentId) };
+        async traceNames(days = 30) {
+            const { from, to } = window(days);
+            const rows = await metrics({ view: "observations", dimensions: [{ field: "traceName" }], metrics: [{ measure: "count", aggregation: "count" }], filters: [generation], fromTimestamp: from, toTimestamp: to, config: { row_limit: 100 } });
+            return rows.filter(row => typeof row.traceName === "string").map(row => ({ traceName: row.traceName, generations: num(row.count_count) ?? 0 })).sort((a, b) => b.generations - a.generations);
+        },
+        async taskEvidence(mapping, days = 30) {
+            const { from, to } = window(days);
+            const byTrace = { column: "traceName", operator: "=", value: mapping.traceName, type: "string" };
+            const usage = await metrics({
+                view: "observations", dimensions: [{ field: "providedModelName" }],
+                metrics: [{ measure: "count", aggregation: "count" }, { measure: "totalCost", aggregation: "sum" }, { measure: "latency", aggregation: "p50" }],
+                filters: [generation, byTrace], fromTimestamp: from, toTimestamp: to, config: { row_limit: 50 },
+            });
+            const scores = mapping.scores.length ? await metrics({
+                view: "scores-numeric", dimensions: [{ field: "name" }], metrics: [{ measure: "value", aggregation: "avg" }, { measure: "count", aggregation: "count" }],
+                filters: [byTrace, { column: "name", operator: "any of", value: mapping.scores, type: "stringOptions" }], fromTimestamp: from, toTimestamp: to, config: { row_limit: mapping.scores.length },
+            }) : [];
+            return {
+                traceName: mapping.traceName, from, to,
+                models: usage.filter(row => typeof row.providedModelName === "string").map(row => ({ model: row.providedModelName, generations: num(row.count_count) ?? 0, costUsd: num(row.sum_totalCost), p50LatencyMs: num(row.p50_latency) })),
+                scores: scores.filter(row => typeof row.name === "string" && num(row.avg_value) !== null).map(row => ({ name: row.name, average: num(row.avg_value), count: num(row.count_count) ?? 0 })),
+            };
         },
-        experimentUrl,
     };
 }

package/dist/daemon/daemon.d.ts

@@ -0,0 +1,81 @@
+import type { Answers, Step } from "../scout/workflow.js";
+export declare const DEFAULT_PORT = 6720;
+export interface DaemonInfo {
+    pid: number;
+    port: number;
+    token: string;
+    version: string;
+    startedAt: string;
+}
+type Env = Record<string, string | undefined>;
+export declare function readDaemonInfo(env?: Env): DaemonInfo | null;
+/** Whether the daemon of `info` runs, has a recent heartbeat and its engine answers. `openmerit daemon status` uses it too. */
+export declare function healthy(info: DaemonInfo, env?: Env): Promise<boolean>;
+/** Runs the daemon in this process until it receives SIGINT or SIGTERM. `openmerit daemon` calls it. */
+export declare function serveDaemon(env?: Env): Promise<void>;
+/** Stops the daemon, if one runs. Its runs stop as interrupted and continue with `openmerit scout --resume`. */
+export declare function stopDaemon(env?: Env): Promise<boolean>;
+/** The daemon could not start. Unlike a lost connection, trying again does not help until its cause is fixed. */
+export declare class DaemonStartError extends Error {
+}
+/** The running daemon of this package version, started in the background when there is none. Throws DaemonStartError when it does not start. */
+export declare function ensureDaemon(env?: Env): Promise<DaemonInfo>;
+export declare function daemonClient(info: DaemonInfo): import("rivetkit/client").Client<import("rivetkit").Registry<{
+    scoutRun: import("rivetkit/client").ActorDefinition<{
+        inFlight: string | null;
+        outcome: import("./registry.js").Outcome | null;
+    }, undefined, undefined, undefined, undefined, undefined, Record<never, never>, {
+        advance: import("./registry.js").AdvanceQueue;
+    }, {
+        attach: (c: import("rivetkit").ActionContext<{
+            inFlight: string | null;
+            outcome: import("./registry.js").Outcome | null;
+        }, undefined, undefined, undefined, undefined, undefined, Record<never, never>, {
+            advance: import("./registry.js").AdvanceQueue;
+        }>, input: {
+            cwd: string;
+            env: Record<string, string | undefined>;
+        }) => {
+            inFlight: string | null;
+            since: number;
+        };
+        poll: (c: import("rivetkit").ActionContext<{
+            inFlight: string | null;
+            outcome: import("./registry.js").Outcome | null;
+        }, undefined, undefined, undefined, undefined, undefined, Record<never, never>, {
+            advance: import("./registry.js").AdvanceQueue;
+        }>, input: {
+            requestId: string;
+            since: number;
+        }) => {
+            outcome: import("./registry.js").Outcome | null;
+            lines: string[];
+            next: number;
+            inFlight: string | null;
+        };
+        stop: (c: import("rivetkit").ActionContext<{
+            inFlight: string | null;
+            outcome: import("./registry.js").Outcome | null;
+        }, undefined, undefined, undefined, undefined, undefined, Record<never, never>, {
+            advance: import("./registry.js").AdvanceQueue;
+        }>) => void;
+    }>;
+}>>;
+/** Reconnect attempts after a lost connection, before the client gives up. */
+export declare const RECONNECTS = 3;
+/**
+ * Advances a Scout run in the daemon: attaches this client's directory and environment, sends the
+ * answers, and waits for the step's outcome while relaying its progress lines. Aborting `signal` stops
+ * the step, which saves the run as interrupted. When the daemon stops answering, the client reconnects
+ * (starting a new daemon if needed) up to RECONNECTS times, then gives up with the resume command. A
+ * daemon that cannot start is not a lost connection: that error ends the call at once, with the resume command.
+ */
+export declare function advanceInDaemon(input: {
+    runId: string;
+    cwd: string;
+    env: Env;
+    answers: Answers;
+    log: (line: string) => void;
+    signal?: AbortSignal;
+}): Promise<Step>;
+export {};