openmerit 0.3.11 → 0.4.0

package/dist/connectors/langfuse/index.js

@@ -0,0 +1,378 @@
+// Langfuse eval connector: reads labelled datasets and experiment results, and records the
+// evaluations Scout runs as Langfuse experiments, on Langfuse Cloud or a self-hosted instance.
+//
+// Read paths: /api/public/projects, /api/public/v2/datasets, /api/public/dataset-items,
+// /api/public/experiments, /api/public/experiment-items (fields=scores), /api/public/v3/scores
+// (trace-level scores that experiment items omit) and /api/public/v2/observations (cost).
+// Write paths: /api/public/v2/datasets and /api/public/dataset-items (ensureDataset), OTLP
+// /api/public/otel/v1/traces with langfuse.experiment.* attributes, and /api/public/scores.
+// The deprecated dataset-run-items API is not used.
+import { createTransport, LangfuseError } from "./client.js";
+import { asText, attribute, exportRequest, hexId, span } from "./otlp.js";
+export { connectionFromEnv, LangfuseError, normalizeBaseUrl } from "./client.js";
+/** Version written into recorded traces' scope. */
+export const CONNECTOR_VERSION = "0.1.0";
+const DAY_MS = 86_400_000;
+const isObject = (value) => !!value && typeof value === "object" && !Array.isArray(value);
+const str = (value) => (typeof value === "string" ? value : null);
+export function evaluatePass(rule, value) {
+    if (value === undefined)
+        return false;
+    if (rule.equals !== undefined)
+        return value === rule.equals || (typeof rule.equals === "boolean" && value === (rule.equals ? 1 : 0));
+    if (rule.min !== undefined || rule.max !== undefined) {
+        if (typeof value !== "number")
+            return false;
+        return (rule.min === undefined || value >= rule.min) && (rule.max === undefined || value <= rule.max);
+    }
+    return value === true || value === 1;
+}
+function scoreValue(score) {
+    const value = score.value;
+    if (score.dataType === "BOOLEAN") {
+        if (typeof value === "boolean")
+            return value;
+        if (value === 1 || value === "1" || value === "true" || value === "True")
+            return true;
+        if (value === 0 || value === "0" || value === "false" || value === "False")
+            return false;
+        return undefined;
+    }
+    if (typeof value === "number" && Number.isFinite(value))
+        return value;
+    if (typeof value === "string" || typeof value === "boolean")
+        return value;
+    return undefined;
+}
+const iso = (since, now) => (since === undefined ? new Date(now - 30 * DAY_MS).toISOString() : new Date(since).toISOString());
+function stable(value) {
+    if (Array.isArray(value))
+        return `[${value.map(stable).join(",")}]`;
+    if (isObject(value))
+        return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stable(value[key])}`).join(",")}}`;
+    return JSON.stringify(value) ?? "null";
+}
+/** Trace ID of one recorded item: the first 32 hex characters of SHA-256 over its experiment and item. */
+export const recordedTraceId = (experimentId, itemId) => hexId(`openmerit:experiment:${experimentId}:item:${itemId}`, 32);
+export function createLangfuseConnector(connection, options = {}) {
+    const http = createTransport(connection, options);
+    const environment = options.environment ?? "openmerit";
+    if (!/^(?!langfuse)[a-z0-9_-]{1,40}$/.test(environment))
+        throw new LangfuseError("The Langfuse environment must be lowercase letters, digits, - or _, and must not start with 'langfuse'.");
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    let project;
+    const get = async (path, query, allowNotFound = false) => (await http.request("GET", path, { ...(query ? { query } : {}), allowNotFound })).data;
+    async function* cursorPages(path, query) {
+        const seen = new Set();
+        let cursor;
+        do {
+            const page = await get(path, { ...query, cursor });
+            if (!page || !Array.isArray(page.data))
+                throw new LangfuseError(`Langfuse GET ${path} returned no data array.`);
+            yield page.data;
+            cursor = str(page.meta?.cursor) ?? undefined;
+            if (cursor && seen.has(cursor))
+                throw new LangfuseError(`Langfuse GET ${path} repeated a pagination cursor.`);
+            if (cursor)
+                seen.add(cursor);
+        } while (cursor);
+    }
+    async function* numberedPages(path, query) {
+        for (let page = 1;; page += 1) {
+            const body = await get(path, { ...query, page, limit: 100 });
+            if (!body || !Array.isArray(body.data))
+                throw new LangfuseError(`Langfuse GET ${path} returned no data array.`);
+            yield body.data;
+            const total = typeof body.meta?.totalPages === "number" ? body.meta.totalPages : 0;
+            if (page >= total || body.data.length === 0)
+                return;
+        }
+    }
+    function getProject() {
+        project ??= (async () => {
+            const body = await get("/api/public/projects");
+            const first = Array.isArray(body?.data) ? body.data[0] : undefined;
+            if (!isObject(first) || !str(first.id))
+                throw new LangfuseError("Langfuse returned no project for this key pair. Use a project-scoped API key.");
+            return { id: String(first.id), name: str(first.name) ?? "" };
+        })();
+        project.catch(() => (project = undefined));
+        return project;
+    }
+    async function getDataset(name) {
+        const body = await get(`/api/public/v2/datasets/${encodeURIComponent(name)}`, undefined, true);
+        if (!body || !str(body.id))
+            throw new LangfuseError(`Langfuse has no dataset named "${name}" in this project.`, 404);
+        return { id: String(body.id), name: str(body.name) ?? name };
+    }
+    async function listExperiments(query = {}) {
+        const datasetId = query.datasetId ?? (query.datasetName ? (await getDataset(query.datasetName)).id : undefined);
+        const experiments = [];
+        for await (const page of cursorPages("/api/public/experiments", { fields: "core,metadata", limit: 100, fromStartTime: iso(query.since, Date.now()), datasetId, name: query.names?.join(","), id: query.ids?.join(",") })) {
+            for (const raw of page) {
+                if (!isObject(raw) || !str(raw.id))
+                    continue;
+                const metadata = isObject(raw.metadata) ? raw.metadata : {};
+                const models = {};
+                for (const [key, value] of Object.entries(metadata))
+                    if (key.startsWith("model_") && typeof value === "string")
+                        models[key.slice("model_".length)] = value;
+                experiments.push({ id: String(raw.id), name: str(raw.name) ?? "", datasetId: str(raw.datasetId), itemCount: typeof raw.itemCount === "number" ? raw.itemCount : null, startTime: str(raw.startTime), metadata, models });
+            }
+        }
+        return experiments;
+    }
+    /** Latest score per name for each trace, from /api/public/v3/scores (trace- and observation-level). */
+    async function scoresByTrace(traceIds, name, since) {
+        const latest = new Map();
+        for (let i = 0; i < traceIds.length; i += 40) {
+            for await (const page of cursorPages("/api/public/v3/scores", { traceId: traceIds.slice(i, i + 40).join(","), name, fields: "subject", limit: 100, fromTimestamp: since })) {
+                for (const raw of page) {
+                    if (!isObject(raw) || !isObject(raw.subject))
+                        continue;
+                    const traceId = raw.subject.kind === "trace" ? str(raw.subject.id) : str(raw.subject.traceId);
+                    const value = scoreValue(raw);
+                    const at = str(raw.timestamp) ?? "";
+                    if (!traceId || value === undefined)
+                        continue;
+                    const old = latest.get(traceId);
+                    if (!old || at > old.at)
+                        latest.set(traceId, { at, value });
+                }
+            }
+        }
+        return new Map([...latest].map(([traceId, { value }]) => [traceId, value]));
+    }
+    async function costsByTrace(traceIds, since) {
+        const costs = new Map();
+        for (let i = 0; i < traceIds.length; i += 40) {
+            const filter = JSON.stringify([
+                { type: "stringOptions", column: "traceId", operator: "any of", value: traceIds.slice(i, i + 40) },
+                { type: "string", column: "type", operator: "=", value: "GENERATION" },
+                { type: "datetime", column: "startTime", operator: ">=", value: since },
+            ]);
+            for await (const page of cursorPages("/api/public/v2/observations", { fields: "core,usage", limit: 100, filter })) {
+                for (const raw of page) {
+                    if (!isObject(raw) || !str(raw.traceId) || typeof raw.totalCost !== "number" || !Number.isFinite(raw.totalCost))
+                        continue;
+                    costs.set(String(raw.traceId), (costs.get(String(raw.traceId)) ?? 0) + raw.totalCost);
+                }
+            }
+        }
+        return costs;
+    }
+    async function experimentResults(query) {
+        if (!query.experimentId === !query.experimentName)
+            throw new LangfuseError("Pass exactly one of experimentId or experimentName.");
+        if (!query.pass?.score)
+            throw new LangfuseError("Pass a pass rule naming the score that decides each item.");
+        const since = iso(query.since, Date.now());
+        const latest = new Map();
+        const experimentIds = new Set();
+        let experimentName = query.experimentName ?? "";
+        let datasetId = null;
+        for await (const page of cursorPages("/api/public/experiment-items", { fields: "core,dataset,scores", limit: 100, scoreLimit: 50, fromStartTime: since, experimentId: query.experimentId, experimentName: query.experimentName })) {
+            for (const raw of page) {
+                if (!isObject(raw) || !str(raw.experimentItemId) || !str(raw.traceId))
+                    continue;
+                experimentIds.add(str(raw.experimentId) ?? "");
+                experimentName = str(raw.experimentName) ?? experimentName;
+                datasetId = str(raw.experimentDatasetId) ?? datasetId;
+                const itemId = String(raw.experimentItemId);
+                const at = str(raw.startTime) ?? "";
+                const old = latest.get(itemId);
+                // A retried item has several experiment items; its latest run counts.
+                if (!old || at > old.at)
+                    latest.set(itemId, { raw, at });
+            }
+        }
+        if (experimentIds.size > 1)
+            throw new LangfuseError(`Experiment name "${query.experimentName}" matches ${experimentIds.size} experiments (${[...experimentIds].join(", ")}). Query by experimentId.`);
+        const wanted = query.itemIds ? new Set(query.itemIds) : null;
+        const items = [];
+        for (const [itemId, { raw }] of latest) {
+            if (wanted && !wanted.has(itemId))
+                continue;
+            const scores = {};
+            for (const score of Array.isArray(raw.scores) ? raw.scores : []) {
+                if (!isObject(score) || !str(score.name))
+                    continue;
+                const value = scoreValue(score);
+                const at = str(score.timestamp) ?? "";
+                const old = scores[String(score.name)];
+                if (value !== undefined && (!old || at > old.at))
+                    scores[String(score.name)] = { at, value };
+            }
+            items.push({ itemId, traceId: String(raw.traceId), passed: false, missingScore: false, scores: Object.fromEntries(Object.entries(scores).map(([name, { value }]) => [name, value])), costUsd: null });
+        }
+        // Experiment items list observation-level scores; scores posted on the trace come from the scores API.
+        const unscored = items.filter((item) => item.scores[query.pass.score] === undefined);
+        if (unscored.length) {
+            const found = await scoresByTrace(unscored.map((item) => item.traceId), query.pass.score, since);
+            for (const item of unscored) {
+                const value = found.get(item.traceId);
+                if (value !== undefined)
+                    item.scores[query.pass.score] = value;
+            }
+        }
+        for (const item of items) {
+            const value = item.scores[query.pass.score];
+            item.missingScore = value === undefined;
+            item.passed = evaluatePass(query.pass, value);
+        }
+        let costUsd = null;
+        if (query.includeCost && items.length) {
+            const costs = await costsByTrace(items.map((item) => item.traceId), since);
+            for (const item of items)
+                item.costUsd = costs.get(item.traceId) ?? null;
+            costUsd = costs.size ? [...costs.values()].reduce((sum, cost) => sum + cost, 0) : null;
+        }
+        items.sort((a, b) => a.itemId.localeCompare(b.itemId));
+        return {
+            experimentId: query.experimentId ?? [...experimentIds][0] ?? "",
+            experimentName,
+            datasetId,
+            items,
+            passes: items.filter((item) => item.passed).length,
+            total: items.length,
+            missing: query.itemIds ? query.itemIds.filter((id) => !latest.has(id)) : [],
+            missingScores: items.filter((item) => item.missingScore).map((item) => item.itemId),
+            costUsd,
+        };
+    }
+    async function experimentUrl(datasetId, experimentId) {
+        const { id } = await getProject();
+        return `${http.baseUrl}/project/${encodeURIComponent(id)}/datasets/${encodeURIComponent(datasetId)}/runs/${encodeURIComponent(experimentId)}`;
+    }
+    return {
+        baseUrl: http.baseUrl,
+        async check() {
+            const { id, name } = await getProject();
+            return { baseUrl: http.baseUrl, projectId: id, projectName: name };
+        },
+        async listDatasets() {
+            const datasets = [];
+            for await (const page of numberedPages("/api/public/v2/datasets", {}))
+                for (const raw of page)
+                    if (isObject(raw) && str(raw.id) && str(raw.name))
+                        datasets.push({ id: String(raw.id), name: String(raw.name) });
+            return datasets;
+        },
+        getDataset,
+        async getDatasetItems(datasetName) {
+            const items = [];
+            for await (const page of numberedPages("/api/public/dataset-items", { datasetName })) {
+                for (const raw of page) {
+                    if (!isObject(raw) || !str(raw.id) || raw.status === "ARCHIVED")
+                        continue;
+                    items.push({ id: String(raw.id), input: raw.input ?? null, expectedOutput: raw.expectedOutput ?? null, metadata: isObject(raw.metadata) ? raw.metadata : {} });
+                }
+            }
+            return items;
+        },
+        async ensureDataset(input) {
+            if (!input.name?.trim())
+                throw new LangfuseError("A dataset needs a name.");
+            const existing = await http.request("GET", `/api/public/v2/datasets/${encodeURIComponent(input.name)}`, { allowNotFound: true });
+            const dataset = existing.data ?? (await http.request("POST", "/api/public/v2/datasets", { body: { name: input.name, description: input.description, metadata: input.metadata } })).data;
+            if (!isObject(dataset) || !str(dataset.id))
+                throw new LangfuseError(`Langfuse returned no ID for dataset "${input.name}".`);
+            const itemIds = [];
+            for (const item of input.items) {
+                const id = item.id ?? `om-${hexId(`${input.name}\n${stable(item.input)}`, 16)}`;
+                // Upserts by ID. Langfuse item IDs are unique per project, so reuse an ID in one dataset only.
+                await http.request("POST", "/api/public/dataset-items", { body: { datasetName: input.name, id, input: item.input, expectedOutput: item.expectedOutput, metadata: item.metadata } });
+                itemIds.push(id);
+            }
+            return { datasetId: String(dataset.id), itemIds };
+        },
+        listExperiments,
+        experimentResults,
+        async waitForExperiment(query, { expectItems, timeoutMs = 600_000, intervalMs = 15_000 }) {
+            const deadline = Date.now() + timeoutMs;
+            for (;;) {
+                const result = await experimentResults(query);
+                const scored = result.total - result.missingScores.length;
+                if (scored >= expectItems)
+                    return result;
+                if (Date.now() + intervalMs > deadline) {
+                    throw new LangfuseError(`Langfuse shows ${scored}/${expectItems} scored items for experiment ${query.experimentId ?? query.experimentName} after ${Math.round(timeoutMs / 1000)} s (missing items: ${result.missing.join(", ") || "none"}; unscored: ${result.missingScores.join(", ") || "none"}).`);
+                }
+                await sleep(intervalMs);
+            }
+        },
+        async recordExperimentItems(recording) {
+            const { datasetId, experimentId, experimentName } = recording;
+            if (![datasetId, experimentId, experimentName].every((value) => typeof value === "string" && value.trim()))
+                throw new LangfuseError("A recording needs datasetId, experimentId and experimentName.");
+            if (!recording.items.length)
+                throw new LangfuseError("A recording needs at least one item.");
+            const traceName = recording.traceName ?? "openmerit-eval";
+            const traceIds = {};
+            const spans = [];
+            const scores = [];
+            for (const item of recording.items) {
+                if (traceIds[item.itemId])
+                    throw new LangfuseError(`Item ${item.itemId} appears twice in one recording.`);
+                const traceId = recordedTraceId(experimentId, item.itemId);
+                const rootId = hexId(`${traceId}:root`, 16);
+                traceIds[item.itemId] = traceId;
+                const common = [
+                    attribute("langfuse.trace.name", traceName),
+                    attribute("langfuse.trace.tags", ["openmerit", `experiment:${experimentId}`]),
+                    attribute("langfuse.environment", environment),
+                    attribute("langfuse.release", `openmerit-langfuse-connector@${CONNECTOR_VERSION}`),
+                    attribute("langfuse.trace.metadata.model", item.model),
+                    ...Object.entries(item.metadata ?? {}).map(([key, value]) => attribute(`langfuse.trace.metadata.${key}`, String(value))),
+                ];
+                spans.push(span({
+                    traceId, spanId: rootId, name: traceName, startMs: item.startMs, endMs: item.endMs, ...(item.error ? { error: item.error } : {}),
+                    attributes: [
+                        ...common,
+                        attribute("langfuse.observation.type", "span"),
+                        attribute("langfuse.observation.input", asText(item.input)),
+                        attribute("langfuse.observation.output", asText(item.output ?? null)),
+                        ...(item.error ? [attribute("langfuse.observation.level", "ERROR"), attribute("langfuse.observation.status_message", item.error.slice(0, 500))] : []),
+                        attribute("langfuse.experiment.id", experimentId),
+                        attribute("langfuse.experiment.name", experimentName),
+                        attribute("langfuse.experiment.dataset.id", datasetId),
+                        attribute("langfuse.experiment.item.id", item.itemId),
+                        attribute("langfuse.experiment.item.root_observation_id", rootId),
+                        ...Object.entries(recording.metadata ?? {}).map(([key, value]) => attribute(`langfuse.experiment.metadata.${key}`, String(value))),
+                    ],
+                }), span({
+                    traceId, spanId: hexId(`${traceId}:generation`, 16), parentSpanId: rootId, name: `${traceName}-model`, startMs: item.startMs, endMs: item.endMs, ...(item.error ? { error: item.error } : {}),
+                    attributes: [
+                        ...common,
+                        attribute("langfuse.observation.type", "generation"),
+                        attribute("langfuse.observation.model.name", item.model),
+                        ...(item.usage ? [attribute("langfuse.observation.usage_details", JSON.stringify(item.usage))] : []),
+                        ...(item.costUsd !== undefined ? [attribute("langfuse.observation.cost_details", JSON.stringify({ total: item.costUsd }))] : []),
+                    ],
+                }));
+                for (const score of item.scores) {
+                    const dataType = score.dataType ?? (typeof score.value === "boolean" ? "BOOLEAN" : typeof score.value === "number" ? "NUMERIC" : "CATEGORICAL");
+                    const value = typeof score.value === "boolean" ? (score.value ? 1 : 0) : score.value;
+                    if (dataType === "CATEGORICAL" ? typeof value !== "string" : typeof value !== "number" || !Number.isFinite(value))
+                        throw new LangfuseError(`Score ${score.name} on item ${item.itemId} has a ${typeof score.value} value for data type ${dataType}.`);
+                    // A deterministic ID makes a retried recording update the score instead of adding a second one.
+                    scores.push({ id: hexId(`openmerit:score:${traceId}:${score.name}`, 32), traceId, observationId: rootId, name: score.name, value, dataType, environment, ...(score.comment ? { comment: score.comment.slice(0, 1000) } : {}), metadata: { source: "openmerit", experiment: experimentId } });
+                }
+            }
+            for (let i = 0; i < spans.length; i += 200) {
+                const response = await http.request("POST", "/api/public/otel/v1/traces", { body: exportRequest(spans.slice(i, i + 200), CONNECTOR_VERSION), headers: { "x-langfuse-ingestion-version": "4" } });
+                const rejected = Number(response.data?.partialSuccess?.rejectedSpans ?? 0);
+                if (rejected > 0)
+                    throw new LangfuseError(`Langfuse rejected ${rejected} spans: ${String(response.data?.partialSuccess?.errorMessage ?? "").slice(0, 300)}`);
+            }
+            for (let i = 0; i < scores.length; i += 100) {
+                const response = await http.request("POST", "/api/public/scores", { body: scores.slice(i, i + 100) });
+                // 207 reports rejected scores with a 2xx status; Langfuse asks clients not to retry the batch.
+                if (response.status === 207)
+                    throw new LangfuseError(`Langfuse rejected some scores (HTTP 207): ${JSON.stringify(response.data?.errors ?? response.data).slice(0, 300)}`, 207);
+            }
+            return { experimentId, traceIds, url: await experimentUrl(datasetId, experimentId) };
+        },
+        experimentUrl,
+    };
+}

package/dist/connectors/langfuse/otlp.d.ts

@@ -0,0 +1,60 @@
+/** Deterministic hex IDs, so a retried recording updates the same trace, spans and scores. */
+export declare const hexId: (value: string, length: 16 | 32) => string;
+type AnyValue = {
+    stringValue: string;
+} | {
+    boolValue: boolean;
+} | {
+    intValue: string;
+} | {
+    doubleValue: number;
+} | {
+    arrayValue: {
+        values: AnyValue[];
+    };
+};
+export interface Attribute {
+    key: string;
+    value: AnyValue;
+}
+export declare function attribute(key: string, value: string | number | boolean | string[]): Attribute;
+export declare const asText: (value: unknown) => string;
+export interface OtlpSpan {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: number;
+    startTimeUnixNano: string;
+    endTimeUnixNano: string;
+    attributes: Attribute[];
+    status: {
+        code: number;
+        message?: string;
+    };
+}
+export declare function span(input: {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    startMs: number;
+    endMs: number;
+    attributes: Attribute[];
+    error?: string;
+}): OtlpSpan;
+export declare function exportRequest(spans: OtlpSpan[], version: string): {
+    resourceSpans: {
+        resource: {
+            attributes: Attribute[];
+        };
+        scopeSpans: {
+            scope: {
+                name: string;
+                version: string;
+            };
+            spans: OtlpSpan[];
+        }[];
+    }[];
+};
+export {};

package/dist/connectors/langfuse/otlp.js

@@ -0,0 +1,40 @@
+// Builds OTLP/HTTP JSON spans with Langfuse's attribute names (ingestion format v4), so a
+// recorded evaluation is a Langfuse experiment item without the deprecated dataset-run-items API.
+// https://langfuse.com/integrations/native/opentelemetry
+import { createHash } from "node:crypto";
+/** Deterministic hex IDs, so a retried recording updates the same trace, spans and scores. */
+export const hexId = (value, length) => createHash("sha256").update(value).digest("hex").slice(0, length);
+export function attribute(key, value) {
+    if (Array.isArray(value))
+        return { key, value: { arrayValue: { values: value.map((item) => ({ stringValue: item })) } } };
+    if (typeof value === "boolean")
+        return { key, value: { boolValue: value } };
+    if (typeof value === "number")
+        return { key, value: Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value } };
+    return { key, value: { stringValue: value } };
+}
+export const asText = (value) => (typeof value === "string" ? value : JSON.stringify(value) ?? "null");
+const nanos = (ms) => `${BigInt(Math.round(ms)) * 1000000n}`;
+export function span(input) {
+    return {
+        traceId: input.traceId,
+        spanId: input.spanId,
+        ...(input.parentSpanId ? { parentSpanId: input.parentSpanId } : {}),
+        name: input.name,
+        kind: 1,
+        startTimeUnixNano: nanos(input.startMs),
+        endTimeUnixNano: nanos(Math.max(input.endMs, input.startMs)),
+        attributes: input.attributes,
+        status: input.error ? { code: 2, message: input.error.slice(0, 500) } : { code: 1 },
+    };
+}
+export function exportRequest(spans, version) {
+    return {
+        resourceSpans: [
+            {
+                resource: { attributes: [attribute("service.name", "openmerit")] },
+                scopeSpans: [{ scope: { name: "openmerit-langfuse-connector", version }, spans }],
+            },
+        ],
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export { rankCandidates, rank, openRouterToolSupport, type Candidate, type CandidateFilter, type RunnableModel } from "./scout/candidates.ts";
+export { loadEci, parseEciScores, ECI_ATTRIBUTION, type EciRow, type EciSource } from "./scout/eci.ts";
+export { scoutSearch, CandidateUnavailable, outward, compareFinals, meanCost, SAMPLE_SIZE, FINALISTS, type EvalItem, type Evaluate, type Outcome, type SearchOptions, type SearchResult, type Stage, type Trial } from "./scout/search.ts";
+export { runScout, type ScoutReport, type ScoutRunOptions, type FinalVerdict, type DatasetSource } from "./scout/pipeline.ts";
+export { reportMarkdown } from "./scout/report.ts";
+export { piEvaluator, jsonMatches, extractJson, type Grader, type ModelCaller, type PiTaskSpec } from "./scout/evaluators/pi.ts";
+export { loadEvaluatorPlugin, type EvaluatorPlugin } from "./scout/evaluators/module.ts";
+export * as scout from "./scout/service.ts";
+export { createScoutExtension, type ScoutExtensionOptions } from "./pi/extension.ts";
+export { configureHarness } from "./pi/harness.ts";
+export * as langfuse from "./connectors/langfuse/index.ts";

package/dist/index.js

@@ -0,0 +1,11 @@
+export { rankCandidates, rank, openRouterToolSupport } from "./scout/candidates.js";
+export { loadEci, parseEciScores, ECI_ATTRIBUTION } from "./scout/eci.js";
+export { scoutSearch, CandidateUnavailable, outward, compareFinals, meanCost, SAMPLE_SIZE, FINALISTS } from "./scout/search.js";
+export { runScout } from "./scout/pipeline.js";
+export { reportMarkdown } from "./scout/report.js";
+export { piEvaluator, jsonMatches, extractJson } from "./scout/evaluators/pi.js";
+export { loadEvaluatorPlugin } from "./scout/evaluators/module.js";
+export * as scout from "./scout/service.js";
+export { createScoutExtension } from "./pi/extension.js";
+export { configureHarness } from "./pi/harness.js";
+export * as langfuse from "./connectors/langfuse/index.js";

package/dist/pi/extension.d.ts

@@ -0,0 +1,13 @@
+import type { ExtensionFactory } from "@earendil-works/pi-coding-agent";
+export interface ScoutExtensionOptions {
+    /** Whether `scout_run` may spend money: true (headless `--execute`), false (plan only) or "ask" (confirm in the UI). */
+    allowSpend?: boolean | "ask";
+    /** Progress lines, for example to stderr in headless runs. */
+    log?: (line: string) => void;
+    /** The `provider/id` Scout runs on, for the judge grader. Defaults to the session's model. */
+    scoutModel?: string;
+}
+/** OpenMerit Scout inside Pi: model-recommendation tools, a `/scout` command and Scout's instructions. */
+export declare function createScoutExtension(options?: ScoutExtensionOptions): ExtensionFactory;
+declare const _default: ExtensionFactory;
+export default _default;