openmerit 0.3.11 → 0.4.1

package/dist/connectors/langfuse/client.js

@@ -0,0 +1,141 @@
+// HTTP transport for the Langfuse public API. It uses only global fetch, so it runs on
+// Node.js 22.19+ against Langfuse Cloud or any self-hosted Langfuse.
+//
+// Secrets are kept out of errors: a message names the method, the path and the status, plus at
+// most 300 characters of Langfuse's own error body, and never the Authorization header.
+export class LangfuseError extends Error {
+    status;
+    constructor(message, status = null) {
+        super(message);
+        this.name = "LangfuseError";
+        this.status = status;
+    }
+}
+const DEFAULT_BASE_URL = "https://cloud.langfuse.com";
+const MAX_BODY_BYTES = 8_000_000;
+/**
+ * Reads LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY and LANGFUSE_BASE_URL (or LANGFUSE_HOST, the
+ * Langfuse SDK's name). Without a base URL it uses Langfuse Cloud EU, like the Langfuse SDKs.
+ */
+export function connectionFromEnv(env = process.env) {
+    const publicKey = env.LANGFUSE_PUBLIC_KEY?.trim();
+    const secretKey = env.LANGFUSE_SECRET_KEY?.trim();
+    const missing = [!publicKey && "LANGFUSE_PUBLIC_KEY", !secretKey && "LANGFUSE_SECRET_KEY"].filter(Boolean);
+    if (missing.length)
+        throw new LangfuseError(`Set ${missing.join(" and ")} to connect Langfuse (and LANGFUSE_BASE_URL for a self-hosted or US instance).`);
+    return { baseUrl: normalizeBaseUrl(env.LANGFUSE_BASE_URL?.trim() || env.LANGFUSE_HOST?.trim() || DEFAULT_BASE_URL), publicKey: publicKey, secretKey: secretKey };
+}
+export function normalizeBaseUrl(value) {
+    let url;
+    try {
+        url = new URL(value);
+    }
+    catch {
+        throw new LangfuseError(`LANGFUSE_BASE_URL is not a URL: "${value.slice(0, 200)}".`);
+    }
+    if (url.protocol !== "https:" && url.protocol !== "http:")
+        throw new LangfuseError("LANGFUSE_BASE_URL must start with https:// or http://.");
+    if (url.username || url.password || url.search || url.hash)
+        throw new LangfuseError("LANGFUSE_BASE_URL must not contain credentials, a query or a fragment.");
+    return `${url.origin}${url.pathname.replace(/\/+$/, "")}`;
+}
+export function createTransport(connection, options = {}) {
+    const baseUrl = normalizeBaseUrl(connection.baseUrl);
+    const fetcher = options.fetch ?? fetch;
+    const timeoutMs = options.timeoutMs ?? 30_000;
+    const retries = options.retries ?? 5;
+    const maxDelay = options.maxRetryDelayMs ?? 120_000;
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const authorization = `Basic ${Buffer.from(`${connection.publicKey}:${connection.secretKey}`).toString("base64")}`;
+    return {
+        baseUrl,
+        async request(method, path, request = {}) {
+            const url = new URL(`${baseUrl}${path}`);
+            for (const [key, value] of Object.entries(request.query ?? {}))
+                if (value !== undefined && value !== null && value !== "")
+                    url.searchParams.set(key, String(value));
+            const label = `Langfuse ${method} ${path}`;
+            for (let attempt = 0;; attempt += 1) {
+                let response;
+                try {
+                    response = await fetcher(url, {
+                        method,
+                        // Never follow a redirect with the key pair attached; a redirect means LANGFUSE_BASE_URL is wrong.
+                        redirect: "manual",
+                        signal: AbortSignal.timeout(timeoutMs),
+                        headers: { authorization, accept: "application/json", ...(request.body === undefined ? {} : { "content-type": "application/json" }), ...request.headers },
+                        ...(request.body === undefined ? {} : { body: JSON.stringify(request.body) }),
+                    });
+                }
+                catch (error) {
+                    const reason = error instanceof Error ? (error.name === "TimeoutError" ? `timed out after ${timeoutMs} ms` : error.message) : String(error);
+                    if (attempt < retries) {
+                        await sleep(Math.min(maxDelay, 1000 * 2 ** attempt));
+                        continue;
+                    }
+                    throw new LangfuseError(`${label} failed: ${reason}. Check LANGFUSE_BASE_URL (${baseUrl}).`);
+                }
+                const status = response.status;
+                if ((status === 429 || status === 502 || status === 503 || status === 504) && attempt < retries) {
+                    await response.body?.cancel();
+                    const retryAfter = Number(response.headers.get("retry-after"));
+                    await sleep(Math.min(maxDelay, Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : 1000 * 2 ** attempt));
+                    continue;
+                }
+                if (status >= 300 && status < 400) {
+                    await response.body?.cancel();
+                    throw new LangfuseError(`${label} was redirected (HTTP ${status}). Set LANGFUSE_BASE_URL to the final Langfuse URL; credentials are not forwarded across redirects.`, status);
+                }
+                if (status === 404 && request.allowNotFound) {
+                    await response.body?.cancel();
+                    return { status, data: null };
+                }
+                const text = await readBounded(response, label);
+                if (!response.ok) {
+                    const hint = status === 401 ? " Check LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY and that LANGFUSE_BASE_URL is the instance or region that issued them." : status === 429 ? " Langfuse kept rate limiting after retries; try again later." : "";
+                    throw new LangfuseError(`${label} failed (HTTP ${status}): ${excerpt(text).replace(/[.\s]*$/, ".")}${hint}`, status);
+                }
+                if (!text)
+                    return { status, data: null };
+                try {
+                    return { status, data: JSON.parse(text) };
+                }
+                catch {
+                    throw new LangfuseError(`${label} returned invalid JSON (HTTP ${status}). Is LANGFUSE_BASE_URL a Langfuse instance?`, status);
+                }
+            }
+        },
+    };
+}
+async function readBounded(response, label) {
+    const reader = response.body?.getReader();
+    if (!reader)
+        return "";
+    const decoder = new TextDecoder();
+    let bytes = 0;
+    let text = "";
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        bytes += value.length;
+        if (bytes > MAX_BODY_BYTES) {
+            await reader.cancel();
+            throw new LangfuseError(`${label} returned more than ${MAX_BODY_BYTES} bytes.`);
+        }
+        text += decoder.decode(value, { stream: true });
+    }
+    return text + decoder.decode();
+}
+function excerpt(text) {
+    let message = text;
+    try {
+        const parsed = JSON.parse(text);
+        if (parsed && typeof parsed === "object" && "message" in parsed && typeof parsed.message === "string")
+            message = parsed.message;
+    }
+    catch {
+        // Not JSON; use the raw text.
+    }
+    return message.replace(/\s+/g, " ").trim().slice(0, 300) || "(empty body)";
+}

package/dist/connectors/langfuse/index.d.ts

@@ -0,0 +1,163 @@
+import { type LangfuseConnection, type TransportOptions } from "./client.js";
+export { connectionFromEnv, type LangfuseConnection, LangfuseError, normalizeBaseUrl, type TransportOptions } from "./client.js";
+export type ScoreValue = number | boolean | string;
+export type Metadata = Record<string, string | number | boolean>;
+export interface EvalItem {
+    id: string;
+    input: unknown;
+    expectedOutput: unknown;
+    metadata: Record<string, unknown>;
+}
+export interface DatasetSummary {
+    id: string;
+    name: string;
+}
+export interface ExperimentSummary {
+    id: string;
+    name: string;
+    datasetId: string | null;
+    itemCount: number | null;
+    startTime: string | null;
+    metadata: Record<string, unknown>;
+    /** Experiment metadata `model_<role>` entries, e.g. { enrollment: "openrouter/anthropic/claude-sonnet-5.5" }. */
+    models: Record<string, string>;
+}
+/**
+ * Decides whether one item passed from one score. With no condition, a BOOLEAN score must be true
+ * and a NUMERIC score must be 1. `equals` compares exactly; `min`/`max` are inclusive numeric bounds.
+ */
+export interface PassRule {
+    score: string;
+    equals?: ScoreValue;
+    min?: number;
+    max?: number;
+}
+export interface ItemResult {
+    itemId: string;
+    traceId: string;
+    passed: boolean;
+    /** The pass score was absent: the item failed to run or was not scored. It never counts as a pass. */
+    missingScore: boolean;
+    scores: Record<string, ScoreValue>;
+    costUsd: number | null;
+}
+export interface ExperimentResult {
+    experimentId: string;
+    experimentName: string;
+    datasetId: string | null;
+    items: ItemResult[];
+    passes: number;
+    total: number;
+    /** Requested item IDs with no experiment item in Langfuse. */
+    missing: string[];
+    /** Item IDs present without the pass score. */
+    missingScores: string[];
+    /** Sum of generation costs Langfuse recorded for the items; null when not requested or none recorded. */
+    costUsd: number | null;
+}
+export interface ExperimentQuery {
+    experimentId?: string;
+    /** Names are not unique (the playground gives its claims and enrollments runs one name); an ambiguous name throws. */
+    experimentName?: string;
+    pass: PassRule;
+    /** Only these items count, e.g. the five screening samples; absent ones are listed in `missing`. */
+    itemIds?: string[];
+    includeCost?: boolean;
+    /** Lower bound for Langfuse's required start-time filter. Default: 30 days ago. */
+    since?: Date | string;
+}
+export interface RecordedScore {
+    name: string;
+    value: ScoreValue;
+    /** Inferred from the value when absent: boolean → BOOLEAN, number → NUMERIC, string → CATEGORICAL. */
+    dataType?: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
+    comment?: string;
+}
+export interface RecordedItem {
+    /** Dataset item ID; it must exist in the experiment's dataset. */
+    itemId: string;
+    /** Model that produced the output, e.g. "openrouter/deepseek/deepseek-v4-pro". */
+    model: string;
+    input: unknown;
+    output: unknown;
+    startMs: number;
+    endMs: number;
+    usage?: {
+        input?: number;
+        output?: number;
+        total?: number;
+    };
+    costUsd?: number;
+    /** Set when the run failed; the root observation gets level ERROR. */
+    error?: string;
+    /** Per-item trace metadata, e.g. { stage: "screen", eci: 142.1 }. */
+    metadata?: Metadata;
+    scores: RecordedScore[];
+}
+export interface ExperimentRecording {
+    datasetId: string;
+    experimentId: string;
+    experimentName: string;
+    /** Experiment metadata. Put the candidate model under `model_<role>` so listExperiments reports it. */
+    metadata?: Metadata;
+    /** Trace and root observation name. Default "openmerit-eval". */
+    traceName?: string;
+    items: RecordedItem[];
+}
+export interface RecordedExperiment {
+    experimentId: string;
+    traceIds: Record<string, string>;
+    url: string;
+}
+export interface DatasetInput {
+    name: string;
+    description?: string;
+    metadata?: Metadata;
+    /** Without an ID, an item gets one derived from the dataset name and its input, so re-pushing a file upserts. */
+    items: {
+        id?: string;
+        input: unknown;
+        expectedOutput: unknown;
+        metadata?: Record<string, unknown>;
+    }[];
+}
+export interface LangfuseConnector {
+    readonly baseUrl: string;
+    check(): Promise<{
+        baseUrl: string;
+        projectId: string;
+        projectName: string;
+    }>;
+    listDatasets(): Promise<DatasetSummary[]>;
+    getDataset(name: string): Promise<DatasetSummary>;
+    getDatasetItems(datasetName: string): Promise<EvalItem[]>;
+    ensureDataset(input: DatasetInput): Promise<{
+        datasetId: string;
+        itemIds: string[];
+    }>;
+    listExperiments(query?: {
+        datasetName?: string;
+        datasetId?: string;
+        names?: string[];
+        ids?: string[];
+        since?: Date | string;
+    }): Promise<ExperimentSummary[]>;
+    experimentResults(query: ExperimentQuery): Promise<ExperimentResult>;
+    waitForExperiment(query: ExperimentQuery, options: {
+        expectItems: number;
+        timeoutMs?: number;
+        intervalMs?: number;
+    }): Promise<ExperimentResult>;
+    recordExperimentItems(recording: ExperimentRecording): Promise<RecordedExperiment>;
+    experimentUrl(datasetId: string, experimentId: string): Promise<string>;
+}
+export interface ConnectorOptions extends TransportOptions {
+    /** Langfuse environment for recorded traces and scores. Default "openmerit". */
+    environment?: string;
+}
+/** Version written into recorded traces' scope. */
+export declare const CONNECTOR_VERSION = "0.1.0";
+export declare function evaluatePass(rule: PassRule, value: ScoreValue | undefined): boolean;
+/** Trace ID of one recorded item: the first 32 hex characters of SHA-256 over its experiment and item. */
+export declare const recordedTraceId: (experimentId: string, itemId: string) => string;
+export declare function createLangfuseConnector(connection: LangfuseConnection, options?: ConnectorOptions): LangfuseConnector;