openmerit 0.4.1 → 0.4.2

package/dist/connect.js

@@ -0,0 +1,158 @@
+import { DEFAULT_CLAUDE_MODEL } from "./harness/claude-bridge.js";
+import { loadCatalogue } from "./catalogue/catalogue.js";
+import { CODEX_PROVIDER, createHarness, detectAgents, harnessRuntime, runClaude, verifyHarness } from "./harness/harness.js";
+import { readGlobalConfig, storeSecret, updateGlobalConfig } from "./state/config.js";
+export const HARNESS_LABEL = { "claude-code": "Claude Code", codex: "Codex (ChatGPT sign-in)", api: "an API key" };
+const interactive = (ctx) => !!ctx.prompter && !ctx.flags.nonInteractive;
+export async function confirm(ctx, question, fallback) { return interactive(ctx) ? ctx.prompter.confirm(question, fallback) : fallback; }
+/** A key from an environment variable (kept as a reference) or typed in (stored in ~/.openmerit/secrets.json). */
+export async function credential(ctx, label, envNames, secretName, flagEnv) {
+    if (flagEnv)
+        return ctx.env[flagEnv] ? { env: flagEnv } : (() => { throw new Error(`$${flagEnv} is not set.`); })();
+    const present = envNames.find(name => ctx.env[name]?.trim());
+    if (present && await confirm(ctx, `Use $${present} for the ${label}?`, true))
+        return { env: present };
+    if (!interactive(ctx))
+        return null;
+    const value = await ctx.prompter.secret(`${label} (input hidden; stored in ~/.openmerit/secrets.json, outside the repository):`);
+    return value ? storeSecret(secretName, value, ctx.env) : null;
+}
+/** Pi's login prompts and notices in the terminal, or on stderr only without one (device code). */
+function authInteraction(ctx) {
+    return {
+        async prompt(prompt) {
+            if (prompt.type === "select") {
+                const ids = prompt.options.map(option => option.id);
+                const device = ids.find(id => id === "device_code");
+                if (device && (ctx.flags.deviceCode || !interactive(ctx)))
+                    return device;
+                if (!interactive(ctx))
+                    return ids[0];
+                return ctx.prompter.choose(prompt.message, prompt.options.map(option => ({ value: option.id, label: option.label })), ids[0]);
+            }
+            if (!interactive(ctx))
+                throw new Error(`${prompt.message} needs a terminal. Rerun with --device-code.`);
+            return prompt.type === "secret" ? ctx.prompter.secret(prompt.message) : ctx.prompter.ask(prompt.message, "", prompt.signal);
+        },
+        notify(event) {
+            if (event.type === "auth_url")
+                ctx.log(`  Open this URL to sign in to ChatGPT:\n  ${event.url}${event.instructions ? `\n  ${event.instructions}` : ""}`);
+            else if (event.type === "device_code")
+                ctx.log(`  On any device, open ${event.verificationUri} and enter the code ${event.userCode}${event.expiresInSeconds ? ` within ${Math.round(event.expiresInSeconds / 60)} minutes` : ""}.`);
+            else
+                ctx.log(`  ${event.message}`);
+        },
+    };
+}
+/** OpenAI's reply when a ChatGPT plan does not include a Codex model. */
+const NOT_ON_PLAN = /not supported when using Codex with a ChatGPT account/i;
+/** Pi's Codex models, most capable (by the bundled catalogue's ECI) first. */
+async function codexModelsByEci(env) {
+    const eci = new Map(loadCatalogue().models.map(model => [model.id, model.eci ?? 0]));
+    const score = (id) => eci.get(`openai/${id}`) ?? 0;
+    return (await harnessRuntime(env)).getModels(CODEX_PROVIDER).map(model => model.id).sort((a, b) => score(b) - score(a));
+}
+/** Makes sure the chosen connection is signed in. Returns false when the user declined. */
+async function signIn(ctx, kind) {
+    if (kind === "api")
+        return true;
+    const agent = (await detectAgents(ctx.env)).find(item => item.kind === kind);
+    if (agent.loggedIn)
+        return true;
+    if (kind === "claude-code") {
+        if (!interactive(ctx))
+            throw new Error("Claude Code is not logged in. Run `openmerit connect claude-code` in a terminal to log in, or on a server set CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token`.");
+        if (!(await confirm(ctx, "Claude Code is not logged in. Log in with your Claude subscription now?", true)))
+            return false;
+        await ctx.prompter.handOver(() => runClaude(["auth", "login"], { env: ctx.env, interactive: true }));
+        return true;
+    }
+    ctx.log("  Signing in to ChatGPT (Plus or Pro) for Codex models. OpenMerit keeps the sign-in in ~/.openmerit/agent/auth.json.");
+    const runtime = await harnessRuntime(ctx.env);
+    await runtime.login(CODEX_PROVIDER, "oauth", authInteraction(ctx));
+    await runtime.settleCredentialWrites();
+    ctx.log("  ✓ Signed in to ChatGPT.");
+    return true;
+}
+/**
+ * Connects OpenMerit Harness. With `kind`, connects that; without it, keeps a working saved
+ * connection when the user agrees, and otherwise asks which to use.
+ */
+export async function connect(ctx, kind) {
+    const existing = readGlobalConfig(ctx.env).harness;
+    if (existing && !kind) {
+        ctx.log(`Checking the current harness connection (${HARNESS_LABEL[existing.kind]}${existing.model ? `, ${existing.model}` : ""})…`);
+        const check = await createHarness(existing, ctx.env).then(harness => verifyHarness(harness, ctx.root), (error) => ({ ok: false, detail: error instanceof Error ? error.message : String(error) }));
+        if (check.ok) {
+            ctx.log(`  ✓ OpenMerit Harness runs on ${HARNESS_LABEL[existing.kind]} (${check.detail}).`);
+            if (await confirm(ctx, "Keep this harness connection?", true)) {
+                const saved = { ...existing, verifiedAt: new Date().toISOString() };
+                updateGlobalConfig({ harness: saved }, ctx.env);
+                return saved;
+            }
+        }
+        else
+            ctx.log(`  ✗ The saved harness connection failed: ${check.detail}`);
+    }
+    for (;;) {
+        if (!kind) {
+            if (!interactive(ctx))
+                throw new Error("Choose how to power OpenMerit Harness: `openmerit connect claude-code`, `codex` or `api`.");
+            const agents = await detectAgents(ctx.env);
+            kind = await ctx.prompter.choose("Which should power OpenMerit Harness?", [
+                ...agents.map(agent => ({ value: agent.kind, label: `${HARNESS_LABEL[agent.kind]} (${agent.detail})` })),
+                { value: "api", label: "An API provider, key and model" },
+            ], agents.find(agent => agent.loggedIn)?.kind ?? "claude-code");
+        }
+        const settings = { kind };
+        if (kind === "api") {
+            const provider = ctx.flags.apiProvider ?? (interactive(ctx) ? await ctx.prompter.ask("Pi provider for the harness (anthropic, openai, openrouter, google, ...)", existing?.api?.provider ?? "anthropic") : "anthropic");
+            const suggested = (existing?.api?.provider === provider ? existing.api.model : undefined) ?? (provider === "openai" ? "gpt-6.1-sol" : provider === "openrouter" ? "anthropic/claude-sonnet-5.5" : "claude-sonnet-5-5");
+            const model = ctx.flags.apiModel ?? (interactive(ctx) ? await ctx.prompter.ask("Model", suggested) : suggested);
+            const envNames = { anthropic: ["ANTHROPIC_API_KEY"], openai: ["OPENAI_API_KEY"], openrouter: ["OPENROUTER_API_KEY"], google: ["GEMINI_API_KEY", "GOOGLE_API_KEY"] }[provider] ?? [];
+            const apiKey = await credential(ctx, `${provider} API key for the harness`, envNames, `harness-${provider}`, ctx.flags.apiKeyEnv);
+            if (!apiKey)
+                throw new Error("The API harness needs a key: pass --api-key-env <VAR>.");
+            settings.api = { provider, model, apiKey };
+        }
+        else {
+            const chosen = ctx.flags.model ?? (existing?.kind === kind ? existing.model : undefined);
+            if (chosen || kind === "claude-code")
+                settings.model = chosen ?? DEFAULT_CLAUDE_MODEL;
+            ctx.log(settings.model ? `  Model: ${settings.model} (change it with --model).` : "  Model: the most capable Codex model your ChatGPT plan allows (choose one with --model).");
+        }
+        let check = { ok: false, detail: "no model to try" };
+        try {
+            if (!(await signIn(ctx, kind)))
+                return null;
+            // Which Codex models a ChatGPT account may use depends on its plan, and OpenAI does not list them: try them by ECI.
+            const models = kind === "codex" && !settings.model ? await codexModelsByEci(ctx.env) : [settings.model];
+            for (const model of models) {
+                if (model)
+                    settings.model = model;
+                ctx.log(`Verifying that ${HARNESS_LABEL[kind]}${model ? ` (${model})` : ""} powers OpenMerit Harness…`);
+                check = await verifyHarness(await createHarness(settings, ctx.env), ctx.root);
+                if (check.ok || !NOT_ON_PLAN.test(check.detail))
+                    break;
+                ctx.log(`  ${model} is not available on this ChatGPT plan.`);
+            }
+        }
+        catch (error) {
+            check = { ok: false, detail: error instanceof Error ? error.message : String(error) };
+        }
+        if (check.ok) {
+            ctx.log(`  ✓ ${HARNESS_LABEL[kind]} answered through OpenMerit Harness (${check.detail}).`);
+            const saved = { ...settings, verifiedAt: new Date().toISOString() };
+            updateGlobalConfig({ harness: saved }, ctx.env);
+            return saved;
+        }
+        ctx.log(`  ✗ ${HARNESS_LABEL[kind]} did not answer: ${check.detail}`);
+        if (!interactive(ctx))
+            throw new Error(`${HARNESS_LABEL[kind]} could not power the harness: ${check.detail}${existing ? ". Your previous harness settings are unchanged." : ""}`);
+        const next = await ctx.prompter.choose("What now?", [{ value: "retry", label: `Retry ${HARNESS_LABEL[kind]}` }, { value: "other", label: "Choose another connection" }, { value: "quit", label: existing ? "Stop and keep the previous settings" : "Stop" }], "retry");
+        if (next === "quit")
+            return null;
+        if (next === "other")
+            kind = undefined;
+    }
+}

package/dist/connectors/langfuse/index.d.ts

@@ -1,27 +1,6 @@
 import { type LangfuseConnection, type TransportOptions } from "./client.js";
 export { connectionFromEnv, type LangfuseConnection, LangfuseError, normalizeBaseUrl, type TransportOptions } from "./client.js";
 export type ScoreValue = number | boolean | string;
-export type Metadata = Record<string, string | number | boolean>;
-export interface EvalItem {
-    id: string;
-    input: unknown;
-    expectedOutput: unknown;
-    metadata: Record<string, unknown>;
-}
-export interface DatasetSummary {
-    id: string;
-    name: string;
-}
-export interface ExperimentSummary {
-    id: string;
-    name: string;
-    datasetId: string | null;
-    itemCount: number | null;
-    startTime: string | null;
-    metadata: Record<string, unknown>;
-    /** Experiment metadata `model_<role>` entries, e.g. { enrollment: "openrouter/anthropic/claude-sonnet-5.5" }. */
-    models: Record<string, string>;
-}
 /**
  * Decides whether one item passed from one score. With no condition, a BOOLEAN score must be true
  * and a NUMERIC score must be 1. `equals` compares exactly; `min`/`max` are inclusive numeric bounds.
@@ -38,12 +17,10 @@ export interface ItemResult {
     passed: boolean;
     /** The pass score was absent: the item failed to run or was not scored. It never counts as a pass. */
     missingScore: boolean;
-    scores: Record<string, ScoreValue>;
-    costUsd: number | null;
 }
 export interface ExperimentResult {
-    experimentId: string;
     experimentName: string;
+    experimentId: string;
     datasetId: string | null;
     items: ItemResult[];
     passes: number;
@@ -52,74 +29,34 @@ export interface ExperimentResult {
     missing: string[];
     /** Item IDs present without the pass score. */
     missingScores: string[];
-    /** Sum of generation costs Langfuse recorded for the items; null when not requested or none recorded. */
-    costUsd: number | null;
 }
 export interface ExperimentQuery {
-    experimentId?: string;
-    /** Names are not unique (the playground gives its claims and enrollments runs one name); an ambiguous name throws. */
-    experimentName?: string;
+    experimentName: string;
     pass: PassRule;
-    /** Only these items count, e.g. the five screening samples; absent ones are listed in `missing`. */
-    itemIds?: string[];
-    includeCost?: boolean;
-    /** Lower bound for Langfuse's required start-time filter. Default: 30 days ago. */
-    since?: Date | string;
+    /** Only these items count; absent ones are listed in `missing`. */
+    itemIds?: readonly string[];
+    /** Lower bound for Langfuse's required start-time filter. */
+    since: string;
 }
-export interface RecordedScore {
-    name: string;
-    value: ScoreValue;
-    /** Inferred from the value when absent: boolean → BOOLEAN, number → NUMERIC, string → CATEGORICAL. */
-    dataType?: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
-    comment?: string;
-}
-export interface RecordedItem {
-    /** Dataset item ID; it must exist in the experiment's dataset. */
-    itemId: string;
-    /** Model that produced the output, e.g. "openrouter/deepseek/deepseek-v4-pro". */
+export interface ModelUsage {
+    /** Model name as the traces report it (Langfuse `providedModelName`). */
     model: string;
-    input: unknown;
-    output: unknown;
-    startMs: number;
-    endMs: number;
-    usage?: {
-        input?: number;
-        output?: number;
-        total?: number;
-    };
-    costUsd?: number;
-    /** Set when the run failed; the root observation gets level ERROR. */
-    error?: string;
-    /** Per-item trace metadata, e.g. { stage: "screen", eci: 142.1 }. */
-    metadata?: Metadata;
-    scores: RecordedScore[];
-}
-export interface ExperimentRecording {
-    datasetId: string;
-    experimentId: string;
-    experimentName: string;
-    /** Experiment metadata. Put the candidate model under `model_<role>` so listExperiments reports it. */
-    metadata?: Metadata;
-    /** Trace and root observation name. Default "openmerit-eval". */
-    traceName?: string;
-    items: RecordedItem[];
-}
-export interface RecordedExperiment {
-    experimentId: string;
-    traceIds: Record<string, string>;
-    url: string;
+    generations: number;
+    costUsd: number | null;
+    p50LatencyMs: number | null;
 }
-export interface DatasetInput {
+export interface ScoreAggregate {
     name: string;
-    description?: string;
-    metadata?: Metadata;
-    /** Without an ID, an item gets one derived from the dataset name and its input, so re-pushing a file upserts. */
-    items: {
-        id?: string;
-        input: unknown;
-        expectedOutput: unknown;
-        metadata?: Record<string, unknown>;
-    }[];
+    average: number;
+    count: number;
+}
+export interface TaskEvidence {
+    traceName: string;
+    from: string;
+    to: string;
+    models: ModelUsage[];
+    /** Score averages over every trace of this name in the window, whichever model produced it. */
+    scores: ScoreAggregate[];
 }
 export interface LangfuseConnector {
     readonly baseUrl: string;
@@ -128,36 +65,26 @@ export interface LangfuseConnector {
         projectId: string;
         projectName: string;
     }>;
-    listDatasets(): Promise<DatasetSummary[]>;
-    getDataset(name: string): Promise<DatasetSummary>;
-    getDatasetItems(datasetName: string): Promise<EvalItem[]>;
-    ensureDataset(input: DatasetInput): Promise<{
-        datasetId: string;
-        itemIds: string[];
-    }>;
-    listExperiments(query?: {
-        datasetName?: string;
-        datasetId?: string;
-        names?: string[];
-        ids?: string[];
-        since?: Date | string;
-    }): Promise<ExperimentSummary[]>;
+    /** Throws unless the key pair is read-only (Viewer). */
+    verifyViewer(): Promise<void>;
+    /** IDs of the dataset's active items. */
+    datasetItemIds(name: string): Promise<string[]>;
     experimentResults(query: ExperimentQuery): Promise<ExperimentResult>;
+    /** Polls until every expected item is present and scored, allowing for ingestion lag; throws on timeout. */
     waitForExperiment(query: ExperimentQuery, options: {
         expectItems: number;
         timeoutMs?: number;
         intervalMs?: number;
     }): Promise<ExperimentResult>;
-    recordExperimentItems(recording: ExperimentRecording): Promise<RecordedExperiment>;
-    experimentUrl(datasetId: string, experimentId: string): Promise<string>;
-}
-export interface ConnectorOptions extends TransportOptions {
-    /** Langfuse environment for recorded traces and scores. Default "openmerit". */
-    environment?: string;
+    /** Trace names with generation counts, to suggest task mappings in setup. */
+    traceNames(days?: number): Promise<{
+        traceName: string;
+        generations: number;
+    }[]>;
+    taskEvidence(mapping: {
+        traceName: string;
+        scores: string[];
+    }, days?: number): Promise<TaskEvidence>;
 }
-/** Version written into recorded traces' scope. */
-export declare const CONNECTOR_VERSION = "0.1.0";
 export declare function evaluatePass(rule: PassRule, value: ScoreValue | undefined): boolean;
-/** Trace ID of one recorded item: the first 32 hex characters of SHA-256 over its experiment and item. */
-export declare const recordedTraceId: (experimentId: string, itemId: string) => string;
-export declare function createLangfuseConnector(connection: LangfuseConnection, options?: ConnectorOptions): LangfuseConnector;
+export declare function createLangfuseConnector(connection: LangfuseConnection, options?: TransportOptions): LangfuseConnector;