openmerit 0.1.0

package/dist/pi-trials.js

@@ -0,0 +1,224 @@
+/** Run a task through pi for each model, using pi's JSON event stream as the measurement source. */
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { judge, judgeWithImages } from "./judge.js";
+import { knownInvoiceScore } from "./invoice-eval.js";
+import { paths, readJson } from "./store.js";
+import { sessionImages, taskInputKey } from "./task-input.js";
+function answerText(content) {
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    return content.filter((x) => !!x && typeof x === "object" && x.type === "text")
+        .map((x) => x.text ?? "").join("\n");
+}
+/** Use model A's completed result from the active pi session when it matches this task. */
+export function recordedActiveTask(task, model, images = [], sessionFile, sessionBytes) {
+    const state = sessionFile ? null : readJson(paths.harnessState(), {});
+    const file = sessionFile ?? state?.sessionFile;
+    if (!file || !existsSync(file))
+        return null;
+    let user = null;
+    const assistants = [];
+    for (const line of readFileSync(file).subarray(0, sessionBytes).toString("utf8").split("\n")) {
+        if (!line.trim())
+            continue;
+        let entry;
+        try {
+            entry = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (entry.type !== "message" || !entry.message)
+            continue;
+        if (entry.message.role === "user") {
+            user = entry.message;
+            assistants.length = 0;
+        }
+        else if (user && entry.message.role === "assistant")
+            assistants.push(entry.message);
+    }
+    const userImages = user ? sessionImages(user.content) : null;
+    if (!user || !userImages || taskInputKey(answerText(user.content), userImages) !==
+        taskInputKey(task, images))
+        return null;
+    const matching = assistants.filter((m) => (m.provider === "openrouter" ? m.model : `${m.provider}/${m.model}`) === model);
+    if (matching.length === 0 || matching.some((m) => m.stopReason === "error"))
+        return null;
+    const answer = matching.map((m) => answerText(m.content)).filter(Boolean).at(-1) ?? "";
+    if (!answer.trim())
+        return null;
+    const first = user.timestamp ?? 0;
+    const last = matching.at(-1)?.timestamp ?? first;
+    return {
+        answer,
+        costUsd: matching.reduce((n, m) => n + (m.usage?.cost?.total ?? 0), 0),
+        latencyMs: Math.max(0, last - first),
+        errors: 0,
+    };
+}
+/** Read the exact latest completed text task from the pi extension's session marker. */
+export function settledActiveTask(snapshot) {
+    const st = snapshot ?? readJson(paths.harnessState(), {});
+    if (!st.currentModel || !st.sessionFile || !st.settledTaskKey || !st.settledAt ||
+        !existsSync(st.sessionFile))
+        return null;
+    let userContent = null;
+    let usedTools = false;
+    for (const line of readFileSync(st.sessionFile).subarray(0, st.sessionBytes).toString("utf8").split("\n")) {
+        if (!line.trim())
+            continue;
+        let entry;
+        try {
+            entry = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (entry.type !== "message" || !entry.message)
+            continue;
+        if (entry.message.role === "user") {
+            userContent = entry.message.content;
+            usedTools = false;
+        }
+        else if (userContent && entry.message.role === "toolResult")
+            usedTools = true;
+        else if (userContent && entry.message.role === "assistant" &&
+            Array.isArray(entry.message.content) && entry.message.content.some((b) => b?.type === "toolCall"))
+            usedTools = true;
+    }
+    if (!userContent || usedTools)
+        return null;
+    const images = sessionImages(userContent);
+    if (!images)
+        return null;
+    const task = answerText(userContent);
+    if (!task.trim() || taskInputKey(task, images) !== st.settledTaskKey)
+        return null;
+    const run = recordedActiveTask(task, st.currentModel, images, st.sessionFile, st.sessionBytes);
+    if (!run)
+        return null;
+    return { task, images, model: st.currentModel, run, sessionFile: st.sessionFile,
+        settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd() };
+}
+/** Limit candidates to models this installed pi can actually select. */
+export function availablePiModels() {
+    const result = spawnSync("pi", ["--offline", "--list-models", "openrouter"], { encoding: "utf8" });
+    if (result.error || result.status !== 0)
+        throw new Error("could not list pi OpenRouter models");
+    return new Set(result.stdout.split("\n").slice(1)
+        .map((line) => line.trim().split(/\s+/)[1])
+        .filter((id) => !!id && !id.startsWith("~")));
+}
+/** Each invocation is a fresh pi run, so model B does not inherit model A's answer. */
+export async function executePiTask(model, task, cwd = process.cwd(), images = []) {
+    const imageDir = images.length ? mkdtempSync(join(tmpdir(), "openmerit-pi-image-")) : null;
+    const suffix = {
+        "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", "image/gif": "gif",
+    };
+    const imageArgs = images.map((image, i) => {
+        if (!imageDir || !suffix[image.mimeType])
+            throw new Error(`unsupported pi image: ${image.mimeType}`);
+        const file = join(imageDir, `image-${i}.${suffix[image.mimeType]}`);
+        writeFileSync(file, Buffer.from(image.data, "base64"));
+        return `@${file}`;
+    });
+    const child = spawn("pi", [
+        "--provider", "openrouter", "--model", model, "--mode", "json",
+        "--offline", "--no-extensions", "--no-tools", "--print", ...imageArgs, task,
+    ], { cwd, stdio: ["ignore", "pipe", "pipe"] });
+    let buffer = "";
+    let stderr = "";
+    let answer = "";
+    let costUsd = 0;
+    let errors = 0;
+    let sessionId;
+    const started = Date.now();
+    function consume(line) {
+        if (!line.trim())
+            return;
+        let event;
+        try {
+            event = JSON.parse(line);
+        }
+        catch {
+            return;
+        }
+        if (event.type === "session" && event.id)
+            sessionId = event.id;
+        if (event.type !== "message_end" || event.message?.role !== "assistant")
+            return;
+        answer = answerText(event.message.content) || answer;
+        costUsd += event.message.usage?.cost?.total ?? 0;
+        if (event.message.stopReason === "error")
+            errors++;
+    }
+    child.stdout.on("data", (chunk) => {
+        buffer += chunk.toString("utf8");
+        let n;
+        while ((n = buffer.indexOf("\n")) >= 0) {
+            consume(buffer.slice(0, n));
+            buffer = buffer.slice(n + 1);
+        }
+    });
+    child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
+    let exitCode;
+    try {
+        exitCode = await new Promise((resolve, reject) => {
+            child.on("error", reject);
+            child.on("close", (code) => resolve(code ?? 1));
+        });
+    }
+    finally {
+        if (imageDir)
+            rmSync(imageDir, { recursive: true, force: true });
+    }
+    consume(buffer);
+    if (exitCode !== 0 || !answer.trim()) {
+        throw new Error(`pi trial ${model} failed: ${stderr.trim().slice(0, 180) || `exit ${exitCode}, empty answer`}`);
+    }
+    return { answer, costUsd, latencyMs: Date.now() - started, errors, sessionId };
+}
+/** Quality uses the same OpenMerit judge; cost and latency come from pi itself. */
+export async function runPiTrial(key, judgeModel, task, rubric, model, entry, cwd = process.cwd(), images = []) {
+    try {
+        const run = await executePiTask(model, task, cwd, images);
+        return await scorePiRun(key, judgeModel, task, rubric, model, entry, run, "pi_trial", images);
+    }
+    catch (e) {
+        const error = String(e);
+        return {
+            point: { model, score: 0, price: entry?.price ?? 0,
+                ts: new Date().toISOString(), source: "pi_trial", why: error.slice(0, 160) },
+            costUsd: 0, error,
+        };
+    }
+}
+export async function scorePiRun(key, judgeModel, task, rubric, model, entry, run, source, images = []) {
+    let score = 0;
+    let why = "judge unavailable";
+    try {
+        const pinned = images.length ? await knownInvoiceScore(task, images, run.answer) : null;
+        const j = pinned ?? (images.length
+            ? await judgeWithImages(key, judgeModel, task, rubric, run.answer, images)
+            : await judge(key, judgeModel, task, rubric, run.answer));
+        score = Math.max(0, Math.min(1, j.score));
+        why = j.why;
+    }
+    catch (e) {
+        why = `judge error: ${String(e).slice(0, 120)}`;
+    }
+    return {
+        point: {
+            model, score, price: entry?.price ?? 0,
+            latencyMs: run.latencyMs, ts: new Date().toISOString(), source,
+            why: `${why}${run.errors ? `; pi errors: ${run.errors}` : ""}`,
+        },
+        costUsd: run.costUsd,
+        sessionId: run.sessionId,
+    };
+}

package/dist/policy.js

@@ -0,0 +1,110 @@
+/** Policy loading, validation, and the swap gate. */
+import { existsSync, readFileSync } from "node:fs";
+export const DEFAULT_POLICY = {
+    version: 1,
+    mode: "recommend",
+    auto_apply: {
+        enabled: false,
+        min_score_gain: 0.1,
+        max_price_ratio: 1.5,
+        require_frontier: true,
+    },
+    budgets: { max_usd_per_trial: 0.25, max_trials_per_day: 20, max_usd_per_day: 5.0 },
+    providers: { allow: ["*"], deny: [] },
+    watch: { catalog_interval_min: 360, traces_interval_sec: 5, trial_interval_min: 30, models_per_task: 3 },
+    fallback: { auto_update: true, min_score: 0.6, apply_on_error: true },
+    judge_model: null,
+    strategist_model: null,
+    max_usd_per_m: 20.0,
+};
+function num(v, fallback) {
+    return typeof v === "number" && Number.isFinite(v) ? v : fallback;
+}
+/** Load a policy file, filling defaults for missing keys. Throws on invalid JSON. */
+export function loadPolicy(path) {
+    if (!existsSync(path))
+        return structuredClone(DEFAULT_POLICY);
+    const raw = JSON.parse(readFileSync(path, "utf8"));
+    const d = DEFAULT_POLICY;
+    return {
+        version: num(raw.version, d.version),
+        mode: raw.mode === "auto" ? "auto" : "recommend",
+        auto_apply: {
+            enabled: Boolean(raw.auto_apply?.enabled ?? d.auto_apply.enabled),
+            min_score_gain: num(raw.auto_apply?.min_score_gain, d.auto_apply.min_score_gain),
+            max_price_ratio: num(raw.auto_apply?.max_price_ratio, d.auto_apply.max_price_ratio),
+            require_frontier: Boolean(raw.auto_apply?.require_frontier ?? d.auto_apply.require_frontier),
+        },
+        budgets: {
+            max_usd_per_trial: num(raw.budgets?.max_usd_per_trial, d.budgets.max_usd_per_trial),
+            max_trials_per_day: num(raw.budgets?.max_trials_per_day, d.budgets.max_trials_per_day),
+            max_usd_per_day: num(raw.budgets?.max_usd_per_day, d.budgets.max_usd_per_day),
+        },
+        providers: {
+            allow: Array.isArray(raw.providers?.allow) ? raw.providers.allow.map(String) : d.providers.allow,
+            deny: Array.isArray(raw.providers?.deny) ? raw.providers.deny.map(String) : d.providers.deny,
+        },
+        watch: {
+            catalog_interval_min: num(raw.watch?.catalog_interval_min, d.watch.catalog_interval_min),
+            traces_interval_sec: num(raw.watch?.traces_interval_sec, d.watch.traces_interval_sec),
+            trial_interval_min: num(raw.watch?.trial_interval_min, d.watch.trial_interval_min),
+            models_per_task: num(raw.watch?.models_per_task, d.watch.models_per_task),
+        },
+        fallback: {
+            auto_update: Boolean(raw.fallback?.auto_update ?? d.fallback.auto_update),
+            min_score: num(raw.fallback?.min_score, d.fallback.min_score),
+            apply_on_error: Boolean(raw.fallback?.apply_on_error ?? d.fallback.apply_on_error),
+        },
+        judge_model: raw.judge_model ?? null,
+        strategist_model: raw.strategist_model ?? null,
+        max_usd_per_m: num(raw.max_usd_per_m, d.max_usd_per_m),
+    };
+}
+/** Is a model id allowed by the provider allow/deny lists? ("*" matches all; "vendor/*" matches a vendor.) */
+export function providerAllowed(policy, modelId) {
+    const vendor = modelId.split("/")[0];
+    const match = (pat) => pat === "*" || pat === modelId || (pat.endsWith("/*") && pat.slice(0, -2) === vendor);
+    if (policy.providers.deny.some(match))
+        return false;
+    return policy.providers.allow.some(match);
+}
+/**
+ * The swap gate: decide whether a recommendation may be applied without
+ * human approval. Always returns the reasons for the decision so the
+ * recommendation can explain itself.
+ */
+export function gate(policy, input) {
+    const reasons = [];
+    if (!input.providerOk) {
+        reasons.push("provider not allowed by policy");
+        return { autoApply: false, reasons };
+    }
+    if (input.baselineMeasured === false) {
+        reasons.push("current model has no measured baseline");
+        return { autoApply: false, reasons };
+    }
+    if (input.sessionBound === false) {
+        reasons.push("recommendation is not bound to a pi session");
+        return { autoApply: false, reasons };
+    }
+    if (policy.mode !== "auto" || !policy.auto_apply.enabled) {
+        reasons.push(`policy mode is "${policy.mode}" (auto_apply ${policy.auto_apply.enabled ? "enabled" : "disabled"})`);
+        return { autoApply: false, reasons };
+    }
+    let ok = true;
+    if (input.scoreGain < policy.auto_apply.min_score_gain) {
+        reasons.push(`score gain ${input.scoreGain.toFixed(3)} < min ${policy.auto_apply.min_score_gain}`);
+        ok = false;
+    }
+    if (input.priceRatio > policy.auto_apply.max_price_ratio) {
+        reasons.push(`price ratio ${input.priceRatio.toFixed(2)}x > max ${policy.auto_apply.max_price_ratio}x`);
+        ok = false;
+    }
+    if (policy.auto_apply.require_frontier && !input.onFrontier) {
+        reasons.push("candidate is not on the pareto frontier");
+        ok = false;
+    }
+    if (ok)
+        reasons.unshift("all auto-apply thresholds met");
+    return { autoApply: ok, reasons };
+}

package/dist/recommend.js

@@ -0,0 +1,63 @@
+/** Build policy-gated recommendations from a task's frontier + the harness's current model. */
+import { createHash } from "node:crypto";
+import { paretoFrontier, pickBest, pickFallback } from "./frontier.js";
+import { gate, providerAllowed } from "./policy.js";
+export function buildRecommendation(taskKey, taskLabel, currentModel, points, policy, sessionFile = null) {
+    const scored = points.filter((p) => p.score > 0);
+    if (scored.length === 0)
+        return null;
+    const best = pickBest(scored);
+    if (!best)
+        return null;
+    if (currentModel && best.model === currentModel)
+        return null; // already on the best model
+    const current = currentModel ? scored.find((p) => p.model === currentModel) : undefined;
+    const scoreGain = current ? best.score - current.score : 0;
+    // pickBest may choose a cheaper or faster model at equal quality.
+    if (current && scoreGain < 0)
+        return null;
+    const priceRatio = current && current.price > 0 ? best.price / current.price : 1;
+    const frontierModels = paretoFrontier(scored).map((p) => p.model);
+    const onFrontier = frontierModels.includes(best.model);
+    const fb = pickFallback(scored, best, policy.fallback.min_score);
+    const decision = gate(policy, {
+        scoreGain,
+        priceRatio,
+        onFrontier,
+        providerOk: providerAllowed(policy, best.model),
+        baselineMeasured: !!current,
+        sessionBound: !!sessionFile,
+    });
+    const reason = `${best.model} scores ${best.score.toFixed(2)} vs ` +
+        (current ? `${current.score.toFixed(2)} for ${current.model}` : "no baseline measured") +
+        ` (${scoreGain > 0 ? `quality gain ${scoreGain.toFixed(2)}` : "equal measured quality"}), at $${best.price.toFixed(2)}/M` +
+        (current ? ` (${priceRatio.toFixed(2)}x current price)` : "") +
+        (best.why ? `. Judge: ${best.why}` : "");
+    return {
+        id: createHash("sha1")
+            .update(`${taskKey}:${best.model}:${Date.now()}`)
+            .digest("hex")
+            .slice(0, 10),
+        ts: new Date().toISOString(),
+        taskKey,
+        taskLabel,
+        sessionFile,
+        currentModel,
+        recommended: { model: best.model, reason },
+        fallback: fb
+            ? {
+                model: fb.model,
+                reason: `score ${fb.score.toFixed(2)} at $${fb.price.toFixed(2)}/M` +
+                    (fb.latencyMs ? `, ~${Math.round(fb.latencyMs)}ms` : ""),
+            }
+            : null,
+        evidence: {
+            scoreGain: Math.round(scoreGain * 1000) / 1000,
+            priceRatio: Math.round(priceRatio * 100) / 100,
+            onFrontier,
+            trials: scored.length,
+        },
+        policy: decision,
+        status: "pending",
+    };
+}

package/dist/store.js

@@ -0,0 +1,55 @@
+/** State directory layout + JSONL persistence. All state lives under OPENMERIT_HOME (default ~/.openmerit). */
+import { createHash } from "node:crypto";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+export function stateDir() {
+    const dir = process.env.OPENMERIT_HOME ?? join(homedir(), ".openmerit");
+    if (!existsSync(dir))
+        mkdirSync(dir, { recursive: true });
+    return dir;
+}
+export const paths = {
+    policy: () => join(stateDir(), "policy.json"),
+    trials: () => join(stateDir(), "trials.jsonl"),
+    recommendations: () => join(stateDir(), "recommendations.jsonl"),
+    catalogSnapshot: () => join(stateDir(), "catalog", "snapshot.json"),
+    candidates: () => join(stateDir(), "catalog", "candidates.json"),
+    benchmarksDigest: () => join(stateDir(), "benchmarks", "digest.json"),
+    traceCursor: () => join(stateDir(), "traces", "cursor.json"),
+    observations: () => join(stateDir(), "traces", "observations.jsonl"),
+    harnessState: () => join(stateDir(), "harness-state.json"),
+    ledger: () => join(stateDir(), "ledger.json"),
+    watchProcessed: () => join(stateDir(), "watch", "processed.json"),
+    sessionJob: (marker) => join(stateDir(), "watch", "jobs", sha1(marker) + ".json"),
+    envFile: () => join(stateDir(), ".env"),
+};
+export function sha1(text) {
+    return createHash("sha1").update(text).digest("hex").slice(0, 12);
+}
+/** Stable task key from a task statement. */
+export function taskKey(taskText) {
+    const normalized = taskText.toLowerCase().replace(/\s+/g, " ").trim();
+    return sha1(normalized);
+}
+export function appendJsonl(file, obj) {
+    mkdirSync(join(file, ".."), { recursive: true });
+    appendFileSync(file, JSON.stringify(obj) + "\n");
+}
+export function readJsonl(file) {
+    if (!existsSync(file))
+        return [];
+    return readFileSync(file, "utf8")
+        .split("\n")
+        .filter((l) => l.trim())
+        .map((l) => JSON.parse(l));
+}
+export function readJson(file, fallback) {
+    if (!existsSync(file))
+        return fallback;
+    return JSON.parse(readFileSync(file, "utf8"));
+}
+export function writeJson(file, obj) {
+    mkdirSync(join(file, ".."), { recursive: true });
+    writeFileSync(file, JSON.stringify(obj, null, 2) + "\n");
+}

package/dist/strategist.js

@@ -0,0 +1,64 @@
+/** Strategist: pick the next untried model to trial, benchmark-aware. */
+import { chat } from "./llm.js";
+import { parseObj } from "./judge.js";
+export const STRAT_PREFS = [
+    "anthropic/claude-3.7-sonnet",
+    "anthropic/claude-3.5-sonnet",
+    "openai/gpt-4o",
+];
+const STRAT_PROMPT = `We are finding the best OpenRouter model for a task via iterative trials.
+TASK: {task}
+RUBRIC: {rubric}
+RELEVANT PUBLIC BENCHMARKS FOR THIS TASK: {benchmarks}
+RESULTS SO FAR (model=score): {results}
+Pick ONE untried model from the catalog below that is likely to do well on this
+task, based on the benchmarks relevant to it. Balance quality against price so
+a pareto frontier emerges.
+Return ONLY json: {{"next_model": "<catalog id>", "why": "<one line>"}}
+CATALOG (id | blended $/1M tokens | ctx):
+{catalog}`;
+/** Pick the next model to trial, or null when the catalog is exhausted. */
+export async function pickNext(key, stratModel, task, rubric, benchmarks, results, cat, tried, maxPrice, failedVendors, extraCandidates) {
+    const ok = (c) => c.price <= maxPrice &&
+        !tried.has(c.id) &&
+        c.ctx >= 4096 &&
+        !failedVendors.has(c.id.split("/")[0]);
+    // Benchmark-shortlisted models get priority in the listing.
+    const pool = [...cat.values()].filter(ok);
+    if (pool.length === 0)
+        return null;
+    const priority = new Set(extraCandidates ?? []);
+    pool.sort((a, b) => {
+        const pa = priority.has(a.id) ? 0 : 1;
+        const pb = priority.has(b.id) ? 0 : 1;
+        return pa - pb || a.price - b.price;
+    });
+    const lines = pool
+        .map((c) => `${c.id} | ${c.price.toFixed(2)} | ${c.ctx}${priority.has(c.id) ? " | BENCHMARK" : ""}`)
+        .join("\n");
+    const res = results
+        .map((r) => `${r.model}=${r.score.toFixed(2)}`)
+        .join("  ") || "none yet";
+    const { content } = await chat(key, stratModel, STRAT_PROMPT.replace("{task}", task)
+        .replace("{rubric}", rubric)
+        .replace("{benchmarks}", benchmarks.join(", "))
+        .replace("{results}", res)
+        .replace("{catalog}", lines), 1024, 0.2);
+    let pick = null;
+    let why = "unparseable strategist output";
+    try {
+        const obj = parseObj(content);
+        pick = typeof obj.next_model === "string" ? obj.next_model : null;
+        why = String(obj.why ?? "");
+    }
+    catch {
+        /* fall through to cheap pick */
+    }
+    if (pick) {
+        const entry = cat.get(pick);
+        if (entry && ok(entry))
+            return { model: pick, why };
+    }
+    const cheapest = pool[0];
+    return { model: cheapest.id, why: `fallback pick: cheapest untried (strategist pick ${pick ?? "none"} unusable)` };
+}

package/dist/task-input.js

@@ -0,0 +1,30 @@
+/** A pi text task may carry images embedded in its saved session message. */
+import { createHash } from "node:crypto";
+import { taskKey } from "./store.js";
+const SUPPORTED = new Set(["image/jpeg", "image/png", "image/webp", "image/gif"]);
+export function sessionImages(content) {
+    if (!Array.isArray(content))
+        return [];
+    const images = [];
+    for (const part of content) {
+        if (!part || typeof part !== "object")
+            return null;
+        if (part.type === "text")
+            continue;
+        if (part.type !== "image" || typeof part.data !== "string" ||
+            typeof part.mimeType !== "string" || !SUPPORTED.has(part.mimeType) ||
+            !/^[A-Za-z0-9+/]+={0,2}$/.test(part.data))
+            return null;
+        images.push({ type: "image", data: part.data, mimeType: part.mimeType });
+    }
+    return images;
+}
+/** Text-only keys remain compatible; image bytes distinguish same-prompt documents. */
+export function taskInputKey(text, images) {
+    if (images.length === 0)
+        return taskKey(text);
+    const normalized = text.toLowerCase().replace(/\s+/g, " ").trim();
+    const hashes = images.map((image) => `${image.mimeType}:` +
+        createHash("sha256").update(Buffer.from(image.data, "base64")).digest("hex"));
+    return taskKey(`${normalized}\nimages:${hashes.join(",")}`);
+}

package/dist/traces.js

@@ -0,0 +1,123 @@
+/**
+ * Trace ingestion: read pi session files (JSONL entry trees) and extract
+ * per-task observations (model, cost, tokens, latency, errors). Incremental
+ * via a per-file byte-offset cursor, so the background track never re-reads.
+ */
+import { readdirSync, readFileSync, existsSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { paths, readJson, taskKey, writeJson } from "./store.js";
+export function defaultSessionsDir() {
+    return join(homedir(), ".pi", "agent", "sessions");
+}
+function listSessionFiles(dir) {
+    if (!existsSync(dir))
+        return [];
+    const out = [];
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const p = join(dir, entry.name);
+        if (entry.isDirectory())
+            out.push(...listSessionFiles(p));
+        else if (entry.name.endsWith(".jsonl"))
+            out.push(p);
+    }
+    return out;
+}
+function textOf(content) {
+    if (typeof content === "string")
+        return content;
+    if (Array.isArray(content)) {
+        return content
+            .map((b) => (b && typeof b === "object" && b.type === "text"
+            ? String(b.text ?? "")
+            : ""))
+            .filter(Boolean)
+            .join(" ");
+    }
+    return "";
+}
+/** Extract observations from the entries of one session file. */
+export function observationsFromEntries(entries, sessionFile) {
+    const obs = [];
+    let current = null;
+    for (const e of entries) {
+        if (e.type !== "message" || !e.message)
+            continue;
+        const m = e.message;
+        if (m.role === "user") {
+            if (current)
+                obs.push(current);
+            const text = textOf(m.content);
+            if (!text.trim()) {
+                current = null;
+                continue;
+            }
+            current = {
+                taskKey: taskKey(text),
+                taskLabel: text.replace(/\s+/g, " ").trim().slice(0, 120),
+                model: null,
+                costUsd: 0,
+                tokens: 0,
+                latencyMs: null,
+                errors: 0,
+                ts: e.timestamp ?? new Date().toISOString(),
+                sessionFile,
+            };
+            continue;
+        }
+        if (!current)
+            continue;
+        if (m.role === "assistant") {
+            if (!current.model && m.provider && m.model)
+                current.model = `${m.provider}/${m.model}`;
+            current.costUsd += m.usage?.cost?.total ?? 0;
+            current.tokens += m.usage?.totalTokens ?? 0;
+            if (m.stopReason === "error")
+                current.errors += 1;
+            if (typeof m.timestamp === "number") {
+                const start = Date.parse(current.ts);
+                if (!Number.isNaN(start))
+                    current.latencyMs = Math.max(0, m.timestamp - start);
+            }
+        }
+        else if (m.role === "toolResult") {
+            const tr = m;
+            if (tr.isError)
+                current.errors += 1;
+        }
+    }
+    if (current)
+        obs.push(current);
+    return obs;
+}
+/** Read new bytes of every session file since the last run; returns fresh observations. */
+export function ingestNewTraces(sessionsDir = defaultSessionsDir()) {
+    const cursor = readJson(paths.traceCursor(), { offsets: {} });
+    const fresh = [];
+    for (const file of listSessionFiles(sessionsDir)) {
+        const size = statSync(file).size;
+        const from = cursor.offsets[file] ?? 0;
+        if (size <= from)
+            continue;
+        const buf = readFileSync(file);
+        const chunk = buf.subarray(from).toString("utf8");
+        // Drop a trailing partial line; it will be re-read next tick.
+        const lastNl = chunk.lastIndexOf("\n");
+        const complete = lastNl >= 0 ? chunk.slice(0, lastNl + 1) : "";
+        const entries = [];
+        for (const line of complete.split("\n")) {
+            if (!line.trim())
+                continue;
+            try {
+                entries.push(JSON.parse(line));
+            }
+            catch {
+                /* skip corrupt line */
+            }
+        }
+        fresh.push(...observationsFromEntries(entries, file));
+        cursor.offsets[file] = from + Buffer.byteLength(complete);
+    }
+    writeJson(paths.traceCursor(), cursor);
+    return fresh;
+}