openmerit 0.1.3 → 0.1.5

package/dist/trials.js

@@ -1,140 +0,0 @@
-/** Shadow trials: run candidate models against observed/declared tasks on the background track. */
-import { directChatClient } from "./llm.js";
-import { judge, parseObj } from "./judge.js";
-import { paths, readJsonReport, writeJson } from "./store.js";
-function today() {
-    return new Date().toISOString().slice(0, 10);
-}
-export function budgetOk(policy) {
-    const report = readJsonReport(paths.ledger(), { date: today(), trials: 0, usd: 0 });
-    if (!report.valid)
-        return { ok: false, reason: "trial ledger is malformed; run `openmerit doctor`" };
-    const ledger = report.value;
-    if (ledger.date !== today())
-        return { ok: true }; // new day resets
-    if (ledger.trials >= policy.budgets.max_trials_per_day)
-        return { ok: false, reason: `daily trial cap reached (${policy.budgets.max_trials_per_day})` };
-    if (ledger.usd >= policy.budgets.max_usd_per_day)
-        return { ok: false, reason: `daily budget exhausted ($${policy.budgets.max_usd_per_day})` };
-    return { ok: true };
-}
-/** Conservative admission estimate for one candidate answer (rough input tokens + 4K output). */
-export function trialBudgetOk(policy, entry, inputChars) {
-    if (entry.priceKnown === false)
-        return { ok: false, reason: "route price is unknown" };
-    const inputTokens = Math.ceil(inputChars / 4);
-    const outputTokens = Math.min(entry.route?.maxTokens ?? 4096, 4096);
-    const estimatedUsd = entry.pp * inputTokens + entry.pc * outputTokens;
-    if (estimatedUsd > policy.budgets.max_usd_per_trial)
-        return {
-            ok: false,
-            estimatedUsd,
-            reason: `estimated trial cost $${estimatedUsd.toFixed(4)} exceeds $${policy.budgets.max_usd_per_trial.toFixed(4)} limit`,
-        };
-    return { ok: true, estimatedUsd };
-}
-export function recordTrialSpend(usd) {
-    const report = readJsonReport(paths.ledger(), { date: today(), trials: 0, usd: 0 });
-    if (!report.valid)
-        throw new Error("trial ledger is malformed; refusing to reset spend");
-    let ledger = report.value;
-    if (ledger.date !== today())
-        ledger = { date: today(), trials: 0, usd: 0 };
-    ledger.trials += 1;
-    ledger.usd = Math.round((ledger.usd + usd) * 1e6) / 1e6;
-    writeJson(paths.ledger(), ledger);
-}
-/** Record non-candidate merit-loop spend without consuming a trial slot. */
-export function recordMeritSpend(usd) {
-    if (!(usd > 0))
-        return;
-    const report = readJsonReport(paths.ledger(), { date: today(), trials: 0, usd: 0 });
-    if (!report.valid)
-        throw new Error("trial ledger is malformed; refusing to reset spend");
-    let ledger = report.value;
-    if (ledger.date !== today())
-        ledger = { date: today(), trials: 0, usd: 0 };
-    ledger.usd = Math.round((ledger.usd + usd) * 1e6) / 1e6;
-    writeJson(paths.ledger(), ledger);
-}
-const RUBRIC_PROMPT = `Write a compact grading rubric for answers to the task below.
-It must list the concrete criteria for a score of 1.0 and how to deduct.
-TASK: {task}
-Return ONLY json: {{"rubric": "<text>"}}`;
-/** Derive (and cache) a grading rubric for a task that came from traces. */
-export async function ensureRubric(key, judgeModel, task, cache, client = directChatClient(key)) {
-    const hit = cache.get(task);
-    if (hit)
-        return hit;
-    const { content } = await client.chat(judgeModel, RUBRIC_PROMPT.replace("{task}", task.slice(0, 8000)), 1024, 0);
-    let rubric = "Score 1.0 for a fully correct, complete, usable answer; deduct for errors, omissions, or unusable output.";
-    try {
-        const obj = parseObj(content);
-        if (typeof obj.rubric === "string" && obj.rubric.trim())
-            rubric = obj.rubric;
-    }
-    catch {
-        /* keep default */
-    }
-    cache.set(task, rubric);
-    return rubric;
-}
-/** Run one shadow trial: generate with `model`, judge the output, return a frontier point + cost. */
-export async function runTrial(key, judgeModel, task, rubric, model, entry, client = directChatClient(key)) {
-    const started = Date.now();
-    let gen;
-    let usage = {};
-    try {
-        const r = await client.chat(entry?.route ?? model, task, 4096, 0.2);
-        gen = r.content;
-        usage = r.usage;
-    }
-    catch (e) {
-        return {
-            point: {
-                schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
-                priceKnown: entry?.priceKnown ?? !!entry,
-                ts: new Date().toISOString(), source: "shadow_trial", why: String(e).slice(0, 160),
-            },
-            costUsd: 0,
-            error: String(e),
-        };
-    }
-    const latencyMs = Date.now() - started;
-    const costUsd = (entry?.pp ?? 0) * (usage.prompt_tokens ?? 0) + (entry?.pc ?? 0) * (usage.completion_tokens ?? 0);
-    if (!gen.trim()) {
-        return {
-            point: {
-                schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
-                priceKnown: entry?.priceKnown ?? !!entry, latencyMs,
-                ts: new Date().toISOString(), source: "shadow_trial", why: "empty response",
-            },
-            costUsd,
-        };
-    }
-    let score = 0;
-    let why = "judge error";
-    try {
-        const j = await judge(key, judgeModel, task, rubric, gen, client);
-        score = j.score;
-        why = j.why;
-    }
-    catch (e) {
-        why = `judge error: ${String(e).slice(0, 120)}`;
-    }
-    return {
-        point: {
-            schemaVersion: 1,
-            model,
-            route: entry?.route,
-            score,
-            price: entry?.price ?? 0,
-            priceKnown: entry?.priceKnown ?? !!entry,
-            latencyMs,
-            ts: new Date().toISOString(),
-            source: "shadow_trial",
-            why,
-        },
-        costUsd,
-    };
-}

package/dist/types.js

@@ -1,2 +0,0 @@
-/** Shared types for the openmerit core. */
-export {};

package/examples/invoice-prompt.txt

@@ -1,19 +0,0 @@
-Extract the invoice into the required JSON schema using only the image.
-
-Return a JSON object with exactly these fields: supplier_name, customer_name,
-invoice_number, po_number, document_date, due_date, document_currency,
-document_total_net, document_total_tax, document_total_amount, and line_items.
-Scalar names, identifiers, dates, and currency are strings or null. Monetary
-totals are JSON numbers or null. line_items is an array of objects, each with
-exactly description (string or null), quantity (number or null), unit_price
-(number or null), and amount (number or null). Do not add any other fields.
-
-Rules:
-- Copy names, identifiers, and descriptions exactly as printed.
-- Return dates as YYYY-MM-DD, regardless of the printed date format.
-- Return currency as its three-letter ISO code.
-- Return monetary values and quantities as JSON numbers without currency symbols or grouping separators.
-- Preserve line-item order and include every printed line item exactly once.
-- Use null only when a scalar field is not present. Never infer a missing value.
-- document_total_net is the subtotal/net amount before tax; document_total_amount is the final amount including tax.
-- Return only the schema-conforming JSON object.

package/examples/task.example.json

@@ -1,7 +0,0 @@
-{
-  "task": "Write a Python function `def word_ladder(begin, end, words)` that returns the SHORTEST transformation sequence from `begin` to `end`, where consecutive words differ by exactly one character and every intermediate word must be in `words`. Return [] if no path exists. If begin == end return [begin]. All words are the same length, lowercase a-z. Output only runnable code, no explanation.",
-  "eval": "Score 1.0 requires ALL of: (a) algorithm guaranteed to find a shortest path (BFS or equivalent, NOT DFS/greedy); (b) returned path starts with begin and ends with end with all intermediates in words; (c) returns [] when impossible; (d) begin==end returns [begin]; (e) begin need not be in words; (f) syntactically valid, runnable Python, no external imports, function named word_ladder; (g) output contains only code. Deduct ~0.2 per missing criterion; non-shortest-path algorithms score at most 0.5.",
-  "initial_model": "openai/gpt-4o-mini",
-  "initial_route": "openai:gpt-4o-mini",
-  "max_usd_per_m": 20
-}