margin-meter 0.2.0 → 0.4.0

package/dist/langsmith.js

@@ -0,0 +1,415 @@
+/**
+ * margin-meter/langsmith — turn a LangSmith `evaluate()` result into outcomes.
+ *
+ * The TypeScript twin of `sdk/python/margin_meter/langsmith.py`. LangSmith is the
+ * eval/observability layer of the LangChain ecosystem — one of the largest
+ * install bases in production LLM apps, and it ships a first-class JS/TS SDK, so a
+ * LangSmith customer is squarely in the TypeScript toolchain. `evaluate()`
+ * attaches one or more feedback scores to each example; a customer serializes that
+ * result — iterating `ExperimentResults` (each row carries
+ * `evaluation_results.results`) or via `result.to_pandas().to_dict(...)` (feedback
+ * lands in `feedback.<key>` columns) — and both are a match for
+ * `POST /api/ingest/outcomes` (`docs/INTEGRATION_LADDER.md` rung 2):
+ *
+ *     import { MarginMeter, langsmith } from "margin-meter";
+ *     const meter = new MarginMeter();
+ *     const data = JSON.parse(await readFile("ls.json", "utf8"));
+ *     await langsmith.recordResults(meter, data, "my-app", { threshold: 0.7 });
+ *
+ * The mapping is identical to the Python adapter's, on purpose — one LangSmith
+ * row → one outcome:
+ *
+ *  - `qualityScore`  ← the mean of the row's numeric feedback scores in [0, 1]
+ *    (booleans count as 1.0 / 0.0). LangSmith attaches a score per evaluator, not
+ *    one per row, so they are aggregated — the same rule the DeepEval, Braintrust
+ *    and Ragas adapters apply. Both serializations are read: the object form
+ *    (`evaluation_results.results = [{key, score}]`) and the pandas form
+ *    (`feedback.<key>` columns).
+ *  - `passed`        ← LangSmith has NO single pass/fail verdict, so it is derived
+ *    HONESTLY, never fabricated: (1) `threshold` given → mean ≥ threshold; (2) no
+ *    threshold but every feedback is binary (0.0/1.0) → every evaluator passed;
+ *    (3) no threshold and continuous scores → the row is SKIPPED with a warning.
+ *  - `qualityMethod`/`graderKind` ← DERIVED from the evaluator KEYS, never assumed:
+ *    a known LangChain LLM-judge evaluator (`correctness`, `qa`, `criteria`,
+ *    `score_string`, …) → `llm_judge`/`judge`; a known code evaluator
+ *    (`string_distance`, `exact_match`, `json_validity`, …) → `ground_truth`/`code`;
+ *    anything else → `unknown`/`unknown` (the weaker-claim-wins honesty rule).
+ *    Most LangSmith keys are user-named, so `unknown` is the honest common default.
+ *  - `judgeModelId`  ← an opt-in `margin_judge_model` field on the ROW (LangSmith
+ *    rows have no metadata envelope), ridden only when the grade is an LLM judge
+ *    (rung 6 provenance). A bare `model` field is NOT used.
+ *  - `link`          ← the row's `example_id`/`id`/`run_id`, else
+ *    `langsmith:<index>` — never the input question (MAR-1548).
+ *  - `valueUsd`      ← opt-in `margin_value` on the ROW (rung 8); absent is the
+ *    cost-per-outcome default, never 0.
+ *
+ * Same two promises as the other adapters: it consumes output the customer already
+ * produces (no new harness), and it never fabricates and is fail-safe.
+ */
+// Grading provenance strings — mirror `src/margin/grading.py`, kept as literals.
+const GROUND_TRUTH = "ground_truth";
+const LLM_JUDGE = "llm_judge";
+const UNKNOWN_METHOD = "unknown";
+const GRADER_CODE = "code";
+const GRADER_JUDGE = "judge";
+const GRADER_UNKNOWN = "unknown";
+const MAX_LINK_LEN = 512;
+const MAX_JUDGE_ID_LEN = 128;
+const VALUE_KEY = "margin_value";
+// The explicit opt-in key for the judging model. A bare `model` is NOT used: on a
+// LangSmith row it is the target model under evaluation, not the judge, and
+// labelling it `judgeModelId` would misstate the provenance (rung 6).
+const JUDGE_MODEL_KEY = "margin_judge_model";
+// Mirror `src/margin/ingest.MAX_VALUE_USD`. A value above this is rejected by
+// ingest and would take the whole row down with it, so the adapter drops the
+// value and keeps the row — the same treatment a malformed value gets.
+const MAX_VALUE_USD = 10_000_000.0;
+// Model ids that name no real model. Normalised lower-case for the check.
+const NON_MODELS = new Set(["", "n/a", "na", "none", "null"]);
+// A finite DECIMAL number — the only string shape accepted for a value tag.
+// Deliberately narrower than JS `Number()` (which parses `0x10`/`Infinity`/`""`):
+// this is stricter than Python `float()` in the honest direction — it also
+// rejects `inf`/`nan`, which Python would coerce into a garbage numerator.
+const DECIMAL_RE = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
+// Feedback columns in the pandas serialization are prefixed with this.
+const FEEDBACK_PREFIX = "feedback.";
+/** LangChain's built-in LLM-as-judge string evaluators, plus common judge keys.
+ *  Normalised lower-case, non-alphanumerics stripped so `score_string`/`scoreString`
+ *  collapse together. */
+const MODEL_EVALUATORS = new Set([
+    "correctness",
+    "qa",
+    "cotqa",
+    "contextqa",
+    "criteria",
+    "labeledcriteria",
+    "scorestring",
+    "labeledscorestring",
+    "embeddingdistance",
+    "helpfulness",
+    "relevance",
+    "coherence",
+    "conciseness",
+    "harmfulness",
+    "maliciousness",
+    "faithfulness",
+    "hallucination",
+]);
+/** Deterministic, code-graded evaluators (no model in the grading loop). */
+const CODE_EVALUATORS = new Set([
+    "stringdistance",
+    "exactmatch",
+    "regex",
+    "jsonvalidity",
+    "jsonequality",
+    "jsonedit",
+    "jsonschema",
+    "levenshtein",
+]);
+const defaultWarn = (message) => {
+    const p = globalThis
+        .process;
+    if (p && typeof p.emitWarning === "function")
+        p.emitWarning(message);
+    else
+        console.warn(message);
+};
+function isObject(x) {
+    return typeof x === "object" && x !== null && !Array.isArray(x);
+}
+/** Normalise an evaluator key for set membership: lower-case, keep only
+ *  alphanumerics (so `string_distance`/`String Distance` collapse), strip a
+ *  leading `not` negation. Returns undefined for a missing/blank name. */
+export function normKey(raw) {
+    if (typeof raw !== "string")
+        return undefined;
+    let t = raw
+        .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9]/g, "");
+    if (!t)
+        return undefined;
+    if (t.startsWith("not") && t !== "not")
+        t = t.slice(3);
+    return t || undefined;
+}
+/** A score coerced to [0, 1]: a bool becomes 1.0/0.0, a number is kept when in
+ *  range, anything else (or out of range) is undefined (dropped). */
+function numInRange(value) {
+    if (typeof value === "boolean")
+        return value ? 1.0 : 0.0;
+    if (typeof value === "number" && !Number.isNaN(value)) {
+        if (value >= 0.0 && value <= 1.0)
+            return value;
+    }
+    return undefined;
+}
+/** The per-evaluator feedback scores of a row, as `{ [key]: score }`.
+ *
+ *  Reads both real serializations: the object form
+ *  (`evaluation_results.results = [{key, score}]`, or a bare top-level `results`
+ *  list) and the pandas form (`feedback.<key>` columns). A null/non-numeric/
+ *  out-of-[0,1] score is dropped so the ingest layer never rejects the row for a
+ *  bad decimal. */
+export function feedbackScores(row) {
+    const out = {};
+    if (!isObject(row))
+        return out;
+    // Object form.
+    let results = undefined;
+    const er = row.evaluation_results;
+    if (isObject(er))
+        results = er.results;
+    // Python tests `results is None`, which is true both when `evaluation_results`
+    // was absent AND when its `results` was explicitly null — so a null there must
+    // also fall through to a top-level `results` list, not just an absent one.
+    if ((results === undefined || results === null) && Array.isArray(row.results))
+        results = row.results;
+    if (Array.isArray(results)) {
+        for (let i = 0; i < results.length; i++) {
+            const item = results[i];
+            if (!isObject(item))
+                continue;
+            const score = numInRange(item.score);
+            if (score === undefined)
+                continue;
+            const key = typeof item.key === "string" && item.key.trim() ? item.key : `eval_${i}`;
+            out[key] = score;
+        }
+    }
+    // Pandas form: feedback.<key> columns.
+    for (const [name, value] of Object.entries(row)) {
+        if (!name.startsWith(FEEDBACK_PREFIX))
+            continue;
+        const score = numInRange(value);
+        if (score === undefined)
+            continue;
+        out[name.slice(FEEDBACK_PREFIX.length) || name] = score;
+    }
+    return out;
+}
+/** Derive `(qualityMethod, graderKind)` from the evaluator keys behind a row,
+ *  weaker-claim-wins:
+ *   - any model evaluator → llm_judge/judge
+ *   - only recognised code evaluators → ground_truth/code
+ *   - anything else → unknown/unknown */
+export function evaluatorSignature(keys) {
+    let hasModel = false;
+    let hasCode = false;
+    let hasUnknown = false;
+    for (const raw of keys) {
+        const t = normKey(raw);
+        if (t === undefined)
+            continue;
+        if (MODEL_EVALUATORS.has(t))
+            hasModel = true;
+        else if (CODE_EVALUATORS.has(t))
+            hasCode = true;
+        else
+            hasUnknown = true;
+    }
+    if (hasModel)
+        return { qualityMethod: LLM_JUDGE, graderKind: GRADER_JUDGE };
+    if (hasCode && !hasUnknown)
+        return { qualityMethod: GROUND_TRUTH, graderKind: GRADER_CODE };
+    return { qualityMethod: UNKNOWN_METHOD, graderKind: GRADER_UNKNOWN };
+}
+/** The judging model, from an explicit `margin_judge_model` opt-in on the row, or
+ *  undefined. A bare `model` field is deliberately NOT used. */
+export function judgeModelFromRow(row) {
+    if (!isObject(row))
+        return undefined;
+    const raw = row[JUDGE_MODEL_KEY];
+    if (typeof raw !== "string")
+        return undefined;
+    if (NON_MODELS.has(raw.trim().toLowerCase()))
+        return undefined;
+    return raw.trim().slice(0, MAX_JUDGE_ID_LEN);
+}
+function coerceValue(raw) {
+    if (typeof raw === "boolean")
+        throw new Error(`${VALUE_KEY} must be a number, got ${JSON.stringify(raw)}`);
+    let val;
+    if (typeof raw === "number") {
+        val = raw;
+    }
+    else if (typeof raw === "string") {
+        const s = raw.trim();
+        if (!DECIMAL_RE.test(s))
+            throw new Error(`${VALUE_KEY} must be a number, got ${JSON.stringify(raw)}`);
+        val = Number(s);
+    }
+    else {
+        throw new Error(`${VALUE_KEY} must be a number, got ${JSON.stringify(raw)}`);
+    }
+    if (!Number.isFinite(val))
+        throw new Error(`${VALUE_KEY} must be a finite number, got ${JSON.stringify(raw)}`);
+    // A loss is a real value (MAR-1198): ingest takes any finite amount within +/-MAX.
+    if (Math.abs(val) > MAX_VALUE_USD)
+        throw new Error(`${VALUE_KEY} implausible (|value| > ${MAX_VALUE_USD}), got ${val}`);
+    return val;
+}
+/** Read an opt-in `margin_value` off a row — the value-per-outcome numerator
+ *  (rung 8). Returns the USD amount, or undefined when absent. Throws on a
+ *  malformed/implausible value (a present-but-null value included) so the
+ *  caller can warn and drop it, matching the Python twin. */
+export function valueFromRow(row) {
+    if (!isObject(row))
+        return undefined;
+    if (!(VALUE_KEY in row))
+        return undefined;
+    return coerceValue(row[VALUE_KEY]);
+}
+/** A traceable link back to the example: its id, else the row index.
+ *  ⛔ Never the input question (MAR-1548): /security says no prompt reaches us. */
+export function linkFromRow(row, index) {
+    if (isObject(row)) {
+        for (const key of ["example_id", "id", "run_id"]) {
+            const val = row[key];
+            if (typeof val === "string" && val.trim())
+                return val.trim().slice(0, MAX_LINK_LEN);
+        }
+    }
+    return `langsmith:${index}`;
+}
+/** Derive the pass/fail for a row, or undefined when it cannot be derived
+ *  honestly (no scores, or continuous scores with no threshold). */
+export function passedFromScores(scores, threshold) {
+    const values = Object.values(scores);
+    if (values.length === 0)
+        return undefined;
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    if (threshold !== undefined)
+        return mean >= threshold;
+    if (values.every((v) => v === 0.0 || v === 1.0))
+        return values.every((v) => v === 1.0);
+    return undefined;
+}
+/** Translate one graded LangSmith row into `MarginMeter.recordOutcome` input.
+ *  `judgeModelId` rides only when the grade is an LLM judge and a model was named;
+ *  `valueUsd` rides only when a value was supplied. */
+export function outcomeInput(args) {
+    const { qualityMethod, graderKind } = evaluatorSignature(args.evaluatorKeys);
+    const input = {
+        workflowId: args.workflowId,
+        passed: Boolean(args.passed),
+        qualityScore: Number(args.qualityScore),
+        qualityMethod,
+        graderKind,
+        link: String(args.link).slice(0, MAX_LINK_LEN),
+        isSimulated: Boolean(args.isSimulated),
+    };
+    if (qualityMethod === LLM_JUDGE && args.judgeModelId)
+        input.judgeModelId = args.judgeModelId.slice(0, MAX_JUDGE_ID_LEN);
+    if (args.valueUsd !== undefined)
+        input.valueUsd = Number(args.valueUsd);
+    return input;
+}
+/** Full translation of one LangSmith row, or undefined if it is not a real
+ *  verdict (no feedback score, or an ambiguous continuous score with no
+ *  threshold). A malformed `margin_value` warns and drops the value while keeping
+ *  the pass/fail. */
+export function rowToOutcomeInput(row, opts) {
+    const onWarn = opts.onWarn ?? defaultWarn;
+    const index = opts.index ?? 0;
+    const scores = feedbackScores(row);
+    const passed = passedFromScores(scores, opts.threshold);
+    if (passed === undefined) {
+        if (Object.keys(scores).length > 0) {
+            onWarn(`${linkFromRow(row, index)}: continuous scores and no threshold; ` +
+                "skipped (no verdict to record)");
+        }
+        return undefined;
+    }
+    const values = Object.values(scores);
+    const quality = values.reduce((a, b) => a + b, 0) / values.length;
+    let value;
+    try {
+        value = valueFromRow(row);
+    }
+    catch (exc) {
+        value = undefined;
+        onWarn(`${linkFromRow(row, index)}: ${exc.message}`);
+    }
+    return outcomeInput({
+        workflowId: opts.workflowId,
+        passed,
+        qualityScore: quality,
+        evaluatorKeys: Object.keys(scores),
+        link: linkFromRow(row, index),
+        judgeModelId: judgeModelFromRow(row),
+        valueUsd: value,
+        isSimulated: opts.isSimulated,
+    });
+}
+/** Pull the flat list of per-example rows out of parsed LangSmith output — a bare
+ *  list, or a `{results|rows|records: [...]}` envelope. */
+export function extractRows(data) {
+    if (Array.isArray(data))
+        return data.filter(isObject);
+    if (!isObject(data))
+        return [];
+    for (const key of ["results", "rows", "records"]) {
+        const block = data[key];
+        if (Array.isArray(block))
+            return block.filter(isObject);
+    }
+    return [];
+}
+/** Yield `recordOutcome` inputs for every real verdict in parsed LangSmith output
+ *  (any envelope shape {@link extractRows} understands). */
+export function iterOutcomes(data, opts) {
+    const out = [];
+    const rows = extractRows(data);
+    for (let i = 0; i < rows.length; i++) {
+        const input = rowToOutcomeInput(rows[i], {
+            workflowId: opts.workflowId,
+            index: i,
+            threshold: opts.threshold,
+            isSimulated: opts.isSimulated,
+            onWarn: opts.onWarn,
+        });
+        if (input !== undefined)
+            out.push(input);
+    }
+    return out;
+}
+/** Emit an outcome per verdict in parsed LangSmith output through `meter`.
+ *  Fail-safe: a delivery fault on one row warns and counts an error; it never
+ *  aborts the batch. Non-verdict rows are counted as skipped. */
+export async function recordResults(meter, data, workflowId, opts = {}) {
+    const onWarn = opts.onWarn ?? defaultWarn;
+    const rows = extractRows(data);
+    let recorded = 0;
+    let errors = 0;
+    let skipped = 0;
+    for (let i = 0; i < rows.length; i++) {
+        const input = rowToOutcomeInput(rows[i], {
+            workflowId,
+            index: i,
+            threshold: opts.threshold,
+            isSimulated: opts.isSimulated,
+            onWarn,
+        });
+        if (input === undefined) {
+            skipped++;
+            continue;
+        }
+        try {
+            await meter.recordOutcome(input);
+            recorded++;
+        }
+        catch (exc) {
+            errors++;
+            onWarn(`failed to record outcome for ${input.link}: ${exc.message}`);
+        }
+    }
+    try {
+        await meter.flush();
+    }
+    catch (exc) {
+        onWarn(`flush failed: ${exc.message}`);
+    }
+    return { recorded, errors, skipped };
+}
+//# sourceMappingURL=langsmith.js.map

package/dist/langsmith.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"langsmith.js","sourceRoot":"","sources":["../src/langsmith.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAIH,iFAAiF;AACjF,MAAM,YAAY,GAAG,cAAc,CAAC;AACpC,MAAM,SAAS,GAAG,WAAW,CAAC;AAC9B,MAAM,cAAc,GAAG,SAAS,CAAC;AACjC,MAAM,WAAW,GAAG,MAAM,CAAC;AAC3B,MAAM,YAAY,GAAG,OAAO,CAAC;AAC7B,MAAM,cAAc,GAAG,SAAS,CAAC;AAEjC,MAAM,YAAY,GAAG,GAAG,CAAC;AACzB,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,MAAM,SAAS,GAAG,cAAc,CAAC;AACjC,kFAAkF;AAClF,4EAA4E;AAC5E,sEAAsE;AACtE,MAAM,eAAe,GAAG,oBAAoB,CAAC;AAE7C,8EAA8E;AAC9E,6EAA6E;AAC7E,uEAAuE;AACvE,MAAM,aAAa,GAAG,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,MAAM,UAAU,GAAwB,IAAI,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAEnF,4EAA4E;AAC5E,kFAAkF;AAClF,2EAA2E;AAC3E,2EAA2E;AAC3E,MAAM,UAAU,GAAG,yCAAyC,CAAC;AAE7D,uEAAuE;AACvE,MAAM,eAAe,GAAG,WAAW,CAAC;AAEpC;;yBAEyB;AACzB,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC;IACpD,aAAa;IACb,IAAI;IACJ,OAAO;IACP,WAAW;IACX,UAAU;IACV,iBAAiB;IACjB,aAAa;IACb,oBAAoB;IACpB,mBAAmB;IACnB,aAAa;IACb,WAAW;IACX,WAAW;IACX,aAAa;IACb,aAAa;IACb,eAAe;IACf,cAAc;IACd,eAAe;CAChB,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC;IACnD,gBAAgB;IAChB,YAAY;IACZ,OAAO;IACP,cAAc;IACd,cAAc;IACd,UAAU;IACV,YAAY;IACZ,aAAa;CACd,CAAC,CAAC;AAMH,MAAM,WAAW,GAAW,CAAC,OAAO,EAAE,EAAE;IACtC,MAAM,CAAC,GAAI,UAAkE;SAC1E,OAAO,CAAC;IACX,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,UAAU;QAAE,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;;QAChE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC,CAAC;AAQF,SAAS,QAAQ,CAAC,CAAU;IAC1B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;0EAE0E;AAC1E,MAAM,UAAU,OAAO,CAAC,GAAY;IAClC,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,CAAC,GAAG,GAAG;SACR,IAAI,EAAE;SACN,WAAW,EAAE;SACb,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAC7B,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzB,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,OAAO,CAAC,IAAI,SAAS,CAAC;AACxB,CAAC;AAED;qEACqE;AACrE,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACzD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG;YAAE,OAAO,KAAK,CAAC;IACjD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;mBAMmB;AACnB,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IAC/B,eAAe;IACf,IAAI,OAAO,GAAY,SAAS,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,kBAAkB,CAAC;IAClC,IAAI,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC;IACvC,+EAA+E;IAC/E,+EAA+E;IAC/E,2EAA2E;IAC3E,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;QAC3E,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IACxB,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YAClC,MAAM,GAAG,GACP,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3E,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,uCAAuC;IACvC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC;YAAE,SAAS;QAChD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAClC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC;IAC1D,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;yCAIyC;AACzC,MAAM,UAAU,kBAAkB,CAAC,IAAuB;IACxD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QAC9B,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,GAAG,IAAI,CAAC;aACxC,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,IAAI,CAAC;;YAC3C,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IACD,IAAI,QAAQ;QAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;IAC5E,IAAI,OAAO,IAAI,CAAC,UAAU;QACxB,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;IAClE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC;AACvE,CAAC;AAED;gEACgE;AAChE,MAAM,UAAU,iBAAiB,CAAC,GAAY;IAC5C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACrC,MAAM,GAAG,GAAG,GAAG,CAAC,eAAe,CAAC,CAAC;IACjC,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/D,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,OAAO,GAAG,KAAK,SAAS;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/E,IAAI,GAAW,CAAC;IAChB,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,GAAG,GAAG,GAAG,CAAC;IACZ,CAAC;SAAM,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/E,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,iCAAiC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtF,mFAAmF;IACnF,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,aAAa;QAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,2BAA2B,aAAa,UAAU,GAAG,EAAE,CAAC,CAAC;IACvF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;6DAG6D;AAC7D,MAAM,UAAU,YAAY,CAAC,GAAY;IACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACrC,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AACrC,CAAC;AAED;kFACkF;AAClF,MAAM,UAAU,WAAW,CAAC,GAAY,EAAE,KAAa;IACrD,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,KAAK,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;YACjD,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE;gBAAE,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IACD,OAAO,aAAa,KAAK,EAAE,CAAC;AAC9B,CAAC;AAED;oEACoE;AACpE,MAAM,UAAU,gBAAgB,CAC9B,MAA8B,EAC9B,SAA6B;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/D,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,IAAI,IAAI,SAAS,CAAC;IACtD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;IACvF,OAAO,SAAS,CAAC;AACnB,CAAC;AAcD;;uDAEuD;AACvD,MAAM,UAAU,YAAY,CAAC,IAAsB;IACjD,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,kBAAkB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC7E,MAAM,KAAK,GAAuB;QAChC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACvC,aAAa;QACb,UAAU;QACV,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC;QAC9C,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;KACvC,CAAC;IACF,IAAI,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,YAAY;QAClD,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACpE,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxE,OAAO,KAAK,CAAC;AACf,CAAC;AAWD;;;qBAGqB;AACrB,MAAM,UAAU,iBAAiB,CAC/B,GAAY,EACZ,IAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC;IAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACxD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CACJ,GAAG,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,wCAAwC;gBAChE,gCAAgC,CACnC,CAAC;QACJ,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAClE,IAAI,KAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,GAAG,SAAS,CAAC;QAClB,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,YAAY,CAAC;QAClB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,MAAM;QACN,YAAY,EAAE,OAAO;QACrB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC;QAC7B,YAAY,EAAE,iBAAiB,CAAC,GAAG,CAAC;QACpC,QAAQ,EAAE,KAAK;QACf,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC,CAAC;AACL,CAAC;AAED;2DAC2D;AAC3D,MAAM,UAAU,WAAW,CAAC,IAAa;IACvC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;4DAC4D;AAC5D,MAAM,UAAU,YAAY,CAC1B,IAAa,EACb,IAAwF;IAExF,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAiBD;;iEAEiE;AACjE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAkB,EAClB,IAAa,EACb,UAAkB,EAClB,OAAuE,EAAE;IAEzE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC;IAC1C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;YACvC,UAAU;YACV,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM;SACP,CAAC,CAAC;QACH,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACjC,QAAQ,EAAE,CAAC;QACb,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,EAAE,CAAC;YACT,MAAM,CAAC,gCAAgC,KAAK,CAAC,IAAI,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,iBAAkB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACvC,CAAC"}

package/dist/promptfoo.d.ts

@@ -0,0 +1,161 @@
+/**
+ * margin-meter/promptfoo — turn a `promptfoo eval` result file into outcomes.
+ *
+ * The TypeScript twin of `sdk/python/margin_meter/promptfoo.py`. promptfoo is a
+ * Node tool, so a promptfoo user is already in the JS toolchain and this SDK is
+ * their natural bridge. `promptfoo eval -o results.json` writes a code-graded
+ * pass/fail file that maps one-to-one onto `POST /api/ingest/outcomes`
+ * (`docs/INTEGRATION_LADDER.md` rung 2), so the whole of rung 2 — "their evals
+ * report pass/fail" — is available with no hand-written POST:
+ *
+ *     promptfoo eval -o results.json
+ *
+ *     import { MarginMeter, promptfoo } from "margin-meter";
+ *     const meter = new MarginMeter();               // MARGIN_INGEST_URL/KEY from env
+ *     const data = JSON.parse(await readFile("results.json", "utf8"));
+ *     await promptfoo.recordResults(meter, data, "my-eval-suite");
+ *
+ * The mapping is identical to the Python adapter's, on purpose — the two SDKs
+ * must not drift:
+ *
+ *  - `passed`        ← `success` (the top-level aggregate verdict).
+ *  - `qualityScore`  ← `score` passed through when in [0, 1]; a score outside
+ *    that range (a raw metric from a custom assert) falls back to the coherent
+ *    binary, because the ingest layer rejects an out-of-range score and losing
+ *    the row is worse than losing the decimal. Always supplied (1.0/0.0 at
+ *    least): the ingest layer discards `quality_method`/`grader_kind` on a
+ *    scoreless outcome, so the score is what lights the grading provenance.
+ *  - `qualityMethod`/`graderKind` ← DERIVED from the assert types, never
+ *    assumed (see the honesty rule below).
+ *  - `link`          ← `testCase.description`, else the prompt id + index — never
+ *    `prompt.label`, which promptfoo defaults to the prompt text (MAR-1548).
+ *  - `valueUsd`      ← opt-in via `vars.margin_value` or
+ *    `testCase.metadata.margin_value` — the only route to value-per-outcome
+ *    (rung 8). Absent is the cost-per-outcome default, never coerced to 0.
+ *  - a result with no boolean `success` is not a verdict and is SKIPPED —
+ *    recording a non-verdict would fabricate an outcome the eval did not
+ *    produce.
+ *
+ * ## The honesty rule
+ *
+ * promptfoo grades two different ways and the adapter must not conflate them:
+ * code-graded asserts (`contains`, `equals`, `is-json`, `javascript`, `regex`,
+ * …) are `ground_truth`/`code`; model-graded asserts (`llm-rubric`,
+ * `factuality`, `similar`, `classifier`, `g-eval`, …) are `llm_judge`/`judge`.
+ * The WEAKER honest claim wins: any model-graded assert makes the whole outcome
+ * a judge grade; a code assert mixed with an assert type we do not recognise is
+ * downgraded to `unknown` (we cannot prove the unknown one was code); nothing
+ * recognisable at all is `unknown`. A rubric-graded eval reported as a
+ * ground-truth code grade would be the exact productivity illusion Margin
+ * exists to puncture.
+ *
+ * Two design promises, same as the Python adapter: it consumes the file the
+ * customer already produces (no new harness), and it never fabricates and is
+ * fail-safe — a delivery fault on one row warns and is counted, never aborting
+ * the batch.
+ */
+import type { RecordOutcomeInput } from "./index.ts";
+/** Where a warning goes. Defaults to `process.emitWarning` so a delivery or
+ *  tagging problem is visible without crashing the host; a test injects a
+ *  collector. */
+export type WarnFn = (message: string) => void;
+/** The graded verdict behind one promptfoo result. */
+export interface Verdict {
+    passed: boolean;
+    score: number;
+}
+/** Grading provenance for one outcome. */
+export interface GraderSignature {
+    qualityMethod: string;
+    graderKind: string;
+}
+/** Normalise a promptfoo assert type for set membership: lower-case, drop
+ *  dashes/underscores/spaces, strip a leading `not` negation (`not-contains`
+ *  grades the same WAY as `contains`). Returns undefined for a missing/blank
+ *  type. */
+export declare function normAssertType(raw: unknown): string | undefined;
+/** Derive `(qualityMethod, graderKind)` from the assert types behind a result,
+ *  applying the weaker-claim-wins honesty rule:
+ *   - any model-graded assert  → llm_judge/judge
+ *   - only recognised code asserts → ground_truth/code
+ *   - anything else (no asserts, or a code assert mixed with an unrecognised
+ *     type we cannot vouch for) → unknown/unknown */
+export declare function gradingSignature(assertTypes: Iterable<unknown>): GraderSignature;
+/** Collect the assert types behind one promptfoo result. Prefers
+ *  `gradingResult.componentResults[].assertion.type` (what actually ran,
+ *  including expanded asserts) and its single top-level `assertion`; falls back
+ *  to the configured `testCase.assert[].type`. */
+export declare function assertTypesFromResult(result: unknown): unknown[];
+/** Read an opt-in `margin_value` off a result's `vars` or its test case's
+ *  `metadata`/`vars` — the value-per-outcome numerator (rung 8). Returns the USD
+ *  amount, or undefined when absent (the normal cost-per-outcome case). Throws
+ *  on a malformed value so the caller can warn and drop it. */
+export declare function valueFromResult(result: unknown): number | undefined;
+/** A traceable link back to the exact eval case: the test description if the
+ *  author wrote one, else the prompt id plus the row index. ⛔ Never
+ *  `prompt.label` (MAR-1548): promptfoo defaults it to the raw prompt text. */
+export declare function linkFromResult(result: unknown, index: number): string;
+/** Return the `{ passed, score }` verdict for a real result, else undefined.
+ *  `passed` comes from a boolean `success`; a result without one is not a
+ *  verdict. `score` is passed through when in [0, 1]; out of that range (a raw
+ *  metric) falls back to the coherent binary and warns, because the ingest layer
+ *  rejects an out-of-range score and a lost row is worse than a lost decimal. */
+export declare function passedAndScore(result: unknown, onWarn?: WarnFn): Verdict | undefined;
+/** Fields for {@link outcomeInput}. */
+export interface OutcomeInputArgs {
+    workflowId: string;
+    passed: boolean;
+    qualityScore: number;
+    assertTypes: Iterable<unknown>;
+    link: string;
+    valueUsd?: number;
+    isSimulated?: boolean;
+}
+/** Translate one graded promptfoo result into `MarginMeter.recordOutcome` input.
+ *  The grader signature is derived from `assertTypes` (never assumed);
+ *  `valueUsd` rides only when a value was supplied. */
+export declare function outcomeInput(args: OutcomeInputArgs): RecordOutcomeInput;
+/** Options shared by the result-level translators. */
+export interface TranslateOptions {
+    workflowId: string;
+    index?: number;
+    isSimulated?: boolean;
+    onWarn?: WarnFn;
+}
+/** Full translation of one promptfoo result, or undefined if it is not a real
+ *  verdict. A malformed `margin_value` warns and drops the value (keeping the
+ *  pass/fail), never throwing into the caller. */
+export declare function resultToOutcomeInput(result: unknown, opts: TranslateOptions): RecordOutcomeInput | undefined;
+/** Pull the flat list of per-test results out of a parsed promptfoo output.
+ *  promptfoo has shipped several envelope shapes (the top-level OutputFile with
+ *  `results.results`, an older `results` list, a bare list); this walks the
+ *  known shapes rather than pinning one version. */
+export declare function extractResults(data: unknown): Record<string, unknown>[];
+/** Yield `recordOutcome` inputs for every real verdict in a parsed promptfoo
+ *  output (any envelope shape {@link extractResults} understands). */
+export declare function iterOutcomes(data: unknown, opts: {
+    workflowId: string;
+    isSimulated?: boolean;
+    onWarn?: WarnFn;
+}): RecordOutcomeInput[];
+/** The minimal meter surface {@link recordResults} needs — satisfied by
+ *  `MarginMeter`, kept as a structural type so this module imports no runtime
+ *  value from `index.ts`. */
+export interface OutcomeSink {
+    recordOutcome(input: RecordOutcomeInput): Promise<unknown>;
+    flush(): Promise<void>;
+}
+/** How one `recordResults` batch landed. */
+export interface RecordSummary {
+    recorded: number;
+    errors: number;
+    skipped: number;
+}
+/** Emit an outcome per verdict in a parsed promptfoo output through `meter`.
+ *  Fail-safe: a delivery fault on one row warns and counts an error; it never
+ *  aborts the batch. Non-verdict rows are counted as skipped. */
+export declare function recordResults(meter: OutcomeSink, data: unknown, workflowId: string, opts?: {
+    isSimulated?: boolean;
+    onWarn?: WarnFn;
+}): Promise<RecordSummary>;
+//# sourceMappingURL=promptfoo.d.ts.map

package/dist/promptfoo.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"promptfoo.d.ts","sourceRoot":"","sources":["../src/promptfoo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AA8ErD;;iBAEiB;AACjB,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;AAU/C,sDAAsD;AACtD,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,0CAA0C;AAC1C,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;YAGY;AACZ,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAM/D;AAED;;;;;qDAKqD;AACrD,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,eAAe,CAehF;AAED;;;kDAGkD;AAClD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,EAAE,CAuBhE;AAmCD;;;+DAG+D;AAC/D,wBAAgB,eAAe,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAmBnE;AAED;;+EAE+E;AAC/E,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAoBrE;AAED;;;;iFAIiF;AACjF,wBAAgB,cAAc,CAC5B,MAAM,EAAE,OAAO,EACf,MAAM,GAAE,MAAoB,GAC3B,OAAO,GAAG,SAAS,CAerB;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;uDAEuD;AACvD,wBAAgB,YAAY,CAAC,IAAI,EAAE,gBAAgB,GAAG,kBAAkB,CAavE;AAED,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;kDAEkD;AAClD,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,OAAO,EACf,IAAI,EAAE,gBAAgB,GACrB,kBAAkB,GAAG,SAAS,CAqBhC;AAED;;;oDAGoD;AACpD,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAWvE;AAED;sEACsE;AACtE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,OAAO,EACb,IAAI,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,kBAAkB,EAAE,CAatB;AAED;;6BAE6B;AAC7B,MAAM,WAAW,WAAW;IAC1B,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,4CAA4C;AAC5C,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;iEAEiE;AACjE,wBAAsB,aAAa,CACjC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,OAAO,EACb,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE;IAAE,WAAW,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GACpD,OAAO,CAAC,aAAa,CAAC,CAiCxB"}