@trazum/core 1.8.0

package/dist/detect.js

@@ -0,0 +1,228 @@
+/**
+ * How each provider announces itself in source.
+ *
+ * Data rather than branches, so adding a provider is a row. Matched as plain
+ * substrings on a lowercased copy — no regex, because this runs over files a
+ * contributor supplies and the pattern-goes-quadratic lesson has been paid for
+ * three times this week already.
+ *
+ * A quoted package name will also match the same string appearing in a prompt's
+ * own text. That is accepted rather than fixed: an import is the *weakest* kind
+ * of evidence here, so a model literal or a base URL overrides it, and the
+ * alternative — enumerating every import spelling in two languages — is how the
+ * single-quoted `require` form came to be missing.
+ */
+const SIGNATURES = [
+    {
+        provider: 'anthropic',
+        imports: ['@anthropic-ai/sdk', 'anthropic-sdk', 'from anthropic', 'import anthropic'],
+        urls: ['api.anthropic.com'],
+    },
+    {
+        provider: 'openai',
+        // The quoted package name covers `from 'openai'`, `require("openai")` and
+        // every other JS spelling without enumerating them — which is how the
+        // single-quoted `require` form got missed. The bare Python forms need their
+        // own entries because Python does not quote its imports.
+        imports: ["'openai'", '"openai"', 'from openai', 'import openai'],
+        urls: ['api.openai.com'],
+    },
+    {
+        provider: 'google',
+        imports: [
+            '@google/genai',
+            '@google/generative-ai',
+            'google.generativeai',
+            'from google import genai',
+        ],
+        urls: ['generativelanguage.googleapis.com', 'aiplatform.googleapis.com'],
+    },
+    {
+        provider: 'moonshot',
+        imports: ['moonshot'],
+        urls: ['api.moonshot.cn', 'api.moonshot.ai'],
+    },
+    {
+        provider: 'deepseek',
+        imports: ['deepseek'],
+        urls: ['api.deepseek.com'],
+    },
+    { provider: 'xai', imports: ['@ai-sdk/xai', 'xai-sdk'], urls: ['api.x.ai'] },
+    { provider: 'mistral', imports: ['@mistralai/mistralai', 'from mistralai'], urls: ['api.mistral.ai'] },
+];
+/**
+ * 1-based line lookup, built once per file.
+ *
+ * The obvious version counts newlines from zero on every call, which is
+ * quadratic in the number of matches — 36 seconds on a file repeating a model
+ * id. That is the third time this week the same shape has appeared, and the
+ * second time I have written it after fixing it in `extract.ts`; the difference
+ * here is that matches arrive out of order across the signature loop, so a
+ * forward-only counter does not work and this is a binary search instead.
+ */
+function lineIndex(source) {
+    const newlines = [];
+    for (let i = 0; i < source.length; i++)
+        if (source[i] === '\n')
+            newlines.push(i);
+    return (offset) => {
+        let low = 0;
+        let high = newlines.length;
+        while (low < high) {
+            const mid = (low + high) >> 1;
+            if (newlines[mid] < offset)
+                low = mid + 1;
+            else
+                high = mid;
+        }
+        return low + 1;
+    };
+}
+/**
+ * Model names written as string literals next to a `model` key.
+ *
+ * Bounded deliberately: `model` then up to a few characters of punctuation and
+ * whitespace, then a quoted value. A looser search would match the word "model"
+ * in a prompt's own text and price the prompt against something it merely
+ * mentions.
+ */
+function modelLiterals(source, known, lineAt) {
+    const found = [];
+    const haystack = source.toLowerCase();
+    for (const model of known) {
+        let from = 0;
+        for (;;) {
+            const at = haystack.indexOf(model.id.toLowerCase(), from);
+            if (at === -1)
+                break;
+            from = at + model.id.length;
+            // It has to be a quoted value: a model id in prose is a mention, not a
+            // call. Checking the character either side is enough and costs nothing.
+            const before = source[at - 1];
+            const after = source[at + model.id.length];
+            const quoted = (before === "'" || before === '"' || before === '`') &&
+                (after === "'" || after === '"' || after === '`');
+            if (!quoted)
+                continue;
+            found.push({
+                kind: 'model-literal',
+                detail: model.id,
+                line: lineAt(at),
+                provider: model.provider,
+                model: model.id,
+            });
+        }
+    }
+    return found;
+}
+/** `model=gpt-5` on a `trazum:prompt` marker line. */
+function markerModels(source, known, lineAt) {
+    const found = [];
+    let from = 0;
+    for (;;) {
+        const at = source.indexOf('trazum:prompt', from);
+        if (at === -1)
+            break;
+        from = at + 13;
+        const lineEnd = source.indexOf('\n', at);
+        const rest = source.slice(from, lineEnd === -1 ? source.length : lineEnd);
+        const key = rest.indexOf('model=');
+        if (key === -1)
+            continue;
+        const value = rest.slice(key + 6).trim().split(/[\s>]/)[0] ?? '';
+        const model = known.find((m) => m.id === value);
+        if (!model)
+            continue;
+        found.push({
+            kind: 'marker',
+            detail: `model=${value}`,
+            line: lineAt(at),
+            provider: model.provider,
+            model: model.id,
+        });
+    }
+    return found;
+}
+/**
+ * Works out which provider a source file talks to, and says why.
+ *
+ * Returns `provider: null` both when nothing was found and when the file named
+ * more than one — the caller cannot act on either, and `conflicts` distinguishes
+ * them for the reader.
+ */
+export function detectFromSource(source, options = {}) {
+    const known = options.models ?? [];
+    const haystack = source.toLowerCase();
+    const lineAt = lineIndex(source);
+    const evidence = [
+        // Strongest first: the author naming a model beats the code naming one,
+        // which beats the code naming only a provider.
+        ...markerModels(source, known, lineAt),
+        ...modelLiterals(source, known, lineAt),
+    ];
+    for (const signature of SIGNATURES) {
+        for (const needle of signature.imports) {
+            const at = haystack.indexOf(needle.toLowerCase());
+            if (at === -1)
+                continue;
+            evidence.push({
+                kind: 'sdk-import',
+                detail: needle,
+                line: lineAt(at),
+                provider: signature.provider,
+            });
+            break;
+        }
+        for (const url of signature.urls) {
+            const at = haystack.indexOf(url);
+            if (at === -1)
+                continue;
+            evidence.push({
+                kind: 'base-url',
+                detail: url,
+                line: lineAt(at),
+                provider: signature.provider,
+            });
+            break;
+        }
+    }
+    if (evidence.length === 0) {
+        return { provider: null, model: null, evidence: [], conflicts: [] };
+    }
+    // A stronger kind overrides a weaker one; only equals can disagree.
+    //
+    // Getting this wrong the first time broke the most common non-OpenAI setup
+    // there is. Moonshot, DeepSeek, xAI, Groq and Together all ship an
+    // OpenAI-compatible endpoint, so their documented usage is the OpenAI SDK
+    // pointed at a different `base_url`. Treating that as a contradiction —
+    // "this file says both openai and deepseek" — would refuse to price a
+    // perfectly ordinary DeepSeek client. The base URL is the specific fact and
+    // the SDK import is the generic one, so the URL wins rather than ties.
+    //
+    // Likewise a `model=` on the marker is the author stating the answer, which
+    // cannot be contradicted by an import they wrote for another reason.
+    const RANK = {
+        marker: 3,
+        'model-literal': 2,
+        'base-url': 1,
+        'sdk-import': 0,
+    };
+    const ranked = [...evidence].sort((a, b) => RANK[b.kind] - RANK[a.kind]);
+    const best = ranked[0];
+    const topRank = RANK[best.kind];
+    // Only evidence of the same strength can conflict. Anything weaker is
+    // context, not contradiction.
+    const conflicts = ranked.filter((e) => RANK[e.kind] === topRank && e.provider !== undefined && e.provider !== best.provider);
+    if (conflicts.length > 0) {
+        return { provider: null, model: null, evidence: ranked, conflicts };
+    }
+    return {
+        provider: best.provider ?? null,
+        // A model only comes from something that named one, and only from the
+        // winning provider: an import tells us who, never which of their models.
+        model: ranked.find((e) => e.model !== undefined && e.provider === best.provider)?.model ?? null,
+        evidence: ranked,
+        conflicts: [],
+    };
+}
+//# sourceMappingURL=detect.js.map

package/dist/detect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect.js","sourceRoot":"","sources":["../src/detect.ts"],"names":[],"mappings":"AA8DA;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,GAAmE;IACjF;QACE,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,CAAC,mBAAmB,EAAE,eAAe,EAAE,gBAAgB,EAAE,kBAAkB,CAAC;QACrF,IAAI,EAAE,CAAC,mBAAmB,CAAC;KAC5B;IACD;QACE,QAAQ,EAAE,QAAQ;QAClB,0EAA0E;QAC1E,sEAAsE;QACtE,4EAA4E;QAC5E,yDAAyD;QACzD,OAAO,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,eAAe,CAAC;QACjE,IAAI,EAAE,CAAC,gBAAgB,CAAC;KACzB;IACD;QACE,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE;YACP,eAAe;YACf,uBAAuB;YACvB,qBAAqB;YACrB,0BAA0B;SAC3B;QACD,IAAI,EAAE,CAAC,mCAAmC,EAAE,2BAA2B,CAAC;KACzE;IACD;QACE,QAAQ,EAAE,UAAU;QACpB,OAAO,EAAE,CAAC,UAAU,CAAC;QACrB,IAAI,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;KAC7C;IACD;QACE,QAAQ,EAAE,UAAU;QACpB,OAAO,EAAE,CAAC,UAAU,CAAC;QACrB,IAAI,EAAE,CAAC,kBAAkB,CAAC;KAC3B;IACD,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,aAAa,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,EAAE;IAC5E,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,sBAAsB,EAAE,gBAAgB,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,CAAC,EAAE;CACvG,CAAC;AAEF;;;;;;;;;GASG;AACH,SAAS,SAAS,CAAC,MAAc;IAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEjF,OAAO,CAAC,MAAM,EAAE,EAAE;QAChB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC3B,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,QAAQ,CAAC,GAAG,CAAE,GAAG,MAAM;gBAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;;gBACtC,IAAI,GAAG,GAAG,CAAC;QAClB,CAAC;QACD,OAAO,GAAG,GAAG,CAAC,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,aAAa,CACpB,MAAc,EACd,KAAqB,EACrB,MAAkC;IAElC,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IAEtC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,SAAS,CAAC;YACR,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,MAAM;YACrB,IAAI,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC;YAE5B,uEAAuE;YACvE,wEAAwE;YACxE,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,MAAM,GACV,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;gBACpD,CAAC,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,CAAC;YACpD,IAAI,CAAC,MAAM;gBAAE,SAAS;YAEtB,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,eAAe;gBACrB,MAAM,EAAE,KAAK,CAAC,EAAE;gBAChB,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,KAAK,EAAE,KAAK,CAAC,EAAE;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,sDAAsD;AACtD,SAAS,YAAY,CACnB,MAAc,EACd,KAAqB,EACrB,MAAkC;IAElC,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,SAAS,CAAC;QACR,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QACjD,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,MAAM;QACrB,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC;QAEf,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,SAAS;QAEzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,SAAS;QAErB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,SAAS,KAAK,EAAE;YACxB,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;YAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,EAAE;SAChB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAOD;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,OAAO,GAAkB,EAAE;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAEjC,MAAM,QAAQ,GAAe;QAC3B,wEAAwE;QACxE,+CAA+C;QAC/C,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC;QACtC,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC;KACxC,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;YAClD,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,SAAS;YACxB,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChB,QAAQ,EAAE,SAAS,CAAC,QAAQ;aAC7B,CAAC,CAAC;YACH,MAAM;QACR,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,SAAS;YACxB,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChB,QAAQ,EAAE,SAAS,CAAC,QAAQ;aAC7B,CAAC,CAAC;YACH,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACtE,CAAC;IAED,oEAAoE;IACpE,EAAE;IACF,2EAA2E;IAC3E,mEAAmE;IACnE,0EAA0E;IAC1E,wEAAwE;IACxE,sEAAsE;IACtE,4EAA4E;IAC5E,uEAAuE;IACvE,EAAE;IACF,4EAA4E;IAC5E,qEAAqE;IACrE,MAAM,IAAI,GAAiC;QACzC,MAAM,EAAE,CAAC;QACT,eAAe,EAAE,CAAC;QAClB,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;KAChB,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;IACxB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhC,sEAAsE;IACtE,8BAA8B;IAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAC5F,CAAC;IAEF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACtE,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;QAC/B,sEAAsE;QACtE,yEAAyE;QACzE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,IAAI,IAAI;QAC/F,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC"}

package/dist/evaluate.d.ts

@@ -0,0 +1,98 @@
+import type { LlmProvider } from './types.js';
+/**
+ * Golden-set evaluation.
+ *
+ * Everything else Trazum reports is arithmetic: tokens, prices, multiplication.
+ * This is the one question arithmetic cannot answer — does the shorter prompt
+ * still do the job? — and the README has been answering it with a caveat
+ * ("the aggressive level can change nuance; read the diff") because a rules
+ * engine genuinely cannot know.
+ *
+ * The trap here is comparing the two prompts' outputs and calling the
+ * difference a regression. A model asked the same question twice does not
+ * answer identically, so "the optimised prompt diverged on 3 of 10 cases" is
+ * meaningless on its own — it might be better than the original manages
+ * against itself.
+ *
+ * So the original is run twice per case first, and that self-agreement is the
+ * yardstick. The optimised prompt is judged against the model's own variance,
+ * not against an imaginary determinism it never had. It costs a third call per
+ * case and it is the only reason the number means anything.
+ */
+export interface EvalCase {
+    /** The input this case feeds the prompt. */
+    input: string;
+    /** The original prompt's two answers, used to measure its own variance. */
+    baseline: [string, string];
+    /** The optimised prompt's answer. */
+    optimized: string;
+    /** How closely the original agreed with itself (0-1). */
+    selfSimilarity: number;
+    /** How closely the optimised answer matched the original's first (0-1). */
+    crossSimilarity: number;
+}
+export type EvalVerdict = 'indistinguishable' | 'within-noise' | 'diverges' | 'inconclusive';
+export interface EvalReport {
+    provider: string;
+    model: string;
+    cases: EvalCase[];
+    /** Mean agreement of the original prompt with itself. The yardstick. */
+    selfAgreement: number;
+    /** Mean agreement between the original and the optimised prompt. */
+    crossAgreement: number;
+    verdict: EvalVerdict;
+    /** Total provider calls made, so the cost is never a surprise. */
+    callsMade: number;
+}
+export interface EvaluateOptions {
+    /**
+     * How many cases to run at once. Kept low by default: this hammers someone
+     * else's endpoint, and a rate limit tripped halfway through wastes every
+     * call already paid for.
+     */
+    concurrency?: number;
+}
+/**
+ * Builds the prompt for one case.
+ *
+ * A template gets its first placeholder filled; anything else gets the input
+ * appended. Substituting is the honest reading of a prompt written with
+ * `{{query}}` — appending would test a prompt nobody runs.
+ */
+export declare function fillPrompt(prompt: string, input: string): string;
+/** Agreement between two answers, 0-1. */
+/**
+ * How closely two answers agree, 0 to 1.
+ *
+ * Exported because `prune.ts` measures the same thing and must measure it the
+ * same way. It was a private function here first, and the copy that appeared in
+ * `prune.ts` was a bag-of-words F1 while this is Jaccard over normalised text —
+ * two different numbers under one name, with a comment in the copy claiming they
+ * were the same measure. Sharing the function is what makes that comment true.
+ */
+export declare function agreement(a: string, b: string): number;
+/**
+ * Turns the two agreement figures into a verdict.
+ *
+ * The comparison is always relative. An optimised prompt agreeing with the
+ * original 0.85 of the time looks alarming until you see the original agrees
+ * with itself 0.86 — at which point the optimisation changed nothing the model
+ * was not already doing on its own.
+ *
+ * `inconclusive` exists because a model that is wildly inconsistent with itself
+ * cannot be used to judge anything. Reporting a confident verdict off that
+ * would be worse than admitting the test does not work here.
+ */
+export declare function verdictFor(selfAgreement: number, crossAgreement: number): EvalVerdict;
+/** Runs `tasks` with a bounded number in flight, preserving order. */
+export declare function pooled<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]>;
+/**
+ * Runs both prompt versions over a set of inputs and reports whether the
+ * optimisation changed the answers.
+ *
+ * Costs **three provider calls per case**: the original twice, the optimised
+ * once. The doubled original is what makes the result interpretable, and
+ * `callsMade` reports the total so the bill is never a surprise.
+ */
+export declare function evaluate(originalPrompt: string, optimizedPrompt: string, inputs: readonly string[], provider: LlmProvider, options?: EvaluateOptions): Promise<EvalReport>;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,QAAQ;IACvB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,cAAc,GAAG,UAAU,GAAG,cAAc,CAAC;AAE7F,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,wEAAwE;IACxE,aAAa,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAMhE;AAED,0CAA0C;AAC1C;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAKtD;AAKD;;;;;;;;;;;GAWG;AACH,wBAAgB,UAAU,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,WAAW,CAOrF;AAED,sEAAsE;AACtE,wBAAsB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAc3F;AAED;;;;;;;GAOG;AACH,wBAAsB,QAAQ,CAC5B,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,QAAQ,EAAE,WAAW,EACrB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,UAAU,CAAC,CAsCrB"}

package/dist/evaluate.js

@@ -0,0 +1,110 @@
+import { segment } from './segment.js';
+import { jaccard, normalizeForCompare } from './similarity.js';
+/**
+ * Builds the prompt for one case.
+ *
+ * A template gets its first placeholder filled; anything else gets the input
+ * appended. Substituting is the honest reading of a prompt written with
+ * `{{query}}` — appending would test a prompt nobody runs.
+ */
+export function fillPrompt(prompt, input) {
+    const placeholder = segment(prompt).find((s) => s.kind === 'protected' && s.protection === 'placeholder');
+    if (!placeholder)
+        return `${prompt.trimEnd()}\n\n${input}`;
+    return prompt.replace(placeholder.text, input);
+}
+/** Agreement between two answers, 0-1. */
+/**
+ * How closely two answers agree, 0 to 1.
+ *
+ * Exported because `prune.ts` measures the same thing and must measure it the
+ * same way. It was a private function here first, and the copy that appeared in
+ * `prune.ts` was a bag-of-words F1 while this is Jaccard over normalised text —
+ * two different numbers under one name, with a comment in the copy claiming they
+ * were the same measure. Sharing the function is what makes that comment true.
+ */
+export function agreement(a, b) {
+    const left = normalizeForCompare(a);
+    const right = normalizeForCompare(b);
+    if (left === right)
+        return 1;
+    return jaccard(left, right);
+}
+const mean = (values) => values.length === 0 ? 0 : values.reduce((sum, v) => sum + v, 0) / values.length;
+/**
+ * Turns the two agreement figures into a verdict.
+ *
+ * The comparison is always relative. An optimised prompt agreeing with the
+ * original 0.85 of the time looks alarming until you see the original agrees
+ * with itself 0.86 — at which point the optimisation changed nothing the model
+ * was not already doing on its own.
+ *
+ * `inconclusive` exists because a model that is wildly inconsistent with itself
+ * cannot be used to judge anything. Reporting a confident verdict off that
+ * would be worse than admitting the test does not work here.
+ */
+export function verdictFor(selfAgreement, crossAgreement) {
+    if (crossAgreement >= 0.999)
+        return 'indistinguishable';
+    if (selfAgreement < 0.5)
+        return 'inconclusive';
+    // Within a small margin of the model's own noise floor, the difference is
+    // not attributable to the prompt.
+    if (crossAgreement >= selfAgreement - 0.05)
+        return 'within-noise';
+    return 'diverges';
+}
+/** Runs `tasks` with a bounded number in flight, preserving order. */
+export async function pooled(tasks, limit) {
+    const results = new Array(tasks.length);
+    let next = 0;
+    const workers = Array.from({ length: Math.min(limit, tasks.length) }, async () => {
+        for (;;) {
+            const index = next++;
+            if (index >= tasks.length)
+                return;
+            results[index] = await tasks[index]();
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+/**
+ * Runs both prompt versions over a set of inputs and reports whether the
+ * optimisation changed the answers.
+ *
+ * Costs **three provider calls per case**: the original twice, the optimised
+ * once. The doubled original is what makes the result interpretable, and
+ * `callsMade` reports the total so the bill is never a surprise.
+ */
+export async function evaluate(originalPrompt, optimizedPrompt, inputs, provider, options = {}) {
+    const concurrency = Math.max(1, options.concurrency ?? 3);
+    const run = (prompt, input) => provider.complete({ system: fillPrompt(prompt, input), user: input });
+    const cases = await pooled(inputs.map((input) => async () => {
+        // Sequential within a case: the two baseline runs exist to measure the
+        // model's variance, and issuing them together invites a provider to
+        // serve one from a cache and report a variance of zero.
+        const baselineA = await run(originalPrompt, input);
+        const baselineB = await run(originalPrompt, input);
+        const optimized = await run(optimizedPrompt, input);
+        return {
+            input,
+            baseline: [baselineA, baselineB],
+            optimized,
+            selfSimilarity: agreement(baselineA, baselineB),
+            crossSimilarity: agreement(baselineA, optimized),
+        };
+    }), concurrency);
+    const selfAgreement = mean(cases.map((c) => c.selfSimilarity));
+    const crossAgreement = mean(cases.map((c) => c.crossSimilarity));
+    return {
+        provider: provider.name,
+        model: provider.model,
+        cases,
+        selfAgreement,
+        crossAgreement,
+        verdict: verdictFor(selfAgreement, crossAgreement),
+        callsMade: cases.length * 3,
+    };
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AA6D/D;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,MAAc,EAAE,KAAa;IACtD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,UAAU,KAAK,aAAa,CAChE,CAAC;IACF,IAAI,CAAC,WAAW;QAAE,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,KAAK,EAAE,CAAC;IAC3D,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACjD,CAAC;AAED,0CAA0C;AAC1C;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,CAAS,EAAE,CAAS;IAC5C,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACrC,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAU,EAAE,CACxC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAElF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,UAAU,CAAC,aAAqB,EAAE,cAAsB;IACtE,IAAI,cAAc,IAAI,KAAK;QAAE,OAAO,mBAAmB,CAAC;IACxD,IAAI,aAAa,GAAG,GAAG;QAAE,OAAO,cAAc,CAAC;IAC/C,0EAA0E;IAC1E,kCAAkC;IAClC,IAAI,cAAc,IAAI,aAAa,GAAG,IAAI;QAAE,OAAO,cAAc,CAAC;IAClE,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,MAAM,CAAI,KAA8B,EAAE,KAAa;IAC3E,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC/E,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;YACrB,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAClC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,KAAK,CAAE,EAAE,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,cAAsB,EACtB,eAAuB,EACvB,MAAyB,EACzB,QAAqB,EACrB,OAAO,GAAoB,EAAE;IAE7B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAE1D,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,KAAa,EAAmB,EAAE,CAC7D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAExE,MAAM,KAAK,GAAG,MAAM,MAAM,CACxB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAuB,EAAE;QAClD,uEAAuE;QACvE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAEpD,OAAO;YACL,KAAK;YACL,QAAQ,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;YAChC,SAAS;YACT,cAAc,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;YAC/C,eAAe,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;SACjD,CAAC;IACJ,CAAC,CAAC,EACF,WAAW,CACZ,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;IAEjE,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,KAAK;QACL,aAAa;QACb,cAAc;QACd,OAAO,EAAE,UAAU,CAAC,aAAa,EAAE,cAAc,CAAC;QAClD,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC;KAC5B,CAAC;AACJ,CAAC"}

package/dist/extract.d.ts

@@ -0,0 +1,81 @@
+/**
+ * Prompts embedded in source files.
+ *
+ * `check` and `diff` read `.txt`, `.md`, `.prompt` and `.tmpl`. Real prompts live
+ * in TypeScript template literals, Python triple-quoted strings and YAML blocks,
+ * so adopting Trazum has meant first refactoring them out into standalone files —
+ * a change to somebody's application as the price of admission.
+ *
+ * **It reads a marker, it does not guess.** `// trazum:prompt` on the line before
+ * the literal, and then the literal by delimiter matching. Guessing which string
+ * in a file is a prompt is a heuristic, and a heuristic inside a tool used as a
+ * CI gate produces failures on strings nobody meant to govern. A marker is one
+ * line of noise in exchange for never being wrong about what it picked up.
+ *
+ * **Interpolation already works and is not a special case.** `${x}` inside a
+ * template literal is exactly the placeholder shape `segment.ts` protects, so an
+ * embedded prompt gets the same cache-prefix analysis, the same protection from
+ * the rules, and the same `--reorder` treatment as a `{{x}}` template. Nothing
+ * here has to know about it.
+ *
+ * **The honest limit:** a prompt assembled from concatenated pieces cannot be
+ * read this way. `` `You are ${role}.` + rules.join('\n') `` is a prompt whose
+ * text does not exist until it runs. This module declines it and says so rather
+ * than governing the half it can see — a budget enforced against a fragment is a
+ * green build for a prompt nobody measured.
+ *
+ * Scanned character by character rather than with a regex. The module it most
+ * resembles shipped two quadratic patterns this week, and delimiter matching over
+ * untrusted source is exactly the shape that goes wrong.
+ */
+export interface ExtractedPrompt {
+    /** Name from `trazum:prompt <name>`, or undefined when the marker was bare. */
+    name?: string;
+    /** 1-based line of the marker comment, which is what an error should cite. */
+    line: number;
+    /** The prompt text, delimiters removed and escapes resolved. */
+    text: string;
+    /** The delimiter it was written with, so a writer could put it back. */
+    quote: '`' | '"' | "'" | '"""' | "'''";
+    /** Offsets of the text within the source, exclusive of the delimiters. */
+    start: number;
+    end: number;
+}
+export interface DeclinedPrompt {
+    line: number;
+    /**
+     * Why this marker produced nothing.
+     *
+     * `concatenated` is the interesting one: the text exists only at runtime.
+     * `no-literal` means the marker was not followed by a string at all.
+     * `unterminated` means the file ends inside the literal, which is a syntax
+     * error in the source rather than a Trazum problem, but worth naming.
+     */
+    reason: 'concatenated' | 'no-literal' | 'unterminated';
+    detail: string;
+}
+export interface ExtractionResult {
+    prompts: ExtractedPrompt[];
+    declined: DeclinedPrompt[];
+}
+/**
+ * Finds every marked prompt in a source file.
+ *
+ * Returns what it found and what it refused, because a marker that produced
+ * nothing is the case the author most needs to hear about: they asked for the
+ * prompt to be governed and it is not being governed.
+ */
+export declare function extractPrompts(source: string): ExtractionResult;
+/**
+ * A stable identifier for an embedded prompt, for budgets and reports.
+ *
+ * `src/prompts.ts#support` when the marker was named, `src/prompts.ts:12` when it
+ * was not. Both are path-prefixed so the glob patterns in `trazum.config.json`
+ * cover embedded prompts without learning a new syntax — `src/**` matches either.
+ */
+export declare function promptId(path: string, prompt: ExtractedPrompt): string;
+/** Whether a file is worth opening for extraction at all. */
+export declare const SOURCE_EXTENSIONS: string[];
+/** Cheap pre-filter: a source file with no marker has nothing to extract. */
+export declare const hasMarker: (source: string) => boolean;
+//# sourceMappingURL=extract.d.ts.map

package/dist/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,MAAM,WAAW,eAAe;IAC9B,+EAA+E;IAC/E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;IACb,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,KAAK,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,KAAK,GAAG,KAAK,CAAC;IACvC,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;OAOG;IACH,MAAM,EAAE,cAAc,GAAG,YAAY,GAAG,cAAc,CAAC;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B;AAkJD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAuF/D;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,GAAG,MAAM,CAEtE;AAED,6DAA6D;AAC7D,eAAO,MAAM,iBAAiB,UAgB7B,CAAC;AAEF,6EAA6E;AAC7E,eAAO,MAAM,SAAS,WAAY,MAAM,KAAG,OAA+B,CAAC"}