@trazum/core 1.8.0

package/dist/promptfoo.js

@@ -0,0 +1,149 @@
+import { getModel } from './pricing.js';
+import { segment } from './segment.js';
+/** The variable a prompt with no `{{placeholder}}` gets. */
+const APPENDED_VAR = 'input';
+/**
+ * promptfoo names providers `<vendor>:<model>`. Only the two vendors whose
+ * shape is certain are mapped; anything else is emitted as-is with a warning,
+ * because a wrong provider id fails at run time with a message about the
+ * harness rather than about this file.
+ */
+function providerId(modelId) {
+    if (modelId === undefined)
+        return { id: 'openai:gpt-4o-mini', warning: null };
+    const model = getModel(modelId);
+    switch (model.provider) {
+        case 'anthropic':
+            return { id: `anthropic:messages:${model.id}`, warning: null };
+        case 'openai':
+            return { id: `openai:${model.id}`, warning: null };
+        default:
+            return {
+                id: model.id,
+                warning: `${model.provider ?? 'this provider'} has no known promptfoo id; "${model.id}" is a guess`,
+            };
+    }
+}
+/** Placeholders in the prompt, in order of appearance, deduplicated. */
+function placeholdersIn(prompt) {
+    const found = segment(prompt)
+        .filter((piece) => piece.kind === 'protected' && piece.protection === 'placeholder')
+        .map((piece) => piece.text);
+    return [...new Set(found)];
+}
+/** `{{query}}` → `query`, or null when promptfoo would not substitute it. */
+function varNameOf(placeholder) {
+    const match = /^\{\{\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}$/.exec(placeholder);
+    return match?.[1] ?? null;
+}
+/**
+ * Whether the prompt demands JSON output.
+ *
+ * Defined narrowly and checkably: a fenced block tagged `json`, or an untagged
+ * fenced block whose body parses as JSON. Nothing about the surrounding prose,
+ * because "return JSON" in a sentence is a phrase, and a phrase is where
+ * guessing starts.
+ *
+ * The first version of this asked `findRestatedFormat`, which was the wrong
+ * question wearing a convenient shape. That function answers "is this prompt
+ * wasting tokens restating its own schema?" — so a prompt demanding JSON
+ * *cleanly* got no assertion while a wasteful one did, which is exactly
+ * backwards.
+ */
+function asksForJson(prompt) {
+    const fenced = /^[ \t]*(?:```|~~~)([A-Za-z0-9]*)[ \t]*\n([\s\S]*?)^[ \t]*(?:```|~~~)/gm;
+    let match;
+    while ((match = fenced.exec(prompt)) !== null) {
+        const lang = (match[1] ?? '').toLowerCase();
+        const body = (match[2] ?? '').trim();
+        if (/^jsonc?5?$/.test(lang))
+            return true;
+        if (lang !== '')
+            continue;
+        if (!body.startsWith('{') && !body.startsWith('['))
+            continue;
+        try {
+            JSON.parse(body);
+            return true;
+        }
+        catch {
+            // An untagged block of something else. Not our business.
+        }
+    }
+    return false;
+}
+export function toPromptfoo(original, optimized, cases, options = {}) {
+    const warnings = [];
+    const placeholders = placeholdersIn(original);
+    const supported = placeholders.map(varNameOf);
+    // The variable the cases fill. `trazum eval` substitutes the *first*
+    // placeholder and appends when there is none; promptfoo is driven the same
+    // way so the two commands are testing the same prompt.
+    let variable = APPENDED_VAR;
+    let originalTemplate = original;
+    let optimizedTemplate = optimized;
+    if (placeholders.length === 0) {
+        variable = APPENDED_VAR;
+        originalTemplate = `${original.trimEnd()}\n\n{{${APPENDED_VAR}}}`;
+        optimizedTemplate = `${optimized.trimEnd()}\n\n{{${APPENDED_VAR}}}`;
+        warnings.push({
+            kind: 'appended-input',
+            detail: 'This prompt has no placeholder, so each case is appended at the end — the same ' +
+                'thing `trazum eval` does. If the prompt is meant to be a template, add ' +
+                '{{a_variable}} and export again.',
+        });
+    }
+    else {
+        const first = supported[0] ?? null;
+        if (first === null) {
+            warnings.push({
+                kind: 'unsupported-placeholder',
+                detail: `promptfoo substitutes {{name}}; this prompt uses ${placeholders[0]}, which it will ` +
+                    'leave untouched. Every case would run against the literal template.',
+            });
+            variable = APPENDED_VAR;
+        }
+        else {
+            variable = first;
+        }
+        const distinct = placeholders.length;
+        if (distinct > 1) {
+            warnings.push({
+                kind: 'multiple-placeholders',
+                detail: `The prompt has ${distinct} placeholders (${placeholders.join(', ')}) and the case ` +
+                    `file supplies one value each, which fills ${placeholders[0]} only. Give the others ` +
+                    'defaults in `defaultTest.vars`, or the run tests a prompt nobody sends.',
+            });
+        }
+    }
+    const { id, warning } = providerId(options.model);
+    if (warning !== null)
+        warnings.push({ kind: 'unmapped-provider', detail: warning });
+    // The one assertion Trazum can make without guessing at the task: if the
+    // prompt asks for JSON, both versions still have to produce it. Everything
+    // else — accuracy, refusal rate, format specifics — is the team's to write,
+    // and inventing it here would be a tool with opinions about somebody else's
+    // product.
+    const assertions = asksForJson(original) ? [{ type: 'is-json' }] : [];
+    const label = options.level ? `after (trazum, ${options.level})` : 'after (trazum)';
+    return {
+        config: {
+            description: 'Generated by trazum. The only difference between the two prompts is the ' +
+                'optimisation — same provider, same cases, same everything else.\n\n' +
+                'The assertions are yours to write: `trazum eval` already answers "does the model ' +
+                'still say the same thing", and this suite exists for the question it cannot ask — ' +
+                'whether your accuracy, your format and your refusal rate survived. Add them under ' +
+                '`defaultTest.assert` or per test.\n\n' +
+                'Docs: https://www.promptfoo.dev/docs/configuration/expected-outputs/',
+            prompts: [
+                { label: 'before', raw: originalTemplate },
+                { label, raw: optimizedTemplate },
+            ],
+            providers: [id],
+            ...(assertions.length > 0 ? { defaultTest: { assert: assertions } } : {}),
+            tests: cases.map((input) => ({ vars: { [variable]: input } })),
+        },
+        warnings,
+    };
+}
+//# sourceMappingURL=promptfoo.js.map

package/dist/promptfoo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"promptfoo.js","sourceRoot":"","sources":["../src/promptfoo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAkDvC,4DAA4D;AAC5D,MAAM,YAAY,GAAG,OAAO,CAAC;AAE7B;;;;;GAKG;AACH,SAAS,UAAU,CAAC,OAA2B;IAC7C,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,oBAAoB,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC9E,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChC,QAAQ,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvB,KAAK,WAAW;YACd,OAAO,EAAE,EAAE,EAAE,sBAAsB,KAAK,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACjE,KAAK,QAAQ;YACX,OAAO,EAAE,EAAE,EAAE,UAAU,KAAK,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACrD;YACE,OAAO;gBACL,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,OAAO,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,eAAe,gCAAgC,KAAK,CAAC,EAAE,cAAc;aACpG,CAAC;IACN,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,SAAS,cAAc,CAAC,MAAc;IACpC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;SAC1B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,UAAU,KAAK,aAAa,CAAC;SACnF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED,6EAA6E;AAC7E,SAAS,SAAS,CAAC,WAAmB;IACpC,MAAM,KAAK,GAAG,2CAA2C,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5E,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AAC5B,CAAC;AAGD;;;;;;;;;;;;;GAaG;AACH,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,MAAM,GAAG,wEAAwE,CAAC;IAExF,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,IAAI,KAAK,EAAE;YAAG,SAAS;QAC3B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC7D,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;QAC3D,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,QAAgB,EAChB,SAAiB,EACjB,KAAwB,EACxB,OAAO,GAAqB,EAAE;IAE9B,MAAM,QAAQ,GAAuB,EAAE,CAAC;IAExC,MAAM,YAAY,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAE9C,qEAAqE;IACrE,2EAA2E;IAC3E,uDAAuD;IACvD,IAAI,QAAQ,GAAG,YAAY,CAAC;IAC5B,IAAI,gBAAgB,GAAG,QAAQ,CAAC;IAChC,IAAI,iBAAiB,GAAG,SAAS,CAAC;IAElC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,QAAQ,GAAG,YAAY,CAAC;QACxB,gBAAgB,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,IAAI,CAAC;QAClE,iBAAiB,GAAG,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,YAAY,IAAI,CAAC;QACpE,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,gBAAgB;YACtB,MAAM,EACJ,iFAAiF;gBACjF,yEAAyE;gBACzE,kCAAkC;SACrC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACnC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,yBAAyB;gBAC/B,MAAM,EACJ,oDAAoD,YAAY,CAAC,CAAC,CAAC,kBAAkB;oBACrF,qEAAqE;aACxE,CAAC,CAAC;YACH,QAAQ,GAAG,YAAY,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,KAAK,CAAC;QACnB,CAAC;QAED,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC;QACrC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,uBAAuB;gBAC7B,MAAM,EACJ,kBAAkB,QAAQ,kBAAkB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB;oBACpF,6CAA6C,YAAY,CAAC,CAAC,CAAC,yBAAyB;oBACrF,yEAAyE;aAC5E,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,OAAO,KAAK,IAAI;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAEpF,yEAAyE;IACzE,2EAA2E;IAC3E,4EAA4E;IAC5E,4EAA4E;IAC5E,WAAW;IACX,MAAM,UAAU,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEtE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC;IAEpF,OAAO;QACL,MAAM,EAAE;YACN,WAAW,EACT,0EAA0E;gBAC1E,qEAAqE;gBACrE,mFAAmF;gBACnF,oFAAoF;gBACpF,oFAAoF;gBACpF,uCAAuC;gBACvC,sEAAsE;YACxE,OAAO,EAAE;gBACP,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,gBAAgB,EAAE;gBAC1C,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE;aAClC;YACD,SAAS,EAAE,CAAC,EAAE,CAAC;YACf,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;SAC/D;QACD,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/prune.d.ts

@@ -0,0 +1,91 @@
+import type { EvalVerdict } from './evaluate.js';
+import type { LlmProvider, TokenCounter } from './types.js';
+/**
+ * Which few-shot examples earn their tokens, measured rather than guessed.
+ *
+ * The `redundant-examples` advisory answers a *textual* question: does this
+ * example look like an earlier one? That catches the way few-shot blocks actually
+ * grow — copy the last one, change two fields — and it is the cheapest useful
+ * thing to say, because it costs nothing.
+ *
+ * This answers a different and much stronger question: **does removing this
+ * example change any answer?** Two examples can be textually unalike and teach the
+ * same thing, and a block nobody has measured is usually where the tokens are: a
+ * few-shot section is routinely most of a prompt.
+ *
+ * The method is leave-one-out against the prompt's own noise floor. Ask the full
+ * prompt twice to find out how much the model disagrees with *itself*, then remove
+ * one example and ask again. If the answer moves no further than the model already
+ * moves on its own, that example is not doing observable work.
+ *
+ * **It spends the caller's money and must never run by default.** The bill is
+ * `2 + examples` calls per input, and `plannedCalls` exists so a caller can print
+ * the figure before deciding rather than discovering it afterwards.
+ *
+ * **What it cannot tell you.** An example may exist for a case these inputs do not
+ * contain — the boundary condition somebody hit in production last March and added
+ * a demonstration for. Removing it would change nothing measurable here and break
+ * that case. So this reports "no effect on these inputs", never "delete this", and
+ * the wording is deliberate: the strength of the claim is bounded by the inputs
+ * given, and only the caller knows whether those cover what matters.
+ */
+export interface ExampleContribution {
+    /** Position in the prompt's example block, from zero. */
+    index: number;
+    /** The example itself, so a report can quote its first line. */
+    text: string;
+    tokens: number;
+    /**
+     * Mean agreement between the full prompt's answer and the answer with this
+     * example removed, across every input.
+     */
+    agreementWithout: number;
+    /**
+     * `indistinguishable` and `within-noise` both mean the removal changed nothing
+     * this measurement can see. `diverges` means it did. `inconclusive` means the
+     * model disagreed with itself too much for any of this to mean anything.
+     */
+    verdict: EvalVerdict;
+}
+export interface PruneReport {
+    provider: string;
+    model: string;
+    /** The model's agreement with itself, given the full prompt. The yardstick. */
+    selfAgreement: number;
+    contributions: ExampleContribution[];
+    /** Tokens held by examples whose removal changed nothing observable. */
+    recoverableTokens: number;
+    /** Calls actually made, so the bill is never a surprise. */
+    callsMade: number;
+}
+export interface PruneOptions {
+    concurrency?: number;
+    countTokens?: TokenCounter;
+}
+/**
+ * What this will cost, before it costs it.
+ *
+ * Pure and exported so a CLI can print the number and let somebody say no. A
+ * feature that spends money and only reports the total afterwards is a feature
+ * people run once.
+ */
+export declare function plannedCalls(examples: number, inputs: number): number;
+/**
+ * The prompt with one example block removed, located by position rather than by
+ * text.
+ *
+ * `prompt.replace(block.text, '')` would be shorter and wrong: two identical
+ * example blocks — which is exactly what a copy-paste few-shot section contains —
+ * would both match the first occurrence, so removing the second would silently
+ * remove the first and the measurement would describe a prompt nobody asked about.
+ * Scanning forward from the end of the previous block gives each block its true
+ * offset.
+ */
+export declare function withoutExample(prompt: string, examples: readonly {
+    text: string;
+}[], index: number): string;
+/** Thrown rather than returned: there is nothing to report and a reason to say. */
+export declare class NothingToPrune extends Error {
+}
+export declare function pruneExamples(prompt: string, inputs: readonly string[], provider: LlmProvider, options?: PruneOptions): Promise<PruneReport>;
+//# sourceMappingURL=prune.d.ts.map

package/dist/prune.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prune.d.ts","sourceRoot":"","sources":["../src/prune.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,mBAAmB;IAClC,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,OAAO,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,+EAA+E;IAC/E,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,mBAAmB,EAAE,CAAC;IACrC,wEAAwE;IACxE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,YAAY,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAMrE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,SAAS;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,EACrC,KAAK,EAAE,MAAM,GACZ,MAAM,CAgBR;AAMD,mFAAmF;AACnF,qBAAa,cAAe,SAAQ,KAAK;CAAG;AAE5C,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,QAAQ,EAAE,WAAW,EACrB,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,CAAC,CA8EtB"}

package/dist/prune.js

@@ -0,0 +1,110 @@
+import { findExamples } from './structure.js';
+import { estimateTokens } from './tokenizer.js';
+import { agreement, pooled, verdictFor } from './evaluate.js';
+/**
+ * What this will cost, before it costs it.
+ *
+ * Pure and exported so a CLI can print the number and let somebody say no. A
+ * feature that spends money and only reports the total afterwards is a feature
+ * people run once.
+ */
+export function plannedCalls(examples, inputs) {
+    if (examples < 2 || inputs < 1)
+        return 0;
+    // Two baseline runs per input to establish the noise floor, then one run per
+    // example removed. The baselines are shared across every example, which is the
+    // only reason this is affordable at all.
+    return inputs * (2 + examples);
+}
+/**
+ * The prompt with one example block removed, located by position rather than by
+ * text.
+ *
+ * `prompt.replace(block.text, '')` would be shorter and wrong: two identical
+ * example blocks — which is exactly what a copy-paste few-shot section contains —
+ * would both match the first occurrence, so removing the second would silently
+ * remove the first and the measurement would describe a prompt nobody asked about.
+ * Scanning forward from the end of the previous block gives each block its true
+ * offset.
+ */
+export function withoutExample(prompt, examples, index) {
+    let cursor = 0;
+    for (let position = 0; position < examples.length; position++) {
+        const text = examples[position]?.text ?? '';
+        const at = prompt.indexOf(text, cursor);
+        if (at === -1)
+            break;
+        if (position === index) {
+            const before = prompt.slice(0, at);
+            const after = prompt.slice(at + text.length);
+            // Collapse the blank lines the removal leaves behind, so the resulting
+            // prompt is one an author might have written rather than one with a hole.
+            return `${before}${after}`.replace(/\n{3,}/g, '\n\n');
+        }
+        cursor = at + text.length;
+    }
+    return prompt;
+}
+function mean(values) {
+    return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+/** Thrown rather than returned: there is nothing to report and a reason to say. */
+export class NothingToPrune extends Error {
+}
+export async function pruneExamples(prompt, inputs, provider, options = {}) {
+    const count = options.countTokens ?? estimateTokens;
+    const concurrency = Math.max(1, options.concurrency ?? 3);
+    const examples = findExamples(prompt, count);
+    if (examples.length < 2) {
+        throw new NothingToPrune('this prompt has fewer than two few-shot examples, so there is nothing to compare');
+    }
+    if (inputs.length === 0) {
+        throw new NothingToPrune('no inputs were given, and leave-one-out needs something to run on');
+    }
+    let callsMade = 0;
+    const run = async (text, input) => {
+        callsMade++;
+        return provider.complete({ system: text.replaceAll('{{input}}', input), user: input });
+    };
+    /**
+     * Baselines first, and sequentially within an input.
+     *
+     * The two runs exist to measure the model's own variance, and issuing them
+     * together invites a provider to serve one from a cache and report a variance of
+     * zero — which would make every example look load-bearing.
+     */
+    const baselines = await pooled(inputs.map((input) => async () => {
+        const first = await run(prompt, input);
+        const second = await run(prompt, input);
+        return { input, first, selfSimilarity: agreement(first, second) };
+    }), concurrency);
+    const selfAgreement = mean(baselines.map((baseline) => baseline.selfSimilarity));
+    const contributions = await pooled(examples.map((example, index) => async () => {
+        const reduced = withoutExample(prompt, examples, index);
+        const answers = await pooled(baselines.map((baseline) => () => run(reduced, baseline.input)), 1);
+        const agreementWithout = mean(answers.map((answer, position) => agreement(baselines[position].first, answer)));
+        return {
+            index,
+            text: example.text,
+            tokens: example.tokens,
+            agreementWithout,
+            // `verdictFor` and `agreement` both come from evaluate.ts rather than
+            // being reimplemented. A repository where two features disagree about what
+            // "within the noise" means is one where the answer depends on which
+            // command you ran — and the first draft of this file did exactly that.
+            verdict: verdictFor(selfAgreement, agreementWithout),
+        };
+    }), concurrency);
+    const recoverableTokens = contributions
+        .filter((contribution) => contribution.verdict === 'indistinguishable' || contribution.verdict === 'within-noise')
+        .reduce((sum, contribution) => sum + contribution.tokens, 0);
+    return {
+        provider: provider.name,
+        model: provider.model,
+        selfAgreement,
+        contributions,
+        recoverableTokens,
+        callsMade,
+    };
+}
+//# sourceMappingURL=prune.js.map

package/dist/prune.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prune.js","sourceRoot":"","sources":["../src/prune.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAsE9D;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,MAAc;IAC3D,IAAI,QAAQ,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACzC,6EAA6E;IAC7E,+EAA+E;IAC/E,yCAAyC;IACzC,OAAO,MAAM,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAc,EACd,QAAqC,EACrC,KAAa;IAEb,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QAC5C,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,MAAM;QACrB,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7C,uEAAuE;YACvE,0EAA0E;YAC1E,OAAO,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,IAAI,CAAC,MAAyB;IACrC,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACjG,CAAC;AAED,mFAAmF;AACnF,MAAM,OAAO,cAAe,SAAQ,KAAK;CAAG;AAE5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,MAAyB,EACzB,QAAqB,EACrB,OAAO,GAAiB,EAAE;IAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,IAAI,cAAc,CAAC;IACpD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAE7C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,cAAc,CACtB,kFAAkF,CACnF,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,cAAc,CAAC,mEAAmE,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,GAAG,GAAG,KAAK,EAAE,IAAY,EAAE,KAAa,EAAmB,EAAE;QACjE,SAAS,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzF,CAAC,CAAC;IAEF;;;;;;OAMG;IACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;IACpE,CAAC,CAAC,EACF,WAAW,CACZ,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;IAEjF,MAAM,aAAa,GAAG,MAAM,MAAM,CAChC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAkC,EAAE;QACxE,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,MAAM,MAAM,CAC1B,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAC/D,CAAC,CACF,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAC3B,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CACjF,CAAC;QACF,OAAO;YACL,KAAK;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB;YACd,sEAAsE;YACxE,2EAA2E;YAC3E,oEAAoE;YACpE,uEAAuE;YACvE,OAAO,EAAE,UAAU,CAAC,aAAa,EAAE,gBAAgB,CAAC;SACrD,CAAC;IACJ,CAAC,CAAC,EACF,WAAW,CACZ,CAAC;IAEF,MAAM,iBAAiB,GAAG,aAAa;SACpC,MAAM,CACL,CAAC,YAAY,EAAE,EAAE,CACf,YAAY,CAAC,OAAO,KAAK,mBAAmB,IAAI,YAAY,CAAC,OAAO,KAAK,cAAc,CAC1F;SACA,MAAM,CAAC,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE,CAAC,GAAG,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAE/D,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,aAAa;QACb,aAAa;QACb,iBAAiB;QACjB,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/reorder.d.ts

@@ -0,0 +1,82 @@
+import type { TokenCounter } from './types.js';
+/**
+ * Moving stable instructions in front of the first placeholder.
+ *
+ * This is the largest saving Trazum knows about and the only one it used to
+ * report without acting on. Prompt caching is a byte-for-byte prefix match, so
+ * everything after the first `{{placeholder}}` is re-read at full price on every
+ * call. Measured on a 1,178-token support prompt: 14 tokens cacheable as written,
+ * 1,174 after rearranging the *same content* — $227.65 a month at 50,000 calls.
+ *
+ * No rule can compete with that, because a rule deletes a few percent of tokens
+ * while this changes the price of 98% of them.
+ *
+ * **It is also the most dangerous thing in this repository**, which is why it is
+ * not a rule and not part of `aggressive`. Every other transformation removes
+ * text whose absence is local. This one moves text, and order carries meaning:
+ * "Summarise the text above" is correct where it sits and nonsense in front of
+ * the text it points at. So the whole design here is about what to *refuse*.
+ *
+ * Three refusals, in order of how much they cost:
+ *
+ * 1. **A block containing a backward reference stays put** — and so does
+ *    everything after it. Moving a later block past a pinned one changes their
+ *    order relative to each other, which is the same class of harm.
+ * 2. **Only whole blocks move.** Blocks are separated by blank lines, so a
+ *    sentence is never severed from the paragraph that qualifies it.
+ * 3. **Nothing moves if the prompt has no placeholder**, or if the resulting
+ *    prefix would not clear the model's cacheable minimum anyway — a
+ *    rearrangement that buys nothing is a diff for its own sake.
+ */
+export interface ReorderedBlock {
+    text: string;
+    tokens: number;
+}
+export interface DeclinedBlock {
+    text: string;
+    /**
+     * Why it stayed. `backward-reference` names the phrase found; `after-pinned`
+     * means an earlier block was pinned and moving this one would reorder the two.
+     */
+    reason: 'backward-reference' | 'after-pinned' | 'uncovered-script';
+    /** The phrase that pinned it, for `backward-reference`. */
+    phrase?: string;
+    /** The script with no phrase list, for `uncovered-script`. */
+    script?: string;
+}
+export interface ReorderResult {
+    /** The rearranged prompt, or the original when nothing could move. */
+    text: string;
+    /** Blocks moved ahead of the first placeholder, in their original order. */
+    moved: ReorderedBlock[];
+    /** Blocks left where they were, with the reason. */
+    declined: DeclinedBlock[];
+    /** Tokens that moved from unpriced-every-call into the cacheable prefix. */
+    tokensMoved: number;
+    /** Cacheable prefix before and after, so the gain is visible rather than claimed. */
+    prefixTokensBefore: number;
+    prefixTokensAfter: number;
+}
+export interface ReorderOptions {
+    count?: TokenCounter;
+    /**
+     * Do not rearrange unless the prefix ends up at least this long. Defaults to
+     * 0 — the caller knows the model's cacheable minimum and this module does not.
+     *
+     * The bar is on the **resulting prefix**, not on the amount moved. Those are
+     * different questions, and asking the second one refuses a real saving: a
+     * prompt whose head already clears the minimum gains from any block that joins
+     * it, however small. Asking "did 200 tokens move?" answers "no" and reports
+     * that nothing could move, which is not what happened.
+     */
+    minPrefixTokens?: number;
+}
+/**
+ * Rearranges a prompt so its stable instructions sit in the cacheable prefix.
+ *
+ * Returns the original text unchanged when nothing can safely move, and always
+ * reports what it declined and why — a saving Trazum silently chose not to take
+ * is one the author cannot evaluate.
+ */
+export declare function reorderForCache(prompt: string, options?: ReorderOptions): ReorderResult;
+//# sourceMappingURL=reorder.d.ts.map

package/dist/reorder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reorder.d.ts","sourceRoot":"","sources":["../src/reorder.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,EAAE,oBAAoB,GAAG,cAAc,GAAG,kBAAkB,CAAC;IACnE,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAC;IACb,4EAA4E;IAC5E,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,oDAAoD;IACpD,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAC;IACpB,qFAAqF;IACrF,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAiGD,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,aAAa,CA0H3F"}