@trazum/core 1.8.0

package/src/promptfoo.ts

@@ -0,0 +1,213 @@
+import { getModel } from './pricing.js';
+import { segment } from './segment.js';
+
+/**
+ * Exports a before/after pair as a suite somebody else's harness can run.
+ *
+ * `trazum eval` answers "does the model still say the same thing?" — semantic
+ * agreement, measured against the model's own variance. That is the question
+ * Trazum is qualified to ask, and it is not the question a team actually needs
+ * answered before shipping. *Their* question is whether the classifier still
+ * hits 94%, whether the JSON still parses, whether the refusal rate moved. Those
+ * are assertions about their task, and Trazum has no business inventing them.
+ *
+ * So this hands over the part it *can* build correctly — a suite in which the
+ * only variable is the prompt, with both versions and every case already wired
+ * — and leaves the assertions where they belong.
+ *
+ * ## Why JSON rather than YAML
+ *
+ * promptfoo reads `promptfooconfig.json` as readily as the YAML. This package
+ * has no dependencies and is not going to acquire a YAML emitter, and a
+ * hand-rolled one would be a quoting bug waiting for the first prompt
+ * containing a colon, a tab, or a line ending in a space. `JSON.stringify`
+ * escapes everything correctly, and JSON is a subset of YAML, so the file can
+ * be renamed if somebody prefers.
+ */
+
+export interface PromptfooExport {
+  /** The config, ready for `JSON.stringify`. */
+  config: Record<string, unknown>;
+  /** Things the reader has to know before trusting the run. */
+  warnings: PromptfooWarning[];
+}
+
+export type PromptfooWarning =
+  /** `${x}` or `{x}`: promptfoo substitutes `{{x}}` and will leave these alone. */
+  | { kind: 'unsupported-placeholder'; detail: string }
+  /** More than one distinct placeholder, and one value per case to fill them. */
+  | { kind: 'multiple-placeholders'; detail: string }
+  /** The provider has no promptfoo id here, so the model string is a guess. */
+  | { kind: 'unmapped-provider'; detail: string }
+  /** No placeholder at all: the case is appended, as `trazum eval` does. */
+  | { kind: 'appended-input'; detail: string };
+
+export interface PromptfooOptions {
+  /** Model id from Trazum's catalogue, mapped to a promptfoo provider. */
+  model?: string;
+  /** Rule level used to produce the optimised prompt, for the label. */
+  level?: string;
+}
+
+/** The variable a prompt with no `{{placeholder}}` gets. */
+const APPENDED_VAR = 'input';
+
+/**
+ * promptfoo names providers `<vendor>:<model>`. Only the two vendors whose
+ * shape is certain are mapped; anything else is emitted as-is with a warning,
+ * because a wrong provider id fails at run time with a message about the
+ * harness rather than about this file.
+ */
+function providerId(modelId: string | undefined): { id: string; warning: string | null } {
+  if (modelId === undefined) return { id: 'openai:gpt-4o-mini', warning: null };
+  const model = getModel(modelId);
+  switch (model.provider) {
+    case 'anthropic':
+      return { id: `anthropic:messages:${model.id}`, warning: null };
+    case 'openai':
+      return { id: `openai:${model.id}`, warning: null };
+    default:
+      return {
+        id: model.id,
+        warning: `${model.provider ?? 'this provider'} has no known promptfoo id; "${model.id}" is a guess`,
+      };
+  }
+}
+
+/** Placeholders in the prompt, in order of appearance, deduplicated. */
+function placeholdersIn(prompt: string): string[] {
+  const found = segment(prompt)
+    .filter((piece) => piece.kind === 'protected' && piece.protection === 'placeholder')
+    .map((piece) => piece.text);
+  return [...new Set(found)];
+}
+
+/** `{{query}}` → `query`, or null when promptfoo would not substitute it. */
+function varNameOf(placeholder: string): string | null {
+  const match = /^\{\{\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}$/.exec(placeholder);
+  return match?.[1] ?? null;
+}
+
+
+/**
+ * Whether the prompt demands JSON output.
+ *
+ * Defined narrowly and checkably: a fenced block tagged `json`, or an untagged
+ * fenced block whose body parses as JSON. Nothing about the surrounding prose,
+ * because "return JSON" in a sentence is a phrase, and a phrase is where
+ * guessing starts.
+ *
+ * The first version of this asked `findRestatedFormat`, which was the wrong
+ * question wearing a convenient shape. That function answers "is this prompt
+ * wasting tokens restating its own schema?" — so a prompt demanding JSON
+ * *cleanly* got no assertion while a wasteful one did, which is exactly
+ * backwards.
+ */
+function asksForJson(prompt: string): boolean {
+  const fenced = /^[ \t]*(?:```|~~~)([A-Za-z0-9]*)[ \t]*\n([\s\S]*?)^[ \t]*(?:```|~~~)/gm;
+
+  let match: RegExpExecArray | null;
+  while ((match = fenced.exec(prompt)) !== null) {
+    const lang = (match[1] ?? '').toLowerCase();
+    const body = (match[2] ?? '').trim();
+    if (/^jsonc?5?$/.test(lang)) return true;
+    if (lang !== '' ) continue;
+    if (!body.startsWith('{') && !body.startsWith('[')) continue;
+    try {
+      JSON.parse(body);
+      return true;
+    } catch {
+      // An untagged block of something else. Not our business.
+    }
+  }
+  return false;
+}
+
+export function toPromptfoo(
+  original: string,
+  optimized: string,
+  cases: readonly string[],
+  options: PromptfooOptions = {},
+): PromptfooExport {
+  const warnings: PromptfooWarning[] = [];
+
+  const placeholders = placeholdersIn(original);
+  const supported = placeholders.map(varNameOf);
+
+  // The variable the cases fill. `trazum eval` substitutes the *first*
+  // placeholder and appends when there is none; promptfoo is driven the same
+  // way so the two commands are testing the same prompt.
+  let variable = APPENDED_VAR;
+  let originalTemplate = original;
+  let optimizedTemplate = optimized;
+
+  if (placeholders.length === 0) {
+    variable = APPENDED_VAR;
+    originalTemplate = `${original.trimEnd()}\n\n{{${APPENDED_VAR}}}`;
+    optimizedTemplate = `${optimized.trimEnd()}\n\n{{${APPENDED_VAR}}}`;
+    warnings.push({
+      kind: 'appended-input',
+      detail:
+        'This prompt has no placeholder, so each case is appended at the end — the same ' +
+        'thing `trazum eval` does. If the prompt is meant to be a template, add ' +
+        '{{a_variable}} and export again.',
+    });
+  } else {
+    const first = supported[0] ?? null;
+    if (first === null) {
+      warnings.push({
+        kind: 'unsupported-placeholder',
+        detail:
+          `promptfoo substitutes {{name}}; this prompt uses ${placeholders[0]}, which it will ` +
+          'leave untouched. Every case would run against the literal template.',
+      });
+      variable = APPENDED_VAR;
+    } else {
+      variable = first;
+    }
+
+    const distinct = placeholders.length;
+    if (distinct > 1) {
+      warnings.push({
+        kind: 'multiple-placeholders',
+        detail:
+          `The prompt has ${distinct} placeholders (${placeholders.join(', ')}) and the case ` +
+          `file supplies one value each, which fills ${placeholders[0]} only. Give the others ` +
+          'defaults in `defaultTest.vars`, or the run tests a prompt nobody sends.',
+      });
+    }
+  }
+
+  const { id, warning } = providerId(options.model);
+  if (warning !== null) warnings.push({ kind: 'unmapped-provider', detail: warning });
+
+  // The one assertion Trazum can make without guessing at the task: if the
+  // prompt asks for JSON, both versions still have to produce it. Everything
+  // else — accuracy, refusal rate, format specifics — is the team's to write,
+  // and inventing it here would be a tool with opinions about somebody else's
+  // product.
+  const assertions = asksForJson(original) ? [{ type: 'is-json' }] : [];
+
+  const label = options.level ? `after (trazum, ${options.level})` : 'after (trazum)';
+
+  return {
+    config: {
+      description:
+        'Generated by trazum. The only difference between the two prompts is the ' +
+        'optimisation — same provider, same cases, same everything else.\n\n' +
+        'The assertions are yours to write: `trazum eval` already answers "does the model ' +
+        'still say the same thing", and this suite exists for the question it cannot ask — ' +
+        'whether your accuracy, your format and your refusal rate survived. Add them under ' +
+        '`defaultTest.assert` or per test.\n\n' +
+        'Docs: https://www.promptfoo.dev/docs/configuration/expected-outputs/',
+      prompts: [
+        { label: 'before', raw: originalTemplate },
+        { label, raw: optimizedTemplate },
+      ],
+      providers: [id],
+      ...(assertions.length > 0 ? { defaultTest: { assert: assertions } } : {}),
+      tests: cases.map((input) => ({ vars: { [variable]: input } })),
+    },
+    warnings,
+  };
+}

package/src/prune.ts

@@ -0,0 +1,211 @@
+import { findExamples } from './structure.js';
+import { estimateTokens } from './tokenizer.js';
+import { agreement, pooled, verdictFor } from './evaluate.js';
+import type { EvalVerdict } from './evaluate.js';
+import type { LlmProvider, TokenCounter } from './types.js';
+
+/**
+ * Which few-shot examples earn their tokens, measured rather than guessed.
+ *
+ * The `redundant-examples` advisory answers a *textual* question: does this
+ * example look like an earlier one? That catches the way few-shot blocks actually
+ * grow — copy the last one, change two fields — and it is the cheapest useful
+ * thing to say, because it costs nothing.
+ *
+ * This answers a different and much stronger question: **does removing this
+ * example change any answer?** Two examples can be textually unalike and teach the
+ * same thing, and a block nobody has measured is usually where the tokens are: a
+ * few-shot section is routinely most of a prompt.
+ *
+ * The method is leave-one-out against the prompt's own noise floor. Ask the full
+ * prompt twice to find out how much the model disagrees with *itself*, then remove
+ * one example and ask again. If the answer moves no further than the model already
+ * moves on its own, that example is not doing observable work.
+ *
+ * **It spends the caller's money and must never run by default.** The bill is
+ * `2 + examples` calls per input, and `plannedCalls` exists so a caller can print
+ * the figure before deciding rather than discovering it afterwards.
+ *
+ * **What it cannot tell you.** An example may exist for a case these inputs do not
+ * contain — the boundary condition somebody hit in production last March and added
+ * a demonstration for. Removing it would change nothing measurable here and break
+ * that case. So this reports "no effect on these inputs", never "delete this", and
+ * the wording is deliberate: the strength of the claim is bounded by the inputs
+ * given, and only the caller knows whether those cover what matters.
+ */
+
+export interface ExampleContribution {
+  /** Position in the prompt's example block, from zero. */
+  index: number;
+  /** The example itself, so a report can quote its first line. */
+  text: string;
+  tokens: number;
+  /**
+   * Mean agreement between the full prompt's answer and the answer with this
+   * example removed, across every input.
+   */
+  agreementWithout: number;
+  /**
+   * `indistinguishable` and `within-noise` both mean the removal changed nothing
+   * this measurement can see. `diverges` means it did. `inconclusive` means the
+   * model disagreed with itself too much for any of this to mean anything.
+   */
+  verdict: EvalVerdict;
+}
+
+export interface PruneReport {
+  provider: string;
+  model: string;
+  /** The model's agreement with itself, given the full prompt. The yardstick. */
+  selfAgreement: number;
+  contributions: ExampleContribution[];
+  /** Tokens held by examples whose removal changed nothing observable. */
+  recoverableTokens: number;
+  /** Calls actually made, so the bill is never a surprise. */
+  callsMade: number;
+}
+
+export interface PruneOptions {
+  concurrency?: number;
+  countTokens?: TokenCounter;
+}
+
+/**
+ * What this will cost, before it costs it.
+ *
+ * Pure and exported so a CLI can print the number and let somebody say no. A
+ * feature that spends money and only reports the total afterwards is a feature
+ * people run once.
+ */
+export function plannedCalls(examples: number, inputs: number): number {
+  if (examples < 2 || inputs < 1) return 0;
+  // Two baseline runs per input to establish the noise floor, then one run per
+  // example removed. The baselines are shared across every example, which is the
+  // only reason this is affordable at all.
+  return inputs * (2 + examples);
+}
+
+/**
+ * The prompt with one example block removed, located by position rather than by
+ * text.
+ *
+ * `prompt.replace(block.text, '')` would be shorter and wrong: two identical
+ * example blocks — which is exactly what a copy-paste few-shot section contains —
+ * would both match the first occurrence, so removing the second would silently
+ * remove the first and the measurement would describe a prompt nobody asked about.
+ * Scanning forward from the end of the previous block gives each block its true
+ * offset.
+ */
+export function withoutExample(
+  prompt: string,
+  examples: readonly { text: string }[],
+  index: number,
+): string {
+  let cursor = 0;
+  for (let position = 0; position < examples.length; position++) {
+    const text = examples[position]?.text ?? '';
+    const at = prompt.indexOf(text, cursor);
+    if (at === -1) break;
+    if (position === index) {
+      const before = prompt.slice(0, at);
+      const after = prompt.slice(at + text.length);
+      // Collapse the blank lines the removal leaves behind, so the resulting
+      // prompt is one an author might have written rather than one with a hole.
+      return `${before}${after}`.replace(/\n{3,}/g, '\n\n');
+    }
+    cursor = at + text.length;
+  }
+  return prompt;
+}
+
+function mean(values: readonly number[]): number {
+  return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+/** Thrown rather than returned: there is nothing to report and a reason to say. */
+export class NothingToPrune extends Error {}
+
+export async function pruneExamples(
+  prompt: string,
+  inputs: readonly string[],
+  provider: LlmProvider,
+  options: PruneOptions = {},
+): Promise<PruneReport> {
+  const count = options.countTokens ?? estimateTokens;
+  const concurrency = Math.max(1, options.concurrency ?? 3);
+  const examples = findExamples(prompt, count);
+
+  if (examples.length < 2) {
+    throw new NothingToPrune(
+      'this prompt has fewer than two few-shot examples, so there is nothing to compare',
+    );
+  }
+  if (inputs.length === 0) {
+    throw new NothingToPrune('no inputs were given, and leave-one-out needs something to run on');
+  }
+
+  let callsMade = 0;
+  const run = async (text: string, input: string): Promise<string> => {
+    callsMade++;
+    return provider.complete({ system: text.replaceAll('{{input}}', input), user: input });
+  };
+
+  /**
+   * Baselines first, and sequentially within an input.
+   *
+   * The two runs exist to measure the model's own variance, and issuing them
+   * together invites a provider to serve one from a cache and report a variance of
+   * zero — which would make every example look load-bearing.
+   */
+  const baselines = await pooled(
+    inputs.map((input) => async () => {
+      const first = await run(prompt, input);
+      const second = await run(prompt, input);
+      return { input, first, selfSimilarity: agreement(first, second) };
+    }),
+    concurrency,
+  );
+
+  const selfAgreement = mean(baselines.map((baseline) => baseline.selfSimilarity));
+
+  const contributions = await pooled(
+    examples.map((example, index) => async (): Promise<ExampleContribution> => {
+      const reduced = withoutExample(prompt, examples, index);
+      const answers = await pooled(
+        baselines.map((baseline) => () => run(reduced, baseline.input)),
+        1,
+      );
+      const agreementWithout = mean(
+        answers.map((answer, position) => agreement(baselines[position]!.first, answer)),
+      );
+      return {
+        index,
+        text: example.text,
+        tokens: example.tokens,
+        agreementWithout,
+          // `verdictFor` and `agreement` both come from evaluate.ts rather than
+        // being reimplemented. A repository where two features disagree about what
+        // "within the noise" means is one where the answer depends on which
+        // command you ran — and the first draft of this file did exactly that.
+        verdict: verdictFor(selfAgreement, agreementWithout),
+      };
+    }),
+    concurrency,
+  );
+
+  const recoverableTokens = contributions
+    .filter(
+      (contribution) =>
+        contribution.verdict === 'indistinguishable' || contribution.verdict === 'within-noise',
+    )
+    .reduce((sum, contribution) => sum + contribution.tokens, 0);
+
+  return {
+    provider: provider.name,
+    model: provider.model,
+    selfAgreement,
+    contributions,
+    recoverableTokens,
+    callsMade,
+  };
+}