@startsimpli/llm 0.4.16 → 0.4.17

package/src/translation/__tests__/quality-gates.test.ts

@@ -0,0 +1,338 @@
+/**
+ * THE MECHANICAL GATES (bd startsim-ob4d, RED items 8–12, 14, 15).
+ *
+ * `startsim-r9q8`'s central move: every measure here compares SOURCE TO OUTPUT
+ * and none of them judges the output's language. That is what makes the product
+ * assessable by the people selling it, neither of whom reads Arabic.
+ *
+ * Their blind spot is stated so nobody mistakes a green run for a good
+ * translation: A TRANSLATION CAN BE 100% TERM-CORRECT AND STILL BE WRONG.
+ * Adherence says the right words appeared. It says nothing about whether the
+ * sentence means what the source meant. That question belongs to the named
+ * reviewer and to gate 4 of the ship floor, and no assertion in this file
+ * substitutes for it.
+ *
+ * Numeric fidelity, do-not-translate and structure are the 100%-or-fail gates;
+ * glossary adherence ships at ≥98% overall because a matcher may legitimately
+ * miss an inflected term. None of that tolerance lives HERE — these are unit
+ * tests over crafted input where the answer is exactly known.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import { stripIsolates } from '@startsimpli/i18n';
+
+import {
+  createTranslationService,
+  extractNumerics,
+  reassembleDocument,
+  renderTranslatePrompt,
+  runMechanicalChecks,
+  segmentDocument,
+  type TranslationRoute,
+} from '../index';
+import type { GlossaryTerm, MemoryUnit, StructuredDocument } from '../types';
+import {
+  DNT_BRAND,
+  GOLDEN_SETS,
+  NUMERICS_IN_FIXTURE,
+  envelopeFor,
+  pairIn,
+  segmentsOf,
+  type GoldenSet,
+} from '../__fixtures__/golden-pairs';
+import { stubProvider } from './stub-provider';
+
+function walledRoute(): TranslationRoute {
+  return {
+    posture: 'walled',
+    provider: 'stub',
+    model: 'stub-model',
+    allowedModels: ['stub-model'],
+    fallback: false,
+  };
+}
+
+describe.each(GOLDEN_SETS)(
+  'mechanical gates [$sourceLocale→$targetLocale]',
+  (set: GoldenSet) => {
+    const binding = set.glossary.find((t) => !t.doNotTranslate)!;
+
+    it('injects only the glossary terms that OCCUR in this batch', () => {
+      // The prompt scales with the TEXT, not with the glossary. A client with a
+      // 4,000-term glossary must not pay for all of it on a two-line page.
+      const absent: GlossaryTerm = {
+        source: 'force majeure',
+        target: 'NOT-IN-THIS-BATCH',
+        binding: true,
+      };
+      const batch = [{ id: 'glossary-term', text: pairIn(set, 'glossary-term').sourceText }];
+
+      const prompt = renderTranslatePrompt(batch, {
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+        glossary: [...set.glossary, absent],
+      });
+
+      expect(prompt).toContain(binding.source);
+      expect(prompt).toContain(binding.target);
+      expect(prompt).not.toContain(absent.source);
+      expect(prompt).not.toContain(absent.target);
+    });
+
+    it('labels memory examples so the model cannot mistake one for input', () => {
+      // `startsim-yfot` names this the single most likely prompt-level failure:
+      // the model translates its own examples.
+      const example: MemoryUnit = {
+        sourceText: 'A previously approved sentence.',
+        targetText: pairIn(set, 'plain').targetText,
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+      };
+      const batch = [{ id: 'plain', text: pairIn(set, 'plain').sourceText }];
+
+      const prompt = renderTranslatePrompt(batch, {
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+        memoryExamples: [example],
+      });
+
+      // The example and the thing to translate must sit in DIFFERENT blocks.
+      const exampleAt = prompt.indexOf(example.sourceText);
+      const inputAt = prompt.indexOf(batch[0]!.text);
+      expect(exampleAt).toBeGreaterThanOrEqual(0);
+      expect(inputAt).toBeGreaterThanOrEqual(0);
+      expect(prompt.slice(0, inputAt)).not.toBe(prompt.slice(0, exampleAt));
+      expect(prompt).toMatch(/<(memory|example)[\s>]/i);
+    });
+
+    it('scores a binding term as adhered when its approved form is present', () => {
+      const pair = pairIn(set, 'glossary-term');
+
+      const checks = runMechanicalChecks(pair.sourceText, pair.targetText, set.glossary);
+
+      expect(checks.glossaryAdherence).toContainEqual({
+        term: binding.source,
+        present: true,
+      });
+      expect(checks.ok).toBe(true);
+    });
+
+    it('scores a binding term as MISSED when the approved form is absent', () => {
+      // The gate has to be able to fail, or it is decoration.
+      const pair = pairIn(set, 'glossary-term');
+      const stripped = pair.targetText.replace(binding.target, 'XXX');
+
+      const checks = runMechanicalChecks(pair.sourceText, stripped, set.glossary);
+
+      expect(checks.glossaryAdherence).toContainEqual({
+        term: binding.source,
+        present: false,
+      });
+      expect(checks.ok).toBe(false);
+    });
+
+    it('requires a do-not-translate brand to survive VERBATIM', () => {
+      const pair = pairIn(set, 'do-not-translate');
+
+      const good = runMechanicalChecks(pair.sourceText, pair.targetText, set.glossary);
+      expect(good.doNotTranslate).toContainEqual({ term: DNT_BRAND, present: true });
+
+      const mangled = pair.targetText.replace(DNT_BRAND, 'نورثويند');
+      const bad = runMechanicalChecks(pair.sourceText, mangled, set.glossary);
+      expect(bad.doNotTranslate).toContainEqual({ term: DNT_BRAND, present: false });
+      expect(bad.ok).toBe(false);
+    });
+
+    it('round-trips numbers, percentages and dates as a multiset', () => {
+      // 100%, non-negotiable: a dropped digit in a business document is a
+      // commercial error, not a stylistic one.
+      const pair = pairIn(set, 'numerics');
+
+      const checks = runMechanicalChecks(pair.sourceText, pair.targetText, set.glossary);
+
+      expect(checks.missingNumerics).toEqual([]);
+      expect(extractNumerics(pair.sourceText).sort()).toEqual(
+        [...NUMERICS_IN_FIXTURE].sort(),
+      );
+    });
+
+    it('catches a dropped numeric', () => {
+      const pair = pairIn(set, 'numerics');
+      const dropped = pair.targetText.replace('$4,200,000', '');
+
+      const checks = runMechanicalChecks(pair.sourceText, dropped, set.glossary);
+
+      expect(checks.missingNumerics).toContain('$4,200,000');
+      expect(checks.ok).toBe(false);
+    });
+
+    it('stores target text with NO bidi isolate characters', () => {
+      // `startsim-jb1z`'s rule: isolation happens at RENDER time. A stored
+      // string carrying U+2066–U+2069 is invisible on screen and very visible
+      // in a failing diff — and it breaks exact memory matching.
+      const provider = stubProvider(envelopeFor(set));
+      const service = createTranslationService(walledRoute(), { provider });
+
+      return service
+        .translate({
+          segments: segmentsOf(set),
+          sourceLocale: set.sourceLocale,
+          targetLocale: set.targetLocale,
+          glossary: set.glossary,
+        })
+        .then((result) => {
+          for (const unit of result.translated) {
+            expect(stripIsolates(unit.targetText)).toBe(unit.targetText);
+          }
+        });
+    });
+
+    it('records the provider, model and prompt version on every unit', () => {
+      // Without provenance a quality regression after a route change is
+      // unattributable, and "translated under the old prompt" becomes a guess.
+      const provider = stubProvider(envelopeFor(set));
+      const service = createTranslationService(walledRoute(), { provider });
+
+      return service
+        .translate({
+          segments: segmentsOf(set),
+          sourceLocale: set.sourceLocale,
+          targetLocale: set.targetLocale,
+        })
+        .then((result) => {
+          for (const unit of result.translated) {
+            expect(unit.provenance.provider).toBe('stub');
+            expect(unit.provenance.model).toBe('stub-model');
+            expect(unit.provenance.promptVersion).toBeTruthy();
+          }
+        });
+    });
+
+    it('serves an exact memory hit and makes ZERO provider calls', async () => {
+      // The cheapest, sharpest test in the suite. An exact hit costs nothing,
+      // which is why the effective rate falls as the memory fills.
+      const pair = pairIn(set, 'plain');
+      const memory: MemoryUnit[] = [
+        {
+          sourceText: pair.sourceText,
+          targetText: pair.targetText,
+          sourceLocale: set.sourceLocale,
+          targetLocale: set.targetLocale,
+        },
+      ];
+      const provider = stubProvider('{"segments":[]}');
+      const service = createTranslationService(walledRoute(), { provider });
+
+      const result = await service.translate({
+        segments: [{ id: 'plain', text: pair.sourceText }],
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+        memory,
+      });
+
+      expect(provider.calls).toHaveLength(0);
+      expect(result.providerCalls).toBe(0);
+      expect(result.translated).toHaveLength(1);
+      expect(result.translated[0]!.fromMemory).toBe(true);
+      expect(result.translated[0]!.targetText).toBe(pair.targetText);
+    });
+  },
+);
+
+/**
+ * STRUCTURE IS DETERMINISTIC BY CONSTRUCTION, because `startsim-jb1z` keeps it
+ * out of the model entirely: the caller segments, the caller reassembles, and
+ * the model only ever sees leaf text. So this is a pure round-trip and there is
+ * no excuse for it being anything but exact.
+ */
+describe('structure round-trip', () => {
+  const document: StructuredDocument = {
+    nodes: [
+      { kind: 'heading', text: 'Quarterly Report' },
+      {
+        kind: 'list',
+        children: [
+          { kind: 'list_item', text: 'Revenue grew 12.5%.' },
+          { kind: 'list_item', text: 'Headcount held flat.' },
+        ],
+      },
+      {
+        kind: 'table',
+        children: [
+          {
+            kind: 'table_row',
+            children: [
+              { kind: 'table_cell', text: 'Region' },
+              { kind: 'table_cell', text: 'Total' },
+            ],
+          },
+        ],
+      },
+      { kind: 'paragraph', text: 'See the enclosed contract.' },
+      // A product code is not prose. It must survive without being offered to
+      // the model at all.
+      { kind: 'code', text: 'NW-LOG-4420-X', translatable: false },
+    ],
+  };
+
+  it('extracts only translatable leaves, and never the untranslatable ones', () => {
+    const { segments } = segmentDocument(document);
+
+    const texts = segments.map((s) => s.text);
+    expect(texts).toContain('Quarterly Report');
+    expect(texts).toContain('Revenue grew 12.5%.');
+    expect(texts).toContain('Region');
+    expect(texts).not.toContain('NW-LOG-4420-X');
+  });
+
+  it('returns the ORIGINAL shape when the leaves are put back unchanged', () => {
+    const { segments, skeleton } = segmentDocument(document);
+    const identity = new Map(segments.map((s) => [s.id, s.text]));
+
+    expect(reassembleDocument(skeleton, identity)).toEqual(document);
+  });
+
+  it('changes only the leaf text when the leaves are translated', () => {
+    const { segments, skeleton } = segmentDocument(document);
+    const translated = new Map(segments.map((s) => [s.id, `⟦${s.text}⟧`]));
+
+    const out = reassembleDocument(skeleton, translated);
+
+    // Same node tree, same kinds, same nesting — only text differs.
+    const kinds = (doc: StructuredDocument): string[] =>
+      doc.nodes.flatMap(function walk(node): string[] {
+        return [node.kind, ...(node.children ?? []).flatMap(walk)];
+      });
+    expect(kinds(out)).toEqual(kinds(document));
+    expect(out.nodes[4]!.text).toBe('NW-LOG-4420-X');
+    expect(out.nodes[0]!.text).toBe('⟦Quarterly Report⟧');
+  });
+});
+
+/**
+ * THE NAMING BAN, ENFORCED (bd startsim-jb1z, via bd startsim-bfem).
+ *
+ * A guard rather than a RED assertion — it passes today because the stubs are
+ * empty, and its job is to fail the day GREEN writes `if (target === 'ar')`.
+ */
+describe('the engine is language-agnostic', () => {
+  it('contains no locale literal anywhere in the translation engine', async () => {
+    const { readdir, readFile } = await import('node:fs/promises');
+    const { join } = await import('node:path');
+    const root = new URL('..', import.meta.url).pathname;
+
+    const entries = (await readdir(root, { withFileTypes: true }))
+      .filter((entry) => entry.isFile() && entry.name.endsWith('.ts'));
+    expect(entries.length).toBeGreaterThan(0);
+
+    for (const entry of entries) {
+      const source = await readFile(join(root, entry.name), 'utf8');
+      const code = source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
+      // A locale compared as a STRING is the leak. Locales are data.
+      expect(code, `${entry.name} branches on a locale code`).not.toMatch(
+        /===\s*['"](ar|zh|en|he|fa|ur)(-[A-Za-z0-9]+)?['"]/,
+      );
+    }
+  });
+});

package/src/translation/__tests__/route-confidentiality.test.ts

@@ -0,0 +1,228 @@
+/**
+ * ROUTING AND THE FAILURE PATH (bd startsim-ob4d, RED items 16–18).
+ *
+ * `startsim-bfem`: "Failure-path assertions matter as much as happy-path ones,
+ * because the failure path is where the confidentiality guarantee actually
+ * breaks." The named precedent is real and still in the estate — a Django
+ * handler that logs the model's raw response content on a JSON-parse failure.
+ * If translation reproduces that, the wall `startsim-hopl` sells is a comment.
+ *
+ * The three properties pinned here:
+ *
+ *   16. A malformed envelope is retried, then FAILS — and no response text and
+ *       no client text reaches any log, on any path.
+ *   17. A `walled` route whose provider will not answer FAILS. It never crosses
+ *       to a vendor under different retention terms. This is a deliberate
+ *       override of `LLMService.getProvider()`'s "fall back to any available",
+ *       whose `isAvailable()` is `Boolean(apiKey)` — and prod is on record
+ *       holding an OpenAI key in `ANTHROPIC_API_KEY`, so a deployment that
+ *       believes it is walled can route client text to another vendor with
+ *       nothing saying so.
+ *   18. The service REFUSES TO BOOT on a model outside the deployment's
+ *       allow-list. `claude-fable-5` is the named case: a Covered Model that
+ *       cannot run under zero-retention terms, advertised by our own provider
+ *       today. A live foot-gun, not a hypothetical.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import {
+  EnvelopeParseError,
+  TranslationConfigError,
+  WalledRouteUnavailableError,
+  createTranslationService,
+  parseTranslationEnvelope,
+  resolveTranslationRoute,
+  type TranslationRoute,
+} from '../index';
+import { GOLDEN_SETS, pairIn, segmentsOf } from '../__fixtures__/golden-pairs';
+import { capturingLogger, stubProvider } from './stub-provider';
+
+/** One locale is enough here — nothing in this file is language-sensitive. */
+const SET = GOLDEN_SETS[0]!;
+
+function walledRoute(overrides: Partial<TranslationRoute> = {}): TranslationRoute {
+  return {
+    posture: 'walled',
+    provider: 'stub',
+    model: 'stub-model',
+    allowedModels: ['stub-model'],
+    fallback: false,
+    ...overrides,
+  };
+}
+
+function request() {
+  return {
+    segments: segmentsOf(SET),
+    sourceLocale: SET.sourceLocale,
+    targetLocale: SET.targetLocale,
+    glossary: SET.glossary,
+  };
+}
+
+describe('a malformed envelope', () => {
+  /**
+   * Prose wrapped around the client's own sentence — what a model actually
+   * returns when it ignores the JSON instruction. The client text is IN here on
+   * purpose: it is the needle the log assertion searches for.
+   */
+  const CLIENT_SENTENCE = pairIn(SET, 'glossary-term').sourceText;
+  const GARBAGE = `Sure! Here is the translation of "${CLIENT_SENTENCE}" — I hope it helps.`;
+
+  it('is retried before the batch is failed', async () => {
+    const provider = stubProvider(GARBAGE);
+    const service = createTranslationService(walledRoute(), {
+      provider,
+      maxEnvelopeRetries: 2,
+    });
+
+    await expect(service.translate(request())).rejects.toThrow(EnvelopeParseError);
+    // One original attempt plus the permitted retries — a parse failure is
+    // worth asking again for, once, with a correction prompt.
+    expect(provider.calls.length).toBeGreaterThan(1);
+    expect(provider.calls.length).toBeLessThanOrEqual(3);
+  });
+
+  it('puts NO response text and NO client text in any log', async () => {
+    const logger = capturingLogger();
+    const provider = stubProvider(GARBAGE);
+    const service = createTranslationService(walledRoute(), {
+      provider,
+      logger,
+      maxEnvelopeRetries: 1,
+    });
+
+    await expect(service.translate(request())).rejects.toThrow(EnvelopeParseError);
+
+    // Asserted against the FIXTURE TEXT rather than by inspecting call sites,
+    // which is the only version of this check that cannot rot.
+    const logged = logger.all();
+    expect(logged).not.toContain(CLIENT_SENTENCE);
+    expect(logged).not.toContain(GARBAGE);
+    for (const pair of SET.pairs) {
+      if (pair.sourceText) expect(logged).not.toContain(pair.sourceText);
+      if (pair.targetText) expect(logged).not.toContain(pair.targetText);
+    }
+  });
+
+  it('carries no response text in the error it throws, either', async () => {
+    // An exception is a log line waiting to happen: it will be caught, printed,
+    // and shipped to an aggregator by code that has never read this file.
+    const provider = stubProvider(GARBAGE);
+    const service = createTranslationService(walledRoute(), {
+      provider,
+      maxEnvelopeRetries: 1,
+    });
+
+    await expect(service.translate(request())).rejects.toThrow(EnvelopeParseError);
+    await service.translate(request()).catch((error: unknown) => {
+      const text = `${(error as Error).message}\n${(error as Error).stack ?? ''}`;
+      expect(text).not.toContain(CLIENT_SENTENCE);
+      expect(text).not.toContain(GARBAGE);
+    });
+  });
+
+  it('rejects unparseable JSON as malformed rather than guessing', () => {
+    const parsed = parseTranslationEnvelope(GARBAGE, ['plain']);
+
+    expect(parsed.ok).toBe(false);
+    if (parsed.ok) return;
+    expect(parsed.reason).toBe('malformed');
+  });
+});
+
+describe('the walled posture', () => {
+  it('FAILS when the configured provider is unavailable, rather than falling back', async () => {
+    const provider = stubProvider('{"segments":[]}', { available: false });
+    const service = createTranslationService(walledRoute(), { provider });
+
+    await expect(service.translate(request())).rejects.toThrow(
+      WalledRouteUnavailableError,
+    );
+    // It must not even have tried: the refusal is the correct outcome, and the
+    // job is retried later against the SAME provider.
+    expect(provider.calls).toHaveLength(0);
+  });
+
+  it('refuses at BOOT to be configured with fallback enabled', () => {
+    // Posture is a property of the deployment and fallback is a property of the
+    // posture. `walled` + `fallback: true` is not a preference, it is a breach
+    // waiting for an outage.
+    expect(() =>
+      createTranslationService(walledRoute({ fallback: true }), {
+        provider: stubProvider('{}'),
+      }),
+    ).toThrow(TranslationConfigError);
+  });
+
+  it('surfaces a provider refusal as a failure, never as an empty document', async () => {
+    const provider = stubProvider('{}', {
+      throws: new Error('provider refused'),
+    });
+    const service = createTranslationService(walledRoute(), { provider });
+
+    await expect(service.translate(request())).rejects.toThrow();
+  });
+});
+
+describe('the boot-time model allow-list', () => {
+  it('refuses to build a service whose model is outside the allow-list', () => {
+    expect(() =>
+      createTranslationService(
+        walledRoute({ model: 'claude-fable-5', allowedModels: ['stub-model'] }),
+        { provider: stubProvider('{}') },
+      ),
+    ).toThrow(TranslationConfigError);
+  });
+
+  it('refuses a Covered Model on a walled deployment, by name', () => {
+    // Fail the boot, not the first client document.
+    expect(() =>
+      resolveTranslationRoute({
+        TRANSLATION_POSTURE: 'walled',
+        TRANSLATION_PROVIDER: 'anthropic',
+        TRANSLATION_MODEL: 'claude-fable-5',
+      }),
+    ).toThrow(TranslationConfigError);
+
+    expect(() =>
+      resolveTranslationRoute({
+        TRANSLATION_POSTURE: 'walled',
+        TRANSLATION_PROVIDER: 'anthropic',
+        TRANSLATION_MODEL: 'claude-mythos-5',
+      }),
+    ).toThrow(TranslationConfigError);
+  });
+
+  it('resolves a walled route with fallback OFF, whatever the environment asks for', () => {
+    const route = resolveTranslationRoute({
+      TRANSLATION_POSTURE: 'walled',
+      TRANSLATION_PROVIDER: 'anthropic',
+      TRANSLATION_MODEL: 'claude-sonnet-5',
+      TRANSLATION_ALLOWED_MODELS: 'claude-sonnet-5',
+      // Asked for, and ignored: the posture decides.
+      TRANSLATION_FALLBACK: 'true',
+    });
+
+    expect(route.posture).toBe('walled');
+    expect(route.fallback).toBe(false);
+    expect(route.model).toBe('claude-sonnet-5');
+  });
+
+  it('permits fallback on the open posture', () => {
+    // OGMC's marketing copy and MCR's artist metadata are not confidential in
+    // the sense the client's material is. Forcing the whole estate through the
+    // walled route would be cargo-culting.
+    const route = resolveTranslationRoute({
+      TRANSLATION_POSTURE: 'open',
+      TRANSLATION_PROVIDER: 'anthropic',
+      TRANSLATION_MODEL: 'claude-sonnet-5',
+      TRANSLATION_ALLOWED_MODELS: 'claude-sonnet-5',
+      TRANSLATION_FALLBACK: 'true',
+    });
+
+    expect(route.posture).toBe('open');
+    expect(route.fallback).toBe(true);
+  });
+});

package/src/translation/__tests__/stub-provider.ts

@@ -0,0 +1,118 @@
+/**
+ * A PURPOSE-BUILT provider stub for the translation suite.
+ *
+ * `MockLLMProvider` is deliberately not used here, and `startsim-bfem` says why:
+ * its behaviour is set through a module-global mutable (`let mockBehavior`, set
+ * via `setMockBehavior()`) carrying a single `customResponse` string. That is
+ * fine for one-response tests and wrong for these, which need a DIFFERENT
+ * envelope per call and must not leak state between parallel test files.
+ *
+ * The engine takes an `ILLMProvider` by injection, so a local stub costs one
+ * file and buys isolation. That is the right trade.
+ */
+
+import type { ILLMProvider } from '../../providers/interface';
+import type { GenerateOptions, LLMResponse } from '../../types';
+import type { TranslationLogger } from '../types';
+
+export interface RecordedCall {
+  userPrompt: string;
+  systemPrompt: string;
+  options?: GenerateOptions;
+}
+
+export interface StubProvider extends ILLMProvider {
+  /** Every call, in order, so a test can assert the BATCH count and the prompt. */
+  readonly calls: RecordedCall[];
+}
+
+export interface StubProviderOptions {
+  name?: string;
+  model?: string;
+  /** `isAvailable()`. False models a provider that is down or unconfigured. */
+  available?: boolean;
+  /** Thrown instead of answering — a provider that refuses. */
+  throws?: Error;
+}
+
+/**
+ * @param script What to return, per call. A string array is consumed in order
+ *   and the last entry repeats; a function receives the 0-based call index.
+ */
+export function stubProvider(
+  script: string | string[] | ((callIndex: number, call: RecordedCall) => string),
+  options: StubProviderOptions = {},
+): StubProvider {
+  const {
+    name = 'stub',
+    model = 'stub-model',
+    available = true,
+    throws,
+  } = options;
+  const calls: RecordedCall[] = [];
+
+  const pick = (index: number, call: RecordedCall): string => {
+    if (typeof script === 'function') return script(index, call);
+    if (typeof script === 'string') return script;
+    return script[Math.min(index, script.length - 1)] ?? '';
+  };
+
+  return {
+    name,
+    availableModels: [model],
+    defaultModel: model,
+    calls,
+    isAvailable: () => available,
+    async generate(
+      userPrompt: string,
+      systemPrompt: string,
+      generateOptions?: GenerateOptions,
+    ): Promise<LLMResponse> {
+      const call: RecordedCall = { userPrompt, systemPrompt, options: generateOptions };
+      calls.push(call);
+      if (throws) throw throws;
+      return {
+        content: pick(calls.length - 1, call),
+        usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+        model,
+      };
+    },
+  };
+}
+
+export interface CapturingLogger extends TranslationLogger {
+  /** Every line and every piece of metadata, flattened for substring search. */
+  readonly written: string[];
+  /** One string carrying everything logged, for a single `not.toContain`. */
+  all(): string;
+}
+
+/**
+ * A logger that keeps everything, so "no client text reaches a log" can be
+ * ASSERTED against the fixture text rather than checked by reading call sites.
+ *
+ * The metadata is serialised into the same haystack on purpose: the leak this
+ * guards against is a raw provider response passed as a structured field, which
+ * is exactly what the Django precedent at `messages/services/llm.py` does.
+ */
+export function capturingLogger(): CapturingLogger {
+  const written: string[] = [];
+  const record = (level: string) => (message: string, meta?: Record<string, unknown>) => {
+    written.push(`${level} ${message}`);
+    if (meta !== undefined) {
+      try {
+        written.push(JSON.stringify(meta));
+      } catch {
+        written.push(String(meta));
+      }
+    }
+  };
+  return {
+    written,
+    all: () => written.join('\n'),
+    debug: record('debug'),
+    info: record('info'),
+    warn: record('warn'),
+    error: record('error'),
+  };
+}