@startsimpli/llm 0.4.16 → 0.4.18

package/src/translation/__tests__/batch-contract.test.ts

@@ -0,0 +1,247 @@
+/**
+ * THE BATCH AND ID-ECHO CONTRACT (bd startsim-ob4d, RED items 1–7, 13, 15).
+ *
+ * `startsim-yfot` states the contract and this file is its enforcement. The
+ * rules being pinned down, in the design's own terms:
+ *
+ *   1. Ids are OPAQUE and round-trip byte-for-byte.
+ *   2. The returned id set EQUALS the requested id set.
+ *   3. A MISSING id fails only its own segment, which is re-queued.
+ *   4. An UNKNOWN id fails the WHOLE batch. Never map positionally.
+ *   5. DUPLICATE ids fail the whole batch.
+ *   6. ORDER is not part of the contract — reassembly is by id.
+ *   7. EMPTY output for a present id is a failure, not an empty translation.
+ *
+ * Rules 4 and 5 are the ones that look like parse pedantry and are not. A model
+ * that invents an id has attributed text to a segment that did not ask for it,
+ * and in a product sold on "one client's material never appears against
+ * another's" that is a correctness alarm.
+ *
+ * EVERY CASE RUNS TWICE, over an RTL and an LTR target, from ONE body. That is
+ * `startsim-bfem`'s enforcement of `startsim-jb1z`'s naming ban: a test that
+ * needs to know which locale it is running has caught a leak.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import {
+  createTranslationService,
+  parseTranslationEnvelope,
+  planBatches,
+  type TranslationRoute,
+} from '../index';
+import { DEFAULT_BATCH_LIMITS, type TranslationSegment } from '../types';
+import {
+  GOLDEN_SETS,
+  envelopeFor,
+  pairIn,
+  segmentsOf,
+  type GoldenSet,
+} from '../__fixtures__/golden-pairs';
+import { stubProvider } from './stub-provider';
+
+/**
+ * A route literal rather than `resolveTranslationRoute(...)`, so a failure in
+ * THIS file always means the batch contract is unimplemented — never that
+ * configuration parsing is. Routing has its own suite.
+ */
+function walledRoute(): TranslationRoute {
+  return {
+    posture: 'walled',
+    provider: 'stub',
+    model: 'stub-model',
+    allowedModels: ['stub-model'],
+    fallback: false,
+  };
+}
+
+function requestFor(set: GoldenSet) {
+  return {
+    segments: segmentsOf(set),
+    sourceLocale: set.sourceLocale,
+    targetLocale: set.targetLocale,
+    glossary: set.glossary,
+  };
+}
+
+describe.each(GOLDEN_SETS)(
+  'batch + id echo [$sourceLocale→$targetLocale]',
+  (set: GoldenSet) => {
+    const ids = set.pairs.map((pair) => pair.id);
+
+    it('packs every segment into exactly ONE provider call', async () => {
+      // The batching rule made enforceable. `packages/llm/README.md`: "Pack N
+      // items into one prompt with echoed ids — never loop".
+      const provider = stubProvider(envelopeFor(set));
+      const service = createTranslationService(walledRoute(), { provider });
+
+      const result = await service.translate(requestFor(set));
+
+      expect(provider.calls).toHaveLength(1);
+      expect(result.providerCalls).toBe(1);
+    });
+
+    it('returns every requested id, byte-identical', async () => {
+      const provider = stubProvider(envelopeFor(set));
+      const service = createTranslationService(walledRoute(), { provider });
+
+      const result = await service.translate(requestFor(set));
+
+      // Set equality, not containment: no more, no fewer.
+      expect([...result.translated.map((s) => s.id)].sort()).toEqual([...ids].sort());
+    });
+
+    it('fails ONLY the missing segment and re-queues it individually', async () => {
+      const kept = ids.filter((id) => id !== 'glossary-term');
+      // First call omits one id; the re-queued single-segment call answers it.
+      const provider = stubProvider([
+        envelopeFor(set, kept),
+        envelopeFor(set, ['glossary-term']),
+      ]);
+      const service = createTranslationService(walledRoute(), { provider });
+
+      const result = await service.translate(requestFor(set));
+
+      expect(result.requeued).toContain('glossary-term');
+      // One difficult sentence must not cost the document.
+      for (const id of kept) {
+        expect(result.translated.map((s) => s.id)).toContain(id);
+      }
+      expect(provider.calls.length).toBeGreaterThan(1);
+    });
+
+    it('fails the WHOLE batch on an unknown id, and maps nothing positionally', async () => {
+      const forged = JSON.stringify({
+        segments: [
+          // Same COUNT and same ORDER as the request, so a positional mapper
+          // would sail straight through this and produce a plausible document.
+          ...ids.slice(1).map((id) => ({ id, text: pairIn(set, id).targetText })),
+          { id: 'id-the-caller-never-sent', text: pairIn(set, ids[0]!).targetText },
+        ],
+      });
+      const provider = stubProvider(forged);
+      const service = createTranslationService(walledRoute(), { provider });
+
+      await expect(service.translate(requestFor(set))).rejects.toThrow(
+        /unknown_id/,
+      );
+    });
+
+    it('fails the WHOLE batch on a duplicate id', async () => {
+      const duplicated = JSON.stringify({
+        segments: [
+          ...ids.map((id) => ({ id, text: pairIn(set, id).targetText })),
+          { id: ids[0], text: pairIn(set, ids[0]!).targetText },
+        ],
+      });
+      const provider = stubProvider(duplicated);
+      const service = createTranslationService(walledRoute(), { provider });
+
+      await expect(service.translate(requestFor(set))).rejects.toThrow(
+        /duplicate_id/,
+      );
+    });
+
+    it('treats empty text for a present id as a failure, not an empty translation', async () => {
+      // A blank string is how a silent truncation looks.
+      const blanked = JSON.stringify({
+        segments: ids.map((id) => ({
+          id,
+          text: id === 'plain' ? '' : pairIn(set, id).targetText,
+        })),
+      });
+      const provider = stubProvider(blanked);
+      const service = createTranslationService(walledRoute(), { provider });
+
+      const result = await service.translate(requestFor(set));
+
+      expect(result.translated.map((s) => s.id)).not.toContain('plain');
+      expect(result.failed).toContainEqual({ id: 'plain', reason: 'empty_text' });
+    });
+
+    it('reassembles correctly when the model returns the segments reordered', async () => {
+      // Order is NOT part of the contract — asserting it would be asserting
+      // model behaviour we do not control. Reassembly is by id.
+      const reversed = JSON.stringify({
+        segments: [...ids].reverse().map((id) => ({ id, text: pairIn(set, id).targetText })),
+      });
+      const provider = stubProvider(reversed);
+      const service = createTranslationService(walledRoute(), { provider });
+
+      const result = await service.translate(requestFor(set));
+
+      for (const pair of set.pairs) {
+        const got = result.translated.find((s) => s.id === pair.id);
+        expect(got?.targetText).toBe(pair.targetText);
+      }
+    });
+
+    it('parses a well-formed envelope without inventing or dropping an id', () => {
+      // The parser on its own, so a batch failure can be localised to it.
+      const parsed = parseTranslationEnvelope(envelopeFor(set), ids);
+
+      expect(parsed.ok).toBe(true);
+      if (!parsed.ok) return;
+      expect(parsed.missing).toEqual([]);
+      expect(parsed.empty).toEqual([]);
+      expect(parsed.segments.map((s) => s.id).sort()).toEqual([...ids].sort());
+    });
+  },
+);
+
+/**
+ * Batching is cut by SIZE, not by count, and the reason is written into the
+ * corpus package it is copied from: "a corpus that grew a transcript overnight
+ * turns yesterday's working slice into today's gateway timeout, and the only
+ * symptom is a 504 with no clue in it."
+ *
+ * Locale-independent by nature — these are character counts.
+ */
+describe('planBatches — cut by character budget', () => {
+  const seg = (id: string, chars: number): TranslationSegment => ({
+    id,
+    text: 'x'.repeat(chars),
+  });
+
+  it('cuts a batch when the character budget is reached', () => {
+    const batches = planBatches(
+      [seg('a', 1_500), seg('b', 1_500), seg('c', 1_500)],
+      { maxCharsPerBatch: 3_200, maxSegmentsPerBatch: 25 },
+    );
+
+    expect(batches.length).toBeGreaterThan(1);
+    expect(batches[0]!.map((s) => s.id)).toEqual(['a', 'b']);
+  });
+
+  it('always includes at least one segment, even when it alone exceeds the budget', () => {
+    // Otherwise an oversized segment is skipped forever — `takeWithinBudget`'s
+    // rule, for exactly the same reason.
+    const batches = planBatches([seg('huge', 50_000), seg('small', 10)], {
+      maxCharsPerBatch: 4_000,
+      maxSegmentsPerBatch: 25,
+    });
+
+    expect(batches[0]).toHaveLength(1);
+    expect(batches[0]![0]!.id).toBe('huge');
+    expect(batches.flat().map((s) => s.id)).toContain('small');
+  });
+
+  it('honours the hard segment ceiling so an id list cannot dominate the call', () => {
+    // A page of one-line table cells must not become a 300-item echo list.
+    const many = Array.from({ length: 60 }, (_, i) => seg(`cell-${i}`, 5));
+
+    const batches = planBatches(many, { maxCharsPerBatch: 4_000, maxSegmentsPerBatch: 25 });
+
+    for (const batch of batches) expect(batch.length).toBeLessThanOrEqual(25);
+    // Nothing is lost in the cutting.
+    expect(batches.flat()).toHaveLength(60);
+  });
+
+  it('loses no segment and duplicates none, whatever the budget', () => {
+    const segments = Array.from({ length: 37 }, (_, i) => seg(`s-${i}`, 200));
+
+    const flat = planBatches(segments, DEFAULT_BATCH_LIMITS).flat();
+
+    expect(flat.map((s) => s.id)).toEqual(segments.map((s) => s.id));
+  });
+});

package/src/translation/__tests__/quality-gates.test.ts

@@ -0,0 +1,338 @@
+/**
+ * THE MECHANICAL GATES (bd startsim-ob4d, RED items 8–12, 14, 15).
+ *
+ * `startsim-r9q8`'s central move: every measure here compares SOURCE TO OUTPUT
+ * and none of them judges the output's language. That is what makes the product
+ * assessable by the people selling it, neither of whom reads Arabic.
+ *
+ * Their blind spot is stated so nobody mistakes a green run for a good
+ * translation: A TRANSLATION CAN BE 100% TERM-CORRECT AND STILL BE WRONG.
+ * Adherence says the right words appeared. It says nothing about whether the
+ * sentence means what the source meant. That question belongs to the named
+ * reviewer and to gate 4 of the ship floor, and no assertion in this file
+ * substitutes for it.
+ *
+ * Numeric fidelity, do-not-translate and structure are the 100%-or-fail gates;
+ * glossary adherence ships at ≥98% overall because a matcher may legitimately
+ * miss an inflected term. None of that tolerance lives HERE — these are unit
+ * tests over crafted input where the answer is exactly known.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import { stripIsolates } from '@startsimpli/i18n';
+
+import {
+  createTranslationService,
+  extractNumerics,
+  reassembleDocument,
+  renderTranslatePrompt,
+  runMechanicalChecks,
+  segmentDocument,
+  type TranslationRoute,
+} from '../index';
+import type { GlossaryTerm, MemoryUnit, StructuredDocument } from '../types';
+import {
+  DNT_BRAND,
+  GOLDEN_SETS,
+  NUMERICS_IN_FIXTURE,
+  envelopeFor,
+  pairIn,
+  segmentsOf,
+  type GoldenSet,
+} from '../__fixtures__/golden-pairs';
+import { stubProvider } from './stub-provider';
+
+function walledRoute(): TranslationRoute {
+  return {
+    posture: 'walled',
+    provider: 'stub',
+    model: 'stub-model',
+    allowedModels: ['stub-model'],
+    fallback: false,
+  };
+}
+
+describe.each(GOLDEN_SETS)(
+  'mechanical gates [$sourceLocale→$targetLocale]',
+  (set: GoldenSet) => {
+    const binding = set.glossary.find((t) => !t.doNotTranslate)!;
+
+    it('injects only the glossary terms that OCCUR in this batch', () => {
+      // The prompt scales with the TEXT, not with the glossary. A client with a
+      // 4,000-term glossary must not pay for all of it on a two-line page.
+      const absent: GlossaryTerm = {
+        source: 'force majeure',
+        target: 'NOT-IN-THIS-BATCH',
+        binding: true,
+      };
+      const batch = [{ id: 'glossary-term', text: pairIn(set, 'glossary-term').sourceText }];
+
+      const prompt = renderTranslatePrompt(batch, {
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+        glossary: [...set.glossary, absent],
+      });
+
+      expect(prompt).toContain(binding.source);
+      expect(prompt).toContain(binding.target);
+      expect(prompt).not.toContain(absent.source);
+      expect(prompt).not.toContain(absent.target);
+    });
+
+    it('labels memory examples so the model cannot mistake one for input', () => {
+      // `startsim-yfot` names this the single most likely prompt-level failure:
+      // the model translates its own examples.
+      const example: MemoryUnit = {
+        sourceText: 'A previously approved sentence.',
+        targetText: pairIn(set, 'plain').targetText,
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+      };
+      const batch = [{ id: 'plain', text: pairIn(set, 'plain').sourceText }];
+
+      const prompt = renderTranslatePrompt(batch, {
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+        memoryExamples: [example],
+      });
+
+      // The example and the thing to translate must sit in DIFFERENT blocks.
+      const exampleAt = prompt.indexOf(example.sourceText);
+      const inputAt = prompt.indexOf(batch[0]!.text);
+      expect(exampleAt).toBeGreaterThanOrEqual(0);
+      expect(inputAt).toBeGreaterThanOrEqual(0);
+      expect(prompt.slice(0, inputAt)).not.toBe(prompt.slice(0, exampleAt));
+      expect(prompt).toMatch(/<(memory|example)[\s>]/i);
+    });
+
+    it('scores a binding term as adhered when its approved form is present', () => {
+      const pair = pairIn(set, 'glossary-term');
+
+      const checks = runMechanicalChecks(pair.sourceText, pair.targetText, set.glossary);
+
+      expect(checks.glossaryAdherence).toContainEqual({
+        term: binding.source,
+        present: true,
+      });
+      expect(checks.ok).toBe(true);
+    });
+
+    it('scores a binding term as MISSED when the approved form is absent', () => {
+      // The gate has to be able to fail, or it is decoration.
+      const pair = pairIn(set, 'glossary-term');
+      const stripped = pair.targetText.replace(binding.target, 'XXX');
+
+      const checks = runMechanicalChecks(pair.sourceText, stripped, set.glossary);
+
+      expect(checks.glossaryAdherence).toContainEqual({
+        term: binding.source,
+        present: false,
+      });
+      expect(checks.ok).toBe(false);
+    });
+
+    it('requires a do-not-translate brand to survive VERBATIM', () => {
+      const pair = pairIn(set, 'do-not-translate');
+
+      const good = runMechanicalChecks(pair.sourceText, pair.targetText, set.glossary);
+      expect(good.doNotTranslate).toContainEqual({ term: DNT_BRAND, present: true });
+
+      const mangled = pair.targetText.replace(DNT_BRAND, 'نورثويند');
+      const bad = runMechanicalChecks(pair.sourceText, mangled, set.glossary);
+      expect(bad.doNotTranslate).toContainEqual({ term: DNT_BRAND, present: false });
+      expect(bad.ok).toBe(false);
+    });
+
+    it('round-trips numbers, percentages and dates as a multiset', () => {
+      // 100%, non-negotiable: a dropped digit in a business document is a
+      // commercial error, not a stylistic one.
+      const pair = pairIn(set, 'numerics');
+
+      const checks = runMechanicalChecks(pair.sourceText, pair.targetText, set.glossary);
+
+      expect(checks.missingNumerics).toEqual([]);
+      expect(extractNumerics(pair.sourceText).sort()).toEqual(
+        [...NUMERICS_IN_FIXTURE].sort(),
+      );
+    });
+
+    it('catches a dropped numeric', () => {
+      const pair = pairIn(set, 'numerics');
+      const dropped = pair.targetText.replace('$4,200,000', '');
+
+      const checks = runMechanicalChecks(pair.sourceText, dropped, set.glossary);
+
+      expect(checks.missingNumerics).toContain('$4,200,000');
+      expect(checks.ok).toBe(false);
+    });
+
+    it('stores target text with NO bidi isolate characters', () => {
+      // `startsim-jb1z`'s rule: isolation happens at RENDER time. A stored
+      // string carrying U+2066–U+2069 is invisible on screen and very visible
+      // in a failing diff — and it breaks exact memory matching.
+      const provider = stubProvider(envelopeFor(set));
+      const service = createTranslationService(walledRoute(), { provider });
+
+      return service
+        .translate({
+          segments: segmentsOf(set),
+          sourceLocale: set.sourceLocale,
+          targetLocale: set.targetLocale,
+          glossary: set.glossary,
+        })
+        .then((result) => {
+          for (const unit of result.translated) {
+            expect(stripIsolates(unit.targetText)).toBe(unit.targetText);
+          }
+        });
+    });
+
+    it('records the provider, model and prompt version on every unit', () => {
+      // Without provenance a quality regression after a route change is
+      // unattributable, and "translated under the old prompt" becomes a guess.
+      const provider = stubProvider(envelopeFor(set));
+      const service = createTranslationService(walledRoute(), { provider });
+
+      return service
+        .translate({
+          segments: segmentsOf(set),
+          sourceLocale: set.sourceLocale,
+          targetLocale: set.targetLocale,
+        })
+        .then((result) => {
+          for (const unit of result.translated) {
+            expect(unit.provenance.provider).toBe('stub');
+            expect(unit.provenance.model).toBe('stub-model');
+            expect(unit.provenance.promptVersion).toBeTruthy();
+          }
+        });
+    });
+
+    it('serves an exact memory hit and makes ZERO provider calls', async () => {
+      // The cheapest, sharpest test in the suite. An exact hit costs nothing,
+      // which is why the effective rate falls as the memory fills.
+      const pair = pairIn(set, 'plain');
+      const memory: MemoryUnit[] = [
+        {
+          sourceText: pair.sourceText,
+          targetText: pair.targetText,
+          sourceLocale: set.sourceLocale,
+          targetLocale: set.targetLocale,
+        },
+      ];
+      const provider = stubProvider('{"segments":[]}');
+      const service = createTranslationService(walledRoute(), { provider });
+
+      const result = await service.translate({
+        segments: [{ id: 'plain', text: pair.sourceText }],
+        sourceLocale: set.sourceLocale,
+        targetLocale: set.targetLocale,
+        memory,
+      });
+
+      expect(provider.calls).toHaveLength(0);
+      expect(result.providerCalls).toBe(0);
+      expect(result.translated).toHaveLength(1);
+      expect(result.translated[0]!.fromMemory).toBe(true);
+      expect(result.translated[0]!.targetText).toBe(pair.targetText);
+    });
+  },
+);
+
+/**
+ * STRUCTURE IS DETERMINISTIC BY CONSTRUCTION, because `startsim-jb1z` keeps it
+ * out of the model entirely: the caller segments, the caller reassembles, and
+ * the model only ever sees leaf text. So this is a pure round-trip and there is
+ * no excuse for it being anything but exact.
+ */
+describe('structure round-trip', () => {
+  const document: StructuredDocument = {
+    nodes: [
+      { kind: 'heading', text: 'Quarterly Report' },
+      {
+        kind: 'list',
+        children: [
+          { kind: 'list_item', text: 'Revenue grew 12.5%.' },
+          { kind: 'list_item', text: 'Headcount held flat.' },
+        ],
+      },
+      {
+        kind: 'table',
+        children: [
+          {
+            kind: 'table_row',
+            children: [
+              { kind: 'table_cell', text: 'Region' },
+              { kind: 'table_cell', text: 'Total' },
+            ],
+          },
+        ],
+      },
+      { kind: 'paragraph', text: 'See the enclosed contract.' },
+      // A product code is not prose. It must survive without being offered to
+      // the model at all.
+      { kind: 'code', text: 'NW-LOG-4420-X', translatable: false },
+    ],
+  };
+
+  it('extracts only translatable leaves, and never the untranslatable ones', () => {
+    const { segments } = segmentDocument(document);
+
+    const texts = segments.map((s) => s.text);
+    expect(texts).toContain('Quarterly Report');
+    expect(texts).toContain('Revenue grew 12.5%.');
+    expect(texts).toContain('Region');
+    expect(texts).not.toContain('NW-LOG-4420-X');
+  });
+
+  it('returns the ORIGINAL shape when the leaves are put back unchanged', () => {
+    const { segments, skeleton } = segmentDocument(document);
+    const identity = new Map(segments.map((s) => [s.id, s.text]));
+
+    expect(reassembleDocument(skeleton, identity)).toEqual(document);
+  });
+
+  it('changes only the leaf text when the leaves are translated', () => {
+    const { segments, skeleton } = segmentDocument(document);
+    const translated = new Map(segments.map((s) => [s.id, `⟦${s.text}⟧`]));
+
+    const out = reassembleDocument(skeleton, translated);
+
+    // Same node tree, same kinds, same nesting — only text differs.
+    const kinds = (doc: StructuredDocument): string[] =>
+      doc.nodes.flatMap(function walk(node): string[] {
+        return [node.kind, ...(node.children ?? []).flatMap(walk)];
+      });
+    expect(kinds(out)).toEqual(kinds(document));
+    expect(out.nodes[4]!.text).toBe('NW-LOG-4420-X');
+    expect(out.nodes[0]!.text).toBe('⟦Quarterly Report⟧');
+  });
+});
+
+/**
+ * THE NAMING BAN, ENFORCED (bd startsim-jb1z, via bd startsim-bfem).
+ *
+ * A guard rather than a RED assertion — it passes today because the stubs are
+ * empty, and its job is to fail the day GREEN writes `if (target === 'ar')`.
+ */
+describe('the engine is language-agnostic', () => {
+  it('contains no locale literal anywhere in the translation engine', async () => {
+    const { readdir, readFile } = await import('node:fs/promises');
+    const { join } = await import('node:path');
+    const root = new URL('..', import.meta.url).pathname;
+
+    const entries = (await readdir(root, { withFileTypes: true }))
+      .filter((entry) => entry.isFile() && entry.name.endsWith('.ts'));
+    expect(entries.length).toBeGreaterThan(0);
+
+    for (const entry of entries) {
+      const source = await readFile(join(root, entry.name), 'utf8');
+      const code = source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
+      // A locale compared as a STRING is the leak. Locales are data.
+      expect(code, `${entry.name} branches on a locale code`).not.toMatch(
+        /===\s*['"](ar|zh|en|he|fa|ur)(-[A-Za-z0-9]+)?['"]/,
+      );
+    }
+  });
+});