@startsimpli/llm 0.4.16 → 0.4.18

package/src/translation/__tests__/route-confidentiality.test.ts

@@ -0,0 +1,228 @@
+/**
+ * ROUTING AND THE FAILURE PATH (bd startsim-ob4d, RED items 16–18).
+ *
+ * `startsim-bfem`: "Failure-path assertions matter as much as happy-path ones,
+ * because the failure path is where the confidentiality guarantee actually
+ * breaks." The named precedent is real and still in the estate — a Django
+ * handler that logs the model's raw response content on a JSON-parse failure.
+ * If translation reproduces that, the wall `startsim-hopl` sells is a comment.
+ *
+ * The three properties pinned here:
+ *
+ *   16. A malformed envelope is retried, then FAILS — and no response text and
+ *       no client text reaches any log, on any path.
+ *   17. A `walled` route whose provider will not answer FAILS. It never crosses
+ *       to a vendor under different retention terms. This is a deliberate
+ *       override of `LLMService.getProvider()`'s "fall back to any available",
+ *       whose `isAvailable()` is `Boolean(apiKey)` — and prod is on record
+ *       holding an OpenAI key in `ANTHROPIC_API_KEY`, so a deployment that
+ *       believes it is walled can route client text to another vendor with
+ *       nothing saying so.
+ *   18. The service REFUSES TO BOOT on a model outside the deployment's
+ *       allow-list. `claude-fable-5` is the named case: a Covered Model that
+ *       cannot run under zero-retention terms, advertised by our own provider
+ *       today. A live foot-gun, not a hypothetical.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import {
+  EnvelopeParseError,
+  TranslationConfigError,
+  WalledRouteUnavailableError,
+  createTranslationService,
+  parseTranslationEnvelope,
+  resolveTranslationRoute,
+  type TranslationRoute,
+} from '../index';
+import { GOLDEN_SETS, pairIn, segmentsOf } from '../__fixtures__/golden-pairs';
+import { capturingLogger, stubProvider } from './stub-provider';
+
+/** One locale is enough here — nothing in this file is language-sensitive. */
+const SET = GOLDEN_SETS[0]!;
+
+function walledRoute(overrides: Partial<TranslationRoute> = {}): TranslationRoute {
+  return {
+    posture: 'walled',
+    provider: 'stub',
+    model: 'stub-model',
+    allowedModels: ['stub-model'],
+    fallback: false,
+    ...overrides,
+  };
+}
+
+function request() {
+  return {
+    segments: segmentsOf(SET),
+    sourceLocale: SET.sourceLocale,
+    targetLocale: SET.targetLocale,
+    glossary: SET.glossary,
+  };
+}
+
+describe('a malformed envelope', () => {
+  /**
+   * Prose wrapped around the client's own sentence — what a model actually
+   * returns when it ignores the JSON instruction. The client text is IN here on
+   * purpose: it is the needle the log assertion searches for.
+   */
+  const CLIENT_SENTENCE = pairIn(SET, 'glossary-term').sourceText;
+  const GARBAGE = `Sure! Here is the translation of "${CLIENT_SENTENCE}" — I hope it helps.`;
+
+  it('is retried before the batch is failed', async () => {
+    const provider = stubProvider(GARBAGE);
+    const service = createTranslationService(walledRoute(), {
+      provider,
+      maxEnvelopeRetries: 2,
+    });
+
+    await expect(service.translate(request())).rejects.toThrow(EnvelopeParseError);
+    // One original attempt plus the permitted retries — a parse failure is
+    // worth asking again for, once, with a correction prompt.
+    expect(provider.calls.length).toBeGreaterThan(1);
+    expect(provider.calls.length).toBeLessThanOrEqual(3);
+  });
+
+  it('puts NO response text and NO client text in any log', async () => {
+    const logger = capturingLogger();
+    const provider = stubProvider(GARBAGE);
+    const service = createTranslationService(walledRoute(), {
+      provider,
+      logger,
+      maxEnvelopeRetries: 1,
+    });
+
+    await expect(service.translate(request())).rejects.toThrow(EnvelopeParseError);
+
+    // Asserted against the FIXTURE TEXT rather than by inspecting call sites,
+    // which is the only version of this check that cannot rot.
+    const logged = logger.all();
+    expect(logged).not.toContain(CLIENT_SENTENCE);
+    expect(logged).not.toContain(GARBAGE);
+    for (const pair of SET.pairs) {
+      if (pair.sourceText) expect(logged).not.toContain(pair.sourceText);
+      if (pair.targetText) expect(logged).not.toContain(pair.targetText);
+    }
+  });
+
+  it('carries no response text in the error it throws, either', async () => {
+    // An exception is a log line waiting to happen: it will be caught, printed,
+    // and shipped to an aggregator by code that has never read this file.
+    const provider = stubProvider(GARBAGE);
+    const service = createTranslationService(walledRoute(), {
+      provider,
+      maxEnvelopeRetries: 1,
+    });
+
+    await expect(service.translate(request())).rejects.toThrow(EnvelopeParseError);
+    await service.translate(request()).catch((error: unknown) => {
+      const text = `${(error as Error).message}\n${(error as Error).stack ?? ''}`;
+      expect(text).not.toContain(CLIENT_SENTENCE);
+      expect(text).not.toContain(GARBAGE);
+    });
+  });
+
+  it('rejects unparseable JSON as malformed rather than guessing', () => {
+    const parsed = parseTranslationEnvelope(GARBAGE, ['plain']);
+
+    expect(parsed.ok).toBe(false);
+    if (parsed.ok) return;
+    expect(parsed.reason).toBe('malformed');
+  });
+});
+
+describe('the walled posture', () => {
+  it('FAILS when the configured provider is unavailable, rather than falling back', async () => {
+    const provider = stubProvider('{"segments":[]}', { available: false });
+    const service = createTranslationService(walledRoute(), { provider });
+
+    await expect(service.translate(request())).rejects.toThrow(
+      WalledRouteUnavailableError,
+    );
+    // It must not even have tried: the refusal is the correct outcome, and the
+    // job is retried later against the SAME provider.
+    expect(provider.calls).toHaveLength(0);
+  });
+
+  it('refuses at BOOT to be configured with fallback enabled', () => {
+    // Posture is a property of the deployment and fallback is a property of the
+    // posture. `walled` + `fallback: true` is not a preference, it is a breach
+    // waiting for an outage.
+    expect(() =>
+      createTranslationService(walledRoute({ fallback: true }), {
+        provider: stubProvider('{}'),
+      }),
+    ).toThrow(TranslationConfigError);
+  });
+
+  it('surfaces a provider refusal as a failure, never as an empty document', async () => {
+    const provider = stubProvider('{}', {
+      throws: new Error('provider refused'),
+    });
+    const service = createTranslationService(walledRoute(), { provider });
+
+    await expect(service.translate(request())).rejects.toThrow();
+  });
+});
+
+describe('the boot-time model allow-list', () => {
+  it('refuses to build a service whose model is outside the allow-list', () => {
+    expect(() =>
+      createTranslationService(
+        walledRoute({ model: 'claude-fable-5', allowedModels: ['stub-model'] }),
+        { provider: stubProvider('{}') },
+      ),
+    ).toThrow(TranslationConfigError);
+  });
+
+  it('refuses a Covered Model on a walled deployment, by name', () => {
+    // Fail the boot, not the first client document.
+    expect(() =>
+      resolveTranslationRoute({
+        TRANSLATION_POSTURE: 'walled',
+        TRANSLATION_PROVIDER: 'anthropic',
+        TRANSLATION_MODEL: 'claude-fable-5',
+      }),
+    ).toThrow(TranslationConfigError);
+
+    expect(() =>
+      resolveTranslationRoute({
+        TRANSLATION_POSTURE: 'walled',
+        TRANSLATION_PROVIDER: 'anthropic',
+        TRANSLATION_MODEL: 'claude-mythos-5',
+      }),
+    ).toThrow(TranslationConfigError);
+  });
+
+  it('resolves a walled route with fallback OFF, whatever the environment asks for', () => {
+    const route = resolveTranslationRoute({
+      TRANSLATION_POSTURE: 'walled',
+      TRANSLATION_PROVIDER: 'anthropic',
+      TRANSLATION_MODEL: 'claude-sonnet-5',
+      TRANSLATION_ALLOWED_MODELS: 'claude-sonnet-5',
+      // Asked for, and ignored: the posture decides.
+      TRANSLATION_FALLBACK: 'true',
+    });
+
+    expect(route.posture).toBe('walled');
+    expect(route.fallback).toBe(false);
+    expect(route.model).toBe('claude-sonnet-5');
+  });
+
+  it('permits fallback on the open posture', () => {
+    // OGMC's marketing copy and MCR's artist metadata are not confidential in
+    // the sense the client's material is. Forcing the whole estate through the
+    // walled route would be cargo-culting.
+    const route = resolveTranslationRoute({
+      TRANSLATION_POSTURE: 'open',
+      TRANSLATION_PROVIDER: 'anthropic',
+      TRANSLATION_MODEL: 'claude-sonnet-5',
+      TRANSLATION_ALLOWED_MODELS: 'claude-sonnet-5',
+      TRANSLATION_FALLBACK: 'true',
+    });
+
+    expect(route.posture).toBe('open');
+    expect(route.fallback).toBe(true);
+  });
+});

package/src/translation/__tests__/stub-provider.ts

@@ -0,0 +1,118 @@
+/**
+ * A PURPOSE-BUILT provider stub for the translation suite.
+ *
+ * `MockLLMProvider` is deliberately not used here, and `startsim-bfem` says why:
+ * its behaviour is set through a module-global mutable (`let mockBehavior`, set
+ * via `setMockBehavior()`) carrying a single `customResponse` string. That is
+ * fine for one-response tests and wrong for these, which need a DIFFERENT
+ * envelope per call and must not leak state between parallel test files.
+ *
+ * The engine takes an `ILLMProvider` by injection, so a local stub costs one
+ * file and buys isolation. That is the right trade.
+ */
+
+import type { ILLMProvider } from '../../providers/interface';
+import type { GenerateOptions, LLMResponse } from '../../types';
+import type { TranslationLogger } from '../types';
+
+export interface RecordedCall {
+  userPrompt: string;
+  systemPrompt: string;
+  options?: GenerateOptions;
+}
+
+export interface StubProvider extends ILLMProvider {
+  /** Every call, in order, so a test can assert the BATCH count and the prompt. */
+  readonly calls: RecordedCall[];
+}
+
+export interface StubProviderOptions {
+  name?: string;
+  model?: string;
+  /** `isAvailable()`. False models a provider that is down or unconfigured. */
+  available?: boolean;
+  /** Thrown instead of answering — a provider that refuses. */
+  throws?: Error;
+}
+
+/**
+ * @param script What to return, per call. A string array is consumed in order
+ *   and the last entry repeats; a function receives the 0-based call index.
+ */
+export function stubProvider(
+  script: string | string[] | ((callIndex: number, call: RecordedCall) => string),
+  options: StubProviderOptions = {},
+): StubProvider {
+  const {
+    name = 'stub',
+    model = 'stub-model',
+    available = true,
+    throws,
+  } = options;
+  const calls: RecordedCall[] = [];
+
+  const pick = (index: number, call: RecordedCall): string => {
+    if (typeof script === 'function') return script(index, call);
+    if (typeof script === 'string') return script;
+    return script[Math.min(index, script.length - 1)] ?? '';
+  };
+
+  return {
+    name,
+    availableModels: [model],
+    defaultModel: model,
+    calls,
+    isAvailable: () => available,
+    async generate(
+      userPrompt: string,
+      systemPrompt: string,
+      generateOptions?: GenerateOptions,
+    ): Promise<LLMResponse> {
+      const call: RecordedCall = { userPrompt, systemPrompt, options: generateOptions };
+      calls.push(call);
+      if (throws) throw throws;
+      return {
+        content: pick(calls.length - 1, call),
+        usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+        model,
+      };
+    },
+  };
+}
+
+export interface CapturingLogger extends TranslationLogger {
+  /** Every line and every piece of metadata, flattened for substring search. */
+  readonly written: string[];
+  /** One string carrying everything logged, for a single `not.toContain`. */
+  all(): string;
+}
+
+/**
+ * A logger that keeps everything, so "no client text reaches a log" can be
+ * ASSERTED against the fixture text rather than checked by reading call sites.
+ *
+ * The metadata is serialised into the same haystack on purpose: the leak this
+ * guards against is a raw provider response passed as a structured field, which
+ * is exactly what the Django precedent at `messages/services/llm.py` does.
+ */
+export function capturingLogger(): CapturingLogger {
+  const written: string[] = [];
+  const record = (level: string) => (message: string, meta?: Record<string, unknown>) => {
+    written.push(`${level} ${message}`);
+    if (meta !== undefined) {
+      try {
+        written.push(JSON.stringify(meta));
+      } catch {
+        written.push(String(meta));
+      }
+    }
+  };
+  return {
+    written,
+    all: () => written.join('\n'),
+    debug: record('debug'),
+    info: record('info'),
+    warn: record('warn'),
+    error: record('error'),
+  };
+}