@trazum/core 1.8.0

package/src/config.ts

@@ -0,0 +1,161 @@
+import { open, stat } from 'node:fs/promises';
+import { dirname, join, resolve, sep } from 'node:path';
+
+import {
+  CONFIG_FILENAME,
+  ConfigError,
+  MAX_CONFIG_BYTES,
+  MAX_CONFIG_SEARCH_DEPTH,
+  parseConfig,
+  validateConfigModel,
+} from './config-schema.js';
+import type { TrazumConfig } from './config-schema.js';
+import { BUNDLED_CATALOGUE } from './pricing.js';
+import type { PricingCatalogue } from './pricing.js';
+import { MAX_PRICING_BYTES, PricingOverlayError, applyPricingOverlay, parsePricingOverlay } from './pricing-overlay.js';
+
+/**
+ * Finding and reading `trazum.config.json`.
+ *
+ * Split from the schema in `config-schema.ts` because this is the only half
+ * that touches the filesystem, and the split is load-bearing: `apps/web`
+ * bundles `@trazum/core` for the browser, where a single `node:fs` import
+ * anywhere in the graph fails the build. Reachable only through
+ * `@trazum/core/node`, which the CLI imports and the web app does not.
+ */
+
+export interface LoadedConfig {
+  config: TrazumConfig;
+  /** Absolute path the config came from, or null when none was found. */
+  path: string | null;
+  /**
+   * Prices to work from: the bundled catalogue, with the config's `pricing`
+   * overlay applied if it named one. Always present, so a caller never has to
+   * decide which of two sources to use.
+   */
+  pricing: PricingCatalogue;
+  /** Absolute path the overlay came from, or null when there was none. */
+  pricingPath: string | null;
+}
+
+/**
+ * Resolves a config's `pricing` overlay, if it names one.
+ *
+ * The path is relative to **the config file**, not the working directory, so a
+ * config found by walking upward still finds its own overlay. Reusing
+ * `readIfPresent` means the overlay gets the same one-handle read and the same
+ * size limit as the config itself.
+ */
+async function resolvePricing(
+  config: TrazumConfig,
+  configPath: string,
+): Promise<{ pricing: PricingCatalogue; pricingPath: string | null }> {
+  if (!config.pricing) return { pricing: BUNDLED_CATALOGUE, pricingPath: null };
+
+  const path = resolve(dirname(configPath), config.pricing);
+  const raw = await readIfPresent(path, MAX_PRICING_BYTES);
+  if (raw === null) {
+    // Named and missing is an error, for the same reason `--config` is: somebody
+    // who points at a price list is not asking for the bundled one.
+    throw new PricingOverlayError('no such pricing overlay', path);
+  }
+  return {
+    pricing: applyPricingOverlay(BUNDLED_CATALOGUE, parsePricingOverlay(raw, path), path),
+    pricingPath: path,
+  };
+}
+
+/**
+ * Reads a config file if it is there, or returns null.
+ *
+ * **The handle is opened once and everything is asked of the handle**, not of
+ * the path again. Checking the size with `stat(path)` and then reading with
+ * `readFile(path)` resolves the name twice, so what gets read is not
+ * necessarily what got measured — a symlink swapped in between the two calls
+ * defeats the size limit entirely, and on a CI runner checking out a pull
+ * request that is not a hypothetical attacker. One `open`, then `fh.stat()` and
+ * `fh.readFile()`, refer to the same file object throughout.
+ */
+async function readIfPresent(path: string, limit = MAX_CONFIG_BYTES): Promise<string | null> {
+  let handle;
+  try {
+    handle = await open(path, 'r');
+  } catch {
+    return null;
+  }
+
+  try {
+    const info = await handle.stat();
+    if (!info.isFile()) return null;
+    if (info.size > limit) {
+      throw new ConfigError(`is ${info.size} bytes, over the ${limit}-byte limit`, path);
+    }
+    return await handle.readFile('utf8');
+  } finally {
+    await handle.close();
+  }
+}
+
+export interface LoadConfigOptions {
+  /** Directory to start the upward search from. Defaults to the process cwd. */
+  from?: string;
+  /**
+   * An explicit path. Given one, no search happens and a missing file is an
+   * error: somebody who names a config file is not asking for defaults.
+   */
+  explicit?: string;
+}
+
+/**
+ * Finds and reads the nearest config file.
+ *
+ * The search walks **upward from the working directory**, which is what makes
+ * `cd packages/thing && trazum check prompt.txt` pick up the repository's
+ * config. It stops at the first hit, at a directory containing `.git`, or after
+ * `MAX_CONFIG_SEARCH_DEPTH` levels — a bound rather than an unlimited climb to
+ * the filesystem root.
+ *
+ * Finding nothing is not an error: the tool is useful with no config at all,
+ * and returning `{config: {}, path: null}` lets the caller say so.
+ */
+export async function loadConfig(options: LoadConfigOptions = {}): Promise<LoadedConfig> {
+  if (options.explicit !== undefined) {
+    const path = resolve(options.explicit);
+    const raw = await readIfPresent(path);
+    if (raw === null) throw new ConfigError('no such config file', path);
+    return finish(parseConfig(raw, path), path);
+  }
+
+  let directory = resolve(options.from ?? process.cwd());
+
+  for (let depth = 0; depth < MAX_CONFIG_SEARCH_DEPTH; depth++) {
+    const candidate = join(directory, CONFIG_FILENAME);
+    const raw = await readIfPresent(candidate);
+    if (raw !== null) return finish(parseConfig(raw, candidate), candidate);
+
+    // A repository root is a deliberate stopping point: reaching past it would
+    // read a config belonging to whatever happens to be above the checkout,
+    // which on a CI runner is not the project's business.
+    const gitDirectory = await stat(join(directory, '.git')).catch(() => null);
+    if (gitDirectory) break;
+
+    const parent = dirname(directory);
+    if (parent === directory || parent === sep) break;
+    directory = parent;
+  }
+
+  return { config: {}, path: null, pricing: BUNDLED_CATALOGUE, pricingPath: null };
+}
+
+/**
+ * Resolves the overlay and checks the model, in that order.
+ *
+ * The order is the point: `usage.model` may name a model the overlay introduces,
+ * so validating it before the overlay is read would reject a config that is
+ * correct.
+ */
+async function finish(config: TrazumConfig, path: string): Promise<LoadedConfig> {
+  const { pricing, pricingPath } = await resolvePricing(config, path);
+  validateConfigModel(config, pricing, path);
+  return { config, path, pricing, pricingPath };
+}

package/src/detect.ts

@@ -0,0 +1,312 @@
+import type { ModelPricing } from './types.js';
+
+/**
+ * Where a prompt is actually going.
+ *
+ * Trazum priced one vendor, so defaulting to Claude cost nothing. Pricing seven
+ * made that default a **wrong number**: `trazum check src/prompts.ts` billed a
+ * file that calls OpenAI against Claude Opus 5 and said so with a straight face.
+ * The fix is not a better default — it is reading what the code already says.
+ *
+ * Everything here is **evidence-first**. A detection that cannot name what it
+ * saw is a guess wearing a result's clothes, and this feeds a command used as a
+ * CI gate. Every answer carries the line it came from, so a wrong one is
+ * arguable rather than mysterious.
+ *
+ * **It declines when the file points two ways.** A module importing both
+ * `openai` and `@anthropic-ai/sdk` is a module Trazum cannot price without
+ * picking a side, and picking silently is how somebody budgets against the wrong
+ * provider for a month. The conflict is reported and the caller falls back to
+ * whatever they configured.
+ *
+ * Detection sits **between config and defaults** in the usual layering: a flag
+ * beats config, config beats detection, detection beats the built-in default.
+ * Reading the code is better than assuming, and worse than being told.
+ */
+
+export type EvidenceKind =
+  /** `model: 'gpt-5'` — names the model outright, so nothing beats it. */
+  | 'model-literal'
+  /** `from 'openai'` — names the provider but not which model. */
+  | 'sdk-import'
+  /** `https://api.deepseek.com` — a base URL pinned in the source. */
+  | 'base-url'
+  /** `// trazum:prompt name model=gpt-5` — the author said so directly. */
+  | 'marker';
+
+export interface Evidence {
+  kind: EvidenceKind;
+  /** The text that produced it, so the reader can go and look. */
+  detail: string;
+  /** 1-based line, when it came from a specific place in the file. */
+  line?: number;
+  provider?: string;
+  model?: string;
+}
+
+export interface Detection {
+  /** The provider, or null when nothing was found or the file disagreed. */
+  provider: string | null;
+  /** The exact model, when something named one. */
+  model: string | null;
+  /** What was found, strongest first. Empty when nothing was. */
+  evidence: Evidence[];
+  /**
+   * Evidence pointing somewhere other than the answer.
+   *
+   * Non-empty means the file names more than one provider, and `provider` is
+   * null: two answers is not a weaker version of one answer.
+   */
+  conflicts: Evidence[];
+}
+
+/**
+ * How each provider announces itself in source.
+ *
+ * Data rather than branches, so adding a provider is a row. Matched as plain
+ * substrings on a lowercased copy — no regex, because this runs over files a
+ * contributor supplies and the pattern-goes-quadratic lesson has been paid for
+ * three times this week already.
+ *
+ * A quoted package name will also match the same string appearing in a prompt's
+ * own text. That is accepted rather than fixed: an import is the *weakest* kind
+ * of evidence here, so a model literal or a base URL overrides it, and the
+ * alternative — enumerating every import spelling in two languages — is how the
+ * single-quoted `require` form came to be missing.
+ */
+const SIGNATURES: Array<{ provider: string; imports: string[]; urls: string[] }> = [
+  {
+    provider: 'anthropic',
+    imports: ['@anthropic-ai/sdk', 'anthropic-sdk', 'from anthropic', 'import anthropic'],
+    urls: ['api.anthropic.com'],
+  },
+  {
+    provider: 'openai',
+    // The quoted package name covers `from 'openai'`, `require("openai")` and
+    // every other JS spelling without enumerating them — which is how the
+    // single-quoted `require` form got missed. The bare Python forms need their
+    // own entries because Python does not quote its imports.
+    imports: ["'openai'", '"openai"', 'from openai', 'import openai'],
+    urls: ['api.openai.com'],
+  },
+  {
+    provider: 'google',
+    imports: [
+      '@google/genai',
+      '@google/generative-ai',
+      'google.generativeai',
+      'from google import genai',
+    ],
+    urls: ['generativelanguage.googleapis.com', 'aiplatform.googleapis.com'],
+  },
+  {
+    provider: 'moonshot',
+    imports: ['moonshot'],
+    urls: ['api.moonshot.cn', 'api.moonshot.ai'],
+  },
+  {
+    provider: 'deepseek',
+    imports: ['deepseek'],
+    urls: ['api.deepseek.com'],
+  },
+  { provider: 'xai', imports: ['@ai-sdk/xai', 'xai-sdk'], urls: ['api.x.ai'] },
+  { provider: 'mistral', imports: ['@mistralai/mistralai', 'from mistralai'], urls: ['api.mistral.ai'] },
+];
+
+/**
+ * 1-based line lookup, built once per file.
+ *
+ * The obvious version counts newlines from zero on every call, which is
+ * quadratic in the number of matches — 36 seconds on a file repeating a model
+ * id. That is the third time this week the same shape has appeared, and the
+ * second time I have written it after fixing it in `extract.ts`; the difference
+ * here is that matches arrive out of order across the signature loop, so a
+ * forward-only counter does not work and this is a binary search instead.
+ */
+function lineIndex(source: string): (offset: number) => number {
+  const newlines: number[] = [];
+  for (let i = 0; i < source.length; i++) if (source[i] === '\n') newlines.push(i);
+
+  return (offset) => {
+    let low = 0;
+    let high = newlines.length;
+    while (low < high) {
+      const mid = (low + high) >> 1;
+      if (newlines[mid]! < offset) low = mid + 1;
+      else high = mid;
+    }
+    return low + 1;
+  };
+}
+
+/**
+ * Model names written as string literals next to a `model` key.
+ *
+ * Bounded deliberately: `model` then up to a few characters of punctuation and
+ * whitespace, then a quoted value. A looser search would match the word "model"
+ * in a prompt's own text and price the prompt against something it merely
+ * mentions.
+ */
+function modelLiterals(
+  source: string,
+  known: ModelPricing[],
+  lineAt: (offset: number) => number,
+): Evidence[] {
+  const found: Evidence[] = [];
+  const haystack = source.toLowerCase();
+
+  for (const model of known) {
+    let from = 0;
+    for (;;) {
+      const at = haystack.indexOf(model.id.toLowerCase(), from);
+      if (at === -1) break;
+      from = at + model.id.length;
+
+      // It has to be a quoted value: a model id in prose is a mention, not a
+      // call. Checking the character either side is enough and costs nothing.
+      const before = source[at - 1];
+      const after = source[at + model.id.length];
+      const quoted =
+        (before === "'" || before === '"' || before === '`') &&
+        (after === "'" || after === '"' || after === '`');
+      if (!quoted) continue;
+
+      found.push({
+        kind: 'model-literal',
+        detail: model.id,
+        line: lineAt(at),
+        provider: model.provider,
+        model: model.id,
+      });
+    }
+  }
+  return found;
+}
+
+/** `model=gpt-5` on a `trazum:prompt` marker line. */
+function markerModels(
+  source: string,
+  known: ModelPricing[],
+  lineAt: (offset: number) => number,
+): Evidence[] {
+  const found: Evidence[] = [];
+  let from = 0;
+  for (;;) {
+    const at = source.indexOf('trazum:prompt', from);
+    if (at === -1) break;
+    from = at + 13;
+
+    const lineEnd = source.indexOf('\n', at);
+    const rest = source.slice(from, lineEnd === -1 ? source.length : lineEnd);
+    const key = rest.indexOf('model=');
+    if (key === -1) continue;
+
+    const value = rest.slice(key + 6).trim().split(/[\s>]/)[0] ?? '';
+    const model = known.find((m) => m.id === value);
+    if (!model) continue;
+
+    found.push({
+      kind: 'marker',
+      detail: `model=${value}`,
+      line: lineAt(at),
+      provider: model.provider,
+      model: model.id,
+    });
+  }
+  return found;
+}
+
+export interface DetectOptions {
+  /** The catalogue to recognise model names from. */
+  models?: ModelPricing[];
+}
+
+/**
+ * Works out which provider a source file talks to, and says why.
+ *
+ * Returns `provider: null` both when nothing was found and when the file named
+ * more than one — the caller cannot act on either, and `conflicts` distinguishes
+ * them for the reader.
+ */
+export function detectFromSource(source: string, options: DetectOptions = {}): Detection {
+  const known = options.models ?? [];
+  const haystack = source.toLowerCase();
+  const lineAt = lineIndex(source);
+
+  const evidence: Evidence[] = [
+    // Strongest first: the author naming a model beats the code naming one,
+    // which beats the code naming only a provider.
+    ...markerModels(source, known, lineAt),
+    ...modelLiterals(source, known, lineAt),
+  ];
+
+  for (const signature of SIGNATURES) {
+    for (const needle of signature.imports) {
+      const at = haystack.indexOf(needle.toLowerCase());
+      if (at === -1) continue;
+      evidence.push({
+        kind: 'sdk-import',
+        detail: needle,
+        line: lineAt(at),
+        provider: signature.provider,
+      });
+      break;
+    }
+    for (const url of signature.urls) {
+      const at = haystack.indexOf(url);
+      if (at === -1) continue;
+      evidence.push({
+        kind: 'base-url',
+        detail: url,
+        line: lineAt(at),
+        provider: signature.provider,
+      });
+      break;
+    }
+  }
+
+  if (evidence.length === 0) {
+    return { provider: null, model: null, evidence: [], conflicts: [] };
+  }
+
+  // A stronger kind overrides a weaker one; only equals can disagree.
+  //
+  // Getting this wrong the first time broke the most common non-OpenAI setup
+  // there is. Moonshot, DeepSeek, xAI, Groq and Together all ship an
+  // OpenAI-compatible endpoint, so their documented usage is the OpenAI SDK
+  // pointed at a different `base_url`. Treating that as a contradiction —
+  // "this file says both openai and deepseek" — would refuse to price a
+  // perfectly ordinary DeepSeek client. The base URL is the specific fact and
+  // the SDK import is the generic one, so the URL wins rather than ties.
+  //
+  // Likewise a `model=` on the marker is the author stating the answer, which
+  // cannot be contradicted by an import they wrote for another reason.
+  const RANK: Record<EvidenceKind, number> = {
+    marker: 3,
+    'model-literal': 2,
+    'base-url': 1,
+    'sdk-import': 0,
+  };
+  const ranked = [...evidence].sort((a, b) => RANK[b.kind] - RANK[a.kind]);
+  const best = ranked[0]!;
+  const topRank = RANK[best.kind];
+
+  // Only evidence of the same strength can conflict. Anything weaker is
+  // context, not contradiction.
+  const conflicts = ranked.filter(
+    (e) => RANK[e.kind] === topRank && e.provider !== undefined && e.provider !== best.provider,
+  );
+
+  if (conflicts.length > 0) {
+    return { provider: null, model: null, evidence: ranked, conflicts };
+  }
+
+  return {
+    provider: best.provider ?? null,
+    // A model only comes from something that named one, and only from the
+    // winning provider: an import tells us who, never which of their models.
+    model: ranked.find((e) => e.model !== undefined && e.provider === best.provider)?.model ?? null,
+    evidence: ranked,
+    conflicts: [],
+  };
+}

package/src/evaluate.ts

@@ -0,0 +1,188 @@
+import { segment } from './segment.js';
+import { jaccard, normalizeForCompare } from './similarity.js';
+import type { LlmProvider } from './types.js';
+
+/**
+ * Golden-set evaluation.
+ *
+ * Everything else Trazum reports is arithmetic: tokens, prices, multiplication.
+ * This is the one question arithmetic cannot answer — does the shorter prompt
+ * still do the job? — and the README has been answering it with a caveat
+ * ("the aggressive level can change nuance; read the diff") because a rules
+ * engine genuinely cannot know.
+ *
+ * The trap here is comparing the two prompts' outputs and calling the
+ * difference a regression. A model asked the same question twice does not
+ * answer identically, so "the optimised prompt diverged on 3 of 10 cases" is
+ * meaningless on its own — it might be better than the original manages
+ * against itself.
+ *
+ * So the original is run twice per case first, and that self-agreement is the
+ * yardstick. The optimised prompt is judged against the model's own variance,
+ * not against an imaginary determinism it never had. It costs a third call per
+ * case and it is the only reason the number means anything.
+ */
+
+export interface EvalCase {
+  /** The input this case feeds the prompt. */
+  input: string;
+  /** The original prompt's two answers, used to measure its own variance. */
+  baseline: [string, string];
+  /** The optimised prompt's answer. */
+  optimized: string;
+  /** How closely the original agreed with itself (0-1). */
+  selfSimilarity: number;
+  /** How closely the optimised answer matched the original's first (0-1). */
+  crossSimilarity: number;
+}
+
+export type EvalVerdict = 'indistinguishable' | 'within-noise' | 'diverges' | 'inconclusive';
+
+export interface EvalReport {
+  provider: string;
+  model: string;
+  cases: EvalCase[];
+  /** Mean agreement of the original prompt with itself. The yardstick. */
+  selfAgreement: number;
+  /** Mean agreement between the original and the optimised prompt. */
+  crossAgreement: number;
+  verdict: EvalVerdict;
+  /** Total provider calls made, so the cost is never a surprise. */
+  callsMade: number;
+}
+
+export interface EvaluateOptions {
+  /**
+   * How many cases to run at once. Kept low by default: this hammers someone
+   * else's endpoint, and a rate limit tripped halfway through wastes every
+   * call already paid for.
+   */
+  concurrency?: number;
+}
+
+/**
+ * Builds the prompt for one case.
+ *
+ * A template gets its first placeholder filled; anything else gets the input
+ * appended. Substituting is the honest reading of a prompt written with
+ * `{{query}}` — appending would test a prompt nobody runs.
+ */
+export function fillPrompt(prompt: string, input: string): string {
+  const placeholder = segment(prompt).find(
+    (s) => s.kind === 'protected' && s.protection === 'placeholder',
+  );
+  if (!placeholder) return `${prompt.trimEnd()}\n\n${input}`;
+  return prompt.replace(placeholder.text, input);
+}
+
+/** Agreement between two answers, 0-1. */
+/**
+ * How closely two answers agree, 0 to 1.
+ *
+ * Exported because `prune.ts` measures the same thing and must measure it the
+ * same way. It was a private function here first, and the copy that appeared in
+ * `prune.ts` was a bag-of-words F1 while this is Jaccard over normalised text —
+ * two different numbers under one name, with a comment in the copy claiming they
+ * were the same measure. Sharing the function is what makes that comment true.
+ */
+export function agreement(a: string, b: string): number {
+  const left = normalizeForCompare(a);
+  const right = normalizeForCompare(b);
+  if (left === right) return 1;
+  return jaccard(left, right);
+}
+
+const mean = (values: number[]): number =>
+  values.length === 0 ? 0 : values.reduce((sum, v) => sum + v, 0) / values.length;
+
+/**
+ * Turns the two agreement figures into a verdict.
+ *
+ * The comparison is always relative. An optimised prompt agreeing with the
+ * original 0.85 of the time looks alarming until you see the original agrees
+ * with itself 0.86 — at which point the optimisation changed nothing the model
+ * was not already doing on its own.
+ *
+ * `inconclusive` exists because a model that is wildly inconsistent with itself
+ * cannot be used to judge anything. Reporting a confident verdict off that
+ * would be worse than admitting the test does not work here.
+ */
+export function verdictFor(selfAgreement: number, crossAgreement: number): EvalVerdict {
+  if (crossAgreement >= 0.999) return 'indistinguishable';
+  if (selfAgreement < 0.5) return 'inconclusive';
+  // Within a small margin of the model's own noise floor, the difference is
+  // not attributable to the prompt.
+  if (crossAgreement >= selfAgreement - 0.05) return 'within-noise';
+  return 'diverges';
+}
+
+/** Runs `tasks` with a bounded number in flight, preserving order. */
+export async function pooled<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> {
+  const results = new Array<T>(tasks.length);
+  let next = 0;
+
+  const workers = Array.from({ length: Math.min(limit, tasks.length) }, async () => {
+    for (;;) {
+      const index = next++;
+      if (index >= tasks.length) return;
+      results[index] = await tasks[index]!();
+    }
+  });
+
+  await Promise.all(workers);
+  return results;
+}
+
+/**
+ * Runs both prompt versions over a set of inputs and reports whether the
+ * optimisation changed the answers.
+ *
+ * Costs **three provider calls per case**: the original twice, the optimised
+ * once. The doubled original is what makes the result interpretable, and
+ * `callsMade` reports the total so the bill is never a surprise.
+ */
+export async function evaluate(
+  originalPrompt: string,
+  optimizedPrompt: string,
+  inputs: readonly string[],
+  provider: LlmProvider,
+  options: EvaluateOptions = {},
+): Promise<EvalReport> {
+  const concurrency = Math.max(1, options.concurrency ?? 3);
+
+  const run = (prompt: string, input: string): Promise<string> =>
+    provider.complete({ system: fillPrompt(prompt, input), user: input });
+
+  const cases = await pooled(
+    inputs.map((input) => async (): Promise<EvalCase> => {
+      // Sequential within a case: the two baseline runs exist to measure the
+      // model's variance, and issuing them together invites a provider to
+      // serve one from a cache and report a variance of zero.
+      const baselineA = await run(originalPrompt, input);
+      const baselineB = await run(originalPrompt, input);
+      const optimized = await run(optimizedPrompt, input);
+
+      return {
+        input,
+        baseline: [baselineA, baselineB],
+        optimized,
+        selfSimilarity: agreement(baselineA, baselineB),
+        crossSimilarity: agreement(baselineA, optimized),
+      };
+    }),
+    concurrency,
+  );
+
+  const selfAgreement = mean(cases.map((c) => c.selfSimilarity));
+  const crossAgreement = mean(cases.map((c) => c.crossSimilarity));
+
+  return {
+    provider: provider.name,
+    model: provider.model,
+    cases,
+    selfAgreement,
+    crossAgreement,
+    verdict: verdictFor(selfAgreement, crossAgreement),
+    callsMade: cases.length * 3,
+  };
+}