@trazum/core 1.8.0

package/src/extract.ts

@@ -0,0 +1,336 @@
+/**
+ * Prompts embedded in source files.
+ *
+ * `check` and `diff` read `.txt`, `.md`, `.prompt` and `.tmpl`. Real prompts live
+ * in TypeScript template literals, Python triple-quoted strings and YAML blocks,
+ * so adopting Trazum has meant first refactoring them out into standalone files —
+ * a change to somebody's application as the price of admission.
+ *
+ * **It reads a marker, it does not guess.** `// trazum:prompt` on the line before
+ * the literal, and then the literal by delimiter matching. Guessing which string
+ * in a file is a prompt is a heuristic, and a heuristic inside a tool used as a
+ * CI gate produces failures on strings nobody meant to govern. A marker is one
+ * line of noise in exchange for never being wrong about what it picked up.
+ *
+ * **Interpolation already works and is not a special case.** `${x}` inside a
+ * template literal is exactly the placeholder shape `segment.ts` protects, so an
+ * embedded prompt gets the same cache-prefix analysis, the same protection from
+ * the rules, and the same `--reorder` treatment as a `{{x}}` template. Nothing
+ * here has to know about it.
+ *
+ * **The honest limit:** a prompt assembled from concatenated pieces cannot be
+ * read this way. `` `You are ${role}.` + rules.join('\n') `` is a prompt whose
+ * text does not exist until it runs. This module declines it and says so rather
+ * than governing the half it can see — a budget enforced against a fragment is a
+ * green build for a prompt nobody measured.
+ *
+ * Scanned character by character rather than with a regex. The module it most
+ * resembles shipped two quadratic patterns this week, and delimiter matching over
+ * untrusted source is exactly the shape that goes wrong.
+ */
+
+export interface ExtractedPrompt {
+  /** Name from `trazum:prompt <name>`, or undefined when the marker was bare. */
+  name?: string;
+  /** 1-based line of the marker comment, which is what an error should cite. */
+  line: number;
+  /** The prompt text, delimiters removed and escapes resolved. */
+  text: string;
+  /** The delimiter it was written with, so a writer could put it back. */
+  quote: '`' | '"' | "'" | '"""' | "'''";
+  /** Offsets of the text within the source, exclusive of the delimiters. */
+  start: number;
+  end: number;
+}
+
+export interface DeclinedPrompt {
+  line: number;
+  /**
+   * Why this marker produced nothing.
+   *
+   * `concatenated` is the interesting one: the text exists only at runtime.
+   * `no-literal` means the marker was not followed by a string at all.
+   * `unterminated` means the file ends inside the literal, which is a syntax
+   * error in the source rather than a Trazum problem, but worth naming.
+   */
+  reason: 'concatenated' | 'no-literal' | 'unterminated';
+  detail: string;
+}
+
+export interface ExtractionResult {
+  prompts: ExtractedPrompt[];
+  declined: DeclinedPrompt[];
+}
+
+/** The marker, in the comment syntaxes that cover the languages prompts live in. */
+const MARKERS = ['//', '#', '--', '<!--'];
+const TAG = 'trazum:prompt';
+
+/**
+ * Both ways an HTML comment can close.
+ *
+ * `--!>` is the "comment end bang" the HTML parser also accepts, and stripping
+ * only `-->` put it in the name: `<!-- trazum:prompt greeting--!>` produced the
+ * name `greeting--!>`. Found by CodeQL, which is the second time this week a
+ * pattern has been right about the case in front of it and wrong about the one
+ * beside it.
+ */
+const COMMENT_CLOSE = /--!?>\s*$/;
+
+/**
+ * What a name may be.
+ *
+ * An identifier charset rather than "whatever is left on the line". The name
+ * flows into `promptId`, which is printed in reports and matched against the
+ * budget patterns in `trazum.config.json` — a name is an identifier, and letting
+ * it be arbitrary text is how a stray terminator became part of one above. A
+ * candidate that does not fit falls back to the `file:line` form, which always
+ * works.
+ */
+const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+
+/**
+ * How far past the marker a literal may start.
+ *
+ * A prompt is expected on the next line or two — `const SYSTEM = \`` and the
+ * occasional type annotation. Scanning further would let a marker attach itself
+ * to an unrelated string much later in the file, which is the false positive this
+ * design exists to avoid.
+ */
+const MAX_GAP_LINES = 3;
+
+/**
+ * A forward-only line counter.
+ *
+ * Markers are found in increasing order, so each character is counted once
+ * across the whole scan. Recomputing the line by counting from zero at every
+ * marker is the obvious version and is quadratic in the number of markers —
+ * 15.5 seconds on a file of 20,000 of them, which the hostile-input tests
+ * caught before this shipped.
+ */
+function lineCounter(source: string): (offset: number) => number {
+  let seen = 0;
+  let line = 1;
+  return (offset) => {
+    for (; seen < offset && seen < source.length; seen++) {
+      if (source[seen] === '\n') line++;
+    }
+    return line;
+  };
+}
+
+const ESCAPES: Record<string, string> = {
+  n: '\n',
+  t: '\t',
+  r: '\r',
+  '\\': '\\',
+  '"': '"',
+  "'": "'",
+  '`': '`',
+  '0': '\0',
+};
+
+/**
+ * Reads a string literal starting at `start`, which must be its opening quote.
+ *
+ * Returns the resolved text and the offset just past the closing delimiter, or
+ * null when the literal never closes. Escapes are resolved for single- and
+ * double-quoted strings, where `\n` is one newline written as two characters;
+ * backtick and triple-quoted literals carry their newlines literally and only
+ * need the delimiter itself unescaped.
+ */
+function readLiteral(
+  source: string,
+  start: number,
+): { text: string; quote: ExtractedPrompt['quote']; end: number; textStart: number } | null {
+  const triple = source.slice(start, start + 3);
+  const quote: ExtractedPrompt['quote'] | null =
+    triple === '"""' || triple === "'''"
+      ? (triple as '"""' | "'''")
+      : source[start] === '`' || source[start] === '"' || source[start] === "'"
+        ? (source[start] as '`' | '"' | "'")
+        : null;
+  if (quote === null) return null;
+
+  const raw = quote === '`' || quote.length === 3;
+  const textStart = start + quote.length;
+  let out = '';
+  let i = textStart;
+
+  while (i < source.length) {
+    const ch = source[i]!;
+
+    if (ch === '\\') {
+      const next = source[i + 1];
+      if (next === undefined) break;
+      // In a raw literal only the delimiter and the backslash itself are
+      // escapes; `\n` there is a literal backslash followed by an n, and
+      // resolving it would invent a newline the prompt does not contain.
+      if (raw) {
+        out += next === quote[0] || next === '\\' ? next : `\\${next}`;
+      } else {
+        out += ESCAPES[next] ?? `\\${next}`;
+      }
+      i += 2;
+      continue;
+    }
+
+    if (source.startsWith(quote, i)) {
+      return { text: out, quote, end: i + quote.length, textStart };
+    }
+
+    // A single-quoted or double-quoted literal cannot span lines. Stopping here
+    // rather than running to the end of the file is what keeps a missing quote
+    // from swallowing the rest of the source.
+    if (quote.length === 1 && quote !== '`' && ch === '\n') return null;
+
+    out += ch;
+    i++;
+  }
+
+  return null;
+}
+
+/** Whether the expression continues past the literal with a concatenation. */
+function isConcatenated(source: string, end: number): boolean {
+  let i = end;
+  while (i < source.length && (source[i] === ' ' || source[i] === '\t')) i++;
+  // `+` in JS/TS, `.` in PHP-ish sources, and a bare newline followed by `+` is
+  // the wrapped form of the same thing. A trailing comma or semicolon is not.
+  if (source[i] === '+') return true;
+  if (source[i] === '\n') {
+    let j = i + 1;
+    while (j < source.length && /[ \t\n]/.test(source[j]!)) j++;
+    return source[j] === '+';
+  }
+  return false;
+}
+
+/**
+ * Finds every marked prompt in a source file.
+ *
+ * Returns what it found and what it refused, because a marker that produced
+ * nothing is the case the author most needs to hear about: they asked for the
+ * prompt to be governed and it is not being governed.
+ */
+export function extractPrompts(source: string): ExtractionResult {
+  const prompts: ExtractedPrompt[] = [];
+  const declined: DeclinedPrompt[] = [];
+
+  const lineAt = lineCounter(source);
+
+  let cursor = 0;
+  while (cursor < source.length) {
+    const at = source.indexOf(TAG, cursor);
+    if (at === -1) break;
+    cursor = at + TAG.length;
+
+    // The tag has to be inside a comment. Without this, the string 'trazum:prompt'
+    // appearing in a prompt's own text would mark the prompt after it.
+    const lineStart = source.lastIndexOf('\n', at) + 1;
+    const before = source.slice(lineStart, at).trimStart();
+    if (!MARKERS.some((m) => before.startsWith(m))) continue;
+
+    const markerLine = lineAt(at);
+
+    // The rest of the marker line is an optional name: `trazum:prompt greeting`.
+    const lineEnd = source.indexOf('\n', at);
+    const rest = source.slice(cursor, lineEnd === -1 ? source.length : lineEnd);
+    const candidate = rest.replace(COMMENT_CLOSE, '').trim().split(/\s+/)[0] ?? '';
+    const name = NAME.test(candidate) ? candidate : undefined;
+
+    // Scan forward for the opening delimiter, bounded so a marker cannot adopt
+    // a string much further down the file.
+    let i = lineEnd === -1 ? source.length : lineEnd + 1;
+    let lines = 0;
+    let opened: ReturnType<typeof readLiteral> = null;
+    let openAt = -1;
+
+    while (i < source.length && lines <= MAX_GAP_LINES) {
+      const ch = source[i]!;
+      if (ch === '\n') {
+        lines++;
+        i++;
+        continue;
+      }
+      if (ch === '`' || ch === '"' || ch === "'") {
+        openAt = i;
+        opened = readLiteral(source, i);
+        break;
+      }
+      i++;
+    }
+
+    if (openAt === -1) {
+      declined.push({
+        line: markerLine,
+        reason: 'no-literal',
+        detail: `no string literal within ${MAX_GAP_LINES} lines of the marker`,
+      });
+      continue;
+    }
+    if (opened === null) {
+      declined.push({
+        line: markerLine,
+        reason: 'unterminated',
+        detail: 'the literal is never closed',
+      });
+      continue;
+    }
+    if (isConcatenated(source, opened.end)) {
+      declined.push({
+        line: markerLine,
+        reason: 'concatenated',
+        detail:
+          'the prompt is built by concatenation, so its text does not exist until it runs',
+      });
+      cursor = opened.end;
+      continue;
+    }
+
+    prompts.push({
+      name,
+      line: markerLine,
+      text: opened.text,
+      quote: opened.quote,
+      start: opened.textStart,
+      end: opened.end - opened.quote.length,
+    });
+    cursor = opened.end;
+  }
+
+  return { prompts, declined };
+}
+
+/**
+ * A stable identifier for an embedded prompt, for budgets and reports.
+ *
+ * `src/prompts.ts#support` when the marker was named, `src/prompts.ts:12` when it
+ * was not. Both are path-prefixed so the glob patterns in `trazum.config.json`
+ * cover embedded prompts without learning a new syntax — `src/**` matches either.
+ */
+export function promptId(path: string, prompt: ExtractedPrompt): string {
+  return prompt.name ? `${path}#${prompt.name}` : `${path}:${prompt.line}`;
+}
+
+/** Whether a file is worth opening for extraction at all. */
+export const SOURCE_EXTENSIONS = [
+  '.ts',
+  '.tsx',
+  '.js',
+  '.jsx',
+  '.mjs',
+  '.py',
+  '.rb',
+  '.go',
+  '.rs',
+  '.java',
+  '.kt',
+  '.cs',
+  '.php',
+  '.yaml',
+  '.yml',
+];
+
+/** Cheap pre-filter: a source file with no marker has nothing to extract. */
+export const hasMarker = (source: string): boolean => source.includes(TAG);

package/src/gcp-auth.ts

@@ -0,0 +1,166 @@
+/**
+ * A Google service-account access token, without the SDK.
+ *
+ * Vertex will not take an API key. It wants an OAuth access token, and the only
+ * way to get one unattended is the two-legged flow: build a JWT, sign it with
+ * the service account's private key, and trade it at the token endpoint.
+ * `google-auth-library` is ninety-odd packages to do that; this library has zero
+ * runtime dependencies and a test that fails the build if one appears, because
+ * every dependency is somebody else's code reading your prompts.
+ *
+ * WebCrypto rather than `node:crypto`, so the browser-safe entry point stays
+ * browser-safe. `RSASSA-PKCS1-v1_5` with SHA-256 is what Google calls RS256.
+ */
+
+/** Base64url, no padding — what a JWT uses and what `btoa` does not produce. */
+function base64url(bytes: Uint8Array): string {
+  let binary = '';
+  for (const byte of bytes) binary += String.fromCharCode(byte);
+  return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+}
+
+const encoder = new TextEncoder();
+
+const encodeJson = (value: unknown): string => base64url(encoder.encode(JSON.stringify(value)));
+
+/**
+ * PEM to the DER bytes `importKey` wants.
+ *
+ * The header, the footer and every newline come out. A PEM that still has them
+ * fails inside WebCrypto with `DataError`, which says nothing about which of the
+ * fourteen things that can be wrong with a key is wrong with this one.
+ */
+export function pkcs8FromPem(pem: string): ArrayBuffer {
+  const body = pem
+    .replace(/-----BEGIN [^-]+-----/, '')
+    .replace(/-----END [^-]+-----/, '')
+    .replace(/\s+/g, '');
+  if (!body) throw new Error('The service account private key is empty.');
+
+  let binary: string;
+  try {
+    binary = atob(body);
+  } catch {
+    throw new Error('The service account private key is not valid base64.');
+  }
+  // Returned as an ArrayBuffer rather than a view: `importKey` accepts a
+  // `BufferSource`, and a `Uint8Array` over a `SharedArrayBuffer` is not one
+  // as far as the type checker is concerned.
+  return Uint8Array.from(binary, (character) => character.charCodeAt(0)).buffer as ArrayBuffer;
+}
+
+export interface ServiceAccount {
+  client_email: string;
+  private_key: string;
+  token_uri?: string;
+}
+
+/**
+ * The assertion Google trades for a token.
+ *
+ * Exported so a test can read it: this is the document where a wrong `aud`, a
+ * clock an hour out, or a scope nobody granted turns into `invalid_grant`, an
+ * error message that names none of the three.
+ */
+export async function signedJwt(
+  account: ServiceAccount,
+  scope: string,
+  now: Date,
+): Promise<string> {
+  const tokenUri = account.token_uri ?? 'https://oauth2.googleapis.com/token';
+  const issued = Math.floor(now.getTime() / 1000);
+
+  const header = encodeJson({ alg: 'RS256', typ: 'JWT' });
+  const claims = encodeJson({
+    iss: account.client_email,
+    scope,
+    aud: tokenUri,
+    iat: issued,
+    // An hour is Google's maximum. Longer is rejected outright rather than
+    // clamped, which is a confusing way to learn about a limit.
+    exp: issued + 3600,
+  });
+
+  const key = await crypto.subtle.importKey(
+    'pkcs8',
+    pkcs8FromPem(account.private_key),
+    { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' },
+    false,
+    ['sign'],
+  );
+
+  const signature = await crypto.subtle.sign(
+    'RSASSA-PKCS1-v1_5',
+    key,
+    encoder.encode(`${header}.${claims}`),
+  );
+
+  return `${header}.${claims}.${base64url(new Uint8Array(signature))}`;
+}
+
+export interface CachedToken {
+  token: string;
+  /** Epoch seconds. */
+  expiresAt: number;
+}
+
+/**
+ * Trades the assertion for an access token, caching until shortly before expiry.
+ *
+ * The cache is the point. A token lasts an hour and `optimize --suggest` over a
+ * directory makes one call per prompt: fetching a token each time turns forty
+ * prompts into eighty requests, half of them to an endpoint that rate-limits.
+ *
+ * Sixty seconds of margin, because a token that expires in flight fails the
+ * request it was fetched for, and the clock here is not the clock there.
+ */
+export async function accessToken(
+  account: ServiceAccount,
+  options: {
+    scope?: string;
+    fetchImpl?: typeof fetch;
+    now?: () => Date;
+    cache?: { current: CachedToken | null };
+  } = {},
+): Promise<string> {
+  const {
+    scope = 'https://www.googleapis.com/auth/cloud-platform',
+    fetchImpl = fetch,
+    now = () => new Date(),
+    cache,
+  } = options;
+
+  const at = now();
+  const seconds = Math.floor(at.getTime() / 1000);
+
+  if (cache?.current && cache.current.expiresAt - 60 > seconds) {
+    return cache.current.token;
+  }
+
+  const assertion = await signedJwt(account, scope, at);
+  const tokenUri = account.token_uri ?? 'https://oauth2.googleapis.com/token';
+
+  const res = await fetchImpl(tokenUri, {
+    method: 'POST',
+    redirect: 'error',
+    headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+      assertion,
+    }).toString(),
+  });
+
+  if (!res.ok) {
+    throw new Error(`Google refused the service account assertion (${res.status}): ${await res.text()}`);
+  }
+
+  const data = (await res.json()) as { access_token?: unknown; expires_in?: unknown };
+  if (typeof data.access_token !== 'string' || !data.access_token) {
+    throw new Error('Google returned no access_token for the service account.');
+  }
+
+  const lifetime = typeof data.expires_in === 'number' ? data.expires_in : 3600;
+  if (cache) cache.current = { token: data.access_token, expiresAt: seconds + lifetime };
+
+  return data.access_token;
+}

package/src/glob.ts

@@ -0,0 +1,160 @@
+/**
+ * A small, bounded glob matcher.
+ *
+ * Trazum has zero runtime dependencies and that is a security property, not a
+ * preference: a prompt optimiser reads your prompts, and every dependency is
+ * someone else's code doing that too. So the config file's budget patterns are
+ * matched here rather than by pulling in a glob library.
+ *
+ * **It is deliberately not a regex translation.** Turning `**` into
+ * `(?:[^/]*\/)*` produces exactly the nested-quantifier shape that makes a
+ * pattern take exponential time on the wrong input, and these patterns come
+ * from a file in a repository — which on a pull request means from whoever
+ * opened it. A segment-wise dynamic program has no such shape: it is O(pattern
+ * segments x path segments), and each segment match is O(pattern chars x path
+ * chars) with no backtracking beyond a single remembered star.
+ *
+ * Supported, and nothing else:
+ *   `*`   any run of characters within one path segment
+ *   `**`  any number of whole path segments, including none
+ *   `?`   exactly one character within one path segment
+ */
+
+/**
+ * Longest pattern and path this will consider.
+ *
+ * The matcher is quadratic in both, so a pattern nobody would write by hand is
+ * declined rather than allowed to spend the CI runner's afternoon. Both limits
+ * are far above any real prompt path.
+ */
+const MAX_PATTERN_LENGTH = 1024;
+const MAX_PATH_LENGTH = 4096;
+
+/** Normalises separators and strips the noise a hand-written path collects. */
+function toSegments(value: string): string[] {
+  return value
+    .replace(/\\/g, '/')
+    .split('/')
+    .filter((segment) => segment.length > 0 && segment !== '.');
+}
+
+/**
+ * Matches one path segment against one pattern segment (`*` and `?` only).
+ *
+ * Linear-with-backtracking over a single remembered star, which is the classic
+ * bounded algorithm: on a mismatch it retreats to the last star and advances
+ * the input by one, so the work is O(pattern x input) and never exponential.
+ */
+function matchSegment(pattern: string, value: string): boolean {
+  let p = 0;
+  let v = 0;
+  let star = -1;
+  let mark = 0;
+
+  while (v < value.length) {
+    if (p < pattern.length && (pattern[p] === '?' || pattern[p] === value[v])) {
+      p++;
+      v++;
+    } else if (p < pattern.length && pattern[p] === '*') {
+      star = p++;
+      mark = v;
+    } else if (star >= 0) {
+      p = star + 1;
+      v = ++mark;
+    } else {
+      return false;
+    }
+  }
+
+  while (p < pattern.length && pattern[p] === '*') p++;
+  return p === pattern.length;
+}
+
+/**
+ * True when `path` matches `pattern`.
+ *
+ * Both sides are treated as relative paths with `/` separators; a leading `./`
+ * and duplicate separators are ignored, and backslashes are read as separators
+ * so a Windows-shaped path still matches a pattern written with slashes.
+ */
+export function matchGlob(pattern: string, path: string): boolean {
+  if (pattern.length > MAX_PATTERN_LENGTH || path.length > MAX_PATH_LENGTH) return false;
+
+  const p = toSegments(pattern);
+  const s = toSegments(path);
+
+  // reachable[j] === true means "the first j pattern segments can consume the
+  // path segments seen so far". Walking the path once and updating this row is
+  // the whole algorithm; `**` is the only segment that can stay put.
+  let reachable = new Array<boolean>(p.length + 1).fill(false);
+  reachable[0] = true;
+  // Leading `**` segments match nothing at all, so they are reachable up front.
+  for (let j = 0; j < p.length && p[j] === '**'; j++) reachable[j + 1] = true;
+
+  for (const segment of s) {
+    const next = new Array<boolean>(p.length + 1).fill(false);
+    for (let j = 0; j < p.length; j++) {
+      if (!reachable[j] && !(p[j] === '**' && next[j])) continue;
+      if (p[j] === '**') {
+        // Consume this segment and stay on the same `**`, or step past it.
+        next[j] = true;
+        next[j + 1] = true;
+      } else if (matchSegment(p[j]!, segment)) {
+        next[j + 1] = true;
+      }
+    }
+    // A `**` that has just been stepped past may be followed by another one,
+    // which can also match nothing. Propagate that before moving on.
+    for (let j = 0; j < p.length; j++) {
+      if (next[j] && p[j] === '**') next[j + 1] = true;
+    }
+    reachable = next;
+  }
+
+  return reachable[p.length] === true;
+}
+
+/**
+ * How specific a pattern is, for deciding which of two matching patterns wins.
+ *
+ * "Most specific" needs a definition that is stated rather than felt, because
+ * a budget silently resolved from the wrong pattern is a budget nobody can
+ * debug. The rule: **more literal characters wins**, and a longer pattern
+ * breaks a tie. So `prompts/system.txt` beats `prompts/*.txt`, which beats
+ * `prompts/**`, which beats `**`.
+ */
+export function specificity(pattern: string): number {
+  const literals = pattern.replace(/[*?]/g, '').length;
+  // The multiplier has to exceed every possible tie-breaker value, or the
+  // primary rule stops holding: at a multiplier of 1,000 a pattern with one
+  // literal and 1,024 characters scores the same as one with two literals and
+  // 24, and the stated "more literals wins" quietly becomes a coin toss.
+  return literals * (MAX_PATTERN_LENGTH + 1) + pattern.length;
+}
+
+/**
+ * The most specific pattern in `patterns` that matches `path`, or null.
+ *
+ * Ties on specificity are broken by lexical order rather than by insertion
+ * order: object key order is easy to reorder by accident, and a budget that
+ * changes because two keys swapped places is the kind of bug that gets blamed
+ * on the tool.
+ */
+export function mostSpecificMatch(
+  patterns: readonly string[],
+  path: string,
+): string | null {
+  let best: string | null = null;
+  let bestScore = -1;
+
+  for (const pattern of patterns) {
+    if (!matchGlob(pattern, path)) continue;
+    const score = specificity(pattern);
+    if (score > bestScore || (score === bestScore && best !== null && pattern < best)) {
+      best = pattern;
+      bestScore = score;
+    }
+  }
+
+  return best;
+}