@trazum/core 1.8.0

package/src/rules.ts

@@ -0,0 +1,324 @@
+import {
+  EMPHASIS_PREFIXES,
+  FILLER,
+  HEDGES,
+  INTENSIFIERS,
+  POLITENESS,
+  SELF_CHECK,
+  SHOUTED_WORDS,
+  VERBOSE_PHRASES,
+} from './phrases.js';
+import { jaccard, normalizeForCompare } from './similarity.js';
+import { EXAMPLE_FIELD_LINE } from './structure.js';
+import type { RuleId } from './i18n/types.js';
+import type { Rule } from './types.js';
+
+/** Escapes a literal so it can go inside a regular expression. */
+function escapeRe(literal: string): string {
+  return literal.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Builds a regex with accent-aware word boundaries.
+ * JavaScript's `\b` only knows ASCII, so "de" would match inside "ánde".
+ * Lookarounds over \p{L} avoid that.
+ */
+function phraseRegex(phrase: string, flags = 'giu'): RegExp {
+  const body = escapeRe(phrase).replace(/\s+/g, '\\s+');
+  return new RegExp(`(?<![\\p{L}\\p{N}_])${body}(?![\\p{L}\\p{N}_])`, flags);
+}
+
+/** Cleans up what deleting a phrase leaves behind: spaces and orphan punctuation. */
+function tidyAfterRemoval(text: string): string {
+  return (
+    text
+      // Repeated spaces inside a line (leading indentation untouched).
+      .replace(/([^\n\S])[^\S\n]+/g, '$1')
+      // Comma or semicolon left with a space in front of it.
+      .replace(/[^\S\n]+([,;:.!?])/g, '$1')
+      // Commas duplicated by the removal.
+      .replace(/,(\s*,)+/g, ',')
+      // Punctuation left at the start of a line.
+      .replace(/^[^\S\n]*[,;:]\s*/gm, '')
+      // Spanish opening exclamation/question mark left with no content.
+      .replace(/[¡¿]+(?=[^\S\n]*(?:[!?.,;:]|$))/gm, '')
+      // Duplicated sentence-final punctuation when two fragments join ("answer.!").
+      .replace(/([.!?])[^\S\n]*[.!?]+/g, '$1')
+      // A line left holding nothing but punctuation.
+      .replace(/^[^\S\n]*[.,;:!?¡¿]+[^\S\n]*$/gm, '')
+      // A single leading space is removal residue, not indentation.
+      .replace(/^[^\S\n](?=\S)/gm, '')
+      // Trailing spaces.
+      .replace(/[^\S\n]+$/gm, '')
+      // Restore the capital when the removal left a sentence starting lowercase.
+      // Sentence starts: beginning of the text, after strong punctuation, and
+      // the start of a new paragraph.
+      .replace(
+        /(^|[.!?]\s+|\n\n[^\S\n]*)(\p{Ll})/gu,
+        (_m, pre: string, ch: string) => pre + ch.toUpperCase(),
+      )
+  );
+}
+
+/**
+ * Deletion regex: on top of the phrase, it swallows the commas that delimited
+ * it. Without this, removing an aside such as "if you don't mind," leaves
+ * "and, classify it".
+ */
+function dropRegex(phrase: string): RegExp {
+  const body = escapeRe(phrase).replace(/\s+/g, '\\s+');
+  // The boundary check goes AFTER the optional comma: in front of it, the "y"
+  // of "y, si no te importa," would fail the match and the comma would stay.
+  return new RegExp(
+    `(?:,[^\\S\\n]*)?(?<![\\p{L}\\p{N}_])${body}[^\\S\\n]*,?(?![\\p{L}\\p{N}_])`,
+    'giu',
+  );
+}
+
+/** A rule that deletes a set of phrases. */
+function dropRule(id: RuleId, level: Rule['level'], phrases: readonly string[]): Rule {
+  // Longest first, so "thank you very much" wins over "thanks".
+  const sorted = [...phrases].sort((a, b) => b.length - a.length);
+  const regexes = sorted.map((p) => dropRegex(p));
+
+  return {
+    id,
+    level,
+    apply(text) {
+      let hits = 0;
+      let out = text;
+      for (const re of regexes) {
+        out = out.replace(re, () => {
+          hits++;
+          return '';
+        });
+      }
+      return { text: hits > 0 ? tidyAfterRemoval(out) : out, hits };
+    },
+  };
+}
+
+/** Keeps the original leading capital when substituting. */
+function matchCase(original: string, replacement: string): string {
+  if (!replacement) return replacement;
+  const firstChar = original[0];
+  if (firstChar && firstChar === firstChar.toUpperCase() && firstChar !== firstChar.toLowerCase()) {
+    return replacement[0]!.toUpperCase() + replacement.slice(1);
+  }
+  return replacement;
+}
+
+const whitespaceRule: Rule = {
+  id: 'whitespace',
+  level: 'safe',
+  apply(text) {
+    let hits = 0;
+    let out = text;
+
+    // The lookbehind is load-bearing, not decoration. Without it the engine
+    // restarts this match at every position inside a whitespace run, and when
+    // the run does not end the line it fails from each one — quadratic, and
+    // 17 seconds on a 100 KB line of spaces that the HTTP API happily accepts.
+    // Anchoring to the start of a run means each run is tried exactly once.
+    out = out.replace(/(?<![^\S\n])[^\S\n]+$/gm, () => {
+      hits++;
+      return '';
+    });
+    // Collapse repeated spaces only after the first non-space character.
+    out = out.replace(/(\S)[^\S\n]{2,}/g, (_m, ch: string) => {
+      hits++;
+      return `${ch} `;
+    });
+    out = out.replace(/\n{3,}/g, () => {
+      hits++;
+      return '\n\n';
+    });
+    out = out.replace(/\t/g, () => {
+      hits++;
+      return ' ';
+    });
+
+    return { text: out, hits };
+  },
+};
+
+const decorationRule: Rule = {
+  id: 'decoration',
+  level: 'safe',
+  apply(text) {
+    let hits = 0;
+    let out = text;
+
+    // A whole line of 8 or more identical decorative characters.
+    out = out.replace(/^[^\S\n]*([=\-_*~#])\1{7,}[^\S\n]*$/gm, () => {
+      hits++;
+      return '';
+    });
+    // Repeated exclamation or question marks, opening ones included.
+    out = out.replace(/([!?¡¿])\1{1,}/g, (_m, ch: string) => {
+      hits++;
+      return ch;
+    });
+    // Clean up the blank lines the deletion just left behind.
+    if (hits > 0) out = out.replace(/\n{3,}/g, '\n\n');
+
+    return { text: out, hits };
+  },
+};
+
+const verbosePhrasesRule: Rule = {
+  id: 'verbose-phrases',
+  level: 'safe',
+  apply(text) {
+    let hits = 0;
+    let out = text;
+    // Longest first so the most specific substitution wins.
+    const entries = [...VERBOSE_PHRASES].sort((a, b) => b[0].length - a[0].length);
+    for (const [from, to] of entries) {
+      const re = phraseRegex(from);
+      out = out.replace(re, (match) => {
+        hits++;
+        return matchCase(match, to);
+      });
+    }
+    return { text: hits > 0 ? tidyAfterRemoval(out) : out, hits };
+  },
+};
+
+const politenessRule = dropRule('politeness', 'safe', POLITENESS);
+const fillerRule = dropRule('filler', 'safe', FILLER);
+const intensifiersRule = dropRule('intensifiers', 'aggressive', INTENSIFIERS);
+const hedgesRule = dropRule('hedges', 'aggressive', HEDGES);
+const selfCheckRule = dropRule('self-check', 'aggressive', SELF_CHECK);
+
+const emphasisRule: Rule = {
+  id: 'emphasis',
+  level: 'aggressive',
+  apply(text) {
+    let hits = 0;
+    let out = text;
+
+    for (const prefix of EMPHASIS_PREFIXES) {
+      const re = new RegExp(`^([^\\S\\n]*)${escapeRe(prefix)}\\s*:\\s*`, 'gm');
+      out = out.replace(re, (_m, indent: string) => {
+        hits++;
+        return indent;
+      });
+    }
+
+    for (const word of SHOUTED_WORDS) {
+      const re = phraseRegex(word, 'gu'); // case-sensitive: only the shouted form
+      out = out.replace(re, (match) => {
+        hits++;
+        return match.toLowerCase();
+      });
+    }
+
+    // Dropping "IMPORTANT:" leaves the sentence starting lowercase: recapitalise.
+    return { text: hits > 0 ? tidyAfterRemoval(out) : out, hits };
+  },
+};
+
+const duplicateLinesRule: Rule = {
+  id: 'duplicate-lines',
+  level: 'safe',
+  apply(text) {
+    const lines = text.split('\n');
+    const seen = new Set<string>();
+    const kept: string[] = [];
+    let hits = 0;
+
+    for (const line of lines) {
+      const normalized = normalizeForCompare(line);
+      // A labelled example field is data, not repetition: two examples sharing
+      // an output line show that two inputs map to the same answer. Dropping
+      // the second leaves that example with no output.
+      if (normalized.length >= 25 && !EXAMPLE_FIELD_LINE.test(line)) {
+        if (seen.has(normalized)) {
+          hits++;
+          continue;
+        }
+        seen.add(normalized);
+      }
+      kept.push(line);
+    }
+
+    let out = kept.join('\n');
+    if (hits > 0) out = out.replace(/\n{3,}/g, '\n\n');
+    return { text: out, hits };
+  },
+};
+
+const duplicateBlocksRule: Rule = {
+  id: 'duplicate-blocks',
+  level: 'safe',
+  apply(text) {
+    const blocks = text.split(/\n{2,}/);
+    const seen = new Set<string>();
+    const kept: string[] = [];
+    let hits = 0;
+
+    for (const block of blocks) {
+      const normalized = normalizeForCompare(block);
+      if (normalized.length >= 40) {
+        if (seen.has(normalized)) {
+          hits++;
+          continue;
+        }
+        seen.add(normalized);
+      }
+      kept.push(block);
+    }
+
+    return { text: kept.join('\n\n'), hits };
+  },
+};
+
+const nearDuplicateBlocksRule: Rule = {
+  id: 'near-duplicate-blocks',
+  level: 'aggressive',
+  apply(text) {
+    const blocks = text.split(/\n{2,}/);
+    const keptNormalized: string[] = [];
+    const kept: string[] = [];
+    let hits = 0;
+
+    for (const block of blocks) {
+      const normalized = normalizeForCompare(block);
+      if (normalized.length >= 60) {
+        const isDuplicate = keptNormalized.some((prev) => jaccard(prev, normalized) >= 0.92);
+        if (isDuplicate) {
+          hits++;
+          continue;
+        }
+        keptNormalized.push(normalized);
+      }
+      kept.push(block);
+    }
+
+    return { text: kept.join('\n\n'), hits };
+  },
+};
+
+/** Every rule, in the order they must run. */
+export const RULES: readonly Rule[] = [
+  // Whole-block deletions first, so the rest works over less text.
+  duplicateBlocksRule,
+  nearDuplicateBlocksRule,
+  duplicateLinesRule,
+  // Then the phrase-level rules.
+  verbosePhrasesRule,
+  politenessRule,
+  fillerRule,
+  hedgesRule,
+  intensifiersRule,
+  selfCheckRule,
+  emphasisRule,
+  // And finally typographic cleanup, which picks up what the others left.
+  decorationRule,
+  whitespaceRule,
+];
+
+export function getRule(id: string): Rule | undefined {
+  return RULES.find((r) => r.id === id);
+}

package/src/savings.ts

@@ -0,0 +1,121 @@
+import {
+  BUNDLED_CATALOGUE,
+  COST_MULTIPLIERS,
+  effectivePricing,
+  modelFrom,
+  multipliersFor,
+} from './pricing.js';
+import type { PricingCatalogue } from './pricing.js';
+import type { CostBreakdown, SavingsReport, UsageProfile } from './types.js';
+
+/**
+ * Cost of a single call.
+ *
+ * `batchDiscount` defaults to Anthropic's 50% rather than being read from the
+ * model, because this function takes prices rather than a model. Callers that
+ * have the model — which is all of them inside this package — pass its own,
+ * and a provider with no batch API passes 1.
+ */
+export function costOfCall(
+  inputTokens: number,
+  outputTokens: number,
+  inputPerMTok: number,
+  outputPerMTok: number,
+  batch: boolean,
+  batchDiscount: number = COST_MULTIPLIERS.batch,
+): CostBreakdown {
+  const discount = batch ? batchDiscount : 1;
+  const inputUsd = (inputTokens / 1_000_000) * inputPerMTok * discount;
+  const outputUsd = (outputTokens / 1_000_000) * outputPerMTok * discount;
+  return { inputUsd, outputUsd, totalUsd: inputUsd + outputUsd };
+}
+
+function scale(cost: CostBreakdown, factor: number): CostBreakdown {
+  return {
+    inputUsd: cost.inputUsd * factor,
+    outputUsd: cost.outputUsd * factor,
+    totalUsd: cost.totalUsd * factor,
+  };
+}
+
+/**
+ * Compares cost before and after optimising.
+ *
+ * Important: output tokens are held constant. A shorter prompt often produces
+ * somewhat shorter answers, but that depends on the task and cannot be
+ * promised, so the saving reported here comes exclusively from input tokens.
+ */
+export function computeSavings(
+  tokensBefore: number,
+  tokensAfter: number,
+  usage: UsageProfile,
+  on: Date = new Date(),
+  pricing: PricingCatalogue = BUNDLED_CATALOGUE,
+): SavingsReport {
+  const model = modelFrom(pricing, usage.model);
+  const { inputPerMTok, outputPerMTok, promoApplied } = effectivePricing(model, on);
+  // A provider with no batch API gets no discount even when the caller ticked
+  // the box: `batchEligible` describes the work, not what the provider sells.
+  const batchDiscount = multipliersFor(model).batch ?? 1;
+
+  const before = costOfCall(
+    tokensBefore,
+    usage.avgOutputTokens,
+    inputPerMTok,
+    outputPerMTok,
+    usage.batchEligible,
+    batchDiscount,
+  );
+  const after = costOfCall(
+    tokensAfter,
+    usage.avgOutputTokens,
+    inputPerMTok,
+    outputPerMTok,
+    usage.batchEligible,
+    batchDiscount,
+  );
+
+  const monthBefore = scale(before, usage.callsPerMonth);
+  const monthAfter = scale(after, usage.callsPerMonth);
+  const monthlySavingsUsd = monthBefore.totalUsd - monthAfter.totalUsd;
+
+  return {
+    model: model.id,
+    modelDisplayName: model.displayName,
+    promoApplied,
+    perCall: { before, after },
+    perMonth: { before: monthBefore, after: monthAfter },
+    monthlySavingsUsd,
+    monthlySavingsPct:
+      monthBefore.totalUsd > 0 ? (monthlySavingsUsd / monthBefore.totalUsd) * 100 : 0,
+  };
+}
+
+/**
+ * Formats a USD amount with precision suited to its magnitude.
+ * Currency formatting stays in `en-US` on purpose: these are US dollar prices
+ * from a US price list, and showing them the same way everywhere avoids
+ * confusion when a report is shared across locales.
+ */
+export function formatUsd(value: number): string {
+  if (value === 0) return '$0';
+  const abs = Math.abs(value);
+  if (abs < 0.01) return `$${value.toFixed(5)}`;
+  if (abs < 1) return `$${value.toFixed(4)}`;
+  if (abs < 1000) return `$${value.toFixed(2)}`;
+  return `$${value.toLocaleString('en-US', { maximumFractionDigits: 0 })}`;
+}
+
+/**
+ * Formats a USD amount that can legitimately be negative, with an explicit
+ * sign.
+ *
+ * `formatUsd` renders a negative as `$-30.80`, which reads as a typo. In a
+ * comparison every cost line can go either way and the sign carries the whole
+ * meaning, so it goes in front of the currency where a reader expects it.
+ */
+export function formatSignedUsd(value: number): string {
+  if (value === 0) return '$0';
+  const rendered = formatUsd(Math.abs(value));
+  return value > 0 ? `+${rendered}` : `-${rendered}`;
+}

package/src/segment.ts

@@ -0,0 +1,106 @@
+import type { ProtectionKind, Segment } from './types.js';
+
+/**
+ * Splits the prompt into mutable and protected slices.
+ *
+ * Protected content is NEVER touched. Compressing a code block, a URL or a
+ * template placeholder would break the prompt, and that is exactly the failure
+ * that makes a prompt optimiser useless.
+ */
+
+interface PatternDef {
+  kind: ProtectionKind;
+  regex: RegExp;
+  /**
+   * Characters to trim from the end of a match. Needed for URLs: the full stop
+   * in "see https://example.com/guide." belongs to the sentence, not the URL,
+   * and protecting it leaves duplicated punctuation once the rest is cleaned.
+   */
+  trimTrailing?: RegExp;
+}
+
+// Order matters: the first pattern to match at a position wins.
+const PATTERNS: PatternDef[] = [
+  // Fenced code blocks with ``` or ~~~ (unclosed blocks run to the end).
+  { kind: 'fenced-code', regex: /(?:```|~~~)[\s\S]*?(?:```|~~~|$)/g },
+  // Indented blocks of 4+ spaces are left to the rules: they are ambiguous in markdown.
+  { kind: 'url', regex: /\b(?:https?|ftp):\/\/[^\s<>"')\]]+/g, trimTrailing: /[.,;:!?]+$/ },
+  // Template placeholders: {{var}}, {var}, ${var}, {% tag %}, <<VAR>>, %(var)s
+  {
+    kind: 'placeholder',
+    regex:
+      /\{\{[^{}]*\}\}|\{%[\s\S]*?%\}|\$\{[^{}]*\}|<<[A-Z0-9_]+>>|%\([^()]*\)[sdfr]|\{[A-Za-z_][A-Za-z0-9_.]*\}/g,
+  },
+  // Inline code spans.
+  { kind: 'inline-code', regex: /`[^`\n]+`/g },
+  // XML/HTML tags, heavily used to structure prompts.
+  { kind: 'xml-tag', regex: /<\/?[A-Za-z][\w:.-]*(?:\s[^<>]*?)?\/?>/g },
+];
+
+interface Match {
+  start: number;
+  end: number;
+  kind: ProtectionKind;
+}
+
+/** Splits the text into segments, marking what must not be modified. */
+export function segment(text: string): Segment[] {
+  const matches: Match[] = [];
+
+  for (const { kind, regex, trimTrailing } of PATTERNS) {
+    // Copy the regex so lastIndex is not shared between calls.
+    const re = new RegExp(regex.source, regex.flags);
+    let m: RegExpExecArray | null;
+    while ((m = re.exec(text)) !== null) {
+      let matched = m[0];
+      if (matched.length === 0) {
+        re.lastIndex++;
+        continue;
+      }
+      if (trimTrailing) matched = matched.replace(trimTrailing, '');
+      if (matched.length === 0) continue;
+      matches.push({ start: m.index, end: m.index + matched.length, kind });
+    }
+  }
+
+  // Sort by start and, on a tie, longest span first.
+  matches.sort((a, b) => a.start - b.start || b.end - a.end);
+
+  // Drop overlaps, keeping the first match at each position.
+  const kept: Match[] = [];
+  let cursor = 0;
+  for (const match of matches) {
+    if (match.start < cursor) continue;
+    kept.push(match);
+    cursor = match.end;
+  }
+
+  const segments: Segment[] = [];
+  let pos = 0;
+  for (const match of kept) {
+    if (match.start > pos) {
+      segments.push({ kind: 'mutable', text: text.slice(pos, match.start) });
+    }
+    segments.push({
+      kind: 'protected',
+      protection: match.kind,
+      text: text.slice(match.start, match.end),
+    });
+    pos = match.end;
+  }
+  if (pos < text.length) {
+    segments.push({ kind: 'mutable', text: text.slice(pos) });
+  }
+
+  return segments;
+}
+
+/** Joins segments back into a single string. */
+export function join(segments: Segment[]): string {
+  return segments.map((s) => s.text).join('');
+}
+
+/** Returns only the text of the protected segments, in order. */
+export function protectedTexts(segments: Segment[]): string[] {
+  return segments.filter((s) => s.kind === 'protected').map((s) => s.text);
+}