@trazum/core 1.8.0

package/src/optimize.ts

@@ -0,0 +1,228 @@
+import { buildAdvisories } from './advisories.js';
+import { extractChanges } from './changes.js';
+import { DEFAULT_LOCALE, getMessages } from './i18n/index.js';
+import type { Locale, RuleId } from './i18n/types.js';
+import { BUNDLED_CATALOGUE, DEFAULT_MODEL } from './pricing.js';
+import type { PricingCatalogue } from './pricing.js';
+import { RULES } from './rules.js';
+import { computeSavings } from './savings.js';
+import { join, segment } from './segment.js';
+import { estimateTokens } from './tokenizer.js';
+import type { AsyncTokenCounter } from './tokenizer.js';
+import type {
+  OptimizationResult,
+  OptimizeOptions,
+  RuleResult,
+  Segment,
+  TokenCounter,
+  UsageProfile,
+} from './types.js';
+
+const MASK_OPEN = '\uE000';
+const MASK_CLOSE = '\uE001';
+const MASK_BASE = 0xe100;
+/** Private-use-area slots available for masks. */
+const MASK_CAPACITY = 0xf8ff - MASK_BASE;
+
+export const DEFAULT_USAGE: UsageProfile = {
+  model: DEFAULT_MODEL,
+  callsPerMonth: 1000,
+  avgOutputTokens: 500,
+  cacheHitRate: 0.9,
+  batchEligible: false,
+};
+
+interface Masked {
+  text: string;
+  /** Original text of each protected segment, indexed by its mask. */
+  vault: string[];
+}
+
+/**
+ * Replaces every protected segment with a 3-character marker from the Unicode
+ * private use area. The rules work on the masked text, so none of them can
+ * touch code, URLs or template placeholders.
+ */
+function mask(segments: Segment[]): Masked {
+  const vault: string[] = [];
+  let text = '';
+
+  for (const seg of segments) {
+    if (seg.kind === 'mutable') {
+      text += seg.text;
+      continue;
+    }
+    if (vault.length >= MASK_CAPACITY) {
+      // A prompt with thousands of protected blocks is left as-is rather than
+      // risking a mask collision.
+      text += seg.text;
+      continue;
+    }
+    const slot = String.fromCharCode(MASK_BASE + vault.length);
+    vault.push(seg.text);
+    text += MASK_OPEN + slot + MASK_CLOSE;
+  }
+
+  return { text, vault };
+}
+
+/** Puts the protected content back. */
+function unmask(text: string, vault: string[]): string {
+  return text.replace(
+    new RegExp(`${MASK_OPEN}([\\s\\S])${MASK_CLOSE}`, 'g'),
+    (whole, slot: string) => {
+      const index = slot.charCodeAt(0) - MASK_BASE;
+      const original = vault[index];
+      return original === undefined ? whole : original;
+    },
+  );
+}
+
+/** Distinct protected texts that must still be present after optimising. */
+function distinctProtected(segments: Segment[]): string[] {
+  return [...new Set(segments.filter((s) => s.kind === 'protected').map((s) => s.text))];
+}
+
+/**
+ * Optimises a prompt by applying deterministic rules.
+ *
+ * The result is reproducible: the same input always yields the same output and
+ * running it costs nothing. The optional LLM pass lives separately, in
+ * `refineWithLlm`.
+ */
+export function optimize(prompt: string, options: OptimizeOptions = {}): OptimizationResult {
+  const level = options.level ?? 'safe';
+  const locale: Locale = options.locale ?? DEFAULT_LOCALE;
+  const t = getMessages(locale);
+  const disabled = new Set<RuleId>(options.disableRules ?? []);
+  const count: TokenCounter = options.tokenCounter ?? estimateTokens;
+
+  const segments = segment(prompt);
+  const mustSurvive = distinctProtected(segments);
+  const { text: maskedOriginal, vault } = mask(segments);
+
+  let current = maskedOriginal;
+  const ruleResults: RuleResult[] = [];
+
+  for (const rule of RULES) {
+    if (disabled.has(rule.id)) continue;
+    if (rule.level === 'aggressive' && level !== 'aggressive') continue;
+
+    const currentUnmasked = unmask(current, vault);
+    const before = count(currentUnmasked);
+    const { text: candidate, hits } = rule.apply(current);
+    if (hits === 0 || candidate === current) continue;
+
+    const candidateUnmasked = unmask(candidate, vault);
+
+    // Safety net: if a rule made protected content disappear, that rule is
+    // dropped rather than returning a broken prompt.
+    const lostProtected = mustSurvive.some((text) => !candidateUnmasked.includes(text));
+    if (lostProtected) continue;
+
+    const after = count(candidateUnmasked);
+    // Captured against the unmasked text on both sides, so the snippets read
+    // as the author wrote them rather than showing private-use markers.
+    const changes = extractChanges(currentUnmasked, candidateUnmasked);
+    current = candidate;
+    const copy = t.rules[rule.id];
+    ruleResults.push({
+      id: rule.id,
+      title: copy.title,
+      rationale: copy.rationale,
+      level: rule.level,
+      hits,
+      tokensSaved: Math.max(0, before - after),
+      changes,
+    });
+  }
+
+  const optimized = unmask(current, vault).trim();
+  const tokensBefore = count(prompt);
+  const tokensAfter = count(optimized);
+
+  const usage: UsageProfile = { ...DEFAULT_USAGE, ...options.usage };
+  const pricing = options.pricing ?? BUNDLED_CATALOGUE;
+  const savings = computeSavings(tokensBefore, tokensAfter, usage, new Date(), pricing);
+  const advisories = buildAdvisories(optimized, tokensAfter, usage, { count, locale, pricing });
+
+  return {
+    original: prompt,
+    optimized,
+    tokensBefore,
+    tokensAfter,
+    tokensSaved: tokensBefore - tokensAfter,
+    reductionPct: tokensBefore > 0 ? ((tokensBefore - tokensAfter) / tokensBefore) * 100 : 0,
+    rules: ruleResults,
+    advisories,
+    savings,
+    usage,
+    locale,
+    tokenSource: options.tokenCounter ? 'external' : 'heuristic',
+    pricingSource: {
+      lastReviewed: pricing.lastReviewed,
+      overriddenModels: pricing.overriddenModels,
+      addedModels: pricing.addedModels,
+    },
+  };
+}
+
+/**
+ * Recomputes the report with an exact token counter (e.g. the official
+ * token-counting endpoint).
+ *
+ * Only the headline numbers and the saving are recomputed. Per-rule token
+ * attribution stays on the heuristic estimate: asking a remote counter once
+ * per rule would multiply the calls without changing any decision.
+ */
+export async function withExactTokenCounts(
+  result: OptimizationResult,
+  counter: AsyncTokenCounter,
+  pricing: PricingCatalogue = BUNDLED_CATALOGUE,
+): Promise<OptimizationResult> {
+  // A result priced against an overlay cannot be recomputed against the bundled
+  // catalogue: the token counts would come from one source and the money from
+  // another, and the report would disagree with itself with nothing to show why.
+  // Throwing is the only honest option — silently reverting to bundled prices is
+  // the exact failure the overlay exists to prevent.
+  const overlaid =
+    result.pricingSource.overriddenModels.length > 0 ||
+    result.pricingSource.addedModels.length > 0;
+  if (overlaid && pricing === BUNDLED_CATALOGUE) {
+    throw new Error(
+      'This result was priced against a pricing overlay, so withExactTokenCounts ' +
+        'needs the same catalogue passed as its third argument. Recomputing against ' +
+        'the bundled prices would make the token counts and the costs disagree.',
+    );
+  }
+
+  const [tokensBefore, tokensAfter] = await Promise.all([
+    counter(result.original),
+    counter(result.optimized),
+  ]);
+
+  const savings = computeSavings(tokensBefore, tokensAfter, result.usage, new Date(), pricing);
+  const advisories = buildAdvisories(result.optimized, tokensAfter, result.usage, {
+    locale: result.locale,
+    pricing,
+  });
+
+  return {
+    ...result,
+    tokensBefore,
+    tokensAfter,
+    tokensSaved: tokensBefore - tokensAfter,
+    reductionPct: tokensBefore > 0 ? ((tokensBefore - tokensAfter) / tokensBefore) * 100 : 0,
+    savings,
+    advisories,
+    tokenSource: 'external',
+    pricingSource: {
+      lastReviewed: pricing.lastReviewed,
+      overriddenModels: pricing.overriddenModels,
+      addedModels: pricing.addedModels,
+    },
+  };
+}
+
+/** Re-exports of internals useful to anyone extending the library. */
+export { join, mask as maskProtected, segment, unmask as unmaskProtected };

package/src/otlp.ts

@@ -0,0 +1,179 @@
+/**
+ * Prompt cost as OpenTelemetry metrics, in the OTLP/HTTP JSON encoding.
+ *
+ * **Trazum writes the payload; it does not send it.** That is a decision, not a
+ * missing feature. Pushing to a collector would mean holding an endpoint and a
+ * credential, and this repository has twice shipped an SSRF where a URL reached
+ * `fetch` without being the URL that was checked. A command that writes a file
+ * has no such failure mode, and `curl --data-binary @metrics.json $OTLP_ENDPOINT`
+ * is one line in the pipeline that already has the credential.
+ *
+ * No dependency on `@opentelemetry/*` either — the JSON encoding is a stable,
+ * documented wire format, and this package has no runtime dependencies by design.
+ *
+ * The two encoding rules that are easy to get wrong and silently produce a
+ * payload a collector drops:
+ *
+ * - **64-bit integers are strings.** `timeUnixNano` and `asInt` are `int64` in
+ *   the protobuf, and JSON numbers cannot hold one exactly, so the JSON mapping
+ *   specifies strings. A collector reading `1786000000000000000` as a double
+ *   loses the last digits of every timestamp.
+ * - **A gauge is `asDouble` or `asInt`, never both**, and a money figure is a
+ *   double. Sending `asInt` for `$41.20` silently reports `41`.
+ */
+
+/** Attributes as OTLP encodes them: a list of key/value, not an object. */
+export interface OtlpAttribute {
+  key: string;
+  value: { stringValue: string } | { intValue: string } | { doubleValue: number };
+}
+
+export interface OtlpDataPoint {
+  attributes: OtlpAttribute[];
+  timeUnixNano: string;
+  asInt?: string;
+  asDouble?: number;
+}
+
+export interface OtlpMetric {
+  name: string;
+  description: string;
+  unit: string;
+  gauge: { dataPoints: OtlpDataPoint[] };
+}
+
+export interface OtlpPayload {
+  resourceMetrics: [
+    {
+      resource: { attributes: OtlpAttribute[] };
+      scopeMetrics: [{ scope: { name: string; version: string }; metrics: OtlpMetric[] }];
+    },
+  ];
+}
+
+/** What `doctor` knows, in the shape this module needs. */
+export interface OtlpInput {
+  /** Every prompt surveyed, with the count a budget would be compared against. */
+  prompts: readonly { path: string; tokens: number; overBudget: boolean; budgeted: boolean }[];
+  /** Advisories rolled up by id, as the survey reports them. */
+  findings: readonly { id: string; prompts: number; monthlyUsd: number | null }[];
+  model: string;
+  callsPerMonth: number;
+}
+
+const str = (key: string, value: string): OtlpAttribute => ({ key, value: { stringValue: value } });
+
+/** Rounded to the nearest nanosecond as an integer string, never a float. */
+function nanos(atMillis: number): string {
+  return `${Math.round(atMillis)}000000`;
+}
+
+/**
+ * `atMillis` is a parameter rather than a call to `Date.now()`.
+ *
+ * The same reason `computeSavings` takes a `Date`: a function that reads the
+ * clock cannot be asserted against a fixed expectation, and every test of it
+ * ends up asserting only its shape. Here the timestamp *is* half the payload.
+ */
+export function toOtlpMetrics(input: OtlpInput, atMillis: number): OtlpPayload {
+  const { prompts, findings, model, callsPerMonth } = input;
+  const time = nanos(atMillis);
+  const metrics: OtlpMetric[] = [];
+
+  const gauge = (
+    name: string,
+    description: string,
+    unit: string,
+    dataPoints: OtlpDataPoint[],
+  ): void => {
+    // An empty metric is not the same as a zero, and a collector charting an
+    // empty series draws nothing rather than a flat line at zero. Omitted.
+    if (dataPoints.length > 0) metrics.push({ name, description, unit, gauge: { dataPoints } });
+  };
+
+  gauge(
+    'trazum.prompt.tokens',
+    'Input tokens in a prompt as written, the figure a budget is compared against.',
+    '{token}',
+    prompts.map((p) => ({
+      attributes: [str('prompt.path', p.path)],
+      timeUnixNano: time,
+      asInt: `${Math.round(p.tokens)}`,
+    })),
+  );
+
+  // Per prompt rather than as one count, so a dashboard can name the offender
+  // instead of showing a number that went from 2 to 3.
+  gauge(
+    'trazum.prompt.over_budget',
+    'Whether this prompt exceeds the budget that covers it. 1 when it does.',
+    '1',
+    prompts
+      .filter((p) => p.budgeted)
+      .map((p) => ({
+        attributes: [str('prompt.path', p.path)],
+        timeUnixNano: time,
+        asInt: p.overBudget ? '1' : '0',
+      })),
+  );
+
+  gauge(
+    'trazum.prompts.unbudgeted',
+    'Prompts no budget pattern matches, so nothing is watching them.',
+    '{prompt}',
+    [
+      {
+        attributes: [],
+        timeUnixNano: time,
+        asInt: `${prompts.filter((p) => !p.budgeted).length}`,
+      },
+    ],
+  );
+
+  // Money is a double. Sending this as asInt would report $41.20 as 41 and the
+  // chart would look plausible.
+  gauge(
+    'trazum.advisory.monthly_usd',
+    'Estimated monthly USD an advisory would recover, summed across the prompts raising it.',
+    'USD',
+    findings
+      .filter((f): f is typeof f & { monthlyUsd: number } => f.monthlyUsd !== null)
+      .map((f) => ({
+        attributes: [str('advisory.id', f.id)],
+        timeUnixNano: time,
+        asDouble: f.monthlyUsd,
+      })),
+  );
+
+  gauge(
+    'trazum.advisory.prompts',
+    'How many prompts raise this advisory.',
+    '{prompt}',
+    findings.map((f) => ({
+      attributes: [str('advisory.id', f.id)],
+      timeUnixNano: time,
+      asInt: `${f.prompts}`,
+    })),
+  );
+
+  return {
+    resourceMetrics: [
+      {
+        // The model and call volume are resource attributes, not metric labels:
+        // every figure above was priced through them, and a dollar amount whose
+        // scenario is not recorded beside it is a number nobody can check later.
+        resource: {
+          attributes: [
+            str('service.name', 'trazum'),
+            str('trazum.model', model),
+            {
+              key: 'trazum.calls_per_month',
+              value: { intValue: `${Math.round(callsPerMonth)}` },
+            },
+          ],
+        },
+        scopeMetrics: [{ scope: { name: 'trazum', version: '1.0.0' }, metrics }],
+      },
+    ],
+  };
+}