@trazum/core 1.81.0 → 1.83.0

package/src/helicone.ts

@@ -0,0 +1,227 @@
+/**
+ * Helicone's request export, read as a usage log.
+ *
+ * The third converter in the pattern `from-claude-code` started: a pure
+ * function turns one tool's export into Trazum's usage-log records, and every
+ * door prices it from there. Helicone sits as a proxy in front of the provider
+ * and keeps every request it saw, so a team using it already has the export.
+ *
+ * **The format is derived, not guessed.** The columns below are the SELECT
+ * that builds Helicone's own request table — `web/lib/api/request/request.ts`
+ * in Helicone/helicone — and the response shape its `POST /v1/request/query`
+ * endpoint documents. A converter written from memory of an API is a converter
+ * that silently mis-reads somebody's bill.
+ *
+ * ## The model, and why it takes three columns to answer
+ *
+ * Helicone carries `request_model`, `model_override` and `response_model`, and
+ * they can disagree: the override exists precisely because a proxy can send a
+ * different model than the caller asked for, and the response says what
+ * actually answered. **The response wins**, then the override, then the
+ * request — the bill is about what was billed, not what was intended — and
+ * `modelDisagreements` counts the rows where they differed so a reader can see
+ * the substitution happened rather than discover it in a total.
+ *
+ * ## What deliberately does not cross
+ *
+ * A Helicone row carries the request body and the response body — the prompt
+ * and the completion — and `request_user_id`, which is an email address in
+ * Helicone's own documented example. **None of it is read.** The converter
+ * names the fields it takes and takes nothing else, and a fixture plants a
+ * marker in each and greps the whole output.
+ *
+ * ## What it refuses to invent
+ *
+ * There is no cache-token split anywhere on the row: `cache_enabled` is a flag
+ * on the analytics table. A converted record therefore carries no cache fields
+ * and the caching questions come back `cannot-tell` rather than answered from
+ * a guess — the same refusal `from-otel` and `from-litellm` make.
+ *
+ * There is no conversation identity either. `request_id` is one call, so the
+ * records carry no `session` and the conversation-shaped findings stay
+ * unavailable. A custom property can carry one, and then it is there because
+ * the operator put it there — never because this file inferred it.
+ */
+
+/** One converted record, shaped exactly as `parseUsageLine` reads it. */
+export interface HeliconeRecord {
+  model: string;
+  ts?: string;
+  label?: string;
+  usage: {
+    input_tokens: number;
+    output_tokens: number;
+  };
+}
+
+export interface HeliconeConversion {
+  records: HeliconeRecord[];
+  /** Rows that were requests and converted. */
+  rows: number;
+  /** Rows naming no model in any of the three columns: counted, never guessed. */
+  unnamedModel: number;
+  /**
+   * Rows where the model that answered was not the model that was asked for.
+   *
+   * The response's model is what the bill rests on, and a proxy substituting
+   * one model for another is a fact worth seeing rather than a difference
+   * that only shows up as an unexplained total.
+   */
+  modelDisagreements: number;
+  /** Rows carrying no token counts at all — a logged request nobody can price. */
+  noTokens: number;
+  /** Rows Helicone served from its cache, with no token split behind the flag. */
+  cacheFlagged: number;
+  /** Lines or documents that did not parse as JSON at all. */
+  unparseable: number;
+}
+
+const asObject = (value: unknown): Record<string, unknown> | null =>
+  typeof value === 'object' && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+
+const tryParse = (text: string): unknown => {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+};
+
+const asCount = (value: unknown): number | undefined => {
+  const n = typeof value === 'string' ? Number(value) : value;
+  return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined;
+};
+
+const asName = (value: unknown): string | null =>
+  typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
+
+/**
+ * The label, from Helicone's custom properties.
+ *
+ * `request_properties` is the documented place for whatever a team tags its
+ * calls with, and the keys are theirs. Trazum looks for the ones a workload
+ * would plausibly be named by, in order, and takes the first — never joins
+ * several, because a workload has one name in a bill.
+ */
+function labelOf(row: Record<string, unknown>): string | undefined {
+  const properties = asObject(row.request_properties) ?? asObject(row.properties);
+  if (properties === null) return undefined;
+  for (const key of ['Helicone-Property-Label', 'label', 'workload', 'app', 'environment']) {
+    const found = asName(properties[key]);
+    if (found !== null) return found;
+  }
+  return undefined;
+}
+
+/** The request's creation time as an ISO instant, or undefined. */
+function timeOf(row: Record<string, unknown>): string | undefined {
+  const raw = row.request_created_at ?? row.created_at ?? row.response_created_at;
+  if (typeof raw !== 'string' && typeof raw !== 'number') return undefined;
+  const ms = typeof raw === 'number' ? raw : Date.parse(raw);
+  if (!Number.isFinite(ms)) return undefined;
+  return new Date(ms).toISOString();
+}
+
+/**
+ * Convert a Helicone request export.
+ *
+ * Accepts a JSON array of rows, a single row, `{ data: [...] }` as the query
+ * endpoint returns it, or newline-delimited rows. Pure over its input.
+ */
+export function heliconeRecords(text: string): HeliconeConversion {
+  const out: HeliconeConversion = {
+    records: [],
+    rows: 0,
+    unnamedModel: 0,
+    modelDisagreements: 0,
+    noTokens: 0,
+    cacheFlagged: 0,
+    unparseable: 0,
+  };
+
+  const rows: unknown[] = [];
+  const whole = tryParse(text);
+  if (whole !== undefined) {
+    if (Array.isArray(whole)) rows.push(...whole);
+    else {
+      const doc = asObject(whole);
+      if (doc !== null && Array.isArray(doc.data)) rows.push(...doc.data);
+      else if (doc !== null) rows.push(doc);
+    }
+  } else {
+    for (const line of text.split('\n')) {
+      if (line.trim() === '') continue;
+      const parsed = tryParse(line);
+      if (parsed === undefined) out.unparseable += 1;
+      else if (Array.isArray(parsed)) rows.push(...parsed);
+      else rows.push(parsed);
+    }
+  }
+
+  for (const raw of rows) {
+    const row = asObject(raw);
+    if (row === null) continue;
+    const input = asCount(row.prompt_tokens);
+    const output = asCount(row.completion_tokens);
+    // A request row is one carrying at least one token column. Anything else
+    // in the same file is not a call and is skipped rather than recorded as a
+    // zero, which would be a call nobody made.
+    if (input === undefined && output === undefined) continue;
+
+    out.rows += 1;
+    if (input === 0 && output === 0) out.noTokens += 1;
+    if (row.cache_enabled === true || row.cache_enabled === 1 || row.cache_enabled === '1') {
+      out.cacheFlagged += 1;
+    }
+
+    /*
+      The response first, and this order is the whole point of counting the
+      disagreement. A proxy that substituted a model billed for the model that
+      answered, so that is what a bill has to price; taking the requested one
+      would report the intention and call it the cost.
+    */
+    const answered = asName(row.response_model);
+    const overridden = asName(row.model_override);
+    const requested = asName(row.request_model);
+    const model = answered ?? overridden ?? requested;
+    if (model === null) {
+      out.unnamedModel += 1;
+      continue;
+    }
+    if (requested !== null && answered !== null && requested !== answered) {
+      out.modelDisagreements += 1;
+    }
+
+    const ts = timeOf(row);
+    const label = labelOf(row);
+    out.records.push({
+      model,
+      ...(ts !== undefined ? { ts } : {}),
+      ...(label !== undefined ? { label } : {}),
+      usage: { input_tokens: input ?? 0, output_tokens: output ?? 0 },
+    });
+  }
+
+  return out;
+}
+
+/**
+ * Whether a file looks like a Helicone request export, by shape.
+ *
+ * Deliberately narrow: a token column beside one of the names only Helicone's
+ * own SELECT produces. A looser test would claim an OpenAI usage response or a
+ * LiteLLM spend log — both of which also carry `prompt_tokens` — and refuse it
+ * in this file's words instead of letting the code that can read it try.
+ */
+export function looksLikeHelicone(text: string, prefixBytes = 8192): boolean {
+  const head = text.slice(0, prefixBytes);
+  if (!head.includes('prompt_tokens') && !head.includes('completion_tokens')) return false;
+  return (
+    head.includes('"request_created_at"') ||
+    head.includes('"request_model"') ||
+    head.includes('"response_model"') ||
+    head.includes('"request_properties"')
+  );
+}

package/src/i18n/en.ts

@@ -154,10 +154,12 @@ export const en: CoreMessages = {
       };
     },
 
-    promptCachingNotWorthIt: () => ({
+    promptCachingNotWorthIt: ({ readPct, writePct, breakEvenPct }) => ({
       title: 'At that hit rate, caching does not pay off',
       detail:
-        'A cache write costs 125% of the input price and a read costs 10%. Below roughly a 28% hit rate you pay more than you save. Either raise prefix reuse or leave caching off.',
+        breakEvenPct === null
+          ? `A cache write costs ${writePct}% of the input price on this model, the same as not caching, and a read costs ${readPct}%. So caching cannot lose you money here: at this hit rate it simply buys nothing. Raise prefix reuse and it starts paying immediately.`
+          : `A cache write costs ${writePct}% of the input price on this model and a read costs ${readPct}%. Below a ${breakEvenPct}% hit rate you pay more than you save. Either raise prefix reuse or leave caching off.`,
     }),
 
     belowCacheMinimum: ({
@@ -198,7 +200,12 @@ export const en: CoreMessages = {
 
     modelDowngrade: ({ modelName, tier, candidateName, currentUsd, candidateUsd }) => ({
       title: `This task may not need ${modelName}`,
-      detail: `By length and vocabulary the prompt looks like "${tier}" complexity. On ${candidateName} you would go from ${currentUsd} to ${candidateUsd} per month. This is a keyword heuristic, not a judgement about answer quality: measure the difference with your own evaluations before switching in production.`,
+      detail: `By length and vocabulary the prompt looks like "${tier}" complexity. On ${candidateName} you would go from ${currentUsd} to ${candidateUsd} per month. This is a keyword heuristic, not a judgement about answer quality. Measure it before switching in production: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <cases> --yes\` sends the same cases to both models and reports whether the cheaper one still does the job.`,
+    }),
+
+    tierSignalsConflict: ({ complexSignals, simpleSignals }) => ({
+      title: 'The prompt asks for depth and for brevity at once',
+      detail: `It carries ${complexSignals} signal${complexSignals === 1 ? '' : 's'} of a hard task and ${simpleSignals} of an easy one, and this heuristic scores those against each other, so they cancel. No tier is recommended here: a number produced by two opposite readings cancelling out looks exactly like a prompt with no signals at all, and they are not the same thing. Decide it by measurement rather than by vocabulary: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <cases> --yes\`.`,
     }),
 
     outputDominated: ({ outputUsd, inputUsd }) => ({
@@ -211,6 +218,15 @@ export const en: CoreMessages = {
       detail: `${modelName} has introductory pricing of ${promoInput}/${promoOutput} per million tokens until ${until}. After that date it moves to ${listInput}/${listOutput}: your bill goes up even if you change nothing.`,
     }),
 
+    modelRetired: ({ modelName, modelId, on, because }) => ({
+      title: `${modelName} is priced here and refused by its provider`,
+      detail:
+        `A real request for \`${modelId}\` was answered with an error on ${on}: "${because}" `
+        + 'The price above is still the right one for calls already in your log, because that '
+        + 'money was spent. A new call on this id will not run. Nothing here proposes what the '
+        + 'replacement costs: that figure comes off the provider\'s own pricing page or not at all.',
+    }),
+
     contradictoryInstructions: ({
       axis,
       firstValue,

package/src/i18n/es.ts

@@ -138,10 +138,12 @@ export const es: CoreMessages = {
       };
     },
 
-    promptCachingNotWorthIt: () => ({
+    promptCachingNotWorthIt: ({ readPct, writePct, breakEvenPct }) => ({
       title: 'Con esa tasa de acierto, la caché no compensa',
       detail:
-        'Escribir en caché cuesta un 125% del precio de entrada y leer un 10%. Por debajo de un ~28% de aciertos pagas más de lo que ahorras. Sube la reutilización del prefijo o deja la caché desactivada.',
+        breakEvenPct === null
+          ? `En este modelo escribir en caché cuesta un ${writePct}% del precio de entrada, lo mismo que no usarla, y leer un ${readPct}%. Así que aquí la caché no puede costarte dinero: con esta tasa de acierto simplemente no te aporta nada. En cuanto subas la reutilización del prefijo, empieza a compensar.`
+          : `En este modelo escribir en caché cuesta un ${writePct}% del precio de entrada y leer un ${readPct}%. Por debajo de un ${breakEvenPct}% de aciertos pagas más de lo que ahorras. Sube la reutilización del prefijo o deja la caché desactivada.`,
     }),
 
     belowCacheMinimum: ({
@@ -182,7 +184,12 @@ export const es: CoreMessages = {
 
     modelDowngrade: ({ modelName, tier, candidateName, currentUsd, candidateUsd }) => ({
       title: `Esta tarea quizá no necesite ${modelName}`,
-      detail: `Por longitud y vocabulario, el prompt parece de complejidad "${tier}". Con ${candidateName} pasarías de ${currentUsd} a ${candidateUsd} al mes. Es una heurística por palabras clave, no un juicio de calidad: mide la diferencia con tus propias evaluaciones antes de cambiar en producción.`,
+      detail: `Por longitud y vocabulario, el prompt parece de complejidad "${tier}". Con ${candidateName} pasarías de ${currentUsd} a ${candidateUsd} al mes. Es una heurística por palabras clave, no un juicio de calidad. Mídelo antes de cambiar en producción: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <casos> --yes\` envía los mismos casos a ambos modelos y dice si el barato sigue haciendo el trabajo.`,
+    }),
+
+    tierSignalsConflict: ({ complexSignals, simpleSignals }) => ({
+      title: 'El prompt pide profundidad y brevedad a la vez',
+      detail: `Lleva ${complexSignals} señal${complexSignals === 1 ? '' : 'es'} de tarea difícil y ${simpleSignals} de tarea fácil, y esta heurística las resta entre sí, así que se anulan. Aquí no se recomienda ningún nivel: un número que sale de dos lecturas opuestas cancelándose se parece exactamente a un prompt sin ninguna señal, y no son lo mismo. Decídelo midiendo y no por vocabulario: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <casos> --yes\`.`,
     }),
 
     outputDominated: ({ outputUsd, inputUsd }) => ({
@@ -195,6 +202,16 @@ export const es: CoreMessages = {
       detail: `${modelName} tiene precio de lanzamiento ${promoInput}/${promoOutput} por millón de tokens hasta el ${until}. A partir de esa fecha pasa a ${listInput}/${listOutput}: tu factura subirá aunque no cambies nada.`,
     }),
 
+    modelRetired: ({ modelName, modelId, on, because }) => ({
+      title: `${modelName} está tarifado aquí y su proveedor lo rechaza`,
+      detail:
+        `Una petición real a \`${modelId}\` se respondió con un error el ${on}: "${because}" `
+        + 'El precio de arriba sigue siendo el correcto para las llamadas que ya están en tu '
+        + 'registro, porque ese dinero se gastó. Una llamada nueva a este id no se ejecutará. '
+        + 'Aquí no se propone lo que cuesta el reemplazo: esa cifra sale de la página de precios '
+        + 'del proveedor o no sale.',
+    }),
+
     contradictoryInstructions: ({
       axis,
       firstValue,

package/src/i18n/types.ts

@@ -160,6 +160,23 @@ export interface OutputDominatedParams {
   inputUsd: string;
 }
 
+/**
+ * The provider's own refusal, carried through to the reader unchanged.
+ *
+ * `because` is quoted rather than summarised: it is the only part of this
+ * message that is evidence, it usually names the replacement, and a paraphrase
+ * of a provider's error is a sentence this repository would then be the author
+ * of.
+ */
+export interface ModelRetiredParams {
+  modelName: string;
+  modelId: string;
+  /** ISO date the refusal was recorded. */
+  on: string;
+  /** The provider's own words. */
+  because: string;
+}
+
 export interface PromoPricingParams {
   modelName: string;
   promoInput: number;
@@ -215,6 +232,49 @@ export interface LlmMessages {
   suspiciousShrink(retainedPct: number): string;
 }
 
+/**
+ * What the caching-does-not-pay advisory needs to state a threshold that is
+ * true about **this** model.
+ *
+ * It used to take nothing and print one sentence for every model: *"a cache
+ * write costs 125% of the input price and a read costs 10%. Below roughly a 28%
+ * hit rate you pay more than you save."* Two of those three numbers were the
+ * Anthropic multipliers stated as universal, and the third was not derivable
+ * from any model in the catalogue: at 125% and 10% the break-even is 21.74%,
+ * not 28%.
+ *
+ * The multipliers were already in hand at the call site and already passed to
+ * the advisory next door. Only this message invented its own.
+ */
+export interface PromptCachingNotWorthItParams {
+  /** The model's cache read multiplier as a percentage of the input price. */
+  readPct: number;
+  /** Its 5-minute cache write multiplier, likewise. */
+  writePct: number;
+  /**
+   * The hit rate at which caching stops costing money, derived rather than
+   * quoted: `(1 - write) / (read - write)`.
+   *
+   * `null` when the write multiplier is 1, which is not a smaller threshold but
+   * a different situation: writing costs exactly what not caching costs, so
+   * caching can never lose money and there is no rate below which it does. Eight
+   * of the eighteen models in the catalogue are that shape, and the old sentence
+   * told all eight to consider turning caching off.
+   */
+  breakEvenPct: number | null;
+}
+
+/**
+ * What the tier heuristic saw when it declined to answer.
+ *
+ * Both counts, because "three of each" and "one of each" are different degrees
+ * of the same refusal and the reader can act on the difference.
+ */
+export interface TierSignalsConflictParams {
+  complexSignals: number;
+  simpleSignals: number;
+}
+
 /** Why a proposed rewrite did not survive checking against the prompt. */
 export interface SuggestMessages {
   'not-found'(): string;
@@ -236,13 +296,15 @@ export interface CoreMessages {
     contextOverflow(p: ContextOverflowParams): LocalizedMessage;
     contextNearLimit(p: ContextNearLimitParams): LocalizedMessage;
     promptCaching(p: PromptCachingParams): LocalizedMessage;
-    promptCachingNotWorthIt(): LocalizedMessage;
+    promptCachingNotWorthIt(p: PromptCachingNotWorthItParams): LocalizedMessage;
     belowCacheMinimum(p: BelowCacheMinimumParams): LocalizedMessage;
     cachePrefixReorder(p: CachePrefixReorderParams): LocalizedMessage;
     batchApi(): LocalizedMessage;
     modelDowngrade(p: ModelDowngradeParams): LocalizedMessage;
+    tierSignalsConflict(p: TierSignalsConflictParams): LocalizedMessage;
     outputDominated(p: OutputDominatedParams): LocalizedMessage;
     promoPricing(p: PromoPricingParams): LocalizedMessage;
+    modelRetired(p: ModelRetiredParams): LocalizedMessage;
     contradictoryInstructions(p: ContradictoryInstructionsParams): LocalizedMessage;
     redundantExamples(p: RedundantExamplesParams): LocalizedMessage;
     restatedOutputFormat(p: RestatedOutputFormatParams): LocalizedMessage;

package/src/index.ts

@@ -52,6 +52,14 @@ export { explainGateFailure, gateMargin, GATE_MARGIN_TIGHT } from './gate-explai
 export { measuredUsage, labelCoverage, MIN_SCALE_DAYS, SCALE_TO_DAYS } from './measured-profile.js';
 export { assignSources, fleetRollup } from './fleet.js';
 export { rollUp } from './rollup.js';
+export { receiptFrom, RECEIPT_LINE_FIELDS } from './receipt.js';
+export type {
+  ReceiptDocument,
+  ReceiptGap,
+  ReceiptLine,
+  ReceiptOptions,
+  ReceiptPricing,
+} from './receipt.js';
 export { heartbeats } from './heartbeat.js';
 export { ruleYield } from './rule-yield.js';
 export { buildPlan, planLabelName, parsePlanDocument, PLAN_ACTION_KINDS } from './plan.js';
@@ -166,7 +174,18 @@ export type {
   GatewayStanding,
   RefuseReason,
 } from './gateway.js';
+export {
+  BANDS,
+  MEASURED_FOREIGN_ERROR_PCT,
+  bandFor,
+  bucketFor,
+  foreignTokenizer,
+  measuredForeignError,
+} from './band.js';
+export type { BandBucket } from './band.js';
 export { conform, CONTRACT_NAMES, requiredFieldsOf } from './conform.js';
+export { readDroppedVerdict, verdictMatchesSlice } from './verdict-bridge.js';
+export type { DroppedVerdict, BridgeReading } from './verdict-bridge.js';
 export { CONTRACT_SCHEMAS, contractSchema } from './schemas.js';
 export type {
   ConformOptions,
@@ -203,6 +222,12 @@ export type { MeasuredIndex } from './position.js';
 export { claudeCodeRecords, looksLikeClaudeCodeTranscript } from './claude-code.js';
 export { otelRecords, looksLikeOtel } from './otel.js';
 export type { OtelConversion, OtelRecord } from './otel.js';
+export { litellmRecords, looksLikeLiteLlm } from './litellm.js';
+export type { LiteLlmConversion, LiteLlmRecord } from './litellm.js';
+export { heliconeRecords, looksLikeHelicone } from './helicone.js';
+export type { HeliconeConversion, HeliconeRecord } from './helicone.js';
+export { langsmithRecords, looksLikeLangsmith } from './langsmith.js';
+export type { LangsmithConversion, LangsmithRecord } from './langsmith.js';
 export type { ClaudeCodeConversion, ClaudeCodeRecord } from './claude-code.js';
 export { positionReport } from './position-report.js';
 export type {
@@ -368,6 +393,8 @@ export {
   DEFAULT_MODEL,
   COST_MULTIPLIERS,
   PRICING_LAST_REVIEWED,
+  STALE_PRICING_DAYS,
+  PROVIDER_REVIEWED,
   reviewAgeDays,
   BUNDLED_CATALOGUE,
   getModel,
@@ -377,6 +404,7 @@ export {
   modelFrom,
   cheapestOfTierIn,
   multipliersFor,
+  isOffered,
 } from './pricing.js';
 export type { PricingCatalogue } from './pricing.js';
 export type { CachingMode, Capability, CostMultipliers } from './types.js';
@@ -424,7 +452,8 @@ export type {
 export { RULES, getRule } from './rules.js';
 export { segment, join, protectedTexts } from './segment.js';
 export { computeSavings, costOfCall, formatUsd, formatSignedUsd } from './savings.js';
-export { buildAdvisories, recommendTier } from './advisories.js';
+export { buildAdvisories, recommendTier, recommendTierDetailed } from './advisories.js';
+export type { TierRecommendation } from './advisories.js';
 export type { AdvisoryOptions } from './advisories.js';
 export { analyzeCachePrefix } from './cache.js';
 
@@ -541,7 +570,7 @@ export { NothingToPrune, plannedCalls, pruneExamples, withoutExample } from './p
 export type { ExampleContribution, PruneOptions, PruneReport } from './prune.js';
 
 // Golden-set evaluation (optional, costs three LLM calls per case)
-export { evaluate, fillPrompt, verdictFor } from './evaluate.js';
+export { agreement, evaluate, fillPrompt, pooled, verdictFor } from './evaluate.js';
 export type { EvalCase, EvalReport, EvalVerdict, EvaluateOptions } from './evaluate.js';
 
 // Comparing two prompt versions