@whyparabola/cabar 0.0.0-stage → 0.1.0

package/src/renderings.ts

@@ -0,0 +1,170 @@
+/**
+ * Prism renderings: deterministic multi-fidelity renderings of a retrieved
+ * unit (the CABAR-2 "Prism" extension of CABAR's all-or-nothing selection).
+ *
+ * CABAR-1 spent budget on a unit at a single fidelity (verbatim, or nothing —
+ * with an extractive tail for the rest). Prism turns that binary into a ladder:
+ * a unit can be rendered at VERBATIM (full content), EXTRACT (query-aware
+ * sentence extract, ≤~40% of verbatim tokens), CLAIMS (its pre-resolved claim
+ * bullets), or MENTION (a one-line dereferenceable pointer). The MCKP selector
+ * (PrismSelector) then picks per-unit fidelities to maximize utility under the
+ * token budget instead of dropping units wholesale.
+ *
+ * Every rendering here is PURE and deterministic: no LLM, no I/O, no clock.
+ * Costs are measured through an injected `TokenCounter` (char/4 by default,
+ * real BPE when you inject one) so the same code costs the same
+ * text the caller will bill for. Claims are pre-fetched by the caller and live
+ * on the unit — this module never fetches them.
+ */
+
+import type { PrismUnit } from './prism';
+import type { TokenCounter } from './shared';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** EXTRACT targets at most this fraction of the verbatim token count. */
+export const EXTRACT_TARGET_RATIO = 0.4;
+/** Positional prior: the opening sentence carries topic-setting signal. */
+export const EXTRACT_FIRST_SENTENCE_BONUS = 1;
+/** MENTION lines truncate their sentence to at most this many tokens. */
+export const MENTION_MAX_TOKENS = 25;
+
+// ---------------------------------------------------------------------------
+// Shared lexical helpers (reused by PrismSelector for novelty Jaccard so the
+// rendering token-sets and the MMR token-sets are tokenized identically — and
+// so verbatim novelty matches CabarSelector's exactly for the equivalence test)
+// ---------------------------------------------------------------------------
+
+/** Lowercased word (3+ chars) token set — same family as CabarSelector.tokenize. */
+export function tokenize(text: string): Set<string> {
+  const matches = text.toLowerCase().match(/[a-z0-9]{3,}/g);
+  return new Set(matches ?? []);
+}
+
+export function jaccard(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 || b.size === 0) return 0;
+  let intersection = 0;
+  for (const t of a) {
+    if (b.has(t)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+/** Sentence split — same regex family as CabarSelector.buildTailSummary. */
+export function splitSentences(text: string): string[] {
+  return text
+    .split(/(?<=[.!?])\s+/)
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+/**
+ * Word-boundary truncation to a token budget, appending an ellipsis only when
+ * the text is actually cut. Uses the injected counter so the cut respects the
+ * caller's tokenization (char/4 or BPE). Always keeps at least the first word.
+ */
+export function truncateToTokens(text: string, maxTokens: number, counter: TokenCounter): string {
+  if (counter.count(text) <= maxTokens) return text;
+  const words = text.split(/\s+/).filter(Boolean);
+  let out = '';
+  for (const w of words) {
+    const candidate = out ? `${out} ${w}` : w;
+    if (counter.count(`${candidate}…`) > maxTokens) break;
+    out = candidate;
+  }
+  if (!out) out = words[0] ?? '';
+  return `${out}…`;
+}
+
+// ---------------------------------------------------------------------------
+// Renderings (each deterministic, no LLM)
+// ---------------------------------------------------------------------------
+
+/** VERBATIM — the unit's full content, unchanged. */
+export function verbatim(u: PrismUnit): string {
+  return u.content;
+}
+
+/**
+ * EXTRACT — query-aware sentence extraction targeting ≤~40% of verbatim tokens.
+ *
+ * Sentences are scored by query-term overlap plus a first-sentence positional
+ * prior, deduped on normalized text, then greedily packed (highest score first,
+ * stable on original order) up to the ~40% token target. Selected sentences are
+ * emitted in ORIGINAL narrative order (unlike the score-ordered tail) so the
+ * extract reads coherently. Never returns empty for non-empty content: if the
+ * target is too small for any sentence, the single top-scored sentence is kept
+ * (a single-sentence unit thus extracts to itself, and is dropped later by the
+ * selector's cost-dominance guard rather than duplicating verbatim).
+ */
+export function extract(u: PrismUnit, query: string, counter: TokenCounter): string {
+  const sentences = splitSentences(u.content);
+  if (sentences.length <= 1) return u.content.trim();
+
+  const queryTokens = tokenize(query);
+  const target = Math.max(1, Math.floor(counter.count(u.content) * EXTRACT_TARGET_RATIO));
+
+  interface Scored {
+    text: string;
+    order: number;
+    score: number;
+    cost: number;
+  }
+  const scored: Scored[] = [];
+  const seen = new Set<string>();
+  sentences.forEach((text, order) => {
+    const normalized = text.toLowerCase().replace(/\s+/g, ' ').trim();
+    if (seen.has(normalized)) return; // exact-normalized dedup, keep first occurrence
+    seen.add(normalized);
+    let overlap = 0;
+    for (const t of tokenize(text)) {
+      if (queryTokens.has(t)) overlap++;
+    }
+    scored.push({
+      text,
+      order,
+      score: overlap + (order === 0 ? EXTRACT_FIRST_SENTENCE_BONUS : 0),
+      cost: counter.count(text),
+    });
+  });
+
+  // Greedy pack by score (stable on original order), staying within the target.
+  const byScore = [...scored].sort((a, b) => b.score - a.score || a.order - b.order);
+  const chosen: Scored[] = [];
+  let used = 0;
+  for (const s of byScore) {
+    if (used + s.cost > target) continue; // skip; a smaller lower-scored sentence may still fit
+    chosen.push(s);
+    used += s.cost;
+  }
+  if (chosen.length === 0) chosen.push(byScore[0]!); // never empty for multi-sentence content
+
+  // Emit in original narrative order.
+  chosen.sort((a, b) => a.order - b.order);
+  return chosen.map((s) => s.text).join(' ');
+}
+
+/**
+ * CLAIMS — the unit's pre-resolved claims as '- <claim>' bullet lines.
+ * UNAVAILABLE (null) when the caller pre-fetched no claims for this unit.
+ */
+export function claimsRendering(u: PrismUnit): string | null {
+  if (!u.claims?.length) return null;
+  return u.claims.map((c) => `- ${c}`).join('\n');
+}
+
+/**
+ * MENTION — a one-line dereferenceable pointer:
+ *   [<id>@<yyyy-mm-dd if timestamp>] <first sentence, ≤25 tokens>
+ * This is the answer-lossless floor: even a unit that loses the budget contest
+ * still leaves a citable trace the caller can dereference back to the full unit.
+ */
+export function mention(u: PrismUnit, counter: TokenCounter): string {
+  const first = splitSentences(u.content)[0] ?? u.content.trim();
+  const snippet = truncateToTokens(first, MENTION_MAX_TOKENS, counter);
+  const idPart = u.timestamp ? `${u.id}@${u.timestamp.toISOString().slice(0, 10)}` : u.id;
+  return `[${idPart}] ${snippet}`;
+}

package/src/shared.ts

@@ -0,0 +1,85 @@
+/**
+ * Small shared helpers: the default token estimator and the recency decay curve.
+ */
+
+/**
+ * Token-count heuristic: ~4 characters per token.
+ *
+ * Not a real tokenizer. Deliberately cheap and provider-agnostic (no
+ * dependency on any one model's BPE vocabulary): good enough for budget
+ * heuristics, not for exact billing. Pass your own `TokenCounter` to either
+ * selector when you need exact counts.
+ */
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * A pluggable token counter. Defaults to the char/4 estimator; inject a real
+ * BPE tokenizer (e.g. `gpt-tokenizer`, tiktoken) for exact costs.
+ */
+export interface TokenCounter {
+  name: string;
+  count(text: string): number;
+}
+
+/** Provider-agnostic default counter: the char/4 heuristic. */
+export const CHAR4_COUNTER: TokenCounter = { name: 'char4', count: estimateTokens };
+
+/**
+ * Similarity between two units in [0, 1], used for the novelty term in place
+ * of the built-in word-set Jaccard. 1 = same content, 0 = unrelated.
+ */
+export type Similarity<T> = (a: T, b: T) => number;
+
+/** Cosine similarity of two equal-length vectors, clamped to [0, 1]. */
+export function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number {
+  if (a.length !== b.length) throw new RangeError('cosineSimilarity: vectors must have the same length');
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i]! * b[i]!;
+    na += a[i]! * a[i]!;
+    nb += b[i]! * b[i]!;
+  }
+  if (na === 0 || nb === 0) return 0;
+  return Math.max(0, Math.min(1, dot / Math.sqrt(na * nb)));
+}
+
+/**
+ * Ready-made `similarity` option: cosine over each unit's `embedding`.
+ * Catches paraphrases that word overlap misses. Throws if a unit has no embedding.
+ */
+export function embeddingSimilarity<T extends { id: string; embedding?: ArrayLike<number> }>(a: T, b: T): number {
+  if (!a.embedding || !b.embedding) throw new Error(`embeddingSimilarity: unit ${!a.embedding ? a.id : b.id} has no embedding`);
+  return cosineSimilarity(a.embedding, b.embedding);
+}
+
+/** Default recency scale in days: score = e^(-age / 365). */
+export const DEFAULT_RECENCY_DECAY_DAYS = 365;
+
+/**
+ * Exponential recency decay in [0, 1]: 1 = brand new, → 0 as the unit ages.
+ * score = e^(-ageInDays / decayDays)
+ */
+export function recencyDecay(ageInDays: number, decayDays: number = DEFAULT_RECENCY_DECAY_DAYS): number {
+  if (ageInDays < 0) throw new RangeError('ageInDays must be >= 0');
+  if (decayDays <= 0) throw new RangeError('decayDays must be > 0');
+  return Math.exp(-ageInDays / decayDays);
+}
+
+/**
+ * Strips explicitly-`undefined`-valued keys from an options object before it
+ * gets spread over defaults. `{ ...DEFAULTS, ...opts }` alone is not enough:
+ * a caller that merges option layers can produce a key that IS present with
+ * value `undefined` rather than omitting it, and object spread lets that
+ * `undefined` win over the default, turning `budgetTokens - guaranteedTokens`
+ * into NaN and silently disabling every budget check (NaN comparisons are
+ * always false). Only present keys with a defined value override a default.
+ */
+export function stripUndefined<T extends object>(obj: T): Partial<T> {
+  const out: Partial<T> = {};
+  for (const key of Object.keys(obj) as (keyof T)[]) {
+    if (obj[key] !== undefined) out[key] = obj[key];
+  }
+  return out;
+}