@whyparabola/cabar 0.0.0-stage → 0.1.1

package/dist/cabar-B-R-rHby.d.ts

@@ -0,0 +1,202 @@
+/**
+ * Small shared helpers: the default token estimator and the recency decay curve.
+ */
+/**
+ * Token-count heuristic: ~4 characters per token.
+ *
+ * Not a real tokenizer. Deliberately cheap and provider-agnostic (no
+ * dependency on any one model's BPE vocabulary): good enough for budget
+ * heuristics, not for exact billing. Pass your own `TokenCounter` to either
+ * selector when you need exact counts.
+ */
+declare function estimateTokens(text: string): number;
+/**
+ * A pluggable token counter. Defaults to the char/4 estimator; inject a real
+ * BPE tokenizer (e.g. `gpt-tokenizer`, tiktoken) for exact costs.
+ */
+interface TokenCounter {
+    name: string;
+    count(text: string): number;
+}
+/** Provider-agnostic default counter: the char/4 heuristic. */
+declare const CHAR4_COUNTER: TokenCounter;
+/**
+ * Similarity between two units in [0, 1], used for the novelty term in place
+ * of the built-in word-set Jaccard. 1 = same content, 0 = unrelated.
+ */
+type Similarity<T> = (a: T, b: T) => number;
+/** Cosine similarity of two equal-length vectors, clamped to [0, 1]. */
+declare function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number;
+/**
+ * Ready-made `similarity` option: cosine over each unit's `embedding`.
+ * Catches paraphrases that word overlap misses. Throws if a unit has no embedding.
+ */
+declare function embeddingSimilarity<T extends {
+    id: string;
+    embedding?: ArrayLike<number>;
+}>(a: T, b: T): number;
+/** Default recency scale in days: score = e^(-age / 365). */
+declare const DEFAULT_RECENCY_DECAY_DAYS = 365;
+/**
+ * Exponential recency decay in [0, 1]: 1 = brand new, → 0 as the unit ages.
+ * score = e^(-ageInDays / decayDays)
+ */
+declare function recencyDecay(ageInDays: number, decayDays?: number): number;
+
+/**
+ * CabarSelector: Context-Budget-Aware Retrieval.
+ *
+ * Pure, deterministic selection of which retrieved units are worth spending
+ * prompt-token budget on, replacing "take the top-K by score" with a greedy
+ * utility/cost knapsack:
+ *
+ *   utility(u, S) = relevance(u) × recency(u) × novelty(u, S)
+ *
+ * where S is the set already selected (novelty is recomputed against the
+ * CURRENT S on every iteration, so a second near-duplicate of an already-picked
+ * unit contributes ~0 marginal utility and is skipped in favor of budget spent
+ * elsewhere). Units below `tauFloor` relevance are dropped entirely; units at
+ * or above the floor that don't make the cut are extractively summarized into
+ * a compact `tailSummary` instead of being discarded outright.
+ *
+ * No LLM calls, no I/O, no `Date.now()`: the caller supplies `now` (defaults
+ * to `new Date()` at the call site) so `select()` is a pure, replayable
+ * function of its inputs.
+ */
+
+interface CabarUnit {
+    id: string;
+    content: string;
+    /** Relevance from your retriever, normalized to [0,1] (top item ideally 1.0). */
+    score: number;
+    timestamp?: Date;
+    /** Optional vector for `similarity: embeddingSimilarity`. */
+    embedding?: ArrayLike<number>;
+}
+interface CabarSelectOptions {
+    /** Token budget for the guaranteed block + selected units. The tail summary is capped separately by tailBudgetTokens. */
+    budgetTokens?: number;
+    /** Hard cap on the number of units selected, independent of budget. */
+    maxUnits?: number;
+    /** Hard token cap for the extractive tail summary (on top of budgetTokens). */
+    tailBudgetTokens?: number;
+    /** Minimum relevance score for a unit to survive into the tail at all (below this: dropped). */
+    tauFloor?: number;
+    /**
+     * Blend factor between "ignore recency" (0) and "full recency effect" (1).
+     * ρ = recencyLambda × decay + (1 − recencyLambda). At 0, ρ ≡ 1 for every
+     * unit regardless of timestamp — required for multi-session safety (a
+     * cross-session query shouldn't be penalized just because the recall is old).
+     */
+    recencyLambda?: number;
+    /** Recency scale in days for the exponential decay e^(-age/days). Default 365. */
+    recencyDecayDays?: number;
+    /**
+     * Blend factor between "ignore novelty" (0) and "full novelty effect" (1)
+     * applied to the raw novelty(u,S) ∈ [0,1] term:
+     *   effectiveNovelty = 1 − noveltyWeight × (1 − novelty(u,S))
+     * Defaults to 1, i.e. utility = r × ρ × novelty(u,S) exactly as specified.
+     */
+    noveltyWeight?: number;
+    /** Tokens pre-charged against budgetTokens before selection starts (e.g. the resolved-claims block). */
+    guaranteedTokens?: number;
+    /** Deterministic "current time" for recency scoring. Defaults to `new Date()`. */
+    now?: Date;
+    /**
+     * How hard long units are penalized: greedy density = utility / tokens^costExponent.
+     * 1 (default) = pure value per token, which favors short units. 0.5 softens it so a
+     * long unit that holds the answer can still win; 0 ignores length (pure utility order).
+     */
+    costExponent?: number;
+    /** Also return `decisions`: why each unit was selected, summarized into the tail, or dropped. */
+    explain?: boolean;
+    /** Injectable token cost model. Defaults to the char/4 estimator. */
+    tokenCounter?: TokenCounter;
+    /**
+     * Similarity used for novelty, in [0, 1]. Defaults to word-set Jaccard.
+     * Pass `embeddingSimilarity` (with `embedding` on each unit) to catch paraphrases.
+     */
+    similarity?: Similarity<CabarUnit>;
+    /**
+     * Adaptive stopping mode. When true, the greedy loop's
+     * stop condition is replaced: instead of the fixed `budgetTokens`/`maxUnits`
+     * caps, it keeps accepting units past `minKeep` only while the next unit's
+     * relevance fraction (score / top-candidate score) is >= `relFloor` AND its
+     * novelty vs the already-selected set is >= `novFloor`; `maxCeilingTokens`
+     * becomes the only hard token bound (`maxUnits` is ignored). This lets the
+     * pool's own relevance-score shape decide how much to keep — noisy pools
+     * (one strong hit, many weak ones) stop near `minKeep`; concentrated/spread
+     * pools with a genuine relevant tail keep more. Default false: behavior is
+     * byte-identical to the fixed-budget path.
+     */
+    adaptive?: boolean;
+    /** Adaptive mode: units accepted before the relFloor/novFloor stop rule can trigger. Default 3. */
+    minKeep?: number;
+    /** Adaptive mode: below this fraction of the top candidate's score, the greedy fill stops. Default 0.4. */
+    relFloor?: number;
+    /** Adaptive mode: below this novelty vs the selected set, the greedy fill stops. Default 0.2. */
+    novFloor?: number;
+    /** Adaptive mode: hard token ceiling — replaces budgetTokens as the bound. Default 4000. */
+    maxCeilingTokens?: number;
+    /**
+     * Adaptive mode: also honor `maxUnits` as a hard ceiling ("capped adaptive").
+     * The relFloor/novFloor early-stop can still stop BELOW maxUnits on cliff-y
+     * (noisy) pools — this only prevents adaptive from keeping MORE than maxUnits on
+     * uniformly-relevant (concentrated) pools, where the uncapped rule over-keeps
+     * with no accuracy payoff. Unifies cabar1's ceiling with adaptive's early stop:
+     * concentrated → cap binds (≡ cabar1); noisy → cliff stops early (≡ adaptive).
+     * Default false (preserves the original uncapped adaptive arm byte-identically).
+     */
+    capToMaxUnits?: boolean;
+}
+interface CabarStats {
+    candidateCount: number;
+    candidateTokens: number;
+    selectedTokens: number;
+    tailSummaryTokens: number;
+    budgetTokens: number;
+    /** guaranteedTokens + selectedTokens + tailSummaryTokens — actual spend against budgetTokens. */
+    utilizedTokens: number;
+}
+interface CabarDecision {
+    id: string;
+    status: 'selected' | 'tail' | 'dropped';
+    /** Plain-English reason, e.g. "redundant with u3 (similarity 0.82)". */
+    reason: string;
+    tokens: number;
+    relevance: number;
+    recency: number;
+    /** 1 − highest similarity to any (other) selected unit. */
+    novelty: number;
+    /** For redundant units: the selected unit it overlaps most. */
+    redundantWith?: string;
+}
+interface CabarSelectResult {
+    selected: CabarUnit[];
+    /** Unselected units at or above tauFloor — summarized into tailSummary, not dropped. */
+    tail: CabarUnit[];
+    tailSummary: string;
+    stats: CabarStats;
+    /** Present when `explain: true`, one entry per input unit in input order. */
+    decisions?: CabarDecision[];
+}
+/** Fixed-mode selector defaults. Exported so `cabar.config.ts` can source the
+ * budget/maxUnits ceilings from one place instead of re-declaring the literals. */
+declare const DEFAULTS: Required<Omit<CabarSelectOptions, 'now' | 'tokenCounter' | 'similarity' | 'explain' | 'adaptive' | 'minKeep' | 'relFloor' | 'novFloor' | 'maxCeilingTokens' | 'capToMaxUnits'>>;
+/** Adaptive-mode-only defaults (fixed global constants, never tuned per query type). */
+declare const ADAPTIVE_DEFAULTS: {
+    minKeep: number;
+    relFloor: number;
+    novFloor: number;
+    maxCeilingTokens: number;
+};
+declare class CabarSelector {
+    /**
+     * Select which units fit the budget, greedily maximizing utility/cost
+     * density, with a best-single-vs-greedy swap (budgeted-max-coverage 2-approx
+     * guarantee) and an extractive tail summary for the rest.
+     */
+    static select(units: CabarUnit[], query: string, opts?: CabarSelectOptions): CabarSelectResult;
+}
+
+export { ADAPTIVE_DEFAULTS as A, type CabarSelectResult as C, DEFAULTS as D, type Similarity as S, type TokenCounter as T, type CabarUnit as a, type CabarSelectOptions as b, CHAR4_COUNTER as c, type CabarDecision as d, CabarSelector as e, type CabarStats as f, DEFAULT_RECENCY_DECAY_DAYS as g, cosineSimilarity as h, embeddingSimilarity as i, estimateTokens as j, recencyDecay as r };

package/dist/chunk-UE4FTY7E.js

@@ -0,0 +1,388 @@
+// src/shared.ts
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+var CHAR4_COUNTER = { name: "char4", count: estimateTokens };
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) throw new RangeError("cosineSimilarity: vectors must have the same length");
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 0;
+  return Math.max(0, Math.min(1, dot / Math.sqrt(na * nb)));
+}
+function embeddingSimilarity(a, b) {
+  if (!a.embedding || !b.embedding) throw new Error(`embeddingSimilarity: unit ${!a.embedding ? a.id : b.id} has no embedding`);
+  return cosineSimilarity(a.embedding, b.embedding);
+}
+var DEFAULT_RECENCY_DECAY_DAYS = 365;
+function recencyDecay(ageInDays, decayDays = DEFAULT_RECENCY_DECAY_DAYS) {
+  if (ageInDays < 0) throw new RangeError("ageInDays must be >= 0");
+  if (decayDays <= 0) throw new RangeError("decayDays must be > 0");
+  return Math.exp(-ageInDays / decayDays);
+}
+function stripUndefined(obj) {
+  const out = {};
+  for (const key of Object.keys(obj)) {
+    if (obj[key] !== void 0) out[key] = obj[key];
+  }
+  return out;
+}
+
+// src/renderings.ts
+var EXTRACT_TARGET_RATIO = 0.4;
+var EXTRACT_FIRST_SENTENCE_BONUS = 1;
+var MENTION_MAX_TOKENS = 25;
+function tokenize(text) {
+  const matches = text.toLowerCase().match(/[a-z0-9]{3,}/g);
+  return new Set(matches ?? []);
+}
+function jaccard(a, b) {
+  if (a.size === 0 || b.size === 0) return 0;
+  let intersection = 0;
+  for (const t of a) {
+    if (b.has(t)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+function splitSentences(text) {
+  return text.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter(Boolean);
+}
+function truncateToTokens(text, maxTokens, counter) {
+  if (counter.count(text) <= maxTokens) return text;
+  const words = text.split(/\s+/).filter(Boolean);
+  let out = "";
+  for (const w of words) {
+    const candidate = out ? `${out} ${w}` : w;
+    if (counter.count(`${candidate}\u2026`) > maxTokens) break;
+    out = candidate;
+  }
+  if (!out) out = words[0] ?? "";
+  return `${out}\u2026`;
+}
+function verbatim(u) {
+  return u.content;
+}
+function extract(u, query, counter) {
+  const sentences = splitSentences(u.content);
+  if (sentences.length <= 1) return u.content.trim();
+  const queryTokens = tokenize(query);
+  const target = Math.max(1, Math.floor(counter.count(u.content) * EXTRACT_TARGET_RATIO));
+  const scored = [];
+  const seen = /* @__PURE__ */ new Set();
+  sentences.forEach((text, order) => {
+    const normalized = text.toLowerCase().replace(/\s+/g, " ").trim();
+    if (seen.has(normalized)) return;
+    seen.add(normalized);
+    let overlap = 0;
+    for (const t of tokenize(text)) {
+      if (queryTokens.has(t)) overlap++;
+    }
+    scored.push({
+      text,
+      order,
+      score: overlap + (order === 0 ? EXTRACT_FIRST_SENTENCE_BONUS : 0),
+      cost: counter.count(text)
+    });
+  });
+  const byScore = [...scored].sort((a, b) => b.score - a.score || a.order - b.order);
+  const chosen = [];
+  let used = 0;
+  for (const s of byScore) {
+    if (used + s.cost > target) continue;
+    chosen.push(s);
+    used += s.cost;
+  }
+  if (chosen.length === 0) chosen.push(byScore[0]);
+  chosen.sort((a, b) => a.order - b.order);
+  return chosen.map((s) => s.text).join(" ");
+}
+function claimsRendering(u) {
+  if (!u.claims?.length) return null;
+  return u.claims.map((c) => `- ${c}`).join("\n");
+}
+function mention(u, counter) {
+  const first = splitSentences(u.content)[0] ?? u.content.trim();
+  const snippet = truncateToTokens(first, MENTION_MAX_TOKENS, counter);
+  const idPart = u.timestamp ? `${u.id}@${u.timestamp.toISOString().slice(0, 10)}` : u.id;
+  return `[${idPart}] ${snippet}`;
+}
+
+// src/cabar.ts
+var DEFAULTS = {
+  budgetTokens: 4e3,
+  maxUnits: 8,
+  tailBudgetTokens: 600,
+  tauFloor: 0.15,
+  recencyLambda: 0.3,
+  recencyDecayDays: DEFAULT_RECENCY_DECAY_DAYS,
+  noveltyWeight: 1,
+  guaranteedTokens: 0,
+  costExponent: 1
+};
+var ADAPTIVE_DEFAULTS = {
+  minKeep: 3,
+  relFloor: 0.4,
+  novFloor: 0.2,
+  maxCeilingTokens: 4e3
+};
+var MS_PER_DAY = 864e5;
+function tokenize2(text) {
+  const matches = text.toLowerCase().match(/[a-z0-9]{3,}/g);
+  return new Set(matches ?? []);
+}
+function jaccard2(a, b) {
+  if (a.size === 0 || b.size === 0) return 0;
+  let intersection = 0;
+  for (const t of a) {
+    if (b.has(t)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+var CabarSelector = class {
+  /**
+   * Select which units fit the budget, greedily maximizing utility/cost
+   * density, with a best-single-vs-greedy swap (budgeted-max-coverage 2-approx
+   * guarantee) and an extractive tail summary for the rest.
+   */
+  static select(units, query, opts = {}) {
+    const cfg = { ...DEFAULTS, ...stripUndefined(opts) };
+    const now = opts.now ?? /* @__PURE__ */ new Date();
+    const counter = opts.tokenCounter ?? CHAR4_COUNTER;
+    const count = (t) => counter.count(t);
+    const adaptive = opts.adaptive === true;
+    const minKeep = opts.minKeep ?? ADAPTIVE_DEFAULTS.minKeep;
+    const relFloor = opts.relFloor ?? ADAPTIVE_DEFAULTS.relFloor;
+    const novFloor = opts.novFloor ?? ADAPTIVE_DEFAULTS.novFloor;
+    const maxCeilingTokens = opts.maxCeilingTokens ?? ADAPTIVE_DEFAULTS.maxCeilingTokens;
+    const capToMaxUnits = opts.capToMaxUnits === true;
+    const topScore = units.length > 0 ? Math.max(...units.map((u) => u.score)) : 1;
+    const costOf = /* @__PURE__ */ new Map();
+    const tokensOf = /* @__PURE__ */ new Map();
+    let candidateTokens = 0;
+    for (const u of units) {
+      const cost = count(u.content);
+      costOf.set(u.id, cost);
+      tokensOf.set(u.id, tokenize2(u.content));
+      candidateTokens += cost;
+    }
+    const effectiveBudget = adaptive ? maxCeilingTokens : cfg.budgetTokens;
+    const remainingBudget = Math.max(0, effectiveBudget - cfg.guaranteedTokens);
+    const recencyOf = (u) => {
+      if (!u.timestamp || cfg.recencyLambda === 0) return 1;
+      const ageInDays = Math.max(0, (now.getTime() - u.timestamp.getTime()) / MS_PER_DAY);
+      const decay = recencyDecay(ageInDays, cfg.recencyDecayDays);
+      return cfg.recencyLambda * decay + (1 - cfg.recencyLambda);
+    };
+    const noveltyOf = (u, selectedSoFar) => {
+      if (selectedSoFar.length === 0) return 1;
+      const a = tokensOf.get(u.id);
+      let maxSim = 0;
+      for (const s of selectedSoFar) {
+        const sim2 = opts.similarity ? opts.similarity(u, s) : jaccard2(a, tokensOf.get(s.id));
+        if (sim2 > maxSim) maxSim = sim2;
+      }
+      return 1 - maxSim;
+    };
+    const utilityOf = (u, selectedSoFar) => {
+      const r = u.score;
+      const rho = recencyOf(u);
+      const noveltyRaw = noveltyOf(u, selectedSoFar);
+      const effectiveNovelty = 1 - cfg.noveltyWeight * (1 - noveltyRaw);
+      return r * rho * effectiveNovelty;
+    };
+    const selected = [];
+    const selectedIds = /* @__PURE__ */ new Set();
+    let selectedTokens = 0;
+    let greedyTotalUtility = 0;
+    while (adaptive && !capToMaxUnits || selected.length < cfg.maxUnits) {
+      let best = null;
+      for (const u of units) {
+        if (selectedIds.has(u.id)) continue;
+        const cost = costOf.get(u.id);
+        if (selectedTokens + cost > remainingBudget) continue;
+        const utility = utilityOf(u, selected);
+        if (utility <= 0) continue;
+        const density = cost > 0 ? utility / Math.pow(cost, cfg.costExponent) : Infinity;
+        if (!best || density > best.density) {
+          best = { unit: u, utility, density };
+        }
+      }
+      if (!best) break;
+      if (adaptive && selected.length >= minKeep) {
+        const relFrac = best.unit.score / (topScore || 1);
+        const nov = noveltyOf(best.unit, selected);
+        if (relFrac < relFloor || nov < novFloor) break;
+      }
+      selected.push(best.unit);
+      selectedIds.add(best.unit.id);
+      selectedTokens += costOf.get(best.unit.id);
+      greedyTotalUtility += best.utility;
+    }
+    let swapped = false;
+    if (adaptive || cfg.maxUnits >= 1) {
+      let bestSingle = null;
+      for (const u of units) {
+        const cost = costOf.get(u.id);
+        if (cost > remainingBudget) continue;
+        const utility = utilityOf(u, []);
+        if (!bestSingle || utility > bestSingle.utility) {
+          bestSingle = { unit: u, utility };
+        }
+      }
+      if (bestSingle && bestSingle.utility > greedyTotalUtility) {
+        swapped = true;
+        selected.length = 0;
+        selected.push(bestSingle.unit);
+        selectedIds.clear();
+        selectedIds.add(bestSingle.unit.id);
+        selectedTokens = costOf.get(bestSingle.unit.id);
+      }
+    }
+    const tail = units.filter((u) => !selectedIds.has(u.id) && u.score >= cfg.tauFloor);
+    const extractive = buildTailSummary(tail, query, cfg.tailBudgetTokens, count);
+    const verbatim2 = tail.map((u) => u.content.trim()).join("\n");
+    let tailSummary = tail.length === 0 ? "" : count(verbatim2) < count(extractive) ? verbatim2 : extractive;
+    if (count(tailSummary) > cfg.tailBudgetTokens) {
+      tailSummary = counter === CHAR4_COUNTER ? truncateToTokens2(tailSummary, cfg.tailBudgetTokens) : truncateToTokens(tailSummary, cfg.tailBudgetTokens, counter);
+    }
+    const tailSummaryTokens = count(tailSummary);
+    const result = {
+      selected,
+      tail,
+      tailSummary,
+      stats: {
+        candidateCount: units.length,
+        candidateTokens,
+        selectedTokens,
+        tailSummaryTokens,
+        budgetTokens: effectiveBudget,
+        utilizedTokens: cfg.guaranteedTokens + selectedTokens + tailSummaryTokens
+      }
+    };
+    if (!opts.explain) return result;
+    const sim = (a, b) => opts.similarity ? opts.similarity(a, b) : jaccard2(tokensOf.get(a.id), tokensOf.get(b.id));
+    const unitCap = !adaptive || capToMaxUnits;
+    result.decisions = units.map((u) => {
+      const tokens = costOf.get(u.id);
+      const others = selected.filter((s) => s.id !== u.id);
+      let maxSim = 0;
+      let closest;
+      for (const s of others) {
+        const v = sim(u, s);
+        if (v > maxSim) [maxSim, closest] = [v, s.id];
+      }
+      const base = { id: u.id, tokens, relevance: u.score, recency: recencyOf(u), novelty: 1 - maxSim };
+      if (selectedIds.has(u.id)) {
+        return { ...base, status: "selected", reason: swapped ? "best single unit: worth more alone than the greedy set" : "selected" };
+      }
+      if (u.score < cfg.tauFloor) {
+        return { ...base, status: "dropped", reason: `relevance ${u.score.toFixed(2)} is below tauFloor ${cfg.tauFloor}` };
+      }
+      let reason;
+      if (swapped) reason = "a single unit beat the greedy set";
+      else if (maxSim >= 1 - (adaptive ? novFloor : 0.2) && closest)
+        reason = `redundant with ${closest} (similarity ${maxSim.toFixed(2)})`;
+      else if (selectedTokens + tokens > remainingBudget) reason = `doesn't fit: needs ${tokens} tokens, ${Math.max(0, remainingBudget - selectedTokens)} left`;
+      else if (unitCap && selected.length >= cfg.maxUnits) reason = `maxUnits (${cfg.maxUnits}) reached`;
+      else if (adaptive && u.score / (topScore || 1) < relFloor) reason = `below the adaptive relevance cliff (${(u.score / (topScore || 1)).toFixed(2)} of the top score)`;
+      else reason = "adaptive stop: lower value per token than what was kept";
+      return { ...base, status: "tail", reason, ...reason.startsWith("redundant") ? { redundantWith: closest } : {} };
+    });
+    return result;
+  }
+};
+function truncateToTokens2(text, budgetTokens) {
+  const maxChars = budgetTokens * 4;
+  if (text.length <= maxChars) return text;
+  const cut = text.slice(0, maxChars);
+  const lastSpace = cut.lastIndexOf(" ");
+  return (lastSpace > maxChars * 0.8 ? cut.slice(0, lastSpace) : cut).trimEnd() + "\u2026";
+}
+function buildTailSummary(tail, query, tailBudgetTokens, count) {
+  if (tail.length === 0 || tailBudgetTokens <= 0) return "";
+  const queryTokens = tokenize2(query);
+  const sentences = [];
+  let order = 0;
+  for (const unit of tail) {
+    const parts = unit.content.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter(Boolean);
+    for (const text of parts) {
+      const tokens = tokenize2(text);
+      let overlap = 0;
+      for (const t of tokens) {
+        if (queryTokens.has(t)) overlap++;
+      }
+      sentences.push({
+        text,
+        score: overlap + unit.score,
+        unitId: unit.id,
+        timestamp: unit.timestamp,
+        order: order++
+      });
+    }
+  }
+  sentences.sort((a, b) => b.score - a.score || a.order - b.order);
+  const seen = /* @__PURE__ */ new Set();
+  const lines = [];
+  let usedTokens = 0;
+  for (const s of sentences) {
+    const normalized = s.text.toLowerCase().replace(/\s+/g, " ").trim();
+    if (seen.has(normalized)) continue;
+    const idPart = s.timestamp ? `${s.unitId}@${s.timestamp.toISOString().slice(0, 10)}` : s.unitId;
+    const line = `[${idPart}] ${s.text}`;
+    const lineTokens = count(lines.length === 0 ? line : `
+${line}`);
+    if (usedTokens + lineTokens > tailBudgetTokens) break;
+    seen.add(normalized);
+    lines.push(line);
+    usedTokens += lineTokens;
+  }
+  return lines.join("\n");
+}
+
+// src/normalize.ts
+function normalizeScores(units, opts = {}) {
+  if (units.length === 0) return [];
+  const method = opts.method ?? "minmax";
+  const raw = units.map((u) => opts.higherIsBetter === false ? -u.score : u.score);
+  for (const s of raw) {
+    if (!Number.isFinite(s)) throw new RangeError("normalizeScores: every score must be a finite number");
+  }
+  const hi = Math.max(...raw);
+  const lo = Math.min(...raw);
+  let map;
+  if (method === "max") {
+    if (lo < 0) throw new RangeError("normalizeScores: method 'max' needs non-negative scores, use 'minmax'");
+    map = (s) => hi === 0 ? 1 : s / hi;
+  } else {
+    map = (s) => hi === lo ? 1 : (s - lo) / (hi - lo);
+  }
+  return units.map((u, i) => ({ ...u, score: map(raw[i]) }));
+}
+
+export {
+  estimateTokens,
+  CHAR4_COUNTER,
+  cosineSimilarity,
+  embeddingSimilarity,
+  DEFAULT_RECENCY_DECAY_DAYS,
+  recencyDecay,
+  stripUndefined,
+  tokenize,
+  jaccard,
+  splitSentences,
+  truncateToTokens,
+  verbatim,
+  extract,
+  claimsRendering,
+  mention,
+  DEFAULTS,
+  ADAPTIVE_DEFAULTS,
+  CabarSelector,
+  normalizeScores
+};