@remnic/core 9.3.688 → 9.3.689

package/dist/recall-pipeline-stages.js

@@ -0,0 +1,61 @@
+import "./chunk-PZ5AY32C.js";
+
+// src/recall-pipeline-stages.ts
+var UNDEFINED_TURN_INDEX_DESC_SENTINEL = -1;
+var UNDEFINED_TURN_INDEX_ASC_SENTINEL = Number.MAX_SAFE_INTEGER;
+function unifiedDedupeAndRank(items, config) {
+  const transformContent = config.transformContent ?? ((content) => content);
+  const direction = config.turnIndexSortDirection ?? "desc";
+  const dedupByContent = config.dedupByContent !== false;
+  const seenIds = /* @__PURE__ */ new Set();
+  const seenContent = /* @__PURE__ */ new Set();
+  const survivors = [];
+  for (const item of items) {
+    const id = resolveItemId(item);
+    if (id && seenIds.has(id)) continue;
+    const transformedContent = transformContent(item.content, config.intents);
+    if (dedupByContent) {
+      const contentKey = transformedContent.toLowerCase().replace(/\s+/g, " ").trim();
+      if (seenContent.has(contentKey)) continue;
+      seenContent.add(contentKey);
+    }
+    if (id) seenIds.add(id);
+    survivors.push({ original: item, transformedContent });
+  }
+  const scored = survivors.map(({ original, transformedContent }) => ({
+    ...original,
+    content: transformedContent,
+    rank: config.scoreEvidence(original, config.query, config.intents)
+  }));
+  const filtered = typeof config.rankThreshold === "number" ? scored.filter((item) => item.rank >= config.rankThreshold) : scored;
+  return filtered.sort(makeComparator(direction));
+}
+function makeComparator(direction) {
+  if (direction === "asc") {
+    return (left, right) => {
+      if (right.rank !== left.rank) return right.rank - left.rank;
+      const leftTurn = typeof left.turnIndex === "number" ? left.turnIndex : UNDEFINED_TURN_INDEX_ASC_SENTINEL;
+      const rightTurn = typeof right.turnIndex === "number" ? right.turnIndex : UNDEFINED_TURN_INDEX_ASC_SENTINEL;
+      if (leftTurn !== rightTurn) return leftTurn - rightTurn;
+      return left.content.localeCompare(right.content);
+    };
+  }
+  return (left, right) => {
+    if (right.rank !== left.rank) return right.rank - left.rank;
+    const leftTurn = typeof left.turnIndex === "number" ? left.turnIndex : UNDEFINED_TURN_INDEX_DESC_SENTINEL;
+    const rightTurn = typeof right.turnIndex === "number" ? right.turnIndex : UNDEFINED_TURN_INDEX_DESC_SENTINEL;
+    if (rightTurn !== leftTurn) return rightTurn - leftTurn;
+    return (right.score ?? 0) - (left.score ?? 0);
+  };
+}
+function resolveItemId(item) {
+  if (item.id) return item.id;
+  if (item.sessionId && typeof item.turnIndex === "number") {
+    return `${item.sessionId}:${item.turnIndex}`;
+  }
+  return void 0;
+}
+export {
+  unifiedDedupeAndRank
+};
+//# sourceMappingURL=recall-pipeline-stages.js.map

package/dist/recall-pipeline-stages.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/recall-pipeline-stages.ts"],"sourcesContent":["/**\n * Issue #1539 PR2 — the staged recall pipeline spine.\n *\n * The four recall pipelines (`targeted-fact-recall.ts`, `response-guidance-\n * recall.ts`, `explicit-cue-recall.ts`, `event-order-recall.ts`) implement\n * the same stage sequence — intent → candidate collection → dedup → rank →\n * filter → slice → metadata insertion → token budgeting — with ~70%\n * near-duplicate code. The most dangerous divergence is the sort\n * comparator: guidance/targeted-fact sort `turnIndex` DESC (recency) while\n * event-order sorts `turnIndex` ASC (chronological). The comparators are\n * byte-identical except for this direction — copy-pasting into a new\n * chronological pipeline silently inverts the ordering.\n *\n * This module centralizes the **dedup + score + threshold-filter + sort**\n * stages behind one declared config object. Per-tier divergence (threshold\n * value, sort direction, content transformation) becomes a declared config\n * field instead of embedded code.\n *\n * **PR 2 scope:** extract the module with NO pipeline changes. PRs 3–6\n * migrate each pipeline to call `unifiedDedupeAndRank` one at a time,\n * re-running the characterization snapshots\n * (`tests/recall-pipeline-unified.test.ts`) to verify byte-for-byte parity.\n *\n * Future PRs will add the remaining stages from the issue's Solution\n * (`mergeAcrossLcmKeys`, `insertMetadata`) once the per-tier hooks are\n * factored out of each pipeline.\n *\n * @see https://github.com/joshuaswarren/remnic/issues/1539\n */\n\nimport type { EvidencePackItem } from \"./evidence-pack.js\";\n\n/** A ranked evidence item: an `EvidencePackItem` with a computed `rank`. */\nexport interface RankedEvidenceItem extends EvidencePackItem {\n  rank: number;\n}\n\n/**\n * Sort direction for the turn-index secondary key. This is the divergence\n * issue #1539 identifies: relevance-ranked pipelines (targeted-fact,\n * response-guidance) sort `turnIndex` DESC (recency — latest first); the\n * chronological pipeline (event-order) sorts `turnIndex` ASC (earliest\n * first). The rank primary key is ALWAYS DESC (higher rank first); only\n * the `turnIndex` tiebreaker flips direction.\n */\nexport type TurnIndexSortDirection = \"desc\" | \"asc\";\n\n/**\n * Configuration for the unified dedup + score + threshold-filter + sort\n * pass. Each pipeline declares its divergences as fields here instead of\n * embedding them in pipeline-specific code.\n *\n * The issue's Solution defines the intended full config object\n * (`RecallPipelineConfig`); this interface is extracted incrementally:\n *   - PR 2 (this PR): dedup/score/threshold/sort only\n *   - PRs 3–6: migrate each pipeline to consume the spine\n *   - future: add `mergeAcrossLcmKeys`, `insertMetadata` once the per-tier\n *     hooks are factored out of each pipeline\n */\nexport interface UnifiedRankConfig<TIntent> {\n  /** The user query — passed to the scorer. */\n  query: string;\n  /**\n   * Per-tier intent classification result (already computed by the caller;\n   * intent classification stays per-tier because it is genuinely\n   * per-tier per issue #1539 Pitfall 1).\n   */\n  intents: TIntent[];\n  /**\n   * Score an item using its ORIGINAL (pre-transform) content. Higher = more\n   * relevant. The return value becomes the item's `rank`. For pipelines\n   * that don't score (explicit-cue), pass a constant scorer — the config\n   * makes the no-scoring policy explicit.\n   */\n  scoreEvidence: (\n    item: EvidencePackItem,\n    query: string,\n    intents: TIntent[],\n  ) => number;\n  /**\n   * Optional content transformation applied to each surviving item's\n   * OUTPUT content (NOT to the content the scorer sees). Per-tier\n   * cue-appenders go here:\n   *   - targeted-fact: `appendNormalizedNumericCues`\n   *   - response-guidance: `appendGuidanceCues`\n   *   - event-order: `appendChronologicalCues`\n   *\n   * The dedup key uses the TRANSFORMED content. This is equivalent to\n   * deduping on original content for all existing pipelines because every\n   * transform is a deterministic append (same original → same transformed →\n   * same key; different originals → different transformed → different key).\n   */\n  transformContent?: (content: string, intents: TIntent[]) => string;\n  /**\n   * Whether to deduplicate by normalized (transformed) content in addition to\n   * id. Default `true` — matches targeted-fact, response-guidance, and\n   * explicit-cue, which all collapse later items sharing a normalized content\n   * key. Event-order sets this to `false`: its rank pass\n   * (`rankAndSelectEventOrderItems`) deduplicates by turn id only and keeps\n   * distinct turns even when two turns share the same cue-appended body\n   * (legitimate repeated turns in a chronological transcript). Making this a\n   * declared field prevents PR 6's migration from silently dropping valid\n   * turns (cursor bugbot a4299851).\n   */\n  dedupByContent?: boolean;\n  /**\n   * Items with `rank` below this threshold are dropped. `undefined` = no\n   * filter. Event-order declares `rankThreshold: 6` here instead of\n   * inlining an undocumented `.filter((item) => item.rank >= 6)` (the\n   * \"hardcoded rank threshold that exists in no config and no other\n   * pipeline\" from issue #1539's audit).\n   */\n  rankThreshold?: number;\n  /**\n   * Sort direction for the `turnIndex` secondary key.\n   *\n   * - `\"desc\"` (default): relevance-ranked pipelines (targeted-fact,\n   *   response-guidance) sort `turnIndex` DESC. Undefined `turnIndex`\n   *   falls to `-1` (bottom of a DESC list). Tertiary tiebreaker:\n   *   `score DESC`.\n   * - `\"asc\"`: the chronological pipeline (event-order) sorts `turnIndex`\n   *   ASC. Undefined `turnIndex` falls to `Number.MAX_SAFE_INTEGER`\n   *   (bottom of an ASC list). Tertiary tiebreaker: `content.localeCompare`.\n   *\n   * The rank primary key is ALWAYS DESC regardless of this setting.\n   */\n  turnIndexSortDirection?: TurnIndexSortDirection;\n}\n\n/**\n * Sentinel for undefined `turnIndex` when sorting DESC. `-1` sorts below\n * every real turn index (which are `>= 0`), so missing `turn_index` always\n * lands at the bottom of a DESC-ordered list — never wins ordering over a\n * real turn.\n */\nconst UNDEFINED_TURN_INDEX_DESC_SENTINEL = -1;\n\n/**\n * Sentinel for undefined `turnIndex` when sorting ASC.\n * `Number.MAX_SAFE_INTEGER` sorts above every real turn index, so missing\n * `turn_index` always lands at the bottom of an ASC-ordered list.\n */\nconst UNDEFINED_TURN_INDEX_ASC_SENTINEL = Number.MAX_SAFE_INTEGER;\n\n/**\n * Deduplicate, score, threshold-filter, and sort evidence items under one\n * unified policy.\n *\n * **Stage order** (matches every existing pipeline's rank/dedupe function):\n *   1. **dedup** by `id` + normalized content (first-seen wins). The dedup\n *      key uses transformed content if `transformContent` is declared.\n *   2. **score** each surviving item on its ORIGINAL content (the transform\n *      does not influence the score).\n *   3. **threshold-filter**: drop items with `rank < rankThreshold` (if\n *      declared).\n *   4. **sort**: `rank DESC` → `turnIndex` (direction-configurable) →\n *      tertiary tiebreaker (`score DESC` for relevance pipelines,\n *      `content.localeCompare` for chronological pipelines).\n *\n * This function does NOT slice (`maxResults`), budget, or format — those\n * stages remain per-tier because they interact with per-tier metadata\n * insertion. The issue's Solution describes a future `insertMetadata` hook\n * that will make budget-adjustment uniform; that lands in the per-tier\n * migration PRs.\n *\n * @example\n * // Relevance-ranked pipeline (targeted-fact shape):\n * unifiedDedupeAndRank(items, {\n *   query,\n *   intents: [],\n *   scoreEvidence: (item, q) => scoreTargetedFact(item, q),\n *   transformContent: (content) => appendNormalizedNumericCues(content),\n *   // turnIndexSortDirection defaults to \"desc\"\n * });\n *\n * @example\n * // Chronological pipeline (event-order shape):\n * unifiedDedupeAndRank(items, {\n *   query,\n *   intents: [],\n *   scoreEvidence: (item, q) => scoreEventOrder(item, q),\n *   transformContent: (content) => appendChronologicalCues(content, query),\n *   rankThreshold: 6,           // declared, not inlined\n *   turnIndexSortDirection: \"asc\",\n * });\n */\nexport function unifiedDedupeAndRank<TIntent>(\n  items: readonly EvidencePackItem[],\n  config: UnifiedRankConfig<TIntent>,\n): RankedEvidenceItem[] {\n  const transformContent = config.transformContent ?? ((content: string) => content);\n  const direction: TurnIndexSortDirection = config.turnIndexSortDirection ?? \"desc\";\n  const dedupByContent = config.dedupByContent !== false;\n\n  // Stage 1: dedup by id (+ normalized transformed content when enabled).\n  // First-seen wins, matching every existing pipeline. Event-order opts out\n  // of content dedup (dedupByContent: false) because it keeps distinct turns\n  // even when two turns share the same cue-appended body.\n  const seenIds = new Set<string>();\n  const seenContent = new Set<string>();\n  const survivors: Array<{ original: EvidencePackItem; transformedContent: string }> = [];\n\n  for (const item of items) {\n    const id = resolveItemId(item);\n    if (id && seenIds.has(id)) continue;\n\n    const transformedContent = transformContent(item.content, config.intents);\n    if (dedupByContent) {\n      const contentKey = transformedContent.toLowerCase().replace(/\\s+/g, \" \").trim();\n      if (seenContent.has(contentKey)) continue;\n      seenContent.add(contentKey);\n    }\n    if (id) seenIds.add(id);\n    survivors.push({ original: item, transformedContent });\n  }\n\n  // Stage 2: score on ORIGINAL content (transforms don't influence the score).\n  const scored: RankedEvidenceItem[] = survivors.map(({ original, transformedContent }) => ({\n    ...original,\n    content: transformedContent,\n    rank: config.scoreEvidence(original, config.query, config.intents),\n  }));\n\n  // Stage 3: threshold-filter (declared, not inlined).\n  const filtered =\n    typeof config.rankThreshold === \"number\"\n      ? scored.filter((item) => item.rank >= (config.rankThreshold as number))\n      : scored;\n\n  // Stage 4: sort.\n  // rank is ALWAYS DESC (higher rank first).\n  // turnIndex direction is configurable: DESC (relevance) or ASC (chronology).\n  // The tertiary tiebreaker follows the direction:\n  //   DESC → score DESC; ASC → content.localeCompare.\n  return filtered.sort(makeComparator(direction));\n}\n\n/**\n * Build the sort comparator for the configured direction. Extracted so the\n * comparator's two shapes (DESC / ASC) can be tested independently and so\n * the direction divergence is visible in ONE place.\n */\nfunction makeComparator(\n  direction: TurnIndexSortDirection,\n): (left: RankedEvidenceItem, right: RankedEvidenceItem) => number {\n  if (direction === \"asc\") {\n    // Chronological (event-order): rank DESC → turnIndex ASC → content localeCompare.\n    // Matches event-order-recall.ts:159-164 (rankedByScore sort) byte-for-byte.\n    return (left, right) => {\n      if (right.rank !== left.rank) return right.rank - left.rank;\n      const leftTurn =\n        typeof left.turnIndex === \"number\"\n          ? left.turnIndex\n          : UNDEFINED_TURN_INDEX_ASC_SENTINEL;\n      const rightTurn =\n        typeof right.turnIndex === \"number\"\n          ? right.turnIndex\n          : UNDEFINED_TURN_INDEX_ASC_SENTINEL;\n      if (leftTurn !== rightTurn) return leftTurn - rightTurn;\n      return left.content.localeCompare(right.content);\n    };\n  }\n  // Relevance-ranked (targeted-fact, response-guidance):\n  // rank DESC → turnIndex DESC → score DESC.\n  // Matches targeted-fact-recall.ts:239-245 and response-guidance-recall.ts:374-380\n  // byte-for-byte.\n  return (left, right) => {\n    if (right.rank !== left.rank) return right.rank - left.rank;\n    const leftTurn =\n      typeof left.turnIndex === \"number\"\n        ? left.turnIndex\n        : UNDEFINED_TURN_INDEX_DESC_SENTINEL;\n    const rightTurn =\n      typeof right.turnIndex === \"number\"\n        ? right.turnIndex\n        : UNDEFINED_TURN_INDEX_DESC_SENTINEL;\n    if (rightTurn !== leftTurn) return rightTurn - leftTurn;\n    return (right.score ?? 0) - (left.score ?? 0);\n  };\n}\n\n\n/**\n * Resolve an evidence item's id. Falls back to `sessionId:turnIndex` when\n * `id` is absent — mirrors every existing pipeline's fallback and\n * `evidence-pack.ts`'s `evidenceItemFallbackId`.\n */\nfunction resolveItemId(item: EvidencePackItem): string | undefined {\n  if (item.id) return item.id;\n  if (item.sessionId && typeof item.turnIndex === \"number\") {\n    return `${item.sessionId}:${item.turnIndex}`;\n  }\n  return undefined;\n}\n"],"mappings":";;;AAuIA,IAAM,qCAAqC;AAO3C,IAAM,oCAAoC,OAAO;AA4C1C,SAAS,qBACd,OACA,QACsB;AACtB,QAAM,mBAAmB,OAAO,qBAAqB,CAAC,YAAoB;AAC1E,QAAM,YAAoC,OAAO,0BAA0B;AAC3E,QAAM,iBAAiB,OAAO,mBAAmB;AAMjD,QAAM,UAAU,oBAAI,IAAY;AAChC,QAAM,cAAc,oBAAI,IAAY;AACpC,QAAM,YAA+E,CAAC;AAEtF,aAAW,QAAQ,OAAO;AACxB,UAAM,KAAK,cAAc,IAAI;AAC7B,QAAI,MAAM,QAAQ,IAAI,EAAE,EAAG;AAE3B,UAAM,qBAAqB,iBAAiB,KAAK,SAAS,OAAO,OAAO;AACxE,QAAI,gBAAgB;AAClB,YAAM,aAAa,mBAAmB,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAC9E,UAAI,YAAY,IAAI,UAAU,EAAG;AACjC,kBAAY,IAAI,UAAU;AAAA,IAC5B;AACA,QAAI,GAAI,SAAQ,IAAI,EAAE;AACtB,cAAU,KAAK,EAAE,UAAU,MAAM,mBAAmB,CAAC;AAAA,EACvD;AAGA,QAAM,SAA+B,UAAU,IAAI,CAAC,EAAE,UAAU,mBAAmB,OAAO;AAAA,IACxF,GAAG;AAAA,IACH,SAAS;AAAA,IACT,MAAM,OAAO,cAAc,UAAU,OAAO,OAAO,OAAO,OAAO;AAAA,EACnE,EAAE;AAGF,QAAM,WACJ,OAAO,OAAO,kBAAkB,WAC5B,OAAO,OAAO,CAAC,SAAS,KAAK,QAAS,OAAO,aAAwB,IACrE;AAON,SAAO,SAAS,KAAK,eAAe,SAAS,CAAC;AAChD;AAOA,SAAS,eACP,WACiE;AACjE,MAAI,cAAc,OAAO;AAGvB,WAAO,CAAC,MAAM,UAAU;AACtB,UAAI,MAAM,SAAS,KAAK,KAAM,QAAO,MAAM,OAAO,KAAK;AACvD,YAAM,WACJ,OAAO,KAAK,cAAc,WACtB,KAAK,YACL;AACN,YAAM,YACJ,OAAO,MAAM,cAAc,WACvB,MAAM,YACN;AACN,UAAI,aAAa,UAAW,QAAO,WAAW;AAC9C,aAAO,KAAK,QAAQ,cAAc,MAAM,OAAO;AAAA,IACjD;AAAA,EACF;AAKA,SAAO,CAAC,MAAM,UAAU;AACtB,QAAI,MAAM,SAAS,KAAK,KAAM,QAAO,MAAM,OAAO,KAAK;AACvD,UAAM,WACJ,OAAO,KAAK,cAAc,WACtB,KAAK,YACL;AACN,UAAM,YACJ,OAAO,MAAM,cAAc,WACvB,MAAM,YACN;AACN,QAAI,cAAc,SAAU,QAAO,YAAY;AAC/C,YAAQ,MAAM,SAAS,MAAM,KAAK,SAAS;AAAA,EAC7C;AACF;AAQA,SAAS,cAAc,MAA4C;AACjE,MAAI,KAAK,GAAI,QAAO,KAAK;AACzB,MAAI,KAAK,aAAa,OAAO,KAAK,cAAc,UAAU;AACxD,WAAO,GAAG,KAAK,SAAS,IAAI,KAAK,SAAS;AAAA,EAC5C;AACA,SAAO;AACT;","names":[]}

package/dist/search/factory.js

@@ -2,19 +2,19 @@ import {
   createConversationIndexRuntime,
   createConversationSearchBackend,
   createSearchBackend
-} from "../chunk-2KAYTPPT.js";
+} from "../chunk-JPETDXED.js";
+import "../chunk-5CEJH5ZN.js";
 import "../chunk-WRFKZEO6.js";
 import "../chunk-CYEPCZN5.js";
 import "../chunk-LQ6JI4VH.js";
 import "../chunk-SANZHXY2.js";
-import "../chunk-EHISUJFN.js";
-import "../chunk-5CEJH5ZN.js";
 import "../chunk-AER6MT24.js";
+import "../chunk-EHISUJFN.js";
 import "../chunk-CINZGPSJ.js";
-import "../chunk-DCWIQFNA.js";
+import "../chunk-T422SYM6.js";
+import "../chunk-SJHM6I4J.js";
 import "../chunk-XBZQRZ6G.js";
 import "../chunk-KCQA46NR.js";
-import "../chunk-SJHM6I4J.js";
 import "../chunk-K43PI6DQ.js";
 import "../chunk-YNQ6DFSV.js";
 import "../chunk-O75CRYGF.js";

package/dist/search/index.js

@@ -2,7 +2,10 @@ import {
   createConversationIndexRuntime,
   createConversationSearchBackend,
   createSearchBackend
-} from "../chunk-2KAYTPPT.js";
+} from "../chunk-JPETDXED.js";
+import {
+  LanceDbBackend
+} from "../chunk-5CEJH5ZN.js";
 import {
   MeilisearchBackend
 } from "../chunk-WRFKZEO6.js";
@@ -15,20 +18,17 @@ import {
 import {
   RemoteSearchBackend
 } from "../chunk-SANZHXY2.js";
-import {
-  EmbedHelper
-} from "../chunk-EHISUJFN.js";
-import {
-  LanceDbBackend
-} from "../chunk-5CEJH5ZN.js";
 import {
   scanMemoryDir
 } from "../chunk-AER6MT24.js";
+import {
+  EmbedHelper
+} from "../chunk-EHISUJFN.js";
 import "../chunk-CINZGPSJ.js";
-import "../chunk-DCWIQFNA.js";
+import "../chunk-T422SYM6.js";
+import "../chunk-SJHM6I4J.js";
 import "../chunk-XBZQRZ6G.js";
 import "../chunk-KCQA46NR.js";
-import "../chunk-SJHM6I4J.js";
 import "../chunk-K43PI6DQ.js";
 import "../chunk-YNQ6DFSV.js";
 import "../chunk-O75CRYGF.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "9.3.688",
+  "version": "9.3.689",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",
@@ -2931,7 +2931,7 @@
     "core"
   ],
   "peerDependencies": {
-    "@remnic/coding-graph": "^9.3.688"
+    "@remnic/coding-graph": "^9.3.689"
   },
   "peerDependenciesMeta": {
     "@remnic/coding-graph": {

package/src/recall-pipeline-stages.test.ts

@@ -0,0 +1,269 @@
+// Issue #1539 PR2 — unit tests for the recall pipeline spine module.
+//
+// These tests verify `unifiedDedupeAndRank` produces the correct results for
+// each declared divergence dimension, WITHOUT changing any existing pipeline.
+// PRs 3–6 will migrate each pipeline to call this function; at that point the
+// characterization snapshots (tests/recall-pipeline-unified.test.ts) verify
+// end-to-end byte-for-byte parity.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { EvidencePackItem } from "./evidence-pack.js";
+import {
+  unifiedDedupeAndRank,
+  type RankedEvidenceItem,
+} from "./recall-pipeline-stages.js";
+
+const NO_INTENTS: never[] = [];
+const constantScorer = (_item: EvidencePackItem): number => 10;
+
+function item(
+  turnIndex: number,
+  content: string,
+  overrides: Partial<EvidencePackItem> = {},
+): EvidencePackItem {
+  return {
+    id: `s1:${turnIndex}`,
+    sessionId: "s1",
+    turnIndex,
+    role: "user",
+    content,
+    ...overrides,
+  };
+}
+
+test("unifiedDedupeAndRank: dedup collapses identical ids", () => {
+  const items = [
+    item(3, "Content A"),
+    item(3, "Content A"), // same id → deduped
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+  });
+  assert.equal(result.length, 1);
+  assert.equal(result[0]?.turnIndex, 3);
+});
+
+test("unifiedDedupeAndRank: dedup collapses identical normalized content under different ids", () => {
+  const items = [
+    item(10, "My monthly expenses are $2,400."),
+    item(11, "MY   MONTHLY   EXPENSES ARE $2,400."), // same normalized content
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+  });
+  assert.equal(result.length, 1, "expected content dedup to collapse the two items");
+  assert.equal(result[0]?.turnIndex, 10, "first-seen wins");
+});
+
+test("unifiedDedupeAndRank: DESC sort (default) orders turnIndex descending on score ties", () => {
+  const items = [
+    item(2, "Oldest"),
+    item(5, "Middle"),
+    item(9, "Newest"),
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer, // all tied → turnIndex DESC
+  });
+  const turns = result.map((r) => r.turnIndex);
+  assert.deepEqual(turns, [9, 5, 2]);
+});
+
+test("unifiedDedupeAndRank: ASC sort orders turnIndex ascending on score ties", () => {
+  const items = [
+    item(30, "Latest"),
+    item(10, "Earliest"),
+    item(20, "Middle"),
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+    turnIndexSortDirection: "asc",
+  });
+  const turns = result.map((r) => r.turnIndex);
+  assert.deepEqual(turns, [10, 20, 30]);
+});
+
+test("unifiedDedupeAndRank: rank primary key is always DESC regardless of turnIndex direction", () => {
+  const items = [
+    item(1, "Low rank", { score: 0 }),
+    item(2, "High rank", { score: 0 }),
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: (i) => (i.turnIndex === 2 ? 100 : 1),
+    turnIndexSortDirection: "asc",
+  });
+  // rank DESC wins over turnIndex ASC: turn 2 (rank 100) comes first despite ASC.
+  assert.equal(result[0]?.turnIndex, 2);
+  assert.equal(result[1]?.turnIndex, 1);
+});
+
+test("unifiedDedupeAndRank: rankThreshold drops items below the declared threshold", () => {
+  const items = [
+    item(1, "Weak", { score: 0 }),
+    item(2, "Strong", { score: 0 }),
+    item(3, "Medium", { score: 0 }),
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: (i) => {
+      if (i.turnIndex === 1) return 3;
+      if (i.turnIndex === 2) return 10;
+      return 6;
+    },
+    rankThreshold: 6,
+  });
+  const turns = result.map((r) => r.turnIndex).sort((a, b) => (a ?? 0) - (b ?? 0));
+  assert.deepEqual(turns, [2, 3], "turn 1 (rank 3) is below threshold 6 and dropped");
+});
+
+test("unifiedDedupeAndRank: transformContent is applied to output but NOT to scorer input", () => {
+  const items = [item(5, "original content")];
+  let scorerSawTransformed = false;
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: (i) => {
+      if (i.content.includes("APPENDED CUE")) scorerSawTransformed = true;
+      return 5;
+    },
+    transformContent: (content) => `${content}\nAPPENDED CUE`,
+  });
+  assert.equal(scorerSawTransformed, false, "scorer must see ORIGINAL content");
+  assert.ok(result[0]?.content.includes("APPENDED CUE"), "output must have transformed content");
+});
+
+test("unifiedDedupeAndRank: undefined turnIndex sorts to the bottom of DESC (-1 sentinel)", () => {
+  const items = [
+    item(5, "Has turn"),
+    { id: "s1:x", sessionId: "s1", role: "user", content: "No turn" }, // no turnIndex
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+    // default DESC
+  });
+  assert.equal(result[1]?.id, "s1:x", "undefined-turnIndex item sorts last in DESC");
+});
+
+test("unifiedDedupeAndRank: undefined turnIndex sorts to the bottom of ASC (MAX sentinel)", () => {
+  const items = [
+    item(5, "Has turn"),
+    { id: "s1:x", sessionId: "s1", role: "user", content: "No turn" },
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+    turnIndexSortDirection: "asc",
+  });
+  assert.equal(result[1]?.id, "s1:x", "undefined-turnIndex item sorts last in ASC");
+});
+
+test("unifiedDedupeAndRank: DESC tertiary tiebreaker is score DESC", () => {
+  // Same rank (all tied via constantScorer) AND same turnIndex → score breaks the tie.
+  const items = [
+    item(5, "Low score", { score: 10 }),
+    item(5, "High score", { score: 90 }), // same turn, deduped by content? No — different content
+  ];
+  // Wait — both have turn 5 so same id "s1:5" → second is deduped. Use different sessions.
+  const itemsDistinct: EvidencePackItem[] = [
+    { id: "s1:5", sessionId: "s1", turnIndex: 5, role: "user", content: "A", score: 10 },
+    { id: "s2:5", sessionId: "s2", turnIndex: 5, role: "user", content: "B", score: 90 },
+  ];
+  const result = unifiedDedupeAndRank(itemsDistinct, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer, // rank tied → turnIndex tied → score DESC
+  });
+  assert.equal(result[0]?.id, "s2:5", "higher score (90) ranks first on DESC tertiary");
+  assert.equal(result[1]?.id, "s1:5");
+});
+
+test("unifiedDedupeAndRank: ASC tertiary tiebreaker is content.localeCompare", () => {
+  const items: EvidencePackItem[] = [
+    { id: "s2:5", sessionId: "s2", turnIndex: 5, role: "user", content: "Banana" },
+    { id: "s1:5", sessionId: "s1", turnIndex: 5, role: "user", content: "Apple" },
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer, // rank tied → turnIndex tied → content localeCompare
+    turnIndexSortDirection: "asc",
+  });
+  assert.equal(result[0]?.content, "Apple", "localeCompare ASC: Apple < Banana");
+  assert.equal(result[1]?.content, "Banana");
+});
+
+test("unifiedDedupeAndRank: RankedEvidenceItem carries the computed rank", () => {
+  const items = [item(1, "content")];
+  const result: RankedEvidenceItem[] = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: () => 42,
+  });
+  assert.equal(result[0]?.rank, 42);
+});
+
+test("unifiedDedupeAndRank: fallback id uses sessionId:turnIndex when id is absent", () => {
+  const items: EvidencePackItem[] = [
+    { sessionId: "s1", turnIndex: 7, role: "user", content: "No explicit id" },
+    { sessionId: "s1", turnIndex: 7, role: "user", content: "Same fallback id → deduped" },
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+  });
+  assert.equal(result.length, 1, "fallback id dedup must collapse the two items");
+});
+
+test("unifiedDedupeAndRank: dedupByContent false keeps distinct ids with identical content (event-order shape)", () => {
+  // Event-order's rankAndSelectEventOrderItems deduplicates by turn id only and
+  // keeps distinct turns even when two turns share the same cue-appended body.
+  // The spine must express that as a declared config field so PR 6's migration
+  // does not silently drop valid turns (cursor bugbot a4299851).
+  const items: EvidencePackItem[] = [
+    item(1, "What time is it?", { id: "s1:1" }),
+    item(5, "What time is it?", { id: "s1:5" }),
+  ];
+  const result = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+    dedupByContent: false,
+  });
+  assert.equal(result.length, 2, "distinct ids with identical content must both survive when dedupByContent is false");
+});
+
+test("unifiedDedupeAndRank: dedupByContent true (default) still collapses identical content under different ids", () => {
+  const items: EvidencePackItem[] = [
+    item(1, "Duplicate body", { id: "s1:1" }),
+    item(2, "Duplicate body", { id: "s1:2" }),
+  ];
+  const resultDefault = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+  });
+  assert.equal(resultDefault.length, 1, "default dedupByContent collapses identical content");
+  const resultExplicit = unifiedDedupeAndRank(items, {
+    query: "q",
+    intents: NO_INTENTS,
+    scoreEvidence: constantScorer,
+    dedupByContent: true,
+  });
+  assert.equal(resultExplicit.length, 1, "explicit dedupByContent: true collapses identical content");
+});