@crewhaus/compaction-curator 0.1.0

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@crewhaus/compaction-curator",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Pillar-2 active context curation — pre-compaction relevance + dedupe pass that lets compaction-autocompact skip the model call when curation alone brings tokens under the trigger threshold",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.0.0"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/compaction-curator"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/compaction-curator#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/index.test.ts

@@ -0,0 +1,168 @@
+import { describe, expect, test } from "bun:test";
+import { type EmbedderFn, type Item, curate, dedupeBySimilarity, rankByRelevance } from "./index";
+
+/**
+ * Tiny canned embedder for deterministic tests: maps each unique input
+ * string to a fixed unit vector. Same input → same vector → cosine of 1.
+ * Different input → orthogonal vector → cosine of 0.
+ */
+function makeCannedEmbedder(): { fn: EmbedderFn; counter: () => number } {
+  const cache = new Map<string, ReadonlyArray<number>>();
+  let calls = 0;
+  const dim = 16;
+  const fn: EmbedderFn = async (texts) => {
+    calls += 1;
+    return texts.map((t) => {
+      const cached = cache.get(t);
+      if (cached !== undefined) return cached;
+      const idx = cache.size % dim;
+      const v = new Array(dim).fill(0);
+      v[idx] = 1;
+      const arr = Object.freeze(v) as ReadonlyArray<number>;
+      cache.set(t, arr);
+      return arr;
+    });
+  };
+  return { fn, counter: () => calls };
+}
+
+describe("curate — happy path", () => {
+  test("returns input unchanged when no dedupe + no query", async () => {
+    const { fn } = makeCannedEmbedder();
+    const items: ReadonlyArray<Item> = [
+      { text: "alpha gamma delta epsilon zeta" },
+      { text: "beta theta iota kappa lambda" },
+      { text: "rho sigma tau upsilon phi" },
+    ];
+    const result = await curate(items, { embedder: fn });
+    expect(result.items.length).toBe(3);
+    expect(result.droppedIndices.length).toBe(0);
+    expect(result.bytesSaved).toBe(0);
+  });
+
+  test("dedupes when two items embed identically", async () => {
+    const embedder: EmbedderFn = async (texts) =>
+      texts.map(() => [1, 0, 0, 0] as ReadonlyArray<number>);
+    const items: ReadonlyArray<Item> = [
+      { text: "alpha gamma delta epsilon zeta one" },
+      { text: "alpha gamma delta epsilon zeta two" },
+      { text: "alpha gamma delta epsilon zeta three" },
+    ];
+    const result = await curate(items, { embedder });
+    expect(result.items.length).toBe(1);
+    expect(result.droppedIndices).toEqual([1, 2]);
+  });
+
+  test("keeps when below dedupe threshold", async () => {
+    const { fn } = makeCannedEmbedder();
+    const items: ReadonlyArray<Item> = [
+      { text: "alpha gamma delta epsilon zeta" },
+      { text: "beta theta iota kappa lambda" },
+    ];
+    const result = await curate(items, { embedder: fn, dedupeThreshold: 0.99 });
+    expect(result.items.length).toBe(2);
+  });
+
+  test("relevance-orders when query supplied", async () => {
+    const items: ReadonlyArray<Item> = [
+      { text: "completely unrelated chitchat", id: "off-topic" },
+      { text: "deep technical content about query", id: "on-topic" },
+    ];
+    // Make on-topic share the query vector
+    const embedder: EmbedderFn = async (texts) => {
+      return texts.map((t) => {
+        if (t === "deep technical content about query" || t === "search query") {
+          return [1, 0, 0] as ReadonlyArray<number>;
+        }
+        return [0, 1, 0] as ReadonlyArray<number>;
+      });
+    };
+    const result = await curate(items, { embedder, query: "search query" });
+    expect(result.items[0]?.id).toBe("on-topic");
+    expect(result.items[1]?.id).toBe("off-topic");
+  });
+
+  test("topK trims after relevance reorder", async () => {
+    const items: ReadonlyArray<Item> = [
+      { text: "irrelevant content one", id: "1" },
+      { text: "highly relevant query match", id: "2" },
+      { text: "irrelevant content two", id: "3" },
+    ];
+    const embedder: EmbedderFn = async (texts) =>
+      texts.map((t) => {
+        if (t === "highly relevant query match" || t === "the search query") {
+          return [1, 0] as ReadonlyArray<number>;
+        }
+        return [0, 1] as ReadonlyArray<number>;
+      });
+    const result = await curate(items, {
+      embedder,
+      query: "the search query",
+      relevanceTopK: 1,
+    });
+    expect(result.items.length).toBe(1);
+    expect(result.items[0]?.id).toBe("2");
+    expect(result.droppedIndices.length).toBe(2);
+  });
+});
+
+describe("curate — pre-embedded path", () => {
+  test("no embedder needed when all items carry embeddings", async () => {
+    const items: ReadonlyArray<Item> = [
+      { text: "first item with embedding precomputed", embedding: [1, 0] },
+      { text: "second item with embedding precomputed", embedding: [0, 1] },
+    ];
+    const result = await curate(items, {}); // no embedder, no query
+    expect(result.items.length).toBe(2);
+  });
+
+  test("throws when items lack embeddings and no embedder is supplied", async () => {
+    const items: ReadonlyArray<Item> = [{ text: "needs embedding plenty of words" }];
+    await expect(curate(items, {})).rejects.toThrow(/lack pre-computed embeddings/);
+  });
+});
+
+describe("curate — edge cases", () => {
+  test("empty input returns empty result", async () => {
+    const result = await curate([], {});
+    expect(result.items.length).toBe(0);
+    expect(result.bytesSaved).toBe(0);
+  });
+
+  test("bytesSaved reflects dropped item lengths", async () => {
+    const embedder: EmbedderFn = async (texts) => texts.map(() => [1, 0] as ReadonlyArray<number>);
+    const items: ReadonlyArray<Item> = [
+      { text: "kept first kept first kept first" }, // 32 bytes
+      { text: "dropped dropped dropped" }, // 23 bytes
+    ];
+    const result = await curate(items, { embedder });
+    expect(result.bytesSaved).toBe(Buffer.byteLength("dropped dropped dropped", "utf8"));
+  });
+});
+
+describe("dedupeBySimilarity (pure)", () => {
+  test("first occurrence wins for duplicates", () => {
+    const items: ReadonlyArray<Item> = [{ text: "a" }, { text: "b" }, { text: "c" }];
+    const embeddings: ReadonlyArray<ReadonlyArray<number>> = [
+      [1, 0],
+      [1, 0], // duplicate of #0
+      [0, 1],
+    ];
+    const r = dedupeBySimilarity(items, embeddings, 0.92);
+    expect(r.kept).toEqual([0, 2]);
+    expect(r.dropped).toEqual([1]);
+  });
+});
+
+describe("rankByRelevance (pure)", () => {
+  test("ranks by descending cosine similarity", async () => {
+    const items: ReadonlyArray<Item> = [{ text: "x" }, { text: "y" }];
+    const embeddings: ReadonlyArray<ReadonlyArray<number>> = [
+      [0, 1],
+      [1, 0],
+    ];
+    const embedder: EmbedderFn = async () => [[1, 0]];
+    const order = await rankByRelevance(items, embeddings, "q", embedder);
+    expect(order).toEqual([1, 0]);
+  });
+});

package/src/index.ts

@@ -0,0 +1,288 @@
+/**
+ * Pillar 2 active context curation — pre-compaction relevance + dedupe.
+ *
+ * `compaction-autocompact` (model summarize-then-replace) and
+ * `compaction-snip` (drop oldest) are *reactive*: they only fire when
+ * context is already near its limit, and they cost a model call (autocompact)
+ * or accept information loss (snip). Routray's "Stop Wasting Money on AI
+ * Context You Don't Need" (Mar 2026) documents 60–80% token reduction
+ * from a pre-compaction pass that:
+ *
+ *   1. Removes semantically duplicate items (embedding cosine ≥ threshold).
+ *   2. Re-orders to front-load high-relevance items (since transformer
+ *      attention favors prompt-start and prompt-end positions).
+ *
+ * This package implements both as pure functions over a generic `Item`
+ * shape (string text + optional embedding). Callers (compaction-autocompact
+ * primarily) wire it as an opt-in stage controlled by `spec.compaction.curate`.
+ * When curation brings the item count below the model-summarize trigger,
+ * the summarization model call is skipped entirely — that's where the
+ * cost savings compound on top of token savings.
+ *
+ * The package does NOT carry an embedder dependency. Callers supply
+ * `EmbedderFn` — typically wrapping `@crewhaus/embedder`'s configured
+ * provider. This keeps `compaction-curator` light enough to be useful
+ * from `target-pipeline` (which already has an embedder) without forcing
+ * a heavy import onto `target-cli` (which doesn't).
+ *
+ * Catalog layer: R6 (extension of §17 compaction primitives, symmetric to
+ * `compaction-autocompact` and `compaction-snip`). Recipe:
+ * demos/walkthroughs/52-context-curation.md.
+ */
+import { CrewhausError } from "@crewhaus/errors";
+
+export class CompactionCuratorError extends CrewhausError {
+  override readonly name = "CompactionCuratorError";
+  constructor(message: string, cause?: unknown) {
+    super("config", message, cause);
+  }
+}
+
+/**
+ * The generic shape this package operates on. Sufficient for two backends:
+ *   - context items: `text` = message body, `embedding` = optional pre-
+ *     computed vector
+ *   - RAG-retrieved chunks: `text` = chunk body, `embedding` = the vector
+ *     already used for retrieval
+ */
+export type Item = {
+  readonly text: string;
+  /** Optional pre-computed embedding. When absent, callers must supply
+   *  `EmbedderFn` to `curate()`. */
+  readonly embedding?: ReadonlyArray<number>;
+  /** Optional client-side identifier carried through unchanged. */
+  readonly id?: string;
+};
+
+/**
+ * Async embedder callback. Returns a vector per input string in the same
+ * order. Implementations typically batch internally for cost efficiency.
+ *
+ * Vectors must be normalized (unit length) so cosine reduces to a dot
+ * product. Most production embedders return normalized vectors by default;
+ * if yours doesn't, normalize before returning here.
+ */
+export type EmbedderFn = (
+  texts: ReadonlyArray<string>,
+) => Promise<ReadonlyArray<ReadonlyArray<number>>>;
+
+export type CurateOptions = {
+  /**
+   * Query / goal string used to score relevance. When absent, the curator
+   * skips the relevance-reorder pass and only does dedupe.
+   */
+  readonly query?: string;
+  /**
+   * Cosine-similarity threshold above which two items are considered
+   * duplicates. Default 0.92 — high enough to avoid false-positive
+   * collapses on near-but-distinct items (a function definition and its
+   * caller share substantial token overlap but should NOT dedupe).
+   */
+  readonly dedupeThreshold?: number;
+  /**
+   * Max items to keep after relevance scoring. Items below this rank are
+   * dropped entirely. When absent, no items are dropped from the
+   * relevance pass — items are only reordered. Set when you want a hard
+   * top-K filter.
+   */
+  readonly relevanceTopK?: number;
+  /**
+   * Embedder supplied by the caller. Required if any item lacks a
+   * pre-computed `embedding` AND either dedupe or relevance reorder is
+   * requested. The function-level types let it stay optional for the
+   * pre-embedded case (most production callers).
+   */
+  readonly embedder?: EmbedderFn;
+};
+
+export const DEFAULT_DEDUPE_THRESHOLD = 0.92;
+
+function cosine(a: ReadonlyArray<number>, b: ReadonlyArray<number>): number {
+  if (a.length !== b.length) {
+    throw new CompactionCuratorError(`embedding dimension mismatch (${a.length} vs ${b.length})`);
+  }
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    const av = a[i] ?? 0;
+    const bv = b[i] ?? 0;
+    dot += av * bv;
+    na += av * av;
+    nb += bv * bv;
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / Math.sqrt(na * nb);
+}
+
+/**
+ * Ensure every input item has an embedding. Items already carrying one
+ * pass through; items missing one are batched into a single embedder
+ * call. Preserves order so the caller's downstream indexing isn't
+ * disturbed.
+ */
+async function ensureEmbeddings(
+  items: ReadonlyArray<Item>,
+  embedder: EmbedderFn | undefined,
+): Promise<ReadonlyArray<ReadonlyArray<number>>> {
+  if (items.length === 0) return [];
+  const missingIdx: number[] = [];
+  const missingTexts: string[] = [];
+  for (let i = 0; i < items.length; i++) {
+    const item = items[i];
+    if (item !== undefined && item.embedding === undefined) {
+      missingIdx.push(i);
+      missingTexts.push(item.text);
+    }
+  }
+  if (missingIdx.length > 0 && embedder === undefined) {
+    throw new CompactionCuratorError(
+      `${missingIdx.length} item(s) lack pre-computed embeddings and no embedder was supplied — pass options.embedder or pre-embed your items`,
+    );
+  }
+  const fresh = embedder !== undefined && missingIdx.length > 0 ? await embedder(missingTexts) : [];
+  return items.map((item, i) => {
+    if (item.embedding !== undefined) return item.embedding;
+    const slot = missingIdx.indexOf(i);
+    if (slot === -1) {
+      throw new CompactionCuratorError(`internal: missing embedding for item ${i}`);
+    }
+    const v = fresh[slot];
+    if (v === undefined) {
+      throw new CompactionCuratorError(`embedder returned no vector for item ${i}`);
+    }
+    return v;
+  });
+}
+
+/**
+ * Pass 1 — semantic dedupe. For each item, if a *prior* item's embedding
+ * has cosine ≥ threshold, drop the current. Order-preserving: the first
+ * occurrence wins (keeps the head of the conversation, which carries
+ * goal-setting context).
+ */
+export function dedupeBySimilarity(
+  items: ReadonlyArray<Item>,
+  embeddings: ReadonlyArray<ReadonlyArray<number>>,
+  threshold: number,
+): { kept: ReadonlyArray<number>; dropped: ReadonlyArray<number> } {
+  const kept: number[] = [];
+  const dropped: number[] = [];
+  for (let i = 0; i < items.length; i++) {
+    let isDuplicate = false;
+    for (const k of kept) {
+      const e1 = embeddings[i];
+      const e2 = embeddings[k];
+      if (e1 === undefined || e2 === undefined) continue;
+      if (cosine(e1, e2) >= threshold) {
+        isDuplicate = true;
+        break;
+      }
+    }
+    if (isDuplicate) dropped.push(i);
+    else kept.push(i);
+  }
+  return { kept, dropped };
+}
+
+/**
+ * Pass 2 — relevance reorder. Compute cosine(query, item) for every
+ * surviving item; return indices in descending similarity order.
+ * Stable sort: items with identical similarity stay in original order.
+ *
+ * When `topK` is supplied, items beyond that rank are dropped.
+ */
+export async function rankByRelevance(
+  items: ReadonlyArray<Item>,
+  itemEmbeddings: ReadonlyArray<ReadonlyArray<number>>,
+  query: string,
+  embedder: EmbedderFn | undefined,
+  topK?: number,
+): Promise<ReadonlyArray<number>> {
+  if (items.length === 0) return [];
+  if (embedder === undefined) {
+    throw new CompactionCuratorError(
+      "rankByRelevance requires an embedder to compute the query vector",
+    );
+  }
+  const queryVecs = await embedder([query]);
+  const queryVec = queryVecs[0];
+  if (queryVec === undefined) {
+    throw new CompactionCuratorError("embedder returned no vector for the query");
+  }
+  const scored: Array<{ idx: number; sim: number; original: number }> = [];
+  for (let i = 0; i < items.length; i++) {
+    const e = itemEmbeddings[i];
+    if (e === undefined) continue;
+    scored.push({ idx: i, sim: cosine(queryVec, e), original: i });
+  }
+  scored.sort((a, b) => {
+    if (a.sim !== b.sim) return b.sim - a.sim;
+    return a.original - b.original; // stable
+  });
+  const ordered = scored.map((s) => s.idx);
+  if (topK !== undefined && ordered.length > topK) return ordered.slice(0, topK);
+  return ordered;
+}
+
+export type CurationResult = {
+  /** Items in the curated order (deduped + relevance-ranked + topK-trimmed). */
+  readonly items: ReadonlyArray<Item>;
+  /** Indices into the *original* input array, in output order. */
+  readonly originalIndices: ReadonlyArray<number>;
+  /** Indices that were dropped (in original order). */
+  readonly droppedIndices: ReadonlyArray<number>;
+  /** Bytes saved as compared to the original total (text only). */
+  readonly bytesSaved: number;
+};
+
+/**
+ * High-level entry: dedupe then relevance-reorder (when a query is
+ * supplied) then top-K filter. The most common call from
+ * `compaction-autocompact` is `curate(items, { query, embedder })`.
+ */
+export async function curate(
+  items: ReadonlyArray<Item>,
+  opts: CurateOptions = {},
+): Promise<CurationResult> {
+  if (items.length === 0) {
+    return { items: [], originalIndices: [], droppedIndices: [], bytesSaved: 0 };
+  }
+  const threshold = opts.dedupeThreshold ?? DEFAULT_DEDUPE_THRESHOLD;
+  const embeddings = await ensureEmbeddings(items, opts.embedder);
+  const { kept, dropped } = dedupeBySimilarity(items, embeddings, threshold);
+  let surviving: ReadonlyArray<number> = kept;
+  const droppedAll: number[] = [...dropped];
+  if (opts.query !== undefined && opts.query.length > 0) {
+    const subItems = surviving.map((i) => items[i]).filter((it): it is Item => it !== undefined);
+    const subEmbeddings = surviving
+      .map((i) => embeddings[i])
+      .filter((e): e is ReadonlyArray<number> => e !== undefined);
+    const localOrder = await rankByRelevance(
+      subItems,
+      subEmbeddings,
+      opts.query,
+      opts.embedder,
+      opts.relevanceTopK,
+    );
+    const localToOriginal: ReadonlyArray<number> = surviving;
+    surviving = localOrder
+      .map((localIdx) => localToOriginal[localIdx])
+      .filter((i): i is number => i !== undefined);
+    if (opts.relevanceTopK !== undefined) {
+      const survivingSet = new Set(surviving);
+      for (const i of kept) if (!survivingSet.has(i)) droppedAll.push(i);
+    }
+  }
+  const outItems = surviving.map((i) => items[i]).filter((it): it is Item => it !== undefined);
+  const droppedTotal = droppedAll
+    .map((i) => items[i])
+    .filter((it): it is Item => it !== undefined)
+    .reduce((acc, it) => acc + Buffer.byteLength(it.text, "utf8"), 0);
+  return {
+    items: outItems,
+    originalIndices: surviving,
+    droppedIndices: [...droppedAll].sort((a, b) => a - b),
+    bytesSaved: droppedTotal,
+  };
+}