@chroxify/eunoe 1.2.1 → 1.3.0

package/src/cli/sessions/index.ts

@@ -2,8 +2,8 @@ import { spawnSync } from "node:child_process"
 import { claudeConfigDirs, readConfig } from "../../config"
 import { FOLD_CAP_CHARS, foldedView, sourceChunks } from "../../context/fold"
 import { recallFor } from "../../context/recall"
-import { DEFAULT_RATIO } from "../../proxy/estimate"
-import { continueToken } from "../../proxy/handoff"
+import { DEFAULT_RATIO } from "../../core/estimate"
+import { continueToken } from "../../core/handoff"
 import { seenSessions } from "../../proxy/sessions"
 import { isSessionId } from "../../transcript/locate"
 import { subagentChunks } from "../../transcript/subagents"

package/src/config/index.ts

@@ -1,29 +1,13 @@
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"
 import { homedir } from "node:os"
 import path from "node:path"
+import { CORE_DEFAULTS, type CoreConfig, type Mode } from "../core/config"
 import { EUNOE_DIR, paths } from "./paths"
 
-export type Mode = "off" | "compact" | "default"
 export type SearchMode = "jsonl" | "markdown" | "xml" | "qmd"
 type Scope = "all" | "sessions"
 
-interface EvalConfig {
-  recallOptions?: { maxChars?: number; maxCharsMulti?: number; perQuery?: number }
-  strategy?: "tail" | "guide" | "trim"
-  rolling?: { keep: number; budget: number }
-  forceCut?: boolean
-  trustHeaders?: boolean
-  evidence?: boolean
-  recall?: boolean
-  rerank?: { model: string; candidates?: number; endpoint?: string }
-}
-
-export interface Tunables {
-  compactAt: number
-  keepTurns: number | "all"
-  keep: number
-  budget: number
-  turnCap: number | "off"
+export interface Tunables extends Pick<CoreConfig, "compactAt" | "keepTurns" | "keep" | "budget" | "turnCap"> {
   search: SearchMode
   rerank: string
 }
@@ -34,27 +18,20 @@ interface SessionSettings extends Partial<Tunables> {
   mode?: Mode
 }
 
-export interface Config extends Tunables {
-  mode: Mode
+export interface Config extends CoreConfig, Tunables {
   scope: Scope
   sessions: Record<string, SessionSettings>
   port: number
   upstream: string
   claudeConfigDirs: string[]
   window?: number
-  eval?: EvalConfig
 }
 
 export const DEFAULTS: Config = {
-  mode: "default",
+  ...CORE_DEFAULTS,
   scope: "all",
   sessions: {},
   port: 38663,
-  compactAt: 0.9,
-  keepTurns: "all",
-  keep: 3,
-  budget: 100_000,
-  turnCap: 100_000,
   search: "markdown",
   rerank: "jev-preview",
   upstream: "https://api.anthropic.com",

package/src/context/content.ts

@@ -45,8 +45,71 @@ export function clean(message: Message): Message {
   return { role: message.role, content: blocks(message).map(withoutCacheControl) }
 }
 
+const SHA256_ROUNDS = new Uint32Array([
+  0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6,
+  0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb,
+  0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, 0x748f82ee,
+  0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+])
+
+const utf8 = new TextEncoder()
+
+const rotate = (x: number, n: number) => (x >>> n) | (x << (32 - n))
+
+export function sha256(text: string): string {
+  const data = utf8.encode(text)
+  const size = ((data.length + 72) >> 6) << 6
+  const bytes = new Uint8Array(size)
+  bytes.set(data)
+  bytes[data.length] = 0x80
+  const view = new DataView(bytes.buffer)
+  view.setUint32(size - 8, Math.floor(data.length / 0x20000000))
+  view.setUint32(size - 4, (data.length << 3) >>> 0)
+  const h = new Uint32Array([0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19])
+  const w = new Uint32Array(64)
+  for (let offset = 0; offset < size; offset += 64) {
+    for (let i = 0; i < 16; i += 1) w[i] = view.getUint32(offset + i * 4)
+    for (let i = 16; i < 64; i += 1) w[i] = (w[i - 16] + (rotate(w[i - 15], 7) ^ rotate(w[i - 15], 18) ^ (w[i - 15] >>> 3)) + w[i - 7] + (rotate(w[i - 2], 17) ^ rotate(w[i - 2], 19) ^ (w[i - 2] >>> 10))) | 0
+    let [a, b, c, d, e, f, g, k] = h
+    for (let i = 0; i < 64; i += 1) {
+      const t1 = (k + (rotate(e, 6) ^ rotate(e, 11) ^ rotate(e, 25)) + ((e & f) ^ (~e & g)) + SHA256_ROUNDS[i] + w[i]) | 0
+      const t2 = ((rotate(a, 2) ^ rotate(a, 13) ^ rotate(a, 22)) + ((a & b) ^ (a & c) ^ (b & c))) | 0
+      k = g
+      g = f
+      f = e
+      e = (d + t1) | 0
+      d = c
+      c = b
+      b = a
+      a = (t1 + t2) | 0
+    }
+    h[0] += a
+    h[1] += b
+    h[2] += c
+    h[3] += d
+    h[4] += e
+    h[5] += f
+    h[6] += g
+    h[7] += k
+  }
+  return Array.from(h, (x) => x.toString(16).padStart(8, "0")).join("")
+}
+
 export function shortHash(json: string, chars: number): string {
-  return new Bun.CryptoHasher("sha256").update(json).digest("hex").slice(0, chars)
+  return sha256(json).slice(0, chars)
+}
+
+export function quickHash(text: string): string {
+  let a = 0xdeadbeef ^ text.length
+  let b = 0x41c6ce57 ^ text.length
+  for (let i = 0; i < text.length; i += 1) {
+    const code = text.charCodeAt(i)
+    a = Math.imul(a ^ code, 2654435761)
+    b = Math.imul(b ^ code, 1597334677)
+  }
+  const high = Math.imul(a ^ (a >>> 16), 2246822507) ^ Math.imul(b ^ (b >>> 13), 3266489909)
+  const low = Math.imul(b ^ (b >>> 16), 2246822507) ^ Math.imul(high ^ (high >>> 13), 3266489909)
+  return `${(low >>> 0).toString(36)}${(high >>> 0).toString(36)}`
 }
 
 export function truncate(text: string, max: number): string {

package/src/context/fold/strategies.ts

@@ -1,6 +1,6 @@
-import { type Block, type Body, blocks, type Message, truncate, withoutCacheControl } from "../content"
+import { type Block, type Body, blocks, clean, type Message, truncate, withoutCacheControl } from "../content"
 import { clearedInput, compactionNotice, truncationNotice } from "../prompts"
-import { hasContent, isHarness, type ReduceOptions, reduceTurn, splitTurns } from "./turns"
+import { hasContent, isHarness, isStaleNote, type ReduceOptions, reduceTurn, splitTurns, turnFingerprint } from "./turns"
 
 export interface CompactOptions extends ReduceOptions {
   keepTurns: number
@@ -8,26 +8,70 @@ export interface CompactOptions extends ReduceOptions {
   guide: string | null
   omittedBefore?: number
   reduce?: (turn: Message[], options: ReduceOptions) => Message[]
+  pinned?: ReadonlySet<string>
 }
 
+const pinnedBy =
+  ({ pinned }: CompactOptions) =>
+  (turn: Message[]) =>
+    pinned !== undefined && pinned.size > 0 && pinned.has(turnFingerprint(turn))
+
+export const countPinned = (turns: Message[][], pinned: ReadonlySet<string>) => (pinned.size > 0 ? turns.filter((turn) => pinned.has(turnFingerprint(turn))).length : 0)
+
 export function trimMessages(messages: Message[], options: ReduceOptions = {}): Message[] {
   const { preamble, turns } = splitTurns(messages)
   if (turns.length < 2) return messages
   return [...preamble, ...turns.slice(0, -1).flatMap((turn) => reduceTurn(turn, options)), ...(turns.at(-1) ?? [])]
 }
 
+export const CARRIED_CHARS = 80_000
+
+function carriedHarness(removed: Message[]): Block[] {
+  const texts = removed
+    .filter((m) => isHarness(m) && hasContent(m))
+    .flatMap((m) => blocks(m).filter((b) => b.type === "text"))
+    .map((b) => String(b.text ?? ""))
+    .filter((text) => text.trim() && !isStaleNote(text))
+  const kept: string[] = []
+  const seen = new Set<string>()
+  let used = 0
+  for (const text of texts.toReversed()) {
+    if (seen.has(text) || used + text.length > CARRIED_CHARS) continue
+    seen.add(text)
+    kept.push(text)
+    used += text.length
+  }
+  return kept.toReversed().map((text) => ({ type: "text", text: `<system-reminder>\n${text}\n</system-reminder>` }))
+}
+
+export const foldsOpening = (opening: Message[], keptFrom: number) => keptFrom === 1 && opening.length > 1
+
 export function compactMessages(messages: Message[], cutIndex: number, options: CompactOptions): Message[] {
   const { preamble, turns } = splitTurns(messages)
   if (cutIndex <= 0 || cutIndex >= turns.length) return messages
   const reduce = options.reduce ?? reduceTurn
+  const isPinned = pinnedBy(options)
   const keptFrom = Math.max(1, cutIndex - options.keepTurns)
-  const notice = compactionNotice({ omitted: keptFrom - 1 + (options.omittedBefore ?? 0), kept: cutIndex - keptFrom, guide: options.guide })
-  const removed = [...turns[0].slice(1), ...turns.slice(1, keptFrom).flat()]
-  const carried = removed
-    .filter((m) => isHarness(m) && hasContent(m))
-    .flatMap((m) => blocks(m).filter((b) => b.type === "text"))
-    .map((b) => ({ type: "text", text: `<system-reminder>\n${b.text}\n</system-reminder>` }))
-  return [...preamble, turns[0][0], { role: "user", content: [{ type: "text", text: notice }, ...carried] }, ...turns.slice(keptFrom, cutIndex).flatMap((turn) => reduce(turn, options)), ...turns.slice(cutIndex).flat()]
+  const [first, ...rest] = turns[0]
+  const openingPinned = isPinned(turns[0])
+  const openingFolded = !openingPinned && foldsOpening(turns[0], keptFrom)
+  const dropped = turns.slice(1, keptFrom)
+  const whole = dropped.filter(isPinned)
+  const pinned = whole.length + turns.slice(keptFrom, cutIndex).filter(isPinned).length + (openingPinned ? 1 : 0)
+  const kept = cutIndex - keptFrom + (openingFolded ? 1 : 0)
+  const notice = compactionNotice({ omitted: dropped.length - whole.length + (options.omittedBefore ?? 0), kept, pinned, guide: options.guide })
+  const removed = [...(openingPinned || openingFolded ? [] : rest), ...dropped.filter((turn) => !isPinned(turn)).flat()]
+  const shown = (turn: Message[]) => (isPinned(turn) ? turn.map(clean) : reduce(turn, options))
+  return [
+    ...preamble,
+    first,
+    ...(openingPinned ? rest.map(clean) : []),
+    { role: "user", content: [{ type: "text", text: notice }, ...carriedHarness(removed)] },
+    ...(openingFolded ? reduce(turns[0], options).slice(1) : []),
+    ...whole.flatMap((turn) => turn.map(clean)),
+    ...turns.slice(keptFrom, cutIndex).flatMap(shown),
+    ...turns.slice(cutIndex).flat(),
+  ]
 }
 
 export function capTurns(messages: Message[], options: CompactOptions): Message[] {
@@ -36,7 +80,8 @@ export function capTurns(messages: Message[], options: CompactOptions): Message[
   const { preamble, turns } = splitTurns(messages)
   if (turns.length < 3) return messages
   const live = turns.length - 1
-  const over = (turn: Message[], i: number) => i > 0 && i < live && JSON.stringify(turn).length > capChars
+  const isPinned = pinnedBy(options)
+  const over = (turn: Message[], i: number) => i > 0 && i < live && !isPinned(turn) && JSON.stringify(turn).length > capChars
   if (!turns.some(over)) return messages
   const reduce = options.reduce ?? reduceTurn
   return [...preamble, ...turns.flatMap((turn, i) => (over(turn, i) ? reduce(turn, options) : turn))]

package/src/context/fold/turns.ts

@@ -6,9 +6,18 @@ export interface ReduceOptions {
   evidence?: boolean
 }
 
+const IMAGE_NOTE = /^\s*\[Image: original \d+x\d+, displayed at \d+x\d+\.[^\]]*\]\s*$/
+
+const STALE_NOTE = /changed on disk since you last read it|was modified, either by the user or by a linter/
+
+export const isStaleNote = (text: unknown) => STALE_NOTE.test(String(text ?? ""))
+
+const isImageNote = (content: Block[]) => content.length > 0 && content.every((block) => block.type === "text" && IMAGE_NOTE.test(String(block.text ?? "")))
+
 export function isPrompt(message: Message) {
   if (message.role !== "user") return false
   const content = blocks(message)
+  if (isImageNote(content)) return false
   return !content.some((block) => block.type === "tool_result") && content.some((block) => block.type === "text" || block.type === "image" || block.type === "document")
 }
 
@@ -39,7 +48,18 @@ export function hasContent(message: Message) {
 const hasText = (block: Block) => block.type === "text" && String(block.text ?? "").trim() !== ""
 
 function mergedHarness(turn: Message[]): Message[] {
-  const content = turn.filter((m) => isHarness(m) && hasContent(m)).flatMap((m) => blocks(m).map(withoutCacheControl))
+  const seen = new Set<string>()
+  const fresh = (block: Block) => {
+    if (block.type !== "text") return true
+    const text = String(block.text ?? "")
+    if (isStaleNote(text) || seen.has(text)) return false
+    seen.add(text)
+    return true
+  }
+  const content = turn
+    .filter((m) => isHarness(m) && hasContent(m))
+    .flatMap((m) => blocks(m).map(withoutCacheControl))
+    .filter(fresh)
   return content.length > 0 ? [{ role: "system", content }] : []
 }
 
@@ -56,7 +76,7 @@ function interrupted(turn: Message[]): string {
 function carriedText(messages: Message[]): Block[] {
   return messages
     .filter((m) => m.role === "user")
-    .flatMap((m) => blocks(m).filter((b) => hasText(b) && !/^\s*(?:<system-reminder>[\s\S]*?<\/system-reminder>\s*)+$/.test(String(b.text))))
+    .flatMap((m) => blocks(m).filter((b) => hasText(b) && !IMAGE_NOTE.test(String(b.text)) && !/^\s*(?:<system-reminder>[\s\S]*?<\/system-reminder>\s*)+$/.test(String(b.text))))
     .map((b) => ({ type: "text", text: b.text }))
 }
 

package/src/context/prompts.ts

@@ -1,9 +1,9 @@
-import type { TranscriptRef } from "../transcript/search"
 import { truncate } from "./content"
 
 interface NoticeCounts {
   omitted: number
   kept: number
+  pinned?: number
   guide: string | null
 }
 
@@ -68,7 +68,7 @@ export function clearedInput(kept: string, cleared: number): string {
   return `${kept}… [${cleared} chars cleared; in the transcript]`
 }
 
-export function compactionNotice({ omitted, kept, guide }: NoticeCounts): string {
+export function compactionNotice({ omitted, kept, pinned = 0, guide }: NoticeCounts): string {
   return [
     "<context-compacted>",
     "Your context was compacted here, so tool calls and their output from before this point were removed.",
@@ -77,6 +77,7 @@ export function compactionNotice({ omitted, kept, guide }: NoticeCounts): string
       : kept > 0
         ? "Every earlier turn is shown below as the user's message and your final reply."
         : "",
+    pinned > 0 ? `${pinned === 1 ? "One earlier turn was" : `${pinned} earlier turns were`} kept whole, with their tool calls and output, because they matter to what comes next.` : "",
     "The turn that was in progress when the context filled up follows in full. Continue from it.",
     "Instructions and preferences the user gave in earlier turns still apply: how they want things done, and what to do or not do on your own. Look back over their messages before you act.",
     "For an exact detail from before this point (an ID, a path, a command, an error, a value, what a tool printed), look it up instead of going from memory. A lookup is cheap; a wrong detail is not. Never say you don't know about something earlier in this session without searching for it first, and if what you remember contradicts what the user says, search before telling them they are wrong.",
@@ -137,7 +138,7 @@ export function searchOnlyInstructions(guide: string): string {
   ].join("\n")
 }
 
-export function searchGuide(ref: TranscriptRef): string {
+export function searchGuide(ref: { mode: string; jsonl: string; dir: string }): string {
   if (ref.mode === "jsonl") {
     return [
       `The full transcript is Claude Code's session file: ${ref.jsonl}`,

package/src/context/recall/index.ts

@@ -50,6 +50,50 @@ function windows(lines: string[], size: number): string[][] {
   return out
 }
 
+interface ChunkedText {
+  text: string
+  head: string
+  turn: number
+  chunks: RecallChunk[]
+}
+
+const CHUNK_CACHE_TERMS = 8_000_000
+
+const chunked = new Map<string, ChunkedText>()
+let chunkedTerms = 0
+
+const termCount = (chunks: RecallChunk[]) => chunks.reduce((sum, chunk) => sum + chunk.terms.length, 0)
+
+function forget(key: string) {
+  const known = chunked.get(key)
+  if (!known) return
+  chunked.delete(key)
+  chunkedTerms -= termCount(known.chunks)
+}
+
+function chunksOf(key: string | null, text: string, head: string, turn: number, build: () => RecallChunk[]): RecallChunk[] {
+  if (key === null) return build()
+  const known = chunked.get(key)
+  if (known && known.text === text && known.head === head) {
+    chunked.delete(key)
+    chunked.set(key, known)
+    if (known.turn !== turn) {
+      known.turn = turn
+      known.chunks = known.chunks.map((chunk) => ({ ...chunk, turn }))
+    }
+    return known.chunks
+  }
+  forget(key)
+  const chunks = build()
+  chunked.set(key, { text, head, turn, chunks })
+  chunkedTerms += termCount(chunks)
+  for (const oldest of chunked.keys()) {
+    if (chunkedTerms <= CHUNK_CACHE_TERMS || oldest === key) break
+    forget(oldest)
+  }
+  return chunks
+}
+
 export function recallChunks(messages: Message[], keptIds: Map<string, string>, options: RecallOptions = {}): RecallChunk[] {
   const size = options.windowLines ?? 6
   const weight = Math.max(1, options.callWeight ?? 1)
@@ -61,8 +105,12 @@ export function recallChunks(messages: Message[], keptIds: Map<string, string>,
         const text = resultText(result)
         if (keptIds.get(call.id) === text || !text.trim()) continue
         const head = describeToolCall(call)
-        const headTerms = Array.from({ length: weight }, () => terms(head)).flat()
-        for (const window of windows(text.split("\n"), size)) chunks.push({ turn: turnIndex + 1, call: head, lines: window, terms: [...headTerms, ...terms(window.join("\n"))] })
+        const key = typeof call.id === "string" && call.id ? `${size}|${weight}|${call.id}` : null
+        const built = chunksOf(key, text, head, turnIndex + 1, () => {
+          const headTerms = Array.from({ length: weight }, () => terms(head)).flat()
+          return windows(text.split("\n"), size).map((window) => ({ turn: turnIndex + 1, call: head, lines: window, terms: [...headTerms, ...terms(window.join("\n"))] }))
+        })
+        for (const chunk of built) chunks.push(chunk)
       }
       if (!(options.narrative ?? true)) return
       const said = turn
@@ -74,34 +122,214 @@ export function recallChunks(messages: Message[], keptIds: Map<string, string>,
             .map((b) => String(b.text ?? "").trim()),
         )
         .filter(Boolean)
-      for (const window of windows(said.join("\n").split("\n"), size)) chunks.push({ turn: turnIndex + 1, call: "agent", lines: window, terms: terms(window.join("\n")) })
+        .join("\n")
+      if (!said) return
+      const built = chunksOf(`${size}|said|${turnIndex}|${said}`, said, "agent", turnIndex + 1, () => windows(said.split("\n"), size).map((window) => ({ turn: turnIndex + 1, call: "agent", lines: window, terms: terms(window.join("\n")) })))
+      for (const chunk of built) chunks.push(chunk)
     })
   return chunks
 }
 
-export function rank(query: string[], chunks: RecallChunk[], minTerms = 2) {
-  const df = new Map<string, number>()
-  for (const chunk of chunks) for (const term of new Set(chunk.terms)) df.set(term, (df.get(term) ?? 0) + 1)
-  const average = chunks.reduce((s, c) => s + c.terms.length, 0) / Math.max(1, chunks.length)
-  const wanted = [...new Set(query)].filter((t) => df.has(t))
-  return chunks
-    .map((chunk, index) => {
-      const tf = new Map<string, number>()
-      for (const term of chunk.terms) tf.set(term, (tf.get(term) ?? 0) + 1)
+interface RankedChunk {
+  index: number
+  score: number
+  matched: number
+}
+
+interface Postings {
+  chunks: RecallChunk[]
+  counts: number[]
+}
+
+interface CorpusIndex {
+  members: Set<RecallChunk>
+  postings: Map<string, Postings>
+}
+
+const INDEXED_FROM = 2_000
+const KEPT_INDEXES = 6
+const SAMPLED = 32
+
+const indexes: CorpusIndex[] = []
+
+function addToIndex(index: CorpusIndex, chunk: RecallChunk) {
+  index.members.add(chunk)
+  const counts = new Map<string, number>()
+  for (const term of chunk.terms) counts.set(term, (counts.get(term) ?? 0) + 1)
+  for (const [term, count] of counts) {
+    const postings = index.postings.get(term)
+    if (postings) {
+      postings.chunks.push(chunk)
+      postings.counts.push(count)
+    } else index.postings.set(term, { chunks: [chunk], counts: [count] })
+  }
+}
+
+function overlap(index: CorpusIndex, chunks: RecallChunk[]): number {
+  const step = Math.max(1, Math.floor(chunks.length / SAMPLED))
+  let found = 0
+  let sampled = 0
+  for (let i = 0; i < chunks.length; i += step) {
+    sampled += 1
+    if (index.members.has(chunks[i])) found += 1
+  }
+  return found / Math.max(1, sampled)
+}
+
+function indexFor(chunks: RecallChunk[]): CorpusIndex {
+  let best: CorpusIndex | null = null
+  let bestOverlap = 0.5
+  for (const index of indexes) {
+    const shared = overlap(index, chunks)
+    if (shared >= bestOverlap) {
+      best = index
+      bestOverlap = shared
+    }
+  }
+  let present = 0
+  if (best) for (const chunk of chunks) if (best.members.has(chunk)) present += 1
+  if (!best || best.members.size - present > chunks.length / 2) {
+    if (best) indexes.splice(indexes.indexOf(best), 1)
+    best = { members: new Set(), postings: new Map() }
+    present = 0
+  } else indexes.splice(indexes.indexOf(best), 1)
+  indexes.push(best)
+  if (indexes.length > KEPT_INDEXES) indexes.shift()
+  if (present < chunks.length) for (const chunk of chunks) if (!best.members.has(chunk)) addToIndex(best, chunk)
+  return best
+}
+
+let warming: { chunks: RecallChunk[]; index: CorpusIndex; next: number } | null = null
+
+export function warmIndex(chunks: RecallChunk[], until: number): boolean {
+  if (chunks.length < INDEXED_FROM) return true
+  if (warming?.chunks !== chunks) {
+    const index: CorpusIndex = { members: new Set(), postings: new Map() }
+    indexes.push(index)
+    if (indexes.length > KEPT_INDEXES) indexes.shift()
+    warming = { chunks, index, next: 0 }
+  }
+  const { index } = warming
+  while (warming.next < chunks.length) {
+    const chunk = chunks[warming.next]
+    warming.next += 1
+    if (!index.members.has(chunk)) addToIndex(index, chunk)
+    if (warming.next < chunks.length && performance.now() > until) return false
+  }
+  warming = null
+  return true
+}
+
+function indexedAll(queries: string[][], chunks: RecallChunk[], minTerms: number): RankedChunk[][] | null {
+  const position = new Map<RecallChunk, number>()
+  let total = 0
+  chunks.forEach((chunk, i) => {
+    position.set(chunk, i)
+    total += chunk.terms.length
+  })
+  if (position.size !== chunks.length) return null
+  const index = indexFor(chunks)
+  const average = total / Math.max(1, chunks.length)
+  const hitsByTerm = new Map<string, Array<[number, number]>>()
+  const hitsOf = (term: string) => {
+    const known = hitsByTerm.get(term)
+    if (known) return known
+    const hits: Array<[number, number]> = []
+    const postings = index.postings.get(term)
+    if (postings)
+      postings.chunks.forEach((chunk, k) => {
+        const at = position.get(chunk)
+        if (at !== undefined) hits.push([at, postings.counts[k]])
+      })
+    hitsByTerm.set(term, hits)
+    return hits
+  }
+  return queries.map((query) => {
+    const lists = [...new Set(query)].map(hitsOf)
+    const needed = Math.max(1, Math.min(minTerms, lists.filter((hits) => hits.length > 0).length))
+    const found = new Map<number, Array<[number, number]>>()
+    lists.forEach((hits, k) => {
+      for (const [at, f] of hits) {
+        const mine = found.get(at)
+        if (mine) mine.push([k, f])
+        else found.set(at, [[k, f]])
+      }
+    })
+    const scored: RankedChunk[] = []
+    for (const at of [...found.keys()].sort((a, b) => a - b)) {
+      const mine = found.get(at) as Array<[number, number]>
+      if (mine.length < needed) continue
+      const length = chunks[at].terms.length
       let score = 0
-      let matched = 0
-      for (const term of wanted) {
-        const f = tf.get(term) ?? 0
-        if (!f) continue
-        matched += 1
-        const n = df.get(term) ?? 0
+      for (const [k, f] of mine) {
+        const n = lists[k].length
         const idf = Math.log(1 + (chunks.length - n + 0.5) / (n + 0.5))
-        score += (idf * (f * (BM25_K1 + 1))) / (f + BM25_K1 * (1 - BM25_B + (BM25_B * chunk.terms.length) / average))
+        score += (idf * (f * (BM25_K1 + 1))) / (f + BM25_K1 * (1 - BM25_B + (BM25_B * length) / average))
       }
-      return { index, score, matched }
+      if (score > 0) scored.push({ index: at, score, matched: mine.length })
+    }
+    return scored.sort((a, b) => b.score - a.score)
+  })
+}
+
+function rankAll(queries: string[][], chunks: RecallChunk[], minTerms = 2): RankedChunk[][] {
+  return (chunks.length >= INDEXED_FROM && indexedAll(queries, chunks, minTerms)) || scanAll(queries, chunks, minTerms)
+}
+
+function scanAll(queries: string[][], chunks: RecallChunk[], minTerms: number): RankedChunk[][] {
+  const ids = new Map<string, number>()
+  for (const query of queries) for (const term of query) if (!ids.has(term)) ids.set(term, ids.size)
+  const df = new Int32Array(ids.size)
+  const counts = new Int32Array(ids.size)
+  const touched: number[] = []
+  const hits: Array<{ index: number; found: Array<[number, number]> }> = []
+  let total = 0
+  chunks.forEach((chunk, index) => {
+    total += chunk.terms.length
+    touched.length = 0
+    for (const term of chunk.terms) {
+      const id = ids.get(term)
+      if (id === undefined) continue
+      if (counts[id] === 0) touched.push(id)
+      counts[id] += 1
+    }
+    if (touched.length === 0) return
+    const found = touched.map((id): [number, number] => {
+      const f = counts[id]
+      counts[id] = 0
+      df[id] += 1
+      return [id, f]
     })
-    .filter((r) => r.matched >= Math.min(minTerms, wanted.length) && r.score > 0)
-    .sort((a, b) => b.score - a.score)
+    hits.push({ index, found })
+  })
+  const average = total / Math.max(1, chunks.length)
+  return queries.map((query) => {
+    const asked = [...new Set(query)].map((term) => ids.get(term) ?? -1)
+    const position = new Int32Array(ids.size).fill(-1)
+    asked.forEach((id, i) => {
+      position[id] = i
+    })
+    const needed = Math.max(1, Math.min(minTerms, asked.filter((id) => df[id] > 0).length))
+    const scored: RankedChunk[] = []
+    for (const { index, found } of hits) {
+      const mine = found.filter(([id]) => position[id] >= 0)
+      if (mine.length < needed) continue
+      if (mine.length > 1) mine.sort((a, b) => position[a[0]] - position[b[0]])
+      const length = chunks[index].terms.length
+      let score = 0
+      for (const [id, f] of mine) {
+        const n = df[id]
+        const idf = Math.log(1 + (chunks.length - n + 0.5) / (n + 0.5))
+        score += (idf * (f * (BM25_K1 + 1))) / (f + BM25_K1 * (1 - BM25_B + (BM25_B * length) / average))
+      }
+      if (score > 0) scored.push({ index, score, matched: mine.length })
+    }
+    return scored.sort((a, b) => b.score - a.score)
+  })
+}
+
+export function rank(query: string[], chunks: RecallChunk[], minTerms = 2): RankedChunk[] {
+  return rankAll([query], chunks, minTerms)[0]
 }
 
 function excerpt(chunk: RecallChunk, query: Set<string>, salient: boolean): string[] {
@@ -136,27 +364,23 @@ function merged(hits: Array<{ query: string[]; indices: number[] }>): RecallPick
   return [...picked.entries()].map(([index, query]) => ({ index, query }))
 }
 
-const ranked = (query: string[], chunks: RecallChunk[], options: RecallOptions, width: number) =>
-  rank(query, chunks, options.minTerms)
-    .slice(0, width)
-    .map((hit) => hit.index)
+function rankedLines(prompt: string, chunks: RecallChunk[], options: RecallOptions, width: number) {
+  const lines = queriesOf(prompt)
+  const queries = lines.map(terms)
+  const results = rankAll(queries, chunks, options.minTerms)
+  return lines.map((line, i) => ({ line, query: queries[i], indices: results[i].slice(0, width).map((hit) => hit.index) }))
+}
 
 function pickRecall(prompt: string, chunks: RecallChunk[], options: RecallOptions): RecallPick[] {
-  return merged(
-    queriesOf(prompt).map((line) => {
-      const query = terms(line)
-      return { query, indices: ranked(query, chunks, options, options.perQuery ?? 12) }
-    }),
-  )
+  return merged(rankedLines(prompt, chunks, options, options.perQuery ?? 12))
 }
 
 export const identityReranker: Reranker = async (_query, _chunks, candidates) => candidates
 
 export async function pickRecallWith(prompt: string, chunks: RecallChunk[], rerank: Reranker, options: RecallOptions = {}): Promise<RecallPick[]> {
   const hits: Array<{ query: string[]; indices: number[] }> = []
-  for (const line of queriesOf(prompt)) {
-    const query = terms(line)
-    const reordered = await rerank(line, chunks, ranked(query, chunks, options, options.candidates ?? 30))
+  for (const { line, query, indices } of rankedLines(prompt, chunks, options, options.candidates ?? 30)) {
+    const reordered = await rerank(line, chunks, indices)
     hits.push({ query, indices: reordered.slice(0, options.perQuery ?? 12) })
   }
   return merged(hits)