dsh-continual-evolve 0.3.0 → 0.5.0

package/lib/inject.js

@@ -2,6 +2,9 @@ import { isArchived } from "./types.js";
 import { mergeHarnessStates } from "./state.js";
 import { entryLine } from "./render.js";
 import { recordInjection } from "./usage.js";
+import { buildRelevanceIndex, relevanceScore, tokenize } from "./search.js";
+/** CJK-bigram tokenizer re-exported for ranking consumers (see search.ts). */
+export { tokenize };
 /** Prompt sections render at most this many entries per kind. */
 export const MAX_INJECTED_ENTRIES_PER_KIND = 6;
 /** Per-entry content budget inside the injected block (matches render.ts). */
@@ -18,31 +21,6 @@ export const MAX_QUERY_CHARS = 400;
 function stableCompare(a, b) {
     return [a.path, a.title, a.id].join("\0").localeCompare([b.path, b.title, b.id].join("\0"));
 }
-/**
- * Lowercase tokenization for the keyword relevance scorer: runs of ASCII
- * alphanumerics and CJK characters become tokens (CJK is not split so whole
- * Chinese words/characters stay comparable), everything else is a separator.
- */
-export function tokenize(text) {
-    return text
-        .toLowerCase()
-        .split(/[^a-z0-9\u4e00-\u9fff]+/)
-        .filter((token) => token.length > 0);
-}
-/**
- * Keyword hit count of `query` tokens inside an entry: title hits weigh 2×,
- * content/path hits 1×. BM25-level relevance without any external service.
- */
-export function relevanceHits(entry, query) {
-    const titleTokens = tokenize(entry.title);
-    const bodyTokens = tokenize(`${entry.content} ${entry.path}`);
-    let hits = 0;
-    for (const token of tokenize(query)) {
-        hits += titleTokens.filter((t) => t === token).length * 2;
-        hits += bodyTokens.filter((t) => t === token).length;
-    }
-    return hits;
-}
 /**
  * Normalized recency in [0, 1]: 1 when the entry was just updated, decaying
  * linearly to 0 after {@link RECENCY_HALF_LIFE_MS}. Unparseable timestamps
@@ -61,20 +39,34 @@ export function recencyScore(entry, now) {
 }
 /**
  * Rank entries for injection, best first. With no query the ranking is pure
- * recency (newest first). With a query, any entry with at least one keyword
- * hit outranks every hit-less entry (`hits * 2 + recency <= 1` for the
- * latter), and hits decide the order among relevant entries; recency then
- * breaks remaining ties, and the stable dictionary order is the final
- * tiebreak, so the result is deterministic.
+ * recency (newest first). With a query, entries are scored once against a
+ * per-call BM25 index (CJK bigrams; field-weighted title ×2 — see
+ * search.ts): any entry with a positive score (≥1 matched token) outranks
+ * every hit-less entry (score exactly 0), scores decide the order among
+ * relevant entries, recency breaks remaining ties, and the stable dictionary
+ * order is the final tiebreak, so the result is deterministic. The input is
+ * never mutated.
  */
 export function rankEntries(entries, query, now = Date.now()) {
     const q = (query ?? "").trim();
-    return [...entries].sort((a, b) => {
-        if (q.length > 0) {
-            const relevanceDelta = relevanceHits(b, q) * 2 - relevanceHits(a, q) * 2;
-            if (relevanceDelta !== 0) {
-                return relevanceDelta;
+    if (q.length === 0) {
+        return [...entries].sort((a, b) => {
+            const recencyDelta = recencyScore(b, now) - recencyScore(a, now);
+            if (recencyDelta !== 0) {
+                return recencyDelta;
             }
+            return stableCompare(a, b);
+        });
+    }
+    // Precompute scores once: the old comparator re-tokenized both sides on
+    // every comparison (O(n log n) tokenizations); one index + one score per
+    // entry turns the pass into table lookups.
+    const index = buildRelevanceIndex(entries);
+    const scores = new Map(entries.map((entry) => [entry, relevanceScore(index, entry, q)]));
+    return [...entries].sort((a, b) => {
+        const relevanceDelta = (scores.get(b) ?? 0) - (scores.get(a) ?? 0);
+        if (relevanceDelta !== 0) {
+            return relevanceDelta;
         }
         const recencyDelta = recencyScore(b, now) - recencyScore(a, now);
         if (recencyDelta !== 0) {
@@ -190,22 +182,45 @@ export function formatSubagentSpecsSection(entries, query) {
  * the model a zero-cost overview of what exists so it can ask for full text
  * via `evolve_list` or `/evolve list`. The directory is appended after the
  * curated top-N injection sections and adds minimal tokens.
+ *
+ * 2026-08-22 throttle: the directory is CAPPED at {@link DEFAULT_DIRECTORY_LINES}
+ * lines (oldest-sorted stable order) with the remainder folded into a single
+ * counter line — an uncapped directory across a polluted global store was
+ * measured at ~2K chars of every build in every project.
  */
+export const DEFAULT_DIRECTORY_LINES = 15;
+/** How many of the variadic arrays are content-section kinds (prompt, subagent). */
+const CONTENT_SECTION_KINDS = 2;
 export function formatEntriesDirectory(...kindEntries) {
+    return formatEntriesDirectoryCapped(DEFAULT_DIRECTORY_LINES, ...kindEntries);
+}
+/** {@link formatEntriesDirectory} with an explicit cap (configurable). */
+export function formatEntriesDirectoryCapped(maxLines, ...kindEntries) {
     const allEntries = kindEntries.flat().filter((e) => !isArchived(e));
     if (allEntries.length === 0) {
         return "";
     }
-    // Skip the directory when it would be redundant (all entries already shown
-    // in the curated sections above — 6/kind cap means ≤6 entries total).
-    const totalCapped = kindEntries.reduce((sum, entries) => sum + Math.min(entries.filter((e) => !isArchived(e)).length, MAX_INJECTED_ENTRIES_PER_KIND), 0);
-    if (allEntries.length <= totalCapped) {
+    // Skip the directory only when EVERY entry is already content-visible.
+    // Only the first two arrays (prompt, subagent) have curated sections —
+    // memories and skills have NO content injection, so they are invisible
+    // unless the directory lists them (pre-2026-08-22 the redundancy check
+    // wrongly counted them as "already shown", hiding small stores entirely).
+    const contentVisible = kindEntries
+        .slice(0, CONTENT_SECTION_KINDS)
+        .reduce((sum, entries) => sum + Math.min(entries.filter((e) => !isArchived(e)).length, MAX_INJECTED_ENTRIES_PER_KIND), 0);
+    if (allEntries.length <= contentVisible) {
         return "";
     }
+    const sorted = [...allEntries].sort((a, b) => `${a.kind}:${a.id}`.localeCompare(`${b.kind}:${b.id}`));
     const lines = ["# Continual Harness — Entry Directory", "All entries (use evolve_list for full text of any entry):"];
-    for (const entry of allEntries.sort((a, b) => `${a.kind}:${a.id}`.localeCompare(`${b.kind}:${b.id}`))) {
+    const shown = sorted.slice(0, Math.max(maxLines, 1));
+    for (const entry of shown) {
         lines.push(`- [${entry.kind}:${entry.id}] ${entry.title}`);
     }
+    const hidden = sorted.length - shown.length;
+    if (hidden > 0) {
+        lines.push(`- …and ${hidden} more entries (evolve_list for the full index)`);
+    }
     return lines.join("\n");
 }
 /**
@@ -236,8 +251,13 @@ export function nearestLocalStateWithEntries(engine, agent) {
  * (relevance first, then recency; see {@link rankEntries}). Returns "" when
  * nothing is injectable — the prompt renderer then drops the section, so an
  * empty store adds zero tokens to every assembly.
+ *
+ * `opts.directoryLines` caps the entry-directory index (2026-08-22 throttle).
+ * Usage recording covers ALL kinds — memories and skills appear as directory
+ * lines, prompts/subagents as content — and is deduped per session so the
+ * counts read "how many sessions saw this", not "how many prompt builds".
  */
-export function entriesSectionText(engine, agent, query) {
+export function entriesSectionText(engine, agent, query, opts) {
     if (!agent) {
         return "";
     }
@@ -250,28 +270,41 @@ export function entriesSectionText(engine, agent, query) {
     // Build injected text and collect which entries were included (gap B1).
     const promptText = formatPromptEntriesSection(promptEntries, relevanceQuery);
     const subagentText = formatSubagentSpecsSection(subagentEntries, relevanceQuery);
-    const injectedKeys = [];
+    const injectedKeys = new Set();
     // Collect keys from the visible (ranked, capped) entries that actually appear.
     const visiblePrompt = promptEntries.filter((e) => !isArchived(e));
     const visibleSubagent = subagentEntries.filter((e) => !isArchived(e));
     for (const entry of rankEntries(visiblePrompt, relevanceQuery).slice(0, MAX_INJECTED_ENTRIES_PER_KIND)) {
-        injectedKeys.push(`prompt:${entry.id}`);
+        injectedKeys.add(`prompt:${entry.id}`);
     }
     for (const entry of rankEntries(visibleSubagent, relevanceQuery).slice(0, MAX_INJECTED_ENTRIES_PER_KIND)) {
-        injectedKeys.push(`subagent:${entry.id}`);
+        injectedKeys.add(`subagent:${entry.id}`);
+    }
+    // Gap B3: lightweight directory of ALL entries (id+title, one line each).
+    // Zero-cost index so the model knows what exists and can ask for full text.
+    const directoryText = formatEntriesDirectoryCapped(opts?.directoryLines ?? DEFAULT_DIRECTORY_LINES, Object.values(merged.entries.prompt), Object.values(merged.entries.memory), Object.values(merged.entries.skill), Object.values(merged.entries.subagent));
+    // Directory-visible keys count too: a memory's injection IS its directory
+    // line. Set semantics keep content-injected entries single-counted.
+    for (const kind of ["prompt", "memory", "skill", "subagent"]) {
+        for (const entry of Object.values(merged.entries[kind])) {
+            if (isArchived(entry))
+                continue;
+            const key = `${kind}:${entry.id}`;
+            if (injectedKeys.has(key) || directoryText.includes(`[${key}]`)) {
+                injectedKeys.add(key);
+            }
+        }
     }
     // Record usage durably (best-effort: failure never blocks injection).
-    if (injectedKeys.length > 0) {
+    // Deduped per session — see recordInjection.
+    if (injectedKeys.size > 0) {
         try {
-            recordInjection(engine.baseDir, injectedKeys);
+            recordInjection(engine.baseDir, [...injectedKeys], agent.id);
         }
         catch {
             // Usage recording is diagnostic; never interrupt the injection path.
         }
     }
-    // Gap B3: lightweight directory of ALL entries (id+title, one line each).
-    // Zero-cost index so the model knows what exists and can ask for full text.
-    const directoryText = formatEntriesDirectory(Object.values(merged.entries.prompt), Object.values(merged.entries.memory), Object.values(merged.entries.skill), Object.values(merged.entries.subagent));
     const parts = [promptText, subagentText, directoryText].filter((part) => part.length > 0);
     return parts.join("\n\n");
 }

package/lib/promotion.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Promotion policy: mechanical, code-enforced gates that decide whether a
+ * local entry MAY be promoted to the cross-session global store ("模型提议，
+ * 代码保证"). The LLM classification proposes; these guards dispose.
+ *
+ * Rationale (2026-08-22 store audit): the global store is shared by every
+ * project on one profile, so promoting project-scoped knowledge taxes every
+ * future session in every project. Measured failure modes:
+ * - absolute paths / session ids in promoted content (project-scoped),
+ * - near-duplicates of existing global entries re-promoted from later
+ *   sessions (title matching alone missed them),
+ * - one-line facts whose framing costs more than their content.
+ *
+ * Pure functions only — the callers (wrapup command, gate local-fate phase)
+ * supply the resolved {@link PromotionPolicy}.
+ */
+import type { HarnessEntry, HarnessState, RefinementKind } from "./types.js";
+export interface PromotionPolicy {
+    /** Content matching any pattern is project-scoped and stays local. */
+    blockPatterns: RegExp[];
+    /** Whole promotions below this content length stay local (chars). */
+    minPromoteChars: number;
+    /** Content overlap above this against a global entry = duplicate. */
+    maxContentOverlap: number;
+}
+export declare const DEFAULT_PROMOTION_POLICY: PromotionPolicy;
+/**
+ * Write-time conflict guard (R2): a global create whose similarity against an
+ * existing same-kind entry reaches this score is rejected outright — a
+ * near-duplicate adds zero information and the model should evolve_update
+ * the existing entry instead.
+ */
+export declare const CONFLICT_BLOCK_SCORE = 0.8;
+/** Similarity at/above this stamps {@link CONFLICT_HINT_KEY} but lets the write proceed. */
+export declare const CONFLICT_WARN_SCORE = 0.5;
+/**
+ * Build a policy from config values (schemastery strings compiled here so
+ * the config layer never touches RegExp). Invalid patterns are skipped —
+ * a broken user pattern must not disable the remaining guards.
+ */
+export declare function resolvePromotionPolicy(options: {
+    blockPatterns?: readonly string[] | undefined;
+    minPromoteChars?: number | undefined;
+    maxContentOverlap?: number | undefined;
+}): PromotionPolicy;
+/**
+ * First reason the content reads as project-scoped, or undefined when it
+ * looks portable. Returns the matched pattern source so skip reports stay
+ * explainable in reviews.jsonl rationales.
+ */
+export declare function projectScopedReason(content: string, policy: PromotionPolicy): string | undefined;
+/**
+ * Tokenize for cheap similarity: ASCII words plus CJK character bigrams
+ * (single CJK chars are too ambiguous; bigrams survive segmentation-free
+ * Chinese text). Lowercased; single-char ASCII tokens dropped as noise.
+ */
+export declare function normalizedTokens(text: string): Set<string>;
+/** Jaccard similarity of two texts' normalized token sets (0..1). */
+export declare function contentOverlap(a: string, b: string): number;
+export interface SimilarEntryHit {
+    id: string;
+    title: string;
+    score: number;
+}
+/**
+ * Human/LLM-readable description of a similarity hit, shared by the block
+ * error and the approval-question suffix so both surfaces explain the same
+ * way.
+ */
+export declare function buildConflictNotice(hit: SimilarEntryHit): string;
+/**
+ * The most similar entry of the list, above `minScore`. Title and content
+ * both feed the comparison (titles are short; content carries the real
+ * signal). Generic form of {@link mostSimilarGlobalEntry} — callers decide
+ * which corpus (global store, merged view) the candidates come from.
+ */
+export declare function mostSimilarEntry(entries: readonly HarnessEntry[], title: string, content: string, minScore: number): SimilarEntryHit | undefined;
+/**
+ * The most similar non-archived global entry of the same kind, above the
+ * policy threshold.
+ */
+export declare function mostSimilarGlobalEntry(globalState: HarnessState, kind: RefinementKind, title: string, content: string, policy: PromotionPolicy): SimilarEntryHit | undefined;
+//# sourceMappingURL=promotion.d.ts.map

package/lib/promotion.js

@@ -0,0 +1,127 @@
+import { isArchived } from "./types.js";
+/** Regex sources that mark content as project-scoped (never global). */
+const DEFAULT_BLOCK_PATTERNS = [
+    String.raw `\/(?:mnt|home|Users)\/`, // absolute POSIX paths: "/home/…", "/mnt/…", "/Users/…"
+    String.raw `\bsession-[0-9a-f]{8}\b`, // session-scoped identifiers
+    String.raw `~/\.dsh\b`, // user harness home references
+];
+export const DEFAULT_PROMOTION_POLICY = {
+    blockPatterns: DEFAULT_BLOCK_PATTERNS.map((source) => new RegExp(source, "i")),
+    minPromoteChars: 100,
+    maxContentOverlap: 0.6,
+};
+/**
+ * Write-time conflict guard (R2): a global create whose similarity against an
+ * existing same-kind entry reaches this score is rejected outright — a
+ * near-duplicate adds zero information and the model should evolve_update
+ * the existing entry instead.
+ */
+export const CONFLICT_BLOCK_SCORE = 0.8;
+/** Similarity at/above this stamps {@link CONFLICT_HINT_KEY} but lets the write proceed. */
+export const CONFLICT_WARN_SCORE = 0.5;
+/**
+ * Build a policy from config values (schemastery strings compiled here so
+ * the config layer never touches RegExp). Invalid patterns are skipped —
+ * a broken user pattern must not disable the remaining guards.
+ */
+export function resolvePromotionPolicy(options) {
+    const patterns = [...(options.blockPatterns ?? [])]
+        .map((source) => {
+        try {
+            return new RegExp(source, "i");
+        }
+        catch {
+            return undefined;
+        }
+    })
+        .filter((pattern) => pattern !== undefined);
+    return {
+        blockPatterns: patterns.length > 0 ? patterns : DEFAULT_PROMOTION_POLICY.blockPatterns,
+        minPromoteChars: options.minPromoteChars ?? DEFAULT_PROMOTION_POLICY.minPromoteChars,
+        maxContentOverlap: options.maxContentOverlap ?? DEFAULT_PROMOTION_POLICY.maxContentOverlap,
+    };
+}
+/**
+ * First reason the content reads as project-scoped, or undefined when it
+ * looks portable. Returns the matched pattern source so skip reports stay
+ * explainable in reviews.jsonl rationales.
+ */
+export function projectScopedReason(content, policy) {
+    for (const pattern of policy.blockPatterns) {
+        if (pattern.test(content)) {
+            return `project-scoped content (matches /${pattern.source}/)`;
+        }
+    }
+    return undefined;
+}
+/**
+ * Tokenize for cheap similarity: ASCII words plus CJK character bigrams
+ * (single CJK chars are too ambiguous; bigrams survive segmentation-free
+ * Chinese text). Lowercased; single-char ASCII tokens dropped as noise.
+ */
+export function normalizedTokens(text) {
+    const lowered = text.toLowerCase();
+    const tokens = new Set();
+    for (const match of lowered.matchAll(/[a-z0-9_]{2,}/g)) {
+        tokens.add(match[0] ?? "");
+    }
+    let previous;
+    for (const match of lowered.matchAll(/[\u3400-\u9fff]/g)) {
+        const char = match[0] ?? "";
+        if (previous !== undefined) {
+            tokens.add(`${previous}${char}`);
+        }
+        previous = char;
+    }
+    tokens.delete("");
+    return tokens;
+}
+/** Jaccard similarity of two texts' normalized token sets (0..1). */
+export function contentOverlap(a, b) {
+    const left = normalizedTokens(a);
+    const right = normalizedTokens(b);
+    if (left.size === 0 || right.size === 0) {
+        return 0;
+    }
+    let intersection = 0;
+    for (const token of left) {
+        if (right.has(token)) {
+            intersection += 1;
+        }
+    }
+    return intersection / (left.size + right.size - intersection);
+}
+/**
+ * Human/LLM-readable description of a similarity hit, shared by the block
+ * error and the approval-question suffix so both surfaces explain the same
+ * way.
+ */
+export function buildConflictNotice(hit) {
+    return `near-duplicate of ${hit.id} 「${hit.title}」 (similarity ${Math.round(hit.score * 100)}%)`;
+}
+/**
+ * The most similar entry of the list, above `minScore`. Title and content
+ * both feed the comparison (titles are short; content carries the real
+ * signal). Generic form of {@link mostSimilarGlobalEntry} — callers decide
+ * which corpus (global store, merged view) the candidates come from.
+ */
+export function mostSimilarEntry(entries, title, content, minScore) {
+    let best;
+    for (const other of entries) {
+        if (isArchived(other))
+            continue;
+        const score = Math.max(contentOverlap(title, other.title), contentOverlap(content, other.content));
+        if (score >= minScore && (best === undefined || score > best.score)) {
+            best = { id: other.id, title: other.title, score };
+        }
+    }
+    return best;
+}
+/**
+ * The most similar non-archived global entry of the same kind, above the
+ * policy threshold.
+ */
+export function mostSimilarGlobalEntry(globalState, kind, title, content, policy) {
+    return mostSimilarEntry(Object.values(globalState.entries[kind]), title, content, policy.maxContentOverlap);
+}
+//# sourceMappingURL=promotion.js.map

package/lib/search.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Retrieval primitives for injection ranking: CJK-aware tokenization plus a
+ * dependency-free, field-weighted BM25 scorer (ADR
+ * `implemented/feature/2026-08-24-cjk-bigram-bm25-retrieval.md`).
+ *
+ * Why this shape:
+ * - the old scorer tokenized whole CJK runs as single tokens compared for
+ *   exact equality, so any rewording of a Chinese query missed every entry
+ *   (「检索升级」 never matched 「全文检索方案」); overlapping character
+ *   bigrams — the standard cjk_bigram analyzer shape (Elasticsearch/
+ *   OpenSearch) — fix recall while keeping English behavior identical;
+ * - raw hit counts weighted common words and distinctive terms alike; BM25's
+ *   IDF separates them. We use Lucene's non-negative variant
+ *   `ln(1 + (N - df + 0.5) / (df + 0.5))` so EVERY matched token scores > 0
+ *   and unmatched documents score exactly 0, preserving rankEntries'
+ *   "relevant outranks recent-but-irrelevant" ordering invariant;
+ * - the index lifetime is one ranking call. Stores hold tens~hundreds of
+ *   entries (measured full rebuild + score: single-digit milliseconds), the
+ *   JSON state files stay the single source of truth, and npm installers
+ *   carry zero native/WASM baggage — better-sqlite3 FTS5, sql.js, MiniSearch,
+ *   FlexSearch were all evaluated and rejected (see the ADR's alternatives).
+ */
+import type { HarnessEntry } from "./types.js";
+/** BM25 term-saturation constant (Okapi k1). */
+export declare const BM25_K1 = 1.5;
+/** BM25 document-length normalization constant (Okapi b). */
+export declare const BM25_B = 0.75;
+/** Multiplier for title-field matches (title hits outweigh body hits 2:1). */
+export declare const TITLE_FIELD_WEIGHT = 2;
+/**
+ * Tokenize text for relevance scoring: lowercase ASCII runs become word
+ * tokens; CJK runs become overlapping character bigrams (a single-character
+ * run stays a unigram). Deterministic and allocation-light — safe to call per
+ * injection build.
+ *
+ * @param text Arbitrary entry or query text.
+ * @returns Tokens in occurrence order (duplicates kept; consumers dedupe when
+ *          order-independent weighting is wanted).
+ */
+export declare function tokenize(text: string): string[];
+/** Per-field corpus statistics for one ranking pass. */
+interface FieldStats {
+    /** Term-frequency map per document, positioned like {@link RelevanceIndex.positions}. */
+    readonly tfs: ReadonlyArray<ReadonlyMap<string, number>>;
+    /** Token count per document (this field's dl for BM25 length normalization). */
+    readonly lengths: ReadonlyArray<number>;
+    /** Document frequency per distinct term across the corpus. */
+    readonly df: ReadonlyMap<string, number>;
+    /** Average token count across the corpus (0 when empty). */
+    readonly averageLength: number;
+}
+/** Precomputed corpus statistics backing {@link relevanceScore}. */
+export interface RelevanceIndex {
+    /** Number of documents in the corpus. */
+    readonly size: number;
+    /** Identity positions: the exact entry objects the index was built from. */
+    readonly positions: ReadonlyMap<object, number>;
+    readonly titles: FieldStats;
+    readonly bodies: FieldStats;
+}
+/**
+ * Build the corpus statistics (per-field tf, df, average length) for one
+ * ranking call. Pure: the index reads its input snapshot and never mutates
+ * the entries.
+ *
+ * @param entries The candidate entries being ranked this call.
+ * @returns An index that answers {@link relevanceScore} for exactly these
+ *          entries (identity-keyed).
+ */
+export declare function buildRelevanceIndex(entries: readonly HarnessEntry[]): RelevanceIndex;
+/**
+ * Field-weighted BM25 score of `entry` against `query` within `index`:
+ * `TITLE_FIELD_WEIGHT × bm25(title) + bm25(body)`. Query tokens are deduped,
+ * so repeating a word adds no weight.
+ *
+ * @returns A score > 0 when at least one query token occurs in the entry
+ *          (either field), and exactly 0 otherwise — the property rankEntries'
+ *          relevance-first ordering relies on. Never throws; an entry outside
+ *          the index simply scores 0.
+ */
+export declare function relevanceScore(index: RelevanceIndex, entry: HarnessEntry, query: string): number;
+export {};
+//# sourceMappingURL=search.d.ts.map

package/lib/search.js

@@ -0,0 +1,136 @@
+/** BM25 term-saturation constant (Okapi k1). */
+export const BM25_K1 = 1.5;
+/** BM25 document-length normalization constant (Okapi b). */
+export const BM25_B = 0.75;
+/** Multiplier for title-field matches (title hits outweigh body hits 2:1). */
+export const TITLE_FIELD_WEIGHT = 2;
+/**
+ * One pass, two run shapes: ASCII alphanumeric words, or maximal CJK runs.
+ * Runs never mix scripts because the alternation splits at the boundary, so
+ * 「深色主题dark主题」 tokenizes as 深色/色主/主题, "dark", 主题.
+ */
+const TOKEN_RUN_PATTERN = /[a-z0-9]+|[\u4e00-\u9fff]+/g;
+const ASCII_RUN_PATTERN = /^[a-z0-9]+$/;
+/**
+ * Tokenize text for relevance scoring: lowercase ASCII runs become word
+ * tokens; CJK runs become overlapping character bigrams (a single-character
+ * run stays a unigram). Deterministic and allocation-light — safe to call per
+ * injection build.
+ *
+ * @param text Arbitrary entry or query text.
+ * @returns Tokens in occurrence order (duplicates kept; consumers dedupe when
+ *          order-independent weighting is wanted).
+ */
+export function tokenize(text) {
+    const tokens = [];
+    for (const match of text.toLowerCase().matchAll(TOKEN_RUN_PATTERN)) {
+        const run = match[0];
+        if (ASCII_RUN_PATTERN.test(run)) {
+            tokens.push(run);
+        }
+        else if (run.length === 1) {
+            tokens.push(run);
+        }
+        else {
+            for (let i = 0; i < run.length - 1; i += 1) {
+                tokens.push(run.slice(i, i + 2));
+            }
+        }
+    }
+    return tokens;
+}
+function buildFieldStats(fieldTokens) {
+    const tfs = fieldTokens.map((tokens) => {
+        const tf = new Map();
+        for (const token of tokens) {
+            tf.set(token, (tf.get(token) ?? 0) + 1);
+        }
+        return tf;
+    });
+    const df = new Map();
+    let totalLength = 0;
+    for (let position = 0; position < tfs.length; position += 1) {
+        const tf = tfs[position];
+        let length = 0;
+        for (const count of tf.values()) {
+            length += count;
+        }
+        totalLength += length;
+        for (const token of tf.keys()) {
+            df.set(token, (df.get(token) ?? 0) + 1);
+        }
+    }
+    return {
+        tfs,
+        lengths: fieldTokens.map((tokens) => tokens.length),
+        df,
+        averageLength: tfs.length === 0 ? 0 : totalLength / tfs.length,
+    };
+}
+/**
+ * Build the corpus statistics (per-field tf, df, average length) for one
+ * ranking call. Pure: the index reads its input snapshot and never mutates
+ * the entries.
+ *
+ * @param entries The candidate entries being ranked this call.
+ * @returns An index that answers {@link relevanceScore} for exactly these
+ *          entries (identity-keyed).
+ */
+export function buildRelevanceIndex(entries) {
+    const titles = [];
+    const bodies = [];
+    for (const entry of entries) {
+        titles.push(tokenize(entry.title));
+        bodies.push(tokenize(`${entry.content} ${entry.path}`));
+    }
+    return {
+        size: entries.length,
+        positions: new Map(entries.map((entry, position) => [entry, position])),
+        titles: buildFieldStats(titles),
+        bodies: buildFieldStats(bodies),
+    };
+}
+/** One field's BM25 contribution for a single term (0 when the term is absent). */
+function bm25Term(termFrequency, documentFrequency, corpusSize, docLength, averageLength) {
+    if (termFrequency <= 0 || documentFrequency <= 0) {
+        return 0;
+    }
+    const inverseDocumentFrequency = Math.log(1 + (corpusSize - documentFrequency + 0.5) / (documentFrequency + 0.5));
+    const normalization = averageLength > 0 ? 1 - BM25_B + BM25_B * (docLength / averageLength) : 1;
+    return inverseDocumentFrequency * ((termFrequency * (BM25_K1 + 1)) / (termFrequency + BM25_K1 * normalization));
+}
+/**
+ * Field-weighted BM25 score of `entry` against `query` within `index`:
+ * `TITLE_FIELD_WEIGHT × bm25(title) + bm25(body)`. Query tokens are deduped,
+ * so repeating a word adds no weight.
+ *
+ * @returns A score > 0 when at least one query token occurs in the entry
+ *          (either field), and exactly 0 otherwise — the property rankEntries'
+ *          relevance-first ordering relies on. Never throws; an entry outside
+ *          the index simply scores 0.
+ */
+export function relevanceScore(index, entry, query) {
+    const position = index.positions.get(entry);
+    if (position === undefined || index.size === 0) {
+        return 0;
+    }
+    const seen = new Set();
+    let titleScore = 0;
+    let bodyScore = 0;
+    for (const token of tokenize(query)) {
+        if (seen.has(token)) {
+            continue;
+        }
+        seen.add(token);
+        const titleDf = index.titles.df.get(token) ?? 0;
+        if (titleDf > 0) {
+            titleScore += bm25Term(index.titles.tfs[position]?.get(token) ?? 0, titleDf, index.size, index.titles.lengths[position] ?? 0, index.titles.averageLength);
+        }
+        const bodyDf = index.bodies.df.get(token) ?? 0;
+        if (bodyDf > 0) {
+            bodyScore += bm25Term(index.bodies.tfs[position]?.get(token) ?? 0, bodyDf, index.size, index.bodies.lengths[position] ?? 0, index.bodies.averageLength);
+        }
+    }
+    return TITLE_FIELD_WEIGHT * titleScore + bodyScore;
+}
+//# sourceMappingURL=search.js.map