@hraness/wordcell 0.22.3 → 0.22.5

package/src/publish-model.ts

@@ -47,7 +47,7 @@ export const WORDCELL_SITE_LIMITS_V1 = Object.freeze({
    * exact lane's substring semantics; larger corpora use postings shards.
    */
   inlineTotalBytes: 2 * 1_024 * 1_024,
-  /** Normalized content text retained per note payload for hydration. */
+  /** Normalized readable content retained per note payload for snippet hydration. */
   noteTextBytes: 32 * 1_024,
   /** Site title and description. */
   titleBytes: 512,
@@ -185,7 +185,7 @@ export type WordcellSiteNoteV1 = {
   readonly type?: string;
   readonly tags: readonly string[];
   readonly summary: string;
-  /** Bounded normalized content text for search hydration. */
+  /** Bounded normalized readable content for snippet hydration, not ranking. */
   readonly text: string;
   readonly textTruncated: boolean;
   readonly links: readonly WordcellSiteNoteLinkV1[];

package/src/publish-reader/reader.ts

@@ -321,7 +321,9 @@ function resultHref(base: string, slug: string): string {
 
 function snippetFor(index: SiteIndex, doc: WordcellSiteDocV1, query: PublishQuery): string {
   const hydrated = index.notes.get(doc.s);
-  const text = hydrated?.text ?? doc.x;
+  // doc.x is a normalized ranking basis, not display text. The eager preview
+  // remains readable while the existing bounded hydration fetch is pending.
+  const text = hydrated?.text;
   return text === undefined ? doc.p : publishSnippet(text, query, doc.p);
 }
 

package/src/publish-search.ts

@@ -392,8 +392,10 @@ export function comparePublishScores(
 }
 
 /**
- * Bounded context window around the first query match in normalized text.
- * Mirrors `exactSnippet`: collapse whitespace, ellipsize edges.
+ * Bounded display context around the first normalized query match. Match
+ * offsets are mapped back to NFC display text before taking a UTF-8 byte
+ * window, so case expansion and supplementary characters cannot split it.
+ * The fallback retains the separate document-preview byte contract.
  */
 export function publishSnippet(
   text: string,
@@ -401,18 +403,47 @@ export function publishSnippet(
   fallback: string,
   windowBytes = 160,
 ): string {
-  const normalized = publishNormalize(text);
-  let offset = query.normalized === "" ? -1 : normalized.indexOf(query.normalized);
-  if (offset < 0) {
+  if (!Number.isSafeInteger(windowBytes) || windowBytes < 0) throw new RangeError("Snippet window must be a nonnegative safe byte count.");
+  const display = text.normalize("NFC");
+  const normalized = publishNormalize(display);
+  let match = query.normalized === "" ? -1 : normalized.indexOf(query.normalized);
+  if (match < 0) {
     for (const term of query.terms) {
-      offset = normalized.indexOf(term);
-      if (offset >= 0) break;
+      match = normalized.indexOf(term);
+      if (match >= 0) break;
     }
   }
-  if (offset < 0) return fallback;
-  const half = Math.floor(windowBytes / 2);
-  const start = Math.max(0, offset - half);
-  const end = Math.min(text.length, offset + half);
-  const snippet = text.slice(start, end).replace(/\s+/gu, " ").trim();
-  return `${start > 0 ? "…" : ""}${snippet}${end < text.length ? "…" : ""}`;
+  if (match < 0) return fallback;
+  let offset = 0, normalizedOffset = 0;
+  for (const character of display) {
+    const width = character.toLocaleLowerCase("en-US").length;
+    if (normalizedOffset + width > match) break;
+    normalizedOffset += width;
+    offset += character.length;
+  }
+  const edge = windowBytes >= 6 ? "…" : "";
+  const budget = windowBytes - 2 * publishUtf8Bytes(edge);
+  const previous = (end: number): number => {
+    const last = display.charCodeAt(end - 1);
+    return end > 1 && last >= 0xdc00 && last <= 0xdfff && display.charCodeAt(end - 2) >= 0xd800 && display.charCodeAt(end - 2) <= 0xdbff ? end - 2 : end - 1;
+  };
+  let start = offset, end = offset, used = 0;
+  while (start > 0) {
+    const before = previous(start), bytes = publishUtf8Bytes(display.slice(before, start));
+    if (used + bytes > Math.floor(budget / 2)) break;
+    start = before; used += bytes;
+  }
+  while (end < display.length) {
+    const character = String.fromCodePoint(display.codePointAt(end) ?? 0), bytes = publishUtf8Bytes(character);
+    if (used + bytes > budget) break;
+    end += character.length; used += bytes;
+  }
+  // Near the end, spend the remaining budget on earlier whole characters.
+  while (start > 0) {
+    const before = previous(start), bytes = publishUtf8Bytes(display.slice(before, start));
+    if (used + bytes > budget) break;
+    start = before; used += bytes;
+  }
+  const snippet = display.slice(start, end).replace(/\s+/gu, " ").trim();
+  return `${start > 0 ? edge : ""}${snippet}${end < display.length ? edge : ""}`;
 }

package/src/publish.ts

@@ -16,6 +16,7 @@ import {
 } from "./graph.js";
 import { buildSiteIndex } from "./publish-index.js";
 import {
+  projectMarkdownText,
   renderMarkdownToHtml,
   type PublishRenderContext,
 } from "./publish-markdown.js";
@@ -307,7 +308,9 @@ export async function projectVault(
   let anyTextTruncated = false;
   for (const note of selection.notes) {
     const slug = slugFor(note.id);
-    const text = boundedNormalized(note.searchableText, WORDCELL_SITE_LIMITS_V1.noteTextBytes);
+    // Display grammar owns citations before normalization can fold distinct
+    // labels together. Ranking continues to use untouched searchableText.
+    const text = boundedNormalized(projectMarkdownText(note.content).text, WORDCELL_SITE_LIMITS_V1.noteTextBytes);
     anyTextTruncated ||= text.truncated;
     const payload: WordcellSiteNoteV1 = {
       format: WORDCELL_SITE_NOTE_FORMAT_V1,

package/src/vault.ts

@@ -13,6 +13,7 @@ import { basename, dirname, join, relative, resolve, sep } from "node:path";
 
 import {
   analyzeVault,
+  analyzeVaultComplete,
   isCanonicalNoteId,
   lookupNote,
   MAX_ANALYZED_NOTES,
@@ -21,6 +22,7 @@ import {
   renderCatalog,
   replaceCatalog,
   type AnalyzeVaultOptions,
+  type CompleteAnalyzeVaultOptions,
   type Note,
   type VaultAnalysis,
 } from "./graph.js";
@@ -90,6 +92,8 @@ export type ScanVaultOptions = Omit<AnalyzeVaultOptions, "mentionScope"> & {
   readonly mentionScope?: string | false;
 };
 
+export type CompleteScanVaultOptions = ScanVaultOptions & Pick<CompleteAnalyzeVaultOptions, "mentionIndexLimits">;
+
 export async function markdownFiles(
   directory: string,
   ignoredDirectories: ReadonlySet<string> = defaultIgnoredDirectories,
@@ -520,8 +524,9 @@ async function atomicReplace(
 
 async function snapshot(
   rootInput: string,
-  options: ScanVaultOptions,
+  options: CompleteScanVaultOptions,
   writeIndex: boolean,
+  completeMentions = false,
 ): Promise<VaultSnapshot> {
   const requestedRoot = resolve(rootInput);
   const root = await realpath(requestedRoot);
@@ -613,7 +618,7 @@ async function snapshot(
     catalogMode,
     index,
     notes,
-    analysis: analyzeVault(notes, {
+    analysis: (completeMentions ? analyzeVaultComplete : analyzeVault)(notes, {
       catalogNoteId,
       ...(options.includeInSuggestions === undefined
         ? {}
@@ -631,6 +636,9 @@ async function snapshot(
       ...(options.maxMentions === undefined
         ? {}
         : { maxMentions: options.maxMentions }),
+      ...(!completeMentions || options.mentionIndexLimits === undefined
+        ? {}
+        : { mentionIndexLimits: options.mentionIndexLimits }),
     }),
   };
 }
@@ -648,3 +656,19 @@ export async function refreshVault(
 ): Promise<VaultSnapshot> {
   return snapshot(root, options, true);
 }
+
+/** Read every admitted note and compute the full graph through the phrase index. */
+export async function scanVaultComplete(
+  root = ".",
+  options: CompleteScanVaultOptions = {},
+): Promise<VaultSnapshot> {
+  return snapshot(root, options, false, true);
+}
+
+/** Refresh the configured catalog while retaining complete indexed analysis. */
+export async function refreshVaultComplete(
+  root = ".",
+  options: CompleteScanVaultOptions = {},
+): Promise<VaultSnapshot> {
+  return snapshot(root, options, true, true);
+}