@hraness/wordcell 0.21.2 → 0.22.0

package/dist/{index-0k2x4nn9.js → index-1tm7bgx7.js}

@@ -334,12 +334,24 @@ async function snapshot(rootInput, options, writeIndex) {
     ...options.maxNoteBytes === undefined ? {} : { maxNoteBytes: options.maxNoteBytes },
     ...options.maxTotalBytes === undefined ? {} : { maxTotalBytes: options.maxTotalBytes }
   });
-  const indexRevision = await readIndexRevision(root, indexPath, options.maxNoteBytes ?? MAX_NOTE_UTF8_BYTES);
-  const currentIndex = indexRevision.content;
-  const indexNote = parseNote(vaultIndexPath, currentIndex);
-  const catalogMode = parsedCatalogMode(options.catalogMode, "ScanVaultOptions.catalogMode") ?? declaredCatalogMode(indexNote) ?? "managed";
+  const configuredCatalogMode = parsedCatalogMode(options.catalogMode, "ScanVaultOptions.catalogMode");
+  let indexRevision;
+  try {
+    indexRevision = await readIndexRevision(root, indexPath, options.maxNoteBytes ?? MAX_NOTE_UTF8_BYTES);
+  } catch (error) {
+    const absent = error instanceof Error && "code" in error && error.code === "ENOENT";
+    if (!absent || options.index !== undefined || configuredCatalogMode === "managed" || notes.some((note) => note.path === vaultIndexPath))
+      throw error;
+    if (writeIndex) {
+      throw new Error("Refresh requires an index.md front door. Add one with kb_catalog: authored to preserve its contents, or catalog markers for a managed catalog. Search and graph commands can read this folder without one.", { cause: error });
+    }
+  }
+  const currentIndex = indexRevision?.content ?? "";
+  const catalogMode = configuredCatalogMode ?? (indexRevision === undefined ? "authored" : declaredCatalogMode(parseNote(vaultIndexPath, currentIndex)) ?? "managed");
   let index = "authored";
   if (catalogMode === "managed") {
+    if (indexRevision === undefined)
+      throw new Error("A managed catalog requires an existing index file.");
     const expectedIndex = replaceCatalog(currentIndex, renderCatalog(notes, catalogNoteId));
     const stale = currentIndex !== expectedIndex;
     index = stale ? "stale" : "current";

package/dist/{index-vdw5yjfa.js → index-2hf27mws.js}

@@ -1,8 +1,4 @@
 // @bun
-import {
-  UntrustedContentBudgetError,
-  createUntrustedToolResult
-} from "./index-4j3tt0c3.js";
 import {
   expandSearchRequest,
   parseSearchRules,
@@ -11,20 +7,26 @@ import {
 import {
   openSemanticSearchSession,
   recommendedEmbeddingModel
-} from "./index-115b07ap.js";
+} from "./index-hcw140eb.js";
+import {
+  UntrustedContentBudgetError,
+  createUntrustedToolResult
+} from "./index-4j3tt0c3.js";
+import {
+  MAX_RERANK_CANDIDATES,
+  MAX_RERANK_SNIPPET_BYTES,
+  applyRerank
+} from "./index-j70m75wd.js";
 import {
   percolateWithGraph
-} from "./index-bgfzwt4h.js";
+} from "./index-t2bs9xdr.js";
 import {
   openGraphAuthority
-} from "./index-bcfn9xah.js";
+} from "./index-famy7fhs.js";
 import {
   validateGraphPercolationOptions,
   validateGraphQueryRequest
 } from "./index-11621h23.js";
-import {
-  scanVault
-} from "./index-0k2x4nn9.js";
 import {
   GitHistoryError,
   gitHistoryForNotes,
@@ -39,14 +41,17 @@ import {
   fuseRankedCandidates,
   searchExactVault,
   validateSearchQuery
-} from "./index-why54q5p.js";
+} from "./index-gm9t95d9.js";
 import {
-  queryVault
-} from "./index-48pz4jpc.js";
+  scanVault
+} from "./index-1tm7bgx7.js";
 import {
   NavigationBudgetError,
   navigateLinks
 } from "./index-d13v9ckt.js";
+import {
+  queryVault
+} from "./index-48pz4jpc.js";
 import {
   lookupNote
 } from "./index-ekpwvbra.js";
@@ -83,6 +88,9 @@ function noteOrThrow(notes, query) {
 function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
+function searchEvidenceRank(item) {
+  return item.kind === "rerank" ? item.rerankRank : item.rank;
+}
 function utf8Prefix(value, maximumBytes) {
   let bytes = 0;
   let end = 0;
@@ -203,6 +211,29 @@ function checkedHistoryRequest(value) {
 function validateKnowledgeBaseSearchHistory(value) {
   checkedHistoryRequest(value);
 }
+function checkedRerankRequest(value, rerankers) {
+  if (value === undefined)
+    return { enabled: false };
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new TypeError("Search rerank options must be an options object.");
+  }
+  const options = value;
+  if (options.engine !== "typesafe") {
+    throw new Error('Search rerank engine must be "typesafe".');
+  }
+  const reranker = rerankers?.find((entry) => entry.id === options.engine);
+  if (reranker === undefined) {
+    throw new Error(`Search rerank engine ${JSON.stringify(options.engine)} is not available in this session.`);
+  }
+  const limit = options.limit ?? MAX_RERANK_CANDIDATES;
+  if (!Number.isSafeInteger(limit) || limit < 2 || limit > MAX_RERANK_CANDIDATES) {
+    throw new RangeError(`Search rerank limit must be an integer from 2 through ${MAX_RERANK_CANDIDATES}.`);
+  }
+  if (options.signal !== undefined && !(options.signal instanceof AbortSignal)) {
+    throw new TypeError("Search rerank signal must be an AbortSignal.");
+  }
+  return { enabled: true, limit, reranker, ...options.signal === undefined ? {} : { signal: options.signal } };
+}
 async function openKnowledgeBase(options, dependencies = {}) {
   const searchRules = options.searchRules === undefined ? null : parseSearchRules(options.searchRules);
   if (options.embeddingModelFile !== undefined && options.embeddingModelLease !== undefined) {
@@ -322,6 +353,7 @@ async function openKnowledgeBase(options, dependencies = {}) {
     }
     const historyRequest = checkedHistoryRequest(effectiveOptions.history);
     const graphOptions = checkedGraphOptions(effectiveOptions.graph);
+    const rerankRequest = checkedRerankRequest(effectiveOptions.rerank, dependencies.rerankers);
     const explicitSeeds = graphOptions === null ? [] : resolvedGraphSeeds(snapshot.notes, graphOptions.related);
     const historyNoteLimit = historyRequest.enabled ? historyRequest.noteLimit : null;
     const allowedIds = new Set(queryVault(snapshot.notes, snapshot.analysis, {
@@ -415,7 +447,8 @@ async function openKnowledgeBase(options, dependencies = {}) {
     const fused = fuseRankedCandidates(lanes);
     const ordered = fused.toSorted((left, right) => Number(exactById.get(right.id)?.hit.identity ?? false) - Number(exactById.get(left.id)?.hit.identity ?? false) || left.rank - right.rank || left.id.localeCompare(right.id));
     const shouldApplyPriority = ordering === "priority-then-relevance" && searchRules !== null && searchRules.priorityRules.length > 0;
-    const relevanceResults = (shouldApplyPriority ? ordered : ordered.slice(0, limit)).map((candidate, index) => {
+    const resultWindow = rerankRequest.enabled ? Math.max(limit, rerankRequest.limit) : limit;
+    const relevanceResults = (shouldApplyPriority ? ordered : ordered.slice(0, resultWindow)).map((candidate, index) => {
       const note = notesById.get(candidate.id);
       if (note === undefined) {
         throw new Error(`Fused retrieval returned unknown note ${JSON.stringify(candidate.id)}.`);
@@ -458,10 +491,67 @@ async function openKnowledgeBase(options, dependencies = {}) {
         contributions: candidate.contributions
       };
     });
+    let rankedResults = relevanceResults;
+    if (rerankRequest.enabled) {
+      const candidates = relevanceResults.slice(0, rerankRequest.limit).map((hit, index) => ({
+        id: hit.id,
+        baselineRank: index + 1,
+        score: hit.score,
+        title: hit.title,
+        path: hit.path,
+        snippet: utf8Prefix(hit.snippet, MAX_RERANK_SNIPPET_BYTES).value
+      }));
+      const outcome = await Promise.resolve().then(() => rerankRequest.reranker.rerank({
+        query: effectiveQuery,
+        candidates,
+        ...rerankRequest.signal === undefined ? {} : { signal: rerankRequest.signal }
+      })).then((result) => ({ ok: true, result }), (error) => ({ ok: false, error }));
+      const applied = applyRerank(relevanceResults, outcome.ok ? outcome.result : {
+        status: "failed",
+        message: "Rerank engine failed before returning a result."
+      }, candidates.map(({ id }) => id));
+      rankedResults = applied.hits.map((hit) => {
+        const placement = applied.placements.get(hit.id);
+        if (placement === undefined)
+          return hit;
+        return {
+          ...hit,
+          evidence: [
+            ...hit.evidence,
+            {
+              kind: "rerank",
+              engine: rerankRequest.reranker.id,
+              baselineRank: placement.baselineRank,
+              rerankRank: placement.rerankRank,
+              ...placement.probability === undefined ? {} : { probability: placement.probability }
+            }
+          ]
+        };
+      });
+      const resultDetails = {
+        ...applied.result.model === undefined ? {} : { model: applied.result.model },
+        ...applied.result.usage === undefined ? {} : { usage: applied.result.usage },
+        ...applied.result.accounting === undefined ? {} : { accounting: applied.result.accounting }
+      };
+      const details = [];
+      if (resultDetails.model !== undefined)
+        details.push(resultDetails.model);
+      if (resultDetails.usage !== undefined) {
+        details.push(`${(resultDetails.usage.inputTokens ?? 0).toLocaleString("en-US")} input tokens, ` + `${(resultDetails.usage.outputTokens ?? 0).toLocaleString("en-US")} output tokens`);
+      }
+      const rerankMessage = applied.status === "ready" ? details.length === 0 ? undefined : details.join("; ") : applied.message ?? "Rerank engine did not return a result.";
+      diagnostics.push({
+        lane: "rerank",
+        ...Object.keys(resultDetails).length === 0 ? {} : { rerank: resultDetails },
+        status: applied.status === "failed" ? "degraded" : applied.status,
+        results: applied.status === "ready" ? applied.placements.size : 0,
+        ...rerankMessage === undefined ? {} : { message: rerankMessage }
+      });
+    }
     let priorityTrace = null;
-    let results = relevanceResults.slice(0, limit);
+    let results = rankedResults.slice(0, limit);
     if (shouldApplyPriority && searchRules !== null) {
-      const prioritized = prioritizeSearchHits(relevanceResults, searchRules, {
+      const prioritized = prioritizeSearchHits(rankedResults, searchRules, {
         ...options.vaultId === undefined ? {} : { vaultId: options.vaultId }
       });
       const selectedHits = prioritized.hits.slice(0, limit);
@@ -696,7 +786,7 @@ Evidence: `))
         return writer.result();
       if (!append("#"))
         return writer.result();
-      if (!append(String(item.rank)))
+      if (!append(String(searchEvidenceRank(item))))
         return writer.result();
     }
     if (!append(`
@@ -1015,4 +1105,4 @@ function packUntrustedSearchContext(result, options = {}) {
   });
 }
 
-export { MAX_SEARCH_RESULTS, MAX_SEARCH_CANDIDATES, DEFAULT_SEARCH_RESULTS, MIN_UNTRUSTED_CONTEXT_BYTES, MAX_SEARCH_RELATED_SEEDS, MAX_SEARCH_NOTE_REFERENCE_BYTES, validateKnowledgeBaseSearchHistory, openKnowledgeBase, packSearchContext, packUntrustedSearchContext };
+export { MAX_SEARCH_RESULTS, MAX_SEARCH_CANDIDATES, DEFAULT_SEARCH_RESULTS, MIN_UNTRUSTED_CONTEXT_BYTES, MAX_SEARCH_RELATED_SEEDS, MAX_SEARCH_NOTE_REFERENCE_BYTES, searchEvidenceRank, validateKnowledgeBaseSearchHistory, openKnowledgeBase, packSearchContext, packUntrustedSearchContext };

package/dist/index-3agn8scn.js

@@ -0,0 +1,306 @@
+// @bun
+// src/publish-search.ts
+var PUBLISH_FIELD_TITLE = 1;
+var PUBLISH_FIELD_ALIAS = 2;
+var PUBLISH_FIELD_PATH = 4;
+var PUBLISH_FIELD_TAG = 8;
+var PUBLISH_FIELD_METADATA = 16;
+var PUBLISH_FIELD_CONTENT = 32;
+var PUBLISH_SEARCH_WEIGHTS_V1 = Object.freeze({
+  identity: Object.freeze({ title: 1000, alias: 950, path: 900 }),
+  phrase: Object.freeze({
+    title: 400,
+    alias: 350,
+    path: 300,
+    tag: 250,
+    content: 150,
+    metadata: 100
+  }),
+  term: Object.freeze({
+    title: 40,
+    alias: 35,
+    path: 30,
+    tag: 30,
+    metadata: 10,
+    content: 5
+  }),
+  coverage: 100
+});
+var MAX_PUBLISH_QUERY_BYTES = 16 * 1024;
+var MAX_PUBLISH_QUERY_TERMS = 64;
+var MAX_PUBLISH_PREFIX_EXPANSIONS = 32;
+var TERM_PATTERN = /[\p{L}\p{N}][\p{L}\p{N}._/-]*/gu;
+var textEncoder = new TextEncoder;
+function publishUtf8Bytes(value) {
+  return textEncoder.encode(value).byteLength;
+}
+function publishNormalize(value) {
+  return value.normalize("NFC").toLocaleLowerCase("en-US");
+}
+var MAX_PUBLISH_QUERY_FILTERS = 8;
+var FILTER_PATTERN = /^(tag|type|path):(\S+)$/u;
+function trimSlashes(value) {
+  let start = 0;
+  let end = value.length;
+  while (start < end && value.charCodeAt(start) === 47)
+    start += 1;
+  while (end > start && value.charCodeAt(end - 1) === 47)
+    end -= 1;
+  return value.slice(start, end);
+}
+function publishQueryParts(raw) {
+  const tags = [];
+  const types = [];
+  const paths = [];
+  const rest = [];
+  for (const token of raw.split(/\s+/u)) {
+    if (token === "")
+      continue;
+    const match = FILTER_PATTERN.exec(token);
+    const value = match === null ? "" : publishNormalize(trimSlashes(match[2] ?? ""));
+    if (match === null || value === "") {
+      rest.push(token);
+      continue;
+    }
+    const bucket = match[1] === "tag" ? tags : match[1] === "type" ? types : paths;
+    if (bucket.length < MAX_PUBLISH_QUERY_FILTERS)
+      bucket.push(value);
+    else
+      rest.push(token);
+  }
+  return { filters: { tags, types, paths }, text: rest.join(" ") };
+}
+function publishDocMatchesFilters(doc, filters, type) {
+  if (filters.tags.length > 0) {
+    const tags = doc.f.g === "" ? [] : doc.f.g.split(`
+`);
+    for (const tag of filters.tags) {
+      if (!tags.includes(publishNormalize(tag)))
+        return false;
+    }
+  }
+  if (filters.types.length > 0) {
+    const docType = publishNormalize(type === undefined || type === "" ? "note" : type);
+    for (const wanted of filters.types) {
+      if (docType !== publishNormalize(wanted))
+        return false;
+    }
+  }
+  if (filters.paths.length > 0) {
+    const slug = publishNormalize(doc.s);
+    const fields = doc.f.p === "" ? [] : doc.f.p.split(`
+`);
+    for (const value of filters.paths) {
+      const prefix = publishNormalize(trimSlashes(value));
+      const boundary = `${prefix}/`;
+      const inSlug = slug === prefix || slug.startsWith(boundary);
+      const inFields = fields.some((line) => line === prefix || line.startsWith(boundary));
+      if (!inSlug && !inFields)
+        return false;
+    }
+  }
+  return true;
+}
+var MAX_PUBLISH_MARK_RANGES = 32;
+function publishMarkRanges(text, terms, maximum = MAX_PUBLISH_MARK_RANGES) {
+  const display = text.normalize("NFC");
+  let normalized = "";
+  const map = [];
+  for (let offset = 0;offset < display.length; ) {
+    const point = display.codePointAt(offset) ?? 0;
+    const character = String.fromCodePoint(point);
+    const lowered = character.toLocaleLowerCase("en-US");
+    for (let index = 0;index < lowered.length; index += 1)
+      map.push(offset);
+    normalized += lowered;
+    offset += character.length;
+  }
+  const ranges = [];
+  for (const term of terms) {
+    const needle = publishNormalize(term);
+    if (needle === "")
+      continue;
+    let from = 0;
+    while (ranges.length < maximum) {
+      const hit = normalized.indexOf(needle, from);
+      if (hit === -1)
+        break;
+      const start = map[hit] ?? 0;
+      const tail = hit + needle.length;
+      const end = tail >= map.length ? display.length : map[tail] ?? display.length;
+      if (end > start)
+        ranges.push({ start, end });
+      from = tail === 0 ? 1 : tail;
+    }
+    if (ranges.length >= maximum)
+      break;
+  }
+  ranges.sort((left, right) => left.start - right.start || left.end - right.end);
+  const merged = [];
+  for (const range of ranges) {
+    const last = merged[merged.length - 1];
+    if (last !== undefined && range.start <= last.end) {
+      merged[merged.length - 1] = { start: last.start, end: Math.max(last.end, range.end) };
+    } else {
+      merged.push(range);
+    }
+  }
+  return merged;
+}
+function publishQuery(value) {
+  if (typeof value !== "string") {
+    throw new TypeError("Search query must be a string.");
+  }
+  if (publishUtf8Bytes(value) > MAX_PUBLISH_QUERY_BYTES) {
+    throw new RangeError(`Search query exceeds the ${MAX_PUBLISH_QUERY_BYTES}-byte limit.`);
+  }
+  const raw = value.trim();
+  const normalized = publishNormalize(raw);
+  const terms = [];
+  const seen = new Set;
+  for (const match of normalized.matchAll(TERM_PATTERN)) {
+    if (seen.has(match[0]))
+      continue;
+    seen.add(match[0]);
+    terms.push(match[0]);
+    if (terms.length > MAX_PUBLISH_QUERY_TERMS) {
+      throw new RangeError(`Search query may contain at most ${MAX_PUBLISH_QUERY_TERMS} unique normalized terms.`);
+    }
+  }
+  return { raw, normalized, terms };
+}
+function publishShardName(term) {
+  let hash = 2166136261;
+  for (let index = 0;index < term.length; index += 1) {
+    hash ^= term.charCodeAt(index);
+    hash = Math.imul(hash, 16777619) >>> 0;
+  }
+  return (hash >>> 24).toString(16).padStart(2, "0");
+}
+function publishPrefixTerms(dictionary, prefix, maximum = MAX_PUBLISH_PREFIX_EXPANSIONS) {
+  if (prefix === "")
+    return [];
+  let low = 0;
+  let high = dictionary.length;
+  while (low < high) {
+    const middle = low + high >> 1;
+    if ((dictionary[middle] ?? "") < prefix)
+      low = middle + 1;
+    else
+      high = middle;
+  }
+  const found = [];
+  for (let index = low;index < dictionary.length && found.length < maximum; index += 1) {
+    const term = dictionary[index] ?? "";
+    if (!term.startsWith(prefix))
+      break;
+    found.push(term);
+  }
+  return found;
+}
+function fieldMasks(term, fields) {
+  let mask = 0;
+  if (fields.t.includes(term))
+    mask |= PUBLISH_FIELD_TITLE;
+  if (fields.a.includes(term))
+    mask |= PUBLISH_FIELD_ALIAS;
+  if (fields.p.includes(term))
+    mask |= PUBLISH_FIELD_PATH;
+  if (fields.g.includes(term))
+    mask |= PUBLISH_FIELD_TAG;
+  if (fields.m.includes(term))
+    mask |= PUBLISH_FIELD_METADATA;
+  return mask;
+}
+function scorePublishDocument(input, query) {
+  const fields = input.doc.f;
+  const phrase = query.normalized;
+  const weights = PUBLISH_SEARCH_WEIGHTS_V1;
+  let score = 0;
+  let identity = false;
+  let phraseMatched = false;
+  if (phrase !== "") {
+    if (fields.t === phrase) {
+      identity = true;
+      score += weights.identity.title;
+    }
+    if (fields.a.split(`
+`).includes(phrase)) {
+      identity = true;
+      score += weights.identity.alias;
+    }
+    if (fields.p.split(`
+`).includes(phrase)) {
+      identity = true;
+      score += weights.identity.path;
+    }
+    const phraseFields = [
+      [PUBLISH_FIELD_TITLE, fields.t, weights.phrase.title],
+      [PUBLISH_FIELD_ALIAS, fields.a, weights.phrase.alias],
+      [PUBLISH_FIELD_PATH, fields.p, weights.phrase.path],
+      [PUBLISH_FIELD_TAG, fields.g, weights.phrase.tag],
+      [PUBLISH_FIELD_METADATA, fields.m, weights.phrase.metadata]
+    ];
+    for (const [, value, weight] of phraseFields) {
+      if (value.includes(phrase)) {
+        phraseMatched = true;
+        score += weight;
+      }
+    }
+    if (input.contentText !== undefined && input.contentText.includes(phrase)) {
+      phraseMatched = true;
+      score += weights.phrase.content;
+    }
+  }
+  let matchedTerms = 0;
+  for (const term of query.terms) {
+    const mask = fieldMasks(term, fields) | (input.contentTerms.has(term) ? PUBLISH_FIELD_CONTENT : 0);
+    if (mask === 0)
+      continue;
+    matchedTerms += 1;
+    if (mask & PUBLISH_FIELD_TITLE)
+      score += weights.term.title;
+    if (mask & PUBLISH_FIELD_ALIAS)
+      score += weights.term.alias;
+    if (mask & PUBLISH_FIELD_PATH)
+      score += weights.term.path;
+    if (mask & PUBLISH_FIELD_TAG)
+      score += weights.term.tag;
+    if (mask & PUBLISH_FIELD_METADATA)
+      score += weights.term.metadata;
+    if (mask & PUBLISH_FIELD_CONTENT)
+      score += weights.term.content;
+  }
+  const requiredTerms = query.terms.length <= 1 ? query.terms.length : Math.min(3, Math.ceil(query.terms.length / 2));
+  if (!identity && !phraseMatched && matchedTerms < requiredTerms)
+    return null;
+  if (query.terms.length > 0) {
+    score += Math.round(matchedTerms / query.terms.length * weights.coverage);
+  }
+  if (score === 0)
+    return null;
+  return { score, identity, phraseMatched, matchedTerms };
+}
+function comparePublishScores(left, right) {
+  return Number(right.identity) - Number(left.identity) || right.score - left.score || left.i - right.i;
+}
+function publishSnippet(text, query, fallback, windowBytes = 160) {
+  const normalized = publishNormalize(text);
+  let offset = query.normalized === "" ? -1 : normalized.indexOf(query.normalized);
+  if (offset < 0) {
+    for (const term of query.terms) {
+      offset = normalized.indexOf(term);
+      if (offset >= 0)
+        break;
+    }
+  }
+  if (offset < 0)
+    return fallback;
+  const half = Math.floor(windowBytes / 2);
+  const start = Math.max(0, offset - half);
+  const end = Math.min(text.length, offset + half);
+  const snippet = text.slice(start, end).replace(/\s+/gu, " ").trim();
+  return `${start > 0 ? "\u2026" : ""}${snippet}${end < text.length ? "\u2026" : ""}`;
+}
+
+export { PUBLISH_FIELD_TITLE, PUBLISH_FIELD_ALIAS, PUBLISH_FIELD_PATH, PUBLISH_FIELD_TAG, PUBLISH_FIELD_METADATA, PUBLISH_FIELD_CONTENT, PUBLISH_SEARCH_WEIGHTS_V1, MAX_PUBLISH_QUERY_BYTES, MAX_PUBLISH_QUERY_TERMS, MAX_PUBLISH_PREFIX_EXPANSIONS, publishUtf8Bytes, publishNormalize, MAX_PUBLISH_QUERY_FILTERS, publishQueryParts, publishDocMatchesFilters, MAX_PUBLISH_MARK_RANGES, publishMarkRanges, publishQuery, publishShardName, publishPrefixTerms, scorePublishDocument, comparePublishScores, publishSnippet };