@hraness/wordcell 0.21.3 → 0.22.1

package/dist/index-j70m75wd.js

@@ -0,0 +1,217 @@
+// @bun
+// src/rerank.ts
+var MAX_RERANK_CANDIDATES = 25;
+var MAX_RERANK_SNIPPET_BYTES = 512;
+var MAX_RERANK_STATE_BYTES = 24 * 1024;
+var MALFORMED_RERANK_RESULT_MESSAGE = "Rerank engine returned a malformed result.";
+var MAX_RERANK_DIAGNOSTIC_BYTES = 512;
+var MAX_RERANK_MODEL_BYTES = 128;
+var MAX_RERANK_TOKEN_COUNT = 1e9;
+function dataRecord(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value))
+    return null;
+  const prototype = Object.getPrototypeOf(value);
+  if (prototype !== Object.prototype && prototype !== null)
+    return null;
+  const keys = Reflect.ownKeys(value);
+  if (keys.length > MAX_RERANK_CANDIDATES)
+    return null;
+  const output = Object.create(null);
+  for (const key of keys) {
+    if (typeof key !== "string")
+      return null;
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    if (descriptor === undefined || !("value" in descriptor) || !descriptor.enumerable)
+      return null;
+    output[key] = descriptor.value;
+  }
+  return output;
+}
+function hasOnlyKeys(value, allowed) {
+  return Object.keys(value).every((key) => allowed.includes(key));
+}
+function boundedDiagnostic(value) {
+  return typeof value === "string" && value !== "" && Buffer.byteLength(value, "utf8") <= MAX_RERANK_DIAGNOSTIC_BYTES && !/[\u0000-\u001f\u007f]/u.test(value);
+}
+function tokenCount(value) {
+  return Number.isSafeInteger(value) && value >= 0 && value <= MAX_RERANK_TOKEN_COUNT;
+}
+function inspectRerankDetails(result) {
+  const model = result["model"];
+  if (model !== undefined && (typeof model !== "string" || !/^[\x20-\x7e]+$/u.test(model) || Buffer.byteLength(model, "utf8") > MAX_RERANK_MODEL_BYTES)) {
+    return null;
+  }
+  const rawUsage = result["usage"];
+  let usage;
+  if (rawUsage !== undefined) {
+    const usageRecord = dataRecord(rawUsage);
+    if (usageRecord === null || !hasOnlyKeys(usageRecord, ["inputTokens", "outputTokens"]) || usageRecord["inputTokens"] !== undefined && !tokenCount(usageRecord["inputTokens"]) || usageRecord["outputTokens"] !== undefined && !tokenCount(usageRecord["outputTokens"])) {
+      return null;
+    }
+    usage = {
+      ...usageRecord["inputTokens"] === undefined ? {} : { inputTokens: usageRecord["inputTokens"] },
+      ...usageRecord["outputTokens"] === undefined ? {} : { outputTokens: usageRecord["outputTokens"] }
+    };
+  }
+  const rawAccounting = result["accounting"];
+  let accounting;
+  if (rawAccounting !== undefined) {
+    const record = dataRecord(rawAccounting);
+    if (record === null || !hasOnlyKeys(record, ["candidates", "attempted", "completed", "elapsedMs", "usageComplete"]))
+      return null;
+    const { candidates, attempted, completed, elapsedMs, usageComplete } = record;
+    if (!Number.isSafeInteger(candidates) || candidates < 0 || candidates > MAX_RERANK_CANDIDATES || !Number.isSafeInteger(attempted) || attempted < 0 || attempted > candidates || !Number.isSafeInteger(completed) || completed < 0 || completed > attempted || typeof elapsedMs !== "number" || !Number.isFinite(elapsedMs) || elapsedMs < 0 || elapsedMs > 86400000 || typeof usageComplete !== "boolean" || usageComplete && (completed !== attempted || attempted > 0 && (usage?.inputTokens === undefined || usage.outputTokens === undefined)))
+      return null;
+    accounting = Object.freeze({
+      candidates,
+      attempted,
+      completed,
+      elapsedMs,
+      usageComplete
+    });
+  }
+  return {
+    ...model === undefined ? {} : { model },
+    ...usage === undefined ? {} : { usage: Object.freeze(usage) },
+    ...accounting === undefined ? {} : { accounting }
+  };
+}
+function inspectRerankResult(value) {
+  let details = {};
+  const malformed = () => ({
+    status: "failed",
+    message: MALFORMED_RERANK_RESULT_MESSAGE,
+    ...details
+  });
+  try {
+    const result = dataRecord(value);
+    if (result === null || typeof result["status"] !== "string") {
+      return malformed();
+    }
+    const inspectedDetails = inspectRerankDetails(result);
+    if (inspectedDetails === null)
+      return malformed();
+    details = inspectedDetails;
+    if (result["status"] === "failed" || result["status"] === "unavailable") {
+      if (!hasOnlyKeys(result, ["status", "message", "model", "usage", "accounting"]) || !boundedDiagnostic(result["message"])) {
+        return malformed();
+      }
+      return { status: result["status"], message: result["message"], ...details };
+    }
+    if (result["status"] !== "ready" || !hasOnlyKeys(result, ["status", "ordering", "probabilities", "confidence", "model", "usage", "accounting"])) {
+      return malformed();
+    }
+    if (!Array.isArray(result["ordering"]) || result["ordering"].length > MAX_RERANK_CANDIDATES) {
+      return malformed();
+    }
+    const ordering = [];
+    const rawOrdering = result["ordering"];
+    for (let index = 0;index < rawOrdering.length; index += 1) {
+      const descriptor = Object.getOwnPropertyDescriptor(rawOrdering, String(index));
+      const id = descriptor !== undefined && "value" in descriptor ? descriptor.value : undefined;
+      if (typeof id !== "string" || id === "" || Buffer.byteLength(id, "utf8") > MAX_RERANK_STATE_BYTES) {
+        return malformed();
+      }
+      ordering.push(id);
+    }
+    const rawProbabilities = dataRecord(result["probabilities"]);
+    if (rawProbabilities === null) {
+      return malformed();
+    }
+    const probabilities = Object.create(null);
+    for (const [id, probability] of Object.entries(rawProbabilities)) {
+      if (typeof probability !== "number" || !Number.isFinite(probability) || probability < 0 || probability > 1) {
+        return malformed();
+      }
+      probabilities[id] = probability;
+    }
+    const confidence = result["confidence"];
+    if (confidence !== undefined && (typeof confidence !== "number" || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)) {
+      return malformed();
+    }
+    return {
+      status: "ready",
+      ordering: Object.freeze(ordering),
+      probabilities,
+      ...confidence === undefined ? {} : { confidence },
+      ...details
+    };
+  } catch {
+    return malformed();
+  }
+}
+function failedRerank(hits, message, details) {
+  const result = {
+    status: "failed",
+    message,
+    ...details.model === undefined ? {} : { model: details.model },
+    ...details.usage === undefined ? {} : { usage: details.usage },
+    ...details.accounting === undefined ? {} : { accounting: details.accounting }
+  };
+  return { status: "failed", hits, placements: new Map, result, message };
+}
+function applyRerank(hits, result, expectedWindowIds) {
+  const inspected = inspectRerankResult(result);
+  if (inspected.status !== "ready") {
+    return {
+      status: inspected.status,
+      hits,
+      placements: new Map,
+      result: inspected,
+      message: inspected.message
+    };
+  }
+  const ordering = inspected.ordering;
+  if (expectedWindowIds !== undefined && (ordering.length !== expectedWindowIds.length || expectedWindowIds.some((id) => !ordering.includes(id)))) {
+    return failedRerank(hits, MALFORMED_RERANK_RESULT_MESSAGE, inspected);
+  }
+  const windowIds = new Set;
+  for (const id of ordering) {
+    if (windowIds.has(id)) {
+      return failedRerank(hits, "Rerank ordering contained a duplicate candidate id.", inspected);
+    }
+    windowIds.add(id);
+  }
+  const window = hits.slice(0, ordering.length);
+  const windowById = new Map(window.map((hit) => [hit.id, hit]));
+  for (const id of ordering) {
+    if (!windowById.has(id)) {
+      return failedRerank(hits, "Rerank ordering contained an unknown candidate id.", inspected);
+    }
+  }
+  for (const hit of window) {
+    if (!windowIds.has(hit.id)) {
+      return failedRerank(hits, "Rerank ordering omitted a window candidate id.", inspected);
+    }
+  }
+  const probabilityKeys = Object.keys(inspected.probabilities);
+  for (const id of probabilityKeys) {
+    if (!windowIds.has(id)) {
+      return failedRerank(hits, "Rerank probabilities contained an unknown candidate id.", inspected);
+    }
+  }
+  for (const id of ordering) {
+    if (inspected.probabilities[id] === undefined) {
+      return failedRerank(hits, "Rerank probability for a candidate id was missing or invalid.", inspected);
+    }
+  }
+  const probabilityOf = (id) => inspected.probabilities[id];
+  const sorted = window.toSorted((left, right) => Number(right.identity === true) - Number(left.identity === true) || probabilityOf(right.id) - probabilityOf(left.id) || left.rank - right.rank || left.id.localeCompare(right.id));
+  const placements = new Map;
+  const reordered = sorted.map((hit, index) => {
+    placements.set(hit.id, {
+      baselineRank: hit.rank,
+      rerankRank: index + 1,
+      probability: probabilityOf(hit.id)
+    });
+    return hit.rank === index + 1 ? hit : { ...hit, rank: index + 1 };
+  });
+  return {
+    status: "ready",
+    hits: Object.freeze([...reordered, ...hits.slice(ordering.length)]),
+    placements,
+    result: inspected
+  };
+}
+
+export { MAX_RERANK_CANDIDATES, MAX_RERANK_SNIPPET_BYTES, MAX_RERANK_STATE_BYTES, applyRerank };

package/dist/{index-nd6nynv2.js → index-jk1sgx36.js}

@@ -5,7 +5,7 @@ import {
   isCanonicalNoteId,
   isCanonicalRelationPredicate,
   lookupNote
-} from "./index-ekpwvbra.js";
+} from "./index-qbssx940.js";
 
 // src/percolate.ts
 import { createHash } from "crypto";

package/dist/{index-3ehtt59c.js → index-n386evxj.js}

@@ -1,7 +1,7 @@
 // @bun
 import {
   fuseRankedCandidates
-} from "./index-why54q5p.js";
+} from "./index-vnybgywh.js";
 
 // src/benchmark.ts
 var MAX_BENCHMARK_CASES = 500;

package/dist/{index-vdw5yjfa.js → index-q7tbg07z.js}

@@ -1,8 +1,4 @@
 // @bun
-import {
-  UntrustedContentBudgetError,
-  createUntrustedToolResult
-} from "./index-4j3tt0c3.js";
 import {
   expandSearchRequest,
   parseSearchRules,
@@ -11,20 +7,26 @@ import {
 import {
   openSemanticSearchSession,
   recommendedEmbeddingModel
-} from "./index-115b07ap.js";
+} from "./index-rr9kzq4n.js";
+import {
+  UntrustedContentBudgetError,
+  createUntrustedToolResult
+} from "./index-4j3tt0c3.js";
+import {
+  MAX_RERANK_CANDIDATES,
+  MAX_RERANK_SNIPPET_BYTES,
+  applyRerank
+} from "./index-j70m75wd.js";
 import {
   percolateWithGraph
-} from "./index-bgfzwt4h.js";
+} from "./index-djrcc1yf.js";
 import {
   openGraphAuthority
-} from "./index-bcfn9xah.js";
+} from "./index-xv80vzfp.js";
 import {
   validateGraphPercolationOptions,
   validateGraphQueryRequest
-} from "./index-11621h23.js";
-import {
-  scanVault
-} from "./index-0k2x4nn9.js";
+} from "./index-tf4wzjew.js";
 import {
   GitHistoryError,
   gitHistoryForNotes,
@@ -39,17 +41,20 @@ import {
   fuseRankedCandidates,
   searchExactVault,
   validateSearchQuery
-} from "./index-why54q5p.js";
+} from "./index-vnybgywh.js";
 import {
-  queryVault
-} from "./index-48pz4jpc.js";
+  scanVault
+} from "./index-233z9wmn.js";
 import {
   NavigationBudgetError,
   navigateLinks
 } from "./index-d13v9ckt.js";
+import {
+  queryVault
+} from "./index-48pz4jpc.js";
 import {
   lookupNote
-} from "./index-ekpwvbra.js";
+} from "./index-qbssx940.js";
 
 // src/sdk.ts
 import { resolve } from "path";
@@ -83,6 +88,9 @@ function noteOrThrow(notes, query) {
 function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
+function searchEvidenceRank(item) {
+  return item.kind === "rerank" ? item.rerankRank : item.rank;
+}
 function utf8Prefix(value, maximumBytes) {
   let bytes = 0;
   let end = 0;
@@ -203,6 +211,29 @@ function checkedHistoryRequest(value) {
 function validateKnowledgeBaseSearchHistory(value) {
   checkedHistoryRequest(value);
 }
+function checkedRerankRequest(value, rerankers) {
+  if (value === undefined)
+    return { enabled: false };
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new TypeError("Search rerank options must be an options object.");
+  }
+  const options = value;
+  if (options.engine !== "typesafe") {
+    throw new Error('Search rerank engine must be "typesafe".');
+  }
+  const reranker = rerankers?.find((entry) => entry.id === options.engine);
+  if (reranker === undefined) {
+    throw new Error(`Search rerank engine ${JSON.stringify(options.engine)} is not available in this session.`);
+  }
+  const limit = options.limit ?? MAX_RERANK_CANDIDATES;
+  if (!Number.isSafeInteger(limit) || limit < 2 || limit > MAX_RERANK_CANDIDATES) {
+    throw new RangeError(`Search rerank limit must be an integer from 2 through ${MAX_RERANK_CANDIDATES}.`);
+  }
+  if (options.signal !== undefined && !(options.signal instanceof AbortSignal)) {
+    throw new TypeError("Search rerank signal must be an AbortSignal.");
+  }
+  return { enabled: true, limit, reranker, ...options.signal === undefined ? {} : { signal: options.signal } };
+}
 async function openKnowledgeBase(options, dependencies = {}) {
   const searchRules = options.searchRules === undefined ? null : parseSearchRules(options.searchRules);
   if (options.embeddingModelFile !== undefined && options.embeddingModelLease !== undefined) {
@@ -322,6 +353,7 @@ async function openKnowledgeBase(options, dependencies = {}) {
     }
     const historyRequest = checkedHistoryRequest(effectiveOptions.history);
     const graphOptions = checkedGraphOptions(effectiveOptions.graph);
+    const rerankRequest = checkedRerankRequest(effectiveOptions.rerank, dependencies.rerankers);
     const explicitSeeds = graphOptions === null ? [] : resolvedGraphSeeds(snapshot.notes, graphOptions.related);
     const historyNoteLimit = historyRequest.enabled ? historyRequest.noteLimit : null;
     const allowedIds = new Set(queryVault(snapshot.notes, snapshot.analysis, {
@@ -415,7 +447,8 @@ async function openKnowledgeBase(options, dependencies = {}) {
     const fused = fuseRankedCandidates(lanes);
     const ordered = fused.toSorted((left, right) => Number(exactById.get(right.id)?.hit.identity ?? false) - Number(exactById.get(left.id)?.hit.identity ?? false) || left.rank - right.rank || left.id.localeCompare(right.id));
     const shouldApplyPriority = ordering === "priority-then-relevance" && searchRules !== null && searchRules.priorityRules.length > 0;
-    const relevanceResults = (shouldApplyPriority ? ordered : ordered.slice(0, limit)).map((candidate, index) => {
+    const resultWindow = rerankRequest.enabled ? Math.max(limit, rerankRequest.limit) : limit;
+    const relevanceResults = (shouldApplyPriority ? ordered : ordered.slice(0, resultWindow)).map((candidate, index) => {
       const note = notesById.get(candidate.id);
       if (note === undefined) {
         throw new Error(`Fused retrieval returned unknown note ${JSON.stringify(candidate.id)}.`);
@@ -458,10 +491,67 @@ async function openKnowledgeBase(options, dependencies = {}) {
         contributions: candidate.contributions
       };
     });
+    let rankedResults = relevanceResults;
+    if (rerankRequest.enabled) {
+      const candidates = relevanceResults.slice(0, rerankRequest.limit).map((hit, index) => ({
+        id: hit.id,
+        baselineRank: index + 1,
+        score: hit.score,
+        title: hit.title,
+        path: hit.path,
+        snippet: utf8Prefix(hit.snippet, MAX_RERANK_SNIPPET_BYTES).value
+      }));
+      const outcome = await Promise.resolve().then(() => rerankRequest.reranker.rerank({
+        query: effectiveQuery,
+        candidates,
+        ...rerankRequest.signal === undefined ? {} : { signal: rerankRequest.signal }
+      })).then((result) => ({ ok: true, result }), (error) => ({ ok: false, error }));
+      const applied = applyRerank(relevanceResults, outcome.ok ? outcome.result : {
+        status: "failed",
+        message: "Rerank engine failed before returning a result."
+      }, candidates.map(({ id }) => id));
+      rankedResults = applied.hits.map((hit) => {
+        const placement = applied.placements.get(hit.id);
+        if (placement === undefined)
+          return hit;
+        return {
+          ...hit,
+          evidence: [
+            ...hit.evidence,
+            {
+              kind: "rerank",
+              engine: rerankRequest.reranker.id,
+              baselineRank: placement.baselineRank,
+              rerankRank: placement.rerankRank,
+              ...placement.probability === undefined ? {} : { probability: placement.probability }
+            }
+          ]
+        };
+      });
+      const resultDetails = {
+        ...applied.result.model === undefined ? {} : { model: applied.result.model },
+        ...applied.result.usage === undefined ? {} : { usage: applied.result.usage },
+        ...applied.result.accounting === undefined ? {} : { accounting: applied.result.accounting }
+      };
+      const details = [];
+      if (resultDetails.model !== undefined)
+        details.push(resultDetails.model);
+      if (resultDetails.usage !== undefined) {
+        details.push(`${(resultDetails.usage.inputTokens ?? 0).toLocaleString("en-US")} input tokens, ` + `${(resultDetails.usage.outputTokens ?? 0).toLocaleString("en-US")} output tokens`);
+      }
+      const rerankMessage = applied.status === "ready" ? details.length === 0 ? undefined : details.join("; ") : applied.message ?? "Rerank engine did not return a result.";
+      diagnostics.push({
+        lane: "rerank",
+        ...Object.keys(resultDetails).length === 0 ? {} : { rerank: resultDetails },
+        status: applied.status === "failed" ? "degraded" : applied.status,
+        results: applied.status === "ready" ? applied.placements.size : 0,
+        ...rerankMessage === undefined ? {} : { message: rerankMessage }
+      });
+    }
     let priorityTrace = null;
-    let results = relevanceResults.slice(0, limit);
+    let results = rankedResults.slice(0, limit);
     if (shouldApplyPriority && searchRules !== null) {
-      const prioritized = prioritizeSearchHits(relevanceResults, searchRules, {
+      const prioritized = prioritizeSearchHits(rankedResults, searchRules, {
         ...options.vaultId === undefined ? {} : { vaultId: options.vaultId }
       });
       const selectedHits = prioritized.hits.slice(0, limit);
@@ -696,7 +786,7 @@ Evidence: `))
         return writer.result();
       if (!append("#"))
         return writer.result();
-      if (!append(String(item.rank)))
+      if (!append(String(searchEvidenceRank(item))))
         return writer.result();
     }
     if (!append(`
@@ -1015,4 +1105,4 @@ function packUntrustedSearchContext(result, options = {}) {
   });
 }
 
-export { MAX_SEARCH_RESULTS, MAX_SEARCH_CANDIDATES, DEFAULT_SEARCH_RESULTS, MIN_UNTRUSTED_CONTEXT_BYTES, MAX_SEARCH_RELATED_SEEDS, MAX_SEARCH_NOTE_REFERENCE_BYTES, validateKnowledgeBaseSearchHistory, openKnowledgeBase, packSearchContext, packUntrustedSearchContext };
+export { MAX_SEARCH_RESULTS, MAX_SEARCH_CANDIDATES, DEFAULT_SEARCH_RESULTS, MIN_UNTRUSTED_CONTEXT_BYTES, MAX_SEARCH_RELATED_SEEDS, MAX_SEARCH_NOTE_REFERENCE_BYTES, searchEvidenceRank, validateKnowledgeBaseSearchHistory, openKnowledgeBase, packSearchContext, packUntrustedSearchContext };

package/dist/{index-ekpwvbra.js → index-qbssx940.js}

@@ -374,8 +374,7 @@ function aliasesFromMetadata(metadata) {
   return value.filter((candidate) => typeof candidate === "string" && candidate.trim() !== "");
 }
 function frontmatterOf(content, path) {
-  const lines = content.split(`
-`);
+  const lines = content.split(/\r?\n/u);
   if (lines[0]?.trim() !== "---") {
     return {
       values: new Map,

package/dist/{index-115b07ap.js → index-rr9kzq4n.js}

@@ -1,15 +1,15 @@
 // @bun
+import {
+  fuseRankedCandidates,
+  validateSearchQuery
+} from "./index-vnybgywh.js";
 import {
   MAX_NOTE_UTF8_BYTES,
   MAX_SCANNED_NOTES,
   MAX_VAULT_UTF8_BYTES,
   VaultScanBudgetError,
   scanVault
-} from "./index-0k2x4nn9.js";
-import {
-  fuseRankedCandidates,
-  validateSearchQuery
-} from "./index-why54q5p.js";
+} from "./index-233z9wmn.js";
 
 // src/semantic.ts
 import { createHash as createHash2 } from "crypto";

package/dist/{index-11621h23.js → index-tf4wzjew.js}

@@ -5,7 +5,7 @@ import {
 import {
   isCanonicalNoteId,
   isCanonicalRelationPredicate
-} from "./index-ekpwvbra.js";
+} from "./index-qbssx940.js";
 
 // src/graph-authority-model.ts
 var GRAPH_LIMITS = Object.freeze({

package/dist/{index-why54q5p.js → index-vnybgywh.js}

@@ -1,13 +1,13 @@
 // @bun
-import {
-  queryVault
-} from "./index-48pz4jpc.js";
 import {
   navigateLinks
 } from "./index-d13v9ckt.js";
+import {
+  queryVault
+} from "./index-48pz4jpc.js";
 import {
   lookupNote
-} from "./index-ekpwvbra.js";
+} from "./index-qbssx940.js";
 
 // src/search.ts
 var MAX_EXACT_RESULTS = 500;

package/dist/{index-bcfn9xah.js → index-xv80vzfp.js}

@@ -3,23 +3,23 @@ import {
   GRAPH_LIMITS,
   GraphAuthorityError,
   validateGraphQueryRequest
-} from "./index-11621h23.js";
+} from "./index-tf4wzjew.js";
 import {
   scanVault
-} from "./index-0k2x4nn9.js";
-import {
-  acquireFileLease
-} from "./index-3rm7cz6h.js";
+} from "./index-233z9wmn.js";
 import {
   analyzeAuthoredRepositoryScopes
 } from "./index-06c9ctr6.js";
+import {
+  acquireFileLease
+} from "./index-3rm7cz6h.js";
 import {
   VaultAnalysisBudgetError,
   analyzeVault,
   documentIdState,
   isCanonicalNoteId,
   parseNote
-} from "./index-ekpwvbra.js";
+} from "./index-qbssx940.js";
 import {
   __require
 } from "./index-z1w83f81.js";
@@ -231,7 +231,7 @@ import { lstat, open } from "fs/promises";
 var graphCacheFileSystem = { lstat, open };
 
 // src/graph-authority.ts
-var openOhGraphAdapter = async (...args) => await (await import("./authority-4sp9d552.js")).openOhGraphAdapter(...args);
+var openOhGraphAdapter = async (...args) => await (await import("./authority-06eqhqam.js")).openOhGraphAdapter(...args);
 var MAX_CACHE_BYTES = 67108864;
 var ignoredCache = `# Disposable Wordcell graph state. Rebuild from Markdown.
 *