@hraness/wordcell 0.21.3 → 0.22.1

package/src/graph.ts

@@ -608,7 +608,7 @@ function aliasesFromMetadata(metadata: MetadataObject): readonly string[] {
 }
 
 function frontmatterOf(content: string, path: string): Frontmatter {
-  const lines = content.split("\n");
+  const lines = content.split(/\r?\n/u);
   if (lines[0]?.trim() !== "---") {
     return {
       values: new Map(),

package/src/index.ts

@@ -11,7 +11,11 @@ export * from "./init.js";
 export * from "./navigation.js";
 export * from "./oh-adoption.js";
 export * from "./percolate.js";
+export * from "./publish.js";
+export * from "./publish-model.js";
 export * from "./query.js";
+export * from "./rerank.js";
+export * from "./rerank-typesafe.js";
 export * from "./repository-memory.js";
 export * from "./search.js";
 export * from "./semantic.js";

package/src/oh/authority.ts

@@ -3,7 +3,7 @@ import { isProxy } from "node:util/types";
 import { canonicalJson, canonicalSha256, type KnowledgeGraphChangeV1, type KnowledgeGraphRecordV1 } from "@hraness/oh";
 import {
   createOhProjectionDatasetV1, createOhProjectionFactV1, createOhProjectionSnapshotV1,
-  evaluateOhProjectionV1, parseOhProjectionResultV1, type OhProjectionProofV1,
+  parseOhProjectionResultV1, type OhProjectionProofV1,
 } from "@hraness/oh/projection";
 import { createOhSqliteStoreAuthorityV1 } from "@hraness/oh/sqlite";
 import { createOhStoreProfileV1, OH_WORKING_STORE_PROFILE_V1, type OhStoreV1 } from "@hraness/oh/store";
@@ -11,6 +11,7 @@ import {
   GRAPH_LIMITS, GraphAuthorityError, type GraphAuthority, type GraphProof,
   type GraphQueryLimits, type GraphQueryRequest, type GraphQueryResult, type GraphSnapshot,
 } from "../graph-authority-model";
+import { evaluateOhProjectionWithFallbackV1, loadProjectionRustEngineV1 } from "./projection-rust";
 import { compileGraphProgram, normalizeGraphQueryRequest } from "./programs";
 import { GRAPH_EXTRACTOR_SHA256, GRAPH_MANIFEST_KEY, manifestGraphRecord, parseGraphSnapshot, sourceGraphRecord } from "./snapshot";
 import { deepFreeze, detachedData, exactKeys } from "./validation";
@@ -95,6 +96,7 @@ export async function openOhGraphAdapter(input: GraphSnapshot, options: OhGraphA
     const materialized = await selectedStore.snapshot({ maximumRecords: GRAPH_LIMITS.notes + 1 });
     if (canonicalJson(materialized.records) !== canonicalJson(expected)) throw new GraphAuthorityError("corrupt-cache", "Graph reconciliation did not retain exact source records.");
     const projectionSnapshot = createOhProjectionSnapshotV1({ ...materialized, spaceId });
+    const rustEngine = await loadProjectionRustEngineV1();
     const recordMap = new Map(records.map((record, index) => [record.key, { record, source: snapshot.records[index]! }]));
     const assertOpen = () => { if (closed) throw new GraphAuthorityError("closed", "The graph authority is closed."); };
     const assertCurrent = async () => {
@@ -141,10 +143,17 @@ export async function openOhGraphAdapter(input: GraphSnapshot, options: OhGraphA
       });
       try {
         const dataset = createOhProjectionDatasetV1({ extractorSha256: GRAPH_EXTRACTOR_SHA256, factPackId: "wordcell.graph-facts", factPackRevision: 1, facts, snapshot: projectionSnapshot });
-        const result = evaluateOhProjectionV1({ ...program, dataset, snapshot: projectionSnapshot, options: {
-          maximumDerivedTuples: limits.derivedTuples, maximumRounds: limits.rounds, maximumWorkUnits: limits.workUnits,
-          maximumProofDepth: limits.proofDepth, maximumProofNodes: limits.proofNodes, maximumTotalProofNodes: limits.totalProofNodes, maximumResultBytes: limits.resultBytes,
-        } });
+        const result = evaluateOhProjectionWithFallbackV1({
+          dataset,
+          engine: rustEngine,
+          rulePack: program.rulePack,
+          query: program.query,
+          snapshot: projectionSnapshot,
+          options: {
+            maximumDerivedTuples: limits.derivedTuples, maximumRounds: limits.rounds, maximumWorkUnits: limits.workUnits,
+            maximumProofDepth: limits.proofDepth, maximumProofNodes: limits.proofNodes, maximumTotalProofNodes: limits.totalProofNodes, maximumResultBytes: limits.resultBytes,
+          },
+        });
         if (parseOhProjectionResultV1(result, result.identity.projectionSha256) === null) throw new GraphAuthorityError("corrupt-cache", "Oh returned an invalid bounded projection result.");
         await assertCurrent();
         const proofBudget = { remaining: limits.resultBytes };

package/src/oh/canonical-rust.ts

@@ -0,0 +1,127 @@
+import { canonicalJson } from "@hraness/oh";
+import {
+  OH_CANONICAL_RAW_WASM_BASE64,
+  OH_CANONICAL_RAW_WASM_SHA256,
+} from "@hraness/oh/canonical-rust/artifact";
+import { emitOhRustFallback } from "@hraness/oh/rust-fallback";
+
+/**
+ * Optional Rust canonical-JSON/digest engine (`oh.canonical.rust.v1`), vendored
+ * from `hraness/oh`. Returns `null` when the engine cannot be loaded or cannot
+ * match the reference `@hraness/oh` output, so callers keep the existing TypeScript
+ * path as the fallback.
+ */
+
+interface OhCanonicalRawExports {
+  readonly memory: WebAssembly.Memory;
+  readonly oh_canonical_alloc: (len: number) => number;
+  readonly oh_canonical_free: (ptr: number, capacity: number) => void;
+  readonly oh_canonical_json: (ptr: number, len: number) => number;
+  readonly oh_canonical_sha256: (ptr: number, len: number) => number;
+}
+
+const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+function decodeBase64(text: string): Uint8Array {
+  const clean = text.replace(/=+$/u, "");
+  const bytes = new Uint8Array(Math.floor((clean.length * 3) / 4));
+  let bits = 0;
+  let bitCount = 0;
+  let offset = 0;
+  for (const character of clean) {
+    const value = BASE64_ALPHABET.indexOf(character);
+    if (value < 0) throw new Error("invalid base64 artifact");
+    bits = (bits << 6) | value;
+    bitCount += 6;
+    if (bitCount >= 8) {
+      bitCount -= 8;
+      bytes[offset] = (bits >> bitCount) & 0xff;
+      offset += 1;
+    }
+  }
+  if (offset !== bytes.length) throw new Error("base64 artifact length mismatch");
+  return bytes;
+}
+
+let cached: OhCanonicalRawExports | null | undefined;
+
+function engine(): OhCanonicalRawExports | null {
+  if (cached !== undefined) return cached;
+  try {
+    const bytes = decodeBase64(OH_CANONICAL_RAW_WASM_BASE64);
+    const instance = new WebAssembly.Instance(new WebAssembly.Module(bytes.buffer as ArrayBuffer));
+    cached = instance.exports as unknown as OhCanonicalRawExports;
+  } catch {
+    cached = null;
+  }
+  return cached;
+}
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+function runEngine(
+  text: string,
+  call: (exports: OhCanonicalRawExports, ptr: number, len: number) => number,
+): string | null {
+  const exports = engine();
+  if (exports === null) return null;
+  const input = encoder.encode(text);
+  if (input.length === 0) return null;
+  const inputPtr = exports.oh_canonical_alloc(input.length);
+  if (inputPtr === 0) return null;
+  let resultPtr = 0;
+  let resultCapacity = 0;
+  try {
+    new Uint8Array(exports.memory.buffer as ArrayBuffer, inputPtr, input.length).set(input);
+    resultPtr = call(exports, inputPtr, input.length);
+    if (resultPtr === 0) return null;
+    const memoryLength = exports.memory.buffer.byteLength;
+    if (resultPtr > memoryLength - 12) return null;
+    const view = new DataView(exports.memory.buffer as ArrayBuffer, resultPtr, 12);
+    const capacity = view.getUint32(0, true);
+    const status = view.getUint32(4, true);
+    const payloadLength = view.getUint32(8, true);
+    if (capacity < 12 || capacity > memoryLength - resultPtr || payloadLength > capacity - 12) return null;
+    resultCapacity = capacity;
+    if (status !== 0 || payloadLength === 0) return null;
+    return decoder.decode(new Uint8Array(exports.memory.buffer as ArrayBuffer, resultPtr + 12, payloadLength));
+  } catch {
+    return null;
+  } finally {
+    if (resultPtr !== 0 && resultCapacity > 0) exports.oh_canonical_free(resultPtr, resultCapacity);
+    exports.oh_canonical_free(inputPtr, input.length);
+  }
+}
+
+/** Canonical JSON text matching `@hraness/oh`, or `null` if the engine cannot match. */
+export function canonicalJsonRust(value: unknown): string | null {
+  const json = JSON.stringify(value);
+  if (json === undefined) return null;
+  const rust = runEngine(json, (exports, ptr, len) => exports.oh_canonical_json(ptr, len));
+  if (rust === null) return null;
+  try {
+    if (rust !== canonicalJson(value)) return null;
+  } catch {
+    return null;
+  }
+  return rust;
+}
+
+/** SHA-256 hex of the canonical JSON, matching `@hraness/oh`, or `null` if the engine cannot match. */
+export function canonicalSha256Rust(value: unknown): string | null {
+  if (canonicalJsonRust(value) === null) return null;
+  const json = JSON.stringify(value);
+  if (json === undefined) return null;
+  return runEngine(json, (exports, ptr, len) => exports.oh_canonical_sha256(ptr, len));
+}
+
+/** Engine identity and artifact digest for diagnostics and tests. */
+export function canonicalRustEngineInfo(): { engine: string; artifactSha256: string; available: boolean } {
+  return { engine: "oh.canonical.rust.v1", artifactSha256: OH_CANONICAL_RAW_WASM_SHA256, available: engine() !== null };
+}
+
+/** Emit a non-fatal telemetry notice when the Rust engine falls back to TS. */
+export function emitCanonicalRustFallback(reason: string, inputClass: string): void {
+  emitOhRustFallback({ tag: "oh-canonical-rust-fallback", reason, inputClass });
+}

package/src/oh/projection-rust.ts

@@ -0,0 +1,63 @@
+import { evaluateOhProjectionV1, type OhProjectionDatasetV1, type OhProjectionEvaluationOptionsV1, type OhProjectionQueryV1, type OhProjectionResultV1, type OhProjectionRulePackV1, type OhProjectionSnapshotV1 } from "@hraness/oh/projection";
+import { emitOhRustFallback } from "@hraness/oh/rust-fallback";
+
+interface ProjectionRustEngineV1 {
+  readonly engine: "oh.projection.rust.v1";
+  evaluate(input: Readonly<{
+    dataset: OhProjectionDatasetV1;
+    options?: OhProjectionEvaluationOptionsV1;
+    query: OhProjectionQueryV1;
+    rulePack: OhProjectionRulePackV1;
+    snapshot: OhProjectionSnapshotV1;
+  }>): OhProjectionResultV1;
+}
+
+let cached: ProjectionRustEngineV1 | null | undefined;
+
+export function emitProjectionRustFallback(reason: "load-failed" | "evaluate-failed"): void {
+  emitOhRustFallback({ tag: "oh-projection-rust-fallback", reason });
+}
+
+const OH_PROJECTION_RUST_MODULE = "@hraness/oh/projection/rust";
+
+async function loadEngine(): Promise<ProjectionRustEngineV1 | null> {
+  if (cached !== undefined) return cached;
+  try {
+    const module = (await import(OH_PROJECTION_RUST_MODULE)) as {
+      loadProjectionRustEngineV1?(): Promise<ProjectionRustEngineV1 | null>;
+    };
+    if (typeof module.loadProjectionRustEngineV1 !== "function") {
+      cached = null;
+      return null;
+    }
+    cached = await module.loadProjectionRustEngineV1();
+  } catch {
+    emitProjectionRustFallback("load-failed");
+    cached = null;
+  }
+  return cached;
+}
+
+/** Load the optional Rust projection engine once per process. */
+export async function loadProjectionRustEngineV1(): Promise<ProjectionRustEngineV1 | null> {
+  return loadEngine();
+}
+
+/** Evaluate a projection, preferring the Rust engine when available. */
+export function evaluateOhProjectionWithFallbackV1(input: Readonly<{
+  dataset: OhProjectionDatasetV1;
+  engine: ProjectionRustEngineV1 | null;
+  options?: OhProjectionEvaluationOptionsV1;
+  query: OhProjectionQueryV1;
+  rulePack: OhProjectionRulePackV1;
+  snapshot: OhProjectionSnapshotV1;
+}>): OhProjectionResultV1 {
+  if (input.engine !== null) {
+    try {
+      return input.engine.evaluate(input);
+    } catch {
+      emitProjectionRustFallback("evaluate-failed");
+    }
+  }
+  return evaluateOhProjectionV1(input);
+}

package/src/oh-adoption.ts

@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
 import { posix } from "node:path";
 
 import { canonicalJson, canonicalSha256 } from "@hraness/oh";
+
+import { canonicalSha256Rust, emitCanonicalRustFallback } from "./oh/canonical-rust.js";
 import {
   parseOhHeadV1,
   parseOhStoreBindingV1,
@@ -419,7 +421,11 @@ function prepareWithPolicy(
     transformations,
     v: 1 as const,
   };
-  const candidateSha256 = canonicalSha256(manifest);
+  const rustSha256 = canonicalSha256Rust(manifest);
+  if (rustSha256 === null) {
+    emitCanonicalRustFallback("canonicalSha256Rust", "object");
+  }
+  const candidateSha256 = rustSha256 ?? canonicalSha256(manifest);
   const markdown = renderMarkdown(manifest, candidateSha256);
   if (Buffer.byteLength(markdown, "utf8") > MAX_CAPSULE_BYTES) {
     throw new RangeError("The adoption candidate exceeds its Markdown byte limit.");

package/src/publish-index.ts

@@ -0,0 +1,195 @@
+import { canonicalJson } from "@hraness/oh";
+
+import type { Note } from "./graph.js";
+import {
+  WORDCELL_SITE_DOCS_FORMAT_V1,
+  WORDCELL_SITE_LIMITS_V1,
+  WORDCELL_SITE_POSTINGS_FORMAT_V1,
+  WORDCELL_SITE_TERMS_FORMAT_V1,
+  type WordcellSiteContentSearchV1,
+  type WordcellSiteDocsV1,
+  type WordcellSiteDocV1,
+  type WordcellSitePostingsV1,
+  type WordcellSiteTermsV1,
+} from "./publish-model.js";
+import {
+  publishNormalize,
+  publishShardName,
+} from "./publish-search.js";
+
+const TERM_PATTERN = /[\p{L}\p{N}][\p{L}\p{N}._/-]*/gu;
+
+export type SiteIndexBuild = {
+  readonly docs: WordcellSiteDocsV1;
+  readonly terms: WordcellSiteTermsV1;
+  readonly postings: ReadonlyMap<string, WordcellSitePostingsV1>;
+  readonly termsCount: number;
+  readonly termsTruncated: boolean;
+  readonly textTruncated: boolean;
+  readonly mode: WordcellSiteContentSearchV1;
+};
+
+function bounded(value: string, maximumBytes: number): { text: string; truncated: boolean } {
+  if (Buffer.byteLength(value, "utf8") <= maximumBytes) return { text: value, truncated: false };
+  const bytes = Buffer.from(value, "utf8");
+  let end = maximumBytes;
+  while (end > 0 && ((bytes[end] ?? 0) & 0xc0) === 0x80) end -= 1;
+  return { text: bytes.subarray(0, end).toString("utf8"), truncated: true };
+}
+
+function previewText(note: Note): string {
+  const basis = note.summary !== "" ? note.summary : note.searchableText;
+  const collapsed = basis.replace(/\s+/gu, " ").trim();
+  return bounded(collapsed, WORDCELL_SITE_LIMITS_V1.docPreviewBytes).text;
+}
+
+function indexableTerms(text: string): readonly string[] {
+  const terms = new Set<string>();
+  for (const match of text.matchAll(TERM_PATTERN)) terms.add(match[0]);
+  return [...terms];
+}
+
+/**
+ * Build the document table, term dictionary, and content postings for a
+ * published selection. The document table always carries normalized eager
+ * fields (title, aliases, path/id, tags, metadata) so browser field matching
+ * reproduces the exact lane's substring semantics. Content is indexed inline
+ * when the whole selection fits the inline budget, otherwise by sharded
+ * postings; `indexContent: false` emits no content index at all.
+ */
+export function buildSiteIndex(
+  notes: readonly Note[],
+  slugById: ReadonlyMap<string, string>,
+  options: { readonly indexContent?: boolean } = {},
+): SiteIndexBuild {
+  const indexContent = options.indexContent !== false;
+  const fields: WordcellSiteDocV1["f"][] = [];
+  const inlineTexts: string[] = [];
+  let textTruncated = false;
+  let inlineTotal = 0;
+
+  for (const note of notes) {
+    const title = bounded(publishNormalize(note.title), WORDCELL_SITE_LIMITS_V1.fieldTextBytes);
+    const aliases = bounded(
+      note.aliases.map(publishNormalize).join("\n"),
+      WORDCELL_SITE_LIMITS_V1.fieldTextBytes,
+    );
+    const path = bounded(
+      `${publishNormalize(note.path)}\n${publishNormalize(note.id)}`,
+      WORDCELL_SITE_LIMITS_V1.fieldTextBytes,
+    );
+    const tags = bounded(
+      note.tags.map(publishNormalize).join("\n"),
+      WORDCELL_SITE_LIMITS_V1.fieldTextBytes,
+    );
+    const metadata = bounded(
+      publishNormalize(canonicalJson(note.metadata)),
+      WORDCELL_SITE_LIMITS_V1.fieldTextBytes,
+    );
+    textTruncated ||= title.truncated || aliases.truncated || path.truncated
+      || tags.truncated || metadata.truncated;
+    fields.push({ t: title.text, a: aliases.text, p: path.text, g: tags.text, m: metadata.text });
+
+    if (indexContent) {
+      // Match the exact lane's basis: searchableText strips frontmatter,
+      // code fences, and comments before indexing.
+      const inline = bounded(
+        publishNormalize(note.searchableText),
+        WORDCELL_SITE_LIMITS_V1.inlineTextBytes,
+      );
+      textTruncated ||= inline.truncated;
+      inlineTexts.push(inline.text);
+      inlineTotal += Buffer.byteLength(inline.text, "utf8");
+    }
+  }
+
+  const mode: WordcellSiteContentSearchV1 = !indexContent
+    ? "none"
+    : inlineTotal <= WORDCELL_SITE_LIMITS_V1.inlineTotalBytes
+      ? "inline"
+      : "shards";
+
+  const allTerms = new Set<string>();
+  const contentPostings = new Map<string, number[]>();
+  const docs: WordcellSiteDocV1[] = [];
+
+  for (const [index, note] of notes.entries()) {
+    const slug = slugById.get(note.id);
+    if (slug === undefined) throw new Error(`Missing published slug for ${note.id}.`);
+    const field = fields[index];
+    if (field === undefined) throw new Error(`Missing indexed fields for ${note.id}.`);
+    const inline = mode === "inline" ? inlineTexts[index] : undefined;
+
+    const fieldTerms = indexableTerms(
+      `${field.t}\n${field.a}\n${field.p}\n${field.g}\n${field.m}`,
+    );
+    for (const term of fieldTerms) allTerms.add(term);
+
+    if (mode !== "none") {
+      const content = inlineTexts[index] ?? "";
+      for (const term of indexableTerms(content)) {
+        allTerms.add(term);
+        if (mode === "shards") {
+          const list = contentPostings.get(term) ?? [];
+          list.push(index);
+          contentPostings.set(term, list);
+        }
+      }
+    }
+
+    docs.push({
+      i: index,
+      s: slug,
+      t: note.title,
+      p: previewText(note),
+      f: field,
+      ...(inline === undefined ? {} : { x: inline }),
+    });
+  }
+
+  let termsTruncated = false;
+  let sortedTerms = [...allTerms].toSorted((left, right) => left.localeCompare(right));
+  if (sortedTerms.length > WORDCELL_SITE_LIMITS_V1.indexTerms) {
+    sortedTerms = sortedTerms.slice(0, WORDCELL_SITE_LIMITS_V1.indexTerms);
+    termsTruncated = true;
+  }
+  const admitted = new Set(sortedTerms);
+
+  const postings = new Map<string, WordcellSitePostingsV1>();
+  if (mode === "shards") {
+    const grouped = new Map<string, Record<string, readonly number[]>>();
+    for (const [term, docIds] of [...contentPostings.entries()].toSorted(([a], [b]) =>
+      a.localeCompare(b))) {
+      if (!admitted.has(term)) continue;
+      const shard = publishShardName(term);
+      const bucket = grouped.get(shard) ?? Object.create(null) as Record<string, readonly number[]>;
+      bucket[term] = docIds;
+      grouped.set(shard, bucket);
+    }
+    for (const [shard, bucket] of [...grouped.entries()].toSorted(([a], [b]) =>
+      a.localeCompare(b))) {
+      postings.set(shard, {
+        format: WORDCELL_SITE_POSTINGS_FORMAT_V1,
+        shard,
+        postings: Object.freeze(bucket),
+      });
+    }
+  }
+
+  return {
+    docs: {
+      format: WORDCELL_SITE_DOCS_FORMAT_V1,
+      content: mode,
+      docs,
+    },
+    terms: {
+      format: WORDCELL_SITE_TERMS_FORMAT_V1,
+      terms: sortedTerms,
+    },
+    postings,
+    termsCount: sortedTerms.length,
+    termsTruncated,
+    textTruncated,
+    mode,
+  };
+}