albex 0.1.0 → 0.6.0

package/src/wasm-bindings.ts

@@ -0,0 +1,349 @@
+/**
+ * Typed interfaces for the two WASM modules Albex ships with.
+ *
+ * These types replace ad-hoc `as Function` casts and give us:
+ *  - argument/return type checking at call sites,
+ *  - autocompletion in IDEs,
+ *  - safe refactors when an export name or signature changes.
+ *
+ * The interfaces mirror the `#[no_mangle] pub extern "C" fn` exports
+ * in `wasm/src/lib.rs` and `pdf-wasm/src/lib.rs`.
+ */
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Main WASM module (albex_wasm_bg.wasm)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface AlbexWasmExports {
+  readonly memory: WebAssembly.Memory;
+
+  // ABI / lifecycle
+  abiVersion(): number;
+  getBuffer(size: number): number;
+  init(): void;
+
+  /** Reset the streaming FNV-1a 64-bit hash state. Optional on the first
+   * hash of a session because the static initialiser is also FNV_OFFSET. */
+  hashBegin(): void;
+  /** Fold `len` bytes of scratchpad into the streaming hash. May be
+   * called repeatedly for files larger than SCRATCHPAD_SIZE. */
+  hashFeed(len: number): void;
+  /** Write the final 8 raw big-endian bytes at scratchpad[0..8] and
+   * reset the state so the next hash can start without an explicit Begin. */
+  hashFinish(): void;
+
+  // Document ingestion
+  setDocumentName(len: number): void;
+  beginDocument(): number;
+  feedXmlBytes(len: number): void;
+  endDocument(): number;
+
+  // XLSX
+  beginXlsx(): number;
+  feedXlsxBytes(len: number): void;
+
+  // Generic text path (PDF, TXT, XML, future formats)
+  feedText(len: number): void;
+  flushParagraph(): void;
+
+  // Search configuration
+  setMaxErrors(errors: number): void;
+  setThreshold(threshold: number): void;
+  setMaxResults(max: number): void;
+
+  // Query parsing (since ABI v2). Single source of truth for tokenization.
+  prepareQuery(len: number): number;
+  getQueryKind(): number;
+  getQueryBranchCount(): number;
+  getQueryBranchPattern(i: number): number;
+  selectQueryBranch(i: number): number;
+
+  // Search execution
+  setPattern(len: number): number;
+  search(): number;
+  // Resumable search (used by frame-budgeted searches and worker shards)
+  searchBegin(): number;
+  searchSlice(maxChunks: number): number;
+  getSearchCursor(): number;
+  getSearchTotal(): number;
+
+  // Result accessors
+  getResultCount(): number;
+  getResultDocId(i: number): number;
+  getResultLocation(i: number): number;
+  getResultScore(i: number): number;
+  getResultStart(i: number): number;
+  getResultEnd(i: number): number;
+  getResultChunkIdx(i: number): number;
+  getResultDocName(i: number): number;
+  getResultMatchCount(i: number): number;
+  getResultMatchStartAt(i: number, k: number): number;
+  getResultMatchEndAt(i: number, k: number): number;
+  getSnippet(i: number): number;
+  getSnippetWindow(i: number, before: number, after: number): number;
+  getSnippetWindowOffset(): number;
+
+  // Stats
+  getStatBloomTested(): number;
+  getStatBloomPassed(): number;
+  getStatBitapMatched(): number;
+  getChunkCount(): number;
+  getDocCount(): number;
+  getTextUsed(): number;
+  getTextCapacity(): number;
+  /** Bitflags of capacity limits hit during the most recent
+   * begin..endDocument cycle: 1 = chunks, 2 = text, 4 = docs, 8 = names.
+   * 0 = everything fit. Read by the host right after endDocument to raise a
+   * typed AlbexCapacityError instead of silently truncating the corpus. */
+  getLastIndexOverflow(): number;
+
+  // Snapshot / restore (v3 protocol; v1 and v2 still load)
+  snapshotSize(): number;
+  snapshotChunk(offset: number, maxLen: number): number;
+  /** Validate header. For v3 also reserves the staging buffer; state is
+   * NOT touched until restoreCommit succeeds. For v1/v2 (legacy) state is
+   * reset and counters are written immediately. */
+  restoreBegin(): number;
+  /** Feed payload bytes. For v3 they accumulate into staging; for v1/v2
+   * they are written straight to the state arrays as before. */
+  restoreFeed(len: number): number;
+  /** Atomic commit for v3 snapshots. Returns 1 if the staged payload was
+   * complete and decoded successfully; 0 otherwise — and in the 0 case
+   * the previous engine state is preserved. For v1/v2 this is a no-op
+   * that always returns 1. */
+  restoreCommit(): number;
+
+  // Incremental / per-doc
+  getDocId(index: number): number;
+  getDocChunkCount(index: number): number;
+  getDocName(index: number): number;
+  isDocDeleted(index: number): number;
+  removeDocument(docId: number): number;
+  compact(): void;
+
+  /**
+   * Per-document content hash (snapshot v2). Returns a pointer to 8 bytes
+   * holding the FNV-1a 64-bit hash of the original file bytes, or 0 if the
+   * doc index is out of range. All-zero bytes mean "hash not available"
+   * — either the host never called setDocumentContentHash for this doc
+   * (legacy code path) or it was restored from a v1 snapshot.
+   */
+  getDocContentHashPtr(index: number): number;
+  /** Always returns 8. Useful as a runtime feature-detect: older binaries
+   * without snapshot v2 will not export this function at all. */
+  getDocContentHashLen(): number;
+  /** Copy a content hash (up to 8 bytes from the scratchpad) into the
+   * pending slot. endDocument() then writes it into doc_hashes[]. */
+  setDocumentContentHash(len: number): void;
+
+  // Stemming
+  setLanguage(lang: number): void;
+
+  // Tier identification
+  getTier(): number;        // 1=mini, 2=std, 3=pro
+  getMaxChunks(): number;
+  getMaxDocs(): number;
+  getNameCapacity(): number;
+
+  // GPU bridge (CD1): zero-copy access to chunk array + candidate mask
+  getChunksPtr(): number;
+  getChunkStructSize(): number;
+  setCandidateMask(byteLen: number): void;
+  clearCandidateMask(): void;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// PDF WASM module (albex_pdf.wasm)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface AlbexPdfExports {
+  readonly memory: WebAssembly.Memory;
+
+  /** ABI version of the PDF module. The host loader refuses any binary
+   * whose abiVersion is outside the supported range. */
+  abiVersion(): number;
+
+  /** Reserve `len` bytes inside the PDF module and return a pointer. */
+  allocInput(len: number): number;
+
+  /**
+   * Parse the PDF. Returns:
+   *   N ≥ 0 — page count,
+   *   -1    — parse error (read with getErrorPtr/getErrorLen),
+   *   -2    — image-only / no extractable text.
+   *
+   * When `-2` is returned, the host can fall through to the scanned-PDF
+   * path via `getPageCount` + `extractPageImages`.
+   */
+  extractPdf(len: number): number;
+
+  getPageLen(page: number): number;
+  getPagePtr(page: number): number;
+  getErrorLen(): number;
+  getErrorPtr(): number;
+
+  // ── Scanned-PDF path (extracts embedded image XObjects) ──────────────────
+  //
+  // Available since pdf-wasm 0.2. Older binaries built before this addition
+  // will not expose these exports — `asAlbexPdfExports` keeps them typed,
+  // but the engine should feature-detect at runtime before relying on them.
+
+  /** Total page count of the loaded PDF; 0 if the input cannot be parsed. */
+  getPageCount(): number;
+
+  /**
+   * Extract every supported image XObject on page `page` (0-based) into the
+   * module's internal buffer.
+   *
+   * Returns:
+   *   N ≥ 0 — number of images extracted on this page,
+   *   -1    — parse error or page index out of range.
+   *
+   * Each extracted image is one of:
+   *   * JPEG (kind = 1, from a `/DCTDecode` filter), or
+   *   * JPEG2000 (kind = 2, from a `/JPXDecode` filter).
+   *
+   * Filters that require Rust-side reconstruction (`FlateDecode`,
+   * `CCITTFaxDecode`, `JBIG2Decode`) are intentionally skipped — they
+   * would roughly double the binary size for ~5 % more coverage.
+   */
+  extractPageImages(page: number): number;
+
+  /** Byte length of extracted image `i` (from the last `extractPageImages`). */
+  getPageImageLen(i: number): number;
+
+  /** Pointer to extracted image `i`'s raw bytes. */
+  getPageImagePtr(i: number): number;
+
+  /** Format tag for extracted image `i`: 1 = JPEG, 2 = JPEG2000, 0 = none. */
+  getPageImageKind(i: number): number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Runtime validators
+// ─────────────────────────────────────────────────────────────────────────────
+//
+// These replace the pre-0.5.0 `as unknown as` casts. They check three
+// things at instantiation time:
+//   1. memory is a WebAssembly.Memory instance.
+//   2. abiVersion() returns a number inside the supported range.
+//   3. every required export exists and is a function.
+//
+// If any of these fails, the loader throws a typed error before the
+// engine returns from init(). This eliminates the audit 3.2 issue:
+// previously a missing export only surfaced when its call site ran.
+
+/** Range of ABI versions this host code understands for the main module.
+ * Update both ends together with the Rust `abiVersion()` constant when
+ * the export surface changes. */
+// 0.6.0 requires ABI 3 (trigram pre-filter + getLastIndexOverflow). The
+// required-exports list below already makes any older binary fail the
+// missing-exports check, so a tolerant lower bound was dead code — the range
+// is pinned to the one ABI this host actually speaks (audit 0.6.0, finding #7).
+const MAIN_ABI_MIN = 3;
+const MAIN_ABI_MAX = 3;
+
+/** Range of ABI versions for the PDF module. */
+const PDF_ABI_MIN = 1;
+const PDF_ABI_MAX = 3;
+
+/** Required function names on the main WASM. Adding a new one here forces
+ * the validator to check it; removing one is a breaking ABI bump. */
+const MAIN_REQUIRED = [
+  'abiVersion', 'getBuffer', 'init',
+  'setDocumentName', 'beginDocument', 'feedXmlBytes', 'endDocument',
+  'beginXlsx', 'feedXlsxBytes',
+  'feedText', 'flushParagraph',
+  'setMaxErrors', 'setThreshold', 'setMaxResults',
+  'prepareQuery', 'getQueryKind', 'getQueryBranchCount',
+  'getQueryBranchPattern', 'selectQueryBranch',
+  'setPattern', 'search',
+  'searchBegin', 'searchSlice', 'getSearchCursor', 'getSearchTotal',
+  'getResultCount',
+  'getResultDocId', 'getResultLocation', 'getResultScore',
+  'getResultStart', 'getResultEnd', 'getResultChunkIdx',
+  'getResultDocName', 'getResultMatchCount',
+  'getResultMatchStartAt', 'getResultMatchEndAt',
+  'getSnippet', 'getSnippetWindow', 'getSnippetWindowOffset',
+  'getStatBloomTested', 'getStatBloomPassed', 'getStatBitapMatched',
+  'getChunkCount', 'getDocCount', 'getTextUsed', 'getTextCapacity',
+  'getLastIndexOverflow',
+  'snapshotSize', 'snapshotChunk',
+  'restoreBegin', 'restoreFeed', 'restoreCommit',
+  'getDocId', 'getDocChunkCount', 'getDocName', 'isDocDeleted',
+  'removeDocument', 'compact',
+  'setLanguage',
+  'getTier', 'getMaxChunks', 'getMaxDocs', 'getNameCapacity',
+  'getChunksPtr', 'getChunkStructSize',
+  'setCandidateMask', 'clearCandidateMask',
+  'getDocContentHashPtr', 'getDocContentHashLen', 'setDocumentContentHash',
+  'hashBegin', 'hashFeed', 'hashFinish',
+] as const;
+
+const PDF_REQUIRED = [
+  'abiVersion', 'allocInput', 'extractPdf',
+  'getPageLen', 'getPagePtr', 'getErrorLen', 'getErrorPtr',
+  'getPageCount', 'extractPageImages',
+  'getPageImageLen', 'getPageImagePtr', 'getPageImageKind',
+] as const;
+
+/** Thrown when an instantiated WASM module fails the ABI contract. */
+export class AlbexAbiMismatchError extends Error {
+  readonly module: 'main' | 'pdf';
+  readonly missing?: readonly string[];
+  readonly version?: number;
+  constructor(module: 'main' | 'pdf', message: string, opts?: { missing?: readonly string[]; version?: number }) {
+    super(message);
+    this.name = 'AlbexAbiMismatchError';
+    this.module = module;
+    if (opts?.missing) this.missing = opts.missing;
+    if (opts?.version !== undefined) this.version = opts.version;
+  }
+}
+
+function validateExports(
+  exports: WebAssembly.Exports,
+  required: readonly string[],
+  module: 'main' | 'pdf',
+  abiMin: number,
+  abiMax: number,
+): void {
+  const mem = (exports as Record<string, unknown>)['memory'];
+  if (!(mem instanceof WebAssembly.Memory)) {
+    throw new AlbexAbiMismatchError(module, `${module}: \`memory\` is missing or not a WebAssembly.Memory instance.`);
+  }
+  const missing: string[] = [];
+  for (const name of required) {
+    if (typeof (exports as Record<string, unknown>)[name] !== 'function') missing.push(name);
+  }
+  if (missing.length) {
+    throw new AlbexAbiMismatchError(
+      module,
+      `${module}: WASM binary missing required exports: ${missing.join(', ')}. ` +
+      `The .wasm was built with an incompatible source — rebuild with the current toolchain.`,
+      { missing },
+    );
+  }
+  const version = ((exports as Record<string, unknown>)['abiVersion'] as () => number)();
+  if (version < abiMin || version > abiMax) {
+    throw new AlbexAbiMismatchError(
+      module,
+      `${module}: abiVersion ${version} outside supported range [${abiMin}..${abiMax}]. ` +
+      `The host TypeScript expects a different binary — upgrade albex or rebuild the WASM.`,
+      { version },
+    );
+  }
+}
+
+/** Validate and narrow `WebAssembly.Exports` to the typed Albex main
+ * interface. Throws `AlbexAbiMismatchError` if the contract is broken. */
+export function asAlbexExports(exports: WebAssembly.Exports): AlbexWasmExports {
+  validateExports(exports, MAIN_REQUIRED, 'main', MAIN_ABI_MIN, MAIN_ABI_MAX);
+  return exports as unknown as AlbexWasmExports;
+}
+
+/** Validate and narrow `WebAssembly.Exports` to the typed PDF interface. */
+export function asAlbexPdfExports(exports: WebAssembly.Exports): AlbexPdfExports {
+  validateExports(exports, PDF_REQUIRED, 'pdf', PDF_ABI_MIN, PDF_ABI_MAX);
+  return exports as unknown as AlbexPdfExports;
+}

package/src/worker-protocol.ts

@@ -0,0 +1,48 @@
+/**
+ * Wire protocol between the main thread and the AlbexEngineWorker runtime.
+ *
+ * One request/response pair per call, identified by `id`. The runtime is
+ * single-threaded so we serialise requests on the main side (one in-flight
+ * call at a time per worker) — keeps the protocol trivial and matches the
+ * actual constraint of `static mut` WASM state.
+ *
+ * `Transferable` is opt-in per op; we use it for `indexFile` to avoid
+ * copying the file bytes into the worker.
+ */
+
+import type { AlbexOptions, IndexedDocument, SearchOptions, SearchResult, EngineStats, SearchStats } from './albex.js';
+
+export type WorkerOp =
+  | { kind: 'init';            opts: AlbexOptions }
+  | { kind: 'indexFile';        name: string; buffer: ArrayBuffer }
+  | { kind: 'search';           query: string; options: SearchOptions }
+  | { kind: 'removeDocument';   id: string }
+  | { kind: 'compact' }
+  | { kind: 'reset' }
+  | { kind: 'getStats' }
+  | { kind: 'getLastSearchStats' }
+  | { kind: 'getDocuments' }
+  | { kind: 'setMaxErrors';     n: 0 | 1 | 2 | 3 }
+  | { kind: 'setThreshold';     n: number }
+  | { kind: 'setMaxResults';    n: number }
+  | { kind: 'setLanguage';      lang: 'off' | 'es' }
+  | { kind: 'save';             name: string }
+  | { kind: 'load';             name: string }
+  | { kind: 'loadOrInit';       name: string }
+  | { kind: 'deleteSnapshot';   name: string }
+  | { kind: 'listSnapshots' };
+
+export interface WorkerRequest {
+  id: number;
+  op: WorkerOp;
+}
+
+export type WorkerResponse =
+  | { id: number; ok: true;  result: unknown }
+  | { id: number; ok: false; error: { name: string; kind?: string; message: string } };
+
+export type IndexFileResult = IndexedDocument;
+export type SearchResultArr = SearchResult[];
+export type StatsResult     = EngineStats;
+export type SearchStatsRes  = SearchStats | null;
+export type DocsResult      = readonly IndexedDocument[];

package/src/worker-runtime.ts

@@ -0,0 +1,106 @@
+/**
+ * Albex worker runtime.
+ *
+ * Loads inside a Web Worker, instantiates an `AlbexEngine`, and serves the
+ * wire protocol from `worker-protocol.ts`. Designed to be referenced as:
+ *
+ *     new Worker(new URL('./worker-runtime.js', import.meta.url),
+ *                { type: 'module' });
+ *
+ * The runtime is intentionally side-effectful (registers `onmessage` at
+ * import time). It is not meant to be imported from the main thread.
+ */
+
+import { AlbexEngine, AlbexError } from './albex.js';
+import type { WorkerRequest, WorkerResponse, WorkerOp } from './worker-protocol.js';
+
+let engine: AlbexEngine | null = null;
+
+function ensureEngine(): AlbexEngine {
+  if (!engine) throw new Error('Worker runtime: init() not called yet');
+  return engine;
+}
+
+async function dispatch(op: WorkerOp): Promise<unknown> {
+  switch (op.kind) {
+    case 'init': {
+      engine = new AlbexEngine(op.opts);
+      await engine.init();
+      return undefined;
+    }
+    case 'indexFile': {
+      // Wrap the transferred buffer in a File-like object so existing
+      // indexers work unchanged.
+      const file = new File([op.buffer], op.name);
+      return ensureEngine().indexFile(file);
+    }
+    case 'search':
+      return ensureEngine().search(op.query, op.options);
+    case 'removeDocument':
+      return ensureEngine().removeDocument(op.id);
+    case 'compact':
+      ensureEngine().compact();
+      return undefined;
+    case 'reset':
+      ensureEngine().reset();
+      return undefined;
+    case 'getStats':
+      return ensureEngine().getStats();
+    case 'getLastSearchStats':
+      return ensureEngine().getLastSearchStats();
+    case 'getDocuments':
+      return ensureEngine().documents.slice();
+    case 'setMaxErrors':
+      ensureEngine().setMaxErrors(op.n);
+      return undefined;
+    case 'setThreshold':
+      ensureEngine().setThreshold(op.n);
+      return undefined;
+    case 'setMaxResults':
+      ensureEngine().setMaxResults(op.n);
+      return undefined;
+    case 'setLanguage':
+      ensureEngine().setLanguage(op.lang);
+      return undefined;
+    case 'save':
+      return ensureEngine().save(op.name);
+    case 'load':
+      return ensureEngine().load(op.name);
+    case 'loadOrInit':
+      return ensureEngine().loadOrInit(op.name);
+    case 'deleteSnapshot':
+      return ensureEngine().deleteSnapshot(op.name);
+    case 'listSnapshots':
+      return ensureEngine().listSnapshots();
+  }
+}
+
+async function handle(req: WorkerRequest): Promise<void> {
+  const { id, op } = req;
+  try {
+    const result = await dispatch(op);
+    const res: WorkerResponse = { id, ok: true, result };
+    (self as unknown as Worker).postMessage(res);
+  } catch (err) {
+    const e = err as Error & { kind?: string };
+    const res: WorkerResponse = {
+      id, ok: false,
+      error: {
+        name:    e.name ?? 'Error',
+        kind:    err instanceof AlbexError ? err.kind : undefined,
+        message: e.message ?? String(err),
+      },
+    };
+    (self as unknown as Worker).postMessage(res);
+  }
+}
+
+// Process messages strictly in arrival order. The engine guards its own
+// state, but a sync `search` arriving mid-`indexFile` await would otherwise
+// be rejected as "busy"; queueing keeps the worker's externally-observable
+// behaviour serial and matches the main-thread engine's serialization.
+let _queue: Promise<void> = Promise.resolve();
+self.onmessage = (ev: MessageEvent<WorkerRequest>) => {
+  const req = ev.data;
+  _queue = _queue.then(() => handle(req));
+};