albex 0.1.0 → 0.3.0

package/src/resource-manager.ts

@@ -0,0 +1,167 @@
+/**
+ * Resource manager — listens to environmental signals and exposes them as
+ * a small event API consumed by `AlbexEngine` (and `AlbexEngineWorker`).
+ *
+ * The signals tracked:
+ *
+ *   - **Visibility** — `document.visibilitychange`. When the tab is hidden
+ *     the engine should pause speculative work (background indexing,
+ *     prefetch of optional binaries) but must still answer in-flight queries.
+ *
+ *   - **Battery** — `navigator.getBattery()`. When level <20% AND not
+ *     charging, switch to low-power mode (smaller worker pool, longer
+ *     frame budget yields, no GPU acceleration).
+ *
+ *   - **Connection** — `navigator.connection.effectiveType` + `saveData`.
+ *     On `'slow-2g'/'2g'` or `saveData === true`, defer optional downloads
+ *     (PDF wasm, embedding model) until the user explicitly needs them.
+ *
+ * The manager is *passive*: it does not call into the engine. Instead it
+ * exposes a `state` snapshot and an `on(event, callback)` subscription so
+ * the engine can react with its own policy. This keeps the dependency
+ * direction one-way and lets the engine be tested without the DOM.
+ */
+
+export type ResourceMode = 'normal' | 'low-power' | 'background' | 'constrained-network';
+
+export interface ResourceState {
+  visible: boolean;
+  lowPower: boolean;
+  constrainedNetwork: boolean;
+  /** Composite mode derived from the three signals above. */
+  mode: ResourceMode;
+}
+
+type Listener = (state: ResourceState) => void;
+
+interface BatteryLike {
+  level: number;
+  charging: boolean;
+  addEventListener?: (type: string, cb: () => void) => void;
+  removeEventListener?: (type: string, cb: () => void) => void;
+}
+
+interface ConnectionLike {
+  effectiveType?: string;
+  saveData?: boolean;
+  addEventListener?: (type: string, cb: () => void) => void;
+  removeEventListener?: (type: string, cb: () => void) => void;
+}
+
+export class ResourceManager {
+  private _state: ResourceState = {
+    visible: true,
+    lowPower: false,
+    constrainedNetwork: false,
+    mode: 'normal',
+  };
+  private _listeners = new Set<Listener>();
+  private _battery: BatteryLike | null = null;
+  private _connection: ConnectionLike | null = null;
+  private _onVisibility = (): void => this._refresh();
+  private _onBatteryChange = (): void => this._refresh();
+  private _onConnChange = (): void => this._refresh();
+  private _started = false;
+
+  get state(): ResourceState { return this._state; }
+
+  /** Subscribe to changes. Returns an unsubscribe function. */
+  on(cb: Listener): () => void {
+    this._listeners.add(cb);
+    return () => this._listeners.delete(cb);
+  }
+
+  /**
+   * Start listening. Idempotent. Safe to call from non-browser environments
+   * (Node tests, Workers without DOM access) — missing APIs are tolerated.
+   */
+  async start(): Promise<void> {
+    if (this._started) return;
+    this._started = true;
+
+    if (typeof document !== 'undefined' && document.addEventListener) {
+      document.addEventListener('visibilitychange', this._onVisibility);
+    }
+
+    try {
+      // @ts-expect-error Battery API is non-standard in TS typings
+      const getBat: (() => Promise<BatteryLike>) | undefined = navigator?.getBattery?.bind(navigator);
+      if (getBat) {
+        const b = await getBat();
+        this._battery = b;
+        b.addEventListener?.('levelchange',    this._onBatteryChange);
+        b.addEventListener?.('chargingchange', this._onBatteryChange);
+      }
+    } catch { /* unavailable; tolerate */ }
+
+    const conn = (navigator as unknown as { connection?: ConnectionLike } | undefined)?.connection;
+    if (conn) {
+      this._connection = conn;
+      conn.addEventListener?.('change', this._onConnChange);
+    }
+
+    this._refresh();
+  }
+
+  /** Tear down listeners. */
+  stop(): void {
+    if (!this._started) return;
+    this._started = false;
+
+    if (typeof document !== 'undefined' && document.removeEventListener) {
+      document.removeEventListener('visibilitychange', this._onVisibility);
+    }
+    this._battery?.removeEventListener?.('levelchange',    this._onBatteryChange);
+    this._battery?.removeEventListener?.('chargingchange', this._onBatteryChange);
+    this._connection?.removeEventListener?.('change', this._onConnChange);
+    this._battery = null;
+    this._connection = null;
+  }
+
+  private _refresh(): void {
+    const visible = typeof document !== 'undefined'
+      ? document.visibilityState === 'visible'
+      : true;
+
+    const lowPower = !!(this._battery
+      && this._battery.level < 0.2
+      && this._battery.charging === false);
+
+    const conn = this._connection;
+    const constrainedNetwork = !!conn && (
+      conn.saveData === true ||
+      conn.effectiveType === 'slow-2g' ||
+      conn.effectiveType === '2g'
+    );
+
+    let mode: ResourceMode = 'normal';
+    if (!visible)               mode = 'background';
+    else if (lowPower)          mode = 'low-power';
+    else if (constrainedNetwork) mode = 'constrained-network';
+
+    const next: ResourceState = { visible, lowPower, constrainedNetwork, mode };
+    if (
+      next.visible === this._state.visible &&
+      next.lowPower === this._state.lowPower &&
+      next.constrainedNetwork === this._state.constrainedNetwork &&
+      next.mode === this._state.mode
+    ) return;
+
+    this._state = next;
+    for (const cb of this._listeners) {
+      try { cb(next); } catch { /* swallow listener errors */ }
+    }
+  }
+}
+
+/**
+ * Singleton accessor. Multiple engines in the same realm share the same
+ * manager — there is no benefit to running the listeners more than once,
+ * and the signal is global to the page anyway.
+ */
+let _instance: ResourceManager | null = null;
+
+export function getResourceManager(): ResourceManager {
+  if (!_instance) _instance = new ResourceManager();
+  return _instance;
+}

package/src/tiered-store.ts

@@ -0,0 +1,259 @@
+/**
+ * Tiered storage layer for Albex.
+ *
+ * The base engine keeps every indexed document in memory inside the BSS
+ * arrays. That works beautifully up to the tier's capacity (4 MB to 128 MB
+ * of indexed text) but breaks when the user wants to search across more.
+ *
+ * `TieredStore` adds two memory tiers behind the engine:
+ *
+ *   HOT  — already-indexed documents living in the WASM arrays.
+ *   WARM — original file blobs serialised in OPFS, NOT in the engine.
+ *
+ * Eviction happens when text capacity climbs above a configurable
+ * threshold (default 85 %). The least-recently-accessed HOT document is
+ * removed from the engine; its blob stays in OPFS so we can promote it
+ * back later without asking the user to re-pick the file.
+ *
+ * Promotion happens explicitly: callers tell the store "I want these
+ * names searchable again" and we re-feed them through `engine.indexFile`.
+ *
+ * **Trade-off vs storing the internal representation in OPFS:** promoting
+ * a doc means re-parsing it (DOCX XML decode, etc.). For typical document
+ * sizes this is 20-200 ms — negligible for an explicit "search across the
+ * archive" action. The win is huge: the persistence format is just the
+ * source files, so it survives engine version bumps without any migration.
+ */
+
+import type { AlbexEngine, IndexedDocument } from './albex.js';
+
+const OPFS_DIR = 'albex-tiered';
+
+interface TieredEntry {
+  name: string;
+  ext: string;
+  lastAccessedMs: number;
+  /** Whether the doc is currently indexed in the engine. */
+  hot: boolean;
+  /** Size of the original blob in bytes, for capacity estimation. */
+  byteSize: number;
+}
+
+export interface TieredStoreOptions {
+  /**
+   * Evict when textUsed exceeds `evictThreshold * textCapacity`.
+   * Default 0.85. Set 1.0 to disable.
+   */
+  evictThreshold?: number;
+  /**
+   * Keep this many documents in the hot tier at minimum. Default 1.
+   * Useful when you want to ensure the most recently added file is
+   * always available without an explicit promote step.
+   */
+  hotFloor?: number;
+}
+
+export class TieredStore {
+  private readonly _engine: AlbexEngine;
+  private _entries = new Map<string, TieredEntry>();
+  private _dir: FileSystemDirectoryHandle | null = null;
+  private readonly _opts: Required<TieredStoreOptions>;
+
+  constructor(engine: AlbexEngine, opts: TieredStoreOptions = {}) {
+    this._engine = engine;
+    this._opts = {
+      evictThreshold: opts.evictThreshold ?? 0.85,
+      hotFloor:       opts.hotFloor       ?? 1,
+    };
+  }
+
+  /** Ensure the OPFS directory exists. Idempotent. Tolerated when OPFS is unavailable. */
+  async init(): Promise<void> {
+    try {
+      const root = await navigator.storage.getDirectory();
+      this._dir = await root.getDirectoryHandle(OPFS_DIR, { create: true });
+    } catch {
+      // No OPFS — warm tier disabled; everything stays hot.
+      this._dir = null;
+    }
+    await this._rehydrateIndex();
+  }
+
+  /**
+   * Index a file AND register it in the warm tier so it survives across
+   * sessions. Equivalent to `engine.indexFile(file)` but with the extra
+   * persistence guarantee.
+   */
+  async indexFile(file: File): Promise<IndexedDocument> {
+    const doc = await this._engine.indexFile(file);
+
+    // Persist the original blob in OPFS so we can promote it back later
+    // without asking the user to re-pick the file.
+    await this._writeBlob(file);
+
+    this._entries.set(doc.name, {
+      name:           doc.name,
+      ext:            doc.ext,
+      lastAccessedMs: Date.now(),
+      hot:            true,
+      byteSize:       file.size,
+    });
+
+    await this._enforceCapacity();
+    return doc;
+  }
+
+  /** Touch an entry to mark it recently used (for LRU). */
+  touch(name: string): void {
+    const e = this._entries.get(name);
+    if (e) e.lastAccessedMs = Date.now();
+  }
+
+  /**
+   * Evict the least-recently-used HOT documents until `textUsed` falls
+   * below the configured threshold. Respects `hotFloor` (never evicts
+   * the last N docs).
+   */
+  async _enforceCapacity(): Promise<void> {
+    const stats = this._engine.getStats();
+    const threshold = stats.textCapacity * this._opts.evictThreshold;
+    if (stats.textUsed <= threshold) return;
+
+    const hot = [...this._entries.values()]
+      .filter(e => e.hot)
+      .sort((a, b) => a.lastAccessedMs - b.lastAccessedMs);
+
+    while (hot.length > this._opts.hotFloor) {
+      const victim = hot.shift()!;
+      this._engine.removeDocument(victim.name);
+      victim.hot = false;
+      // Reclaim storage now so subsequent index calls see real headroom.
+      this._engine.compact();
+      const after = this._engine.getStats();
+      if (after.textUsed <= threshold) break;
+    }
+  }
+
+  /**
+   * Bring a warm document back into the engine. No-op if already hot.
+   * Returns the resulting `IndexedDocument` or `null` if the doc isn't known.
+   */
+  async promote(name: string): Promise<IndexedDocument | null> {
+    const e = this._entries.get(name);
+    if (!e) return null;
+    if (e.hot) {
+      this.touch(name);
+      const stats = this._engine.getStats();
+      void stats;
+      return this._engine.documents.find(d => d.name === name) ?? null;
+    }
+
+    const blob = await this._readBlob(name);
+    if (!blob) return null;
+
+    // Re-create the File so `engine.indexFile` sees the original metadata.
+    const file = new File([blob], name);
+    const doc = await this._engine.indexFile(file);
+    e.hot = true;
+    e.lastAccessedMs = Date.now();
+    return doc;
+  }
+
+  /**
+   * Forget a document entirely: remove from engine and delete its OPFS blob.
+   * Returns whether the entry existed.
+   */
+  async forget(name: string): Promise<boolean> {
+    const had = this._entries.has(name);
+    if (this._entries.get(name)?.hot) this._engine.removeDocument(name);
+    this._entries.delete(name);
+    await this._deleteBlob(name);
+    return had;
+  }
+
+  /** Names of all known documents, hot or warm. */
+  list(): { name: string; hot: boolean; byteSize: number }[] {
+    return [...this._entries.values()].map(e => ({
+      name: e.name, hot: e.hot, byteSize: e.byteSize,
+    }));
+  }
+
+  /** Aggregate storage stats. */
+  getTierStats(): { hot: number; warm: number; totalBytes: number } {
+    let hot = 0, warm = 0, totalBytes = 0;
+    for (const e of this._entries.values()) {
+      if (e.hot) hot++; else warm++;
+      totalBytes += e.byteSize;
+    }
+    return { hot, warm, totalBytes };
+  }
+
+  // ── OPFS plumbing ─────────────────────────────────────────────────────
+
+  private _safeName(name: string): string {
+    // Strip path separators just in case; OPFS requires plain file names.
+    return name.replace(/[/\\]/g, '_');
+  }
+
+  private async _writeBlob(file: File): Promise<void> {
+    if (!this._dir) return;
+    try {
+      const handle = await this._dir.getFileHandle(this._safeName(file.name), { create: true });
+      const w = await handle.createWritable();
+      const bytes = new Uint8Array(await file.arrayBuffer());
+      const plain = new Uint8Array(bytes.byteLength);
+      plain.set(bytes);
+      await w.write(plain);
+      await w.close();
+    } catch (e) {
+      console.warn(`[albex] failed to persist blob for ${file.name}:`, e);
+    }
+  }
+
+  private async _readBlob(name: string): Promise<Blob | null> {
+    if (!this._dir) return null;
+    try {
+      const handle = await this._dir.getFileHandle(this._safeName(name));
+      return await handle.getFile();
+    } catch {
+      return null;
+    }
+  }
+
+  private async _deleteBlob(name: string): Promise<void> {
+    if (!this._dir) return;
+    try { await this._dir.removeEntry(this._safeName(name)); } catch { /* not found */ }
+  }
+
+  /**
+   * TC39 explicit-resource-management hook. Drops the in-memory index and
+   * the OPFS directory handle. The underlying OPFS blobs are NOT deleted —
+   * disposal only frees JS-side state. Use `forget()` per-doc or
+   * `deleteOpfsAll()` if you want to wipe persisted data.
+   */
+  [Symbol.dispose](): void {
+    this._entries.clear();
+    this._dir = null;
+  }
+
+  private async _rehydrateIndex(): Promise<void> {
+    if (!this._dir) return;
+    try {
+      // @ts-expect-error async-iterable on FileSystemDirectoryHandle
+      for await (const [name, handle] of this._dir.entries()) {
+        if (handle.kind !== 'file') continue;
+        const file = await handle.getFile();
+        if (this._entries.has(name)) continue;
+        this._entries.set(name, {
+          name,
+          ext:            (name.split('.').pop() ?? '').toLowerCase(),
+          lastAccessedMs: 0, // never accessed in this session yet
+          hot:            false,
+          byteSize:       file.size,
+        });
+      }
+    } catch (e) {
+      console.warn('[albex] could not rehydrate tiered index:', e);
+    }
+  }
+}

package/src/wasm-bindings.ts

@@ -0,0 +1,200 @@
+/**
+ * Typed interfaces for the two WASM modules Albex ships with.
+ *
+ * These types replace ad-hoc `as Function` casts and give us:
+ *  - argument/return type checking at call sites,
+ *  - autocompletion in IDEs,
+ *  - safe refactors when an export name or signature changes.
+ *
+ * The interfaces mirror the `#[no_mangle] pub extern "C" fn` exports
+ * in `wasm/src/lib.rs` and `pdf-wasm/src/lib.rs`.
+ */
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Main WASM module (albex_wasm_bg.wasm)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface AlbexWasmExports {
+  readonly memory: WebAssembly.Memory;
+
+  // Scratchpad / lifecycle
+  getBuffer(size: number): number;
+  init(): void;
+
+  // Document ingestion
+  setDocumentName(len: number): void;
+  beginDocument(): number;
+  feedXmlBytes(len: number): void;
+  endDocument(): number;
+
+  // XLSX
+  beginXlsx(): number;
+  feedXlsxBytes(len: number): void;
+
+  // Generic text path (PDF, TXT, XML, future formats)
+  feedText(len: number): void;
+  flushParagraph(): void;
+
+  // Search configuration
+  setMaxErrors(errors: number): void;
+  setThreshold(threshold: number): void;
+  setMaxResults(max: number): void;
+
+  // Search execution
+  setPattern(len: number): number;
+  search(): number;
+  // Resumable search (used by frame-budgeted searches and worker shards)
+  searchBegin(): number;
+  searchSlice(maxChunks: number): number;
+  getSearchCursor(): number;
+  getSearchTotal(): number;
+
+  // Result accessors
+  getResultCount(): number;
+  getResultDocId(i: number): number;
+  getResultLocation(i: number): number;
+  getResultScore(i: number): number;
+  getResultStart(i: number): number;
+  getResultEnd(i: number): number;
+  getResultChunkIdx(i: number): number;
+  getResultDocName(i: number): number;
+  getResultMatchCount(i: number): number;
+  getResultMatchStartAt(i: number, k: number): number;
+  getResultMatchEndAt(i: number, k: number): number;
+  getSnippet(i: number): number;
+  getSnippetWindow(i: number, before: number, after: number): number;
+  getSnippetWindowOffset(): number;
+
+  // Stats
+  getStatBloomTested(): number;
+  getStatBloomPassed(): number;
+  getStatBitapMatched(): number;
+  getChunkCount(): number;
+  getDocCount(): number;
+  getTextUsed(): number;
+  getTextCapacity(): number;
+
+  // Snapshot / restore
+  snapshotSize(): number;
+  snapshotChunk(offset: number, maxLen: number): number;
+  restoreBegin(): number;
+  restoreFeed(len: number): number;
+
+  // Incremental / per-doc
+  getDocId(index: number): number;
+  getDocChunkCount(index: number): number;
+  getDocName(index: number): number;
+  isDocDeleted(index: number): number;
+  removeDocument(docId: number): number;
+  compact(): void;
+
+  /**
+   * Per-document content hash (snapshot v2). Returns a pointer to 8 bytes
+   * holding the FNV-1a 64-bit hash of the original file bytes, or 0 if the
+   * doc index is out of range. All-zero bytes mean "hash not available"
+   * — either the host never called setDocumentContentHash for this doc
+   * (legacy code path) or it was restored from a v1 snapshot.
+   */
+  getDocContentHashPtr(index: number): number;
+  /** Always returns 8. Useful as a runtime feature-detect: older binaries
+   * without snapshot v2 will not export this function at all. */
+  getDocContentHashLen(): number;
+  /** Copy a content hash (up to 8 bytes from the scratchpad) into the
+   * pending slot. endDocument() then writes it into doc_hashes[]. */
+  setDocumentContentHash(len: number): void;
+
+  // Stemming
+  setLanguage(lang: number): void;
+
+  // Tier identification
+  getTier(): number;        // 1=mini, 2=std, 3=pro
+  getMaxChunks(): number;
+  getMaxDocs(): number;
+  getNameCapacity(): number;
+
+  // GPU bridge (CD1): zero-copy access to chunk array + candidate mask
+  getChunksPtr(): number;
+  getChunkStructSize(): number;
+  setCandidateMask(byteLen: number): void;
+  clearCandidateMask(): void;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// PDF WASM module (albex_pdf.wasm)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface AlbexPdfExports {
+  readonly memory: WebAssembly.Memory;
+
+  /** Reserve `len` bytes inside the PDF module and return a pointer. */
+  allocInput(len: number): number;
+
+  /**
+   * Parse the PDF. Returns:
+   *   N ≥ 0 — page count,
+   *   -1    — parse error (read with getErrorPtr/getErrorLen),
+   *   -2    — image-only / no extractable text.
+   *
+   * When `-2` is returned, the host can fall through to the scanned-PDF
+   * path via `getPageCount` + `extractPageImages`.
+   */
+  extractPdf(len: number): number;
+
+  getPageLen(page: number): number;
+  getPagePtr(page: number): number;
+  getErrorLen(): number;
+  getErrorPtr(): number;
+
+  // ── Scanned-PDF path (extracts embedded image XObjects) ──────────────────
+  //
+  // Available since pdf-wasm 0.2. Older binaries built before this addition
+  // will not expose these exports — `asAlbexPdfExports` keeps them typed,
+  // but the engine should feature-detect at runtime before relying on them.
+
+  /** Total page count of the loaded PDF; 0 if the input cannot be parsed. */
+  getPageCount(): number;
+
+  /**
+   * Extract every supported image XObject on page `page` (0-based) into the
+   * module's internal buffer.
+   *
+   * Returns:
+   *   N ≥ 0 — number of images extracted on this page,
+   *   -1    — parse error or page index out of range.
+   *
+   * Each extracted image is one of:
+   *   * JPEG (kind = 1, from a `/DCTDecode` filter), or
+   *   * JPEG2000 (kind = 2, from a `/JPXDecode` filter).
+   *
+   * Filters that require Rust-side reconstruction (`FlateDecode`,
+   * `CCITTFaxDecode`, `JBIG2Decode`) are intentionally skipped — they
+   * would roughly double the binary size for ~5 % more coverage.
+   */
+  extractPageImages(page: number): number;
+
+  /** Byte length of extracted image `i` (from the last `extractPageImages`). */
+  getPageImageLen(i: number): number;
+
+  /** Pointer to extracted image `i`'s raw bytes. */
+  getPageImagePtr(i: number): number;
+
+  /** Format tag for extracted image `i`: 1 = JPEG, 2 = JPEG2000, 0 = none. */
+  getPageImageKind(i: number): number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Narrowing helpers for instantiation results
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Cast `WebAssembly.Exports` to the typed Albex main interface.
+ * Runtime check is intentionally minimal — if the .wasm doesn't match,
+ * the first call site that touches a missing function throws naturally.
+ */
+export function asAlbexExports(exports: WebAssembly.Exports): AlbexWasmExports {
+  return exports as unknown as AlbexWasmExports;
+}
+
+export function asAlbexPdfExports(exports: WebAssembly.Exports): AlbexPdfExports {
+  return exports as unknown as AlbexPdfExports;
+}

package/src/worker-protocol.ts

@@ -0,0 +1,48 @@
+/**
+ * Wire protocol between the main thread and the AlbexEngineWorker runtime.
+ *
+ * One request/response pair per call, identified by `id`. The runtime is
+ * single-threaded so we serialise requests on the main side (one in-flight
+ * call at a time per worker) — keeps the protocol trivial and matches the
+ * actual constraint of `static mut` WASM state.
+ *
+ * `Transferable` is opt-in per op; we use it for `indexFile` to avoid
+ * copying the file bytes into the worker.
+ */
+
+import type { AlbexOptions, IndexedDocument, SearchOptions, SearchResult, EngineStats, SearchStats } from './albex.js';
+
+export type WorkerOp =
+  | { kind: 'init';            opts: AlbexOptions }
+  | { kind: 'indexFile';        name: string; buffer: ArrayBuffer }
+  | { kind: 'search';           query: string; options: SearchOptions }
+  | { kind: 'removeDocument';   id: string }
+  | { kind: 'compact' }
+  | { kind: 'reset' }
+  | { kind: 'getStats' }
+  | { kind: 'getLastSearchStats' }
+  | { kind: 'getDocuments' }
+  | { kind: 'setMaxErrors';     n: 0 | 1 | 2 | 3 }
+  | { kind: 'setThreshold';     n: number }
+  | { kind: 'setMaxResults';    n: number }
+  | { kind: 'setLanguage';      lang: 'off' | 'es' }
+  | { kind: 'save';             name: string }
+  | { kind: 'load';             name: string }
+  | { kind: 'loadOrInit';       name: string }
+  | { kind: 'deleteSnapshot';   name: string }
+  | { kind: 'listSnapshots' };
+
+export interface WorkerRequest {
+  id: number;
+  op: WorkerOp;
+}
+
+export type WorkerResponse =
+  | { id: number; ok: true;  result: unknown }
+  | { id: number; ok: false; error: { name: string; kind?: string; message: string } };
+
+export type IndexFileResult = IndexedDocument;
+export type SearchResultArr = SearchResult[];
+export type StatsResult     = EngineStats;
+export type SearchStatsRes  = SearchStats | null;
+export type DocsResult      = readonly IndexedDocument[];