albex 0.1.0 → 0.6.0

package/src/resource-manager.ts

@@ -0,0 +1,167 @@
+/**
+ * Resource manager — listens to environmental signals and exposes them as
+ * a small event API consumed by `AlbexEngine` (and `AlbexEngineWorker`).
+ *
+ * The signals tracked:
+ *
+ *   - **Visibility** — `document.visibilitychange`. When the tab is hidden
+ *     the engine should pause speculative work (background indexing,
+ *     prefetch of optional binaries) but must still answer in-flight queries.
+ *
+ *   - **Battery** — `navigator.getBattery()`. When level <20% AND not
+ *     charging, switch to low-power mode (smaller worker pool, longer
+ *     frame budget yields, no GPU acceleration).
+ *
+ *   - **Connection** — `navigator.connection.effectiveType` + `saveData`.
+ *     On `'slow-2g'/'2g'` or `saveData === true`, defer optional downloads
+ *     (PDF wasm, embedding model) until the user explicitly needs them.
+ *
+ * The manager is *passive*: it does not call into the engine. Instead it
+ * exposes a `state` snapshot and an `on(event, callback)` subscription so
+ * the engine can react with its own policy. This keeps the dependency
+ * direction one-way and lets the engine be tested without the DOM.
+ */
+
+export type ResourceMode = 'normal' | 'low-power' | 'background' | 'constrained-network';
+
+export interface ResourceState {
+  visible: boolean;
+  lowPower: boolean;
+  constrainedNetwork: boolean;
+  /** Composite mode derived from the three signals above. */
+  mode: ResourceMode;
+}
+
+type Listener = (state: ResourceState) => void;
+
+interface BatteryLike {
+  level: number;
+  charging: boolean;
+  addEventListener?: (type: string, cb: () => void) => void;
+  removeEventListener?: (type: string, cb: () => void) => void;
+}
+
+interface ConnectionLike {
+  effectiveType?: string;
+  saveData?: boolean;
+  addEventListener?: (type: string, cb: () => void) => void;
+  removeEventListener?: (type: string, cb: () => void) => void;
+}
+
+export class ResourceManager {
+  private _state: ResourceState = {
+    visible: true,
+    lowPower: false,
+    constrainedNetwork: false,
+    mode: 'normal',
+  };
+  private _listeners = new Set<Listener>();
+  private _battery: BatteryLike | null = null;
+  private _connection: ConnectionLike | null = null;
+  private _onVisibility = (): void => this._refresh();
+  private _onBatteryChange = (): void => this._refresh();
+  private _onConnChange = (): void => this._refresh();
+  private _started = false;
+
+  get state(): ResourceState { return this._state; }
+
+  /** Subscribe to changes. Returns an unsubscribe function. */
+  on(cb: Listener): () => void {
+    this._listeners.add(cb);
+    return () => this._listeners.delete(cb);
+  }
+
+  /**
+   * Start listening. Idempotent. Safe to call from non-browser environments
+   * (Node tests, Workers without DOM access) — missing APIs are tolerated.
+   */
+  async start(): Promise<void> {
+    if (this._started) return;
+    this._started = true;
+
+    if (typeof document !== 'undefined' && document.addEventListener) {
+      document.addEventListener('visibilitychange', this._onVisibility);
+    }
+
+    try {
+      // @ts-expect-error Battery API is non-standard in TS typings
+      const getBat: (() => Promise<BatteryLike>) | undefined = navigator?.getBattery?.bind(navigator);
+      if (getBat) {
+        const b = await getBat();
+        this._battery = b;
+        b.addEventListener?.('levelchange',    this._onBatteryChange);
+        b.addEventListener?.('chargingchange', this._onBatteryChange);
+      }
+    } catch { /* unavailable; tolerate */ }
+
+    const conn = (navigator as unknown as { connection?: ConnectionLike } | undefined)?.connection;
+    if (conn) {
+      this._connection = conn;
+      conn.addEventListener?.('change', this._onConnChange);
+    }
+
+    this._refresh();
+  }
+
+  /** Tear down listeners. */
+  stop(): void {
+    if (!this._started) return;
+    this._started = false;
+
+    if (typeof document !== 'undefined' && document.removeEventListener) {
+      document.removeEventListener('visibilitychange', this._onVisibility);
+    }
+    this._battery?.removeEventListener?.('levelchange',    this._onBatteryChange);
+    this._battery?.removeEventListener?.('chargingchange', this._onBatteryChange);
+    this._connection?.removeEventListener?.('change', this._onConnChange);
+    this._battery = null;
+    this._connection = null;
+  }
+
+  private _refresh(): void {
+    const visible = typeof document !== 'undefined'
+      ? document.visibilityState === 'visible'
+      : true;
+
+    const lowPower = !!(this._battery
+      && this._battery.level < 0.2
+      && this._battery.charging === false);
+
+    const conn = this._connection;
+    const constrainedNetwork = !!conn && (
+      conn.saveData === true ||
+      conn.effectiveType === 'slow-2g' ||
+      conn.effectiveType === '2g'
+    );
+
+    let mode: ResourceMode = 'normal';
+    if (!visible)               mode = 'background';
+    else if (lowPower)          mode = 'low-power';
+    else if (constrainedNetwork) mode = 'constrained-network';
+
+    const next: ResourceState = { visible, lowPower, constrainedNetwork, mode };
+    if (
+      next.visible === this._state.visible &&
+      next.lowPower === this._state.lowPower &&
+      next.constrainedNetwork === this._state.constrainedNetwork &&
+      next.mode === this._state.mode
+    ) return;
+
+    this._state = next;
+    for (const cb of this._listeners) {
+      try { cb(next); } catch { /* swallow listener errors */ }
+    }
+  }
+}
+
+/**
+ * Singleton accessor. Multiple engines in the same realm share the same
+ * manager — there is no benefit to running the listeners more than once,
+ * and the signal is global to the page anyway.
+ */
+let _instance: ResourceManager | null = null;
+
+export function getResourceManager(): ResourceManager {
+  if (!_instance) _instance = new ResourceManager();
+  return _instance;
+}

package/src/tiered-store.ts

@@ -0,0 +1,259 @@
+/**
+ * Tiered storage layer for Albex.
+ *
+ * The base engine keeps every indexed document in memory inside the BSS
+ * arrays. That works beautifully up to the tier's capacity (4 MB to 128 MB
+ * of indexed text) but breaks when the user wants to search across more.
+ *
+ * `TieredStore` adds two memory tiers behind the engine:
+ *
+ *   HOT  — already-indexed documents living in the WASM arrays.
+ *   WARM — original file blobs serialised in OPFS, NOT in the engine.
+ *
+ * Eviction happens when text capacity climbs above a configurable
+ * threshold (default 85 %). The least-recently-accessed HOT document is
+ * removed from the engine; its blob stays in OPFS so we can promote it
+ * back later without asking the user to re-pick the file.
+ *
+ * Promotion happens explicitly: callers tell the store "I want these
+ * names searchable again" and we re-feed them through `engine.indexFile`.
+ *
+ * **Trade-off vs storing the internal representation in OPFS:** promoting
+ * a doc means re-parsing it (DOCX XML decode, etc.). For typical document
+ * sizes this is 20-200 ms — negligible for an explicit "search across the
+ * archive" action. The win is huge: the persistence format is just the
+ * source files, so it survives engine version bumps without any migration.
+ */
+
+import type { AlbexEngine, IndexedDocument } from './albex.js';
+
+const OPFS_DIR = 'albex-tiered';
+
+interface TieredEntry {
+  name: string;
+  ext: string;
+  lastAccessedMs: number;
+  /** Whether the doc is currently indexed in the engine. */
+  hot: boolean;
+  /** Size of the original blob in bytes, for capacity estimation. */
+  byteSize: number;
+}
+
+export interface TieredStoreOptions {
+  /**
+   * Evict when textUsed exceeds `evictThreshold * textCapacity`.
+   * Default 0.85. Set 1.0 to disable.
+   */
+  evictThreshold?: number;
+  /**
+   * Keep this many documents in the hot tier at minimum. Default 1.
+   * Useful when you want to ensure the most recently added file is
+   * always available without an explicit promote step.
+   */
+  hotFloor?: number;
+}
+
+export class TieredStore {
+  private readonly _engine: AlbexEngine;
+  private _entries = new Map<string, TieredEntry>();
+  private _dir: FileSystemDirectoryHandle | null = null;
+  private readonly _opts: Required<TieredStoreOptions>;
+
+  constructor(engine: AlbexEngine, opts: TieredStoreOptions = {}) {
+    this._engine = engine;
+    this._opts = {
+      evictThreshold: opts.evictThreshold ?? 0.85,
+      hotFloor:       opts.hotFloor       ?? 1,
+    };
+  }
+
+  /** Ensure the OPFS directory exists. Idempotent. Tolerated when OPFS is unavailable. */
+  async init(): Promise<void> {
+    try {
+      const root = await navigator.storage.getDirectory();
+      this._dir = await root.getDirectoryHandle(OPFS_DIR, { create: true });
+    } catch {
+      // No OPFS — warm tier disabled; everything stays hot.
+      this._dir = null;
+    }
+    await this._rehydrateIndex();
+  }
+
+  /**
+   * Index a file AND register it in the warm tier so it survives across
+   * sessions. Equivalent to `engine.indexFile(file)` but with the extra
+   * persistence guarantee.
+   */
+  async indexFile(file: File): Promise<IndexedDocument> {
+    const doc = await this._engine.indexFile(file);
+
+    // Persist the original blob in OPFS so we can promote it back later
+    // without asking the user to re-pick the file.
+    await this._writeBlob(file);
+
+    this._entries.set(doc.name, {
+      name:           doc.name,
+      ext:            doc.ext,
+      lastAccessedMs: Date.now(),
+      hot:            true,
+      byteSize:       file.size,
+    });
+
+    await this._enforceCapacity();
+    return doc;
+  }
+
+  /** Touch an entry to mark it recently used (for LRU). */
+  touch(name: string): void {
+    const e = this._entries.get(name);
+    if (e) e.lastAccessedMs = Date.now();
+  }
+
+  /**
+   * Evict the least-recently-used HOT documents until `textUsed` falls
+   * below the configured threshold. Respects `hotFloor` (never evicts
+   * the last N docs).
+   */
+  async _enforceCapacity(): Promise<void> {
+    const stats = this._engine.getStats();
+    const threshold = stats.textCapacity * this._opts.evictThreshold;
+    if (stats.textUsed <= threshold) return;
+
+    const hot = [...this._entries.values()]
+      .filter(e => e.hot)
+      .sort((a, b) => a.lastAccessedMs - b.lastAccessedMs);
+
+    while (hot.length > this._opts.hotFloor) {
+      const victim = hot.shift()!;
+      this._engine.removeDocument(victim.name);
+      victim.hot = false;
+      // Reclaim storage now so subsequent index calls see real headroom.
+      this._engine.compact();
+      const after = this._engine.getStats();
+      if (after.textUsed <= threshold) break;
+    }
+  }
+
+  /**
+   * Bring a warm document back into the engine. No-op if already hot.
+   * Returns the resulting `IndexedDocument` or `null` if the doc isn't known.
+   */
+  async promote(name: string): Promise<IndexedDocument | null> {
+    const e = this._entries.get(name);
+    if (!e) return null;
+    if (e.hot) {
+      this.touch(name);
+      const stats = this._engine.getStats();
+      void stats;
+      return this._engine.documents.find(d => d.name === name) ?? null;
+    }
+
+    const blob = await this._readBlob(name);
+    if (!blob) return null;
+
+    // Re-create the File so `engine.indexFile` sees the original metadata.
+    const file = new File([blob], name);
+    const doc = await this._engine.indexFile(file);
+    e.hot = true;
+    e.lastAccessedMs = Date.now();
+    return doc;
+  }
+
+  /**
+   * Forget a document entirely: remove from engine and delete its OPFS blob.
+   * Returns whether the entry existed.
+   */
+  async forget(name: string): Promise<boolean> {
+    const had = this._entries.has(name);
+    if (this._entries.get(name)?.hot) this._engine.removeDocument(name);
+    this._entries.delete(name);
+    await this._deleteBlob(name);
+    return had;
+  }
+
+  /** Names of all known documents, hot or warm. */
+  list(): { name: string; hot: boolean; byteSize: number }[] {
+    return [...this._entries.values()].map(e => ({
+      name: e.name, hot: e.hot, byteSize: e.byteSize,
+    }));
+  }
+
+  /** Aggregate storage stats. */
+  getTierStats(): { hot: number; warm: number; totalBytes: number } {
+    let hot = 0, warm = 0, totalBytes = 0;
+    for (const e of this._entries.values()) {
+      if (e.hot) hot++; else warm++;
+      totalBytes += e.byteSize;
+    }
+    return { hot, warm, totalBytes };
+  }
+
+  // ── OPFS plumbing ─────────────────────────────────────────────────────
+
+  private _safeName(name: string): string {
+    // Strip path separators just in case; OPFS requires plain file names.
+    return name.replace(/[/\\]/g, '_');
+  }
+
+  private async _writeBlob(file: File): Promise<void> {
+    if (!this._dir) return;
+    try {
+      const handle = await this._dir.getFileHandle(this._safeName(file.name), { create: true });
+      const w = await handle.createWritable();
+      const bytes = new Uint8Array(await file.arrayBuffer());
+      const plain = new Uint8Array(bytes.byteLength);
+      plain.set(bytes);
+      await w.write(plain);
+      await w.close();
+    } catch (e) {
+      console.warn(`[albex] failed to persist blob for ${file.name}:`, e);
+    }
+  }
+
+  private async _readBlob(name: string): Promise<Blob | null> {
+    if (!this._dir) return null;
+    try {
+      const handle = await this._dir.getFileHandle(this._safeName(name));
+      return await handle.getFile();
+    } catch {
+      return null;
+    }
+  }
+
+  private async _deleteBlob(name: string): Promise<void> {
+    if (!this._dir) return;
+    try { await this._dir.removeEntry(this._safeName(name)); } catch { /* not found */ }
+  }
+
+  /**
+   * TC39 explicit-resource-management hook. Drops the in-memory index and
+   * the OPFS directory handle. The underlying OPFS blobs are NOT deleted —
+   * disposal only frees JS-side state. Use `forget()` per-doc or
+   * `deleteOpfsAll()` if you want to wipe persisted data.
+   */
+  [Symbol.dispose](): void {
+    this._entries.clear();
+    this._dir = null;
+  }
+
+  private async _rehydrateIndex(): Promise<void> {
+    if (!this._dir) return;
+    try {
+      // @ts-expect-error async-iterable on FileSystemDirectoryHandle
+      for await (const [name, handle] of this._dir.entries()) {
+        if (handle.kind !== 'file') continue;
+        const file = await handle.getFile();
+        if (this._entries.has(name)) continue;
+        this._entries.set(name, {
+          name,
+          ext:            (name.split('.').pop() ?? '').toLowerCase(),
+          lastAccessedMs: 0, // never accessed in this session yet
+          hot:            false,
+          byteSize:       file.size,
+        });
+      }
+    } catch (e) {
+      console.warn('[albex] could not rehydrate tiered index:', e);
+    }
+  }
+}