albex 0.1.0 → 0.3.0

package/dist/worker-runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-runtime.js","sourceRoot":"","sources":["../src/worker-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGrD,IAAI,MAAM,GAAuB,IAAI,CAAC;AAEtC,SAAS,YAAY;IACnB,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACtE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,EAAY;IAClC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QAChB,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,gEAAgE;YAChE,2BAA2B;YAC3B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YAC5C,OAAO,YAAY,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,QAAQ;YACX,OAAO,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QACrD,KAAK,gBAAgB;YACnB,OAAO,YAAY,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,KAAK,SAAS;YACZ,YAAY,EAAE,CAAC,OAAO,EAAE,CAAC;YACzB,OAAO,SAAS,CAAC;QACnB,KAAK,OAAO;YACV,YAAY,EAAE,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACnB,KAAK,UAAU;YACb,OAAO,YAAY,EAAE,CAAC,QAAQ,EAAE,CAAC;QACnC,KAAK,oBAAoB;YACvB,OAAO,YAAY,EAAE,CAAC,kBAAkB,EAAE,CAAC;QAC7C,KAAK,cAAc;YACjB,OAAO,YAAY,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1C,KAAK,cAAc;YACjB,YAAY,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,OAAO,SAAS,CAAC;QACnB,KAAK,cAAc;YACjB,YAAY,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,OAAO,SAAS,CAAC;QACnB,KAAK,eAAe;YAClB,YAAY,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACnC,OAAO,SAAS,CAAC;QACnB,KAAK,aAAa;YAChB,YAAY,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACpC,OAAO,SAAS,CAAC;QACnB,KAAK,MAAM;YACT,OAAO,YAAY,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACtC,KAAK,MAAM;YACT,OAAO,YAAY,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACtC,KAAK,YAAY;YACf,OAAO,YAAY,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC5C,KAAK,gBAAgB;YACnB,OAAO,YAAY,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAChD,KAAK,eAAe;YAClB,OAAO,YAAY,EAAE,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC;AACH,CAAC;AAED,IAAI,CAAC,SAAS,GAAG,KAAK,EAAE,EAA+B,EAAE,EAAE;IACzD,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,GAAG,GAAmB,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACpD,IAA0B,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,GAAG,GAAgC,CAAC;QAC3C,MAAM,GAAG,GAAmB;YAC1B,EAAE,EAAE,EAAE,EAAE,KAAK;YACb,KAAK,EAAE;gBACL,IAAI,EAAK,CAAC,CAAC,IAAI,IAAI,OAAO;gBAC1B,IAAI,EAAK,GAAG,YAAY,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACzD,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;aAClC;SACF,CAAC;QACD,IAA0B,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "albex",
-  "version": "0.1.0",
-  "description": "Local full-text search engine — documents never leave the browser",
+  "version": "0.3.0",
+  "description": "Zero-config local full-text search for documents — runs entirely in the browser, no server, no upload.",
   "type": "module",
   "main": "./dist/albex.js",
   "module": "./dist/albex.js",
@@ -10,21 +10,50 @@
     ".": {
       "import": "./dist/albex.js",
       "types": "./dist/albex.d.ts"
+    },
+    "./worker": {
+      "import": "./dist/albex-worker.js",
+      "types": "./dist/albex-worker.d.ts"
+    },
+    "./worker-runtime": {
+      "import": "./dist/worker-runtime.js",
+      "types": "./dist/worker-runtime.d.ts"
+    },
+    "./pool": {
+      "import": "./dist/pool/coordinator.js",
+      "types": "./dist/pool/coordinator.d.ts"
+    },
+    "./gpu": {
+      "import": "./dist/gpu/bloom-runtime.js",
+      "types": "./dist/gpu/bloom-runtime.d.ts"
+    },
+    "./tiered": {
+      "import": "./dist/tiered-store.js",
+      "types": "./dist/tiered-store.d.ts"
     }
   },
   "files": [
     "dist/",
     "src/",
-    "wasm/pkg/albex_wasm_bg.wasm",
-    "wasm/pkg/albex_pdf.wasm"
+    "wasm/pkg/*.wasm",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && node scripts/banner.mjs",
     "prepublishOnly": "npm run build",
-    "build:wasm": "cd wasm && cargo build --target wasm32-unknown-unknown --release && cp ../target/wasm32-unknown-unknown/release/albex_wasm.wasm pkg/albex_wasm_bg.wasm",
-    "build:pdf-wasm": "cd pdf-wasm && cargo build --target wasm32-unknown-unknown --release && cp ../target/wasm32-unknown-unknown/release/albex_pdf.wasm ../wasm/pkg/albex_pdf.wasm",
-    "build:all": "npm run build:wasm && npm run build:pdf-wasm && npm run build",
-    "typecheck": "tsc --noEmit"
+    "build:wasm": "node scripts/build-wasm.mjs std && node scripts/wasm-banner.mjs",
+    "build:wasm:mini": "node scripts/build-wasm.mjs mini && node scripts/wasm-banner.mjs",
+    "build:wasm:pro":  "node scripts/build-wasm.mjs pro && node scripts/wasm-banner.mjs",
+    "build:wasm:tiers": "node scripts/build-wasm.mjs && node scripts/wasm-banner.mjs",
+    "build:pdf-wasm": "cd pdf-wasm && cargo build --target wasm32-unknown-unknown --release && cp ../target/wasm32-unknown-unknown/release/albex_pdf.wasm ../wasm/pkg/albex_pdf.wasm && node scripts/wasm-banner.mjs",
+    "build:all": "npm run build:wasm:tiers && npm run build:pdf-wasm && npm run build",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "bench": "vitest bench --run",
+    "serve": "npx --yes serve@14 -p 5173 -L ."
   },
   "repository": {
     "type": "git",
@@ -38,13 +67,26 @@
     "search",
     "full-text",
     "wasm",
+    "webassembly",
     "browser",
+    "client-side",
     "offline",
+    "privacy",
+    "fuzzy",
+    "bitap",
+    "bloom-filter",
     "docx",
-    "pdf",
     "xlsx",
-    "fuzzy",
-    "privacy"
+    "pdf",
+    "markdown",
+    "html",
+    "json",
+    "csv",
+    "eml",
+    "rtf",
+    "zero-config",
+    "opfs",
+    "webgpu"
   ],
   "author": "RafaCalRob",
   "license": "MIT",
@@ -52,6 +94,7 @@
     "node": ">=18"
   },
   "devDependencies": {
-    "typescript": "^5.4.0"
+    "typescript": "^5.4.0",
+    "vitest": "^2.0.0"
   }
 }

package/src/albex-worker.ts

@@ -0,0 +1,187 @@
+/**
+ * `AlbexEngineWorker` — a main-thread wrapper that runs the engine inside a
+ * Web Worker. Mirrors the surface of `AlbexEngine` so it can be swapped in
+ * without code changes.
+ *
+ * Usage:
+ *
+ *     const engine = new AlbexEngineWorker({
+ *       wasmUrl:    '/assets/albex_wasm_bg.wasm',
+ *       pdfWasmUrl: '/assets/albex_pdf.wasm',
+ *       // Provide the URL to the bundled worker runtime.
+ *       workerUrl:  new URL('./worker-runtime.js', import.meta.url),
+ *     });
+ *     await engine.init();
+ *
+ * Why: a `search()` over 100k chunks can take 10–50 ms. On main thread that
+ * is visible jank for every keystroke. Off-main-thread keeps the UI at 60 fps.
+ *
+ * The runtime is single-threaded WASM, so requests are serialised: only one
+ * call is in flight at a time. This matches the actual `static mut` model
+ * inside the .wasm and is fine for an interactive search UI (each keystroke
+ * replaces the previous query).
+ */
+
+import type {
+  AlbexOptions,
+  IndexedDocument,
+  SearchOptions,
+  SearchResult,
+  EngineStats,
+  SearchStats,
+} from './albex.js';
+import type {
+  WorkerRequest,
+  WorkerResponse,
+  WorkerOp,
+} from './worker-protocol.js';
+import {
+  AlbexError,
+  AlbexInitError,
+  AlbexUnsupportedFormatError,
+  AlbexParseError,
+  AlbexCapacityError,
+} from './errors.js';
+
+export interface AlbexWorkerOptions extends AlbexOptions {
+  /** URL to the bundled worker runtime script (worker-runtime.js). */
+  workerUrl: string | URL;
+}
+
+interface Pending {
+  resolve: (v: unknown) => void;
+  reject:  (e: unknown) => void;
+}
+
+let _workerSearchStreamWarned = false;
+
+export class AlbexEngineWorker {
+  private readonly _opts: AlbexWorkerOptions;
+  private _worker!: Worker;
+  private _nextId = 1;
+  private _pending = new Map<number, Pending>();
+  private _docsCache: IndexedDocument[] = [];
+
+  constructor(opts: AlbexWorkerOptions) {
+    this._opts = opts;
+  }
+
+  async init(): Promise<void> {
+    this._worker = new Worker(this._opts.workerUrl, { type: 'module' });
+    this._worker.onmessage = (ev: MessageEvent<WorkerResponse>) => {
+      const { id } = ev.data;
+      const p = this._pending.get(id);
+      if (!p) return;
+      this._pending.delete(id);
+      if (ev.data.ok) p.resolve(ev.data.result);
+      else            p.reject(rehydrateError(ev.data.error));
+    };
+    this._worker.onerror = (e) => {
+      // Surface the error to every in-flight call.
+      const err = new AlbexInitError(`Worker crashed: ${e.message}`);
+      for (const [, p] of this._pending) p.reject(err);
+      this._pending.clear();
+    };
+    await this._send({ kind: 'init', opts: {
+      wasmUrl:    this._opts.wasmUrl,
+      pdfWasmUrl: this._opts.pdfWasmUrl,
+    } });
+  }
+
+  private _send<T = unknown>(op: WorkerOp, transfer: Transferable[] = []): Promise<T> {
+    const id = this._nextId++;
+    const req: WorkerRequest = { id, op };
+    return new Promise<T>((resolve, reject) => {
+      this._pending.set(id, { resolve: resolve as (v: unknown) => void, reject });
+      this._worker.postMessage(req, transfer);
+    });
+  }
+
+  async indexFile(file: File): Promise<IndexedDocument> {
+    const buffer = await file.arrayBuffer();
+    // Transfer the buffer to avoid a copy.
+    const doc = await this._send<IndexedDocument>(
+      { kind: 'indexFile', name: file.name, buffer },
+      [buffer],
+    );
+    this._docsCache.push(doc);
+    return doc;
+  }
+
+  search(query: string, opts: SearchOptions = {}): Promise<SearchResult[]> {
+    return this._send<SearchResult[]>({ kind: 'search', query, options: opts });
+  }
+
+  /**
+   * Cooperative variant of `search`. Today the wire still sends a single
+   * batch — the result array is fetched in one round-trip from the worker
+   * and then exposed as an async iterator so callers can `break` early.
+   * A future iteration may use a `MessagePort` to stream individual results
+   * from the worker side; the iterator shape is preserved across that
+   * transition.
+   */
+  async *searchCooperative(query: string, opts: SearchOptions = {}): AsyncIterable<SearchResult> {
+    const results = await this.search(query, opts);
+    for (const r of results) yield r;
+  }
+
+  /**
+   * @deprecated Renamed to `searchCooperative` in 0.3.0. Alias removed in 0.4.0.
+   */
+  async *searchStream(query: string, opts: SearchOptions = {}): AsyncIterable<SearchResult> {
+    if (!_workerSearchStreamWarned) {
+      _workerSearchStreamWarned = true;
+      console.warn('[albex] `AlbexEngineWorker.searchStream` is deprecated; rename to `searchCooperative`. Alias removed in 0.4.0.');
+    }
+    yield* this.searchCooperative(query, opts);
+  }
+
+  async removeDocument(id: string): Promise<boolean> {
+    const ok = await this._send<boolean>({ kind: 'removeDocument', id });
+    if (ok) this._docsCache = this._docsCache.filter(d => d.name !== id && d.contentHash !== id);
+    return ok;
+  }
+
+  async compact(): Promise<void> { await this._send({ kind: 'compact' }); }
+  async reset(): Promise<void>   {
+    await this._send({ kind: 'reset' });
+    this._docsCache = [];
+  }
+
+  getStats():          Promise<EngineStats>       { return this._send({ kind: 'getStats' }); }
+  getLastSearchStats(): Promise<SearchStats | null> { return this._send({ kind: 'getLastSearchStats' }); }
+  getDocuments():       Promise<readonly IndexedDocument[]> { return this._send({ kind: 'getDocuments' }); }
+
+  async setMaxErrors(n: 0 | 1 | 2 | 3): Promise<void> { await this._send({ kind: 'setMaxErrors',  n }); }
+  async setThreshold(n: number):       Promise<void> { await this._send({ kind: 'setThreshold', n }); }
+  async setMaxResults(n: number):      Promise<void> { await this._send({ kind: 'setMaxResults', n }); }
+  async setLanguage(lang: 'off' | 'es'): Promise<void> { await this._send({ kind: 'setLanguage', lang }); }
+
+  // Persistence — mirror of AlbexEngine.
+  async save(name: string):           Promise<void>    { await this._send({ kind: 'save', name }); }
+  async load(name: string):           Promise<boolean> { return this._send({ kind: 'load', name }); }
+  async loadOrInit(name: string):     Promise<boolean> { return this._send({ kind: 'loadOrInit', name }); }
+  async deleteSnapshot(name: string): Promise<void>    { await this._send({ kind: 'deleteSnapshot', name }); }
+  async listSnapshots():              Promise<string[]>{ return this._send({ kind: 'listSnapshots' }); }
+
+  [Symbol.dispose](): void {
+    for (const [, p] of this._pending) p.reject(new AlbexError('disposed', 'Engine disposed'));
+    this._pending.clear();
+    this._worker?.terminate();
+    this._docsCache = [];
+  }
+}
+
+function rehydrateError(e: { name: string; kind?: string; message: string }): Error {
+  switch (e.kind) {
+    case 'init':               return new AlbexInitError(e.message);
+    case 'unsupported_format': return new AlbexUnsupportedFormatError(e.message.replace(/^Unsupported format: \./, ''));
+    case 'parse':              return new AlbexParseError('unknown', e.message);
+    case 'capacity':           return new AlbexCapacityError(e.message);
+    default: {
+      const err = new Error(e.message);
+      err.name = e.name;
+      return err;
+    }
+  }
+}