albex 0.3.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +466 -0
- package/README.md +32 -19
- package/dist/albex-worker.d.ts +65 -2
- package/dist/albex-worker.d.ts.map +1 -1
- package/dist/albex-worker.js +97 -20
- package/dist/albex-worker.js.map +1 -1
- package/dist/albex.d.ts +359 -55
- package/dist/albex.d.ts.map +1 -1
- package/dist/albex.js +766 -312
- package/dist/albex.js.map +1 -1
- package/dist/errors.d.ts +47 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +41 -3
- package/dist/errors.js.map +1 -1
- package/dist/persistence.js +1 -1
- package/dist/pool/coordinator.d.ts +14 -6
- package/dist/pool/coordinator.d.ts.map +1 -1
- package/dist/pool/coordinator.js +65 -28
- package/dist/pool/coordinator.js.map +1 -1
- package/dist/profile.d.ts +11 -6
- package/dist/profile.d.ts.map +1 -1
- package/dist/profile.js +6 -13
- package/dist/profile.js.map +1 -1
- package/dist/resource-manager.js +1 -1
- package/dist/tiered-store.js +1 -1
- package/dist/wasm-bindings.d.ts +96 -6
- package/dist/wasm-bindings.d.ts.map +1 -1
- package/dist/wasm-bindings.js +110 -7
- package/dist/wasm-bindings.js.map +1 -1
- package/dist/worker-protocol.d.ts +23 -2
- package/dist/worker-protocol.d.ts.map +1 -1
- package/dist/worker-protocol.js +1 -1
- package/dist/worker-runtime.js +27 -3
- package/dist/worker-runtime.js.map +1 -1
- package/package.json +13 -9
- package/src/albex-worker.ts +103 -18
- package/src/albex.ts +2937 -2292
- package/src/errors.ts +63 -2
- package/src/pool/coordinator.ts +61 -34
- package/src/profile.ts +11 -10
- package/src/wasm-bindings.ts +225 -10
- package/src/worker-protocol.ts +12 -2
- package/src/worker-runtime.ts +28 -3
- package/wasm/pkg/albex_pdf.wasm +0 -0
- package/wasm/pkg/albex_wasm.wasm +0 -0
- package/wasm/pkg/albex_wasm_bg.wasm +0 -0
- package/wasm/pkg/albex_wasm_simd.wasm +0 -0
- package/wasm/pkg/albex_wasm_mini.wasm +0 -0
- package/wasm/pkg/albex_wasm_mini_simd.wasm +0 -0
- package/wasm/pkg/albex_wasm_pro.wasm +0 -0
- package/wasm/pkg/albex_wasm_pro_simd.wasm +0 -0
- package/wasm/pkg/albex_wasm_std.wasm +0 -0
- package/wasm/pkg/albex_wasm_std_simd.wasm +0 -0
package/src/errors.ts
CHANGED
|
@@ -51,10 +51,71 @@ export class AlbexParseError extends AlbexError {
|
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Thrown when an indexing operation does not fit: either the scratchpad was
|
|
56
|
+
* too small for a single write, or one of the engine's pools (chunks, text,
|
|
57
|
+
* documents, names) ran out of room mid-document. Before 0.6.0 the latter was
|
|
58
|
+
* silent — the corpus was truncated with no signal.
|
|
59
|
+
*
|
|
60
|
+
* `limit` names which pool overflowed (or `'scratchpad'`, or `'file'` when an
|
|
61
|
+
* input file exceeds `maxFileBytes` before any byte is read), so callers can
|
|
62
|
+
* branch — e.g. start a fresh shard, `compact()`, or surface "library full".
|
|
63
|
+
* When a capacity error is raised during `indexFile`, the engine may hold a
|
|
64
|
+
* partially-indexed copy of the offending document; treat the index as full
|
|
65
|
+
* and stop adding. A `'file'` capacity error is raised BEFORE the file is
|
|
66
|
+
* read, so the index is untouched and fully usable.
|
|
67
|
+
*/
|
|
68
|
+
export type AlbexCapacityLimit = 'chunks' | 'text' | 'docs' | 'names' | 'scratchpad' | 'file';
|
|
69
|
+
|
|
55
70
|
export class AlbexCapacityError extends AlbexError {
|
|
56
|
-
|
|
71
|
+
/** Which pool overflowed. Undefined for older call sites that didn't set it. */
|
|
72
|
+
readonly limit?: AlbexCapacityLimit;
|
|
73
|
+
/**
|
|
74
|
+
* The RUNTIME numeric capacity of the pool named by `limit`, as the
|
|
75
|
+
* engine is actually configured (e.g. `4` when `capacity: { maxDocs: 4 }`
|
|
76
|
+
* overflows its document table, `128` for the std default). Units: docs
|
|
77
|
+
* for `'docs'`, chunks for `'chunks'`, bytes for `'text'`/`'names'`/
|
|
78
|
+
* `'scratchpad'`/`'file'`. Undefined when the limit is not known at the
|
|
79
|
+
* throw site.
|
|
80
|
+
*/
|
|
81
|
+
readonly max?: number;
|
|
82
|
+
constructor(message: string, limit?: AlbexCapacityLimit, max?: number) {
|
|
57
83
|
super('capacity', message);
|
|
58
84
|
this.name = 'AlbexCapacityError';
|
|
85
|
+
if (limit) this.limit = limit;
|
|
86
|
+
if (max !== undefined) this.max = max;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Default `maxFileBytes` for `indexFile`: 256 MiB. Far above anything the
|
|
92
|
+
* ~16–21 MB text pool could ever absorb, so legitimate documents are never
|
|
93
|
+
* rejected — the guard only exists to refuse pathological inputs (a 2 GB
|
|
94
|
+
* file would otherwise be fully buffered AND hashed before the first
|
|
95
|
+
* capacity check could fire).
|
|
96
|
+
*/
|
|
97
|
+
export const DEFAULT_MAX_FILE_BYTES = 256 * 1024 * 1024;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Pre-read size guard for `indexFile`. Throws a typed
|
|
101
|
+
* {@link AlbexCapacityError} (`limit: 'file'`) when `file.size` exceeds the
|
|
102
|
+
* configured cap — BEFORE any byte of the file is read into memory
|
|
103
|
+
* (`File`/`Blob` expose `size` without reading). Shared by the engine, the
|
|
104
|
+
* worker wrapper and the pool coordinator so the guard fires on whichever
|
|
105
|
+
* thread would otherwise buffer the bytes.
|
|
106
|
+
*/
|
|
107
|
+
export function assertFileSizeWithinLimit(
|
|
108
|
+
file: { name: string; size: number },
|
|
109
|
+
maxFileBytes?: number,
|
|
110
|
+
): void {
|
|
111
|
+
const cap = maxFileBytes ?? DEFAULT_MAX_FILE_BYTES;
|
|
112
|
+
if (file.size > cap) {
|
|
113
|
+
throw new AlbexCapacityError(
|
|
114
|
+
`"${file.name}" is ${file.size} bytes, above the maxFileBytes limit of ` +
|
|
115
|
+
`${cap}. The file was not read or indexed. Raise \`maxFileBytes\` in ` +
|
|
116
|
+
`AlbexOptions if this is intentional.`,
|
|
117
|
+
'file',
|
|
118
|
+
cap,
|
|
119
|
+
);
|
|
59
120
|
}
|
|
60
121
|
}
|
package/src/pool/coordinator.ts
CHANGED
|
@@ -30,9 +30,8 @@ import type {
|
|
|
30
30
|
EngineStats,
|
|
31
31
|
SearchStats,
|
|
32
32
|
} from '../albex.js';
|
|
33
|
-
import type { Tier } from '../profile.js';
|
|
34
33
|
import { detectProfile, pickWorkerCount } from '../profile.js';
|
|
35
|
-
import { AlbexInitError, AlbexError } from '../errors.js';
|
|
34
|
+
import { AlbexInitError, AlbexError, assertFileSizeWithinLimit } from '../errors.js';
|
|
36
35
|
import type {
|
|
37
36
|
WorkerRequest,
|
|
38
37
|
WorkerResponse,
|
|
@@ -77,7 +76,9 @@ export class AlbexPool {
|
|
|
77
76
|
private _docsCache: IndexedDocument[] = [];
|
|
78
77
|
private _rrCursor = 0;
|
|
79
78
|
private _lastSearch: SearchStats | null = null;
|
|
80
|
-
|
|
79
|
+
/** Global result cap applied AFTER the cross-shard merge. Mirrors the
|
|
80
|
+
* last `setMaxResults` call (the WASM engine default is 50). */
|
|
81
|
+
private _maxResults = 50;
|
|
81
82
|
|
|
82
83
|
constructor(opts: AlbexPoolOptions) {
|
|
83
84
|
this._opts = opts;
|
|
@@ -98,23 +99,21 @@ export class AlbexPool {
|
|
|
98
99
|
console.warn('[albex] pool mode=shared requested but cross-origin isolation is not active; falling back to replicated');
|
|
99
100
|
}
|
|
100
101
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
102
|
+
// Forward every serializable engine option to the shards; strip the
|
|
103
|
+
// pool-only fields (workerUrl/workers/mode) and anything non-clonable.
|
|
104
|
+
// Same policy as AlbexEngineWorker.init (audit 1.4).
|
|
105
|
+
const shardOpts: AlbexOptions = {};
|
|
106
|
+
for (const [k, v] of Object.entries(this._opts)) {
|
|
107
|
+
if (k === 'workerUrl' || k === 'workers' || k === 'mode') continue;
|
|
108
|
+
if (v === undefined || typeof v === 'function') continue;
|
|
109
|
+
(shardOpts as Record<string, unknown>)[k] = v;
|
|
110
|
+
}
|
|
108
111
|
|
|
109
112
|
for (let i = 0; i < n; i++) {
|
|
110
113
|
const shard = this._spawnShard();
|
|
111
114
|
await this._send(shard, { kind: 'init', opts: shardOpts });
|
|
112
115
|
this._shards.push(shard);
|
|
113
116
|
}
|
|
114
|
-
|
|
115
|
-
// Tier is the same across shards — capture it from shard 0 stats.
|
|
116
|
-
const stats0 = await this._send<EngineStats>(this._shards[0]!, { kind: 'getStats' });
|
|
117
|
-
this._tier = stats0.tier;
|
|
118
117
|
}
|
|
119
118
|
|
|
120
119
|
// ── Shard plumbing ─────────────────────────────────────────────────────
|
|
@@ -168,6 +167,9 @@ export class AlbexPool {
|
|
|
168
167
|
|
|
169
168
|
async indexFile(file: File): Promise<IndexedDocument> {
|
|
170
169
|
if (this._shards.length === 0) throw new AlbexInitError('Pool not initialised');
|
|
170
|
+
// Size guard BEFORE reading — same limit the shard engine enforces, but
|
|
171
|
+
// checked here so an oversized file is never buffered on the main thread.
|
|
172
|
+
assertFileSizeWithinLimit(file, this._opts.maxFileBytes);
|
|
171
173
|
const idx = this._rrCursor++ % this._shards.length;
|
|
172
174
|
const shard = this._shards[idx]!;
|
|
173
175
|
const buffer = await file.arrayBuffer();
|
|
@@ -185,30 +187,50 @@ export class AlbexPool {
|
|
|
185
187
|
* Map-reduce search:
|
|
186
188
|
* 1. broadcast the query to every shard,
|
|
187
189
|
* 2. each shard runs its local Bloom→Bitap→top-K,
|
|
188
|
-
* 3. coordinator merges the K-best from each shard,
|
|
190
|
+
* 3. coordinator merges the K-best from each shard, deduplicating
|
|
191
|
+
* identical hits (same document name, chunk id and match offset —
|
|
192
|
+
* the case where the same file was indexed into two shards),
|
|
189
193
|
* 4. global top-K returned in descending score order.
|
|
190
194
|
*
|
|
191
195
|
* Capped to `setMaxResults` results (default 50) AFTER merge.
|
|
196
|
+
*
|
|
197
|
+
* Per-shard search stats are requested in the same posting batch as the
|
|
198
|
+
* search itself: each worker processes its queue in arrival order, so the
|
|
199
|
+
* stats reply is guaranteed to belong to THIS query even when several
|
|
200
|
+
* `search()` calls overlap.
|
|
192
201
|
*/
|
|
193
202
|
async search(query: string, opts: SearchOptions = {}): Promise<SearchResult[]> {
|
|
194
203
|
if (this._shards.length === 0) return [];
|
|
195
204
|
const t0 = performance.now();
|
|
196
|
-
|
|
197
|
-
|
|
205
|
+
// Post `search` and `getLastSearchStats` back-to-back (synchronously,
|
|
206
|
+
// per shard) so no other request can slot between them in the worker's
|
|
207
|
+
// FIFO queue — the stats round can't race a subsequent search.
|
|
208
|
+
const perShard = await Promise.all(
|
|
209
|
+
this._shards.map(s => {
|
|
210
|
+
const results = this._send<SearchResult[]>(s, { kind: 'search', query, options: opts });
|
|
211
|
+
const stats = this._send<SearchStats | null>(s, { kind: 'getLastSearchStats' });
|
|
212
|
+
return Promise.all([results, stats] as const);
|
|
213
|
+
}),
|
|
198
214
|
);
|
|
199
215
|
|
|
200
|
-
// Merge:
|
|
201
|
-
//
|
|
202
|
-
//
|
|
203
|
-
|
|
216
|
+
// Merge: flatten + dedup + sort (descending) + global cap. Shard-local
|
|
217
|
+
// docIds collide across shards, so the dedup identity also includes the
|
|
218
|
+
// document name: it only collapses true duplicates (the same document
|
|
219
|
+
// indexed into more than one shard yields identical name/chunkId/offset).
|
|
220
|
+
const seen = new Set<string>();
|
|
221
|
+
const merged: SearchResult[] = [];
|
|
222
|
+
for (const [bucket] of perShard) {
|
|
223
|
+
for (const r of bucket) {
|
|
224
|
+
const key = `${r.documentName}:${r.chunkId}:${r.matchStart}`;
|
|
225
|
+
if (!seen.has(key)) { seen.add(key); merged.push(r); }
|
|
226
|
+
}
|
|
227
|
+
}
|
|
204
228
|
merged.sort((a, b) => b.score - a.score);
|
|
229
|
+
const capped = merged.slice(0, this._maxResults);
|
|
205
230
|
|
|
206
231
|
// Aggregate search stats across shards.
|
|
207
|
-
const stats = await Promise.all(
|
|
208
|
-
this._shards.map(s => this._send<SearchStats | null>(s, { kind: 'getLastSearchStats' })),
|
|
209
|
-
);
|
|
210
232
|
let bloomTested = 0, bloomPassed = 0, bitapMatched = 0;
|
|
211
|
-
for (const s of
|
|
233
|
+
for (const [, s] of perShard) {
|
|
212
234
|
if (!s) continue;
|
|
213
235
|
bloomTested += s.bloomTested;
|
|
214
236
|
bloomPassed += s.bloomPassed;
|
|
@@ -217,11 +239,11 @@ export class AlbexPool {
|
|
|
217
239
|
this._lastSearch = {
|
|
218
240
|
query,
|
|
219
241
|
timeMs: performance.now() - t0,
|
|
220
|
-
results:
|
|
242
|
+
results: capped.length,
|
|
221
243
|
bloomTested, bloomPassed, bitapMatched,
|
|
222
244
|
};
|
|
223
245
|
|
|
224
|
-
return
|
|
246
|
+
return capped;
|
|
225
247
|
}
|
|
226
248
|
|
|
227
249
|
/**
|
|
@@ -273,11 +295,13 @@ export class AlbexPool {
|
|
|
273
295
|
for (const s of this._shards) s.docCount = 0;
|
|
274
296
|
}
|
|
275
297
|
|
|
276
|
-
/** Aggregate engine stats across all shards.
|
|
298
|
+
/** Aggregate engine stats across all shards. Capacities are the RUNTIME
|
|
299
|
+
* per-shard capacities summed (each shard was initialised with the same
|
|
300
|
+
* `capacity` option, forwarded through the wire protocol). */
|
|
277
301
|
async getStats(): Promise<EngineStats> {
|
|
278
302
|
const all = await this._broadcast<EngineStats>({ kind: 'getStats' });
|
|
279
303
|
let documents = 0, chunks = 0, textUsed = 0, textCapacity = 0, wasmMemoryBytes = 0;
|
|
280
|
-
let maxChunks = 0, maxDocs = 0;
|
|
304
|
+
let maxChunks = 0, maxDocs = 0, namePoolBytes = 0;
|
|
281
305
|
for (const s of all) {
|
|
282
306
|
documents += s.documents;
|
|
283
307
|
chunks += s.chunks;
|
|
@@ -286,10 +310,11 @@ export class AlbexPool {
|
|
|
286
310
|
wasmMemoryBytes += s.wasmMemoryBytes;
|
|
287
311
|
maxChunks += s.maxChunks;
|
|
288
312
|
maxDocs += s.maxDocs;
|
|
313
|
+
namePoolBytes += s.namePoolBytes;
|
|
289
314
|
}
|
|
290
315
|
return {
|
|
291
316
|
documents, chunks, textUsed, textCapacity, wasmMemoryBytes,
|
|
292
|
-
|
|
317
|
+
maxChunks, maxDocs, namePoolBytes,
|
|
293
318
|
};
|
|
294
319
|
}
|
|
295
320
|
|
|
@@ -303,15 +328,17 @@ export class AlbexPool {
|
|
|
303
328
|
|
|
304
329
|
async setMaxErrors(n: 0 | 1 | 2 | 3): Promise<void> { await this._broadcast({ kind: 'setMaxErrors', n }); }
|
|
305
330
|
async setThreshold(n: number): Promise<void> { await this._broadcast({ kind: 'setThreshold', n }); }
|
|
306
|
-
async setMaxResults(n: number): Promise<void> {
|
|
331
|
+
async setMaxResults(n: number): Promise<void> {
|
|
332
|
+
// Track the effective cap (same clamp as the WASM engine) so search()
|
|
333
|
+
// can enforce it globally after the cross-shard merge.
|
|
334
|
+
this._maxResults = Math.max(1, Math.min(200, Math.floor(n)));
|
|
335
|
+
await this._broadcast({ kind: 'setMaxResults', n });
|
|
336
|
+
}
|
|
307
337
|
async setLanguage(lang: 'off' | 'es'): Promise<void> { await this._broadcast({ kind: 'setLanguage', lang }); }
|
|
308
338
|
|
|
309
339
|
/** Number of shards currently running. */
|
|
310
340
|
get workerCount(): number { return this._shards.length; }
|
|
311
341
|
|
|
312
|
-
/** Tier loaded by the shards (same value across all of them). */
|
|
313
|
-
get tier(): Tier | null { return this._tier; }
|
|
314
|
-
|
|
315
342
|
[Symbol.dispose](): void {
|
|
316
343
|
for (const s of this._shards) {
|
|
317
344
|
for (const [, p] of s.pending) p.reject(new AlbexError('disposed', 'Pool disposed'));
|
package/src/profile.ts
CHANGED
|
@@ -233,19 +233,20 @@ export async function detectProfile(opts: { fresh?: boolean } = {}): Promise<Dev
|
|
|
233
233
|
|
|
234
234
|
// ── Tier selection ───────────────────────────────────────────────────────────
|
|
235
235
|
|
|
236
|
-
|
|
236
|
+
/**
|
|
237
|
+
* @deprecated The tier system was removed in 0.5.0 (audit 4.1: "6
|
|
238
|
+
* binaries × no proven benefit"). The type remains exported as `'std'`
|
|
239
|
+
* for backwards compatibility with code that read `engine.getStats().tier`.
|
|
240
|
+
*/
|
|
241
|
+
export type Tier = 'std';
|
|
237
242
|
|
|
238
243
|
/**
|
|
239
|
-
*
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
244
|
+
* @deprecated Always returns `'std'` as of 0.5.0. Albex ships exactly
|
|
245
|
+
* two main binaries (baseline + SIMD); the only runtime variant is the
|
|
246
|
+
* SIMD probe, not a capacity tier. Kept callable so existing integrators
|
|
247
|
+
* don't break, but the value has no operational meaning anymore.
|
|
243
248
|
*/
|
|
244
|
-
export function pickTier(
|
|
245
|
-
const m = profile.memoryGB;
|
|
246
|
-
if (m === null) return 'std';
|
|
247
|
-
if (m <= 1) return 'mini';
|
|
248
|
-
if (m >= 8) return 'pro';
|
|
249
|
+
export function pickTier(_profile: DeviceProfile): Tier {
|
|
249
250
|
return 'std';
|
|
250
251
|
}
|
|
251
252
|
|
package/src/wasm-bindings.ts
CHANGED
|
@@ -17,9 +17,36 @@
|
|
|
17
17
|
export interface AlbexWasmExports {
|
|
18
18
|
readonly memory: WebAssembly.Memory;
|
|
19
19
|
|
|
20
|
-
//
|
|
20
|
+
// ABI / lifecycle
|
|
21
|
+
abiVersion(): number;
|
|
21
22
|
getBuffer(size: number): number;
|
|
23
|
+
/** Reset with the std default capacities (128 docs · 100k chunks · 16 MB
|
|
24
|
+
* text · 32 KB names) — identical behaviour to every pre-ABI-7 release. */
|
|
22
25
|
init(): void;
|
|
26
|
+
/** (Re-)initialise the engine with runtime capacities (ABI 7, decision
|
|
27
|
+
* A16). Allocates the capacity-dependent pools on the WASM heap. Returns
|
|
28
|
+
* 1 on success; 0 on invalid parameters (floors/ceilings documented in
|
|
29
|
+
* wasm/src/lib.rs: ≥1 doc, docs ≤ 65 536, chunks ≥ docs and ≤ 4 M, text
|
|
30
|
+
* pool 4 KiB–1 GiB, name pool 256 B–16 MiB) or on allocation failure —
|
|
31
|
+
* never traps. Re-init with the same capacities is a plain reset; with
|
|
32
|
+
* different capacities the pools are freed and re-allocated (no leak,
|
|
33
|
+
* but the linear-memory high-water mark never shrinks). */
|
|
34
|
+
initWithCapacity(
|
|
35
|
+
maxDocs: number,
|
|
36
|
+
maxChunks: number,
|
|
37
|
+
textPoolBytes: number,
|
|
38
|
+
namePoolBytes: number,
|
|
39
|
+
): number;
|
|
40
|
+
|
|
41
|
+
/** Reset the streaming FNV-1a 64-bit hash state. Optional on the first
|
|
42
|
+
* hash of a session because the static initialiser is also FNV_OFFSET. */
|
|
43
|
+
hashBegin(): void;
|
|
44
|
+
/** Fold `len` bytes of scratchpad into the streaming hash. May be
|
|
45
|
+
* called repeatedly for files larger than SCRATCHPAD_SIZE. */
|
|
46
|
+
hashFeed(len: number): void;
|
|
47
|
+
/** Write the final 8 raw big-endian bytes at scratchpad[0..8] and
|
|
48
|
+
* reset the state so the next hash can start without an explicit Begin. */
|
|
49
|
+
hashFinish(): void;
|
|
23
50
|
|
|
24
51
|
// Document ingestion
|
|
25
52
|
setDocumentName(len: number): void;
|
|
@@ -40,6 +67,18 @@ export interface AlbexWasmExports {
|
|
|
40
67
|
setThreshold(threshold: number): void;
|
|
41
68
|
setMaxResults(max: number): void;
|
|
42
69
|
|
|
70
|
+
// Query parsing (since ABI v2). Single source of truth for tokenization.
|
|
71
|
+
prepareQuery(len: number): number;
|
|
72
|
+
getQueryKind(): number;
|
|
73
|
+
getQueryBranchCount(): number;
|
|
74
|
+
getQueryBranchPattern(i: number): number;
|
|
75
|
+
selectQueryBranch(i: number): number;
|
|
76
|
+
/** Bitflags of what the most recent prepareQuery dropped or clipped
|
|
77
|
+
* (ABI 5): 1 = OR branches beyond 8 discarded, 2 = tokens dropped
|
|
78
|
+
* (> 4 per branch) or clipped (> 64 bytes), 4 = raw query cut at
|
|
79
|
+
* 1024 bytes. 0 = compiled in full. */
|
|
80
|
+
getQueryTruncationFlags(): number;
|
|
81
|
+
|
|
43
82
|
// Search execution
|
|
44
83
|
setPattern(len: number): number;
|
|
45
84
|
search(): number;
|
|
@@ -50,6 +89,15 @@ export interface AlbexWasmExports {
|
|
|
50
89
|
getSearchTotal(): number;
|
|
51
90
|
|
|
52
91
|
// Result accessors
|
|
92
|
+
/** Base pointer of the `#[repr(C)]` RESULTS array (ABI 6). Read
|
|
93
|
+
* `getResultCount()` records of `getResultStride()` bytes each with one
|
|
94
|
+
* DataView pass instead of ~12 frontier calls per result. Field offsets
|
|
95
|
+
* are documented (and compile-time asserted) in wasm/src/lib.rs. Copy
|
|
96
|
+
* everything out before any call that could grow WASM memory. */
|
|
97
|
+
getResultsPtr(): number;
|
|
98
|
+
/** Byte stride between consecutive RESULTS records (= sizeof(DocMatch),
|
|
99
|
+
* 60 today). Exported so the host never hardcodes the struct size. */
|
|
100
|
+
getResultStride(): number;
|
|
53
101
|
getResultCount(): number;
|
|
54
102
|
getResultDocId(i: number): number;
|
|
55
103
|
getResultLocation(i: number): number;
|
|
@@ -73,12 +121,27 @@ export interface AlbexWasmExports {
|
|
|
73
121
|
getDocCount(): number;
|
|
74
122
|
getTextUsed(): number;
|
|
75
123
|
getTextCapacity(): number;
|
|
124
|
+
/** Bitflags of capacity limits hit during the most recent
|
|
125
|
+
* begin..endDocument cycle: 1 = chunks, 2 = text, 4 = docs, 8 = names.
|
|
126
|
+
* 0 = everything fit. Read by the host right after endDocument to raise a
|
|
127
|
+
* typed AlbexCapacityError instead of silently truncating the corpus. */
|
|
128
|
+
getLastIndexOverflow(): number;
|
|
76
129
|
|
|
77
|
-
// Snapshot / restore
|
|
130
|
+
// Snapshot / restore (v3 protocol; v1 and v2 still load)
|
|
78
131
|
snapshotSize(): number;
|
|
79
132
|
snapshotChunk(offset: number, maxLen: number): number;
|
|
133
|
+
/** Validate header. For v3 also reserves the staging buffer; state is
|
|
134
|
+
* NOT touched until restoreCommit succeeds. For v1/v2 (legacy) state is
|
|
135
|
+
* reset and counters are written immediately. */
|
|
80
136
|
restoreBegin(): number;
|
|
137
|
+
/** Feed payload bytes. For v3 they accumulate into staging; for v1/v2
|
|
138
|
+
* they are written straight to the state arrays as before. */
|
|
81
139
|
restoreFeed(len: number): number;
|
|
140
|
+
/** Atomic commit for v3 snapshots. Returns 1 if the staged payload was
|
|
141
|
+
* complete and decoded successfully; 0 otherwise — and in the 0 case
|
|
142
|
+
* the previous engine state is preserved. For v1/v2 this is a no-op
|
|
143
|
+
* that always returns 1. */
|
|
144
|
+
restoreCommit(): number;
|
|
82
145
|
|
|
83
146
|
// Incremental / per-doc
|
|
84
147
|
getDocId(index: number): number;
|
|
@@ -88,6 +151,27 @@ export interface AlbexWasmExports {
|
|
|
88
151
|
removeDocument(docId: number): number;
|
|
89
152
|
compact(): void;
|
|
90
153
|
|
|
154
|
+
// Authoritative chunk enumeration (ABI 4). Address a document's chunks by
|
|
155
|
+
// (doc slot, ordinal). The compact()-stable key is (doc_id, ordinal).
|
|
156
|
+
/** First CHUNKS[] index for the document at slot `index` (= chunk_start).
|
|
157
|
+
* `ord = resultChunkIdx - getDocChunkBase(slot)` gives the doc-relative
|
|
158
|
+
* ordinal of a search hit. */
|
|
159
|
+
getDocChunkBase(index: number): number;
|
|
160
|
+
/** `location` (paragraph/page) of the `ord`-th chunk; u32::MAX if OOB. */
|
|
161
|
+
getChunkLocationAt(index: number, ord: number): number;
|
|
162
|
+
/** Byte length of the `ord`-th chunk's text; 0 if OOB. */
|
|
163
|
+
getChunkByteLenAt(index: number, ord: number): number;
|
|
164
|
+
/** Copy the `ord`-th chunk's UTF-8 text into the scratchpad; returns byte
|
|
165
|
+
* length (0 if OOB). Lets a host enumerate a doc's authoritative chunks
|
|
166
|
+
* right after indexing, with no query. */
|
|
167
|
+
getChunkTextAt(index: number, ord: number): number;
|
|
168
|
+
/** Batch chunk enumeration (ABI 6). Packs up to `maxChunks` records
|
|
169
|
+
* `[u32 text_len][u32 location][text bytes]` (LE, tightly packed) into
|
|
170
|
+
* the scratchpad starting at ordinal `startOrd`; returns how many were
|
|
171
|
+
* written. One frontier call per scratchpad-full instead of 2-3 per
|
|
172
|
+
* chunk. */
|
|
173
|
+
listChunksBatch(index: number, startOrd: number, maxChunks: number): number;
|
|
174
|
+
|
|
91
175
|
/**
|
|
92
176
|
* Per-document content hash (snapshot v2). Returns a pointer to 8 bytes
|
|
93
177
|
* holding the FNV-1a 64-bit hash of the original file bytes, or 0 if the
|
|
@@ -106,8 +190,8 @@ export interface AlbexWasmExports {
|
|
|
106
190
|
// Stemming
|
|
107
191
|
setLanguage(lang: number): void;
|
|
108
192
|
|
|
109
|
-
//
|
|
110
|
-
|
|
193
|
+
// Runtime capacity identification (ABI 7). Report the capacities the
|
|
194
|
+
// engine was last initialised with — `init()` = the std defaults.
|
|
111
195
|
getMaxChunks(): number;
|
|
112
196
|
getMaxDocs(): number;
|
|
113
197
|
getNameCapacity(): number;
|
|
@@ -117,6 +201,13 @@ export interface AlbexWasmExports {
|
|
|
117
201
|
getChunkStructSize(): number;
|
|
118
202
|
setCandidateMask(byteLen: number): void;
|
|
119
203
|
clearCandidateMask(): void;
|
|
204
|
+
/** Low 32 bits of the active pattern's aggregate character Bloom
|
|
205
|
+
* (ABI 6). Computed WASM-side in setPattern through the same pipeline
|
|
206
|
+
* searchBegin uses (split → optional stemming → fold), so the GPU
|
|
207
|
+
* pre-filter tests exactly the bits the CPU path would. */
|
|
208
|
+
getPatternBloomLo(): number;
|
|
209
|
+
/** High 32 bits of the active pattern's aggregate character Bloom. */
|
|
210
|
+
getPatternBloomHi(): number;
|
|
120
211
|
}
|
|
121
212
|
|
|
122
213
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -126,6 +217,10 @@ export interface AlbexWasmExports {
|
|
|
126
217
|
export interface AlbexPdfExports {
|
|
127
218
|
readonly memory: WebAssembly.Memory;
|
|
128
219
|
|
|
220
|
+
/** ABI version of the PDF module. The host loader refuses any binary
|
|
221
|
+
* whose abiVersion is outside the supported range. */
|
|
222
|
+
abiVersion(): number;
|
|
223
|
+
|
|
129
224
|
/** Reserve `len` bytes inside the PDF module and return a pointer. */
|
|
130
225
|
allocInput(len: number): number;
|
|
131
226
|
|
|
@@ -183,18 +278,138 @@ export interface AlbexPdfExports {
|
|
|
183
278
|
}
|
|
184
279
|
|
|
185
280
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
186
|
-
//
|
|
281
|
+
// Runtime validators
|
|
187
282
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
283
|
+
//
|
|
284
|
+
// These replace the pre-0.5.0 `as unknown as` casts. They check three
|
|
285
|
+
// things at instantiation time:
|
|
286
|
+
// 1. memory is a WebAssembly.Memory instance.
|
|
287
|
+
// 2. abiVersion() returns a number inside the supported range.
|
|
288
|
+
// 3. every required export exists and is a function.
|
|
289
|
+
//
|
|
290
|
+
// If any of these fails, the loader throws a typed error before the
|
|
291
|
+
// engine returns from init(). This eliminates the audit 3.2 issue:
|
|
292
|
+
// previously a missing export only surfaced when its call site ran.
|
|
188
293
|
|
|
189
|
-
/**
|
|
190
|
-
*
|
|
191
|
-
*
|
|
192
|
-
|
|
193
|
-
|
|
294
|
+
/** Range of ABI versions this host code understands for the main module.
|
|
295
|
+
* Update both ends together with the Rust `abiVersion()` constant when
|
|
296
|
+
* the export surface changes. */
|
|
297
|
+
// ABI 7 adds runtime capacity (initWithCapacity, decision A16) and removes
|
|
298
|
+
// the compile-time tier system (`getTier` is gone), on top of ABI 6's batch
|
|
299
|
+
// frontier reads, ABI 5's truncation signalling and ABI 4's authoritative
|
|
300
|
+
// chunk enumeration. The required-exports list below already makes any
|
|
301
|
+
// older binary fail the missing-exports check, so a tolerant lower bound
|
|
302
|
+
// was dead code — the range is pinned to the one ABI this host actually
|
|
303
|
+
// speaks (audit 0.6.0, finding #7). The .wasm ships inside this package
|
|
304
|
+
// (files: wasm/pkg/*.wasm), so host TS and binary are always
|
|
305
|
+
// version-matched.
|
|
306
|
+
const MAIN_ABI_MIN = 7;
|
|
307
|
+
const MAIN_ABI_MAX = 7;
|
|
308
|
+
|
|
309
|
+
/** Range of ABI versions for the PDF module. */
|
|
310
|
+
const PDF_ABI_MIN = 1;
|
|
311
|
+
const PDF_ABI_MAX = 3;
|
|
312
|
+
|
|
313
|
+
/** Required function names on the main WASM. Adding a new one here forces
|
|
314
|
+
* the validator to check it; removing one is a breaking ABI bump. */
|
|
315
|
+
const MAIN_REQUIRED = [
|
|
316
|
+
'abiVersion', 'getBuffer', 'init', 'initWithCapacity',
|
|
317
|
+
'setDocumentName', 'beginDocument', 'feedXmlBytes', 'endDocument',
|
|
318
|
+
'beginXlsx', 'feedXlsxBytes',
|
|
319
|
+
'feedText', 'flushParagraph',
|
|
320
|
+
'setMaxErrors', 'setThreshold', 'setMaxResults',
|
|
321
|
+
'prepareQuery', 'getQueryKind', 'getQueryBranchCount',
|
|
322
|
+
'getQueryBranchPattern', 'selectQueryBranch', 'getQueryTruncationFlags',
|
|
323
|
+
'setPattern', 'search',
|
|
324
|
+
'searchBegin', 'searchSlice', 'getSearchCursor', 'getSearchTotal',
|
|
325
|
+
'getResultCount', 'getResultsPtr', 'getResultStride',
|
|
326
|
+
'getResultDocId', 'getResultLocation', 'getResultScore',
|
|
327
|
+
'getResultStart', 'getResultEnd', 'getResultChunkIdx',
|
|
328
|
+
'getResultDocName', 'getResultMatchCount',
|
|
329
|
+
'getResultMatchStartAt', 'getResultMatchEndAt',
|
|
330
|
+
'getSnippet', 'getSnippetWindow', 'getSnippetWindowOffset',
|
|
331
|
+
'getStatBloomTested', 'getStatBloomPassed', 'getStatBitapMatched',
|
|
332
|
+
'getChunkCount', 'getDocCount', 'getTextUsed', 'getTextCapacity',
|
|
333
|
+
'getLastIndexOverflow',
|
|
334
|
+
'snapshotSize', 'snapshotChunk',
|
|
335
|
+
'restoreBegin', 'restoreFeed', 'restoreCommit',
|
|
336
|
+
'getDocId', 'getDocChunkCount', 'getDocName', 'isDocDeleted',
|
|
337
|
+
'removeDocument', 'compact',
|
|
338
|
+
'getDocChunkBase', 'getChunkLocationAt', 'getChunkByteLenAt', 'getChunkTextAt',
|
|
339
|
+
'listChunksBatch',
|
|
340
|
+
'setLanguage',
|
|
341
|
+
'getMaxChunks', 'getMaxDocs', 'getNameCapacity',
|
|
342
|
+
'getChunksPtr', 'getChunkStructSize',
|
|
343
|
+
'setCandidateMask', 'clearCandidateMask',
|
|
344
|
+
'getPatternBloomLo', 'getPatternBloomHi',
|
|
345
|
+
'getDocContentHashPtr', 'getDocContentHashLen', 'setDocumentContentHash',
|
|
346
|
+
'hashBegin', 'hashFeed', 'hashFinish',
|
|
347
|
+
] as const;
|
|
348
|
+
|
|
349
|
+
const PDF_REQUIRED = [
|
|
350
|
+
'abiVersion', 'allocInput', 'extractPdf',
|
|
351
|
+
'getPageLen', 'getPagePtr', 'getErrorLen', 'getErrorPtr',
|
|
352
|
+
'getPageCount', 'extractPageImages',
|
|
353
|
+
'getPageImageLen', 'getPageImagePtr', 'getPageImageKind',
|
|
354
|
+
] as const;
|
|
355
|
+
|
|
356
|
+
/** Thrown when an instantiated WASM module fails the ABI contract. */
|
|
357
|
+
export class AlbexAbiMismatchError extends Error {
|
|
358
|
+
readonly module: 'main' | 'pdf';
|
|
359
|
+
readonly missing?: readonly string[];
|
|
360
|
+
readonly version?: number;
|
|
361
|
+
constructor(module: 'main' | 'pdf', message: string, opts?: { missing?: readonly string[]; version?: number }) {
|
|
362
|
+
super(message);
|
|
363
|
+
this.name = 'AlbexAbiMismatchError';
|
|
364
|
+
this.module = module;
|
|
365
|
+
if (opts?.missing) this.missing = opts.missing;
|
|
366
|
+
if (opts?.version !== undefined) this.version = opts.version;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function validateExports(
|
|
371
|
+
exports: WebAssembly.Exports,
|
|
372
|
+
required: readonly string[],
|
|
373
|
+
module: 'main' | 'pdf',
|
|
374
|
+
abiMin: number,
|
|
375
|
+
abiMax: number,
|
|
376
|
+
): void {
|
|
377
|
+
const mem = (exports as Record<string, unknown>)['memory'];
|
|
378
|
+
if (!(mem instanceof WebAssembly.Memory)) {
|
|
379
|
+
throw new AlbexAbiMismatchError(module, `${module}: \`memory\` is missing or not a WebAssembly.Memory instance.`);
|
|
380
|
+
}
|
|
381
|
+
const missing: string[] = [];
|
|
382
|
+
for (const name of required) {
|
|
383
|
+
if (typeof (exports as Record<string, unknown>)[name] !== 'function') missing.push(name);
|
|
384
|
+
}
|
|
385
|
+
if (missing.length) {
|
|
386
|
+
throw new AlbexAbiMismatchError(
|
|
387
|
+
module,
|
|
388
|
+
`${module}: WASM binary missing required exports: ${missing.join(', ')}. ` +
|
|
389
|
+
`The .wasm was built with an incompatible source — rebuild with the current toolchain.`,
|
|
390
|
+
{ missing },
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
const version = ((exports as Record<string, unknown>)['abiVersion'] as () => number)();
|
|
394
|
+
if (version < abiMin || version > abiMax) {
|
|
395
|
+
throw new AlbexAbiMismatchError(
|
|
396
|
+
module,
|
|
397
|
+
`${module}: abiVersion ${version} outside supported range [${abiMin}..${abiMax}]. ` +
|
|
398
|
+
`The host TypeScript expects a different binary — upgrade albex or rebuild the WASM.`,
|
|
399
|
+
{ version },
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/** Validate and narrow `WebAssembly.Exports` to the typed Albex main
|
|
405
|
+
* interface. Throws `AlbexAbiMismatchError` if the contract is broken. */
|
|
194
406
|
export function asAlbexExports(exports: WebAssembly.Exports): AlbexWasmExports {
|
|
407
|
+
validateExports(exports, MAIN_REQUIRED, 'main', MAIN_ABI_MIN, MAIN_ABI_MAX);
|
|
195
408
|
return exports as unknown as AlbexWasmExports;
|
|
196
409
|
}
|
|
197
410
|
|
|
411
|
+
/** Validate and narrow `WebAssembly.Exports` to the typed PDF interface. */
|
|
198
412
|
export function asAlbexPdfExports(exports: WebAssembly.Exports): AlbexPdfExports {
|
|
413
|
+
validateExports(exports, PDF_REQUIRED, 'pdf', PDF_ABI_MIN, PDF_ABI_MAX);
|
|
199
414
|
return exports as unknown as AlbexPdfExports;
|
|
200
415
|
}
|
package/src/worker-protocol.ts
CHANGED
|
@@ -10,13 +10,19 @@
|
|
|
10
10
|
* copying the file bytes into the worker.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import type { AlbexOptions, IndexedDocument, SearchOptions, SearchResult, EngineStats, SearchStats } from './albex.js';
|
|
13
|
+
import type { AlbexDiagnostic, AlbexOptions, AuthoritativeChunk, IndexedDocument, SearchOptions, SearchResult, EngineStats, SearchStats } from './albex.js';
|
|
14
14
|
|
|
15
15
|
export type WorkerOp =
|
|
16
16
|
| { kind: 'init'; opts: AlbexOptions }
|
|
17
17
|
| { kind: 'indexFile'; name: string; buffer: ArrayBuffer }
|
|
18
18
|
| { kind: 'search'; query: string; options: SearchOptions }
|
|
19
|
+
| { kind: 'listChunks'; docId: number }
|
|
19
20
|
| { kind: 'removeDocument'; id: string }
|
|
21
|
+
/** Replace doc `name` with new content. `fileName` is the replacement
|
|
22
|
+
* file's own name (may differ from `name`); the bytes travel as a
|
|
23
|
+
* transferred ArrayBuffer like `indexFile`. */
|
|
24
|
+
| { kind: 'replaceDocument'; name: string; fileName: string; buffer: ArrayBuffer }
|
|
25
|
+
| { kind: 'takeDiagnostics' }
|
|
20
26
|
| { kind: 'compact' }
|
|
21
27
|
| { kind: 'reset' }
|
|
22
28
|
| { kind: 'getStats' }
|
|
@@ -39,10 +45,14 @@ export interface WorkerRequest {
|
|
|
39
45
|
|
|
40
46
|
export type WorkerResponse =
|
|
41
47
|
| { id: number; ok: true; result: unknown }
|
|
42
|
-
|
|
48
|
+
/** `limit`/`max` are populated for capacity errors so the rehydrated
|
|
49
|
+
* AlbexCapacityError keeps reporting the runtime limit that overflowed. */
|
|
50
|
+
| { id: number; ok: false; error: { name: string; kind?: string; message: string; limit?: string; max?: number } };
|
|
43
51
|
|
|
44
52
|
export type IndexFileResult = IndexedDocument;
|
|
45
53
|
export type SearchResultArr = SearchResult[];
|
|
54
|
+
export type ChunksResult = AuthoritativeChunk[];
|
|
46
55
|
export type StatsResult = EngineStats;
|
|
47
56
|
export type SearchStatsRes = SearchStats | null;
|
|
48
57
|
export type DocsResult = readonly IndexedDocument[];
|
|
58
|
+
export type DiagnosticsRes = AlbexDiagnostic[];
|