@bendyline/gezel-retrieval 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +44 -2
- package/dist/index.d.ts +254 -0
- package/dist/index.js +849 -0
- package/dist/sql-BFQtPDha.d.ts +24 -0
- package/dist/sqlite-wasm.d.ts +69 -0
- package/dist/sqlite-wasm.js +68 -0
- package/package.json +68 -3
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The slice of a synchronous SQLite binding the live index uses. Node's
|
|
3
|
+
* `node:sqlite` `DatabaseSync` satisfies it as is; `./sqlite-wasm` adapts
|
|
4
|
+
* sqlite-wasm's oo1 API for a Web Worker or WebView. Synchronous on purpose:
|
|
5
|
+
* the index runs in a worker or a Node process, off any UI thread.
|
|
6
|
+
*
|
|
7
|
+
* The binding must include FTS5 and the JSON functions (both are in Node's
|
|
8
|
+
* build and the official sqlite-wasm build) and support `RETURNING`.
|
|
9
|
+
*/
|
|
10
|
+
type SqlValue = string | number | bigint | Uint8Array | null;
|
|
11
|
+
type SqlRow = Record<string, SqlValue>;
|
|
12
|
+
interface SqlStatement {
|
|
13
|
+
run(...params: SqlValue[]): {
|
|
14
|
+
readonly changes: number | bigint;
|
|
15
|
+
};
|
|
16
|
+
get(...params: SqlValue[]): SqlRow | undefined;
|
|
17
|
+
all(...params: SqlValue[]): SqlRow[];
|
|
18
|
+
}
|
|
19
|
+
interface SqlDatabase {
|
|
20
|
+
exec(sql: string): void;
|
|
21
|
+
prepare(sql: string): SqlStatement;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export type { SqlDatabase as S, SqlRow as a, SqlStatement as b, SqlValue as c };
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { c as SqlValue, S as SqlDatabase } from './sql-BFQtPDha.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `@bendyline/gezel-retrieval/sqlite-wasm`: a read-write `SqlDatabase` over
|
|
5
|
+
* the official sqlite-wasm build (`@sqlite.org/sqlite-wasm`), for a Web
|
|
6
|
+
* Worker or a mobile WebView, and the origin-private (OPFS) pool that keeps
|
|
7
|
+
* an index across launches.
|
|
8
|
+
*
|
|
9
|
+
* Typed structurally: the app passes in the sqlite-wasm module it bundles
|
|
10
|
+
* (and serves `sqlite3.wasm` for), so this package does not depend on it.
|
|
11
|
+
* The pool VFS needs a dedicated Worker; a page or a second window that
|
|
12
|
+
* cannot own the pool uses `openMemoryDatabase` instead.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
interface WasmPreparedStatement {
|
|
16
|
+
readonly columnCount: number;
|
|
17
|
+
bind(values: SqlValue[]): unknown;
|
|
18
|
+
clearBindings(): unknown;
|
|
19
|
+
step(): boolean;
|
|
20
|
+
get(index: number): unknown;
|
|
21
|
+
getColumnName(index: number): string | undefined;
|
|
22
|
+
reset(): unknown;
|
|
23
|
+
}
|
|
24
|
+
interface WasmDatabase {
|
|
25
|
+
exec(sql: string): unknown;
|
|
26
|
+
prepare(sql: string): WasmPreparedStatement;
|
|
27
|
+
changes(): number | bigint;
|
|
28
|
+
close(): unknown;
|
|
29
|
+
}
|
|
30
|
+
interface WasmSahPool {
|
|
31
|
+
readonly OpfsSAHPoolDb: new (filename: string) => WasmDatabase;
|
|
32
|
+
getFileNames(): string[];
|
|
33
|
+
unlink(filename: string): boolean;
|
|
34
|
+
}
|
|
35
|
+
interface WasmSqlite3 {
|
|
36
|
+
readonly oo1: {
|
|
37
|
+
readonly DB: new (options: {
|
|
38
|
+
filename: string;
|
|
39
|
+
flags?: string;
|
|
40
|
+
vfs?: string;
|
|
41
|
+
}) => WasmDatabase;
|
|
42
|
+
};
|
|
43
|
+
installOpfsSAHPoolVfs?(options: {
|
|
44
|
+
name?: string;
|
|
45
|
+
directory?: string;
|
|
46
|
+
}): Promise<WasmSahPool>;
|
|
47
|
+
}
|
|
48
|
+
interface WasmSqlDatabase extends SqlDatabase {
|
|
49
|
+
close(): void;
|
|
50
|
+
}
|
|
51
|
+
/** Wrap an open sqlite-wasm oo1 database. */
|
|
52
|
+
declare function wrapWasmDatabase(db: WasmDatabase): WasmSqlDatabase;
|
|
53
|
+
/** An in-memory database: nothing persists, for a window that cannot own the pool. */
|
|
54
|
+
declare function openMemoryDatabase(sqlite3: Pick<WasmSqlite3, 'oo1'>): WasmSqlDatabase;
|
|
55
|
+
/**
|
|
56
|
+
* A database in an origin-private storage pool (sqlite-wasm's `opfs-sahpool`
|
|
57
|
+
* VFS), which persists across launches. Only one context may hold a pool at
|
|
58
|
+
* a time, so the host serializes owners (with a Web Lock, say). Throws where
|
|
59
|
+
* the pool cannot be installed (no OPFS, not a dedicated Worker).
|
|
60
|
+
*/
|
|
61
|
+
declare function openPoolDatabase(sqlite3: WasmSqlite3, options: {
|
|
62
|
+
readonly pool: string;
|
|
63
|
+
readonly filename: string;
|
|
64
|
+
}): Promise<{
|
|
65
|
+
readonly db: WasmSqlDatabase;
|
|
66
|
+
readonly pool: WasmSahPool;
|
|
67
|
+
}>;
|
|
68
|
+
|
|
69
|
+
export { type WasmDatabase, type WasmPreparedStatement, type WasmSahPool, type WasmSqlDatabase, type WasmSqlite3, openMemoryDatabase, openPoolDatabase, wrapWasmDatabase };
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// src/sqlite-wasm.ts
|
|
2
|
+
function bind(statement, params) {
|
|
3
|
+
statement.clearBindings();
|
|
4
|
+
if (params.length > 0) statement.bind(params);
|
|
5
|
+
}
|
|
6
|
+
function readRow(statement) {
|
|
7
|
+
const row = {};
|
|
8
|
+
for (let index = 0; index < statement.columnCount; index++) {
|
|
9
|
+
const value = statement.get(index);
|
|
10
|
+
row[statement.getColumnName(index) ?? String(index)] = value instanceof Uint8Array ? value.slice() : value;
|
|
11
|
+
}
|
|
12
|
+
return row;
|
|
13
|
+
}
|
|
14
|
+
function wrapWasmDatabase(db) {
|
|
15
|
+
const statement = (sql) => {
|
|
16
|
+
const prepared = db.prepare(sql);
|
|
17
|
+
const all = (params) => {
|
|
18
|
+
const rows = [];
|
|
19
|
+
try {
|
|
20
|
+
bind(prepared, params);
|
|
21
|
+
while (prepared.step()) rows.push(readRow(prepared));
|
|
22
|
+
return rows;
|
|
23
|
+
} finally {
|
|
24
|
+
prepared.reset();
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
return {
|
|
28
|
+
run: (...params) => {
|
|
29
|
+
all(params);
|
|
30
|
+
return { changes: db.changes() };
|
|
31
|
+
},
|
|
32
|
+
get: (...params) => {
|
|
33
|
+
try {
|
|
34
|
+
bind(prepared, params);
|
|
35
|
+
return prepared.step() ? readRow(prepared) : void 0;
|
|
36
|
+
} finally {
|
|
37
|
+
prepared.reset();
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
all: (...params) => all(params)
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
return {
|
|
44
|
+
exec: (sql) => {
|
|
45
|
+
db.exec(sql);
|
|
46
|
+
},
|
|
47
|
+
prepare: statement,
|
|
48
|
+
// oo1's close() finalizes every statement the database prepared.
|
|
49
|
+
close: () => {
|
|
50
|
+
db.close();
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function openMemoryDatabase(sqlite3) {
|
|
55
|
+
return wrapWasmDatabase(new sqlite3.oo1.DB({ filename: ":memory:", flags: "c" }));
|
|
56
|
+
}
|
|
57
|
+
async function openPoolDatabase(sqlite3, options) {
|
|
58
|
+
if (!sqlite3.installOpfsSAHPoolVfs) {
|
|
59
|
+
throw new Error("This sqlite-wasm build has no opfs-sahpool VFS");
|
|
60
|
+
}
|
|
61
|
+
const pool = await sqlite3.installOpfsSAHPoolVfs({ name: options.pool });
|
|
62
|
+
return { db: wrapWasmDatabase(new pool.OpfsSAHPoolDb(options.filename)), pool };
|
|
63
|
+
}
|
|
64
|
+
export {
|
|
65
|
+
openMemoryDatabase,
|
|
66
|
+
openPoolDatabase,
|
|
67
|
+
wrapWasmDatabase
|
|
68
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,71 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezel-retrieval",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "A mutable, browser-safe retrieval index for documents an app owns: heading-aware chunks and SQLite FTS5 over any synchronous SQLite handle (node:sqlite, or sqlite-wasm on origin-private storage).",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"gezel",
|
|
8
|
+
"retrieval",
|
|
9
|
+
"search",
|
|
10
|
+
"rag",
|
|
11
|
+
"sqlite",
|
|
12
|
+
"fts5",
|
|
13
|
+
"opfs"
|
|
14
|
+
],
|
|
15
|
+
"homepage": "https://github.com/bendyline/gezel",
|
|
16
|
+
"bugs": {
|
|
17
|
+
"url": "https://github.com/bendyline/gezel/issues"
|
|
18
|
+
},
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/bendyline/gezel.git",
|
|
22
|
+
"directory": "packages/retrieval"
|
|
23
|
+
},
|
|
24
|
+
"author": {
|
|
25
|
+
"name": "Bendyline",
|
|
26
|
+
"email": "support@bendyline.com"
|
|
27
|
+
},
|
|
28
|
+
"publishConfig": {
|
|
29
|
+
"access": "public",
|
|
30
|
+
"provenance": true
|
|
31
|
+
},
|
|
32
|
+
"type": "module",
|
|
33
|
+
"engines": {
|
|
34
|
+
"node": ">=24.0.0"
|
|
35
|
+
},
|
|
36
|
+
"main": "./dist/index.js",
|
|
37
|
+
"module": "./dist/index.js",
|
|
38
|
+
"types": "./dist/index.d.ts",
|
|
39
|
+
"exports": {
|
|
40
|
+
".": {
|
|
41
|
+
"types": "./dist/index.d.ts",
|
|
42
|
+
"import": "./dist/index.js"
|
|
43
|
+
},
|
|
44
|
+
"./sqlite-wasm": {
|
|
45
|
+
"types": "./dist/sqlite-wasm.d.ts",
|
|
46
|
+
"import": "./dist/sqlite-wasm.js"
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
"files": [
|
|
50
|
+
"dist",
|
|
51
|
+
"!dist/**/*.map"
|
|
52
|
+
],
|
|
53
|
+
"dependencies": {
|
|
54
|
+
"@bendyline/gezel-knowledge": "1.2.11",
|
|
55
|
+
"@bendyline/gezk": "1.0.7"
|
|
56
|
+
},
|
|
57
|
+
"devDependencies": {
|
|
58
|
+
"@sqlite.org/sqlite-wasm": "3.53.4-build2",
|
|
59
|
+
"@types/node": "^24.10.1",
|
|
60
|
+
"tsup": "^8.5.1",
|
|
61
|
+
"typescript": "^6.0.3",
|
|
62
|
+
"vitest": "^4.1.10"
|
|
63
|
+
},
|
|
64
|
+
"scripts": {
|
|
65
|
+
"build": "tsup",
|
|
66
|
+
"dev": "tsup --watch",
|
|
67
|
+
"typecheck": "tsc --noEmit",
|
|
68
|
+
"test": "vitest run",
|
|
69
|
+
"clean": "node ../../scripts/clean-dirs.mjs dist .turbo"
|
|
70
|
+
}
|
|
6
71
|
}
|