@statewalker/db-sqlite-browser 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -55
- package/dist/index.d.mts +10 -7
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +43 -30
- package/dist/index.mjs.map +1 -1
- package/package.json +6 -4
- package/src/browser-sqlite-db.browser.test.ts +72 -72
- package/src/browser-sqlite-db.test.ts +10 -19
- package/src/browser-sqlite-db.ts +74 -40
package/README.md
CHANGED
|
@@ -2,16 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
## What it is
|
|
4
4
|
|
|
5
|
-
A
|
|
6
|
-
`@
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
A SQLite driver for the browser that implements `Db` from `@statewalker/db-api` on the official
|
|
6
|
+
WebAssembly build, `@sqlite.org/sqlite-wasm` (SQLite 3.53, FTS5 with the `unicode61`, `porter`
|
|
7
|
+
and `trigram` tokenizers). It opens an in-memory database anywhere, and a persistent database in
|
|
8
|
+
the origin-private file system (OPFS) from a Dedicated Worker. Its Db works with `inTransaction`
|
|
9
|
+
from `@statewalker/db-sqlite`.
|
|
9
10
|
|
|
10
11
|
## Why it exists
|
|
11
12
|
|
|
12
|
-
It
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
It lets code written against `Db` run on SQLite in the browser with the same SQL as on Node.js
|
|
14
|
+
(`@statewalker/db-sqlite-node`) and in Durable Objects (`@statewalker/db-sqlite-do`), so search
|
|
15
|
+
and storage code is written and tested once.
|
|
15
16
|
|
|
16
17
|
## How to use
|
|
17
18
|
|
|
@@ -19,70 +20,69 @@ search and storage code runs unchanged in both places.
|
|
|
19
20
|
pnpm add @statewalker/db-sqlite-browser
|
|
20
21
|
```
|
|
21
22
|
|
|
22
|
-
You must serve the engine's `sqlite3.wasm`. It is in
|
|
23
|
-
`@libsql/libsql-wasm-experimental/sqlite-wasm/jswasm/`. The driver loads it
|
|
24
|
-
from `/sqlite3.wasm` unless you pass `wasmUrl`.
|
|
25
|
-
|
|
26
23
|
| Import | Gives | Environment |
|
|
27
24
|
| --- | --- | --- |
|
|
28
|
-
| `@statewalker/db-sqlite-browser` | `newBrowserSqliteDb
|
|
29
|
-
|
|
30
|
-
`newBrowserSqliteDb(options?)` returns `Promise<Db>`.
|
|
25
|
+
| `@statewalker/db-sqlite-browser` | `newBrowserSqliteDb`; types `BrowserDbOptions`, `Db`, `DbEntry`, `DbOptions` | browser: main thread (in-memory) or Dedicated Worker (OPFS); Node.js (in-memory) |
|
|
31
26
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
placeholders.
|
|
27
|
+
`newBrowserSqliteDb({ path?, wasmUrl?, capacity? })` returns `Promise<Db>`. Without `path` the
|
|
28
|
+
database is in memory; that also works under Node.js (where a `wasmUrl`, if given, is a file path).
|
|
29
|
+
With `path` it lives in OPFS through the `opfs-sahpool` VFS, which works only in a Dedicated Worker.
|
|
30
|
+
`capacity` raises the number of files that VFS can hold (see below).
|
|
31
|
+
Use `?` placeholders. Bundlers must not pre-bundle `@sqlite.org/sqlite-wasm` (Vite:
|
|
32
|
+
`optimizeDeps.exclude`), or pass `wasmUrl`.
|
|
37
33
|
|
|
38
34
|
## Examples
|
|
39
35
|
|
|
40
|
-
Full-text search:
|
|
41
|
-
|
|
42
36
|
```ts
|
|
37
|
+
// worker.ts — started with new Worker(new URL("./worker.ts", import.meta.url), { type: "module" })
|
|
43
38
|
import { newBrowserSqliteDb } from "@statewalker/db-sqlite-browser";
|
|
44
39
|
|
|
45
|
-
const db = await newBrowserSqliteDb({
|
|
46
|
-
await db.exec("CREATE VIRTUAL TABLE
|
|
47
|
-
await db.
|
|
48
|
-
const hits = await db.query<{ content: string }>(
|
|
49
|
-
"SELECT content FROM docs WHERE docs MATCH ?",
|
|
50
|
-
["hello"],
|
|
51
|
-
);
|
|
52
|
-
await db.close();
|
|
40
|
+
const db = await newBrowserSqliteDb({ path: "/notes.db" });
|
|
41
|
+
await db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes USING fts5(body)");
|
|
42
|
+
const hits = await db.query("SELECT rowid FROM notes WHERE notes MATCH ?", ["sqlite"]);
|
|
53
43
|
```
|
|
54
44
|
|
|
55
|
-
Vector search:
|
|
56
|
-
|
|
57
45
|
```ts
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const db = await newBrowserSqliteDb();
|
|
61
|
-
await db.exec("CREATE TABLE emb (id INTEGER PRIMARY KEY, vec F32_BLOB(3))");
|
|
62
|
-
await db.exec("CREATE INDEX emb_idx ON emb (libsql_vector_idx(vec))");
|
|
63
|
-
await db.exec("INSERT INTO emb VALUES (1, vector32('[1,0,0]')), (2, vector32('[0,1,0]'))");
|
|
64
|
-
const nearest = await db.query<{ id: number }>(
|
|
65
|
-
"SELECT e.id FROM vector_top_k('emb_idx', '[1,0.1,0]', 1) AS v JOIN emb e ON e.rowid = v.id",
|
|
66
|
-
);
|
|
46
|
+
// A worker that keeps one database per project: reserve room before the default 6 files run out.
|
|
47
|
+
const projectId = "p1";
|
|
48
|
+
const db = await newBrowserSqliteDb({ path: `/projects/${projectId}.db`, capacity: 64 });
|
|
67
49
|
```
|
|
68
50
|
|
|
51
|
+
## How many databases fit: the pool holds a fixed number of files
|
|
52
|
+
|
|
53
|
+
`opfs-sahpool` keeps a pool of OPFS files open for as long as the worker lives, and every file
|
|
54
|
+
SQLite creates takes one: each database file, whether open or closed, and its rollback journal
|
|
55
|
+
while a write transaction runs. The pool starts with **6** files, so a worker can keep about
|
|
56
|
+
five databases if their writes do not overlap.
|
|
57
|
+
|
|
58
|
+
- **What a full pool looks like.** Creating a database, or the journal of its next write, fails with
|
|
59
|
+
`SQLITE_CANTOPEN: sqlite3 result code 14: unable to open database file`. It looks like a bad path
|
|
60
|
+
or a locking problem, but it is the pool running out of files. Opening a database that already
|
|
61
|
+
exists needs no new file.
|
|
62
|
+
- **`capacity` reserves room.** It is the minimum number of files in the pool. It is stored in OPFS
|
|
63
|
+
with the pool, never shrinks, and costs one open file handle per slot from the moment the worker
|
|
64
|
+
starts, used or not, which is why the default is small. Size it at about two files per database
|
|
65
|
+
(the database and its journal), plus a few spare; tens are cheap, hundreds slow the start.
|
|
66
|
+
- **Closing a database does not free its file.** The data stays in the pool until the file is
|
|
67
|
+
deleted, and this driver does not delete files.
|
|
68
|
+
- **One pool per worker.** All databases a worker opens share the pool, and `capacity` applies to
|
|
69
|
+
that pool. Each call that passes `capacity` grows it if needed.
|
|
70
|
+
|
|
69
71
|
## Internals
|
|
70
72
|
|
|
71
|
-
- **
|
|
72
|
-
|
|
73
|
-
a
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
- **
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
- **
|
|
83
|
-
|
|
84
|
-
- **Dependencies:** `@libsql/libsql-wasm-experimental` (the engine),
|
|
85
|
-
`@statewalker/db-api` (the interface types).
|
|
73
|
+
- **Why OPFS only in a Dedicated Worker.** `opfs-sahpool` uses synchronous OPFS access handles,
|
|
74
|
+
which browsers provide only in workers. It needs no `SharedArrayBuffer`, so no COOP/COEP
|
|
75
|
+
headers. Outside a Dedicated Worker (the main thread, Node.js) a `path` rejects at once with
|
|
76
|
+
`newBrowserSqliteDb: { path } needs a Dedicated Worker ...` rather than falling back to memory.
|
|
77
|
+
- **One worker owns a database.** The pool holds its OPFS files open while the worker lives; a
|
|
78
|
+
second worker (or tab) opening the same `path` fails with an OPFS lock error until the first
|
|
79
|
+
one is terminated.
|
|
80
|
+
- **Calls are queued** and transactions open with `BEGIN IMMEDIATE` (see `@statewalker/db-sqlite`).
|
|
81
|
+
Rows are plain objects; BLOBs are `Uint8Array`.
|
|
82
|
+
- **The engine loads once per `wasmUrl`.** A failed load (for example a wrong `wasmUrl`) is not
|
|
83
|
+
retried for that URL.
|
|
84
|
+
- **Dependencies:** `@sqlite.org/sqlite-wasm` (the engine), `@statewalker/db-sqlite` (queue and
|
|
85
|
+
transactions), `@statewalker/db-api` (types).
|
|
86
86
|
|
|
87
87
|
## License
|
|
88
88
|
|
package/dist/index.d.mts
CHANGED
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
import { Db, Db as Db$1, DbEntry, DbOptions, DbOptions as DbOptions$1 } from "@statewalker/db-api";
|
|
2
2
|
//#region src/browser-sqlite-db.d.ts
|
|
3
3
|
type BrowserDbOptions = DbOptions$1 & {
|
|
4
|
-
/** URL
|
|
4
|
+
/** URL of `sqlite3.wasm`. Default: the package's own file, located by the bundler. */
|
|
5
5
|
wasmUrl?: string;
|
|
6
|
+
/**
|
|
7
|
+
* Minimum number of files the OPFS pool can hold, when `path` is set. Each open database takes
|
|
8
|
+
* one file and its rollback journal another while it writes. Default: the pool's own (6).
|
|
9
|
+
*/
|
|
10
|
+
capacity?: number;
|
|
6
11
|
};
|
|
7
12
|
/**
|
|
8
|
-
* Create a
|
|
13
|
+
* Create a SQLite {@link Db} on `@sqlite.org/sqlite-wasm`.
|
|
9
14
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* @param options.path File path for persistent storage. Omit for in-memory.
|
|
14
|
-
* @param options.wasmUrl URL to the sqlite3.wasm file. Defaults to "/sqlite3.wasm".
|
|
15
|
+
* @param options.path OPFS database name; Dedicated Worker only. Omit for in-memory.
|
|
16
|
+
* @param options.wasmUrl URL of `sqlite3.wasm`, when the bundler cannot locate it.
|
|
17
|
+
* @param options.capacity Minimum OPFS pool size in files, with `path`; it only ever grows.
|
|
15
18
|
*/
|
|
16
19
|
export declare function newBrowserSqliteDb(options?: BrowserDbOptions): Promise<Db$1>;
|
|
17
20
|
//#endregion
|
package/dist/index.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.mts","names":["DbOptions","Db"],"sources":["../src/browser-sqlite-db.ts"],"mappings":";;
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":["DbOptions","Db"],"sources":["../src/browser-sqlite-db.ts"],"mappings":";;KASY,mBAAmBA;;EAE7B;;;;;EAKA;;;;;;;;;wBAuDoB,mBAAmB,UAAU,mBAAmB,QAAQC"}
|
package/dist/index.mjs
CHANGED
|
@@ -1,43 +1,56 @@
|
|
|
1
|
-
import sqlite3InitModule from "@
|
|
1
|
+
import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
|
|
2
|
+
import { newSerialisedSqliteDb } from "@statewalker/db-sqlite";
|
|
2
3
|
//#region src/browser-sqlite-db.ts
|
|
3
|
-
|
|
4
|
-
|
|
4
|
+
const MAIN_THREAD_PATH = "newBrowserSqliteDb: { path } needs a Dedicated Worker (the \"opfs-sahpool\" OPFS VFS runs only there). On the main thread, omit path for an in-memory database.";
|
|
5
|
+
/** One engine per wasmUrl, started on first use; one OPFS pool per engine. */
|
|
6
|
+
const engines = /* @__PURE__ */ new Map();
|
|
7
|
+
const pools = /* @__PURE__ */ new WeakMap();
|
|
5
8
|
function getSqlite3(wasmUrl) {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
9
|
+
const key = wasmUrl ?? "";
|
|
10
|
+
let engine = engines.get(key);
|
|
11
|
+
if (!engine) {
|
|
12
|
+
engine = sqlite3InitModule(wasmUrl ? { locateFile: (file) => file.endsWith(".wasm") ? wasmUrl : file } : void 0);
|
|
13
|
+
engines.set(key, engine);
|
|
10
14
|
}
|
|
11
|
-
return
|
|
15
|
+
return engine;
|
|
12
16
|
}
|
|
17
|
+
function getPool(sqlite3) {
|
|
18
|
+
let pool = pools.get(sqlite3);
|
|
19
|
+
if (!pool) {
|
|
20
|
+
pool = sqlite3.installOpfsSAHPoolVfs({});
|
|
21
|
+
pools.set(sqlite3, pool);
|
|
22
|
+
}
|
|
23
|
+
return pool;
|
|
24
|
+
}
|
|
25
|
+
async function openPooled(sqlite3, path, capacity) {
|
|
26
|
+
const pool = await getPool(sqlite3);
|
|
27
|
+
if (capacity !== void 0) await pool.reserveMinimumCapacity(capacity);
|
|
28
|
+
return new pool.OpfsSAHPoolDb(path);
|
|
29
|
+
}
|
|
30
|
+
const inDedicatedWorker = () => typeof globalThis.DedicatedWorkerGlobalScope !== "undefined";
|
|
13
31
|
/**
|
|
14
|
-
* Create a
|
|
32
|
+
* Create a SQLite {@link Db} on `@sqlite.org/sqlite-wasm`.
|
|
15
33
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* @param options.path File path for persistent storage. Omit for in-memory.
|
|
20
|
-
* @param options.wasmUrl URL to the sqlite3.wasm file. Defaults to "/sqlite3.wasm".
|
|
34
|
+
* @param options.path OPFS database name; Dedicated Worker only. Omit for in-memory.
|
|
35
|
+
* @param options.wasmUrl URL of `sqlite3.wasm`, when the bundler cannot locate it.
|
|
36
|
+
* @param options.capacity Minimum OPFS pool size in files, with `path`; it only ever grows.
|
|
21
37
|
*/
|
|
22
38
|
async function newBrowserSqliteDb(options) {
|
|
23
|
-
|
|
24
|
-
const
|
|
25
|
-
const db = new sqlite3.oo1.DB(
|
|
26
|
-
return {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
async exec(sql) {
|
|
39
|
+
if (options?.path && !inDedicatedWorker()) throw new Error(MAIN_THREAD_PATH);
|
|
40
|
+
const sqlite3 = await getSqlite3(options?.wasmUrl);
|
|
41
|
+
const db = options?.path ? await openPooled(sqlite3, options.path, options.capacity) : new sqlite3.oo1.DB(":memory:", "c");
|
|
42
|
+
return newSerialisedSqliteDb({
|
|
43
|
+
all: (sql, params) => db.exec({
|
|
44
|
+
sql,
|
|
45
|
+
bind: params.length > 0 ? params : void 0,
|
|
46
|
+
rowMode: "object",
|
|
47
|
+
returnValue: "resultRows"
|
|
48
|
+
}),
|
|
49
|
+
exec: (sql) => {
|
|
35
50
|
db.exec(sql);
|
|
36
51
|
},
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
}
|
|
40
|
-
};
|
|
52
|
+
close: () => db.close()
|
|
53
|
+
});
|
|
41
54
|
}
|
|
42
55
|
//#endregion
|
|
43
56
|
export { newBrowserSqliteDb };
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","names":[],"sources":["../src/browser-sqlite-db.ts"],"sourcesContent":["import
|
|
1
|
+
{"version":3,"file":"index.mjs","names":[],"sources":["../src/browser-sqlite-db.ts"],"sourcesContent":["import sqlite3InitModule, {\n type BindableValue,\n type Database,\n type SAHPoolUtil,\n type Sqlite3Static,\n} from \"@sqlite.org/sqlite-wasm\";\nimport type { Db, DbOptions } from \"@statewalker/db-api\";\nimport { newSerialisedSqliteDb } from \"@statewalker/db-sqlite\";\n\nexport type BrowserDbOptions = DbOptions & {\n /** URL of `sqlite3.wasm`. Default: the package's own file, located by the bundler. */\n wasmUrl?: string;\n /**\n * Minimum number of files the OPFS pool can hold, when `path` is set. Each open database takes\n * one file and its rollback journal another while it writes. Default: the pool's own (6).\n */\n capacity?: number;\n};\n\nconst MAIN_THREAD_PATH =\n 'newBrowserSqliteDb: { path } needs a Dedicated Worker (the \"opfs-sahpool\" OPFS VFS runs only ' +\n \"there). On the main thread, omit path for an in-memory database.\";\n\ntype InitModule = (module?: { locateFile?: (file: string) => string }) => Promise<Sqlite3Static>;\n\n/** One engine per wasmUrl, started on first use; one OPFS pool per engine. */\nconst engines = new Map<string, Promise<Sqlite3Static>>();\nconst pools = new WeakMap<Sqlite3Static, Promise<SAHPoolUtil>>();\n\nfunction getSqlite3(wasmUrl: string | undefined): Promise<Sqlite3Static> {\n const key = wasmUrl ?? \"\";\n let engine = engines.get(key);\n if (!engine) {\n engine = (sqlite3InitModule as InitModule)(\n wasmUrl ? { locateFile: (file) => (file.endsWith(\".wasm\") ? wasmUrl : file) } : undefined,\n );\n engines.set(key, engine);\n }\n return engine;\n}\n\nfunction getPool(sqlite3: Sqlite3Static): Promise<SAHPoolUtil> {\n let pool = pools.get(sqlite3);\n if (!pool) {\n pool = sqlite3.installOpfsSAHPoolVfs({});\n pools.set(sqlite3, pool);\n }\n return pool;\n}\n\nasync function openPooled(\n sqlite3: Sqlite3Static,\n path: string,\n capacity: number | undefined,\n): Promise<Database> {\n const pool = await getPool(sqlite3);\n if (capacity !== undefined) await pool.reserveMinimumCapacity(capacity);\n return new pool.OpfsSAHPoolDb(path);\n}\n\nconst inDedicatedWorker = () =>\n typeof (globalThis as { DedicatedWorkerGlobalScope?: unknown }).DedicatedWorkerGlobalScope !==\n \"undefined\";\n\n/**\n * Create a SQLite {@link Db} on `@sqlite.org/sqlite-wasm`.\n *\n * @param options.path OPFS database name; Dedicated Worker only. Omit for in-memory.\n * @param options.wasmUrl URL of `sqlite3.wasm`, when the bundler cannot locate it.\n * @param options.capacity Minimum OPFS pool size in files, with `path`; it only ever grows.\n */\nexport async function newBrowserSqliteDb(options?: BrowserDbOptions): Promise<Db> {\n if (options?.path && !inDedicatedWorker()) throw new Error(MAIN_THREAD_PATH);\n const sqlite3 = await getSqlite3(options?.wasmUrl);\n const db: Database = options?.path\n ? await openPooled(sqlite3, options.path, options.capacity)\n : new sqlite3.oo1.DB(\":memory:\", \"c\");\n return newSerialisedSqliteDb({\n all: (sql, params) =>\n db.exec({\n sql,\n bind: params.length > 0 ? (params as BindableValue[]) : undefined,\n rowMode: \"object\",\n returnValue: \"resultRows\",\n }) as Record<string, unknown>[],\n exec: (sql) => {\n db.exec(sql);\n },\n close: () => db.close(),\n });\n}\n"],"mappings":";;;AAmBA,MAAM,mBACJ;;AAMF,MAAM,0BAAU,IAAI,IAAoC;AACxD,MAAM,wBAAQ,IAAI,QAA6C;AAE/D,SAAS,WAAW,SAAqD;CACvE,MAAM,MAAM,WAAW;CACvB,IAAI,SAAS,QAAQ,IAAI,GAAG;CAC5B,IAAI,CAAC,QAAQ;EACX,SAAU,kBACR,UAAU,EAAE,aAAa,SAAU,KAAK,SAAS,OAAO,IAAI,UAAU,KAAM,IAAI,KAAA,CAClF;EACA,QAAQ,IAAI,KAAK,MAAM;CACzB;CACA,OAAO;AACT;AAEA,SAAS,QAAQ,SAA8C;CAC7D,IAAI,OAAO,MAAM,IAAI,OAAO;CAC5B,IAAI,CAAC,MAAM;EACT,OAAO,QAAQ,sBAAsB,CAAC,CAAC;EACvC,MAAM,IAAI,SAAS,IAAI;CACzB;CACA,OAAO;AACT;AAEA,eAAe,WACb,SACA,MACA,UACmB;CACnB,MAAM,OAAO,MAAM,QAAQ,OAAO;CAClC,IAAI,aAAa,KAAA,GAAW,MAAM,KAAK,uBAAuB,QAAQ;CACtE,OAAO,IAAI,KAAK,cAAc,IAAI;AACpC;AAEA,MAAM,0BACJ,OAAQ,WAAwD,+BAChE;;;;;;;;AASF,eAAsB,mBAAmB,SAAyC;CAChF,IAAI,SAAS,QAAQ,CAAC,kBAAkB,GAAG,MAAM,IAAI,MAAM,gBAAgB;CAC3E,MAAM,UAAU,MAAM,WAAW,SAAS,OAAO;CACjD,MAAM,KAAe,SAAS,OAC1B,MAAM,WAAW,SAAS,QAAQ,MAAM,QAAQ,QAAQ,IACxD,IAAI,QAAQ,IAAI,GAAG,YAAY,GAAG;CACtC,OAAO,sBAAsB;EAC3B,MAAM,KAAK,WACT,GAAG,KAAK;GACN;GACA,MAAM,OAAO,SAAS,IAAK,SAA6B,KAAA;GACxD,SAAS;GACT,aAAa;EACf,CAAC;EACH,OAAO,QAAQ;GACb,GAAG,KAAK,GAAG;EACb;EACA,aAAa,GAAG,MAAM;CACxB,CAAC;AACH"}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/db-sqlite-browser",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
|
-
"description": "
|
|
6
|
+
"description": "SQLite WASM (@sqlite.org/sqlite-wasm) driver for @statewalker/db-api in the browser: in-memory anywhere, OPFS persistence in a Dedicated Worker.",
|
|
7
7
|
"bugs": {
|
|
8
8
|
"url": "https://github.com/statewalker/statewalker-db/issues"
|
|
9
9
|
},
|
|
@@ -30,11 +30,13 @@
|
|
|
30
30
|
"src"
|
|
31
31
|
],
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@
|
|
34
|
-
"@statewalker/db-
|
|
33
|
+
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
34
|
+
"@statewalker/db-sqlite": "^0.1.0",
|
|
35
|
+
"@statewalker/db-api": "^0.1.4"
|
|
35
36
|
},
|
|
36
37
|
"devDependencies": {
|
|
37
38
|
"@playwright/test": "^1.63.0",
|
|
39
|
+
"@types/node": "^26.6.4",
|
|
38
40
|
"@vitest/browser": "^5.0.3",
|
|
39
41
|
"@vitest/browser-playwright": "^5.0.3",
|
|
40
42
|
"playwright": "^1.63.0",
|
|
@@ -1,30 +1,21 @@
|
|
|
1
1
|
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { runDbConformance } from "@statewalker/db-tests";
|
|
2
|
+
import { runDbConformance, runSqliteTransactionConformance } from "@statewalker/db-tests";
|
|
3
3
|
import { afterEach, describe, expect, it } from "vitest";
|
|
4
4
|
import { newBrowserSqliteDb } from "./browser-sqlite-db.js";
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* The
|
|
7
|
+
* The sqlite-wasm adapter, exercised in a REAL browser (Vitest browser
|
|
8
8
|
* mode + Playwright/Chromium). Launched only by `vitest.browser.config.ts` (the
|
|
9
9
|
* `test:browser` script) — the Node config excludes `*.browser.test.ts`.
|
|
10
10
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* file-persistence scenario is skipped via `skipFilePersistence`.
|
|
11
|
+
* Everything runs in-memory except the OPFS tests, which use Dedicated Workers;
|
|
12
|
+
* the node-only file-persistence scenario is skipped via `skipFilePersistence`.
|
|
14
13
|
*/
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
runDbConformance(() => newBrowserSqliteDb(), {
|
|
19
|
-
// libsql dialect: `?` placeholder; the spread normalizer drops libsql Row's
|
|
20
|
-
// non-enumerable positional keys so a plain `toEqual` matches.
|
|
21
|
-
placeholder: "?",
|
|
22
|
-
normalizeRow: (row) => ({ ...row }),
|
|
23
|
-
skipFilePersistence: true,
|
|
24
|
-
});
|
|
15
|
+
runDbConformance(() => newBrowserSqliteDb(), { placeholder: "?", skipFilePersistence: true });
|
|
16
|
+
runSqliteTransactionConformance(() => newBrowserSqliteDb());
|
|
25
17
|
|
|
26
|
-
// Dialect-specific browser behavior the shared suite does not cover:
|
|
27
|
-
// FTS5 full-text search and native vector index.
|
|
18
|
+
// Dialect-specific browser behavior the shared suite does not cover: FTS5 full-text search.
|
|
28
19
|
describe("newBrowserSqliteDb", () => {
|
|
29
20
|
let db: Db;
|
|
30
21
|
|
|
@@ -35,6 +26,14 @@ describe("newBrowserSqliteDb", () => {
|
|
|
35
26
|
});
|
|
36
27
|
|
|
37
28
|
describe("FTS5", () => {
|
|
29
|
+
it("has the trigram tokenizer (substring search)", async () => {
|
|
30
|
+
db = await newBrowserSqliteDb();
|
|
31
|
+
await db.exec("CREATE VIRTUAL TABLE g USING fts5(c, tokenize='trigram')");
|
|
32
|
+
await db.exec("INSERT INTO g VALUES ('idx_lefthand')");
|
|
33
|
+
const rows = await db.query<{ c: string }>("SELECT c FROM g WHERE g MATCH 'x_le'");
|
|
34
|
+
expect(rows.map((r) => r.c)).toEqual(["idx_lefthand"]);
|
|
35
|
+
});
|
|
36
|
+
|
|
38
37
|
it("returns empty results for unmatched search terms", async () => {
|
|
39
38
|
db = await newBrowserSqliteDb();
|
|
40
39
|
await db.exec("CREATE VIRTUAL TABLE docs2_fts USING fts5(content)");
|
|
@@ -93,67 +92,68 @@ describe("newBrowserSqliteDb", () => {
|
|
|
93
92
|
expect(rowids).toContain(3);
|
|
94
93
|
});
|
|
95
94
|
});
|
|
95
|
+
});
|
|
96
96
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
it("returns k nearest neighbors ordered by distance", async () => {
|
|
119
|
-
db = await newBrowserSqliteDb();
|
|
120
|
-
|
|
121
|
-
await db.exec("CREATE TABLE vectors (id INTEGER PRIMARY KEY, vec F32_BLOB(3))");
|
|
122
|
-
await db.exec("CREATE INDEX vec_idx2 ON vectors (libsql_vector_idx(vec))");
|
|
123
|
-
await db.exec(`
|
|
124
|
-
INSERT INTO vectors VALUES
|
|
125
|
-
(1, vector32('[1.0, 0.0, 0.0]')),
|
|
126
|
-
(2, vector32('[0.9, 0.1, 0.0]')),
|
|
127
|
-
(3, vector32('[0.0, 1.0, 0.0]')),
|
|
128
|
-
(4, vector32('[0.0, 0.0, 1.0]'))
|
|
129
|
-
`);
|
|
130
|
-
|
|
131
|
-
const rows = await db.query<{ id: number }>(
|
|
132
|
-
"SELECT e.id FROM vector_top_k('vec_idx2', '[1.0, 0.0, 0.0]', 2) AS v JOIN vectors e ON e.rowid = v.id",
|
|
133
|
-
);
|
|
134
|
-
|
|
135
|
-
expect(rows).toHaveLength(2);
|
|
136
|
-
const ids = rows.map((r) => r.id);
|
|
137
|
-
expect(ids).toContain(1);
|
|
138
|
-
expect(ids).toContain(2);
|
|
97
|
+
type TestWorker = {
|
|
98
|
+
onmessage: ((event: { data: { count?: number; error?: string } }) => void) | null;
|
|
99
|
+
onerror: ((event: { message: string }) => void) | null;
|
|
100
|
+
postMessage(message: unknown): void;
|
|
101
|
+
terminate(): void;
|
|
102
|
+
};
|
|
103
|
+
const WorkerClass = (
|
|
104
|
+
globalThis as unknown as {
|
|
105
|
+
Worker: new (url: URL, options: { type: "module" }) => TestWorker;
|
|
106
|
+
}
|
|
107
|
+
).Worker;
|
|
108
|
+
|
|
109
|
+
/** Run one action in a fresh Dedicated Worker, then terminate it (as a page reload would). */
|
|
110
|
+
function inWorker(
|
|
111
|
+
message:
|
|
112
|
+
| { path: string; action: "write" | "count" }
|
|
113
|
+
| { path: string; action: "open-many"; files: number; capacity?: number },
|
|
114
|
+
): Promise<number> {
|
|
115
|
+
return new Promise((resolve, reject) => {
|
|
116
|
+
const worker = new WorkerClass(new URL("../test/opfs-test-worker.ts", import.meta.url), {
|
|
117
|
+
type: "module",
|
|
139
118
|
});
|
|
119
|
+
worker.onmessage = ({ data }) => {
|
|
120
|
+
worker.terminate();
|
|
121
|
+
if (data.error) reject(new Error(data.error));
|
|
122
|
+
else resolve(Number(data.count));
|
|
123
|
+
};
|
|
124
|
+
worker.onerror = (event) => {
|
|
125
|
+
worker.terminate();
|
|
126
|
+
reject(new Error(event.message));
|
|
127
|
+
};
|
|
128
|
+
worker.postMessage(message);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
140
131
|
|
|
141
|
-
|
|
142
|
-
|
|
132
|
+
describe("OPFS persistence (opfs-sahpool)", () => {
|
|
133
|
+
it("rejects a persistent path on the main thread", async () => {
|
|
134
|
+
await expect(newBrowserSqliteDb({ path: "/main-thread.db" })).rejects.toThrow(
|
|
135
|
+
/Dedicated Worker/,
|
|
136
|
+
);
|
|
137
|
+
});
|
|
143
138
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
`);
|
|
139
|
+
it("keeps a path's data from one worker to the next", async () => {
|
|
140
|
+
const path = `/persist-${Date.now()}.db`;
|
|
141
|
+
expect(await inWorker({ path, action: "write" })).toBe(1);
|
|
142
|
+
expect(await inWorker({ path, action: "count" })).toBe(1);
|
|
143
|
+
});
|
|
150
144
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
145
|
+
// Order matters: the pool's capacity is stored in OPFS and only grows, so the default-size
|
|
146
|
+
// case must run before any test raises it. Files stay in the pool after close, so the
|
|
147
|
+
// second case leaves room for the ones the first left behind.
|
|
148
|
+
it("runs out of pool files with the default capacity", async () => {
|
|
149
|
+
const path = `/full-${Date.now()}`;
|
|
150
|
+
await expect(inWorker({ path, action: "open-many", files: 10 })).rejects.toThrow(
|
|
151
|
+
/SQLITE_CANTOPEN/,
|
|
152
|
+
);
|
|
153
|
+
});
|
|
154
154
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
});
|
|
155
|
+
it("opens more databases once capacity is raised", async () => {
|
|
156
|
+
const path = `/many-${Date.now()}`;
|
|
157
|
+
expect(await inWorker({ path, action: "open-many", files: 10, capacity: 32 })).toBe(10);
|
|
158
158
|
});
|
|
159
159
|
});
|
|
@@ -1,29 +1,20 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import { runDbConformance, runSqliteTransactionConformance } from "@statewalker/db-tests";
|
|
1
3
|
import { describe, expect, it } from "vitest";
|
|
2
4
|
import { newBrowserSqliteDb } from "./browser-sqlite-db.js";
|
|
3
5
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
* browser mode (`pnpm test:browser`). Under Node the engine's loader reads
|
|
8
|
-
* `wasmUrl` as a file path, which is enough to check how the engine is loaded.
|
|
9
|
-
*/
|
|
10
|
-
const wasmPath = decodeURIComponent(
|
|
11
|
-
new URL(
|
|
12
|
-
"../node_modules/@libsql/libsql-wasm-experimental/sqlite-wasm/jswasm/sqlite3.wasm",
|
|
13
|
-
import.meta.url,
|
|
14
|
-
).pathname,
|
|
15
|
-
);
|
|
6
|
+
// The same WASM engine as in the browser, in memory: the shared suites run under Node too.
|
|
7
|
+
runDbConformance(() => newBrowserSqliteDb(), { placeholder: "?", skipFilePersistence: true });
|
|
8
|
+
runSqliteTransactionConformance(() => newBrowserSqliteDb());
|
|
16
9
|
|
|
17
|
-
describe("
|
|
18
|
-
it("
|
|
19
|
-
|
|
20
|
-
expect(typeof mod.newBrowserSqliteDb).toBe("function");
|
|
10
|
+
describe("newBrowserSqliteDb under Node", () => {
|
|
11
|
+
it("rejects a persistent path outside a Dedicated Worker", async () => {
|
|
12
|
+
await expect(newBrowserSqliteDb({ path: "/data.db" })).rejects.toThrow(/Dedicated Worker/);
|
|
21
13
|
});
|
|
22
14
|
|
|
23
15
|
it("honors a different wasmUrl after an earlier call", async () => {
|
|
24
|
-
await expect(newBrowserSqliteDb({ wasmUrl: "/nonexistent/sqlite3.wasm" })).rejects.toThrow(
|
|
25
|
-
|
|
26
|
-
);
|
|
16
|
+
await expect(newBrowserSqliteDb({ wasmUrl: "/nonexistent/sqlite3.wasm" })).rejects.toThrow();
|
|
17
|
+
const wasmPath = createRequire(import.meta.url).resolve("@sqlite.org/sqlite-wasm/sqlite3.wasm");
|
|
27
18
|
const db = await newBrowserSqliteDb({ wasmUrl: wasmPath });
|
|
28
19
|
try {
|
|
29
20
|
expect(await db.query("SELECT 1 AS x")).toEqual([{ x: 1 }]);
|
package/src/browser-sqlite-db.ts
CHANGED
|
@@ -1,57 +1,91 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import sqlite3InitModule, {
|
|
2
|
+
type BindableValue,
|
|
3
|
+
type Database,
|
|
4
|
+
type SAHPoolUtil,
|
|
5
|
+
type Sqlite3Static,
|
|
6
|
+
} from "@sqlite.org/sqlite-wasm";
|
|
7
|
+
import type { Db, DbOptions } from "@statewalker/db-api";
|
|
8
|
+
import { newSerialisedSqliteDb } from "@statewalker/db-sqlite";
|
|
4
9
|
|
|
5
10
|
export type BrowserDbOptions = DbOptions & {
|
|
6
|
-
/** URL
|
|
11
|
+
/** URL of `sqlite3.wasm`. Default: the package's own file, located by the bundler. */
|
|
7
12
|
wasmUrl?: string;
|
|
13
|
+
/**
|
|
14
|
+
* Minimum number of files the OPFS pool can hold, when `path` is set. Each open database takes
|
|
15
|
+
* one file and its rollback journal another while it writes. Default: the pool's own (6).
|
|
16
|
+
*/
|
|
17
|
+
capacity?: number;
|
|
8
18
|
};
|
|
9
19
|
|
|
10
|
-
|
|
11
|
-
|
|
20
|
+
const MAIN_THREAD_PATH =
|
|
21
|
+
'newBrowserSqliteDb: { path } needs a Dedicated Worker (the "opfs-sahpool" OPFS VFS runs only ' +
|
|
22
|
+
"there). On the main thread, omit path for an in-memory database.";
|
|
12
23
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
24
|
+
type InitModule = (module?: { locateFile?: (file: string) => string }) => Promise<Sqlite3Static>;
|
|
25
|
+
|
|
26
|
+
/** One engine per wasmUrl, started on first use; one OPFS pool per engine. */
|
|
27
|
+
const engines = new Map<string, Promise<Sqlite3Static>>();
|
|
28
|
+
const pools = new WeakMap<Sqlite3Static, Promise<SAHPoolUtil>>();
|
|
29
|
+
|
|
30
|
+
function getSqlite3(wasmUrl: string | undefined): Promise<Sqlite3Static> {
|
|
31
|
+
const key = wasmUrl ?? "";
|
|
32
|
+
let engine = engines.get(key);
|
|
33
|
+
if (!engine) {
|
|
34
|
+
engine = (sqlite3InitModule as InitModule)(
|
|
35
|
+
wasmUrl ? { locateFile: (file) => (file.endsWith(".wasm") ? wasmUrl : file) } : undefined,
|
|
36
|
+
);
|
|
37
|
+
engines.set(key, engine);
|
|
38
|
+
}
|
|
39
|
+
return engine;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function getPool(sqlite3: Sqlite3Static): Promise<SAHPoolUtil> {
|
|
43
|
+
let pool = pools.get(sqlite3);
|
|
44
|
+
if (!pool) {
|
|
45
|
+
pool = sqlite3.installOpfsSAHPoolVfs({});
|
|
46
|
+
pools.set(sqlite3, pool);
|
|
20
47
|
}
|
|
21
|
-
return
|
|
48
|
+
return pool;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function openPooled(
|
|
52
|
+
sqlite3: Sqlite3Static,
|
|
53
|
+
path: string,
|
|
54
|
+
capacity: number | undefined,
|
|
55
|
+
): Promise<Database> {
|
|
56
|
+
const pool = await getPool(sqlite3);
|
|
57
|
+
if (capacity !== undefined) await pool.reserveMinimumCapacity(capacity);
|
|
58
|
+
return new pool.OpfsSAHPoolDb(path);
|
|
22
59
|
}
|
|
23
60
|
|
|
61
|
+
const inDedicatedWorker = () =>
|
|
62
|
+
typeof (globalThis as { DedicatedWorkerGlobalScope?: unknown }).DedicatedWorkerGlobalScope !==
|
|
63
|
+
"undefined";
|
|
64
|
+
|
|
24
65
|
/**
|
|
25
|
-
* Create a
|
|
66
|
+
* Create a SQLite {@link Db} on `@sqlite.org/sqlite-wasm`.
|
|
26
67
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* @param options.path File path for persistent storage. Omit for in-memory.
|
|
31
|
-
* @param options.wasmUrl URL to the sqlite3.wasm file. Defaults to "/sqlite3.wasm".
|
|
68
|
+
* @param options.path OPFS database name; Dedicated Worker only. Omit for in-memory.
|
|
69
|
+
* @param options.wasmUrl URL of `sqlite3.wasm`, when the bundler cannot locate it.
|
|
70
|
+
* @param options.capacity Minimum OPFS pool size in files, with `path`; it only ever grows.
|
|
32
71
|
*/
|
|
33
72
|
export async function newBrowserSqliteDb(options?: BrowserDbOptions): Promise<Db> {
|
|
34
|
-
|
|
35
|
-
const sqlite3 = await getSqlite3(wasmUrl);
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
return {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
73
|
+
if (options?.path && !inDedicatedWorker()) throw new Error(MAIN_THREAD_PATH);
|
|
74
|
+
const sqlite3 = await getSqlite3(options?.wasmUrl);
|
|
75
|
+
const db: Database = options?.path
|
|
76
|
+
? await openPooled(sqlite3, options.path, options.capacity)
|
|
77
|
+
: new sqlite3.oo1.DB(":memory:", "c");
|
|
78
|
+
return newSerialisedSqliteDb({
|
|
79
|
+
all: (sql, params) =>
|
|
80
|
+
db.exec({
|
|
81
|
+
sql,
|
|
82
|
+
bind: params.length > 0 ? (params as BindableValue[]) : undefined,
|
|
43
83
|
rowMode: "object",
|
|
44
|
-
|
|
45
|
-
})
|
|
46
|
-
|
|
47
|
-
},
|
|
48
|
-
|
|
49
|
-
async exec(sql: string): Promise<void> {
|
|
84
|
+
returnValue: "resultRows",
|
|
85
|
+
}) as Record<string, unknown>[],
|
|
86
|
+
exec: (sql) => {
|
|
50
87
|
db.exec(sql);
|
|
51
88
|
},
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
db.close();
|
|
55
|
-
},
|
|
56
|
-
};
|
|
89
|
+
close: () => db.close(),
|
|
90
|
+
});
|
|
57
91
|
}
|