@aztec/kv-store 0.0.1-commit.aada20e3 → 0.0.1-commit.b2a5d0dd1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/bench/shared_map_bench.d.ts +19 -0
- package/dest/bench/shared_map_bench.d.ts.map +1 -0
- package/dest/bench/shared_map_bench.js +91 -0
- package/dest/indexeddb/index.d.ts +2 -2
- package/dest/indexeddb/index.d.ts.map +1 -1
- package/dest/indexeddb/map.d.ts +7 -1
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +12 -2
- package/dest/indexeddb/multi_map.js +1 -1
- package/dest/indexeddb/store.d.ts +1 -1
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +6 -4
- package/dest/lmdb/index.d.ts +2 -2
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +3 -3
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +12 -8
- package/dest/lmdb-v2/factory.d.ts +2 -2
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +21 -19
- package/dest/sqlite-opfs/array.d.ts +21 -0
- package/dest/sqlite-opfs/array.d.ts.map +1 -0
- package/dest/sqlite-opfs/array.js +128 -0
- package/dest/sqlite-opfs/index.d.ts +7 -0
- package/dest/sqlite-opfs/index.d.ts.map +1 -0
- package/dest/sqlite-opfs/index.js +13 -0
- package/dest/sqlite-opfs/map.d.ts +35 -0
- package/dest/sqlite-opfs/map.d.ts.map +1 -0
- package/dest/sqlite-opfs/map.js +163 -0
- package/dest/sqlite-opfs/messages.d.ts +58 -0
- package/dest/sqlite-opfs/messages.d.ts.map +1 -0
- package/dest/sqlite-opfs/messages.js +5 -0
- package/dest/sqlite-opfs/multi_map.d.ts +16 -0
- package/dest/sqlite-opfs/multi_map.d.ts.map +1 -0
- package/dest/sqlite-opfs/multi_map.js +67 -0
- package/dest/sqlite-opfs/set.d.ts +13 -0
- package/dest/sqlite-opfs/set.d.ts.map +1 -0
- package/dest/sqlite-opfs/set.js +19 -0
- package/dest/sqlite-opfs/singleton.d.ts +13 -0
- package/dest/sqlite-opfs/singleton.d.ts.map +1 -0
- package/dest/sqlite-opfs/singleton.js +48 -0
- package/dest/sqlite-opfs/store.d.ts +70 -0
- package/dest/sqlite-opfs/store.d.ts.map +1 -0
- package/dest/sqlite-opfs/store.js +242 -0
- package/dest/sqlite-opfs/worker.d.ts +2 -0
- package/dest/sqlite-opfs/worker.d.ts.map +1 -0
- package/dest/sqlite-opfs/worker.js +194 -0
- package/package.json +17 -15
- package/src/bench/shared_map_bench.ts +111 -0
- package/src/indexeddb/index.ts +1 -1
- package/src/indexeddb/map.ts +14 -2
- package/src/indexeddb/multi_map.ts +1 -1
- package/src/indexeddb/store.ts +6 -4
- package/src/lmdb/index.ts +1 -1
- package/src/lmdb/store.ts +12 -8
- package/src/lmdb-v2/factory.ts +1 -1
- package/src/lmdb-v2/read_transaction.ts +23 -23
- package/src/sqlite-opfs/array.ts +124 -0
- package/src/sqlite-opfs/index.ts +27 -0
- package/src/sqlite-opfs/map.ts +163 -0
- package/src/sqlite-opfs/messages.ts +28 -0
- package/src/sqlite-opfs/multi_map.ts +74 -0
- package/src/sqlite-opfs/set.ts +29 -0
- package/src/sqlite-opfs/singleton.ts +48 -0
- package/src/sqlite-opfs/store.ts +248 -0
- package/src/sqlite-opfs/worker.ts +162 -0
- package/dest/config.d.ts +0 -17
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -26
- package/src/config.ts +0 -36
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { SerialQueue } from '@aztec/foundation/queue';
|
|
2
|
+
import { SQLiteOPFSAztecArray } from './array.js';
|
|
3
|
+
import { SQLiteOPFSAztecMap } from './map.js';
|
|
4
|
+
import { SQLiteOPFSAztecMultiMap } from './multi_map.js';
|
|
5
|
+
import { SQLiteOPFSAztecSet } from './set.js';
|
|
6
|
+
import { SQLiteOPFSAztecSingleton } from './singleton.js';
|
|
7
|
+
/**
|
|
8
|
+
* Main-thread handle for a SQLite database persisted to OPFS via the `opfs-sahpool`
|
|
9
|
+
* VFS. Owns a dedicated Web Worker (the SAH Pool VFS requires Worker context) and
|
|
10
|
+
* routes every SQL op through it via typed postMessage RPC.
|
|
11
|
+
*
|
|
12
|
+
* Transaction ordering is guaranteed by a `SerialQueue` on the main thread combined
|
|
13
|
+
* with an `#inTx` flag: outside a `transactionAsync` block, each op acquires the
|
|
14
|
+
* queue for its own auto-commit; inside a block, the outer call holds the queue and
|
|
15
|
+
* nested ops bypass it to avoid deadlock.
|
|
16
|
+
*/ export class AztecSQLiteOPFSStore {
|
|
17
|
+
isEphemeral;
|
|
18
|
+
#worker;
|
|
19
|
+
#pending;
|
|
20
|
+
#txQueue;
|
|
21
|
+
#name;
|
|
22
|
+
#log;
|
|
23
|
+
#nextId;
|
|
24
|
+
#inTx;
|
|
25
|
+
#closed;
|
|
26
|
+
constructor(worker, name, log, isEphemeral){
|
|
27
|
+
this.isEphemeral = isEphemeral;
|
|
28
|
+
this.#pending = new Map();
|
|
29
|
+
this.#txQueue = new SerialQueue();
|
|
30
|
+
this.#nextId = 0;
|
|
31
|
+
this.#inTx = false;
|
|
32
|
+
this.#closed = false;
|
|
33
|
+
this.#worker = worker;
|
|
34
|
+
this.#name = name;
|
|
35
|
+
this.#log = log;
|
|
36
|
+
this.#worker.onmessage = (ev)=>{
|
|
37
|
+
const { id } = ev.data;
|
|
38
|
+
const handler = this.#pending.get(id);
|
|
39
|
+
if (!handler) {
|
|
40
|
+
this.#log.warn(`SQLite worker: no pending handler for id ${id}`);
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
this.#pending.delete(id);
|
|
44
|
+
handler.resolve(ev.data);
|
|
45
|
+
};
|
|
46
|
+
this.#worker.onerror = (ev)=>{
|
|
47
|
+
this.#log.error(`SQLite worker crashed: ${ev.message}`);
|
|
48
|
+
this.#rejectPending(`SQLite worker crashed: ${ev.message}`);
|
|
49
|
+
};
|
|
50
|
+
this.#txQueue.start();
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Opens (or creates) a SQLite database stored in the OPFS SAH Pool. When `ephemeral`
|
|
54
|
+
* is true the database lives only in memory and is lost when the worker terminates.
|
|
55
|
+
* Pass `poolDirectory` to place the SAH Pool in a non-default OPFS subdirectory —
|
|
56
|
+
* required when multiple stores coexist in the same tab, because the SAH Pool holds
|
|
57
|
+
* an exclusive lock on its directory.
|
|
58
|
+
*/ static async open(log, name, ephemeral = false, poolDirectory) {
|
|
59
|
+
const dbName = name && !ephemeral ? name : `tmp-${globalThis.crypto.getRandomValues(new Uint8Array(8)).join('')}`;
|
|
60
|
+
log.debug(`Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}database ${dbName}`);
|
|
61
|
+
const worker = new Worker(new URL('./worker.js', import.meta.url), {
|
|
62
|
+
type: 'module'
|
|
63
|
+
});
|
|
64
|
+
const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral);
|
|
65
|
+
await store.#sendRequest({
|
|
66
|
+
type: 'init',
|
|
67
|
+
id: store.#allocId(),
|
|
68
|
+
dbName,
|
|
69
|
+
ephemeral,
|
|
70
|
+
poolDirectory
|
|
71
|
+
});
|
|
72
|
+
return store;
|
|
73
|
+
}
|
|
74
|
+
openMap(name) {
|
|
75
|
+
return new SQLiteOPFSAztecMap(this, name);
|
|
76
|
+
}
|
|
77
|
+
openSet(name) {
|
|
78
|
+
return new SQLiteOPFSAztecSet(this, name);
|
|
79
|
+
}
|
|
80
|
+
openMultiMap(name) {
|
|
81
|
+
return new SQLiteOPFSAztecMultiMap(this, name);
|
|
82
|
+
}
|
|
83
|
+
openCounter(_name) {
|
|
84
|
+
throw new Error('Method not implemented.');
|
|
85
|
+
}
|
|
86
|
+
openArray(name) {
|
|
87
|
+
return new SQLiteOPFSAztecArray(this, name);
|
|
88
|
+
}
|
|
89
|
+
openSingleton(name) {
|
|
90
|
+
return new SQLiteOPFSAztecSingleton(this, name);
|
|
91
|
+
}
|
|
92
|
+
transactionAsync(callback) {
|
|
93
|
+
// Nested calls join the outer transaction — SQLite does not support nested BEGIN,
|
|
94
|
+
// and re-acquiring the SerialQueue while the outer call holds it would deadlock.
|
|
95
|
+
// Errors in the nested callback propagate to the outer catch, which rolls back the
|
|
96
|
+
// whole thing (the standard "nested tx = savepoint-free join" semantic).
|
|
97
|
+
if (this.#inTx) {
|
|
98
|
+
return callback();
|
|
99
|
+
}
|
|
100
|
+
return this.#txQueue.put(async ()=>{
|
|
101
|
+
this.#inTx = true;
|
|
102
|
+
await this.#sendRequest({
|
|
103
|
+
type: 'begin',
|
|
104
|
+
id: this.#allocId()
|
|
105
|
+
});
|
|
106
|
+
try {
|
|
107
|
+
const result = await callback();
|
|
108
|
+
await this.#sendRequest({
|
|
109
|
+
type: 'commit',
|
|
110
|
+
id: this.#allocId()
|
|
111
|
+
});
|
|
112
|
+
return result;
|
|
113
|
+
} catch (err) {
|
|
114
|
+
await this.#sendRequest({
|
|
115
|
+
type: 'rollback',
|
|
116
|
+
id: this.#allocId()
|
|
117
|
+
}).catch((rollbackErr)=>this.#log.warn(`SQLite ROLLBACK failed: ${rollbackErr instanceof Error ? rollbackErr.message : rollbackErr}`));
|
|
118
|
+
throw err;
|
|
119
|
+
} finally{
|
|
120
|
+
this.#inTx = false;
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
async clear() {
|
|
125
|
+
await this.runAsync('DELETE FROM data');
|
|
126
|
+
}
|
|
127
|
+
async delete() {
|
|
128
|
+
if (this.#closed) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
this.#closed = true;
|
|
132
|
+
await this.#txQueue.end();
|
|
133
|
+
await this.#sendRequest({
|
|
134
|
+
type: 'deleteDb',
|
|
135
|
+
id: this.#allocId(),
|
|
136
|
+
dbName: this.#name
|
|
137
|
+
}).catch((err)=>this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`));
|
|
138
|
+
this.#worker.terminate();
|
|
139
|
+
this.#rejectPending('SQLite store deleted');
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Placeholder — returns zeros to mirror the IndexedDB backend. SQLite exposes real
|
|
143
|
+
* numbers cheaply via `PRAGMA page_count` / `page_size` / `freelist_count` and
|
|
144
|
+
* `SELECT COUNT(*) FROM data`, which would populate `physicalFileSize`, `actualSize`,
|
|
145
|
+
* and `numItems` meaningfully (`mappingSize` stays 0 — it's an LMDB mmap concept).
|
|
146
|
+
* Upgrade when any caller actually consumes these values; all current consumers
|
|
147
|
+
* tolerate zeros.
|
|
148
|
+
*/ estimateSize() {
|
|
149
|
+
return Promise.resolve({
|
|
150
|
+
mappingSize: 0,
|
|
151
|
+
physicalFileSize: 0,
|
|
152
|
+
actualSize: 0,
|
|
153
|
+
numItems: 0
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
async close() {
|
|
157
|
+
if (this.#closed) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
this.#closed = true;
|
|
161
|
+
await this.#txQueue.end();
|
|
162
|
+
await this.#sendRequest({
|
|
163
|
+
type: 'close',
|
|
164
|
+
id: this.#allocId()
|
|
165
|
+
}).catch(()=>{});
|
|
166
|
+
this.#worker.terminate();
|
|
167
|
+
this.#rejectPending('SQLite store closed');
|
|
168
|
+
}
|
|
169
|
+
backupTo(_dstPath, _compact) {
|
|
170
|
+
throw new Error('Method not implemented.');
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Returns a raw SQLite image (bytes suitable for writing as a `.sqlite` file and
|
|
174
|
+
* opening in any SQLite tool). Works only for non-ephemeral DBs because the OPFS
|
|
175
|
+
* SAH Pool has to be initialized. Useful for inspection/debugging.
|
|
176
|
+
*/ async exportDb() {
|
|
177
|
+
const resp = await this.#sendRequest({
|
|
178
|
+
type: 'export',
|
|
179
|
+
id: this.#allocId()
|
|
180
|
+
});
|
|
181
|
+
if (!('bytes' in resp) || !resp.bytes) {
|
|
182
|
+
throw new Error('exportDb: worker returned no bytes');
|
|
183
|
+
}
|
|
184
|
+
return resp.bytes;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Runs a write statement (INSERT/UPDATE/DELETE/DDL). If called inside a
|
|
188
|
+
* `transactionAsync` block, bypasses the queue; otherwise acquires it so the
|
|
189
|
+
* op runs in its own auto-commit.
|
|
190
|
+
*/ runAsync(sql, bind) {
|
|
191
|
+
const send = ()=>this.#sendRequest({
|
|
192
|
+
type: 'run',
|
|
193
|
+
id: this.#allocId(),
|
|
194
|
+
sql,
|
|
195
|
+
bind
|
|
196
|
+
}).then((r)=>({
|
|
197
|
+
changes: 'changes' in r ? r.changes ?? 0 : 0
|
|
198
|
+
}));
|
|
199
|
+
return this.#inTx ? send() : this.#txQueue.put(send);
|
|
200
|
+
}
|
|
201
|
+
/** Runs a SELECT statement and returns rows in array row-mode. */ allAsync(sql, bind) {
|
|
202
|
+
const send = ()=>this.#sendRequest({
|
|
203
|
+
type: 'all',
|
|
204
|
+
id: this.#allocId(),
|
|
205
|
+
sql,
|
|
206
|
+
bind
|
|
207
|
+
}).then((r)=>'rows' in r ? r.rows ?? [] : []);
|
|
208
|
+
return this.#inTx ? send() : this.#txQueue.put(send);
|
|
209
|
+
}
|
|
210
|
+
#allocId() {
|
|
211
|
+
return ++this.#nextId;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Reject any in-flight requests with `reason`, so callers awaiting a response to a
|
|
215
|
+
* request sent to a now-terminated worker don't hang forever. Called from
|
|
216
|
+
* close()/delete() and from the worker.onerror handler.
|
|
217
|
+
*/ #rejectPending(reason) {
|
|
218
|
+
if (this.#pending.size === 0) {
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
const err = new Error(reason);
|
|
222
|
+
for (const { reject } of this.#pending.values()){
|
|
223
|
+
reject(err);
|
|
224
|
+
}
|
|
225
|
+
this.#pending.clear();
|
|
226
|
+
}
|
|
227
|
+
#sendRequest(req) {
|
|
228
|
+
return new Promise((resolve, reject)=>{
|
|
229
|
+
this.#pending.set(req.id, {
|
|
230
|
+
resolve: (resp)=>{
|
|
231
|
+
if (resp.type === 'err') {
|
|
232
|
+
reject(new Error(resp.message));
|
|
233
|
+
} else {
|
|
234
|
+
resolve(resp);
|
|
235
|
+
}
|
|
236
|
+
},
|
|
237
|
+
reject
|
|
238
|
+
});
|
|
239
|
+
this.#worker.postMessage(req);
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/sqlite-opfs/worker.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/// <reference lib="webworker" />
|
|
2
|
+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
3
|
+
const SCHEMA_SQL = `
|
|
4
|
+
CREATE TABLE IF NOT EXISTS data (
|
|
5
|
+
slot TEXT NOT NULL PRIMARY KEY,
|
|
6
|
+
container TEXT NOT NULL,
|
|
7
|
+
key BLOB NOT NULL,
|
|
8
|
+
key_count INTEGER NOT NULL,
|
|
9
|
+
hash TEXT NOT NULL,
|
|
10
|
+
value BLOB
|
|
11
|
+
) WITHOUT ROWID;
|
|
12
|
+
|
|
13
|
+
CREATE INDEX IF NOT EXISTS idx_container_key ON data(container, key);
|
|
14
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_count ON data(container, key, key_count);
|
|
15
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
|
|
16
|
+
`;
|
|
17
|
+
const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
|
|
18
|
+
const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
|
|
19
|
+
let sqlite3;
|
|
20
|
+
let pool;
|
|
21
|
+
let db;
|
|
22
|
+
let dbPath;
|
|
23
|
+
async function ensurePool(directory) {
|
|
24
|
+
sqlite3 ??= await sqlite3InitModule();
|
|
25
|
+
if (!pool) {
|
|
26
|
+
pool = await sqlite3.installOpfsSAHPoolVfs({
|
|
27
|
+
name: SAH_POOL_VFS_NAME,
|
|
28
|
+
directory,
|
|
29
|
+
initialCapacity: 8
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return pool;
|
|
33
|
+
}
|
|
34
|
+
async function handleInit(dbName, ephemeral, directory) {
|
|
35
|
+
sqlite3 ??= await sqlite3InitModule();
|
|
36
|
+
if (ephemeral) {
|
|
37
|
+
db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
38
|
+
} else {
|
|
39
|
+
const p = await ensurePool(directory ?? DEFAULT_SAH_POOL_DIRECTORY);
|
|
40
|
+
dbPath = normalizeDbPath(dbName);
|
|
41
|
+
db = new p.OpfsSAHPoolDb(dbPath);
|
|
42
|
+
}
|
|
43
|
+
runSql(SCHEMA_SQL);
|
|
44
|
+
}
|
|
45
|
+
function handleClose() {
|
|
46
|
+
db?.close();
|
|
47
|
+
db = undefined;
|
|
48
|
+
dbPath = undefined;
|
|
49
|
+
}
|
|
50
|
+
async function handleExport() {
|
|
51
|
+
if (!db || !dbPath) {
|
|
52
|
+
throw new Error('SQLite worker: no database open to export');
|
|
53
|
+
}
|
|
54
|
+
if (!pool) {
|
|
55
|
+
throw new Error('SQLite worker: no SAH Pool available (ephemeral DBs cannot be exported)');
|
|
56
|
+
}
|
|
57
|
+
return await pool.exportFile(dbPath);
|
|
58
|
+
}
|
|
59
|
+
function handleDeleteDb(dbName) {
|
|
60
|
+
const path = normalizeDbPath(dbName);
|
|
61
|
+
if (db && dbPath === path) {
|
|
62
|
+
db.close();
|
|
63
|
+
db = undefined;
|
|
64
|
+
dbPath = undefined;
|
|
65
|
+
}
|
|
66
|
+
// Ephemeral :memory: DBs never back a file — skip installing a pool just to unlink
|
|
67
|
+
// nothing. installOpfsSAHPoolVfs acquires an exclusive lock on the OPFS SAH
|
|
68
|
+
// directory, and under heavy test churn that can contend with workers from
|
|
69
|
+
// previous tests whose OPFS handles Chromium hasn't yet released, hanging the RPC
|
|
70
|
+
// and then the whole test run.
|
|
71
|
+
if (!pool) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
pool.unlink(path);
|
|
76
|
+
} catch {
|
|
77
|
+
// File may not exist; ignore.
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
function requireDb() {
|
|
81
|
+
if (!db) {
|
|
82
|
+
throw new Error('SQLite worker: no database open');
|
|
83
|
+
}
|
|
84
|
+
return db;
|
|
85
|
+
}
|
|
86
|
+
function runSql(sql, bind) {
|
|
87
|
+
const conn = requireDb();
|
|
88
|
+
conn.exec({
|
|
89
|
+
sql,
|
|
90
|
+
bind
|
|
91
|
+
});
|
|
92
|
+
return {
|
|
93
|
+
changes: conn.changes()
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function selectAll(sql, bind) {
|
|
97
|
+
const conn = requireDb();
|
|
98
|
+
const rows = [];
|
|
99
|
+
conn.exec({
|
|
100
|
+
sql,
|
|
101
|
+
bind,
|
|
102
|
+
rowMode: 'array',
|
|
103
|
+
resultRows: rows
|
|
104
|
+
});
|
|
105
|
+
return rows;
|
|
106
|
+
}
|
|
107
|
+
function normalizeDbPath(dbName) {
|
|
108
|
+
return dbName.startsWith('/') ? dbName : `/${dbName}`;
|
|
109
|
+
}
|
|
110
|
+
function respond(msg) {
|
|
111
|
+
self.postMessage(msg);
|
|
112
|
+
}
|
|
113
|
+
self.onmessage = async (ev)=>{
|
|
114
|
+
const req = ev.data;
|
|
115
|
+
try {
|
|
116
|
+
switch(req.type){
|
|
117
|
+
case 'init':
|
|
118
|
+
await handleInit(req.dbName, req.ephemeral, req.poolDirectory);
|
|
119
|
+
return respond({
|
|
120
|
+
type: 'ok',
|
|
121
|
+
id: req.id
|
|
122
|
+
});
|
|
123
|
+
case 'close':
|
|
124
|
+
handleClose();
|
|
125
|
+
return respond({
|
|
126
|
+
type: 'ok',
|
|
127
|
+
id: req.id
|
|
128
|
+
});
|
|
129
|
+
case 'deleteDb':
|
|
130
|
+
handleDeleteDb(req.dbName);
|
|
131
|
+
return respond({
|
|
132
|
+
type: 'ok',
|
|
133
|
+
id: req.id
|
|
134
|
+
});
|
|
135
|
+
case 'run':
|
|
136
|
+
{
|
|
137
|
+
const { changes } = runSql(req.sql, req.bind);
|
|
138
|
+
return respond({
|
|
139
|
+
type: 'ok',
|
|
140
|
+
id: req.id,
|
|
141
|
+
changes
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
case 'all':
|
|
145
|
+
{
|
|
146
|
+
const rows = selectAll(req.sql, req.bind);
|
|
147
|
+
return respond({
|
|
148
|
+
type: 'ok',
|
|
149
|
+
id: req.id,
|
|
150
|
+
rows
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
case 'export':
|
|
154
|
+
{
|
|
155
|
+
const bytes = await handleExport();
|
|
156
|
+
return respond({
|
|
157
|
+
type: 'ok',
|
|
158
|
+
id: req.id,
|
|
159
|
+
bytes
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
case 'begin':
|
|
163
|
+
runSql('BEGIN');
|
|
164
|
+
return respond({
|
|
165
|
+
type: 'ok',
|
|
166
|
+
id: req.id
|
|
167
|
+
});
|
|
168
|
+
case 'commit':
|
|
169
|
+
runSql('COMMIT');
|
|
170
|
+
return respond({
|
|
171
|
+
type: 'ok',
|
|
172
|
+
id: req.id
|
|
173
|
+
});
|
|
174
|
+
case 'rollback':
|
|
175
|
+
runSql('ROLLBACK');
|
|
176
|
+
return respond({
|
|
177
|
+
type: 'ok',
|
|
178
|
+
id: req.id
|
|
179
|
+
});
|
|
180
|
+
default:
|
|
181
|
+
{
|
|
182
|
+
const _exhaustive = req;
|
|
183
|
+
throw new Error(`Unknown request: ${JSON.stringify(_exhaustive)}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
} catch (err) {
|
|
187
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
188
|
+
respond({
|
|
189
|
+
type: 'err',
|
|
190
|
+
id: req.id,
|
|
191
|
+
message
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/kv-store",
|
|
3
|
-
"version": "0.0.1-commit.
|
|
3
|
+
"version": "0.0.1-commit.b2a5d0dd1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./dest/interfaces/index.js",
|
|
@@ -8,16 +8,17 @@
|
|
|
8
8
|
"./lmdb": "./dest/lmdb/index.js",
|
|
9
9
|
"./lmdb-v2": "./dest/lmdb-v2/index.js",
|
|
10
10
|
"./indexeddb": "./dest/indexeddb/index.js",
|
|
11
|
-
"./
|
|
12
|
-
"./
|
|
11
|
+
"./sqlite-opfs": "./dest/sqlite-opfs/index.js",
|
|
12
|
+
"./stores": "./dest/stores/index.js"
|
|
13
13
|
},
|
|
14
14
|
"scripts": {
|
|
15
15
|
"build": "yarn clean && ../scripts/tsc.sh",
|
|
16
16
|
"build:dev": "../scripts/tsc.sh --watch",
|
|
17
17
|
"clean": "rm -rf ./dest .tsbuildinfo",
|
|
18
18
|
"test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
|
|
19
|
-
"test:browser": "
|
|
20
|
-
"
|
|
19
|
+
"test:browser": "vitest run --config ./vitest.config.ts",
|
|
20
|
+
"bench:browser": "VITE_BENCH=1 vitest run --config ./vitest.config.ts src/bench",
|
|
21
|
+
"test": "yarn test:node",
|
|
21
22
|
"test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
|
|
22
23
|
},
|
|
23
24
|
"inherits": [
|
|
@@ -25,11 +26,12 @@
|
|
|
25
26
|
"./package.local.json"
|
|
26
27
|
],
|
|
27
28
|
"dependencies": {
|
|
28
|
-
"@aztec/constants": "0.0.1-commit.
|
|
29
|
-
"@aztec/ethereum": "0.0.1-commit.
|
|
30
|
-
"@aztec/foundation": "0.0.1-commit.
|
|
31
|
-
"@aztec/native": "0.0.1-commit.
|
|
32
|
-
"@aztec/stdlib": "0.0.1-commit.
|
|
29
|
+
"@aztec/constants": "0.0.1-commit.b2a5d0dd1",
|
|
30
|
+
"@aztec/ethereum": "0.0.1-commit.b2a5d0dd1",
|
|
31
|
+
"@aztec/foundation": "0.0.1-commit.b2a5d0dd1",
|
|
32
|
+
"@aztec/native": "0.0.1-commit.b2a5d0dd1",
|
|
33
|
+
"@aztec/stdlib": "0.0.1-commit.b2a5d0dd1",
|
|
34
|
+
"@sqlite.org/sqlite-wasm": "3.50.4-build1",
|
|
33
35
|
"idb": "^8.0.0",
|
|
34
36
|
"lmdb": "^3.2.0",
|
|
35
37
|
"msgpackr": "^1.11.2",
|
|
@@ -46,19 +48,19 @@
|
|
|
46
48
|
"@types/node": "^22.15.17",
|
|
47
49
|
"@types/sinon": "^17.0.3",
|
|
48
50
|
"@typescript/native-preview": "7.0.0-dev.20260113.1",
|
|
49
|
-
"@
|
|
50
|
-
"
|
|
51
|
-
"@web/test-runner": "^0.19.0",
|
|
52
|
-
"@web/test-runner-playwright": "^0.11.0",
|
|
51
|
+
"@vitest/browser-playwright": "^4.0.0",
|
|
52
|
+
"buffer": "^6.0.3",
|
|
53
53
|
"chai": "^5.1.2",
|
|
54
54
|
"chai-as-promised": "^8.0.1",
|
|
55
55
|
"jest": "^30.0.0",
|
|
56
56
|
"mocha": "^10.8.2",
|
|
57
57
|
"mocha-each": "^2.0.1",
|
|
58
|
+
"playwright": "1.49.0",
|
|
58
59
|
"sinon": "^19.0.2",
|
|
59
60
|
"ts-node": "^10.9.1",
|
|
60
61
|
"typescript": "^5.3.3",
|
|
61
|
-
"util": "^0.12.5"
|
|
62
|
+
"util": "^0.12.5",
|
|
63
|
+
"vitest": "^4.0.0"
|
|
62
64
|
},
|
|
63
65
|
"files": [
|
|
64
66
|
"dest",
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { Timer } from '@aztec/foundation/timer';
|
|
3
|
+
|
|
4
|
+
import type { Key } from '../interfaces/common.js';
|
|
5
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
6
|
+
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
7
|
+
|
|
8
|
+
/** One benchmark measurement. */
|
|
9
|
+
export type BenchResult = {
|
|
10
|
+
/** Benchmark name (includes the backend prefix for disambiguation). */
|
|
11
|
+
name: string;
|
|
12
|
+
value: number;
|
|
13
|
+
unit: 'ms' | 'us';
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
|
|
20
|
+
* populates `results`, and calls `reporter` in `afterAll`.
|
|
21
|
+
*
|
|
22
|
+
* Kept free of Node-only deps (`fs`, `path`) so the same runner works under
|
|
23
|
+
* vitest-browser for IndexedDB and SQLite-OPFS.
|
|
24
|
+
*/
|
|
25
|
+
export function describeAztecMapBench(
|
|
26
|
+
backendPrefix: string,
|
|
27
|
+
getStore: () => Promise<AztecAsyncKVStore>,
|
|
28
|
+
logger: Logger,
|
|
29
|
+
reporter: BenchReporter,
|
|
30
|
+
) {
|
|
31
|
+
describe(`${backendPrefix} Map benchmarks`, () => {
|
|
32
|
+
let store: AztecAsyncKVStore;
|
|
33
|
+
let map: AztecAsyncMap<Key, string>;
|
|
34
|
+
|
|
35
|
+
const results: BenchResult[] = [];
|
|
36
|
+
|
|
37
|
+
const generateKeyValuePairs = (count: number, offset = 0) => {
|
|
38
|
+
const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
|
|
39
|
+
const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
|
|
40
|
+
return keys.map((key, i) => ({ key, value: values[i] }));
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const record = (name: string, value: number, unit: BenchResult['unit']) => {
|
|
44
|
+
results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
beforeEach(async () => {
|
|
48
|
+
store = await getStore();
|
|
49
|
+
map = store.openMap<Key, string>('test');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
afterEach(async () => {
|
|
53
|
+
await store.delete();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
afterAll(async () => {
|
|
57
|
+
const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
|
|
58
|
+
logger.info(`\n${pretty}\n`);
|
|
59
|
+
await reporter(results);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('adds individual values', async () => {
|
|
63
|
+
const pairs = generateKeyValuePairs(1000);
|
|
64
|
+
const timer = new Timer();
|
|
65
|
+
for (const pair of pairs) {
|
|
66
|
+
await map.set(pair.key, pair.value);
|
|
67
|
+
}
|
|
68
|
+
record('Individual insertion', timer.ms() / pairs.length, 'ms');
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('adds batched values', async () => {
|
|
72
|
+
const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
|
|
73
|
+
const timer = new Timer();
|
|
74
|
+
for (const batch of batches) {
|
|
75
|
+
await map.setMany(batch);
|
|
76
|
+
}
|
|
77
|
+
record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('reads individual values', async () => {
|
|
81
|
+
const pairs = generateKeyValuePairs(10000);
|
|
82
|
+
await map.setMany(pairs);
|
|
83
|
+
const timer = new Timer();
|
|
84
|
+
for (const pair of pairs) {
|
|
85
|
+
await map.getAsync(pair.key);
|
|
86
|
+
}
|
|
87
|
+
record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('reads via a cursor', async () => {
|
|
91
|
+
const pairs = generateKeyValuePairs(10000);
|
|
92
|
+
await map.setMany(pairs);
|
|
93
|
+
const timer = new Timer();
|
|
94
|
+
for await (const _ of map.entriesAsync()) {
|
|
95
|
+
// consume
|
|
96
|
+
}
|
|
97
|
+
record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('reads the size of the map', async () => {
|
|
101
|
+
const numIterations = 1000;
|
|
102
|
+
const pairs = generateKeyValuePairs(10000);
|
|
103
|
+
await map.setMany(pairs);
|
|
104
|
+
const timer = new Timer();
|
|
105
|
+
for (let i = 0; i < numIterations; i++) {
|
|
106
|
+
await map.sizeAsync();
|
|
107
|
+
}
|
|
108
|
+
record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
package/src/indexeddb/index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
2
3
|
|
|
3
|
-
import type { DataStoreConfig } from '../config.js';
|
|
4
4
|
import { initStoreForRollupAndSchemaVersion } from '../utils.js';
|
|
5
5
|
import { AztecIndexedDBStore } from './store.js';
|
|
6
6
|
|
package/src/indexeddb/map.ts
CHANGED
|
@@ -31,7 +31,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
31
31
|
|
|
32
32
|
async getAsync(key: K): Promise<V | undefined> {
|
|
33
33
|
const data = await this.db.get(this.slot(key));
|
|
34
|
-
return data
|
|
34
|
+
return data ? this.restoreBuffers(data.value as V) : undefined;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
async hasAsync(key: K): Promise<boolean> {
|
|
@@ -94,7 +94,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
94
94
|
if (range.limit && count >= range.limit) {
|
|
95
95
|
return;
|
|
96
96
|
}
|
|
97
|
-
yield [this.#denormalizeKey(cursor.value.key), cursor.value.value] as [K, V];
|
|
97
|
+
yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
|
|
98
98
|
count++;
|
|
99
99
|
}
|
|
100
100
|
}
|
|
@@ -111,6 +111,18 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
111
111
|
}
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
+
/**
|
|
115
|
+
* IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
|
|
116
|
+
* This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
|
|
117
|
+
* comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
|
|
118
|
+
*/
|
|
119
|
+
protected restoreBuffers(val: V): V {
|
|
120
|
+
if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
|
|
121
|
+
return Buffer.from(val) as V;
|
|
122
|
+
}
|
|
123
|
+
return val;
|
|
124
|
+
}
|
|
125
|
+
|
|
114
126
|
#denormalizeKey(key: (string | number | Uint8Array)[]): K {
|
|
115
127
|
return (key.length > 1 ? key : key[0]) as K;
|
|
116
128
|
}
|