@aztec/kv-store 0.0.1-commit.9372f48 → 0.0.1-commit.949a33fd8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/bench/shared_map_bench.d.ts +19 -0
- package/dest/bench/shared_map_bench.d.ts.map +1 -0
- package/dest/bench/shared_map_bench.js +91 -0
- package/dest/indexeddb/array.js +18 -6
- package/dest/indexeddb/index.d.ts +2 -2
- package/dest/indexeddb/index.d.ts.map +1 -1
- package/dest/indexeddb/map.d.ts +8 -2
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +23 -13
- package/dest/indexeddb/multi_map.d.ts +2 -1
- package/dest/indexeddb/multi_map.d.ts.map +1 -1
- package/dest/indexeddb/multi_map.js +16 -1
- package/dest/indexeddb/singleton.js +3 -1
- package/dest/indexeddb/store.d.ts +3 -3
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +6 -4
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +48 -2
- package/dest/interfaces/multi_map_test_suite.d.ts +1 -1
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/multi_map_test_suite.js +25 -0
- package/dest/lmdb/index.d.ts +2 -2
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +3 -3
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +12 -8
- package/dest/lmdb-v2/factory.d.ts +2 -2
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +21 -19
- package/dest/sqlite-opfs/array.d.ts +21 -0
- package/dest/sqlite-opfs/array.d.ts.map +1 -0
- package/dest/sqlite-opfs/array.js +128 -0
- package/dest/sqlite-opfs/index.d.ts +7 -0
- package/dest/sqlite-opfs/index.d.ts.map +1 -0
- package/dest/sqlite-opfs/index.js +13 -0
- package/dest/sqlite-opfs/map.d.ts +35 -0
- package/dest/sqlite-opfs/map.d.ts.map +1 -0
- package/dest/sqlite-opfs/map.js +163 -0
- package/dest/sqlite-opfs/messages.d.ts +58 -0
- package/dest/sqlite-opfs/messages.d.ts.map +1 -0
- package/dest/sqlite-opfs/messages.js +5 -0
- package/dest/sqlite-opfs/multi_map.d.ts +16 -0
- package/dest/sqlite-opfs/multi_map.d.ts.map +1 -0
- package/dest/sqlite-opfs/multi_map.js +67 -0
- package/dest/sqlite-opfs/set.d.ts +13 -0
- package/dest/sqlite-opfs/set.d.ts.map +1 -0
- package/dest/sqlite-opfs/set.js +19 -0
- package/dest/sqlite-opfs/singleton.d.ts +13 -0
- package/dest/sqlite-opfs/singleton.d.ts.map +1 -0
- package/dest/sqlite-opfs/singleton.js +48 -0
- package/dest/sqlite-opfs/store.d.ts +70 -0
- package/dest/sqlite-opfs/store.d.ts.map +1 -0
- package/dest/sqlite-opfs/store.js +242 -0
- package/dest/sqlite-opfs/worker.d.ts +2 -0
- package/dest/sqlite-opfs/worker.d.ts.map +1 -0
- package/dest/sqlite-opfs/worker.js +194 -0
- package/dest/utils.d.ts +1 -1
- package/dest/utils.d.ts.map +1 -1
- package/dest/utils.js +7 -1
- package/package.json +17 -13
- package/src/bench/shared_map_bench.ts +111 -0
- package/src/indexeddb/array.ts +4 -4
- package/src/indexeddb/index.ts +1 -1
- package/src/indexeddb/map.ts +24 -11
- package/src/indexeddb/multi_map.ts +15 -1
- package/src/indexeddb/singleton.ts +1 -1
- package/src/indexeddb/store.ts +13 -6
- package/src/interfaces/map_test_suite.ts +30 -2
- package/src/interfaces/multi_map_test_suite.ts +32 -0
- package/src/lmdb/index.ts +1 -1
- package/src/lmdb/store.ts +12 -8
- package/src/lmdb-v2/factory.ts +1 -1
- package/src/lmdb-v2/read_transaction.ts +23 -23
- package/src/sqlite-opfs/array.ts +124 -0
- package/src/sqlite-opfs/index.ts +27 -0
- package/src/sqlite-opfs/map.ts +163 -0
- package/src/sqlite-opfs/messages.ts +28 -0
- package/src/sqlite-opfs/multi_map.ts +74 -0
- package/src/sqlite-opfs/set.ts +29 -0
- package/src/sqlite-opfs/singleton.ts +48 -0
- package/src/sqlite-opfs/store.ts +248 -0
- package/src/sqlite-opfs/worker.ts +162 -0
- package/src/utils.ts +10 -0
- package/dest/config.d.ts +0 -17
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -26
- package/src/config.ts +0 -36
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/// <reference lib="webworker" />
|
|
2
|
+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
3
|
+
const SCHEMA_SQL = `
|
|
4
|
+
CREATE TABLE IF NOT EXISTS data (
|
|
5
|
+
slot TEXT NOT NULL PRIMARY KEY,
|
|
6
|
+
container TEXT NOT NULL,
|
|
7
|
+
key BLOB NOT NULL,
|
|
8
|
+
key_count INTEGER NOT NULL,
|
|
9
|
+
hash TEXT NOT NULL,
|
|
10
|
+
value BLOB
|
|
11
|
+
) WITHOUT ROWID;
|
|
12
|
+
|
|
13
|
+
CREATE INDEX IF NOT EXISTS idx_container_key ON data(container, key);
|
|
14
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_count ON data(container, key, key_count);
|
|
15
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
|
|
16
|
+
`;
|
|
17
|
+
const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
|
|
18
|
+
const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
|
|
19
|
+
let sqlite3;
|
|
20
|
+
let pool;
|
|
21
|
+
let db;
|
|
22
|
+
let dbPath;
|
|
23
|
+
async function ensurePool(directory) {
|
|
24
|
+
sqlite3 ??= await sqlite3InitModule();
|
|
25
|
+
if (!pool) {
|
|
26
|
+
pool = await sqlite3.installOpfsSAHPoolVfs({
|
|
27
|
+
name: SAH_POOL_VFS_NAME,
|
|
28
|
+
directory,
|
|
29
|
+
initialCapacity: 8
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return pool;
|
|
33
|
+
}
|
|
34
|
+
async function handleInit(dbName, ephemeral, directory) {
|
|
35
|
+
sqlite3 ??= await sqlite3InitModule();
|
|
36
|
+
if (ephemeral) {
|
|
37
|
+
db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
38
|
+
} else {
|
|
39
|
+
const p = await ensurePool(directory ?? DEFAULT_SAH_POOL_DIRECTORY);
|
|
40
|
+
dbPath = normalizeDbPath(dbName);
|
|
41
|
+
db = new p.OpfsSAHPoolDb(dbPath);
|
|
42
|
+
}
|
|
43
|
+
runSql(SCHEMA_SQL);
|
|
44
|
+
}
|
|
45
|
+
function handleClose() {
|
|
46
|
+
db?.close();
|
|
47
|
+
db = undefined;
|
|
48
|
+
dbPath = undefined;
|
|
49
|
+
}
|
|
50
|
+
async function handleExport() {
|
|
51
|
+
if (!db || !dbPath) {
|
|
52
|
+
throw new Error('SQLite worker: no database open to export');
|
|
53
|
+
}
|
|
54
|
+
if (!pool) {
|
|
55
|
+
throw new Error('SQLite worker: no SAH Pool available (ephemeral DBs cannot be exported)');
|
|
56
|
+
}
|
|
57
|
+
return await pool.exportFile(dbPath);
|
|
58
|
+
}
|
|
59
|
+
function handleDeleteDb(dbName) {
|
|
60
|
+
const path = normalizeDbPath(dbName);
|
|
61
|
+
if (db && dbPath === path) {
|
|
62
|
+
db.close();
|
|
63
|
+
db = undefined;
|
|
64
|
+
dbPath = undefined;
|
|
65
|
+
}
|
|
66
|
+
// Ephemeral :memory: DBs never back a file — skip installing a pool just to unlink
|
|
67
|
+
// nothing. installOpfsSAHPoolVfs acquires an exclusive lock on the OPFS SAH
|
|
68
|
+
// directory, and under heavy test churn that can contend with workers from
|
|
69
|
+
// previous tests whose OPFS handles Chromium hasn't yet released, hanging the RPC
|
|
70
|
+
// and then the whole test run.
|
|
71
|
+
if (!pool) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
pool.unlink(path);
|
|
76
|
+
} catch {
|
|
77
|
+
// File may not exist; ignore.
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
function requireDb() {
|
|
81
|
+
if (!db) {
|
|
82
|
+
throw new Error('SQLite worker: no database open');
|
|
83
|
+
}
|
|
84
|
+
return db;
|
|
85
|
+
}
|
|
86
|
+
function runSql(sql, bind) {
|
|
87
|
+
const conn = requireDb();
|
|
88
|
+
conn.exec({
|
|
89
|
+
sql,
|
|
90
|
+
bind
|
|
91
|
+
});
|
|
92
|
+
return {
|
|
93
|
+
changes: conn.changes()
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function selectAll(sql, bind) {
|
|
97
|
+
const conn = requireDb();
|
|
98
|
+
const rows = [];
|
|
99
|
+
conn.exec({
|
|
100
|
+
sql,
|
|
101
|
+
bind,
|
|
102
|
+
rowMode: 'array',
|
|
103
|
+
resultRows: rows
|
|
104
|
+
});
|
|
105
|
+
return rows;
|
|
106
|
+
}
|
|
107
|
+
function normalizeDbPath(dbName) {
|
|
108
|
+
return dbName.startsWith('/') ? dbName : `/${dbName}`;
|
|
109
|
+
}
|
|
110
|
+
function respond(msg) {
|
|
111
|
+
self.postMessage(msg);
|
|
112
|
+
}
|
|
113
|
+
self.onmessage = async (ev)=>{
|
|
114
|
+
const req = ev.data;
|
|
115
|
+
try {
|
|
116
|
+
switch(req.type){
|
|
117
|
+
case 'init':
|
|
118
|
+
await handleInit(req.dbName, req.ephemeral, req.poolDirectory);
|
|
119
|
+
return respond({
|
|
120
|
+
type: 'ok',
|
|
121
|
+
id: req.id
|
|
122
|
+
});
|
|
123
|
+
case 'close':
|
|
124
|
+
handleClose();
|
|
125
|
+
return respond({
|
|
126
|
+
type: 'ok',
|
|
127
|
+
id: req.id
|
|
128
|
+
});
|
|
129
|
+
case 'deleteDb':
|
|
130
|
+
handleDeleteDb(req.dbName);
|
|
131
|
+
return respond({
|
|
132
|
+
type: 'ok',
|
|
133
|
+
id: req.id
|
|
134
|
+
});
|
|
135
|
+
case 'run':
|
|
136
|
+
{
|
|
137
|
+
const { changes } = runSql(req.sql, req.bind);
|
|
138
|
+
return respond({
|
|
139
|
+
type: 'ok',
|
|
140
|
+
id: req.id,
|
|
141
|
+
changes
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
case 'all':
|
|
145
|
+
{
|
|
146
|
+
const rows = selectAll(req.sql, req.bind);
|
|
147
|
+
return respond({
|
|
148
|
+
type: 'ok',
|
|
149
|
+
id: req.id,
|
|
150
|
+
rows
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
case 'export':
|
|
154
|
+
{
|
|
155
|
+
const bytes = await handleExport();
|
|
156
|
+
return respond({
|
|
157
|
+
type: 'ok',
|
|
158
|
+
id: req.id,
|
|
159
|
+
bytes
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
case 'begin':
|
|
163
|
+
runSql('BEGIN');
|
|
164
|
+
return respond({
|
|
165
|
+
type: 'ok',
|
|
166
|
+
id: req.id
|
|
167
|
+
});
|
|
168
|
+
case 'commit':
|
|
169
|
+
runSql('COMMIT');
|
|
170
|
+
return respond({
|
|
171
|
+
type: 'ok',
|
|
172
|
+
id: req.id
|
|
173
|
+
});
|
|
174
|
+
case 'rollback':
|
|
175
|
+
runSql('ROLLBACK');
|
|
176
|
+
return respond({
|
|
177
|
+
type: 'ok',
|
|
178
|
+
id: req.id
|
|
179
|
+
});
|
|
180
|
+
default:
|
|
181
|
+
{
|
|
182
|
+
const _exhaustive = req;
|
|
183
|
+
throw new Error(`Unknown request: ${JSON.stringify(_exhaustive)}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
} catch (err) {
|
|
187
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
188
|
+
respond({
|
|
189
|
+
type: 'err',
|
|
190
|
+
id: req.id,
|
|
191
|
+
message
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
};
|
package/dest/utils.d.ts
CHANGED
|
@@ -12,4 +12,4 @@ import type { AztecAsyncKVStore, AztecKVStore } from './interfaces/store.js';
|
|
|
12
12
|
* @returns The store (cleared if necessary)
|
|
13
13
|
*/
|
|
14
14
|
export declare function initStoreForRollupAndSchemaVersion<T extends AztecKVStore | AztecAsyncKVStore>(store: T, schemaVersion: number | undefined, rollupAddress: EthAddress | undefined, log?: Logger): Promise<T>;
|
|
15
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
15
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidXRpbHMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy91dGlscy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsVUFBVSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDM0QsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFJcEQsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsWUFBWSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFHN0U7Ozs7Ozs7OztHQVNHO0FBQ0gsd0JBQXNCLGtDQUFrQyxDQUFDLENBQUMsU0FBUyxZQUFZLEdBQUcsaUJBQWlCLEVBQ2pHLEtBQUssRUFBRSxDQUFDLEVBQ1IsYUFBYSxFQUFFLE1BQU0sR0FBRyxTQUFTLEVBQ2pDLGFBQWEsRUFBRSxVQUFVLEdBQUcsU0FBUyxFQUNyQyxHQUFHLENBQUMsRUFBRSxNQUFNLEdBQ1gsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQXFDWiJ9
|
package/dest/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIpD,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAG7E;;;;;;;;;GASG;AACH,wBAAsB,kCAAkC,CAAC,CAAC,SAAS,YAAY,GAAG,iBAAiB,EACjG,KAAK,EAAE,CAAC,EACR,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,aAAa,EAAE,UAAU,GAAG,SAAS,EACrC,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIpD,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAG7E;;;;;;;;;GASG;AACH,wBAAsB,kCAAkC,CAAC,CAAC,SAAS,YAAY,GAAG,iBAAiB,EACjG,KAAK,EAAE,CAAC,EACR,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,aAAa,EAAE,UAAU,GAAG,SAAS,EACrC,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,CAAC,CAAC,CAqCZ"}
|
package/dest/utils.js
CHANGED
|
@@ -17,7 +17,13 @@ import { isSyncStore } from './interfaces/utils.js';
|
|
|
17
17
|
// DB version: database schema version + rollup address combined)
|
|
18
18
|
const dbVersion = store.openSingleton('dbVersion');
|
|
19
19
|
const storedDatabaseVersion = isSyncStore(store) ? dbVersion.get() : await dbVersion.getAsync();
|
|
20
|
-
if (
|
|
20
|
+
// Check if this is an old format store (has rollupAddress singleton but no dbVersion)
|
|
21
|
+
const oldRollupSingleton = store.openSingleton('rollupAddress');
|
|
22
|
+
const hasOldFormat = isSyncStore(store) ? !storedDatabaseVersion && !!oldRollupSingleton.get() : !storedDatabaseVersion && !!await oldRollupSingleton.getAsync();
|
|
23
|
+
if (hasOldFormat || doesStoreNeedToBeCleared(targetDatabaseVersion, storedDatabaseVersion, targetSchemaVersion, targetRollupAddress, log)) {
|
|
24
|
+
if (hasOldFormat) {
|
|
25
|
+
log?.warn('Detected old store format without dbVersion, clearing database');
|
|
26
|
+
}
|
|
21
27
|
await store.clear();
|
|
22
28
|
}
|
|
23
29
|
await dbVersion.set(targetDatabaseVersion.toBuffer().toString('utf-8'));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/kv-store",
|
|
3
|
-
"version": "0.0.1-commit.
|
|
3
|
+
"version": "0.0.1-commit.949a33fd8",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./dest/interfaces/index.js",
|
|
@@ -8,15 +8,16 @@
|
|
|
8
8
|
"./lmdb": "./dest/lmdb/index.js",
|
|
9
9
|
"./lmdb-v2": "./dest/lmdb-v2/index.js",
|
|
10
10
|
"./indexeddb": "./dest/indexeddb/index.js",
|
|
11
|
-
"./
|
|
12
|
-
"./
|
|
11
|
+
"./sqlite-opfs": "./dest/sqlite-opfs/index.js",
|
|
12
|
+
"./stores": "./dest/stores/index.js"
|
|
13
13
|
},
|
|
14
14
|
"scripts": {
|
|
15
15
|
"build": "yarn clean && ../scripts/tsc.sh",
|
|
16
16
|
"build:dev": "../scripts/tsc.sh --watch",
|
|
17
17
|
"clean": "rm -rf ./dest .tsbuildinfo",
|
|
18
18
|
"test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
|
|
19
|
-
"test:browser": "
|
|
19
|
+
"test:browser": "vitest run --config ./vitest.config.ts",
|
|
20
|
+
"bench:browser": "VITE_BENCH=1 vitest run --config ./vitest.config.ts src/bench",
|
|
20
21
|
"test": "yarn test:node",
|
|
21
22
|
"test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
|
|
22
23
|
},
|
|
@@ -25,11 +26,12 @@
|
|
|
25
26
|
"./package.local.json"
|
|
26
27
|
],
|
|
27
28
|
"dependencies": {
|
|
28
|
-
"@aztec/constants": "0.0.1-commit.
|
|
29
|
-
"@aztec/ethereum": "0.0.1-commit.
|
|
30
|
-
"@aztec/foundation": "0.0.1-commit.
|
|
31
|
-
"@aztec/native": "0.0.1-commit.
|
|
32
|
-
"@aztec/stdlib": "0.0.1-commit.
|
|
29
|
+
"@aztec/constants": "0.0.1-commit.949a33fd8",
|
|
30
|
+
"@aztec/ethereum": "0.0.1-commit.949a33fd8",
|
|
31
|
+
"@aztec/foundation": "0.0.1-commit.949a33fd8",
|
|
32
|
+
"@aztec/native": "0.0.1-commit.949a33fd8",
|
|
33
|
+
"@aztec/stdlib": "0.0.1-commit.949a33fd8",
|
|
34
|
+
"@sqlite.org/sqlite-wasm": "3.50.4-build1",
|
|
33
35
|
"idb": "^8.0.0",
|
|
34
36
|
"lmdb": "^3.2.0",
|
|
35
37
|
"msgpackr": "^1.11.2",
|
|
@@ -46,17 +48,19 @@
|
|
|
46
48
|
"@types/node": "^22.15.17",
|
|
47
49
|
"@types/sinon": "^17.0.3",
|
|
48
50
|
"@typescript/native-preview": "7.0.0-dev.20260113.1",
|
|
49
|
-
"@
|
|
50
|
-
"
|
|
51
|
-
"@web/test-runner-playwright": "^0.11.0",
|
|
51
|
+
"@vitest/browser-playwright": "^4.0.0",
|
|
52
|
+
"buffer": "^6.0.3",
|
|
52
53
|
"chai": "^5.1.2",
|
|
53
54
|
"chai-as-promised": "^8.0.1",
|
|
54
55
|
"jest": "^30.0.0",
|
|
55
56
|
"mocha": "^10.8.2",
|
|
56
57
|
"mocha-each": "^2.0.1",
|
|
58
|
+
"playwright": "1.49.0",
|
|
57
59
|
"sinon": "^19.0.2",
|
|
58
60
|
"ts-node": "^10.9.1",
|
|
59
|
-
"typescript": "^5.3.3"
|
|
61
|
+
"typescript": "^5.3.3",
|
|
62
|
+
"util": "^0.12.5",
|
|
63
|
+
"vitest": "^4.0.0"
|
|
60
64
|
},
|
|
61
65
|
"files": [
|
|
62
66
|
"dest",
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { Timer } from '@aztec/foundation/timer';
|
|
3
|
+
|
|
4
|
+
import type { Key } from '../interfaces/common.js';
|
|
5
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
6
|
+
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
7
|
+
|
|
8
|
+
/** One benchmark measurement. */
|
|
9
|
+
export type BenchResult = {
|
|
10
|
+
/** Benchmark name (includes the backend prefix for disambiguation). */
|
|
11
|
+
name: string;
|
|
12
|
+
value: number;
|
|
13
|
+
unit: 'ms' | 'us';
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
|
|
20
|
+
* populates `results`, and calls `reporter` in `afterAll`.
|
|
21
|
+
*
|
|
22
|
+
* Kept free of Node-only deps (`fs`, `path`) so the same runner works under
|
|
23
|
+
* vitest-browser for IndexedDB and SQLite-OPFS.
|
|
24
|
+
*/
|
|
25
|
+
export function describeAztecMapBench(
|
|
26
|
+
backendPrefix: string,
|
|
27
|
+
getStore: () => Promise<AztecAsyncKVStore>,
|
|
28
|
+
logger: Logger,
|
|
29
|
+
reporter: BenchReporter,
|
|
30
|
+
) {
|
|
31
|
+
describe(`${backendPrefix} Map benchmarks`, () => {
|
|
32
|
+
let store: AztecAsyncKVStore;
|
|
33
|
+
let map: AztecAsyncMap<Key, string>;
|
|
34
|
+
|
|
35
|
+
const results: BenchResult[] = [];
|
|
36
|
+
|
|
37
|
+
const generateKeyValuePairs = (count: number, offset = 0) => {
|
|
38
|
+
const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
|
|
39
|
+
const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
|
|
40
|
+
return keys.map((key, i) => ({ key, value: values[i] }));
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const record = (name: string, value: number, unit: BenchResult['unit']) => {
|
|
44
|
+
results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
beforeEach(async () => {
|
|
48
|
+
store = await getStore();
|
|
49
|
+
map = store.openMap<Key, string>('test');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
afterEach(async () => {
|
|
53
|
+
await store.delete();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
afterAll(async () => {
|
|
57
|
+
const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
|
|
58
|
+
logger.info(`\n${pretty}\n`);
|
|
59
|
+
await reporter(results);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('adds individual values', async () => {
|
|
63
|
+
const pairs = generateKeyValuePairs(1000);
|
|
64
|
+
const timer = new Timer();
|
|
65
|
+
for (const pair of pairs) {
|
|
66
|
+
await map.set(pair.key, pair.value);
|
|
67
|
+
}
|
|
68
|
+
record('Individual insertion', timer.ms() / pairs.length, 'ms');
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('adds batched values', async () => {
|
|
72
|
+
const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
|
|
73
|
+
const timer = new Timer();
|
|
74
|
+
for (const batch of batches) {
|
|
75
|
+
await map.setMany(batch);
|
|
76
|
+
}
|
|
77
|
+
record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('reads individual values', async () => {
|
|
81
|
+
const pairs = generateKeyValuePairs(10000);
|
|
82
|
+
await map.setMany(pairs);
|
|
83
|
+
const timer = new Timer();
|
|
84
|
+
for (const pair of pairs) {
|
|
85
|
+
await map.getAsync(pair.key);
|
|
86
|
+
}
|
|
87
|
+
record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('reads via a cursor', async () => {
|
|
91
|
+
const pairs = generateKeyValuePairs(10000);
|
|
92
|
+
await map.setMany(pairs);
|
|
93
|
+
const timer = new Timer();
|
|
94
|
+
for await (const _ of map.entriesAsync()) {
|
|
95
|
+
// consume
|
|
96
|
+
}
|
|
97
|
+
record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('reads the size of the map', async () => {
|
|
101
|
+
const numIterations = 1000;
|
|
102
|
+
const pairs = generateKeyValuePairs(10000);
|
|
103
|
+
await map.setMany(pairs);
|
|
104
|
+
const timer = new Timer();
|
|
105
|
+
for (let i = 0; i < numIterations; i++) {
|
|
106
|
+
await map.sizeAsync();
|
|
107
|
+
}
|
|
108
|
+
record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
package/src/indexeddb/array.ts
CHANGED
|
@@ -32,7 +32,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
32
32
|
return (
|
|
33
33
|
(await this.db
|
|
34
34
|
.index('key')
|
|
35
|
-
.count(IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]))) ?? 0
|
|
35
|
+
.count(IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]))) ?? 0
|
|
36
36
|
);
|
|
37
37
|
}
|
|
38
38
|
|
|
@@ -43,7 +43,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
43
43
|
value: val,
|
|
44
44
|
hash: hash(val),
|
|
45
45
|
container: this.#container,
|
|
46
|
-
key: this.#name,
|
|
46
|
+
key: [this.#name],
|
|
47
47
|
keyCount: length + 1,
|
|
48
48
|
slot: this.#slot(length),
|
|
49
49
|
});
|
|
@@ -91,7 +91,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
91
91
|
value: val,
|
|
92
92
|
hash: hash(val),
|
|
93
93
|
container: this.#container,
|
|
94
|
-
key: this.#name,
|
|
94
|
+
key: [this.#name],
|
|
95
95
|
keyCount: index + 1,
|
|
96
96
|
slot: this.#slot(index),
|
|
97
97
|
});
|
|
@@ -100,7 +100,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
100
100
|
|
|
101
101
|
async *entriesAsync(): AsyncIterableIterator<[number, T]> {
|
|
102
102
|
const index = this.db.index('key');
|
|
103
|
-
const rangeQuery = IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]);
|
|
103
|
+
const rangeQuery = IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]);
|
|
104
104
|
for await (const cursor of index.iterate(rangeQuery)) {
|
|
105
105
|
yield [cursor.value.keyCount - 1, cursor.value.value] as [number, T];
|
|
106
106
|
}
|
package/src/indexeddb/index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
2
3
|
|
|
3
|
-
import type { DataStoreConfig } from '../config.js';
|
|
4
4
|
import { initStoreForRollupAndSchemaVersion } from '../utils.js';
|
|
5
5
|
import { AztecIndexedDBStore } from './store.js';
|
|
6
6
|
|
package/src/indexeddb/map.ts
CHANGED
|
@@ -31,7 +31,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
31
31
|
|
|
32
32
|
async getAsync(key: K): Promise<V | undefined> {
|
|
33
33
|
const data = await this.db.get(this.slot(key));
|
|
34
|
-
return data
|
|
34
|
+
return data ? this.restoreBuffers(data.value as V) : undefined;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
async hasAsync(key: K): Promise<boolean> {
|
|
@@ -41,7 +41,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
41
41
|
|
|
42
42
|
async sizeAsync(): Promise<number> {
|
|
43
43
|
const index = this.db.index('key');
|
|
44
|
-
const rangeQuery = IDBKeyRange.bound([this.container
|
|
44
|
+
const rangeQuery = IDBKeyRange.bound([this.container], [this.container + '\uffff'], true, true);
|
|
45
45
|
return await index.count(rangeQuery);
|
|
46
46
|
}
|
|
47
47
|
|
|
@@ -80,9 +80,12 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
80
80
|
|
|
81
81
|
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
|
|
82
82
|
const index = this.db.index('key');
|
|
83
|
+
const startKey = range.start ? this.normalizeKey(range.start) : [];
|
|
84
|
+
const endKey = range.end ? this.normalizeKey(range.end) : ['\uffff'];
|
|
85
|
+
|
|
83
86
|
const rangeQuery = IDBKeyRange.bound(
|
|
84
|
-
[this.container,
|
|
85
|
-
[this.container,
|
|
87
|
+
[this.container, startKey],
|
|
88
|
+
[this.container, endKey],
|
|
86
89
|
!!range.reverse,
|
|
87
90
|
!range.reverse,
|
|
88
91
|
);
|
|
@@ -91,7 +94,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
91
94
|
if (range.limit && count >= range.limit) {
|
|
92
95
|
return;
|
|
93
96
|
}
|
|
94
|
-
yield [this.#denormalizeKey(cursor.value.key), cursor.value.value] as [K, V];
|
|
97
|
+
yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
|
|
95
98
|
count++;
|
|
96
99
|
}
|
|
97
100
|
}
|
|
@@ -108,14 +111,24 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
108
111
|
}
|
|
109
112
|
}
|
|
110
113
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
+
/**
|
|
115
|
+
* IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
|
|
116
|
+
* This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
|
|
117
|
+
* comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
|
|
118
|
+
*/
|
|
119
|
+
protected restoreBuffers(val: V): V {
|
|
120
|
+
if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
|
|
121
|
+
return Buffer.from(val) as V;
|
|
122
|
+
}
|
|
123
|
+
return val;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
#denormalizeKey(key: (string | number | Uint8Array)[]): K {
|
|
127
|
+
return (key.length > 1 ? key : key[0]) as K;
|
|
114
128
|
}
|
|
115
129
|
|
|
116
|
-
protected normalizeKey(key: K): string {
|
|
117
|
-
|
|
118
|
-
return (arrayKey as K[]).map((element: K) => (typeof element === 'number' ? `n_${element}` : element)).join(',');
|
|
130
|
+
protected normalizeKey(key: K): (string | number | Uint8Array)[] {
|
|
131
|
+
return Array.isArray(key) ? key : [key];
|
|
119
132
|
}
|
|
120
133
|
|
|
121
134
|
protected slot(key: K, index: number = 0): string {
|
|
@@ -57,7 +57,7 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
|
|
|
57
57
|
false,
|
|
58
58
|
);
|
|
59
59
|
for await (const cursor of index.iterate(rangeQuery)) {
|
|
60
|
-
yield cursor.value.value as V;
|
|
60
|
+
yield this.restoreBuffers(cursor.value.value as V);
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
|
|
@@ -88,4 +88,18 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
|
|
|
88
88
|
await this.db.delete(fullKey);
|
|
89
89
|
}
|
|
90
90
|
}
|
|
91
|
+
|
|
92
|
+
override async delete(key: K): Promise<void> {
|
|
93
|
+
const index = this.db.index('keyCount');
|
|
94
|
+
const rangeQuery = IDBKeyRange.bound(
|
|
95
|
+
[this.container, this.normalizeKey(key), 0],
|
|
96
|
+
[this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
97
|
+
false,
|
|
98
|
+
false,
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
for await (const cursor of index.iterate(rangeQuery)) {
|
|
102
|
+
await cursor.delete();
|
|
103
|
+
}
|
|
104
|
+
}
|
|
91
105
|
}
|
|
@@ -37,7 +37,7 @@ export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingl
|
|
|
37
37
|
const result = await this.db.put({
|
|
38
38
|
container: this.#container,
|
|
39
39
|
slot: this.#slot,
|
|
40
|
-
key: this.#slot,
|
|
40
|
+
key: [this.#slot],
|
|
41
41
|
keyCount: 1,
|
|
42
42
|
value: val,
|
|
43
43
|
hash: hash(val),
|
package/src/indexeddb/store.ts
CHANGED
|
@@ -20,7 +20,7 @@ import { IndexedDBAztecSingleton } from './singleton.js';
|
|
|
20
20
|
export type StoredData<V extends Value> = {
|
|
21
21
|
value: V;
|
|
22
22
|
container: string;
|
|
23
|
-
key: string;
|
|
23
|
+
key: (string | number | Uint8Array)[];
|
|
24
24
|
keyCount: number;
|
|
25
25
|
slot: string;
|
|
26
26
|
hash: string;
|
|
@@ -30,7 +30,12 @@ export interface AztecIDBSchema extends DBSchema {
|
|
|
30
30
|
data: {
|
|
31
31
|
value: StoredData<any>;
|
|
32
32
|
key: string;
|
|
33
|
-
indexes: {
|
|
33
|
+
indexes: {
|
|
34
|
+
container: string;
|
|
35
|
+
key: any; // Runtime: (string | number | Uint8Array)[] - idb types don't support arrays
|
|
36
|
+
keyCount: number;
|
|
37
|
+
hash: string;
|
|
38
|
+
};
|
|
34
39
|
};
|
|
35
40
|
}
|
|
36
41
|
|
|
@@ -187,18 +192,20 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
187
192
|
}
|
|
188
193
|
|
|
189
194
|
/** Deletes this store and removes the database */
|
|
190
|
-
delete() {
|
|
195
|
+
async delete() {
|
|
191
196
|
this.#containers.clear();
|
|
197
|
+
await this.#txQueue.end();
|
|
192
198
|
this.#rootDB.close();
|
|
193
|
-
|
|
199
|
+
await deleteDB(this.#name);
|
|
194
200
|
}
|
|
195
201
|
|
|
196
202
|
estimateSize(): Promise<StoreSize> {
|
|
197
203
|
return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
|
|
198
204
|
}
|
|
199
205
|
|
|
200
|
-
close(): Promise<void> {
|
|
201
|
-
|
|
206
|
+
async close(): Promise<void> {
|
|
207
|
+
await this.#txQueue.end();
|
|
208
|
+
this.#rootDB.close();
|
|
202
209
|
}
|
|
203
210
|
|
|
204
211
|
backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
|
|
@@ -107,6 +107,33 @@ export function describeAztecMap(
|
|
|
107
107
|
expect(await size()).to.equal(1);
|
|
108
108
|
});
|
|
109
109
|
|
|
110
|
+
it('returns 0 for empty map size', async () => {
|
|
111
|
+
expect(await size()).to.equal(0);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('calculates size correctly across multiple operations', async () => {
|
|
115
|
+
expect(await size()).to.equal(0);
|
|
116
|
+
|
|
117
|
+
// Add items
|
|
118
|
+
await map.set('a', 'value1');
|
|
119
|
+
await map.set('b', 'value2');
|
|
120
|
+
await map.set('c', 'value3');
|
|
121
|
+
expect(await size()).to.equal(3);
|
|
122
|
+
|
|
123
|
+
// Update existing (size should not change)
|
|
124
|
+
await map.set('b', 'updated');
|
|
125
|
+
expect(await size()).to.equal(3);
|
|
126
|
+
|
|
127
|
+
// Delete some
|
|
128
|
+
await map.delete('a');
|
|
129
|
+
expect(await size()).to.equal(2);
|
|
130
|
+
|
|
131
|
+
// Delete all
|
|
132
|
+
await map.delete('b');
|
|
133
|
+
await map.delete('c');
|
|
134
|
+
expect(await size()).to.equal(0);
|
|
135
|
+
});
|
|
136
|
+
|
|
110
137
|
it('should be able to iterate over entries when there are no keys', async () => {
|
|
111
138
|
expect(await entries()).to.deep.equal([]);
|
|
112
139
|
});
|
|
@@ -145,10 +172,11 @@ export function describeAztecMap(
|
|
|
145
172
|
for (const [name, data] of [
|
|
146
173
|
['chars', ['a', 'b', 'c', 'd']],
|
|
147
174
|
['numbers', [1, 2, 3, 4]],
|
|
148
|
-
|
|
149
|
-
// ['negative numbers', [-4, -3, -2, -1]],
|
|
175
|
+
['negative numbers', [-4, -3, -2, -1]],
|
|
150
176
|
['strings', ['aaa', 'bbb', 'ccc', 'ddd']],
|
|
151
177
|
['zero-based numbers', [0, 1, 2, 3]],
|
|
178
|
+
['large numbers', [100, 999, 1000, 1001]],
|
|
179
|
+
['mixed negative and positive', [-1000, -1, 1, 1000]],
|
|
152
180
|
]) {
|
|
153
181
|
it(`supports range queries over ${name} keys`, async () => {
|
|
154
182
|
const [a, b, c, d] = data;
|