@aztec/kv-store 0.0.1-commit.b655e406 → 0.0.1-commit.b9865e97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -1
- package/dest/bench/shared_map_bench.d.ts +19 -0
- package/dest/bench/shared_map_bench.d.ts.map +1 -0
- package/dest/bench/shared_map_bench.js +91 -0
- package/dest/indexeddb/array.d.ts +1 -1
- package/dest/indexeddb/array.d.ts.map +1 -1
- package/dest/indexeddb/array.js +21 -7
- package/dest/indexeddb/index.d.ts +3 -3
- package/dest/indexeddb/index.d.ts.map +1 -1
- package/dest/indexeddb/index.js +3 -6
- package/dest/indexeddb/map.d.ts +8 -2
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +23 -13
- package/dest/indexeddb/multi_map.d.ts +3 -1
- package/dest/indexeddb/multi_map.d.ts.map +1 -1
- package/dest/indexeddb/multi_map.js +30 -1
- package/dest/indexeddb/set.d.ts +1 -1
- package/dest/indexeddb/set.d.ts.map +1 -1
- package/dest/indexeddb/singleton.d.ts +1 -1
- package/dest/indexeddb/singleton.d.ts.map +1 -1
- package/dest/indexeddb/singleton.js +3 -1
- package/dest/indexeddb/store.d.ts +3 -3
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +6 -4
- package/dest/interfaces/array.d.ts +1 -1
- package/dest/interfaces/array_test_suite.d.ts +1 -1
- package/dest/interfaces/array_test_suite.d.ts.map +1 -1
- package/dest/interfaces/array_test_suite.js +33 -34
- package/dest/interfaces/common.d.ts +1 -1
- package/dest/interfaces/counter.d.ts +1 -1
- package/dest/interfaces/index.d.ts +2 -2
- package/dest/interfaces/index.d.ts.map +1 -1
- package/dest/interfaces/map.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +74 -29
- package/dest/interfaces/multi_map.d.ts +7 -1
- package/dest/interfaces/multi_map.d.ts.map +1 -1
- package/dest/interfaces/multi_map_test_suite.d.ts +1 -1
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/multi_map_test_suite.js +154 -36
- package/dest/interfaces/set.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts.map +1 -1
- package/dest/interfaces/set_test_suite.js +13 -14
- package/dest/interfaces/singleton.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts.map +1 -1
- package/dest/interfaces/singleton_test_suite.js +6 -7
- package/dest/interfaces/store.d.ts +1 -1
- package/dest/interfaces/utils.d.ts +2 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/interfaces/utils.js +2 -1
- package/dest/lmdb/array.d.ts +1 -1
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/array.js +4 -2
- package/dest/lmdb/counter.d.ts +1 -1
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/index.d.ts +3 -3
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/index.js +4 -4
- package/dest/lmdb/map.d.ts +1 -1
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/multi_map.d.ts +2 -1
- package/dest/lmdb/multi_map.d.ts.map +1 -1
- package/dest/lmdb/multi_map.js +15 -0
- package/dest/lmdb/set.d.ts +1 -1
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/singleton.d.ts +1 -1
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +4 -5
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +13 -9
- package/dest/lmdb-v2/array.d.ts +2 -2
- package/dest/lmdb-v2/array.d.ts.map +1 -1
- package/dest/lmdb-v2/array.js +4 -3
- package/dest/lmdb-v2/factory.d.ts +32 -10
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +71 -16
- package/dest/lmdb-v2/index.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts +2 -2
- package/dest/lmdb-v2/map.d.ts.map +1 -1
- package/dest/lmdb-v2/map.js +1 -2
- package/dest/lmdb-v2/message.d.ts +1 -1
- package/dest/lmdb-v2/multi_map.d.ts +3 -2
- package/dest/lmdb-v2/multi_map.d.ts.map +1 -1
- package/dest/lmdb-v2/multi_map.js +11 -1
- package/dest/lmdb-v2/read_transaction.d.ts +1 -1
- package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +21 -19
- package/dest/lmdb-v2/set.d.ts +1 -1
- package/dest/lmdb-v2/set.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.d.ts +2 -2
- package/dest/lmdb-v2/singleton.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.js +1 -2
- package/dest/lmdb-v2/store.d.ts +4 -5
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/store.js +6 -28
- package/dest/lmdb-v2/tx-helpers.d.ts +6 -0
- package/dest/lmdb-v2/tx-helpers.d.ts.map +1 -0
- package/dest/lmdb-v2/tx-helpers.js +21 -0
- package/dest/lmdb-v2/utils.d.ts +1 -10
- package/dest/lmdb-v2/utils.d.ts.map +1 -1
- package/dest/lmdb-v2/utils.js +0 -94
- package/dest/lmdb-v2/write_transaction.d.ts +1 -1
- package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/write_transaction.js +9 -5
- package/dest/sqlite-opfs/array.d.ts +21 -0
- package/dest/sqlite-opfs/array.d.ts.map +1 -0
- package/dest/sqlite-opfs/array.js +128 -0
- package/dest/sqlite-opfs/errors.d.ts +27 -0
- package/dest/sqlite-opfs/errors.d.ts.map +1 -0
- package/dest/sqlite-opfs/errors.js +34 -0
- package/dest/sqlite-opfs/index.d.ts +16 -0
- package/dest/sqlite-opfs/index.d.ts.map +1 -0
- package/dest/sqlite-opfs/index.js +22 -0
- package/dest/sqlite-opfs/internal/ordered-binary-browser.d.ts +32 -0
- package/dest/sqlite-opfs/internal/ordered-binary-browser.d.ts.map +1 -0
- package/dest/sqlite-opfs/internal/ordered-binary-browser.js +448 -0
- package/dest/sqlite-opfs/map.d.ts +35 -0
- package/dest/sqlite-opfs/map.d.ts.map +1 -0
- package/dest/sqlite-opfs/map.js +163 -0
- package/dest/sqlite-opfs/messages.d.ts +66 -0
- package/dest/sqlite-opfs/messages.d.ts.map +1 -0
- package/dest/sqlite-opfs/messages.js +5 -0
- package/dest/sqlite-opfs/multi_map.d.ts +16 -0
- package/dest/sqlite-opfs/multi_map.d.ts.map +1 -0
- package/dest/sqlite-opfs/multi_map.js +67 -0
- package/dest/sqlite-opfs/set.d.ts +13 -0
- package/dest/sqlite-opfs/set.d.ts.map +1 -0
- package/dest/sqlite-opfs/set.js +19 -0
- package/dest/sqlite-opfs/singleton.d.ts +13 -0
- package/dest/sqlite-opfs/singleton.d.ts.map +1 -0
- package/dest/sqlite-opfs/singleton.js +48 -0
- package/dest/sqlite-opfs/store.d.ts +79 -0
- package/dest/sqlite-opfs/store.d.ts.map +1 -0
- package/dest/sqlite-opfs/store.js +281 -0
- package/dest/sqlite-opfs/worker.d.ts +2 -0
- package/dest/sqlite-opfs/worker.d.ts.map +1 -0
- package/dest/sqlite-opfs/worker.js +249 -0
- package/dest/stores/index.d.ts +1 -1
- package/dest/stores/l2_tips_store.d.ts +25 -10
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +62 -53
- package/dest/utils.d.ts +9 -6
- package/dest/utils.d.ts.map +1 -1
- package/dest/utils.js +51 -16
- package/package.json +33 -26
- package/src/bench/shared_map_bench.ts +111 -0
- package/src/indexeddb/array.ts +4 -4
- package/src/indexeddb/index.ts +9 -7
- package/src/indexeddb/map.ts +24 -11
- package/src/indexeddb/multi_map.ts +27 -1
- package/src/indexeddb/singleton.ts +1 -1
- package/src/indexeddb/store.ts +13 -6
- package/src/interfaces/array_test_suite.ts +33 -35
- package/src/interfaces/index.ts +1 -1
- package/src/interfaces/map_test_suite.ts +56 -30
- package/src/interfaces/multi_map.ts +7 -0
- package/src/interfaces/multi_map_test_suite.ts +167 -37
- package/src/interfaces/set_test_suite.ts +13 -15
- package/src/interfaces/singleton_test_suite.ts +6 -8
- package/src/interfaces/utils.ts +1 -0
- package/src/lmdb/index.ts +10 -5
- package/src/lmdb/multi_map.ts +16 -0
- package/src/lmdb/store.ts +13 -9
- package/src/lmdb-v2/array.ts +2 -2
- package/src/lmdb-v2/factory.ts +93 -19
- package/src/lmdb-v2/map.ts +2 -2
- package/src/lmdb-v2/multi_map.ts +10 -1
- package/src/lmdb-v2/read_transaction.ts +23 -23
- package/src/lmdb-v2/singleton.ts +2 -2
- package/src/lmdb-v2/store.ts +9 -33
- package/src/lmdb-v2/tx-helpers.ts +29 -0
- package/src/lmdb-v2/utils.ts +0 -118
- package/src/lmdb-v2/write_transaction.ts +9 -8
- package/src/sqlite-opfs/array.ts +124 -0
- package/src/sqlite-opfs/errors.ts +44 -0
- package/src/sqlite-opfs/index.ts +39 -0
- package/src/sqlite-opfs/internal/ordered-binary-browser.js +465 -0
- package/src/sqlite-opfs/map.ts +163 -0
- package/src/sqlite-opfs/messages.ts +39 -0
- package/src/sqlite-opfs/multi_map.ts +74 -0
- package/src/sqlite-opfs/set.ts +29 -0
- package/src/sqlite-opfs/singleton.ts +48 -0
- package/src/sqlite-opfs/store.ts +296 -0
- package/src/sqlite-opfs/worker.ts +231 -0
- package/src/stores/l2_tips_store.ts +66 -56
- package/src/utils.ts +79 -21
- package/dest/config.d.ts +0 -17
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -26
- package/src/config.ts +0 -36
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/// <reference lib="webworker" />
|
|
2
|
+
import sqlite3InitModule, { type Database, type SAHPoolUtil, type Sqlite3Static } from '@aztec/sqlite3mc-wasm';
|
|
3
|
+
|
|
4
|
+
import { SqliteEncryptionError, type SqliteEncryptionErrorCode, isDecryptFailureMessage } from './errors.js';
|
|
5
|
+
import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
|
|
6
|
+
|
|
7
|
+
const SCHEMA_SQL = `
|
|
8
|
+
CREATE TABLE IF NOT EXISTS data (
|
|
9
|
+
slot TEXT NOT NULL PRIMARY KEY,
|
|
10
|
+
container TEXT NOT NULL,
|
|
11
|
+
key BLOB NOT NULL,
|
|
12
|
+
key_count INTEGER NOT NULL,
|
|
13
|
+
hash TEXT NOT NULL,
|
|
14
|
+
value BLOB
|
|
15
|
+
) WITHOUT ROWID;
|
|
16
|
+
|
|
17
|
+
CREATE INDEX IF NOT EXISTS idx_container_key ON data(container, key);
|
|
18
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_count ON data(container, key, key_count);
|
|
19
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
|
|
20
|
+
`;
|
|
21
|
+
|
|
22
|
+
const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
|
|
23
|
+
const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
|
|
24
|
+
const MC_SAH_POOL_VFS_NAME = `multipleciphers-${SAH_POOL_VFS_NAME}`;
|
|
25
|
+
|
|
26
|
+
let sqlite3: Sqlite3Static | undefined;
|
|
27
|
+
let pool: SAHPoolUtil | undefined;
|
|
28
|
+
let db: Database | undefined;
|
|
29
|
+
let dbPath: string | undefined;
|
|
30
|
+
|
|
31
|
+
async function ensurePool(directory: string): Promise<SAHPoolUtil> {
|
|
32
|
+
sqlite3 ??= await sqlite3InitModule();
|
|
33
|
+
const s = sqlite3;
|
|
34
|
+
if (!pool) {
|
|
35
|
+
pool = await s.installOpfsSAHPoolVfs({
|
|
36
|
+
name: SAH_POOL_VFS_NAME,
|
|
37
|
+
directory,
|
|
38
|
+
initialCapacity: 8,
|
|
39
|
+
});
|
|
40
|
+
// Register a sqlite3mc-wrapped VFS pointing at our SAH Pool VFS.
|
|
41
|
+
// Encrypted DBs must be opened through this wrapper so sqlite3mc can
|
|
42
|
+
// intercept file I/O; plain DBs continue using the SAH Pool VFS directly.
|
|
43
|
+
// The wrapper name is `multipleciphers-<underlying>`.
|
|
44
|
+
(s.capi as unknown as { sqlite3mc_vfs_create(name: string, makeDefault: number): number }).sqlite3mc_vfs_create(
|
|
45
|
+
SAH_POOL_VFS_NAME,
|
|
46
|
+
0,
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
return pool!;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Applies sqlite3mc's ChaCha20 page cipher using a pre-derived 32-byte key.
|
|
54
|
+
* The PRAGMAs must run before any schema DDL so sqlite3mc can decrypt existing
|
|
55
|
+
* pages and encrypt new ones. Zeroes the caller-held key array after the PRAGMA
|
|
56
|
+
* completes to minimize residency of the raw key bytes outside sqlite3mc's heap.
|
|
57
|
+
*/
|
|
58
|
+
function applyEncryptionKey(conn: Database, key: Uint8Array): void {
|
|
59
|
+
const hex = Array.from(key, b => b.toString(16).padStart(2, '0')).join('');
|
|
60
|
+
conn.exec(`PRAGMA cipher = 'chacha20'`);
|
|
61
|
+
conn.exec(`PRAGMA key = "x'${hex}'"`);
|
|
62
|
+
key.fill(0);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function handleInit(
|
|
66
|
+
dbName: string,
|
|
67
|
+
ephemeral: boolean,
|
|
68
|
+
directory?: string,
|
|
69
|
+
encryptionKey?: Uint8Array,
|
|
70
|
+
): Promise<void> {
|
|
71
|
+
sqlite3 ??= await sqlite3InitModule();
|
|
72
|
+
const s = sqlite3;
|
|
73
|
+
if (encryptionKey !== undefined && ephemeral) {
|
|
74
|
+
throw new SqliteEncryptionError(
|
|
75
|
+
'encryption_not_supported_for_ephemeral',
|
|
76
|
+
'encryptionKey is not supported for ephemeral (:memory:) stores',
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
if (ephemeral) {
|
|
80
|
+
db = new s.oo1.DB(':memory:', 'c');
|
|
81
|
+
} else {
|
|
82
|
+
await ensurePool(directory ?? DEFAULT_SAH_POOL_DIRECTORY);
|
|
83
|
+
dbPath = normalizeDbPath(dbName);
|
|
84
|
+
if (encryptionKey !== undefined) {
|
|
85
|
+
db = new s.oo1.DB({ filename: dbPath, flags: 'c', vfs: MC_SAH_POOL_VFS_NAME });
|
|
86
|
+
applyEncryptionKey(db, encryptionKey);
|
|
87
|
+
} else {
|
|
88
|
+
db = new pool!.OpfsSAHPoolDb(dbPath);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
runSql(SCHEMA_SQL);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function handleClose(): void {
|
|
95
|
+
db?.close();
|
|
96
|
+
db = undefined;
|
|
97
|
+
dbPath = undefined;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function handleExport(): Promise<Uint8Array> {
|
|
101
|
+
if (!db || !dbPath) {
|
|
102
|
+
throw new Error('SQLite worker: no database open to export');
|
|
103
|
+
}
|
|
104
|
+
if (!pool) {
|
|
105
|
+
throw new Error('SQLite worker: no SAH Pool available (ephemeral DBs cannot be exported)');
|
|
106
|
+
}
|
|
107
|
+
return await pool.exportFile(dbPath);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function handleDeleteDb(dbName: string): void {
|
|
111
|
+
const path = normalizeDbPath(dbName);
|
|
112
|
+
if (db && dbPath === path) {
|
|
113
|
+
db.close();
|
|
114
|
+
db = undefined;
|
|
115
|
+
dbPath = undefined;
|
|
116
|
+
}
|
|
117
|
+
// Ephemeral :memory: DBs never back a file — skip installing a pool just to unlink
|
|
118
|
+
// nothing. installOpfsSAHPoolVfs acquires an exclusive lock on the OPFS SAH
|
|
119
|
+
// directory, and under heavy test churn that can contend with workers from
|
|
120
|
+
// previous tests whose OPFS handles Chromium hasn't yet released, hanging the RPC
|
|
121
|
+
// and then the whole test run.
|
|
122
|
+
if (!pool) {
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
pool.unlink(path);
|
|
127
|
+
} catch {
|
|
128
|
+
// File may not exist; ignore.
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function requireDb(): Database {
|
|
133
|
+
if (!db) {
|
|
134
|
+
throw new Error('SQLite worker: no database open');
|
|
135
|
+
}
|
|
136
|
+
return db;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function runSql(sql: string, bind?: SqlValue[]): { changes: number } {
|
|
140
|
+
const conn = requireDb();
|
|
141
|
+
conn.exec({ sql, bind });
|
|
142
|
+
return { changes: conn.changes() };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function selectAll(sql: string, bind?: SqlValue[]): ResultRow[] {
|
|
146
|
+
const conn = requireDb();
|
|
147
|
+
const rows: ResultRow[] = [];
|
|
148
|
+
conn.exec({ sql, bind, rowMode: 'array', resultRows: rows });
|
|
149
|
+
return rows;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function normalizeDbPath(dbName: string): string {
|
|
153
|
+
return dbName.startsWith('/') ? dbName : `/${dbName}`;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function respond(msg: WorkerResponse): void {
|
|
157
|
+
(self as DedicatedWorkerGlobalScope).postMessage(msg);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
(self as DedicatedWorkerGlobalScope).onmessage = async (ev: MessageEvent<WorkerRequest>) => {
|
|
161
|
+
const req = ev.data;
|
|
162
|
+
try {
|
|
163
|
+
switch (req.type) {
|
|
164
|
+
case 'init':
|
|
165
|
+
await handleInit(req.dbName, req.ephemeral, req.poolDirectory, req.encryptionKey);
|
|
166
|
+
return respond({ type: 'ok', id: req.id });
|
|
167
|
+
case 'close':
|
|
168
|
+
handleClose();
|
|
169
|
+
return respond({ type: 'ok', id: req.id });
|
|
170
|
+
case 'deleteDb':
|
|
171
|
+
handleDeleteDb(req.dbName);
|
|
172
|
+
return respond({ type: 'ok', id: req.id });
|
|
173
|
+
case 'run': {
|
|
174
|
+
const { changes } = runSql(req.sql, req.bind);
|
|
175
|
+
return respond({ type: 'ok', id: req.id, changes });
|
|
176
|
+
}
|
|
177
|
+
case 'all': {
|
|
178
|
+
const rows = selectAll(req.sql, req.bind);
|
|
179
|
+
return respond({ type: 'ok', id: req.id, rows });
|
|
180
|
+
}
|
|
181
|
+
case 'export': {
|
|
182
|
+
const bytes = await handleExport();
|
|
183
|
+
return respond({ type: 'ok', id: req.id, bytes });
|
|
184
|
+
}
|
|
185
|
+
case 'begin':
|
|
186
|
+
runSql('BEGIN');
|
|
187
|
+
return respond({ type: 'ok', id: req.id });
|
|
188
|
+
case 'commit':
|
|
189
|
+
runSql('COMMIT');
|
|
190
|
+
return respond({ type: 'ok', id: req.id });
|
|
191
|
+
case 'rollback':
|
|
192
|
+
runSql('ROLLBACK');
|
|
193
|
+
return respond({ type: 'ok', id: req.id });
|
|
194
|
+
default: {
|
|
195
|
+
const _exhaustive: never = req;
|
|
196
|
+
throw new Error(`Unknown request: ${JSON.stringify(_exhaustive)}`);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
} catch (err) {
|
|
200
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
201
|
+
respond({ type: 'err', id: req.id, message, encryptionCode: detectEncryptionCode(req, err, message) });
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Maps a thrown error during request handling to a typed encryption code, so the
|
|
207
|
+
* main thread can re-hydrate it as a {@link SqliteEncryptionError}. Returns
|
|
208
|
+
* `undefined` for non-encryption errors (preserves the existing untyped path).
|
|
209
|
+
*
|
|
210
|
+
* Two sources:
|
|
211
|
+
* - The error was already a `SqliteEncryptionError` (pre-flight throws inside
|
|
212
|
+
* this worker — e.g. ephemeral + encryptionKey). Forward its code as-is.
|
|
213
|
+
* - The error came from SQLite/sqlite3mc with a known decrypt-failure message
|
|
214
|
+
* during an `init` request. Both "wrong key supplied" and "no key supplied
|
|
215
|
+
* to an encrypted DB" surface as SQLITE_NOTADB ("file is not a database") —
|
|
216
|
+
* we don't constrain on `req.encryptionKey` because the no-key-on-encrypted-DB
|
|
217
|
+
* case is exactly when the caller most needs the typed signal.
|
|
218
|
+
*/
|
|
219
|
+
function detectEncryptionCode(
|
|
220
|
+
req: WorkerRequest,
|
|
221
|
+
err: unknown,
|
|
222
|
+
message: string,
|
|
223
|
+
): SqliteEncryptionErrorCode | undefined {
|
|
224
|
+
if (err instanceof SqliteEncryptionError) {
|
|
225
|
+
return err.code;
|
|
226
|
+
}
|
|
227
|
+
if (req.type === 'init' && isDecryptFailureMessage(message)) {
|
|
228
|
+
return 'decrypt_failed';
|
|
229
|
+
}
|
|
230
|
+
return undefined;
|
|
231
|
+
}
|
|
@@ -1,79 +1,89 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
L2BlockStreamEventHandler,
|
|
5
|
-
L2BlockStreamLocalDataProvider,
|
|
6
|
-
L2BlockTag,
|
|
7
|
-
L2Tips,
|
|
8
|
-
} from '@aztec/stdlib/block';
|
|
1
|
+
import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
2
|
+
import { type BlockHash, type L2BlockTag, L2TipsStoreBase } from '@aztec/stdlib/block';
|
|
3
|
+
import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
9
4
|
|
|
10
5
|
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
11
6
|
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
12
7
|
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Persistent implementation of L2 tips store backed by a KV store.
|
|
10
|
+
* Used by nodes that need to persist chain state across restarts.
|
|
11
|
+
*/
|
|
12
|
+
export class L2TipsKVStore extends L2TipsStoreBase {
|
|
13
|
+
private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, BlockNumber>;
|
|
14
|
+
private readonly l2BlockHashesStore: AztecAsyncMap<BlockNumber, string>;
|
|
15
|
+
private readonly l2BlockNumberToCheckpointNumberStore: AztecAsyncMap<BlockNumber, CheckpointNumber>;
|
|
16
|
+
private readonly l2CheckpointStore: AztecAsyncMap<CheckpointNumber, Buffer>;
|
|
17
17
|
|
|
18
|
-
constructor(
|
|
18
|
+
constructor(
|
|
19
|
+
private store: AztecAsyncKVStore,
|
|
20
|
+
namespace: string,
|
|
21
|
+
initialBlockHash: BlockHash,
|
|
22
|
+
) {
|
|
23
|
+
super(initialBlockHash);
|
|
19
24
|
this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_'));
|
|
20
25
|
this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_'));
|
|
26
|
+
this.l2BlockNumberToCheckpointNumberStore = store.openMap(
|
|
27
|
+
[namespace, 'l2_block_number_to_checkpoint_number'].join('_'),
|
|
28
|
+
);
|
|
29
|
+
this.l2CheckpointStore = store.openMap([namespace, 'l2_checkpoint_store'].join('_'));
|
|
21
30
|
}
|
|
22
31
|
|
|
23
|
-
|
|
24
|
-
return this.
|
|
32
|
+
protected getTip(tag: L2BlockTag): Promise<BlockNumber | undefined> {
|
|
33
|
+
return this.l2TipsStore.getAsync(tag);
|
|
25
34
|
}
|
|
26
35
|
|
|
27
|
-
|
|
28
|
-
return
|
|
29
|
-
latest: await this.getL2Tip('latest'),
|
|
30
|
-
finalized: await this.getL2Tip('finalized'),
|
|
31
|
-
proven: await this.getL2Tip('proven'),
|
|
32
|
-
};
|
|
36
|
+
protected setTip(tag: L2BlockTag, blockNumber: BlockNumber): Promise<void> {
|
|
37
|
+
return this.l2TipsStore.set(tag, blockNumber);
|
|
33
38
|
}
|
|
34
39
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
protected getStoredBlockHash(blockNumber: BlockNumber): Promise<string | undefined> {
|
|
41
|
+
return this.l2BlockHashesStore.getAsync(blockNumber);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
protected setBlockHash(blockNumber: BlockNumber, hash: string): Promise<void> {
|
|
45
|
+
return this.l2BlockHashesStore.set(blockNumber, hash);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
protected async deleteBlockHashesBefore(blockNumber: BlockNumber): Promise<void> {
|
|
49
|
+
for await (const key of this.l2BlockHashesStore.keysAsync({ end: blockNumber })) {
|
|
50
|
+
await this.l2BlockHashesStore.delete(key);
|
|
43
51
|
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
protected getCheckpointNumberForBlock(blockNumber: BlockNumber): Promise<CheckpointNumber | undefined> {
|
|
55
|
+
return this.l2BlockNumberToCheckpointNumberStore.getAsync(blockNumber);
|
|
56
|
+
}
|
|
44
57
|
|
|
45
|
-
|
|
58
|
+
protected setCheckpointNumberForBlock(blockNumber: BlockNumber, checkpointNumber: CheckpointNumber): Promise<void> {
|
|
59
|
+
return this.l2BlockNumberToCheckpointNumberStore.set(blockNumber, checkpointNumber);
|
|
46
60
|
}
|
|
47
61
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const blocks = event.blocks.map(b => b.block);
|
|
52
|
-
for (const block of blocks) {
|
|
53
|
-
await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString());
|
|
54
|
-
}
|
|
55
|
-
await this.l2TipsStore.set('latest', blocks.at(-1)!.number);
|
|
56
|
-
break;
|
|
57
|
-
}
|
|
58
|
-
case 'chain-pruned':
|
|
59
|
-
await this.saveTag('latest', event.block);
|
|
60
|
-
break;
|
|
61
|
-
case 'chain-proven':
|
|
62
|
-
await this.saveTag('proven', event.block);
|
|
63
|
-
break;
|
|
64
|
-
case 'chain-finalized':
|
|
65
|
-
await this.saveTag('finalized', event.block);
|
|
66
|
-
for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.block.number })) {
|
|
67
|
-
await this.l2BlockHashesStore.delete(key);
|
|
68
|
-
}
|
|
69
|
-
break;
|
|
62
|
+
protected async deleteBlockToCheckpointBefore(blockNumber: BlockNumber): Promise<void> {
|
|
63
|
+
for await (const key of this.l2BlockNumberToCheckpointNumberStore.keysAsync({ end: blockNumber })) {
|
|
64
|
+
await this.l2BlockNumberToCheckpointNumberStore.delete(key);
|
|
70
65
|
}
|
|
71
66
|
}
|
|
72
67
|
|
|
73
|
-
|
|
74
|
-
await this.
|
|
75
|
-
if (
|
|
76
|
-
|
|
68
|
+
protected async getCheckpoint(checkpointNumber: CheckpointNumber): Promise<PublishedCheckpoint | undefined> {
|
|
69
|
+
const buffer = await this.l2CheckpointStore.getAsync(checkpointNumber);
|
|
70
|
+
if (!buffer) {
|
|
71
|
+
return undefined;
|
|
77
72
|
}
|
|
73
|
+
return PublishedCheckpoint.fromBuffer(buffer);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
protected saveCheckpointData(checkpoint: PublishedCheckpoint): Promise<void> {
|
|
77
|
+
return this.l2CheckpointStore.set(checkpoint.checkpoint.number, checkpoint.toBuffer());
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
protected async deleteCheckpointsBefore(checkpointNumber: CheckpointNumber): Promise<void> {
|
|
81
|
+
for await (const key of this.l2CheckpointStore.keysAsync({ end: checkpointNumber })) {
|
|
82
|
+
await this.l2CheckpointStore.delete(key);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
protected runInTransaction<T>(fn: () => Promise<T>): Promise<T> {
|
|
87
|
+
return this.store.transactionAsync(fn);
|
|
78
88
|
}
|
|
79
89
|
}
|
package/src/utils.ts
CHANGED
|
@@ -1,40 +1,98 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
2
2
|
import type { Logger } from '@aztec/foundation/log';
|
|
3
|
+
import { DatabaseVersion } from '@aztec/stdlib/database-version/version';
|
|
3
4
|
|
|
4
5
|
import type { AztecAsyncSingleton, AztecSingleton } from './interfaces/singleton.js';
|
|
5
6
|
import type { AztecAsyncKVStore, AztecKVStore } from './interfaces/store.js';
|
|
6
7
|
import { isSyncStore } from './interfaces/utils.js';
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
|
-
* Clears the store if the rollup address does not match the one stored in the database.
|
|
10
|
-
*
|
|
10
|
+
* Clears the store if the schema version or rollup address does not match the one stored in the database.
|
|
11
|
+
* Also clears if migrating from an older store format that didn't track schema version.
|
|
12
|
+
* This is to prevent data from being accidentally mixed up between different rollup instances or schema versions.
|
|
11
13
|
* @param store - The store to check
|
|
14
|
+
* @param targetSchemaVersion - The current schema version
|
|
12
15
|
* @param rollupAddress - The ETH address of the rollup contract
|
|
13
|
-
* @
|
|
16
|
+
* @param log - Optional logger
|
|
17
|
+
* @returns The store (cleared if necessary)
|
|
14
18
|
*/
|
|
15
|
-
export async function
|
|
19
|
+
export async function initStoreForRollupAndSchemaVersion<T extends AztecKVStore | AztecAsyncKVStore>(
|
|
16
20
|
store: T,
|
|
17
|
-
|
|
21
|
+
schemaVersion: number | undefined,
|
|
22
|
+
rollupAddress: EthAddress | undefined,
|
|
18
23
|
log?: Logger,
|
|
19
24
|
): Promise<T> {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
25
|
+
const targetSchemaVersion = schemaVersion ?? 0;
|
|
26
|
+
const targetRollupAddress = rollupAddress ?? EthAddress.ZERO;
|
|
27
|
+
const targetDatabaseVersion = new DatabaseVersion(targetSchemaVersion, targetRollupAddress);
|
|
28
|
+
|
|
29
|
+
// DB version: database schema version + rollup address combined)
|
|
30
|
+
const dbVersion = store.openSingleton<string>('dbVersion');
|
|
31
|
+
|
|
32
|
+
const storedDatabaseVersion = isSyncStore(store)
|
|
33
|
+
? (dbVersion as AztecSingleton<string>).get()
|
|
34
|
+
: await (dbVersion as AztecAsyncSingleton<string>).getAsync();
|
|
35
|
+
|
|
36
|
+
// Check if this is an old format store (has rollupAddress singleton but no dbVersion)
|
|
37
|
+
const oldRollupSingleton = store.openSingleton<string>('rollupAddress');
|
|
38
|
+
const hasOldFormat = isSyncStore(store)
|
|
39
|
+
? !storedDatabaseVersion && !!(oldRollupSingleton as AztecSingleton<string>).get()
|
|
40
|
+
: !storedDatabaseVersion && !!(await (oldRollupSingleton as AztecAsyncSingleton<string>).getAsync());
|
|
34
41
|
|
|
42
|
+
if (
|
|
43
|
+
hasOldFormat ||
|
|
44
|
+
doesStoreNeedToBeCleared(
|
|
45
|
+
targetDatabaseVersion,
|
|
46
|
+
storedDatabaseVersion,
|
|
47
|
+
targetSchemaVersion,
|
|
48
|
+
targetRollupAddress,
|
|
49
|
+
log,
|
|
50
|
+
)
|
|
51
|
+
) {
|
|
52
|
+
if (hasOldFormat) {
|
|
53
|
+
log?.warn('Detected old store format without dbVersion, clearing database');
|
|
54
|
+
}
|
|
35
55
|
await store.clear();
|
|
36
56
|
}
|
|
37
57
|
|
|
38
|
-
await
|
|
58
|
+
await dbVersion.set(targetDatabaseVersion.toBuffer().toString('utf-8'));
|
|
59
|
+
|
|
39
60
|
return store;
|
|
40
61
|
}
|
|
62
|
+
|
|
63
|
+
function doesStoreNeedToBeCleared(
|
|
64
|
+
targetDatabaseVersion: DatabaseVersion,
|
|
65
|
+
storedDatabaseVersion: string | undefined,
|
|
66
|
+
targetSchemaVersion: number,
|
|
67
|
+
targetRollupAddress: EthAddress,
|
|
68
|
+
log?: Logger,
|
|
69
|
+
) {
|
|
70
|
+
if (storedDatabaseVersion) {
|
|
71
|
+
try {
|
|
72
|
+
const storedVersion = DatabaseVersion.fromBuffer(Buffer.from(storedDatabaseVersion, 'utf-8'));
|
|
73
|
+
const cmp = storedVersion.cmp(targetDatabaseVersion);
|
|
74
|
+
|
|
75
|
+
if (cmp === undefined) {
|
|
76
|
+
log?.warn('Rollup address changed, clearing database', {
|
|
77
|
+
stored: storedVersion.rollupAddress.toString(),
|
|
78
|
+
current: targetRollupAddress.toString(),
|
|
79
|
+
});
|
|
80
|
+
return true;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (cmp !== 0) {
|
|
84
|
+
log?.warn('Schema version changed, clearing database', {
|
|
85
|
+
stored: storedVersion.schemaVersion,
|
|
86
|
+
current: targetSchemaVersion,
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
} catch (err) {
|
|
92
|
+
log?.warn('Failed to parse stored version, clearing database', { err });
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return false;
|
|
98
|
+
}
|
package/dest/config.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { type ConfigMappingsType } from '@aztec/foundation/config';
|
|
2
|
-
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
3
|
-
export type DataStoreConfig = {
|
|
4
|
-
dataDirectory: string | undefined;
|
|
5
|
-
dataStoreMapSizeKb: number;
|
|
6
|
-
l1Contracts?: {
|
|
7
|
-
rollupAddress: EthAddress;
|
|
8
|
-
};
|
|
9
|
-
};
|
|
10
|
-
export declare const dataConfigMappings: ConfigMappingsType<DataStoreConfig>;
|
|
11
|
-
/**
|
|
12
|
-
* Returns the archiver configuration from the environment variables.
|
|
13
|
-
* Note: If an environment variable is not set, the default value is used.
|
|
14
|
-
* @returns The archiver configuration.
|
|
15
|
-
*/
|
|
16
|
-
export declare function getDataConfigFromEnv(): DataStoreConfig;
|
|
17
|
-
//# sourceMappingURL=config.d.ts.map
|
package/dest/config.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,kBAAkB,EAA6C,MAAM,0BAA0B,CAAC;AAC9G,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAEhE,MAAM,MAAM,eAAe,GAAG;IAC5B,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE;QAAE,aAAa,EAAE,UAAU,CAAA;KAAE,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,kBAAkB,CAAC,eAAe,CAgBlE,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,oBAAoB,IAAI,eAAe,CAEtD"}
|
package/dest/config.js
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
|
|
2
|
-
import { getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
|
|
3
|
-
export const dataConfigMappings = {
|
|
4
|
-
dataDirectory: {
|
|
5
|
-
env: 'DATA_DIRECTORY',
|
|
6
|
-
description: 'Optional dir to store data. If omitted will store in memory.'
|
|
7
|
-
},
|
|
8
|
-
dataStoreMapSizeKb: {
|
|
9
|
-
env: 'DATA_STORE_MAP_SIZE_KB',
|
|
10
|
-
description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
|
|
11
|
-
...numberConfigHelper(128 * 1_024 * 1_024)
|
|
12
|
-
},
|
|
13
|
-
l1Contracts: {
|
|
14
|
-
description: 'The deployed L1 contract addresses',
|
|
15
|
-
nested: {
|
|
16
|
-
rollupAddress: l1ContractAddressesMapping.rollupAddress
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
};
|
|
20
|
-
/**
|
|
21
|
-
* Returns the archiver configuration from the environment variables.
|
|
22
|
-
* Note: If an environment variable is not set, the default value is used.
|
|
23
|
-
* @returns The archiver configuration.
|
|
24
|
-
*/ export function getDataConfigFromEnv() {
|
|
25
|
-
return getConfigFromMappings(dataConfigMappings);
|
|
26
|
-
}
|
package/src/config.ts
DELETED
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
|
|
2
|
-
import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
|
|
3
|
-
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
|
-
|
|
5
|
-
export type DataStoreConfig = {
|
|
6
|
-
dataDirectory: string | undefined;
|
|
7
|
-
dataStoreMapSizeKb: number;
|
|
8
|
-
l1Contracts?: { rollupAddress: EthAddress };
|
|
9
|
-
};
|
|
10
|
-
|
|
11
|
-
export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
|
|
12
|
-
dataDirectory: {
|
|
13
|
-
env: 'DATA_DIRECTORY',
|
|
14
|
-
description: 'Optional dir to store data. If omitted will store in memory.',
|
|
15
|
-
},
|
|
16
|
-
dataStoreMapSizeKb: {
|
|
17
|
-
env: 'DATA_STORE_MAP_SIZE_KB',
|
|
18
|
-
description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
|
|
19
|
-
...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
|
|
20
|
-
},
|
|
21
|
-
l1Contracts: {
|
|
22
|
-
description: 'The deployed L1 contract addresses',
|
|
23
|
-
nested: {
|
|
24
|
-
rollupAddress: l1ContractAddressesMapping.rollupAddress,
|
|
25
|
-
},
|
|
26
|
-
},
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Returns the archiver configuration from the environment variables.
|
|
31
|
-
* Note: If an environment variable is not set, the default value is used.
|
|
32
|
-
* @returns The archiver configuration.
|
|
33
|
-
*/
|
|
34
|
-
export function getDataConfigFromEnv(): DataStoreConfig {
|
|
35
|
-
return getConfigFromMappings<DataStoreConfig>(dataConfigMappings);
|
|
36
|
-
}
|