@optimystic/db-p2p-storage-rn 1.15.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -0
- package/dist/src/identity.d.ts +3 -1
- package/dist/src/identity.d.ts.map +1 -1
- package/dist/src/identity.js +4 -2
- package/dist/src/identity.js.map +1 -1
- package/dist/src/index.d.ts +2 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/keys.d.ts +22 -0
- package/dist/src/keys.d.ts.map +1 -1
- package/dist/src/keys.js +47 -0
- package/dist/src/keys.js.map +1 -1
- package/dist/src/leveldb-kv-store.d.ts +24 -2
- package/dist/src/leveldb-kv-store.d.ts.map +1 -1
- package/dist/src/leveldb-kv-store.js +61 -3
- package/dist/src/leveldb-kv-store.js.map +1 -1
- package/dist/src/leveldb-like.d.ts +23 -4
- package/dist/src/leveldb-like.d.ts.map +1 -1
- package/dist/src/leveldb-like.js +6 -0
- package/dist/src/leveldb-like.js.map +1 -1
- package/dist/src/leveldb-storage.d.ts +70 -40
- package/dist/src/leveldb-storage.d.ts.map +1 -1
- package/dist/src/leveldb-storage.js +164 -103
- package/dist/src/leveldb-storage.js.map +1 -1
- package/dist/src/rn-opener.d.ts.map +1 -1
- package/dist/src/rn-opener.js +9 -7
- package/dist/src/rn-opener.js.map +1 -1
- package/package.json +3 -3
- package/src/identity.ts +4 -2
- package/src/index.ts +2 -2
- package/src/keys.ts +57 -0
- package/src/leveldb-kv-store.ts +67 -6
- package/src/leveldb-like.ts +24 -4
- package/src/leveldb-storage.ts +192 -127
- package/src/rn-opener.ts +11 -9
package/src/identity.ts
CHANGED
|
@@ -7,7 +7,9 @@ export const DEFAULT_PEER_KEY_NAME = 'peer-private-key';
|
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Loads the persisted React Native peer's libp2p private key, or generates a
|
|
10
|
-
* fresh Ed25519 key on first call and persists it
|
|
10
|
+
* fresh Ed25519 key on first call and persists it in one batch, synced where
|
|
11
|
+
* the handle can sync, so a power cut right after first start cannot give the
|
|
12
|
+
* node a different peer id next time. An existing key is read as it is.
|
|
11
13
|
*
|
|
12
14
|
* The key is stored as raw protobuf bytes under a dedicated identity tag —
|
|
13
15
|
* LevelDB stores arbitrary byte sequences natively, so there is no base64
|
|
@@ -28,6 +30,6 @@ export async function loadOrCreateRNPeerKey(
|
|
|
28
30
|
}
|
|
29
31
|
const key = await generateKeyPair('Ed25519');
|
|
30
32
|
const bytes = privateKeyToProtobuf(key);
|
|
31
|
-
await db.put(identityKey(keyName), bytes);
|
|
33
|
+
await db.batch().put(identityKey(keyName), bytes).write({ sync: db.canSync });
|
|
32
34
|
return key;
|
|
33
35
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { LevelDBRawStorage,
|
|
2
|
-
export { LevelDBKVStore } from './leveldb-kv-store.js';
|
|
1
|
+
export { LevelDBRawStorage, LevelDBJournalStore, type LevelDBRawStorageOptions } from './leveldb-storage.js';
|
|
2
|
+
export { LevelDBKVStore, type LevelDBKVStoreOptions } from './leveldb-kv-store.js';
|
|
3
3
|
export { loadOrCreateRNPeerKey, DEFAULT_PEER_KEY_NAME } from './identity.js';
|
|
4
4
|
export {
|
|
5
5
|
openOptimysticRNDb,
|
package/src/keys.ts
CHANGED
|
@@ -16,6 +16,10 @@
|
|
|
16
16
|
* - proofs: rev (8-byte big-endian unsigned, same as revisions)
|
|
17
17
|
* - materialized: actionId UTF-8 (terminal)
|
|
18
18
|
*
|
|
19
|
+
* The journal (`TAG_JOURNAL`) has no `blockId` envelope: a record key is the
|
|
20
|
+
* tag then its `seq` as 8 bytes big-endian, and the applied-through mark is the
|
|
21
|
+
* bare tag, which sorts before every numbered key.
|
|
22
|
+
*
|
|
19
23
|
* `kv` and `identity` keys are flat — no `blockId` envelope — under their
|
|
20
24
|
* own tag bytes (`TAG_KV`, `TAG_IDENTITY`) and use UTF-8 of the full key.
|
|
21
25
|
*
|
|
@@ -30,6 +34,7 @@ export const TAG_PENDING = 0x03;
|
|
|
30
34
|
export const TAG_TRANSACTIONS = 0x04;
|
|
31
35
|
export const TAG_MATERIALIZED = 0x05;
|
|
32
36
|
export const TAG_PROOFS = 0x06;
|
|
37
|
+
export const TAG_JOURNAL = 0x07;
|
|
33
38
|
export const TAG_KV = 0x10;
|
|
34
39
|
export const TAG_IDENTITY = 0x20;
|
|
35
40
|
|
|
@@ -150,6 +155,58 @@ export function actionIdFromKey(key: Uint8Array, blockId: string): string {
|
|
|
150
155
|
return textDecoder().decode(key.subarray(suffixOffset));
|
|
151
156
|
}
|
|
152
157
|
|
|
158
|
+
// --- the journal ---
|
|
159
|
+
|
|
160
|
+
const SEQ_BYTES = 8;
|
|
161
|
+
|
|
162
|
+
/** `seq` as 8 bytes big-endian, the suffix of a record key and the value of the mark. */
|
|
163
|
+
function encodeSeq(seq: number): Uint8Array {
|
|
164
|
+
const out = new Uint8Array(SEQ_BYTES);
|
|
165
|
+
new DataView(out.buffer).setBigUint64(0, BigInt(seq), false);
|
|
166
|
+
return out;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** The number 8 bytes big-endian hold, or `undefined` for any other length. */
|
|
170
|
+
function decodeSeq(bytes: Uint8Array): number | undefined {
|
|
171
|
+
if (bytes.byteLength !== SEQ_BYTES) return undefined;
|
|
172
|
+
return Number(new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getBigUint64(0, false));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** `TAG_JOURNAL || seq (8 BE)`: the key of the journal record numbered `seq`. */
|
|
176
|
+
export function journalRecordKey(seq: number): Uint8Array {
|
|
177
|
+
return concat(Uint8Array.of(TAG_JOURNAL), encodeSeq(seq));
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** The bare tag: the key of the applied-through mark, which sorts before every record key. */
|
|
181
|
+
export function journalMarkKey(): Uint8Array {
|
|
182
|
+
return Uint8Array.of(TAG_JOURNAL);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** The mark's value: the number of the last record applied, as 8 bytes big-endian. */
|
|
186
|
+
export function encodeJournalMark(appliedThrough: number): Uint8Array {
|
|
187
|
+
return encodeSeq(appliedThrough);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** The number a mark value holds, or `undefined` when it is not 8 bytes. */
|
|
191
|
+
export function decodeJournalMark(value: Uint8Array): number | undefined {
|
|
192
|
+
return decodeSeq(value);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Returns the inclusive lower / exclusive upper range covering the mark and every record key. */
|
|
196
|
+
export function journalRange(): { gte: Uint8Array; lt: Uint8Array } {
|
|
197
|
+
return { gte: Uint8Array.of(TAG_JOURNAL), lt: Uint8Array.of(TAG_JOURNAL + 1) };
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Whether a key inside `journalRange` is the mark's (the bare tag) rather than a record's. */
|
|
201
|
+
export function isJournalMarkKey(key: Uint8Array): boolean {
|
|
202
|
+
return key.byteLength === 1;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** The `seq` a record key names, or `undefined` for a key of another shape. */
|
|
206
|
+
export function seqFromJournalKey(key: Uint8Array): number | undefined {
|
|
207
|
+
return decodeSeq(key.subarray(1));
|
|
208
|
+
}
|
|
209
|
+
|
|
153
210
|
export function kvKey(key: string): Uint8Array {
|
|
154
211
|
return concat(Uint8Array.of(TAG_KV), textEncoder.encode(key));
|
|
155
212
|
}
|
package/src/leveldb-kv-store.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { WriteGroup, type CommittedBatch, type FlushOptions, type IKVStore, type WriteGroupEntry, type WriteGroupOptions } from '@optimystic/db-p2p';
|
|
2
2
|
import { drain, type LevelDBLike } from './leveldb-like.js';
|
|
3
3
|
import { kvKey, kvKeyToString, kvPrefixRange } from './keys.js';
|
|
4
4
|
|
|
5
|
+
/** The write group's options (`idleFlushMs`, `maxUnsyncedBytes`, the test-only `setTimer`). */
|
|
6
|
+
export type LevelDBKVStoreOptions = WriteGroupOptions;
|
|
7
|
+
|
|
5
8
|
const textEncoder = new TextEncoder();
|
|
6
9
|
// Built on first use, not at module load: Hermes (React Native) has no native `TextDecoder`, so a
|
|
7
10
|
// module-scope construction fails the import itself whenever it runs ahead of the host's polyfill.
|
|
@@ -16,29 +19,87 @@ const textDecoder = (): TextDecoder => textDecoderInstance ??= new TextDecoder()
|
|
|
16
19
|
* so the three subsystems can't collide regardless of the user-chosen
|
|
17
20
|
* `prefix`. `list(prefix)` is a range-bounded scan — never a full-database
|
|
18
21
|
* iteration plus JS-side filter — so listing latency stays bounded.
|
|
22
|
+
*
|
|
23
|
+
* Writes go through a {@link WriteGroup} over the plain keys, as `FileKVStore` does: a `set` or
|
|
24
|
+
* `delete` resolves into memory, served from there to `get` and `list`, and `flush()` commits
|
|
25
|
+
* everything written since the last flush as ONE LevelDB batch, synced wherever the handle can
|
|
26
|
+
* sync (`canSync`), so the writes are on the disk before the flush resolves. Every flush the node
|
|
27
|
+
* makes of this store is strict, and so is the group's own idle flush, so there is no relaxed path
|
|
28
|
+
* to journal: a flush given a window is treated as strict, as a store that cannot defer a sync
|
|
29
|
+
* treats every window, and each batch carries its own data. A failed batch latches the store, after
|
|
30
|
+
* which every write and flush throws `StoreFailedError` (rule 5 of the flush contract on
|
|
31
|
+
* `RawStoreDriver.flush`); reads keep serving the overlay.
|
|
19
32
|
*/
|
|
20
33
|
export class LevelDBKVStore implements IKVStore {
|
|
21
|
-
private readonly
|
|
34
|
+
private readonly group: WriteGroup<string>;
|
|
22
35
|
|
|
23
|
-
constructor(
|
|
24
|
-
|
|
36
|
+
constructor(
|
|
37
|
+
private readonly db: LevelDBLike,
|
|
38
|
+
private readonly prefix: string = 'optimystic:txn:',
|
|
39
|
+
options: LevelDBKVStoreOptions = {},
|
|
40
|
+
) {
|
|
41
|
+
this.group = new WriteGroup<string>(key => key, {
|
|
42
|
+
commit: entries => this.commit(entries),
|
|
43
|
+
// Every commit is as durable as this store can make it when it resolves, so nothing is ever owed a sync.
|
|
44
|
+
sync: () => Promise.resolve(),
|
|
45
|
+
}, options);
|
|
25
46
|
}
|
|
26
47
|
|
|
27
48
|
async get(key: string): Promise<string | undefined> {
|
|
49
|
+
const held = this.group.lookup(key);
|
|
50
|
+
if (held !== undefined) return held === null ? undefined : textDecoder().decode(held);
|
|
28
51
|
const bytes = await this.db.get(kvKey(this.prefix + key));
|
|
29
52
|
if (!bytes) return undefined;
|
|
30
53
|
return textDecoder().decode(bytes);
|
|
31
54
|
}
|
|
32
55
|
|
|
33
56
|
async set(key: string, value: string): Promise<void> {
|
|
34
|
-
|
|
57
|
+
this.group.put(key, textEncoder.encode(value));
|
|
35
58
|
}
|
|
36
59
|
|
|
37
60
|
async delete(key: string): Promise<void> {
|
|
38
|
-
|
|
61
|
+
this.group.delete(key);
|
|
39
62
|
}
|
|
40
63
|
|
|
64
|
+
/** Every key under `prefix`: the store's, plus those written and not yet flushed, less those deleted and not yet flushed. */
|
|
41
65
|
async list(prefix: string): Promise<string[]> {
|
|
66
|
+
// Taken before awaiting the store: a commit resolving meanwhile drops entries from the overlay
|
|
67
|
+
// that the listing below may have been taken without.
|
|
68
|
+
const held = [...this.group.entries()].filter(([key]) => key.startsWith(prefix));
|
|
69
|
+
const keys = new Set(await this.listStored(prefix));
|
|
70
|
+
for (const [key, value] of held) {
|
|
71
|
+
if (value === null) keys.delete(key);
|
|
72
|
+
else keys.add(key);
|
|
73
|
+
}
|
|
74
|
+
return [...keys];
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Make every write resolved before this call durable: one batch, synced where the handle can, whatever `options.withinMs` says. The contract is `RawStoreDriver.flush`'s. */
|
|
78
|
+
flush(options?: FlushOptions): Promise<void> {
|
|
79
|
+
return this.group.flush(options);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Resolves once every write covered by a flush that resolved before this call is durable; `RawStoreDriver.whenDurable`'s contract. */
|
|
83
|
+
whenDurable(): Promise<void> {
|
|
84
|
+
return this.group.whenDurable();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Flush what is still held and disarm the idle timer. The host closes the shared handle after this resolves. */
|
|
88
|
+
close(): Promise<void> {
|
|
89
|
+
return this.group.close();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private async commit(entries: readonly WriteGroupEntry<string>[]): Promise<CommittedBatch> {
|
|
93
|
+
const batch = this.db.batch();
|
|
94
|
+
for (const { key, value } of entries) {
|
|
95
|
+
const encoded = kvKey(this.prefix + key);
|
|
96
|
+
if (value === undefined) batch.delete(encoded); else batch.put(encoded, value);
|
|
97
|
+
}
|
|
98
|
+
await batch.write({ sync: this.db.canSync });
|
|
99
|
+
return { durable: Promise.resolve() };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
private async listStored(prefix: string): Promise<string[]> {
|
|
42
103
|
const range = kvPrefixRange(this.prefix + prefix);
|
|
43
104
|
const entries = await drain(this.db.iterator({ gte: range.gte, lt: range.lt, keys: true }));
|
|
44
105
|
return entries.map(([rawKey]) => kvKeyToString(rawKey).slice(this.prefix.length));
|
package/src/leveldb-like.ts
CHANGED
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
* Only `openOptimysticRNDb`, the user-facing constructors, and the identity
|
|
11
11
|
* helper are exported from `index.ts`; the interfaces in this file are
|
|
12
12
|
* internal — consumers never see them.
|
|
13
|
+
*
|
|
14
|
+
* Every write this package makes is a `batch().write(...)`: the surface has no
|
|
15
|
+
* single `put` or `delete`, on purpose. A direct write lands unsynced in
|
|
16
|
+
* LevelDB's current log file, which LevelDB closes unsynced when it switches to
|
|
17
|
+
* a new one, so a later synced write does not cover it — exactly the write the
|
|
18
|
+
* journal under `LevelDBRawStorage` exists to keep out of the store.
|
|
13
19
|
*/
|
|
14
20
|
|
|
15
21
|
/** Range options for a `LevelDBLike.iterator` scan. */
|
|
@@ -38,12 +44,22 @@ export interface LevelDBIteratorLike {
|
|
|
38
44
|
close(): Promise<void>;
|
|
39
45
|
}
|
|
40
46
|
|
|
47
|
+
/** How a batch is written. */
|
|
48
|
+
export interface LevelDBWriteOptions {
|
|
49
|
+
/**
|
|
50
|
+
* Write with LevelDB's `WriteOptions.sync`: the batch is on the disk before the write resolves.
|
|
51
|
+
* Only valid on a handle whose `canSync` is true; asking a handle that cannot is a caller bug,
|
|
52
|
+
* and the adapter throws rather than write unsynced and report otherwise.
|
|
53
|
+
*/
|
|
54
|
+
readonly sync?: boolean;
|
|
55
|
+
}
|
|
56
|
+
|
|
41
57
|
/** Atomic batch of `put` / `delete` operations against a single database. */
|
|
42
58
|
export interface LevelDBWriteBatchLike {
|
|
43
59
|
put(key: Uint8Array, value: Uint8Array): this;
|
|
44
60
|
delete(key: Uint8Array): this;
|
|
45
|
-
/** Commit the batch atomically. */
|
|
46
|
-
write(): Promise<void>;
|
|
61
|
+
/** Commit the batch atomically; unsynced unless `options.sync`. The batch is consumed either way. */
|
|
62
|
+
write(options?: LevelDBWriteOptions): Promise<void>;
|
|
47
63
|
}
|
|
48
64
|
|
|
49
65
|
/**
|
|
@@ -55,9 +71,13 @@ export interface LevelDBWriteBatchLike {
|
|
|
55
71
|
* uniform with `classic-level`'s native async API.
|
|
56
72
|
*/
|
|
57
73
|
export interface LevelDBLike {
|
|
74
|
+
/**
|
|
75
|
+
* Whether `batch().write({ sync: true })` is honoured. Fixed for the life of the handle; it is
|
|
76
|
+
* what `LevelDBRawStorage` reports its `durability` from. `rn-leveldb` cannot sync today
|
|
77
|
+
* (`react-native-storage-syncs-once-rn-leveldb-can`); `classic-level` can.
|
|
78
|
+
*/
|
|
79
|
+
readonly canSync: boolean;
|
|
58
80
|
get(key: Uint8Array): Promise<Uint8Array | undefined>;
|
|
59
|
-
put(key: Uint8Array, value: Uint8Array): Promise<void>;
|
|
60
|
-
delete(key: Uint8Array): Promise<void>;
|
|
61
81
|
batch(): LevelDBWriteBatchLike;
|
|
62
82
|
iterator(options?: LevelDBIteratorOptions): LevelDBIteratorLike;
|
|
63
83
|
/** Release the underlying database handle. */
|
package/src/leveldb-storage.ts
CHANGED
|
@@ -1,11 +1,33 @@
|
|
|
1
1
|
import type { ActionId, BlockId } from '@optimystic/db-core';
|
|
2
|
-
import {
|
|
3
|
-
|
|
2
|
+
import {
|
|
3
|
+
JournalBatchBackend,
|
|
4
|
+
JournalRecordDecodeError,
|
|
5
|
+
KvRawStorage,
|
|
6
|
+
WriteGroupDriver,
|
|
7
|
+
decodeJournalRecord,
|
|
8
|
+
encodeJournalRecord,
|
|
9
|
+
identityForHandle,
|
|
10
|
+
type JournalBatchBackendOptions,
|
|
11
|
+
type JournalContents,
|
|
12
|
+
type JournalRecord,
|
|
13
|
+
type JournalStore,
|
|
14
|
+
type StoreDurability,
|
|
15
|
+
type StoreIdentity,
|
|
16
|
+
type StoreKey,
|
|
17
|
+
type WriteGroupOptions,
|
|
18
|
+
} from '@optimystic/db-p2p';
|
|
19
|
+
import { drain, type LevelDBLike, type LevelDBWriteBatchLike } from './leveldb-like.js';
|
|
4
20
|
import {
|
|
5
21
|
TAG_PENDING,
|
|
6
22
|
actionIdFromKey,
|
|
7
23
|
blockEnvelopeRange,
|
|
8
24
|
blockIdFromMetadataKey,
|
|
25
|
+
decodeJournalMark,
|
|
26
|
+
encodeJournalMark,
|
|
27
|
+
isJournalMarkKey,
|
|
28
|
+
journalMarkKey,
|
|
29
|
+
journalRange,
|
|
30
|
+
journalRecordKey,
|
|
9
31
|
materializedKey,
|
|
10
32
|
metadataKey,
|
|
11
33
|
metadataRange,
|
|
@@ -13,27 +35,71 @@ import {
|
|
|
13
35
|
proofKey,
|
|
14
36
|
revisionFromKey,
|
|
15
37
|
revisionKey,
|
|
38
|
+
seqFromJournalKey,
|
|
16
39
|
transactionKey,
|
|
17
40
|
} from './keys.js';
|
|
18
41
|
import { createLogger } from './logger.js';
|
|
19
42
|
|
|
20
43
|
const log = createLogger('storage:leveldb');
|
|
21
44
|
|
|
45
|
+
/** The LevelDB key a `StoreKey` addresses: the tag-prefixed byte scheme of `keys.ts`, unchanged from the per-write driver before it. */
|
|
46
|
+
function keyBytes(key: StoreKey): Uint8Array {
|
|
47
|
+
switch (key.store) {
|
|
48
|
+
case 'metadata':
|
|
49
|
+
return metadataKey(key.blockId);
|
|
50
|
+
case 'revisions':
|
|
51
|
+
return revisionKey(key.blockId, key.rev);
|
|
52
|
+
case 'proofs':
|
|
53
|
+
return proofKey(key.blockId, key.rev);
|
|
54
|
+
case 'pending':
|
|
55
|
+
return pendingKey(key.blockId, key.actionId);
|
|
56
|
+
case 'transactions':
|
|
57
|
+
return transactionKey(key.blockId, key.actionId);
|
|
58
|
+
case 'materialized':
|
|
59
|
+
return materializedKey(key.blockId, key.actionId);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
22
63
|
/**
|
|
23
|
-
* LevelDB {@link
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
64
|
+
* LevelDB {@link JournalStore}: the six logical block-storage stores mapped into a single ordered
|
|
65
|
+
* byte keyspace, partitioned by a leading tag byte (see `./keys.ts`), read exactly as the per-write
|
|
66
|
+
* driver before it read them, so data an earlier build wrote reads unchanged; and the journal under
|
|
67
|
+
* its own tag byte, which `JournalBatchBackend` drives.
|
|
68
|
+
*
|
|
69
|
+
* `KvRawStorage` owns all JSON/UTF-8 serialization, so this store only ever reads/writes raw
|
|
70
|
+
* `Uint8Array` values; a journal record is bytes too, through `encodeJournalRecord`. Everything
|
|
71
|
+
* LevelDB-specific stays here: the tag-range scans (drained snapshot-first before yielding, so a
|
|
72
|
+
* native iterator never straddles a consumer's `await`), and the one `WriteBatch` each journal
|
|
73
|
+
* write is.
|
|
28
74
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
75
|
+
* ### Why a group survives, and in order
|
|
76
|
+
*
|
|
77
|
+
* A LevelDB batch is atomic, but LevelDB syncs only its current log file, closes an old one
|
|
78
|
+
* unsynced when it switches, and under default options skips a damaged log record and keeps the
|
|
79
|
+
* ones after it, so a later write can survive a power cut while an earlier one is lost — which is
|
|
80
|
+
* why the backend journals (see `JournalBatchBackend`). Here an append is one unsynced batch
|
|
81
|
+
* putting the record under its sequence, and an apply is one batch over the data tags and the
|
|
82
|
+
* journal — every record's entries in order, the applied-through mark, and the delete of each
|
|
83
|
+
* record applied or discarded — written synced wherever the handle can sync (`canSync`). The apply
|
|
84
|
+
* carries every byte it applies, so its durability depends on no earlier batch, and the backend
|
|
85
|
+
* syncs each apply before the next starts: what a power cut leaves is a prefix of the groups, and
|
|
86
|
+
* `durability` is `'power-loss'`. Nothing this package writes bypasses a batch, so no unsynced
|
|
87
|
+
* direct write can sit in an old log file outside the journal's reach.
|
|
88
|
+
*
|
|
89
|
+
* `rn-leveldb` cannot sync today, so on a phone the store reports `'process-crash'` and the backend
|
|
90
|
+
* applies each group at once as one unsynced batch, never journaling: a group lands whole or not
|
|
91
|
+
* at all, and the ordering half of the contract arrives with the binding change and nothing here.
|
|
92
|
+
*
|
|
93
|
+
* The handle is shared with `LevelDBKVStore` and `loadOrCreateRNPeerKey`, so there is no `close`
|
|
94
|
+
* here: the host closes the handle after `storage.close()` has resolved.
|
|
35
95
|
*/
|
|
36
|
-
export class
|
|
96
|
+
export class LevelDBJournalStore implements JournalStore {
|
|
97
|
+
// NOTE: `'process-crash'` holds only while the handle cannot sync (flush contract rule 3 is then
|
|
98
|
+
// not kept across a power cut: LevelDB's recovery skips a damaged log record and keeps the ones
|
|
99
|
+
// after it, and no apply is synced). The synced path is built and tested over `classic-level`;
|
|
100
|
+
// `react-native-storage-syncs-once-rn-leveldb-can` is the binding change that turns it on.
|
|
101
|
+
readonly durability: StoreDurability;
|
|
102
|
+
|
|
37
103
|
// NOTE: identity is the HANDLE object, so two handles opened over the same LevelDB PATH read
|
|
38
104
|
// as two identities. Resolving that would mean naming the underlying on-disk path, which
|
|
39
105
|
// `LevelDBLike` does not expose. The package opener (`rn-opener.ts`) hands out one handle per
|
|
@@ -41,6 +107,7 @@ export class LevelDBStoreDriver implements RawStoreDriver {
|
|
|
41
107
|
private readonly identity: StoreIdentity;
|
|
42
108
|
|
|
43
109
|
constructor(private readonly db: LevelDBLike) {
|
|
110
|
+
this.durability = db.canSync ? 'power-loss' : 'process-crash';
|
|
44
111
|
this.identity = identityForHandle('leveldb-handle', db);
|
|
45
112
|
}
|
|
46
113
|
|
|
@@ -48,119 +115,30 @@ export class LevelDBStoreDriver implements RawStoreDriver {
|
|
|
48
115
|
return this.identity;
|
|
49
116
|
}
|
|
50
117
|
|
|
51
|
-
// ---
|
|
118
|
+
// --- reads: the applied stores ---
|
|
52
119
|
|
|
53
|
-
async
|
|
54
|
-
return this.db.get(
|
|
120
|
+
async get(key: StoreKey): Promise<Uint8Array | undefined> {
|
|
121
|
+
return this.db.get(keyBytes(key));
|
|
55
122
|
}
|
|
56
123
|
|
|
57
|
-
async
|
|
58
|
-
await this.db.put(metadataKey(blockId), value);
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
// --- revisions ---
|
|
62
|
-
|
|
63
|
-
async getRevision(blockId: BlockId, rev: number): Promise<Uint8Array | undefined> {
|
|
64
|
-
return this.db.get(revisionKey(blockId, rev));
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
async putRevision(blockId: BlockId, rev: number, value: Uint8Array): Promise<void> {
|
|
68
|
-
await this.db.put(revisionKey(blockId, rev), value);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
async *rangeRevisions(blockId: BlockId, lo: number, hi: number, reverse: boolean): AsyncIterable<[number, Uint8Array]> {
|
|
124
|
+
async rangeRevisions(blockId: BlockId, lo: number, hi: number): Promise<[number, Uint8Array][]> {
|
|
72
125
|
// Revs are encoded 8-byte big-endian (`revisionKey`), so byte order == numeric
|
|
73
126
|
// order and `revisionKey(blockId, hi)` is exactly the inclusive upper bound;
|
|
74
127
|
// LevelDB's upper bound is exclusive (`lt`), so use `lt = revisionKey(hi + 1)`.
|
|
75
128
|
const gte = revisionKey(blockId, lo);
|
|
76
129
|
const lt = revisionKey(blockId, hi + 1);
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
const entries = await drain(this.db.iterator({ gte, lt
|
|
80
|
-
|
|
81
|
-
yield [revisionFromKey(key), value];
|
|
82
|
-
}
|
|
130
|
+
// Drained before returning: a native LevelDB iterator must not stay open across the
|
|
131
|
+
// consumer's awaits (the kernel's drain-before-yield contract).
|
|
132
|
+
const entries = await drain(this.db.iterator({ gte, lt }));
|
|
133
|
+
return entries.map(([key, value]) => [revisionFromKey(key), value]);
|
|
83
134
|
}
|
|
84
135
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
async getPending(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
88
|
-
return this.db.get(pendingKey(blockId, actionId));
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
async putPending(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
92
|
-
await this.db.put(pendingKey(blockId, actionId), value);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
async deletePending(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
96
|
-
await this.db.delete(pendingKey(blockId, actionId));
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
async *listPendingActionIds(blockId: BlockId): AsyncIterable<ActionId> {
|
|
136
|
+
async listPendingActionIds(blockId: BlockId): Promise<ActionId[]> {
|
|
100
137
|
const range = blockEnvelopeRange(TAG_PENDING, blockId);
|
|
101
|
-
// Drain before yielding, same rationale as rangeRevisions.
|
|
102
138
|
const entries = await drain(this.db.iterator({ gte: range.gte, lt: range.lt, keys: true }));
|
|
103
|
-
|
|
104
|
-
yield actionIdFromKey(key, blockId) as ActionId;
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
// --- transactions ---
|
|
109
|
-
|
|
110
|
-
async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
111
|
-
return this.db.get(transactionKey(blockId, actionId));
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
async putTransaction(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
115
|
-
await this.db.put(transactionKey(blockId, actionId), value);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
// --- proofs (own tag byte; no delete — see RawStoreDriver.getProof) ---
|
|
119
|
-
|
|
120
|
-
async getProof(blockId: BlockId, rev: number): Promise<Uint8Array | undefined> {
|
|
121
|
-
return this.db.get(proofKey(blockId, rev));
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
async putProof(blockId: BlockId, rev: number, value: Uint8Array): Promise<void> {
|
|
125
|
-
await this.db.put(proofKey(blockId, rev), value);
|
|
139
|
+
return entries.map(([key]) => actionIdFromKey(key, blockId) as ActionId);
|
|
126
140
|
}
|
|
127
141
|
|
|
128
|
-
// --- materialized ---
|
|
129
|
-
|
|
130
|
-
async getMaterialized(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
131
|
-
return this.db.get(materializedKey(blockId, actionId));
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
async putMaterialized(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
135
|
-
await this.db.put(materializedKey(blockId, actionId), value);
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
// The kernel owns the put-or-delete branch of `saveMaterializedBlock`, so the
|
|
139
|
-
// driver exposes delete as a separate op.
|
|
140
|
-
async deleteMaterialized(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
141
|
-
await this.db.delete(materializedKey(blockId, actionId));
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
// --- promote (the only cross-key atomic op) ---
|
|
145
|
-
|
|
146
|
-
async promote(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
147
|
-
const pKey = pendingKey(blockId, actionId);
|
|
148
|
-
const value = await this.db.get(pKey);
|
|
149
|
-
if (!value) {
|
|
150
|
-
throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
|
|
151
|
-
}
|
|
152
|
-
// The single `WriteBatch` IS the atomic move: a crash leaves either the
|
|
153
|
-
// pending or the committed entry, never both/neither.
|
|
154
|
-
const tKey = transactionKey(blockId, actionId);
|
|
155
|
-
await this.db
|
|
156
|
-
.batch()
|
|
157
|
-
.put(tKey, value)
|
|
158
|
-
.delete(pKey)
|
|
159
|
-
.write();
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// --- optional passthroughs ---
|
|
163
|
-
|
|
164
142
|
async *listBlockIds(): AsyncIterable<BlockId> {
|
|
165
143
|
// Scan the whole TAG_METADATA keyspace; each metadata key is one distinct
|
|
166
144
|
// block id (empty suffix). keys:true skips values — we only need the ids.
|
|
@@ -197,31 +175,118 @@ export class LevelDBStoreDriver implements RawStoreDriver {
|
|
|
197
175
|
}
|
|
198
176
|
}
|
|
199
177
|
|
|
178
|
+
// --- the journal ---
|
|
179
|
+
|
|
180
|
+
/** One unsynced batch: the record's bytes under its `seq`. `JournalStore.appendJournal`'s contract. */
|
|
181
|
+
async appendJournal(record: JournalRecord): Promise<void> {
|
|
182
|
+
await this.db.batch().put(journalRecordKey(record.seq), encodeJournalRecord(record)).write();
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* One batch over the data tags and the journal, synced wherever the handle can sync: each
|
|
187
|
+
* record's entries in order, then the mark, then the delete of every numbered record. A number
|
|
188
|
+
* with no record present (a strict group's, never appended) deletes nothing.
|
|
189
|
+
* `JournalStore.applyJournal`'s contract.
|
|
190
|
+
*/
|
|
191
|
+
// NOTE: on a handle that cannot sync every group is applied this way, so each flush also puts the
|
|
192
|
+
// mark and deletes a record key that was never appended (a LevelDB tombstone): two small extra
|
|
193
|
+
// operations per flush, unmeasured. If flush cost on phones ever shows in profiles, have the
|
|
194
|
+
// backend hand the store only the numbers it appended, rather than every number applied.
|
|
195
|
+
async applyJournal(apply: readonly JournalRecord[], appliedThrough: number, discard: readonly number[]): Promise<void> {
|
|
196
|
+
const batch = this.db.batch();
|
|
197
|
+
for (const record of apply) {
|
|
198
|
+
putEntries(batch, record);
|
|
199
|
+
}
|
|
200
|
+
batch.put(journalMarkKey(), encodeJournalMark(appliedThrough));
|
|
201
|
+
for (const { seq } of apply) batch.delete(journalRecordKey(seq));
|
|
202
|
+
for (const seq of discard) batch.delete(journalRecordKey(seq));
|
|
203
|
+
await batch.write({ sync: this.db.canSync });
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* One drained scan over the journal's tag: the mark and every numbered record, ascending (the
|
|
208
|
+
* sequence is 8 bytes big-endian, so key order is sequence order). A record that does not
|
|
209
|
+
* decode, or whose decoded number is not the one its key names, is left out, which recovery
|
|
210
|
+
* treats as a gap; a mark that is not 8 bytes reads as 0. `JournalStore.readJournal`'s contract.
|
|
211
|
+
*/
|
|
212
|
+
async readJournal(): Promise<JournalContents> {
|
|
213
|
+
const range = journalRange();
|
|
214
|
+
const entries = await drain(this.db.iterator({ gte: range.gte, lt: range.lt }));
|
|
215
|
+
let appliedThrough = 0;
|
|
216
|
+
const records: JournalRecord[] = [];
|
|
217
|
+
for (const [key, value] of entries) {
|
|
218
|
+
if (isJournalMarkKey(key)) {
|
|
219
|
+
appliedThrough = this.readMark(value);
|
|
220
|
+
} else {
|
|
221
|
+
const record = this.readRecord(key, value);
|
|
222
|
+
if (record) records.push(record);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return { appliedThrough, records };
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
private readMark(value: Uint8Array): number {
|
|
229
|
+
const mark = decodeJournalMark(value);
|
|
230
|
+
if (mark === undefined || !Number.isSafeInteger(mark)) {
|
|
231
|
+
log('read-journal: the applied-through mark of %s is not a number this build can read (%d bytes) and reads as 0', this.identity, value.byteLength);
|
|
232
|
+
return 0;
|
|
233
|
+
}
|
|
234
|
+
return mark;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
private readRecord(key: Uint8Array, value: Uint8Array): JournalRecord | undefined {
|
|
238
|
+
const seq = seqFromJournalKey(key);
|
|
239
|
+
try {
|
|
240
|
+
const record = decodeJournalRecord(value);
|
|
241
|
+
if (record.seq === seq) return record;
|
|
242
|
+
log('read-journal: a journal record of %s is numbered %d but filed under %s and is left out; recovery treats its number as missing', this.identity, record.seq, String(seq));
|
|
243
|
+
return undefined;
|
|
244
|
+
} catch (err) {
|
|
245
|
+
if (!(err instanceof JournalRecordDecodeError)) throw err;
|
|
246
|
+
log('read-journal: journal record %s of %s is not one this build can read and is left out (%s); recovery treats its number as missing', String(seq), this.identity, err.message);
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
200
251
|
// NOTE: intentionally NO close(). The `LevelDBLike` handle is shared with
|
|
201
252
|
// `LevelDBKVStore` and `loadOrCreateRNPeerKey` (one db per RN peer — see the
|
|
202
|
-
// README usage), so the
|
|
203
|
-
// would break the other two. Same rationale as the IndexedDB
|
|
204
|
-
// kernel never wires the optional close() regardless.
|
|
253
|
+
// README usage), so the store must not own its lifecycle; closing it here
|
|
254
|
+
// would break the other two. Same rationale as the IndexedDB store.
|
|
205
255
|
}
|
|
206
256
|
|
|
257
|
+
/** The record's entries onto the batch, in order: a put stores the value, a delete removes the key. */
|
|
258
|
+
function putEntries(batch: LevelDBWriteBatchLike, record: JournalRecord): void {
|
|
259
|
+
for (const { key, value } of record.entries) {
|
|
260
|
+
const encoded = keyBytes(key);
|
|
261
|
+
if (value === undefined) batch.delete(encoded); else batch.put(encoded, value);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** The write group's and the journal backend's options (`idleFlushMs`, `maxUnsyncedBytes`, the test-only `setTimer` and `now`). */
|
|
266
|
+
export type LevelDBRawStorageOptions = WriteGroupOptions & JournalBatchBackendOptions;
|
|
267
|
+
|
|
207
268
|
/**
|
|
208
|
-
* LevelDB-backed {@link IRawStorage} for React Native peers
|
|
209
|
-
* over
|
|
210
|
-
*
|
|
211
|
-
*
|
|
212
|
-
*
|
|
269
|
+
* LevelDB-backed {@link IRawStorage} for React Native peers: the shared {@link KvRawStorage}
|
|
270
|
+
* kernel over a {@link WriteGroupDriver} over a {@link JournalBatchBackend} over a
|
|
271
|
+
* {@link LevelDBJournalStore}. Every write resolves into the driver's in-memory overlay; `flush()`
|
|
272
|
+
* hands the group to the backend as one LevelDB batch — applied before the flush resolves, or,
|
|
273
|
+
* for a flush with a window on a handle that can sync, appended as one journal record and applied
|
|
274
|
+
* by the deadline. The public name/constructor (`new LevelDBRawStorage(db)`) is unchanged so
|
|
275
|
+
* existing imports keep resolving.
|
|
213
276
|
*
|
|
214
|
-
* `listBlockIds`/`getApproximateBytesUsed` are re-declared here as
|
|
215
|
-
* (the
|
|
216
|
-
*
|
|
217
|
-
* them.
|
|
277
|
+
* `listBlockIds`/`getApproximateBytesUsed`/`getStoreIdentity`/`close` are re-declared here as
|
|
278
|
+
* always-present (the store always implements the first three, so the backend, driver and kernel
|
|
279
|
+
* always wire them, and the driver always has a `close`) — the base declares them optional, but
|
|
280
|
+
* every RN consumer relies on them. `close()` applies what is still held and disarms the timers;
|
|
281
|
+
* it does not close the shared handle, which the host closes afterwards.
|
|
218
282
|
*/
|
|
219
283
|
export class LevelDBRawStorage extends KvRawStorage {
|
|
220
284
|
declare listBlockIds: () => AsyncIterable<BlockId>;
|
|
221
285
|
declare getApproximateBytesUsed: () => Promise<number>;
|
|
222
286
|
declare getStoreIdentity: () => StoreIdentity;
|
|
287
|
+
declare close: () => Promise<void>;
|
|
223
288
|
|
|
224
|
-
constructor(db: LevelDBLike) {
|
|
225
|
-
super(new
|
|
289
|
+
constructor(db: LevelDBLike, options: LevelDBRawStorageOptions = {}) {
|
|
290
|
+
super(new WriteGroupDriver(new JournalBatchBackend(new LevelDBJournalStore(db), options), options));
|
|
226
291
|
}
|
|
227
292
|
}
|