@optimystic/db-p2p 0.22.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/client.d.ts +10 -0
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +30 -1
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +39 -14
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +226 -119
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/service.d.ts +32 -1
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +43 -2
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cohort-topic/host.js +34 -11
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +37 -7
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +77 -19
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +9 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +88 -2
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +134 -28
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +25 -1
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/logger.d.ts +17 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -2
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.js +3 -3
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/owned-block-seed.d.ts +6 -3
- package/dist/src/owned-block-seed.d.ts.map +1 -1
- package/dist/src/owned-block-seed.js +16 -3
- package/dist/src/owned-block-seed.js.map +1 -1
- package/dist/src/peer-address-book.d.ts +72 -0
- package/dist/src/peer-address-book.d.ts.map +1 -0
- package/dist/src/peer-address-book.js +123 -0
- package/dist/src/peer-address-book.js.map +1 -0
- package/dist/src/reactivity/notify-transport.d.ts +4 -4
- package/dist/src/reactivity/notify-transport.js +6 -6
- package/dist/src/reactivity/notify-transport.js.map +1 -1
- package/dist/src/reactivity/push-state-gossip.js +2 -2
- package/dist/src/reactivity/push-state-gossip.js.map +1 -1
- package/dist/src/reactivity/recover-transport.d.ts +6 -2
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +7 -3
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +11 -2
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +30 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +95 -3
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +62 -9
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +242 -73
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/rn.d.ts +3 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +3 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +83 -0
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
- package/dist/src/storage/cached-raw-storage.js +152 -0
- package/dist/src/storage/cached-raw-storage.js.map +1 -0
- package/dist/src/storage/cached-store-driver.d.ts +186 -0
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
- package/dist/src/storage/cached-store-driver.js +775 -0
- package/dist/src/storage/cached-store-driver.js.map +1 -0
- package/dist/src/storage/i-raw-storage.d.ts +12 -5
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +234 -0
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
- package/dist/src/storage/shared-cache-pool.js +354 -0
- package/dist/src/storage/shared-cache-pool.js.map +1 -0
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts +13 -6
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/cohort-topic-mesh-harness.js +15 -6
- package/dist/src/testing/cohort-topic-mesh-harness.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +35 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +3 -3
- package/readme.md +668 -668
- package/src/cluster/block-transfer.ts +424 -424
- package/src/cluster/client.ts +119 -88
- package/src/cluster/cluster-error.ts +64 -64
- package/src/cluster/cluster-policy.ts +203 -203
- package/src/cluster/cluster-repo.ts +242 -122
- package/src/cluster/cluster-size-coupling.ts +45 -45
- package/src/cluster/commit-cert.ts +139 -139
- package/src/cluster/i-transaction-state-store.ts +43 -43
- package/src/cluster/memory-transaction-state-store.ts +56 -56
- package/src/cluster/peer-key-binding.ts +37 -37
- package/src/cluster/persistent-transaction-state-store.ts +92 -92
- package/src/cluster/quorum-restore.ts +223 -223
- package/src/cluster/reconcile-block.ts +203 -203
- package/src/cluster/service.ts +293 -241
- package/src/cluster/supermajority-coupling.ts +37 -37
- package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
- package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
- package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
- package/src/cohort-topic/change-bridge.ts +109 -109
- package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
- package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
- package/src/cohort-topic/fret-trust-anchor.ts +153 -153
- package/src/cohort-topic/host.ts +42 -11
- package/src/cohort-topic/index.ts +13 -13
- package/src/cohort-topic/membership-publish-sink.ts +20 -20
- package/src/cohort-topic/membership-source.ts +68 -68
- package/src/cohort-topic/peer-codec.ts +31 -31
- package/src/cohort-topic/peer-sig.ts +86 -86
- package/src/cohort-topic/protocols.ts +71 -71
- package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
- package/src/cohort-topic/size-estimator.ts +16 -16
- package/src/cohort-topic/stream-util.ts +79 -19
- package/src/cohort-topic/threshold-crypto.ts +239 -239
- package/src/cohort-topic/topic-router.ts +77 -77
- package/src/dispute/arbitrator-selection.ts +138 -138
- package/src/dispute/cascade.ts +524 -524
- package/src/dispute/dispute-service.ts +11 -5
- package/src/dispute/invalidation.ts +625 -625
- package/src/inbound-authorization.ts +190 -190
- package/src/index.ts +52 -49
- package/src/libp2p-key-network.ts +1120 -990
- package/src/libp2p-node-base.ts +1675 -1651
- package/src/libp2p-node-rn.ts +30 -30
- package/src/libp2p-node.ts +36 -36
- package/src/logger.ts +19 -2
- package/src/matchmaking/aggregate-counts.ts +104 -104
- package/src/matchmaking/index.ts +20 -20
- package/src/matchmaking/module.ts +363 -363
- package/src/matchmaking/protocols.ts +51 -51
- package/src/matchmaking/provider-manager.ts +95 -95
- package/src/matchmaking/query-handler.ts +88 -88
- package/src/matchmaking/query-transport.ts +3 -3
- package/src/matchmaking/seeker-manager.ts +64 -64
- package/src/matchmaking/seeker-walk-client.ts +293 -293
- package/src/matchmaking/traffic-validation.ts +195 -195
- package/src/optimystic-node.ts +36 -36
- package/src/owned-block-seed.ts +53 -40
- package/src/peer-address-book.ts +149 -0
- package/src/protocol-limits.ts +33 -33
- package/src/reactivity/forwarder-host.ts +438 -438
- package/src/reactivity/index.ts +19 -19
- package/src/reactivity/notify-transport.ts +144 -144
- package/src/reactivity/origination-manager.ts +192 -192
- package/src/reactivity/protocols.ts +61 -61
- package/src/reactivity/push-state-gossip.ts +291 -291
- package/src/reactivity/recover-transport.ts +7 -3
- package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
- package/src/reactivity/subscriber-registry.ts +96 -96
- package/src/reactivity/subscription-manager.ts +450 -450
- package/src/reactivity/topic-bytes.ts +37 -37
- package/src/repo/client.ts +12 -2
- package/src/repo/cluster-coordinator.ts +99 -3
- package/src/repo/coordinator-repo.ts +281 -74
- package/src/repo/types.ts +7 -7
- package/src/rn.ts +39 -36
- package/src/rpc-deadline.ts +45 -45
- package/src/storage/arachnode-partition.ts +74 -74
- package/src/storage/cached-raw-storage.ts +180 -0
- package/src/storage/cached-store-driver.ts +859 -0
- package/src/storage/i-kv-store.ts +8 -8
- package/src/storage/i-raw-storage.ts +12 -5
- package/src/storage/kv-raw-storage.ts +135 -135
- package/src/storage/memory-kv-store.ts +28 -28
- package/src/storage/memory-storage.ts +25 -25
- package/src/storage/memory-store-driver.ts +157 -157
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +80 -80
- package/src/storage/ring-selector.ts +317 -317
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/shared-cache-pool.ts +452 -0
- package/src/storage/storage-repo.ts +1014 -1014
- package/src/testing/cohort-topic-mesh-harness.ts +673 -663
- package/src/testing/index.ts +8 -8
- package/src/testing/matchmaking-mesh-harness.ts +475 -475
- package/src/testing/raw-storage-conformance.ts +453 -417
- package/src/testing/reactivity-mesh-harness.ts +922 -922
- package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
- package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
- package/dist/src/storage/restoration-coordinator-v2.js +0 -172
- package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
|
@@ -1,157 +1,157 @@
|
|
|
1
|
-
import type { BlockId, ActionId } from "@optimystic/db-core";
|
|
2
|
-
import type { RawStoreDriver } from "./raw-store-driver.js";
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* In-memory {@link RawStoreDriver}: the five logical block-storage stores as
|
|
6
|
-
* `Map`s of `Uint8Array` values. `KvRawStorage` hands this driver bytes produced
|
|
7
|
-
* by `JSON`-encode and reads them back via `JSON`-decode, so every get yields a
|
|
8
|
-
* fresh object and every save stored an independent byte snapshot BY
|
|
9
|
-
* CONSTRUCTION — the old `structuredClone`-on-every-get/put discipline is now
|
|
10
|
-
* structural, not a rule a maintainer has to remember. The driver therefore
|
|
11
|
-
* stores the byte reference directly: those bytes are never mutated by the
|
|
12
|
-
* kernel or exposed to callers as a mutable handle.
|
|
13
|
-
*/
|
|
14
|
-
export class MemoryStoreDriver implements RawStoreDriver {
|
|
15
|
-
private readonly metadata = new Map<BlockId, Uint8Array>();
|
|
16
|
-
private readonly revisions = new Map<string, Uint8Array>(); // `${blockId}:${rev}` -> actionId bytes
|
|
17
|
-
private readonly pending = new Map<string, Uint8Array>(); // `${blockId}:${actionId}` -> transform bytes
|
|
18
|
-
private readonly transactions = new Map<string, Uint8Array>();
|
|
19
|
-
private readonly materialized = new Map<string, Uint8Array>();
|
|
20
|
-
|
|
21
|
-
private revisionKey(blockId: BlockId, rev: number): string {
|
|
22
|
-
return `${blockId}:${rev}`;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
private actionKey(blockId: BlockId, actionId: ActionId): string {
|
|
26
|
-
return `${blockId}:${actionId}`;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
// --- metadata ---
|
|
30
|
-
|
|
31
|
-
async getMetadata(blockId: BlockId): Promise<Uint8Array | undefined> {
|
|
32
|
-
return this.metadata.get(blockId);
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
async putMetadata(blockId: BlockId, value: Uint8Array): Promise<void> {
|
|
36
|
-
this.metadata.set(blockId, value);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
// --- revisions ---
|
|
40
|
-
|
|
41
|
-
async getRevision(blockId: BlockId, rev: number): Promise<Uint8Array | undefined> {
|
|
42
|
-
return this.revisions.get(this.revisionKey(blockId, rev));
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
async putRevision(blockId: BlockId, rev: number, value: Uint8Array): Promise<void> {
|
|
46
|
-
this.revisions.set(this.revisionKey(blockId, rev), value);
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
async *rangeRevisions(blockId: BlockId, lo: number, hi: number, reverse: boolean): AsyncIterable<[number, Uint8Array]> {
|
|
50
|
-
// Drain into an array before yielding (drain-before-yield contract): the
|
|
51
|
-
// consumer awaits between yields, and a Map has no live cursor to pin, but
|
|
52
|
-
// this keeps the memory driver's semantics identical to the native backends.
|
|
53
|
-
const results: [number, Uint8Array][] = [];
|
|
54
|
-
for (let rev = lo; rev <= hi; rev++) {
|
|
55
|
-
const value = this.revisions.get(this.revisionKey(blockId, rev));
|
|
56
|
-
if (value !== undefined) {
|
|
57
|
-
results.push([rev, value]);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
if (reverse) {
|
|
61
|
-
results.reverse();
|
|
62
|
-
}
|
|
63
|
-
for (const result of results) {
|
|
64
|
-
yield result;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// --- pending ---
|
|
69
|
-
|
|
70
|
-
async getPending(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
71
|
-
return this.pending.get(this.actionKey(blockId, actionId));
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
async putPending(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
75
|
-
this.pending.set(this.actionKey(blockId, actionId), value);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
async deletePending(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
79
|
-
this.pending.delete(this.actionKey(blockId, actionId));
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
async *listPendingActionIds(blockId: BlockId): AsyncIterable<ActionId> {
|
|
83
|
-
const prefix = `${blockId}:`;
|
|
84
|
-
// Snapshot before yielding (drain-before-yield): a concurrent putPending during
|
|
85
|
-
// the scan must not invalidate a live map iterator.
|
|
86
|
-
const ids: ActionId[] = [];
|
|
87
|
-
for (const key of Array.from(this.pending.keys())) {
|
|
88
|
-
if (key.startsWith(prefix)) {
|
|
89
|
-
ids.push(key.substring(prefix.length) as ActionId);
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
for (const id of ids) {
|
|
93
|
-
yield id;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
// --- transactions ---
|
|
98
|
-
|
|
99
|
-
async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
100
|
-
return this.transactions.get(this.actionKey(blockId, actionId));
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
async putTransaction(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
104
|
-
this.transactions.set(this.actionKey(blockId, actionId), value);
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
// --- materialized ---
|
|
108
|
-
|
|
109
|
-
async getMaterialized(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
110
|
-
return this.materialized.get(this.actionKey(blockId, actionId));
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
async putMaterialized(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
114
|
-
this.materialized.set(this.actionKey(blockId, actionId), value);
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
async deleteMaterialized(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
118
|
-
this.materialized.delete(this.actionKey(blockId, actionId));
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// --- promote (atomic move; synchronous Map ops make it indivisible here) ---
|
|
122
|
-
|
|
123
|
-
async promote(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
124
|
-
const key = this.actionKey(blockId, actionId);
|
|
125
|
-
const value = this.pending.get(key);
|
|
126
|
-
if (value === undefined) {
|
|
127
|
-
throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
|
|
128
|
-
}
|
|
129
|
-
this.transactions.set(key, value);
|
|
130
|
-
this.pending.delete(key);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// --- optional passthroughs ---
|
|
134
|
-
|
|
135
|
-
async *listBlockIds(): AsyncIterable<BlockId> {
|
|
136
|
-
// Snapshot the keys before yielding so a concurrent putMetadata during the scan
|
|
137
|
-
// doesn't invalidate a live map iterator. Fresh in-memory storage is empty, so at
|
|
138
|
-
// real process startup this yields nothing — it exists so the seed path can be
|
|
139
|
-
// unit-tested against a pre-populated store.
|
|
140
|
-
for (const blockId of Array.from(this.metadata.keys())) {
|
|
141
|
-
yield blockId;
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
async approximateBytesUsed(): Promise<number> {
|
|
146
|
-
let total = 0;
|
|
147
|
-
for (const [blockId, value] of this.metadata) {
|
|
148
|
-
total += blockId.length + value.byteLength;
|
|
149
|
-
}
|
|
150
|
-
for (const store of [this.revisions, this.pending, this.transactions, this.materialized]) {
|
|
151
|
-
for (const [key, value] of store) {
|
|
152
|
-
total += key.length + value.byteLength;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
return total;
|
|
156
|
-
}
|
|
157
|
-
}
|
|
1
|
+
import type { BlockId, ActionId } from "@optimystic/db-core";
|
|
2
|
+
import type { RawStoreDriver } from "./raw-store-driver.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* In-memory {@link RawStoreDriver}: the five logical block-storage stores as
|
|
6
|
+
* `Map`s of `Uint8Array` values. `KvRawStorage` hands this driver bytes produced
|
|
7
|
+
* by `JSON`-encode and reads them back via `JSON`-decode, so every get yields a
|
|
8
|
+
* fresh object and every save stored an independent byte snapshot BY
|
|
9
|
+
* CONSTRUCTION — the old `structuredClone`-on-every-get/put discipline is now
|
|
10
|
+
* structural, not a rule a maintainer has to remember. The driver therefore
|
|
11
|
+
* stores the byte reference directly: those bytes are never mutated by the
|
|
12
|
+
* kernel or exposed to callers as a mutable handle.
|
|
13
|
+
*/
|
|
14
|
+
export class MemoryStoreDriver implements RawStoreDriver {
|
|
15
|
+
private readonly metadata = new Map<BlockId, Uint8Array>();
|
|
16
|
+
private readonly revisions = new Map<string, Uint8Array>(); // `${blockId}:${rev}` -> actionId bytes
|
|
17
|
+
private readonly pending = new Map<string, Uint8Array>(); // `${blockId}:${actionId}` -> transform bytes
|
|
18
|
+
private readonly transactions = new Map<string, Uint8Array>();
|
|
19
|
+
private readonly materialized = new Map<string, Uint8Array>();
|
|
20
|
+
|
|
21
|
+
private revisionKey(blockId: BlockId, rev: number): string {
|
|
22
|
+
return `${blockId}:${rev}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
private actionKey(blockId: BlockId, actionId: ActionId): string {
|
|
26
|
+
return `${blockId}:${actionId}`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// --- metadata ---
|
|
30
|
+
|
|
31
|
+
async getMetadata(blockId: BlockId): Promise<Uint8Array | undefined> {
|
|
32
|
+
return this.metadata.get(blockId);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async putMetadata(blockId: BlockId, value: Uint8Array): Promise<void> {
|
|
36
|
+
this.metadata.set(blockId, value);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// --- revisions ---
|
|
40
|
+
|
|
41
|
+
async getRevision(blockId: BlockId, rev: number): Promise<Uint8Array | undefined> {
|
|
42
|
+
return this.revisions.get(this.revisionKey(blockId, rev));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async putRevision(blockId: BlockId, rev: number, value: Uint8Array): Promise<void> {
|
|
46
|
+
this.revisions.set(this.revisionKey(blockId, rev), value);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async *rangeRevisions(blockId: BlockId, lo: number, hi: number, reverse: boolean): AsyncIterable<[number, Uint8Array]> {
|
|
50
|
+
// Drain into an array before yielding (drain-before-yield contract): the
|
|
51
|
+
// consumer awaits between yields, and a Map has no live cursor to pin, but
|
|
52
|
+
// this keeps the memory driver's semantics identical to the native backends.
|
|
53
|
+
const results: [number, Uint8Array][] = [];
|
|
54
|
+
for (let rev = lo; rev <= hi; rev++) {
|
|
55
|
+
const value = this.revisions.get(this.revisionKey(blockId, rev));
|
|
56
|
+
if (value !== undefined) {
|
|
57
|
+
results.push([rev, value]);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (reverse) {
|
|
61
|
+
results.reverse();
|
|
62
|
+
}
|
|
63
|
+
for (const result of results) {
|
|
64
|
+
yield result;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// --- pending ---
|
|
69
|
+
|
|
70
|
+
async getPending(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
71
|
+
return this.pending.get(this.actionKey(blockId, actionId));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async putPending(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
75
|
+
this.pending.set(this.actionKey(blockId, actionId), value);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async deletePending(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
79
|
+
this.pending.delete(this.actionKey(blockId, actionId));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async *listPendingActionIds(blockId: BlockId): AsyncIterable<ActionId> {
|
|
83
|
+
const prefix = `${blockId}:`;
|
|
84
|
+
// Snapshot before yielding (drain-before-yield): a concurrent putPending during
|
|
85
|
+
// the scan must not invalidate a live map iterator.
|
|
86
|
+
const ids: ActionId[] = [];
|
|
87
|
+
for (const key of Array.from(this.pending.keys())) {
|
|
88
|
+
if (key.startsWith(prefix)) {
|
|
89
|
+
ids.push(key.substring(prefix.length) as ActionId);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
for (const id of ids) {
|
|
93
|
+
yield id;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// --- transactions ---
|
|
98
|
+
|
|
99
|
+
async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
100
|
+
return this.transactions.get(this.actionKey(blockId, actionId));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async putTransaction(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
104
|
+
this.transactions.set(this.actionKey(blockId, actionId), value);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// --- materialized ---
|
|
108
|
+
|
|
109
|
+
async getMaterialized(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
110
|
+
return this.materialized.get(this.actionKey(blockId, actionId));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async putMaterialized(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
114
|
+
this.materialized.set(this.actionKey(blockId, actionId), value);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async deleteMaterialized(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
118
|
+
this.materialized.delete(this.actionKey(blockId, actionId));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// --- promote (atomic move; synchronous Map ops make it indivisible here) ---
|
|
122
|
+
|
|
123
|
+
async promote(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
124
|
+
const key = this.actionKey(blockId, actionId);
|
|
125
|
+
const value = this.pending.get(key);
|
|
126
|
+
if (value === undefined) {
|
|
127
|
+
throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
|
|
128
|
+
}
|
|
129
|
+
this.transactions.set(key, value);
|
|
130
|
+
this.pending.delete(key);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// --- optional passthroughs ---
|
|
134
|
+
|
|
135
|
+
async *listBlockIds(): AsyncIterable<BlockId> {
|
|
136
|
+
// Snapshot the keys before yielding so a concurrent putMetadata during the scan
|
|
137
|
+
// doesn't invalidate a live map iterator. Fresh in-memory storage is empty, so at
|
|
138
|
+
// real process startup this yields nothing — it exists so the seed path can be
|
|
139
|
+
// unit-tested against a pre-populated store.
|
|
140
|
+
for (const blockId of Array.from(this.metadata.keys())) {
|
|
141
|
+
yield blockId;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async approximateBytesUsed(): Promise<number> {
|
|
146
|
+
let total = 0;
|
|
147
|
+
for (const [blockId, value] of this.metadata) {
|
|
148
|
+
total += blockId.length + value.byteLength;
|
|
149
|
+
}
|
|
150
|
+
for (const store of [this.revisions, this.pending, this.transactions, this.materialized]) {
|
|
151
|
+
for (const [key, value] of store) {
|
|
152
|
+
total += key.length + value.byteLength;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return total;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
@@ -1,42 +1,42 @@
|
|
|
1
|
-
import type { ActionId } from "@optimystic/db-core";
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Value codec shared by every `KvRawStorage`-backed store. Values cross the
|
|
5
|
-
* `RawStoreDriver` boundary as `Uint8Array`, so this module is the single place
|
|
6
|
-
* that turns `BlockMetadata`/`Transform`/`IBlock` (via JSON) and the `ActionId`
|
|
7
|
-
* string (via UTF-8) into bytes and back.
|
|
8
|
-
*
|
|
9
|
-
* Because a get always decodes fresh bytes into a NEW object and a save always
|
|
10
|
-
* encodes an independent byte copy, the clone-on-store / clone-on-read invariant
|
|
11
|
-
* that in-memory storage used to enforce by hand (`structuredClone`) is now
|
|
12
|
-
* structural for kernel-backed stores — see docs/internals.md "Storage Returns
|
|
13
|
-
* References".
|
|
14
|
-
*
|
|
15
|
-
* Round-trip fidelity matters for `BlockMetadata`: an open-ended `RevisionRange`
|
|
16
|
-
* is encoded `[E]` (one element, upper bound `undefined`). `JSON.stringify([5])`
|
|
17
|
-
* → `"[5]"` → `JSON.parse` → `[5]`, so the open-ended encoding survives byte-exact.
|
|
18
|
-
* Do NOT normalize ranges here.
|
|
19
|
-
*/
|
|
20
|
-
|
|
21
|
-
const encoder = new TextEncoder();
|
|
22
|
-
const decoder = new TextDecoder();
|
|
23
|
-
|
|
24
|
-
/** JSON-encode a value (`BlockMetadata` / `Transform` / `IBlock`) to UTF-8 bytes. */
|
|
25
|
-
export function encodeJson<T>(value: T): Uint8Array {
|
|
26
|
-
return encoder.encode(JSON.stringify(value));
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
/** Decode UTF-8 JSON bytes back into a value. Callers guard the miss (`undefined`) first. */
|
|
30
|
-
export function decodeJson<T>(bytes: Uint8Array): T {
|
|
31
|
-
return JSON.parse(decoder.decode(bytes)) as T;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/** Encode an `ActionId` string to UTF-8 bytes (stored as the revisions-store value). */
|
|
35
|
-
export function encodeActionId(actionId: ActionId): Uint8Array {
|
|
36
|
-
return encoder.encode(actionId);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/** Decode UTF-8 bytes back into an `ActionId` string. */
|
|
40
|
-
export function decodeActionId(bytes: Uint8Array): ActionId {
|
|
41
|
-
return decoder.decode(bytes) as ActionId;
|
|
42
|
-
}
|
|
1
|
+
import type { ActionId } from "@optimystic/db-core";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Value codec shared by every `KvRawStorage`-backed store. Values cross the
|
|
5
|
+
* `RawStoreDriver` boundary as `Uint8Array`, so this module is the single place
|
|
6
|
+
* that turns `BlockMetadata`/`Transform`/`IBlock` (via JSON) and the `ActionId`
|
|
7
|
+
* string (via UTF-8) into bytes and back.
|
|
8
|
+
*
|
|
9
|
+
* Because a get always decodes fresh bytes into a NEW object and a save always
|
|
10
|
+
* encodes an independent byte copy, the clone-on-store / clone-on-read invariant
|
|
11
|
+
* that in-memory storage used to enforce by hand (`structuredClone`) is now
|
|
12
|
+
* structural for kernel-backed stores — see docs/internals.md "Storage Returns
|
|
13
|
+
* References".
|
|
14
|
+
*
|
|
15
|
+
* Round-trip fidelity matters for `BlockMetadata`: an open-ended `RevisionRange`
|
|
16
|
+
* is encoded `[E]` (one element, upper bound `undefined`). `JSON.stringify([5])`
|
|
17
|
+
* → `"[5]"` → `JSON.parse` → `[5]`, so the open-ended encoding survives byte-exact.
|
|
18
|
+
* Do NOT normalize ranges here.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const encoder = new TextEncoder();
|
|
22
|
+
const decoder = new TextDecoder();
|
|
23
|
+
|
|
24
|
+
/** JSON-encode a value (`BlockMetadata` / `Transform` / `IBlock`) to UTF-8 bytes. */
|
|
25
|
+
export function encodeJson<T>(value: T): Uint8Array {
|
|
26
|
+
return encoder.encode(JSON.stringify(value));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Decode UTF-8 JSON bytes back into a value. Callers guard the miss (`undefined`) first. */
|
|
30
|
+
export function decodeJson<T>(bytes: Uint8Array): T {
|
|
31
|
+
return JSON.parse(decoder.decode(bytes)) as T;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Encode an `ActionId` string to UTF-8 bytes (stored as the revisions-store value). */
|
|
35
|
+
export function encodeActionId(actionId: ActionId): Uint8Array {
|
|
36
|
+
return encoder.encode(actionId);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Decode UTF-8 bytes back into an `ActionId` string. */
|
|
40
|
+
export function decodeActionId(bytes: Uint8Array): ActionId {
|
|
41
|
+
return decoder.decode(bytes) as ActionId;
|
|
42
|
+
}
|
|
@@ -1,80 +1,80 @@
|
|
|
1
|
-
import type { BlockId, ActionId } from "@optimystic/db-core";
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Bytes-valued, per-logical-store driver surface. Each backend implements the
|
|
5
|
-
* five block-storage stores (metadata, revisions, pending, transactions,
|
|
6
|
-
* materialized) over its native mechanism (LevelDB tag-ranges, five SQLite
|
|
7
|
-
* tables, five IndexedDB object stores, five filesystem subdirectories, five
|
|
8
|
-
* in-memory maps). `KvRawStorage` layers all JSON serialization and call
|
|
9
|
-
* orchestration on top — drivers never (de)serialize values and never see the
|
|
10
|
-
* `BlockMetadata`/`Transform`/`IBlock` types. Drivers speak only
|
|
11
|
-
* `Uint8Array`/`BlockId`/`ActionId`/`number`.
|
|
12
|
-
*
|
|
13
|
-
* ### Iteration semantics (drain-before-yield)
|
|
14
|
-
*
|
|
15
|
-
* `rangeRevisions` and `listPendingActionIds` return an `AsyncIterable`, but a
|
|
16
|
-
* driver MUST drain its native cursor/iterator into memory BEFORE yielding to
|
|
17
|
-
* the consumer. A live LevelDB iterator, IndexedDB transaction, or SQLite
|
|
18
|
-
* cursor must not straddle the consumer's `await`s: IndexedDB auto-commits an
|
|
19
|
-
* idle transaction, SQLite would hold its mutex slot, and LevelDB pins native
|
|
20
|
-
* resources. The kernel encodes this as a contract (not a shared implementation)
|
|
21
|
-
* because the drain is backend-specific. The conformance suite exercises it by
|
|
22
|
-
* interleaving other awaits between yielded items.
|
|
23
|
-
*
|
|
24
|
-
* ### Promote atomicity
|
|
25
|
-
*
|
|
26
|
-
* `promote` is the ONLY cross-key atomic operation the kernel requires. Every
|
|
27
|
-
* other write is a single put/delete. Each backend satisfies `promote` with its
|
|
28
|
-
* native atomic mechanism (LevelDB batch, SQLite transaction, IndexedDB
|
|
29
|
-
* readwrite transaction, filesystem rename); the kernel never assumes an
|
|
30
|
-
* atomicity a backend cannot deliver.
|
|
31
|
-
*/
|
|
32
|
-
export interface RawStoreDriver {
|
|
33
|
-
// metadata store — keyed by blockId
|
|
34
|
-
getMetadata(blockId: BlockId): Promise<Uint8Array | undefined>;
|
|
35
|
-
putMetadata(blockId: BlockId, value: Uint8Array): Promise<void>;
|
|
36
|
-
|
|
37
|
-
// revisions store — keyed by (blockId, rev), ORDERED BY rev
|
|
38
|
-
getRevision(blockId: BlockId, rev: number): Promise<Uint8Array | undefined>;
|
|
39
|
-
putRevision(blockId: BlockId, rev: number, value: Uint8Array): Promise<void>;
|
|
40
|
-
/**
|
|
41
|
-
* Yield `[rev, value]` for every present rev in `[lo, hi]` (both inclusive),
|
|
42
|
-
* ascending when `reverse` is false, descending when `reverse` is true. The
|
|
43
|
-
* driver MUST drain any native cursor into memory before yielding — see the
|
|
44
|
-
* "drain-before-yield" contract above.
|
|
45
|
-
*/
|
|
46
|
-
rangeRevisions(blockId: BlockId, lo: number, hi: number, reverse: boolean): AsyncIterable<[number, Uint8Array]>;
|
|
47
|
-
|
|
48
|
-
// pending store — keyed by (blockId, actionId)
|
|
49
|
-
getPending(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined>;
|
|
50
|
-
putPending(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void>;
|
|
51
|
-
deletePending(blockId: BlockId, actionId: ActionId): Promise<void>;
|
|
52
|
-
/** Yield each present pending actionId for the block. MUST drain before yielding (see above). */
|
|
53
|
-
listPendingActionIds(blockId: BlockId): AsyncIterable<ActionId>;
|
|
54
|
-
|
|
55
|
-
// transactions store — keyed by (blockId, actionId)
|
|
56
|
-
getTransaction(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined>;
|
|
57
|
-
putTransaction(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void>;
|
|
58
|
-
|
|
59
|
-
// materialized store — keyed by (blockId, actionId)
|
|
60
|
-
getMaterialized(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined>;
|
|
61
|
-
putMaterialized(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void>;
|
|
62
|
-
deleteMaterialized(blockId: BlockId, actionId: ActionId): Promise<void>;
|
|
63
|
-
|
|
64
|
-
/**
|
|
65
|
-
* Atomically move `pending(blockId, actionId)` → `transactions(blockId, actionId)`:
|
|
66
|
-
* write the transactions entry and remove the pending entry as one indivisible
|
|
67
|
-
* step (batch / DB transaction / rename). A crash must leave exactly one of the
|
|
68
|
-
* two states, never both/neither. Throw
|
|
69
|
-
* `Pending action <actionId> not found for block <blockId>` when no pending
|
|
70
|
-
* entry exists. This is the ONLY cross-key atomic operation the kernel requires.
|
|
71
|
-
*/
|
|
72
|
-
promote(blockId: BlockId, actionId: ActionId): Promise<void>;
|
|
73
|
-
|
|
74
|
-
/** Optional — enumerate block ids with durable metadata (startup seed). Passed through by the kernel. */
|
|
75
|
-
listBlockIds?(): AsyncIterable<BlockId>;
|
|
76
|
-
/** Optional — best cheap byte estimate. Passed through by the kernel. */
|
|
77
|
-
approximateBytesUsed?(): Promise<number>;
|
|
78
|
-
/** Optional — release the underlying handle. */
|
|
79
|
-
close?(): Promise<void>;
|
|
80
|
-
}
|
|
1
|
+
import type { BlockId, ActionId } from "@optimystic/db-core";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Bytes-valued, per-logical-store driver surface. Each backend implements the
|
|
5
|
+
* five block-storage stores (metadata, revisions, pending, transactions,
|
|
6
|
+
* materialized) over its native mechanism (LevelDB tag-ranges, five SQLite
|
|
7
|
+
* tables, five IndexedDB object stores, five filesystem subdirectories, five
|
|
8
|
+
* in-memory maps). `KvRawStorage` layers all JSON serialization and call
|
|
9
|
+
* orchestration on top — drivers never (de)serialize values and never see the
|
|
10
|
+
* `BlockMetadata`/`Transform`/`IBlock` types. Drivers speak only
|
|
11
|
+
* `Uint8Array`/`BlockId`/`ActionId`/`number`.
|
|
12
|
+
*
|
|
13
|
+
* ### Iteration semantics (drain-before-yield)
|
|
14
|
+
*
|
|
15
|
+
* `rangeRevisions` and `listPendingActionIds` return an `AsyncIterable`, but a
|
|
16
|
+
* driver MUST drain its native cursor/iterator into memory BEFORE yielding to
|
|
17
|
+
* the consumer. A live LevelDB iterator, IndexedDB transaction, or SQLite
|
|
18
|
+
* cursor must not straddle the consumer's `await`s: IndexedDB auto-commits an
|
|
19
|
+
* idle transaction, SQLite would hold its mutex slot, and LevelDB pins native
|
|
20
|
+
* resources. The kernel encodes this as a contract (not a shared implementation)
|
|
21
|
+
* because the drain is backend-specific. The conformance suite exercises it by
|
|
22
|
+
* interleaving other awaits between yielded items.
|
|
23
|
+
*
|
|
24
|
+
* ### Promote atomicity
|
|
25
|
+
*
|
|
26
|
+
* `promote` is the ONLY cross-key atomic operation the kernel requires. Every
|
|
27
|
+
* other write is a single put/delete. Each backend satisfies `promote` with its
|
|
28
|
+
* native atomic mechanism (LevelDB batch, SQLite transaction, IndexedDB
|
|
29
|
+
* readwrite transaction, filesystem rename); the kernel never assumes an
|
|
30
|
+
* atomicity a backend cannot deliver.
|
|
31
|
+
*/
|
|
32
|
+
export interface RawStoreDriver {
|
|
33
|
+
// metadata store — keyed by blockId
|
|
34
|
+
getMetadata(blockId: BlockId): Promise<Uint8Array | undefined>;
|
|
35
|
+
putMetadata(blockId: BlockId, value: Uint8Array): Promise<void>;
|
|
36
|
+
|
|
37
|
+
// revisions store — keyed by (blockId, rev), ORDERED BY rev
|
|
38
|
+
getRevision(blockId: BlockId, rev: number): Promise<Uint8Array | undefined>;
|
|
39
|
+
putRevision(blockId: BlockId, rev: number, value: Uint8Array): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Yield `[rev, value]` for every present rev in `[lo, hi]` (both inclusive),
|
|
42
|
+
* ascending when `reverse` is false, descending when `reverse` is true. The
|
|
43
|
+
* driver MUST drain any native cursor into memory before yielding — see the
|
|
44
|
+
* "drain-before-yield" contract above.
|
|
45
|
+
*/
|
|
46
|
+
rangeRevisions(blockId: BlockId, lo: number, hi: number, reverse: boolean): AsyncIterable<[number, Uint8Array]>;
|
|
47
|
+
|
|
48
|
+
// pending store — keyed by (blockId, actionId)
|
|
49
|
+
getPending(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined>;
|
|
50
|
+
putPending(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void>;
|
|
51
|
+
deletePending(blockId: BlockId, actionId: ActionId): Promise<void>;
|
|
52
|
+
/** Yield each present pending actionId for the block. MUST drain before yielding (see above). */
|
|
53
|
+
listPendingActionIds(blockId: BlockId): AsyncIterable<ActionId>;
|
|
54
|
+
|
|
55
|
+
// transactions store — keyed by (blockId, actionId)
|
|
56
|
+
getTransaction(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined>;
|
|
57
|
+
putTransaction(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void>;
|
|
58
|
+
|
|
59
|
+
// materialized store — keyed by (blockId, actionId)
|
|
60
|
+
getMaterialized(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined>;
|
|
61
|
+
putMaterialized(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void>;
|
|
62
|
+
deleteMaterialized(blockId: BlockId, actionId: ActionId): Promise<void>;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Atomically move `pending(blockId, actionId)` → `transactions(blockId, actionId)`:
|
|
66
|
+
* write the transactions entry and remove the pending entry as one indivisible
|
|
67
|
+
* step (batch / DB transaction / rename). A crash must leave exactly one of the
|
|
68
|
+
* two states, never both/neither. Throw
|
|
69
|
+
* `Pending action <actionId> not found for block <blockId>` when no pending
|
|
70
|
+
* entry exists. This is the ONLY cross-key atomic operation the kernel requires.
|
|
71
|
+
*/
|
|
72
|
+
promote(blockId: BlockId, actionId: ActionId): Promise<void>;
|
|
73
|
+
|
|
74
|
+
/** Optional — enumerate block ids with durable metadata (startup seed). Passed through by the kernel. */
|
|
75
|
+
listBlockIds?(): AsyncIterable<BlockId>;
|
|
76
|
+
/** Optional — best cheap byte estimate. Passed through by the kernel. */
|
|
77
|
+
approximateBytesUsed?(): Promise<number>;
|
|
78
|
+
/** Optional — release the underlying handle. */
|
|
79
|
+
close?(): Promise<void>;
|
|
80
|
+
}
|