@peerbit/shared-log 13.2.3 → 13.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +12 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +18 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +477 -13
- package/dist/src/index.js.map +1 -1
- package/package.json +14 -13
- package/src/exchange-heads.ts +13 -0
- package/src/index.ts +549 -15
package/dist/src/index.js
CHANGED
|
@@ -36,7 +36,7 @@ import { BorshError, deserialize, field, serialize, variant, } from "@dao-xyz/bo
|
|
|
36
36
|
import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
|
|
37
37
|
import { cidifyString } from "@peerbit/blocks-interface";
|
|
38
38
|
import { Cache } from "@peerbit/cache";
|
|
39
|
-
import { AccessError, Ed25519Keypair, Ed25519PublicKey, PublicSignKey, Secp256k1PublicKey, getPublicKeyFromPeerId, sha256Base64Sync, sha256Sync, } from "@peerbit/crypto";
|
|
39
|
+
import { AccessError, Ed25519Keypair, Ed25519PublicKey, PublicSignKey, Secp256k1PublicKey, getPublicKeyFromPeerId, sha256Base64Sync, sha256Sync, toHexString, } from "@peerbit/crypto";
|
|
40
40
|
import { And, ByteMatchQuery, NotStartedError as IndexNotStartedError, Or, Sort, StringMatch, toId, } from "@peerbit/indexer-interface";
|
|
41
41
|
import { Entry, EntryType, LamportClock, Log, Meta, ShallowEntry, ShallowMeta, Timestamp, verifyEd25519Batch, verifyEntryV0Ed25519BatchFromEntries, } from "@peerbit/log";
|
|
42
42
|
import { logger as loggerFn } from "@peerbit/logger";
|
|
@@ -120,6 +120,295 @@ const canUseOptionalNativeModuleImports = () => {
|
|
|
120
120
|
!!scope.registration &&
|
|
121
121
|
typeof scope.skipWaiting === "function"));
|
|
122
122
|
};
|
|
123
|
+
/**
|
|
124
|
+
* Build the per-program coordinate persistence directory under a node's
|
|
125
|
+
* durable storage root: `<nodeDirectory>/coordinates/<fsSafeLogId>`. Uses
|
|
126
|
+
* forward-slash joining (accepted by Node's `fs` on every platform) so
|
|
127
|
+
* shared-log does not need to statically import `node:path`, which is not
|
|
128
|
+
* available in browser bundles. Trailing separators on the root are trimmed
|
|
129
|
+
* to avoid doubled slashes.
|
|
130
|
+
*/
|
|
131
|
+
const joinNativeCoordinateDirectory = (nodeDirectory, fsSafeLogId) => `${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
|
|
132
|
+
/**
|
|
133
|
+
* Write-through block store bridging the native backbone's in-wasm-memory block
|
|
134
|
+
* store to a durable per-program {@link AnyBlockStore}.
|
|
135
|
+
*
|
|
136
|
+
* WHY: when the native backbone is active the log's entry blocks live only in
|
|
137
|
+
* the native wasm block map (`NativeBackboneBlockStore.persisted() === false`).
|
|
138
|
+
* On a restart that map is empty, so the native graph cannot reload heads the
|
|
139
|
+
* durable heads index still lists ("Failed to load entry from head"). This
|
|
140
|
+
* wrapper mirrors every write into the same durable `blocks` sublevel the
|
|
141
|
+
* non-native path uses. On a native miss, reads fall through to durable storage
|
|
142
|
+
* and lazily repopulate the wasm map so the native graph can walk the DAG again.
|
|
143
|
+
*
|
|
144
|
+
* The native store stays the authoritative hot store the native graph reads
|
|
145
|
+
* from: reads hit native first and only fall back to durable (repopulating
|
|
146
|
+
* native on a hit so subsequent native-graph reads succeed).
|
|
147
|
+
*
|
|
148
|
+
* METHOD SURFACE (see #1006): `RemoteBlocks` and the log feature-detect the
|
|
149
|
+
* optional batch methods (`putMany`/`putKnown`/`putKnownMany`/
|
|
150
|
+
* `putKnownManyColumns`/`rmMany`). To keep the receive-fusion / columnar fast
|
|
151
|
+
* paths engaged this wrapper exposes EXACTLY the methods the native store
|
|
152
|
+
* exposes — including `putKnownManyColumns`, which `AnyBlockStore` does not have
|
|
153
|
+
* — and delegates each. It deliberately does not add methods the native store
|
|
154
|
+
* lacks (`getBlockResponsePayload`/`getNativeLogBlockStoreHandle` stay absent so
|
|
155
|
+
* the optional-chained probes in `RemoteBlocks` behave exactly as today).
|
|
156
|
+
*/
|
|
157
|
+
class NativeBackboneWriteThroughBlockStore {
|
|
158
|
+
native;
|
|
159
|
+
durable;
|
|
160
|
+
constructor(native, durable) {
|
|
161
|
+
this.native = native;
|
|
162
|
+
this.durable = durable;
|
|
163
|
+
}
|
|
164
|
+
// A durable mirror write that cannot be awaited at its call site (the
|
|
165
|
+
// columnar putKnownManyColumns fast path must return a synchronous string[]
|
|
166
|
+
// because RemoteBlocks.putKnownManyColumns treats the result as sync) is
|
|
167
|
+
// tracked here instead of being silently `void`ed. Its rejection is stored
|
|
168
|
+
// and re-thrown on the next awaited wrapper method (and on stop()), so a
|
|
169
|
+
// failed durable write (IO/disk-full) surfaces as an error rather than
|
|
170
|
+
// vanishing and leaving the block out of durable while native/log report
|
|
171
|
+
// success. `.catch` also prevents unhandled-rejection noise.
|
|
172
|
+
pendingDurableWrites = new Set();
|
|
173
|
+
durableWriteError;
|
|
174
|
+
trackDurable(result) {
|
|
175
|
+
// The durable store may answer synchronously (an in-memory or already
|
|
176
|
+
// resolved store); only a real pending promise needs tracking. A sync
|
|
177
|
+
// success is already durable; a sync throw would have propagated already.
|
|
178
|
+
if (!isPromiseLike(result)) {
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
const tracked = Promise.resolve(result).then(() => {
|
|
182
|
+
this.pendingDurableWrites.delete(tracked);
|
|
183
|
+
}, (error) => {
|
|
184
|
+
this.pendingDurableWrites.delete(tracked);
|
|
185
|
+
if (this.durableWriteError === undefined) {
|
|
186
|
+
this.durableWriteError = error;
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
this.pendingDurableWrites.add(tracked);
|
|
190
|
+
}
|
|
191
|
+
// Wait for every tracked (sync-path) durable write to settle, then surface
|
|
192
|
+
// the first failure. Awaited methods call this so a prior columnar durable
|
|
193
|
+
// failure propagates as back-pressure to the next caller.
|
|
194
|
+
async drainDurable() {
|
|
195
|
+
while (this.pendingDurableWrites.size > 0) {
|
|
196
|
+
await Promise.allSettled([...this.pendingDurableWrites]);
|
|
197
|
+
}
|
|
198
|
+
if (this.durableWriteError !== undefined) {
|
|
199
|
+
const error = this.durableWriteError;
|
|
200
|
+
this.durableWriteError = undefined;
|
|
201
|
+
throw error;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
// --- lifecycle -------------------------------------------------------
|
|
205
|
+
// The native store's lifecycle hooks are no-ops; only the durable store
|
|
206
|
+
// needs starting/stopping. The wasm map is NOT eagerly rehydrated from disk:
|
|
207
|
+
// entry blocks are pulled back lazily on demand through the read fallback in
|
|
208
|
+
// getMany()/get() (durable hit -> repopulate the wasm map), which is what the
|
|
209
|
+
// log's DAG walk (EntryIndex.resolveMany -> store.getMany) exercises. Keeping
|
|
210
|
+
// the wasm map cold on open is required by the strict-native resident
|
|
211
|
+
// coordinate-state optimization: a reopened non-replicating native node must
|
|
212
|
+
// report hasBlock(head) === false and answer a same-signer append from the
|
|
213
|
+
// persisted coordinate + signer facts without resolving the entry block.
|
|
214
|
+
async start() {
|
|
215
|
+
await this.durable.start();
|
|
216
|
+
}
|
|
217
|
+
async stop() {
|
|
218
|
+
// Surface any tracked (sync-path) durable write failure and ensure all
|
|
219
|
+
// mirror writes have settled before the durable store is torn down.
|
|
220
|
+
await this.drainDurable();
|
|
221
|
+
await this.durable.stop();
|
|
222
|
+
}
|
|
223
|
+
status() {
|
|
224
|
+
return this.durable.status();
|
|
225
|
+
}
|
|
226
|
+
waitFor() {
|
|
227
|
+
return Promise.resolve([]);
|
|
228
|
+
}
|
|
229
|
+
// Mirror a single already-committed block (present in the wasm map) to the
|
|
230
|
+
// durable store ONLY. Used by the native commit-only append fast path: the
|
|
231
|
+
// native prepare commits the entry block into the wasm map and returns no raw
|
|
232
|
+
// bytes, and the strict-native resident-coordinate path deliberately does NOT
|
|
233
|
+
// route the block through the log's finishBlocks/putKnown* (that would disturb
|
|
234
|
+
// the commit-only append path the RCS optimization depends on). Instead the
|
|
235
|
+
// caller reads the committed bytes back and calls this so the block lands in
|
|
236
|
+
// durable directly. Like putKnownManyColumns' durable mirror, the write is
|
|
237
|
+
// tracked (not fire-and-forget) so an IO/disk-full failure surfaces on the
|
|
238
|
+
// next awaited wrapper method / stop() rather than being swallowed.
|
|
239
|
+
mirrorToDurable(cid, bytes) {
|
|
240
|
+
this.trackDurable(this.durable.putKnown(cid, bytes));
|
|
241
|
+
}
|
|
242
|
+
// --- writes (apply to BOTH: native first for the hot path, then durable) ---
|
|
243
|
+
async put(data) {
|
|
244
|
+
await this.drainDurable();
|
|
245
|
+
const cid = await this.native.put(data);
|
|
246
|
+
// The native store computes a raw-codec CID for a `Uint8Array`, storing
|
|
247
|
+
// the bytes verbatim (raw codec is identity), and stores `block.bytes`
|
|
248
|
+
// for the pre-CIDed object form. Either way the input bytes match what
|
|
249
|
+
// native stored, so feed durable the known cid+bytes without recomputing.
|
|
250
|
+
const value = data instanceof Uint8Array
|
|
251
|
+
? data
|
|
252
|
+
: data.block.bytes;
|
|
253
|
+
await this.durable.putKnown(cid, value);
|
|
254
|
+
return cid;
|
|
255
|
+
}
|
|
256
|
+
async putMany(blocks) {
|
|
257
|
+
await this.drainDurable();
|
|
258
|
+
const cids = await this.native.putMany(blocks);
|
|
259
|
+
const durableBlocks = cids.map((cid, index) => {
|
|
260
|
+
const block = blocks[index];
|
|
261
|
+
const value = block instanceof Uint8Array
|
|
262
|
+
? block
|
|
263
|
+
: block.block.bytes;
|
|
264
|
+
return [cid, value];
|
|
265
|
+
});
|
|
266
|
+
await this.durable.putKnownMany(durableBlocks);
|
|
267
|
+
return cids;
|
|
268
|
+
}
|
|
269
|
+
// Native put is synchronous (the authoritative hot store); the durable mirror
|
|
270
|
+
// is awaited so the returned promise resolves only after BOTH native and
|
|
271
|
+
// durable succeed and a durable IO/disk-full failure rejects here instead of
|
|
272
|
+
// being swallowed. RemoteBlocks.putKnown and the log's putKnownEntryBytesBatch
|
|
273
|
+
// both await this method, so returning a promise is compatible.
|
|
274
|
+
async putKnown(cid, bytes) {
|
|
275
|
+
await this.drainDurable();
|
|
276
|
+
const stored = this.native.putKnown(cid, bytes);
|
|
277
|
+
await this.durable.putKnown(cid, bytes);
|
|
278
|
+
return stored;
|
|
279
|
+
}
|
|
280
|
+
async putKnownMany(blocks) {
|
|
281
|
+
await this.drainDurable();
|
|
282
|
+
const cids = this.native.putKnownMany(blocks);
|
|
283
|
+
await this.durable.putKnownMany(blocks);
|
|
284
|
+
return cids;
|
|
285
|
+
}
|
|
286
|
+
putKnownManyColumns(cids, bytes) {
|
|
287
|
+
const stored = this.native.putKnownManyColumns(cids, bytes);
|
|
288
|
+
// AnyBlockStore has no columnar method; mirror via putKnownMany, which
|
|
289
|
+
// takes [cid, bytes] tuples and hits the same batched store path.
|
|
290
|
+
// This method must return a synchronous string[] (RemoteBlocks.putKnownManyColumns
|
|
291
|
+
// consumes the result synchronously), so the durable write cannot be awaited
|
|
292
|
+
// inline. Track it instead of `void`ing it so a durable rejection surfaces
|
|
293
|
+
// on the next awaited wrapper method / stop() rather than being swallowed.
|
|
294
|
+
const durableBlocks = cids.map((cid, index) => [cid, bytes[index]]);
|
|
295
|
+
this.trackDurable(this.durable.putKnownMany(durableBlocks));
|
|
296
|
+
return stored;
|
|
297
|
+
}
|
|
298
|
+
// --- reads (native/wasm first; on miss, durable fallback + repopulate) ---
|
|
299
|
+
// RemoteBlocks.get awaits this single-get, so on a native miss consult the
|
|
300
|
+
// durable store (like getMany does) and repopulate the native map, rather
|
|
301
|
+
// than returning undefined and only scheduling a background repopulate. That
|
|
302
|
+
// avoids a spurious miss (which would otherwise fall through to a remote
|
|
303
|
+
// read) for a block that is present on disk.
|
|
304
|
+
async get(cid, _options) {
|
|
305
|
+
const local = this.native.get(cid);
|
|
306
|
+
if (local != null) {
|
|
307
|
+
return local;
|
|
308
|
+
}
|
|
309
|
+
const durableValue = await this.durable.get(cid);
|
|
310
|
+
if (durableValue != null) {
|
|
311
|
+
// Repopulate the native map so the native graph reads it next time.
|
|
312
|
+
this.native.putKnownManyColumns([cid], [durableValue]);
|
|
313
|
+
return durableValue;
|
|
314
|
+
}
|
|
315
|
+
return undefined;
|
|
316
|
+
}
|
|
317
|
+
async getMany(cids) {
|
|
318
|
+
const results = await this.native.getMany(cids);
|
|
319
|
+
const missing = [];
|
|
320
|
+
for (let i = 0; i < results.length; i++) {
|
|
321
|
+
if (results[i] == null) {
|
|
322
|
+
missing.push(cids[i]);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
if (missing.length === 0) {
|
|
326
|
+
return results;
|
|
327
|
+
}
|
|
328
|
+
const durableValues = await this.durable.getMany(missing);
|
|
329
|
+
const repopulateCids = [];
|
|
330
|
+
const repopulateBytes = [];
|
|
331
|
+
let missingIndex = 0;
|
|
332
|
+
for (let i = 0; i < results.length; i++) {
|
|
333
|
+
if (results[i] != null) {
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
const value = durableValues[missingIndex++];
|
|
337
|
+
if (value != null) {
|
|
338
|
+
results[i] = value;
|
|
339
|
+
repopulateCids.push(cids[i]);
|
|
340
|
+
repopulateBytes.push(value);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
if (repopulateCids.length > 0) {
|
|
344
|
+
// Repopulate the native map so the native graph sees these blocks.
|
|
345
|
+
this.native.putKnownManyColumns(repopulateCids, repopulateBytes);
|
|
346
|
+
}
|
|
347
|
+
return results;
|
|
348
|
+
}
|
|
349
|
+
async has(cid) {
|
|
350
|
+
if (this.native.has(cid)) {
|
|
351
|
+
return true;
|
|
352
|
+
}
|
|
353
|
+
// Mirror getMany/hasMany: a block absent from the native wasm map may still
|
|
354
|
+
// be present in the durable store (e.g. persisted on disk but not yet
|
|
355
|
+
// repopulated into wasm). Consult durable on a native miss so presence
|
|
356
|
+
// checks agree with the resolves that getMany/hasMany already durable-fall
|
|
357
|
+
// back on. `Blocks.has` is declared `MaybePromise<boolean>`, so returning a
|
|
358
|
+
// promise here is contract-compatible.
|
|
359
|
+
return this.durable.has(cid);
|
|
360
|
+
}
|
|
361
|
+
async hasMany(cids) {
|
|
362
|
+
const nativeHas = await this.native.hasMany(cids);
|
|
363
|
+
const missing = [];
|
|
364
|
+
for (let i = 0; i < nativeHas.length; i++) {
|
|
365
|
+
if (!nativeHas[i]) {
|
|
366
|
+
missing.push(cids[i]);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
if (missing.length === 0) {
|
|
370
|
+
return nativeHas;
|
|
371
|
+
}
|
|
372
|
+
const durableHas = await this.durable.hasMany(missing);
|
|
373
|
+
let missingIndex = 0;
|
|
374
|
+
for (let i = 0; i < nativeHas.length; i++) {
|
|
375
|
+
if (!nativeHas[i]) {
|
|
376
|
+
nativeHas[i] = durableHas[missingIndex++];
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
return nativeHas;
|
|
380
|
+
}
|
|
381
|
+
// --- removes (apply to BOTH) ----------------------------------------
|
|
382
|
+
// Native rm is synchronous; the durable rm is awaited so the returned promise
|
|
383
|
+
// resolves only after both succeed and a durable failure rejects here rather
|
|
384
|
+
// than being swallowed. All rm callers (RemoteBlocks.rm, the log) await it.
|
|
385
|
+
async rm(cid) {
|
|
386
|
+
await this.drainDurable();
|
|
387
|
+
this.native.rm(cid);
|
|
388
|
+
await this.durable.rm(cid);
|
|
389
|
+
}
|
|
390
|
+
del(cid) {
|
|
391
|
+
void this.rm(cid);
|
|
392
|
+
}
|
|
393
|
+
async rmMany(cids) {
|
|
394
|
+
await this.drainDurable();
|
|
395
|
+
const removed = await this.native.rmMany(cids);
|
|
396
|
+
await this.durable.rmMany(cids);
|
|
397
|
+
return removed;
|
|
398
|
+
}
|
|
399
|
+
// --- misc ------------------------------------------------------------
|
|
400
|
+
async *iterator() {
|
|
401
|
+
yield* this.native.iterator();
|
|
402
|
+
}
|
|
403
|
+
size() {
|
|
404
|
+
return this.native.size();
|
|
405
|
+
}
|
|
406
|
+
persisted() {
|
|
407
|
+
// The blocks are now mirrored to a durable store, so report persisted so
|
|
408
|
+
// callers that gate durable-only behavior on this flag behave correctly.
|
|
409
|
+
return true;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
123
412
|
const countTruthyValues = (values) => {
|
|
124
413
|
if (!values) {
|
|
125
414
|
return 0;
|
|
@@ -4017,6 +4306,20 @@ let SharedLog = (() => {
|
|
|
4017
4306
|
return undefined;
|
|
4018
4307
|
}
|
|
4019
4308
|
const backbone = this._nativeBackbone;
|
|
4309
|
+
// When the durable write-through wrapper is active the log's block store
|
|
4310
|
+
// (this.remoteBlocks.localStore) is the wrapper, NOT the raw wasm block map
|
|
4311
|
+
// (backbone.blocks), so this comparison is false. In that case the block
|
|
4312
|
+
// must be mirrored to durable directly; see the guarded handling in the
|
|
4313
|
+
// prepare callback below.
|
|
4314
|
+
const durableWrapperActive = this.remoteBlocks?.localStore !== backbone.blocks;
|
|
4315
|
+
// The write-through wrapper instance, captured only when active, so the
|
|
4316
|
+
// commit-only block can be mirrored to durable WITHOUT routing it through
|
|
4317
|
+
// the log's finishBlocks/putKnown* (which would disturb the strict-native
|
|
4318
|
+
// resident-coordinate append path). `mirrorToDurable` writes to the durable
|
|
4319
|
+
// side only and tracks the write for error-surfacing.
|
|
4320
|
+
const durableWrapper = durableWrapperActive
|
|
4321
|
+
? this.remoteBlocks?.localStore
|
|
4322
|
+
: undefined;
|
|
4020
4323
|
let nativeBackboneDocumentIndexCommitted = false;
|
|
4021
4324
|
let committedNativeBackboneDocumentIndex;
|
|
4022
4325
|
const nativeCommitProperties = {
|
|
@@ -4065,9 +4368,38 @@ let SharedLog = (() => {
|
|
|
4065
4368
|
}
|
|
4066
4369
|
: {}),
|
|
4067
4370
|
}, backbone.blocks);
|
|
4068
|
-
if (prepared &&
|
|
4069
|
-
|
|
4070
|
-
|
|
4371
|
+
if (prepared && !prepared.bytes) {
|
|
4372
|
+
if (durableWrapper) {
|
|
4373
|
+
// The durable write-through wrapper is active, so the block store
|
|
4374
|
+
// the log writes through (this.remoteBlocks.localStore) is the
|
|
4375
|
+
// wrapper, NOT the raw wasm block map. prepareEntryV0PlainEntryCommit
|
|
4376
|
+
// committed the block into the wasm map ONLY and returned no raw
|
|
4377
|
+
// bytes, so on its own the block would never reach durable (log's
|
|
4378
|
+
// finishBlocks only calls putKnown* when prepared.bytes is set) and
|
|
4379
|
+
// a non-replicating native node would lose it on restart.
|
|
4380
|
+
//
|
|
4381
|
+
// Mirror the just-committed block (read back from the wasm store)
|
|
4382
|
+
// straight to the DURABLE side of the wrapper. Crucially we do NOT
|
|
4383
|
+
// attach prepared.bytes: doing so would make the log's finishBlocks
|
|
4384
|
+
// call putKnown*, which changes the commit-only append path and
|
|
4385
|
+
// breaks the strict-native resident-coordinate optimization (the
|
|
4386
|
+
// reopen tests assert the append stays native and resolves no entry
|
|
4387
|
+
// block). Instead the prepared result is returned exactly as the
|
|
4388
|
+
// memory-only branch below (getBytes only, no bytes), so the log's
|
|
4389
|
+
// finishBlocks path is UNCHANGED, and the block is made durable
|
|
4390
|
+
// out-of-band here. mirrorToDurable tracks the write so an IO/
|
|
4391
|
+
// disk-full failure surfaces on a later awaited wrapper method.
|
|
4392
|
+
const preparedHash = prepared.cid ?? prepared.hash;
|
|
4393
|
+
const committedBytes = preparedHash
|
|
4394
|
+
? backbone.blocks.get(preparedHash)
|
|
4395
|
+
: undefined;
|
|
4396
|
+
if (committedBytes && durableWrapper.mirrorToDurable) {
|
|
4397
|
+
durableWrapper.mirrorToDurable(preparedHash, committedBytes);
|
|
4398
|
+
}
|
|
4399
|
+
}
|
|
4400
|
+
// The block lives in the wasm map; expose getBytes for materialization
|
|
4401
|
+
// (identical to the memory-only native node path). When the durable
|
|
4402
|
+
// wrapper is active the durable copy was mirrored just above.
|
|
4071
4403
|
return {
|
|
4072
4404
|
...prepared,
|
|
4073
4405
|
getBytes: (hash) => backbone.blocks.get(hash),
|
|
@@ -4134,7 +4466,22 @@ let SharedLog = (() => {
|
|
|
4134
4466
|
return mapMaybePromise(this.createLeaderSelectionContext(), (context) => {
|
|
4135
4467
|
const nativeLeaderOptions = this.createNativeLeaderOptions(context);
|
|
4136
4468
|
let backboneAppend;
|
|
4137
|
-
|
|
4469
|
+
// The write-through durable wrapper, when active, is captured here so a
|
|
4470
|
+
// just-committed block can be mirrored to durable without disturbing the
|
|
4471
|
+
// strict-native commit path. When the wrapper is active the log's block
|
|
4472
|
+
// store is the wrapper (not the raw wasm map), so `localStore ===
|
|
4473
|
+
// backbone.blocks` is false — but the store is still native-backed, so we
|
|
4474
|
+
// must commit blocks in the backbone (the committed native prepare variant
|
|
4475
|
+
// is the one that emits the document-signer journal record; the
|
|
4476
|
+
// block-deferring variant does not, which would otherwise leave
|
|
4477
|
+
// document-signers.wal unwritten and break same-signer facts after
|
|
4478
|
+
// reopen). The block is then mirrored to durable out-of-band below.
|
|
4479
|
+
const durableWrapperActive = this.remoteBlocks?.localStore !== backbone.blocks;
|
|
4480
|
+
const durableWrapper = durableWrapperActive
|
|
4481
|
+
? this.remoteBlocks?.localStore
|
|
4482
|
+
: undefined;
|
|
4483
|
+
const commitBlocksInBackbone = this.remoteBlocks?.localStore === backbone.blocks ||
|
|
4484
|
+
durableWrapperActive;
|
|
4138
4485
|
let nativeBackboneDocumentIndexCommitted = false;
|
|
4139
4486
|
let nativeBackboneDocumentDeleteCommitted = false;
|
|
4140
4487
|
let committedNativeBackboneDocumentIndex;
|
|
@@ -4226,6 +4573,22 @@ let SharedLog = (() => {
|
|
|
4226
4573
|
nativeBackboneDocumentDeleteCommitted =
|
|
4227
4574
|
!!nativeBackboneDocumentDeleteKey;
|
|
4228
4575
|
const useTrimmedHashesOnly = properties?.resolveTrimmedEntries === false;
|
|
4576
|
+
if (durableWrapper?.mirrorToDurable) {
|
|
4577
|
+
// The block was committed into the wasm map (commitBlocksInBackbone is
|
|
4578
|
+
// true) but not into durable, because the log's finishBlocks path is
|
|
4579
|
+
// left UNCHANGED for strict-native mode (getBytes only, no bytes, so
|
|
4580
|
+
// no putKnown* through the wrapper). Mirror the just-committed block
|
|
4581
|
+
// straight to the durable side so a non-replicating native node keeps
|
|
4582
|
+
// it across a restart. mirrorToDurable tracks the write for
|
|
4583
|
+
// error-surfacing.
|
|
4584
|
+
const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
4585
|
+
const committedBytes = committedHash
|
|
4586
|
+
? backbone.blocks.get(committedHash)
|
|
4587
|
+
: undefined;
|
|
4588
|
+
if (committedBytes) {
|
|
4589
|
+
durableWrapper.mirrorToDurable(committedHash, committedBytes);
|
|
4590
|
+
}
|
|
4591
|
+
}
|
|
4229
4592
|
return {
|
|
4230
4593
|
...backboneAppend.entry,
|
|
4231
4594
|
gid: backboneAppend.coordinate.gid,
|
|
@@ -4586,11 +4949,21 @@ let SharedLog = (() => {
|
|
|
4586
4949
|
options?.target !== "none" ||
|
|
4587
4950
|
options?.replicate === true ||
|
|
4588
4951
|
(options?.delivery !== undefined && options.delivery !== false) ||
|
|
4589
|
-
this.remoteBlocks?.localStore !== backbone.blocks ||
|
|
4590
4952
|
!this.canUseNativeBackboneResidentCoordinateState() ||
|
|
4591
4953
|
properties?.nexts?.some((nexts) => nexts.length > 0)) {
|
|
4592
4954
|
return undefined;
|
|
4593
4955
|
}
|
|
4956
|
+
// When the durable write-through wrapper is active the log's block store is
|
|
4957
|
+
// the wrapper, not the raw wasm map, so `localStore === backbone.blocks` is
|
|
4958
|
+
// false. This batch path always commits blocks in the backbone (the
|
|
4959
|
+
// committed native batch prepare variants), so it is safe with the wrapper:
|
|
4960
|
+
// the blocks land in wasm and are mirrored to durable per-entry below. This
|
|
4961
|
+
// preserves the resident-coordinate fast batch path (and its meta.next
|
|
4962
|
+
// linking) after a reopen instead of bailing to a slow generic path.
|
|
4963
|
+
const durableWrapperActive = this.remoteBlocks?.localStore !== backbone.blocks;
|
|
4964
|
+
const durableWrapper = durableWrapperActive
|
|
4965
|
+
? this.remoteBlocks?.localStore
|
|
4966
|
+
: undefined;
|
|
4594
4967
|
const usesLatestDocumentContext = documentIndexes.every((index) => index.useLatestContext === true);
|
|
4595
4968
|
if (!usesLatestDocumentContext &&
|
|
4596
4969
|
documentIndexes.some((index) => index.useLatestContext === true)) {
|
|
@@ -4684,6 +5057,20 @@ let SharedLog = (() => {
|
|
|
4684
5057
|
for (let i = 0; i < appended.appendFacts.length; i++) {
|
|
4685
5058
|
const facts = appended.appendFacts[i];
|
|
4686
5059
|
const backboneAppend = backboneAppends[i];
|
|
5060
|
+
if (durableWrapper?.mirrorToDurable) {
|
|
5061
|
+
// The batch prepare committed each block into the wasm map; the log's
|
|
5062
|
+
// finishBlocks path is left unchanged (getBytes only) to preserve
|
|
5063
|
+
// strict-native mode, so mirror the committed block straight to durable
|
|
5064
|
+
// so a non-replicating native node keeps it across a restart. Tracked
|
|
5065
|
+
// for error-surfacing by mirrorToDurable.
|
|
5066
|
+
const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
5067
|
+
const committedBytes = committedHash
|
|
5068
|
+
? backbone.blocks.get(committedHash)
|
|
5069
|
+
: undefined;
|
|
5070
|
+
if (committedBytes) {
|
|
5071
|
+
durableWrapper.mirrorToDurable(committedHash, committedBytes);
|
|
5072
|
+
}
|
|
5073
|
+
}
|
|
4687
5074
|
const coordinateFields = this.createCoordinateFieldsFromNativePlanFacts({
|
|
4688
5075
|
appendFacts: facts,
|
|
4689
5076
|
plan: backboneAppend.coordinate,
|
|
@@ -5604,9 +5991,27 @@ let SharedLog = (() => {
|
|
|
5604
5991
|
this._wireSyncSession = wireSyncSession;
|
|
5605
5992
|
wireSyncSession.registerTopic(this.topic);
|
|
5606
5993
|
}
|
|
5607
|
-
|
|
5608
|
-
|
|
5609
|
-
|
|
5994
|
+
// Block store selection:
|
|
5995
|
+
// - No native backbone: durable per-program cache (unchanged default).
|
|
5996
|
+
// - Native backbone WITHOUT a durable directory (memory-only node): the
|
|
5997
|
+
// wasm-memory native store only (unchanged prior behavior).
|
|
5998
|
+
// - Native backbone WITH a durable directory: a write-through wrapper that
|
|
5999
|
+
// mirrors the native wasm store to the SAME durable `blocks` sublevel the
|
|
6000
|
+
// default path uses, and rehydrates the wasm map from disk on open. This
|
|
6001
|
+
// is what makes native entry blocks survive a restart so heads reload.
|
|
6002
|
+
let localBlocks;
|
|
6003
|
+
if (this._nativeBackbone) {
|
|
6004
|
+
if (this.node.directory != null) {
|
|
6005
|
+
const durable = new AnyBlockStore(await storage.sublevel("blocks"));
|
|
6006
|
+
localBlocks = new NativeBackboneWriteThroughBlockStore(this._nativeBackbone.blocks, durable);
|
|
6007
|
+
}
|
|
6008
|
+
else {
|
|
6009
|
+
localBlocks = this._nativeBackbone.blocks;
|
|
6010
|
+
}
|
|
6011
|
+
}
|
|
6012
|
+
else {
|
|
6013
|
+
localBlocks = new AnyBlockStore(await storage.sublevel("blocks"));
|
|
6014
|
+
}
|
|
5610
6015
|
this.remoteBlocks = new RemoteBlocks({
|
|
5611
6016
|
local: localBlocks,
|
|
5612
6017
|
publish: (message, options) => this.rpc.send(new BlocksMessage(message), options),
|
|
@@ -6127,16 +6532,29 @@ let SharedLog = (() => {
|
|
|
6127
6532
|
return undefined;
|
|
6128
6533
|
}
|
|
6129
6534
|
try {
|
|
6130
|
-
const
|
|
6535
|
+
const nativeBackboneModule = await import(
|
|
6536
|
+
/* @vite-ignore */ "@peerbit/native-backbone");
|
|
6537
|
+
const { createNativeBackboneCoordinatePersistence, createNativePeerbitBackbone, } = nativeBackboneModule;
|
|
6131
6538
|
const backbone = await createNativePeerbitBackbone({
|
|
6132
6539
|
resolution: this.domain.resolution,
|
|
6133
6540
|
clockId: this.node.identity.publicKey.bytes,
|
|
6134
6541
|
privateKey: this.node.identity.privateKey.privateKey,
|
|
6135
6542
|
publicKey: this.node.identity.publicKey.publicKey,
|
|
6136
6543
|
});
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6544
|
+
// Backward compatible: an explicitly supplied coordinate persistence
|
|
6545
|
+
// config always wins and is used unchanged. Otherwise, when the node
|
|
6546
|
+
// runs on durable on-disk storage, auto-derive a per-program store so
|
|
6547
|
+
// replication coordinates survive a clean stop -> restart without a
|
|
6548
|
+
// peer to re-derive from. Memory-only nodes (no directory) keep the
|
|
6549
|
+
// previous in-memory behavior.
|
|
6550
|
+
if (options.coordinatePersistence) {
|
|
6551
|
+
this._nativeBackboneCoordinatePersistence =
|
|
6552
|
+
createNativeBackboneCoordinatePersistence(options.coordinatePersistence);
|
|
6553
|
+
}
|
|
6554
|
+
else {
|
|
6555
|
+
this._nativeBackboneCoordinatePersistence =
|
|
6556
|
+
await this.createAutoDerivedCoordinatePersistence(nativeBackboneModule);
|
|
6557
|
+
}
|
|
6140
6558
|
return backbone;
|
|
6141
6559
|
}
|
|
6142
6560
|
catch (error) {
|
|
@@ -6147,6 +6565,52 @@ let SharedLog = (() => {
|
|
|
6147
6565
|
return undefined;
|
|
6148
6566
|
}
|
|
6149
6567
|
}
|
|
6568
|
+
/**
|
|
6569
|
+
* When the node has a durable on-disk storage directory, build a
|
|
6570
|
+
* per-program coordinate persistence store rooted under it so replication
|
|
6571
|
+
* coordinates auto-persist and survive a clean stop -> restart. Returns
|
|
6572
|
+
* `undefined` for memory-only nodes (no directory), preserving the prior
|
|
6573
|
+
* in-memory behavior.
|
|
6574
|
+
*
|
|
6575
|
+
* Namespacing: `<nodeDirectory>/coordinates/<fsSafe(log.id)>`. The log id
|
|
6576
|
+
* is the same identity used for `storage.sublevel`/`indexer.scope`
|
|
6577
|
+
* (see the `sha256Base64Sync(this.log.id)` above), but that base64 form is
|
|
6578
|
+
* not filesystem-path-safe, so the directory segment uses the hex encoding
|
|
6579
|
+
* of `this.log.id` (only `[0-9a-f]`, no `/`, `+`, or padding).
|
|
6580
|
+
*
|
|
6581
|
+
* Node vs OPFS is chosen with the same signal native-backbone uses to load
|
|
6582
|
+
* its wasm (`globalThis.process?.versions?.node`): Node gets the on-disk
|
|
6583
|
+
* store, browsers get the OPFS store.
|
|
6584
|
+
*/
|
|
6585
|
+
async createAutoDerivedCoordinatePersistence(nativeBackboneModule) {
|
|
6586
|
+
const directory = this.node.directory;
|
|
6587
|
+
if (directory == null) {
|
|
6588
|
+
// Memory-only node: keep prior in-memory behavior.
|
|
6589
|
+
return undefined;
|
|
6590
|
+
}
|
|
6591
|
+
const { createNativeBackboneCoordinatePersistence, NativeBackboneNodeCoordinatePersistenceStore, NativeBackboneOPFSCoordinatePersistenceStore, } = nativeBackboneModule;
|
|
6592
|
+
const namespace = toHexString(this.log.id);
|
|
6593
|
+
const isNode = !!globalThis.process?.versions?.node;
|
|
6594
|
+
let store;
|
|
6595
|
+
if (isNode) {
|
|
6596
|
+
const coordinateDirectory = joinNativeCoordinateDirectory(directory, namespace);
|
|
6597
|
+
store = new NativeBackboneNodeCoordinatePersistenceStore(coordinateDirectory);
|
|
6598
|
+
}
|
|
6599
|
+
else {
|
|
6600
|
+
// OPFS stores address by directory parts relative to the OPFS root,
|
|
6601
|
+
// not by an absolute filesystem path, so the node `directory` only
|
|
6602
|
+
// gates activation; the per-program namespace segments keep programs
|
|
6603
|
+
// isolated within the browser's origin-private file system.
|
|
6604
|
+
store = await NativeBackboneOPFSCoordinatePersistenceStore.create({
|
|
6605
|
+
directory: ["coordinates", namespace],
|
|
6606
|
+
});
|
|
6607
|
+
}
|
|
6608
|
+
return createNativeBackboneCoordinatePersistence({
|
|
6609
|
+
store,
|
|
6610
|
+
buffered: true,
|
|
6611
|
+
flushOnAppend: true,
|
|
6612
|
+
});
|
|
6613
|
+
}
|
|
6150
6614
|
async updateTimestampOfOwnedReplicationRanges(timestamp = +new Date()) {
|
|
6151
6615
|
const all = await this.replicationIndex
|
|
6152
6616
|
.iterate({
|