@optimystic/db-core 1.15.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/structs.d.ts +11 -1
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/collection/collection.d.ts +80 -32
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +121 -54
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +13 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/repo-protocol.d.ts +28 -15
- package/dist/src/network/repo-protocol.d.ts.map +1 -1
- package/dist/src/network/stale-failure.d.ts +10 -7
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +10 -7
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/transactor/block-floors.d.ts +2 -2
- package/dist/src/transactor/network-transactor.d.ts +21 -1
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +45 -13
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/utility/batch-coordinator.d.ts +55 -1
- package/dist/src/utility/batch-coordinator.d.ts.map +1 -1
- package/dist/src/utility/batch-coordinator.js +82 -12
- package/dist/src/utility/batch-coordinator.js.map +1 -1
- package/dist/src/utility/unref-timer.d.ts +2 -0
- package/dist/src/utility/unref-timer.d.ts.map +1 -1
- package/dist/src/utility/unref-timer.js +4 -0
- package/dist/src/utility/unref-timer.js.map +1 -1
- package/package.json +1 -1
- package/src/cluster/structs.ts +11 -1
- package/src/collection/collection.ts +125 -55
- package/src/network/i-repo.ts +13 -0
- package/src/network/repo-protocol.ts +22 -7
- package/src/network/stale-failure.ts +10 -7
- package/src/transactor/block-floors.ts +2 -2
- package/src/transactor/network-transactor.ts +67 -15
- package/src/utility/batch-coordinator.ts +124 -18
- package/src/utility/unref-timer.ts +5 -0
|
@@ -325,15 +325,15 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
325
325
|
// wires up and the log-attach-time lines included — names the instance the same way
|
|
326
326
|
// post-construction ones do.
|
|
327
327
|
const instanceTag = Collection.newInstanceTag();
|
|
328
|
-
const { source, sourceCache, tracker, floors,
|
|
329
|
-
if (!
|
|
328
|
+
const { source, sourceCache, tracker, floors, ends } = await Collection.probeHeader(transactor, id, instanceTag);
|
|
329
|
+
if (!ends) {
|
|
330
330
|
// Return before anything is staged: the tracker's transforms stay empty, so a caller
|
|
331
331
|
// that ignores the undefined cannot later sync a phantom collection into existence.
|
|
332
332
|
return undefined;
|
|
333
333
|
}
|
|
334
|
-
await Collection.attachToLog<TAction>(source,
|
|
334
|
+
await Collection.attachToLog<TAction>(source, sourceCache, tracker, floors, id, instanceTag, ends);
|
|
335
335
|
const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.filterConflict, instanceTag);
|
|
336
|
-
collection.logTailId = header.tailId;
|
|
336
|
+
collection.logTailId = ends.header.tailId;
|
|
337
337
|
return collection;
|
|
338
338
|
}
|
|
339
339
|
|
|
@@ -346,10 +346,10 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
346
346
|
static async createOrOpen<TAction>(transactor: ITransactor, id: CollectionId, init: CollectionInitOptions<TAction>): Promise<Collection<TAction>> {
|
|
347
347
|
// Pre-construction for the same reason as in open(): see the comment there.
|
|
348
348
|
const instanceTag = Collection.newInstanceTag();
|
|
349
|
-
const { source, sourceCache, tracker, floors,
|
|
349
|
+
const { source, sourceCache, tracker, floors, ends } = await Collection.probeHeader(transactor, id, instanceTag);
|
|
350
350
|
|
|
351
|
-
if (
|
|
352
|
-
await Collection.attachToLog<TAction>(source,
|
|
351
|
+
if (ends) { // Collection already exists
|
|
352
|
+
await Collection.attachToLog<TAction>(source, sourceCache, tracker, floors, id, instanceTag, ends);
|
|
353
353
|
} else { // Collection does not exist
|
|
354
354
|
log('collection:invented id=%s — no committed header found; staging a fresh empty collection', id);
|
|
355
355
|
const headerBlock = init.createHeaderBlock(id, tracker);
|
|
@@ -359,7 +359,7 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
359
359
|
}
|
|
360
360
|
|
|
361
361
|
const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.filterConflict, instanceTag);
|
|
362
|
-
collection.logTailId = header
|
|
362
|
+
collection.logTailId = ends?.header.tailId;
|
|
363
363
|
return collection;
|
|
364
364
|
}
|
|
365
365
|
|
|
@@ -383,14 +383,24 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
383
383
|
return tailId === undefined || rev === undefined ? undefined : { tailId, rev };
|
|
384
384
|
}
|
|
385
385
|
|
|
386
|
-
/** The per-instance read wiring every open path needs, plus the
|
|
387
|
-
*
|
|
386
|
+
/** The per-instance read wiring every open path needs, plus the collection's log ends — the
|
|
387
|
+
* header, then the log tail block it names — read by {@link readLogEnds}, the reader every
|
|
388
|
+
* refresh uses. Two requests: the tail id is known only from the header, so an open cannot ask
|
|
389
|
+
* for both in one the way a refresh can. `ends` is `undefined` when the header is
|
|
390
|
+
* authoritatively absent. Shared by {@link open} and {@link createOrOpen} so the two cannot drift.
|
|
391
|
+
*
|
|
392
|
+
* The header used to be probed through `source.tryGet`, an unpinned read on a handle with no
|
|
393
|
+
* floors yet. That decided nothing {@link readLogEnds} does not decide — `answeredBlock(id, entry,
|
|
394
|
+
* undefined)` is the whole of its verdict for such a read, so an unretrievable or doubted header
|
|
395
|
+
* throws exactly as it did — except recording the header as a read dependency, which the walk's
|
|
396
|
+
* first read of it through the handle's cache now records, at the revision it was served at
|
|
397
|
+
* (see {@link attachToLog}). */
|
|
388
398
|
private static async probeHeader(transactor: ITransactor, id: CollectionId, instanceTag: string): Promise<{
|
|
389
399
|
source: TransactorSource<IBlock>,
|
|
390
400
|
sourceCache: CacheSource<IBlock>,
|
|
391
401
|
tracker: Tracker<IBlock>,
|
|
392
402
|
floors: BlockFloors,
|
|
393
|
-
|
|
403
|
+
ends: LogEnds | undefined,
|
|
394
404
|
}> {
|
|
395
405
|
// Start with a context that has an infinite revision number to ensure that we always fetch the latest log information.
|
|
396
406
|
// One shared read-dependency collector feeds both the source (direct structural reads) and the cache (every
|
|
@@ -400,8 +410,8 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
400
410
|
const source = new TransactorSource(id, transactor, undefined, collector, floors);
|
|
401
411
|
const sourceCache = new CacheSource(source, undefined, collector);
|
|
402
412
|
const tracker = new Tracker(sourceCache);
|
|
403
|
-
const
|
|
404
|
-
return { source, sourceCache, tracker, floors,
|
|
413
|
+
const ends = await Collection.readLogEnds(transactor, id, undefined);
|
|
414
|
+
return { source, sourceCache, tracker, floors, ends };
|
|
405
415
|
}
|
|
406
416
|
|
|
407
417
|
/** A new handle's floors: none yet (opening walks no entries), wired to report every below-floor
|
|
@@ -419,21 +429,27 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
419
429
|
|
|
420
430
|
/** Walk an existing collection's log and point the source at its latest action context.
|
|
421
431
|
* A header we just probed successfully but whose log will not open is a fault, not an
|
|
422
|
-
* absence — throw rather than let the collection read as empty. (
|
|
423
|
-
* the tracker/cache, so it can disagree with the probe when
|
|
432
|
+
* absence — throw rather than let the collection read as empty. (A header {@link keepLogEnds}
|
|
433
|
+
* declines to keep is re-read through the tracker/cache, so it can disagree with the probe when
|
|
434
|
+
* storage is flaky mid-open.) */
|
|
424
435
|
private static async attachToLog<TAction>(
|
|
425
436
|
source: TransactorSource<IBlock>,
|
|
426
|
-
|
|
437
|
+
sourceCache: CacheSource<IBlock>,
|
|
427
438
|
tracker: Tracker<IBlock>,
|
|
439
|
+
floors: BlockFloors,
|
|
428
440
|
id: CollectionId,
|
|
429
441
|
/** The tag the calling open path minted for the Collection it is ABOUT to construct, so a
|
|
430
442
|
* diagnostic emitted here carries the same instance name as every post-construction one. */
|
|
431
443
|
instanceTag: string,
|
|
432
|
-
|
|
444
|
+
ends: LogEnds,
|
|
433
445
|
): Promise<void> {
|
|
434
446
|
// Bootstrap ActionContext from the committed tail before walking the chain.
|
|
435
447
|
// This allows the transactor to serve pending non-tail blocks during Log.open.
|
|
436
|
-
Collection.bootstrapContext(source,
|
|
448
|
+
Collection.bootstrapContext(source, ends.tail);
|
|
449
|
+
// The walk starts from the header and the tail, which the probe just read: keep them, so the
|
|
450
|
+
// walk reads neither again. Its first read of each through the cache then records the read
|
|
451
|
+
// dependency the probe's own read of the header used to.
|
|
452
|
+
Collection.keepLogEnds(sourceCache, floors, source.actionContext, ends);
|
|
437
453
|
|
|
438
454
|
const collectionLog = await Log.open<Action<TAction>>(tracker, id);
|
|
439
455
|
if (!collectionLog) {
|
|
@@ -1008,8 +1024,11 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1008
1024
|
// the entry walk and the invalidation walk each start again from the header and the tail, and
|
|
1009
1025
|
// would otherwise fetch both every time. The seed came from UNPINNED reads and is served to
|
|
1010
1026
|
// reads pinned at the tail's claim, which is sound: the tail's content at its own claimed
|
|
1011
|
-
// revision is that pinned view, and the header
|
|
1012
|
-
//
|
|
1027
|
+
// revision is that pinned view, and the only part of the header a log walk reads — where the
|
|
1028
|
+
// chain starts and ends — changes only when the tail block fills, which Chain.getTail already
|
|
1029
|
+
// tolerates by following `nextId` from whichever tail the header names. (The rest of the header,
|
|
1030
|
+
// a tree's root for one, is not read here; the handle's own cache keeps the header only under
|
|
1031
|
+
// the stricter rule in {@link keepLogEnds}.)
|
|
1013
1032
|
// NOTE: the cache keeps its default size (128 blocks, about 4,000 log entries). The entry walk
|
|
1014
1033
|
// reads every log block back to the head (no checkpoints), newest first, so past that size
|
|
1015
1034
|
// the newest blocks are evicted first and the invalidation walk fetches them again. If logs
|
|
@@ -1063,7 +1082,7 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1063
1082
|
anyConflicts = true;
|
|
1064
1083
|
}
|
|
1065
1084
|
|
|
1066
|
-
this.forgetAndAdopt(latest, entryRevs, revertedBlockIds, ends
|
|
1085
|
+
this.forgetAndAdopt(latest, entryRevs, revertedBlockIds, ends);
|
|
1067
1086
|
|
|
1068
1087
|
Collection.reportShortfall(this.id, this.instanceTag, tailRev, actionContext?.rev, this.source.actionContext?.rev);
|
|
1069
1088
|
|
|
@@ -1083,7 +1102,7 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1083
1102
|
|
|
1084
1103
|
/** Forget every block the refresh saw change, floor the ones a log entry names, adopt the
|
|
1085
1104
|
* revision the log is at, and keep what the refresh already read
|
|
1086
|
-
* ({@link
|
|
1105
|
+
* ({@link keepLogEnds}) — ONE synchronous step, which must stay free of any `await`.
|
|
1087
1106
|
*
|
|
1088
1107
|
* Reads are not latched, so one can run while a refresh is under way. Forgetting a block while
|
|
1089
1108
|
* this handle still reads at the revision it is LEAVING invites exactly the wrong re-read: the
|
|
@@ -1105,7 +1124,7 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1105
1124
|
latest: GetFromResult<Action<TAction>> | undefined,
|
|
1106
1125
|
entryRevs: ReadonlyMap<ActionId, number>,
|
|
1107
1126
|
revertedBlockIds: BlockId[],
|
|
1108
|
-
|
|
1127
|
+
ends: LogEnds,
|
|
1109
1128
|
): void {
|
|
1110
1129
|
for (const entry of latest?.entries ?? []) {
|
|
1111
1130
|
this.sourceCache.clear(entry.blockIds);
|
|
@@ -1113,63 +1132,113 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1113
1132
|
}
|
|
1114
1133
|
this.sourceCache.clear(revertedBlockIds);
|
|
1115
1134
|
Collection.advanceContext(this.source, this.id, this.instanceTag, 'refresh', latest?.context);
|
|
1116
|
-
this.
|
|
1135
|
+
Collection.keepLogEnds(this.sourceCache, this.floors, this.source.actionContext, ends);
|
|
1117
1136
|
}
|
|
1118
1137
|
|
|
1119
|
-
/** Put the header and log tail block {@link readLogEnds} already fetched into
|
|
1138
|
+
/** Put the header and log tail block {@link readLogEnds} already fetched into a handle's own
|
|
1120
1139
|
* cache, so the next read of either is a hit rather than a second fetch of a block just
|
|
1121
|
-
* received.
|
|
1122
|
-
*
|
|
1123
|
-
*
|
|
1124
|
-
*
|
|
1125
|
-
*
|
|
1126
|
-
*
|
|
1127
|
-
*
|
|
1128
|
-
*
|
|
1129
|
-
*
|
|
1130
|
-
*
|
|
1131
|
-
*
|
|
1132
|
-
*
|
|
1140
|
+
* received. Two sites, one rule:
|
|
1141
|
+
* - **An open** ({@link attachToLog}), before the walk: `Log.open` and the tail lookup behind
|
|
1142
|
+
* the context start from exactly these two blocks, and would otherwise fetch both again.
|
|
1143
|
+
* - **A refresh that walked** ({@link forgetAndAdopt}). Without it, a write that follows ANOTHER
|
|
1144
|
+
* handle's commit fetches the log tail twice: the refresh reads it, the walked entry names it
|
|
1145
|
+
* (a commit's blocks include the log block its entry was appended to) so the clear drops it,
|
|
1146
|
+
* and `Chain.add` then reads it again to append this write's own entry. Last in that one
|
|
1147
|
+
* synchronous step, after BOTH clears — offering before either would simply be undone by it.
|
|
1148
|
+
* An id that an invalidation reverted is therefore re-offered rather than left forgotten, which
|
|
1149
|
+
* is sound because the revert is itself a commit and {@link readLogEnds} read unpinned: what it
|
|
1150
|
+
* read already includes the revert.
|
|
1151
|
+
*
|
|
1152
|
+
* Static, taking the cache, the floors and the context rather than `this`, because an open keeps
|
|
1153
|
+
* before the handle is constructed.
|
|
1154
|
+
*
|
|
1155
|
+
* An offer is made only when all of these hold:
|
|
1156
|
+
* - **The handle holds a context.** With none adopted, every later read is unpinned and may
|
|
1133
1157
|
* legitimately be answered newer than what was read here, so there is nothing to judge the
|
|
1134
1158
|
* offer against.
|
|
1135
1159
|
* - **The served revision is at or below the adopted one.** {@link readLogEnds} reads unpinned,
|
|
1136
|
-
* so it can come back newer than the view
|
|
1160
|
+
* so it can come back newer than the view the handle now reads at — exactly the
|
|
1137
1161
|
* `collection:context-short-of-tail` case. Serving that to a read pinned lower would fabricate
|
|
1138
1162
|
* a view that never existed. At or below the pin it is sound: a block whose newest content is
|
|
1139
1163
|
* at `rev <= context.rev` has the same content at `context.rev`.
|
|
1140
1164
|
* - **The served revision meets any applicable floor.** A cache must never keep a below-floor
|
|
1141
1165
|
* answer.
|
|
1166
|
+
* - **For the header, nothing rewrote it after it was read** ({@link headerCurrentThroughTail}).
|
|
1167
|
+
* "Newest at `rev`" above holds for each block at the moment it was read, and the header and
|
|
1168
|
+
* the tail are not read at one moment.
|
|
1142
1169
|
*
|
|
1143
1170
|
* NOTE: the floor check lives here because {@link readLogEnds} reads around
|
|
1144
1171
|
* {@link TransactorSource}, which is what applies floors on every other read — nothing else on
|
|
1145
1172
|
* this path would apply one. It asks {@link BlockFloors.meetsApplicableFloor} rather than
|
|
1146
1173
|
* spelling the comparison out, so it cannot drift from the one `answeredBelowFloor` makes, and
|
|
1147
1174
|
* asks the non-reporting face because this is not an answer being served to a reader — the
|
|
1148
|
-
* `collection:block-below-floor` line would misdescribe it.
|
|
1149
|
-
*
|
|
1150
|
-
*
|
|
1151
|
-
*
|
|
1152
|
-
*
|
|
1153
|
-
*
|
|
1154
|
-
*
|
|
1175
|
+
* `collection:block-below-floor` line would misdescribe it. An opening handle has no floors yet,
|
|
1176
|
+
* so there the check passes by construction.
|
|
1177
|
+
*
|
|
1178
|
+
* The early return at {@link tailShowsNothingNewer} deliberately does not keep: it fires when
|
|
1179
|
+
* the log has not moved, and the cache is then already warm with the tail (folded in by this
|
|
1180
|
+
* handle's own commit, or kept from its last walk or its open), so keeping there would buy a
|
|
1181
|
+
* fetch back only when the LRU had evicted it — at the cost of putting the one-synchronous-step
|
|
1182
|
+
* reasoning in a second place. */
|
|
1155
1183
|
// NOTE: an offer whose revision equals what the cache already holds is still kept, so a walking
|
|
1156
1184
|
// refresh bumps the header's generation even when the header was neither cleared nor changed.
|
|
1157
1185
|
// Over-bumping is safe (see `CacheSource.bump`) and costs a re-materialize of that block on its
|
|
1158
1186
|
// next read; if that ever shows up, skip the offer when the cache already RETAINS the id at the
|
|
1159
1187
|
// same revision — but note it would then also stop correcting a cached block whose content was
|
|
1160
1188
|
// folded forward locally at that revision.
|
|
1161
|
-
private
|
|
1162
|
-
const context = this.source.actionContext;
|
|
1189
|
+
private static keepLogEnds(cache: CacheSource<IBlock>, floors: BlockFloors, context: ActionContext | undefined, ends: LogEnds): void {
|
|
1163
1190
|
if (context === undefined) {
|
|
1164
1191
|
return;
|
|
1165
1192
|
}
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1193
|
+
const headerId = ends.header.header.id;
|
|
1194
|
+
const keepHeader = Collection.headerCurrentThroughTail(ends);
|
|
1195
|
+
for (const [blockId, block, rev] of ends.served) {
|
|
1196
|
+
if ((blockId !== headerId || keepHeader) && rev <= context.rev && floors.meetsApplicableFloor(blockId, context, rev)) {
|
|
1197
|
+
cache.offerServed(blockId, block, rev);
|
|
1169
1198
|
}
|
|
1170
1199
|
}
|
|
1171
1200
|
}
|
|
1172
1201
|
|
|
1202
|
+
/** Whether the header {@link readLogEnds} read is still the header at the revision of the tail
|
|
1203
|
+
* block it read — what {@link keepLogEnds}' revision test cannot see for the header.
|
|
1204
|
+
*
|
|
1205
|
+
* That test keeps a block served at or below the adopted revision because a block whose newest
|
|
1206
|
+
* content is at `rev` has the same content at every later revision. "Newest" held when the block
|
|
1207
|
+
* was read, and the header and the tail are not read at one moment: an open asks for the tail in
|
|
1208
|
+
* a second request, so does a refresh whose remembered tail is out of date, and even the
|
|
1209
|
+
* refresh's one batched request may be answered by two machines. A commit that lands between the
|
|
1210
|
+
* two reads and rewrites the header — a log roll-over (the header names the tail block) or a tree
|
|
1211
|
+
* whose root moved (a tree's header names its root) — leaves the context adopted from the tail
|
|
1212
|
+
* at that commit's revision while the header read before it is served there as if current. Kept,
|
|
1213
|
+
* the handle would read the tree through a root it no longer has, and a write that changes the
|
|
1214
|
+
* header again would be refused by storage's base guard and retried.
|
|
1215
|
+
*
|
|
1216
|
+
* Every commit and every invalidation appends an entry naming the blocks it changed, and only
|
|
1217
|
+
* ever on the tail block; starting a new tail block rewrites the header. So every commit above
|
|
1218
|
+
* the header's served revision sits in the tail block the header names, or past it. The header is
|
|
1219
|
+
* current through that tail when the tail is still the last block (no `nextId`, so nothing rolled
|
|
1220
|
+
* over since) and none of its entries above the header's revision changed the header. Otherwise
|
|
1221
|
+
* the header is not kept, and the next read of it goes to storage pinned at the adopted revision.
|
|
1222
|
+
*
|
|
1223
|
+
* Cautious by design: an entry above the ADOPTED revision refuses too, as does a header served by
|
|
1224
|
+
* a machine behind the log. Either costs one read of the header; neither keeps a wrong one. */
|
|
1225
|
+
private static headerCurrentThroughTail(ends: LogEnds): boolean {
|
|
1226
|
+
const tail = ends.tail?.block as LogBlock<unknown> | undefined;
|
|
1227
|
+
const headerId = ends.header.header.id;
|
|
1228
|
+
const headerRev = ends.served.find(([blockId]) => blockId === headerId)?.[2];
|
|
1229
|
+
if (!tail || tail.nextId !== undefined || headerRev === undefined) {
|
|
1230
|
+
return false;
|
|
1231
|
+
}
|
|
1232
|
+
return !tail.entries.some(entry => entry.rev > headerRev && Collection.entryChanges(entry, headerId));
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
/** Whether the log `entry` changed `blockId`: an action that wrote it, or an invalidation that
|
|
1236
|
+
* reverted it. A checkpoint changes no block but the log block it is appended to. */
|
|
1237
|
+
private static entryChanges(entry: LogEntry<unknown>, blockId: BlockId): boolean {
|
|
1238
|
+
return (entry.action?.blockIds.includes(blockId) ?? false)
|
|
1239
|
+
|| (entry.invalidation?.reverted.some(reverted => reverted.blockId === blockId) ?? false);
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1173
1242
|
/** The revision each action in `context` committed at, keyed by action id. `Log.getFrom` returns
|
|
1174
1243
|
* entries without their revisions; the context the same walk built is where they are restated. */
|
|
1175
1244
|
private static revisionsByAction(context: ActionContext | undefined): Map<ActionId, number> {
|
|
@@ -2219,11 +2288,12 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
2219
2288
|
* letting that walk stop short, read the known tail in its own request instead.
|
|
2220
2289
|
*
|
|
2221
2290
|
* `tailFloor` (see {@link updateInternal}) goes on whichever request reads the tail — never on
|
|
2222
|
-
* the header, which is rewritten only when the tail rolls over
|
|
2223
|
-
*
|
|
2224
|
-
*
|
|
2225
|
-
*
|
|
2226
|
-
*
|
|
2291
|
+
* the header, which is rewritten only when the tail rolls over or the collection changes it
|
|
2292
|
+
* itself (a tree's root moving), so its revision says nothing about the collection's. It is not
|
|
2293
|
+
* raised into the handle's {@link floors}: a standing floor on the tail id this handle remembers
|
|
2294
|
+
* would be wrong once that tail rolls over, because a filled tail's last revision is the
|
|
2295
|
+
* roll-over's, below the floor, so every later read of it (the chain walk, the invalidation walk)
|
|
2296
|
+
* would be judged too old for the life of the handle.
|
|
2227
2297
|
*
|
|
2228
2298
|
* @returns undefined when the header is authoritatively absent. */
|
|
2229
2299
|
private static async readLogEnds(transactor: ITransactor, id: CollectionId, knownTailId: BlockId | undefined, tailFloor?: number): Promise<LogEnds | undefined> {
|
package/src/network/i-repo.ts
CHANGED
|
@@ -64,4 +64,17 @@ export type IRepo = {
|
|
|
64
64
|
pend(request: PendRequest, options?: MessageOptions): Promise<PendResult>;
|
|
65
65
|
cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void>;
|
|
66
66
|
commit(request: RepoCommitRequest, options?: MessageOptions): Promise<CommitResult>;
|
|
67
|
+
/**
|
|
68
|
+
* Pend every request as ONE consensus record. The requests are one action's (same action id), name
|
|
69
|
+
* disjoint blocks, and are answered together: success only if every one was accepted, and then
|
|
70
|
+
* `PendSuccess.blockIds` is the union of their blocks. A list that is empty, names two actions or
|
|
71
|
+
* names one block twice is a sender's bug and throws.
|
|
72
|
+
*
|
|
73
|
+
* Optional: only a repo that coordinates consensus implements it (`CoordinatorRepo`, `RepoClient`
|
|
74
|
+
* and the proxies in front of them). A member applies a batch one request at a time through
|
|
75
|
+
* {@link pend}, so local storage does not.
|
|
76
|
+
*/
|
|
77
|
+
pendMany?(requests: readonly PendRequest[], options?: MessageOptions): Promise<PendResult>;
|
|
78
|
+
/** Commit every request as ONE consensus record, on the same terms as {@link pendMany}. */
|
|
79
|
+
commitMany?(requests: readonly RepoCommitRequest[], options?: MessageOptions): Promise<CommitResult>;
|
|
67
80
|
}
|
|
@@ -1,13 +1,28 @@
|
|
|
1
1
|
import type { ActionBlocks, PendRequest, CommitRequest, BlockGets, InvalidateRequest } from "../index.js";
|
|
2
2
|
|
|
3
|
+
/** One operation a {@link RepoMessage} asks for. */
|
|
4
|
+
export type RepoOperation =
|
|
5
|
+
{ get: BlockGets } |
|
|
6
|
+
{ pend: PendRequest } |
|
|
7
|
+
{ cancel: { actionRef: ActionBlocks } } |
|
|
8
|
+
{ commit: CommitRequest } |
|
|
9
|
+
{ invalidate: InvalidateRequest };
|
|
10
|
+
|
|
3
11
|
export type RepoMessage = {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
12
|
+
/**
|
|
13
|
+
* What the message asks for. A message of more than one operation must be a **batch**, the several
|
|
14
|
+
* collections of one action written together:
|
|
15
|
+
* - every operation is a `pend`, or every operation is a `commit` — no `get`, `cancel` or
|
|
16
|
+
* `invalidate`, and no mixing;
|
|
17
|
+
* - every operation names the same action id;
|
|
18
|
+
* - no block id appears in two operations: collections never share blocks, and a duplicate would
|
|
19
|
+
* pend or commit one block twice under one action, at two revisions.
|
|
20
|
+
*
|
|
21
|
+
* Each operation keeps its own request — its own revision, log tail, bases and digests — and is
|
|
22
|
+
* applied one at a time. A one-operation message is trivially a batch. `batchShapeFault` in
|
|
23
|
+
* `packages/db-p2p/src/cluster/record-operations.ts` is the one implementation of the rule.
|
|
24
|
+
*/
|
|
25
|
+
operations: [RepoOperation, ...RepoOperation[]],
|
|
11
26
|
expiration?: number,
|
|
12
27
|
coordinatingBlockIds?: string[],
|
|
13
28
|
};
|
|
@@ -41,13 +41,16 @@ export function isExpiryFailure(failure: WriteFailure): failure is ExpiredFailur
|
|
|
41
41
|
* the transactor's aggregate meaningful: if a producer reported an arbitrary block instead of its
|
|
42
42
|
* highest, taking the max across producers would still understate the constraint.
|
|
43
43
|
*
|
|
44
|
-
* NOTE: comparing revisions across blocks is only
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* counter.
|
|
44
|
+
* NOTE: comparing revisions across blocks is meaningful only within one collection, where every
|
|
45
|
+
* candidate comes from the same revision counter. A batch (`IRepo.pendMany` / `commitMany`) spans
|
|
46
|
+
* collections, and `CoordinatorRepo` still picks the highest across them, so its `staleAt` names the
|
|
47
|
+
* block — and so the collection — of one loser among several counters. That is harmless while the
|
|
48
|
+
* multi-collection coordinator, the only writer that sends a batch, does not read the number (backlog
|
|
49
|
+
* `debt-multi-collection-retry-cannot-see-the-taken-revision`). A consumer that compares it against a
|
|
50
|
+
* revision counter, as `Collection.syncAttempts` does against its OWN next revision to decide whether a
|
|
51
|
+
* retry could differ (`SyncRevisionStalledError`), must compare it against the counter of the
|
|
52
|
+
* collection owning `staleAt.blockId` — and once one does for a batch, selection must become
|
|
53
|
+
* per-collection.
|
|
51
54
|
*/
|
|
52
55
|
export function highestStaleAt(candidates: readonly StaleFailure['staleAt'][]): StaleFailure['staleAt'] {
|
|
53
56
|
let best: StaleFailure['staleAt'];
|
|
@@ -41,8 +41,8 @@ export interface BlockFloorCheck {
|
|
|
41
41
|
|
|
42
42
|
/** The same comparison {@link answeredBelowFloor} makes, without reporting it — for content that
|
|
43
43
|
* is not an answer being served to a reader, where the `collection:block-below-floor` line would
|
|
44
|
-
* misdescribe what happened. `Collection.
|
|
45
|
-
*
|
|
44
|
+
* misdescribe what happened. `Collection.keepLogEnds` weighs the header and log tail an open or a
|
|
45
|
+
* refresh read around `TransactorSource` this way.
|
|
46
46
|
*
|
|
47
47
|
* @param servedRev the revision the content is.
|
|
48
48
|
* @returns `true` when no floor applies or the content meets the one that does. */
|
|
@@ -15,7 +15,7 @@ import { Log } from "../log/log.js";
|
|
|
15
15
|
import { groupBy } from "../utility/groupby.js";
|
|
16
16
|
import { routingKeyForBlock } from "../network/routing-key.js";
|
|
17
17
|
import { isRecordEmpty } from "../utility/is-record-empty.js";
|
|
18
|
-
import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
|
|
18
|
+
import { type CoordinatorBatch, type RetryBackoff, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload, retryWaitWithinBudget, retrySleep } from "../utility/batch-coordinator.js";
|
|
19
19
|
import { abortableDelay, jitteredBackoffMs } from "../utility/backoff.js";
|
|
20
20
|
import { OwedCancels, type OwedCancelClock } from "./owed-cancels.js";
|
|
21
21
|
import { createLogger, verbose } from "../logger.js";
|
|
@@ -55,6 +55,18 @@ type NetworkTransactorInit = {
|
|
|
55
55
|
* which is the nearest.
|
|
56
56
|
*/
|
|
57
57
|
localPeerId?: PeerId;
|
|
58
|
+
/**
|
|
59
|
+
* The first wait, in ms, before a retry of a request that went unanswered: one declared link round
|
|
60
|
+
* trip (`LinkDeadlines.retryBackoffBaseMs` on a db-p2p node), doubling for each further coordinator
|
|
61
|
+
* the retry's chain tries, and never less than the failed request was outstanding when the retry
|
|
62
|
+
* goes out through this node's own send queue to the same peer (`retryWaitMs` in
|
|
63
|
+
* `utility/batch-coordinator.ts`). A request given up on at a deadline is usually still in that
|
|
64
|
+
* queue, so a retry sent at once only lengthens it. Omit, or pass 0, on a node that declared no round
|
|
65
|
+
* trip: every retry then goes out at once, as before the wait existed.
|
|
66
|
+
*/
|
|
67
|
+
retryBackoffBaseMs?: number;
|
|
68
|
+
/** Test seam for the retry waits; omit in production (an unref'd `setTimeout`). */
|
|
69
|
+
retrySleep?: (ms: number) => Promise<void>;
|
|
58
70
|
/**
|
|
59
71
|
* Time for the re-sends of cancels this transactor could not discharge ({@link OwedCancels}).
|
|
60
72
|
* Omit in production (`Date.now` and an unref'd `setTimeout`); a test injects a fake to fire the
|
|
@@ -81,6 +93,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
81
93
|
private readonly localChangeNotifier: IBlockChangeNotifier | undefined;
|
|
82
94
|
/** `NetworkTransactorInit.localPeerId` as a string, the form cohort peer ids arrive in. */
|
|
83
95
|
private readonly localPeerId: string | undefined;
|
|
96
|
+
/** The pace of every retry this transactor sends: each {@link processBatches} call, and {@link get}'s retry round. */
|
|
97
|
+
private readonly retryBackoff: RetryBackoff;
|
|
84
98
|
|
|
85
99
|
/**
|
|
86
100
|
* Per-transaction coordinator cache: `actionId → (blockId → resolved coordinator)`.
|
|
@@ -122,6 +136,7 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
122
136
|
this.getRepo = init.getRepo;
|
|
123
137
|
this.localChangeNotifier = init.localChangeNotifier;
|
|
124
138
|
this.localPeerId = init.localPeerId?.toString();
|
|
139
|
+
this.retryBackoff = { baseMs: init.retryBackoffBaseMs ?? 0, localPeerId: init.localPeerId, sleep: init.retrySleep };
|
|
125
140
|
this.owedCancels = new OwedCancels({
|
|
126
141
|
attempt: actionRef => this.resendOwedCancel(actionRef),
|
|
127
142
|
clock: init.owedCancelClock
|
|
@@ -193,7 +208,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
193
208
|
batch => batch.payload,
|
|
194
209
|
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
|
|
195
210
|
expiration,
|
|
196
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' })
|
|
211
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' }),
|
|
212
|
+
this.retryBackoff
|
|
197
213
|
);
|
|
198
214
|
} catch (e) {
|
|
199
215
|
error = e as Error;
|
|
@@ -258,7 +274,12 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
258
274
|
};
|
|
259
275
|
|
|
260
276
|
// Retry only genuine no-response / partial-response batches. An authoritative
|
|
261
|
-
// absent answer is not retried.
|
|
277
|
+
// absent answer is not retried. Neither is a batch `processBatches` already
|
|
278
|
+
// re-batched: a coordinator that threw has retries in the tree that carry the chain
|
|
279
|
+
// on with it excluded, and they are judged here in its place. Asking it again —
|
|
280
|
+
// after a fresh backoff wait, under the outstanding floor a whole request deadline
|
|
281
|
+
// when this node coordinates — would only add a request and latency. Skipping it also
|
|
282
|
+
// leaves the retried batches covering disjoint blocks, so no block is asked twice.
|
|
262
283
|
// NOTE: a 'cohort-unreachable' entry earns this retry like any other flagged entry,
|
|
263
284
|
// and on a genuinely isolated node the retry re-picks the same node (the
|
|
264
285
|
// findCoordinator:all-excluded path) and repeats the same futile consult. Fine
|
|
@@ -272,15 +293,21 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
272
293
|
// the below-floor content is the correct content and every machine agrees on it
|
|
273
294
|
// (see the accepted-tradeoff NOTE at `TransactorSource.mayRetain`). The reader
|
|
274
295
|
// judges it again and returns it uncached.
|
|
275
|
-
const retryable = Array.from(allBatches(batches)).filter(b =>
|
|
276
|
-
!isAuthoritative(b as any)
|
|
277
|
-
) as CoordinatorBatch<BlockId[], GetBlockResults>[];
|
|
296
|
+
const retryable = Array.from(allBatches(batches)).filter(b => !b.rebatched && !isAuthoritative(b));
|
|
278
297
|
|
|
279
298
|
if (retryable.length > 0 && Date.now() < expiration) {
|
|
280
299
|
log('get:retry retryable=%d', retryable.length);
|
|
281
|
-
// Fan out the per-batch retries concurrently. Each
|
|
282
|
-
// excluded-peer set and attaches its own `subsumedBy`,
|
|
283
|
-
//
|
|
300
|
+
// Fan out the per-batch retries concurrently. Each retried batch builds its own
|
|
301
|
+
// excluded-peer set and attaches its own `subsumedBy`, and retried batches cover
|
|
302
|
+
// disjoint blocks (above), so the retry rounds are independent and safe to run in parallel.
|
|
303
|
+
// Only a batch with NO valid response (its request threw or timed out) waits before its
|
|
304
|
+
// retry: its frames are likely still in this node's send queue to that peer, and a retry
|
|
305
|
+
// sent at once would queue behind them. Such a batch reaches this round only when
|
|
306
|
+
// `processBatches` did not re-batch it: its retry lookup threw, or its wait did not fit
|
|
307
|
+
// the budget. A batch that got a response which was not authoritative (an entry
|
|
308
|
+
// `unavailable`, `unconfirmedAheadRev`, below its floor, or missing) retries at once —
|
|
309
|
+
// its predecessor is out of the queue, so waiting would add latency and buy nothing.
|
|
310
|
+
// That narrows "wait before each retry" deliberately.
|
|
284
311
|
const retryOutcomes = await Promise.allSettled(retryable.map(async b => {
|
|
285
312
|
const excluded = new Set<PeerId>([b.peerId, ...((b.excludedPeers ?? []) as PeerId[])]);
|
|
286
313
|
const retries = await createBatchesForPayload<BlockId[], GetBlockResults>(
|
|
@@ -290,7 +317,7 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
290
317
|
Array.from(excluded),
|
|
291
318
|
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' })
|
|
292
319
|
);
|
|
293
|
-
if (retries.length > 0) {
|
|
320
|
+
if (retries.length > 0 && (hasValidResponse(b) || await this.waitedForReadRetry(b, excluded.size, retries, expiration))) {
|
|
294
321
|
b.subsumedBy = [...(b.subsumedBy ?? []), ...retries];
|
|
295
322
|
await processBatches(
|
|
296
323
|
retries,
|
|
@@ -298,7 +325,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
298
325
|
batch => batch.payload,
|
|
299
326
|
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
|
|
300
327
|
expiration,
|
|
301
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' })
|
|
328
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' }),
|
|
329
|
+
this.retryBackoff
|
|
302
330
|
);
|
|
303
331
|
}
|
|
304
332
|
}));
|
|
@@ -416,6 +444,27 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
416
444
|
return Object.fromEntries(resultEntries) as GetBlockResults;
|
|
417
445
|
}
|
|
418
446
|
|
|
447
|
+
/**
|
|
448
|
+
* Waits out the backoff before {@link get}'s second-chance round re-sends `failed` — a batch with no
|
|
449
|
+
* valid response — to `retries`, whose chain has tried `attempt` coordinators. False when the wait
|
|
450
|
+
* would reach `expiration`: the retry is then not sent, and the batch keeps its own failure.
|
|
451
|
+
*/
|
|
452
|
+
private async waitedForReadRetry(
|
|
453
|
+
failed: CoordinatorBatch<BlockId[], GetBlockResults>,
|
|
454
|
+
attempt: number,
|
|
455
|
+
retries: CoordinatorBatch<BlockId[], GetBlockResults>[],
|
|
456
|
+
expiration: number
|
|
457
|
+
): Promise<boolean> {
|
|
458
|
+
const outstandingMs = failed.outstandingMs ?? 0;
|
|
459
|
+
const waitMs = retryWaitWithinBudget(this.retryBackoff, { attempt, failedPeer: failed.peerId, outstandingMs, retryPeers: retries.map(r => r.peerId) }, expiration);
|
|
460
|
+
if (waitMs === undefined) return false;
|
|
461
|
+
if (waitMs > 0) {
|
|
462
|
+
log('get:retry-wait peer=%s excluded=%d waitMs=%d outstandingMs=%d', failed.peerId.toString(), attempt, waitMs, outstandingMs);
|
|
463
|
+
await retrySleep(this.retryBackoff)(waitMs);
|
|
464
|
+
}
|
|
465
|
+
return true;
|
|
466
|
+
}
|
|
467
|
+
|
|
419
468
|
async getStatus(blockActions: ActionBlocks[]): Promise<BlockActionStatus[]> {
|
|
420
469
|
// Collect all unique block IDs across all action refs
|
|
421
470
|
const allBlockIds = [...new Set(blockActions.flatMap(ref => ref.blockIds))];
|
|
@@ -716,7 +765,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
716
765
|
batch => blockIdsForTransforms(batch.payload),
|
|
717
766
|
transformForBlock,
|
|
718
767
|
expiration,
|
|
719
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
|
|
768
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options),
|
|
769
|
+
this.retryBackoff
|
|
720
770
|
);
|
|
721
771
|
// Cache resolved coordinators for follow-up commit to hit the same peers
|
|
722
772
|
try {
|
|
@@ -1047,7 +1097,7 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1047
1097
|
* record, so the cancel is a no-op there;
|
|
1048
1098
|
* - it did not land ⇒ the cancel is exactly the repair;
|
|
1049
1099
|
* - its consensus is still in flight and lands AFTER the cancel ⇒ the member meets a missing
|
|
1050
|
-
* pend, which `ClusterMember.
|
|
1100
|
+
* pend, which `ClusterMember.applyCommitToStorage` already treats as "behind" divergence and
|
|
1051
1101
|
* cures by reconciling the block from a cohort peer.
|
|
1052
1102
|
*
|
|
1053
1103
|
* Confirmed blocks are excluded only to skip a pointless consensus round — cancelling one would
|
|
@@ -1150,7 +1200,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1150
1200
|
batch => batch.payload,
|
|
1151
1201
|
mergeBlocks,
|
|
1152
1202
|
expiration,
|
|
1153
|
-
async (blockId, options) => this.resolveCoordinator(blockId, options, actionId)
|
|
1203
|
+
async (blockId, options) => this.resolveCoordinator(blockId, options, actionId),
|
|
1204
|
+
this.retryBackoff
|
|
1154
1205
|
);
|
|
1155
1206
|
} catch (e) {
|
|
1156
1207
|
error = e as Error;
|
|
@@ -1405,7 +1456,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1405
1456
|
batch => batch.payload,
|
|
1406
1457
|
mergeBlocks,
|
|
1407
1458
|
deadline,
|
|
1408
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
|
|
1459
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options),
|
|
1460
|
+
this.retryBackoff
|
|
1409
1461
|
);
|
|
1410
1462
|
} catch (e) {
|
|
1411
1463
|
error = asError(e);
|