@optimystic/db-core 1.18.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/src/blocks/helpers.d.ts +7 -0
- package/dist/src/blocks/helpers.d.ts.map +1 -1
- package/dist/src/blocks/helpers.js +7 -1
- package/dist/src/blocks/helpers.js.map +1 -1
- package/dist/src/btree/btree.d.ts +100 -4
- package/dist/src/btree/btree.d.ts.map +1 -1
- package/dist/src/btree/btree.js +252 -90
- package/dist/src/btree/btree.js.map +1 -1
- package/dist/src/btree/node-division.d.ts +37 -0
- package/dist/src/btree/node-division.d.ts.map +1 -0
- package/dist/src/btree/node-division.js +133 -0
- package/dist/src/btree/node-division.js.map +1 -0
- package/dist/src/chain/chain.d.ts +23 -0
- package/dist/src/chain/chain.d.ts.map +1 -1
- package/dist/src/chain/chain.js +43 -6
- package/dist/src/chain/chain.js.map +1 -1
- package/dist/src/collection/collection.d.ts +38 -5
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +126 -11
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +20 -0
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +5 -3
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/log/log.d.ts +5 -1
- package/dist/src/log/log.d.ts.map +1 -1
- package/dist/src/log/log.js +7 -1
- package/dist/src/log/log.js.map +1 -1
- package/dist/src/network/block-size.d.ts +67 -0
- package/dist/src/network/block-size.d.ts.map +1 -0
- package/dist/src/network/block-size.js +82 -0
- package/dist/src/network/block-size.js.map +1 -0
- package/dist/src/network/index.d.ts +2 -0
- package/dist/src/network/index.d.ts.map +1 -1
- package/dist/src/network/index.js +2 -0
- package/dist/src/network/index.js.map +1 -1
- package/dist/src/network/struct.d.ts +13 -11
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/network/transaction-size.d.ts +59 -0
- package/dist/src/network/transaction-size.d.ts.map +1 -0
- package/dist/src/network/transaction-size.js +74 -0
- package/dist/src/network/transaction-size.js.map +1 -0
- package/dist/src/reactivity/config.d.ts +8 -4
- package/dist/src/reactivity/config.d.ts.map +1 -1
- package/dist/src/reactivity/config.js +7 -3
- package/dist/src/reactivity/config.js.map +1 -1
- package/dist/src/reactivity/rotation.d.ts +9 -3
- package/dist/src/reactivity/rotation.d.ts.map +1 -1
- package/dist/src/reactivity/rotation.js +8 -2
- package/dist/src/reactivity/rotation.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +2 -0
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +10 -0
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +72 -3
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +320 -146
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transactor/change-notifier.d.ts +13 -4
- package/dist/src/transactor/change-notifier.d.ts.map +1 -1
- package/dist/src/transactor/change-notifier.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +57 -8
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +185 -24
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor.d.ts +14 -0
- package/dist/src/transactor/transactor.d.ts.map +1 -1
- package/dist/src/transform/digest.d.ts +20 -0
- package/dist/src/transform/digest.d.ts.map +1 -1
- package/dist/src/transform/digest.js +23 -3
- package/dist/src/transform/digest.js.map +1 -1
- package/package.json +1 -1
- package/src/blocks/helpers.ts +8 -1
- package/src/btree/btree.ts +266 -88
- package/src/btree/node-division.ts +155 -0
- package/src/chain/chain.ts +55 -6
- package/src/collection/collection.ts +135 -13
- package/src/collection/struct.ts +18 -0
- package/src/collections/tree/tree.ts +4 -2
- package/src/log/log.ts +7 -1
- package/src/network/block-size.ts +81 -0
- package/src/network/index.ts +2 -0
- package/src/network/struct.ts +15 -13
- package/src/network/transaction-size.ts +73 -0
- package/src/reactivity/config.ts +8 -4
- package/src/reactivity/rotation.ts +9 -3
- package/src/testing/test-transactor.ts +12 -0
- package/src/transaction/coordinator.ts +383 -154
- package/src/transactor/change-notifier.ts +13 -4
- package/src/transactor/network-transactor.ts +192 -29
- package/src/transactor/transactor.ts +16 -0
- package/src/transform/digest.ts +36 -3
|
@@ -67,10 +67,19 @@ export type CommitCert = {
|
|
|
67
67
|
|
|
68
68
|
export interface IBlockChangeNotifier {
|
|
69
69
|
/**
|
|
70
|
-
* Subscribe to commits that mutate the given collection. Returns an
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
70
|
+
* Subscribe to commits that mutate the given collection. Returns an idempotent unsubscribe.
|
|
71
|
+
* Listeners are invoked AFTER the commit's critical section (locks released), in commit order,
|
|
72
|
+
* never from inside it; a throwing listener must not break the commit or other listeners (log +
|
|
73
|
+
* continue).
|
|
74
|
+
*
|
|
75
|
+
* A listener may be invoked later than the call that landed the change: a transaction that
|
|
76
|
+
* writes several collections (a table and its indexes) commits each as its own request under one
|
|
77
|
+
* action id, and a node's storage holds one collection's event while another collection of the
|
|
78
|
+
* same action is still pending there, releasing it when that collection lands, is cancelled, or
|
|
79
|
+
* a bound elapses (`StorageRepo.emitCollectionChanges` in
|
|
80
|
+
* `packages/db-p2p/src/storage/storage-repo.ts`). So an event may arrive from a later call of
|
|
81
|
+
* the storage, or from a timer, and a held event can fire after the event of a commit that landed
|
|
82
|
+
* later. Every change still produces exactly one event.
|
|
74
83
|
*/
|
|
75
84
|
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void;
|
|
76
85
|
}
|
|
@@ -4,6 +4,7 @@ import { highestStaleAt, isConflictFailure, isExpiryFailure } from "../network/s
|
|
|
4
4
|
import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struct.js";
|
|
5
5
|
import { mergeDurability, withTornBlocks } from "../network/durability.js";
|
|
6
6
|
import { judgeCohortLineage, type CohortLineage, type MemberLineage } from "../network/lineage.js";
|
|
7
|
+
import { assertTransactionFits, transactionBytes, MAX_TRANSACTION_BYTES } from "../network/transaction-size.js";
|
|
7
8
|
import type { ActionTransforms, ActionBlocks, ActionLineage, BlockActionStatus, ITransactor, PendSuccess, CommitSuccess, StaleFailure, WriteFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, GetBlockResult, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests, BlockBaseRevs, WriteDurability } from "../index.js";
|
|
8
9
|
import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
|
|
9
10
|
import { transformForBlockId, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../transform/helpers.js";
|
|
@@ -640,16 +641,13 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
640
641
|
return judgeCohortLineage(members);
|
|
641
642
|
}
|
|
642
643
|
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
// peers that appear in the most clusters — reducing round trips when
|
|
651
|
-
// blocks share cluster members.
|
|
652
|
-
|
|
644
|
+
/**
|
|
645
|
+
* Which coordinator drives which blocks, keyed by peer id string. For each block, find its full
|
|
646
|
+
* cluster, then greedily assign blocks to the peer appearing in the most clusters
|
|
647
|
+
* ({@link bestCoveringPeer}), reducing round trips when blocks share cluster members; a block whose
|
|
648
|
+
* cluster could not be looked up, or that no member covers, goes to `findCoordinator`.
|
|
649
|
+
*/
|
|
650
|
+
private async coordinatorCover(blockIds: readonly BlockId[]): Promise<Map<string, BlockId[]>> {
|
|
653
651
|
// Step 1: Get cluster peer sets for each block
|
|
654
652
|
const blockClusterPeerIds: Map<BlockId, Set<string>> = new Map();
|
|
655
653
|
const fallbackBlocks: BlockId[] = [];
|
|
@@ -699,8 +697,16 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
699
697
|
existing.push(blockId);
|
|
700
698
|
assignments.set(key, existing);
|
|
701
699
|
}
|
|
700
|
+
return assignments;
|
|
701
|
+
}
|
|
702
702
|
|
|
703
|
-
|
|
703
|
+
/** One pend batch per coordinator of {@link coordinatorCover}, carrying that coordinator's share of `transforms`. */
|
|
704
|
+
private async consolidateCoordinators(
|
|
705
|
+
blockIds: BlockId[],
|
|
706
|
+
transforms: Transforms,
|
|
707
|
+
transformForBlock: (payload: Transforms, blockId: BlockId, mergeWith?: Transforms) => Transforms
|
|
708
|
+
): Promise<CoordinatorBatch<Transforms, PendResult>[]> {
|
|
709
|
+
const assignments = await this.coordinatorCover(blockIds);
|
|
704
710
|
const batches: CoordinatorBatch<Transforms, PendResult>[] = [];
|
|
705
711
|
for (const [peerIdStr, consolidatedBlocks] of assignments) {
|
|
706
712
|
const peerId = peerIdFromString(peerIdStr);
|
|
@@ -724,6 +730,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
724
730
|
}
|
|
725
731
|
|
|
726
732
|
async pend(blockAction: PendRequest): Promise<PendResult> {
|
|
733
|
+
// Refused before anything is sent, so no cancel is owed. The whole pend is judged, not each
|
|
734
|
+
// coordinator's share: a share is a subset, so a whole that fits means every share does.
|
|
735
|
+
assertTransactionFits([blockAction]);
|
|
727
736
|
const t0 = Date.now();
|
|
728
737
|
const transformForBlock = (payload: Transforms, blockId: BlockId, mergeWithPayload: Transforms | undefined): Transforms => {
|
|
729
738
|
const filteredTransform = transformForBlockId(payload, blockId);
|
|
@@ -769,11 +778,7 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
769
778
|
this.retryBackoff
|
|
770
779
|
);
|
|
771
780
|
// Cache resolved coordinators for follow-up commit to hit the same peers
|
|
772
|
-
|
|
773
|
-
for (const b of Array.from(allBatches(batches))) {
|
|
774
|
-
this.keyNetwork.recordCoordinator?.(routingKeyForBlock(b.blockId), b.peerId);
|
|
775
|
-
}
|
|
776
|
-
} catch (e) { log('WARN: Failed to record coordinator hint %o', e); }
|
|
781
|
+
this.recordCoordinatorHints(Array.from(allBatches(batches), b => [b.blockId, b.peerId] as const));
|
|
777
782
|
} catch (e) {
|
|
778
783
|
error = e as Error;
|
|
779
784
|
}
|
|
@@ -860,6 +865,116 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
860
865
|
};
|
|
861
866
|
}
|
|
862
867
|
|
|
868
|
+
/**
|
|
869
|
+
* Pends every request as ONE request to the one coordinator covering all their blocks — one consensus
|
|
870
|
+
* record, answered together (`IRepo.pendMany`). Each request goes whole, `soleReservation` and bases
|
|
871
|
+
* exactly as given, with one expiration and the union of their blocks as the blocks that coordinator
|
|
872
|
+
* drives.
|
|
873
|
+
*
|
|
874
|
+
* Returns `undefined` before sending anything when they cannot go as one: their blocks need more than
|
|
875
|
+
* one coordinator by the cover {@link pend} computes ({@link coordinatorCover}), a lookup failed, that
|
|
876
|
+
* coordinator's repo takes no batch, or together they are over the per-transaction size limit (which
|
|
877
|
+
* binds one record, so each may still fit on its own). The caller then pends them its usual way, and
|
|
878
|
+
* each of those pends repeats the lookups and the size check for itself.
|
|
879
|
+
*
|
|
880
|
+
* Answered as {@link pend} answers, cancelling every block first on a failure: a RETURNED refusal is
|
|
881
|
+
* rebuilt by {@link refusalFrom}. A THROW returns `undefined` too: unlike a pend's batches, this one
|
|
882
|
+
* request cannot be re-batched onto other coordinators, so it is the caller's per-request pends —
|
|
883
|
+
* which {@link processBatches} retries elsewhere — that try again. A cancel that did not discharge is
|
|
884
|
+
* owed ({@link owedCancels}), and those pends, of the same action, supersede it on the blocks they
|
|
885
|
+
* write afresh.
|
|
886
|
+
*/
|
|
887
|
+
async pendMany(requests: readonly PendRequest[]): Promise<PendResult | undefined> {
|
|
888
|
+
if (requests.length === 0) throw new Error('pendMany: no requests to pend');
|
|
889
|
+
const actionId = requests[0]!.actionId;
|
|
890
|
+
const blockIds = requests.flatMap(request => blockIdsForTransforms(request.transforms));
|
|
891
|
+
if (transactionBytes(requests) > MAX_TRANSACTION_BYTES) {
|
|
892
|
+
log('pendMany:inapplicable actionId=%s reason=too-large', actionId);
|
|
893
|
+
return undefined;
|
|
894
|
+
}
|
|
895
|
+
const coordinator = await this.solePendCoordinator(blockIds, actionId);
|
|
896
|
+
const repo = coordinator === undefined ? undefined : this.getRepo(coordinator);
|
|
897
|
+
if (coordinator === undefined || !repo?.pendMany) {
|
|
898
|
+
if (coordinator !== undefined) log('pendMany:inapplicable actionId=%s reason=no-batch-repo', actionId);
|
|
899
|
+
return undefined;
|
|
900
|
+
}
|
|
901
|
+
await this.owedCancels.beforePend(actionId, blockIds);
|
|
902
|
+
log('pendMany actionId=%s requests=%d blockIds=%d', actionId, requests.length, blockIds.length);
|
|
903
|
+
|
|
904
|
+
let answer: PendResult;
|
|
905
|
+
try {
|
|
906
|
+
answer = await repo.pendMany(requests, {
|
|
907
|
+
expiration: Date.now() + this.timeoutMs,
|
|
908
|
+
dialTimeoutMs: this.dialTimeoutMs,
|
|
909
|
+
coordinatingBlockIds: blockIds
|
|
910
|
+
});
|
|
911
|
+
} catch (e) {
|
|
912
|
+
log('WARN: batched pend threw; pending each request on its own: actionId=%s peer=%s error=%s',
|
|
913
|
+
actionId, coordinator.toString(), errorMessage(e));
|
|
914
|
+
await this.cancelBatchedPend(coordinator, actionId, blockIds);
|
|
915
|
+
return undefined;
|
|
916
|
+
}
|
|
917
|
+
if (!answer.success) {
|
|
918
|
+
await this.cancelBatchedPend(coordinator, actionId, blockIds);
|
|
919
|
+
return refusalFrom([answer]);
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
const txnCoordinators = this.txnCoordinatorsFor(actionId);
|
|
923
|
+
for (const bid of blockIds) txnCoordinators.set(bid, coordinator);
|
|
924
|
+
this.recordCoordinatorHints([[blockIds[0]!, coordinator]]);
|
|
925
|
+
return { success: true, pending: answer.pending, blockIds, durability: mergeDurability([answer.durability]) };
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/**
|
|
929
|
+
* The one coordinator {@link coordinatorCover} assigns every block in `blockIds` to, or `undefined`
|
|
930
|
+
* when it assigns more than one, or a lookup failed (logged).
|
|
931
|
+
*/
|
|
932
|
+
private async solePendCoordinator(blockIds: BlockId[], actionId: ActionId): Promise<PeerId | undefined> {
|
|
933
|
+
let cover: Map<string, BlockId[]>;
|
|
934
|
+
try {
|
|
935
|
+
cover = await this.coordinatorCover(blockIds);
|
|
936
|
+
} catch (e) {
|
|
937
|
+
log('pendMany:unplanned actionId=%s error=%s', actionId, errorMessage(e));
|
|
938
|
+
return undefined;
|
|
939
|
+
}
|
|
940
|
+
const [only, ...others] = cover.keys();
|
|
941
|
+
if (only === undefined || others.length > 0) {
|
|
942
|
+
log('pendMany:inapplicable actionId=%s coordinators=%d', actionId, cover.size);
|
|
943
|
+
return undefined;
|
|
944
|
+
}
|
|
945
|
+
return peerIdFromString(only);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* Discharges every block a batched pend named, its first round sent to the coordinator the pend went
|
|
950
|
+
* to. Awaited, and never thrown: as in {@link pend}'s failure path, a cancel that did not discharge is
|
|
951
|
+
* logged and owed, and must not replace the pend's own answer.
|
|
952
|
+
*/
|
|
953
|
+
private async cancelBatchedPend(coordinator: PeerId, actionId: ActionId, blockIds: BlockId[]): Promise<void> {
|
|
954
|
+
const seed: CoordinatorBatch<BlockId[], PendResult> = {
|
|
955
|
+
peerId: coordinator,
|
|
956
|
+
blockId: blockIds[0]!,
|
|
957
|
+
payload: blockIds,
|
|
958
|
+
coordinatingBlockIds: blockIds,
|
|
959
|
+
excludedPeers: []
|
|
960
|
+
};
|
|
961
|
+
try {
|
|
962
|
+
await this.cancelBatch([seed], { actionId, blockIds });
|
|
963
|
+
} catch (cancelError) {
|
|
964
|
+
log('WARN: cancel after batched pend did not discharge: %o', cancelError);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
/** Tells the key network which coordinator answered for each block, so later requests reach the same
|
|
969
|
+
* peer. A hint: a failure is logged, never thrown. */
|
|
970
|
+
private recordCoordinatorHints(hints: Iterable<readonly [BlockId, PeerId]>): void {
|
|
971
|
+
try {
|
|
972
|
+
for (const [blockId, peerId] of hints) {
|
|
973
|
+
this.keyNetwork.recordCoordinator?.(routingKeyForBlock(blockId), peerId);
|
|
974
|
+
}
|
|
975
|
+
} catch (e) { log('WARN: Failed to record coordinator hint %o', e); }
|
|
976
|
+
}
|
|
977
|
+
|
|
863
978
|
/**
|
|
864
979
|
* Discharges every pending record `actionRef` left behind, and THROWS if it could not.
|
|
865
980
|
*
|
|
@@ -967,6 +1082,47 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
967
1082
|
: refusalFrom([result]);
|
|
968
1083
|
}
|
|
969
1084
|
|
|
1085
|
+
/**
|
|
1086
|
+
* Commits every request as ONE request to the one coordinator covering all their blocks — one
|
|
1087
|
+
* consensus record, answered together (`IRepo.commitMany`). Each request lists its tail first and its
|
|
1088
|
+
* other blocks after, as {@link commitInOneRound} sends one action's, and is built where every commit
|
|
1089
|
+
* request is ({@link repoCommitRequest}), so its digests name only its own blocks.
|
|
1090
|
+
*
|
|
1091
|
+
* Returns `undefined` when they cannot go as one — their blocks need more than one coordinator
|
|
1092
|
+
* ({@link soleCommitCoordinator}, which prefers what {@link pendMany} cached), or that coordinator's
|
|
1093
|
+
* repo takes no batch — and when the one request THREW: the caller's per-request commits then re-send
|
|
1094
|
+
* everything, and whatever the thrown request landed is this action's own revision, which storage
|
|
1095
|
+
* treats as already done (`isOwnRevision`), the reasoning {@link commitInOneRound}'s fallback rests on.
|
|
1096
|
+
* A RETURNED answer is the batch's verdict.
|
|
1097
|
+
*/
|
|
1098
|
+
async commitMany(requests: readonly CommitRequest[]): Promise<CommitResult | undefined> {
|
|
1099
|
+
if (requests.length === 0) throw new Error('commitMany: no requests to commit');
|
|
1100
|
+
const actionId = requests[0]!.actionId;
|
|
1101
|
+
const blockLists = requests.map(tailFirst);
|
|
1102
|
+
const coordinator = await this.soleCommitCoordinator(blockLists.flat(), actionId);
|
|
1103
|
+
const repo = coordinator === undefined ? undefined : this.getRepo(coordinator);
|
|
1104
|
+
if (coordinator === undefined || !repo?.commitMany) {
|
|
1105
|
+
if (coordinator !== undefined) log('commitMany:inapplicable actionId=%s reason=no-batch-repo', actionId);
|
|
1106
|
+
return undefined;
|
|
1107
|
+
}
|
|
1108
|
+
log('commitMany actionId=%s requests=%d', actionId, requests.length);
|
|
1109
|
+
|
|
1110
|
+
let answer: CommitResult;
|
|
1111
|
+
try {
|
|
1112
|
+
answer = await repo.commitMany(
|
|
1113
|
+
requests.map((request, i) => repoCommitRequest(blockLists[i]!, request)),
|
|
1114
|
+
{ expiration: Date.now() + this.timeoutMs, dialTimeoutMs: this.dialTimeoutMs }
|
|
1115
|
+
);
|
|
1116
|
+
} catch (e) {
|
|
1117
|
+
log('WARN: batched commit threw; committing each request on its own: actionId=%s peer=%s error=%s',
|
|
1118
|
+
actionId, coordinator.toString(), errorMessage(e));
|
|
1119
|
+
return undefined;
|
|
1120
|
+
}
|
|
1121
|
+
return answer.success
|
|
1122
|
+
? { success: true, durability: mergeDurability([answer.durability]) }
|
|
1123
|
+
: refusalFrom([answer]);
|
|
1124
|
+
}
|
|
1125
|
+
|
|
970
1126
|
/**
|
|
971
1127
|
* The one coordinator covering every block in `blockIds` — resolved as {@link commitBlocks}
|
|
972
1128
|
* resolves them, preferring what the pend already resolved — or `undefined` when they need more
|
|
@@ -1169,19 +1325,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1169
1325
|
return stale.length === 0 ? undefined : refusalFrom(stale.map(b => b.request!.response! as WriteFailure));
|
|
1170
1326
|
}
|
|
1171
1327
|
|
|
1172
|
-
/**
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
* narrowed to `blockIds` here, at send time, never up front: each request becomes its own cluster
|
|
1176
|
-
* record, so shipping the whole map would make one cohort sign for blocks it is not responsible
|
|
1177
|
-
* for; and `processBatches` re-batches failed blocks onto different coordinators, so only a
|
|
1178
|
-
* send-time subset stays correct across retries.
|
|
1179
|
-
*/
|
|
1180
|
-
private sendCommit(peerId: PeerId, blockIds: BlockId[], { actionId, rev, tailId, blockDigests }: RepoCommitRequest, expiration: number): Promise<CommitResult> {
|
|
1181
|
-
return this.getRepo(peerId).commit(
|
|
1182
|
-
{ actionId, blockIds, rev, tailId, ...digestsFor(blockDigests, blockIds) },
|
|
1183
|
-
{ expiration, dialTimeoutMs: this.dialTimeoutMs }
|
|
1184
|
-
);
|
|
1328
|
+
/** Sends `request`'s commit of `blockIds` to `peerId` — shared by {@link commitInOneRound} and {@link commitBlocks}. */
|
|
1329
|
+
private sendCommit(peerId: PeerId, blockIds: BlockId[], request: RepoCommitRequest, expiration: number): Promise<CommitResult> {
|
|
1330
|
+
return this.getRepo(peerId).commit(repoCommitRequest(blockIds, request), { expiration, dialTimeoutMs: this.dialTimeoutMs });
|
|
1185
1331
|
}
|
|
1186
1332
|
|
|
1187
1333
|
/** Attempts to commit a set of blocks, and handles failures and errors. */
|
|
@@ -1581,6 +1727,23 @@ function refusalFrom(responses: WriteFailure[]): WriteFailure {
|
|
|
1581
1727
|
};
|
|
1582
1728
|
}
|
|
1583
1729
|
|
|
1730
|
+
/**
|
|
1731
|
+
* The one place a commit request to a coordinator is built, for `blockIds` of `request`'s action —
|
|
1732
|
+
* every commit {@link NetworkTransactor} sends, batched or not. `blockDigests` arrives as the action's
|
|
1733
|
+
* FULL declaration map and is narrowed to `blockIds` here, at send time, never up front: each request
|
|
1734
|
+
* becomes its own cluster record, or its own operation of one, so shipping the whole map would make a
|
|
1735
|
+
* cohort sign for blocks the request is not committing; and `processBatches` re-batches failed blocks
|
|
1736
|
+
* onto different coordinators, so only a send-time subset stays correct across retries.
|
|
1737
|
+
*/
|
|
1738
|
+
function repoCommitRequest(blockIds: BlockId[], { actionId, rev, tailId, blockDigests }: RepoCommitRequest): RepoCommitRequest {
|
|
1739
|
+
return { actionId, blockIds, rev, tailId, ...digestsFor(blockDigests, blockIds) };
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
/** `request`'s blocks with its log tail first, each once — the order a coordinator commits them in. */
|
|
1743
|
+
function tailFirst(request: CommitRequest): BlockId[] {
|
|
1744
|
+
return [request.tailId, ...new Set(request.blockIds)].filter((blockId, i) => i === 0 || blockId !== request.tailId);
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1584
1747
|
/** The entries of `all` whose ids appear in `batchBlockIds` — the per-batch narrowing both
|
|
1585
1748
|
* per-block declarations share ({@link digestsFor}, {@link baseRevsFor}). */
|
|
1586
1749
|
function subsetOf<T>(all: Record<BlockId, T>, batchBlockIds: BlockId[]): Record<BlockId, T> {
|
|
@@ -58,4 +58,20 @@ export type ITransactor = {
|
|
|
58
58
|
unable to establish the outcome, which it reports as such rather than guessing either way.
|
|
59
59
|
*/
|
|
60
60
|
getLineage?(ref: ActionBlocks & { rev: number }): Promise<ActionLineage>;
|
|
61
|
+
|
|
62
|
+
/** Pend every request (one action's, disjoint blocks) as ONE request to one coordinator, answered
|
|
63
|
+
together — one consensus record for the whole set (`IRepo.pendMany`).
|
|
64
|
+
- `undefined`: they could not go as one — nothing of them is left pended by this call, and the
|
|
65
|
+
caller pends them its usual way. Never a verdict.
|
|
66
|
+
- Optional: without it, or without {@link commitMany} beside it, a writer pends each request on its
|
|
67
|
+
own — a batched pend promises its commit goes as one request too (`PendRequest.soleReservation`).
|
|
68
|
+
*/
|
|
69
|
+
pendMany?(requests: readonly PendRequest[]): Promise<PendResult | undefined>;
|
|
70
|
+
|
|
71
|
+
/** Commit every request as ONE request to one coordinator, answered together (`IRepo.commitMany`).
|
|
72
|
+
- `undefined`: they could not go as one, or the one request threw; the caller commits them its
|
|
73
|
+
usual way. Never a verdict.
|
|
74
|
+
- Optional, like {@link pendMany}.
|
|
75
|
+
*/
|
|
76
|
+
commitMany?(requests: readonly CommitRequest[]): Promise<CommitResult | undefined>;
|
|
61
77
|
}
|
package/src/transform/digest.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { BlockId, IBlock } from "../index.js";
|
|
2
2
|
import type { BlockBaseRevs, BlockContentDigests } from "../network/struct.js";
|
|
3
|
-
import {
|
|
3
|
+
import { canonicalBlockHashAndBytes } from "../blocks/helpers.js";
|
|
4
4
|
import { createLogger } from "../logger.js";
|
|
5
5
|
import { isRecordEmpty } from "../utility/is-record-empty.js";
|
|
6
6
|
import type { Tracker } from "./tracker.js";
|
|
@@ -32,16 +32,49 @@ export async function computeBlockContentDigests<T extends IBlock>(
|
|
|
32
32
|
tracker: Tracker<T>,
|
|
33
33
|
blockIds: BlockId[]
|
|
34
34
|
): Promise<BlockContentDigests> {
|
|
35
|
+
return (await describeTouchedBlocks(tracker, blockIds)).digests;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** What a writer can say about the blocks a write touches, from one local materialization of each. */
|
|
39
|
+
export type TouchedBlocks = {
|
|
40
|
+
/** The commit's declaration of what each block will contain ({@link computeBlockContentDigests}). */
|
|
41
|
+
digests: BlockContentDigests;
|
|
42
|
+
/** What each of the same blocks' post-write content encodes to, by `jsonEncodedBytes` — the measure
|
|
43
|
+
* `oversizedBlock` judges against `MAX_BLOCK_BYTES`. */
|
|
44
|
+
bytes: Record<BlockId, number>;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
/** {@link computeBlockContentDigests}, plus the size of every block it declares, taken from the
|
|
48
|
+
* serialization the digest already hashes — the one pass both write paths make over their touched
|
|
49
|
+
* blocks before anything is sent. Judging the sizes is the caller's separate step (`oversizedBlock`),
|
|
50
|
+
* so declaring content still never breaks committing it.
|
|
51
|
+
*
|
|
52
|
+
* A block this pass cannot materialize is in neither map: neither declared nor measured. Those are the
|
|
53
|
+
* omissions listed at {@link computeBlockContentDigests} — a deleted block, a blind update to a block
|
|
54
|
+
* this node never read, an update whose base was evicted before it was staged, a base that moved under
|
|
55
|
+
* its staged operations — and a base the staged operations would not apply to (`peekOrSkip`). So the
|
|
56
|
+
* per-block size bound has the same gaps as the declaration: a block that outgrows `MAX_BLOCK_BYTES`
|
|
57
|
+
* through an update this node cannot materialize is not refused by the writer. */
|
|
58
|
+
// NOTE: if a block is ever seen past MAX_BLOCK_BYTES after growing through one of those gaps, measure an
|
|
59
|
+
// unmeasured update-only block here by reading its base — the one exception that would make to a commit
|
|
60
|
+
// never paying a network read to describe itself. Its digest can stay undeclared.
|
|
61
|
+
export async function describeTouchedBlocks<T extends IBlock>(
|
|
62
|
+
tracker: Tracker<T>,
|
|
63
|
+
blockIds: BlockId[]
|
|
64
|
+
): Promise<TouchedBlocks> {
|
|
35
65
|
const digests: BlockContentDigests = {};
|
|
66
|
+
const bytes: Record<BlockId, number> = {};
|
|
36
67
|
for (const id of blockIds) {
|
|
37
68
|
const peeked = peekOrSkip(tracker, id);
|
|
38
69
|
if (!peeked) continue;
|
|
70
|
+
const described = await canonicalBlockHashAndBytes(peeked.block);
|
|
39
71
|
digests[id] = {
|
|
40
|
-
digest:
|
|
72
|
+
digest: described.hash,
|
|
41
73
|
...(peeked.baseRev !== undefined ? { baseRev: peeked.baseRev } : {}),
|
|
42
74
|
};
|
|
75
|
+
bytes[id] = described.bytes;
|
|
43
76
|
}
|
|
44
|
-
return digests;
|
|
77
|
+
return { digests, bytes };
|
|
45
78
|
}
|
|
46
79
|
|
|
47
80
|
/** Wraps `blockDigests` so it spreads onto a request only when there is something to declare. The
|