@peerbit/shared-log 13.2.28 → 13.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +11 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +244 -219
- package/dist/src/index.js.map +1 -1
- package/package.json +10 -7
- package/src/index.ts +345 -296
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@peerbit/shared-log",
|
|
3
|
-
"version": "13.2.
|
|
3
|
+
"version": "13.2.30",
|
|
4
4
|
"description": "Shared log",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"type": "module",
|
|
@@ -63,22 +63,22 @@
|
|
|
63
63
|
"pino": "^9.4.0",
|
|
64
64
|
"uint8arrays": "^5.1.0",
|
|
65
65
|
"@peerbit/any-store": "2.2.14",
|
|
66
|
-
"@peerbit/blocks-interface": "2.1.5",
|
|
67
66
|
"@peerbit/cache": "3.1.1",
|
|
68
|
-
"@peerbit/crypto": "3.1.5",
|
|
69
67
|
"@peerbit/blocks": "4.2.10",
|
|
68
|
+
"@peerbit/blocks-interface": "2.1.5",
|
|
69
|
+
"@peerbit/crypto": "3.1.5",
|
|
70
70
|
"@peerbit/diagnostics": "0.0.1",
|
|
71
71
|
"@peerbit/indexer-sqlite3": "3.0.14",
|
|
72
|
+
"@peerbit/indexer-interface": "3.0.9",
|
|
72
73
|
"@peerbit/log": "6.2.16",
|
|
73
74
|
"@peerbit/logger": "2.0.1",
|
|
74
75
|
"@peerbit/pubsub": "5.4.0",
|
|
75
|
-
"@peerbit/indexer-interface": "3.0.9",
|
|
76
|
-
"@peerbit/stream-interface": "6.0.15",
|
|
77
76
|
"@peerbit/pubsub-interface": "5.2.0",
|
|
78
77
|
"@peerbit/riblt": "1.2.0",
|
|
79
|
-
"@peerbit/
|
|
78
|
+
"@peerbit/rpc": "6.1.16",
|
|
79
|
+
"@peerbit/stream-interface": "6.0.15",
|
|
80
80
|
"@peerbit/program": "6.0.48",
|
|
81
|
-
"@peerbit/
|
|
81
|
+
"@peerbit/time": "3.0.1"
|
|
82
82
|
},
|
|
83
83
|
"optionalDependencies": {
|
|
84
84
|
"@peerbit/native-backbone": "0.2.5",
|
|
@@ -112,6 +112,9 @@
|
|
|
112
112
|
"benchmark:network-preset-e2e": "node --loader ts-node/esm ./benchmark/network-preset-e2e.ts",
|
|
113
113
|
"benchmark:receive-prune": "node --loader ts-node/esm ./benchmark/receive-prune.ts",
|
|
114
114
|
"benchmark:sync-catchup": "node --loader ts-node/esm ./benchmark/sync-catchup.ts",
|
|
115
|
+
"benchmark:sync-peer-state": "node --loader ts-node/esm ./benchmark/sync-peer-state.ts",
|
|
116
|
+
"benchmark:sync-peer-state:compare": "node ./benchmark/sync-peer-state-compare.mjs",
|
|
117
|
+
"benchmark:sync-peer-state:compare:test": "node --test ./benchmark/sync-peer-state-compare.test.mjs",
|
|
115
118
|
"benchmark:sync-phase-profile": "node --loader ts-node/esm ./benchmark/sync-phase-profile.ts",
|
|
116
119
|
"test": "aegir test --target node",
|
|
117
120
|
"test:shared-log-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!u32-simple replication degree commit options control per commmit put before join converges under deterministic pubsub chaos$)(?!.*(does not fetch missing entries from remotes|will prune once reaching max replicas|repairs when joiner request responses are dropped|time out when pending IHave are never resolved|does not confirm checked prune from a shallow-only entry|replication degree update ))(append delivery options |join |replicate |(u32-simple|u64-iblt) (replication references |replication replication (one way|two way) |redundancy only sends entries once|canReplicate |replication degree |start/stop replicate on connect|sync ))' -- --forbid-pending",
|
package/src/index.ts
CHANGED
|
@@ -16786,6 +16786,339 @@ export class SharedLog<
|
|
|
16786
16786
|
});
|
|
16787
16787
|
}
|
|
16788
16788
|
|
|
16789
|
+
/**
|
|
16790
|
+
* Normalize a raw exchange-heads receive into the regular exchange message
|
|
16791
|
+
* consumed by the rest of the receive path. An undefined result means the
|
|
16792
|
+
* message was fully handled (all heads were already present or the native
|
|
16793
|
+
* receive plan dropped every head).
|
|
16794
|
+
*
|
|
16795
|
+
* This helper intentionally runs inside `onMessage`'s shared try/finally
|
|
16796
|
+
* envelope so receive errors keep their existing classification and a
|
|
16797
|
+
* wire-backed message keeps its single outer stash-release boundary.
|
|
16798
|
+
*/
|
|
16799
|
+
private async materializeRawReceiveMessage(
|
|
16800
|
+
msg: RawExchangeHeadsMessage,
|
|
16801
|
+
properties: {
|
|
16802
|
+
from: PublicSignKey;
|
|
16803
|
+
stashBackedRawMessage?: StashBackedRawExchangeHeadsMessage;
|
|
16804
|
+
syncProfile?: SyncProfileFn;
|
|
16805
|
+
receiveOwnershipRevision: number;
|
|
16806
|
+
},
|
|
16807
|
+
): Promise<
|
|
16808
|
+
| {
|
|
16809
|
+
message: ExchangeHeadsMessage<any>;
|
|
16810
|
+
preparedSelection: NativeBackboneRawReceiveSelectionPlan | undefined;
|
|
16811
|
+
}
|
|
16812
|
+
| undefined
|
|
16813
|
+
> {
|
|
16814
|
+
const {
|
|
16815
|
+
from: rawFrom,
|
|
16816
|
+
stashBackedRawMessage,
|
|
16817
|
+
syncProfile,
|
|
16818
|
+
receiveOwnershipRevision,
|
|
16819
|
+
} = properties;
|
|
16820
|
+
const fromIsSelf = rawFrom.equals(this.node.identity.publicKey);
|
|
16821
|
+
if (syncProfile && !stashBackedRawMessage) {
|
|
16822
|
+
// Per-message JS-side entry decode: the heads were
|
|
16823
|
+
// borsh-decoded in TS (regular RPC path) instead of being
|
|
16824
|
+
// resolved from the native wire stash. Zero on the fused
|
|
16825
|
+
// hot path.
|
|
16826
|
+
emitSyncProfileEvent(syncProfile, {
|
|
16827
|
+
name: "sharedLog.rawReceive.jsEntryDecode",
|
|
16828
|
+
component: "shared-log",
|
|
16829
|
+
entries: msg.heads.length,
|
|
16830
|
+
messages: 1,
|
|
16831
|
+
});
|
|
16832
|
+
}
|
|
16833
|
+
const rawExistingStartedAt = syncProfileStart(syncProfile);
|
|
16834
|
+
const rawExistingHashes = await this.log.hasMany(
|
|
16835
|
+
msg.heads.map((head) => head.hash),
|
|
16836
|
+
);
|
|
16837
|
+
if (syncProfile) {
|
|
16838
|
+
emitSyncProfileDuration(syncProfile, rawExistingStartedAt, {
|
|
16839
|
+
name: "sharedLog.rawReceive.existingHeads",
|
|
16840
|
+
component: "shared-log",
|
|
16841
|
+
entries: msg.heads.length,
|
|
16842
|
+
messages: 1,
|
|
16843
|
+
});
|
|
16844
|
+
}
|
|
16845
|
+
const rawMissingHeads = [];
|
|
16846
|
+
const rawConfirmedHashes = new Set<string>();
|
|
16847
|
+
let rawMissingBytes = 0;
|
|
16848
|
+
for (const head of msg.heads) {
|
|
16849
|
+
if (rawExistingHashes.has(head.hash)) {
|
|
16850
|
+
rawConfirmedHashes.add(head.hash);
|
|
16851
|
+
} else {
|
|
16852
|
+
rawMissingHeads.push(head);
|
|
16853
|
+
rawMissingBytes += getRawExchangeHeadByteLength(head);
|
|
16854
|
+
}
|
|
16855
|
+
}
|
|
16856
|
+
if (rawConfirmedHashes.size > 0 && !fromIsSelf) {
|
|
16857
|
+
const rawConfirmStartedAt = syncProfileStart(syncProfile);
|
|
16858
|
+
this.markEntriesKnownByPeer(rawConfirmedHashes, rawFrom.hashcode());
|
|
16859
|
+
await this.sendRepairConfirmation(rawFrom, rawConfirmedHashes);
|
|
16860
|
+
if (syncProfile) {
|
|
16861
|
+
emitSyncProfileDuration(syncProfile, rawConfirmStartedAt, {
|
|
16862
|
+
name: "sharedLog.rawReceive.confirmExisting",
|
|
16863
|
+
component: "shared-log",
|
|
16864
|
+
entries: rawConfirmedHashes.size,
|
|
16865
|
+
messages: 1,
|
|
16866
|
+
});
|
|
16867
|
+
}
|
|
16868
|
+
}
|
|
16869
|
+
if (rawMissingHeads.length === 0) {
|
|
16870
|
+
return undefined;
|
|
16871
|
+
}
|
|
16872
|
+
const rawIsRepairHint =
|
|
16873
|
+
(msg.reserved[0] & EXCHANGE_HEADS_REPAIR_HINT) !== 0;
|
|
16874
|
+
const rawPrepareVerifySetting =
|
|
16875
|
+
this._logProperties?.sync?.rawExchangeHeadsVerifySignaturesDuringPrepare;
|
|
16876
|
+
// A program-level canAppend hook must observe every entry before
|
|
16877
|
+
// it commits, so the native join commit (which validates and
|
|
16878
|
+
// commits entirely in wasm) is not used for programs that
|
|
16879
|
+
// register one; those joins run through the lower-log batch
|
|
16880
|
+
// join where the hook fires per entry.
|
|
16881
|
+
const programCanAppend = !!this._logProperties?.canAppend;
|
|
16882
|
+
const canVerifyPreparedRawReceiveOnCommit =
|
|
16883
|
+
!programCanAppend &&
|
|
16884
|
+
!!this._nativeBackbone?.graph.commitVerifiedPreparedRawReceiveJoinBatch;
|
|
16885
|
+
const canDeferRawReceiveVerificationUntilNativeSelection =
|
|
16886
|
+
!rawIsRepairHint &&
|
|
16887
|
+
!!this._nativeBackbone?.verifyPreparedRawReceiveEntries &&
|
|
16888
|
+
!this._isReplicating &&
|
|
16889
|
+
!this.keep &&
|
|
16890
|
+
!this.closed &&
|
|
16891
|
+
!!this.syncronizer.onReceivedEntryHashes &&
|
|
16892
|
+
rawMissingHeads.every((head) => head.gidRefrences.length === 0);
|
|
16893
|
+
const verifyNativeBackboneSignaturesDuringPrepare =
|
|
16894
|
+
rawPrepareVerifySetting === true ||
|
|
16895
|
+
(rawPrepareVerifySetting !== false &&
|
|
16896
|
+
(canDeferRawReceiveVerificationUntilNativeSelection ||
|
|
16897
|
+
(this._isReplicating &&
|
|
16898
|
+
!rawIsRepairHint &&
|
|
16899
|
+
!canVerifyPreparedRawReceiveOnCommit)));
|
|
16900
|
+
const deferNativeBackboneSignatureVerificationUntilSelection =
|
|
16901
|
+
verifyNativeBackboneSignaturesDuringPrepare &&
|
|
16902
|
+
canDeferRawReceiveVerificationUntilNativeSelection;
|
|
16903
|
+
const deferNativeBackboneSignatureVerificationUntilCommit =
|
|
16904
|
+
deferNativeBackboneSignatureVerificationUntilSelection &&
|
|
16905
|
+
!programCanAppend &&
|
|
16906
|
+
!!this._nativeBackbone?.graph.commitVerifiedPreparedRawReceiveJoinBatch;
|
|
16907
|
+
let rawPreparedReceiveSelectionValue:
|
|
16908
|
+
| NativeBackboneRawReceiveSelectionPlan
|
|
16909
|
+
| undefined;
|
|
16910
|
+
let rawPreparedReceiveSelection:
|
|
16911
|
+
| Promise<NativeBackboneRawReceiveSelectionPlan | undefined>
|
|
16912
|
+
| undefined;
|
|
16913
|
+
const getRawPreparedReceiveSelection = async (
|
|
16914
|
+
heads: RawEntryWithRefs[],
|
|
16915
|
+
hashes: string[],
|
|
16916
|
+
) => {
|
|
16917
|
+
if (rawPreparedReceiveSelectionValue) {
|
|
16918
|
+
return rawPreparedReceiveSelectionValue;
|
|
16919
|
+
}
|
|
16920
|
+
rawPreparedReceiveSelection ??=
|
|
16921
|
+
this.planNativePreparedRawReceiveSelection({
|
|
16922
|
+
heads,
|
|
16923
|
+
hashes,
|
|
16924
|
+
from: rawFrom,
|
|
16925
|
+
});
|
|
16926
|
+
rawPreparedReceiveSelectionValue = await rawPreparedReceiveSelection;
|
|
16927
|
+
return rawPreparedReceiveSelectionValue;
|
|
16928
|
+
};
|
|
16929
|
+
// Receive fusion: when this message was resolved from the wire
|
|
16930
|
+
// stash, the prepared receive reads entry block bytes straight
|
|
16931
|
+
// out of wasm memory (indexed into the stashed frame) instead
|
|
16932
|
+
// of copying a JS blocks array across the boundary.
|
|
16933
|
+
const rawStashIndexes = stashBackedRawMessage
|
|
16934
|
+
? getRawExchangeHeadStashIndexes(rawMissingHeads)
|
|
16935
|
+
: undefined;
|
|
16936
|
+
const prepareNativeBackboneExpectedColumns =
|
|
16937
|
+
stashBackedRawMessage && rawStashIndexes
|
|
16938
|
+
? ({
|
|
16939
|
+
hashes,
|
|
16940
|
+
verifySignatures,
|
|
16941
|
+
}: {
|
|
16942
|
+
hashes: string[];
|
|
16943
|
+
verifySignatures: boolean;
|
|
16944
|
+
}) => {
|
|
16945
|
+
const backbone = this._nativeBackbone;
|
|
16946
|
+
const wireSession = this._wireSyncSession;
|
|
16947
|
+
if (!backbone || !wireSession) {
|
|
16948
|
+
return undefined;
|
|
16949
|
+
}
|
|
16950
|
+
try {
|
|
16951
|
+
return backbone.prepareStashedRawReceiveExpectedColumnsBatch(
|
|
16952
|
+
wireSession,
|
|
16953
|
+
stashBackedRawMessage.messageId,
|
|
16954
|
+
rawStashIndexes,
|
|
16955
|
+
hashes,
|
|
16956
|
+
{ verifySignatures },
|
|
16957
|
+
);
|
|
16958
|
+
} catch {
|
|
16959
|
+
return undefined;
|
|
16960
|
+
}
|
|
16961
|
+
}
|
|
16962
|
+
: undefined;
|
|
16963
|
+
const prepareNativeBackboneExpectedColumnsAndSelection = rawIsRepairHint
|
|
16964
|
+
? undefined
|
|
16965
|
+
: async ({
|
|
16966
|
+
blocks,
|
|
16967
|
+
hashes,
|
|
16968
|
+
verifySignatures,
|
|
16969
|
+
}: {
|
|
16970
|
+
blocks: () => Uint8Array[];
|
|
16971
|
+
hashes: string[];
|
|
16972
|
+
verifySignatures: boolean;
|
|
16973
|
+
}) => {
|
|
16974
|
+
if (
|
|
16975
|
+
verifySignatures ||
|
|
16976
|
+
!canDeferRawReceiveVerificationUntilNativeSelection
|
|
16977
|
+
) {
|
|
16978
|
+
return undefined;
|
|
16979
|
+
}
|
|
16980
|
+
try {
|
|
16981
|
+
const replicaOptions = {
|
|
16982
|
+
minReplicas: this.replicas.min?.getValue(this) || 1,
|
|
16983
|
+
maxReplicas: this.replicas.max?.getValue(this),
|
|
16984
|
+
};
|
|
16985
|
+
const leaderSelectionContext =
|
|
16986
|
+
await this.createLeaderSelectionContext();
|
|
16987
|
+
const prepareOptions = {
|
|
16988
|
+
verifySignatures: false as const,
|
|
16989
|
+
...replicaOptions,
|
|
16990
|
+
leaderOptions: this.createNativeLeaderOptions(
|
|
16991
|
+
leaderSelectionContext,
|
|
16992
|
+
),
|
|
16993
|
+
fromHash: rawFrom.hashcode(),
|
|
16994
|
+
};
|
|
16995
|
+
let prepared:
|
|
16996
|
+
| ReturnType<
|
|
16997
|
+
NativePeerbitBackbone["prepareRawReceiveExpectedColumnsAndSelectionBatch"]
|
|
16998
|
+
>
|
|
16999
|
+
| undefined;
|
|
17000
|
+
const wireSession = this._wireSyncSession;
|
|
17001
|
+
if (
|
|
17002
|
+
stashBackedRawMessage &&
|
|
17003
|
+
rawStashIndexes &&
|
|
17004
|
+
wireSession &&
|
|
17005
|
+
this._nativeBackbone
|
|
17006
|
+
) {
|
|
17007
|
+
prepared =
|
|
17008
|
+
this._nativeBackbone.prepareStashedRawReceiveExpectedColumnsAndSelectionBatch(
|
|
17009
|
+
wireSession,
|
|
17010
|
+
stashBackedRawMessage.messageId,
|
|
17011
|
+
rawStashIndexes,
|
|
17012
|
+
hashes,
|
|
17013
|
+
prepareOptions,
|
|
17014
|
+
);
|
|
17015
|
+
}
|
|
17016
|
+
if (
|
|
17017
|
+
!prepared &&
|
|
17018
|
+
this._nativeBackbone
|
|
17019
|
+
?.prepareRawReceiveExpectedColumnsAndSelectionBatch
|
|
17020
|
+
) {
|
|
17021
|
+
prepared =
|
|
17022
|
+
this._nativeBackbone.prepareRawReceiveExpectedColumnsAndSelectionBatch(
|
|
17023
|
+
blocks(),
|
|
17024
|
+
hashes,
|
|
17025
|
+
prepareOptions,
|
|
17026
|
+
);
|
|
17027
|
+
}
|
|
17028
|
+
if (!prepared) {
|
|
17029
|
+
return undefined;
|
|
17030
|
+
}
|
|
17031
|
+
rawPreparedReceiveSelectionValue = prepared.selection;
|
|
17032
|
+
rawPreparedReceiveSelection = Promise.resolve(
|
|
17033
|
+
rawPreparedReceiveSelectionValue,
|
|
17034
|
+
);
|
|
17035
|
+
return { columns: prepared.columns };
|
|
17036
|
+
} catch {
|
|
17037
|
+
this.throwIfReplicationOwnershipPoisoned();
|
|
17038
|
+
return undefined;
|
|
17039
|
+
}
|
|
17040
|
+
};
|
|
17041
|
+
const rawMaterializeStartedAt = syncProfileStart(syncProfile);
|
|
17042
|
+
const materializedRawMessage =
|
|
17043
|
+
await materializeVerifiedRawExchangeHeadsMessage(
|
|
17044
|
+
new RawExchangeHeadsMessage({
|
|
17045
|
+
heads: rawMissingHeads,
|
|
17046
|
+
reserved: msg.reserved,
|
|
17047
|
+
}),
|
|
17048
|
+
this.log,
|
|
17049
|
+
syncProfile,
|
|
17050
|
+
{
|
|
17051
|
+
nativeBackbone: this._nativeBackbone,
|
|
17052
|
+
verifyNativeBackboneSignaturesDuringPrepare:
|
|
17053
|
+
verifyNativeBackboneSignaturesDuringPrepare,
|
|
17054
|
+
deferNativeBackboneSignatureVerificationUntilSelection:
|
|
17055
|
+
deferNativeBackboneSignatureVerificationUntilSelection,
|
|
17056
|
+
deferNativeBackboneSignatureVerificationUntilCommit:
|
|
17057
|
+
deferNativeBackboneSignatureVerificationUntilCommit,
|
|
17058
|
+
prepareNativeBackboneExpectedColumnsAndSelection:
|
|
17059
|
+
prepareNativeBackboneExpectedColumnsAndSelection,
|
|
17060
|
+
prepareNativeBackboneExpectedColumns:
|
|
17061
|
+
prepareNativeBackboneExpectedColumns,
|
|
17062
|
+
tryPreparedRawReceiveFastDrop: rawIsRepairHint
|
|
17063
|
+
? undefined
|
|
17064
|
+
: async ({ heads, hashes }) =>
|
|
17065
|
+
this.tryFastDropPreparedRawReceive({
|
|
17066
|
+
heads,
|
|
17067
|
+
hashes,
|
|
17068
|
+
from: rawFrom,
|
|
17069
|
+
fromIsSelf,
|
|
17070
|
+
syncProfile,
|
|
17071
|
+
selection: await getRawPreparedReceiveSelection(
|
|
17072
|
+
heads,
|
|
17073
|
+
hashes,
|
|
17074
|
+
),
|
|
17075
|
+
receiveOwnershipRevision,
|
|
17076
|
+
}),
|
|
17077
|
+
selectPreparedRawReceiveHashes: rawIsRepairHint
|
|
17078
|
+
? undefined
|
|
17079
|
+
: async ({ heads, hashes }) =>
|
|
17080
|
+
this.selectNativePreparedRawReceiveHashes({
|
|
17081
|
+
heads,
|
|
17082
|
+
hashes,
|
|
17083
|
+
from: rawFrom,
|
|
17084
|
+
fromIsSelf,
|
|
17085
|
+
syncProfile,
|
|
17086
|
+
selection: await getRawPreparedReceiveSelection(
|
|
17087
|
+
heads,
|
|
17088
|
+
hashes,
|
|
17089
|
+
),
|
|
17090
|
+
receiveOwnershipRevision,
|
|
17091
|
+
}),
|
|
17092
|
+
},
|
|
17093
|
+
);
|
|
17094
|
+
if (materializedRawMessage === undefined) {
|
|
17095
|
+
if (syncProfile) {
|
|
17096
|
+
emitSyncProfileDuration(syncProfile, rawMaterializeStartedAt, {
|
|
17097
|
+
name: "sharedLog.rawReceive.materialize",
|
|
17098
|
+
component: "shared-log",
|
|
17099
|
+
entries: rawMissingHeads.length,
|
|
17100
|
+
bytes: rawMissingBytes,
|
|
17101
|
+
messages: 1,
|
|
17102
|
+
details: { nativeFastDropEarly: true },
|
|
17103
|
+
});
|
|
17104
|
+
}
|
|
17105
|
+
return undefined;
|
|
17106
|
+
}
|
|
17107
|
+
if (syncProfile) {
|
|
17108
|
+
emitSyncProfileDuration(syncProfile, rawMaterializeStartedAt, {
|
|
17109
|
+
name: "sharedLog.rawReceive.materialize",
|
|
17110
|
+
component: "shared-log",
|
|
17111
|
+
entries: rawMissingHeads.length,
|
|
17112
|
+
bytes: rawMissingBytes,
|
|
17113
|
+
messages: 1,
|
|
17114
|
+
});
|
|
17115
|
+
}
|
|
17116
|
+
return {
|
|
17117
|
+
message: materializedRawMessage,
|
|
17118
|
+
preparedSelection: rawPreparedReceiveSelectionValue,
|
|
17119
|
+
};
|
|
17120
|
+
}
|
|
17121
|
+
|
|
16789
17122
|
// Callback for receiving a message from the network
|
|
16790
17123
|
async onMessage(
|
|
16791
17124
|
msg: TransportMessage,
|
|
@@ -16863,307 +17196,23 @@ export class SharedLog<
|
|
|
16863
17196
|
| NativeBackboneRawReceiveSelectionPlan
|
|
16864
17197
|
| undefined;
|
|
16865
17198
|
if (msg instanceof RawExchangeHeadsMessage) {
|
|
16866
|
-
const
|
|
16867
|
-
|
|
16868
|
-
|
|
16869
|
-
|
|
16870
|
-
|
|
16871
|
-
// resolved from the native wire stash. Zero on the fused
|
|
16872
|
-
// hot path.
|
|
16873
|
-
emitSyncProfileEvent(syncProfile, {
|
|
16874
|
-
name: "sharedLog.rawReceive.jsEntryDecode",
|
|
16875
|
-
component: "shared-log",
|
|
16876
|
-
entries: msg.heads.length,
|
|
16877
|
-
messages: 1,
|
|
16878
|
-
});
|
|
16879
|
-
}
|
|
16880
|
-
const rawExistingStartedAt = syncProfileStart(syncProfile);
|
|
16881
|
-
const rawExistingHashes = await this.log.hasMany(
|
|
16882
|
-
msg.heads.map((head) => head.hash),
|
|
16883
|
-
);
|
|
16884
|
-
if (syncProfile) {
|
|
16885
|
-
emitSyncProfileDuration(syncProfile, rawExistingStartedAt, {
|
|
16886
|
-
name: "sharedLog.rawReceive.existingHeads",
|
|
16887
|
-
component: "shared-log",
|
|
16888
|
-
entries: msg.heads.length,
|
|
16889
|
-
messages: 1,
|
|
16890
|
-
});
|
|
16891
|
-
}
|
|
16892
|
-
const rawMissingHeads = [];
|
|
16893
|
-
const rawConfirmedHashes = new Set<string>();
|
|
16894
|
-
let rawMissingBytes = 0;
|
|
16895
|
-
for (const head of msg.heads) {
|
|
16896
|
-
if (rawExistingHashes.has(head.hash)) {
|
|
16897
|
-
rawConfirmedHashes.add(head.hash);
|
|
16898
|
-
} else {
|
|
16899
|
-
rawMissingHeads.push(head);
|
|
16900
|
-
rawMissingBytes += getRawExchangeHeadByteLength(head);
|
|
16901
|
-
}
|
|
16902
|
-
}
|
|
16903
|
-
if (rawConfirmedHashes.size > 0 && !fromIsSelf) {
|
|
16904
|
-
const rawConfirmStartedAt = syncProfileStart(syncProfile);
|
|
16905
|
-
this.markEntriesKnownByPeer(rawConfirmedHashes, rawFrom.hashcode());
|
|
16906
|
-
await this.sendRepairConfirmation(rawFrom, rawConfirmedHashes);
|
|
16907
|
-
if (syncProfile) {
|
|
16908
|
-
emitSyncProfileDuration(syncProfile, rawConfirmStartedAt, {
|
|
16909
|
-
name: "sharedLog.rawReceive.confirmExisting",
|
|
16910
|
-
component: "shared-log",
|
|
16911
|
-
entries: rawConfirmedHashes.size,
|
|
16912
|
-
messages: 1,
|
|
16913
|
-
});
|
|
16914
|
-
}
|
|
16915
|
-
}
|
|
16916
|
-
if (rawMissingHeads.length === 0) {
|
|
16917
|
-
return;
|
|
16918
|
-
}
|
|
16919
|
-
const rawIsRepairHint =
|
|
16920
|
-
(msg.reserved[0] & EXCHANGE_HEADS_REPAIR_HINT) !== 0;
|
|
16921
|
-
const rawPrepareVerifySetting =
|
|
16922
|
-
this._logProperties?.sync
|
|
16923
|
-
?.rawExchangeHeadsVerifySignaturesDuringPrepare;
|
|
16924
|
-
// A program-level canAppend hook must observe every entry before
|
|
16925
|
-
// it commits, so the native join commit (which validates and
|
|
16926
|
-
// commits entirely in wasm) is not used for programs that
|
|
16927
|
-
// register one; those joins run through the lower-log batch
|
|
16928
|
-
// join where the hook fires per entry.
|
|
16929
|
-
const programCanAppend = !!this._logProperties?.canAppend;
|
|
16930
|
-
const canVerifyPreparedRawReceiveOnCommit =
|
|
16931
|
-
!programCanAppend &&
|
|
16932
|
-
!!this._nativeBackbone?.graph
|
|
16933
|
-
.commitVerifiedPreparedRawReceiveJoinBatch;
|
|
16934
|
-
const canDeferRawReceiveVerificationUntilNativeSelection =
|
|
16935
|
-
!rawIsRepairHint &&
|
|
16936
|
-
!!this._nativeBackbone?.verifyPreparedRawReceiveEntries &&
|
|
16937
|
-
!this._isReplicating &&
|
|
16938
|
-
!this.keep &&
|
|
16939
|
-
!this.closed &&
|
|
16940
|
-
!!this.syncronizer.onReceivedEntryHashes &&
|
|
16941
|
-
rawMissingHeads.every((head) => head.gidRefrences.length === 0);
|
|
16942
|
-
const verifyNativeBackboneSignaturesDuringPrepare =
|
|
16943
|
-
rawPrepareVerifySetting === true ||
|
|
16944
|
-
(rawPrepareVerifySetting !== false &&
|
|
16945
|
-
(canDeferRawReceiveVerificationUntilNativeSelection ||
|
|
16946
|
-
(this._isReplicating &&
|
|
16947
|
-
!rawIsRepairHint &&
|
|
16948
|
-
!canVerifyPreparedRawReceiveOnCommit)));
|
|
16949
|
-
const deferNativeBackboneSignatureVerificationUntilSelection =
|
|
16950
|
-
verifyNativeBackboneSignaturesDuringPrepare &&
|
|
16951
|
-
canDeferRawReceiveVerificationUntilNativeSelection;
|
|
16952
|
-
const deferNativeBackboneSignatureVerificationUntilCommit =
|
|
16953
|
-
deferNativeBackboneSignatureVerificationUntilSelection &&
|
|
16954
|
-
!programCanAppend &&
|
|
16955
|
-
!!this._nativeBackbone?.graph
|
|
16956
|
-
.commitVerifiedPreparedRawReceiveJoinBatch;
|
|
16957
|
-
let rawPreparedReceiveSelection:
|
|
16958
|
-
| Promise<NativeBackboneRawReceiveSelectionPlan | undefined>
|
|
16959
|
-
| undefined;
|
|
16960
|
-
const getRawPreparedReceiveSelection = async (
|
|
16961
|
-
heads: RawEntryWithRefs[],
|
|
16962
|
-
hashes: string[],
|
|
16963
|
-
) => {
|
|
16964
|
-
if (rawPreparedReceiveSelectionValue) {
|
|
16965
|
-
return rawPreparedReceiveSelectionValue;
|
|
16966
|
-
}
|
|
16967
|
-
rawPreparedReceiveSelection ??=
|
|
16968
|
-
this.planNativePreparedRawReceiveSelection({
|
|
16969
|
-
heads,
|
|
16970
|
-
hashes,
|
|
16971
|
-
from: rawFrom,
|
|
16972
|
-
});
|
|
16973
|
-
rawPreparedReceiveSelectionValue = await rawPreparedReceiveSelection;
|
|
16974
|
-
return rawPreparedReceiveSelectionValue;
|
|
16975
|
-
};
|
|
16976
|
-
// Receive fusion: when this message was resolved from the wire
|
|
16977
|
-
// stash, the prepared receive reads entry block bytes straight
|
|
16978
|
-
// out of wasm memory (indexed into the stashed frame) instead
|
|
16979
|
-
// of copying a JS blocks array across the boundary.
|
|
16980
|
-
const rawStashIndexes = stashBackedRawMessage
|
|
16981
|
-
? getRawExchangeHeadStashIndexes(rawMissingHeads)
|
|
16982
|
-
: undefined;
|
|
16983
|
-
const prepareNativeBackboneExpectedColumns =
|
|
16984
|
-
stashBackedRawMessage && rawStashIndexes
|
|
16985
|
-
? ({
|
|
16986
|
-
hashes,
|
|
16987
|
-
verifySignatures,
|
|
16988
|
-
}: {
|
|
16989
|
-
hashes: string[];
|
|
16990
|
-
verifySignatures: boolean;
|
|
16991
|
-
}) => {
|
|
16992
|
-
const backbone = this._nativeBackbone;
|
|
16993
|
-
const wireSession = this._wireSyncSession;
|
|
16994
|
-
if (!backbone || !wireSession) {
|
|
16995
|
-
return undefined;
|
|
16996
|
-
}
|
|
16997
|
-
try {
|
|
16998
|
-
return backbone.prepareStashedRawReceiveExpectedColumnsBatch(
|
|
16999
|
-
wireSession,
|
|
17000
|
-
stashBackedRawMessage.messageId,
|
|
17001
|
-
rawStashIndexes,
|
|
17002
|
-
hashes,
|
|
17003
|
-
{ verifySignatures },
|
|
17004
|
-
);
|
|
17005
|
-
} catch {
|
|
17006
|
-
return undefined;
|
|
17007
|
-
}
|
|
17008
|
-
}
|
|
17009
|
-
: undefined;
|
|
17010
|
-
const prepareNativeBackboneExpectedColumnsAndSelection = rawIsRepairHint
|
|
17011
|
-
? undefined
|
|
17012
|
-
: async ({
|
|
17013
|
-
blocks,
|
|
17014
|
-
hashes,
|
|
17015
|
-
verifySignatures,
|
|
17016
|
-
}: {
|
|
17017
|
-
blocks: () => Uint8Array[];
|
|
17018
|
-
hashes: string[];
|
|
17019
|
-
verifySignatures: boolean;
|
|
17020
|
-
}) => {
|
|
17021
|
-
if (
|
|
17022
|
-
verifySignatures ||
|
|
17023
|
-
!canDeferRawReceiveVerificationUntilNativeSelection
|
|
17024
|
-
) {
|
|
17025
|
-
return undefined;
|
|
17026
|
-
}
|
|
17027
|
-
try {
|
|
17028
|
-
const replicaOptions = {
|
|
17029
|
-
minReplicas: this.replicas.min?.getValue(this) || 1,
|
|
17030
|
-
maxReplicas: this.replicas.max?.getValue(this),
|
|
17031
|
-
};
|
|
17032
|
-
const leaderSelectionContext =
|
|
17033
|
-
await this.createLeaderSelectionContext();
|
|
17034
|
-
const prepareOptions = {
|
|
17035
|
-
verifySignatures: false as const,
|
|
17036
|
-
...replicaOptions,
|
|
17037
|
-
leaderOptions: this.createNativeLeaderOptions(
|
|
17038
|
-
leaderSelectionContext,
|
|
17039
|
-
),
|
|
17040
|
-
fromHash: rawFrom.hashcode(),
|
|
17041
|
-
};
|
|
17042
|
-
let prepared:
|
|
17043
|
-
| ReturnType<
|
|
17044
|
-
NativePeerbitBackbone["prepareRawReceiveExpectedColumnsAndSelectionBatch"]
|
|
17045
|
-
>
|
|
17046
|
-
| undefined;
|
|
17047
|
-
const wireSession = this._wireSyncSession;
|
|
17048
|
-
if (
|
|
17049
|
-
stashBackedRawMessage &&
|
|
17050
|
-
rawStashIndexes &&
|
|
17051
|
-
wireSession &&
|
|
17052
|
-
this._nativeBackbone
|
|
17053
|
-
) {
|
|
17054
|
-
prepared =
|
|
17055
|
-
this._nativeBackbone.prepareStashedRawReceiveExpectedColumnsAndSelectionBatch(
|
|
17056
|
-
wireSession,
|
|
17057
|
-
stashBackedRawMessage.messageId,
|
|
17058
|
-
rawStashIndexes,
|
|
17059
|
-
hashes,
|
|
17060
|
-
prepareOptions,
|
|
17061
|
-
);
|
|
17062
|
-
}
|
|
17063
|
-
if (
|
|
17064
|
-
!prepared &&
|
|
17065
|
-
this._nativeBackbone
|
|
17066
|
-
?.prepareRawReceiveExpectedColumnsAndSelectionBatch
|
|
17067
|
-
) {
|
|
17068
|
-
prepared =
|
|
17069
|
-
this._nativeBackbone.prepareRawReceiveExpectedColumnsAndSelectionBatch(
|
|
17070
|
-
blocks(),
|
|
17071
|
-
hashes,
|
|
17072
|
-
prepareOptions,
|
|
17073
|
-
);
|
|
17074
|
-
}
|
|
17075
|
-
if (!prepared) {
|
|
17076
|
-
return undefined;
|
|
17077
|
-
}
|
|
17078
|
-
rawPreparedReceiveSelectionValue = prepared.selection;
|
|
17079
|
-
rawPreparedReceiveSelection = Promise.resolve(
|
|
17080
|
-
rawPreparedReceiveSelectionValue,
|
|
17081
|
-
);
|
|
17082
|
-
return { columns: prepared.columns };
|
|
17083
|
-
} catch {
|
|
17084
|
-
this.throwIfReplicationOwnershipPoisoned();
|
|
17085
|
-
return undefined;
|
|
17086
|
-
}
|
|
17087
|
-
};
|
|
17088
|
-
const rawMaterializeStartedAt = syncProfileStart(syncProfile);
|
|
17089
|
-
const materializedRawMessage =
|
|
17090
|
-
await materializeVerifiedRawExchangeHeadsMessage(
|
|
17091
|
-
new RawExchangeHeadsMessage({
|
|
17092
|
-
heads: rawMissingHeads,
|
|
17093
|
-
reserved: msg.reserved,
|
|
17094
|
-
}),
|
|
17095
|
-
this.log,
|
|
17199
|
+
const materializedRawReceive = await this.materializeRawReceiveMessage(
|
|
17200
|
+
msg,
|
|
17201
|
+
{
|
|
17202
|
+
from: context.from,
|
|
17203
|
+
stashBackedRawMessage,
|
|
17096
17204
|
syncProfile,
|
|
17097
|
-
|
|
17098
|
-
|
|
17099
|
-
|
|
17100
|
-
|
|
17101
|
-
deferNativeBackboneSignatureVerificationUntilSelection:
|
|
17102
|
-
deferNativeBackboneSignatureVerificationUntilSelection,
|
|
17103
|
-
deferNativeBackboneSignatureVerificationUntilCommit:
|
|
17104
|
-
deferNativeBackboneSignatureVerificationUntilCommit,
|
|
17105
|
-
prepareNativeBackboneExpectedColumnsAndSelection:
|
|
17106
|
-
prepareNativeBackboneExpectedColumnsAndSelection,
|
|
17107
|
-
prepareNativeBackboneExpectedColumns:
|
|
17108
|
-
prepareNativeBackboneExpectedColumns,
|
|
17109
|
-
tryPreparedRawReceiveFastDrop: rawIsRepairHint
|
|
17110
|
-
? undefined
|
|
17111
|
-
: async ({ heads, hashes }) =>
|
|
17112
|
-
this.tryFastDropPreparedRawReceive({
|
|
17113
|
-
heads,
|
|
17114
|
-
hashes,
|
|
17115
|
-
from: rawFrom,
|
|
17116
|
-
fromIsSelf,
|
|
17117
|
-
syncProfile,
|
|
17118
|
-
selection: await getRawPreparedReceiveSelection(
|
|
17119
|
-
heads,
|
|
17120
|
-
hashes,
|
|
17121
|
-
),
|
|
17122
|
-
receiveOwnershipRevision,
|
|
17123
|
-
}),
|
|
17124
|
-
selectPreparedRawReceiveHashes: rawIsRepairHint
|
|
17125
|
-
? undefined
|
|
17126
|
-
: async ({ heads, hashes }) =>
|
|
17127
|
-
this.selectNativePreparedRawReceiveHashes({
|
|
17128
|
-
heads,
|
|
17129
|
-
hashes,
|
|
17130
|
-
from: rawFrom,
|
|
17131
|
-
fromIsSelf,
|
|
17132
|
-
syncProfile,
|
|
17133
|
-
selection: await getRawPreparedReceiveSelection(
|
|
17134
|
-
heads,
|
|
17135
|
-
hashes,
|
|
17136
|
-
),
|
|
17137
|
-
receiveOwnershipRevision,
|
|
17138
|
-
}),
|
|
17139
|
-
},
|
|
17140
|
-
);
|
|
17141
|
-
if (materializedRawMessage === undefined) {
|
|
17142
|
-
if (syncProfile) {
|
|
17143
|
-
emitSyncProfileDuration(syncProfile, rawMaterializeStartedAt, {
|
|
17144
|
-
name: "sharedLog.rawReceive.materialize",
|
|
17145
|
-
component: "shared-log",
|
|
17146
|
-
entries: rawMissingHeads.length,
|
|
17147
|
-
bytes: rawMissingBytes,
|
|
17148
|
-
messages: 1,
|
|
17149
|
-
details: { nativeFastDropEarly: true },
|
|
17150
|
-
});
|
|
17151
|
-
}
|
|
17205
|
+
receiveOwnershipRevision,
|
|
17206
|
+
},
|
|
17207
|
+
);
|
|
17208
|
+
if (materializedRawReceive === undefined) {
|
|
17152
17209
|
return;
|
|
17153
17210
|
}
|
|
17154
|
-
msg =
|
|
17211
|
+
msg = materializedRawReceive.message;
|
|
17212
|
+
rawPreparedReceiveSelectionValue =
|
|
17213
|
+
materializedRawReceive.preparedSelection;
|
|
17155
17214
|
rawMaterializedKnownMissing = true;
|
|
17156
|
-
if (syncProfile) {
|
|
17157
|
-
emitSyncProfileDuration(syncProfile, rawMaterializeStartedAt, {
|
|
17158
|
-
name: "sharedLog.rawReceive.materialize",
|
|
17159
|
-
component: "shared-log",
|
|
17160
|
-
entries: rawMissingHeads.length,
|
|
17161
|
-
bytes: rawMissingBytes,
|
|
17162
|
-
messages: 1,
|
|
17163
|
-
});
|
|
17164
|
-
}
|
|
17165
17215
|
}
|
|
17166
|
-
|
|
17167
17216
|
if (msg instanceof ExchangeHeadsMessage) {
|
|
17168
17217
|
/**
|
|
17169
17218
|
* I have received heads from someone else.
|