@peerbit/shared-log 16.0.23 → 16.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/exchange-heads.d.ts +8 -0
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +8 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +292 -26
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-info-v2-receive.d.ts +23 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -1
- package/dist/src/replication-info-v2-receive.js +91 -0
- package/dist/src/replication-info-v2-receive.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts +6 -0
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +13 -0
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/dist/src/sync/index.d.ts +3 -1
- package/dist/src/sync/index.d.ts.map +1 -1
- package/package.json +16 -16
- package/src/exchange-heads.ts +9 -0
- package/src/index.ts +386 -35
- package/src/replication-info-v2-receive.ts +122 -0
- package/src/replication-info-v2-send.ts +16 -0
- package/src/sync/index.ts +3 -1
package/dist/src/index.js
CHANGED
|
@@ -55,7 +55,7 @@ import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isProm
|
|
|
55
55
|
import { CPUUsageIntervalLag } from "./cpu.js";
|
|
56
56
|
import { debouncedAccumulatorMap, } from "./debounce.js";
|
|
57
57
|
import { CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, PersistedDeliveryError, isNotStartedError, } from "./errors.js";
|
|
58
|
-
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
58
|
+
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
59
59
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
60
60
|
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
61
61
|
import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
|
|
@@ -81,6 +81,54 @@ import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from
|
|
|
81
81
|
import { ConfirmEntriesMessage, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS, RequestPersistedEntriesV1, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
|
|
82
82
|
import { groupByGid, tryGroupByGidSync } from "./utils.js";
|
|
83
83
|
const getSharedLogFanoutService = (services) => services.fanout;
|
|
84
|
+
const FANOUT_OPEN_METRICS = [
|
|
85
|
+
["joinReqSent", "joinReqSent"],
|
|
86
|
+
["joinAcceptReceived", "joinAcceptReceived"],
|
|
87
|
+
["joinRejectReceived", "joinRejectReceived"],
|
|
88
|
+
["bootstrapDialAttempts", "joinBootstrapDialAttempts"],
|
|
89
|
+
["bootstrapDialFailures", "joinBootstrapDialFailures"],
|
|
90
|
+
["candidateDialAttempts", "joinCandidateDialAttempts"],
|
|
91
|
+
["candidateDialFailures", "joinCandidateDialFailures"],
|
|
92
|
+
["connectedCandidateAttempts", "joinConnectedCandidateAttempts"],
|
|
93
|
+
["unconnectedCandidateAttempts", "joinUnconnectedCandidateAttempts"],
|
|
94
|
+
["joinReqTimeouts", "joinReqTimeouts"],
|
|
95
|
+
["deadlineExpirations", "joinDeadlineExpirations"],
|
|
96
|
+
];
|
|
97
|
+
const snapshotFanoutOpenMetrics = (service, topic, root) => {
|
|
98
|
+
try {
|
|
99
|
+
const metrics = service.getChannelMetrics(topic, root);
|
|
100
|
+
return Object.fromEntries(FANOUT_OPEN_METRICS.map(([name, source]) => [name, metrics[source] ?? 0]));
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
const emitFanoutOpenProfile = (properties) => {
|
|
107
|
+
if (!properties.profile)
|
|
108
|
+
return;
|
|
109
|
+
try {
|
|
110
|
+
const after = snapshotFanoutOpenMetrics(properties.service, properties.topic, properties.root);
|
|
111
|
+
const deltas = Object.fromEntries(FANOUT_OPEN_METRICS.map(([name]) => [
|
|
112
|
+
name,
|
|
113
|
+
(after?.[name] ?? 0) - (properties.before?.[name] ?? 0),
|
|
114
|
+
]));
|
|
115
|
+
emitSyncProfileDuration(properties.profile, properties.startedAt, {
|
|
116
|
+
name: "sharedLog.open.fanout",
|
|
117
|
+
component: "shared-log",
|
|
118
|
+
messages: deltas.joinReqSent,
|
|
119
|
+
details: {
|
|
120
|
+
configured: true,
|
|
121
|
+
mode: properties.mode,
|
|
122
|
+
outcome: properties.outcome,
|
|
123
|
+
configuredTimeoutMs: properties.timeoutMs,
|
|
124
|
+
...deltas,
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// Diagnostics must not affect open correctness.
|
|
130
|
+
}
|
|
131
|
+
};
|
|
84
132
|
const createOneShotPeerReceiveLease = (releaseFn) => {
|
|
85
133
|
let released = false;
|
|
86
134
|
return {
|
|
@@ -133,6 +181,14 @@ export { ExchangeHeadsMessage, RawExchangeHeadsMessage, StashBackedRawExchangeHe
|
|
|
133
181
|
export const logger = loggerFn("peerbit:shared-log");
|
|
134
182
|
const warn = logger.newScope("warn");
|
|
135
183
|
const traceLogger = logger.trace;
|
|
184
|
+
const emitAdvisorySyncProfileDuration = (profile, startedAt, event) => {
|
|
185
|
+
try {
|
|
186
|
+
emitSyncProfileDuration(profile, startedAt, event);
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
// Diagnostics must not change open or provider-resolution correctness.
|
|
190
|
+
}
|
|
191
|
+
};
|
|
136
192
|
const canUseOptionalNativeModuleImports = () => {
|
|
137
193
|
const scope = globalThis;
|
|
138
194
|
const serviceWorkerGlobalScope = scope.ServiceWorkerGlobalScope;
|
|
@@ -1968,7 +2024,10 @@ let SharedLog = (() => {
|
|
|
1968
2024
|
const peerHash = properties.target.hashcode();
|
|
1969
2025
|
const receiverTransportSession = this.ownTransportSession();
|
|
1970
2026
|
await this.rpc.send(new SyncCapabilitiesMessage({
|
|
1971
|
-
capabilities: this.replicationInfoV2ReceiveCapabilities()
|
|
2027
|
+
capabilities: this.replicationInfoV2ReceiveCapabilities() |
|
|
2028
|
+
(properties.requestRemoteFullRearm
|
|
2029
|
+
? SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM
|
|
2030
|
+
: 0),
|
|
1972
2031
|
}), {
|
|
1973
2032
|
mode: new AcknowledgeDelivery({
|
|
1974
2033
|
redundancy: 1,
|
|
@@ -2022,6 +2081,7 @@ let SharedLog = (() => {
|
|
|
2022
2081
|
peerSession: properties.peerSession,
|
|
2023
2082
|
receiveEpoch: properties.receiveEpoch,
|
|
2024
2083
|
signal: properties.signal,
|
|
2084
|
+
requestRemoteFullRearm: properties.requestRemoteFullRearm,
|
|
2025
2085
|
}),
|
|
2026
2086
|
onRequestError: (error) => {
|
|
2027
2087
|
if (isNotStartedError(error) ||
|
|
@@ -2252,18 +2312,40 @@ let SharedLog = (() => {
|
|
|
2252
2312
|
void this._onFanoutUnicast(detail).catch((error) => logger.error(error));
|
|
2253
2313
|
});
|
|
2254
2314
|
channel.addEventListener("unicast", this._onFanoutUnicastFn);
|
|
2315
|
+
const profile = this._logProperties?.sync?.profile;
|
|
2316
|
+
const startedAt = syncProfileStart(profile);
|
|
2317
|
+
const mode = resolvedRoot === fanoutService.publicKeyHash ? "root" : "node";
|
|
2318
|
+
const before = profile
|
|
2319
|
+
? snapshotFanoutOpenMetrics(fanoutService, this.topic, resolvedRoot)
|
|
2320
|
+
: undefined;
|
|
2321
|
+
let outcome = "error";
|
|
2255
2322
|
try {
|
|
2256
2323
|
const channelOptions = this.getFanoutChannelOptions(options);
|
|
2257
|
-
if (
|
|
2324
|
+
if (mode === "root") {
|
|
2258
2325
|
await channel.openAsRoot(channelOptions);
|
|
2326
|
+
outcome = "opened";
|
|
2259
2327
|
return;
|
|
2260
2328
|
}
|
|
2261
2329
|
await channel.join(channelOptions, options.join);
|
|
2330
|
+
outcome = "joined";
|
|
2262
2331
|
}
|
|
2263
2332
|
catch (error) {
|
|
2264
2333
|
this._closeFanoutChannel();
|
|
2265
2334
|
throw error;
|
|
2266
2335
|
}
|
|
2336
|
+
finally {
|
|
2337
|
+
emitFanoutOpenProfile({
|
|
2338
|
+
profile,
|
|
2339
|
+
startedAt,
|
|
2340
|
+
service: fanoutService,
|
|
2341
|
+
topic: this.topic,
|
|
2342
|
+
root: resolvedRoot,
|
|
2343
|
+
mode,
|
|
2344
|
+
outcome,
|
|
2345
|
+
timeoutMs: options.join?.timeoutMs,
|
|
2346
|
+
before,
|
|
2347
|
+
});
|
|
2348
|
+
}
|
|
2267
2349
|
}
|
|
2268
2350
|
_closeFanoutChannel() {
|
|
2269
2351
|
if (this._fanoutChannel) {
|
|
@@ -3369,7 +3451,7 @@ let SharedLog = (() => {
|
|
|
3369
3451
|
})) {
|
|
3370
3452
|
return undefined;
|
|
3371
3453
|
}
|
|
3372
|
-
return { capabilitySession, peerSession };
|
|
3454
|
+
return { capabilitySession, peerSession, receiveEpoch };
|
|
3373
3455
|
}
|
|
3374
3456
|
async waitPersistedReceiptRetry(signal, ms) {
|
|
3375
3457
|
try {
|
|
@@ -11083,6 +11165,8 @@ let SharedLog = (() => {
|
|
|
11083
11165
|
const recoveringNativeDurableFailure = this._nativeDurableCommitFailure !== undefined;
|
|
11084
11166
|
options = applySharedLogNativeDefaults(options, this.node
|
|
11085
11167
|
.sharedLogNativeDefaults);
|
|
11168
|
+
const openProfile = options?.sync?.profile;
|
|
11169
|
+
const openStartedAt = syncProfileStart(openProfile);
|
|
11086
11170
|
this.replicas = {
|
|
11087
11171
|
min: options?.replicas?.min != null
|
|
11088
11172
|
? typeof options?.replicas?.min === "number"
|
|
@@ -11249,6 +11333,7 @@ let SharedLog = (() => {
|
|
|
11249
11333
|
this._isTrustedReplicator = options?.canReplicate;
|
|
11250
11334
|
this.keep = options?.keep;
|
|
11251
11335
|
this.pendingMaturity = new Map();
|
|
11336
|
+
const localStateStartedAt = syncProfileStart(openProfile);
|
|
11252
11337
|
const id = sha256Base64Sync(this.log.id);
|
|
11253
11338
|
const [storage, logScope] = await Promise.all([
|
|
11254
11339
|
this.node.storage.sublevel(id),
|
|
@@ -11303,6 +11388,11 @@ let SharedLog = (() => {
|
|
|
11303
11388
|
this._entryCoordinatesIndex = await replicationIndex.init({
|
|
11304
11389
|
schema: this.indexableDomain.constructorEntry,
|
|
11305
11390
|
});
|
|
11391
|
+
emitAdvisorySyncProfileDuration(openProfile, localStateStartedAt, {
|
|
11392
|
+
name: "sharedLog.open.localState",
|
|
11393
|
+
component: "shared-log",
|
|
11394
|
+
});
|
|
11395
|
+
const blockStoreStartedAt = syncProfileStart(openProfile);
|
|
11306
11396
|
const deferStandaloneNativeRangePlanner = !!options?.nativeBackbone && options.nativeRangePlanner == null;
|
|
11307
11397
|
await this.openNativeRangePlanner(deferStandaloneNativeRangePlanner ? false : options?.nativeRangePlanner);
|
|
11308
11398
|
this._nativeBackbone = await this.openNativeBackbone(options?.nativeBackbone);
|
|
@@ -11343,6 +11433,14 @@ let SharedLog = (() => {
|
|
|
11343
11433
|
else {
|
|
11344
11434
|
localBlocks = await createDefaultDurableBlockStore(storage);
|
|
11345
11435
|
}
|
|
11436
|
+
emitAdvisorySyncProfileDuration(openProfile, blockStoreStartedAt, {
|
|
11437
|
+
name: "sharedLog.open.blockStore",
|
|
11438
|
+
component: "shared-log",
|
|
11439
|
+
details: {
|
|
11440
|
+
nativeBackbone: this._nativeBackbone != null,
|
|
11441
|
+
directoryConfigured: this.node.directory != null,
|
|
11442
|
+
},
|
|
11443
|
+
});
|
|
11346
11444
|
this.remoteBlocks = new RemoteBlocks({
|
|
11347
11445
|
local: localBlocks,
|
|
11348
11446
|
publish: (message, options) => this.rpc.send(new BlocksMessage(message), options),
|
|
@@ -11352,30 +11450,61 @@ let SharedLog = (() => {
|
|
|
11352
11450
|
// compatible eager path with bounded validation and storage budgets.
|
|
11353
11451
|
eagerBlocks: options?.eagerBlocks ?? false,
|
|
11354
11452
|
resolveProviders: async (cid, opts) => {
|
|
11453
|
+
const profile = this._logProperties?.sync?.profile;
|
|
11355
11454
|
const maxPeers = 8;
|
|
11455
|
+
const excluded = new Set((opts?.exclude ?? []).slice(0, maxPeers));
|
|
11456
|
+
const lookupPeers = opts?.refresh
|
|
11457
|
+
? Math.min(maxPeers * 2, maxPeers + excluded.size)
|
|
11458
|
+
: maxPeers;
|
|
11459
|
+
const resolutionStartedAt = syncProfileStart(profile);
|
|
11356
11460
|
const localCandidates = (await this.resolveCandidatePeersForHash(cid, {
|
|
11357
11461
|
signal: opts?.signal,
|
|
11358
|
-
maxPeers,
|
|
11462
|
+
maxPeers: lookupPeers,
|
|
11359
11463
|
})) ?? [];
|
|
11360
|
-
|
|
11464
|
+
const emitResolution = profile
|
|
11465
|
+
? (status, targets, directoryCandidates = 0, reachableCandidates = 0) => emitAdvisorySyncProfileDuration(profile, resolutionStartedAt, {
|
|
11466
|
+
name: "sharedLog.blocks.resolveProviders",
|
|
11467
|
+
component: "shared-log",
|
|
11468
|
+
count: targets,
|
|
11469
|
+
targets,
|
|
11470
|
+
details: {
|
|
11471
|
+
status,
|
|
11472
|
+
refresh: opts?.refresh === true,
|
|
11473
|
+
excluded: excluded.size,
|
|
11474
|
+
lookupPeers,
|
|
11475
|
+
localCandidates: localCandidates.length,
|
|
11476
|
+
directoryCandidates,
|
|
11477
|
+
reachableCandidates,
|
|
11478
|
+
},
|
|
11479
|
+
})
|
|
11480
|
+
: undefined;
|
|
11481
|
+
if (opts?.signal?.aborted) {
|
|
11482
|
+
emitResolution?.("aborted", 0);
|
|
11361
11483
|
return [];
|
|
11484
|
+
}
|
|
11362
11485
|
const locallyReachable = new Set(await this._getLocalReachablePeerHashes(this.topic));
|
|
11486
|
+
if (opts?.signal?.aborted) {
|
|
11487
|
+
emitResolution?.("aborted", 0, 0, locallyReachable.size);
|
|
11488
|
+
return [];
|
|
11489
|
+
}
|
|
11363
11490
|
const confirmed = this._checkedPrune.getConfirmedReplicators(cid);
|
|
11364
11491
|
const contacted = this._checkedPrune.getContactedReplicators(cid);
|
|
11365
|
-
const
|
|
11366
|
-
|
|
11367
|
-
|
|
11368
|
-
|
|
11492
|
+
const hasProviderEvidence = (peer) => confirmed?.has(peer) === true ||
|
|
11493
|
+
contacted?.has(peer) ||
|
|
11494
|
+
this.uniqueReplicators.has(peer);
|
|
11495
|
+
const hasLiveCandidate = localCandidates.some((peer) => locallyReachable.has(peer) && hasProviderEvidence(peer));
|
|
11369
11496
|
// Only reachability corroborated by provider/replicator evidence may
|
|
11370
11497
|
// bypass the initial CID lookup. Arbitrary bootstrap connections are
|
|
11371
11498
|
// useful fallbacks, but are not evidence that they hold this block.
|
|
11372
11499
|
if (hasLiveCandidate && !opts?.refresh) {
|
|
11373
|
-
|
|
11500
|
+
const selected = localCandidates.slice(0, maxPeers);
|
|
11501
|
+
emitResolution?.("local", selected.length, 0, locallyReachable.size);
|
|
11502
|
+
return selected;
|
|
11374
11503
|
}
|
|
11375
11504
|
let directoryProviders = [];
|
|
11376
11505
|
try {
|
|
11377
11506
|
const query = (namespace) => fanoutService?.queryProviders(namespace, {
|
|
11378
|
-
want:
|
|
11507
|
+
want: lookupPeers,
|
|
11379
11508
|
timeoutMs: 2_000,
|
|
11380
11509
|
queryTimeoutMs: 500,
|
|
11381
11510
|
bootstrapMaxPeers: 2,
|
|
@@ -11387,13 +11516,17 @@ let SharedLog = (() => {
|
|
|
11387
11516
|
]);
|
|
11388
11517
|
for (const result of results) {
|
|
11389
11518
|
if (result.status === "fulfilled") {
|
|
11390
|
-
directoryProviders.push(...result.value);
|
|
11519
|
+
directoryProviders.push(...result.value.slice(0, lookupPeers));
|
|
11391
11520
|
}
|
|
11392
11521
|
}
|
|
11393
11522
|
}
|
|
11394
11523
|
catch {
|
|
11395
11524
|
// Ignore discovery failures; local evidence remains usable.
|
|
11396
11525
|
}
|
|
11526
|
+
if (opts?.signal?.aborted) {
|
|
11527
|
+
emitResolution?.("aborted", 0, directoryProviders.length, locallyReachable.size);
|
|
11528
|
+
return [];
|
|
11529
|
+
}
|
|
11397
11530
|
const selected = [];
|
|
11398
11531
|
const selectedSet = new Set();
|
|
11399
11532
|
const add = (peer) => {
|
|
@@ -11403,11 +11536,44 @@ let SharedLog = (() => {
|
|
|
11403
11536
|
selectedSet.add(peer);
|
|
11404
11537
|
selected.push(peer);
|
|
11405
11538
|
};
|
|
11406
|
-
|
|
11407
|
-
(
|
|
11408
|
-
|
|
11409
|
-
|
|
11539
|
+
const append = (providers, includeExcluded, predicate) => {
|
|
11540
|
+
for (const provider of providers) {
|
|
11541
|
+
if (selected.length >= maxPeers)
|
|
11542
|
+
return;
|
|
11543
|
+
if (excluded.has(provider) === includeExcluded &&
|
|
11544
|
+
(!predicate || predicate(provider))) {
|
|
11545
|
+
add(provider);
|
|
11546
|
+
}
|
|
11547
|
+
}
|
|
11548
|
+
};
|
|
11549
|
+
const appendInterleaved = (includeExcluded) => {
|
|
11550
|
+
for (let index = 0; selected.length < maxPeers &&
|
|
11551
|
+
(index < localCandidates.length ||
|
|
11552
|
+
index < directoryProviders.length); index++) {
|
|
11553
|
+
const local = localCandidates[index];
|
|
11554
|
+
if (local && excluded.has(local) === includeExcluded)
|
|
11555
|
+
add(local);
|
|
11556
|
+
const directory = directoryProviders[index];
|
|
11557
|
+
if (directory && excluded.has(directory) === includeExcluded) {
|
|
11558
|
+
add(directory);
|
|
11559
|
+
}
|
|
11560
|
+
}
|
|
11561
|
+
};
|
|
11562
|
+
if (opts?.refresh) {
|
|
11563
|
+
// Retry results are wider than the regular eight-peer window. Prefer
|
|
11564
|
+
// untried reachable holders, then the remaining fresh directory
|
|
11565
|
+
// evidence, without discarding attempted peers as bounded transient-
|
|
11566
|
+
// failure fallbacks.
|
|
11567
|
+
append(directoryProviders, false, (peer) => locallyReachable.has(peer));
|
|
11568
|
+
append(localCandidates, false, (peer) => locallyReachable.has(peer) && hasProviderEvidence(peer));
|
|
11569
|
+
append(directoryProviders, false);
|
|
11570
|
+
append(localCandidates, false);
|
|
11571
|
+
}
|
|
11572
|
+
else {
|
|
11573
|
+
appendInterleaved(false);
|
|
11410
11574
|
}
|
|
11575
|
+
appendInterleaved(true);
|
|
11576
|
+
emitResolution?.("directory", selected.length, directoryProviders.length, locallyReachable.size);
|
|
11411
11577
|
return selected;
|
|
11412
11578
|
},
|
|
11413
11579
|
watchProviders: fanoutService
|
|
@@ -11454,7 +11620,13 @@ let SharedLog = (() => {
|
|
|
11454
11620
|
}
|
|
11455
11621
|
: undefined,
|
|
11456
11622
|
});
|
|
11457
|
-
const
|
|
11623
|
+
const remoteBlocksStartedAt = syncProfileStart(openProfile);
|
|
11624
|
+
const remoteBlocksStartPromise = this.remoteBlocks.start().then(() => {
|
|
11625
|
+
emitAdvisorySyncProfileDuration(openProfile, remoteBlocksStartedAt, {
|
|
11626
|
+
name: "sharedLog.open.remoteBlocks",
|
|
11627
|
+
component: "shared-log",
|
|
11628
|
+
});
|
|
11629
|
+
});
|
|
11458
11630
|
const hasIndexedReplicationInfo = (await this.replicationIndex.count({
|
|
11459
11631
|
query: [
|
|
11460
11632
|
new StringMatch({
|
|
@@ -11613,6 +11785,7 @@ let SharedLog = (() => {
|
|
|
11613
11785
|
// joins rely on: a replicate:false observer syncing a head whose parents
|
|
11614
11786
|
// are not local would fail block resolution, and Log.join treats that as
|
|
11615
11787
|
// recoverable and skips the entry without persisting anything.
|
|
11788
|
+
const lowerLogStartedAt = syncProfileStart(openProfile);
|
|
11616
11789
|
await this.log.open(this.remoteBlocks, this.node.identity, {
|
|
11617
11790
|
keychain: this.node.services.keychain,
|
|
11618
11791
|
resolveRemotePeers: (hash, options) => this.resolveCandidatePeersForHash(hash, {
|
|
@@ -11643,6 +11816,10 @@ let SharedLog = (() => {
|
|
|
11643
11816
|
},
|
|
11644
11817
|
indexer: logIndex,
|
|
11645
11818
|
});
|
|
11819
|
+
emitAdvisorySyncProfileDuration(openProfile, lowerLogStartedAt, {
|
|
11820
|
+
name: "sharedLog.open.lowerLog",
|
|
11821
|
+
component: "shared-log",
|
|
11822
|
+
});
|
|
11646
11823
|
this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
|
|
11647
11824
|
try {
|
|
11648
11825
|
const recovered = await this.recoverNativeStrictDurableTransactionIntent();
|
|
@@ -11694,6 +11871,7 @@ let SharedLog = (() => {
|
|
|
11694
11871
|
((event) => {
|
|
11695
11872
|
void this.runSubscriptionChangeCallback(() => this._onUnsubscription(event));
|
|
11696
11873
|
});
|
|
11874
|
+
const communicationStartedAt = syncProfileStart(openProfile);
|
|
11697
11875
|
await Promise.all([
|
|
11698
11876
|
this.rpc.open({
|
|
11699
11877
|
queryType: TransportMessage,
|
|
@@ -11705,6 +11883,11 @@ let SharedLog = (() => {
|
|
|
11705
11883
|
this.node.services.pubsub.addEventListener("subscribe", this._onSubscriptionFn),
|
|
11706
11884
|
this.node.services.pubsub.addEventListener("unsubscribe", this._onUnsubscriptionFn),
|
|
11707
11885
|
]);
|
|
11886
|
+
emitAdvisorySyncProfileDuration(openProfile, communicationStartedAt, {
|
|
11887
|
+
name: "sharedLog.open.rpcSubscriptions",
|
|
11888
|
+
component: "shared-log",
|
|
11889
|
+
});
|
|
11890
|
+
const providerChannelStartedAt = syncProfileStart(openProfile);
|
|
11708
11891
|
const fanoutOpenPromise = this._openFanoutChannel(options?.fanout);
|
|
11709
11892
|
// Mark previously-owned replication ranges as "new" only when they already exist.
|
|
11710
11893
|
// Fresh opens have nothing to touch here, so skip the extra scan/write entirely.
|
|
@@ -11712,30 +11895,57 @@ let SharedLog = (() => {
|
|
|
11712
11895
|
? this.updateTimestampOfOwnedReplicationRanges()
|
|
11713
11896
|
: Promise.resolve();
|
|
11714
11897
|
await Promise.all([fanoutOpenPromise, updateOwnedReplicationPromise]);
|
|
11898
|
+
emitAdvisorySyncProfileDuration(openProfile, providerChannelStartedAt, {
|
|
11899
|
+
name: "sharedLog.open.providerAndOwnership",
|
|
11900
|
+
component: "shared-log",
|
|
11901
|
+
details: { indexedReplicationInfo: hasIndexedReplicationInfo },
|
|
11902
|
+
});
|
|
11715
11903
|
// if we had a previous session with replication info, and new replication info dictates that we unreplicate
|
|
11716
11904
|
// we should do that. Otherwise if options is a unreplication we dont need to do anything because
|
|
11717
11905
|
// we are already unreplicated (as we are just opening)
|
|
11718
11906
|
const isUnreplicationOptionsDefined = isUnreplicationOptions(options?.replicate);
|
|
11719
11907
|
const canResumeReplication = hasIndexedReplicationInfo &&
|
|
11720
11908
|
(await isReplicationOptionsDependentOnPreviousState(options?.replicate, this.replicationIndex, this.node.identity.publicKey));
|
|
11909
|
+
const replicationStartedAt = syncProfileStart(openProfile);
|
|
11910
|
+
let replicationAction;
|
|
11721
11911
|
if (hasIndexedReplicationInfo && isUnreplicationOptionsDefined) {
|
|
11912
|
+
replicationAction = "replace";
|
|
11722
11913
|
await this.replicate(options?.replicate, { checkDuplicates: true });
|
|
11723
11914
|
}
|
|
11724
11915
|
else if (canResumeReplication) {
|
|
11916
|
+
replicationAction = "resume";
|
|
11725
11917
|
// dont do anthing since we are alread replicating stuff
|
|
11726
11918
|
}
|
|
11727
11919
|
else {
|
|
11920
|
+
replicationAction = "reset";
|
|
11728
11921
|
await this.replicate(options?.replicate, {
|
|
11729
11922
|
checkDuplicates: true,
|
|
11730
11923
|
reset: true,
|
|
11731
11924
|
});
|
|
11732
11925
|
}
|
|
11926
|
+
emitAdvisorySyncProfileDuration(openProfile, replicationStartedAt, {
|
|
11927
|
+
name: "sharedLog.open.replication",
|
|
11928
|
+
component: "shared-log",
|
|
11929
|
+
details: {
|
|
11930
|
+
hadIndexedState: hasIndexedReplicationInfo,
|
|
11931
|
+
action: replicationAction,
|
|
11932
|
+
},
|
|
11933
|
+
});
|
|
11934
|
+
const synchronizerStartedAt = syncProfileStart(openProfile);
|
|
11733
11935
|
await this.syncronizer.open();
|
|
11936
|
+
emitAdvisorySyncProfileDuration(openProfile, synchronizerStartedAt, {
|
|
11937
|
+
name: "sharedLog.open.synchronizer",
|
|
11938
|
+
component: "shared-log",
|
|
11939
|
+
});
|
|
11734
11940
|
this.interval = setInterval(() => {
|
|
11735
11941
|
void this.rebalanceParticipationDebounced?.call();
|
|
11736
11942
|
}, RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL);
|
|
11737
11943
|
this._instanceLifecycle.markOpenComplete();
|
|
11738
11944
|
this.scheduleReplicationStatusRefresh();
|
|
11945
|
+
emitAdvisorySyncProfileDuration(openProfile, openStartedAt, {
|
|
11946
|
+
name: "sharedLog.open.total",
|
|
11947
|
+
component: "shared-log",
|
|
11948
|
+
});
|
|
11739
11949
|
}
|
|
11740
11950
|
toNativeReplicationRange(range) {
|
|
11741
11951
|
return {
|
|
@@ -15760,6 +15970,13 @@ let SharedLog = (() => {
|
|
|
15760
15970
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
15761
15971
|
const capabilityTransportSession = context.message?.header?.session;
|
|
15762
15972
|
const capabilityTimestamp = context.message?.header?.timestamp;
|
|
15973
|
+
const previousCapabilitySession = this._peerSyncCapabilitySessions.get(receiveFromHash);
|
|
15974
|
+
const previousCapabilityTimestamp = this._peerSyncCapabilityTimestamps.get(receiveFromHash);
|
|
15975
|
+
const requestsRemoteFullRearm = (msg.capabilities & SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM) !==
|
|
15976
|
+
0;
|
|
15977
|
+
// The rearm bit is a one-shot authenticated command, not a sticky
|
|
15978
|
+
// negotiated capability. Store and promote only the steady bits.
|
|
15979
|
+
const steadyCapabilities = msg.capabilities & ~SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM;
|
|
15763
15980
|
// No await separates this from the capture above, so the captured
|
|
15764
15981
|
// window state is exact: the legacy re-read of the opening map here
|
|
15765
15982
|
// could never observe a different value.
|
|
@@ -15770,7 +15987,7 @@ let SharedLog = (() => {
|
|
|
15770
15987
|
// cleanup cannot erase it before the opening transition commits.
|
|
15771
15988
|
this.observePeerSyncCapabilities({
|
|
15772
15989
|
peerHash: receiveFromHash,
|
|
15773
|
-
capabilities:
|
|
15990
|
+
capabilities: steadyCapabilities,
|
|
15774
15991
|
transportSession: capabilityTransportSession,
|
|
15775
15992
|
timestamp: capabilityTimestamp,
|
|
15776
15993
|
openingSession: receiveSession,
|
|
@@ -15779,12 +15996,34 @@ let SharedLog = (() => {
|
|
|
15779
15996
|
else {
|
|
15780
15997
|
const observed = this.observePeerSyncCapabilities({
|
|
15781
15998
|
peerHash: receiveFromHash,
|
|
15782
|
-
capabilities:
|
|
15999
|
+
capabilities: steadyCapabilities,
|
|
15783
16000
|
transportSession: capabilityTransportSession,
|
|
15784
16001
|
timestamp: capabilityTimestamp,
|
|
15785
16002
|
});
|
|
15786
16003
|
if (observed && receiveSession?.phase === "open") {
|
|
15787
16004
|
this.promoteReplicationInfoV2ReceiveCapability(context.from, receiveSession);
|
|
16005
|
+
const freshExactRearm = requestsRemoteFullRearm &&
|
|
16006
|
+
capabilityTransportSession !== undefined &&
|
|
16007
|
+
capabilityTimestamp !== undefined &&
|
|
16008
|
+
previousCapabilitySession === capabilityTransportSession &&
|
|
16009
|
+
previousCapabilityTimestamp !== undefined &&
|
|
16010
|
+
capabilityTimestamp > previousCapabilityTimestamp;
|
|
16011
|
+
if (freshExactRearm &&
|
|
16012
|
+
this._v2Receive.isCurrentActive({
|
|
16013
|
+
peerHash: receiveFromHash,
|
|
16014
|
+
peerSession: receiveSession,
|
|
16015
|
+
receiveEpoch: this._peerSessions.receiveEpoch(receiveFromHash),
|
|
16016
|
+
senderTransportSession: capabilityTransportSession,
|
|
16017
|
+
})) {
|
|
16018
|
+
// Rotate the receiver grant/challenge before requesting Full. A
|
|
16019
|
+
// sender rebuilt from no state starts at sequence one, which an
|
|
16020
|
+
// active receiver's old sequence fence must otherwise reject.
|
|
16021
|
+
this._v2Receive.advanceRecovery({
|
|
16022
|
+
peerHash: receiveFromHash,
|
|
16023
|
+
peerSession: receiveSession,
|
|
16024
|
+
receiveEpoch: this._peerSessions.receiveEpoch(receiveFromHash),
|
|
16025
|
+
});
|
|
16026
|
+
}
|
|
15788
16027
|
}
|
|
15789
16028
|
else if (observed && receiveSession === null) {
|
|
15790
16029
|
// A capability can arrive before the sender's topic Subscribe after
|
|
@@ -16649,8 +16888,8 @@ let SharedLog = (() => {
|
|
|
16649
16888
|
}
|
|
16650
16889
|
throwIfInactive();
|
|
16651
16890
|
}
|
|
16652
|
-
nudgePersistedReceiptPeerReadiness(publicKey) {
|
|
16653
|
-
if (this.closed)
|
|
16891
|
+
nudgePersistedReceiptPeerReadiness(publicKey, signal) {
|
|
16892
|
+
if (this.closed || signal.aborted)
|
|
16654
16893
|
return;
|
|
16655
16894
|
const peerHash = publicKey.hashcode();
|
|
16656
16895
|
const peerSession = this._peerSessions.current(peerHash);
|
|
@@ -16680,6 +16919,20 @@ let SharedLog = (() => {
|
|
|
16680
16919
|
peerSession,
|
|
16681
16920
|
receiveEpoch,
|
|
16682
16921
|
});
|
|
16922
|
+
const receiptTarget = this.persistedReceiptPeerSession(peerHash);
|
|
16923
|
+
if (receiptTarget &&
|
|
16924
|
+
!this._v2Send.hasCurrentStateForPeer({
|
|
16925
|
+
peerHash,
|
|
16926
|
+
peerSession: receiptTarget.peerSession,
|
|
16927
|
+
receiverTransportSession: receiptTarget.capabilitySession,
|
|
16928
|
+
})) {
|
|
16929
|
+
this._v2Receive.reAdvertiseLocalCapabilityForRemoteFull({
|
|
16930
|
+
peerHash,
|
|
16931
|
+
peerSession: receiptTarget.peerSession,
|
|
16932
|
+
receiveEpoch: receiptTarget.receiveEpoch,
|
|
16933
|
+
signal,
|
|
16934
|
+
});
|
|
16935
|
+
}
|
|
16683
16936
|
this.scheduleReplicationInfoV2Recovery(publicKey);
|
|
16684
16937
|
}
|
|
16685
16938
|
/**
|
|
@@ -16873,17 +17126,25 @@ let SharedLog = (() => {
|
|
|
16873
17126
|
throw new AbortError("Persisted-receipt readiness wait settled");
|
|
16874
17127
|
}
|
|
16875
17128
|
};
|
|
17129
|
+
const recoveryDelayMs = Math.max(50, Math.min(1_000, this.waitForReplicatorRequestIntervalMs));
|
|
16876
17130
|
const armRecoveryTick = () => {
|
|
16877
17131
|
if (settled || recoveryTimer)
|
|
16878
17132
|
return;
|
|
16879
|
-
const delayMs = Math.max(50, Math.min(1_000, this.waitForReplicatorRequestIntervalMs));
|
|
16880
17133
|
recoveryTimer = setTimeout(() => {
|
|
16881
17134
|
recoveryTimer = undefined;
|
|
16882
17135
|
if (!continueWait())
|
|
16883
17136
|
return;
|
|
16884
|
-
this.nudgePersistedReceiptPeerReadiness(key);
|
|
17137
|
+
this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
|
|
17138
|
+
if (checkInFlight) {
|
|
17139
|
+
// Confirmation can legitimately span several recovery intervals.
|
|
17140
|
+
// Keep repairing the exact current generation without aborting its
|
|
17141
|
+
// one application-confirmation waiter on every tick.
|
|
17142
|
+
rerun = true;
|
|
17143
|
+
armRecoveryTick();
|
|
17144
|
+
return;
|
|
17145
|
+
}
|
|
16885
17146
|
scheduleCheck();
|
|
16886
|
-
},
|
|
17147
|
+
}, recoveryDelayMs);
|
|
16887
17148
|
recoveryTimer.unref?.();
|
|
16888
17149
|
};
|
|
16889
17150
|
const runCheck = async () => {
|
|
@@ -16913,6 +17174,11 @@ let SharedLog = (() => {
|
|
|
16913
17174
|
snapshot.reason === "replication-confirmation-pending") {
|
|
16914
17175
|
const target = this.persistedReceiptPeerSession(peerHash);
|
|
16915
17176
|
if (target) {
|
|
17177
|
+
// A confirmation waiter cannot create an outbound V2 stream when
|
|
17178
|
+
// that exact session state is missing. Nudge first, then keep the
|
|
17179
|
+
// recovery tick alive while the single confirmation wait remains.
|
|
17180
|
+
this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
|
|
17181
|
+
armRecoveryTick();
|
|
16916
17182
|
const currentConfirmationController = new AbortController();
|
|
16917
17183
|
confirmationController = currentConfirmationController;
|
|
16918
17184
|
try {
|
|
@@ -16957,7 +17223,7 @@ let SharedLog = (() => {
|
|
|
16957
17223
|
}
|
|
16958
17224
|
if (!continueWait())
|
|
16959
17225
|
return;
|
|
16960
|
-
this.nudgePersistedReceiptPeerReadiness(key);
|
|
17226
|
+
this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
|
|
16961
17227
|
}
|
|
16962
17228
|
catch (error) {
|
|
16963
17229
|
if (!settled)
|