@peerbit/shared-log 16.0.22 → 16.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +78 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +780 -31
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts +6 -0
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +15 -0
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/dist/src/sync/index.d.ts +3 -1
- package/dist/src/sync/index.d.ts.map +1 -1
- package/package.json +16 -16
- package/src/index.ts +1085 -39
- package/src/replication-info-v2-send.ts +18 -0
- package/src/sync/index.ts +3 -1
package/src/index.ts
CHANGED
|
@@ -322,6 +322,86 @@ type SharedLogServicesWithFanout = {
|
|
|
322
322
|
const getSharedLogFanoutService = (services: unknown): FanoutTree | undefined =>
|
|
323
323
|
(services as SharedLogServicesWithFanout).fanout;
|
|
324
324
|
|
|
325
|
+
const FANOUT_OPEN_METRICS = [
|
|
326
|
+
["joinReqSent", "joinReqSent"],
|
|
327
|
+
["joinAcceptReceived", "joinAcceptReceived"],
|
|
328
|
+
["joinRejectReceived", "joinRejectReceived"],
|
|
329
|
+
["bootstrapDialAttempts", "joinBootstrapDialAttempts"],
|
|
330
|
+
["bootstrapDialFailures", "joinBootstrapDialFailures"],
|
|
331
|
+
["candidateDialAttempts", "joinCandidateDialAttempts"],
|
|
332
|
+
["candidateDialFailures", "joinCandidateDialFailures"],
|
|
333
|
+
["connectedCandidateAttempts", "joinConnectedCandidateAttempts"],
|
|
334
|
+
["unconnectedCandidateAttempts", "joinUnconnectedCandidateAttempts"],
|
|
335
|
+
["joinReqTimeouts", "joinReqTimeouts"],
|
|
336
|
+
["deadlineExpirations", "joinDeadlineExpirations"],
|
|
337
|
+
] as const;
|
|
338
|
+
|
|
339
|
+
type FanoutOpenMetric = (typeof FANOUT_OPEN_METRICS)[number][0];
|
|
340
|
+
type FanoutOpenMetricSnapshot = Record<FanoutOpenMetric, number>;
|
|
341
|
+
|
|
342
|
+
const snapshotFanoutOpenMetrics = (
|
|
343
|
+
service: FanoutTree,
|
|
344
|
+
topic: string,
|
|
345
|
+
root: string,
|
|
346
|
+
): FanoutOpenMetricSnapshot | undefined => {
|
|
347
|
+
try {
|
|
348
|
+
const metrics = (
|
|
349
|
+
service as FanoutTree & {
|
|
350
|
+
getChannelMetrics: (
|
|
351
|
+
topic: string,
|
|
352
|
+
root: string,
|
|
353
|
+
) => Record<string, number>;
|
|
354
|
+
}
|
|
355
|
+
).getChannelMetrics(topic, root);
|
|
356
|
+
return Object.fromEntries(
|
|
357
|
+
FANOUT_OPEN_METRICS.map(([name, source]) => [name, metrics[source] ?? 0]),
|
|
358
|
+
) as FanoutOpenMetricSnapshot;
|
|
359
|
+
} catch {
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
};
|
|
363
|
+
|
|
364
|
+
const emitFanoutOpenProfile = (properties: {
|
|
365
|
+
profile?: SyncProfileFn;
|
|
366
|
+
startedAt: number;
|
|
367
|
+
service: FanoutTree;
|
|
368
|
+
topic: string;
|
|
369
|
+
root: string;
|
|
370
|
+
mode: "root" | "node";
|
|
371
|
+
outcome: "error" | "opened" | "joined";
|
|
372
|
+
timeoutMs?: number;
|
|
373
|
+
before?: FanoutOpenMetricSnapshot;
|
|
374
|
+
}) => {
|
|
375
|
+
if (!properties.profile) return;
|
|
376
|
+
try {
|
|
377
|
+
const after = snapshotFanoutOpenMetrics(
|
|
378
|
+
properties.service,
|
|
379
|
+
properties.topic,
|
|
380
|
+
properties.root,
|
|
381
|
+
);
|
|
382
|
+
const deltas = Object.fromEntries(
|
|
383
|
+
FANOUT_OPEN_METRICS.map(([name]) => [
|
|
384
|
+
name,
|
|
385
|
+
(after?.[name] ?? 0) - (properties.before?.[name] ?? 0),
|
|
386
|
+
]),
|
|
387
|
+
) as FanoutOpenMetricSnapshot;
|
|
388
|
+
emitSyncProfileDuration(properties.profile, properties.startedAt, {
|
|
389
|
+
name: "sharedLog.open.fanout",
|
|
390
|
+
component: "shared-log",
|
|
391
|
+
messages: deltas.joinReqSent,
|
|
392
|
+
details: {
|
|
393
|
+
configured: true,
|
|
394
|
+
mode: properties.mode,
|
|
395
|
+
outcome: properties.outcome,
|
|
396
|
+
configuredTimeoutMs: properties.timeoutMs,
|
|
397
|
+
...deltas,
|
|
398
|
+
},
|
|
399
|
+
});
|
|
400
|
+
} catch {
|
|
401
|
+
// Diagnostics must not affect open correctness.
|
|
402
|
+
}
|
|
403
|
+
};
|
|
404
|
+
|
|
325
405
|
type PendingIHave<T> = {
|
|
326
406
|
resetTimeout: () => void;
|
|
327
407
|
requesting: Map<string, Uint8Array>;
|
|
@@ -471,6 +551,18 @@ export const logger = loggerFn("peerbit:shared-log");
|
|
|
471
551
|
const warn = logger.newScope("warn");
|
|
472
552
|
const traceLogger = logger.trace as typeof logger.trace & { enabled?: boolean };
|
|
473
553
|
|
|
554
|
+
const emitAdvisorySyncProfileDuration = (
|
|
555
|
+
profile: SyncProfileFn | undefined,
|
|
556
|
+
startedAt: number,
|
|
557
|
+
event: Parameters<typeof emitSyncProfileDuration>[2],
|
|
558
|
+
): void => {
|
|
559
|
+
try {
|
|
560
|
+
emitSyncProfileDuration(profile, startedAt, event);
|
|
561
|
+
} catch {
|
|
562
|
+
// Diagnostics must not change open or provider-resolution correctness.
|
|
563
|
+
}
|
|
564
|
+
};
|
|
565
|
+
|
|
474
566
|
const canUseOptionalNativeModuleImports = (): boolean => {
|
|
475
567
|
const scope = globalThis as {
|
|
476
568
|
ServiceWorkerGlobalScope?: unknown;
|
|
@@ -1752,6 +1844,71 @@ export type DeliveryOptions = {
|
|
|
1752
1844
|
signal?: AbortSignal;
|
|
1753
1845
|
};
|
|
1754
1846
|
|
|
1847
|
+
export type PersistedReceiptPeerReadinessPendingReason =
|
|
1848
|
+
| "closed"
|
|
1849
|
+
| "no-current-session"
|
|
1850
|
+
| "session-opening"
|
|
1851
|
+
| "capability-pending"
|
|
1852
|
+
| "replication-state-pending"
|
|
1853
|
+
| "replication-confirmation-pending"
|
|
1854
|
+
| "not-replicating"
|
|
1855
|
+
| "not-entry-leader"
|
|
1856
|
+
| "ownership-changing";
|
|
1857
|
+
|
|
1858
|
+
export type PersistedReceiptPeerReadinessUnsupportedReason =
|
|
1859
|
+
| "persisted-receipts-unsupported"
|
|
1860
|
+
| "replication-confirmation-unsupported";
|
|
1861
|
+
|
|
1862
|
+
/**
|
|
1863
|
+
* Detached view of one public key's current persisted-receipt generation.
|
|
1864
|
+
* `generation` is opaque: callers may compare it for equality, but must not
|
|
1865
|
+
* interpret its contents or use it as a future-session capability. Equal
|
|
1866
|
+
* generations mean the connection/receive/capability binding is unchanged;
|
|
1867
|
+
* leadership and outbound confirmation can still change within a generation.
|
|
1868
|
+
*/
|
|
1869
|
+
export type PersistedReceiptPeerReadiness =
|
|
1870
|
+
| Readonly<{
|
|
1871
|
+
status: "ready";
|
|
1872
|
+
generation: string;
|
|
1873
|
+
}>
|
|
1874
|
+
| Readonly<{
|
|
1875
|
+
status: "pending";
|
|
1876
|
+
reason: PersistedReceiptPeerReadinessPendingReason;
|
|
1877
|
+
generation?: string;
|
|
1878
|
+
}>
|
|
1879
|
+
| Readonly<{
|
|
1880
|
+
status: "unsupported";
|
|
1881
|
+
reason: PersistedReceiptPeerReadinessUnsupportedReason;
|
|
1882
|
+
generation: string;
|
|
1883
|
+
}>;
|
|
1884
|
+
|
|
1885
|
+
export type PersistedReceiptPeerReady = Extract<
|
|
1886
|
+
PersistedReceiptPeerReadiness,
|
|
1887
|
+
{ status: "ready" }
|
|
1888
|
+
>;
|
|
1889
|
+
|
|
1890
|
+
export type PersistedReceiptPeerReadinessOptions<
|
|
1891
|
+
T,
|
|
1892
|
+
R extends "u32" | "u64",
|
|
1893
|
+
> = Readonly<{
|
|
1894
|
+
/** Require this peer to be a freshly planned leader for every entry. */
|
|
1895
|
+
entries?: readonly (ShallowOrFullEntry<T> | EntryReplicated<R>)[];
|
|
1896
|
+
/**
|
|
1897
|
+
* Total leader-plan replica degree used for `entries`; defaults to this log's
|
|
1898
|
+
* configured minimum. This is not the persisted delivery `minAcks` count.
|
|
1899
|
+
*/
|
|
1900
|
+
replicas?: number;
|
|
1901
|
+
}>;
|
|
1902
|
+
|
|
1903
|
+
export type WaitForPersistedReceiptPeerReadinessOptions<
|
|
1904
|
+
T,
|
|
1905
|
+
R extends "u32" | "u64",
|
|
1906
|
+
> = PersistedReceiptPeerReadinessOptions<T, R> &
|
|
1907
|
+
Readonly<{
|
|
1908
|
+
timeout?: number;
|
|
1909
|
+
signal?: AbortSignal;
|
|
1910
|
+
}>;
|
|
1911
|
+
|
|
1755
1912
|
type PersistedDeliveryOptions = Readonly<{
|
|
1756
1913
|
reliability: "persisted";
|
|
1757
1914
|
minAcks: number;
|
|
@@ -1793,6 +1950,7 @@ const PERSISTED_RECEIPT_RETRY_MS = 50;
|
|
|
1793
1950
|
const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
|
|
1794
1951
|
const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
|
|
1795
1952
|
const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
|
|
1953
|
+
const MAX_PERSISTED_RECEIPT_READINESS_WAITERS = 1_024;
|
|
1796
1954
|
const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
|
|
1797
1955
|
const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
|
|
1798
1956
|
const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
|
|
@@ -2167,6 +2325,10 @@ export type ReplicatorLeaveEvent = { publicKey: PublicSignKey };
|
|
|
2167
2325
|
export type ReplicationChangeEvent = { publicKey: PublicSignKey };
|
|
2168
2326
|
export type ReplicatorMatureEvent = { publicKey: PublicSignKey };
|
|
2169
2327
|
export type ReplicationStatusEvent = ReplicationStatus;
|
|
2328
|
+
/** `peerHash` is the result of `PublicSignKey.hashcode()`. */
|
|
2329
|
+
export type PersistedReceiptPeerReadinessEvent = Readonly<{
|
|
2330
|
+
peerHash: string;
|
|
2331
|
+
}>;
|
|
2170
2332
|
|
|
2171
2333
|
class ReplicationStatusSnapshotChangedError extends Error {
|
|
2172
2334
|
constructor() {
|
|
@@ -2189,6 +2351,12 @@ export interface SharedLogEvents extends ProgramEvents {
|
|
|
2189
2351
|
"replication:change": CustomEvent<ReplicationChangeEvent>;
|
|
2190
2352
|
"replicator:mature": CustomEvent<ReplicatorMatureEvent>;
|
|
2191
2353
|
"replication:status": CustomEvent<ReplicationStatusEvent>;
|
|
2354
|
+
/**
|
|
2355
|
+
* Non-exhaustive wake hint that a peer may now produce a new readiness
|
|
2356
|
+
* snapshot. Consumers must re-read the snapshot; this event is deliberately
|
|
2357
|
+
* not a durable transition log and a `ready` result remains advisory.
|
|
2358
|
+
*/
|
|
2359
|
+
"persisted-receipt:readiness": CustomEvent<PersistedReceiptPeerReadinessEvent>;
|
|
2192
2360
|
}
|
|
2193
2361
|
|
|
2194
2362
|
export type SharedLogRuntimeSnapshot = Readonly<{
|
|
@@ -3742,6 +3910,23 @@ export class SharedLog<
|
|
|
3742
3910
|
// parallel map so existing capability-number consumers remain unchanged.
|
|
3743
3911
|
private _peerSyncCapabilitySessions!: Map<string, bigint>;
|
|
3744
3912
|
private _peerSyncCapabilityTimestamps!: Map<string, bigint>;
|
|
3913
|
+
// design-note: these fields cache a stable, public diagnostics token for the
|
|
3914
|
+
// composite of PeerSession identity, receive epoch, and signed capability
|
|
3915
|
+
// session. They are not consulted to admit or fence asynchronous work. A
|
|
3916
|
+
// separate opaque token is necessary because exposing any of those internal
|
|
3917
|
+
// identities would leak protocol/session values, while PeerSession alone does
|
|
3918
|
+
// not change when receive or capability state is replaced.
|
|
3919
|
+
private _persistedReceiptReadinessGenerations!: WeakMap<
|
|
3920
|
+
PeerSession,
|
|
3921
|
+
{
|
|
3922
|
+
receiveEpoch: object | null;
|
|
3923
|
+
capabilitySession?: bigint;
|
|
3924
|
+
generation: string;
|
|
3925
|
+
}
|
|
3926
|
+
>;
|
|
3927
|
+
private _persistedReceiptReadinessGenerationPrefix!: string;
|
|
3928
|
+
private _persistedReceiptReadinessGenerationCounter!: number;
|
|
3929
|
+
private _persistedReceiptReadinessWaiters!: Set<object>;
|
|
3745
3930
|
private _persistedReceiptStorage?: PersistedReceiptStorage;
|
|
3746
3931
|
private _persistedReceiptRequestsInFlight!: Map<string, number>;
|
|
3747
3932
|
private _persistedReceiptRequestsInFlightTotal!: number;
|
|
@@ -4099,6 +4284,12 @@ export class SharedLog<
|
|
|
4099
4284
|
this._peerSyncCapabilities = new Map();
|
|
4100
4285
|
this._peerSyncCapabilitySessions = new Map();
|
|
4101
4286
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
4287
|
+
this._persistedReceiptReadinessGenerations = new WeakMap();
|
|
4288
|
+
this._persistedReceiptReadinessGenerationPrefix = toHexString(
|
|
4289
|
+
randomBytes(8),
|
|
4290
|
+
);
|
|
4291
|
+
this._persistedReceiptReadinessGenerationCounter = 0;
|
|
4292
|
+
this._persistedReceiptReadinessWaiters = new Set();
|
|
4102
4293
|
this._persistedReceiptStorage = undefined;
|
|
4103
4294
|
this._persistedReceiptRequestsInFlight = new Map();
|
|
4104
4295
|
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
@@ -4242,16 +4433,38 @@ export class SharedLog<
|
|
|
4242
4433
|
});
|
|
4243
4434
|
channel.addEventListener("unicast", this._onFanoutUnicastFn);
|
|
4244
4435
|
|
|
4436
|
+
const profile = this._logProperties?.sync?.profile;
|
|
4437
|
+
const startedAt = syncProfileStart(profile);
|
|
4438
|
+
const mode =
|
|
4439
|
+
resolvedRoot === fanoutService.publicKeyHash ? "root" : "node";
|
|
4440
|
+
const before = profile
|
|
4441
|
+
? snapshotFanoutOpenMetrics(fanoutService, this.topic, resolvedRoot)
|
|
4442
|
+
: undefined;
|
|
4443
|
+
let outcome: "error" | "opened" | "joined" = "error";
|
|
4245
4444
|
try {
|
|
4246
4445
|
const channelOptions = this.getFanoutChannelOptions(options);
|
|
4247
|
-
if (
|
|
4446
|
+
if (mode === "root") {
|
|
4248
4447
|
await channel.openAsRoot(channelOptions);
|
|
4448
|
+
outcome = "opened";
|
|
4249
4449
|
return;
|
|
4250
4450
|
}
|
|
4251
4451
|
await channel.join(channelOptions, options.join);
|
|
4452
|
+
outcome = "joined";
|
|
4252
4453
|
} catch (error) {
|
|
4253
4454
|
this._closeFanoutChannel();
|
|
4254
4455
|
throw error;
|
|
4456
|
+
} finally {
|
|
4457
|
+
emitFanoutOpenProfile({
|
|
4458
|
+
profile,
|
|
4459
|
+
startedAt,
|
|
4460
|
+
service: fanoutService,
|
|
4461
|
+
topic: this.topic,
|
|
4462
|
+
root: resolvedRoot,
|
|
4463
|
+
mode,
|
|
4464
|
+
outcome,
|
|
4465
|
+
timeoutMs: options.join?.timeoutMs,
|
|
4466
|
+
before,
|
|
4467
|
+
});
|
|
4255
4468
|
}
|
|
4256
4469
|
}
|
|
4257
4470
|
|
|
@@ -5061,15 +5274,23 @@ export class SharedLog<
|
|
|
5061
5274
|
) {
|
|
5062
5275
|
return false;
|
|
5063
5276
|
}
|
|
5277
|
+
const nextCapabilities = previous.capabilities | capabilities;
|
|
5278
|
+
const nextTimestamp =
|
|
5279
|
+
previous.timestamp === undefined || timestamp > previous.timestamp
|
|
5280
|
+
? timestamp
|
|
5281
|
+
: previous.timestamp;
|
|
5064
5282
|
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
5065
5283
|
epoch: openingSession,
|
|
5066
|
-
capabilities:
|
|
5284
|
+
capabilities: nextCapabilities,
|
|
5067
5285
|
transportSession,
|
|
5068
|
-
timestamp:
|
|
5069
|
-
previous.timestamp === undefined || timestamp > previous.timestamp
|
|
5070
|
-
? timestamp
|
|
5071
|
-
: previous.timestamp,
|
|
5286
|
+
timestamp: nextTimestamp,
|
|
5072
5287
|
});
|
|
5288
|
+
if (
|
|
5289
|
+
previous.capabilities !== nextCapabilities ||
|
|
5290
|
+
previous.timestamp === undefined
|
|
5291
|
+
) {
|
|
5292
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5293
|
+
}
|
|
5073
5294
|
return true;
|
|
5074
5295
|
}
|
|
5075
5296
|
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
@@ -5078,17 +5299,25 @@ export class SharedLog<
|
|
|
5078
5299
|
transportSession,
|
|
5079
5300
|
timestamp,
|
|
5080
5301
|
});
|
|
5302
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5081
5303
|
return true;
|
|
5082
5304
|
}
|
|
5083
5305
|
|
|
5084
5306
|
if (transportSession === undefined || timestamp === undefined) {
|
|
5085
5307
|
// Test/in-process synthetic contexts predate signed envelope captures.
|
|
5086
5308
|
// They may exercise capability-number behavior, but can never authorize V2.
|
|
5309
|
+
const readinessChanged =
|
|
5310
|
+
this._peerSyncCapabilities.get(peerHash) !== capabilities ||
|
|
5311
|
+
this._peerSyncCapabilitySessions.has(peerHash) ||
|
|
5312
|
+
this._peerSyncCapabilityTimestamps.has(peerHash);
|
|
5087
5313
|
this._peerSyncCapabilities.set(peerHash, capabilities);
|
|
5088
5314
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
5089
5315
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
5090
5316
|
this._v2Send.advancePeerCapability(peerHash);
|
|
5091
5317
|
this._v2Receive.revokePeerCapability(peerHash);
|
|
5318
|
+
if (readinessChanged) {
|
|
5319
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5320
|
+
}
|
|
5092
5321
|
return true;
|
|
5093
5322
|
}
|
|
5094
5323
|
|
|
@@ -5113,6 +5342,10 @@ export class SharedLog<
|
|
|
5113
5342
|
!sameTransportSession ||
|
|
5114
5343
|
(previousCapabilities & senderGrantCapabilityMask) !==
|
|
5115
5344
|
(nextCapabilities & senderGrantCapabilityMask);
|
|
5345
|
+
const readinessChanged =
|
|
5346
|
+
!sameTransportSession ||
|
|
5347
|
+
previousTimestamp === undefined ||
|
|
5348
|
+
previousCapabilities !== nextCapabilities;
|
|
5116
5349
|
this._peerSyncCapabilities.set(peerHash, nextCapabilities);
|
|
5117
5350
|
this._peerSyncCapabilitySessions.set(peerHash, transportSession);
|
|
5118
5351
|
this._peerSyncCapabilityTimestamps.set(
|
|
@@ -5129,6 +5362,9 @@ export class SharedLog<
|
|
|
5129
5362
|
// recovery re-solicitation may restart from the base interval.
|
|
5130
5363
|
this.resetReplicationInfoV2RecoveryEscalation(peerHash);
|
|
5131
5364
|
}
|
|
5365
|
+
if (readinessChanged) {
|
|
5366
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5367
|
+
}
|
|
5132
5368
|
return true;
|
|
5133
5369
|
}
|
|
5134
5370
|
|
|
@@ -5478,9 +5714,148 @@ export class SharedLog<
|
|
|
5478
5714
|
return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
|
|
5479
5715
|
}
|
|
5480
5716
|
|
|
5717
|
+
private persistedReceiptReadinessGeneration(
|
|
5718
|
+
peerSession: PeerSession,
|
|
5719
|
+
receiveEpoch: object | null,
|
|
5720
|
+
capabilitySession: bigint | undefined,
|
|
5721
|
+
): string {
|
|
5722
|
+
const current = this._persistedReceiptReadinessGenerations.get(peerSession);
|
|
5723
|
+
if (
|
|
5724
|
+
current?.receiveEpoch === receiveEpoch &&
|
|
5725
|
+
current.capabilitySession === capabilitySession
|
|
5726
|
+
) {
|
|
5727
|
+
return current.generation;
|
|
5728
|
+
}
|
|
5729
|
+
const generation = `${this._persistedReceiptReadinessGenerationPrefix}:${(++this
|
|
5730
|
+
._persistedReceiptReadinessGenerationCounter).toString(36)}`;
|
|
5731
|
+
this._persistedReceiptReadinessGenerations.set(peerSession, {
|
|
5732
|
+
receiveEpoch,
|
|
5733
|
+
capabilitySession,
|
|
5734
|
+
generation,
|
|
5735
|
+
});
|
|
5736
|
+
return generation;
|
|
5737
|
+
}
|
|
5738
|
+
|
|
5739
|
+
private pendingPersistedReceiptReadiness(
|
|
5740
|
+
reason: PersistedReceiptPeerReadinessPendingReason,
|
|
5741
|
+
generation?: string,
|
|
5742
|
+
): PersistedReceiptPeerReadiness {
|
|
5743
|
+
return Object.freeze({
|
|
5744
|
+
status: "pending" as const,
|
|
5745
|
+
reason,
|
|
5746
|
+
...(generation === undefined ? {} : { generation }),
|
|
5747
|
+
});
|
|
5748
|
+
}
|
|
5749
|
+
|
|
5750
|
+
private unsupportedPersistedReceiptReadiness(
|
|
5751
|
+
reason: PersistedReceiptPeerReadinessUnsupportedReason,
|
|
5752
|
+
generation: string,
|
|
5753
|
+
): PersistedReceiptPeerReadiness {
|
|
5754
|
+
return Object.freeze({
|
|
5755
|
+
status: "unsupported" as const,
|
|
5756
|
+
reason,
|
|
5757
|
+
generation,
|
|
5758
|
+
});
|
|
5759
|
+
}
|
|
5760
|
+
|
|
5761
|
+
private dispatchPersistedReceiptReadinessChange(peerHash: string): void {
|
|
5762
|
+
this.events.dispatchEvent(
|
|
5763
|
+
new CustomEvent<PersistedReceiptPeerReadinessEvent>(
|
|
5764
|
+
"persisted-receipt:readiness",
|
|
5765
|
+
{ detail: Object.freeze({ peerHash }) },
|
|
5766
|
+
),
|
|
5767
|
+
);
|
|
5768
|
+
}
|
|
5769
|
+
|
|
5770
|
+
private persistedReceiptReadinessCandidate(peerHash: string):
|
|
5771
|
+
| {
|
|
5772
|
+
capabilitySession: bigint;
|
|
5773
|
+
peerSession: PeerSession;
|
|
5774
|
+
receiveEpoch: object | null;
|
|
5775
|
+
generation: string;
|
|
5776
|
+
}
|
|
5777
|
+
| PersistedReceiptPeerReadiness {
|
|
5778
|
+
if (this.closed) {
|
|
5779
|
+
return this.pendingPersistedReceiptReadiness("closed");
|
|
5780
|
+
}
|
|
5781
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
5782
|
+
if (!peerSession) {
|
|
5783
|
+
return this.pendingPersistedReceiptReadiness("no-current-session");
|
|
5784
|
+
}
|
|
5785
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
5786
|
+
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
5787
|
+
const generation = this.persistedReceiptReadinessGeneration(
|
|
5788
|
+
peerSession,
|
|
5789
|
+
receiveEpoch,
|
|
5790
|
+
capabilitySession,
|
|
5791
|
+
);
|
|
5792
|
+
if (
|
|
5793
|
+
peerSession.phase !== "open" ||
|
|
5794
|
+
!peerSession.isActive() ||
|
|
5795
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
5796
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash)
|
|
5797
|
+
) {
|
|
5798
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5799
|
+
"session-opening",
|
|
5800
|
+
generation,
|
|
5801
|
+
);
|
|
5802
|
+
}
|
|
5803
|
+
if (
|
|
5804
|
+
capabilitySession === undefined ||
|
|
5805
|
+
!this._peerSyncCapabilityTimestamps.has(peerHash)
|
|
5806
|
+
) {
|
|
5807
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5808
|
+
"capability-pending",
|
|
5809
|
+
generation,
|
|
5810
|
+
);
|
|
5811
|
+
}
|
|
5812
|
+
const capabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
5813
|
+
if ((capabilities & SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) === 0) {
|
|
5814
|
+
return this.unsupportedPersistedReceiptReadiness(
|
|
5815
|
+
"persisted-receipts-unsupported",
|
|
5816
|
+
generation,
|
|
5817
|
+
);
|
|
5818
|
+
}
|
|
5819
|
+
if ((capabilities & SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) === 0) {
|
|
5820
|
+
return this.unsupportedPersistedReceiptReadiness(
|
|
5821
|
+
"replication-confirmation-unsupported",
|
|
5822
|
+
generation,
|
|
5823
|
+
);
|
|
5824
|
+
}
|
|
5825
|
+
if (
|
|
5826
|
+
!this._v2Receive.isCurrentActive({
|
|
5827
|
+
peerHash,
|
|
5828
|
+
peerSession,
|
|
5829
|
+
receiveEpoch,
|
|
5830
|
+
senderTransportSession: capabilitySession,
|
|
5831
|
+
})
|
|
5832
|
+
) {
|
|
5833
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5834
|
+
"replication-state-pending",
|
|
5835
|
+
generation,
|
|
5836
|
+
);
|
|
5837
|
+
}
|
|
5838
|
+
if (!this.uniqueReplicators.has(peerHash)) {
|
|
5839
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5840
|
+
"not-replicating",
|
|
5841
|
+
generation,
|
|
5842
|
+
);
|
|
5843
|
+
}
|
|
5844
|
+
return {
|
|
5845
|
+
capabilitySession,
|
|
5846
|
+
peerSession,
|
|
5847
|
+
receiveEpoch,
|
|
5848
|
+
generation,
|
|
5849
|
+
};
|
|
5850
|
+
}
|
|
5851
|
+
|
|
5481
5852
|
private persistedReceiptPeerSession(
|
|
5482
5853
|
peerHash: string,
|
|
5483
5854
|
): { capabilitySession: bigint; peerSession: PeerSession } | undefined {
|
|
5855
|
+
// This is a hot receipt/transfer-loop predicate. Keep it allocation-light,
|
|
5856
|
+
// while mirroring every exact-session gate in
|
|
5857
|
+
// persistedReceiptReadinessCandidate (which additionally creates public
|
|
5858
|
+
// reason/generation snapshots).
|
|
5484
5859
|
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
5485
5860
|
const peerSession = this._peerSessions.current(peerHash);
|
|
5486
5861
|
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
@@ -5488,10 +5863,14 @@ export class SharedLog<
|
|
|
5488
5863
|
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS |
|
|
5489
5864
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM;
|
|
5490
5865
|
if (
|
|
5866
|
+
this.closed ||
|
|
5491
5867
|
capabilitySession == null ||
|
|
5492
5868
|
!peerSession ||
|
|
5493
5869
|
peerSession.phase !== "open" ||
|
|
5494
|
-
!
|
|
5870
|
+
!peerSession.isActive() ||
|
|
5871
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
5872
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
5873
|
+
!this.uniqueReplicators.has(peerHash) ||
|
|
5495
5874
|
!this._peerSyncCapabilityTimestamps.has(peerHash) ||
|
|
5496
5875
|
((this._peerSyncCapabilities.get(peerHash) ?? 0) &
|
|
5497
5876
|
requiredCapabilities) !==
|
|
@@ -8181,8 +8560,11 @@ export class SharedLog<
|
|
|
8181
8560
|
? checkedPruneCoordinator.fencePeerRemoval(keyHash)
|
|
8182
8561
|
: undefined;
|
|
8183
8562
|
const blockPeerReceiveAdmission = () => {
|
|
8184
|
-
releaseReceiveCleanupGate
|
|
8185
|
-
|
|
8563
|
+
if (!releaseReceiveCleanupGate) {
|
|
8564
|
+
releaseReceiveCleanupGate =
|
|
8565
|
+
this._peerSessions.acquireReceiveCleanupGate(keyHash);
|
|
8566
|
+
this.dispatchPersistedReceiptReadinessChange(keyHash);
|
|
8567
|
+
}
|
|
8186
8568
|
};
|
|
8187
8569
|
if (!isMe && !isSpeculativePeerRemoval) {
|
|
8188
8570
|
// Revoke this peer's receipts synchronously, before this removal can
|
|
@@ -8423,7 +8805,10 @@ export class SharedLog<
|
|
|
8423
8805
|
});
|
|
8424
8806
|
removalCallCompleted = true;
|
|
8425
8807
|
} finally {
|
|
8426
|
-
releaseReceiveCleanupGate
|
|
8808
|
+
if (releaseReceiveCleanupGate) {
|
|
8809
|
+
releaseReceiveCleanupGate();
|
|
8810
|
+
this.dispatchPersistedReceiptReadinessChange(keyHash);
|
|
8811
|
+
}
|
|
8427
8812
|
if (
|
|
8428
8813
|
replicationInfoRecoveryEpochAdvanced &&
|
|
8429
8814
|
ownsReplicationOwnershipLifecycle() &&
|
|
@@ -16817,6 +17202,8 @@ export class SharedLog<
|
|
|
16817
17202
|
(this.node as unknown as NodeWithSharedLogNativeDefaults)
|
|
16818
17203
|
.sharedLogNativeDefaults,
|
|
16819
17204
|
);
|
|
17205
|
+
const openProfile = options?.sync?.profile;
|
|
17206
|
+
const openStartedAt = syncProfileStart(openProfile);
|
|
16820
17207
|
this.replicas = {
|
|
16821
17208
|
min:
|
|
16822
17209
|
options?.replicas?.min != null
|
|
@@ -16904,6 +17291,12 @@ export class SharedLog<
|
|
|
16904
17291
|
this._peerSyncCapabilities = new Map();
|
|
16905
17292
|
this._peerSyncCapabilitySessions = new Map();
|
|
16906
17293
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
17294
|
+
this._persistedReceiptReadinessGenerations = new WeakMap();
|
|
17295
|
+
this._persistedReceiptReadinessGenerationPrefix = toHexString(
|
|
17296
|
+
randomBytes(8),
|
|
17297
|
+
);
|
|
17298
|
+
this._persistedReceiptReadinessGenerationCounter = 0;
|
|
17299
|
+
this._persistedReceiptReadinessWaiters = new Set();
|
|
16907
17300
|
this._persistedReceiptStorage = undefined;
|
|
16908
17301
|
this._persistedReceiptRequestsInFlight = new Map();
|
|
16909
17302
|
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
@@ -17041,6 +17434,7 @@ export class SharedLog<
|
|
|
17041
17434
|
this.keep = options?.keep;
|
|
17042
17435
|
this.pendingMaturity = new Map();
|
|
17043
17436
|
|
|
17437
|
+
const localStateStartedAt = syncProfileStart(openProfile);
|
|
17044
17438
|
const id = sha256Base64Sync(this.log.id);
|
|
17045
17439
|
const [storage, logScope] = await Promise.all([
|
|
17046
17440
|
this.node.storage.sublevel(id),
|
|
@@ -17096,6 +17490,11 @@ export class SharedLog<
|
|
|
17096
17490
|
this._entryCoordinatesIndex = await replicationIndex.init({
|
|
17097
17491
|
schema: this.indexableDomain.constructorEntry,
|
|
17098
17492
|
});
|
|
17493
|
+
emitAdvisorySyncProfileDuration(openProfile, localStateStartedAt, {
|
|
17494
|
+
name: "sharedLog.open.localState",
|
|
17495
|
+
component: "shared-log",
|
|
17496
|
+
});
|
|
17497
|
+
const blockStoreStartedAt = syncProfileStart(openProfile);
|
|
17099
17498
|
const deferStandaloneNativeRangePlanner =
|
|
17100
17499
|
!!options?.nativeBackbone && options.nativeRangePlanner == null;
|
|
17101
17500
|
await this.openNativeRangePlanner(
|
|
@@ -17146,6 +17545,14 @@ export class SharedLog<
|
|
|
17146
17545
|
storage as unknown as DurableBlockSublevelStore,
|
|
17147
17546
|
);
|
|
17148
17547
|
}
|
|
17548
|
+
emitAdvisorySyncProfileDuration(openProfile, blockStoreStartedAt, {
|
|
17549
|
+
name: "sharedLog.open.blockStore",
|
|
17550
|
+
component: "shared-log",
|
|
17551
|
+
details: {
|
|
17552
|
+
nativeBackbone: this._nativeBackbone != null,
|
|
17553
|
+
directoryConfigured: this.node.directory != null,
|
|
17554
|
+
},
|
|
17555
|
+
});
|
|
17149
17556
|
this.remoteBlocks = new RemoteBlocks({
|
|
17150
17557
|
local: localBlocks,
|
|
17151
17558
|
publish: (message, options) =>
|
|
@@ -17156,38 +17563,76 @@ export class SharedLog<
|
|
|
17156
17563
|
// compatible eager path with bounded validation and storage budgets.
|
|
17157
17564
|
eagerBlocks: options?.eagerBlocks ?? false,
|
|
17158
17565
|
resolveProviders: async (cid, opts) => {
|
|
17566
|
+
const profile = this._logProperties?.sync?.profile;
|
|
17159
17567
|
const maxPeers = 8;
|
|
17568
|
+
const excluded = new Set((opts?.exclude ?? []).slice(0, maxPeers));
|
|
17569
|
+
const lookupPeers = opts?.refresh
|
|
17570
|
+
? Math.min(maxPeers * 2, maxPeers + excluded.size)
|
|
17571
|
+
: maxPeers;
|
|
17572
|
+
const resolutionStartedAt = syncProfileStart(profile);
|
|
17160
17573
|
const localCandidates =
|
|
17161
17574
|
(await this.resolveCandidatePeersForHash(cid, {
|
|
17162
17575
|
signal: opts?.signal,
|
|
17163
|
-
maxPeers,
|
|
17576
|
+
maxPeers: lookupPeers,
|
|
17164
17577
|
})) ?? [];
|
|
17165
|
-
|
|
17578
|
+
const emitResolution = profile
|
|
17579
|
+
? (
|
|
17580
|
+
status: "aborted" | "local" | "directory",
|
|
17581
|
+
targets: number,
|
|
17582
|
+
directoryCandidates = 0,
|
|
17583
|
+
reachableCandidates = 0,
|
|
17584
|
+
) =>
|
|
17585
|
+
emitAdvisorySyncProfileDuration(profile, resolutionStartedAt, {
|
|
17586
|
+
name: "sharedLog.blocks.resolveProviders",
|
|
17587
|
+
component: "shared-log",
|
|
17588
|
+
count: targets,
|
|
17589
|
+
targets,
|
|
17590
|
+
details: {
|
|
17591
|
+
status,
|
|
17592
|
+
refresh: opts?.refresh === true,
|
|
17593
|
+
excluded: excluded.size,
|
|
17594
|
+
lookupPeers,
|
|
17595
|
+
localCandidates: localCandidates.length,
|
|
17596
|
+
directoryCandidates,
|
|
17597
|
+
reachableCandidates,
|
|
17598
|
+
},
|
|
17599
|
+
})
|
|
17600
|
+
: undefined;
|
|
17601
|
+
if (opts?.signal?.aborted) {
|
|
17602
|
+
emitResolution?.("aborted", 0);
|
|
17603
|
+
return [];
|
|
17604
|
+
}
|
|
17166
17605
|
const locallyReachable = new Set(
|
|
17167
17606
|
await this._getLocalReachablePeerHashes(this.topic),
|
|
17168
17607
|
);
|
|
17608
|
+
if (opts?.signal?.aborted) {
|
|
17609
|
+
emitResolution?.("aborted", 0, 0, locallyReachable.size);
|
|
17610
|
+
return [];
|
|
17611
|
+
}
|
|
17169
17612
|
const confirmed = this._checkedPrune.getConfirmedReplicators(cid);
|
|
17170
17613
|
const contacted = this._checkedPrune.getContactedReplicators(cid);
|
|
17614
|
+
const hasProviderEvidence = (peer: string) =>
|
|
17615
|
+
confirmed?.has(peer) === true ||
|
|
17616
|
+
contacted?.has(peer) ||
|
|
17617
|
+
this.uniqueReplicators.has(peer);
|
|
17171
17618
|
const hasLiveCandidate = localCandidates.some(
|
|
17172
|
-
(peer) =>
|
|
17173
|
-
locallyReachable.has(peer) &&
|
|
17174
|
-
(confirmed?.has(peer) ||
|
|
17175
|
-
contacted?.has(peer) ||
|
|
17176
|
-
this.uniqueReplicators.has(peer)),
|
|
17619
|
+
(peer) => locallyReachable.has(peer) && hasProviderEvidence(peer),
|
|
17177
17620
|
);
|
|
17178
17621
|
|
|
17179
17622
|
// Only reachability corroborated by provider/replicator evidence may
|
|
17180
17623
|
// bypass the initial CID lookup. Arbitrary bootstrap connections are
|
|
17181
17624
|
// useful fallbacks, but are not evidence that they hold this block.
|
|
17182
17625
|
if (hasLiveCandidate && !opts?.refresh) {
|
|
17183
|
-
|
|
17626
|
+
const selected = localCandidates.slice(0, maxPeers);
|
|
17627
|
+
emitResolution?.("local", selected.length, 0, locallyReachable.size);
|
|
17628
|
+
return selected;
|
|
17184
17629
|
}
|
|
17185
17630
|
|
|
17186
17631
|
let directoryProviders: string[] = [];
|
|
17187
17632
|
try {
|
|
17188
17633
|
const query = (namespace: string) =>
|
|
17189
17634
|
fanoutService?.queryProviders(namespace, {
|
|
17190
|
-
want:
|
|
17635
|
+
want: lookupPeers,
|
|
17191
17636
|
timeoutMs: 2_000,
|
|
17192
17637
|
queryTimeoutMs: 500,
|
|
17193
17638
|
bootstrapMaxPeers: 2,
|
|
@@ -17199,12 +17644,21 @@ export class SharedLog<
|
|
|
17199
17644
|
]);
|
|
17200
17645
|
for (const result of results) {
|
|
17201
17646
|
if (result.status === "fulfilled") {
|
|
17202
|
-
directoryProviders.push(...result.value);
|
|
17647
|
+
directoryProviders.push(...result.value.slice(0, lookupPeers));
|
|
17203
17648
|
}
|
|
17204
17649
|
}
|
|
17205
17650
|
} catch {
|
|
17206
17651
|
// Ignore discovery failures; local evidence remains usable.
|
|
17207
17652
|
}
|
|
17653
|
+
if (opts?.signal?.aborted) {
|
|
17654
|
+
emitResolution?.(
|
|
17655
|
+
"aborted",
|
|
17656
|
+
0,
|
|
17657
|
+
directoryProviders.length,
|
|
17658
|
+
locallyReachable.size,
|
|
17659
|
+
);
|
|
17660
|
+
return [];
|
|
17661
|
+
}
|
|
17208
17662
|
|
|
17209
17663
|
const selected: string[] = [];
|
|
17210
17664
|
const selectedSet = new Set<string>();
|
|
@@ -17215,15 +17669,62 @@ export class SharedLog<
|
|
|
17215
17669
|
selectedSet.add(peer);
|
|
17216
17670
|
selected.push(peer);
|
|
17217
17671
|
};
|
|
17218
|
-
|
|
17219
|
-
|
|
17220
|
-
|
|
17221
|
-
(
|
|
17222
|
-
|
|
17223
|
-
|
|
17224
|
-
|
|
17225
|
-
|
|
17226
|
-
|
|
17672
|
+
const append = (
|
|
17673
|
+
providers: readonly string[],
|
|
17674
|
+
includeExcluded: boolean,
|
|
17675
|
+
predicate?: (provider: string) => boolean,
|
|
17676
|
+
) => {
|
|
17677
|
+
for (const provider of providers) {
|
|
17678
|
+
if (selected.length >= maxPeers) return;
|
|
17679
|
+
if (
|
|
17680
|
+
excluded.has(provider) === includeExcluded &&
|
|
17681
|
+
(!predicate || predicate(provider))
|
|
17682
|
+
) {
|
|
17683
|
+
add(provider);
|
|
17684
|
+
}
|
|
17685
|
+
}
|
|
17686
|
+
};
|
|
17687
|
+
const appendInterleaved = (includeExcluded: boolean) => {
|
|
17688
|
+
for (
|
|
17689
|
+
let index = 0;
|
|
17690
|
+
selected.length < maxPeers &&
|
|
17691
|
+
(index < localCandidates.length ||
|
|
17692
|
+
index < directoryProviders.length);
|
|
17693
|
+
index++
|
|
17694
|
+
) {
|
|
17695
|
+
const local = localCandidates[index];
|
|
17696
|
+
if (local && excluded.has(local) === includeExcluded) add(local);
|
|
17697
|
+
const directory = directoryProviders[index];
|
|
17698
|
+
if (directory && excluded.has(directory) === includeExcluded) {
|
|
17699
|
+
add(directory);
|
|
17700
|
+
}
|
|
17701
|
+
}
|
|
17702
|
+
};
|
|
17703
|
+
if (opts?.refresh) {
|
|
17704
|
+
// Retry results are wider than the regular eight-peer window. Prefer
|
|
17705
|
+
// untried reachable holders, then the remaining fresh directory
|
|
17706
|
+
// evidence, without discarding attempted peers as bounded transient-
|
|
17707
|
+
// failure fallbacks.
|
|
17708
|
+
append(directoryProviders, false, (peer) =>
|
|
17709
|
+
locallyReachable.has(peer),
|
|
17710
|
+
);
|
|
17711
|
+
append(
|
|
17712
|
+
localCandidates,
|
|
17713
|
+
false,
|
|
17714
|
+
(peer) => locallyReachable.has(peer) && hasProviderEvidence(peer),
|
|
17715
|
+
);
|
|
17716
|
+
append(directoryProviders, false);
|
|
17717
|
+
append(localCandidates, false);
|
|
17718
|
+
} else {
|
|
17719
|
+
appendInterleaved(false);
|
|
17720
|
+
}
|
|
17721
|
+
appendInterleaved(true);
|
|
17722
|
+
emitResolution?.(
|
|
17723
|
+
"directory",
|
|
17724
|
+
selected.length,
|
|
17725
|
+
directoryProviders.length,
|
|
17726
|
+
locallyReachable.size,
|
|
17727
|
+
);
|
|
17227
17728
|
return selected;
|
|
17228
17729
|
},
|
|
17229
17730
|
watchProviders: fanoutService
|
|
@@ -17271,7 +17772,13 @@ export class SharedLog<
|
|
|
17271
17772
|
: undefined,
|
|
17272
17773
|
});
|
|
17273
17774
|
|
|
17274
|
-
const
|
|
17775
|
+
const remoteBlocksStartedAt = syncProfileStart(openProfile);
|
|
17776
|
+
const remoteBlocksStartPromise = this.remoteBlocks.start().then(() => {
|
|
17777
|
+
emitAdvisorySyncProfileDuration(openProfile, remoteBlocksStartedAt, {
|
|
17778
|
+
name: "sharedLog.open.remoteBlocks",
|
|
17779
|
+
component: "shared-log",
|
|
17780
|
+
});
|
|
17781
|
+
});
|
|
17275
17782
|
const hasIndexedReplicationInfo =
|
|
17276
17783
|
(await this.replicationIndex.count({
|
|
17277
17784
|
query: [
|
|
@@ -17487,6 +17994,7 @@ export class SharedLog<
|
|
|
17487
17994
|
// joins rely on: a replicate:false observer syncing a head whose parents
|
|
17488
17995
|
// are not local would fail block resolution, and Log.join treats that as
|
|
17489
17996
|
// recoverable and skips the entry without persisting anything.
|
|
17997
|
+
const lowerLogStartedAt = syncProfileStart(openProfile);
|
|
17490
17998
|
await this.log.open(this.remoteBlocks, this.node.identity, {
|
|
17491
17999
|
keychain: this.node.services.keychain,
|
|
17492
18000
|
resolveRemotePeers: (hash, options) =>
|
|
@@ -17518,6 +18026,10 @@ export class SharedLog<
|
|
|
17518
18026
|
},
|
|
17519
18027
|
indexer: logIndex,
|
|
17520
18028
|
});
|
|
18029
|
+
emitAdvisorySyncProfileDuration(openProfile, lowerLogStartedAt, {
|
|
18030
|
+
name: "sharedLog.open.lowerLog",
|
|
18031
|
+
component: "shared-log",
|
|
18032
|
+
});
|
|
17521
18033
|
this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
|
|
17522
18034
|
try {
|
|
17523
18035
|
const recovered =
|
|
@@ -17582,6 +18094,7 @@ export class SharedLog<
|
|
|
17582
18094
|
this._onUnsubscription(event),
|
|
17583
18095
|
);
|
|
17584
18096
|
});
|
|
18097
|
+
const communicationStartedAt = syncProfileStart(openProfile);
|
|
17585
18098
|
await Promise.all([
|
|
17586
18099
|
this.rpc.open({
|
|
17587
18100
|
queryType: TransportMessage,
|
|
@@ -17600,7 +18113,12 @@ export class SharedLog<
|
|
|
17600
18113
|
this._onUnsubscriptionFn,
|
|
17601
18114
|
),
|
|
17602
18115
|
]);
|
|
18116
|
+
emitAdvisorySyncProfileDuration(openProfile, communicationStartedAt, {
|
|
18117
|
+
name: "sharedLog.open.rpcSubscriptions",
|
|
18118
|
+
component: "shared-log",
|
|
18119
|
+
});
|
|
17603
18120
|
|
|
18121
|
+
const providerChannelStartedAt = syncProfileStart(openProfile);
|
|
17604
18122
|
const fanoutOpenPromise = this._openFanoutChannel(options?.fanout);
|
|
17605
18123
|
// Mark previously-owned replication ranges as "new" only when they already exist.
|
|
17606
18124
|
// Fresh opens have nothing to touch here, so skip the extra scan/write entirely.
|
|
@@ -17608,6 +18126,11 @@ export class SharedLog<
|
|
|
17608
18126
|
? this.updateTimestampOfOwnedReplicationRanges()
|
|
17609
18127
|
: Promise.resolve();
|
|
17610
18128
|
await Promise.all([fanoutOpenPromise, updateOwnedReplicationPromise]);
|
|
18129
|
+
emitAdvisorySyncProfileDuration(openProfile, providerChannelStartedAt, {
|
|
18130
|
+
name: "sharedLog.open.providerAndOwnership",
|
|
18131
|
+
component: "shared-log",
|
|
18132
|
+
details: { indexedReplicationInfo: hasIndexedReplicationInfo },
|
|
18133
|
+
});
|
|
17611
18134
|
|
|
17612
18135
|
// if we had a previous session with replication info, and new replication info dictates that we unreplicate
|
|
17613
18136
|
// we should do that. Otherwise if options is a unreplication we dont need to do anything because
|
|
@@ -17625,17 +18148,35 @@ export class SharedLog<
|
|
|
17625
18148
|
this.node.identity.publicKey,
|
|
17626
18149
|
));
|
|
17627
18150
|
|
|
18151
|
+
const replicationStartedAt = syncProfileStart(openProfile);
|
|
18152
|
+
let replicationAction: "replace" | "resume" | "reset";
|
|
17628
18153
|
if (hasIndexedReplicationInfo && isUnreplicationOptionsDefined) {
|
|
18154
|
+
replicationAction = "replace";
|
|
17629
18155
|
await this.replicate(options?.replicate, { checkDuplicates: true });
|
|
17630
18156
|
} else if (canResumeReplication) {
|
|
18157
|
+
replicationAction = "resume";
|
|
17631
18158
|
// dont do anthing since we are alread replicating stuff
|
|
17632
18159
|
} else {
|
|
18160
|
+
replicationAction = "reset";
|
|
17633
18161
|
await this.replicate(options?.replicate, {
|
|
17634
18162
|
checkDuplicates: true,
|
|
17635
18163
|
reset: true,
|
|
17636
18164
|
});
|
|
17637
18165
|
}
|
|
18166
|
+
emitAdvisorySyncProfileDuration(openProfile, replicationStartedAt, {
|
|
18167
|
+
name: "sharedLog.open.replication",
|
|
18168
|
+
component: "shared-log",
|
|
18169
|
+
details: {
|
|
18170
|
+
hadIndexedState: hasIndexedReplicationInfo,
|
|
18171
|
+
action: replicationAction,
|
|
18172
|
+
},
|
|
18173
|
+
});
|
|
18174
|
+
const synchronizerStartedAt = syncProfileStart(openProfile);
|
|
17638
18175
|
await this.syncronizer.open();
|
|
18176
|
+
emitAdvisorySyncProfileDuration(openProfile, synchronizerStartedAt, {
|
|
18177
|
+
name: "sharedLog.open.synchronizer",
|
|
18178
|
+
component: "shared-log",
|
|
18179
|
+
});
|
|
17639
18180
|
|
|
17640
18181
|
this.interval = setInterval(() => {
|
|
17641
18182
|
void this.rebalanceParticipationDebounced?.call();
|
|
@@ -17643,6 +18184,10 @@ export class SharedLog<
|
|
|
17643
18184
|
|
|
17644
18185
|
this._instanceLifecycle!.markOpenComplete();
|
|
17645
18186
|
this.scheduleReplicationStatusRefresh();
|
|
18187
|
+
emitAdvisorySyncProfileDuration(openProfile, openStartedAt, {
|
|
18188
|
+
name: "sharedLog.open.total",
|
|
18189
|
+
component: "shared-log",
|
|
18190
|
+
});
|
|
17646
18191
|
}
|
|
17647
18192
|
|
|
17648
18193
|
private toNativeReplicationRange(
|
|
@@ -18580,6 +19125,7 @@ export class SharedLog<
|
|
|
18580
19125
|
ownershipLifecycleController,
|
|
18581
19126
|
this._checkedPrune,
|
|
18582
19127
|
);
|
|
19128
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
18583
19129
|
}
|
|
18584
19130
|
|
|
18585
19131
|
private cleanupPendingIHavePeer(peerHash: string) {
|
|
@@ -18604,6 +19150,7 @@ export class SharedLog<
|
|
|
18604
19150
|
receiveEpoch,
|
|
18605
19151
|
});
|
|
18606
19152
|
}
|
|
19153
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
18607
19154
|
}
|
|
18608
19155
|
|
|
18609
19156
|
private async resolveCandidatePeersForHash(
|
|
@@ -20045,6 +20592,7 @@ export class SharedLog<
|
|
|
20045
20592
|
this._peerSyncCapabilities?.clear();
|
|
20046
20593
|
this._peerSyncCapabilitySessions?.clear();
|
|
20047
20594
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
20595
|
+
this._persistedReceiptReadinessGenerations = new WeakMap();
|
|
20048
20596
|
this._persistedReceiptStorage = undefined;
|
|
20049
20597
|
this._persistedReceiptRequestsInFlight?.clear();
|
|
20050
20598
|
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
@@ -22601,10 +23149,14 @@ export class SharedLog<
|
|
|
22601
23149
|
}
|
|
22602
23150
|
return;
|
|
22603
23151
|
} else if (msg instanceof ReplicationInfoV2AppliedMessage) {
|
|
22604
|
-
|
|
22605
|
-
|
|
22606
|
-
|
|
22607
|
-
|
|
23152
|
+
if (
|
|
23153
|
+
this._v2Send.acceptApplied(msg, {
|
|
23154
|
+
from: context.from,
|
|
23155
|
+
receiverTransportSession: context.message.header.session,
|
|
23156
|
+
})
|
|
23157
|
+
) {
|
|
23158
|
+
this.dispatchPersistedReceiptReadinessChange(receiveFromHash);
|
|
23159
|
+
}
|
|
22608
23160
|
return;
|
|
22609
23161
|
} else if (isReplicationInfoV2Message(msg)) {
|
|
22610
23162
|
await this.handleReplicationInfoV2Announcement(
|
|
@@ -23420,6 +23972,7 @@ export class SharedLog<
|
|
|
23420
23972
|
// A committed V2 announcement is applied progress: the peer answers,
|
|
23421
23973
|
// so recovery re-solicitation may restart from the base interval.
|
|
23422
23974
|
this.resetReplicationInfoV2RecoveryEscalation(fromHash);
|
|
23975
|
+
this.dispatchPersistedReceiptReadinessChange(fromHash);
|
|
23423
23976
|
});
|
|
23424
23977
|
} finally {
|
|
23425
23978
|
this._v2Receive.release(admission);
|
|
@@ -23774,6 +24327,465 @@ export class SharedLog<
|
|
|
23774
24327
|
throwIfInactive();
|
|
23775
24328
|
}
|
|
23776
24329
|
|
|
24330
|
+
private nudgePersistedReceiptPeerReadiness(publicKey: PublicSignKey): void {
|
|
24331
|
+
if (this.closed) return;
|
|
24332
|
+
const peerHash = publicKey.hashcode();
|
|
24333
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
24334
|
+
if (
|
|
24335
|
+
!peerSession ||
|
|
24336
|
+
peerSession.phase === "departing" ||
|
|
24337
|
+
(peerSession.phase === "opening" &&
|
|
24338
|
+
!peerSession.openingBarrierActive)
|
|
24339
|
+
) {
|
|
24340
|
+
// A barrier rejection deliberately leaves the current session in its
|
|
24341
|
+
// fail-closed opening phase after the barrier window has settled. Ask the
|
|
24342
|
+
// authenticated peer for a fresh subscriber snapshot so the replacement
|
|
24343
|
+
// session can recover; never rotate a barrier that is still in flight.
|
|
24344
|
+
this.requestSubscriberSnapshotForCapability(publicKey);
|
|
24345
|
+
return;
|
|
24346
|
+
}
|
|
24347
|
+
if (peerSession.phase !== "open" || !peerSession.isActive()) {
|
|
24348
|
+
return;
|
|
24349
|
+
}
|
|
24350
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
24351
|
+
this.promoteReplicationInfoV2ReceiveCapability(publicKey, peerSession);
|
|
24352
|
+
this._v2Receive.reAdvertiseLocalCapabilityForRecovery({
|
|
24353
|
+
peerHash,
|
|
24354
|
+
peerSession,
|
|
24355
|
+
receiveEpoch,
|
|
24356
|
+
});
|
|
24357
|
+
this._v2Receive.ensureRequestProgress({
|
|
24358
|
+
peerHash,
|
|
24359
|
+
peerSession,
|
|
24360
|
+
receiveEpoch,
|
|
24361
|
+
});
|
|
24362
|
+
this.scheduleReplicationInfoV2Recovery(publicKey);
|
|
24363
|
+
}
|
|
24364
|
+
|
|
24365
|
+
/**
|
|
24366
|
+
* Inspect whether one public key's exact current connection generation can
|
|
24367
|
+
* supply persisted-receipt evidence. The returned object is frozen and never
|
|
24368
|
+
* exposes the internal PeerSession token. When `entries` are supplied, the
|
|
24369
|
+
* peer must also be present in a fresh leader plan for every entry.
|
|
24370
|
+
*
|
|
24371
|
+
* This is advisory preflight state. Persisted delivery repeats every
|
|
24372
|
+
* generation, leadership, ownership and storage check at receipt time; a
|
|
24373
|
+
* `ready` snapshot is never itself authority to dispose a source copy.
|
|
24374
|
+
*/
|
|
24375
|
+
async getPersistedReceiptPeerReadiness(
|
|
24376
|
+
key: PublicSignKey,
|
|
24377
|
+
options: PersistedReceiptPeerReadinessOptions<T, R> = {},
|
|
24378
|
+
): Promise<PersistedReceiptPeerReadiness> {
|
|
24379
|
+
return this.inspectPersistedReceiptPeerReadiness(key, options);
|
|
24380
|
+
}
|
|
24381
|
+
|
|
24382
|
+
private async inspectPersistedReceiptPeerReadiness(
|
|
24383
|
+
key: PublicSignKey,
|
|
24384
|
+
options: PersistedReceiptPeerReadinessOptions<T, R>,
|
|
24385
|
+
assertContinue?: () => void,
|
|
24386
|
+
): Promise<PersistedReceiptPeerReadiness> {
|
|
24387
|
+
// Capture and validate caller-owned planning input before consulting live
|
|
24388
|
+
// peer state. Invalid options must not appear to work merely because the
|
|
24389
|
+
// peer is currently absent, then fail later when the same session connects.
|
|
24390
|
+
const entries = options.entries ? [...options.entries] : [];
|
|
24391
|
+
const replicas =
|
|
24392
|
+
options.replicas ??
|
|
24393
|
+
(entries.length > 0 ? this.replicas.min.getValue(this) : undefined);
|
|
24394
|
+
if (replicas !== undefined) {
|
|
24395
|
+
if (!Number.isSafeInteger(replicas) || replicas <= 0) {
|
|
24396
|
+
throw new RangeError(
|
|
24397
|
+
"Persisted-receipt readiness replicas must be a positive integer",
|
|
24398
|
+
);
|
|
24399
|
+
}
|
|
24400
|
+
checkMinReplicasLimit(replicas);
|
|
24401
|
+
}
|
|
24402
|
+
|
|
24403
|
+
const peerHash = key.hashcode();
|
|
24404
|
+
const captured = this.persistedReceiptReadinessCandidate(peerHash);
|
|
24405
|
+
if ("status" in captured) {
|
|
24406
|
+
return captured;
|
|
24407
|
+
}
|
|
24408
|
+
assertContinue?.();
|
|
24409
|
+
|
|
24410
|
+
if (entries.length > 0) {
|
|
24411
|
+
const ownershipLifecycleController =
|
|
24412
|
+
this.captureReplicationOwnershipLifecycle();
|
|
24413
|
+
const ownershipRevision =
|
|
24414
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
24415
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
24416
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24417
|
+
"ownership-changing",
|
|
24418
|
+
captured.generation,
|
|
24419
|
+
);
|
|
24420
|
+
}
|
|
24421
|
+
for (const entry of entries) {
|
|
24422
|
+
assertContinue?.();
|
|
24423
|
+
const leaders = await this.findLeadersFromEntry(
|
|
24424
|
+
entry,
|
|
24425
|
+
replicas!,
|
|
24426
|
+
{ freshLeaderPlan: true },
|
|
24427
|
+
ownershipLifecycleController,
|
|
24428
|
+
);
|
|
24429
|
+
assertContinue?.();
|
|
24430
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
24431
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24432
|
+
"ownership-changing",
|
|
24433
|
+
captured.generation,
|
|
24434
|
+
);
|
|
24435
|
+
}
|
|
24436
|
+
const current = this.persistedReceiptReadinessCandidate(peerHash);
|
|
24437
|
+
if ("status" in current) {
|
|
24438
|
+
return current;
|
|
24439
|
+
}
|
|
24440
|
+
if (
|
|
24441
|
+
current.peerSession !== captured.peerSession ||
|
|
24442
|
+
current.receiveEpoch !== captured.receiveEpoch ||
|
|
24443
|
+
current.capabilitySession !== captured.capabilitySession
|
|
24444
|
+
) {
|
|
24445
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24446
|
+
"replication-state-pending",
|
|
24447
|
+
current.generation,
|
|
24448
|
+
);
|
|
24449
|
+
}
|
|
24450
|
+
if (!leaders.has(peerHash)) {
|
|
24451
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24452
|
+
"not-entry-leader",
|
|
24453
|
+
captured.generation,
|
|
24454
|
+
);
|
|
24455
|
+
}
|
|
24456
|
+
}
|
|
24457
|
+
}
|
|
24458
|
+
|
|
24459
|
+
assertContinue?.();
|
|
24460
|
+
const current = this.persistedReceiptReadinessCandidate(peerHash);
|
|
24461
|
+
if ("status" in current) {
|
|
24462
|
+
return current;
|
|
24463
|
+
}
|
|
24464
|
+
if (
|
|
24465
|
+
current.peerSession !== captured.peerSession ||
|
|
24466
|
+
current.receiveEpoch !== captured.receiveEpoch ||
|
|
24467
|
+
current.capabilitySession !== captured.capabilitySession
|
|
24468
|
+
) {
|
|
24469
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24470
|
+
"replication-state-pending",
|
|
24471
|
+
current.generation,
|
|
24472
|
+
);
|
|
24473
|
+
}
|
|
24474
|
+
if (
|
|
24475
|
+
!this._v2Send.isLatestConfirmedForPeer({
|
|
24476
|
+
peerHash,
|
|
24477
|
+
peerSession: captured.peerSession,
|
|
24478
|
+
receiverTransportSession: captured.capabilitySession,
|
|
24479
|
+
})
|
|
24480
|
+
) {
|
|
24481
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24482
|
+
"replication-confirmation-pending",
|
|
24483
|
+
captured.generation,
|
|
24484
|
+
);
|
|
24485
|
+
}
|
|
24486
|
+
return Object.freeze({
|
|
24487
|
+
status: "ready" as const,
|
|
24488
|
+
generation: captured.generation,
|
|
24489
|
+
});
|
|
24490
|
+
}
|
|
24491
|
+
|
|
24492
|
+
/**
|
|
24493
|
+
* Wait for a public key's current (or replacement) connection generation to
|
|
24494
|
+
* become persisted-receipt ready. Transition listeners are installed before
|
|
24495
|
+
* the first asynchronous inspection, and a bounded recovery tick repairs
|
|
24496
|
+
* missed subscriber/capability wakes without retaining stale PeerSessions.
|
|
24497
|
+
* This waiter is advisory only; the following persisted delivery remains the
|
|
24498
|
+
* operation that proves the requested remote durability quorum.
|
|
24499
|
+
*/
|
|
24500
|
+
async waitForPersistedReceiptPeerReadiness(
|
|
24501
|
+
key: PublicSignKey,
|
|
24502
|
+
options: WaitForPersistedReceiptPeerReadinessOptions<T, R> = {},
|
|
24503
|
+
): Promise<PersistedReceiptPeerReady> {
|
|
24504
|
+
if (this.closed) {
|
|
24505
|
+
throw new ClosedError();
|
|
24506
|
+
}
|
|
24507
|
+
const timeoutMs = options.timeout ?? this.waitForReplicatorTimeout;
|
|
24508
|
+
if (
|
|
24509
|
+
!Number.isSafeInteger(timeoutMs) ||
|
|
24510
|
+
timeoutMs <= 0 ||
|
|
24511
|
+
timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS
|
|
24512
|
+
) {
|
|
24513
|
+
throw new RangeError(
|
|
24514
|
+
`Persisted-receipt readiness timeout must be an integer from 1 to ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS} milliseconds`,
|
|
24515
|
+
);
|
|
24516
|
+
}
|
|
24517
|
+
if (options.signal?.aborted) {
|
|
24518
|
+
throw options.signal.reason instanceof Error
|
|
24519
|
+
? options.signal.reason
|
|
24520
|
+
: new AbortError("Persisted-receipt readiness wait aborted");
|
|
24521
|
+
}
|
|
24522
|
+
|
|
24523
|
+
// Capture caller-owned inputs before reserving a waiter slot. A throwing
|
|
24524
|
+
// iterator/key implementation must not strand capacity permanently.
|
|
24525
|
+
const entries = options.entries ? [...options.entries] : undefined;
|
|
24526
|
+
const inspectOptions: PersistedReceiptPeerReadinessOptions<T, R> = {
|
|
24527
|
+
...(entries ? { entries } : {}),
|
|
24528
|
+
...(options.replicas === undefined ? {} : { replicas: options.replicas }),
|
|
24529
|
+
};
|
|
24530
|
+
const peerHash = key.hashcode();
|
|
24531
|
+
const waiterSet = this._persistedReceiptReadinessWaiters;
|
|
24532
|
+
if (waiterSet.size >= MAX_PERSISTED_RECEIPT_READINESS_WAITERS) {
|
|
24533
|
+
throw new RangeError(
|
|
24534
|
+
`Too many pending persisted-receipt readiness waits (maximum ${MAX_PERSISTED_RECEIPT_READINESS_WAITERS})`,
|
|
24535
|
+
);
|
|
24536
|
+
}
|
|
24537
|
+
const waiterToken = {};
|
|
24538
|
+
waiterSet.add(waiterToken);
|
|
24539
|
+
const deadline = Date.now() + timeoutMs;
|
|
24540
|
+
const closeSignal = this._closeController.signal;
|
|
24541
|
+
const operationController = new AbortController();
|
|
24542
|
+
const operationSignal = AbortSignal.any(
|
|
24543
|
+
[options.signal, closeSignal, operationController.signal].filter(
|
|
24544
|
+
(value): value is AbortSignal => value !== undefined,
|
|
24545
|
+
),
|
|
24546
|
+
);
|
|
24547
|
+
const deferred = pDefer<PersistedReceiptPeerReady>();
|
|
24548
|
+
let settled = false;
|
|
24549
|
+
let checkScheduled = false;
|
|
24550
|
+
let checkInFlight = false;
|
|
24551
|
+
let rerun = false;
|
|
24552
|
+
let recoveryTimer: ReturnType<typeof setTimeout> | undefined;
|
|
24553
|
+
let confirmationController: AbortController | undefined;
|
|
24554
|
+
let lastSnapshot: PersistedReceiptPeerReadiness | undefined;
|
|
24555
|
+
const createTimeoutError = () => {
|
|
24556
|
+
const suffix = lastSnapshot
|
|
24557
|
+
? ` (last status: ${lastSnapshot.status}${
|
|
24558
|
+
"reason" in lastSnapshot ? `/${lastSnapshot.reason}` : ""
|
|
24559
|
+
})`
|
|
24560
|
+
: "";
|
|
24561
|
+
return new TimeoutError(
|
|
24562
|
+
`Timeout waiting for persisted-receipt readiness from ${peerHash}${suffix}`,
|
|
24563
|
+
);
|
|
24564
|
+
};
|
|
24565
|
+
|
|
24566
|
+
const cleanup = () => {
|
|
24567
|
+
waiterSet.delete(waiterToken);
|
|
24568
|
+
this.events.removeEventListener(
|
|
24569
|
+
"persisted-receipt:readiness",
|
|
24570
|
+
onReadinessChange,
|
|
24571
|
+
);
|
|
24572
|
+
this.events.removeEventListener("replication:change", onRoleChange);
|
|
24573
|
+
this.events.removeEventListener("replicator:mature", onRoleChange);
|
|
24574
|
+
options.signal?.removeEventListener("abort", onCallerAbort);
|
|
24575
|
+
closeSignal.removeEventListener("abort", onClose);
|
|
24576
|
+
if (recoveryTimer) {
|
|
24577
|
+
clearTimeout(recoveryTimer);
|
|
24578
|
+
recoveryTimer = undefined;
|
|
24579
|
+
}
|
|
24580
|
+
confirmationController?.abort(
|
|
24581
|
+
new AbortError("Persisted-receipt readiness generation changed"),
|
|
24582
|
+
);
|
|
24583
|
+
confirmationController = undefined;
|
|
24584
|
+
operationController.abort(
|
|
24585
|
+
new AbortError("Persisted-receipt readiness wait settled"),
|
|
24586
|
+
);
|
|
24587
|
+
};
|
|
24588
|
+
const resolve = (snapshot: PersistedReceiptPeerReady) => {
|
|
24589
|
+
if (settled) return;
|
|
24590
|
+
settled = true;
|
|
24591
|
+
cleanup();
|
|
24592
|
+
deferred.resolve(snapshot);
|
|
24593
|
+
};
|
|
24594
|
+
const reject = (error: unknown) => {
|
|
24595
|
+
if (settled) return;
|
|
24596
|
+
settled = true;
|
|
24597
|
+
cleanup();
|
|
24598
|
+
deferred.reject(
|
|
24599
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
24600
|
+
);
|
|
24601
|
+
};
|
|
24602
|
+
const onCallerAbort = () =>
|
|
24603
|
+
reject(
|
|
24604
|
+
options.signal?.reason instanceof Error
|
|
24605
|
+
? options.signal.reason
|
|
24606
|
+
: new AbortError("Persisted-receipt readiness wait aborted"),
|
|
24607
|
+
);
|
|
24608
|
+
const onClose = () => reject(new ClosedError());
|
|
24609
|
+
const continueWait = () => {
|
|
24610
|
+
if (settled) return false;
|
|
24611
|
+
if (closeSignal.aborted) {
|
|
24612
|
+
onClose();
|
|
24613
|
+
return false;
|
|
24614
|
+
}
|
|
24615
|
+
if (options.signal?.aborted) {
|
|
24616
|
+
onCallerAbort();
|
|
24617
|
+
return false;
|
|
24618
|
+
}
|
|
24619
|
+
if (Date.now() >= deadline) {
|
|
24620
|
+
reject(createTimeoutError());
|
|
24621
|
+
return false;
|
|
24622
|
+
}
|
|
24623
|
+
return true;
|
|
24624
|
+
};
|
|
24625
|
+
const assertInspectionCurrent = () => {
|
|
24626
|
+
if (!continueWait()) {
|
|
24627
|
+
throw new AbortError("Persisted-receipt readiness wait settled");
|
|
24628
|
+
}
|
|
24629
|
+
};
|
|
24630
|
+
const armRecoveryTick = () => {
|
|
24631
|
+
if (settled || recoveryTimer) return;
|
|
24632
|
+
const delayMs = Math.max(
|
|
24633
|
+
50,
|
|
24634
|
+
Math.min(1_000, this.waitForReplicatorRequestIntervalMs),
|
|
24635
|
+
);
|
|
24636
|
+
recoveryTimer = setTimeout(() => {
|
|
24637
|
+
recoveryTimer = undefined;
|
|
24638
|
+
if (!continueWait()) return;
|
|
24639
|
+
this.nudgePersistedReceiptPeerReadiness(key);
|
|
24640
|
+
scheduleCheck();
|
|
24641
|
+
}, delayMs);
|
|
24642
|
+
recoveryTimer.unref?.();
|
|
24643
|
+
};
|
|
24644
|
+
const runCheck = async () => {
|
|
24645
|
+
checkScheduled = false;
|
|
24646
|
+
if (!continueWait()) return;
|
|
24647
|
+
if (checkInFlight) {
|
|
24648
|
+
rerun = true;
|
|
24649
|
+
return;
|
|
24650
|
+
}
|
|
24651
|
+
checkInFlight = true;
|
|
24652
|
+
try {
|
|
24653
|
+
let snapshot = await this.inspectPersistedReceiptPeerReadiness(
|
|
24654
|
+
key,
|
|
24655
|
+
inspectOptions,
|
|
24656
|
+
assertInspectionCurrent,
|
|
24657
|
+
);
|
|
24658
|
+
lastSnapshot = snapshot;
|
|
24659
|
+
if (!continueWait()) return;
|
|
24660
|
+
if (rerun) return;
|
|
24661
|
+
if (snapshot.status === "ready") {
|
|
24662
|
+
// A wake observed while the asynchronous inspection was running may
|
|
24663
|
+
// already have invalidated this snapshot. Drain that coalesced wake
|
|
24664
|
+
// before publishing readiness.
|
|
24665
|
+
resolve(snapshot);
|
|
24666
|
+
return;
|
|
24667
|
+
}
|
|
24668
|
+
if (
|
|
24669
|
+
snapshot.status === "pending" &&
|
|
24670
|
+
snapshot.reason === "replication-confirmation-pending"
|
|
24671
|
+
) {
|
|
24672
|
+
const target = this.persistedReceiptPeerSession(peerHash);
|
|
24673
|
+
if (target) {
|
|
24674
|
+
const currentConfirmationController = new AbortController();
|
|
24675
|
+
confirmationController = currentConfirmationController;
|
|
24676
|
+
try {
|
|
24677
|
+
await this._v2Send.confirmLatestForPeer(
|
|
24678
|
+
{
|
|
24679
|
+
peerHash,
|
|
24680
|
+
peerSession: target.peerSession,
|
|
24681
|
+
receiverTransportSession: target.capabilitySession,
|
|
24682
|
+
},
|
|
24683
|
+
{
|
|
24684
|
+
timeout: Math.max(1, deadline - Date.now()),
|
|
24685
|
+
signal: AbortSignal.any([
|
|
24686
|
+
operationSignal,
|
|
24687
|
+
currentConfirmationController.signal,
|
|
24688
|
+
]),
|
|
24689
|
+
},
|
|
24690
|
+
);
|
|
24691
|
+
} catch (error) {
|
|
24692
|
+
if (!continueWait()) return;
|
|
24693
|
+
if (!(error instanceof AbortError)) {
|
|
24694
|
+
throw error;
|
|
24695
|
+
}
|
|
24696
|
+
rerun = true;
|
|
24697
|
+
} finally {
|
|
24698
|
+
if (confirmationController === currentConfirmationController) {
|
|
24699
|
+
confirmationController = undefined;
|
|
24700
|
+
}
|
|
24701
|
+
}
|
|
24702
|
+
if (!continueWait()) return;
|
|
24703
|
+
snapshot = await this.inspectPersistedReceiptPeerReadiness(
|
|
24704
|
+
key,
|
|
24705
|
+
inspectOptions,
|
|
24706
|
+
assertInspectionCurrent,
|
|
24707
|
+
);
|
|
24708
|
+
lastSnapshot = snapshot;
|
|
24709
|
+
if (!continueWait()) return;
|
|
24710
|
+
if (rerun) return;
|
|
24711
|
+
if (snapshot.status === "ready") {
|
|
24712
|
+
resolve(snapshot);
|
|
24713
|
+
return;
|
|
24714
|
+
}
|
|
24715
|
+
}
|
|
24716
|
+
}
|
|
24717
|
+
if (!continueWait()) return;
|
|
24718
|
+
this.nudgePersistedReceiptPeerReadiness(key);
|
|
24719
|
+
} catch (error) {
|
|
24720
|
+
if (!settled) reject(error);
|
|
24721
|
+
} finally {
|
|
24722
|
+
checkInFlight = false;
|
|
24723
|
+
if (!settled && rerun) {
|
|
24724
|
+
rerun = false;
|
|
24725
|
+
scheduleCheck();
|
|
24726
|
+
} else {
|
|
24727
|
+
armRecoveryTick();
|
|
24728
|
+
}
|
|
24729
|
+
}
|
|
24730
|
+
};
|
|
24731
|
+
const scheduleCheck = (interruptConfirmation = false) => {
|
|
24732
|
+
if (settled) return;
|
|
24733
|
+
if (recoveryTimer) {
|
|
24734
|
+
clearTimeout(recoveryTimer);
|
|
24735
|
+
recoveryTimer = undefined;
|
|
24736
|
+
}
|
|
24737
|
+
if (checkInFlight) {
|
|
24738
|
+
rerun = true;
|
|
24739
|
+
if (interruptConfirmation) {
|
|
24740
|
+
confirmationController?.abort(
|
|
24741
|
+
new AbortError(
|
|
24742
|
+
"Persisted-receipt readiness changed during confirmation",
|
|
24743
|
+
),
|
|
24744
|
+
);
|
|
24745
|
+
}
|
|
24746
|
+
return;
|
|
24747
|
+
}
|
|
24748
|
+
if (checkScheduled) return;
|
|
24749
|
+
checkScheduled = true;
|
|
24750
|
+
void Promise.resolve().then(runCheck);
|
|
24751
|
+
};
|
|
24752
|
+
const onReadinessChange = (
|
|
24753
|
+
event: CustomEvent<PersistedReceiptPeerReadinessEvent>,
|
|
24754
|
+
) => {
|
|
24755
|
+
if (event.detail.peerHash === peerHash) scheduleCheck(true);
|
|
24756
|
+
};
|
|
24757
|
+
const onRoleChange = (event: CustomEvent<ReplicationChangeEvent>) => {
|
|
24758
|
+
if (
|
|
24759
|
+
(entries?.length ?? 0) > 0 ||
|
|
24760
|
+
event.detail.publicKey.hashcode() === peerHash
|
|
24761
|
+
) {
|
|
24762
|
+
scheduleCheck(true);
|
|
24763
|
+
}
|
|
24764
|
+
};
|
|
24765
|
+
|
|
24766
|
+
// Register wake sources before the first state inspection. EventTarget does
|
|
24767
|
+
// not replay a transition that fired between an async check and registration.
|
|
24768
|
+
this.events.addEventListener(
|
|
24769
|
+
"persisted-receipt:readiness",
|
|
24770
|
+
onReadinessChange,
|
|
24771
|
+
);
|
|
24772
|
+
this.events.addEventListener("replication:change", onRoleChange);
|
|
24773
|
+
this.events.addEventListener("replicator:mature", onRoleChange);
|
|
24774
|
+
options.signal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
24775
|
+
closeSignal.addEventListener("abort", onClose, { once: true });
|
|
24776
|
+
if (options.signal?.aborted) {
|
|
24777
|
+
onCallerAbort();
|
|
24778
|
+
} else if (closeSignal.aborted) {
|
|
24779
|
+
onClose();
|
|
24780
|
+
} else {
|
|
24781
|
+
scheduleCheck();
|
|
24782
|
+
}
|
|
24783
|
+
|
|
24784
|
+
const timeout = setTimeout(() => reject(createTimeoutError()), timeoutMs);
|
|
24785
|
+
timeout.unref?.();
|
|
24786
|
+
return deferred.promise.finally(() => clearTimeout(timeout));
|
|
24787
|
+
}
|
|
24788
|
+
|
|
23777
24789
|
async waitForReplicator(
|
|
23778
24790
|
key: PublicSignKey,
|
|
23779
24791
|
options?: {
|
|
@@ -23783,19 +24795,28 @@ export class SharedLog<
|
|
|
23783
24795
|
timeout?: number;
|
|
23784
24796
|
},
|
|
23785
24797
|
) {
|
|
24798
|
+
if (options?.signal?.aborted) {
|
|
24799
|
+
throw new AbortError();
|
|
24800
|
+
}
|
|
23786
24801
|
const deferred = pDefer<void>();
|
|
23787
24802
|
const timeoutMs = options?.timeout ?? this.waitForReplicatorTimeout;
|
|
23788
24803
|
const resolvedRoleAge = options?.eager
|
|
23789
24804
|
? undefined
|
|
23790
24805
|
: (options?.roleAge ?? (await this.getDefaultMinRoleAge()));
|
|
24806
|
+
if (options?.signal?.aborted) {
|
|
24807
|
+
throw new AbortError();
|
|
24808
|
+
}
|
|
23791
24809
|
|
|
23792
24810
|
let settled = false;
|
|
23793
24811
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
23794
24812
|
let requestTimer: ReturnType<typeof setTimeout> | undefined;
|
|
24813
|
+
let checkInFlight = false;
|
|
24814
|
+
let checkAgain = false;
|
|
23795
24815
|
|
|
23796
24816
|
const clear = () => {
|
|
23797
|
-
|
|
23798
|
-
this.events.removeEventListener("
|
|
24817
|
+
checkAgain = false;
|
|
24818
|
+
this.events.removeEventListener("replicator:mature", runCheck);
|
|
24819
|
+
this.events.removeEventListener("replication:change", runCheck);
|
|
23799
24820
|
options?.signal?.removeEventListener("abort", onAbort);
|
|
23800
24821
|
if (timer != null) {
|
|
23801
24822
|
clearTimeout(timer);
|
|
@@ -23931,11 +24952,34 @@ export class SharedLog<
|
|
|
23931
24952
|
await iterator?.close();
|
|
23932
24953
|
}
|
|
23933
24954
|
};
|
|
24955
|
+
const runCheck = () => {
|
|
24956
|
+
if (settled) return;
|
|
24957
|
+
if (checkInFlight) {
|
|
24958
|
+
checkAgain = true;
|
|
24959
|
+
return;
|
|
24960
|
+
}
|
|
24961
|
+
// Reserve synchronously before `check()` can dispatch/re-enter from an
|
|
24962
|
+
// index implementation's first `next()` call.
|
|
24963
|
+
checkInFlight = true;
|
|
24964
|
+
void check()
|
|
24965
|
+
.catch((error) =>
|
|
24966
|
+
reject(error instanceof Error ? error : new Error(String(error))),
|
|
24967
|
+
)
|
|
24968
|
+
.finally(() => {
|
|
24969
|
+
checkInFlight = false;
|
|
24970
|
+
if (!settled && checkAgain) {
|
|
24971
|
+
checkAgain = false;
|
|
24972
|
+
runCheck();
|
|
24973
|
+
}
|
|
24974
|
+
});
|
|
24975
|
+
};
|
|
23934
24976
|
|
|
24977
|
+
// Register before the first asynchronous index read. EventTarget does not
|
|
24978
|
+
// replay a maturity/change event that fires while that read is in flight.
|
|
24979
|
+
this.events.addEventListener("replicator:mature", runCheck);
|
|
24980
|
+
this.events.addEventListener("replication:change", runCheck);
|
|
23935
24981
|
requestReplicationInfo();
|
|
23936
|
-
|
|
23937
|
-
this.events.addEventListener("replicator:mature", check);
|
|
23938
|
-
this.events.addEventListener("replication:change", check);
|
|
24982
|
+
runCheck();
|
|
23939
24983
|
|
|
23940
24984
|
return deferred.promise.finally(clear);
|
|
23941
24985
|
}
|
|
@@ -26825,6 +27869,7 @@ export class SharedLog<
|
|
|
26825
27869
|
if (!ownsSubscriptionEpoch()) {
|
|
26826
27870
|
return;
|
|
26827
27871
|
}
|
|
27872
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
26828
27873
|
// A reconnect can arrive before the previous exact-session recovery tick
|
|
26829
27874
|
// observes its stale session. Retire that job synchronously so it cannot
|
|
26830
27875
|
// suppress the replacement session's scheduler in the shared peer slot.
|
|
@@ -26997,6 +28042,7 @@ export class SharedLog<
|
|
|
26997
28042
|
publicKey,
|
|
26998
28043
|
replicationLifecycleController,
|
|
26999
28044
|
);
|
|
28045
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
27000
28046
|
}
|
|
27001
28047
|
|
|
27002
28048
|
private getClampedReplicas(customValue?: MinReplicas) {
|