p2p-fret 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +23 -0
- package/dist/src/service/fret-service.d.ts +62 -7
- package/dist/src/service/fret-service.js +179 -51
- package/dist/src/service/idle-backoff.d.ts +5 -4
- package/dist/src/service/idle-backoff.js +5 -4
- package/dist/src/service/libp2p-fret-service.d.ts +1 -0
- package/dist/src/service/libp2p-fret-service.js +4 -1
- package/dist/src/store/digitree-store.d.ts +37 -0
- package/dist/src/store/digitree-store.js +18 -1
- package/dist/src/store/relevance.js +15 -9
- package/dist/src/store/tenure.d.ts +72 -0
- package/dist/src/store/tenure.js +91 -0
- package/package.json +1 -1
- package/src/index.ts +23 -0
- package/src/service/fret-service.ts +190 -50
- package/src/service/idle-backoff.ts +5 -4
- package/src/service/libp2p-fret-service.ts +5 -1
- package/src/store/digitree-store.ts +756 -699
- package/src/store/relevance.ts +16 -9
- package/src/store/tenure.ts +100 -0
package/dist/src/index.d.ts
CHANGED
|
@@ -48,6 +48,22 @@ export interface FretConfig {
|
|
|
48
48
|
* Must be a positive finite number; the constructor throws otherwise.
|
|
49
49
|
*/
|
|
50
50
|
maxPassiveIntervalMs?: number;
|
|
51
|
+
/**
|
|
52
|
+
* The longest grace, in milliseconds, a peer this node has watched stay up for a long time is
|
|
53
|
+
* given to come back before it is marked `dead` (default 300 000 ms — 5 minutes).
|
|
54
|
+
*
|
|
55
|
+
* A peer is normally marked dead after `deadAfterFailures` spaced failed contacts — a few
|
|
56
|
+
* seconds. A peer with long *proven tenure* (how long this node itself has watched it stay up)
|
|
57
|
+
* is instead marked dead only once its failure run has also lasted `tenure / 12`, capped here:
|
|
58
|
+
* nothing below 10 minutes of tenure, 50 s at 10 minutes, 2.5 minutes at 30, the full cap from
|
|
59
|
+
* an hour on. So a long-running server down for a few minutes of maintenance keeps its place in
|
|
60
|
+
* the ring rather than having the ring re-form around it twice. While in grace the peer is still
|
|
61
|
+
* a live member — routing steers around it by its probe backoff, and cohorts keep it as a
|
|
62
|
+
* member that is merely unavailable. `0` disables grace for every peer. Must be a finite number
|
|
63
|
+
* between 0 and 15 minutes (the tenure gap tolerance — a longer grace could keep a peer "alive"
|
|
64
|
+
* across a silence long enough to restart its session); the constructor throws otherwise.
|
|
65
|
+
*/
|
|
66
|
+
maxDeadGraceMs?: number;
|
|
51
67
|
/**
|
|
52
68
|
* This node's libp2p private key — the key `node.peerId` was derived from.
|
|
53
69
|
*
|
|
@@ -174,6 +190,13 @@ export interface FretService {
|
|
|
174
190
|
id: string;
|
|
175
191
|
metadata?: Record<string, unknown>;
|
|
176
192
|
}>;
|
|
193
|
+
/**
|
|
194
|
+
* How long, in ms, this node has itself watched `peerId` stay continuously up — measured from
|
|
195
|
+
* proof of life only, never gossiped (see *Tenure* in docs/fret.md). 0 for a peer the table
|
|
196
|
+
* holds but has never proven alive; `undefined` for one it does not hold. Cohort order does not
|
|
197
|
+
* use it — a higher layer may, e.g. to choose which cohort members to ask first.
|
|
198
|
+
*/
|
|
199
|
+
getPeerTenure(peerId: string): number | undefined;
|
|
177
200
|
reportNetworkSize(estimate: number, confidence: number, source?: string): void;
|
|
178
201
|
getNetworkSizeEstimate(): {
|
|
179
202
|
size_estimate: number;
|
|
@@ -389,7 +389,7 @@ export declare class FretService implements IFretService, Startable {
|
|
|
389
389
|
*
|
|
390
390
|
* **Scoring never creates** — a peer we hold no entry for is not scored, and the call returns
|
|
391
391
|
* before the `selfCoord()` await. See {@link applyTouch} for why, and for where creation does
|
|
392
|
-
* belong. The
|
|
392
|
+
* belong. The `applyMembershipSignal` / `noteProofOfLife` land inside that guard and
|
|
393
393
|
* so stop running for a missing peer, which loses nothing: both already no-op on one.
|
|
394
394
|
*
|
|
395
395
|
* `latencyMs` is optional and must be supplied **only** when the caller timed a round trip
|
|
@@ -428,7 +428,16 @@ export declare class FretService implements IFretService, Startable {
|
|
|
428
428
|
*/
|
|
429
429
|
private applyContactFailure;
|
|
430
430
|
/**
|
|
431
|
-
* Count one failed contact against `id`, marking it `dead` once the run completes
|
|
431
|
+
* Count one failed contact against `id`, marking it `dead` once the run completes — and, for a
|
|
432
|
+
* peer with long proven tenure, has also lasted that peer's grace.
|
|
433
|
+
*
|
|
434
|
+
* **The grace** (`deadGraceMs` in `store/tenure.ts`, capped by `maxDeadGraceMs`) is time, not
|
|
435
|
+
* strikes: strikes arrive at the pass rate, so a larger count would stretch seconds, never the
|
|
436
|
+
* minutes a server's maintenance takes. A peer under 10 minutes of tenure has none and is judged
|
|
437
|
+
* exactly as before. Tenure is frozen during the outage (no proof of life arrives), so the grace
|
|
438
|
+
* is stable across the run without being snapshotted at its start. A peer held in grace is still
|
|
439
|
+
* a live member: routing steers around it by its probe backoff, and cohorts keep it as a member
|
|
440
|
+
* that is merely unavailable.
|
|
432
441
|
*
|
|
433
442
|
* **Synchronous by construction.** `applySuccess` / `applyFailure` read an entry, await the
|
|
434
443
|
* self coordinate, then write a value derived before the await — so two concurrent chains
|
|
@@ -446,7 +455,7 @@ export declare class FretService implements IFretService, Startable {
|
|
|
446
455
|
private applyContactStrike;
|
|
447
456
|
/**
|
|
448
457
|
* Record that `id` is demonstrably alive: clear its contact-failure run and, if it was marked
|
|
449
|
-
* `dead`, restore it to a live state.
|
|
458
|
+
* `dead`, restore it to a live state — and advance its session tenure (`store/tenure.ts`).
|
|
450
459
|
*
|
|
451
460
|
* Called from every site that proves liveness: the RPC sites through {@link noteRpcEvidence},
|
|
452
461
|
* and a fresh transport connection (`peer:connect`) directly. The counter reset is the
|
|
@@ -456,8 +465,15 @@ export declare class FretService implements IFretService, Startable {
|
|
|
456
465
|
*
|
|
457
466
|
* Relevance is deliberately *not* reset to a baseline. `scoreSuccess` already up-ranks on
|
|
458
467
|
* success, and wiping the health counters would erase the record of a peer that flaps.
|
|
468
|
+
*
|
|
469
|
+
* This is the **only** place tenure advances, so it counts proof of life and nothing else:
|
|
470
|
+
* hearsay (`noteDiscovered`, address hints, the peerStore listeners) never reaches it. The
|
|
471
|
+
* stamp rides the same write as the liveness patch, and a stamp alone is **not** a change —
|
|
472
|
+
* otherwise every RPC from a settled neighbour would snap the idle backoff back to its base.
|
|
459
473
|
*/
|
|
460
474
|
private noteProofOfLife;
|
|
475
|
+
/** What proof of life changes about `e`'s liveness bookkeeping — empty when already settled. */
|
|
476
|
+
private livenessPatch;
|
|
461
477
|
/**
|
|
462
478
|
* The remote *answered on this network's namespaced protocol* — a framed reply came back,
|
|
463
479
|
* whatever it said: `ok: false`, `busy`, or bytes that would not decode.
|
|
@@ -622,7 +638,9 @@ export declare class FretService implements IFretService, Startable {
|
|
|
622
638
|
* Classify `id` from a libp2p-reported protocol list (available once identify has
|
|
623
639
|
* run). Member if any of our namespaced protocols appears; foreign if the list is
|
|
624
640
|
* non-empty but contains none of them; left unknown if empty (identify pending).
|
|
625
|
-
* The demotion arm is deliberately weak — see `applyMembershipSignal`.
|
|
641
|
+
* The demotion arm is deliberately weak — see `applyMembershipSignal`. A non-empty list also
|
|
642
|
+
* records whether it named ours (`PeerEntry.advertisedForeign`), which the label alone cannot
|
|
643
|
+
* say for a member — see {@link noteAdvertisedProtocols}.
|
|
626
644
|
*
|
|
627
645
|
* NOTE: the positive arm resets the negotiate-failure run (identify listing one of our protocols
|
|
628
646
|
* is strong evidence), so a stale-positive list re-applied often enough would keep a peer that
|
|
@@ -640,6 +658,16 @@ export declare class FretService implements IFretService, Startable {
|
|
|
640
658
|
* service-only-outage block still runs on identify nodes, the realistic rig.
|
|
641
659
|
*/
|
|
642
660
|
private classifyByProtocols;
|
|
661
|
+
/**
|
|
662
|
+
* Record on `id`'s entry whether its latest non-empty advertised protocol list named ours
|
|
663
|
+
* (`PeerEntry.advertisedForeign`) — whatever its label, which the evidence-strength ordering
|
|
664
|
+
* settles separately: a confirmed member stays `member` on a list lacking ours, but the flag
|
|
665
|
+
* still says what it now advertises. Never a change on its own: clearing it coincides with the
|
|
666
|
+
* `identify-member` promotion, which counts for itself, and setting it on a member moves nothing
|
|
667
|
+
* a pass acts on — the flag only helps keep an *off-ring* peer out of the re-probe arms (see
|
|
668
|
+
* {@link isSettledForeign}).
|
|
669
|
+
*/
|
|
670
|
+
private noteAdvertisedProtocols;
|
|
643
671
|
/**
|
|
644
672
|
* Admit a peer libp2p's peerStore told us about, then classify it off `protocols` — the shared
|
|
645
673
|
* step of the `peer:update` / `peer:identify` listeners and the start walk.
|
|
@@ -1384,6 +1412,9 @@ export declare class FretService implements IFretService, Startable {
|
|
|
1384
1412
|
*
|
|
1385
1413
|
* The gate lives here rather than inside the selectors so a bare selector call (from a test)
|
|
1386
1414
|
* still exercises the real selection logic against the live table.
|
|
1415
|
+
*
|
|
1416
|
+
* The two selections come back apart ({@link PhaseTwoTargets}) because only the classify arm
|
|
1417
|
+
* holds the idle backoff at its base; the tick pools them together.
|
|
1387
1418
|
*/
|
|
1388
1419
|
private phaseTwoTargets;
|
|
1389
1420
|
/**
|
|
@@ -1422,9 +1453,12 @@ export declare class FretService implements IFretService, Startable {
|
|
|
1422
1453
|
* - **`foreign`** — a same-network peer *mislabeled* foreign (e.g. identify completed before
|
|
1423
1454
|
* it registered our protocol handlers) is re-admitted by a successful namespaced ping.
|
|
1424
1455
|
* Before ring-membership gating this self-healed for free (`probeNeighborLatency` pinged
|
|
1425
|
-
* near ring peers regardless of label); gating closed that path.
|
|
1426
|
-
*
|
|
1427
|
-
* backstop.
|
|
1456
|
+
* near ring peers regardless of label); gating closed that path. For a peer demoted by
|
|
1457
|
+
* negotiate strikes whose latest advertised list still names ours (a same-network peer
|
|
1458
|
+
* mid-restart on a host without `identifyPush`) this is the dependable backstop. A peer whose
|
|
1459
|
+
* latest list names none of ours is re-probed only until it has refused a full negotiate run
|
|
1460
|
+
* (see {@link isSettledForeign}); after that it comes back through identify delivering a list
|
|
1461
|
+
* that does, or its own inbound RPC.
|
|
1428
1462
|
* - **`dead`** — a peer marked dead by a run of failed contacts is out of every ring view, so
|
|
1429
1463
|
* `stabilizeOnce` (which draws its probe targets from `getNeighbors`) never touches it
|
|
1430
1464
|
* again. Without this arm a peer that recovers but never dials us and never forms a
|
|
@@ -1442,6 +1476,18 @@ export declare class FretService implements IFretService, Startable {
|
|
|
1442
1476
|
* `entries` defaults to a fresh `store.list()`, exactly as {@link classifyTargets}'s does.
|
|
1443
1477
|
*/
|
|
1444
1478
|
private reprobeExcludedTargets;
|
|
1479
|
+
/**
|
|
1480
|
+
* Whether `e` is foreign on both kinds of evidence at once: its latest advertised protocol list
|
|
1481
|
+
* names none of ours (`advertisedForeign`), *and* a full run of spaced negotiate refusals since
|
|
1482
|
+
* the last positive proof says it really answers none of them. Neither alone stops re-probing.
|
|
1483
|
+
* The list can be stale — captured before the remote's FRET registered its handlers, the
|
|
1484
|
+
* ordinary shape for a host that starts the core service by hand after its node — and without
|
|
1485
|
+
* `identifyPush` nothing refreshes it: two such peers trusting the list alone would never contact
|
|
1486
|
+
* each other at all (`test/address-hints.relay.spec.ts` is that rig). A refusal run alone is what
|
|
1487
|
+
* a same-network peer mid-restart produces, which is what the foreign arm exists to recover.
|
|
1488
|
+
* Costs a relay {@link NEGOTIATE_FAILURE_THRESHOLD} failed pings per run, then silence.
|
|
1489
|
+
*/
|
|
1490
|
+
private isSettledForeign;
|
|
1445
1491
|
/**
|
|
1446
1492
|
* Select one bounded re-probe arm's targets — peers matching `isCandidate` — the shared
|
|
1447
1493
|
* mechanics of both arms of {@link reprobeExcludedTargets}: off-backoff and reachable candidates
|
|
@@ -1453,6 +1499,14 @@ export declare class FretService implements IFretService, Startable {
|
|
|
1453
1499
|
* re-probed at most ~once per backoff window (doubling to a 32× cap) rather than every tick.
|
|
1454
1500
|
* That growing factor is also what rotates the list under tick truncation: a probed peer's
|
|
1455
1501
|
* factor grows, so it sorts behind the ones a truncated tick never reached.
|
|
1502
|
+
*
|
|
1503
|
+
* A peer that is foreign on both kinds of evidence ({@link isSettledForeign} — a relay, another
|
|
1504
|
+
* network's node, once it has refused a run of our probes) is skipped by **both** arms: answering
|
|
1505
|
+
* our ping cannot make it a ring member, so probing it — and redialing it when disconnected —
|
|
1506
|
+
* was traffic forever with nothing to recover. If it does start serving this network, identify
|
|
1507
|
+
* delivers the new list (`identifyPush`, or a reconnect re-running identify), or it contacts us
|
|
1508
|
+
* itself; a dead one that comes back is revived by `peer:connect`. The filter only removes
|
|
1509
|
+
* candidates, so the separate per-arm budgets stand.
|
|
1456
1510
|
*/
|
|
1457
1511
|
private reprobeOffRingTargets;
|
|
1458
1512
|
/**
|
|
@@ -1712,6 +1766,7 @@ export declare class FretService implements IFretService, Startable {
|
|
|
1712
1766
|
id: string;
|
|
1713
1767
|
metadata?: Record<string, unknown>;
|
|
1714
1768
|
}>;
|
|
1769
|
+
getPeerTenure(peerId: string): number | undefined;
|
|
1715
1770
|
exportTable(): SerializedTable;
|
|
1716
1771
|
importTable(table: SerializedTable): Promise<number>;
|
|
1717
1772
|
/**
|
|
@@ -28,6 +28,7 @@ import { NearExchange, snapshotDigest } from './near-exchange.js';
|
|
|
28
28
|
import { IdleBackoff } from './idle-backoff.js';
|
|
29
29
|
import { ringNeighborsBothSides } from '../ring/ring-walk.js';
|
|
30
30
|
import { createSparsityModel, normalizedLogDistance, sparsityBonus, touch as scoreTouch, recordSuccess as scoreSuccess, recordFailure as scoreFailure, initialRelevance, } from '../store/relevance.js';
|
|
31
|
+
import { deadGraceMs, noteProven, provenTenureMs, TENURE_GAP_TOLERANCE_MS } from '../store/tenure.js';
|
|
31
32
|
import { createLogger } from '../logger.js';
|
|
32
33
|
const log = createLogger('service:fret');
|
|
33
34
|
function isPlainObject(value) {
|
|
@@ -118,6 +119,20 @@ function requirePositiveMs(value, fallback, name) {
|
|
|
118
119
|
}
|
|
119
120
|
return value;
|
|
120
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* The configured dead-verdict grace ceiling (`FretConfig.maxDeadGraceMs`): a finite number of ms
|
|
124
|
+
* from 0 (the off switch) up to the tenure gap tolerance; absent takes `fallback`. Past the
|
|
125
|
+
* tolerance a grace could hold a peer "not dead" across a silence long enough that its session
|
|
126
|
+
* restarts on return — a contradiction, so it fails here rather than being clamped.
|
|
127
|
+
*/
|
|
128
|
+
function requireDeadGraceMs(value, fallback) {
|
|
129
|
+
if (value === undefined)
|
|
130
|
+
return fallback;
|
|
131
|
+
if (!Number.isFinite(value) || value < 0 || value > TENURE_GAP_TOLERANCE_MS) {
|
|
132
|
+
throw new Error(`FretService: maxDeadGraceMs must be a finite number of milliseconds from 0 to ${TENURE_GAP_TOLERANCE_MS}, got ${String(value)}`);
|
|
133
|
+
}
|
|
134
|
+
return value;
|
|
135
|
+
}
|
|
121
136
|
/**
|
|
122
137
|
* The peer ids named by configured bootstrap entries: a multiaddr's `p2p` component, otherwise
|
|
123
138
|
* the entry itself as a bare peer id. An entry that yields no parseable id is logged and skipped
|
|
@@ -485,6 +500,13 @@ export class FretService {
|
|
|
485
500
|
* sent). A redial cancelled mid-flight by `stop()` counts in neither verdict.
|
|
486
501
|
*/
|
|
487
502
|
redials: { started: 0, connected: 0, failed: 0 },
|
|
503
|
+
/**
|
|
504
|
+
* Contact strikes that completed a failure run against a live peer but did not mark it
|
|
505
|
+
* `dead`, because the run had not yet lasted the peer's tenure-earned grace (see
|
|
506
|
+
* `applyContactStrike`). One per such strike, so a long-tenured peer that stays silent
|
|
507
|
+
* through its grace counts once per pass until the verdict lands.
|
|
508
|
+
*/
|
|
509
|
+
deadGraceHeld: 0,
|
|
488
510
|
/**
|
|
489
511
|
* The stabilization loop's idle backoff, as of the last completed pass (see `IdleBackoff`):
|
|
490
512
|
* `delayMs` is the passive delay the loop armed after it — the base while things change,
|
|
@@ -564,6 +586,7 @@ export class FretService {
|
|
|
564
586
|
// Validated like `dialTimeoutMs`: a host that set it meant a number. A value at or
|
|
565
587
|
// below the base interval is the off switch — `IdleBackoff` clamps the cap to the base.
|
|
566
588
|
maxPassiveIntervalMs: requirePositiveMs(cfg?.maxPassiveIntervalMs, 60_000, 'maxPassiveIntervalMs'),
|
|
589
|
+
maxDeadGraceMs: requireDeadGraceMs(cfg?.maxDeadGraceMs, 300_000),
|
|
567
590
|
};
|
|
568
591
|
this.idle = new IdleBackoff({ baseMs: FretService.PASSIVE_INTERVAL_MS, capMs: this.cfg.maxPassiveIntervalMs });
|
|
569
592
|
// A key that does not derive this node's peer id would seal records no receiver accepts
|
|
@@ -771,7 +794,7 @@ export class FretService {
|
|
|
771
794
|
*
|
|
772
795
|
* **Scoring never creates** — a peer we hold no entry for is not scored, and the call returns
|
|
773
796
|
* before the `selfCoord()` await. See {@link applyTouch} for why, and for where creation does
|
|
774
|
-
* belong. The
|
|
797
|
+
* belong. The `applyMembershipSignal` / `noteProofOfLife` land inside that guard and
|
|
775
798
|
* so stop running for a missing peer, which loses nothing: both already no-op on one.
|
|
776
799
|
*
|
|
777
800
|
* `latencyMs` is optional and must be supplied **only** when the caller timed a round trip
|
|
@@ -789,6 +812,16 @@ export class FretService {
|
|
|
789
812
|
* inside the store instead of patching a value derived outside it.
|
|
790
813
|
*/
|
|
791
814
|
async applySuccess(id, latencyMs) {
|
|
815
|
+
if (!this.store.getById(id))
|
|
816
|
+
return;
|
|
817
|
+
// Every applySuccess call in this service follows a completed RPC over this
|
|
818
|
+
// network's namespaced protocol (ping or maybeAct), which proves the peer
|
|
819
|
+
// serves this network → confirm membership for free off normal traffic…
|
|
820
|
+
this.applyMembershipSignal(id, 'rpc-success');
|
|
821
|
+
// …and proves it is reachable, which clears any contact-failure run and resurrects it if
|
|
822
|
+
// a previous run had marked it dead. Before the score, so the score reads the tenure this
|
|
823
|
+
// very proof extended or restarted — not a session from before a long absence.
|
|
824
|
+
this.noteRpcEvidence(id);
|
|
792
825
|
const entry = this.store.getById(id);
|
|
793
826
|
if (!entry)
|
|
794
827
|
return;
|
|
@@ -806,13 +839,6 @@ export class FretService {
|
|
|
806
839
|
// has nothing to say about.
|
|
807
840
|
...(latencyMs === undefined ? {} : { avgLatencyMs: next.avgLatencyMs }),
|
|
808
841
|
});
|
|
809
|
-
// Every applySuccess call in this service follows a completed RPC over this
|
|
810
|
-
// network's namespaced protocol (ping or maybeAct), which proves the peer
|
|
811
|
-
// serves this network → confirm membership for free off normal traffic.
|
|
812
|
-
this.applyMembershipSignal(id, 'rpc-success');
|
|
813
|
-
// …and proves it is reachable, which clears any contact-failure run and resurrects it if
|
|
814
|
-
// a previous run had marked it dead.
|
|
815
|
-
this.noteRpcEvidence(id);
|
|
816
842
|
}
|
|
817
843
|
/**
|
|
818
844
|
* Record a failed interaction with `id`.
|
|
@@ -851,7 +877,16 @@ export class FretService {
|
|
|
851
877
|
this.applyContactStrike(id);
|
|
852
878
|
}
|
|
853
879
|
/**
|
|
854
|
-
* Count one failed contact against `id`, marking it `dead` once the run completes
|
|
880
|
+
* Count one failed contact against `id`, marking it `dead` once the run completes — and, for a
|
|
881
|
+
* peer with long proven tenure, has also lasted that peer's grace.
|
|
882
|
+
*
|
|
883
|
+
* **The grace** (`deadGraceMs` in `store/tenure.ts`, capped by `maxDeadGraceMs`) is time, not
|
|
884
|
+
* strikes: strikes arrive at the pass rate, so a larger count would stretch seconds, never the
|
|
885
|
+
* minutes a server's maintenance takes. A peer under 10 minutes of tenure has none and is judged
|
|
886
|
+
* exactly as before. Tenure is frozen during the outage (no proof of life arrives), so the grace
|
|
887
|
+
* is stable across the run without being snapshotted at its start. A peer held in grace is still
|
|
888
|
+
* a live member: routing steers around it by its probe backoff, and cohorts keep it as a member
|
|
889
|
+
* that is merely unavailable.
|
|
855
890
|
*
|
|
856
891
|
* **Synchronous by construction.** `applySuccess` / `applyFailure` read an entry, await the
|
|
857
892
|
* self coordinate, then write a value derived before the await — so two concurrent chains
|
|
@@ -884,20 +919,30 @@ export class FretService {
|
|
|
884
919
|
// Clamped at the threshold so the counter stays bounded for a peer we keep re-probing;
|
|
885
920
|
// a peer already dead simply stays dead.
|
|
886
921
|
const failures = Math.min(e.contactFailures + 1, threshold);
|
|
887
|
-
|
|
888
|
-
|
|
922
|
+
// A run carrying no start stamp (counter written outside this method) reads as long over, so
|
|
923
|
+
// its verdict falls back to the strike count alone — today's rule, never a stuck grace.
|
|
924
|
+
const runStart = e.contactFailures === 0 ? now : e.contactRunStartedAt;
|
|
925
|
+
const completed = failures >= threshold && e.state !== 'dead';
|
|
926
|
+
const toDead = completed && now - runStart >= deadGraceMs(provenTenureMs(e), this.cfg.maxDeadGraceMs);
|
|
927
|
+
const patch = { contactFailures: failures, lastContactFailureAt: now, contactRunStartedAt: runStart };
|
|
928
|
+
if (toDead)
|
|
889
929
|
patch.state = 'dead';
|
|
890
930
|
this.store.update(id, patch);
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
//
|
|
894
|
-
//
|
|
895
|
-
|
|
896
|
-
|
|
931
|
+
if (completed && !toDead)
|
|
932
|
+
this.diag.deadGraceHeld++;
|
|
933
|
+
// A change is a strike that progressed the count, or the one that marked the peer dead. A
|
|
934
|
+
// grace-held strike at the clamped threshold moves nothing — it can repeat every pass for
|
|
935
|
+
// minutes, and counting it would pin the idle backoff at its base for the whole grace. So
|
|
936
|
+
// the verdict lands at most one passive interval after the grace expires. A strike on a peer
|
|
937
|
+
// already `dead` (the dead arm re-probing it) is never a change, or every dead-arm probe
|
|
938
|
+
// would reset the delay forever.
|
|
939
|
+
if (e.state !== 'dead' && (failures > e.contactFailures || toDead)) {
|
|
940
|
+
this.noteChange(toDead ? `contact run completed on ${id}` : `contact strike ${failures} on ${id}`);
|
|
941
|
+
}
|
|
897
942
|
}
|
|
898
943
|
/**
|
|
899
944
|
* Record that `id` is demonstrably alive: clear its contact-failure run and, if it was marked
|
|
900
|
-
* `dead`, restore it to a live state.
|
|
945
|
+
* `dead`, restore it to a live state — and advance its session tenure (`store/tenure.ts`).
|
|
901
946
|
*
|
|
902
947
|
* Called from every site that proves liveness: the RPC sites through {@link noteRpcEvidence},
|
|
903
948
|
* and a fresh transport connection (`peer:connect`) directly. The counter reset is the
|
|
@@ -907,28 +952,45 @@ export class FretService {
|
|
|
907
952
|
*
|
|
908
953
|
* Relevance is deliberately *not* reset to a baseline. `scoreSuccess` already up-ranks on
|
|
909
954
|
* success, and wiping the health counters would erase the record of a peer that flaps.
|
|
955
|
+
*
|
|
956
|
+
* This is the **only** place tenure advances, so it counts proof of life and nothing else:
|
|
957
|
+
* hearsay (`noteDiscovered`, address hints, the peerStore listeners) never reaches it. The
|
|
958
|
+
* stamp rides the same write as the liveness patch, and a stamp alone is **not** a change —
|
|
959
|
+
* otherwise every RPC from a settled neighbour would snap the idle backoff back to its base.
|
|
910
960
|
*/
|
|
911
961
|
noteProofOfLife(id) {
|
|
912
962
|
const e = this.store.getById(id);
|
|
913
963
|
if (!e)
|
|
914
964
|
return;
|
|
965
|
+
const patch = this.livenessPatch(e);
|
|
966
|
+
const settled = Object.keys(patch).length === 0;
|
|
967
|
+
// Self never reaches here by RPC; the guard keeps a meaningless tenure off its entry anyway.
|
|
968
|
+
const tenure = id === this.selfIdStr ? undefined : noteProven(e, Date.now());
|
|
969
|
+
if (settled && tenure === undefined)
|
|
970
|
+
return;
|
|
971
|
+
this.store.update(id, { ...patch, ...tenure });
|
|
972
|
+
// Only when the liveness half wrote — a run cleared or a `dead` peer revived. Proof of life
|
|
973
|
+
// from a peer already settled (an idle neighbour's ping) at most stamped tenure, which is
|
|
974
|
+
// not something the next pass could act on.
|
|
975
|
+
if (settled)
|
|
976
|
+
return;
|
|
977
|
+
this.noteChange(patch.state === undefined ? `contact run cleared for ${id}` : `${id} revived`);
|
|
978
|
+
}
|
|
979
|
+
/** What proof of life changes about `e`'s liveness bookkeeping — empty when already settled. */
|
|
980
|
+
livenessPatch(e) {
|
|
915
981
|
const patch = {};
|
|
916
982
|
// The spacing stamp is cleared alongside the count, because it only means anything
|
|
917
983
|
// *within* a run: leaving it behind makes the first failure of the next run land inside
|
|
918
984
|
// the spacing window of a failure from the run just ended, so that strike is discarded
|
|
919
985
|
// and the peer gets a free miss after every recovery.
|
|
920
|
-
if (e.contactFailures !== 0 || e.lastContactFailureAt !== 0) {
|
|
986
|
+
if (e.contactFailures !== 0 || e.lastContactFailureAt !== 0 || e.contactRunStartedAt !== 0) {
|
|
921
987
|
patch.contactFailures = 0;
|
|
922
988
|
patch.lastContactFailureAt = 0;
|
|
989
|
+
patch.contactRunStartedAt = 0;
|
|
923
990
|
}
|
|
924
991
|
if (e.state === 'dead')
|
|
925
|
-
patch.state = this.isConnected(id) ? 'connected' : 'disconnected';
|
|
926
|
-
|
|
927
|
-
return; // already settled
|
|
928
|
-
this.store.update(id, patch);
|
|
929
|
-
// Only when it wrote — a run cleared or a `dead` peer revived. Proof of life from a peer
|
|
930
|
-
// already settled (an idle neighbour's ping) returned above and is not a change.
|
|
931
|
-
this.noteChange(patch.state === undefined ? `contact run cleared for ${id}` : `${id} revived`);
|
|
992
|
+
patch.state = this.isConnected(e.id) ? 'connected' : 'disconnected';
|
|
993
|
+
return patch;
|
|
932
994
|
}
|
|
933
995
|
/**
|
|
934
996
|
* The remote *answered on this network's namespaced protocol* — a framed reply came back,
|
|
@@ -1221,7 +1283,9 @@ export class FretService {
|
|
|
1221
1283
|
* Classify `id` from a libp2p-reported protocol list (available once identify has
|
|
1222
1284
|
* run). Member if any of our namespaced protocols appears; foreign if the list is
|
|
1223
1285
|
* non-empty but contains none of them; left unknown if empty (identify pending).
|
|
1224
|
-
* The demotion arm is deliberately weak — see `applyMembershipSignal`.
|
|
1286
|
+
* The demotion arm is deliberately weak — see `applyMembershipSignal`. A non-empty list also
|
|
1287
|
+
* records whether it named ours (`PeerEntry.advertisedForeign`), which the label alone cannot
|
|
1288
|
+
* say for a member — see {@link noteAdvertisedProtocols}.
|
|
1225
1289
|
*
|
|
1226
1290
|
* NOTE: the positive arm resets the negotiate-failure run (identify listing one of our protocols
|
|
1227
1291
|
* is strong evidence), so a stale-positive list re-applied often enough would keep a peer that
|
|
@@ -1242,8 +1306,24 @@ export class FretService {
|
|
|
1242
1306
|
if (!protocols || protocols.length === 0)
|
|
1243
1307
|
return; // identify not complete → stay unknown
|
|
1244
1308
|
const mine = Object.values(this.protocols);
|
|
1245
|
-
const
|
|
1246
|
-
this.
|
|
1309
|
+
const advertisesOurs = protocols.some((p) => mine.includes(p));
|
|
1310
|
+
this.noteAdvertisedProtocols(id, advertisesOurs);
|
|
1311
|
+
this.applyMembershipSignal(id, advertisesOurs ? 'identify-member' : 'identify-foreign');
|
|
1312
|
+
}
|
|
1313
|
+
/**
|
|
1314
|
+
* Record on `id`'s entry whether its latest non-empty advertised protocol list named ours
|
|
1315
|
+
* (`PeerEntry.advertisedForeign`) — whatever its label, which the evidence-strength ordering
|
|
1316
|
+
* settles separately: a confirmed member stays `member` on a list lacking ours, but the flag
|
|
1317
|
+
* still says what it now advertises. Never a change on its own: clearing it coincides with the
|
|
1318
|
+
* `identify-member` promotion, which counts for itself, and setting it on a member moves nothing
|
|
1319
|
+
* a pass acts on — the flag only helps keep an *off-ring* peer out of the re-probe arms (see
|
|
1320
|
+
* {@link isSettledForeign}).
|
|
1321
|
+
*/
|
|
1322
|
+
noteAdvertisedProtocols(id, advertisesOurs) {
|
|
1323
|
+
const advertisedForeign = !advertisesOurs;
|
|
1324
|
+
const e = this.store.getById(id);
|
|
1325
|
+
if (e && e.advertisedForeign !== advertisedForeign)
|
|
1326
|
+
this.store.update(id, { advertisedForeign });
|
|
1247
1327
|
}
|
|
1248
1328
|
/**
|
|
1249
1329
|
* Admit a peer libp2p's peerStore told us about, then classify it off `protocols` — the shared
|
|
@@ -2305,6 +2385,9 @@ export class FretService {
|
|
|
2305
2385
|
// dead self has no path back short of a restart — the same guard `applyContactStrike` keeps.
|
|
2306
2386
|
if (id === this.selfIdStr)
|
|
2307
2387
|
return;
|
|
2388
|
+
// NOTE: immediate whatever the sender's tenure — no dead-verdict grace — because the notice is
|
|
2389
|
+
// the peer's own statement that it is going. A "back in N minutes" leave flavour that earns
|
|
2390
|
+
// grace needs leave authentication first (tickets/backlog/22-debt-leave-authentication.md).
|
|
2308
2391
|
this.store.update(id, { state: 'dead' });
|
|
2309
2392
|
}
|
|
2310
2393
|
/**
|
|
@@ -2932,26 +3015,36 @@ export class FretService {
|
|
|
2932
3015
|
// buys phase 2 its turn. This return still fires for a `stop()` or a tick that really
|
|
2933
3016
|
// is out of time.
|
|
2934
3017
|
if (budget.signal.aborted) {
|
|
2935
|
-
// Work left undone that no event will announce —
|
|
2936
|
-
//
|
|
3018
|
+
// Work left undone that no event will announce — phase 2 never selected, so nothing
|
|
3019
|
+
// can tell whether unknowns were waiting. Harmless on a `stop()`, which resets the
|
|
3020
|
+
// backoff anyway.
|
|
2937
3021
|
this.idle.notePending();
|
|
2938
3022
|
return;
|
|
2939
3023
|
}
|
|
2940
|
-
const
|
|
2941
|
-
// Pending on the
|
|
2942
|
-
//
|
|
2943
|
-
//
|
|
2944
|
-
//
|
|
2945
|
-
//
|
|
2946
|
-
//
|
|
2947
|
-
//
|
|
2948
|
-
//
|
|
2949
|
-
//
|
|
2950
|
-
//
|
|
2951
|
-
//
|
|
2952
|
-
//
|
|
2953
|
-
|
|
3024
|
+
const { classify, reprobe } = this.phaseTwoTargets();
|
|
3025
|
+
// Pending on the classify arm's selection alone. An `unknown` peer is a transient
|
|
3026
|
+
// population the ring is waiting on, so the next pass should carry on classifying at the
|
|
3027
|
+
// base. A re-probe's outcome lands inside this pass: a success is a membership or revival
|
|
3028
|
+
// change, a failure that progresses a run is a change, and one that progresses nothing
|
|
3029
|
+
// needs no follow-up pass — so holding for it only kept a ring with a permanently
|
|
3030
|
+
// off-ring peer (a dead phone, a relay) restarting the ladder every backoff window. A
|
|
3031
|
+
// redial this pass outlives is held by `runStabilizationPass` on its own.
|
|
3032
|
+
// NOTE: accepted tradeoff — once the loop has backed off, the re-probe arms service their
|
|
3033
|
+
// budget (Core 2 / Edge 1 per arm) per *interval*, not per 1.5 s: a dead or
|
|
3034
|
+
// negotiate-foreign peer whose window expires during a backed-off interval is re-probed up
|
|
3035
|
+
// to one interval late, and with more such peers than the budget each waits about
|
|
3036
|
+
// ceil(count / budget) intervals — at the 60 s cap, a Core ring holding 10 dead entries
|
|
3037
|
+
// re-probes each about every 5 minutes while idle. Accepted because a recovered neighbour
|
|
3038
|
+
// contacts us itself (an inbound RPC revives it), a reconnect revives it through
|
|
3039
|
+
// `peer:connect`, and any change returns the loop to the base. Holding pending whenever an
|
|
3040
|
+
// arm was truncated by its budget was rejected: at Edge's budget of 1, two dead entries
|
|
3041
|
+
// whose windows align truncate every window and restart the ladder about every 32 s.
|
|
3042
|
+
// Revisit if dead-recovery latency on large idle rings shows up as a problem — then hold
|
|
3043
|
+
// pending when an off-ring arm was budget-truncated, accepting that saturated arms keep the
|
|
3044
|
+
// loop awake.
|
|
3045
|
+
if (classify.length > 0)
|
|
2954
3046
|
this.idle.notePending();
|
|
3047
|
+
const targets = [...classify, ...reprobe];
|
|
2955
3048
|
const phaseTwoPool = { concurrency: this.maintenanceConcurrency, signal: budget.signal };
|
|
2956
3049
|
const probed = await runPooled(targets.map((id) => () => this.probeMembership(id, budget.signal)), phaseTwoPool);
|
|
2957
3050
|
logRejected(probed, targets, 'probeMembership');
|
|
@@ -3298,12 +3391,15 @@ export class FretService {
|
|
|
3298
3391
|
*
|
|
3299
3392
|
* The gate lives here rather than inside the selectors so a bare selector call (from a test)
|
|
3300
3393
|
* still exercises the real selection logic against the live table.
|
|
3394
|
+
*
|
|
3395
|
+
* The two selections come back apart ({@link PhaseTwoTargets}) because only the classify arm
|
|
3396
|
+
* holds the idle backoff at its base; the tick pools them together.
|
|
3301
3397
|
*/
|
|
3302
3398
|
phaseTwoTargets() {
|
|
3303
3399
|
if (this.store.countByMembership('unknown') === 0
|
|
3304
3400
|
&& this.store.countByMembership('foreign') === 0
|
|
3305
3401
|
&& this.store.countByState('dead') === 0)
|
|
3306
|
-
return [];
|
|
3402
|
+
return { classify: [], reprobe: [] };
|
|
3307
3403
|
// NOTE: the walk is captured once and `put` writes a *new* entry object rather than editing
|
|
3308
3404
|
// in place, so a store mutation made after this line is invisible to the arms that run
|
|
3309
3405
|
// after it. Sound today: all three arms are pure selection — they read the backoff state,
|
|
@@ -3311,7 +3407,7 @@ export class FretService {
|
|
|
3311
3407
|
// happens later, in the pooled phase. If an arm ever writes to the store while selecting,
|
|
3312
3408
|
// re-walk per arm rather than leaving the later arms on a stale array.
|
|
3313
3409
|
const entries = this.store.list();
|
|
3314
|
-
return
|
|
3410
|
+
return { classify: this.classifyTargets(entries), reprobe: this.reprobeExcludedTargets(entries) };
|
|
3315
3411
|
}
|
|
3316
3412
|
/**
|
|
3317
3413
|
* Bounded classification probe pass over `unknown` peers.
|
|
@@ -3362,9 +3458,12 @@ export class FretService {
|
|
|
3362
3458
|
* - **`foreign`** — a same-network peer *mislabeled* foreign (e.g. identify completed before
|
|
3363
3459
|
* it registered our protocol handlers) is re-admitted by a successful namespaced ping.
|
|
3364
3460
|
* Before ring-membership gating this self-healed for free (`probeNeighborLatency` pinged
|
|
3365
|
-
* near ring peers regardless of label); gating closed that path.
|
|
3366
|
-
*
|
|
3367
|
-
* backstop.
|
|
3461
|
+
* near ring peers regardless of label); gating closed that path. For a peer demoted by
|
|
3462
|
+
* negotiate strikes whose latest advertised list still names ours (a same-network peer
|
|
3463
|
+
* mid-restart on a host without `identifyPush`) this is the dependable backstop. A peer whose
|
|
3464
|
+
* latest list names none of ours is re-probed only until it has refused a full negotiate run
|
|
3465
|
+
* (see {@link isSettledForeign}); after that it comes back through identify delivering a list
|
|
3466
|
+
* that does, or its own inbound RPC.
|
|
3368
3467
|
* - **`dead`** — a peer marked dead by a run of failed contacts is out of every ring view, so
|
|
3369
3468
|
* `stabilizeOnce` (which draws its probe targets from `getNeighbors`) never touches it
|
|
3370
3469
|
* again. Without this arm a peer that recovers but never dials us and never forms a
|
|
@@ -3400,6 +3499,20 @@ export class FretService {
|
|
|
3400
3499
|
...this.reprobeOffRingTargets(entries, (e) => e.state === 'dead', budget),
|
|
3401
3500
|
];
|
|
3402
3501
|
}
|
|
3502
|
+
/**
|
|
3503
|
+
* Whether `e` is foreign on both kinds of evidence at once: its latest advertised protocol list
|
|
3504
|
+
* names none of ours (`advertisedForeign`), *and* a full run of spaced negotiate refusals since
|
|
3505
|
+
* the last positive proof says it really answers none of them. Neither alone stops re-probing.
|
|
3506
|
+
* The list can be stale — captured before the remote's FRET registered its handlers, the
|
|
3507
|
+
* ordinary shape for a host that starts the core service by hand after its node — and without
|
|
3508
|
+
* `identifyPush` nothing refreshes it: two such peers trusting the list alone would never contact
|
|
3509
|
+
* each other at all (`test/address-hints.relay.spec.ts` is that rig). A refusal run alone is what
|
|
3510
|
+
* a same-network peer mid-restart produces, which is what the foreign arm exists to recover.
|
|
3511
|
+
* Costs a relay {@link NEGOTIATE_FAILURE_THRESHOLD} failed pings per run, then silence.
|
|
3512
|
+
*/
|
|
3513
|
+
isSettledForeign(e) {
|
|
3514
|
+
return e.advertisedForeign && e.negotiateFailures >= FretService.NEGOTIATE_FAILURE_THRESHOLD;
|
|
3515
|
+
}
|
|
3403
3516
|
/**
|
|
3404
3517
|
* Select one bounded re-probe arm's targets — peers matching `isCandidate` — the shared
|
|
3405
3518
|
* mechanics of both arms of {@link reprobeExcludedTargets}: off-backoff and reachable candidates
|
|
@@ -3411,10 +3524,18 @@ export class FretService {
|
|
|
3411
3524
|
* re-probed at most ~once per backoff window (doubling to a 32× cap) rather than every tick.
|
|
3412
3525
|
* That growing factor is also what rotates the list under tick truncation: a probed peer's
|
|
3413
3526
|
* factor grows, so it sorts behind the ones a truncated tick never reached.
|
|
3527
|
+
*
|
|
3528
|
+
* A peer that is foreign on both kinds of evidence ({@link isSettledForeign} — a relay, another
|
|
3529
|
+
* network's node, once it has refused a run of our probes) is skipped by **both** arms: answering
|
|
3530
|
+
* our ping cannot make it a ring member, so probing it — and redialing it when disconnected —
|
|
3531
|
+
* was traffic forever with nothing to recover. If it does start serving this network, identify
|
|
3532
|
+
* delivers the new list (`identifyPush`, or a reconnect re-running identify), or it contacts us
|
|
3533
|
+
* itself; a dead one that comes back is revived by `peer:connect`. The filter only removes
|
|
3534
|
+
* candidates, so the separate per-arm budgets stand.
|
|
3414
3535
|
*/
|
|
3415
3536
|
reprobeOffRingTargets(entries, isCandidate, budget) {
|
|
3416
3537
|
const selfStr = this.selfIdStr;
|
|
3417
|
-
const candidates = entries.filter((e) => e.id !== selfStr && !this.backoff.isBackedOff(e.id) && isCandidate(e));
|
|
3538
|
+
const candidates = entries.filter((e) => e.id !== selfStr && !this.isSettledForeign(e) && !this.backoff.isBackedOff(e.id) && isCandidate(e));
|
|
3418
3539
|
if (candidates.length === 0)
|
|
3419
3540
|
return [];
|
|
3420
3541
|
// Only reachable peers are probeable; prefer connected over has-addresses.
|
|
@@ -4101,6 +4222,9 @@ export class FretService {
|
|
|
4101
4222
|
backoffPenalty: (id) => this.backoff.penalty(id),
|
|
4102
4223
|
};
|
|
4103
4224
|
}
|
|
4225
|
+
// NOTE: link quality is the success rate alone; session tenure (`store/tenure.ts`) is available
|
|
4226
|
+
// on the entry but deliberately not folded in. If routes are ever measured failing mid-path on
|
|
4227
|
+
// short-lived hops, weight this by `stabilityFactor(provenTenureMs(entry))`.
|
|
4104
4228
|
linkQuality(id) {
|
|
4105
4229
|
const entry = this.store.getById(id);
|
|
4106
4230
|
if (!entry)
|
|
@@ -4425,6 +4549,10 @@ export class FretService {
|
|
|
4425
4549
|
metadata: entry.metadata
|
|
4426
4550
|
}));
|
|
4427
4551
|
}
|
|
4552
|
+
getPeerTenure(peerId) {
|
|
4553
|
+
const entry = this.store.getById(peerId);
|
|
4554
|
+
return entry ? provenTenureMs(entry) : undefined;
|
|
4555
|
+
}
|
|
4428
4556
|
exportTable() {
|
|
4429
4557
|
const now = Date.now();
|
|
4430
4558
|
return {
|
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
* base the moment a **change** is noted — anything the next pass could act on (a connection
|
|
10
10
|
* forming or dropping, a new entry, a label or liveness transition, an address becoming known) —
|
|
11
11
|
* and holds at the base, without counting as a change, while a pass left work **pending**
|
|
12
|
-
* (
|
|
12
|
+
* (a near probe it skipped on recent evidence, unknown peers it selected for classification, a
|
|
13
|
+
* redial it outlived, a phase its tick budget cut off).
|
|
13
14
|
*
|
|
14
15
|
* A change is recorded as a bump of a monotonic counter, and a pass snapshots that counter when it
|
|
15
16
|
* opens: {@link endPass} compares the two, so a change that lands *during* a pass is seen at that
|
|
@@ -52,10 +53,10 @@ export declare class IdleBackoff {
|
|
|
52
53
|
/** Open a pass: snapshot the change counter and clear the pending flag. */
|
|
53
54
|
beginPass(): void;
|
|
54
55
|
/**
|
|
55
|
-
* The open pass left work no event will announce the outcome of — it selected
|
|
56
|
-
*
|
|
56
|
+
* The open pass left work no event will announce the outcome of — it selected unknown peers to
|
|
57
|
+
* classify, or a redial it started is still running. Holds the next delay at the base. Not a
|
|
57
58
|
* change: nothing happened yet, and counting it would reset the quiet count on every pass that
|
|
58
|
-
* merely *
|
|
59
|
+
* merely *selected* a peer to probe.
|
|
59
60
|
*/
|
|
60
61
|
notePending(): void;
|
|
61
62
|
/**
|