@peerbit/shared-log 13.1.1 → 13.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/exchange-heads.d.ts +1 -0
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +2 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +22 -4
- package/dist/src/index.js.map +1 -1
- package/dist/src/pid.d.ts.map +1 -1
- package/dist/src/pid.js +27 -10
- package/dist/src/pid.js.map +1 -1
- package/package.json +10 -10
- package/src/exchange-heads.ts +3 -0
- package/src/index.ts +29 -3
- package/src/pid.ts +37 -10
package/dist/src/pid.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pid.d.ts","sourceRoot":"","sources":["../../src/pid.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"pid.d.ts","sourceRoot":"","sources":["../../src/pid.ts"],"names":[],"mappings":"AAEA,qBAAa,wBAAwB;IAWnC,QAAQ,CAAC,EAAE,EAAE,MAAM;IAVpB,QAAQ,EAAG,MAAM,CAAC;IAClB,SAAS,EAAG,MAAM,CAAC;IACnB,eAAe,EAAG,MAAM,CAAC;IACzB,eAAe,EAAG,MAAM,CAAC;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;gBAEX,EAAE,EAAE,MAAM,EACnB,OAAO,GAAE;QACR,OAAO,CAAC,EAAE;YAAE,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;QAC1B,GAAG,CAAC,EAAE;YAAE,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;QACtB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,MAAM,CAAC;KACP;IAWP;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;KAC7B;IA0LD,KAAK;CAKL"}
|
package/dist/src/pid.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
const MIN_MEMORY_HEADROOM_BALANCE_SCALER = 0.25;
|
|
1
2
|
export class PIDReplicationController {
|
|
2
3
|
id;
|
|
3
4
|
integral;
|
|
@@ -35,13 +36,12 @@ export class PIDReplicationController {
|
|
|
35
36
|
const effectiveMemoryLimit = this.maxMemoryLimit > 0 ? this.maxMemoryLimit * 0.95 : 0;
|
|
36
37
|
errorMemory =
|
|
37
38
|
currentFactor > 0 && memoryUsage > 0
|
|
38
|
-
? Math.max(Math.min(1, effectiveMemoryLimit / estimatedTotalSize), 0) -
|
|
39
|
-
currentFactor
|
|
39
|
+
? Math.max(Math.min(1, effectiveMemoryLimit / estimatedTotalSize), 0) - currentFactor
|
|
40
40
|
: 0;
|
|
41
41
|
// Math.max(Math.min((this.maxMemoryLimit - memoryUsage) / 100e5, 1), -1)// Math.min(Math.max((this.maxMemoryLimit - memoryUsage, 0) / 10e5, 0), 1);
|
|
42
42
|
}
|
|
43
43
|
const errorCoverageUnmodified = Math.min(1 - totalFactor, 1);
|
|
44
|
-
|
|
44
|
+
let errorCoverage = (this.maxMemoryLimit ? 1 - Math.sqrt(Math.abs(errorMemory)) : 1) *
|
|
45
45
|
errorCoverageUnmodified;
|
|
46
46
|
const errorFromEven = 1 / peerCount - currentFactor;
|
|
47
47
|
// When the network is under-covered (`totalFactor < 1`) balancing "down" (negative
|
|
@@ -51,12 +51,19 @@ export class PIDReplicationController {
|
|
|
51
51
|
// Use a soft clamp: only suppress negative balance strongly when the coverage deficit
|
|
52
52
|
// is material. This avoids oscillations around `totalFactor ~= 1`.
|
|
53
53
|
const coverageDeficit = Math.max(0, errorCoverageUnmodified); // ~= max(0, 1 - totalFactor)
|
|
54
|
-
const negativeBalanceScale = coverageDeficit <= 0
|
|
55
|
-
? 1
|
|
56
|
-
: 1 - Math.min(1, coverageDeficit / 0.1); // full clamp at 10% deficit
|
|
54
|
+
const negativeBalanceScale = coverageDeficit <= 0 ? 1 : 1 - Math.min(1, coverageDeficit / 0.1); // full clamp at 10% deficit
|
|
57
55
|
const errorFromEvenForBalance = errorFromEven >= 0 ? errorFromEven : errorFromEven * negativeBalanceScale;
|
|
56
|
+
const hasMemoryHeadroom = this.maxMemoryLimit != null && this.maxMemoryLimit > 0 && errorMemory > 0;
|
|
57
|
+
if (hasMemoryHeadroom && errorFromEvenForBalance > 0) {
|
|
58
|
+
// Coverage surplus often means another peer has not pruned yet. Do not let
|
|
59
|
+
// that transient surplus cancel a constrained peer that is still below an
|
|
60
|
+
// even share and has storage headroom to take more work.
|
|
61
|
+
errorCoverage = Math.max(errorCoverage, 0);
|
|
62
|
+
}
|
|
58
63
|
const balanceErrorScaler = this.maxMemoryLimit
|
|
59
|
-
?
|
|
64
|
+
? hasMemoryHeadroom
|
|
65
|
+
? Math.max(Math.abs(errorMemory), MIN_MEMORY_HEADROOM_BALANCE_SCALER)
|
|
66
|
+
: Math.abs(errorMemory)
|
|
60
67
|
: 1 - Math.abs(errorCoverage);
|
|
61
68
|
// Balance should be symmetric (allow negative error) so a peer can *reduce*
|
|
62
69
|
// participation when peerCount increases. Otherwise early joiners can get
|
|
@@ -87,8 +94,7 @@ export class PIDReplicationController {
|
|
|
87
94
|
// range can increase gap-boundary assignments and make local memory usage
|
|
88
95
|
// worse, not better. Let the coverage term dominate until the floor is
|
|
89
96
|
// restored, while preserving the hard shrink behavior for zero-capacity peers.
|
|
90
|
-
errorMemoryFactor = Math.max(0.2, errorMemoryFactor -
|
|
91
|
-
0.7 * Math.min(1, coverageDeficit / 0.25));
|
|
97
|
+
errorMemoryFactor = Math.max(0.2, errorMemoryFactor - 0.7 * Math.min(1, coverageDeficit / 0.25));
|
|
92
98
|
}
|
|
93
99
|
totalError =
|
|
94
100
|
errorMemory * errorMemoryFactor + totalError * (1 - errorMemoryFactor);
|
|
@@ -113,7 +119,18 @@ export class PIDReplicationController {
|
|
|
113
119
|
const dTerm = this.kd * derivative;
|
|
114
120
|
// Calculate the new replication factor
|
|
115
121
|
const change = pTerm + iTerm + dTerm;
|
|
116
|
-
|
|
122
|
+
let newFactor = currentFactor + change;
|
|
123
|
+
if (this.maxCPUUsage != null && this.maxMemoryLimit == null) {
|
|
124
|
+
// CPU pressure may shed surplus replicas, but it must not create a
|
|
125
|
+
// coverage gap where the network no longer has one full copy.
|
|
126
|
+
const coverageSurplus = Math.max(0, totalFactor - 1);
|
|
127
|
+
if (newFactor < currentFactor) {
|
|
128
|
+
newFactor =
|
|
129
|
+
coverageSurplus <= 0
|
|
130
|
+
? currentFactor
|
|
131
|
+
: Math.max(newFactor, currentFactor - coverageSurplus);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
117
134
|
// Update state for the next iteration
|
|
118
135
|
this.prevError = totalError;
|
|
119
136
|
// reset integral term if we are "way" out of bounds
|
package/dist/src/pid.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pid.js","sourceRoot":"","sources":["../../src/pid.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,wBAAwB;IAW1B;IAVV,QAAQ,CAAU;IAClB,SAAS,CAAU;IACnB,eAAe,CAAU;IACzB,eAAe,CAAU;IACzB,EAAE,CAAS;IACX,EAAE,CAAS;IACX,EAAE,CAAS;IACX,cAAc,CAAU;IACxB,WAAW,CAAU;IACrB,YACU,EAAU,EACnB,UAMI,EAAE;QAPG,OAAE,GAAF,EAAE,CAAQ;QASnB,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,KAAK,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAC1E,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,GAAG,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,GAAG,EAAE,GAAG,CAAC;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,UAMJ;QACA,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,GACnE,UAAU,CAAC;QAEZ,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC;QAEnC,MAAM,kBAAkB,GACvB,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC;QAEvD,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YACjC,yEAAyE;YACzE,4EAA4E;YAC5E,8DAA8D;YAC9D,MAAM,oBAAoB,GACzB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1D,WAAW;gBACV,aAAa,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC;oBACnC,CAAC,CAAC,IAAI,CAAC,GAAG,
|
|
1
|
+
{"version":3,"file":"pid.js","sourceRoot":"","sources":["../../src/pid.ts"],"names":[],"mappings":"AAAA,MAAM,kCAAkC,GAAG,IAAI,CAAC;AAEhD,MAAM,OAAO,wBAAwB;IAW1B;IAVV,QAAQ,CAAU;IAClB,SAAS,CAAU;IACnB,eAAe,CAAU;IACzB,eAAe,CAAU;IACzB,EAAE,CAAS;IACX,EAAE,CAAS;IACX,EAAE,CAAS;IACX,cAAc,CAAU;IACxB,WAAW,CAAU;IACrB,YACU,EAAU,EACnB,UAMI,EAAE;QAPG,OAAE,GAAF,EAAE,CAAQ;QASnB,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,KAAK,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAC1E,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,GAAG,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,GAAG,EAAE,GAAG,CAAC;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,UAMJ;QACA,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,GACnE,UAAU,CAAC;QAEZ,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC;QAEnC,MAAM,kBAAkB,GACvB,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC;QAEvD,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YACjC,yEAAyE;YACzE,4EAA4E;YAC5E,8DAA8D;YAC9D,MAAM,oBAAoB,GACzB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1D,WAAW;gBACV,aAAa,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC;oBACnC,CAAC,CAAC,IAAI,CAAC,GAAG,CACR,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,GAAG,kBAAkB,CAAC,EACtD,CAAC,CACD,GAAG,aAAa;oBAClB,CAAC,CAAC,CAAC,CAAC;YACN,oJAAoJ;QACrJ,CAAC;QAED,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC;QAC7D,IAAI,aAAa,GAChB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChE,uBAAuB,CAAC;QAEzB,MAAM,aAAa,GAAG,CAAC,GAAG,SAAS,GAAG,aAAa,CAAC;QACpD,mFAAmF;QACnF,sFAAsF;QACtF,0DAA0D;QAC1D,EAAE;QACF,sFAAsF;QACtF,mEAAmE;QACnE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC,CAAC,6BAA6B;QAC3F,MAAM,oBAAoB,GACzB,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,GAAG,GAAG,CAAC,CAAC,CAAC,4BAA4B;QAChG,MAAM,uBAAuB,GAC5B,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,GAAG,oBAAoB,CAAC;QAE3E,MAAM,iBAAiB,GACtB,IAAI,CAAC,cAAc,IAAI,IAAI,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC;QAC3E,IAAI,iBAAiB,IAAI,uBAAuB,GAAG,CAAC,EAAE,CAAC;YACtD,2EAA2E;YAC3E,0EAA0E;YAC1E,yDAAyD;YACzD,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc;YAC7C,CAAC,CAAC,iBAAiB;gBAClB,CAAC,CAAC,IAAI,CAAC,GAAG,CACR,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EACrB,kCAAkC,CAClC;gBACF,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACxB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAE/B,4EAA4E;QAC5E,0EAA0E;QAC1E,6EAA6E;QAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc;YACvC,CAAC,CAAC,kEAAkE;gBACnE,wEAAwE;gBACxE,WAAW,GAAG,CAAC;oBACf,CAAC,CAAC,kBAAkB,GAAG,uBAAuB;oBAC9C,CAAC,CAAC,CAAC;YACJ,CAAC,CAAC,kBAAkB,GAAG,uBAAuB,CAAC;QAEhD,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE1D,4BAA4B;QAC5B,8EAA8E;QAC9E,kCAAkC;QAElC,IAAI,UAAkB,CAAC;QACvB,IAAI,iBAAiB,GAAG,GAAG,CAAC;QAC5B,MAAM,kBAAkB,GAAG,GAAG,CAAC;QAE/B,UAAU;YACT,YAAY,GAAG,kBAAkB;gBACjC,aAAa,GAAG,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC;QAE1C,gCAAgC;QAChC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACrB,IACC,IAAI,CAAC,cAAc,IAAI,IAAI;gBAC3B,IAAI,CAAC,cAAc,GAAG,CAAC;gBACvB,eAAe,GAAG,CAAC,EAClB,CAAC;gBACF,wEAAwE;gBACxE,0EAA0E;gBAC1E,uEAAuE;gBACvE,+EAA+E;gBAC/E,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAC3B,GAAG,EACH,iBAAiB,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC,CAC7D,CAAC;YACH,CAAC;YACD,UAAU;gBACT,WAAW,GAAG,iBAAiB,GAAG,UAAU,GAAG,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC;QACzE,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACpE,MAAM,cAAc,GAAG,GAAG,CAAC;YAC3B,UAAU;gBACT,UAAU,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC;oBACjC,cAAc,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAY,CAAC,CAAC;QAClD,CAAC;QAED,gBAAgB;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC;QAEnC,gBAAgB;QAChB,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC;QAE5B,mEAAmE;QACnE,MAAM,IAAI,GAAG,GAAG,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE/D,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QAEtC,kBAAkB;QAClB,MAAM,UAAU,GAAG,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC;QAEnC,uCAAuC;QACvC,MAAM,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;QACrC,IAAI,SAAS,GAAG,aAAa,GAAG,MAAM,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YAC7D,mEAAmE;YACnE,8DAA8D;YAC9D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;YACrD,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;gBAC/B,SAAS;oBACR,eAAe,IAAI,CAAC;wBACnB,CAAC,CAAC,aAAa;wBACf,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,GAAG,eAAe,CAAC,CAAC;YAC1D,CAAC;QACF,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC;QAE5B,oDAAoD;QACpD,2DAA2D;QAC3D,oBAAoB;QACpB,IAAI,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,IAAI,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACnB,CAAC;QAED;;;;;;;;;;;;;;;;;;;;kBAoBO;QAEP,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;IAC1B,CAAC;CACD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@peerbit/shared-log",
|
|
3
|
-
"version": "13.1.
|
|
3
|
+
"version": "13.1.3",
|
|
4
4
|
"description": "Shared log",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"type": "module",
|
|
@@ -62,27 +62,27 @@
|
|
|
62
62
|
"pino": "^9.4.0",
|
|
63
63
|
"uint8arrays": "^5.1.0",
|
|
64
64
|
"@peerbit/any-store": "2.2.9",
|
|
65
|
-
"@peerbit/blocks-interface": "2.0.8",
|
|
66
|
-
"@peerbit/blocks": "4.1.0",
|
|
67
65
|
"@peerbit/cache": "3.0.0",
|
|
68
|
-
"@peerbit/
|
|
66
|
+
"@peerbit/blocks-interface": "2.0.8",
|
|
69
67
|
"@peerbit/indexer-interface": "3.0.3",
|
|
68
|
+
"@peerbit/crypto": "3.1.1",
|
|
69
|
+
"@peerbit/blocks": "4.1.1",
|
|
70
70
|
"@peerbit/indexer-sqlite3": "3.0.6",
|
|
71
|
-
"@peerbit/log": "6.0.22",
|
|
72
71
|
"@peerbit/logger": "2.0.1",
|
|
73
|
-
"@peerbit/program": "6.0.
|
|
72
|
+
"@peerbit/program": "6.0.19",
|
|
74
73
|
"@peerbit/pubsub-interface": "5.1.1",
|
|
74
|
+
"@peerbit/pubsub": "5.2.0",
|
|
75
|
+
"@peerbit/rpc": "6.0.23",
|
|
75
76
|
"@peerbit/riblt": "1.2.0",
|
|
76
|
-
"@peerbit/rpc": "6.0.22",
|
|
77
77
|
"@peerbit/stream-interface": "6.0.7",
|
|
78
|
-
"@peerbit/
|
|
79
|
-
"@peerbit/
|
|
78
|
+
"@peerbit/log": "6.0.24",
|
|
79
|
+
"@peerbit/time": "3.0.0"
|
|
80
80
|
},
|
|
81
81
|
"devDependencies": {
|
|
82
82
|
"@types/libsodium-wrappers": "^0.7.14",
|
|
83
83
|
"@types/pidusage": "^2.0.5",
|
|
84
84
|
"uuid": "^10.0.0",
|
|
85
|
-
"@peerbit/test-utils": "3.0.
|
|
85
|
+
"@peerbit/test-utils": "3.0.23"
|
|
86
86
|
},
|
|
87
87
|
"repository": {
|
|
88
88
|
"type": "git",
|
package/src/exchange-heads.ts
CHANGED
|
@@ -7,6 +7,9 @@ import { TransportMessage } from "./message.js";
|
|
|
7
7
|
const logger = loggerFn("peerbit:shared-log:exchange-heads");
|
|
8
8
|
const warn = logger.newScope("warn");
|
|
9
9
|
|
|
10
|
+
// Stored in the reserved bytes so older peers ignore the hint.
|
|
11
|
+
export const EXCHANGE_HEADS_REPAIR_HINT = 1;
|
|
12
|
+
|
|
10
13
|
/**
|
|
11
14
|
* This thing allows use to faster sync since we can provide
|
|
12
15
|
* references that can be read concurrently to
|
package/src/index.ts
CHANGED
|
@@ -88,6 +88,7 @@ const getSharedLogFanoutService = (
|
|
|
88
88
|
): FanoutTree | undefined =>
|
|
89
89
|
(services as SharedLogServicesWithFanout).fanout;
|
|
90
90
|
import {
|
|
91
|
+
EXCHANGE_HEADS_REPAIR_HINT,
|
|
91
92
|
EntryWithRefs,
|
|
92
93
|
ExchangeHeadsMessage,
|
|
93
94
|
RequestIPrune,
|
|
@@ -2902,6 +2903,7 @@ export class SharedLog<
|
|
|
2902
2903
|
this.log,
|
|
2903
2904
|
[...entries.keys()],
|
|
2904
2905
|
)) {
|
|
2906
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
2905
2907
|
await this.rpc.send(message, {
|
|
2906
2908
|
priority: 1,
|
|
2907
2909
|
mode: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
@@ -5150,6 +5152,8 @@ export class SharedLog<
|
|
|
5150
5152
|
*/
|
|
5151
5153
|
|
|
5152
5154
|
const { heads } = msg;
|
|
5155
|
+
const isRepairHint =
|
|
5156
|
+
(msg.reserved[0] & EXCHANGE_HEADS_REPAIR_HINT) !== 0;
|
|
5153
5157
|
|
|
5154
5158
|
logger.trace(
|
|
5155
5159
|
`${this.node.identity.publicKey.hashcode()}: Recieved heads: ${
|
|
@@ -5270,8 +5274,15 @@ export class SharedLog<
|
|
|
5270
5274
|
|
|
5271
5275
|
let maybeDelete: EntryWithRefs<any>[][] | undefined;
|
|
5272
5276
|
let toMerge: Entry<any>[] = [];
|
|
5277
|
+
let toPersist: Entry<any>[] = [];
|
|
5273
5278
|
let toDelete: Entry<any>[] | undefined;
|
|
5274
|
-
|
|
5279
|
+
// Targeted repair is sent only to peers the sender currently believes
|
|
5280
|
+
// should store the entry. Accept it while local membership catches up;
|
|
5281
|
+
// the normal checked-prune path below can still remove it if this peer
|
|
5282
|
+
// truly no longer owns the entry.
|
|
5283
|
+
const acceptsTargetedRepair = isRepairHint && fromIsLeader;
|
|
5284
|
+
const keepAsLeader = isLeader || acceptsTargetedRepair;
|
|
5285
|
+
if (keepAsLeader) {
|
|
5275
5286
|
for (const entry of entries) {
|
|
5276
5287
|
this.pruneDebouncedFn.delete(entry.entry.hash);
|
|
5277
5288
|
this.removePruneRequestSent(entry.entry.hash);
|
|
@@ -5292,8 +5303,9 @@ export class SharedLog<
|
|
|
5292
5303
|
}
|
|
5293
5304
|
|
|
5294
5305
|
outer: for (const entry of entries) {
|
|
5295
|
-
if (
|
|
5306
|
+
if (keepAsLeader || (await this.keep?.(entry.entry))) {
|
|
5296
5307
|
toMerge.push(entry.entry);
|
|
5308
|
+
toPersist.push(entry.entry);
|
|
5297
5309
|
} else {
|
|
5298
5310
|
for (const ref of entry.gidRefrences) {
|
|
5299
5311
|
const map = await this.log.entryIndex.getHeads(ref).all();
|
|
@@ -5322,6 +5334,16 @@ export class SharedLog<
|
|
|
5322
5334
|
context.from!.hashcode(),
|
|
5323
5335
|
);
|
|
5324
5336
|
await this.log.join(toMerge);
|
|
5337
|
+
// Network joins bypass SharedLog.join(), but churn repair scans
|
|
5338
|
+
// the coordinate index to redistribute entries after membership changes.
|
|
5339
|
+
for (const entry of toPersist) {
|
|
5340
|
+
const replicas = decodeReplicas(entry).getValue(this);
|
|
5341
|
+
await this.findLeaders(
|
|
5342
|
+
await this.createCoordinates(entry, replicas),
|
|
5343
|
+
entry,
|
|
5344
|
+
{ roleAge: 0, persist: {} },
|
|
5345
|
+
);
|
|
5346
|
+
}
|
|
5325
5347
|
for (const merged of toMerge) {
|
|
5326
5348
|
confirmedHashes.add(merged.hash);
|
|
5327
5349
|
}
|
|
@@ -5396,7 +5418,11 @@ export class SharedLog<
|
|
|
5396
5418
|
const indexedEntry = await this.log.entryIndex.getShallow(hash);
|
|
5397
5419
|
let isLeader = false;
|
|
5398
5420
|
|
|
5399
|
-
if (
|
|
5421
|
+
if (
|
|
5422
|
+
indexedEntry &&
|
|
5423
|
+
!this._pendingDeletes.has(hash) &&
|
|
5424
|
+
(await this.log.blocks.has(hash))
|
|
5425
|
+
) {
|
|
5400
5426
|
this.removePeerFromGidPeerHistory(
|
|
5401
5427
|
context.from!.hashcode(),
|
|
5402
5428
|
indexedEntry!.value.meta.gid,
|
package/src/pid.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
const MIN_MEMORY_HEADROOM_BALANCE_SCALER = 0.25;
|
|
2
|
+
|
|
1
3
|
export class PIDReplicationController {
|
|
2
4
|
integral!: number;
|
|
3
5
|
prevError!: number;
|
|
@@ -56,14 +58,16 @@ export class PIDReplicationController {
|
|
|
56
58
|
this.maxMemoryLimit > 0 ? this.maxMemoryLimit * 0.95 : 0;
|
|
57
59
|
errorMemory =
|
|
58
60
|
currentFactor > 0 && memoryUsage > 0
|
|
59
|
-
? Math.max(
|
|
60
|
-
|
|
61
|
+
? Math.max(
|
|
62
|
+
Math.min(1, effectiveMemoryLimit / estimatedTotalSize),
|
|
63
|
+
0,
|
|
64
|
+
) - currentFactor
|
|
61
65
|
: 0;
|
|
62
66
|
// Math.max(Math.min((this.maxMemoryLimit - memoryUsage) / 100e5, 1), -1)// Math.min(Math.max((this.maxMemoryLimit - memoryUsage, 0) / 10e5, 0), 1);
|
|
63
67
|
}
|
|
64
68
|
|
|
65
69
|
const errorCoverageUnmodified = Math.min(1 - totalFactor, 1);
|
|
66
|
-
|
|
70
|
+
let errorCoverage =
|
|
67
71
|
(this.maxMemoryLimit ? 1 - Math.sqrt(Math.abs(errorMemory)) : 1) *
|
|
68
72
|
errorCoverageUnmodified;
|
|
69
73
|
|
|
@@ -76,14 +80,26 @@ export class PIDReplicationController {
|
|
|
76
80
|
// is material. This avoids oscillations around `totalFactor ~= 1`.
|
|
77
81
|
const coverageDeficit = Math.max(0, errorCoverageUnmodified); // ~= max(0, 1 - totalFactor)
|
|
78
82
|
const negativeBalanceScale =
|
|
79
|
-
coverageDeficit <= 0
|
|
80
|
-
? 1
|
|
81
|
-
: 1 - Math.min(1, coverageDeficit / 0.1); // full clamp at 10% deficit
|
|
83
|
+
coverageDeficit <= 0 ? 1 : 1 - Math.min(1, coverageDeficit / 0.1); // full clamp at 10% deficit
|
|
82
84
|
const errorFromEvenForBalance =
|
|
83
85
|
errorFromEven >= 0 ? errorFromEven : errorFromEven * negativeBalanceScale;
|
|
84
86
|
|
|
87
|
+
const hasMemoryHeadroom =
|
|
88
|
+
this.maxMemoryLimit != null && this.maxMemoryLimit > 0 && errorMemory > 0;
|
|
89
|
+
if (hasMemoryHeadroom && errorFromEvenForBalance > 0) {
|
|
90
|
+
// Coverage surplus often means another peer has not pruned yet. Do not let
|
|
91
|
+
// that transient surplus cancel a constrained peer that is still below an
|
|
92
|
+
// even share and has storage headroom to take more work.
|
|
93
|
+
errorCoverage = Math.max(errorCoverage, 0);
|
|
94
|
+
}
|
|
95
|
+
|
|
85
96
|
const balanceErrorScaler = this.maxMemoryLimit
|
|
86
|
-
?
|
|
97
|
+
? hasMemoryHeadroom
|
|
98
|
+
? Math.max(
|
|
99
|
+
Math.abs(errorMemory),
|
|
100
|
+
MIN_MEMORY_HEADROOM_BALANCE_SCALER,
|
|
101
|
+
)
|
|
102
|
+
: Math.abs(errorMemory)
|
|
87
103
|
: 1 - Math.abs(errorCoverage);
|
|
88
104
|
|
|
89
105
|
// Balance should be symmetric (allow negative error) so a peer can *reduce*
|
|
@@ -124,8 +140,7 @@ export class PIDReplicationController {
|
|
|
124
140
|
// restored, while preserving the hard shrink behavior for zero-capacity peers.
|
|
125
141
|
errorMemoryFactor = Math.max(
|
|
126
142
|
0.2,
|
|
127
|
-
errorMemoryFactor -
|
|
128
|
-
0.7 * Math.min(1, coverageDeficit / 0.25),
|
|
143
|
+
errorMemoryFactor - 0.7 * Math.min(1, coverageDeficit / 0.25),
|
|
129
144
|
);
|
|
130
145
|
}
|
|
131
146
|
totalError =
|
|
@@ -158,7 +173,19 @@ export class PIDReplicationController {
|
|
|
158
173
|
|
|
159
174
|
// Calculate the new replication factor
|
|
160
175
|
const change = pTerm + iTerm + dTerm;
|
|
161
|
-
|
|
176
|
+
let newFactor = currentFactor + change;
|
|
177
|
+
|
|
178
|
+
if (this.maxCPUUsage != null && this.maxMemoryLimit == null) {
|
|
179
|
+
// CPU pressure may shed surplus replicas, but it must not create a
|
|
180
|
+
// coverage gap where the network no longer has one full copy.
|
|
181
|
+
const coverageSurplus = Math.max(0, totalFactor - 1);
|
|
182
|
+
if (newFactor < currentFactor) {
|
|
183
|
+
newFactor =
|
|
184
|
+
coverageSurplus <= 0
|
|
185
|
+
? currentFactor
|
|
186
|
+
: Math.max(newFactor, currentFactor - coverageSurplus);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
162
189
|
|
|
163
190
|
// Update state for the next iteration
|
|
164
191
|
this.prevError = totalError;
|