@peerbit/shared-log 4.0.11 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -48,7 +48,7 @@ import { CustomEvent } from "@libp2p/interface";
48
48
  import yallist from "yallist";
49
49
  import {
50
50
  AcknowledgeDelivery,
51
- AnyWhere,
51
+ DeliveryMode,
52
52
  SeekDelivery,
53
53
  SilentDelivery
54
54
  } from "@peerbit/stream-interface";
@@ -121,22 +121,25 @@ const isAdaptiveReplicatorOption = (
121
121
  return false;
122
122
  };
123
123
 
124
- export type SharedLogOptions = {
124
+ export type SharedLogOptions<T> = {
125
125
  role?: RoleOptions;
126
126
  replicas?: ReplicationLimitsOptions;
127
127
  respondToIHaveTimeout?: number;
128
128
  canReplicate?: (publicKey: PublicSignKey) => Promise<boolean> | boolean;
129
+ sync?: (entry: Entry<T>) => boolean;
130
+ timeUntilRoleMaturity?: number;
129
131
  };
130
132
 
131
133
  export const DEFAULT_MIN_REPLICAS = 2;
132
134
  export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
133
135
  export const WAIT_FOR_ROLE_MATURITY = 5000;
134
- const REBALANCE_DEBOUNCE_INTERAVAL = 30;
136
+ const REBALANCE_DEBOUNCE_INTERVAL = 100;
135
137
 
136
- export type Args<T> = LogProperties<T> & LogEvents<T> & SharedLogOptions;
138
+ export type Args<T> = LogProperties<T> & LogEvents<T> & SharedLogOptions<T>;
137
139
 
138
140
  export type SharedAppendOptions<T> = AppendOptions<T> & {
139
141
  replicas?: AbsoluteReplicas | number;
142
+ target?: "all" | "replicators";
140
143
  };
141
144
 
142
145
  type UpdateRoleEvent = { publicKey: PublicSignKey; role: Role };
@@ -195,12 +198,16 @@ export class SharedLog<T = Uint8Array> extends Program<
195
198
 
196
199
  private openTime: number;
197
200
 
201
+ private sync?: (entry: Entry<T>) => boolean;
202
+
198
203
  private rebalanceParticipationDebounced:
199
204
  | ReturnType<typeof debounce>
200
205
  | undefined;
201
206
 
202
207
  replicas: ReplicationLimits;
203
208
 
209
+ timeUntilRoleMaturity: number;
210
+
204
211
  constructor(properties?: { id?: Uint8Array }) {
205
212
  super();
206
213
  this.log = new Log(properties);
@@ -219,9 +226,8 @@ export class SharedLog<T = Uint8Array> extends Program<
219
226
  this.rebalanceParticipationDebounced = debounce(
220
227
  () => this.rebalanceParticipation(),
221
228
  Math.max(
222
- REBALANCE_DEBOUNCE_INTERAVAL,
223
- (this.getReplicatorsSorted()?.length || 0) *
224
- REBALANCE_DEBOUNCE_INTERAVAL
229
+ REBALANCE_DEBOUNCE_INTERVAL,
230
+ (this.getReplicatorsSorted()?.length || 0) * REBALANCE_DEBOUNCE_INTERVAL
225
231
  )
226
232
  );
227
233
  }
@@ -229,10 +235,13 @@ export class SharedLog<T = Uint8Array> extends Program<
229
235
  this.rebalanceParticipationDebounced = undefined;
230
236
 
231
237
  const setupDebouncedRebalancing = (options?: AdaptiveReplicatorOptions) => {
232
- this.replicationController = new PIDReplicationController({
233
- targetMemoryLimit: options?.limits?.memory,
234
- errorFunction: options?.error
235
- });
238
+ this.replicationController = new PIDReplicationController(
239
+ this.node.identity.publicKey.hashcode(),
240
+ {
241
+ targetMemoryLimit: options?.limits?.memory,
242
+ errorFunction: options?.error
243
+ }
244
+ );
236
245
 
237
246
  this.setupRebalanceDebounceFunction();
238
247
  };
@@ -250,7 +259,10 @@ export class SharedLog<T = Uint8Array> extends Program<
250
259
  setupDebouncedRebalancing(options);
251
260
  this._roleOptions = options;
252
261
  } else {
253
- this._roleOptions = new Replicator({ factor: options.factor });
262
+ this._roleOptions = new Replicator({
263
+ factor: options.factor,
264
+ offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
265
+ });
254
266
  }
255
267
  } else {
256
268
  this._roleOptions = new Observer();
@@ -272,11 +284,13 @@ export class SharedLog<T = Uint8Array> extends Program<
272
284
  if (this._roleOptions.limits) {
273
285
  this._role = new Replicator({
274
286
  // initial role in a dynamic setup
275
- factor: 1
287
+ factor: 1,
288
+ offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
276
289
  });
277
290
  } else {
278
291
  this._role = new Replicator({
279
- factor: 1
292
+ factor: 1,
293
+ offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
280
294
  });
281
295
  }
282
296
  }
@@ -338,11 +352,20 @@ export class SharedLog<T = Uint8Array> extends Program<
338
352
  }
339
353
 
340
354
  const result = await this.log.append(data, appendOptions);
341
- const leaders = await this.findLeaders(
342
- result.entry.meta.gid,
343
- decodeReplicas(result.entry).getValue(this)
344
- );
345
- const isLeader = leaders.includes(this.node.identity.publicKey.hashcode());
355
+ let mode: DeliveryMode | undefined = undefined;
356
+
357
+ if (options?.target === "replicators" || !options?.target) {
358
+ const leaders = await this.findLeaders(
359
+ result.entry.meta.gid,
360
+ decodeReplicas(result.entry).getValue(this)
361
+ );
362
+ const isLeader = leaders.includes(
363
+ this.node.identity.publicKey.hashcode()
364
+ );
365
+ mode = isLeader
366
+ ? new SilentDelivery({ redundancy: 1, to: leaders })
367
+ : new AcknowledgeDelivery({ redundancy: 1, to: leaders });
368
+ }
346
369
 
347
370
  await this.rpc.send(
348
371
  await createExchangeHeadsMessage(
@@ -351,9 +374,7 @@ export class SharedLog<T = Uint8Array> extends Program<
351
374
  this._gidParentCache
352
375
  ),
353
376
  {
354
- mode: isLeader
355
- ? new SilentDelivery({ redundancy: 1, to: leaders })
356
- : new AcknowledgeDelivery({ redundancy: 1, to: leaders })
377
+ mode
357
378
  }
358
379
  );
359
380
 
@@ -381,11 +402,13 @@ export class SharedLog<T = Uint8Array> extends Program<
381
402
  this._pendingIHave = new Map();
382
403
  this.latestRoleMessages = new Map();
383
404
  this.openTime = +new Date();
384
-
405
+ this.timeUntilRoleMaturity =
406
+ options?.timeUntilRoleMaturity || WAIT_FOR_ROLE_MATURITY;
385
407
  this._gidParentCache = new Cache({ max: 1000 });
386
408
  this._closeController = new AbortController();
387
409
 
388
410
  this._canReplicate = options?.canReplicate;
411
+ this.sync = options?.sync;
389
412
  this._logProperties = options;
390
413
 
391
414
  this.setupRole(options?.role);
@@ -414,17 +437,6 @@ export class SharedLog<T = Uint8Array> extends Program<
414
437
  this._sortedPeersCache = yallist.create();
415
438
  this._gidPeersHistory = new Map();
416
439
 
417
- await this.node.services.pubsub.addEventListener(
418
- "subscribe",
419
- this._onSubscriptionFn
420
- );
421
-
422
- this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
423
- await this.node.services.pubsub.addEventListener(
424
- "unsubscribe",
425
- this._onUnsubscriptionFn
426
- );
427
-
428
440
  const cache = await storage.sublevel("cache");
429
441
 
430
442
  await this.log.open(this.remoteBlocks, this.node.identity, {
@@ -484,6 +496,17 @@ export class SharedLog<T = Uint8Array> extends Program<
484
496
  topic: this.topic
485
497
  });
486
498
 
499
+ await this.node.services.pubsub.addEventListener(
500
+ "subscribe",
501
+ this._onSubscriptionFn
502
+ );
503
+
504
+ this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
505
+ await this.node.services.pubsub.addEventListener(
506
+ "unsubscribe",
507
+ this._onUnsubscriptionFn
508
+ );
509
+
487
510
  await this.log.load();
488
511
  }
489
512
 
@@ -612,8 +635,6 @@ export class SharedLog<T = Uint8Array> extends Program<
612
635
  const groupedByGid = await groupByGid(filteredHeads);
613
636
  const promises: Promise<void>[] = [];
614
637
 
615
- /// console.log("ADD CACHE", this.node.identity.publicKey.hashcode(), context.from!.hashcode(), groupedByGid.size)
616
-
617
638
  for (const [gid, entries] of groupedByGid) {
618
639
  const fn = async () => {
619
640
  const headsWithGid = this.log.headsIndex.gids.get(gid);
@@ -628,10 +649,15 @@ export class SharedLog<T = Uint8Array> extends Program<
628
649
  entries.map((x) => x.entry)
629
650
  );
630
651
 
631
- const leaders = await this.waitForIsLeader(
632
- gid,
633
- Math.max(maxReplicasFromHead, maxReplicasFromNewEntries)
634
- );
652
+ const leaders = await (this.role instanceof Observer
653
+ ? this.findLeaders(
654
+ gid,
655
+ Math.max(maxReplicasFromHead, maxReplicasFromNewEntries)
656
+ )
657
+ : this.waitForIsLeader(
658
+ gid,
659
+ Math.max(maxReplicasFromHead, maxReplicasFromNewEntries)
660
+ ));
635
661
  const isLeader = !!leaders;
636
662
  if (isLeader) {
637
663
  if (leaders.find((x) => x === context.from!.hashcode())) {
@@ -649,7 +675,7 @@ export class SharedLog<T = Uint8Array> extends Program<
649
675
  }
650
676
 
651
677
  outer: for (const entry of entries) {
652
- if (isLeader) {
678
+ if (isLeader || this.sync?.(entry.entry)) {
653
679
  toMerge.push(entry);
654
680
  } else {
655
681
  for (const ref of entry.references) {
@@ -754,7 +780,6 @@ export class SharedLog<T = Uint8Array> extends Program<
754
780
  }
755
781
 
756
782
  prevPendingIHave && prevPendingIHave.callback(entry);
757
-
758
783
  this._pendingIHave.delete(entry.hash);
759
784
  }
760
785
  };
@@ -804,8 +829,6 @@ export class SharedLog<T = Uint8Array> extends Program<
804
829
  timeout: WAIT_FOR_REPLICATOR_TIMEOUT
805
830
  })
806
831
  .then(async () => {
807
- /* await delay(1000 * Math.random()) */
808
-
809
832
  const prev = this.latestRoleMessages.get(context.from!.hashcode());
810
833
  if (prev && prev > context.timestamp) {
811
834
  return;
@@ -982,7 +1005,7 @@ export class SharedLog<T = Uint8Array> extends Program<
982
1005
  const startNode = currentNode;
983
1006
  const diffs: { diff: number; rect: ReplicatorRect }[] = [];
984
1007
  while (currentNode && !done()) {
985
- const start = currentNode.value.offset % width;
1008
+ const start = currentNode.value.role.offset % width;
986
1009
  const absDelta = Math.abs(start - point());
987
1010
  const diff = Math.min(absDelta, width - absDelta);
988
1011
 
@@ -1041,7 +1064,7 @@ export class SharedLog<T = Uint8Array> extends Program<
1041
1064
  const t = +new Date();
1042
1065
  const roleAge =
1043
1066
  options?.roleAge ??
1044
- Math.min(WAIT_FOR_ROLE_MATURITY, +new Date() - this.openTime);
1067
+ Math.min(this.timeUntilRoleMaturity, +new Date() - this.openTime);
1045
1068
 
1046
1069
  for (let i = 0; i < numberOfLeaders; i++) {
1047
1070
  const point = ((cursor + i / numberOfLeaders) % 1) * width;
@@ -1064,7 +1087,7 @@ export class SharedLog<T = Uint8Array> extends Program<
1064
1087
  *
1065
1088
  * @returns groups where at least one in any group will have the entry you are looking for
1066
1089
  */
1067
- getReplicatorUnion(roleAge: number = WAIT_FOR_ROLE_MATURITY) {
1090
+ getReplicatorUnion(roleAge: number = this.timeUntilRoleMaturity) {
1068
1091
  if (this.closed === true) {
1069
1092
  throw new Error("Closed");
1070
1093
  }
@@ -1112,7 +1135,7 @@ export class SharedLog<T = Uint8Array> extends Program<
1112
1135
  return [];
1113
1136
  }
1114
1137
 
1115
- let nextPoint = startNode.value.offset;
1138
+ let nextPoint = startNode.value.role.offset;
1116
1139
  const t = +new Date();
1117
1140
  this.collectNodesAroundPoint(
1118
1141
  t,
@@ -1158,7 +1181,7 @@ export class SharedLog<T = Uint8Array> extends Program<
1158
1181
  this._closeController.signal.removeEventListener("abort", listener);
1159
1182
  await this.rebalanceParticipationDebounced?.();
1160
1183
  this.distribute();
1161
- }, WAIT_FOR_ROLE_MATURITY + 2000);
1184
+ }, this.timeUntilRoleMaturity + 2000);
1162
1185
 
1163
1186
  const listener = () => {
1164
1187
  clearTimeout(timer);
@@ -1179,14 +1202,13 @@ export class SharedLog<T = Uint8Array> extends Program<
1179
1202
  publicKey: PublicSignKey
1180
1203
  ) {
1181
1204
  const update = await this._modifyReplicators(role, publicKey);
1205
+
1182
1206
  if (update.changed !== "none") {
1183
1207
  if (update.changed === "added" || update.changed === "removed") {
1184
1208
  this.setupRebalanceDebounceFunction();
1185
1209
  }
1186
1210
 
1187
- if (this.rebalanceParticipationDebounced) {
1188
- await this.rebalanceParticipationDebounced?.(); /* await this.rebalanceParticipation(false); */
1189
- }
1211
+ await this.rebalanceParticipationDebounced?.(); /* await this.rebalanceParticipation(false); */
1190
1212
  if (update.changed === "added") {
1191
1213
  await this.rpc.send(new ResponseRoleMessage({ role: this._role }), {
1192
1214
  mode: new SeekDelivery({
@@ -1236,10 +1258,8 @@ export class SharedLog<T = Uint8Array> extends Program<
1236
1258
 
1237
1259
  if (isOnline) {
1238
1260
  // insert or if already there do nothing
1239
- const code = hashToUniformNumber(publicKey.bytes);
1240
1261
  const rect: ReplicatorRect = {
1241
1262
  publicKey,
1242
- offset: code,
1243
1263
  role
1244
1264
  };
1245
1265
 
@@ -1261,7 +1281,7 @@ export class SharedLog<T = Uint8Array> extends Program<
1261
1281
  return { prev: prev.role, changed: "updated" };
1262
1282
  }
1263
1283
 
1264
- if (code > currentNode.value.offset) {
1284
+ if (role.offset > currentNode.value.role.offset) {
1265
1285
  const next = currentNode?.next;
1266
1286
  if (next) {
1267
1287
  currentNode = next;
@@ -1306,29 +1326,28 @@ export class SharedLog<T = Uint8Array> extends Program<
1306
1326
  changes: string[],
1307
1327
  subscribed: boolean
1308
1328
  ) {
1309
- if (subscribed) {
1310
- if (this.role instanceof Replicator) {
1311
- for (const subscription of changes) {
1312
- if (this.log.idString !== subscription) {
1313
- continue;
1314
- }
1315
- this.rpc
1316
- .send(new ResponseRoleMessage({ role: this._role }), {
1317
- mode: new SeekDelivery({ redundancy: 1, to: [publicKey] })
1318
- })
1319
- .catch((e) => logger.error(e.toString()));
1320
- }
1329
+ for (const topic of changes) {
1330
+ if (this.log.idString !== topic) {
1331
+ continue;
1321
1332
  }
1333
+ }
1322
1334
 
1323
- //if(evt.detail.subscriptions.map((x) => x.topic).includes())
1324
- } else {
1325
- for (const topic of changes) {
1326
- if (this.log.idString !== topic) {
1327
- continue;
1328
- }
1335
+ if (!subscribed) {
1336
+ for (const [_a, b] of this._gidPeersHistory) {
1337
+ b.delete(publicKey.hashcode());
1338
+ }
1339
+ }
1329
1340
 
1330
- await this.modifyReplicators(new Observer(), publicKey);
1341
+ if (subscribed) {
1342
+ if (this.role instanceof Replicator) {
1343
+ this.rpc
1344
+ .send(new ResponseRoleMessage({ role: this._role }), {
1345
+ mode: new SeekDelivery({ redundancy: 1, to: [publicKey] })
1346
+ })
1347
+ .catch((e) => logger.error(e.toString()));
1331
1348
  }
1349
+ } else {
1350
+ await this.modifyReplicators(new Observer(), publicKey);
1332
1351
  }
1333
1352
  }
1334
1353
 
@@ -1469,7 +1488,7 @@ export class SharedLog<T = Uint8Array> extends Program<
1469
1488
  _queue: PQueue;
1470
1489
  async distribute() {
1471
1490
  if (this._queue?.size > 0) {
1472
- return;
1491
+ return this._queue.onEmpty();
1473
1492
  }
1474
1493
  (this._queue || (this._queue = new PQueue({ concurrency: 1 }))).add(() =>
1475
1494
  this._distribute()
@@ -1507,6 +1526,7 @@ export class SharedLog<T = Uint8Array> extends Program<
1507
1526
  gid,
1508
1527
  maxReplicas(this, entries) // pick max replication policy of all entries, so all information is treated equally important as the most important
1509
1528
  );
1529
+
1510
1530
  const isLeader = currentPeers.find(
1511
1531
  (x) => x === this.node.identity.publicKey.hashcode()
1512
1532
  );
@@ -1552,8 +1572,6 @@ export class SharedLog<T = Uint8Array> extends Program<
1552
1572
  }
1553
1573
 
1554
1574
  for (const [target, entries] of uncheckedDeliver) {
1555
- const promise: Promise<any> = Promise.resolve();
1556
-
1557
1575
  // TODO better choice of step size
1558
1576
  for (let i = 0; i < entries.length; i += 100) {
1559
1577
  const message = await createExchangeHeadsMessage(
@@ -1646,10 +1664,14 @@ export class SharedLog<T = Uint8Array> extends Program<
1646
1664
  );
1647
1665
  }
1648
1666
 
1667
+ xxx: number;
1649
1668
  async rebalanceParticipation(onRoleChange = true) {
1650
1669
  // update more participation rate to converge to the average expected rate or bounded by
1651
1670
  // resources such as memory and or cpu
1652
1671
 
1672
+ const t = +new Date();
1673
+ // console.log(t - this.xxx)
1674
+ this.xxx = t;
1653
1675
  if (this.closed) {
1654
1676
  return false;
1655
1677
  }
@@ -1675,15 +1697,16 @@ export class SharedLog<T = Uint8Array> extends Program<
1675
1697
  peers?.length || 1
1676
1698
  );
1677
1699
 
1678
- const newRole = new Replicator({
1679
- factor: newFactor,
1680
- timestamp: this._role.timestamp
1681
- });
1682
-
1683
1700
  const relativeDifference =
1684
- Math.abs(this._role.factor - newRole.factor) / this._role.factor;
1701
+ Math.abs(this._role.factor - newFactor) / this._role.factor;
1685
1702
 
1686
1703
  if (relativeDifference > 0.0001) {
1704
+ const newRole = new Replicator({
1705
+ factor: newFactor,
1706
+ timestamp: this._role.timestamp,
1707
+ offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
1708
+ });
1709
+
1687
1710
  const canReplicate =
1688
1711
  !this._canReplicate ||
1689
1712
  (await this._canReplicate(this.node.identity.publicKey, newRole));
@@ -1692,7 +1715,11 @@ export class SharedLog<T = Uint8Array> extends Program<
1692
1715
  }
1693
1716
 
1694
1717
  await this._updateRole(newRole, onRoleChange);
1718
+ this.rebalanceParticipationDebounced?.();
1719
+
1695
1720
  return true;
1721
+ } else {
1722
+ this.rebalanceParticipationDebounced?.();
1696
1723
  }
1697
1724
  return false;
1698
1725
  }
package/src/pid.ts CHANGED
@@ -8,13 +8,14 @@ export class PIDReplicationController {
8
8
  integral: number;
9
9
  prevError: number;
10
10
  prevMemoryUsage: number;
11
- lastTs: number;
11
+ prevTotalFactor: number;
12
12
  kp: number;
13
13
  ki: number;
14
14
  kd: number;
15
15
  errorFunction: ReplicationErrorFunction;
16
16
  targetMemoryLimit?: number;
17
17
  constructor(
18
+ readonly id: string,
18
19
  options: {
19
20
  errorFunction?: ReplicationErrorFunction;
20
21
  targetMemoryLimit?: number;
@@ -25,11 +26,14 @@ export class PIDReplicationController {
25
26
  ) {
26
27
  const {
27
28
  targetMemoryLimit,
28
- kp = 0.5,
29
- ki = 0.1,
30
- kd = 0.25,
31
- errorFunction = ({ balance, coverage, memory }) =>
32
- memory * 0.8 + balance * 0.1 + coverage * 0.1
29
+ kp = 0.7,
30
+ ki = 0.025,
31
+ kd = 0.05,
32
+ errorFunction = ({ balance, coverage, memory }) => {
33
+ return memory < 0
34
+ ? memory * 0.9 + balance * 0.06 + coverage * 0.04
35
+ : balance * 0.6 + coverage * 0.4;
36
+ }
33
37
  } = options;
34
38
  this.reset();
35
39
  this.kp = kp;
@@ -39,18 +43,27 @@ export class PIDReplicationController {
39
43
  this.errorFunction = errorFunction;
40
44
  }
41
45
 
46
+ /**
47
+ * Call this function on a period interval since it does not track time passed
48
+ * @param memoryUsage
49
+ * @param currentFactor
50
+ * @param totalFactor
51
+ * @param peerCount
52
+ * @returns
53
+ */
42
54
  async adjustReplicationFactor(
43
55
  memoryUsage: number,
44
56
  currentFactor: number,
45
57
  totalFactor: number,
46
58
  peerCount: number
47
59
  ) {
60
+ const totalFactorDiff = totalFactor - this.prevTotalFactor;
61
+ this.prevTotalFactor = totalFactor;
48
62
  this.prevMemoryUsage = memoryUsage;
49
63
 
50
64
  const estimatedTotalSize = memoryUsage / currentFactor;
51
65
 
52
66
  let errorMemory = 0;
53
- const errorTarget = 1 / peerCount - currentFactor;
54
67
 
55
68
  if (this.targetMemoryLimit != null) {
56
69
  errorMemory =
@@ -59,42 +72,39 @@ export class PIDReplicationController {
59
72
  Math.min(1, this.targetMemoryLimit / estimatedTotalSize),
60
73
  0
61
74
  ) - currentFactor
62
- : 0.0001;
75
+ : 0;
63
76
  }
64
77
 
65
78
  const errorCoverageUnmodified = Math.min(1 - totalFactor, 1);
66
- const includeCoverageError =
67
- Math.max(Math.abs(errorTarget), Math.abs(errorMemory)) < 0.01;
68
- const errorCoverage = includeCoverageError ? errorCoverageUnmodified : 0; /// 1 / (Math.max(Math.abs(errorTarget), Math.abs(errorMemory))) * errorCoverage / 100;
79
+ const errorCoverage =
80
+ (this.targetMemoryLimit ? 1 - Math.sqrt(Math.abs(errorMemory)) : 1) *
81
+ errorCoverageUnmodified;
69
82
 
70
- let totalError = this.errorFunction({
71
- balance: errorTarget,
83
+ const errorFromEven = 1 / peerCount - currentFactor;
84
+
85
+ const balanceErrorScaler = this.targetMemoryLimit
86
+ ? Math.abs(errorMemory)
87
+ : 1 - Math.abs(errorCoverage);
88
+
89
+ const errorBalance = (this.targetMemoryLimit ? errorMemory > -0.01 : true)
90
+ ? errorFromEven > 0
91
+ ? balanceErrorScaler * errorFromEven
92
+ : 0
93
+ : 0;
94
+
95
+ const totalError = this.errorFunction({
96
+ balance: errorBalance,
72
97
  coverage: errorCoverage,
73
98
  memory: errorMemory
74
99
  });
75
100
 
76
- if (totalError === 0 && !includeCoverageError) {
77
- totalError = this.errorFunction({
78
- balance: errorTarget,
79
- coverage: errorCoverageUnmodified,
80
- memory: errorMemory
81
- });
82
- }
83
-
84
- if (this.lastTs === 0) {
85
- this.lastTs = +new Date();
86
- }
87
- const kpAdjusted = Math.min(
88
- Math.max(this.kp, (+new Date() - this.lastTs) / 100),
89
- 0.8
90
- );
91
- const pTerm = kpAdjusted * totalError;
92
-
93
- this.lastTs = +new Date();
101
+ const pTerm = this.kp * totalError;
94
102
 
95
103
  // Integral term
96
104
  this.integral += totalError;
97
- const beta = 0.5;
105
+
106
+ // Beta controls how much of the accumulated error we should forget
107
+ const beta = 0.8;
98
108
  this.integral = beta * totalError + (1 - beta) * this.integral;
99
109
 
100
110
  const iTerm = this.ki * this.integral;
@@ -114,19 +124,30 @@ export class PIDReplicationController {
114
124
  this.integral = 0;
115
125
  }
116
126
 
127
+ // prevent drift when everone wants to do less
128
+ /* if (newFactor < currentFactor && totalFactorDiff < 0 && totalFactor < 0.5) {
129
+ newFactor = currentFactor;
130
+ this.integral = 0;
131
+ }
132
+ */
133
+
117
134
  /* console.log({
118
- newFactor,
135
+ id: this.id,
119
136
  currentFactor,
137
+ newFactor,
138
+ factorDiff: newFactor - currentFactor,
120
139
  pTerm,
121
140
  dTerm,
122
141
  iTerm,
123
- kpAdjusted,
124
142
  totalError,
125
- errorTarget,
143
+ errorTarget: errorBalance,
126
144
  errorCoverage,
127
145
  errorMemory,
128
146
  peerCount,
129
- totalFactor
147
+ totalFactor,
148
+ totalFactorDiff,
149
+ targetScaler: balanceErrorScaler,
150
+ estimatedTotalSize
130
151
  }); */
131
152
 
132
153
  return Math.max(Math.min(newFactor, 1), 0);
@@ -136,6 +157,5 @@ export class PIDReplicationController {
136
157
  this.prevError = 0;
137
158
  this.integral = 0;
138
159
  this.prevMemoryUsage = 0;
139
- this.lastTs = 0;
140
160
  }
141
161
  }
@@ -15,7 +15,6 @@ export type ReplicationLimits = { min: MinReplicas; max?: MinReplicas };
15
15
 
16
16
  export type ReplicatorRect = {
17
17
  publicKey: PublicSignKey;
18
- offset: number;
19
18
  role: Replicator;
20
19
  };
21
20
 
package/src/role.ts CHANGED
@@ -31,13 +31,13 @@ class ReplicationSegment {
31
31
  @field({ type: "u32" })
32
32
  private factorNominator: number;
33
33
 
34
- @field({ type: option("u32") })
35
- private offsetNominator?: number;
34
+ @field({ type: "u32" })
35
+ private offsetNominator: number;
36
36
 
37
37
  constructor(properties: {
38
38
  factor: number;
39
+ offset: number;
39
40
  timestamp?: bigint;
40
- offset?: number;
41
41
  }) {
42
42
  const { factor, timestamp, offset } = properties;
43
43
  if (factor > 1 || factor < 0) {
@@ -47,24 +47,18 @@ class ReplicationSegment {
47
47
  this.timestamp = timestamp ?? BigInt(+new Date());
48
48
  this.factorNominator = Math.round(4294967295 * factor);
49
49
 
50
- if (offset != null) {
51
- if (offset > 1 || offset < 0) {
52
- throw new Error(
53
- "Expecting offset to be between 0 and 1, got: " + offset
54
- );
55
- }
56
- this.offsetNominator = Math.round(4294967295 * offset);
50
+ if (offset > 1 || offset < 0) {
51
+ throw new Error("Expecting offset to be between 0 and 1, got: " + offset);
57
52
  }
53
+ this.offsetNominator = Math.round(4294967295 * offset);
58
54
  }
59
55
 
60
56
  get factor(): number {
61
57
  return this.factorNominator / 4294967295;
62
58
  }
63
59
 
64
- get offset(): number | undefined {
65
- return this.offsetNominator != null
66
- ? this.offsetNominator / 4294967295
67
- : undefined;
60
+ get offset(): number {
61
+ return this.offsetNominator / 4294967295;
68
62
  }
69
63
  }
70
64
 
@@ -76,7 +70,7 @@ export class Replicator extends Role {
76
70
  constructor(properties: {
77
71
  factor: number;
78
72
  timestamp?: bigint;
79
- offset?: number;
73
+ offset: number;
80
74
  }) {
81
75
  super();
82
76
  const segment: ReplicationSegment = new ReplicationSegment(properties);
@@ -87,7 +81,7 @@ export class Replicator extends Role {
87
81
  return this.segments[0]!.factor;
88
82
  }
89
83
 
90
- get offset(): number | undefined {
84
+ get offset(): number {
91
85
  return this.segments[0]!.offset;
92
86
  }
93
87