@activeledger/activenetwork 5.3.0 → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,7 @@ import { Neighbour } from "./neighbour";
14
14
  import { ActiveInterfaces } from "./utils";
15
15
  import { Endpoints, Maintain } from "./index";
16
16
  import { Locker } from "./locker";
17
+ import { withJsonFallback } from "./ipc";
17
18
  import { PhysicalCores } from "./cpus";
18
19
  import { DEFAULT_WORKER_COUNT, workerCount } from "./workercount";
19
20
  import { workerExecArgv } from "./workerheap";
@@ -28,10 +29,15 @@ import { gunzip } from "zlib";
28
29
  const GZIP_MIN_RESPONSE_BYTES = 16384;
29
30
  const gunzipAsync = promisify(gunzip);
30
31
  const TIMER_QUEUE_INTERVAL = 2 * 1000;
32
+ const RESENT_LOAD_WAIT_MS = 5 * 1000;
33
+ const STORED_WINDOW_MS = Number(process.env.AL_STORED_WINDOW_MS ?? 5 * 60 * 1000);
34
+ const STORED_MAX = Number(process.env.AL_STORED_MAX ?? 200000);
35
+ const STORED_UNKNOWN_MAX = Number(process.env.AL_STORED_UNKNOWN_MAX ?? 10000);
31
36
  const GRACEFUL_PROC_SHUTDOWN = 7 * 60 * 1000;
32
37
  const KILL_PROC_SHUTDOWN = 2.5 * 1000;
33
38
  const QUEUE_MAX_WAIT_MS = Number(process.env.AL_QUEUE_MAX_WAIT_MS || 7_000);
34
39
  const QUEUE_RECHECK_MS = Number(process.env.AL_QUEUE_RECHECK_MS ?? 50);
40
+ const QUEUE_SCAN_MIN_MS = Number(process.env.AL_QUEUE_SCAN_MIN_MS ?? 20);
35
41
  const MAX_RETRIES = 35;
36
42
  function peekBytes(chunks, n) {
37
43
  if (chunks.length > 0 && chunks[0].length >= n) {
@@ -265,51 +271,55 @@ export class Host extends Home {
265
271
  }
266
272
  });
267
273
  }
274
+ static p2pServerWanted() {
275
+ return ActiveOptions.get("p2pStreamServer", false);
276
+ }
277
+ p2pConnection(socket) {
278
+ socket.setNoDelay(true);
279
+ ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
280
+ let chunks = [];
281
+ let bufferLength = 0;
282
+ socket.on("data", async (data) => {
283
+ chunks.push(data);
284
+ bufferLength += data.length;
285
+ while (bufferLength >= 44) {
286
+ const header = peekBytes(chunks, 44);
287
+ const length = header.readUInt32BE(40);
288
+ if (bufferLength >= 44 + length) {
289
+ const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
290
+ const senderRef = fullBuffer.slice(0, 40).toString().trim();
291
+ const item = fullBuffer.slice(44, 44 + length);
292
+ const remaining = fullBuffer.slice(44 + length);
293
+ chunks = remaining.length > 0 ? [remaining] : [];
294
+ bufferLength = remaining.length;
295
+ let tx;
296
+ try {
297
+ tx = await ActiveClone.deserialize(item);
298
+ }
299
+ catch (error) {
300
+ ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
301
+ continue;
302
+ }
303
+ this.processEndpoints({
304
+ headers: { "X-Activeledger": senderRef },
305
+ method: "POST",
306
+ url: "/a/init",
307
+ connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
308
+ }, null, tx, senderRef, true);
309
+ }
310
+ else {
311
+ break;
312
+ }
313
+ }
314
+ });
315
+ socket.on("error", (err) => {
316
+ ActiveLogger.error(err, "P2P socket error");
317
+ });
318
+ }
268
319
  constructor() {
269
320
  super();
270
- if (ActiveOptions.get("p2pStreamServer", false)) {
271
- this.p2pServer = net.createServer((socket) => {
272
- socket.setNoDelay(true);
273
- ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
274
- let chunks = [];
275
- let bufferLength = 0;
276
- socket.on("data", async (data) => {
277
- chunks.push(data);
278
- bufferLength += data.length;
279
- while (bufferLength >= 44) {
280
- const header = peekBytes(chunks, 44);
281
- const length = header.readUInt32BE(40);
282
- if (bufferLength >= 44 + length) {
283
- const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
284
- const senderRef = fullBuffer.slice(0, 40).toString().trim();
285
- const item = fullBuffer.slice(44, 44 + length);
286
- const remaining = fullBuffer.slice(44 + length);
287
- chunks = remaining.length > 0 ? [remaining] : [];
288
- bufferLength = remaining.length;
289
- let tx;
290
- try {
291
- tx = await ActiveClone.deserialize(item);
292
- }
293
- catch (error) {
294
- ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
295
- continue;
296
- }
297
- this.processEndpoints({
298
- headers: { "X-Activeledger": senderRef },
299
- method: "POST",
300
- url: "/a/init",
301
- connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
302
- }, null, tx, senderRef, true);
303
- }
304
- else {
305
- break;
306
- }
307
- }
308
- });
309
- socket.on("error", (err) => {
310
- ActiveLogger.error(err, "P2P socket error");
311
- });
312
- });
321
+ if (Host.p2pServerWanted()) {
322
+ this.p2pServer = net.createServer((socket) => this.p2pConnection(socket));
313
323
  const p2pPort = parseInt(ActiveInterfaces.getBindingDetails("port")) + 1;
314
324
  this.p2pServer.listen(p2pPort, () => {
315
325
  ActiveLogger.info(`P2P TCP server listening on port ${p2pPort}`);
@@ -322,6 +332,8 @@ export class Host extends Home {
322
332
  this.dbErrorConnection.info();
323
333
  this.dbEventConnection = new ActiveDSConnect(db.url + "/" + db.event);
324
334
  this.dbEventConnection.info();
335
+ void this.loadResentGuard();
336
+ void this.loadStoredMarker();
325
337
  this.api = App();
326
338
  this.hybridRegistry = new LedgerHybridRegistry(this.dbConnection);
327
339
  const hybridApi = new HybridApi({
@@ -609,13 +621,57 @@ export class Host extends Home {
609
621
  },
610
622
  };
611
623
  }
624
+ static ipcSerialization() {
625
+ return ActiveOptions.get("ipcSerialization", "json") === "advanced"
626
+ ? "advanced"
627
+ : "json";
628
+ }
629
+ batchBroadcasts(child) {
630
+ if (ActiveOptions.get("ipcBatching", false) !== true)
631
+ return;
632
+ const rawSend = child.send.bind(child);
633
+ let queued = [];
634
+ let scheduled = false;
635
+ const flush = () => {
636
+ scheduled = false;
637
+ if (!queued.length)
638
+ return true;
639
+ const out = queued;
640
+ queued = [];
641
+ try {
642
+ return out.length === 1
643
+ ? rawSend(out[0])
644
+ : rawSend({ type: "batch", data: { messages: out } });
645
+ }
646
+ catch (error) {
647
+ ActiveLogger.warn(error, "Worker message not sent");
648
+ return false;
649
+ }
650
+ };
651
+ child.send = (message, ...rest) => {
652
+ if (message && message.type === "broadcast" && !rest.length) {
653
+ queued.push(message);
654
+ if (!scheduled) {
655
+ scheduled = true;
656
+ setImmediate(flush);
657
+ }
658
+ return true;
659
+ }
660
+ flush();
661
+ return rawSend(message, ...rest);
662
+ };
663
+ }
612
664
  createProcessor(cpuTotal) {
613
665
  const pFork = fork(`${__dirname}/process.js`, [], {
614
666
  cwd: process.cwd(),
615
667
  execArgv: workerExecArgv(),
616
668
  stdio: "inherit",
669
+ serialization: Host.ipcSerialization(),
617
670
  });
618
671
  pFork.stop = false;
672
+ if (Host.ipcSerialization() === "advanced")
673
+ withJsonFallback(pFork);
674
+ this.batchBroadcasts(pFork);
619
675
  let unloadHandled = false;
620
676
  const unloadProcessorSafely = (...error) => {
621
677
  if (unloadHandled || !this.listenSocket) {
@@ -690,8 +746,12 @@ export class Host extends Home {
690
746
  recordFailed(this.dbEventConnection, this.dbConnection, pending.entry, ActiveOptions.get("build", 0));
691
747
  break;
692
748
  case "commited":
749
+ if (m.data.broadcast) {
750
+ this.broadcast(m.data.umid, false);
751
+ }
693
752
  if (!pending)
694
753
  return;
754
+ this.noteStored(pending.entry.$umid, pending.entry.$tx?.$expire);
695
755
  pending.resolve({
696
756
  status: 200,
697
757
  data: { ...pending.entry, ...m.data.entry },
@@ -1026,6 +1086,217 @@ export class Host extends Home {
1026
1086
  }
1027
1087
  return robin;
1028
1088
  }
1089
+ storedByMinute = new Map();
1090
+ storedCount = 0;
1091
+ storedUnknown = new Set();
1092
+ storedUnknownAllUntil = 0;
1093
+ storedColdUntil = Date.now() + STORED_WINDOW_MS;
1094
+ storedSwept = 0;
1095
+ storedMarkerLoaded = false;
1096
+ storedMarkerLoading;
1097
+ storedMarkerTried = 0;
1098
+ storedMarkerWritten = new Set();
1099
+ static minuteOf(ms) {
1100
+ return Math.floor(ms / 60000);
1101
+ }
1102
+ noteStored(umid, expire) {
1103
+ if (STORED_WINDOW_MS <= 0 || !expire)
1104
+ return;
1105
+ const until = new Date(expire).getTime();
1106
+ const now = Date.now();
1107
+ if (!Number.isFinite(until) || until <= now)
1108
+ return;
1109
+ const minute = Host.minuteOf(until);
1110
+ if (until > now + STORED_WINDOW_MS || this.storedCount >= STORED_MAX) {
1111
+ this.markUnknown(minute, until);
1112
+ return;
1113
+ }
1114
+ if (now - this.storedSwept > 1000) {
1115
+ this.storedSwept = now;
1116
+ const current = Host.minuteOf(now);
1117
+ for (const [m, set] of this.storedByMinute) {
1118
+ if (m < current) {
1119
+ this.storedCount -= set.size;
1120
+ this.storedByMinute.delete(m);
1121
+ }
1122
+ }
1123
+ for (const m of this.storedUnknown)
1124
+ if (m < current)
1125
+ this.storedUnknown.delete(m);
1126
+ }
1127
+ let set = this.storedByMinute.get(minute);
1128
+ if (!set)
1129
+ this.storedByMinute.set(minute, (set = new Set()));
1130
+ if (!set.has(umid)) {
1131
+ set.add(umid);
1132
+ this.storedCount++;
1133
+ }
1134
+ }
1135
+ markUnknown(minute, until) {
1136
+ this.writeStoredMarker(minute);
1137
+ if (this.storedUnknown.size >= STORED_UNKNOWN_MAX && !this.storedUnknown.has(minute)) {
1138
+ this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, until);
1139
+ return;
1140
+ }
1141
+ this.storedUnknown.add(minute);
1142
+ }
1143
+ writeStoredMarker(minute) {
1144
+ if (this.storedMarkerWritten.has(minute))
1145
+ return;
1146
+ this.storedMarkerWritten.add(minute);
1147
+ this.dbEventConnection
1148
+ .post({ _id: `storedfar:${minute}`, minute, until: (minute + 1) * 60000 })
1149
+ .catch((error) => {
1150
+ this.storedMarkerWritten.delete(minute);
1151
+ ActiveLogger.error({ error }, "Could not record a far-off $expire commit; a restart would forget it");
1152
+ });
1153
+ }
1154
+ loadStoredMarker() {
1155
+ if (STORED_WINDOW_MS <= 0 || this.storedMarkerLoaded)
1156
+ return Promise.resolve();
1157
+ if (this.storedMarkerLoading)
1158
+ return this.storedMarkerLoading;
1159
+ if (Date.now() - this.storedMarkerTried < 5000)
1160
+ return Promise.resolve();
1161
+ this.storedMarkerTried = Date.now();
1162
+ this.storedMarkerLoading = (async () => {
1163
+ try {
1164
+ const list = await this.dbEventConnection.allDocs({ startkey: "storedfar:", endkey: "storedfar:x", include_docs: true });
1165
+ if (!list || !Array.isArray(list.rows))
1166
+ throw new Error("no answer");
1167
+ const now = Date.now();
1168
+ for (const row of list.rows) {
1169
+ const doc = row?.doc ?? row;
1170
+ if (!doc?._id)
1171
+ continue;
1172
+ if (Number(doc.until) > now) {
1173
+ this.storedMarkerWritten.add(Number(doc.minute));
1174
+ if (this.storedUnknown.size < STORED_UNKNOWN_MAX)
1175
+ this.storedUnknown.add(Number(doc.minute));
1176
+ else
1177
+ this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, Number(doc.until));
1178
+ }
1179
+ else
1180
+ await this.dbEventConnection.purge(doc).catch(() => undefined);
1181
+ }
1182
+ this.storedMarkerLoaded = true;
1183
+ }
1184
+ catch (error) {
1185
+ ActiveLogger.warn(`Could not read the far-off $expire rows, asking the storage meanwhile: ${error?.message ?? error}`);
1186
+ }
1187
+ finally {
1188
+ this.storedMarkerLoading = undefined;
1189
+ }
1190
+ })();
1191
+ return this.storedMarkerLoading;
1192
+ }
1193
+ mayBeStored(umid, expire) {
1194
+ if (STORED_WINDOW_MS <= 0)
1195
+ return true;
1196
+ if (!this.storedMarkerLoaded) {
1197
+ void this.loadStoredMarker();
1198
+ return true;
1199
+ }
1200
+ const now = Date.now();
1201
+ if (now < this.storedColdUntil || now < this.storedUnknownAllUntil)
1202
+ return true;
1203
+ if (!(expire <= now + STORED_WINDOW_MS))
1204
+ return true;
1205
+ const minute = Host.minuteOf(expire);
1206
+ if (this.storedUnknown.has(minute))
1207
+ return true;
1208
+ return this.storedByMinute.get(minute)?.has(umid) === true;
1209
+ }
1210
+ resentGuardUntil = 0;
1211
+ resentInFlight = new Map();
1212
+ resentTimer;
1213
+ resentLoaded = false;
1214
+ resentLoading;
1215
+ rememberResent(replaced, replacement, expire) {
1216
+ const until = expire ? new Date(expire).getTime() : NaN;
1217
+ if (!Number.isFinite(until) || until <= Date.now())
1218
+ return;
1219
+ this.resentGuardUntil = Math.max(this.resentGuardUntil, until);
1220
+ this.resentInFlight.set(replaced, replacement);
1221
+ this.dbEventConnection
1222
+ .post({ _id: `resent:${replaced}`, to: replacement, until })
1223
+ .then(() => void this.resentInFlight.delete(replaced), (error) => {
1224
+ ActiveLogger.error({ replaced, error }, "Could not record a resent transaction; refusing a repost from memory only");
1225
+ setTimeout(() => this.resentInFlight.delete(replaced), Math.max(0, until - Date.now())).unref();
1226
+ });
1227
+ this.startResentHousekeeping();
1228
+ }
1229
+ async resentAs(umid) {
1230
+ if (this.resentLoading && !this.resentLoaded)
1231
+ await this.waitForResentList();
1232
+ if (Date.now() >= this.resentGuardUntil)
1233
+ return undefined;
1234
+ const inFlight = this.resentInFlight.get(umid);
1235
+ if (inFlight)
1236
+ return inFlight;
1237
+ const row = await this.dbEventConnection.get(`resent:${umid}`).catch(() => null);
1238
+ if (row === null || row === undefined)
1239
+ throw new Error("could not check whether the transaction was resent");
1240
+ return typeof row.to === "string" && Number(row.until) > Date.now() ? row.to : undefined;
1241
+ }
1242
+ loadResentGuard() {
1243
+ this.resentLoading = this.refreshResentGuard(false).then(() => undefined);
1244
+ return this.resentLoading;
1245
+ }
1246
+ resentWaiting;
1247
+ waitForResentList() {
1248
+ this.resentWaiting ??= (async () => {
1249
+ try {
1250
+ await this.resentLoading;
1251
+ for (let attempt = 0; !this.resentLoaded && attempt < RESENT_LOAD_WAIT_MS / 500; attempt++) {
1252
+ await new Promise((resolve) => setTimeout(resolve, 500).unref());
1253
+ await this.refreshResentGuard(false);
1254
+ }
1255
+ }
1256
+ finally {
1257
+ this.resentWaiting = undefined;
1258
+ }
1259
+ if (!this.resentLoaded)
1260
+ throw new Error("the list of resent transactions has not been read yet");
1261
+ })();
1262
+ return this.resentWaiting;
1263
+ }
1264
+ async refreshResentGuard(retryLater = true) {
1265
+ let latest = 0;
1266
+ try {
1267
+ const list = await this.dbEventConnection.allDocs({ startkey: "resent:", endkey: "resent:x", include_docs: true });
1268
+ if (!list || !Array.isArray(list.rows))
1269
+ throw new Error("no answer");
1270
+ const now = Date.now();
1271
+ for (const row of list.rows) {
1272
+ const doc = row?.doc ?? row;
1273
+ if (!doc?._id)
1274
+ continue;
1275
+ if (Number(doc.until) > now)
1276
+ latest = Math.max(latest, Number(doc.until));
1277
+ else
1278
+ await this.dbEventConnection.purge(doc).catch(() => undefined);
1279
+ }
1280
+ }
1281
+ catch (error) {
1282
+ if (retryLater) {
1283
+ ActiveLogger.warn(`Could not read the resent transactions, trying again in 30s: ${error?.message ?? error}`);
1284
+ setTimeout(() => void this.refreshResentGuard(), 30_000).unref();
1285
+ }
1286
+ return false;
1287
+ }
1288
+ this.resentGuardUntil = Math.max(latest, this.resentInFlight.size ? this.resentGuardUntil : 0);
1289
+ this.resentLoaded = true;
1290
+ if (latest > 0)
1291
+ this.startResentHousekeeping();
1292
+ return true;
1293
+ }
1294
+ startResentHousekeeping() {
1295
+ if (this.resentTimer)
1296
+ return;
1297
+ this.resentTimer = setInterval(() => void this.refreshResentGuard(), 60 * 60 * 1000);
1298
+ this.resentTimer.unref();
1299
+ }
1029
1300
  release(umid, owner) {
1030
1301
  if (owner && this.processPending[umid] && this.processPending[umid].entry !== owner) {
1031
1302
  ActiveLogger.warn(umid, "Not releasing, pending entry belongs to another submission");
@@ -1052,20 +1323,49 @@ export class Host extends Home {
1052
1323
  this.destroy(umid);
1053
1324
  }
1054
1325
  }, hold);
1055
- if (QUEUE_RECHECK_MS > 0) {
1056
- setTimeout(() => this.processQueue(), QUEUE_RECHECK_MS);
1057
- }
1058
- else {
1059
- setImmediate(() => this.processQueue());
1060
- }
1326
+ this.scheduleQueueRecheck(QUEUE_RECHECK_MS);
1061
1327
  }
1062
1328
  else {
1063
1329
  ActiveLogger.warn(umid, "Trying to release unknown umid");
1064
1330
  }
1065
1331
  }
1066
1332
  processingBLQ = false;
1333
+ lastQueueScan = 0;
1334
+ queueRecheckPending = false;
1335
+ scheduleQueueRecheck(ms) {
1336
+ if (this.queueRecheckPending)
1337
+ return;
1338
+ this.queueRecheckPending = true;
1339
+ const run = () => {
1340
+ this.queueRecheckPending = false;
1341
+ this.processQueue();
1342
+ };
1343
+ if (ms > 0) {
1344
+ setTimeout(run, ms);
1345
+ }
1346
+ else {
1347
+ setImmediate(run);
1348
+ }
1349
+ }
1067
1350
  processQueue(next, internal = false) {
1068
1351
  if (!this.processingBLQ) {
1352
+ const queued = this.busyLocksQueue.internal.length + this.busyLocksQueue.external.length;
1353
+ if (!queued) {
1354
+ if (next)
1355
+ this.hold(next);
1356
+ return;
1357
+ }
1358
+ const since = Date.now() - this.lastQueueScan;
1359
+ if (QUEUE_SCAN_MIN_MS > 0 && since < QUEUE_SCAN_MIN_MS) {
1360
+ if (next) {
1361
+ this.hold(next);
1362
+ }
1363
+ else {
1364
+ this.scheduleQueueRecheck(QUEUE_SCAN_MIN_MS - since);
1365
+ }
1366
+ return;
1367
+ }
1368
+ this.lastQueueScan = Date.now();
1069
1369
  this.processingBLQ = true;
1070
1370
  const checked = new Set();
1071
1371
  if (this.busyLocksQueue.internal.length) {