@activeledger/activenetwork 5.3.0 → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,6 +50,7 @@ const neighbour_1 = require("./neighbour");
50
50
  const utils_1 = require("./utils");
51
51
  const index_1 = require("./index");
52
52
  const locker_1 = require("./locker");
53
+ const ipc_1 = require("./ipc");
53
54
  const cpus_1 = require("./cpus");
54
55
  const workercount_1 = require("./workercount");
55
56
  const workerheap_1 = require("./workerheap");
@@ -64,10 +65,15 @@ const zlib_1 = require("zlib");
64
65
  const GZIP_MIN_RESPONSE_BYTES = 16384;
65
66
  const gunzipAsync = (0, util_1.promisify)(zlib_1.gunzip);
66
67
  const TIMER_QUEUE_INTERVAL = 2 * 1000;
68
+ const RESENT_LOAD_WAIT_MS = 5 * 1000;
69
+ const STORED_WINDOW_MS = Number(process.env.AL_STORED_WINDOW_MS ?? 5 * 60 * 1000);
70
+ const STORED_MAX = Number(process.env.AL_STORED_MAX ?? 200000);
71
+ const STORED_UNKNOWN_MAX = Number(process.env.AL_STORED_UNKNOWN_MAX ?? 10000);
67
72
  const GRACEFUL_PROC_SHUTDOWN = 7 * 60 * 1000;
68
73
  const KILL_PROC_SHUTDOWN = 2.5 * 1000;
69
74
  const QUEUE_MAX_WAIT_MS = Number(process.env.AL_QUEUE_MAX_WAIT_MS || 7_000);
70
75
  const QUEUE_RECHECK_MS = Number(process.env.AL_QUEUE_RECHECK_MS ?? 50);
76
+ const QUEUE_SCAN_MIN_MS = Number(process.env.AL_QUEUE_SCAN_MIN_MS ?? 20);
71
77
  const MAX_RETRIES = 35;
72
78
  function peekBytes(chunks, n) {
73
79
  if (chunks.length > 0 && chunks[0].length >= n) {
@@ -301,51 +307,55 @@ class Host extends home_1.Home {
301
307
  }
302
308
  });
303
309
  }
310
+ static p2pServerWanted() {
311
+ return activeoptions_1.ActiveOptions.get("p2pStreamServer", false);
312
+ }
313
+ p2pConnection(socket) {
314
+ socket.setNoDelay(true);
315
+ activelogger_1.ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
316
+ let chunks = [];
317
+ let bufferLength = 0;
318
+ socket.on("data", async (data) => {
319
+ chunks.push(data);
320
+ bufferLength += data.length;
321
+ while (bufferLength >= 44) {
322
+ const header = peekBytes(chunks, 44);
323
+ const length = header.readUInt32BE(40);
324
+ if (bufferLength >= 44 + length) {
325
+ const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
326
+ const senderRef = fullBuffer.slice(0, 40).toString().trim();
327
+ const item = fullBuffer.slice(44, 44 + length);
328
+ const remaining = fullBuffer.slice(44 + length);
329
+ chunks = remaining.length > 0 ? [remaining] : [];
330
+ bufferLength = remaining.length;
331
+ let tx;
332
+ try {
333
+ tx = await activeoptions_1.ActiveClone.deserialize(item);
334
+ }
335
+ catch (error) {
336
+ activelogger_1.ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
337
+ continue;
338
+ }
339
+ this.processEndpoints({
340
+ headers: { "X-Activeledger": senderRef },
341
+ method: "POST",
342
+ url: "/a/init",
343
+ connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
344
+ }, null, tx, senderRef, true);
345
+ }
346
+ else {
347
+ break;
348
+ }
349
+ }
350
+ });
351
+ socket.on("error", (err) => {
352
+ activelogger_1.ActiveLogger.error(err, "P2P socket error");
353
+ });
354
+ }
304
355
  constructor() {
305
356
  super();
306
- if (activeoptions_1.ActiveOptions.get("p2pStreamServer", false)) {
307
- this.p2pServer = net.createServer((socket) => {
308
- socket.setNoDelay(true);
309
- activelogger_1.ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
310
- let chunks = [];
311
- let bufferLength = 0;
312
- socket.on("data", async (data) => {
313
- chunks.push(data);
314
- bufferLength += data.length;
315
- while (bufferLength >= 44) {
316
- const header = peekBytes(chunks, 44);
317
- const length = header.readUInt32BE(40);
318
- if (bufferLength >= 44 + length) {
319
- const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
320
- const senderRef = fullBuffer.slice(0, 40).toString().trim();
321
- const item = fullBuffer.slice(44, 44 + length);
322
- const remaining = fullBuffer.slice(44 + length);
323
- chunks = remaining.length > 0 ? [remaining] : [];
324
- bufferLength = remaining.length;
325
- let tx;
326
- try {
327
- tx = await activeoptions_1.ActiveClone.deserialize(item);
328
- }
329
- catch (error) {
330
- activelogger_1.ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
331
- continue;
332
- }
333
- this.processEndpoints({
334
- headers: { "X-Activeledger": senderRef },
335
- method: "POST",
336
- url: "/a/init",
337
- connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
338
- }, null, tx, senderRef, true);
339
- }
340
- else {
341
- break;
342
- }
343
- }
344
- });
345
- socket.on("error", (err) => {
346
- activelogger_1.ActiveLogger.error(err, "P2P socket error");
347
- });
348
- });
357
+ if (Host.p2pServerWanted()) {
358
+ this.p2pServer = net.createServer((socket) => this.p2pConnection(socket));
349
359
  const p2pPort = parseInt(utils_1.ActiveInterfaces.getBindingDetails("port")) + 1;
350
360
  this.p2pServer.listen(p2pPort, () => {
351
361
  activelogger_1.ActiveLogger.info(`P2P TCP server listening on port ${p2pPort}`);
@@ -358,6 +368,8 @@ class Host extends home_1.Home {
358
368
  this.dbErrorConnection.info();
359
369
  this.dbEventConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.event);
360
370
  this.dbEventConnection.info();
371
+ void this.loadResentGuard();
372
+ void this.loadStoredMarker();
361
373
  this.api = (0, uWebSockets_js_1.App)();
362
374
  this.hybridRegistry = new hybridApi_1.LedgerHybridRegistry(this.dbConnection);
363
375
  const hybridApi = new hybridApi_1.HybridApi({
@@ -645,13 +657,57 @@ class Host extends home_1.Home {
645
657
  },
646
658
  };
647
659
  }
660
+ static ipcSerialization() {
661
+ return activeoptions_1.ActiveOptions.get("ipcSerialization", "json") === "advanced"
662
+ ? "advanced"
663
+ : "json";
664
+ }
665
+ batchBroadcasts(child) {
666
+ if (activeoptions_1.ActiveOptions.get("ipcBatching", false) !== true)
667
+ return;
668
+ const rawSend = child.send.bind(child);
669
+ let queued = [];
670
+ let scheduled = false;
671
+ const flush = () => {
672
+ scheduled = false;
673
+ if (!queued.length)
674
+ return true;
675
+ const out = queued;
676
+ queued = [];
677
+ try {
678
+ return out.length === 1
679
+ ? rawSend(out[0])
680
+ : rawSend({ type: "batch", data: { messages: out } });
681
+ }
682
+ catch (error) {
683
+ activelogger_1.ActiveLogger.warn(error, "Worker message not sent");
684
+ return false;
685
+ }
686
+ };
687
+ child.send = (message, ...rest) => {
688
+ if (message && message.type === "broadcast" && !rest.length) {
689
+ queued.push(message);
690
+ if (!scheduled) {
691
+ scheduled = true;
692
+ setImmediate(flush);
693
+ }
694
+ return true;
695
+ }
696
+ flush();
697
+ return rawSend(message, ...rest);
698
+ };
699
+ }
648
700
  createProcessor(cpuTotal) {
649
701
  const pFork = (0, child_process_1.fork)(`${__dirname}/process.js`, [], {
650
702
  cwd: process.cwd(),
651
703
  execArgv: (0, workerheap_1.workerExecArgv)(),
652
704
  stdio: "inherit",
705
+ serialization: Host.ipcSerialization(),
653
706
  });
654
707
  pFork.stop = false;
708
+ if (Host.ipcSerialization() === "advanced")
709
+ (0, ipc_1.withJsonFallback)(pFork);
710
+ this.batchBroadcasts(pFork);
655
711
  let unloadHandled = false;
656
712
  const unloadProcessorSafely = (...error) => {
657
713
  if (unloadHandled || !this.listenSocket) {
@@ -726,8 +782,12 @@ class Host extends home_1.Home {
726
782
  (0, hybridApi_1.recordFailed)(this.dbEventConnection, this.dbConnection, pending.entry, activeoptions_1.ActiveOptions.get("build", 0));
727
783
  break;
728
784
  case "commited":
785
+ if (m.data.broadcast) {
786
+ this.broadcast(m.data.umid, false);
787
+ }
729
788
  if (!pending)
730
789
  return;
790
+ this.noteStored(pending.entry.$umid, pending.entry.$tx?.$expire);
731
791
  pending.resolve({
732
792
  status: 200,
733
793
  data: { ...pending.entry, ...m.data.entry },
@@ -1062,6 +1122,217 @@ class Host extends home_1.Home {
1062
1122
  }
1063
1123
  return robin;
1064
1124
  }
1125
+ storedByMinute = new Map();
1126
+ storedCount = 0;
1127
+ storedUnknown = new Set();
1128
+ storedUnknownAllUntil = 0;
1129
+ storedColdUntil = Date.now() + STORED_WINDOW_MS;
1130
+ storedSwept = 0;
1131
+ storedMarkerLoaded = false;
1132
+ storedMarkerLoading;
1133
+ storedMarkerTried = 0;
1134
+ storedMarkerWritten = new Set();
1135
+ static minuteOf(ms) {
1136
+ return Math.floor(ms / 60000);
1137
+ }
1138
+ noteStored(umid, expire) {
1139
+ if (STORED_WINDOW_MS <= 0 || !expire)
1140
+ return;
1141
+ const until = new Date(expire).getTime();
1142
+ const now = Date.now();
1143
+ if (!Number.isFinite(until) || until <= now)
1144
+ return;
1145
+ const minute = Host.minuteOf(until);
1146
+ if (until > now + STORED_WINDOW_MS || this.storedCount >= STORED_MAX) {
1147
+ this.markUnknown(minute, until);
1148
+ return;
1149
+ }
1150
+ if (now - this.storedSwept > 1000) {
1151
+ this.storedSwept = now;
1152
+ const current = Host.minuteOf(now);
1153
+ for (const [m, set] of this.storedByMinute) {
1154
+ if (m < current) {
1155
+ this.storedCount -= set.size;
1156
+ this.storedByMinute.delete(m);
1157
+ }
1158
+ }
1159
+ for (const m of this.storedUnknown)
1160
+ if (m < current)
1161
+ this.storedUnknown.delete(m);
1162
+ }
1163
+ let set = this.storedByMinute.get(minute);
1164
+ if (!set)
1165
+ this.storedByMinute.set(minute, (set = new Set()));
1166
+ if (!set.has(umid)) {
1167
+ set.add(umid);
1168
+ this.storedCount++;
1169
+ }
1170
+ }
1171
+ markUnknown(minute, until) {
1172
+ this.writeStoredMarker(minute);
1173
+ if (this.storedUnknown.size >= STORED_UNKNOWN_MAX && !this.storedUnknown.has(minute)) {
1174
+ this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, until);
1175
+ return;
1176
+ }
1177
+ this.storedUnknown.add(minute);
1178
+ }
1179
+ writeStoredMarker(minute) {
1180
+ if (this.storedMarkerWritten.has(minute))
1181
+ return;
1182
+ this.storedMarkerWritten.add(minute);
1183
+ this.dbEventConnection
1184
+ .post({ _id: `storedfar:${minute}`, minute, until: (minute + 1) * 60000 })
1185
+ .catch((error) => {
1186
+ this.storedMarkerWritten.delete(minute);
1187
+ activelogger_1.ActiveLogger.error({ error }, "Could not record a far-off $expire commit; a restart would forget it");
1188
+ });
1189
+ }
1190
+ loadStoredMarker() {
1191
+ if (STORED_WINDOW_MS <= 0 || this.storedMarkerLoaded)
1192
+ return Promise.resolve();
1193
+ if (this.storedMarkerLoading)
1194
+ return this.storedMarkerLoading;
1195
+ if (Date.now() - this.storedMarkerTried < 5000)
1196
+ return Promise.resolve();
1197
+ this.storedMarkerTried = Date.now();
1198
+ this.storedMarkerLoading = (async () => {
1199
+ try {
1200
+ const list = await this.dbEventConnection.allDocs({ startkey: "storedfar:", endkey: "storedfar:x", include_docs: true });
1201
+ if (!list || !Array.isArray(list.rows))
1202
+ throw new Error("no answer");
1203
+ const now = Date.now();
1204
+ for (const row of list.rows) {
1205
+ const doc = row?.doc ?? row;
1206
+ if (!doc?._id)
1207
+ continue;
1208
+ if (Number(doc.until) > now) {
1209
+ this.storedMarkerWritten.add(Number(doc.minute));
1210
+ if (this.storedUnknown.size < STORED_UNKNOWN_MAX)
1211
+ this.storedUnknown.add(Number(doc.minute));
1212
+ else
1213
+ this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, Number(doc.until));
1214
+ }
1215
+ else
1216
+ await this.dbEventConnection.purge(doc).catch(() => undefined);
1217
+ }
1218
+ this.storedMarkerLoaded = true;
1219
+ }
1220
+ catch (error) {
1221
+ activelogger_1.ActiveLogger.warn(`Could not read the far-off $expire rows, asking the storage meanwhile: ${error?.message ?? error}`);
1222
+ }
1223
+ finally {
1224
+ this.storedMarkerLoading = undefined;
1225
+ }
1226
+ })();
1227
+ return this.storedMarkerLoading;
1228
+ }
1229
+ mayBeStored(umid, expire) {
1230
+ if (STORED_WINDOW_MS <= 0)
1231
+ return true;
1232
+ if (!this.storedMarkerLoaded) {
1233
+ void this.loadStoredMarker();
1234
+ return true;
1235
+ }
1236
+ const now = Date.now();
1237
+ if (now < this.storedColdUntil || now < this.storedUnknownAllUntil)
1238
+ return true;
1239
+ if (!(expire <= now + STORED_WINDOW_MS))
1240
+ return true;
1241
+ const minute = Host.minuteOf(expire);
1242
+ if (this.storedUnknown.has(minute))
1243
+ return true;
1244
+ return this.storedByMinute.get(minute)?.has(umid) === true;
1245
+ }
1246
+ resentGuardUntil = 0;
1247
+ resentInFlight = new Map();
1248
+ resentTimer;
1249
+ resentLoaded = false;
1250
+ resentLoading;
1251
+ rememberResent(replaced, replacement, expire) {
1252
+ const until = expire ? new Date(expire).getTime() : NaN;
1253
+ if (!Number.isFinite(until) || until <= Date.now())
1254
+ return;
1255
+ this.resentGuardUntil = Math.max(this.resentGuardUntil, until);
1256
+ this.resentInFlight.set(replaced, replacement);
1257
+ this.dbEventConnection
1258
+ .post({ _id: `resent:${replaced}`, to: replacement, until })
1259
+ .then(() => void this.resentInFlight.delete(replaced), (error) => {
1260
+ activelogger_1.ActiveLogger.error({ replaced, error }, "Could not record a resent transaction; refusing a repost from memory only");
1261
+ setTimeout(() => this.resentInFlight.delete(replaced), Math.max(0, until - Date.now())).unref();
1262
+ });
1263
+ this.startResentHousekeeping();
1264
+ }
1265
+ async resentAs(umid) {
1266
+ if (this.resentLoading && !this.resentLoaded)
1267
+ await this.waitForResentList();
1268
+ if (Date.now() >= this.resentGuardUntil)
1269
+ return undefined;
1270
+ const inFlight = this.resentInFlight.get(umid);
1271
+ if (inFlight)
1272
+ return inFlight;
1273
+ const row = await this.dbEventConnection.get(`resent:${umid}`).catch(() => null);
1274
+ if (row === null || row === undefined)
1275
+ throw new Error("could not check whether the transaction was resent");
1276
+ return typeof row.to === "string" && Number(row.until) > Date.now() ? row.to : undefined;
1277
+ }
1278
+ loadResentGuard() {
1279
+ this.resentLoading = this.refreshResentGuard(false).then(() => undefined);
1280
+ return this.resentLoading;
1281
+ }
1282
+ resentWaiting;
1283
+ waitForResentList() {
1284
+ this.resentWaiting ??= (async () => {
1285
+ try {
1286
+ await this.resentLoading;
1287
+ for (let attempt = 0; !this.resentLoaded && attempt < RESENT_LOAD_WAIT_MS / 500; attempt++) {
1288
+ await new Promise((resolve) => setTimeout(resolve, 500).unref());
1289
+ await this.refreshResentGuard(false);
1290
+ }
1291
+ }
1292
+ finally {
1293
+ this.resentWaiting = undefined;
1294
+ }
1295
+ if (!this.resentLoaded)
1296
+ throw new Error("the list of resent transactions has not been read yet");
1297
+ })();
1298
+ return this.resentWaiting;
1299
+ }
1300
+ async refreshResentGuard(retryLater = true) {
1301
+ let latest = 0;
1302
+ try {
1303
+ const list = await this.dbEventConnection.allDocs({ startkey: "resent:", endkey: "resent:x", include_docs: true });
1304
+ if (!list || !Array.isArray(list.rows))
1305
+ throw new Error("no answer");
1306
+ const now = Date.now();
1307
+ for (const row of list.rows) {
1308
+ const doc = row?.doc ?? row;
1309
+ if (!doc?._id)
1310
+ continue;
1311
+ if (Number(doc.until) > now)
1312
+ latest = Math.max(latest, Number(doc.until));
1313
+ else
1314
+ await this.dbEventConnection.purge(doc).catch(() => undefined);
1315
+ }
1316
+ }
1317
+ catch (error) {
1318
+ if (retryLater) {
1319
+ activelogger_1.ActiveLogger.warn(`Could not read the resent transactions, trying again in 30s: ${error?.message ?? error}`);
1320
+ setTimeout(() => void this.refreshResentGuard(), 30_000).unref();
1321
+ }
1322
+ return false;
1323
+ }
1324
+ this.resentGuardUntil = Math.max(latest, this.resentInFlight.size ? this.resentGuardUntil : 0);
1325
+ this.resentLoaded = true;
1326
+ if (latest > 0)
1327
+ this.startResentHousekeeping();
1328
+ return true;
1329
+ }
1330
+ startResentHousekeeping() {
1331
+ if (this.resentTimer)
1332
+ return;
1333
+ this.resentTimer = setInterval(() => void this.refreshResentGuard(), 60 * 60 * 1000);
1334
+ this.resentTimer.unref();
1335
+ }
1065
1336
  release(umid, owner) {
1066
1337
  if (owner && this.processPending[umid] && this.processPending[umid].entry !== owner) {
1067
1338
  activelogger_1.ActiveLogger.warn(umid, "Not releasing, pending entry belongs to another submission");
@@ -1088,20 +1359,49 @@ class Host extends home_1.Home {
1088
1359
  this.destroy(umid);
1089
1360
  }
1090
1361
  }, hold);
1091
- if (QUEUE_RECHECK_MS > 0) {
1092
- setTimeout(() => this.processQueue(), QUEUE_RECHECK_MS);
1093
- }
1094
- else {
1095
- setImmediate(() => this.processQueue());
1096
- }
1362
+ this.scheduleQueueRecheck(QUEUE_RECHECK_MS);
1097
1363
  }
1098
1364
  else {
1099
1365
  activelogger_1.ActiveLogger.warn(umid, "Trying to release unknown umid");
1100
1366
  }
1101
1367
  }
1102
1368
  processingBLQ = false;
1369
+ lastQueueScan = 0;
1370
+ queueRecheckPending = false;
1371
+ scheduleQueueRecheck(ms) {
1372
+ if (this.queueRecheckPending)
1373
+ return;
1374
+ this.queueRecheckPending = true;
1375
+ const run = () => {
1376
+ this.queueRecheckPending = false;
1377
+ this.processQueue();
1378
+ };
1379
+ if (ms > 0) {
1380
+ setTimeout(run, ms);
1381
+ }
1382
+ else {
1383
+ setImmediate(run);
1384
+ }
1385
+ }
1103
1386
  processQueue(next, internal = false) {
1104
1387
  if (!this.processingBLQ) {
1388
+ const queued = this.busyLocksQueue.internal.length + this.busyLocksQueue.external.length;
1389
+ if (!queued) {
1390
+ if (next)
1391
+ this.hold(next);
1392
+ return;
1393
+ }
1394
+ const since = Date.now() - this.lastQueueScan;
1395
+ if (QUEUE_SCAN_MIN_MS > 0 && since < QUEUE_SCAN_MIN_MS) {
1396
+ if (next) {
1397
+ this.hold(next);
1398
+ }
1399
+ else {
1400
+ this.scheduleQueueRecheck(QUEUE_SCAN_MIN_MS - since);
1401
+ }
1402
+ return;
1403
+ }
1404
+ this.lastQueueScan = Date.now();
1105
1405
  this.processingBLQ = true;
1106
1406
  const checked = new Set();
1107
1407
  if (this.busyLocksQueue.internal.length) {