@activeledger/activenetwork 5.3.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/es/network/endpoints.d.ts +1 -0
- package/es/network/endpoints.d.ts.map +1 -1
- package/es/network/endpoints.js +32 -4
- package/es/network/endpoints.js.map +1 -1
- package/es/network/host.d.ts +35 -0
- package/es/network/host.d.ts.map +1 -1
- package/es/network/host.js +349 -49
- package/es/network/host.js.map +1 -1
- package/es/network/ipc.d.ts +6 -0
- package/es/network/ipc.d.ts.map +1 -0
- package/es/network/ipc.js +18 -0
- package/es/network/ipc.js.map +1 -0
- package/es/network/locker.d.ts.map +1 -1
- package/es/network/locker.js +21 -19
- package/es/network/locker.js.map +1 -1
- package/es/network/process.js +50 -10
- package/es/network/process.js.map +1 -1
- package/lib/network/endpoints.d.ts +1 -0
- package/lib/network/endpoints.d.ts.map +1 -1
- package/lib/network/endpoints.js +32 -4
- package/lib/network/endpoints.js.map +1 -1
- package/lib/network/host.d.ts +35 -0
- package/lib/network/host.d.ts.map +1 -1
- package/lib/network/host.js +349 -49
- package/lib/network/host.js.map +1 -1
- package/lib/network/ipc.d.ts +6 -0
- package/lib/network/ipc.d.ts.map +1 -0
- package/lib/network/ipc.js +24 -0
- package/lib/network/ipc.js.map +1 -0
- package/lib/network/locker.d.ts.map +1 -1
- package/lib/network/locker.js +21 -19
- package/lib/network/locker.js.map +1 -1
- package/lib/network/process.js +50 -10
- package/lib/network/process.js.map +1 -1
- package/package.json +9 -9
package/lib/network/host.js
CHANGED
|
@@ -50,6 +50,7 @@ const neighbour_1 = require("./neighbour");
|
|
|
50
50
|
const utils_1 = require("./utils");
|
|
51
51
|
const index_1 = require("./index");
|
|
52
52
|
const locker_1 = require("./locker");
|
|
53
|
+
const ipc_1 = require("./ipc");
|
|
53
54
|
const cpus_1 = require("./cpus");
|
|
54
55
|
const workercount_1 = require("./workercount");
|
|
55
56
|
const workerheap_1 = require("./workerheap");
|
|
@@ -64,10 +65,15 @@ const zlib_1 = require("zlib");
|
|
|
64
65
|
const GZIP_MIN_RESPONSE_BYTES = 16384;
|
|
65
66
|
const gunzipAsync = (0, util_1.promisify)(zlib_1.gunzip);
|
|
66
67
|
const TIMER_QUEUE_INTERVAL = 2 * 1000;
|
|
68
|
+
const RESENT_LOAD_WAIT_MS = 5 * 1000;
|
|
69
|
+
const STORED_WINDOW_MS = Number(process.env.AL_STORED_WINDOW_MS ?? 5 * 60 * 1000);
|
|
70
|
+
const STORED_MAX = Number(process.env.AL_STORED_MAX ?? 200000);
|
|
71
|
+
const STORED_UNKNOWN_MAX = Number(process.env.AL_STORED_UNKNOWN_MAX ?? 10000);
|
|
67
72
|
const GRACEFUL_PROC_SHUTDOWN = 7 * 60 * 1000;
|
|
68
73
|
const KILL_PROC_SHUTDOWN = 2.5 * 1000;
|
|
69
74
|
const QUEUE_MAX_WAIT_MS = Number(process.env.AL_QUEUE_MAX_WAIT_MS || 7_000);
|
|
70
75
|
const QUEUE_RECHECK_MS = Number(process.env.AL_QUEUE_RECHECK_MS ?? 50);
|
|
76
|
+
const QUEUE_SCAN_MIN_MS = Number(process.env.AL_QUEUE_SCAN_MIN_MS ?? 20);
|
|
71
77
|
const MAX_RETRIES = 35;
|
|
72
78
|
function peekBytes(chunks, n) {
|
|
73
79
|
if (chunks.length > 0 && chunks[0].length >= n) {
|
|
@@ -301,51 +307,55 @@ class Host extends home_1.Home {
|
|
|
301
307
|
}
|
|
302
308
|
});
|
|
303
309
|
}
|
|
310
|
+
static p2pServerWanted() {
|
|
311
|
+
return activeoptions_1.ActiveOptions.get("p2pStreamServer", false);
|
|
312
|
+
}
|
|
313
|
+
p2pConnection(socket) {
|
|
314
|
+
socket.setNoDelay(true);
|
|
315
|
+
activelogger_1.ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
|
|
316
|
+
let chunks = [];
|
|
317
|
+
let bufferLength = 0;
|
|
318
|
+
socket.on("data", async (data) => {
|
|
319
|
+
chunks.push(data);
|
|
320
|
+
bufferLength += data.length;
|
|
321
|
+
while (bufferLength >= 44) {
|
|
322
|
+
const header = peekBytes(chunks, 44);
|
|
323
|
+
const length = header.readUInt32BE(40);
|
|
324
|
+
if (bufferLength >= 44 + length) {
|
|
325
|
+
const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
|
|
326
|
+
const senderRef = fullBuffer.slice(0, 40).toString().trim();
|
|
327
|
+
const item = fullBuffer.slice(44, 44 + length);
|
|
328
|
+
const remaining = fullBuffer.slice(44 + length);
|
|
329
|
+
chunks = remaining.length > 0 ? [remaining] : [];
|
|
330
|
+
bufferLength = remaining.length;
|
|
331
|
+
let tx;
|
|
332
|
+
try {
|
|
333
|
+
tx = await activeoptions_1.ActiveClone.deserialize(item);
|
|
334
|
+
}
|
|
335
|
+
catch (error) {
|
|
336
|
+
activelogger_1.ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
this.processEndpoints({
|
|
340
|
+
headers: { "X-Activeledger": senderRef },
|
|
341
|
+
method: "POST",
|
|
342
|
+
url: "/a/init",
|
|
343
|
+
connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
|
|
344
|
+
}, null, tx, senderRef, true);
|
|
345
|
+
}
|
|
346
|
+
else {
|
|
347
|
+
break;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
socket.on("error", (err) => {
|
|
352
|
+
activelogger_1.ActiveLogger.error(err, "P2P socket error");
|
|
353
|
+
});
|
|
354
|
+
}
|
|
304
355
|
constructor() {
|
|
305
356
|
super();
|
|
306
|
-
if (
|
|
307
|
-
this.p2pServer = net.createServer((socket) =>
|
|
308
|
-
socket.setNoDelay(true);
|
|
309
|
-
activelogger_1.ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
|
|
310
|
-
let chunks = [];
|
|
311
|
-
let bufferLength = 0;
|
|
312
|
-
socket.on("data", async (data) => {
|
|
313
|
-
chunks.push(data);
|
|
314
|
-
bufferLength += data.length;
|
|
315
|
-
while (bufferLength >= 44) {
|
|
316
|
-
const header = peekBytes(chunks, 44);
|
|
317
|
-
const length = header.readUInt32BE(40);
|
|
318
|
-
if (bufferLength >= 44 + length) {
|
|
319
|
-
const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
|
|
320
|
-
const senderRef = fullBuffer.slice(0, 40).toString().trim();
|
|
321
|
-
const item = fullBuffer.slice(44, 44 + length);
|
|
322
|
-
const remaining = fullBuffer.slice(44 + length);
|
|
323
|
-
chunks = remaining.length > 0 ? [remaining] : [];
|
|
324
|
-
bufferLength = remaining.length;
|
|
325
|
-
let tx;
|
|
326
|
-
try {
|
|
327
|
-
tx = await activeoptions_1.ActiveClone.deserialize(item);
|
|
328
|
-
}
|
|
329
|
-
catch (error) {
|
|
330
|
-
activelogger_1.ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
|
|
331
|
-
continue;
|
|
332
|
-
}
|
|
333
|
-
this.processEndpoints({
|
|
334
|
-
headers: { "X-Activeledger": senderRef },
|
|
335
|
-
method: "POST",
|
|
336
|
-
url: "/a/init",
|
|
337
|
-
connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
|
|
338
|
-
}, null, tx, senderRef, true);
|
|
339
|
-
}
|
|
340
|
-
else {
|
|
341
|
-
break;
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
});
|
|
345
|
-
socket.on("error", (err) => {
|
|
346
|
-
activelogger_1.ActiveLogger.error(err, "P2P socket error");
|
|
347
|
-
});
|
|
348
|
-
});
|
|
357
|
+
if (Host.p2pServerWanted()) {
|
|
358
|
+
this.p2pServer = net.createServer((socket) => this.p2pConnection(socket));
|
|
349
359
|
const p2pPort = parseInt(utils_1.ActiveInterfaces.getBindingDetails("port")) + 1;
|
|
350
360
|
this.p2pServer.listen(p2pPort, () => {
|
|
351
361
|
activelogger_1.ActiveLogger.info(`P2P TCP server listening on port ${p2pPort}`);
|
|
@@ -358,6 +368,8 @@ class Host extends home_1.Home {
|
|
|
358
368
|
this.dbErrorConnection.info();
|
|
359
369
|
this.dbEventConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.event);
|
|
360
370
|
this.dbEventConnection.info();
|
|
371
|
+
void this.loadResentGuard();
|
|
372
|
+
void this.loadStoredMarker();
|
|
361
373
|
this.api = (0, uWebSockets_js_1.App)();
|
|
362
374
|
this.hybridRegistry = new hybridApi_1.LedgerHybridRegistry(this.dbConnection);
|
|
363
375
|
const hybridApi = new hybridApi_1.HybridApi({
|
|
@@ -645,13 +657,57 @@ class Host extends home_1.Home {
|
|
|
645
657
|
},
|
|
646
658
|
};
|
|
647
659
|
}
|
|
660
|
+
static ipcSerialization() {
|
|
661
|
+
return activeoptions_1.ActiveOptions.get("ipcSerialization", "json") === "advanced"
|
|
662
|
+
? "advanced"
|
|
663
|
+
: "json";
|
|
664
|
+
}
|
|
665
|
+
batchBroadcasts(child) {
|
|
666
|
+
if (activeoptions_1.ActiveOptions.get("ipcBatching", false) !== true)
|
|
667
|
+
return;
|
|
668
|
+
const rawSend = child.send.bind(child);
|
|
669
|
+
let queued = [];
|
|
670
|
+
let scheduled = false;
|
|
671
|
+
const flush = () => {
|
|
672
|
+
scheduled = false;
|
|
673
|
+
if (!queued.length)
|
|
674
|
+
return true;
|
|
675
|
+
const out = queued;
|
|
676
|
+
queued = [];
|
|
677
|
+
try {
|
|
678
|
+
return out.length === 1
|
|
679
|
+
? rawSend(out[0])
|
|
680
|
+
: rawSend({ type: "batch", data: { messages: out } });
|
|
681
|
+
}
|
|
682
|
+
catch (error) {
|
|
683
|
+
activelogger_1.ActiveLogger.warn(error, "Worker message not sent");
|
|
684
|
+
return false;
|
|
685
|
+
}
|
|
686
|
+
};
|
|
687
|
+
child.send = (message, ...rest) => {
|
|
688
|
+
if (message && message.type === "broadcast" && !rest.length) {
|
|
689
|
+
queued.push(message);
|
|
690
|
+
if (!scheduled) {
|
|
691
|
+
scheduled = true;
|
|
692
|
+
setImmediate(flush);
|
|
693
|
+
}
|
|
694
|
+
return true;
|
|
695
|
+
}
|
|
696
|
+
flush();
|
|
697
|
+
return rawSend(message, ...rest);
|
|
698
|
+
};
|
|
699
|
+
}
|
|
648
700
|
createProcessor(cpuTotal) {
|
|
649
701
|
const pFork = (0, child_process_1.fork)(`${__dirname}/process.js`, [], {
|
|
650
702
|
cwd: process.cwd(),
|
|
651
703
|
execArgv: (0, workerheap_1.workerExecArgv)(),
|
|
652
704
|
stdio: "inherit",
|
|
705
|
+
serialization: Host.ipcSerialization(),
|
|
653
706
|
});
|
|
654
707
|
pFork.stop = false;
|
|
708
|
+
if (Host.ipcSerialization() === "advanced")
|
|
709
|
+
(0, ipc_1.withJsonFallback)(pFork);
|
|
710
|
+
this.batchBroadcasts(pFork);
|
|
655
711
|
let unloadHandled = false;
|
|
656
712
|
const unloadProcessorSafely = (...error) => {
|
|
657
713
|
if (unloadHandled || !this.listenSocket) {
|
|
@@ -726,8 +782,12 @@ class Host extends home_1.Home {
|
|
|
726
782
|
(0, hybridApi_1.recordFailed)(this.dbEventConnection, this.dbConnection, pending.entry, activeoptions_1.ActiveOptions.get("build", 0));
|
|
727
783
|
break;
|
|
728
784
|
case "commited":
|
|
785
|
+
if (m.data.broadcast) {
|
|
786
|
+
this.broadcast(m.data.umid, false);
|
|
787
|
+
}
|
|
729
788
|
if (!pending)
|
|
730
789
|
return;
|
|
790
|
+
this.noteStored(pending.entry.$umid, pending.entry.$tx?.$expire);
|
|
731
791
|
pending.resolve({
|
|
732
792
|
status: 200,
|
|
733
793
|
data: { ...pending.entry, ...m.data.entry },
|
|
@@ -1062,6 +1122,217 @@ class Host extends home_1.Home {
|
|
|
1062
1122
|
}
|
|
1063
1123
|
return robin;
|
|
1064
1124
|
}
|
|
1125
|
+
storedByMinute = new Map();
|
|
1126
|
+
storedCount = 0;
|
|
1127
|
+
storedUnknown = new Set();
|
|
1128
|
+
storedUnknownAllUntil = 0;
|
|
1129
|
+
storedColdUntil = Date.now() + STORED_WINDOW_MS;
|
|
1130
|
+
storedSwept = 0;
|
|
1131
|
+
storedMarkerLoaded = false;
|
|
1132
|
+
storedMarkerLoading;
|
|
1133
|
+
storedMarkerTried = 0;
|
|
1134
|
+
storedMarkerWritten = new Set();
|
|
1135
|
+
static minuteOf(ms) {
|
|
1136
|
+
return Math.floor(ms / 60000);
|
|
1137
|
+
}
|
|
1138
|
+
noteStored(umid, expire) {
|
|
1139
|
+
if (STORED_WINDOW_MS <= 0 || !expire)
|
|
1140
|
+
return;
|
|
1141
|
+
const until = new Date(expire).getTime();
|
|
1142
|
+
const now = Date.now();
|
|
1143
|
+
if (!Number.isFinite(until) || until <= now)
|
|
1144
|
+
return;
|
|
1145
|
+
const minute = Host.minuteOf(until);
|
|
1146
|
+
if (until > now + STORED_WINDOW_MS || this.storedCount >= STORED_MAX) {
|
|
1147
|
+
this.markUnknown(minute, until);
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
if (now - this.storedSwept > 1000) {
|
|
1151
|
+
this.storedSwept = now;
|
|
1152
|
+
const current = Host.minuteOf(now);
|
|
1153
|
+
for (const [m, set] of this.storedByMinute) {
|
|
1154
|
+
if (m < current) {
|
|
1155
|
+
this.storedCount -= set.size;
|
|
1156
|
+
this.storedByMinute.delete(m);
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
for (const m of this.storedUnknown)
|
|
1160
|
+
if (m < current)
|
|
1161
|
+
this.storedUnknown.delete(m);
|
|
1162
|
+
}
|
|
1163
|
+
let set = this.storedByMinute.get(minute);
|
|
1164
|
+
if (!set)
|
|
1165
|
+
this.storedByMinute.set(minute, (set = new Set()));
|
|
1166
|
+
if (!set.has(umid)) {
|
|
1167
|
+
set.add(umid);
|
|
1168
|
+
this.storedCount++;
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
markUnknown(minute, until) {
|
|
1172
|
+
this.writeStoredMarker(minute);
|
|
1173
|
+
if (this.storedUnknown.size >= STORED_UNKNOWN_MAX && !this.storedUnknown.has(minute)) {
|
|
1174
|
+
this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, until);
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
this.storedUnknown.add(minute);
|
|
1178
|
+
}
|
|
1179
|
+
writeStoredMarker(minute) {
|
|
1180
|
+
if (this.storedMarkerWritten.has(minute))
|
|
1181
|
+
return;
|
|
1182
|
+
this.storedMarkerWritten.add(minute);
|
|
1183
|
+
this.dbEventConnection
|
|
1184
|
+
.post({ _id: `storedfar:${minute}`, minute, until: (minute + 1) * 60000 })
|
|
1185
|
+
.catch((error) => {
|
|
1186
|
+
this.storedMarkerWritten.delete(minute);
|
|
1187
|
+
activelogger_1.ActiveLogger.error({ error }, "Could not record a far-off $expire commit; a restart would forget it");
|
|
1188
|
+
});
|
|
1189
|
+
}
|
|
1190
|
+
loadStoredMarker() {
|
|
1191
|
+
if (STORED_WINDOW_MS <= 0 || this.storedMarkerLoaded)
|
|
1192
|
+
return Promise.resolve();
|
|
1193
|
+
if (this.storedMarkerLoading)
|
|
1194
|
+
return this.storedMarkerLoading;
|
|
1195
|
+
if (Date.now() - this.storedMarkerTried < 5000)
|
|
1196
|
+
return Promise.resolve();
|
|
1197
|
+
this.storedMarkerTried = Date.now();
|
|
1198
|
+
this.storedMarkerLoading = (async () => {
|
|
1199
|
+
try {
|
|
1200
|
+
const list = await this.dbEventConnection.allDocs({ startkey: "storedfar:", endkey: "storedfar:x", include_docs: true });
|
|
1201
|
+
if (!list || !Array.isArray(list.rows))
|
|
1202
|
+
throw new Error("no answer");
|
|
1203
|
+
const now = Date.now();
|
|
1204
|
+
for (const row of list.rows) {
|
|
1205
|
+
const doc = row?.doc ?? row;
|
|
1206
|
+
if (!doc?._id)
|
|
1207
|
+
continue;
|
|
1208
|
+
if (Number(doc.until) > now) {
|
|
1209
|
+
this.storedMarkerWritten.add(Number(doc.minute));
|
|
1210
|
+
if (this.storedUnknown.size < STORED_UNKNOWN_MAX)
|
|
1211
|
+
this.storedUnknown.add(Number(doc.minute));
|
|
1212
|
+
else
|
|
1213
|
+
this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, Number(doc.until));
|
|
1214
|
+
}
|
|
1215
|
+
else
|
|
1216
|
+
await this.dbEventConnection.purge(doc).catch(() => undefined);
|
|
1217
|
+
}
|
|
1218
|
+
this.storedMarkerLoaded = true;
|
|
1219
|
+
}
|
|
1220
|
+
catch (error) {
|
|
1221
|
+
activelogger_1.ActiveLogger.warn(`Could not read the far-off $expire rows, asking the storage meanwhile: ${error?.message ?? error}`);
|
|
1222
|
+
}
|
|
1223
|
+
finally {
|
|
1224
|
+
this.storedMarkerLoading = undefined;
|
|
1225
|
+
}
|
|
1226
|
+
})();
|
|
1227
|
+
return this.storedMarkerLoading;
|
|
1228
|
+
}
|
|
1229
|
+
mayBeStored(umid, expire) {
|
|
1230
|
+
if (STORED_WINDOW_MS <= 0)
|
|
1231
|
+
return true;
|
|
1232
|
+
if (!this.storedMarkerLoaded) {
|
|
1233
|
+
void this.loadStoredMarker();
|
|
1234
|
+
return true;
|
|
1235
|
+
}
|
|
1236
|
+
const now = Date.now();
|
|
1237
|
+
if (now < this.storedColdUntil || now < this.storedUnknownAllUntil)
|
|
1238
|
+
return true;
|
|
1239
|
+
if (!(expire <= now + STORED_WINDOW_MS))
|
|
1240
|
+
return true;
|
|
1241
|
+
const minute = Host.minuteOf(expire);
|
|
1242
|
+
if (this.storedUnknown.has(minute))
|
|
1243
|
+
return true;
|
|
1244
|
+
return this.storedByMinute.get(minute)?.has(umid) === true;
|
|
1245
|
+
}
|
|
1246
|
+
resentGuardUntil = 0;
|
|
1247
|
+
resentInFlight = new Map();
|
|
1248
|
+
resentTimer;
|
|
1249
|
+
resentLoaded = false;
|
|
1250
|
+
resentLoading;
|
|
1251
|
+
rememberResent(replaced, replacement, expire) {
|
|
1252
|
+
const until = expire ? new Date(expire).getTime() : NaN;
|
|
1253
|
+
if (!Number.isFinite(until) || until <= Date.now())
|
|
1254
|
+
return;
|
|
1255
|
+
this.resentGuardUntil = Math.max(this.resentGuardUntil, until);
|
|
1256
|
+
this.resentInFlight.set(replaced, replacement);
|
|
1257
|
+
this.dbEventConnection
|
|
1258
|
+
.post({ _id: `resent:${replaced}`, to: replacement, until })
|
|
1259
|
+
.then(() => void this.resentInFlight.delete(replaced), (error) => {
|
|
1260
|
+
activelogger_1.ActiveLogger.error({ replaced, error }, "Could not record a resent transaction; refusing a repost from memory only");
|
|
1261
|
+
setTimeout(() => this.resentInFlight.delete(replaced), Math.max(0, until - Date.now())).unref();
|
|
1262
|
+
});
|
|
1263
|
+
this.startResentHousekeeping();
|
|
1264
|
+
}
|
|
1265
|
+
async resentAs(umid) {
|
|
1266
|
+
if (this.resentLoading && !this.resentLoaded)
|
|
1267
|
+
await this.waitForResentList();
|
|
1268
|
+
if (Date.now() >= this.resentGuardUntil)
|
|
1269
|
+
return undefined;
|
|
1270
|
+
const inFlight = this.resentInFlight.get(umid);
|
|
1271
|
+
if (inFlight)
|
|
1272
|
+
return inFlight;
|
|
1273
|
+
const row = await this.dbEventConnection.get(`resent:${umid}`).catch(() => null);
|
|
1274
|
+
if (row === null || row === undefined)
|
|
1275
|
+
throw new Error("could not check whether the transaction was resent");
|
|
1276
|
+
return typeof row.to === "string" && Number(row.until) > Date.now() ? row.to : undefined;
|
|
1277
|
+
}
|
|
1278
|
+
loadResentGuard() {
|
|
1279
|
+
this.resentLoading = this.refreshResentGuard(false).then(() => undefined);
|
|
1280
|
+
return this.resentLoading;
|
|
1281
|
+
}
|
|
1282
|
+
resentWaiting;
|
|
1283
|
+
waitForResentList() {
|
|
1284
|
+
this.resentWaiting ??= (async () => {
|
|
1285
|
+
try {
|
|
1286
|
+
await this.resentLoading;
|
|
1287
|
+
for (let attempt = 0; !this.resentLoaded && attempt < RESENT_LOAD_WAIT_MS / 500; attempt++) {
|
|
1288
|
+
await new Promise((resolve) => setTimeout(resolve, 500).unref());
|
|
1289
|
+
await this.refreshResentGuard(false);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
finally {
|
|
1293
|
+
this.resentWaiting = undefined;
|
|
1294
|
+
}
|
|
1295
|
+
if (!this.resentLoaded)
|
|
1296
|
+
throw new Error("the list of resent transactions has not been read yet");
|
|
1297
|
+
})();
|
|
1298
|
+
return this.resentWaiting;
|
|
1299
|
+
}
|
|
1300
|
+
async refreshResentGuard(retryLater = true) {
|
|
1301
|
+
let latest = 0;
|
|
1302
|
+
try {
|
|
1303
|
+
const list = await this.dbEventConnection.allDocs({ startkey: "resent:", endkey: "resent:x", include_docs: true });
|
|
1304
|
+
if (!list || !Array.isArray(list.rows))
|
|
1305
|
+
throw new Error("no answer");
|
|
1306
|
+
const now = Date.now();
|
|
1307
|
+
for (const row of list.rows) {
|
|
1308
|
+
const doc = row?.doc ?? row;
|
|
1309
|
+
if (!doc?._id)
|
|
1310
|
+
continue;
|
|
1311
|
+
if (Number(doc.until) > now)
|
|
1312
|
+
latest = Math.max(latest, Number(doc.until));
|
|
1313
|
+
else
|
|
1314
|
+
await this.dbEventConnection.purge(doc).catch(() => undefined);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
catch (error) {
|
|
1318
|
+
if (retryLater) {
|
|
1319
|
+
activelogger_1.ActiveLogger.warn(`Could not read the resent transactions, trying again in 30s: ${error?.message ?? error}`);
|
|
1320
|
+
setTimeout(() => void this.refreshResentGuard(), 30_000).unref();
|
|
1321
|
+
}
|
|
1322
|
+
return false;
|
|
1323
|
+
}
|
|
1324
|
+
this.resentGuardUntil = Math.max(latest, this.resentInFlight.size ? this.resentGuardUntil : 0);
|
|
1325
|
+
this.resentLoaded = true;
|
|
1326
|
+
if (latest > 0)
|
|
1327
|
+
this.startResentHousekeeping();
|
|
1328
|
+
return true;
|
|
1329
|
+
}
|
|
1330
|
+
startResentHousekeeping() {
|
|
1331
|
+
if (this.resentTimer)
|
|
1332
|
+
return;
|
|
1333
|
+
this.resentTimer = setInterval(() => void this.refreshResentGuard(), 60 * 60 * 1000);
|
|
1334
|
+
this.resentTimer.unref();
|
|
1335
|
+
}
|
|
1065
1336
|
release(umid, owner) {
|
|
1066
1337
|
if (owner && this.processPending[umid] && this.processPending[umid].entry !== owner) {
|
|
1067
1338
|
activelogger_1.ActiveLogger.warn(umid, "Not releasing, pending entry belongs to another submission");
|
|
@@ -1088,20 +1359,49 @@ class Host extends home_1.Home {
|
|
|
1088
1359
|
this.destroy(umid);
|
|
1089
1360
|
}
|
|
1090
1361
|
}, hold);
|
|
1091
|
-
|
|
1092
|
-
setTimeout(() => this.processQueue(), QUEUE_RECHECK_MS);
|
|
1093
|
-
}
|
|
1094
|
-
else {
|
|
1095
|
-
setImmediate(() => this.processQueue());
|
|
1096
|
-
}
|
|
1362
|
+
this.scheduleQueueRecheck(QUEUE_RECHECK_MS);
|
|
1097
1363
|
}
|
|
1098
1364
|
else {
|
|
1099
1365
|
activelogger_1.ActiveLogger.warn(umid, "Trying to release unknown umid");
|
|
1100
1366
|
}
|
|
1101
1367
|
}
|
|
1102
1368
|
processingBLQ = false;
|
|
1369
|
+
lastQueueScan = 0;
|
|
1370
|
+
queueRecheckPending = false;
|
|
1371
|
+
scheduleQueueRecheck(ms) {
|
|
1372
|
+
if (this.queueRecheckPending)
|
|
1373
|
+
return;
|
|
1374
|
+
this.queueRecheckPending = true;
|
|
1375
|
+
const run = () => {
|
|
1376
|
+
this.queueRecheckPending = false;
|
|
1377
|
+
this.processQueue();
|
|
1378
|
+
};
|
|
1379
|
+
if (ms > 0) {
|
|
1380
|
+
setTimeout(run, ms);
|
|
1381
|
+
}
|
|
1382
|
+
else {
|
|
1383
|
+
setImmediate(run);
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1103
1386
|
processQueue(next, internal = false) {
|
|
1104
1387
|
if (!this.processingBLQ) {
|
|
1388
|
+
const queued = this.busyLocksQueue.internal.length + this.busyLocksQueue.external.length;
|
|
1389
|
+
if (!queued) {
|
|
1390
|
+
if (next)
|
|
1391
|
+
this.hold(next);
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
const since = Date.now() - this.lastQueueScan;
|
|
1395
|
+
if (QUEUE_SCAN_MIN_MS > 0 && since < QUEUE_SCAN_MIN_MS) {
|
|
1396
|
+
if (next) {
|
|
1397
|
+
this.hold(next);
|
|
1398
|
+
}
|
|
1399
|
+
else {
|
|
1400
|
+
this.scheduleQueueRecheck(QUEUE_SCAN_MIN_MS - since);
|
|
1401
|
+
}
|
|
1402
|
+
return;
|
|
1403
|
+
}
|
|
1404
|
+
this.lastQueueScan = Date.now();
|
|
1105
1405
|
this.processingBLQ = true;
|
|
1106
1406
|
const checked = new Set();
|
|
1107
1407
|
if (this.busyLocksQueue.internal.length) {
|