@activeledger/activenetwork 5.3.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/es/network/endpoints.d.ts +1 -0
- package/es/network/endpoints.d.ts.map +1 -1
- package/es/network/endpoints.js +32 -4
- package/es/network/endpoints.js.map +1 -1
- package/es/network/host.d.ts +35 -0
- package/es/network/host.d.ts.map +1 -1
- package/es/network/host.js +349 -49
- package/es/network/host.js.map +1 -1
- package/es/network/ipc.d.ts +6 -0
- package/es/network/ipc.d.ts.map +1 -0
- package/es/network/ipc.js +18 -0
- package/es/network/ipc.js.map +1 -0
- package/es/network/locker.d.ts.map +1 -1
- package/es/network/locker.js +21 -19
- package/es/network/locker.js.map +1 -1
- package/es/network/process.js +50 -10
- package/es/network/process.js.map +1 -1
- package/lib/network/endpoints.d.ts +1 -0
- package/lib/network/endpoints.d.ts.map +1 -1
- package/lib/network/endpoints.js +32 -4
- package/lib/network/endpoints.js.map +1 -1
- package/lib/network/host.d.ts +35 -0
- package/lib/network/host.d.ts.map +1 -1
- package/lib/network/host.js +349 -49
- package/lib/network/host.js.map +1 -1
- package/lib/network/ipc.d.ts +6 -0
- package/lib/network/ipc.d.ts.map +1 -0
- package/lib/network/ipc.js +24 -0
- package/lib/network/ipc.js.map +1 -0
- package/lib/network/locker.d.ts.map +1 -1
- package/lib/network/locker.js +21 -19
- package/lib/network/locker.js.map +1 -1
- package/lib/network/process.js +50 -10
- package/lib/network/process.js.map +1 -1
- package/package.json +9 -9
package/es/network/host.js
CHANGED
|
@@ -14,6 +14,7 @@ import { Neighbour } from "./neighbour";
|
|
|
14
14
|
import { ActiveInterfaces } from "./utils";
|
|
15
15
|
import { Endpoints, Maintain } from "./index";
|
|
16
16
|
import { Locker } from "./locker";
|
|
17
|
+
import { withJsonFallback } from "./ipc";
|
|
17
18
|
import { PhysicalCores } from "./cpus";
|
|
18
19
|
import { DEFAULT_WORKER_COUNT, workerCount } from "./workercount";
|
|
19
20
|
import { workerExecArgv } from "./workerheap";
|
|
@@ -28,10 +29,15 @@ import { gunzip } from "zlib";
|
|
|
28
29
|
const GZIP_MIN_RESPONSE_BYTES = 16384;
|
|
29
30
|
const gunzipAsync = promisify(gunzip);
|
|
30
31
|
const TIMER_QUEUE_INTERVAL = 2 * 1000;
|
|
32
|
+
const RESENT_LOAD_WAIT_MS = 5 * 1000;
|
|
33
|
+
const STORED_WINDOW_MS = Number(process.env.AL_STORED_WINDOW_MS ?? 5 * 60 * 1000);
|
|
34
|
+
const STORED_MAX = Number(process.env.AL_STORED_MAX ?? 200000);
|
|
35
|
+
const STORED_UNKNOWN_MAX = Number(process.env.AL_STORED_UNKNOWN_MAX ?? 10000);
|
|
31
36
|
const GRACEFUL_PROC_SHUTDOWN = 7 * 60 * 1000;
|
|
32
37
|
const KILL_PROC_SHUTDOWN = 2.5 * 1000;
|
|
33
38
|
const QUEUE_MAX_WAIT_MS = Number(process.env.AL_QUEUE_MAX_WAIT_MS || 7_000);
|
|
34
39
|
const QUEUE_RECHECK_MS = Number(process.env.AL_QUEUE_RECHECK_MS ?? 50);
|
|
40
|
+
const QUEUE_SCAN_MIN_MS = Number(process.env.AL_QUEUE_SCAN_MIN_MS ?? 20);
|
|
35
41
|
const MAX_RETRIES = 35;
|
|
36
42
|
function peekBytes(chunks, n) {
|
|
37
43
|
if (chunks.length > 0 && chunks[0].length >= n) {
|
|
@@ -265,51 +271,55 @@ export class Host extends Home {
|
|
|
265
271
|
}
|
|
266
272
|
});
|
|
267
273
|
}
|
|
274
|
+
static p2pServerWanted() {
|
|
275
|
+
return ActiveOptions.get("p2pStreamServer", false);
|
|
276
|
+
}
|
|
277
|
+
p2pConnection(socket) {
|
|
278
|
+
socket.setNoDelay(true);
|
|
279
|
+
ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
|
|
280
|
+
let chunks = [];
|
|
281
|
+
let bufferLength = 0;
|
|
282
|
+
socket.on("data", async (data) => {
|
|
283
|
+
chunks.push(data);
|
|
284
|
+
bufferLength += data.length;
|
|
285
|
+
while (bufferLength >= 44) {
|
|
286
|
+
const header = peekBytes(chunks, 44);
|
|
287
|
+
const length = header.readUInt32BE(40);
|
|
288
|
+
if (bufferLength >= 44 + length) {
|
|
289
|
+
const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
|
|
290
|
+
const senderRef = fullBuffer.slice(0, 40).toString().trim();
|
|
291
|
+
const item = fullBuffer.slice(44, 44 + length);
|
|
292
|
+
const remaining = fullBuffer.slice(44 + length);
|
|
293
|
+
chunks = remaining.length > 0 ? [remaining] : [];
|
|
294
|
+
bufferLength = remaining.length;
|
|
295
|
+
let tx;
|
|
296
|
+
try {
|
|
297
|
+
tx = await ActiveClone.deserialize(item);
|
|
298
|
+
}
|
|
299
|
+
catch (error) {
|
|
300
|
+
ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
this.processEndpoints({
|
|
304
|
+
headers: { "X-Activeledger": senderRef },
|
|
305
|
+
method: "POST",
|
|
306
|
+
url: "/a/init",
|
|
307
|
+
connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
|
|
308
|
+
}, null, tx, senderRef, true);
|
|
309
|
+
}
|
|
310
|
+
else {
|
|
311
|
+
break;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
});
|
|
315
|
+
socket.on("error", (err) => {
|
|
316
|
+
ActiveLogger.error(err, "P2P socket error");
|
|
317
|
+
});
|
|
318
|
+
}
|
|
268
319
|
constructor() {
|
|
269
320
|
super();
|
|
270
|
-
if (
|
|
271
|
-
this.p2pServer = net.createServer((socket) =>
|
|
272
|
-
socket.setNoDelay(true);
|
|
273
|
-
ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
|
|
274
|
-
let chunks = [];
|
|
275
|
-
let bufferLength = 0;
|
|
276
|
-
socket.on("data", async (data) => {
|
|
277
|
-
chunks.push(data);
|
|
278
|
-
bufferLength += data.length;
|
|
279
|
-
while (bufferLength >= 44) {
|
|
280
|
-
const header = peekBytes(chunks, 44);
|
|
281
|
-
const length = header.readUInt32BE(40);
|
|
282
|
-
if (bufferLength >= 44 + length) {
|
|
283
|
-
const fullBuffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
|
|
284
|
-
const senderRef = fullBuffer.slice(0, 40).toString().trim();
|
|
285
|
-
const item = fullBuffer.slice(44, 44 + length);
|
|
286
|
-
const remaining = fullBuffer.slice(44 + length);
|
|
287
|
-
chunks = remaining.length > 0 ? [remaining] : [];
|
|
288
|
-
bufferLength = remaining.length;
|
|
289
|
-
let tx;
|
|
290
|
-
try {
|
|
291
|
-
tx = await ActiveClone.deserialize(item);
|
|
292
|
-
}
|
|
293
|
-
catch (error) {
|
|
294
|
-
ActiveLogger.warn(error, `P2P dropped malformed frame from ${socket.remoteAddress} (sender ${senderRef})`);
|
|
295
|
-
continue;
|
|
296
|
-
}
|
|
297
|
-
this.processEndpoints({
|
|
298
|
-
headers: { "X-Activeledger": senderRef },
|
|
299
|
-
method: "POST",
|
|
300
|
-
url: "/a/init",
|
|
301
|
-
connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
|
|
302
|
-
}, null, tx, senderRef, true);
|
|
303
|
-
}
|
|
304
|
-
else {
|
|
305
|
-
break;
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
});
|
|
309
|
-
socket.on("error", (err) => {
|
|
310
|
-
ActiveLogger.error(err, "P2P socket error");
|
|
311
|
-
});
|
|
312
|
-
});
|
|
321
|
+
if (Host.p2pServerWanted()) {
|
|
322
|
+
this.p2pServer = net.createServer((socket) => this.p2pConnection(socket));
|
|
313
323
|
const p2pPort = parseInt(ActiveInterfaces.getBindingDetails("port")) + 1;
|
|
314
324
|
this.p2pServer.listen(p2pPort, () => {
|
|
315
325
|
ActiveLogger.info(`P2P TCP server listening on port ${p2pPort}`);
|
|
@@ -322,6 +332,8 @@ export class Host extends Home {
|
|
|
322
332
|
this.dbErrorConnection.info();
|
|
323
333
|
this.dbEventConnection = new ActiveDSConnect(db.url + "/" + db.event);
|
|
324
334
|
this.dbEventConnection.info();
|
|
335
|
+
void this.loadResentGuard();
|
|
336
|
+
void this.loadStoredMarker();
|
|
325
337
|
this.api = App();
|
|
326
338
|
this.hybridRegistry = new LedgerHybridRegistry(this.dbConnection);
|
|
327
339
|
const hybridApi = new HybridApi({
|
|
@@ -609,13 +621,57 @@ export class Host extends Home {
|
|
|
609
621
|
},
|
|
610
622
|
};
|
|
611
623
|
}
|
|
624
|
+
static ipcSerialization() {
|
|
625
|
+
return ActiveOptions.get("ipcSerialization", "json") === "advanced"
|
|
626
|
+
? "advanced"
|
|
627
|
+
: "json";
|
|
628
|
+
}
|
|
629
|
+
batchBroadcasts(child) {
|
|
630
|
+
if (ActiveOptions.get("ipcBatching", false) !== true)
|
|
631
|
+
return;
|
|
632
|
+
const rawSend = child.send.bind(child);
|
|
633
|
+
let queued = [];
|
|
634
|
+
let scheduled = false;
|
|
635
|
+
const flush = () => {
|
|
636
|
+
scheduled = false;
|
|
637
|
+
if (!queued.length)
|
|
638
|
+
return true;
|
|
639
|
+
const out = queued;
|
|
640
|
+
queued = [];
|
|
641
|
+
try {
|
|
642
|
+
return out.length === 1
|
|
643
|
+
? rawSend(out[0])
|
|
644
|
+
: rawSend({ type: "batch", data: { messages: out } });
|
|
645
|
+
}
|
|
646
|
+
catch (error) {
|
|
647
|
+
ActiveLogger.warn(error, "Worker message not sent");
|
|
648
|
+
return false;
|
|
649
|
+
}
|
|
650
|
+
};
|
|
651
|
+
child.send = (message, ...rest) => {
|
|
652
|
+
if (message && message.type === "broadcast" && !rest.length) {
|
|
653
|
+
queued.push(message);
|
|
654
|
+
if (!scheduled) {
|
|
655
|
+
scheduled = true;
|
|
656
|
+
setImmediate(flush);
|
|
657
|
+
}
|
|
658
|
+
return true;
|
|
659
|
+
}
|
|
660
|
+
flush();
|
|
661
|
+
return rawSend(message, ...rest);
|
|
662
|
+
};
|
|
663
|
+
}
|
|
612
664
|
createProcessor(cpuTotal) {
|
|
613
665
|
const pFork = fork(`${__dirname}/process.js`, [], {
|
|
614
666
|
cwd: process.cwd(),
|
|
615
667
|
execArgv: workerExecArgv(),
|
|
616
668
|
stdio: "inherit",
|
|
669
|
+
serialization: Host.ipcSerialization(),
|
|
617
670
|
});
|
|
618
671
|
pFork.stop = false;
|
|
672
|
+
if (Host.ipcSerialization() === "advanced")
|
|
673
|
+
withJsonFallback(pFork);
|
|
674
|
+
this.batchBroadcasts(pFork);
|
|
619
675
|
let unloadHandled = false;
|
|
620
676
|
const unloadProcessorSafely = (...error) => {
|
|
621
677
|
if (unloadHandled || !this.listenSocket) {
|
|
@@ -690,8 +746,12 @@ export class Host extends Home {
|
|
|
690
746
|
recordFailed(this.dbEventConnection, this.dbConnection, pending.entry, ActiveOptions.get("build", 0));
|
|
691
747
|
break;
|
|
692
748
|
case "commited":
|
|
749
|
+
if (m.data.broadcast) {
|
|
750
|
+
this.broadcast(m.data.umid, false);
|
|
751
|
+
}
|
|
693
752
|
if (!pending)
|
|
694
753
|
return;
|
|
754
|
+
this.noteStored(pending.entry.$umid, pending.entry.$tx?.$expire);
|
|
695
755
|
pending.resolve({
|
|
696
756
|
status: 200,
|
|
697
757
|
data: { ...pending.entry, ...m.data.entry },
|
|
@@ -1026,6 +1086,217 @@ export class Host extends Home {
|
|
|
1026
1086
|
}
|
|
1027
1087
|
return robin;
|
|
1028
1088
|
}
|
|
1089
|
+
storedByMinute = new Map();
|
|
1090
|
+
storedCount = 0;
|
|
1091
|
+
storedUnknown = new Set();
|
|
1092
|
+
storedUnknownAllUntil = 0;
|
|
1093
|
+
storedColdUntil = Date.now() + STORED_WINDOW_MS;
|
|
1094
|
+
storedSwept = 0;
|
|
1095
|
+
storedMarkerLoaded = false;
|
|
1096
|
+
storedMarkerLoading;
|
|
1097
|
+
storedMarkerTried = 0;
|
|
1098
|
+
storedMarkerWritten = new Set();
|
|
1099
|
+
static minuteOf(ms) {
|
|
1100
|
+
return Math.floor(ms / 60000);
|
|
1101
|
+
}
|
|
1102
|
+
noteStored(umid, expire) {
|
|
1103
|
+
if (STORED_WINDOW_MS <= 0 || !expire)
|
|
1104
|
+
return;
|
|
1105
|
+
const until = new Date(expire).getTime();
|
|
1106
|
+
const now = Date.now();
|
|
1107
|
+
if (!Number.isFinite(until) || until <= now)
|
|
1108
|
+
return;
|
|
1109
|
+
const minute = Host.minuteOf(until);
|
|
1110
|
+
if (until > now + STORED_WINDOW_MS || this.storedCount >= STORED_MAX) {
|
|
1111
|
+
this.markUnknown(minute, until);
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
if (now - this.storedSwept > 1000) {
|
|
1115
|
+
this.storedSwept = now;
|
|
1116
|
+
const current = Host.minuteOf(now);
|
|
1117
|
+
for (const [m, set] of this.storedByMinute) {
|
|
1118
|
+
if (m < current) {
|
|
1119
|
+
this.storedCount -= set.size;
|
|
1120
|
+
this.storedByMinute.delete(m);
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
for (const m of this.storedUnknown)
|
|
1124
|
+
if (m < current)
|
|
1125
|
+
this.storedUnknown.delete(m);
|
|
1126
|
+
}
|
|
1127
|
+
let set = this.storedByMinute.get(minute);
|
|
1128
|
+
if (!set)
|
|
1129
|
+
this.storedByMinute.set(minute, (set = new Set()));
|
|
1130
|
+
if (!set.has(umid)) {
|
|
1131
|
+
set.add(umid);
|
|
1132
|
+
this.storedCount++;
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
markUnknown(minute, until) {
|
|
1136
|
+
this.writeStoredMarker(minute);
|
|
1137
|
+
if (this.storedUnknown.size >= STORED_UNKNOWN_MAX && !this.storedUnknown.has(minute)) {
|
|
1138
|
+
this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, until);
|
|
1139
|
+
return;
|
|
1140
|
+
}
|
|
1141
|
+
this.storedUnknown.add(minute);
|
|
1142
|
+
}
|
|
1143
|
+
writeStoredMarker(minute) {
|
|
1144
|
+
if (this.storedMarkerWritten.has(minute))
|
|
1145
|
+
return;
|
|
1146
|
+
this.storedMarkerWritten.add(minute);
|
|
1147
|
+
this.dbEventConnection
|
|
1148
|
+
.post({ _id: `storedfar:${minute}`, minute, until: (minute + 1) * 60000 })
|
|
1149
|
+
.catch((error) => {
|
|
1150
|
+
this.storedMarkerWritten.delete(minute);
|
|
1151
|
+
ActiveLogger.error({ error }, "Could not record a far-off $expire commit; a restart would forget it");
|
|
1152
|
+
});
|
|
1153
|
+
}
|
|
1154
|
+
loadStoredMarker() {
|
|
1155
|
+
if (STORED_WINDOW_MS <= 0 || this.storedMarkerLoaded)
|
|
1156
|
+
return Promise.resolve();
|
|
1157
|
+
if (this.storedMarkerLoading)
|
|
1158
|
+
return this.storedMarkerLoading;
|
|
1159
|
+
if (Date.now() - this.storedMarkerTried < 5000)
|
|
1160
|
+
return Promise.resolve();
|
|
1161
|
+
this.storedMarkerTried = Date.now();
|
|
1162
|
+
this.storedMarkerLoading = (async () => {
|
|
1163
|
+
try {
|
|
1164
|
+
const list = await this.dbEventConnection.allDocs({ startkey: "storedfar:", endkey: "storedfar:x", include_docs: true });
|
|
1165
|
+
if (!list || !Array.isArray(list.rows))
|
|
1166
|
+
throw new Error("no answer");
|
|
1167
|
+
const now = Date.now();
|
|
1168
|
+
for (const row of list.rows) {
|
|
1169
|
+
const doc = row?.doc ?? row;
|
|
1170
|
+
if (!doc?._id)
|
|
1171
|
+
continue;
|
|
1172
|
+
if (Number(doc.until) > now) {
|
|
1173
|
+
this.storedMarkerWritten.add(Number(doc.minute));
|
|
1174
|
+
if (this.storedUnknown.size < STORED_UNKNOWN_MAX)
|
|
1175
|
+
this.storedUnknown.add(Number(doc.minute));
|
|
1176
|
+
else
|
|
1177
|
+
this.storedUnknownAllUntil = Math.max(this.storedUnknownAllUntil, Number(doc.until));
|
|
1178
|
+
}
|
|
1179
|
+
else
|
|
1180
|
+
await this.dbEventConnection.purge(doc).catch(() => undefined);
|
|
1181
|
+
}
|
|
1182
|
+
this.storedMarkerLoaded = true;
|
|
1183
|
+
}
|
|
1184
|
+
catch (error) {
|
|
1185
|
+
ActiveLogger.warn(`Could not read the far-off $expire rows, asking the storage meanwhile: ${error?.message ?? error}`);
|
|
1186
|
+
}
|
|
1187
|
+
finally {
|
|
1188
|
+
this.storedMarkerLoading = undefined;
|
|
1189
|
+
}
|
|
1190
|
+
})();
|
|
1191
|
+
return this.storedMarkerLoading;
|
|
1192
|
+
}
|
|
1193
|
+
mayBeStored(umid, expire) {
|
|
1194
|
+
if (STORED_WINDOW_MS <= 0)
|
|
1195
|
+
return true;
|
|
1196
|
+
if (!this.storedMarkerLoaded) {
|
|
1197
|
+
void this.loadStoredMarker();
|
|
1198
|
+
return true;
|
|
1199
|
+
}
|
|
1200
|
+
const now = Date.now();
|
|
1201
|
+
if (now < this.storedColdUntil || now < this.storedUnknownAllUntil)
|
|
1202
|
+
return true;
|
|
1203
|
+
if (!(expire <= now + STORED_WINDOW_MS))
|
|
1204
|
+
return true;
|
|
1205
|
+
const minute = Host.minuteOf(expire);
|
|
1206
|
+
if (this.storedUnknown.has(minute))
|
|
1207
|
+
return true;
|
|
1208
|
+
return this.storedByMinute.get(minute)?.has(umid) === true;
|
|
1209
|
+
}
|
|
1210
|
+
resentGuardUntil = 0;
|
|
1211
|
+
resentInFlight = new Map();
|
|
1212
|
+
resentTimer;
|
|
1213
|
+
resentLoaded = false;
|
|
1214
|
+
resentLoading;
|
|
1215
|
+
rememberResent(replaced, replacement, expire) {
|
|
1216
|
+
const until = expire ? new Date(expire).getTime() : NaN;
|
|
1217
|
+
if (!Number.isFinite(until) || until <= Date.now())
|
|
1218
|
+
return;
|
|
1219
|
+
this.resentGuardUntil = Math.max(this.resentGuardUntil, until);
|
|
1220
|
+
this.resentInFlight.set(replaced, replacement);
|
|
1221
|
+
this.dbEventConnection
|
|
1222
|
+
.post({ _id: `resent:${replaced}`, to: replacement, until })
|
|
1223
|
+
.then(() => void this.resentInFlight.delete(replaced), (error) => {
|
|
1224
|
+
ActiveLogger.error({ replaced, error }, "Could not record a resent transaction; refusing a repost from memory only");
|
|
1225
|
+
setTimeout(() => this.resentInFlight.delete(replaced), Math.max(0, until - Date.now())).unref();
|
|
1226
|
+
});
|
|
1227
|
+
this.startResentHousekeeping();
|
|
1228
|
+
}
|
|
1229
|
+
async resentAs(umid) {
|
|
1230
|
+
if (this.resentLoading && !this.resentLoaded)
|
|
1231
|
+
await this.waitForResentList();
|
|
1232
|
+
if (Date.now() >= this.resentGuardUntil)
|
|
1233
|
+
return undefined;
|
|
1234
|
+
const inFlight = this.resentInFlight.get(umid);
|
|
1235
|
+
if (inFlight)
|
|
1236
|
+
return inFlight;
|
|
1237
|
+
const row = await this.dbEventConnection.get(`resent:${umid}`).catch(() => null);
|
|
1238
|
+
if (row === null || row === undefined)
|
|
1239
|
+
throw new Error("could not check whether the transaction was resent");
|
|
1240
|
+
return typeof row.to === "string" && Number(row.until) > Date.now() ? row.to : undefined;
|
|
1241
|
+
}
|
|
1242
|
+
loadResentGuard() {
|
|
1243
|
+
this.resentLoading = this.refreshResentGuard(false).then(() => undefined);
|
|
1244
|
+
return this.resentLoading;
|
|
1245
|
+
}
|
|
1246
|
+
resentWaiting;
|
|
1247
|
+
waitForResentList() {
|
|
1248
|
+
this.resentWaiting ??= (async () => {
|
|
1249
|
+
try {
|
|
1250
|
+
await this.resentLoading;
|
|
1251
|
+
for (let attempt = 0; !this.resentLoaded && attempt < RESENT_LOAD_WAIT_MS / 500; attempt++) {
|
|
1252
|
+
await new Promise((resolve) => setTimeout(resolve, 500).unref());
|
|
1253
|
+
await this.refreshResentGuard(false);
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
finally {
|
|
1257
|
+
this.resentWaiting = undefined;
|
|
1258
|
+
}
|
|
1259
|
+
if (!this.resentLoaded)
|
|
1260
|
+
throw new Error("the list of resent transactions has not been read yet");
|
|
1261
|
+
})();
|
|
1262
|
+
return this.resentWaiting;
|
|
1263
|
+
}
|
|
1264
|
+
async refreshResentGuard(retryLater = true) {
|
|
1265
|
+
let latest = 0;
|
|
1266
|
+
try {
|
|
1267
|
+
const list = await this.dbEventConnection.allDocs({ startkey: "resent:", endkey: "resent:x", include_docs: true });
|
|
1268
|
+
if (!list || !Array.isArray(list.rows))
|
|
1269
|
+
throw new Error("no answer");
|
|
1270
|
+
const now = Date.now();
|
|
1271
|
+
for (const row of list.rows) {
|
|
1272
|
+
const doc = row?.doc ?? row;
|
|
1273
|
+
if (!doc?._id)
|
|
1274
|
+
continue;
|
|
1275
|
+
if (Number(doc.until) > now)
|
|
1276
|
+
latest = Math.max(latest, Number(doc.until));
|
|
1277
|
+
else
|
|
1278
|
+
await this.dbEventConnection.purge(doc).catch(() => undefined);
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
catch (error) {
|
|
1282
|
+
if (retryLater) {
|
|
1283
|
+
ActiveLogger.warn(`Could not read the resent transactions, trying again in 30s: ${error?.message ?? error}`);
|
|
1284
|
+
setTimeout(() => void this.refreshResentGuard(), 30_000).unref();
|
|
1285
|
+
}
|
|
1286
|
+
return false;
|
|
1287
|
+
}
|
|
1288
|
+
this.resentGuardUntil = Math.max(latest, this.resentInFlight.size ? this.resentGuardUntil : 0);
|
|
1289
|
+
this.resentLoaded = true;
|
|
1290
|
+
if (latest > 0)
|
|
1291
|
+
this.startResentHousekeeping();
|
|
1292
|
+
return true;
|
|
1293
|
+
}
|
|
1294
|
+
startResentHousekeeping() {
|
|
1295
|
+
if (this.resentTimer)
|
|
1296
|
+
return;
|
|
1297
|
+
this.resentTimer = setInterval(() => void this.refreshResentGuard(), 60 * 60 * 1000);
|
|
1298
|
+
this.resentTimer.unref();
|
|
1299
|
+
}
|
|
1029
1300
|
release(umid, owner) {
|
|
1030
1301
|
if (owner && this.processPending[umid] && this.processPending[umid].entry !== owner) {
|
|
1031
1302
|
ActiveLogger.warn(umid, "Not releasing, pending entry belongs to another submission");
|
|
@@ -1052,20 +1323,49 @@ export class Host extends Home {
|
|
|
1052
1323
|
this.destroy(umid);
|
|
1053
1324
|
}
|
|
1054
1325
|
}, hold);
|
|
1055
|
-
|
|
1056
|
-
setTimeout(() => this.processQueue(), QUEUE_RECHECK_MS);
|
|
1057
|
-
}
|
|
1058
|
-
else {
|
|
1059
|
-
setImmediate(() => this.processQueue());
|
|
1060
|
-
}
|
|
1326
|
+
this.scheduleQueueRecheck(QUEUE_RECHECK_MS);
|
|
1061
1327
|
}
|
|
1062
1328
|
else {
|
|
1063
1329
|
ActiveLogger.warn(umid, "Trying to release unknown umid");
|
|
1064
1330
|
}
|
|
1065
1331
|
}
|
|
1066
1332
|
processingBLQ = false;
|
|
1333
|
+
lastQueueScan = 0;
|
|
1334
|
+
queueRecheckPending = false;
|
|
1335
|
+
scheduleQueueRecheck(ms) {
|
|
1336
|
+
if (this.queueRecheckPending)
|
|
1337
|
+
return;
|
|
1338
|
+
this.queueRecheckPending = true;
|
|
1339
|
+
const run = () => {
|
|
1340
|
+
this.queueRecheckPending = false;
|
|
1341
|
+
this.processQueue();
|
|
1342
|
+
};
|
|
1343
|
+
if (ms > 0) {
|
|
1344
|
+
setTimeout(run, ms);
|
|
1345
|
+
}
|
|
1346
|
+
else {
|
|
1347
|
+
setImmediate(run);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1067
1350
|
processQueue(next, internal = false) {
|
|
1068
1351
|
if (!this.processingBLQ) {
|
|
1352
|
+
const queued = this.busyLocksQueue.internal.length + this.busyLocksQueue.external.length;
|
|
1353
|
+
if (!queued) {
|
|
1354
|
+
if (next)
|
|
1355
|
+
this.hold(next);
|
|
1356
|
+
return;
|
|
1357
|
+
}
|
|
1358
|
+
const since = Date.now() - this.lastQueueScan;
|
|
1359
|
+
if (QUEUE_SCAN_MIN_MS > 0 && since < QUEUE_SCAN_MIN_MS) {
|
|
1360
|
+
if (next) {
|
|
1361
|
+
this.hold(next);
|
|
1362
|
+
}
|
|
1363
|
+
else {
|
|
1364
|
+
this.scheduleQueueRecheck(QUEUE_SCAN_MIN_MS - since);
|
|
1365
|
+
}
|
|
1366
|
+
return;
|
|
1367
|
+
}
|
|
1368
|
+
this.lastQueueScan = Date.now();
|
|
1069
1369
|
this.processingBLQ = true;
|
|
1070
1370
|
const checked = new Set();
|
|
1071
1371
|
if (this.busyLocksQueue.internal.length) {
|