@activeledger/activenetwork 2.15.6 → 4.3.3

package/lib/network/host.js

@@ -1,10 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Host = void 0;
-const http_1 = require("http");
 const child_process_1 = require("child_process");
 const fs_1 = require("fs");
 const path_1 = require("path");
+const net = require("net");
 const activeoptions_1 = require("@activeledger/activeoptions");
 const activecrypto_1 = require("@activeledger/activecrypto");
 const activelogger_1 = require("@activeledger/activelogger");
@@ -15,80 +15,72 @@ const index_1 = require("./index");
 const locker_1 = require("./locker");
 const cpus_1 = require("./cpus");
 const process = require("process");
-const RELEASE_SHUTDOWN_TIMEOUT = 5 * 60 * 1000;
-const RELEASE_DELETE_TIMEOUT = 1 * 60 * 1000;
-const TIMER_QUEUE_INTERVAL = 10 * 1000;
+const uWebSockets_js_1 = require("uWebSockets.js");
+const RELEASE_SHUTDOWN_TIMEOUT = 1 * 60 * 1000;
+const RELEASE_DELETE_TIMEOUT = 2 * 60 * 1000;
+const TIMER_QUEUE_INTERVAL = 2 * 1000;
 const GRACEFUL_PROC_SHUTDOWN = 7 * 60 * 1000;
 const KILL_PROC_SHUTDOWN = 2.5 * 1000;
+const MAX_RETRIES = 35;
+function peekBytes(chunks, n) {
+    if (chunks.length > 0 && chunks[0].length >= n) {
+        return chunks[0];
+    }
+    let total = 0;
+    let end = 0;
+    while (end < chunks.length && total < n) {
+        total += chunks[end].length;
+        end++;
+    }
+    return Buffer.concat(chunks.slice(0, end));
+}
 class Host extends home_1.Home {
-    constructor() {
-        super();
-        this.processPending = {};
-        this.cpuReady = 0;
-        this.busyLocksQueue = [];
-        let db = activeoptions_1.ActiveOptions.get("db", {});
-        this.dbConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.database);
-        this.dbConnection.info();
-        this.dbErrorConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.error);
-        this.dbErrorConnection.info();
-        this.dbEventConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.event);
-        this.dbEventConnection.info();
-        this.hybridHosts = activeoptions_1.ActiveOptions.get("hybrid", []);
-        if (this.hybridHosts.length) {
-            for (let i = this.hybridHosts.length; i--;) {
-                const hybrid = this.hybridHosts[i];
-                hybrid.docName = activecrypto_1.ActiveCrypto.Hash.getHash(hybrid.url + hybrid.auth);
-            }
+    p2pServer;
+    listenSocket;
+    api;
+    dbConnection;
+    dbErrorConnection;
+    dbEventConnection;
+    processPending = {};
+    cpuReady = 0;
+    hybridHosts;
+    busyLocksQueue = { internal: [], external: [] };
+    shutdown() {
+        if (this.listenSocket) {
+            (0, uWebSockets_js_1.us_listen_socket_close)(this.listenSocket);
+            this.listenSocket = null;
+            setTimeout(() => {
+                activelogger_1.ActiveLogger.info("Stopping HTTP Server");
+                this.api.close();
+                activelogger_1.ActiveLogger.info("Shutting down...");
+                process.exit(0);
+            }, 1300);
         }
-        this.api = (0, http_1.createServer)((req, res) => {
-            activelogger_1.ActiveLogger.debug(`Request - ${req.connection.remoteAddress} @ ${req.method}:${req.url}`);
-            if (req.method == "POST") {
-                const body = [];
-                req.on("data", (chunk) => {
-                    body.push(chunk);
-                });
-                req.on("end", async () => {
-                    let data = Buffer.concat(body);
-                    if (req.headers["content-encoding"] == "gzip" ||
-                        (data[0] == 0x1f && data[1] == 0x8b)) {
-                        try {
-                            data = await activeoptions_1.ActiveGZip.ungzip(data);
-                        }
-                        catch (_a) {
+    }
+    pending(entry, remoteAddr, internal = false, forceRestart = false, isP2P = false) {
+        return new Promise(async (resolve, reject) => {
+            if (entry.$broadcast && entry.$nodes) {
+                const nodeSpoofCheck = Object.keys(entry.$nodes);
+                if (nodeSpoofCheck.length) {
+                    for (let i = nodeSpoofCheck.length; i--;) {
+                        const nodeCheck = nodeSpoofCheck[i];
+                        if (!this.neighbourhood.checkFirewall(remoteAddr, nodeCheck)) {
+                            return reject("Bad Neighbour Payload");
                         }
                     }
-                    index_1.Endpoints.postConvertor(this, data.toString(), req.headers["x-activeledger-encrypt"] ||
-                        false)
-                        .then((body) => {
-                        this.processEndpoints(req, res, body.body, body.from);
-                    })
-                        .catch((error) => {
-                        this.writeResponse(res, error.statusCode || 500, JSON.stringify(error.content || {}), req.headers["accept-encoding"]);
-                    });
-                });
+                }
             }
-            else {
-                this.processEndpoints(req, res);
+            if (forceRestart && this.processPending[entry.$umid]) {
+                this.destroy(entry.$umid, true);
+                delete this.processPending[entry.$umid];
             }
-        });
-        const cpuTotal = cpus_1.PhysicalCores.count();
-        const latestSetupMsg = this.getLatestSetup();
-        for (let i = 0; i < cpuTotal; i++) {
-            const processor = this.createProcessor(cpuTotal);
-            this.processors.push(processor);
-            processor.send(latestSetupMsg);
-        }
-        this.standbyProcess = this.createProcessor(cpuTotal);
-        this.standbyProcess.send(latestSetupMsg);
-        this.processorIterator = this.processors[Symbol.iterator]();
-        this.timerQueue();
-    }
-    pending(entry, internal = false) {
-        return new Promise((resolve, reject) => {
             if (entry.$broadcast) {
                 if (this.processPending[entry.$umid]) {
-                    activelogger_1.ActiveLogger.debug("Broadcast Recieved : " + entry.$umid);
-                    if (this.processPending[entry.$umid].pid) {
+                    if (!this.processPending[entry.$umid]?.finished &&
+                        this.processPending[entry.$umid]?.pid &&
+                        !this.processPending[entry.$umid]?.entry?.$nodes?.[this.reference]
+                            ?.leader) {
+                        activelogger_1.ActiveLogger.debug(`Broadcast Recieved [${isP2P ? "P2P" : "HTTP"}] : ` + entry.$umid);
                         const processor = this.findProcessor(this.processPending[entry.$umid].pid);
                         if (processor) {
                             processor.send({
@@ -102,35 +94,327 @@ class Host extends home_1.Home {
                         else {
                         }
                     }
+                    else {
+                        const pendingEntry = this.processPending[entry.$umid]?.entry;
+                        if (pendingEntry) {
+                            delete entry.$nodes[this.reference];
+                            pendingEntry.$nodes = { ...pendingEntry.$nodes, ...entry.$nodes };
+                        }
+                    }
+                    if (!entry.$$noreply) {
+                        this.broadcast(entry.$umid, false, true);
+                    }
                     return resolve({
                         status: 200,
                         data: this.processPending[entry.$umid].entry,
+                        dontRelease: true,
                     });
                 }
             }
             if (!this.processPending[entry.$umid]) {
                 this.processPending[entry.$umid] = {
-                    entry: entry,
+                    entry,
                     resolve: (response) => {
-                        this.release({ entry, resolve: null, reject: null, pid: 0 });
-                        resolve(response);
+                        if (internal && this.processPending[entry.$umid]?.responded) {
+                            this.release(entry.$umid);
+                        }
+                        if (this.processPending[entry.$umid]) {
+                            this.processPending[entry.$umid].finished = true;
+                        }
+                        if (!this.processPending[entry.$umid]?.responded) {
+                            resolve(response);
+                            try {
+                                this.processPending[entry.$umid].responded = true;
+                                activelogger_1.ActiveLogger.debug("Client Response TX : " + entry.$umid);
+                            }
+                            catch { }
+                        }
                     },
                     reject: (response) => {
-                        this.release({ entry, resolve: null, reject: null, pid: 0 });
-                        reject(response);
+                        if (this.processPending[entry.$umid]) {
+                            this.processPending[entry.$umid].finished = true;
+                        }
+                        if (!this.processPending[entry.$umid]?.responded) {
+                            reject(response);
+                            try {
+                                this.processPending[entry.$umid].responded = true;
+                            }
+                            catch { }
+                        }
                     },
                     pid: 0,
+                    finished: false,
+                    responded: false,
                 };
+                if (entry.$tx.$expire && remoteAddr !== this.host) {
+                    if (await this.dbConnection.exists(`${entry.$umid}:umid`)) {
+                        if (entry.$nodes) {
+                            entry.$nodes[this.reference] = {
+                                vote: false,
+                                commit: false,
+                                error: `Transaction Exists : ${entry.$umid}`,
+                            };
+                        }
+                        else {
+                            entry.$nodes = {
+                                [this.reference]: {
+                                    vote: false,
+                                    commit: false,
+                                    error: `Transaction Exists : ${entry.$umid}`,
+                                },
+                            };
+                        }
+                        return this.processPending[entry.$umid].resolve({
+                            status: 200,
+                            data: entry,
+                        });
+                    }
+                }
                 this.processQueue(entry, internal);
+                if (internal && entry.$broadcast) {
+                    this.processPending[entry.$umid].responded = true;
+                    return resolve({
+                        status: 200,
+                        data: { ok: true },
+                        dontRelease: true,
+                    });
+                }
             }
             else {
                 return resolve({
                     status: 200,
                     data: this.processPending[entry.$umid].entry,
+                    dontRelease: true,
                 });
             }
         });
     }
+    constructor() {
+        super();
+        if (activeoptions_1.ActiveOptions.get("p2pStreamServer", false)) {
+            this.p2pServer = net.createServer((socket) => {
+                activelogger_1.ActiveLogger.info(`P2P TCP connection accepted from ${socket.remoteAddress}`);
+                let chunks = [];
+                let bufferLength = 0;
+                socket.on("data", async (data) => {
+                    chunks.push(data);
+                    bufferLength += data.length;
+                    while (bufferLength >= 44) {
+                        const header = peekBytes(chunks, 44);
+                        const length = header.readUInt32BE(40);
+                        if (bufferLength >= 44 + length) {
+                            const fullBuffer = Buffer.concat(chunks);
+                            const senderRef = fullBuffer.slice(0, 40).toString().trim();
+                            const item = fullBuffer.slice(44, 44 + length);
+                            const remaining = fullBuffer.slice(44 + length);
+                            chunks = remaining.length > 0 ? [remaining] : [];
+                            bufferLength = remaining.length;
+                            const tx = await activeoptions_1.ActiveClone.deserialize(item);
+                            this.processEndpoints({
+                                headers: { "X-Activeledger": senderRef },
+                                method: "POST",
+                                url: "/a/init",
+                                connection: { remoteAddress: socket.remoteAddress || "127.0.0.1" },
+                            }, null, tx, senderRef, true);
+                        }
+                        else {
+                            break;
+                        }
+                    }
+                });
+                socket.on("error", (err) => {
+                    activelogger_1.ActiveLogger.error(err, "P2P socket error");
+                });
+            });
+            const p2pPort = parseInt(utils_1.ActiveInterfaces.getBindingDetails("port")) + 1;
+            this.p2pServer.listen(p2pPort, () => {
+                activelogger_1.ActiveLogger.info(`P2P TCP server listening on port ${p2pPort}`);
+            });
+        }
+        let db = activeoptions_1.ActiveOptions.get("db", {});
+        this.dbConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.database);
+        this.dbConnection.info();
+        this.dbErrorConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.error);
+        this.dbErrorConnection.info();
+        this.dbEventConnection = new activeoptions_1.ActiveDSConnect(db.url + "/" + db.event);
+        this.dbEventConnection.info();
+        this.hybridHosts = activeoptions_1.ActiveOptions.get("hybrid", []);
+        if (this.hybridHosts.length) {
+            for (let i = this.hybridHosts.length; i--;) {
+                const hybrid = this.hybridHosts[i];
+                hybrid.docName = activecrypto_1.ActiveCrypto.Hash.getHash(hybrid.url + hybrid.auth);
+            }
+        }
+        this.api = (0, uWebSockets_js_1.App)();
+        this.api.any("/*", async (res, req) => {
+            res.onAborted(() => {
+                res.writable = false;
+            });
+            res.writable = true;
+            const headers = {
+                "Accept-Encoding": req.getHeader("accept-encoding"),
+                "X-Activeledger": req.getHeader("x-activeledger"),
+                "x-activeledger-encrypt": req.getHeader("x-activeledger-encrypt"),
+                "content-encoding": req.getHeader("content-encoding"),
+                "X-Bundle": req.getHeader("x-bundle"),
+            };
+            const method = req.getMethod().toUpperCase();
+            const url = req.getUrl();
+            let ipFrom = Buffer.from(res.getProxiedRemoteAddressAsText().byteLength
+                ? res.getProxiedRemoteAddressAsText()
+                : res.getRemoteAddressAsText()).toString();
+            if (ipFrom.indexOf(":") !== -1) {
+                const ip6 = this.parseIp6(ipFrom);
+                ipFrom =
+                    (ip6[6] >> 8) +
+                        "." +
+                        (ip6[6] & 0xff) +
+                        "." +
+                        (ip6[7] >> 8) +
+                        "." +
+                        (ip6[7] & 0xff);
+            }
+            if (method === "POST") {
+                let body = await this.readBuffer(res);
+                if (headers["content-encoding"] == "gzip" ||
+                    (body[0] == 0x1f && body[1] == 0x8b)) {
+                    try {
+                        body = await activeoptions_1.ActiveGZip.ungzip(body);
+                    }
+                    catch (e) {
+                    }
+                }
+                const bodyString = (await this.makeSureNotBuffer(body));
+                let bundles;
+                if (headers["X-Bundle"]) {
+                    bundles = bodyString.split(":$ALB:");
+                    res.cork(() => {
+                        res.writeStatus("200");
+                        res.writeHeader("Content-Type", "application/json");
+                        res.end("{}");
+                    });
+                    res.writable = false;
+                }
+                else {
+                    bundles = [bodyString];
+                }
+                for (let i = bundles.length; i--;) {
+                    if (bundles[i]) {
+                        index_1.Endpoints.postConvertor(this, bundles[i], headers["x-activeledger-encrypt"] || false)
+                            .then((body) => {
+                            this.processEndpoints({
+                                headers,
+                                method,
+                                url,
+                                connection: {
+                                    remoteAddress: ipFrom,
+                                },
+                            }, res, body.body, body.from);
+                        })
+                            .catch((error) => {
+                            activelogger_1.ActiveLogger.error(error, "Server POST Parser 500");
+                            this.writeResponse(res, error.statusCode || 500, JSON.stringify(error.content || { error: 1 }), headers["Accept-Encoding"]);
+                        })
+                            .finally(() => {
+                        });
+                    }
+                }
+            }
+            else {
+                if (method === "OPTIONS") {
+                    await this.writeResponse(res, 200, "{}", "", true);
+                }
+                else {
+                    await this.processEndpoints({
+                        headers,
+                        method,
+                        url,
+                        connection: {
+                            remoteAddress: ipFrom,
+                        },
+                    }, res);
+                }
+            }
+        });
+        const cpuTotal = parseInt(activeoptions_1.ActiveOptions.get("cpus", "0")) || cpus_1.PhysicalCores.count();
+        const latestSetupMsg = this.getLatestSetup();
+        for (let i = 0; i < cpuTotal; i++) {
+            const processor = this.createProcessor(cpuTotal);
+            this.processors.push(processor);
+            processor.send(latestSetupMsg);
+        }
+        this.standbyProcess = this.createProcessor(cpuTotal);
+        this.standbyProcess.send(latestSetupMsg);
+        this.processorIterator = this.processors[Symbol.iterator]();
+        this.timerQueue();
+    }
+    readBuffer(res) {
+        return new Promise((resolve, reject) => {
+            const chunks = [];
+            res.onData((ab, isLast) => {
+                if (res.aborted) {
+                    reject(new Error("Request aborted"));
+                    return;
+                }
+                if (ab.byteLength > 0) {
+                    const buffer = Buffer.allocUnsafe(ab.byteLength);
+                    buffer.set(new Uint8Array(ab));
+                    chunks.push(buffer);
+                }
+                if (isLast) {
+                    resolve(chunks.length === 0
+                        ? Buffer.alloc(0)
+                        : chunks.length === 1
+                            ? chunks[0]
+                            : Buffer.concat(chunks));
+                }
+            });
+        });
+    }
+    parseIp6(str) {
+        var ar = new Array();
+        for (var i = 0; i < 8; i++)
+            ar[i] = 0;
+        if (str == "::")
+            return ar;
+        var sar = str.split(":");
+        var slen = sar.length;
+        if (slen > 8)
+            slen = 8;
+        var j = 0;
+        for (var i = 0; i < slen; i++) {
+            if (i && sar[i] == "") {
+                j = 9 - slen + i;
+                continue;
+            }
+            ar[j] = parseInt("0x0" + sar[i]);
+            j++;
+        }
+        return ar;
+    }
+    async makeSureNotBuffer(obj) {
+        if (obj.type === "Buffer" && obj.data?.length) {
+            if (obj.data[0] == 0x1f && obj.data[1] == 0x8b) {
+                return (await activeoptions_1.ActiveGZip.ungzip(Buffer.from(obj.data))).toString();
+            }
+            return Buffer.from(obj.data).toString();
+        }
+        if (Buffer.isBuffer(obj)) {
+            if (obj[0] == 0x1f && obj[1] == 0x8b) {
+                return (await activeoptions_1.ActiveGZip.ungzip(obj)).toString();
+            }
+            const tmp = obj.toString();
+            if (tmp.startsWith('{"type":"Buffer"')) {
+                const asBufferObj = JSON.parse(tmp);
+                if (asBufferObj.data[0] == 0x1f && asBufferObj.data[1] == 0x8b) {
+                    return (await activeoptions_1.ActiveGZip.ungzip(Buffer.from(asBufferObj.data))).toString();
+                }
+                return Buffer.from(asBufferObj.data).toString();
+            }
+            return obj.toString();
+        }
+        return obj;
+    }
     getLatestSetup() {
         return {
             type: "setup",
@@ -154,29 +438,33 @@ class Host extends home_1.Home {
         pFork.stop = false;
         let unloadHandled = false;
         const unloadProcessorSafely = (...error) => {
-            if (unloadHandled) {
-                activelogger_1.ActiveLogger.fatal(error, "Processor Crashed - Already Shutting Down");
+            if (unloadHandled || !this.listenSocket) {
             }
             else {
                 pFork.stop = unloadHandled = true;
                 activelogger_1.ActiveLogger.fatal(error, "Processor Crashed");
                 this.processors.push(this.standbyProcess);
                 this.standbyProcess = this.createProcessor();
-                this.standbyProcess.send(this.getLatestSetup());
+                setTimeout(() => {
+                    this.standbyProcess.send(this.getLatestSetup(), (e) => {
+                        if (e) {
+                            activelogger_1.ActiveLogger.fatal(e, "Issue with new Standby Process");
+                        }
+                    });
+                }, 1000);
                 activelogger_1.ActiveLogger.fatal(pFork, "Will Gracefully Shutdown in " + GRACEFUL_PROC_SHUTDOWN);
                 setTimeout(() => {
                     activelogger_1.ActiveLogger.fatal(pFork, "Starting Graceful Shutdown");
                     const pendings = Object.keys(this.processPending);
                     pendings.forEach((key) => {
                         let pending = this.processPending[key];
-                        if ((pending === null || pending === void 0 ? void 0 : pending.pid) === pFork.pid) {
+                        if (pending?.pid === pFork.pid) {
                             pending.entry.$nodes[this.reference].error =
                                 "(Contract Thread Error) Unknown - Try Again";
                             pending.resolve({
                                 status: 200,
                                 data: pending.entry,
                             });
-                            this.release(pending);
                         }
                     });
                     activelogger_1.ActiveLogger.fatal(pFork, "Will Kill in " + KILL_PROC_SHUTDOWN);
@@ -194,10 +482,9 @@ class Host extends home_1.Home {
             }
         };
         pFork.on("message", (m) => {
-            var _a;
             const pending = this.processPending[m.data.umid];
             if (pending) {
-                if (m.data.nodes) {
+                if (m.data.nodes && !m.data.early) {
                     pending.entry.$nodes = {
                         ...pending.entry.$nodes,
                         ...m.data.nodes,
@@ -216,12 +503,10 @@ class Host extends home_1.Home {
                         return;
                     pending.resolve({
                         status: 200,
-                        data: pending.entry,
+                        data: m.data.entry
+                            ? { ...pending.entry, ...m.data.entry }
+                            : pending.entry,
                     });
-                    this.release(pending);
-                    if (m.data && this.hybridHosts.length) {
-                        this.processHybridNodes(pending.entry);
-                    }
                     break;
                 case "commited":
                     if (!pending)
@@ -230,9 +515,8 @@ class Host extends home_1.Home {
                         status: 200,
                         data: { ...pending.entry, ...m.data.entry },
                     });
-                    this.release(pending);
                     if (this.hybridHosts.length) {
-                        this.processHybridNodes(pending.entry, (_a = m.data.entry) === null || _a === void 0 ? void 0 : _a.$streams);
+                        this.processHybridNodes(pending.entry, m.data.entry?.$streams);
                     }
                     break;
                 case "broadcast":
@@ -244,8 +528,10 @@ class Host extends home_1.Home {
                 case "ready":
                     if (cpuTotal) {
                         this.cpuReady++;
-                        if (!this.api.listening && this.cpuReady >= cpuTotal) {
-                            this.api.listen(utils_1.ActiveInterfaces.getBindingDetails("port"), () => {
+                        if (!this.listeing && this.cpuReady >= cpuTotal) {
+                            this.listeing = true;
+                            this.api.listen(utils_1.ActiveInterfaces.getBindingDetails("port"), (token) => {
+                                this.listenSocket = token;
                                 activelogger_1.ActiveLogger.info("Activeledger listening on port " +
                                     utils_1.ActiveInterfaces.getBindingDetails("port"));
                             });
@@ -259,10 +545,12 @@ class Host extends home_1.Home {
                             status: 200,
                             data: { ...pending.entry, ...m.data.entry },
                         });
-                        this.release(pending);
                     }
-                    unloadProcessorSafely("unhandledrejection - Already Handled, Tidying up processes");
+                    if (!m.data.recoverable) {
+                        unloadProcessorSafely("unhandledrejection - Already Handled, Tidying up processes");
+                    }
                     break;
+                case "contractData":
                 case "contractLatestVersion":
                     this.processors.forEach((processor) => {
                         processor.send(m);
@@ -279,6 +567,7 @@ class Host extends home_1.Home {
         pFork.on("error", unloadProcessorSafely);
         return pFork;
     }
+    listeing = false;
     reload() {
         activeoptions_1.ActiveOptions.extendConfig()
             .then((config) => {
@@ -307,57 +596,73 @@ class Host extends home_1.Home {
             activelogger_1.ActiveLogger.info(e, "Failed to reload Neighbourhood");
         });
     }
-    destroy(umid) {
-        var _a;
+    destroy(umid, skipTimeout = false) {
         if (this.processPending[umid]) {
             this.processPending[umid].shutdown = true;
-            (_a = this.findProcessor(this.processPending[umid].pid)) === null || _a === void 0 ? void 0 : _a.send({
+            this.findProcessor(this.processPending[umid].pid)?.send({
                 type: "destory",
                 data: {
                     umid,
+                    skipTimeout,
                 },
             });
-            setTimeout(() => {
-                delete this.processPending[umid];
-            }, RELEASE_DELETE_TIMEOUT);
+            if (!skipTimeout) {
+                setTimeout(() => {
+                    delete this.processPending[umid];
+                }, RELEASE_DELETE_TIMEOUT);
+            }
         }
     }
-    broadcast(umid, early = false) {
-        var _a;
-        if (((_a = this.processPending[umid]) === null || _a === void 0 ? void 0 : _a.entry) &&
+    broadcast(umid, early = false, noreply = false) {
+        if (this.processPending[umid] && this.processPending[umid].broadcasting) {
+            return;
+        }
+        if (this.processPending[umid]?.entry &&
             this.processPending[umid].entry.$broadcast &&
             this.processPending[umid].entry.$nodes &&
-            this.processPending[umid].entry.$nodes[this.reference]) {
-            activelogger_1.ActiveLogger.debug("Broadcasting TX : " + umid);
+            (early ||
+                (this.processPending[umid].entry.$nodes[this.reference] &&
+                    "vote" in this.processPending[umid].entry.$nodes[this.reference]))) {
+            this.processPending[umid].broadcasting = true;
+            activelogger_1.ActiveLogger.debug(`Broadcasting TX ($$NR ${noreply}) : ` + umid);
             let neighbourhood = this.neighbourhood.get();
             let nodes = this.neighbourhood.keys();
-            let i = nodes.length;
             let promises = [];
-            const data = !early
+            const selfEntry = this.processPending[umid].entry.$nodes[this.reference];
+            const data = selfEntry && !selfEntry.early
                 ? Object.assign(this.processPending[umid].entry, {
                     $nodes: {
-                        [this.reference]: this.processPending[umid].entry.$nodes[this.reference],
+                        [this.reference]: selfEntry,
                     },
                 })
                 : Object.assign(this.processPending[umid].entry, {
                     $nodes: {},
                 });
             if (data) {
-                while (i--) {
+                data.$$noreply = noreply;
+                if (early) {
+                    data.$$noreply = true;
+                }
+                for (let i = nodes.length; i--;) {
                     let node = neighbourhood[nodes[i]];
-                    if (node.isHome && node.reference !== this.reference) {
-                        promises.push(node.knock("init", data));
+                    if (node.isHome &&
+                        node.reference !==
+                            this
+                                .reference) {
+                        promises.push(node.knock("init", data, false, 0, true));
                     }
                 }
+                Promise.all(promises)
+                    .then(() => {
+                    this.processPending[umid].broadcasting = false;
+                })
+                    .catch(() => {
+                    this.processPending[umid].broadcasting = false;
+                    setTimeout(() => {
+                        this.broadcastResolver(umid);
+                    }, 250);
+                });
             }
-            Promise.all(promises)
-                .then(() => {
-            })
-                .catch(() => {
-                setTimeout(() => {
-                    this.broadcastResolver(umid);
-                }, 500);
-            });
         }
     }
     broadcastResolver(umid) {
@@ -377,14 +682,18 @@ class Host extends home_1.Home {
                 },
                 reason: "Failed to rebroadcast while in memory",
             };
-            this.dbErrorConnection.post(doc);
+            this.dbErrorConnection.post(doc).catch((error) => {
+                activelogger_1.ActiveLogger.error(error, "Failed to write rebroadcast-failure error document");
+            });
         }
     }
     labelOrKey(txIO) {
         const keys = Object.keys(txIO || {});
         const out = [];
         for (let i = keys.length; i--;) {
-            out.push(this.filterPrefix(txIO[keys[i]].$stream || keys[i]));
+            if (!txIO[keys[i]].$sigOnly) {
+                out.push(this.filterPrefix(txIO[keys[i]].$stream || keys[i]));
+            }
         }
         return out;
     }
@@ -395,19 +704,27 @@ class Host extends home_1.Home {
         return streamId;
     }
     hold(v, retries = 0) {
-        if (v.$selfsign ||
-            locker_1.Locker.hold([
-                ...new Set([
-                    ...this.labelOrKey(v.$tx.$i),
-                    ...this.labelOrKey(v.$tx.$o),
-                ]),
-            ])) {
+        if (!v.$$labelOrKey) {
+            const outputs = this.labelOrKey(v.$tx.$o);
+            if (!v.$selfsign) {
+                v.$$labelOrKey = [
+                    ...new Set([...this.labelOrKey(v.$tx.$i), ...outputs]),
+                ];
+            }
+            else {
+                v.$$labelOrKey = outputs;
+            }
+        }
+        if (locker_1.Locker.hold(v.$$labelOrKey, v.$umid)) {
             const robin = this.getRobin();
             if (!this.processPending[v.$umid].entry.$nodes)
-                this.processPending[v.$umid].entry.$nodes = {};
+                if (!this.processPending[v.$umid].entry.$nodes) {
+                    this.processPending[v.$umid].entry.$nodes = {};
+                }
             this.processPending[v.$umid].entry.$nodes[home_1.Home.reference] = {
                 vote: false,
                 commit: false,
+                early: true
             };
             this.processPending[v.$umid].pid = robin.pid || 0;
             robin.send({
@@ -417,60 +734,70 @@ class Host extends home_1.Home {
             return true;
         }
         else {
-            if (retries === 0) {
-                this.busyLocksQueue.push({
-                    entry: v,
-                    retry: 1,
-                });
-            }
-            else {
-                const internal = v.$revs ? true : false;
-                const maxRetries = internal
-                    ? 2
-                    : activeoptions_1.ActiveOptions.get("queue_retry", 5);
-                if (retries > maxRetries) {
-                    if (internal) {
-                        if (!v.$nodes) {
-                            v.$nodes = {};
-                        }
-                        v.$nodes[this.reference] = {
+            if (v.$nolock) {
+                if (v.$broadcast) {
+                    this.processPending[v.$umid].entry.$nodes = {
+                        [this.reference]: {
                             vote: false,
                             commit: false,
                             error: "Busy Locks",
-                        };
-                        const doc = {
-                            code: 1100,
-                            processed: false,
-                            umid: v.$umid,
-                            transaction: v,
-                            locker: locker_1.Locker.getLocks(),
-                            reason: "Internal Busy Locks",
-                        };
-                        this.dbErrorConnection.post(doc);
-                        if (!v.$broadcast && home_1.Home.right.reference != v.$origin) {
-                            (async () => {
-                                const next = await home_1.Home.right.knock("init", v);
+                        },
+                    };
+                    this.broadcast(v.$umid, false, true);
+                    activelogger_1.ActiveLogger.warn(this.processPending[v.$umid], `${v.$umid} busy lock broadcasting that fact`);
+                }
+                this.processPending[v.$umid].reject({
+                    status: 100,
+                    error: "Busy Locks",
+                });
+            }
+            else {
+                if (retries === 0) {
+                    (v.$revs
+                        ? this.busyLocksQueue.internal
+                        : this.busyLocksQueue.external).push({
+                        running: false,
+                        entry: v,
+                        retry: 1,
+                    });
+                }
+                else {
+                    const internal = v.$revs ? true : false;
+                    if (retries > activeoptions_1.ActiveOptions.get("queue_retry", MAX_RETRIES)) {
+                        if (internal) {
+                            if (!v.$nodes) {
+                                v.$nodes = {};
+                            }
+                            v.$nodes[this.reference] = {
+                                vote: false,
+                                commit: false,
+                                error: "IBL01",
+                            };
+                            if (!v.$broadcast && home_1.Home.right.reference != v.$origin) {
+                                (async () => {
+                                    const next = await home_1.Home.right.knock("init", v);
+                                    this.processPending[v.$umid].resolve({
+                                        status: 200,
+                                        data: { ...v, ...next.data },
+                                    });
+                                })();
+                            }
+                            else {
+                                this.broadcast(v.$umid, false, true);
                                 this.processPending[v.$umid].resolve({
                                     status: 200,
-                                    data: { ...v, ...next.data },
+                                    data: v,
                                 });
-                            })();
+                            }
                         }
                         else {
-                            this.processPending[v.$umid].resolve({
-                                status: 200,
-                                data: v,
+                            this.processPending[v.$umid].reject({
+                                status: 100,
+                                error: "Busy Locks",
                             });
                         }
+                        return true;
                     }
-                    else {
-                        this.processPending[v.$umid].reject({
-                            status: 100,
-                            error: "Busy Locks",
-                        });
-                    }
-                    this.release(this.processPending[v.$umid]);
-                    return true;
                 }
             }
             return false;
@@ -487,36 +814,68 @@ class Host extends home_1.Home {
         }
         return robin;
     }
-    release(pending) {
-        locker_1.Locker.release([
-            ...this.labelOrKey(pending.entry.$tx.$i),
-            ...this.labelOrKey(pending.entry.$tx.$o),
-        ]);
-        setTimeout(() => {
-            if (pending.entry) {
-                this.destroy(pending.entry.$umid);
-            }
-        }, RELEASE_SHUTDOWN_TIMEOUT);
-        setTimeout(() => {
-            this.processQueue();
-        }, 100);
+    release(umid) {
+        if (this.processPending[umid]) {
+            const entry = this.processPending[umid].entry;
+            locker_1.Locker.release(entry.$$labelOrKey || [
+                ...this.labelOrKey(entry.$tx.$i),
+                ...this.labelOrKey(entry.$tx.$o),
+            ], entry.$umid);
+            setTimeout(() => {
+                if (entry) {
+                    this.destroy(entry.$umid);
+                }
+            }, RELEASE_SHUTDOWN_TIMEOUT);
+            setTimeout(() => {
+                this.processQueue();
+            }, 200);
+        }
+        else {
+            activelogger_1.ActiveLogger.warn(umid, "Trying to release unknown umid");
+        }
     }
+    processingBLQ = false;
     processQueue(next, internal = false) {
-        let skipped = false;
-        if (next && internal && !next.$broadcast) {
-            this.hold(next);
-            skipped = true;
-        }
-        if (this.busyLocksQueue.length) {
-            for (let i = 0; i < this.busyLocksQueue.length; i++) {
-                if (this.hold(this.busyLocksQueue[i].entry, this.busyLocksQueue[i].retry++)) {
-                    delete this.busyLocksQueue[i];
+        if (!this.processingBLQ) {
+            this.processingBLQ = true;
+            const checked = new Set();
+            if (this.busyLocksQueue.internal.length) {
+                const stillPending = [];
+                busyQueueInternal: for (let i = 0; i < this.busyLocksQueue.internal.length; i++) {
+                    const labelOrKey = this.busyLocksQueue.internal[i].entry.$$labelOrKey;
+                    if (labelOrKey?.length) {
+                        if (labelOrKey.some((io) => checked.has(io))) {
+                            continue busyQueueInternal;
+                        }
+                        labelOrKey.forEach((io) => checked.add(io));
+                    }
+                    if (!this.hold(this.busyLocksQueue.internal[i].entry, this.busyLocksQueue.internal[i].retry++)) {
+                        stillPending.push(this.busyLocksQueue.internal[i]);
+                    }
                 }
+                this.busyLocksQueue.internal = stillPending;
+            }
+            if (next && internal) {
+                this.hold(next);
             }
-            this.busyLocksQueue = this.busyLocksQueue.filter((n) => n);
+            if (this.busyLocksQueue.external.length) {
+                const stillPending = [];
+                for (let i = 0; i < this.busyLocksQueue.external.length; i++) {
+                    if (!this.hold(this.busyLocksQueue.external[i].entry, this.busyLocksQueue.external[i].retry++)) {
+                        stillPending.push(this.busyLocksQueue.external[i]);
+                    }
+                }
+                this.busyLocksQueue.external = stillPending;
+            }
+            if (next && !internal) {
+                this.hold(next);
+            }
+            this.processingBLQ = false;
         }
-        if (next && !skipped) {
-            this.hold(next);
+        else {
+            if (next) {
+                this.hold(next);
+            }
         }
     }
     timerQueue() {
@@ -550,12 +909,12 @@ class Host extends home_1.Home {
                             const data = response.data;
                             if (data.status !== "ok") {
                                 const updated = [
-                                    ...((activityStreams === null || activityStreams === void 0 ? void 0 : activityStreams.new) || []),
-                                    ...((activityStreams === null || activityStreams === void 0 ? void 0 : activityStreams.updated) || []),
+                                    ...(activityStreams?.new || []),
+                                    ...(activityStreams?.updated || []),
                                 ].map((stream) => stream.id);
-                                const input = tx.$tx.$i
-                                    ? this.hybridLabelKeyId(tx.$tx.$i)
-                                    : [];
+                                const input = tx.$tx.$i ?
+                                    this.hybridLabelKeyId(tx.$tx.$i) :
+                                    [];
                                 const output = tx.$tx.$o
                                     ? this.hybridLabelKeyId(tx.$tx.$o)
                                     : [];
@@ -570,8 +929,8 @@ class Host extends home_1.Home {
                                     }
                                 }
                                 const tmp = [];
-                                for (let i = keys.length; i--;) {
-                                    tmp.push(keys[i] + ":stream");
+                                for (const key of keys) {
+                                    tmp.push(key + ":stream");
                                 }
                                 keys.push(...tmp);
                                 return this.dbConnection
@@ -623,15 +982,16 @@ class Host extends home_1.Home {
             return streams;
         }
     }
-    processEndpoints(req, res, body, from) {
-        let requester = req.headers["x-activeledger"] || "NA";
+    processEndpoints(req, res, body, from, isP2P = false) {
+        let requester = req.headers["X-Activeledger"] || "NA";
         let response;
-        let gzipAccepted = req.headers["accept-encoding"];
+        const started = Date.now();
+        let gzipAccepted = req.headers["Accept-Encoding"];
         switch (req.method) {
             case "GET":
                 switch (req.url) {
                     case "/a/locks":
-                        return this.writeResponse(res, 200, JSON.stringify(locker_1.Locker.getLocks()), gzipAccepted);
+                        return this.writeResponse(res, 200, JSON.stringify({ L: locker_1.Locker.getLocks(), Q: this.busyLocksQueue }), gzipAccepted);
                     case "/a/locks/check":
                         locker_1.Locker.checkLocks();
                         return this.writeResponse(res, 200, JSON.stringify({ checked: true }), gzipAccepted);
@@ -639,15 +999,39 @@ class Host extends home_1.Home {
                         response = index_1.Endpoints.status(this, requester);
                         break;
                     case "/a/all":
-                        if (this.firewallCheck(requester, req)) {
+                        if (this.firewallCheck(requester, req.connection.remoteAddress)) {
                             response = index_1.Endpoints.all(this.dbConnection);
                         }
                         else {
                             return this.writeResponse(res, 403, "Forbidden", gzipAccepted);
                         }
                         break;
+                    case "/a/admin-reload":
+                        if (!activeoptions_1.ActiveOptions.get("remote", false)) {
+                            return this.writeResponse(res, 200, JSON.stringify({
+                                status: "failed",
+                                message: "remote disabled"
+                            }), gzipAccepted);
+                        }
+                        try {
+                            activeoptions_1.ActiveOptions.parseConfig();
+                            activelogger_1.ActiveLogger.enableDebug = activeoptions_1.ActiveOptions.get("debug", false);
+                            this.reload();
+                            activelogger_1.ActiveLogger.info("Host and child processes config.json reloaded successfully via admin API");
+                            return this.writeResponse(res, 200, JSON.stringify({
+                                status: "success",
+                                build: activeoptions_1.ActiveOptions.get("build", 0),
+                            }), gzipAccepted);
+                        }
+                        catch (error) {
+                            activelogger_1.ActiveLogger.error(error, "Admin reload error");
+                            return this.writeResponse(res, 200, JSON.stringify({
+                                status: "failed",
+                                message: error.message || "Internal Error"
+                            }), gzipAccepted);
+                        }
                     default:
-                        if (this.firewallCheck(requester, req)) {
+                        if (this.firewallCheck(requester, req.connection.remoteAddress)) {
                             if (req.url) {
                                 let match = req.url.substr(0, 7);
                                 switch (match) {
@@ -673,41 +1057,27 @@ class Host extends home_1.Home {
             case "POST":
                 switch (req.url) {
                     case "/":
-                        response = index_1.Endpoints.ExternalInitalise(this, body, req.connection.remoteAddress || "unknown");
+                        response = index_1.Endpoints.ExternalInitalise(this, body, req.connection.remoteAddress || "unknown", this.dbConnection);
                         break;
                     case "/a/encrypt":
                         response = index_1.Endpoints.ExternalEncrypt(this, body, req.headers["x-activeledger-encrypt"] ||
                             false, this.dbConnection);
                         break;
                     case "/a/init":
-                        if (this.firewallCheck(requester, req)) {
-                            response = index_1.Endpoints.InternalInitalise(this, body);
+                        if (this.firewallCheck(requester, req.connection.remoteAddress)) {
+                            response = index_1.Endpoints.InternalInitalise(this, body, req.connection.remoteAddress, false, isP2P);
                         }
                         else {
                             return this.writeResponse(res, 403, "Forbidden", gzipAccepted);
                         }
                         break;
                     case "/a/stream":
-                        if (this.firewallCheck(requester, req)) {
-                            response = index_1.Endpoints.streams(this.dbConnection, body);
-                        }
-                        else {
-                            return this.writeResponse(res, 403, "Forbidden", gzipAccepted);
-                        }
+                        response = index_1.Endpoints.streams(this.dbConnection, body);
                         break;
                     default:
                         return this.writeResponse(res, 404, "Not Found", gzipAccepted);
                 }
                 break;
-            case "OPTIONS":
-                res.writeHead(200, {
-                    "Access-Control-Allow-Origin": "*",
-                    "Access-Control-Allow-Methods": "GET, POST",
-                    "Access-Control-Allow-Headers": req.headers["access-control-request-headers"] || "*",
-                    "X-Powered-By": "Activeledger",
-                });
-                res.end();
-                return;
             default:
                 return this.writeResponse(res, 404, "Not Found", gzipAccepted);
         }
@@ -722,31 +1092,52 @@ class Host extends home_1.Home {
                     $enc: true,
                 };
             }
+            if (data.$umid) {
+                const TT = Date.now() - started;
+                if (TT > 5) {
+                    activelogger_1.ActiveLogger.info(`[RR] ${data.$umid ? data.$umid : "No Umid"} : S=${started}, TT=${TT}ms ${TT > 30000 ? "TTLR" : " OK"}`);
+                }
+            }
             this.writeResponse(res, response.statusCode, JSON.stringify(data), gzipAccepted);
         })
             .catch((error) => {
+            activelogger_1.ActiveLogger.error(error, "Failed to send response back");
+            activelogger_1.ActiveLogger.info(`[RR] ERROR : S=${started}, TT=${Date.now() - started}ms`);
             this.writeResponse(res, error.statusCode || 500, JSON.stringify(error.content || "Something has gone wrong"), gzipAccepted);
         });
     }
-    async writeResponse(res, statusCode, content, encoding) {
-        let headers = {
-            "Content-Type": "application/json",
-            "Content-Encoding": "none",
-            "Access-Control-Allow-Origin": "*",
-            "X-Powered-By": "Activeledger",
-        };
-        if (encoding == "gzip") {
-            headers["Content-Encoding"] = "gzip";
-            content = await activeoptions_1.ActiveGZip.gzip(content);
+    async writeResponse(res, statusCode, content, encoding, cors = false) {
+        if (!res || !res.writable) {
+            return;
         }
-        res.writeHead(statusCode, headers);
-        res.write(content);
-        res.end();
+        if (content) {
+            if (encoding == "gzip") {
+                content = await activeoptions_1.ActiveGZip.gzip(content);
+            }
+        }
+        res.cork(() => {
+            res.writeStatus(`${statusCode}`);
+            res.writeHeader("Access-Control-Allow-Origin", "*");
+            res.writeHeader("X-Content-Type-Options", "nosniff");
+            res.writeHeader("X-Frame-Options", "DENY");
+            res.writeHeader("Strict-Transport-Security", "max-age=63072000; includeSubDomains; preload");
+            res.writeHeader("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none';");
+            if (cors) {
+                res.writeHeader("Access-Control-Allow-Methods", "GET, POST");
+                res.writeHeader("Access-Control-Allow-Headers", "*");
+            }
+            if (content) {
+                res.writeHeader("Content-Type", "application/json");
+                if (encoding == "gzip") {
+                    res.writeHeader("Content-Encoding", "gzip");
+                }
+                res.end(content);
+                res.writable = false;
+            }
+        });
     }
-    firewallCheck(requester, req) {
-        return (requester !== "NA" &&
-            this.neighbourhood.checkFirewall(req.headers["x-forwarded-for"] ||
-                req.connection.remoteAddress));
+    firewallCheck(requester, remoteAddr) {
+        return this.neighbourhood.checkFirewall(remoteAddr, requester);
     }
 }
 exports.Host = Host;