@activeledger/activenetwork 2.15.7 → 4.3.3

package/lib/network/endpoints.js

@@ -2,132 +2,402 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Endpoints = void 0;
 const activeoptions_1 = require("@activeledger/activeoptions");
+const activeutilities_1 = require("@activeledger/activeutilities");
 const activelogger_1 = require("@activeledger/activelogger");
 const activedefinitions_1 = require("@activeledger/activedefinitions");
 const activecrypto_1 = require("@activeledger/activecrypto");
 const home_1 = require("./home");
-const neighbourhood_1 = require("./neighbourhood");
 const maintain_1 = require("./maintain");
+const locker_1 = require("./locker");
+const MAX_COUNTERS = 10;
 class Endpoints {
-    static ExternalInitalise(host, body, ip) {
-        return new Promise((resolve, reject) => {
-            if (body && activedefinitions_1.ActiveDefinitions.LedgerTypeChecks.isEntry(body)) {
-                let tx = body;
-                tx.$datetime = new Date();
-                if (tx.$tx.$expire && new Date(tx.$tx.$expire) <= tx.$datetime) {
-                    return resolve(this.successfulFailure(`Transaction Expired : ${tx.$tx.$expire}`));
-                }
-                if (!tx.$sigs) {
-                    return resolve(this.successfulFailure(`$sigs not found`));
-                }
-                tx.$origin = host.reference;
-                tx.$umid = activecrypto_1.ActiveCrypto.Hash.getHash(JSON.stringify(tx));
-                tx.$remoteAddr = ip;
-                if (tx.$multi) {
-                    return resolve({
-                        statusCode: 400,
-                        content: "Multiple Transaction Not Implemented",
-                    });
-                }
-                if (host.neighbourhood.count() < 4) {
-                    tx.$broadcast = false;
-                }
-                else if (!tx.$territoriality && !tx.$broadcast) {
-                    tx.$broadcast = true;
-                }
-                Endpoints.DirectInternalInitalise(host, tx)
-                    .then((response) => {
-                    if (response.status == "200" && !response.data.error) {
-                        let tx = response.data;
-                        let summary = {
-                            total: 0,
-                            vote: 0,
-                            commit: 0,
-                        };
-                        let responses = [];
-                        let nodes = Object.keys(tx.$nodes);
-                        for (let i = nodes.length; i--;) {
-                            summary.total++;
-                            if (tx.$nodes[nodes[i]].vote)
-                                summary.vote++;
-                            if (tx.$nodes[nodes[i]].commit)
-                                summary.commit++;
-                            if (tx.$nodes[nodes[i]].error) {
-                                if (summary.errors) {
-                                    summary.errors.push(tx.$nodes[nodes[i]].error);
-                                }
-                                else {
-                                    summary.errors = [tx.$nodes[nodes[i]].error];
-                                }
-                            }
-                            if (tx.$nodes[nodes[i]].return) {
-                                responses.push(tx.$nodes[nodes[i]].return);
-                            }
-                            if (!tx.$streams && tx.$nodes[nodes[i]].streams) {
-                                tx.$streams = tx.$nodes[nodes[i]].streams;
-                            }
+    static rebaseThrottle = 0;
+    static ExternalInitalise(host, body, ip, db) {
+        return new Promise(async (resolve, reject) => {
+            const process = (body) => {
+                return new Promise(async (resolve, reject) => {
+                    if (body && activedefinitions_1.ActiveDefinitions.LedgerTypeChecks.isEntry(body)) {
+                        let tx = body;
+                        const now = new Date();
+                        if (tx.$datetime) {
+                            this.successfulFailure(`$datetime cannot be preset`);
                         }
-                        if (tx.$broadcast && summary.vote) {
-                            const percent = tx.$unanimous
-                                ? 100
-                                : activeoptions_1.ActiveOptions.get("consensus", {}).reached;
-                            if ((summary.vote / summary.total) * 100 >= percent || false) {
-                                summary.commit = summary.vote;
+                        if (tx.$tx.$expire) {
+                            if (new Date(tx.$tx.$expire) <= now) {
+                                return resolve(this.successfulFailure(`Transaction Expired : ${tx.$tx.$expire}`));
+                            }
+                            else {
+                                tx.$umid = activecrypto_1.ActiveCrypto.Hash.getHash(JSON.stringify(tx));
+                                if (await db.exists(`${tx.$umid}:umid`)) {
+                                    return resolve(this.successfulFailure(`Transaction Exists : ${tx.$umid}`));
+                                }
+                                tx.$datetime = now;
                             }
                         }
-                        let output = {
-                            $umid: tx.$umid,
-                            $summary: summary,
-                            $streams: tx.$streams,
-                        };
-                        if (responses.length) {
-                            output.$responses = responses;
+                        else {
+                            tx.$datetime = now;
+                            tx.$umid = activecrypto_1.ActiveCrypto.Hash.getHash(JSON.stringify(tx));
                         }
-                        if (activeoptions_1.ActiveOptions.get("debug", false)) {
-                            output.$debug = tx;
+                        if (!tx.$sigs) {
+                            return resolve(this.successfulFailure(`$sigs not found`));
                         }
-                        return resolve({
-                            statusCode: 200,
-                            content: output,
-                        });
-                    }
-                    else {
-                        if (response.rebroadcasted) {
-                            return resolve({
-                                statusCode: 200,
-                                content: response.data,
-                            });
+                        tx.$origin = host.reference;
+                        tx.$remoteAddr = ip;
+                        if (!tx.$territoriality && !tx.$broadcast) {
+                            tx.$broadcast = true;
                         }
-                        else {
-                            return resolve({
-                                statusCode: response.status,
-                                content: response.data,
+                        activelogger_1.ActiveLogger.debug("Client Sent TX : " + tx.$umid);
+                        const resendable = (initTx, counter = 0) => {
+                            Endpoints.DirectInternalInitalise(host, initTx)
+                                .then(async (response) => {
+                                if (response.status == "200" && !response.data?.error) {
+                                    let tx = response.data;
+                                    if (!tx) {
+                                        delete initTx.$nodes;
+                                        delete initTx.$revs;
+                                        delete initTx.$streams;
+                                        activelogger_1.ActiveLogger.warn(initTx.$tx, `SPI NOTX (Empty Response) Resending ${counter}`);
+                                        setTimeout(() => {
+                                            if (!response.dontRelease) {
+                                                host.release(initTx.$umid);
+                                            }
+                                            resendable(initTx, ++counter);
+                                        }, 50);
+                                        return;
+                                    }
+                                    let summary = {
+                                        total: 0,
+                                        vote: 0,
+                                        commit: 0,
+                                    };
+                                    let responses = [];
+                                    if (tx?.$nodes) {
+                                        let nodes = Object.keys(tx.$nodes);
+                                        for (let i = nodes.length; i--;) {
+                                            summary.total++;
+                                            if (tx.$nodes[nodes[i]].vote)
+                                                summary.vote++;
+                                            if (tx.$nodes[nodes[i]].commit)
+                                                summary.commit++;
+                                            if (tx.$nodes[nodes[i]]?.error) {
+                                                if (summary.errors) {
+                                                    summary.errors.push(tx.$nodes[nodes[i]].error);
+                                                }
+                                                else {
+                                                    summary.errors = [
+                                                        tx.$nodes[nodes[i]].error,
+                                                    ];
+                                                }
+                                            }
+                                            if (tx.$nodes[nodes[i]].return) {
+                                                responses.push(tx.$nodes[nodes[i]].return);
+                                            }
+                                            if (!tx.$streams && tx.$nodes[nodes[i]].streams) {
+                                                tx.$streams = tx.$nodes[nodes[i]].streams;
+                                            }
+                                        }
+                                        if (!summary.commit &&
+                                            summary.total > 1 &&
+                                            summary.errors &&
+                                            counter <= MAX_COUNTERS) {
+                                            activelogger_1.ActiveLogger.warn(summary, `SPI Checking - Origin Node, Is it wrong? umid: ${tx.$umid}`);
+                                            if (Endpoints.shouldTriggerSpiLookup(summary.errors, tx.$nodes[home_1.Home.reference].error)) {
+                                                activelogger_1.ActiveLogger.warn(`SPI Checked - Origin Node, Wrong. Starting lookup umid: ${tx.$umid}`);
+                                                setTimeout(async () => {
+                                                    let rewroteSomething = false;
+                                                    const streams = [
+                                                        ...new Set([
+                                                            ...this.labelOrKey(tx.$tx.$i),
+                                                            ...this.labelOrKey(tx.$tx.$o),
+                                                        ]),
+                                                    ];
+                                                    if (streams.length) {
+                                                        const rewrote = activeoptions_1.ActiveCacheManager.fetch("rewrote", 1500);
+                                                        for (let i = streams.length; i--;) {
+                                                            streams.push(`${streams[i]}:stream`);
+                                                        }
+                                                        streams.push(`${tx.$tx.$contract.substring(0, 64)}:data`);
+                                                        if (streams.length) {
+                                                            const networkStreams = await host.neighbourhood.knockAll("stream", {
+                                                                $streams: streams,
+                                                            });
+                                                            const consensusReached = Math.ceil((activeoptions_1.ActiveOptions.get("consensus", {})
+                                                                .reached /
+                                                                100) *
+                                                                host.neighbourhood.count() -
+                                                                1);
+                                                            if (networkStreams.length >= consensusReached) {
+                                                                const consensus = {};
+                                                                for (let i = networkStreams.length; i--;) {
+                                                                    const nodeStreams = networkStreams[i];
+                                                                    if (nodeStreams?.length) {
+                                                                        for (let ii = nodeStreams.length; ii--;) {
+                                                                            const streamDoc = nodeStreams[ii];
+                                                                            if (streamDoc && streamDoc._id && streamDoc._rev) {
+                                                                                if (!consensus[streamDoc._id]) {
+                                                                                    consensus[streamDoc._id] = {};
+                                                                                }
+                                                                                if (consensus[streamDoc._id][streamDoc._rev]) {
+                                                                                    consensus[streamDoc._id][streamDoc._rev].votes++;
+                                                                                }
+                                                                                else {
+                                                                                    consensus[streamDoc._id][streamDoc._rev] = {
+                                                                                        votes: 1,
+                                                                                        doc: streamDoc,
+                                                                                    };
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                                const docs = Object.keys(consensus);
+                                                                for (let g = docs.length; g--;) {
+                                                                    const doc = consensus[docs[g]];
+                                                                    if (rewrote.has(docs[g])) {
+                                                                        continue;
+                                                                    }
+                                                                    let max = 0;
+                                                                    let winner = "";
+                                                                    for (let x in doc) {
+                                                                        if (doc[x].votes >= consensusReached) {
+                                                                            activelogger_1.ActiveLogger.warn(`SPI ${doc[x].votes} >= ${consensusReached} for ${docs[g]}@${x}`);
+                                                                            if (doc[x].votes > max) {
+                                                                                max = doc[x].votes;
+                                                                                winner = x;
+                                                                            }
+                                                                            else if (doc[x].votes === max) {
+                                                                                const [xPos] = x.split("-");
+                                                                                const [wPos] = winner.split("-");
+                                                                                if (+xPos > +wPos) {
+                                                                                    activelogger_1.ActiveLogger.warn(`SPI matching max (${max}) but has higher position`);
+                                                                                    winner = x;
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                    if (winner && !rewrote.has(docs[g])) {
+                                                                        const winningDoc = doc[winner].doc;
+                                                                        rewrote.set(winningDoc._id, winningDoc._rev);
+                                                                        const dblCheck = await host.dbConnection.get(winningDoc._id);
+                                                                        if (dblCheck._rev !== winningDoc._rev) {
+                                                                            activelogger_1.ActiveLogger.error(`SPI REWRITING #1 ${winningDoc._id} @ ${winningDoc._rev} NOT ${dblCheck._rev} : ${tx.$umid} CACHE : ${rewrote.get(winningDoc._id)}`);
+                                                                            await host.dbConnection.bulkDocs([winningDoc], {
+                                                                                new_edits: true,
+                                                                                force_rev: winningDoc._rev,
+                                                                            });
+                                                                            rewroteSomething = true;
+                                                                        }
+                                                                    }
+                                                                    else {
+                                                                        if (!rewrote.has(docs[g])) {
+                                                                            activelogger_1.ActiveLogger.warn(`SPI NOWINNER #1 - ${docs[g]}`);
+                                                                        }
+                                                                    }
+                                                                }
+                                                                if (rewroteSomething) {
+                                                                    delete initTx.$nodes;
+                                                                    delete initTx.$revs;
+                                                                    delete initTx.$streams;
+                                                                    if (!response.dontRelease) {
+                                                                        host.release(initTx.$umid);
+                                                                    }
+                                                                    initTx.$umid = activecrypto_1.ActiveCrypto.Hash.getHash(JSON.stringify(initTx) + counter);
+                                                                    activelogger_1.ActiveLogger.warn(initTx, `SPI (Rewrite) Resending #1 ${counter}`);
+                                                                    setTimeout(() => {
+                                                                        resendable(initTx, ++counter);
+                                                                    }, 50);
+                                                                    return;
+                                                                }
+                                                                else {
+                                                                    if (!response.dontRelease) {
+                                                                        host.release(tx.$umid);
+                                                                    }
+                                                                    const output = {
+                                                                        $umid: tx.$umid,
+                                                                        $summary: summary,
+                                                                        $streams: tx.$streams,
+                                                                    };
+                                                                    if (responses.length) {
+                                                                        output.$responses = responses;
+                                                                    }
+                                                                    if (activeoptions_1.ActiveOptions.get("debugToClient", false)) {
+                                                                        output.$debug = tx;
+                                                                    }
+                                                                    activelogger_1.ActiveLogger.warn(output, `SPI Failed to find an issue returning to client`);
+                                                                    return resolve({
+                                                                        statusCode: 200,
+                                                                        content: output,
+                                                                    });
+                                                                }
+                                                            }
+                                                            else {
+                                                                activelogger_1.ActiveLogger.warn(`SPI Skipped not enough returned for consensus yet (${networkStreams.length}/${consensusReached})`);
+                                                                return resolve({
+                                                                    statusCode: 200,
+                                                                    content: output,
+                                                                });
+                                                            }
+                                                        }
+                                                    }
+                                                }, 100);
+                                                return;
+                                            }
+                                            else {
+                                                if (summary.errors?.some((e) => e.indexOf("Busy Locks") !== -1 ||
+                                                    e.indexOf("IBL01") !== -1) &&
+                                                    !initTx.$nolock) {
+                                                    delete initTx.$nodes;
+                                                    delete initTx.$revs;
+                                                    delete initTx.$streams;
+                                                    if (!response.dontRelease) {
+                                                        host.release(initTx.$umid);
+                                                    }
+                                                    initTx.$umid = activecrypto_1.ActiveCrypto.Hash.getHash(JSON.stringify(initTx) + counter);
+                                                    activelogger_1.ActiveLogger.warn(initTx.$tx, `SPI Resending #2 ${counter} in 5s`);
+                                                    setTimeout(() => {
+                                                        resendable(initTx, ++counter);
+                                                    }, 50);
+                                                    return;
+                                                }
+                                            }
+                                        }
+                                    }
+                                    if (!response.dontRelease) {
+                                        host.release(tx.$umid);
+                                    }
+                                    const output = {
+                                        $umid: tx.$umid,
+                                        $summary: summary,
+                                        $streams: tx.$streams,
+                                    };
+                                    if (responses.length) {
+                                        output.$responses = responses;
+                                    }
+                                    if (activeoptions_1.ActiveOptions.get("debugToClient", false)) {
+                                        output.$debug = tx;
+                                    }
+                                    return resolve({
+                                        statusCode: 200,
+                                        content: output,
+                                    });
+                                }
+                                else {
+                                    if (response.rebroadcasted) {
+                                        return resolve({
+                                            statusCode: 200,
+                                            content: response.data,
+                                        });
+                                    }
+                                    else {
+                                        return resolve({
+                                            statusCode: response.status,
+                                            content: response.data,
+                                        });
+                                    }
+                                }
+                            })
+                                .catch((error) => {
+                                host.release(initTx.$umid);
+                                if (error?.status == 100 && error.error) {
+                                    if (counter <= MAX_COUNTERS &&
+                                        error.error === "Busy Locks" &&
+                                        !initTx.$nolock) {
+                                        delete initTx.$nodes;
+                                        delete initTx.$revs;
+                                        delete initTx.$streams;
+                                        initTx.$umid = activecrypto_1.ActiveCrypto.Hash.getHash(JSON.stringify(initTx) + counter);
+                                        activelogger_1.ActiveLogger.warn(initTx.$tx, `SPI Resending #3 ${counter}`);
+                                        setTimeout(() => {
+                                            resendable(initTx, ++counter);
+                                        }, 250);
+                                        return;
+                                    }
+                                    else {
+                                        return resolve(this.successfulFailure(error.error || error, 0));
+                                    }
+                                }
+                                else {
+                                    activelogger_1.ActiveLogger.error(error, "Sent 500 Response (1000)");
+                                    return reject({
+                                        statusCode: 500,
+                                        content: error instanceof Error
+                                            ? { error: error.message }
+                                            : error,
+                                    });
+                                }
                             });
-                        }
-                    }
-                })
-                    .catch((error) => {
-                    if ((error === null || error === void 0 ? void 0 : error.status) == "100" && error.error) {
-                        return resolve(this.successfulFailure(error.error || error));
+                        };
+                        resendable(tx);
                     }
                     else {
-                        activelogger_1.ActiveLogger.error(error, "Sent 500 Response (1000)");
+                        activelogger_1.ActiveLogger.error("Sent 500 Response (1200)");
                         return reject({
                             statusCode: 500,
-                            content: error,
+                            content: "Invalid Transaction",
                         });
                     }
                 });
+            };
+            if (body.$multi) {
+                const results = [];
+                if (body.$seq) {
+                    for (let i = body.$multi.length; i--;) {
+                        results.push(await process(body.$multi[i]));
+                    }
+                    const response = [];
+                    for (let i = results.length; i--;) {
+                        response.push(results[i].content);
+                    }
+                    resolve({
+                        statusCode: 200,
+                        content: response,
+                    });
+                }
+                else {
+                    for (let i = body.$multi.length; i--;) {
+                        results.push(process(body.$multi[i]));
+                    }
+                    const results2 = (await Promise.all(results));
+                    const response = [];
+                    for (let i = results2.length; i--;) {
+                        response.push(results2[i].content);
+                    }
+                    resolve({
+                        statusCode: 200,
+                        content: response,
+                    });
+                }
             }
             else {
-                activelogger_1.ActiveLogger.error("Sent 500 Response (1200)");
-                return reject({
-                    statusCode: 500,
-                    content: "Invalid Transaction",
-                });
+                process(body).then(resolve).catch(reject);
             }
         });
     }
+    static shouldTriggerSpiLookup(errors, selfError) {
+        const spiErrorCount = errors?.filter((e) => e.indexOf("Stream Position Incorrect") !== -1)
+            .length || 0;
+        return ((spiErrorCount > 0 &&
+            spiErrorCount >= Math.floor((errors?.length || 0) / 3)) ||
+            (selfError !== undefined &&
+                selfError.indexOf("Stream Position Incorrect") !== -1));
+    }
+    static labelOrKey(txIO) {
+        const keys = Object.keys(txIO || {});
+        const out = [];
+        for (let i = keys.length; i--;) {
+            const addr = this.filterPrefix(txIO[keys[i]].$stream || keys[i]);
+            if (addr.length === 64) {
+                out.push(addr);
+            }
+        }
+        return out;
+    }
+    static filterPrefix(streamId) {
+        if (streamId.length > 64) {
+            streamId = streamId.slice(-64);
+        }
+        return streamId;
+    }
     static ExternalEncrypt(host, body, encHeader, db) {
         return new Promise((resolve, reject) => {
             if (encHeader) {
@@ -161,27 +431,247 @@ class Endpoints {
             }
         });
     }
-    static InternalInitalise(host, body) {
+    static InternalInitalise(host, body, remoteAddr, retried = false, isP2P = false) {
         return new Promise((resolve, reject) => {
-            if (host.getStatus() != neighbourhood_1.NeighbourStatus.Stable) {
-                activelogger_1.ActiveLogger.error("Sent 500 Response (1500)");
-                return resolve({
-                    statusCode: 500,
-                    content: "Network Not Stable",
-                });
-            }
             let tx = body;
+            if (!retried) {
+                tx.$spiRetry = false;
+            }
             host
-                .pending(tx, true)
-                .then((ledger) => {
-                return resolve({
-                    statusCode: ledger.status,
-                    content: ledger.data,
-                });
+                .pending(tx, remoteAddr, true, retried, isP2P)
+                .then(async (ledger) => {
+                let canRetry = false;
+                let resolved = false;
+                if (!tx.$unanimous || retried) {
+                    resolved = true;
+                    resolve({
+                        statusCode: ledger.status,
+                        content: ledger.data,
+                    });
+                }
+                const rewrote = activeoptions_1.ActiveCacheManager.fetch("rewrote", 500);
+                if (ledger?.data?.$nodes && !rewrote.has(tx.$umid)) {
+                    if (ledger?.data?.$nodes[home_1.Home.reference] &&
+                        ledger.data.$nodes[home_1.Home.reference].error) {
+                        const spiError = ledger.data.$nodes[home_1.Home.reference].error.indexOf("Position Incorrect") !== -1;
+                        const spi404Error = ledger.data.$nodes[home_1.Home.reference].error.indexOf("Stream(s) not found") !== -1;
+                        if (spiError || spi404Error) {
+                            rewrote.set(tx.$umid, 1);
+                            activelogger_1.ActiveLogger.warn(tx.$umid, "SPI NON Origin - Position Incorrect or 404");
+                            let check = false;
+                            let posCount = 0;
+                            let myPos = false;
+                            if (spiError) {
+                                const nodes = Object.keys(ledger.data.$nodes);
+                                for (let i = nodes.length; i--;) {
+                                    if (ledger.data.$nodes[nodes[i]].commit) {
+                                        check = true;
+                                        break;
+                                    }
+                                    if (ledger.data.$nodes[nodes[i]].error?.indexOf("Stream Position Incorrect") !== -1) {
+                                        posCount++;
+                                        if (nodes[i] === home_1.Home.reference) {
+                                            myPos = true;
+                                        }
+                                    }
+                                }
+                            }
+                            else {
+                                check = spi404Error;
+                            }
+                            if (check || (posCount >= 1 && myPos)) {
+                                activelogger_1.ActiveLogger.warn(tx.$umid, "SPI NON Origin - Must Check");
+                                const streams = [
+                                    ...new Set([
+                                        ...this.labelOrKey(ledger.data.$tx.$i),
+                                        ...this.labelOrKey(ledger.data.$tx.$o),
+                                    ]),
+                                ];
+                                if (streams.length) {
+                                    for (let i = streams.length; i--;) {
+                                        streams.push(`${streams[i]}:stream`);
+                                    }
+                                    streams.push(`${ledger.data.$tx.$contract.substring(0, 64)}:data`);
+                                    if (streams.length) {
+                                        const tmp = async () => {
+                                            const networkStreams = await host.neighbourhood.knockAll("stream", {
+                                                $streams: streams,
+                                            });
+                                            const consensusReached = Math.ceil((activeoptions_1.ActiveOptions.get("consensus", {}).reached /
+                                                100) *
+                                                host.neighbourhood.count() -
+                                                1);
+                                            if (networkStreams.length < consensusReached) {
+                                                activelogger_1.ActiveLogger.warn(`SPI Skipped not enough returned for consensus yet (${networkStreams.length}/${consensusReached})`);
+                                                return;
+                                            }
+                                            const consensus = {};
+                                            for (let i = networkStreams.length; i--;) {
+                                                const nodeStreams = networkStreams[i];
+                                                if (nodeStreams?.length) {
+                                                    for (let ii = nodeStreams.length; ii--;) {
+                                                        const streamDoc = nodeStreams[ii];
+                                                        if (streamDoc && streamDoc._id && streamDoc._rev) {
+                                                            if (!consensus[streamDoc._id]) {
+                                                                consensus[streamDoc._id] = {};
+                                                            }
+                                                            if (consensus[streamDoc._id][streamDoc._rev]) {
+                                                                consensus[streamDoc._id][streamDoc._rev].votes++;
+                                                            }
+                                                            else {
+                                                                consensus[streamDoc._id][streamDoc._rev] = {
+                                                                    votes: 1,
+                                                                    doc: streamDoc,
+                                                                };
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            const docs = Object.keys(consensus);
+                                            for (let g = docs.length; g--;) {
+                                                const doc = consensus[docs[g]];
+                                                if (rewrote.has(docs[g])) {
+                                                    continue;
+                                                }
+                                                let max = 0;
+                                                let winner = "";
+                                                for (let x in doc) {
+                                                    if (doc[x].votes >= consensusReached) {
+                                                        activelogger_1.ActiveLogger.warn(`SPI ${doc[x].votes} >= ${consensusReached} for ${docs[g]}@${x}`);
+                                                        if (doc[x].votes > max) {
+                                                            max = doc[x].votes;
+                                                            winner = x;
+                                                        }
+                                                        else if (doc[x].votes === max) {
+                                                            const [xPos] = x.split("-");
+                                                            const [wPos] = winner.split("-");
+                                                            if (+xPos > +wPos) {
+                                                                activelogger_1.ActiveLogger.warn(`SPI matching max (${max}) but has higher position`);
+                                                                winner = x;
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                                if (winner && !rewrote.has(docs[g])) {
+                                                    const winningDoc = doc[winner].doc;
+                                                    rewrote.set(tx.$umid, true);
+                                                    rewrote.set(winningDoc._id, winningDoc._rev);
+                                                    const dblCheck = await host.dbConnection.get(winningDoc._id);
+                                                    if (dblCheck._rev !== winningDoc._rev) {
+                                                        activelogger_1.ActiveLogger.error(`SPI REWRITING #2 ${winningDoc._id} @ ${winningDoc._rev} NOT ${dblCheck._rev} : ${tx.$umid} CACHE : ${rewrote.get(winningDoc._id)}`);
+                                                        if (!dblCheck._rev) {
+                                                            try {
+                                                                await host.dbConnection.put(winningDoc);
+                                                                activelogger_1.ActiveLogger.warn(`SPI 404 - Create Base for ${winningDoc._id}`);
+                                                            }
+                                                            catch {
+                                                                activelogger_1.ActiveLogger.error(`SPI 404 - Failed to create ${winningDoc._id}`);
+                                                            }
+                                                        }
+                                                        await host.dbConnection.bulkDocs([winningDoc], {
+                                                            new_edits: true,
+                                                            force_rev: winningDoc._rev,
+                                                        });
+                                                        canRetry = true;
+                                                    }
+                                                }
+                                                else {
+                                                    if (!rewrote.has(docs[g])) {
+                                                        activelogger_1.ActiveLogger.warn(`SPI NOWINNER #2 - ${docs[g]}`);
+                                                    }
+                                                }
+                                            }
+                                            if (rewrote.has(tx.$umid)) {
+                                                activelogger_1.ActiveLogger.warn(tx.$umid, `SPI Adding 950 Checker`);
+                                                await host.dbErrorConnection.post({
+                                                    _id: `${tx.$umid}:${Date.now()}`,
+                                                    code: 950,
+                                                    processed: false,
+                                                    umid: tx.$umid,
+                                                    transaction: {
+                                                        $broadcast: true,
+                                                        $tx: {},
+                                                        $revs: {},
+                                                    },
+                                                    reason: 'Vote Failure - "SPI#2 UMID not found',
+                                                });
+                                            }
+                                            if (!ledger.dontRelease) {
+                                                host.release(tx.$umid);
+                                            }
+                                        };
+                                        if (resolved) {
+                                            setTimeout(async () => {
+                                                activelogger_1.ActiveLogger.warn(`SPI WAITING - ${tx.$umid}`);
+                                                tmp();
+                                            }, 500);
+                                        }
+                                        else {
+                                            activelogger_1.ActiveLogger.warn(`SPI NOW - ${tx.$umid}`);
+                                            await tmp();
+                                            if (canRetry && tx.$unanimous) {
+                                                activelogger_1.ActiveLogger.warn(tx.$umid, `SPI RETRY as it was unanimous and written`);
+                                                tx.$spiRetry = true;
+                                                this.InternalInitalise(host, tx, remoteAddr, true)
+                                                    .then(resolve)
+                                                    .catch(reject);
+                                            }
+                                            else {
+                                                activelogger_1.ActiveLogger.warn(tx.$umid, `SPI Delay resolved`);
+                                                return resolve({
+                                                    statusCode: ledger.status,
+                                                    content: ledger.data,
+                                                });
+                                            }
+                                        }
+                                    }
+                                    else {
+                                        if (!ledger.dontRelease) {
+                                            host.release(tx.$umid);
+                                        }
+                                    }
+                                }
+                                else {
+                                    if (!ledger.dontRelease) {
+                                        host.release(tx.$umid);
+                                    }
+                                }
+                            }
+                            else {
+                                if (!ledger.dontRelease) {
+                                    host.release(tx.$umid);
+                                }
+                            }
+                        }
+                        else {
+                            if (!ledger.dontRelease) {
+                                host.release(tx.$umid);
+                            }
+                        }
+                    }
+                    else {
+                        if (!ledger.dontRelease) {
+                            host.release(tx.$umid);
+                        }
+                        if (!resolved) {
+                            resolve({
+                                statusCode: ledger.status,
+                                content: ledger.data,
+                            });
+                        }
+                    }
+                }
+                else {
+                    if (!ledger.dontRelease) {
+                        host.release(tx.$umid);
+                    }
+                }
             })
                 .catch((error) => {
                 activelogger_1.ActiveLogger.error(tx, "Transaction error");
                 activelogger_1.ActiveLogger.error(error, "Sent 500 Response (1600)");
+                host.release(tx.$umid);
                 return reject({
                     statusCode: 500,
                     content: error,
@@ -191,11 +681,6 @@ class Endpoints {
     }
     static DirectInternalInitalise(host, tx) {
         return new Promise((resolve, reject) => {
-            if (host.getStatus() != neighbourhood_1.NeighbourStatus.Stable)
-                return reject({
-                    status: 100,
-                    error: "Network Not Stable",
-                });
             if (tx.$territoriality) {
                 if (tx.$broadcast) {
                     return reject({
@@ -205,11 +690,11 @@ class Endpoints {
                 }
                 let sending = host.terriMap(tx.$territoriality);
                 if (sending) {
-                    activelogger_1.ActiveLogger.info("Rebroadcasting to : " + sending);
                     if (sending !== host.reference) {
+                        activelogger_1.ActiveLogger.info("Rebroadcasting to : " + sending);
                         let rebroadcast = host.neighbourhood.get(sending);
                         rebroadcast
-                            .knock("", tx, true)
+                            .knock("", tx, true, 0, false)
                             .then((ledger) => {
                             ledger.rebroadcasted = true;
                             resolve(ledger);
@@ -225,9 +710,14 @@ class Endpoints {
                 }
             }
             host
-                .pending(tx)
+                .pending(tx, home_1.Home.host)
                 .then((ledger) => resolve(ledger))
-                .catch((error) => reject(error));
+                .catch((error) => {
+                activelogger_1.ActiveLogger.fatal(tx, "last tx sent in");
+                activelogger_1.ActiveLogger.fatal(error, "error that is bubbling");
+                host.release(tx.$umid);
+                reject(error);
+            });
         });
     }
     static status(host, requester) {
@@ -283,42 +773,54 @@ class Endpoints {
     static streams(db, body) {
         return new Promise((resolve, reject) => {
             if (body.$streams) {
-                let i = body.$streams.length;
-                let fetchStream = [];
-                while (i--) {
+                const fetchStream = [];
+                for (let i = body.$streams.length; i--;) {
                     if (body.$streams[i].indexOf(":volatile") !== -1) {
                         return reject({
                             statusCode: 403,
                             content: "Request not allowed",
                         });
                     }
-                    fetchStream.push(db.get(body.$streams[i]).catch((error) => {
-                        return { _error: error };
-                    }));
-                }
-                Promise.all(fetchStream)
-                    .then((docs) => {
-                    let i = docs.length;
-                    let streams = [];
-                    while (i--) {
-                        if (docs[i]._id) {
-                            streams.push({
-                                _id: docs[i]._id,
-                                _rev: docs[i]._rev,
-                            });
+                    const holdValue = body.$streams[i].replace(":stream", "");
+                    if (locker_1.Locker.hold(holdValue, "SPI")) {
+                        setTimeout(() => {
+                            locker_1.Locker.release(holdValue, "SPI");
+                        }, 1000);
+                        fetchStream.push(db.get(body.$streams[i]));
+                    }
+                    else {
+                        if (locker_1.Locker.is(holdValue, "SPI")) {
+                            fetchStream.push(db.get(body.$streams[i]));
                         }
                     }
-                    return resolve({
-                        statusCode: 200,
-                        content: streams,
+                }
+                if (fetchStream.length) {
+                    Promise.all(fetchStream)
+                        .then((docs) => {
+                        const streams = [];
+                        for (let i = docs.length; i--;) {
+                            if (docs[i]._id) {
+                                streams.push(docs[i]);
+                            }
+                        }
+                        return resolve({
+                            statusCode: 200,
+                            content: streams,
+                        });
+                    })
+                        .catch(() => {
+                        return resolve({
+                            statusCode: 200,
+                            content: [],
+                        });
                     });
-                })
-                    .catch(() => {
+                }
+                else {
                     return resolve({
                         statusCode: 200,
                         content: [],
                     });
-                });
+                }
             }
             else {
                 if (body.$stream && body.$rev) {
@@ -374,7 +876,7 @@ class Endpoints {
                 });
             })
                 .catch(() => {
-                reject({});
+                reject({ error: 3 });
             });
         });
     }
@@ -388,7 +890,7 @@ class Endpoints {
                 });
             })
                 .catch(() => {
-                reject({});
+                reject({ error: 2 });
             });
         });
     }
@@ -399,13 +901,14 @@ class Endpoints {
         return !(row.id.indexOf(":volatile") !== -1);
     }
     static postConvertor(host, body, encryptHeader) {
-        return new Promise((resolve, reject) => {
+        return new Promise(async (resolve, reject) => {
+            let data;
             if (encryptHeader) {
                 activelogger_1.ActiveLogger.info("Encrypted Transaction Inbound");
                 try {
-                    resolve(JSON.parse(Buffer.from(host.decrypt(body), "base64").toString()));
+                    data = Buffer.from(host.decrypt(body), "base64");
                 }
-                catch (_a) {
+                catch {
                     activelogger_1.ActiveLogger.error("Sent 500 Response (1700)");
                     return reject({
                         statusCode: 500,
@@ -414,20 +917,23 @@ class Endpoints {
                 }
             }
             else {
-                const bodyObject = this.makeSureNotBuffer(JSON.parse(body));
-                if (bodyObject.$neighbour && bodyObject.$packet) {
-                    activelogger_1.ActiveLogger.debug("Converting Signed for Post");
-                    if (bodyObject.$neighbour.reference != host.reference) {
-                        if (bodyObject.$enc ||
+                data = typeof body === "string" ? Buffer.from(body) : body;
+            }
+            try {
+                const bodyObject = await activeutilities_1.ActiveClone.deserialize(data);
+                if (bodyObject && bodyObject.$neighbour && bodyObject.$packet) {
+                    const bodyObj = bodyObject;
+                    if (bodyObj.$neighbour.reference != host.reference) {
+                        if (bodyObj.$enc ||
                             activeoptions_1.ActiveOptions.get("security", {}).encryptedConsensus) {
-                            bodyObject.$packet = JSON.parse(Buffer.from(host.decrypt(bodyObject.$packet), "base64").toString());
+                            bodyObj.$packet = await activeutilities_1.ActiveClone.deserialize(Buffer.from(host.decrypt(bodyObj.$packet), "base64"));
                         }
                     }
-                    if (bodyObject.$neighbour.signature ||
+                    if (bodyObj.$neighbour.signature ||
                         activeoptions_1.ActiveOptions.get("security", {}).signedConsensus) {
                         if (!host.neighbourhood
-                            .get(bodyObject.$neighbour.reference)
-                            .verifySignature(bodyObject.$neighbour.signature, bodyObject.$packet)) {
+                            .get(bodyObj.$neighbour.reference)
+                            .verifySignature(bodyObj.$neighbour.signature, bodyObj.$packet)) {
                             activelogger_1.ActiveLogger.error("Sent 500 Response (1800)");
                             return reject({
                                 statusCode: 500,
@@ -436,26 +942,33 @@ class Endpoints {
                         }
                     }
                     return resolve({
-                        from: bodyObject.$neighbour.reference,
-                        body: bodyObject.$packet,
+                        from: bodyObj.$neighbour.reference,
+                        body: bodyObj.$packet,
                     });
                 }
                 else {
                     resolve({ body: bodyObject });
                 }
             }
+            catch (e) {
+                activelogger_1.ActiveLogger.error(e, "Deserialization Error");
+                return reject({
+                    statusCode: 500,
+                    content: "Deserialization Error",
+                });
+            }
         });
     }
-    static makeSureNotBuffer(obj) {
-        var _a;
-        if (obj.type === "Buffer" && ((_a = obj.data) === null || _a === void 0 ? void 0 : _a.length)) {
-            const tmp = JSON.parse(Buffer.from(obj.data).toString());
-            activelogger_1.ActiveLogger.error(tmp, "Buffer Found");
-            return tmp;
+    static async makeSureNotBuffer(obj) {
+        if (obj.type === "Buffer" && obj.data?.length) {
+            if (obj.data[0] == 0x1f && obj.data[1] == 0x8b) {
+                return JSON.parse((await activeoptions_1.ActiveGZip.ungzip(Buffer.from(obj.data))).toString());
+            }
+            return JSON.parse(Buffer.from(obj.data).toString());
         }
         return obj;
     }
-    static successfulFailure(error) {
+    static successfulFailure(error, counter = 0) {
         return {
             statusCode: 200,
             content: {
@@ -465,6 +978,7 @@ class Endpoints {
                     vote: 0,
                     commit: 0,
                     errors: [error],
+                    counter,
                 },
                 $streams: {
                     new: [],
@@ -475,5 +989,4 @@ class Endpoints {
     }
 }
 exports.Endpoints = Endpoints;
-Endpoints.rebaseThrottle = 0;
 //# sourceMappingURL=endpoints.js.map