@harperfast/harper 5.0.0-alpha.10

package/dist/server/threads/threadServer.js

@@ -0,0 +1,288 @@
+'use strict';
+const { isMainThread, parentPort, threadId, workerData } = require('node:worker_threads');
+const { createServer: createSocketServer } = require('node:net');
+const { unlinkSync, existsSync } = require('fs');
+let componentsLoadedResolve;
+exports.whenComponentsLoaded = new Promise((resolve) => {
+    componentsLoadedResolve = resolve;
+});
+const harperLogger = require('../../utility/logging/harper_logger.js');
+const env = require('../../utility/environment/environmentManager.js');
+const terms = require("../../utility/hdbTerms.js");
+const { server } = require("../Server.js");
+let { createServer: createSecureSocketServer } = require('node:tls');
+const { restartNumber, getWorkerIndex } = require('./manageThreads.js');
+const { createReuseportFd } = require("../serverHelpers/Request.js");
+const { createTLSSelector } = require('../../security/keys.js');
+const { startupLog } = require('../../bin/run.js');
+const { SERVERS, setPortServerMap, portServer } = require("../serverRegistry.js");
+const httpComponent = require("../http.js");
+const globals = require('../../globals.js');
+const debugThreads = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG);
+const sessionAffinity = env.get(terms.CONFIG_PARAMS.HTTP_SESSIONAFFINITY);
+server.socket = onSocket;
+if (debugThreads) {
+    let port;
+    if (isMainThread) {
+        port = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_PORT) ?? 9229;
+        process.on(['SIGINT', 'SIGTERM', 'SIGQUIT', 'exit'], () => {
+            try {
+                require('inspector').close();
+            }
+            catch (error) {
+                harperLogger.info('Could not close debugger', error);
+            }
+        });
+    }
+    else {
+        const startingPort = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_STARTINGPORT);
+        if (startingPort && getWorkerIndex() >= 0) {
+            port = startingPort + getWorkerIndex();
+        }
+    }
+    if (port) {
+        const host = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_HOST);
+        const waitForDebugger = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_WAITFORDEBUGGER);
+        try {
+            require('inspector').open(port, host, waitForDebugger);
+        }
+        catch (error) {
+            harperLogger.trace(`Could not start debugging on port ${port}, you may already be debugging:`, error.message);
+        }
+    }
+}
+else if (process.env.DEV_MODE && isMainThread) {
+    try {
+        require('inspector').open(9229);
+    }
+    catch (error) {
+        if (restartNumber <= 1)
+            harperLogger.trace('Could not start debugging on port 9229, you may already be debugging:', error.message);
+    }
+}
+process.on('uncaughtException', (error) => {
+    if (error.isHandled)
+        return;
+    if (error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED')
+        return; // that's what network connections do
+    if (error.message === 'write EIO')
+        return; // that means the terminal is closed
+    harperLogger.error('uncaughtException', error);
+});
+env.initSync();
+exports.globals = globals;
+exports.listenOnPorts = listenOnPorts;
+exports.startServers = startServers;
+function startServers() {
+    const rootPath = env.get(terms.CONFIG_PARAMS.ROOTPATH);
+    if (rootPath) {
+        try {
+            process.chdir(rootPath);
+        }
+        catch {
+            // ignore any errors with this; just a best effort for now
+        }
+    }
+    let loaded = require('../loadRootComponents.js')
+        .loadRootComponents(true)
+        .then(() => {
+        parentPort
+            ?.on('message', (message) => {
+            const { port, fd, data } = message;
+            if (fd) {
+                // Create a socket from the file descriptor for the socket that was routed to us.
+                httpComponent.deliverSocket(fd, port, data);
+            }
+            else if (message.requestId) {
+                // Windows doesn't support passing file descriptors, so we have to resort to manually proxying the socket
+                // data for each request
+                httpComponent.proxyRequest(message);
+            }
+            else if (message.type === terms.ITC_EVENT_TYPES.SHUTDOWN) {
+                harperLogger.trace('received shutdown request', threadId);
+                // shutdown (for these threads) means stop listening for incoming requests (finish what we are working) and
+                // close connections as possible, then let the event loop complete
+                for (let port in SERVERS) {
+                    const server = SERVERS[port];
+                    let closeAllTimer;
+                    if (server.closeIdleConnections) {
+                        // Here we attempt to gracefully close all outstanding keep-alive connections,
+                        // repeatedly closing any connections that are idle. This allows any active requests
+                        // to finish sending their response, then we close their connections.
+                        let symbols = Object.getOwnPropertySymbols(server);
+                        let connectionsSymbol = symbols.find((symbol) => symbol.description.includes('connections'));
+                        let closeAttempts = 0;
+                        let timer = setInterval(() => {
+                            closeAttempts++;
+                            const forceClose = closeAttempts >= 100;
+                            if (!server[connectionsSymbol]) {
+                                if (forceClose)
+                                    server.closeAllConnections?.();
+                                clearInterval(timer);
+                                return;
+                            }
+                            const connections = server[connectionsSymbol][forceClose ? 'all' : 'idle']?.() || [];
+                            if (connections.length === 0) {
+                                if (forceClose)
+                                    clearInterval(timer);
+                                return;
+                            }
+                            if (closeAttempts === 1)
+                                harperLogger.info(`Closing ${connections.length} idle connections`);
+                            else if (forceClose)
+                                harperLogger.warn(`Forcefully closing ${connections.length} active connections`);
+                            for (let i = 0, l = connections.length; i < l; i++) {
+                                const socket = connections[i].socket;
+                                if (socket._httpMessage && !socket._httpMessage.finished && !forceClose) {
+                                    continue;
+                                }
+                                if (forceClose)
+                                    socket.destroySoon();
+                                else
+                                    socket.end('HTTP/1.1 408 Request Timeout\r\nConnection: close\r\n\r\n');
+                            }
+                        }, 25).unref();
+                    }
+                    // And we tell the server not to accept any more incoming connections
+                    server.close?.(() => {
+                        clearInterval(closeAllTimer);
+                        // We hope for a graceful exit once all connections have been closed, and no
+                        // more incoming connections are accepted, but if we need to, we eventually will exit
+                        setTimeout(() => {
+                            console.log('forced close server', port, threadId);
+                            if (!server.cantCleanupProperly)
+                                harperLogger.warn('Had to forcefully exit the thread', threadId);
+                            process.exit(0);
+                        }, 5000).unref();
+                    });
+                }
+                if (debugThreads || process.env.DEV_MODE) {
+                    try {
+                        require('inspector').close();
+                    }
+                    catch (error) {
+                        harperLogger.info('Could not close debugger', error);
+                    }
+                }
+            }
+        })
+            .ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles
+        let listening;
+        if (createReuseportFd && !sessionAffinity) {
+            listening = listenOnPorts();
+        }
+        // notify that we are now ready to start receiving requests
+        Promise.resolve(listening).then(() => {
+            if (getWorkerIndex() === 0) {
+                try {
+                    startupLog(portServer);
+                }
+                catch (err) {
+                    console.error('Error displaying start-up log', err);
+                }
+            }
+            parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED });
+        });
+    });
+    componentsLoadedResolve(loaded);
+    return loaded;
+}
+function listenOnPorts() {
+    const listening = [];
+    for (let port in SERVERS) {
+        const server = SERVERS[port];
+        // If server is unix domain socket
+        if (port.includes?.('/') && getWorkerIndex() == 0) {
+            if (existsSync(port))
+                unlinkSync(port);
+            listening.push(new Promise((resolve, reject) => {
+                server
+                    .listen({ path: port }, () => {
+                    resolve({ port, name: server.name, protocol_name: server.protocol_name });
+                    harperLogger.info('Domain socket listening on ' + port);
+                })
+                    .on('error', reject);
+            }));
+            continue;
+        }
+        let listen_on;
+        const threadRange = env.get(terms.CONFIG_PARAMS.HTTP_THREADRANGE);
+        if (threadRange) {
+            let threadRangeArray = typeof threadRange === 'string' ? threadRange.split('-') : threadRange;
+            let threadIndex = getWorkerIndex();
+            if (threadIndex < threadRangeArray[0] || threadIndex > threadRangeArray[1]) {
+                continue;
+            }
+        }
+        try {
+            const lastColon = port.lastIndexOf(':');
+            if (lastColon > 0)
+                if (createReuseportFd)
+                    // if there is a colon, we assume it is a host:port pair, and then strip brackets as that is a common way to
+                    // specify an IPv6 address
+                    listen_on = {
+                        fd: createReuseportFd(+port.slice(lastColon + 1).replace(/[[\]]/g, ''), port.slice(0, lastColon)),
+                    };
+                else
+                    listen_on = { host: +port.slice(lastColon + 1).replace(/[[\]]/g, ''), port: port.slice(0, lastColon) };
+            else if (createReuseportFd)
+                listen_on = { fd: createReuseportFd(+port, '::') };
+            else
+                listen_on = { port };
+        }
+        catch (error) {
+            harperLogger.error(`Unable to bind to port ${port}`, error);
+            continue;
+        }
+        listening.push(new Promise((resolve, reject) => {
+            server
+                .listen(listen_on, () => {
+                resolve({ port, name: server.name, protocol_name: server.protocol_name });
+                harperLogger.trace('Listening on port ' + port, threadId);
+            })
+                .on('error', reject);
+        }));
+    }
+    return Promise.all(listening);
+}
+if (!isMainThread && !workerData?.noServerStart) {
+    startServers();
+}
+/**
+ * Direct socket listener
+ * @param listener
+ * @param options
+ */
+function onSocket(listener, options) {
+    let getComponentName = require("../../components/componentLoader.js").getComponentName;
+    let socketServer;
+    if (options.securePort) {
+        setPortServerMap(options.securePort, { protocol_name: 'TLS', name: getComponentName() });
+        const SNICallback = createTLSSelector('server', options.mtls);
+        const tlsConfig = env.get('tls');
+        socketServer = createSecureSocketServer({
+            rejectUnauthorized: Boolean(options.mtls?.required),
+            requestCert: Boolean(options.mtls),
+            noDelay: true, // don't delay for Nagle's algorithm, it is a relic of the past that slows things down: https://brooker.co.za/blog/2024/05/09/nagle.html
+            keepAlive: true,
+            keepAliveInitialDelay: 600, // 10 minute keep-alive, want to be proactive about closing unused connections
+            // For some reason ciphers doesn't work from the secure context, despite node docs claiming it would. Lost
+            // count of how many node TLS bugs that makes
+            ciphers: tlsConfig.ciphers ?? tlsConfig[0]?.ciphers,
+            SNICallback,
+        }, listener);
+        SNICallback.initialize(socketServer);
+        SERVERS[options.securePort] = socketServer;
+    }
+    if (options.port) {
+        setPortServerMap(options.port, { protocol_name: 'TCP', name: getComponentName() });
+        socketServer = createSocketServer(listener, {
+            noDelay: true,
+            keepAlive: true,
+            keepAliveInitialDelay: 600,
+        });
+        SERVERS[options.port] = socketServer;
+    }
+    return socketServer;
+}
+//# sourceMappingURL=threadServer.js.map

package/dist/server/threads/threadServer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"threadServer.js","sourceRoot":"","sources":["../../../server/threads/threadServer.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC1F,MAAM,EAAE,YAAY,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AACjE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACjD,IAAI,uBAAuB,CAAC;AAC5B,OAAO,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;IACtD,uBAAuB,GAAG,OAAO,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,OAAO,CAAC,wCAAwC,CAAC,CAAC;AACvE,MAAM,GAAG,GAAG,OAAO,CAAC,iDAAiD,CAAC,CAAC;AACvE,MAAM,KAAK,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AACnD,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AAC3C,IAAI,EAAE,YAAY,EAAE,wBAAwB,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AACrE,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AACxE,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AACrE,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAChE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;AACnD,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAClF,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAC5C,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAE5C,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;AAChE,MAAM,eAAe,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;AAC1E,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC;AAEzB,IAAI,YAAY,EAAE,CAAC;IAClB,IAAI,IAAI,CAAC;IACT,IAAI,YAAY,EAAE,CAAC;QAClB,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC;QAC/D,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE;YACzD,IAAI,CAAC;gBACJ,OAAO,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,YAAY,CAAC,IAAI,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;YACtD,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;SAAM,CAAC;QACP,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,0BAA0B,CAAC,CAAC;QAC7E,IAAI,YAAY,IAAI,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC;YAC3C,IAAI,GAAG,YAAY,GAAG,cAAc,EAAE,CAAC;QACxC,CAAC;IACF,CAAC;IACD,IAAI,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;QAC7D,MAAM,eAAe,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,6BAA6B,CAAC,CAAC;QACnF,IAAI,CAAC;YACJ,OAAO,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,YAAY,CAAC,KAAK,CAAC,qCAAqC,IAAI,iCAAiC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/G,CAAC;IACF,CAAC;AACF,CAAC;KAAM,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,YAAY,EAAE,CAAC;IACjD,IAAI,CAAC;QACJ,OAAO,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,aAAa,IAAI,CAAC;YACrB,YAAY,CAAC,KAAK,CAAC,uEAAuE,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7G,CAAC;AACF,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE;IACzC,IAAI,KAAK,CAAC,SAAS;QAAE,OAAO;IAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc;QAAE,OAAO,CAAC,qCAAqC;IAC/G,IAAI,KAAK,CAAC,OAAO,KAAK,WAAW;QAAE,OAAO,CAAC,oCAAoC;IAC/E,YAAY,CAAC,KAAK,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AACH,GAAG,CAAC,QAAQ,EAAE,CAAC;AACf,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;AAC1B,OAAO,CAAC,aAAa,GAAG,aAAa,CAAC;AACtC,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;AAEpC,SAAS,YAAY;IACpB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACvD,IAAI,QAAQ,EAAE,CAAC;QACd,IAAI,CAAC;YACJ,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,0DAA0D;QAC3D,CAAC;IACF,CAAC;IACD,IAAI,MAAM,GAAG,OAAO,CAAC,0BAA0B,CAAC;SAC9C,kBAAkB,CAAC,IAAI,CAAC;SACxB,IAAI,CAAC,GAAG,EAAE;QACV,UAAU;YACT,EAAE,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC3B,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;YACnC,IAAI,EAAE,EAAE,CAAC;gBACR,iFAAiF;gBACjF,aAAa,CAAC,aAAa,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7C,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC9B,yGAAyG;gBACzG,wBAAwB;gBACxB,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC;iBAAM,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;gBAC5D,YAAY,CAAC,KAAK,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;gBAC1D,2GAA2G;gBAC3G,kEAAkE;gBAClE,KAAK,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;oBAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC7B,IAAI,aAAa,CAAC;oBAClB,IAAI,MAAM,CAAC,oBAAoB,EAAE,CAAC;wBACjC,8EAA8E;wBAC9E,oFAAoF;wBACpF,qEAAqE;wBACrE,IAAI,OAAO,GAAG,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;wBACnD,IAAI,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;wBAC7F,IAAI,aAAa,GAAG,CAAC,CAAC;wBACtB,IAAI,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;4BAC5B,aAAa,EAAE,CAAC;4BAChB,MAAM,UAAU,GAAG,aAAa,IAAI,GAAG,CAAC;4BACxC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC;gCAChC,IAAI,UAAU;oCAAE,MAAM,CAAC,mBAAmB,EAAE,EAAE,CAAC;gCAC/C,aAAa,CAAC,KAAK,CAAC,CAAC;gCACrB,OAAO;4BACR,CAAC;4BACD,MAAM,WAAW,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC;4BACrF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gCAC9B,IAAI,UAAU;oCAAE,aAAa,CAAC,KAAK,CAAC,CAAC;gCACrC,OAAO;4BACR,CAAC;4BACD,IAAI,aAAa,KAAK,CAAC;gCAAE,YAAY,CAAC,IAAI,CAAC,WAAW,WAAW,CAAC,MAAM,mBAAmB,CAAC,CAAC;iCACxF,IAAI,UAAU;gCAAE,YAAY,CAAC,IAAI,CAAC,sBAAsB,WAAW,CAAC,MAAM,qBAAqB,CAAC,CAAC;4BACtG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gCACpD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gCACrC,IAAI,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,IAAI,CAAC,UAAU,EAAE,CAAC;oCACzE,SAAS;gCACV,CAAC;gCACD,IAAI,UAAU;oCAAE,MAAM,CAAC,WAAW,EAAE,CAAC;;oCAChC,MAAM,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;4BAC9E,CAAC;wBACF,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;oBAChB,CAAC;oBACD,qEAAqE;oBACrE,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE;wBACnB,aAAa,CAAC,aAAa,CAAC,CAAC;wBAC7B,4EAA4E;wBAC5E,qFAAqF;wBACrF,UAAU,CAAC,GAAG,EAAE;4BACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;4BACnD,IAAI,CAAC,MAAM,CAAC,mBAAmB;gCAAE,YAAY,CAAC,IAAI,CAAC,mCAAmC,EAAE,QAAQ,CAAC,CAAC;4BAClG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;wBACjB,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;oBAClB,CAAC,CAAC,CAAC;gBACJ,CAAC;gBACD,IAAI,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;oBAC1C,IAAI,CAAC;wBACJ,OAAO,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;oBAC9B,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,YAAY,CAAC,IAAI,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;oBACtD,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC,CAAC;aACD,GAAG,EAAE,CAAC,CAAC,0FAA0F;QACnG,IAAI,SAAS,CAAC;QACd,IAAI,iBAAiB,IAAI,CAAC,eAAe,EAAE,CAAC;YAC3C,SAAS,GAAG,aAAa,EAAE,CAAC;QAC7B,CAAC;QAED,2DAA2D;QAC3D,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;YACpC,IAAI,cAAc,EAAE,KAAK,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACJ,UAAU,CAAC,UAAU,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC;gBACrD,CAAC;YACF,CAAC;YACD,UAAU,EAAE,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IACJ,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AACf,CAAC;AACD,SAAS,aAAa;IACrB,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,KAAK,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAE7B,kCAAkC;QAClC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,IAAI,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC;YACnD,IAAI,UAAU,CAAC,IAAI,CAAC;gBAAE,UAAU,CAAC,IAAI,CAAC,CAAC;YACvC,SAAS,CAAC,IAAI,CACb,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC/B,MAAM;qBACJ,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE;oBAC5B,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;oBAC1E,YAAY,CAAC,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,CAAC;gBACzD,CAAC,CAAC;qBACD,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACvB,CAAC,CAAC,CACF,CAAC;YACF,SAAS;QACV,CAAC;QACD,IAAI,SAAS,CAAC;QACd,MAAM,WAAW,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QAClE,IAAI,WAAW,EAAE,CAAC;YACjB,IAAI,gBAAgB,GAAG,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;YAC9F,IAAI,WAAW,GAAG,cAAc,EAAE,CAAC;YACnC,IAAI,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5E,SAAS;YACV,CAAC;QACF,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACxC,IAAI,SAAS,GAAG,CAAC;gBAChB,IAAI,iBAAiB;oBACpB,4GAA4G;oBAC5G,0BAA0B;oBAC1B,SAAS,GAAG;wBACX,EAAE,EAAE,iBAAiB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;qBACjG,CAAC;;oBACE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;iBACxG,IAAI,iBAAiB;gBAAE,SAAS,GAAG,EAAE,EAAE,EAAE,iBAAiB,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;;gBAC1E,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,YAAY,CAAC,KAAK,CAAC,0BAA0B,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;YAC5D,SAAS;QACV,CAAC;QACD,SAAS,CAAC,IAAI,CACb,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/B,MAAM;iBACJ,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE;gBACvB,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;gBAC1E,YAAY,CAAC,KAAK,CAAC,oBAAoB,GAAG,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC3D,CAAC,CAAC;iBACD,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACvB,CAAC,CAAC,CACF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC/B,CAAC;AACD,IAAI,CAAC,YAAY,IAAI,CAAC,UAAU,EAAE,aAAa,EAAE,CAAC;IACjD,YAAY,EAAE,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,QAAQ,EAAE,OAAO;IAClC,IAAI,gBAAgB,GAAG,OAAO,CAAC,qCAAqC,CAAC,CAAC,gBAAgB,CAAC;IACvF,IAAI,YAAY,CAAC;IACjB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,gBAAgB,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAC;QACzF,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjC,YAAY,GAAG,wBAAwB,CACtC;YACC,kBAAkB,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;YACnD,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,wIAAwI;YACvJ,SAAS,EAAE,IAAI;YACf,qBAAqB,EAAE,GAAG,EAAE,8EAA8E;YAC1G,0GAA0G;YAC1G,6CAA6C;YAC7C,OAAO,EAAE,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO;YACnD,WAAW;SACX,EACD,QAAQ,CACR,CAAC;QACF,WAAW,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QACrC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,YAAY,CAAC;IAC5C,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAC;QACnF,YAAY,GAAG,kBAAkB,CAAC,QAAQ,EAAE;YAC3C,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI;YACf,qBAAqB,EAAE,GAAG;SAC1B,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;IACtC,CAAC;IACD,OAAO,YAAY,CAAC;AACrB,CAAC"}

package/dist/server/throttle.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Throttle function to limit the number of calls to a function so that the event queue doesn't get overwhelmed.
+ * @param fn
+ * @param onLimitExceeded
+ * @param maxQueueTimeLimit
+ */
+export declare function throttle(fn: (...args: any) => any, onLimitExceeded?: (...args: any) => any, maxQueueTimeLimit?: number): (...args: any[]) => any;

package/dist/server/throttle.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.throttle = throttle;
+const logger_ts_1 = require("../utility/logging/logger.js");
+const MAX_EVENT_DELAY_TIME = 3000;
+const DEFAULT_MAX_QUEUE_TIME = 20_000; // 20 seconds
+let lastWarning = 0;
+const WARNING_INTERVAL = 30000;
+const EVENT_QUEUE_MONITORING_INTERVAL = 3000;
+let lastEventQueueCheck = performance.now() + EVENT_QUEUE_MONITORING_INTERVAL;
+let averageEventCycleTime = 0;
+/**
+ * Throttle function to limit the number of calls to a function so that the event queue doesn't get overwhelmed.
+ * @param fn
+ * @param onLimitExceeded
+ * @param maxQueueTimeLimit
+ */
+function throttle(fn, onLimitExceeded, maxQueueTimeLimit = DEFAULT_MAX_QUEUE_TIME) {
+    let queuedCalls;
+    return function (...args) {
+        if (queuedCalls) {
+            // this is an estimate of the time an event will take to process, based on the average event cycle time and the queue depth
+            if (queuedCalls.length * averageEventCycleTime > maxQueueTimeLimit) {
+                return onLimitExceeded(...args);
+            }
+            return new Promise((resolve, reject) => {
+                queuedCalls.push({
+                    args,
+                    fn() {
+                        try {
+                            const result = fn(...args);
+                            resolve(result);
+                        }
+                        catch (e) {
+                            reject(e);
+                        }
+                    },
+                });
+            });
+        }
+        queuedCalls = [];
+        waitForNextCycle(performance.now());
+        return fn(...args);
+    };
+    function waitForNextCycle(startTime) {
+        setImmediate(() => {
+            const now = performance.now();
+            // get the decaying/running average of the event cycle time
+            averageEventCycleTime = (averageEventCycleTime * 4 + now - startTime) / 5;
+            const nextCall = queuedCalls.shift();
+            if (nextCall) {
+                const { fn: nextFunction } = nextCall;
+                nextFunction();
+                waitForNextCycle(now);
+            }
+            else {
+                queuedCalls = null;
+            }
+        });
+    }
+}
+setInterval(() => {
+    const now = performance.now();
+    if (now - lastEventQueueCheck - EVENT_QUEUE_MONITORING_INTERVAL > MAX_EVENT_DELAY_TIME &&
+        lastWarning + WARNING_INTERVAL < now) {
+        logger_ts_1.logger.warn?.(`JavaScript execution has taken too long and is not allowing proper event queue cycling, consider using 'await new Promise(setImmediate)' in code that will execute for a long duration`);
+        lastWarning = now;
+    }
+    lastEventQueueCheck = now;
+}, EVENT_QUEUE_MONITORING_INTERVAL).unref();
+//# sourceMappingURL=throttle.js.map

package/dist/server/throttle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"throttle.js","sourceRoot":"","sources":["../../server/throttle.ts"],"names":[],"mappings":";;AAcA,4BA6CC;AA3DD,4DAAsD;AACtD,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,sBAAsB,GAAG,MAAM,CAAC,CAAC,aAAa;AACpD,IAAI,WAAW,GAAG,CAAC,CAAC;AACpB,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,MAAM,+BAA+B,GAAG,IAAI,CAAC;AAC7C,IAAI,mBAAmB,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,+BAA+B,CAAC;AAC9E,IAAI,qBAAqB,GAAG,CAAC,CAAC;AAC9B;;;;;GAKG;AACH,SAAgB,QAAQ,CACvB,EAAyB,EACzB,eAAuC,EACvC,iBAAiB,GAAG,sBAAsB;IAE1C,IAAI,WAAkB,CAAC;IACvB,OAAO,UAAU,GAAG,IAAW;QAC9B,IAAI,WAAW,EAAE,CAAC;YACjB,2HAA2H;YAC3H,IAAI,WAAW,CAAC,MAAM,GAAG,qBAAqB,GAAG,iBAAiB,EAAE,CAAC;gBACpE,OAAO,eAAe,CAAC,GAAG,IAAI,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACtC,WAAW,CAAC,IAAI,CAAC;oBAChB,IAAI;oBACJ,EAAE;wBACD,IAAI,CAAC;4BACJ,MAAM,MAAM,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;4BAC3B,OAAO,CAAC,MAAM,CAAC,CAAC;wBACjB,CAAC;wBAAC,OAAO,CAAC,EAAE,CAAC;4BACZ,MAAM,CAAC,CAAC,CAAC,CAAC;wBACX,CAAC;oBACF,CAAC;iBACD,CAAC,CAAC;YACJ,CAAC,CAAC,CAAC;QACJ,CAAC;QACD,WAAW,GAAG,EAAE,CAAC;QACjB,gBAAgB,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QACpC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IACF,SAAS,gBAAgB,CAAC,SAAiB;QAC1C,YAAY,CAAC,GAAG,EAAE;YACjB,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC9B,2DAA2D;YAC3D,qBAAqB,GAAG,CAAC,qBAAqB,GAAG,CAAC,GAAG,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC1E,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,QAAQ,EAAE,CAAC;gBACd,MAAM,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,QAAQ,CAAC;gBACtC,YAAY,EAAE,CAAC;gBACf,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACP,WAAW,GAAG,IAAI,CAAC;YACpB,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;AACF,CAAC;AACD,WAAW,CAAC,GAAG,EAAE;IAChB,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC9B,IACC,GAAG,GAAG,mBAAmB,GAAG,+BAA+B,GAAG,oBAAoB;QAClF,WAAW,GAAG,gBAAgB,GAAG,GAAG,EACnC,CAAC;QACF,kBAAM,CAAC,IAAI,EAAE,CACZ,wLAAwL,CACxL,CAAC;QACF,WAAW,GAAG,GAAG,CAAC;IACnB,CAAC;IACD,mBAAmB,GAAG,GAAG,CAAC;AAC3B,CAAC,EAAE,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC"}

package/dist/sqlTranslator/SelectValidator.d.ts

@@ -0,0 +1,79 @@
+export = SelectValidator;
+/**
+ * Validates the tables and attributes against the actual schema
+ * Validates general SQL rules
+ */
+declare class SelectValidator {
+    constructor(statement: any);
+    statement: any;
+    attributes: any[];
+    /**
+     * entry point for validation
+     * @returns {*}
+     */
+    validate(): any;
+    /**
+     * if the statement has columns in it:
+     * loops thru the from and join arrays of the AST and passes individual entries into validateTable
+     */
+    [validateTables](): void;
+    /**
+     * Checks that the table exists in the schema, adds all of it's attributes to the class level collection this.attributes
+     * @param table
+     */
+    [validateTable](table: any): void;
+    /**
+     * validates the column against the schema
+     * @param column
+     * @returns {*[]}
+     */
+    [findColumn](column: any): any[];
+    /**
+     * detects * in the select, if found adds all columns to the select
+     */
+    [checkColumnsForAsterisk](): void;
+    /**
+     * takes a table and adds all of it's columns to the select. if no table it adds every column from every table in the select
+     * @param tableName
+     */
+    [setColumnsForTable](tableName: any): void;
+    /**
+     * passes segments to ValidateSegment for validation
+     */
+    [validateAllColumns](): void;
+    /**
+     * iterates the attributes in a segment and validates them against the schema
+     * @param segment
+     * @param isOrderBy
+     * @returns {*}
+     */
+    [validateSegment](segment: any, isOrderBy: any): any;
+    /**
+     * validation specific for GROUP BY
+     * makes sure that the non-aggregate functions and columns from the select are represented in the group by and the columns match the schema
+     * @param segment
+     */
+    [validateGroupBy](segment: any): void;
+    /**
+     * Order BY specific validation
+     *
+     * @param column
+     */
+    [validateOrderBy](column: any): void;
+    /**
+     * validates a column to the schema
+     * @param column
+     * @returns {*}
+     */
+    [validateColumn](column: any): any;
+}
+declare const validateTables: unique symbol;
+declare const validateTable: unique symbol;
+declare const findColumn: unique symbol;
+declare const checkColumnsForAsterisk: unique symbol;
+declare const setColumnsForTable: unique symbol;
+declare const validateAllColumns: unique symbol;
+declare const validateSegment: unique symbol;
+declare const validateGroupBy: unique symbol;
+declare const validateOrderBy: unique symbol;
+declare const validateColumn: unique symbol;