@koeroesi86/node-worker-express 22.3.12-fd2e588 → 23.3.4361459321-main

package/examples/staticWorker.js

@@ -1,237 +0,0 @@
-const path = require('path');
-const fs = require('fs');
-const crypto = require('crypto');
-const { execSync } = require('child_process');
-const { Writable } = require('stream');
-
-let timer;
-
-const features = {
-  enableEmit: false,
-};
-
-function debounce(fn = () => {}, timeout = 0) {
-  if (timer) clearTimeout(timer);
-  timer = setTimeout(() => {
-    timer = undefined;
-    fn();
-  }, timeout);
-}
-
-function getContentType(extension = '') {
-  const contentTypes = {
-    '.aac':	'audio/aac',
-    '.abw':	'application/x-abiword',
-    '.arc':	'application/x-freearc',
-    '.avi':	'video/x-msvideo',
-    '.azw':	'application/vnd.amazon.ebook',
-    '.bin':	'application/octet-stream',
-    '.bmp':	'image/bmp',
-    '.bz': 'application/x-bzip',
-    '.bz2': 'application/x-bzip2',
-    '.csh': 'application/x-csh',
-    '.css': 'text/css',
-    '.csv': 'text/csv',
-    '.doc': 'application/msword',
-    '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-    '.eot': 'application/vnd.ms-fontobject',
-    '.epub': 'application/epub+zip',
-    '.gz': 'application/gzip',
-    '.gif': 'image/gif',
-    '.htm': 'text/html',
-    '.html': 'text/html',
-    '.ico': 'image/vnd.microsoft.icon',
-    '.ics': 'text/calendar',
-    '.jar': 'application/java-archive',
-    '.jpeg': 'image/jpeg',
-    '.jpg': 'image/jpeg',
-    '.js': 'text/javascript',
-    '.json': 'application/json',
-    '.jsonld': 'application/ld+json',
-    '.mid': 'audio/midi',
-    '.midi': 'audio/x-midi',
-    '.mjs': 'text/javascript',
-    '.mp3': 'audio/mpeg',
-    '.mpeg': 'video/mpeg',
-    '.mpkg': 'application/vnd.apple.installer+xml',
-    '.odp': 'application/vnd.oasis.opendocument.presentation',
-    '.ods': 'application/vnd.oasis.opendocument.spreadsheet',
-    '.odt': 'application/vnd.oasis.opendocument.text',
-    '.oga': 'audio/ogg',
-    '.ogv': 'video/ogg',
-    '.ogx': 'application/ogg',
-    '.opus': 'audio/opus',
-    '.otf': 'font/otf',
-    '.png': 'image/png',
-    '.pdf': 'application/pdf',
-    '.php': 'application/php',
-    '.ppt': 'application/vnd.ms-powerpoint',
-    '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-    '.rar': 'application/vnd.rar',
-    '.rtf': 'application/rtf',
-    '.sh': 'application/x-sh',
-    '.svg': 'image/svg+xml',
-    '.swf': 'application/x-shockwave-flash',
-    '.tar': 'application/x-tar',
-    '.tif': 'image/tiff',
-    '.tiff': 'image/tiff',
-    '.ts': 'video/mp2t',
-    '.ttf': 'font/ttf',
-    '.txt': 'text/plain',
-    '.vsd': 'application/vnd.visio',
-    '.wav': 'audio/wav',
-    '.weba': 'audio/webm',
-    '.webm': 'video/webm',
-    '.webp': 'image/webp',
-    '.woff': 'font/woff',
-    '.woff2': 'font/woff2',
-    '.xhtml': 'application/xhtml+xml',
-    '.xls': 'application/vnd.ms-excel',
-    '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-    '.xml': 'application/xml',
-    '.xul': 'application/vnd.mozilla.xul+xml',
-    '.zip': 'application/zip',
-    '.3gp': 'video/3gpp',
-    '.3g2': 'video/3gpp2',
-    '.7z': 'application/x-7z-compressed',
-};
-
-  return contentTypes[extension] || contentTypes['.txt'];
-}
-
-function etag(body) {
-  if (body.length === 0) {
-    // fast-path empty
-    return '"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk"';
-  }
-
-  // compute hash of entity
-  const hash = crypto
-    .createHash('sha1')
-    .update(body, 'utf8')
-    .digest('base64')
-    .substring(0, 27);
-
-  // compute length of entity
-  const len = typeof body === 'string'
-    ? Buffer.byteLength(body, 'utf8')
-    : body.length;
-
-  return `"${len.toString(16)}-${hash}"`
-}
-
-/**
- * TODO: implement
- * @param {Buffer} body
- * @param {string} fileName
- * @returns {boolean|string}
- */
-function getCharset(body, fileName) {
-  // Byte order mark
-  if (body[0] === 0xEF && body[1] === 0xBB && body[2] === 0xBF)
-    return 'utf8';
-  if (body[0] === 0xFE && body[1] === 0xFF)
-    return 'utf16be';
-  if (body[0] === 0xFF && body[1] === 0xFE)
-    return 'utf16le';
-  if (body.indexOf('') === 0)
-    return 'iso-8859-1';
-
-  try {
-    return execSync(`file -b --mime-encoding ${fileName}`).toString('utf8').trim();
-  } catch (e) {
-    //
-  }
-
-  return false;
-}
-
-const staticWorker = (event, callback = () => {}) => {
-  debounce(() => {
-    console.log('Exiting static worker.');
-    process.exit(0);
-  }, 1000 * 60 * 30);
-  const currentPath = `${event.path.replace(/\.{2,}/, '')}${/\/$/.test(event.path) ? 'index.html' : ''}`;
-  const fileName = path.resolve(event.rootPath, `.${currentPath}`);
-  if (['GET', 'HEAD'].includes(event.httpMethod) && fs.existsSync(fileName)) {
-    // TODO: range request
-    const bodyBuffer = fs.readFileSync(fileName);
-    const stats = fs.statSync(fileName);
-    const extension = path.extname(fileName);
-    const contentType = getContentType(extension);
-    const currentEtag = etag(bodyBuffer);
-    const charset = getCharset(bodyBuffer, fileName);
-
-    const lastModified = new Date(stats.mtime);
-    let isModified = true;
-
-    if (event.headers['if-none-match'] === currentEtag) {
-      isModified = false;
-    } else if (event.headers['if-modified-since']) {
-      const ifModifiedSince = new Date(event.headers['if-modified-since']);
-      if (ifModifiedSince.toUTCString() === lastModified.toUTCString()) {
-        isModified = false;
-      }
-    }
-    const response = {
-      statusCode: isModified ? 200 : 304,
-      headers: {
-        'Content-Type': `${contentType}${charset ? `; charset=${charset}` : ''}`,
-        'Cache-Control': 'public, max-age=0',
-        'Content-Length': bodyBuffer.byteLength,
-        'ETag': currentEtag,
-        ...(stats.mtime && { 'Last-Modified': lastModified.toUTCString() }),
-      },
-      body: isModified ? bodyBuffer.toString('base64') : '',
-      isBase64Encoded: true,
-    };
-
-    if (features.enableEmit && isModified) {
-      const writer = new Writable({
-        /**
-         * @param {Buffer} chunk
-         * @param {string} encoding
-         * @param {Function} next
-         */
-        write(chunk, encoding, next) {
-          response.emit = true;
-          response.isBase64Encoded = true;
-          response.body = chunk.toString('base64');
-          // console.log('sending', chunk.length, 'bytes')
-          callback(response);
-          next();
-        },
-        destroy() {
-          response.isBase64Encoded = false;
-          response.emit = true;
-          response.body = null;
-          callback(response);
-        },
-      });
-
-      fs.createReadStream(fileName, {
-        // encoding: 'binary',
-        // encoding: 'utf-8',
-        emitClose: false,
-        autoClose: false,
-        // highWaterMark: 16,
-        start: 0,
-        end: Infinity,
-      }).pipe(writer);
-    } else {
-      callback(response);
-    }
-  } else {
-    callback({
-      statusCode: 404,
-      headers: {
-        'Content-Type': 'text/plain',
-        'Cache-Control': 'public, max-age=0',
-      },
-      body:`${fileName} does not exist`,
-      isBase64Encoded: false,
-    });
-  }
-};
-
-module.exports = staticWorker;

package/index.d.ts

@@ -1,64 +0,0 @@
-type MiddlewareOptions = {
-  root: string,
-  limit?: number,
-  limitPerPath?: number|((path: string) => number),
-  limitRequestBody?: number,
-  limitRequestTimeout?: number,
-  idleCheckTimeout?: number,
-  onStdout?: (data: Buffer) => void,
-  onStderr?: (data: Buffer) => void,
-  onExit?: void,
-  index?: string[],
-  env?: object,
-  staticWorker?: string,
-  cwd?: string,
-}
-
-declare namespace Middleware {
-  type RequestEvent = {
-    httpMethod: string;
-    protocol: string;
-    path: string;
-    pathFragments: string[];
-    queryStringParameters: { [key: string]: string };
-    headers: { [key: string]: string };
-    remoteAddress: string;
-    body: string;
-    rootPath: string;
-    closed?: boolean;
-    frame?: string;
-  }
-
-  type WorkerInputEvent = {
-    type: string;
-    requestId: string;
-    event?: RequestEvent;
-  }
-
-  type WorkerOutputEvent = {
-    type: string;
-    requestId: string;
-    event?: ResponseEvent|WSFrameEvent;
-  }
-
-  type ResponseEvent = {
-    statusCode: number;
-    headers?: { [key: string]: string };
-    isBase64Encoded?: boolean;
-    emit?: boolean;
-    body?: string;
-  }
-
-  type WSFrameEvent = {
-    sendWsMessage: boolean;
-    frame: string;
-  }
-
-  type InvokableWorker = (event: RequestEvent, callback: (e: ResponseEvent) => void) => void;
-}
-
-declare module '@koeroesi86/node-worker-express' {
-  import { RequestHandler } from "express";
-
-  export function middleware(options: MiddlewareOptions): RequestHandler;
-}

package/index.js

@@ -1,3 +0,0 @@
-const middleware = require('./middleware');
-
-module.exports.middleware = middleware;

package/middleware/bodyParser.js

@@ -1,36 +0,0 @@
-const ignoredMethods = ['get', 'delete', 'options', 'head'];
-
-const bodyParser = (config = { limitRequestBody: 1000000, shouldError: true }) => (request, response, next) => {
-  let body = '';
-
-  if (ignoredMethods.includes(request.method.toLowerCase())) {
-    request.body = body;
-    next();
-    return;
-  }
-
-  request.on('data', data => {
-    body += data;
-
-    // Too much POST data, kill the connection!
-    // 1e6 === 1 * Math.pow(10, 6) === 1 * 1000000 ~~~ 1MB
-    if (body.length > (config.limitRequestBody || 1000000)) {
-      body = "";
-
-      if (config.shouldError) {
-        response.writeHead(413, { 'Content-Type': 'text/plain' });
-        response.end();
-        request.connection.destroy();
-      } else {
-        next();
-      }
-    }
-  });
-
-  request.on('end', () => {
-    request.body = body;
-    next();
-  });
-};
-
-module.exports = bodyParser;

package/middleware/index.js

@@ -1,288 +0,0 @@
-const { v4: uuid } = require('uuid');
-const path = require('path');
-const fs = require('fs');
-const url = require('url');
-const { WORKER_EVENT } = require('../constants');
-const WorkerPool = require('../utils/workerPool');
-const isWebSocket = require('../utils/isWebSocket');
-const parseWsMessage = require('../utils/parseWsMessage');
-const constructWsMessage = require('../utils/constructWsMessage');
-const getClientIp = require('../utils/getClientIp');
-const createBodyParser = require('./bodyParser');
-
-const ForbiddenPaths = [
-  '..'
-];
-
-const Protocols = {
-  http: 'HTTP',
-  websocket: 'WS',
-};
-
-const DefaultOptions = {
-  root: null,
-  limit: 0,
-  limitPerPath: 0,
-  limitRequestBody: 1000000,
-  limitRequestTimeout: 5000,
-  idleCheckTimeout: 5,
-  onStdout: () => {
-  },
-  onStderr: () => {
-  },
-  onExit: () => {
-  },
-  onForbiddenPath: (request, response) => {
-    response.writeHead(500, { 'Content-Type': 'text/plain' });
-    response.end();
-    request.connection.destroy();
-  },
-  index: [],
-  env: {},
-  staticWorker: path.resolve(__dirname, '../examples/staticWorker.js'),
-  cwd: process.cwd(),
-};
-
-/**
- * @namespace Middleware
- * @param {MiddlewareOptions} [options]
- * @returns {RequestHandler}
- */
-const workerMiddleware = (options) => {
-  const config = {
-    ...DefaultOptions,
-    ...(options && options),
-  };
-  if (!config.root) {
-    throw new Error('No root path defined in configuration!');
-  }
-  const rootPath = path.resolve(config.root);
-  const workerPool = new WorkerPool({
-    overallLimit: config.limit,
-    onExit: config.onExit,
-    idleCheckTimeout: config.idleCheckTimeout,
-  });
-  const bodyParser = createBodyParser({ limitRequestBody: config.limitRequestBody });
-  const aliasCache = {};
-  const workerCache = {};
-
-  return async (request, response, next) => {
-    const {
-      query: queryStringParameters,
-      pathname
-    } = url.parse(request.url, true);
-    request.pathFragments = pathname.split(/\//gi).filter(Boolean);
-
-    try {
-      let isWorker = false;
-      const pathFragments = pathname.split(/\//gi).filter(Boolean);
-      let currentPathFragments = pathFragments.slice(0);
-      let pathExists = false;
-
-      await new Promise((resolve, reject) => {
-        if (request.pathFragments.find(p => ForbiddenPaths.includes(p))) {
-          config.onForbiddenPath(request, response);
-          reject();
-        }
-        resolve();
-      });
-
-      await Promise.race([
-        new Promise((res, rej) => setTimeout(rej, config.limitRequestTimeout)),
-        new Promise(res => bodyParser(request, response, res)),
-      ]);
-      let indexPath;
-
-      if (aliasCache[pathname] && fs.existsSync(aliasCache[pathname])) {
-        isWorker = true;
-        indexPath = aliasCache[pathname];
-      } else if (workerCache[pathname] && fs.existsSync(workerCache[pathname])) {
-        isWorker = true;
-        indexPath = workerCache[pathname];
-      } else {
-        indexPath = path.join(rootPath, ...currentPathFragments)
-
-        for (let i = currentPathFragments.length; i >= 0 && !pathExists; i--) {
-          if (pathExists) continue;
-          currentPathFragments.splice(i);
-          const currentPath = path.join(rootPath, ...currentPathFragments);
-          pathExists = fs.existsSync(currentPath);
-          if (!pathExists) continue;
-          const currentStats = fs.statSync(currentPath);
-
-          if (pathExists && currentStats.isDirectory()) {
-            // index fallback
-            pathExists = !!config.index.find(indexFile => {
-              const checkIndexFilePath = path.join(currentPath, indexFile);
-              if (fs.existsSync(checkIndexFilePath)) {
-                isWorker = true;
-                indexPath = checkIndexFilePath;
-                aliasCache[pathname] = checkIndexFilePath;
-                setTimeout(() => {
-                  aliasCache[pathname] = null;
-                  delete aliasCache[pathname];
-                }, 5000);
-                return true;
-              }
-            });
-            if (pathExists) break;
-          } else if (pathExists && currentStats.isFile()) {
-            if (config.index.includes(currentPathFragments[currentPathFragments.length - 1])) {
-              isWorker = true;
-              workerCache[pathname] = currentPath;
-              setTimeout(() => {
-                workerCache[pathname] = null;
-                delete workerCache[pathname];
-              }, 5000);
-            }
-          }
-        }
-      }
-
-      /** @type {RequestEvent} */
-      const event = {
-        httpMethod: request.method.toUpperCase(),
-        protocol: isWebSocket(request) ? Protocols.websocket : Protocols.http,
-        path: pathname,
-        pathFragments: pathFragments,
-        queryStringParameters: JSON.parse(JSON.stringify(queryStringParameters)),
-        headers: request.headers,
-        remoteAddress: getClientIp(request),
-        body: `${request.body}`,
-        rootPath: rootPath,
-      };
-
-      const worker = isWorker ? await workerPool.getWorker(
-        indexPath,
-        {
-          stdio: [ 'pipe', 'pipe', 'pipe', 'ipc' ],
-          env: { ...process.env, ...config.env },
-          cwd: config.cwd,
-        },
-        typeof config.limitPerPath === "function" ? config.limitPerPath(indexPath) : config.limitPerPath
-      ) : await workerPool.getWorker(
-        config.staticWorker,
-        {
-          cwd: process.cwd(),
-          stdio: [ 'pipe', 'pipe', 'pipe', 'ipc' ],
-        },
-        1
-      );
-
-      const requestId = uuid();
-
-      let firstReceived = false;
-      const requestSocketListener = data => {
-        const frame = parseWsMessage(data);
-
-        // TODO: filter open frame better
-        if (data.length === 8 && !firstReceived) {
-          firstReceived = true;
-          return;
-        }
-
-        firstReceived = true;
-
-        worker.postMessage({
-          type: WORKER_EVENT.WS_MESSAGE_RECEIVE,
-          requestId,
-          event: { ...event, frame }
-        });
-      };
-      if (event.protocol === Protocols.websocket) {
-        request.socket.on('data', requestSocketListener);
-      }
-
-      worker.instance.stdout.off('data', config.onStdout);
-      worker.instance.stdout.on('data', config.onStdout);
-
-      worker.instance.stderr.off('data', config.onStderr);
-      worker.instance.stderr.on('data', config.onStderr);
-
-      const messageListener = responseEvent => {
-        if (responseEvent.requestId === requestId) {
-          if (responseEvent.type === WORKER_EVENT.RESPONSE) {
-            worker.postMessage({
-              type: WORKER_EVENT.RESPONSE_ACKNOWLEDGE,
-              requestId,
-            });
-            /** @type {ResponseEvent|WSFrameEvent} event */
-            const { event } = responseEvent;
-            const bufferEncoding = event.isBase64Encoded ? 'base64' : 'utf8';
-
-            response.writeHead(event.statusCode, event.headers);
-            response.write(Buffer.from(event.body, bufferEncoding));
-            response.end();
-          }
-
-          if (responseEvent.type === WORKER_EVENT.RESPONSE_EMIT) {
-            worker.postMessage({
-              type: WORKER_EVENT.RESPONSE_ACKNOWLEDGE,
-              requestId,
-            });
-            /** @type {ResponseEvent|WSFrameEvent} event */
-            const { event } = responseEvent;
-            const bufferEncoding = event.isBase64Encoded ? 'base64' : 'utf8';
-
-            if (!response.headersSent) {
-              response.writeHead(event.statusCode, event.headers);
-            }
-            if (event.body !== null) {
-              response.write(Buffer.from(event.body, bufferEncoding).toString());
-            } else {
-              response.end();
-            }
-          }
-
-          if (responseEvent.type === WORKER_EVENT.WS_MESSAGE_SEND) {
-            request.socket.write(constructWsMessage(responseEvent.event.frame));
-          }
-
-          if (responseEvent.type === WORKER_EVENT.REQUEST_ACKNOWLEDGE) {
-            worker.busy = false;
-          }
-        }
-      };
-
-      const requestCloseListener = () => {
-        worker.postMessage({
-          type: WORKER_EVENT.WS_CONNECTION_CLOSE,
-          requestId,
-          event
-        });
-        if (request.socket && request.socket.off) request.socket.off('close', requestCloseListener);
-        worker.removeEventListener('message', messageListener);
-      };
-      request.socket.on('close', requestCloseListener);
-
-      worker.addEventListener('message', messageListener);
-      worker.postMessage({
-        type: WORKER_EVENT.REQUEST,
-        requestId,
-        event,
-      });
-
-      const cleanupConnection = () => {
-        worker.removeEventListener('message', messageListener);
-        if (request && request.off) {
-          request.off('close', cleanupConnection);
-          request.off('aborted', cleanupConnection);
-        }
-
-        if (request.socket && request.socket.off) {
-          request.socket.off('data', requestSocketListener);
-          request.socket.off('close', requestCloseListener);
-        }
-      };
-      request.on('aborted', cleanupConnection);
-      if (event.protocol === Protocols.http) {
-        request.on('close', cleanupConnection);
-        response.on('finish', cleanupConnection)
-      }
-    } catch (e) {
-      next(e);
-    }
-  };
-};
-
-module.exports = workerMiddleware;

package/middleware/workerInvoke.js

@@ -1,54 +0,0 @@
-const { WORKER_EVENT } = require('../constants');
-
-/** @namespace Middleware */
-
-/** @type {InvokableWorker} worker */
-const worker = require(process.argv[2]);
-
-/** @type {WorkerInputEvent} message */
-function messageListener(message) {
-  if (message.type === WORKER_EVENT.REQUEST)  {
-    process.send({
-      type: WORKER_EVENT.REQUEST_ACKNOWLEDGE,
-      requestId: message.requestId,
-    });
-    const callback = responseEvent => {
-      /** @type {WorkerOutputEvent} e */
-      const e = {
-        type: WORKER_EVENT.RESPONSE,
-        requestId: message.requestId,
-        event: responseEvent,
-      };
-      if (responseEvent.sendWsMessage) e.type = WORKER_EVENT.WS_MESSAGE_SEND;
-      if (responseEvent.emit) e.type = WORKER_EVENT.RESPONSE_EMIT;
-      process.send(e);
-    };
-    worker(message.event, callback);
-  }
-
-  if (message.type === WORKER_EVENT.WS_MESSAGE_RECEIVE) {
-    const callback = responseEvent => {
-      /** @type {WorkerOutputEvent} e */
-      const e = {
-        type: WORKER_EVENT.WS_MESSAGE_SEND,
-        requestId: message.requestId,
-        event: responseEvent,
-      }
-      process.send(e);
-    };
-    worker(message.event, callback);
-  }
-
-  if (message.type === WORKER_EVENT.WS_CONNECTION_CLOSE) {
-    /** @type {ResponseEvent} responseEvent */
-    worker({ ...message.event, closed: true }, responseEvent => {
-      process.send({
-        type: WORKER_EVENT.WS_CONNECTION_CLOSE_ACKNOWLEDGE,
-        requestId: message.requestId,
-        event: responseEvent,
-      });
-    });
-  }
-}
-
-process.on('message', messageListener);

package/nodemon.json

@@ -1,6 +0,0 @@
-{
-  "ignore": [
-    ".git"
-  ],
-  "signal": "SIGINT"
-}

package/scripts/version.js

@@ -1,9 +0,0 @@
-const { execSync } = require('child_process');
-
-const { TRAVIS_TAG, TRAVIS_BUILD_NUMBER } = process.env;
-
-const now = new Date();
-const version = `${(now.getFullYear() + '').substring(2)}.${((now.getMonth() + 1) + '').padStart(2, '0')}.${TRAVIS_BUILD_NUMBER}-${TRAVIS_TAG || 'release'}`;
-
-execSync(`yarn version --no-git-tag-version --new-version "v${version}"`, { shell: true });
-console.log('version set', version);