@naturalcycles/backend-lib 9.63.0 → 9.64.0

package/dist/express/startServer.d.ts

@@ -32,7 +32,7 @@ export interface StartServerCfg extends DefaultAppCfg {
      */
     onShutdown?: () => Promise<void>;
     /**
-     * @default 3000
+     * @default 8_000
      */
     forceShutdownTimeout?: number;
     sentryService?: SentrySharedService;

package/dist/express/startServer.js

@@ -45,8 +45,13 @@ export class BackendServer {
                 resolve(server);
             });
         });
-        // This is to fix GCP LoadBalancer race condition
-        this.server.keepAliveTimeout = 600 * 1000; // 10 minutes
+        // This is to fix GCP LoadBalancer race condition:
+        // GCLB's backend keep-alive timeout is fixed at 600s, and the backend must use
+        // a strictly higher value, otherwise the server can close an idle connection at the
+        // same moment GCLB reuses it, resulting in intermittent 502s.
+        // https://cloud.google.com/load-balancing/docs/https#timeouts_and_retries
+        this.server.keepAliveTimeout = 620 * 1000; // slightly above GCLB's 600s
+        this.server.headersTimeout = 630 * 1000; // conventionally kept above keepAliveTimeout
         let address = `http://localhost:${port}`; // default
         const addr = this.server.address();
         if (addr) {
@@ -77,7 +82,7 @@ export class BackendServer {
         const shutdownTimeout = setTimeout(() => {
             console.log(boldGrey('Forceful shutdown after timeout'));
             process.exit(0);
-        }, this.cfg.forceShutdownTimeout ?? 10_000);
+        }, this.cfg.forceShutdownTimeout ?? 8000);
         try {
             await Promise.all([
                 this.server && new Promise(r => this.server.close(r)),

package/dist/upload/getUploadedFiles.d.ts

@@ -0,0 +1,43 @@
+import type { NumberOfBytes } from '@naturalcycles/js-lib/types';
+import type { BackendRequest } from '../server/server.model.js';
+/**
+ * Parses a `multipart/form-data` request body (on top of `busboy`), populates `req.body` with the
+ * form fields, and returns the requested files keyed by their form field name:
+ *
+ *     const { csv } = await getUploadedFiles(req, { files: ['csv'] })
+ *     const { csv, avatar } = await getUploadedFiles(req, {
+ *       files: ['csv'],            // required   -> UploadedFile
+ *       optionalFiles: ['avatar'], // optional   -> UploadedFile | undefined
+ *     })
+ *
+ * `files` are required: each is asserted to be present and non-empty (throws 400 otherwise).
+ * `optionalFiles` are returned as `UploadedFile | undefined` — `undefined` when the field is
+ * absent or empty (e.g. a blank file input), so "optional" is genuinely optional.
+ *
+ * Throws 413 as soon as ANY uploaded file exceeds `maxFileSize` bytes (default: 10 MB), aborting
+ * the parse immediately without buffering the rest of the body.
+ *
+ * Meant to be called from inside a request handler, AFTER authentication — so an
+ * unauthenticated/unauthorized request is rejected before its (potentially large) multipart body
+ * is buffered into memory.
+ */
+export declare function getUploadedFiles<const FILENAMES extends string = never, const OPTIONAL_FILENAMES extends string = never>(req: BackendRequest, opt: {
+    files?: readonly FILENAMES[];
+    optionalFiles?: readonly OPTIONAL_FILENAMES[];
+    maxFileSize?: NumberOfBytes;
+}): Promise<Record<FILENAMES, UploadedFile> & Record<OPTIONAL_FILENAMES, UploadedFile | undefined>>;
+/**
+ * A single uploaded file, buffered in memory.
+ */
+export interface UploadedFile {
+    /** Original filename from the client. */
+    name: string;
+    /** Content-Type of the file part. */
+    mimeType: string;
+    /** Content-Transfer-Encoding of the file part. */
+    encoding: string;
+    /** File contents. */
+    data: Buffer;
+    /** Size of `data` in bytes. */
+    size: NumberOfBytes;
+}

package/dist/upload/getUploadedFiles.js

@@ -0,0 +1,94 @@
+import { _assert, AppError } from '@naturalcycles/js-lib/error';
+import busboy from 'busboy';
+/**
+ * Parses a `multipart/form-data` request body (on top of `busboy`), populates `req.body` with the
+ * form fields, and returns the requested files keyed by their form field name:
+ *
+ *     const { csv } = await getUploadedFiles(req, { files: ['csv'] })
+ *     const { csv, avatar } = await getUploadedFiles(req, {
+ *       files: ['csv'],            // required   -> UploadedFile
+ *       optionalFiles: ['avatar'], // optional   -> UploadedFile | undefined
+ *     })
+ *
+ * `files` are required: each is asserted to be present and non-empty (throws 400 otherwise).
+ * `optionalFiles` are returned as `UploadedFile | undefined` — `undefined` when the field is
+ * absent or empty (e.g. a blank file input), so "optional" is genuinely optional.
+ *
+ * Throws 413 as soon as ANY uploaded file exceeds `maxFileSize` bytes (default: 10 MB), aborting
+ * the parse immediately without buffering the rest of the body.
+ *
+ * Meant to be called from inside a request handler, AFTER authentication — so an
+ * unauthenticated/unauthorized request is rejected before its (potentially large) multipart body
+ * is buffered into memory.
+ */
+export async function getUploadedFiles(req, opt) {
+    const { files: requiredNames = [], optionalFiles = [], maxFileSize = 10 * MB } = opt;
+    const uploadedFiles = await parseMultipart(req, maxFileSize);
+    const result = {};
+    for (const name of requiredNames) {
+        const file = uploadedFiles[name];
+        _assert(file, `Uploaded file "${name}" is missing`, { backendResponseStatusCode: 400 });
+        _assert(file.size > 0, `Uploaded file "${name}" is empty`, { backendResponseStatusCode: 400 });
+        result[name] = file;
+    }
+    for (const name of optionalFiles) {
+        const file = uploadedFiles[name];
+        // Absent or empty (e.g. a blank file input) is treated as "not provided".
+        result[name] = file && file.size > 0 ? file : undefined;
+    }
+    return result;
+}
+/**
+ * Streams the request through busboy, buffering each file into memory and collecting the form
+ * fields onto `req.body`. Resolves with the uploaded files keyed by their form field name.
+ *
+ * Rejects (aborting the parse) the moment a file exceeds `maxFileSize`.
+ */
+async function parseMultipart(req, maxFileSize) {
+    return new Promise((resolve, reject) => {
+        const files = {};
+        const body = {};
+        let bb;
+        try {
+            bb = busboy({ headers: req.headers, limits: { fileSize: maxFileSize } });
+        }
+        catch {
+            // Not a multipart request (or malformed Content-Type) — no files to parse
+            resolve(files);
+            return;
+        }
+        bb.on('field', (name, value) => {
+            body[name] = value;
+        });
+        bb.on('file', (name, stream, info) => {
+            const chunks = [];
+            stream.on('data', (chunk) => chunks.push(chunk));
+            // Abort as soon as the size limit is hit: stop feeding busboy and reject right away,
+            // rather than buffering the whole over-limit body and reporting it at the end.
+            stream.on('limit', () => {
+                req.unpipe(bb);
+                req.resume(); // drain the remainder so the connection can close cleanly
+                reject(new AppError(`Uploaded file "${name}" exceeds the size limit of ${maxFileSize} bytes`, {
+                    backendResponseStatusCode: 413,
+                }));
+            });
+            stream.on('close', () => {
+                const data = Buffer.concat(chunks);
+                files[name] = {
+                    name: info.filename,
+                    mimeType: info.mimeType,
+                    encoding: info.encoding,
+                    data,
+                    size: data.length,
+                };
+            });
+        });
+        bb.on('error', reject);
+        bb.on('close', () => {
+            req.body = body;
+            resolve(files);
+        });
+        req.pipe(bb);
+    });
+}
+const MB = 1024 * 1024;

package/package.json

@@ -1,27 +1,28 @@
 {
   "name": "@naturalcycles/backend-lib",
   "type": "module",
-  "version": "9.63.0",
+  "version": "9.64.0",
   "dependencies": {
     "@naturalcycles/db-lib": "^10",
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15",
     "@types/body-parser": "^1",
-    "@types/compressible": "^2",
     "@types/cookie-parser": "^1",
     "@types/cors": "^2",
     "@types/express": "^5",
+    "@types/compressible": "^2",
     "@types/on-finished": "^2",
     "@types/on-headers": "^1",
     "@types/vary": "^1",
+    "busboy": "^1",
     "compressible": "^2",
     "cookie-parser": "^1",
     "cors": "^2",
     "express": "^5",
     "helmet": "^8",
     "negotiator": "^1",
-    "on-finished": "^2",
     "on-headers": "^1",
+    "on-finished": "^2",
     "tslib": "^2",
     "vary": "^1"
   },
@@ -43,9 +44,10 @@
   },
   "devDependencies": {
     "@sentry/node-core": "^10",
-    "fastify": "^5",
+    "@types/busboy": "^1",
     "typescript": "rc",
-    "@naturalcycles/dev-lib": "20.50.0"
+    "fastify": "^5",
+    "@naturalcycles/dev-lib": "0.0.0"
   },
   "exports": {
     ".": "./dist/index.js",
@@ -55,6 +57,7 @@
     "./deploy": "./dist/deploy/index.js",
     "./deploy/*.js": "./dist/deploy/*.js",
     "./express/*.js": "./dist/express/*.js",
+    "./upload": "./dist/upload/getUploadedFiles.js",
     "./validateRequest": "./dist/validation/ajv/validateRequest.js",
     "./onFinished": "./dist/onFinished.js",
     "./testing": "./dist/testing/index.js"

package/src/express/startServer.ts

@@ -53,8 +53,13 @@ export class BackendServer {
       })
     })
 
-    // This is to fix GCP LoadBalancer race condition
-    this.server.keepAliveTimeout = 600 * 1000 // 10 minutes
+    // This is to fix GCP LoadBalancer race condition:
+    // GCLB's backend keep-alive timeout is fixed at 600s, and the backend must use
+    // a strictly higher value, otherwise the server can close an idle connection at the
+    // same moment GCLB reuses it, resulting in intermittent 502s.
+    // https://cloud.google.com/load-balancing/docs/https#timeouts_and_retries
+    this.server.keepAliveTimeout = 620 * 1000 // slightly above GCLB's 600s
+    this.server.headersTimeout = 630 * 1000 // conventionally kept above keepAliveTimeout
 
     let address = `http://localhost:${port}` // default
 
@@ -97,7 +102,7 @@ export class BackendServer {
     const shutdownTimeout = setTimeout(() => {
       console.log(boldGrey('Forceful shutdown after timeout'))
       process.exit(0)
-    }, this.cfg.forceShutdownTimeout ?? 10_000)
+    }, this.cfg.forceShutdownTimeout ?? 8000)
 
     try {
       await Promise.all([
@@ -146,7 +151,7 @@ export interface StartServerCfg extends DefaultAppCfg {
   onShutdown?: () => Promise<void>
 
   /**
-   * @default 3000
+   * @default 8_000
    */
   forceShutdownTimeout?: number
 

package/src/upload/getUploadedFiles.ts

@@ -0,0 +1,140 @@
+import { _assert, AppError } from '@naturalcycles/js-lib/error'
+import type { NumberOfBytes } from '@naturalcycles/js-lib/types'
+import busboy from 'busboy'
+import type { Busboy, FileInfo } from 'busboy'
+import type { BackendRequest } from '../server/server.model.js'
+
+/**
+ * Parses a `multipart/form-data` request body (on top of `busboy`), populates `req.body` with the
+ * form fields, and returns the requested files keyed by their form field name:
+ *
+ *     const { csv } = await getUploadedFiles(req, { files: ['csv'] })
+ *     const { csv, avatar } = await getUploadedFiles(req, {
+ *       files: ['csv'],            // required   -> UploadedFile
+ *       optionalFiles: ['avatar'], // optional   -> UploadedFile | undefined
+ *     })
+ *
+ * `files` are required: each is asserted to be present and non-empty (throws 400 otherwise).
+ * `optionalFiles` are returned as `UploadedFile | undefined` — `undefined` when the field is
+ * absent or empty (e.g. a blank file input), so "optional" is genuinely optional.
+ *
+ * Throws 413 as soon as ANY uploaded file exceeds `maxFileSize` bytes (default: 10 MB), aborting
+ * the parse immediately without buffering the rest of the body.
+ *
+ * Meant to be called from inside a request handler, AFTER authentication — so an
+ * unauthenticated/unauthorized request is rejected before its (potentially large) multipart body
+ * is buffered into memory.
+ */
+export async function getUploadedFiles<
+  const FILENAMES extends string = never,
+  const OPTIONAL_FILENAMES extends string = never,
+>(
+  req: BackendRequest,
+  opt: {
+    files?: readonly FILENAMES[]
+    optionalFiles?: readonly OPTIONAL_FILENAMES[]
+    maxFileSize?: NumberOfBytes
+  },
+): Promise<Record<FILENAMES, UploadedFile> & Record<OPTIONAL_FILENAMES, UploadedFile | undefined>> {
+  const { files: requiredNames = [], optionalFiles = [], maxFileSize = 10 * MB } = opt
+
+  const uploadedFiles = await parseMultipart(req, maxFileSize)
+  const result: Record<string, UploadedFile | undefined> = {}
+
+  for (const name of requiredNames) {
+    const file = uploadedFiles[name]
+    _assert(file, `Uploaded file "${name}" is missing`, { backendResponseStatusCode: 400 })
+    _assert(file.size > 0, `Uploaded file "${name}" is empty`, { backendResponseStatusCode: 400 })
+    result[name] = file
+  }
+
+  for (const name of optionalFiles) {
+    const file = uploadedFiles[name]
+    // Absent or empty (e.g. a blank file input) is treated as "not provided".
+    result[name] = file && file.size > 0 ? file : undefined
+  }
+
+  return result as Record<FILENAMES, UploadedFile> &
+    Record<OPTIONAL_FILENAMES, UploadedFile | undefined>
+}
+
+/**
+ * Streams the request through busboy, buffering each file into memory and collecting the form
+ * fields onto `req.body`. Resolves with the uploaded files keyed by their form field name.
+ *
+ * Rejects (aborting the parse) the moment a file exceeds `maxFileSize`.
+ */
+async function parseMultipart(
+  req: BackendRequest,
+  maxFileSize: NumberOfBytes,
+): Promise<Record<string, UploadedFile>> {
+  return new Promise((resolve, reject) => {
+    const files: Record<string, UploadedFile> = {}
+    const body: Record<string, string> = {}
+
+    let bb: Busboy
+    try {
+      bb = busboy({ headers: req.headers, limits: { fileSize: maxFileSize } })
+    } catch {
+      // Not a multipart request (or malformed Content-Type) — no files to parse
+      resolve(files)
+      return
+    }
+
+    bb.on('field', (name, value) => {
+      body[name] = value
+    })
+
+    bb.on('file', (name: string, stream, info: FileInfo) => {
+      const chunks: Buffer[] = []
+      stream.on('data', (chunk: Buffer) => chunks.push(chunk))
+      // Abort as soon as the size limit is hit: stop feeding busboy and reject right away,
+      // rather than buffering the whole over-limit body and reporting it at the end.
+      stream.on('limit', () => {
+        req.unpipe(bb)
+        req.resume() // drain the remainder so the connection can close cleanly
+        reject(
+          new AppError(`Uploaded file "${name}" exceeds the size limit of ${maxFileSize} bytes`, {
+            backendResponseStatusCode: 413,
+          }),
+        )
+      })
+      stream.on('close', () => {
+        const data = Buffer.concat(chunks)
+        files[name] = {
+          name: info.filename,
+          mimeType: info.mimeType,
+          encoding: info.encoding,
+          data,
+          size: data.length,
+        }
+      })
+    })
+
+    bb.on('error', reject)
+    bb.on('close', () => {
+      req.body = body
+      resolve(files)
+    })
+
+    req.pipe(bb)
+  })
+}
+
+const MB = 1024 * 1024
+
+/**
+ * A single uploaded file, buffered in memory.
+ */
+export interface UploadedFile {
+  /** Original filename from the client. */
+  name: string
+  /** Content-Type of the file part. */
+  mimeType: string
+  /** Content-Transfer-Encoding of the file part. */
+  encoding: string
+  /** File contents. */
+  data: Buffer
+  /** Size of `data` in bytes. */
+  size: NumberOfBytes
+}