@eggjs/multipart 4.0.0

package/src/app/extend/context.ts

@@ -0,0 +1,368 @@
+import assert from 'node:assert';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import fs from 'node:fs/promises';
+import { createWriteStream } from 'node:fs';
+import { Readable, PassThrough } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+// @ts-expect-error no types
+import parse from 'co-busboy';
+import dayjs from 'dayjs';
+import { Context } from '@eggjs/core';
+import { humanizeBytes } from '../../lib/utils.js';
+import { LimitError } from '../../lib/LimitError.js';
+import { MultipartFileTooLargeError } from '../../lib/MultipartFileTooLargeError.js';
+
+const HAS_CONSUMED = Symbol('Context#multipartHasConsumed');
+
+export interface EggFile {
+  field: string;
+  filename: string;
+  encoding: string;
+  mime: string;
+  filepath: string;
+}
+
+export interface MultipartFileStream extends Readable {
+  fields: Record<string, any>;
+  filename: string;
+  fieldname: string;
+  mime: string;
+  mimeType: string;
+  transferEncoding: string;
+  encoding: string;
+  truncated: boolean;
+}
+
+export interface MultipartOptions {
+  autoFields?: boolean;
+  /**
+   * required file submit, default is true
+   */
+  requireFile?: boolean;
+  /**
+   * default charset encoding
+   */
+  defaultCharset?: string;
+  /**
+   * compatible with defaultCharset
+   * @deprecated use `defaultCharset` instead
+   */
+  defCharset?: string;
+  defaultParamCharset?: string;
+  /**
+   * compatible with defaultParamCharset
+   * @deprecated use `defaultParamCharset` instead
+   */
+  defParamCharset?: string;
+  limits?: {
+    fieldNameSize?: number;
+    fieldSize?: number;
+    fields?: number;
+    fileSize?: number;
+    files?: number;
+    parts?: number;
+    headerPairs?: number;
+  };
+  checkFile?(
+    fieldname: string,
+    file: any,
+    filename: string,
+    encoding: string,
+    mimetype: string
+  ): void | Error;
+}
+
+export default class MultipartContext extends Context {
+  /**
+   * create multipart.parts instance, to get separated files.
+   * @function Context#multipart
+   * @param {Object} [options] - override default multipart configurations
+   *  - {Boolean} options.autoFields
+   *  - {String} options.defaultCharset
+   *  - {String} options.defaultParamCharset
+   *  - {Object} options.limits
+   *  - {Function} options.checkFile
+   * @return {Yieldable | AsyncIterable<Yieldable>} parts
+   */
+  multipart(options: MultipartOptions = {}): AsyncIterable<MultipartFileStream> {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const ctx = this;
+    // multipart/form-data
+    if (!ctx.is('multipart')) ctx.throw(400, 'Content-Type must be multipart/*');
+
+    assert(!ctx[HAS_CONSUMED], 'the multipart request can\'t be consumed twice');
+    ctx[HAS_CONSUMED] = true;
+
+    const { autoFields, defaultCharset, defaultParamCharset, checkFile } = ctx.app.config.multipart;
+    const { fieldNameSize, fieldSize, fields, fileSize, files } = ctx.app.config.multipart;
+    options = extractOptions(options);
+
+    const parseOptions = Object.assign({
+      autoFields,
+      defCharset: defaultCharset,
+      defParamCharset: defaultParamCharset,
+      checkFile,
+    }, options);
+
+    // https://github.com/mscdex/busboy#busboy-methods
+    // merge limits
+    parseOptions.limits = Object.assign({
+      fieldNameSize,
+      fieldSize,
+      fields,
+      fileSize,
+      files,
+    }, options.limits);
+
+    // mount asyncIterator, so we can use `for await` to get parts
+    const parts = parse(this, parseOptions);
+    parts[Symbol.asyncIterator] = async function* () {
+      let part: MultipartFileStream | undefined;
+      do {
+        part = await parts();
+
+        if (!part) continue;
+
+        if (Array.isArray(part)) {
+          if (part[3]) throw new LimitError('Request_fieldSize_limit', 'Reach fieldSize limit');
+          // TODO: still not support at busboy 1.x (only support at urlencoded)
+          // https://github.com/mscdex/busboy/blob/v0.3.1/lib/types/multipart.js#L5
+          // https://github.com/mscdex/busboy/blob/master/lib/types/multipart.js#L251
+          // if (part[2]) throw new LimitError('Request_fieldNameSize_limit', 'Reach fieldNameSize limit');
+        } else {
+          // user click `upload` before choose a file, `part` will be file stream, but `part.filename` is empty must handler this, such as log error.
+          if (!part.filename) {
+            ctx.coreLogger.debug('[egg-multipart] file field `%s` is upload without file stream, will drop it.', part.fieldname);
+            await pipeline(part, new PassThrough());
+            continue;
+          }
+          // TODO: check whether filename is malicious input
+
+          // busboy only set truncated when consume the stream
+          if (part.truncated) {
+            // in case of emit 'limit' too fast
+            throw new LimitError('Request_fileSize_limit', 'Reach fileSize limit');
+          } else {
+            part.once('limit', function(this: MultipartFileStream) {
+              this.emit('error', new LimitError('Request_fileSize_limit', 'Reach fileSize limit'));
+              this.resume();
+            });
+          }
+        }
+
+        // dispatch part to outter logic such as for-await-of
+        yield part;
+
+      } while (part !== undefined);
+    };
+    return parts;
+  }
+
+  /**
+   * save request multipart data and files to `ctx.request`
+   * @function Context#saveRequestFiles
+   * @param {Object} options - { limits, checkFile, ... }
+   */
+  async saveRequestFiles(options: MultipartOptions = {}) {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const ctx = this;
+
+    const allowArrayField = ctx.app.config.multipart.allowArrayField;
+
+    let storeDir: string | undefined;
+
+    const requestBody: Record<string, any> = {};
+    const requestFiles: EggFile[] = [];
+
+    options.autoFields = false;
+    const parts = ctx.multipart(options);
+
+    try {
+      for await (const part of parts) {
+        if (Array.isArray(part)) {
+          // fields
+          const [ fieldName, fieldValue ] = part;
+          if (!allowArrayField) {
+            requestBody[fieldName] = fieldValue;
+          } else {
+            if (!requestBody[fieldName]) {
+              requestBody[fieldName] = fieldValue;
+            } else if (!Array.isArray(requestBody[fieldName])) {
+              requestBody[fieldName] = [ requestBody[fieldName], fieldValue ];
+            } else {
+              requestBody[fieldName].push(fieldValue);
+            }
+          }
+        } else {
+          // stream
+          const { filename, fieldname, encoding, mime } = part;
+
+          if (!storeDir) {
+            // ${tmpdir}/YYYY/MM/DD/HH
+            storeDir = path.join(ctx.app.config.multipart.tmpdir, dayjs().format('YYYY/MM/DD/HH'));
+            await fs.mkdir(storeDir, { recursive: true });
+          }
+
+          // write to tmp file
+          const filepath = path.join(storeDir, randomUUID() + path.extname(filename));
+          const target = createWriteStream(filepath);
+          await pipeline(part, target);
+
+          const meta = {
+            filepath,
+            field: fieldname,
+            filename,
+            encoding,
+            mime,
+            // keep same property name as file stream, https://github.com/cojs/busboy/blob/master/index.js#L114
+            fieldname,
+            transferEncoding: encoding,
+            mimeType: mime,
+          };
+
+          requestFiles.push(meta);
+        }
+      }
+    } catch (err) {
+      await ctx.cleanupRequestFiles(requestFiles);
+      throw err;
+    }
+
+    ctx.request.body = requestBody;
+    ctx.request.files = requestFiles;
+  }
+
+  /**
+   * get upload file stream
+   * @example
+   * ```js
+   * const stream = await ctx.getFileStream();
+   * // get other fields
+   * console.log(stream.fields);
+   * ```
+   * @function Context#getFileStream
+   * @param {Object} options
+   *  - {Boolean} options.requireFile - required file submit, default is true
+   *  - {String} options.defaultCharset
+   *  - {String} options.defaultParamCharset
+   *  - {Object} options.limits
+   *  - {Function} options.checkFile
+   * @return {ReadStream} stream
+   * @since 1.0.0
+   * @deprecated Not safe enough, use `ctx.multipart()` instead
+   */
+  async getFileStream(options: MultipartOptions = {}): Promise<MultipartFileStream> {
+    options.autoFields = true;
+    const parts: any = this.multipart(options);
+    let stream: MultipartFileStream = await parts();
+
+    if (options.requireFile !== false) {
+      // stream not exists, treat as an exception
+      if (!stream || !stream.filename) {
+        this.throw(400, 'Can\'t found upload file');
+      }
+    }
+
+    if (!stream) {
+      stream = Readable.from([]) as MultipartFileStream;
+    }
+
+    if (stream.truncated) {
+      throw new LimitError('Request_fileSize_limit', 'Request file too large, please check multipart config');
+    }
+
+    stream.fields = parts.field;
+    stream.once('limit', () => {
+      const err = new MultipartFileTooLargeError(
+        'Request file too large, please check multipart config', stream.fields, stream.filename);
+      if (stream.listenerCount('error') > 0) {
+        stream.emit('error', err);
+        this.coreLogger.warn(err);
+      } else {
+        this.coreLogger.error(err);
+        // ignore next error event
+        stream.on('error', () => {});
+      }
+      // ignore all data
+      stream.resume();
+    });
+    return stream;
+  }
+
+  /**
+   * clean up request tmp files helper
+   * @function Context#cleanupRequestFiles
+   * @param {Array<String>} [files] - file paths need to cleanup, default is `ctx.request.files`.
+   */
+  async cleanupRequestFiles(files?: EggFile[]) {
+    if (!files || !files.length) {
+      files = this.request.files;
+    }
+    if (Array.isArray(files)) {
+      for (const file of files) {
+        try {
+          await fs.rm(file.filepath, { force: true, recursive: true });
+        } catch (err) {
+          // warning log
+          this.coreLogger.warn('[egg-multipart-cleanupRequestFiles-error] file: %j, error: %s', file, err);
+        }
+      }
+    }
+  }
+}
+
+function extractOptions(options: MultipartOptions = {}) {
+  const opts: MultipartOptions = {};
+  if (typeof options.autoFields === 'boolean') {
+    opts.autoFields = options.autoFields;
+  }
+  if (options.limits) {
+    opts.limits = options.limits;
+  }
+  if (options.checkFile) {
+    opts.checkFile = options.checkFile;
+  }
+
+  if (options.defCharset) {
+    opts.defCharset = options.defCharset;
+  }
+  if (options.defParamCharset) {
+    opts.defParamCharset = options.defParamCharset;
+  }
+  // compatible with config names
+  if (options.defaultCharset) {
+    opts.defCharset = options.defaultCharset;
+  }
+  if (options.defaultParamCharset) {
+    opts.defParamCharset = options.defaultParamCharset;
+  }
+
+  // limits
+  if (options.limits) {
+    const limits: Record<string, number | undefined> = opts.limits = { ...options.limits };
+    for (const key in limits) {
+      if (key.endsWith('Size') && limits[key]) {
+        limits[key] = humanizeBytes(limits[key]);
+      }
+    }
+  }
+
+  return opts;
+}
+
+declare module '@eggjs/core' {
+  interface Request {
+    /**
+     * Files Object Array
+     */
+    files?: EggFile[];
+  }
+
+  interface Context {
+    saveRequestFiles(options?: MultipartOptions): Promise<void>;
+    getFileStream(options?: MultipartOptions): Promise<MultipartFileStream>;
+    cleanupRequestFiles(files?: EggFile[]): Promise<void>;
+  }
+}
+

package/src/app/middleware/multipart.ts

@@ -0,0 +1,20 @@
+import { pathMatching } from 'egg-path-matching';
+import type { Context, Next, EggCore } from '@eggjs/core';
+import type { MultipartConfig } from '../../config/config.default.js';
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export default (options: MultipartConfig, _app: EggCore) => {
+  // normalize options
+  const matchFn = options.fileModeMatch && pathMatching({
+    match: options.fileModeMatch,
+    // pathToRegexpModule: app.options.pathToRegexpModule,
+  });
+
+  return async function multipart(ctx: Context, next: Next) {
+    if (!ctx.is('multipart')) return next();
+    if (matchFn && !matchFn(ctx)) return next();
+
+    await ctx.saveRequestFiles();
+    return next();
+  };
+};

package/src/app/schedule/clean_tmpdir.ts

@@ -0,0 +1,55 @@
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import dayjs from 'dayjs';
+import { EggCore } from '@eggjs/core';
+
+export default (app: EggCore): any => {
+  return class CleanTmpdir extends app.Subscription {
+    static get schedule() {
+      return {
+        type: 'worker',
+        cron: app.config.multipart.cleanSchedule.cron,
+        disable: app.config.multipart.cleanSchedule.disable,
+        immediate: false,
+      };
+    }
+
+    async _remove(dir: string) {
+      const { ctx } = this;
+      if (await fs.access(dir).then(() => true, () => false)) {
+        ctx.coreLogger.info('[@eggjs/multipart:CleanTmpdir] removing tmpdir: %j', dir);
+        try {
+          await fs.rm(dir, { force: true, recursive: true });
+          ctx.coreLogger.info('[@eggjs/multipart:CleanTmpdir:success] tmpdir: %j has been removed', dir);
+        } catch (err) {
+          /* c8 ignore next 3 */
+          ctx.coreLogger.error('[@eggjs/multipart:CleanTmpdir:error] remove tmpdir: %j error: %s', dir, err);
+          ctx.coreLogger.error(err);
+        }
+      }
+    }
+
+    async subscribe() {
+      const { ctx } = this;
+      const config = ctx.app.config;
+      ctx.coreLogger.info('[@eggjs/multipart:CleanTmpdir] start clean tmpdir: %j', config.multipart.tmpdir);
+      // last year
+      const lastYear = dayjs().subtract(1, 'years');
+      const lastYearDir = path.join(config.multipart.tmpdir, lastYear.format('YYYY'));
+      await this._remove(lastYearDir);
+      // 3 months
+      for (let i = 1; i <= 3; i++) {
+        const date = dayjs().subtract(i, 'months');
+        const dir = path.join(config.multipart.tmpdir, date.format('YYYY/MM'));
+        await this._remove(dir);
+      }
+      // 7 days
+      for (let i = 1; i <= 7; i++) {
+        const date = dayjs().subtract(i, 'days');
+        const dir = path.join(config.multipart.tmpdir, date.format('YYYY/MM/DD'));
+        await this._remove(dir);
+      }
+      ctx.coreLogger.info('[@eggjs/multipart:CleanTmpdir] end');
+    }
+  };
+};

package/src/app.ts

@@ -0,0 +1,20 @@
+import type { EggCore, ILifecycleBoot } from '@eggjs/core';
+import { normalizeOptions } from './lib/utils.js';
+
+export default class AppBootHook implements ILifecycleBoot {
+  constructor(private app: EggCore) {}
+
+  configWillLoad() {
+    this.app.config.multipart = normalizeOptions(this.app.config.multipart);
+    const options = this.app.config.multipart;
+
+    this.app.coreLogger.info('[@eggjs/multipart] %s mode enable', options.mode);
+    if (options.mode === 'file' || options.fileModeMatch) {
+      this.app.coreLogger.info('[@eggjs/multipart] will save temporary files to %j, cleanup job cron: %j',
+        options.tmpdir, options.cleanSchedule.cron);
+      // enable multipart middleware
+      this.app.config.coreMiddleware.push('multipart');
+    }
+  }
+}
+

package/src/config/config.default.ts

@@ -0,0 +1,130 @@
+import os from 'node:os';
+import path from 'node:path';
+import type { Context, EggAppInfo } from '@eggjs/core';
+import type { PathMatchingPattern } from 'egg-path-matching';
+
+export type MatchItem = string | RegExp | ((ctx: Context) => boolean);
+
+/**
+ * multipart parser options
+ * @member Config#multipart
+ */
+export interface MultipartConfig {
+  /**
+   * which mode to handle multipart request, default is `stream`, the hard way.
+   * If set mode to `file`, it's the easy way to handle multipart request and save it to local files.
+   * If you don't know the Node.js Stream work, maybe you should use the `file` mode to get started.
+   */
+  mode: 'stream' | 'file';
+  /**
+   * special url to use file mode when global `mode` is `stream`.
+   */
+  fileModeMatch?: PathMatchingPattern;
+  /**
+   * Auto set fields to parts, default is `false`.
+   * Only work on `stream` mode.
+   * If set true，all fields will be auto handle and can access by `parts.fields`
+   */
+  autoFields: boolean;
+  /**
+   * default charset encoding, don't change it before you real know about it
+   * Default is `utf8`
+   */
+  defaultCharset: string;
+  /**
+   * For multipart forms, the default character set to use for values of part header parameters (e.g. filename)
+   * that are not extended parameters (that contain an explicit charset), don't change it before you real know about it
+   * Default is `utf8`
+   */
+  defaultParamCharset: string;
+  /**
+   * Max field name size (in bytes), default is `100`
+   */
+  fieldNameSize: number;
+  /**
+   * Max field value size (in bytes), default is `100kb`
+   */
+  fieldSize: string | number;
+  /**
+   * Max number of non-file fields, default is `10`
+   */
+  fields: number;
+  /**
+   * Max file size (in bytes), default is `10mb`
+   */
+  fileSize: string | number;
+  /**
+   * Max number of file fields, default is `10`
+   */
+  files: number;
+  /**
+   * Add more ext file names to the `whitelist`, default is `[]`, only valid when `whitelist` is `null`
+   */
+  fileExtensions: string[];
+  /**
+   * The white ext file names, default is `null`
+   */
+  whitelist: string[] | ((filename: string) => boolean) | null;
+  /**
+   * Allow array field, default is `false`
+   */
+  allowArrayField: boolean;
+  /**
+   * The directory for temporary files. Only work on `file` mode.
+   * Default is `os.tmpdir()/egg-multipart-tmp/${appInfo.name}`
+   */
+  tmpdir: string;
+  /**
+   * The schedule for cleaning temporary files. Only work on `file` mode.
+   */
+  cleanSchedule: {
+    /**
+     * The cron expression for the schedule.
+     * Default is `0 30 4 * * *`
+     * @see https://github.com/eggjs/egg-schedule#cron-style-scheduling
+     */
+    cron: string;
+    /**
+     * Default is `false`
+     */
+    disable: boolean;
+  };
+  checkFile?(
+    fieldname: string,
+    file: any,
+    filename: string,
+    encoding: string,
+    mimetype: string
+  ): void | Error;
+}
+
+export default (appInfo: EggAppInfo) => {
+  return {
+    multipart: {
+      mode: 'stream',
+      autoFields: false,
+      defaultCharset: 'utf8',
+      defaultParamCharset: 'utf8',
+      fieldNameSize: 100,
+      fieldSize: '100kb',
+      fields: 10,
+      fileSize: '10mb',
+      files: 10,
+      fileExtensions: [],
+      whitelist: null,
+      allowArrayField: false,
+      tmpdir: path.join(os.tmpdir(), 'egg-multipart-tmp', appInfo.name),
+      cleanSchedule: {
+        cron: '0 30 4 * * *',
+        disable: false,
+      },
+    } as MultipartConfig,
+  };
+};
+
+declare module '@eggjs/core' {
+  // add EggAppConfig overrides types
+  interface EggAppConfig {
+    multipart: MultipartConfig;
+  }
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+import './config/config.default.js';
+import './app/extend/context.js';

package/src/lib/LimitError.ts

@@ -0,0 +1,12 @@
+export class LimitError extends Error {
+  code: string;
+  status: number;
+
+  constructor(code: string, message: string) {
+    super(message);
+    this.code = code;
+    this.status = 413;
+    this.name = this.constructor.name;
+    Error.captureStackTrace(this, this.constructor);
+  }
+}

package/src/lib/MultipartFileTooLargeError.ts

@@ -0,0 +1,14 @@
+export class MultipartFileTooLargeError extends Error {
+  status: number;
+  fields: Record<string, any>;
+  filename: string;
+
+  constructor(message: string, fields: Record<string, any>, filename: string) {
+    super(message);
+    this.name = this.constructor.name;
+    this.status = 413;
+    this.fields = fields;
+    this.filename = filename;
+    Error.captureStackTrace(this, this.constructor);
+  }
+}