formstream 1.4.0 → 1.5.0

package/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2012 - 2014 fengmk2 <fengmk2@gmail.com>
+Copyright (c) 2014-present node-modules and other contributors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,10 +1,14 @@
 # formstream
 
 [![NPM version][npm-image]][npm-url]
+[![CI](https://github.com/node-modules/formstream/actions/workflows/ci.yml/badge.svg)](https://github.com/node-modules/formstream/actions/workflows/ci.yml)
+[![Test coverage][codecov-image]][codecov-url]
 [![npm download][download-image]][download-url]
 
 [npm-image]: https://img.shields.io/npm/v/formstream.svg?style=flat-square
 [npm-url]: https://npmjs.org/package/formstream
+[codecov-image]: https://codecov.io/github/node-modules/formstream/coverage.svg?branch=master
+[codecov-url]: https://codecov.io/github/node-modules/formstream?branch=master
 [download-image]: https://img.shields.io/npm/dm/formstream.svg?style=flat-square
 [download-url]: https://npmjs.org/package/formstream
 
@@ -13,7 +17,7 @@ A [multipart/form-data](http://tools.ietf.org/html/rfc2388) encoded stream, help
 ## Install
 
 ```bash
-$ npm install formstream
+npm install formstream
 ```
 
 ## Quick Start
@@ -58,19 +62,44 @@ var formstream = require('formstream');
 
 var filepath = './logo.png';
 fs.stat(filepath, function (err, stat) {
-  formstream().field('status', 'share picture')
-      .field('access_token', 'your access token')
-      .file('pic', filepath, 'logo.png', stat.size)
-      .pipe(process.stdout); // your request stream
+  formstream()
+    .field('status', 'share picture')
+    .field('access_token', 'your access token')
+    .file('pic', filepath, 'logo.png', stat.size)
+    .pipe(process.stdout); // your request stream
+});
+```
+
+### Set min chunk buffer size
+
+Some web servers have a limit on the number of chunks, and you can set `minChunkSize` to ensure the size of chunk sent to the server.
+
+```js
+var fs = require('fs');
+var FormStream = require('formstream');
+
+var filepath = './big-file.zip';
+fs.stat(filepath, function (err, stat) {
+  new FormStream({
+    // send >= 2MB chunk buffer size to the server
+    minChunkSize: 1024 * 1024 * 2,
+  }).field('status', 'share file')
+    .field('access_token', 'your access token')
+    .file('file', filepath, 'big-file.zip', stat.size)
+    .pipe(process.stdout); // your request stream
 });
 ```
 
 ## API Doc
 
-### formstream()
+### formstream([options])
 
 Create a form instance.
 
+#### Arguments
+
+- **options.minChunkSize** Number - min chunk size to emit data event
+
 #### Returns
 
 Form - form instance
@@ -140,7 +169,7 @@ Get headers for the request.
 
 #### Arguments
 
-- ***headers*** Object - Additional headers
+- **headers** Object - Additional headers
 
 #### Example
 
@@ -173,29 +202,8 @@ See [Node.js Documentation](http://nodejs.org/api/stream.html#stream_event_end)
 
 ## License
 
-(The MIT License)
-
-Copyright (c) 2012 - 2014 fengmk2 <fengmk2@gmail.com>
-Copyright(c) node-modules and other contributors.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+[MIT](LICENSE)
+
 <!-- GITCONTRIBUTOR_START -->
 
 ## Contributors
@@ -206,4 +214,4 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Wed May 15 2024 00:34:12 GMT+0800`.
 
-<!-- GITCONTRIBUTOR_END -->
+<!-- GITCONTRIBUTOR_END -->

package/lib/formstream.js

@@ -2,6 +2,7 @@
  * Form Data format:
  *
 
+```txt
 --FormStreamBoundary1349886663601\r\n
 Content-Disposition: form-data; name="foo"\r\n
 \r\n
@@ -15,19 +16,24 @@ Content-Type: application/json\r\n
 Content-Disposition: form-data; name="file"; filename="formstream.test.js"\r\n
 Content-Type: application/javascript\r\n
 \r\n
-<FILE-CONTENT>\r\n
+<FILE-CONTENT-CHUNK-1>
+...
+<FILE-CONTENT-CHUNK-N>
+\r\n
 --FormStreamBoundary1349886663601\r\n
 Content-Disposition: form-data; name="pic"; filename="fawave.png"\r\n
 Content-Type: image/png\r\n
 \r\n
 <IMAGE-CONTENT>\r\n
 --FormStreamBoundary1349886663601--
+```
 
  *
  */
 
 'use strict';
 
+var debug = require('util').debuglog('formstream');
 var Stream = require('stream');
 var parseStream = require('pause-stream');
 var util = require('util');
@@ -35,14 +41,15 @@ var mime = require('mime');
 var path = require('path');
 var fs = require('fs');
 var destroy = require('destroy');
+var hex = require('node-hex');
 
 var PADDING = '--';
 var NEW_LINE = '\r\n';
 var NEW_LINE_BUFFER =  Buffer.from(NEW_LINE);
 
-function FormStream() {
+function FormStream(options) {
   if (!(this instanceof FormStream)) {
-    return new FormStream();
+    return new FormStream(options);
   }
 
   FormStream.super_.call(this);
@@ -54,8 +61,10 @@ function FormStream() {
   this._contentLength = 0;
   this._isAllStreamSizeKnown = true;
   this._knownStreamSize = 0;
+  this._minChunkSize = options && options.minChunkSize || 0;
 
   this.isFormStream = true;
+  debug('start boundary\n%s', this._boundary);
 }
 
 util.inherits(FormStream, Stream);
@@ -83,12 +92,12 @@ FormStream.prototype.setTotalStreamSize = function (size) {
 
   for (var i = 0; i < this._streams.length; i++) {
     size += this._streams[i][0].length;
-    size += NEW_LINE_BUFFER.length; // stream field end pedding size
+    size += NEW_LINE_BUFFER.length; // stream field end padding size
   }
 
   this._knownStreamSize = size;
   this._isAllStreamSizeKnown = true;
-
+  debug('set total size: %s', size);
   return this;
 };
 
@@ -99,7 +108,6 @@ FormStream.prototype.headers = function (options) {
 
   // calculate total stream size
   this._contentLength += this._knownStreamSize;
-
   // calculate length of end padding
   this._contentLength += this._endData.length;
 
@@ -113,19 +121,20 @@ FormStream.prototype.headers = function (options) {
     }
   }
 
+  debug('headers: %j', headers);
   return headers;
 };
 
 FormStream.prototype.file = function (name, filepath, filename, filesize) {
-  var mimeType = mime.getType(filepath);
-
   if (typeof filename === 'number' && !filesize) {
     filesize = filename;
     filename = path.basename(filepath);
-  } else if (!filename) {
+  }
+  if (!filename) {
     filename = path.basename(filepath);
   }
 
+  var mimeType = mime.getType(filename);
   var stream = fs.createReadStream(filepath);
 
   return this.stream(name, stream, filename, mimeType, filesize);
@@ -198,38 +207,37 @@ FormStream.prototype.buffer = function (name, buffer, filename, mimeType) {
 
   var leading = this._leading(disposition, mimeType);
 
-  this._buffers.push([leading, buffer]);
-
   // plus buffer length to total content-length
-  this._contentLength += leading.length;
-  this._contentLength += buffer.length;
-  this._contentLength += NEW_LINE_BUFFER.length;
+  var bufferSize = leading.length + buffer.length + NEW_LINE_BUFFER.length;
+  this._buffers.push(Buffer.concat([leading, buffer, NEW_LINE_BUFFER], bufferSize));  
+  this._contentLength += bufferSize;
 
   process.nextTick(this.resume.bind(this));
-
+  if (debug.enabled) {
+    debug('new buffer field, content size: %d\n%s%s',
+      buffer.length, leading.toString(), hex(buffer));
+  }
   return this;
 };
 
 FormStream.prototype._leading = function (disposition, type) {
   var leading = [PADDING + this._boundary];
 
-  var disps = [];
+  var dispositions = [];
 
   if (disposition) {
     for (var k in disposition) {
-      disps.push(k + '="' + disposition[k] + '"');
+      dispositions.push(k + '="' + disposition[k] + '"');
     }
   }
 
-  leading.push('Content-Disposition: form-data; ' + disps.join('; '));
-
+  leading.push('Content-Disposition: form-data; ' + dispositions.join('; '));
   if (type) {
     leading.push('Content-Type: ' + type);
   }
 
   leading.push('');
   leading.push('');
-
   return Buffer.from(leading.join(NEW_LINE));
 };
 
@@ -239,26 +247,101 @@ FormStream.prototype._emitBuffers = function () {
   }
 
   for (var i = 0; i < this._buffers.length; i++) {
-    var item = this._buffers[i];
-    this.emit('data', item[0]); // part leading
-    this.emit('data', item[1]); // part content
-    this.emit('data', NEW_LINE_BUFFER);
+    this.emit('data', this._buffers[i]);
   }
-
   this._buffers = [];
 };
 
 FormStream.prototype._emitStream = function (item) {
   var self = this;
-  // item: [ fieldData, stream ]
-  self.emit('data', item[0]);
+  // item: [ leading, stream ]
+  var streamSize = 0;
+  var chunkCount = 0;
+  const leading = item[0];
+  self.emit('data', leading);
+  chunkCount++;
+  if (debug.enabled) {
+    debug('new stream, chunk index %d\n%s', chunkCount, leading.toString());
+  }
 
   var stream = item[1];
   stream.on('data', function (data) {
     self.emit('data', data);
+    streamSize += leading.length;
+    chunkCount++;
+    if (debug.enabled) {
+      if (data.length > 512) {
+        debug('stream chunk, size %d, chunk index %d, stream size %d\n%s......   only show 512 bytes   ......',
+          data.length, chunkCount, streamSize, hex(data.slice(0, 512)));
+      } else {
+        debug('stream chunk, size %d, chunk index %d, stream size %d\n%s',
+          data.length, chunkCount, streamSize, hex(data));
+      }
+    }
   });
   stream.on('end', function () {
     self.emit('data', NEW_LINE_BUFFER);
+    chunkCount++;
+    debug('stream end, chunk index %d, stream size %d', chunkCount, streamSize);
+    return process.nextTick(self.drain.bind(self));
+  });
+  stream.resume();
+};
+
+FormStream.prototype._emitStreamWithChunkSize = function (item, minChunkSize) {
+  var self = this;
+  // item: [ leading, stream ]
+  var streamSize = 0;
+  var chunkCount = 0;
+  var bufferSize = 0;
+  var buffers = [];
+  const leading = item[0];
+  buffers.push(leading);
+  bufferSize += leading.length;
+  if (debug.enabled) {
+    debug('new stream, with min chunk size: %d\n%s', minChunkSize, leading.toString());
+  }
+
+  var stream = item[1];
+  stream.on('data', function (data) {
+    if (typeof data === 'string') {
+      data = Buffer.from(data, 'utf-8');
+    }
+    buffers.push(data);
+    bufferSize += data.length;
+    streamSize += data.length;
+    debug('got stream data size %d, buffer size %d, stream size %d',
+      data.length, bufferSize, streamSize);
+    if (bufferSize >= minChunkSize) {
+      const chunk = Buffer.concat(buffers, bufferSize);
+      buffers = [];
+      bufferSize = 0;
+      self.emit('data', chunk);
+      chunkCount++;
+      if (debug.enabled) {
+        if (chunk.length > 512) {
+          debug('stream chunk, size %d, chunk index %d, stream size %d\n%s......   only show 512 bytes   ......',
+            chunk.length, chunkCount, streamSize, hex(chunk.slice(0, 512)));
+        } else {
+          debug('stream chunk, size %d, chunk index %d, stream size %d\n%s',
+            chunk.length, chunkCount, streamSize, hex(chunk));
+        }
+      }
+    }
+  });
+  stream.on('end', function () {
+    buffers.push(NEW_LINE_BUFFER);
+    bufferSize += NEW_LINE_BUFFER.length;
+    const chunk = Buffer.concat(buffers, bufferSize);
+    self.emit('data', chunk);
+    chunkCount++;
+    if (chunk.length > 512) {
+      debug('stream end, size %d, chunk index %d, stream size %d\n%s......   only show 512 bytes   ......',
+        chunk.length, chunkCount, streamSize, hex(chunk.slice(0, 512)));
+    } else {
+      debug('stream end, size %d, chunk index %d, stream size %d\n%s',
+        chunk.length, chunkCount, streamSize, hex(chunk));
+    }
     return process.nextTick(self.drain.bind(self));
   });
   stream.resume();
@@ -270,14 +353,22 @@ FormStream.prototype._emitEnd = function () {
   // --{boundary}--\r\n
   this.emit('data', this._endData);
   this.emit('end');
+  if (debug.enabled) {
+    debug('end boundary\n%s', this._endData.toString());
+  }
 };
 
 FormStream.prototype.drain = function () {
+  // debug('drain');
   this._emitBuffers();
 
   var item = this._streams.shift();
   if (item) {
-    this._emitStream(item);
+    if (this._minChunkSize && this._minChunkSize > 0) {
+      this._emitStreamWithChunkSize(item, this._minChunkSize);
+    } else {
+      this._emitStream(item);
+    }
   } else {
     this._emitEnd();
   }
@@ -286,6 +377,7 @@ FormStream.prototype.drain = function () {
 };
 
 FormStream.prototype.resume = function () {
+  // debug('resume');
   this.paused = false;
 
   if (!this._draining) {
@@ -298,4 +390,5 @@ FormStream.prototype.resume = function () {
 
 FormStream.prototype.close = FormStream.prototype.destroy = function () {
   this.emit('destroy');
+  // debug('destroy or close');
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "formstream",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "A multipart/form-data encoded stream, helper for file upload.",
   "main": "lib/formstream.js",
   "types": "types/formstream.d.ts",
@@ -11,7 +11,7 @@
   "scripts": {
     "test": "egg-bin test",
     "cov": "egg-bin cov",
-    "ci": "npm run lint && npm run tsd && npm run cov",
+    "ci": "npm run lint && npm run tsd && npm run cov && NODE_DEBUG=formstream npm run cov",
     "lint": "jshint .",
     "tsd": "tsd",
     "contributor": "git-contributor"
@@ -32,6 +32,7 @@
   "dependencies": {
     "destroy": "^1.0.4",
     "mime": "^2.5.2",
+    "node-hex": "^1.0.1",
     "pause-stream": "~0.0.11"
   },
   "devDependencies": {

package/types/formstream.d.ts

@@ -64,9 +64,14 @@ declare class FormStream extends Stream {
   headers(additionalHeaders?: Record<string, any>): Record<string, any>
 }
 
+interface FormStreamOptions {
+  /** min chunk size to emit data event */
+  minChunkSize?: number;
+}
+
 declare const formStream: {
-  new (): FormStream
-  (): FormStream
+  new (options?: FormStreamOptions): FormStream
+  (options?: FormStreamOptions): FormStream
 }
 
 interface formStream extends FormStream {}