msgpackr 1.7.0-alpha2 → 1.7.0-alpha5

package/SECURITY.md

@@ -1,11 +1,11 @@
-# Security Policy
-
-## Supported Versions
-
-| Version | Supported          |
-| ------- | ------------------ |
-| 1.4.x   | :white_check_mark: |
-
-## Reporting a Vulnerability
-
-Please report security vulnerabilities to kriszyp@gmail.com.
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 1.4.x   | :white_check_mark: |
+
+## Reporting a Vulnerability
+
+Please report security vulnerabilities to kriszyp@gmail.com.

package/benchmark.md

@@ -1,67 +1,67 @@
-Here are more comprehensive benchmarks. This is comparison with the next fastest JS projects using the benchmark tool from `msgpack-lite` (and data is from some clinical research data we use that has a good mix of different value types and structures). It also includes comparison to V8 native JSON functionality, and JavaScript Avro (`avsc`, a very optimized Avro implementation):
-
-operation                                                  |   op   |   ms  |  op/s
----------------------------------------------------------- | ------: | ----: | -----:
-buf = Buffer(JSON.stringify(obj));                         |   82000 |  5004 |  16386
-obj = JSON.parse(buf);                                     |   88600 |  5000 |  17720
-require("msgpackr").pack(obj);                             |  161500 |  5002 |  32287
-require("msgpackr").unpack(buf);                           |   94600 |  5004 |  18904
-msgpackr w/ shared structures: packr.pack(obj);            |  178400 |  5002 |  35665
-msgpackr w/ shared structures: packr.unpack(buf);          |  376700 |  5000 |  75340
-buf = require("msgpack-lite").encode(obj);                 |   30100 |  5012 |   6005
-obj = require("msgpack-lite").decode(buf);                 |   16200 |  5001 |   3239
-buf = require("notepack").encode(obj);                     |   62600 |  5005 |  12507
-obj = require("notepack").decode(buf);                     |   32400 |  5007 |   6470
-require("what-the-pack")... encoder.encode(obj);           |   63500 |  5002 |  12694
-require("what-the-pack")... encoder.decode(buf);           |   32000 |  5001 |   6398
-require("avsc")...make schema/type...type.toBuffer(obj);   |   84600 |  5003 |  16909
-require("avsc")...make schema/type...type.toBuffer(obj);   |   99300 |  5001 |  19856
-
-(`avsc` is schema-based and more comparable in style to msgpackr with shared structures).
-
-Here is a benchmark of streaming data (again borrowed from `msgpack-lite`'s benchmarking), where msgpackr is able to take advantage of the structured record extension and really pull away from other tools:
-
-operation (1000000 x 2)                          |   op    |  ms   |  op/s
------------------------------------------------- | ------: | ----: | -----:
-new PackrStream().write(obj);                    | 1000000 |   372 | 2688172
-new UnpackrStream().write(buf);                  | 1000000 |   247 | 4048582
-stream.write(msgpack.encode(obj));               | 1000000 |  2898 | 345065
-stream.write(msgpack.decode(buf));               | 1000000 |  1969 | 507872
-stream.write(notepack.encode(obj));              | 1000000 |   901 | 1109877
-stream.write(notepack.decode(buf));              | 1000000 |  1012 | 988142
-msgpack.Encoder().on("data",ondata).encode(obj); | 1000000 |  1763 | 567214
-msgpack.createDecodeStream().write(buf);         | 1000000 |  2222 | 450045
-msgpack.createEncodeStream().write(obj);         | 1000000 |  1577 | 634115
-msgpack.Decoder().on("data",ondata).decode(buf); | 1000000 |  2246 | 445235
-
-
-
-These are the benchmarks from notepack package. The larger test data for these benchmarks is very heavily weighted with large binary/buffer data and objects with extreme numbers of keys (much more than I typically see with real-world data, but YMMV):
-
-node ./benchmarks/encode
-
-library          |   tiny            |  small          | medium         | large
----------------- | ----------------: | --------------: | ---------------| -------:
-notepack         | 2,171,621 ops/sec | 546,905 ops/sec | 29,578 ops/sec | 265 ops/sec   
-msgpack-js       | 967,682 ops/sec   | 184,455 ops/sec | 20,556 ops/sec | 259 ops/sec   
-msgpackr         | 2,392,826 ops/sec | 556,915 ops/sec | 70,573 ops/sec | 313 ops/sec   
-msgpack-lite     | 553,143 ops/sec   | 132,318 ops/sec | 11,816 ops/sec | 186 ops/sec   
-@msgpack/msgpack | 2,157,655 ops/sec | 573,236 ops/sec | 25,864 ops/sec | 90.26 ops/sec 
-
-
-node ./benchmarks/decode
-
-library          |   tiny            |  small          | medium          | large
----------------- | ----------------: | --------------: | --------------- | -------:
-notepack         | 2,220,904 ops/sec | 560,630 ops/sec | 28,177 ops/sec  | 275 ops/sec   
-msgpack-js       | 965,719 ops/sec   | 222,047 ops/sec | 21,431 ops/sec  | 257 ops/sec   
-msgpackr         | 2,320,046 ops/sec | 589,167 ops/sec | 70,299 ops/sec  | 329 ops/sec   
-msgpackr records | 3,750,547 ops/sec | 912,419 ops/sec | 136,853 ops/sec | 733 ops/sec   
-msgpack-lite     | 569,222 ops/sec   | 129,008 ops/sec | 12,424 ops/sec  | 180 ops/sec   
-@msgpack/msgpack | 2,089,697 ops/sec | 557,507 ops/sec | 20,256 ops/sec  | 85.03 ops/sec 
-
-This was run by adding the msgpackr to the benchmarks for notepack.
-
-All benchmarks were performed on Node 14.8.0 (Windows i7-4770 3.4Ghz). They can be run with:
-npm install --no-save msgpack msgpack-js @msgpack/msgpack msgpack-lite notepack avsc
-node tests/benchmark
+Here are more comprehensive benchmarks. This is comparison with the next fastest JS projects using the benchmark tool from `msgpack-lite` (and data is from some clinical research data we use that has a good mix of different value types and structures). It also includes comparison to V8 native JSON functionality, and JavaScript Avro (`avsc`, a very optimized Avro implementation):
+
+operation                                                  |   op   |   ms  |  op/s
+---------------------------------------------------------- | ------: | ----: | -----:
+buf = Buffer(JSON.stringify(obj));                         |   82000 |  5004 |  16386
+obj = JSON.parse(buf);                                     |   88600 |  5000 |  17720
+require("msgpackr").pack(obj);                             |  161500 |  5002 |  32287
+require("msgpackr").unpack(buf);                           |   94600 |  5004 |  18904
+msgpackr w/ shared structures: packr.pack(obj);            |  178400 |  5002 |  35665
+msgpackr w/ shared structures: packr.unpack(buf);          |  376700 |  5000 |  75340
+buf = require("msgpack-lite").encode(obj);                 |   30100 |  5012 |   6005
+obj = require("msgpack-lite").decode(buf);                 |   16200 |  5001 |   3239
+buf = require("notepack").encode(obj);                     |   62600 |  5005 |  12507
+obj = require("notepack").decode(buf);                     |   32400 |  5007 |   6470
+require("what-the-pack")... encoder.encode(obj);           |   63500 |  5002 |  12694
+require("what-the-pack")... encoder.decode(buf);           |   32000 |  5001 |   6398
+require("avsc")...make schema/type...type.toBuffer(obj);   |   84600 |  5003 |  16909
+require("avsc")...make schema/type...type.toBuffer(obj);   |   99300 |  5001 |  19856
+
+(`avsc` is schema-based and more comparable in style to msgpackr with shared structures).
+
+Here is a benchmark of streaming data (again borrowed from `msgpack-lite`'s benchmarking), where msgpackr is able to take advantage of the structured record extension and really pull away from other tools:
+
+operation (1000000 x 2)                          |   op    |  ms   |  op/s
+------------------------------------------------ | ------: | ----: | -----:
+new PackrStream().write(obj);                    | 1000000 |   372 | 2688172
+new UnpackrStream().write(buf);                  | 1000000 |   247 | 4048582
+stream.write(msgpack.encode(obj));               | 1000000 |  2898 | 345065
+stream.write(msgpack.decode(buf));               | 1000000 |  1969 | 507872
+stream.write(notepack.encode(obj));              | 1000000 |   901 | 1109877
+stream.write(notepack.decode(buf));              | 1000000 |  1012 | 988142
+msgpack.Encoder().on("data",ondata).encode(obj); | 1000000 |  1763 | 567214
+msgpack.createDecodeStream().write(buf);         | 1000000 |  2222 | 450045
+msgpack.createEncodeStream().write(obj);         | 1000000 |  1577 | 634115
+msgpack.Decoder().on("data",ondata).decode(buf); | 1000000 |  2246 | 445235
+
+
+
+These are the benchmarks from notepack package. The larger test data for these benchmarks is very heavily weighted with large binary/buffer data and objects with extreme numbers of keys (much more than I typically see with real-world data, but YMMV):
+
+node ./benchmarks/encode
+
+library          |   tiny            |  small          | medium         | large
+---------------- | ----------------: | --------------: | ---------------| -------:
+notepack         | 2,171,621 ops/sec | 546,905 ops/sec | 29,578 ops/sec | 265 ops/sec   
+msgpack-js       | 967,682 ops/sec   | 184,455 ops/sec | 20,556 ops/sec | 259 ops/sec   
+msgpackr         | 2,392,826 ops/sec | 556,915 ops/sec | 70,573 ops/sec | 313 ops/sec   
+msgpack-lite     | 553,143 ops/sec   | 132,318 ops/sec | 11,816 ops/sec | 186 ops/sec   
+@msgpack/msgpack | 2,157,655 ops/sec | 573,236 ops/sec | 25,864 ops/sec | 90.26 ops/sec 
+
+
+node ./benchmarks/decode
+
+library          |   tiny            |  small          | medium          | large
+---------------- | ----------------: | --------------: | --------------- | -------:
+notepack         | 2,220,904 ops/sec | 560,630 ops/sec | 28,177 ops/sec  | 275 ops/sec   
+msgpack-js       | 965,719 ops/sec   | 222,047 ops/sec | 21,431 ops/sec  | 257 ops/sec   
+msgpackr         | 2,320,046 ops/sec | 589,167 ops/sec | 70,299 ops/sec  | 329 ops/sec   
+msgpackr records | 3,750,547 ops/sec | 912,419 ops/sec | 136,853 ops/sec | 733 ops/sec   
+msgpack-lite     | 569,222 ops/sec   | 129,008 ops/sec | 12,424 ops/sec  | 180 ops/sec   
+@msgpack/msgpack | 2,089,697 ops/sec | 557,507 ops/sec | 20,256 ops/sec  | 85.03 ops/sec 
+
+This was run by adding the msgpackr to the benchmarks for notepack.
+
+All benchmarks were performed on Node 14.8.0 (Windows i7-4770 3.4Ghz). They can be run with:
+npm install --no-save msgpack msgpack-js @msgpack/msgpack msgpack-lite notepack avsc
+node tests/benchmark

package/dist/index.js

@@ -59,16 +59,21 @@
 			}
 			Object.assign(this, options);
 		}
-		unpack(source, end) {
+		unpack(source, options) {
 			if (src) {
 				// re-entrant execution, save the state and restore it after we do this unpack
 				return saveState(() => {
 					clearSource();
-					return this ? this.unpack(source, end) : Unpackr.prototype.unpack.call(defaultOptions, source, end)
+					return this ? this.unpack(source, options) : Unpackr.prototype.unpack.call(defaultOptions, source, options)
 				})
 			}
-			srcEnd = end > -1 ? end : source.length;
-			position = 0;
+			if (typeof options === 'object') {
+				srcEnd = options.end || source.length;
+				position = options.start || 0;
+			} else {
+				position = 0;
+				srcEnd = options > -1 ? options : source.length;
+			}
 			srcStringEnd = 0;
 			srcString = null;
 			bundledStrings = null;
@@ -89,7 +94,7 @@
 				currentUnpackr = this;
 				if (this.structures) {
 					currentStructures = this.structures;
-					return checkedRead()
+					return checkedRead(options)
 				} else if (!currentStructures || currentStructures.length > 0) {
 					currentStructures = [];
 				}
@@ -98,7 +103,7 @@
 				if (!currentStructures || currentStructures.length > 0)
 					currentStructures = [];
 			}
-			return checkedRead()
+			return checkedRead(options)
 		}
 		unpackMultiple(source, forEach) {
 			let values, lastPosition = 0;
@@ -134,6 +139,8 @@
 		}
 		_mergeStructures(loadedStructures, existingStructures) {
 			loadedStructures = loadedStructures || [];
+			if (Object.isFrozen(loadedStructures))
+				loadedStructures = loadedStructures.map(structure => structure.slice(0));
 			for (let i = 0, l = loadedStructures.length; i < l; i++) {
 				let structure = loadedStructures[i];
 				if (structure) {
@@ -160,7 +167,7 @@
 			return this.unpack(source, end)
 		}
 	}
-	function checkedRead() {
+	function checkedRead(options) {
 		try {
 			if (!currentUnpackr.trusted && !sequentialMode) {
 				let sharedLength = currentStructures.sharedLength || 0;
@@ -168,9 +175,10 @@
 					currentStructures.length = sharedLength;
 			}
 			let result;
-			if (currentUnpackr.randomAccessStructure && src[position] < 0x40 && readStruct) {
-				let id = (src[position++] << 8) + src[position++];
-				result = readStruct(src, position, srcEnd, currentStructures[id - 0x40] || loadStructures()[id - 0x40], currentUnpackr);
+			if (currentUnpackr.randomAccessStructure && src[position] < 0x40 && src[position] >= 0x20 && readStruct) {
+				result = readStruct(src, position, srcEnd, currentUnpackr);
+				if (!(options && options.lazy) && result)
+					result = result.toJSON();
 				position = srcEnd;
 			} else
 				result = read();
@@ -787,7 +795,15 @@
 	function readExt(length) {
 		let type = src[position++];
 		if (currentExtensions[type]) {
-			return currentExtensions[type](src.subarray(position, position += length))
+			let end;
+			return currentExtensions[type](src.subarray(position, end = (position += length)), (readPosition) => {
+				position = readPosition;
+				try {
+					return read();
+				} finally {
+					position = end;
+				}
+			})
 		}
 		else
 			throw new Error('Unknown extension type ' + type)
@@ -1054,7 +1070,6 @@
 			let hasSharedUpdate;
 			let structures;
 			let referenceMap;
-			let lastSharedStructuresLength = 0;
 			let encodeUtf8 = ByteArray.prototype.utf8Write ? function(string, position) {
 				return target.utf8Write(string, position, 0xffffffff)
 			} : (textEncoder && textEncoder.encodeInto) ?
@@ -1140,7 +1155,7 @@
 							}
 							transition[RECORD_SYMBOL] = i + 0x40;
 						}
-						lastSharedStructuresLength = sharedLength;
+						this.lastNamedStructuresLength = sharedLength;
 					}
 					if (!isSequential) {
 						structures.nextId = sharedLength + 0x40;
@@ -1176,7 +1191,7 @@
 					if (structures) {
 						if (serializationsSinceTransitionRebuild < 10)
 							serializationsSinceTransitionRebuild++;
-						let sharedLength = structures.sharedLength || maxSharedStructures;
+						let sharedLength = structures.sharedLength || 0;
 						if (structures.length > sharedLength)
 							structures.length = sharedLength;
 						if (transitionsCount > 10000) {
@@ -1195,12 +1210,12 @@
 						if (hasSharedUpdate && packr.saveStructures) {
 							// we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
 							let returnBuffer = target.subarray(start, position$1);
-							if (packr.saveStructures(structures, lastSharedStructuresLength) === false) {
+							let newSharedData = prepareStructures(structures, packr);
+							if (packr.saveStructures(newSharedData, newSharedData.isCompatible) === false) {
 								// get updated structures and try again if the update failed
-								packr._mergeStructures(packr.getStructures());
 								return packr.pack(value)
 							}
-							lastSharedStructuresLength = sharedLength;
+							packr.lastNamedStructuresLength = sharedLength;
 							return returnBuffer
 						}
 					}
@@ -1318,7 +1333,7 @@
 				} else if (type === 'number') {
 					if (value >>> 0 === value) {// positive integer, 32-bit or less
 						// positive uint
-						if (value < 0x40 || (value < 0x80 && this.useRecords === false)) {
+						if (value < 0x20 || (value < 0x80 && this.useRecords === false) || (value < 0x40 && !this.randomAccessStructure)) {
 							target[position$1++] = value;
 						} else if (value < 0x100) {
 							target[position$1++] = 0xcc;
@@ -1733,6 +1748,8 @@
 		clearSharedData() {
 			if (this.structures)
 				this.structures = [];
+			if (this.typedStructs)
+				this.typedStructs = [];
 		}
 	}
 
@@ -1952,6 +1969,15 @@
 		}
 		addExtension(extension);
 	}
+	function prepareStructures(structures, packr) {
+		structures.isCompatible = (existingStructures) => {
+			let compatible = !existingStructures || ((packr.lastNamedStructuresLength || 0) === existingStructures.length);
+			if (!compatible) // we want to merge these existing structures immediately since we already have it and we are in the right transaction
+				packr._mergeStructures(existingStructures);
+			return compatible;
+		};
+		return structures
+	}
 
 	let defaultPackr = new Packr({ useRecords: false });
 	const pack = defaultPackr.pack;
@@ -1961,92 +1987,92 @@
 	const REUSE_BUFFER_MODE = 512;
 	const RESET_BUFFER_MODE = 1024;
 
-	/**
-	 * Given an Iterable first argument, returns an Iterable where each value is packed as a Buffer
-	 * If the argument is only Async Iterable, the return value will be an Async Iterable.
-	 * @param {Iterable|Iterator|AsyncIterable|AsyncIterator} objectIterator - iterable source, like a Readable object stream, an array, Set, or custom object
-	 * @param {options} [options] - msgpackr pack options
-	 * @returns {IterableIterator|Promise.<AsyncIterableIterator>}
-	 */
-	function packIter (objectIterator, options = {}) {
-	  if (!objectIterator || typeof objectIterator !== 'object') {
-	    throw new Error('first argument must be an Iterable, Async Iterable, or a Promise for an Async Iterable')
-	  } else if (typeof objectIterator[Symbol.iterator] === 'function') {
-	    return packIterSync(objectIterator, options)
-	  } else if (typeof objectIterator.then === 'function' || typeof objectIterator[Symbol.asyncIterator] === 'function') {
-	    return packIterAsync(objectIterator, options)
-	  } else {
-	    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a Promise')
-	  }
-	}
-
-	function * packIterSync (objectIterator, options) {
-	  const packr = new Packr(options);
-	  for (const value of objectIterator) {
-	    yield packr.pack(value);
-	  }
-	}
-
-	async function * packIterAsync (objectIterator, options) {
-	  const packr = new Packr(options);
-	  for await (const value of objectIterator) {
-	    yield packr.pack(value);
-	  }
-	}
-
-	/**
-	 * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
-	 * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
-	 * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
-	 * @param {object} [options] - unpackr options
-	 * @returns {IterableIterator|Promise.<AsyncIterableIterator}
-	 */
-	function unpackIter (bufferIterator, options = {}) {
-	  if (!bufferIterator || typeof bufferIterator !== 'object') {
-	    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a promise')
-	  }
-
-	  const unpackr = new Unpackr(options);
-	  let incomplete;
-	  const parser = (chunk) => {
-	    let yields;
-	    // if there's incomplete data from previous chunk, concatinate and try again
-	    if (incomplete) {
-	      chunk = Buffer.concat([incomplete, chunk]);
-	      incomplete = undefined;
-	    }
-
-	    try {
-	      yields = unpackr.unpackMultiple(chunk);
-	    } catch (err) {
-	      if (err.incomplete) {
-	        incomplete = chunk.slice(err.lastPosition);
-	        yields = err.values;
-	      } else {
-	        throw err
-	      }
-	    }
-	    return yields
-	  };
-
-	  if (typeof bufferIterator[Symbol.iterator] === 'function') {
-	    return (function * iter () {
-	      for (const value of bufferIterator) {
-	        yield * parser(value);
-	      }
-	    })()
-	  } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
-	    return (async function * iter () {
-	      for await (const value of bufferIterator) {
-	        yield * parser(value);
-	      }
-	    })()
-	  }
-	}
-	const decodeIter = unpackIter;
+	/**
+	 * Given an Iterable first argument, returns an Iterable where each value is packed as a Buffer
+	 * If the argument is only Async Iterable, the return value will be an Async Iterable.
+	 * @param {Iterable|Iterator|AsyncIterable|AsyncIterator} objectIterator - iterable source, like a Readable object stream, an array, Set, or custom object
+	 * @param {options} [options] - msgpackr pack options
+	 * @returns {IterableIterator|Promise.<AsyncIterableIterator>}
+	 */
+	function packIter (objectIterator, options = {}) {
+	  if (!objectIterator || typeof objectIterator !== 'object') {
+	    throw new Error('first argument must be an Iterable, Async Iterable, or a Promise for an Async Iterable')
+	  } else if (typeof objectIterator[Symbol.iterator] === 'function') {
+	    return packIterSync(objectIterator, options)
+	  } else if (typeof objectIterator.then === 'function' || typeof objectIterator[Symbol.asyncIterator] === 'function') {
+	    return packIterAsync(objectIterator, options)
+	  } else {
+	    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a Promise')
+	  }
+	}
+
+	function * packIterSync (objectIterator, options) {
+	  const packr = new Packr(options);
+	  for (const value of objectIterator) {
+	    yield packr.pack(value);
+	  }
+	}
+
+	async function * packIterAsync (objectIterator, options) {
+	  const packr = new Packr(options);
+	  for await (const value of objectIterator) {
+	    yield packr.pack(value);
+	  }
+	}
+
+	/**
+	 * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
+	 * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
+	 * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
+	 * @param {object} [options] - unpackr options
+	 * @returns {IterableIterator|Promise.<AsyncIterableIterator}
+	 */
+	function unpackIter (bufferIterator, options = {}) {
+	  if (!bufferIterator || typeof bufferIterator !== 'object') {
+	    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a promise')
+	  }
+
+	  const unpackr = new Unpackr(options);
+	  let incomplete;
+	  const parser = (chunk) => {
+	    let yields;
+	    // if there's incomplete data from previous chunk, concatinate and try again
+	    if (incomplete) {
+	      chunk = Buffer.concat([incomplete, chunk]);
+	      incomplete = undefined;
+	    }
+
+	    try {
+	      yields = unpackr.unpackMultiple(chunk);
+	    } catch (err) {
+	      if (err.incomplete) {
+	        incomplete = chunk.slice(err.lastPosition);
+	        yields = err.values;
+	      } else {
+	        throw err
+	      }
+	    }
+	    return yields
+	  };
+
+	  if (typeof bufferIterator[Symbol.iterator] === 'function') {
+	    return (function * iter () {
+	      for (const value of bufferIterator) {
+	        yield * parser(value);
+	      }
+	    })()
+	  } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
+	    return (async function * iter () {
+	      for await (const value of bufferIterator) {
+	        yield * parser(value);
+	      }
+	    })()
+	  }
+	}
+	const decodeIter = unpackIter;
 	const encodeIter = packIter;
 
-	const useRecords = false;
+	const useRecords = false;
 	const mapsAsObjects = true;
 
 	exports.ALWAYS = ALWAYS;