msgpackr 1.5.2 → 1.5.3

package/README.md

@@ -7,8 +7,9 @@
 [![module](https://img.shields.io/badge/module-ESM%2FCJS-blue)](README.md)
 [![license](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE)
 
+<img align="right" src="./assets/performance.png" width="380"/>
 
-The msgpackr package is an extremely fast MessagePack NodeJS/JavaScript implementation. Currently, it is significantly faster than any other known implementations, faster than Avro (for JS), and generally faster than native V8 JSON.stringify/parse, on NodeJS. It also includes an optional record extension (the `r` in msgpackr), for defining record structures that makes MessagePack even faster and more compact, often over twice as fast as even native JSON functions, several times faster than other JS implementations, and 15-50% more compact. See the performance section for more details. Structured cloning (with support for cyclical references) is also supported through optional extensions.<img align="right" src="./assets/performance.png" width="380"/>
+The msgpackr package is an extremely fast MessagePack NodeJS/JavaScript implementation. Currently, it is significantly faster than any other known implementations, faster than Avro (for JS), and generally faster than native V8 JSON.stringify/parse, on NodeJS. It also includes an optional record extension (the `r` in msgpackr), for defining record structures that makes MessagePack even faster and more compact, often over twice as fast as even native JSON functions, several times faster than other JS implementations, and 15-50% more compact. See the performance section for more details. Structured cloning (with support for cyclical references) is also supported through optional extensions.
 
 ## Basic Usage
 

package/dist/index.js

@@ -152,6 +152,9 @@
 					currentStructures.length = sharedLength;
 			}
 			let result = read();
+			if (bundledStrings) // bundled strings to skip past
+				position = bundledStrings.postBundlePosition;
+
 			if (position == srcEnd) {
 				// finished reading this source, cleanup references
 				if (currentStructures.restoreStructures)
@@ -715,6 +718,36 @@
 		}
 	}
 
+	function readOnlyJSString() {
+		let token = src[position++];
+		let length;
+		if (token < 0xc0) {
+			// fixstr
+			length = token - 0xa0;
+		} else {
+			switch(token) {
+				case 0xd9:
+				// str 8
+					length = src[position++];
+					break
+				case 0xda:
+				// str 16
+					length = dataView.getUint16(position);
+					position += 2;
+					break
+				case 0xdb:
+				// str 32
+					length = dataView.getUint32(position);
+					position += 4;
+					break
+				default:
+					throw new Error('Expected string')
+			}
+		}
+		return readStringJS(length)
+	}
+
+
 	function readBin(length) {
 		return currentUnpackr.copyBuffers ?
 			// specifically use the copying slice (not the node one)
@@ -866,21 +899,18 @@
 		let data = read();
 		return new RegExp(data[0], data[1])
 	};
-
+	const TEMP_BUNDLE = [];
 	currentExtensions[0x62] = (data) => {
 		let dataSize = (data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3];
 		let dataPosition = position;
-		position += dataSize - 4;
-		bundledStrings = [read(), read()];
+		position += dataSize - data.length;
+		bundledStrings = TEMP_BUNDLE;
+		bundledStrings = [readOnlyJSString(), readOnlyJSString()];
 		bundledStrings.position0 = 0;
 		bundledStrings.position1 = 0;
-		let postBundlePosition = position;
+		bundledStrings.postBundlePosition = position;
 		position = dataPosition;
-		try {
-			return read()
-		} finally {
-			position = postBundlePosition;
-		}
+		return read()
 	};
 
 	currentExtensions[0xff] = (data) => {
@@ -977,11 +1007,12 @@
 	const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array;
 	const ByteArray = hasNodeBuffer ? Buffer : Uint8Array;
 	const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000;
-	let target;
+	let target, keysTarget;
 	let targetView;
 	let position$1 = 0;
 	let safeEnd;
 	let bundledStrings$1 = null;
+	const MAX_BUNDLE_SIZE = 0xf000;
 	const hasNonLatin = /[\u0080-\uFFFF]/;
 	const RECORD_SYMBOL = Symbol('record-id');
 	class Packr extends Unpackr {
@@ -1044,12 +1075,9 @@
 					position$1 = (position$1 + 7) & 0x7ffffff8; // Word align to make any future copying of this buffer faster
 				start = position$1;
 				referenceMap = packr.structuredClone ? new Map() : null;
-				if (packr.bundleStrings) {
-					bundledStrings$1 = ['', ''];
-					target[position$1++] = 0xd6;
-					target[position$1++] = 0x62; // 'b'
-					bundledStrings$1.position = position$1 - start;
-					position$1 += 4;
+				if (packr.bundleStrings && typeof value !== 'string') {
+					bundledStrings$1 = [];
+					bundledStrings$1.size = Infinity; // force a new bundle start on first string
 				} else
 					bundledStrings$1 = null;
 				sharedStructures = packr.structures;
@@ -1087,15 +1115,11 @@
 				}
 				if (hasSharedUpdate)
 					hasSharedUpdate = false;
-				structures = sharedStructures || [];
+				structures = sharedStructures || (packr.structures = []);
 				try {
 					pack(value);
 					if (bundledStrings$1) {
-						targetView.setUint32(bundledStrings$1.position + start, position$1 - bundledStrings$1.position - start);
-						let writeStrings = bundledStrings$1;
-						bundledStrings$1 = null;
-						pack(writeStrings[0]);
-						pack(writeStrings[1]);
+						writeBundles(start, pack);
 					}
 					packr.offset = position$1; // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 					if (referenceMap && referenceMap.idsToInsert) {
@@ -1117,6 +1141,8 @@
 					if (sharedStructures) {
 						if (serializationsSinceTransitionRebuild < 10)
 							serializationsSinceTransitionRebuild++;
+						if (sharedStructures.length > maxSharedStructures)
+							sharedStructures.length = maxSharedStructures;
 						if (transitionsCount > 10000) {
 							// force a rebuild occasionally after a lot of transitions so it can get cleaned up
 							sharedStructures.transitions = null;
@@ -1158,7 +1184,30 @@
 				var length;
 				if (type === 'string') {
 					let strLength = value.length;
-					if (bundledStrings$1 && strLength >= 8 && strLength < 0x1000) {
+					if (bundledStrings$1 && strLength >= 4 && strLength < 0x1000) {
+						if ((bundledStrings$1.size += strLength) > MAX_BUNDLE_SIZE) {
+							let extStart;
+							let maxBytes = (bundledStrings$1[0] ? bundledStrings$1[0].length * 3 + bundledStrings$1[1].length : 0) + 10;
+							if (position$1 + maxBytes > safeEnd)
+								target = makeRoom(position$1 + maxBytes);
+							if (bundledStrings$1.position) { // here we use the 0x62 extension to write the last bundle and reserve sapce for the reference pointer to the next/current bundle
+								target[position$1] = 0xc8; // ext 16
+								position$1 += 3; // reserve for the writing bundle size
+								target[position$1++] = 0x62; // 'b'
+								extStart = position$1 - start;
+								position$1 += 4; // reserve for writing bundle reference
+								writeBundles(start, pack); // write the last bundles
+								targetView.setUint16(extStart + start - 3, position$1 - start - extStart);
+							} else { // here we use the 0x62 extension just to reserve the space for the reference pointer to the bundle (will be updated once the bundle is written)
+								target[position$1++] = 0xd6; // fixext 4
+								target[position$1++] = 0x62; // 'b'
+								extStart = position$1 - start;
+								position$1 += 4; // reserve for writing bundle reference
+							}
+							bundledStrings$1 = ['', '']; // create new ones
+							bundledStrings$1.size = 0;
+							bundledStrings$1.position = extStart;
+						}
 						let twoByte = hasNonLatin.test(value);
 						bundledStrings$1[twoByte ? 0 : 1] += value;
 						target[position$1++] = 0xc1;
@@ -1463,53 +1512,55 @@
 				target[objectOffset++ + start] = size >> 8;
 				target[objectOffset + start] = size & 0xff;
 			} :
-
-		/*	sharedStructures ?  // For highly stable structures, using for-in can a little bit faster
+			(options.progressiveRecords && !useTwoByteRecords) ?  // this is about 2% faster for highly stable structures, since it only requires one for-in loop (but much more expensive when new structure needs to be written)
 			(object, safePrototype) => {
-				let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null))
-				let objectOffset = position++ - start
-				let wroteKeys
+				let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null));
+				let objectOffset = position$1++ - start;
+				let wroteKeys;
 				for (let key in object) {
 					if (safePrototype || object.hasOwnProperty(key)) {
-						nextTransition = transition[key]
-						if (!nextTransition) {
-							nextTransition = transition[key] = Object.create(null)
-							nextTransition.__keys__ = (transition.__keys__ || []).concat([key])
-							/*let keys = Object.keys(object)
-							if 
-							let size = 0
-							let startBranch = transition.__keys__ ? transition.__keys__.length : 0
-							for (let i = 0, l = keys.length; i++) {
-								let key = keys[i]
-								size += key.length << 2
-								if (i >= startBranch) {
-									nextTransition = nextTransition[key] = Object.create(null)
-									nextTransition.__keys__ = keys.slice(0, i + 1)
+						nextTransition = transition[key];
+						if (nextTransition)
+							transition = nextTransition;
+						else {
+							// record doesn't exist, create full new record and insert it
+							let keys = Object.keys(object);
+							let lastTransition = transition;
+							transition = structures.transitions;
+							let newTransitions = 0;
+							for (let i = 0, l = keys.length; i < l; i++) {
+								let key = keys[i];
+								nextTransition = transition[key];
+								if (!nextTransition) {
+									nextTransition = transition[key] = Object.create(null);
+									newTransitions++;
 								}
+								transition = nextTransition;
 							}
-							makeRoom(position + size)
-							nextTransition = transition[key]
-							target.copy(target, )
-							objectOffset
+							if (objectOffset + start + 1 == position$1) {
+								// first key, so we don't need to insert, we can just write record directly
+								position$1--;
+								newRecord(transition, keys, newTransitions);
+							} else // otherwise we need to insert the record, moving existing data after the record
+								insertNewRecord(transition, keys, objectOffset, newTransitions);
+							wroteKeys = true;
+							transition = lastTransition[key];
 						}
-						transition = nextTransition
-						pack(object[key])
+						pack(object[key]);
 					}
 				}
-				let id = transition.id
-				if (!id) {
-					id = transition.id = structures.push(transition.__keys__) + 63
-					if (sharedStructures.onUpdate)
-						sharedStructures.onUpdate(id, transition.__keys__)
+				if (!wroteKeys) {
+					let recordId = transition[RECORD_SYMBOL];
+					if (recordId)
+						target[objectOffset + start] = recordId;
+					else
+						insertNewRecord(transition, Object.keys(object), objectOffset, 0);
 				}
-				target[objectOffset + start] = id
-			}*/
-			(object) => {
-				let keys = Object.keys(object);
+			} :
+			(object, safePrototype) => {
 				let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null));
 				let newTransitions = 0;
-				for (let i = 0, l = keys.length; i < l; i++) {
-					let key = keys[i];
+				for (let key in object) if (safePrototype || object.hasOwnProperty(key)) {
 					nextTransition = transition[key];
 					if (!nextTransition) {
 						nextTransition = transition[key] = Object.create(null);
@@ -1525,57 +1576,12 @@
 					} else
 						target[position$1++] = recordId;
 				} else {
-					recordId = structures.nextId;
-					if (!recordId)
-						recordId = 0x40;
-					if (recordId < sharedLimitId && this.shouldShareStructure && !this.shouldShareStructure(keys)) {
-						recordId = structures.nextOwnId;
-						if (!(recordId < maxStructureId))
-							recordId = sharedLimitId;
-						structures.nextOwnId = recordId + 1;
-					} else {
-						if (recordId >= maxStructureId)// cycle back around
-							recordId = sharedLimitId;
-						structures.nextId = recordId + 1;
-					}
-					let highByte = keys.highByte = recordId >= 0x60 && useTwoByteRecords ? (recordId - 0x60) >> 5 : -1;
-					transition[RECORD_SYMBOL] = recordId;
-					structures[recordId - 0x40] = keys;
-
-					if (recordId < sharedLimitId) {
-						keys.isShared = true;
-						structures.sharedLength = recordId - 0x3f;
-						hasSharedUpdate = true;
-						if (highByte >= 0) {
-							target[position$1++] = (recordId & 0x1f) + 0x60;
-							target[position$1++] = highByte;
-						} else {
-							target[position$1++] = recordId;
-						}
-					} else {
-						if (highByte >= 0) {
-							target[position$1++] = 0xd5; // fixext 2
-							target[position$1++] = 0x72; // "r" record defintion extension type
-							target[position$1++] = (recordId & 0x1f) + 0x60;
-							target[position$1++] = highByte;
-						} else {
-							target[position$1++] = 0xd4; // fixext 1
-							target[position$1++] = 0x72; // "r" record defintion extension type
-							target[position$1++] = recordId;
-						}
-
-						if (newTransitions)
-							transitionsCount += serializationsSinceTransitionRebuild * newTransitions;
-						// record the removal of the id, we can maintain our shared structure
-						if (recordIdsToRemove.length >= maxOwnStructures)
-							recordIdsToRemove.shift()[RECORD_SYMBOL] = 0; // we are cycling back through, and have to remove old ones
-						recordIdsToRemove.push(transition);
-						pack(keys);
-					}
+					newRecord(transition, transition.__keys__ || Object.keys(object), newTransitions);
 				}
 				// now write the values
-				for (let i = 0, l = keys.length; i < l; i++)
-					pack(object[keys[i]]);
+				for (let key in object)
+					if (safePrototype || object.hasOwnProperty(key))
+						pack(object[key]);
 			};
 			const makeRoom = (end) => {
 				let newSize;
@@ -1598,6 +1604,86 @@
 				safeEnd = newBuffer.length - 10;
 				return target = newBuffer
 			};
+			const newRecord = (transition, keys, newTransitions) => {
+				let recordId = structures.nextId;
+				if (!recordId)
+					recordId = 0x40;
+				if (recordId < sharedLimitId && this.shouldShareStructure && !this.shouldShareStructure(keys)) {
+					recordId = structures.nextOwnId;
+					if (!(recordId < maxStructureId))
+						recordId = sharedLimitId;
+					structures.nextOwnId = recordId + 1;
+				} else {
+					if (recordId >= maxStructureId)// cycle back around
+						recordId = sharedLimitId;
+					structures.nextId = recordId + 1;
+				}
+				let highByte = keys.highByte = recordId >= 0x60 && useTwoByteRecords ? (recordId - 0x60) >> 5 : -1;
+				transition[RECORD_SYMBOL] = recordId;
+				transition.__keys__ = keys;
+				structures[recordId - 0x40] = keys;
+
+				if (recordId < sharedLimitId) {
+					keys.isShared = true;
+					structures.sharedLength = recordId - 0x3f;
+					hasSharedUpdate = true;
+					if (highByte >= 0) {
+						target[position$1++] = (recordId & 0x1f) + 0x60;
+						target[position$1++] = highByte;
+					} else {
+						target[position$1++] = recordId;
+					}
+				} else {
+					if (highByte >= 0) {
+						target[position$1++] = 0xd5; // fixext 2
+						target[position$1++] = 0x72; // "r" record defintion extension type
+						target[position$1++] = (recordId & 0x1f) + 0x60;
+						target[position$1++] = highByte;
+					} else {
+						target[position$1++] = 0xd4; // fixext 1
+						target[position$1++] = 0x72; // "r" record defintion extension type
+						target[position$1++] = recordId;
+					}
+
+					if (newTransitions)
+						transitionsCount += serializationsSinceTransitionRebuild * newTransitions;
+					// record the removal of the id, we can maintain our shared structure
+					if (recordIdsToRemove.length >= maxOwnStructures)
+						recordIdsToRemove.shift()[RECORD_SYMBOL] = 0; // we are cycling back through, and have to remove old ones
+					recordIdsToRemove.push(transition);
+					pack(keys);
+				}
+			};
+			const insertNewRecord = (transition, keys, insertionOffset, newTransitions) => {
+				let mainTarget = target;
+				let mainPosition = position$1;
+				let mainSafeEnd = safeEnd;
+				let mainStart = start;
+				target = keysTarget;
+				position$1 = 0;
+				start = 0;
+				if (!target)
+					keysTarget = target = new ByteArrayAllocate(8192);
+				safeEnd = target.length - 10;
+				newRecord(transition, keys, newTransitions);
+				keysTarget = target;
+				let keysPosition = position$1;
+				target = mainTarget;
+				position$1 = mainPosition;
+				safeEnd = mainSafeEnd;
+				start = mainStart;
+				if (keysPosition > 1) {
+					let newEnd = position$1 + keysPosition - 1;
+					if (newEnd > safeEnd)
+						makeRoom(newEnd);
+					let insertionPosition = insertionOffset + start;
+					target.copyWithin(insertionPosition + keysPosition, insertionPosition + 1, position$1);
+					target.set(keysTarget.slice(0, keysPosition), insertionPosition);
+					position$1 = newEnd;
+				} else {
+					target[insertionOffset + start] = keysTarget[0];
+				}
+			};
 		}
 		useBuffer(buffer) {
 			// this means we are finished using our own buffer and we can write over it safely
@@ -1605,6 +1691,10 @@
 			targetView = new DataView(target.buffer, target.byteOffset, target.byteLength);
 			position$1 = 0;
 		}
+		clearSharedData() {
+			if (this.structures)
+				this.structures = [];
+		}
 	}
 
 	extensionClasses = [ Date, Set, Error, RegExp, ArrayBuffer, Object.getPrototypeOf(Uint8Array.prototype).constructor /*TypedArray*/, C1Type ];
@@ -1802,6 +1892,14 @@
 		return serialized
 	}
 
+	function writeBundles(start, pack) {
+		targetView.setUint32(bundledStrings$1.position + start, position$1 - bundledStrings$1.position - start);
+		let writeStrings = bundledStrings$1;
+		bundledStrings$1 = null;
+		pack(writeStrings[0]);
+		pack(writeStrings[1]);
+	}
+
 	function addExtension$1(extension) {
 		if (extension.Class) {
 			if (!extension.pack && !extension.write)