msgpackr 1.5.2 → 1.5.3

package/dist/node.cjs

@@ -158,6 +158,9 @@ function checkedRead() {
 				currentStructures.length = sharedLength;
 		}
 		let result = read();
+		if (bundledStrings) // bundled strings to skip past
+			position = bundledStrings.postBundlePosition;
+
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
 			if (currentStructures.restoreStructures)
@@ -500,6 +503,8 @@ function setExtractor(extractStrings) {
 		return function readString(length) {
 			let string = strings[stringPosition++];
 			if (string == null) {
+				if (bundledStrings)
+					return readStringJS(length)
 				let extraction = extractStrings(position - headerLength, srcEnd, src);
 				if (typeof extraction == 'string') {
 					string = extraction;
@@ -758,6 +763,36 @@ function shortStringInJS(length) {
 	}
 }
 
+function readOnlyJSString() {
+	let token = src[position++];
+	let length;
+	if (token < 0xc0) {
+		// fixstr
+		length = token - 0xa0;
+	} else {
+		switch(token) {
+			case 0xd9:
+			// str 8
+				length = src[position++];
+				break
+			case 0xda:
+			// str 16
+				length = dataView.getUint16(position);
+				position += 2;
+				break
+			case 0xdb:
+			// str 32
+				length = dataView.getUint32(position);
+				position += 4;
+				break
+			default:
+				throw new Error('Expected string')
+		}
+	}
+	return readStringJS(length)
+}
+
+
 function readBin(length) {
 	return currentUnpackr.copyBuffers ?
 		// specifically use the copying slice (not the node one)
@@ -909,21 +944,18 @@ currentExtensions[0x78] = () => {
 	let data = read();
 	return new RegExp(data[0], data[1])
 };
-
+const TEMP_BUNDLE = [];
 currentExtensions[0x62] = (data) => {
 	let dataSize = (data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3];
 	let dataPosition = position;
-	position += dataSize - 4;
-	bundledStrings = [read(), read()];
+	position += dataSize - data.length;
+	bundledStrings = TEMP_BUNDLE;
+	bundledStrings = [readOnlyJSString(), readOnlyJSString()];
 	bundledStrings.position0 = 0;
 	bundledStrings.position1 = 0;
-	let postBundlePosition = position;
+	bundledStrings.postBundlePosition = position;
 	position = dataPosition;
-	try {
-		return read()
-	} finally {
-		position = postBundlePosition;
-	}
+	return read()
 };
 
 currentExtensions[0xff] = (data) => {
@@ -1024,11 +1056,12 @@ const hasNodeBuffer = typeof Buffer !== 'undefined';
 const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array;
 const ByteArray = hasNodeBuffer ? Buffer : Uint8Array;
 const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000;
-let target;
+let target, keysTarget;
 let targetView;
 let position$1 = 0;
 let safeEnd;
 let bundledStrings$1 = null;
+const MAX_BUNDLE_SIZE = 0xf000;
 const hasNonLatin = /[\u0080-\uFFFF]/;
 const RECORD_SYMBOL = Symbol('record-id');
 class Packr extends Unpackr {
@@ -1091,12 +1124,9 @@ class Packr extends Unpackr {
 				position$1 = (position$1 + 7) & 0x7ffffff8; // Word align to make any future copying of this buffer faster
 			start = position$1;
 			referenceMap = packr.structuredClone ? new Map() : null;
-			if (packr.bundleStrings) {
-				bundledStrings$1 = ['', ''];
-				target[position$1++] = 0xd6;
-				target[position$1++] = 0x62; // 'b'
-				bundledStrings$1.position = position$1 - start;
-				position$1 += 4;
+			if (packr.bundleStrings && typeof value !== 'string') {
+				bundledStrings$1 = [];
+				bundledStrings$1.size = Infinity; // force a new bundle start on first string
 			} else
 				bundledStrings$1 = null;
 			sharedStructures = packr.structures;
@@ -1134,15 +1164,11 @@ class Packr extends Unpackr {
 			}
 			if (hasSharedUpdate)
 				hasSharedUpdate = false;
-			structures = sharedStructures || [];
+			structures = sharedStructures || (packr.structures = []);
 			try {
 				pack(value);
 				if (bundledStrings$1) {
-					targetView.setUint32(bundledStrings$1.position + start, position$1 - bundledStrings$1.position - start);
-					let writeStrings = bundledStrings$1;
-					bundledStrings$1 = null;
-					pack(writeStrings[0]);
-					pack(writeStrings[1]);
+					writeBundles(start, pack);
 				}
 				packr.offset = position$1; // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 				if (referenceMap && referenceMap.idsToInsert) {
@@ -1164,6 +1190,8 @@ class Packr extends Unpackr {
 				if (sharedStructures) {
 					if (serializationsSinceTransitionRebuild < 10)
 						serializationsSinceTransitionRebuild++;
+					if (sharedStructures.length > maxSharedStructures)
+						sharedStructures.length = maxSharedStructures;
 					if (transitionsCount > 10000) {
 						// force a rebuild occasionally after a lot of transitions so it can get cleaned up
 						sharedStructures.transitions = null;
@@ -1205,7 +1233,30 @@ class Packr extends Unpackr {
 			var length;
 			if (type === 'string') {
 				let strLength = value.length;
-				if (bundledStrings$1 && strLength >= 8 && strLength < 0x1000) {
+				if (bundledStrings$1 && strLength >= 4 && strLength < 0x1000) {
+					if ((bundledStrings$1.size += strLength) > MAX_BUNDLE_SIZE) {
+						let extStart;
+						let maxBytes = (bundledStrings$1[0] ? bundledStrings$1[0].length * 3 + bundledStrings$1[1].length : 0) + 10;
+						if (position$1 + maxBytes > safeEnd)
+							target = makeRoom(position$1 + maxBytes);
+						if (bundledStrings$1.position) { // here we use the 0x62 extension to write the last bundle and reserve sapce for the reference pointer to the next/current bundle
+							target[position$1] = 0xc8; // ext 16
+							position$1 += 3; // reserve for the writing bundle size
+							target[position$1++] = 0x62; // 'b'
+							extStart = position$1 - start;
+							position$1 += 4; // reserve for writing bundle reference
+							writeBundles(start, pack); // write the last bundles
+							targetView.setUint16(extStart + start - 3, position$1 - start - extStart);
+						} else { // here we use the 0x62 extension just to reserve the space for the reference pointer to the bundle (will be updated once the bundle is written)
+							target[position$1++] = 0xd6; // fixext 4
+							target[position$1++] = 0x62; // 'b'
+							extStart = position$1 - start;
+							position$1 += 4; // reserve for writing bundle reference
+						}
+						bundledStrings$1 = ['', '']; // create new ones
+						bundledStrings$1.size = 0;
+						bundledStrings$1.position = extStart;
+					}
 					let twoByte = hasNonLatin.test(value);
 					bundledStrings$1[twoByte ? 0 : 1] += value;
 					target[position$1++] = 0xc1;
@@ -1510,53 +1561,55 @@ class Packr extends Unpackr {
 			target[objectOffset++ + start] = size >> 8;
 			target[objectOffset + start] = size & 0xff;
 		} :
-
-	/*	sharedStructures ?  // For highly stable structures, using for-in can a little bit faster
+		(options.progressiveRecords && !useTwoByteRecords) ?  // this is about 2% faster for highly stable structures, since it only requires one for-in loop (but much more expensive when new structure needs to be written)
 		(object, safePrototype) => {
-			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null))
-			let objectOffset = position++ - start
-			let wroteKeys
+			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null));
+			let objectOffset = position$1++ - start;
+			let wroteKeys;
 			for (let key in object) {
 				if (safePrototype || object.hasOwnProperty(key)) {
-					nextTransition = transition[key]
-					if (!nextTransition) {
-						nextTransition = transition[key] = Object.create(null)
-						nextTransition.__keys__ = (transition.__keys__ || []).concat([key])
-						/*let keys = Object.keys(object)
-						if 
-						let size = 0
-						let startBranch = transition.__keys__ ? transition.__keys__.length : 0
-						for (let i = 0, l = keys.length; i++) {
-							let key = keys[i]
-							size += key.length << 2
-							if (i >= startBranch) {
-								nextTransition = nextTransition[key] = Object.create(null)
-								nextTransition.__keys__ = keys.slice(0, i + 1)
+					nextTransition = transition[key];
+					if (nextTransition)
+						transition = nextTransition;
+					else {
+						// record doesn't exist, create full new record and insert it
+						let keys = Object.keys(object);
+						let lastTransition = transition;
+						transition = structures.transitions;
+						let newTransitions = 0;
+						for (let i = 0, l = keys.length; i < l; i++) {
+							let key = keys[i];
+							nextTransition = transition[key];
+							if (!nextTransition) {
+								nextTransition = transition[key] = Object.create(null);
+								newTransitions++;
 							}
+							transition = nextTransition;
 						}
-						makeRoom(position + size)
-						nextTransition = transition[key]
-						target.copy(target, )
-						objectOffset
+						if (objectOffset + start + 1 == position$1) {
+							// first key, so we don't need to insert, we can just write record directly
+							position$1--;
+							newRecord(transition, keys, newTransitions);
+						} else // otherwise we need to insert the record, moving existing data after the record
+							insertNewRecord(transition, keys, objectOffset, newTransitions);
+						wroteKeys = true;
+						transition = lastTransition[key];
 					}
-					transition = nextTransition
-					pack(object[key])
+					pack(object[key]);
 				}
 			}
-			let id = transition.id
-			if (!id) {
-				id = transition.id = structures.push(transition.__keys__) + 63
-				if (sharedStructures.onUpdate)
-					sharedStructures.onUpdate(id, transition.__keys__)
+			if (!wroteKeys) {
+				let recordId = transition[RECORD_SYMBOL];
+				if (recordId)
+					target[objectOffset + start] = recordId;
+				else
+					insertNewRecord(transition, Object.keys(object), objectOffset, 0);
 			}
-			target[objectOffset + start] = id
-		}*/
-		(object) => {
-			let keys = Object.keys(object);
+		} :
+		(object, safePrototype) => {
 			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null));
 			let newTransitions = 0;
-			for (let i = 0, l = keys.length; i < l; i++) {
-				let key = keys[i];
+			for (let key in object) if (safePrototype || object.hasOwnProperty(key)) {
 				nextTransition = transition[key];
 				if (!nextTransition) {
 					nextTransition = transition[key] = Object.create(null);
@@ -1572,57 +1625,12 @@ class Packr extends Unpackr {
 				} else
 					target[position$1++] = recordId;
 			} else {
-				recordId = structures.nextId;
-				if (!recordId)
-					recordId = 0x40;
-				if (recordId < sharedLimitId && this.shouldShareStructure && !this.shouldShareStructure(keys)) {
-					recordId = structures.nextOwnId;
-					if (!(recordId < maxStructureId))
-						recordId = sharedLimitId;
-					structures.nextOwnId = recordId + 1;
-				} else {
-					if (recordId >= maxStructureId)// cycle back around
-						recordId = sharedLimitId;
-					structures.nextId = recordId + 1;
-				}
-				let highByte = keys.highByte = recordId >= 0x60 && useTwoByteRecords ? (recordId - 0x60) >> 5 : -1;
-				transition[RECORD_SYMBOL] = recordId;
-				structures[recordId - 0x40] = keys;
-
-				if (recordId < sharedLimitId) {
-					keys.isShared = true;
-					structures.sharedLength = recordId - 0x3f;
-					hasSharedUpdate = true;
-					if (highByte >= 0) {
-						target[position$1++] = (recordId & 0x1f) + 0x60;
-						target[position$1++] = highByte;
-					} else {
-						target[position$1++] = recordId;
-					}
-				} else {
-					if (highByte >= 0) {
-						target[position$1++] = 0xd5; // fixext 2
-						target[position$1++] = 0x72; // "r" record defintion extension type
-						target[position$1++] = (recordId & 0x1f) + 0x60;
-						target[position$1++] = highByte;
-					} else {
-						target[position$1++] = 0xd4; // fixext 1
-						target[position$1++] = 0x72; // "r" record defintion extension type
-						target[position$1++] = recordId;
-					}
-
-					if (newTransitions)
-						transitionsCount += serializationsSinceTransitionRebuild * newTransitions;
-					// record the removal of the id, we can maintain our shared structure
-					if (recordIdsToRemove.length >= maxOwnStructures)
-						recordIdsToRemove.shift()[RECORD_SYMBOL] = 0; // we are cycling back through, and have to remove old ones
-					recordIdsToRemove.push(transition);
-					pack(keys);
-				}
+				newRecord(transition, transition.__keys__ || Object.keys(object), newTransitions);
 			}
 			// now write the values
-			for (let i = 0, l = keys.length; i < l; i++)
-				pack(object[keys[i]]);
+			for (let key in object)
+				if (safePrototype || object.hasOwnProperty(key))
+					pack(object[key]);
 		};
 		const makeRoom = (end) => {
 			let newSize;
@@ -1645,6 +1653,86 @@ class Packr extends Unpackr {
 			safeEnd = newBuffer.length - 10;
 			return target = newBuffer
 		};
+		const newRecord = (transition, keys, newTransitions) => {
+			let recordId = structures.nextId;
+			if (!recordId)
+				recordId = 0x40;
+			if (recordId < sharedLimitId && this.shouldShareStructure && !this.shouldShareStructure(keys)) {
+				recordId = structures.nextOwnId;
+				if (!(recordId < maxStructureId))
+					recordId = sharedLimitId;
+				structures.nextOwnId = recordId + 1;
+			} else {
+				if (recordId >= maxStructureId)// cycle back around
+					recordId = sharedLimitId;
+				structures.nextId = recordId + 1;
+			}
+			let highByte = keys.highByte = recordId >= 0x60 && useTwoByteRecords ? (recordId - 0x60) >> 5 : -1;
+			transition[RECORD_SYMBOL] = recordId;
+			transition.__keys__ = keys;
+			structures[recordId - 0x40] = keys;
+
+			if (recordId < sharedLimitId) {
+				keys.isShared = true;
+				structures.sharedLength = recordId - 0x3f;
+				hasSharedUpdate = true;
+				if (highByte >= 0) {
+					target[position$1++] = (recordId & 0x1f) + 0x60;
+					target[position$1++] = highByte;
+				} else {
+					target[position$1++] = recordId;
+				}
+			} else {
+				if (highByte >= 0) {
+					target[position$1++] = 0xd5; // fixext 2
+					target[position$1++] = 0x72; // "r" record defintion extension type
+					target[position$1++] = (recordId & 0x1f) + 0x60;
+					target[position$1++] = highByte;
+				} else {
+					target[position$1++] = 0xd4; // fixext 1
+					target[position$1++] = 0x72; // "r" record defintion extension type
+					target[position$1++] = recordId;
+				}
+
+				if (newTransitions)
+					transitionsCount += serializationsSinceTransitionRebuild * newTransitions;
+				// record the removal of the id, we can maintain our shared structure
+				if (recordIdsToRemove.length >= maxOwnStructures)
+					recordIdsToRemove.shift()[RECORD_SYMBOL] = 0; // we are cycling back through, and have to remove old ones
+				recordIdsToRemove.push(transition);
+				pack(keys);
+			}
+		};
+		const insertNewRecord = (transition, keys, insertionOffset, newTransitions) => {
+			let mainTarget = target;
+			let mainPosition = position$1;
+			let mainSafeEnd = safeEnd;
+			let mainStart = start;
+			target = keysTarget;
+			position$1 = 0;
+			start = 0;
+			if (!target)
+				keysTarget = target = new ByteArrayAllocate(8192);
+			safeEnd = target.length - 10;
+			newRecord(transition, keys, newTransitions);
+			keysTarget = target;
+			let keysPosition = position$1;
+			target = mainTarget;
+			position$1 = mainPosition;
+			safeEnd = mainSafeEnd;
+			start = mainStart;
+			if (keysPosition > 1) {
+				let newEnd = position$1 + keysPosition - 1;
+				if (newEnd > safeEnd)
+					makeRoom(newEnd);
+				let insertionPosition = insertionOffset + start;
+				target.copyWithin(insertionPosition + keysPosition, insertionPosition + 1, position$1);
+				target.set(keysTarget.slice(0, keysPosition), insertionPosition);
+				position$1 = newEnd;
+			} else {
+				target[insertionOffset + start] = keysTarget[0];
+			}
+		};
 	}
 	useBuffer(buffer) {
 		// this means we are finished using our own buffer and we can write over it safely
@@ -1652,6 +1740,10 @@ class Packr extends Unpackr {
 		targetView = new DataView(target.buffer, target.byteOffset, target.byteLength);
 		position$1 = 0;
 	}
+	clearSharedData() {
+		if (this.structures)
+			this.structures = [];
+	}
 }
 
 extensionClasses = [ Date, Set, Error, RegExp, ArrayBuffer, Object.getPrototypeOf(Uint8Array.prototype).constructor /*TypedArray*/, C1Type ];
@@ -1849,6 +1941,14 @@ function insertIds(serialized, idsToInsert) {
 	return serialized
 }
 
+function writeBundles(start, pack) {
+	targetView.setUint32(bundledStrings$1.position + start, position$1 - bundledStrings$1.position - start);
+	let writeStrings = bundledStrings$1;
+	bundledStrings$1 = null;
+	pack(writeStrings[0]);
+	pack(writeStrings[1]);
+}
+
 function addExtension$1(extension) {
 	if (extension.Class) {
 		if (!extension.pack && !extension.write)

package/dist/test.js

@@ -634,10 +634,29 @@
 			let structures = [];
 			let packr = new Packr({ structures });
 			var serialized = packr.pack(data);
+			serialized = packr.pack(data);
+			serialized = packr.pack(data);
 			var deserialized = packr.unpack(serialized);
 			assert.deepEqual(deserialized, data);
 		});
 
+		test('mixed structures', function(){
+			let data1 = { a: 1, b: 2, c: 3 };
+			let data2 = { a: 1, b: 2, d: 4 };
+			let data3 = { a: 1, b: 2, e: 5 };
+			let structures = [];
+			let packr = new Packr({ structures });
+			var serialized = packr.pack(data1);
+			var deserialized = packr.unpack(serialized);
+			assert.deepEqual(deserialized, data1);
+			var serialized = packr.pack(data2);
+			var deserialized = packr.unpack(serialized);
+			assert.deepEqual(deserialized, data2);
+			var serialized = packr.pack(data3);
+			var deserialized = packr.unpack(serialized);
+			assert.deepEqual(deserialized, data3);
+		});
+
 		test('mixed array', function(){
 			var data = [
 				'one',
@@ -686,8 +705,7 @@
 		});
 		test('pack/unpack sample data with bundled strings', function(){
 			var data = sampleData;
-			let structures = [];
-			let packr = new Packr({ structures, bundleStrings: true });
+			let packr = new Packr({ /*structures,*/ useRecords: false, bundleStrings: true });
 			var serialized = packr.pack(data);
 			var deserialized = packr.unpack(serialized);
 			assert.deepEqual(deserialized, data);