msgpackr 1.5.2 → 1.5.3

package/pack.js

@@ -9,11 +9,12 @@ const hasNodeBuffer = typeof Buffer !== 'undefined'
 const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array
 const ByteArray = hasNodeBuffer ? Buffer : Uint8Array
 const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000
-let target
+let target, keysTarget
 let targetView
 let position = 0
 let safeEnd
 let bundledStrings = null
+const MAX_BUNDLE_SIZE = 0xf000
 const hasNonLatin = /[\u0080-\uFFFF]/
 const RECORD_SYMBOL = Symbol('record-id')
 export class Packr extends Unpackr {
@@ -77,12 +78,9 @@ export class Packr extends Unpackr {
 				position = (position + 7) & 0x7ffffff8 // Word align to make any future copying of this buffer faster
 			start = position
 			referenceMap = packr.structuredClone ? new Map() : null
-			if (packr.bundleStrings) {
-				bundledStrings = ['', '']
-				target[position++] = 0xd6
-				target[position++] = 0x62 // 'b'
-				bundledStrings.position = position - start
-				position += 4
+			if (packr.bundleStrings && typeof value !== 'string') {
+				bundledStrings = []
+				bundledStrings.size = Infinity // force a new bundle start on first string
 			} else
 				bundledStrings = null
 			sharedStructures = packr.structures
@@ -120,15 +118,11 @@ export class Packr extends Unpackr {
 			}
 			if (hasSharedUpdate)
 				hasSharedUpdate = false
-			structures = sharedStructures || []
+			structures = sharedStructures || (packr.structures = [])
 			try {
 				pack(value)
 				if (bundledStrings) {
-					targetView.setUint32(bundledStrings.position + start, position - bundledStrings.position - start)
-					let writeStrings = bundledStrings
-					bundledStrings = null
-					pack(writeStrings[0])
-					pack(writeStrings[1])
+					writeBundles(start, pack)
 				}
 				packr.offset = position // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 				if (referenceMap && referenceMap.idsToInsert) {
@@ -150,6 +144,8 @@ export class Packr extends Unpackr {
 				if (sharedStructures) {
 					if (serializationsSinceTransitionRebuild < 10)
 						serializationsSinceTransitionRebuild++
+					if (sharedStructures.length > maxSharedStructures)
+						sharedStructures.length = maxSharedStructures
 					if (transitionsCount > 10000) {
 						// force a rebuild occasionally after a lot of transitions so it can get cleaned up
 						sharedStructures.transitions = null
@@ -191,7 +187,30 @@ export class Packr extends Unpackr {
 			var length
 			if (type === 'string') {
 				let strLength = value.length
-				if (bundledStrings && strLength >= 8 && strLength < 0x1000) {
+				if (bundledStrings && strLength >= 4 && strLength < 0x1000) {
+					if ((bundledStrings.size += strLength) > MAX_BUNDLE_SIZE) {
+						let extStart
+						let maxBytes = (bundledStrings[0] ? bundledStrings[0].length * 3 + bundledStrings[1].length : 0) + 10
+						if (position + maxBytes > safeEnd)
+							target = makeRoom(position + maxBytes)
+						if (bundledStrings.position) { // here we use the 0x62 extension to write the last bundle and reserve sapce for the reference pointer to the next/current bundle
+							target[position] = 0xc8 // ext 16
+							position += 3 // reserve for the writing bundle size
+							target[position++] = 0x62 // 'b'
+							extStart = position - start
+							position += 4 // reserve for writing bundle reference
+							writeBundles(start, pack) // write the last bundles
+							targetView.setUint16(extStart + start - 3, position - start - extStart)
+						} else { // here we use the 0x62 extension just to reserve the space for the reference pointer to the bundle (will be updated once the bundle is written)
+							target[position++] = 0xd6 // fixext 4
+							target[position++] = 0x62 // 'b'
+							extStart = position - start
+							position += 4 // reserve for writing bundle reference
+						}
+						bundledStrings = ['', ''] // create new ones
+						bundledStrings.size = 0
+						bundledStrings.position = extStart
+					}
 					let twoByte = hasNonLatin.test(value)
 					bundledStrings[twoByte ? 0 : 1] += value
 					target[position++] = 0xc1
@@ -496,8 +515,7 @@ export class Packr extends Unpackr {
 			target[objectOffset++ + start] = size >> 8
 			target[objectOffset + start] = size & 0xff
 		} :
-
-	/*	sharedStructures ?  // For highly stable structures, using for-in can a little bit faster
+		(options.progressiveRecords && !useTwoByteRecords) ?  // this is about 2% faster for highly stable structures, since it only requires one for-in loop (but much more expensive when new structure needs to be written)
 		(object, safePrototype) => {
 			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null))
 			let objectOffset = position++ - start
@@ -505,44 +523,47 @@ export class Packr extends Unpackr {
 			for (let key in object) {
 				if (safePrototype || object.hasOwnProperty(key)) {
 					nextTransition = transition[key]
-					if (!nextTransition) {
-						nextTransition = transition[key] = Object.create(null)
-						nextTransition.__keys__ = (transition.__keys__ || []).concat([key])
-						/*let keys = Object.keys(object)
-						if 
-						let size = 0
-						let startBranch = transition.__keys__ ? transition.__keys__.length : 0
-						for (let i = 0, l = keys.length; i++) {
+					if (nextTransition)
+						transition = nextTransition
+					else {
+						// record doesn't exist, create full new record and insert it
+						let keys = Object.keys(object)
+						let lastTransition = transition
+						transition = structures.transitions
+						let newTransitions = 0
+						for (let i = 0, l = keys.length; i < l; i++) {
 							let key = keys[i]
-							size += key.length << 2
-							if (i >= startBranch) {
-								nextTransition = nextTransition[key] = Object.create(null)
-								nextTransition.__keys__ = keys.slice(0, i + 1)
+							nextTransition = transition[key]
+							if (!nextTransition) {
+								nextTransition = transition[key] = Object.create(null)
+								newTransitions++
 							}
+							transition = nextTransition
 						}
-						makeRoom(position + size)
-						nextTransition = transition[key]
-						target.copy(target, )
-						objectOffset
+						if (objectOffset + start + 1 == position) {
+							// first key, so we don't need to insert, we can just write record directly
+							position--
+							newRecord(transition, keys, newTransitions)
+						} else // otherwise we need to insert the record, moving existing data after the record
+							insertNewRecord(transition, keys, objectOffset, newTransitions)
+						wroteKeys = true
+						transition = lastTransition[key]
 					}
-					transition = nextTransition
 					pack(object[key])
 				}
 			}
-			let id = transition.id
-			if (!id) {
-				id = transition.id = structures.push(transition.__keys__) + 63
-				if (sharedStructures.onUpdate)
-					sharedStructures.onUpdate(id, transition.__keys__)
+			if (!wroteKeys) {
+				let recordId = transition[RECORD_SYMBOL]
+				if (recordId)
+					target[objectOffset + start] = recordId
+				else
+					insertNewRecord(transition, Object.keys(object), objectOffset, 0)
 			}
-			target[objectOffset + start] = id
-		}*/
-		(object) => {
-			let keys = Object.keys(object)
+		} :
+		(object, safePrototype) => {
 			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null))
 			let newTransitions = 0
-			for (let i = 0, l = keys.length; i < l; i++) {
-				let key = keys[i]
+			for (let key in object) if (safePrototype || object.hasOwnProperty(key)) {
 				nextTransition = transition[key]
 				if (!nextTransition) {
 					nextTransition = transition[key] = Object.create(null)
@@ -558,57 +579,12 @@ export class Packr extends Unpackr {
 				} else
 					target[position++] = recordId
 			} else {
-				recordId = structures.nextId
-				if (!recordId)
-					recordId = 0x40
-				if (recordId < sharedLimitId && this.shouldShareStructure && !this.shouldShareStructure(keys)) {
-					recordId = structures.nextOwnId
-					if (!(recordId < maxStructureId))
-						recordId = sharedLimitId
-					structures.nextOwnId = recordId + 1
-				} else {
-					if (recordId >= maxStructureId)// cycle back around
-						recordId = sharedLimitId
-					structures.nextId = recordId + 1
-				}
-				let highByte = keys.highByte = recordId >= 0x60 && useTwoByteRecords ? (recordId - 0x60) >> 5 : -1
-				transition[RECORD_SYMBOL] = recordId
-				structures[recordId - 0x40] = keys
-
-				if (recordId < sharedLimitId) {
-					keys.isShared = true
-					structures.sharedLength = recordId - 0x3f
-					hasSharedUpdate = true
-					if (highByte >= 0) {
-						target[position++] = (recordId & 0x1f) + 0x60
-						target[position++] = highByte
-					} else {
-						target[position++] = recordId
-					}
-				} else {
-					if (highByte >= 0) {
-						target[position++] = 0xd5 // fixext 2
-						target[position++] = 0x72 // "r" record defintion extension type
-						target[position++] = (recordId & 0x1f) + 0x60
-						target[position++] = highByte
-					} else {
-						target[position++] = 0xd4 // fixext 1
-						target[position++] = 0x72 // "r" record defintion extension type
-						target[position++] = recordId
-					}
-
-					if (newTransitions)
-						transitionsCount += serializationsSinceTransitionRebuild * newTransitions
-					// record the removal of the id, we can maintain our shared structure
-					if (recordIdsToRemove.length >= maxOwnStructures)
-						recordIdsToRemove.shift()[RECORD_SYMBOL] = 0 // we are cycling back through, and have to remove old ones
-					recordIdsToRemove.push(transition)
-					pack(keys)
-				}
+				newRecord(transition, transition.__keys__ || Object.keys(object), newTransitions)
 			}
 			// now write the values
-			for (let i = 0, l = keys.length; i < l; i++)
-				pack(object[keys[i]])
+			for (let key in object)
+				if (safePrototype || object.hasOwnProperty(key))
+					pack(object[key])
 		}
 		const makeRoom = (end) => {
 			let newSize
@@ -631,6 +607,86 @@ export class Packr extends Unpackr {
 			safeEnd = newBuffer.length - 10
 			return target = newBuffer
 		}
+		const newRecord = (transition, keys, newTransitions) => {
+			let recordId = structures.nextId
+			if (!recordId)
+				recordId = 0x40
+			if (recordId < sharedLimitId && this.shouldShareStructure && !this.shouldShareStructure(keys)) {
+				recordId = structures.nextOwnId
+				if (!(recordId < maxStructureId))
+					recordId = sharedLimitId
+				structures.nextOwnId = recordId + 1
+			} else {
+				if (recordId >= maxStructureId)// cycle back around
+					recordId = sharedLimitId
+				structures.nextId = recordId + 1
+			}
+			let highByte = keys.highByte = recordId >= 0x60 && useTwoByteRecords ? (recordId - 0x60) >> 5 : -1
+			transition[RECORD_SYMBOL] = recordId
+			transition.__keys__ = keys
+			structures[recordId - 0x40] = keys
+
+			if (recordId < sharedLimitId) {
+				keys.isShared = true
+				structures.sharedLength = recordId - 0x3f
+				hasSharedUpdate = true
+				if (highByte >= 0) {
+					target[position++] = (recordId & 0x1f) + 0x60
+					target[position++] = highByte
+				} else {
+					target[position++] = recordId
+				}
+			} else {
+				if (highByte >= 0) {
+					target[position++] = 0xd5 // fixext 2
+					target[position++] = 0x72 // "r" record defintion extension type
+					target[position++] = (recordId & 0x1f) + 0x60
+					target[position++] = highByte
+				} else {
+					target[position++] = 0xd4 // fixext 1
+					target[position++] = 0x72 // "r" record defintion extension type
+					target[position++] = recordId
+				}
+
+				if (newTransitions)
+					transitionsCount += serializationsSinceTransitionRebuild * newTransitions
+				// record the removal of the id, we can maintain our shared structure
+				if (recordIdsToRemove.length >= maxOwnStructures)
+					recordIdsToRemove.shift()[RECORD_SYMBOL] = 0 // we are cycling back through, and have to remove old ones
+				recordIdsToRemove.push(transition)
+				pack(keys)
+			}
+		}
+		const insertNewRecord = (transition, keys, insertionOffset, newTransitions) => {
+			let mainTarget = target
+			let mainPosition = position
+			let mainSafeEnd = safeEnd
+			let mainStart = start
+			target = keysTarget
+			position = 0
+			start = 0
+			if (!target)
+				keysTarget = target = new ByteArrayAllocate(8192)
+			safeEnd = target.length - 10
+			newRecord(transition, keys, newTransitions)
+			keysTarget = target
+			let keysPosition = position
+			target = mainTarget
+			position = mainPosition
+			safeEnd = mainSafeEnd
+			start = mainStart
+			if (keysPosition > 1) {
+				let newEnd = position + keysPosition - 1
+				if (newEnd > safeEnd)
+					makeRoom(newEnd)
+				let insertionPosition = insertionOffset + start
+				target.copyWithin(insertionPosition + keysPosition, insertionPosition + 1, position)
+				target.set(keysTarget.slice(0, keysPosition), insertionPosition)
+				position = newEnd
+			} else {
+				target[insertionOffset + start] = keysTarget[0]
+			}
+		}
 	}
 	useBuffer(buffer) {
 		// this means we are finished using our own buffer and we can write over it safely
@@ -638,6 +694,10 @@ export class Packr extends Unpackr {
 		targetView = new DataView(target.buffer, target.byteOffset, target.byteLength)
 		position = 0
 	}
+	clearSharedData() {
+		if (this.structures)
+			this.structures = []
+	}
 }
 
 function copyBinary(source, target, targetOffset, offset, endOffset) {
@@ -841,6 +901,15 @@ function insertIds(serialized, idsToInsert) {
 	return serialized
 }
 
+function writeBundles(start, pack) {
+	targetView.setUint32(bundledStrings.position + start, position - bundledStrings.position - start)
+	let writeStrings = bundledStrings
+	bundledStrings = null
+	let startPosition = position
+	pack(writeStrings[0])
+	pack(writeStrings[1])
+}
+
 export function addExtension(extension) {
 	if (extension.Class) {
 		if (!extension.pack && !extension.write)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "msgpackr",
   "author": "Kris Zyp",
-  "version": "1.5.2",
+  "version": "1.5.3",
   "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",

package/unpack.js

@@ -156,6 +156,9 @@ export function checkedRead() {
 				currentStructures.length = sharedLength
 		}
 		let result = read()
+		if (bundledStrings) // bundled strings to skip past
+			position = bundledStrings.postBundlePosition
+
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
 			if (currentStructures.restoreStructures)
@@ -498,6 +501,8 @@ export function setExtractor(extractStrings) {
 		return function readString(length) {
 			let string = strings[stringPosition++]
 			if (string == null) {
+				if (bundledStrings)
+					return readStringJS(length)
 				let extraction = extractStrings(position - headerLength, srcEnd, src)
 				if (typeof extraction == 'string') {
 					string = extraction
@@ -756,6 +761,36 @@ function shortStringInJS(length) {
 	}
 }
 
+function readOnlyJSString() {
+	let token = src[position++]
+	let length
+	if (token < 0xc0) {
+		// fixstr
+		length = token - 0xa0
+	} else {
+		switch(token) {
+			case 0xd9:
+			// str 8
+				length = src[position++]
+				break
+			case 0xda:
+			// str 16
+				length = dataView.getUint16(position)
+				position += 2
+				break
+			case 0xdb:
+			// str 32
+				length = dataView.getUint32(position)
+				position += 4
+				break
+			default:
+				throw new Error('Expected string')
+		}
+	}
+	return readStringJS(length)
+}
+
+
 function readBin(length) {
 	return currentUnpackr.copyBuffers ?
 		// specifically use the copying slice (not the node one)
@@ -907,21 +942,18 @@ currentExtensions[0x78] = () => {
 	let data = read()
 	return new RegExp(data[0], data[1])
 }
-
+const TEMP_BUNDLE = []
 currentExtensions[0x62] = (data) => {
 	let dataSize = (data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]
 	let dataPosition = position
-	position += dataSize - 4
-	bundledStrings = [read(), read()]
+	position += dataSize - data.length
+	bundledStrings = TEMP_BUNDLE
+	bundledStrings = [readOnlyJSString(), readOnlyJSString()]
 	bundledStrings.position0 = 0
 	bundledStrings.position1 = 0
-	let postBundlePosition = position
+	bundledStrings.postBundlePosition = position
 	position = dataPosition
-	try {
-		return read()
-	} finally {
-		position = postBundlePosition
-	}
+	return read()
 }
 
 currentExtensions[0xff] = (data) => {