msgpackr 1.6.2 → 1.7.0-alpha1

package/node-index.js

@@ -1,5 +1,6 @@
 export { Packr, Encoder, addExtension, pack, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } from './pack.js'
 export { Unpackr, Decoder, C1, unpack, unpackMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './unpack.js'
+import './struct.js'
 export { PackrStream, UnpackrStream, PackrStream as EncoderStream, UnpackrStream as DecoderStream } from './stream.js'
 export { decodeIter, encodeIter } from './iterators.js'
 export const useRecords = false

package/pack.js

@@ -15,9 +15,10 @@ let targetView
 let position = 0
 let safeEnd
 let bundledStrings = null
+let writeStructSlots
 const MAX_BUNDLE_SIZE = 0xf000
 const hasNonLatin = /[\u0080-\uFFFF]/
-const RECORD_SYMBOL = Symbol('record-id')
+export const RECORD_SYMBOL = Symbol('record-id')
 export class Packr extends Unpackr {
 	constructor(options) {
 		super(options)
@@ -67,14 +68,14 @@ export class Packr extends Unpackr {
 		this.pack = this.encode = function(value, encodeOptions) {
 			if (!target) {
 				target = new ByteArrayAllocate(8192)
-				targetView = new DataView(target.buffer, 0, 8192)
+				targetView = target.dataView = new DataView(target.buffer, 0, 8192)
 				position = 0
 			}
 			safeEnd = target.length - 10
 			if (safeEnd - position < 0x800) {
 				// don't start too close to the end, 
 				target = new ByteArrayAllocate(target.length)
-				targetView = new DataView(target.buffer, 0, target.length)
+				targetView = target.dataView = new DataView(target.buffer, 0, target.length)
 				safeEnd = target.length - 10
 				position = 0
 			} else
@@ -122,7 +123,10 @@ export class Packr extends Unpackr {
 			if (hasSharedUpdate)
 				hasSharedUpdate = false
 			try {
-				pack(value)
+				if (packr.randomAccessStructure)
+					writeStruct(value);
+				else
+					pack(value)
 				if (bundledStrings) {
 					writeBundles(start, pack)
 				}
@@ -596,7 +600,7 @@ export class Packr extends Unpackr {
 			} else // faster handling for smaller buffers
 				newSize = ((Math.max((end - start) << 2, target.length - 1) >> 12) + 1) << 12
 			let newBuffer = new ByteArrayAllocate(newSize)
-			targetView = new DataView(newBuffer.buffer, 0, newSize)
+			targetView = newBuffer.dataView = new DataView(newBuffer.buffer, 0, newSize)
 			end = Math.min(end, target.length)
 			if (target.copy)
 				target.copy(newBuffer, 0, start, end)
@@ -687,6 +691,21 @@ export class Packr extends Unpackr {
 				target[insertionOffset + start] = keysTarget[0]
 			}
 		}
+		const writeStruct = (object, safePrototype) => {
+			let newPosition = writeStructSlots(object, target, position, structures, makeRoom, (value, newPosition) => {
+				position = newPosition;
+				if (start > 0) {
+					pack(value);
+					if (start == 0)
+						return { position, targetView }; // indicate the buffer was re-allocated
+				} else
+					pack(value);
+				return position;
+			})
+			if (newPosition === 0) // bail and go to a msgpack object
+				return writeObject(object, true);
+			position = newPosition;
+		}
 	}
 	useBuffer(buffer) {
 		// this means we are finished using our own buffer and we can write over it safely
@@ -923,6 +942,9 @@ export function addExtension(extension) {
 	}
 	unpackAddExtension(extension)
 }
+export function setWriteStructSlots(func) {
+	writeStructSlots = func;
+}
 
 let defaultPackr = new Packr({ useRecords: false })
 export const pack = defaultPackr.pack

package/package.json

@@ -1,83 +1,83 @@
-{
-  "name": "msgpackr",
-  "author": "Kris Zyp",
-  "version": "1.6.2",
-  "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
-  "license": "MIT",
-  "types": "./index.d.ts",
-  "main": "./dist/node.cjs",
-  "module": "./index.js",
-  "keywords": [
-    "MessagePack",
-    "msgpack",
-    "performance",
-    "structured",
-    "clone"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "http://github.com/kriszyp/msgpackr"
-  },
-  "scripts": {
-    "benchmark": "node ./tests/benchmark.cjs",
-    "build": "rollup -c",
-    "dry-run": "npm publish --dry-run",
-    "prepare": "npm run build",
-    "test": "mocha tests/test**.*js -u tdd --experimental-json-modules"
-  },
-  "type": "module",
-  "exports": {
-    ".": {
-      "node": {
-        "require": "./dist/node.cjs",
-        "import": "./node-index.js"
-      },
-      "bun": {
-        "require": "./dist/node.cjs",
-        "import": "./node-index.js"
-      },
-      "default": "./index.js"
-    },
-    "./pack": {
-      "node": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "bun": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "default": "./pack.js"
-    },
-    "./unpack": {
-      "node": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "bun": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "default": "./unpack.js"
-    }
-  },
-  "files": [
-    "/dist",
-    "*.md",
-    "/*.js",
-    "/*.ts"
-  ],
-  "optionalDependencies": {
-    "msgpackr-extract": "^2.0.2"
-  },
-  "devDependencies": {
-    "@rollup/plugin-json": "^4.1.0",
-    "@types/node": "latest",
-    "async": "^3",
-    "chai": "^4.3.4",
-    "esm": "^3.2.25",
-    "mocha": "^8.1.3",
-    "rollup": "^1.20.3",
-    "rollup-plugin-babel-minify": "^9.0.0"
-  }
-}
+{
+  "name": "msgpackr",
+  "author": "Kris Zyp",
+  "version": "1.7.0-alpha1",
+  "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
+  "license": "MIT",
+  "types": "./index.d.ts",
+  "main": "./dist/node.cjs",
+  "module": "./index.js",
+  "keywords": [
+    "MessagePack",
+    "msgpack",
+    "performance",
+    "structured",
+    "clone"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "http://github.com/kriszyp/msgpackr"
+  },
+  "scripts": {
+    "benchmark": "node ./tests/benchmark.cjs",
+    "build": "rollup -c",
+    "dry-run": "npm publish --dry-run",
+    "prepare": "npm run build",
+    "test": "mocha tests/test**.*js -u tdd --experimental-json-modules"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "node": {
+        "require": "./dist/node.cjs",
+        "import": "./node-index.js"
+      },
+      "bun": {
+        "require": "./dist/node.cjs",
+        "import": "./node-index.js"
+      },
+      "default": "./index.js"
+    },
+    "./pack": {
+      "node": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "bun": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "default": "./pack.js"
+    },
+    "./unpack": {
+      "node": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "bun": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "default": "./unpack.js"
+    }
+  },
+  "files": [
+    "/dist",
+    "*.md",
+    "/*.js",
+    "/*.ts"
+  ],
+  "optionalDependencies": {
+    "msgpackr-extract": "^2.1.1"
+  },
+  "devDependencies": {
+    "@rollup/plugin-json": "^4.1.0",
+    "@types/node": "latest",
+    "async": "^3",
+    "chai": "^4.3.4",
+    "esm": "^3.2.25",
+    "mocha": "^8.1.3",
+    "rollup": "^1.20.3",
+    "rollup-plugin-babel-minify": "^9.0.0"
+  }
+}

package/struct.js

@@ -0,0 +1,260 @@
+// first four bits
+// 0000 - unsigned int
+// 0010 - float32
+// 0011 - float32
+// 0100 - float32
+// 0101 - float32
+// 0110 - latin string reference
+// 0111 - plain reference
+// 1000 - structure reference
+// 1001 - random access structure reference
+// 1010 - float32
+// 1011 - float32
+// 1100 - float32
+// 1101 - float32
+// 1110 - constants and 3-byte strings
+// 1111 - negative int
+
+// first three bits
+// 000 - unsigned int
+// 001 - float32
+// 010 - float32
+// 011 - latin string reference
+// 100 - reference
+// 101 - float32
+// 110 - float32
+// 111 - constants and 3-byte strings
+
+import { setWriteStructSlots, RECORD_SYMBOL } from './pack.js'
+import { setReadStruct, unpack, mult10 } from './unpack.js';
+const hasNonLatin = /[\u0080-\uFFFF]/;
+const float32Headers = [false, true, true, false, false, true, true, false]
+setWriteStructSlots(writeStruct);
+function writeStruct(object, target, position, structures, makeRoom, pack) {
+	let transition = structures.transitions || false
+	let newTransitions = 0
+	let keyCount = 0;
+	let start = position;
+	position += 4;
+	let queuedReferences = [];
+	let uint32 = target.uint32 || (target.uint32 = new Uint32Array(target.buffer));
+	let targetView = target.dataView;
+	let encoded;
+	let stringData = '';
+	let safeEnd = target.length - 10;
+	for (let key in object) {
+		let nextTransition = transition[key]
+		if (!nextTransition) {
+			return 0; // bail
+			//nextTransition = transition[key] = Object.create(null)
+			//newTransitions++
+		}
+		if (position > safeEnd) {
+			let newPosition = position - start;
+			target = makeRoom(position)
+			position = newPosition;
+			start = 0
+			safeEnd = target.length - 10
+		}
+		transition = nextTransition
+		let value = object[key];
+		switch (typeof value) {
+			case 'number':
+				if (value >>> 0 === value && value < 0x20000000) {
+					encoded = value;
+					break;
+				} else if (value < 0x100000000 && value >= -0x80000000) {
+					targetView.setFloat32(position, value, true)
+					if (float32Headers[target[position + 3] >>> 5]) {
+						let xShifted
+						// this checks for rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
+						if (((xShifted = value * mult10[((target[position + 3] & 0x7f) << 1) | (target[position + 2] >> 7)]) >> 0) === xShifted) {
+							position += 4;
+							continue;
+						}
+					}
+				}
+				// fall back to msgpack encoding
+				queuedReferences.push(value, position - start);
+				position += 4;
+				continue;
+			case 'string':
+				if (hasNonLatin.test(value)) {
+					queuedReferences.push(value, position - start);
+					position += 4;
+					continue;
+				}
+				if (value.length < 4) { // we can inline really small strings
+					encoded = 0xf8000000 + (value.length << 24) + (value.charCodeAt(0) << 16) + (value.charCodeAt(1) << 8) + (value.charCodeAt(2) || 0)
+					// TODO: determining remaining and make max value be a ratio of that (probably 1/256th)
+				} else if (value.length < 256 && stringData.length < 61440) {
+					// bundle these strings
+					encoded = 0x60000000 | (value.length << 16) | stringData.length;
+					stringData += value;
+				} else { // else queue it
+					queuedReferences.push(value, position - start);
+					position += 4;
+					continue;
+				}
+				break;
+			case 'object':
+				if (value) {
+					queuedReferences.push(value, position - start);
+					position += 4;
+					continue;
+				} else { // null
+					encoded = 0xe0000000;
+				}
+				break;
+			case 'boolean':
+				encoded = value ? 0xe3000000 : 0xe2000000;
+				break;
+			case 'undefined':
+				encoded = 0xe1000000;
+				break;
+		}
+		targetView.setUint32(position, encoded, true);
+		position += 4;
+	}
+	let recordId = transition[RECORD_SYMBOL]
+	if (!(recordId < 1024)) {
+		// for now just punt and go back to writeObject
+		return 0;
+		// newRecord(transition, transition.__keys__ || Object.keys(object), newTransitions, true)
+	}
+	let stringLength = stringData.length;
+	if (stringData) {
+		if (position + stringLength > safeEnd) {
+			target = makeRoom(position + stringLength);
+		}
+		position += target.latin1Write(stringData, position, 0xffffffff);
+	}
+	target[start] = recordId >> 8;
+	target[start + 1] = recordId & 0xff;
+	target[start + 2] = stringLength >> 8;
+	target[start + 3] = stringLength & 0xff;
+	let queued32BitReferences;
+	for (let i = 0, l = queuedReferences.length; i < l;) {
+		let value = queuedReferences[i++];
+		let slotOffset = queuedReferences[i++] + start;
+		let offset = position - slotOffset;
+		if (offset < 0x1f000000) {
+			targetView.setUint32(slotOffset, 0x80000000 | (offset), true);
+		} else {
+			if (!queued32BitReferences)
+				queued32BitReferences = [];
+			queued32BitReferences.push({slotOffset, offset: position - start});
+		}
+		let newPosition = pack(value, position);
+		if (typeof newPosition === 'object') {
+			// re-allocated
+			position = newPosition.position;
+			targetView = newPosition.targetView;
+			start = 0;
+		} else
+			position = newPosition;
+	}
+	if (queued32BitReferences) {
+		// TODO: makeRoom
+		for (let i = 0, l = queued32BitReferences.length; i < l; i++) {
+			let ref = queued32BitReferences[i];
+			targetView.setUint32(ref.slotOffset, 0xa0000000 - ((l - i) << 2), true);
+			targetView.setUint32(position, ref.offset, true);
+			position += 4;
+		}
+	}
+
+	return position;
+}
+var sourceSymbol = Symbol('source')
+function readStruct(src, position, srcEnd, structure, unpackr) {
+	var stringLength = (src[position++] << 8) | src[position++];
+	var construct = structure.construct;
+	var srcString;
+	if (!construct) {
+		construct = structure.construct = function() {
+		}
+		var prototype = construct.prototype;
+		Object.defineProperty(prototype, 'toJSON', {
+			get() {
+				// return an enumerable object with own properties to JSON stringify
+				let resolved = {};
+				for (let i = 0, l = structure.length; i < l; i++) {
+					let key = structure[i];
+					resolved[key] = this[key];
+				}
+				return resolved;
+			},
+			// not enumerable or anything 
+		});
+		for (let i = 0, l = structure.length; i < l; i++) {
+			let key = structure[i];
+			Object.defineProperty(prototype, key, {
+				get() {
+					let source = this[sourceSymbol];
+					let src = source.src;
+					//let uint32 = src.uint32 || (src.uint32 = new Uint32Array(src.buffer, src.byteOffset, src.byteLength));
+					let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
+					let position = source.position + (i << 2);
+					let value = dataView.getUint32(position, true);
+					let start;
+					switch (value >>> 29) {
+						case 0:
+							return value;
+						case 3:
+							if (value & 0x10000000) {
+								start = (value & 0xffff) + position;
+								return src.toString('utf8', start, start + ((value >> 16) & 0x7ff));
+							} else {
+								if (!srcString) {
+									start = source.position + (l << 2);
+									srcString = src.toString('latin1', start, start + stringLength);
+								}
+								start = value & 0xffff;
+								return srcString.slice(start, start + ((value >> 16) & 0x7ff));
+							}
+						case 4:
+							start = (0x1fffffff & value) + position;
+							let end = srcEnd;
+							for (let next = i + 1; next < l; next++) {
+								position = source.position + (next << 2);
+								let nextValue = dataView.getUint32(position, true);;
+								if ((nextValue & 0xe0000000) == -0x80000000) {
+									end = (0x1fffffff & nextValue) + position;
+									break;
+								}
+							}
+							return unpackr.unpack(src.slice(start, end));
+						case 1: case 2: case 5: case 6:
+							let fValue = dataView.getFloat32(position, true);
+							// this does rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
+							let multiplier = mult10[((src[position + 3] & 0x7f) << 1) | (src[position + 2] >> 7)]
+							return ((multiplier * fValue + (fValue > 0 ? 0.5 : -0.5)) >> 0) / multiplier;
+						case 7:
+							switch((value >> 24) & 0x1f) {
+								case 0: return null;
+								case 1: return undefined;
+								case 2: return false;
+								case 3: return true;
+								case 8: return dataView.getFloat64(position + (value & 0x3ffffff), true);
+								case 0x18: return '';
+								case 0x19: return String.fromCharCode((value >> 16) & 0xff);
+								case 0x20: return String.fromCharCode((value >> 16) & 0xff, (value >> 8) & 0xff);
+								case 0x21: return String.fromCharCode((value >> 16) & 0xff, (value >> 8) & 0xff, value & 0xff);
+								default: throw new Error('Unknown constant');
+							}
+					}
+				},
+				enumerable: true,
+			});
+		}
+	}
+	var instance = new construct();
+	instance[sourceSymbol] = {
+		src,
+		uint32: src.uint32,
+		position,
+	}
+	return instance;
+}
+setReadStruct(readStruct)

package/unpack.js

@@ -28,6 +28,7 @@ export const C1 = new C1Type()
 C1.name = 'MessagePack 0xC1'
 var sequentialMode = false
 var inlineObjectReadThreshold = 2
+var readStruct
 try {
 	new Function('')
 } catch(error) {
@@ -170,7 +171,13 @@ export function checkedRead() {
 			if (sharedLength < currentStructures.length)
 				currentStructures.length = sharedLength
 		}
-		let result = read()
+		let result
+		if (currentUnpackr.randomAccessStructure && src[position] < 0x40 && readStruct) {
+			let id = (src[position++] << 8) + src[position++];
+			result = readStruct(src, position, srcEnd, currentStructures[id - 0x40] || loadStructures()[id - 0x40], currentUnpackr)
+			position = srcEnd
+		} else
+			result = read()
 		if (bundledStrings) // bundled strings to skip past
 			position = bundledStrings.postBundlePosition
 
@@ -247,6 +254,8 @@ export function read() {
 			for (let i = 0; i < token; i++) {
 				array[i] = read()
 			}
+			if (currentUnpackr.freezeData)
+				return Object.freeze(array)
 			return array
 		}
 	} else if (token < 0xc0) {
@@ -457,7 +466,8 @@ function createStructureReader(structure, firstId) {
 	function readObject() {
 		// This initial function is quick to instantiate, but runs slower. After several iterations pay the cost to build the faster function
 		if (readObject.count++ > inlineObjectReadThreshold) {
-			let readObject = structure.read = (new Function('r', 'return function(){return {' + structure.map(key => validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '}}'))(read)
+			let readObject = structure.read = (new Function('r', 'return function(){return ' + (currentUnpackr.freezeData ? 'Object.freeze' : '') +
+				'({' + structure.map(key => validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '})}'))(read)
 			if (structure.highByte === 0)
 				structure.read = createSecondByteReader(firstId, structure.read)
 			return readObject() // second byte is already read, if there is one so immediately read object
@@ -467,6 +477,8 @@ function createStructureReader(structure, firstId) {
 			let key = structure[i]
 			object[key] = read()
 		}
+		if (currentUnpackr.freezeData)
+			return Object.freeze(object);
 		return object
 	}
 	readObject.count = 0
@@ -604,6 +616,8 @@ function readArray(length) {
 	for (let i = 0; i < length; i++) {
 		array[i] = read()
 	}
+	if (currentUnpackr.freezeData)
+		return Object.freeze(array)
 	return array
 }
 
@@ -1059,3 +1073,6 @@ export function roundFloat32(float32Number) {
 	let multiplier = mult10[((u8Array[3] & 0x7f) << 1) | (u8Array[2] >> 7)]
 	return ((multiplier * float32Number + (float32Number > 0 ? 0.5 : -0.5)) >> 0) / multiplier
 }
+export function setReadStruct(func) {
+	readStruct = func;
+}