msgpackr 1.7.0-beta1 → 1.7.1

package/index.d.ts

@@ -1,17 +1,65 @@
-export {
-	Unpackr,
-	Decoder,
-	unpack,
-	unpackMultiple,
-	decode,
-	addExtension,
-	FLOAT32_OPTIONS,
-	clearSource,
-	roundFloat32,
-	isNativeAccelerationEnabled,
-} from './unpack'
-import { Options } from './unpack'
-export { Packr, Encoder, pack, encode } from './pack'
+export enum FLOAT32_OPTIONS {
+	NEVER = 0,
+	ALWAYS = 1,
+	DECIMAL_ROUND = 3,
+	DECIMAL_FIT = 4
+}
+
+export interface Options {
+	useFloat32?: FLOAT32_OPTIONS
+	useRecords?: boolean
+	structures?: {}[]
+	moreTypes?: boolean
+	structuredClone?: boolean
+	mapsAsObjects?: boolean
+	variableMapSize?: boolean
+	copyBuffers?: boolean
+	bundleStrings?: boolean
+	useTimestamp32?: boolean
+	largeBigIntToFloat?: boolean
+	encodeUndefinedAsNil?: boolean
+	maxSharedStructures?: number
+	maxOwnStructures?: number
+	int64AsNumber?: boolean
+	shouldShareStructure?: (keys: string[]) => boolean
+	getStructures?(): {}[]
+	saveStructures?(structures: {}[]): boolean | void
+	onInvalidDate?: () => any
+}
+interface Extension {
+	Class: Function
+	type: number
+	pack?(value: any): Buffer | Uint8Array
+	unpack?(messagePack: Buffer | Uint8Array): any	
+	read?(datum: any): any
+	write?(instance: any): any
+}
+export class Unpackr {
+	constructor(options?: Options)
+	unpack(messagePack: Buffer | Uint8Array): any
+	decode(messagePack: Buffer | Uint8Array): any
+	unpackMultiple(messagePack: Buffer | Uint8Array): any[]
+	unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
+}
+export class Decoder extends Unpackr {}
+export function unpack(messagePack: Buffer | Uint8Array): any
+export function unpackMultiple(messagePack: Buffer | Uint8Array): any[]
+export function unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
+export function decode(messagePack: Buffer | Uint8Array): any
+export function addExtension(extension: Extension): void
+export function clearSource(): void
+export function roundFloat32(float32Number: number): number
+export const C1: {}
+export let isNativeAccelerationEnabled: boolean
+
+export class Packr extends Unpackr {
+	pack(value: any): Buffer
+	encode(value: any): Buffer
+}
+export class Encoder extends Packr {}
+export function pack(value: any): Buffer
+export function encode(value: any): Buffer
+
 import { Transform, Readable } from 'stream'
 
 export as namespace msgpackr;

package/pack.d.ts

@@ -1,9 +1 @@
-import { Unpackr } from './unpack'
-export { addExtension, FLOAT32_OPTIONS } from './unpack'
-export class Packr extends Unpackr {
-	pack(value: any): Buffer
-	encode(value: any): Buffer
-}
-export class Encoder extends Packr {}
-export function pack(value: any): Buffer
-export function encode(value: any): Buffer
+export { Unpackr, Decoder, Packr, Encoder, pack, encode, unpack, decode, addExtension, FLOAT32_OPTIONS } from '.'

package/pack.js

@@ -1,4 +1,3 @@
-"use strict"
 import { Unpackr, mult10, C1Type, typedArrays, addExtension as unpackAddExtension } from './unpack.js'
 let textEncoder
 try {
@@ -126,12 +125,11 @@ export class Packr extends Unpackr {
 					writeStruct(value);
 				else
 					pack(value)
-				if (bundledStrings) {
-					writeBundles(start, pack)
-				}
-				packr.offset = position // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 				if (referenceMap && referenceMap.idsToInsert) {
-					position += referenceMap.idsToInsert.length * 6
+					let incrementPosition = referenceMap.idsToInsert.length * 6;
+					if (bundledStrings)
+						writeBundles(start, pack, incrementPosition)
+					position += incrementPosition
 					if (position > safeEnd)
 						makeRoom(position)
 					packr.offset = position
@@ -139,6 +137,9 @@ export class Packr extends Unpackr {
 					referenceMap = null
 					return serialized
 				}
+				if (bundledStrings)
+					writeBundles(start, pack, 0)
+				packr.offset = position // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 				if (encodeOptions & REUSE_BUFFER_MODE) {
 					target.start = start
 					target.end = position
@@ -923,12 +924,11 @@ function insertIds(serialized, idsToInsert) {
 	return serialized
 }
 
-function writeBundles(start, pack) {
+function writeBundles(start, pack, incrementPosition) {
 	if (bundledStrings.length > 0) {
-		targetView.setUint32(bundledStrings.position + start, position - bundledStrings.position - start)
+		targetView.setUint32(bundledStrings.position + start, position + incrementPosition - bundledStrings.position - start)
 		let writeStrings = bundledStrings
 		bundledStrings = null
-		let startPosition = position
 		pack(writeStrings[0])
 		pack(writeStrings[1])
 	}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "msgpackr",
   "author": "Kris Zyp",
-  "version": "1.7.0-beta1",
+  "version": "1.7.1",
   "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",
@@ -20,7 +20,7 @@
   },
   "scripts": {
     "benchmark": "node ./tests/benchmark.cjs",
-    "build": "rollup -c",
+    "build": "rollup -c && cpy index.d.ts . --rename=index.d.cts",
     "dry-run": "npm publish --dry-run",
     "prepare": "npm run build",
     "test": "mocha tests/test**.*js -u tdd --experimental-json-modules"
@@ -36,6 +36,10 @@
         "require": "./dist/node.cjs",
         "import": "./node-index.js"
       },
+      "types": {
+        "require": "./index.d.cts",
+        "import": "./index.d.ts"
+      },
       "default": "./index.js"
     },
     "./pack": {
@@ -68,13 +72,14 @@
     "/*.ts"
   ],
   "optionalDependencies": {
-    "msgpackr-extract": "^2.1.1"
+    "msgpackr-extract": "^2.1.2"
   },
   "devDependencies": {
     "@rollup/plugin-json": "^4.1.0",
     "@types/node": "latest",
     "async": "^3",
     "chai": "^4.3.4",
+    "cpy-cli": "^4.1.0",
     "esm": "^3.2.25",
     "mocha": "^8.1.3",
     "rollup": "^1.20.3",

package/struct.js

@@ -45,6 +45,14 @@ const DATE = 16;
 const TYPE_NAMES = ['num', 'object', 'string', 'ascii'];
 TYPE_NAMES[DATE] = 'date';
 const float32Headers = [false, true, true, false, false, true, true, false];
+let evalSupported;
+try {
+	new Function('');
+	evalSupported = true;
+} catch(error) {
+	// if eval variants are not supported, do not create inline object readers ever
+}
+
 let updatedPosition;
 const hasNodeBuffer = typeof Buffer !== 'undefined'
 let textEncoder, currentSource;
@@ -484,7 +492,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 			case 27: recordId = src[position++] + (src[position++] << 8) + (src[position++] << 16) + (src[position++] << 24); break;
 		}
 	}
-	let structure = unpackr.typedStructs?.[recordId];
+	let structure = unpackr.typedStructs && unpackr.typedStructs[recordId];
 	if (!structure) {
 		// copy src buffer because getStructures will override it
 		src = Uint8Array.prototype.slice.call(src, position, srcEnd);
@@ -504,23 +512,13 @@ function readStruct(src, position, srcEnd, unpackr) {
 		}
 		var prototype = construct.prototype;
 		let properties = [];
-		Object.defineProperty(prototype, 'toJSON', {
-			value() {
-				// return an enumerable object with own properties to JSON stringify
-				let resolved = {};
-				for (let i = 0, l = properties.length; i < l; i++) {
-					let key = properties[i].key;
-					resolved[key] = this[key];
-				}
-				return resolved;
-			},
-			// not enumerable or anything 
-		});
 		let currentOffset = 0;
 		let lastRefProperty;
 		for (let i = 0, l = structure.length; i < l; i++) {
 			let definition = structure[i];
 			let [ type, size, key, enumerationOffset ] = definition;
+			if (key === '__proto__')
+				key = '__proto_';
 			let property = {
 				key,
 				offset: currentOffset,
@@ -564,8 +562,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 						lastRefProperty.next = property;
 					lastRefProperty = property;
 					property.multiGetCount = 0;
-					get = function() {
-						let source = this[sourceSymbol];
+					get = function(source) {
 						let src = source.bytes;
 						let position = source.position;
 						let refStart = currentOffset + position;
@@ -616,8 +613,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 					if (lastRefProperty && !lastRefProperty.next)
 						lastRefProperty.next = property;
 					lastRefProperty = property;
-					get = function() {
-						let source = this[sourceSymbol];
+					get = function(source) {
 						let position = source.position;
 						let refStart = currentOffset + position;
 						let ref = getRef(source, position);
@@ -649,8 +645,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 				case NUMBER:
 					switch(size) {
 						case 4:
-							get = function () {
-								let source = this[sourceSymbol];
+							get = function (source) {
 								let src = source.bytes;
 								let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 								let position = source.position + property.offset;
@@ -668,8 +663,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 							};
 							break;
 						case 8:
-							get = function () {
-								let source = this[sourceSymbol];
+							get = function (source) {
 								let src = source.bytes;
 								let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 								let value = dataView.getFloat64(source.position + property.offset, true);
@@ -682,8 +676,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 							};
 							break;
 						case 1:
-							get = function () {
-								let source = this[sourceSymbol];
+							get = function (source) {
 								let src = source.bytes;
 								let value = src[source.position + property.offset];
 								return value < 0xf6 ? value : toConstant(value);
@@ -692,8 +685,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 					}
 					break;
 				case DATE:
-					get = function () {
-						let source = this[sourceSymbol];
+					get = function (source) {
 						let src = source.bytes;
 						let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 						return new Date(dataView.getFloat64(source.position + property.offset, true));
@@ -703,8 +695,38 @@ function readStruct(src, position, srcEnd, unpackr) {
 			}
 			property.get = get;
 		}
-		for (let property of properties) // assign in enumeration order
-			Object.defineProperty(prototype, property.key, { get: property.get, enumerable: true });
+		// TODO: load the srcString for faster string decoding on toJSON
+		if (evalSupported) {
+			let objectLiteralProperties = [];
+			let args = [];
+			let i = 0;
+			for (let property of properties) { // assign in enumeration order
+				Object.defineProperty(prototype, property.key, { get: withSource(property.get), enumerable: true });
+				let valueFunction = 'v' + i++;
+				args.push(valueFunction);
+				objectLiteralProperties.push('[' + JSON.stringify(property.key) + ']:' + valueFunction + '(s)');
+			}
+			let toObject = (new Function(...args, 'return function(s){return{' + objectLiteralProperties.join(',') + '}}')).apply(null, properties.map(prop => prop.get));
+			Object.defineProperty(prototype, 'toJSON', {
+				value() {
+					return toObject(this[sourceSymbol]);
+				}
+			});
+		} else {
+			Object.defineProperty(prototype, 'toJSON', {
+				value() {
+					// return an enumerable object with own properties to JSON stringify
+					let resolved = {};
+					for (let i = 0, l = properties.length; i < l; i++) {
+						let key = properties[i].key;
+
+						resolved[key] = this[key];
+					}
+					return resolved;
+				},
+				// not enumerable or anything
+			});
+		}
 	}
 	var instance = new construct();
 	instance[sourceSymbol] = {
@@ -724,6 +746,11 @@ function toConstant(code) {
 	}
 	throw new Error('Unknown constant');
 }
+function withSource(get) {
+	return function() {
+		return get(this[sourceSymbol]);
+	}
+}
 
 function saveState() {
 	if (currentSource) {
@@ -757,7 +784,7 @@ function prepareStructures(structures, packr) {
 			packr._mergeStructures(existing);
 		return compatible;
 	};
-	packr.lastTypedStructuresLength = packr.typedStructs?.length;
+	packr.lastTypedStructuresLength = packr.typedStructs && packr.typedStructs.length;
 	return structures;
 }
 

package/unpack.d.ts

@@ -1,53 +1,2 @@
-export enum FLOAT32_OPTIONS {
-	NEVER = 0,
-	ALWAYS = 1,
-	DECIMAL_ROUND = 3,
-	DECIMAL_FIT = 4
-}
-
-export interface Options {
-	useFloat32?: FLOAT32_OPTIONS
-	useRecords?: boolean
-	structures?: {}[]
-	moreTypes?: boolean
-	structuredClone?: boolean
-	mapsAsObjects?: boolean
-	variableMapSize?: boolean
-	copyBuffers?: boolean
-	bundleStrings?: boolean
-	useTimestamp32?: boolean
-	largeBigIntToFloat?: boolean
-	encodeUndefinedAsNil?: boolean
-	maxSharedStructures?: number
-	maxOwnStructures?: number
-	int64AsNumber?: boolean
-	shouldShareStructure?: (keys: string[]) => boolean
-	getStructures?(): {}[]
-	saveStructures?(structures: {}[]): boolean | void
-	onInvalidDate?: () => any
-}
-interface Extension {
-	Class: Function
-	type: number
-	pack?(value: any): Buffer | Uint8Array
-	unpack?(messagePack: Buffer | Uint8Array): any	
-	read?(datum: any): any
-	write?(instance: any): any
-}
-export class Unpackr {
-	constructor(options?: Options)
-	unpack(messagePack: Buffer | Uint8Array): any
-	decode(messagePack: Buffer | Uint8Array): any
-	unpackMultiple(messagePack: Buffer | Uint8Array): any[]
-	unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
-}
-export class Decoder extends Unpackr {}
-export function unpack(messagePack: Buffer | Uint8Array): any
-export function unpackMultiple(messagePack: Buffer | Uint8Array): any[]
-export function unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
-export function decode(messagePack: Buffer | Uint8Array): any
-export function addExtension(extension: Extension): void
-export function clearSource(): void
-export function roundFloat32(float32Number: number): number
-export const C1: {}
-export let isNativeAccelerationEnabled: boolean
+export { Unpackr, Decoder, unpack, unpackMultiple, decode,
+	addExtension, FLOAT32_OPTIONS, Options, Extension, clearSource, roundFloat32 } from '.'

package/unpack.js

@@ -1,4 +1,3 @@
-"use strict"
 var decoder
 try {
 	decoder = new TextDecoder()
@@ -183,6 +182,7 @@ export function checkedRead(options) {
 		let result
 		if (currentUnpackr.randomAccessStructure && src[position] < 0x40 && src[position] >= 0x20 && readStruct) {
 			result = readStruct(src, position, srcEnd, currentUnpackr)
+			src = null // dispose of this so that recursive unpack calls don't save state
 			if (!(options && options.lazy) && result)
 				result = result.toJSON()
 			position = srcEnd
@@ -193,7 +193,7 @@ export function checkedRead(options) {
 
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
-			if (currentStructures.restoreStructures)
+			if (currentStructures && currentStructures.restoreStructures)
 				restoreStructures()
 			currentStructures = null
 			src = null
@@ -208,7 +208,7 @@ export function checkedRead(options) {
 		// else more to read, but we are reading sequentially, so don't clear source yet
 		return result
 	} catch(error) {
-		if (currentStructures?.restoreStructures)
+		if (currentStructures && currentStructures.restoreStructures)
 			restoreStructures()
 		clearSource()
 		if (error instanceof RangeError || error.message.startsWith('Unexpected end of buffer') || position > srcEnd) {
@@ -248,7 +248,10 @@ export function read() {
 			if (currentUnpackr.mapsAsObjects) {
 				let object = {}
 				for (let i = 0; i < token; i++) {
-					object[readKey()] = read()
+					let key = readKey()
+					if (key === '__proto__')
+						key = '__proto_'
+					object[key] = read()
 				}
 				return object
 			} else {
@@ -477,7 +480,7 @@ function createStructureReader(structure, firstId) {
 		// This initial function is quick to instantiate, but runs slower. After several iterations pay the cost to build the faster function
 		if (readObject.count++ > inlineObjectReadThreshold) {
 			let readObject = structure.read = (new Function('r', 'return function(){return ' + (currentUnpackr.freezeData ? 'Object.freeze' : '') +
-				'({' + structure.map(key => validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '})}'))(read)
+				'({' + structure.map(key => key === '__proto__' ? '__proto_:r()' : validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '})}'))(read)
 			if (structure.highByte === 0)
 				structure.read = createSecondByteReader(firstId, structure.read)
 			return readObject() // second byte is already read, if there is one so immediately read object
@@ -485,6 +488,8 @@ function createStructureReader(structure, firstId) {
 		let object = {}
 		for (let i = 0, l = structure.length; i < l; i++) {
 			let key = structure[i]
+			if (key === '__proto__')
+				key = '__proto_'
 			object[key] = read()
 		}
 		if (currentUnpackr.freezeData)
@@ -645,7 +650,10 @@ function readMap(length) {
 	if (currentUnpackr.mapsAsObjects) {
 		let object = {}
 		for (let i = 0; i < length; i++) {
-			object[readKey()] = read()
+			let key = readKey()
+			if (key === '__proto__')
+				key = '__proto_';
+			object[key] = read()
 		}
 		return object
 	} else {