msgpackr 1.7.0-alpha7 → 1.7.0

package/index.d.ts

@@ -1,17 +1,65 @@
-export {
-	Unpackr,
-	Decoder,
-	unpack,
-	unpackMultiple,
-	decode,
-	addExtension,
-	FLOAT32_OPTIONS,
-	clearSource,
-	roundFloat32,
-	isNativeAccelerationEnabled,
-} from './unpack'
-import { Options } from './unpack'
-export { Packr, Encoder, pack, encode } from './pack'
+export enum FLOAT32_OPTIONS {
+	NEVER = 0,
+	ALWAYS = 1,
+	DECIMAL_ROUND = 3,
+	DECIMAL_FIT = 4
+}
+
+export interface Options {
+	useFloat32?: FLOAT32_OPTIONS
+	useRecords?: boolean
+	structures?: {}[]
+	moreTypes?: boolean
+	structuredClone?: boolean
+	mapsAsObjects?: boolean
+	variableMapSize?: boolean
+	copyBuffers?: boolean
+	bundleStrings?: boolean
+	useTimestamp32?: boolean
+	largeBigIntToFloat?: boolean
+	encodeUndefinedAsNil?: boolean
+	maxSharedStructures?: number
+	maxOwnStructures?: number
+	int64AsNumber?: boolean
+	shouldShareStructure?: (keys: string[]) => boolean
+	getStructures?(): {}[]
+	saveStructures?(structures: {}[]): boolean | void
+	onInvalidDate?: () => any
+}
+interface Extension {
+	Class: Function
+	type: number
+	pack?(value: any): Buffer | Uint8Array
+	unpack?(messagePack: Buffer | Uint8Array): any	
+	read?(datum: any): any
+	write?(instance: any): any
+}
+export class Unpackr {
+	constructor(options?: Options)
+	unpack(messagePack: Buffer | Uint8Array): any
+	decode(messagePack: Buffer | Uint8Array): any
+	unpackMultiple(messagePack: Buffer | Uint8Array): any[]
+	unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
+}
+export class Decoder extends Unpackr {}
+export function unpack(messagePack: Buffer | Uint8Array): any
+export function unpackMultiple(messagePack: Buffer | Uint8Array): any[]
+export function unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
+export function decode(messagePack: Buffer | Uint8Array): any
+export function addExtension(extension: Extension): void
+export function clearSource(): void
+export function roundFloat32(float32Number: number): number
+export const C1: {}
+export let isNativeAccelerationEnabled: boolean
+
+export class Packr extends Unpackr {
+	pack(value: any): Buffer
+	encode(value: any): Buffer
+}
+export class Encoder extends Packr {}
+export function pack(value: any): Buffer
+export function encode(value: any): Buffer
+
 import { Transform, Readable } from 'stream'
 
 export as namespace msgpackr;

package/pack.d.ts

@@ -1,9 +1 @@
-import { Unpackr } from './unpack'
-export { addExtension, FLOAT32_OPTIONS } from './unpack'
-export class Packr extends Unpackr {
-	pack(value: any): Buffer
-	encode(value: any): Buffer
-}
-export class Encoder extends Packr {}
-export function pack(value: any): Buffer
-export function encode(value: any): Buffer
+export { Unpackr, Decoder, Packr, Encoder, pack, encode, unpack, decode, addExtension, FLOAT32_OPTIONS } from '.'

package/pack.js

@@ -1,4 +1,3 @@
-"use strict"
 import { Unpackr, mult10, C1Type, typedArrays, addExtension as unpackAddExtension } from './unpack.js'
 let textEncoder
 try {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "msgpackr",
   "author": "Kris Zyp",
-  "version": "1.7.0-alpha7",
+  "version": "1.7.0",
   "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",
@@ -20,7 +20,7 @@
   },
   "scripts": {
     "benchmark": "node ./tests/benchmark.cjs",
-    "build": "rollup -c",
+    "build": "rollup -c && cpy index.d.ts . --rename=index.d.cts",
     "dry-run": "npm publish --dry-run",
     "prepare": "npm run build",
     "test": "mocha tests/test**.*js -u tdd --experimental-json-modules"
@@ -36,6 +36,10 @@
         "require": "./dist/node.cjs",
         "import": "./node-index.js"
       },
+      "types": {
+        "require": "./index.d.cts",
+        "import": "./index.d.ts"
+      },
       "default": "./index.js"
     },
     "./pack": {
@@ -68,13 +72,14 @@
     "/*.ts"
   ],
   "optionalDependencies": {
-    "msgpackr-extract": "^2.1.1"
+    "msgpackr-extract": "^2.1.2"
   },
   "devDependencies": {
     "@rollup/plugin-json": "^4.1.0",
     "@types/node": "latest",
     "async": "^3",
     "chai": "^4.3.4",
+    "cpy-cli": "^4.1.0",
     "esm": "^3.2.25",
     "mocha": "^8.1.3",
     "rollup": "^1.20.3",

package/struct.js

@@ -45,6 +45,14 @@ const DATE = 16;
 const TYPE_NAMES = ['num', 'object', 'string', 'ascii'];
 TYPE_NAMES[DATE] = 'date';
 const float32Headers = [false, true, true, false, false, true, true, false];
+let evalSupported;
+try {
+	new Function('');
+	evalSupported = true;
+} catch(error) {
+	// if eval variants are not supported, do not create inline object readers ever
+}
+
 let updatedPosition;
 const hasNodeBuffer = typeof Buffer !== 'undefined'
 let textEncoder, currentSource;
@@ -472,9 +480,8 @@ function onLoadedStructures(sharedData) {
 	this.lastTypedStructuresLength = typed.length;
 	return named;
 }
-var sourceSymbol = Symbol('source')
+var sourceSymbol = Symbol.for('source')
 function readStruct(src, position, srcEnd, unpackr) {
-//	var stringLength = (src[position++] << 8) | src[position++];
 	let recordId = src[position++] - 0x20;
 	if (recordId >= 24) {
 		switch(recordId) {
@@ -505,23 +512,13 @@ function readStruct(src, position, srcEnd, unpackr) {
 		}
 		var prototype = construct.prototype;
 		let properties = [];
-		Object.defineProperty(prototype, 'toJSON', {
-			value() {
-				// return an enumerable object with own properties to JSON stringify
-				let resolved = {};
-				for (let i = 0, l = properties.length; i < l; i++) {
-					let key = properties[i].key;
-					resolved[key] = this[key];
-				}
-				return resolved;
-			},
-			// not enumerable or anything 
-		});
 		let currentOffset = 0;
 		let lastRefProperty;
 		for (let i = 0, l = structure.length; i < l; i++) {
 			let definition = structure[i];
 			let [ type, size, key, enumerationOffset ] = definition;
+			if (key === '__proto__')
+				key = '__proto_';
 			let property = {
 				key,
 				offset: currentOffset,
@@ -535,13 +532,13 @@ function readStruct(src, position, srcEnd, unpackr) {
 				case 0: getRef = () => 0; break;
 				case 1:
 					getRef = (source, position) => {
-						let ref = source.src[position + property.offset];
+						let ref = source.bytes[position + property.offset];
 						return ref >= 0xf6 ? toConstant(ref) : ref;
 					};
 					break;
 				case 2:
 					getRef = (source, position) => {
-						let src = source.src;
+						let src = source.bytes;
 						let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 						let ref = dataView.getUint16(position + property.offset, true);
 						return ref >= 0xff00 ? toConstant(ref & 0xff) : ref;
@@ -549,7 +546,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 					break;
 				case 4:
 					getRef = (source, position) => {
-						let src = source.src;
+						let src = source.bytes;
 						let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 						let ref = dataView.getUint32(position + property.offset, true);
 						return ref >= 0xffffff00 ? toConstant(ref & 0xff) : ref;
@@ -565,9 +562,8 @@ function readStruct(src, position, srcEnd, unpackr) {
 						lastRefProperty.next = property;
 					lastRefProperty = property;
 					property.multiGetCount = 0;
-					get = function() {
-						let source = this[sourceSymbol];
-						let src = source.src;
+					get = function(source) {
+						let src = source.bytes;
 						let position = source.position;
 						let refStart = currentOffset + position;
 						let ref = getRef(source, position);
@@ -583,7 +579,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 							next = next.next;
 						}
 						if (end == null)
-							end = source.srcEnd - refStart;
+							end = source.bytesEnd - refStart;
 						if (source.srcString) {
 							return source.srcString.slice(ref, end);
 						}
@@ -599,7 +595,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 									asciiEnd = null;
 							} while((next = next.next));
 							if (asciiEnd == null)
-								asciiEnd = source.srcEnd - refStart
+								asciiEnd = source.bytesEnd - refStart
 							source.srcString = src.toString('latin1', refStart, refStart + asciiEnd);
 							return source.srcString.slice(ref, end);
 						}
@@ -617,13 +613,12 @@ function readStruct(src, position, srcEnd, unpackr) {
 					if (lastRefProperty && !lastRefProperty.next)
 						lastRefProperty.next = property;
 					lastRefProperty = property;
-					get = function() {
-						let source = this[sourceSymbol];
+					get = function(source) {
 						let position = source.position;
 						let refStart = currentOffset + position;
 						let ref = getRef(source, position);
 						if (typeof ref !== 'number') return ref;
-						let src = source.src;
+						let src = source.bytes;
 						let end, next = property.next;
 						while(next) {
 							end = next.getRef(source, position);
@@ -634,7 +629,7 @@ function readStruct(src, position, srcEnd, unpackr) {
 							next = next.next;
 						}
 						if (end == null)
-							end = source.srcEnd - refStart;
+							end = source.bytesEnd - refStart;
 						if (type === UTF8) {
 							return src.toString('utf8', ref + refStart, end + refStart);
 						} else {
@@ -650,9 +645,8 @@ function readStruct(src, position, srcEnd, unpackr) {
 				case NUMBER:
 					switch(size) {
 						case 4:
-							get = function () {
-								let source = this[sourceSymbol];
-								let src = source.src;
+							get = function (source) {
+								let src = source.bytes;
 								let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 								let position = source.position + property.offset;
 								let value = dataView.getInt32(position, true)
@@ -669,9 +663,8 @@ function readStruct(src, position, srcEnd, unpackr) {
 							};
 							break;
 						case 8:
-							get = function () {
-								let source = this[sourceSymbol];
-								let src = source.src;
+							get = function (source) {
+								let src = source.bytes;
 								let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 								let value = dataView.getFloat64(source.position + property.offset, true);
 								if (isNaN(value)) {
@@ -683,9 +676,8 @@ function readStruct(src, position, srcEnd, unpackr) {
 							};
 							break;
 						case 1:
-							get = function () {
-								let source = this[sourceSymbol];
-								let src = source.src;
+							get = function (source) {
+								let src = source.bytes;
 								let value = src[source.position + property.offset];
 								return value < 0xf6 ? value : toConstant(value);
 							};
@@ -693,9 +685,8 @@ function readStruct(src, position, srcEnd, unpackr) {
 					}
 					break;
 				case DATE:
-					get = function () {
-						let source = this[sourceSymbol];
-						let src = source.src;
+					get = function (source) {
+						let src = source.bytes;
 						let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
 						return new Date(dataView.getFloat64(source.position + property.offset, true));
 					};
@@ -704,15 +695,45 @@ function readStruct(src, position, srcEnd, unpackr) {
 			}
 			property.get = get;
 		}
-		for (let property of properties) // assign in enumeration order
-			Object.defineProperty(prototype, property.key, { get: property.get, enumerable: true });
+		// TODO: load the srcString for faster string decoding on toJSON
+		if (evalSupported) {
+			let objectLiteralProperties = [];
+			let args = [];
+			let i = 0;
+			for (let property of properties) { // assign in enumeration order
+				Object.defineProperty(prototype, property.key, { get: withSource(property.get), enumerable: true });
+				let valueFunction = 'v' + i++;
+				args.push(valueFunction);
+				objectLiteralProperties.push('[' + JSON.stringify(property.key) + ']:' + valueFunction + '(s)');
+			}
+			let toObject = (new Function(...args, 'return function(s){return{' + objectLiteralProperties.join(',') + '}}')).apply(null, properties.map(prop => prop.get));
+			Object.defineProperty(prototype, 'toJSON', {
+				value() {
+					return toObject(this[sourceSymbol]);
+				}
+			});
+		} else {
+			Object.defineProperty(prototype, 'toJSON', {
+				value() {
+					// return an enumerable object with own properties to JSON stringify
+					let resolved = {};
+					for (let i = 0, l = properties.length; i < l; i++) {
+						let key = properties[i].key;
+
+						resolved[key] = this[key];
+					}
+					return resolved;
+				},
+				// not enumerable or anything
+			});
+		}
 	}
 	var instance = new construct();
 	instance[sourceSymbol] = {
-		src,
+		bytes: src,
 		position,
 		srcString: '',
-		srcEnd
+		bytesEnd: srcEnd
 	}
 	return instance;
 }
@@ -725,12 +746,17 @@ function toConstant(code) {
 	}
 	throw new Error('Unknown constant');
 }
+function withSource(get) {
+	return function() {
+		return get(this[sourceSymbol]);
+	}
+}
 
 function saveState() {
 	if (currentSource) {
-		currentSource.src = Uint8Array.prototype.slice.call(currentSource.src, currentSource.position, currentSource.srcEnd);
+		currentSource.bytes = Uint8Array.prototype.slice.call(currentSource.bytes, currentSource.position, currentSource.bytesEnd);
 		currentSource.position = 0;
-		currentSource.srcEnd = currentSource.src.length;
+		currentSource.bytesEnd = currentSource.bytes.length;
 	}
 }
 function prepareStructures(structures, packr) {

package/unpack.d.ts

@@ -1,53 +1,2 @@
-export enum FLOAT32_OPTIONS {
-	NEVER = 0,
-	ALWAYS = 1,
-	DECIMAL_ROUND = 3,
-	DECIMAL_FIT = 4
-}
-
-export interface Options {
-	useFloat32?: FLOAT32_OPTIONS
-	useRecords?: boolean
-	structures?: {}[]
-	moreTypes?: boolean
-	structuredClone?: boolean
-	mapsAsObjects?: boolean
-	variableMapSize?: boolean
-	copyBuffers?: boolean
-	bundleStrings?: boolean
-	useTimestamp32?: boolean
-	largeBigIntToFloat?: boolean
-	encodeUndefinedAsNil?: boolean
-	maxSharedStructures?: number
-	maxOwnStructures?: number
-	int64AsNumber?: boolean
-	shouldShareStructure?: (keys: string[]) => boolean
-	getStructures?(): {}[]
-	saveStructures?(structures: {}[]): boolean | void
-	onInvalidDate?: () => any
-}
-interface Extension {
-	Class: Function
-	type: number
-	pack?(value: any): Buffer | Uint8Array
-	unpack?(messagePack: Buffer | Uint8Array): any	
-	read?(datum: any): any
-	write?(instance: any): any
-}
-export class Unpackr {
-	constructor(options?: Options)
-	unpack(messagePack: Buffer | Uint8Array): any
-	decode(messagePack: Buffer | Uint8Array): any
-	unpackMultiple(messagePack: Buffer | Uint8Array): any[]
-	unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
-}
-export class Decoder extends Unpackr {}
-export function unpack(messagePack: Buffer | Uint8Array): any
-export function unpackMultiple(messagePack: Buffer | Uint8Array): any[]
-export function unpackMultiple(messagePack: Buffer | Uint8Array, forEach: (value: any) => any): void
-export function decode(messagePack: Buffer | Uint8Array): any
-export function addExtension(extension: Extension): void
-export function clearSource(): void
-export function roundFloat32(float32Number: number): number
-export const C1: {}
-export let isNativeAccelerationEnabled: boolean
+export { Unpackr, Decoder, unpack, unpackMultiple, decode,
+	addExtension, FLOAT32_OPTIONS, Options, Extension, clearSource, roundFloat32 } from '.'

package/unpack.js

@@ -1,4 +1,3 @@
-"use strict"
 var decoder
 try {
 	decoder = new TextDecoder()
@@ -183,6 +182,7 @@ export function checkedRead(options) {
 		let result
 		if (currentUnpackr.randomAccessStructure && src[position] < 0x40 && src[position] >= 0x20 && readStruct) {
 			result = readStruct(src, position, srcEnd, currentUnpackr)
+			src = null // dispose of this so that recursive unpack calls don't save state
 			if (!(options && options.lazy) && result)
 				result = result.toJSON()
 			position = srcEnd
@@ -193,7 +193,7 @@ export function checkedRead(options) {
 
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
-			if (currentStructures.restoreStructures)
+			if (currentStructures?.restoreStructures)
 				restoreStructures()
 			currentStructures = null
 			src = null
@@ -248,7 +248,10 @@ export function read() {
 			if (currentUnpackr.mapsAsObjects) {
 				let object = {}
 				for (let i = 0; i < token; i++) {
-					object[readKey()] = read()
+					let key = readKey()
+					if (key === '__proto__')
+						key = '__proto_'
+					object[key] = read()
 				}
 				return object
 			} else {
@@ -477,7 +480,7 @@ function createStructureReader(structure, firstId) {
 		// This initial function is quick to instantiate, but runs slower. After several iterations pay the cost to build the faster function
 		if (readObject.count++ > inlineObjectReadThreshold) {
 			let readObject = structure.read = (new Function('r', 'return function(){return ' + (currentUnpackr.freezeData ? 'Object.freeze' : '') +
-				'({' + structure.map(key => validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '})}'))(read)
+				'({' + structure.map(key => key === '__proto__' ? '__proto_:r()' : validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '})}'))(read)
 			if (structure.highByte === 0)
 				structure.read = createSecondByteReader(firstId, structure.read)
 			return readObject() // second byte is already read, if there is one so immediately read object
@@ -485,6 +488,8 @@ function createStructureReader(structure, firstId) {
 		let object = {}
 		for (let i = 0, l = structure.length; i < l; i++) {
 			let key = structure[i]
+			if (key === '__proto__')
+				key = '__proto_'
 			object[key] = read()
 		}
 		if (currentUnpackr.freezeData)
@@ -645,7 +650,10 @@ function readMap(length) {
 	if (currentUnpackr.mapsAsObjects) {
 		let object = {}
 		for (let i = 0; i < length; i++) {
-			object[readKey()] = read()
+			let key = readKey()
+			if (key === '__proto__')
+				key = '__proto_';
+			object[key] = read()
 		}
 		return object
 	} else {
@@ -930,7 +938,16 @@ function readKey() {
 
 // the registration of the record definition extension (as "r")
 const recordDefinition = (id, highByte) => {
-	var structure = read()
+	let structure
+	if (currentUnpackr.freezeData) {
+		currentUnpackr.freezeData = false;
+		try {
+			structure = read()
+		} finally {
+			currentUnpackr.freezeData = true;
+		}
+	} else
+		structure = read()
 	let firstByte = id
 	if (highByte !== undefined) {
 		id = id < 32 ? -((highByte << 5) + id) : ((highByte << 5) + id)