cbor-x 1.5.8 → 1.6.0

package/README.md

@@ -228,6 +228,18 @@ Note, that the performance is decreased with decimal rounding by about 20-25%, a
 
 In addition, msgpackr exports a `roundFloat32(number)` function that can be used to round floating point numbers to the maximum significant decimal digits that can be stored in 32-bit float, just as DECIMAL_ROUND does when decoding. This can be useful for determining how a number will be decoded prior to encoding it.
 
+### Setting Size Limits
+You can set size limits on objects, arrays, and maps to prevent resource exhaustion when decoding. This can be done by calling the setMaxLimits export. Each of the properties are optional (only provide 
+the properties you want to change), for example (with the defaults):
+```JavaScript
+import { setMaxLimits } from 'cbor-x';
+setMaxLimits({
+	maxArraySize: 112810000, 
+	maxMapSize: 16810000,
+	maxObjectSize : 16710000
+});
+```
+
 ## Performance
 Cbor-x is fast. Really fast. Here is comparison with the next fastest JS projects using the benchmark tool from `msgpack-lite` (and the sample data is from some clinical research data we use that has a good mix of different value types and structures). It also includes comparison to V8 native JSON functionality, and JavaScript Avro (`avsc`, a very optimized Avro implementation):
 

package/decode.d.ts

@@ -1,2 +1,2 @@
 export { decode, decodeMultiple, Decoder, addExtension, clearSource,roundFloat32, isNativeAccelerationEnabled,
-	Extension, Options, FLOAT32_OPTIONS} from '.'
+	Extension, Options, FLOAT32_OPTIONS, setMaxLimits, MAX_LIMITS_OPTIONS } from '.'

package/decode.js

@@ -14,6 +14,13 @@ const BUNDLED_STRINGS_ID = 0xdff9
 const PACKED_TABLE_TAG_ID = 51
 const PACKED_REFERENCE_TAG_ID = 6
 const STOP_CODE = {}
+let maxArraySize = 112810000 // This is the maximum array size in V8. We would potentially detect and set it higher
+// for JSC, but this is pretty large and should be sufficient for most use cases
+let maxMapSize = 16810000 // JavaScript has a fixed maximum map size of about 16710000, but JS itself enforces this,
+// so we don't need to
+
+let maxObjectSize = 16710000; // This is the maximum number of keys in a Map. It takes over a minute to create this
+// many keys in an object, so also probably a reasonable choice there.
 let strings = EMPTY_ARRAY
 let stringPosition = 0
 let currentDecoder = {}
@@ -294,6 +301,7 @@ export function read() {
 						let array = []
 						let value, i = 0
 						while ((value = read()) != STOP_CODE) {
+							if (i >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 							array[i++] = value
 						}
 						return majorType == 4 ? array : majorType == 3 ? array.join('') : Buffer.concat(array)
@@ -301,8 +309,19 @@ export function read() {
 						let key
 						if (currentDecoder.mapsAsObjects) {
 							let object = {}
-							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) object[safeKey(currentDecoder.decodeKey(key))] = read()
-							else while ((key = read()) != STOP_CODE) object[safeKey(key)] = read()
+							let i = 0;
+							if (currentDecoder.keyMap) {
+								while((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+									object[safeKey(currentDecoder.decodeKey(key))] = read()
+								}
+							}
+							else {
+								while ((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+									object[safeKey(key)] = read()
+								}
+							}
 							return object
 						} else {
 							if (restoreMapsAsObject) {
@@ -310,8 +329,24 @@ export function read() {
 								restoreMapsAsObject = false
 							}
 							let map = new Map()
-							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) map.set(currentDecoder.decodeKey(key), read())
-							else while ((key = read()) != STOP_CODE) map.set(key, read())
+							if (currentDecoder.keyMap) {
+								let i = 0;
+								while((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) {
+										throw new Error(`Map size exceeds ${maxMapSize}`);
+									}
+									map.set(currentDecoder.decodeKey(key), read())
+								}
+							}
+							else {
+								let i = 0;
+								while ((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) {
+										throw new Error(`Map size exceeds ${maxMapSize}`);
+									}
+									map.set(key, read())
+								}
+							}
 							return map
 						}
 					case 7:
@@ -342,12 +377,14 @@ export function read() {
 			}
 			return readFixedString(token)
 		case 4: // array
+			if (token >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 			let array = new Array(token)
 		  //if (currentDecoder.keyMap) for (let i = 0; i < token; i++) array[i] = currentDecoder.decodeKey(read())	
 			//else 
 			for (let i = 0; i < token; i++) array[i] = read()
 			return array
 		case 5: // map
+			if (token >= maxMapSize) throw new Error(`Map size exceeds ${maxArraySize}`)
 			if (currentDecoder.mapsAsObjects) {
 				let object = {}
 				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[safeKey(currentDecoder.decodeKey(read()))] = read()
@@ -449,6 +486,7 @@ export function read() {
 }
 const validName = /^[a-zA-Z_$][a-zA-Z\d_$]*$/
 function createStructureReader(structure) {
+	if (!structure) throw new Error('Structure is required in record definition');
 	function readObject() {
 		// get the array size from the header
 		let length = src[position++]
@@ -504,7 +542,8 @@ function createStructureReader(structure) {
 function safeKey(key) {
 	// protect against prototype pollution
 	if (typeof key === 'string') return key === '__proto__' ? '__proto_' : key
-	if (typeof key !== 'object') return key.toString()
+	if (typeof key === 'number' || typeof key === 'boolean' || typeof key === 'bigint') return key.toString();
+	if (key == null) return key + '';
 	// protect against expensive (DoS) string conversions
 	throw new Error('Invalid property name type ' + typeof key);
 }
@@ -885,7 +924,7 @@ currentExtensions[2] = (buffer) => {
 	// bigint extension
 	let value = BigInt(0)
 	for (let i = 0, l = buffer.byteLength; i < l; i++) {
-		value = BigInt(buffer[i]) + value << BigInt(8)
+		value = BigInt(buffer[i]) + (value << BigInt(8))
 	}
 	return value
 }
@@ -1000,6 +1039,7 @@ currentExtensions[28] = (read) => {
 		referenceMap.id = 0
 	}
 	let id = referenceMap.id++
+	let startingPosition = position
 	let token = src[position]
 	let target
 	// TODO: handle Maps, Sets, and other types that can cycle; this is complicated, because you potentially need to read
@@ -1012,8 +1052,20 @@ currentExtensions[28] = (read) => {
 	let refEntry = { target } // a placeholder object
 	referenceMap.set(id, refEntry)
 	let targetProperties = read() // read the next value as the target object to id
-	if (refEntry.used) // there is a cycle, so we have to assign properties to original target
+	if (refEntry.used) {// there is a cycle, so we have to assign properties to original target
+		if (Object.getPrototypeOf(target) !== Object.getPrototypeOf(targetProperties)) {
+			// this means that the returned target does not match the targetProperties, so we need rerun the read to
+			// have the correctly create instance be assigned as a reference, then we do the copy the properties back to the
+			// target
+			// reset the position so that the read can be repeated
+			position = startingPosition
+			// the returned instance is our new target for references
+			target = targetProperties
+			referenceMap.set(id, { target })
+			targetProperties = read()
+		}
 		return Object.assign(target, targetProperties)
+	}
 	refEntry.target = targetProperties // the placeholder wasn't used, replace with the deserialized one
 	return targetProperties // no cycle, can just use the returned read object
 }
@@ -1095,7 +1147,7 @@ function registerTypedArray(TypedArray, tag) {
 	for (let littleEndian = 0; littleEndian < 2; littleEndian++) {
 		if (!littleEndian && bytesPerElement == 1)
 			continue
-		let sizeShift = bytesPerElement == 2 ? 1 : bytesPerElement == 4 ? 2 : 3
+		let sizeShift = bytesPerElement == 2 ? 1 : bytesPerElement == 4 ? 2 : bytesPerElement == 8 ? 3 : 0
 		currentExtensions[littleEndian ? tag : (tag - 4)] = (bytesPerElement == 1 || littleEndian == isLittleEndianMachine) ? (buffer) => {
 			if (!TypedArray)
 				throw new Error('Could not find typed array for code ' + tag)
@@ -1105,7 +1157,7 @@ function registerTypedArray(TypedArray, tag) {
 					bytesPerElement === 2 && !(buffer.byteOffset & 1) ||
 					bytesPerElement === 4 && !(buffer.byteOffset & 3) ||
 					bytesPerElement === 8 && !(buffer.byteOffset & 7))
-					return new TypedArray(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+					return new TypedArray(buffer.buffer, buffer.byteOffset, buffer.byteLength >> sizeShift);
 			}
 			// we have to slice/copy here to get a new ArrayBuffer, if we are not word/byte aligned
 			return new TypedArray(Uint8Array.prototype.slice.call(buffer, 0).buffer)
@@ -1222,6 +1274,12 @@ export function addExtension(extension) {
 	currentExtensions[extension.tag] = extension.decode
 }
 
+export function setSizeLimits(limits) {
+	if (limits.maxMapSize) maxMapSize = limits.maxMapSize;
+	if (limits.maxArraySize) maxArraySize = limits.maxArraySize;
+	if (limits.maxObjectSize) maxObjectSize = limits.maxObjectSize;
+}
+
 export const mult10 = new Array(147) // this is a table matching binary exponents to the multiplier to determine significant digit rounding
 for (let i = 0; i < 256; i++) {
 	mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103))

package/dist/decode-no-eval.cjs

@@ -18,6 +18,10 @@
 	const BUNDLED_STRINGS_ID = 0xdff9;
 	const PACKED_REFERENCE_TAG_ID = 6;
 	const STOP_CODE = {};
+	let maxArraySize = 112810000; // This is the maximum array size in V8. We would potentially detect and set it higher
+	// for JSC, but this is pretty large and should be sufficient for most use cases
+	let maxMapSize = 16810000; // JavaScript has a fixed maximum map size of about 16710000, but JS itself enforces this,
+	// many keys in an object, so also probably a reasonable choice there.
 	let strings = EMPTY_ARRAY;
 	let stringPosition = 0;
 	let currentDecoder = {};
@@ -298,6 +302,7 @@
 							let array = [];
 							let value, i = 0;
 							while ((value = read()) != STOP_CODE) {
+								if (i >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 								array[i++] = value;
 							}
 							return majorType == 4 ? array : majorType == 3 ? array.join('') : Buffer.concat(array)
@@ -305,8 +310,19 @@
 							let key;
 							if (currentDecoder.mapsAsObjects) {
 								let object = {};
-								if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) object[safeKey(currentDecoder.decodeKey(key))] = read();
-								else while ((key = read()) != STOP_CODE) object[safeKey(key)] = read();
+								let i = 0;
+								if (currentDecoder.keyMap) {
+									while((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+										object[safeKey(currentDecoder.decodeKey(key))] = read();
+									}
+								}
+								else {
+									while ((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+										object[safeKey(key)] = read();
+									}
+								}
 								return object
 							} else {
 								if (restoreMapsAsObject) {
@@ -314,8 +330,24 @@
 									restoreMapsAsObject = false;
 								}
 								let map = new Map();
-								if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) map.set(currentDecoder.decodeKey(key), read());
-								else while ((key = read()) != STOP_CODE) map.set(key, read());
+								if (currentDecoder.keyMap) {
+									let i = 0;
+									while((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) {
+											throw new Error(`Map size exceeds ${maxMapSize}`);
+										}
+										map.set(currentDecoder.decodeKey(key), read());
+									}
+								}
+								else {
+									let i = 0;
+									while ((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) {
+											throw new Error(`Map size exceeds ${maxMapSize}`);
+										}
+										map.set(key, read());
+									}
+								}
 								return map
 							}
 						case 7:
@@ -346,12 +378,14 @@
 				}
 				return readFixedString(token)
 			case 4: // array
+				if (token >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 				let array = new Array(token);
 			  //if (currentDecoder.keyMap) for (let i = 0; i < token; i++) array[i] = currentDecoder.decodeKey(read())	
 				//else 
 				for (let i = 0; i < token; i++) array[i] = read();
 				return array
 			case 5: // map
+				if (token >= maxMapSize) throw new Error(`Map size exceeds ${maxArraySize}`)
 				if (currentDecoder.mapsAsObjects) {
 					let object = {};
 					if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[safeKey(currentDecoder.decodeKey(read()))] = read();
@@ -453,6 +487,7 @@
 	}
 	const validName = /^[a-zA-Z_$][a-zA-Z\d_$]*$/;
 	function createStructureReader(structure) {
+		if (!structure) throw new Error('Structure is required in record definition');
 		function readObject() {
 			// get the array size from the header
 			let length = src[position++];
@@ -508,7 +543,8 @@
 	function safeKey(key) {
 		// protect against prototype pollution
 		if (typeof key === 'string') return key === '__proto__' ? '__proto_' : key
-		if (typeof key !== 'object') return key.toString()
+		if (typeof key === 'number' || typeof key === 'boolean' || typeof key === 'bigint') return key.toString();
+		if (key == null) return key + '';
 		// protect against expensive (DoS) string conversions
 		throw new Error('Invalid property name type ' + typeof key);
 	}
@@ -813,7 +849,7 @@
 		// bigint extension
 		let value = BigInt(0);
 		for (let i = 0, l = buffer.byteLength; i < l; i++) {
-			value = BigInt(buffer[i]) + value << BigInt(8);
+			value = BigInt(buffer[i]) + (value << BigInt(8));
 		}
 		return value
 	};
@@ -928,6 +964,7 @@
 			referenceMap.id = 0;
 		}
 		let id = referenceMap.id++;
+		let startingPosition = position;
 		let token = src[position];
 		let target;
 		// TODO: handle Maps, Sets, and other types that can cycle; this is complicated, because you potentially need to read
@@ -940,8 +977,20 @@
 		let refEntry = { target }; // a placeholder object
 		referenceMap.set(id, refEntry);
 		let targetProperties = read(); // read the next value as the target object to id
-		if (refEntry.used) // there is a cycle, so we have to assign properties to original target
+		if (refEntry.used) {// there is a cycle, so we have to assign properties to original target
+			if (Object.getPrototypeOf(target) !== Object.getPrototypeOf(targetProperties)) {
+				// this means that the returned target does not match the targetProperties, so we need rerun the read to
+				// have the correctly create instance be assigned as a reference, then we do the copy the properties back to the
+				// target
+				// reset the position so that the read can be repeated
+				position = startingPosition;
+				// the returned instance is our new target for references
+				target = targetProperties;
+				referenceMap.set(id, { target });
+				targetProperties = read();
+			}
 			return Object.assign(target, targetProperties)
+		}
 		refEntry.target = targetProperties; // the placeholder wasn't used, replace with the deserialized one
 		return targetProperties // no cycle, can just use the returned read object
 	};
@@ -1023,7 +1072,7 @@
 		for (let littleEndian = 0; littleEndian < 2; littleEndian++) {
 			if (!littleEndian && bytesPerElement == 1)
 				continue
-			let sizeShift = bytesPerElement == 2 ? 1 : bytesPerElement == 4 ? 2 : 3;
+			let sizeShift = bytesPerElement == 2 ? 1 : bytesPerElement == 4 ? 2 : bytesPerElement == 8 ? 3 : 0;
 			currentExtensions[littleEndian ? tag : (tag - 4)] = (bytesPerElement == 1 || littleEndian == isLittleEndianMachine) ? (buffer) => {
 				if (!TypedArray)
 					throw new Error('Could not find typed array for code ' + tag)
@@ -1033,7 +1082,7 @@
 						bytesPerElement === 2 && !(buffer.byteOffset & 1) ||
 						bytesPerElement === 4 && !(buffer.byteOffset & 3) ||
 						bytesPerElement === 8 && !(buffer.byteOffset & 7))
-						return new TypedArray(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+						return new TypedArray(buffer.buffer, buffer.byteOffset, buffer.byteLength >> sizeShift);
 				}
 				// we have to slice/copy here to get a new ArrayBuffer, if we are not word/byte aligned
 				return new TypedArray(Uint8Array.prototype.slice.call(buffer, 0).buffer)
@@ -1150,6 +1199,12 @@
 		currentExtensions[extension.tag] = extension.decode;
 	}
 
+	function setSizeLimits(limits) {
+		if (limits.maxMapSize) maxMapSize = limits.maxMapSize;
+		if (limits.maxArraySize) maxArraySize = limits.maxArraySize;
+		if (limits.maxObjectSize) limits.maxObjectSize;
+	}
+
 	const mult10 = new Array(147); // this is a table matching binary exponents to the multiplier to determine significant digit rounding
 	for (let i = 0; i < 256; i++) {
 		mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103));
@@ -1182,6 +1237,7 @@
 	exports.read = read;
 	exports.roundFloat32 = roundFloat32;
 	exports.setExtractor = setExtractor;
+	exports.setSizeLimits = setSizeLimits;
 	exports.typedArrays = typedArrays;
 
 }));