cbor-x 1.5.9 → 1.6.3

package/README.md

@@ -29,7 +29,7 @@ This `encode` function will generate standard CBOR without any extensions that s
 Cbor-x modules are standard ESM modules and can be loaded directly from the [deno.land registry for cbor](https://deno.land/x/cbor) for use in Deno. The standard encode and decode functionality is available on Deno, like other platforms.
 
 ### Streams
-We can use the including streaming functionality (which further improves performance). The `EncoderStream` is a NodeJS transform stream that can be used to serialize objects to a binary stream (writing to network/socket, IPC, etc.), and the `DecoderStream` can be used to deserialize objects from a binary sream (reading from network/socket, etc.):
+We can use the including streaming functionality (which further improves performance). The `EncoderStream` is a NodeJS transform stream that can be used to serialize objects to a binary stream (writing to network/socket, IPC, etc.), and the `DecoderStream` can be used to deserialize objects from a binary stream (reading from network/socket, etc.):
 
 ```JavaScript
 import { EncoderStream } from 'cbor-x';
@@ -89,23 +89,22 @@ import { decode } from 'cbor-x/decode' // if you only need to decode
 ```
 
 ## Structured Cloning
-You can also use cbor-x for [structured cloning](https://html.spec.whatwg.org/multipage/structured-data.html). By enabling the `structuredClone` option, you can include references to other objects or cyclic references, and object identity will be preserved. Structured cloning also enables preserving certain typed objects like `Error`, `Set`, `RegExp` and TypedArray instances, using [registered CBOR tag extensions](https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml). For example:
+You can also use cbor-x for [structured cloning](https://html.spec.whatwg.org/multipage/structured-data.html). By enabling the `structuredClone` option, you can include references to other objects or cyclic references, and object identity will be preserved.For example:
 ```JavaScript
 let obj = {
-	set: new Set(['a', 'b']),
-	regular: /a\spattern/
 };
 obj.self = obj;
 let encoder = new Encoder({ structuredClone: true });
 let serialized = encoder.encode(obj);
 let copy = encoder.decode(serialized);
 copy.self === copy // true
-copy.set.has('a') // true
 
 ```
 
 This option is disabled by default because reference checking degrades performance (by about 25-30%). (Note this implementation doesn't serialize every class/type specified in the HTML specification since not all of them make sense for storing across platforms.)
 
+cbor-x also preserves certain typed objects like `Error`, `Set`, `RegExp` and TypedArray instances, using [registered CBOR tag extensions](https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml). This works with or without structured cloning enabled.
+
 ## Record / Object Structures
 There is a critical difference between maps (or dictionaries) that hold an arbitrary set of keys and values (JavaScript `Map` is designed for these), and records or object structures that have a well-defined set of fields. Typical JS objects/records may have many instances re(use) the same structure. By using the record extension, this distinction is preserved in CBOR and the encoding can reuse structures and not only provides better type preservation, but yield much more compact encodings and increase decoding performance by 2-3x. Cbor-x automatically generates record definitions that are reused and referenced by objects with the same structure. Records use CBOR's tags to align well CBOR's tag/extension mechanism. There are a number of ways to use this to our advantage. For large object structures with repeating nested objects with similar structures, simply serializing with the record extension can yield significant benefits. To use the record structures extension, we create a new `Encoder` instance. By default a new `Encoder` instance will have the record extension enabled:
 ```JavaScript
@@ -166,7 +165,7 @@ The principle driver for this feature is to support `application/senml+cbor`cont
 Records are also supported in conjunction with keyMaps, but these are disabled by default when keyMaps are specified as use of the two features does not introduce any additional compression efficiency unless that the data arrays are quite large (> 10 items).
 
 ```JavaScript
-import { Encoder } from 'cbor-x'
+import { Decoder, Encoder } from 'cbor-x'
 const data = [ 
 	{ bn: '/3303/0/5700', bt: 1278887, v: 35.5 },
 	{ t: 10, v: 34 },
@@ -183,7 +182,8 @@ let senmlBuff = senmlCbor.encode(data)
 let basicBuff = basicCbor.encode(data)
 console.log('Senml CBOR size:', senmlBuff.length) // 77
 console.log('Basic CBOR size:', basicBuff.length) // 90
-assert.deepEqual(senmlEncoder.decode(senmlBuff), data)
+let senmlDecoder = new Decoder({ keyMap: senmlKeys }); 
+assert.deepEqual(senmlDecoder.decode(senmlBuff), data)
 
 ```
 
@@ -213,6 +213,7 @@ The following options properties can be provided to the Encoder or Decoder const
 * `useTag259ForMaps` - This flag indicates if [tag 259 (explicit maps)](https://github.com/shanewholloway/js-cbor-codec/blob/master/docs/CBOR-259-spec--explicit-maps.md) should be used to encode JS `Map`s. When using records is enabled, this is disabled by default, since plain objects are encoded with record structures and unambigiously differentiated from `Map`s, which are encoded as CBOR maps. Without using records, this enabled by default and is necessary to distinguish plain objects from `Map`s (but can be disabled by setting this to `false`).
 * `tagUint8Array` - Indicates if tag 64 should be used for `Uint8Array`s. 
 * `int64AsNumber` - This will decode uint64 and int64 numbers as standard JS numbers rather than as bigint numbers.
+* `skipFunction`  - This skip functions in encode object.
 
 ### 32-bit Float Options
 By default all non-integer numbers are serialized as 64-bit float (double). This is fast, and ensures maximum precision. However, often real-world data doesn't not need 64-bits of precision, and using 32-bit encoding can be much more space efficient. There are several options that provide more efficient encodings. Using the decimal rounding options for encoding and decoding provides lossless storage of common decimal representations like 7.99, in more efficient 32-bit format (rather than 64-bit). The `useFloat32` property has several possible options, available from the module as constants:
@@ -228,6 +229,18 @@ Note, that the performance is decreased with decimal rounding by about 20-25%, a
 
 In addition, msgpackr exports a `roundFloat32(number)` function that can be used to round floating point numbers to the maximum significant decimal digits that can be stored in 32-bit float, just as DECIMAL_ROUND does when decoding. This can be useful for determining how a number will be decoded prior to encoding it.
 
+### Setting Size Limits
+You can set size limits on objects, arrays, and maps to prevent resource exhaustion when decoding. This can be done by calling the setMaxLimits export. Each of the properties are optional (only provide 
+the properties you want to change), for example (with the defaults):
+```JavaScript
+import { setMaxLimits } from 'cbor-x';
+setMaxLimits({
+	maxArraySize: 112810000, 
+	maxMapSize: 16810000,
+	maxObjectSize : 16710000
+});
+```
+
 ## Performance
 Cbor-x is fast. Really fast. Here is comparison with the next fastest JS projects using the benchmark tool from `msgpack-lite` (and the sample data is from some clinical research data we use that has a good mix of different value types and structures). It also includes comparison to V8 native JSON functionality, and JavaScript Avro (`avsc`, a very optimized Avro implementation):
 

package/decode.d.ts

@@ -1,2 +1,2 @@
 export { decode, decodeMultiple, Decoder, addExtension, clearSource,roundFloat32, isNativeAccelerationEnabled,
-	Extension, Options, FLOAT32_OPTIONS} from '.'
+	Extension, Options, FLOAT32_OPTIONS, setMaxLimits, MAX_LIMITS_OPTIONS } from '.'

package/decode.js

@@ -14,6 +14,13 @@ const BUNDLED_STRINGS_ID = 0xdff9
 const PACKED_TABLE_TAG_ID = 51
 const PACKED_REFERENCE_TAG_ID = 6
 const STOP_CODE = {}
+let maxArraySize = 112810000 // This is the maximum array size in V8. We would potentially detect and set it higher
+// for JSC, but this is pretty large and should be sufficient for most use cases
+let maxMapSize = 16810000 // JavaScript has a fixed maximum map size of about 16710000, but JS itself enforces this,
+// so we don't need to
+
+let maxObjectSize = 16710000; // This is the maximum number of keys in a Map. It takes over a minute to create this
+// many keys in an object, so also probably a reasonable choice there.
 let strings = EMPTY_ARRAY
 let stringPosition = 0
 let currentDecoder = {}
@@ -266,6 +273,7 @@ export function read() {
 				}
 				token = dataView.getUint32(position)
 				position += 4
+				if (majorType === 1) return -1 - token; // can't safely use negation operator here
 				break
 			case 0x1b:
 				if (majorType == 7) {
@@ -280,8 +288,7 @@ export function read() {
 				} else if (currentDecoder.int64AsNumber) {
 					token = dataView.getUint32(position) * 0x100000000
 					token += dataView.getUint32(position + 4)
-				} else
-					token = dataView.getBigUint64(position)
+				} else token = dataView.getBigUint64(position)
 				position += 8
 				break
 			case 0x1f: 
@@ -294,6 +301,7 @@ export function read() {
 						let array = []
 						let value, i = 0
 						while ((value = read()) != STOP_CODE) {
+							if (i >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 							array[i++] = value
 						}
 						return majorType == 4 ? array : majorType == 3 ? array.join('') : Buffer.concat(array)
@@ -301,8 +309,19 @@ export function read() {
 						let key
 						if (currentDecoder.mapsAsObjects) {
 							let object = {}
-							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) object[safeKey(currentDecoder.decodeKey(key))] = read()
-							else while ((key = read()) != STOP_CODE) object[safeKey(key)] = read()
+							let i = 0;
+							if (currentDecoder.keyMap) {
+								while((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+									object[safeKey(currentDecoder.decodeKey(key))] = read()
+								}
+							}
+							else {
+								while ((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+									object[safeKey(key)] = read()
+								}
+							}
 							return object
 						} else {
 							if (restoreMapsAsObject) {
@@ -310,8 +329,24 @@ export function read() {
 								restoreMapsAsObject = false
 							}
 							let map = new Map()
-							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) map.set(currentDecoder.decodeKey(key), read())
-							else while ((key = read()) != STOP_CODE) map.set(key, read())
+							if (currentDecoder.keyMap) {
+								let i = 0;
+								while((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) {
+										throw new Error(`Map size exceeds ${maxMapSize}`);
+									}
+									map.set(currentDecoder.decodeKey(key), read())
+								}
+							}
+							else {
+								let i = 0;
+								while ((key = read()) != STOP_CODE) {
+									if (i++ >= maxMapSize) {
+										throw new Error(`Map size exceeds ${maxMapSize}`);
+									}
+									map.set(key, read())
+								}
+							}
 							return map
 						}
 					case 7:
@@ -342,12 +377,14 @@ export function read() {
 			}
 			return readFixedString(token)
 		case 4: // array
+			if (token >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 			let array = new Array(token)
 		  //if (currentDecoder.keyMap) for (let i = 0; i < token; i++) array[i] = currentDecoder.decodeKey(read())	
 			//else 
 			for (let i = 0; i < token; i++) array[i] = read()
 			return array
 		case 5: // map
+			if (token >= maxMapSize) throw new Error(`Map size exceeds ${maxArraySize}`)
 			if (currentDecoder.mapsAsObjects) {
 				let object = {}
 				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[safeKey(currentDecoder.decodeKey(read()))] = read()
@@ -449,6 +486,7 @@ export function read() {
 }
 const validName = /^[a-zA-Z_$][a-zA-Z\d_$]*$/
 function createStructureReader(structure) {
+	if (!structure) throw new Error('Structure is required in record definition');
 	function readObject() {
 		// get the array size from the header
 		let length = src[position++]
@@ -886,7 +924,7 @@ currentExtensions[2] = (buffer) => {
 	// bigint extension
 	let value = BigInt(0)
 	for (let i = 0, l = buffer.byteLength; i < l; i++) {
-		value = BigInt(buffer[i]) + value << BigInt(8)
+		value = BigInt(buffer[i]) + (value << BigInt(8))
 	}
 	return value
 }
@@ -1236,6 +1274,12 @@ export function addExtension(extension) {
 	currentExtensions[extension.tag] = extension.decode
 }
 
+export function setSizeLimits(limits) {
+	if (limits.maxMapSize) maxMapSize = limits.maxMapSize;
+	if (limits.maxArraySize) maxArraySize = limits.maxArraySize;
+	if (limits.maxObjectSize) maxObjectSize = limits.maxObjectSize;
+}
+
 export const mult10 = new Array(147) // this is a table matching binary exponents to the multiplier to determine significant digit rounding
 for (let i = 0; i < 256; i++) {
 	mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103))

package/dist/decode-no-eval.cjs

@@ -18,6 +18,10 @@
 	const BUNDLED_STRINGS_ID = 0xdff9;
 	const PACKED_REFERENCE_TAG_ID = 6;
 	const STOP_CODE = {};
+	let maxArraySize = 112810000; // This is the maximum array size in V8. We would potentially detect and set it higher
+	// for JSC, but this is pretty large and should be sufficient for most use cases
+	let maxMapSize = 16810000; // JavaScript has a fixed maximum map size of about 16710000, but JS itself enforces this,
+	// many keys in an object, so also probably a reasonable choice there.
 	let strings = EMPTY_ARRAY;
 	let stringPosition = 0;
 	let currentDecoder = {};
@@ -270,6 +274,7 @@
 					}
 					token = dataView.getUint32(position);
 					position += 4;
+					if (majorType === 1) return -1 - token; // can't safely use negation operator here
 					break
 				case 0x1b:
 					if (majorType == 7) {
@@ -284,8 +289,7 @@
 					} else if (currentDecoder.int64AsNumber) {
 						token = dataView.getUint32(position) * 0x100000000;
 						token += dataView.getUint32(position + 4);
-					} else
-						token = dataView.getBigUint64(position);
+					} else token = dataView.getBigUint64(position);
 					position += 8;
 					break
 				case 0x1f: 
@@ -298,6 +302,7 @@
 							let array = [];
 							let value, i = 0;
 							while ((value = read()) != STOP_CODE) {
+								if (i >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 								array[i++] = value;
 							}
 							return majorType == 4 ? array : majorType == 3 ? array.join('') : Buffer.concat(array)
@@ -305,8 +310,19 @@
 							let key;
 							if (currentDecoder.mapsAsObjects) {
 								let object = {};
-								if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) object[safeKey(currentDecoder.decodeKey(key))] = read();
-								else while ((key = read()) != STOP_CODE) object[safeKey(key)] = read();
+								let i = 0;
+								if (currentDecoder.keyMap) {
+									while((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+										object[safeKey(currentDecoder.decodeKey(key))] = read();
+									}
+								}
+								else {
+									while ((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) throw new Error(`Property count exceeds ${maxMapSize}`)
+										object[safeKey(key)] = read();
+									}
+								}
 								return object
 							} else {
 								if (restoreMapsAsObject) {
@@ -314,8 +330,24 @@
 									restoreMapsAsObject = false;
 								}
 								let map = new Map();
-								if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) map.set(currentDecoder.decodeKey(key), read());
-								else while ((key = read()) != STOP_CODE) map.set(key, read());
+								if (currentDecoder.keyMap) {
+									let i = 0;
+									while((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) {
+											throw new Error(`Map size exceeds ${maxMapSize}`);
+										}
+										map.set(currentDecoder.decodeKey(key), read());
+									}
+								}
+								else {
+									let i = 0;
+									while ((key = read()) != STOP_CODE) {
+										if (i++ >= maxMapSize) {
+											throw new Error(`Map size exceeds ${maxMapSize}`);
+										}
+										map.set(key, read());
+									}
+								}
 								return map
 							}
 						case 7:
@@ -346,12 +378,14 @@
 				}
 				return readFixedString(token)
 			case 4: // array
+				if (token >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 				let array = new Array(token);
 			  //if (currentDecoder.keyMap) for (let i = 0; i < token; i++) array[i] = currentDecoder.decodeKey(read())	
 				//else 
 				for (let i = 0; i < token; i++) array[i] = read();
 				return array
 			case 5: // map
+				if (token >= maxMapSize) throw new Error(`Map size exceeds ${maxArraySize}`)
 				if (currentDecoder.mapsAsObjects) {
 					let object = {};
 					if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[safeKey(currentDecoder.decodeKey(read()))] = read();
@@ -453,6 +487,7 @@
 	}
 	const validName = /^[a-zA-Z_$][a-zA-Z\d_$]*$/;
 	function createStructureReader(structure) {
+		if (!structure) throw new Error('Structure is required in record definition');
 		function readObject() {
 			// get the array size from the header
 			let length = src[position++];
@@ -814,7 +849,7 @@
 		// bigint extension
 		let value = BigInt(0);
 		for (let i = 0, l = buffer.byteLength; i < l; i++) {
-			value = BigInt(buffer[i]) + value << BigInt(8);
+			value = BigInt(buffer[i]) + (value << BigInt(8));
 		}
 		return value
 	};
@@ -1164,6 +1199,12 @@
 		currentExtensions[extension.tag] = extension.decode;
 	}
 
+	function setSizeLimits(limits) {
+		if (limits.maxMapSize) maxMapSize = limits.maxMapSize;
+		if (limits.maxArraySize) maxArraySize = limits.maxArraySize;
+		if (limits.maxObjectSize) limits.maxObjectSize;
+	}
+
 	const mult10 = new Array(147); // this is a table matching binary exponents to the multiplier to determine significant digit rounding
 	for (let i = 0; i < 256; i++) {
 		mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103));
@@ -1196,6 +1237,7 @@
 	exports.read = read;
 	exports.roundFloat32 = roundFloat32;
 	exports.setExtractor = setExtractor;
+	exports.setSizeLimits = setSizeLimits;
 	exports.typedArrays = typedArrays;
 
 }));