cbor-x 1.4.0 → 1.5.0

package/README.md

@@ -52,6 +52,26 @@ receivingStream.on('data', (data) => {
 ```
 The `EncoderStream` and `DecoderStream` instances  will have also the record structure extension enabled by default (see below).
 
+### Iterables
+In addition to using CBOR with streams, CBOR can also encode to an iterable that can be iterated as a sequence of binary chunks with `encodeAsIterable`, which facilitates progressive encoding:
+```JavaScript
+import { encodeAsIterable } from 'cbor-x';
+
+for (let binaryChunk of encodeAsIterable(data)){
+	// progressively get binary chunks as data is encoded
+}
+```
+And `encodeAsAsyncIterable` is also available, which returns an async iterable, and can be used to encode data from async iterables as well as Blob data.
+
+```JavaScript
+import { encodeAsAsyncIterable } from 'cbor-x';
+
+let data = { blob: new Blob(...) };
+for await (let binaryChunk of encodeAsAsyncIterable(data)){
+	// progressively get binary chunks as asynchronous data source is encoded
+}
+```
+
 ## Deno Usage
 Cbor-x modules are standard ESM modules and can be loaded directly from the [deno.land registry for cbor](https://deno.land/x/cbor) for use in Deno. The standard pack/encode and unpack/decode functionality is available on Deno, like other platforms.
 
@@ -168,7 +188,7 @@ assert.deepEqual(senmlEncoder.decode(senmlBuff), data)
 ```
 
 ### CBOR Packing
-[Packed CBOR](https://datatracker.ietf.org/doc/html/draft-ietf-cbor-packed) is additional specification for CBOR which allows for compact encoding of data that has repeated values. Cbor-x supports decoding packed CBOR, no or flags/options needed. Cbor-x can also optionally generate packed CBOR (with the `pack` option), which will cause the encoder to look for repeated strings in a data structure that is being encoded, and store the strings in a packed table that can be referenced, to reduce encoding size. This involves extra overhead and reduces encoding performance, and generally does not yield as much compaction as standard compression tools. However, this is can be much faster than encoding plus compression, while still providing some level of reduction in encoding size. In addition to size reduction, packed CBOR is also usually faster to decode (assuming that some repetitive values could be found/packed).
+[Packed CBOR](https://datatracker.ietf.org/doc/html/draft-ietf-cbor-packed) is additional specification for CBOR which allows for compact encoding of data that has repeated values. Cbor-x supports decoding packed CBOR, no flags or options needed. Cbor-x can also optionally generate packed CBOR (with the `pack` option), which will cause the encoder to look for repeated strings in a data structure that is being encoded, and store the strings in a packed table that can be referenced, to reduce encoding size. This involves extra overhead and reduces encoding performance, and generally does not yield as much compaction as standard compression tools. However, this is can be much faster than encoding plus compression, while still providing some level of reduction in encoding size. In addition to size reduction, packed CBOR is also usually faster to decode (assuming that some repetitive values could be found/packed).
 
 Cbor-x also has in-progress effort to support shared packed tables.
 

package/decode.js

@@ -84,7 +84,7 @@ export class Decoder {
 		}
 		let res = {}
 		//map.forEach((v,k) => res[Object.keys(this._keyMap)[Object.values(this._keyMap).indexOf(k)] || k] = v)
-		map.forEach((v,k) => res[this._mapKey.has(k) ? this._mapKey.get(k) : k] =  v)
+		map.forEach((v,k) => res[safeKey(this._mapKey.has(k) ? this._mapKey.get(k) : k)] =  v)
 		return res
 	}
 	
@@ -197,6 +197,7 @@ export function checkedRead() {
 			}
 			// bundled strings to skip past
 			position = bundledStrings.postBundlePosition;
+			bundledStrings = null;
 		}
 
 		if (position == srcEnd) {
@@ -289,8 +290,8 @@ export function read() {
 						let key
 						if (currentDecoder.mapsAsObjects) {
 							let object = {}
-							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) object[currentDecoder.decodeKey(key)] = read()
-							else while ((key = read()) != STOP_CODE) object[key] = read()
+							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) object[safeKey(currentDecoder.decodeKey(key))] = read()
+							else while ((key = read()) != STOP_CODE) object[safeKey(key)] = read()
 							return object
 						} else {
 							if (restoreMapsAsObject) {
@@ -338,8 +339,8 @@ export function read() {
 		case 5: // map
 			if (currentDecoder.mapsAsObjects) {
 				let object = {}
-				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[currentDecoder.decodeKey(read())] = read()
-				else for (let i = 0; i < token; i++) object[read()] = read()
+				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[safeKey(currentDecoder.decodeKey(read()))] = read()
+				else for (let i = 0; i < token; i++) object[safeKey(read())] = read()
 				return object
 			} else {
 				if (restoreMapsAsObject) {
@@ -455,8 +456,8 @@ function createStructureReader(structure) {
 		if (this.slowReads++ >= 3) { // create a fast compiled reader
 			let array = this.length == length ? this : this.slice(0, length)
 			compiledReader = currentDecoder.keyMap 
-			? new Function('r', 'return {' + array.map(k => currentDecoder.decodeKey(k)).map(k => validName.test(k) ? k + ':r()' : ('[' + JSON.stringify(k) + ']:r()')).join(',') + '}')
-			: new Function('r', 'return {' + array.map(key => validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '}')
+			? new Function('r', 'return {' + array.map(k => currentDecoder.decodeKey(k)).map(k => validName.test(k) ? safeKey(k) + ':r()' : ('[' + JSON.stringify(k) + ']:r()')).join(',') + '}')
+			: new Function('r', 'return {' + array.map(key => validName.test(key) ? safeKey(key) + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '}')
 			if (this.compiledReader)
 				compiledReader.next = this.compiledReader // if there is an existing one, we store multiple readers as a linked list because it is usually pretty rare to have multiple readers (of different length) for the same structure
 			compiledReader.propertyCount = length
@@ -464,14 +465,20 @@ function createStructureReader(structure) {
 			return compiledReader(read)
 		}
 		let object = {}
-		if (currentDecoder.keyMap) for (let i = 0; i < length; i++) object[currentDecoder.decodeKey(this[i])] = read()
-		else for (let i = 0; i < length; i++) object[this[i]] = read()
+		if (currentDecoder.keyMap) for (let i = 0; i < length; i++) object[safeKey(currentDecoder.decodeKey(this[i]))] = read()
+		else for (let i = 0; i < length; i++) {
+			object[safeKey(this[i])] = read();
+		}
 		return object
 	}
 	structure.slowReads = 0
 	return readObject
 }
 
+function safeKey(key) {
+	return key === '__proto__' ? '__proto_' : key
+}
+
 let readFixedString = readStringJS
 let readString8 = readStringJS
 let readString16 = readStringJS
@@ -737,18 +744,30 @@ function readExt(length) {
 	else
 		throw new Error('Unknown extension type ' + type)
 }
-
+let f32Array = new Float32Array(1)
+let u8Array = new Uint8Array(f32Array.buffer, 0, 4)
 function getFloat16() {
 	let byte0 = src[position++]
 	let byte1 = src[position++]
-	let half = (byte0 << 8) + byte1
-	let exp = (half >> 10) & 0x1f
-	let mant = half & 0x3ff
-	let val
-	if (exp == 0) val = Math.exp(mant, -24)
-	else if (exp != 31) val = Math.exp(mant + 1024, exp - 25)
-	else val = mant == 0 ? Infinity : NaN
-	return half & 0x8000 ? -val : val
+	let exponent = (byte0 & 0x7f) >> 2;
+	if (exponent === 0x1f) { // specials
+		if (byte1 || (byte0 & 3))
+			return NaN;
+		return (byte0 & 0x80) ? -Infinity : Infinity;
+	}
+	if (exponent === 0) { // sub-normals
+		// significand with 10 fractional bits and divided by 2^14
+		let abs = (((byte0 & 3) << 8) | byte1) / (1 << 24)
+		return (byte0 & 0x80) ? -abs : abs
+	}
+
+	u8Array[3] = (byte0 & 0x80) | // sign bit
+		((exponent >> 1) + 56) // 4 of 5 of the exponent bits, re-offset-ed
+	u8Array[2] = ((byte0 & 7) << 5) | // last exponent bit and first two mantissa bits
+		(byte1 >> 3) // next 5 bits of mantissa
+	u8Array[1] = byte1 << 5; // last three bits of mantissa
+	u8Array[0] = 0;
+	return f32Array[0];
 }
 
 let keyCache = new Array(4096)
@@ -869,11 +888,11 @@ const recordDefinition = (definition) => {
 	let object = {}
 	if (currentDecoder.keyMap) for (let i = 2,l = definition.length; i < l; i++) {
 			let key = currentDecoder.decodeKey(structure[i - 2])
-			object[key] = definition[i]
+			object[safeKey(key)] = definition[i]
 		}
 	else for (let i = 2,l = definition.length; i < l; i++) {
 			let key = structure[i - 2]
-			object[key] = definition[i]
+			object[safeKey(key)] = definition[i]
 		}
 	return object
 }
@@ -915,18 +934,23 @@ currentExtensions[PACKED_REFERENCE_TAG_ID] = (data) => { // packed reference
 		return packedValues[16 + (data >= 0 ? 2 * data : (-2 * data - 1))]
 	throw new Error('No support for non-integer packed references yet')
 }
-currentExtensions[25] = (id) => {
-	return stringRefs[id]
-}
-currentExtensions[256] = (read) => {
-	stringRefs = []
-	try {
-		return read()
-	} finally {
-		stringRefs = null
-	}
-}
-currentExtensions[256].handlesRead = true
+
+// The following code is an incomplete implementation of http://cbor.schmorp.de/stringref
+// the real thing would need to implemennt more logic to populate the stringRefs table and
+// maintain a stack of stringRef "namespaces".
+//
+// currentExtensions[25] = (id) => {
+// 	return stringRefs[id]
+// }
+// currentExtensions[256] = (read) => {
+// 	stringRefs = []
+// 	try {
+// 		return read()
+// 	} finally {
+// 		stringRefs = null
+// 	}
+// }
+// currentExtensions[256].handlesRead = true
 
 currentExtensions[28] = (read) => { 
 	// shareable http://cbor.schmorp.de/value-sharing (for structured clones)
@@ -1160,8 +1184,6 @@ export const FLOAT32_OPTIONS = {
 	DECIMAL_ROUND: 3,
 	DECIMAL_FIT: 4
 }
-let f32Array = new Float32Array(1)
-let u8Array = new Uint8Array(f32Array.buffer, 0, 4)
 export function roundFloat32(float32Number) {
 	f32Array[0] = float32Number
 	let multiplier = mult10[((u8Array[3] & 0x7f) << 1) | (u8Array[2] >> 7)]