cbor-x 1.6.7 → 1.6.9

package/README.md

@@ -52,6 +52,10 @@ receivingStream.on('data', (data) => {
 ```
 The `EncoderStream` and `DecoderStream` instances  will have also the record structure extension enabled by default (see below).
 
+When a chunk ends inside a value, `DecoderStream` keeps the bytes of that value until the chunks that complete it arrive. The `maxIncompleteBufferSize` option limits how many bytes it keeps for one incomplete value (default 64 MB, `0x4000000`); past that, the stream emits a "Maximum incomplete buffer size exceeded" error. A value larger than this limit still decodes when it arrives in a single chunk, so raise the limit for streams that carry larger values. If the input ends inside a value, the stream emits an "Unexpected end of CBOR data" error with `incomplete: true`. `decodeIter` takes the same option and reports an incomplete final value the same way.
+
+Incomplete values are scanned incrementally without constructing objects, then decoded when their complete bytes arrive. This keeps decoding work linear in the size of a value split across many chunks. All built-in tags and custom extensions that consume exactly one tagged CBOR item are supported. An undocumented custom `handlesRead` decoder that consumes multiple sibling items cannot be framed on fragmented input; it throws an error naming the tag. Whole-buffer decoding of these extensions is unchanged.
+
 ### Iterables
 In addition to using CBOR with streams, CBOR can also encode to an iterable that can be iterated as a sequence of binary chunks with `encodeAsIterable`, which facilitates progressive encoding:
 ```JavaScript

package/decode.js

@@ -13,7 +13,6 @@ const RECORD_INLINE_ID = 0xdfff // temporary first-come first-serve tag // propo
 const BUNDLED_STRINGS_ID = 0xdff9
 const PACKED_TABLE_TAG_ID = 51
 const PACKED_REFERENCE_TAG_ID = 6
-const STOP_CODE = {}
 let maxArraySize = 112810000 // This is the maximum array size in V8. We would potentially detect and set it higher
 // for JSC, but this is pretty large and should be sufficient for most use cases
 let maxMapSize = 16810000 // JavaScript has a fixed maximum map size of about 16710000, but JS itself enforces this,
@@ -125,16 +124,11 @@ export class Decoder {
 			// re-entrant execution, save the state and restore it after we do this decode
 			return saveState(() => {
 				clearSource()
-				try {
-					return this ? this.decode(source, end) : Decoder.prototype.decode.call(defaultOptions, source, end)
-				} catch (error) {
-					if (error && typeof error == 'object')
-						nestedDecodeErrors.add(error)
-					throw error
-				}
+				return this ? this.decode(source, end) : Decoder.prototype.decode.call(defaultOptions, source, end)
 			})
 		}
-		srcEnd = end > -1 ? end : source.length
+		// past the view, the JS string readers would decode the missing bytes as zeros instead of failing as truncated
+		srcEnd = end > -1 && end < source.length ? end : source.length
 		position = 0
 		stringPosition = 0
 		srcStringEnd = 0
@@ -174,18 +168,28 @@ export class Decoder {
 		return checkedRead()
 	}
 	decodeMultiple(source, forEach) {
+		if (src) {
+			// Keep the complete nested sequence inside its own decoder state.
+			return saveState(() => {
+				clearSource()
+				return this ? this.decodeMultiple(source, forEach) : defaultDecoder.decodeMultiple(source, forEach)
+			})
+		}
 		let values, lastPosition = 0
 		try {
 			let size = source.length
 			sequentialMode = true
 			let value = this ? this.decode(source, size) : defaultDecoder.decode(source, size)
 			if (forEach) {
+				let end = position
 				if (forEach(value) === false) {
 					return
 				}
-				while(position < size) {
-					lastPosition = position
-					if (forEach(checkedRead()) === false) {
+				while(end < size) {
+					lastPosition = end
+					value = checkedRead()
+					end = position
+					if (forEach(value) === false) {
 						return
 					}
 				}
@@ -199,8 +203,10 @@ export class Decoder {
 				return values
 			}
 		} catch(error) {
-			error.lastPosition = lastPosition
-			error.values = values
+			if (error && (typeof error == 'object' || typeof error == 'function')) {
+				error.lastPosition = lastPosition
+				error.values = values
+			}
 			throw error
 		} finally {
 			sequentialMode = false
@@ -212,8 +218,10 @@ export function getPosition() {
 	return position
 }
 const nestedDecodeErrors = new WeakSet()
+let arrayAllocationBudget
 export function checkedRead() {
 	try {
+		arrayAllocationBudget = srcEnd - position
 		let result = read()
 		if (bundledStrings) {
 			if (position >= bundledStrings.postBundlePosition)
@@ -255,9 +263,21 @@ export function checkedRead() {
 	}
 }
 
-function endOfCBORError() {
+function readBreak() {
+	if (position < srcEnd && src[position] === 0xff) {
+		position++
+		return true
+	}
+	return false
+}
+// neededEnd, from a length check, is a position that the source must extend to before the item can decode. It is a
+// lower bound for any completion of the source: every byte read before the check was within the source, so a longer
+// source reaches the same check with the same length. Streams wait for it instead of decoding on every chunk.
+function endOfCBORError(neededEnd) {
 	let error = new Error('Unexpected end of CBOR data')
 	error.incomplete = true
+	if (neededEnd)
+		error.neededEnd = neededEnd
 	return error
 }
 
@@ -309,6 +329,9 @@ export function read() {
 				} else if (currentDecoder.int64AsNumber) {
 					token = dataView.getUint32(position) * 0x100000000
 					token += dataView.getUint32(position + 4)
+					position += 8
+					if (majorType === 1) return -1 - token
+					break
 				} else token = dataView.getBigUint64(position)
 				position += 8
 				break
@@ -320,10 +343,10 @@ export function read() {
 						throw new Error('Indefinite length not supported for byte or text strings')
 					case 4: // array
 						let array = []
-						let value, i = 0
-						while ((value = read()) != STOP_CODE) {
+						let i = 0
+						while (!readBreak()) {
 							if (i >= maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
-							array[i++] = value
+							array[i++] = read()
 						}
 						return majorType == 4 ? array : majorType == 3 ? array.join('') : Buffer.concat(array)
 					case 5: // map
@@ -332,13 +355,15 @@ export function read() {
 							let object = {}
 							let i = 0;
 							if (currentDecoder.keyMap) {
-								while((key = read()) != STOP_CODE) {
+								while (!readBreak()) {
+									key = read()
 									if (i++ >= objectSizeLimit) throw new Error(`Property count exceeds ${objectSizeLimit}`)
 									object[safeKey(currentDecoder.decodeKey(key))] = read()
 								}
 							}
 							else {
-								while ((key = read()) != STOP_CODE) {
+								while (!readBreak()) {
+									key = read()
 									if (i++ >= objectSizeLimit) throw new Error(`Property count exceeds ${objectSizeLimit}`)
 									object[safeKey(key)] = read()
 								}
@@ -352,7 +377,8 @@ export function read() {
 							let map = new Map()
 							if (currentDecoder.keyMap) {
 								let i = 0;
-								while((key = read()) != STOP_CODE) {
+								while (!readBreak()) {
+									key = read()
 									if (i++ >= maxMapSize) {
 										throw new Error(`Map size exceeds ${maxMapSize}`);
 									}
@@ -361,7 +387,8 @@ export function read() {
 							}
 							else {
 								let i = 0;
-								while ((key = read()) != STOP_CODE) {
+								while (!readBreak()) {
+									key = read()
 									if (i++ >= maxMapSize) {
 										throw new Error(`Map size exceeds ${maxMapSize}`);
 									}
@@ -371,7 +398,8 @@ export function read() {
 							return map
 						}
 					case 7:
-						return STOP_CODE
+						// readBreak consumes every break that ends an indefinite-length item
+						throw new Error('Unexpected break stop code outside of an indefinite-length item')
 					default:
 						throw new Error('Invalid major type for indefinite length ' + majorType)
 				}
@@ -385,13 +413,13 @@ export function read() {
 		case 1: // negative int
 			return ~token
 		case 2: // buffer
-			if (token > srcEnd - position) throw endOfCBORError()
+			if (token > srcEnd - position) throw endOfCBORError(position + token)
 			return readBin(token)
 		case 3: // string
 			if (srcStringEnd >= position) { // already extracted, which only covers complete strings
 				return srcString.slice(position - srcStringStart, (position += token) - srcStringStart)
 			}
-			if (token > srcEnd - position) throw endOfCBORError()
+			if (token > srcEnd - position) throw endOfCBORError(position + token)
 			if (srcStringEnd == 0 && srcEnd < 140 && token < 32) {
 				// for small blocks, avoiding the overhead of the extract call is helpful
 				let string = token < 16 ? shortStringInJS(token) : longStringInJS(token)
@@ -403,7 +431,9 @@ export function read() {
 			if (token > maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
 			// every element occupies at least one byte, so a length beyond what remains in the source can
 			// never be satisfied; check before allocating so a tiny header can not force a huge allocation
-			if (token > srcEnd - position) throw endOfCBORError()
+			// nested arrays each pass the check above yet still preallocate, so also spend from a
+			// per-decode budget of the remaining bytes; the summed lengths of any valid tree stay under it
+			if (token > srcEnd - position || (arrayAllocationBudget -= token) < 0) throw endOfCBORError(position + token)
 			let array = new Array(token)
 		  //if (currentDecoder.keyMap) for (let i = 0; i < token; i++) array[i] = currentDecoder.decodeKey(read())	
 			//else 
@@ -412,7 +442,7 @@ export function read() {
 		case 5: // map
 			if (currentDecoder.mapsAsObjects) {
 				if (token > objectSizeLimit) throw new Error(`Property count exceeds ${objectSizeLimit}`)
-				if (token > (srcEnd - position) / 2) throw endOfCBORError() // each entry needs at least two bytes
+				if (token > (srcEnd - position) / 2) throw endOfCBORError(position + 2 * token) // each entry needs at least two bytes
 				let object = {}
 				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[safeKey(currentDecoder.decodeKey(read()))] = read()
 				else for (let i = 0; i < token; i++) object[safeKey(read())] = read()
@@ -423,7 +453,7 @@ export function read() {
 					restoreMapsAsObject = false
 				}
 				if (token > maxMapSize) throw new Error(`Map size exceeds ${maxMapSize}`)
-				if (token > (srcEnd - position) / 2) throw endOfCBORError()
+				if (token > (srcEnd - position) / 2) throw endOfCBORError(position + 2 * token)
 				let map = new Map()
 				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) map.set(currentDecoder.decodeKey(read()),read())
 				else for (let i = 0; i < token; i++) map.set(read(), read())
@@ -1016,6 +1046,7 @@ currentExtensions[LEGACY_RECORD_INLINE_ID] = (data) => {
 	}
 	return object
 }
+const legacyRecordDefinition = currentExtensions[LEGACY_RECORD_INLINE_ID]
 currentExtensions[14] = (value) => {
 	if (bundledStrings)
 		return bundledStrings[0].slice(bundledStrings.position0, bundledStrings.position0 += value)
@@ -1288,6 +1319,9 @@ function loadShared() {
 }
 
 function saveState(callback) {
+	let savedFragmentEnd = fragmentEnd
+	let savedFragmentReader = fragmentReadString
+	let savedReadFixedString = readFixedString
 	let savedSrcEnd = srcEnd
 	let savedPosition = position
 	let savedStringPosition = stringPosition
@@ -1297,6 +1331,7 @@ function saveState(callback) {
 	let savedStrings = strings
 	let savedReferenceMap = referenceMap
 	let savedBundledStrings = bundledStrings
+	let savedPackedValues = packedValues
 
 	// TODO: We may need to revisit this if we do more external calls to user code (since it could be slow)
 	let savedSrc = new Uint8Array(src.slice(0, srcEnd)) // we copy the data in case it changes while external data is processed
@@ -1304,10 +1339,23 @@ function saveState(callback) {
 	let savedDecoder = currentDecoder
 	let savedSequentialMode = sequentialMode
 	let savedRestoreMapsAsObject = restoreMapsAsObject
+	let savedArrayAllocationBudget = arrayAllocationBudget
+	fragmentEnd = 0
+	if (fragmentReadString) readFixedString = fragmentReadString
+	fragmentReadString = null
 	restoreMapsAsObject = false
 	try {
 		return callback()
+	} catch (error) {
+		// the callback decodes its own source (a shared structures blob, or an extension's embedded data), so a
+		// truncation there cannot be completed by more of the outer input; checkedRead() clears the flag
+		if (error && (typeof error == 'object' || typeof error == 'function'))
+			nestedDecodeErrors.add(error)
+		throw error
 	} finally {
+		fragmentEnd = savedFragmentEnd
+		fragmentReadString = savedFragmentReader
+		readFixedString = savedReadFixedString
 		srcEnd = savedSrcEnd
 		position = savedPosition
 		stringPosition = savedStringPosition
@@ -1317,11 +1365,13 @@ function saveState(callback) {
 		strings = savedStrings
 		referenceMap = savedReferenceMap
 		bundledStrings = savedBundledStrings
+		packedValues = savedPackedValues
 		src = savedSrc
 		sequentialMode = savedSequentialMode
 		currentStructures = savedStructures
 		currentDecoder = savedDecoder
 		restoreMapsAsObject = savedRestoreMapsAsObject
+		arrayAllocationBudget = savedArrayAllocationBudget
 		dataView = new DataView(src.buffer, src.byteOffset, src.byteLength)
 	}
 }
@@ -1331,8 +1381,87 @@ export function clearSource() {
 	currentStructures = null
 }
 
+const registeredExtensions = new Map()
+let fragmentEnd = 0
+let fragmentReadString
+
+const builtinScanTags = new Map([[LEGACY_RECORD_INLINE_ID, legacyRecordDefinition], [259, currentExtensions[259]]])
+function isBuiltinScanTag(tag) {
+	return builtinScanTags.has(tag) && currentExtensions[tag] === builtinScanTags.get(tag)
+}
+
+function getScanTag(tag, decoder) {
+	return tag >= BUNDLED_STRINGS_ID && decoder.structures && decoder.structures[tag & 0x1fff]
+}
+
+function checkScanContainer(major, length, mapsAsObjects) {
+	if (major === 4) {
+		if (length > maxArraySize) throw new Error(`Array length exceeds ${maxArraySize}`)
+	} else if (mapsAsObjects) {
+		if (length > objectSizeLimit) throw new Error(`Property count exceeds ${objectSizeLimit}`)
+	} else if (length > maxMapSize) throw new Error(`Map size exceeds ${maxMapSize}`)
+}
+
+function decodeScanned(decoder, source, end) {
+	const boundStrings = isNativeAccelerationEnabled && end < source.length
+	if (!registeredExtensions.size && !boundStrings) return decoder.decodeMultiple(source)
+	const savedEnd = fragmentEnd
+	const savedReader = readFixedString
+	const savedFragmentReader = fragmentReadString
+	const originals = []
+	fragmentEnd = end
+	fragmentReadString = savedReader
+	try {
+		if (boundStrings) readFixedString = (length) => {
+			if (!fragmentEnd || position >= fragmentEnd) return savedReader(length)
+			const savedSrcEnd = srcEnd
+			srcEnd = fragmentEnd
+			try { return savedReader(length) } finally { srcEnd = savedSrcEnd }
+		}
+		for (const [tag, extension] of registeredExtensions) {
+			if (!extension.handlesRead) continue
+			const original = currentExtensions[tag]
+			const guarded = (read) => {
+				if (!fragmentEnd || position >= fragmentEnd) return extension(read)
+				let calls = 0, framingError
+				const value = extension(() => {
+					if (calls++) {
+						framingError = new Error('Tag ' + tag + ': fragmented decoding cannot frame multi-item handlesRead extensions')
+						throw framingError
+					}
+					return read()
+				})
+				if (framingError) throw framingError
+				return value
+			}
+			guarded.handlesRead = true
+			originals.push([tag, original, guarded])
+			currentExtensions[tag] = guarded
+		}
+		return decoder.decodeMultiple(source)
+	} finally {
+		fragmentEnd = savedEnd
+		fragmentReadString = savedFragmentReader
+		readFixedString = savedReader
+		for (const [tag, original, guarded] of originals) {
+			if (currentExtensions[tag] === guarded) currentExtensions[tag] = original
+		}
+	}
+}
+
+// Internal streaming hooks; not part of the public decoder API.
+export const _stream = {
+	getTag: getScanTag,
+	isBuiltinTag: isBuiltinScanTag,
+	checkContainer: checkScanContainer,
+	decode: decodeScanned,
+	get readThreshold() { return inlineObjectReadThreshold }
+}
+
 export function addExtension(extension) {
 	currentExtensions[extension.tag] = extension.decode
+	if (typeof extension.decode === 'function') registeredExtensions.set(extension.tag, extension.decode)
+	else registeredExtensions.delete(extension.tag)
 }
 
 export function setSizeLimits(limits) {