cbor-x 0.8.2 → 0.9.2

package/dist/node.cjs

@@ -14,6 +14,7 @@ let srcEnd;
 let position = 0;
 const EMPTY_ARRAY = [];
 const RECORD_TAG_ID = 0x69;
+const PACKED_REFERENCE_TAG_ID = 6;
 const STOP_CODE = {};
 let strings = EMPTY_ARRAY;
 let stringPosition = 0;
@@ -24,11 +25,15 @@ let srcStringStart = 0;
 let srcStringEnd = 0;
 let referenceMap;
 let currentExtensions = [];
+let currentExtensionRanges = [];
+let packedValues;
 let dataView;
+let restoreMapsAsObject;
 let defaultOptions = {
 	useRecords: false,
 	mapsAsObjects: true
 };
+let sequentialMode = false;
 
 class Decoder {
 	constructor(options) {
@@ -40,12 +45,12 @@ class Decoder {
 		}
 		Object.assign(this, options);
 	}
-	decode(source, end, continueReading) {
+	decode(source, end) {
 		if (src) {
 			// re-entrant execution, save the state and restore it after we do this decode
 			return saveState(() => {
 				clearSource();
-				return this ? this.decode(source, end, continueReading) : Decoder.prototype.decode.call(defaultOptions, source, end, continueReading)
+				return this ? this.decode(source, end) : Decoder.prototype.decode.call(defaultOptions, source, end)
 			})
 		}
 		srcEnd = end > -1 ? end : source.length;
@@ -61,19 +66,12 @@ class Decoder {
 		dataView = source.dataView || (source.dataView = new DataView(source.buffer, source.byteOffset, source.byteLength));
 		if (this) {
 			currentDecoder = this;
+			packedValues = this.sharedValues &&
+				(this.pack ? new Array(this.maxPrivatePackedValues || 16).concat(this.sharedValues) :
+				this.sharedValues);
 			if (this.structures) {
 				currentStructures = this.structures;
-				try {
-					return read()
-				} finally {
-					if (position >= srcEnd || !continueReading) {
-						// finished reading this source, cleanup references
-						currentStructures = null;
-						src = null;
-						if (referenceMap)
-							referenceMap = null;
-					}
-				}
+				return checkedRead()
 			} else if (!currentStructures || currentStructures.length > 0) {
 				currentStructures = [];
 			}
@@ -81,27 +79,23 @@ class Decoder {
 			currentDecoder = defaultOptions;
 			if (!currentStructures || currentStructures.length > 0)
 				currentStructures = [];
+			packedValues = null;
 		}
-		try {
-			return read()
-		} finally {
-			if (position >= srcEnd || !continueReading) {
-				src = null;
-				if (referenceMap)
-					referenceMap = null;
-			}
-		}
+		return checkedRead()
 	}
 	decodeMultiple(source, forEach) {
+		let values, lastPosition = 0;
 		try {
-			let decoder = this;
 			let size = source.length;
-			let value = this ? this.decode(source, size, true) : defaultDecoder.decode(source, size, true);
-			let values;
+			sequentialMode = true;
+			let value = this ? this.decode(source, size) : defaultDecoder.decode(source, size);
 			if (forEach) {
-				forEach(value);
+				if (forEach(value) === false) {
+					return
+				}
 				while(position < size) {
-					if (forEach(read()) === false) {
+					lastPosition = position;
+					if (forEach(checkedRead()) === false) {
 						return
 					}
 				}
@@ -109,17 +103,47 @@ class Decoder {
 			else {
 				values = [ value ];
 				while(position < size) {
-					values.push(read());
+					lastPosition = position;
+					values.push(checkedRead());
 				}
 				return values
 			}
+		} catch(error) {
+			error.lastPosition = lastPosition;
+			error.values = values;
+			throw error
 		} finally {
+			sequentialMode = false;
 			clearSource();
 		}
 	}
 }
-function getPosition() {
-	return position
+function checkedRead() {
+	try {
+		let result = read();
+		if (position == srcEnd) {
+			// finished reading this source, cleanup references
+			currentStructures = null;
+			src = null;
+			if (referenceMap)
+				referenceMap = null;
+		} else if (position > srcEnd) {
+			// over read
+			let error = new Error('Unexpected end of CBOR data');
+			error.incomplete = true;
+			throw error
+		} else if (!sequentialMode) {
+			throw new Error('Data read, but end of buffer not reached')
+		}
+		// else more to read, but we are reading sequentially, so don't clear source yet
+		return result
+	} catch(error) {
+		clearSource();
+		if (error instanceof RangeError || error.message.startsWith('Unexpected end of buffer')) {
+			error.incomplete = true;
+		}
+		throw error
+	}
 }
 
 function read() {
@@ -185,6 +209,10 @@ function read() {
 								object[key] = read();
 							return object
 						} else {
+							if (restoreMapsAsObject) {
+								currentDecoder.mapsAsObjects = true;
+								restoreMapsAsObject = false;
+							}
 							let map = new Map();
 							while ((key = read()) != STOP_CODE)
 								map.set(key, read());
@@ -231,6 +259,10 @@ function read() {
 				}
 				return object
 			} else {
+				if (restoreMapsAsObject) {
+					currentDecoder.mapsAsObjects = true;
+					restoreMapsAsObject = false;
+				}
 				let map = new Map();
 				for (let i = 0; i < token; i++) {
 					map.set(read(), read());
@@ -270,9 +302,15 @@ function read() {
 						return extension(read)
 					else
 						return extension(read())
+				} else {
+					let input = read();
+					for (let i = 0; i < currentExtensionRanges.length; i++) {
+						let value = currentExtensionRanges[i](token, input);
+						if (value !== undefined)
+							return value
+					}
+					return new Tag(input)
 				}
-				else
-					return new Tag(read())
 			}
 		case 7: // fixed value
 			switch (token) {
@@ -282,6 +320,9 @@ function read() {
 				case 0x17: return; // undefined
 				case 0x1f: 
 				default:
+					let packedValue = packedValues[token];
+					if (packedValue !== undefined)
+						return packedValue
 					throw new Error('Unknown token ' + token)
 			}
 		default: // negative int
@@ -704,6 +745,23 @@ currentExtensions[RECORD_TAG_ID] = recordDefinition;
 currentExtensions[27] = (data) => { // http://cbor.schmorp.de/generic-object
 	return (glbl[data[0]] || Error)(data[1], data[2])
 };
+const packedTable = (read) => {
+	if (src[position++] != 0x84)
+		throw new Error('Packed values structure must be followed by 4 element array')
+	let newPackedValues = read(); // packed values
+	packedValues = packedValues ? newPackedValues.concat(packedValues.slice(newPackedValues.length)) : newPackedValues;
+	packedValues.prefixes = read();
+	packedValues.suffixes = read();
+	return read() // read the rump
+};
+packedTable.handlesRead = true;
+currentExtensions[51] = packedTable;
+
+currentExtensions[PACKED_REFERENCE_TAG_ID] = (data) => { // packed reference
+	if (typeof data == 'number')
+		return packedValues[16 + (data >= 0 ? 2 * data : (-2 * data - 1))]
+	throw new Error('No support for non-integer packed references yet')
+};
 
 currentExtensions[40009] = (id) => {
 	// id extension (for structured clones)
@@ -735,6 +793,36 @@ currentExtensions[40010] = (id) => {
 };
 
 currentExtensions[258] = (array) => new Set(array); // https://github.com/input-output-hk/cbor-sets-spec/blob/master/CBOR_SETS.md
+(currentExtensions[259] = (read) => {
+	// https://github.com/shanewholloway/js-cbor-codec/blob/master/docs/CBOR-259-spec
+	// for decoding as a standard Map
+	if (currentDecoder.mapsAsObjects) {
+		currentDecoder.mapsAsObjects = false;
+		restoreMapsAsObject = true;
+	}
+	return read()
+}).handlesRead = true;
+function combine(a, b) {
+	if (typeof a === 'string')
+		return a + b
+	if (a instanceof Array)
+		return a.concat(b)
+	return Object.assign({}, a, b)
+}
+currentExtensionRanges.push((tag, input) => {
+	if (tag >= 225 && tag <= 255)
+		return combine(packedTable.prefixes[tag - 224], input)
+	if (tag >= 28704 && tag <= 32767)
+		return combine(packedTable.prefixes[tag - 28672], input)
+	if (tag >= 1879052288 && tag <= 2147483647)
+		return combine(packedTable.prefixes[tag - 1879048192], input)
+	if (tag >= 216 && tag <= 223)
+		return combine(input, packedTable.suffixes[tag - 216])
+	if (tag >= 27647 && tag <= 28671)
+		return combine(input, packedTable.suffixes[tag - 27639])
+	if (tag >= 1811940352 && tag <= 1879048191)
+		return combine(input, packedTable.suffixes[tag - 1811939328])
+});
 
 const typedArrays = ['Uint8', 'Uint8Clamped', 'Uint16', 'Uint32', 'BigUint64','Int8', 'Int16', 'Int32', 'BigInt64', 'Float32', 'Float64'].map(type => type + 'Array');
 const typedArrayTags = [64, 68, 69, 70, 71, 72, 77, 78, 79, 81, 82];
@@ -763,6 +851,7 @@ function saveState(callback) {
 	let savedSrc = new Uint8Array(src.slice(0, srcEnd)); // we copy the data in case it changes while external data is processed
 	let savedStructures = currentStructures;
 	let savedDecoder = currentDecoder;
+	let savedSequentialMode = sequentialMode;
 	let value = callback();
 	srcEnd = savedSrcEnd;
 	position = savedPosition;
@@ -773,6 +862,7 @@ function saveState(callback) {
 	strings = savedStrings;
 	referenceMap = savedReferenceMap;
 	src = savedSrc;
+	sequentialMode = savedSequentialMode;
 	currentStructures = savedStructures;
 	currentDecoder = savedDecoder;
 	dataView = new DataView(src.buffer, src.byteOffset, src.byteLength);
@@ -788,7 +878,7 @@ function addExtension(extension) {
 	currentExtensions[extension.tag] = extension.decode;
 }
 
-let mult10 = new Array(147); // this is a table matching binary exponents to the multiplier to determine significant digit rounding
+const mult10 = new Array(147); // this is a table matching binary exponents to the multiplier to determine significant digit rounding
 for (let i = 0; i < 256; i++) {
 	mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103));
 }
@@ -800,7 +890,14 @@ const FLOAT32_OPTIONS = {
 	ALWAYS: 1,
 	DECIMAL_ROUND: 3,
 	DECIMAL_FIT: 4
-};
+};
+let f32Array = new Float32Array(1);
+let u8Array = new Uint8Array(f32Array.buffer, 0, 4);
+function roundFloat32(float32Number) {
+	f32Array[0] = float32Number;
+	let multiplier = mult10[((u8Array[3] & 0x7f) << 1) | (u8Array[2] >> 7)];
+	return ((multiplier * float32Number + (float32Number > 0 ? 0.5 : -0.5)) >> 0) / multiplier
+}
 
 let textEncoder;
 try {
@@ -827,6 +924,7 @@ class Encoder extends Decoder {
 		let hasSharedUpdate;
 		let structures;
 		let referenceMap;
+		options = options || {};
 		let lastSharedStructuresLength = 0;
 		let encodeUtf8 = ByteArray.prototype.utf8Write ? function(string, position, maxBytes) {
 			return target.utf8Write(string, position, maxBytes)
@@ -837,19 +935,25 @@ class Encoder extends Decoder {
 
 		let encoder = this;
 		let maxSharedStructures = 64;
-		let isSequential = options && options.sequential;
+		let isSequential = options.sequential;
 		if (isSequential) {
 			maxSharedStructures = 0;
 			this.structures = [];
 		}
+		let samplingPackedValues, packedObjectMap, sharedValues = options.sharedValues;
+		let sharedPackedObjectMap;
+		if (sharedValues) {
+			sharedPackedObjectMap = Object.create(null);
+			for (let i = 0, l = sharedValues.length; i < l; i++) {
+				sharedPackedObjectMap[sharedValues[i]] = i;
+			}
+
+		}
 		let recordIdsToRemove = [];
 		let transitionsCount = 0;
 		let serializationsSinceTransitionRebuild = 0;
-		if (this.structures && this.structures.length > maxSharedStructures) {
-			throw new Error('Too many shared structures')
-		}
 
-		this.encode = function(value) {
+		this.encode = function(value, encodeOptions) {
 			if (!target) {
 				target = new ByteArrayAllocate(8192);
 				targetView = new DataView(target.buffer, 0, 8192);
@@ -862,7 +966,8 @@ class Encoder extends Decoder {
 				targetView = new DataView(target.buffer, 0, target.length);
 				safeEnd = target.length - 10;
 				position$1 = 0;
-			}
+			} else if (encodeOptions === REUSE_BUFFER_MODE)
+				position$1 = (position$1 + 7) & 0x7ffffff8; // Word align to make any future copying of this buffer faster
 			start = position$1;
 			referenceMap = encoder.structuredClone ? new Map() : null;
 			sharedStructures = encoder.structures;
@@ -870,7 +975,7 @@ class Encoder extends Decoder {
 				if (sharedStructures.uninitialized)
 					encoder.structures = sharedStructures = encoder.getStructures();
 				let sharedStructuresLength = sharedStructures.length;
-				if (sharedStructuresLength >  maxSharedStructures && !isSequential)
+				if (sharedStructuresLength > maxSharedStructures && !isSequential)
 					sharedStructuresLength = maxSharedStructures;
 				if (!sharedStructures.transitions) {
 					// rebuild our structure transitions
@@ -880,8 +985,8 @@ class Encoder extends Decoder {
 						if (!keys)
 							continue
 						let nextTransition, transition = sharedStructures.transitions;
-						for (let i =0, l = keys.length; i < l; i++) {
-							let key = keys[i];
+						for (let j = 0, l = keys.length; j < l; j++) {
+							let key = keys[j];
 							nextTransition = transition[key];
 							if (!nextTransition) {
 								nextTransition = transition[key] = Object.create(null);
@@ -898,6 +1003,29 @@ class Encoder extends Decoder {
 			if (hasSharedUpdate)
 				hasSharedUpdate = false;
 			structures = sharedStructures || [];
+			packedObjectMap = sharedPackedObjectMap;
+			if (options.pack) {
+				let packedValues = new Map();
+				packedValues.values = [];
+				packedValues.encoder = encoder;
+				packedValues.maxValues = options.maxPrivatePackedValues || (sharedPackedObjectMap ? 16 : Infinity);
+				packedValues.objectMap = sharedPackedObjectMap || false;
+				packedValues.samplingPackedValues = samplingPackedValues;
+				findRepetitiveStrings(value, packedValues);
+				if (packedValues.values.length > 0) {
+					target[position$1++] = 0xd8; // one-byte tag
+					target[position$1++] = 51; // tag 51 for packed shared structures https://www.potaroo.net/ietf/ids/draft-ietf-cbor-packed-03.txt
+					writeArrayHeader(4);
+					let valuesArray = packedValues.values;
+					encode(valuesArray);
+					writeArrayHeader(0); // prefixes
+					writeArrayHeader(0); // suffixes
+					packedObjectMap = Object.create(sharedPackedObjectMap || null);
+					for (let i = 0, l = valuesArray.length; i < l; i++) {
+						packedObjectMap[valuesArray[i]] = i;
+					}
+				}
+			}
 			try {
 				encode(value);
 				encoder.offset = position$1; // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
@@ -910,7 +1038,12 @@ class Encoder extends Decoder {
 					referenceMap = null;
 					return serialized
 				}
-				return target.subarray(start, position$1) // position can change if we call encode again in saveStructures, so we get the buffer now
+				if (encodeOptions === REUSE_BUFFER_MODE) {
+					target.start = start;
+					target.end = position$1;
+					return target
+				}
+				return target.subarray(start, position$1) // position can change if we call pack again in saveStructures, so we get the buffer now
 			} finally {
 				if (sharedStructures) {
 					if (serializationsSinceTransitionRebuild < 10)
@@ -924,7 +1057,7 @@ class Encoder extends Decoder {
 							recordIdsToRemove = [];
 					} else if (recordIdsToRemove.length > 0 && !isSequential) {
 						for (let i = 0, l = recordIdsToRemove.length; i < l; i++) {
-							recordIdsToRemove[i][RECORD_SYMBOL] = 0;
+							recordIdsToRemove[i][RECORD_SYMBOL] = undefined;
 						}
 						recordIdsToRemove = [];
 					}
@@ -932,17 +1065,43 @@ class Encoder extends Decoder {
 						if (encoder.structures.length > maxSharedStructures) {
 							encoder.structures = encoder.structures.slice(0, maxSharedStructures);
 						}
+						// we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
+						let returnBuffer = target.subarray(start, position$1);
+						let shared = encoder.structures || [];
+						if (sharedValues) {
+							shared = shared.concat(sharedValues);
+						}
 
 						if (encoder.saveStructures(encoder.structures, lastSharedStructuresLength) === false) {
 							// get updated structures and try again if the update failed
 							encoder.structures = encoder.getStructures() || [];
 							return encoder.encode(value)
 						}
-						lastSharedStructuresLength = encoder.structures.length;
+						lastSharedStructuresLength = shared.length;
+						return returnBuffer
 					}
 				}
 			}
 		};
+		this.findCommonStringsToPack = () => {
+			samplingPackedValues = new Map();
+			if (!sharedPackedObjectMap)
+				sharedPackedObjectMap = Object.create(null);
+			return ({ threshold }) => {
+				threshold = threshold || 4;
+				let position = this.pack ? options.maxPrivatePackedValues || 16 : 0;
+				if (!sharedValues)
+					sharedValues = this.sharedValues = [];
+				for (let [ key, status ] of samplingPackedValues) {
+					if (status.count > threshold) {
+						sharedPackedObjectMap[key] = position++;
+						sharedValues.push(key);
+						hasSharedUpdate = true;
+					}
+				}
+				samplingPackedValues = null;
+			}
+		};
 		const encode = (value) => {
 			if (position$1 > safeEnd)
 				target = makeRoom(position$1);
@@ -950,6 +1109,43 @@ class Encoder extends Decoder {
 			var type = typeof value;
 			var length;
 			if (type === 'string') {
+				if (packedObjectMap) {
+					let packedPosition = packedObjectMap[value];
+					if (packedPosition >= 0) {
+						if (packedPosition < 16)
+							target[position$1++] = packedPosition + 0xe0; // simple values, defined in https://www.potaroo.net/ietf/ids/draft-ietf-cbor-packed-03.txt
+						else {
+							target[position$1++] = 0xc6; // tag 6 defined in https://www.potaroo.net/ietf/ids/draft-ietf-cbor-packed-03.txt
+							if (packedPosition & 1)
+								encode((15 - packedPosition) >> 1);
+							else
+								encode((packedPosition - 16) >> 1);
+						}
+						return
+/*						} else if (packedStatus.serializationId != serializationId) {
+							packedStatus.serializationId = serializationId
+							packedStatus.count = 1
+							if (options.sharedPack) {
+								let sharedCount = packedStatus.sharedCount = (packedStatus.sharedCount || 0) + 1
+								if (shareCount > (options.sharedPack.threshold || 5)) {
+									let sharedPosition = packedStatus.position = packedStatus.nextSharedPosition
+									hasSharedUpdate = true
+									if (sharedPosition < 16)
+										target[position++] = sharedPosition + 0xc0
+
+								}
+							}
+						} // else any in-doc incrementation?*/
+					} else if (samplingPackedValues && !options.pack) {
+						let status = samplingPackedValues.get(value);
+						if (status)
+							status.count++;
+						else
+							samplingPackedValues.set(value, {
+								count: 1,
+							});
+					}
+				}
 				let strLength = value.length;
 				let headerSize;
 				// first we estimate the header size, so we can write to the correct location
@@ -1059,7 +1255,7 @@ class Encoder extends Decoder {
 						targetView.setFloat32(position$1, value);
 						let xShifted;
 						if (useFloat32 < 4 ||
-							// this checks for  rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
+								// this checks for rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
 								((xShifted = value * mult10[((target[position$1] & 0x7f) << 1) | (target[position$1 + 1] >> 7)]) >> 0) === xShifted) {
 							position$1 += 4;
 							return
@@ -1105,6 +1301,12 @@ class Encoder extends Decoder {
 							encode(value[i]);
 						}
 					} else if (constructor === Map) {
+						if (this.mapsAsObjects ? this.useTag259ForMaps !== false : this.useTag259ForMaps) {
+							// use Tag 259 (https://github.com/shanewholloway/js-cbor-codec/blob/master/docs/CBOR-259-spec--explicit-maps.md) for maps if the user wants it that way
+							target[position$1++] = 0xd9;
+							target[position$1++] = 1;
+							target[position$1++] = 3;
+						}
 						length = value.size;
 						if (length < 0x18) {
 							target[position$1++] = 0xa0 | length;
@@ -1182,10 +1384,9 @@ class Encoder extends Decoder {
 				}
 				position$1 += 8;
 			} else if (type === 'undefined') {
-				//target[position++] = 0xc1 // this is the "never-used" byte
 				target[position$1++] = 0xf7;
 			} else {
-				throw new Error('Unknown type ' + type)
+				throw new Error('Unknown type: ' + type)
 			}
 		};
 
@@ -1274,7 +1475,7 @@ class Encoder extends Decoder {
 			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null));
 			let newTransitions = 0;
 			let length = keys.length;
-			for (let i =0; i < length; i++) {
+			for (let i = 0; i < length; i++) {
 				let key = keys[i];
 				nextTransition = transition[key];
 				if (!nextTransition) {
@@ -1284,7 +1485,7 @@ class Encoder extends Decoder {
 				transition = nextTransition;
 			}
 			let recordId = transition[RECORD_SYMBOL];
-			if (recordId) {
+			if (recordId !== undefined) {
 				target[position$1++] = 0xd9; // tag two byte
 				target[position$1++] = RECORD_STARTING_ID_PREFIX;
 				target[position$1++] = recordId;
@@ -1311,12 +1512,9 @@ class Encoder extends Decoder {
 						transitionsCount += serializationsSinceTransitionRebuild * newTransitions;
 					// record the removal of the id, we can maintain our shared structure
 					if (recordIdsToRemove.length >= MAX_STRUCTURES - maxSharedStructures)
-						recordIdsToRemove.shift()[RECORD_SYMBOL] = 0; // we are cycling back through, and have to remove old ones
+						recordIdsToRemove.shift()[RECORD_SYMBOL] = undefined; // we are cycling back through, and have to remove old ones
 					recordIdsToRemove.push(transition);
-					if (length < 0x16)
-						target[position$1++] = 0x82 + length; // array header, length of values + 2
-					else
-						writeArrayHeader(length + 2);
+					writeArrayHeader(length + 2);
 					encode(keys);
 					target[position$1++] = 0x19; // uint16
 					target[position$1++] = RECORD_STARTING_ID_PREFIX;
@@ -1366,7 +1564,9 @@ class Encoder extends Decoder {
 }
 
 function writeArrayHeader(length) {
-	if (length < 0x100) {
+	if (length < 0x18)
+		target[position$1++] = 0x80 | length;
+	else if (length < 0x100) {
 		target[position$1++] = 0x98;
 		target[position$1++] = length;
 	} else if (length < 0x10000) {
@@ -1380,6 +1580,56 @@ function writeArrayHeader(length) {
 	}
 }
 
+function findRepetitiveStrings(value, packedValues) {
+	switch(typeof value) {
+		case 'string':
+			if (value.length > 3) {
+				if (packedValues.objectMap[value] > -1 || packedValues.values.length >= packedValues.maxValues)
+					return
+				let packedStatus = packedValues.get(value);
+				if (packedStatus) {
+					if (++packedStatus.count == 2) {
+						packedValues.values.push(value);
+					}
+				} else {
+					packedValues.set(value, {
+						count: 1,
+					});
+					if (packedValues.samplingPackedValues) {
+						let status = packedValues.samplingPackedValues.get(value);
+						if (status)
+							status.count++;
+						else
+							packedValues.samplingPackedValues.set(value, {
+								count: 1,
+							});
+					}
+				}
+			}
+			break
+		case 'object':
+			if (value) {
+				if (value instanceof Array) {
+					for (let i = 0, l = value.length; i < l; i++) {
+						findRepetitiveStrings(value[i], packedValues);
+					}
+
+				} else {
+					let includeKeys = !packedValues.encoder.useRecords;
+					for (var key in value) {
+						if (value.hasOwnProperty(key)) {
+							if (includeKeys)
+								findRepetitiveStrings(key, packedValues);
+							findRepetitiveStrings(value[key], packedValues);
+						}
+					}
+				}
+			}
+			break
+		case 'function': console.log(value);
+	}
+}
+
 extensionClasses = [ Date, Set, Error, RegExp, ArrayBuffer, ByteArray,
 	Uint8Array, Uint8ClampedArray, Uint16Array, Uint32Array, BigUint64Array, Int8Array, Int16Array, Int32Array, BigInt64Array,
 	Float32Array, Float64Array];
@@ -1451,7 +1701,9 @@ function typedArrayEncoder(tag) {
 }
 function writeBuffer(buffer, makeRoom) {
 	let length = buffer.byteLength;
-	if (length < 0x100) {
+	if (length < 0x18) {
+		target[position$1++] = 0x40 + length;
+	} else if (length < 0x100) {
 		target[position$1++] = 0x58;
 		target[position$1++] = length;
 	} else if (length < 0x10000) {
@@ -1506,24 +1758,21 @@ function addExtension$1(extension) {
 }
 let defaultEncoder = new Encoder({ useRecords: false });
 const encode = defaultEncoder.encode;
-const { NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } = FLOAT32_OPTIONS;
+const { NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } = FLOAT32_OPTIONS;
+const REUSE_BUFFER_MODE = 1000;
 
 class EncoderStream extends stream.Transform {
 	constructor(options) {
 		if (!options)
 			options = {};
+		options.writableObjectMode = true;
 		super(options);
 		options.sequential = true;
-		this.encoder = new Encoder(options);
+		this.encoder = options.encoder || new Encoder(options);
 	}
-	write(value) {
+	_transform(value, encoding, callback) {
 		this.push(this.encoder.encode(value));
-	}
-
-	end(value) {
-		if (value != null)
-			this.push(this.encoder.encode(value));
-		this.push(null);
+		callback();
 	}
 }
 
@@ -1534,33 +1783,118 @@ class DecoderStream extends stream.Transform {
 		options.objectMode = true;
 		super(options);
 		options.structures = [];
-		this.decoder = new Decoder(options);
+		this.decoder = options.decoder || new Decoder(options);
 	}
 	_transform(chunk, encoding, callback) {
 		if (this.incompleteBuffer) {
 			chunk = Buffer.concat([this.incompleteBuffer, chunk]);
 			this.incompleteBuffer = null;
 		}
-		let position = 0;
-		let size = chunk.length;
+		let values;
 		try {
-			this.push(this.decoder.decode(chunk, size, true));
-			position = getPosition();
-			while (position < size) {
-				let value = read();
-				this.push(value);
-				position = getPosition();
-			}
+			values = this.decoder.decodeMultiple(chunk);
 		} catch(error) {
-			if (error.incomplete)
-				this.incompleteBuffer = chunk.slice(position);
+			if (error.incomplete) {
+				this.incompleteBuffer = chunk.slice(error.lastPosition);
+				values = error.values;
+			}
 			else
 				throw error
 		} finally {
-			clearSource();
+			for (let value of values || []) {
+				if (value === null)
+					value = this.getNullValue();
+				this.push(value);
+			}
 		}
 		if (callback) callback();
 	}
+	getNullValue() {
+		return Symbol.for(null)
+	}
+}
+
+/**
+ * Given an Iterable first argument, returns an Iterable where each value is encoded as a Buffer
+ * If the argument is only Async Iterable, the return value will be an Async Iterable.
+ * @param {Iterable|Iterator|AsyncIterable|AsyncIterator} objectIterator - iterable source, like a Readable object stream, an array, Set, or custom object
+ * @param {options} [options] - cbor-x Encoder options
+ * @returns {IterableIterator|Promise.<AsyncIterableIterator>}
+ */
+function encodeIter (objectIterator, options = {}) {
+  if (!objectIterator || typeof objectIterator !== 'object') {
+    throw new Error('first argument must be an Iterable, Async Iterable, or a Promise for an Async Iterable')
+  } else if (typeof objectIterator[Symbol.iterator] === 'function') {
+    return encodeIterSync(objectIterator, options)
+  } else if (typeof objectIterator.then === 'function' || typeof objectIterator[Symbol.asyncIterator] === 'function') {
+    return encodeIterAsync(objectIterator, options)
+  } else {
+    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a Promise')
+  }
+}
+
+function * encodeIterSync (objectIterator, options) {
+  const encoder = new Encoder(options);
+  for (const value of objectIterator) {
+    yield encoder.encode(value);
+  }
+}
+
+async function * encodeIterAsync (objectIterator, options) {
+  const encoder = new Encoder(options);
+  for await (const value of objectIterator) {
+    yield encoder.encode(value);
+  }
+}
+
+/**
+ * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
+ * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
+ * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
+ * @param {object} [options] - Decoder options
+ * @returns {IterableIterator|Promise.<AsyncIterableIterator}
+ */
+function decodeIter (bufferIterator, options = {}) {
+  if (!bufferIterator || typeof bufferIterator !== 'object') {
+    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a promise')
+  }
+
+  const decoder = new Decoder(options);
+  let incomplete;
+  const parser = (chunk) => {
+    let yields;
+    // if there's incomplete data from previous chunk, concatinate and try again
+    if (incomplete) {
+      chunk = Buffer.concat([incomplete, chunk]);
+      incomplete = undefined;
+    }
+
+    try {
+      yields = decoder.decodeMultiple(chunk);
+    } catch (err) {
+      if (err.incomplete) {
+        incomplete = chunk.slice(err.lastPosition);
+        yields = err.values;
+      } else {
+        throw err
+      }
+    }
+    return yields
+  };
+
+  if (typeof bufferIterator[Symbol.iterator] === 'function') {
+    return (function * iter () {
+      for (const value of bufferIterator) {
+        yield * parser(value);
+      }
+    })()
+  } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
+    return (async function * iter () {
+      for await (const value of bufferIterator) {
+        yield * parser(value);
+      }
+    })()
+  }
 }
 
 const useRecords = false;
@@ -1591,10 +1925,15 @@ exports.Encoder = Encoder;
 exports.EncoderStream = EncoderStream;
 exports.FLOAT32_OPTIONS = FLOAT32_OPTIONS;
 exports.NEVER = NEVER;
+exports.REUSE_BUFFER_MODE = REUSE_BUFFER_MODE;
 exports.Tag = Tag;
 exports.addExtension = addExtension$1;
+exports.clearSource = clearSource;
 exports.decode = decode;
+exports.decodeIter = decodeIter;
 exports.decodeMultiple = decodeMultiple;
 exports.encode = encode;
+exports.encodeIter = encodeIter;
 exports.mapsAsObjects = mapsAsObjects;
+exports.roundFloat32 = roundFloat32;
 exports.useRecords = useRecords;