cbor-x 0.9.4 → 1.0.0

package/dist/node.cjs

@@ -13,7 +13,10 @@ let src;
 let srcEnd;
 let position = 0;
 const EMPTY_ARRAY = [];
-const RECORD_TAG_ID = 0x69;
+const LEGACY_RECORD_INLINE_ID = 105;
+const RECORD_DEFINITIONS_ID = 0xdffe;
+const RECORD_INLINE_ID = 0xdfff; // temporary first-come first-serve tag // proposed tag: 0x7265 // 're'
+const BUNDLED_STRINGS_ID = 0xdff9;
 const PACKED_REFERENCE_TAG_ID = 6;
 const STOP_CODE = {};
 let strings = EMPTY_ARRAY;
@@ -23,6 +26,7 @@ let currentStructures;
 let srcString;
 let srcStringStart = 0;
 let srcStringEnd = 0;
+let bundledStrings;
 let referenceMap;
 let currentExtensions = [];
 let currentExtensionRanges = [];
@@ -38,13 +42,70 @@ let sequentialMode = false;
 class Decoder {
 	constructor(options) {
 		if (options) {
+			if ((options.keyMap || options._keyMap) && !options.useRecords) {
+				options.useRecords = false;
+				options.mapsAsObjects = true;
+			}
 			if (options.useRecords === false && options.mapsAsObjects === undefined)
 				options.mapsAsObjects = true;
-			if (options.getStructures && !options.structures)
+			if (options.getStructures)
+				options.getShared = options.getStructures;
+			if (options.getShared && !options.structures)
 				(options.structures = []).uninitialized = true; // this is what we use to denote an uninitialized structures
+			if (options.keyMap) {
+				this.mapKey = new Map();
+				for (let [k,v] of Object.entries(options.keyMap)) this.mapKey.set(v,k);
+			}
 		}
 		Object.assign(this, options);
 	}
+	/*
+	decodeKey(key) {
+		return this.keyMap
+			? Object.keys(this.keyMap)[Object.values(this.keyMap).indexOf(key)] || key
+			: key
+	}
+	*/
+	decodeKey(key) {
+		return this.keyMap ? this.mapKey.get(key) || key : key
+	}
+	
+	encodeKey(key) {
+		return this.keyMap && this.keyMap.hasOwnProperty(key) ? this.keyMap[key] : key
+	}
+
+	encodeKeys(rec) {
+		if (!this._keyMap) return rec
+		let map = new Map();
+		for (let [k,v] of Object.entries(rec)) map.set((this._keyMap.hasOwnProperty(k) ? this._keyMap[k] : k), v);
+		return map
+	}
+
+	decodeKeys(map) {
+		if (!this._keyMap || map.constructor.name != 'Map') return map
+		if (!this._mapKey) {
+			this._mapKey = new Map();
+			for (let [k,v] of Object.entries(this._keyMap)) this._mapKey.set(v,k);
+		}
+		let res = {};
+		//map.forEach((v,k) => res[Object.keys(this._keyMap)[Object.values(this._keyMap).indexOf(k)] || k] = v)
+		map.forEach((v,k) => res[this._mapKey.has(k) ? this._mapKey.get(k) : k] =  v);
+		return res
+	}
+	
+	mapDecode(source, end) {
+	
+		let res = this.decode(source);
+		if (this._keyMap) { 
+			//Experiemntal support for Optimised KeyMap  decoding 
+			switch (res.constructor.name) {
+				case 'Array': return res.map(r => this.decodeKeys(r))
+				//case 'Map': return this.decodeKeys(res)
+			}
+		}
+		return res
+	}
+
 	decode(source, end) {
 		if (src) {
 			// re-entrant execution, save the state and restore it after we do this decode
@@ -59,6 +120,7 @@ class Decoder {
 		srcStringEnd = 0;
 		srcString = null;
 		strings = EMPTY_ARRAY;
+		bundledStrings = null;
 		src = source;
 		// this provides cached access to the data view for a buffer if it is getting reused, which is a recommend
 		// technique for getting data from a database where it can be copied into an existing buffer instead of creating
@@ -129,6 +191,9 @@ class Decoder {
 function checkedRead() {
 	try {
 		let result = read();
+		if (bundledStrings) // bundled strings to skip past
+			position = bundledStrings.postBundlePosition;
+
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
 			currentStructures = null;
@@ -203,6 +268,7 @@ function read() {
 				switch(majorType) {
 					case 2: // byte string
 					case 3: // text string
+						throw new Error('Indefinite length not supported for byte or text strings')
 					case 4: // array
 						let array = [];
 						let value, i = 0;
@@ -214,8 +280,8 @@ function read() {
 						let key;
 						if (currentDecoder.mapsAsObjects) {
 							let object = {};
-							while ((key = readKey()) != STOP_CODE)
-								object[key] = read();
+							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) object[currentDecoder.decodeKey(key)] = read();
+							else while ((key = read()) != STOP_CODE) object[key] = read();
 							return object
 						} else {
 							if (restoreMapsAsObject) {
@@ -223,8 +289,8 @@ function read() {
 								restoreMapsAsObject = false;
 							}
 							let map = new Map();
-							while ((key = read()) != STOP_CODE)
-								map.set(key, read());
+							if (currentDecoder.keyMap) while((key = read()) != STOP_CODE) map.set(currentDecoder.decodeKey(key), read());
+							else while ((key = read()) != STOP_CODE) map.set(key, read());
 							return map
 						}
 					case 7:
@@ -256,16 +322,15 @@ function read() {
 			return readFixedString(token)
 		case 4: // array
 			let array = new Array(token);
-			for (let i = 0; i < token; i++) {
-				array[i] = read();
-			}
+		  //if (currentDecoder.keyMap) for (let i = 0; i < token; i++) array[i] = currentDecoder.decodeKey(read())	
+			//else 
+			for (let i = 0; i < token; i++) array[i] = read();
 			return array
 		case 5: // map
 			if (currentDecoder.mapsAsObjects) {
 				let object = {};
-				for (let i = 0; i < token; i++) {
-					object[readKey()] = read();
-				}
+				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) object[currentDecoder.decodeKey(read())] = read();
+				else for (let i = 0; i < token; i++) object[read()] = read();
 				return object
 			} else {
 				if (restoreMapsAsObject) {
@@ -273,63 +338,66 @@ function read() {
 					restoreMapsAsObject = false;
 				}
 				let map = new Map();
-				for (let i = 0; i < token; i++) {
-					map.set(read(), read());
-				}
+				if (currentDecoder.keyMap) for (let i = 0; i < token; i++) map.set(currentDecoder.decodeKey(read()),read());
+				else for (let i = 0; i < token; i++) map.set(read(), read());
 				return map
 			}
 		case 6: // extension
-			if ((token >> 8) == RECORD_TAG_ID) { // record structures
-				let structure = currentStructures[token & 0xff];
+			if (token >= BUNDLED_STRINGS_ID) {
+				let structure = currentStructures[token & 0x1fff]; // check record structures first
+				// At some point we may provide an option for dynamic tag assignment with a range like token >= 8 && (token < 16 || (token > 0x80 && token < 0xc0) || (token > 0x130 && token < 0x4000))
 				if (structure) {
-					if (!structure.read)
-						structure.read = createStructureReader(structure);
+					if (!structure.read) structure.read = createStructureReader(structure);
 					return structure.read()
-				} else if (currentDecoder.getStructures) {
-					let updatedStructures = saveState(() => {
-						// save the state in case getStructures modifies our buffer
-						src = null;
-						return currentDecoder.getStructures()
-					});
-					if (currentStructures === true)
-						currentDecoder.structures = currentStructures = updatedStructures;
-					else
-						currentStructures.splice.apply(currentStructures, [0, updatedStructures.length].concat(updatedStructures));
-					structure = currentStructures[token & 0xff];
-					if (structure) {
-						if (!structure.read)
-							structure.read = createStructureReader(structure);
-						return structure.read()
-					} else
-						return token
-				} else
-					return token
-			} else {
-				let extension = currentExtensions[token];
-				if (extension) {
-					if (extension.handlesRead)
-						return extension(read)
-					else
-						return extension(read())
-				} else {
-					let input = read();
-					for (let i = 0; i < currentExtensionRanges.length; i++) {
-						let value = currentExtensionRanges[i](token, input);
-						if (value !== undefined)
-							return value
+				}
+				if (token < 0x10000) {
+					if (token == RECORD_INLINE_ID) // we do a special check for this so that we can keep the currentExtensions as densely stored array (v8 stores arrays densely under about 3000 elements)
+						return recordDefinition(read())
+					else if (token == RECORD_DEFINITIONS_ID) {
+						let length = readJustLength();
+						let id = read();
+						for (let i = 2; i < length; i++) {
+							recordDefinition([id++, read()]);
+						}
+						return read()
+					} else if (token == BUNDLED_STRINGS_ID) {
+						return readBundleExt()
+					}
+					if (currentDecoder.getShared) {
+						loadShared();
+						structure = currentStructures[token & 0x1fff];
+						if (structure) {
+							if (!structure.read)
+								structure.read = createStructureReader(structure);
+							return structure.read()
+						}
 					}
-					return new Tag(input)
 				}
 			}
+			let extension = currentExtensions[token];
+			if (extension) {
+				if (extension.handlesRead)
+					return extension(read)
+				else
+					return extension(read())
+			} else {
+				let input = read();
+				for (let i = 0; i < currentExtensionRanges.length; i++) {
+					let value = currentExtensionRanges[i](token, input);
+					if (value !== undefined)
+						return value
+				}
+				return new Tag(input, token)
+			}
 		case 7: // fixed value
 			switch (token) {
 				case 0x14: return false
 				case 0x15: return true
 				case 0x16: return null
 				case 0x17: return; // undefined
-				case 0x1f: 
+				case 0x1f:
 				default:
-					let packedValue = packedValues[token];
+					let packedValue = (packedValues || getPackedValues())[token];
 					if (packedValue !== undefined)
 						return packedValue
 					throw new Error('Unknown token ' + token)
@@ -345,41 +413,47 @@ function read() {
 }
 const validName = /^[a-zA-Z_$][a-zA-Z\d_$]*$/;
 function createStructureReader(structure) {
-	let l = structure.length;
 	function readObject() {
+		// get the array size from the header
+		let length = src[position++];
+		//let majorType = token >> 5
+		length = length & 0x1f;
+		if (length > 0x17) {
+			switch (length) {
+				case 0x18:
+					length = src[position++];
+					break
+				case 0x19:
+					length = dataView.getUint16(position);
+					position += 2;
+					break
+				case 0x1a:
+					length = dataView.getUint32(position);
+					position += 4;
+					break
+				default:
+					throw new Error('Expected array header, but got ' + src[position - 1])
+			}
+		}
 		// This initial function is quick to instantiate, but runs slower. After several iterations pay the cost to build the faster function
-		if (readObject.count++ > 2) {
-			this.read = (new Function('a', 'r', 'return function(){a();return {' + structure.map(key => validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '}}'))(readArrayHeader, read);
-			return this.read()
+		if (this.objectLiteralSize === length) // we have a fast object literal reader, use it (assuming it is the right length)
+			return this.objectLiteral(read)
+		if (this.count++ == 3) { // create a fast reader
+			this.objectLiteralSize = length;
+			this.objectLiteral = currentDecoder.keyMap 
+				? new Function('r', 'return {' + this.map(k => currentDecoder.decodeKey(k)).map(k => validName.test(k) ? k + ':r()' : ('[' + JSON.stringify(k) + ']:r()')).join(',') + '}')
+				: new Function('r', 'return {' + this.map(key => validName.test(key) ? key + ':r()' : ('[' + JSON.stringify(key) + ']:r()')).join(',') + '}');
+			return this.objectLiteral(read)
 		}
-		readArrayHeader();
 		let object = {};
-		for (let i = 0; i < l; i++) {
-			let key = structure[i];
-			object[key] = read();
-		}
+		if (currentDecoder.keyMap) for (let i = 0; i < length; i++) object[currentDecoder.decodeKey(this[i])] = read();
+		else for (let i = 0; i < length; i++) object[this[i]] = read();
 		return object
 	}
-	readObject.count = 0;
+	structure.count = 0;
 	return readObject
 }
 
-function readArrayHeader(expectedLength) {
-	// consume the array header, TODO: check expected length
-	let token = src[position++];
-	//let majorType = token >> 5
-	token = token & 0x1f;
-	if (token > 0x17) {
-		switch (token) {
-			case 0x18: position++;
-				break
-			case 0x19: position += 2;
-				break
-			case 0x1a: position += 4;
-		}
-	}
-}
-
 let readFixedString = readStringJS;
 
 exports.isNativeAccelerationEnabled = false;
@@ -390,6 +464,8 @@ function setExtractor(extractStrings) {
 		return function readString(length) {
 			let string = strings[stringPosition++];
 			if (string == null) {
+				if (bundledStrings)
+					return readStringJS(length)
 				let extraction = extractStrings(position, srcEnd, length, src);
 				if (typeof extraction == 'string') {
 					string = extraction;
@@ -644,76 +720,15 @@ function getFloat16() {
 }
 
 let keyCache = new Array(4096);
-function readKey() {
-	let length = src[position++];
-	if (length >= 0x60 && length < 0x78) {
-		// fixstr, potentially use key cache
-		length = length - 0x60;
-		if (srcStringEnd >= position) // if it has been extracted, must use it (and faster anyway)
-			return srcString.slice(position - srcStringStart, (position += length) - srcStringStart)
-		else if (!(srcStringEnd == 0 && srcEnd < 180))
-			return readFixedString(length)
-	} else { // not cacheable, go back and do a standard read
-		position--;
-		return read()
-	}
-	let key = ((length << 5) ^ (length > 1 ? dataView.getUint16(position) : length > 0 ? src[position] : 0)) & 0xfff;
-	let entry = keyCache[key];
-	let checkPosition = position;
-	let end = position + length - 3;
-	let chunk;
-	let i = 0;
-	if (entry && entry.bytes == length) {
-		while (checkPosition < end) {
-			chunk = dataView.getUint32(checkPosition);
-			if (chunk != entry[i++]) {
-				checkPosition = 0x70000000;
-				break
-			}
-			checkPosition += 4;
-		}
-		end += 3;
-		while (checkPosition < end) {
-			chunk = src[checkPosition++];
-			if (chunk != entry[i++]) {
-				checkPosition = 0x70000000;
-				break
-			}
-		}
-		if (checkPosition === end) {
-			position = checkPosition;
-			return entry.string
-		}
-		end -= 3;
-		checkPosition = position;
-	}
-	entry = [];
-	keyCache[key] = entry;
-	entry.bytes = length;
-	while (checkPosition < end) {
-		chunk = dataView.getUint32(checkPosition);
-		entry.push(chunk);
-		checkPosition += 4;
-	}
-	end += 3;
-	while (checkPosition < end) {
-		chunk = src[checkPosition++];
-		entry.push(chunk);
-	}
-	// for small blocks, avoiding the overhead of the extract call is helpful
-	let string = length < 16 ? shortStringInJS(length) : longStringInJS(length);
-	if (string != null)
-		return entry.string = string
-	return entry.string = readFixedString(length)
-}
 
 class Tag {
-	constructor(value) {
+	constructor(value, tag) {
 		this.value = value;
+		this.tag = tag;
 	}
 }
 
-let glbl = typeof window == 'object' ? window : global;
+let glbl = typeof self == 'object' ? self : global;
 
 currentExtensions[0] = (dateString) => {
 	// string date extension
@@ -727,38 +742,62 @@ currentExtensions[1] = (epochSec) => {
 
 currentExtensions[2] = (buffer) => {
 	// bigint extension
-	return new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength).getBigUint64(0)
+	let value = BigInt(0);
+	for (let i = 0, l = buffer.byteLength; i < l; i++) {
+		value = BigInt(buffer[i]) + value << BigInt(8);
+	}
+	return value
 };
 
 currentExtensions[3] = (buffer) => {
 	// negative bigint extension
-	return BigInt(-1) - (new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength).getBigUint64(0))
+	return BigInt(-1) - currentExtensions[2](buffer)
+};
+currentExtensions[4] = (fraction) => {
+	// best to reparse to maintain accuracy
+	return +(fraction[1] + 'e' + fraction[0])
+};
+
+currentExtensions[5] = (fraction) => {
+	// probably not sufficiently accurate
+	return fraction[1] * Math.exp(fraction[0] * Math.log(2))
 };
 
 // the registration of the record definition extension (tag 105)
-const recordDefinition = () => {
-	let definition = read();
-	let structure = definition[0];
-	let id = definition[1];
-	currentStructures[id & 0xff] = structure;
+const recordDefinition = (definition) => {
+	let id = definition[0] - 0xe000;
+	let structure = definition[1];
+	currentStructures[id] = structure;
 	structure.read = createStructureReader(structure);
 	let object = {};
-	for (let i = 2,l = definition.length; i < l; i++) {
-		let key = structure[i - 2];
-		object[key] = definition[i];
-	}
+	if (currentDecoder.keyMap) for (let i = 2,l = definition.length; i < l; i++) {
+			let key = currentDecoder.decodeKey(structure[i - 2]);
+			object[key] = definition[i];
+		}
+	else for (let i = 2,l = definition.length; i < l; i++) {
+			let key = structure[i - 2];
+			object[key] = definition[i];
+		}
 	return object
 };
-
-recordDefinition.handlesRead = true;
-currentExtensions[RECORD_TAG_ID] = recordDefinition;
+currentExtensions[LEGACY_RECORD_INLINE_ID] = recordDefinition;
+currentExtensions[14] = (value) => {
+	if (bundledStrings)
+		return bundledStrings[0].slice(bundledStrings.position0, bundledStrings.position0 += value)
+	return new Tag(value, 14)
+};
+currentExtensions[15] = (value) => {
+	if (bundledStrings)
+		return bundledStrings[1].slice(bundledStrings.position1, bundledStrings.position1 += value)
+	return new Tag(value, 15)
+};
 
 currentExtensions[27] = (data) => { // http://cbor.schmorp.de/generic-object
 	return (glbl[data[0]] || Error)(data[1], data[2])
 };
 const packedTable = (read) => {
 	if (src[position++] != 0x84)
-		throw new Error('Packed values structure must be followed by 4 element array')
+		throw new Error('Packed values structure must be followed by a 4 element array')
 	let newPackedValues = read(); // packed values
 	packedValues = packedValues ? newPackedValues.concat(packedValues.slice(newPackedValues.length)) : newPackedValues;
 	packedValues.prefixes = read();
@@ -769,15 +808,36 @@ packedTable.handlesRead = true;
 currentExtensions[51] = packedTable;
 
 currentExtensions[PACKED_REFERENCE_TAG_ID] = (data) => { // packed reference
+	if (!packedValues) {
+		if (currentDecoder.getShared)
+			loadShared();
+		else
+			return new Tag(data, PACKED_REFERENCE_TAG_ID)
+	}
 	if (typeof data == 'number')
 		return packedValues[16 + (data >= 0 ? 2 * data : (-2 * data - 1))]
 	throw new Error('No support for non-integer packed references yet')
 };
+currentExtensions[25] = (id) => {
+	return stringRefs[id]
+};
+currentExtensions[256] = (read) => {
+	stringRefs = [];
+	try {
+		return read()
+	} finally {
+		stringRefs = null;
+	}
+};
+currentExtensions[256].handlesRead = true;
 
-currentExtensions[40009] = (id) => {
-	// id extension (for structured clones)
-	if (!referenceMap)
+currentExtensions[28] = (read) => { 
+	// shareable http://cbor.schmorp.de/value-sharing (for structured clones)
+	if (!referenceMap) {
 		referenceMap = new Map();
+		referenceMap.id = 0;
+	}
+	let id = referenceMap.id++;
 	let token = src[position];
 	let target;
 	// TODO: handle Maps, Sets, and other types that can cycle; this is complicated, because you potentially need to read
@@ -795,9 +855,10 @@ currentExtensions[40009] = (id) => {
 	refEntry.target = targetProperties; // the placeholder wasn't used, replace with the deserialized one
 	return targetProperties // no cycle, can just use the returned read object
 };
+currentExtensions[28].handlesRead = true;
 
-currentExtensions[40010] = (id) => {
-	// pointer extension (for structured clones)
+currentExtensions[29] = (id) => {
+	// sharedref http://cbor.schmorp.de/value-sharing (for structured clones)
 	let refEntry = referenceMap.get(id);
 	refEntry.used = true;
 	return refEntry.target
@@ -820,19 +881,36 @@ function combine(a, b) {
 		return a.concat(b)
 	return Object.assign({}, a, b)
 }
+function getPackedValues() {
+	if (!packedValues) {
+		if (currentDecoder.getShared)
+			loadShared();
+		else
+			throw new Error('No packed values available')
+	}
+	return packedValues
+}
+const SHARED_DATA_TAG_ID = 0x53687264; // ascii 'Shrd'
 currentExtensionRanges.push((tag, input) => {
 	if (tag >= 225 && tag <= 255)
-		return combine(packedTable.prefixes[tag - 224], input)
+		return combine(getPackedValues().prefixes[tag - 224], input)
 	if (tag >= 28704 && tag <= 32767)
-		return combine(packedTable.prefixes[tag - 28672], input)
+		return combine(getPackedValues().prefixes[tag - 28672], input)
 	if (tag >= 1879052288 && tag <= 2147483647)
-		return combine(packedTable.prefixes[tag - 1879048192], input)
+		return combine(getPackedValues().prefixes[tag - 1879048192], input)
 	if (tag >= 216 && tag <= 223)
-		return combine(input, packedTable.suffixes[tag - 216])
+		return combine(input, getPackedValues().suffixes[tag - 216])
 	if (tag >= 27647 && tag <= 28671)
-		return combine(input, packedTable.suffixes[tag - 27639])
+		return combine(input, getPackedValues().suffixes[tag - 27639])
 	if (tag >= 1811940352 && tag <= 1879048191)
-		return combine(input, packedTable.suffixes[tag - 1811939328])
+		return combine(input, getPackedValues().suffixes[tag - 1811939328])
+	if (tag == SHARED_DATA_TAG_ID) {// we do a special check for this so that we can keep the currentExtensions as densely stored array (v8 stores arrays densely under about 3000 elements)
+		return {
+			packedValues: packedValues,
+			structures: currentStructures.slice(0),
+			version: input,
+		}
+	}
 });
 
 const typedArrays = ['Uint8', 'Uint8Clamped', 'Uint16', 'Uint32', 'BigUint64','Int8', 'Int16', 'Int32', 'BigInt64', 'Float32', 'Float64'].map(type => type + 'Array');
@@ -849,6 +927,61 @@ function registerTypedArray(typedArrayName, tag) {
 	};
 }
 
+function readBundleExt() {
+	let length = readJustLength();
+	let bundlePosition = position + read();
+	for (let i = 2; i < length; i++) {
+		// skip past bundles that were already read
+		let bundleLength = readJustLength(); // this will increment position, so must add to position afterwards
+		position += bundleLength;
+	}
+	let dataPosition = position;
+	position = bundlePosition;
+	bundledStrings = [readStringJS(readJustLength()), readStringJS(readJustLength())];
+	bundledStrings.position0 = 0;
+	bundledStrings.position1 = 0;
+	bundledStrings.postBundlePosition = position;
+	position = dataPosition;
+	return read()
+}
+
+function readJustLength() {
+	let token = src[position++] & 0x1f;
+	if (token > 0x17) {
+		switch (token) {
+			case 0x18:
+				token = src[position++];
+				break
+			case 0x19:
+				token = dataView.getUint16(position);
+				position += 2;
+				break
+			case 0x1a:
+				token = dataView.getUint32(position);
+				position += 4;
+				break
+		}
+	}
+	return token
+}
+
+function loadShared() {
+	if (currentDecoder.getShared) {
+		let sharedData = saveState(() => {
+			// save the state in case getShared modifies our buffer
+			src = null;
+			return currentDecoder.getShared()
+		}) || {};
+		let updatedStructures = sharedData.structures || [];
+		currentDecoder.sharedVersion = sharedData.version;
+		packedValues = currentDecoder.sharedValues = sharedData.packedValues;
+		if (currentStructures === true)
+			currentDecoder.structures = currentStructures = updatedStructures;
+		else
+			currentStructures.splice.apply(currentStructures, [0, updatedStructures.length].concat(updatedStructures));
+	}
+}
+
 function saveState(callback) {
 	let savedSrcEnd = srcEnd;
 	let savedPosition = position;
@@ -858,6 +991,8 @@ function saveState(callback) {
 	let savedSrcString = srcString;
 	let savedStrings = strings;
 	let savedReferenceMap = referenceMap;
+	let savedBundledStrings = bundledStrings;
+
 	// TODO: We may need to revisit this if we do more external calls to user code (since it could be slow)
 	let savedSrc = new Uint8Array(src.slice(0, srcEnd)); // we copy the data in case it changes while external data is processed
 	let savedStructures = currentStructures;
@@ -872,6 +1007,7 @@ function saveState(callback) {
 	srcString = savedSrcString;
 	strings = savedStrings;
 	referenceMap = savedReferenceMap;
+	bundledStrings = savedBundledStrings;
 	src = savedSrc;
 	sequentialMode = savedSequentialMode;
 	currentStructures = savedStructures;
@@ -918,13 +1054,15 @@ let extensions, extensionClasses;
 const hasNodeBuffer = typeof Buffer !== 'undefined';
 const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array;
 const ByteArray = hasNodeBuffer ? Buffer : Uint8Array;
-const RECORD_STARTING_ID_PREFIX = 0x69; // tag 105/0x69
 const MAX_STRUCTURES = 0x100;
 const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000;
 let target;
 let targetView;
 let position$1 = 0;
 let safeEnd;
+let bundledStrings$1 = null;
+const MAX_BUNDLE_SIZE = 0xf000;
+const hasNonLatin = /[\u0080-\uFFFF]/;
 const RECORD_SYMBOL = Symbol('record-id');
 class Encoder extends Decoder {
 	constructor(options) {
@@ -936,7 +1074,6 @@ class Encoder extends Decoder {
 		let structures;
 		let referenceMap;
 		options = options || {};
-		let lastSharedStructuresLength = 0;
 		let encodeUtf8 = ByteArray.prototype.utf8Write ? function(string, position, maxBytes) {
 			return target.utf8Write(string, position, maxBytes)
 		} : (textEncoder && textEncoder.encodeInto) ?
@@ -945,12 +1082,14 @@ class Encoder extends Decoder {
 			} : false;
 
 		let encoder = this;
-		let maxSharedStructures = 64;
+		let maxSharedStructures = options.maxSharedStructures || 128;
 		let isSequential = options.sequential;
 		if (isSequential) {
 			maxSharedStructures = 0;
 			this.structures = [];
 		}
+		if (this.saveStructures)
+			this.saveShared = this.saveStructures;
 		let samplingPackedValues, packedObjectMap, sharedValues = options.sharedValues;
 		let sharedPackedObjectMap;
 		if (sharedValues) {
@@ -958,13 +1097,29 @@ class Encoder extends Decoder {
 			for (let i = 0, l = sharedValues.length; i < l; i++) {
 				sharedPackedObjectMap[sharedValues[i]] = i;
 			}
-
 		}
 		let recordIdsToRemove = [];
 		let transitionsCount = 0;
 		let serializationsSinceTransitionRebuild = 0;
-
-		this.encode = function(value, encodeOptions) {
+		
+		this.mapEncode = function(value, encodeOptions) {
+			// Experimental support for premapping keys using _keyMap instad of keyMap - not optiimised yet)
+			if (this._keyMap && !this._mapped) {
+				//console.log('encoding ', value)
+				switch (value.constructor.name) {
+					case 'Array': 
+						value = value.map(r => this.encodeKeys(r));
+						break
+					//case 'Map': 
+					//	value = this.encodeKeys(value)
+					//	break
+				}
+				//this._mapped = true
+			}
+			return this.encode(value, encodeOptions)
+		};
+		
+		this.encode = function(value, encodeOptions)	{
 			if (!target) {
 				target = new ByteArrayAllocate(8192);
 				targetView = new DataView(target.buffer, 0, 8192);
@@ -981,10 +1136,25 @@ class Encoder extends Decoder {
 				position$1 = (position$1 + 7) & 0x7ffffff8; // Word align to make any future copying of this buffer faster
 			start = position$1;
 			referenceMap = encoder.structuredClone ? new Map() : null;
+			if (encoder.bundleStrings && typeof value !== 'string') {
+				bundledStrings$1 = [];
+				bundledStrings$1.size = Infinity; // force a new bundle start on first string
+			} else
+				bundledStrings$1 = null;
+
 			sharedStructures = encoder.structures;
 			if (sharedStructures) {
-				if (sharedStructures.uninitialized)
-					encoder.structures = sharedStructures = encoder.getStructures();
+				if (sharedStructures.uninitialized) {
+					let sharedData = encoder.getShared() || {};
+					encoder.structures = sharedStructures = sharedData.structures || [];
+					encoder.sharedVersion = sharedData.version;
+					let sharedValues = encoder.sharedValues = sharedData.packedValues;
+					if (sharedValues) {
+						sharedPackedObjectMap = {};
+						for (let i = 0, l = sharedValues.length; i < l; i++)
+							sharedPackedObjectMap[sharedValues[i]] = i;
+					}
+				}
 				let sharedStructuresLength = sharedStructures.length;
 				if (sharedStructuresLength > maxSharedStructures && !isSequential)
 					sharedStructuresLength = maxSharedStructures;
@@ -993,6 +1163,7 @@ class Encoder extends Decoder {
 					sharedStructures.transitions = Object.create(null);
 					for (let i = 0; i < sharedStructuresLength; i++) {
 						let keys = sharedStructures[i];
+						//console.log('shared struct keys:', keys)
 						if (!keys)
 							continue
 						let nextTransition, transition = sharedStructures.transitions;
@@ -1006,7 +1177,6 @@ class Encoder extends Decoder {
 						}
 						transition[RECORD_SYMBOL] = i;
 					}
-					lastSharedStructuresLength = sharedStructures.length;
 				}
 				if (!isSequential)
 					sharedStructures.nextId = sharedStructuresLength;
@@ -1039,9 +1209,12 @@ class Encoder extends Decoder {
 			}
 			try {
 				encode(value);
+				if (bundledStrings$1) {
+					writeBundles(start, encode);
+				}
 				encoder.offset = position$1; // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 				if (referenceMap && referenceMap.idsToInsert) {
-					position$1 += referenceMap.idsToInsert.length * 8;
+					position$1 += referenceMap.idsToInsert.length * 2;
 					if (position$1 > safeEnd)
 						makeRoom(position$1);
 					encoder.offset = position$1;
@@ -1049,12 +1222,12 @@ class Encoder extends Decoder {
 					referenceMap = null;
 					return serialized
 				}
-				if (encodeOptions === REUSE_BUFFER_MODE) {
+				if (encodeOptions & REUSE_BUFFER_MODE) {
 					target.start = start;
 					target.end = position$1;
 					return target
 				}
-				return target.subarray(start, position$1) // position can change if we call pack again in saveStructures, so we get the buffer now
+				return target.subarray(start, position$1) // position can change if we call pack again in saveShared, so we get the buffer now
 			} finally {
 				if (sharedStructures) {
 					if (serializationsSinceTransitionRebuild < 10)
@@ -1072,34 +1245,27 @@ class Encoder extends Decoder {
 						}
 						recordIdsToRemove = [];
 					}
-					if (hasSharedUpdate && encoder.saveStructures) {
-						if (encoder.structures.length > maxSharedStructures) {
-							encoder.structures = encoder.structures.slice(0, maxSharedStructures);
-						}
-						// we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
-						let returnBuffer = target.subarray(start, position$1);
-						let shared = encoder.structures || [];
-						if (sharedValues) {
-							shared = shared.concat(sharedValues);
-						}
-
-						if (encoder.saveStructures(encoder.structures, lastSharedStructuresLength) === false) {
-							// get updated structures and try again if the update failed
-							encoder.structures = encoder.getStructures() || [];
-							return encoder.encode(value)
-						}
-						lastSharedStructuresLength = shared.length;
-						return returnBuffer
+				}
+				if (hasSharedUpdate && encoder.saveShared) {
+					if (encoder.structures.length > maxSharedStructures) {
+						encoder.structures = encoder.structures.slice(0, maxSharedStructures);
 					}
+					// we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
+					let returnBuffer = target.subarray(start, position$1);
+					if (encoder.updateSharedData() === false)
+						return encoder.encode(value) // re-encode if it fails
+					return returnBuffer
 				}
+				if (encodeOptions & RESET_BUFFER_MODE)
+					position$1 = start;
 			}
 		};
 		this.findCommonStringsToPack = () => {
 			samplingPackedValues = new Map();
 			if (!sharedPackedObjectMap)
 				sharedPackedObjectMap = Object.create(null);
-			return ({ threshold }) => {
-				threshold = threshold || 4;
+			return (options) => {
+				let threshold = options && options.threshold || 4;
 				let position = this.pack ? options.maxPrivatePackedValues || 16 : 0;
 				if (!sharedValues)
 					sharedValues = this.sharedValues = [];
@@ -1110,6 +1276,7 @@ class Encoder extends Decoder {
 						hasSharedUpdate = true;
 					}
 				}
+				while (this.saveShared && this.updateSharedData() === false) {}
 				samplingPackedValues = null;
 			}
 		};
@@ -1158,6 +1325,33 @@ class Encoder extends Decoder {
 					}
 				}
 				let strLength = value.length;
+				if (bundledStrings$1 && strLength >= 4 && strLength < 0x400) {
+					if ((bundledStrings$1.size += strLength) > MAX_BUNDLE_SIZE) {
+						let extStart;
+						let maxBytes = (bundledStrings$1[0] ? bundledStrings$1[0].length * 3 + bundledStrings$1[1].length : 0) + 10;
+						if (position$1 + maxBytes > safeEnd)
+							target = makeRoom(position$1 + maxBytes);
+						target[position$1++] = 0xd9; // tag 16-bit
+						target[position$1++] = 0xdf; // tag 0xdff9
+						target[position$1++] = 0xf9;
+						// TODO: If we only have one bundle with any string data, only write one string bundle
+						target[position$1++] = bundledStrings$1.position ? 0x84 : 0x82; // array of 4 or 2 elements depending on if we write bundles
+						target[position$1++] = 0x1a; // 32-bit unsigned int
+						extStart = position$1 - start;
+						position$1 += 4; // reserve for writing bundle reference
+						if (bundledStrings$1.position) {
+							writeBundles(start, encode); // write the last bundles
+						}
+						bundledStrings$1 = ['', '']; // create new ones
+						bundledStrings$1.size = 0;
+						bundledStrings$1.position = extStart;
+					}
+					let twoByte = hasNonLatin.test(value);
+					bundledStrings$1[twoByte ? 0 : 1] += value;
+					target[position$1++] = twoByte ? 0xce : 0xcf;
+					encode(strLength);
+					return
+				}
 				let headerSize;
 				// first we estimate the header size, so we can write to the correct location
 				if (strLength < 0x20) {
@@ -1284,16 +1478,16 @@ class Encoder extends Decoder {
 					if (referenceMap) {
 						let referee = referenceMap.get(value);
 						if (referee) {
-							if (!referee.id) {
+							target[position$1++] = 0xd8;
+							target[position$1++] = 29; // http://cbor.schmorp.de/value-sharing
+							target[position$1++] = 0x19; // 16-bit uint
+							if (!referee.references) {
 								let idsToInsert = referenceMap.idsToInsert || (referenceMap.idsToInsert = []);
-								referee.id = idsToInsert.push(referee);
+								referee.references = [];
+								idsToInsert.push(referee);
 							}
-							target[position$1++] = 0xd9;
-							target[position$1++] = 40010 >> 8;
-							target[position$1++] = 40010 & 0xff;
-							target[position$1++] = 0x1a; // uint32
-							targetView.setUint32(position$1, referee.id);
-							position$1 += 4;
+							referee.references.push(position$1 - start);
+							position$1 += 2; // TODO: also support 32-bit
 							return
 						} else 
 							referenceMap.set(value, { offset: position$1 - start });
@@ -1333,16 +1527,23 @@ class Encoder extends Decoder {
 							targetView.setUint32(position$1, length);
 							position$1 += 4;
 						}
-						for (let [ key, entryValue ] of value) {
-							encode(key);
-							encode(entryValue);
+						if (encoder.keyMap) { 
+							for (let [ key, entryValue ] of value) {
+								encode(encoder.encodeKey(key));
+								encode(entryValue);
+							} 
+						} else { 
+							for (let [ key, entryValue ] of value) {
+								encode(key); 
+								encode(entryValue);
+							} 	
 						}
-					} else {	
+					} else {
 						for (let i = 0, l = extensions.length; i < l; i++) {
 							let extensionClass = extensionClasses[i];
 							if (value instanceof extensionClass) {
 								let extension = extensions[i];
-								let tag = extension.tag;
+								let tag = extension.tag || extension.getTag && extension.getTag(value);
 								if (tag < 0x18) {
 									target[position$1++] = 0xc0 | tag;
 								} else if (tag < 0x100) {
@@ -1404,6 +1605,7 @@ class Encoder extends Decoder {
 		const writeObject = this.useRecords === false ? this.variableMapSize ? (object) => {
 			// this method is slightly slower, but generates "preferred serialization" (optimally small for smaller objects)
 			let keys = Object.keys(object);
+			let vals = Object.values(object);
 			let length = keys.length;
 			if (length < 0x18) {
 				target[position$1++] = 0xa0 | length;
@@ -1419,10 +1621,16 @@ class Encoder extends Decoder {
 				targetView.setUint32(position$1, length);
 				position$1 += 4;
 			}
-			let key;
-			for (let i = 0; i < length; i++) {
-				encode(key = keys[i]);
-				encode(object[key]);
+			if (encoder.keyMap) { 
+				for (let i = 0; i < length; i++) {
+					encode(encodeKey(keys[i]));
+					encode(vals[i]);
+				}
+			} else {
+				for (let i = 0; i < length; i++) {
+					encode(keys[i]);
+					encode(vals[i]);
+				}
 			}
 		} :
 		(object, safePrototype) => {
@@ -1430,12 +1638,18 @@ class Encoder extends Decoder {
 			let objectOffset = position$1 - start;
 			position$1 += 2;
 			let size = 0;
-			for (let key in object) {
-				if (safePrototype || object.hasOwnProperty(key)) {
-					encode(key);
+			if (encoder.keyMap) { 
+				for (let key in object) if (safePrototype || object.hasOwnProperty(key)) {
+					encode(encoder.encodeKey(key));
 					encode(object[key]);
 					size++;
 				}
+			} else { 
+				for (let key in object) if (safePrototype || object.hasOwnProperty(key)) {
+						encode(key);
+						encode(object[key]);
+					size++;
+				}
 			}
 			target[objectOffset++ + start] = size >> 8;
 			target[objectOffset + start] = size & 0xff;
@@ -1483,11 +1697,16 @@ class Encoder extends Decoder {
 		}*/
 		(object) => {
 			let keys = Object.keys(object);
+			let vals = Object.values(object);
+			if (this.keyMap) keys = keys.map(k => this.encodeKey(k));
 			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null));
 			let newTransitions = 0;
 			let length = keys.length;
+			//let parentRecordId
 			for (let i = 0; i < length; i++) {
+				//if (!parentRecordId)
 				let key = keys[i];
+				//	parentRecordId = transition[RECORD_SYMBOL]
 				nextTransition = transition[key];
 				if (!nextTransition) {
 					nextTransition = transition[key] = Object.create(null);
@@ -1497,9 +1716,9 @@ class Encoder extends Decoder {
 			}
 			let recordId = transition[RECORD_SYMBOL];
 			if (recordId !== undefined) {
-				target[position$1++] = 0xd9; // tag two byte
-				target[position$1++] = RECORD_STARTING_ID_PREFIX;
-				target[position$1++] = recordId;
+				target[position$1++] = 0xd9;
+				target[position$1++] = (recordId >> 8) | 0xe0;
+				target[position$1++] = recordId & 0xff;
 			} else {
 				recordId = structures.nextId++;
 				if (!recordId) {
@@ -1512,13 +1731,13 @@ class Encoder extends Decoder {
 				transition[RECORD_SYMBOL] = recordId;
 				structures[recordId] = keys;
 				if (sharedStructures && sharedStructures.length <= maxSharedStructures) {
-					target[position$1++] = 0xd9; // tag two byte
-					target[position$1++] = RECORD_STARTING_ID_PREFIX;
-					target[position$1++] = recordId; // tag number
+					target[position$1++] = 0xd9;
+					target[position$1++] = (recordId >> 8) | 0xe0;
+					target[position$1++] = recordId & 0xff;
 					hasSharedUpdate = true;
 				} else {
-					target[position$1++] = 0xd8;
-					target[position$1++] = RECORD_STARTING_ID_PREFIX;
+					targetView.setUint32(position$1, 0xd9dfff00); // tag two byte, then record definition id
+					position$1 += 3;
 					if (newTransitions)
 						transitionsCount += serializationsSinceTransitionRebuild * newTransitions;
 					// record the removal of the id, we can maintain our shared structure
@@ -1526,13 +1745,9 @@ class Encoder extends Decoder {
 						recordIdsToRemove.shift()[RECORD_SYMBOL] = undefined; // we are cycling back through, and have to remove old ones
 					recordIdsToRemove.push(transition);
 					writeArrayHeader(length + 2);
+					encode(0xe000 + recordId);
 					encode(keys);
-					target[position$1++] = 0x19; // uint16
-					target[position$1++] = RECORD_STARTING_ID_PREFIX;
-					target[position$1++] = recordId;
-					// now write the values
-					for (let i =0; i < length; i++)
-						encode(object[keys[i]]);
+					for (let v of Object.values(object)) encode(v);
 					return
 				}
 			}
@@ -1541,8 +1756,7 @@ class Encoder extends Decoder {
 			} else {
 				writeArrayHeader(length);
 			}
-			for (let i =0; i < length; i++)
-				encode(object[keys[i]]);
+			for (let i =0; i < length; i++) encode(vals[i]);
 		};
 		const makeRoom = (end) => {
 			let newSize;
@@ -1572,6 +1786,33 @@ class Encoder extends Decoder {
 		targetView = new DataView(target.buffer, target.byteOffset, target.byteLength);
 		position$1 = 0;
 	}
+	clearSharedData() {
+		if (this.structures)
+			this.structures = [];
+		if (this.sharedValues)
+			this.sharedValues = undefined;
+	}
+	updateSharedData() {
+		let lastVersion = this.sharedVersion || 0;
+		this.sharedVersion = lastVersion + 1;
+		let saveResults = this.saveShared(new SharedData(this.structures, this.sharedValues, this.sharedVersion),
+				existingShared => (existingShared && existingShared.version || 0) == lastVersion);
+		if (saveResults === false) {
+			// get updated structures and try again if the update failed
+			let sharedData = this.getShared() || {};
+			this.structures = sharedData.structures || [];
+			this.sharedValues = sharedData.packedValues;
+			this.sharedVersion = sharedData.version;
+		}
+		return saveResults
+	}
+}
+class SharedData {
+	constructor(structures, values, version) {
+		this.structures = structures;
+		this.packedValues = values;
+		this.version = version;
+	}
 }
 
 function writeArrayHeader(length) {
@@ -1641,11 +1882,11 @@ function findRepetitiveStrings(value, packedValues) {
 	}
 }
 
-extensionClasses = [ Date, Set, Error, RegExp, ArrayBuffer, ByteArray,
+extensionClasses = [ Date, Set, Error, RegExp, Tag, ArrayBuffer, ByteArray,
 	Uint8Array, Uint8ClampedArray, Uint16Array, Uint32Array,
 	typeof BigUint64Array == 'undefined' ? function() {} : BigUint64Array, Int8Array, Int16Array, Int32Array,
 	typeof BigInt64Array == 'undefined' ? function() {} : BigInt64Array,
-	Float32Array, Float64Array];
+	Float32Array, Float64Array, SharedData];
 
 //Object.getPrototypeOf(Uint8Array.prototype).constructor /*TypedArray*/
 extensions = [{
@@ -1680,6 +1921,13 @@ extensions = [{
 	encode(regex, encode) {
 		encode([ 'RegExp', regex.source, regex.flags ]);
 	}
+}, {
+	getTag(tag) {
+		return tag.tag
+	},
+	encode(tag, encode) {
+		encode(tag.value);
+	}
 }, {
 	encode(arrayBuffer, encode, makeRoom) {
 		writeBuffer(arrayBuffer, makeRoom);
@@ -1698,7 +1946,35 @@ extensions = [{
 	typedArrayEncoder(78),
 	typedArrayEncoder(79),
 	typedArrayEncoder(81),
-	typedArrayEncoder(82)];
+	typedArrayEncoder(82),
+{
+	encode(sharedData, encode) { // write SharedData
+		let packedValues = sharedData.packedValues || [];
+		let sharedStructures = sharedData.structures || [];
+		if (packedValues.values.length > 0) {
+			target[position$1++] = 0xd8; // one-byte tag
+			target[position$1++] = 51; // tag 51 for packed shared structures https://www.potaroo.net/ietf/ids/draft-ietf-cbor-packed-03.txt
+			writeArrayHeader(4);
+			let valuesArray = packedValues.values;
+			encode(valuesArray);
+			writeArrayHeader(0); // prefixes
+			writeArrayHeader(0); // suffixes
+			packedObjectMap = Object.create(sharedPackedObjectMap || null);
+			for (let i = 0, l = valuesArray.length; i < l; i++) {
+				packedObjectMap[valuesArray[i]] = i;
+			}
+		}
+		if (sharedStructures) {
+			targetView.setUint32(position$1, 0xd9dffe00);
+			position$1 += 3;
+			let definitions = sharedStructures.slice(0);
+			definitions.unshift(0xe000);
+			definitions.push(new Tag(sharedData.version, 0x53687264));
+			encode(definitions);
+		} else
+			encode(new Tag(sharedData.version, 0x53687264));
+		}
+	}];
 
 function typedArrayEncoder(tag) {
 	return {
@@ -1738,27 +2014,35 @@ function writeBuffer(buffer, makeRoom) {
 function insertIds(serialized, idsToInsert) {
 	// insert the ids that need to be referenced for structured clones
 	let nextId;
-	let distanceToMove = idsToInsert.length * 8;
+	let distanceToMove = idsToInsert.length * 2;
 	let lastEnd = serialized.length - distanceToMove;
 	idsToInsert.sort((a, b) => a.offset > b.offset ? 1 : -1);
+	for (let id = 0; id < idsToInsert.length; id++) {
+		let referee = idsToInsert[id];
+		referee.id = id;
+		for (let position of referee.references) {
+			serialized[position++] = id >> 8;
+			serialized[position] = id & 0xff;
+		}
+	}
 	while (nextId = idsToInsert.pop()) {
 		let offset = nextId.offset;
-		let id = nextId.id;
 		serialized.copyWithin(offset + distanceToMove, offset, lastEnd);
-		distanceToMove -= 8;
+		distanceToMove -= 2;
 		let position = offset + distanceToMove;
-		serialized[position++] = 0xd9;
-		serialized[position++] = 40009 >> 8;
-		serialized[position++] = 40009 & 0xff;
-		serialized[position++] = 0x1a; // uint32
-		serialized[position++] = id >> 24;
-		serialized[position++] = (id >> 16) & 0xff;
-		serialized[position++] = (id >> 8) & 0xff;
-		serialized[position++] = id & 0xff;
+		serialized[position++] = 0xd8;
+		serialized[position++] = 28; // http://cbor.schmorp.de/value-sharing
 		lastEnd = offset;
 	}
 	return serialized
 }
+function writeBundles(start, encode) {
+	targetView.setUint32(bundledStrings$1.position + start, position$1 - bundledStrings$1.position - start + 1); // the offset to bundle
+	let writeStrings = bundledStrings$1;
+	bundledStrings$1 = null;
+	encode(writeStrings[0]);
+	encode(writeStrings[1]);
+}
 
 function addExtension$1(extension) {
 	if (extension.Class) {
@@ -1772,7 +2056,8 @@ function addExtension$1(extension) {
 let defaultEncoder = new Encoder({ useRecords: false });
 const encode = defaultEncoder.encode;
 const { NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } = FLOAT32_OPTIONS;
-const REUSE_BUFFER_MODE = 1000;
+const REUSE_BUFFER_MODE = 512;
+const RESET_BUFFER_MODE = 1024;
 
 class EncoderStream extends stream.Transform {
 	constructor(options) {