msgpackr 1.7.0-alpha2 → 1.7.0-alpha5

package/dist/test.js

@@ -1,7 +1,8 @@
-(function (msgpackr, chai, fs) {
+(function (msgpackr, chai, inspector, fs) {
 	'use strict';
 
 	chai = chai && Object.prototype.hasOwnProperty.call(chai, 'default') ? chai['default'] : chai;
+	inspector = inspector && Object.prototype.hasOwnProperty.call(inspector, 'default') ? inspector['default'] : inspector;
 
 	var decoder;
 	try {
@@ -28,7 +29,7 @@
 	C1.name = 'MessagePack 0xC1';
 	var sequentialMode = false;
 	var inlineObjectReadThreshold = 2;
-	var readStruct;
+	var readStruct, onLoadedStructures;
 	try {
 		new Function('');
 	} catch(error) {
@@ -58,16 +59,21 @@
 			}
 			Object.assign(this, options);
 		}
-		unpack(source, end) {
+		unpack(source, options) {
 			if (src) {
 				// re-entrant execution, save the state and restore it after we do this unpack
 				return saveState(() => {
 					clearSource();
-					return this ? this.unpack(source, end) : Unpackr.prototype.unpack.call(defaultOptions, source, end)
+					return this ? this.unpack(source, options) : Unpackr.prototype.unpack.call(defaultOptions, source, options)
 				})
 			}
-			srcEnd = end > -1 ? end : source.length;
-			position = 0;
+			if (typeof options === 'object') {
+				srcEnd = options.end || source.length;
+				position = options.start || 0;
+			} else {
+				position = 0;
+				srcEnd = options > -1 ? options : source.length;
+			}
 			srcStringEnd = 0;
 			srcString = null;
 			bundledStrings = null;
@@ -88,7 +94,7 @@
 				currentUnpackr = this;
 				if (this.structures) {
 					currentStructures = this.structures;
-					return checkedRead()
+					return checkedRead(options)
 				} else if (!currentStructures || currentStructures.length > 0) {
 					currentStructures = [];
 				}
@@ -97,7 +103,7 @@
 				if (!currentStructures || currentStructures.length > 0)
 					currentStructures = [];
 			}
-			return checkedRead()
+			return checkedRead(options)
 		}
 		unpackMultiple(source, forEach) {
 			let values, lastPosition = 0;
@@ -132,7 +138,11 @@
 			}
 		}
 		_mergeStructures(loadedStructures, existingStructures) {
+			if (onLoadedStructures)
+				loadedStructures = onLoadedStructures.call(this, loadedStructures);
 			loadedStructures = loadedStructures || [];
+			if (Object.isFrozen(loadedStructures))
+				loadedStructures = loadedStructures.map(structure => structure.slice(0));
 			for (let i = 0, l = loadedStructures.length; i < l; i++) {
 				let structure = loadedStructures[i];
 				if (structure) {
@@ -159,7 +169,7 @@
 			return this.unpack(source, end)
 		}
 	}
-	function checkedRead() {
+	function checkedRead(options) {
 		try {
 			if (!currentUnpackr.trusted && !sequentialMode) {
 				let sharedLength = currentStructures.sharedLength || 0;
@@ -167,9 +177,10 @@
 					currentStructures.length = sharedLength;
 			}
 			let result;
-			if (currentUnpackr.randomAccessStructure && src[position] < 0x40 && readStruct) {
-				let id = (src[position++] << 8) + src[position++];
-				result = readStruct(src, position, srcEnd, currentStructures[id - 0x40] || loadStructures()[id - 0x40], currentUnpackr);
+			if (currentUnpackr.randomAccessStructure && src[position] < 0x40 && src[position] >= 0x20 && readStruct) {
+				result = readStruct(src, position, srcEnd, currentUnpackr);
+				if (!(options && options.lazy) && result)
+					result = result.toJSON();
 				position = srcEnd;
 			} else
 				result = read();
@@ -565,6 +576,16 @@
 
 		return result
 	}
+	function readString(source, start, length) {
+		let existingSrc = src;
+		src = source;
+		position = start;
+		try {
+			return readStringJS(length);
+		} finally {
+			src = existingSrc;
+		}
+	}
 
 	function readArray(length) {
 		let array = new Array(length);
@@ -785,7 +806,15 @@
 	function readExt(length) {
 		let type = src[position++];
 		if (currentExtensions[type]) {
-			return currentExtensions[type](src.subarray(position, position += length))
+			let end;
+			return currentExtensions[type](src.subarray(position, end = (position += length)), (readPosition) => {
+				position = readPosition;
+				try {
+					return read();
+				} finally {
+					position = end;
+				}
+			})
 		}
 		else
 			throw new Error('Unknown extension type ' + type)
@@ -1000,8 +1029,9 @@
 		mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103));
 	}
 	var defaultUnpackr = new Unpackr({ useRecords: false });
-	function setReadStruct(func) {
-		readStruct = func;
+	function setReadStruct(updatedReadStruct, loadedStructs) {
+		readStruct = updatedReadStruct;
+		onLoadedStructures = loadedStructs;
 	}
 
 	let textEncoder;
@@ -1031,7 +1061,6 @@
 			let hasSharedUpdate;
 			let structures;
 			let referenceMap;
-			let lastSharedStructuresLength = 0;
 			let encodeUtf8 = ByteArray.prototype.utf8Write ? function(string, position) {
 				return target.utf8Write(string, position, 0xffffffff)
 			} : (textEncoder && textEncoder.encodeInto) ?
@@ -1117,7 +1146,7 @@
 							}
 							transition[RECORD_SYMBOL] = i + 0x40;
 						}
-						lastSharedStructuresLength = sharedLength;
+						this.lastNamedStructuresLength = sharedLength;
 					}
 					if (!isSequential) {
 						structures.nextId = sharedLength + 0x40;
@@ -1153,7 +1182,7 @@
 					if (structures) {
 						if (serializationsSinceTransitionRebuild < 10)
 							serializationsSinceTransitionRebuild++;
-						let sharedLength = structures.sharedLength || maxSharedStructures;
+						let sharedLength = structures.sharedLength || 0;
 						if (structures.length > sharedLength)
 							structures.length = sharedLength;
 						if (transitionsCount > 10000) {
@@ -1172,12 +1201,12 @@
 						if (hasSharedUpdate && packr.saveStructures) {
 							// we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
 							let returnBuffer = target.subarray(start, position$1);
-							if (packr.saveStructures(structures, lastSharedStructuresLength) === false) {
+							let newSharedData = prepareStructures(structures, packr);
+							if (packr.saveStructures(newSharedData, newSharedData.isCompatible) === false) {
 								// get updated structures and try again if the update failed
-								packr._mergeStructures(packr.getStructures());
 								return packr.pack(value)
 							}
-							lastSharedStructuresLength = sharedLength;
+							packr.lastNamedStructuresLength = sharedLength;
 							return returnBuffer
 						}
 					}
@@ -1295,7 +1324,7 @@
 				} else if (type === 'number') {
 					if (value >>> 0 === value) {// positive integer, 32-bit or less
 						// positive uint
-						if (value < 0x40 || (value < 0x80 && this.useRecords === false)) {
+						if (value < 0x20 || (value < 0x80 && this.useRecords === false) || (value < 0x40 && !this.randomAccessStructure)) {
 							target[position$1++] = value;
 						} else if (value < 0x100) {
 							target[position$1++] = 0xcc;
@@ -1695,16 +1724,18 @@
 				}
 			};
 			const writeStruct = (object, safePrototype) => {
-				let newPosition = writeStructSlots(object, target, position$1, structures, makeRoom, (value, newPosition) => {
+				let newPosition = writeStructSlots(object, target, position$1, structures, makeRoom, (value, newPosition, notifySharedUpdate) => {
+					if (notifySharedUpdate)
+						return hasSharedUpdate = true;
 					position$1 = newPosition;
 					if (start > 0) {
 						pack(value);
 						if (start == 0)
-							return { position: position$1, targetView }; // indicate the buffer was re-allocated
+							return { position: position$1, targetView, target }; // indicate the buffer was re-allocated
 					} else
 						pack(value);
 					return position$1;
-				});
+				}, this);
 				if (newPosition === 0) // bail and go to a msgpack object
 					return writeObject(object, true);
 				position$1 = newPosition;
@@ -1719,6 +1750,8 @@
 		clearSharedData() {
 			if (this.structures)
 				this.structures = [];
+			if (this.typedStructs)
+				this.typedStructs = [];
 		}
 	}
 
@@ -1926,247 +1959,699 @@
 			pack(writeStrings[1]);
 		}
 	}
-	function setWriteStructSlots(func) {
-		writeStructSlots = func;
+	function prepareStructures(structures, packr) {
+		structures.isCompatible = (existingStructures) => {
+			let compatible = !existingStructures || ((packr.lastNamedStructuresLength || 0) === existingStructures.length);
+			if (!compatible) // we want to merge these existing structures immediately since we already have it and we are in the right transaction
+				packr._mergeStructures(existingStructures);
+			return compatible;
+		};
+		return structures
+	}
+	function setWriteStructSlots(writeSlots, makeStructures) {
+		writeStructSlots = writeSlots;
+		prepareStructures = makeStructures;
 	}
 
 	let defaultPackr = new Packr({ useRecords: false });
 	const REUSE_BUFFER_MODE = 512;
 	const RESET_BUFFER_MODE = 1024;
 
-	// first four bits
-	const hasNonLatin$1 = /[\u0080-\uFFFF]/;
+	const ASCII = 3; // the MIBenum from https://www.iana.org/assignments/character-sets/character-sets.xhtml (and other character encodings could be referenced by MIBenum)
+	const NUMBER = 0;
+	const UTF8 = 2;
+	const OBJECT_DATA = 1;
+	const TYPE_NAMES = ['num', 'object', 'string', 'ascii'];
 	const float32Headers = [false, true, true, false, false, true, true, false];
-	setWriteStructSlots(writeStruct);
-	function writeStruct(object, target, position, structures, makeRoom, pack) {
-		let transition = structures.transitions || false;
-		let start = position;
-		position += 4;
-		let queuedReferences = [];
-		let uint32 = target.uint32 || (target.uint32 = new Uint32Array(target.buffer));
+	let updatedPosition;
+	const hasNodeBuffer$1 = typeof Buffer !== 'undefined';
+	let textEncoder$1;
+	try {
+		textEncoder$1 = new TextEncoder();
+	} catch (error) {}
+	const encodeUtf8 = hasNodeBuffer$1 ? function(target, string, position) {
+		return target.utf8Write(string, position, 0xffffffff)
+	} : (textEncoder$1 && textEncoder$1.encodeInto) ?
+		function(target, string, position) {
+			return textEncoder$1.encodeInto(string, target.subarray(position)).written
+		} : false;
+	setWriteStructSlots(writeStruct, prepareStructures$1);
+	function writeStruct(object, target, position, structures, makeRoom, pack, packr) {
+		let typedStructs = packr.typedStructs || (packr.typedStructs = []);
+		// note that we rely on pack.js to load stored structures before we get to this point
 		let targetView = target.dataView;
-		let encoded;
-		let stringData = '';
+		let refsStartPosition = (typedStructs.lastStringStart || 100) + position;
 		let safeEnd = target.length - 10;
+		let start = position;
+		if (position > safeEnd) {
+			let lastStart = start;
+			target = makeRoom(position);
+			targetView = target.dataView;
+			position -= lastStart;
+			refsStartPosition -= lastStart;
+			start = 0;
+			safeEnd = target.length - 10;
+		}
+
+		let refOffset, refPosition = refsStartPosition;
+
+		let transition = typedStructs.transitions || (typedStructs.transitions = Object.create(null));
+		let nextId = typedStructs.nextId || typedStructs.length;
+		let headerSize =
+			nextId < 0xf ? 1 :
+				nextId < 0xf0 ? 2 :
+					nextId < 0xf000 ? 3 :
+						nextId < 0xf00000 ? 4 : 0;
+		if (headerSize === 0)
+			return 0;
+		position += headerSize;
+		let queuedReferences = [];
+		let keyIndex = 0;
 		for (let key in object) {
+			let value = object[key];
 			let nextTransition = transition[key];
 			if (!nextTransition) {
-				return 0; // bail
-				//nextTransition = transition[key] = Object.create(null)
-				//newTransitions++
+				transition[key] = nextTransition = {
+					key,
+					parent: transition,
+					enumerationOffset: 0,
+					ascii0: null,
+					ascii8: null,
+					num8: null,
+					string16: null,
+					object16: null,
+					num32: null,
+					float64: null
+				};
 			}
 			if (position > safeEnd) {
-				let newPosition = position - start;
+				let lastStart = start;
 				target = makeRoom(position);
-				position = newPosition;
+				targetView = target.dataView;
+				position -= lastStart;
+				refsStartPosition -= lastStart;
+				refPosition -= lastStart;
 				start = 0;
 				safeEnd = target.length - 10;
 			}
-			transition = nextTransition;
-			let value = object[key];
 			switch (typeof value) {
 				case 'number':
-					if (value >>> 0 === value && value < 0x20000000) {
-						encoded = value;
+					let number = value;
+					if (number >> 0 === number && number < 0x20000000 && number > -0x1f000000) {
+						if (number < 0xf6 && number >= 0 && (nextTransition.num8 || number < 0x20 && !nextTransition.num32)) {
+							transition = nextTransition.num8 || createTypeTransition(nextTransition, NUMBER, 1);
+							target[position++] = number;
+						} else {
+							transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4);
+							targetView.setUint32(position, number, true);
+							position += 4;
+						}
 						break;
-					} else if (value < 0x100000000 && value >= -0x80000000) {
-						targetView.setFloat32(position, value, true);
+					} else if (number < 0x100000000 && number >= -0x80000000) {
+						targetView.setFloat32(position, number, true);
 						if (float32Headers[target[position + 3] >>> 5]) {
 							let xShifted;
 							// this checks for rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
-							if (((xShifted = value * mult10[((target[position + 3] & 0x7f) << 1) | (target[position + 2] >> 7)]) >> 0) === xShifted) {
+							if (((xShifted = number * mult10[((target[position + 3] & 0x7f) << 1) | (target[position + 2] >> 7)]) >> 0) === xShifted) {
+								transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4);
 								position += 4;
-								continue;
+								break;
 							}
 						}
 					}
-					// fall back to msgpack encoding
-					queuedReferences.push(value, position - start);
-					position += 4;
-					continue;
+					transition = nextTransition.num64 || createTypeTransition(nextTransition, NUMBER, 8);
+					targetView.setFloat64(position, number, true);
+					position += 8;
+					break;
 				case 'string':
-					if (hasNonLatin$1.test(value)) {
-						queuedReferences.push(value, position - start);
-						position += 4;
-						continue;
+					let strLength = value.length;
+					refOffset = refPosition - refsStartPosition;
+					if ((strLength << 2) + position > safeEnd) {
+						let lastStart = start;
+						target = makeRoom(refPosition);
+						targetView = target.dataView;
+						position -= lastStart;
+						refsStartPosition -= lastStart;
+						refPosition -= lastStart;
+						start = 0;
+						safeEnd = target.length - 10;
 					}
-					if (value.length < 4) { // we can inline really small strings
-						encoded = 0xf8000000 + (value.length << 24) + (value.charCodeAt(0) << 16) + (value.charCodeAt(1) << 8) + (value.charCodeAt(2) || 0);
-						// TODO: determining remaining and make max value be a ratio of that (probably 1/256th)
-					} else if (value.length < 256 && stringData.length < 61440) {
-						// bundle these strings
-						encoded = 0x60000000 | (value.length << 16) | stringData.length;
-						stringData += value;
-					} else { // else queue it
-						queuedReferences.push(value, position - start);
-						position += 4;
-						continue;
+					if (strLength > ((0xff00 + refOffset) >> 2)) {
+						queuedReferences.push(key, value, position - start);
+						break;
+					}
+					let isNotAscii;
+					let strStart = refPosition;
+					if (strLength < 0x40) {
+						let i, c1, c2;
+						for (i = 0; i < strLength; i++) {
+							c1 = value.charCodeAt(i);
+							if (c1 < 0x80) {
+								target[refPosition++] = c1;
+							} else if (c1 < 0x800) {
+								isNotAscii = true;
+								target[refPosition++] = c1 >> 6 | 0xc0;
+								target[refPosition++] = c1 & 0x3f | 0x80;
+							} else if (
+								(c1 & 0xfc00) === 0xd800 &&
+								((c2 = value.charCodeAt(i + 1)) & 0xfc00) === 0xdc00
+							) {
+								isNotAscii = true;
+								c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff);
+								i++;
+								target[refPosition++] = c1 >> 18 | 0xf0;
+								target[refPosition++] = c1 >> 12 & 0x3f | 0x80;
+								target[refPosition++] = c1 >> 6 & 0x3f | 0x80;
+								target[refPosition++] = c1 & 0x3f | 0x80;
+							} else {
+								isNotAscii = true;
+								target[refPosition++] = c1 >> 12 | 0xe0;
+								target[refPosition++] = c1 >> 6 & 0x3f | 0x80;
+								target[refPosition++] = c1 & 0x3f | 0x80;
+							}
+						}
+					} else {
+						refPosition += encodeUtf8(target, value, refPosition);
+						isNotAscii = refPosition - strStart > strLength;
+					}
+					if (refOffset < 0x100) {
+						if (isNotAscii)
+							transition = nextTransition.string8 || createTypeTransition(nextTransition, UTF8, 1);
+						else
+							transition = nextTransition.ascii8 || createTypeTransition(nextTransition, ASCII, 1);
+						target[position++] = refOffset;
+					} else {
+						if (isNotAscii)
+							transition = nextTransition.string16 || createTypeTransition(nextTransition, UTF8, 2);
+						else
+							transition = nextTransition.ascii16 || createTypeTransition(nextTransition, ASCII, 2);
+						targetView.setUint16(position, refOffset, true);
+						position += 2;
 					}
 					break;
 				case 'object':
 					if (value) {
-						queuedReferences.push(value, position - start);
-						position += 4;
-						continue;
+						//transition = nextTransition.object16 || createTypeTransition(nextTransition, OBJECT_DATA, 2);
+						queuedReferences.push(key, value, keyIndex);
+						break;
 					} else { // null
-						encoded = 0xe0000000;
+						nextTransition = anyType(nextTransition, position, targetView, -10); // match CBOR with this
+						if (nextTransition) {
+							transition = nextTransition;
+							position = updatedPosition;
+						} else queuedReferences.push(key, value, keyIndex);
 					}
 					break;
 				case 'boolean':
-					encoded = value ? 0xe3000000 : 0xe2000000;
+					transition = nextTransition.num8 || nextTransition.ascii8 || createTypeTransition(nextTransition, NUMBER, 1);
+					target[position++] = value ? 0xf9 : 0xf8; // match CBOR with these
 					break;
 				case 'undefined':
-					encoded = 0xe1000000;
+					nextTransition = anyType(nextTransition, position, targetView, -9); // match CBOR with this
+					if (nextTransition) {
+						transition = nextTransition;
+						position = updatedPosition;
+					} else queuedReferences.push(key, value, keyIndex);
 					break;
 			}
-			targetView.setUint32(position, encoded, true);
-			position += 4;
+			keyIndex++;
 		}
-		let recordId = transition[RECORD_SYMBOL];
-		if (!(recordId < 1024)) {
-			// for now just punt and go back to writeObject
-			return 0;
-			// newRecord(transition, transition.__keys__ || Object.keys(object), newTransitions, true)
-		}
-		let stringLength = stringData.length;
-		if (stringData) {
-			if (position + stringLength > safeEnd) {
-				target = makeRoom(position + stringLength);
-			}
-			position += target.latin1Write(stringData, position, 0xffffffff);
-		}
-		target[start] = recordId >> 8;
-		target[start + 1] = recordId & 0xff;
-		target[start + 2] = stringLength >> 8;
-		target[start + 3] = stringLength & 0xff;
-		let queued32BitReferences;
+
 		for (let i = 0, l = queuedReferences.length; i < l;) {
+			let key = queuedReferences[i++];
 			let value = queuedReferences[i++];
-			let slotOffset = queuedReferences[i++] + start;
-			let offset = position - slotOffset;
-			if (offset < 0x1f000000) {
-				targetView.setUint32(slotOffset, 0x80000000 | (offset), true);
+			let propertyIndex = queuedReferences[i++];
+			let nextTransition = transition[key];
+			if (!nextTransition) {
+				transition[key] = nextTransition = {
+					key,
+					parent: transition,
+					enumerationOffset: propertyIndex - keyIndex,
+					ascii0: null,
+					ascii8: null,
+					num8: null,
+					string16: null,
+					object16: null,
+					num32: null,
+					float64: null
+				};
+			}
+			let newPosition;
+			if (value) {
+				/*if (typeof value === 'string') { // TODO: we could re-enable long strings
+					if (position + value.length * 3 > safeEnd) {
+						target = makeRoom(position + value.length * 3);
+						position -= start;
+						targetView = target.dataView;
+						start = 0;
+					}
+					newPosition = position + target.utf8Write(value, position, 0xffffffff);
+				} else { */
+				let size;
+				refOffset = refPosition - refsStartPosition;
+				if (refOffset < 0xff00) {
+					transition = nextTransition.object16;
+					if (transition)
+						size = 2;
+					else if ((transition = nextTransition.object32))
+						size = 4;
+					else {
+						transition = createTypeTransition(nextTransition, OBJECT_DATA, 2);
+						size = 2;
+					}
+				} else {
+					transition = nextTransition.object32 || createTypeTransition(nextTransition, OBJECT_DATA, 4);
+					size = 4;
+				}
+				newPosition = pack(value, refPosition);
+				//}
+				if (typeof newPosition === 'object') {
+					// re-allocated
+					refPosition = newPosition.position;
+					targetView = newPosition.targetView;
+					target = newPosition.target;
+					refsStartPosition -= start;
+					position -= start;
+					start = 0;
+				} else
+					refPosition = newPosition;
+				if (size === 2) {
+					targetView.setUint16(position, refOffset, true);
+					position += 2;
+				} else {
+					targetView.setUint32(position, refOffset, true);
+					position += 4;
+				}
 			} else {
-				if (!queued32BitReferences)
-					queued32BitReferences = [];
-				queued32BitReferences.push({slotOffset, offset: position - start});
+				transition = nextTransition.object16 || createTypeTransition(nextTransition, OBJECT_DATA, 2);
+				targetView.setInt16(position, value === null ? -10 : -9, true);
+				position += 2;
 			}
-			let newPosition = pack(value, position);
-			if (typeof newPosition === 'object') {
-				// re-allocated
-				position = newPosition.position;
-				targetView = newPosition.targetView;
-				start = 0;
-			} else
-				position = newPosition;
+			keyIndex++;
 		}
-		if (queued32BitReferences) {
-			// TODO: makeRoom
-			for (let i = 0, l = queued32BitReferences.length; i < l; i++) {
-				let ref = queued32BitReferences[i];
-				targetView.setUint32(ref.slotOffset, 0xa0000000 - ((l - i) << 2), true);
-				targetView.setUint32(position, ref.offset, true);
-				position += 4;
+
+
+		let recordId = transition[RECORD_SYMBOL];
+		if (recordId == null) {
+			recordId = packr.typedStructs.length;
+			let structure = [];
+			let nextTransition = transition;
+			let key, type;
+			while ((type = nextTransition.__type) !== undefined) {
+				let size = nextTransition.__size;
+				nextTransition = nextTransition.__parent;
+				key = nextTransition.key;
+				let property = [type, size, key];
+				if (nextTransition.enumerationOffset)
+					property.push(nextTransition.enumerationOffset);
+				structure.push(property);
+				nextTransition = nextTransition.parent;
 			}
+			structure.reverse();
+			transition[RECORD_SYMBOL] = recordId;
+			packr.typedStructs[recordId] = structure;
+			pack(null, 0, true); // special call to notify that structures have been updated
+		}
+
+
+		switch (headerSize) {
+			case 1:
+				if (recordId >= 0x10) return 0;
+				target[start] = recordId + 0x20;
+				break;
+			case 2:
+				if (recordId >= 0x100) return 0;
+				target[start] = 0x38;
+				target[start + 1] = recordId;
+				break;
+			case 3:
+				if (recordId >= 0x10000) return 0;
+				target[start] = 0x39;
+				target.setUint16(start + 1, recordId, true);
+				break;
+			case 4:
+				if (recordId >= 0x1000000) return 0;
+				target.setUint32(start, (recordId << 8) + 0x3a, true);
+				break;
 		}
 
-		return position;
+		if (position < refsStartPosition) {
+			if (refsStartPosition === refPosition)
+				return position; // no refs
+			// adjust positioning
+			target.copyWithin(position, refsStartPosition, refPosition);
+			refPosition += position - refsStartPosition;
+			typedStructs.lastStringStart = position - start;
+		} else if (position > refsStartPosition) {
+			if (refsStartPosition === refPosition)
+				return position; // no refs
+			typedStructs.lastStringStart = position - start;
+			return writeStruct(object, target, start, structures, makeRoom, pack, packr);
+		}
+		return refPosition;
+	}
+	function anyType(transition, position, targetView, value) {
+		let nextTransition;
+		if ((nextTransition = transition.ascii8 || transition.num8)) {
+			targetView.setInt8(position, value, true);
+			updatedPosition = position + 1;
+			return nextTransition;
+		}
+		if ((nextTransition = transition.string16 || transition.object16)) {
+			targetView.setInt16(position, value, true);
+			updatedPosition = position + 2;
+			return nextTransition;
+		}
+		if (nextTransition = transition.num32) {
+			targetView.setUint32(position, 0xe0000100 + value, true);
+			updatedPosition = position + 4;
+			return nextTransition;
+		}
+		// transition.float64
+		if (nextTransition = transition.num64) {
+			targetView.setFloat64(position, NaN, true);
+			targetView.setInt8(position, value);
+			updatedPosition = position + 8;
+			return nextTransition;
+		}
+		updatedPosition = position;
+		// TODO: can we do an "any" type where we defer the decision?
+		return;
+	}
+	function createTypeTransition(transition, type, size) {
+		let typeName = TYPE_NAMES[type] + (size << 3);
+		let newTransition = transition[typeName] || (transition[typeName] = Object.create(null));
+		newTransition.__type = type;
+		newTransition.__size = size;
+		newTransition.__parent = transition;
+		return newTransition;
+	}
+	function onLoadedStructures$1(sharedData) {
+		if (!(sharedData instanceof Map))
+			return sharedData;
+		let typed = sharedData.get('typed') || [];
+		if (Object.isFrozen(typed))
+			typed = typed.map(structure => structure.slice(0));
+		let named = sharedData.get('named');
+		let transitions = Object.create(null);
+		for (let i = 0, l = typed.length; i < l; i++) {
+			let structure = typed[i];
+			let transition = transitions;
+			for (let [type, size, key] of structure) {
+				let nextTransition = transition[key];
+				if (!nextTransition) {
+					transition[key] = nextTransition = {
+						key,
+						parent: transition,
+						enumerationOffset: 0,
+						ascii0: null,
+						ascii8: null,
+						num8: null,
+						string16: null,
+						object16: null,
+						num32: null,
+						float64: null
+					};
+				}
+				transition = createTypeTransition(nextTransition, type, size);
+			}
+			transition[RECORD_SYMBOL] = i;
+		}
+		typed.transitions = transitions;
+		this.typedStructs = typed;
+		this.lastTypedStructuresLength = typed.length;
+		return named;
 	}
 	var sourceSymbol = Symbol('source');
-	function readStruct$1(src, position, srcEnd, structure, unpackr) {
-		var stringLength = (src[position++] << 8) | src[position++];
+	function readStruct$1(src, position, srcEnd, unpackr) {
+	//	var stringLength = (src[position++] << 8) | src[position++];
+		let recordId = src[position++] - 0x20;
+		if (recordId >= 24) {
+			switch(recordId) {
+				case 24: recordId = src[position++]; break;
+				// little endian:
+				case 25: recordId = src[position++] + (src[position++] << 8); break;
+				case 26: recordId = src[position++] + (src[position++] << 8) + (src[position++] << 16); break;
+				case 27: recordId = src[position++] + (src[position++] << 8) + (src[position++] << 16) + (src[position++] << 24); break;
+			}
+		}
+		let structure = unpackr.typedStructs?.[recordId];
+		if (!structure) {
+			// copy src buffer because getStructures will override it
+			src = Uint8Array.prototype.slice.call(src, position, srcEnd);
+			srcEnd -= position;
+			position = 0;
+			unpackr._mergeStructures(unpackr.getStructures());
+			if (!unpackr.typedStructs)
+				throw new Error('Could not find any shared typed structures');
+			unpackr.lastTypedStructuresLength = unpackr.typedStructs.length;
+			structure = unpackr.typedStructs[recordId];
+			if (!structure)
+				throw new Error('Could not find typed structure ' + recordId);
+		}
 		var construct = structure.construct;
 		if (!construct) {
-			construct = structure.construct = function() {
+			construct = structure.construct = function LazyObject() {
 			};
 			var prototype = construct.prototype;
+			let properties = [];
 			Object.defineProperty(prototype, 'toJSON', {
-				get() {
+				value() {
 					// return an enumerable object with own properties to JSON stringify
 					let resolved = {};
-					for (let i = 0, l = structure.length; i < l; i++) {
-						let key = structure[i];
+					for (let i = 0, l = properties.length; i < l; i++) {
+						let key = properties[i].key;
 						resolved[key] = this[key];
 					}
 					return resolved;
 				},
 				// not enumerable or anything 
 			});
+			let currentOffset = 0;
+			let lastRefProperty;
 			for (let i = 0, l = structure.length; i < l; i++) {
-				let key = structure[i];
-				Object.defineProperty(prototype, key, {
-					get() {
-						let source = this[sourceSymbol];
-						let src = source.src;
-						//let uint32 = src.uint32 || (src.uint32 = new Uint32Array(src.buffer, src.byteOffset, src.byteLength));
-						let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
-						let position = source.position + (i << 2);
-						let value = dataView.getUint32(position, true);
-						let start;
-						switch (value >>> 29) {
-							case 0:
-								return value;
-							case 3:
-								if (value & 0x10000000) {
-									start = (value & 0xffff) + position;
-									return src.toString('utf8', start, start + ((value >> 16) & 0x7ff));
-								} else {
-									if (!source.srcString) {
-										start = source.position + (l << 2);
-										source.srcString = src.toString('latin1', start, start + source.stringLength);
-									}
-									start = value & 0xffff;
-									return source.srcString.slice(start, start + ((value >> 16) & 0x7ff));
-								}
-							case 4:
-								start = (0x1fffffff & value) + position;
-								let end = source.srcEnd;
-								for (let next = i + 1; next < l; next++) {
-									position = source.position + (next << 2);
-									let nextValue = dataView.getUint32(position, true);								if ((nextValue & 0xe0000000) == -0x80000000) {
-										end = (0x1fffffff & nextValue) + position;
+				let definition = structure[i];
+				let [ type, size, key, enumerationOffset ] = definition;
+				let property = {
+					key,
+					offset: currentOffset,
+				};
+				if (enumerationOffset)
+					properties.splice(i + enumerationOffset, 0, property);
+				else
+					properties.push(property);
+				let getRef;
+				switch(size) { // TODO: Move into a separate function
+					case 0: getRef = () => 0; break;
+					case 1:
+						getRef = (source, position) => {
+							let ref = source.src[position + property.offset];
+							return ref >= 0xf6 ? toConstant(ref) : ref;
+						};
+						break;
+					case 2:
+						getRef = (source, position) => {
+							let src = source.src;
+							let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
+							let ref = dataView.getUint16(position + property.offset, true);
+							return ref >= 0xff00 ? toConstant(ref & 0xff) : ref;
+						};
+						break;
+					case 4:
+						getRef = (source, position) => {
+							let src = source.src;
+							let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
+							let ref = dataView.getUint32(position + property.offset, true);
+							return ref >= 0xffffff00 ? toConstant(ref & 0xff) : ref;
+						};
+						break;
+				}
+				property.getRef = getRef;
+				currentOffset += size;
+				let get;
+				switch(type) {
+					case ASCII:
+						if (lastRefProperty && !lastRefProperty.next)
+							lastRefProperty.next = property;
+						lastRefProperty = property;
+						property.multiGetCount = 0;
+						get = function() {
+							let source = this[sourceSymbol];
+							let src = source.src;
+							let position = source.position;
+							let refStart = currentOffset + position;
+							let ref = getRef(source, position);
+							if (typeof ref !== 'number') return ref;
+
+							let end, next = property.next;
+							while(next) {
+								end = next.getRef(source, position);
+								if (typeof end === 'number')
+									break;
+								else
+									end = null;
+								next = next.next;
+							}
+							if (end == null)
+								end = source.srcEnd - refStart;
+							if (source.srcString) {
+								return source.srcString.slice(ref, end);
+							}
+							/*if (property.multiGetCount > 0) {
+								let asciiEnd;
+								next = firstRefProperty;
+								let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
+								do {
+									asciiEnd = dataView.getUint16(source.position + next.offset, true);
+									if (asciiEnd < 0xff00)
 										break;
+									else
+										asciiEnd = null;
+								} while((next = next.next));
+								if (asciiEnd == null)
+									asciiEnd = source.srcEnd - refStart
+								source.srcString = src.toString('latin1', refStart, refStart + asciiEnd);
+								return source.srcString.slice(ref, end);
+							}
+							if (source.prevStringGet) {
+								source.prevStringGet.multiGetCount += 2;
+							} else {
+								source.prevStringGet = property;
+								property.multiGetCount--;
+							}*/
+							return readString(src, ref + refStart, end - ref);
+							//return src.toString('latin1', ref + refStart, end + refStart);
+						};
+						break;
+					case UTF8: case OBJECT_DATA:
+						if (lastRefProperty && !lastRefProperty.next)
+							lastRefProperty.next = property;
+						lastRefProperty = property;
+						get = function() {
+							let source = this[sourceSymbol];
+							let position = source.position;
+							let refStart = currentOffset + position;
+							let ref = getRef(source, position);
+							if (typeof ref !== 'number') return ref;
+							let src = source.src;
+							let end, next = property.next;
+							while(next) {
+								end = next.getRef(source, position);
+								if (typeof end === 'number')
+									break;
+								else
+									end = null;
+								next = next.next;
+							}
+							if (end == null)
+								end = source.srcEnd - refStart;
+							if (type === UTF8) {
+								return src.toString('utf8', ref + refStart, end + refStart);
+							} else {
+								return unpackr.unpack(src, { start: ref + refStart, end: end  + refStart }); // could reuse this object
+							}
+						};
+						break;
+					case NUMBER:
+						switch(size) {
+							case 4:
+								get = function () {
+									let source = this[sourceSymbol];
+									let src = source.src;
+									let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
+									let position = source.position + property.offset;
+									let value = dataView.getInt32(position, true);
+									if (value < 0x20000000) {
+										if (value > -0x1f000000)
+											return value;
+										if (value > -0x20000000)
+											return toConstant(value & 0xff);
 									}
-								}
-								return unpackr.unpack(src.slice(start, end));
-							case 1: case 2: case 5: case 6:
-								let fValue = dataView.getFloat32(position, true);
-								// this does rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
-								let multiplier = mult10[((src[position + 3] & 0x7f) << 1) | (src[position + 2] >> 7)];
-								return ((multiplier * fValue + (fValue > 0 ? 0.5 : -0.5)) >> 0) / multiplier;
-							case 7:
-								switch((value >> 24) & 0x1f) {
-									case 0: return null;
-									case 1: return undefined;
-									case 2: return false;
-									case 3: return true;
-									case 8: return dataView.getFloat64(position + (value & 0x3ffffff), true);
-									case 0x18: return '';
-									case 0x19: return String.fromCharCode((value >> 16) & 0xff);
-									case 0x1a: return String.fromCharCode((value >> 16) & 0xff, (value >> 8) & 0xff);
-									case 0x1b: return String.fromCharCode((value >> 16) & 0xff, (value >> 8) & 0xff, value & 0xff);
-									default: throw new Error('Unknown constant');
-								}
+									let fValue = dataView.getFloat32(position, true);
+									// this does rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
+									let multiplier = mult10[((src[position + 3] & 0x7f) << 1) | (src[position + 2] >> 7)];
+									return ((multiplier * fValue + (fValue > 0 ? 0.5 : -0.5)) >> 0) / multiplier;
+								};
+								break;
+							case 8:
+								get = function () {
+									let source = this[sourceSymbol];
+									let src = source.src;
+									let dataView = src.dataView || (src.dataView = new DataView(src.buffer, src.byteOffset, src.byteLength));
+									let value = dataView.getFloat64(source.position + property.offset, true);
+									if (isNaN(value)) {
+										let byte = src[source.position + property.offset];
+										if (byte >= 0xf6)
+											return toConstant(byte);
+									}
+									return value;
+								};
+								break;
+							case 1:
+								get = function () {
+									let source = this[sourceSymbol];
+									let src = source.src;
+									let value = src[source.position + property.offset];
+									return value < 0xf6 ? value : toConstant(value);
+								};
+								break;
 						}
-					},
-					enumerable: true,
-				});
+				}
+				property.get = get;
 			}
+			for (let property of properties) // assign in enumeration order
+				Object.defineProperty(prototype, property.key, { get: property.get, enumerable: true });
 		}
 		var instance = new construct();
 		instance[sourceSymbol] = {
 			src,
-			uint32: src.uint32,
 			position,
 			srcString: '',
-			srcEnd,
-			stringLength
+			srcEnd
 		};
 		return instance;
 	}
-	setReadStruct(readStruct$1);
+	function toConstant(code) {
+		switch(code) {
+			case 0xf6: return null;
+			case 0xf7: return undefined;
+			case 0xf8: return false;
+			case 0xf9: return true;
+		}
+		throw new Error('Unknown constant');
+	}
+	function prepareStructures$1(structures, packr) {
+		if (packr.typedStructs) {
+			let structMap = new Map();
+			structMap.set('named', structures);
+			structMap.set('typed', packr.typedStructs);
+			structures = structMap;
+		}
+		let lastTypedStructuresLength = packr.lastTypedStructuresLength || 0;
+		structures.isCompatible = existing => {
+			let compatible = true;
+			if (existing instanceof Map) {
+				let named = existing.get('named') || [];
+				if (named.length !== (packr.lastNamedStructuresLength || 0))
+					compatible = false;
+				let typed = existing.get('typed') || [];
+				if (typed.length !== lastTypedStructuresLength)
+					compatible = false;
+			} else if (existing instanceof Array) {
+				if (existing.length !== (packr.lastNamedStructuresLength || 0))
+					compatible = false;
+			}
+			if (!compatible)
+				packr._mergeStructures(existing);
+			return compatible;
+		};
+		packr.lastTypedStructuresLength = packr.typedStructs?.length;
+		return structures;
+	}
 
-	const allSampleData = [];
+	setReadStruct(readStruct$1, onLoadedStructures$1);
+
+	let allSampleData = [];
 	for (let i = 1; i < 6; i++) {
 		allSampleData.push(JSON.parse(fs.readFileSync(new URL(`./example${i > 1 ? i : ''}.json`, (document.currentScript && document.currentScript.src || new URL('test.js', document.baseURI).href)))));
 	}
@@ -2178,6 +2663,13 @@
 			return {}
 		}
 	}
+
+	let seed = 0;
+	function random() {
+		seed++;
+		let a = seed * 15485863;
+		return (a * a * a % 2038074743) / 2038074743;
+	}
 	//if (typeof chai === 'undefined') { chai = require('chai') }
 	var assert = chai.assert;
 	//if (typeof msgpackr === 'undefined') { msgpackr = require('..') }
@@ -2276,6 +2768,36 @@
 			var deserialized = unpack(serialized);
 			assert.equal(deserialized, data);
 		});
+		test('pack/unpack varying data with random access structures', function() {
+			let structures = [];
+			let packr = new Packr$1({ structures, useRecords: true, randomAccessStructure: true, freezeData: true, saveStructures(structures) {
+				}, getStructures() {
+					console.log('getStructures');
+				} });
+			for (let i = 0; i < 20; i++) {
+				let data = {};
+				let props = ['foo', 'bar', 'a', 'b', 'c','name', 'age', 'd'];
+				function makeString() {
+					let str = '';
+					while (random() < 0.9) {
+						str += random() < 0.8 ? 'hello world' : String.fromCharCode(300);
+					}
+					return str;
+				}
+				for (let i = 0; i < random() * 20; i++) {
+					data[props[Math.floor(random() * 8)]] =
+						random() < 0.3 ? Math.floor(random() * 400) / 2 :
+							random() < 0.3 ? makeString() : random() < 0.3 ? true : random() < 0.3 ? sampleData : null;
+				}
+				var serialized = packr.pack(data);
+				var deserialized = packr.unpack(serialized);
+				for (let key in deserialized) {
+					let a = deserialized[key];
+				}
+				assert.deepEqual(deserialized, data);
+			}
+		});
+
 		for (let sampleData of allSampleData) {
 			let snippet = JSON.stringify(sampleData).slice(0, 20) + '...';
 			test('pack/unpack sample data ' + snippet, function(){
@@ -2290,11 +2812,18 @@
 			test('pack/unpack sample data with random access structures ' + snippet, function() {
 				var data = sampleData;
 				let structures = [];
-				let packr = new Packr$1({ structures, useRecords: true, randomAccessStructure: true, freezeData: true });
+				let packr = new Packr$1({ structures, useRecords: true, randomAccessStructure: true, freezeData: true, saveStructures(structures) {
+				}, getStructures() {
+					console.log('getStructures');
+				} });
 				for (let i = 0; i < 20; i++) {
 					var serialized = packr.pack(data);
-					var deserialized = packr.unpack(serialized);
-					assert.deepEqual(deserialized, data);
+					var deserialized = packr.unpack(serialized, { lazy: true });
+					var copied = {};
+					for (let key in deserialized) {
+						copied[key] = deserialized[key];
+					}
+					assert.deepEqual(copied, data);
 				}
 			});
 			test('pack/unpack sample data with bundled strings ' + snippet, function(){
@@ -2304,6 +2833,7 @@
 				var deserialized = packr.unpack(serialized);
 				assert.deepEqual(deserialized, data);
 			});
+			break;
 		}
 
 		test('pack/unpack empty data with bundled strings', function(){
@@ -2886,5 +3416,5 @@
 		});
 	});
 
-}(msgpackr, chai, fs));
+}(msgpackr, chai, inspector, fs));
 //# sourceMappingURL=test.js.map