msgpackr 1.11.12 → 1.11.13

package/dist/test.js

@@ -2399,9 +2399,16 @@
 			return textEncoder.encodeInto(string, target.subarray(position)).written
 		} : false;
 	setWriteStructSlots(writeStruct, prepareStructures);
-	function writeStruct(object, target, encodingStart, position, structures, makeRoom, pack, packr) {
+	function writeStruct(object, target, encodingStart, position, structures, makeRoom, pack, packr, structureKnown) {
 		let typedStructs = packr.typedStructs || (packr.typedStructs = []);
 		// note that we rely on pack.js to load stored structures before we get to this point
+		// structureKnown is set only on the internal layout-retry below: attempt 1 already minted
+		// this record's structure, so the retry re-encodes a known shape and must not re-apply the
+		// cap (which could otherwise bail after attempt 1 already packed refs → corrupt fallback).
+		// `frozen` is a local (from this instance's typedStructs) — never a shared global — so a
+		// re-entrant encode on another instance (e.g. via an enumerable getter) can't flip it.
+		const cap = packr.maxOwnStructures ?? Infinity;
+		const frozen = !structureKnown && typedStructs.length >= cap;
 		let targetView = target.dataView;
 		let refsStartPosition = (typedStructs.lastStringStart || 100) + position;
 		let safeEnd = target.length - 10;
@@ -2432,9 +2439,12 @@
 		let usedAscii0;
 		let keyIndex = 0;
 		for (let key in object) {
-			let value = object[key];
 			let nextTransition = transition[key];
+			// Resolve the key transition BEFORE reading the value: when frozen and the key is new we
+			// bail here, so an enumerable getter isn't invoked during this (failed) struct attempt and
+			// then again by the plain fallback (which would double-read a side-effecting accessor).
 			if (!nextTransition) {
+				if (frozen) return 0;
 				transition[key] = nextTransition = {
 					key,
 					parent: transition,
@@ -2449,6 +2459,7 @@
 					date64: null
 				};
 			}
+			let value = object[key];
 			if (position > safeEnd) {
 				target = makeRoom(position);
 				targetView = target.dataView;
@@ -2466,10 +2477,10 @@
 					if (nextId < 200 || !nextTransition.num64) {
 						if (number >> 0 === number && number < 0x20000000 && number > -0x1f000000) {
 							if (number < 0xf6 && number >= 0 && (nextTransition.num8 && !(nextId > 200 && nextTransition.num32) || number < 0x20 && !nextTransition.num32)) {
-								transition = nextTransition.num8 || createTypeTransition(nextTransition, NUMBER, 1);
+								transition = nextTransition.num8 || createTypeTransition(nextTransition, NUMBER, 1, frozen);
 								target[position++] = number;
 							} else {
-								transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4);
+								transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4, frozen);
 								targetView.setUint32(position, number, true);
 								position += 4;
 							}
@@ -2480,14 +2491,14 @@
 								let xShifted;
 								// this checks for rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
 								if (((xShifted = number * mult10[((target[position + 3] & 0x7f) << 1) | (target[position + 2] >> 7)]) >> 0) === xShifted) {
-									transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4);
+									transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4, frozen);
 									position += 4;
 									break;
 								}
 							}
 						}
 					}
-					transition = nextTransition.num64 || createTypeTransition(nextTransition, NUMBER, 8);
+					transition = nextTransition.num64 || createTypeTransition(nextTransition, NUMBER, 8, frozen);
 					targetView.setFloat64(position, number, true);
 					position += 8;
 					break;
@@ -2553,21 +2564,21 @@
 									nextTransition.string8 = transition;
 									pack(null, 0, true); // special call to notify that structures have been updated
 								} else {
-									transition = createTypeTransition(nextTransition, UTF8, 1);
+									transition = createTypeTransition(nextTransition, UTF8, 1, frozen);
 								}
 							}
 						} else if (refOffset === 0 && !usedAscii0) {
 							usedAscii0 = true;
-							transition = nextTransition.ascii0 || createTypeTransition(nextTransition, ASCII, 0);
+							transition = nextTransition.ascii0 || createTypeTransition(nextTransition, ASCII, 0, frozen);
 							break; // don't increment position
 						}// else ascii:
 						else if (!(transition = nextTransition.ascii8) && !(typedStructs.length > 10 && (transition = nextTransition.string8)))
-							transition = createTypeTransition(nextTransition, ASCII, 1);
+							transition = createTypeTransition(nextTransition, ASCII, 1, frozen);
 						target[position++] = refOffset;
 					} else {
 						// TODO: Enable ascii16 at some point, but get the logic right
 						//if (isNotAscii)
-							transition = nextTransition.string16 || createTypeTransition(nextTransition, UTF8, 2);
+							transition = nextTransition.string16 || createTypeTransition(nextTransition, UTF8, 2, frozen);
 						//else
 							//transition = nextTransition.ascii16 || createTypeTransition(nextTransition, ASCII, 2);
 						targetView.setUint16(position, refOffset, true);
@@ -2577,7 +2588,7 @@
 				case 'object':
 					if (value) {
 						if (value.constructor === Date) {
-							transition = nextTransition.date64 || createTypeTransition(nextTransition, DATE, 8);
+							transition = nextTransition.date64 || createTypeTransition(nextTransition, DATE, 8, frozen);
 							targetView.setFloat64(position, value.getTime(), true);
 							position += 8;
 						} else {
@@ -2593,7 +2604,7 @@
 					}
 					break;
 				case 'boolean':
-					transition = nextTransition.num8 || nextTransition.ascii8 || createTypeTransition(nextTransition, NUMBER, 1);
+					transition = nextTransition.num8 || nextTransition.ascii8 || createTypeTransition(nextTransition, NUMBER, 1, frozen);
 					target[position++] = value ? 0xf9 : 0xf8; // match CBOR with these
 					break;
 				case 'undefined':
@@ -2606,9 +2617,41 @@
 				default:
 					queuedReferences.push(key, value, keyIndex);
 			}
+			if (transition === undefined) return 0; // frozen: structure cap reached
 			keyIndex++;
 		}
 
+		// Cap enforcement for queued (nested-object / null) references. pack() advances msgpackr's
+		// shared write position and we cannot cleanly bail afterward, so preflight the whole queued
+		// chain through EXISTING transitions first: if the cap is reached and any field would need a
+		// new structure, fall back to plain encoding now (return 0) — before touching the shared
+		// position. Uses a FRESH length read (not the entry-time `frozen`): a getter invoked while
+		// reading values above may have minted on this same instance since entry.
+		if (!structureKnown && queuedReferences.length > 0 && typedStructs.length >= cap) {
+			let t = transition;
+			for (let i = 0, l = queuedReferences.length; i < l; i += 3) {
+				// A non-null (object/Date) ref is pack()ed into the shared buffer, advancing
+				// msgpackr's write position. Its structure variant (object16 vs object32) depends on
+				// the runtime ref-section offset (inline strings + earlier refs), which we can't know
+				// before packing — and we can't bail after a pack without corrupting the fallback. So
+				// under the cap, any record with a packing ref falls back to plain encoding now,
+				// before any pack(). null/undefined refs don't pack, so they're walked normally.
+				if (queuedReferences[i + 1] != null) return 0;
+				const nt = t[queuedReferences[i]];
+				if (!nt) return 0;
+				const next = nt.object16; // null/undefined ref → OBJECT_DATA size 2
+				if (!next) return 0;
+				t = next;
+			}
+			if (t[RECORD_SYMBOL] == null) return 0; // exact structure not yet minted
+		}
+
+		// Past the preflight the chain is known, so no minting happens — except a rare offset
+		// divergence (a known shape whose ref section now crosses 0xff00 and needs object32 where
+		// the preflight matched object16). Once a ref is packed we can no longer bail, so we finish
+		// via the unfrozen forceTypeTransition: a bounded, self-converging overshoot for that one
+		// record. packedRef keeps the record-id mint from bailing after a pack.
+		let packedRef = false;
 		for (let i = 0, l = queuedReferences.length; i < l;) {
 			let key = queuedReferences[i++];
 			let value = queuedReferences[i++];
@@ -2630,15 +2673,6 @@
 			}
 			let newPosition;
 			if (value) {
-				/*if (typeof value === 'string') { // TODO: we could re-enable long strings
-					if (position + value.length * 3 > safeEnd) {
-						target = makeRoom(position + value.length * 3);
-						position -= start;
-						targetView = target.dataView;
-						start = 0;
-					}
-					newPosition = position + target.utf8Write(value, position, 0xffffffff);
-				} else { */
 				let size;
 				refOffset = refPosition - refsStartPosition;
 				if (refOffset < 0xff00) {
@@ -2648,15 +2682,15 @@
 					else if ((transition = nextTransition.object32))
 						size = 4;
 					else {
-						transition = createTypeTransition(nextTransition, OBJECT_DATA, 2);
+						transition = forceTypeTransition(nextTransition, OBJECT_DATA, 2);
 						size = 2;
 					}
 				} else {
-					transition = nextTransition.object32 || createTypeTransition(nextTransition, OBJECT_DATA, 4);
+					transition = nextTransition.object32 || forceTypeTransition(nextTransition, OBJECT_DATA, 4);
 					size = 4;
 				}
 				newPosition = pack(value, refPosition);
-				//}
+				packedRef = true;
 				if (typeof newPosition === 'object') {
 					// re-allocated
 					refPosition = newPosition.position;
@@ -2676,16 +2710,19 @@
 					position += 4;
 				}
 			} else { // null or undefined
-				transition = nextTransition.object16 || createTypeTransition(nextTransition, OBJECT_DATA, 2);
+				transition = nextTransition.object16 || forceTypeTransition(nextTransition, OBJECT_DATA, 2);
 				targetView.setInt16(position, value === null ? -10 : -9, true);
 				position += 2;
 			}
 			keyIndex++;
 		}
 
-
 		let recordId = transition[RECORD_SYMBOL];
 		if (recordId == null) {
+			// Flat records (no queued refs) reach here without packing, so the cap is enforced
+			// cleanly. Records that packed nested refs already passed the preflight; either way
+			// bailing now after refs were packed would corrupt the fallback.
+			if (!packedRef && typedStructs.length >= cap) return 0;
 			recordId = packr.typedStructs.length;
 			let structure = [];
 			let nextTransition = transition;
@@ -2739,7 +2776,11 @@
 			if (refsStartPosition === refPosition)
 				return position; // no refs
 			typedStructs.lastStringStart = position - start;
-			return writeStruct(object, target, encodingStart, start, structures, makeRoom, pack, packr);
+			// Fixed section overflowed our estimate — retry with the corrected size. The structure
+			// is already minted at this point, so pass structureKnown=true to skip the cap check
+			// (otherwise a record that became frozen during attempt 1 would bail mid-retry, after
+			// refs were already packed, and corrupt the fallback).
+			return writeStruct(object, target, encodingStart, start, structures, makeRoom, pack, packr, true);
 		}
 		return refPosition;
 	}
@@ -2771,9 +2812,36 @@
 		// TODO: can we do an "any" type where we defer the decision?
 		return;
 	}
-	function createTypeTransition(transition, type, size) {
+	// When the typed-structure dictionary reaches maxOwnStructures we stop minting new
+	// structures/transitions. typedStructs is append-only and pinned on the long-lived
+	// encoder (records reference structures by recordId), so an unbounded shape space —
+	// e.g. a wide, sparsely/variably-populated schema — would otherwise grow the
+	// dictionary + transition trie without limit. `frozen` is passed in (derived from the
+	// encoding instance's own typedStructs.length, never a shared global) so a re-entrant
+	// encode on another instance can't flip it; while frozen, a missing transition returns
+	// undefined so the caller bails and the record falls back to plain encoding.
+	function createTypeTransition(transition, type, size, frozen) {
+		let typeName = TYPE_NAMES[type] + (size << 3);
+		let newTransition = transition[typeName];
+		if (newTransition) return newTransition;
+		if (frozen) return undefined;
+		newTransition = transition[typeName] = Object.create(null);
+		newTransition.__type = type;
+		newTransition.__size = size;
+		newTransition.__parent = transition;
+		return newTransition;
+	}
+
+	// Unfrozen variant: always mints. Used in the queued-ref loop once a nested value has
+	// already been pack()ed — at that point pack() has advanced msgpackr's shared write
+	// position, so bailing with `return 0` would corrupt the fallback. We must finish the
+	// encode instead, even if that means minting a (bounded) handful of structures past the
+	// cap. The cap is still enforced up front via the preflight, before the first pack().
+	function forceTypeTransition(transition, type, size) {
 		let typeName = TYPE_NAMES[type] + (size << 3);
-		let newTransition = transition[typeName] || (transition[typeName] = Object.create(null));
+		let newTransition = transition[typeName];
+		if (newTransition) return newTransition;
+		newTransition = transition[typeName] = Object.create(null);
 		newTransition.__type = type;
 		newTransition.__size = size;
 		newTransition.__parent = transition;
@@ -2807,7 +2875,8 @@
 						date64: null,
 					};
 				}
-				transition = createTypeTransition(nextTransition, type, size);
+				// Replaying persisted structures is never subject to the cap — always mint.
+				transition = createTypeTransition(nextTransition, type, size, false);
 			}
 			transition[RECORD_SYMBOL] = i;
 		}
@@ -4564,5 +4633,189 @@
 		});
 	});
 
+	suite('msgpackr – maxOwnStructures cap (randomAccessStructure)', function () {
+		// Helper: create a width-heterogeneous record generator using a deterministic PRNG.
+		// Objects have up to 6 fields drawn from a sparse set, each value is an integer whose
+		// width (num8 / num32 / num64) varies per value, producing many distinct typed structures.
+		// useRecords: false keeps the classic named-record encoder out of the way so all structure
+		// creation goes through the typed-struct path and the cap is exercised in isolation.
+		function makeCapRunner(cap) {
+			const fields = ['a','b','c','d','e','f','g','h'];
+			let seed = 42;
+			const rnd = () => { seed = (seed * 1664525 + 1013904223) >>> 0; return seed / 0xffffffff; };
+			const packr = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				maxOwnStructures: cap,
+			});
+			const norm = r => packr.unpack(packr.pack(r));
+			for (let i = 0; i < 4000; i++) {
+				const r = {};
+				for (let j = 0; j < Math.ceil(rnd() * 6); j++) {
+					// values cycle across num8 / num32 / float64 ranges to force distinct structures
+					const mag = rnd() < 0.33 ? 10 : rnd() < 0.5 ? 400000 : 1e13;
+					r[fields[Math.floor(rnd() * 8)]] = Math.floor(rnd() * mag);
+				}
+				assert.deepEqual(norm(r), r);
+			}
+			return packr.typedStructs ? packr.typedStructs.length : 0;
+		}
+
+		test('uncapped (default) grows well past 256 for width-heterogeneous records', function () {
+			assert.ok(makeCapRunner(undefined) > 256, 'expected uncapped typedStructs to exceed 256');
+		});
+
+		test('cap=64 bounds typedStructs.length and preserves round-trips', function () {
+			assert.ok(makeCapRunner(64) <= 64, 'typedStructs should not exceed cap of 64');
+		});
+
+		test('cap=256 bounds typedStructs.length and preserves round-trips', function () {
+			assert.ok(makeCapRunner(256) <= 256, 'typedStructs should not exceed cap of 256');
+		});
+
+		test('flat-record streams stay a strict hard bound', function () {
+			// With maxOwnStructures=16, pack 2000 flat records; typedStructs must never exceed 16.
+			const packr = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				maxOwnStructures: 16,
+			});
+			let seed = 99;
+			const rnd = () => { seed = (seed * 1664525 + 1013904223) >>> 0; return seed / 0xffffffff; };
+			for (let i = 0; i < 2000; i++) {
+				const r = { x: Math.floor(rnd() * 1e6), y: Math.floor(rnd() * 200), z: Math.floor(rnd() * 1e12) };
+				assert.deepEqual(packr.unpack(packr.pack(r)), r);
+			}
+			assert.ok(packr.typedStructs.length <= 16, 'flat records must stay within cap, got ' + packr.typedStructs.length);
+		});
+
+		test('capped-out records fall back to plain encoding and still round-trip', function () {
+			// Once the cap is hit, novel shapes must decode correctly via plain msgpack fallback.
+			const packr = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				maxOwnStructures: 2,
+			});
+			const norm = r => packr.unpack(packr.pack(r));
+			assert.deepEqual(norm({ a: 1 }), { a: 1 });         // mints structure 0
+			assert.deepEqual(norm({ b: 'hello' }), { b: 'hello' }); // mints structure 1, cap hit
+			// These novel shapes fall back to plain msgpack — must still decode correctly:
+			assert.deepEqual(norm({ c: 42 }), { c: 42 });
+			assert.deepEqual(norm({ a: 1, b: 'hi', c: 99 }), { a: 1, b: 'hi', c: 99 });
+			assert.strictEqual(packr.typedStructs.length, 2, 'cap must be exact');
+		});
+
+		test('a known key later seen as a nested object falls back cleanly', function () {
+			// Once frozen, a previously-learned scalar key carrying an object must bail BEFORE pack()
+			// advances the shared encoder position — otherwise the plain fallback gets corrupt bytes.
+			const packr = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				maxOwnStructures: 1,
+			});
+			const norm = r => packr.unpack(packr.pack(r));
+			assert.deepEqual(norm({ a: 1 }), { a: 1 }); // mints structure 0, cap hit
+			const r2 = { a: { x: 1 } };
+			assert.deepEqual(norm(r2), r2); // 'a' known but now carries object — must fall back cleanly
+		});
+
+		test('nested records do not overshoot the cap and still round-trip', function () {
+			// A nested object mints its own structure before the outer record, so a stale frozen flag
+			// could push the outer record past the cap. The record-id mint guard re-checks the live
+			// length, keeping typedStructs.length a strict bound.
+			const packr = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				maxOwnStructures: 4,
+			});
+			let seed = 7;
+			const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; };
+			for (let i = 0; i < 1000; i++) {
+				const r = { outer: { inner: (rnd() * 1e7 | 0) * 1000 }, tag: 't' + (i % 20), n: (rnd() * 300 | 0) };
+				assert.deepEqual(packr.unpack(packr.pack(r)), r);
+			}
+			assert.ok(packr.typedStructs.length <= 4, 'nested encodes must not push typedStructs past the cap, got ' + packr.typedStructs.length);
+		});
+
+		test('persisted typed structures still load after a capped encoder froze the dictionary', function () {
+			// Replaying persisted structures in onLoadedStructures must always succeed, regardless of
+			// maxOwnStructures — the cap only limits minting NEW structures during encode.
+			let saved = null;
+			const writer = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				saveStructures(s) { saved = s; return true; },
+				getStructures() { return saved; },
+			});
+			const buf = writer.pack({ name: 'Alice', age: 30 });
+
+			// Warm up a capped encoder so the module-global-if-any freeze state is set.
+			const capped = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				maxOwnStructures: 1,
+			});
+			capped.pack({ x: 1 });
+			capped.pack({ y: 2, z: 3 }); // cap reached
+
+			// A fresh reader must still rebuild the transition trie from saved structures.
+			const reader = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				getStructures() { return saved; },
+			});
+			const result = reader.unpack(buf);
+			assert.equal(result.name, 'Alice');
+			assert.equal(result.age, 30);
+		});
+
+		test('the cap is per-instance: an uncapped sibling cannot lift this instance\'s cap', function () {
+			// frozen is derived from each encoder's own typedStructs.length — not a shared global —
+			// so an uncapped sibling churning out structures cannot lift the cap on the bounded one.
+			const uncapped = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+			});
+			const capped = new Packr({
+				structures: [],
+				useRecords: false,
+				randomAccessStructure: true,
+				maxOwnStructures: 2,
+			});
+			let seed = 1;
+			const rnd = () => { seed = (seed * 1664525 + 1013904223) >>> 0; return seed / 0xffffffff; };
+			const mk = () => { const o = {}; for (let f = 0; f < 20; f++) if (rnd() < 0.5) o['f' + f] = Math.floor(rnd() * 1e7); return o; };
+			for (let i = 0; i < 500; i++) {
+				uncapped.pack(mk()); // grows the sibling's dictionary freely
+				const r = mk();
+				assert.deepEqual(capped.unpack(capped.pack(r)), r);
+			}
+			assert.ok(capped.typedStructs.length <= 2, 'capped must stay bounded, got ' + capped.typedStructs.length);
+			assert.ok(uncapped.typedStructs.length > 2, 'uncapped sibling should grow freely');
+		});
+
+		test('a layout-retry record (large fixed section + nested refs) does not corrupt', function () {
+			// A large fixed section overflows the ref-start estimate and triggers the internal retry
+			// (which re-invokes writeStruct after refs were packed). The retry passes structureKnown=true
+			// to re-encode the already-minted structure rather than bailing under the now-reached cap —
+			// bailing there would write the fallback at an advanced position and corrupt the bytes.
+			const packr = new Packr({ structures: [], useRecords: false, randomAccessStructure: true, maxOwnStructures: 1 });
+			const norm = r => packr.unpack(packr.pack(r));
+			const mk = base => { const r = {}; for (let i = 0; i < 40; i++) r['n' + i] = base + i; r.a = { x: base }; r.b = { y: base + 1 }; return r; };
+			assert.deepEqual(norm(mk(1000000)), mk(1000000));
+			assert.deepEqual(norm(mk(2000000)), mk(2000000));
+			assert.ok(packr.typedStructs.length <= 1, 'retry-path records must not exceed the cap, got ' + packr.typedStructs.length);
+		});
+	});
+
 })(chai, null, module, fs);
 //# sourceMappingURL=test.js.map