msgpackr 1.7.1 → 1.8.0

package/index.d.ts

@@ -20,7 +20,9 @@ export interface Options {
 	encodeUndefinedAsNil?: boolean
 	maxSharedStructures?: number
 	maxOwnStructures?: number
+	/** @deprecated use int64AsType: 'number' */
 	int64AsNumber?: boolean
+	int64AsType?: 'bigint' | 'number' | 'string'
 	shouldShareStructure?: (keys: string[]) => boolean
 	getStructures?(): {}[]
 	saveStructures?(structures: {}[]): boolean | void

package/pack.js

@@ -66,14 +66,14 @@ export class Packr extends Unpackr {
 		this.pack = this.encode = function(value, encodeOptions) {
 			if (!target) {
 				target = new ByteArrayAllocate(8192)
-				targetView = target.dataView = new DataView(target.buffer, 0, 8192)
+				targetView = target.dataView || (target.dataView = new DataView(target.buffer, 0, 8192))
 				position = 0
 			}
 			safeEnd = target.length - 10
 			if (safeEnd - position < 0x800) {
 				// don't start too close to the end, 
 				target = new ByteArrayAllocate(target.length)
-				targetView = target.dataView = new DataView(target.buffer, 0, target.length)
+				targetView = target.dataView || (target.dataView = new DataView(target.buffer, 0, target.length))
 				safeEnd = target.length - 10
 				position = 0
 			} else
@@ -125,20 +125,44 @@ export class Packr extends Unpackr {
 					writeStruct(value);
 				else
 					pack(value)
+				let lastBundle = bundledStrings;
+				if (bundledStrings)
+					writeBundles(start, pack, 0)
 				if (referenceMap && referenceMap.idsToInsert) {
-					let incrementPosition = referenceMap.idsToInsert.length * 6;
-					if (bundledStrings)
-						writeBundles(start, pack, incrementPosition)
-					position += incrementPosition
+					let idsToInsert = referenceMap.idsToInsert.sort((a, b) => a.offset > b.offset ? 1 : -1);
+					let i = idsToInsert.length;
+					let incrementPosition = -1;
+					while (lastBundle && i > 0) {
+						let insertionPoint = idsToInsert[--i].offset + start;
+						if (insertionPoint < (lastBundle.stringsPosition + start) && incrementPosition === -1)
+							incrementPosition = 0;
+						if (insertionPoint > (lastBundle.position + start)) {
+							if (incrementPosition >= 0)
+								incrementPosition += 6;
+						} else {
+							if (incrementPosition >= 0) {
+								// update the bundle reference now
+								targetView.setUint32(lastBundle.position + start,
+									targetView.getUint32(lastBundle.position + start) + incrementPosition)
+								incrementPosition = -1; // reset
+							}
+							lastBundle = lastBundle.previous;
+							i++;
+						}
+					}
+					if (incrementPosition >= 0 && lastBundle) {
+						// update the bundle reference now
+						targetView.setUint32(lastBundle.position + start,
+							targetView.getUint32(lastBundle.position + start) + incrementPosition)
+					}
+					position += idsToInsert.length * 6;
 					if (position > safeEnd)
 						makeRoom(position)
 					packr.offset = position
-					let serialized = insertIds(target.subarray(start, position), referenceMap.idsToInsert)
+					let serialized = insertIds(target.subarray(start, position), idsToInsert)
 					referenceMap = null
 					return serialized
 				}
-				if (bundledStrings)
-					writeBundles(start, pack, 0)
 				packr.offset = position // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 				if (encodeOptions & REUSE_BUFFER_MODE) {
 					target.start = start
@@ -196,13 +220,15 @@ export class Packr extends Unpackr {
 						let maxBytes = (bundledStrings[0] ? bundledStrings[0].length * 3 + bundledStrings[1].length : 0) + 10
 						if (position + maxBytes > safeEnd)
 							target = makeRoom(position + maxBytes)
-						if (bundledStrings.position) { // here we use the 0x62 extension to write the last bundle and reserve sapce for the reference pointer to the next/current bundle
+						let lastBundle
+						if (bundledStrings.position) { // here we use the 0x62 extension to write the last bundle and reserve space for the reference pointer to the next/current bundle
+							lastBundle = bundledStrings
 							target[position] = 0xc8 // ext 16
 							position += 3 // reserve for the writing bundle size
 							target[position++] = 0x62 // 'b'
 							extStart = position - start
 							position += 4 // reserve for writing bundle reference
-							writeBundles(start, pack) // write the last bundles
+							writeBundles(start, pack, 0) // write the last bundles
 							targetView.setUint16(extStart + start - 3, position - start - extStart)
 						} else { // here we use the 0x62 extension just to reserve the space for the reference pointer to the bundle (will be updated once the bundle is written)
 							target[position++] = 0xd6 // fixext 4
@@ -211,6 +237,7 @@ export class Packr extends Unpackr {
 							position += 4 // reserve for writing bundle reference
 						}
 						bundledStrings = ['', ''] // create new ones
+						bundledStrings.previous = lastBundle;
 						bundledStrings.size = 0
 						bundledStrings.position = extStart
 					}
@@ -361,7 +388,7 @@ export class Packr extends Unpackr {
 					let constructor = value.constructor
 					if (constructor === Object) {
 						writeObject(value, true)
-					} else if (constructor === Array) {
+					} else if (constructor === Array || Array.isArray(value)) {
 						length = value.length
 						if (length < 0x10) {
 							target[position++] = 0x90 | length
@@ -600,7 +627,7 @@ export class Packr extends Unpackr {
 			} else // faster handling for smaller buffers
 				newSize = ((Math.max((end - start) << 2, target.length - 1) >> 12) + 1) << 12
 			let newBuffer = new ByteArrayAllocate(newSize)
-			targetView = newBuffer.dataView = new DataView(newBuffer.buffer, 0, newSize)
+			targetView = newBuffer.dataView || (newBuffer.dataView = new DataView(newBuffer.buffer, 0, newSize))
 			end = Math.min(end, target.length)
 			if (target.copy)
 				target.copy(newBuffer, 0, start, end)
@@ -906,7 +933,6 @@ function insertIds(serialized, idsToInsert) {
 	let nextId
 	let distanceToMove = idsToInsert.length * 6
 	let lastEnd = serialized.length - distanceToMove
-	idsToInsert.sort((a, b) => a.offset > b.offset ? 1 : -1)
 	while (nextId = idsToInsert.pop()) {
 		let offset = nextId.offset
 		let id = nextId.id
@@ -927,6 +953,7 @@ function insertIds(serialized, idsToInsert) {
 function writeBundles(start, pack, incrementPosition) {
 	if (bundledStrings.length > 0) {
 		targetView.setUint32(bundledStrings.position + start, position + incrementPosition - bundledStrings.position - start)
+		bundledStrings.stringsPosition = position - start;
 		let writeStrings = bundledStrings
 		bundledStrings = null
 		pack(writeStrings[0])

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "msgpackr",
   "author": "Kris Zyp",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",
@@ -72,17 +72,18 @@
     "/*.ts"
   ],
   "optionalDependencies": {
-    "msgpackr-extract": "^2.1.2"
+    "msgpackr-extract": "^2.2.0"
   },
   "devDependencies": {
-    "@rollup/plugin-json": "^4.1.0",
+    "@rollup/plugin-json": "^5.0.1",
+    "@rollup/plugin-replace": "^5.0.1",
     "@types/node": "latest",
     "async": "^3",
     "chai": "^4.3.4",
     "cpy-cli": "^4.1.0",
     "esm": "^3.2.25",
-    "mocha": "^8.1.3",
-    "rollup": "^1.20.3",
-    "rollup-plugin-babel-minify": "^9.0.0"
+    "mocha": "^10.1.0",
+    "rollup": "^3.2.5",
+    "@rollup/plugin-terser": "^0.1.0"
   }
 }

package/rollup.config.js

@@ -1,5 +1,6 @@
-import minify from "rollup-plugin-babel-minify";
-import json  from "@rollup/plugin-json";
+import terser from '@rollup/plugin-terser';
+import json from "@rollup/plugin-json";
+import replace from "@rollup/plugin-replace";
 
 export default [
     {
@@ -20,11 +21,24 @@ export default [
             name: "msgpackr",
             sourcemap: true
         }
-    },    
+    },
+    {
+        input: "index.js",
+        plugins: [
+            replace({ Function: 'BlockedFunction '})
+        ],
+        output: {
+            file: "dist/index-no-eval.js",
+            format: "umd",
+            name: "msgpackr",
+            sourcemap: true
+        },
+    },
     {
         input: "index.js",
-        plugins: [minify({
-        })],
+        plugins: [
+            terser({})
+        ],
         output: {
             file: "dist/index.min.js",
             format: "umd",
@@ -32,6 +46,19 @@ export default [
             sourcemap: true
         }
     },
+    {
+        input: "index.js",
+        plugins: [
+            replace({ Function: 'BlockedFunction '}),
+            terser({})
+        ],
+        output: {
+            file: "dist/index-no-eval.min.js",
+            format: "umd",
+            name: "msgpackr",
+            sourcemap: true
+        }
+    },
     {
         input: "tests/test.js",
         plugins: [json()],

package/struct.js

@@ -161,9 +161,9 @@ function writeStruct(object, target, position, structures, makeRoom, pack, packr
 			case 'string':
 				let strLength = value.length;
 				refOffset = refPosition - refsStartPosition;
-				if ((strLength << 2) + position > safeEnd) {
+				if ((strLength << 2) + refPosition > safeEnd) {
 					let lastStart = start;
-					target = makeRoom(refPosition);
+					target = makeRoom((strLength << 2) + refPosition);
 					targetView = target.dataView;
 					position -= lastStart;
 					refsStartPosition -= lastStart;
@@ -776,7 +776,7 @@ function prepareStructures(structures, packr) {
 			let typed = existing.get('typed') || [];
 			if (typed.length !== lastTypedStructuresLength)
 				compatible = false;
-		} else if (existing instanceof Array) {
+		} else if (existing instanceof Array || Array.isArray(existing)) {
 			if (existing.length !== (packr.lastNamedStructuresLength || 0))
 				compatible = false;
 		}

package/unpack.js

@@ -28,6 +28,8 @@ C1.name = 'MessagePack 0xC1'
 var sequentialMode = false
 var inlineObjectReadThreshold = 2
 var readStruct, onLoadedStructures, onSaveState
+var BlockedFunction // we use search and replace to change the next call to BlockedFunction to avoid CSP issues for
+// no-eval build
 try {
 	new Function('')
 } catch(error) {
@@ -54,6 +56,9 @@ export class Unpackr {
 				(options.structures = []).uninitialized = true // this is what we use to denote an uninitialized structures
 				options.structures.sharedLength = 0
 			}
+			if (options.int64AsNumber) {
+				options.int64AsType = 'number'
+			}
 		}
 		Object.assign(this, options)
 	}
@@ -188,8 +193,10 @@ export function checkedRead(options) {
 			position = srcEnd
 		} else
 			result = read()
-		if (bundledStrings) // bundled strings to skip past
+		if (bundledStrings) { // bundled strings to skip past
 			position = bundledStrings.postBundlePosition
+			bundledStrings = null
+		}
 
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
@@ -354,9 +361,11 @@ export function read() {
 				position += 4
 				return value
 			case 0xcf:
-				if (currentUnpackr.int64AsNumber) {
+				if (currentUnpackr.int64AsType === 'number') {
 					value = dataView.getUint32(position) * 0x100000000
 					value += dataView.getUint32(position + 4)
+				} else if (currentUnpackr.int64AsType === 'string') {
+					value = dataView.getBigUint64(position).toString()
 				} else
 					value = dataView.getBigUint64(position)
 				position += 8
@@ -374,9 +383,11 @@ export function read() {
 				position += 4
 				return value
 			case 0xd3:
-				if (currentUnpackr.int64AsNumber) {
+				if (currentUnpackr.int64AsType === 'number') {
 					value = dataView.getInt32(position) * 0x100000000
 					value += dataView.getUint32(position + 4)
+				} else if (currentUnpackr.int64AsType === 'string') {
+					value = dataView.getBigInt64(position).toString()
 				} else
 					value = dataView.getBigInt64(position)
 				position += 8
@@ -884,7 +895,7 @@ function readKey() {
 			return readFixedString(length)
 	} else { // not cacheable, go back and do a standard read
 		position--
-		return read()
+		return read().toString()
 	}
 	let key = ((length << 5) ^ (length > 1 ? dataView.getUint16(position) : length > 0 ? src[position] : 0)) & 0xfff
 	let entry = keyCache[key]
@@ -938,16 +949,7 @@ function readKey() {
 
 // the registration of the record definition extension (as "r")
 const recordDefinition = (id, highByte) => {
-	let structure
-	if (currentUnpackr.freezeData) {
-		currentUnpackr.freezeData = false;
-		try {
-			structure = read()
-		} finally {
-			currentUnpackr.freezeData = true;
-		}
-	} else
-		structure = read()
+	let structure = read().map(property => property.toString()) // ensure that all keys are strings and that the array is mutable
 	let firstByte = id
 	if (highByte !== undefined) {
 		id = id < 32 ? -((highByte << 5) + id) : ((highByte << 5) + id)