msgpackr 1.6.2 → 1.7.0-alpha3

package/index.js

@@ -1,5 +1,5 @@
-export { Packr, Encoder, addExtension, pack, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './pack.js'
-export { Unpackr, Decoder, C1, unpack, unpackMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './unpack.js'
-export { decodeIter, encodeIter } from './iterators.js'
-export const useRecords = false
-export const mapsAsObjects = true
+export { Packr, Encoder, addExtension, pack, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './pack.js'
+export { Unpackr, Decoder, C1, unpack, unpackMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './unpack.js'
+export { decodeIter, encodeIter } from './iterators.js'
+export const useRecords = false
+export const mapsAsObjects = true

package/iterators.js

@@ -1,87 +1,87 @@
-import { Packr } from './pack.js'
-import { Unpackr } from './unpack.js'
-
-/**
- * Given an Iterable first argument, returns an Iterable where each value is packed as a Buffer
- * If the argument is only Async Iterable, the return value will be an Async Iterable.
- * @param {Iterable|Iterator|AsyncIterable|AsyncIterator} objectIterator - iterable source, like a Readable object stream, an array, Set, or custom object
- * @param {options} [options] - msgpackr pack options
- * @returns {IterableIterator|Promise.<AsyncIterableIterator>}
- */
-export function packIter (objectIterator, options = {}) {
-  if (!objectIterator || typeof objectIterator !== 'object') {
-    throw new Error('first argument must be an Iterable, Async Iterable, or a Promise for an Async Iterable')
-  } else if (typeof objectIterator[Symbol.iterator] === 'function') {
-    return packIterSync(objectIterator, options)
-  } else if (typeof objectIterator.then === 'function' || typeof objectIterator[Symbol.asyncIterator] === 'function') {
-    return packIterAsync(objectIterator, options)
-  } else {
-    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a Promise')
-  }
-}
-
-function * packIterSync (objectIterator, options) {
-  const packr = new Packr(options)
-  for (const value of objectIterator) {
-    yield packr.pack(value)
-  }
-}
-
-async function * packIterAsync (objectIterator, options) {
-  const packr = new Packr(options)
-  for await (const value of objectIterator) {
-    yield packr.pack(value)
-  }
-}
-
-/**
- * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
- * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
- * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
- * @param {object} [options] - unpackr options
- * @returns {IterableIterator|Promise.<AsyncIterableIterator}
- */
-export function unpackIter (bufferIterator, options = {}) {
-  if (!bufferIterator || typeof bufferIterator !== 'object') {
-    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a promise')
-  }
-
-  const unpackr = new Unpackr(options)
-  let incomplete
-  const parser = (chunk) => {
-    let yields
-    // if there's incomplete data from previous chunk, concatinate and try again
-    if (incomplete) {
-      chunk = Buffer.concat([incomplete, chunk])
-      incomplete = undefined
-    }
-
-    try {
-      yields = unpackr.unpackMultiple(chunk)
-    } catch (err) {
-      if (err.incomplete) {
-        incomplete = chunk.slice(err.lastPosition)
-        yields = err.values
-      } else {
-        throw err
-      }
-    }
-    return yields
-  }
-
-  if (typeof bufferIterator[Symbol.iterator] === 'function') {
-    return (function * iter () {
-      for (const value of bufferIterator) {
-        yield * parser(value)
-      }
-    })()
-  } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
-    return (async function * iter () {
-      for await (const value of bufferIterator) {
-        yield * parser(value)
-      }
-    })()
-  }
-}
-export const decodeIter = unpackIter
+import { Packr } from './pack.js'
+import { Unpackr } from './unpack.js'
+
+/**
+ * Given an Iterable first argument, returns an Iterable where each value is packed as a Buffer
+ * If the argument is only Async Iterable, the return value will be an Async Iterable.
+ * @param {Iterable|Iterator|AsyncIterable|AsyncIterator} objectIterator - iterable source, like a Readable object stream, an array, Set, or custom object
+ * @param {options} [options] - msgpackr pack options
+ * @returns {IterableIterator|Promise.<AsyncIterableIterator>}
+ */
+export function packIter (objectIterator, options = {}) {
+  if (!objectIterator || typeof objectIterator !== 'object') {
+    throw new Error('first argument must be an Iterable, Async Iterable, or a Promise for an Async Iterable')
+  } else if (typeof objectIterator[Symbol.iterator] === 'function') {
+    return packIterSync(objectIterator, options)
+  } else if (typeof objectIterator.then === 'function' || typeof objectIterator[Symbol.asyncIterator] === 'function') {
+    return packIterAsync(objectIterator, options)
+  } else {
+    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a Promise')
+  }
+}
+
+function * packIterSync (objectIterator, options) {
+  const packr = new Packr(options)
+  for (const value of objectIterator) {
+    yield packr.pack(value)
+  }
+}
+
+async function * packIterAsync (objectIterator, options) {
+  const packr = new Packr(options)
+  for await (const value of objectIterator) {
+    yield packr.pack(value)
+  }
+}
+
+/**
+ * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
+ * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
+ * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
+ * @param {object} [options] - unpackr options
+ * @returns {IterableIterator|Promise.<AsyncIterableIterator}
+ */
+export function unpackIter (bufferIterator, options = {}) {
+  if (!bufferIterator || typeof bufferIterator !== 'object') {
+    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a promise')
+  }
+
+  const unpackr = new Unpackr(options)
+  let incomplete
+  const parser = (chunk) => {
+    let yields
+    // if there's incomplete data from previous chunk, concatinate and try again
+    if (incomplete) {
+      chunk = Buffer.concat([incomplete, chunk])
+      incomplete = undefined
+    }
+
+    try {
+      yields = unpackr.unpackMultiple(chunk)
+    } catch (err) {
+      if (err.incomplete) {
+        incomplete = chunk.slice(err.lastPosition)
+        yields = err.values
+      } else {
+        throw err
+      }
+    }
+    return yields
+  }
+
+  if (typeof bufferIterator[Symbol.iterator] === 'function') {
+    return (function * iter () {
+      for (const value of bufferIterator) {
+        yield * parser(value)
+      }
+    })()
+  } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
+    return (async function * iter () {
+      for await (const value of bufferIterator) {
+        yield * parser(value)
+      }
+    })()
+  }
+}
+export const decodeIter = unpackIter
 export const encodeIter = packIter

package/node-index.js

@@ -1,5 +1,6 @@
 export { Packr, Encoder, addExtension, pack, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } from './pack.js'
 export { Unpackr, Decoder, C1, unpack, unpackMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './unpack.js'
+import './struct.js'
 export { PackrStream, UnpackrStream, PackrStream as EncoderStream, UnpackrStream as DecoderStream } from './stream.js'
 export { decodeIter, encodeIter } from './iterators.js'
 export const useRecords = false

package/pack.d.ts

@@ -1,9 +1,9 @@
-import { Unpackr } from './unpack'
-export { addExtension, FLOAT32_OPTIONS } from './unpack'
-export class Packr extends Unpackr {
-	pack(value: any): Buffer
-	encode(value: any): Buffer
-}
-export class Encoder extends Packr {}
-export function pack(value: any): Buffer
-export function encode(value: any): Buffer
+import { Unpackr } from './unpack'
+export { addExtension, FLOAT32_OPTIONS } from './unpack'
+export class Packr extends Unpackr {
+	pack(value: any): Buffer
+	encode(value: any): Buffer
+}
+export class Encoder extends Packr {}
+export function pack(value: any): Buffer
+export function encode(value: any): Buffer

package/pack.js

@@ -15,9 +15,10 @@ let targetView
 let position = 0
 let safeEnd
 let bundledStrings = null
+let writeStructSlots, prepareStructures
 const MAX_BUNDLE_SIZE = 0xf000
 const hasNonLatin = /[\u0080-\uFFFF]/
-const RECORD_SYMBOL = Symbol('record-id')
+export const RECORD_SYMBOL = Symbol('record-id')
 export class Packr extends Unpackr {
 	constructor(options) {
 		super(options)
@@ -27,7 +28,6 @@ export class Packr extends Unpackr {
 		let hasSharedUpdate
 		let structures
 		let referenceMap
-		let lastSharedStructuresLength = 0
 		let encodeUtf8 = ByteArray.prototype.utf8Write ? function(string, position) {
 			return target.utf8Write(string, position, 0xffffffff)
 		} : (textEncoder && textEncoder.encodeInto) ?
@@ -67,14 +67,14 @@ export class Packr extends Unpackr {
 		this.pack = this.encode = function(value, encodeOptions) {
 			if (!target) {
 				target = new ByteArrayAllocate(8192)
-				targetView = new DataView(target.buffer, 0, 8192)
+				targetView = target.dataView = new DataView(target.buffer, 0, 8192)
 				position = 0
 			}
 			safeEnd = target.length - 10
 			if (safeEnd - position < 0x800) {
 				// don't start too close to the end, 
 				target = new ByteArrayAllocate(target.length)
-				targetView = new DataView(target.buffer, 0, target.length)
+				targetView = target.dataView = new DataView(target.buffer, 0, target.length)
 				safeEnd = target.length - 10
 				position = 0
 			} else
@@ -113,7 +113,7 @@ export class Packr extends Unpackr {
 						}
 						transition[RECORD_SYMBOL] = i + 0x40
 					}
-					lastSharedStructuresLength = sharedLength
+					this.lastNamedStructuresLength = sharedLength
 				}
 				if (!isSequential) {
 					structures.nextId = sharedLength + 0x40
@@ -122,7 +122,10 @@ export class Packr extends Unpackr {
 			if (hasSharedUpdate)
 				hasSharedUpdate = false
 			try {
-				pack(value)
+				if (packr.randomAccessStructure && value.constructor && value.constructor === Object)
+					writeStruct(value);
+				else
+					pack(value)
 				if (bundledStrings) {
 					writeBundles(start, pack)
 				}
@@ -165,12 +168,13 @@ export class Packr extends Unpackr {
 					if (hasSharedUpdate && packr.saveStructures) {
 						// we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
 						let returnBuffer = target.subarray(start, position)
-						if (packr.saveStructures(structures, lastSharedStructuresLength) === false) {
+						let newSharedData = prepareStructures ? prepareStructures(structures, packr) : structures;
+						if (packr.saveStructures(newSharedData, newSharedData.isCompatible || packr.lastNamedStructuresLength || 0) === false) {
 							// get updated structures and try again if the update failed
 							packr._mergeStructures(packr.getStructures())
 							return packr.pack(value)
 						}
-						lastSharedStructuresLength = sharedLength
+						packr.lastNamedStructuresLength = sharedLength
 						return returnBuffer
 					}
 				}
@@ -288,7 +292,7 @@ export class Packr extends Unpackr {
 			} else if (type === 'number') {
 				if (value >>> 0 === value) {// positive integer, 32-bit or less
 					// positive uint
-					if (value < 0x40 || (value < 0x80 && this.useRecords === false)) {
+					if (value < 0x20 || (value < 0x80 && this.useRecords === false) || (value < 0x40 && !this.randomAccessStructure)) {
 						target[position++] = value
 					} else if (value < 0x100) {
 						target[position++] = 0xcc
@@ -596,7 +600,7 @@ export class Packr extends Unpackr {
 			} else // faster handling for smaller buffers
 				newSize = ((Math.max((end - start) << 2, target.length - 1) >> 12) + 1) << 12
 			let newBuffer = new ByteArrayAllocate(newSize)
-			targetView = new DataView(newBuffer.buffer, 0, newSize)
+			targetView = newBuffer.dataView = new DataView(newBuffer.buffer, 0, newSize)
 			end = Math.min(end, target.length)
 			if (target.copy)
 				target.copy(newBuffer, 0, start, end)
@@ -687,6 +691,23 @@ export class Packr extends Unpackr {
 				target[insertionOffset + start] = keysTarget[0]
 			}
 		}
+		const writeStruct = (object, safePrototype) => {
+			let newPosition = writeStructSlots(object, target, position, structures, makeRoom, (value, newPosition, notifySharedUpdate) => {
+				if (notifySharedUpdate)
+					hasSharedUpdate = true;
+				position = newPosition;
+				if (start > 0) {
+					pack(value);
+					if (start == 0)
+						return { position, targetView, target }; // indicate the buffer was re-allocated
+				} else
+					pack(value);
+				return position;
+			}, this);
+			if (newPosition === 0) // bail and go to a msgpack object
+				return writeObject(object, true);
+			position = newPosition;
+		}
 	}
 	useBuffer(buffer) {
 		// this means we are finished using our own buffer and we can write over it safely
@@ -697,6 +718,8 @@ export class Packr extends Unpackr {
 	clearSharedData() {
 		if (this.structures)
 			this.structures = []
+		if (this.typedStructs)
+			this.typedStructs = []
 	}
 }
 
@@ -923,6 +946,10 @@ export function addExtension(extension) {
 	}
 	unpackAddExtension(extension)
 }
+export function setWriteStructSlots(writeSlots, makeStructures) {
+	writeStructSlots = writeSlots;
+	prepareStructures = makeStructures;
+}
 
 let defaultPackr = new Packr({ useRecords: false })
 export const pack = defaultPackr.pack

package/package.json

@@ -1,83 +1,83 @@
-{
-  "name": "msgpackr",
-  "author": "Kris Zyp",
-  "version": "1.6.2",
-  "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
-  "license": "MIT",
-  "types": "./index.d.ts",
-  "main": "./dist/node.cjs",
-  "module": "./index.js",
-  "keywords": [
-    "MessagePack",
-    "msgpack",
-    "performance",
-    "structured",
-    "clone"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "http://github.com/kriszyp/msgpackr"
-  },
-  "scripts": {
-    "benchmark": "node ./tests/benchmark.cjs",
-    "build": "rollup -c",
-    "dry-run": "npm publish --dry-run",
-    "prepare": "npm run build",
-    "test": "mocha tests/test**.*js -u tdd --experimental-json-modules"
-  },
-  "type": "module",
-  "exports": {
-    ".": {
-      "node": {
-        "require": "./dist/node.cjs",
-        "import": "./node-index.js"
-      },
-      "bun": {
-        "require": "./dist/node.cjs",
-        "import": "./node-index.js"
-      },
-      "default": "./index.js"
-    },
-    "./pack": {
-      "node": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "bun": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "default": "./pack.js"
-    },
-    "./unpack": {
-      "node": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "bun": {
-        "import": "./index.js",
-        "require": "./dist/node.cjs"
-      },
-      "default": "./unpack.js"
-    }
-  },
-  "files": [
-    "/dist",
-    "*.md",
-    "/*.js",
-    "/*.ts"
-  ],
-  "optionalDependencies": {
-    "msgpackr-extract": "^2.0.2"
-  },
-  "devDependencies": {
-    "@rollup/plugin-json": "^4.1.0",
-    "@types/node": "latest",
-    "async": "^3",
-    "chai": "^4.3.4",
-    "esm": "^3.2.25",
-    "mocha": "^8.1.3",
-    "rollup": "^1.20.3",
-    "rollup-plugin-babel-minify": "^9.0.0"
-  }
-}
+{
+  "name": "msgpackr",
+  "author": "Kris Zyp",
+  "version": "1.7.0-alpha3",
+  "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
+  "license": "MIT",
+  "types": "./index.d.ts",
+  "main": "./dist/node.cjs",
+  "module": "./index.js",
+  "keywords": [
+    "MessagePack",
+    "msgpack",
+    "performance",
+    "structured",
+    "clone"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "http://github.com/kriszyp/msgpackr"
+  },
+  "scripts": {
+    "benchmark": "node ./tests/benchmark.cjs",
+    "build": "rollup -c",
+    "dry-run": "npm publish --dry-run",
+    "prepare": "npm run build",
+    "test": "mocha tests/test**.*js -u tdd --experimental-json-modules"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "node": {
+        "require": "./dist/node.cjs",
+        "import": "./node-index.js"
+      },
+      "bun": {
+        "require": "./dist/node.cjs",
+        "import": "./node-index.js"
+      },
+      "default": "./index.js"
+    },
+    "./pack": {
+      "node": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "bun": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "default": "./pack.js"
+    },
+    "./unpack": {
+      "node": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "bun": {
+        "import": "./index.js",
+        "require": "./dist/node.cjs"
+      },
+      "default": "./unpack.js"
+    }
+  },
+  "files": [
+    "/dist",
+    "*.md",
+    "/*.js",
+    "/*.ts"
+  ],
+  "optionalDependencies": {
+    "msgpackr-extract": "^2.1.1"
+  },
+  "devDependencies": {
+    "@rollup/plugin-json": "^4.1.0",
+    "@types/node": "latest",
+    "async": "^3",
+    "chai": "^4.3.4",
+    "esm": "^3.2.25",
+    "mocha": "^8.1.3",
+    "rollup": "^1.20.3",
+    "rollup-plugin-babel-minify": "^9.0.0"
+  }
+}

package/rollup.config.js

@@ -1,45 +1,49 @@
-import minify from "rollup-plugin-babel-minify";
-import json  from "@rollup/plugin-json";
-
-export default [
-    {
-        input: "node-index.js",
-        output: [
-            {
-                file: "dist/node.cjs",
-                format: "cjs"
-            }
-        ]
-    },
-    {
-        input: "index.js",
-        output: {
-            file: "dist/index.js",
-            format: "umd",
-            name: "msgpackr"
-        }
-    },    
-    {
-        input: "index.js",
-        plugins: [minify({
-        })],
-        output: {
-            file: "dist/index.min.js",
-            format: "umd",
-            name: "msgpackr"
-        }
-    },
-    {
-        input: "tests/test.js",
-        plugins: [json()],
-        external: ['chai', '../index.js'],
-        output: {
-            file: "dist/test.js",
-            format: "iife",
-            globals: {
-                chai: 'chai',
-                './index.js': 'msgpackr',
-            },
-        }
-    }
-];
+import minify from "rollup-plugin-babel-minify";
+import json  from "@rollup/plugin-json";
+
+export default [
+    {
+        input: "node-index.js",
+        output: [
+            {
+                file: "dist/node.cjs",
+                format: "cjs",
+                sourcemap: true
+            }
+        ]
+    },
+    {
+        input: "index.js",
+        output: {
+            file: "dist/index.js",
+            format: "umd",
+            name: "msgpackr",
+            sourcemap: true
+        }
+    },    
+    {
+        input: "index.js",
+        plugins: [minify({
+        })],
+        output: {
+            file: "dist/index.min.js",
+            format: "umd",
+            name: "msgpackr",
+            sourcemap: true
+        }
+    },
+    {
+        input: "tests/test.js",
+        plugins: [json()],
+        external: ['chai', '../index.js'],
+        output: {
+            file: "dist/test.js",
+            format: "iife",
+            sourcemap: true,
+            globals: {
+                chai: 'chai',
+                './index.js': 'msgpackr',
+            },
+        }
+    }
+];