ordered-binary 1.4.1 → 1.5.1

package/dist/index.cjs

@@ -2,6 +2,8 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
+require('constants');
+
 /*
 control character types:
 1 - metadata
@@ -14,9 +16,6 @@ control character types:
 0 - multipart separator
 > 27 normal string characters
 */
-/*
-* Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
-*/
 
 const float64Array = new Float64Array(2);
 const int32Array = new Int32Array(float64Array.buffer, 0, 4);
@@ -114,7 +113,7 @@ function writeKey(key, target, position, inSequence) {
 				position += key.length;
 				break
 			} else {
-				throw new Error('Unable to serialize object as a key')
+				throw new Error('Unable to serialize object as a key: ' + JSON.stringify(key))
 			}
 		} else // null
 			target[position++] = 0;
@@ -123,7 +122,44 @@ function writeKey(key, target, position, inSequence) {
 		targetView.setUint32(position++, key ? 7 : 6, true);
 		return position
 	case 'bigint':
-		return writeKey(Number(key), target, position, inSequence)
+		let asFloat = Number(key);
+		if (BigInt(asFloat) > key) {
+			float64Array[0] = asFloat;
+			if (asFloat > 0) {
+				if (int32Array[0])
+					int32Array[0]--;
+				else {
+					int32Array[1]--;
+					int32Array[0] = 0xffffffff;
+				}
+			} else {
+				if (int32Array[0] < 0xffffffff)
+					int32Array[0]++;
+				else {
+					int32Array[1]++;
+					int32Array[0] = 0;
+				}
+			}
+			asFloat = float64Array[0];
+		}
+		let difference = key - BigInt(asFloat);
+		if (difference === 0n)
+			return writeKey(asFloat, target, position, inSequence)
+		writeKey(asFloat, target, position, inSequence);
+		position += 9; // always increment by 9 if we are adding fractional bits
+		let exponent = BigInt((int32Array[1] >> 20 & 0x7ff) - 1079);
+		let nextByte = difference >> exponent;
+		target[position - 1] |= Number(nextByte);
+		difference -= nextByte << exponent;
+		let first = true;
+		while (difference || first) {
+			first = false;
+			exponent -= 7n;
+			let nextByte = difference >> exponent;
+			target[position++] = Number(nextByte) | 0x80;
+			difference -= nextByte << exponent;
+		}
+		return position;
 	case 'undefined':
 		return position
 	// undefined is interpreted as the absence of a key, signified by zero length
@@ -145,45 +181,63 @@ function readKey(buffer, start, end, inSequence) {
 	let controlByte = buffer[position];
 	let value;
 	if (controlByte < 24) {
-	if (controlByte < 8) {
-		position++;
-		if (controlByte == 6) {
-			value = false;
-		} else if (controlByte == 7) {
-			value = true;
-		} else if (controlByte == 0) {
-			value = null;
-		} else if (controlByte == 2) {
-			value = Symbol.for(readString(buffer));
-		} else
-			return Uint8Array.prototype.slice.call(buffer, start, end)
-	} else {
-		let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
-		let highInt = dataView.getInt32(position) << 4;
-		let size = end - position;
-		let lowInt;
-		if (size > 4) {
-			lowInt = dataView.getInt32(position + 4);
-			highInt |= lowInt >>> 28;
-			if (size <= 6) { // clear the last bits
-				lowInt &= -0x1000;
+		if (controlByte < 8) {
+			position++;
+			if (controlByte == 6) {
+				value = false;
+			} else if (controlByte == 7) {
+				value = true;
+			} else if (controlByte == 0) {
+				value = null;
+			} else if (controlByte == 2) {
+				value = Symbol.for(readString(buffer));
+			} else
+				return Uint8Array.prototype.slice.call(buffer, start, end)
+		} else {
+			let dataView;
+			try {
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
+			} catch(error) {
+				// if it is write at the end of the ArrayBuffer, we may need to retry with the exact remaining bytes
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.buffer.byteLength - buffer.byteOffset));
+			}
+
+			let highInt = dataView.getInt32(position) << 4;
+			let size = end - position;
+			let lowInt;
+			if (size > 4) {
+				lowInt = dataView.getInt32(position + 4);
+				highInt |= lowInt >>> 28;
+				if (size <= 6) { // clear the last bits
+					lowInt &= -0x1000;
+				}
+				lowInt = lowInt << 4;
+				if (size > 8) {
+					lowInt = lowInt | buffer[position + 8] >> 4;
+				}
+			} else
+				lowInt = 0;
+			if (controlByte < 16) {
+				// negative gets negated
+				highInt = highInt ^ 0x7fffffff;
+				lowInt = ~lowInt;
 			}
-			lowInt = lowInt << 4;
-			if (size > 8) {
-				lowInt = lowInt | buffer[position + 8] >> 4;
+			int32Array[1] = highInt;
+			int32Array[0] = lowInt;
+			value = float64Array[0];
+			position += 9;
+			if (size > 9 && buffer[position] > 0) {
+				// convert the float to bigint, and then we will add precision as we enumerate through the
+				// extra bytes
+				value = BigInt(value);
+				let exponent = highInt >> 20 & 0x7ff;
+				let next_byte = buffer[position - 1] & 0xf;
+				value += BigInt(next_byte) << BigInt(exponent - 1079);
+				while ((next_byte = buffer[position]) > 0 && position++ < end) {
+					value += BigInt(next_byte & 0x7f) << BigInt((start - position) * 7 + exponent - 1016);
+				}
 			}
-		} else
-			lowInt = 0;
-		if (controlByte < 16) {
-			// negative gets negated
-			highInt = highInt ^ 0x7fffffff;
-			lowInt = ~lowInt;
 		}
-		int32Array[1] = highInt;
-		int32Array[0] = lowInt;
-		value = float64Array[0];
-		position += 9;
-	}
 	} else {
 		if (controlByte == 27) {
 			position++;

package/index.js

@@ -14,6 +14,8 @@ control character types:
 * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
 */
 
+import exp from "constants"
+
 const float64Array = new Float64Array(2)
 const int32Array = new Int32Array(float64Array.buffer, 0, 4)
 let nullTerminate = false
@@ -110,7 +112,7 @@ export function writeKey(key, target, position, inSequence) {
 				position += key.length
 				break
 			} else {
-				throw new Error('Unable to serialize object as a key')
+				throw new Error('Unable to serialize object as a key: ' + JSON.stringify(key))
 			}
 		} else // null
 			target[position++] = 0
@@ -119,7 +121,44 @@ export function writeKey(key, target, position, inSequence) {
 		targetView.setUint32(position++, key ? 7 : 6, true)
 		return position
 	case 'bigint':
-		return writeKey(Number(key), target, position, inSequence)
+		let asFloat = Number(key)
+		if (BigInt(asFloat) > key) {
+			float64Array[0] = asFloat;
+			if (asFloat > 0) {
+				if (int32Array[0])
+					int32Array[0]--;
+				else {
+					int32Array[1]--;
+					int32Array[0] = 0xffffffff;
+				}
+			} else {
+				if (int32Array[0] < 0xffffffff)
+					int32Array[0]++;
+				else {
+					int32Array[1]++;
+					int32Array[0] = 0;
+				}
+			}
+			asFloat = float64Array[0];
+		}
+		let difference = key - BigInt(asFloat);
+		if (difference === 0n)
+			return writeKey(asFloat, target, position, inSequence)
+		writeKey(asFloat, target, position, inSequence)
+		position += 9; // always increment by 9 if we are adding fractional bits
+		let exponent = BigInt((int32Array[1] >> 20 & 0x7ff) - 1079);
+		let nextByte = difference >> exponent;
+		target[position - 1] |= Number(nextByte);
+		difference -= nextByte << exponent;
+		let first = true;
+		while (difference || first) {
+			first = false;
+			exponent -= 7n;
+			let nextByte = difference >> exponent;
+			target[position++] = Number(nextByte) | 0x80;
+			difference -= nextByte << exponent;
+		}
+		return position;
 	case 'undefined':
 		return position
 	// undefined is interpreted as the absence of a key, signified by zero length
@@ -141,45 +180,63 @@ export function readKey(buffer, start, end, inSequence) {
 	let controlByte = buffer[position]
 	let value
 	if (controlByte < 24) {
-	if (controlByte < 8) {
-		position++
-		if (controlByte == 6) {
-			value = false
-		} else if (controlByte == 7) {
-			value = true
-		} else if (controlByte == 0) {
-			value = null
-		} else if (controlByte == 2) {
-			value = Symbol.for(readString(buffer))
-		} else
-			return Uint8Array.prototype.slice.call(buffer, start, end)
-	} else {
-		let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2))
-		let highInt = dataView.getInt32(position) << 4
-		let size = end - position
-		let lowInt
-		if (size > 4) {
-			lowInt = dataView.getInt32(position + 4)
-			highInt |= lowInt >>> 28
-			if (size <= 6) { // clear the last bits
-				lowInt &= -0x1000
+		if (controlByte < 8) {
+			position++
+			if (controlByte == 6) {
+				value = false
+			} else if (controlByte == 7) {
+				value = true
+			} else if (controlByte == 0) {
+				value = null
+			} else if (controlByte == 2) {
+				value = Symbol.for(readString(buffer))
+			} else
+				return Uint8Array.prototype.slice.call(buffer, start, end)
+		} else {
+			let dataView;
+			try {
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2))
+			} catch(error) {
+				// if it is write at the end of the ArrayBuffer, we may need to retry with the exact remaining bytes
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.buffer.byteLength - buffer.byteOffset))
 			}
-			lowInt = lowInt << 4
-			if (size > 8) {
-				lowInt = lowInt | buffer[position + 8] >> 4
+
+			let highInt = dataView.getInt32(position) << 4
+			let size = end - position
+			let lowInt
+			if (size > 4) {
+				lowInt = dataView.getInt32(position + 4)
+				highInt |= lowInt >>> 28
+				if (size <= 6) { // clear the last bits
+					lowInt &= -0x1000
+				}
+				lowInt = lowInt << 4
+				if (size > 8) {
+					lowInt = lowInt | buffer[position + 8] >> 4
+				}
+			} else
+				lowInt = 0
+			if (controlByte < 16) {
+				// negative gets negated
+				highInt = highInt ^ 0x7fffffff
+				lowInt = ~lowInt
+			}
+			int32Array[1] = highInt
+			int32Array[0] = lowInt
+			value = float64Array[0]
+			position += 9
+			if (size > 9 && buffer[position] > 0) {
+				// convert the float to bigint, and then we will add precision as we enumerate through the
+				// extra bytes
+				value = BigInt(value);
+				let exponent = highInt >> 20 & 0x7ff;
+				let next_byte = buffer[position - 1] & 0xf;
+				value += BigInt(next_byte) << BigInt(exponent - 1079);
+				while ((next_byte = buffer[position]) > 0 && position++ < end) {
+					value += BigInt(next_byte & 0x7f) << BigInt((start - position) * 7 + exponent - 1016);
+				}
 			}
-		} else
-			lowInt = 0
-		if (controlByte < 16) {
-			// negative gets negated
-			highInt = highInt ^ 0x7fffffff
-			lowInt = ~lowInt
 		}
-		int32Array[1] = highInt
-		int32Array[0] = lowInt
-		value = float64Array[0]
-		position += 9
-	}
 	} else {
 		if (controlByte == 27) {
 			position++

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "ordered-binary",
   "author": "Kris Zyp",
-  "version": "1.4.1",
+  "version": "1.5.1",
   "description": "Conversion of JavaScript primitives to and from Buffer with binary order matching natural primitive order",
   "license": "MIT",
   "repository": {

package/tests/test.js

@@ -13,6 +13,12 @@ function assertBufferComparison(lesser, greater) {
   }
 }
 //var inspector = require('inspector'); inspector.open(9330, null, true); debugger
+let seed = 0;
+export function random() {
+	seed++;
+	let a = seed * 15485863;
+	return ((a * a * a) % 2038074743) / 2038074743;
+}
 
 suite('key buffers', () => {
 
@@ -40,6 +46,30 @@ suite('key buffers', () => {
     assertBufferComparison(toBufferKey(-0.001), toBufferKey(-0.000001))
     assertBufferComparison(toBufferKey(-5236532532532), toBufferKey(-5236532532531))
   })
+  test('bigint equivalence', () => {
+    assert.strictEqual(fromBufferKey(toBufferKey(-35913040084491349n)), -35913040084491349n)
+    assert.strictEqual(fromBufferKey(toBufferKey(6135421331404949076605986n)), 6135421331404949076605986n)
+    assert.strictEqual(fromBufferKey(toBufferKey(0xfffffffffffffffffffffn)), 0xfffffffffffffffffffffn)
+    assert.strictEqual(fromBufferKey(toBufferKey(12345678901234567890n)), 12345678901234567890n)
+    assert.deepEqual(fromBufferKey(toBufferKey([12345678901234567890n, 44])), [12345678901234567890n, 44])
+    assert.deepEqual(fromBufferKey(toBufferKey(['hi', 12345678901234567890n])), ['hi', 12345678901234567890n])
+    assert.deepEqual(fromBufferKey(toBufferKey([6135421331404949076605986n, 'after'])), [6135421331404949076605986n, 'after'])
+    assert.strictEqual(fromBufferKey(toBufferKey(132923456789012345678903533235253252353211125n)), 132923456789012345678903533235253252353211125n)
+    assert.strictEqual(fromBufferKey(toBufferKey(352n)), 352)
+    let num = 5325n
+    for (let i = 0; i < 1100; i++) {
+      num *= BigInt(Math.floor(random() * 3 + 1));
+      num -= BigInt(Math.floor(random() * 1000));
+      assert.strictEqual(BigInt(fromBufferKey(toBufferKey(num))), num)
+      assert.strictEqual(BigInt(fromBufferKey(toBufferKey(-num))), -num)
+    }
+    assert.strictEqual(fromBufferKey(toBufferKey(-352n)), -352)
+  })
+  test('bigint comparison', () => {
+    assertBufferComparison(toBufferKey(0xfffffffffffffffffffffn), toBufferKey(0x100fffffffffffffffffffn))
+    assertBufferComparison(toBufferKey(12345678901234567890), toBufferKey(12345678901234567890n))
+  })
+
   test('string equivalence', () => {
     assert.strictEqual(fromBufferKey(toBufferKey('4')), '4')
     assert.strictEqual(fromBufferKey(toBufferKey('hello')), 'hello')