ordered-binary 1.4.0 → 1.5.0

package/.nyc_output/5b6bd808-0399-4ee6-9395-344688761dfa.json

@@ -0,0 +1 @@
+{}

package/.nyc_output/processinfo/5b6bd808-0399-4ee6-9395-344688761dfa.json

@@ -0,0 +1 @@
+{"parent":null,"pid":240712,"argv":["/home/kzyp/.nvm/versions/node/v20.0.0/bin/node","/home/kzyp/dev/ordered-binary/node_modules/mocha/bin/mocha","--ui","tdd","--reporter","/home/kzyp/.local/share/JetBrains/Toolbox/apps/CLion/ch-0/222.4167.35.plugins/NodeJS/js/mocha-intellij/lib/mochaIntellijReporter.js","/home/kzyp/dev/ordered-binary/tests/test.js"],"execArgv":[],"cwd":"/home/kzyp/dev/ordered-binary","time":1688411366364,"ppid":240701,"coverageFilename":"/home/kzyp/dev/ordered-binary/.nyc_output/5b6bd808-0399-4ee6-9395-344688761dfa.json","externalId":"","uuid":"5b6bd808-0399-4ee6-9395-344688761dfa","files":[]}

package/.nyc_output/processinfo/index.json

@@ -0,0 +1 @@
+{"processes":{"5b6bd808-0399-4ee6-9395-344688761dfa":{"parent":null,"children":[]}},"files":{},"externalIds":{}}

package/dist/index.cjs

@@ -2,6 +2,8 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
+require('constants');
+
 /*
 control character types:
 1 - metadata
@@ -14,9 +16,6 @@ control character types:
 0 - multipart separator
 > 27 normal string characters
 */
-/*
-* Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
-*/
 
 const float64Array = new Float64Array(2);
 const int32Array = new Int32Array(float64Array.buffer, 0, 4);
@@ -69,7 +68,7 @@ function writeKey(key, target, position, inSequence) {
 			}
 		} else {
 			if (target.utf8Write)
-				position += target.utf8Write(key, position);
+				position += target.utf8Write(key, position, 0xffffffff);
 			else
 				position += textEncoder.encodeInto(key, target.subarray(position)).written;
 			if (position > target.length - 4)
@@ -114,7 +113,7 @@ function writeKey(key, target, position, inSequence) {
 				position += key.length;
 				break
 			} else {
-				throw new Error('Unable to serialize object as a key')
+				throw new Error('Unable to serialize object as a key: ' + JSON.stringify(key))
 			}
 		} else // null
 			target[position++] = 0;
@@ -123,7 +122,35 @@ function writeKey(key, target, position, inSequence) {
 		targetView.setUint32(position++, key ? 7 : 6, true);
 		return position
 	case 'bigint':
-		return writeKey(Number(key), target, position, inSequence)
+		let asFloat = Number(key);
+		if (BigInt(asFloat) > key) {
+			float64Array[0] = asFloat;
+			if (int32Array[0])
+				int32Array[0]--;
+			else {
+				int32Array[1]--;
+				int32Array[0] = 0xffffffff;
+			}
+			asFloat = float64Array[0];
+		}
+		let difference = key - BigInt(asFloat);
+		if (difference === 0n)
+			return writeKey(asFloat, target, position, inSequence)
+		writeKey(asFloat, target, position, inSequence);
+		position += 9; // always increment by 9 if we are adding fractional bits
+		let exponent = BigInt((int32Array[1] >> 20) - 1079);
+		let nextByte = difference >> exponent;
+		target[position - 1] |= Number(nextByte);
+		difference -= nextByte << exponent;
+		let first = true;
+		while (difference || first) {
+			first = false;
+			exponent -= 7n;
+			let nextByte = difference >> exponent;
+			target[position++] = Number(nextByte) | 0x80;
+			difference -= nextByte << exponent;
+		}
+		return position;
 	case 'undefined':
 		return position
 	// undefined is interpreted as the absence of a key, signified by zero length
@@ -145,45 +172,57 @@ function readKey(buffer, start, end, inSequence) {
 	let controlByte = buffer[position];
 	let value;
 	if (controlByte < 24) {
-	if (controlByte < 8) {
-		position++;
-		if (controlByte == 6) {
-			value = false;
-		} else if (controlByte == 7) {
-			value = true;
-		} else if (controlByte == 0) {
-			value = null;
-		} else if (controlByte == 2) {
-			value = Symbol.for(readString(buffer));
-		} else
-			return Uint8Array.prototype.slice.call(buffer, start, end)
-	} else {
-		let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
-		let highInt = dataView.getInt32(position) << 4;
-		let size = end - position;
-		let lowInt;
-		if (size > 4) {
-			lowInt = dataView.getInt32(position + 4);
-			highInt |= lowInt >>> 28;
-			if (size <= 6) { // clear the last bits
-				lowInt &= -0x1000;
+		if (controlByte < 8) {
+			position++;
+			if (controlByte == 6) {
+				value = false;
+			} else if (controlByte == 7) {
+				value = true;
+			} else if (controlByte == 0) {
+				value = null;
+			} else if (controlByte == 2) {
+				value = Symbol.for(readString(buffer));
+			} else
+				return Uint8Array.prototype.slice.call(buffer, start, end)
+		} else {
+			let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
+			let highInt = dataView.getInt32(position) << 4;
+			let size = end - position;
+			let lowInt;
+			if (size > 4) {
+				lowInt = dataView.getInt32(position + 4);
+				highInt |= lowInt >>> 28;
+				if (size <= 6) { // clear the last bits
+					lowInt &= -0x1000;
+				}
+				lowInt = lowInt << 4;
+				if (size > 8) {
+					lowInt = lowInt | buffer[position + 8] >> 4;
+				}
+			} else
+				lowInt = 0;
+			if (controlByte < 16) {
+				// negative gets negated
+				highInt = highInt ^ 0x7fffffff;
+				lowInt = ~lowInt;
 			}
-			lowInt = lowInt << 4;
-			if (size > 8) {
-				lowInt = lowInt | buffer[position + 8] >> 4;
+			int32Array[1] = highInt;
+			int32Array[0] = lowInt;
+			value = float64Array[0];
+			position += 9;
+			if (size > 9 && buffer[position] > 0) {
+				// convert the float to bigint, and then we will add precision as we enumerate through the
+				// extra bytes
+				value = BigInt(value);
+				let exponent = highInt >> 20;
+				// TODO: negatives?
+				let next_byte = buffer[position - 1] & 0xf;
+				value += BigInt(next_byte) << BigInt(exponent - 1079);
+				while ((next_byte = buffer[position]) > 0 && position++ < end) {
+					value += BigInt(next_byte & 0x7f) << BigInt((start - position) * 7 + exponent - 1016);
+				}
 			}
-		} else
-			lowInt = 0;
-		if (controlByte < 16) {
-			// negative gets negated
-			highInt = highInt ^ 0x7fffffff;
-			lowInt = ~lowInt;
 		}
-		int32Array[1] = highInt;
-		int32Array[0] = lowInt;
-		value = float64Array[0];
-		position += 9;
-	}
 	} else {
 		if (controlByte == 27) {
 			position++;
@@ -288,20 +327,19 @@ function finishUtf8(byte1, src) {
 	} else if ((byte1 & 0xf8) === 0xf0) {
 		// 4 bytes
 		if (pendingSurrogate) {
-		byte1 = pendingSurrogate;
-		pendingSurrogate = null;
-		position += 3;
-		return byte1
+			byte1 = pendingSurrogate;
+			pendingSurrogate = null;
+			position += 3;
+			return byte1
 		}
 		const byte2 = src[position++] & 0x3f;
 		const byte3 = src[position++] & 0x3f;
 		const byte4 = src[position++] & 0x3f;
 		let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4;
 		if (unit > 0xffff) {
-		unit -= 0x10000;
-		unit = 0xdc00 | (unit & 0x3ff);
-		pendingSurrogate = ((unit >>> 10) & 0x3ff) | 0xd800;
-		position -= 4; // reset so we can return the next part of the surrogate pair
+			pendingSurrogate = 0xdc00 | (unit & 0x3ff);
+			unit = (((unit - 0x10000) >>> 10) & 0x3ff) | 0xd800;
+			position -= 4; // reset so we can return the next part of the surrogate pair
 		}
 		return unit
 	} else {
@@ -309,7 +347,23 @@ function finishUtf8(byte1, src) {
 	}
 }
 
-const readString = eval(makeStringBuilder());
+const readString =
+	typeof process !== 'undefined' && process.isBun ? // the eval in bun doesn't properly closure on position, so we
+		// have to manually update it
+		(function(reading) {
+			let { setPosition, getPosition, readString } = reading;
+			return (source) => {
+				setPosition(position);
+				let value = readString(source);
+				position = getPosition();
+				return value;
+			};
+		})((new Function('fromCharCode', 'let position; let readString = ' + makeStringBuilder() +
+			';return {' +
+			'setPosition(p) { position = p },' +
+			'getPosition() { return position },' +
+			'readString }'))(fromCharCode)) :
+		eval(makeStringBuilder());
 
 function compareKeys(a, b) {
 	// compare with type consistency that matches binary comparison

package/index.js

@@ -14,6 +14,8 @@ control character types:
 * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
 */
 
+import exp from "constants"
+
 const float64Array = new Float64Array(2)
 const int32Array = new Int32Array(float64Array.buffer, 0, 4)
 let nullTerminate = false
@@ -65,7 +67,7 @@ export function writeKey(key, target, position, inSequence) {
 			}
 		} else {
 			if (target.utf8Write)
-				position += target.utf8Write(key, position)
+				position += target.utf8Write(key, position, 0xffffffff)
 			else
 				position += textEncoder.encodeInto(key, target.subarray(position)).written
 			if (position > target.length - 4)
@@ -110,7 +112,7 @@ export function writeKey(key, target, position, inSequence) {
 				position += key.length
 				break
 			} else {
-				throw new Error('Unable to serialize object as a key')
+				throw new Error('Unable to serialize object as a key: ' + JSON.stringify(key))
 			}
 		} else // null
 			target[position++] = 0
@@ -119,7 +121,35 @@ export function writeKey(key, target, position, inSequence) {
 		targetView.setUint32(position++, key ? 7 : 6, true)
 		return position
 	case 'bigint':
-		return writeKey(Number(key), target, position, inSequence)
+		let asFloat = Number(key)
+		if (BigInt(asFloat) > key) {
+			float64Array[0] = asFloat;
+			if (int32Array[0])
+				int32Array[0]--;
+			else {
+				int32Array[1]--;
+				int32Array[0] = 0xffffffff;
+			}
+			asFloat = float64Array[0];
+		}
+		let difference = key - BigInt(asFloat);
+		if (difference === 0n)
+			return writeKey(asFloat, target, position, inSequence)
+		writeKey(asFloat, target, position, inSequence)
+		position += 9; // always increment by 9 if we are adding fractional bits
+		let exponent = BigInt((int32Array[1] >> 20) - 1079);
+		let nextByte = difference >> exponent;
+		target[position - 1] |= Number(nextByte);
+		difference -= nextByte << exponent;
+		let first = true;
+		while (difference || first) {
+			first = false;
+			exponent -= 7n;
+			let nextByte = difference >> exponent;
+			target[position++] = Number(nextByte) | 0x80;
+			difference -= nextByte << exponent;
+		}
+		return position;
 	case 'undefined':
 		return position
 	// undefined is interpreted as the absence of a key, signified by zero length
@@ -141,45 +171,57 @@ export function readKey(buffer, start, end, inSequence) {
 	let controlByte = buffer[position]
 	let value
 	if (controlByte < 24) {
-	if (controlByte < 8) {
-		position++
-		if (controlByte == 6) {
-			value = false
-		} else if (controlByte == 7) {
-			value = true
-		} else if (controlByte == 0) {
-			value = null
-		} else if (controlByte == 2) {
-			value = Symbol.for(readString(buffer))
-		} else
-			return Uint8Array.prototype.slice.call(buffer, start, end)
-	} else {
-		let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2))
-		let highInt = dataView.getInt32(position) << 4
-		let size = end - position
-		let lowInt
-		if (size > 4) {
-			lowInt = dataView.getInt32(position + 4)
-			highInt |= lowInt >>> 28
-			if (size <= 6) { // clear the last bits
-				lowInt &= -0x1000
+		if (controlByte < 8) {
+			position++
+			if (controlByte == 6) {
+				value = false
+			} else if (controlByte == 7) {
+				value = true
+			} else if (controlByte == 0) {
+				value = null
+			} else if (controlByte == 2) {
+				value = Symbol.for(readString(buffer))
+			} else
+				return Uint8Array.prototype.slice.call(buffer, start, end)
+		} else {
+			let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2))
+			let highInt = dataView.getInt32(position) << 4
+			let size = end - position
+			let lowInt
+			if (size > 4) {
+				lowInt = dataView.getInt32(position + 4)
+				highInt |= lowInt >>> 28
+				if (size <= 6) { // clear the last bits
+					lowInt &= -0x1000
+				}
+				lowInt = lowInt << 4
+				if (size > 8) {
+					lowInt = lowInt | buffer[position + 8] >> 4
+				}
+			} else
+				lowInt = 0
+			if (controlByte < 16) {
+				// negative gets negated
+				highInt = highInt ^ 0x7fffffff
+				lowInt = ~lowInt
 			}
-			lowInt = lowInt << 4
-			if (size > 8) {
-				lowInt = lowInt | buffer[position + 8] >> 4
+			int32Array[1] = highInt
+			int32Array[0] = lowInt
+			value = float64Array[0]
+			position += 9
+			if (size > 9 && buffer[position] > 0) {
+				// convert the float to bigint, and then we will add precision as we enumerate through the
+				// extra bytes
+				value = BigInt(value);
+				let exponent = highInt >> 20;
+				// TODO: negatives?
+				let next_byte = buffer[position - 1] & 0xf;
+				value += BigInt(next_byte) << BigInt(exponent - 1079);
+				while ((next_byte = buffer[position]) > 0 && position++ < end) {
+					value += BigInt(next_byte & 0x7f) << BigInt((start - position) * 7 + exponent - 1016);
+				}
 			}
-		} else
-			lowInt = 0
-		if (controlByte < 16) {
-			// negative gets negated
-			highInt = highInt ^ 0x7fffffff
-			lowInt = ~lowInt
 		}
-		int32Array[1] = highInt
-		int32Array[0] = lowInt
-		value = float64Array[0]
-		position += 9
-	}
 	} else {
 		if (controlByte == 27) {
 			position++
@@ -284,20 +326,19 @@ function finishUtf8(byte1, src) {
 	} else if ((byte1 & 0xf8) === 0xf0) {
 		// 4 bytes
 		if (pendingSurrogate) {
-		byte1 = pendingSurrogate
-		pendingSurrogate = null
-		position += 3
-		return byte1
+			byte1 = pendingSurrogate
+			pendingSurrogate = null
+			position += 3
+			return byte1
 		}
 		const byte2 = src[position++] & 0x3f
 		const byte3 = src[position++] & 0x3f
 		const byte4 = src[position++] & 0x3f
 		let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4
 		if (unit > 0xffff) {
-		unit -= 0x10000
-		unit = 0xdc00 | (unit & 0x3ff)
-		pendingSurrogate = ((unit >>> 10) & 0x3ff) | 0xd800
-		position -= 4 // reset so we can return the next part of the surrogate pair
+			pendingSurrogate = 0xdc00 | (unit & 0x3ff)
+			unit = (((unit - 0x10000) >>> 10) & 0x3ff) | 0xd800
+			position -= 4 // reset so we can return the next part of the surrogate pair
 		}
 		return unit
 	} else {
@@ -305,7 +346,23 @@ function finishUtf8(byte1, src) {
 	}
 }
 
-const readString = eval(makeStringBuilder())
+const readString =
+	typeof process !== 'undefined' && process.isBun ? // the eval in bun doesn't properly closure on position, so we
+		// have to manually update it
+		(function(reading) {
+			let { setPosition, getPosition, readString } = reading;
+			return (source) => {
+				setPosition(position);
+				let value = readString(source);
+				position = getPosition();
+				return value;
+			};
+		})((new Function('fromCharCode', 'let position; let readString = ' + makeStringBuilder() +
+			';return {' +
+			'setPosition(p) { position = p },' +
+			'getPosition() { return position },' +
+			'readString }'))(fromCharCode)) :
+		eval(makeStringBuilder())
 
 export function compareKeys(a, b) {
 	// compare with type consistency that matches binary comparison

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "ordered-binary",
   "author": "Kris Zyp",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "Conversion of JavaScript primitives to and from Buffer with binary order matching natural primitive order",
   "license": "MIT",
   "repository": {

package/tests/test.js

@@ -13,6 +13,12 @@ function assertBufferComparison(lesser, greater) {
   }
 }
 //var inspector = require('inspector'); inspector.open(9330, null, true); debugger
+let seed = 0;
+export function random() {
+	seed++;
+	let a = seed * 15485863;
+	return ((a * a * a) % 2038074743) / 2038074743;
+}
 
 suite('key buffers', () => {
 
@@ -40,12 +46,35 @@ suite('key buffers', () => {
     assertBufferComparison(toBufferKey(-0.001), toBufferKey(-0.000001))
     assertBufferComparison(toBufferKey(-5236532532532), toBufferKey(-5236532532531))
   })
+  test('bigint equivalence', () => {
+    assert.strictEqual(fromBufferKey(toBufferKey(6135421331404949076605986n)), 6135421331404949076605986n)
+    assert.strictEqual(fromBufferKey(toBufferKey(0xfffffffffffffffffffffn)), 0xfffffffffffffffffffffn)
+    assert.strictEqual(fromBufferKey(toBufferKey(12345678901234567890n)), 12345678901234567890n)
+    assert.deepEqual(fromBufferKey(toBufferKey([12345678901234567890n, 44])), [12345678901234567890n, 44])
+    assert.deepEqual(fromBufferKey(toBufferKey(['hi', 12345678901234567890n])), ['hi', 12345678901234567890n])
+    assert.deepEqual(fromBufferKey(toBufferKey([6135421331404949076605986n, 'after'])), [6135421331404949076605986n, 'after'])
+    assert.strictEqual(fromBufferKey(toBufferKey(132923456789012345678903533235253252353211125n)), 132923456789012345678903533235253252353211125n)
+    assert.strictEqual(fromBufferKey(toBufferKey(352n)), 352)
+    let num = 5325n
+    for (let i = 0; i < 1100; i++) {
+      num *= BigInt(Math.floor(random() * 3 + 1));
+      num -= BigInt(Math.floor(random() * 1000));
+      assert.strictEqual(BigInt(fromBufferKey(toBufferKey(num))), num)
+    }
+    assert.strictEqual(fromBufferKey(toBufferKey(-352n)), -352)
+  })
+  test('bigint comparison', () => {
+    assertBufferComparison(toBufferKey(0xfffffffffffffffffffffn), toBufferKey(0x100fffffffffffffffffffn))
+    assertBufferComparison(toBufferKey(12345678901234567890), toBufferKey(12345678901234567890n))
+  })
+
   test('string equivalence', () => {
     assert.strictEqual(fromBufferKey(toBufferKey('4')), '4')
     assert.strictEqual(fromBufferKey(toBufferKey('hello')), 'hello')
     assert.strictEqual(fromBufferKey(toBufferKey('')), '')
     assert.strictEqual(fromBufferKey(toBufferKey('\x00')), '\x00')
     assert.strictEqual(fromBufferKey(toBufferKey('\x03test\x01\x00')), '\x03test\x01\x00')
+    assert.strictEqual(fromBufferKey(toBufferKey('prance 🧚🏻‍♀️🩷')), 'prance 🧚🏻‍♀️🩷')
   })
   test('string comparison', () => {
     assertBufferComparison(toBufferKey('4'), toBufferKey('5'))