@cloudpss/ubjson 0.3.6 → 0.3.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloudpss/ubjson",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "author": "CloudPSS",
   "license": "MIT",
   "keywords": [
@@ -14,8 +14,14 @@
   "module": "dist/index.js",
   "types": "dist/index.d.ts",
   "exports": {
-    "types": "./dist/index.d.ts",
-    "default": "./dist/index.js"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./stream": {
+      "types": "./dist/stream/index.d.ts",
+      "default": "./dist/stream/index.js"
+    }
   },
   "scripts": {
     "start": "yarn clean && tsc --watch",
@@ -23,11 +29,13 @@
     "prepublishOnly": "yarn build",
     "clean": "rimraf dist",
     "benchmark": "node ./benchmark",
-    "test": "c8 --all --src ./src tape-es \"./test/**/*.js\""
+    "test": "c8 --all --src ./src tape-es \"./test/**/*.js\"",
+    "test:file": "c8 --all --src ./src tape-es"
   },
   "devDependencies": {
-    "c8": "^7.10.0",
-    "tape": "^5.3.2",
+    "@msgpack/msgpack": "^2.7.2",
+    "c8": "^7.11.3",
+    "tape": "^5.5.3",
     "tape-es": "^1.2.17"
   }
 }

package/src/{decoder.ts → common/decoder.ts}

@@ -2,7 +2,7 @@ import * as constants from './constants.js';
 import { StringDecoder } from './string-decoder.js';
 
 /** decoder */
-class Decoder {
+export class Decoder {
     private readonly view: DataView;
     /** 当前读指针位置 */
     private offset = 0;
@@ -93,11 +93,12 @@ class Decoder {
             case constants.INT64:
                 length = this.readInt64Data();
                 break;
+            default:
+                throw new Error(`Unexpected marker '${String.fromCharCode(marker)}'(${marker}) for int length`);
         }
-        if (length === undefined) {
-            throw new Error(`Unexpected marker '${String.fromCharCode(marker)}'(${marker}) for int length`);
+        if (length < 0) {
+            throw new Error('Invalid length');
         }
-        if (length < 0) throw new Error('Invalid length');
         return length;
     }
 
@@ -107,9 +108,9 @@ class Decoder {
     private readStringData(): string {
         const length = this.readIntLength();
         this.ensureData(length);
-        const buffer = this.data.subarray(this.offset, this.offset + length);
+        const begin = this.offset;
         this.offset += length;
-        return this.stringDecoder.decode(buffer);
+        return this.stringDecoder.decode(this.data, begin, this.offset);
     }
     /** readHighPrecisionNumberData */
     private readHighPrecisionNumberData(): never {
@@ -247,7 +248,8 @@ class Decoder {
                 this.ensureData(count * 8);
                 return BigInt64Array.from({ length: count }, () => this.readInt64Data());
         }
-        const array = new Array<unknown>(count);
+        const array: unknown[] = [];
+        array.length = count;
         for (let i = 0; i < count; i++) {
             array[i] = type == null ? this.read() : this.readData(type);
         }
@@ -262,24 +264,17 @@ class Decoder {
             const object: Array<[string, unknown]> = [];
             while (this.readMarker() !== constants.OBJECT_END) {
                 this.offset--;
-                const key = this.readStringData();
-                object.push([key, this.read()]);
+                object.push([this.readStringData(), this.read()]);
             }
             return Object.fromEntries(object);
         }
 
         const { count, type } = markers;
-        const object = new Array<[string, unknown]>(count);
+        const object: Array<[string, unknown]> = [];
+        object.length = count;
         for (let i = 0; i < count; i++) {
-            const key = this.readStringData();
-            object[i] = [key, type == null ? this.read() : this.readData(type)];
+            object[i] = [this.readStringData(), type == null ? this.read() : this.readData(type)];
         }
         return Object.fromEntries(object);
     }
 }
-
-/** 解码 */
-export function decode(value: Uint8Array): unknown {
-    const decoder = new Decoder(value);
-    return decoder.decode();
-}

package/src/common/encoder.ts

@@ -0,0 +1,305 @@
+/* eslint-disable @typescript-eslint/unbound-method */
+import * as constants from './constants.js';
+import { StringEncoder } from './string-encoder.js';
+
+/** 编码至 ubjson */
+export abstract class EncoderBase {
+    /** 字符串编码 */
+    protected stringEncoder = new StringEncoder();
+    /** 当前写指针位置 */
+    protected length = 0;
+    /** 数据 */
+    protected buffer!: Uint8Array;
+    /** buffer 的 DataView */
+    protected view!: DataView;
+
+    constructor(readonly value: unknown) {}
+    /**
+     * 确保 buffer 还有 capacity 的空闲空间
+     */
+    protected abstract ensureCapacity(capacity: number): void;
+    /** 写入一个对象 */
+    protected write(value: unknown): void {
+        switch (typeof value) {
+            case 'undefined':
+                return this.writeNoOp();
+            case 'boolean':
+                return this.writeBoolean(value);
+            case 'number':
+                return this.writeNumber(value);
+            case 'string':
+                if (value.length === 1 && value.charCodeAt(0) < 255) {
+                    // 1 byte string
+                    return this.writeChar(value);
+                }
+                return this.writeString(value);
+            case 'object': {
+                if (value === null) {
+                    return this.writeNull();
+                }
+                if (Array.isArray(value)) {
+                    return this.writeArray(value);
+                }
+                if (ArrayBuffer.isView(value)) {
+                    return this.writeTypedArray(value);
+                }
+                return this.writeObject(value as Record<string, unknown>);
+            }
+        }
+        throw new Error(`Unsupported type ${Object.prototype.toString.call(value)}`);
+    }
+    /** 写入 marker */
+    protected writeMarker(marker: number): void {
+        this.ensureCapacity(1);
+        this.view.setUint8(this.length, marker);
+        this.length += 1;
+    }
+
+    /** 写入 marker */
+    protected writeNull(): void {
+        this.writeMarker(constants.NULL);
+    }
+
+    /** writeNoOp */
+    protected writeNoOp(): void {
+        this.writeMarker(constants.NO_OP);
+    }
+
+    /** writeBoolean */
+    protected writeBoolean(value: boolean): void {
+        this.writeMarker(value ? constants.TRUE : constants.FALSE);
+    }
+
+    /** writeInt8 */
+    protected writeInt8(value: number): void {
+        this.writeMarker(constants.INT8);
+        this.writeInt8Data(value);
+    }
+
+    /** writeInt8Data */
+    protected writeInt8Data(value: number): void {
+        this.ensureCapacity(1);
+        this.view.setInt8(this.length, value);
+        this.length += 1;
+    }
+
+    /** writeUint8 */
+    protected writeUint8(value: number): void {
+        this.writeMarker(constants.UINT8);
+        this.writeUint8Data(value);
+    }
+
+    /** writeUint8Data */
+    protected writeUint8Data(value: number): void {
+        this.ensureCapacity(1);
+        this.view.setUint8(this.length, value);
+        this.length += 1;
+    }
+
+    /** writeInt16 */
+    protected writeInt16(value: number): void {
+        this.writeMarker(constants.INT16);
+        this.writeInt16Data(value);
+    }
+
+    /** writeInt16Data */
+    protected writeInt16Data(value: number): void {
+        this.ensureCapacity(2);
+        this.view.setInt16(this.length, value);
+        this.length += 2;
+    }
+
+    /** writeInt32 */
+    protected writeInt32(value: number): void {
+        this.writeMarker(constants.INT32);
+        this.writeInt32Data(value);
+    }
+
+    /** writeInt32Data */
+    protected writeInt32Data(value: number): void {
+        this.ensureCapacity(4);
+        this.view.setInt32(this.length, value);
+        this.length += 4;
+    }
+
+    /** writeFloat32 */
+    protected writeFloat32(value: number): void {
+        this.writeMarker(constants.FLOAT32);
+        this.writeFloat32Data(value);
+    }
+
+    /** writeFloat32Data */
+    protected writeFloat32Data(value: number): void {
+        this.ensureCapacity(4);
+        this.view.setFloat32(this.length, value);
+        this.length += 4;
+    }
+
+    /** writeFloat64 */
+    protected writeFloat64(value: number): void {
+        this.writeMarker(constants.FLOAT64);
+        this.writeFloat64Data(value);
+    }
+
+    /** writeFloat64Data */
+    protected writeFloat64Data(value: number): void {
+        this.ensureCapacity(8);
+        this.view.setFloat64(this.length, value);
+        this.length += 8;
+    }
+
+    /** writeChar */
+    protected writeChar(value: string): void {
+        this.writeMarker(constants.CHAR);
+        this.writeCharData(value);
+    }
+
+    /** writeCharData */
+    protected writeCharData(value: string): void {
+        this.ensureCapacity(1);
+        this.view.setUint8(this.length, value.charCodeAt(0));
+        this.length += 1;
+    }
+
+    /** writeString */
+    protected writeString(value: string): void {
+        this.writeMarker(constants.STRING);
+        this.writeStringData(value);
+    }
+
+    /** writeStringData */
+    protected writeStringData(value: string): void {
+        // 优化小字符串
+        if (value.length < 32) {
+            const buf = this.stringEncoder.encode(value);
+            this.writeInt(buf.length);
+            this.ensureCapacity(buf.length);
+            this.buffer.set(buf, this.length);
+            this.length += buf.length;
+            return;
+        }
+        if (this.stringEncoder.encodeInto != null) {
+            const maxUsage = value.length * 3;
+            const currentPos = this.length;
+            // 一次性分配 writeInt 和 encodeInto 的空间，避免无法回溯
+            this.ensureCapacity(maxUsage + 4);
+            // 先写入最大大小
+            const lengthWriter = this.writeInt(maxUsage);
+            // 写入文本数据
+            const { written } = this.stringEncoder.encodeInto(value, this.buffer.subarray(this.length));
+            // 回溯，写入实际大小
+            this.length = currentPos;
+            lengthWriter.call(this, written as number);
+            // 移动指针到写入末尾
+            this.length += written as number;
+            return;
+        }
+        const data = this.stringEncoder.encode(value);
+        this.writeInt(data.length);
+        this.ensureCapacity(data.length);
+        this.buffer.set(data, this.length);
+        this.length += data.length;
+    }
+
+    /**
+     * 写入整形数字，选取合适的大小
+     *
+     * @throws 无法在 int32 范围内表示
+     */
+    protected writeInt(value: number): (this: this, value: number) => void {
+        value = value | 0;
+        if (value >= -128 && value <= 127) {
+            this.writeInt8(value);
+            return this.writeInt8;
+        }
+        if (value >= -32768 && value <= 32767) {
+            this.writeInt16(value);
+            return this.writeInt16;
+        }
+        if (value >= -2147483648 && value <= 2147483647) {
+            this.writeInt32(value);
+            return this.writeInt32;
+        }
+        /* c8 ignore next 2 */
+        throw new Error(`Unsupported int value ${value}: out of range`);
+    }
+
+    /** 写入数字，选取合适的大小 */
+    protected writeNumber(value: number): void {
+        if (Number.isInteger(value) && value >= -2147483648 && value <= 2147483647) {
+            if (value >= 0 && value <= 255) return this.writeUint8(value);
+            this.writeInt(value);
+            return;
+        }
+        if (!(Number.isNaN(value) || Math.fround(value) === value)) {
+            return this.writeFloat64(value);
+        }
+        // 如果不会损失精度，则使用 32 位浮点
+        return this.writeFloat32(value);
+    }
+
+    /** writeObject */
+    protected writeObject(value: Record<string, unknown>): void {
+        this.writeMarker(constants.OBJECT);
+        // 生成稳定的结果以便 hash 计算
+        for (const key of Object.keys(value).sort()) {
+            const element = value[key];
+            if (element === undefined) continue;
+            this.writeStringData(key);
+            this.write(element);
+        }
+        this.writeMarker(constants.OBJECT_END);
+    }
+
+    /** writeArray */
+    protected writeArray(value: unknown[]): void {
+        this.writeMarker(constants.ARRAY);
+        for (const v of value) {
+            // 在数组中 undefined 也被视作 null 进行序列化
+            if (v == null) this.writeNull();
+            else this.write(v);
+        }
+        this.writeMarker(constants.ARRAY_END);
+    }
+
+    /** writeArray */
+    protected writeTypedArray(value: ArrayBufferView): void {
+        this.writeMarker(constants.ARRAY);
+        this.writeMarker(constants.TYPE_MARKER);
+        if (value instanceof Uint8Array || value instanceof Int8Array) {
+            // fast path for typed arrays with `BYTES_PER_ELEMENT` of 1
+            this.writeMarker(value instanceof Uint8Array ? constants.UINT8 : constants.INT8);
+            this.writeMarker(constants.COUNT_MARKER);
+            this.writeInt(value.length);
+            this.ensureCapacity(value.byteLength);
+            this.buffer.set(value, this.length);
+            this.length += value.byteLength;
+            return;
+        }
+        /** 用于写入的 setter */
+        let setValue;
+        if (value instanceof Int16Array) {
+            this.writeMarker(constants.INT16);
+            setValue = this.view.setInt16;
+        } else if (value instanceof Int32Array) {
+            this.writeMarker(constants.INT32);
+            setValue = this.view.setInt32;
+        } else if (value instanceof Float32Array) {
+            this.writeMarker(constants.FLOAT32);
+            setValue = this.view.setFloat32;
+        } else if (value instanceof Float64Array) {
+            this.writeMarker(constants.FLOAT64);
+            setValue = this.view.setFloat64;
+        } else {
+            throw new Error(`Unsupported typed array type ${Object.prototype.toString.call(value)}`);
+        }
+        this.writeMarker(constants.COUNT_MARKER);
+        this.writeInt(value.length);
+        this.ensureCapacity(value.byteLength);
+        // 不要在前面 bind，this.ensureCapacity 有可能导致 this.view 指向新的对象
+        for (const v of value) {
+            setValue.call(this.view, this.length, v);
+            this.length += value.BYTES_PER_ELEMENT;
+        }
+    }
+}

package/src/common/string-decoder.ts

@@ -0,0 +1,107 @@
+export const textDecoder =
+    typeof TextDecoder == 'function' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: false }) : null;
+
+export const TEXT_ENCODER_THRESHOLD = textDecoder == null ? 0xffff_ffff : 200;
+
+const CHUNK_SIZE = 0x1_000;
+/** 解码 */
+export function decodeJs(bytes: Uint8Array, begin: number, end: number): string {
+    let offset = begin;
+
+    const units: number[] = [];
+    let result = '';
+    while (offset < end) {
+        const byte1 = bytes[offset++];
+        if ((byte1 & 0x80) === 0) {
+            // 1 byte
+            units.push(byte1);
+        } else if ((byte1 & 0xe0) === 0xc0) {
+            // 2 bytes
+            const byte2 = bytes[offset++] & 0x3f;
+            units.push(((byte1 & 0x1f) << 6) | byte2);
+        } else if ((byte1 & 0xf0) === 0xe0) {
+            // 3 bytes
+            const byte2 = bytes[offset++] & 0x3f;
+            const byte3 = bytes[offset++] & 0x3f;
+            units.push(((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3);
+        } else if ((byte1 & 0xf8) === 0xf0) {
+            // 4 bytes
+            const byte2 = bytes[offset++] & 0x3f;
+            const byte3 = bytes[offset++] & 0x3f;
+            const byte4 = bytes[offset++] & 0x3f;
+            let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4;
+            if (unit > 0xffff) {
+                unit -= 0x10000;
+                units.push(((unit >>> 10) & 0x3ff) | 0xd800);
+                unit = 0xdc00 | (unit & 0x3ff);
+            }
+            units.push(unit);
+        } else {
+            units.push(byte1);
+        }
+
+        if (units.length >= CHUNK_SIZE) {
+            result += String.fromCharCode(...units);
+            units.length = 0;
+        }
+    }
+
+    if (units.length > 0) {
+        result += String.fromCharCode(...units);
+    }
+
+    return result;
+}
+
+/** 解码 */
+export function decode(data: Uint8Array, begin: number, end: number): string {
+    if (
+        end - begin <
+        TEXT_ENCODER_THRESHOLD // 只有小字符串有优化价值，见 benchmark-string.js
+    ) {
+        // 为小字符串优化
+        return decodeJs(data, begin, end);
+    }
+    // 使用系统解码
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return textDecoder!.decode(data.subarray(begin, end));
+}
+
+/** 特别优化字符串解码速度 */
+export class StringDecoder {
+    /** 小字符串缓存 */
+    private readonly cache = [
+        undefined, // 0 字节字符串，直接返回
+        undefined, // 1 字节字符串，使用 String.fromCharCode，兼容 UTF8
+        new Map<number, string>(), // 2 字节
+        new Map<number, string>(), // 3 字节
+        new Map<number, string>(), // 4 字节
+        new Map<number, string>(), // 5 字节
+        new Map<number, string>(), // 6 字节
+    ] as const;
+
+    /** 字符串解码 */
+    decode(data: Uint8Array, begin: number, end: number): string {
+        const length = (end - begin) as 0 | 1 | 2 | 3 | 4 | 5 | 6;
+        // 这里，length 类型为 Uint32
+        if (length > 6) return decode(data, begin, end);
+
+        // 这里，length 类型为 0 | 1 | 2 | 3 | 4 | 5 | 6
+        // 为小字符串优化
+        if (length === 0) return '';
+        if (length === 1) return String.fromCharCode(data[begin]);
+        // number 最多存储 6 字节数据
+        const cache = this.cache[length];
+        // 计算缓存 key
+        let key = 0;
+        for (let i = begin; i < end; i++) {
+            // 不要用位运算，JS 位运算只支持 32 位
+            key = key * 0xff + data[i];
+        }
+        const match = cache.get(key);
+        if (match != null) return match;
+        const string = decodeJs(data, begin, end);
+        cache.set(key, string);
+        return string;
+    }
+}