@cloudpss/ubjson 0.5.34 → 0.5.36

package/src/common/string-encoder.ts

@@ -1,9 +1,9 @@
 // https://github.com/ehmicky/string-byte-length/blob/main/src/char_code.js
 
 // Last ASCII character (1 byte)
-const LAST_ASCII_CODEPOINT = 0x7f;
+const LAST_ASCII_CODEPOINT = 0x80;
 // Last 2-bytes character
-const LAST_TWO_BYTES_CODEPOINT = 0x7ff;
+const LAST_TWO_BYTES_CODEPOINT = 0x800;
 // Others are 3 bytes characters
 // However, U+d800 to U+dbff:
 //  - Followed by U+dc00 to U+dfff -> 4 bytes together (astral character)
@@ -13,18 +13,18 @@ const LAST_HIGH_SURROGATE = 0xdbff;
 const FIRST_LOW_SURROGATE = 0xdc00;
 const LAST_LOW_SURROGATE = 0xdfff;
 /** 计算使用的空间 */
-function getCharCodeByteLength(string: string): number {
+export function jsStringByteLength(string: string): number {
     const charLength = string.length;
     let byteLength = charLength;
 
     for (let charIndex = 0; charIndex < charLength; charIndex += 1) {
         const codepoint = string.charCodeAt(charIndex);
 
-        if (codepoint <= LAST_ASCII_CODEPOINT) {
+        if (codepoint < LAST_ASCII_CODEPOINT) {
             continue;
         }
 
-        if (codepoint <= LAST_TWO_BYTES_CODEPOINT) {
+        if (codepoint < LAST_TWO_BYTES_CODEPOINT) {
             byteLength += 1;
             continue;
         }
@@ -52,15 +52,104 @@ function getCharCodeByteLength(string: string): number {
 
     return byteLength;
 }
+/** 计算使用的空间 */
+export function nodeStringByteLength(string: string): number {
+    return Buffer.byteLength(string, 'utf8');
+}
+
+/** 计算使用的空间 */
+export let stringByteLength: (v: string) => number;
 
-export const getStringByteLength = (): ((v: string) => number) =>
-    typeof Buffer == 'function' && typeof Buffer.byteLength == 'function'
-        ? (v) => Buffer.byteLength(v, 'utf8')
-        : getCharCodeByteLength;
+/**
+ * Encode string into utf-8.
+ * Provided `buf` MUST have enough space.
+ */
+export function jsEncodeInto(v: string, buf: Uint8Array, offset: number): number {
+    let ptr = offset;
+    const strLen = v.length;
+    for (let i = 0; i < strLen; i++) {
+        const c1 = v.charCodeAt(i);
+        if (c1 < LAST_ASCII_CODEPOINT) {
+            buf[ptr++] = c1;
+        } else if (c1 < LAST_TWO_BYTES_CODEPOINT) {
+            buf[ptr++] = 0xc0 | (c1 >> 6);
+            buf[ptr++] = 0x80 | (c1 & 0x3f);
+        } else if (c1 < FIRST_HIGH_SURROGATE || c1 > LAST_LOW_SURROGATE) {
+            buf[ptr++] = 0xe0 | (c1 >> 12);
+            buf[ptr++] = 0x80 | ((c1 >> 6) & 0x3f);
+            buf[ptr++] = 0x80 | (c1 & 0x3f);
+        } else if (c1 > LAST_HIGH_SURROGATE) {
+            // low surrogate without high surrogate
+            buf[ptr++] = 0xef;
+            buf[ptr++] = 0xbf;
+            buf[ptr++] = 0xbd;
+        } else {
+            const c2 = v.charCodeAt(++i);
+            if (i >= strLen || c2 < FIRST_LOW_SURROGATE || c2 > LAST_LOW_SURROGATE) {
+                // high surrogate not followed by low surrogate
+                buf[ptr++] = 0xef;
+                buf[ptr++] = 0xbf;
+                buf[ptr++] = 0xbd;
+                i--;
+                continue;
+            }
+            const c = ((c1 & 0x3ff) << 10) + (c2 & 0x3ff) + 0x1_0000;
+            buf[ptr++] = 0xf0 | (c >> 18);
+            buf[ptr++] = 0x80 | ((c >> 12) & 0x3f);
+            buf[ptr++] = 0x80 | ((c >> 6) & 0x3f);
+            buf[ptr++] = 0x80 | (c & 0x3f);
+        }
+    }
+    return ptr - offset;
+}
 
 /* c8 ignore next 1 */
-const encoder = typeof TextEncoder == 'function' ? new TextEncoder() : undefined;
-export const getEncodeInto = (): ((v: string, buf: Uint8Array, offset: number) => number) | undefined =>
-    typeof encoder?.encodeInto == 'function'
-        ? (v, buf, offset) => encoder.encodeInto(v, buf.subarray(offset)).written
-        : undefined;
+let TEXT_ENCODER: TextEncoder | null;
+/**
+ * Encode string into utf-8.
+ * Provided `buf` MUST have enough space.
+ */
+export function nativeEncodeInto(v: string, buf: Uint8Array, offset: number): number {
+    const encoded = TEXT_ENCODER!.encodeInto(v, buf.subarray(offset));
+    return encoded.written;
+}
+/**
+ * Encode string into utf-8.
+ * Provided `buf` MUST have enough space.
+ */
+export function myEncodeInto(v: string, buf: Uint8Array, offset: number): number {
+    if (v.length < 55) {
+        return jsEncodeInto(v, buf, offset);
+    }
+    return nativeEncodeInto(v, buf, offset);
+}
+/**
+ * Encode string into utf-8.
+ * Provided `buf` MUST have enough space.
+ */
+export let encodeInto: (v: string, buf: Uint8Array, offset: number) => number;
+/**
+ * Encode string into utf-8.
+ */
+export let encode: (v: string) => Uint8Array;
+
+/** 重设环境 */
+export function resetEnv(): void {
+    TEXT_ENCODER = typeof TextEncoder == 'function' ? new TextEncoder() : null;
+    stringByteLength =
+        typeof Buffer == 'function' && typeof Buffer.byteLength == 'function'
+            ? nodeStringByteLength
+            : jsStringByteLength;
+    encodeInto = typeof TEXT_ENCODER?.encodeInto == 'function' ? myEncodeInto : jsEncodeInto;
+    encode =
+        typeof Buffer == 'function' && Buffer.from
+            ? (v) => Buffer.from(v, 'utf8')
+            : TEXT_ENCODER
+              ? (v) => TEXT_ENCODER!.encode(v)
+              : (v) => {
+                    const buf = new Uint8Array(stringByteLength(v));
+                    jsEncodeInto(v, buf, 0);
+                    return buf;
+                };
+}
+resetEnv();

package/src/stream-helper/encoder.ts

@@ -1,33 +1,49 @@
+import { constants } from '../common/constants.js';
 import { EncoderBase } from '../common/encoder.js';
+import { unsupportedView } from '../common/errors.js';
+import { encode, stringByteLength } from '../common/string-encoder.js';
 
-const BLOCK_SIZE = 1024 * 8; // 8 KiB
-const MAX_SIZE = 1024 * 1024 * 256; // 256 MiB
+const BLOCK_SIZE = 1024 * 64; // 64 KiB
+const MAX_SIZE = 1024 * 1024 * 32; // 32 MiB
 
 /** 保存一个内存池以减少重复分配 */
 let POOL: Uint8Array | null = null;
 
+/** 获取内存池 */
+function alloc(size: number): Uint8Array {
+    if (POOL == null || size !== BLOCK_SIZE) {
+        return new Uint8Array(size);
+    }
+    const pool = POOL;
+    POOL = null;
+    return pool;
+}
+
+/** 归还内存池 */
+function free(buf: Uint8Array): boolean {
+    if (POOL == null && buf.byteLength === BLOCK_SIZE) {
+        POOL = buf;
+        return true;
+    }
+    return false;
+}
+
 /** 流式编码 UBJSON */
 export class StreamEncoderHelper extends EncoderBase {
     constructor(protected readonly onChunk: (chunk: Uint8Array) => void) {
         super();
-        if (POOL != null) {
-            this.pool = POOL;
-            POOL = null;
-        } else {
-            this.pool = new Uint8Array(BLOCK_SIZE);
-        }
+        this.buffer = alloc(BLOCK_SIZE);
+        this.view = new DataView(this.buffer.buffer);
     }
     /**
      * 销毁实例，释放内存池
      */
     destroy(): void {
-        POOL ??= this.pool;
-        const self = this as unknown as { pool: Uint8Array | null; buffer: Uint8Array | null };
-        self.pool = null;
+        free(this.buffer);
+        const self = this as unknown as { view: DataView | null; buffer: Uint8Array | null };
+        self.view = null;
         self.buffer = null;
     }
-    /** 通过内存池减少分配 */
-    private readonly pool;
     /**
      * 确保 buffer 还有 capacity 的空闲空间
      */
@@ -36,45 +52,108 @@ export class StreamEncoderHelper extends EncoderBase {
             // 超过最大尺寸限制
             throw new Error('Buffer has exceed max size');
         }
-        if (capacity < 0) {
-            // 结束流
-            if (this.buffer === this.pool) {
-                this.onChunk(this.buffer.slice(0, this.length));
-            } else {
-                this.onChunk(this.buffer.subarray(0, this.length));
-            }
-            return;
-        }
         // 无需扩容
-        if (this.buffer.byteLength >= this.length + capacity) return;
+        if (capacity >= 0 && this.buffer.byteLength >= this.length + capacity) return;
 
+        const CURRENT_SIZE = this.buffer.byteLength;
+        const NEXT_SIZE = capacity < BLOCK_SIZE ? BLOCK_SIZE : capacity;
+        const REUSE_BUF =
+            CURRENT_SIZE >= NEXT_SIZE && // 满足容量需求
+            CURRENT_SIZE - BLOCK_SIZE < NEXT_SIZE; // 不过于浪费
         // 提交目前的数据
-        if (this.buffer === this.pool) {
+        if (REUSE_BUF) {
             this.onChunk(this.buffer.slice(0, this.length));
         } else {
-            this.onChunk(this.buffer.subarray(0, this.length));
+            if (free(this.buffer)) {
+                // 归还内存池成功，buffer 可能重用，需要拷贝数据
+                this.onChunk(this.buffer.slice(0, this.length));
+            } else {
+                this.onChunk(this.buffer.subarray(0, this.length));
+            }
+            // 重新分配缓冲区
+            this.buffer = alloc(NEXT_SIZE);
+            this.view = new DataView(this.buffer.buffer);
         }
+        this.length = 0;
+    }
+    /** @inheritdoc */
+    protected override writeLargeStringData(value: string): void {
+        const strLen = value.length;
+        const binLen = stringByteLength(value);
+        this.ensureCapacity(5);
+        this.buffer[this.length++] = constants.INT32;
+        this.view.setInt32(this.length, binLen);
+        this.length += 4;
+        this.ensureCapacity(-1);
 
-        // 重新分配缓冲区
-        if (capacity < BLOCK_SIZE) capacity = BLOCK_SIZE;
-        this.allocUnsafe(capacity);
+        // divide string to 64k chunks
+        for (let i = 0; i < strLen; i += BLOCK_SIZE) {
+            let end = i + BLOCK_SIZE;
+            // avoid split surrogate pair
+            const endAtSurrogate = end < strLen && (value.charCodeAt(end) & 0xfc00) === 0xdc00;
+            if (endAtSurrogate) {
+                end--;
+            }
+            const chunk = value.slice(i, end);
+            this.onChunk(encode(chunk));
+            if (endAtSurrogate) {
+                i--;
+            }
+        }
     }
-    /** 分配 buffer */
-    private allocUnsafe(size: number): void {
-        if (size === this.pool.byteLength) {
-            // 从 pool 中获取
-            this.buffer = this.pool;
-            this.view = new DataView(this.buffer.buffer);
-            this.length = 0;
+    /** @inheritdoc */
+    protected override writeLargeTypedArrayData(value: ArrayBufferView): void {
+        this.ensureCapacity(-1);
+        if (value instanceof Uint8Array || value instanceof Int8Array) {
+            // fast path for typed arrays with `BYTES_PER_ELEMENT` of 1
+            // divide buffer to 64k chunks
+            for (let i = 0; i < value.byteLength; i += BLOCK_SIZE) {
+                this.onChunk(
+                    new Uint8Array(value.buffer.slice(value.byteOffset + i, value.byteOffset + i + BLOCK_SIZE)),
+                );
+            }
             return;
         }
-        this.buffer = new Uint8Array(size);
-        this.view = new DataView(this.buffer.buffer);
-        this.length = 0;
+
+        const arrayLength = (value as Int16Array | Int32Array | BigInt64Array | Float32Array | Float64Array).length;
+        const elementSize = (value as Int16Array | Int32Array | BigInt64Array | Float32Array | Float64Array)
+            .BYTES_PER_ELEMENT;
+        if (value instanceof Int16Array) {
+            for (let i = 0; i < arrayLength; i++) {
+                this.ensureCapacity(elementSize);
+                this.view.setInt16(this.length, value[i]!);
+                this.length += elementSize;
+            }
+        } else if (value instanceof Int32Array) {
+            for (let i = 0; i < arrayLength; i++) {
+                this.ensureCapacity(elementSize);
+                this.view.setInt32(this.length, value[i]!);
+                this.length += elementSize;
+            }
+        } else if (value instanceof Float32Array) {
+            for (let i = 0; i < arrayLength; i++) {
+                this.ensureCapacity(elementSize);
+                this.view.setFloat32(this.length, value[i]!);
+                this.length += elementSize;
+            }
+        } else if (value instanceof Float64Array) {
+            for (let i = 0; i < arrayLength; i++) {
+                this.ensureCapacity(elementSize);
+                this.view.setFloat64(this.length, value[i]!);
+                this.length += elementSize;
+            }
+        } else if (value instanceof BigInt64Array) {
+            for (let i = 0; i < arrayLength; i++) {
+                this.ensureCapacity(elementSize);
+                this.view.setBigInt64(this.length, value[i]!);
+                this.length += elementSize;
+            }
+        } else {
+            unsupportedView(value);
+        }
     }
     /** 获取写入结果 */
     encode(value: unknown): void {
-        this.allocUnsafe(BLOCK_SIZE);
         this.writeValue(value);
         this.ensureCapacity(-1);
     }

package/tests/.utils.js

@@ -1,3 +1,13 @@
+import { resetEnv as resetDecoderEnv } from '../dist/common/string-decoder.js';
+import { resetEnv as resetEncoderEnv } from '../dist/common/string-encoder.js';
+import { resetEncoder } from '../dist/encoder.js';
+
+export function resetEnv() {
+    resetDecoderEnv();
+    resetEncoderEnv();
+    resetEncoder();
+}
+
 /**
  * 输入转为数字数组以便比较
  *