@cloudpss/ubjson 0.4.32 → 0.4.34

package/src/common/string-decoder.ts

@@ -1,3 +1,5 @@
+import { UnexpectedEof } from '../utils.js';
+
 /* c8 ignore next 2: TextDecoder always present, fallback tested */
 export const textDecoder =
     typeof TextDecoder == 'function' ? new TextDecoder('utf8', { ignoreBOM: true, fatal: false }) : null;
@@ -8,6 +10,8 @@ export const TEXT_ENCODER_THRESHOLD = textDecoder == null ? 0xffff_ffff : 200;
 const CHUNK_SIZE = 0x1000;
 const REPLACE_CHAR = 0xfffd;
 
+const fromCharCode = String.fromCharCode;
+
 /** 解码 */
 export function decodeJs(bytes: Uint8Array, begin: number, end: number): string {
     let offset = begin;
@@ -45,67 +49,195 @@ export function decodeJs(bytes: Uint8Array, begin: number, end: number): string
         }
 
         if (units.length >= CHUNK_SIZE) {
-            result += String.fromCharCode(...units);
+            result += fromCharCode(...units);
             units.length = 0;
         }
     }
 
     if (units.length > 0) {
-        result += String.fromCharCode(...units);
+        result += fromCharCode(...units);
     }
 
     return result;
 }
 
-/** 解码 */
+/** 解码 Ascii */
+function longStringInJS(buf: Uint8Array, begin: number, length: number): string | undefined {
+    const bytes = Array.from<number>({ length });
+    for (let i = 0; i < length; i++) {
+        const byte = buf[begin++];
+        if ((byte & 0x80) > 0) {
+            return;
+        }
+        bytes[i] = byte;
+    }
+    return fromCharCode(...bytes);
+}
+/** 解码 Ascii */
+function shortStringInJS(buf: Uint8Array, begin: number, length: number): string | undefined {
+    if (length < 4) {
+        if (length < 2) {
+            if (length === 0) return '';
+            else {
+                const a = buf[begin++];
+                if ((a & 0x80) > 1) {
+                    return;
+                }
+                return fromCharCode(a);
+            }
+        } else {
+            const a = buf[begin++];
+            const b = buf[begin++];
+            if ((a & 0x80) > 0 || (b & 0x80) > 0) {
+                return;
+            }
+            if (length < 3) return fromCharCode(a, b);
+            const c = buf[begin++];
+            if ((c & 0x80) > 0) {
+                return;
+            }
+            return fromCharCode(a, b, c);
+        }
+    } else {
+        const a = buf[begin++];
+        const b = buf[begin++];
+        const c = buf[begin++];
+        const d = buf[begin++];
+        if ((a & 0x80) > 0 || (b & 0x80) > 0 || (c & 0x80) > 0 || (d & 0x80) > 0) {
+            return;
+        }
+        if (length < 6) {
+            if (length === 4) return fromCharCode(a, b, c, d);
+            else {
+                const e = buf[begin++];
+                if ((e & 0x80) > 0) {
+                    return;
+                }
+                return fromCharCode(a, b, c, d, e);
+            }
+        } else if (length < 8) {
+            const e = buf[begin++];
+            const f = buf[begin++];
+            if ((e & 0x80) > 0 || (f & 0x80) > 0) {
+                return;
+            }
+            if (length < 7) return fromCharCode(a, b, c, d, e, f);
+            const g = buf[begin++];
+            if ((g & 0x80) > 0) {
+                return;
+            }
+            return fromCharCode(a, b, c, d, e, f, g);
+        } else {
+            const e = buf[begin++];
+            const f = buf[begin++];
+            const g = buf[begin++];
+            const h = buf[begin++];
+            if ((e & 0x80) > 0 || (f & 0x80) > 0 || (g & 0x80) > 0 || (h & 0x80) > 0) {
+                return;
+            }
+            if (length < 10) {
+                if (length === 8) return fromCharCode(a, b, c, d, e, f, g, h);
+                else {
+                    const i = buf[begin++];
+                    if ((i & 0x80) > 0) {
+                        return;
+                    }
+                    return fromCharCode(a, b, c, d, e, f, g, h, i);
+                }
+            } else if (length < 12) {
+                const i = buf[begin++];
+                const j = buf[begin++];
+                if ((i & 0x80) > 0 || (j & 0x80) > 0) {
+                    return;
+                }
+                if (length < 11) return fromCharCode(a, b, c, d, e, f, g, h, i, j);
+                const k = buf[begin++];
+                if ((k & 0x80) > 0) {
+                    return;
+                }
+                return fromCharCode(a, b, c, d, e, f, g, h, i, j, k);
+            } else {
+                const i = buf[begin++];
+                const j = buf[begin++];
+                const k = buf[begin++];
+                const l = buf[begin++];
+                if ((i & 0x80) > 0 || (j & 0x80) > 0 || (k & 0x80) > 0 || (l & 0x80) > 0) {
+                    return;
+                }
+                if (length < 14) {
+                    if (length === 12) return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l);
+                    else {
+                        const m = buf[begin++];
+                        if ((m & 0x80) > 0) {
+                            begin -= 13;
+                            return;
+                        }
+                        return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l, m);
+                    }
+                } else {
+                    const m = buf[begin++];
+                    const n = buf[begin++];
+                    if ((m & 0x80) > 0 || (n & 0x80) > 0) {
+                        return;
+                    }
+                    if (length < 15) return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l, m, n);
+                    const o = buf[begin++];
+                    if ((o & 0x80) > 0) {
+                        return;
+                    }
+                    return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o);
+                }
+            }
+        }
+    }
+}
+
+/** 字符串解码，无缓存 */
 export function decode(data: Uint8Array, begin: number, end: number): string {
-    if (
-        end - begin <
-        TEXT_ENCODER_THRESHOLD // 只有小字符串有优化价值，见 benchmark-string.js
-    ) {
+    if (end > data.length) throw new UnexpectedEof();
+    const length = end - begin;
+    if (length < 16) {
+        const result = shortStringInJS(data, begin, length);
+        if (result != null) return result;
+    }
+    // 只有小字符串有优化价值，见 benchmark-string.js
+    if (length < TEXT_ENCODER_THRESHOLD) {
         // 为小字符串优化
         return decodeJs(data, begin, end);
     }
     // 使用系统解码
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
     return textDecoder!.decode(data.subarray(begin, end));
 }
 
-/** 特别优化字符串解码速度 */
-export class StringDecoder {
-    /** 小字符串缓存 */
-    private readonly cache = [
-        undefined, // 0 字节字符串，直接返回
-        undefined, // 1 字节字符串，使用 String.fromCharCode，兼容 UTF8
-        new Map<number, string>(), // 2 字节
-        new Map<number, string>(), // 3 字节
-        new Map<number, string>(), // 4 字节
-        new Map<number, string>(), // 5 字节
-        new Map<number, string>(), // 6 字节
-    ] as const;
+const keyCache = Array.from<{ value: string; buffer: Uint8Array } | undefined>({ length: 4096 });
 
-    /** 字符串解码 */
-    decode(data: Uint8Array, begin: number, end: number): string {
-        const length = (end - begin) as 0 | 1 | 2 | 3 | 4 | 5 | 6;
-        // 这里，length 类型为 Uint32
-        if (length > 6) return decode(data, begin, end);
+/** 字符串解码，使用缓存 */
+export function decodeKey(data: Uint8Array, begin: number, end: number): string {
+    const length = end - begin;
+    const cacheKey =
+        ((length << 5) ^ (length > 1 ? data[begin] & (data[begin + 1] << 8) : length > 0 ? data[begin] : 0)) & 0xfff;
+    let entry = keyCache[cacheKey];
+    if (entry != null && entry.buffer.byteLength === length) {
+        let i = 0;
+        for (; i < length; i++) {
+            if (entry.buffer[i] !== data[begin + i]) break;
+        }
+        if (i === length) return entry.value;
+    }
 
-        // 这里，length 类型为 0 | 1 | 2 | 3 | 4 | 5 | 6
-        // 为小字符串优化
-        if (length === 0) return '';
-        if (length === 1) return String.fromCharCode(data[begin]);
-        // number 最多存储 6 字节数据
-        const cache = this.cache[length];
-        // 计算缓存 key
-        let key = 0;
-        for (let i = begin; i < end; i++) {
-            // 不要用位运算，JS 位运算只支持 32 位
-            key = key * 0xff + data[i];
+    if (end > data.length) throw new UnexpectedEof();
+    let str = length < 16 ? shortStringInJS(data, begin, length) : longStringInJS(data, begin, length);
+    if (str == null) {
+        // 只有小字符串有优化价值，见 benchmark-string.js
+        if (length < TEXT_ENCODER_THRESHOLD) {
+            // 为小字符串优化
+            str = decodeJs(data, begin, end);
+        } else {
+            // 使用系统解码
+            str = textDecoder!.decode(data.subarray(begin, end));
         }
-        const match = cache.get(key);
-        if (match != null) return match;
-        const string = decodeJs(data, begin, end);
-        cache.set(key, string);
-        return string;
     }
+    entry = { value: str, buffer: data.slice(begin, end) };
+    keyCache[cacheKey] = entry;
+    return str;
 }

package/src/common/string-encoder.ts

@@ -53,45 +53,14 @@ function getCharCodeByteLength(string: string): number {
     return byteLength;
 }
 
-/** 字符串编码器 */
-export class StringEncoder extends TextEncoder {
-    private readonly cache = new Map<string, Uint8Array>();
+export const getStringByteLength = (): ((v: string) => number) =>
+    typeof Buffer == 'function' && typeof Buffer.byteLength == 'function'
+        ? (v) => Buffer.byteLength(v, 'utf8')
+        : getCharCodeByteLength;
 
-    /** 编码字符串 */
-    override encode(value: string): Uint8Array {
-        if (value.length > 16) {
-            return super.encode(value);
-        }
-        // 优化小字符串
-        let buf = this.cache.get(value);
-        if (buf == null) {
-            let isAscii = true;
-            for (let index = 0; index < value.length; index++) {
-                const ch = value.charCodeAt(index);
-                if (ch >= 128) {
-                    isAscii = false;
-                    break;
-                }
-            }
-            if (isAscii) {
-                buf = new Uint8Array(value.length);
-                for (let index = 0; index < value.length; index++) {
-                    const ch = value.charCodeAt(index);
-                    buf[index] = ch;
-                }
-            } else {
-                buf = super.encode(value);
-            }
-            this.cache.set(value, buf);
-        }
-        return buf;
-    }
-
-    /** 计算使用的空间 */
-    byteLength(value: string): number {
-        if (typeof Buffer == 'function' && typeof Buffer.byteLength == 'function') {
-            return Buffer.byteLength(value, 'utf8');
-        }
-        return getCharCodeByteLength(value);
-    }
-}
+/* c8 ignore next 1 */
+const encoder = typeof TextEncoder == 'function' ? new TextEncoder() : undefined;
+export const getEncodeInto = (): ((v: string, buf: Uint8Array, offset: number) => number) | undefined =>
+    typeof encoder?.encodeInto == 'function'
+        ? (v, buf, offset) => encoder.encodeInto(v, buf.subarray(offset)).written
+        : undefined;

package/src/encoder.ts

@@ -34,22 +34,8 @@ export class Encoder extends EncoderBase {
 
     /** 获取写入结果 */
     encode(): Uint8Array {
-        if (ArrayBuffer.isView(this.value)) {
-            // 为根 typed array 优化，减少不必要的分配
-            if (this.value.byteLength < 128) this.ensureCapacity(6 + this.value.byteLength);
-            else if (this.value.byteLength < 32768) this.ensureCapacity(7 + this.value.byteLength);
-            else if (this.value.byteLength < 2_147_483_648) this.ensureCapacity(9 + this.value.byteLength);
-            else this.ensureCapacity(13 + this.value.byteLength);
-            this.writeTypedArray(this.value);
-            // 取消注释后运行测试
-            // if (this.buffer.byteLength !== this.length) {
-            //     // 这里永远无法到达
-            //     throw new Error('Bad buffer size');
-            // }
-        } else {
-            this.ensureCapacity(BLOCK_SIZE);
-            this.write(this.value);
-        }
+        this.ensureCapacity(BLOCK_SIZE);
+        this.writeValue();
         if (this.flushedBuffers.length === 0) {
             if (this.buffer.byteLength === this.length) {
                 // 无需再拷贝一次

package/src/index.ts

@@ -1,6 +1,6 @@
 import { Encoder } from './encoder.js';
 import { Decoder } from './decoder.js';
-import { toUint8Array } from './utils.js';
+export { UnexpectedEof } from './utils.js';
 
 /** 编码为 UBJSON */
 export function encode(value: unknown): Uint8Array {
@@ -10,9 +10,6 @@ export function encode(value: unknown): Uint8Array {
 
 /** 解码 UBJSON */
 export function decode(value: BinaryData): unknown {
-    const decoder = new Decoder(toUint8Array(value));
+    const decoder = new Decoder(value);
     return decoder.decode();
 }
-
-export { StringEncoder } from './common/string-encoder.js';
-export { StringDecoder } from './common/string-decoder.js';

package/src/rxjs/decoder.ts

@@ -15,7 +15,7 @@ export function decode(): OperatorFunction<BinaryData, unknown> {
             let end = 0;
             return observable.subscribe({
                 next(value) {
-                    const data = toUint8Array(value);
+                    const data = toUint8Array(value, false);
                     if (buffer.length >= end + data.length) {
                         buffer.set(data, end);
                         end += data.length;
@@ -30,7 +30,7 @@ export function decode(): OperatorFunction<BinaryData, unknown> {
                     }
                     while (end - begin > 0) {
                         try {
-                            const helper = new StreamDecoderHelper(buffer.subarray(begin, end));
+                            const helper = new StreamDecoderHelper(buffer.slice(begin, end));
                             const result = helper.decode();
                             if (result !== undefined) {
                                 subscriber.next(result);

package/src/rxjs/index.ts

@@ -1,2 +1,3 @@
 export { encode } from './encoder.js';
 export { decode } from './decoder.js';
+export { UnexpectedEof } from '../utils.js';

package/src/stream/index.ts

@@ -2,6 +2,8 @@ import { Readable, Transform } from 'node:stream';
 import { StreamEncoder } from './encoder.js';
 import { StreamDecoder } from './decoder.js';
 
+export { UnexpectedEof } from '../utils.js';
+
 /** 编码为 UBJSON */
 export function encode(value: unknown): Readable {
     if (value == null) {

package/src/stream-helper/decoder.ts

@@ -12,4 +12,4 @@ export class StreamDecoderHelper extends Decoder {
     }
 }
 
-export { UnexpectedEof } from '../common/decoder.js';
+export { UnexpectedEof } from '../utils.js';

package/src/stream-helper/encoder.ts

@@ -48,11 +48,11 @@ export class StreamEncoderHelper extends EncoderBase {
     encode(): void {
         if (typeof this.value != 'object' || this.value == null || ArrayBuffer.isView(this.value)) {
             this.ensureCapacity(20);
-            this.write(this.value);
+            this.writeValue();
             this.ensureCapacity(-1);
         } else {
             this.ensureCapacity(BLOCK_SIZE);
-            this.write(this.value);
+            this.writeValue();
             this.ensureCapacity(-1);
         }
     }

package/src/utils.ts

@@ -1,10 +1,20 @@
 /** 支持的数据转为 Uint8Array */
-export function toUint8Array(data: BinaryData): Uint8Array {
+export function toUint8Array(data: BinaryData, exact: boolean): Uint8Array {
     if (data == null || typeof data != 'object' || typeof data.byteLength != 'number') {
         throw new TypeError('Invalid data');
     }
-    if (ArrayBuffer.isView(data)) {
-        return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+    if (!ArrayBuffer.isView(data)) {
+        return new Uint8Array(data);
+    }
+    if (exact && (data.byteOffset !== 0 || data.buffer.byteLength !== data.byteLength)) {
+        return new Uint8Array(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
+    }
+    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+}
+
+/** 未结束的流 */
+export class UnexpectedEof extends Error {
+    constructor() {
+        super('Unexpected EOF');
     }
-    return new Uint8Array(data, 0, data.byteLength);
 }

package/tests/decode.js

@@ -2,7 +2,7 @@
  * Tests from https://bitbucket.org/shelacek/ubjson
  */
 
-import { decode } from '../dist/index.js';
+import { decode, UnexpectedEof } from '../dist/index.js';
 import { toBuffer } from './.utils.js';
 
 test('decode unsupported type', () => {
@@ -66,14 +66,50 @@ test('decode float64', () => {
     expect(decode(toBuffer('D', 0x40, 0xf8, 0x6a, 0x00, 0x10, 0x00, 0x00, 0x00))).toBe(100_000.003_906_25);
 });
 
+test('decode int8 [unexpected eof]', () => {
+    expect(() => decode(toBuffer('i'))).toThrowError(UnexpectedEof);
+});
+
+test('decode uint8 [unexpected eof]', () => {
+    expect(() => decode(toBuffer('U'))).toThrowError(UnexpectedEof);
+});
+
+test('decode int16 [unexpected eof]', () => {
+    expect(() => decode(toBuffer('I', 0x12))).toThrowError(UnexpectedEof);
+});
+
+test('decode int32 [unexpected eof]', () => {
+    expect(() => decode(toBuffer('l', 0x12, 0x34, 0x56))).toThrowError(UnexpectedEof);
+});
+
+test('decode int64 [unexpected eof]', () => {
+    expect(() => decode(toBuffer('L', 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde))).toThrowError(UnexpectedEof);
+});
+
+test('decode float32 [unexpected eof]', () => {
+    expect(() => decode(toBuffer('d', 0x3f, 0x80, 0x80))).toThrowError(UnexpectedEof);
+});
+
+test('decode float64 [unexpected eof]', () => {
+    expect(() => decode(toBuffer('D', 0x40, 0xf8, 0x6a, 0x00, 0x10, 0x00, 0x00))).toThrowError(UnexpectedEof);
+});
+
 test('decode high-precision number [error]', () => {
     expect(() => decode(toBuffer('H', 'i', 3, '1', '.', '1'))).toThrow();
 });
 
+test('decode high-precision number [unexpected eof]', () => {
+    expect(() => decode(toBuffer('H', 'i', 3, '1', '.'))).toThrowError(UnexpectedEof);
+});
+
 test('decode char', () => {
     expect(decode(toBuffer('C', 'a'))).toBe('a');
 });
 
+test('decode char [unexpected eof]', () => {
+    expect(() => decode(toBuffer('C'))).toThrowError(UnexpectedEof);
+});
+
 test('decode string', () => {
     expect(decode(toBuffer('S', 'i', 6, 'u', 'b', 'j', 's', 'o', 'n'))).toBe('ubjson');
 });
@@ -90,8 +126,16 @@ test('decode string (bad size) [error]', () => {
     expect(() => decode(toBuffer('S', 'i', 0xff, 'x'))).toThrow(/Invalid length/);
 });
 
-test('decode string (unexpected eof) [error]', () => {
-    expect(() => decode(toBuffer('S', 'i', 2, 'x'))).toThrow(/Unexpected EOF/);
+test('decode short string (unexpected eof) [error]', () => {
+    expect(() => decode(toBuffer('S', 'i', 2, 'x'))).toThrow(UnexpectedEof);
+});
+
+test('decode long string (unexpected eof) [error]', () => {
+    expect(() => decode(toBuffer('S', 'i', 20, 'x'))).toThrow(UnexpectedEof);
+});
+
+test('decode huge string (unexpected eof) [error]', () => {
+    expect(() => decode(toBuffer('S', 'U', 200, 'x'))).toThrow(UnexpectedEof);
 });
 
 test('decode ascii string', () => {
@@ -276,8 +320,24 @@ test('decode array of objects of arrays (optimized)', () => {
     ]);
 });
 
-test('decode array (strongly typed, unexpected eof, optimized)', () => {
-    expect(() => decode(toBuffer('[', '$', 'i', '#', 'i', 3, 1, 2))).toThrow();
+test('decode array (strongly typed i8, unexpected eof, optimized)', () => {
+    expect(() => decode(toBuffer('[', '$', 'i', '#', 'i', 3, 1, 2))).toThrow(UnexpectedEof);
+});
+
+test('decode array (strongly typed u8, unexpected eof, optimized)', () => {
+    expect(() => decode(toBuffer('[', '$', 'U', '#', 'i', 3, 1, 2))).toThrow(UnexpectedEof);
+});
+
+test('decode array (strongly typed i16, unexpected eof, optimized)', () => {
+    expect(() => decode(toBuffer('[', '$', 'I', '#', 'i', 3, 1, 2))).toThrow(UnexpectedEof);
+});
+
+test('decode array (strongly typed i32, unexpected eof, optimized)', () => {
+    expect(() => decode(toBuffer('[', '$', 'l', '#', 'i', 3, 1, 2))).toThrow(UnexpectedEof);
+});
+
+test('decode array (strongly typed i64, unexpected eof, optimized)', () => {
+    expect(() => decode(toBuffer('[', '$', 'L', '#', 'i', 3, 1, 2))).toThrow(UnexpectedEof);
 });
 
 test('decode array (strongly typed, invalid length value, optimized)', () => {
@@ -334,6 +394,10 @@ test('decode array (int64, strongly typed, optimized) [use typed array]', () =>
     ).toThrow();
 });
 
+test('decode array (int64, strongly typed, optimized, empty) [use typed array]', () => {
+    expect(() => decode(toBuffer('[', '$', 'L', '#', 'i', 0))).toThrow();
+});
+
 test('decode array (float32, strongly typed, optimized) [use typed array]', () => {
     const actual = decode(toBuffer('[', '$', 'd', '#', 'i', 2, 0x3e, 0x80, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00));
     expect(actual).toBeInstanceOf(Float32Array);

package/tests/e2e.js

@@ -299,7 +299,6 @@ test('encode/decode model', () => {
 test('encode/decode complex object (no encodeInto)', () => {
     // eslint-disable-next-line @typescript-eslint/unbound-method
     const encodeInto = TextEncoder.prototype.encodeInto;
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-expect-error 移除 encodeInto 以测试兼容性
     TextEncoder.prototype.encodeInto = undefined;
     const expected = {
@@ -399,3 +398,15 @@ test('decode Int8Array', () => {
     const obj = { a: 1, b: undefined, c: { d: 2, e: undefined, f: null } };
     expect(decode(new Int8Array(encode(obj).buffer))).toEqual(obj);
 });
+
+test('decode invalid __proto__', () => {
+    const obj = decode(encode(JSON.parse('{"__proto__":"xxx"}')));
+    expect(obj).toEqual(JSON.parse('{"__proto__":"xxx"}'));
+    expect(Object.getPrototypeOf(obj)).toBe(Object.prototype);
+});
+
+test('decode null __proto__', () => {
+    const obj = decode(encode(JSON.parse('{"__proto__":null}')));
+    expect(obj).toEqual(JSON.parse('{"__proto__":null}'));
+    expect(Object.getPrototypeOf(obj)).toBe(Object.prototype);
+});