@cloudpss/ubjson 0.5.40 → 0.5.41

package/src/rxjs/decoder.ts

@@ -1,69 +1,95 @@
 import { Observable, type OperatorFunction } from 'rxjs';
-import { StreamDecoderHelper, kEof, UnexpectedEof } from '../stream-helper/decoder.js';
+import { UnexpectedEof } from '../stream-helper/decoder.js';
 import { toUint8Array } from '../helper/utils.js';
+import { read } from '../helper/decode-ae.js';
 
-const DEFAULT_BUFFER_SIZE = 16 * 1024 * 1024; // 16 MiB
 const EMPTY_BUFFER = new Uint8Array(0);
+const EMPTY_VIEW = new DataView(EMPTY_BUFFER.buffer);
 
 /** 流式解码 UBJSON */
 export function decode(): OperatorFunction<BinaryData, unknown> {
     return (observable) => {
         return new Observable<unknown>((subscriber) => {
-            let chunkSize = DEFAULT_BUFFER_SIZE;
-            let buffer = new Uint8Array(chunkSize);
-            let begin = 0;
-            let end = 0;
+            const data = EMPTY_BUFFER;
+            const cursor = {
+                view: EMPTY_VIEW,
+                data,
+                capacity: 0,
+                size: 0,
+                offset: 0,
+            };
+            /** reader 返回的还需接收的字节数 */
+            let required = 1;
+            let reader = read(cursor);
             return observable.subscribe({
                 next(value) {
-                    const data = toUint8Array(value);
-                    if (buffer.length >= end + data.length) {
-                        buffer.set(data, end);
-                        end += data.length;
+                    const chunk = toUint8Array(value);
+                    const chunkSize = chunk.byteLength;
+                    if (cursor.capacity - cursor.size < chunkSize) {
+                        // 当前缓冲区不足，需要扩容
+                        const newSize = Math.max(
+                            // 不缩小缓冲区
+                            cursor.capacity,
+                            // 扩大缓冲区到足够容纳 2 倍当前数据
+                            chunkSize * 2 + cursor.size - cursor.offset,
+                        );
+                        if (newSize > cursor.capacity) {
+                            // 需要增大缓冲区
+                            const newData = new Uint8Array(newSize);
+                            newData.set(cursor.data.subarray(cursor.offset, cursor.size), 0);
+                            newData.set(chunk, cursor.size - cursor.offset);
+                            cursor.data = newData;
+                            cursor.view = new DataView(newData.buffer, newData.byteOffset, newData.byteLength);
+                            cursor.capacity = newSize;
+                        } else {
+                            // 无需增大缓冲区，直接移动数据
+                            cursor.data.copyWithin(0, cursor.offset, cursor.size);
+                            cursor.data.set(chunk, cursor.size - cursor.offset);
+                        }
+                        cursor.size = cursor.size - cursor.offset + chunkSize;
+                        cursor.offset = 0;
                     } else {
-                        chunkSize = Math.max(chunkSize, data.length * 2 + end - begin);
-                        const allocSize = Math.max(chunkSize, data.length + end - begin);
-                        const newBuffer =
-                            allocSize > buffer.byteLength
-                                ? new Uint8Array(Math.max(chunkSize, data.length + end - begin))
-                                : buffer;
-                        newBuffer.set(buffer.subarray(begin, end), 0);
-                        newBuffer.set(data, end - begin);
-                        buffer = newBuffer;
-                        end = end - begin + data.length;
-                        begin = 0;
+                        // 当前缓冲区足够，直接写入
+                        cursor.data.set(chunk, cursor.size);
+                        cursor.size += chunkSize;
                     }
-                    while (end - begin > 0) {
-                        try {
-                            const helper = new StreamDecoderHelper(buffer.slice(begin, end));
-                            const result = helper.decode();
-                            if (result !== undefined) {
-                                subscriber.next(result);
-                            }
-                            begin += helper.readLength;
-                        } catch (ex) {
-                            if (ex === kEof) {
-                                return;
+
+                    required -= chunkSize;
+                    // 未读够数据，继续等待
+                    if (required > 0) return;
+
+                    try {
+                        for (;;) {
+                            const result = reader.next();
+                            if (result.done) {
+                                // 读取完成，新建 reader 读取下一个值
+                                subscriber.next(result.value);
+                                reader = read(cursor);
+                                if (cursor.offset === cursor.size) {
+                                    break;
+                                }
                             } else {
-                                subscriber.error(ex as Error);
-                                buffer = EMPTY_BUFFER;
-                                return;
+                                required = result.value;
+                                break;
                             }
                         }
+                    } catch (ex) {
+                        subscriber.error(ex);
                     }
-                    // 完全消费了 Buffer 内容，重置 Buffer
-                    begin = end = 0;
                 },
                 error(err) {
                     subscriber.error(err);
-                    buffer = EMPTY_BUFFER;
+                    cursor.data = EMPTY_BUFFER;
+                    cursor.view = EMPTY_VIEW;
                 },
                 complete() {
-                    if (end - begin > 0) {
+                    if (cursor.size > cursor.offset) {
                         subscriber.error(new UnexpectedEof());
                     } else {
                         subscriber.complete();
                     }
-                    buffer = EMPTY_BUFFER;
+                    cursor.data = EMPTY_BUFFER;
+                    cursor.view = EMPTY_VIEW;
                 },
             });
         });

package/src/rxjs/encoder.ts

@@ -1,11 +1,12 @@
 import { Observable, type OperatorFunction } from 'rxjs';
 import { StreamEncoderHelper } from '../stream-helper/encoder.js';
+import type { EncodeOptions } from '../options.js';
 
 /** 流式编码 UBJSON */
-export function encode(): OperatorFunction<unknown, Uint8Array> {
+export function encode(options?: EncodeOptions): OperatorFunction<unknown, Uint8Array> {
     return (observable) => {
         return new Observable<Uint8Array>((subscriber) => {
-            const helper = new StreamEncoderHelper((chunk: Uint8Array): void => subscriber.next(chunk));
+            const helper = new StreamEncoderHelper(options, (chunk: Uint8Array): void => subscriber.next(chunk));
             const sub = observable.subscribe({
                 next(value) {
                     try {

package/src/rxjs/index.ts

@@ -1,3 +1,4 @@
 export { encode } from './encoder.js';
 export { decode } from './decoder.js';
 export { UnexpectedEofError as UnexpectedEof } from '../helper/errors.js';
+export type { EncodeOptions } from '../options.js';

package/src/stream/encoder.ts

@@ -1,14 +1,15 @@
 import { Transform, type TransformCallback } from 'node:stream';
 import { StreamEncoderHelper } from '../stream-helper/encoder.js';
+import type { EncodeOptions } from '../options.js';
 
 /** 流式编码 UBJSON */
 export class StreamEncoder extends Transform {
-    constructor() {
+    constructor(options?: EncodeOptions) {
         super({
             readableObjectMode: false,
             writableObjectMode: true,
         });
-        this.helper = new StreamEncoderHelper((binary) => this.push(binary));
+        this.helper = new StreamEncoderHelper(options, (binary) => this.push(binary));
     }
 
     private readonly helper;
@@ -24,7 +25,7 @@ export class StreamEncoder extends Transform {
     }
 
     /** @inheritdoc */
-    override _destroy(error: Error | null, callback: (error?: Error | null | undefined) => void): void {
+    override _destroy(error: Error | null, callback: (error?: Error | null) => void): void {
         this.helper.destroy();
         super._destroy(error, callback);
     }

package/src/stream/index.ts

@@ -1,22 +1,24 @@
 import { Readable, type Transform } from 'node:stream';
 import { StreamEncoder } from './encoder.js';
 import { StreamDecoder } from './decoder.js';
+import type { EncodeOptions } from '../options.js';
 
 export { UnexpectedEofError as UnexpectedEof } from '../helper/errors.js';
+export type { EncodeOptions };
 
 /** 编码为 UBJSON */
-export function encode(value: unknown): Readable {
+export function encode(value: unknown, options?: EncodeOptions): Readable {
     if (value == null) {
         return Readable.from([value === null ? 'Z' : 'N'], { objectMode: false });
     }
-    const encoder = new StreamEncoder();
+    const encoder = new StreamEncoder(options);
     encoder.write(value);
     encoder.end();
     return encoder;
 }
 /** 编码为 UBJSON */
-export function encodeMany(value: AsyncIterable<unknown>): Readable {
-    const encoder = new StreamEncoder();
+export function encodeMany(value: AsyncIterable<unknown>, options?: EncodeOptions): Readable {
+    const encoder = new StreamEncoder(options);
     void (async () => {
         try {
             for await (const v of value) {
@@ -32,8 +34,8 @@ export function encodeMany(value: AsyncIterable<unknown>): Readable {
 }
 
 /** 编码为 UBJSON */
-export function encoder(): Transform {
-    return new StreamEncoder();
+export function encoder(options?: EncodeOptions): Transform {
+    return new StreamEncoder(options);
 }
 
 /** 解码 UBJSON */

package/src/stream-helper/encoder.ts

@@ -2,6 +2,7 @@ import { constants } from '../helper/constants.js';
 import { encode, stringByteLength } from '../helper/string-encoder.js';
 import { EncoderBase } from '../base/encoder.js';
 import type { TypedArrayType } from '../helper/encode.js';
+import type { EncodeOptions } from '../options.js';
 
 const BLOCK_SIZE = 1024 * 64; // 64 KiB
 const MAX_SIZE = 1024 * 1024 * 32; // 32 MiB
@@ -30,8 +31,12 @@ function free(buf: Uint8Array): boolean {
 
 /** 流式编码 UBJSON */
 export class StreamEncoderHelper extends EncoderBase {
-    constructor(protected readonly onChunk: (chunk: Uint8Array) => void) {
+    constructor(
+        options: EncodeOptions | null | undefined,
+        protected readonly onChunk: (chunk: Uint8Array) => void,
+    ) {
         super();
+        this.sortObjectKeys = options?.sortObjectKeys ?? false;
         this.data = alloc(BLOCK_SIZE);
         this.view = new DataView(this.data.buffer);
     }

package/tests/.utils.js

@@ -2,6 +2,7 @@ import { resetEnv as resetDecoderEnv } from '../dist/helper/string-decoder.js';
 import { resetEnv as resetEncoderEnv } from '../dist/helper/string-encoder.js';
 import { resetEncoder } from '../dist/encoder.js';
 import '../dist/helper/constants.js';
+import '../dist/options.js';
 
 /**
  * 重设所有环境

package/tests/e2e/stream.js

@@ -13,7 +13,19 @@ describe('stream', () => {
                 expect(decoded).toEqual(expected);
             }).rejects.toThrow(expected);
         } else {
-            const decoded = await decodeStream(encodeStream(input));
+            const encoded = await encodeStream(input).toArray();
+            const data = encoded.flatMap((/** @type {Buffer} */ chunk) => {
+                // split to random chunks
+                const chunks = [];
+                let offset = 0;
+                while (offset < chunk.length) {
+                    const size = Math.floor(Math.random() * chunk.length);
+                    chunks.push(chunk.subarray(offset, offset + size));
+                    offset += size;
+                }
+                return chunks;
+            });
+            const decoded = await decodeStream(Readable.from(data));
             expect(decoded).toEqual(expected);
         }
     });

package/tests/encode.js

@@ -408,7 +408,7 @@ test('encode object', () => {
 });
 
 test('encode object (keep order)', () => {
-    expect(toArray(encode({ b: 2, a: 1, c: 3 }))).toEqual(
+    expect(toArray(encode({ b: 2, a: 1, c: 3 }, { sortObjectKeys: true }))).toEqual(
         toArray('{', 'i', 1, 'a', 'U', 1, 'i', 1, 'b', 'U', 2, 'i', 1, 'c', 'U', 3, '}'),
     );
     // @ts-expect-error Access private property