@cloudpss/ubjson 0.5.9 → 0.5.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/benchmark-small.js +81 -0
- package/dist/common/decoder.js +13 -7
- package/dist/common/decoder.js.map +1 -1
- package/dist/common/encoder.d.ts +1 -3
- package/dist/common/encoder.js +3 -7
- package/dist/common/encoder.js.map +1 -1
- package/dist/common/string-decoder.js +0 -5
- package/dist/common/string-decoder.js.map +1 -1
- package/dist/encoder.d.ts +9 -1
- package/dist/encoder.js +41 -20
- package/dist/encoder.js.map +1 -1
- package/dist/index.js +2 -3
- package/dist/index.js.map +1 -1
- package/dist/rxjs/decoder.js +5 -2
- package/dist/rxjs/decoder.js.map +1 -1
- package/dist/rxjs/encoder.js +5 -4
- package/dist/rxjs/encoder.js.map +1 -1
- package/dist/stream/encoder.d.ts +3 -0
- package/dist/stream/encoder.js +8 -2
- package/dist/stream/encoder.js.map +1 -1
- package/dist/stream-helper/encoder.d.ts +10 -2
- package/dist/stream-helper/encoder.js +50 -24
- package/dist/stream-helper/encoder.js.map +1 -1
- package/dist/utils.d.ts +1 -1
- package/dist/utils.js +1 -4
- package/dist/utils.js.map +1 -1
- package/package.json +6 -15
- package/src/common/decoder.ts +11 -6
- package/src/common/encoder.ts +3 -5
- package/src/common/string-decoder.ts +0 -4
- package/src/encoder.ts +44 -21
- package/src/index.ts +2 -3
- package/src/rxjs/decoder.ts +6 -2
- package/src/rxjs/encoder.ts +5 -4
- package/src/stream/encoder.ts +10 -2
- package/src/stream-helper/encoder.ts +48 -26
- package/src/utils.ts +1 -4
- package/tests/e2e.js +3 -0
- package/tests/encode.js +142 -0
- package/tests/huge-string.js +3 -0
- package/tests/stream/decode.js +15 -0
- package/tests/stream/encode.js +7 -0
- package/tests/string-encoding.js +2 -2
|
@@ -2,11 +2,19 @@ import { EncoderBase } from '../common/encoder.js';
|
|
|
2
2
|
/** 流式编码 UBJSON */
|
|
3
3
|
export declare class StreamEncoderHelper extends EncoderBase {
|
|
4
4
|
protected readonly onChunk: (chunk: Uint8Array) => void;
|
|
5
|
-
constructor(
|
|
5
|
+
constructor(onChunk: (chunk: Uint8Array) => void);
|
|
6
|
+
/**
|
|
7
|
+
* 销毁实例,释放内存池
|
|
8
|
+
*/
|
|
9
|
+
destroy(): void;
|
|
10
|
+
/** 通过内存池减少分配 */
|
|
11
|
+
private readonly pool;
|
|
6
12
|
/**
|
|
7
13
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
8
14
|
*/
|
|
9
15
|
protected ensureCapacity(capacity: number): void;
|
|
16
|
+
/** 分配 buffer */
|
|
17
|
+
private allocUnsafe;
|
|
10
18
|
/** 获取写入结果 */
|
|
11
|
-
encode(): void;
|
|
19
|
+
encode(value: unknown): void;
|
|
12
20
|
}
|
|
@@ -1,13 +1,33 @@
|
|
|
1
1
|
import { EncoderBase } from '../common/encoder.js';
|
|
2
2
|
const BLOCK_SIZE = 1024 * 8; // 8 KiB
|
|
3
3
|
const MAX_SIZE = 1024 * 1024 * 256; // 256 MiB
|
|
4
|
+
/** 保存一个内存池以减少重复分配 */
|
|
5
|
+
let POOL = null;
|
|
4
6
|
/** 流式编码 UBJSON */
|
|
5
7
|
export class StreamEncoderHelper extends EncoderBase {
|
|
6
8
|
onChunk;
|
|
7
|
-
constructor(
|
|
8
|
-
super(
|
|
9
|
+
constructor(onChunk) {
|
|
10
|
+
super();
|
|
9
11
|
this.onChunk = onChunk;
|
|
12
|
+
if (POOL != null) {
|
|
13
|
+
this.pool = POOL;
|
|
14
|
+
POOL = null;
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
this.pool = new Uint8Array(BLOCK_SIZE);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* 销毁实例,释放内存池
|
|
22
|
+
*/
|
|
23
|
+
destroy() {
|
|
24
|
+
POOL ??= this.pool;
|
|
25
|
+
const self = this;
|
|
26
|
+
self.pool = null;
|
|
27
|
+
self.buffer = null;
|
|
10
28
|
}
|
|
29
|
+
/** 通过内存池减少分配 */
|
|
30
|
+
pool;
|
|
11
31
|
/**
|
|
12
32
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
13
33
|
*/
|
|
@@ -16,43 +36,49 @@ export class StreamEncoderHelper extends EncoderBase {
|
|
|
16
36
|
// 超过最大尺寸限制
|
|
17
37
|
throw new Error('Buffer has exceed max size');
|
|
18
38
|
}
|
|
19
|
-
if (this.buffer == null) {
|
|
20
|
-
this.buffer = new Uint8Array(capacity);
|
|
21
|
-
this.view = new DataView(this.buffer.buffer);
|
|
22
|
-
return;
|
|
23
|
-
}
|
|
24
39
|
if (capacity < 0) {
|
|
25
40
|
// 结束流
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
41
|
+
if (this.buffer === this.pool) {
|
|
42
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
46
|
+
}
|
|
30
47
|
return;
|
|
31
48
|
}
|
|
32
49
|
// 无需扩容
|
|
33
50
|
if (this.buffer.byteLength >= this.length + capacity)
|
|
34
51
|
return;
|
|
35
52
|
// 提交目前的数据
|
|
36
|
-
|
|
53
|
+
if (this.buffer === this.pool) {
|
|
54
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
58
|
+
}
|
|
37
59
|
// 重新分配缓冲区
|
|
38
60
|
if (capacity < BLOCK_SIZE)
|
|
39
61
|
capacity = BLOCK_SIZE;
|
|
40
|
-
this.
|
|
62
|
+
this.allocUnsafe(capacity);
|
|
63
|
+
}
|
|
64
|
+
/** 分配 buffer */
|
|
65
|
+
allocUnsafe(size) {
|
|
66
|
+
if (size === this.pool.byteLength) {
|
|
67
|
+
// 从 pool 中获取
|
|
68
|
+
this.buffer = this.pool;
|
|
69
|
+
this.view = new DataView(this.buffer.buffer);
|
|
70
|
+
this.length = 0;
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
this.buffer = new Uint8Array(size);
|
|
41
74
|
this.view = new DataView(this.buffer.buffer);
|
|
42
75
|
this.length = 0;
|
|
43
76
|
}
|
|
44
77
|
/** 获取写入结果 */
|
|
45
|
-
encode() {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
this.ensureCapacity(-1);
|
|
50
|
-
}
|
|
51
|
-
else {
|
|
52
|
-
this.ensureCapacity(BLOCK_SIZE);
|
|
53
|
-
this.writeValue();
|
|
54
|
-
this.ensureCapacity(-1);
|
|
55
|
-
}
|
|
78
|
+
encode(value) {
|
|
79
|
+
this.allocUnsafe(BLOCK_SIZE);
|
|
80
|
+
this.writeValue(value);
|
|
81
|
+
this.ensureCapacity(-1);
|
|
56
82
|
}
|
|
57
83
|
}
|
|
58
84
|
//# sourceMappingURL=encoder.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../../src/stream-helper/encoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ;AACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,UAAU;AAE9C,kBAAkB;AAClB,MAAM,OAAO,mBAAoB,SAAQ,WAAW;
|
|
1
|
+
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../../src/stream-helper/encoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ;AACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,UAAU;AAE9C,qBAAqB;AACrB,IAAI,IAAI,GAAsB,IAAI,CAAC;AAEnC,kBAAkB;AAClB,MAAM,OAAO,mBAAoB,SAAQ,WAAW;IACjB;IAA/B,YAA+B,OAAoC;QAC/D,KAAK,EAAE,CAAC;QADmB,YAAO,GAAP,OAAO,CAA6B;QAE/D,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,GAAG,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,IAAI,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;IACL,CAAC;IACD;;OAEG;IACH,OAAO;QACH,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC;QACnB,MAAM,IAAI,GAAG,IAAyE,CAAC;QACvF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,gBAAgB;IACC,IAAI,CAAC;IACtB;;OAEG;IACO,cAAc,CAAC,QAAgB;QACrC,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACtB,WAAW;YACX,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACf,MAAM;YACN,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACvD,CAAC;YACD,OAAO;QACX,CAAC;QACD,OAAO;QACP,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ;YAAE,OAAO;QAE7D,UAAU;QACV,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACvD,CAAC;QAED,UAAU;QACV,IAAI,QAAQ,GAAG,UAAU;YAAE,QAAQ,GAAG,UAAU,CAAC;QACjD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IACD,gBAAgB;IACR,WAAW,CAAC,IAAY;QAC5B,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,aAAa;YACb,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;YACxB,IAAI,CAAC,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,aAAa;IACb,MAAM,CAAC,KAAc;QACjB,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACvB,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;CACJ"}
|
package/dist/utils.d.ts
CHANGED
package/dist/utils.js
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
/** 支持的数据转为 Uint8Array */
|
|
2
|
-
export function toUint8Array(data
|
|
2
|
+
export function toUint8Array(data) {
|
|
3
3
|
if (data == null || typeof data != 'object' || typeof data.byteLength != 'number') {
|
|
4
4
|
throw new TypeError('Invalid data');
|
|
5
5
|
}
|
|
6
6
|
if (!ArrayBuffer.isView(data)) {
|
|
7
7
|
return new Uint8Array(data);
|
|
8
8
|
}
|
|
9
|
-
if (exact && (data.byteOffset !== 0 || data.buffer.byteLength !== data.byteLength)) {
|
|
10
|
-
return new Uint8Array(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
|
|
11
|
-
}
|
|
12
9
|
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
13
10
|
}
|
|
14
11
|
/** 未结束的流 */
|
package/dist/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,MAAM,UAAU,YAAY,CAAC,IAAgB
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,MAAM,UAAU,YAAY,CAAC,IAAgB;IACzC,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,IAAI,OAAO,IAAI,CAAC,UAAU,IAAI,QAAQ,EAAE,CAAC;QAChF,MAAM,IAAI,SAAS,CAAC,cAAc,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;AACzE,CAAC;AAED,YAAY;AACZ,MAAM,OAAO,aAAc,SAAQ,KAAK;IACpC;QACI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC5B,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cloudpss/ubjson",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.11",
|
|
4
4
|
"author": "CloudPSS",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -27,23 +27,14 @@
|
|
|
27
27
|
}
|
|
28
28
|
},
|
|
29
29
|
"exports": {
|
|
30
|
-
".":
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
},
|
|
34
|
-
"./stream": {
|
|
35
|
-
"types": "./dist/stream/index.d.ts",
|
|
36
|
-
"default": "./dist/stream/index.js"
|
|
37
|
-
},
|
|
38
|
-
"./rxjs": {
|
|
39
|
-
"types": "./dist/rxjs/index.d.ts",
|
|
40
|
-
"default": "./dist/rxjs/index.js"
|
|
41
|
-
}
|
|
30
|
+
".": "./dist/index.js",
|
|
31
|
+
"./stream": "./dist/stream/index.js",
|
|
32
|
+
"./rxjs": "./dist/rxjs/index.js"
|
|
42
33
|
},
|
|
43
34
|
"devDependencies": {
|
|
44
|
-
"@msgpack/msgpack": "3.0.0-beta2",
|
|
45
35
|
"@types/lodash": "^4.14.202",
|
|
46
|
-
"
|
|
36
|
+
"lodash": "^4.17.21",
|
|
37
|
+
"ref-impl": "npm:@cloudpss/ubjson@0.5.9"
|
|
47
38
|
},
|
|
48
39
|
"dependencies": {
|
|
49
40
|
"rxjs": "^7.8.1"
|
package/src/common/decoder.ts
CHANGED
|
@@ -11,7 +11,7 @@ export class Decoder {
|
|
|
11
11
|
/** 当前读指针位置 */
|
|
12
12
|
protected offset = 0;
|
|
13
13
|
constructor(data: BinaryData) {
|
|
14
|
-
this.data = toUint8Array(data
|
|
14
|
+
this.data = toUint8Array(data);
|
|
15
15
|
this.view = new DataView(this.data.buffer, this.data.byteOffset, this.data.byteLength);
|
|
16
16
|
}
|
|
17
17
|
|
|
@@ -53,8 +53,11 @@ export class Decoder {
|
|
|
53
53
|
case constants.STRING: {
|
|
54
54
|
const length = this.readIntLength();
|
|
55
55
|
const begin = this.offset;
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
const end = begin + length;
|
|
57
|
+
this.offset = end;
|
|
58
|
+
const data = this.data;
|
|
59
|
+
if (end > data.length) this.eof();
|
|
60
|
+
return decode(data, begin, end);
|
|
58
61
|
}
|
|
59
62
|
case constants.OBJECT: {
|
|
60
63
|
const markers = this.readOptimizedFormatMarkers();
|
|
@@ -210,7 +213,6 @@ export class Decoder {
|
|
|
210
213
|
/** 读取一个大于 0 的整数 */
|
|
211
214
|
private readIntLength(): number {
|
|
212
215
|
const marker = this.readMarker();
|
|
213
|
-
if (marker === undefined) this.eof();
|
|
214
216
|
let length;
|
|
215
217
|
switch (marker) {
|
|
216
218
|
case constants.INT8:
|
|
@@ -241,8 +243,11 @@ export class Decoder {
|
|
|
241
243
|
private readKey(): string {
|
|
242
244
|
const length = this.readIntLength();
|
|
243
245
|
const begin = this.offset;
|
|
244
|
-
|
|
245
|
-
|
|
246
|
+
const end = begin + length;
|
|
247
|
+
this.offset = end;
|
|
248
|
+
const data = this.data;
|
|
249
|
+
if (end > data.length) this.eof();
|
|
250
|
+
return decodeKey(data, begin, end);
|
|
246
251
|
}
|
|
247
252
|
/** readInt8Data */
|
|
248
253
|
private readInt8Data(): number {
|
package/src/common/encoder.ts
CHANGED
|
@@ -11,21 +11,19 @@ export abstract class EncoderBase {
|
|
|
11
11
|
protected buffer!: Uint8Array;
|
|
12
12
|
/** buffer 的 DataView */
|
|
13
13
|
protected view!: DataView;
|
|
14
|
-
|
|
15
|
-
constructor(readonly value: unknown) {}
|
|
16
14
|
/**
|
|
17
15
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
18
16
|
*/
|
|
19
17
|
protected abstract ensureCapacity(capacity: number): void;
|
|
20
18
|
|
|
21
19
|
/** 编码至 ubjson */
|
|
22
|
-
protected writeValue(): void {
|
|
23
|
-
if (
|
|
20
|
+
protected writeValue(value: unknown): void {
|
|
21
|
+
if (value === undefined) {
|
|
24
22
|
this.ensureCapacity(1);
|
|
25
23
|
this.buffer[this.length++] = constants.NO_OP;
|
|
26
24
|
return;
|
|
27
25
|
}
|
|
28
|
-
this.write(
|
|
26
|
+
this.write(value);
|
|
29
27
|
}
|
|
30
28
|
/** 写入一个对象 */
|
|
31
29
|
private write(value: unknown): void {
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { UnexpectedEof } from '../utils.js';
|
|
2
|
-
|
|
3
1
|
/* c8 ignore next 2: TextDecoder always present, fallback tested */
|
|
4
2
|
export const textDecoder =
|
|
5
3
|
typeof TextDecoder == 'function' ? new TextDecoder('utf8', { ignoreBOM: true, fatal: false }) : null;
|
|
@@ -194,7 +192,6 @@ function shortStringInJS(buf: Uint8Array, begin: number, length: number): string
|
|
|
194
192
|
|
|
195
193
|
/** 字符串解码,无缓存 */
|
|
196
194
|
export function decode(data: Uint8Array, begin: number, end: number): string {
|
|
197
|
-
if (end > data.length) throw new UnexpectedEof();
|
|
198
195
|
const length = end - begin;
|
|
199
196
|
if (length < 16) {
|
|
200
197
|
const result = shortStringInJS(data, begin, length);
|
|
@@ -225,7 +222,6 @@ export function decodeKey(data: Uint8Array, begin: number, end: number): string
|
|
|
225
222
|
if (i === length) return entry.value;
|
|
226
223
|
}
|
|
227
224
|
|
|
228
|
-
if (end > data.length) throw new UnexpectedEof();
|
|
229
225
|
let str = length < 16 ? shortStringInJS(data, begin, length) : longStringInJS(data, begin, length);
|
|
230
226
|
if (str == null) {
|
|
231
227
|
// 只有小字符串有优化价值,见 benchmark-string.js
|
package/src/encoder.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { EncoderBase } from './common/encoder.js';
|
|
2
2
|
|
|
3
|
-
const BLOCK_SIZE = 1024 * 16; //
|
|
3
|
+
const BLOCK_SIZE = 1024 * 16; // 16 KiB
|
|
4
4
|
const MAX_SIZE = 1024 * 1024 * 128; //128 MiB
|
|
5
5
|
|
|
6
6
|
/** 编码至 ubjson */
|
|
7
7
|
export class Encoder extends EncoderBase {
|
|
8
8
|
private readonly flushedBuffers: Uint8Array[] = [];
|
|
9
|
+
/** 通过内存池减少分配 */
|
|
10
|
+
private readonly pool = new Uint8Array(BLOCK_SIZE);
|
|
9
11
|
/**
|
|
10
12
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
11
13
|
*/
|
|
@@ -14,43 +16,49 @@ export class Encoder extends EncoderBase {
|
|
|
14
16
|
// 超过最大尺寸限制
|
|
15
17
|
throw new Error('Buffer has exceed max size');
|
|
16
18
|
}
|
|
17
|
-
if (this.buffer == null) {
|
|
18
|
-
this.buffer = new Uint8Array(capacity);
|
|
19
|
-
this.view = new DataView(this.buffer.buffer);
|
|
20
|
-
return;
|
|
21
|
-
}
|
|
22
19
|
// 无需扩容
|
|
23
20
|
if (this.buffer.byteLength >= this.length + capacity) return;
|
|
24
21
|
|
|
25
22
|
// 提交目前的数据
|
|
26
|
-
|
|
23
|
+
if (this.buffer === this.pool) {
|
|
24
|
+
this.flushedBuffers.push(this.buffer.slice(0, this.length));
|
|
25
|
+
} else {
|
|
26
|
+
this.flushedBuffers.push(this.buffer.subarray(0, this.length));
|
|
27
|
+
}
|
|
27
28
|
|
|
28
29
|
// 重新分配缓冲区
|
|
29
30
|
if (capacity < BLOCK_SIZE) capacity = BLOCK_SIZE;
|
|
30
|
-
this.
|
|
31
|
+
this.allocUnsafe(capacity);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** 分配 buffer */
|
|
35
|
+
private allocUnsafe(size: number): void {
|
|
36
|
+
if (size === this.pool.byteLength) {
|
|
37
|
+
// 从 pool 中获取
|
|
38
|
+
this.buffer = this.pool;
|
|
39
|
+
this.view = new DataView(this.buffer.buffer);
|
|
40
|
+
this.length = 0;
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
this.buffer = new Uint8Array(size);
|
|
31
44
|
this.view = new DataView(this.buffer.buffer);
|
|
32
45
|
this.length = 0;
|
|
33
46
|
}
|
|
34
47
|
|
|
35
48
|
/** 获取写入结果 */
|
|
36
|
-
encode(): Uint8Array {
|
|
37
|
-
this.
|
|
38
|
-
this.writeValue();
|
|
49
|
+
encode(value: unknown): Uint8Array {
|
|
50
|
+
this.allocUnsafe(BLOCK_SIZE);
|
|
51
|
+
this.writeValue(value);
|
|
39
52
|
if (this.flushedBuffers.length === 0) {
|
|
40
|
-
|
|
41
|
-
// 无需再拷贝一次
|
|
42
|
-
return this.buffer;
|
|
43
|
-
} else {
|
|
44
|
-
return this.buffer.slice(0, this.length);
|
|
45
|
-
}
|
|
53
|
+
return this.buffer.slice(0, this.length);
|
|
46
54
|
}
|
|
47
55
|
|
|
48
56
|
// 合并缓冲区
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
);
|
|
57
|
+
const flushedBuffers = this.flushedBuffers.splice(0);
|
|
58
|
+
const size = flushedBuffers.reduce((sum, buffer) => sum + buffer.byteLength, this.length);
|
|
59
|
+
const result = new Uint8Array(size);
|
|
52
60
|
let offset = 0;
|
|
53
|
-
for (const buffer of
|
|
61
|
+
for (const buffer of flushedBuffers) {
|
|
54
62
|
result.set(buffer, offset);
|
|
55
63
|
offset += buffer.byteLength;
|
|
56
64
|
}
|
|
@@ -58,3 +66,18 @@ export class Encoder extends EncoderBase {
|
|
|
58
66
|
return result;
|
|
59
67
|
}
|
|
60
68
|
}
|
|
69
|
+
|
|
70
|
+
let _ENCODER: Encoder | undefined;
|
|
71
|
+
|
|
72
|
+
/** 获取默认的编码器 */
|
|
73
|
+
export function getEncoder(): Encoder {
|
|
74
|
+
if (_ENCODER == null) {
|
|
75
|
+
_ENCODER = new Encoder();
|
|
76
|
+
}
|
|
77
|
+
return _ENCODER;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** 重置编码器, For testing only */
|
|
81
|
+
export function resetEncoder(): void {
|
|
82
|
+
_ENCODER = undefined;
|
|
83
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getEncoder } from './encoder.js';
|
|
2
2
|
import { Decoder } from './decoder.js';
|
|
3
3
|
export { UnexpectedEof } from './utils.js';
|
|
4
4
|
|
|
5
5
|
/** 编码为 UBJSON */
|
|
6
6
|
export function encode(value: unknown): Uint8Array {
|
|
7
|
-
|
|
8
|
-
return encoder.encode();
|
|
7
|
+
return getEncoder().encode(value);
|
|
9
8
|
}
|
|
10
9
|
|
|
11
10
|
/** 解码 UBJSON */
|
package/src/rxjs/decoder.ts
CHANGED
|
@@ -15,13 +15,17 @@ export function decode(): OperatorFunction<BinaryData, unknown> {
|
|
|
15
15
|
let end = 0;
|
|
16
16
|
return observable.subscribe({
|
|
17
17
|
next(value) {
|
|
18
|
-
const data = toUint8Array(value
|
|
18
|
+
const data = toUint8Array(value);
|
|
19
19
|
if (buffer.length >= end + data.length) {
|
|
20
20
|
buffer.set(data, end);
|
|
21
21
|
end += data.length;
|
|
22
22
|
} else {
|
|
23
23
|
chunkSize = Math.max(chunkSize, data.length * 2 + end - begin);
|
|
24
|
-
const
|
|
24
|
+
const allocSize = Math.max(chunkSize, data.length + end - begin);
|
|
25
|
+
const newBuffer =
|
|
26
|
+
allocSize > buffer.byteLength
|
|
27
|
+
? new Uint8Array(Math.max(chunkSize, data.length + end - begin))
|
|
28
|
+
: buffer;
|
|
25
29
|
newBuffer.set(buffer.subarray(begin, end), 0);
|
|
26
30
|
newBuffer.set(data, end - begin);
|
|
27
31
|
buffer = newBuffer;
|
package/src/rxjs/encoder.ts
CHANGED
|
@@ -5,12 +5,11 @@ import { StreamEncoderHelper } from '../stream-helper/encoder.js';
|
|
|
5
5
|
export function encode(): OperatorFunction<unknown, Uint8Array> {
|
|
6
6
|
return (observable) => {
|
|
7
7
|
return new Observable<Uint8Array>((subscriber) => {
|
|
8
|
-
const
|
|
9
|
-
|
|
8
|
+
const helper = new StreamEncoderHelper((chunk: Uint8Array): void => subscriber.next(chunk));
|
|
9
|
+
const sub = observable.subscribe({
|
|
10
10
|
next(value) {
|
|
11
11
|
try {
|
|
12
|
-
|
|
13
|
-
helper.encode();
|
|
12
|
+
helper.encode(value);
|
|
14
13
|
} catch (ex) {
|
|
15
14
|
subscriber.error(ex as Error);
|
|
16
15
|
}
|
|
@@ -22,6 +21,8 @@ export function encode(): OperatorFunction<unknown, Uint8Array> {
|
|
|
22
21
|
subscriber.complete();
|
|
23
22
|
},
|
|
24
23
|
});
|
|
24
|
+
sub.add(() => helper.destroy());
|
|
25
|
+
return sub;
|
|
25
26
|
});
|
|
26
27
|
};
|
|
27
28
|
}
|
package/src/stream/encoder.ts
CHANGED
|
@@ -8,16 +8,24 @@ export class StreamEncoder extends Transform {
|
|
|
8
8
|
readableObjectMode: false,
|
|
9
9
|
writableObjectMode: true,
|
|
10
10
|
});
|
|
11
|
+
this.helper = new StreamEncoderHelper((binary) => this.push(binary));
|
|
11
12
|
}
|
|
12
13
|
|
|
14
|
+
private readonly helper;
|
|
15
|
+
|
|
13
16
|
/** @inheritdoc */
|
|
14
17
|
override _transform(obj: unknown, _encoding: BufferEncoding, callback: TransformCallback): void {
|
|
15
|
-
const helper = new StreamEncoderHelper(obj, (binary) => this.push(binary));
|
|
16
18
|
try {
|
|
17
|
-
helper.encode();
|
|
19
|
+
this.helper.encode(obj);
|
|
18
20
|
callback();
|
|
19
21
|
} catch (ex) {
|
|
20
22
|
callback(ex as Error);
|
|
21
23
|
}
|
|
22
24
|
}
|
|
25
|
+
|
|
26
|
+
/** @inheritdoc */
|
|
27
|
+
override _destroy(error: Error | null, callback: (error?: Error | null | undefined) => void): void {
|
|
28
|
+
this.helper.destroy();
|
|
29
|
+
super._destroy(error, callback);
|
|
30
|
+
}
|
|
23
31
|
}
|
|
@@ -3,14 +3,31 @@ import { EncoderBase } from '../common/encoder.js';
|
|
|
3
3
|
const BLOCK_SIZE = 1024 * 8; // 8 KiB
|
|
4
4
|
const MAX_SIZE = 1024 * 1024 * 256; // 256 MiB
|
|
5
5
|
|
|
6
|
+
/** 保存一个内存池以减少重复分配 */
|
|
7
|
+
let POOL: Uint8Array | null = null;
|
|
8
|
+
|
|
6
9
|
/** 流式编码 UBJSON */
|
|
7
10
|
export class StreamEncoderHelper extends EncoderBase {
|
|
8
|
-
constructor(
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
constructor(protected readonly onChunk: (chunk: Uint8Array) => void) {
|
|
12
|
+
super();
|
|
13
|
+
if (POOL != null) {
|
|
14
|
+
this.pool = POOL;
|
|
15
|
+
POOL = null;
|
|
16
|
+
} else {
|
|
17
|
+
this.pool = new Uint8Array(BLOCK_SIZE);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* 销毁实例,释放内存池
|
|
22
|
+
*/
|
|
23
|
+
destroy(): void {
|
|
24
|
+
POOL ??= this.pool;
|
|
25
|
+
const self = this as unknown as { pool: Uint8Array | null; buffer: Uint8Array | null };
|
|
26
|
+
self.pool = null;
|
|
27
|
+
self.buffer = null;
|
|
13
28
|
}
|
|
29
|
+
/** 通过内存池减少分配 */
|
|
30
|
+
private readonly pool;
|
|
14
31
|
/**
|
|
15
32
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
16
33
|
*/
|
|
@@ -19,41 +36,46 @@ export class StreamEncoderHelper extends EncoderBase {
|
|
|
19
36
|
// 超过最大尺寸限制
|
|
20
37
|
throw new Error('Buffer has exceed max size');
|
|
21
38
|
}
|
|
22
|
-
if (this.buffer == null) {
|
|
23
|
-
this.buffer = new Uint8Array(capacity);
|
|
24
|
-
this.view = new DataView(this.buffer.buffer);
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
39
|
if (capacity < 0) {
|
|
28
40
|
// 结束流
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
41
|
+
if (this.buffer === this.pool) {
|
|
42
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
43
|
+
} else {
|
|
44
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
45
|
+
}
|
|
33
46
|
return;
|
|
34
47
|
}
|
|
35
48
|
// 无需扩容
|
|
36
49
|
if (this.buffer.byteLength >= this.length + capacity) return;
|
|
37
50
|
|
|
38
51
|
// 提交目前的数据
|
|
39
|
-
|
|
52
|
+
if (this.buffer === this.pool) {
|
|
53
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
54
|
+
} else {
|
|
55
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
56
|
+
}
|
|
40
57
|
|
|
41
58
|
// 重新分配缓冲区
|
|
42
59
|
if (capacity < BLOCK_SIZE) capacity = BLOCK_SIZE;
|
|
43
|
-
this.
|
|
60
|
+
this.allocUnsafe(capacity);
|
|
61
|
+
}
|
|
62
|
+
/** 分配 buffer */
|
|
63
|
+
private allocUnsafe(size: number): void {
|
|
64
|
+
if (size === this.pool.byteLength) {
|
|
65
|
+
// 从 pool 中获取
|
|
66
|
+
this.buffer = this.pool;
|
|
67
|
+
this.view = new DataView(this.buffer.buffer);
|
|
68
|
+
this.length = 0;
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
this.buffer = new Uint8Array(size);
|
|
44
72
|
this.view = new DataView(this.buffer.buffer);
|
|
45
73
|
this.length = 0;
|
|
46
74
|
}
|
|
47
75
|
/** 获取写入结果 */
|
|
48
|
-
encode(): void {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
this.ensureCapacity(-1);
|
|
53
|
-
} else {
|
|
54
|
-
this.ensureCapacity(BLOCK_SIZE);
|
|
55
|
-
this.writeValue();
|
|
56
|
-
this.ensureCapacity(-1);
|
|
57
|
-
}
|
|
76
|
+
encode(value: unknown): void {
|
|
77
|
+
this.allocUnsafe(BLOCK_SIZE);
|
|
78
|
+
this.writeValue(value);
|
|
79
|
+
this.ensureCapacity(-1);
|
|
58
80
|
}
|
|
59
81
|
}
|
package/src/utils.ts
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
/** 支持的数据转为 Uint8Array */
|
|
2
|
-
export function toUint8Array(data: BinaryData
|
|
2
|
+
export function toUint8Array(data: BinaryData): Uint8Array {
|
|
3
3
|
if (data == null || typeof data != 'object' || typeof data.byteLength != 'number') {
|
|
4
4
|
throw new TypeError('Invalid data');
|
|
5
5
|
}
|
|
6
6
|
if (!ArrayBuffer.isView(data)) {
|
|
7
7
|
return new Uint8Array(data);
|
|
8
8
|
}
|
|
9
|
-
if (exact && (data.byteOffset !== 0 || data.buffer.byteLength !== data.byteLength)) {
|
|
10
|
-
return new Uint8Array(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
|
|
11
|
-
}
|
|
12
9
|
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
13
10
|
}
|
|
14
11
|
|
package/tests/e2e.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Tests from https://bitbucket.org/shelacek/ubjson
|
|
3
3
|
*/
|
|
4
4
|
import { encode, decode } from '../dist/index.js';
|
|
5
|
+
import { resetEncoder } from '../dist/encoder.js';
|
|
5
6
|
|
|
6
7
|
test('encode/decode complex object', () => {
|
|
7
8
|
const expected = {
|
|
@@ -301,6 +302,7 @@ test('encode/decode complex object (no encodeInto)', () => {
|
|
|
301
302
|
const encodeInto = TextEncoder.prototype.encodeInto;
|
|
302
303
|
// @ts-expect-error 移除 encodeInto 以测试兼容性
|
|
303
304
|
TextEncoder.prototype.encodeInto = undefined;
|
|
305
|
+
resetEncoder();
|
|
304
306
|
const expected = {
|
|
305
307
|
hello: 'world',
|
|
306
308
|
from: ['UBJSON'],
|
|
@@ -372,6 +374,7 @@ test('encode/decode complex object (no encodeInto)', () => {
|
|
|
372
374
|
const actual = decode(encoded);
|
|
373
375
|
expect(actual).toEqual(expected);
|
|
374
376
|
TextEncoder.prototype.encodeInto = encodeInto;
|
|
377
|
+
resetEncoder();
|
|
375
378
|
});
|
|
376
379
|
|
|
377
380
|
test('encode/decode large array', () => {
|