@cloudpss/ubjson 0.5.10 → 0.5.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/benchmark-small.js +81 -0
- package/dist/common/decoder.js +1 -3
- package/dist/common/decoder.js.map +1 -1
- package/dist/common/encoder.d.ts +1 -3
- package/dist/common/encoder.js +3 -7
- package/dist/common/encoder.js.map +1 -1
- package/dist/encoder.d.ts +9 -1
- package/dist/encoder.js +41 -20
- package/dist/encoder.js.map +1 -1
- package/dist/index.js +2 -3
- package/dist/index.js.map +1 -1
- package/dist/rxjs/decoder.js +5 -2
- package/dist/rxjs/decoder.js.map +1 -1
- package/dist/rxjs/encoder.js +5 -4
- package/dist/rxjs/encoder.js.map +1 -1
- package/dist/stream/encoder.d.ts +3 -0
- package/dist/stream/encoder.js +8 -2
- package/dist/stream/encoder.js.map +1 -1
- package/dist/stream-helper/encoder.d.ts +10 -2
- package/dist/stream-helper/encoder.js +50 -24
- package/dist/stream-helper/encoder.js.map +1 -1
- package/dist/utils.d.ts +1 -1
- package/dist/utils.js +1 -4
- package/dist/utils.js.map +1 -1
- package/package.json +6 -15
- package/src/common/decoder.ts +1 -2
- package/src/common/encoder.ts +3 -5
- package/src/encoder.ts +44 -21
- package/src/index.ts +2 -3
- package/src/rxjs/decoder.ts +6 -2
- package/src/rxjs/encoder.ts +5 -4
- package/src/stream/encoder.ts +10 -2
- package/src/stream-helper/encoder.ts +48 -26
- package/src/utils.ts +1 -4
- package/tests/e2e.js +3 -0
- package/tests/encode.js +142 -0
- package/tests/huge-string.js +3 -0
- package/tests/stream/decode.js +15 -0
- package/tests/stream/encode.js +7 -0
- package/tests/string-encoding.js +2 -2
|
@@ -1,13 +1,33 @@
|
|
|
1
1
|
import { EncoderBase } from '../common/encoder.js';
|
|
2
2
|
const BLOCK_SIZE = 1024 * 8; // 8 KiB
|
|
3
3
|
const MAX_SIZE = 1024 * 1024 * 256; // 256 MiB
|
|
4
|
+
/** 保存一个内存池以减少重复分配 */
|
|
5
|
+
let POOL = null;
|
|
4
6
|
/** 流式编码 UBJSON */
|
|
5
7
|
export class StreamEncoderHelper extends EncoderBase {
|
|
6
8
|
onChunk;
|
|
7
|
-
constructor(
|
|
8
|
-
super(
|
|
9
|
+
constructor(onChunk) {
|
|
10
|
+
super();
|
|
9
11
|
this.onChunk = onChunk;
|
|
12
|
+
if (POOL != null) {
|
|
13
|
+
this.pool = POOL;
|
|
14
|
+
POOL = null;
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
this.pool = new Uint8Array(BLOCK_SIZE);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* 销毁实例,释放内存池
|
|
22
|
+
*/
|
|
23
|
+
destroy() {
|
|
24
|
+
POOL ??= this.pool;
|
|
25
|
+
const self = this;
|
|
26
|
+
self.pool = null;
|
|
27
|
+
self.buffer = null;
|
|
10
28
|
}
|
|
29
|
+
/** 通过内存池减少分配 */
|
|
30
|
+
pool;
|
|
11
31
|
/**
|
|
12
32
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
13
33
|
*/
|
|
@@ -16,43 +36,49 @@ export class StreamEncoderHelper extends EncoderBase {
|
|
|
16
36
|
// 超过最大尺寸限制
|
|
17
37
|
throw new Error('Buffer has exceed max size');
|
|
18
38
|
}
|
|
19
|
-
if (this.buffer == null) {
|
|
20
|
-
this.buffer = new Uint8Array(capacity);
|
|
21
|
-
this.view = new DataView(this.buffer.buffer);
|
|
22
|
-
return;
|
|
23
|
-
}
|
|
24
39
|
if (capacity < 0) {
|
|
25
40
|
// 结束流
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
41
|
+
if (this.buffer === this.pool) {
|
|
42
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
46
|
+
}
|
|
30
47
|
return;
|
|
31
48
|
}
|
|
32
49
|
// 无需扩容
|
|
33
50
|
if (this.buffer.byteLength >= this.length + capacity)
|
|
34
51
|
return;
|
|
35
52
|
// 提交目前的数据
|
|
36
|
-
|
|
53
|
+
if (this.buffer === this.pool) {
|
|
54
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
58
|
+
}
|
|
37
59
|
// 重新分配缓冲区
|
|
38
60
|
if (capacity < BLOCK_SIZE)
|
|
39
61
|
capacity = BLOCK_SIZE;
|
|
40
|
-
this.
|
|
62
|
+
this.allocUnsafe(capacity);
|
|
63
|
+
}
|
|
64
|
+
/** 分配 buffer */
|
|
65
|
+
allocUnsafe(size) {
|
|
66
|
+
if (size === this.pool.byteLength) {
|
|
67
|
+
// 从 pool 中获取
|
|
68
|
+
this.buffer = this.pool;
|
|
69
|
+
this.view = new DataView(this.buffer.buffer);
|
|
70
|
+
this.length = 0;
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
this.buffer = new Uint8Array(size);
|
|
41
74
|
this.view = new DataView(this.buffer.buffer);
|
|
42
75
|
this.length = 0;
|
|
43
76
|
}
|
|
44
77
|
/** 获取写入结果 */
|
|
45
|
-
encode() {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
this.ensureCapacity(-1);
|
|
50
|
-
}
|
|
51
|
-
else {
|
|
52
|
-
this.ensureCapacity(BLOCK_SIZE);
|
|
53
|
-
this.writeValue();
|
|
54
|
-
this.ensureCapacity(-1);
|
|
55
|
-
}
|
|
78
|
+
encode(value) {
|
|
79
|
+
this.allocUnsafe(BLOCK_SIZE);
|
|
80
|
+
this.writeValue(value);
|
|
81
|
+
this.ensureCapacity(-1);
|
|
56
82
|
}
|
|
57
83
|
}
|
|
58
84
|
//# sourceMappingURL=encoder.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../../src/stream-helper/encoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ;AACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,UAAU;AAE9C,kBAAkB;AAClB,MAAM,OAAO,mBAAoB,SAAQ,WAAW;
|
|
1
|
+
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../../src/stream-helper/encoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ;AACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,UAAU;AAE9C,qBAAqB;AACrB,IAAI,IAAI,GAAsB,IAAI,CAAC;AAEnC,kBAAkB;AAClB,MAAM,OAAO,mBAAoB,SAAQ,WAAW;IACjB;IAA/B,YAA+B,OAAoC;QAC/D,KAAK,EAAE,CAAC;QADmB,YAAO,GAAP,OAAO,CAA6B;QAE/D,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,GAAG,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,IAAI,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;IACL,CAAC;IACD;;OAEG;IACH,OAAO;QACH,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC;QACnB,MAAM,IAAI,GAAG,IAAyE,CAAC;QACvF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,gBAAgB;IACC,IAAI,CAAC;IACtB;;OAEG;IACO,cAAc,CAAC,QAAgB;QACrC,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACtB,WAAW;YACX,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACf,MAAM;YACN,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACvD,CAAC;YACD,OAAO;QACX,CAAC;QACD,OAAO;QACP,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ;YAAE,OAAO;QAE7D,UAAU;QACV,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACvD,CAAC;QAED,UAAU;QACV,IAAI,QAAQ,GAAG,UAAU;YAAE,QAAQ,GAAG,UAAU,CAAC;QACjD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IACD,gBAAgB;IACR,WAAW,CAAC,IAAY;QAC5B,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,aAAa;YACb,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;YACxB,IAAI,CAAC,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,aAAa;IACb,MAAM,CAAC,KAAc;QACjB,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACvB,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;CACJ"}
|
package/dist/utils.d.ts
CHANGED
package/dist/utils.js
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
/** 支持的数据转为 Uint8Array */
|
|
2
|
-
export function toUint8Array(data
|
|
2
|
+
export function toUint8Array(data) {
|
|
3
3
|
if (data == null || typeof data != 'object' || typeof data.byteLength != 'number') {
|
|
4
4
|
throw new TypeError('Invalid data');
|
|
5
5
|
}
|
|
6
6
|
if (!ArrayBuffer.isView(data)) {
|
|
7
7
|
return new Uint8Array(data);
|
|
8
8
|
}
|
|
9
|
-
if (exact && (data.byteOffset !== 0 || data.buffer.byteLength !== data.byteLength)) {
|
|
10
|
-
return new Uint8Array(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
|
|
11
|
-
}
|
|
12
9
|
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
13
10
|
}
|
|
14
11
|
/** 未结束的流 */
|
package/dist/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,MAAM,UAAU,YAAY,CAAC,IAAgB
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,MAAM,UAAU,YAAY,CAAC,IAAgB;IACzC,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,IAAI,OAAO,IAAI,CAAC,UAAU,IAAI,QAAQ,EAAE,CAAC;QAChF,MAAM,IAAI,SAAS,CAAC,cAAc,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;AACzE,CAAC;AAED,YAAY;AACZ,MAAM,OAAO,aAAc,SAAQ,KAAK;IACpC;QACI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC5B,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cloudpss/ubjson",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.11",
|
|
4
4
|
"author": "CloudPSS",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -27,23 +27,14 @@
|
|
|
27
27
|
}
|
|
28
28
|
},
|
|
29
29
|
"exports": {
|
|
30
|
-
".":
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
},
|
|
34
|
-
"./stream": {
|
|
35
|
-
"types": "./dist/stream/index.d.ts",
|
|
36
|
-
"default": "./dist/stream/index.js"
|
|
37
|
-
},
|
|
38
|
-
"./rxjs": {
|
|
39
|
-
"types": "./dist/rxjs/index.d.ts",
|
|
40
|
-
"default": "./dist/rxjs/index.js"
|
|
41
|
-
}
|
|
30
|
+
".": "./dist/index.js",
|
|
31
|
+
"./stream": "./dist/stream/index.js",
|
|
32
|
+
"./rxjs": "./dist/rxjs/index.js"
|
|
42
33
|
},
|
|
43
34
|
"devDependencies": {
|
|
44
|
-
"@msgpack/msgpack": "3.0.0-beta2",
|
|
45
35
|
"@types/lodash": "^4.14.202",
|
|
46
|
-
"
|
|
36
|
+
"lodash": "^4.17.21",
|
|
37
|
+
"ref-impl": "npm:@cloudpss/ubjson@0.5.9"
|
|
47
38
|
},
|
|
48
39
|
"dependencies": {
|
|
49
40
|
"rxjs": "^7.8.1"
|
package/src/common/decoder.ts
CHANGED
|
@@ -11,7 +11,7 @@ export class Decoder {
|
|
|
11
11
|
/** 当前读指针位置 */
|
|
12
12
|
protected offset = 0;
|
|
13
13
|
constructor(data: BinaryData) {
|
|
14
|
-
this.data = toUint8Array(data
|
|
14
|
+
this.data = toUint8Array(data);
|
|
15
15
|
this.view = new DataView(this.data.buffer, this.data.byteOffset, this.data.byteLength);
|
|
16
16
|
}
|
|
17
17
|
|
|
@@ -213,7 +213,6 @@ export class Decoder {
|
|
|
213
213
|
/** 读取一个大于 0 的整数 */
|
|
214
214
|
private readIntLength(): number {
|
|
215
215
|
const marker = this.readMarker();
|
|
216
|
-
if (marker === undefined) this.eof();
|
|
217
216
|
let length;
|
|
218
217
|
switch (marker) {
|
|
219
218
|
case constants.INT8:
|
package/src/common/encoder.ts
CHANGED
|
@@ -11,21 +11,19 @@ export abstract class EncoderBase {
|
|
|
11
11
|
protected buffer!: Uint8Array;
|
|
12
12
|
/** buffer 的 DataView */
|
|
13
13
|
protected view!: DataView;
|
|
14
|
-
|
|
15
|
-
constructor(readonly value: unknown) {}
|
|
16
14
|
/**
|
|
17
15
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
18
16
|
*/
|
|
19
17
|
protected abstract ensureCapacity(capacity: number): void;
|
|
20
18
|
|
|
21
19
|
/** 编码至 ubjson */
|
|
22
|
-
protected writeValue(): void {
|
|
23
|
-
if (
|
|
20
|
+
protected writeValue(value: unknown): void {
|
|
21
|
+
if (value === undefined) {
|
|
24
22
|
this.ensureCapacity(1);
|
|
25
23
|
this.buffer[this.length++] = constants.NO_OP;
|
|
26
24
|
return;
|
|
27
25
|
}
|
|
28
|
-
this.write(
|
|
26
|
+
this.write(value);
|
|
29
27
|
}
|
|
30
28
|
/** 写入一个对象 */
|
|
31
29
|
private write(value: unknown): void {
|
package/src/encoder.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { EncoderBase } from './common/encoder.js';
|
|
2
2
|
|
|
3
|
-
const BLOCK_SIZE = 1024 * 16; //
|
|
3
|
+
const BLOCK_SIZE = 1024 * 16; // 16 KiB
|
|
4
4
|
const MAX_SIZE = 1024 * 1024 * 128; //128 MiB
|
|
5
5
|
|
|
6
6
|
/** 编码至 ubjson */
|
|
7
7
|
export class Encoder extends EncoderBase {
|
|
8
8
|
private readonly flushedBuffers: Uint8Array[] = [];
|
|
9
|
+
/** 通过内存池减少分配 */
|
|
10
|
+
private readonly pool = new Uint8Array(BLOCK_SIZE);
|
|
9
11
|
/**
|
|
10
12
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
11
13
|
*/
|
|
@@ -14,43 +16,49 @@ export class Encoder extends EncoderBase {
|
|
|
14
16
|
// 超过最大尺寸限制
|
|
15
17
|
throw new Error('Buffer has exceed max size');
|
|
16
18
|
}
|
|
17
|
-
if (this.buffer == null) {
|
|
18
|
-
this.buffer = new Uint8Array(capacity);
|
|
19
|
-
this.view = new DataView(this.buffer.buffer);
|
|
20
|
-
return;
|
|
21
|
-
}
|
|
22
19
|
// 无需扩容
|
|
23
20
|
if (this.buffer.byteLength >= this.length + capacity) return;
|
|
24
21
|
|
|
25
22
|
// 提交目前的数据
|
|
26
|
-
|
|
23
|
+
if (this.buffer === this.pool) {
|
|
24
|
+
this.flushedBuffers.push(this.buffer.slice(0, this.length));
|
|
25
|
+
} else {
|
|
26
|
+
this.flushedBuffers.push(this.buffer.subarray(0, this.length));
|
|
27
|
+
}
|
|
27
28
|
|
|
28
29
|
// 重新分配缓冲区
|
|
29
30
|
if (capacity < BLOCK_SIZE) capacity = BLOCK_SIZE;
|
|
30
|
-
this.
|
|
31
|
+
this.allocUnsafe(capacity);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** 分配 buffer */
|
|
35
|
+
private allocUnsafe(size: number): void {
|
|
36
|
+
if (size === this.pool.byteLength) {
|
|
37
|
+
// 从 pool 中获取
|
|
38
|
+
this.buffer = this.pool;
|
|
39
|
+
this.view = new DataView(this.buffer.buffer);
|
|
40
|
+
this.length = 0;
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
this.buffer = new Uint8Array(size);
|
|
31
44
|
this.view = new DataView(this.buffer.buffer);
|
|
32
45
|
this.length = 0;
|
|
33
46
|
}
|
|
34
47
|
|
|
35
48
|
/** 获取写入结果 */
|
|
36
|
-
encode(): Uint8Array {
|
|
37
|
-
this.
|
|
38
|
-
this.writeValue();
|
|
49
|
+
encode(value: unknown): Uint8Array {
|
|
50
|
+
this.allocUnsafe(BLOCK_SIZE);
|
|
51
|
+
this.writeValue(value);
|
|
39
52
|
if (this.flushedBuffers.length === 0) {
|
|
40
|
-
|
|
41
|
-
// 无需再拷贝一次
|
|
42
|
-
return this.buffer;
|
|
43
|
-
} else {
|
|
44
|
-
return this.buffer.slice(0, this.length);
|
|
45
|
-
}
|
|
53
|
+
return this.buffer.slice(0, this.length);
|
|
46
54
|
}
|
|
47
55
|
|
|
48
56
|
// 合并缓冲区
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
);
|
|
57
|
+
const flushedBuffers = this.flushedBuffers.splice(0);
|
|
58
|
+
const size = flushedBuffers.reduce((sum, buffer) => sum + buffer.byteLength, this.length);
|
|
59
|
+
const result = new Uint8Array(size);
|
|
52
60
|
let offset = 0;
|
|
53
|
-
for (const buffer of
|
|
61
|
+
for (const buffer of flushedBuffers) {
|
|
54
62
|
result.set(buffer, offset);
|
|
55
63
|
offset += buffer.byteLength;
|
|
56
64
|
}
|
|
@@ -58,3 +66,18 @@ export class Encoder extends EncoderBase {
|
|
|
58
66
|
return result;
|
|
59
67
|
}
|
|
60
68
|
}
|
|
69
|
+
|
|
70
|
+
let _ENCODER: Encoder | undefined;
|
|
71
|
+
|
|
72
|
+
/** 获取默认的编码器 */
|
|
73
|
+
export function getEncoder(): Encoder {
|
|
74
|
+
if (_ENCODER == null) {
|
|
75
|
+
_ENCODER = new Encoder();
|
|
76
|
+
}
|
|
77
|
+
return _ENCODER;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** 重置编码器, For testing only */
|
|
81
|
+
export function resetEncoder(): void {
|
|
82
|
+
_ENCODER = undefined;
|
|
83
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getEncoder } from './encoder.js';
|
|
2
2
|
import { Decoder } from './decoder.js';
|
|
3
3
|
export { UnexpectedEof } from './utils.js';
|
|
4
4
|
|
|
5
5
|
/** 编码为 UBJSON */
|
|
6
6
|
export function encode(value: unknown): Uint8Array {
|
|
7
|
-
|
|
8
|
-
return encoder.encode();
|
|
7
|
+
return getEncoder().encode(value);
|
|
9
8
|
}
|
|
10
9
|
|
|
11
10
|
/** 解码 UBJSON */
|
package/src/rxjs/decoder.ts
CHANGED
|
@@ -15,13 +15,17 @@ export function decode(): OperatorFunction<BinaryData, unknown> {
|
|
|
15
15
|
let end = 0;
|
|
16
16
|
return observable.subscribe({
|
|
17
17
|
next(value) {
|
|
18
|
-
const data = toUint8Array(value
|
|
18
|
+
const data = toUint8Array(value);
|
|
19
19
|
if (buffer.length >= end + data.length) {
|
|
20
20
|
buffer.set(data, end);
|
|
21
21
|
end += data.length;
|
|
22
22
|
} else {
|
|
23
23
|
chunkSize = Math.max(chunkSize, data.length * 2 + end - begin);
|
|
24
|
-
const
|
|
24
|
+
const allocSize = Math.max(chunkSize, data.length + end - begin);
|
|
25
|
+
const newBuffer =
|
|
26
|
+
allocSize > buffer.byteLength
|
|
27
|
+
? new Uint8Array(Math.max(chunkSize, data.length + end - begin))
|
|
28
|
+
: buffer;
|
|
25
29
|
newBuffer.set(buffer.subarray(begin, end), 0);
|
|
26
30
|
newBuffer.set(data, end - begin);
|
|
27
31
|
buffer = newBuffer;
|
package/src/rxjs/encoder.ts
CHANGED
|
@@ -5,12 +5,11 @@ import { StreamEncoderHelper } from '../stream-helper/encoder.js';
|
|
|
5
5
|
export function encode(): OperatorFunction<unknown, Uint8Array> {
|
|
6
6
|
return (observable) => {
|
|
7
7
|
return new Observable<Uint8Array>((subscriber) => {
|
|
8
|
-
const
|
|
9
|
-
|
|
8
|
+
const helper = new StreamEncoderHelper((chunk: Uint8Array): void => subscriber.next(chunk));
|
|
9
|
+
const sub = observable.subscribe({
|
|
10
10
|
next(value) {
|
|
11
11
|
try {
|
|
12
|
-
|
|
13
|
-
helper.encode();
|
|
12
|
+
helper.encode(value);
|
|
14
13
|
} catch (ex) {
|
|
15
14
|
subscriber.error(ex as Error);
|
|
16
15
|
}
|
|
@@ -22,6 +21,8 @@ export function encode(): OperatorFunction<unknown, Uint8Array> {
|
|
|
22
21
|
subscriber.complete();
|
|
23
22
|
},
|
|
24
23
|
});
|
|
24
|
+
sub.add(() => helper.destroy());
|
|
25
|
+
return sub;
|
|
25
26
|
});
|
|
26
27
|
};
|
|
27
28
|
}
|
package/src/stream/encoder.ts
CHANGED
|
@@ -8,16 +8,24 @@ export class StreamEncoder extends Transform {
|
|
|
8
8
|
readableObjectMode: false,
|
|
9
9
|
writableObjectMode: true,
|
|
10
10
|
});
|
|
11
|
+
this.helper = new StreamEncoderHelper((binary) => this.push(binary));
|
|
11
12
|
}
|
|
12
13
|
|
|
14
|
+
private readonly helper;
|
|
15
|
+
|
|
13
16
|
/** @inheritdoc */
|
|
14
17
|
override _transform(obj: unknown, _encoding: BufferEncoding, callback: TransformCallback): void {
|
|
15
|
-
const helper = new StreamEncoderHelper(obj, (binary) => this.push(binary));
|
|
16
18
|
try {
|
|
17
|
-
helper.encode();
|
|
19
|
+
this.helper.encode(obj);
|
|
18
20
|
callback();
|
|
19
21
|
} catch (ex) {
|
|
20
22
|
callback(ex as Error);
|
|
21
23
|
}
|
|
22
24
|
}
|
|
25
|
+
|
|
26
|
+
/** @inheritdoc */
|
|
27
|
+
override _destroy(error: Error | null, callback: (error?: Error | null | undefined) => void): void {
|
|
28
|
+
this.helper.destroy();
|
|
29
|
+
super._destroy(error, callback);
|
|
30
|
+
}
|
|
23
31
|
}
|
|
@@ -3,14 +3,31 @@ import { EncoderBase } from '../common/encoder.js';
|
|
|
3
3
|
const BLOCK_SIZE = 1024 * 8; // 8 KiB
|
|
4
4
|
const MAX_SIZE = 1024 * 1024 * 256; // 256 MiB
|
|
5
5
|
|
|
6
|
+
/** 保存一个内存池以减少重复分配 */
|
|
7
|
+
let POOL: Uint8Array | null = null;
|
|
8
|
+
|
|
6
9
|
/** 流式编码 UBJSON */
|
|
7
10
|
export class StreamEncoderHelper extends EncoderBase {
|
|
8
|
-
constructor(
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
constructor(protected readonly onChunk: (chunk: Uint8Array) => void) {
|
|
12
|
+
super();
|
|
13
|
+
if (POOL != null) {
|
|
14
|
+
this.pool = POOL;
|
|
15
|
+
POOL = null;
|
|
16
|
+
} else {
|
|
17
|
+
this.pool = new Uint8Array(BLOCK_SIZE);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* 销毁实例,释放内存池
|
|
22
|
+
*/
|
|
23
|
+
destroy(): void {
|
|
24
|
+
POOL ??= this.pool;
|
|
25
|
+
const self = this as unknown as { pool: Uint8Array | null; buffer: Uint8Array | null };
|
|
26
|
+
self.pool = null;
|
|
27
|
+
self.buffer = null;
|
|
13
28
|
}
|
|
29
|
+
/** 通过内存池减少分配 */
|
|
30
|
+
private readonly pool;
|
|
14
31
|
/**
|
|
15
32
|
* 确保 buffer 还有 capacity 的空闲空间
|
|
16
33
|
*/
|
|
@@ -19,41 +36,46 @@ export class StreamEncoderHelper extends EncoderBase {
|
|
|
19
36
|
// 超过最大尺寸限制
|
|
20
37
|
throw new Error('Buffer has exceed max size');
|
|
21
38
|
}
|
|
22
|
-
if (this.buffer == null) {
|
|
23
|
-
this.buffer = new Uint8Array(capacity);
|
|
24
|
-
this.view = new DataView(this.buffer.buffer);
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
39
|
if (capacity < 0) {
|
|
28
40
|
// 结束流
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
41
|
+
if (this.buffer === this.pool) {
|
|
42
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
43
|
+
} else {
|
|
44
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
45
|
+
}
|
|
33
46
|
return;
|
|
34
47
|
}
|
|
35
48
|
// 无需扩容
|
|
36
49
|
if (this.buffer.byteLength >= this.length + capacity) return;
|
|
37
50
|
|
|
38
51
|
// 提交目前的数据
|
|
39
|
-
|
|
52
|
+
if (this.buffer === this.pool) {
|
|
53
|
+
this.onChunk(this.buffer.slice(0, this.length));
|
|
54
|
+
} else {
|
|
55
|
+
this.onChunk(this.buffer.subarray(0, this.length));
|
|
56
|
+
}
|
|
40
57
|
|
|
41
58
|
// 重新分配缓冲区
|
|
42
59
|
if (capacity < BLOCK_SIZE) capacity = BLOCK_SIZE;
|
|
43
|
-
this.
|
|
60
|
+
this.allocUnsafe(capacity);
|
|
61
|
+
}
|
|
62
|
+
/** 分配 buffer */
|
|
63
|
+
private allocUnsafe(size: number): void {
|
|
64
|
+
if (size === this.pool.byteLength) {
|
|
65
|
+
// 从 pool 中获取
|
|
66
|
+
this.buffer = this.pool;
|
|
67
|
+
this.view = new DataView(this.buffer.buffer);
|
|
68
|
+
this.length = 0;
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
this.buffer = new Uint8Array(size);
|
|
44
72
|
this.view = new DataView(this.buffer.buffer);
|
|
45
73
|
this.length = 0;
|
|
46
74
|
}
|
|
47
75
|
/** 获取写入结果 */
|
|
48
|
-
encode(): void {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
this.ensureCapacity(-1);
|
|
53
|
-
} else {
|
|
54
|
-
this.ensureCapacity(BLOCK_SIZE);
|
|
55
|
-
this.writeValue();
|
|
56
|
-
this.ensureCapacity(-1);
|
|
57
|
-
}
|
|
76
|
+
encode(value: unknown): void {
|
|
77
|
+
this.allocUnsafe(BLOCK_SIZE);
|
|
78
|
+
this.writeValue(value);
|
|
79
|
+
this.ensureCapacity(-1);
|
|
58
80
|
}
|
|
59
81
|
}
|
package/src/utils.ts
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
/** 支持的数据转为 Uint8Array */
|
|
2
|
-
export function toUint8Array(data: BinaryData
|
|
2
|
+
export function toUint8Array(data: BinaryData): Uint8Array {
|
|
3
3
|
if (data == null || typeof data != 'object' || typeof data.byteLength != 'number') {
|
|
4
4
|
throw new TypeError('Invalid data');
|
|
5
5
|
}
|
|
6
6
|
if (!ArrayBuffer.isView(data)) {
|
|
7
7
|
return new Uint8Array(data);
|
|
8
8
|
}
|
|
9
|
-
if (exact && (data.byteOffset !== 0 || data.buffer.byteLength !== data.byteLength)) {
|
|
10
|
-
return new Uint8Array(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
|
|
11
|
-
}
|
|
12
9
|
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
13
10
|
}
|
|
14
11
|
|
package/tests/e2e.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Tests from https://bitbucket.org/shelacek/ubjson
|
|
3
3
|
*/
|
|
4
4
|
import { encode, decode } from '../dist/index.js';
|
|
5
|
+
import { resetEncoder } from '../dist/encoder.js';
|
|
5
6
|
|
|
6
7
|
test('encode/decode complex object', () => {
|
|
7
8
|
const expected = {
|
|
@@ -301,6 +302,7 @@ test('encode/decode complex object (no encodeInto)', () => {
|
|
|
301
302
|
const encodeInto = TextEncoder.prototype.encodeInto;
|
|
302
303
|
// @ts-expect-error 移除 encodeInto 以测试兼容性
|
|
303
304
|
TextEncoder.prototype.encodeInto = undefined;
|
|
305
|
+
resetEncoder();
|
|
304
306
|
const expected = {
|
|
305
307
|
hello: 'world',
|
|
306
308
|
from: ['UBJSON'],
|
|
@@ -372,6 +374,7 @@ test('encode/decode complex object (no encodeInto)', () => {
|
|
|
372
374
|
const actual = decode(encoded);
|
|
373
375
|
expect(actual).toEqual(expected);
|
|
374
376
|
TextEncoder.prototype.encodeInto = encodeInto;
|
|
377
|
+
resetEncoder();
|
|
375
378
|
});
|
|
376
379
|
|
|
377
380
|
test('encode/decode large array', () => {
|