@lodestar/beacon-node 1.38.0-dev.1f2a3a4524 → 1.38.0-dev.34d4b7352f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoding.d.ts","sourceRoot":"","sources":["../../../src/network/gossip/encoding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"encoding.d.ts","sourceRoot":"","sources":["../../../src/network/gossip/encoding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI1C,OAAO,EAAC,GAAG,EAAC,MAAM,qCAAqC,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,mCAAmC,CAAC;AAKhE,OAAO,EAAC,oBAAoB,EAAC,MAAM,cAAc,CAAC;AAClD,OAAO,EAAC,gBAAgB,EAAmB,MAAM,YAAY,CAAC;AAe9D;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,GAAG,MAAM,CAKvD;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAItD;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,GAAG,EAAE,OAAO,GAAG,UAAU,CA0BpF;AAED,qBAAa,mBAAoB,YAAW,aAAa;IAErD,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAFP,gBAAgB,EAAE,gBAAgB,EAClC,iBAAiB,EAAE,MAAM,EACzB,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAGvD;;;;;OAKG;IACH,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,UAAU;IAyBhE;;;OAGG;IACH,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,UAAU;CAYlE"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// snappyjs is better for compression for smaller payloads
|
|
2
|
-
import { compress, uncompress } from "snappyjs";
|
|
3
2
|
import xxhashFactory from "xxhash-wasm";
|
|
4
3
|
import { digest } from "@chainsafe/as-sha256";
|
|
4
|
+
import snappyWasm from "@chainsafe/snappy-wasm";
|
|
5
5
|
import { ForkName } from "@lodestar/params";
|
|
6
6
|
import { intToBytes } from "@lodestar/utils";
|
|
7
7
|
import { MESSAGE_DOMAIN_VALID_SNAPPY } from "./constants.js";
|
|
@@ -10,6 +10,9 @@ import { getGossipSSZType } from "./topic.js";
|
|
|
10
10
|
const xxhash = await xxhashFactory();
|
|
11
11
|
// Use salt to prevent msgId from being mined for collisions
|
|
12
12
|
const h64Seed = BigInt(Math.floor(Math.random() * 1e9));
|
|
13
|
+
// create singleton snappy encoder + decoder
|
|
14
|
+
const encoder = new snappyWasm.Encoder();
|
|
15
|
+
const decoder = new snappyWasm.Decoder();
|
|
13
16
|
// Shared buffer to convert msgId to string
|
|
14
17
|
const sharedMsgIdBuf = Buffer.alloc(20);
|
|
15
18
|
/**
|
|
@@ -71,10 +74,11 @@ export class DataTransformSnappy {
|
|
|
71
74
|
* - `outboundTransform()`: compress snappy payload
|
|
72
75
|
*/
|
|
73
76
|
inboundTransform(topicStr, data) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
77
|
+
// check uncompressed data length before we actually decompress
|
|
78
|
+
const uncompressedDataLength = snappyWasm.decompress_len(data);
|
|
79
|
+
if (uncompressedDataLength > this.maxSizePerMessage) {
|
|
80
|
+
throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${this.maxSizePerMessage}`);
|
|
81
|
+
}
|
|
78
82
|
const topic = this.gossipTopicCache.getTopic(topicStr);
|
|
79
83
|
const sszType = getGossipSSZType(topic);
|
|
80
84
|
this.metrics?.dataTransform.inbound.inc({ type: topic.type });
|
|
@@ -84,6 +88,10 @@ export class DataTransformSnappy {
|
|
|
84
88
|
if (uncompressedDataLength > sszType.maxSize) {
|
|
85
89
|
throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`);
|
|
86
90
|
}
|
|
91
|
+
// Only after sanity length checks, we can decompress the data
|
|
92
|
+
// Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
|
|
93
|
+
const uncompressedData = Buffer.alloc(uncompressedDataLength);
|
|
94
|
+
decoder.decompress_into(data, uncompressedData);
|
|
87
95
|
return uncompressedData;
|
|
88
96
|
}
|
|
89
97
|
/**
|
|
@@ -96,8 +104,10 @@ export class DataTransformSnappy {
|
|
|
96
104
|
if (data.length > this.maxSizePerMessage) {
|
|
97
105
|
throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`);
|
|
98
106
|
}
|
|
99
|
-
//
|
|
100
|
-
|
|
107
|
+
// Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
|
|
108
|
+
const compressedData = Buffer.alloc(snappyWasm.max_compress_len(data.length));
|
|
109
|
+
const compressedLen = encoder.compress_into(data, compressedData);
|
|
110
|
+
return compressedData.subarray(0, compressedLen);
|
|
101
111
|
}
|
|
102
112
|
}
|
|
103
113
|
//# sourceMappingURL=encoding.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoding.js","sourceRoot":"","sources":["../../../src/network/gossip/encoding.ts"],"names":[],"mappings":"AACA,0DAA0D;AAC1D,OAAO,
|
|
1
|
+
{"version":3,"file":"encoding.js","sourceRoot":"","sources":["../../../src/network/gossip/encoding.ts"],"names":[],"mappings":"AACA,0DAA0D;AAC1D,OAAO,aAAa,MAAM,aAAa,CAAC;AACxC,OAAO,EAAC,MAAM,EAAC,MAAM,sBAAsB,CAAC;AAG5C,OAAO,UAAU,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAC,2BAA2B,EAAC,MAAM,gBAAgB,CAAC;AAE3D,OAAO,EAAmB,gBAAgB,EAAC,MAAM,YAAY,CAAC;AAE9D,YAAY;AACZ,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAC;AAErC,4DAA4D;AAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;AAExD,4CAA4C;AAC5C,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;AACzC,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;AAEzC,2CAA2C;AAC3C,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAExC;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,MAAmB;IAC7C,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB;IAC5C,mEAAmE;IACnE,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1B,OAAO,KAAK,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,OAAO,CAAC,gBAAkC,EAAE,GAAY;IACtE,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAEnD,IAAI,GAAiB,CAAC;IAEtB,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC5C,yBAAyB;QACzB,MAAM;QACN,6EAA6E;QAC7E,MAAM;QACN,GAAG,GAAG,CAAC,2BAA2B,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;SAAM,CAAC;QACN,qDAAqD;QACrD,MAAM;QACN,UAAU;QACV,kCAAkC;QAClC,gDAAgD;QAChD,oBAAoB;QACpB,oCAAoC;QACpC,SAAS;QACT,MAAM;QACN,iHAAiH;QACjH,GAAG,GAAG,CAAC,2BAA2B,EAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,OAAO,mBAAmB;IAEX;IACA;IACA;IAHnB,YACmB,gBAAkC,EAClC,iBAAyB,EACzB,OAAoC;QAFpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,sBAAiB,GAAjB,iBAAiB,CAAQ;QACzB,YAAO,GAAP,OAAO,CAA6B;IACpD,CAAC;IAEJ;;;;;OAKG;IACH,gBAAgB,CAAC,QAAgB,EAAE,IAAgB;QACjD,+DAA+D;QAC/D,MAAM,sBAAsB,GAAG,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,sBAAsB,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACpD,MAAM,KAAK,CAAC,kCAAkC,sBAAsB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;QACtG,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAC,CAAC,CAAC;QAE5D,IAAI,sBAAsB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAC7C,MAAM,KAAK,CAAC,kCAAkC,sBAAsB,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,sBAAsB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAC7C,MAAM,KAAK,CAAC,kCAAkC,sBAAsB,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,8DAA8D;QAC9D,kHAAkH;QAClH,MAAM,gBAAgB,GAAG,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC9D,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAChD,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,iBAAiB,CAAC,QAAgB,EAAE,IAAgB;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAC,CAAC,CAAC;QAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC,kCAAkC,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,kHAAkH;QAClH,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9E,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QAClE,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IACnD,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"bugs": {
|
|
12
12
|
"url": "https://github.com/ChainSafe/lodestar/issues"
|
|
13
13
|
},
|
|
14
|
-
"version": "1.38.0-dev.
|
|
14
|
+
"version": "1.38.0-dev.34d4b7352f",
|
|
15
15
|
"type": "module",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": {
|
|
@@ -123,6 +123,7 @@
|
|
|
123
123
|
"@chainsafe/persistent-merkle-tree": "^1.2.1",
|
|
124
124
|
"@chainsafe/prometheus-gc-stats": "^1.0.0",
|
|
125
125
|
"@chainsafe/pubkey-index-map": "^3.0.0",
|
|
126
|
+
"@chainsafe/snappy-wasm": "^0.5.0",
|
|
126
127
|
"@chainsafe/ssz": "^1.2.2",
|
|
127
128
|
"@chainsafe/threads": "^1.11.3",
|
|
128
129
|
"@crate-crypto/node-eth-kzg": "0.9.1",
|
|
@@ -140,18 +141,18 @@
|
|
|
140
141
|
"@libp2p/peer-id": "^5.1.0",
|
|
141
142
|
"@libp2p/prometheus-metrics": "^4.3.15",
|
|
142
143
|
"@libp2p/tcp": "^10.1.8",
|
|
143
|
-
"@lodestar/api": "1.38.0-dev.
|
|
144
|
-
"@lodestar/config": "1.38.0-dev.
|
|
145
|
-
"@lodestar/db": "1.38.0-dev.
|
|
146
|
-
"@lodestar/fork-choice": "1.38.0-dev.
|
|
147
|
-
"@lodestar/light-client": "1.38.0-dev.
|
|
148
|
-
"@lodestar/logger": "1.38.0-dev.
|
|
149
|
-
"@lodestar/params": "1.38.0-dev.
|
|
150
|
-
"@lodestar/reqresp": "1.38.0-dev.
|
|
151
|
-
"@lodestar/state-transition": "1.38.0-dev.
|
|
152
|
-
"@lodestar/types": "1.38.0-dev.
|
|
153
|
-
"@lodestar/utils": "1.38.0-dev.
|
|
154
|
-
"@lodestar/validator": "1.38.0-dev.
|
|
144
|
+
"@lodestar/api": "1.38.0-dev.34d4b7352f",
|
|
145
|
+
"@lodestar/config": "1.38.0-dev.34d4b7352f",
|
|
146
|
+
"@lodestar/db": "1.38.0-dev.34d4b7352f",
|
|
147
|
+
"@lodestar/fork-choice": "1.38.0-dev.34d4b7352f",
|
|
148
|
+
"@lodestar/light-client": "1.38.0-dev.34d4b7352f",
|
|
149
|
+
"@lodestar/logger": "1.38.0-dev.34d4b7352f",
|
|
150
|
+
"@lodestar/params": "1.38.0-dev.34d4b7352f",
|
|
151
|
+
"@lodestar/reqresp": "1.38.0-dev.34d4b7352f",
|
|
152
|
+
"@lodestar/state-transition": "1.38.0-dev.34d4b7352f",
|
|
153
|
+
"@lodestar/types": "1.38.0-dev.34d4b7352f",
|
|
154
|
+
"@lodestar/utils": "1.38.0-dev.34d4b7352f",
|
|
155
|
+
"@lodestar/validator": "1.38.0-dev.34d4b7352f",
|
|
155
156
|
"@multiformats/multiaddr": "^12.1.3",
|
|
156
157
|
"datastore-core": "^10.0.2",
|
|
157
158
|
"datastore-fs": "^10.0.6",
|
|
@@ -166,7 +167,6 @@
|
|
|
166
167
|
"multiformats": "^11.0.1",
|
|
167
168
|
"prom-client": "^15.1.0",
|
|
168
169
|
"qs": "^6.11.1",
|
|
169
|
-
"snappyjs": "^0.7.0",
|
|
170
170
|
"strict-event-emitter-types": "^2.0.0",
|
|
171
171
|
"systeminformation": "^5.22.9",
|
|
172
172
|
"uint8arraylist": "^2.4.7",
|
|
@@ -187,5 +187,5 @@
|
|
|
187
187
|
"beacon",
|
|
188
188
|
"blockchain"
|
|
189
189
|
],
|
|
190
|
-
"gitHead": "
|
|
190
|
+
"gitHead": "d5fc6ada940df2cf5fe42c2a1b688f9ed5bcf735"
|
|
191
191
|
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {Message} from "@libp2p/interface";
|
|
2
2
|
// snappyjs is better for compression for smaller payloads
|
|
3
|
-
import {compress, uncompress} from "snappyjs";
|
|
4
3
|
import xxhashFactory from "xxhash-wasm";
|
|
5
4
|
import {digest} from "@chainsafe/as-sha256";
|
|
6
5
|
import {RPC} from "@chainsafe/libp2p-gossipsub/message";
|
|
7
6
|
import {DataTransform} from "@chainsafe/libp2p-gossipsub/types";
|
|
7
|
+
import snappyWasm from "@chainsafe/snappy-wasm";
|
|
8
8
|
import {ForkName} from "@lodestar/params";
|
|
9
9
|
import {intToBytes} from "@lodestar/utils";
|
|
10
10
|
import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
|
|
@@ -17,6 +17,10 @@ const xxhash = await xxhashFactory();
|
|
|
17
17
|
// Use salt to prevent msgId from being mined for collisions
|
|
18
18
|
const h64Seed = BigInt(Math.floor(Math.random() * 1e9));
|
|
19
19
|
|
|
20
|
+
// create singleton snappy encoder + decoder
|
|
21
|
+
const encoder = new snappyWasm.Encoder();
|
|
22
|
+
const decoder = new snappyWasm.Decoder();
|
|
23
|
+
|
|
20
24
|
// Shared buffer to convert msgId to string
|
|
21
25
|
const sharedMsgIdBuf = Buffer.alloc(20);
|
|
22
26
|
|
|
@@ -82,11 +86,12 @@ export class DataTransformSnappy implements DataTransform {
|
|
|
82
86
|
* - `outboundTransform()`: compress snappy payload
|
|
83
87
|
*/
|
|
84
88
|
inboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
|
|
85
|
-
|
|
89
|
+
// check uncompressed data length before we actually decompress
|
|
90
|
+
const uncompressedDataLength = snappyWasm.decompress_len(data);
|
|
91
|
+
if (uncompressedDataLength > this.maxSizePerMessage) {
|
|
92
|
+
throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${this.maxSizePerMessage}`);
|
|
93
|
+
}
|
|
86
94
|
|
|
87
|
-
// check uncompressed data length before we extract beacon block root, slot or
|
|
88
|
-
// attestation data at later steps
|
|
89
|
-
const uncompressedDataLength = uncompressedData.length;
|
|
90
95
|
const topic = this.gossipTopicCache.getTopic(topicStr);
|
|
91
96
|
const sszType = getGossipSSZType(topic);
|
|
92
97
|
this.metrics?.dataTransform.inbound.inc({type: topic.type});
|
|
@@ -98,6 +103,10 @@ export class DataTransformSnappy implements DataTransform {
|
|
|
98
103
|
throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`);
|
|
99
104
|
}
|
|
100
105
|
|
|
106
|
+
// Only after sanity length checks, we can decompress the data
|
|
107
|
+
// Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
|
|
108
|
+
const uncompressedData = Buffer.alloc(uncompressedDataLength);
|
|
109
|
+
decoder.decompress_into(data, uncompressedData);
|
|
101
110
|
return uncompressedData;
|
|
102
111
|
}
|
|
103
112
|
|
|
@@ -111,7 +120,10 @@ export class DataTransformSnappy implements DataTransform {
|
|
|
111
120
|
if (data.length > this.maxSizePerMessage) {
|
|
112
121
|
throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`);
|
|
113
122
|
}
|
|
114
|
-
|
|
115
|
-
|
|
123
|
+
|
|
124
|
+
// Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
|
|
125
|
+
const compressedData = Buffer.alloc(snappyWasm.max_compress_len(data.length));
|
|
126
|
+
const compressedLen = encoder.compress_into(data, compressedData);
|
|
127
|
+
return compressedData.subarray(0, compressedLen);
|
|
116
128
|
}
|
|
117
129
|
}
|