@shelby-protocol/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/index.d.ts +9 -9
- package/dist/browser/index.mjs +1068 -589
- package/dist/core/clients/ShelbyBlobClient.d.ts +155 -40
- package/dist/core/clients/ShelbyBlobClient.mjs +314 -112
- package/dist/core/clients/ShelbyClient.d.ts +27 -4
- package/dist/core/clients/ShelbyClient.mjs +869 -551
- package/dist/core/clients/ShelbyClientConfig.d.ts +9 -1
- package/dist/core/clients/ShelbyMetadataClient.d.ts +38 -6
- package/dist/core/clients/ShelbyMetadataClient.mjs +95 -13
- package/dist/core/clients/ShelbyMicropaymentChannelClient.d.ts +42 -0
- package/dist/core/clients/ShelbyMicropaymentChannelClient.mjs +62 -7
- package/dist/core/clients/ShelbyPlacementGroupClient.mjs +10 -7
- package/dist/core/clients/ShelbyRPCClient.d.ts +55 -66
- package/dist/core/clients/ShelbyRPCClient.mjs +344 -360
- package/dist/core/clients/index.d.ts +3 -2
- package/dist/core/clients/index.mjs +933 -551
- package/dist/core/clients/utils.mjs +1 -1
- package/dist/core/commitments.d.ts +12 -1
- package/dist/core/commitments.mjs +7 -0
- package/dist/core/constants.d.ts +1 -1
- package/dist/core/constants.mjs +1 -1
- package/dist/core/erasure/constants.d.ts +16 -1
- package/dist/core/erasure/constants.mjs +15 -1
- package/dist/core/erasure/index.d.ts +1 -1
- package/dist/core/erasure/index.mjs +15 -1
- package/dist/core/errors.d.ts +20 -1
- package/dist/core/errors.mjs +29 -0
- package/dist/core/index.d.ts +9 -9
- package/dist/core/index.mjs +1068 -589
- package/dist/core/operations/generated/sdk.d.ts +798 -42
- package/dist/core/operations/generated/sdk.mjs +93 -9
- package/dist/core/operations/index.d.ts +1 -1
- package/dist/core/operations/index.mjs +94 -10
- package/dist/core/rpc-responses.d.ts +1 -57
- package/dist/core/rpc-responses.mjs +1 -21
- package/dist/core/sp/chunk_proof.d.ts +23 -0
- package/dist/core/sp/chunk_proof.mjs +113 -0
- package/dist/core/sp/index.d.ts +3 -0
- package/dist/core/sp/index.mjs +402 -0
- package/dist/core/sp/sp_write_client.d.ts +53 -0
- package/dist/core/sp/sp_write_client.mjs +302 -0
- package/dist/core/types/blobs.d.ts +24 -5
- package/dist/core/types/blobs.mjs +24 -0
- package/dist/core/types/index.d.ts +2 -2
- package/dist/core/types/index.mjs +46 -2
- package/dist/core/types/payments.mjs +1 -1
- package/dist/core/types/storage_providers.d.ts +32 -6
- package/dist/core/types/storage_providers.mjs +22 -0
- package/dist/gen/rpc_server_pb.d.ts +295 -0
- package/dist/gen/rpc_server_pb.mjs +28 -0
- package/dist/node/clients/ShelbyNodeClient.d.ts +1 -0
- package/dist/node/clients/ShelbyNodeClient.mjs +869 -551
- package/dist/node/clients/index.d.ts +1 -0
- package/dist/node/clients/index.mjs +867 -551
- package/dist/node/index.d.ts +9 -9
- package/dist/node/index.mjs +1068 -589
- package/dist/node/parallel/commitment_worker.mjs +36 -249
- package/dist/node/parallel/commitment_worker_pool.mjs +56 -3
- package/dist/node/parallel/coverage_flush.d.ts +16 -0
- package/dist/node/parallel/coverage_flush.mjs +43 -0
- package/dist/node/parallel/default_commitment_worker_pool.mjs +56 -3
- package/dist/node/parallel/index.d.ts +1 -0
- package/dist/node/parallel/index.mjs +72 -4
- package/dist/node/parallel/parallel_commitments.mjs +56 -3
- package/dist/node/parallel/worker_pool.mjs +56 -3
- package/package.json +11 -4
|
@@ -1,64 +1,6 @@
|
|
|
1
1
|
// src/node/parallel/commitment_worker.ts
|
|
2
2
|
import { parentPort } from "worker_threads";
|
|
3
|
-
import { Hex as Hex3 } from "@aptos-labs/ts-sdk";
|
|
4
|
-
|
|
5
|
-
// src/core/commitments.ts
|
|
6
3
|
import { Hex as Hex2 } from "@aptos-labs/ts-sdk";
|
|
7
|
-
import { z } from "zod";
|
|
8
|
-
|
|
9
|
-
// src/core/erasure/constants.ts
|
|
10
|
-
var ERASURE_CODE_PARAMS = {
|
|
11
|
-
["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
|
|
12
|
-
// total chunks (data + parity)
|
|
13
|
-
erasure_n: 16,
|
|
14
|
-
// data chunks
|
|
15
|
-
erasure_k: 10,
|
|
16
|
-
// helper nodes
|
|
17
|
-
erasure_d: 13,
|
|
18
|
-
// enum index
|
|
19
|
-
enumIndex: 0
|
|
20
|
-
},
|
|
21
|
-
["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
|
|
22
|
-
// total chunks (data + parity)
|
|
23
|
-
erasure_n: 4,
|
|
24
|
-
// data chunks
|
|
25
|
-
erasure_k: 2,
|
|
26
|
-
// helper nodes
|
|
27
|
-
erasure_d: 3,
|
|
28
|
-
// enum index
|
|
29
|
-
enumIndex: 1
|
|
30
|
-
}
|
|
31
|
-
};
|
|
32
|
-
var DEFAULT_ERASURE_N = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_n;
|
|
33
|
-
var DEFAULT_ERASURE_K = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_k;
|
|
34
|
-
var DEFAULT_ERASURE_D = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_d;
|
|
35
|
-
var DEFAULT_ERASURE_M = DEFAULT_ERASURE_N - DEFAULT_ERASURE_K;
|
|
36
|
-
|
|
37
|
-
// src/core/chunk.ts
|
|
38
|
-
var CHUNK_SIZE_PARAMS = {
|
|
39
|
-
["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */]: {
|
|
40
|
-
// 1MiB
|
|
41
|
-
chunkSizeBytes: 1 * 1024 * 1024,
|
|
42
|
-
// 10MiB
|
|
43
|
-
chunksetSizeBytes: 10 * 1024 * 1024
|
|
44
|
-
},
|
|
45
|
-
["ChunkSet2MiB_Chunk1MiB" /* ChunkSet2MiB_Chunk1MiB */]: {
|
|
46
|
-
// 1MiB
|
|
47
|
-
chunkSizeBytes: 1 * 1024 * 1024,
|
|
48
|
-
// 2MiB
|
|
49
|
-
chunksetSizeBytes: 2 * 1024 * 1024
|
|
50
|
-
}
|
|
51
|
-
};
|
|
52
|
-
var DEFAULT_CHUNK_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunkSizeBytes;
|
|
53
|
-
var DEFAULT_CHUNKSET_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunksetSizeBytes;
|
|
54
|
-
var ERASURE_CODE_AND_CHUNK_MAPPING = {
|
|
55
|
-
["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
|
|
56
|
-
...CHUNK_SIZE_PARAMS.ChunkSet10MiB_Chunk1MiB
|
|
57
|
-
},
|
|
58
|
-
["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
|
|
59
|
-
...CHUNK_SIZE_PARAMS.ChunkSet2MiB_Chunk1MiB
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
4
|
|
|
63
5
|
// src/core/utils.ts
|
|
64
6
|
import {
|
|
@@ -76,211 +18,56 @@ function zeroPadBytes(buffer, desiredLength) {
|
|
|
76
18
|
paddedBuffer.set(buffer);
|
|
77
19
|
return paddedBuffer;
|
|
78
20
|
}
|
|
79
|
-
async function concatHashes(parts) {
|
|
80
|
-
const chunks = parts.map((part) => Hex.fromHexInput(part).toUint8Array());
|
|
81
|
-
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
|
|
82
|
-
const combined = new Uint8Array(totalLength);
|
|
83
|
-
let offset = 0;
|
|
84
|
-
for (const chunk of chunks) {
|
|
85
|
-
combined.set(chunk, offset);
|
|
86
|
-
offset += chunk.byteLength;
|
|
87
|
-
}
|
|
88
|
-
return Hex.fromHexInput(
|
|
89
|
-
new Uint8Array(await crypto.subtle.digest("SHA-256", combined))
|
|
90
|
-
);
|
|
91
|
-
}
|
|
92
21
|
|
|
93
|
-
// src/
|
|
94
|
-
var
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
})
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
{
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
}
|
|
107
|
-
);
|
|
108
|
-
function expectedTotalChunksets(rawSize, chunksetSize = DEFAULT_CHUNKSET_SIZE_BYTES) {
|
|
109
|
-
if (chunksetSize <= 0) {
|
|
110
|
-
throw new Error("chunksetSize must be positive");
|
|
111
|
-
}
|
|
112
|
-
if (rawSize === 0) return 1;
|
|
113
|
-
return Math.ceil(rawSize / chunksetSize);
|
|
114
|
-
}
|
|
115
|
-
var BlobCommitmentsSchema = z.object({
|
|
116
|
-
schema_version: z.string(),
|
|
117
|
-
raw_data_size: z.number(),
|
|
118
|
-
// FIXME I am not sure about this being here, or if it should be somewhere else
|
|
119
|
-
blob_merkle_root: z.string(),
|
|
120
|
-
chunkset_commitments: z.array(ChunksetCommitmentSchema)
|
|
121
|
-
}).refine(
|
|
122
|
-
(data) => {
|
|
123
|
-
return expectedTotalChunksets(data.raw_data_size) === data.chunkset_commitments.length;
|
|
124
|
-
},
|
|
125
|
-
{
|
|
126
|
-
message: "Total chunkset count mismatches with raw data size",
|
|
127
|
-
// FIXME put more details in here
|
|
128
|
-
path: ["chunkset_commitments"]
|
|
129
|
-
}
|
|
130
|
-
);
|
|
131
|
-
async function generateMerkleRoot(leafHashes) {
|
|
132
|
-
if (!leafHashes.length) {
|
|
133
|
-
throw new Error(
|
|
134
|
-
"An empty array cannot be used to construct a Merkle tree."
|
|
135
|
-
);
|
|
136
|
-
}
|
|
137
|
-
const zeroArray = new Uint8Array(leafHashes[0].toUint8Array().length);
|
|
138
|
-
const zeroBytes = Hex2.fromHexInput(zeroArray);
|
|
139
|
-
let currentLeaves = leafHashes;
|
|
140
|
-
while (currentLeaves.length > 1) {
|
|
141
|
-
if (currentLeaves.length % 2 !== 0) {
|
|
142
|
-
currentLeaves.push(zeroBytes);
|
|
143
|
-
}
|
|
144
|
-
const nextLeaves = [];
|
|
145
|
-
for (let i = 0; i < currentLeaves.length; i += 2) {
|
|
146
|
-
nextLeaves.push(
|
|
147
|
-
await concatHashes([
|
|
148
|
-
currentLeaves[i].toUint8Array(),
|
|
149
|
-
currentLeaves[i + 1].toUint8Array()
|
|
150
|
-
])
|
|
151
|
-
);
|
|
152
|
-
}
|
|
153
|
-
currentLeaves = nextLeaves;
|
|
22
|
+
// src/node/parallel/commitment_worker.ts
|
|
23
|
+
var ffiAvailable = null;
|
|
24
|
+
async function tryCreateFFIEncoder(opts) {
|
|
25
|
+
if (ffiAvailable === false) return null;
|
|
26
|
+
try {
|
|
27
|
+
const ffiPath = "@shelby-protocol/clay-codes-ffi";
|
|
28
|
+
const { createEncoder: createFFIEncoder } = await import(ffiPath);
|
|
29
|
+
const encoder = await createFFIEncoder(opts);
|
|
30
|
+
ffiAvailable = true;
|
|
31
|
+
return encoder;
|
|
32
|
+
} catch {
|
|
33
|
+
ffiAvailable = false;
|
|
34
|
+
return null;
|
|
154
35
|
}
|
|
155
|
-
return currentLeaves[0];
|
|
156
36
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
createDecoder,
|
|
161
|
-
createEncoder
|
|
162
|
-
} from "@shelby-protocol/clay-codes";
|
|
163
|
-
function getTotalChunks(config) {
|
|
164
|
-
return config.erasure_n;
|
|
37
|
+
async function createWASMEncoder(opts) {
|
|
38
|
+
const { createEncoder } = await import("@shelby-protocol/clay-codes");
|
|
39
|
+
return createEncoder(opts);
|
|
165
40
|
}
|
|
166
|
-
var ClayErasureCodingProvider = class _ClayErasureCodingProvider {
|
|
167
|
-
config;
|
|
168
|
-
encoderCache;
|
|
169
|
-
decoderCache;
|
|
170
|
-
lastFunction;
|
|
171
|
-
constructor(config) {
|
|
172
|
-
this.config = config;
|
|
173
|
-
this.lastFunction = "none" /* NONE */;
|
|
174
|
-
}
|
|
175
|
-
/**
|
|
176
|
-
* Static factory method to create an initialized ClayErasureCodingProvider
|
|
177
|
-
*/
|
|
178
|
-
static async create(config) {
|
|
179
|
-
const provider = new _ClayErasureCodingProvider(config);
|
|
180
|
-
[provider.encoderCache, provider.decoderCache] = await Promise.all([
|
|
181
|
-
createEncoder({
|
|
182
|
-
n: getTotalChunks(config),
|
|
183
|
-
k: config.erasure_k,
|
|
184
|
-
d: config.erasure_d,
|
|
185
|
-
chunkSizeBytes: config.chunkSizeBytes
|
|
186
|
-
}),
|
|
187
|
-
createDecoder({
|
|
188
|
-
n: getTotalChunks(config),
|
|
189
|
-
k: config.erasure_k,
|
|
190
|
-
d: config.erasure_d,
|
|
191
|
-
chunkSizeBytes: config.chunkSizeBytes,
|
|
192
|
-
erasedChunkIndexes: []
|
|
193
|
-
})
|
|
194
|
-
]);
|
|
195
|
-
return provider;
|
|
196
|
-
}
|
|
197
|
-
encode(data) {
|
|
198
|
-
const { erasure_k, chunkSizeBytes } = this.config;
|
|
199
|
-
if (!this.encoderCache) {
|
|
200
|
-
throw new Error("Encoder cache is missing");
|
|
201
|
-
}
|
|
202
|
-
const systematicCapacity = erasure_k * chunkSizeBytes;
|
|
203
|
-
if (data.byteLength > systematicCapacity) {
|
|
204
|
-
throw new Error(
|
|
205
|
-
`Data size ${data.byteLength} bytes exceeds systematic capacity ${systematicCapacity} bytes (erasure_k=${erasure_k} * chunkSizeBytes=${chunkSizeBytes}).`
|
|
206
|
-
);
|
|
207
|
-
}
|
|
208
|
-
if (this.requiresPadding(data.length)) {
|
|
209
|
-
throw new Error(
|
|
210
|
-
`Data size ${data.length} bytes does not match systematic capacity ${systematicCapacity} bytes (erasure_k=${erasure_k} * chunkSizeBytes=${chunkSizeBytes}). Provide pre-padded data before encoding.`
|
|
211
|
-
);
|
|
212
|
-
}
|
|
213
|
-
this.lastFunction = "encoded" /* ENCODED */;
|
|
214
|
-
return this.encoderCache.erasureCode(data);
|
|
215
|
-
}
|
|
216
|
-
decode(available, config) {
|
|
217
|
-
if (!this.decoderCache) {
|
|
218
|
-
throw new Error("Decoder cache is missing");
|
|
219
|
-
}
|
|
220
|
-
const { erasure_k, chunkSizeBytes } = this.config;
|
|
221
|
-
if (available.length < erasure_k) {
|
|
222
|
-
throw new Error(
|
|
223
|
-
`Insufficient available chunks: need at least ${erasure_k}, received ${available.length}.`
|
|
224
|
-
);
|
|
225
|
-
}
|
|
226
|
-
for (let i = 0; i < available.length; i++) {
|
|
227
|
-
const chunk = available[i];
|
|
228
|
-
if (chunk.length !== chunkSizeBytes) {
|
|
229
|
-
throw new Error(
|
|
230
|
-
`Chunk ${i} size ${chunk.length} bytes does not match expected chunkSizeBytes ${chunkSizeBytes}.`
|
|
231
|
-
);
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
this.lastFunction = "decoded" /* DECODED */;
|
|
235
|
-
return this.decoderCache.decode(available, config);
|
|
236
|
-
}
|
|
237
|
-
getChunkMerkleRoots() {
|
|
238
|
-
if (this.decoderCache && this.lastFunction === "decoded" /* DECODED */)
|
|
239
|
-
return this.decoderCache.getChunkMerkleRoots();
|
|
240
|
-
if (this.encoderCache && this.lastFunction === "encoded" /* ENCODED */)
|
|
241
|
-
return this.encoderCache.getChunkMerkleRoots();
|
|
242
|
-
throw new Error(
|
|
243
|
-
"You must call encode or decode before calling getChunkMerkleRoots"
|
|
244
|
-
);
|
|
245
|
-
}
|
|
246
|
-
/**
|
|
247
|
-
* Determines if data can be erasure coded as-is or requires padding.
|
|
248
|
-
*
|
|
249
|
-
* Data can be erasure coded without padding if its size exactly matches
|
|
250
|
-
* the total systematic data capacity (k * chunkSizeBytes).
|
|
251
|
-
*
|
|
252
|
-
* @param dataSize - Size of the data in bytes
|
|
253
|
-
* @returns true if data needs padding, false if it can be coded as-is
|
|
254
|
-
*/
|
|
255
|
-
requiresPadding(dataSize) {
|
|
256
|
-
const { erasure_k, chunkSizeBytes } = this.config;
|
|
257
|
-
const systematicCapacity = erasure_k * chunkSizeBytes;
|
|
258
|
-
return dataSize !== systematicCapacity;
|
|
259
|
-
}
|
|
260
|
-
};
|
|
261
|
-
|
|
262
|
-
// src/node/parallel/commitment_worker.ts
|
|
263
41
|
async function encodeChunksetForCommitment(chunksetData, encoding) {
|
|
264
42
|
const chunksetSizeBytes = encoding.erasure_k * encoding.chunkSizeBytes;
|
|
265
43
|
const paddedChunkset = zeroPadBytes(chunksetData, chunksetSizeBytes);
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
const
|
|
269
|
-
const
|
|
44
|
+
const encoder = await getEncoder(encoding);
|
|
45
|
+
encoder.erasureCode(paddedChunkset);
|
|
46
|
+
const commitment = encoder.getMerkleCommitment();
|
|
47
|
+
const chunkRoots = commitment.chunkRoots.map(
|
|
48
|
+
(root) => Hex2.fromHexInput(root).toString()
|
|
49
|
+
);
|
|
50
|
+
const chunksetRoot = Hex2.fromHexInput(commitment.chunksetRoot).toString();
|
|
270
51
|
return {
|
|
271
|
-
chunkset_root: chunksetRoot
|
|
272
|
-
chunk_commitments: chunkRoots
|
|
52
|
+
chunkset_root: chunksetRoot,
|
|
53
|
+
chunk_commitments: chunkRoots
|
|
273
54
|
};
|
|
274
55
|
}
|
|
275
|
-
var
|
|
56
|
+
var cachedEncoder;
|
|
276
57
|
var cachedEncodingKey;
|
|
277
|
-
async function
|
|
58
|
+
async function getEncoder(encoding) {
|
|
278
59
|
const key = `${encoding.erasure_n},${encoding.erasure_k},${encoding.erasure_d},${encoding.chunkSizeBytes},${encoding.enumIndex}`;
|
|
279
|
-
if (!
|
|
280
|
-
|
|
60
|
+
if (!cachedEncoder || cachedEncodingKey !== key) {
|
|
61
|
+
const opts = {
|
|
62
|
+
n: encoding.erasure_n,
|
|
63
|
+
k: encoding.erasure_k,
|
|
64
|
+
d: encoding.erasure_d,
|
|
65
|
+
chunkSizeBytes: encoding.chunkSizeBytes
|
|
66
|
+
};
|
|
67
|
+
cachedEncoder = await tryCreateFFIEncoder(opts) ?? await createWASMEncoder(opts);
|
|
281
68
|
cachedEncodingKey = key;
|
|
282
69
|
}
|
|
283
|
-
return
|
|
70
|
+
return cachedEncoder;
|
|
284
71
|
}
|
|
285
72
|
parentPort?.on("message", async (input) => {
|
|
286
73
|
try {
|
|
@@ -3,6 +3,35 @@ import fs from "fs";
|
|
|
3
3
|
import path from "path";
|
|
4
4
|
import { fileURLToPath } from "url";
|
|
5
5
|
import { Worker } from "worker_threads";
|
|
6
|
+
|
|
7
|
+
// src/node/parallel/coverage_flush.ts
|
|
8
|
+
import { takeCoverage } from "v8";
|
|
9
|
+
var FLUSH_REQUEST = "__shelby_cov_flush_request__";
|
|
10
|
+
var FLUSH_ACK = "__shelby_cov_flush_ack__";
|
|
11
|
+
var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
|
|
12
|
+
var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
|
|
13
|
+
function flushWorkerCoverage(worker, timeoutMs = 2e3) {
|
|
14
|
+
if (!coverageEnabled()) return Promise.resolve();
|
|
15
|
+
return new Promise((resolve) => {
|
|
16
|
+
const onMessage = (msg) => {
|
|
17
|
+
if (isType(msg, FLUSH_ACK)) finish();
|
|
18
|
+
};
|
|
19
|
+
const finish = () => {
|
|
20
|
+
clearTimeout(timer);
|
|
21
|
+
worker.off("message", onMessage);
|
|
22
|
+
resolve();
|
|
23
|
+
};
|
|
24
|
+
const timer = setTimeout(finish, timeoutMs);
|
|
25
|
+
worker.on("message", onMessage);
|
|
26
|
+
try {
|
|
27
|
+
worker.postMessage({ type: FLUSH_REQUEST });
|
|
28
|
+
} catch {
|
|
29
|
+
finish();
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// src/node/parallel/worker_pool.ts
|
|
6
35
|
var WorkerPool = class {
|
|
7
36
|
workers = [];
|
|
8
37
|
queue = [];
|
|
@@ -39,6 +68,29 @@ var WorkerPool = class {
|
|
|
39
68
|
const dir = path.dirname(currentPath);
|
|
40
69
|
const ext = path.extname(currentPath);
|
|
41
70
|
const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
|
|
71
|
+
if (workerExt === ".ts") {
|
|
72
|
+
const srcSegment = `${path.sep}src${path.sep}`;
|
|
73
|
+
const srcIdx = currentPath.indexOf(srcSegment);
|
|
74
|
+
if (srcIdx !== -1) {
|
|
75
|
+
const packageRoot = currentPath.slice(0, srcIdx);
|
|
76
|
+
const relFromSrc = path.relative(path.join(packageRoot, "src"), dir);
|
|
77
|
+
return path.join(
|
|
78
|
+
packageRoot,
|
|
79
|
+
"dist",
|
|
80
|
+
"workers",
|
|
81
|
+
relFromSrc,
|
|
82
|
+
`${scriptName}.js`
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
let workerPath = path.join(dir, `${scriptName}${workerExt}`);
|
|
87
|
+
if (fs.existsSync(workerPath)) {
|
|
88
|
+
return workerPath;
|
|
89
|
+
}
|
|
90
|
+
workerPath = path.join(dir, "workers", `${scriptName}${workerExt}`);
|
|
91
|
+
if (fs.existsSync(workerPath)) {
|
|
92
|
+
return workerPath;
|
|
93
|
+
}
|
|
42
94
|
return path.join(dir, `${scriptName}${workerExt}`);
|
|
43
95
|
}
|
|
44
96
|
initialize() {
|
|
@@ -210,9 +262,10 @@ var WorkerPool = class {
|
|
|
210
262
|
for (const waiter of waiters) {
|
|
211
263
|
waiter.reject(terminationError);
|
|
212
264
|
}
|
|
213
|
-
const terminatePromises = this.workers.map((workerWrapper) => {
|
|
214
|
-
|
|
215
|
-
|
|
265
|
+
const terminatePromises = this.workers.map(async (workerWrapper) => {
|
|
266
|
+
const worker = workerWrapper.worker;
|
|
267
|
+
await flushWorkerCoverage(worker);
|
|
268
|
+
await new Promise((resolve) => {
|
|
216
269
|
let resolved = false;
|
|
217
270
|
const doResolve = () => {
|
|
218
271
|
if (!resolved) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { MessagePort, Worker } from 'node:worker_threads';
|
|
2
|
+
|
|
3
|
+
declare function isCoverageFlushMessage(msg: unknown): boolean;
|
|
4
|
+
/**
|
|
5
|
+
* Worker side: handle flush requests by writing this thread's V8 coverage. Call
|
|
6
|
+
* once near the top of each worker entry. No-op unless NODE_V8_COVERAGE is set.
|
|
7
|
+
*/
|
|
8
|
+
declare function registerWorkerCoverageFlush(port: MessagePort | null): void;
|
|
9
|
+
/**
|
|
10
|
+
* Pool side: ask a worker to flush its V8 coverage and wait (briefly) for the ack,
|
|
11
|
+
* so the write completes before the worker is terminated. No-op unless
|
|
12
|
+
* NODE_V8_COVERAGE is set; always resolves (on ack or timeout) and never throws.
|
|
13
|
+
*/
|
|
14
|
+
declare function flushWorkerCoverage(worker: Worker, timeoutMs?: number): Promise<void>;
|
|
15
|
+
|
|
16
|
+
export { flushWorkerCoverage, isCoverageFlushMessage, registerWorkerCoverageFlush };
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// src/node/parallel/coverage_flush.ts
|
|
2
|
+
import { takeCoverage } from "v8";
|
|
3
|
+
var FLUSH_REQUEST = "__shelby_cov_flush_request__";
|
|
4
|
+
var FLUSH_ACK = "__shelby_cov_flush_ack__";
|
|
5
|
+
var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
|
|
6
|
+
var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
|
|
7
|
+
function isCoverageFlushMessage(msg) {
|
|
8
|
+
return isType(msg, FLUSH_REQUEST) || isType(msg, FLUSH_ACK);
|
|
9
|
+
}
|
|
10
|
+
function registerWorkerCoverageFlush(port) {
|
|
11
|
+
if (!coverageEnabled() || !port) return;
|
|
12
|
+
port.on("message", (msg) => {
|
|
13
|
+
if (isType(msg, FLUSH_REQUEST)) {
|
|
14
|
+
takeCoverage();
|
|
15
|
+
port.postMessage({ type: FLUSH_ACK });
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
function flushWorkerCoverage(worker, timeoutMs = 2e3) {
|
|
20
|
+
if (!coverageEnabled()) return Promise.resolve();
|
|
21
|
+
return new Promise((resolve) => {
|
|
22
|
+
const onMessage = (msg) => {
|
|
23
|
+
if (isType(msg, FLUSH_ACK)) finish();
|
|
24
|
+
};
|
|
25
|
+
const finish = () => {
|
|
26
|
+
clearTimeout(timer);
|
|
27
|
+
worker.off("message", onMessage);
|
|
28
|
+
resolve();
|
|
29
|
+
};
|
|
30
|
+
const timer = setTimeout(finish, timeoutMs);
|
|
31
|
+
worker.on("message", onMessage);
|
|
32
|
+
try {
|
|
33
|
+
worker.postMessage({ type: FLUSH_REQUEST });
|
|
34
|
+
} catch {
|
|
35
|
+
finish();
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
export {
|
|
40
|
+
flushWorkerCoverage,
|
|
41
|
+
isCoverageFlushMessage,
|
|
42
|
+
registerWorkerCoverageFlush
|
|
43
|
+
};
|
|
@@ -6,6 +6,35 @@ import fs from "fs";
|
|
|
6
6
|
import path from "path";
|
|
7
7
|
import { fileURLToPath } from "url";
|
|
8
8
|
import { Worker } from "worker_threads";
|
|
9
|
+
|
|
10
|
+
// src/node/parallel/coverage_flush.ts
|
|
11
|
+
import { takeCoverage } from "v8";
|
|
12
|
+
var FLUSH_REQUEST = "__shelby_cov_flush_request__";
|
|
13
|
+
var FLUSH_ACK = "__shelby_cov_flush_ack__";
|
|
14
|
+
var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
|
|
15
|
+
var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
|
|
16
|
+
function flushWorkerCoverage(worker, timeoutMs = 2e3) {
|
|
17
|
+
if (!coverageEnabled()) return Promise.resolve();
|
|
18
|
+
return new Promise((resolve) => {
|
|
19
|
+
const onMessage = (msg) => {
|
|
20
|
+
if (isType(msg, FLUSH_ACK)) finish();
|
|
21
|
+
};
|
|
22
|
+
const finish = () => {
|
|
23
|
+
clearTimeout(timer);
|
|
24
|
+
worker.off("message", onMessage);
|
|
25
|
+
resolve();
|
|
26
|
+
};
|
|
27
|
+
const timer = setTimeout(finish, timeoutMs);
|
|
28
|
+
worker.on("message", onMessage);
|
|
29
|
+
try {
|
|
30
|
+
worker.postMessage({ type: FLUSH_REQUEST });
|
|
31
|
+
} catch {
|
|
32
|
+
finish();
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// src/node/parallel/worker_pool.ts
|
|
9
38
|
var WorkerPool = class {
|
|
10
39
|
workers = [];
|
|
11
40
|
queue = [];
|
|
@@ -42,6 +71,29 @@ var WorkerPool = class {
|
|
|
42
71
|
const dir = path.dirname(currentPath);
|
|
43
72
|
const ext = path.extname(currentPath);
|
|
44
73
|
const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
|
|
74
|
+
if (workerExt === ".ts") {
|
|
75
|
+
const srcSegment = `${path.sep}src${path.sep}`;
|
|
76
|
+
const srcIdx = currentPath.indexOf(srcSegment);
|
|
77
|
+
if (srcIdx !== -1) {
|
|
78
|
+
const packageRoot = currentPath.slice(0, srcIdx);
|
|
79
|
+
const relFromSrc = path.relative(path.join(packageRoot, "src"), dir);
|
|
80
|
+
return path.join(
|
|
81
|
+
packageRoot,
|
|
82
|
+
"dist",
|
|
83
|
+
"workers",
|
|
84
|
+
relFromSrc,
|
|
85
|
+
`${scriptName}.js`
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
let workerPath = path.join(dir, `${scriptName}${workerExt}`);
|
|
90
|
+
if (fs.existsSync(workerPath)) {
|
|
91
|
+
return workerPath;
|
|
92
|
+
}
|
|
93
|
+
workerPath = path.join(dir, "workers", `${scriptName}${workerExt}`);
|
|
94
|
+
if (fs.existsSync(workerPath)) {
|
|
95
|
+
return workerPath;
|
|
96
|
+
}
|
|
45
97
|
return path.join(dir, `${scriptName}${workerExt}`);
|
|
46
98
|
}
|
|
47
99
|
initialize() {
|
|
@@ -213,9 +265,10 @@ var WorkerPool = class {
|
|
|
213
265
|
for (const waiter of waiters) {
|
|
214
266
|
waiter.reject(terminationError);
|
|
215
267
|
}
|
|
216
|
-
const terminatePromises = this.workers.map((workerWrapper) => {
|
|
217
|
-
|
|
218
|
-
|
|
268
|
+
const terminatePromises = this.workers.map(async (workerWrapper) => {
|
|
269
|
+
const worker = workerWrapper.worker;
|
|
270
|
+
await flushWorkerCoverage(worker);
|
|
271
|
+
await new Promise((resolve) => {
|
|
219
272
|
let resolved = false;
|
|
220
273
|
const doResolve = () => {
|
|
221
274
|
if (!resolved) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { CommitmentEncoding } from './commitment_worker.js';
|
|
2
2
|
export { CommitmentWorkerPool } from './commitment_worker_pool.js';
|
|
3
|
+
export { flushWorkerCoverage, isCoverageFlushMessage, registerWorkerCoverageFlush } from './coverage_flush.js';
|
|
3
4
|
export { CommitmentWorkerPoolOptions, createDefaultCommitmentWorkerPool, defaultCommitmentWorkerCount } from './default_commitment_worker_pool.js';
|
|
4
5
|
export { generateCommitmentsParallel } from './parallel_commitments.js';
|
|
5
6
|
export { WorkerPool, WorkerPoolTask } from './worker_pool.js';
|
|
@@ -3,6 +3,47 @@ import fs from "fs";
|
|
|
3
3
|
import path from "path";
|
|
4
4
|
import { fileURLToPath } from "url";
|
|
5
5
|
import { Worker } from "worker_threads";
|
|
6
|
+
|
|
7
|
+
// src/node/parallel/coverage_flush.ts
|
|
8
|
+
import { takeCoverage } from "v8";
|
|
9
|
+
var FLUSH_REQUEST = "__shelby_cov_flush_request__";
|
|
10
|
+
var FLUSH_ACK = "__shelby_cov_flush_ack__";
|
|
11
|
+
var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
|
|
12
|
+
var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
|
|
13
|
+
function isCoverageFlushMessage(msg) {
|
|
14
|
+
return isType(msg, FLUSH_REQUEST) || isType(msg, FLUSH_ACK);
|
|
15
|
+
}
|
|
16
|
+
function registerWorkerCoverageFlush(port) {
|
|
17
|
+
if (!coverageEnabled() || !port) return;
|
|
18
|
+
port.on("message", (msg) => {
|
|
19
|
+
if (isType(msg, FLUSH_REQUEST)) {
|
|
20
|
+
takeCoverage();
|
|
21
|
+
port.postMessage({ type: FLUSH_ACK });
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
function flushWorkerCoverage(worker, timeoutMs = 2e3) {
|
|
26
|
+
if (!coverageEnabled()) return Promise.resolve();
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
const onMessage = (msg) => {
|
|
29
|
+
if (isType(msg, FLUSH_ACK)) finish();
|
|
30
|
+
};
|
|
31
|
+
const finish = () => {
|
|
32
|
+
clearTimeout(timer);
|
|
33
|
+
worker.off("message", onMessage);
|
|
34
|
+
resolve();
|
|
35
|
+
};
|
|
36
|
+
const timer = setTimeout(finish, timeoutMs);
|
|
37
|
+
worker.on("message", onMessage);
|
|
38
|
+
try {
|
|
39
|
+
worker.postMessage({ type: FLUSH_REQUEST });
|
|
40
|
+
} catch {
|
|
41
|
+
finish();
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// src/node/parallel/worker_pool.ts
|
|
6
47
|
var WorkerPool = class {
|
|
7
48
|
workers = [];
|
|
8
49
|
queue = [];
|
|
@@ -39,6 +80,29 @@ var WorkerPool = class {
|
|
|
39
80
|
const dir = path.dirname(currentPath);
|
|
40
81
|
const ext = path.extname(currentPath);
|
|
41
82
|
const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
|
|
83
|
+
if (workerExt === ".ts") {
|
|
84
|
+
const srcSegment = `${path.sep}src${path.sep}`;
|
|
85
|
+
const srcIdx = currentPath.indexOf(srcSegment);
|
|
86
|
+
if (srcIdx !== -1) {
|
|
87
|
+
const packageRoot = currentPath.slice(0, srcIdx);
|
|
88
|
+
const relFromSrc = path.relative(path.join(packageRoot, "src"), dir);
|
|
89
|
+
return path.join(
|
|
90
|
+
packageRoot,
|
|
91
|
+
"dist",
|
|
92
|
+
"workers",
|
|
93
|
+
relFromSrc,
|
|
94
|
+
`${scriptName}.js`
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
let workerPath = path.join(dir, `${scriptName}${workerExt}`);
|
|
99
|
+
if (fs.existsSync(workerPath)) {
|
|
100
|
+
return workerPath;
|
|
101
|
+
}
|
|
102
|
+
workerPath = path.join(dir, "workers", `${scriptName}${workerExt}`);
|
|
103
|
+
if (fs.existsSync(workerPath)) {
|
|
104
|
+
return workerPath;
|
|
105
|
+
}
|
|
42
106
|
return path.join(dir, `${scriptName}${workerExt}`);
|
|
43
107
|
}
|
|
44
108
|
initialize() {
|
|
@@ -210,9 +274,10 @@ var WorkerPool = class {
|
|
|
210
274
|
for (const waiter of waiters) {
|
|
211
275
|
waiter.reject(terminationError);
|
|
212
276
|
}
|
|
213
|
-
const terminatePromises = this.workers.map((workerWrapper) => {
|
|
214
|
-
|
|
215
|
-
|
|
277
|
+
const terminatePromises = this.workers.map(async (workerWrapper) => {
|
|
278
|
+
const worker = workerWrapper.worker;
|
|
279
|
+
await flushWorkerCoverage(worker);
|
|
280
|
+
await new Promise((resolve) => {
|
|
216
281
|
let resolved = false;
|
|
217
282
|
const doResolve = () => {
|
|
218
283
|
if (!resolved) {
|
|
@@ -559,5 +624,8 @@ export {
|
|
|
559
624
|
WorkerPool,
|
|
560
625
|
createDefaultCommitmentWorkerPool,
|
|
561
626
|
defaultCommitmentWorkerCount,
|
|
562
|
-
|
|
627
|
+
flushWorkerCoverage,
|
|
628
|
+
generateCommitmentsParallel,
|
|
629
|
+
isCoverageFlushMessage,
|
|
630
|
+
registerWorkerCoverageFlush
|
|
563
631
|
};
|