@shelby-protocol/sdk 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/index.d.ts +12 -3
- package/dist/browser/index.mjs +3717 -142
- package/dist/core/aptos-explorer.mjs +46 -5
- package/dist/core/blobs.mjs +12 -4
- package/dist/core/chunk.mjs +58 -9
- package/dist/core/clients/ShelbyBlobClient.d.ts +0 -43
- package/dist/core/clients/ShelbyBlobClient.mjs +1084 -17
- package/dist/core/clients/ShelbyClient.d.ts +12 -2
- package/dist/core/clients/ShelbyClient.mjs +2915 -23
- package/dist/core/clients/ShelbyClientConfig.mjs +0 -1
- package/dist/core/clients/ShelbyMetadataClient.d.ts +0 -18
- package/dist/core/clients/ShelbyMetadataClient.mjs +306 -6
- package/dist/core/clients/ShelbyMicropaymentChannelClient.d.ts +7 -49
- package/dist/core/clients/ShelbyMicropaymentChannelClient.mjs +839 -13
- package/dist/core/clients/ShelbyPlacementGroupClient.mjs +307 -8
- package/dist/core/clients/ShelbyRPCClient.d.ts +106 -3
- package/dist/core/clients/ShelbyRPCClient.mjs +1261 -12
- package/dist/core/clients/index.d.ts +2 -1
- package/dist/core/clients/index.mjs +3420 -39
- package/dist/core/clients/utils.mjs +65 -6
- package/dist/core/commitments.mjs +270 -11
- package/dist/core/constants.d.ts +2 -2
- package/dist/core/constants.mjs +39 -16
- package/dist/core/erasure/clay-codes.mjs +129 -4
- package/dist/core/erasure/constants.mjs +33 -10
- package/dist/core/erasure/default.mjs +191 -9
- package/dist/core/erasure/index.mjs +266 -23
- package/dist/core/erasure/reed-solomon.mjs +68 -4
- package/dist/core/erasure/utils.mjs +0 -2
- package/dist/core/errors.mjs +29 -8
- package/dist/core/index.d.ts +1 -1
- package/dist/core/index.mjs +3711 -142
- package/dist/core/layout.mjs +37 -5
- package/dist/core/networks.mjs +10 -5
- package/dist/core/operations/generated/sdk.d.ts +9 -0
- package/dist/core/operations/generated/sdk.mjs +183 -18
- package/dist/core/operations/index.mjs +284 -24
- package/dist/core/promises.mjs +4 -4
- package/dist/core/rpc-responses.mjs +29 -10
- package/dist/core/shelby-explorer.mjs +31 -6
- package/dist/core/strings.mjs +0 -2
- package/dist/core/types/blobs.mjs +0 -1
- package/dist/core/types/index.mjs +230 -9
- package/dist/core/types/payments.d.ts +6 -6
- package/dist/core/types/payments.mjs +230 -5
- package/dist/core/types/placement_groups.mjs +0 -1
- package/dist/core/types/storage_providers.mjs +0 -1
- package/dist/core/utils.mjs +94 -8
- package/dist/node/clients/ShelbyNodeClient.d.ts +1 -0
- package/dist/node/clients/ShelbyNodeClient.mjs +2919 -24
- package/dist/node/clients/index.d.ts +1 -0
- package/dist/node/clients/index.mjs +2920 -26
- package/dist/node/index.d.ts +1 -1
- package/dist/node/index.mjs +3734 -149
- package/dist/node/parallel/commitment_worker.d.ts +9 -0
- package/dist/node/parallel/commitment_worker.mjs +298 -0
- package/dist/node/parallel/commitment_worker_pool.d.ts +35 -0
- package/dist/node/parallel/commitment_worker_pool.mjs +286 -0
- package/dist/node/parallel/default_commitment_worker_pool.d.ts +24 -0
- package/dist/node/parallel/default_commitment_worker_pool.mjs +302 -0
- package/dist/node/parallel/index.d.ts +11 -0
- package/dist/node/parallel/index.mjs +563 -0
- package/dist/node/parallel/parallel_commitments.d.ts +32 -0
- package/dist/node/parallel/parallel_commitments.mjs +561 -0
- package/dist/node/parallel/worker_pool.d.ts +74 -0
- package/dist/node/parallel/worker_pool.mjs +248 -0
- package/dist/node/testUtil.mjs +8 -4
- package/package.json +6 -2
- package/dist/chunk-3NRBHSMQ.mjs +0 -12
- package/dist/chunk-5BVSWXQJ.mjs +0 -801
- package/dist/chunk-6FJNAUU5.mjs +0 -278
- package/dist/chunk-6XSZRDLJ.mjs +0 -375
- package/dist/chunk-7P6ASYW6.mjs +0 -9
- package/dist/chunk-A4IG6GSE.mjs +0 -21
- package/dist/chunk-AD2G3QYD.mjs +0 -0
- package/dist/chunk-AUQDI5BS.mjs +0 -43
- package/dist/chunk-AY3HBWXA.mjs +0 -51
- package/dist/chunk-BDSW5PHM.mjs +0 -37
- package/dist/chunk-C6RQ3AEU.mjs +0 -51
- package/dist/chunk-CQ6QPIZK.mjs +0 -37
- package/dist/chunk-D6GQHO6G.mjs +0 -15
- package/dist/chunk-EM67QTMR.mjs +0 -0
- package/dist/chunk-I6NG5GNL.mjs +0 -8
- package/dist/chunk-JTXYKO3U.mjs +0 -38
- package/dist/chunk-KG2TMLEV.mjs +0 -207
- package/dist/chunk-M3CP5TFP.mjs +0 -39
- package/dist/chunk-M7W43K2Q.mjs +0 -505
- package/dist/chunk-MB7C7VQF.mjs +0 -0
- package/dist/chunk-MNCQBEMS.mjs +0 -55
- package/dist/chunk-MQUVYMNQ.mjs +0 -0
- package/dist/chunk-NEO7Z3ZQ.mjs +0 -49
- package/dist/chunk-NHWWORCH.mjs +0 -72
- package/dist/chunk-OEK24ON7.mjs +0 -135
- package/dist/chunk-QQ57OGQ2.mjs +0 -0
- package/dist/chunk-RNNJXTX2.mjs +0 -154
- package/dist/chunk-S6Q4OG7H.mjs +0 -83
- package/dist/chunk-TET3DJEO.mjs +0 -199
- package/dist/chunk-UEZNZBJO.mjs +0 -104
- package/dist/chunk-V46D5NBW.mjs +0 -11
- package/dist/chunk-V4B3GAMQ.mjs +0 -42
- package/dist/chunk-WFWRBNTW.mjs +0 -412
- package/dist/chunk-WTICJPDB.mjs +0 -0
- package/dist/chunk-XNEIWM4O.mjs +0 -0
- package/dist/chunk-Z4FZ7W6L.mjs +0 -39
- package/dist/chunk-Z7RFCADT.mjs +0 -0
- package/dist/chunk-ZHXCVRZX.mjs +0 -0
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
// src/node/parallel/commitment_worker.ts
|
|
2
|
+
import { parentPort } from "worker_threads";
|
|
3
|
+
import { Hex as Hex3 } from "@aptos-labs/ts-sdk";
|
|
4
|
+
|
|
5
|
+
// src/core/commitments.ts
|
|
6
|
+
import { Hex as Hex2 } from "@aptos-labs/ts-sdk";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
|
|
9
|
+
// src/core/erasure/constants.ts
|
|
10
|
+
var ERASURE_CODE_PARAMS = {
|
|
11
|
+
["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
|
|
12
|
+
// total chunks (data + parity)
|
|
13
|
+
erasure_n: 16,
|
|
14
|
+
// data chunks
|
|
15
|
+
erasure_k: 10,
|
|
16
|
+
// helper nodes
|
|
17
|
+
erasure_d: 13,
|
|
18
|
+
// enum index
|
|
19
|
+
enumIndex: 0
|
|
20
|
+
},
|
|
21
|
+
["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
|
|
22
|
+
// total chunks (data + parity)
|
|
23
|
+
erasure_n: 4,
|
|
24
|
+
// data chunks
|
|
25
|
+
erasure_k: 2,
|
|
26
|
+
// helper nodes
|
|
27
|
+
erasure_d: 3,
|
|
28
|
+
// enum index
|
|
29
|
+
enumIndex: 1
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
var DEFAULT_ERASURE_N = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_n;
|
|
33
|
+
var DEFAULT_ERASURE_K = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_k;
|
|
34
|
+
var DEFAULT_ERASURE_D = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_d;
|
|
35
|
+
var DEFAULT_ERASURE_M = DEFAULT_ERASURE_N - DEFAULT_ERASURE_K;
|
|
36
|
+
|
|
37
|
+
// src/core/chunk.ts
|
|
38
|
+
var CHUNK_SIZE_PARAMS = {
|
|
39
|
+
["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */]: {
|
|
40
|
+
// 1MiB
|
|
41
|
+
chunkSizeBytes: 1 * 1024 * 1024,
|
|
42
|
+
// 10MiB
|
|
43
|
+
chunksetSizeBytes: 10 * 1024 * 1024
|
|
44
|
+
},
|
|
45
|
+
["ChunkSet2MiB_Chunk1MiB" /* ChunkSet2MiB_Chunk1MiB */]: {
|
|
46
|
+
// 1MiB
|
|
47
|
+
chunkSizeBytes: 1 * 1024 * 1024,
|
|
48
|
+
// 2MiB
|
|
49
|
+
chunksetSizeBytes: 2 * 1024 * 1024
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
var DEFAULT_CHUNK_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunkSizeBytes;
|
|
53
|
+
var DEFAULT_CHUNKSET_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunksetSizeBytes;
|
|
54
|
+
var ERASURE_CODE_AND_CHUNK_MAPPING = {
|
|
55
|
+
["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
|
|
56
|
+
...CHUNK_SIZE_PARAMS.ChunkSet10MiB_Chunk1MiB
|
|
57
|
+
},
|
|
58
|
+
["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
|
|
59
|
+
...CHUNK_SIZE_PARAMS.ChunkSet2MiB_Chunk1MiB
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
// src/core/utils.ts
|
|
64
|
+
import {
|
|
65
|
+
AccountAddress,
|
|
66
|
+
Hex
|
|
67
|
+
} from "@aptos-labs/ts-sdk";
|
|
68
|
+
function zeroPadBytes(buffer, desiredLength) {
|
|
69
|
+
if (buffer.byteLength === desiredLength) {
|
|
70
|
+
return buffer;
|
|
71
|
+
}
|
|
72
|
+
if (buffer.byteLength > desiredLength) {
|
|
73
|
+
return buffer.subarray(0, desiredLength);
|
|
74
|
+
}
|
|
75
|
+
const paddedBuffer = new Uint8Array(desiredLength);
|
|
76
|
+
paddedBuffer.set(buffer);
|
|
77
|
+
return paddedBuffer;
|
|
78
|
+
}
|
|
79
|
+
async function concatHashes(parts) {
|
|
80
|
+
const chunks = parts.map((part) => Hex.fromHexInput(part).toUint8Array());
|
|
81
|
+
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
|
|
82
|
+
const combined = new Uint8Array(totalLength);
|
|
83
|
+
let offset = 0;
|
|
84
|
+
for (const chunk of chunks) {
|
|
85
|
+
combined.set(chunk, offset);
|
|
86
|
+
offset += chunk.byteLength;
|
|
87
|
+
}
|
|
88
|
+
return Hex.fromHexInput(
|
|
89
|
+
new Uint8Array(await crypto.subtle.digest("SHA-256", combined))
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// src/core/commitments.ts
|
|
94
|
+
var ChunksetCommitmentSchema = z.object({
|
|
95
|
+
// Chunkset root (vector commitment of child chunks)
|
|
96
|
+
chunkset_root: z.string(),
|
|
97
|
+
// the size is known statically from the current configuration
|
|
98
|
+
chunk_commitments: z.array(z.string())
|
|
99
|
+
}).refine(
|
|
100
|
+
(data) => {
|
|
101
|
+
return data.chunk_commitments.length === DEFAULT_ERASURE_K + DEFAULT_ERASURE_M;
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
message: `Chunkset must have exactly ${DEFAULT_ERASURE_K + DEFAULT_ERASURE_M} chunks (ERASURE_K + ERASURE_M = ${DEFAULT_ERASURE_K} + ${DEFAULT_ERASURE_M})`,
|
|
105
|
+
path: ["chunk_commitments"]
|
|
106
|
+
}
|
|
107
|
+
);
|
|
108
|
+
function expectedTotalChunksets(rawSize, chunksetSize = DEFAULT_CHUNKSET_SIZE_BYTES) {
|
|
109
|
+
if (chunksetSize <= 0) {
|
|
110
|
+
throw new Error("chunksetSize must be positive");
|
|
111
|
+
}
|
|
112
|
+
if (rawSize === 0) return 1;
|
|
113
|
+
return Math.ceil(rawSize / chunksetSize);
|
|
114
|
+
}
|
|
115
|
+
var BlobCommitmentsSchema = z.object({
|
|
116
|
+
schema_version: z.string(),
|
|
117
|
+
raw_data_size: z.number(),
|
|
118
|
+
// FIXME I am not sure about this being here, or if it should be somewhere else
|
|
119
|
+
blob_merkle_root: z.string(),
|
|
120
|
+
chunkset_commitments: z.array(ChunksetCommitmentSchema)
|
|
121
|
+
}).refine(
|
|
122
|
+
(data) => {
|
|
123
|
+
return expectedTotalChunksets(data.raw_data_size) === data.chunkset_commitments.length;
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
message: "Total chunkset count mismatches with raw data size",
|
|
127
|
+
// FIXME put more details in here
|
|
128
|
+
path: ["chunkset_commitments"]
|
|
129
|
+
}
|
|
130
|
+
);
|
|
131
|
+
async function generateMerkleRoot(leafHashes) {
|
|
132
|
+
if (!leafHashes.length) {
|
|
133
|
+
throw new Error(
|
|
134
|
+
"An empty array cannot be used to construct a Merkle tree."
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
const zeroArray = new Uint8Array(leafHashes[0].toUint8Array().length);
|
|
138
|
+
const zeroBytes = Hex2.fromHexInput(zeroArray);
|
|
139
|
+
let currentLeaves = leafHashes;
|
|
140
|
+
while (currentLeaves.length > 1) {
|
|
141
|
+
if (currentLeaves.length % 2 !== 0) {
|
|
142
|
+
currentLeaves.push(zeroBytes);
|
|
143
|
+
}
|
|
144
|
+
const nextLeaves = [];
|
|
145
|
+
for (let i = 0; i < currentLeaves.length; i += 2) {
|
|
146
|
+
nextLeaves.push(
|
|
147
|
+
await concatHashes([
|
|
148
|
+
currentLeaves[i].toUint8Array(),
|
|
149
|
+
currentLeaves[i + 1].toUint8Array()
|
|
150
|
+
])
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
currentLeaves = nextLeaves;
|
|
154
|
+
}
|
|
155
|
+
return currentLeaves[0];
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// src/core/erasure/clay-codes.ts
|
|
159
|
+
import {
|
|
160
|
+
createDecoder,
|
|
161
|
+
createEncoder
|
|
162
|
+
} from "@shelby-protocol/clay-codes";
|
|
163
|
+
function getTotalChunks(config) {
|
|
164
|
+
return config.erasure_n;
|
|
165
|
+
}
|
|
166
|
+
var ClayErasureCodingProvider = class _ClayErasureCodingProvider {
|
|
167
|
+
config;
|
|
168
|
+
encoderCache;
|
|
169
|
+
decoderCache;
|
|
170
|
+
lastFunction;
|
|
171
|
+
constructor(config) {
|
|
172
|
+
this.config = config;
|
|
173
|
+
this.lastFunction = "none" /* NONE */;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Static factory method to create an initialized ClayErasureCodingProvider
|
|
177
|
+
*/
|
|
178
|
+
static async create(config) {
|
|
179
|
+
const provider = new _ClayErasureCodingProvider(config);
|
|
180
|
+
[provider.encoderCache, provider.decoderCache] = await Promise.all([
|
|
181
|
+
createEncoder({
|
|
182
|
+
n: getTotalChunks(config),
|
|
183
|
+
k: config.erasure_k,
|
|
184
|
+
d: config.erasure_d,
|
|
185
|
+
chunkSizeBytes: config.chunkSizeBytes
|
|
186
|
+
}),
|
|
187
|
+
createDecoder({
|
|
188
|
+
n: getTotalChunks(config),
|
|
189
|
+
k: config.erasure_k,
|
|
190
|
+
d: config.erasure_d,
|
|
191
|
+
chunkSizeBytes: config.chunkSizeBytes,
|
|
192
|
+
erasedChunkIndexes: []
|
|
193
|
+
})
|
|
194
|
+
]);
|
|
195
|
+
return provider;
|
|
196
|
+
}
|
|
197
|
+
encode(data) {
|
|
198
|
+
const { erasure_k, chunkSizeBytes } = this.config;
|
|
199
|
+
if (!this.encoderCache) {
|
|
200
|
+
throw new Error("Encoder cache is missing");
|
|
201
|
+
}
|
|
202
|
+
const systematicCapacity = erasure_k * chunkSizeBytes;
|
|
203
|
+
if (data.byteLength > systematicCapacity) {
|
|
204
|
+
throw new Error(
|
|
205
|
+
`Data size ${data.byteLength} bytes exceeds systematic capacity ${systematicCapacity} bytes (erasure_k=${erasure_k} * chunkSizeBytes=${chunkSizeBytes}).`
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
if (this.requiresPadding(data.length)) {
|
|
209
|
+
throw new Error(
|
|
210
|
+
`Data size ${data.length} bytes does not match systematic capacity ${systematicCapacity} bytes (erasure_k=${erasure_k} * chunkSizeBytes=${chunkSizeBytes}). Provide pre-padded data before encoding.`
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
this.lastFunction = "encoded" /* ENCODED */;
|
|
214
|
+
return this.encoderCache.erasureCode(data);
|
|
215
|
+
}
|
|
216
|
+
decode(available, config) {
|
|
217
|
+
if (!this.decoderCache) {
|
|
218
|
+
throw new Error("Decoder cache is missing");
|
|
219
|
+
}
|
|
220
|
+
const { erasure_k, chunkSizeBytes } = this.config;
|
|
221
|
+
if (available.length < erasure_k) {
|
|
222
|
+
throw new Error(
|
|
223
|
+
`Insufficient available chunks: need at least ${erasure_k}, received ${available.length}.`
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
for (let i = 0; i < available.length; i++) {
|
|
227
|
+
const chunk = available[i];
|
|
228
|
+
if (chunk.length !== chunkSizeBytes) {
|
|
229
|
+
throw new Error(
|
|
230
|
+
`Chunk ${i} size ${chunk.length} bytes does not match expected chunkSizeBytes ${chunkSizeBytes}.`
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
this.lastFunction = "decoded" /* DECODED */;
|
|
235
|
+
return this.decoderCache.decode(available, config);
|
|
236
|
+
}
|
|
237
|
+
getChunkMerkleRoots() {
|
|
238
|
+
if (this.decoderCache && this.lastFunction === "decoded" /* DECODED */)
|
|
239
|
+
return this.decoderCache.getChunkMerkleRoots();
|
|
240
|
+
if (this.encoderCache && this.lastFunction === "encoded" /* ENCODED */)
|
|
241
|
+
return this.encoderCache.getChunkMerkleRoots();
|
|
242
|
+
throw new Error(
|
|
243
|
+
"You must call encode or decode before calling getChunkMerkleRoots"
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Determines if data can be erasure coded as-is or requires padding.
|
|
248
|
+
*
|
|
249
|
+
* Data can be erasure coded without padding if its size exactly matches
|
|
250
|
+
* the total systematic data capacity (k * chunkSizeBytes).
|
|
251
|
+
*
|
|
252
|
+
* @param dataSize - Size of the data in bytes
|
|
253
|
+
* @returns true if data needs padding, false if it can be coded as-is
|
|
254
|
+
*/
|
|
255
|
+
requiresPadding(dataSize) {
|
|
256
|
+
const { erasure_k, chunkSizeBytes } = this.config;
|
|
257
|
+
const systematicCapacity = erasure_k * chunkSizeBytes;
|
|
258
|
+
return dataSize !== systematicCapacity;
|
|
259
|
+
}
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
// src/node/parallel/commitment_worker.ts
|
|
263
|
+
async function encodeChunksetForCommitment(chunksetData, encoding) {
|
|
264
|
+
const chunksetSizeBytes = encoding.erasure_k * encoding.chunkSizeBytes;
|
|
265
|
+
const paddedChunkset = zeroPadBytes(chunksetData, chunksetSizeBytes);
|
|
266
|
+
const provider = await getProvider(encoding);
|
|
267
|
+
provider.encode(paddedChunkset);
|
|
268
|
+
const chunkRoots = provider.getChunkMerkleRoots().map((root) => Hex3.fromHexInput(root));
|
|
269
|
+
const chunksetRoot = await generateMerkleRoot(chunkRoots);
|
|
270
|
+
return {
|
|
271
|
+
chunkset_root: chunksetRoot.toString(),
|
|
272
|
+
chunk_commitments: chunkRoots.map((r) => r.toString())
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
var cachedProvider;
|
|
276
|
+
var cachedEncodingKey;
|
|
277
|
+
async function getProvider(encoding) {
|
|
278
|
+
const key = `${encoding.erasure_n},${encoding.erasure_k},${encoding.erasure_d},${encoding.chunkSizeBytes},${encoding.enumIndex}`;
|
|
279
|
+
if (!cachedProvider || cachedEncodingKey !== key) {
|
|
280
|
+
cachedProvider = await ClayErasureCodingProvider.create(encoding);
|
|
281
|
+
cachedEncodingKey = key;
|
|
282
|
+
}
|
|
283
|
+
return cachedProvider;
|
|
284
|
+
}
|
|
285
|
+
parentPort?.on("message", async (input) => {
|
|
286
|
+
try {
|
|
287
|
+
const result = await encodeChunksetForCommitment(
|
|
288
|
+
input.chunksetData,
|
|
289
|
+
input.encoding
|
|
290
|
+
);
|
|
291
|
+
parentPort?.postMessage({ success: true, result });
|
|
292
|
+
} catch (error) {
|
|
293
|
+
parentPort?.postMessage({
|
|
294
|
+
success: false,
|
|
295
|
+
error: error instanceof Error ? error.message : String(error)
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
});
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Worker } from 'node:worker_threads';
|
|
2
|
+
import { ChunksetCommitment } from '../../core/commitments.js';
|
|
3
|
+
import { CommitmentEncoding } from './commitment_worker.js';
|
|
4
|
+
import { WorkerPool, WorkerPoolTask } from './worker_pool.js';
|
|
5
|
+
import '@aptos-labs/ts-sdk';
|
|
6
|
+
import 'zod';
|
|
7
|
+
import '../../clay-codes-DdXABBDx.js';
|
|
8
|
+
import '@shelby-protocol/clay-codes';
|
|
9
|
+
|
|
10
|
+
interface CommitmentTask extends WorkerPoolTask<ChunksetCommitment> {
|
|
11
|
+
chunksetIdx: number;
|
|
12
|
+
chunksetData: Uint8Array;
|
|
13
|
+
encoding: CommitmentEncoding;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Worker pool that encodes chunksets in parallel and returns their commitments.
|
|
17
|
+
*
|
|
18
|
+
* Unlike the RPC server's EncodeWorkerPool, this pool does not write encoded
|
|
19
|
+
* chunks to disk — it only computes the ChunksetCommitment. This makes it
|
|
20
|
+
* suitable for the CLI and any other caller that only needs commitments.
|
|
21
|
+
*/
|
|
22
|
+
declare class CommitmentWorkerPool extends WorkerPool<CommitmentTask, ChunksetCommitment> {
|
|
23
|
+
constructor(maxWorkers: number);
|
|
24
|
+
protected postTaskMessage(worker: Worker, task: CommitmentTask): void;
|
|
25
|
+
/**
|
|
26
|
+
* Dispatch a chunkset to a worker for encoding. Returns a promise that
|
|
27
|
+
* resolves to the chunkset's commitment once the worker finishes.
|
|
28
|
+
*
|
|
29
|
+
* The caller must ensure `chunksetData` is not used after this call
|
|
30
|
+
* (its underlying buffer is transferred to the worker thread).
|
|
31
|
+
*/
|
|
32
|
+
encodeChunkset(chunksetIdx: number, chunksetData: Uint8Array, encoding: CommitmentEncoding): Promise<ChunksetCommitment>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export { CommitmentWorkerPool };
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
// src/node/parallel/worker_pool.ts
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { fileURLToPath } from "url";
|
|
5
|
+
import { Worker } from "worker_threads";
|
|
6
|
+
var WorkerPool = class {
|
|
7
|
+
workers = [];
|
|
8
|
+
queue = [];
|
|
9
|
+
maxWorkers;
|
|
10
|
+
workerPath;
|
|
11
|
+
isTerminating = false;
|
|
12
|
+
activeTasks = /* @__PURE__ */ new Set();
|
|
13
|
+
inFlightCount = 0;
|
|
14
|
+
slotWaiters = [];
|
|
15
|
+
constructor(maxWorkers, workerPath) {
|
|
16
|
+
if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) {
|
|
17
|
+
throw new Error(
|
|
18
|
+
`maxWorkers must be a positive safe integer, got ${maxWorkers}`
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
this.maxWorkers = maxWorkers;
|
|
22
|
+
this.workerPath = workerPath;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Resolve the path to a worker script relative to the calling module.
|
|
26
|
+
*
|
|
27
|
+
* Pass `import.meta.url` from the subclass file along with the script name
|
|
28
|
+
* (without extension). The extension is inferred from the calling module's
|
|
29
|
+
* own extension so that both the TypeScript source (.ts) and the compiled
|
|
30
|
+
* output (.mjs / .js) are handled correctly.
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* constructor(maxWorkers: number) {
|
|
34
|
+
* super(maxWorkers, WorkerPool.resolveWorkerPath(import.meta.url, "my_worker"));
|
|
35
|
+
* }
|
|
36
|
+
*/
|
|
37
|
+
static resolveWorkerPath(importMetaUrl, scriptName) {
|
|
38
|
+
const currentPath = fileURLToPath(importMetaUrl);
|
|
39
|
+
const dir = path.dirname(currentPath);
|
|
40
|
+
const ext = path.extname(currentPath);
|
|
41
|
+
const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
|
|
42
|
+
return path.join(dir, `${scriptName}${workerExt}`);
|
|
43
|
+
}
|
|
44
|
+
initialize() {
|
|
45
|
+
this.isTerminating = false;
|
|
46
|
+
if (!fs.existsSync(this.workerPath)) {
|
|
47
|
+
throw new Error(
|
|
48
|
+
`Could not find worker script at ${this.workerPath}. Running as ${path.extname(this.workerPath)}, expected worker at same extension.`
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
for (let i = 0; i < this.maxWorkers; i++) {
|
|
52
|
+
this.workers.push({ worker: new Worker(this.workerPath), busy: false });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
isInitialized() {
|
|
56
|
+
return this.workers.length > 0;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Acquire a back-pressure slot before submitting work to the pool.
|
|
60
|
+
* Resolves immediately if fewer than `maxWorkers` chunksets are in flight;
|
|
61
|
+
* otherwise waits until a slot is released. Callers must call `releaseSlot()`
|
|
62
|
+
* when the submitted task settles (resolve or reject).
|
|
63
|
+
*
|
|
64
|
+
* Moving back-pressure into the pool ensures a single global bound on
|
|
65
|
+
* buffered memory regardless of how many concurrent callers share the pool.
|
|
66
|
+
*/
|
|
67
|
+
acquireSlot() {
|
|
68
|
+
if (this.inFlightCount < this.maxWorkers) {
|
|
69
|
+
this.inFlightCount++;
|
|
70
|
+
return Promise.resolve();
|
|
71
|
+
}
|
|
72
|
+
return new Promise((resolve, reject) => {
|
|
73
|
+
this.slotWaiters.push({
|
|
74
|
+
resolve: () => {
|
|
75
|
+
this.inFlightCount++;
|
|
76
|
+
resolve();
|
|
77
|
+
},
|
|
78
|
+
reject
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
releaseSlot() {
|
|
83
|
+
this.inFlightCount--;
|
|
84
|
+
const next = this.slotWaiters.shift();
|
|
85
|
+
if (next) {
|
|
86
|
+
next.resolve();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
enqueueTask(task) {
|
|
90
|
+
this.activeTasks.add(task);
|
|
91
|
+
if (this.isTerminating) {
|
|
92
|
+
this.activeTasks.delete(task);
|
|
93
|
+
task.reject(
|
|
94
|
+
new Error("Worker pool is being terminated - cannot process new tasks")
|
|
95
|
+
);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const availableWorker = this.workers.find((w) => !w.busy);
|
|
99
|
+
if (availableWorker) {
|
|
100
|
+
this.executeTask(availableWorker, task);
|
|
101
|
+
} else {
|
|
102
|
+
this.queue.push(task);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
processNextQueuedTask(preferredWorker) {
|
|
106
|
+
const nextTask = this.queue.shift();
|
|
107
|
+
if (!nextTask) return;
|
|
108
|
+
if (preferredWorker && !preferredWorker.busy && preferredWorker.worker) {
|
|
109
|
+
this.executeTask(preferredWorker, nextTask);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const availableWorker = this.workers.find((w) => !w.busy && w.worker);
|
|
113
|
+
if (availableWorker) {
|
|
114
|
+
this.executeTask(availableWorker, nextTask);
|
|
115
|
+
} else {
|
|
116
|
+
this.queue.unshift(nextTask);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
rejectAllQueuedTasks(error) {
|
|
120
|
+
const tasks = this.queue.splice(0);
|
|
121
|
+
for (const task of tasks) {
|
|
122
|
+
this.activeTasks.delete(task);
|
|
123
|
+
task.reject(error);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
rejectAllActiveTasks(error) {
|
|
127
|
+
const tasks = Array.from(this.activeTasks);
|
|
128
|
+
this.activeTasks.clear();
|
|
129
|
+
for (const task of tasks) {
|
|
130
|
+
task.reject(error);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
executeTask(workerWrapper, task) {
|
|
134
|
+
workerWrapper.busy = true;
|
|
135
|
+
const cleanup = () => {
|
|
136
|
+
workerWrapper.worker.off("message", messageHandler);
|
|
137
|
+
workerWrapper.worker.off("error", errorHandler);
|
|
138
|
+
workerWrapper.worker.off("exit", exitHandler);
|
|
139
|
+
workerWrapper.busy = false;
|
|
140
|
+
};
|
|
141
|
+
const messageHandler = (message) => {
|
|
142
|
+
cleanup();
|
|
143
|
+
this.activeTasks.delete(task);
|
|
144
|
+
if (message.success && message.result !== void 0) {
|
|
145
|
+
task.resolve(message.result);
|
|
146
|
+
} else {
|
|
147
|
+
task.reject(new Error(message.error ?? "Worker processing failed"));
|
|
148
|
+
}
|
|
149
|
+
this.processNextQueuedTask(workerWrapper);
|
|
150
|
+
};
|
|
151
|
+
const errorHandler = (error) => {
|
|
152
|
+
workerWrapper.worker.off("message", messageHandler);
|
|
153
|
+
workerWrapper.worker.off("error", errorHandler);
|
|
154
|
+
workerWrapper.busy = false;
|
|
155
|
+
this.activeTasks.delete(task);
|
|
156
|
+
task.reject(error);
|
|
157
|
+
};
|
|
158
|
+
const exitHandler = (exitCode) => {
|
|
159
|
+
workerWrapper.worker.off("message", messageHandler);
|
|
160
|
+
workerWrapper.worker.off("error", errorHandler);
|
|
161
|
+
workerWrapper.worker.off("exit", exitHandler);
|
|
162
|
+
workerWrapper.busy = false;
|
|
163
|
+
this.activeTasks.delete(task);
|
|
164
|
+
task.reject(
|
|
165
|
+
new Error(`Worker terminated unexpectedly with exit code ${exitCode}`)
|
|
166
|
+
);
|
|
167
|
+
if (this.isTerminating) return;
|
|
168
|
+
try {
|
|
169
|
+
const newWorker = new Worker(this.workerPath);
|
|
170
|
+
if (this.isTerminating) {
|
|
171
|
+
newWorker.terminate().catch(() => {
|
|
172
|
+
});
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
workerWrapper.worker = newWorker;
|
|
176
|
+
this.processNextQueuedTask(workerWrapper);
|
|
177
|
+
} catch (_error) {
|
|
178
|
+
const index = this.workers.indexOf(workerWrapper);
|
|
179
|
+
if (index > -1) this.workers.splice(index, 1);
|
|
180
|
+
if (this.workers.length === 0) {
|
|
181
|
+
this.rejectAllQueuedTasks(
|
|
182
|
+
new Error(
|
|
183
|
+
"All workers have died and could not be replaced - worker pool is non-functional"
|
|
184
|
+
)
|
|
185
|
+
);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
this.processNextQueuedTask();
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
workerWrapper.worker.on("message", messageHandler);
|
|
192
|
+
workerWrapper.worker.on("error", errorHandler);
|
|
193
|
+
workerWrapper.worker.on("exit", exitHandler);
|
|
194
|
+
this.postTaskMessage(workerWrapper.worker, task);
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Terminate all workers and reject all pending tasks.
|
|
198
|
+
*
|
|
199
|
+
* This pool is NOT designed to be reused after termination.
|
|
200
|
+
* Create a new instance if a fresh pool is needed.
|
|
201
|
+
*/
|
|
202
|
+
async terminate() {
|
|
203
|
+
this.isTerminating = true;
|
|
204
|
+
const terminationError = new Error(
|
|
205
|
+
"Worker pool is being terminated - task cancelled"
|
|
206
|
+
);
|
|
207
|
+
this.rejectAllQueuedTasks(terminationError);
|
|
208
|
+
this.rejectAllActiveTasks(terminationError);
|
|
209
|
+
const waiters = this.slotWaiters.splice(0);
|
|
210
|
+
for (const waiter of waiters) {
|
|
211
|
+
waiter.reject(terminationError);
|
|
212
|
+
}
|
|
213
|
+
const terminatePromises = this.workers.map((workerWrapper) => {
|
|
214
|
+
return new Promise((resolve) => {
|
|
215
|
+
const worker = workerWrapper.worker;
|
|
216
|
+
let resolved = false;
|
|
217
|
+
const doResolve = () => {
|
|
218
|
+
if (!resolved) {
|
|
219
|
+
resolved = true;
|
|
220
|
+
resolve();
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
let timeoutId;
|
|
224
|
+
const exitHandler = () => {
|
|
225
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
226
|
+
doResolve();
|
|
227
|
+
};
|
|
228
|
+
worker.removeAllListeners("message");
|
|
229
|
+
worker.removeAllListeners("error");
|
|
230
|
+
worker.once("exit", exitHandler);
|
|
231
|
+
timeoutId = setTimeout(() => {
|
|
232
|
+
worker.removeListener("exit", exitHandler);
|
|
233
|
+
doResolve();
|
|
234
|
+
}, 2e3);
|
|
235
|
+
worker.terminate().catch(() => {
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
});
|
|
239
|
+
await Promise.all(terminatePromises);
|
|
240
|
+
this.workers = [];
|
|
241
|
+
this.queue = [];
|
|
242
|
+
this.inFlightCount = 0;
|
|
243
|
+
this.isTerminating = false;
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
// src/node/parallel/commitment_worker_pool.ts
|
|
248
|
+
var CommitmentWorkerPool = class extends WorkerPool {
|
|
249
|
+
constructor(maxWorkers) {
|
|
250
|
+
super(
|
|
251
|
+
maxWorkers,
|
|
252
|
+
WorkerPool.resolveWorkerPath(import.meta.url, "commitment_worker")
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
postTaskMessage(worker, task) {
|
|
256
|
+
worker.postMessage(
|
|
257
|
+
{
|
|
258
|
+
chunksetData: task.chunksetData,
|
|
259
|
+
encoding: task.encoding
|
|
260
|
+
},
|
|
261
|
+
[task.chunksetData.buffer]
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Dispatch a chunkset to a worker for encoding. Returns a promise that
|
|
266
|
+
* resolves to the chunkset's commitment once the worker finishes.
|
|
267
|
+
*
|
|
268
|
+
* The caller must ensure `chunksetData` is not used after this call
|
|
269
|
+
* (its underlying buffer is transferred to the worker thread).
|
|
270
|
+
*/
|
|
271
|
+
encodeChunkset(chunksetIdx, chunksetData, encoding) {
|
|
272
|
+
return new Promise((resolve, reject) => {
|
|
273
|
+
const task = {
|
|
274
|
+
chunksetIdx,
|
|
275
|
+
chunksetData,
|
|
276
|
+
encoding,
|
|
277
|
+
resolve,
|
|
278
|
+
reject
|
|
279
|
+
};
|
|
280
|
+
this.enqueueTask(task);
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
export {
|
|
285
|
+
CommitmentWorkerPool
|
|
286
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { CommitmentWorkerPool } from './commitment_worker_pool.js';
|
|
2
|
+
import 'node:worker_threads';
|
|
3
|
+
import '../../core/commitments.js';
|
|
4
|
+
import '@aptos-labs/ts-sdk';
|
|
5
|
+
import 'zod';
|
|
6
|
+
import '../../clay-codes-DdXABBDx.js';
|
|
7
|
+
import '@shelby-protocol/clay-codes';
|
|
8
|
+
import './commitment_worker.js';
|
|
9
|
+
import './worker_pool.js';
|
|
10
|
+
|
|
11
|
+
interface CommitmentWorkerPoolOptions {
|
|
12
|
+
/** Worker thread count. Defaults to availableParallelism() - 1, minimum 1. */
|
|
13
|
+
workers?: number;
|
|
14
|
+
}
|
|
15
|
+
declare function defaultCommitmentWorkerCount(): number;
|
|
16
|
+
/**
|
|
17
|
+
* Create and initialize a caller-owned commitment worker pool.
|
|
18
|
+
*
|
|
19
|
+
* The returned pool is ready to pass to `generateCommitmentsParallel`.
|
|
20
|
+
* Callers own its lifecycle and must call `terminate()` during shutdown.
|
|
21
|
+
*/
|
|
22
|
+
declare function createDefaultCommitmentWorkerPool(options?: CommitmentWorkerPoolOptions): CommitmentWorkerPool;
|
|
23
|
+
|
|
24
|
+
export { type CommitmentWorkerPoolOptions, createDefaultCommitmentWorkerPool, defaultCommitmentWorkerCount };
|