@shelby-protocol/sdk 0.2.4 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/dist/browser/index.d.ts +13 -4
  2. package/dist/browser/index.mjs +3719 -140
  3. package/dist/{clay-codes-DHP-bYcP.d.ts → clay-codes-DdXABBDx.d.ts} +0 -7
  4. package/dist/core/aptos-explorer.mjs +46 -5
  5. package/dist/core/blobs.d.ts +1 -1
  6. package/dist/core/blobs.mjs +12 -4
  7. package/dist/core/chunk.mjs +58 -9
  8. package/dist/core/clients/ShelbyBlobClient.d.ts +9 -44
  9. package/dist/core/clients/ShelbyBlobClient.mjs +1084 -17
  10. package/dist/core/clients/ShelbyClient.d.ts +12 -2
  11. package/dist/core/clients/ShelbyClient.mjs +2915 -23
  12. package/dist/core/clients/ShelbyClientConfig.d.ts +8 -0
  13. package/dist/core/clients/ShelbyClientConfig.mjs +0 -1
  14. package/dist/core/clients/ShelbyMetadataClient.d.ts +0 -18
  15. package/dist/core/clients/ShelbyMetadataClient.mjs +306 -6
  16. package/dist/core/clients/ShelbyMicropaymentChannelClient.d.ts +7 -49
  17. package/dist/core/clients/ShelbyMicropaymentChannelClient.mjs +839 -13
  18. package/dist/core/clients/ShelbyPlacementGroupClient.mjs +307 -8
  19. package/dist/core/clients/ShelbyRPCClient.d.ts +107 -4
  20. package/dist/core/clients/ShelbyRPCClient.mjs +1261 -12
  21. package/dist/core/clients/index.d.ts +3 -2
  22. package/dist/core/clients/index.mjs +3421 -40
  23. package/dist/core/clients/utils.mjs +65 -6
  24. package/dist/core/commitments.d.ts +3 -12
  25. package/dist/core/commitments.mjs +270 -11
  26. package/dist/core/constants.d.ts +4 -4
  27. package/dist/core/constants.mjs +39 -16
  28. package/dist/core/erasure/clay-codes.d.ts +1 -1
  29. package/dist/core/erasure/clay-codes.mjs +129 -4
  30. package/dist/core/erasure/constants.mjs +33 -10
  31. package/dist/core/erasure/default.d.ts +1 -1
  32. package/dist/core/erasure/default.mjs +191 -9
  33. package/dist/core/erasure/index.d.ts +1 -1
  34. package/dist/core/erasure/index.mjs +266 -23
  35. package/dist/core/erasure/provider.d.ts +1 -1
  36. package/dist/core/erasure/reed-solomon.d.ts +1 -1
  37. package/dist/core/erasure/reed-solomon.mjs +68 -4
  38. package/dist/core/erasure/utils.d.ts +1 -1
  39. package/dist/core/erasure/utils.mjs +0 -2
  40. package/dist/core/errors.mjs +29 -8
  41. package/dist/core/index.d.ts +4 -4
  42. package/dist/core/index.mjs +3713 -140
  43. package/dist/core/layout.d.ts +1 -1
  44. package/dist/core/layout.mjs +37 -5
  45. package/dist/core/networks.mjs +10 -5
  46. package/dist/core/operations/generated/sdk.d.ts +9 -0
  47. package/dist/core/operations/generated/sdk.mjs +183 -18
  48. package/dist/core/operations/index.mjs +284 -24
  49. package/dist/core/promises.mjs +4 -4
  50. package/dist/core/rpc-responses.d.ts +38 -1
  51. package/dist/core/rpc-responses.mjs +31 -8
  52. package/dist/core/shelby-explorer.mjs +31 -6
  53. package/dist/core/strings.mjs +0 -2
  54. package/dist/core/types/blobs.d.ts +1 -1
  55. package/dist/core/types/blobs.mjs +0 -1
  56. package/dist/core/types/index.d.ts +1 -1
  57. package/dist/core/types/index.mjs +230 -9
  58. package/dist/core/types/payments.d.ts +6 -6
  59. package/dist/core/types/payments.mjs +230 -5
  60. package/dist/core/types/placement_groups.mjs +0 -1
  61. package/dist/core/types/storage_providers.mjs +0 -1
  62. package/dist/core/utils.mjs +94 -8
  63. package/dist/node/clients/ShelbyNodeClient.d.ts +2 -1
  64. package/dist/node/clients/ShelbyNodeClient.mjs +2919 -24
  65. package/dist/node/clients/index.d.ts +2 -1
  66. package/dist/node/clients/index.mjs +2920 -26
  67. package/dist/node/index.d.ts +4 -4
  68. package/dist/node/index.mjs +3736 -147
  69. package/dist/node/parallel/commitment_worker.d.ts +9 -0
  70. package/dist/node/parallel/commitment_worker.mjs +298 -0
  71. package/dist/node/parallel/commitment_worker_pool.d.ts +35 -0
  72. package/dist/node/parallel/commitment_worker_pool.mjs +286 -0
  73. package/dist/node/parallel/default_commitment_worker_pool.d.ts +24 -0
  74. package/dist/node/parallel/default_commitment_worker_pool.mjs +302 -0
  75. package/dist/node/parallel/index.d.ts +11 -0
  76. package/dist/node/parallel/index.mjs +563 -0
  77. package/dist/node/parallel/parallel_commitments.d.ts +32 -0
  78. package/dist/node/parallel/parallel_commitments.mjs +561 -0
  79. package/dist/node/parallel/worker_pool.d.ts +74 -0
  80. package/dist/node/parallel/worker_pool.mjs +248 -0
  81. package/dist/node/testUtil.mjs +8 -4
  82. package/package.json +9 -6
  83. package/dist/chunk-3NRBHSMQ.mjs +0 -12
  84. package/dist/chunk-3PCG7PNP.mjs +0 -375
  85. package/dist/chunk-7OV5ZYW6.mjs +0 -55
  86. package/dist/chunk-7P6ASYW6.mjs +0 -9
  87. package/dist/chunk-A4IG6GSE.mjs +0 -21
  88. package/dist/chunk-AD2G3QYD.mjs +0 -0
  89. package/dist/chunk-AEDVYYGD.mjs +0 -11
  90. package/dist/chunk-AUQDI5BS.mjs +0 -43
  91. package/dist/chunk-BDSW5PHM.mjs +0 -37
  92. package/dist/chunk-BKL7NCUB.mjs +0 -412
  93. package/dist/chunk-BUWAD67L.mjs +0 -505
  94. package/dist/chunk-BXEVML7N.mjs +0 -169
  95. package/dist/chunk-C6RQ3AEU.mjs +0 -51
  96. package/dist/chunk-CQ6QPIZK.mjs +0 -37
  97. package/dist/chunk-D6GQHO6G.mjs +0 -15
  98. package/dist/chunk-EM67QTMR.mjs +0 -0
  99. package/dist/chunk-I6NG5GNL.mjs +0 -8
  100. package/dist/chunk-IE6LYVIA.mjs +0 -26
  101. package/dist/chunk-IKRWBDG5.mjs +0 -42
  102. package/dist/chunk-JTXYKO3U.mjs +0 -38
  103. package/dist/chunk-KYQRWJ3U.mjs +0 -49
  104. package/dist/chunk-MB7C7VQF.mjs +0 -0
  105. package/dist/chunk-MQUVYMNQ.mjs +0 -0
  106. package/dist/chunk-NHWWORCH.mjs +0 -72
  107. package/dist/chunk-NI6XBZLI.mjs +0 -83
  108. package/dist/chunk-OGKZ575S.mjs +0 -136
  109. package/dist/chunk-QQ57OGQ2.mjs +0 -0
  110. package/dist/chunk-SSE4MFNN.mjs +0 -278
  111. package/dist/chunk-TET3DJEO.mjs +0 -199
  112. package/dist/chunk-TVLV7C74.mjs +0 -207
  113. package/dist/chunk-UEZNZBJO.mjs +0 -104
  114. package/dist/chunk-WJTVYLKW.mjs +0 -782
  115. package/dist/chunk-WTICJPDB.mjs +0 -0
  116. package/dist/chunk-XNEIWM4O.mjs +0 -0
  117. package/dist/chunk-XRLHLAOW.mjs +0 -51
  118. package/dist/chunk-Z4FZ7W6L.mjs +0 -39
  119. package/dist/chunk-Z7RFCADT.mjs +0 -0
  120. package/dist/chunk-ZHXCVRZX.mjs +0 -0
  121. package/dist/core/operations/generated/types.d.ts +0 -908
  122. package/dist/core/operations/generated/types.mjs +0 -63
  123. package/dist/core/types/encodings.d.ts +0 -11
  124. package/dist/core/types/encodings.mjs +0 -0
@@ -0,0 +1,9 @@
1
+ interface CommitmentEncoding {
2
+ erasure_n: number;
3
+ erasure_k: number;
4
+ erasure_d: number;
5
+ chunkSizeBytes: number;
6
+ enumIndex: number;
7
+ }
8
+
9
+ export type { CommitmentEncoding };
@@ -0,0 +1,298 @@
1
+ // src/node/parallel/commitment_worker.ts
2
+ import { parentPort } from "worker_threads";
3
+ import { Hex as Hex3 } from "@aptos-labs/ts-sdk";
4
+
5
+ // src/core/commitments.ts
6
+ import { Hex as Hex2 } from "@aptos-labs/ts-sdk";
7
+ import { z } from "zod";
8
+
9
+ // src/core/erasure/constants.ts
10
+ var ERASURE_CODE_PARAMS = {
11
+ ["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
12
+ // total chunks (data + parity)
13
+ erasure_n: 16,
14
+ // data chunks
15
+ erasure_k: 10,
16
+ // helper nodes
17
+ erasure_d: 13,
18
+ // enum index
19
+ enumIndex: 0
20
+ },
21
+ ["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
22
+ // total chunks (data + parity)
23
+ erasure_n: 4,
24
+ // data chunks
25
+ erasure_k: 2,
26
+ // helper nodes
27
+ erasure_d: 3,
28
+ // enum index
29
+ enumIndex: 1
30
+ }
31
+ };
32
+ var DEFAULT_ERASURE_N = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_n;
33
+ var DEFAULT_ERASURE_K = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_k;
34
+ var DEFAULT_ERASURE_D = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_d;
35
+ var DEFAULT_ERASURE_M = DEFAULT_ERASURE_N - DEFAULT_ERASURE_K;
36
+
37
+ // src/core/chunk.ts
38
+ var CHUNK_SIZE_PARAMS = {
39
+ ["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */]: {
40
+ // 1MiB
41
+ chunkSizeBytes: 1 * 1024 * 1024,
42
+ // 10MiB
43
+ chunksetSizeBytes: 10 * 1024 * 1024
44
+ },
45
+ ["ChunkSet2MiB_Chunk1MiB" /* ChunkSet2MiB_Chunk1MiB */]: {
46
+ // 1MiB
47
+ chunkSizeBytes: 1 * 1024 * 1024,
48
+ // 2MiB
49
+ chunksetSizeBytes: 2 * 1024 * 1024
50
+ }
51
+ };
52
+ var DEFAULT_CHUNK_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunkSizeBytes;
53
+ var DEFAULT_CHUNKSET_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunksetSizeBytes;
54
+ var ERASURE_CODE_AND_CHUNK_MAPPING = {
55
+ ["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
56
+ ...CHUNK_SIZE_PARAMS.ChunkSet10MiB_Chunk1MiB
57
+ },
58
+ ["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
59
+ ...CHUNK_SIZE_PARAMS.ChunkSet2MiB_Chunk1MiB
60
+ }
61
+ };
62
+
63
+ // src/core/utils.ts
64
+ import {
65
+ AccountAddress,
66
+ Hex
67
+ } from "@aptos-labs/ts-sdk";
68
+ function zeroPadBytes(buffer, desiredLength) {
69
+ if (buffer.byteLength === desiredLength) {
70
+ return buffer;
71
+ }
72
+ if (buffer.byteLength > desiredLength) {
73
+ return buffer.subarray(0, desiredLength);
74
+ }
75
+ const paddedBuffer = new Uint8Array(desiredLength);
76
+ paddedBuffer.set(buffer);
77
+ return paddedBuffer;
78
+ }
79
+ async function concatHashes(parts) {
80
+ const chunks = parts.map((part) => Hex.fromHexInput(part).toUint8Array());
81
+ const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
82
+ const combined = new Uint8Array(totalLength);
83
+ let offset = 0;
84
+ for (const chunk of chunks) {
85
+ combined.set(chunk, offset);
86
+ offset += chunk.byteLength;
87
+ }
88
+ return Hex.fromHexInput(
89
+ new Uint8Array(await crypto.subtle.digest("SHA-256", combined))
90
+ );
91
+ }
92
+
93
+ // src/core/commitments.ts
94
+ var ChunksetCommitmentSchema = z.object({
95
+ // Chunkset root (vector commitment of child chunks)
96
+ chunkset_root: z.string(),
97
+ // the size is known statically from the current configuration
98
+ chunk_commitments: z.array(z.string())
99
+ }).refine(
100
+ (data) => {
101
+ return data.chunk_commitments.length === DEFAULT_ERASURE_K + DEFAULT_ERASURE_M;
102
+ },
103
+ {
104
+ message: `Chunkset must have exactly ${DEFAULT_ERASURE_K + DEFAULT_ERASURE_M} chunks (ERASURE_K + ERASURE_M = ${DEFAULT_ERASURE_K} + ${DEFAULT_ERASURE_M})`,
105
+ path: ["chunk_commitments"]
106
+ }
107
+ );
108
+ function expectedTotalChunksets(rawSize, chunksetSize = DEFAULT_CHUNKSET_SIZE_BYTES) {
109
+ if (chunksetSize <= 0) {
110
+ throw new Error("chunksetSize must be positive");
111
+ }
112
+ if (rawSize === 0) return 1;
113
+ return Math.ceil(rawSize / chunksetSize);
114
+ }
115
+ var BlobCommitmentsSchema = z.object({
116
+ schema_version: z.string(),
117
+ raw_data_size: z.number(),
118
+ // FIXME I am not sure about this being here, or if it should be somewhere else
119
+ blob_merkle_root: z.string(),
120
+ chunkset_commitments: z.array(ChunksetCommitmentSchema)
121
+ }).refine(
122
+ (data) => {
123
+ return expectedTotalChunksets(data.raw_data_size) === data.chunkset_commitments.length;
124
+ },
125
+ {
126
+ message: "Total chunkset count mismatches with raw data size",
127
+ // FIXME put more details in here
128
+ path: ["chunkset_commitments"]
129
+ }
130
+ );
131
+ async function generateMerkleRoot(leafHashes) {
132
+ if (!leafHashes.length) {
133
+ throw new Error(
134
+ "An empty array cannot be used to construct a Merkle tree."
135
+ );
136
+ }
137
+ const zeroArray = new Uint8Array(leafHashes[0].toUint8Array().length);
138
+ const zeroBytes = Hex2.fromHexInput(zeroArray);
139
+ let currentLeaves = leafHashes;
140
+ while (currentLeaves.length > 1) {
141
+ if (currentLeaves.length % 2 !== 0) {
142
+ currentLeaves.push(zeroBytes);
143
+ }
144
+ const nextLeaves = [];
145
+ for (let i = 0; i < currentLeaves.length; i += 2) {
146
+ nextLeaves.push(
147
+ await concatHashes([
148
+ currentLeaves[i].toUint8Array(),
149
+ currentLeaves[i + 1].toUint8Array()
150
+ ])
151
+ );
152
+ }
153
+ currentLeaves = nextLeaves;
154
+ }
155
+ return currentLeaves[0];
156
+ }
157
+
158
+ // src/core/erasure/clay-codes.ts
159
+ import {
160
+ createDecoder,
161
+ createEncoder
162
+ } from "@shelby-protocol/clay-codes";
163
+ function getTotalChunks(config) {
164
+ return config.erasure_n;
165
+ }
166
+ var ClayErasureCodingProvider = class _ClayErasureCodingProvider {
167
+ config;
168
+ encoderCache;
169
+ decoderCache;
170
+ lastFunction;
171
+ constructor(config) {
172
+ this.config = config;
173
+ this.lastFunction = "none" /* NONE */;
174
+ }
175
+ /**
176
+ * Static factory method to create an initialized ClayErasureCodingProvider
177
+ */
178
+ static async create(config) {
179
+ const provider = new _ClayErasureCodingProvider(config);
180
+ [provider.encoderCache, provider.decoderCache] = await Promise.all([
181
+ createEncoder({
182
+ n: getTotalChunks(config),
183
+ k: config.erasure_k,
184
+ d: config.erasure_d,
185
+ chunkSizeBytes: config.chunkSizeBytes
186
+ }),
187
+ createDecoder({
188
+ n: getTotalChunks(config),
189
+ k: config.erasure_k,
190
+ d: config.erasure_d,
191
+ chunkSizeBytes: config.chunkSizeBytes,
192
+ erasedChunkIndexes: []
193
+ })
194
+ ]);
195
+ return provider;
196
+ }
197
+ encode(data) {
198
+ const { erasure_k, chunkSizeBytes } = this.config;
199
+ if (!this.encoderCache) {
200
+ throw new Error("Encoder cache is missing");
201
+ }
202
+ const systematicCapacity = erasure_k * chunkSizeBytes;
203
+ if (data.byteLength > systematicCapacity) {
204
+ throw new Error(
205
+ `Data size ${data.byteLength} bytes exceeds systematic capacity ${systematicCapacity} bytes (erasure_k=${erasure_k} * chunkSizeBytes=${chunkSizeBytes}).`
206
+ );
207
+ }
208
+ if (this.requiresPadding(data.length)) {
209
+ throw new Error(
210
+ `Data size ${data.length} bytes does not match systematic capacity ${systematicCapacity} bytes (erasure_k=${erasure_k} * chunkSizeBytes=${chunkSizeBytes}). Provide pre-padded data before encoding.`
211
+ );
212
+ }
213
+ this.lastFunction = "encoded" /* ENCODED */;
214
+ return this.encoderCache.erasureCode(data);
215
+ }
216
+ decode(available, config) {
217
+ if (!this.decoderCache) {
218
+ throw new Error("Decoder cache is missing");
219
+ }
220
+ const { erasure_k, chunkSizeBytes } = this.config;
221
+ if (available.length < erasure_k) {
222
+ throw new Error(
223
+ `Insufficient available chunks: need at least ${erasure_k}, received ${available.length}.`
224
+ );
225
+ }
226
+ for (let i = 0; i < available.length; i++) {
227
+ const chunk = available[i];
228
+ if (chunk.length !== chunkSizeBytes) {
229
+ throw new Error(
230
+ `Chunk ${i} size ${chunk.length} bytes does not match expected chunkSizeBytes ${chunkSizeBytes}.`
231
+ );
232
+ }
233
+ }
234
+ this.lastFunction = "decoded" /* DECODED */;
235
+ return this.decoderCache.decode(available, config);
236
+ }
237
+ getChunkMerkleRoots() {
238
+ if (this.decoderCache && this.lastFunction === "decoded" /* DECODED */)
239
+ return this.decoderCache.getChunkMerkleRoots();
240
+ if (this.encoderCache && this.lastFunction === "encoded" /* ENCODED */)
241
+ return this.encoderCache.getChunkMerkleRoots();
242
+ throw new Error(
243
+ "You must call encode or decode before calling getChunkMerkleRoots"
244
+ );
245
+ }
246
+ /**
247
+ * Determines if data can be erasure coded as-is or requires padding.
248
+ *
249
+ * Data can be erasure coded without padding if its size exactly matches
250
+ * the total systematic data capacity (k * chunkSizeBytes).
251
+ *
252
+ * @param dataSize - Size of the data in bytes
253
+ * @returns true if data needs padding, false if it can be coded as-is
254
+ */
255
+ requiresPadding(dataSize) {
256
+ const { erasure_k, chunkSizeBytes } = this.config;
257
+ const systematicCapacity = erasure_k * chunkSizeBytes;
258
+ return dataSize !== systematicCapacity;
259
+ }
260
+ };
261
+
262
+ // src/node/parallel/commitment_worker.ts
263
+ async function encodeChunksetForCommitment(chunksetData, encoding) {
264
+ const chunksetSizeBytes = encoding.erasure_k * encoding.chunkSizeBytes;
265
+ const paddedChunkset = zeroPadBytes(chunksetData, chunksetSizeBytes);
266
+ const provider = await getProvider(encoding);
267
+ provider.encode(paddedChunkset);
268
+ const chunkRoots = provider.getChunkMerkleRoots().map((root) => Hex3.fromHexInput(root));
269
+ const chunksetRoot = await generateMerkleRoot(chunkRoots);
270
+ return {
271
+ chunkset_root: chunksetRoot.toString(),
272
+ chunk_commitments: chunkRoots.map((r) => r.toString())
273
+ };
274
+ }
275
+ var cachedProvider;
276
+ var cachedEncodingKey;
277
+ async function getProvider(encoding) {
278
+ const key = `${encoding.erasure_n},${encoding.erasure_k},${encoding.erasure_d},${encoding.chunkSizeBytes},${encoding.enumIndex}`;
279
+ if (!cachedProvider || cachedEncodingKey !== key) {
280
+ cachedProvider = await ClayErasureCodingProvider.create(encoding);
281
+ cachedEncodingKey = key;
282
+ }
283
+ return cachedProvider;
284
+ }
285
+ parentPort?.on("message", async (input) => {
286
+ try {
287
+ const result = await encodeChunksetForCommitment(
288
+ input.chunksetData,
289
+ input.encoding
290
+ );
291
+ parentPort?.postMessage({ success: true, result });
292
+ } catch (error) {
293
+ parentPort?.postMessage({
294
+ success: false,
295
+ error: error instanceof Error ? error.message : String(error)
296
+ });
297
+ }
298
+ });
@@ -0,0 +1,35 @@
1
+ import { Worker } from 'node:worker_threads';
2
+ import { ChunksetCommitment } from '../../core/commitments.js';
3
+ import { CommitmentEncoding } from './commitment_worker.js';
4
+ import { WorkerPool, WorkerPoolTask } from './worker_pool.js';
5
+ import '@aptos-labs/ts-sdk';
6
+ import 'zod';
7
+ import '../../clay-codes-DdXABBDx.js';
8
+ import '@shelby-protocol/clay-codes';
9
+
10
+ interface CommitmentTask extends WorkerPoolTask<ChunksetCommitment> {
11
+ chunksetIdx: number;
12
+ chunksetData: Uint8Array;
13
+ encoding: CommitmentEncoding;
14
+ }
15
+ /**
16
+ * Worker pool that encodes chunksets in parallel and returns their commitments.
17
+ *
18
+ * Unlike the RPC server's EncodeWorkerPool, this pool does not write encoded
19
+ * chunks to disk — it only computes the ChunksetCommitment. This makes it
20
+ * suitable for the CLI and any other caller that only needs commitments.
21
+ */
22
+ declare class CommitmentWorkerPool extends WorkerPool<CommitmentTask, ChunksetCommitment> {
23
+ constructor(maxWorkers: number);
24
+ protected postTaskMessage(worker: Worker, task: CommitmentTask): void;
25
+ /**
26
+ * Dispatch a chunkset to a worker for encoding. Returns a promise that
27
+ * resolves to the chunkset's commitment once the worker finishes.
28
+ *
29
+ * The caller must ensure `chunksetData` is not used after this call
30
+ * (its underlying buffer is transferred to the worker thread).
31
+ */
32
+ encodeChunkset(chunksetIdx: number, chunksetData: Uint8Array, encoding: CommitmentEncoding): Promise<ChunksetCommitment>;
33
+ }
34
+
35
+ export { CommitmentWorkerPool };
@@ -0,0 +1,286 @@
1
+ // src/node/parallel/worker_pool.ts
2
+ import fs from "fs";
3
+ import path from "path";
4
+ import { fileURLToPath } from "url";
5
+ import { Worker } from "worker_threads";
6
+ var WorkerPool = class {
7
+ workers = [];
8
+ queue = [];
9
+ maxWorkers;
10
+ workerPath;
11
+ isTerminating = false;
12
+ activeTasks = /* @__PURE__ */ new Set();
13
+ inFlightCount = 0;
14
+ slotWaiters = [];
15
+ constructor(maxWorkers, workerPath) {
16
+ if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) {
17
+ throw new Error(
18
+ `maxWorkers must be a positive safe integer, got ${maxWorkers}`
19
+ );
20
+ }
21
+ this.maxWorkers = maxWorkers;
22
+ this.workerPath = workerPath;
23
+ }
24
+ /**
25
+ * Resolve the path to a worker script relative to the calling module.
26
+ *
27
+ * Pass `import.meta.url` from the subclass file along with the script name
28
+ * (without extension). The extension is inferred from the calling module's
29
+ * own extension so that both the TypeScript source (.ts) and the compiled
30
+ * output (.mjs / .js) are handled correctly.
31
+ *
32
+ * @example
33
+ * constructor(maxWorkers: number) {
34
+ * super(maxWorkers, WorkerPool.resolveWorkerPath(import.meta.url, "my_worker"));
35
+ * }
36
+ */
37
+ static resolveWorkerPath(importMetaUrl, scriptName) {
38
+ const currentPath = fileURLToPath(importMetaUrl);
39
+ const dir = path.dirname(currentPath);
40
+ const ext = path.extname(currentPath);
41
+ const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
42
+ return path.join(dir, `${scriptName}${workerExt}`);
43
+ }
44
+ initialize() {
45
+ this.isTerminating = false;
46
+ if (!fs.existsSync(this.workerPath)) {
47
+ throw new Error(
48
+ `Could not find worker script at ${this.workerPath}. Running as ${path.extname(this.workerPath)}, expected worker at same extension.`
49
+ );
50
+ }
51
+ for (let i = 0; i < this.maxWorkers; i++) {
52
+ this.workers.push({ worker: new Worker(this.workerPath), busy: false });
53
+ }
54
+ }
55
+ isInitialized() {
56
+ return this.workers.length > 0;
57
+ }
58
+ /**
59
+ * Acquire a back-pressure slot before submitting work to the pool.
60
+ * Resolves immediately if fewer than `maxWorkers` chunksets are in flight;
61
+ * otherwise waits until a slot is released. Callers must call `releaseSlot()`
62
+ * when the submitted task settles (resolve or reject).
63
+ *
64
+ * Moving back-pressure into the pool ensures a single global bound on
65
+ * buffered memory regardless of how many concurrent callers share the pool.
66
+ */
67
+ acquireSlot() {
68
+ if (this.inFlightCount < this.maxWorkers) {
69
+ this.inFlightCount++;
70
+ return Promise.resolve();
71
+ }
72
+ return new Promise((resolve, reject) => {
73
+ this.slotWaiters.push({
74
+ resolve: () => {
75
+ this.inFlightCount++;
76
+ resolve();
77
+ },
78
+ reject
79
+ });
80
+ });
81
+ }
82
+ releaseSlot() {
83
+ this.inFlightCount--;
84
+ const next = this.slotWaiters.shift();
85
+ if (next) {
86
+ next.resolve();
87
+ }
88
+ }
89
+ enqueueTask(task) {
90
+ this.activeTasks.add(task);
91
+ if (this.isTerminating) {
92
+ this.activeTasks.delete(task);
93
+ task.reject(
94
+ new Error("Worker pool is being terminated - cannot process new tasks")
95
+ );
96
+ return;
97
+ }
98
+ const availableWorker = this.workers.find((w) => !w.busy);
99
+ if (availableWorker) {
100
+ this.executeTask(availableWorker, task);
101
+ } else {
102
+ this.queue.push(task);
103
+ }
104
+ }
105
+ processNextQueuedTask(preferredWorker) {
106
+ const nextTask = this.queue.shift();
107
+ if (!nextTask) return;
108
+ if (preferredWorker && !preferredWorker.busy && preferredWorker.worker) {
109
+ this.executeTask(preferredWorker, nextTask);
110
+ return;
111
+ }
112
+ const availableWorker = this.workers.find((w) => !w.busy && w.worker);
113
+ if (availableWorker) {
114
+ this.executeTask(availableWorker, nextTask);
115
+ } else {
116
+ this.queue.unshift(nextTask);
117
+ }
118
+ }
119
+ rejectAllQueuedTasks(error) {
120
+ const tasks = this.queue.splice(0);
121
+ for (const task of tasks) {
122
+ this.activeTasks.delete(task);
123
+ task.reject(error);
124
+ }
125
+ }
126
+ rejectAllActiveTasks(error) {
127
+ const tasks = Array.from(this.activeTasks);
128
+ this.activeTasks.clear();
129
+ for (const task of tasks) {
130
+ task.reject(error);
131
+ }
132
+ }
133
+ executeTask(workerWrapper, task) {
134
+ workerWrapper.busy = true;
135
+ const cleanup = () => {
136
+ workerWrapper.worker.off("message", messageHandler);
137
+ workerWrapper.worker.off("error", errorHandler);
138
+ workerWrapper.worker.off("exit", exitHandler);
139
+ workerWrapper.busy = false;
140
+ };
141
+ const messageHandler = (message) => {
142
+ cleanup();
143
+ this.activeTasks.delete(task);
144
+ if (message.success && message.result !== void 0) {
145
+ task.resolve(message.result);
146
+ } else {
147
+ task.reject(new Error(message.error ?? "Worker processing failed"));
148
+ }
149
+ this.processNextQueuedTask(workerWrapper);
150
+ };
151
+ const errorHandler = (error) => {
152
+ workerWrapper.worker.off("message", messageHandler);
153
+ workerWrapper.worker.off("error", errorHandler);
154
+ workerWrapper.busy = false;
155
+ this.activeTasks.delete(task);
156
+ task.reject(error);
157
+ };
158
+ const exitHandler = (exitCode) => {
159
+ workerWrapper.worker.off("message", messageHandler);
160
+ workerWrapper.worker.off("error", errorHandler);
161
+ workerWrapper.worker.off("exit", exitHandler);
162
+ workerWrapper.busy = false;
163
+ this.activeTasks.delete(task);
164
+ task.reject(
165
+ new Error(`Worker terminated unexpectedly with exit code ${exitCode}`)
166
+ );
167
+ if (this.isTerminating) return;
168
+ try {
169
+ const newWorker = new Worker(this.workerPath);
170
+ if (this.isTerminating) {
171
+ newWorker.terminate().catch(() => {
172
+ });
173
+ return;
174
+ }
175
+ workerWrapper.worker = newWorker;
176
+ this.processNextQueuedTask(workerWrapper);
177
+ } catch (_error) {
178
+ const index = this.workers.indexOf(workerWrapper);
179
+ if (index > -1) this.workers.splice(index, 1);
180
+ if (this.workers.length === 0) {
181
+ this.rejectAllQueuedTasks(
182
+ new Error(
183
+ "All workers have died and could not be replaced - worker pool is non-functional"
184
+ )
185
+ );
186
+ return;
187
+ }
188
+ this.processNextQueuedTask();
189
+ }
190
+ };
191
+ workerWrapper.worker.on("message", messageHandler);
192
+ workerWrapper.worker.on("error", errorHandler);
193
+ workerWrapper.worker.on("exit", exitHandler);
194
+ this.postTaskMessage(workerWrapper.worker, task);
195
+ }
196
+ /**
197
+ * Terminate all workers and reject all pending tasks.
198
+ *
199
+ * This pool is NOT designed to be reused after termination.
200
+ * Create a new instance if a fresh pool is needed.
201
+ */
202
+ async terminate() {
203
+ this.isTerminating = true;
204
+ const terminationError = new Error(
205
+ "Worker pool is being terminated - task cancelled"
206
+ );
207
+ this.rejectAllQueuedTasks(terminationError);
208
+ this.rejectAllActiveTasks(terminationError);
209
+ const waiters = this.slotWaiters.splice(0);
210
+ for (const waiter of waiters) {
211
+ waiter.reject(terminationError);
212
+ }
213
+ const terminatePromises = this.workers.map((workerWrapper) => {
214
+ return new Promise((resolve) => {
215
+ const worker = workerWrapper.worker;
216
+ let resolved = false;
217
+ const doResolve = () => {
218
+ if (!resolved) {
219
+ resolved = true;
220
+ resolve();
221
+ }
222
+ };
223
+ let timeoutId;
224
+ const exitHandler = () => {
225
+ if (timeoutId) clearTimeout(timeoutId);
226
+ doResolve();
227
+ };
228
+ worker.removeAllListeners("message");
229
+ worker.removeAllListeners("error");
230
+ worker.once("exit", exitHandler);
231
+ timeoutId = setTimeout(() => {
232
+ worker.removeListener("exit", exitHandler);
233
+ doResolve();
234
+ }, 2e3);
235
+ worker.terminate().catch(() => {
236
+ });
237
+ });
238
+ });
239
+ await Promise.all(terminatePromises);
240
+ this.workers = [];
241
+ this.queue = [];
242
+ this.inFlightCount = 0;
243
+ this.isTerminating = false;
244
+ }
245
+ };
246
+
247
+ // src/node/parallel/commitment_worker_pool.ts
248
+ var CommitmentWorkerPool = class extends WorkerPool {
249
+ constructor(maxWorkers) {
250
+ super(
251
+ maxWorkers,
252
+ WorkerPool.resolveWorkerPath(import.meta.url, "commitment_worker")
253
+ );
254
+ }
255
+ postTaskMessage(worker, task) {
256
+ worker.postMessage(
257
+ {
258
+ chunksetData: task.chunksetData,
259
+ encoding: task.encoding
260
+ },
261
+ [task.chunksetData.buffer]
262
+ );
263
+ }
264
+ /**
265
+ * Dispatch a chunkset to a worker for encoding. Returns a promise that
266
+ * resolves to the chunkset's commitment once the worker finishes.
267
+ *
268
+ * The caller must ensure `chunksetData` is not used after this call
269
+ * (its underlying buffer is transferred to the worker thread).
270
+ */
271
+ encodeChunkset(chunksetIdx, chunksetData, encoding) {
272
+ return new Promise((resolve, reject) => {
273
+ const task = {
274
+ chunksetIdx,
275
+ chunksetData,
276
+ encoding,
277
+ resolve,
278
+ reject
279
+ };
280
+ this.enqueueTask(task);
281
+ });
282
+ }
283
+ };
284
+ export {
285
+ CommitmentWorkerPool
286
+ };
@@ -0,0 +1,24 @@
1
+ import { CommitmentWorkerPool } from './commitment_worker_pool.js';
2
+ import 'node:worker_threads';
3
+ import '../../core/commitments.js';
4
+ import '@aptos-labs/ts-sdk';
5
+ import 'zod';
6
+ import '../../clay-codes-DdXABBDx.js';
7
+ import '@shelby-protocol/clay-codes';
8
+ import './commitment_worker.js';
9
+ import './worker_pool.js';
10
+
11
+ interface CommitmentWorkerPoolOptions {
12
+ /** Worker thread count. Defaults to availableParallelism() - 1, minimum 1. */
13
+ workers?: number;
14
+ }
15
+ declare function defaultCommitmentWorkerCount(): number;
16
+ /**
17
+ * Create and initialize a caller-owned commitment worker pool.
18
+ *
19
+ * The returned pool is ready to pass to `generateCommitmentsParallel`.
20
+ * Callers own its lifecycle and must call `terminate()` during shutdown.
21
+ */
22
+ declare function createDefaultCommitmentWorkerPool(options?: CommitmentWorkerPoolOptions): CommitmentWorkerPool;
23
+
24
+ export { type CommitmentWorkerPoolOptions, createDefaultCommitmentWorkerPool, defaultCommitmentWorkerCount };