@shelby-protocol/sdk 0.3.0 → 0.3.1

package/dist/node/parallel/index.mjs

@@ -0,0 +1,563 @@
+// src/node/parallel/worker_pool.ts
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { Worker } from "worker_threads";
+var WorkerPool = class {
+  workers = [];
+  queue = [];
+  maxWorkers;
+  workerPath;
+  isTerminating = false;
+  activeTasks = /* @__PURE__ */ new Set();
+  inFlightCount = 0;
+  slotWaiters = [];
+  constructor(maxWorkers, workerPath) {
+    if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) {
+      throw new Error(
+        `maxWorkers must be a positive safe integer, got ${maxWorkers}`
+      );
+    }
+    this.maxWorkers = maxWorkers;
+    this.workerPath = workerPath;
+  }
+  /**
+   * Resolve the path to a worker script relative to the calling module.
+   *
+   * Pass `import.meta.url` from the subclass file along with the script name
+   * (without extension). The extension is inferred from the calling module's
+   * own extension so that both the TypeScript source (.ts) and the compiled
+   * output (.mjs / .js) are handled correctly.
+   *
+   * @example
+   * constructor(maxWorkers: number) {
+   *   super(maxWorkers, WorkerPool.resolveWorkerPath(import.meta.url, "my_worker"));
+   * }
+   */
+  static resolveWorkerPath(importMetaUrl, scriptName) {
+    const currentPath = fileURLToPath(importMetaUrl);
+    const dir = path.dirname(currentPath);
+    const ext = path.extname(currentPath);
+    const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
+    return path.join(dir, `${scriptName}${workerExt}`);
+  }
+  initialize() {
+    this.isTerminating = false;
+    if (!fs.existsSync(this.workerPath)) {
+      throw new Error(
+        `Could not find worker script at ${this.workerPath}. Running as ${path.extname(this.workerPath)}, expected worker at same extension.`
+      );
+    }
+    for (let i = 0; i < this.maxWorkers; i++) {
+      this.workers.push({ worker: new Worker(this.workerPath), busy: false });
+    }
+  }
+  isInitialized() {
+    return this.workers.length > 0;
+  }
+  /**
+   * Acquire a back-pressure slot before submitting work to the pool.
+   * Resolves immediately if fewer than `maxWorkers` chunksets are in flight;
+   * otherwise waits until a slot is released. Callers must call `releaseSlot()`
+   * when the submitted task settles (resolve or reject).
+   *
+   * Moving back-pressure into the pool ensures a single global bound on
+   * buffered memory regardless of how many concurrent callers share the pool.
+   */
+  acquireSlot() {
+    if (this.inFlightCount < this.maxWorkers) {
+      this.inFlightCount++;
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      this.slotWaiters.push({
+        resolve: () => {
+          this.inFlightCount++;
+          resolve();
+        },
+        reject
+      });
+    });
+  }
+  releaseSlot() {
+    this.inFlightCount--;
+    const next = this.slotWaiters.shift();
+    if (next) {
+      next.resolve();
+    }
+  }
+  enqueueTask(task) {
+    this.activeTasks.add(task);
+    if (this.isTerminating) {
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error("Worker pool is being terminated - cannot process new tasks")
+      );
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy);
+    if (availableWorker) {
+      this.executeTask(availableWorker, task);
+    } else {
+      this.queue.push(task);
+    }
+  }
+  processNextQueuedTask(preferredWorker) {
+    const nextTask = this.queue.shift();
+    if (!nextTask) return;
+    if (preferredWorker && !preferredWorker.busy && preferredWorker.worker) {
+      this.executeTask(preferredWorker, nextTask);
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy && w.worker);
+    if (availableWorker) {
+      this.executeTask(availableWorker, nextTask);
+    } else {
+      this.queue.unshift(nextTask);
+    }
+  }
+  rejectAllQueuedTasks(error) {
+    const tasks = this.queue.splice(0);
+    for (const task of tasks) {
+      this.activeTasks.delete(task);
+      task.reject(error);
+    }
+  }
+  rejectAllActiveTasks(error) {
+    const tasks = Array.from(this.activeTasks);
+    this.activeTasks.clear();
+    for (const task of tasks) {
+      task.reject(error);
+    }
+  }
+  executeTask(workerWrapper, task) {
+    workerWrapper.busy = true;
+    const cleanup = () => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+    };
+    const messageHandler = (message) => {
+      cleanup();
+      this.activeTasks.delete(task);
+      if (message.success && message.result !== void 0) {
+        task.resolve(message.result);
+      } else {
+        task.reject(new Error(message.error ?? "Worker processing failed"));
+      }
+      this.processNextQueuedTask(workerWrapper);
+    };
+    const errorHandler = (error) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(error);
+    };
+    const exitHandler = (exitCode) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error(`Worker terminated unexpectedly with exit code ${exitCode}`)
+      );
+      if (this.isTerminating) return;
+      try {
+        const newWorker = new Worker(this.workerPath);
+        if (this.isTerminating) {
+          newWorker.terminate().catch(() => {
+          });
+          return;
+        }
+        workerWrapper.worker = newWorker;
+        this.processNextQueuedTask(workerWrapper);
+      } catch (_error) {
+        const index = this.workers.indexOf(workerWrapper);
+        if (index > -1) this.workers.splice(index, 1);
+        if (this.workers.length === 0) {
+          this.rejectAllQueuedTasks(
+            new Error(
+              "All workers have died and could not be replaced - worker pool is non-functional"
+            )
+          );
+          return;
+        }
+        this.processNextQueuedTask();
+      }
+    };
+    workerWrapper.worker.on("message", messageHandler);
+    workerWrapper.worker.on("error", errorHandler);
+    workerWrapper.worker.on("exit", exitHandler);
+    this.postTaskMessage(workerWrapper.worker, task);
+  }
+  /**
+   * Terminate all workers and reject all pending tasks.
+   *
+   * This pool is NOT designed to be reused after termination.
+   * Create a new instance if a fresh pool is needed.
+   */
+  async terminate() {
+    this.isTerminating = true;
+    const terminationError = new Error(
+      "Worker pool is being terminated - task cancelled"
+    );
+    this.rejectAllQueuedTasks(terminationError);
+    this.rejectAllActiveTasks(terminationError);
+    const waiters = this.slotWaiters.splice(0);
+    for (const waiter of waiters) {
+      waiter.reject(terminationError);
+    }
+    const terminatePromises = this.workers.map((workerWrapper) => {
+      return new Promise((resolve) => {
+        const worker = workerWrapper.worker;
+        let resolved = false;
+        const doResolve = () => {
+          if (!resolved) {
+            resolved = true;
+            resolve();
+          }
+        };
+        let timeoutId;
+        const exitHandler = () => {
+          if (timeoutId) clearTimeout(timeoutId);
+          doResolve();
+        };
+        worker.removeAllListeners("message");
+        worker.removeAllListeners("error");
+        worker.once("exit", exitHandler);
+        timeoutId = setTimeout(() => {
+          worker.removeListener("exit", exitHandler);
+          doResolve();
+        }, 2e3);
+        worker.terminate().catch(() => {
+        });
+      });
+    });
+    await Promise.all(terminatePromises);
+    this.workers = [];
+    this.queue = [];
+    this.inFlightCount = 0;
+    this.isTerminating = false;
+  }
+};
+
+// src/node/parallel/commitment_worker_pool.ts
+var CommitmentWorkerPool = class extends WorkerPool {
+  constructor(maxWorkers) {
+    super(
+      maxWorkers,
+      WorkerPool.resolveWorkerPath(import.meta.url, "commitment_worker")
+    );
+  }
+  postTaskMessage(worker, task) {
+    worker.postMessage(
+      {
+        chunksetData: task.chunksetData,
+        encoding: task.encoding
+      },
+      [task.chunksetData.buffer]
+    );
+  }
+  /**
+   * Dispatch a chunkset to a worker for encoding. Returns a promise that
+   * resolves to the chunkset's commitment once the worker finishes.
+   *
+   * The caller must ensure `chunksetData` is not used after this call
+   * (its underlying buffer is transferred to the worker thread).
+   */
+  encodeChunkset(chunksetIdx, chunksetData, encoding) {
+    return new Promise((resolve, reject) => {
+      const task = {
+        chunksetIdx,
+        chunksetData,
+        encoding,
+        resolve,
+        reject
+      };
+      this.enqueueTask(task);
+    });
+  }
+};
+
+// src/node/parallel/default_commitment_worker_pool.ts
+import { availableParallelism } from "os";
+function defaultCommitmentWorkerCount() {
+  return Math.max(1, availableParallelism() - 1);
+}
+function createDefaultCommitmentWorkerPool(options = {}) {
+  const pool = new CommitmentWorkerPool(
+    options.workers ?? defaultCommitmentWorkerCount()
+  );
+  pool.initialize();
+  return pool;
+}
+
+// src/node/parallel/parallel_commitments.ts
+import { Hex as Hex3 } from "@aptos-labs/ts-sdk";
+
+// src/core/commitments.ts
+import { Hex as Hex2 } from "@aptos-labs/ts-sdk";
+import { z } from "zod";
+
+// src/core/erasure/constants.ts
+var ERASURE_CODE_PARAMS = {
+  ["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
+    // total chunks (data + parity)
+    erasure_n: 16,
+    // data chunks
+    erasure_k: 10,
+    // helper nodes
+    erasure_d: 13,
+    // enum index
+    enumIndex: 0
+  },
+  ["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
+    // total chunks (data + parity)
+    erasure_n: 4,
+    // data chunks
+    erasure_k: 2,
+    // helper nodes
+    erasure_d: 3,
+    // enum index
+    enumIndex: 1
+  }
+};
+var DEFAULT_ERASURE_N = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_n;
+var DEFAULT_ERASURE_K = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_k;
+var DEFAULT_ERASURE_D = ERASURE_CODE_PARAMS["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */].erasure_d;
+var DEFAULT_ERASURE_M = DEFAULT_ERASURE_N - DEFAULT_ERASURE_K;
+
+// src/core/chunk.ts
+var CHUNK_SIZE_PARAMS = {
+  ["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */]: {
+    // 1MiB
+    chunkSizeBytes: 1 * 1024 * 1024,
+    // 10MiB
+    chunksetSizeBytes: 10 * 1024 * 1024
+  },
+  ["ChunkSet2MiB_Chunk1MiB" /* ChunkSet2MiB_Chunk1MiB */]: {
+    // 1MiB
+    chunkSizeBytes: 1 * 1024 * 1024,
+    // 2MiB
+    chunksetSizeBytes: 2 * 1024 * 1024
+  }
+};
+var DEFAULT_CHUNK_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunkSizeBytes;
+var DEFAULT_CHUNKSET_SIZE_BYTES = CHUNK_SIZE_PARAMS["ChunkSet10MiB_Chunk1MiB" /* ChunkSet10MiB_Chunk1MiB */].chunksetSizeBytes;
+var ERASURE_CODE_AND_CHUNK_MAPPING = {
+  ["ClayCode_16Total_10Data_13Helper" /* ClayCode_16Total_10Data_13Helper */]: {
+    ...CHUNK_SIZE_PARAMS.ChunkSet10MiB_Chunk1MiB
+  },
+  ["ClayCode_4Total_2Data_3Helper" /* ClayCode_4Total_2Data_3Helper */]: {
+    ...CHUNK_SIZE_PARAMS.ChunkSet2MiB_Chunk1MiB
+  }
+};
+
+// src/core/utils.ts
+import {
+  AccountAddress,
+  Hex
+} from "@aptos-labs/ts-sdk";
+async function* readInChunks(input, chunkSize) {
+  let idx = 0;
+  if (isReadableStream(input)) {
+    const reader = input.getReader();
+    let buffer = new Uint8Array(chunkSize);
+    let bufferWriteOffset = 0;
+    try {
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        if (value === void 0) continue;
+        let srcOffset = 0;
+        while (srcOffset < value.length) {
+          const remainingCapacity = chunkSize - bufferWriteOffset;
+          const bytesToCopy = Math.min(
+            remainingCapacity,
+            value.length - srcOffset
+          );
+          buffer.set(
+            value.subarray(srcOffset, srcOffset + bytesToCopy),
+            bufferWriteOffset
+          );
+          bufferWriteOffset += bytesToCopy;
+          srcOffset += bytesToCopy;
+          if (bufferWriteOffset >= chunkSize) {
+            yield [idx++, buffer];
+            buffer = new Uint8Array(chunkSize);
+            bufferWriteOffset = 0;
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+    if (bufferWriteOffset > 0) {
+      yield [idx++, buffer.subarray(0, bufferWriteOffset)];
+    }
+    return;
+  }
+  const view = toUint8Array(input);
+  for (let offset = 0; offset < view.byteLength; offset += chunkSize) {
+    yield [
+      idx++,
+      view.subarray(offset, Math.min(offset + chunkSize, view.byteLength))
+    ];
+  }
+}
+async function concatHashes(parts) {
+  const chunks = parts.map((part) => Hex.fromHexInput(part).toUint8Array());
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+  const combined = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    combined.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return Hex.fromHexInput(
+    new Uint8Array(await crypto.subtle.digest("SHA-256", combined))
+  );
+}
+function isReadableStream(value) {
+  return typeof value === "object" && value !== null && "getReader" in value && typeof value.getReader === "function";
+}
+function toUint8Array(view) {
+  return view instanceof Uint8Array ? view : new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
+}
+
+// src/core/commitments.ts
+var COMMITMENT_SCHEMA_VERSION = "1.3";
+var ChunksetCommitmentSchema = z.object({
+  // Chunkset root (vector commitment of child chunks)
+  chunkset_root: z.string(),
+  // the size is known statically from the current configuration
+  chunk_commitments: z.array(z.string())
+}).refine(
+  (data) => {
+    return data.chunk_commitments.length === DEFAULT_ERASURE_K + DEFAULT_ERASURE_M;
+  },
+  {
+    message: `Chunkset must have exactly ${DEFAULT_ERASURE_K + DEFAULT_ERASURE_M} chunks (ERASURE_K + ERASURE_M = ${DEFAULT_ERASURE_K} + ${DEFAULT_ERASURE_M})`,
+    path: ["chunk_commitments"]
+  }
+);
+function expectedTotalChunksets(rawSize, chunksetSize = DEFAULT_CHUNKSET_SIZE_BYTES) {
+  if (chunksetSize <= 0) {
+    throw new Error("chunksetSize must be positive");
+  }
+  if (rawSize === 0) return 1;
+  return Math.ceil(rawSize / chunksetSize);
+}
+var BlobCommitmentsSchema = z.object({
+  schema_version: z.string(),
+  raw_data_size: z.number(),
+  // FIXME I am not sure about this being here, or if it should be somewhere else
+  blob_merkle_root: z.string(),
+  chunkset_commitments: z.array(ChunksetCommitmentSchema)
+}).refine(
+  (data) => {
+    return expectedTotalChunksets(data.raw_data_size) === data.chunkset_commitments.length;
+  },
+  {
+    message: "Total chunkset count mismatches with raw data size",
+    // FIXME put more details in here
+    path: ["chunkset_commitments"]
+  }
+);
+async function generateMerkleRoot(leafHashes) {
+  if (!leafHashes.length) {
+    throw new Error(
+      "An empty array cannot be used to construct a Merkle tree."
+    );
+  }
+  const zeroArray = new Uint8Array(leafHashes[0].toUint8Array().length);
+  const zeroBytes = Hex2.fromHexInput(zeroArray);
+  let currentLeaves = leafHashes;
+  while (currentLeaves.length > 1) {
+    if (currentLeaves.length % 2 !== 0) {
+      currentLeaves.push(zeroBytes);
+    }
+    const nextLeaves = [];
+    for (let i = 0; i < currentLeaves.length; i += 2) {
+      nextLeaves.push(
+        await concatHashes([
+          currentLeaves[i].toUint8Array(),
+          currentLeaves[i + 1].toUint8Array()
+        ])
+      );
+    }
+    currentLeaves = nextLeaves;
+  }
+  return currentLeaves[0];
+}
+
+// src/node/parallel/parallel_commitments.ts
+function toCommitmentEncoding(provider) {
+  const cfg = provider.config;
+  if (typeof cfg.erasure_d !== "number" || typeof cfg.enumIndex !== "number") {
+    throw new Error(
+      "generateCommitmentsParallel requires a Clay erasure coding provider (erasure_d and enumIndex must be set on provider.config)"
+    );
+  }
+  return {
+    erasure_n: cfg.erasure_n,
+    erasure_k: cfg.erasure_k,
+    erasure_d: cfg.erasure_d,
+    chunkSizeBytes: cfg.chunkSizeBytes,
+    enumIndex: cfg.enumIndex
+  };
+}
+async function generateCommitmentsParallel(provider, data, options) {
+  const encoding = toCommitmentEncoding(provider);
+  const chunksetSize = encoding.erasure_k * encoding.chunkSizeBytes;
+  const ownPool = options?.pool === void 0;
+  const pool = options?.pool ?? createDefaultCommitmentWorkerPool({ workers: options?.concurrency });
+  if (!pool.isInitialized()) {
+    throw new Error(
+      "The provided CommitmentWorkerPool has not been initialized. Call pool.initialize() before passing it to generateCommitmentsParallel."
+    );
+  }
+  try {
+    const pending = [];
+    let rawDataSize = 0;
+    for await (const [idx, chunkData] of readInChunks(data, chunksetSize)) {
+      rawDataSize += chunkData.length;
+      await pool.acquireSlot();
+      const copy = new Uint8Array(chunkData);
+      const p = pool.encodeChunkset(idx, copy, encoding);
+      pending[idx] = p;
+      p.then(pool.releaseSlot.bind(pool), pool.releaseSlot.bind(pool));
+    }
+    if (rawDataSize === 0) {
+      pending[0] = pool.encodeChunkset(
+        0,
+        new Uint8Array(chunksetSize),
+        encoding
+      );
+    }
+    const chunksetCommitments = await Promise.all(pending);
+    const chunksetHashes = chunksetCommitments.map(
+      (c) => Hex3.fromHexInput(c.chunkset_root)
+    );
+    return {
+      schema_version: COMMITMENT_SCHEMA_VERSION,
+      raw_data_size: rawDataSize,
+      blob_merkle_root: (await generateMerkleRoot(chunksetHashes)).toString(),
+      chunkset_commitments: chunksetCommitments
+    };
+  } finally {
+    if (ownPool) {
+      await pool.terminate();
+    }
+  }
+}
+export {
+  CommitmentWorkerPool,
+  WorkerPool,
+  createDefaultCommitmentWorkerPool,
+  defaultCommitmentWorkerCount,
+  generateCommitmentsParallel
+};

package/dist/node/parallel/parallel_commitments.d.ts

@@ -0,0 +1,32 @@
+import { BlobCommitments } from '../../core/commitments.js';
+import { a as ErasureCodingProvider } from '../../clay-codes-DdXABBDx.js';
+import { CommitmentWorkerPool } from './commitment_worker_pool.js';
+import '@aptos-labs/ts-sdk';
+import 'zod';
+import '@shelby-protocol/clay-codes';
+import 'node:worker_threads';
+import './commitment_worker.js';
+import './worker_pool.js';
+
+/**
+ * Parallel drop-in replacement for `generateCommitments`.
+ *
+ * Chunksets are dispatched to worker threads as they are read from the stream,
+ * so encoding of chunkset N overlaps with reading of chunkset N+1. The final
+ * `BlobCommitments` is identical to what `generateCommitments` would produce.
+ *
+ * @param provider - Erasure coding provider (must be a Clay provider).
+ * @param data - Input data as a stream or buffer.
+ * @param options.concurrency - Number of worker threads. Defaults to
+ *   `availableParallelism() - 1` (minimum 1).
+ * @param options.pool - A caller-managed `CommitmentWorkerPool`. When provided,
+ *   the pool is used as-is and is NOT terminated when the function returns —
+ *   the caller is responsible for calling `pool.terminate()`. When omitted, an
+ *   internal pool is created and terminated automatically.
+ */
+declare function generateCommitmentsParallel(provider: ErasureCodingProvider, data: ReadableStream<Uint8Array> | Uint8Array, options?: {
+    concurrency?: number;
+    pool?: CommitmentWorkerPool;
+}): Promise<BlobCommitments>;
+
+export { generateCommitmentsParallel };