@shelby-protocol/sdk 0.3.0 → 0.4.0

package/dist/node/parallel/commitment_worker.d.ts

@@ -0,0 +1,9 @@
+interface CommitmentEncoding {
+    erasure_n: number;
+    erasure_k: number;
+    erasure_d: number;
+    chunkSizeBytes: number;
+    enumIndex: number;
+}
+
+export type { CommitmentEncoding };

package/dist/node/parallel/commitment_worker.mjs

@@ -0,0 +1,85 @@
+// src/node/parallel/commitment_worker.ts
+import { parentPort } from "worker_threads";
+import { Hex as Hex2 } from "@aptos-labs/ts-sdk";
+
+// src/core/utils.ts
+import {
+  AccountAddress,
+  Hex
+} from "@aptos-labs/ts-sdk";
+function zeroPadBytes(buffer, desiredLength) {
+  if (buffer.byteLength === desiredLength) {
+    return buffer;
+  }
+  if (buffer.byteLength > desiredLength) {
+    return buffer.subarray(0, desiredLength);
+  }
+  const paddedBuffer = new Uint8Array(desiredLength);
+  paddedBuffer.set(buffer);
+  return paddedBuffer;
+}
+
+// src/node/parallel/commitment_worker.ts
+var ffiAvailable = null;
+async function tryCreateFFIEncoder(opts) {
+  if (ffiAvailable === false) return null;
+  try {
+    const ffiPath = "@shelby-protocol/clay-codes-ffi";
+    const { createEncoder: createFFIEncoder } = await import(ffiPath);
+    const encoder = await createFFIEncoder(opts);
+    ffiAvailable = true;
+    return encoder;
+  } catch {
+    ffiAvailable = false;
+    return null;
+  }
+}
+async function createWASMEncoder(opts) {
+  const { createEncoder } = await import("@shelby-protocol/clay-codes");
+  return createEncoder(opts);
+}
+async function encodeChunksetForCommitment(chunksetData, encoding) {
+  const chunksetSizeBytes = encoding.erasure_k * encoding.chunkSizeBytes;
+  const paddedChunkset = zeroPadBytes(chunksetData, chunksetSizeBytes);
+  const encoder = await getEncoder(encoding);
+  encoder.erasureCode(paddedChunkset);
+  const commitment = encoder.getMerkleCommitment();
+  const chunkRoots = commitment.chunkRoots.map(
+    (root) => Hex2.fromHexInput(root).toString()
+  );
+  const chunksetRoot = Hex2.fromHexInput(commitment.chunksetRoot).toString();
+  return {
+    chunkset_root: chunksetRoot,
+    chunk_commitments: chunkRoots
+  };
+}
+var cachedEncoder;
+var cachedEncodingKey;
+async function getEncoder(encoding) {
+  const key = `${encoding.erasure_n},${encoding.erasure_k},${encoding.erasure_d},${encoding.chunkSizeBytes},${encoding.enumIndex}`;
+  if (!cachedEncoder || cachedEncodingKey !== key) {
+    const opts = {
+      n: encoding.erasure_n,
+      k: encoding.erasure_k,
+      d: encoding.erasure_d,
+      chunkSizeBytes: encoding.chunkSizeBytes
+    };
+    cachedEncoder = await tryCreateFFIEncoder(opts) ?? await createWASMEncoder(opts);
+    cachedEncodingKey = key;
+  }
+  return cachedEncoder;
+}
+parentPort?.on("message", async (input) => {
+  try {
+    const result = await encodeChunksetForCommitment(
+      input.chunksetData,
+      input.encoding
+    );
+    parentPort?.postMessage({ success: true, result });
+  } catch (error) {
+    parentPort?.postMessage({
+      success: false,
+      error: error instanceof Error ? error.message : String(error)
+    });
+  }
+});

package/dist/node/parallel/commitment_worker_pool.d.ts

@@ -0,0 +1,35 @@
+import { Worker } from 'node:worker_threads';
+import { ChunksetCommitment } from '../../core/commitments.js';
+import { CommitmentEncoding } from './commitment_worker.js';
+import { WorkerPool, WorkerPoolTask } from './worker_pool.js';
+import '@aptos-labs/ts-sdk';
+import 'zod';
+import '../../clay-codes-DdXABBDx.js';
+import '@shelby-protocol/clay-codes';
+
+interface CommitmentTask extends WorkerPoolTask<ChunksetCommitment> {
+    chunksetIdx: number;
+    chunksetData: Uint8Array;
+    encoding: CommitmentEncoding;
+}
+/**
+ * Worker pool that encodes chunksets in parallel and returns their commitments.
+ *
+ * Unlike the RPC server's EncodeWorkerPool, this pool does not write encoded
+ * chunks to disk — it only computes the ChunksetCommitment. This makes it
+ * suitable for the CLI and any other caller that only needs commitments.
+ */
+declare class CommitmentWorkerPool extends WorkerPool<CommitmentTask, ChunksetCommitment> {
+    constructor(maxWorkers: number);
+    protected postTaskMessage(worker: Worker, task: CommitmentTask): void;
+    /**
+     * Dispatch a chunkset to a worker for encoding. Returns a promise that
+     * resolves to the chunkset's commitment once the worker finishes.
+     *
+     * The caller must ensure `chunksetData` is not used after this call
+     * (its underlying buffer is transferred to the worker thread).
+     */
+    encodeChunkset(chunksetIdx: number, chunksetData: Uint8Array, encoding: CommitmentEncoding): Promise<ChunksetCommitment>;
+}
+
+export { CommitmentWorkerPool };

package/dist/node/parallel/commitment_worker_pool.mjs

@@ -0,0 +1,339 @@
+// src/node/parallel/worker_pool.ts
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { Worker } from "worker_threads";
+
+// src/node/parallel/coverage_flush.ts
+import { takeCoverage } from "v8";
+var FLUSH_REQUEST = "__shelby_cov_flush_request__";
+var FLUSH_ACK = "__shelby_cov_flush_ack__";
+var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
+var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
+function flushWorkerCoverage(worker, timeoutMs = 2e3) {
+  if (!coverageEnabled()) return Promise.resolve();
+  return new Promise((resolve) => {
+    const onMessage = (msg) => {
+      if (isType(msg, FLUSH_ACK)) finish();
+    };
+    const finish = () => {
+      clearTimeout(timer);
+      worker.off("message", onMessage);
+      resolve();
+    };
+    const timer = setTimeout(finish, timeoutMs);
+    worker.on("message", onMessage);
+    try {
+      worker.postMessage({ type: FLUSH_REQUEST });
+    } catch {
+      finish();
+    }
+  });
+}
+
+// src/node/parallel/worker_pool.ts
+var WorkerPool = class {
+  workers = [];
+  queue = [];
+  maxWorkers;
+  workerPath;
+  isTerminating = false;
+  activeTasks = /* @__PURE__ */ new Set();
+  inFlightCount = 0;
+  slotWaiters = [];
+  constructor(maxWorkers, workerPath) {
+    if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) {
+      throw new Error(
+        `maxWorkers must be a positive safe integer, got ${maxWorkers}`
+      );
+    }
+    this.maxWorkers = maxWorkers;
+    this.workerPath = workerPath;
+  }
+  /**
+   * Resolve the path to a worker script relative to the calling module.
+   *
+   * Pass `import.meta.url` from the subclass file along with the script name
+   * (without extension). The extension is inferred from the calling module's
+   * own extension so that both the TypeScript source (.ts) and the compiled
+   * output (.mjs / .js) are handled correctly.
+   *
+   * @example
+   * constructor(maxWorkers: number) {
+   *   super(maxWorkers, WorkerPool.resolveWorkerPath(import.meta.url, "my_worker"));
+   * }
+   */
+  static resolveWorkerPath(importMetaUrl, scriptName) {
+    const currentPath = fileURLToPath(importMetaUrl);
+    const dir = path.dirname(currentPath);
+    const ext = path.extname(currentPath);
+    const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
+    if (workerExt === ".ts") {
+      const srcSegment = `${path.sep}src${path.sep}`;
+      const srcIdx = currentPath.indexOf(srcSegment);
+      if (srcIdx !== -1) {
+        const packageRoot = currentPath.slice(0, srcIdx);
+        const relFromSrc = path.relative(path.join(packageRoot, "src"), dir);
+        return path.join(
+          packageRoot,
+          "dist",
+          "workers",
+          relFromSrc,
+          `${scriptName}.js`
+        );
+      }
+    }
+    let workerPath = path.join(dir, `${scriptName}${workerExt}`);
+    if (fs.existsSync(workerPath)) {
+      return workerPath;
+    }
+    workerPath = path.join(dir, "workers", `${scriptName}${workerExt}`);
+    if (fs.existsSync(workerPath)) {
+      return workerPath;
+    }
+    return path.join(dir, `${scriptName}${workerExt}`);
+  }
+  initialize() {
+    this.isTerminating = false;
+    if (!fs.existsSync(this.workerPath)) {
+      throw new Error(
+        `Could not find worker script at ${this.workerPath}. Running as ${path.extname(this.workerPath)}, expected worker at same extension.`
+      );
+    }
+    for (let i = 0; i < this.maxWorkers; i++) {
+      this.workers.push({ worker: new Worker(this.workerPath), busy: false });
+    }
+  }
+  isInitialized() {
+    return this.workers.length > 0;
+  }
+  /**
+   * Acquire a back-pressure slot before submitting work to the pool.
+   * Resolves immediately if fewer than `maxWorkers` chunksets are in flight;
+   * otherwise waits until a slot is released. Callers must call `releaseSlot()`
+   * when the submitted task settles (resolve or reject).
+   *
+   * Moving back-pressure into the pool ensures a single global bound on
+   * buffered memory regardless of how many concurrent callers share the pool.
+   */
+  acquireSlot() {
+    if (this.inFlightCount < this.maxWorkers) {
+      this.inFlightCount++;
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      this.slotWaiters.push({
+        resolve: () => {
+          this.inFlightCount++;
+          resolve();
+        },
+        reject
+      });
+    });
+  }
+  releaseSlot() {
+    this.inFlightCount--;
+    const next = this.slotWaiters.shift();
+    if (next) {
+      next.resolve();
+    }
+  }
+  enqueueTask(task) {
+    this.activeTasks.add(task);
+    if (this.isTerminating) {
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error("Worker pool is being terminated - cannot process new tasks")
+      );
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy);
+    if (availableWorker) {
+      this.executeTask(availableWorker, task);
+    } else {
+      this.queue.push(task);
+    }
+  }
+  processNextQueuedTask(preferredWorker) {
+    const nextTask = this.queue.shift();
+    if (!nextTask) return;
+    if (preferredWorker && !preferredWorker.busy && preferredWorker.worker) {
+      this.executeTask(preferredWorker, nextTask);
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy && w.worker);
+    if (availableWorker) {
+      this.executeTask(availableWorker, nextTask);
+    } else {
+      this.queue.unshift(nextTask);
+    }
+  }
+  rejectAllQueuedTasks(error) {
+    const tasks = this.queue.splice(0);
+    for (const task of tasks) {
+      this.activeTasks.delete(task);
+      task.reject(error);
+    }
+  }
+  rejectAllActiveTasks(error) {
+    const tasks = Array.from(this.activeTasks);
+    this.activeTasks.clear();
+    for (const task of tasks) {
+      task.reject(error);
+    }
+  }
+  executeTask(workerWrapper, task) {
+    workerWrapper.busy = true;
+    const cleanup = () => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+    };
+    const messageHandler = (message) => {
+      cleanup();
+      this.activeTasks.delete(task);
+      if (message.success && message.result !== void 0) {
+        task.resolve(message.result);
+      } else {
+        task.reject(new Error(message.error ?? "Worker processing failed"));
+      }
+      this.processNextQueuedTask(workerWrapper);
+    };
+    const errorHandler = (error) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(error);
+    };
+    const exitHandler = (exitCode) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error(`Worker terminated unexpectedly with exit code ${exitCode}`)
+      );
+      if (this.isTerminating) return;
+      try {
+        const newWorker = new Worker(this.workerPath);
+        if (this.isTerminating) {
+          newWorker.terminate().catch(() => {
+          });
+          return;
+        }
+        workerWrapper.worker = newWorker;
+        this.processNextQueuedTask(workerWrapper);
+      } catch (_error) {
+        const index = this.workers.indexOf(workerWrapper);
+        if (index > -1) this.workers.splice(index, 1);
+        if (this.workers.length === 0) {
+          this.rejectAllQueuedTasks(
+            new Error(
+              "All workers have died and could not be replaced - worker pool is non-functional"
+            )
+          );
+          return;
+        }
+        this.processNextQueuedTask();
+      }
+    };
+    workerWrapper.worker.on("message", messageHandler);
+    workerWrapper.worker.on("error", errorHandler);
+    workerWrapper.worker.on("exit", exitHandler);
+    this.postTaskMessage(workerWrapper.worker, task);
+  }
+  /**
+   * Terminate all workers and reject all pending tasks.
+   *
+   * This pool is NOT designed to be reused after termination.
+   * Create a new instance if a fresh pool is needed.
+   */
+  async terminate() {
+    this.isTerminating = true;
+    const terminationError = new Error(
+      "Worker pool is being terminated - task cancelled"
+    );
+    this.rejectAllQueuedTasks(terminationError);
+    this.rejectAllActiveTasks(terminationError);
+    const waiters = this.slotWaiters.splice(0);
+    for (const waiter of waiters) {
+      waiter.reject(terminationError);
+    }
+    const terminatePromises = this.workers.map(async (workerWrapper) => {
+      const worker = workerWrapper.worker;
+      await flushWorkerCoverage(worker);
+      await new Promise((resolve) => {
+        let resolved = false;
+        const doResolve = () => {
+          if (!resolved) {
+            resolved = true;
+            resolve();
+          }
+        };
+        let timeoutId;
+        const exitHandler = () => {
+          if (timeoutId) clearTimeout(timeoutId);
+          doResolve();
+        };
+        worker.removeAllListeners("message");
+        worker.removeAllListeners("error");
+        worker.once("exit", exitHandler);
+        timeoutId = setTimeout(() => {
+          worker.removeListener("exit", exitHandler);
+          doResolve();
+        }, 2e3);
+        worker.terminate().catch(() => {
+        });
+      });
+    });
+    await Promise.all(terminatePromises);
+    this.workers = [];
+    this.queue = [];
+    this.inFlightCount = 0;
+    this.isTerminating = false;
+  }
+};
+
+// src/node/parallel/commitment_worker_pool.ts
+var CommitmentWorkerPool = class extends WorkerPool {
+  constructor(maxWorkers) {
+    super(
+      maxWorkers,
+      WorkerPool.resolveWorkerPath(import.meta.url, "commitment_worker")
+    );
+  }
+  postTaskMessage(worker, task) {
+    worker.postMessage(
+      {
+        chunksetData: task.chunksetData,
+        encoding: task.encoding
+      },
+      [task.chunksetData.buffer]
+    );
+  }
+  /**
+   * Dispatch a chunkset to a worker for encoding. Returns a promise that
+   * resolves to the chunkset's commitment once the worker finishes.
+   *
+   * The caller must ensure `chunksetData` is not used after this call
+   * (its underlying buffer is transferred to the worker thread).
+   */
+  encodeChunkset(chunksetIdx, chunksetData, encoding) {
+    return new Promise((resolve, reject) => {
+      const task = {
+        chunksetIdx,
+        chunksetData,
+        encoding,
+        resolve,
+        reject
+      };
+      this.enqueueTask(task);
+    });
+  }
+};
+export {
+  CommitmentWorkerPool
+};

package/dist/node/parallel/coverage_flush.d.ts

@@ -0,0 +1,16 @@
+import { MessagePort, Worker } from 'node:worker_threads';
+
+declare function isCoverageFlushMessage(msg: unknown): boolean;
+/**
+ * Worker side: handle flush requests by writing this thread's V8 coverage. Call
+ * once near the top of each worker entry. No-op unless NODE_V8_COVERAGE is set.
+ */
+declare function registerWorkerCoverageFlush(port: MessagePort | null): void;
+/**
+ * Pool side: ask a worker to flush its V8 coverage and wait (briefly) for the ack,
+ * so the write completes before the worker is terminated. No-op unless
+ * NODE_V8_COVERAGE is set; always resolves (on ack or timeout) and never throws.
+ */
+declare function flushWorkerCoverage(worker: Worker, timeoutMs?: number): Promise<void>;
+
+export { flushWorkerCoverage, isCoverageFlushMessage, registerWorkerCoverageFlush };

package/dist/node/parallel/coverage_flush.mjs

@@ -0,0 +1,43 @@
+// src/node/parallel/coverage_flush.ts
+import { takeCoverage } from "v8";
+var FLUSH_REQUEST = "__shelby_cov_flush_request__";
+var FLUSH_ACK = "__shelby_cov_flush_ack__";
+var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
+var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
+function isCoverageFlushMessage(msg) {
+  return isType(msg, FLUSH_REQUEST) || isType(msg, FLUSH_ACK);
+}
+function registerWorkerCoverageFlush(port) {
+  if (!coverageEnabled() || !port) return;
+  port.on("message", (msg) => {
+    if (isType(msg, FLUSH_REQUEST)) {
+      takeCoverage();
+      port.postMessage({ type: FLUSH_ACK });
+    }
+  });
+}
+function flushWorkerCoverage(worker, timeoutMs = 2e3) {
+  if (!coverageEnabled()) return Promise.resolve();
+  return new Promise((resolve) => {
+    const onMessage = (msg) => {
+      if (isType(msg, FLUSH_ACK)) finish();
+    };
+    const finish = () => {
+      clearTimeout(timer);
+      worker.off("message", onMessage);
+      resolve();
+    };
+    const timer = setTimeout(finish, timeoutMs);
+    worker.on("message", onMessage);
+    try {
+      worker.postMessage({ type: FLUSH_REQUEST });
+    } catch {
+      finish();
+    }
+  });
+}
+export {
+  flushWorkerCoverage,
+  isCoverageFlushMessage,
+  registerWorkerCoverageFlush
+};

package/dist/node/parallel/default_commitment_worker_pool.d.ts

@@ -0,0 +1,24 @@
+import { CommitmentWorkerPool } from './commitment_worker_pool.js';
+import 'node:worker_threads';
+import '../../core/commitments.js';
+import '@aptos-labs/ts-sdk';
+import 'zod';
+import '../../clay-codes-DdXABBDx.js';
+import '@shelby-protocol/clay-codes';
+import './commitment_worker.js';
+import './worker_pool.js';
+
+interface CommitmentWorkerPoolOptions {
+    /** Worker thread count. Defaults to availableParallelism() - 1, minimum 1. */
+    workers?: number;
+}
+declare function defaultCommitmentWorkerCount(): number;
+/**
+ * Create and initialize a caller-owned commitment worker pool.
+ *
+ * The returned pool is ready to pass to `generateCommitmentsParallel`.
+ * Callers own its lifecycle and must call `terminate()` during shutdown.
+ */
+declare function createDefaultCommitmentWorkerPool(options?: CommitmentWorkerPoolOptions): CommitmentWorkerPool;
+
+export { type CommitmentWorkerPoolOptions, createDefaultCommitmentWorkerPool, defaultCommitmentWorkerCount };