@shelby-protocol/sdk 0.3.0 → 0.4.0

package/dist/node/parallel/default_commitment_worker_pool.mjs

@@ -0,0 +1,355 @@
+// src/node/parallel/default_commitment_worker_pool.ts
+import { availableParallelism } from "os";
+
+// src/node/parallel/worker_pool.ts
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { Worker } from "worker_threads";
+
+// src/node/parallel/coverage_flush.ts
+import { takeCoverage } from "v8";
+var FLUSH_REQUEST = "__shelby_cov_flush_request__";
+var FLUSH_ACK = "__shelby_cov_flush_ack__";
+var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
+var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
+function flushWorkerCoverage(worker, timeoutMs = 2e3) {
+  if (!coverageEnabled()) return Promise.resolve();
+  return new Promise((resolve) => {
+    const onMessage = (msg) => {
+      if (isType(msg, FLUSH_ACK)) finish();
+    };
+    const finish = () => {
+      clearTimeout(timer);
+      worker.off("message", onMessage);
+      resolve();
+    };
+    const timer = setTimeout(finish, timeoutMs);
+    worker.on("message", onMessage);
+    try {
+      worker.postMessage({ type: FLUSH_REQUEST });
+    } catch {
+      finish();
+    }
+  });
+}
+
+// src/node/parallel/worker_pool.ts
+var WorkerPool = class {
+  workers = [];
+  queue = [];
+  maxWorkers;
+  workerPath;
+  isTerminating = false;
+  activeTasks = /* @__PURE__ */ new Set();
+  inFlightCount = 0;
+  slotWaiters = [];
+  constructor(maxWorkers, workerPath) {
+    if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) {
+      throw new Error(
+        `maxWorkers must be a positive safe integer, got ${maxWorkers}`
+      );
+    }
+    this.maxWorkers = maxWorkers;
+    this.workerPath = workerPath;
+  }
+  /**
+   * Resolve the path to a worker script relative to the calling module.
+   *
+   * Pass `import.meta.url` from the subclass file along with the script name
+   * (without extension). The extension is inferred from the calling module's
+   * own extension so that both the TypeScript source (.ts) and the compiled
+   * output (.mjs / .js) are handled correctly.
+   *
+   * @example
+   * constructor(maxWorkers: number) {
+   *   super(maxWorkers, WorkerPool.resolveWorkerPath(import.meta.url, "my_worker"));
+   * }
+   */
+  static resolveWorkerPath(importMetaUrl, scriptName) {
+    const currentPath = fileURLToPath(importMetaUrl);
+    const dir = path.dirname(currentPath);
+    const ext = path.extname(currentPath);
+    const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
+    if (workerExt === ".ts") {
+      const srcSegment = `${path.sep}src${path.sep}`;
+      const srcIdx = currentPath.indexOf(srcSegment);
+      if (srcIdx !== -1) {
+        const packageRoot = currentPath.slice(0, srcIdx);
+        const relFromSrc = path.relative(path.join(packageRoot, "src"), dir);
+        return path.join(
+          packageRoot,
+          "dist",
+          "workers",
+          relFromSrc,
+          `${scriptName}.js`
+        );
+      }
+    }
+    let workerPath = path.join(dir, `${scriptName}${workerExt}`);
+    if (fs.existsSync(workerPath)) {
+      return workerPath;
+    }
+    workerPath = path.join(dir, "workers", `${scriptName}${workerExt}`);
+    if (fs.existsSync(workerPath)) {
+      return workerPath;
+    }
+    return path.join(dir, `${scriptName}${workerExt}`);
+  }
+  initialize() {
+    this.isTerminating = false;
+    if (!fs.existsSync(this.workerPath)) {
+      throw new Error(
+        `Could not find worker script at ${this.workerPath}. Running as ${path.extname(this.workerPath)}, expected worker at same extension.`
+      );
+    }
+    for (let i = 0; i < this.maxWorkers; i++) {
+      this.workers.push({ worker: new Worker(this.workerPath), busy: false });
+    }
+  }
+  isInitialized() {
+    return this.workers.length > 0;
+  }
+  /**
+   * Acquire a back-pressure slot before submitting work to the pool.
+   * Resolves immediately if fewer than `maxWorkers` chunksets are in flight;
+   * otherwise waits until a slot is released. Callers must call `releaseSlot()`
+   * when the submitted task settles (resolve or reject).
+   *
+   * Moving back-pressure into the pool ensures a single global bound on
+   * buffered memory regardless of how many concurrent callers share the pool.
+   */
+  acquireSlot() {
+    if (this.inFlightCount < this.maxWorkers) {
+      this.inFlightCount++;
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      this.slotWaiters.push({
+        resolve: () => {
+          this.inFlightCount++;
+          resolve();
+        },
+        reject
+      });
+    });
+  }
+  releaseSlot() {
+    this.inFlightCount--;
+    const next = this.slotWaiters.shift();
+    if (next) {
+      next.resolve();
+    }
+  }
+  enqueueTask(task) {
+    this.activeTasks.add(task);
+    if (this.isTerminating) {
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error("Worker pool is being terminated - cannot process new tasks")
+      );
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy);
+    if (availableWorker) {
+      this.executeTask(availableWorker, task);
+    } else {
+      this.queue.push(task);
+    }
+  }
+  processNextQueuedTask(preferredWorker) {
+    const nextTask = this.queue.shift();
+    if (!nextTask) return;
+    if (preferredWorker && !preferredWorker.busy && preferredWorker.worker) {
+      this.executeTask(preferredWorker, nextTask);
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy && w.worker);
+    if (availableWorker) {
+      this.executeTask(availableWorker, nextTask);
+    } else {
+      this.queue.unshift(nextTask);
+    }
+  }
+  rejectAllQueuedTasks(error) {
+    const tasks = this.queue.splice(0);
+    for (const task of tasks) {
+      this.activeTasks.delete(task);
+      task.reject(error);
+    }
+  }
+  rejectAllActiveTasks(error) {
+    const tasks = Array.from(this.activeTasks);
+    this.activeTasks.clear();
+    for (const task of tasks) {
+      task.reject(error);
+    }
+  }
+  executeTask(workerWrapper, task) {
+    workerWrapper.busy = true;
+    const cleanup = () => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+    };
+    const messageHandler = (message) => {
+      cleanup();
+      this.activeTasks.delete(task);
+      if (message.success && message.result !== void 0) {
+        task.resolve(message.result);
+      } else {
+        task.reject(new Error(message.error ?? "Worker processing failed"));
+      }
+      this.processNextQueuedTask(workerWrapper);
+    };
+    const errorHandler = (error) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(error);
+    };
+    const exitHandler = (exitCode) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error(`Worker terminated unexpectedly with exit code ${exitCode}`)
+      );
+      if (this.isTerminating) return;
+      try {
+        const newWorker = new Worker(this.workerPath);
+        if (this.isTerminating) {
+          newWorker.terminate().catch(() => {
+          });
+          return;
+        }
+        workerWrapper.worker = newWorker;
+        this.processNextQueuedTask(workerWrapper);
+      } catch (_error) {
+        const index = this.workers.indexOf(workerWrapper);
+        if (index > -1) this.workers.splice(index, 1);
+        if (this.workers.length === 0) {
+          this.rejectAllQueuedTasks(
+            new Error(
+              "All workers have died and could not be replaced - worker pool is non-functional"
+            )
+          );
+          return;
+        }
+        this.processNextQueuedTask();
+      }
+    };
+    workerWrapper.worker.on("message", messageHandler);
+    workerWrapper.worker.on("error", errorHandler);
+    workerWrapper.worker.on("exit", exitHandler);
+    this.postTaskMessage(workerWrapper.worker, task);
+  }
+  /**
+   * Terminate all workers and reject all pending tasks.
+   *
+   * This pool is NOT designed to be reused after termination.
+   * Create a new instance if a fresh pool is needed.
+   */
+  async terminate() {
+    this.isTerminating = true;
+    const terminationError = new Error(
+      "Worker pool is being terminated - task cancelled"
+    );
+    this.rejectAllQueuedTasks(terminationError);
+    this.rejectAllActiveTasks(terminationError);
+    const waiters = this.slotWaiters.splice(0);
+    for (const waiter of waiters) {
+      waiter.reject(terminationError);
+    }
+    const terminatePromises = this.workers.map(async (workerWrapper) => {
+      const worker = workerWrapper.worker;
+      await flushWorkerCoverage(worker);
+      await new Promise((resolve) => {
+        let resolved = false;
+        const doResolve = () => {
+          if (!resolved) {
+            resolved = true;
+            resolve();
+          }
+        };
+        let timeoutId;
+        const exitHandler = () => {
+          if (timeoutId) clearTimeout(timeoutId);
+          doResolve();
+        };
+        worker.removeAllListeners("message");
+        worker.removeAllListeners("error");
+        worker.once("exit", exitHandler);
+        timeoutId = setTimeout(() => {
+          worker.removeListener("exit", exitHandler);
+          doResolve();
+        }, 2e3);
+        worker.terminate().catch(() => {
+        });
+      });
+    });
+    await Promise.all(terminatePromises);
+    this.workers = [];
+    this.queue = [];
+    this.inFlightCount = 0;
+    this.isTerminating = false;
+  }
+};
+
+// src/node/parallel/commitment_worker_pool.ts
+var CommitmentWorkerPool = class extends WorkerPool {
+  constructor(maxWorkers) {
+    super(
+      maxWorkers,
+      WorkerPool.resolveWorkerPath(import.meta.url, "commitment_worker")
+    );
+  }
+  postTaskMessage(worker, task) {
+    worker.postMessage(
+      {
+        chunksetData: task.chunksetData,
+        encoding: task.encoding
+      },
+      [task.chunksetData.buffer]
+    );
+  }
+  /**
+   * Dispatch a chunkset to a worker for encoding. Returns a promise that
+   * resolves to the chunkset's commitment once the worker finishes.
+   *
+   * The caller must ensure `chunksetData` is not used after this call
+   * (its underlying buffer is transferred to the worker thread).
+   */
+  encodeChunkset(chunksetIdx, chunksetData, encoding) {
+    return new Promise((resolve, reject) => {
+      const task = {
+        chunksetIdx,
+        chunksetData,
+        encoding,
+        resolve,
+        reject
+      };
+      this.enqueueTask(task);
+    });
+  }
+};
+
+// src/node/parallel/default_commitment_worker_pool.ts
+function defaultCommitmentWorkerCount() {
+  return Math.max(1, availableParallelism() - 1);
+}
+function createDefaultCommitmentWorkerPool(options = {}) {
+  const pool = new CommitmentWorkerPool(
+    options.workers ?? defaultCommitmentWorkerCount()
+  );
+  pool.initialize();
+  return pool;
+}
+export {
+  createDefaultCommitmentWorkerPool,
+  defaultCommitmentWorkerCount
+};

package/dist/node/parallel/index.d.ts

@@ -0,0 +1,12 @@
+export { CommitmentEncoding } from './commitment_worker.js';
+export { CommitmentWorkerPool } from './commitment_worker_pool.js';
+export { flushWorkerCoverage, isCoverageFlushMessage, registerWorkerCoverageFlush } from './coverage_flush.js';
+export { CommitmentWorkerPoolOptions, createDefaultCommitmentWorkerPool, defaultCommitmentWorkerCount } from './default_commitment_worker_pool.js';
+export { generateCommitmentsParallel } from './parallel_commitments.js';
+export { WorkerPool, WorkerPoolTask } from './worker_pool.js';
+import 'node:worker_threads';
+import '../../core/commitments.js';
+import '@aptos-labs/ts-sdk';
+import 'zod';
+import '../../clay-codes-DdXABBDx.js';
+import '@shelby-protocol/clay-codes';