@shelby-protocol/sdk 0.3.0 → 0.4.0

package/dist/node/parallel/worker_pool.mjs

@@ -0,0 +1,301 @@
+// src/node/parallel/worker_pool.ts
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { Worker } from "worker_threads";
+
+// src/node/parallel/coverage_flush.ts
+import { takeCoverage } from "v8";
+var FLUSH_REQUEST = "__shelby_cov_flush_request__";
+var FLUSH_ACK = "__shelby_cov_flush_ack__";
+var coverageEnabled = () => Boolean(process.env.NODE_V8_COVERAGE);
+var isType = (msg, type) => typeof msg === "object" && msg !== null && msg.type === type;
+function flushWorkerCoverage(worker, timeoutMs = 2e3) {
+  if (!coverageEnabled()) return Promise.resolve();
+  return new Promise((resolve) => {
+    const onMessage = (msg) => {
+      if (isType(msg, FLUSH_ACK)) finish();
+    };
+    const finish = () => {
+      clearTimeout(timer);
+      worker.off("message", onMessage);
+      resolve();
+    };
+    const timer = setTimeout(finish, timeoutMs);
+    worker.on("message", onMessage);
+    try {
+      worker.postMessage({ type: FLUSH_REQUEST });
+    } catch {
+      finish();
+    }
+  });
+}
+
+// src/node/parallel/worker_pool.ts
+var WorkerPool = class {
+  workers = [];
+  queue = [];
+  maxWorkers;
+  workerPath;
+  isTerminating = false;
+  activeTasks = /* @__PURE__ */ new Set();
+  inFlightCount = 0;
+  slotWaiters = [];
+  constructor(maxWorkers, workerPath) {
+    if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) {
+      throw new Error(
+        `maxWorkers must be a positive safe integer, got ${maxWorkers}`
+      );
+    }
+    this.maxWorkers = maxWorkers;
+    this.workerPath = workerPath;
+  }
+  /**
+   * Resolve the path to a worker script relative to the calling module.
+   *
+   * Pass `import.meta.url` from the subclass file along with the script name
+   * (without extension). The extension is inferred from the calling module's
+   * own extension so that both the TypeScript source (.ts) and the compiled
+   * output (.mjs / .js) are handled correctly.
+   *
+   * @example
+   * constructor(maxWorkers: number) {
+   *   super(maxWorkers, WorkerPool.resolveWorkerPath(import.meta.url, "my_worker"));
+   * }
+   */
+  static resolveWorkerPath(importMetaUrl, scriptName) {
+    const currentPath = fileURLToPath(importMetaUrl);
+    const dir = path.dirname(currentPath);
+    const ext = path.extname(currentPath);
+    const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
+    if (workerExt === ".ts") {
+      const srcSegment = `${path.sep}src${path.sep}`;
+      const srcIdx = currentPath.indexOf(srcSegment);
+      if (srcIdx !== -1) {
+        const packageRoot = currentPath.slice(0, srcIdx);
+        const relFromSrc = path.relative(path.join(packageRoot, "src"), dir);
+        return path.join(
+          packageRoot,
+          "dist",
+          "workers",
+          relFromSrc,
+          `${scriptName}.js`
+        );
+      }
+    }
+    let workerPath = path.join(dir, `${scriptName}${workerExt}`);
+    if (fs.existsSync(workerPath)) {
+      return workerPath;
+    }
+    workerPath = path.join(dir, "workers", `${scriptName}${workerExt}`);
+    if (fs.existsSync(workerPath)) {
+      return workerPath;
+    }
+    return path.join(dir, `${scriptName}${workerExt}`);
+  }
+  initialize() {
+    this.isTerminating = false;
+    if (!fs.existsSync(this.workerPath)) {
+      throw new Error(
+        `Could not find worker script at ${this.workerPath}. Running as ${path.extname(this.workerPath)}, expected worker at same extension.`
+      );
+    }
+    for (let i = 0; i < this.maxWorkers; i++) {
+      this.workers.push({ worker: new Worker(this.workerPath), busy: false });
+    }
+  }
+  isInitialized() {
+    return this.workers.length > 0;
+  }
+  /**
+   * Acquire a back-pressure slot before submitting work to the pool.
+   * Resolves immediately if fewer than `maxWorkers` chunksets are in flight;
+   * otherwise waits until a slot is released. Callers must call `releaseSlot()`
+   * when the submitted task settles (resolve or reject).
+   *
+   * Moving back-pressure into the pool ensures a single global bound on
+   * buffered memory regardless of how many concurrent callers share the pool.
+   */
+  acquireSlot() {
+    if (this.inFlightCount < this.maxWorkers) {
+      this.inFlightCount++;
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      this.slotWaiters.push({
+        resolve: () => {
+          this.inFlightCount++;
+          resolve();
+        },
+        reject
+      });
+    });
+  }
+  releaseSlot() {
+    this.inFlightCount--;
+    const next = this.slotWaiters.shift();
+    if (next) {
+      next.resolve();
+    }
+  }
+  enqueueTask(task) {
+    this.activeTasks.add(task);
+    if (this.isTerminating) {
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error("Worker pool is being terminated - cannot process new tasks")
+      );
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy);
+    if (availableWorker) {
+      this.executeTask(availableWorker, task);
+    } else {
+      this.queue.push(task);
+    }
+  }
+  processNextQueuedTask(preferredWorker) {
+    const nextTask = this.queue.shift();
+    if (!nextTask) return;
+    if (preferredWorker && !preferredWorker.busy && preferredWorker.worker) {
+      this.executeTask(preferredWorker, nextTask);
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy && w.worker);
+    if (availableWorker) {
+      this.executeTask(availableWorker, nextTask);
+    } else {
+      this.queue.unshift(nextTask);
+    }
+  }
+  rejectAllQueuedTasks(error) {
+    const tasks = this.queue.splice(0);
+    for (const task of tasks) {
+      this.activeTasks.delete(task);
+      task.reject(error);
+    }
+  }
+  rejectAllActiveTasks(error) {
+    const tasks = Array.from(this.activeTasks);
+    this.activeTasks.clear();
+    for (const task of tasks) {
+      task.reject(error);
+    }
+  }
+  executeTask(workerWrapper, task) {
+    workerWrapper.busy = true;
+    const cleanup = () => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+    };
+    const messageHandler = (message) => {
+      cleanup();
+      this.activeTasks.delete(task);
+      if (message.success && message.result !== void 0) {
+        task.resolve(message.result);
+      } else {
+        task.reject(new Error(message.error ?? "Worker processing failed"));
+      }
+      this.processNextQueuedTask(workerWrapper);
+    };
+    const errorHandler = (error) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(error);
+    };
+    const exitHandler = (exitCode) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error(`Worker terminated unexpectedly with exit code ${exitCode}`)
+      );
+      if (this.isTerminating) return;
+      try {
+        const newWorker = new Worker(this.workerPath);
+        if (this.isTerminating) {
+          newWorker.terminate().catch(() => {
+          });
+          return;
+        }
+        workerWrapper.worker = newWorker;
+        this.processNextQueuedTask(workerWrapper);
+      } catch (_error) {
+        const index = this.workers.indexOf(workerWrapper);
+        if (index > -1) this.workers.splice(index, 1);
+        if (this.workers.length === 0) {
+          this.rejectAllQueuedTasks(
+            new Error(
+              "All workers have died and could not be replaced - worker pool is non-functional"
+            )
+          );
+          return;
+        }
+        this.processNextQueuedTask();
+      }
+    };
+    workerWrapper.worker.on("message", messageHandler);
+    workerWrapper.worker.on("error", errorHandler);
+    workerWrapper.worker.on("exit", exitHandler);
+    this.postTaskMessage(workerWrapper.worker, task);
+  }
+  /**
+   * Terminate all workers and reject all pending tasks.
+   *
+   * This pool is NOT designed to be reused after termination.
+   * Create a new instance if a fresh pool is needed.
+   */
+  async terminate() {
+    this.isTerminating = true;
+    const terminationError = new Error(
+      "Worker pool is being terminated - task cancelled"
+    );
+    this.rejectAllQueuedTasks(terminationError);
+    this.rejectAllActiveTasks(terminationError);
+    const waiters = this.slotWaiters.splice(0);
+    for (const waiter of waiters) {
+      waiter.reject(terminationError);
+    }
+    const terminatePromises = this.workers.map(async (workerWrapper) => {
+      const worker = workerWrapper.worker;
+      await flushWorkerCoverage(worker);
+      await new Promise((resolve) => {
+        let resolved = false;
+        const doResolve = () => {
+          if (!resolved) {
+            resolved = true;
+            resolve();
+          }
+        };
+        let timeoutId;
+        const exitHandler = () => {
+          if (timeoutId) clearTimeout(timeoutId);
+          doResolve();
+        };
+        worker.removeAllListeners("message");
+        worker.removeAllListeners("error");
+        worker.once("exit", exitHandler);
+        timeoutId = setTimeout(() => {
+          worker.removeListener("exit", exitHandler);
+          doResolve();
+        }, 2e3);
+        worker.terminate().catch(() => {
+        });
+      });
+    });
+    await Promise.all(terminatePromises);
+    this.workers = [];
+    this.queue = [];
+    this.inFlightCount = 0;
+    this.isTerminating = false;
+  }
+};
+export {
+  WorkerPool
+};

package/dist/node/testUtil.mjs

@@ -1,7 +1,11 @@
-import {
-  makeChunk
-} from "../chunk-A4IG6GSE.mjs";
-import "../chunk-7P6ASYW6.mjs";
+// src/node/testUtil.ts
+function makeChunk(n) {
+  const c = Buffer.alloc(n);
+  for (let i = 0; i < n; ++i) {
+    c[i] = i % 256;
+  }
+  return c;
+}
 export {
   makeChunk
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shelby-protocol/sdk",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -14,19 +14,28 @@
       "types": "./dist/node/index.d.ts",
       "import": "./dist/node/index.mjs"
     },
+    "./node/parallel": {
+      "types": "./dist/node/parallel/index.d.ts",
+      "import": "./dist/node/parallel/index.mjs"
+    },
+    "./sp": {
+      "types": "./dist/core/sp/index.d.ts",
+      "import": "./dist/core/sp/index.mjs"
+    },
     "./browser": {
       "types": "./dist/browser/index.d.ts",
       "import": "./dist/browser/index.mjs"
     }
   },
   "dependencies": {
+    "@bufbuild/protobuf": "^2.8.0",
     "graphql": "^16.11.0",
     "graphql-request": "^7.2.0",
     "graphql-tag": "^2.12.6",
     "p-limit": "7.1.1",
     "zod": "^3.24.3",
-    "@shelby-protocol/clay-codes": "0.0.2",
-    "@shelby-protocol/reed-solomon": "0.0.1"
+    "@shelby-protocol/reed-solomon": "0.0.1",
+    "@shelby-protocol/clay-codes": "0.0.2"
   },
   "devDependencies": {
     "@aptos-labs/ts-sdk": "5.2.1",
@@ -49,12 +58,14 @@
   "scripts": {
     "lint": "biome check .",
     "fmt": "biome check . --write",
-    "generate": "graphql-codegen",
+    "generate": "graphql-codegen && (cd ../.. && pnpm exec buf generate --template buf.gen.server.yaml --path proto/rpc_server.proto)",
     "generate:watch": "graphql-codegen --watch",
+    "generate:schema": "node scripts/dump-schema.mjs",
     "verify:config": "tsx scripts/verify-config.ts",
     "build": "tsc --noEmit && (rimraf dist; tsup) && pnpm run verify:config",
     "test": "vitest dev",
-    "test:once": "vitest run",
+    "test:once": "pnpm run build && vitest run",
+    "test:integration": "vitest run --config vitest.integration.config.ts --reporter=verbose",
     "benchmark:erasure": "tsx benchmark/erasure.ts"
   }
 }

package/dist/chunk-3NRBHSMQ.mjs

@@ -1,12 +0,0 @@
-import {
-  normalizeAddress
-} from "./chunk-UEZNZBJO.mjs";
-
-// src/core/blobs.ts
-var createBlobKey = (params) => {
-  return `@${normalizeAddress(params.account).toStringLongWithoutPrefix()}/${params.blobName}`;
-};
-
-export {
-  createBlobKey
-};