@shelby-protocol/sdk 0.2.4 → 0.3.1

package/dist/node/parallel/worker_pool.mjs

@@ -0,0 +1,248 @@
+// src/node/parallel/worker_pool.ts
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { Worker } from "worker_threads";
+var WorkerPool = class {
+  workers = [];
+  queue = [];
+  maxWorkers;
+  workerPath;
+  isTerminating = false;
+  activeTasks = /* @__PURE__ */ new Set();
+  inFlightCount = 0;
+  slotWaiters = [];
+  constructor(maxWorkers, workerPath) {
+    if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) {
+      throw new Error(
+        `maxWorkers must be a positive safe integer, got ${maxWorkers}`
+      );
+    }
+    this.maxWorkers = maxWorkers;
+    this.workerPath = workerPath;
+  }
+  /**
+   * Resolve the path to a worker script relative to the calling module.
+   *
+   * Pass `import.meta.url` from the subclass file along with the script name
+   * (without extension). The extension is inferred from the calling module's
+   * own extension so that both the TypeScript source (.ts) and the compiled
+   * output (.mjs / .js) are handled correctly.
+   *
+   * @example
+   * constructor(maxWorkers: number) {
+   *   super(maxWorkers, WorkerPool.resolveWorkerPath(import.meta.url, "my_worker"));
+   * }
+   */
+  static resolveWorkerPath(importMetaUrl, scriptName) {
+    const currentPath = fileURLToPath(importMetaUrl);
+    const dir = path.dirname(currentPath);
+    const ext = path.extname(currentPath);
+    const workerExt = ext === ".mjs" ? ".mjs" : ext === ".js" ? ".js" : ".ts";
+    return path.join(dir, `${scriptName}${workerExt}`);
+  }
+  initialize() {
+    this.isTerminating = false;
+    if (!fs.existsSync(this.workerPath)) {
+      throw new Error(
+        `Could not find worker script at ${this.workerPath}. Running as ${path.extname(this.workerPath)}, expected worker at same extension.`
+      );
+    }
+    for (let i = 0; i < this.maxWorkers; i++) {
+      this.workers.push({ worker: new Worker(this.workerPath), busy: false });
+    }
+  }
+  isInitialized() {
+    return this.workers.length > 0;
+  }
+  /**
+   * Acquire a back-pressure slot before submitting work to the pool.
+   * Resolves immediately if fewer than `maxWorkers` chunksets are in flight;
+   * otherwise waits until a slot is released. Callers must call `releaseSlot()`
+   * when the submitted task settles (resolve or reject).
+   *
+   * Moving back-pressure into the pool ensures a single global bound on
+   * buffered memory regardless of how many concurrent callers share the pool.
+   */
+  acquireSlot() {
+    if (this.inFlightCount < this.maxWorkers) {
+      this.inFlightCount++;
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      this.slotWaiters.push({
+        resolve: () => {
+          this.inFlightCount++;
+          resolve();
+        },
+        reject
+      });
+    });
+  }
+  releaseSlot() {
+    this.inFlightCount--;
+    const next = this.slotWaiters.shift();
+    if (next) {
+      next.resolve();
+    }
+  }
+  enqueueTask(task) {
+    this.activeTasks.add(task);
+    if (this.isTerminating) {
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error("Worker pool is being terminated - cannot process new tasks")
+      );
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy);
+    if (availableWorker) {
+      this.executeTask(availableWorker, task);
+    } else {
+      this.queue.push(task);
+    }
+  }
+  processNextQueuedTask(preferredWorker) {
+    const nextTask = this.queue.shift();
+    if (!nextTask) return;
+    if (preferredWorker && !preferredWorker.busy && preferredWorker.worker) {
+      this.executeTask(preferredWorker, nextTask);
+      return;
+    }
+    const availableWorker = this.workers.find((w) => !w.busy && w.worker);
+    if (availableWorker) {
+      this.executeTask(availableWorker, nextTask);
+    } else {
+      this.queue.unshift(nextTask);
+    }
+  }
+  rejectAllQueuedTasks(error) {
+    const tasks = this.queue.splice(0);
+    for (const task of tasks) {
+      this.activeTasks.delete(task);
+      task.reject(error);
+    }
+  }
+  rejectAllActiveTasks(error) {
+    const tasks = Array.from(this.activeTasks);
+    this.activeTasks.clear();
+    for (const task of tasks) {
+      task.reject(error);
+    }
+  }
+  executeTask(workerWrapper, task) {
+    workerWrapper.busy = true;
+    const cleanup = () => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+    };
+    const messageHandler = (message) => {
+      cleanup();
+      this.activeTasks.delete(task);
+      if (message.success && message.result !== void 0) {
+        task.resolve(message.result);
+      } else {
+        task.reject(new Error(message.error ?? "Worker processing failed"));
+      }
+      this.processNextQueuedTask(workerWrapper);
+    };
+    const errorHandler = (error) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(error);
+    };
+    const exitHandler = (exitCode) => {
+      workerWrapper.worker.off("message", messageHandler);
+      workerWrapper.worker.off("error", errorHandler);
+      workerWrapper.worker.off("exit", exitHandler);
+      workerWrapper.busy = false;
+      this.activeTasks.delete(task);
+      task.reject(
+        new Error(`Worker terminated unexpectedly with exit code ${exitCode}`)
+      );
+      if (this.isTerminating) return;
+      try {
+        const newWorker = new Worker(this.workerPath);
+        if (this.isTerminating) {
+          newWorker.terminate().catch(() => {
+          });
+          return;
+        }
+        workerWrapper.worker = newWorker;
+        this.processNextQueuedTask(workerWrapper);
+      } catch (_error) {
+        const index = this.workers.indexOf(workerWrapper);
+        if (index > -1) this.workers.splice(index, 1);
+        if (this.workers.length === 0) {
+          this.rejectAllQueuedTasks(
+            new Error(
+              "All workers have died and could not be replaced - worker pool is non-functional"
+            )
+          );
+          return;
+        }
+        this.processNextQueuedTask();
+      }
+    };
+    workerWrapper.worker.on("message", messageHandler);
+    workerWrapper.worker.on("error", errorHandler);
+    workerWrapper.worker.on("exit", exitHandler);
+    this.postTaskMessage(workerWrapper.worker, task);
+  }
+  /**
+   * Terminate all workers and reject all pending tasks.
+   *
+   * This pool is NOT designed to be reused after termination.
+   * Create a new instance if a fresh pool is needed.
+   */
+  async terminate() {
+    this.isTerminating = true;
+    const terminationError = new Error(
+      "Worker pool is being terminated - task cancelled"
+    );
+    this.rejectAllQueuedTasks(terminationError);
+    this.rejectAllActiveTasks(terminationError);
+    const waiters = this.slotWaiters.splice(0);
+    for (const waiter of waiters) {
+      waiter.reject(terminationError);
+    }
+    const terminatePromises = this.workers.map((workerWrapper) => {
+      return new Promise((resolve) => {
+        const worker = workerWrapper.worker;
+        let resolved = false;
+        const doResolve = () => {
+          if (!resolved) {
+            resolved = true;
+            resolve();
+          }
+        };
+        let timeoutId;
+        const exitHandler = () => {
+          if (timeoutId) clearTimeout(timeoutId);
+          doResolve();
+        };
+        worker.removeAllListeners("message");
+        worker.removeAllListeners("error");
+        worker.once("exit", exitHandler);
+        timeoutId = setTimeout(() => {
+          worker.removeListener("exit", exitHandler);
+          doResolve();
+        }, 2e3);
+        worker.terminate().catch(() => {
+        });
+      });
+    });
+    await Promise.all(terminatePromises);
+    this.workers = [];
+    this.queue = [];
+    this.inFlightCount = 0;
+    this.isTerminating = false;
+  }
+};
+export {
+  WorkerPool
+};

package/dist/node/testUtil.mjs

@@ -1,7 +1,11 @@
-import {
-  makeChunk
-} from "../chunk-A4IG6GSE.mjs";
-import "../chunk-7P6ASYW6.mjs";
+// src/node/testUtil.ts
+function makeChunk(n) {
+  const c = Buffer.alloc(n);
+  for (let i = 0; i < n; ++i) {
+    c[i] = i % 256;
+  }
+  return c;
+}
 export {
   makeChunk
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shelby-protocol/sdk",
-  "version": "0.2.4",
+  "version": "0.3.1",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -14,6 +14,10 @@
       "types": "./dist/node/index.d.ts",
       "import": "./dist/node/index.mjs"
     },
+    "./node/parallel": {
+      "types": "./dist/node/parallel/index.d.ts",
+      "import": "./dist/node/parallel/index.mjs"
+    },
     "./browser": {
       "types": "./dist/browser/index.d.ts",
       "import": "./dist/browser/index.mjs"
@@ -25,8 +29,8 @@
     "graphql-tag": "^2.12.6",
     "p-limit": "7.1.1",
     "zod": "^3.24.3",
-    "@shelby-protocol/reed-solomon": "0.0.1",
-    "@shelby-protocol/clay-codes": "0.0.2"
+    "@shelby-protocol/clay-codes": "0.0.2",
+    "@shelby-protocol/reed-solomon": "0.0.1"
   },
   "devDependencies": {
     "@aptos-labs/ts-sdk": "5.2.1",
@@ -44,7 +48,7 @@
     "vitest": "^3.1.2"
   },
   "peerDependencies": {
-    "@aptos-labs/ts-sdk": "^2.0.1 || ^3.0.0 || ^4.0.0 || ^5.0.0"
+    "@aptos-labs/ts-sdk": "^5.2.1 || ^6.0.0"
   },
   "scripts": {
     "lint": "biome check .",
@@ -54,8 +58,7 @@
     "verify:config": "tsx scripts/verify-config.ts",
     "build": "tsc --noEmit && (rimraf dist; tsup) && pnpm run verify:config",
     "test": "vitest dev",
-    "test:once": "vitest run",
-    "test:integration": "vitest run --config vitest.integration.config.ts --reporter=verbose",
+    "test:once": "pnpm run build && vitest run",
     "benchmark:erasure": "tsx benchmark/erasure.ts"
   }
 }

package/dist/chunk-3NRBHSMQ.mjs

@@ -1,12 +0,0 @@
-import {
-  normalizeAddress
-} from "./chunk-UEZNZBJO.mjs";
-
-// src/core/blobs.ts
-var createBlobKey = (params) => {
-  return `@${normalizeAddress(params.account).toStringLongWithoutPrefix()}/${params.blobName}`;
-};
-
-export {
-  createBlobKey
-};

package/dist/chunk-3PCG7PNP.mjs

@@ -1,375 +0,0 @@
-import {
-  ShelbyBlobClient
-} from "./chunk-WJTVYLKW.mjs";
-import {
-  defaultErasureCodingConfig
-} from "./chunk-XRLHLAOW.mjs";
-import {
-  ClayErasureCodingProvider
-} from "./chunk-OGKZ575S.mjs";
-import {
-  ShelbyMetadataClient
-} from "./chunk-SSE4MFNN.mjs";
-import {
-  ShelbyRPCClient
-} from "./chunk-BKL7NCUB.mjs";
-import {
-  getAptosConfig,
-  getShelbyRPCBaseUrl
-} from "./chunk-IKRWBDG5.mjs";
-import {
-  createBlobKey
-} from "./chunk-3NRBHSMQ.mjs";
-import {
-  generateCommitments
-} from "./chunk-BXEVML7N.mjs";
-import {
-  normalizeAddress
-} from "./chunk-UEZNZBJO.mjs";
-
-// src/core/clients/ShelbyClient.ts
-import {
-  Aptos,
-  DEFAULT_TXN_TIMEOUT_SEC,
-  Network,
-  TransactionResponseType
-} from "@aptos-labs/ts-sdk";
-import pLimit from "p-limit";
-var ShelbyClient = class {
-  /**
-   * The coordination client is used to interact with the Aptos blockchain which handles the commitments
-   * and metadata for blobs.
-   */
-  coordination;
-  /**
-   * The metadata client is used for protocol-level metadata queries.
-   */
-  metadata;
-  /**
-   * The RPC client is used to interact with the Shelby RPC node which can be responsible for storing,
-   * confirming, and retrieving blobs from the storage layer.
-   *
-   * If not provided, the default RPC client will be created.
-   */
-  rpc;
-  /**
-   * The configuration for the Shelby client.
-   */
-  config;
-  /**
-   * The Aptos client.
-   *
-   * If not provided, a default Aptos client will be created.
-   */
-  aptos;
-  /**
-   * The erasure coding provider used for encoding/decoding operations.
-   * Lazily initialized on first use if not provided.
-   */
-  _provider;
-  /**
-   * Creates a new ShelbyClient instance for interacting with the Shelby Protocol.
-   * This client combines blockchain operations (via coordination) and storage operations (via RPC).
-   *
-   * @param config - The client configuration object.
-   * @param config.aptos.config - The Aptos network configuration.
-   * @param config.shelby.rpc.baseUrl - The base URL of the Shelby RPC node (optional, defaults to devnet).
-   * @param config.shelby.indexer - The indexer configuration for GraphQL queries.
-   * @param provider - Optional erasure coding provider for encoding/decoding operations.
-   *                   If not provided, a ClayErasureCodingProvider will be created on first use.
-   *                   Pass a shared provider to reuse across multiple clients.
-   *
-   * @example
-   * ```typescript
-   * // Basic usage (provider created automatically)
-   * const client = new ShelbyClient({
-   *   network: Network.SHELBYNET,
-   * });
-   *
-   * // Advanced: Share provider across multiple clients
-   * const provider = await ClayErasureCodingProvider.create();
-   * const mainnetClient = new ShelbyClient(mainnetConfig, provider);
-   * const devnetClient = new ShelbyClient(devnetConfig, provider);
-   * ```
-   */
-  constructor(config, provider) {
-    this.config = config;
-    this.aptos = new Aptos(getAptosConfig(config));
-    this.coordination = new ShelbyBlobClient(config);
-    this.metadata = new ShelbyMetadataClient(config);
-    this.rpc = new ShelbyRPCClient(config);
-    this._provider = provider;
-  }
-  /**
-   * Get the erasure coding provider, creating it if necessary.
-   * This allows lazy initialization for users who don't provide a provider.
-   */
-  async getProvider() {
-    if (!this._provider) {
-      this._provider = await ClayErasureCodingProvider.create(
-        defaultErasureCodingConfig()
-      );
-    }
-    return this._provider;
-  }
-  /**
-   * The base URL for the Shelby RPC node.
-   */
-  get baseUrl() {
-    return getShelbyRPCBaseUrl(this.config);
-  }
-  /**
-   * Uploads a blob to the Shelby network.
-   * This method handles the complete upload flow including commitment generation,
-   * blockchain registration, and storage upload.
-   *
-   * Note: This method accepts only `Uint8Array` and buffers the entire blob in memory.
-   * For streaming uploads of large files (e.g. >2 GiB), orchestrate the steps manually
-   * using `generateCommitments()`, `coordination.registerBlob()`, and `rpc.putBlob()`
-   * with a `ReadableStream`.
-   *
-   * @param params.blobData - The raw data to upload as a Uint8Array.
-   * @param params.signer - The account that signs and pays for the transaction.
-   * @param params.blobName - The name/path of the blob (e.g. "folder/file.txt").
-   * @param params.expirationMicros - The expiration time in microseconds since Unix epoch.
-   * @param params.options - Optional upload configuration.
-   * @param params.options.chunksetSizeBytes - Custom chunkset size.
-   * @param params.options.build - Additional Aptos transaction options.
-   *
-   * @returns The transaction and generated blob commitments (when implemented).
-   *
-   * @example
-   * ```typescript
-   * await client.upload({
-   *   blobData: Buffer.from("Hello, World!"),
-   *   signer: account,
-   *   blobName: "hello.txt",
-   *   expirationMicros: Date.now() * 1000 + 86400_000_000 // 24 hours from now
-   * });
-   * ```
-   */
-  async upload(params) {
-    const existingBlobMetadata = await this.coordination.getBlobMetadata({
-      account: params.signer.accountAddress,
-      name: params.blobName
-    });
-    if (!existingBlobMetadata) {
-      const provider = await this.getProvider();
-      const blobCommitments = await generateCommitments(
-        provider,
-        params.blobData
-      );
-      const { transaction: pendingRegisterBlobTransaction } = await this.coordination.registerBlob({
-        account: params.signer,
-        blobName: params.blobName,
-        blobMerkleRoot: blobCommitments.blob_merkle_root,
-        size: params.blobData.length,
-        expirationMicros: params.expirationMicros,
-        config: provider.config,
-        options: params.options
-      });
-      await this.coordination.aptos.waitForTransaction({
-        transactionHash: pendingRegisterBlobTransaction.hash
-      });
-    }
-    await this.rpc.putBlob({
-      account: params.signer.accountAddress,
-      blobName: params.blobName,
-      blobData: params.blobData
-    });
-  }
-  /**
-   * Uploads a batch of blobs to the Shelby network.
-   * This method handles the complete upload flow including commitment generation,
-   * blockchain registration, and storage upload.
-   *
-   * Note: This method accepts only `Uint8Array` and buffers each blob in memory.
-   * For streaming uploads of large files, orchestrate the steps manually using
-   * `generateCommitments()`, `coordination.registerBlob()`, and `rpc.putBlob()`
-   * with a `ReadableStream`.
-   *
-   * @param params.blobs - The blobs to upload.
-   * @param params.blobs.blobData - The raw data to upload as a Uint8Array.
-   * @param params.blobs.blobName - The name/path of the blob (e.g. "folder/file.txt").
-   * @param params.expirationMicros - The expiration time in microseconds since Unix epoch.
-   * @param params.signer - The account that signs and pays for the transaction.
-   * @param params.options - Optional upload configuration.
-   * @param params.options.chunksetSizeBytes - Custom chunkset size.
-   * @param params.options.build - Additional Aptos transaction options.
-   *
-   * @returns The transaction and generated blob commitments (when implemented).
-   *
-   * @example
-   * ```typescript
-   * await client.batchUpload({
-   *   blobs: [
-   *     { blobData: Buffer.from("Hello, World!"), blobName: "hello.txt" },
-   *     { blobData: Buffer.from("Hello, World 2!"), blobName: "hello2.txt" },
-   *   ],
-   *   expirationMicros: Date.now() * 1000 + 86400_000_000 // 24 hours from now
-   * });
-   * ```
-   */
-  async batchUpload(params) {
-    const existingBlobs = await this.coordination.getBlobs({
-      where: {
-        blob_name: {
-          _in: params.blobs.map(
-            (blob) => createBlobKey({
-              account: params.signer.accountAddress,
-              blobName: blob.blobName
-            })
-          )
-        }
-      }
-    });
-    const blobsToRegister = params.blobs.filter(
-      (blob) => !existingBlobs.some(
-        (existingBlob) => existingBlob.name === createBlobKey({
-          account: params.signer.accountAddress,
-          blobName: blob.blobName
-        })
-      )
-    );
-    if (blobsToRegister.length > 0) {
-      const provider = await this.getProvider();
-      const blobCommitments = await Promise.all(
-        blobsToRegister.map(
-          async (blob) => generateCommitments(provider, blob.blobData)
-        )
-      );
-      const { transaction: pendingRegisterBlobTransaction } = await this.coordination.batchRegisterBlobs({
-        account: params.signer,
-        expirationMicros: params.expirationMicros,
-        blobs: blobsToRegister.map((blob, index) => ({
-          blobName: blob.blobName,
-          blobSize: blob.blobData.length,
-          blobMerkleRoot: blobCommitments[index].blob_merkle_root
-        })),
-        config: provider.config,
-        options: params.options
-      });
-      await this.coordination.aptos.waitForTransaction({
-        transactionHash: pendingRegisterBlobTransaction.hash,
-        options: { waitForIndexer: true }
-      });
-    }
-    const limit = pLimit(3);
-    await Promise.all(
-      params.blobs.map(
-        (blob) => limit(
-          () => this.rpc.putBlob({
-            account: params.signer.accountAddress,
-            blobName: blob.blobName,
-            blobData: blob.blobData
-          })
-        )
-      )
-    );
-  }
-  /**
-   * Downloads a blob from the Shelby RPC node.
-   *
-   * @param params.account - The account namespace the blob is stored in (e.g. "0x1")
-   * @param params.blobName - The name of the blob (e.g. "foo/bar")
-   * @param params.range - The range of the blob to download.
-   *
-   * @returns A `ShelbyBlob` object containing the blob data.
-   *
-   * @example
-   * ```typescript
-   * const blob = await client.download({
-   *   account,
-   *   blobName: "foo/bar.txt",
-   * });
-   * ```
-   */
-  async download(params) {
-    return await this.rpc.getBlob(params);
-  }
-  /**
-   *
-   * Funds an account with ShelbyUSD tokens.
-   *
-   * @param params.address - The address to fund.
-   * @param params.amount - The amount to fund.
-   * @returns The transaction hash of the funded account.
-   *
-   * @example
-   * ```typescript
-   * const hash = await client.fundAccountWithShelbyUSD({
-   *   address: "0x1",
-   *   amount: 100000000,
-   * });
-   * ```
-   */
-  async fundAccountWithShelbyUSD(params) {
-    if (this.config.network === Network.TESTNET) {
-      throw new Error(
-        `ShelbyUSD cannot be minted programmatically on ${Network.TESTNET}. Please use the faucet at https://docs.shelby.xyz/apis/faucet/shelbyusd instead.`
-      );
-    }
-    const { address, amount } = params;
-    try {
-      const faucet = this.config.faucet?.baseUrl ?? "https://faucet.shelbynet.shelby.xyz/fund?asset=shelbyusd";
-      const authToken = this.config.faucet?.authToken;
-      const response = await fetch(`${faucet}`, {
-        method: "POST",
-        body: JSON.stringify({
-          address: normalizeAddress(address).toString(),
-          amount
-        }),
-        headers: {
-          "Content-Type": "application/json",
-          ...authToken ? { Authorization: `Bearer ${authToken}` } : {}
-        }
-      });
-      if (!response.ok) {
-        const errorBody = await response.text();
-        throw new Error(`Failed to fund account: ${errorBody}`);
-      }
-      const json = await response.json();
-      const res = await this.aptos.waitForTransaction({
-        transactionHash: json.txn_hashes[0],
-        options: {
-          timeoutSecs: DEFAULT_TXN_TIMEOUT_SEC,
-          checkSuccess: true
-        }
-      });
-      if (res.type === TransactionResponseType.User) {
-        return res.hash;
-      }
-      throw new Error(
-        `Unexpected transaction received for fund account: ${res.type}`
-      );
-    } catch (error) {
-      throw new Error(`Failed to fund account: ${error}`);
-    }
-  }
-  /**
-   * Fund an account with APT tokens
-   *
-   * @param params.address - The address to fund
-   * @param params.amount - The amount to fund
-   * @returns The transaction hash of the funded account
-   *
-   * @example
-   * ```typescript
-   * const hash = await client.fundAccountWithAPT({
-   *   address: "0x1",
-   *   amount: 100000000,
-   * });
-   * ```
-   */
-  async fundAccountWithAPT(params) {
-    const transaction = await this.aptos.fundAccount({
-      accountAddress: params.address,
-      amount: params.amount
-    });
-    return transaction.hash;
-  }
-};
-
-export {
-  ShelbyClient
-};