@shelby-protocol/sdk 0.0.1-experimental.0

package/README.md

@@ -0,0 +1,60 @@
+# @shelby-protocol/sdk
+
+A TypeScript SDK providing core encoding and decoding utilities for both Node.js and browser environments.
+
+## Installation
+
+```bash
+# Installing the shelby sdk
+pnpm install @shelby-protocol/sdk @aptos-labs/ts-sdk
+
+# If you plan to use the Node.js entrypoint, also install its peer dependencies:
+pnpm install fs-extra glob node-fetch yaml
+```
+
+## Usage
+
+Depending on your runtime environment, import from one of the following entrypoints:
+
+### Node.js
+
+```ts
+import { ShelbyNodeClient } from '@shelby-protocol/sdk/node';
+
+// Your code here...
+```
+
+### Browser
+
+```ts
+import { ShelbyBlob } from '@shelby-protocol/sdk/browser';
+
+// Your code here...
+```
+
+## Peer Dependencies
+
+- `@aptos-labs/ts-sdk`
+
+When using the **Node.js** entrypoint (`@shelby-protocol/sdk/node`), make sure you have the following installed in your project:
+
+- `fs-extra`
+- `glob`
+- `node-fetch`
+- `yaml`
+
+These are marked as **peerDependencies** and are **optional** for browser usage.
+
+## Scripts
+
+```bash
+npm run build      # compile TypeScript to dist/
+npm run test       # watch mode with Vitest
+npm run test:once  # single-run tests
+npm run lint       # run biome diagnostics
+npm run fmt        # auto-format with biome
+```
+
+---
+
+> Please import the entrypoint that matches your environment (`/node`, `/browser`, or the root) to ensure you get the correct implementation and avoid missing-module errors.

package/dist/browser/index.d.ts

@@ -0,0 +1,10 @@
+export { BlobCommitments, BlobCommitmentsSchema, ChunksetCommitment, ChunksetCommitmentSchema } from '../core/commitments.js';
+export { BlobName, BlobNameSchema, ChunkKey, allChunksForBlob, roundSize } from '../core/layout.js';
+export { CHUNKSET_SIZE_BYTES, DEFAULT_PROJECT_DESCRIPTION, DEFAULT_PROJECT_NAME, DEFAULT_SHELBY_BASE_URL, ERASURE_K, ERASURE_M, SHELBY_DEPLOYER, TOKEN_DEPLOYER, TOKEN_OBJECT_ADDRESS, getChunkSizeBytes, getChunksetSizeBytes } from '../core/constants.js';
+export { BlobChunk, BlobEncoding, BlobMetadata, ChunkLocation, ClayEncoding, PendingChunkLocation, SignedChunkCommitment, StoredChunkLocation } from '../core/types/blobs.js';
+export { EncodingOptions } from '../core/types/encodings.js';
+export { ShelbyBlob, createBlobKey } from '../core/blobs.js';
+export { ShelbyClient, ShelbyClientConfig } from '../core/ShelbyClient.js';
+import 'zod';
+import '@aptos-labs/ts-sdk';
+import 'node:stream';

package/dist/browser/index.mjs

@@ -0,0 +1,55 @@
+import "../chunk-PZPMURE4.mjs";
+import "../chunk-MWDW4ROU.mjs";
+import "../chunk-ZHXCVRZX.mjs";
+import "../chunk-IHTPXUYI.mjs";
+import {
+  ShelbyClient
+} from "../chunk-YV67F5NY.mjs";
+import {
+  createBlobKey
+} from "../chunk-QEMIORTL.mjs";
+import {
+  BlobCommitmentsSchema,
+  ChunksetCommitmentSchema
+} from "../chunk-NPM7WXK3.mjs";
+import {
+  BlobNameSchema,
+  ChunkKey,
+  allChunksForBlob,
+  roundSize
+} from "../chunk-GGYTHP5F.mjs";
+import {
+  CHUNKSET_SIZE_BYTES,
+  DEFAULT_PROJECT_DESCRIPTION,
+  DEFAULT_PROJECT_NAME,
+  DEFAULT_SHELBY_BASE_URL,
+  ERASURE_K,
+  ERASURE_M,
+  SHELBY_DEPLOYER,
+  TOKEN_DEPLOYER,
+  TOKEN_OBJECT_ADDRESS,
+  getChunkSizeBytes,
+  getChunksetSizeBytes
+} from "../chunk-D2FERD4A.mjs";
+import "../chunk-7P6ASYW6.mjs";
+export {
+  BlobCommitmentsSchema,
+  BlobNameSchema,
+  CHUNKSET_SIZE_BYTES,
+  ChunkKey,
+  ChunksetCommitmentSchema,
+  DEFAULT_PROJECT_DESCRIPTION,
+  DEFAULT_PROJECT_NAME,
+  DEFAULT_SHELBY_BASE_URL,
+  ERASURE_K,
+  ERASURE_M,
+  SHELBY_DEPLOYER,
+  ShelbyClient,
+  TOKEN_DEPLOYER,
+  TOKEN_OBJECT_ADDRESS,
+  allChunksForBlob,
+  createBlobKey,
+  getChunkSizeBytes,
+  getChunksetSizeBytes,
+  roundSize
+};

package/dist/chunk-7P6ASYW6.mjs

@@ -0,0 +1,9 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+export {
+  __export
+};

package/dist/chunk-A4IG6GSE.mjs

@@ -0,0 +1,21 @@
+import {
+  __export
+} from "./chunk-7P6ASYW6.mjs";
+
+// src/node/testUtil.ts
+var testUtil_exports = {};
+__export(testUtil_exports, {
+  makeChunk: () => makeChunk
+});
+function makeChunk(n) {
+  const c = Buffer.alloc(n);
+  for (let i = 0; i < n; ++i) {
+    c[i] = i % 256;
+  }
+  return c;
+}
+
+export {
+  makeChunk,
+  testUtil_exports
+};

package/dist/chunk-D2FERD4A.mjs

@@ -0,0 +1,39 @@
+// src/core/constants.ts
+var DEFAULT_PROJECT_NAME = "shelby";
+var DEFAULT_PROJECT_DESCRIPTION = "High performance, decentralized storage";
+var DEFAULT_SHELBY_BASE_URL = "https://api.shelby.dev";
+var SHELBY_DEPLOYER = "0xc63d6a5efb0080a6029403131715bd4971e1149f7cc099aac69bb0069b3ddbf5";
+var TOKEN_DEPLOYER = "0x33009e852be7f93762dd0bf303383c2cb2c5cab7a30d8238ca5f9f177ae75124";
+var TOKEN_OBJECT_ADDRESS = "0x249f5c642a63885ff88a5113b3ba0079840af5a1357706f8c7f3bfc5dd12511f";
+var ERASURE_K = 2;
+var ERASURE_M = 1;
+var CHUNKSET_SIZE_BYTES = 10 * 1024 * 1024;
+function computeChunkSizeBytes(k, chunksetSize) {
+  const ret = chunksetSize / k;
+  if (!Number.isInteger(ret)) {
+    throw new Error(
+      `ChunksetSize and erasure coding number of chunks are incompatible: ${chunksetSize}/${k} is not an integer`
+    );
+  }
+  return ret;
+}
+function getChunksetSizeBytes() {
+  return CHUNKSET_SIZE_BYTES;
+}
+function getChunkSizeBytes() {
+  return computeChunkSizeBytes(ERASURE_K, getChunksetSizeBytes());
+}
+
+export {
+  DEFAULT_PROJECT_NAME,
+  DEFAULT_PROJECT_DESCRIPTION,
+  DEFAULT_SHELBY_BASE_URL,
+  SHELBY_DEPLOYER,
+  TOKEN_DEPLOYER,
+  TOKEN_OBJECT_ADDRESS,
+  ERASURE_K,
+  ERASURE_M,
+  CHUNKSET_SIZE_BYTES,
+  getChunksetSizeBytes,
+  getChunkSizeBytes
+};

package/dist/chunk-DBTBUKCW.mjs

@@ -0,0 +1,235 @@
+import {
+  ShelbyClient
+} from "./chunk-YV67F5NY.mjs";
+import {
+  BlobNameSchema
+} from "./chunk-GGYTHP5F.mjs";
+import {
+  sleep
+} from "./chunk-I6NG5GNL.mjs";
+
+// src/node/clients/ShelbyRPCClient.ts
+import { Readable, Transform } from "stream";
+import { AccountAddress } from "@aptos-labs/ts-sdk";
+function encodeURIComponentKeepSlashes(str) {
+  return encodeURIComponent(str).replace(/%2F/g, "/");
+}
+var ShelbyRPCClient = class extends ShelbyClient {
+  /**
+   * The ShelbyRPCClient is used to interact with the Shelby RPC node. This
+   * includes functions like uploading blobs after they have been committed to the
+   * blockchain and downloading blobs.
+   *
+   * @param config.aptos.config - The Aptos config.
+   * @param config.shelby.baseUrl - The base URL of the Shelby RPC node.
+   *
+   * @example
+   * ```typescript
+   * const aptos = new Aptos(new AptosConfig({ network: Network.DEVNET }));
+   * const client = new ShelbyRPCClient({ aptos, shelby: { baseUrl: "https://api.shelby.dev" } });
+   * ```
+   */
+  // biome-ignore lint/complexity/noUselessConstructor: Add JSDoc
+  constructor(config) {
+    super(config);
+  }
+  async #uploadPart(uploadId, partIdx, partData) {
+    const nRetries = 5;
+    for (let i = 0; i < nRetries; ++i) {
+      const partResponse = await fetch(
+        `${this.baseUrl}/api/v1/multipartUpload/uploadPart/${uploadId}/${partIdx}`,
+        {
+          method: "PUT",
+          headers: {
+            "Content-Type": "application/octet-stream"
+          },
+          body: partData
+        }
+      );
+      if (partResponse.ok) return;
+      if (i < nRetries - 1) {
+        const delay = 2 ** i * 100;
+        await sleep(delay);
+      }
+    }
+    throw new Error(`Failed to upload part ${partIdx}.`);
+  }
+  async #putBlobMultipart(account, blobName, blobData, partSize = 5 * 1024 * 1024) {
+    const startResponse = await fetch(
+      `${this.baseUrl}/api/v1/multipartUpload/start`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+          rawAccount: account.toString(),
+          rawBlobName: blobName,
+          rawPartSize: partSize
+        })
+      }
+    );
+    if (!startResponse.ok) {
+      let errorBodyText = "Could not read error body";
+      try {
+        errorBodyText = await startResponse.text();
+      } catch (e) {
+      }
+      throw new Error(
+        `Failed to start multipart upload! status: ${startResponse.status}, body: ${errorBodyText}`
+      );
+    }
+    const { uploadId } = await startResponse.json();
+    const totalParts = Math.ceil(blobData.length / partSize);
+    for (let partIdx = 0; partIdx < totalParts; partIdx++) {
+      const start = partIdx * partSize;
+      const end = Math.min(start + partSize, blobData.length);
+      const partData = blobData.subarray(start, end);
+      await this.#uploadPart(uploadId, partIdx, partData);
+    }
+    const completeResponse = await fetch(
+      `${this.baseUrl}/api/v1/multipartUpload/complete/${uploadId}`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json"
+        }
+      }
+    );
+    if (!completeResponse.ok) {
+      let errorBodyText = "Could not read error body";
+      try {
+        errorBodyText = await completeResponse.text();
+      } catch (e) {
+      }
+      throw new Error(
+        `Failed to complete multipart upload! status: ${completeResponse.status}, body: ${errorBodyText}`
+      );
+    }
+  }
+  /**
+   * Uploads blob data to the Shelby RPC node to be stored and confirmed by storage providers. This should be performed after the blob
+   * commitments have been written to the blockchain.
+   *
+   * @param params.account - The account namespace the blob is stored in (e.g. "0x1")
+   * @param params.blobName - The name of the blob (e.g. "foo/bar")
+   * @param params.blobData - The data to upload.
+   *
+   * @example
+   * ```typescript
+   * const blobData = Buffer.from("Hello, world!");
+   *
+   * await client.putBlob({ account, blobName: "foo/bar.txt", blobData });
+   * ```
+   */
+  async putBlob(params) {
+    BlobNameSchema.parse(params.blobName);
+    await this.#putBlobMultipart(
+      params.account,
+      params.blobName,
+      params.blobData
+    );
+  }
+  // FIXME make this possible to stream in put ^^^
+  /**
+   * Downloads a blob from the Shelby RPC node.
+   *
+   * @param params.account - The account namespace the blob is stored in (e.g. "0x1")
+   * @param params.blobName - The name of the blob (e.g. "foo/bar")
+   * @param params.range - The range of the blob to download.
+   *
+   * @returns A `ShelbyBlob` object containing the blob data.
+   *
+   * @example
+   * ```typescript
+   * const blob = await client.getBlob({
+   *   account,
+   *   blobName: "foo/bar.txt",
+   * });
+   * ```
+   */
+  async getBlob(params) {
+    BlobNameSchema.parse(params.blobName);
+    const url = `${this.baseUrl}/storage/${params.account.toString()}/${encodeURIComponentKeepSlashes(params.blobName)}`;
+    const requestInit = {};
+    if (params.range !== void 0) {
+      const headers = new Headers();
+      const { start, end } = params.range;
+      if (end === void 0) {
+        headers.set("Range", `bytes=${start}-`);
+      } else {
+        if (end < start) {
+          throw new Error("Range end cannot be less than start.");
+        }
+        headers.set("Range", `bytes=${start}-${end}`);
+      }
+      requestInit.headers = headers;
+    }
+    const response = await fetch(url, requestInit);
+    if (!response.ok) {
+      throw new Error(
+        `Failed to download blob: ${response.status} ${response.statusText}`
+      );
+    }
+    if (!response.body) {
+      throw new Error("Response body is null");
+    }
+    const contentLengthHeader = response.headers.get("content-length");
+    if (contentLengthHeader === null) {
+      throw new Error(
+        "Response did not have content-length header, which is required"
+      );
+    }
+    const expectedContentLength = Number.parseInt(contentLengthHeader, 10);
+    if (Number.isNaN(expectedContentLength)) {
+      throw new Error(
+        `Invalid content-length header received: ${contentLengthHeader}`
+      );
+    }
+    let bytesReceived = 0;
+    const reader = response.body.getReader();
+    const sourceStream = new Readable({
+      async read() {
+        try {
+          const { done, value } = await reader.read();
+          if (done) {
+            this.push(null);
+          } else {
+            this.push(Buffer.from(value));
+          }
+        } catch (error) {
+          this.destroy(error);
+        }
+      }
+    });
+    const validationTransform = new Transform({
+      transform(chunk, encoding, callback) {
+        bytesReceived += chunk.length;
+        callback(null, chunk);
+      },
+      flush(callback) {
+        if (bytesReceived !== expectedContentLength) {
+          callback(
+            new Error(
+              `Downloaded data size (${bytesReceived} bytes) does not match content-length header (${expectedContentLength} bytes). This might indicate a partial or corrupted download.`
+            )
+          );
+        } else {
+          callback();
+        }
+      }
+    });
+    sourceStream.pipe(validationTransform);
+    return {
+      account: AccountAddress.from(params.account),
+      name: params.blobName,
+      readable: response.body,
+      stream: validationTransform,
+      contentLength: expectedContentLength
+    };
+  }
+};
+
+export {
+  ShelbyRPCClient
+};

package/dist/chunk-E3QOKRQ4.mjs

@@ -0,0 +1,50 @@
+import {
+  __export
+} from "./chunk-7P6ASYW6.mjs";
+
+// src/node/readableUtil.ts
+var readableUtil_exports = {};
+__export(readableUtil_exports, {
+  readInChunks: () => readInChunks,
+  zeroPadBuffer: () => zeroPadBuffer
+});
+async function* readInChunks(r, chunkSize) {
+  let buffer = Buffer.alloc(chunkSize);
+  let bufferWriteOffset = 0;
+  let idx = 0;
+  for await (const readableChunk of r) {
+    let bufferRemain = chunkSize - bufferWriteOffset;
+    let st = 0;
+    let ed = Math.min(readableChunk.length, st + bufferRemain);
+    while (st < ed) {
+      const slice = readableChunk.slice(st, ed);
+      slice.copy(buffer, bufferWriteOffset);
+      bufferWriteOffset += slice.length;
+      if (bufferWriteOffset >= chunkSize) {
+        yield [idx++, buffer];
+        buffer = Buffer.alloc(chunkSize);
+        bufferWriteOffset = 0;
+      }
+      bufferRemain = chunkSize - bufferWriteOffset;
+      st = ed;
+      ed = Math.min(readableChunk.length, st + bufferRemain);
+    }
+  }
+  if (bufferWriteOffset > 0) {
+    yield [idx++, buffer.subarray(0, bufferWriteOffset)];
+  }
+}
+function zeroPadBuffer(buffer, desiredLength) {
+  if (buffer.length >= desiredLength) {
+    return buffer;
+  }
+  const paddedBuffer = Buffer.alloc(desiredLength);
+  buffer.copy(paddedBuffer);
+  return paddedBuffer;
+}
+
+export {
+  readInChunks,
+  zeroPadBuffer,
+  readableUtil_exports
+};