@shelby-protocol/sdk 0.3.0 → 0.4.0

package/dist/core/sp/chunk_proof.mjs

@@ -0,0 +1,113 @@
+// src/core/sp/chunk_proof.ts
+import { createHash } from "crypto";
+
+// src/core/utils.ts
+import {
+  AccountAddress,
+  Hex
+} from "@aptos-labs/ts-sdk";
+async function concatHashes(parts) {
+  const chunks = parts.map((part) => Hex.fromHexInput(part).toUint8Array());
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+  const combined = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    combined.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return Hex.fromHexInput(
+    new Uint8Array(await crypto.subtle.digest("SHA-256", combined))
+  );
+}
+
+// src/core/sp/chunk_proof.ts
+var SP_WRITE_SUCCESS_ERROR_CODE = 1n;
+function concatHashesSync(left, right) {
+  const combined = new Uint8Array(64);
+  combined.set(left, 0);
+  combined.set(right, 32);
+  const hash = createHash("sha256");
+  hash.update(combined);
+  return new Uint8Array(hash.digest());
+}
+async function generateChunksetInclusionProof(chunksetRoots, chunksetIndex) {
+  if (chunksetRoots.length === 0) {
+    throw new Error("Cannot generate inclusion proof for empty chunkset roots");
+  }
+  if (chunksetIndex < 0 || chunksetIndex >= chunksetRoots.length) {
+    throw new Error(
+      `Chunkset index ${chunksetIndex} out of range [0, ${chunksetRoots.length})`
+    );
+  }
+  if (chunksetRoots.length === 1) {
+    return [];
+  }
+  const zeroHash = new Uint8Array(32);
+  const siblings = [];
+  let currentLeaves = chunksetRoots.map((h) => h.toUint8Array());
+  let currentIdx = chunksetIndex;
+  while (currentLeaves.length > 1) {
+    if (currentLeaves.length % 2 !== 0) {
+      currentLeaves.push(zeroHash);
+    }
+    const siblingIdx = currentIdx % 2 === 0 ? currentIdx + 1 : currentIdx - 1;
+    siblings.push(currentLeaves[siblingIdx]);
+    const nextLeaves = [];
+    for (let i = 0; i < currentLeaves.length; i += 2) {
+      const combined = await concatHashes([
+        currentLeaves[i],
+        currentLeaves[i + 1]
+      ]);
+      nextLeaves.push(combined.toUint8Array());
+    }
+    currentLeaves = nextLeaves;
+    currentIdx = Math.floor(currentIdx / 2);
+  }
+  return siblings;
+}
+function serializeInclusionProofSiblings(siblings) {
+  const totalLength = siblings.reduce((sum, s) => sum + s.length, 0);
+  const proof = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const sibling of siblings) {
+    proof.set(sibling, offset);
+    offset += sibling.length;
+  }
+  return proof;
+}
+function buildChunkProof(chunkIndex, chunkRoots, chunksetToBlobProof) {
+  const chunkToChunksetSiblings = [];
+  let currentIdx = chunkIndex;
+  let currentLevel = [...chunkRoots];
+  while (currentLevel.length > 1) {
+    if (currentLevel.length % 2 !== 0) {
+      currentLevel.push(new Uint8Array(32));
+    }
+    const siblingIdx = currentIdx % 2 === 0 ? currentIdx + 1 : currentIdx - 1;
+    if (siblingIdx < currentLevel.length) {
+      chunkToChunksetSiblings.push(currentLevel[siblingIdx]);
+    }
+    const nextLevel = [];
+    for (let i = 0; i < currentLevel.length; i += 2) {
+      nextLevel.push(concatHashesSync(currentLevel[i], currentLevel[i + 1]));
+    }
+    currentLevel = nextLevel;
+    currentIdx = Math.floor(currentIdx / 2);
+  }
+  const totalLength = chunkToChunksetSiblings.reduce((sum, s) => sum + s.length, 0) + chunksetToBlobProof.length;
+  const fullProof = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const sibling of chunkToChunksetSiblings) {
+    fullProof.set(sibling, offset);
+    offset += sibling.length;
+  }
+  fullProof.set(chunksetToBlobProof, offset);
+  return fullProof;
+}
+export {
+  SP_WRITE_SUCCESS_ERROR_CODE,
+  buildChunkProof,
+  concatHashesSync,
+  generateChunksetInclusionProof,
+  serializeInclusionProofSiblings
+};

package/dist/core/sp/index.d.ts

@@ -0,0 +1,3 @@
+export { SP_WRITE_SUCCESS_ERROR_CODE, buildChunkProof, concatHashesSync, generateChunksetInclusionProof, serializeInclusionProofSiblings } from './chunk_proof.js';
+export { SpReadChunkParams, SpReadResult, SpWriteChunkParams, SpWriteClient, SpWriteResult } from './sp_write_client.js';
+import '@aptos-labs/ts-sdk';

package/dist/core/sp/index.mjs

@@ -0,0 +1,402 @@
+// src/core/sp/chunk_proof.ts
+import { createHash } from "crypto";
+
+// src/core/utils.ts
+import {
+  AccountAddress,
+  Hex
+} from "@aptos-labs/ts-sdk";
+async function concatHashes(parts) {
+  const chunks = parts.map((part) => Hex.fromHexInput(part).toUint8Array());
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+  const combined = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    combined.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return Hex.fromHexInput(
+    new Uint8Array(await crypto.subtle.digest("SHA-256", combined))
+  );
+}
+
+// src/core/sp/chunk_proof.ts
+var SP_WRITE_SUCCESS_ERROR_CODE = 1n;
+function concatHashesSync(left, right) {
+  const combined = new Uint8Array(64);
+  combined.set(left, 0);
+  combined.set(right, 32);
+  const hash = createHash("sha256");
+  hash.update(combined);
+  return new Uint8Array(hash.digest());
+}
+async function generateChunksetInclusionProof(chunksetRoots, chunksetIndex) {
+  if (chunksetRoots.length === 0) {
+    throw new Error("Cannot generate inclusion proof for empty chunkset roots");
+  }
+  if (chunksetIndex < 0 || chunksetIndex >= chunksetRoots.length) {
+    throw new Error(
+      `Chunkset index ${chunksetIndex} out of range [0, ${chunksetRoots.length})`
+    );
+  }
+  if (chunksetRoots.length === 1) {
+    return [];
+  }
+  const zeroHash = new Uint8Array(32);
+  const siblings = [];
+  let currentLeaves = chunksetRoots.map((h) => h.toUint8Array());
+  let currentIdx = chunksetIndex;
+  while (currentLeaves.length > 1) {
+    if (currentLeaves.length % 2 !== 0) {
+      currentLeaves.push(zeroHash);
+    }
+    const siblingIdx = currentIdx % 2 === 0 ? currentIdx + 1 : currentIdx - 1;
+    siblings.push(currentLeaves[siblingIdx]);
+    const nextLeaves = [];
+    for (let i = 0; i < currentLeaves.length; i += 2) {
+      const combined = await concatHashes([
+        currentLeaves[i],
+        currentLeaves[i + 1]
+      ]);
+      nextLeaves.push(combined.toUint8Array());
+    }
+    currentLeaves = nextLeaves;
+    currentIdx = Math.floor(currentIdx / 2);
+  }
+  return siblings;
+}
+function serializeInclusionProofSiblings(siblings) {
+  const totalLength = siblings.reduce((sum, s) => sum + s.length, 0);
+  const proof = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const sibling of siblings) {
+    proof.set(sibling, offset);
+    offset += sibling.length;
+  }
+  return proof;
+}
+function buildChunkProof(chunkIndex, chunkRoots, chunksetToBlobProof) {
+  const chunkToChunksetSiblings = [];
+  let currentIdx = chunkIndex;
+  let currentLevel = [...chunkRoots];
+  while (currentLevel.length > 1) {
+    if (currentLevel.length % 2 !== 0) {
+      currentLevel.push(new Uint8Array(32));
+    }
+    const siblingIdx = currentIdx % 2 === 0 ? currentIdx + 1 : currentIdx - 1;
+    if (siblingIdx < currentLevel.length) {
+      chunkToChunksetSiblings.push(currentLevel[siblingIdx]);
+    }
+    const nextLevel = [];
+    for (let i = 0; i < currentLevel.length; i += 2) {
+      nextLevel.push(concatHashesSync(currentLevel[i], currentLevel[i + 1]));
+    }
+    currentLevel = nextLevel;
+    currentIdx = Math.floor(currentIdx / 2);
+  }
+  const totalLength = chunkToChunksetSiblings.reduce((sum, s) => sum + s.length, 0) + chunksetToBlobProof.length;
+  const fullProof = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const sibling of chunkToChunksetSiblings) {
+    fullProof.set(sibling, offset);
+    offset += sibling.length;
+  }
+  fullProof.set(chunksetToBlobProof, offset);
+  return fullProof;
+}
+
+// src/core/sp/sp_write_client.ts
+import * as net from "net";
+import { create, fromBinary, toBinary } from "@bufbuild/protobuf";
+
+// src/gen/rpc_server_pb.ts
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
+var file_rpc_server = /* @__PURE__ */ fileDesc("ChBycGNfc2VydmVyLnByb3RvEgNjYXYiGgoHUGF5bWVudBIPCgdwYXltZW50GAEgASgMIhkKCVByb29mTm9kZRIMCgRoYXNoGAEgASgMIiwKBVByb29mEiMKC3Byb29mX25vZGVzGAEgAygLMg4uY2F2LlByb29mTm9kZSJFCghDaHVua0tleRIQCghibG9iX3VpZBgBIAEoBBIUCgxjaHVua3NldF9pZHgYAiABKA0SEQoJY2h1bmtfaWR4GAMgASgNIk8KC1JlYWRSZXF1ZXN0EiEKCmlkZW50aWZpZXIYASABKAsyDS5jYXYuQ2h1bmtLZXkSHQoHcGF5bWVudBgCIAEoCzIMLmNhdi5QYXltZW50InEKGUNsYXlIZWxwZXJEYXRhUmVhZFJlcXVlc3QSIQoKaWRlbnRpZmllchgBIAEoCzINLmNhdi5DaHVua0tleRIYChB0YXJnZXRfY2h1bmtfaWR4GAIgASgNEhcKD2hlbHBlcl9zZXRfbWFzaxgDIAEoBCJgCgxXcml0ZVJlcXVlc3QSIQoKaWRlbnRpZmllchgBIAEoCzINLmNhdi5DaHVua0tleRIMCgRib2R5GAIgASgMEhkKBXByb29mGAMgASgLMgouY2F2LlByb29mSgQIBBAFIsQBCgdSZXF1ZXN0EiQKCHJlYWRfcmVxGAEgASgLMhAuY2F2LlJlYWRSZXF1ZXN0SAASJgoJd3JpdGVfcmVxGAIgASgLMhEuY2F2LldyaXRlUmVxdWVzdEgAEjkKD2hlbHBlcl9yZWFkX3JlcRgFIAEoCzIeLmNhdi5DbGF5SGVscGVyRGF0YVJlYWRSZXF1ZXN0SAASEgoKcmVxdWVzdF9pZBgDIAEoBBIRCglzaWduYXR1cmUYBCABKAxCCQoHcmVxdWVzdCJXCgxSZWFkUmVzcG9uc2USDAoEYm9keRgBIAEoDBIZCgVwcm9vZhgCIAEoCzIKLmNhdi5Qcm9vZhIeChZoZWxwZXJfY2h1bmtfcm9vdF9oYXNoGAMgASgMIk0KDVdyaXRlUmVzcG9uc2USGgoSYmxvYl9hY2tfc2lnbmF0dXJlGAIgASgMEiAKGG1pc3NpbmdfY2h1bmtzZXRfaW5kaWNlcxgDIAMoDSKWAQoIUmVzcG9uc2USEgoKcmVxdWVzdF9pZBgBIAEoBBISCgplcnJvcl9jb2RlGAIgASgEEiYKCXJlYWRfcmVzcBgDIAEoCzIRLmNhdi5SZWFkUmVzcG9uc2VIABIoCgp3cml0ZV9yZXNwGAQgASgLMhIuY2F2LldyaXRlUmVzcG9uc2VIAEIKCghyZXNwb25zZUoECAUQBmIGcHJvdG8z");
+var ProofNodeSchema = /* @__PURE__ */ messageDesc(file_rpc_server, 1);
+var ProofSchema = /* @__PURE__ */ messageDesc(file_rpc_server, 2);
+var ChunkKeySchema = /* @__PURE__ */ messageDesc(file_rpc_server, 3);
+var ReadRequestSchema = /* @__PURE__ */ messageDesc(file_rpc_server, 4);
+var WriteRequestSchema = /* @__PURE__ */ messageDesc(file_rpc_server, 6);
+var RequestSchema = /* @__PURE__ */ messageDesc(file_rpc_server, 7);
+var ResponseSchema = /* @__PURE__ */ messageDesc(file_rpc_server, 10);
+
+// src/core/sp/sp_write_client.ts
+var SP_CLIENT_LOG = process.env.SHELBY_SP_CLIENT_LOG === "1";
+var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
+var DEFAULT_CONNECTIONS = 4;
+var SpWriteClient = class {
+  constructor(host, port, numConnections = DEFAULT_CONNECTIONS, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
+    this.host = host;
+    this.port = port;
+    this.numConnections = numConnections;
+    this.requestTimeoutMs = requestTimeoutMs;
+  }
+  connections = [];
+  nextConnectionIdx = 0;
+  nextRequestId = 1n;
+  connected = false;
+  async connect() {
+    if (this.connected) {
+      return;
+    }
+    const connectPromises = [];
+    for (let i = 0; i < this.numConnections; i++) {
+      connectPromises.push(this.createConnection());
+    }
+    await Promise.all(connectPromises);
+    this.connected = true;
+  }
+  createConnection() {
+    return new Promise((resolve, reject) => {
+      const socket = new net.Socket();
+      let settled = false;
+      const state = {
+        socket,
+        responseBuffer: Buffer.alloc(0),
+        expectedSize: null,
+        pending: /* @__PURE__ */ new Map(),
+        writeQueue: [],
+        draining: false,
+        closed: false
+      };
+      socket.on("connect", () => {
+        this.connections.push(state);
+        settled = true;
+        resolve();
+      });
+      socket.on("data", (data) => {
+        this.handleData(state, data);
+      });
+      socket.on("error", (err) => {
+        if (SP_CLIENT_LOG)
+          console.error(
+            `[SpWriteClient] socket error ${this.host}:${this.port}: ${err.message}`
+          );
+        if (!settled) {
+          settled = true;
+          reject(err);
+        }
+        this.handleDisconnect(state, err);
+      });
+      socket.on("close", (hadError) => {
+        if (SP_CLIENT_LOG)
+          console.error(
+            `[SpWriteClient] socket closed ${this.host}:${this.port} (hadError=${hadError})`
+          );
+        this.handleDisconnect(state, new Error("SP connection closed"));
+      });
+      socket.on("timeout", () => {
+        if (SP_CLIENT_LOG)
+          console.error(
+            `[SpWriteClient] socket idle timeout ${this.host}:${this.port} after ${this.requestTimeoutMs}ms`
+          );
+        socket.destroy(new Error("SP connection timed out"));
+      });
+      socket.connect(this.port, this.host);
+    });
+  }
+  handleData(state, data) {
+    state.responseBuffer = Buffer.concat([state.responseBuffer, data]);
+    while (true) {
+      if (state.expectedSize === null) {
+        if (state.responseBuffer.length < 8) {
+          return;
+        }
+        state.expectedSize = Number(state.responseBuffer.readBigUInt64LE(0));
+      }
+      const totalNeeded = 8 + state.expectedSize;
+      if (state.responseBuffer.length < totalNeeded) {
+        return;
+      }
+      const responseBytes = state.responseBuffer.subarray(8, totalNeeded);
+      state.responseBuffer = state.responseBuffer.subarray(totalNeeded);
+      state.expectedSize = null;
+      try {
+        const response = fromBinary(ResponseSchema, responseBytes);
+        const pending = state.pending.get(response.requestId);
+        if (pending) {
+          clearTimeout(pending.timeoutId);
+          state.pending.delete(response.requestId);
+          const success = response.errorCode === SP_WRITE_SUCCESS_ERROR_CODE;
+          if (pending.resolver.kind === "write") {
+            const blobAckSignature = response.response.case === "writeResp" ? response.response.value.blobAckSignature : new Uint8Array(0);
+            pending.resolver.resolve({
+              success,
+              errorCode: response.errorCode,
+              blobAckSignature
+            });
+          } else {
+            const body = response.response.case === "readResp" ? response.response.value.body : new Uint8Array(0);
+            pending.resolver.resolve({
+              success,
+              errorCode: response.errorCode,
+              bytesRead: body.byteLength
+            });
+          }
+        }
+      } catch (error) {
+        for (const [, pending] of state.pending) {
+          clearTimeout(pending.timeoutId);
+          pending.reject(
+            error instanceof Error ? error : new Error(String(error))
+          );
+        }
+        state.pending.clear();
+      }
+    }
+  }
+  handleDisconnect(state, error) {
+    if (state.closed) {
+      return;
+    }
+    state.closed = true;
+    for (const [, pending] of state.pending) {
+      clearTimeout(pending.timeoutId);
+      pending.reject(error);
+    }
+    state.pending.clear();
+    const idx = this.connections.indexOf(state);
+    if (idx !== -1) {
+      this.connections.splice(idx, 1);
+    }
+  }
+  drainConnection(state) {
+    if (state.draining || state.writeQueue.length === 0 || state.closed) {
+      return;
+    }
+    state.draining = true;
+    const frame = state.writeQueue.shift();
+    const canContinue = state.socket.write(frame, () => {
+      state.draining = false;
+      this.drainConnection(state);
+    });
+    if (!canContinue) {
+      state.socket.once("drain", () => {
+        state.draining = false;
+        this.drainConnection(state);
+      });
+    } else {
+      state.draining = false;
+      this.drainConnection(state);
+    }
+  }
+  selectConnection() {
+    if (this.connections.length === 0) {
+      throw new Error("Not connected to SP");
+    }
+    for (let i = 0; i < this.connections.length; i++) {
+      const idx = (this.nextConnectionIdx + i) % this.connections.length;
+      const conn = this.connections[idx];
+      if (!conn.closed) {
+        this.nextConnectionIdx = (idx + 1) % this.connections.length;
+        return conn;
+      }
+    }
+    throw new Error("No available SP connections");
+  }
+  async writeChunk(params) {
+    if (!this.connected) {
+      await this.connect();
+    }
+    const conn = this.selectConnection();
+    const requestId = this.nextRequestId++;
+    const chunkKey = create(ChunkKeySchema, {
+      blobUid: params.blobUid,
+      chunksetIdx: params.chunksetIdx,
+      chunkIdx: params.chunkIdx
+    });
+    const proofNodes = [];
+    for (let i = 0; i < params.proof.length; i += 32) {
+      proofNodes.push(
+        create(ProofNodeSchema, {
+          hash: params.proof.slice(i, i + 32)
+        })
+      );
+    }
+    const request = create(RequestSchema, {
+      request: {
+        case: "writeReq",
+        value: create(WriteRequestSchema, {
+          identifier: chunkKey,
+          body: params.chunkData,
+          proof: create(ProofSchema, { proofNodes })
+        })
+      },
+      requestId,
+      signature: new Uint8Array(64)
+    });
+    const requestBytes = toBinary(RequestSchema, request);
+    const lengthBuffer = Buffer.alloc(8);
+    lengthBuffer.writeBigUInt64LE(BigInt(requestBytes.length), 0);
+    const frame = Buffer.concat([lengthBuffer, Buffer.from(requestBytes)]);
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        conn.pending.delete(requestId);
+        reject(new Error(`SP write request ${requestId} timed out`));
+      }, this.requestTimeoutMs);
+      conn.pending.set(requestId, {
+        resolver: { kind: "write", resolve },
+        reject,
+        timeoutId
+      });
+      conn.writeQueue.push(frame);
+      this.drainConnection(conn);
+    });
+  }
+  async readChunk(params) {
+    if (!this.connected) {
+      await this.connect();
+    }
+    const conn = this.selectConnection();
+    const requestId = this.nextRequestId++;
+    const chunkKey = create(ChunkKeySchema, {
+      blobUid: params.blobUid,
+      chunksetIdx: params.chunksetIdx,
+      chunkIdx: params.chunkIdx
+    });
+    const request = create(RequestSchema, {
+      request: {
+        case: "readReq",
+        value: create(ReadRequestSchema, {
+          identifier: chunkKey
+        })
+      },
+      requestId,
+      signature: new Uint8Array(64)
+    });
+    const requestBytes = toBinary(RequestSchema, request);
+    const lengthBuffer = Buffer.alloc(8);
+    lengthBuffer.writeBigUInt64LE(BigInt(requestBytes.length), 0);
+    const frame = Buffer.concat([lengthBuffer, Buffer.from(requestBytes)]);
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        conn.pending.delete(requestId);
+        reject(new Error(`SP read request ${requestId} timed out`));
+      }, this.requestTimeoutMs);
+      conn.pending.set(requestId, {
+        resolver: { kind: "read", resolve },
+        reject,
+        timeoutId
+      });
+      conn.writeQueue.push(frame);
+      this.drainConnection(conn);
+    });
+  }
+  // biome-ignore lint/suspicious/useAwait: callers rely on Promise return
+  async close() {
+    this.connected = false;
+    for (const conn of this.connections) {
+      conn.closed = true;
+      conn.socket.destroy();
+    }
+    this.connections = [];
+  }
+};
+export {
+  SP_WRITE_SUCCESS_ERROR_CODE,
+  SpWriteClient,
+  buildChunkProof,
+  concatHashesSync,
+  generateChunksetInclusionProof,
+  serializeInclusionProofSiblings
+};

package/dist/core/sp/sp_write_client.d.ts

@@ -0,0 +1,53 @@
+interface SpWriteChunkParams {
+    blobUid: bigint;
+    chunksetIdx: number;
+    chunkIdx: number;
+    chunkData: Uint8Array;
+    proof: Uint8Array;
+}
+interface SpWriteResult {
+    success: boolean;
+    errorCode: bigint;
+    /**
+     * BLS blob-ack signature returned by the SP when it has received all
+     * chunksets for the blob. Empty (zero-length) if the blob is not yet
+     * complete on this SP. Present only for the last write that completes the
+     * blob on this particular SP.
+     */
+    blobAckSignature: Uint8Array;
+}
+interface SpReadChunkParams {
+    blobUid: bigint;
+    chunksetIdx: number;
+    chunkIdx: number;
+}
+interface SpReadResult {
+    success: boolean;
+    errorCode: bigint;
+    bytesRead: number;
+}
+/**
+ * Client for writing erasure chunks directly to a Storage Provider over TCP/protobuf.
+ */
+declare class SpWriteClient {
+    private readonly host;
+    private readonly port;
+    private readonly numConnections;
+    private readonly requestTimeoutMs;
+    private connections;
+    private nextConnectionIdx;
+    private nextRequestId;
+    private connected;
+    constructor(host: string, port: number, numConnections?: number, requestTimeoutMs?: number);
+    connect(): Promise<void>;
+    private createConnection;
+    private handleData;
+    private handleDisconnect;
+    private drainConnection;
+    private selectConnection;
+    writeChunk(params: SpWriteChunkParams): Promise<SpWriteResult>;
+    readChunk(params: SpReadChunkParams): Promise<SpReadResult>;
+    close(): Promise<void>;
+}
+
+export { type SpReadChunkParams, type SpReadResult, type SpWriteChunkParams, SpWriteClient, type SpWriteResult };