@ardrive/turbo-sdk 1.30.0 → 1.31.0-alpha.1

package/bundles/web.bundle.min.js

@@ -197377,7 +197377,7 @@ var require_axios = __commonJS({
       kind === "object" && isFunction4(thing.toString) && thing.toString() === "[object FormData]"));
     };
     var isURLSearchParams2 = kindOfTest2("URLSearchParams");
-    var [isReadableStream2, isRequest2, isResponse2, isHeaders2] = ["ReadableStream", "Request", "Response", "Headers"].map(kindOfTest2);
+    var [isReadableStream3, isRequest2, isResponse2, isHeaders2] = ["ReadableStream", "Request", "Response", "Headers"].map(kindOfTest2);
     var trim2 = (str) => str.trim ? str.trim() : str.replace(/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, "");
     function forEach3(obj, fn2, { allOwnKeys = false } = {}) {
       if (obj === null || typeof obj === "undefined") {
@@ -197662,7 +197662,7 @@ var require_axios = __commonJS({
       isBoolean: isBoolean4,
       isObject: isObject5,
       isPlainObject: isPlainObject2,
-      isReadableStream: isReadableStream2,
+      isReadableStream: isReadableStream3,
       isRequest: isRequest2,
       isResponse: isResponse2,
       isHeaders: isHeaders2,
@@ -220948,6 +220948,7 @@ function isSolanaWalletAdapter(walletAdapter) {
 function isEthereumWalletAdapter(walletAdapter) {
   return "getSigner" in walletAdapter;
 }
+var validChunkingModes = ["force", "disabled", "auto"];
 
 // src/common/logger.ts
 init_dirname();
@@ -220959,7 +220960,7 @@ var import_winston = __toESM(require_winston(), 1);
 init_dirname();
 init_buffer2();
 init_process2();
-var version21 = "1.30.0-alpha.1";
+var version21 = "1.30.0";
 
 // src/common/logger.ts
 var TurboWinstonLogger = class _TurboWinstonLogger {
@@ -225543,6 +225544,7 @@ var TurboHTTPService = class {
     }
     const { body, duplex } = await toFetchBody(data);
     try {
+      this.logger.debug("Posting data via fetch", { endpoint, headers });
       const res = await fetch(this.axios.defaults.baseURL + endpoint, {
         method: "POST",
         headers,
@@ -274373,6 +274375,11 @@ function pLimit(concurrency) {
   return generator;
 }
 
+// src/common/chunked.ts
+init_dirname();
+init_buffer2();
+init_process2();
+
 // src/common/events.ts
 init_dirname();
 init_buffer2();
@@ -274595,6 +274602,340 @@ function createStreamWithSigningEvents({
   });
 }
 
+// src/common/chunked.ts
+var fiveMiB = 5 * 1024 * 1024;
+var fiveHundredMiB = fiveMiB * 100;
+var defaultMaxChunkConcurrency = 5;
+var minChunkByteCount = fiveMiB;
+var defaultChunkByteCount = minChunkByteCount;
+var backlogQueueFactor = 2;
+var chunkingHeader = { "x-chunking-version": "2" };
+var ChunkedUploader = class {
+  constructor({
+    http,
+    token,
+    maxChunkConcurrency = defaultMaxChunkConcurrency,
+    chunkByteCount = defaultChunkByteCount,
+    logger: logger19 = TurboWinstonLogger.default,
+    chunkingMode = "auto",
+    dataItemByteCount
+  }) {
+    this.chunkByteCount = chunkByteCount;
+    this.maxChunkConcurrency = maxChunkConcurrency;
+    this.http = http;
+    this.token = token;
+    this.logger = logger19;
+    this.assertChunkParams({
+      chunkByteCount,
+      chunkingMode,
+      maxChunkConcurrency
+    });
+    this.shouldUseChunkUploader = this.shouldChunkUpload({
+      chunkByteCount,
+      chunkingMode,
+      dataItemByteCount
+    });
+    this.maxBacklogQueue = this.maxChunkConcurrency * backlogQueueFactor;
+  }
+  shouldChunkUpload({
+    chunkByteCount,
+    chunkingMode,
+    dataItemByteCount
+  }) {
+    if (chunkingMode === "disabled") {
+      return false;
+    }
+    if (chunkingMode === "force") {
+      return true;
+    }
+    const isMoreThanTwoChunksOfData = dataItemByteCount > chunkByteCount * 2;
+    return isMoreThanTwoChunksOfData;
+  }
+  assertChunkParams({
+    chunkByteCount,
+    chunkingMode,
+    maxChunkConcurrency
+  }) {
+    if (Number.isNaN(maxChunkConcurrency) || !Number.isInteger(maxChunkConcurrency) || maxChunkConcurrency < 1) {
+      throw new Error(
+        "Invalid max chunk concurrency. Must be an integer of at least 1."
+      );
+    }
+    if (Number.isNaN(chunkByteCount) || !Number.isInteger(chunkByteCount) || chunkByteCount < fiveMiB || chunkByteCount > fiveHundredMiB) {
+      throw new Error(
+        "Invalid chunk size. Must be an integer between 5 MiB and 500 MiB."
+      );
+    }
+    if (typeof chunkingMode !== "string" || !validChunkingModes.includes(chunkingMode)) {
+      throw new Error(
+        `Invalid chunking mode. Must be one of: ${validChunkingModes.join(
+          ", "
+        )}`
+      );
+    }
+  }
+  /**
+   * Initialize or resume an upload session, returning the upload ID.
+   */
+  async initUpload() {
+    const res = await this.http.get({
+      endpoint: `/chunks/${this.token}/-1/-1?chunkSize=${this.chunkByteCount}`,
+      headers: chunkingHeader
+    });
+    if (res.chunkSize !== this.chunkByteCount) {
+      this.logger.warn("Chunk size mismatch! Overriding with server value.", {
+        expected: this.chunkByteCount,
+        actual: res.chunkSize
+      });
+      this.chunkByteCount = res.chunkSize;
+    }
+    return res.id;
+  }
+  async upload({
+    dataItemSizeFactory,
+    dataItemStreamFactory,
+    dataItemOpts,
+    signal,
+    events
+  }) {
+    const uploadId = await this.initUpload();
+    const dataItemByteCount = dataItemSizeFactory();
+    const emitter = new TurboEventEmitter(events);
+    const { stream: stream2, resume } = createStreamWithUploadEvents({
+      data: dataItemStreamFactory(),
+      dataSize: dataItemByteCount,
+      emitter
+    });
+    this.logger.debug(`Starting chunked upload`, {
+      token: this.token,
+      uploadId,
+      totalSize: dataItemByteCount,
+      chunkByteCount: this.chunkByteCount,
+      maxChunkConcurrency: this.maxChunkConcurrency,
+      inputStreamType: isReadableStream2(stream2) ? "ReadableStream" : "Readable"
+    });
+    const inFlight = /* @__PURE__ */ new Set();
+    const internalAbort = new AbortController();
+    const combinedSignal = combineAbortSignals([internalAbort.signal, signal]);
+    const limit = pLimit(this.maxChunkConcurrency);
+    let currentOffset = 0;
+    let currentChunkPartNumber = 0;
+    let firstError;
+    let uploadedBytes = 0;
+    const chunks = splitIntoChunks(stream2, this.chunkByteCount);
+    resume();
+    for await (const chunk of chunks) {
+      if (combinedSignal?.aborted) {
+        internalAbort.abort();
+        await Promise.allSettled(inFlight);
+        firstError ??= new CanceledError2();
+        break;
+      }
+      const chunkPartNumber = ++currentChunkPartNumber;
+      const chunkByteCount = chunk.length;
+      const chunkOffset = currentOffset;
+      currentOffset += chunkByteCount;
+      this.logger.debug("Queueing chunk", {
+        chunkPartNumber,
+        chunkOffset,
+        chunkByteCount
+      });
+      const promise = limit(async () => {
+        if (firstError !== void 0) {
+          return;
+        }
+        this.logger.debug("Uploading chunk", {
+          chunkPartNumber,
+          chunkOffset,
+          chunkByteCount
+        });
+        await this.http.post({
+          endpoint: `/chunks/${this.token}/${uploadId}/${chunkOffset}`,
+          data: chunk,
+          headers: {
+            "Content-Type": "application/octet-stream",
+            ...chunkingHeader
+          },
+          signal: combinedSignal
+        });
+        uploadedBytes += chunkByteCount;
+        this.logger.debug("Chunk uploaded", {
+          chunkPartNumber,
+          chunkOffset,
+          chunkByteCount
+        });
+        emitter.emit("upload-progress", {
+          processedBytes: uploadedBytes,
+          totalBytes: dataItemByteCount
+        });
+      }).catch((err) => {
+        this.logger.error("Chunk upload failed", {
+          id: chunkPartNumber,
+          offset: chunkOffset,
+          size: chunkByteCount,
+          err
+        });
+        emitter.emit("upload-error", err);
+        internalAbort.abort(err);
+        firstError = firstError ?? err;
+      });
+      inFlight.add(promise);
+      promise.finally(() => inFlight.delete(promise));
+      if (inFlight.size >= this.maxBacklogQueue) {
+        await Promise.race(inFlight);
+        if (combinedSignal?.aborted) {
+          internalAbort.abort();
+          await Promise.allSettled(inFlight);
+          firstError ??= new CanceledError2();
+          break;
+        }
+      }
+    }
+    await Promise.all(inFlight);
+    if (firstError !== void 0) {
+      throw firstError;
+    }
+    const paidByHeader = {};
+    if (dataItemOpts?.paidBy !== void 0) {
+      paidByHeader["x-paid-by"] = Array.isArray(dataItemOpts.paidBy) ? dataItemOpts.paidBy.join(",") : dataItemOpts.paidBy;
+    }
+    const finalizeResponse = await this.http.post({
+      endpoint: `/chunks/${this.token}/${uploadId}/-1`,
+      data: Buffer2.alloc(0),
+      headers: {
+        "Content-Type": "application/octet-stream",
+        ...paidByHeader,
+        ...chunkingHeader
+      },
+      signal: combinedSignal
+    });
+    emitter.emit("upload-success");
+    return finalizeResponse;
+  }
+};
+async function* splitIntoChunks(source, chunkByteCount) {
+  if (isReadableStream2(source)) {
+    yield* splitReadableStreamIntoChunks(source, chunkByteCount);
+  } else {
+    yield* splitReadableIntoChunks(source, chunkByteCount);
+  }
+}
+async function* splitReadableIntoChunks(source, chunkByteCount) {
+  const queue2 = [];
+  let total = 0;
+  let encoder;
+  for await (const piece of source) {
+    const u82 = piece instanceof Uint8Array ? new Uint8Array(piece.buffer, piece.byteOffset, piece.byteLength) : (encoder ??= new TextEncoder()).encode(String(piece));
+    queue2.push(u82);
+    total += u82.length;
+    while (total >= chunkByteCount) {
+      const out = new Uint8Array(chunkByteCount);
+      let remaining = out.length;
+      let off2 = 0;
+      while (remaining > 0) {
+        const head = queue2[0];
+        const take = Math.min(remaining, head.length);
+        out.set(head.subarray(0, take), off2);
+        off2 += take;
+        remaining -= take;
+        if (take === head.length) {
+          queue2.shift();
+        } else {
+          queue2[0] = head.subarray(take);
+        }
+      }
+      total -= chunkByteCount;
+      yield Buffer2.from(out.buffer, out.byteOffset, out.byteLength);
+    }
+  }
+  if (total > 0) {
+    const out = new Uint8Array(total);
+    let off2 = 0;
+    while (queue2.length > 0) {
+      const head = queue2.shift();
+      out.set(head, off2);
+      off2 += head.length;
+    }
+    yield Buffer2.from(out.buffer, out.byteOffset, out.byteLength);
+  }
+}
+async function* splitReadableStreamIntoChunks(source, chunkByteCount) {
+  const reader = source.getReader();
+  const queue2 = [];
+  let total = 0;
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done)
+        break;
+      const u82 = new Uint8Array(
+        value.buffer,
+        value.byteOffset,
+        value.byteLength
+      );
+      queue2.push(u82);
+      total += u82.length;
+      while (total >= chunkByteCount) {
+        const out = new Uint8Array(chunkByteCount);
+        let remaining = out.length;
+        let off2 = 0;
+        while (remaining > 0) {
+          const head = queue2[0];
+          const take = Math.min(remaining, head.length);
+          out.set(head.subarray(0, take), off2);
+          off2 += take;
+          remaining -= take;
+          if (take === head.length) {
+            queue2.shift();
+          } else {
+            queue2[0] = head.subarray(take);
+          }
+        }
+        total -= chunkByteCount;
+        yield Buffer2.from(out.buffer, out.byteOffset, out.byteLength);
+      }
+    }
+    if (total > 0) {
+      const out = new Uint8Array(total);
+      let off2 = 0;
+      while (queue2.length > 0) {
+        const head = queue2.shift();
+        out.set(head, off2);
+        off2 += head.length;
+      }
+      yield Buffer2.from(out.buffer, out.byteOffset, out.byteLength);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+}
+function isReadableStream2(source) {
+  if (typeof ReadableStream !== "undefined" && source instanceof ReadableStream) {
+    return true;
+  }
+  return source !== null && typeof source === "object" && "getReader" in source && typeof source.getReader === "function";
+}
+function combineAbortSignals(signals) {
+  const real = signals.filter(Boolean);
+  if (real.length === 0)
+    return void 0;
+  const anyFn = AbortSignal.any;
+  if (typeof anyFn === "function") {
+    return anyFn(real);
+  }
+  const controller = new AbortController();
+  for (const s7 of real) {
+    const sig = s7;
+    if (sig.aborted) {
+      controller.abort(sig.reason);
+      break;
+    }
+    const onAbort = () => controller.abort(sig.reason);
+    s7.addEventListener("abort", onAbort, { once: true });
+  }
+  return controller.signal;
+}
+
 // src/common/upload.ts
 function isTurboUploadFileWithStreamFactoryParams(params) {
   return "fileStreamFactory" in params;
@@ -274679,7 +275020,10 @@ var TurboAuthenticatedBaseUploadService = class extends TurboUnauthenticatedUplo
     data,
     dataItemOpts,
     signal,
-    events
+    events,
+    chunkByteCount,
+    chunkingMode,
+    maxChunkConcurrency
   }) {
     if (isBlob2(data)) {
       const streamFactory = () => data.stream();
@@ -274705,7 +275049,10 @@ var TurboAuthenticatedBaseUploadService = class extends TurboUnauthenticatedUplo
       fileSizeFactory: () => dataBuffer.byteLength,
       signal,
       dataItemOpts,
-      events
+      events,
+      chunkByteCount,
+      chunkingMode,
+      maxChunkConcurrency
     });
   }
   resolveUploadFileConfig(params) {
@@ -274737,17 +275084,37 @@ var TurboAuthenticatedBaseUploadService = class extends TurboUnauthenticatedUplo
     let lastError = void 0;
     let lastStatusCode = void 0;
     const emitter = new TurboEventEmitter(events);
-    const { dataItemStreamFactory, dataItemSizeFactory } = await this.signer.signDataItem({
-      fileStreamFactory,
-      fileSizeFactory,
-      dataItemOpts,
-      emitter
-    });
     while (retries < maxRetries) {
       if (signal?.aborted) {
         throw new CanceledError2();
       }
+      const { dataItemStreamFactory, dataItemSizeFactory } = await this.signer.signDataItem({
+        fileStreamFactory,
+        fileSizeFactory,
+        dataItemOpts,
+        emitter
+      });
       try {
+        const { chunkByteCount, maxChunkConcurrency } = params;
+        const chunkedUploader = new ChunkedUploader({
+          http: this.httpService,
+          token: this.token,
+          maxChunkConcurrency,
+          chunkByteCount,
+          logger: this.logger,
+          dataItemByteCount: dataItemSizeFactory(),
+          chunkingMode: params.chunkingMode
+        });
+        if (chunkedUploader.shouldUseChunkUploader) {
+          const response2 = await chunkedUploader.upload({
+            dataItemStreamFactory,
+            dataItemSizeFactory,
+            dataItemOpts,
+            signal,
+            events
+          });
+          return response2;
+        }
         const response = await this.uploadSignedDataItem({
           dataItemStreamFactory,
           dataItemSizeFactory,
@@ -274847,7 +275214,10 @@ ${lastError instanceof Error ? lastError.message : lastError}`;
       signal,
       manifestOptions = {},
       maxConcurrentUploads = 1,
-      throwOnFailure = true
+      throwOnFailure = true,
+      maxChunkConcurrency,
+      chunkByteCount,
+      chunkingMode
     } = params;
     const { disableManifest, indexFile, fallbackFile } = manifestOptions;
     const paths = {};
@@ -274873,7 +275243,10 @@ ${lastError instanceof Error ? lastError.message : lastError}`;
           fileStreamFactory: () => this.getFileStreamForFile(file),
           fileSizeFactory: () => this.getFileSize(file),
           signal,
-          dataItemOpts: dataItemOptsWithContentType
+          dataItemOpts: dataItemOptsWithContentType,
+          chunkByteCount,
+          maxChunkConcurrency,
+          chunkingMode
         });
         const relativePath = this.getRelativePath(file, params);
         paths[relativePath] = { id: result2.id };
@@ -274916,7 +275289,10 @@ ${lastError instanceof Error ? lastError.message : lastError}`;
       fileStreamFactory: () => this.createManifestStream(manifestBuffer),
       fileSizeFactory: () => manifestBuffer.byteLength,
       signal,
-      dataItemOpts: { ...dataItemOpts, tags: tagsWithManifestContentType }
+      dataItemOpts: { ...dataItemOpts, tags: tagsWithManifestContentType },
+      chunkByteCount,
+      maxChunkConcurrency,
+      chunkingMode
     });
     return {
       ...response,
@@ -275164,9 +275540,20 @@ var TurboAuthenticatedClient = class extends TurboUnauthenticatedClient {
     data,
     dataItemOpts,
     signal,
-    events
+    events,
+    chunkByteCount,
+    chunkingMode,
+    maxChunkConcurrency
   }) {
-    return this.uploadService.upload({ data, dataItemOpts, signal, events });
+    return this.uploadService.upload({
+      data,
+      dataItemOpts,
+      signal,
+      events,
+      chunkByteCount,
+      chunkingMode,
+      maxChunkConcurrency
+    });
   }
   uploadFile(params) {
     return this.uploadService.uploadFile(params);
@@ -275952,6 +276339,7 @@ export {
   tokenToBaseMap,
   tokenTypes,
   ukyveToTokenAmount,
+  validChunkingModes,
   weiToTokenAmount
 };
 /*! Bundled license information:

package/lib/cjs/cli/commands/uploadFile.js

@@ -32,6 +32,7 @@ async function uploadFile(options) {
         fileStreamFactory: () => (0, fs_1.createReadStream)(filePath),
         fileSizeFactory: () => fileSize,
         dataItemOpts: { tags: [...constants_js_1.turboCliTags, ...customTags], paidBy },
+        ...(0, utils_js_1.getChunkingOptions)(options),
     });
     console.log('Uploaded file:', JSON.stringify(result, null, 2));
 }

package/lib/cjs/cli/commands/uploadFolder.js

@@ -21,7 +21,7 @@ const utils_js_1 = require("../utils.js");
 async function uploadFolder(options) {
     const turbo = await (0, utils_js_1.turboFromOptions)(options);
     const paidBy = await (0, utils_js_1.paidByFromOptions)(options, turbo);
-    const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, } = (0, utils_js_1.getUploadFolderOptions)(options);
+    const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, chunkByteCount, chunkingMode, maxChunkConcurrency, } = (0, utils_js_1.getUploadFolderOptions)(options);
     const customTags = (0, utils_js_1.getTagsFromOptions)(options);
     const result = await turbo.uploadFolder({
         folderPath: folderPath,
@@ -32,6 +32,9 @@ async function uploadFolder(options) {
             fallbackFile,
         },
         maxConcurrentUploads,
+        chunkByteCount,
+        chunkingMode,
+        maxChunkConcurrency,
     });
     console.log('Uploaded folder:', JSON.stringify(result, null, 2));
 }

package/lib/cjs/cli/options.js

@@ -134,7 +134,7 @@ exports.optionMap = {
     },
     maxConcurrency: {
         alias: '--max-concurrency <maxConcurrency>',
-        description: 'Maximum number of concurrent uploads',
+        description: 'Maximum number of concurrent file uploads',
     },
     paidBy: {
         alias: '--paid-by <paidBy...>',
@@ -159,6 +159,19 @@ exports.optionMap = {
         alias: '--byte-count <byteCount>',
         description: 'Number of bytes to use for the action',
     },
+    maxChunkConcurrency: {
+        alias: '--max-chunk-concurrency <maxChunkConcurrency>',
+        description: 'Maximum number of concurrent chunks to upload per file',
+    },
+    chunkByteCount: {
+        alias: '--chunk-byte-count <chunkByteCount>',
+        description: 'Size of each chunk in bytes',
+    },
+    chunkingMode: {
+        alias: '--chunking-mode <chunkingMode>',
+        description: 'Chunking mode to use for the upload. Can be "auto", "force" or "disabled". Defaults to "auto".',
+        default: 'auto',
+    },
 };
 exports.walletOptions = [
     exports.optionMap.walletFile,
@@ -182,6 +195,9 @@ exports.uploadOptions = [
     exports.optionMap.ignoreApprovals,
     exports.optionMap.useSignerBalanceFirst,
     exports.optionMap.tags,
+    exports.optionMap.maxChunkConcurrency,
+    exports.optionMap.chunkByteCount,
+    exports.optionMap.chunkingMode,
 ];
 exports.uploadFolderOptions = [
     ...exports.uploadOptions,

package/lib/cjs/cli/types.js

@@ -1,17 +1,2 @@
 "use strict";
-/**
- * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 Object.defineProperty(exports, "__esModule", { value: true });

package/lib/cjs/cli/utils.js

@@ -20,6 +20,7 @@ exports.parseTags = parseTags;
 exports.getTagsFromOptions = getTagsFromOptions;
 exports.currencyFromOptions = currencyFromOptions;
 exports.requiredByteCountFromOptions = requiredByteCountFromOptions;
+exports.getChunkingOptions = getChunkingOptions;
 /**
  * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
  *
@@ -236,6 +237,7 @@ function getUploadFolderOptions(options) {
         fallbackFile: options.fallbackFile,
         disableManifest: !options.manifest,
         maxConcurrentUploads: +(options.maxConcurrency ?? 1),
+        ...getChunkingOptions(options),
     };
 }
 /**
@@ -283,3 +285,14 @@ function requiredByteCountFromOptions({ byteCount, }) {
     }
     return byteCountValue;
 }
+function getChunkingOptions(options) {
+    return {
+        chunkingMode: options.chunkingMode,
+        chunkByteCount: options.chunkByteCount !== undefined
+            ? +options.chunkByteCount
+            : undefined,
+        maxChunkConcurrency: options.maxChunkConcurrency !== undefined
+            ? +options.maxChunkConcurrency
+            : undefined,
+    };
+}