@ardrive/turbo-sdk 1.30.0 → 1.31.0-alpha.2

package/lib/cjs/cli/commands/uploadFolder.js

@@ -21,7 +21,7 @@ const utils_js_1 = require("../utils.js");
 async function uploadFolder(options) {
     const turbo = await (0, utils_js_1.turboFromOptions)(options);
     const paidBy = await (0, utils_js_1.paidByFromOptions)(options, turbo);
-    const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, } = (0, utils_js_1.getUploadFolderOptions)(options);
+    const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, chunkByteCount, chunkingMode, maxChunkConcurrency, maxFinalizeMs, } = (0, utils_js_1.getUploadFolderOptions)(options);
     const customTags = (0, utils_js_1.getTagsFromOptions)(options);
     const result = await turbo.uploadFolder({
         folderPath: folderPath,
@@ -32,6 +32,10 @@ async function uploadFolder(options) {
             fallbackFile,
         },
         maxConcurrentUploads,
+        chunkByteCount,
+        chunkingMode,
+        maxChunkConcurrency,
+        maxFinalizeMs,
     });
     console.log('Uploaded folder:', JSON.stringify(result, null, 2));
 }

package/lib/cjs/cli/options.js

@@ -134,7 +134,7 @@ exports.optionMap = {
     },
     maxConcurrency: {
         alias: '--max-concurrency <maxConcurrency>',
-        description: 'Maximum number of concurrent uploads',
+        description: 'Maximum number of concurrent file uploads',
     },
     paidBy: {
         alias: '--paid-by <paidBy...>',
@@ -159,6 +159,23 @@ exports.optionMap = {
         alias: '--byte-count <byteCount>',
         description: 'Number of bytes to use for the action',
     },
+    maxChunkConcurrency: {
+        alias: '--max-chunk-concurrency <maxChunkConcurrency>',
+        description: 'Maximum number of concurrent chunks to upload per file',
+    },
+    maxFinalizeMs: {
+        alias: '--max-finalize-ms <maxFinalizeMs>',
+        description: 'Maximum time in milliseconds to wait for the finalization of all chunks after the last chunk is uploaded. Defaults to 1 minute per GiB of the total file size.',
+    },
+    chunkByteCount: {
+        alias: '--chunk-byte-count <chunkByteCount>',
+        description: 'Size of each chunk in bytes',
+    },
+    chunkingMode: {
+        alias: '--chunking-mode <chunkingMode>',
+        description: 'Chunking mode to use for the upload. Can be "auto", "force" or "disabled". Defaults to "auto".',
+        default: 'auto',
+    },
 };
 exports.walletOptions = [
     exports.optionMap.walletFile,
@@ -182,6 +199,10 @@ exports.uploadOptions = [
     exports.optionMap.ignoreApprovals,
     exports.optionMap.useSignerBalanceFirst,
     exports.optionMap.tags,
+    exports.optionMap.maxChunkConcurrency,
+    exports.optionMap.maxFinalizeMs,
+    exports.optionMap.chunkByteCount,
+    exports.optionMap.chunkingMode,
 ];
 exports.uploadFolderOptions = [
     ...exports.uploadOptions,

package/lib/cjs/cli/types.js

@@ -1,17 +1,2 @@
 "use strict";
-/**
- * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 Object.defineProperty(exports, "__esModule", { value: true });

package/lib/cjs/cli/utils.js

@@ -20,6 +20,7 @@ exports.parseTags = parseTags;
 exports.getTagsFromOptions = getTagsFromOptions;
 exports.currencyFromOptions = currencyFromOptions;
 exports.requiredByteCountFromOptions = requiredByteCountFromOptions;
+exports.getChunkingOptions = getChunkingOptions;
 /**
  * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
  *
@@ -236,6 +237,7 @@ function getUploadFolderOptions(options) {
         fallbackFile: options.fallbackFile,
         disableManifest: !options.manifest,
         maxConcurrentUploads: +(options.maxConcurrency ?? 1),
+        ...getChunkingOptions(options),
     };
 }
 /**
@@ -283,3 +285,15 @@ function requiredByteCountFromOptions({ byteCount, }) {
     }
     return byteCountValue;
 }
+function getChunkingOptions(options) {
+    return {
+        chunkingMode: options.chunkingMode,
+        chunkByteCount: options.chunkByteCount !== undefined
+            ? +options.chunkByteCount
+            : undefined,
+        maxChunkConcurrency: options.maxChunkConcurrency !== undefined
+            ? +options.maxChunkConcurrency
+            : undefined,
+        maxFinalizeMs: options.maxFinalizeMs !== undefined ? +options.maxFinalizeMs : undefined,
+    };
+}

package/lib/cjs/common/chunked.js

@@ -0,0 +1,414 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ChunkedUploader = exports.defaultChunkByteCount = exports.minChunkByteCount = exports.maxChunkByteCount = exports.defaultMaxChunkConcurrency = void 0;
+exports.splitIntoChunks = splitIntoChunks;
+exports.splitReadableIntoChunks = splitReadableIntoChunks;
+exports.splitReadableStreamIntoChunks = splitReadableStreamIntoChunks;
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const axios_1 = require("axios");
+const plimit_lit_1 = require("plimit-lit");
+const types_js_1 = require("../types.js");
+const common_js_1 = require("../utils/common.js");
+const errors_js_1 = require("../utils/errors.js");
+const events_js_1 = require("./events.js");
+const logger_js_1 = require("./logger.js");
+const fiveMiB = 5 * 1024 * 1024; // 5 MiB
+const fiveHundredMiB = fiveMiB * 100; // 500 MiB
+exports.defaultMaxChunkConcurrency = 5;
+exports.maxChunkByteCount = fiveHundredMiB;
+exports.minChunkByteCount = fiveMiB;
+exports.defaultChunkByteCount = exports.minChunkByteCount;
+const backlogQueueFactor = 2;
+const chunkingHeader = { 'x-chunking-version': '2' };
+/**
+ * Performs a chunked upload by splitting the stream into fixed-size buffers,
+ * uploading them in parallel, and emitting progress/error events.
+ */
+class ChunkedUploader {
+    constructor({ http, token, maxChunkConcurrency = exports.defaultMaxChunkConcurrency, maxFinalizeMs, chunkByteCount = exports.defaultChunkByteCount, logger = logger_js_1.TurboWinstonLogger.default, chunkingMode = 'auto', dataItemByteCount, }) {
+        this.assertChunkParams({
+            chunkByteCount,
+            chunkingMode,
+            maxChunkConcurrency,
+            maxFinalizeMs,
+        });
+        this.chunkByteCount = chunkByteCount;
+        this.maxChunkConcurrency = maxChunkConcurrency;
+        this.maxFinalizeMs = maxFinalizeMs;
+        this.http = http;
+        this.token = token;
+        this.logger = logger;
+        this.shouldUseChunkUploader = this.shouldChunkUpload({
+            chunkByteCount,
+            chunkingMode,
+            dataItemByteCount,
+        });
+        this.maxBacklogQueue = this.maxChunkConcurrency * backlogQueueFactor;
+    }
+    shouldChunkUpload({ chunkByteCount, chunkingMode, dataItemByteCount, }) {
+        if (chunkingMode === 'disabled') {
+            return false;
+        }
+        if (chunkingMode === 'force') {
+            return true;
+        }
+        const isMoreThanTwoChunksOfData = dataItemByteCount > chunkByteCount * 2;
+        return isMoreThanTwoChunksOfData;
+    }
+    assertChunkParams({ chunkByteCount, chunkingMode, maxChunkConcurrency, maxFinalizeMs, }) {
+        if (maxFinalizeMs !== undefined &&
+            (Number.isNaN(maxFinalizeMs) ||
+                !Number.isInteger(maxFinalizeMs) ||
+                maxFinalizeMs < 0)) {
+            throw new Error('Invalid max finalization wait time. Must be a non-negative integer.');
+        }
+        if (Number.isNaN(maxChunkConcurrency) ||
+            !Number.isInteger(maxChunkConcurrency) ||
+            maxChunkConcurrency < 1) {
+            throw new Error('Invalid max chunk concurrency. Must be an integer of at least 1.');
+        }
+        if (Number.isNaN(chunkByteCount) ||
+            !Number.isInteger(chunkByteCount) ||
+            chunkByteCount < fiveMiB ||
+            chunkByteCount > fiveHundredMiB) {
+            throw new Error('Invalid chunk size. Must be an integer between 5 MiB and 500 MiB.');
+        }
+        if (typeof chunkingMode !== 'string' ||
+            !types_js_1.validChunkingModes.includes(chunkingMode)) {
+            throw new Error(`Invalid chunking mode. Must be one of: ${types_js_1.validChunkingModes.join(', ')}`);
+        }
+    }
+    /**
+     * Initialize or resume an upload session, returning the upload ID.
+     */
+    async initUpload() {
+        const res = await this.http.get({
+            endpoint: `/chunks/${this.token}/-1/-1?chunkSize=${this.chunkByteCount}`,
+            headers: chunkingHeader,
+        });
+        if (res.chunkSize !== this.chunkByteCount) {
+            this.logger.warn('Chunk size mismatch! Overriding with server value.', {
+                clientExpected: this.chunkByteCount,
+                serverReturned: res.chunkSize,
+            });
+            this.chunkByteCount = res.chunkSize;
+        }
+        return res.id;
+    }
+    async upload({ dataItemSizeFactory, dataItemStreamFactory, dataItemOpts, signal, events, }) {
+        const uploadId = await this.initUpload();
+        const dataItemByteCount = dataItemSizeFactory();
+        const emitter = new events_js_1.TurboEventEmitter(events);
+        const { stream, resume } = (0, events_js_1.createStreamWithUploadEvents)({
+            data: dataItemStreamFactory(),
+            dataSize: dataItemByteCount,
+            emitter,
+        });
+        this.logger.debug(`Starting chunked upload`, {
+            token: this.token,
+            uploadId,
+            totalSize: dataItemByteCount,
+            chunkByteCount: this.chunkByteCount,
+            maxChunkConcurrency: this.maxChunkConcurrency,
+            inputStreamType: isReadableStream(stream) ? 'ReadableStream' : 'Readable',
+        });
+        const inFlight = new Set();
+        const internalAbort = new AbortController();
+        const combinedSignal = combineAbortSignals([internalAbort.signal, signal]);
+        const limit = (0, plimit_lit_1.pLimit)(this.maxChunkConcurrency);
+        let currentOffset = 0;
+        let currentChunkPartNumber = 0;
+        let firstError;
+        let uploadedBytes = 0;
+        const chunks = splitIntoChunks(stream, this.chunkByteCount);
+        resume();
+        for await (const chunk of chunks) {
+            if (combinedSignal?.aborted) {
+                internalAbort.abort();
+                await Promise.allSettled(inFlight);
+                firstError ??= new axios_1.CanceledError();
+                break;
+            }
+            const chunkPartNumber = ++currentChunkPartNumber;
+            const chunkByteCount = chunk.length;
+            const chunkOffset = currentOffset;
+            currentOffset += chunkByteCount;
+            const promise = limit(async () => {
+                if (firstError !== undefined) {
+                    return;
+                }
+                this.logger.debug('Uploading chunk', {
+                    chunkPartNumber,
+                    chunkOffset,
+                    chunkByteCount,
+                });
+                await this.http.post({
+                    endpoint: `/chunks/${this.token}/${uploadId}/${chunkOffset}`,
+                    data: chunk,
+                    headers: {
+                        'Content-Type': 'application/octet-stream',
+                        ...chunkingHeader,
+                    },
+                    signal: combinedSignal,
+                });
+                uploadedBytes += chunkByteCount;
+                this.logger.debug('Chunk uploaded', {
+                    chunkPartNumber,
+                    chunkOffset,
+                    chunkByteCount,
+                });
+                emitter.emit('upload-progress', {
+                    processedBytes: uploadedBytes,
+                    totalBytes: dataItemByteCount,
+                });
+            }).catch((err) => {
+                this.logger.error('Chunk upload failed', {
+                    id: chunkPartNumber,
+                    offset: chunkOffset,
+                    size: chunkByteCount,
+                    err,
+                });
+                emitter.emit('upload-error', err);
+                internalAbort.abort(err);
+                firstError = firstError ?? err;
+            });
+            inFlight.add(promise);
+            promise.finally(() => inFlight.delete(promise));
+            if (inFlight.size >= this.maxBacklogQueue) {
+                await Promise.race(inFlight);
+                if (combinedSignal?.aborted) {
+                    internalAbort.abort();
+                    await Promise.allSettled(inFlight);
+                    firstError ??= new axios_1.CanceledError();
+                    break;
+                }
+            }
+        }
+        await Promise.all(inFlight);
+        if (firstError !== undefined) {
+            throw firstError;
+        }
+        const finalizeResponse = await this.finalizeUpload(uploadId, dataItemByteCount, dataItemOpts?.paidBy, combinedSignal);
+        emitter.emit('upload-success');
+        return finalizeResponse;
+    }
+    toGiB(bytes) {
+        return bytes / 1024 ** 3;
+    }
+    async finalizeUpload(uploadId, dataItemByteCount, paidBy, signal) {
+        // Wait up to 1 minute per GiB of data for the upload to finalize
+        const fileSizeInGiB = Math.ceil(this.toGiB(dataItemByteCount));
+        const defaultMaxWaitTimeMins = fileSizeInGiB;
+        const maxWaitTimeMs = this.maxFinalizeMs ?? defaultMaxWaitTimeMins * 60 * 1000;
+        const minimumWaitPerStepMs = 
+        // Per step, files smaller than 100MB will wait 2 second,
+        dataItemByteCount < 1024 * 1024 * 100
+            ? 2000
+            : // files smaller than 3 GiB will wait 3 seconds,
+                dataItemByteCount < 1024 * 1024 * 1024 * 3
+                    ? 3000
+                    : // and larger files will wait 1 second per GiB with max of 10 seconds
+                        Math.max(1000 * fileSizeInGiB, 10000);
+        const paidByHeader = {};
+        if (paidBy !== undefined) {
+            paidByHeader['x-paid-by'] = Array.isArray(paidBy)
+                ? paidBy.join(',')
+                : paidBy;
+        }
+        await this.http.post({
+            endpoint: `/chunks/${this.token}/${uploadId}/finalize`,
+            data: Buffer.alloc(0),
+            headers: {
+                'Content-Type': 'application/octet-stream',
+                ...paidByHeader,
+                ...chunkingHeader,
+            },
+            signal,
+        });
+        this.logger.debug(`Confirming upload to Turbo with uploadId ${uploadId} for up to ${defaultMaxWaitTimeMins} minutes.`);
+        const startTime = Date.now();
+        const cutoffTime = startTime + maxWaitTimeMs;
+        let attempts = 0;
+        while (Date.now() < cutoffTime) {
+            // Wait for 3/4 of the time remaining per attempt or minimum step
+            const waitTimeMs = Math.min(Math.floor((cutoffTime - Date.now()) * (3 / 4)), minimumWaitPerStepMs);
+            await (0, common_js_1.sleep)(waitTimeMs);
+            if (signal?.aborted) {
+                this.logger.warn(`Upload finalization aborted by signal.`);
+                throw new axios_1.CanceledError();
+            }
+            const response = await this.http.get({
+                endpoint: `/chunks/${this.token}/${uploadId}/status`,
+                signal,
+            });
+            this.logger.debug(`Upload status found: ${response.status}`, {
+                status: response.status,
+                attempts: attempts++,
+                maxWaitTimeMs,
+                minimumWaitPerStepMs,
+                waitTimeMs,
+                elapsedMs: Date.now() - startTime,
+            });
+            if (response.status === 'FINALIZED') {
+                this.logger.debug(`Upload finalized successfully.`);
+                return response.receipt;
+            }
+            if (response.status === 'UNDERFUNDED') {
+                throw new errors_js_1.FailedRequestError(`Insufficient balance`, 402);
+            }
+        }
+        throw new Error(`Upload multi-part finalization has timed out for Upload ID ${uploadId}`);
+    }
+}
+exports.ChunkedUploader = ChunkedUploader;
+/**
+ * Yield Buffers of up to `chunkByteCount`, coalescing whatever small pieces
+ * the source produces into proper slices.
+ */
+async function* splitIntoChunks(source, chunkByteCount) {
+    if (isReadableStream(source)) {
+        yield* splitReadableStreamIntoChunks(source, chunkByteCount);
+    }
+    else {
+        yield* splitReadableIntoChunks(source, chunkByteCount);
+    }
+}
+async function* splitReadableIntoChunks(source, chunkByteCount) {
+    const queue = [];
+    let total = 0;
+    let encoder;
+    for await (const piece of source) {
+        const u8 = piece instanceof Uint8Array
+            ? new Uint8Array(piece.buffer, piece.byteOffset, piece.byteLength)
+            : (encoder ??= new TextEncoder()).encode(String(piece));
+        queue.push(u8);
+        total += u8.length;
+        // Emit full chunks
+        while (total >= chunkByteCount) {
+            const out = new Uint8Array(chunkByteCount);
+            let remaining = out.length;
+            let off = 0;
+            while (remaining > 0) {
+                const head = queue[0];
+                const take = Math.min(remaining, head.length);
+                out.set(head.subarray(0, take), off);
+                off += take;
+                remaining -= take;
+                if (take === head.length) {
+                    queue.shift();
+                }
+                else {
+                    queue[0] = head.subarray(take);
+                }
+            }
+            total -= chunkByteCount;
+            // Yield a Buffer view (no copy)
+            yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+        }
+    }
+    // Remainder
+    if (total > 0) {
+        const out = new Uint8Array(total);
+        let off = 0;
+        while (queue.length > 0) {
+            const head = queue.shift(); // safe due to loop condition
+            out.set(head, off);
+            off += head.length;
+        }
+        yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+    }
+}
+async function* splitReadableStreamIntoChunks(source, chunkByteCount) {
+    const reader = source.getReader();
+    const queue = [];
+    let total = 0;
+    try {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            // Ensure we keep a plain view (avoids surprises if the producer reuses buffers)
+            const u8 = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+            queue.push(u8);
+            total += u8.length;
+            while (total >= chunkByteCount) {
+                const out = new Uint8Array(chunkByteCount);
+                let remaining = out.length;
+                let off = 0;
+                while (remaining > 0) {
+                    const head = queue[0];
+                    const take = Math.min(remaining, head.length);
+                    out.set(head.subarray(0, take), off);
+                    off += take;
+                    remaining -= take;
+                    if (take === head.length) {
+                        queue.shift();
+                    }
+                    else {
+                        queue[0] = head.subarray(take);
+                    }
+                }
+                total -= chunkByteCount;
+                yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+            }
+        }
+        if (total > 0) {
+            const out = new Uint8Array(total);
+            let off = 0;
+            while (queue.length > 0) {
+                const head = queue.shift(); // safe due to loop condition
+                out.set(head, off);
+                off += head.length;
+            }
+            yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+function isReadableStream(source) {
+    // Prefer instanceof if available, otherwise use a safe duck-typing check
+    if (typeof ReadableStream !== 'undefined' &&
+        source instanceof ReadableStream) {
+        return true;
+    }
+    return (source !== null &&
+        typeof source === 'object' &&
+        'getReader' in source &&
+        typeof source.getReader === 'function');
+}
+function combineAbortSignals(signals) {
+    const real = signals.filter(Boolean);
+    if (real.length === 0)
+        return undefined;
+    const anyFn = AbortSignal.any;
+    if (typeof anyFn === 'function') {
+        return anyFn(real);
+    }
+    const controller = new AbortController();
+    for (const s of real) {
+        const sig = s;
+        if (sig.aborted) {
+            controller.abort(sig.reason);
+            break;
+        }
+        const onAbort = () => controller.abort(sig.reason);
+        s.addEventListener('abort', onAbort, { once: true });
+    }
+    return controller.signal;
+}

package/lib/cjs/common/http.js

@@ -45,6 +45,7 @@ class TurboHTTPService {
         // See: https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API#body
         const { body, duplex } = await toFetchBody(data);
         try {
+            this.logger.debug('Posting data via fetch', { endpoint, headers });
             const res = await fetch(this.axios.defaults.baseURL + endpoint, {
                 method: 'POST',
                 headers,

package/lib/cjs/common/turbo.js

@@ -160,8 +160,16 @@ class TurboAuthenticatedClient extends TurboUnauthenticatedClient {
     /**
      * Signs and uploads raw data to the Turbo Upload Service.
      */
-    upload({ data, dataItemOpts, signal, events, }) {
-        return this.uploadService.upload({ data, dataItemOpts, signal, events });
+    upload({ data, dataItemOpts, signal, events, chunkByteCount, chunkingMode, maxChunkConcurrency, }) {
+        return this.uploadService.upload({
+            data,
+            dataItemOpts,
+            signal,
+            events,
+            chunkByteCount,
+            chunkingMode,
+            maxChunkConcurrency,
+        });
     }
     uploadFile(params) {
         return this.uploadService.uploadFile(params);

package/lib/cjs/common/upload.js

@@ -21,6 +21,7 @@ const plimit_lit_1 = require("plimit-lit");
 const axiosClient_js_1 = require("../utils/axiosClient.js");
 const common_js_1 = require("../utils/common.js");
 const errors_js_1 = require("../utils/errors.js");
+const chunked_js_1 = require("./chunked.js");
 const events_js_1 = require("./events.js");
 const http_js_1 = require("./http.js");
 const logger_js_1 = require("./logger.js");
@@ -95,7 +96,7 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
     /**
      * Signs and uploads raw data to the Turbo Upload Service.
      */
-    upload({ data, dataItemOpts, signal, events, }) {
+    upload({ data, dataItemOpts, signal, events, chunkByteCount, chunkingMode, maxChunkConcurrency, }) {
         // This function is intended to be usable in both Node and browser environments.
         if ((0, common_js_1.isBlob)(data)) {
             const streamFactory = () => data.stream();
@@ -123,6 +124,9 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
             signal,
             dataItemOpts,
             events,
+            chunkByteCount,
+            chunkingMode,
+            maxChunkConcurrency,
         });
     }
     resolveUploadFileConfig(params) {
@@ -163,22 +167,43 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
         let lastStatusCode = undefined; // Store the last status code for throwing
         const emitter = new events_js_1.TurboEventEmitter(events);
         // avoid duplicating signing on failures here - these errors will immediately be thrown
-        // TODO: create a SigningError class and throw that instead of the generic Error
-        const { dataItemStreamFactory, dataItemSizeFactory } = await this.signer.signDataItem({
-            fileStreamFactory,
-            fileSizeFactory,
-            dataItemOpts,
-            emitter,
-        });
         // TODO: move the retry implementation to the http class, and avoid awaiting here. This will standardize the retry logic across all upload methods.
         while (retries < maxRetries) {
             if (signal?.aborted) {
                 throw new axios_1.CanceledError();
             }
+            // TODO: create a SigningError class and throw that instead of the generic Error
+            const { dataItemStreamFactory, dataItemSizeFactory } = await this.signer.signDataItem({
+                fileStreamFactory,
+                fileSizeFactory,
+                dataItemOpts,
+                emitter,
+            });
             // Now that we have the signed data item, we can upload it using the uploadSignedDataItem method
             // which will create a new emitter with upload events. We await
             // this result due to the wrapped retry logic of this method.
             try {
+                const { chunkByteCount, maxChunkConcurrency } = params;
+                const chunkedUploader = new chunked_js_1.ChunkedUploader({
+                    http: this.httpService,
+                    token: this.token,
+                    maxChunkConcurrency,
+                    chunkByteCount,
+                    logger: this.logger,
+                    dataItemByteCount: dataItemSizeFactory(),
+                    chunkingMode: params.chunkingMode,
+                    maxFinalizeMs: params.maxFinalizeMs,
+                });
+                if (chunkedUploader.shouldUseChunkUploader) {
+                    const response = await chunkedUploader.upload({
+                        dataItemStreamFactory,
+                        dataItemSizeFactory,
+                        dataItemOpts,
+                        signal,
+                        events,
+                    });
+                    return response;
+                }
                 const response = await this.uploadSignedDataItem({
                     dataItemStreamFactory,
                     dataItemSizeFactory,
@@ -216,7 +241,7 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
                 ]);
             }
         }
-        const msg = `Failed to upload file after ${maxRetries + 1} attempts\n${lastError instanceof Error ? lastError.message : lastError}`;
+        const msg = `Failed to upload file after ${retries + 1} attempts\n${lastError instanceof Error ? lastError.message : lastError}`;
         // After all retries, throw the last error for catching
         if (lastError instanceof errors_js_1.FailedRequestError) {
             lastError.message = msg;
@@ -267,7 +292,7 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
      */
     async uploadFolder(params) {
         this.logger.debug('Uploading folder...', { params });
-        const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, } = params;
+        const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, maxFinalizeMs, } = params;
         const { disableManifest, indexFile, fallbackFile } = manifestOptions;
         const paths = {};
         const response = {
@@ -291,6 +316,9 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
                     fileSizeFactory: () => this.getFileSize(file),
                     signal,
                     dataItemOpts: dataItemOptsWithContentType,
+                    chunkByteCount,
+                    maxChunkConcurrency,
+                    chunkingMode,
                 });
                 const relativePath = this.getRelativePath(file, params);
                 paths[relativePath] = { id: result.id };
@@ -336,6 +364,10 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
             fileSizeFactory: () => manifestBuffer.byteLength,
             signal,
             dataItemOpts: { ...dataItemOpts, tags: tagsWithManifestContentType },
+            chunkByteCount,
+            maxChunkConcurrency,
+            maxFinalizeMs,
+            chunkingMode,
         });
         return {
             ...response,