@ardrive/turbo-sdk 1.30.0 → 1.31.0-alpha.2

package/lib/cjs/types.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isJWK = exports.isWebUploadFolderParams = exports.isNodeUploadFolderParams = exports.tokenTypes = exports.fiatCurrencyTypes = void 0;
+exports.validChunkingModes = exports.isJWK = exports.isWebUploadFolderParams = exports.isNodeUploadFolderParams = exports.multipartFinalizedStatus = exports.multipartFailedStatus = exports.multipartPendingStatus = exports.tokenTypes = exports.fiatCurrencyTypes = void 0;
 exports.isCurrency = isCurrency;
 exports.isKyvePrivateKey = isKyvePrivateKey;
 exports.isEthPrivateKey = isEthPrivateKey;
@@ -31,6 +31,18 @@ exports.tokenTypes = [
     'pol',
     'base-eth',
 ];
+exports.multipartPendingStatus = [
+    'ASSEMBLING',
+    'VALIDATING',
+    'FINALIZING',
+];
+exports.multipartFailedStatus = [
+    'UNDERFUNDED',
+    'INVALID',
+    'APPROVAL_FAILED',
+    'REVOKE_FAILED',
+];
+exports.multipartFinalizedStatus = ['FINALIZED'];
 const isNodeUploadFolderParams = (p) => p.folderPath !== undefined;
 exports.isNodeUploadFolderParams = isNodeUploadFolderParams;
 const isWebUploadFolderParams = (p) => p.files !== undefined;
@@ -54,3 +66,4 @@ function isSolanaWalletAdapter(walletAdapter) {
 function isEthereumWalletAdapter(walletAdapter) {
     return 'getSigner' in walletAdapter;
 }
+exports.validChunkingModes = ['force', 'disabled', 'auto'];

package/lib/cjs/version.js

@@ -17,4 +17,4 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.version = void 0;
 // AUTOMATICALLY GENERATED FILE - DO NOT TOUCH
-exports.version = '1.30.0';
+exports.version = '1.31.0-alpha.2';

package/lib/esm/cli/commands/uploadFile.js

@@ -15,7 +15,7 @@
  */
 import { createReadStream, statSync } from 'fs';
 import { turboCliTags } from '../constants.js';
-import { getTagsFromOptions, paidByFromOptions, turboFromOptions, } from '../utils.js';
+import { getChunkingOptions, getTagsFromOptions, paidByFromOptions, turboFromOptions, } from '../utils.js';
 export async function uploadFile(options) {
     const { filePath } = options;
     if (filePath === undefined) {
@@ -29,6 +29,7 @@ export async function uploadFile(options) {
         fileStreamFactory: () => createReadStream(filePath),
         fileSizeFactory: () => fileSize,
         dataItemOpts: { tags: [...turboCliTags, ...customTags], paidBy },
+        ...getChunkingOptions(options),
     });
     console.log('Uploaded file:', JSON.stringify(result, null, 2));
 }

package/lib/esm/cli/commands/uploadFolder.js

@@ -18,7 +18,7 @@ import { getTagsFromOptions, getUploadFolderOptions, paidByFromOptions, turboFro
 export async function uploadFolder(options) {
     const turbo = await turboFromOptions(options);
     const paidBy = await paidByFromOptions(options, turbo);
-    const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, } = getUploadFolderOptions(options);
+    const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, chunkByteCount, chunkingMode, maxChunkConcurrency, maxFinalizeMs, } = getUploadFolderOptions(options);
     const customTags = getTagsFromOptions(options);
     const result = await turbo.uploadFolder({
         folderPath: folderPath,
@@ -29,6 +29,10 @@ export async function uploadFolder(options) {
             fallbackFile,
         },
         maxConcurrentUploads,
+        chunkByteCount,
+        chunkingMode,
+        maxChunkConcurrency,
+        maxFinalizeMs,
     });
     console.log('Uploaded folder:', JSON.stringify(result, null, 2));
 }

package/lib/esm/cli/options.js

@@ -131,7 +131,7 @@ export const optionMap = {
     },
     maxConcurrency: {
         alias: '--max-concurrency <maxConcurrency>',
-        description: 'Maximum number of concurrent uploads',
+        description: 'Maximum number of concurrent file uploads',
     },
     paidBy: {
         alias: '--paid-by <paidBy...>',
@@ -156,6 +156,23 @@ export const optionMap = {
         alias: '--byte-count <byteCount>',
         description: 'Number of bytes to use for the action',
     },
+    maxChunkConcurrency: {
+        alias: '--max-chunk-concurrency <maxChunkConcurrency>',
+        description: 'Maximum number of concurrent chunks to upload per file',
+    },
+    maxFinalizeMs: {
+        alias: '--max-finalize-ms <maxFinalizeMs>',
+        description: 'Maximum time in milliseconds to wait for the finalization of all chunks after the last chunk is uploaded. Defaults to 1 minute per GiB of the total file size.',
+    },
+    chunkByteCount: {
+        alias: '--chunk-byte-count <chunkByteCount>',
+        description: 'Size of each chunk in bytes',
+    },
+    chunkingMode: {
+        alias: '--chunking-mode <chunkingMode>',
+        description: 'Chunking mode to use for the upload. Can be "auto", "force" or "disabled". Defaults to "auto".',
+        default: 'auto',
+    },
 };
 export const walletOptions = [
     optionMap.walletFile,
@@ -179,6 +196,10 @@ export const uploadOptions = [
     optionMap.ignoreApprovals,
     optionMap.useSignerBalanceFirst,
     optionMap.tags,
+    optionMap.maxChunkConcurrency,
+    optionMap.maxFinalizeMs,
+    optionMap.chunkByteCount,
+    optionMap.chunkingMode,
 ];
 export const uploadFolderOptions = [
     ...uploadOptions,

package/lib/esm/cli/types.js

@@ -1,16 +1 @@
-/**
- * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 export {};

package/lib/esm/cli/utils.js

@@ -214,6 +214,7 @@ export function getUploadFolderOptions(options) {
         fallbackFile: options.fallbackFile,
         disableManifest: !options.manifest,
         maxConcurrentUploads: +(options.maxConcurrency ?? 1),
+        ...getChunkingOptions(options),
     };
 }
 /**
@@ -261,3 +262,15 @@ export function requiredByteCountFromOptions({ byteCount, }) {
     }
     return byteCountValue;
 }
+export function getChunkingOptions(options) {
+    return {
+        chunkingMode: options.chunkingMode,
+        chunkByteCount: options.chunkByteCount !== undefined
+            ? +options.chunkByteCount
+            : undefined,
+        maxChunkConcurrency: options.maxChunkConcurrency !== undefined
+            ? +options.maxChunkConcurrency
+            : undefined,
+        maxFinalizeMs: options.maxFinalizeMs !== undefined ? +options.maxFinalizeMs : undefined,
+    };
+}

package/lib/esm/common/chunked.js

@@ -0,0 +1,407 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { CanceledError } from 'axios';
+import { pLimit } from 'plimit-lit';
+import { validChunkingModes, } from '../types.js';
+import { sleep } from '../utils/common.js';
+import { FailedRequestError } from '../utils/errors.js';
+import { TurboEventEmitter, createStreamWithUploadEvents } from './events.js';
+import { TurboWinstonLogger } from './logger.js';
+const fiveMiB = 5 * 1024 * 1024; // 5 MiB
+const fiveHundredMiB = fiveMiB * 100; // 500 MiB
+export const defaultMaxChunkConcurrency = 5;
+export const maxChunkByteCount = fiveHundredMiB;
+export const minChunkByteCount = fiveMiB;
+export const defaultChunkByteCount = minChunkByteCount;
+const backlogQueueFactor = 2;
+const chunkingHeader = { 'x-chunking-version': '2' };
+/**
+ * Performs a chunked upload by splitting the stream into fixed-size buffers,
+ * uploading them in parallel, and emitting progress/error events.
+ */
+export class ChunkedUploader {
+    constructor({ http, token, maxChunkConcurrency = defaultMaxChunkConcurrency, maxFinalizeMs, chunkByteCount = defaultChunkByteCount, logger = TurboWinstonLogger.default, chunkingMode = 'auto', dataItemByteCount, }) {
+        this.assertChunkParams({
+            chunkByteCount,
+            chunkingMode,
+            maxChunkConcurrency,
+            maxFinalizeMs,
+        });
+        this.chunkByteCount = chunkByteCount;
+        this.maxChunkConcurrency = maxChunkConcurrency;
+        this.maxFinalizeMs = maxFinalizeMs;
+        this.http = http;
+        this.token = token;
+        this.logger = logger;
+        this.shouldUseChunkUploader = this.shouldChunkUpload({
+            chunkByteCount,
+            chunkingMode,
+            dataItemByteCount,
+        });
+        this.maxBacklogQueue = this.maxChunkConcurrency * backlogQueueFactor;
+    }
+    shouldChunkUpload({ chunkByteCount, chunkingMode, dataItemByteCount, }) {
+        if (chunkingMode === 'disabled') {
+            return false;
+        }
+        if (chunkingMode === 'force') {
+            return true;
+        }
+        const isMoreThanTwoChunksOfData = dataItemByteCount > chunkByteCount * 2;
+        return isMoreThanTwoChunksOfData;
+    }
+    assertChunkParams({ chunkByteCount, chunkingMode, maxChunkConcurrency, maxFinalizeMs, }) {
+        if (maxFinalizeMs !== undefined &&
+            (Number.isNaN(maxFinalizeMs) ||
+                !Number.isInteger(maxFinalizeMs) ||
+                maxFinalizeMs < 0)) {
+            throw new Error('Invalid max finalization wait time. Must be a non-negative integer.');
+        }
+        if (Number.isNaN(maxChunkConcurrency) ||
+            !Number.isInteger(maxChunkConcurrency) ||
+            maxChunkConcurrency < 1) {
+            throw new Error('Invalid max chunk concurrency. Must be an integer of at least 1.');
+        }
+        if (Number.isNaN(chunkByteCount) ||
+            !Number.isInteger(chunkByteCount) ||
+            chunkByteCount < fiveMiB ||
+            chunkByteCount > fiveHundredMiB) {
+            throw new Error('Invalid chunk size. Must be an integer between 5 MiB and 500 MiB.');
+        }
+        if (typeof chunkingMode !== 'string' ||
+            !validChunkingModes.includes(chunkingMode)) {
+            throw new Error(`Invalid chunking mode. Must be one of: ${validChunkingModes.join(', ')}`);
+        }
+    }
+    /**
+     * Initialize or resume an upload session, returning the upload ID.
+     */
+    async initUpload() {
+        const res = await this.http.get({
+            endpoint: `/chunks/${this.token}/-1/-1?chunkSize=${this.chunkByteCount}`,
+            headers: chunkingHeader,
+        });
+        if (res.chunkSize !== this.chunkByteCount) {
+            this.logger.warn('Chunk size mismatch! Overriding with server value.', {
+                clientExpected: this.chunkByteCount,
+                serverReturned: res.chunkSize,
+            });
+            this.chunkByteCount = res.chunkSize;
+        }
+        return res.id;
+    }
+    async upload({ dataItemSizeFactory, dataItemStreamFactory, dataItemOpts, signal, events, }) {
+        const uploadId = await this.initUpload();
+        const dataItemByteCount = dataItemSizeFactory();
+        const emitter = new TurboEventEmitter(events);
+        const { stream, resume } = createStreamWithUploadEvents({
+            data: dataItemStreamFactory(),
+            dataSize: dataItemByteCount,
+            emitter,
+        });
+        this.logger.debug(`Starting chunked upload`, {
+            token: this.token,
+            uploadId,
+            totalSize: dataItemByteCount,
+            chunkByteCount: this.chunkByteCount,
+            maxChunkConcurrency: this.maxChunkConcurrency,
+            inputStreamType: isReadableStream(stream) ? 'ReadableStream' : 'Readable',
+        });
+        const inFlight = new Set();
+        const internalAbort = new AbortController();
+        const combinedSignal = combineAbortSignals([internalAbort.signal, signal]);
+        const limit = pLimit(this.maxChunkConcurrency);
+        let currentOffset = 0;
+        let currentChunkPartNumber = 0;
+        let firstError;
+        let uploadedBytes = 0;
+        const chunks = splitIntoChunks(stream, this.chunkByteCount);
+        resume();
+        for await (const chunk of chunks) {
+            if (combinedSignal?.aborted) {
+                internalAbort.abort();
+                await Promise.allSettled(inFlight);
+                firstError ??= new CanceledError();
+                break;
+            }
+            const chunkPartNumber = ++currentChunkPartNumber;
+            const chunkByteCount = chunk.length;
+            const chunkOffset = currentOffset;
+            currentOffset += chunkByteCount;
+            const promise = limit(async () => {
+                if (firstError !== undefined) {
+                    return;
+                }
+                this.logger.debug('Uploading chunk', {
+                    chunkPartNumber,
+                    chunkOffset,
+                    chunkByteCount,
+                });
+                await this.http.post({
+                    endpoint: `/chunks/${this.token}/${uploadId}/${chunkOffset}`,
+                    data: chunk,
+                    headers: {
+                        'Content-Type': 'application/octet-stream',
+                        ...chunkingHeader,
+                    },
+                    signal: combinedSignal,
+                });
+                uploadedBytes += chunkByteCount;
+                this.logger.debug('Chunk uploaded', {
+                    chunkPartNumber,
+                    chunkOffset,
+                    chunkByteCount,
+                });
+                emitter.emit('upload-progress', {
+                    processedBytes: uploadedBytes,
+                    totalBytes: dataItemByteCount,
+                });
+            }).catch((err) => {
+                this.logger.error('Chunk upload failed', {
+                    id: chunkPartNumber,
+                    offset: chunkOffset,
+                    size: chunkByteCount,
+                    err,
+                });
+                emitter.emit('upload-error', err);
+                internalAbort.abort(err);
+                firstError = firstError ?? err;
+            });
+            inFlight.add(promise);
+            promise.finally(() => inFlight.delete(promise));
+            if (inFlight.size >= this.maxBacklogQueue) {
+                await Promise.race(inFlight);
+                if (combinedSignal?.aborted) {
+                    internalAbort.abort();
+                    await Promise.allSettled(inFlight);
+                    firstError ??= new CanceledError();
+                    break;
+                }
+            }
+        }
+        await Promise.all(inFlight);
+        if (firstError !== undefined) {
+            throw firstError;
+        }
+        const finalizeResponse = await this.finalizeUpload(uploadId, dataItemByteCount, dataItemOpts?.paidBy, combinedSignal);
+        emitter.emit('upload-success');
+        return finalizeResponse;
+    }
+    toGiB(bytes) {
+        return bytes / 1024 ** 3;
+    }
+    async finalizeUpload(uploadId, dataItemByteCount, paidBy, signal) {
+        // Wait up to 1 minute per GiB of data for the upload to finalize
+        const fileSizeInGiB = Math.ceil(this.toGiB(dataItemByteCount));
+        const defaultMaxWaitTimeMins = fileSizeInGiB;
+        const maxWaitTimeMs = this.maxFinalizeMs ?? defaultMaxWaitTimeMins * 60 * 1000;
+        const minimumWaitPerStepMs = 
+        // Per step, files smaller than 100MB will wait 2 second,
+        dataItemByteCount < 1024 * 1024 * 100
+            ? 2000
+            : // files smaller than 3 GiB will wait 3 seconds,
+                dataItemByteCount < 1024 * 1024 * 1024 * 3
+                    ? 3000
+                    : // and larger files will wait 1 second per GiB with max of 10 seconds
+                        Math.max(1000 * fileSizeInGiB, 10000);
+        const paidByHeader = {};
+        if (paidBy !== undefined) {
+            paidByHeader['x-paid-by'] = Array.isArray(paidBy)
+                ? paidBy.join(',')
+                : paidBy;
+        }
+        await this.http.post({
+            endpoint: `/chunks/${this.token}/${uploadId}/finalize`,
+            data: Buffer.alloc(0),
+            headers: {
+                'Content-Type': 'application/octet-stream',
+                ...paidByHeader,
+                ...chunkingHeader,
+            },
+            signal,
+        });
+        this.logger.debug(`Confirming upload to Turbo with uploadId ${uploadId} for up to ${defaultMaxWaitTimeMins} minutes.`);
+        const startTime = Date.now();
+        const cutoffTime = startTime + maxWaitTimeMs;
+        let attempts = 0;
+        while (Date.now() < cutoffTime) {
+            // Wait for 3/4 of the time remaining per attempt or minimum step
+            const waitTimeMs = Math.min(Math.floor((cutoffTime - Date.now()) * (3 / 4)), minimumWaitPerStepMs);
+            await sleep(waitTimeMs);
+            if (signal?.aborted) {
+                this.logger.warn(`Upload finalization aborted by signal.`);
+                throw new CanceledError();
+            }
+            const response = await this.http.get({
+                endpoint: `/chunks/${this.token}/${uploadId}/status`,
+                signal,
+            });
+            this.logger.debug(`Upload status found: ${response.status}`, {
+                status: response.status,
+                attempts: attempts++,
+                maxWaitTimeMs,
+                minimumWaitPerStepMs,
+                waitTimeMs,
+                elapsedMs: Date.now() - startTime,
+            });
+            if (response.status === 'FINALIZED') {
+                this.logger.debug(`Upload finalized successfully.`);
+                return response.receipt;
+            }
+            if (response.status === 'UNDERFUNDED') {
+                throw new FailedRequestError(`Insufficient balance`, 402);
+            }
+        }
+        throw new Error(`Upload multi-part finalization has timed out for Upload ID ${uploadId}`);
+    }
+}
+/**
+ * Yield Buffers of up to `chunkByteCount`, coalescing whatever small pieces
+ * the source produces into proper slices.
+ */
+export async function* splitIntoChunks(source, chunkByteCount) {
+    if (isReadableStream(source)) {
+        yield* splitReadableStreamIntoChunks(source, chunkByteCount);
+    }
+    else {
+        yield* splitReadableIntoChunks(source, chunkByteCount);
+    }
+}
+export async function* splitReadableIntoChunks(source, chunkByteCount) {
+    const queue = [];
+    let total = 0;
+    let encoder;
+    for await (const piece of source) {
+        const u8 = piece instanceof Uint8Array
+            ? new Uint8Array(piece.buffer, piece.byteOffset, piece.byteLength)
+            : (encoder ??= new TextEncoder()).encode(String(piece));
+        queue.push(u8);
+        total += u8.length;
+        // Emit full chunks
+        while (total >= chunkByteCount) {
+            const out = new Uint8Array(chunkByteCount);
+            let remaining = out.length;
+            let off = 0;
+            while (remaining > 0) {
+                const head = queue[0];
+                const take = Math.min(remaining, head.length);
+                out.set(head.subarray(0, take), off);
+                off += take;
+                remaining -= take;
+                if (take === head.length) {
+                    queue.shift();
+                }
+                else {
+                    queue[0] = head.subarray(take);
+                }
+            }
+            total -= chunkByteCount;
+            // Yield a Buffer view (no copy)
+            yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+        }
+    }
+    // Remainder
+    if (total > 0) {
+        const out = new Uint8Array(total);
+        let off = 0;
+        while (queue.length > 0) {
+            const head = queue.shift(); // safe due to loop condition
+            out.set(head, off);
+            off += head.length;
+        }
+        yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+    }
+}
+export async function* splitReadableStreamIntoChunks(source, chunkByteCount) {
+    const reader = source.getReader();
+    const queue = [];
+    let total = 0;
+    try {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            // Ensure we keep a plain view (avoids surprises if the producer reuses buffers)
+            const u8 = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+            queue.push(u8);
+            total += u8.length;
+            while (total >= chunkByteCount) {
+                const out = new Uint8Array(chunkByteCount);
+                let remaining = out.length;
+                let off = 0;
+                while (remaining > 0) {
+                    const head = queue[0];
+                    const take = Math.min(remaining, head.length);
+                    out.set(head.subarray(0, take), off);
+                    off += take;
+                    remaining -= take;
+                    if (take === head.length) {
+                        queue.shift();
+                    }
+                    else {
+                        queue[0] = head.subarray(take);
+                    }
+                }
+                total -= chunkByteCount;
+                yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+            }
+        }
+        if (total > 0) {
+            const out = new Uint8Array(total);
+            let off = 0;
+            while (queue.length > 0) {
+                const head = queue.shift(); // safe due to loop condition
+                out.set(head, off);
+                off += head.length;
+            }
+            yield Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+function isReadableStream(source) {
+    // Prefer instanceof if available, otherwise use a safe duck-typing check
+    if (typeof ReadableStream !== 'undefined' &&
+        source instanceof ReadableStream) {
+        return true;
+    }
+    return (source !== null &&
+        typeof source === 'object' &&
+        'getReader' in source &&
+        typeof source.getReader === 'function');
+}
+function combineAbortSignals(signals) {
+    const real = signals.filter(Boolean);
+    if (real.length === 0)
+        return undefined;
+    const anyFn = AbortSignal.any;
+    if (typeof anyFn === 'function') {
+        return anyFn(real);
+    }
+    const controller = new AbortController();
+    for (const s of real) {
+        const sig = s;
+        if (sig.aborted) {
+            controller.abort(sig.reason);
+            break;
+        }
+        const onAbort = () => controller.abort(sig.reason);
+        s.addEventListener('abort', onAbort, { once: true });
+    }
+    return controller.signal;
+}

package/lib/esm/common/http.js

@@ -42,6 +42,7 @@ export class TurboHTTPService {
         // See: https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API#body
         const { body, duplex } = await toFetchBody(data);
         try {
+            this.logger.debug('Posting data via fetch', { endpoint, headers });
             const res = await fetch(this.axios.defaults.baseURL + endpoint, {
                 method: 'POST',
                 headers,

package/lib/esm/common/turbo.js

@@ -156,8 +156,16 @@ export class TurboAuthenticatedClient extends TurboUnauthenticatedClient {
     /**
      * Signs and uploads raw data to the Turbo Upload Service.
      */
-    upload({ data, dataItemOpts, signal, events, }) {
-        return this.uploadService.upload({ data, dataItemOpts, signal, events });
+    upload({ data, dataItemOpts, signal, events, chunkByteCount, chunkingMode, maxChunkConcurrency, }) {
+        return this.uploadService.upload({
+            data,
+            dataItemOpts,
+            signal,
+            events,
+            chunkByteCount,
+            chunkingMode,
+            maxChunkConcurrency,
+        });
     }
     uploadFile(params) {
         return this.uploadService.uploadFile(params);