@ardrive/turbo-sdk 1.42.0 → 1.43.0-alpha.2

package/lib/esm/common/upload.js

@@ -16,13 +16,14 @@
 import { BigNumber } from 'bignumber.js';
 import { pLimit } from 'plimit-lit';
 import { ExistingBalanceFunding, OnDemandFunding, X402Funding, } from '../types.js';
-import { isBlob, sleep } from '../utils/common.js';
+import { isBlob, isValidArweaveBase64URL, sleep } from '../utils/common.js';
 import { AbortError } from '../utils/errors.js';
 import { FailedRequestError } from '../utils/errors.js';
+import { contentHashFromFolderIndexKey, contentHashTagName, folderIndexKey, } from '../utils/folderIndex.js';
 import { ChunkedUploader } from './chunked.js';
 import { TurboEventEmitter, createStreamWithUploadEvents } from './events.js';
 import { defaultRetryConfig } from './http.js';
-import { TurboHTTPService } from './http.js';
+import { TurboHTTPService, x402UploadEndpoints } from './http.js';
 import { exponentMap, tokenToBaseMap } from './index.js';
 import { Logger } from './logger.js';
 import { makeX402Signer } from './signer.js';
@@ -120,7 +121,9 @@ export class TurboUnauthenticatedUploadService {
             };
         return this.httpService.post({
             data: dataBuffer,
-            endpoint: '/x402/data-item/unsigned',
+            // Only reached when no signer was supplied; the x402 path recomputes this
+            // from `unsignedData`. Both must name the same route.
+            endpoint: x402UploadEndpoints.unsigned,
             signal,
             headers: tags !== undefined
                 ? { 'x-data-item-tags': JSON.stringify(tags) }
@@ -138,7 +141,6 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             'solana',
             'base-eth',
             'base-usdc',
-            'base-ario',
         ];
         this.signer = signer;
         this.paymentService = paymentService;
@@ -220,9 +222,6 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             !this.x402EnabledTokens.includes(this.token)) {
             throw new Error('x402 uploads are not supported for token: ' + this.token);
         }
-        if (params.chunkingMode === 'force' && fundingMode instanceof X402Funding) {
-            throw new Error("Chunking mode 'force' is not supported when x402 is enabled");
-        }
         this.logger.debug('Starting file upload', { params });
         let retries = 0;
         const maxRetries = this.retryConfig.retries ?? 3;
@@ -258,6 +257,19 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             // this result due to the wrapped retry logic of this method.
             try {
                 const { chunkByteCount, maxChunkConcurrency } = params;
+                // Built here rather than after the chunked branch so BOTH paths can
+                // pay with it. Chunked x402 uploads used to be impossible — the
+                // bundler had no way to charge for a multipart upload — so this branch
+                // deliberately fell back to a single request, which capped x402 at the
+                // single-item limit no matter how the client chunked. The bundler now
+                // settles at create, so the fallback is no longer needed.
+                const x402Options = fundingMode instanceof X402Funding
+                    ? {
+                        signer: fundingMode.signer ??
+                            (await makeX402Signer(this.signer.signer)),
+                        maxMUSDCAmount: fundingMode.maxMUSDCAmount,
+                    }
+                    : undefined;
                 const chunkedUploader = new ChunkedUploader({
                     http: this.httpService,
                     token: this.token,
@@ -267,9 +279,17 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                     dataItemByteCount: dataItemSizeFactory(),
                     chunkingMode: params.chunkingMode,
                     maxFinalizeMs: params.maxFinalizeMs,
+                    ...(x402Options ? { x402: x402Options } : {}),
+                    ...(x402Options
+                        ? {
+                            x402RefundIdentity: {
+                                address: await this.signer.getNativeAddress(),
+                                signatureType: this.signer.signer.signatureType,
+                            },
+                        }
+                        : {}),
                 });
-                if (chunkedUploader.shouldUseChunkUploader &&
-                    !(fundingMode instanceof X402Funding)) {
+                if (chunkedUploader.shouldUseChunkUploader) {
                     const response = await chunkedUploader.upload({
                         dataItemStreamFactory,
                         dataItemSizeFactory,
@@ -279,13 +299,6 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                     });
                     return { ...response, cryptoFundResult };
                 }
-                const x402Options = fundingMode instanceof X402Funding
-                    ? {
-                        signer: fundingMode.signer ??
-                            (await makeX402Signer(this.signer.signer)),
-                        maxMUSDCAmount: fundingMode.maxMUSDCAmount,
-                    }
-                    : undefined;
                 const response = await this.uploadSignedDataItem({
                     dataItemStreamFactory,
                     dataItemSizeFactory,
@@ -354,6 +367,214 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
         };
         return manifest;
     }
+    /**
+     * The sha-256 of a file's bytes, as lowercase hex. The key of a
+     * {@link TurboFolderUploadIndex}.
+     *
+     * This default consumes the file's stream and digests it with the platform
+     * WebCrypto implementation, which serves the browser. NodeJS overrides it
+     * with a streaming digest so a large file is never held in memory.
+     */
+    async computeContentHash(file) {
+        const stream = this.getFileStreamForFile(file);
+        const chunks = [];
+        if ('getReader' in stream) {
+            const reader = stream.getReader();
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                chunks.push(value);
+            }
+        }
+        else {
+            for await (const chunk of stream) {
+                chunks.push(new Uint8Array(chunk));
+            }
+        }
+        const digest = await crypto.subtle.digest('SHA-256', Buffer.concat(chunks));
+        return Buffer.from(digest).toString('hex');
+    }
+    /**
+     * The exact tag set a folder upload writes for one file. Shared by the
+     * planner and the uploader, so that the tags a folder index key is computed
+     * from are, without question, the tags the data item ends up carrying.
+     */
+    folderFileTags({ file, dataItemOpts, contentHash, hashTagName = contentHashTagName, }) {
+        return [
+            ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type' &&
+                // Only stripped when this upload is going to write its own. Without a
+                // folder index, a caller's File-SHA256 tag is theirs to keep.
+                (contentHash === undefined || tag.name !== hashTagName)) ?? []),
+            {
+                name: 'Content-Type',
+                value: this.getContentType(file, dataItemOpts),
+            },
+            ...(contentHash !== undefined
+                ? [{ name: hashTagName, value: contentHash }]
+                : []),
+        ];
+    }
+    /**
+     * A folder index is a cache, so a layer that is unreachable must mean a miss
+     * and not an aborted deploy. The write side is forgiving for the same reason.
+     */
+    async readFolderIndex(folderIndex, keys, signal) {
+        const knownIds = new Map();
+        const unresolved = [];
+        for (const key of keys) {
+            let id;
+            try {
+                id = await folderIndex.get(key);
+            }
+            catch (error) {
+                this.logger.error('Failed to read from folder index', error);
+            }
+            if (id !== undefined && isValidArweaveBase64URL(id)) {
+                knownIds.set(key, id);
+            }
+            else {
+                unresolved.push(key);
+            }
+        }
+        if (unresolved.length > 0 && folderIndex.resolve !== undefined) {
+            try {
+                const resolved = await folderIndex.resolve(unresolved, { signal });
+                for (const [key, id] of Object.entries(resolved)) {
+                    if (isValidArweaveBase64URL(id)) {
+                        knownIds.set(key, id);
+                    }
+                }
+            }
+            catch (error) {
+                this.logger.error('Failed to resolve from folder index', error);
+            }
+        }
+        return knownIds;
+    }
+    /**
+     * Hashes every file and derives its folder index key, then works out which of
+     * those keys already have a data item id -- from the index, or from an
+     * identical file earlier in this same folder.
+     */
+    async planFolderIndex({ files, folderIndex, dataItemOpts, limit, signal, }) {
+        const contentHashes = new Map();
+        const itemKeys = new Map();
+        await Promise.all(files.map((file) => limit(async () => {
+            let contentHash;
+            try {
+                contentHash = await this.computeContentHash(file);
+            }
+            catch (error) {
+                // A file that cannot be read is left un-keyed and falls through to
+                // the per-file upload path, which reports it through `errors` under
+                // `throwOnFailure`. Rejecting here instead would abort the whole
+                // folder before any upload starts, so passing a folder index would
+                // silently change the failure contract.
+                this.logger.error('Failed to hash file for the folder index', {
+                    file: this.getFileName(file),
+                    error,
+                });
+                return;
+            }
+            contentHashes.set(file, contentHash);
+            itemKeys.set(file, await folderIndexKey({
+                contentHash,
+                tags: this.folderFileTags({
+                    file,
+                    dataItemOpts,
+                    contentHash,
+                    hashTagName: folderIndex.hashTagName,
+                }),
+            }));
+        })));
+        const knownIds = await this.readFolderIndex(folderIndex, [...new Set(itemKeys.values())], signal);
+        // Two files that would produce the same data item share a single upload, so
+        // the plan deduplicates within the folder as well as against the index.
+        const claimed = new Set();
+        const filesToUpload = files.filter((file) => {
+            const key = itemKeys.get(file);
+            // An un-keyed file could not be hashed. It is never a reuse candidate and
+            // must not share the dedup slot with the next un-keyed file, so it skips
+            // the claimed set entirely and always uploads.
+            if (key === undefined) {
+                return true;
+            }
+            if (knownIds.has(key) || claimed.has(key)) {
+                return false;
+            }
+            claimed.add(key);
+            return true;
+        });
+        this.logger.debug('Planned folder index', {
+            folderIndex: folderIndex.name,
+            totalFiles: files.length,
+            filesToUpload: filesToUpload.length,
+        });
+        await this.warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds });
+        return { contentHashes, itemKeys, knownIds, filesToUpload };
+    }
+    /**
+     * A key covers the tags on a file as well as its bytes, so a tag in
+     * `dataItemOpts` that changes between deploys re-uploads the whole folder at
+     * full price. That is the right answer -- a reused data item is never one
+     * this call would not have made -- but on its own it is a silent cost cliff:
+     * a successful deploy, a full bill, and nothing saying why.
+     *
+     * The signature of that mistake is exact, and it is already in hand. A key is
+     * `<bytes>.<tags>`, so a file whose *bytes half* the index knows under some
+     * other tags half is a file whose content is already paid for and whose tags
+     * moved. Nothing else produces that: a folder the index has never seen has
+     * unknown bytes, and a layer that could not be reached reports nothing known.
+     * It also catches one file in a hundred, not just all of them.
+     */
+    async warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds, }) {
+        if (folderIndex.knownContentHashes === undefined) {
+            return;
+        }
+        // Counted in files, not in keys: the message is about a bill, and the bill
+        // is per file. Two files that share a key are one upload but the reader is
+        // looking for their own file count.
+        const filesPerKey = new Map();
+        for (const key of itemKeys.values()) {
+            filesPerKey.set(key, (filesPerKey.get(key) ?? 0) + 1);
+        }
+        const missed = [...filesPerKey.keys()].filter((key) => !knownIds.has(key));
+        if (missed.length === 0) {
+            return;
+        }
+        const filesFor = (keys) => keys.reduce((count, key) => count + (filesPerKey.get(key) ?? 0), 0);
+        const missedFiles = filesFor(missed);
+        const missedByContentHash = new Map();
+        for (const key of missed) {
+            const contentHash = contentHashFromFolderIndexKey(key);
+            missedByContentHash.set(contentHash, [
+                ...(missedByContentHash.get(contentHash) ?? []),
+                key,
+            ]);
+        }
+        let staleTagged = [];
+        try {
+            staleTagged =
+                (await folderIndex.knownContentHashes([
+                    ...missedByContentHash.keys(),
+                ])) ?? [];
+        }
+        catch (error) {
+            this.logger.error('Failed to read from folder index', error);
+            return;
+        }
+        const affected = staleTagged.reduce((count, contentHash) => count + filesFor(missedByContentHash.get(contentHash) ?? []), 0);
+        if (affected === 0) {
+            return;
+        }
+        this.logger.warn(`${affected} of the ${missedFiles} file(s) this run is about to upload are already on Arweave byte for byte, under a different set of tags. ` +
+            'Their content has not changed but their tags have, so they are being paid for again. A folder index key covers the tags ' +
+            'on a file as well as its bytes. That is usually a tag in dataItemOpts whose value changes between deploys -- a commit ' +
+            'sha, a build number, a timestamp -- in which case move it to manifestDataItemOpts rather than paying for these files ' +
+            'again. It can also be a file that kept its content but changed its Content-Type, through a rename or a new extension, ' +
+            'which is expected and costs one upload.');
+    }
     getContentType(file, dataItemOpts) {
         const userDefinedContentType = dataItemOpts?.tags?.find((tag) => tag.name === 'Content-Type')?.value;
         if (userDefinedContentType !== undefined) {
@@ -363,7 +584,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
     }
     async uploadFolder(params) {
         this.logger.debug('Uploading folder...', { params });
-        const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
+        const { dataItemOpts, manifestDataItemOpts = dataItemOpts, folderIndex, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
         const { disableManifest, indexFile, fallbackFile } = manifestOptions;
         // Create event emitter from events parameter
         const emitter = new TurboEventEmitter(events);
@@ -372,15 +593,37 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             fileResponses: [],
         };
         const errors = [];
+        const limit = pLimit(maxConcurrentUploads);
         // Get files and calculate total bytes upfront for progress tracking
         const files = await this.getFiles(params);
-        const totalFiles = files.length;
+        // With an index, only the files whose bytes are not already on Arweave are
+        // signed, uploaded and paid for. Progress totals cover that subset, since
+        // it is all this call actually does.
+        let contentHashes = new Map();
+        let itemKeys = new Map();
+        let knownIds = new Map();
+        let filesToUpload = files;
+        if (folderIndex !== undefined) {
+            ({ contentHashes, itemKeys, knownIds, filesToUpload } =
+                await this.planFolderIndex({
+                    files,
+                    folderIndex,
+                    dataItemOpts,
+                    limit,
+                    signal,
+                }));
+        }
+        const totalFiles = filesToUpload.length;
         let totalBytes = 0;
+        let folderBytes = 0;
         const fileSizes = new Map();
         files.forEach((file) => {
             const size = this.getFileSize(file);
             fileSizes.set(file, size);
-            totalBytes += size;
+            folderBytes += size;
+        });
+        filesToUpload.forEach((file) => {
+            totalBytes += fileSizes.get(file) ?? 0;
         });
         // Track progress across all files
         let processedFiles = 0;
@@ -395,13 +638,15 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                 fileIndex,
                 totalFiles,
             });
-            const contentType = this.getContentType(file, dataItemOpts);
+            const contentHash = contentHashes.get(file);
             const dataItemOptsWithContentType = {
                 ...dataItemOpts,
-                tags: [
-                    ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ?? []),
-                    { name: 'Content-Type', value: contentType },
-                ],
+                tags: this.folderFileTags({
+                    file,
+                    dataItemOpts,
+                    contentHash,
+                    hashTagName: folderIndex?.hashTagName,
+                }),
             };
             try {
                 const result = await this.uploadFile({
@@ -446,8 +691,24 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                     },
                     fundingMode,
                 });
-                const relativePath = this.getRelativePath(file, params);
-                paths[relativePath] = { id: result.id };
+                const itemKey = itemKeys.get(file);
+                if (itemKey !== undefined) {
+                    // Recorded before anything else can fail, so a run that is killed
+                    // part way through never loses a file it has already paid for.
+                    knownIds.set(itemKey, result.id);
+                    try {
+                        await folderIndex?.set(itemKey, result.id);
+                    }
+                    catch (error) {
+                        // A failed index write costs the next run a re-upload; it must not
+                        // fail this one, which has already paid for and landed the bytes.
+                        this.logger.error('Failed to write to folder index', error);
+                    }
+                }
+                else {
+                    const relativePath = this.getRelativePath(file, params);
+                    paths[relativePath] = { id: result.id };
+                }
                 response.fileResponses.push(result);
                 // Update processed counts after file completes
                 processedFiles++;
@@ -483,10 +744,9 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                 errors.push(error);
             }
         };
-        const limit = pLimit(maxConcurrentUploads);
         let cryptoFundResult;
         if (fundingMode instanceof OnDemandFunding) {
-            const totalByteCount = files.reduce((acc, file) => {
+            const totalByteCount = filesToUpload.reduce((acc, file) => {
                 return acc + this.getFileSize(file) + 1200; // allow extra per file for ANS-104 headers
             }, 0);
             cryptoFundResult = await this.onDemand({
@@ -494,15 +754,36 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                 onDemandFunding: fundingMode,
             });
         }
-        await Promise.all(files.map((file, index) => limit(() => uploadFile(file, index))));
+        await Promise.all(filesToUpload.map((file, index) => limit(() => uploadFile(file, index))));
         this.logger.debug('Finished uploading files', {
-            numFiles: files.length,
+            numFiles: filesToUpload.length,
             numErrors: errors.length,
             results: response.fileResponses,
         });
         if (errors.length > 0) {
             response.errors = errors;
         }
+        if (folderIndex !== undefined) {
+            // Built in folder order rather than in completion order, so an unchanged
+            // folder produces byte identical manifest bytes -- and therefore an
+            // identical manifest data item id -- run after run.
+            for (const file of files) {
+                const id = knownIds.get(itemKeys.get(file));
+                if (id !== undefined) {
+                    paths[this.getRelativePath(file, params)] = { id };
+                }
+            }
+            response.folderIndexSummary = {
+                totalFiles: files.length,
+                totalBytes: folderBytes,
+                // What landed, not what was planned. With throwOnFailure: false a file
+                // can be planned, paid for nothing, and never arrive.
+                uploadedFiles: processedFiles,
+                uploadedBytes: processedBytes,
+                reusedFiles: files.length - filesToUpload.length,
+                reusedBytes: folderBytes - totalBytes,
+            };
+        }
         if (disableManifest) {
             emitter.emit('folder-success');
             return response;
@@ -521,8 +802,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             fallbackFile,
         });
         const tagsWithManifestContentType = [
-            ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ??
-                []),
+            ...(manifestDataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ?? []),
             { name: 'Content-Type', value: 'application/x.arweave-manifest+json' },
         ];
         const manifestBuffer = Buffer.from(JSON.stringify(manifest));
@@ -532,7 +812,10 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             fileStreamFactory: () => this.createManifestStream(manifestBuffer),
             fileSizeFactory: () => manifestBuffer.byteLength,
             signal,
-            dataItemOpts: { ...dataItemOpts, tags: tagsWithManifestContentType },
+            dataItemOpts: {
+                ...manifestDataItemOpts,
+                tags: tagsWithManifestContentType,
+            },
             chunkByteCount,
             maxChunkConcurrency,
             maxFinalizeMs,

package/lib/esm/node/folderIndex.js

@@ -0,0 +1,129 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync, } from 'node:fs';
+import { dirname, resolve as resolvePath } from 'node:path';
+import { isValidArweaveBase64URL } from '../utils/common.js';
+import { contentHashFromFolderIndexKey, isValidFolderIndexKey, } from '../utils/folderIndex.js';
+const fileFolderIndexVersion = 2;
+/**
+ * A local record of folder index key to data item id, for uploads on a machine
+ * that keeps its working directory between deploys.
+ *
+ * The file is an append-only log, one JSON record per line, compacted when it
+ * is loaded. That shape falls out of two requirements pulling against each
+ * other:
+ *
+ * - It has to be written after every single upload, not once at the end of the
+ *   run. An upload that has been paid for but forgotten is money burnt, and a
+ *   deploy killed part way through is the normal case rather than the
+ *   exceptional one.
+ * - Rewriting the whole file per upload is quadratic. A first deploy or a
+ *   migration of a few thousand files spends minutes and gigabytes of writes on
+ *   nothing but bookkeeping.
+ *
+ * Appending one line is constant work, and it is *more* crash safe than a
+ * rewrite rather than less: a process killed mid write can only damage the last
+ * line, which is dropped on load, where a torn rewrite loses every id in the
+ * file. Compaction happens once per load, through a temp file and a rename, so
+ * the readable file is never the partial one.
+ *
+ * Malformed records are dropped. A hand edited index must never be able to
+ * publish a manifest pointing at nothing.
+ */
+export function createFileFolderIndex({ filePath, logger, }) {
+    const absolutePath = resolvePath(filePath);
+    const tempPath = `${absolutePath}.tmp`;
+    const map = new Map();
+    const record = (key, id) => `${JSON.stringify({ k: key, i: id })}\n`;
+    const header = `${JSON.stringify({ folderIndex: fileFolderIndexVersion })}\n`;
+    /** Rewrites the log as one record per live entry, atomically. */
+    const compact = () => {
+        mkdirSync(dirname(absolutePath), { recursive: true });
+        const body = [...map]
+            // Code units, not `localeCompare`, for the same reason the key encoding
+            // avoids it: it is not a total order over distinct strings and depends on
+            // the host's ICU data. Only the file's readability rides on this one, but
+            // a sort that can call two distinct keys equal has no business here.
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([key, id]) => record(key, id))
+            .join('');
+        try {
+            writeFileSync(tempPath, `${header}${body}`);
+            renameSync(tempPath, absolutePath);
+        }
+        catch (error) {
+            rmSync(tempPath, { force: true });
+            throw error;
+        }
+    };
+    let lineCount = 0;
+    if (existsSync(absolutePath)) {
+        let malformed = 0;
+        // A line at a time, so a truncated tail costs one entry rather than all of
+        // them.
+        for (const line of readFileSync(absolutePath, 'utf-8').split('\n')) {
+            if (line.length === 0) {
+                continue;
+            }
+            lineCount++;
+            try {
+                const parsed = JSON.parse(line);
+                if (parsed?.folderIndex !== undefined) {
+                    continue; // header
+                }
+                if (isValidFolderIndexKey(parsed?.k) &&
+                    isValidArweaveBase64URL(parsed?.i)) {
+                    map.set(parsed.k, parsed.i); // last write wins
+                }
+                else {
+                    malformed++;
+                }
+            }
+            catch {
+                malformed++;
+            }
+        }
+        if (malformed > 0) {
+            logger?.warn(`Dropped ${malformed} unreadable record(s) from the folder index at ${absolutePath}`);
+        }
+        // Only worth rewriting when the log has actually accumulated slack.
+        if (lineCount > map.size + 1) {
+            try {
+                compact();
+            }
+            catch (error) {
+                logger?.warn(`Could not compact the folder index at ${absolutePath}`, error);
+            }
+        }
+    }
+    return {
+        name: `file:${absolutePath}`,
+        get: (key) => map.get(key),
+        set: (key, id) => {
+            map.set(key, id);
+            mkdirSync(dirname(absolutePath), { recursive: true });
+            if (!existsSync(absolutePath)) {
+                writeFileSync(absolutePath, header);
+            }
+            appendFileSync(absolutePath, record(key, id));
+        },
+        knownContentHashes: (contentHashes) => {
+            const known = new Set([...map.keys()].map(contentHashFromFolderIndexKey));
+            return contentHashes.filter((contentHash) => known.has(contentHash));
+        },
+        entries: () => Object.fromEntries(map),
+    };
+}

package/lib/esm/node/index.js

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 export * from './factory.js';
+export * from './folderIndex.js';
 export * from './signer.js';
 export * from './upload.js';
 export * from '../types.js';

package/lib/esm/node/upload.js

@@ -15,6 +15,8 @@
  */
 import { createReadStream, promises, statSync } from 'fs';
 import { lookup } from 'mime-types';
+import { createHash } from 'node:crypto';
+import { pipeline } from 'node:stream/promises';
 import { join } from 'path';
 import { Readable } from 'stream';
 import { TurboAuthenticatedBaseUploadService, defaultUploadServiceURL, } from '../common/upload.js';
@@ -84,4 +86,9 @@ export class TurboAuthenticatedUploadService extends TurboAuthenticatedBaseUploa
     createManifestStream(manifestBuffer) {
         return Readable.from(manifestBuffer);
     }
+    async computeContentHash(file) {
+        const hash = createHash('sha256');
+        await pipeline(createReadStream(file), hash);
+        return hash.digest('hex');
+    }
 }