@ardrive/turbo-sdk 1.42.0-alpha.13 → 1.42.0-alpha.14

package/lib/esm/common/upload.js

@@ -16,9 +16,10 @@
 import { BigNumber } from 'bignumber.js';
 import { pLimit } from 'plimit-lit';
 import { ExistingBalanceFunding, OnDemandFunding, X402Funding, } from '../types.js';
-import { isBlob, sleep } from '../utils/common.js';
+import { isBlob, isValidArweaveBase64URL, sleep } from '../utils/common.js';
 import { AbortError } from '../utils/errors.js';
 import { FailedRequestError } from '../utils/errors.js';
+import { contentHashFromFolderIndexKey, contentHashTagName, folderIndexKey, } from '../utils/folderIndex.js';
 import { ChunkedUploader } from './chunked.js';
 import { TurboEventEmitter, createStreamWithUploadEvents } from './events.js';
 import { defaultRetryConfig } from './http.js';
@@ -366,6 +367,193 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
         };
         return manifest;
     }
+    /**
+     * The sha-256 of a file's bytes, as lowercase hex. The key of a
+     * {@link TurboFolderUploadIndex}.
+     *
+     * This default consumes the file's stream and digests it with the platform
+     * WebCrypto implementation, which serves the browser. NodeJS overrides it
+     * with a streaming digest so a large file is never held in memory.
+     */
+    async computeContentHash(file) {
+        const stream = this.getFileStreamForFile(file);
+        const chunks = [];
+        if ('getReader' in stream) {
+            const reader = stream.getReader();
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                chunks.push(value);
+            }
+        }
+        else {
+            for await (const chunk of stream) {
+                chunks.push(new Uint8Array(chunk));
+            }
+        }
+        const digest = await crypto.subtle.digest('SHA-256', Buffer.concat(chunks));
+        return Buffer.from(digest).toString('hex');
+    }
+    /**
+     * The exact tag set a folder upload writes for one file. Shared by the
+     * planner and the uploader, so that the tags a folder index key is computed
+     * from are, without question, the tags the data item ends up carrying.
+     */
+    folderFileTags({ file, dataItemOpts, contentHash, hashTagName = contentHashTagName, }) {
+        return [
+            ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type' &&
+                // Only stripped when this upload is going to write its own. Without a
+                // folder index, a caller's File-SHA256 tag is theirs to keep.
+                (contentHash === undefined || tag.name !== hashTagName)) ?? []),
+            {
+                name: 'Content-Type',
+                value: this.getContentType(file, dataItemOpts),
+            },
+            ...(contentHash !== undefined
+                ? [{ name: hashTagName, value: contentHash }]
+                : []),
+        ];
+    }
+    /**
+     * A folder index is a cache, so a layer that is unreachable must mean a miss
+     * and not an aborted deploy. The write side is forgiving for the same reason.
+     */
+    async readFolderIndex(folderIndex, keys, signal) {
+        const knownIds = new Map();
+        const unresolved = [];
+        for (const key of keys) {
+            let id;
+            try {
+                id = await folderIndex.get(key);
+            }
+            catch (error) {
+                this.logger.error('Failed to read from folder index', error);
+            }
+            if (id !== undefined && isValidArweaveBase64URL(id)) {
+                knownIds.set(key, id);
+            }
+            else {
+                unresolved.push(key);
+            }
+        }
+        if (unresolved.length > 0 && folderIndex.resolve !== undefined) {
+            try {
+                const resolved = await folderIndex.resolve(unresolved, { signal });
+                for (const [key, id] of Object.entries(resolved)) {
+                    if (isValidArweaveBase64URL(id)) {
+                        knownIds.set(key, id);
+                    }
+                }
+            }
+            catch (error) {
+                this.logger.error('Failed to resolve from folder index', error);
+            }
+        }
+        return knownIds;
+    }
+    /**
+     * Hashes every file and derives its folder index key, then works out which of
+     * those keys already have a data item id -- from the index, or from an
+     * identical file earlier in this same folder.
+     */
+    async planFolderIndex({ files, folderIndex, dataItemOpts, limit, signal, }) {
+        const contentHashes = new Map();
+        const itemKeys = new Map();
+        await Promise.all(files.map((file) => limit(async () => {
+            const contentHash = await this.computeContentHash(file);
+            contentHashes.set(file, contentHash);
+            itemKeys.set(file, await folderIndexKey({
+                contentHash,
+                tags: this.folderFileTags({
+                    file,
+                    dataItemOpts,
+                    contentHash,
+                    hashTagName: folderIndex.hashTagName,
+                }),
+            }));
+        })));
+        const knownIds = await this.readFolderIndex(folderIndex, [...new Set(itemKeys.values())], signal);
+        // Two files that would produce the same data item share a single upload, so
+        // the plan deduplicates within the folder as well as against the index.
+        const claimed = new Set();
+        const filesToUpload = files.filter((file) => {
+            const key = itemKeys.get(file);
+            if (knownIds.has(key) || claimed.has(key)) {
+                return false;
+            }
+            claimed.add(key);
+            return true;
+        });
+        this.logger.debug('Planned folder index', {
+            folderIndex: folderIndex.name,
+            totalFiles: files.length,
+            filesToUpload: filesToUpload.length,
+        });
+        await this.warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds });
+        return { contentHashes, itemKeys, knownIds, filesToUpload };
+    }
+    /**
+     * A key covers the tags on a file as well as its bytes, so a tag in
+     * `dataItemOpts` that changes between deploys re-uploads the whole folder at
+     * full price. That is the right answer -- a reused data item is never one
+     * this call would not have made -- but on its own it is a silent cost cliff:
+     * a successful deploy, a full bill, and nothing saying why.
+     *
+     * The signature of that mistake is exact, and it is already in hand. A key is
+     * `<bytes>.<tags>`, so a file whose *bytes half* the index knows under some
+     * other tags half is a file whose content is already paid for and whose tags
+     * moved. Nothing else produces that: a folder the index has never seen has
+     * unknown bytes, and a layer that could not be reached reports nothing known.
+     * It also catches one file in a hundred, not just all of them.
+     */
+    async warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds, }) {
+        if (folderIndex.knownContentHashes === undefined) {
+            return;
+        }
+        // Counted in files, not in keys: the message is about a bill, and the bill
+        // is per file. Two files that share a key are one upload but the reader is
+        // looking for their own file count.
+        const filesPerKey = new Map();
+        for (const key of itemKeys.values()) {
+            filesPerKey.set(key, (filesPerKey.get(key) ?? 0) + 1);
+        }
+        const missed = [...filesPerKey.keys()].filter((key) => !knownIds.has(key));
+        if (missed.length === 0) {
+            return;
+        }
+        const filesFor = (keys) => keys.reduce((count, key) => count + (filesPerKey.get(key) ?? 0), 0);
+        const missedFiles = filesFor(missed);
+        const missedByContentHash = new Map();
+        for (const key of missed) {
+            const contentHash = contentHashFromFolderIndexKey(key);
+            missedByContentHash.set(contentHash, [
+                ...(missedByContentHash.get(contentHash) ?? []),
+                key,
+            ]);
+        }
+        let staleTagged = [];
+        try {
+            staleTagged =
+                (await folderIndex.knownContentHashes([
+                    ...missedByContentHash.keys(),
+                ])) ?? [];
+        }
+        catch (error) {
+            this.logger.error('Failed to read from folder index', error);
+            return;
+        }
+        const affected = staleTagged.reduce((count, contentHash) => count + filesFor(missedByContentHash.get(contentHash) ?? []), 0);
+        if (affected === 0) {
+            return;
+        }
+        this.logger.warn(`${affected} of the ${missedFiles} file(s) this run is about to upload are already on Arweave byte for byte, under a different set of tags. ` +
+            'Their content has not changed but their tags have, so they are being paid for again. A folder index key covers the tags ' +
+            'on a file as well as its bytes. That is usually a tag in dataItemOpts whose value changes between deploys -- a commit ' +
+            'sha, a build number, a timestamp -- in which case move it to manifestDataItemOpts rather than paying for these files ' +
+            'again. It can also be a file that kept its content but changed its Content-Type, through a rename or a new extension, ' +
+            'which is expected and costs one upload.');
+    }
     getContentType(file, dataItemOpts) {
         const userDefinedContentType = dataItemOpts?.tags?.find((tag) => tag.name === 'Content-Type')?.value;
         if (userDefinedContentType !== undefined) {
@@ -375,7 +563,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
     }
     async uploadFolder(params) {
         this.logger.debug('Uploading folder...', { params });
-        const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
+        const { dataItemOpts, manifestDataItemOpts = dataItemOpts, folderIndex, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
         const { disableManifest, indexFile, fallbackFile } = manifestOptions;
         // Create event emitter from events parameter
         const emitter = new TurboEventEmitter(events);
@@ -384,15 +572,37 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             fileResponses: [],
         };
         const errors = [];
+        const limit = pLimit(maxConcurrentUploads);
         // Get files and calculate total bytes upfront for progress tracking
         const files = await this.getFiles(params);
-        const totalFiles = files.length;
+        // With an index, only the files whose bytes are not already on Arweave are
+        // signed, uploaded and paid for. Progress totals cover that subset, since
+        // it is all this call actually does.
+        let contentHashes = new Map();
+        let itemKeys = new Map();
+        let knownIds = new Map();
+        let filesToUpload = files;
+        if (folderIndex !== undefined) {
+            ({ contentHashes, itemKeys, knownIds, filesToUpload } =
+                await this.planFolderIndex({
+                    files,
+                    folderIndex,
+                    dataItemOpts,
+                    limit,
+                    signal,
+                }));
+        }
+        const totalFiles = filesToUpload.length;
         let totalBytes = 0;
+        let folderBytes = 0;
         const fileSizes = new Map();
         files.forEach((file) => {
             const size = this.getFileSize(file);
             fileSizes.set(file, size);
-            totalBytes += size;
+            folderBytes += size;
+        });
+        filesToUpload.forEach((file) => {
+            totalBytes += fileSizes.get(file) ?? 0;
         });
         // Track progress across all files
         let processedFiles = 0;
@@ -407,13 +617,15 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                 fileIndex,
                 totalFiles,
             });
-            const contentType = this.getContentType(file, dataItemOpts);
+            const contentHash = contentHashes.get(file);
             const dataItemOptsWithContentType = {
                 ...dataItemOpts,
-                tags: [
-                    ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ?? []),
-                    { name: 'Content-Type', value: contentType },
-                ],
+                tags: this.folderFileTags({
+                    file,
+                    dataItemOpts,
+                    contentHash,
+                    hashTagName: folderIndex?.hashTagName,
+                }),
             };
             try {
                 const result = await this.uploadFile({
@@ -458,8 +670,24 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                     },
                     fundingMode,
                 });
-                const relativePath = this.getRelativePath(file, params);
-                paths[relativePath] = { id: result.id };
+                const itemKey = itemKeys.get(file);
+                if (itemKey !== undefined) {
+                    // Recorded before anything else can fail, so a run that is killed
+                    // part way through never loses a file it has already paid for.
+                    knownIds.set(itemKey, result.id);
+                    try {
+                        await folderIndex?.set(itemKey, result.id);
+                    }
+                    catch (error) {
+                        // A failed index write costs the next run a re-upload; it must not
+                        // fail this one, which has already paid for and landed the bytes.
+                        this.logger.error('Failed to write to folder index', error);
+                    }
+                }
+                else {
+                    const relativePath = this.getRelativePath(file, params);
+                    paths[relativePath] = { id: result.id };
+                }
                 response.fileResponses.push(result);
                 // Update processed counts after file completes
                 processedFiles++;
@@ -495,10 +723,9 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                 errors.push(error);
             }
         };
-        const limit = pLimit(maxConcurrentUploads);
         let cryptoFundResult;
         if (fundingMode instanceof OnDemandFunding) {
-            const totalByteCount = files.reduce((acc, file) => {
+            const totalByteCount = filesToUpload.reduce((acc, file) => {
                 return acc + this.getFileSize(file) + 1200; // allow extra per file for ANS-104 headers
             }, 0);
             cryptoFundResult = await this.onDemand({
@@ -506,15 +733,36 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
                 onDemandFunding: fundingMode,
             });
         }
-        await Promise.all(files.map((file, index) => limit(() => uploadFile(file, index))));
+        await Promise.all(filesToUpload.map((file, index) => limit(() => uploadFile(file, index))));
         this.logger.debug('Finished uploading files', {
-            numFiles: files.length,
+            numFiles: filesToUpload.length,
             numErrors: errors.length,
             results: response.fileResponses,
         });
         if (errors.length > 0) {
             response.errors = errors;
         }
+        if (folderIndex !== undefined) {
+            // Built in folder order rather than in completion order, so an unchanged
+            // folder produces byte identical manifest bytes -- and therefore an
+            // identical manifest data item id -- run after run.
+            for (const file of files) {
+                const id = knownIds.get(itemKeys.get(file));
+                if (id !== undefined) {
+                    paths[this.getRelativePath(file, params)] = { id };
+                }
+            }
+            response.folderIndexSummary = {
+                totalFiles: files.length,
+                totalBytes: folderBytes,
+                // What landed, not what was planned. With throwOnFailure: false a file
+                // can be planned, paid for nothing, and never arrive.
+                uploadedFiles: processedFiles,
+                uploadedBytes: processedBytes,
+                reusedFiles: files.length - filesToUpload.length,
+                reusedBytes: folderBytes - totalBytes,
+            };
+        }
         if (disableManifest) {
             emitter.emit('folder-success');
             return response;
@@ -533,8 +781,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             fallbackFile,
         });
         const tagsWithManifestContentType = [
-            ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ??
-                []),
+            ...(manifestDataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ?? []),
             { name: 'Content-Type', value: 'application/x.arweave-manifest+json' },
         ];
         const manifestBuffer = Buffer.from(JSON.stringify(manifest));
@@ -544,7 +791,10 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
             fileStreamFactory: () => this.createManifestStream(manifestBuffer),
             fileSizeFactory: () => manifestBuffer.byteLength,
             signal,
-            dataItemOpts: { ...dataItemOpts, tags: tagsWithManifestContentType },
+            dataItemOpts: {
+                ...manifestDataItemOpts,
+                tags: tagsWithManifestContentType,
+            },
             chunkByteCount,
             maxChunkConcurrency,
             maxFinalizeMs,

package/lib/esm/node/folderIndex.js

@@ -0,0 +1,129 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync, } from 'node:fs';
+import { dirname, resolve as resolvePath } from 'node:path';
+import { isValidArweaveBase64URL } from '../utils/common.js';
+import { contentHashFromFolderIndexKey, isValidFolderIndexKey, } from '../utils/folderIndex.js';
+const fileFolderIndexVersion = 2;
+/**
+ * A local record of folder index key to data item id, for uploads on a machine
+ * that keeps its working directory between deploys.
+ *
+ * The file is an append-only log, one JSON record per line, compacted when it
+ * is loaded. That shape falls out of two requirements pulling against each
+ * other:
+ *
+ * - It has to be written after every single upload, not once at the end of the
+ *   run. An upload that has been paid for but forgotten is money burnt, and a
+ *   deploy killed part way through is the normal case rather than the
+ *   exceptional one.
+ * - Rewriting the whole file per upload is quadratic. A first deploy or a
+ *   migration of a few thousand files spends minutes and gigabytes of writes on
+ *   nothing but bookkeeping.
+ *
+ * Appending one line is constant work, and it is *more* crash safe than a
+ * rewrite rather than less: a process killed mid write can only damage the last
+ * line, which is dropped on load, where a torn rewrite loses every id in the
+ * file. Compaction happens once per load, through a temp file and a rename, so
+ * the readable file is never the partial one.
+ *
+ * Malformed records are dropped. A hand edited index must never be able to
+ * publish a manifest pointing at nothing.
+ */
+export function createFileFolderIndex({ filePath, logger, }) {
+    const absolutePath = resolvePath(filePath);
+    const tempPath = `${absolutePath}.tmp`;
+    const map = new Map();
+    const record = (key, id) => `${JSON.stringify({ k: key, i: id })}\n`;
+    const header = `${JSON.stringify({ folderIndex: fileFolderIndexVersion })}\n`;
+    /** Rewrites the log as one record per live entry, atomically. */
+    const compact = () => {
+        mkdirSync(dirname(absolutePath), { recursive: true });
+        const body = [...map]
+            // Code units, not `localeCompare`, for the same reason the key encoding
+            // avoids it: it is not a total order over distinct strings and depends on
+            // the host's ICU data. Only the file's readability rides on this one, but
+            // a sort that can call two distinct keys equal has no business here.
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([key, id]) => record(key, id))
+            .join('');
+        try {
+            writeFileSync(tempPath, `${header}${body}`);
+            renameSync(tempPath, absolutePath);
+        }
+        catch (error) {
+            rmSync(tempPath, { force: true });
+            throw error;
+        }
+    };
+    let lineCount = 0;
+    if (existsSync(absolutePath)) {
+        let malformed = 0;
+        // A line at a time, so a truncated tail costs one entry rather than all of
+        // them.
+        for (const line of readFileSync(absolutePath, 'utf-8').split('\n')) {
+            if (line.length === 0) {
+                continue;
+            }
+            lineCount++;
+            try {
+                const parsed = JSON.parse(line);
+                if (parsed?.folderIndex !== undefined) {
+                    continue; // header
+                }
+                if (isValidFolderIndexKey(parsed?.k) &&
+                    isValidArweaveBase64URL(parsed?.i)) {
+                    map.set(parsed.k, parsed.i); // last write wins
+                }
+                else {
+                    malformed++;
+                }
+            }
+            catch {
+                malformed++;
+            }
+        }
+        if (malformed > 0) {
+            logger?.warn(`Dropped ${malformed} unreadable record(s) from the folder index at ${absolutePath}`);
+        }
+        // Only worth rewriting when the log has actually accumulated slack.
+        if (lineCount > map.size + 1) {
+            try {
+                compact();
+            }
+            catch (error) {
+                logger?.warn(`Could not compact the folder index at ${absolutePath}`, error);
+            }
+        }
+    }
+    return {
+        name: `file:${absolutePath}`,
+        get: (key) => map.get(key),
+        set: (key, id) => {
+            map.set(key, id);
+            mkdirSync(dirname(absolutePath), { recursive: true });
+            if (!existsSync(absolutePath)) {
+                writeFileSync(absolutePath, header);
+            }
+            appendFileSync(absolutePath, record(key, id));
+        },
+        knownContentHashes: (contentHashes) => {
+            const known = new Set([...map.keys()].map(contentHashFromFolderIndexKey));
+            return contentHashes.filter((contentHash) => known.has(contentHash));
+        },
+        entries: () => Object.fromEntries(map),
+    };
+}

package/lib/esm/node/index.js

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 export * from './factory.js';
+export * from './folderIndex.js';
 export * from './signer.js';
 export * from './upload.js';
 export * from '../types.js';

package/lib/esm/node/upload.js

@@ -15,6 +15,8 @@
  */
 import { createReadStream, promises, statSync } from 'fs';
 import { lookup } from 'mime-types';
+import { createHash } from 'node:crypto';
+import { pipeline } from 'node:stream/promises';
 import { join } from 'path';
 import { Readable } from 'stream';
 import { TurboAuthenticatedBaseUploadService, defaultUploadServiceURL, } from '../common/upload.js';
@@ -84,4 +86,9 @@ export class TurboAuthenticatedUploadService extends TurboAuthenticatedBaseUploa
     createManifestStream(manifestBuffer) {
         return Readable.from(manifestBuffer);
     }
+    async computeContentHash(file) {
+        const hash = createHash('sha256');
+        await pipeline(createReadStream(file), hash);
+        return hash.digest('hex');
+    }
 }

package/lib/esm/utils/folderIndex.js

@@ -0,0 +1,81 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Internals shared by the folder index layers and by `uploadFolder`. Not part
+ * of the package's public surface.
+ */
+/** Tag carrying the sha-256 of a file's own bytes, as lowercase hex. */
+export const contentHashTagName = 'File-SHA256';
+const contentHashRegex = /^[0-9a-f]{64}$/;
+const folderIndexKeyRegex = /^[0-9a-f]{64}\.[0-9a-f]{64}$/;
+/** True for a lowercase hex sha-256 digest of a file's bytes. */
+export function isValidContentHash(hash) {
+    return hash !== undefined && contentHashRegex.test(hash);
+}
+/**
+ * A folder index key is `<sha-256 of the bytes>.<sha-256 of the tags>`.
+ *
+ * Both halves matter. Keying on the bytes alone would reuse an item whose tags
+ * are not the ones the caller asked for: an empty `a.css` and an empty `b.js`
+ * hash identically, and reusing one for the other would serve JavaScript as
+ * `text/css`. Keying on the tags as well means a reused data item is always
+ * exactly the data item this call would otherwise have created.
+ *
+ * The bytes half is kept in the clear so a layer that queries a gateway can
+ * recover it and filter on the `File-SHA256` tag.
+ */
+export function isValidFolderIndexKey(key) {
+    return key !== undefined && folderIndexKeyRegex.test(key);
+}
+/** The bytes half of a folder index key. */
+export function contentHashFromFolderIndexKey(key) {
+    return key.slice(0, 64);
+}
+async function sha256Hex(data) {
+    const digest = await crypto.subtle.digest('SHA-256', data);
+    return Buffer.from(digest).toString('hex');
+}
+/**
+ * Canonical encoding of a tag set: the same tags in a different order encode
+ * identically, because two data items that differ only in tag order are the
+ * same item as far as reuse goes.
+ *
+ * The sort compares code units rather than using `localeCompare`, which is not
+ * a total order over distinct strings and depends on the host's ICU data --
+ * NFC and NFD spellings of the same word compare equal without being equal, and
+ * two locales disagree on where an accented letter sorts. Either would let one
+ * tag set hash differently on two machines, which costs a reuse.
+ *
+ * The encoding has to be injective, not merely stable. An ANS-104 tag name and
+ * value are arbitrary byte arrays, so a delimiter-joined string collides: with
+ * a 0x00/0x01 join, a single tag whose value embeds both delimiters encodes
+ * exactly like two separate tags. A collision here is a wrong reuse, which is
+ * the one failure the composite key exists to make impossible. JSON escapes
+ * the delimiters, so no value can forge a record boundary.
+ */
+function canonicalTags(tags) {
+    const pairs = tags
+        .map((tag) => [tag.name, tag.value])
+        .sort(([aName, aValue], [bName, bValue]) => {
+        const [a, b] = aName === bName ? [aValue, bValue] : [aName, bName];
+        return a < b ? -1 : a > b ? 1 : 0;
+    });
+    return new TextEncoder().encode(JSON.stringify(pairs));
+}
+/** The folder index key for a file with these bytes and these final tags. */
+export async function folderIndexKey({ contentHash, tags, }) {
+    return `${contentHash}.${await sha256Hex(canonicalTags(tags))}`;
+}

package/lib/types/common/folderIndex.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ChainFolderUploadIndexParams, ComposeFolderUploadIndexParams, TurboFolderUploadIndex } from '../types.js';
+/**
+ * An in-memory folder index.
+ *
+ * On its own it only deduplicates identical files within a single
+ * `uploadFolder` call, which `uploadFolder` already does. Its real use is as
+ * the writable floor of a {@link composeFolderIndex} stack, or seeded from a
+ * mapping the caller persisted itself.
+ */
+export declare function createMemoryFolderIndex(seed?: Record<string, string>): TurboFolderUploadIndex;
+/**
+ * Rebuilds a folder index by sweeping the uploader's own past uploads over a
+ * gateway's GraphQL endpoint, filtering on the `File-SHA256` tag that
+ * index-backed uploads always write.
+ *
+ * Read only, and the only layer that survives a fresh checkout on a machine
+ * that has never deployed before -- a CI runner, most obviously.
+ *
+ * Known limitation: gateways index an upload minutes after it lands, so two
+ * machines deploying the same new file at the same moment will each pay for it
+ * once. The manifest is correct either way; only the bill is affected, and only
+ * for genuinely new bytes.
+ */
+export declare function createChainFolderIndex({ owner, appName, gatewayUrl, hashTagName, maxPages, pageSize, timeoutMs, fetchImpl, logger, }: ChainFolderUploadIndexParams): TurboFolderUploadIndex;
+/**
+ * Layers folder indexes: reads fall through in order, writes go to every layer
+ * that is not read only.
+ *
+ * The usual stack is a local cache in front of a chain index -- the cache
+ * answers instantly on a developer machine, and the chain index is what a CI
+ * runner with an empty working directory falls back to.
+ *
+ * A layer that throws is skipped, not propagated. That is the whole point of
+ * stacking them: a full disk under the file layer must not stop the memory
+ * layer from holding ids the run has already paid for, and an unreachable
+ * gateway must not stop the local cache from answering.
+ */
+export declare function composeFolderIndex(layers: (TurboFolderUploadIndex | undefined)[], { logger }?: ComposeFolderUploadIndexParams): TurboFolderUploadIndex;
+//# sourceMappingURL=folderIndex.d.ts.map

package/lib/types/common/folderIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"folderIndex.d.ts","sourceRoot":"","sources":["../../../src/common/folderIndex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAEL,4BAA4B,EAC5B,8BAA8B,EAC9B,sBAAsB,EACvB,MAAM,aAAa,CAAC;AAkIrB;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,GAChC,sBAAsB,CAmBxB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,KAAK,EACL,OAAO,EACP,UAAkC,EAClC,WAAgC,EAChC,QAAa,EACb,QAAc,EACd,SAAkB,EAClB,SAAiB,EACjB,MAAM,GACP,EAAE,4BAA4B,GAAG,sBAAsB,CAsKvD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,CAAC,sBAAsB,GAAG,SAAS,CAAC,EAAE,EAC9C,EAAE,MAAM,EAAE,GAAE,8BAAmC,GAC9C,sBAAsB,CAsIxB"}