@aztec/stdlib 3.0.0-nightly.20251005 → 3.0.0-nightly.20251007

package/dest/file-store/interface.d.ts

@@ -14,9 +14,15 @@ export type FileStoreSaveOptions = {
 };
 /** Simple file store. */
 export interface FileStore extends ReadOnlyFileStore {
-    /** Saves contents to the given path. Returns an URI that can be used later to `read` the file. */
+    /**
+     * Saves contents to the given path. Returns an URI that can be used later to `read` the file.
+     * Default: `compress` is false unless explicitly set.
+     */
     save(path: string, data: Buffer, opts?: FileStoreSaveOptions): Promise<string>;
-    /** Uploads contents from a local file. Returns an URI that can be used later to `read` the file. */
+    /**
+     * Uploads contents from a local file. Returns an URI that can be used later to `read` the file.
+     * Default: `compress` is true unless explicitly set to false.
+     */
     upload(destPath: string, srcPath: string, opts?: FileStoreSaveOptions): Promise<string>;
 }
 //# sourceMappingURL=interface.d.ts.map

package/dest/file-store/interface.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/file-store/interface.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,MAAM,WAAW,iBAAiB;IAChC,iGAAiG;IACjG,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,wGAAwG;IACxG,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChE,8DAA8D;IAC9D,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7C;AAED,MAAM,MAAM,oBAAoB,GAAG;IAAE,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE/G,yBAAyB;AACzB,MAAM,WAAW,SAAU,SAAQ,iBAAiB;IAClD,kGAAkG;IAClG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/E,oGAAoG;IACpG,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzF"}
+{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/file-store/interface.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,MAAM,WAAW,iBAAiB;IAChC,iGAAiG;IACjG,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,wGAAwG;IACxG,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChE,8DAA8D;IAC9D,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7C;AAED,MAAM,MAAM,oBAAoB,GAAG;IAAE,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE/G,yBAAyB;AACzB,MAAM,WAAW,SAAU,SAAQ,iBAAiB;IAClD;;;OAGG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/E;;;OAGG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzF"}

package/dest/file-store/s3.d.ts

@@ -18,6 +18,7 @@ export declare class S3FileStore implements FileStore {
     download(pathOrUrlStr: string, destPath: string): Promise<void>;
     exists(pathOrUrlStr: string): Promise<boolean>;
     private extractUserMetadata;
+    private detectContentType;
     private buildReturnedUrl;
     private getBucketAndKey;
     private getFullPath;

package/dest/file-store/s3.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../src/file-store/s3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAgBlE,OAAO,KAAK,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAMtE,qBAAa,WAAY,YAAW,SAAS;IAOzC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAEzB,OAAO,CAAC,QAAQ,CAAC,GAAG;IATtB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAW;IAC9B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;gBAGrB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EACjC,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,EAClC,GAAG,GAAE,MAA6C;IAiBxD,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,oBAAyB,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBlF,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,oBAAyB,GAAG,OAAO,CAAC,MAAM,CAAC;IAoB3F,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAW3C,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/D,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe3D,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,gBAAgB;IA0BxB,OAAO,CAAC,eAAe;IA2BvB,OAAO,CAAC,WAAW;CAKpB"}
+{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../src/file-store/s3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAiBlE,OAAO,KAAK,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAMtE,qBAAa,WAAY,YAAW,SAAS;IAOzC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAEzB,OAAO,CAAC,QAAQ,CAAC,GAAG;IATtB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAW;IAC9B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;gBAGrB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EACjC,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,EAClC,GAAG,GAAE,MAA6C;IAiBxD,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,oBAAyB,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBlF,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,oBAAyB,GAAG,OAAO,CAAC,MAAM,CAAC;IA6D3F,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAW3C,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/D,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe3D,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,gBAAgB;IA0BxB,OAAO,CAAC,eAAe;IA2BvB,OAAO,CAAC,WAAW;CAKpB"}

package/dest/file-store/s3.js

@@ -1,8 +1,9 @@
 import { createLogger } from '@aztec/foundation/log';
 import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
 import { createReadStream, createWriteStream } from 'fs';
-import { mkdir } from 'fs/promises';
-import { dirname, join } from 'path';
+import { mkdir, mkdtemp, stat, unlink } from 'fs/promises';
+import { tmpdir } from 'os';
+import { basename, dirname, join } from 'path';
 import { finished } from 'stream/promises';
 import { createGzip } from 'zlib';
 function normalizeBasePath(path) {
@@ -35,15 +36,18 @@ export class S3FileStore {
     }
     async save(path, data, opts = {}) {
         const key = this.getFullPath(path);
-        const shouldCompress = !opts.compress;
+        const shouldCompress = !!opts.compress;
         const body = shouldCompress ? (await import('zlib')).gzipSync(data) : data;
+        const contentLength = body.length;
+        const contentType = this.detectContentType(key, shouldCompress);
         const put = new PutObjectCommand({
             Bucket: this.bucketName,
             Key: key,
             Body: body,
-            ContentEncoding: shouldCompress ? 'gzip' : undefined,
+            ContentType: contentType,
             CacheControl: opts.metadata?.['Cache-control'],
-            Metadata: this.extractUserMetadata(opts.metadata)
+            Metadata: this.extractUserMetadata(opts.metadata),
+            ContentLength: contentLength
         });
         await this.s3.send(put);
         return this.buildReturnedUrl(key, !!opts.public);
@@ -54,17 +58,57 @@ export class S3FileStore {
         await mkdir(dirname(srcPath), {
             recursive: true
         }).catch(()=>undefined);
-        const source = createReadStream(srcPath);
-        const bodyStream = shouldCompress ? source.pipe(createGzip()) : source;
-        const put = new PutObjectCommand({
-            Bucket: this.bucketName,
-            Key: key,
-            Body: bodyStream,
-            ContentEncoding: shouldCompress ? 'gzip' : undefined,
-            CacheControl: opts.metadata?.['Cache-control'],
-            Metadata: this.extractUserMetadata(opts.metadata)
-        });
-        await this.s3.send(put);
+        let contentLength;
+        let bodyPath = srcPath;
+        // We don't set Content-Encoding and we avoid SigV4 streaming (aws-chunked).
+        // With AWS SigV4 streaming uploads (Content-Encoding: aws-chunked[,gzip]), servers require
+        // x-amz-decoded-content-length (the size of the decoded payload) and an exact Content-Length
+        // that includes chunk metadata. For on-the-fly compression, providing
+        // those values without buffering or a pre-pass is impractical. Instead, we pre-gzip to a temp file
+        // to know ContentLength up-front and upload the gzipped bytes as-is, omitting Content-Encoding.
+        // Reference: AWS SigV4 streaming (chunked upload) requirements —
+        // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+        if (shouldCompress) {
+            // Pre-gzip to a temp file so we know the exact length for R2/S3 headers
+            const tmpDir = await mkdtemp(join(tmpdir(), 's3-upload-'));
+            const gzPath = join(tmpDir, `${basename(srcPath)}.gz`);
+            const source = createReadStream(srcPath);
+            const gz = createGzip();
+            const out = createWriteStream(gzPath);
+            try {
+                await finished(source.pipe(gz).pipe(out));
+                const st = await stat(gzPath);
+                contentLength = st.size;
+                bodyPath = gzPath;
+            } catch (err) {
+                // Ensure temp file is removed on failure
+                await unlink(gzPath).catch(()=>undefined);
+                throw err;
+            }
+        } else {
+            const st = await stat(srcPath);
+            contentLength = st.size;
+            bodyPath = srcPath;
+        }
+        const bodyStream = createReadStream(bodyPath);
+        const contentType = this.detectContentType(key, shouldCompress);
+        try {
+            const put = new PutObjectCommand({
+                Bucket: this.bucketName,
+                Key: key,
+                Body: bodyStream,
+                ContentType: contentType,
+                CacheControl: opts.metadata?.['Cache-control'],
+                Metadata: this.extractUserMetadata(opts.metadata),
+                // Explicitly set ContentLength so R2 can compute x-amz-decoded-content-length correctly
+                ContentLength: contentLength
+            });
+            await this.s3.send(put);
+        } finally{
+            if (shouldCompress && bodyPath !== srcPath) {
+                await unlink(bodyPath).catch(()=>undefined);
+            }
+        }
         return this.buildReturnedUrl(key, !!opts.public);
     }
     async read(pathOrUrlStr) {
@@ -116,6 +160,27 @@ export class S3FileStore {
         const { ['Cache-control']: _ignored, ...rest } = meta;
         return Object.keys(rest).length ? rest : undefined;
     }
+    detectContentType(key, isCompressed) {
+        // Basic content type inference
+        const lower = key.toLowerCase();
+        if (lower.endsWith('.json') || lower.endsWith('.json.gz')) {
+            return 'application/json';
+        }
+        if (lower.endsWith('.txt') || lower.endsWith('.log') || lower.endsWith('.csv') || lower.endsWith('.md')) {
+            return 'text/plain; charset=utf-8';
+        }
+        if (lower.endsWith('.db') || lower.endsWith('.sqlite') || lower.endsWith('.bin')) {
+            return 'application/octet-stream';
+        }
+        if (lower.endsWith('.wasm') || lower.endsWith('.wasm.gz')) {
+            return 'application/wasm';
+        }
+        // If compressed, prefer octet-stream unless known
+        if (isCompressed) {
+            return 'application/octet-stream';
+        }
+        return undefined;
+    }
     buildReturnedUrl(key, makePublic) {
         if (!makePublic) {
             return `s3://${this.bucketName}/${key}`;

package/dest/snapshots/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/snapshots/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAElE,OAAO,EACL,gBAAgB,EAChB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAE5B,MAAM,YAAY,CAAC;AAEpB,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAarC;AAED,wBAAsB,yBAAyB,CAC7C,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAGvC;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAEpE;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAE7E;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,gBAAgB,CAGnE;AAED,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAC5C,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,IAAI,CAAC,CAEf"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/snapshots/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAQlE,OAAO,EACL,gBAAgB,EAChB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAE5B,MAAM,YAAY,CAAC;AAEpB,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAcrC;AAED,wBAAsB,yBAAyB,CAC7C,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAGvC;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAEpE;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAE7E;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,gBAAgB,CAGnE;AAoCD,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAC5C,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,IAAI,CAAC,CAyBf"}

package/dest/snapshots/download.js

@@ -1,5 +1,10 @@
 import { fromEntries, getEntries, maxBy } from '@aztec/foundation/collection';
 import { jsonParseWithSchema } from '@aztec/foundation/json-rpc';
+import { createReadStream, createWriteStream } from 'fs';
+import fs from 'fs/promises';
+import pathMod from 'path';
+import { pipeline } from 'stream/promises';
+import { createGunzip, gunzipSync } from 'zlib';
 import { SnapshotDataKeys, SnapshotsIndexSchema } from './types.js';
 export async function getSnapshotIndex(metadata, store) {
     const basePath = getBasePath(metadata);
@@ -7,7 +12,8 @@ export async function getSnapshotIndex(metadata, store) {
     try {
         if (await store.exists(snapshotIndexPath)) {
             const snapshotIndexData = await store.read(snapshotIndexPath);
-            return jsonParseWithSchema(snapshotIndexData.toString(), SnapshotsIndexSchema);
+            const buf = maybeGunzip(snapshotIndexData);
+            return jsonParseWithSchema(buf.toString('utf-8'), SnapshotsIndexSchema);
         } else {
             return undefined;
         }
@@ -32,6 +38,56 @@ export function makeSnapshotPaths(baseDir) {
             `${baseDir}/${key}.db`
         ]));
 }
+function isGzipMagic(data) {
+    return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+}
+function maybeGunzip(data) {
+    const magicNumberIndicatesGzip = isGzipMagic(data);
+    if (magicNumberIndicatesGzip) {
+        try {
+            const out = gunzipSync(data);
+            return out;
+        } catch (err) {
+            throw new Error(`Decompression of gzipped data failed: ${err.message}`);
+        }
+    }
+    return data;
+}
+async function detectGzip(localFilePathToPeek) {
+    // Peek the actual bytes we downloaded.
+    try {
+        const fd = await fs.open(localFilePathToPeek, 'r');
+        try {
+            const header = Buffer.alloc(2);
+            const { bytesRead } = await fd.read(header, 0, 2, 0);
+            return bytesRead >= 2 && isGzipMagic(header);
+        } finally{
+            await fd.close();
+        }
+    } catch  {
+        return false;
+    }
+}
 export async function downloadSnapshot(snapshot, localPaths, store) {
-    await Promise.all(getEntries(localPaths).map(([key, path])=>store.download(snapshot.dataUrls[key], path)));
+    await Promise.all(getEntries(localPaths).map(async ([key, path])=>{
+        await fs.mkdir(pathMod.dirname(path), {
+            recursive: true
+        });
+        const tmpPath = `${path}.download`;
+        try {
+            const url = snapshot.dataUrls[key];
+            await store.download(url, tmpPath);
+            const isGzip = await detectGzip(tmpPath);
+            const read = createReadStream(tmpPath);
+            const write = createWriteStream(path);
+            if (isGzip) {
+                const gunzip = createGunzip();
+                await pipeline(read, gunzip, write);
+            } else {
+                await pipeline(read, write);
+            }
+        } finally{
+            await fs.unlink(tmpPath).catch(()=>undefined);
+        }
+    }));
 }

package/dest/snapshots/upload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/snapshots/upload.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAG1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAkB,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAE7G,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAC5C,cAAc,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,EAClD,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,SAAS,EAChB,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5E,OAAO,CAAC,gBAAgB,CAAC,CAsB3B;AAED,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAC5C,cAAc,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,EAClD,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,SAAS,GACf,OAAO,CAAC,gBAAgB,CAAC,CAU3B"}
+{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/snapshots/upload.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAG1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAkB,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAE7G,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAC5C,cAAc,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,EAClD,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,SAAS,EAChB,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5E,OAAO,CAAC,gBAAgB,CAAC,CAsB3B;AAED,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAC5C,cAAc,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,EAClD,QAAQ,EAAE,sBAAsB,EAChC,KAAK,EAAE,SAAS,GACf,OAAO,CAAC,gBAAgB,CAAC,CAW3B"}

package/dest/snapshots/upload.js

@@ -27,6 +27,7 @@ export async function uploadSnapshotToIndex(localPaths, schemaVersions, metadata
     snapshotsIndex.snapshots.unshift(newSnapshotMetadata);
     await store.save(getSnapshotIndexPath(metadata), Buffer.from(jsonStringify(snapshotsIndex, true)), {
         public: true,
+        compress: false,
         metadata: {
             ['Cache-control']: 'no-store'
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/stdlib",
-  "version": "3.0.0-nightly.20251005",
+  "version": "3.0.0-nightly.20251007",
   "type": "module",
   "inherits": [
     "../package.common.json",
@@ -70,13 +70,13 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.892.0",
-    "@aztec/bb.js": "3.0.0-nightly.20251005",
-    "@aztec/blob-lib": "3.0.0-nightly.20251005",
-    "@aztec/constants": "3.0.0-nightly.20251005",
-    "@aztec/ethereum": "3.0.0-nightly.20251005",
-    "@aztec/foundation": "3.0.0-nightly.20251005",
-    "@aztec/l1-artifacts": "3.0.0-nightly.20251005",
-    "@aztec/noir-noirc_abi": "3.0.0-nightly.20251005",
+    "@aztec/bb.js": "3.0.0-nightly.20251007",
+    "@aztec/blob-lib": "3.0.0-nightly.20251007",
+    "@aztec/constants": "3.0.0-nightly.20251007",
+    "@aztec/ethereum": "3.0.0-nightly.20251007",
+    "@aztec/foundation": "3.0.0-nightly.20251007",
+    "@aztec/l1-artifacts": "3.0.0-nightly.20251007",
+    "@aztec/noir-noirc_abi": "3.0.0-nightly.20251007",
     "@google-cloud/storage": "^7.15.0",
     "axios": "^1.12.0",
     "json-stringify-deterministic": "1.0.12",

package/src/file-store/interface.ts

@@ -12,8 +12,14 @@ export type FileStoreSaveOptions = { public?: boolean; metadata?: Record<string,
 
 /** Simple file store. */
 export interface FileStore extends ReadOnlyFileStore {
-  /** Saves contents to the given path. Returns an URI that can be used later to `read` the file. */
+  /**
+   * Saves contents to the given path. Returns an URI that can be used later to `read` the file.
+   * Default: `compress` is false unless explicitly set.
+   */
   save(path: string, data: Buffer, opts?: FileStoreSaveOptions): Promise<string>;
-  /** Uploads contents from a local file. Returns an URI that can be used later to `read` the file. */
+  /**
+   * Uploads contents from a local file. Returns an URI that can be used later to `read` the file.
+   * Default: `compress` is true unless explicitly set to false.
+   */
   upload(destPath: string, srcPath: string, opts?: FileStoreSaveOptions): Promise<string>;
 }

package/src/file-store/s3.ts

@@ -8,8 +8,9 @@ import {
   S3Client,
 } from '@aws-sdk/client-s3';
 import { createReadStream, createWriteStream } from 'fs';
-import { mkdir } from 'fs/promises';
-import { dirname, join } from 'path';
+import { mkdir, mkdtemp, stat, unlink } from 'fs/promises';
+import { tmpdir } from 'os';
+import { basename, dirname, join } from 'path';
 import { Readable } from 'stream';
 import { finished } from 'stream/promises';
 import { createGzip } from 'zlib';
@@ -49,15 +50,19 @@ export class S3FileStore implements FileStore {
 
   public async save(path: string, data: Buffer, opts: FileStoreSaveOptions = {}): Promise<string> {
     const key = this.getFullPath(path);
-    const shouldCompress = !opts.compress;
+    const shouldCompress = !!opts.compress;
+
     const body = shouldCompress ? (await import('zlib')).gzipSync(data) : data;
+    const contentLength = body.length;
+    const contentType = this.detectContentType(key, shouldCompress);
     const put = new PutObjectCommand({
       Bucket: this.bucketName,
       Key: key,
       Body: body,
-      ContentEncoding: shouldCompress ? 'gzip' : undefined,
+      ContentType: contentType,
       CacheControl: opts.metadata?.['Cache-control'],
       Metadata: this.extractUserMetadata(opts.metadata),
+      ContentLength: contentLength,
     });
     await this.s3.send(put);
     return this.buildReturnedUrl(key, !!opts.public);
@@ -68,18 +73,59 @@ export class S3FileStore implements FileStore {
     const shouldCompress = opts.compress !== false; // default true like GCS impl
 
     await mkdir(dirname(srcPath), { recursive: true }).catch(() => undefined);
+    let contentLength: number | undefined;
+    let bodyPath = srcPath;
+
+    // We don't set Content-Encoding and we avoid SigV4 streaming (aws-chunked).
+    // With AWS SigV4 streaming uploads (Content-Encoding: aws-chunked[,gzip]), servers require
+    // x-amz-decoded-content-length (the size of the decoded payload) and an exact Content-Length
+    // that includes chunk metadata. For on-the-fly compression, providing
+    // those values without buffering or a pre-pass is impractical. Instead, we pre-gzip to a temp file
+    // to know ContentLength up-front and upload the gzipped bytes as-is, omitting Content-Encoding.
+    // Reference: AWS SigV4 streaming (chunked upload) requirements —
+    // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+    if (shouldCompress) {
+      // Pre-gzip to a temp file so we know the exact length for R2/S3 headers
+      const tmpDir = await mkdtemp(join(tmpdir(), 's3-upload-'));
+      const gzPath = join(tmpDir, `${basename(srcPath)}.gz`);
+      const source = createReadStream(srcPath);
+      const gz = createGzip();
+      const out = createWriteStream(gzPath);
+      try {
+        await finished(source.pipe(gz).pipe(out));
+        const st = await stat(gzPath);
+        contentLength = st.size;
+        bodyPath = gzPath;
+      } catch (err) {
+        // Ensure temp file is removed on failure
+        await unlink(gzPath).catch(() => undefined);
+        throw err;
+      }
+    } else {
+      const st = await stat(srcPath);
+      contentLength = st.size;
+      bodyPath = srcPath;
+    }
 
-    const source = createReadStream(srcPath);
-    const bodyStream = shouldCompress ? source.pipe(createGzip()) : source;
-    const put = new PutObjectCommand({
-      Bucket: this.bucketName,
-      Key: key,
-      Body: bodyStream as any,
-      ContentEncoding: shouldCompress ? 'gzip' : undefined,
-      CacheControl: opts.metadata?.['Cache-control'],
-      Metadata: this.extractUserMetadata(opts.metadata),
-    });
-    await this.s3.send(put);
+    const bodyStream = createReadStream(bodyPath);
+    const contentType = this.detectContentType(key, shouldCompress);
+    try {
+      const put = new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: key,
+        Body: bodyStream as any,
+        ContentType: contentType,
+        CacheControl: opts.metadata?.['Cache-control'],
+        Metadata: this.extractUserMetadata(opts.metadata),
+        // Explicitly set ContentLength so R2 can compute x-amz-decoded-content-length correctly
+        ContentLength: contentLength,
+      } as any);
+      await this.s3.send(put);
+    } finally {
+      if (shouldCompress && bodyPath !== srcPath) {
+        await unlink(bodyPath).catch(() => undefined);
+      }
+    }
     return this.buildReturnedUrl(key, !!opts.public);
   }
 
@@ -125,6 +171,28 @@ export class S3FileStore implements FileStore {
     return Object.keys(rest).length ? rest : undefined;
   }
 
+  private detectContentType(key: string, isCompressed: boolean | undefined): string | undefined {
+    // Basic content type inference
+    const lower = key.toLowerCase();
+    if (lower.endsWith('.json') || lower.endsWith('.json.gz')) {
+      return 'application/json';
+    }
+    if (lower.endsWith('.txt') || lower.endsWith('.log') || lower.endsWith('.csv') || lower.endsWith('.md')) {
+      return 'text/plain; charset=utf-8';
+    }
+    if (lower.endsWith('.db') || lower.endsWith('.sqlite') || lower.endsWith('.bin')) {
+      return 'application/octet-stream';
+    }
+    if (lower.endsWith('.wasm') || lower.endsWith('.wasm.gz')) {
+      return 'application/wasm';
+    }
+    // If compressed, prefer octet-stream unless known
+    if (isCompressed) {
+      return 'application/octet-stream';
+    }
+    return undefined;
+  }
+
   private buildReturnedUrl(key: string, makePublic: boolean): string {
     if (!makePublic) {
       return `s3://${this.bucketName}/${key}`;

package/src/snapshots/download.ts

@@ -2,6 +2,12 @@ import { fromEntries, getEntries, maxBy } from '@aztec/foundation/collection';
 import { jsonParseWithSchema } from '@aztec/foundation/json-rpc';
 import type { ReadOnlyFileStore } from '@aztec/stdlib/file-store';
 
+import { createReadStream, createWriteStream } from 'fs';
+import fs from 'fs/promises';
+import pathMod from 'path';
+import { pipeline } from 'stream/promises';
+import { createGunzip, gunzipSync } from 'zlib';
+
 import {
   SnapshotDataKeys,
   type SnapshotDataUrls,
@@ -20,7 +26,8 @@ export async function getSnapshotIndex(
   try {
     if (await store.exists(snapshotIndexPath)) {
       const snapshotIndexData = await store.read(snapshotIndexPath);
-      return jsonParseWithSchema(snapshotIndexData.toString(), SnapshotsIndexSchema);
+      const buf = maybeGunzip(snapshotIndexData);
+      return jsonParseWithSchema(buf.toString('utf-8'), SnapshotsIndexSchema);
     } else {
       return undefined;
     }
@@ -50,10 +57,67 @@ export function makeSnapshotPaths(baseDir: string): SnapshotDataUrls {
   return fromEntries(SnapshotDataKeys.map(key => [key, `${baseDir}/${key}.db`]));
 }
 
+function isGzipMagic(data: Buffer): boolean {
+  return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+}
+
+function maybeGunzip(data: Buffer): Buffer {
+  const magicNumberIndicatesGzip = isGzipMagic(data);
+
+  if (magicNumberIndicatesGzip) {
+    try {
+      const out = gunzipSync(data);
+      return out;
+    } catch (err) {
+      throw new Error(`Decompression of gzipped data failed: ${(err as Error).message}`);
+    }
+  }
+  return data;
+}
+
+async function detectGzip(localFilePathToPeek: string): Promise<boolean> {
+  // Peek the actual bytes we downloaded.
+  try {
+    const fd = await fs.open(localFilePathToPeek, 'r');
+    try {
+      const header = Buffer.alloc(2);
+      const { bytesRead } = await fd.read(header, 0, 2, 0);
+      return bytesRead >= 2 && isGzipMagic(header);
+    } finally {
+      await fd.close();
+    }
+  } catch {
+    return false;
+  }
+}
+
 export async function downloadSnapshot(
   snapshot: Pick<SnapshotMetadata, 'dataUrls'>,
   localPaths: Record<SnapshotDataKeys, string>,
   store: ReadOnlyFileStore,
 ): Promise<void> {
-  await Promise.all(getEntries(localPaths).map(([key, path]) => store.download(snapshot.dataUrls[key], path)));
+  await Promise.all(
+    getEntries(localPaths).map(async ([key, path]) => {
+      await fs.mkdir(pathMod.dirname(path), { recursive: true });
+
+      const tmpPath = `${path}.download`;
+      try {
+        const url = snapshot.dataUrls[key];
+        await store.download(url, tmpPath);
+
+        const isGzip = await detectGzip(tmpPath);
+
+        const read = createReadStream(tmpPath);
+        const write = createWriteStream(path);
+        if (isGzip) {
+          const gunzip = createGunzip();
+          await pipeline(read, gunzip, write);
+        } else {
+          await pipeline(read, write);
+        }
+      } finally {
+        await fs.unlink(tmpPath).catch(() => undefined);
+      }
+    }),
+  );
 }

package/src/snapshots/upload.ts

@@ -48,6 +48,7 @@ export async function uploadSnapshotToIndex(
 
   await store.save(getSnapshotIndexPath(metadata), Buffer.from(jsonStringify(snapshotsIndex, true)), {
     public: true, // Make the index publicly accessible
+    compress: false, // Ensure index.json is not gzipped
     metadata: { ['Cache-control']: 'no-store' }, // Do not cache object versions
   });
   return newSnapshotMetadata;