deeplake 0.3.31 → 0.3.33

package/dist/node/node/storage.js

@@ -1,13 +1,13 @@
 "use strict";
 /**
- * Node.js S3 storage client factory for the WASM deeplake engine.
+ * Node.js storage client factory for the WASM deeplake engine.
  *
  * The C++ WASM code calls js_storage::client_factory()(type, config, mode?)
  * to create a JS client object with methods like download, exists, list, etc.
- * This module provides the factory that creates those client objects using
- * the AWS SDK v3.
+ * This module provides the factory that creates those client objects for
+ * S3 (AWS SDK v3), GCS (REST API), and Azure Blob Storage (REST API).
  *
- * Credential rotation: STS credentials are refreshed automatically before
+ * Credential rotation: credentials are refreshed automatically before
  * they expire (5 minutes before expiry, matching the native C++ behavior).
  * The C++ side passes refresh metadata (orgId, dsName, token) through the
  * config object so the JS side can call the backend API for fresh creds.
@@ -204,86 +204,42 @@ function releaseStorageClients() {
 }
 function createStorageFactory(wasmModule) {
     return function clientFactory(type, config, mode) {
-        if (type !== 'aws') {
-            // GCS and Azure storage types are supported in the browser visualizer
-            // (via GCSGetter/AzureGetter) but not yet in the Node.js SDK.
-            // The C++ side may request these types when datasets are backed by
-            // non-S3 storage. For now, throw a clear error.
-            throw new Error(`Unsupported storage type: ${type}. The Node.js SDK currently supports "aws" only. ` +
-                `GCS and Azure require additional dependencies (@azure/storage-blob, etc.).`);
-        }
-        const credentials = config?.credentials;
-        const region = config?.region ||
-            process.env.AWS_REGION ||
-            process.env.AWS_DEFAULT_REGION ||
-            'us-east-1';
-        const endpoint = config?.endpoint ?? undefined;
-        // Resolve credentials: prefer config from C++, fall back to process.env.
-        // In WASM, std::getenv() doesn't read Node.js process.env, so the C++
-        // credential chain can't pick up env vars — we must do it here.
-        let resolvedCreds;
-        if (credentials?.accessKeyId && credentials?.secretAccessKey) {
-            resolvedCreds = {
-                accessKeyId: credentials.accessKeyId,
-                secretAccessKey: credentials.secretAccessKey,
-                sessionToken: credentials.sessionToken,
-            };
-        }
-        else if (process.env.AWS_ACCESS_KEY_ID &&
-            process.env.AWS_SECRET_ACCESS_KEY) {
-            resolvedCreds = {
-                accessKeyId: process.env.AWS_ACCESS_KEY_ID,
-                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
-                sessionToken: process.env.AWS_SESSION_TOKEN,
-            };
-        }
-        // Build refresh metadata from C++ params (org_id, ds_name, token)
-        const refresh = config?.refresh;
-        let refreshMeta = null;
-        if (refresh?.orgId && refresh?.token && (refresh?.dsName || refresh?.credsKey)) {
-            refreshMeta = {
-                orgId: refresh.orgId,
-                dsName: refresh.dsName,
-                credsKey: refresh.credsKey,
-                token: refresh.token,
-                apiEndpoint: resolveApiEndpoint(wasmModule),
-            };
-        }
-        const s3Config = {
-            region,
-            credentials: resolvedCreds,
-            // Match C++ native client: 8s request/connect timeouts (s3_utils.hpp:313-314)
-            requestHandler: new (require('@smithy/node-http-handler').NodeHttpHandler)({
-                requestTimeout: 8_000,
-                connectionTimeout: 8_000,
-            }),
-            // Match C++ native client: custom retry (s3_retry_strategy.hpp)
-            // 200ms interval, ~6s max → roughly 30 attempts
-            maxAttempts: 30,
-            // Disable IMDS lookups to avoid 5-10s hangs when no instance role
-            // is available (matches C++ shouldDisableIMDS=true)
-            disableHostPrefix: false,
-            followRegionRedirects: true,
-        };
-        if (endpoint) {
-            s3Config.endpoint = endpoint;
-            // Match C++ native client: use path-style addressing for custom
-            // endpoints (s3_utils.hpp:329 — use_virtual_addressing=false)
-            s3Config.forcePathStyle = true;
-        }
         const dbg = !!process.env.DEEPLAKE_STORAGE_DEBUG;
-        if (dbg) {
-            console.log(`[storage] clientFactory type=${type} mode=${mode ?? 'r'} region=${region} creds=${resolvedCreds ? 'yes' : 'no'} refresh=${refreshMeta ? 'yes' : 'no'}`);
-        }
-        const rotatingClient = new RotatingS3Client(s3Config, region, endpoint, refreshMeta, dbg);
         let result;
-        if (mode === 'w') {
-            result = createWriter(rotatingClient, wasmModule);
+        let destroyable = null;
+        if (type === 'aws') {
+            const built = buildS3Client(config, wasmModule, dbg);
+            destroyable = built.rotatingClient;
+            if (mode === 'w') {
+                result = createS3Writer(built.rotatingClient, wasmModule);
+            }
+            else {
+                result = createS3Getter(built.rotatingClient, wasmModule);
+            }
+        }
+        else if (type === 'gcs') {
+            const rotatingCreds = buildGcsCredentials(config, wasmModule, dbg);
+            if (mode === 'w') {
+                result = createGcsWriter(rotatingCreds, wasmModule);
+            }
+            else {
+                result = createGcsGetter(rotatingCreds, wasmModule);
+            }
+        }
+        else if (type === 'azure') {
+            const rotatingCreds = buildAzureCredentials(config, wasmModule, dbg);
+            if (mode === 'w') {
+                result = createAzureWriter(rotatingCreds, wasmModule);
+            }
+            else {
+                result = createAzureGetter(rotatingCreds, wasmModule);
+            }
         }
         else {
-            result = createGetter(rotatingClient, wasmModule);
+            throw new Error(`Unsupported storage type: ${type}. Supported types: "aws", "gcs", "azure".`);
         }
         if (dbg) {
+            console.log(`[storage] clientFactory type=${type} mode=${mode ?? 'r'}`);
             // Wrap in Proxy to trace ALL method accesses and calls
             const label = mode === 'w' ? 'writer' : 'reader';
             result = new Proxy(result, {
@@ -291,13 +247,13 @@ function createStorageFactory(wasmModule) {
                     const val = target[prop];
                     if (typeof val === 'function') {
                         return function (...args) {
-                            console.log(`[storage:${label}:${prop}] called with ${args.length} args`);
+                            console.log(`[storage:${type}:${label}:${prop}] called with ${args.length} args`);
                             try {
                                 const r = val.apply(target, args);
                                 return r;
                             }
                             catch (e) {
-                                console.log(`[storage:${label}:${prop}] threw: ${e.message}`);
+                                console.log(`[storage:${type}:${label}:${prop}] threw: ${e.message}`);
                                 throw e;
                             }
                         };
@@ -313,14 +269,76 @@ function createStorageFactory(wasmModule) {
         // when the C++ dataset handle is freed.
         result._destroy = () => {
             _liveObjects.delete(result);
-            if (rotatingClient && typeof rotatingClient.destroy === 'function') {
-                rotatingClient.destroy();
+            if (destroyable && typeof destroyable.destroy === 'function') {
+                destroyable.destroy();
             }
         };
         return result;
     };
 }
-function removeScheme(key) {
+// ---------------------------------------------------------------------------
+// S3 client builder
+// ---------------------------------------------------------------------------
+function buildS3Client(config, wasmModule, dbg) {
+    const credentials = config?.credentials;
+    const region = config?.region ||
+        process.env.AWS_REGION ||
+        process.env.AWS_DEFAULT_REGION ||
+        'us-east-1';
+    const endpoint = config?.endpoint ?? undefined;
+    // Resolve credentials: prefer config from C++, fall back to process.env.
+    let resolvedCreds;
+    if (credentials?.accessKeyId && credentials?.secretAccessKey) {
+        resolvedCreds = {
+            accessKeyId: credentials.accessKeyId,
+            secretAccessKey: credentials.secretAccessKey,
+            sessionToken: credentials.sessionToken,
+        };
+    }
+    else if (process.env.AWS_ACCESS_KEY_ID &&
+        process.env.AWS_SECRET_ACCESS_KEY) {
+        resolvedCreds = {
+            accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+            secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+            sessionToken: process.env.AWS_SESSION_TOKEN,
+        };
+    }
+    const refresh = config?.refresh;
+    let refreshMeta = null;
+    if (refresh?.orgId && refresh?.token && (refresh?.dsName || refresh?.credsKey)) {
+        refreshMeta = {
+            orgId: refresh.orgId,
+            dsName: refresh.dsName,
+            credsKey: refresh.credsKey,
+            token: refresh.token,
+            apiEndpoint: resolveApiEndpoint(wasmModule),
+        };
+    }
+    const s3Config = {
+        region,
+        credentials: resolvedCreds,
+        requestHandler: new (require('@smithy/node-http-handler').NodeHttpHandler)({
+            requestTimeout: 8_000,
+            connectionTimeout: 8_000,
+        }),
+        maxAttempts: 30,
+        disableHostPrefix: false,
+        followRegionRedirects: true,
+    };
+    if (endpoint) {
+        s3Config.endpoint = endpoint;
+        s3Config.forcePathStyle = true;
+    }
+    if (dbg) {
+        console.log(`[storage:s3] region=${region} creds=${resolvedCreds ? 'yes' : 'no'} refresh=${refreshMeta ? 'yes' : 'no'}`);
+    }
+    const rotatingClient = new RotatingS3Client(s3Config, region, endpoint, refreshMeta, dbg);
+    return { rotatingClient };
+}
+// ---------------------------------------------------------------------------
+// S3 helpers
+// ---------------------------------------------------------------------------
+function removeS3Scheme(key) {
     if (key.startsWith('s3://')) {
         return key.substring(5);
     }
@@ -335,11 +353,11 @@ function splitBucketPrefix(root) {
     const epos = root.lastIndexOf('/');
     return [root.substring(0, pos), root.substring(pos + 1, epos)];
 }
-function createGetter(rotatingClient, wasmModule) {
+function createS3Getter(rotatingClient, wasmModule) {
     const debug = !!process.env.DEEPLAKE_STORAGE_DEBUG;
     const getter = {
         generateGetPresignedUrl(key, completion) {
-            key = removeScheme(key);
+            key = removeS3Scheme(key);
             const c1 = (0, inflight_tracker_1.trackClone)(completion);
             const [bucket, objKey] = splitBucketKey(key);
             rotatingClient.getClient().then((client) => (0, s3_request_presigner_1.getSignedUrl)(client, new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: objKey }), {
@@ -355,7 +373,7 @@ function createGetter(rotatingClient, wasmModule) {
             }));
         },
         exists(key, completion) {
-            key = removeScheme(key);
+            key = removeS3Scheme(key);
             const c1 = (0, inflight_tracker_1.trackClone)(completion);
             const [bucket, objKey] = splitBucketKey(key);
             if (debug) {
@@ -373,7 +391,7 @@ function createGetter(rotatingClient, wasmModule) {
             }));
         },
         download(key, range, index, completion) {
-            key = removeScheme(key);
+            key = removeS3Scheme(key);
             const c1 = (0, inflight_tracker_1.trackClone)(completion);
             const [bucket, objKey] = splitBucketKey(key);
             if (debug) {
@@ -529,11 +547,11 @@ function createGetter(rotatingClient, wasmModule) {
     };
     return getter;
 }
-function createWriter(rotatingClient, wasmModule) {
+function createS3Writer(rotatingClient, wasmModule) {
     const debug = !!process.env.DEEPLAKE_STORAGE_DEBUG;
     const writer = {
         saveWithCompletion(key, content, completion) {
-            key = removeScheme(key);
+            key = removeS3Scheme(key);
             const c1 = (0, inflight_tracker_1.trackClone)(completion);
             const [bucket, objKey] = splitBucketKey(key);
             if (debug) {
@@ -544,11 +562,13 @@ function createWriter(rotatingClient, wasmModule) {
             rotatingClient.getClient().then((client) => client
                 .send(new client_s3_1.PutObjectCommand({ Bucket: bucket, Key: objKey, Body: body }))
                 .then((result) => {
+                const etag = result.ETag ? result.ETag.replace(/"/g, '') : '';
                 if (debug) {
-                    console.log(`[storage:write:done] key=${objKey} code=${result.$metadata.httpStatusCode}`);
+                    console.log(`[storage:write:done] key=${objKey} code=${result.$metadata.httpStatusCode} etag=${etag}`);
                 }
                 c1.call({
                     responseCode: result.$metadata.httpStatusCode ?? 200,
+                    etag,
                 });
                 c1.delete();
             })
@@ -563,8 +583,85 @@ function createWriter(rotatingClient, wasmModule) {
                 c1.delete();
             }));
         },
+        /**
+         * Conditional write: only succeeds if the object's current ETag matches.
+         * Used by CAS counter and commit operations for optimistic concurrency.
+         * Returns 412 (Precondition Failed) if ETag doesn't match.
+         */
+        saveConditional(key, content, ifMatchEtag, completion) {
+            key = removeS3Scheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const [bucket, objKey] = splitBucketKey(key);
+            const body = typeof content === 'string' ? Buffer.from(content) : content;
+            const quotedEtag = ifMatchEtag.startsWith('"') ? ifMatchEtag : `"${ifMatchEtag}"`;
+            if (debug) {
+                const size = typeof content === 'string' ? content.length : content.byteLength;
+                console.log(`[storage:write_if_match] bucket=${bucket} key=${objKey} size=${size} ifMatch=${quotedEtag}`);
+            }
+            rotatingClient.getClient().then((client) => client
+                .send(new client_s3_1.PutObjectCommand({ Bucket: bucket, Key: objKey, Body: body, IfMatch: quotedEtag }))
+                .then((result) => {
+                const etag = result.ETag ? result.ETag.replace(/"/g, '') : '';
+                if (debug) {
+                    console.log(`[storage:write_if_match:done] key=${objKey} code=${result.$metadata.httpStatusCode} etag=${etag}`);
+                }
+                c1.call({
+                    responseCode: result.$metadata.httpStatusCode ?? 200,
+                    etag,
+                });
+                c1.delete();
+            })
+                .catch((err) => {
+                const code = err.$metadata?.httpStatusCode ?? 1000;
+                const name = err.name ?? err.Code ?? '';
+                const msg = name ? `${name}: ${err.message ?? 'unknown_error'}` : (err.message ?? 'unknown_error');
+                if (debug) {
+                    console.log(`[storage:write_if_match:error] key=${objKey} code=${code} err=${msg}`);
+                }
+                c1.call({ responseCode: code, message: msg });
+                c1.delete();
+            }));
+        },
+        /**
+         * Exclusive write: only succeeds if the object does NOT exist.
+         * Used to prevent concurrent creation of the same file.
+         * Returns 412 (Precondition Failed) if the object already exists.
+         */
+        saveExclusive(key, content, completion) {
+            key = removeS3Scheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const [bucket, objKey] = splitBucketKey(key);
+            const body = typeof content === 'string' ? Buffer.from(content) : content;
+            if (debug) {
+                const size = typeof content === 'string' ? content.length : content.byteLength;
+                console.log(`[storage:write_exclusive] bucket=${bucket} key=${objKey} size=${size}`);
+            }
+            rotatingClient.getClient().then((client) => client
+                .send(new client_s3_1.PutObjectCommand({ Bucket: bucket, Key: objKey, Body: body, IfNoneMatch: '*' }))
+                .then((result) => {
+                const etag = result.ETag ? result.ETag.replace(/"/g, '') : '';
+                if (debug) {
+                    console.log(`[storage:write_exclusive:done] key=${objKey} code=${result.$metadata.httpStatusCode} etag=${etag}`);
+                }
+                c1.call({
+                    responseCode: result.$metadata.httpStatusCode ?? 200,
+                    etag,
+                });
+                c1.delete();
+            })
+                .catch((err) => {
+                const code = err.$metadata?.httpStatusCode ?? 1000;
+                const name = err.name ?? err.Code ?? '';
+                const msg = name ? `${name}: ${err.message ?? 'unknown_error'}` : (err.message ?? 'unknown_error');
+                if (debug) {
+                    console.log(`[storage:write_exclusive:error] key=${objKey} code=${code} err=${msg}`);
+                }
+                c1.call({ responseCode: code, message: msg });
+                c1.delete();
+            }));
+        },
         remove(key, completion) {
-            key = removeScheme(key);
+            key = removeS3Scheme(key);
             const c1 = (0, inflight_tracker_1.trackClone)(completion);
             const [bucket, objKey] = splitBucketKey(key);
             rotatingClient.getClient().then((client) => client
@@ -589,4 +686,829 @@ function createWriter(rotatingClient, wasmModule) {
     };
     return writer;
 }
+// ---------------------------------------------------------------------------
+// Rotating credentials for GCS and Azure
+// ---------------------------------------------------------------------------
+/**
+ * Generic rotating credential holder.  Works like RotatingS3Client but
+ * stores plain credential objects instead of SDK client instances.
+ */
+class RotatingCredentials {
+    creds;
+    refreshMeta;
+    applyCreds;
+    createdAt;
+    lastFailureAt = 0;
+    refreshInProgress = null;
+    debug;
+    constructor(initial, refreshMeta, applyCreds, debug) {
+        this.creds = initial;
+        this.refreshMeta = refreshMeta;
+        this.applyCreds = applyCreds;
+        this.createdAt = Date.now();
+        this.debug = debug;
+    }
+    async get() {
+        if (!this.refreshMeta)
+            return this.creds;
+        const now = Date.now();
+        const age = now - this.createdAt;
+        if (age <= DEFAULT_CRED_LIFETIME_MS - REFRESH_BUFFER_MS)
+            return this.creds;
+        if (this.lastFailureAt && now - this.lastFailureAt < REFRESH_FAILURE_COOLDOWN_MS) {
+            return this.creds;
+        }
+        if (this.refreshInProgress) {
+            await this.refreshInProgress;
+            return this.creds;
+        }
+        this.refreshInProgress = this.doRefresh();
+        try {
+            await this.refreshInProgress;
+        }
+        finally {
+            this.refreshInProgress = null;
+        }
+        return this.creds;
+    }
+    async doRefresh() {
+        const meta = this.refreshMeta;
+        if (this.debug) {
+            console.log(`[storage] refreshing credentials for ds=${meta.dsName ?? ''} credsKey=${meta.credsKey ?? ''}`);
+        }
+        try {
+            const resp = await fetchCredentials(meta);
+            this.creds = this.applyCreds(resp.creds);
+            this.createdAt = Date.now();
+            this.lastFailureAt = 0;
+            if (this.debug) {
+                console.log(`[storage] credential refresh succeeded (repo_type=${resp.repoType})`);
+            }
+        }
+        catch (e) {
+            this.lastFailureAt = Date.now();
+            console.warn(`[storage] credential refresh failed: ${e.message ?? e}`);
+        }
+    }
+}
+function buildRefreshMeta(config, wasmModule) {
+    const refresh = config?.refresh;
+    if (refresh?.orgId && refresh?.token && (refresh?.dsName || refresh?.credsKey)) {
+        return {
+            orgId: refresh.orgId,
+            dsName: refresh.dsName,
+            credsKey: refresh.credsKey,
+            token: refresh.token,
+            apiEndpoint: resolveApiEndpoint(wasmModule),
+        };
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// GCS (Google Cloud Storage) — fetch-based, no extra dependencies
+// ---------------------------------------------------------------------------
+/** Convert string or Uint8Array to a value accepted by Node.js fetch as body. */
+function toFetchBody(content) {
+    if (typeof content === 'string')
+        return Buffer.from(content);
+    return Buffer.isBuffer(content) ? content : Buffer.from(content.buffer, content.byteOffset, content.byteLength);
+}
+const GCS_API_BASE = 'https://storage.googleapis.com';
+function buildGcsCredentials(config, wasmModule, dbg) {
+    const initial = { accessToken: config?.accessToken ?? '' };
+    const refreshMeta = buildRefreshMeta(config, wasmModule);
+    if (dbg) {
+        console.log(`[storage:gcs] token=${initial.accessToken ? 'yes' : 'no'} refresh=${refreshMeta ? 'yes' : 'no'}`);
+    }
+    return new RotatingCredentials(initial, refreshMeta, (raw) => ({
+        accessToken: raw.gcs_oauth_token ?? raw.oauth_token ?? raw.access_token ?? '',
+    }), dbg);
+}
+function gcsHeaders(creds) {
+    const h = {};
+    if (creds.accessToken) {
+        h['Authorization'] = `Bearer ${creds.accessToken}`;
+    }
+    return h;
+}
+function removeGcsScheme(key) {
+    if (key.startsWith('gs://'))
+        return key.substring(5);
+    if (key.startsWith('gcs://'))
+        return key.substring(6);
+    return key;
+}
+function createGcsGetter(rotatingCreds, wasmModule) {
+    const getter = {
+        generateGetPresignedUrl(_key, completion) {
+            // GCS presigned URLs require service account private keys, which
+            // aren't available here.  Return empty like the browser implementation.
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            c1.call('');
+            c1.delete();
+        },
+        exists(key, completion) {
+            key = removeGcsScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                fetch(`${GCS_API_BASE}/${key}`, {
+                    method: 'HEAD',
+                    headers: gcsHeaders(creds),
+                })
+                    .then((resp) => {
+                    if (resp.ok) {
+                        c1.call(Number(resp.headers.get('content-length') ?? 0));
+                    }
+                    else {
+                        c1.call(-1);
+                    }
+                    c1.delete();
+                })
+                    .catch(() => {
+                    c1.call(-1);
+                    c1.delete();
+                });
+            });
+        },
+        download(key, range, index, completion) {
+            key = removeGcsScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                const headers = gcsHeaders(creds);
+                if (range)
+                    headers['Range'] = range;
+                const controller = new AbortController();
+                wasmModule.storeJsObject(index, { abort() { controller.abort(); } });
+                fetch(`${GCS_API_BASE}/${key}`, { headers, signal: controller.signal })
+                    .then(async (resp) => {
+                    if (!resp.ok) {
+                        c1.call({ responseCode: resp.status, message: resp.statusText || 'download_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    const buf = Buffer.from(await resp.arrayBuffer());
+                    const view = wasmModule.provideResponseBuffer(index, buf.length);
+                    view.set(buf);
+                    c1.call({ responseCode: 200 });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        list(root, key, startAfter, completion) {
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const [bucket, prefix] = splitBucketKey(root);
+            const finalPrefix = (prefix.endsWith('/') ? prefix : `${prefix}/`) + key;
+            const results = [];
+            rotatingCreds.get().then((creds) => {
+                const impl = (pageToken) => {
+                    let url = `${GCS_API_BASE}/storage/v1/b/${bucket}/o?prefix=${encodeURIComponent(finalPrefix)}`;
+                    if (startAfter)
+                        url += `&startOffset=${encodeURIComponent(startAfter)}`;
+                    if (pageToken)
+                        url += `&pageToken=${encodeURIComponent(pageToken)}`;
+                    fetch(url, { headers: gcsHeaders(creds) })
+                        .then((resp) => {
+                        if (!resp.ok) {
+                            c1.call({ responseCode: resp.status, message: 'list_request_failed' });
+                            c1.delete();
+                            return;
+                        }
+                        return resp.json();
+                    })
+                        .then((json) => {
+                        if (!json)
+                            return; // error already handled
+                        for (const item of json.items ?? []) {
+                            results.push({
+                                path: item.name.substring(prefix.length + 1),
+                                size: Number(item.size),
+                                lastModified: new Date(item.updated),
+                                etag: item.etag ?? '',
+                            });
+                        }
+                        if (json.nextPageToken) {
+                            impl(json.nextPageToken);
+                        }
+                        else {
+                            c1.call({ responseCode: 200, data: results });
+                            c1.delete();
+                        }
+                    })
+                        .catch((err) => {
+                        c1.call({ responseCode: 1000, message: err.message ?? 'list_request_failed' });
+                        c1.delete();
+                    });
+                };
+                impl();
+            });
+        },
+        listDirs(root, prefix, completion) {
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const [bucket, rootPrefix] = splitBucketPrefix(root);
+            const finalPrefix = (rootPrefix.endsWith('/') ? rootPrefix : `${rootPrefix}/`) + prefix;
+            rotatingCreds.get().then((creds) => {
+                const url = `${GCS_API_BASE}/storage/v1/b/${bucket}/o?prefix=${encodeURIComponent(finalPrefix)}&delimiter=/`;
+                fetch(url, { headers: gcsHeaders(creds) })
+                    .then((resp) => {
+                    if (!resp.ok) {
+                        c1.call({ responseCode: resp.status, message: 'list_dirs_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    return resp.json();
+                })
+                    .then((json) => {
+                    if (!json)
+                        return;
+                    const dirs = (json.prefixes ?? []);
+                    c1.call({ responseCode: 200, data: dirs });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'list_dirs_failed' });
+                    c1.delete();
+                });
+            });
+        },
+        metadata(root, key, completion) {
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const [bucket, prefix] = splitBucketPrefix(root);
+            const objName = (prefix.endsWith('/') ? prefix : `${prefix}/`) + key;
+            rotatingCreds.get().then((creds) => {
+                const url = `${GCS_API_BASE}/storage/v1/b/${bucket}/o/${encodeURIComponent(objName)}`;
+                fetch(url, { headers: gcsHeaders(creds) })
+                    .then((resp) => {
+                    if (!resp.ok) {
+                        c1.call({ responseCode: resp.status, message: 'metadata_request_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    return resp.json();
+                })
+                    .then((json) => {
+                    if (!json)
+                        return;
+                    c1.call({
+                        responseCode: 200,
+                        data: {
+                            path: json.name.substring(prefix.length + 1),
+                            size: Number(json.size),
+                            lastModified: new Date(json.updated),
+                            etag: json.etag ?? '',
+                        },
+                    });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'metadata_request_failed' });
+                    c1.delete();
+                });
+            });
+        },
+        copyGetter() {
+            return getter;
+        },
+    };
+    return getter;
+}
+function createGcsWriter(rotatingCreds, _wasmModule) {
+    const writer = {
+        saveWithCompletion(key, content, completion) {
+            key = removeGcsScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const body = toFetchBody(content);
+            rotatingCreds.get().then((creds) => {
+                // Upload via GCS JSON API resumable or simple upload
+                const [bucket, objKey] = splitBucketKey(key);
+                const url = `${GCS_API_BASE}/upload/storage/v1/b/${bucket}/o?uploadType=media&name=${encodeURIComponent(objKey)}`;
+                fetch(url, {
+                    method: 'POST',
+                    headers: { ...gcsHeaders(creds), 'Content-Type': 'application/octet-stream' },
+                    body,
+                })
+                    .then(async (resp) => {
+                    if (!resp.ok) {
+                        const text = await resp.text().catch(() => '');
+                        c1.call({ responseCode: resp.status, message: text || 'upload_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    const json = await resp.json().catch(() => ({}));
+                    c1.call({ responseCode: 200, etag: json.etag ?? '' });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        saveConditional(key, content, ifMatchEtag, completion) {
+            key = removeGcsScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const body = toFetchBody(content);
+            const [bucket, objKey] = splitBucketKey(key);
+            const normalizedEtag = ifMatchEtag.replace(/"/g, '');
+            rotatingCreds.get().then((creds) => {
+                // First, fetch the object's metadata to get its generation number.
+                // GCS CAS requires ifGenerationMatch=<generation>, not HTTP If-Match.
+                const metaUrl = `${GCS_API_BASE}/storage/v1/b/${bucket}/o/${encodeURIComponent(objKey)}`;
+                fetch(metaUrl, { headers: gcsHeaders(creds) })
+                    .then((resp) => {
+                    if (!resp.ok) {
+                        // Object doesn't exist or other error — CAS fails
+                        c1.call({ responseCode: resp.status === 404 ? 412 : resp.status, message: 'conditional_write_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    return resp.json();
+                })
+                    .then((meta) => {
+                    if (!meta)
+                        return; // error already handled
+                    const currentEtag = (meta.etag ?? '').replace(/"/g, '');
+                    if (currentEtag !== normalizedEtag) {
+                        // ETag mismatch — another writer changed the object
+                        c1.call({ responseCode: 412, message: 'conditional_write_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    // ETag matches — upload with ifGenerationMatch to guarantee atomicity
+                    const uploadUrl = `${GCS_API_BASE}/upload/storage/v1/b/${bucket}/o?uploadType=media&name=${encodeURIComponent(objKey)}&ifGenerationMatch=${meta.generation}`;
+                    fetch(uploadUrl, {
+                        method: 'POST',
+                        headers: {
+                            ...gcsHeaders(creds),
+                            'Content-Type': 'application/octet-stream',
+                        },
+                        body,
+                    })
+                        .then(async (resp) => {
+                        if (!resp.ok) {
+                            c1.call({ responseCode: resp.status === 412 ? 412 : resp.status, message: 'conditional_write_failed' });
+                            c1.delete();
+                            return;
+                        }
+                        const json = await resp.json().catch(() => ({}));
+                        c1.call({ responseCode: 200, etag: json.etag ?? '' });
+                        c1.delete();
+                    })
+                        .catch((err) => {
+                        c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                        c1.delete();
+                    });
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        saveExclusive(key, content, completion) {
+            key = removeGcsScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const body = toFetchBody(content);
+            const [bucket, objKey] = splitBucketKey(key);
+            rotatingCreds.get().then((creds) => {
+                // ifGenerationMatch=0 means "only if the object does not exist"
+                const url = `${GCS_API_BASE}/upload/storage/v1/b/${bucket}/o?uploadType=media&name=${encodeURIComponent(objKey)}&ifGenerationMatch=0`;
+                fetch(url, {
+                    method: 'POST',
+                    headers: { ...gcsHeaders(creds), 'Content-Type': 'application/octet-stream' },
+                    body,
+                })
+                    .then(async (resp) => {
+                    if (!resp.ok) {
+                        c1.call({ responseCode: resp.status === 412 ? 412 : resp.status, message: 'exclusive_write_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    const json = await resp.json().catch(() => ({}));
+                    c1.call({ responseCode: 200, etag: json.etag ?? '' });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        remove(key, completion) {
+            key = removeGcsScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const [bucket, objKey] = splitBucketKey(key);
+            rotatingCreds.get().then((creds) => {
+                const url = `${GCS_API_BASE}/storage/v1/b/${bucket}/o/${encodeURIComponent(objKey)}`;
+                fetch(url, {
+                    method: 'DELETE',
+                    headers: gcsHeaders(creds),
+                })
+                    .then((resp) => {
+                    c1.call({ responseCode: resp.ok ? 200 : resp.status });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        copyWriter() {
+            return writer;
+        },
+    };
+    return writer;
+}
+function buildAzureCredentials(config, wasmModule, dbg) {
+    const initial = {
+        accountName: config?.accountName ?? '',
+        sasToken: config?.blobSasToken ?? '',
+        containerName: config?.containerName ?? '',
+    };
+    const refreshMeta = buildRefreshMeta(config, wasmModule);
+    if (dbg) {
+        console.log(`[storage:azure] account=${initial.accountName} container=${initial.containerName} sas=${initial.sasToken ? 'yes' : 'no'} refresh=${refreshMeta ? 'yes' : 'no'}`);
+    }
+    return new RotatingCredentials(initial, refreshMeta, (raw) => ({
+        accountName: raw.account_name ?? initial.accountName,
+        sasToken: raw.sas_token ?? '',
+        containerName: raw.container_name ?? initial.containerName,
+    }), dbg);
+}
+function azureBlobUrl(creds, blobPath) {
+    const base = `https://${creds.accountName}.blob.core.windows.net/${creds.containerName}/${blobPath}`;
+    return creds.sasToken ? `${base}?${creds.sasToken}` : base;
+}
+function azureListUrl(creds, params) {
+    const base = `https://${creds.accountName}.blob.core.windows.net/${creds.containerName}`;
+    const sep = creds.sasToken ? `?${creds.sasToken}&` : '?';
+    return `${base}${sep}${params}`;
+}
+function removeAzureScheme(key) {
+    if (key.startsWith('azure://'))
+        return key.substring(8);
+    if (key.startsWith('az://'))
+        return key.substring(5);
+    return key;
+}
+/**
+ * Parse an Azure blob path.  The C++ side sends paths as:
+ *   accountName/containerName/blob/path
+ * We need to extract just the blob path portion since accountName
+ * and containerName come from the credentials.
+ */
+function azureBlobPath(key, creds) {
+    const prefix = `${creds.accountName}/${creds.containerName}/`;
+    if (key.startsWith(prefix)) {
+        return key.substring(prefix.length);
+    }
+    // May already be just the blob path
+    return key;
+}
+/** Parse a minimal XML list-blobs response */
+function parseAzureListBlobsXml(xml) {
+    const blobs = [];
+    const blobRegex = /<Blob>[\s\S]*?<\/Blob>/g;
+    let match;
+    while ((match = blobRegex.exec(xml)) !== null) {
+        const blob = match[0];
+        const name = blob.match(/<Name>([\s\S]*?)<\/Name>/)?.[1] ?? '';
+        const size = Number(blob.match(/<Content-Length>([\s\S]*?)<\/Content-Length>/)?.[1] ?? 0);
+        const lastMod = blob.match(/<Last-Modified>([\s\S]*?)<\/Last-Modified>/)?.[1] ?? '';
+        const etag = (blob.match(/<Etag>([\s\S]*?)<\/Etag>/)?.[1] ?? '').replace(/"/g, '');
+        blobs.push({ name, size, lastModified: new Date(lastMod), etag });
+    }
+    const nextMarker = xml.match(/<NextMarker>([\s\S]*?)<\/NextMarker>/)?.[1] || undefined;
+    return { blobs, nextMarker };
+}
+/** Parse directory prefixes from Azure list-blobs XML with delimiter */
+function parseAzureBlobPrefixes(xml) {
+    const prefixes = [];
+    const prefixRegex = /<BlobPrefix><Name>([\s\S]*?)<\/Name><\/BlobPrefix>/g;
+    let match;
+    while ((match = prefixRegex.exec(xml)) !== null) {
+        prefixes.push(match[1]);
+    }
+    return prefixes;
+}
+function createAzureGetter(rotatingCreds, wasmModule) {
+    const getter = {
+        generateGetPresignedUrl(key, completion) {
+            // With SAS token, the URL is already "presigned"
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                key = removeAzureScheme(key);
+                const blobP = azureBlobPath(key, creds);
+                c1.call(azureBlobUrl(creds, blobP));
+                c1.delete();
+            });
+        },
+        exists(key, completion) {
+            key = removeAzureScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                const blobP = azureBlobPath(key, creds);
+                fetch(azureBlobUrl(creds, blobP), { method: 'HEAD' })
+                    .then((resp) => {
+                    if (resp.ok) {
+                        c1.call(Number(resp.headers.get('content-length') ?? 0));
+                    }
+                    else {
+                        c1.call(-1);
+                    }
+                    c1.delete();
+                })
+                    .catch(() => {
+                    c1.call(-1);
+                    c1.delete();
+                });
+            });
+        },
+        download(key, range, index, completion) {
+            key = removeAzureScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                const blobP = azureBlobPath(key, creds);
+                const headers = {};
+                if (range)
+                    headers['Range'] = range;
+                // Azure requires this header for range requests
+                if (range)
+                    headers['x-ms-range'] = range;
+                const controller = new AbortController();
+                wasmModule.storeJsObject(index, { abort() { controller.abort(); } });
+                fetch(azureBlobUrl(creds, blobP), { headers, signal: controller.signal })
+                    .then(async (resp) => {
+                    if (!resp.ok) {
+                        c1.call({ responseCode: resp.status, message: resp.statusText || 'download_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    const buf = Buffer.from(await resp.arrayBuffer());
+                    const view = wasmModule.provideResponseBuffer(index, buf.length);
+                    view.set(buf);
+                    c1.call({ responseCode: 200 });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    if (err.name === 'AbortError') {
+                        c1.call({ responseCode: 1000, message: 'aborted' });
+                    }
+                    else {
+                        c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    }
+                    c1.delete();
+                });
+            });
+        },
+        list(root, key, _startAfter, completion) {
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const results = [];
+            rotatingCreds.get().then((creds) => {
+                // root comes as accountName/containerName/prefix/
+                // We need to extract the prefix portion
+                const rootClean = removeAzureScheme(root);
+                const prefixPart = azureBlobPath(rootClean, creds);
+                const finalPrefix = (prefixPart.endsWith('/') ? prefixPart : `${prefixPart}/`) + key;
+                const impl = (marker) => {
+                    let params = `restype=container&comp=list&prefix=${encodeURIComponent(finalPrefix)}`;
+                    if (marker)
+                        params += `&marker=${encodeURIComponent(marker)}`;
+                    fetch(azureListUrl(creds, params))
+                        .then((resp) => {
+                        if (!resp.ok) {
+                            c1.call({ responseCode: resp.status, message: 'list_request_failed' });
+                            c1.delete();
+                            return;
+                        }
+                        return resp.text();
+                    })
+                        .then((xml) => {
+                        if (!xml)
+                            return;
+                        const parsed = parseAzureListBlobsXml(xml);
+                        for (const blob of parsed.blobs) {
+                            results.push({
+                                path: blob.name.substring(prefixPart.length + 1),
+                                size: blob.size,
+                                lastModified: blob.lastModified,
+                                etag: blob.etag,
+                            });
+                        }
+                        if (parsed.nextMarker) {
+                            impl(parsed.nextMarker);
+                        }
+                        else {
+                            c1.call({ responseCode: 200, data: results });
+                            c1.delete();
+                        }
+                    })
+                        .catch((err) => {
+                        c1.call({ responseCode: 1000, message: err.message ?? 'list_request_failed' });
+                        c1.delete();
+                    });
+                };
+                impl();
+            });
+        },
+        listDirs(root, prefix, completion) {
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                const rootClean = removeAzureScheme(root);
+                const rootPrefix = azureBlobPath(rootClean, creds);
+                const finalPrefix = (rootPrefix.endsWith('/') ? rootPrefix : `${rootPrefix}/`) + prefix;
+                let params = `restype=container&comp=list&prefix=${encodeURIComponent(finalPrefix)}&delimiter=/`;
+                fetch(azureListUrl(creds, params))
+                    .then((resp) => {
+                    if (!resp.ok) {
+                        c1.call({ responseCode: resp.status, message: 'list_dirs_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    return resp.text();
+                })
+                    .then((xml) => {
+                    if (!xml)
+                        return;
+                    const dirs = parseAzureBlobPrefixes(xml);
+                    c1.call({ responseCode: 200, data: dirs });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'list_dirs_failed' });
+                    c1.delete();
+                });
+            });
+        },
+        metadata(root, key, completion) {
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                const rootClean = removeAzureScheme(root);
+                const rootPrefix = azureBlobPath(rootClean, creds);
+                const blobName = (rootPrefix.endsWith('/') ? rootPrefix : `${rootPrefix}/`) + key;
+                // Use a list call with exact prefix to get metadata
+                let params = `restype=container&comp=list&prefix=${encodeURIComponent(blobName)}`;
+                fetch(azureListUrl(creds, params))
+                    .then((resp) => {
+                    if (!resp.ok) {
+                        c1.call({ responseCode: resp.status, message: 'metadata_request_failed' });
+                        c1.delete();
+                        return;
+                    }
+                    return resp.text();
+                })
+                    .then((xml) => {
+                    if (!xml)
+                        return;
+                    const parsed = parseAzureListBlobsXml(xml);
+                    if (parsed.blobs.length === 0) {
+                        c1.call({ responseCode: 404, message: 'resource_not_found' });
+                        c1.delete();
+                        return;
+                    }
+                    const blob = parsed.blobs[0];
+                    c1.call({
+                        responseCode: 200,
+                        data: {
+                            path: blob.name.substring(rootPrefix.length + 1),
+                            size: blob.size,
+                            lastModified: blob.lastModified,
+                            etag: blob.etag,
+                        },
+                    });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'metadata_request_failed' });
+                    c1.delete();
+                });
+            });
+        },
+        copyGetter() {
+            return getter;
+        },
+    };
+    return getter;
+}
+function createAzureWriter(rotatingCreds, _wasmModule) {
+    const writer = {
+        saveWithCompletion(key, content, completion) {
+            key = removeAzureScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const body = toFetchBody(content);
+            rotatingCreds.get().then((creds) => {
+                const blobP = azureBlobPath(key, creds);
+                fetch(azureBlobUrl(creds, blobP), {
+                    method: 'PUT',
+                    headers: {
+                        'x-ms-blob-type': 'BlockBlob',
+                        'Content-Type': 'application/octet-stream',
+                    },
+                    body,
+                })
+                    .then((resp) => {
+                    const etag = (resp.headers.get('etag') ?? '').replace(/"/g, '');
+                    if (resp.ok) {
+                        c1.call({ responseCode: resp.status, etag });
+                    }
+                    else {
+                        c1.call({ responseCode: resp.status, message: 'upload_failed' });
+                    }
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        saveConditional(key, content, ifMatchEtag, completion) {
+            key = removeAzureScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const body = toFetchBody(content);
+            const quotedEtag = ifMatchEtag.startsWith('"') ? ifMatchEtag : `"${ifMatchEtag}"`;
+            rotatingCreds.get().then((creds) => {
+                const blobP = azureBlobPath(key, creds);
+                fetch(azureBlobUrl(creds, blobP), {
+                    method: 'PUT',
+                    headers: {
+                        'x-ms-blob-type': 'BlockBlob',
+                        'Content-Type': 'application/octet-stream',
+                        'If-Match': quotedEtag,
+                    },
+                    body,
+                })
+                    .then((resp) => {
+                    const etag = (resp.headers.get('etag') ?? '').replace(/"/g, '');
+                    if (resp.ok) {
+                        c1.call({ responseCode: resp.status, etag });
+                    }
+                    else {
+                        c1.call({ responseCode: resp.status === 412 ? 412 : resp.status, message: 'conditional_write_failed' });
+                    }
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        saveExclusive(key, content, completion) {
+            key = removeAzureScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            const body = toFetchBody(content);
+            rotatingCreds.get().then((creds) => {
+                const blobP = azureBlobPath(key, creds);
+                fetch(azureBlobUrl(creds, blobP), {
+                    method: 'PUT',
+                    headers: {
+                        'x-ms-blob-type': 'BlockBlob',
+                        'Content-Type': 'application/octet-stream',
+                        'If-None-Match': '*',
+                    },
+                    body,
+                })
+                    .then((resp) => {
+                    const etag = (resp.headers.get('etag') ?? '').replace(/"/g, '');
+                    if (resp.ok) {
+                        c1.call({ responseCode: resp.status, etag });
+                    }
+                    else {
+                        c1.call({ responseCode: resp.status === 412 ? 412 : resp.status, message: 'exclusive_write_failed' });
+                    }
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        remove(key, completion) {
+            key = removeAzureScheme(key);
+            const c1 = (0, inflight_tracker_1.trackClone)(completion);
+            rotatingCreds.get().then((creds) => {
+                const blobP = azureBlobPath(key, creds);
+                fetch(azureBlobUrl(creds, blobP), { method: 'DELETE' })
+                    .then((resp) => {
+                    c1.call({ responseCode: resp.ok ? 200 : resp.status });
+                    c1.delete();
+                })
+                    .catch((err) => {
+                    c1.call({ responseCode: 1000, message: err.message ?? 'unknown_error' });
+                    c1.delete();
+                });
+            });
+        },
+        copyWriter() {
+            return writer;
+        },
+    };
+    return writer;
+}
 //# sourceMappingURL=storage.js.map