@ar.io/sdk 3.12.0-beta.6 → 3.12.0-beta.8

package/lib/cjs/common/wayfinder/verification/strategies/data-root-verifier.js

@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DataRootVerificationStrategy = exports.convertReadableToDataRoot = void 0;
-exports.convertBufferToDataRoot = convertBufferToDataRoot;
+exports.DataRootVerificationStrategy = exports.convertDataStreamToDataRoot = void 0;
 /**
  * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
  *
@@ -22,39 +21,16 @@ exports.convertBufferToDataRoot = convertBufferToDataRoot;
  */
 const arweave_1 = __importDefault(require("arweave"));
 const merkle_js_1 = require("arweave/node/lib/merkle.js");
-const node_stream_1 = require("node:stream");
 const base64_js_1 = require("../../../../utils/base64.js");
-async function convertBufferToDataRoot({ buffer, }) {
-    const chunks = [];
-    let cursor = 0;
-    let offset = 0;
-    while (offset < buffer.byteLength) {
-        let chunkSize = Math.min(merkle_js_1.MAX_CHUNK_SIZE, buffer.byteLength - offset);
-        const remainder = buffer.byteLength - offset - chunkSize;
-        if (remainder > 0 && remainder < merkle_js_1.MIN_CHUNK_SIZE) {
-            chunkSize = Math.ceil((buffer.byteLength - offset) / 2);
-        }
-        // subarray does not exist on web Buffer type
-        const slice = buffer.subarray(offset, offset + chunkSize);
-        const hash = await crypto.subtle.digest('SHA-256', slice);
-        const hashArray = new Uint8Array(hash);
-        chunks.push({
-            dataHash: hashArray,
-            minByteRange: cursor,
-            maxByteRange: cursor + chunkSize,
-        });
-        cursor += chunkSize;
-        offset += chunkSize;
-    }
-    const leaves = await (0, merkle_js_1.generateLeaves)(chunks);
-    const result = await (0, merkle_js_1.buildLayers)(leaves);
-    return Buffer.from(result.id).toString('base64url');
-}
-const convertReadableToDataRoot = async ({ iterable, }) => {
+const hash_js_1 = require("../../../../utils/hash.js");
+const convertDataStreamToDataRoot = async ({ dataStream, }) => {
     const chunks = [];
     let leftover = new Uint8Array(0);
     let cursor = 0;
-    for await (const data of iterable) {
+    const asyncIterable = (0, hash_js_1.isAsyncIterable)(dataStream)
+        ? dataStream
+        : (0, hash_js_1.readableStreamToAsyncIterable)(dataStream);
+    for await (const data of asyncIterable) {
         const inputChunk = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
         const combined = new Uint8Array(leftover.length + inputChunk.length);
         combined.set(leftover, 0);
@@ -79,6 +55,7 @@ const convertReadableToDataRoot = async ({ iterable, }) => {
         leftover = combined.slice(startIndex);
     }
     if (leftover.length > 0) {
+        // TODO: ensure a web friendly crypto hash function is used in web
         const dataHash = await arweave_1.default.crypto.hash(leftover);
         chunks.push({
             dataHash,
@@ -90,27 +67,21 @@ const convertReadableToDataRoot = async ({ iterable, }) => {
     const root = await (0, merkle_js_1.buildLayers)(leaves);
     return (0, base64_js_1.toB64Url)(Buffer.from(root.id));
 };
-exports.convertReadableToDataRoot = convertReadableToDataRoot;
+exports.convertDataStreamToDataRoot = convertDataStreamToDataRoot;
 class DataRootVerificationStrategy {
     trustedDataRootProvider;
     constructor({ trustedDataRootProvider, }) {
         this.trustedDataRootProvider = trustedDataRootProvider;
     }
     async verifyData({ data, txId, }) {
-        const trustedDataRootPromise = this.trustedDataRootProvider.getDataRoot({
-            txId,
-        });
-        let computedDataRoot;
-        if (Buffer.isBuffer(data)) {
-            computedDataRoot = await convertBufferToDataRoot({ buffer: data });
-        }
-        else if (data instanceof node_stream_1.Readable || data instanceof ReadableStream) {
-            computedDataRoot = await (0, exports.convertReadableToDataRoot)({ iterable: data });
-        }
-        if (computedDataRoot === undefined) {
-            throw new Error('Data root could not be computed');
-        }
-        const trustedDataRoot = await trustedDataRootPromise;
+        const [computedDataRoot, trustedDataRoot] = await Promise.all([
+            (0, exports.convertDataStreamToDataRoot)({
+                dataStream: data,
+            }),
+            this.trustedDataRootProvider.getDataRoot({
+                txId,
+            }),
+        ]);
         if (computedDataRoot !== trustedDataRoot) {
             throw new Error('Data root does not match', {
                 cause: { computedDataRoot, trustedDataRoot },

package/lib/cjs/common/wayfinder/verification/strategies/hash-verifier.js

@@ -1,22 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HashVerificationStrategy = void 0;
-/**
- * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-const node_stream_1 = require("node:stream");
 const hash_js_1 = require("../../../../utils/hash.js");
 class HashVerificationStrategy {
     trustedHashProvider;
@@ -24,25 +8,18 @@ class HashVerificationStrategy {
         this.trustedHashProvider = trustedHashProvider;
     }
     async verifyData({ data, txId, }) {
-        const hashPromise = this.trustedHashProvider.getHash({ txId });
-        let computedHash;
-        if (Buffer.isBuffer(data)) {
-            computedHash = (0, hash_js_1.hashBufferToB64Url)(data);
-        }
-        else if (data instanceof node_stream_1.Readable) {
-            computedHash = await (0, hash_js_1.hashReadableToB64Url)(data);
-        }
-        else if (data instanceof ReadableStream) {
-            computedHash = await (0, hash_js_1.hashReadableStreamToB64Url)(data);
-        }
+        // kick off the hash computation, but don't wait for it until we compute our own hash
+        const [computedHash, fetchedHash] = await Promise.all([
+            (0, hash_js_1.hashDataStreamToB64Url)(data),
+            this.trustedHashProvider.getHash({ txId }),
+        ]);
         // await on the hash promise and compare to get a little concurrency when computing hashes over larger data
-        const { hash } = await hashPromise;
         if (computedHash === undefined) {
             throw new Error('Hash could not be computed');
         }
-        if (computedHash !== hash) {
+        if (computedHash !== fetchedHash.hash) {
             throw new Error('Hash does not match', {
-                cause: { computedHash, trustedHash: hash },
+                cause: { computedHash, trustedHash: fetchedHash },
             });
         }
     }

package/lib/cjs/common/wayfinder/verification/trusted.js

@@ -27,14 +27,14 @@ class TrustedGatewaysHashProvider {
         const gateways = await this.gatewaysProvider.getGateways();
         const hashes = await Promise.all(gateways.map(async (gateway) => {
             const sandbox = (0, wayfinder_js_1.sandboxFromId)(txId);
-            const urlWithSandbox = `${gateway.protocol}//${sandbox}.${gateway.hostname}/${txId}/`;
+            const urlWithSandbox = `${gateway.protocol}//${sandbox}.${gateway.hostname}/${txId}`;
             let txIdHash;
             /**
              * This is a problem because we're not able to verify the hash of the data item if the gateway doesn't have the data in its cache.
              * We should add the ability to send a HEAD request to trigger a GET request to hydrate the cache on the trusted gateway via a header.
              * For now, we'll just do a GET request to hydrate the cache if the HEAD request doesn't contain the digest.
              */
-            for (const method of ['HEAD', 'GET']) {
+            for (const method of ['HEAD', 'GET', 'GET']) {
                 const response = await fetch(urlWithSandbox, {
                     method,
                     redirect: 'follow',

package/lib/cjs/common/wayfinder/wayfinder.js

@@ -1,10 +1,7 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Wayfinder = exports.createWayfinderClient = exports.WayfinderEmitter = exports.resolveWayfinderUrl = exports.txIdRegex = exports.arnsRegex = void 0;
-exports.tapAndVerifyStream = tapAndVerifyStream;
+exports.tapAndVerifyReadableStream = tapAndVerifyReadableStream;
 exports.sandboxFromId = sandboxFromId;
 /**
  * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
@@ -21,8 +18,7 @@ exports.sandboxFromId = sandboxFromId;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const node_events_1 = __importDefault(require("node:events"));
-const node_stream_1 = require("node:stream");
+const eventemitter3_1 = require("eventemitter3");
 const rfc4648_1 = require("rfc4648");
 const io_js_1 = require("../io.js");
 const logger_js_1 = require("../logger.js");
@@ -72,22 +68,20 @@ const resolveWayfinderUrl = ({ originalUrl, selectedGateway, logger, }) => {
             return new URL(rest.length > 0 ? rest.join('/') : '', arnsUrl);
         }
         // TODO: support .eth addresses
-        // TODO: "gasless" routing via DNS TXT records (e.g. ar://gatewaypie.com -> TXT record lookup for TX ID and redirect to that gateway)
+        // TODO: "gasless" routing via DNS TXT records
     }
     logger?.debug('No wayfinder routing protocol applied', {
         originalUrl,
     });
-    // return the original url if it's not a wayfinder url (allows you to use the wayfinder client with non-wayfinder urls)
+    // return the original url if it's not a wayfinder url
     return new URL(originalUrl);
 };
 exports.resolveWayfinderUrl = resolveWayfinderUrl;
-class WayfinderEmitter extends node_events_1.default {
-    constructor({ onVerificationPassed, onVerificationFailed, onVerificationProgress,
-    // TODO: continue this pattern for all events
-     } = {}) {
+class WayfinderEmitter extends eventemitter3_1.EventEmitter {
+    constructor({ onVerificationSucceeded, onVerificationFailed, onVerificationProgress, } = {}) {
         super();
-        if (onVerificationPassed) {
-            this.on('verification-succeeded', onVerificationPassed);
+        if (onVerificationSucceeded) {
+            this.on('verification-succeeded', onVerificationSucceeded);
         }
         if (onVerificationFailed) {
             this.on('verification-failed', onVerificationFailed);
@@ -96,84 +90,18 @@ class WayfinderEmitter extends node_events_1.default {
             this.on('verification-progress', onVerificationProgress);
         }
     }
-    emit(event, payload) {
-        return super.emit(event, payload);
-    }
-    on(event, listener) {
-        return super.on(event, listener);
-    }
 }
 exports.WayfinderEmitter = WayfinderEmitter;
-function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, emitter, strict = false, }) {
-    // taps node streams
-    if (originalStream instanceof node_stream_1.Readable &&
-        typeof originalStream.pipe === 'function') {
-        const tappedClientStream = new node_stream_1.PassThrough();
-        const streamToVerify = new node_stream_1.PassThrough();
-        // kick off the verification promise, this will be awaited when the original stream ends
-        const verificationPromise = verifyData({
-            data: streamToVerify,
-            txId,
-        });
-        let bytesProcessed = 0;
-        // pipe the original stream to the verifier and the client stream
-        originalStream.on('data', (chunk) => {
-            streamToVerify.write(chunk);
-            tappedClientStream.write(chunk);
-            bytesProcessed += chunk.length;
-            // only emit if contentLength is not 0
-            if (contentLength !== 0) {
-                emitter?.emit('verification-progress', {
-                    txId,
-                    totalBytes: contentLength,
-                    processedBytes: bytesProcessed,
-                });
-            }
-        });
-        originalStream.on('end', async () => {
-            streamToVerify.end(); // triggers verifier completion and completes the verification promise
-            if (strict) {
-                // in strict mode, we wait for verification to complete before ending the client stream
-                try {
-                    await verificationPromise;
-                    emitter?.emit('verification-succeeded', { txId });
-                    tappedClientStream.end();
-                }
-                catch (error) {
-                    emitter?.emit('verification-failed', { error, txId });
-                    // In strict mode, destroy the client stream with the error
-                    tappedClientStream.destroy(new Error('Verification failed', { cause: error }));
-                }
-            }
-            else {
-                // in non-strict mode, we end the client stream immediately and handle verification asynchronously
-                tappedClientStream.end();
-                // trigger the verification promise and emit events for the result
-                verificationPromise
-                    .then(() => {
-                    emitter?.emit('verification-succeeded', { txId });
-                })
-                    .catch((error) => {
-                    emitter?.emit('verification-failed', { error, txId });
-                });
-            }
-        });
-        originalStream.on('error', (err) => {
-            // emit the verification failed event
-            emitter?.emit('verification-failed', {
-                error: err,
-                txId,
-            });
-            // destroy both streams and propagate the original stream error
-            streamToVerify.destroy(err);
-            tappedClientStream.destroy(err);
-        });
-        // send the stream to the verify function and if it errors end the client stream
-        return tappedClientStream;
-    }
-    // taps web readable streams
+function tapAndVerifyReadableStream({ originalStream, contentLength, verifyData, txId, emitter, strict = false, }) {
     if (originalStream instanceof ReadableStream &&
         typeof originalStream.tee === 'function') {
+        /**
+         * NOTE: tee requires the streams both streams to be consumed, so we need to make sure we consume the client branch
+         * by the caller. This means when `request` is called, the client stream must be consumed by the caller via await request.text()
+         * for verification to complete.
+         *
+         * It is feasible to make the verification stream not to depend on the client branch being consumed, should the DX not be obvious.
+         */
         const [verifyBranch, clientBranch] = originalStream.tee();
         // setup our promise to verify the data
         const verificationPromise = verifyData({
@@ -194,10 +122,8 @@ function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, e
                             controller.close();
                         }
                         catch (err) {
-                            emitter?.emit('verification-failed', {
-                                txId,
-                                error: err,
-                            });
+                            // emit the verification failed event
+                            emitter?.emit('verification-failed', err);
                             // In strict mode, we report the error to the client stream
                             controller.error(new Error('Verification failed', { cause: err }));
                         }
@@ -209,10 +135,7 @@ function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, e
                             emitter?.emit('verification-succeeded', { txId });
                         })
                             .catch((error) => {
-                            emitter?.emit('verification-failed', {
-                                txId,
-                                error,
-                            });
+                            emitter?.emit('verification-failed', error);
                         });
                         // in non-strict mode, we close the controller immediately and handle verification asynchronously
                         controller.close();
@@ -240,8 +163,6 @@ function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, e
                         },
                     }),
                 });
-                // note: we don't block or throw errors here even in strict mode
-                // since the stream is already being cancelled by the client
             },
         });
         return clientStreamWithVerification;
@@ -268,146 +189,128 @@ function sandboxFromId(id) {
  * @returns a wrapped fetch function that supports ar:// protocol and always returns Response
  */
 const createWayfinderClient = ({ resolveUrl, verifyData, selectGateway, emitter = new WayfinderEmitter(), logger, strict = false, }) => {
-    // Create a function that will handle the redirection logic for ar:// URLs
-    const wayfinderRedirect = async (url, init) => {
-        // If the url is not a string or URL (e.g., it's a Request object), extract the URL from it
-        const originalUrl = url instanceof Request ? url.url : url.toString();
-        // If it's not an ar:// URL, pass it directly to fetch
-        if (!originalUrl.startsWith('ar://')) {
-            logger?.debug('Not a wayfinder URL, passing to fetch directly', {
-                originalUrl,
+    return async (url, init) => {
+        if (typeof url !== 'string') {
+            logger?.debug('URL is not a string, skipping routing', {
+                url,
             });
             emitter?.emit('routing-skipped', {
-                originalUrl,
+                originalUrl: JSON.stringify(url),
             });
-            // we don't do anything special with non-ar:// urls, just pass them through
             return fetch(url, init);
         }
-        // Start the routing process
         emitter?.emit('routing-started', {
-            originalUrl,
+            originalUrl: url.toString(),
         });
-        // Retry logic for gateway selection
         const maxRetries = 3;
         const retryDelay = 1000;
         for (let i = 0; i < maxRetries; i++) {
             try {
                 // select the target gateway
-                // TODO: we may want to provide the `path` to select gateway so the HEAD checks in routers check the existence of the actual path/request
                 const selectedGateway = await selectGateway();
                 logger?.debug('Selected gateway', {
-                    originalUrl,
+                    originalUrl: url,
                     selectedGateway: selectedGateway.toString(),
                 });
                 // route the request to the target gateway
                 const redirectUrl = resolveUrl({
-                    originalUrl,
+                    originalUrl: url,
                     selectedGateway,
                     logger,
                 });
                 emitter?.emit('routing-succeeded', {
-                    originalUrl,
+                    originalUrl: url,
                     selectedGateway: selectedGateway.toString(),
                     redirectUrl: redirectUrl.toString(),
                 });
                 logger?.debug(`Redirecting request`, {
-                    originalUrl,
+                    originalUrl: url,
                     redirectUrl: redirectUrl.toString(),
                 });
-                // Make the request to the target gateway
+                // make the request to the target gateway using the redirect url
                 const response = await fetch(redirectUrl.toString(), {
-                    // follow redirects as gateways use sandboxing on /txId requests
+                    // enforce CORS given we're likely going to a different origin, but always allow the client to override
                     redirect: 'follow',
                     mode: 'cors',
-                    // allow requestor to override and any additional request configuration
                     ...init,
                 });
-                // TODO: update any caching we use for the request and gateway response
                 logger?.debug(`Successfully routed request to gateway`, {
                     redirectUrl: redirectUrl.toString(),
-                    originalUrl,
+                    originalUrl: url.toString(),
                 });
-                // return the response right away if no redirect was made
-                if (redirectUrl.toString() === originalUrl) {
-                    return response;
-                }
-                // return the response right away if no verification is needed or if there is no body
-                if (!verifyData) {
-                    return response;
-                }
-                // the txId is either in the response headers or the path of the request as the first parameter
-                const txId = response.headers.get('x-arns-resolved-id') ??
-                    redirectUrl.pathname.split('/')[1];
-                const contentLength = +(response.headers.get('content-length') ?? 0);
-                if (!exports.txIdRegex.test(txId)) {
-                    // No transaction ID found, skip verification
-                    logger?.debug('No transaction ID found, skipping verification', {
-                        redirectUrl: redirectUrl.toString(),
-                        originalUrl,
-                    });
-                    emitter?.emit('verification-skipped', {
-                        originalUrl,
-                    });
-                    return response;
-                }
-                emitter?.emit('identified-transaction-id', {
-                    originalUrl,
-                    selectedGateway: redirectUrl.toString(),
-                    txId,
-                });
-                if (!response.body) {
-                    logger?.debug('No body, skipping verification', {
-                        redirectUrl: redirectUrl.toString(),
-                        originalUrl,
-                    });
-                    emitter?.emit('verification-skipped', {
-                        originalUrl,
-                    });
-                    return response;
+                // only verify data if the redirect url is different from the original url
+                if (redirectUrl.toString() !== url.toString()) {
+                    if (verifyData) {
+                        const headers = response.headers;
+                        // transaction id is either in the response headers or the path of the request as the first parameter
+                        const txId = headers.get('x-arns-resolved-id') ??
+                            redirectUrl.pathname.split('/')[1];
+                        const contentLength = +(headers.get('content-length') ?? 0);
+                        if (!exports.txIdRegex.test(txId)) {
+                            // no transaction id found, skip verification
+                            logger?.debug('No transaction id found, skipping verification', {
+                                redirectUrl: redirectUrl.toString(),
+                                originalUrl: url,
+                            });
+                            emitter?.emit('verification-skipped', {
+                                originalUrl: url,
+                            });
+                            return response;
+                        }
+                        emitter?.emit('identified-transaction-id', {
+                            originalUrl: url,
+                            selectedGateway: redirectUrl.toString(),
+                            txId,
+                        });
+                        // Check if the response has a body
+                        if (response.body) {
+                            const newClientStream = tapAndVerifyReadableStream({
+                                originalStream: response.body,
+                                contentLength,
+                                verifyData,
+                                txId,
+                                emitter,
+                                strict,
+                            });
+                            return new Response(newClientStream, {
+                                status: response.status,
+                                statusText: response.statusText,
+                                headers: response.headers,
+                            });
+                        }
+                        else {
+                            // No response body to verify, skip verification
+                            logger?.debug('No response body to verify', {
+                                redirectUrl: redirectUrl.toString(),
+                                originalUrl: url,
+                                txId,
+                            });
+                            return response;
+                        }
+                    }
                 }
-                const verifiedStream = tapAndVerifyStream({
-                    originalStream: response.body,
-                    contentLength,
-                    verifyData,
-                    txId,
-                    emitter,
-                    strict,
-                });
-                // wrap the response with the verified stream
-                return new Response(verifiedStream, {
-                    status: response.status,
-                    statusText: response.statusText,
-                    // TODO: we could add identified transaction id to the headers here, but it would be changing information from the original response
-                    headers: response.headers,
-                });
+                return response;
             }
             catch (error) {
                 logger?.debug('Failed to route request', {
                     error: error.message,
                     stack: error.stack,
-                    originalUrl,
+                    originalUrl: url,
                     attempt: i + 1,
                     maxRetries,
                 });
                 if (i < maxRetries - 1) {
                     await new Promise((resolve) => setTimeout(resolve, retryDelay));
                 }
-                else {
-                    emitter?.emit('routing-failed', {
-                        originalUrl,
-                        error,
-                    });
-                }
             }
         }
         throw new Error('Failed to route request after max retries', {
             cause: {
-                originalUrl,
+                originalUrl: url,
                 maxRetries,
             },
         });
     };
-    return wayfinderRedirect;
 };
 exports.createWayfinderClient = createWayfinderClient;
 /**
@@ -559,7 +462,7 @@ class Wayfinder {
             }),
         }),
     }), events = {
-        onVerificationPassed: (event) => {
+        onVerificationSucceeded: (event) => {
             logger.debug('Verification passed!', event);
         },
         onVerificationFailed: (event) => {
@@ -568,7 +471,7 @@ class Wayfinder {
         onVerificationProgress: (event) => {
             logger.debug('Verification progress!', event);
         },
-    }, strict = false, }) {
+    }, strict = false, } = {}) {
         this.routingStrategy = routingStrategy;
         this.gatewaysProvider = gatewaysProvider;
         this.emitter = new WayfinderEmitter(events);