@ar.io/sdk 3.12.0-beta.5 → 3.12.0-beta.7

package/lib/esm/common/wayfinder/wayfinder.js

@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import EventEmitter from 'node:events';
-import { PassThrough, Readable } from 'node:stream';
+import { EventEmitter } from 'eventemitter3';
+import { base32 } from 'rfc4648';
 import { ARIO } from '../io.js';
 import { Logger } from '../logger.js';
 import { NetworkGatewaysProvider } from './gateways/network.js';
@@ -50,7 +50,8 @@ export const resolveWayfinderUrl = ({ originalUrl, selectedGateway, logger, }) =
         const [firstPart, ...rest] = path.split('/');
         // TODO: this breaks 43 character named arns names - we should check a a local name cache list before resolving raw transaction ids
         if (txIdRegex.test(firstPart)) {
-            return new URL(`${firstPart}${rest.length > 0 ? '/' + rest.join('/') : ''}`, selectedGateway);
+            const sandbox = sandboxFromId(firstPart);
+            return new URL(`${firstPart}${rest.length > 0 ? '/' + rest.join('/') : ''}`, `${selectedGateway.protocol}//${sandbox}.${selectedGateway.hostname}`);
         }
         if (arnsRegex.test(firstPart)) {
             // TODO: tests to ensure arns names support query params and paths
@@ -62,21 +63,19 @@ export const resolveWayfinderUrl = ({ originalUrl, selectedGateway, logger, }) =
             return new URL(rest.length > 0 ? rest.join('/') : '', arnsUrl);
         }
         // TODO: support .eth addresses
-        // TODO: "gasless" routing via DNS TXT records (e.g. ar://gatewaypie.com -> TXT record lookup for TX ID and redirect to that gateway)
+        // TODO: "gasless" routing via DNS TXT records
     }
     logger?.debug('No wayfinder routing protocol applied', {
         originalUrl,
     });
-    // return the original url if it's not a wayfinder url (allows you to use the wayfinder client with non-wayfinder urls)
+    // return the original url if it's not a wayfinder url
     return new URL(originalUrl);
 };
 export class WayfinderEmitter extends EventEmitter {
-    constructor({ onVerificationPassed, onVerificationFailed, onVerificationProgress,
-    // TODO: continue this pattern for all events
-     } = {}) {
+    constructor({ onVerificationSucceeded, onVerificationFailed, onVerificationProgress, } = {}) {
         super();
-        if (onVerificationPassed) {
-            this.on('verification-succeeded', onVerificationPassed);
+        if (onVerificationSucceeded) {
+            this.on('verification-succeeded', onVerificationSucceeded);
         }
         if (onVerificationFailed) {
             this.on('verification-failed', onVerificationFailed);
@@ -85,83 +84,17 @@ export class WayfinderEmitter extends EventEmitter {
             this.on('verification-progress', onVerificationProgress);
         }
     }
-    emit(event, payload) {
-        return super.emit(event, payload);
-    }
-    on(event, listener) {
-        return super.on(event, listener);
-    }
 }
-export function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, emitter, strict = false, }) {
-    // taps node streams
-    if (originalStream instanceof Readable &&
-        typeof originalStream.pipe === 'function') {
-        const tappedClientStream = new PassThrough();
-        const streamToVerify = new PassThrough();
-        // kick off the verification promise, this will be awaited when the original stream ends
-        const verificationPromise = verifyData({
-            data: streamToVerify,
-            txId,
-        });
-        let bytesProcessed = 0;
-        // pipe the original stream to the verifier and the client stream
-        originalStream.on('data', (chunk) => {
-            streamToVerify.write(chunk);
-            tappedClientStream.write(chunk);
-            bytesProcessed += chunk.length;
-            // only emit if contentLength is not 0
-            if (contentLength !== 0) {
-                emitter?.emit('verification-progress', {
-                    txId,
-                    totalBytes: contentLength,
-                    processedBytes: bytesProcessed,
-                });
-            }
-        });
-        originalStream.on('end', async () => {
-            streamToVerify.end(); // triggers verifier completion and completes the verification promise
-            if (strict) {
-                // in strict mode, we wait for verification to complete before ending the client stream
-                try {
-                    await verificationPromise;
-                    emitter?.emit('verification-succeeded', { txId });
-                    tappedClientStream.end();
-                }
-                catch (error) {
-                    emitter?.emit('verification-failed', { error, txId });
-                    // In strict mode, destroy the client stream with the error
-                    tappedClientStream.destroy(new Error('Verification failed', { cause: error }));
-                }
-            }
-            else {
-                // in non-strict mode, we end the client stream immediately and handle verification asynchronously
-                tappedClientStream.end();
-                // trigger the verification promise and emit events for the result
-                verificationPromise
-                    .then(() => {
-                    emitter?.emit('verification-succeeded', { txId });
-                })
-                    .catch((error) => {
-                    emitter?.emit('verification-failed', { error, txId });
-                });
-            }
-        });
-        originalStream.on('error', (err) => {
-            // emit the verification failed event
-            emitter?.emit('verification-failed', {
-                error: err,
-                txId,
-            });
-            // destroy both streams and propagate the original stream error
-            streamToVerify.destroy(err);
-            tappedClientStream.destroy(err);
-        });
-        // send the stream to the verify function and if it errors end the client stream
-        return tappedClientStream;
-    }
-    // taps web readable streams
+export function tapAndVerifyReadableStream({ originalStream, contentLength, verifyData, txId, emitter, strict = false, }) {
     if (originalStream instanceof ReadableStream &&
         typeof originalStream.tee === 'function') {
+        /**
+         * NOTE: tee requires the streams both streams to be consumed, so we need to make sure we consume the client branch
+         * by the caller. This means when `request` is called, the client stream must be consumed by the caller via await request.text()
+         * for verification to complete.
+         *
+         * It is feasible to make the verification stream not to depend on the client branch being consumed, should the DX not be obvious.
+         */
         const [verifyBranch, clientBranch] = originalStream.tee();
         // setup our promise to verify the data
         const verificationPromise = verifyData({
@@ -182,10 +115,8 @@ export function tapAndVerifyStream({ originalStream, contentLength, verifyData,
                             controller.close();
                         }
                         catch (err) {
-                            emitter?.emit('verification-failed', {
-                                txId,
-                                error: err,
-                            });
+                            // emit the verification failed event
+                            emitter?.emit('verification-failed', err);
                             // In strict mode, we report the error to the client stream
                             controller.error(new Error('Verification failed', { cause: err }));
                         }
@@ -197,10 +128,7 @@ export function tapAndVerifyStream({ originalStream, contentLength, verifyData,
                             emitter?.emit('verification-succeeded', { txId });
                         })
                             .catch((error) => {
-                            emitter?.emit('verification-failed', {
-                                txId,
-                                error,
-                            });
+                            emitter?.emit('verification-failed', error);
                         });
                         // in non-strict mode, we close the controller immediately and handle verification asynchronously
                         controller.close();
@@ -228,14 +156,20 @@ export function tapAndVerifyStream({ originalStream, contentLength, verifyData,
                         },
                     }),
                 });
-                // note: we don't block or throw errors here even in strict mode
-                // since the stream is already being cancelled by the client
             },
         });
         return clientStreamWithVerification;
     }
     throw new Error('Unsupported body type for cloning');
 }
+/**
+ * Gets the sandbox hash for a given transaction id
+ */
+export function sandboxFromId(id) {
+    return base32
+        .stringify(Buffer.from(id, 'base64'), { pad: false })
+        .toLowerCase();
+}
 /**
  * Creates a wrapped fetch function that supports ar:// protocol
  *
@@ -248,146 +182,128 @@ export function tapAndVerifyStream({ originalStream, contentLength, verifyData,
  * @returns a wrapped fetch function that supports ar:// protocol and always returns Response
  */
 export const createWayfinderClient = ({ resolveUrl, verifyData, selectGateway, emitter = new WayfinderEmitter(), logger, strict = false, }) => {
-    // Create a function that will handle the redirection logic for ar:// URLs
-    const wayfinderRedirect = async (url, init) => {
-        // If the url is not a string or URL (e.g., it's a Request object), extract the URL from it
-        const originalUrl = url instanceof Request ? url.url : url.toString();
-        // If it's not an ar:// URL, pass it directly to fetch
-        if (!originalUrl.startsWith('ar://')) {
-            logger?.debug('Not a wayfinder URL, passing to fetch directly', {
-                originalUrl,
+    return async (url, init) => {
+        if (typeof url !== 'string') {
+            logger?.debug('URL is not a string, skipping routing', {
+                url,
             });
             emitter?.emit('routing-skipped', {
-                originalUrl,
+                originalUrl: JSON.stringify(url),
             });
-            // we don't do anything special with non-ar:// urls, just pass them through
             return fetch(url, init);
         }
-        // Start the routing process
         emitter?.emit('routing-started', {
-            originalUrl,
+            originalUrl: url.toString(),
         });
-        // Retry logic for gateway selection
         const maxRetries = 3;
         const retryDelay = 1000;
         for (let i = 0; i < maxRetries; i++) {
             try {
                 // select the target gateway
-                // TODO: we may want to provide the `path` to select gateway so the HEAD checks in routers check the existence of the actual path/request
                 const selectedGateway = await selectGateway();
                 logger?.debug('Selected gateway', {
-                    originalUrl,
+                    originalUrl: url,
                     selectedGateway: selectedGateway.toString(),
                 });
                 // route the request to the target gateway
                 const redirectUrl = resolveUrl({
-                    originalUrl,
+                    originalUrl: url,
                     selectedGateway,
                     logger,
                 });
                 emitter?.emit('routing-succeeded', {
-                    originalUrl,
+                    originalUrl: url,
                     selectedGateway: selectedGateway.toString(),
                     redirectUrl: redirectUrl.toString(),
                 });
                 logger?.debug(`Redirecting request`, {
-                    originalUrl,
+                    originalUrl: url,
                     redirectUrl: redirectUrl.toString(),
                 });
-                // Make the request to the target gateway
+                // make the request to the target gateway using the redirect url
                 const response = await fetch(redirectUrl.toString(), {
-                    // follow redirects as gateways use sandboxing on /txId requests
+                    // enforce CORS given we're likely going to a different origin, but always allow the client to override
                     redirect: 'follow',
                     mode: 'cors',
-                    // allow requestor to override and any additional request configuration
                     ...init,
                 });
-                // TODO: update any caching we use for the request and gateway response
                 logger?.debug(`Successfully routed request to gateway`, {
                     redirectUrl: redirectUrl.toString(),
-                    originalUrl,
+                    originalUrl: url.toString(),
                 });
-                // return the response right away if no redirect was made
-                if (redirectUrl.toString() === originalUrl) {
-                    return response;
-                }
-                // return the response right away if no verification is needed or if there is no body
-                if (!verifyData) {
-                    return response;
-                }
-                // the txId is either in the response headers or the path of the request as the first parameter
-                const txId = response.headers.get('x-arns-resolved-id') ??
-                    redirectUrl.pathname.split('/')[1];
-                const contentLength = +(response.headers.get('content-length') ?? 0);
-                if (!txIdRegex.test(txId)) {
-                    // No transaction ID found, skip verification
-                    logger?.debug('No transaction ID found, skipping verification', {
-                        redirectUrl: redirectUrl.toString(),
-                        originalUrl,
-                    });
-                    emitter?.emit('verification-skipped', {
-                        originalUrl,
-                    });
-                    return response;
-                }
-                emitter?.emit('identified-transaction-id', {
-                    originalUrl,
-                    selectedGateway: redirectUrl.toString(),
-                    txId,
-                });
-                if (!response.body) {
-                    logger?.debug('No body, skipping verification', {
-                        redirectUrl: redirectUrl.toString(),
-                        originalUrl,
-                    });
-                    emitter?.emit('verification-skipped', {
-                        originalUrl,
-                    });
-                    return response;
+                // only verify data if the redirect url is different from the original url
+                if (redirectUrl.toString() !== url.toString()) {
+                    if (verifyData) {
+                        const headers = response.headers;
+                        // transaction id is either in the response headers or the path of the request as the first parameter
+                        const txId = headers.get('x-arns-resolved-id') ??
+                            redirectUrl.pathname.split('/')[1];
+                        const contentLength = +(headers.get('content-length') ?? 0);
+                        if (!txIdRegex.test(txId)) {
+                            // no transaction id found, skip verification
+                            logger?.debug('No transaction id found, skipping verification', {
+                                redirectUrl: redirectUrl.toString(),
+                                originalUrl: url,
+                            });
+                            emitter?.emit('verification-skipped', {
+                                originalUrl: url,
+                            });
+                            return response;
+                        }
+                        emitter?.emit('identified-transaction-id', {
+                            originalUrl: url,
+                            selectedGateway: redirectUrl.toString(),
+                            txId,
+                        });
+                        // Check if the response has a body
+                        if (response.body) {
+                            const newClientStream = tapAndVerifyReadableStream({
+                                originalStream: response.body,
+                                contentLength,
+                                verifyData,
+                                txId,
+                                emitter,
+                                strict,
+                            });
+                            return new Response(newClientStream, {
+                                status: response.status,
+                                statusText: response.statusText,
+                                headers: response.headers,
+                            });
+                        }
+                        else {
+                            // No response body to verify, skip verification
+                            logger?.debug('No response body to verify', {
+                                redirectUrl: redirectUrl.toString(),
+                                originalUrl: url,
+                                txId,
+                            });
+                            return response;
+                        }
+                    }
                 }
-                const verifiedStream = tapAndVerifyStream({
-                    originalStream: response.body,
-                    contentLength,
-                    verifyData,
-                    txId,
-                    emitter,
-                    strict,
-                });
-                // wrap the response with the verified stream
-                return new Response(verifiedStream, {
-                    status: response.status,
-                    statusText: response.statusText,
-                    // TODO: we could add identified transaction id to the headers here, but it would be changing information from the original response
-                    headers: response.headers,
-                });
+                return response;
             }
             catch (error) {
                 logger?.debug('Failed to route request', {
                     error: error.message,
                     stack: error.stack,
-                    originalUrl,
+                    originalUrl: url,
                     attempt: i + 1,
                     maxRetries,
                 });
                 if (i < maxRetries - 1) {
                     await new Promise((resolve) => setTimeout(resolve, retryDelay));
                 }
-                else {
-                    emitter?.emit('routing-failed', {
-                        originalUrl,
-                        error,
-                    });
-                }
             }
         }
         throw new Error('Failed to route request after max retries', {
             cause: {
-                originalUrl,
+                originalUrl: url,
                 maxRetries,
             },
         });
     };
-    return wayfinderRedirect;
 };
 /**
  * The main class for the wayfinder
@@ -538,7 +454,7 @@ export class Wayfinder {
             }),
         }),
     }), events = {
-        onVerificationPassed: (event) => {
+        onVerificationSucceeded: (event) => {
             logger.debug('Verification passed!', event);
         },
         onVerificationFailed: (event) => {
@@ -547,7 +463,7 @@ export class Wayfinder {
         onVerificationProgress: (event) => {
             logger.debug('Verification progress!', event);
         },
-    }, strict = false, }) {
+    }, strict = false, } = {}) {
         this.routingStrategy = routingStrategy;
         this.gatewaysProvider = gatewaysProvider;
         this.emitter = new WayfinderEmitter(events);

package/lib/esm/utils/hash.js

@@ -13,38 +13,85 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import Arweave from 'arweave';
+import { MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, buildLayers, generateLeaves, } from 'arweave/node/lib/merkle.js';
 import { createHash } from 'crypto';
 import { toB64Url } from './base64.js';
-export const hashReadableToB64Url = (stream, algorithm = 'sha256') => {
-    return new Promise((resolve, reject) => {
-        const hash = createHash(algorithm);
-        stream.on('data', (chunk) => hash.update(chunk));
-        stream.on('end', () => resolve(toB64Url(hash.digest())));
-        stream.on('error', (err) => reject(err));
-    });
+export const isAsyncIterable = (x) => {
+    return x && typeof x[Symbol.asyncIterator] === 'function';
 };
-export const hashReadableStreamToB64Url = (stream, algorithm = 'sha256') => {
-    return new Promise((resolve, reject) => {
-        const hash = createHash(algorithm);
+export const isReadableStream = (x) => {
+    return x && typeof x.getReader === 'function';
+};
+// convert ReadableStream to async iterable
+export const readableStreamToAsyncIterable = (stream) => ({
+    async *[Symbol.asyncIterator]() {
         const reader = stream.getReader();
-        const read = async () => {
-            try {
+        try {
+            while (true) {
                 const { done, value } = await reader.read();
-                if (done) {
-                    resolve(toB64Url(hash.digest()));
-                }
-                else {
-                    hash.update(value);
-                    read();
-                }
-            }
-            catch (err) {
-                reject(err);
+                if (done)
+                    break;
+                if (value !== undefined)
+                    yield value;
             }
-        };
-        read().catch(reject);
-    });
+        }
+        finally {
+            reader.releaseLock();
+        }
+    },
+});
+export const hashDataStreamToB64Url = async (stream, algorithm = 'sha256') => {
+    const asyncIterable = isAsyncIterable(stream)
+        ? stream
+        : readableStreamToAsyncIterable(stream);
+    const hash = createHash(algorithm);
+    for await (const chunk of asyncIterable) {
+        hash.update(chunk);
+    }
+    return toB64Url(hash.digest());
 };
-export const hashBufferToB64Url = (buffer, algorithm = 'sha256') => {
-    return toB64Url(createHash(algorithm).update(buffer).digest());
+export const convertDataStreamToDataRoot = async ({ dataStream, }) => {
+    const chunks = [];
+    let leftover = new Uint8Array(0);
+    let cursor = 0;
+    const asyncIterable = isAsyncIterable(dataStream)
+        ? dataStream
+        : readableStreamToAsyncIterable(dataStream);
+    for await (const data of asyncIterable) {
+        const inputChunk = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+        const combined = new Uint8Array(leftover.length + inputChunk.length);
+        combined.set(leftover, 0);
+        combined.set(inputChunk, leftover.length);
+        let startIndex = 0;
+        while (combined.length - startIndex >= MAX_CHUNK_SIZE) {
+            let chunkSize = MAX_CHUNK_SIZE;
+            const remainderAfterThis = combined.length - startIndex - MAX_CHUNK_SIZE;
+            if (remainderAfterThis > 0 && remainderAfterThis < MIN_CHUNK_SIZE) {
+                chunkSize = Math.ceil((combined.length - startIndex) / 2);
+            }
+            const chunkData = combined.slice(startIndex, startIndex + chunkSize);
+            const dataHash = await Arweave.crypto.hash(chunkData);
+            chunks.push({
+                dataHash,
+                minByteRange: cursor,
+                maxByteRange: cursor + chunkSize,
+            });
+            cursor += chunkSize;
+            startIndex += chunkSize;
+        }
+        leftover = combined.slice(startIndex);
+    }
+    if (leftover.length > 0) {
+        // TODO: ensure a web friendly crypto hash function is used in web
+        const dataHash = await Arweave.crypto.hash(leftover);
+        chunks.push({
+            dataHash,
+            minByteRange: cursor,
+            maxByteRange: cursor + leftover.length,
+        });
+    }
+    const leaves = await generateLeaves(chunks);
+    const root = await buildLayers(leaves);
+    return toB64Url(Buffer.from(root.id));
 };

package/lib/esm/version.js

@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 // AUTOMATICALLY GENERATED FILE - DO NOT TOUCH
-export const version = '3.12.0-beta.5';
+export const version = '3.12.0-beta.7';

package/lib/types/common/wayfinder/verification/strategies/data-root-verifier.d.ts

@@ -1,10 +1,6 @@
-import { Readable } from 'node:stream';
-import { DataRootProvider, DataVerificationStrategy } from '../../../../types/wayfinder.js';
-export declare function convertBufferToDataRoot({ buffer, }: {
-    buffer: Buffer;
-}): Promise<string>;
-export declare const convertReadableToDataRoot: <T extends AsyncIterable<Uint8Array>>({ iterable, }: {
-    iterable: T;
+import { DataRootProvider, DataStream, DataVerificationStrategy } from '../../../../types/wayfinder.js';
+export declare const convertDataStreamToDataRoot: ({ dataStream, }: {
+    dataStream: DataStream;
 }) => Promise<string>;
 export declare class DataRootVerificationStrategy implements DataVerificationStrategy {
     private readonly trustedDataRootProvider;
@@ -12,7 +8,7 @@ export declare class DataRootVerificationStrategy implements DataVerificationStr
         trustedDataRootProvider: DataRootProvider;
     });
     verifyData({ data, txId, }: {
-        data: Buffer | Readable | ReadableStream;
+        data: DataStream;
         txId: string;
     }): Promise<void>;
 }

package/lib/types/common/wayfinder/verification/strategies/hash-verifier.d.ts

@@ -13,15 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Readable } from 'node:stream';
-import { DataHashProvider, DataVerificationStrategy } from '../../../../types/wayfinder.js';
+import { DataHashProvider, DataStream, DataVerificationStrategy } from '../../../../types/wayfinder.js';
 export declare class HashVerificationStrategy implements DataVerificationStrategy {
     private readonly trustedHashProvider;
     constructor({ trustedHashProvider, }: {
         trustedHashProvider: DataHashProvider;
     });
     verifyData({ data, txId, }: {
-        data: Buffer | Readable | ReadableStream;
+        data: DataStream;
         txId: string;
     }): Promise<void>;
 }