@ar.io/sdk 3.11.0-alpha.1 → 3.11.0-alpha.10

package/lib/cjs/common/wayfinder/wayfinder.js

@@ -1,44 +1,236 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Wayfinder = exports.createWayfinderClient = exports.resolveWayfinderUrl = exports.txIdRegex = exports.arnsRegex = void 0;
+exports.Wayfinder = exports.createWayfinderClient = exports.WayfinderEmitter = exports.resolveWayfinderUrl = exports.txIdRegex = exports.arnsRegex = void 0;
+exports.tapAndVerifyStream = tapAndVerifyStream;
+exports.wrapVerifiedResponse = wrapVerifiedResponse;
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const node_events_1 = __importDefault(require("node:events"));
+const node_stream_1 = require("node:stream");
 const io_js_1 = require("../io.js");
+const logger_js_1 = require("../logger.js");
 const gateways_js_1 = require("./gateways.js");
+const trusted_gateways_js_1 = require("./gateways/trusted-gateways.js");
 const random_js_1 = require("./routers/random.js");
+const hash_verifier_js_1 = require("./verification/hash-verifier.js");
 // known regexes for wayfinder urls
 exports.arnsRegex = /^[a-z0-9_-]{1,51}$/;
-exports.txIdRegex = /^[a-z0-9]{43}$/;
+exports.txIdRegex = /^[A-Za-z0-9_-]{43}$/;
 /**
  * Core function to resolve a wayfinder url against a target gateway
  * @param originalUrl - the wayfinder url to resolve
  * @param targetGateway - the target gateway to resolve the url against
  * @returns the resolved url that can be used to make a request
  */
-const resolveWayfinderUrl = ({ originalUrl, targetGateway, }) => {
+const resolveWayfinderUrl = async ({ originalUrl, targetGateway, logger, }) => {
     if (originalUrl.toString().startsWith('ar://')) {
+        logger?.debug(`Applying wayfinder routing protocol to ${originalUrl}`, {
+            originalUrl,
+        });
+        const targetGatewayUrl = new URL(await targetGateway());
+        logger?.debug(`Selected target gateway: ${targetGatewayUrl}`, {
+            originalUrl,
+            targetGateway: targetGatewayUrl,
+        });
         const [, path] = originalUrl.toString().split('ar://');
         // e.g. ar:///info should route to the info endpoint of the target gateway
         if (path.startsWith('/')) {
-            return new URL(path.slice(1), targetGateway);
+            logger?.debug(`Routing to ${path.slice(1)} on ${targetGatewayUrl}`, {
+                originalUrl,
+                targetGateway: targetGatewayUrl,
+            });
+            return new URL(path.slice(1), targetGatewayUrl);
         }
         // TODO: this breaks 43 character named arns names - we should check a a local name cache list before resolving raw transaction ids
         if (exports.txIdRegex.test(path)) {
             const [txId, ...rest] = path.split('/');
-            return new URL(`${txId}${rest.join('/')}`, targetGateway);
+            return new URL(`${txId}${rest.join('/')}`, targetGatewayUrl);
         }
         if (exports.arnsRegex.test(path)) {
             // TODO: tests to ensure arns names support query params and paths
             const [name, ...rest] = path.split('/');
-            const targetGatewayUrl = new URL(targetGateway);
             const arnsUrl = `${targetGatewayUrl.protocol}//${name}.${targetGatewayUrl.hostname}${targetGatewayUrl.port ? `:${targetGatewayUrl.port}` : ''}`;
+            logger?.debug(`Routing to ${path} on ${arnsUrl}`, {
+                originalUrl,
+                targetGateway: targetGatewayUrl,
+            });
             return new URL(rest.join('/'), arnsUrl);
         }
         // TODO: support .eth addresses
         // TODO: "gasless" routing via DNS TXT records (e.g. ar://gatewaypie.com -> TXT record lookup for TX ID and redirect to that gateway)
     }
+    logger?.debug('No wayfinder routing protocol applied', {
+        originalUrl,
+    });
     // return the original url if it's not a wayfinder url (allows you to use the wayfinder client with non-wayfinder urls)
     return new URL(originalUrl);
 };
 exports.resolveWayfinderUrl = resolveWayfinderUrl;
+class WayfinderEmitter extends node_events_1.default {
+    constructor({ onVerificationPassed, onVerificationFailed, onVerificationProgress,
+    // TODO: continue this pattern for all events
+     } = {}) {
+        super();
+        if (onVerificationPassed) {
+            this.on('verification-passed', onVerificationPassed);
+        }
+        if (onVerificationFailed) {
+            this.on('verification-failed', onVerificationFailed);
+        }
+        if (onVerificationProgress) {
+            this.on('verification-progress', onVerificationProgress);
+        }
+    }
+    emit(event, payload) {
+        return super.emit(event, payload);
+    }
+    on(event, listener) {
+        return super.on(event, listener);
+    }
+}
+exports.WayfinderEmitter = WayfinderEmitter;
+function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, emitter, }) {
+    // taps node streams
+    if (originalStream instanceof node_stream_1.Readable &&
+        typeof originalStream.pipe === 'function') {
+        const tappedClientStream = new node_stream_1.PassThrough();
+        const streamToVerify = new node_stream_1.PassThrough();
+        // kick off the verification promise, this will be awaited when the original stream ends
+        const verificationPromise = verifyData({
+            data: streamToVerify,
+            txId,
+        });
+        let bytesProcessed = 0;
+        // pipe the original stream to the verifier and the client stream
+        originalStream.on('data', (chunk) => {
+            streamToVerify.write(chunk);
+            tappedClientStream.write(chunk);
+            bytesProcessed += chunk.length;
+            // only emit if contentLength is not 0
+            if (contentLength !== 0) {
+                emitter?.emit('verification-progress', {
+                    txId,
+                    totalBytes: contentLength,
+                    processedBytes: bytesProcessed,
+                });
+            }
+        });
+        originalStream.on('end', async () => {
+            streamToVerify.end(); // triggers verifier completion and completes the verification promise
+            try {
+                await verificationPromise;
+                emitter?.emit('verification-passed', {
+                    txId,
+                });
+                tappedClientStream.end();
+            }
+            catch (error) {
+                emitter?.emit('verification-failed', {
+                    error,
+                    txId,
+                });
+                tappedClientStream.destroy(error);
+            }
+        });
+        originalStream.on('error', (err) => {
+            emitter?.emit('verification-failed', {
+                error: err,
+                txId,
+            });
+            streamToVerify.destroy(err);
+            tappedClientStream.destroy(err);
+        });
+        // send the stream to the verify function and if it errors end the client stream
+        return tappedClientStream;
+    }
+    // taps web readable streams
+    if (originalStream instanceof ReadableStream &&
+        typeof originalStream.tee === 'function') {
+        const [verifyBranch, clientBranch] = originalStream.tee();
+        // setup our promise to verify the data
+        const verificationPromise = verifyData({
+            data: verifyBranch,
+            txId,
+        });
+        let bytesProcessed = 0;
+        const reader = clientBranch.getReader();
+        const clientStreamWithVerification = new ReadableStream({
+            async pull(controller) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    try {
+                        // due to backpressure, if the client does not consume the stream, the verification will not complete (particularly important for fetch, where the response body needs to be awaited for verification to complete)
+                        await verificationPromise;
+                        emitter?.emit('verification-passed', {
+                            txId,
+                        });
+                        controller.close();
+                    }
+                    catch (err) {
+                        emitter?.emit('verification-failed', {
+                            txId,
+                            error: err,
+                        });
+                        controller.error(err);
+                    }
+                }
+                else {
+                    bytesProcessed += value.length;
+                    emitter?.emit('verification-progress', {
+                        txId,
+                        totalBytes: contentLength,
+                        processedBytes: bytesProcessed,
+                    });
+                    controller.enqueue(value);
+                }
+            },
+            cancel(reason) {
+                reader.cancel(reason);
+                emitter?.emit('verification-failed', {
+                    txId,
+                    error: new Error('Verification cancelled', {
+                        cause: {
+                            reason,
+                        },
+                    }),
+                });
+            },
+        });
+        return clientStreamWithVerification;
+    }
+    throw new Error('Unsupported body type for cloning');
+}
+function wrapVerifiedResponse(original, newBody, txId) {
+    // Clone headers (Header objects aren't serializable)
+    const headers = new Headers();
+    original.headers.forEach((value, key) => headers.set(key, value));
+    // Create a new Response with the new body and cloned headers
+    const wrapped = new Response(newBody, {
+        status: original.status,
+        statusText: original.statusText,
+        headers,
+    });
+    // Attach txId for downstream tracking
+    wrapped.txId = txId;
+    wrapped.redirectedFrom = original.url;
+    return wrapped;
+}
 /**
  * Creates a wrapped http client that supports ar:// protocol
  *
@@ -52,24 +244,171 @@ exports.resolveWayfinderUrl = resolveWayfinderUrl;
  * @param resolveUrl - the function to construct the redirect url for ar:// requests
  * @returns a wrapped http client that supports ar:// protocol
  */
-const createWayfinderClient = ({ httpClient, resolveUrl, }) => {
+const createWayfinderClient = ({ httpClient, resolveUrl, verifyData, emitter = new WayfinderEmitter(), logger, }) => {
     const wayfinderRedirect = async (fn, rawArgs) => {
         // TODO: handle if first arg is not a string (i.e. just return the result of the function call)
         const [originalUrl, ...rest] = rawArgs;
+        if (typeof originalUrl !== 'string') {
+            logger?.debug('Original URL is not a string, skipping routing', {
+                originalUrl,
+            });
+            return fn(...rawArgs);
+        }
+        emitter?.emit('routing-started', {
+            originalUrl: originalUrl.toString(),
+        });
         // route the request to the target gateway
         const redirectUrl = await resolveUrl({
             originalUrl,
+            logger,
+        });
+        emitter?.emit('routing-succeeded', {
+            originalUrl,
+            targetGateway: redirectUrl.toString(),
+        });
+        logger?.debug(`Redirecting request to ${redirectUrl}`, {
+            originalUrl,
+            redirectUrl,
         });
         // make the request to the target gateway using the redirect url and http client
         const response = await fn(redirectUrl.toString(), ...rest);
-        // TODO: if verifyDataHash is provided, verify the data hash before returning
+        // TODO: trigger a routing event with the raw response object?
+        logger?.debug(`Successfully routed request to ${redirectUrl}`, {
+            redirectUrl,
+            originalUrl,
+        });
+        // only verify data if the redirect url is different from the original url
+        if (response && redirectUrl.toString() !== originalUrl.toString()) {
+            if (verifyData) {
+                // if the headers do not have .get on them, we need to parse the headers manually
+                const headers = new Headers();
+                let headersObject = response.headers ?? {};
+                if (typeof headersObject.get !== 'function') {
+                    headersObject = Object.fromEntries(headersObject);
+                    for (const [key, value] of Object.entries(headersObject)) {
+                        headers.set(key, value);
+                    }
+                }
+                else {
+                    for (const [key, value] of headersObject.entries()) {
+                        headers.set(key, value);
+                    }
+                }
+                // transaction id is either in the response headers or the path of the request as the first parameter
+                // TODO: we may want to move this parsing to be returned by the resolveUrl function depending on the redirect URL we've constructed
+                const txId = headers.get('x-arns-resolved-id') ??
+                    redirectUrl.pathname.split('/')[1];
+                // TODO: validate nodes return content length for all responses
+                const contentLength = +(headers.get('content-length') ?? 0);
+                if (!exports.txIdRegex.test(txId)) {
+                    // no transaction id found, skip verification
+                    logger?.debug('No transaction id found, skipping verification', {
+                        redirectUrl,
+                        originalUrl,
+                    });
+                    emitter?.emit('verification-skipped', {
+                        originalUrl,
+                    });
+                    return response;
+                }
+                emitter?.emit('identified-transaction-id', {
+                    originalUrl,
+                    targetGateway: redirectUrl.toString(),
+                    txId,
+                });
+                // parse out the key that contains the response body, we'll use it later when updating the response object
+                const responseDataKey = response.body
+                    ? 'body'
+                    : response.data
+                        ? 'data'
+                        : undefined;
+                if (responseDataKey === undefined) {
+                    throw new Error('No data body or data provided, skipping verification', {
+                        cause: {
+                            redirectUrl: redirectUrl.toString(),
+                            originalUrl: originalUrl.toString(),
+                        },
+                    });
+                }
+                const responseBody = response[responseDataKey];
+                // TODO: determine if it is data item or L1 transaction, and tell the verifier accordingly, just drop in hit to graphql now
+                if (txId === undefined) {
+                    throw new Error('Failed to parse data hash from response headers', {
+                        cause: {
+                            redirectUrl: redirectUrl.toString(),
+                            originalUrl: originalUrl.toString(),
+                            txId,
+                        },
+                    });
+                }
+                else if (responseBody === undefined) {
+                    throw new Error('No data body provided, skipping verification', {
+                        cause: {
+                            redirectUrl: redirectUrl.toString(),
+                            originalUrl: originalUrl.toString(),
+                            txId,
+                        },
+                    });
+                }
+                else {
+                    logger?.debug('Verifying data hash for txId', {
+                        redirectUrl: redirectUrl.toString(),
+                        originalUrl: originalUrl.toString(),
+                        txId,
+                    });
+                    const newClientStream = tapAndVerifyStream({
+                        originalStream: responseBody,
+                        contentLength,
+                        verifyData,
+                        txId,
+                        emitter,
+                    });
+                    if (responseBody instanceof ReadableStream) {
+                        // specific to fetch
+                        return wrapVerifiedResponse(response, newClientStream, txId);
+                    }
+                    else if (responseBody instanceof node_stream_1.Readable) {
+                        // overwrite the response body with the new client stream
+                        response.txId = txId;
+                        response.body = newClientStream;
+                        return response;
+                    }
+                    else {
+                        // TODO: content-application/json and it's smaller than 10mb
+                        // TODO: add tests and verify this works for all non-Readable/streamed responses
+                        try {
+                            // if strict set to true
+                            await verifyData({
+                                data: responseBody,
+                                txId,
+                            });
+                            emitter?.emit('verification-passed', {
+                                txId,
+                            });
+                        }
+                        catch (error) {
+                            logger?.debug('Failed to verify data hash', {
+                                error,
+                                txId,
+                            });
+                            emitter?.emit('verification-failed', {
+                                txId,
+                                error,
+                            });
+                        }
+                        return response;
+                    }
+                }
+            }
+        }
+        // TODO: if strict - wait for verification to finish and succeed before returning the response
         return response;
     };
     return new Proxy(httpClient, {
         // support direct calls: fetch('ar://…', options)
         // axios() or got()
         apply: (_target, _thisArg, argArray) => wayfinderRedirect(httpClient, argArray),
-        // support http clients that use methods like `got.get`, `got.post`, `axios.get`, etc. while still using the wayfinder redirect function
+        // support http clients that use functions like `got.get`, `got.post`, `axios.get`, etc. while still using the wayfinder redirect function
         get: (target, prop, receiver) => {
             const value = Reflect.get(target, prop, receiver);
             if (typeof value === 'function') {
@@ -92,18 +431,32 @@ class Wayfinder {
      *
      * @example
      * const wayfinder = new Wayfinder({
-     *   router: new RandomGatewayRouter({ ario: ARIO.mainnet() }),
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
      * });
+     *
+     * // Returns a target gateway based on the routing strategy
+     * const targetGateway = await wayfinder.router.getTargetGateway();
      */
     router;
     /**
-     * The http client to use for requests
+     * The native http client used by wayfinder
      *
      * @example
      * const wayfinder = new Wayfinder({
-     *   router: new RandomGatewayRouter({ ario: ARIO.mainnet() }),
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
      *   httpClient: axios,
      * });
+     *
      */
     httpClient;
     /**
@@ -111,10 +464,16 @@ class Wayfinder {
      *
      * @example
      * const wayfinder = new Wayfinder({
-     *   router: new RandomGatewayRouter({ ario: ARIO.mainnet() }),
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
      *   httpClient: axios,
      * });
      *
+     * // returns the redirected URL based on the routing strategy and the original url
      * const redirectUrl = await wayfinder.resolveUrl({ originalUrl: 'ar://example' });
      */
     resolveUrl;
@@ -123,12 +482,16 @@ class Wayfinder {
      *
      * @example
      * const { request: wayfind } = new Wayfinder({
-     *   router: new RandomGatewayRouter({ ario: ARIO.mainnet() }),
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
      *   httpClient: axios,
      * });;
      *
-     * TODO: consider a top level function that supports wayfinder routing under the hood
-     * const response = await wayfind('ar://', {
+     * const response = await wayfind('ar://example', {
      *   method: 'POST',
      *   data: {
      *     name: 'John Doe',
@@ -138,30 +501,96 @@ class Wayfinder {
     request;
     // TODO: stats provider
     // TODO: metricsProvider for otel/prom support
-    // TODO: private verificationSettings: {
-    //   trustedGateways: URL[];
-    //   method: 'local' | 'remote';
-    // };
-    constructor({ 
+    verifyData;
+    /**
+     * The event emitter for wayfinder that emits verification events.
+     *
+     * const wayfinder = new Wayfinder()
+     *
+     * wayfinder.emitter.on('verification-passed', (event) => {
+     *   console.log('Verification passed!', event);
+     * })
+     *
+     * wayfinder.emitter.on('verification-failed', (event) => {
+     *   console.log('Verification failed!', event);
+     * })
+     *
+     * or implement the events interface and pass it in, using callback functions
+     *
+     * const wayfinder = new Wayfinder({
+     *   events: {
+     *     onVerificationPassed: (event) => {
+     *       console.log('Verification passed!', event);
+     *     },
+     *     onVerificationFailed: (event) => {
+     *       console.log('Verification failed!', event);
+     *     },
+     *     onVerificationProgress: (event) => {
+     *       console.log('Verification progress!', event);
+     *     },
+     *   }
+     * })
+     *
+     * const response = await wayfind('ar://example');
+     */
+    // TODO: consider changing this to events or event emitter
+    emitter;
+    constructor({ httpClient, 
     // TODO: consider changing router to routingStrategy or strategy
     router = new random_js_1.RandomGatewayRouter({
-        // optionally use a cache gateways provider to reduce the number of requests to the contract
-        gatewaysProvider: new gateways_js_1.ARIOGatewaysProvider({ ario: io_js_1.ARIO.mainnet() }),
-    }), httpClient,
-    // TODO: add verifier interface that provides a verifyDataHash function
+        gatewaysProvider: new gateways_js_1.SimpleCacheGatewaysProvider({
+            gatewaysProvider: new gateways_js_1.NetworkGatewaysProvider({ ario: io_js_1.ARIO.mainnet() }),
+            ttlSeconds: 60 * 60 * 24, // 1 day
+        }),
+    }), logger = logger_js_1.Logger.default, 
+    // TODO: support disabling verification or create some PassThroughVerifier like thing
+    verifier = new hash_verifier_js_1.HashVerifier({
+        trustedHashProvider: new trusted_gateways_js_1.TrustedGatewaysHashProvider({
+            gatewaysProvider: new gateways_js_1.StaticGatewaysProvider({
+                gateways: ['https://permagate.io'],
+            }),
+        }),
+    }), events,
     // TODO: stats provider
      }) {
         this.router = router;
         this.httpClient = httpClient;
-        this.resolveUrl = async ({ originalUrl }) => (0, exports.resolveWayfinderUrl)({
-            originalUrl,
-            targetGateway: await this.router.getTargetGateway(),
-        });
+        this.emitter = new WayfinderEmitter(events);
+        this.verifyData = verifier.verifyData.bind(verifier);
+        this.resolveUrl = async ({ originalUrl, logger }) => {
+            return (0, exports.resolveWayfinderUrl)({
+                originalUrl,
+                targetGateway: async () => await this.router.getTargetGateway(),
+                logger,
+            });
+        };
         this.request = (0, exports.createWayfinderClient)({
             httpClient,
             resolveUrl: this.resolveUrl,
-            // TODO: provide the verifyDataHash function from the verifier to the wayfinder client along with verificationSettings
+            verifyData: this.verifyData,
+            emitter: this.emitter,
+            logger,
         });
+        logger?.debug(`Wayfinder initialized with ${router.name} routing strategy`);
     }
 }
 exports.Wayfinder = Wayfinder;
+// TODO: add a chart for verification strategies and what they do
+// include complexity, performance, and security
+// explain use cases that each strategy is best for
+// e.g.
+/**
+ *
+ *  type    | complexity | performance | security
+ * ---------|------------|-------------|---------
+ *  hash    | low        | high        | low
+ * ---------|------------|-------------|---------
+ *  data root    | medium       | medium      | low | only L1
+ * ---------|------------|-------------|---------
+ * signature    | medium       | medium      | medium
+ * ---------|------------|-------------|---------
+ *  composite | high       | low         | high
+ * ---------|------------|-------------|---------
+ *
+ *
+ */