@ar.io/sdk 3.10.2 → 3.11.0-alpha.10

package/lib/cjs/common/wayfinder/wayfinder.js

@@ -0,0 +1,596 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Wayfinder = exports.createWayfinderClient = exports.WayfinderEmitter = exports.resolveWayfinderUrl = exports.txIdRegex = exports.arnsRegex = void 0;
+exports.tapAndVerifyStream = tapAndVerifyStream;
+exports.wrapVerifiedResponse = wrapVerifiedResponse;
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const node_events_1 = __importDefault(require("node:events"));
+const node_stream_1 = require("node:stream");
+const io_js_1 = require("../io.js");
+const logger_js_1 = require("../logger.js");
+const gateways_js_1 = require("./gateways.js");
+const trusted_gateways_js_1 = require("./gateways/trusted-gateways.js");
+const random_js_1 = require("./routers/random.js");
+const hash_verifier_js_1 = require("./verification/hash-verifier.js");
+// known regexes for wayfinder urls
+exports.arnsRegex = /^[a-z0-9_-]{1,51}$/;
+exports.txIdRegex = /^[A-Za-z0-9_-]{43}$/;
+/**
+ * Core function to resolve a wayfinder url against a target gateway
+ * @param originalUrl - the wayfinder url to resolve
+ * @param targetGateway - the target gateway to resolve the url against
+ * @returns the resolved url that can be used to make a request
+ */
+const resolveWayfinderUrl = async ({ originalUrl, targetGateway, logger, }) => {
+    if (originalUrl.toString().startsWith('ar://')) {
+        logger?.debug(`Applying wayfinder routing protocol to ${originalUrl}`, {
+            originalUrl,
+        });
+        const targetGatewayUrl = new URL(await targetGateway());
+        logger?.debug(`Selected target gateway: ${targetGatewayUrl}`, {
+            originalUrl,
+            targetGateway: targetGatewayUrl,
+        });
+        const [, path] = originalUrl.toString().split('ar://');
+        // e.g. ar:///info should route to the info endpoint of the target gateway
+        if (path.startsWith('/')) {
+            logger?.debug(`Routing to ${path.slice(1)} on ${targetGatewayUrl}`, {
+                originalUrl,
+                targetGateway: targetGatewayUrl,
+            });
+            return new URL(path.slice(1), targetGatewayUrl);
+        }
+        // TODO: this breaks 43 character named arns names - we should check a a local name cache list before resolving raw transaction ids
+        if (exports.txIdRegex.test(path)) {
+            const [txId, ...rest] = path.split('/');
+            return new URL(`${txId}${rest.join('/')}`, targetGatewayUrl);
+        }
+        if (exports.arnsRegex.test(path)) {
+            // TODO: tests to ensure arns names support query params and paths
+            const [name, ...rest] = path.split('/');
+            const arnsUrl = `${targetGatewayUrl.protocol}//${name}.${targetGatewayUrl.hostname}${targetGatewayUrl.port ? `:${targetGatewayUrl.port}` : ''}`;
+            logger?.debug(`Routing to ${path} on ${arnsUrl}`, {
+                originalUrl,
+                targetGateway: targetGatewayUrl,
+            });
+            return new URL(rest.join('/'), arnsUrl);
+        }
+        // TODO: support .eth addresses
+        // TODO: "gasless" routing via DNS TXT records (e.g. ar://gatewaypie.com -> TXT record lookup for TX ID and redirect to that gateway)
+    }
+    logger?.debug('No wayfinder routing protocol applied', {
+        originalUrl,
+    });
+    // return the original url if it's not a wayfinder url (allows you to use the wayfinder client with non-wayfinder urls)
+    return new URL(originalUrl);
+};
+exports.resolveWayfinderUrl = resolveWayfinderUrl;
+class WayfinderEmitter extends node_events_1.default {
+    constructor({ onVerificationPassed, onVerificationFailed, onVerificationProgress,
+    // TODO: continue this pattern for all events
+     } = {}) {
+        super();
+        if (onVerificationPassed) {
+            this.on('verification-passed', onVerificationPassed);
+        }
+        if (onVerificationFailed) {
+            this.on('verification-failed', onVerificationFailed);
+        }
+        if (onVerificationProgress) {
+            this.on('verification-progress', onVerificationProgress);
+        }
+    }
+    emit(event, payload) {
+        return super.emit(event, payload);
+    }
+    on(event, listener) {
+        return super.on(event, listener);
+    }
+}
+exports.WayfinderEmitter = WayfinderEmitter;
+function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, emitter, }) {
+    // taps node streams
+    if (originalStream instanceof node_stream_1.Readable &&
+        typeof originalStream.pipe === 'function') {
+        const tappedClientStream = new node_stream_1.PassThrough();
+        const streamToVerify = new node_stream_1.PassThrough();
+        // kick off the verification promise, this will be awaited when the original stream ends
+        const verificationPromise = verifyData({
+            data: streamToVerify,
+            txId,
+        });
+        let bytesProcessed = 0;
+        // pipe the original stream to the verifier and the client stream
+        originalStream.on('data', (chunk) => {
+            streamToVerify.write(chunk);
+            tappedClientStream.write(chunk);
+            bytesProcessed += chunk.length;
+            // only emit if contentLength is not 0
+            if (contentLength !== 0) {
+                emitter?.emit('verification-progress', {
+                    txId,
+                    totalBytes: contentLength,
+                    processedBytes: bytesProcessed,
+                });
+            }
+        });
+        originalStream.on('end', async () => {
+            streamToVerify.end(); // triggers verifier completion and completes the verification promise
+            try {
+                await verificationPromise;
+                emitter?.emit('verification-passed', {
+                    txId,
+                });
+                tappedClientStream.end();
+            }
+            catch (error) {
+                emitter?.emit('verification-failed', {
+                    error,
+                    txId,
+                });
+                tappedClientStream.destroy(error);
+            }
+        });
+        originalStream.on('error', (err) => {
+            emitter?.emit('verification-failed', {
+                error: err,
+                txId,
+            });
+            streamToVerify.destroy(err);
+            tappedClientStream.destroy(err);
+        });
+        // send the stream to the verify function and if it errors end the client stream
+        return tappedClientStream;
+    }
+    // taps web readable streams
+    if (originalStream instanceof ReadableStream &&
+        typeof originalStream.tee === 'function') {
+        const [verifyBranch, clientBranch] = originalStream.tee();
+        // setup our promise to verify the data
+        const verificationPromise = verifyData({
+            data: verifyBranch,
+            txId,
+        });
+        let bytesProcessed = 0;
+        const reader = clientBranch.getReader();
+        const clientStreamWithVerification = new ReadableStream({
+            async pull(controller) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    try {
+                        // due to backpressure, if the client does not consume the stream, the verification will not complete (particularly important for fetch, where the response body needs to be awaited for verification to complete)
+                        await verificationPromise;
+                        emitter?.emit('verification-passed', {
+                            txId,
+                        });
+                        controller.close();
+                    }
+                    catch (err) {
+                        emitter?.emit('verification-failed', {
+                            txId,
+                            error: err,
+                        });
+                        controller.error(err);
+                    }
+                }
+                else {
+                    bytesProcessed += value.length;
+                    emitter?.emit('verification-progress', {
+                        txId,
+                        totalBytes: contentLength,
+                        processedBytes: bytesProcessed,
+                    });
+                    controller.enqueue(value);
+                }
+            },
+            cancel(reason) {
+                reader.cancel(reason);
+                emitter?.emit('verification-failed', {
+                    txId,
+                    error: new Error('Verification cancelled', {
+                        cause: {
+                            reason,
+                        },
+                    }),
+                });
+            },
+        });
+        return clientStreamWithVerification;
+    }
+    throw new Error('Unsupported body type for cloning');
+}
+function wrapVerifiedResponse(original, newBody, txId) {
+    // Clone headers (Header objects aren't serializable)
+    const headers = new Headers();
+    original.headers.forEach((value, key) => headers.set(key, value));
+    // Create a new Response with the new body and cloned headers
+    const wrapped = new Response(newBody, {
+        status: original.status,
+        statusText: original.statusText,
+        headers,
+    });
+    // Attach txId for downstream tracking
+    wrapped.txId = txId;
+    wrapped.redirectedFrom = original.url;
+    return wrapped;
+}
+/**
+ * Creates a wrapped http client that supports ar:// protocol
+ *
+ * This function leverages a Proxy to intercept calls to the http client
+ * and redirects them to the target gateway using the resolveUrl function url.
+ * It also supports the http client methods like get(), post(), put(), delete(), etc.
+ *
+ * Any URLs provided that are not wayfinder urls will be returned as is.
+ *
+ * @param httpClient - the http client to wrap (e.g. axios, fetch, got, etc.)
+ * @param resolveUrl - the function to construct the redirect url for ar:// requests
+ * @returns a wrapped http client that supports ar:// protocol
+ */
+const createWayfinderClient = ({ httpClient, resolveUrl, verifyData, emitter = new WayfinderEmitter(), logger, }) => {
+    const wayfinderRedirect = async (fn, rawArgs) => {
+        // TODO: handle if first arg is not a string (i.e. just return the result of the function call)
+        const [originalUrl, ...rest] = rawArgs;
+        if (typeof originalUrl !== 'string') {
+            logger?.debug('Original URL is not a string, skipping routing', {
+                originalUrl,
+            });
+            return fn(...rawArgs);
+        }
+        emitter?.emit('routing-started', {
+            originalUrl: originalUrl.toString(),
+        });
+        // route the request to the target gateway
+        const redirectUrl = await resolveUrl({
+            originalUrl,
+            logger,
+        });
+        emitter?.emit('routing-succeeded', {
+            originalUrl,
+            targetGateway: redirectUrl.toString(),
+        });
+        logger?.debug(`Redirecting request to ${redirectUrl}`, {
+            originalUrl,
+            redirectUrl,
+        });
+        // make the request to the target gateway using the redirect url and http client
+        const response = await fn(redirectUrl.toString(), ...rest);
+        // TODO: trigger a routing event with the raw response object?
+        logger?.debug(`Successfully routed request to ${redirectUrl}`, {
+            redirectUrl,
+            originalUrl,
+        });
+        // only verify data if the redirect url is different from the original url
+        if (response && redirectUrl.toString() !== originalUrl.toString()) {
+            if (verifyData) {
+                // if the headers do not have .get on them, we need to parse the headers manually
+                const headers = new Headers();
+                let headersObject = response.headers ?? {};
+                if (typeof headersObject.get !== 'function') {
+                    headersObject = Object.fromEntries(headersObject);
+                    for (const [key, value] of Object.entries(headersObject)) {
+                        headers.set(key, value);
+                    }
+                }
+                else {
+                    for (const [key, value] of headersObject.entries()) {
+                        headers.set(key, value);
+                    }
+                }
+                // transaction id is either in the response headers or the path of the request as the first parameter
+                // TODO: we may want to move this parsing to be returned by the resolveUrl function depending on the redirect URL we've constructed
+                const txId = headers.get('x-arns-resolved-id') ??
+                    redirectUrl.pathname.split('/')[1];
+                // TODO: validate nodes return content length for all responses
+                const contentLength = +(headers.get('content-length') ?? 0);
+                if (!exports.txIdRegex.test(txId)) {
+                    // no transaction id found, skip verification
+                    logger?.debug('No transaction id found, skipping verification', {
+                        redirectUrl,
+                        originalUrl,
+                    });
+                    emitter?.emit('verification-skipped', {
+                        originalUrl,
+                    });
+                    return response;
+                }
+                emitter?.emit('identified-transaction-id', {
+                    originalUrl,
+                    targetGateway: redirectUrl.toString(),
+                    txId,
+                });
+                // parse out the key that contains the response body, we'll use it later when updating the response object
+                const responseDataKey = response.body
+                    ? 'body'
+                    : response.data
+                        ? 'data'
+                        : undefined;
+                if (responseDataKey === undefined) {
+                    throw new Error('No data body or data provided, skipping verification', {
+                        cause: {
+                            redirectUrl: redirectUrl.toString(),
+                            originalUrl: originalUrl.toString(),
+                        },
+                    });
+                }
+                const responseBody = response[responseDataKey];
+                // TODO: determine if it is data item or L1 transaction, and tell the verifier accordingly, just drop in hit to graphql now
+                if (txId === undefined) {
+                    throw new Error('Failed to parse data hash from response headers', {
+                        cause: {
+                            redirectUrl: redirectUrl.toString(),
+                            originalUrl: originalUrl.toString(),
+                            txId,
+                        },
+                    });
+                }
+                else if (responseBody === undefined) {
+                    throw new Error('No data body provided, skipping verification', {
+                        cause: {
+                            redirectUrl: redirectUrl.toString(),
+                            originalUrl: originalUrl.toString(),
+                            txId,
+                        },
+                    });
+                }
+                else {
+                    logger?.debug('Verifying data hash for txId', {
+                        redirectUrl: redirectUrl.toString(),
+                        originalUrl: originalUrl.toString(),
+                        txId,
+                    });
+                    const newClientStream = tapAndVerifyStream({
+                        originalStream: responseBody,
+                        contentLength,
+                        verifyData,
+                        txId,
+                        emitter,
+                    });
+                    if (responseBody instanceof ReadableStream) {
+                        // specific to fetch
+                        return wrapVerifiedResponse(response, newClientStream, txId);
+                    }
+                    else if (responseBody instanceof node_stream_1.Readable) {
+                        // overwrite the response body with the new client stream
+                        response.txId = txId;
+                        response.body = newClientStream;
+                        return response;
+                    }
+                    else {
+                        // TODO: content-application/json and it's smaller than 10mb
+                        // TODO: add tests and verify this works for all non-Readable/streamed responses
+                        try {
+                            // if strict set to true
+                            await verifyData({
+                                data: responseBody,
+                                txId,
+                            });
+                            emitter?.emit('verification-passed', {
+                                txId,
+                            });
+                        }
+                        catch (error) {
+                            logger?.debug('Failed to verify data hash', {
+                                error,
+                                txId,
+                            });
+                            emitter?.emit('verification-failed', {
+                                txId,
+                                error,
+                            });
+                        }
+                        return response;
+                    }
+                }
+            }
+        }
+        // TODO: if strict - wait for verification to finish and succeed before returning the response
+        return response;
+    };
+    return new Proxy(httpClient, {
+        // support direct calls: fetch('ar://…', options)
+        // axios() or got()
+        apply: (_target, _thisArg, argArray) => wayfinderRedirect(httpClient, argArray),
+        // support http clients that use functions like `got.get`, `got.post`, `axios.get`, etc. while still using the wayfinder redirect function
+        get: (target, prop, receiver) => {
+            const value = Reflect.get(target, prop, receiver);
+            if (typeof value === 'function') {
+                return (...inner) => wayfinderRedirect(value.bind(target), inner);
+            }
+            return value; // numbers, objects, symbols pass through untouched
+        },
+    });
+};
+exports.createWayfinderClient = createWayfinderClient;
+/**
+ * The main class for the wayfinder
+ * @param router - the router to use for requests
+ * @param httpClient - the http client to use for requests
+ * @param blocklist - the blocklist of gateways to avoid
+ */
+class Wayfinder {
+    /**
+     * The router to use for requests
+     *
+     * @example
+     * const wayfinder = new Wayfinder({
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
+     * });
+     *
+     * // Returns a target gateway based on the routing strategy
+     * const targetGateway = await wayfinder.router.getTargetGateway();
+     */
+    router;
+    /**
+     * The native http client used by wayfinder
+     *
+     * @example
+     * const wayfinder = new Wayfinder({
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
+     *   httpClient: axios,
+     * });
+     *
+     */
+    httpClient;
+    /**
+     * The function that resolves the redirect url for ar:// requests to a target gateway
+     *
+     * @example
+     * const wayfinder = new Wayfinder({
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
+     *   httpClient: axios,
+     * });
+     *
+     * // returns the redirected URL based on the routing strategy and the original url
+     * const redirectUrl = await wayfinder.resolveUrl({ originalUrl: 'ar://example' });
+     */
+    resolveUrl;
+    /**
+     * A wrapped http client that supports ar:// protocol
+     *
+     * @example
+     * const { request: wayfind } = new Wayfinder({
+     *   router: new RandomGatewayRouter({
+     *     gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *       gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *       ttlSeconds: 60 * 60 * 24, // 1 day
+     *     }),
+     *   }),
+     *   httpClient: axios,
+     * });;
+     *
+     * const response = await wayfind('ar://example', {
+     *   method: 'POST',
+     *   data: {
+     *     name: 'John Doe',
+     *   },
+     * })
+     */
+    request;
+    // TODO: stats provider
+    // TODO: metricsProvider for otel/prom support
+    verifyData;
+    /**
+     * The event emitter for wayfinder that emits verification events.
+     *
+     * const wayfinder = new Wayfinder()
+     *
+     * wayfinder.emitter.on('verification-passed', (event) => {
+     *   console.log('Verification passed!', event);
+     * })
+     *
+     * wayfinder.emitter.on('verification-failed', (event) => {
+     *   console.log('Verification failed!', event);
+     * })
+     *
+     * or implement the events interface and pass it in, using callback functions
+     *
+     * const wayfinder = new Wayfinder({
+     *   events: {
+     *     onVerificationPassed: (event) => {
+     *       console.log('Verification passed!', event);
+     *     },
+     *     onVerificationFailed: (event) => {
+     *       console.log('Verification failed!', event);
+     *     },
+     *     onVerificationProgress: (event) => {
+     *       console.log('Verification progress!', event);
+     *     },
+     *   }
+     * })
+     *
+     * const response = await wayfind('ar://example');
+     */
+    // TODO: consider changing this to events or event emitter
+    emitter;
+    constructor({ httpClient, 
+    // TODO: consider changing router to routingStrategy or strategy
+    router = new random_js_1.RandomGatewayRouter({
+        gatewaysProvider: new gateways_js_1.SimpleCacheGatewaysProvider({
+            gatewaysProvider: new gateways_js_1.NetworkGatewaysProvider({ ario: io_js_1.ARIO.mainnet() }),
+            ttlSeconds: 60 * 60 * 24, // 1 day
+        }),
+    }), logger = logger_js_1.Logger.default, 
+    // TODO: support disabling verification or create some PassThroughVerifier like thing
+    verifier = new hash_verifier_js_1.HashVerifier({
+        trustedHashProvider: new trusted_gateways_js_1.TrustedGatewaysHashProvider({
+            gatewaysProvider: new gateways_js_1.StaticGatewaysProvider({
+                gateways: ['https://permagate.io'],
+            }),
+        }),
+    }), events,
+    // TODO: stats provider
+     }) {
+        this.router = router;
+        this.httpClient = httpClient;
+        this.emitter = new WayfinderEmitter(events);
+        this.verifyData = verifier.verifyData.bind(verifier);
+        this.resolveUrl = async ({ originalUrl, logger }) => {
+            return (0, exports.resolveWayfinderUrl)({
+                originalUrl,
+                targetGateway: async () => await this.router.getTargetGateway(),
+                logger,
+            });
+        };
+        this.request = (0, exports.createWayfinderClient)({
+            httpClient,
+            resolveUrl: this.resolveUrl,
+            verifyData: this.verifyData,
+            emitter: this.emitter,
+            logger,
+        });
+        logger?.debug(`Wayfinder initialized with ${router.name} routing strategy`);
+    }
+}
+exports.Wayfinder = Wayfinder;
+// TODO: add a chart for verification strategies and what they do
+// include complexity, performance, and security
+// explain use cases that each strategy is best for
+// e.g.
+/**
+ *
+ *  type    | complexity | performance | security
+ * ---------|------------|-------------|---------
+ *  hash    | low        | high        | low
+ * ---------|------------|-------------|---------
+ *  data root    | medium       | medium      | low | only L1
+ * ---------|------------|-------------|---------
+ * signature    | medium       | medium      | medium
+ * ---------|------------|-------------|---------
+ *  composite | high       | low         | high
+ * ---------|------------|-------------|---------
+ *
+ *
+ */