@ar.io/sdk 3.12.2 → 3.13.0-beta.1

package/lib/cjs/common/wayfinder/verification/trusted.js

@@ -0,0 +1,125 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TrustedGatewaysHashProvider = void 0;
+const wayfinder_js_1 = require("../wayfinder.js");
+const arioGatewayHeaders = {
+    digest: 'x-ar-io-digest',
+    verified: 'x-ar-io-verified',
+    txId: 'x-arns-resolved-tx-id',
+    processId: 'x-arns-resolved-process-id',
+};
+class TrustedGatewaysHashProvider {
+    gatewaysProvider;
+    constructor({ gatewaysProvider,
+    // TODO: add threshold for allowed hash difference (i.e. by count or ratio of total gateways checked)
+     }) {
+        this.gatewaysProvider = gatewaysProvider;
+    }
+    /**
+     * Gets the digest for a given txId from all trusted gateways and ensures they all match.
+     * @param txId - The txId to get the digest for.
+     * @returns The digest for the given txId.
+     */
+    async getHash({ txId, }) {
+        // get the hash from every gateway, and ensure they all match
+        const hashSet = new Set();
+        const hashResults = [];
+        const gateways = await this.gatewaysProvider.getGateways();
+        const hashes = await Promise.all(gateways.map(async (gateway) => {
+            const sandbox = (0, wayfinder_js_1.sandboxFromId)(txId);
+            const urlWithSandbox = `${gateway.protocol}//${sandbox}.${gateway.hostname}/${txId}`;
+            let txIdHash;
+            /**
+             * This is a problem because we're not able to verify the hash of the data item if the gateway doesn't have the data in its cache.
+             * We should add the ability to send a HEAD request to trigger a GET request to hydrate the cache on the trusted gateway via a header.
+             * For now, we'll just do a GET request to hydrate the cache if the HEAD request doesn't contain the digest.
+             */
+            for (const method of ['GET', 'GET']) {
+                const response = await fetch(urlWithSandbox, {
+                    method,
+                    redirect: 'follow',
+                    mode: 'cors',
+                    headers: {
+                        'Cache-Control': 'no-cache',
+                    },
+                });
+                if (!response.ok) {
+                    // skip if the request failed or the digest is not present
+                    return;
+                }
+                const fetchedTxIdHash = response.headers.get(arioGatewayHeaders.digest);
+                if (fetchedTxIdHash !== null && fetchedTxIdHash !== undefined) {
+                    txIdHash = fetchedTxIdHash;
+                    break;
+                }
+            }
+            if (txIdHash === undefined) {
+                // skip this gateway if we didn't get a hash
+                return undefined;
+            }
+            hashResults.push({
+                gateway: gateway.hostname,
+                txIdHash,
+            });
+            return txIdHash;
+        }));
+        for (const hash of hashes) {
+            if (hash !== undefined) {
+                hashSet.add(hash);
+            }
+        }
+        if (hashSet.size === 0) {
+            throw new Error(`No trusted gateways returned a hash for txId ${txId}`);
+        }
+        if (hashSet.size > 1) {
+            throw new Error(`Failed to get consistent hash from all trusted gateways. ${JSON.stringify(hashResults)}`);
+        }
+        return { hash: hashResults[0].txIdHash, algorithm: 'sha256' };
+    }
+    /**
+     * Get the data root for a given txId from all trusted gateways and ensure they all match.
+     * @param txId - The txId to get the data root for.
+     * @returns The data root for the given txId.
+     */
+    async getDataRoot({ txId }) {
+        const dataRootSet = new Set();
+        const dataRootResults = [];
+        const gateways = await this.gatewaysProvider.getGateways();
+        const dataRoots = await Promise.all(gateways.map(async (gateway) => {
+            const response = await fetch(`${gateway.toString()}tx/${txId}/data_root`);
+            if (!response.ok) {
+                // skip this gateway
+                return undefined;
+            }
+            const dataRoot = await response.text();
+            dataRootResults.push({
+                gateway: gateway.hostname,
+                dataRoot,
+            });
+            return dataRoot;
+        }));
+        for (const dataRoot of dataRoots) {
+            if (dataRoot !== undefined) {
+                dataRootSet.add(dataRoot);
+            }
+        }
+        if (dataRootSet.size > 1) {
+            throw new Error(`Failed to get consistent data root from all trusted gateways. ${JSON.stringify(dataRootResults)}`);
+        }
+        return dataRootSet.values().next().value;
+    }
+}
+exports.TrustedGatewaysHashProvider = TrustedGatewaysHashProvider;
+// client could check hashes of data items, match expected hash
+// if the gateway has the hash and they've verified it, you can trust the data item and offset
+// you would be only trusting the gateway that it is a valid bundle
+// you can request the offset from the gateway to verify the id
+/**
+ * Note from @djwhitt
+ *
+ * Calculating data roots this way is fine, but it may not reproduce the original data root.
+ * We could also implement a data root verifier that pulls all the chunks, checks that they
+ * reproduce the expected data root, and then compares the concatenated chunk data to the
+ * original data retrieved. That would take a while, but it should be able to verify any L1
+ * data where we can find the chunks.
+ */

package/lib/cjs/common/wayfinder/wayfinder.js

@@ -0,0 +1,508 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Wayfinder = exports.createWayfinderClient = exports.WayfinderEmitter = exports.resolveWayfinderUrl = exports.txIdRegex = exports.arnsRegex = void 0;
+exports.tapAndVerifyReadableStream = tapAndVerifyReadableStream;
+exports.sandboxFromId = sandboxFromId;
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const eventemitter3_1 = require("eventemitter3");
+const rfc4648_1 = require("rfc4648");
+const io_js_1 = require("../io.js");
+const logger_js_1 = require("../logger.js");
+const network_js_1 = require("./gateways/network.js");
+const simple_cache_js_1 = require("./gateways/simple-cache.js");
+const static_js_1 = require("./gateways/static.js");
+const ping_js_1 = require("./routing/strategies/ping.js");
+const hash_verifier_js_1 = require("./verification/strategies/hash-verifier.js");
+const trusted_js_1 = require("./verification/trusted.js");
+// known regexes for wayfinder urls
+exports.arnsRegex = /^[a-z0-9_-]{1,51}$/;
+exports.txIdRegex = /^[A-Za-z0-9_-]{43}$/;
+/**
+ * Core function that converts a wayfinder url to the proper ar-io gateway URL
+ * @param originalUrl - the wayfinder url to resolve
+ * @param selectedGateway - the target gateway to resolve the url against
+ * @returns the resolved url that can be used to make a request
+ */
+const resolveWayfinderUrl = ({ originalUrl, selectedGateway, logger, }) => {
+    if (originalUrl.toString().startsWith('ar://')) {
+        logger?.debug(`Applying wayfinder routing protocol to ${originalUrl}`, {
+            originalUrl,
+        });
+        const [, path] = originalUrl.toString().split('ar://');
+        // e.g. ar:///info should route to the info endpoint of the target gateway
+        if (path.startsWith('/')) {
+            logger?.debug(`Routing to ${path.slice(1)} on ${selectedGateway}`, {
+                originalUrl,
+                selectedGateway,
+            });
+            return new URL(path.slice(1), selectedGateway);
+        }
+        // Split path to get the first part (name/txId) and remaining path components
+        const [firstPart, ...rest] = path.split('/');
+        // TODO: this breaks 43 character named arns names - we should check a a local name cache list before resolving raw transaction ids
+        if (exports.txIdRegex.test(firstPart)) {
+            const sandbox = sandboxFromId(firstPart);
+            return new URL(`${firstPart}${rest.length > 0 ? '/' + rest.join('/') : ''}`, `${selectedGateway.protocol}//${sandbox}.${selectedGateway.hostname}`);
+        }
+        if (exports.arnsRegex.test(firstPart)) {
+            // TODO: tests to ensure arns names support query params and paths
+            const arnsUrl = `${selectedGateway.protocol}//${firstPart}.${selectedGateway.hostname}${selectedGateway.port ? `:${selectedGateway.port}` : ''}`;
+            logger?.debug(`Routing to ${path} on ${arnsUrl}`, {
+                originalUrl,
+                selectedGateway,
+            });
+            return new URL(rest.length > 0 ? rest.join('/') : '', arnsUrl);
+        }
+        // TODO: support .eth addresses
+        // TODO: "gasless" routing via DNS TXT records
+    }
+    logger?.debug('No wayfinder routing protocol applied', {
+        originalUrl,
+    });
+    // return the original url if it's not a wayfinder url
+    return new URL(originalUrl);
+};
+exports.resolveWayfinderUrl = resolveWayfinderUrl;
+class WayfinderEmitter extends eventemitter3_1.EventEmitter {
+    constructor({ onVerificationSucceeded, onVerificationFailed, onVerificationProgress, } = {}) {
+        super();
+        if (onVerificationSucceeded) {
+            this.on('verification-succeeded', onVerificationSucceeded);
+        }
+        if (onVerificationFailed) {
+            this.on('verification-failed', onVerificationFailed);
+        }
+        if (onVerificationProgress) {
+            this.on('verification-progress', onVerificationProgress);
+        }
+    }
+}
+exports.WayfinderEmitter = WayfinderEmitter;
+function tapAndVerifyReadableStream({ originalStream, contentLength, verifyData, txId, emitter, strict = false, }) {
+    if (originalStream instanceof ReadableStream &&
+        typeof originalStream.tee === 'function') {
+        /**
+         * NOTE: tee requires the streams both streams to be consumed, so we need to make sure we consume the client branch
+         * by the caller. This means when `request` is called, the client stream must be consumed by the caller via await request.text()
+         * for verification to complete.
+         *
+         * It is feasible to make the verification stream not to depend on the client branch being consumed, should the DX not be obvious.
+         */
+        const [verifyBranch, clientBranch] = originalStream.tee();
+        // setup our promise to verify the data
+        const verificationPromise = verifyData({
+            data: verifyBranch,
+            txId,
+        });
+        let bytesProcessed = 0;
+        const reader = clientBranch.getReader();
+        const clientStreamWithVerification = new ReadableStream({
+            async pull(controller) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    if (strict) {
+                        // in strict mode, we wait for verification to complete before closing the controller
+                        try {
+                            await verificationPromise;
+                            emitter?.emit('verification-succeeded', { txId });
+                            controller.close();
+                        }
+                        catch (err) {
+                            // emit the verification failed event
+                            emitter?.emit('verification-failed', err);
+                            // In strict mode, we report the error to the client stream
+                            controller.error(new Error('Verification failed', { cause: err }));
+                        }
+                    }
+                    else {
+                        // trigger the verification promise and emit events for the result
+                        verificationPromise
+                            .then(() => {
+                            emitter?.emit('verification-succeeded', { txId });
+                        })
+                            .catch((error) => {
+                            emitter?.emit('verification-failed', error);
+                        });
+                        // in non-strict mode, we close the controller immediately and handle verification asynchronously
+                        controller.close();
+                    }
+                }
+                else {
+                    bytesProcessed += value.length;
+                    emitter?.emit('verification-progress', {
+                        txId,
+                        totalBytes: contentLength,
+                        processedBytes: bytesProcessed,
+                    });
+                    controller.enqueue(value);
+                }
+            },
+            cancel(reason) {
+                // cancel the reader regardless of verification status
+                reader.cancel(reason);
+                // emit the verification cancellation event
+                emitter?.emit('verification-failed', {
+                    txId,
+                    error: new Error('Verification cancelled', {
+                        cause: {
+                            reason,
+                        },
+                    }),
+                });
+            },
+        });
+        return clientStreamWithVerification;
+    }
+    throw new Error('Unsupported body type for cloning');
+}
+/**
+ * Gets the sandbox hash for a given transaction id
+ */
+function sandboxFromId(id) {
+    return rfc4648_1.base32
+        .stringify(Buffer.from(id, 'base64'), { pad: false })
+        .toLowerCase();
+}
+/**
+ * Creates a wrapped fetch function that supports ar:// protocol
+ *
+ * This function leverages a Proxy to intercept calls to fetch
+ * and redirects them to the target gateway using the resolveUrl function.
+ *
+ * Any URLs provided that are not wayfinder urls will be passed directly to fetch.
+ *
+ * @param resolveUrl - the function to construct the redirect url for ar:// requests
+ * @returns a wrapped fetch function that supports ar:// protocol and always returns Response
+ */
+const createWayfinderClient = ({ resolveUrl, verifyData, selectGateway, emitter = new WayfinderEmitter(), logger, strict = false, }) => {
+    return async (url, init) => {
+        if (typeof url !== 'string') {
+            logger?.debug('URL is not a string, skipping routing', {
+                url,
+            });
+            emitter?.emit('routing-skipped', {
+                originalUrl: JSON.stringify(url),
+            });
+            return fetch(url, init);
+        }
+        emitter?.emit('routing-started', {
+            originalUrl: url.toString(),
+        });
+        const maxRetries = 3;
+        const retryDelay = 1000;
+        for (let i = 0; i < maxRetries; i++) {
+            try {
+                // select the target gateway
+                const selectedGateway = await selectGateway();
+                logger?.debug('Selected gateway', {
+                    originalUrl: url,
+                    selectedGateway: selectedGateway.toString(),
+                });
+                // route the request to the target gateway
+                const redirectUrl = resolveUrl({
+                    originalUrl: url,
+                    selectedGateway,
+                    logger,
+                });
+                emitter?.emit('routing-succeeded', {
+                    originalUrl: url,
+                    selectedGateway: selectedGateway.toString(),
+                    redirectUrl: redirectUrl.toString(),
+                });
+                logger?.debug(`Redirecting request`, {
+                    originalUrl: url,
+                    redirectUrl: redirectUrl.toString(),
+                });
+                // make the request to the target gateway using the redirect url
+                const response = await fetch(redirectUrl.toString(), {
+                    // enforce CORS given we're likely going to a different origin, but always allow the client to override
+                    redirect: 'follow',
+                    mode: 'cors',
+                    ...init,
+                });
+                logger?.debug(`Successfully routed request to gateway`, {
+                    redirectUrl: redirectUrl.toString(),
+                    originalUrl: url.toString(),
+                });
+                // only verify data if the redirect url is different from the original url
+                if (redirectUrl.toString() !== url.toString()) {
+                    if (verifyData) {
+                        const headers = response.headers;
+                        // transaction id is either in the response headers or the path of the request as the first parameter
+                        const txId = headers.get('x-arns-resolved-id') ??
+                            redirectUrl.pathname.split('/')[1];
+                        const contentLength = +(headers.get('content-length') ?? 0);
+                        if (!exports.txIdRegex.test(txId)) {
+                            // no transaction id found, skip verification
+                            logger?.debug('No transaction id found, skipping verification', {
+                                redirectUrl: redirectUrl.toString(),
+                                originalUrl: url,
+                            });
+                            emitter?.emit('verification-skipped', {
+                                originalUrl: url,
+                            });
+                            return response;
+                        }
+                        emitter?.emit('identified-transaction-id', {
+                            originalUrl: url,
+                            selectedGateway: redirectUrl.toString(),
+                            txId,
+                        });
+                        // Check if the response has a body
+                        if (response.body) {
+                            const newClientStream = tapAndVerifyReadableStream({
+                                originalStream: response.body,
+                                contentLength,
+                                verifyData,
+                                txId,
+                                emitter,
+                                strict,
+                            });
+                            return new Response(newClientStream, {
+                                status: response.status,
+                                statusText: response.statusText,
+                                headers: response.headers,
+                            });
+                        }
+                        else {
+                            // No response body to verify, skip verification
+                            logger?.debug('No response body to verify', {
+                                redirectUrl: redirectUrl.toString(),
+                                originalUrl: url,
+                                txId,
+                            });
+                            return response;
+                        }
+                    }
+                }
+                return response;
+            }
+            catch (error) {
+                logger?.debug('Failed to route request', {
+                    error: error.message,
+                    stack: error.stack,
+                    originalUrl: url,
+                    attempt: i + 1,
+                    maxRetries,
+                });
+                if (i < maxRetries - 1) {
+                    await new Promise((resolve) => setTimeout(resolve, retryDelay));
+                }
+            }
+        }
+        throw new Error('Failed to route request after max retries', {
+            cause: {
+                originalUrl: url,
+                maxRetries,
+            },
+        });
+    };
+};
+exports.createWayfinderClient = createWayfinderClient;
+/**
+ * The main class for the wayfinder
+ */
+class Wayfinder {
+    /**
+     * The gateways provider is responsible for providing the list of gateways to use for routing requests.
+     *
+     * @example
+     * const wayfinder = new Wayfinder({
+     *   gatewaysProvider: new SimpleCacheGatewaysProvider({
+     *     gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
+     *     ttlSeconds: 60 * 60 * 24, // 1 day
+     *   }),
+     * });
+     */
+    gatewaysProvider;
+    /**
+     * The routing strategy to use when routing requests.
+     *
+     * @example
+     * const wayfinder = new Wayfinder({
+     *   strategy: new FastestPingStrategy({
+     *     timeoutMs: 1000,
+     *   }),
+     * });
+     */
+    routingStrategy;
+    /**
+     * A helper function that resolves the redirect url for ar:// requests to a target gateway.
+     *
+     * Note: no verification is done when resolving an ar://<path> url to a wayfinder route.
+     * In order to verify the data, you must use the `request` function or request the data and
+     * verify it yourself via the `verifyData` function.
+     *
+     * @example
+     * const { resolveUrl } = new Wayfinder();
+     *
+     * // returns the redirected URL based on the routing strategy and the original url
+     * const redirectUrl = await resolveUrl({ originalUrl: 'ar://example' });
+     *
+     * window.open(redirectUrl.toString(), '_blank');
+     */
+    resolveUrl;
+    /**
+     * A wrapped fetch function that supports ar:// protocol. If a verification strategy is provided,
+     * the request will be verified and events will be emitted as the request is processed.
+     *
+     * @example
+     * const wayfinder = new Wayfinder({
+     *   verificationStrategy: new HashVerificationStrategy({
+     *     trustedHashProvider: new TrustedGatewaysHashProvider({
+     *       gatewaysProvider: new StaticGatewaysProvider({
+     *         gateways: ['https://permagate.io'],
+     *       }),
+     *     }),
+     *   }),
+     * })
+     *
+     * // request an arns name
+     * const response = await wayfinder.request('ar://ardrive')
+     *
+     * // request a transaction id
+     * const response = await wayfinder.request('ar://1234567890')
+     *
+     * // Set strict mode to true to make verification blocking
+     * const wayfinder = new Wayfinder({
+     *   strict: true,
+     * });
+     *
+     * // This will throw an error if verification fails
+     * try {
+     *   const response = await wayfinder.request('ar://1234567890');
+     * } catch (error) {
+     *   console.error('Verification failed', error);
+     * }
+     */
+    request;
+    /**
+     * The function that verifies the data hash for a given transaction id.
+     *
+     * @example
+     * const wayfinder = new Wayfinder({
+     *   verifyData: (data, txId) => {
+     *     // some custom verification logic
+     *     return true;
+     *   },
+     * });
+     */
+    verifyData;
+    /**
+     * Whether verification should be strict (blocking) or not.
+     * If true, verification failures will cause requests to fail.
+     * If false, verification will be performed asynchronously and failures will only emit events.
+     */
+    strict;
+    /**
+     * The event emitter for wayfinder that emits verification events.
+     *
+     * const wayfinder = new Wayfinder()
+     *
+     * wayfinder.emitter.on('verification-succeeded', (event) => {
+     *   console.log('Verification passed!', event);
+     * })
+     *
+     * wayfinder.emitter.on('verification-failed', (event) => {
+     *   console.log('Verification failed!', event);
+     * })
+     *
+     * or implement the events interface and pass it in, using callback functions
+     *
+     * const wayfinder = new Wayfinder({
+     *   events: {
+     *     onVerificationPassed: (event) => {
+     *       console.log('Verification passed!', event);
+     *     },
+     *     onVerificationFailed: (event) => {
+     *       console.log('Verification failed!', event);
+     *     },
+     *     onVerificationProgress: (event) => {
+     *       console.log('Verification progress!', event);
+     *     },
+     *   }
+     * })
+     *
+     * const response = await wayfind('ar://example');
+     */
+    emitter;
+    /**
+     * The constructor for the wayfinder
+     * @param routingStrategy - the routing strategy to use for requests
+     * @param verificationStrategy - the verification strategy to use for requests
+     * @param gatewaysProvider - the gateways provider to use for routing requests
+     * @param logger - the logger to use for logging
+     * @param strict - if true, verification will be blocking and will fail requests if verification fails; if false, verification will be non-blocking
+     */
+    constructor({ logger = logger_js_1.Logger.default, gatewaysProvider = new simple_cache_js_1.SimpleCacheGatewaysProvider({
+        gatewaysProvider: new network_js_1.NetworkGatewaysProvider({
+            ario: io_js_1.ARIO.mainnet(),
+        }),
+        ttlSeconds: 60 * 60, // 1 hour
+    }), routingStrategy = new ping_js_1.FastestPingRoutingStrategy({
+        timeoutMs: 1000,
+    }), verificationStrategy = new hash_verifier_js_1.HashVerificationStrategy({
+        trustedHashProvider: new trusted_js_1.TrustedGatewaysHashProvider({
+            gatewaysProvider: new static_js_1.StaticGatewaysProvider({
+                gateways: ['https://permagate.io'],
+            }),
+        }),
+    }), events = {
+        onVerificationSucceeded: (event) => {
+            logger.debug('Verification passed!', event);
+        },
+        onVerificationFailed: (event) => {
+            logger.error('Verification failed!', event);
+        },
+        onVerificationProgress: (event) => {
+            logger.debug('Verification progress!', event);
+        },
+    }, strict = false, } = {}) {
+        this.routingStrategy = routingStrategy;
+        this.gatewaysProvider = gatewaysProvider;
+        this.emitter = new WayfinderEmitter(events);
+        this.verifyData =
+            verificationStrategy.verifyData.bind(verificationStrategy);
+        this.strict = strict;
+        // top level function to easily resolve wayfinder urls using the routing strategy and gateways provider
+        this.resolveUrl = async ({ originalUrl, logger }) => {
+            const selectedGateway = await this.routingStrategy.selectGateway({
+                gateways: await this.gatewaysProvider.getGateways(),
+            });
+            return (0, exports.resolveWayfinderUrl)({
+                originalUrl,
+                selectedGateway,
+                logger,
+            });
+        };
+        // create a wayfinder client with the routing strategy and gateways provider
+        this.request = (0, exports.createWayfinderClient)({
+            selectGateway: async () => {
+                return this.routingStrategy.selectGateway({
+                    gateways: await this.gatewaysProvider.getGateways(),
+                });
+            },
+            resolveUrl: exports.resolveWayfinderUrl,
+            verifyData: this.verifyData,
+            emitter: this.emitter,
+            logger,
+            strict,
+        });
+        logger?.debug(`Wayfinder initialized with ${routingStrategy.constructor.name} routing strategy`);
+    }
+}
+exports.Wayfinder = Wayfinder;

package/lib/cjs/types/wayfinder.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+// TODO: add an offset provider that returns offsets for data items so we can use them to verify the signatures of a data item within a bundle