@ar.io/sdk 3.11.0-alpha.7 → 3.11.0-alpha.9

package/lib/esm/common/wayfinder/wayfinder.test.js

@@ -1,18 +1,22 @@
 import axios from 'axios';
 import got from 'got';
 import assert from 'node:assert';
+import { EventEmitter } from 'node:events';
+import { Readable } from 'node:stream';
 import { buffer } from 'node:stream/consumers';
 import { before, describe, it } from 'node:test';
 import { Logger } from '../../common/logger.js';
 import { RandomGatewayRouter } from './routers/random.js';
-import { Wayfinder } from './wayfinder.js';
+import { StaticGatewayRouter } from './routers/static.js';
+import { Wayfinder, tapAndVerifyStream } from './wayfinder.js';
+// TODO: replace with locally running gateway
 const gatewayUrl = 'permagate.io';
 const stubbedGatewaysProvider = {
-    getGateways: async () => [new URL(`https://${gatewayUrl}`)],
+    getGateways: async () => [new URL(`http://${gatewayUrl}`)],
 };
-Logger.default.setLogLevel('debug');
+Logger.default.setLogLevel('none');
 describe('Wayfinder', () => {
-    describe('http wrapper', () => {
+    describe.skip('http wrapper', () => {
         describe('fetch', () => {
             let wayfinder;
             before(() => {
@@ -233,4 +237,259 @@ describe('Wayfinder', () => {
             });
         });
     });
+    describe('events', () => {
+        it('should emit events on the wayfinder event emitter', async () => {
+            const wayfinder = new Wayfinder({
+                httpClient: fetch,
+                router: new StaticGatewayRouter({
+                    gateway: `http://${gatewayUrl}`,
+                }),
+                verifier: {
+                    // @ts-expect-error
+                    verifyData: async (params) => {
+                        return;
+                    },
+                },
+            });
+            const events = [];
+            wayfinder.emitter.on('verification-failed', (event) => {
+                events.push({ type: 'verification-failed', ...event });
+            });
+            wayfinder.emitter.on('verification-progress', (event) => {
+                events.push({ type: 'verification-progress', ...event });
+            });
+            wayfinder.emitter.on('verification-passed', (event) => {
+                events.push({ type: 'verification-passed', ...event });
+            });
+            // request data and assert the event is emitted
+            const response = await wayfinder.request('ar://c7wkwt6TKgcWJUfgvpJ5q5qi4DIZyJ1_TqhjXgURh0U', {
+                redirect: 'follow',
+            });
+            // read the full response body to ensure the stream is fully consumed
+            await response.text();
+            assert.strictEqual(response.status, 200);
+            assert.ok(events.find((e) => e.type === 'verification-passed'), 'Should emit at least one verification-passed');
+        });
+        it('should execute callbacks provided to the wayfinder constructor', async () => {
+            let verificationFailed = false;
+            let verificationProgress = false;
+            let verificationPassed = false;
+            const wayfinder = new Wayfinder({
+                httpClient: fetch,
+                router: new StaticGatewayRouter({
+                    gateway: `http://${gatewayUrl}`,
+                }),
+                events: {
+                    onVerificationFailed: () => {
+                        verificationFailed = true;
+                    },
+                    onVerificationProgress: () => {
+                        verificationProgress = true;
+                    },
+                    onVerificationPassed: () => {
+                        verificationPassed = true;
+                    },
+                },
+            });
+            const response = await wayfinder.request('ar://c7wkwt6TKgcWJUfgvpJ5q5qi4DIZyJ1_TqhjXgURh0U', {
+                redirect: 'follow',
+            });
+            // read the full response body to ensure the stream is fully consumed
+            await response.text();
+            assert.strictEqual(response.status, 200);
+            assert.ok(verificationFailed === false, 'Should not emit verification-failed');
+            assert.ok(verificationProgress, 'Should emit verification-progress');
+            assert.ok(verificationPassed, 'Should emit verification-passed');
+        });
+    });
+    describe.skip('tapAndVerifyRequest', () => {
+        describe('Readable', () => {
+            it('should duplicate the stream, verify the first and return the second if verification passes', async () => {
+                // create a simple readable
+                const chunks = [
+                    Buffer.from('foo'),
+                    Buffer.from('bar'),
+                    Buffer.from('baz'),
+                ];
+                const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
+                // a stream that will emit chunks
+                const originalStream = Readable.from(chunks);
+                let seen = Buffer.alloc(0);
+                const verifyData = async ({ data, 
+                // @ts-expect-error
+                txId, }) => {
+                    return new Promise((resolve, reject) => {
+                        data.on('data', (chunk) => {
+                            seen = Buffer.concat([seen, chunk]);
+                        });
+                        data.on('end', () => {
+                            // Should have seen exactly the full payload
+                            assert.strictEqual(seen.length, contentLength);
+                            resolve();
+                        });
+                        data.on('error', reject);
+                    });
+                };
+                const txId = 'test-tx-1';
+                const emitter = new EventEmitter();
+                const events = [];
+                emitter.on('verification-progress', (e) => {
+                    console.log('verification-progress', e);
+                    events.push({ type: 'verification-progress', ...e });
+                });
+                emitter.on('verification-passed', (e) => events.push({ type: 'verification-passed', ...e }));
+                // tap with verification
+                const tapped = tapAndVerifyStream({
+                    originalStream,
+                    contentLength,
+                    verifyData,
+                    txId,
+                    emitter,
+                });
+                // read the stream
+                const out = [];
+                for await (const chunk of tapped) {
+                    out.push(chunk);
+                }
+                // assert the stream is the same
+                assert.strictEqual(Buffer.concat(out).toString(), Buffer.concat(chunks).toString(), 'The tapped stream should emit exactly the original data');
+                assert.ok(events.find((e) => e.type === 'verification-progress'), 'Should emit at least one verification-progress');
+                assert.ok(events.find((e) => e.type === 'verification-passed' && e.txId === txId), 'Should emit at least one verification-passed');
+            });
+            it('should throw an error on the client stream if verification fails', async () => {
+                const chunks = [
+                    Buffer.from('foo'),
+                    Buffer.from('bar'),
+                    Buffer.from('baz'),
+                ];
+                const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
+                // a stream that will emit chunks
+                const originalStream = Readable.from(chunks);
+                const verifyData = async ({ 
+                // @ts-expect-error
+                data, txId, }) => {
+                    throw new Error('Verification failed for txId: ' + txId);
+                };
+                const txId = 'test-tx-1';
+                const emitter = new EventEmitter();
+                const events = [];
+                emitter.on('verification-progress', (e) => events.push({ type: 'verification-progress', ...e }));
+                emitter.on('verification-failed', (e) => events.push({ type: 'verification-failed', ...e }));
+                // tap with verification
+                const tapped = tapAndVerifyStream({
+                    originalStream,
+                    contentLength,
+                    verifyData,
+                    txId,
+                    emitter,
+                });
+                // read the stream
+                try {
+                    const out = [];
+                    for await (const chunk of tapped) {
+                        out.push(chunk);
+                    }
+                }
+                catch (error) {
+                    assert.ok(events.find((e) => e.type === 'verification-failed' && e.txId === txId), 'Should emit at least one verification-failed');
+                    // stream should be closed
+                    assert.ok(tapped.closed);
+                }
+            });
+        });
+        describe('ReadableStream', () => {
+            it('should duplicate the ReadableStream, verify the first and return the second if verification passes', async () => {
+                // create a simple readable
+                const chunks = [
+                    Buffer.from('foo'),
+                    Buffer.from('bar'),
+                    Buffer.from('baz'),
+                ];
+                const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
+                // a stream that will emit chunks
+                const originalStream = ReadableStream.from(chunks);
+                let seen = Buffer.alloc(0);
+                const verifyData = async ({ data, 
+                // @ts-expect-error
+                txId, }) => {
+                    return new Promise(async (resolve, reject) => {
+                        const reader = data.getReader();
+                        while (true) {
+                            try {
+                                const { done, value } = await reader.read();
+                                if (done) {
+                                    resolve();
+                                    break;
+                                }
+                                seen = Buffer.concat([seen, value]);
+                            }
+                            catch (error) {
+                                reject(error);
+                            }
+                        }
+                    });
+                };
+                const txId = 'test-tx-1';
+                const emitter = new EventEmitter();
+                const events = [];
+                emitter.on('verification-progress', (e) => events.push({ type: 'verification-progress', ...e }));
+                emitter.on('verification-passed', (e) => events.push({ type: 'verification-passed', ...e }));
+                // tap with verification
+                const tapped = tapAndVerifyStream({
+                    originalStream,
+                    contentLength,
+                    verifyData,
+                    txId,
+                    emitter,
+                });
+                // read the stream
+                const out = [];
+                for await (const chunk of tapped) {
+                    out.push(chunk);
+                }
+                // assert the stream is the same
+                assert.strictEqual(Buffer.concat(out).toString(), Buffer.concat(chunks).toString(), 'The tapped stream should emit exactly the original data');
+                assert.ok(events.find((e) => e.type === 'verification-progress'), 'Should emit at least one verification-progress');
+                assert.ok(events.find((e) => e.type === 'verification-passed' && e.txId === txId), 'Should emit at least one verification-passed');
+            });
+            it('should throw an error on the client stream if verification fails', async () => {
+                const chunks = [
+                    Buffer.from('foo'),
+                    Buffer.from('bar'),
+                    Buffer.from('baz'),
+                ];
+                const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
+                // a stream that will emit chunks
+                const originalStream = ReadableStream.from(chunks);
+                const verifyData = async ({ 
+                // @ts-expect-error
+                data, txId, }) => {
+                    throw new Error('Verification failed for txId: ' + txId);
+                };
+                const txId = 'test-tx-1';
+                const emitter = new EventEmitter();
+                const events = [];
+                emitter.on('verification-progress', (e) => events.push({ type: 'verification-progress', ...e }));
+                emitter.on('verification-failed', (e) => events.push({ type: 'verification-failed', ...e }));
+                // tap with verification
+                const tapped = tapAndVerifyStream({
+                    originalStream,
+                    contentLength,
+                    verifyData,
+                    txId,
+                    emitter,
+                });
+                // read the stream
+                try {
+                    const out = [];
+                    for await (const chunk of tapped) {
+                        out.push(chunk);
+                    }
+                }
+                catch (error) {
+                    assert.ok(events.find((e) => e.type === 'verification-failed' && e.txId === txId), 'Should emit at least one verification-failed');
+                }
+            });
+        });
+    });
 });

package/lib/esm/types/wayfinder.js

@@ -1 +1,2 @@
 export {};
+// TODO: add an offset provider that returns offsets for data items so we can use them to verify the signatures of a data item within a bundle

package/lib/esm/utils/hash.js

@@ -0,0 +1,50 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { createHash } from 'crypto';
+import { toB64Url } from './base64.js';
+export const hashReadableToB64Url = (stream, algorithm = 'sha256') => {
+    return new Promise((resolve, reject) => {
+        const hash = createHash(algorithm);
+        stream.on('data', (chunk) => hash.update(chunk));
+        stream.on('end', () => resolve(toB64Url(hash.digest())));
+        stream.on('error', (err) => reject(err));
+    });
+};
+export const hashReadableStreamToB64Url = (stream, algorithm = 'sha256') => {
+    return new Promise((resolve, reject) => {
+        const hash = createHash(algorithm);
+        const reader = stream.getReader();
+        const read = async () => {
+            try {
+                const { done, value } = await reader.read();
+                if (done) {
+                    resolve(toB64Url(hash.digest()));
+                }
+                else {
+                    hash.update(value);
+                    read();
+                }
+            }
+            catch (err) {
+                reject(err);
+            }
+        };
+        read().catch(reject);
+    });
+};
+export const hashBufferToB64Url = (buffer, algorithm = 'sha256') => {
+    return toB64Url(createHash(algorithm).update(buffer).digest());
+};

package/lib/esm/version.js

@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 // AUTOMATICALLY GENERATED FILE - DO NOT TOUCH
-export const version = '3.11.0-alpha.7';
+export const version = '3.11.0-alpha.9';

package/lib/types/cli/commands/readCommands.d.ts

@@ -68,3 +68,4 @@ export declare function getAllGatewayVaults(o: PaginationCLIOptions): Promise<im
     message: string;
 }>;
 export declare function getVault(o: AddressAndVaultIdCLIOptions): Promise<import("../../types/io.js").AoVaultData>;
+export declare function resolveArNSName(o: NameCLIOptions): Promise<import("../../types/io.js").ArNSNameResolutionData>;

package/lib/types/common/io.d.ts

@@ -1,5 +1,5 @@
 import Arweave from 'arweave';
-import { ARIOWithFaucet, AoARIORead, AoARIOWrite, AoAllDelegates, AoAllGatewayVaults, AoArNSNameData, AoArNSNameDataWithName, AoArNSPurchaseParams, AoArNSReservedNameData, AoArNSReservedNameDataWithName, AoBalanceWithAddress, AoBuyRecordParams, AoCreateVaultParams, AoDelegation, AoEligibleDistribution, AoEpochData, AoEpochDistributed, AoEpochDistributionData, AoEpochDistributionTotalsData, AoEpochObservationData, AoEpochSettings, AoExtendLeaseParams, AoExtendVaultParams, AoGateway, AoGatewayDelegateWithAddress, AoGatewayRegistrySettings, AoGatewayVault, AoGatewayWithAddress, AoGetCostDetailsParams, AoIncreaseUndernameLimitParams, AoIncreaseVaultParams, AoJoinNetworkParams, AoMessageResult, AoPaginatedAddressParams, AoPrimaryName, AoPrimaryNameRequest, AoRedelegationFeeInfo, AoRegistrationFees, AoReturnedName, AoRevokeVaultParams, AoTokenSupplyData, AoUpdateGatewaySettingsParams, AoVaultData, AoVaultedTransferParams, AoWalletVault, AoWeightedObserver, CostDetailsResult, DemandFactorSettings, EpochInput, OptionalArweave, OptionalPaymentUrl, PaginationParams, PaginationResult, ProcessConfiguration, TransactionId, WalletAddress, WithSigner, WriteOptions, mARIOToken } from '../types/index.js';
+import { ARIOWithFaucet, AoARIORead, AoARIOWrite, AoAllDelegates, AoAllGatewayVaults, AoArNSNameData, AoArNSNameDataWithName, AoArNSPurchaseParams, AoArNSReservedNameData, AoArNSReservedNameDataWithName, AoBalanceWithAddress, AoBuyRecordParams, AoCreateVaultParams, AoDelegation, AoEligibleDistribution, AoEpochData, AoEpochDistributed, AoEpochDistributionData, AoEpochDistributionTotalsData, AoEpochObservationData, AoEpochSettings, AoExtendLeaseParams, AoExtendVaultParams, AoGateway, AoGatewayDelegateWithAddress, AoGatewayRegistrySettings, AoGatewayVault, AoGatewayWithAddress, AoGetCostDetailsParams, AoIncreaseUndernameLimitParams, AoIncreaseVaultParams, AoJoinNetworkParams, AoMessageResult, AoPaginatedAddressParams, AoPrimaryName, AoPrimaryNameRequest, AoRedelegationFeeInfo, AoRegistrationFees, AoReturnedName, AoRevokeVaultParams, AoTokenSupplyData, AoUpdateGatewaySettingsParams, AoVaultData, AoVaultedTransferParams, AoWalletVault, AoWeightedObserver, ArNSNameResolutionData, ArNSNameResolver, CostDetailsResult, DemandFactorSettings, EpochInput, OptionalArweave, OptionalPaymentUrl, PaginationParams, PaginationResult, ProcessConfiguration, TransactionId, WalletAddress, WithSigner, WriteOptions, mARIOToken } from '../types/index.js';
 import { AOProcess } from './contracts/ao-process.js';
 import { TurboArNSPaymentProviderAuthenticated, TurboArNSPaymentProviderUnauthenticated } from './turbo.js';
 type ARIOConfigNoSigner = OptionalPaymentUrl<OptionalArweave<ProcessConfiguration>>;
@@ -19,7 +19,7 @@ export declare class ARIO {
         faucetUrl?: string;
     }): ARIOWithFaucet<AoARIOWrite>;
 }
-export declare class ARIOReadable implements AoARIORead {
+export declare class ARIOReadable implements AoARIORead, ArNSNameResolver {
     readonly process: AOProcess;
     protected epochSettings: AoEpochSettings | undefined;
     protected arweave: Arweave;
@@ -135,6 +135,9 @@ export declare class ARIOReadable implements AoARIORead {
     getGatewayRegistrySettings(): Promise<AoGatewayRegistrySettings>;
     getAllDelegates(params?: PaginationParams<AoAllDelegates>): Promise<PaginationResult<AoAllDelegates>>;
     getAllGatewayVaults(params?: PaginationParams<AoAllGatewayVaults>): Promise<PaginationResult<AoAllGatewayVaults>>;
+    resolveArNSName({ name, }: {
+        name: string;
+    }): Promise<ArNSNameResolutionData>;
 }
 export declare class ARIOWriteable extends ARIOReadable implements AoARIOWrite {
     private signer;

package/lib/types/common/wayfinder/gateways/trusted-gateways.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { DataHashProvider, DataRootProvider } from '../../../types/wayfinder.js';
+import { GatewaysProvider } from '../gateways.js';
+export declare class TrustedGatewaysHashProvider implements DataHashProvider, DataRootProvider {
+    private gatewaysProvider;
+    constructor({ gatewaysProvider, }: {
+        gatewaysProvider: GatewaysProvider;
+    });
+    /**
+     * Gets the digest for a given txId from all trusted gateways and ensures they all match.
+     * @param txId - The txId to get the digest for.
+     * @returns The digest for the given txId.
+     */
+    getHash({ txId, }: {
+        txId: string;
+    }): Promise<{
+        hash: string;
+        algorithm: 'sha256';
+    }>;
+    /**
+     * Get the data root for a given txId from all trusted gateways and ensure they all match.
+     * @param txId - The txId to get the data root for.
+     * @returns The data root for the given txId.
+     */
+    getDataRoot({ txId }: {
+        txId: string;
+    }): Promise<string>;
+}
+/**
+ * Note from @djwhitt
+ *
+ * Calculating data roots this way is fine, but it may not reproduce the original data root.
+ * We could also implement a data root verifier that pulls all the chunks, checks that they
+ * reproduce the expected data root, and then compares the concatenated chunk data to the
+ * original data retrieved. That would take a while, but it should be able to verify any L1
+ * data where we can find the chunks.
+ */

package/lib/types/common/wayfinder/index.d.ts

@@ -18,3 +18,6 @@ export * from './routers/random.js';
 export * from './routers/priority.js';
 export * from './routers/static.js';
 export * from './gateways.js';
+export * from './gateways/trusted-gateways.js';
+export * from './verification/data-root-verifier.js';
+export * from './verification/hash-verifier.js';

package/lib/types/common/wayfinder/verification/data-root-verifier.d.ts

@@ -0,0 +1,31 @@
+import { Readable } from 'node:stream';
+import { DataRootProvider, DataVerifier } from '../../../types/wayfinder.js';
+export declare function convertBufferToDataRoot({ buffer, }: {
+    buffer: Buffer;
+}): Promise<string>;
+export declare const convertReadableToDataRoot: <T extends AsyncIterable<Uint8Array>>({ iterable, }: {
+    iterable: T;
+}) => Promise<string>;
+export declare class DataRootVerifier implements DataVerifier {
+    private readonly trustedDataRootProvider;
+    constructor({ trustedDataRootProvider, }: {
+        trustedDataRootProvider: DataRootProvider;
+    });
+    verifyData({ data, txId, }: {
+        data: Buffer | Readable | ReadableStream;
+        txId: string;
+    }): Promise<void>;
+}
+/**
+ * - when you get a signature of a data item, you can only verify the owner
+ * - you still need to verify it's going back to the bundle, unpack it, and verify the data item exists at the offset
+ * - you need to the location of the chunks for the data item, and prove it's in the chunk and then prove the data root of the bundle, then you have fully verified the data verifier
+ * - how to prove the data item is on arweave - verify the merkle hash that the chunks for the data item, fit within the expected tree of the parent bundle
+ *
+ * Composite verifier - you'll want to be very efficient with streams
+ * - hash verifier
+ * - parent chunks verifier --> for any range of data within a single transaction, tell me that it's correct
+ * - signature verifier
+ * - offset verifier
+ * - data item verifier
+ */

package/lib/types/common/wayfinder/verification/hash-verifier.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Readable } from 'node:stream';
+import { DataHashProvider, DataVerifier } from '../../../types/wayfinder.js';
+export declare class HashVerifier implements DataVerifier {
+    private readonly trustedHashProvider;
+    constructor({ trustedHashProvider, }: {
+        trustedHashProvider: DataHashProvider;
+    });
+    verifyData({ data, txId, }: {
+        data: Buffer | Readable | ReadableStream;
+        txId: string;
+    }): Promise<void>;
+}