@ar.io/sdk 3.11.0-alpha.7 → 3.11.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -0
- package/bundles/web.bundle.min.js +119 -116
- package/lib/cjs/cli/cli.js +137 -122
- package/lib/cjs/cli/commands/readCommands.js +6 -0
- package/lib/cjs/common/ant.js +5 -5
- package/lib/cjs/common/io.js +37 -0
- package/lib/cjs/common/wayfinder/gateways/trusted-gateways.js +106 -0
- package/lib/cjs/common/wayfinder/index.js +6 -0
- package/lib/cjs/common/wayfinder/verification/data-root-verifier.js +139 -0
- package/lib/cjs/common/wayfinder/verification/hash-verifier.js +50 -0
- package/lib/cjs/common/wayfinder/wayfinder.js +407 -18
- package/lib/cjs/common/wayfinder/wayfinder.test.js +262 -3
- package/lib/cjs/types/wayfinder.js +1 -0
- package/lib/cjs/utils/hash.js +56 -0
- package/lib/cjs/version.js +1 -1
- package/lib/esm/cli/cli.js +138 -123
- package/lib/esm/cli/commands/readCommands.js +5 -0
- package/lib/esm/common/ant.js +5 -5
- package/lib/esm/common/io.js +37 -0
- package/lib/esm/common/wayfinder/gateways/trusted-gateways.js +102 -0
- package/lib/esm/common/wayfinder/index.js +6 -0
- package/lib/esm/common/wayfinder/verification/data-root-verifier.js +130 -0
- package/lib/esm/common/wayfinder/verification/hash-verifier.js +46 -0
- package/lib/esm/common/wayfinder/wayfinder.js +401 -18
- package/lib/esm/common/wayfinder/wayfinder.test.js +263 -4
- package/lib/esm/types/wayfinder.js +1 -0
- package/lib/esm/utils/hash.js +50 -0
- package/lib/esm/version.js +1 -1
- package/lib/types/cli/commands/readCommands.d.ts +1 -0
- package/lib/types/common/io.d.ts +5 -2
- package/lib/types/common/wayfinder/gateways/trusted-gateways.d.ts +51 -0
- package/lib/types/common/wayfinder/index.d.ts +3 -0
- package/lib/types/common/wayfinder/verification/data-root-verifier.d.ts +31 -0
- package/lib/types/common/wayfinder/verification/hash-verifier.d.ts +27 -0
- package/lib/types/common/wayfinder/wayfinder.d.ts +148 -10
- package/lib/types/types/io.d.ts +16 -1
- package/lib/types/types/wayfinder.d.ts +43 -0
- package/lib/types/utils/hash.d.ts +4 -0
- package/lib/types/version.d.ts +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import Arweave from 'arweave';
|
|
17
|
+
import { MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, buildLayers, generateLeaves, } from 'arweave/node/lib/merkle.js';
|
|
18
|
+
import { Readable } from 'node:stream';
|
|
19
|
+
import { toB64Url } from '../../../utils/base64.js';
|
|
20
|
+
export async function convertBufferToDataRoot({ buffer, }) {
|
|
21
|
+
const chunks = [];
|
|
22
|
+
let cursor = 0;
|
|
23
|
+
let offset = 0;
|
|
24
|
+
while (offset < buffer.byteLength) {
|
|
25
|
+
let chunkSize = Math.min(MAX_CHUNK_SIZE, buffer.byteLength - offset);
|
|
26
|
+
const remainder = buffer.byteLength - offset - chunkSize;
|
|
27
|
+
if (remainder > 0 && remainder < MIN_CHUNK_SIZE) {
|
|
28
|
+
chunkSize = Math.ceil((buffer.byteLength - offset) / 2);
|
|
29
|
+
}
|
|
30
|
+
// subarray does not exist on web Buffer type
|
|
31
|
+
const slice = buffer.subarray(offset, offset + chunkSize);
|
|
32
|
+
const hash = await crypto.subtle.digest('SHA-256', slice);
|
|
33
|
+
const hashArray = new Uint8Array(hash);
|
|
34
|
+
chunks.push({
|
|
35
|
+
dataHash: hashArray,
|
|
36
|
+
minByteRange: cursor,
|
|
37
|
+
maxByteRange: cursor + chunkSize,
|
|
38
|
+
});
|
|
39
|
+
cursor += chunkSize;
|
|
40
|
+
offset += chunkSize;
|
|
41
|
+
}
|
|
42
|
+
const leaves = await generateLeaves(chunks);
|
|
43
|
+
const result = await buildLayers(leaves);
|
|
44
|
+
return Buffer.from(result.id).toString('base64url');
|
|
45
|
+
}
|
|
46
|
+
export const convertReadableToDataRoot = async ({ iterable, }) => {
|
|
47
|
+
const chunks = [];
|
|
48
|
+
let leftover = new Uint8Array(0);
|
|
49
|
+
let cursor = 0;
|
|
50
|
+
for await (const data of iterable) {
|
|
51
|
+
const inputChunk = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
52
|
+
const combined = new Uint8Array(leftover.length + inputChunk.length);
|
|
53
|
+
combined.set(leftover, 0);
|
|
54
|
+
combined.set(inputChunk, leftover.length);
|
|
55
|
+
let startIndex = 0;
|
|
56
|
+
while (combined.length - startIndex >= MAX_CHUNK_SIZE) {
|
|
57
|
+
let chunkSize = MAX_CHUNK_SIZE;
|
|
58
|
+
const remainderAfterThis = combined.length - startIndex - MAX_CHUNK_SIZE;
|
|
59
|
+
if (remainderAfterThis > 0 && remainderAfterThis < MIN_CHUNK_SIZE) {
|
|
60
|
+
chunkSize = Math.ceil((combined.length - startIndex) / 2);
|
|
61
|
+
}
|
|
62
|
+
const chunkData = combined.slice(startIndex, startIndex + chunkSize);
|
|
63
|
+
const dataHash = await Arweave.crypto.hash(chunkData);
|
|
64
|
+
chunks.push({
|
|
65
|
+
dataHash,
|
|
66
|
+
minByteRange: cursor,
|
|
67
|
+
maxByteRange: cursor + chunkSize,
|
|
68
|
+
});
|
|
69
|
+
cursor += chunkSize;
|
|
70
|
+
startIndex += chunkSize;
|
|
71
|
+
}
|
|
72
|
+
leftover = combined.slice(startIndex);
|
|
73
|
+
}
|
|
74
|
+
if (leftover.length > 0) {
|
|
75
|
+
const dataHash = await Arweave.crypto.hash(leftover);
|
|
76
|
+
chunks.push({
|
|
77
|
+
dataHash,
|
|
78
|
+
minByteRange: cursor,
|
|
79
|
+
maxByteRange: cursor + leftover.length,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
const leaves = await generateLeaves(chunks);
|
|
83
|
+
const root = await buildLayers(leaves);
|
|
84
|
+
return toB64Url(Buffer.from(root.id));
|
|
85
|
+
};
|
|
86
|
+
export class DataRootVerifier {
|
|
87
|
+
trustedDataRootProvider;
|
|
88
|
+
constructor({ trustedDataRootProvider, }) {
|
|
89
|
+
this.trustedDataRootProvider = trustedDataRootProvider;
|
|
90
|
+
}
|
|
91
|
+
async verifyData({ data, txId, }) {
|
|
92
|
+
const trustedDataRootPromise = this.trustedDataRootProvider.getDataRoot({
|
|
93
|
+
txId,
|
|
94
|
+
});
|
|
95
|
+
let computedDataRoot;
|
|
96
|
+
if (Buffer.isBuffer(data)) {
|
|
97
|
+
computedDataRoot = await convertBufferToDataRoot({ buffer: data });
|
|
98
|
+
}
|
|
99
|
+
else if (data instanceof Readable || data instanceof ReadableStream) {
|
|
100
|
+
computedDataRoot = await convertReadableToDataRoot({ iterable: data });
|
|
101
|
+
}
|
|
102
|
+
if (computedDataRoot === undefined) {
|
|
103
|
+
throw new Error('Data root could not be computed');
|
|
104
|
+
}
|
|
105
|
+
const trustedDataRoot = await trustedDataRootPromise;
|
|
106
|
+
if (computedDataRoot !== trustedDataRoot) {
|
|
107
|
+
throw new Error('Data root does not match', {
|
|
108
|
+
cause: { computedDataRoot, trustedDataRoot },
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// some data item options
|
|
114
|
+
// compute and verify data root, use offsets from server and verify the signature that the data item at the offset matches the signature
|
|
115
|
+
// does not give you assurance of valid bundle, but gives verification that the data item itself is valid
|
|
116
|
+
// reading from offsets is the only way for the client to compute and verify the signature
|
|
117
|
+
/**
|
|
118
|
+
* - when you get a signature of a data item, you can only verify the owner
|
|
119
|
+
* - you still need to verify it's going back to the bundle, unpack it, and verify the data item exists at the offset
|
|
120
|
+
* - you need to the location of the chunks for the data item, and prove it's in the chunk and then prove the data root of the bundle, then you have fully verified the data verifier
|
|
121
|
+
* - how to prove the data item is on arweave - verify the merkle hash that the chunks for the data item, fit within the expected tree of the parent bundle
|
|
122
|
+
*
|
|
123
|
+
* Composite verifier - you'll want to be very efficient with streams
|
|
124
|
+
* - hash verifier
|
|
125
|
+
* - parent chunks verifier --> for any range of data within a single transaction, tell me that it's correct
|
|
126
|
+
* - signature verifier
|
|
127
|
+
* - offset verifier
|
|
128
|
+
* - data item verifier
|
|
129
|
+
*/
|
|
130
|
+
// introduce a composite verifier that determines where/how to lookup the hash
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import { Readable } from 'node:stream';
|
|
17
|
+
import { hashBufferToB64Url, hashReadableStreamToB64Url, hashReadableToB64Url, } from '../../../utils/hash.js';
|
|
18
|
+
export class HashVerifier {
|
|
19
|
+
trustedHashProvider;
|
|
20
|
+
constructor({ trustedHashProvider, }) {
|
|
21
|
+
this.trustedHashProvider = trustedHashProvider;
|
|
22
|
+
}
|
|
23
|
+
async verifyData({ data, txId, }) {
|
|
24
|
+
const hashPromise = this.trustedHashProvider.getHash({ txId });
|
|
25
|
+
let computedHash;
|
|
26
|
+
if (Buffer.isBuffer(data)) {
|
|
27
|
+
computedHash = hashBufferToB64Url(data);
|
|
28
|
+
}
|
|
29
|
+
else if (data instanceof Readable) {
|
|
30
|
+
computedHash = await hashReadableToB64Url(data);
|
|
31
|
+
}
|
|
32
|
+
else if (data instanceof ReadableStream) {
|
|
33
|
+
computedHash = await hashReadableStreamToB64Url(data);
|
|
34
|
+
}
|
|
35
|
+
// await on the hash promise and compare to get a little concurrency when computing hashes over larger data
|
|
36
|
+
const { hash } = await hashPromise;
|
|
37
|
+
if (computedHash === undefined) {
|
|
38
|
+
throw new Error('Hash could not be computed');
|
|
39
|
+
}
|
|
40
|
+
if (computedHash !== hash) {
|
|
41
|
+
throw new Error('Hash does not match', {
|
|
42
|
+
cause: { computedHash, trustedHash: hash },
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -1,7 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import EventEmitter from 'node:events';
|
|
17
|
+
import { PassThrough, Readable } from 'node:stream';
|
|
1
18
|
import { ARIO } from '../io.js';
|
|
2
19
|
import { Logger } from '../logger.js';
|
|
3
|
-
import { NetworkGatewaysProvider } from './gateways.js';
|
|
20
|
+
import { NetworkGatewaysProvider, SimpleCacheGatewaysProvider, StaticGatewaysProvider, } from './gateways.js';
|
|
21
|
+
import { TrustedGatewaysHashProvider } from './gateways/trusted-gateways.js';
|
|
4
22
|
import { RandomGatewayRouter } from './routers/random.js';
|
|
23
|
+
import { HashVerifier } from './verification/hash-verifier.js';
|
|
5
24
|
// known regexes for wayfinder urls
|
|
6
25
|
export const arnsRegex = /^[a-z0-9_-]{1,51}$/;
|
|
7
26
|
export const txIdRegex = /^[A-Za-z0-9_-]{43}$/;
|
|
@@ -54,6 +73,154 @@ export const resolveWayfinderUrl = async ({ originalUrl, targetGateway, logger,
|
|
|
54
73
|
// return the original url if it's not a wayfinder url (allows you to use the wayfinder client with non-wayfinder urls)
|
|
55
74
|
return new URL(originalUrl);
|
|
56
75
|
};
|
|
76
|
+
export class WayfinderEmitter extends EventEmitter {
|
|
77
|
+
constructor({ onVerificationPassed, onVerificationFailed, onVerificationProgress,
|
|
78
|
+
// TODO: continue this pattern for all events
|
|
79
|
+
} = {}) {
|
|
80
|
+
super();
|
|
81
|
+
if (onVerificationPassed) {
|
|
82
|
+
this.on('verification-passed', onVerificationPassed);
|
|
83
|
+
}
|
|
84
|
+
if (onVerificationFailed) {
|
|
85
|
+
this.on('verification-failed', onVerificationFailed);
|
|
86
|
+
}
|
|
87
|
+
if (onVerificationProgress) {
|
|
88
|
+
this.on('verification-progress', onVerificationProgress);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
emit(event, payload) {
|
|
92
|
+
return super.emit(event, payload);
|
|
93
|
+
}
|
|
94
|
+
on(event, listener) {
|
|
95
|
+
return super.on(event, listener);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
export function tapAndVerifyStream({ originalStream, contentLength, verifyData, txId, emitter, }) {
|
|
99
|
+
// taps node streams
|
|
100
|
+
if (originalStream instanceof Readable &&
|
|
101
|
+
typeof originalStream.pipe === 'function') {
|
|
102
|
+
const tappedClientStream = new PassThrough();
|
|
103
|
+
const streamToVerify = new PassThrough();
|
|
104
|
+
// kick off the verification promise, this will be awaited when the original stream ends
|
|
105
|
+
const verificationPromise = verifyData({
|
|
106
|
+
data: streamToVerify,
|
|
107
|
+
txId,
|
|
108
|
+
});
|
|
109
|
+
let bytesProcessed = 0;
|
|
110
|
+
// pipe the original stream to the verifier and the client stream
|
|
111
|
+
originalStream.on('data', (chunk) => {
|
|
112
|
+
streamToVerify.write(chunk);
|
|
113
|
+
tappedClientStream.write(chunk);
|
|
114
|
+
bytesProcessed += chunk.length;
|
|
115
|
+
// only emit if contentLength is not 0
|
|
116
|
+
if (contentLength !== 0) {
|
|
117
|
+
emitter?.emit('verification-progress', {
|
|
118
|
+
txId,
|
|
119
|
+
totalBytes: contentLength,
|
|
120
|
+
processedBytes: bytesProcessed,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
originalStream.on('end', async () => {
|
|
125
|
+
streamToVerify.end(); // triggers verifier completion and completes the verification promise
|
|
126
|
+
try {
|
|
127
|
+
await verificationPromise;
|
|
128
|
+
emitter?.emit('verification-passed', {
|
|
129
|
+
txId,
|
|
130
|
+
});
|
|
131
|
+
tappedClientStream.end();
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
emitter?.emit('verification-failed', {
|
|
135
|
+
error,
|
|
136
|
+
txId,
|
|
137
|
+
});
|
|
138
|
+
tappedClientStream.destroy(error);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
originalStream.on('error', (err) => {
|
|
142
|
+
emitter?.emit('verification-failed', {
|
|
143
|
+
error: err,
|
|
144
|
+
txId,
|
|
145
|
+
});
|
|
146
|
+
streamToVerify.destroy(err);
|
|
147
|
+
tappedClientStream.destroy(err);
|
|
148
|
+
});
|
|
149
|
+
// send the stream to the verify function and if it errors end the client stream
|
|
150
|
+
return tappedClientStream;
|
|
151
|
+
}
|
|
152
|
+
// taps web readable streams
|
|
153
|
+
if (originalStream instanceof ReadableStream &&
|
|
154
|
+
typeof originalStream.tee === 'function') {
|
|
155
|
+
const [verifyBranch, clientBranch] = originalStream.tee();
|
|
156
|
+
// setup our promise to verify the data
|
|
157
|
+
const verificationPromise = verifyData({
|
|
158
|
+
data: verifyBranch,
|
|
159
|
+
txId,
|
|
160
|
+
});
|
|
161
|
+
let bytesProcessed = 0;
|
|
162
|
+
const reader = clientBranch.getReader();
|
|
163
|
+
const clientStreamWithVerification = new ReadableStream({
|
|
164
|
+
async pull(controller) {
|
|
165
|
+
const { done, value } = await reader.read();
|
|
166
|
+
if (done) {
|
|
167
|
+
try {
|
|
168
|
+
// due to backpressure, if the client does not consume the stream, the verification will not complete (particularly important for fetch, where the response body needs to be awaited for verification to complete)
|
|
169
|
+
await verificationPromise;
|
|
170
|
+
emitter?.emit('verification-passed', {
|
|
171
|
+
txId,
|
|
172
|
+
});
|
|
173
|
+
controller.close();
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
emitter?.emit('verification-failed', {
|
|
177
|
+
txId,
|
|
178
|
+
error: err,
|
|
179
|
+
});
|
|
180
|
+
controller.error(err);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
bytesProcessed += value.length;
|
|
185
|
+
emitter?.emit('verification-progress', {
|
|
186
|
+
txId,
|
|
187
|
+
totalBytes: contentLength,
|
|
188
|
+
processedBytes: bytesProcessed,
|
|
189
|
+
});
|
|
190
|
+
controller.enqueue(value);
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
cancel(reason) {
|
|
194
|
+
reader.cancel(reason);
|
|
195
|
+
emitter?.emit('verification-failed', {
|
|
196
|
+
txId,
|
|
197
|
+
error: new Error('Verification cancelled', {
|
|
198
|
+
cause: {
|
|
199
|
+
reason,
|
|
200
|
+
},
|
|
201
|
+
}),
|
|
202
|
+
});
|
|
203
|
+
},
|
|
204
|
+
});
|
|
205
|
+
return clientStreamWithVerification;
|
|
206
|
+
}
|
|
207
|
+
throw new Error('Unsupported body type for cloning');
|
|
208
|
+
}
|
|
209
|
+
export function wrapVerifiedResponse(original, newBody, txId) {
|
|
210
|
+
// Clone headers (Header objects aren't serializable)
|
|
211
|
+
const headers = new Headers();
|
|
212
|
+
original.headers.forEach((value, key) => headers.set(key, value));
|
|
213
|
+
// Create a new Response with the new body and cloned headers
|
|
214
|
+
const wrapped = new Response(newBody, {
|
|
215
|
+
status: original.status,
|
|
216
|
+
statusText: original.statusText,
|
|
217
|
+
headers,
|
|
218
|
+
});
|
|
219
|
+
// Attach txId for downstream tracking
|
|
220
|
+
wrapped.txId = txId;
|
|
221
|
+
wrapped.redirectedFrom = original.url;
|
|
222
|
+
return wrapped;
|
|
223
|
+
}
|
|
57
224
|
/**
|
|
58
225
|
* Creates a wrapped http client that supports ar:// protocol
|
|
59
226
|
*
|
|
@@ -67,26 +234,164 @@ export const resolveWayfinderUrl = async ({ originalUrl, targetGateway, logger,
|
|
|
67
234
|
* @param resolveUrl - the function to construct the redirect url for ar:// requests
|
|
68
235
|
* @returns a wrapped http client that supports ar:// protocol
|
|
69
236
|
*/
|
|
70
|
-
export const createWayfinderClient = ({ httpClient, resolveUrl, logger, }) => {
|
|
237
|
+
export const createWayfinderClient = ({ httpClient, resolveUrl, verifyData, emitter = new WayfinderEmitter(), logger, }) => {
|
|
71
238
|
const wayfinderRedirect = async (fn, rawArgs) => {
|
|
72
239
|
// TODO: handle if first arg is not a string (i.e. just return the result of the function call)
|
|
73
240
|
const [originalUrl, ...rest] = rawArgs;
|
|
241
|
+
if (typeof originalUrl !== 'string') {
|
|
242
|
+
logger?.debug('Original URL is not a string, skipping routing', {
|
|
243
|
+
originalUrl,
|
|
244
|
+
});
|
|
245
|
+
return fn(...rawArgs);
|
|
246
|
+
}
|
|
247
|
+
emitter?.emit('routing-started', {
|
|
248
|
+
originalUrl: originalUrl.toString(),
|
|
249
|
+
});
|
|
74
250
|
// route the request to the target gateway
|
|
75
251
|
const redirectUrl = await resolveUrl({
|
|
76
252
|
originalUrl,
|
|
77
253
|
logger,
|
|
78
254
|
});
|
|
255
|
+
emitter?.emit('routing-succeeded', {
|
|
256
|
+
originalUrl,
|
|
257
|
+
targetGateway: redirectUrl.toString(),
|
|
258
|
+
});
|
|
79
259
|
logger?.debug(`Redirecting request to ${redirectUrl}`, {
|
|
80
260
|
originalUrl,
|
|
81
261
|
redirectUrl,
|
|
82
262
|
});
|
|
83
263
|
// make the request to the target gateway using the redirect url and http client
|
|
84
264
|
const response = await fn(redirectUrl.toString(), ...rest);
|
|
265
|
+
// TODO: trigger a routing event with the raw response object?
|
|
85
266
|
logger?.debug(`Successfully routed request to ${redirectUrl}`, {
|
|
86
267
|
redirectUrl,
|
|
87
268
|
originalUrl,
|
|
88
269
|
});
|
|
89
|
-
//
|
|
270
|
+
// only verify data if the redirect url is different from the original url
|
|
271
|
+
if (response && redirectUrl.toString() !== originalUrl.toString()) {
|
|
272
|
+
if (verifyData) {
|
|
273
|
+
// if the headers do not have .get on them, we need to parse the headers manually
|
|
274
|
+
const headers = new Headers();
|
|
275
|
+
let headersObject = response.headers ?? {};
|
|
276
|
+
if (typeof headersObject.get !== 'function') {
|
|
277
|
+
headersObject = Object.fromEntries(headersObject);
|
|
278
|
+
for (const [key, value] of Object.entries(headersObject)) {
|
|
279
|
+
headers.set(key, value);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
else {
|
|
283
|
+
for (const [key, value] of headersObject.entries()) {
|
|
284
|
+
headers.set(key, value);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
// transaction id is either in the response headers or the path of the request as the first parameter
|
|
288
|
+
// TODO: we may want to move this parsing to be returned by the resolveUrl function depending on the redirect URL we've constructed
|
|
289
|
+
const txId = headers.get('x-arns-resolved-id') ??
|
|
290
|
+
redirectUrl.pathname.split('/')[1];
|
|
291
|
+
// TODO: validate nodes return content length for all responses
|
|
292
|
+
const contentLength = +(headers.get('content-length') ?? 0);
|
|
293
|
+
if (!txIdRegex.test(txId)) {
|
|
294
|
+
// no transaction id found, skip verification
|
|
295
|
+
logger?.debug('No transaction id found, skipping verification', {
|
|
296
|
+
redirectUrl,
|
|
297
|
+
originalUrl,
|
|
298
|
+
});
|
|
299
|
+
emitter?.emit('verification-skipped', {
|
|
300
|
+
originalUrl,
|
|
301
|
+
});
|
|
302
|
+
return response;
|
|
303
|
+
}
|
|
304
|
+
emitter?.emit('identified-transaction-id', {
|
|
305
|
+
originalUrl,
|
|
306
|
+
targetGateway: redirectUrl.toString(),
|
|
307
|
+
txId,
|
|
308
|
+
});
|
|
309
|
+
// parse out the key that contains the response body, we'll use it later when updating the response object
|
|
310
|
+
const responseDataKey = response.body
|
|
311
|
+
? 'body'
|
|
312
|
+
: response.data
|
|
313
|
+
? 'data'
|
|
314
|
+
: undefined;
|
|
315
|
+
if (responseDataKey === undefined) {
|
|
316
|
+
throw new Error('No data body or data provided, skipping verification', {
|
|
317
|
+
cause: {
|
|
318
|
+
redirectUrl: redirectUrl.toString(),
|
|
319
|
+
originalUrl: originalUrl.toString(),
|
|
320
|
+
},
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
const responseBody = response[responseDataKey];
|
|
324
|
+
// TODO: determine if it is data item or L1 transaction, and tell the verifier accordingly, just drop in hit to graphql now
|
|
325
|
+
if (txId === undefined) {
|
|
326
|
+
throw new Error('Failed to parse data hash from response headers', {
|
|
327
|
+
cause: {
|
|
328
|
+
redirectUrl: redirectUrl.toString(),
|
|
329
|
+
originalUrl: originalUrl.toString(),
|
|
330
|
+
txId,
|
|
331
|
+
},
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
else if (responseBody === undefined) {
|
|
335
|
+
throw new Error('No data body provided, skipping verification', {
|
|
336
|
+
cause: {
|
|
337
|
+
redirectUrl: redirectUrl.toString(),
|
|
338
|
+
originalUrl: originalUrl.toString(),
|
|
339
|
+
txId,
|
|
340
|
+
},
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
else {
|
|
344
|
+
logger?.debug('Verifying data hash for txId', {
|
|
345
|
+
redirectUrl: redirectUrl.toString(),
|
|
346
|
+
originalUrl: originalUrl.toString(),
|
|
347
|
+
txId,
|
|
348
|
+
});
|
|
349
|
+
const newClientStream = tapAndVerifyStream({
|
|
350
|
+
originalStream: responseBody,
|
|
351
|
+
contentLength,
|
|
352
|
+
verifyData,
|
|
353
|
+
txId,
|
|
354
|
+
emitter,
|
|
355
|
+
});
|
|
356
|
+
if (responseBody instanceof ReadableStream) {
|
|
357
|
+
// specific to fetch
|
|
358
|
+
return wrapVerifiedResponse(response, newClientStream, txId);
|
|
359
|
+
}
|
|
360
|
+
else if (responseBody instanceof Readable) {
|
|
361
|
+
// overwrite the response body with the new client stream
|
|
362
|
+
response.txId = txId;
|
|
363
|
+
response.body = newClientStream;
|
|
364
|
+
return response;
|
|
365
|
+
}
|
|
366
|
+
else {
|
|
367
|
+
// TODO: content-application/json and it's smaller than 10mb
|
|
368
|
+
// TODO: add tests and verify this works for all non-Readable/streamed responses
|
|
369
|
+
try {
|
|
370
|
+
// if strict set to true
|
|
371
|
+
await verifyData({
|
|
372
|
+
data: responseBody,
|
|
373
|
+
txId,
|
|
374
|
+
});
|
|
375
|
+
emitter?.emit('verification-passed', {
|
|
376
|
+
txId,
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
catch (error) {
|
|
380
|
+
logger?.debug('Failed to verify data hash', {
|
|
381
|
+
error,
|
|
382
|
+
txId,
|
|
383
|
+
});
|
|
384
|
+
emitter?.emit('verification-failed', {
|
|
385
|
+
txId,
|
|
386
|
+
error,
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
return response;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
// TODO: if strict - wait for verification to finish and succeed before returning the response
|
|
90
395
|
return response;
|
|
91
396
|
};
|
|
92
397
|
return new Proxy(httpClient, {
|
|
@@ -116,21 +421,31 @@ export class Wayfinder {
|
|
|
116
421
|
* @example
|
|
117
422
|
* const wayfinder = new Wayfinder({
|
|
118
423
|
* router: new RandomGatewayRouter({
|
|
119
|
-
* gatewaysProvider: new
|
|
424
|
+
* gatewaysProvider: new SimpleCacheGatewaysProvider({
|
|
425
|
+
* gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
|
|
426
|
+
* ttlSeconds: 60 * 60 * 24, // 1 day
|
|
427
|
+
* }),
|
|
120
428
|
* }),
|
|
121
429
|
* });
|
|
430
|
+
*
|
|
431
|
+
* // Returns a target gateway based on the routing strategy
|
|
432
|
+
* const targetGateway = await wayfinder.router.getTargetGateway();
|
|
122
433
|
*/
|
|
123
434
|
router;
|
|
124
435
|
/**
|
|
125
|
-
* The http client
|
|
436
|
+
* The native http client used by wayfinder
|
|
126
437
|
*
|
|
127
438
|
* @example
|
|
128
439
|
* const wayfinder = new Wayfinder({
|
|
129
440
|
* router: new RandomGatewayRouter({
|
|
130
|
-
* gatewaysProvider: new
|
|
441
|
+
* gatewaysProvider: new SimpleCacheGatewaysProvider({
|
|
442
|
+
* gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
|
|
443
|
+
* ttlSeconds: 60 * 60 * 24, // 1 day
|
|
444
|
+
* }),
|
|
131
445
|
* }),
|
|
132
446
|
* httpClient: axios,
|
|
133
447
|
* });
|
|
448
|
+
*
|
|
134
449
|
*/
|
|
135
450
|
httpClient;
|
|
136
451
|
/**
|
|
@@ -139,11 +454,15 @@ export class Wayfinder {
|
|
|
139
454
|
* @example
|
|
140
455
|
* const wayfinder = new Wayfinder({
|
|
141
456
|
* router: new RandomGatewayRouter({
|
|
142
|
-
* gatewaysProvider: new
|
|
457
|
+
* gatewaysProvider: new SimpleCacheGatewaysProvider({
|
|
458
|
+
* gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
|
|
459
|
+
* ttlSeconds: 60 * 60 * 24, // 1 day
|
|
460
|
+
* }),
|
|
143
461
|
* }),
|
|
144
462
|
* httpClient: axios,
|
|
145
463
|
* });
|
|
146
464
|
*
|
|
465
|
+
* // returns the redirected URL based on the routing strategy and the original url
|
|
147
466
|
* const redirectUrl = await wayfinder.resolveUrl({ originalUrl: 'ar://example' });
|
|
148
467
|
*/
|
|
149
468
|
resolveUrl;
|
|
@@ -153,7 +472,10 @@ export class Wayfinder {
|
|
|
153
472
|
* @example
|
|
154
473
|
* const { request: wayfind } = new Wayfinder({
|
|
155
474
|
* router: new RandomGatewayRouter({
|
|
156
|
-
* gatewaysProvider: new
|
|
475
|
+
* gatewaysProvider: new SimpleCacheGatewaysProvider({
|
|
476
|
+
* gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
|
|
477
|
+
* ttlSeconds: 60 * 60 * 24, // 1 day
|
|
478
|
+
* }),
|
|
157
479
|
* }),
|
|
158
480
|
* httpClient: axios,
|
|
159
481
|
* });;
|
|
@@ -168,21 +490,62 @@ export class Wayfinder {
|
|
|
168
490
|
request;
|
|
169
491
|
// TODO: stats provider
|
|
170
492
|
// TODO: metricsProvider for otel/prom support
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
493
|
+
verifyData;
|
|
494
|
+
/**
|
|
495
|
+
* The event emitter for wayfinder that emits verification events.
|
|
496
|
+
*
|
|
497
|
+
* const wayfinder = new Wayfinder()
|
|
498
|
+
*
|
|
499
|
+
* wayfinder.emitter.on('verification-passed', (event) => {
|
|
500
|
+
* console.log('Verification passed!', event);
|
|
501
|
+
* })
|
|
502
|
+
*
|
|
503
|
+
* wayfinder.emitter.on('verification-failed', (event) => {
|
|
504
|
+
* console.log('Verification failed!', event);
|
|
505
|
+
* })
|
|
506
|
+
*
|
|
507
|
+
* or implement the events interface and pass it in, using callback functions
|
|
508
|
+
*
|
|
509
|
+
* const wayfinder = new Wayfinder({
|
|
510
|
+
* events: {
|
|
511
|
+
* onVerificationPassed: (event) => {
|
|
512
|
+
* console.log('Verification passed!', event);
|
|
513
|
+
* },
|
|
514
|
+
* onVerificationFailed: (event) => {
|
|
515
|
+
* console.log('Verification failed!', event);
|
|
516
|
+
* },
|
|
517
|
+
* onVerificationProgress: (event) => {
|
|
518
|
+
* console.log('Verification progress!', event);
|
|
519
|
+
* },
|
|
520
|
+
* }
|
|
521
|
+
* })
|
|
522
|
+
*
|
|
523
|
+
* const response = await wayfind('ar://example');
|
|
524
|
+
*/
|
|
525
|
+
// TODO: consider changing this to events or event emitter
|
|
526
|
+
emitter;
|
|
527
|
+
constructor({ httpClient,
|
|
176
528
|
// TODO: consider changing router to routingStrategy or strategy
|
|
177
529
|
router = new RandomGatewayRouter({
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
530
|
+
gatewaysProvider: new SimpleCacheGatewaysProvider({
|
|
531
|
+
gatewaysProvider: new NetworkGatewaysProvider({ ario: ARIO.mainnet() }),
|
|
532
|
+
ttlSeconds: 60 * 60 * 24, // 1 day
|
|
533
|
+
}),
|
|
534
|
+
}), logger = Logger.default,
|
|
535
|
+
// TODO: support disabling verification or create some PassThroughVerifier like thing
|
|
536
|
+
verifier = new HashVerifier({
|
|
537
|
+
trustedHashProvider: new TrustedGatewaysHashProvider({
|
|
538
|
+
gatewaysProvider: new StaticGatewaysProvider({
|
|
539
|
+
gateways: ['https://permagate.io'],
|
|
540
|
+
}),
|
|
541
|
+
}),
|
|
542
|
+
}), events,
|
|
182
543
|
// TODO: stats provider
|
|
183
544
|
}) {
|
|
184
545
|
this.router = router;
|
|
185
546
|
this.httpClient = httpClient;
|
|
547
|
+
this.emitter = new WayfinderEmitter(events);
|
|
548
|
+
this.verifyData = verifier.verifyData.bind(verifier);
|
|
186
549
|
this.resolveUrl = async ({ originalUrl, logger }) => {
|
|
187
550
|
return resolveWayfinderUrl({
|
|
188
551
|
originalUrl,
|
|
@@ -193,9 +556,29 @@ export class Wayfinder {
|
|
|
193
556
|
this.request = createWayfinderClient({
|
|
194
557
|
httpClient,
|
|
195
558
|
resolveUrl: this.resolveUrl,
|
|
559
|
+
verifyData: this.verifyData,
|
|
560
|
+
emitter: this.emitter,
|
|
196
561
|
logger,
|
|
197
|
-
// TODO: provide the verifyDataHash function from the verifier to the wayfinder client along with verificationSettings
|
|
198
562
|
});
|
|
199
563
|
logger?.debug(`Wayfinder initialized with ${router.name} routing strategy`);
|
|
200
564
|
}
|
|
201
565
|
}
|
|
566
|
+
// TODO: add a chart for verification strategies and what they do
|
|
567
|
+
// include complexity, performance, and security
|
|
568
|
+
// explain use cases that each strategy is best for
|
|
569
|
+
// e.g.
|
|
570
|
+
/**
|
|
571
|
+
*
|
|
572
|
+
* type | complexity | performance | security
|
|
573
|
+
* ---------|------------|-------------|---------
|
|
574
|
+
* hash | low | high | low
|
|
575
|
+
* ---------|------------|-------------|---------
|
|
576
|
+
* data root | medium | medium | low | only L1
|
|
577
|
+
* ---------|------------|-------------|---------
|
|
578
|
+
* signature | medium | medium | medium
|
|
579
|
+
* ---------|------------|-------------|---------
|
|
580
|
+
* composite | high | low | high
|
|
581
|
+
* ---------|------------|-------------|---------
|
|
582
|
+
*
|
|
583
|
+
*
|
|
584
|
+
*/
|