@ar.io/sdk 3.11.0-alpha.6 → 3.11.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundles/web.bundle.min.js +119 -116
- package/lib/cjs/common/io.js +8 -3
- package/lib/cjs/common/wayfinder/gateways/trusted-gateways.js +106 -0
- package/lib/cjs/common/wayfinder/gateways.js +19 -8
- package/lib/cjs/common/wayfinder/index.js +7 -1
- package/lib/cjs/common/wayfinder/routers/priority.js +11 -20
- package/lib/cjs/common/wayfinder/routers/priority.test.js +5 -5
- package/lib/cjs/common/wayfinder/routers/random.js +2 -2
- package/lib/cjs/common/wayfinder/routers/random.test.js +5 -84
- package/lib/cjs/common/wayfinder/routers/{fixed.js → static.js} +5 -5
- package/lib/cjs/common/wayfinder/routers/{fixed.test.js → static.test.js} +4 -4
- package/lib/cjs/common/wayfinder/verification/data-root-verifier.js +139 -0
- package/lib/cjs/common/wayfinder/verification/hash-verifier.js +50 -0
- package/lib/cjs/common/wayfinder/wayfinder.js +408 -19
- package/lib/cjs/common/wayfinder/wayfinder.test.js +296 -49
- package/lib/cjs/types/wayfinder.js +1 -0
- package/lib/cjs/utils/arweave.js +1 -1
- package/lib/cjs/utils/hash.js +56 -0
- package/lib/cjs/version.js +1 -1
- package/lib/esm/common/io.js +8 -3
- package/lib/esm/common/wayfinder/gateways/trusted-gateways.js +102 -0
- package/lib/esm/common/wayfinder/gateways.js +17 -6
- package/lib/esm/common/wayfinder/index.js +7 -1
- package/lib/esm/common/wayfinder/routers/priority.js +11 -20
- package/lib/esm/common/wayfinder/routers/priority.test.js +5 -5
- package/lib/esm/common/wayfinder/routers/random.js +2 -2
- package/lib/esm/common/wayfinder/routers/random.test.js +5 -84
- package/lib/esm/common/wayfinder/routers/{fixed.js → static.js} +3 -3
- package/lib/esm/common/wayfinder/routers/{fixed.test.js → static.test.js} +4 -4
- package/lib/esm/common/wayfinder/verification/data-root-verifier.js +130 -0
- package/lib/esm/common/wayfinder/verification/hash-verifier.js +46 -0
- package/lib/esm/common/wayfinder/wayfinder.js +402 -19
- package/lib/esm/common/wayfinder/wayfinder.test.js +297 -50
- package/lib/esm/types/wayfinder.js +1 -0
- package/lib/esm/utils/arweave.js +1 -1
- package/lib/esm/utils/hash.js +50 -0
- package/lib/esm/version.js +1 -1
- package/lib/types/common/wayfinder/gateways/trusted-gateways.d.ts +51 -0
- package/lib/types/common/wayfinder/gateways.d.ts +16 -7
- package/lib/types/common/wayfinder/index.d.ts +4 -1
- package/lib/types/common/wayfinder/routers/priority.d.ts +8 -12
- package/lib/types/common/wayfinder/routers/{fixed.d.ts → static.d.ts} +3 -3
- package/lib/types/common/wayfinder/verification/data-root-verifier.d.ts +31 -0
- package/lib/types/common/wayfinder/verification/hash-verifier.d.ts +27 -0
- package/lib/types/common/wayfinder/wayfinder.d.ts +148 -10
- package/lib/types/types/wayfinder.d.ts +43 -0
- package/lib/types/utils/hash.d.ts +4 -0
- package/lib/types/version.d.ts +1 -1
- package/package.json +1 -1
- /package/lib/types/common/wayfinder/routers/{fixed.test.d.ts → static.test.d.ts} +0 -0
|
@@ -1,27 +1,22 @@
|
|
|
1
1
|
import axios from 'axios';
|
|
2
2
|
import got from 'got';
|
|
3
3
|
import assert from 'node:assert';
|
|
4
|
+
import { EventEmitter } from 'node:events';
|
|
5
|
+
import { Readable } from 'node:stream';
|
|
4
6
|
import { buffer } from 'node:stream/consumers';
|
|
5
7
|
import { before, describe, it } from 'node:test';
|
|
8
|
+
import { Logger } from '../../common/logger.js';
|
|
6
9
|
import { RandomGatewayRouter } from './routers/random.js';
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
operatorStake: 1000,
|
|
12
|
-
totalDelegatedStake: 1000,
|
|
13
|
-
startTimestamp: 1000,
|
|
14
|
-
settings: {
|
|
15
|
-
protocol: 'https',
|
|
16
|
-
fqdn: 'arweave.net',
|
|
17
|
-
port: 443,
|
|
18
|
-
},
|
|
19
|
-
};
|
|
10
|
+
import { StaticGatewayRouter } from './routers/static.js';
|
|
11
|
+
import { Wayfinder, tapAndVerifyStream } from './wayfinder.js';
|
|
12
|
+
// TODO: replace with locally running gateway
|
|
13
|
+
const gatewayUrl = 'permagate.io';
|
|
20
14
|
const stubbedGatewaysProvider = {
|
|
21
|
-
getGateways: async () => [
|
|
15
|
+
getGateways: async () => [new URL(`http://${gatewayUrl}`)],
|
|
22
16
|
};
|
|
17
|
+
Logger.default.setLogLevel('none');
|
|
23
18
|
describe('Wayfinder', () => {
|
|
24
|
-
describe('http wrapper', () => {
|
|
19
|
+
describe.skip('http wrapper', () => {
|
|
25
20
|
describe('fetch', () => {
|
|
26
21
|
let wayfinder;
|
|
27
22
|
before(() => {
|
|
@@ -33,7 +28,7 @@ describe('Wayfinder', () => {
|
|
|
33
28
|
});
|
|
34
29
|
});
|
|
35
30
|
it('should fetch the data using the selected gateway', async () => {
|
|
36
|
-
const nativeFetch = await fetch(
|
|
31
|
+
const nativeFetch = await fetch(`https://ao.${gatewayUrl}`);
|
|
37
32
|
const response = await wayfinder.request('ar://ao');
|
|
38
33
|
assert.strictEqual(response.status, 200);
|
|
39
34
|
assert.strictEqual(response.status, nativeFetch.status);
|
|
@@ -41,44 +36,48 @@ describe('Wayfinder', () => {
|
|
|
41
36
|
const arnsHeaders = Array.from(response.headers.entries()).filter(([key]) => key.startsWith('x-arns-'));
|
|
42
37
|
const nativeFetchHeaders = Array.from(nativeFetch.headers.entries()).filter(([key]) => key.startsWith('x-arns-'));
|
|
43
38
|
assert.deepStrictEqual(arnsHeaders, nativeFetchHeaders);
|
|
44
|
-
|
|
39
|
+
});
|
|
40
|
+
it('should fetch a tx id using the selected gateway', async () => {
|
|
41
|
+
const nativeFetch = await fetch(`https://${gatewayUrl}/KKmRbIfrc7wiLcG0zvY1etlO0NBx1926dSCksxCIN3A`,
|
|
42
|
+
// follow redirects
|
|
43
|
+
{ redirect: 'follow' });
|
|
44
|
+
const response = await wayfinder.request('ar://KKmRbIfrc7wiLcG0zvY1etlO0NBx1926dSCksxCIN3A', { redirect: 'follow' });
|
|
45
|
+
assert.strictEqual(response.status, 200);
|
|
46
|
+
assert.strictEqual(response.status, nativeFetch.status);
|
|
45
47
|
});
|
|
46
48
|
it('should route a non-ar:// url as a normal fetch', async () => {
|
|
47
49
|
const [nativeFetch, response] = await Promise.all([
|
|
48
|
-
fetch(
|
|
50
|
+
fetch(`https://${gatewayUrl}/`, {
|
|
49
51
|
method: 'HEAD',
|
|
50
52
|
}),
|
|
51
|
-
wayfinder.request(
|
|
53
|
+
wayfinder.request(`https://${gatewayUrl}/`, {
|
|
52
54
|
method: 'HEAD',
|
|
53
55
|
}),
|
|
54
56
|
]);
|
|
55
57
|
assert.strictEqual(response.status, 200);
|
|
56
58
|
assert.strictEqual(response.status, nativeFetch.status);
|
|
57
59
|
// TODO: ensure the headers are the same excluding unique headers
|
|
58
|
-
assert.deepStrictEqual(await response.text(), await nativeFetch.text());
|
|
59
60
|
});
|
|
60
|
-
for (const api of ['/info', '/
|
|
61
|
+
for (const api of ['/info', '/block/current']) {
|
|
61
62
|
it(`supports native arweave node apis ${api}`, async () => {
|
|
62
63
|
const [nativeFetch, response] = await Promise.all([
|
|
63
|
-
fetch(`https
|
|
64
|
-
wayfinder.request(`ar
|
|
64
|
+
fetch(`https://${gatewayUrl}${api}`),
|
|
65
|
+
wayfinder.request(`ar://${api}`),
|
|
65
66
|
]);
|
|
66
67
|
assert.strictEqual(response.status, 200);
|
|
67
68
|
assert.strictEqual(response.status, nativeFetch.status);
|
|
68
69
|
// TODO: ensure the headers are the same excluding unique headers
|
|
69
|
-
assert.deepStrictEqual(await response.text(), await nativeFetch.text());
|
|
70
70
|
});
|
|
71
71
|
}
|
|
72
|
-
for (const api of ['/ar-io/info'
|
|
72
|
+
for (const api of ['/ar-io/info']) {
|
|
73
73
|
it(`supports native ario node gateway apis ${api}`, async () => {
|
|
74
74
|
const [nativeFetch, response] = await Promise.all([
|
|
75
|
-
fetch(`https
|
|
75
|
+
fetch(`https://${gatewayUrl}${api}`),
|
|
76
76
|
wayfinder.request(`ar:///${api}`),
|
|
77
77
|
]);
|
|
78
78
|
assert.strictEqual(response.status, 200);
|
|
79
79
|
assert.strictEqual(response.status, nativeFetch.status);
|
|
80
80
|
// TODO: ensure the headers are the same excluding unique headers
|
|
81
|
-
assert.deepStrictEqual(await response.text(), await nativeFetch.text());
|
|
82
81
|
});
|
|
83
82
|
}
|
|
84
83
|
it('supports a post request to graphql', async () => {
|
|
@@ -117,14 +116,13 @@ describe('Wayfinder', () => {
|
|
|
117
116
|
});
|
|
118
117
|
assert.strictEqual(response.status, 200);
|
|
119
118
|
});
|
|
120
|
-
it('returns the error from the target gateway if the route is not found', async () => {
|
|
119
|
+
it.skip('returns the error from the target gateway if the route is not found', async () => {
|
|
121
120
|
const [nativeFetch, response] = await Promise.all([
|
|
122
|
-
fetch(
|
|
123
|
-
wayfinder.request('
|
|
121
|
+
fetch(`https://${gatewayUrl}/ar-io/not-found`),
|
|
122
|
+
wayfinder.request('ar:///not-found'),
|
|
124
123
|
]);
|
|
125
124
|
assert.strictEqual(response.status, nativeFetch.status);
|
|
126
125
|
assert.strictEqual(response.statusText, nativeFetch.statusText);
|
|
127
|
-
assert.deepStrictEqual(await response.text(), await nativeFetch.text());
|
|
128
126
|
});
|
|
129
127
|
});
|
|
130
128
|
describe('axios', () => {
|
|
@@ -139,7 +137,7 @@ describe('Wayfinder', () => {
|
|
|
139
137
|
});
|
|
140
138
|
it('should fetch the data using axios default function against the target gateway', async () => {
|
|
141
139
|
const [nativeAxios, response] = await Promise.all([
|
|
142
|
-
axios(
|
|
140
|
+
axios(`https://ao.${gatewayUrl}`),
|
|
143
141
|
wayfinder.request('ar://ao'),
|
|
144
142
|
]);
|
|
145
143
|
assert.strictEqual(response.status, 200);
|
|
@@ -150,11 +148,10 @@ describe('Wayfinder', () => {
|
|
|
150
148
|
.filter(([key]) => key.startsWith('x-arns-'));
|
|
151
149
|
const nativeAxiosHeaders = Object.entries(nativeAxios.headers).filter(([key]) => key.startsWith('x-arns-'));
|
|
152
150
|
assert.deepStrictEqual(arnsHeaders.sort(), nativeAxiosHeaders.sort());
|
|
153
|
-
assert.deepStrictEqual(response.data, nativeAxios.data);
|
|
154
151
|
});
|
|
155
152
|
it('should fetch the data using the axios.get method against the target gateway', async () => {
|
|
156
153
|
const [nativeAxios, response] = await Promise.all([
|
|
157
|
-
axios.get(
|
|
154
|
+
axios.get(`https://ao.${gatewayUrl}`),
|
|
158
155
|
wayfinder.request.get('ar://ao'),
|
|
159
156
|
]);
|
|
160
157
|
assert.strictEqual(response.status, 200);
|
|
@@ -165,43 +162,39 @@ describe('Wayfinder', () => {
|
|
|
165
162
|
.filter(([key]) => key.startsWith('x-arns-'));
|
|
166
163
|
const nativeAxiosHeaders = Object.entries(nativeAxios.headers).filter(([key]) => key.startsWith('x-arns-'));
|
|
167
164
|
assert.deepStrictEqual(arnsHeaders.sort(), nativeAxiosHeaders.sort());
|
|
168
|
-
assert.deepStrictEqual(response.data, nativeAxios.data);
|
|
169
165
|
});
|
|
170
166
|
it('should route a non-ar:// url as a normal axios request', async () => {
|
|
171
167
|
const [nativeAxios, response] = await Promise.all([
|
|
172
|
-
axios(
|
|
173
|
-
wayfinder.request(
|
|
168
|
+
axios(`https://${gatewayUrl}/`),
|
|
169
|
+
wayfinder.request(`https://${gatewayUrl}/`),
|
|
174
170
|
]);
|
|
175
171
|
assert.strictEqual(response.status, 200);
|
|
176
172
|
assert.strictEqual(response.status, nativeAxios.status);
|
|
177
|
-
assert.deepStrictEqual(response.data, nativeAxios.data);
|
|
178
173
|
// TODO: ensure the headers are the same excluding unique headers
|
|
179
174
|
});
|
|
180
|
-
for (const api of ['/info', '/
|
|
175
|
+
for (const api of ['/info', '/block/current']) {
|
|
181
176
|
it(`supports native arweave node apis ${api}`, async () => {
|
|
182
177
|
const [nativeAxios, response] = await Promise.all([
|
|
183
|
-
axios(`https
|
|
184
|
-
wayfinder.request(`ar
|
|
178
|
+
axios(`https://${gatewayUrl}${api}`),
|
|
179
|
+
wayfinder.request(`ar://${api}`),
|
|
185
180
|
]);
|
|
186
181
|
assert.strictEqual(response.status, 200);
|
|
187
182
|
assert.strictEqual(response.status, nativeAxios.status);
|
|
188
183
|
// TODO: ensure the headers are the same excluding unique headers
|
|
189
|
-
assert.deepStrictEqual(response.data, nativeAxios.data);
|
|
190
184
|
});
|
|
191
185
|
}
|
|
192
186
|
for (const api of ['/ar-io/info', '/ar-io/__gateway_metrics']) {
|
|
193
187
|
it(`supports native ario node gateway apis ${api}`, async () => {
|
|
194
188
|
const [nativeAxios, response] = await Promise.all([
|
|
195
|
-
axios(`https
|
|
189
|
+
axios(`https://${gatewayUrl}${api}`),
|
|
196
190
|
wayfinder.request(`ar:///${api}`),
|
|
197
191
|
]);
|
|
198
192
|
assert.strictEqual(response.status, 200);
|
|
199
193
|
assert.strictEqual(response.status, nativeAxios.status);
|
|
200
194
|
// TODO: ensure the headers are the same excluding unique headers
|
|
201
|
-
assert.deepStrictEqual(response.data, nativeAxios.data);
|
|
202
195
|
});
|
|
203
196
|
}
|
|
204
|
-
it('should return the error from the target gateway if the route is not found', async () => {
|
|
197
|
+
it.skip('should return the error from the target gateway if the route is not found', async () => {
|
|
205
198
|
const axiosInstance = axios.create({
|
|
206
199
|
validateStatus: () => true, // don't throw so we can compare axios result with wrapped axios result
|
|
207
200
|
});
|
|
@@ -212,11 +205,10 @@ describe('Wayfinder', () => {
|
|
|
212
205
|
}),
|
|
213
206
|
});
|
|
214
207
|
const [nativeAxios, response] = await Promise.all([
|
|
215
|
-
axiosInstance(
|
|
216
|
-
wayfinder.request('
|
|
208
|
+
axiosInstance(`https://${gatewayUrl}/ar-io/not-found`),
|
|
209
|
+
wayfinder.request('ar:///not-found'),
|
|
217
210
|
]);
|
|
218
211
|
assert.strictEqual(response.status, nativeAxios.status);
|
|
219
|
-
assert.strictEqual(response.data, nativeAxios.data);
|
|
220
212
|
});
|
|
221
213
|
});
|
|
222
214
|
describe('got', () => {
|
|
@@ -231,7 +223,7 @@ describe('Wayfinder', () => {
|
|
|
231
223
|
});
|
|
232
224
|
it('should fetch the data using the got default function against the target gateway', async () => {
|
|
233
225
|
const [nativeGot, response] = await Promise.all([
|
|
234
|
-
got(
|
|
226
|
+
got(`https://ao.${gatewayUrl}`),
|
|
235
227
|
wayfinder.request('ar://ao'),
|
|
236
228
|
]);
|
|
237
229
|
assert.strictEqual(response.statusCode, 200);
|
|
@@ -239,10 +231,265 @@ describe('Wayfinder', () => {
|
|
|
239
231
|
assert.deepStrictEqual(response.body, nativeGot.body);
|
|
240
232
|
});
|
|
241
233
|
it('should stream the data using got.stream against the selected target gateway', async () => {
|
|
242
|
-
const nativeBuffer = await buffer(await got.stream(
|
|
234
|
+
const nativeBuffer = await buffer(await got.stream(`https://ao.${gatewayUrl}`, { decompress: false }));
|
|
243
235
|
const wayfinderBuffer = await buffer(await wayfinder.request.stream('ar://ao', { decompress: false }));
|
|
244
236
|
assert.deepStrictEqual(wayfinderBuffer, nativeBuffer);
|
|
245
237
|
});
|
|
246
238
|
});
|
|
247
239
|
});
|
|
240
|
+
describe('events', () => {
|
|
241
|
+
it('should emit events on the wayfinder event emitter', async () => {
|
|
242
|
+
const wayfinder = new Wayfinder({
|
|
243
|
+
httpClient: fetch,
|
|
244
|
+
router: new StaticGatewayRouter({
|
|
245
|
+
gateway: `http://${gatewayUrl}`,
|
|
246
|
+
}),
|
|
247
|
+
verifier: {
|
|
248
|
+
// @ts-expect-error
|
|
249
|
+
verifyData: async (params) => {
|
|
250
|
+
return;
|
|
251
|
+
},
|
|
252
|
+
},
|
|
253
|
+
});
|
|
254
|
+
const events = [];
|
|
255
|
+
wayfinder.emitter.on('verification-failed', (event) => {
|
|
256
|
+
events.push({ type: 'verification-failed', ...event });
|
|
257
|
+
});
|
|
258
|
+
wayfinder.emitter.on('verification-progress', (event) => {
|
|
259
|
+
events.push({ type: 'verification-progress', ...event });
|
|
260
|
+
});
|
|
261
|
+
wayfinder.emitter.on('verification-passed', (event) => {
|
|
262
|
+
events.push({ type: 'verification-passed', ...event });
|
|
263
|
+
});
|
|
264
|
+
// request data and assert the event is emitted
|
|
265
|
+
const response = await wayfinder.request('ar://c7wkwt6TKgcWJUfgvpJ5q5qi4DIZyJ1_TqhjXgURh0U', {
|
|
266
|
+
redirect: 'follow',
|
|
267
|
+
});
|
|
268
|
+
// read the full response body to ensure the stream is fully consumed
|
|
269
|
+
await response.text();
|
|
270
|
+
assert.strictEqual(response.status, 200);
|
|
271
|
+
assert.ok(events.find((e) => e.type === 'verification-passed'), 'Should emit at least one verification-passed');
|
|
272
|
+
});
|
|
273
|
+
it('should execute callbacks provided to the wayfinder constructor', async () => {
|
|
274
|
+
let verificationFailed = false;
|
|
275
|
+
let verificationProgress = false;
|
|
276
|
+
let verificationPassed = false;
|
|
277
|
+
const wayfinder = new Wayfinder({
|
|
278
|
+
httpClient: fetch,
|
|
279
|
+
router: new StaticGatewayRouter({
|
|
280
|
+
gateway: `http://${gatewayUrl}`,
|
|
281
|
+
}),
|
|
282
|
+
events: {
|
|
283
|
+
onVerificationFailed: () => {
|
|
284
|
+
verificationFailed = true;
|
|
285
|
+
},
|
|
286
|
+
onVerificationProgress: () => {
|
|
287
|
+
verificationProgress = true;
|
|
288
|
+
},
|
|
289
|
+
onVerificationPassed: () => {
|
|
290
|
+
verificationPassed = true;
|
|
291
|
+
},
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
const response = await wayfinder.request('ar://c7wkwt6TKgcWJUfgvpJ5q5qi4DIZyJ1_TqhjXgURh0U', {
|
|
295
|
+
redirect: 'follow',
|
|
296
|
+
});
|
|
297
|
+
// read the full response body to ensure the stream is fully consumed
|
|
298
|
+
await response.text();
|
|
299
|
+
assert.strictEqual(response.status, 200);
|
|
300
|
+
assert.ok(verificationFailed === false, 'Should not emit verification-failed');
|
|
301
|
+
assert.ok(verificationProgress, 'Should emit verification-progress');
|
|
302
|
+
assert.ok(verificationPassed, 'Should emit verification-passed');
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
describe.skip('tapAndVerifyRequest', () => {
|
|
306
|
+
describe('Readable', () => {
|
|
307
|
+
it('should duplicate the stream, verify the first and return the second if verification passes', async () => {
|
|
308
|
+
// create a simple readable
|
|
309
|
+
const chunks = [
|
|
310
|
+
Buffer.from('foo'),
|
|
311
|
+
Buffer.from('bar'),
|
|
312
|
+
Buffer.from('baz'),
|
|
313
|
+
];
|
|
314
|
+
const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
315
|
+
// a stream that will emit chunks
|
|
316
|
+
const originalStream = Readable.from(chunks);
|
|
317
|
+
let seen = Buffer.alloc(0);
|
|
318
|
+
const verifyData = async ({ data,
|
|
319
|
+
// @ts-expect-error
|
|
320
|
+
txId, }) => {
|
|
321
|
+
return new Promise((resolve, reject) => {
|
|
322
|
+
data.on('data', (chunk) => {
|
|
323
|
+
seen = Buffer.concat([seen, chunk]);
|
|
324
|
+
});
|
|
325
|
+
data.on('end', () => {
|
|
326
|
+
// Should have seen exactly the full payload
|
|
327
|
+
assert.strictEqual(seen.length, contentLength);
|
|
328
|
+
resolve();
|
|
329
|
+
});
|
|
330
|
+
data.on('error', reject);
|
|
331
|
+
});
|
|
332
|
+
};
|
|
333
|
+
const txId = 'test-tx-1';
|
|
334
|
+
const emitter = new EventEmitter();
|
|
335
|
+
const events = [];
|
|
336
|
+
emitter.on('verification-progress', (e) => {
|
|
337
|
+
console.log('verification-progress', e);
|
|
338
|
+
events.push({ type: 'verification-progress', ...e });
|
|
339
|
+
});
|
|
340
|
+
emitter.on('verification-passed', (e) => events.push({ type: 'verification-passed', ...e }));
|
|
341
|
+
// tap with verification
|
|
342
|
+
const tapped = tapAndVerifyStream({
|
|
343
|
+
originalStream,
|
|
344
|
+
contentLength,
|
|
345
|
+
verifyData,
|
|
346
|
+
txId,
|
|
347
|
+
emitter,
|
|
348
|
+
});
|
|
349
|
+
// read the stream
|
|
350
|
+
const out = [];
|
|
351
|
+
for await (const chunk of tapped) {
|
|
352
|
+
out.push(chunk);
|
|
353
|
+
}
|
|
354
|
+
// assert the stream is the same
|
|
355
|
+
assert.strictEqual(Buffer.concat(out).toString(), Buffer.concat(chunks).toString(), 'The tapped stream should emit exactly the original data');
|
|
356
|
+
assert.ok(events.find((e) => e.type === 'verification-progress'), 'Should emit at least one verification-progress');
|
|
357
|
+
assert.ok(events.find((e) => e.type === 'verification-passed' && e.txId === txId), 'Should emit at least one verification-passed');
|
|
358
|
+
});
|
|
359
|
+
it('should throw an error on the client stream if verification fails', async () => {
|
|
360
|
+
const chunks = [
|
|
361
|
+
Buffer.from('foo'),
|
|
362
|
+
Buffer.from('bar'),
|
|
363
|
+
Buffer.from('baz'),
|
|
364
|
+
];
|
|
365
|
+
const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
366
|
+
// a stream that will emit chunks
|
|
367
|
+
const originalStream = Readable.from(chunks);
|
|
368
|
+
const verifyData = async ({
|
|
369
|
+
// @ts-expect-error
|
|
370
|
+
data, txId, }) => {
|
|
371
|
+
throw new Error('Verification failed for txId: ' + txId);
|
|
372
|
+
};
|
|
373
|
+
const txId = 'test-tx-1';
|
|
374
|
+
const emitter = new EventEmitter();
|
|
375
|
+
const events = [];
|
|
376
|
+
emitter.on('verification-progress', (e) => events.push({ type: 'verification-progress', ...e }));
|
|
377
|
+
emitter.on('verification-failed', (e) => events.push({ type: 'verification-failed', ...e }));
|
|
378
|
+
// tap with verification
|
|
379
|
+
const tapped = tapAndVerifyStream({
|
|
380
|
+
originalStream,
|
|
381
|
+
contentLength,
|
|
382
|
+
verifyData,
|
|
383
|
+
txId,
|
|
384
|
+
emitter,
|
|
385
|
+
});
|
|
386
|
+
// read the stream
|
|
387
|
+
try {
|
|
388
|
+
const out = [];
|
|
389
|
+
for await (const chunk of tapped) {
|
|
390
|
+
out.push(chunk);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
catch (error) {
|
|
394
|
+
assert.ok(events.find((e) => e.type === 'verification-failed' && e.txId === txId), 'Should emit at least one verification-failed');
|
|
395
|
+
// stream should be closed
|
|
396
|
+
assert.ok(tapped.closed);
|
|
397
|
+
}
|
|
398
|
+
});
|
|
399
|
+
});
|
|
400
|
+
describe('ReadableStream', () => {
|
|
401
|
+
it('should duplicate the ReadableStream, verify the first and return the second if verification passes', async () => {
|
|
402
|
+
// create a simple readable
|
|
403
|
+
const chunks = [
|
|
404
|
+
Buffer.from('foo'),
|
|
405
|
+
Buffer.from('bar'),
|
|
406
|
+
Buffer.from('baz'),
|
|
407
|
+
];
|
|
408
|
+
const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
409
|
+
// a stream that will emit chunks
|
|
410
|
+
const originalStream = ReadableStream.from(chunks);
|
|
411
|
+
let seen = Buffer.alloc(0);
|
|
412
|
+
const verifyData = async ({ data,
|
|
413
|
+
// @ts-expect-error
|
|
414
|
+
txId, }) => {
|
|
415
|
+
return new Promise(async (resolve, reject) => {
|
|
416
|
+
const reader = data.getReader();
|
|
417
|
+
while (true) {
|
|
418
|
+
try {
|
|
419
|
+
const { done, value } = await reader.read();
|
|
420
|
+
if (done) {
|
|
421
|
+
resolve();
|
|
422
|
+
break;
|
|
423
|
+
}
|
|
424
|
+
seen = Buffer.concat([seen, value]);
|
|
425
|
+
}
|
|
426
|
+
catch (error) {
|
|
427
|
+
reject(error);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
});
|
|
431
|
+
};
|
|
432
|
+
const txId = 'test-tx-1';
|
|
433
|
+
const emitter = new EventEmitter();
|
|
434
|
+
const events = [];
|
|
435
|
+
emitter.on('verification-progress', (e) => events.push({ type: 'verification-progress', ...e }));
|
|
436
|
+
emitter.on('verification-passed', (e) => events.push({ type: 'verification-passed', ...e }));
|
|
437
|
+
// tap with verification
|
|
438
|
+
const tapped = tapAndVerifyStream({
|
|
439
|
+
originalStream,
|
|
440
|
+
contentLength,
|
|
441
|
+
verifyData,
|
|
442
|
+
txId,
|
|
443
|
+
emitter,
|
|
444
|
+
});
|
|
445
|
+
// read the stream
|
|
446
|
+
const out = [];
|
|
447
|
+
for await (const chunk of tapped) {
|
|
448
|
+
out.push(chunk);
|
|
449
|
+
}
|
|
450
|
+
// assert the stream is the same
|
|
451
|
+
assert.strictEqual(Buffer.concat(out).toString(), Buffer.concat(chunks).toString(), 'The tapped stream should emit exactly the original data');
|
|
452
|
+
assert.ok(events.find((e) => e.type === 'verification-progress'), 'Should emit at least one verification-progress');
|
|
453
|
+
assert.ok(events.find((e) => e.type === 'verification-passed' && e.txId === txId), 'Should emit at least one verification-passed');
|
|
454
|
+
});
|
|
455
|
+
it('should throw an error on the client stream if verification fails', async () => {
|
|
456
|
+
const chunks = [
|
|
457
|
+
Buffer.from('foo'),
|
|
458
|
+
Buffer.from('bar'),
|
|
459
|
+
Buffer.from('baz'),
|
|
460
|
+
];
|
|
461
|
+
const contentLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
462
|
+
// a stream that will emit chunks
|
|
463
|
+
const originalStream = ReadableStream.from(chunks);
|
|
464
|
+
const verifyData = async ({
|
|
465
|
+
// @ts-expect-error
|
|
466
|
+
data, txId, }) => {
|
|
467
|
+
throw new Error('Verification failed for txId: ' + txId);
|
|
468
|
+
};
|
|
469
|
+
const txId = 'test-tx-1';
|
|
470
|
+
const emitter = new EventEmitter();
|
|
471
|
+
const events = [];
|
|
472
|
+
emitter.on('verification-progress', (e) => events.push({ type: 'verification-progress', ...e }));
|
|
473
|
+
emitter.on('verification-failed', (e) => events.push({ type: 'verification-failed', ...e }));
|
|
474
|
+
// tap with verification
|
|
475
|
+
const tapped = tapAndVerifyStream({
|
|
476
|
+
originalStream,
|
|
477
|
+
contentLength,
|
|
478
|
+
verifyData,
|
|
479
|
+
txId,
|
|
480
|
+
emitter,
|
|
481
|
+
});
|
|
482
|
+
// read the stream
|
|
483
|
+
try {
|
|
484
|
+
const out = [];
|
|
485
|
+
for await (const chunk of tapped) {
|
|
486
|
+
out.push(chunk);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
catch (error) {
|
|
490
|
+
assert.ok(events.find((e) => e.type === 'verification-failed' && e.txId === txId), 'Should emit at least one verification-failed');
|
|
491
|
+
}
|
|
492
|
+
});
|
|
493
|
+
});
|
|
494
|
+
});
|
|
248
495
|
});
|
package/lib/esm/utils/arweave.js
CHANGED
|
@@ -107,7 +107,7 @@ export const getEpochDataFromGqlFallback = async ({ ao, epochIndex, processId =
|
|
|
107
107
|
if (!messageResult) {
|
|
108
108
|
continue;
|
|
109
109
|
}
|
|
110
|
-
for (const message of messageResult
|
|
110
|
+
for (const message of messageResult?.Messages ?? []) {
|
|
111
111
|
const data = JSON.parse(message.Data);
|
|
112
112
|
const tags = message.Tags;
|
|
113
113
|
// check if the message results include epoch-distribution-notice for the requested epoch index
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import { createHash } from 'crypto';
|
|
17
|
+
import { toB64Url } from './base64.js';
|
|
18
|
+
export const hashReadableToB64Url = (stream, algorithm = 'sha256') => {
|
|
19
|
+
return new Promise((resolve, reject) => {
|
|
20
|
+
const hash = createHash(algorithm);
|
|
21
|
+
stream.on('data', (chunk) => hash.update(chunk));
|
|
22
|
+
stream.on('end', () => resolve(toB64Url(hash.digest())));
|
|
23
|
+
stream.on('error', (err) => reject(err));
|
|
24
|
+
});
|
|
25
|
+
};
|
|
26
|
+
export const hashReadableStreamToB64Url = (stream, algorithm = 'sha256') => {
|
|
27
|
+
return new Promise((resolve, reject) => {
|
|
28
|
+
const hash = createHash(algorithm);
|
|
29
|
+
const reader = stream.getReader();
|
|
30
|
+
const read = async () => {
|
|
31
|
+
try {
|
|
32
|
+
const { done, value } = await reader.read();
|
|
33
|
+
if (done) {
|
|
34
|
+
resolve(toB64Url(hash.digest()));
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
hash.update(value);
|
|
38
|
+
read();
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
reject(err);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
read().catch(reject);
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
export const hashBufferToB64Url = (buffer, algorithm = 'sha256') => {
|
|
49
|
+
return toB64Url(createHash(algorithm).update(buffer).digest());
|
|
50
|
+
};
|
package/lib/esm/version.js
CHANGED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import { DataHashProvider, DataRootProvider } from '../../../types/wayfinder.js';
|
|
17
|
+
import { GatewaysProvider } from '../gateways.js';
|
|
18
|
+
export declare class TrustedGatewaysHashProvider implements DataHashProvider, DataRootProvider {
|
|
19
|
+
private gatewaysProvider;
|
|
20
|
+
constructor({ gatewaysProvider, }: {
|
|
21
|
+
gatewaysProvider: GatewaysProvider;
|
|
22
|
+
});
|
|
23
|
+
/**
|
|
24
|
+
* Gets the digest for a given txId from all trusted gateways and ensures they all match.
|
|
25
|
+
* @param txId - The txId to get the digest for.
|
|
26
|
+
* @returns The digest for the given txId.
|
|
27
|
+
*/
|
|
28
|
+
getHash({ txId, }: {
|
|
29
|
+
txId: string;
|
|
30
|
+
}): Promise<{
|
|
31
|
+
hash: string;
|
|
32
|
+
algorithm: 'sha256';
|
|
33
|
+
}>;
|
|
34
|
+
/**
|
|
35
|
+
* Get the data root for a given txId from all trusted gateways and ensure they all match.
|
|
36
|
+
* @param txId - The txId to get the data root for.
|
|
37
|
+
* @returns The data root for the given txId.
|
|
38
|
+
*/
|
|
39
|
+
getDataRoot({ txId }: {
|
|
40
|
+
txId: string;
|
|
41
|
+
}): Promise<string>;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Note from @djwhitt
|
|
45
|
+
*
|
|
46
|
+
* Calculating data roots this way is fine, but it may not reproduce the original data root.
|
|
47
|
+
* We could also implement a data root verifier that pulls all the chunks, checks that they
|
|
48
|
+
* reproduce the expected data root, and then compares the concatenated chunk data to the
|
|
49
|
+
* original data retrieved. That would take a while, but it should be able to verify any L1
|
|
50
|
+
* data where we can find the chunks.
|
|
51
|
+
*/
|
|
@@ -15,21 +15,30 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { AoARIORead, AoGatewayWithAddress } from '../../types/io.js';
|
|
17
17
|
export interface GatewaysProvider {
|
|
18
|
-
getGateways(): Promise<
|
|
18
|
+
getGateways(): Promise<URL[]>;
|
|
19
19
|
}
|
|
20
|
-
export declare class
|
|
20
|
+
export declare class NetworkGatewaysProvider implements GatewaysProvider {
|
|
21
21
|
private ario;
|
|
22
|
-
|
|
22
|
+
private sortBy;
|
|
23
|
+
private sortOrder;
|
|
24
|
+
private limit;
|
|
25
|
+
private filter;
|
|
26
|
+
constructor({ ario, sortBy, sortOrder, limit, filter, }: {
|
|
23
27
|
ario: AoARIORead;
|
|
28
|
+
sortBy?: 'totalDelegatedStake' | 'operatorStake' | 'startTimestamp';
|
|
29
|
+
sortOrder?: 'asc' | 'desc';
|
|
30
|
+
limit?: number;
|
|
31
|
+
blocklist?: string[];
|
|
32
|
+
filter?: (gateway: AoGatewayWithAddress) => boolean;
|
|
24
33
|
});
|
|
25
|
-
getGateways(): Promise<
|
|
34
|
+
getGateways(): Promise<URL[]>;
|
|
26
35
|
}
|
|
27
36
|
export declare class StaticGatewaysProvider implements GatewaysProvider {
|
|
28
37
|
private gateways;
|
|
29
38
|
constructor({ gateways }: {
|
|
30
|
-
gateways:
|
|
39
|
+
gateways: string[];
|
|
31
40
|
});
|
|
32
|
-
getGateways(): Promise<
|
|
41
|
+
getGateways(): Promise<URL[]>;
|
|
33
42
|
}
|
|
34
43
|
export declare class SimpleCacheGatewaysProvider implements GatewaysProvider {
|
|
35
44
|
private gatewaysProvider;
|
|
@@ -40,5 +49,5 @@ export declare class SimpleCacheGatewaysProvider implements GatewaysProvider {
|
|
|
40
49
|
gatewaysProvider: GatewaysProvider;
|
|
41
50
|
ttlSeconds?: number;
|
|
42
51
|
});
|
|
43
|
-
getGateways(): Promise<
|
|
52
|
+
getGateways(): Promise<URL[]>;
|
|
44
53
|
}
|
|
@@ -16,5 +16,8 @@
|
|
|
16
16
|
export * from './wayfinder.js';
|
|
17
17
|
export * from './routers/random.js';
|
|
18
18
|
export * from './routers/priority.js';
|
|
19
|
-
export * from './routers/
|
|
19
|
+
export * from './routers/static.js';
|
|
20
20
|
export * from './gateways.js';
|
|
21
|
+
export * from './gateways/trusted-gateways.js';
|
|
22
|
+
export * from './verification/data-root-verifier.js';
|
|
23
|
+
export * from './verification/hash-verifier.js';
|