@jinn-network/core 0.1.2-canary.b91a66df → 0.1.2-canary.bebeb033
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/corpus-read/acquire.js +19 -1
- package/dist/corpus-read/ipfs.d.ts +74 -0
- package/dist/corpus-read/ipfs.js +372 -24
- package/package.json +2 -2
|
@@ -8,11 +8,24 @@
|
|
|
8
8
|
import { createHash } from 'node:crypto';
|
|
9
9
|
import { fetchArtifactContent } from './fetch-artifact.js';
|
|
10
10
|
import { AcquireError, HashMismatchError } from './types.js';
|
|
11
|
-
import { fetchFromIpfs as defaultFetchFromIpfs } from './ipfs.js';
|
|
11
|
+
import { classifyIpfsFetchFailure, fetchFromIpfs as defaultFetchFromIpfs } from './ipfs.js';
|
|
12
12
|
const DONATION_ARTIFACT_ENCODING = 'jinn.artifact.donation.v1';
|
|
13
13
|
function sha256Hex(buf) {
|
|
14
14
|
return createHash('sha256').update(buf).digest('hex');
|
|
15
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* Log an IPFS read that the acquisition chain is about to fall through, unless
|
|
18
|
+
* the gateway actually answered "not there" (#3441). A cap refusal is positive
|
|
19
|
+
* evidence the content exists, so it must not read like an ordinary miss.
|
|
20
|
+
*/
|
|
21
|
+
function warnIpfsFallThrough(subject, error) {
|
|
22
|
+
const classification = classifyIpfsFetchFailure(error);
|
|
23
|
+
if (classification === 'not-found')
|
|
24
|
+
return;
|
|
25
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
26
|
+
console.warn(`[corpus-read] IPFS source for ${subject} could not be used (${classification}), `
|
|
27
|
+
+ `falling through to the next source: ${detail}`);
|
|
28
|
+
}
|
|
16
29
|
function decodeDonationArtifact(raw, expectedSha256) {
|
|
17
30
|
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
18
31
|
throw new Error('donation artifact payload is not an object');
|
|
@@ -84,6 +97,11 @@ export async function acquireArtifactContent(args) {
|
|
|
84
97
|
// Donated IPFS is an opportunistic fast path. Gateway failures and
|
|
85
98
|
// malformed donation payloads should not block the acquisition chain;
|
|
86
99
|
// only verified byte hash mismatches remain fatal.
|
|
100
|
+
//
|
|
101
|
+
// Swallowing the failure silently, though, turns a size-policy refusal or
|
|
102
|
+
// a blocked redirect into an invisible data gap (#3441). Classify before
|
|
103
|
+
// swallowing so anything that is not proven absence reaches the operator.
|
|
104
|
+
warnIpfsFallThrough(`donation artifact ${sha256}`, err);
|
|
87
105
|
}
|
|
88
106
|
}
|
|
89
107
|
// 3. Self-store fast path
|
|
@@ -1,3 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default cap on any single gateway response, JSON or raw; a caller that
|
|
3
|
+
* legitimately reads larger payloads may raise it per call through
|
|
4
|
+
* {@link FetchFromIpfsOptions.maxResponseBytes} (#3441). Corpus manifests and
|
|
5
|
+
* donation artifacts are JSON envelopes orders of magnitude smaller than this
|
|
6
|
+
* (#3410); the raw-bytes path added in #3438 also carries source-bundle files
|
|
7
|
+
* and sealed documents, which are larger but nowhere near this. A response
|
|
8
|
+
* above it is hostile or broken, and buffering it would exhaust memory — so
|
|
9
|
+
* size this against the largest legitimate *source file*, not against the
|
|
10
|
+
* JSON envelopes alone.
|
|
11
|
+
*/
|
|
12
|
+
export declare const DEFAULT_MAX_IPFS_RESPONSE_BYTES: number;
|
|
13
|
+
/**
|
|
14
|
+
* A gateway response the byte cap refused. FAILURE IS NOT ABSENCE (#2647, #3451): the gateway
|
|
15
|
+
* answered FOR this content, so the refusal is positive evidence that it EXISTS and is merely
|
|
16
|
+
* larger than policy allows. (On the declared-`content-length` path the body is discarded unread,
|
|
17
|
+
* so the evidence is the headers rather than the bytes — still an answer, not a miss.) A caller
|
|
18
|
+
* that reports it as "not on IPFS" turns a size-policy decision into a silent data gap.
|
|
19
|
+
*/
|
|
20
|
+
export declare class IpfsResponseTooLargeError extends Error {
|
|
21
|
+
readonly limitBytes: number;
|
|
22
|
+
readonly declaredBytes?: number | undefined;
|
|
23
|
+
readonly name = "IpfsResponseTooLargeError";
|
|
24
|
+
constructor(limitBytes: number, declaredBytes?: number | undefined);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The gateway answered that this content is not there. The one failure mode that IS absence
|
|
28
|
+
* (#3451) — every other one leaves the question unanswered.
|
|
29
|
+
*/
|
|
30
|
+
export declare class IpfsContentNotFoundError extends Error {
|
|
31
|
+
readonly status: number;
|
|
32
|
+
readonly name = "IpfsContentNotFoundError";
|
|
33
|
+
constructor(message: string, status: number);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Every gateway x CID candidate failed. Carries the per-candidate causes so a caller can classify
|
|
37
|
+
* the whole operation with {@link classifyIpfsFetchFailure} instead of string-matching the joined
|
|
38
|
+
* message, which is a display artifact and not a contract.
|
|
39
|
+
*/
|
|
40
|
+
export declare class IpfsFetchFailedError extends Error {
|
|
41
|
+
readonly causes: readonly unknown[];
|
|
42
|
+
readonly name = "IpfsFetchFailedError";
|
|
43
|
+
constructor(message: string, causes: readonly unknown[]);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Classify a failed IPFS fetch into the three answers a caller can act on differently:
|
|
47
|
+
*
|
|
48
|
+
* - `'too-large'` — at least one candidate served the content and the byte cap refused it. This
|
|
49
|
+
* dominates: a size refusal is positive proof of presence, so it outranks any number of
|
|
50
|
+
* not-found answers from other candidates.
|
|
51
|
+
* - `'not-found'` — every candidate answered, and every answer was "not there". Genuine absence.
|
|
52
|
+
* - `'unavailable'` — anything else (transport error, timeout, malformed candidate URL). Nothing
|
|
53
|
+
* was learned about whether the content exists.
|
|
54
|
+
*
|
|
55
|
+
* `'not-found'` is deliberately strict, and in production it is therefore the rarest answer: a
|
|
56
|
+
* default fetch also tries the `ipfs.io` fallback, which typically stalls into a 504 or an abort
|
|
57
|
+
* for an unpinned digest rather than answering 404. Such a run classifies `'unavailable'` — the
|
|
58
|
+
* safe direction, since absence is never claimed without proof of it.
|
|
59
|
+
*/
|
|
60
|
+
export declare function classifyIpfsFetchFailure(error: unknown): 'too-large' | 'not-found' | 'unavailable';
|
|
1
61
|
export declare function normalizeIpfsGatewayBase(gatewayUrl: string): string;
|
|
2
62
|
export declare function buildIpfsHexCidCandidatesFromPartialHex(hex: string): string[];
|
|
3
63
|
export declare function buildIpfsFetchCidPathCandidates(cidOrPath: string): string[];
|
|
@@ -9,6 +69,20 @@ export type FetchFromIpfsOptions = {
|
|
|
9
69
|
* - string → alternate fallback (normalized via `normalizeIpfsGatewayBase`)
|
|
10
70
|
*/
|
|
11
71
|
fallbackGatewayBase?: string | false;
|
|
72
|
+
/**
|
|
73
|
+
* Byte cap for one gateway response, defaulting to
|
|
74
|
+
* {@link DEFAULT_MAX_IPFS_RESPONSE_BYTES}. Raise it only for a call site that
|
|
75
|
+
* legitimately reads larger payloads (#3441) — the default is sized for JSON
|
|
76
|
+
* envelopes and source files, and a caller that raises it accepts buffering
|
|
77
|
+
* that many bytes. Values below 1 fall back to the default.
|
|
78
|
+
*/
|
|
79
|
+
maxResponseBytes?: number;
|
|
12
80
|
};
|
|
13
81
|
/** Read-only multi-codec, primary-plus-fallback IPFS JSON fetch. */
|
|
14
82
|
export declare function fetchFromIpfs(gatewayUrl: string, cid: string, opts?: FetchFromIpfsOptions): Promise<unknown>;
|
|
83
|
+
/**
|
|
84
|
+
* Read-only multi-codec, primary-plus-fallback IPFS fetch returning the exact
|
|
85
|
+
* bytes stored at the CID — no JSON parse/re-encode roundtrip. Use this
|
|
86
|
+
* whenever the bytes will be hashed, or whenever the content is not JSON.
|
|
87
|
+
*/
|
|
88
|
+
export declare function fetchBytesFromIpfs(gatewayUrl: string, cid: string, opts?: FetchFromIpfsOptions): Promise<Uint8Array>;
|
package/dist/corpus-read/ipfs.js
CHANGED
|
@@ -1,9 +1,113 @@
|
|
|
1
1
|
const IPFS_FETCH_TIMEOUT_MS = 15_000;
|
|
2
|
+
/**
|
|
3
|
+
* Bound on one whole `fetchFromIpfs` / `fetchBytesFromIpfs` call. The
|
|
4
|
+
* per-attempt timer above is re-armed for every gateway x CID candidate, so
|
|
5
|
+
* without this a single call could legitimately run for the product of the
|
|
6
|
+
* two. Every candidate is still attempted: the per-attempt timer is clamped
|
|
7
|
+
* to an equal share of whatever budget remains.
|
|
8
|
+
*/
|
|
9
|
+
const IPFS_TOTAL_FETCH_TIMEOUT_MS = 45_000;
|
|
10
|
+
/**
|
|
11
|
+
* Default cap on any single gateway response, JSON or raw; a caller that
|
|
12
|
+
* legitimately reads larger payloads may raise it per call through
|
|
13
|
+
* {@link FetchFromIpfsOptions.maxResponseBytes} (#3441). Corpus manifests and
|
|
14
|
+
* donation artifacts are JSON envelopes orders of magnitude smaller than this
|
|
15
|
+
* (#3410); the raw-bytes path added in #3438 also carries source-bundle files
|
|
16
|
+
* and sealed documents, which are larger but nowhere near this. A response
|
|
17
|
+
* above it is hostile or broken, and buffering it would exhaust memory — so
|
|
18
|
+
* size this against the largest legitimate *source file*, not against the
|
|
19
|
+
* JSON envelopes alone.
|
|
20
|
+
*/
|
|
21
|
+
export const DEFAULT_MAX_IPFS_RESPONSE_BYTES = 8 * 1024 * 1024;
|
|
22
|
+
const MAX_IPFS_REDIRECT_HOPS = 3;
|
|
23
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
2
24
|
const FALLBACK_IPFS_GATEWAY_BASE = 'https://ipfs.io/ipfs/';
|
|
25
|
+
/**
|
|
26
|
+
* A gateway response the byte cap refused. FAILURE IS NOT ABSENCE (#2647, #3451): the gateway
|
|
27
|
+
* answered FOR this content, so the refusal is positive evidence that it EXISTS and is merely
|
|
28
|
+
* larger than policy allows. (On the declared-`content-length` path the body is discarded unread,
|
|
29
|
+
* so the evidence is the headers rather than the bytes — still an answer, not a miss.) A caller
|
|
30
|
+
* that reports it as "not on IPFS" turns a size-policy decision into a silent data gap.
|
|
31
|
+
*/
|
|
32
|
+
export class IpfsResponseTooLargeError extends Error {
|
|
33
|
+
limitBytes;
|
|
34
|
+
declaredBytes;
|
|
35
|
+
name = 'IpfsResponseTooLargeError';
|
|
36
|
+
constructor(limitBytes, declaredBytes) {
|
|
37
|
+
super(`IPFS response exceeds the ${limitBytes}-byte cap`
|
|
38
|
+
+ (declaredBytes === undefined ? '' : ` (content-length ${declaredBytes})`));
|
|
39
|
+
this.limitBytes = limitBytes;
|
|
40
|
+
this.declaredBytes = declaredBytes;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The gateway answered that this content is not there. The one failure mode that IS absence
|
|
45
|
+
* (#3451) — every other one leaves the question unanswered.
|
|
46
|
+
*/
|
|
47
|
+
export class IpfsContentNotFoundError extends Error {
|
|
48
|
+
status;
|
|
49
|
+
name = 'IpfsContentNotFoundError';
|
|
50
|
+
constructor(message, status) {
|
|
51
|
+
super(message);
|
|
52
|
+
this.status = status;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Every gateway x CID candidate failed. Carries the per-candidate causes so a caller can classify
|
|
57
|
+
* the whole operation with {@link classifyIpfsFetchFailure} instead of string-matching the joined
|
|
58
|
+
* message, which is a display artifact and not a contract.
|
|
59
|
+
*/
|
|
60
|
+
export class IpfsFetchFailedError extends Error {
|
|
61
|
+
causes;
|
|
62
|
+
name = 'IpfsFetchFailedError';
|
|
63
|
+
constructor(message, causes) {
|
|
64
|
+
super(message);
|
|
65
|
+
this.causes = causes;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Classify a failed IPFS fetch into the three answers a caller can act on differently:
|
|
70
|
+
*
|
|
71
|
+
* - `'too-large'` — at least one candidate served the content and the byte cap refused it. This
|
|
72
|
+
* dominates: a size refusal is positive proof of presence, so it outranks any number of
|
|
73
|
+
* not-found answers from other candidates.
|
|
74
|
+
* - `'not-found'` — every candidate answered, and every answer was "not there". Genuine absence.
|
|
75
|
+
* - `'unavailable'` — anything else (transport error, timeout, malformed candidate URL). Nothing
|
|
76
|
+
* was learned about whether the content exists.
|
|
77
|
+
*
|
|
78
|
+
* `'not-found'` is deliberately strict, and in production it is therefore the rarest answer: a
|
|
79
|
+
* default fetch also tries the `ipfs.io` fallback, which typically stalls into a 504 or an abort
|
|
80
|
+
* for an unpinned digest rather than answering 404. Such a run classifies `'unavailable'` — the
|
|
81
|
+
* safe direction, since absence is never claimed without proof of it.
|
|
82
|
+
*/
|
|
83
|
+
export function classifyIpfsFetchFailure(error) {
|
|
84
|
+
const causes = error instanceof IpfsFetchFailedError ? error.causes : [error];
|
|
85
|
+
if (causes.some((cause) => cause instanceof IpfsResponseTooLargeError))
|
|
86
|
+
return 'too-large';
|
|
87
|
+
if (causes.length > 0 && causes.every((cause) => cause instanceof IpfsContentNotFoundError)) {
|
|
88
|
+
return 'not-found';
|
|
89
|
+
}
|
|
90
|
+
return 'unavailable';
|
|
91
|
+
}
|
|
3
92
|
export function normalizeIpfsGatewayBase(gatewayUrl) {
|
|
4
93
|
let normalized = gatewayUrl.trim();
|
|
5
94
|
if (normalized === '')
|
|
6
95
|
normalized = 'https://gateway.autonolas.tech';
|
|
96
|
+
// Drop any userinfo at the source. `fetch` rejects a credentialed URL
|
|
97
|
+
// outright, and its own error message quotes the URL back — so a gateway
|
|
98
|
+
// configured Infura-style would otherwise put its secret into every
|
|
99
|
+
// aggregated fetch error, which callers log.
|
|
100
|
+
try {
|
|
101
|
+
const parsed = new URL(normalized);
|
|
102
|
+
if (parsed.username !== '' || parsed.password !== '') {
|
|
103
|
+
parsed.username = '';
|
|
104
|
+
parsed.password = '';
|
|
105
|
+
normalized = parsed.toString();
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
// Not an absolute URL; leave it to the caller's own failure path.
|
|
110
|
+
}
|
|
7
111
|
normalized = normalized.replace(/\/+$/, '');
|
|
8
112
|
if (!normalized.toLowerCase().endsWith('/ipfs'))
|
|
9
113
|
normalized = `${normalized}/ipfs`;
|
|
@@ -29,15 +133,164 @@ export function buildIpfsFetchCidPathCandidates(cidOrPath) {
|
|
|
29
133
|
}
|
|
30
134
|
return [value];
|
|
31
135
|
}
|
|
32
|
-
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
136
|
+
function isHostInGatewayFamily(host, gatewayHost) {
|
|
137
|
+
const candidate = host.toLowerCase();
|
|
138
|
+
const gateway = gatewayHost.toLowerCase();
|
|
139
|
+
return candidate === gateway || candidate.endsWith(`.${gateway}`);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* The port a request to `url` actually reaches. `URL.port` is the empty string
|
|
143
|
+
* whenever the port is the scheme default, so comparing it raw makes
|
|
144
|
+
* `http://gw.example` (80) and `https://gw.example` (443) look identical
|
|
145
|
+
* (#3439). Resolving the default first keeps an http-to-https hop
|
|
146
|
+
* distinguishable, which the downgrade rule cannot catch because it only fires
|
|
147
|
+
* in the other direction.
|
|
148
|
+
*/
|
|
149
|
+
function effectivePort(url) {
|
|
150
|
+
if (url.port !== '')
|
|
151
|
+
return url.port;
|
|
152
|
+
return url.protocol === 'https:' ? '443' : '80';
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* A gateway may legitimately redirect the path form of a CID to its subdomain
|
|
156
|
+
* form, so hops are allowed inside the configured gateway's host family. They
|
|
157
|
+
* are never allowed to leave it, to reach a different port on it, to downgrade
|
|
158
|
+
* the transport, or to smuggle credentials.
|
|
159
|
+
*/
|
|
160
|
+
function assertRedirectAllowed(next, current, gateway) {
|
|
161
|
+
if (next.protocol !== 'http:' && next.protocol !== 'https:') {
|
|
162
|
+
throw new Error(`IPFS redirect uses an unsupported scheme (${next.protocol})`);
|
|
36
163
|
}
|
|
164
|
+
if (current.protocol === 'https:' && next.protocol === 'http:') {
|
|
165
|
+
throw new Error('IPFS redirect downgrades https to http');
|
|
166
|
+
}
|
|
167
|
+
if (next.username !== '' || next.password !== '') {
|
|
168
|
+
throw new Error('IPFS redirect carries embedded credentials');
|
|
169
|
+
}
|
|
170
|
+
if (!isHostInGatewayFamily(next.hostname, gateway.hostname)) {
|
|
171
|
+
throw new Error(`IPFS redirect leaves the configured gateway (${next.hostname})`);
|
|
172
|
+
}
|
|
173
|
+
// Host family alone would let a self-hosted gateway (`http://127.0.0.1:8080/ipfs/`)
|
|
174
|
+
// pivot onto any other service on the same host. Compare the ports the
|
|
175
|
+
// requests actually reach, not the literal `URL.port` (#3439).
|
|
176
|
+
const nextPort = effectivePort(next);
|
|
177
|
+
if (nextPort !== effectivePort(gateway)) {
|
|
178
|
+
throw new Error(`IPFS redirect changes the gateway port (${nextPort})`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/** Location for an error message, with any configured gateway credentials dropped. */
|
|
182
|
+
function displayUrl(url) {
|
|
183
|
+
return `${url.origin}${url.pathname}`.slice(0, 100);
|
|
184
|
+
}
|
|
185
|
+
async function discardBody(response) {
|
|
186
|
+
try {
|
|
187
|
+
await response.body?.cancel();
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
// A body that refuses to cancel is not worth failing the request over.
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/** Read a response body as bytes, refusing anything past `maxBytes`. */
|
|
194
|
+
async function readBoundedBytes(response, maxBytes) {
|
|
195
|
+
const declared = Number(response.headers.get('content-length'));
|
|
196
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
197
|
+
await discardBody(response);
|
|
198
|
+
throw new IpfsResponseTooLargeError(maxBytes, declared);
|
|
199
|
+
}
|
|
200
|
+
const body = response.body;
|
|
201
|
+
if (!body)
|
|
202
|
+
return new Uint8Array(0);
|
|
203
|
+
const reader = body.getReader();
|
|
204
|
+
const chunks = [];
|
|
205
|
+
let total = 0;
|
|
206
|
+
try {
|
|
207
|
+
for (;;) {
|
|
208
|
+
const { done, value } = await reader.read();
|
|
209
|
+
if (done)
|
|
210
|
+
break;
|
|
211
|
+
if (!value)
|
|
212
|
+
continue;
|
|
213
|
+
total += value.byteLength;
|
|
214
|
+
if (total > maxBytes) {
|
|
215
|
+
throw new IpfsResponseTooLargeError(maxBytes);
|
|
216
|
+
}
|
|
217
|
+
chunks.push(value);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
try {
|
|
222
|
+
await reader.cancel();
|
|
223
|
+
}
|
|
224
|
+
catch {
|
|
225
|
+
// Already terminal; the read result (or throw) above is what matters.
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
const joined = new Uint8Array(total);
|
|
229
|
+
let offset = 0;
|
|
230
|
+
for (const chunk of chunks) {
|
|
231
|
+
joined.set(chunk, offset);
|
|
232
|
+
offset += chunk.byteLength;
|
|
233
|
+
}
|
|
234
|
+
return joined;
|
|
235
|
+
}
|
|
236
|
+
/** Read a response body as text, refusing anything past `maxBytes`. */
|
|
237
|
+
async function readBoundedText(response, maxBytes) {
|
|
238
|
+
const joined = await readBoundedBytes(response, maxBytes);
|
|
239
|
+
try {
|
|
240
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(joined);
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
throw new Error('IPFS response is not valid UTF-8');
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Request `url`, resolving redirects here rather than in `fetch`, so every hop
|
|
248
|
+
* is revalidated against the configured gateway before it is requested.
|
|
249
|
+
* Returns the first non-redirect, ok response; its body is still unread.
|
|
250
|
+
*/
|
|
251
|
+
async function fetchThroughGateway(url, signal) {
|
|
252
|
+
const gateway = url;
|
|
253
|
+
let current = new URL(url);
|
|
254
|
+
for (let hop = 0;; hop += 1) {
|
|
255
|
+
const response = await fetch(current, { method: 'GET', redirect: 'manual', signal });
|
|
256
|
+
if (REDIRECT_STATUSES.has(response.status)) {
|
|
257
|
+
await discardBody(response);
|
|
258
|
+
if (hop >= MAX_IPFS_REDIRECT_HOPS) {
|
|
259
|
+
throw new Error(`IPFS fetch exceeded ${MAX_IPFS_REDIRECT_HOPS} redirects`);
|
|
260
|
+
}
|
|
261
|
+
const location = response.headers.get('location');
|
|
262
|
+
if (location === null || location.trim() === '') {
|
|
263
|
+
throw new Error(`IPFS gateway returned ${response.status} without a Location header`);
|
|
264
|
+
}
|
|
265
|
+
let next;
|
|
266
|
+
try {
|
|
267
|
+
next = new URL(location, current);
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
throw new Error('IPFS redirect Location is not a valid URL');
|
|
271
|
+
}
|
|
272
|
+
assertRedirectAllowed(next, current, gateway);
|
|
273
|
+
current = next;
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
if (!response.ok) {
|
|
277
|
+
await discardBody(response);
|
|
278
|
+
const message = `IPFS fetch failed: ${response.status} ${response.statusText} ` +
|
|
279
|
+
`(${displayUrl(current)}…)`;
|
|
280
|
+
// 404/410 is the gateway ANSWERING "not there" — the only status that carries absence.
|
|
281
|
+
// Every other one (429, 5xx, …) leaves the question open and stays a plain failure.
|
|
282
|
+
if (response.status === 404 || response.status === 410) {
|
|
283
|
+
throw new IpfsContentNotFoundError(message, response.status);
|
|
284
|
+
}
|
|
285
|
+
throw new Error(message);
|
|
286
|
+
}
|
|
287
|
+
return response;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
async function fetchJson(url, signal, maxBytes) {
|
|
291
|
+
const response = await fetchThroughGateway(url, signal);
|
|
37
292
|
const contentType = response.headers.get('content-type') ?? '';
|
|
38
|
-
|
|
39
|
-
return response.json();
|
|
40
|
-
const text = await response.text();
|
|
293
|
+
const text = await readBoundedText(response, maxBytes);
|
|
41
294
|
try {
|
|
42
295
|
return JSON.parse(text);
|
|
43
296
|
}
|
|
@@ -45,6 +298,16 @@ async function fetchJson(url, signal) {
|
|
|
45
298
|
throw new Error(`IPFS response is not JSON (content-type: ${contentType || 'none'})`);
|
|
46
299
|
}
|
|
47
300
|
}
|
|
301
|
+
async function fetchBytes(url, signal, maxBytes) {
|
|
302
|
+
return readBoundedBytes(await fetchThroughGateway(url, signal), maxBytes);
|
|
303
|
+
}
|
|
304
|
+
function resolveMaxResponseBytes(opts) {
|
|
305
|
+
const requested = opts?.maxResponseBytes;
|
|
306
|
+
if (typeof requested !== 'number' || !Number.isFinite(requested) || requested < 1) {
|
|
307
|
+
return DEFAULT_MAX_IPFS_RESPONSE_BYTES;
|
|
308
|
+
}
|
|
309
|
+
return Math.floor(requested);
|
|
310
|
+
}
|
|
48
311
|
function resolveFallbackGatewayBases(opts) {
|
|
49
312
|
if (opts?.fallbackGatewayBase === false)
|
|
50
313
|
return [];
|
|
@@ -53,29 +316,114 @@ function resolveFallbackGatewayBases(opts) {
|
|
|
53
316
|
}
|
|
54
317
|
return [['fallback', FALLBACK_IPFS_GATEWAY_BASE]];
|
|
55
318
|
}
|
|
56
|
-
/**
|
|
57
|
-
|
|
319
|
+
/**
|
|
320
|
+
* Resolve one CID path against its gateway base and refuse anything that leaves
|
|
321
|
+
* the base's `/ipfs/` prefix (#3440).
|
|
322
|
+
*
|
|
323
|
+
* The CID reaches this from a manifest, so plain interpolation let
|
|
324
|
+
* `../../evil` walk out of the prefix and issue `GET /evil` against the
|
|
325
|
+
* gateway. Resolving through `URL` normalizes the `..` segments; comparing the
|
|
326
|
+
* result back against the base then also rejects the absolute-path (`/admin`)
|
|
327
|
+
* and absolute-URL (`https://evil/`) shapes. A strict CID regex would be the
|
|
328
|
+
* wrong guard: `buildIpfsFetchCidPathCandidates` documents `<cid>/path/to/file`
|
|
329
|
+
* as a legitimate gateway path.
|
|
330
|
+
*
|
|
331
|
+
* `URL` does not decode percent-encoded segments, so a `%2e%2e%2f` spelling
|
|
332
|
+
* still leaves the prefix intact here and is resolved, if at all, by the
|
|
333
|
+
* gateway. That is unchanged from the pre-guard behaviour and stays bounded by
|
|
334
|
+
* the same two properties: the origin is pinned, and the bytes are sha256-gated
|
|
335
|
+
* downstream in `acquire.ts`.
|
|
336
|
+
*/
|
|
337
|
+
function resolveGatewayCandidateUrl(baseUrl, cidPath) {
|
|
338
|
+
let base;
|
|
339
|
+
let resolved;
|
|
340
|
+
try {
|
|
341
|
+
base = new URL(baseUrl);
|
|
342
|
+
resolved = new URL(cidPath, base);
|
|
343
|
+
}
|
|
344
|
+
catch {
|
|
345
|
+
return { reason: 'candidate URL could not be parsed' };
|
|
346
|
+
}
|
|
347
|
+
if (resolved.origin !== base.origin || !resolved.pathname.startsWith(base.pathname)) {
|
|
348
|
+
return { reason: 'candidate URL escapes the gateway path prefix' };
|
|
349
|
+
}
|
|
350
|
+
return { url: resolved };
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Run every gateway x CID candidate through `read`, under one whole-operation
|
|
354
|
+
* deadline. Shared by the JSON and raw-bytes entry points so both inherit the
|
|
355
|
+
* same redirect revalidation, byte cap, and deadline (#3438).
|
|
356
|
+
*/
|
|
357
|
+
async function fetchCandidatesFromIpfs(gatewayUrl, cid, opts, read, failureLabel) {
|
|
58
358
|
const primary = normalizeIpfsGatewayBase(gatewayUrl);
|
|
59
359
|
const gateways = [
|
|
60
360
|
['primary', primary],
|
|
61
361
|
...resolveFallbackGatewayBases(opts),
|
|
62
362
|
];
|
|
63
|
-
const
|
|
363
|
+
const maxBytes = resolveMaxResponseBytes(opts);
|
|
364
|
+
const attempts = [];
|
|
64
365
|
for (const cidPath of buildIpfsFetchCidPathCandidates(cid)) {
|
|
65
|
-
for (const [name, baseUrl] of gateways)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
366
|
+
for (const [name, baseUrl] of gateways)
|
|
367
|
+
attempts.push([name, baseUrl, cidPath]);
|
|
368
|
+
}
|
|
369
|
+
const errors = [];
|
|
370
|
+
// Parallel to `errors`: the display message is for the operator, the cause is what
|
|
371
|
+
// `classifyIpfsFetchFailure` reads to tell a size refusal and a transport failure apart from
|
|
372
|
+
// genuine absence (#3451).
|
|
373
|
+
const causes = [];
|
|
374
|
+
const deadline = Date.now() + IPFS_TOTAL_FETCH_TIMEOUT_MS;
|
|
375
|
+
for (let index = 0; index < attempts.length; index += 1) {
|
|
376
|
+
const [name, baseUrl, cidPath] = attempts[index];
|
|
377
|
+
const remainingMs = deadline - Date.now();
|
|
378
|
+
if (remainingMs <= 0) {
|
|
379
|
+
const message = `whole-operation timeout after ${IPFS_TOTAL_FETCH_TIMEOUT_MS}ms`;
|
|
380
|
+
errors.push(message);
|
|
381
|
+
causes.push(new Error(message));
|
|
382
|
+
break;
|
|
383
|
+
}
|
|
384
|
+
// Share what is left of the budget across the candidates still to try, so a
|
|
385
|
+
// run of slow early candidates cannot starve a later one that would have
|
|
386
|
+
// succeeded. Without this the whole-operation bound would silently narrow
|
|
387
|
+
// the candidate matrix instead of only bounding it.
|
|
388
|
+
const attemptMs = Math.min(IPFS_FETCH_TIMEOUT_MS, Math.ceil(remainingMs / (attempts.length - index)));
|
|
389
|
+
const controller = new AbortController();
|
|
390
|
+
const timer = setTimeout(() => controller.abort(), attemptMs);
|
|
391
|
+
// Resolved once here rather than inside the catch, so a candidate that
|
|
392
|
+
// cannot be parsed or that escapes the gateway prefix is reported as a
|
|
393
|
+
// candidate failure instead of escaping as a bare throw that discards the
|
|
394
|
+
// other candidates' errors.
|
|
395
|
+
const candidate = resolveGatewayCandidateUrl(baseUrl, cidPath);
|
|
396
|
+
if (!('url' in candidate)) {
|
|
397
|
+
const message = `${name}: ${candidate.reason}`;
|
|
398
|
+
errors.push(message);
|
|
399
|
+
causes.push(new Error(message));
|
|
400
|
+
clearTimeout(timer);
|
|
401
|
+
continue;
|
|
402
|
+
}
|
|
403
|
+
const target = candidate.url;
|
|
404
|
+
try {
|
|
405
|
+
return await read(target, controller.signal, maxBytes);
|
|
406
|
+
}
|
|
407
|
+
catch (error) {
|
|
408
|
+
errors.push(`${name}:${displayUrl(target)}: ` +
|
|
409
|
+
`${error instanceof Error ? error.message : String(error)}`);
|
|
410
|
+
causes.push(error);
|
|
411
|
+
}
|
|
412
|
+
finally {
|
|
413
|
+
clearTimeout(timer);
|
|
78
414
|
}
|
|
79
415
|
}
|
|
80
|
-
throw new
|
|
416
|
+
throw new IpfsFetchFailedError(`${failureLabel}: ${errors.join(' | ')}`, causes);
|
|
417
|
+
}
|
|
418
|
+
/** Read-only multi-codec, primary-plus-fallback IPFS JSON fetch. */
|
|
419
|
+
export async function fetchFromIpfs(gatewayUrl, cid, opts) {
|
|
420
|
+
return fetchCandidatesFromIpfs(gatewayUrl, cid, opts, fetchJson, 'IPFS JSON fetch failed after all candidates');
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Read-only multi-codec, primary-plus-fallback IPFS fetch returning the exact
|
|
424
|
+
* bytes stored at the CID — no JSON parse/re-encode roundtrip. Use this
|
|
425
|
+
* whenever the bytes will be hashed, or whenever the content is not JSON.
|
|
426
|
+
*/
|
|
427
|
+
export async function fetchBytesFromIpfs(gatewayUrl, cid, opts) {
|
|
428
|
+
return fetchCandidatesFromIpfs(gatewayUrl, cid, opts, fetchBytes, 'IPFS raw bytes fetch failed after all candidates');
|
|
81
429
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jinn-network/core",
|
|
3
|
-
"version": "0.1.2-canary.
|
|
3
|
+
"version": "0.1.2-canary.bebeb033",
|
|
4
4
|
"description": "Domain core for Jinn — evidence/contribution stores, scrub, trajectory parsing, and corpus reads. Independent of operator/src.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "yarn@4.13.0",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@huggingface/transformers": "^4.2.0",
|
|
46
|
-
"@jinn-network/plugin": "0.1.2-canary.
|
|
46
|
+
"@jinn-network/plugin": "0.1.2-canary.bebeb033",
|
|
47
47
|
"@lmoe/gliner-onnx": "0.1.0",
|
|
48
48
|
"@noble/hashes": "^2.2.0",
|
|
49
49
|
"@secretlint/core": "^13.0.2",
|