@fedify/vocab-runtime 2.4.0-pr.936.41 → 2.5.0-dev.2271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +4 -2
- package/dist/{tests/docloader-Ck8bcBir.mjs → contexts-BK1CqBR5.cjs} +164 -275
- package/dist/{tests/docloader-S_gQZ2tt.cjs → contexts-DPuJ4UYL.js} +160 -294
- package/dist/{docloader-C_dir7Xb.d.ts → docloader-CYwqh5Df.d.cts} +66 -2
- package/dist/{docloader-D2DTRiyA.d.cts → docloader-CYwqh5Df.d.ts} +66 -2
- package/dist/internal/jsonld-cache.cjs +70 -1
- package/dist/internal/jsonld-cache.d.cts +35 -2
- package/dist/internal/jsonld-cache.d.ts +35 -2
- package/dist/internal/jsonld-cache.js +67 -2
- package/dist/internal/portable-dereference.cjs +721 -0
- package/dist/internal/portable-dereference.d.cts +349 -0
- package/dist/internal/portable-dereference.d.ts +349 -0
- package/dist/internal/portable-dereference.js +702 -0
- package/dist/internal/signed-representation.cjs +378 -0
- package/dist/internal/signed-representation.d.cts +144 -0
- package/dist/internal/signed-representation.d.ts +144 -0
- package/dist/internal/signed-representation.js +371 -0
- package/dist/jsonld.cjs +2 -2
- package/dist/mod.cjs +711 -4527
- package/dist/mod.d.cts +400 -8
- package/dist/mod.d.ts +400 -9
- package/dist/mod.js +692 -4522
- package/dist/portable-DqtfLy_1.d.cts +160 -0
- package/dist/portable-DtWsu2yU.d.ts +160 -0
- package/dist/tests/body-CVR1bABr.mjs +130 -0
- package/dist/tests/body-T_yQWql3.cjs +153 -0
- package/dist/tests/body.test.cjs +89 -0
- package/dist/tests/body.test.d.cts +1 -0
- package/dist/tests/body.test.d.mts +1 -0
- package/dist/tests/body.test.mjs +90 -0
- package/dist/tests/contexts-CIKsin4e.mjs +4512 -0
- package/dist/tests/contexts-DizzBjz4.cjs +4523 -0
- package/dist/tests/decimal.test.cjs +8 -7
- package/dist/tests/decimal.test.mjs +8 -6
- package/dist/tests/digest-3FeH2Y-Q.cjs +176 -0
- package/dist/tests/digest-COC7xDiQ.mjs +141 -0
- package/dist/tests/digest.test.cjs +102 -0
- package/dist/tests/digest.test.d.cts +1 -0
- package/dist/tests/digest.test.d.mts +1 -0
- package/dist/tests/digest.test.mjs +103 -0
- package/dist/tests/docloader-C6yApEts.mjs +373 -0
- package/dist/tests/docloader-CJK2w3l-.cjs +396 -0
- package/dist/tests/docloader.test.cjs +692 -39
- package/dist/tests/docloader.test.mjs +686 -34
- package/dist/tests/internal/multicodec.test.cjs +2 -3
- package/dist/tests/internal/multicodec.test.mjs +2 -2
- package/dist/tests/internal/portable-dereference.test.cjs +172 -0
- package/dist/tests/internal/portable-dereference.test.d.cts +1 -0
- package/dist/tests/internal/portable-dereference.test.d.mts +1 -0
- package/dist/tests/internal/portable-dereference.test.mjs +173 -0
- package/dist/tests/jsonld-cache-BPQmOZWD.mjs +342 -0
- package/dist/tests/jsonld-cache-C07AyNOY.cjs +397 -0
- package/dist/tests/jsonld-cache.test.cjs +101 -298
- package/dist/tests/jsonld-cache.test.mjs +94 -289
- package/dist/tests/{key-_wXwomh_.cjs → key-C-AYkdJJ.cjs} +11 -4
- package/dist/tests/{key-CDGDH_vC.mjs → key-C2Db_TAJ.mjs} +11 -3
- package/dist/tests/key.test.cjs +6 -5
- package/dist/tests/key.test.mjs +6 -4
- package/dist/tests/langstr.test.cjs +4 -4
- package/dist/tests/langstr.test.mjs +2 -2
- package/dist/tests/link.test.cjs +2 -3
- package/dist/tests/link.test.mjs +2 -2
- package/dist/tests/multibase/multibase.test.cjs +8 -9
- package/dist/tests/multibase/multibase.test.mjs +6 -6
- package/dist/tests/{multibase-Bz_UUDtL.cjs → multibase-B5Mea7Ip.cjs} +19 -2
- package/dist/tests/{multibase-B4bvakyA.mjs → multibase-BPnF_L4e.mjs} +12 -1
- package/dist/tests/portable-dereference-BNXtgg5U.cjs +278 -0
- package/dist/tests/portable-dereference-DRE5bz-l.mjs +249 -0
- package/dist/tests/portable-media-C4LEXJk4.mjs +148 -0
- package/dist/tests/portable-media-D05JJIJg.cjs +153 -0
- package/dist/tests/portable-media.test.cjs +222 -0
- package/dist/tests/portable-media.test.d.cts +1 -0
- package/dist/tests/portable-media.test.d.mts +1 -0
- package/dist/tests/portable-media.test.mjs +223 -0
- package/dist/tests/portable-workers.test.cjs +36 -0
- package/dist/tests/portable-workers.test.d.cts +2 -0
- package/dist/tests/portable-workers.test.d.mts +2 -0
- package/dist/tests/portable-workers.test.mjs +35 -0
- package/dist/tests/{request-uk51rkhO.cjs → request-C-VZ9TLj.cjs} +10 -4
- package/dist/tests/{request-C8CaGwtt.mjs → request-Cf-A3DU-.mjs} +8 -2
- package/dist/tests/request.test.cjs +7 -4
- package/dist/tests/request.test.mjs +5 -2
- package/dist/tests/signed-representation.test.cjs +588 -0
- package/dist/tests/signed-representation.test.d.cts +1 -0
- package/dist/tests/signed-representation.test.d.mts +1 -0
- package/dist/tests/signed-representation.test.mjs +589 -0
- package/dist/tests/temporal.test.cjs +1 -2
- package/dist/tests/temporal.test.mjs +1 -1
- package/dist/tests/url-BNakuZ8k.cjs +998 -0
- package/dist/tests/url-DMxmp7ZG.mjs +859 -0
- package/dist/tests/url.test.cjs +489 -6
- package/dist/tests/url.test.mjs +489 -5
- package/dist/url-DrGTR8yv.cjs +993 -0
- package/dist/url-Dzyp-NsC.js +860 -0
- package/package.json +27 -4
- package/scripts/test-bun.mjs +17 -0
- package/src/body.test.ts +125 -0
- package/src/body.ts +152 -0
- package/src/contexts/cid-v1.json +114 -0
- package/src/contexts/fep-22cd.json +21 -0
- package/src/contexts/miscellany.json +17 -0
- package/src/contexts.ts +39 -1
- package/src/digest.test.ts +220 -0
- package/src/digest.ts +229 -0
- package/src/docloader.test.ts +918 -27
- package/src/docloader.ts +320 -105
- package/src/internal/jsonld-cache.ts +97 -1
- package/src/internal/portable-dereference.test.ts +254 -0
- package/src/internal/portable-dereference.ts +1000 -0
- package/src/internal/signed-representation.ts +565 -0
- package/src/jsonld-cache.test.ts +89 -0
- package/src/key.test.ts +9 -0
- package/src/key.ts +11 -1
- package/src/mod.ts +28 -0
- package/src/multibase/multibase.test.ts +5 -5
- package/src/portable-media.test.ts +293 -0
- package/src/portable-media.ts +264 -0
- package/src/portable-workers.test.ts +81 -0
- package/src/portable.ts +180 -0
- package/src/preprocessor.ts +7 -0
- package/src/request.test.ts +10 -1
- package/src/request.ts +9 -1
- package/src/signed-representation.test.ts +338 -0
- package/src/url.test.ts +823 -3
- package/src/url.ts +700 -23
- package/tsdown.config.ts +2 -0
- package/dist/tests/url-CsOV_B_P.cjs +0 -482
- package/dist/tests/url-Du7RQQgP.mjs +0 -392
- package/dist/url-DD4F0ULf.cjs +0 -483
- package/dist/url-DGVbSVVi.js +0 -393
- /package/dist/{chunk-M78iaK0I.cjs → rolldown-runtime-B7lfambq.cjs} +0 -0
- /package/dist/tests/{langstr-CbAxaeEZ.cjs → langstr-C4Fl80ae.cjs} +0 -0
- /package/dist/tests/{langstr-Di5AvKpB.mjs → langstr-CQ26J_L7.mjs} +0 -0
- /package/dist/tests/{link-NUUWCdnK.mjs → link-Cevmc87v.mjs} +0 -0
- /package/dist/tests/{link-FguCydMA.cjs → link-DlKm8bEr.cjs} +0 -0
- /package/dist/tests/{multicodec-CxGVGa91.cjs → multicodec-CLRPeW4N.cjs} +0 -0
- /package/dist/tests/{multicodec-CyFp54fI.mjs → multicodec-CRIj_05H.mjs} +0 -0
- /package/dist/tests/{chunk-C2EiDwsr.cjs → rolldown-runtime-emK7D4bc.cjs} +0 -0
package/src/docloader.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { getLogger } from "@logtape/logtape";
|
|
2
2
|
import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
3
3
|
import metadata from "../deno.json" with { type: "json" };
|
|
4
|
+
import {
|
|
5
|
+
BodyTooLargeError,
|
|
6
|
+
MAX_BODY_SIZE,
|
|
7
|
+
readBoundedBytes,
|
|
8
|
+
readBoundedText,
|
|
9
|
+
} from "./body.ts";
|
|
4
10
|
import preloadedContexts from "./contexts.ts";
|
|
5
11
|
import { HttpHeaderLink } from "./link.ts";
|
|
6
12
|
import {
|
|
@@ -14,6 +20,11 @@ import { UrlError, validatePublicUrl } from "./url.ts";
|
|
|
14
20
|
const logger = getLogger(["fedify", "runtime", "docloader"]);
|
|
15
21
|
const DEFAULT_MAX_REDIRECTION = 20;
|
|
16
22
|
const MAX_HTML_SIZE = 1024 * 1024; // 1MB
|
|
23
|
+
const MAX_ERROR_BODY_SIZE = 1024 * 1024; // 1MB
|
|
24
|
+
const DEFAULT_TIMEOUT = 10_000; // 10 seconds
|
|
25
|
+
// The maximum delay setTimeout() accepts; larger values fire immediately
|
|
26
|
+
// on some runtimes:
|
|
27
|
+
const MAX_TIMEOUT = 2_147_483_647;
|
|
17
28
|
|
|
18
29
|
/**
|
|
19
30
|
* A remote JSON-LD document and its context fetched by
|
|
@@ -46,6 +57,14 @@ export interface DocumentLoaderOptions {
|
|
|
46
57
|
* @since 1.8.0
|
|
47
58
|
*/
|
|
48
59
|
signal?: AbortSignal;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Whether to lower error-level logs for recoverable document loading
|
|
63
|
+
* failures to warning-level logs. The loader still throws the error.
|
|
64
|
+
* @default `false`
|
|
65
|
+
* @since 2.4.0
|
|
66
|
+
*/
|
|
67
|
+
suppressError?: boolean;
|
|
49
68
|
}
|
|
50
69
|
|
|
51
70
|
/**
|
|
@@ -96,6 +115,29 @@ export interface DocumentLoaderFactoryOptions {
|
|
|
96
115
|
* @since 2.2.0
|
|
97
116
|
*/
|
|
98
117
|
maxRedirection?: number;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The timeout in milliseconds for each call of the created document
|
|
121
|
+
* loader. The timeout is shared by all the steps of a call, including
|
|
122
|
+
* URL validation, every HTTP redirect and alternate document link it
|
|
123
|
+
* follows, retries, and reading the response body; it does not restart
|
|
124
|
+
* for each of them. It does not interrupt synchronous work such as
|
|
125
|
+
* parsing a document that has already been received.
|
|
126
|
+
*
|
|
127
|
+
* When a call times out, the loader throws a {@link FetchError} without
|
|
128
|
+
* a {@link FetchError.response}, whose `cause` is a `DOMException` named
|
|
129
|
+
* `"TimeoutError"`. An `AbortSignal` passed through
|
|
130
|
+
* {@link DocumentLoaderOptions.signal} still cancels a call; in that case
|
|
131
|
+
* the loader throws the signal's reason as before.
|
|
132
|
+
*
|
|
133
|
+
* Fractional values are rounded up. Set it to `null` to turn off the
|
|
134
|
+
* timeout.
|
|
135
|
+
* @default `10000` (10 seconds)
|
|
136
|
+
* @throws {RangeError} If the value is not a positive finite number or is
|
|
137
|
+
* greater than 2,147,483,647 (about 24.8 days).
|
|
138
|
+
* @since 2.4.0
|
|
139
|
+
*/
|
|
140
|
+
timeout?: number | null;
|
|
99
141
|
}
|
|
100
142
|
|
|
101
143
|
/**
|
|
@@ -121,49 +163,59 @@ function createResponseMetadata(response: Response): Response {
|
|
|
121
163
|
});
|
|
122
164
|
}
|
|
123
165
|
|
|
124
|
-
|
|
125
|
-
if (response.body != null) {
|
|
126
|
-
await response.body.cancel();
|
|
127
|
-
}
|
|
128
|
-
}
|
|
166
|
+
const NULL_BODY_STATUSES: ReadonlySet<number> = new Set([204, 205, 304]);
|
|
129
167
|
|
|
130
|
-
|
|
168
|
+
/**
|
|
169
|
+
* Reads the body of an error response while the document loader is still
|
|
170
|
+
* running, so that its timeout and `AbortSignal` also bound the read, and
|
|
171
|
+
* nothing reading {@link FetchError.response} later waits on the network.
|
|
172
|
+
* The body is kept byte for byte, unless it is too large or cannot be read;
|
|
173
|
+
* then only the status and headers are kept. A response whose status
|
|
174
|
+
* the `Response` constructor does not accept (e.g., 999) is kept as a clone
|
|
175
|
+
* whose body has been read in full; if its body is too large or cannot be
|
|
176
|
+
* read, no response is kept at all.
|
|
177
|
+
*/
|
|
178
|
+
async function bufferErrorResponse(
|
|
131
179
|
response: Response,
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
if (
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
180
|
+
url: string,
|
|
181
|
+
signal?: AbortSignal,
|
|
182
|
+
): Promise<Response | undefined> {
|
|
183
|
+
if (response.status < 200 || response.status > 599) {
|
|
184
|
+
// Such a response cannot be rebuilt, so keep a clone instead, and read
|
|
185
|
+
// the original to the end so that the clone's body is buffered too:
|
|
186
|
+
const clone = response.clone();
|
|
187
|
+
try {
|
|
188
|
+
await readBoundedBytes(response, MAX_ERROR_BODY_SIZE, url);
|
|
189
|
+
} catch (error) {
|
|
190
|
+
await clone.body?.cancel().catch(() => {});
|
|
191
|
+
if (signal?.aborted) throw error;
|
|
192
|
+
logger.debug(
|
|
193
|
+
"Failed to read the error response body from {url}: {error}",
|
|
194
|
+
{ url, error },
|
|
195
|
+
);
|
|
196
|
+
return undefined;
|
|
140
197
|
}
|
|
198
|
+
return clone;
|
|
141
199
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const decoder = new TextDecoder();
|
|
147
|
-
let text = "";
|
|
148
|
-
let size = 0;
|
|
200
|
+
if (response.body == null || NULL_BODY_STATUSES.has(response.status)) {
|
|
201
|
+
return createResponseMetadata(response);
|
|
202
|
+
}
|
|
203
|
+
let body: Uint8Array<ArrayBuffer>;
|
|
149
204
|
try {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
}
|
|
159
|
-
size += chunkSize;
|
|
160
|
-
text += decoder.decode(result.value, { stream: true });
|
|
161
|
-
}
|
|
162
|
-
text += decoder.decode();
|
|
163
|
-
return { text, size, tooLarge: false };
|
|
164
|
-
} finally {
|
|
165
|
-
reader.releaseLock();
|
|
205
|
+
body = await readBoundedBytes(response, MAX_ERROR_BODY_SIZE, url);
|
|
206
|
+
} catch (error) {
|
|
207
|
+
if (signal?.aborted) throw error;
|
|
208
|
+
logger.debug(
|
|
209
|
+
"Failed to read the error response body from {url}: {error}",
|
|
210
|
+
{ url, error },
|
|
211
|
+
);
|
|
212
|
+
return createResponseMetadata(response);
|
|
166
213
|
}
|
|
214
|
+
return new Response(body, {
|
|
215
|
+
headers: response.headers,
|
|
216
|
+
status: response.status,
|
|
217
|
+
statusText: response.statusText,
|
|
218
|
+
});
|
|
167
219
|
}
|
|
168
220
|
|
|
169
221
|
/**
|
|
@@ -171,6 +223,7 @@ async function readBoundedText(
|
|
|
171
223
|
* @param url The URL of the document to load.
|
|
172
224
|
* @param response The response to get the document from.
|
|
173
225
|
* @param fetch The function to fetch the document.
|
|
226
|
+
* @param options The options for loading the document.
|
|
174
227
|
* @returns The loaded remote document.
|
|
175
228
|
* @throws {FetchError} If the response is not OK.
|
|
176
229
|
* @internal
|
|
@@ -182,22 +235,35 @@ export async function getRemoteDocument(
|
|
|
182
235
|
url: string,
|
|
183
236
|
options?: DocumentLoaderOptions,
|
|
184
237
|
) => Promise<RemoteDocument>,
|
|
238
|
+
options?: DocumentLoaderOptions,
|
|
185
239
|
): Promise<RemoteDocument> {
|
|
186
240
|
const documentUrl = response.url === "" ? url : response.url;
|
|
187
241
|
const docUrl = new URL(documentUrl);
|
|
188
242
|
if (!response.ok) {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
243
|
+
if (options?.suppressError) {
|
|
244
|
+
logger.warn(
|
|
245
|
+
"Failed to fetch document: {status} {url} {headers}",
|
|
246
|
+
{
|
|
247
|
+
status: response.status,
|
|
248
|
+
url: documentUrl,
|
|
249
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
250
|
+
},
|
|
251
|
+
);
|
|
252
|
+
} else {
|
|
253
|
+
logger.error(
|
|
254
|
+
"Failed to fetch document: {status} {url} {headers}",
|
|
255
|
+
{
|
|
256
|
+
status: response.status,
|
|
257
|
+
url: documentUrl,
|
|
258
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
259
|
+
},
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
|
|
197
263
|
throw new FetchError(
|
|
198
264
|
documentUrl,
|
|
199
265
|
`HTTP ${response.status}: ${documentUrl}`,
|
|
200
|
-
response
|
|
266
|
+
await bufferErrorResponse(response, documentUrl, options?.signal),
|
|
201
267
|
);
|
|
202
268
|
}
|
|
203
269
|
const contentType = response.headers.get("Content-Type");
|
|
@@ -242,7 +308,7 @@ export async function getRemoteDocument(
|
|
|
242
308
|
"Found alternate document: {alternateUrl} from {url}",
|
|
243
309
|
{ alternateUrl: altUri.href, url: documentUrl },
|
|
244
310
|
);
|
|
245
|
-
return await fetch(altUri.href);
|
|
311
|
+
return await fetch(altUri.href, options);
|
|
246
312
|
}
|
|
247
313
|
}
|
|
248
314
|
}
|
|
@@ -255,11 +321,14 @@ export async function getRemoteDocument(
|
|
|
255
321
|
contentType?.startsWith("application/xhtml+xml;"))
|
|
256
322
|
) {
|
|
257
323
|
const errorResponse = createResponseMetadata(response);
|
|
258
|
-
|
|
259
|
-
|
|
324
|
+
let html: string;
|
|
325
|
+
try {
|
|
326
|
+
html = await readBoundedText(response, MAX_HTML_SIZE, documentUrl);
|
|
327
|
+
} catch (error) {
|
|
328
|
+
if (!(error instanceof BodyTooLargeError)) throw error;
|
|
260
329
|
logger.warn(
|
|
261
330
|
"HTML response too large, skipping alternate link discovery: {url}",
|
|
262
|
-
{ url: documentUrl,
|
|
331
|
+
{ url: documentUrl, maxBytes: MAX_HTML_SIZE },
|
|
263
332
|
);
|
|
264
333
|
throw new FetchError(
|
|
265
334
|
documentUrl,
|
|
@@ -267,58 +336,58 @@ export async function getRemoteDocument(
|
|
|
267
336
|
`(Content-Type: ${contentType})`,
|
|
268
337
|
errorResponse,
|
|
269
338
|
);
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
/([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
278
|
-
|
|
279
|
-
let tagMatch: RegExpExecArray | null;
|
|
280
|
-
while ((tagMatch = tagPattern.exec(html.text)) !== null) {
|
|
281
|
-
const tagContent = tagMatch[2];
|
|
282
|
-
let attrMatch: RegExpExecArray | null;
|
|
283
|
-
const attribs: Record<string, string> = {};
|
|
339
|
+
}
|
|
340
|
+
// Safe regex patterns without nested quantifiers to prevent ReDoS
|
|
341
|
+
// (CVE-2025-68475)
|
|
342
|
+
// Step 1: Extract <a ...> or <link ...> tags
|
|
343
|
+
const tagPattern = /<(a|link)\s+([^>]*?)\s*\/?>/gi;
|
|
344
|
+
// Step 2: Parse attributes
|
|
345
|
+
const attrPattern = /([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
284
346
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
attribs[key] = value;
|
|
291
|
-
}
|
|
347
|
+
let tagMatch: RegExpExecArray | null;
|
|
348
|
+
while ((tagMatch = tagPattern.exec(html)) !== null) {
|
|
349
|
+
const tagContent = tagMatch[2];
|
|
350
|
+
let attrMatch: RegExpExecArray | null;
|
|
351
|
+
const attribs: Record<string, string> = {};
|
|
292
352
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
new URL(attribs.href, docUrl).href !== docUrl.href
|
|
300
|
-
) {
|
|
301
|
-
logger.debug(
|
|
302
|
-
"Found alternate document: {alternateUrl} from {url}",
|
|
303
|
-
{ alternateUrl: attribs.href, url: documentUrl },
|
|
304
|
-
);
|
|
305
|
-
return await fetch(new URL(attribs.href, docUrl).href);
|
|
306
|
-
}
|
|
353
|
+
// Reset regex state for attribute parsing
|
|
354
|
+
attrPattern.lastIndex = 0;
|
|
355
|
+
while ((attrMatch = attrPattern.exec(tagContent)) !== null) {
|
|
356
|
+
const key = attrMatch[1].toLowerCase();
|
|
357
|
+
const value = attrMatch[2] ?? attrMatch[3] ?? attrMatch[4] ?? "";
|
|
358
|
+
attribs[key] = value;
|
|
307
359
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
360
|
+
|
|
361
|
+
if (
|
|
362
|
+
attribs.rel === "alternate" && "type" in attribs && (
|
|
363
|
+
attribs.type === "application/activity+json" ||
|
|
364
|
+
attribs.type === "application/ld+json" ||
|
|
365
|
+
attribs.type.startsWith("application/ld+json;")
|
|
366
|
+
) && "href" in attribs &&
|
|
367
|
+
new URL(attribs.href, docUrl).href !== docUrl.href
|
|
368
|
+
) {
|
|
369
|
+
logger.debug(
|
|
370
|
+
"Found alternate document: {alternateUrl} from {url}",
|
|
371
|
+
{ alternateUrl: attribs.href, url: documentUrl },
|
|
317
372
|
);
|
|
373
|
+
return await fetch(new URL(attribs.href, docUrl).href, options);
|
|
318
374
|
}
|
|
319
375
|
}
|
|
376
|
+
try {
|
|
377
|
+
document = JSON.parse(html);
|
|
378
|
+
} catch (error) {
|
|
379
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
380
|
+
throw new FetchError(
|
|
381
|
+
documentUrl,
|
|
382
|
+
`HTML document has no ActivityPub alternate link ` +
|
|
383
|
+
`(Content-Type: ${contentType})`,
|
|
384
|
+
errorResponse,
|
|
385
|
+
);
|
|
386
|
+
}
|
|
320
387
|
} else {
|
|
321
|
-
document =
|
|
388
|
+
document = JSON.parse(
|
|
389
|
+
await readBoundedText(response, MAX_BODY_SIZE, documentUrl),
|
|
390
|
+
);
|
|
322
391
|
}
|
|
323
392
|
logger.debug(
|
|
324
393
|
"Fetched document: {status} {url} {headers}",
|
|
@@ -331,6 +400,104 @@ export async function getRemoteDocument(
|
|
|
331
400
|
return { contextUrl, document, documentUrl };
|
|
332
401
|
}
|
|
333
402
|
|
|
403
|
+
/**
|
|
404
|
+
* Resolves {@link DocumentLoaderFactoryOptions.timeout} into milliseconds.
|
|
405
|
+
* @param timeout The timeout option. `undefined` means the default timeout,
|
|
406
|
+
* and `null` means no timeout.
|
|
407
|
+
* @returns The timeout in milliseconds, or `null` if it is turned off.
|
|
408
|
+
* @throws {RangeError} If the timeout is invalid.
|
|
409
|
+
* @internal
|
|
410
|
+
*/
|
|
411
|
+
export function resolveDocumentLoaderTimeout(
|
|
412
|
+
timeout: number | null | undefined,
|
|
413
|
+
): number | null {
|
|
414
|
+
if (timeout === undefined) return DEFAULT_TIMEOUT;
|
|
415
|
+
if (timeout === null) return null;
|
|
416
|
+
if (
|
|
417
|
+
typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0
|
|
418
|
+
) {
|
|
419
|
+
throw new RangeError(
|
|
420
|
+
`The document loader timeout must be a positive finite number of ` +
|
|
421
|
+
`milliseconds, but got ${String(timeout)}.`,
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
const ms = Math.ceil(timeout);
|
|
425
|
+
if (ms > MAX_TIMEOUT) {
|
|
426
|
+
throw new RangeError(
|
|
427
|
+
`The document loader timeout must not be greater than ${MAX_TIMEOUT} ` +
|
|
428
|
+
`milliseconds, but got ${timeout}.`,
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
return ms;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* Bounds each call of the given document loader by the given timeout.
|
|
436
|
+
* The timeout is combined with the caller's `signal`, and the combined
|
|
437
|
+
* signal is passed to the loader. The call settles no later than the
|
|
438
|
+
* timeout even if the loader is stuck in a step that cannot be aborted,
|
|
439
|
+
* e.g., a DNS lookup.
|
|
440
|
+
*
|
|
441
|
+
* A timed-out call throws a {@link FetchError} without a response, whose
|
|
442
|
+
* `cause` is a `DOMException` named `"TimeoutError"`. If the caller's
|
|
443
|
+
* signal is aborted, its reason is thrown instead.
|
|
444
|
+
* @param loader The document loader to bound.
|
|
445
|
+
* @param timeout The timeout in milliseconds, or `null` for no timeout.
|
|
446
|
+
* It is assumed to have been resolved by
|
|
447
|
+
* {@link resolveDocumentLoaderTimeout}.
|
|
448
|
+
* @returns The bounded document loader.
|
|
449
|
+
* @internal
|
|
450
|
+
*/
|
|
451
|
+
export function withDocumentLoaderTimeout(
|
|
452
|
+
loader: DocumentLoader,
|
|
453
|
+
timeout: number | null,
|
|
454
|
+
): DocumentLoader {
|
|
455
|
+
if (timeout == null) return loader;
|
|
456
|
+
return async (url, options) => {
|
|
457
|
+
const callerSignal = options?.signal;
|
|
458
|
+
callerSignal?.throwIfAborted();
|
|
459
|
+
const controller = new AbortController();
|
|
460
|
+
const timeoutReason = new DOMException(
|
|
461
|
+
`The document loader timed out after ${timeout} ms.`,
|
|
462
|
+
"TimeoutError",
|
|
463
|
+
);
|
|
464
|
+
let timedOut = false;
|
|
465
|
+
const timer = setTimeout(() => {
|
|
466
|
+
timedOut = true;
|
|
467
|
+
controller.abort(timeoutReason);
|
|
468
|
+
}, timeout);
|
|
469
|
+
const onCallerAbort = () => controller.abort(callerSignal?.reason);
|
|
470
|
+
callerSignal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
471
|
+
let onAbort: (() => void) | undefined;
|
|
472
|
+
const aborted = new Promise<never>((_, reject) => {
|
|
473
|
+
onAbort = () => reject(controller.signal.reason);
|
|
474
|
+
controller.signal.addEventListener("abort", onAbort, { once: true });
|
|
475
|
+
});
|
|
476
|
+
const loading = loader(url, { ...options, signal: controller.signal });
|
|
477
|
+
// If the abort wins the race, the loader's late rejection is ignored:
|
|
478
|
+
loading.catch(() => {});
|
|
479
|
+
try {
|
|
480
|
+
return await Promise.race([loading, aborted]);
|
|
481
|
+
} catch (error) {
|
|
482
|
+
if (callerSignal?.aborted) throw callerSignal.reason;
|
|
483
|
+
if (!timedOut) throw error;
|
|
484
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
485
|
+
"Timed out after {timeout} ms while fetching document: {url}",
|
|
486
|
+
{ timeout, url },
|
|
487
|
+
);
|
|
488
|
+
const fetchError = new FetchError(url, `Timed out after ${timeout} ms`);
|
|
489
|
+
fetchError.cause = timeoutReason;
|
|
490
|
+
throw fetchError;
|
|
491
|
+
} finally {
|
|
492
|
+
clearTimeout(timer);
|
|
493
|
+
callerSignal?.removeEventListener("abort", onCallerAbort);
|
|
494
|
+
if (onAbort != null) {
|
|
495
|
+
controller.signal.removeEventListener("abort", onAbort);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
|
|
334
501
|
/**
|
|
335
502
|
* Options for {@link getDocumentLoader}.
|
|
336
503
|
* @since 1.3.0
|
|
@@ -344,6 +511,10 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
|
|
|
344
511
|
|
|
345
512
|
/**
|
|
346
513
|
* Creates a JSON-LD document loader that utilizes the browser's `fetch` API.
|
|
514
|
+
* At most 20 HTTP redirects and alternate document links are followed in total
|
|
515
|
+
* per call. Revisiting a URL within that chain throws a {@link FetchError}.
|
|
516
|
+
* Each call times out after 10 seconds by default; see
|
|
517
|
+
* {@link DocumentLoaderFactoryOptions.timeout}.
|
|
347
518
|
*
|
|
348
519
|
* The created loader preloads the below frequently used contexts by default
|
|
349
520
|
* (unless `options.skipPreloadedContexts` is set to `true`):
|
|
@@ -353,8 +524,10 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
|
|
|
353
524
|
* - <https://w3id.org/security/data-integrity/v1>
|
|
354
525
|
* - <https://w3id.org/security/data-integrity/v2>
|
|
355
526
|
* - <https://www.w3.org/ns/did/v1>
|
|
527
|
+
* - <https://www.w3.org/ns/cid/v1>
|
|
356
528
|
* - <https://w3id.org/security/multikey/v1>
|
|
357
529
|
* - <https://w3id.org/fep/ef61>
|
|
530
|
+
* - <https://w3id.org/fep/7aa9>
|
|
358
531
|
* - <https://purl.archive.org/socialweb/webfinger>
|
|
359
532
|
* - <http://schema.org/>
|
|
360
533
|
* @param options Options for the document loader.
|
|
@@ -362,9 +535,15 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
|
|
|
362
535
|
* @since 1.3.0
|
|
363
536
|
*/
|
|
364
537
|
export function getDocumentLoader(
|
|
365
|
-
{
|
|
366
|
-
|
|
538
|
+
{
|
|
539
|
+
allowPrivateAddress,
|
|
540
|
+
maxRedirection,
|
|
541
|
+
skipPreloadedContexts,
|
|
542
|
+
timeout,
|
|
543
|
+
userAgent,
|
|
544
|
+
}: GetDocumentLoaderOptions = {},
|
|
367
545
|
): DocumentLoader {
|
|
546
|
+
const resolvedTimeout = resolveDocumentLoaderTimeout(timeout);
|
|
368
547
|
const tracerProvider = trace.getTracerProvider();
|
|
369
548
|
const tracer = tracerProvider.getTracer(metadata.name, metadata.version);
|
|
370
549
|
const maximumRedirection = maxRedirection ?? DEFAULT_MAX_REDIRECTION;
|
|
@@ -390,13 +569,26 @@ export function getDocumentLoader(
|
|
|
390
569
|
await validatePublicUrl(currentUrl);
|
|
391
570
|
} catch (error) {
|
|
392
571
|
if (error instanceof UrlError) {
|
|
393
|
-
|
|
394
|
-
url
|
|
395
|
-
|
|
396
|
-
|
|
572
|
+
if (error.reason === "dns") {
|
|
573
|
+
logger.debug("DNS lookup failed for {url}", {
|
|
574
|
+
url: currentUrl,
|
|
575
|
+
error,
|
|
576
|
+
});
|
|
577
|
+
} else {
|
|
578
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
579
|
+
"Disallowed private URL: {url}",
|
|
580
|
+
{
|
|
581
|
+
url: currentUrl,
|
|
582
|
+
error,
|
|
583
|
+
},
|
|
584
|
+
);
|
|
585
|
+
}
|
|
397
586
|
}
|
|
398
587
|
throw error;
|
|
399
588
|
}
|
|
589
|
+
// The DNS lookup cannot be aborted, so do not go on if the call was
|
|
590
|
+
// aborted or timed out in the meantime:
|
|
591
|
+
options?.signal?.throwIfAborted();
|
|
400
592
|
}
|
|
401
593
|
visited.add(currentUrl);
|
|
402
594
|
|
|
@@ -427,7 +619,7 @@ export function getDocumentLoader(
|
|
|
427
619
|
response.headers.has("Location")
|
|
428
620
|
) {
|
|
429
621
|
if (redirected >= maximumRedirection) {
|
|
430
|
-
logger
|
|
622
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
431
623
|
"Too many redirections ({redirections}) while fetching document.",
|
|
432
624
|
{ redirections: redirected + 1, url: currentUrl },
|
|
433
625
|
);
|
|
@@ -442,7 +634,7 @@ export function getDocumentLoader(
|
|
|
442
634
|
).href;
|
|
443
635
|
span.setAttribute("http.redirect.url", redirectUrl);
|
|
444
636
|
if (visited.has(redirectUrl)) {
|
|
445
|
-
logger
|
|
637
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
446
638
|
"Detected a redirect loop while fetching document: {url} -> " +
|
|
447
639
|
"{redirectUrl}",
|
|
448
640
|
{ url: currentUrl, redirectUrl },
|
|
@@ -455,7 +647,27 @@ export function getDocumentLoader(
|
|
|
455
647
|
return await load(redirectUrl, options, redirected + 1, visited);
|
|
456
648
|
}
|
|
457
649
|
|
|
458
|
-
const result = await getRemoteDocument(
|
|
650
|
+
const result = await getRemoteDocument(
|
|
651
|
+
currentUrl,
|
|
652
|
+
response,
|
|
653
|
+
async (alternateUrl) => {
|
|
654
|
+
options?.signal?.throwIfAborted();
|
|
655
|
+
if (redirected >= DEFAULT_MAX_REDIRECTION) {
|
|
656
|
+
throw new FetchError(
|
|
657
|
+
currentUrl,
|
|
658
|
+
`Too many redirections (${redirected + 1})`,
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
if (visited.has(alternateUrl)) {
|
|
662
|
+
throw new FetchError(
|
|
663
|
+
currentUrl,
|
|
664
|
+
`Redirect loop detected: ${alternateUrl}`,
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
return await load(alternateUrl, options, redirected + 1, visited);
|
|
668
|
+
},
|
|
669
|
+
options,
|
|
670
|
+
);
|
|
459
671
|
span.setAttribute("docloader.document_url", result.documentUrl);
|
|
460
672
|
if (result.contextUrl != null) {
|
|
461
673
|
span.setAttribute("docloader.context_url", result.contextUrl);
|
|
@@ -474,5 +686,8 @@ export function getDocumentLoader(
|
|
|
474
686
|
},
|
|
475
687
|
);
|
|
476
688
|
}
|
|
477
|
-
return
|
|
689
|
+
return withDocumentLoaderTimeout(
|
|
690
|
+
(url, options) => load(url, options),
|
|
691
|
+
resolvedTimeout,
|
|
692
|
+
);
|
|
478
693
|
}
|
|
@@ -1,9 +1,105 @@
|
|
|
1
1
|
import type { DocumentLoader } from "../docloader.ts";
|
|
2
2
|
import jsonld from "../jsonld.ts";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
formatIri,
|
|
5
|
+
fromCompatibleEf61Id,
|
|
6
|
+
getFe34Origin,
|
|
7
|
+
haveSameFe34Origin,
|
|
8
|
+
haveSameIriOrigin,
|
|
9
|
+
parseIri,
|
|
10
|
+
} from "../url.ts";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Reads expanded gateways, including an unmapped term on portable actors.
|
|
14
|
+
* The original values and their order are preserved for URL validation by
|
|
15
|
+
* the caller. An explicitly mapped FEP-ef61 property always takes precedence.
|
|
16
|
+
* @internal Used by generated vocabulary classes and portable proof checks.
|
|
17
|
+
*/
|
|
18
|
+
export function getPortableActorGateways(
|
|
19
|
+
node: Record<string, unknown>,
|
|
20
|
+
allowUnmapped = true,
|
|
21
|
+
): unknown[] | undefined {
|
|
22
|
+
const property = "https://w3id.org/fep/ef61/gateways";
|
|
23
|
+
if (globalThis.Object.hasOwn(node, property)) {
|
|
24
|
+
return Array.isArray(node[property]) ? node[property] : undefined;
|
|
25
|
+
}
|
|
26
|
+
if (
|
|
27
|
+
!allowUnmapped || typeof node["@id"] !== "string" ||
|
|
28
|
+
!("http://www.w3.org/ns/ldp#inbox" in node) ||
|
|
29
|
+
!("https://www.w3.org/ns/activitystreams#outbox" in node)
|
|
30
|
+
) return undefined;
|
|
31
|
+
try {
|
|
32
|
+
const id = fromCompatibleEf61Id(node["@id"]) ?? node["@id"];
|
|
33
|
+
if (!getFe34Origin(parseIri(id)).startsWith("did:")) return undefined;
|
|
34
|
+
} catch (error) {
|
|
35
|
+
if (error instanceof TypeError) return undefined;
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
return Array.isArray(node["_:gateways"]) ? node["_:gateways"] : undefined;
|
|
39
|
+
}
|
|
4
40
|
|
|
5
41
|
const noJsonLdContext = Symbol("noJsonLdContext");
|
|
6
42
|
|
|
43
|
+
const documentLoaderWrappers = new WeakMap<
|
|
44
|
+
DocumentLoader,
|
|
45
|
+
{ readonly base: DocumentLoader; released: boolean }
|
|
46
|
+
>();
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Registers a loader wrapper with the shared released-wrapper registry.
|
|
50
|
+
* Both suppression and snapshot wrappers must use this registry so mixed
|
|
51
|
+
* chains can be unwrapped. The caller updates the state on release.
|
|
52
|
+
* @internal Not a public API.
|
|
53
|
+
*/
|
|
54
|
+
export function registerDocumentLoaderWrapper(
|
|
55
|
+
loader: DocumentLoader,
|
|
56
|
+
state: { readonly base: DocumentLoader; released: boolean },
|
|
57
|
+
): void {
|
|
58
|
+
documentLoaderWrappers.set(loader, state);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Removes released loader wrappers, stopping at the first active wrapper.
|
|
63
|
+
* Active wrappers retain their operation's suppression or snapshot cache.
|
|
64
|
+
* @internal Not a public API.
|
|
65
|
+
*/
|
|
66
|
+
export function unwrapReleasedDocumentLoader(
|
|
67
|
+
base: DocumentLoader,
|
|
68
|
+
): DocumentLoader {
|
|
69
|
+
for (
|
|
70
|
+
let state = documentLoaderWrappers.get(base);
|
|
71
|
+
state?.released;
|
|
72
|
+
state = documentLoaderWrappers.get(base)
|
|
73
|
+
) {
|
|
74
|
+
base = state.base;
|
|
75
|
+
}
|
|
76
|
+
return base;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Lowers context loading failure logs for one accessor operation. Parsed
|
|
81
|
+
* objects retain the loader, so release it before returning to the caller.
|
|
82
|
+
* Released wrappers are unwrapped to avoid accumulating loader chains.
|
|
83
|
+
* @internal Used by generated vocabulary accessors; not a public API.
|
|
84
|
+
*/
|
|
85
|
+
export function createScopedContextLoader(
|
|
86
|
+
base: DocumentLoader,
|
|
87
|
+
suppressError?: boolean,
|
|
88
|
+
): { loader: DocumentLoader; release: () => void } {
|
|
89
|
+
base = unwrapReleasedDocumentLoader(base);
|
|
90
|
+
if (!suppressError) return { loader: base, release: () => {} };
|
|
91
|
+
const state = { base, released: false };
|
|
92
|
+
const loader: DocumentLoader = (url, options) =>
|
|
93
|
+
base(url, state.released ? options : { ...options, suppressError: true });
|
|
94
|
+
registerDocumentLoaderWrapper(loader, state);
|
|
95
|
+
return {
|
|
96
|
+
loader,
|
|
97
|
+
release: () => {
|
|
98
|
+
state.released = true;
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
7
103
|
/**
|
|
8
104
|
* Options for deciding whether two IRIs should be treated as same-origin.
|
|
9
105
|
*
|