@fedify/vocab-runtime 2.4.0-pr.934.40 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +4 -2
- package/dist/{tests/docloader-Xi61Le8w.mjs → contexts-BK1CqBR5.cjs} +164 -269
- package/dist/{tests/docloader-BFedvemb.cjs → contexts-DPuJ4UYL.js} +160 -288
- package/dist/{docloader-DnUMWHaJ.d.cts → docloader-CYwqh5Df.d.cts} +68 -2
- package/dist/{docloader-xRGn1azD.d.ts → docloader-CYwqh5Df.d.ts} +68 -2
- package/dist/internal/jsonld-cache.cjs +70 -1
- package/dist/internal/jsonld-cache.d.cts +35 -2
- package/dist/internal/jsonld-cache.d.ts +35 -2
- package/dist/internal/jsonld-cache.js +67 -2
- package/dist/internal/portable-dereference.cjs +721 -0
- package/dist/internal/portable-dereference.d.cts +349 -0
- package/dist/internal/portable-dereference.d.ts +349 -0
- package/dist/internal/portable-dereference.js +702 -0
- package/dist/internal/signed-representation.cjs +378 -0
- package/dist/internal/signed-representation.d.cts +144 -0
- package/dist/internal/signed-representation.d.ts +144 -0
- package/dist/internal/signed-representation.js +371 -0
- package/dist/jsonld.cjs +2 -2
- package/dist/mod.cjs +715 -4523
- package/dist/mod.d.cts +406 -6
- package/dist/mod.d.ts +406 -7
- package/dist/mod.js +694 -4518
- package/dist/portable-DqtfLy_1.d.cts +160 -0
- package/dist/portable-DtWsu2yU.d.ts +160 -0
- package/dist/tests/body-CYiu9CO-.mjs +130 -0
- package/dist/tests/body-CkEwROsn.cjs +153 -0
- package/dist/tests/body.test.cjs +89 -0
- package/dist/tests/body.test.d.cts +1 -0
- package/dist/tests/body.test.d.mts +1 -0
- package/dist/tests/body.test.mjs +90 -0
- package/dist/tests/contexts-CIKsin4e.mjs +4512 -0
- package/dist/tests/contexts-DizzBjz4.cjs +4523 -0
- package/dist/tests/decimal.test.cjs +8 -7
- package/dist/tests/decimal.test.mjs +8 -6
- package/dist/tests/digest-3FeH2Y-Q.cjs +176 -0
- package/dist/tests/digest-COC7xDiQ.mjs +141 -0
- package/dist/tests/digest.test.cjs +102 -0
- package/dist/tests/digest.test.d.cts +1 -0
- package/dist/tests/digest.test.d.mts +1 -0
- package/dist/tests/digest.test.mjs +103 -0
- package/dist/tests/docloader-C3YFl6y4.cjs +396 -0
- package/dist/tests/docloader-CvmS4sVq.mjs +373 -0
- package/dist/tests/docloader.test.cjs +692 -39
- package/dist/tests/docloader.test.mjs +686 -34
- package/dist/tests/internal/multicodec.test.cjs +2 -3
- package/dist/tests/internal/multicodec.test.mjs +2 -2
- package/dist/tests/internal/portable-dereference.test.cjs +172 -0
- package/dist/tests/internal/portable-dereference.test.d.cts +1 -0
- package/dist/tests/internal/portable-dereference.test.d.mts +1 -0
- package/dist/tests/internal/portable-dereference.test.mjs +173 -0
- package/dist/tests/jsonld-cache-BPQmOZWD.mjs +342 -0
- package/dist/tests/jsonld-cache-C07AyNOY.cjs +397 -0
- package/dist/tests/jsonld-cache.test.cjs +101 -298
- package/dist/tests/jsonld-cache.test.mjs +94 -289
- package/dist/tests/{key-_wXwomh_.cjs → key-C-AYkdJJ.cjs} +11 -4
- package/dist/tests/{key-CDGDH_vC.mjs → key-C2Db_TAJ.mjs} +11 -3
- package/dist/tests/key.test.cjs +6 -5
- package/dist/tests/key.test.mjs +6 -4
- package/dist/tests/langstr.test.cjs +4 -4
- package/dist/tests/langstr.test.mjs +2 -2
- package/dist/tests/link.test.cjs +2 -3
- package/dist/tests/link.test.mjs +2 -2
- package/dist/tests/multibase/multibase.test.cjs +8 -9
- package/dist/tests/multibase/multibase.test.mjs +6 -6
- package/dist/tests/{multibase-Bz_UUDtL.cjs → multibase-B5Mea7Ip.cjs} +19 -2
- package/dist/tests/{multibase-B4bvakyA.mjs → multibase-BPnF_L4e.mjs} +12 -1
- package/dist/tests/portable-dereference-BNXtgg5U.cjs +278 -0
- package/dist/tests/portable-dereference-DRE5bz-l.mjs +249 -0
- package/dist/tests/portable-media-BLKllU56.cjs +153 -0
- package/dist/tests/portable-media-DRSGBORB.mjs +148 -0
- package/dist/tests/portable-media.test.cjs +222 -0
- package/dist/tests/portable-media.test.d.cts +1 -0
- package/dist/tests/portable-media.test.d.mts +1 -0
- package/dist/tests/portable-media.test.mjs +223 -0
- package/dist/tests/portable-workers.test.cjs +36 -0
- package/dist/tests/portable-workers.test.d.cts +2 -0
- package/dist/tests/portable-workers.test.d.mts +2 -0
- package/dist/tests/portable-workers.test.mjs +35 -0
- package/dist/tests/{request-cCPgOxYG.cjs → request-8vPWtV1-.cjs} +10 -4
- package/dist/tests/{request-BOS-hNaf.mjs → request-DRaOaTqD.mjs} +8 -2
- package/dist/tests/request.test.cjs +7 -4
- package/dist/tests/request.test.mjs +5 -2
- package/dist/tests/signed-representation.test.cjs +588 -0
- package/dist/tests/signed-representation.test.d.cts +1 -0
- package/dist/tests/signed-representation.test.d.mts +1 -0
- package/dist/tests/signed-representation.test.mjs +589 -0
- package/dist/tests/temporal.test.cjs +1 -2
- package/dist/tests/temporal.test.mjs +1 -1
- package/dist/tests/url-BNakuZ8k.cjs +998 -0
- package/dist/tests/url-DMxmp7ZG.mjs +859 -0
- package/dist/tests/url.test.cjs +503 -3
- package/dist/tests/url.test.mjs +503 -2
- package/dist/url-DrGTR8yv.cjs +993 -0
- package/dist/url-Dzyp-NsC.js +860 -0
- package/package.json +27 -4
- package/scripts/test-bun.mjs +17 -0
- package/src/body.test.ts +125 -0
- package/src/body.ts +152 -0
- package/src/contexts/cid-v1.json +114 -0
- package/src/contexts/fep-22cd.json +21 -0
- package/src/contexts/fep-ef61.json +10 -0
- package/src/contexts/miscellany.json +17 -0
- package/src/contexts/security-data-integrity-v1.json +0 -4
- package/src/contexts.ts +40 -0
- package/src/digest.test.ts +220 -0
- package/src/digest.ts +229 -0
- package/src/docloader.test.ts +918 -27
- package/src/docloader.ts +322 -105
- package/src/internal/jsonld-cache.ts +97 -1
- package/src/internal/portable-dereference.test.ts +254 -0
- package/src/internal/portable-dereference.ts +1000 -0
- package/src/internal/signed-representation.ts +565 -0
- package/src/jsonld-cache.test.ts +89 -0
- package/src/key.test.ts +9 -0
- package/src/key.ts +11 -1
- package/src/mod.ts +30 -0
- package/src/multibase/multibase.test.ts +5 -5
- package/src/portable-media.test.ts +293 -0
- package/src/portable-media.ts +264 -0
- package/src/portable-workers.test.ts +81 -0
- package/src/portable.ts +180 -0
- package/src/preprocessor.ts +7 -0
- package/src/request.test.ts +10 -1
- package/src/request.ts +9 -1
- package/src/signed-representation.test.ts +338 -0
- package/src/url.test.ts +844 -1
- package/src/url.ts +717 -17
- package/tsdown.config.ts +2 -0
- package/dist/tests/url-BvjYQdxL.cjs +0 -456
- package/dist/tests/url-a2D8NAgh.mjs +0 -378
- package/dist/url-Ck3dGEwH.cjs +0 -457
- package/dist/url-m1YxGNZ0.js +0 -379
- /package/dist/{chunk-M78iaK0I.cjs → rolldown-runtime-B7lfambq.cjs} +0 -0
- /package/dist/tests/{langstr-CbAxaeEZ.cjs → langstr-C4Fl80ae.cjs} +0 -0
- /package/dist/tests/{langstr-Di5AvKpB.mjs → langstr-CQ26J_L7.mjs} +0 -0
- /package/dist/tests/{link-NUUWCdnK.mjs → link-Cevmc87v.mjs} +0 -0
- /package/dist/tests/{link-FguCydMA.cjs → link-DlKm8bEr.cjs} +0 -0
- /package/dist/tests/{multicodec-CxGVGa91.cjs → multicodec-CLRPeW4N.cjs} +0 -0
- /package/dist/tests/{multicodec-CyFp54fI.mjs → multicodec-CRIj_05H.mjs} +0 -0
- /package/dist/tests/{chunk-C2EiDwsr.cjs → rolldown-runtime-emK7D4bc.cjs} +0 -0
package/src/docloader.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { getLogger } from "@logtape/logtape";
|
|
2
2
|
import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
3
3
|
import metadata from "../deno.json" with { type: "json" };
|
|
4
|
+
import {
|
|
5
|
+
BodyTooLargeError,
|
|
6
|
+
MAX_BODY_SIZE,
|
|
7
|
+
readBoundedBytes,
|
|
8
|
+
readBoundedText,
|
|
9
|
+
} from "./body.ts";
|
|
4
10
|
import preloadedContexts from "./contexts.ts";
|
|
5
11
|
import { HttpHeaderLink } from "./link.ts";
|
|
6
12
|
import {
|
|
@@ -14,6 +20,11 @@ import { UrlError, validatePublicUrl } from "./url.ts";
|
|
|
14
20
|
const logger = getLogger(["fedify", "runtime", "docloader"]);
|
|
15
21
|
const DEFAULT_MAX_REDIRECTION = 20;
|
|
16
22
|
const MAX_HTML_SIZE = 1024 * 1024; // 1MB
|
|
23
|
+
const MAX_ERROR_BODY_SIZE = 1024 * 1024; // 1MB
|
|
24
|
+
const DEFAULT_TIMEOUT = 10_000; // 10 seconds
|
|
25
|
+
// The maximum delay setTimeout() accepts; larger values fire immediately
|
|
26
|
+
// on some runtimes:
|
|
27
|
+
const MAX_TIMEOUT = 2_147_483_647;
|
|
17
28
|
|
|
18
29
|
/**
|
|
19
30
|
* A remote JSON-LD document and its context fetched by
|
|
@@ -46,6 +57,14 @@ export interface DocumentLoaderOptions {
|
|
|
46
57
|
* @since 1.8.0
|
|
47
58
|
*/
|
|
48
59
|
signal?: AbortSignal;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Whether to lower error-level logs for recoverable document loading
|
|
63
|
+
* failures to warning-level logs. The loader still throws the error.
|
|
64
|
+
* @default `false`
|
|
65
|
+
* @since 2.4.0
|
|
66
|
+
*/
|
|
67
|
+
suppressError?: boolean;
|
|
49
68
|
}
|
|
50
69
|
|
|
51
70
|
/**
|
|
@@ -96,6 +115,29 @@ export interface DocumentLoaderFactoryOptions {
|
|
|
96
115
|
* @since 2.2.0
|
|
97
116
|
*/
|
|
98
117
|
maxRedirection?: number;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The timeout in milliseconds for each call of the created document
|
|
121
|
+
* loader. The timeout is shared by all the steps of a call, including
|
|
122
|
+
* URL validation, every HTTP redirect and alternate document link it
|
|
123
|
+
* follows, retries, and reading the response body; it does not restart
|
|
124
|
+
* for each of them. It does not interrupt synchronous work such as
|
|
125
|
+
* parsing a document that has already been received.
|
|
126
|
+
*
|
|
127
|
+
* When a call times out, the loader throws a {@link FetchError} without
|
|
128
|
+
* a {@link FetchError.response}, whose `cause` is a `DOMException` named
|
|
129
|
+
* `"TimeoutError"`. An `AbortSignal` passed through
|
|
130
|
+
* {@link DocumentLoaderOptions.signal} still cancels a call; in that case
|
|
131
|
+
* the loader throws the signal's reason as before.
|
|
132
|
+
*
|
|
133
|
+
* Fractional values are rounded up. Set it to `null` to turn off the
|
|
134
|
+
* timeout.
|
|
135
|
+
* @default `10000` (10 seconds)
|
|
136
|
+
* @throws {RangeError} If the value is not a positive finite number or is
|
|
137
|
+
* greater than 2,147,483,647 (about 24.8 days).
|
|
138
|
+
* @since 2.4.0
|
|
139
|
+
*/
|
|
140
|
+
timeout?: number | null;
|
|
99
141
|
}
|
|
100
142
|
|
|
101
143
|
/**
|
|
@@ -121,49 +163,59 @@ function createResponseMetadata(response: Response): Response {
|
|
|
121
163
|
});
|
|
122
164
|
}
|
|
123
165
|
|
|
124
|
-
|
|
125
|
-
if (response.body != null) {
|
|
126
|
-
await response.body.cancel();
|
|
127
|
-
}
|
|
128
|
-
}
|
|
166
|
+
const NULL_BODY_STATUSES: ReadonlySet<number> = new Set([204, 205, 304]);
|
|
129
167
|
|
|
130
|
-
|
|
168
|
+
/**
|
|
169
|
+
* Reads the body of an error response while the document loader is still
|
|
170
|
+
* running, so that its timeout and `AbortSignal` also bound the read, and
|
|
171
|
+
* nothing reading {@link FetchError.response} later waits on the network.
|
|
172
|
+
* The body is kept byte for byte, unless it is too large or cannot be read;
|
|
173
|
+
* then only the status and headers are kept. A response whose status
|
|
174
|
+
* the `Response` constructor does not accept (e.g., 999) is kept as a clone
|
|
175
|
+
* whose body has been read in full; if its body is too large or cannot be
|
|
176
|
+
* read, no response is kept at all.
|
|
177
|
+
*/
|
|
178
|
+
async function bufferErrorResponse(
|
|
131
179
|
response: Response,
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
if (
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
180
|
+
url: string,
|
|
181
|
+
signal?: AbortSignal,
|
|
182
|
+
): Promise<Response | undefined> {
|
|
183
|
+
if (response.status < 200 || response.status > 599) {
|
|
184
|
+
// Such a response cannot be rebuilt, so keep a clone instead, and read
|
|
185
|
+
// the original to the end so that the clone's body is buffered too:
|
|
186
|
+
const clone = response.clone();
|
|
187
|
+
try {
|
|
188
|
+
await readBoundedBytes(response, MAX_ERROR_BODY_SIZE, url);
|
|
189
|
+
} catch (error) {
|
|
190
|
+
await clone.body?.cancel().catch(() => {});
|
|
191
|
+
if (signal?.aborted) throw error;
|
|
192
|
+
logger.debug(
|
|
193
|
+
"Failed to read the error response body from {url}: {error}",
|
|
194
|
+
{ url, error },
|
|
195
|
+
);
|
|
196
|
+
return undefined;
|
|
140
197
|
}
|
|
198
|
+
return clone;
|
|
141
199
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const decoder = new TextDecoder();
|
|
147
|
-
let text = "";
|
|
148
|
-
let size = 0;
|
|
200
|
+
if (response.body == null || NULL_BODY_STATUSES.has(response.status)) {
|
|
201
|
+
return createResponseMetadata(response);
|
|
202
|
+
}
|
|
203
|
+
let body: Uint8Array<ArrayBuffer>;
|
|
149
204
|
try {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
}
|
|
159
|
-
size += chunkSize;
|
|
160
|
-
text += decoder.decode(result.value, { stream: true });
|
|
161
|
-
}
|
|
162
|
-
text += decoder.decode();
|
|
163
|
-
return { text, size, tooLarge: false };
|
|
164
|
-
} finally {
|
|
165
|
-
reader.releaseLock();
|
|
205
|
+
body = await readBoundedBytes(response, MAX_ERROR_BODY_SIZE, url);
|
|
206
|
+
} catch (error) {
|
|
207
|
+
if (signal?.aborted) throw error;
|
|
208
|
+
logger.debug(
|
|
209
|
+
"Failed to read the error response body from {url}: {error}",
|
|
210
|
+
{ url, error },
|
|
211
|
+
);
|
|
212
|
+
return createResponseMetadata(response);
|
|
166
213
|
}
|
|
214
|
+
return new Response(body, {
|
|
215
|
+
headers: response.headers,
|
|
216
|
+
status: response.status,
|
|
217
|
+
statusText: response.statusText,
|
|
218
|
+
});
|
|
167
219
|
}
|
|
168
220
|
|
|
169
221
|
/**
|
|
@@ -171,6 +223,7 @@ async function readBoundedText(
|
|
|
171
223
|
* @param url The URL of the document to load.
|
|
172
224
|
* @param response The response to get the document from.
|
|
173
225
|
* @param fetch The function to fetch the document.
|
|
226
|
+
* @param options The options for loading the document.
|
|
174
227
|
* @returns The loaded remote document.
|
|
175
228
|
* @throws {FetchError} If the response is not OK.
|
|
176
229
|
* @internal
|
|
@@ -182,22 +235,35 @@ export async function getRemoteDocument(
|
|
|
182
235
|
url: string,
|
|
183
236
|
options?: DocumentLoaderOptions,
|
|
184
237
|
) => Promise<RemoteDocument>,
|
|
238
|
+
options?: DocumentLoaderOptions,
|
|
185
239
|
): Promise<RemoteDocument> {
|
|
186
240
|
const documentUrl = response.url === "" ? url : response.url;
|
|
187
241
|
const docUrl = new URL(documentUrl);
|
|
188
242
|
if (!response.ok) {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
243
|
+
if (options?.suppressError) {
|
|
244
|
+
logger.warn(
|
|
245
|
+
"Failed to fetch document: {status} {url} {headers}",
|
|
246
|
+
{
|
|
247
|
+
status: response.status,
|
|
248
|
+
url: documentUrl,
|
|
249
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
250
|
+
},
|
|
251
|
+
);
|
|
252
|
+
} else {
|
|
253
|
+
logger.error(
|
|
254
|
+
"Failed to fetch document: {status} {url} {headers}",
|
|
255
|
+
{
|
|
256
|
+
status: response.status,
|
|
257
|
+
url: documentUrl,
|
|
258
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
259
|
+
},
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
|
|
197
263
|
throw new FetchError(
|
|
198
264
|
documentUrl,
|
|
199
265
|
`HTTP ${response.status}: ${documentUrl}`,
|
|
200
|
-
response
|
|
266
|
+
await bufferErrorResponse(response, documentUrl, options?.signal),
|
|
201
267
|
);
|
|
202
268
|
}
|
|
203
269
|
const contentType = response.headers.get("Content-Type");
|
|
@@ -242,7 +308,7 @@ export async function getRemoteDocument(
|
|
|
242
308
|
"Found alternate document: {alternateUrl} from {url}",
|
|
243
309
|
{ alternateUrl: altUri.href, url: documentUrl },
|
|
244
310
|
);
|
|
245
|
-
return await fetch(altUri.href);
|
|
311
|
+
return await fetch(altUri.href, options);
|
|
246
312
|
}
|
|
247
313
|
}
|
|
248
314
|
}
|
|
@@ -255,11 +321,14 @@ export async function getRemoteDocument(
|
|
|
255
321
|
contentType?.startsWith("application/xhtml+xml;"))
|
|
256
322
|
) {
|
|
257
323
|
const errorResponse = createResponseMetadata(response);
|
|
258
|
-
|
|
259
|
-
|
|
324
|
+
let html: string;
|
|
325
|
+
try {
|
|
326
|
+
html = await readBoundedText(response, MAX_HTML_SIZE, documentUrl);
|
|
327
|
+
} catch (error) {
|
|
328
|
+
if (!(error instanceof BodyTooLargeError)) throw error;
|
|
260
329
|
logger.warn(
|
|
261
330
|
"HTML response too large, skipping alternate link discovery: {url}",
|
|
262
|
-
{ url: documentUrl,
|
|
331
|
+
{ url: documentUrl, maxBytes: MAX_HTML_SIZE },
|
|
263
332
|
);
|
|
264
333
|
throw new FetchError(
|
|
265
334
|
documentUrl,
|
|
@@ -267,58 +336,58 @@ export async function getRemoteDocument(
|
|
|
267
336
|
`(Content-Type: ${contentType})`,
|
|
268
337
|
errorResponse,
|
|
269
338
|
);
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
/([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
278
|
-
|
|
279
|
-
let tagMatch: RegExpExecArray | null;
|
|
280
|
-
while ((tagMatch = tagPattern.exec(html.text)) !== null) {
|
|
281
|
-
const tagContent = tagMatch[2];
|
|
282
|
-
let attrMatch: RegExpExecArray | null;
|
|
283
|
-
const attribs: Record<string, string> = {};
|
|
339
|
+
}
|
|
340
|
+
// Safe regex patterns without nested quantifiers to prevent ReDoS
|
|
341
|
+
// (CVE-2025-68475)
|
|
342
|
+
// Step 1: Extract <a ...> or <link ...> tags
|
|
343
|
+
const tagPattern = /<(a|link)\s+([^>]*?)\s*\/?>/gi;
|
|
344
|
+
// Step 2: Parse attributes
|
|
345
|
+
const attrPattern = /([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
284
346
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
attribs[key] = value;
|
|
291
|
-
}
|
|
347
|
+
let tagMatch: RegExpExecArray | null;
|
|
348
|
+
while ((tagMatch = tagPattern.exec(html)) !== null) {
|
|
349
|
+
const tagContent = tagMatch[2];
|
|
350
|
+
let attrMatch: RegExpExecArray | null;
|
|
351
|
+
const attribs: Record<string, string> = {};
|
|
292
352
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
new URL(attribs.href, docUrl).href !== docUrl.href
|
|
300
|
-
) {
|
|
301
|
-
logger.debug(
|
|
302
|
-
"Found alternate document: {alternateUrl} from {url}",
|
|
303
|
-
{ alternateUrl: attribs.href, url: documentUrl },
|
|
304
|
-
);
|
|
305
|
-
return await fetch(new URL(attribs.href, docUrl).href);
|
|
306
|
-
}
|
|
353
|
+
// Reset regex state for attribute parsing
|
|
354
|
+
attrPattern.lastIndex = 0;
|
|
355
|
+
while ((attrMatch = attrPattern.exec(tagContent)) !== null) {
|
|
356
|
+
const key = attrMatch[1].toLowerCase();
|
|
357
|
+
const value = attrMatch[2] ?? attrMatch[3] ?? attrMatch[4] ?? "";
|
|
358
|
+
attribs[key] = value;
|
|
307
359
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
360
|
+
|
|
361
|
+
if (
|
|
362
|
+
attribs.rel === "alternate" && "type" in attribs && (
|
|
363
|
+
attribs.type === "application/activity+json" ||
|
|
364
|
+
attribs.type === "application/ld+json" ||
|
|
365
|
+
attribs.type.startsWith("application/ld+json;")
|
|
366
|
+
) && "href" in attribs &&
|
|
367
|
+
new URL(attribs.href, docUrl).href !== docUrl.href
|
|
368
|
+
) {
|
|
369
|
+
logger.debug(
|
|
370
|
+
"Found alternate document: {alternateUrl} from {url}",
|
|
371
|
+
{ alternateUrl: attribs.href, url: documentUrl },
|
|
317
372
|
);
|
|
373
|
+
return await fetch(new URL(attribs.href, docUrl).href, options);
|
|
318
374
|
}
|
|
319
375
|
}
|
|
376
|
+
try {
|
|
377
|
+
document = JSON.parse(html);
|
|
378
|
+
} catch (error) {
|
|
379
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
380
|
+
throw new FetchError(
|
|
381
|
+
documentUrl,
|
|
382
|
+
`HTML document has no ActivityPub alternate link ` +
|
|
383
|
+
`(Content-Type: ${contentType})`,
|
|
384
|
+
errorResponse,
|
|
385
|
+
);
|
|
386
|
+
}
|
|
320
387
|
} else {
|
|
321
|
-
document =
|
|
388
|
+
document = JSON.parse(
|
|
389
|
+
await readBoundedText(response, MAX_BODY_SIZE, documentUrl),
|
|
390
|
+
);
|
|
322
391
|
}
|
|
323
392
|
logger.debug(
|
|
324
393
|
"Fetched document: {status} {url} {headers}",
|
|
@@ -331,6 +400,104 @@ export async function getRemoteDocument(
|
|
|
331
400
|
return { contextUrl, document, documentUrl };
|
|
332
401
|
}
|
|
333
402
|
|
|
403
|
+
/**
|
|
404
|
+
* Resolves {@link DocumentLoaderFactoryOptions.timeout} into milliseconds.
|
|
405
|
+
* @param timeout The timeout option. `undefined` means the default timeout,
|
|
406
|
+
* and `null` means no timeout.
|
|
407
|
+
* @returns The timeout in milliseconds, or `null` if it is turned off.
|
|
408
|
+
* @throws {RangeError} If the timeout is invalid.
|
|
409
|
+
* @internal
|
|
410
|
+
*/
|
|
411
|
+
export function resolveDocumentLoaderTimeout(
|
|
412
|
+
timeout: number | null | undefined,
|
|
413
|
+
): number | null {
|
|
414
|
+
if (timeout === undefined) return DEFAULT_TIMEOUT;
|
|
415
|
+
if (timeout === null) return null;
|
|
416
|
+
if (
|
|
417
|
+
typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0
|
|
418
|
+
) {
|
|
419
|
+
throw new RangeError(
|
|
420
|
+
`The document loader timeout must be a positive finite number of ` +
|
|
421
|
+
`milliseconds, but got ${String(timeout)}.`,
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
const ms = Math.ceil(timeout);
|
|
425
|
+
if (ms > MAX_TIMEOUT) {
|
|
426
|
+
throw new RangeError(
|
|
427
|
+
`The document loader timeout must not be greater than ${MAX_TIMEOUT} ` +
|
|
428
|
+
`milliseconds, but got ${timeout}.`,
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
return ms;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* Bounds each call of the given document loader by the given timeout.
|
|
436
|
+
* The timeout is combined with the caller's `signal`, and the combined
|
|
437
|
+
* signal is passed to the loader. The call settles no later than the
|
|
438
|
+
* timeout even if the loader is stuck in a step that cannot be aborted,
|
|
439
|
+
* e.g., a DNS lookup.
|
|
440
|
+
*
|
|
441
|
+
* A timed-out call throws a {@link FetchError} without a response, whose
|
|
442
|
+
* `cause` is a `DOMException` named `"TimeoutError"`. If the caller's
|
|
443
|
+
* signal is aborted, its reason is thrown instead.
|
|
444
|
+
* @param loader The document loader to bound.
|
|
445
|
+
* @param timeout The timeout in milliseconds, or `null` for no timeout.
|
|
446
|
+
* It is assumed to have been resolved by
|
|
447
|
+
* {@link resolveDocumentLoaderTimeout}.
|
|
448
|
+
* @returns The bounded document loader.
|
|
449
|
+
* @internal
|
|
450
|
+
*/
|
|
451
|
+
export function withDocumentLoaderTimeout(
|
|
452
|
+
loader: DocumentLoader,
|
|
453
|
+
timeout: number | null,
|
|
454
|
+
): DocumentLoader {
|
|
455
|
+
if (timeout == null) return loader;
|
|
456
|
+
return async (url, options) => {
|
|
457
|
+
const callerSignal = options?.signal;
|
|
458
|
+
callerSignal?.throwIfAborted();
|
|
459
|
+
const controller = new AbortController();
|
|
460
|
+
const timeoutReason = new DOMException(
|
|
461
|
+
`The document loader timed out after ${timeout} ms.`,
|
|
462
|
+
"TimeoutError",
|
|
463
|
+
);
|
|
464
|
+
let timedOut = false;
|
|
465
|
+
const timer = setTimeout(() => {
|
|
466
|
+
timedOut = true;
|
|
467
|
+
controller.abort(timeoutReason);
|
|
468
|
+
}, timeout);
|
|
469
|
+
const onCallerAbort = () => controller.abort(callerSignal?.reason);
|
|
470
|
+
callerSignal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
471
|
+
let onAbort: (() => void) | undefined;
|
|
472
|
+
const aborted = new Promise<never>((_, reject) => {
|
|
473
|
+
onAbort = () => reject(controller.signal.reason);
|
|
474
|
+
controller.signal.addEventListener("abort", onAbort, { once: true });
|
|
475
|
+
});
|
|
476
|
+
const loading = loader(url, { ...options, signal: controller.signal });
|
|
477
|
+
// If the abort wins the race, the loader's late rejection is ignored:
|
|
478
|
+
loading.catch(() => {});
|
|
479
|
+
try {
|
|
480
|
+
return await Promise.race([loading, aborted]);
|
|
481
|
+
} catch (error) {
|
|
482
|
+
if (callerSignal?.aborted) throw callerSignal.reason;
|
|
483
|
+
if (!timedOut) throw error;
|
|
484
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
485
|
+
"Timed out after {timeout} ms while fetching document: {url}",
|
|
486
|
+
{ timeout, url },
|
|
487
|
+
);
|
|
488
|
+
const fetchError = new FetchError(url, `Timed out after ${timeout} ms`);
|
|
489
|
+
fetchError.cause = timeoutReason;
|
|
490
|
+
throw fetchError;
|
|
491
|
+
} finally {
|
|
492
|
+
clearTimeout(timer);
|
|
493
|
+
callerSignal?.removeEventListener("abort", onCallerAbort);
|
|
494
|
+
if (onAbort != null) {
|
|
495
|
+
controller.signal.removeEventListener("abort", onAbort);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
|
|
334
501
|
/**
|
|
335
502
|
* Options for {@link getDocumentLoader}.
|
|
336
503
|
* @since 1.3.0
|
|
@@ -344,6 +511,10 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
|
|
|
344
511
|
|
|
345
512
|
/**
|
|
346
513
|
* Creates a JSON-LD document loader that utilizes the browser's `fetch` API.
|
|
514
|
+
* At most 20 HTTP redirects and alternate document links are followed in total
|
|
515
|
+
* per call. Revisiting a URL within that chain throws a {@link FetchError}.
|
|
516
|
+
* Each call times out after 10 seconds by default; see
|
|
517
|
+
* {@link DocumentLoaderFactoryOptions.timeout}.
|
|
347
518
|
*
|
|
348
519
|
* The created loader preloads the below frequently used contexts by default
|
|
349
520
|
* (unless `options.skipPreloadedContexts` is set to `true`):
|
|
@@ -351,8 +522,12 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
|
|
|
351
522
|
* - <https://www.w3.org/ns/activitystreams>
|
|
352
523
|
* - <https://w3id.org/security/v1>
|
|
353
524
|
* - <https://w3id.org/security/data-integrity/v1>
|
|
525
|
+
* - <https://w3id.org/security/data-integrity/v2>
|
|
354
526
|
* - <https://www.w3.org/ns/did/v1>
|
|
527
|
+
* - <https://www.w3.org/ns/cid/v1>
|
|
355
528
|
* - <https://w3id.org/security/multikey/v1>
|
|
529
|
+
* - <https://w3id.org/fep/ef61>
|
|
530
|
+
* - <https://w3id.org/fep/7aa9>
|
|
356
531
|
* - <https://purl.archive.org/socialweb/webfinger>
|
|
357
532
|
* - <http://schema.org/>
|
|
358
533
|
* @param options Options for the document loader.
|
|
@@ -360,9 +535,15 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
|
|
|
360
535
|
* @since 1.3.0
|
|
361
536
|
*/
|
|
362
537
|
export function getDocumentLoader(
|
|
363
|
-
{
|
|
364
|
-
|
|
538
|
+
{
|
|
539
|
+
allowPrivateAddress,
|
|
540
|
+
maxRedirection,
|
|
541
|
+
skipPreloadedContexts,
|
|
542
|
+
timeout,
|
|
543
|
+
userAgent,
|
|
544
|
+
}: GetDocumentLoaderOptions = {},
|
|
365
545
|
): DocumentLoader {
|
|
546
|
+
const resolvedTimeout = resolveDocumentLoaderTimeout(timeout);
|
|
366
547
|
const tracerProvider = trace.getTracerProvider();
|
|
367
548
|
const tracer = tracerProvider.getTracer(metadata.name, metadata.version);
|
|
368
549
|
const maximumRedirection = maxRedirection ?? DEFAULT_MAX_REDIRECTION;
|
|
@@ -388,13 +569,26 @@ export function getDocumentLoader(
|
|
|
388
569
|
await validatePublicUrl(currentUrl);
|
|
389
570
|
} catch (error) {
|
|
390
571
|
if (error instanceof UrlError) {
|
|
391
|
-
|
|
392
|
-
url
|
|
393
|
-
|
|
394
|
-
|
|
572
|
+
if (error.reason === "dns") {
|
|
573
|
+
logger.debug("DNS lookup failed for {url}", {
|
|
574
|
+
url: currentUrl,
|
|
575
|
+
error,
|
|
576
|
+
});
|
|
577
|
+
} else {
|
|
578
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
579
|
+
"Disallowed private URL: {url}",
|
|
580
|
+
{
|
|
581
|
+
url: currentUrl,
|
|
582
|
+
error,
|
|
583
|
+
},
|
|
584
|
+
);
|
|
585
|
+
}
|
|
395
586
|
}
|
|
396
587
|
throw error;
|
|
397
588
|
}
|
|
589
|
+
// The DNS lookup cannot be aborted, so do not go on if the call was
|
|
590
|
+
// aborted or timed out in the meantime:
|
|
591
|
+
options?.signal?.throwIfAborted();
|
|
398
592
|
}
|
|
399
593
|
visited.add(currentUrl);
|
|
400
594
|
|
|
@@ -425,7 +619,7 @@ export function getDocumentLoader(
|
|
|
425
619
|
response.headers.has("Location")
|
|
426
620
|
) {
|
|
427
621
|
if (redirected >= maximumRedirection) {
|
|
428
|
-
logger
|
|
622
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
429
623
|
"Too many redirections ({redirections}) while fetching document.",
|
|
430
624
|
{ redirections: redirected + 1, url: currentUrl },
|
|
431
625
|
);
|
|
@@ -440,7 +634,7 @@ export function getDocumentLoader(
|
|
|
440
634
|
).href;
|
|
441
635
|
span.setAttribute("http.redirect.url", redirectUrl);
|
|
442
636
|
if (visited.has(redirectUrl)) {
|
|
443
|
-
logger
|
|
637
|
+
logger[options?.suppressError ? "warn" : "error"](
|
|
444
638
|
"Detected a redirect loop while fetching document: {url} -> " +
|
|
445
639
|
"{redirectUrl}",
|
|
446
640
|
{ url: currentUrl, redirectUrl },
|
|
@@ -453,7 +647,27 @@ export function getDocumentLoader(
|
|
|
453
647
|
return await load(redirectUrl, options, redirected + 1, visited);
|
|
454
648
|
}
|
|
455
649
|
|
|
456
|
-
const result = await getRemoteDocument(
|
|
650
|
+
const result = await getRemoteDocument(
|
|
651
|
+
currentUrl,
|
|
652
|
+
response,
|
|
653
|
+
async (alternateUrl) => {
|
|
654
|
+
options?.signal?.throwIfAborted();
|
|
655
|
+
if (redirected >= DEFAULT_MAX_REDIRECTION) {
|
|
656
|
+
throw new FetchError(
|
|
657
|
+
currentUrl,
|
|
658
|
+
`Too many redirections (${redirected + 1})`,
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
if (visited.has(alternateUrl)) {
|
|
662
|
+
throw new FetchError(
|
|
663
|
+
currentUrl,
|
|
664
|
+
`Redirect loop detected: ${alternateUrl}`,
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
return await load(alternateUrl, options, redirected + 1, visited);
|
|
668
|
+
},
|
|
669
|
+
options,
|
|
670
|
+
);
|
|
457
671
|
span.setAttribute("docloader.document_url", result.documentUrl);
|
|
458
672
|
if (result.contextUrl != null) {
|
|
459
673
|
span.setAttribute("docloader.context_url", result.contextUrl);
|
|
@@ -472,5 +686,8 @@ export function getDocumentLoader(
|
|
|
472
686
|
},
|
|
473
687
|
);
|
|
474
688
|
}
|
|
475
|
-
return
|
|
689
|
+
return withDocumentLoaderTimeout(
|
|
690
|
+
(url, options) => load(url, options),
|
|
691
|
+
resolvedTimeout,
|
|
692
|
+
);
|
|
476
693
|
}
|
|
@@ -1,9 +1,105 @@
|
|
|
1
1
|
import type { DocumentLoader } from "../docloader.ts";
|
|
2
2
|
import jsonld from "../jsonld.ts";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
formatIri,
|
|
5
|
+
fromCompatibleEf61Id,
|
|
6
|
+
getFe34Origin,
|
|
7
|
+
haveSameFe34Origin,
|
|
8
|
+
haveSameIriOrigin,
|
|
9
|
+
parseIri,
|
|
10
|
+
} from "../url.ts";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Reads expanded gateways, including an unmapped term on portable actors.
|
|
14
|
+
* The original values and their order are preserved for URL validation by
|
|
15
|
+
* the caller. An explicitly mapped FEP-ef61 property always takes precedence.
|
|
16
|
+
* @internal Used by generated vocabulary classes and portable proof checks.
|
|
17
|
+
*/
|
|
18
|
+
export function getPortableActorGateways(
|
|
19
|
+
node: Record<string, unknown>,
|
|
20
|
+
allowUnmapped = true,
|
|
21
|
+
): unknown[] | undefined {
|
|
22
|
+
const property = "https://w3id.org/fep/ef61/gateways";
|
|
23
|
+
if (globalThis.Object.hasOwn(node, property)) {
|
|
24
|
+
return Array.isArray(node[property]) ? node[property] : undefined;
|
|
25
|
+
}
|
|
26
|
+
if (
|
|
27
|
+
!allowUnmapped || typeof node["@id"] !== "string" ||
|
|
28
|
+
!("http://www.w3.org/ns/ldp#inbox" in node) ||
|
|
29
|
+
!("https://www.w3.org/ns/activitystreams#outbox" in node)
|
|
30
|
+
) return undefined;
|
|
31
|
+
try {
|
|
32
|
+
const id = fromCompatibleEf61Id(node["@id"]) ?? node["@id"];
|
|
33
|
+
if (!getFe34Origin(parseIri(id)).startsWith("did:")) return undefined;
|
|
34
|
+
} catch (error) {
|
|
35
|
+
if (error instanceof TypeError) return undefined;
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
return Array.isArray(node["_:gateways"]) ? node["_:gateways"] : undefined;
|
|
39
|
+
}
|
|
4
40
|
|
|
5
41
|
const noJsonLdContext = Symbol("noJsonLdContext");
|
|
6
42
|
|
|
43
|
+
const documentLoaderWrappers = new WeakMap<
|
|
44
|
+
DocumentLoader,
|
|
45
|
+
{ readonly base: DocumentLoader; released: boolean }
|
|
46
|
+
>();
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Registers a loader wrapper with the shared released-wrapper registry.
|
|
50
|
+
* Both suppression and snapshot wrappers must use this registry so mixed
|
|
51
|
+
* chains can be unwrapped. The caller updates the state on release.
|
|
52
|
+
* @internal Not a public API.
|
|
53
|
+
*/
|
|
54
|
+
export function registerDocumentLoaderWrapper(
|
|
55
|
+
loader: DocumentLoader,
|
|
56
|
+
state: { readonly base: DocumentLoader; released: boolean },
|
|
57
|
+
): void {
|
|
58
|
+
documentLoaderWrappers.set(loader, state);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Removes released loader wrappers, stopping at the first active wrapper.
|
|
63
|
+
* Active wrappers retain their operation's suppression or snapshot cache.
|
|
64
|
+
* @internal Not a public API.
|
|
65
|
+
*/
|
|
66
|
+
export function unwrapReleasedDocumentLoader(
|
|
67
|
+
base: DocumentLoader,
|
|
68
|
+
): DocumentLoader {
|
|
69
|
+
for (
|
|
70
|
+
let state = documentLoaderWrappers.get(base);
|
|
71
|
+
state?.released;
|
|
72
|
+
state = documentLoaderWrappers.get(base)
|
|
73
|
+
) {
|
|
74
|
+
base = state.base;
|
|
75
|
+
}
|
|
76
|
+
return base;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Lowers context loading failure logs for one accessor operation. Parsed
|
|
81
|
+
* objects retain the loader, so release it before returning to the caller.
|
|
82
|
+
* Released wrappers are unwrapped to avoid accumulating loader chains.
|
|
83
|
+
* @internal Used by generated vocabulary accessors; not a public API.
|
|
84
|
+
*/
|
|
85
|
+
export function createScopedContextLoader(
|
|
86
|
+
base: DocumentLoader,
|
|
87
|
+
suppressError?: boolean,
|
|
88
|
+
): { loader: DocumentLoader; release: () => void } {
|
|
89
|
+
base = unwrapReleasedDocumentLoader(base);
|
|
90
|
+
if (!suppressError) return { loader: base, release: () => {} };
|
|
91
|
+
const state = { base, released: false };
|
|
92
|
+
const loader: DocumentLoader = (url, options) =>
|
|
93
|
+
base(url, state.released ? options : { ...options, suppressError: true });
|
|
94
|
+
registerDocumentLoaderWrapper(loader, state);
|
|
95
|
+
return {
|
|
96
|
+
loader,
|
|
97
|
+
release: () => {
|
|
98
|
+
state.released = true;
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
7
103
|
/**
|
|
8
104
|
* Options for deciding whether two IRIs should be treated as same-origin.
|
|
9
105
|
*
|