@fedify/vocab-runtime 2.4.0-dev.1528 → 2.4.0-dev.1531
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +1 -1
- package/dist/mod.cjs +73 -8
- package/dist/mod.js +73 -8
- package/dist/tests/decimal.test.cjs +1 -1
- package/dist/tests/decimal.test.mjs +1 -1
- package/dist/tests/{docloader-9JRkIgYh.mjs → docloader-Bfj7NaT_.mjs} +74 -9
- package/dist/tests/{docloader-yxXccHZv.cjs → docloader-Cvdl8PIZ.cjs} +79 -8
- package/dist/tests/docloader.test.cjs +57 -5
- package/dist/tests/docloader.test.mjs +57 -5
- package/dist/tests/{request-4-bNhrbG.mjs → request-B5Su2gl0.mjs} +1 -1
- package/dist/tests/{request-_LWX0dnv.cjs → request-DJvOZdo7.cjs} +1 -1
- package/dist/tests/request.test.cjs +1 -1
- package/dist/tests/request.test.mjs +1 -1
- package/package.json +1 -1
- package/src/docloader.test.ts +102 -6
- package/src/docloader.ts +76 -8
package/deno.json
CHANGED
package/dist/mod.cjs
CHANGED
|
@@ -4345,7 +4345,7 @@ const preloadedContexts = {
|
|
|
4345
4345
|
//#endregion
|
|
4346
4346
|
//#region deno.json
|
|
4347
4347
|
var name = "@fedify/vocab-runtime";
|
|
4348
|
-
var version = "2.4.0-dev.
|
|
4348
|
+
var version = "2.4.0-dev.1531+0896bc51";
|
|
4349
4349
|
//#endregion
|
|
4350
4350
|
//#region src/link.ts
|
|
4351
4351
|
const parametersNeedLowerCase = ["rel", "type"];
|
|
@@ -4605,6 +4605,66 @@ const logger = (0, _logtape_logtape.getLogger)([
|
|
|
4605
4605
|
"docloader"
|
|
4606
4606
|
]);
|
|
4607
4607
|
const DEFAULT_MAX_REDIRECTION = 20;
|
|
4608
|
+
const MAX_HTML_SIZE = 1024 * 1024;
|
|
4609
|
+
function createResponseMetadata(response) {
|
|
4610
|
+
return new Response(null, {
|
|
4611
|
+
headers: response.headers,
|
|
4612
|
+
status: response.status,
|
|
4613
|
+
statusText: response.statusText
|
|
4614
|
+
});
|
|
4615
|
+
}
|
|
4616
|
+
async function cancelResponseBody(response) {
|
|
4617
|
+
if (response.body != null) await response.body.cancel();
|
|
4618
|
+
}
|
|
4619
|
+
async function readBoundedText(response, maxBytes) {
|
|
4620
|
+
const contentLength = response.headers.get("Content-Length");
|
|
4621
|
+
if (contentLength != null) {
|
|
4622
|
+
const size = Number(contentLength);
|
|
4623
|
+
if (size > maxBytes) {
|
|
4624
|
+
await cancelResponseBody(response);
|
|
4625
|
+
return {
|
|
4626
|
+
text: "",
|
|
4627
|
+
size,
|
|
4628
|
+
tooLarge: true
|
|
4629
|
+
};
|
|
4630
|
+
}
|
|
4631
|
+
}
|
|
4632
|
+
if (response.body == null) return {
|
|
4633
|
+
text: "",
|
|
4634
|
+
size: 0,
|
|
4635
|
+
tooLarge: false
|
|
4636
|
+
};
|
|
4637
|
+
const reader = response.body.getReader();
|
|
4638
|
+
const decoder = new TextDecoder();
|
|
4639
|
+
let text = "";
|
|
4640
|
+
let size = 0;
|
|
4641
|
+
try {
|
|
4642
|
+
while (true) {
|
|
4643
|
+
const result = await reader.read();
|
|
4644
|
+
if (result.done) break;
|
|
4645
|
+
const chunkSize = result.value.byteLength;
|
|
4646
|
+
if (size + chunkSize > maxBytes) {
|
|
4647
|
+
size += chunkSize;
|
|
4648
|
+
await reader.cancel();
|
|
4649
|
+
return {
|
|
4650
|
+
text: "",
|
|
4651
|
+
size,
|
|
4652
|
+
tooLarge: true
|
|
4653
|
+
};
|
|
4654
|
+
}
|
|
4655
|
+
size += chunkSize;
|
|
4656
|
+
text += decoder.decode(result.value, { stream: true });
|
|
4657
|
+
}
|
|
4658
|
+
text += decoder.decode();
|
|
4659
|
+
return {
|
|
4660
|
+
text,
|
|
4661
|
+
size,
|
|
4662
|
+
tooLarge: false
|
|
4663
|
+
};
|
|
4664
|
+
} finally {
|
|
4665
|
+
reader.releaseLock();
|
|
4666
|
+
}
|
|
4667
|
+
}
|
|
4608
4668
|
/**
|
|
4609
4669
|
* Gets a {@link RemoteDocument} from the given response.
|
|
4610
4670
|
* @param url The URL of the document to load.
|
|
@@ -4659,19 +4719,19 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4659
4719
|
}
|
|
4660
4720
|
let document;
|
|
4661
4721
|
if (!jsonLd && (contentType === "text/html" || contentType?.startsWith("text/html;") || contentType === "application/xhtml+xml" || contentType?.startsWith("application/xhtml+xml;"))) {
|
|
4662
|
-
const
|
|
4663
|
-
const html = await response
|
|
4664
|
-
if (html.
|
|
4722
|
+
const errorResponse = createResponseMetadata(response);
|
|
4723
|
+
const html = await readBoundedText(response, MAX_HTML_SIZE);
|
|
4724
|
+
if (html.tooLarge) {
|
|
4665
4725
|
logger.warn("HTML response too large, skipping alternate link discovery: {url}", {
|
|
4666
4726
|
url: documentUrl,
|
|
4667
|
-
size: html.
|
|
4727
|
+
size: html.size
|
|
4668
4728
|
});
|
|
4669
|
-
document
|
|
4729
|
+
throw new FetchError(documentUrl, `HTML document is too large to scan for an ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4670
4730
|
} else {
|
|
4671
4731
|
const tagPattern = /<(a|link)\s+([^>]*?)\s*\/?>/gi;
|
|
4672
4732
|
const attrPattern = /([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
4673
4733
|
let tagMatch;
|
|
4674
|
-
while ((tagMatch = tagPattern.exec(html)) !== null) {
|
|
4734
|
+
while ((tagMatch = tagPattern.exec(html.text)) !== null) {
|
|
4675
4735
|
const tagContent = tagMatch[2];
|
|
4676
4736
|
let attrMatch;
|
|
4677
4737
|
const attribs = {};
|
|
@@ -4688,7 +4748,12 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4688
4748
|
return await fetch(new URL(attribs.href, docUrl).href);
|
|
4689
4749
|
}
|
|
4690
4750
|
}
|
|
4691
|
-
|
|
4751
|
+
try {
|
|
4752
|
+
document = JSON.parse(html.text);
|
|
4753
|
+
} catch (error) {
|
|
4754
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
4755
|
+
throw new FetchError(documentUrl, `HTML document has no ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4756
|
+
}
|
|
4692
4757
|
}
|
|
4693
4758
|
} else document = await response.json();
|
|
4694
4759
|
logger.debug("Fetched document: {status} {url} {headers}", {
|
package/dist/mod.js
CHANGED
|
@@ -4341,7 +4341,7 @@ const preloadedContexts = {
|
|
|
4341
4341
|
//#endregion
|
|
4342
4342
|
//#region deno.json
|
|
4343
4343
|
var name = "@fedify/vocab-runtime";
|
|
4344
|
-
var version = "2.4.0-dev.
|
|
4344
|
+
var version = "2.4.0-dev.1531+0896bc51";
|
|
4345
4345
|
//#endregion
|
|
4346
4346
|
//#region src/link.ts
|
|
4347
4347
|
const parametersNeedLowerCase = ["rel", "type"];
|
|
@@ -4601,6 +4601,66 @@ const logger = getLogger([
|
|
|
4601
4601
|
"docloader"
|
|
4602
4602
|
]);
|
|
4603
4603
|
const DEFAULT_MAX_REDIRECTION = 20;
|
|
4604
|
+
const MAX_HTML_SIZE = 1024 * 1024;
|
|
4605
|
+
function createResponseMetadata(response) {
|
|
4606
|
+
return new Response(null, {
|
|
4607
|
+
headers: response.headers,
|
|
4608
|
+
status: response.status,
|
|
4609
|
+
statusText: response.statusText
|
|
4610
|
+
});
|
|
4611
|
+
}
|
|
4612
|
+
async function cancelResponseBody(response) {
|
|
4613
|
+
if (response.body != null) await response.body.cancel();
|
|
4614
|
+
}
|
|
4615
|
+
async function readBoundedText(response, maxBytes) {
|
|
4616
|
+
const contentLength = response.headers.get("Content-Length");
|
|
4617
|
+
if (contentLength != null) {
|
|
4618
|
+
const size = Number(contentLength);
|
|
4619
|
+
if (size > maxBytes) {
|
|
4620
|
+
await cancelResponseBody(response);
|
|
4621
|
+
return {
|
|
4622
|
+
text: "",
|
|
4623
|
+
size,
|
|
4624
|
+
tooLarge: true
|
|
4625
|
+
};
|
|
4626
|
+
}
|
|
4627
|
+
}
|
|
4628
|
+
if (response.body == null) return {
|
|
4629
|
+
text: "",
|
|
4630
|
+
size: 0,
|
|
4631
|
+
tooLarge: false
|
|
4632
|
+
};
|
|
4633
|
+
const reader = response.body.getReader();
|
|
4634
|
+
const decoder = new TextDecoder();
|
|
4635
|
+
let text = "";
|
|
4636
|
+
let size = 0;
|
|
4637
|
+
try {
|
|
4638
|
+
while (true) {
|
|
4639
|
+
const result = await reader.read();
|
|
4640
|
+
if (result.done) break;
|
|
4641
|
+
const chunkSize = result.value.byteLength;
|
|
4642
|
+
if (size + chunkSize > maxBytes) {
|
|
4643
|
+
size += chunkSize;
|
|
4644
|
+
await reader.cancel();
|
|
4645
|
+
return {
|
|
4646
|
+
text: "",
|
|
4647
|
+
size,
|
|
4648
|
+
tooLarge: true
|
|
4649
|
+
};
|
|
4650
|
+
}
|
|
4651
|
+
size += chunkSize;
|
|
4652
|
+
text += decoder.decode(result.value, { stream: true });
|
|
4653
|
+
}
|
|
4654
|
+
text += decoder.decode();
|
|
4655
|
+
return {
|
|
4656
|
+
text,
|
|
4657
|
+
size,
|
|
4658
|
+
tooLarge: false
|
|
4659
|
+
};
|
|
4660
|
+
} finally {
|
|
4661
|
+
reader.releaseLock();
|
|
4662
|
+
}
|
|
4663
|
+
}
|
|
4604
4664
|
/**
|
|
4605
4665
|
* Gets a {@link RemoteDocument} from the given response.
|
|
4606
4666
|
* @param url The URL of the document to load.
|
|
@@ -4655,19 +4715,19 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4655
4715
|
}
|
|
4656
4716
|
let document;
|
|
4657
4717
|
if (!jsonLd && (contentType === "text/html" || contentType?.startsWith("text/html;") || contentType === "application/xhtml+xml" || contentType?.startsWith("application/xhtml+xml;"))) {
|
|
4658
|
-
const
|
|
4659
|
-
const html = await response
|
|
4660
|
-
if (html.
|
|
4718
|
+
const errorResponse = createResponseMetadata(response);
|
|
4719
|
+
const html = await readBoundedText(response, MAX_HTML_SIZE);
|
|
4720
|
+
if (html.tooLarge) {
|
|
4661
4721
|
logger.warn("HTML response too large, skipping alternate link discovery: {url}", {
|
|
4662
4722
|
url: documentUrl,
|
|
4663
|
-
size: html.
|
|
4723
|
+
size: html.size
|
|
4664
4724
|
});
|
|
4665
|
-
document
|
|
4725
|
+
throw new FetchError(documentUrl, `HTML document is too large to scan for an ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4666
4726
|
} else {
|
|
4667
4727
|
const tagPattern = /<(a|link)\s+([^>]*?)\s*\/?>/gi;
|
|
4668
4728
|
const attrPattern = /([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
4669
4729
|
let tagMatch;
|
|
4670
|
-
while ((tagMatch = tagPattern.exec(html)) !== null) {
|
|
4730
|
+
while ((tagMatch = tagPattern.exec(html.text)) !== null) {
|
|
4671
4731
|
const tagContent = tagMatch[2];
|
|
4672
4732
|
let attrMatch;
|
|
4673
4733
|
const attribs = {};
|
|
@@ -4684,7 +4744,12 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4684
4744
|
return await fetch(new URL(attribs.href, docUrl).href);
|
|
4685
4745
|
}
|
|
4686
4746
|
}
|
|
4687
|
-
|
|
4747
|
+
try {
|
|
4748
|
+
document = JSON.parse(html.text);
|
|
4749
|
+
} catch (error) {
|
|
4750
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
4751
|
+
throw new FetchError(documentUrl, `HTML document has no ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4752
|
+
}
|
|
4688
4753
|
}
|
|
4689
4754
|
} else document = await response.json();
|
|
4690
4755
|
logger.debug("Fetched document: {status} {url} {headers}", {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as name, i as logRequest, n as createActivityPubRequest, o as version, t as FetchError } from "./request-
|
|
1
|
+
import { a as name, i as logRequest, n as createActivityPubRequest, o as version, t as FetchError } from "./request-B5Su2gl0.mjs";
|
|
2
2
|
import { t as HttpHeaderLink } from "./link-NUUWCdnK.mjs";
|
|
3
3
|
import { l as validatePublicUrl, t as UrlError } from "./url-YWJbnRlf.mjs";
|
|
4
4
|
import { getLogger } from "@logtape/logtape";
|
|
@@ -4340,6 +4340,66 @@ const logger = getLogger([
|
|
|
4340
4340
|
"docloader"
|
|
4341
4341
|
]);
|
|
4342
4342
|
const DEFAULT_MAX_REDIRECTION = 20;
|
|
4343
|
+
const MAX_HTML_SIZE = 1024 * 1024;
|
|
4344
|
+
function createResponseMetadata(response) {
|
|
4345
|
+
return new Response(null, {
|
|
4346
|
+
headers: response.headers,
|
|
4347
|
+
status: response.status,
|
|
4348
|
+
statusText: response.statusText
|
|
4349
|
+
});
|
|
4350
|
+
}
|
|
4351
|
+
async function cancelResponseBody(response) {
|
|
4352
|
+
if (response.body != null) await response.body.cancel();
|
|
4353
|
+
}
|
|
4354
|
+
async function readBoundedText(response, maxBytes) {
|
|
4355
|
+
const contentLength = response.headers.get("Content-Length");
|
|
4356
|
+
if (contentLength != null) {
|
|
4357
|
+
const size = Number(contentLength);
|
|
4358
|
+
if (size > maxBytes) {
|
|
4359
|
+
await cancelResponseBody(response);
|
|
4360
|
+
return {
|
|
4361
|
+
text: "",
|
|
4362
|
+
size,
|
|
4363
|
+
tooLarge: true
|
|
4364
|
+
};
|
|
4365
|
+
}
|
|
4366
|
+
}
|
|
4367
|
+
if (response.body == null) return {
|
|
4368
|
+
text: "",
|
|
4369
|
+
size: 0,
|
|
4370
|
+
tooLarge: false
|
|
4371
|
+
};
|
|
4372
|
+
const reader = response.body.getReader();
|
|
4373
|
+
const decoder = new TextDecoder();
|
|
4374
|
+
let text = "";
|
|
4375
|
+
let size = 0;
|
|
4376
|
+
try {
|
|
4377
|
+
while (true) {
|
|
4378
|
+
const result = await reader.read();
|
|
4379
|
+
if (result.done) break;
|
|
4380
|
+
const chunkSize = result.value.byteLength;
|
|
4381
|
+
if (size + chunkSize > maxBytes) {
|
|
4382
|
+
size += chunkSize;
|
|
4383
|
+
await reader.cancel();
|
|
4384
|
+
return {
|
|
4385
|
+
text: "",
|
|
4386
|
+
size,
|
|
4387
|
+
tooLarge: true
|
|
4388
|
+
};
|
|
4389
|
+
}
|
|
4390
|
+
size += chunkSize;
|
|
4391
|
+
text += decoder.decode(result.value, { stream: true });
|
|
4392
|
+
}
|
|
4393
|
+
text += decoder.decode();
|
|
4394
|
+
return {
|
|
4395
|
+
text,
|
|
4396
|
+
size,
|
|
4397
|
+
tooLarge: false
|
|
4398
|
+
};
|
|
4399
|
+
} finally {
|
|
4400
|
+
reader.releaseLock();
|
|
4401
|
+
}
|
|
4402
|
+
}
|
|
4343
4403
|
/**
|
|
4344
4404
|
* Gets a {@link RemoteDocument} from the given response.
|
|
4345
4405
|
* @param url The URL of the document to load.
|
|
@@ -4394,19 +4454,19 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4394
4454
|
}
|
|
4395
4455
|
let document;
|
|
4396
4456
|
if (!jsonLd && (contentType === "text/html" || contentType?.startsWith("text/html;") || contentType === "application/xhtml+xml" || contentType?.startsWith("application/xhtml+xml;"))) {
|
|
4397
|
-
const
|
|
4398
|
-
const html = await response
|
|
4399
|
-
if (html.
|
|
4457
|
+
const errorResponse = createResponseMetadata(response);
|
|
4458
|
+
const html = await readBoundedText(response, MAX_HTML_SIZE);
|
|
4459
|
+
if (html.tooLarge) {
|
|
4400
4460
|
logger.warn("HTML response too large, skipping alternate link discovery: {url}", {
|
|
4401
4461
|
url: documentUrl,
|
|
4402
|
-
size: html.
|
|
4462
|
+
size: html.size
|
|
4403
4463
|
});
|
|
4404
|
-
document
|
|
4464
|
+
throw new FetchError(documentUrl, `HTML document is too large to scan for an ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4405
4465
|
} else {
|
|
4406
4466
|
const tagPattern = /<(a|link)\s+([^>]*?)\s*\/?>/gi;
|
|
4407
4467
|
const attrPattern = /([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
4408
4468
|
let tagMatch;
|
|
4409
|
-
while ((tagMatch = tagPattern.exec(html)) !== null) {
|
|
4469
|
+
while ((tagMatch = tagPattern.exec(html.text)) !== null) {
|
|
4410
4470
|
const tagContent = tagMatch[2];
|
|
4411
4471
|
let attrMatch;
|
|
4412
4472
|
const attribs = {};
|
|
@@ -4423,7 +4483,12 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4423
4483
|
return await fetch(new URL(attribs.href, docUrl).href);
|
|
4424
4484
|
}
|
|
4425
4485
|
}
|
|
4426
|
-
|
|
4486
|
+
try {
|
|
4487
|
+
document = JSON.parse(html.text);
|
|
4488
|
+
} catch (error) {
|
|
4489
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
4490
|
+
throw new FetchError(documentUrl, `HTML document has no ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4491
|
+
}
|
|
4427
4492
|
}
|
|
4428
4493
|
} else document = await response.json();
|
|
4429
4494
|
logger.debug("Fetched document: {status} {url} {headers}", {
|
|
@@ -4528,4 +4593,4 @@ function getDocumentLoader({ allowPrivateAddress, maxRedirection, skipPreloadedC
|
|
|
4528
4593
|
return load;
|
|
4529
4594
|
}
|
|
4530
4595
|
//#endregion
|
|
4531
|
-
export {
|
|
4596
|
+
export { getRemoteDocument as n, preloadedContexts as r, getDocumentLoader as t };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
require("./chunk-C2EiDwsr.cjs");
|
|
2
|
-
const require_request = require("./request-
|
|
2
|
+
const require_request = require("./request-DJvOZdo7.cjs");
|
|
3
3
|
const require_link = require("./link-FguCydMA.cjs");
|
|
4
4
|
const require_url = require("./url-2XwVbUS_.cjs");
|
|
5
5
|
let _logtape_logtape = require("@logtape/logtape");
|
|
@@ -4341,6 +4341,66 @@ const logger = (0, _logtape_logtape.getLogger)([
|
|
|
4341
4341
|
"docloader"
|
|
4342
4342
|
]);
|
|
4343
4343
|
const DEFAULT_MAX_REDIRECTION = 20;
|
|
4344
|
+
const MAX_HTML_SIZE = 1024 * 1024;
|
|
4345
|
+
function createResponseMetadata(response) {
|
|
4346
|
+
return new Response(null, {
|
|
4347
|
+
headers: response.headers,
|
|
4348
|
+
status: response.status,
|
|
4349
|
+
statusText: response.statusText
|
|
4350
|
+
});
|
|
4351
|
+
}
|
|
4352
|
+
async function cancelResponseBody(response) {
|
|
4353
|
+
if (response.body != null) await response.body.cancel();
|
|
4354
|
+
}
|
|
4355
|
+
async function readBoundedText(response, maxBytes) {
|
|
4356
|
+
const contentLength = response.headers.get("Content-Length");
|
|
4357
|
+
if (contentLength != null) {
|
|
4358
|
+
const size = Number(contentLength);
|
|
4359
|
+
if (size > maxBytes) {
|
|
4360
|
+
await cancelResponseBody(response);
|
|
4361
|
+
return {
|
|
4362
|
+
text: "",
|
|
4363
|
+
size,
|
|
4364
|
+
tooLarge: true
|
|
4365
|
+
};
|
|
4366
|
+
}
|
|
4367
|
+
}
|
|
4368
|
+
if (response.body == null) return {
|
|
4369
|
+
text: "",
|
|
4370
|
+
size: 0,
|
|
4371
|
+
tooLarge: false
|
|
4372
|
+
};
|
|
4373
|
+
const reader = response.body.getReader();
|
|
4374
|
+
const decoder = new TextDecoder();
|
|
4375
|
+
let text = "";
|
|
4376
|
+
let size = 0;
|
|
4377
|
+
try {
|
|
4378
|
+
while (true) {
|
|
4379
|
+
const result = await reader.read();
|
|
4380
|
+
if (result.done) break;
|
|
4381
|
+
const chunkSize = result.value.byteLength;
|
|
4382
|
+
if (size + chunkSize > maxBytes) {
|
|
4383
|
+
size += chunkSize;
|
|
4384
|
+
await reader.cancel();
|
|
4385
|
+
return {
|
|
4386
|
+
text: "",
|
|
4387
|
+
size,
|
|
4388
|
+
tooLarge: true
|
|
4389
|
+
};
|
|
4390
|
+
}
|
|
4391
|
+
size += chunkSize;
|
|
4392
|
+
text += decoder.decode(result.value, { stream: true });
|
|
4393
|
+
}
|
|
4394
|
+
text += decoder.decode();
|
|
4395
|
+
return {
|
|
4396
|
+
text,
|
|
4397
|
+
size,
|
|
4398
|
+
tooLarge: false
|
|
4399
|
+
};
|
|
4400
|
+
} finally {
|
|
4401
|
+
reader.releaseLock();
|
|
4402
|
+
}
|
|
4403
|
+
}
|
|
4344
4404
|
/**
|
|
4345
4405
|
* Gets a {@link RemoteDocument} from the given response.
|
|
4346
4406
|
* @param url The URL of the document to load.
|
|
@@ -4395,19 +4455,19 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4395
4455
|
}
|
|
4396
4456
|
let document;
|
|
4397
4457
|
if (!jsonLd && (contentType === "text/html" || contentType?.startsWith("text/html;") || contentType === "application/xhtml+xml" || contentType?.startsWith("application/xhtml+xml;"))) {
|
|
4398
|
-
const
|
|
4399
|
-
const html = await response
|
|
4400
|
-
if (html.
|
|
4458
|
+
const errorResponse = createResponseMetadata(response);
|
|
4459
|
+
const html = await readBoundedText(response, MAX_HTML_SIZE);
|
|
4460
|
+
if (html.tooLarge) {
|
|
4401
4461
|
logger.warn("HTML response too large, skipping alternate link discovery: {url}", {
|
|
4402
4462
|
url: documentUrl,
|
|
4403
|
-
size: html.
|
|
4463
|
+
size: html.size
|
|
4404
4464
|
});
|
|
4405
|
-
|
|
4465
|
+
throw new require_request.FetchError(documentUrl, `HTML document is too large to scan for an ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4406
4466
|
} else {
|
|
4407
4467
|
const tagPattern = /<(a|link)\s+([^>]*?)\s*\/?>/gi;
|
|
4408
4468
|
const attrPattern = /([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
4409
4469
|
let tagMatch;
|
|
4410
|
-
while ((tagMatch = tagPattern.exec(html)) !== null) {
|
|
4470
|
+
while ((tagMatch = tagPattern.exec(html.text)) !== null) {
|
|
4411
4471
|
const tagContent = tagMatch[2];
|
|
4412
4472
|
let attrMatch;
|
|
4413
4473
|
const attribs = {};
|
|
@@ -4424,7 +4484,12 @@ async function getRemoteDocument(url, response, fetch) {
|
|
|
4424
4484
|
return await fetch(new URL(attribs.href, docUrl).href);
|
|
4425
4485
|
}
|
|
4426
4486
|
}
|
|
4427
|
-
|
|
4487
|
+
try {
|
|
4488
|
+
document = JSON.parse(html.text);
|
|
4489
|
+
} catch (error) {
|
|
4490
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
4491
|
+
throw new require_request.FetchError(documentUrl, `HTML document has no ActivityPub alternate link (Content-Type: ${contentType})`, errorResponse);
|
|
4492
|
+
}
|
|
4428
4493
|
}
|
|
4429
4494
|
} else document = await response.json();
|
|
4430
4495
|
logger.debug("Fetched document: {status} {url} {headers}", {
|
|
@@ -4535,6 +4600,12 @@ Object.defineProperty(exports, "getDocumentLoader", {
|
|
|
4535
4600
|
return getDocumentLoader;
|
|
4536
4601
|
}
|
|
4537
4602
|
});
|
|
4603
|
+
Object.defineProperty(exports, "getRemoteDocument", {
|
|
4604
|
+
enumerable: true,
|
|
4605
|
+
get: function() {
|
|
4606
|
+
return getRemoteDocument;
|
|
4607
|
+
}
|
|
4608
|
+
});
|
|
4538
4609
|
Object.defineProperty(exports, "preloadedContexts", {
|
|
4539
4610
|
enumerable: true,
|
|
4540
4611
|
get: function() {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const require_chunk = require("./chunk-C2EiDwsr.cjs");
|
|
2
|
-
const require_docloader = require("./docloader-
|
|
3
|
-
const require_request = require("./request-
|
|
2
|
+
const require_docloader = require("./docloader-Cvdl8PIZ.cjs");
|
|
3
|
+
const require_request = require("./request-DJvOZdo7.cjs");
|
|
4
4
|
const require_url = require("./url-2XwVbUS_.cjs");
|
|
5
5
|
let node_assert = require("node:assert");
|
|
6
6
|
let node_test = require("node:test");
|
|
@@ -1267,7 +1267,7 @@ var esm_default = new class FetchMock {
|
|
|
1267
1267
|
type: "Object"
|
|
1268
1268
|
},
|
|
1269
1269
|
headers: {
|
|
1270
|
-
"Content-Type": "
|
|
1270
|
+
"Content-Type": "application/activity+json",
|
|
1271
1271
|
Link: "<https://example.com/object>; rel=\"alternate\"; type=\"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"\""
|
|
1272
1272
|
}
|
|
1273
1273
|
});
|
|
@@ -1399,6 +1399,27 @@ var esm_default = new class FetchMock {
|
|
|
1399
1399
|
}
|
|
1400
1400
|
});
|
|
1401
1401
|
});
|
|
1402
|
+
esm_default.get("https://example.com/html-no-alternate", {
|
|
1403
|
+
body: `<!DOCTYPE html>
|
|
1404
|
+
<html>
|
|
1405
|
+
<head>
|
|
1406
|
+
<title>Not an ActivityPub document</title>
|
|
1407
|
+
</head>
|
|
1408
|
+
<body>Not found</body>
|
|
1409
|
+
</html>`,
|
|
1410
|
+
headers: { "Content-Type": "text/html; charset=utf-8" }
|
|
1411
|
+
});
|
|
1412
|
+
await t.test("HTML without ActivityPub alternate link", async () => {
|
|
1413
|
+
await (0, node_assert.rejects)(() => fetchDocumentLoader("https://example.com/html-no-alternate"), (error) => {
|
|
1414
|
+
(0, node_assert.ok)(error instanceof require_request.FetchError);
|
|
1415
|
+
(0, node_assert.ok)(error.message.includes("HTML document has no ActivityPub alternate link"));
|
|
1416
|
+
(0, node_assert.ok)(error.message.includes("Content-Type: text/html; charset=utf-8"));
|
|
1417
|
+
(0, node_assert.deepStrictEqual)(error.url, new URL("https://example.com/html-no-alternate"));
|
|
1418
|
+
(0, node_assert.ok)(error.response != null);
|
|
1419
|
+
(0, node_assert.deepStrictEqual)(error.response.headers.get("Content-Type"), "text/html; charset=utf-8");
|
|
1420
|
+
return true;
|
|
1421
|
+
});
|
|
1422
|
+
});
|
|
1402
1423
|
esm_default.get("https://example.com/wrong-content-type", {
|
|
1403
1424
|
body: {
|
|
1404
1425
|
"@context": "https://www.w3.org/ns/activitystreams",
|
|
@@ -1408,7 +1429,7 @@ var esm_default = new class FetchMock {
|
|
|
1408
1429
|
},
|
|
1409
1430
|
headers: { "Content-Type": "text/html; charset=utf-8" }
|
|
1410
1431
|
});
|
|
1411
|
-
await t.test("
|
|
1432
|
+
await t.test("wrong Content-Type with JSON body", async () => {
|
|
1412
1433
|
(0, node_assert.deepStrictEqual)(await fetchDocumentLoader("https://example.com/wrong-content-type"), {
|
|
1413
1434
|
contextUrl: null,
|
|
1414
1435
|
documentUrl: "https://example.com/wrong-content-type",
|
|
@@ -1420,6 +1441,37 @@ var esm_default = new class FetchMock {
|
|
|
1420
1441
|
}
|
|
1421
1442
|
});
|
|
1422
1443
|
});
|
|
1444
|
+
esm_default.get("https://example.com/large-html", {
|
|
1445
|
+
body: "<!DOCTYPE html>",
|
|
1446
|
+
headers: {
|
|
1447
|
+
"Content-Length": String(1048577),
|
|
1448
|
+
"Content-Type": "text/html; charset=utf-8"
|
|
1449
|
+
}
|
|
1450
|
+
});
|
|
1451
|
+
await t.test("HTML Content-Length over limit", async () => {
|
|
1452
|
+
await (0, node_assert.rejects)(() => fetchDocumentLoader("https://example.com/large-html"), (error) => {
|
|
1453
|
+
(0, node_assert.ok)(error instanceof require_request.FetchError);
|
|
1454
|
+
(0, node_assert.ok)(error.message.includes("HTML document is too large to scan for an ActivityPub alternate link"));
|
|
1455
|
+
(0, node_assert.ok)(error.response != null);
|
|
1456
|
+
(0, node_assert.deepStrictEqual)(error.response.status, 200);
|
|
1457
|
+
(0, node_assert.deepStrictEqual)(error.response.headers.get("Content-Type"), "text/html; charset=utf-8");
|
|
1458
|
+
return true;
|
|
1459
|
+
});
|
|
1460
|
+
});
|
|
1461
|
+
await t.test("HTML Content-Length over limit cancels body", async () => {
|
|
1462
|
+
let canceled = false;
|
|
1463
|
+
const response = new Response("<!DOCTYPE html>", { headers: {
|
|
1464
|
+
"Content-Length": String(1048577),
|
|
1465
|
+
"Content-Type": "text/html; charset=utf-8"
|
|
1466
|
+
} });
|
|
1467
|
+
Object.defineProperty(response, "body", { value: { cancel: () => {
|
|
1468
|
+
canceled = true;
|
|
1469
|
+
} } });
|
|
1470
|
+
await (0, node_assert.rejects)(() => require_docloader.getRemoteDocument("https://example.com/large-html-cancel", response, () => {
|
|
1471
|
+
throw new Error("unexpected alternate fetch");
|
|
1472
|
+
}), require_request.FetchError);
|
|
1473
|
+
(0, node_assert.deepStrictEqual)(canceled, true);
|
|
1474
|
+
});
|
|
1423
1475
|
esm_default.get("https://example.com/404", { status: 404 });
|
|
1424
1476
|
await t.test("not ok", async () => {
|
|
1425
1477
|
await (0, node_assert.rejects)(() => fetchDocumentLoader("https://example.com/404"), require_request.FetchError, "HTTP 404: https://example.com/404");
|
|
@@ -1536,7 +1588,7 @@ var esm_default = new class FetchMock {
|
|
|
1536
1588
|
});
|
|
1537
1589
|
await t.test("ReDoS resistance (CVE-2025-68475)", async () => {
|
|
1538
1590
|
const start = performance.now();
|
|
1539
|
-
await (0, node_assert.rejects)(() => fetchDocumentLoader("https://example.com/redos"),
|
|
1591
|
+
await (0, node_assert.rejects)(() => fetchDocumentLoader("https://example.com/redos"), require_request.FetchError);
|
|
1540
1592
|
const elapsed = performance.now() - start;
|
|
1541
1593
|
(0, node_assert.ok)(elapsed < 1e3, `Potential ReDoS vulnerability detected: ${elapsed}ms (expected < 1000ms)`);
|
|
1542
1594
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { n as preloadedContexts, t as getDocumentLoader } from "./docloader-
|
|
2
|
-
import { t as FetchError } from "./request-
|
|
1
|
+
import { n as getRemoteDocument, r as preloadedContexts, t as getDocumentLoader } from "./docloader-Bfj7NaT_.mjs";
|
|
2
|
+
import { t as FetchError } from "./request-B5Su2gl0.mjs";
|
|
3
3
|
import { t as UrlError } from "./url-YWJbnRlf.mjs";
|
|
4
4
|
import { deepStrictEqual, ok, rejects } from "node:assert";
|
|
5
5
|
import { test } from "node:test";
|
|
@@ -1286,7 +1286,7 @@ test("getDocumentLoader()", async (t) => {
|
|
|
1286
1286
|
type: "Object"
|
|
1287
1287
|
},
|
|
1288
1288
|
headers: {
|
|
1289
|
-
"Content-Type": "
|
|
1289
|
+
"Content-Type": "application/activity+json",
|
|
1290
1290
|
Link: "<https://example.com/object>; rel=\"alternate\"; type=\"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"\""
|
|
1291
1291
|
}
|
|
1292
1292
|
});
|
|
@@ -1418,6 +1418,27 @@ test("getDocumentLoader()", async (t) => {
|
|
|
1418
1418
|
}
|
|
1419
1419
|
});
|
|
1420
1420
|
});
|
|
1421
|
+
esm_default.get("https://example.com/html-no-alternate", {
|
|
1422
|
+
body: `<!DOCTYPE html>
|
|
1423
|
+
<html>
|
|
1424
|
+
<head>
|
|
1425
|
+
<title>Not an ActivityPub document</title>
|
|
1426
|
+
</head>
|
|
1427
|
+
<body>Not found</body>
|
|
1428
|
+
</html>`,
|
|
1429
|
+
headers: { "Content-Type": "text/html; charset=utf-8" }
|
|
1430
|
+
});
|
|
1431
|
+
await t.test("HTML without ActivityPub alternate link", async () => {
|
|
1432
|
+
await rejects(() => fetchDocumentLoader("https://example.com/html-no-alternate"), (error) => {
|
|
1433
|
+
ok(error instanceof FetchError);
|
|
1434
|
+
ok(error.message.includes("HTML document has no ActivityPub alternate link"));
|
|
1435
|
+
ok(error.message.includes("Content-Type: text/html; charset=utf-8"));
|
|
1436
|
+
deepStrictEqual(error.url, new URL("https://example.com/html-no-alternate"));
|
|
1437
|
+
ok(error.response != null);
|
|
1438
|
+
deepStrictEqual(error.response.headers.get("Content-Type"), "text/html; charset=utf-8");
|
|
1439
|
+
return true;
|
|
1440
|
+
});
|
|
1441
|
+
});
|
|
1421
1442
|
esm_default.get("https://example.com/wrong-content-type", {
|
|
1422
1443
|
body: {
|
|
1423
1444
|
"@context": "https://www.w3.org/ns/activitystreams",
|
|
@@ -1427,7 +1448,7 @@ test("getDocumentLoader()", async (t) => {
|
|
|
1427
1448
|
},
|
|
1428
1449
|
headers: { "Content-Type": "text/html; charset=utf-8" }
|
|
1429
1450
|
});
|
|
1430
|
-
await t.test("
|
|
1451
|
+
await t.test("wrong Content-Type with JSON body", async () => {
|
|
1431
1452
|
deepStrictEqual(await fetchDocumentLoader("https://example.com/wrong-content-type"), {
|
|
1432
1453
|
contextUrl: null,
|
|
1433
1454
|
documentUrl: "https://example.com/wrong-content-type",
|
|
@@ -1439,6 +1460,37 @@ test("getDocumentLoader()", async (t) => {
|
|
|
1439
1460
|
}
|
|
1440
1461
|
});
|
|
1441
1462
|
});
|
|
1463
|
+
esm_default.get("https://example.com/large-html", {
|
|
1464
|
+
body: "<!DOCTYPE html>",
|
|
1465
|
+
headers: {
|
|
1466
|
+
"Content-Length": String(1048577),
|
|
1467
|
+
"Content-Type": "text/html; charset=utf-8"
|
|
1468
|
+
}
|
|
1469
|
+
});
|
|
1470
|
+
await t.test("HTML Content-Length over limit", async () => {
|
|
1471
|
+
await rejects(() => fetchDocumentLoader("https://example.com/large-html"), (error) => {
|
|
1472
|
+
ok(error instanceof FetchError);
|
|
1473
|
+
ok(error.message.includes("HTML document is too large to scan for an ActivityPub alternate link"));
|
|
1474
|
+
ok(error.response != null);
|
|
1475
|
+
deepStrictEqual(error.response.status, 200);
|
|
1476
|
+
deepStrictEqual(error.response.headers.get("Content-Type"), "text/html; charset=utf-8");
|
|
1477
|
+
return true;
|
|
1478
|
+
});
|
|
1479
|
+
});
|
|
1480
|
+
await t.test("HTML Content-Length over limit cancels body", async () => {
|
|
1481
|
+
let canceled = false;
|
|
1482
|
+
const response = new Response("<!DOCTYPE html>", { headers: {
|
|
1483
|
+
"Content-Length": String(1048577),
|
|
1484
|
+
"Content-Type": "text/html; charset=utf-8"
|
|
1485
|
+
} });
|
|
1486
|
+
Object.defineProperty(response, "body", { value: { cancel: () => {
|
|
1487
|
+
canceled = true;
|
|
1488
|
+
} } });
|
|
1489
|
+
await rejects(() => getRemoteDocument("https://example.com/large-html-cancel", response, () => {
|
|
1490
|
+
throw new Error("unexpected alternate fetch");
|
|
1491
|
+
}), FetchError);
|
|
1492
|
+
deepStrictEqual(canceled, true);
|
|
1493
|
+
});
|
|
1442
1494
|
esm_default.get("https://example.com/404", { status: 404 });
|
|
1443
1495
|
await t.test("not ok", async () => {
|
|
1444
1496
|
await rejects(() => fetchDocumentLoader("https://example.com/404"), FetchError, "HTTP 404: https://example.com/404");
|
|
@@ -1555,7 +1607,7 @@ test("getDocumentLoader()", async (t) => {
|
|
|
1555
1607
|
});
|
|
1556
1608
|
await t.test("ReDoS resistance (CVE-2025-68475)", async () => {
|
|
1557
1609
|
const start = performance.now();
|
|
1558
|
-
await rejects(() => fetchDocumentLoader("https://example.com/redos"),
|
|
1610
|
+
await rejects(() => fetchDocumentLoader("https://example.com/redos"), FetchError);
|
|
1559
1611
|
const elapsed = performance.now() - start;
|
|
1560
1612
|
ok(elapsed < 1e3, `Potential ReDoS vulnerability detected: ${elapsed}ms (expected < 1000ms)`);
|
|
1561
1613
|
});
|
|
@@ -3,7 +3,7 @@ let node_process = require("node:process");
|
|
|
3
3
|
node_process = require_chunk.__toESM(node_process, 1);
|
|
4
4
|
//#region deno.json
|
|
5
5
|
var name = "@fedify/vocab-runtime";
|
|
6
|
-
var version = "2.4.0-dev.
|
|
6
|
+
var version = "2.4.0-dev.1531+0896bc51";
|
|
7
7
|
//#endregion
|
|
8
8
|
//#region src/request.ts
|
|
9
9
|
/**
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const require_chunk = require("./chunk-C2EiDwsr.cjs");
|
|
2
|
-
const require_request = require("./request-
|
|
2
|
+
const require_request = require("./request-DJvOZdo7.cjs");
|
|
3
3
|
let node_assert = require("node:assert");
|
|
4
4
|
let node_test = require("node:test");
|
|
5
5
|
let node_process = require("node:process");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { o as version, r as getUserAgent } from "./request-
|
|
1
|
+
import { o as version, r as getUserAgent } from "./request-B5Su2gl0.mjs";
|
|
2
2
|
import { deepStrictEqual } from "node:assert";
|
|
3
3
|
import { test } from "node:test";
|
|
4
4
|
import process from "node:process";
|
package/package.json
CHANGED
package/src/docloader.test.ts
CHANGED
|
@@ -2,7 +2,7 @@ import fetchMock from "fetch-mock";
|
|
|
2
2
|
import { deepStrictEqual, ok, rejects } from "node:assert";
|
|
3
3
|
import { test } from "node:test";
|
|
4
4
|
import preloadedContexts from "./contexts.ts";
|
|
5
|
-
import { getDocumentLoader } from "./docloader.ts";
|
|
5
|
+
import { getDocumentLoader, getRemoteDocument } from "./docloader.ts";
|
|
6
6
|
import { FetchError } from "./request.ts";
|
|
7
7
|
import { UrlError } from "./url.ts";
|
|
8
8
|
|
|
@@ -90,7 +90,7 @@ test("getDocumentLoader()", async (t) => {
|
|
|
90
90
|
type: "Object",
|
|
91
91
|
},
|
|
92
92
|
headers: {
|
|
93
|
-
"Content-Type": "
|
|
93
|
+
"Content-Type": "application/activity+json",
|
|
94
94
|
Link: '<https://example.com/object>; rel="alternate"; ' +
|
|
95
95
|
'type="application/ld+json; profile="https://www.w3.org/ns/activitystreams""',
|
|
96
96
|
},
|
|
@@ -247,6 +247,44 @@ test("getDocumentLoader()", async (t) => {
|
|
|
247
247
|
});
|
|
248
248
|
});
|
|
249
249
|
|
|
250
|
+
fetchMock.get("https://example.com/html-no-alternate", {
|
|
251
|
+
body: `<!DOCTYPE html>
|
|
252
|
+
<html>
|
|
253
|
+
<head>
|
|
254
|
+
<title>Not an ActivityPub document</title>
|
|
255
|
+
</head>
|
|
256
|
+
<body>Not found</body>
|
|
257
|
+
</html>`,
|
|
258
|
+
headers: { "Content-Type": "text/html; charset=utf-8" },
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
await t.test("HTML without ActivityPub alternate link", async () => {
|
|
262
|
+
await rejects(
|
|
263
|
+
() => fetchDocumentLoader("https://example.com/html-no-alternate"),
|
|
264
|
+
(error) => {
|
|
265
|
+
ok(error instanceof FetchError);
|
|
266
|
+
ok(
|
|
267
|
+
error.message.includes(
|
|
268
|
+
"HTML document has no ActivityPub alternate link",
|
|
269
|
+
),
|
|
270
|
+
);
|
|
271
|
+
ok(
|
|
272
|
+
error.message.includes("Content-Type: text/html; charset=utf-8"),
|
|
273
|
+
);
|
|
274
|
+
deepStrictEqual(
|
|
275
|
+
error.url,
|
|
276
|
+
new URL("https://example.com/html-no-alternate"),
|
|
277
|
+
);
|
|
278
|
+
ok(error.response != null);
|
|
279
|
+
deepStrictEqual(
|
|
280
|
+
error.response.headers.get("Content-Type"),
|
|
281
|
+
"text/html; charset=utf-8",
|
|
282
|
+
);
|
|
283
|
+
return true;
|
|
284
|
+
},
|
|
285
|
+
);
|
|
286
|
+
});
|
|
287
|
+
|
|
250
288
|
fetchMock.get("https://example.com/wrong-content-type", {
|
|
251
289
|
body: {
|
|
252
290
|
"@context": "https://www.w3.org/ns/activitystreams",
|
|
@@ -257,7 +295,7 @@ test("getDocumentLoader()", async (t) => {
|
|
|
257
295
|
headers: { "Content-Type": "text/html; charset=utf-8" },
|
|
258
296
|
});
|
|
259
297
|
|
|
260
|
-
await t.test("
|
|
298
|
+
await t.test("wrong Content-Type with JSON body", async () => {
|
|
261
299
|
deepStrictEqual(
|
|
262
300
|
await fetchDocumentLoader("https://example.com/wrong-content-type"),
|
|
263
301
|
{
|
|
@@ -273,6 +311,64 @@ test("getDocumentLoader()", async (t) => {
|
|
|
273
311
|
);
|
|
274
312
|
});
|
|
275
313
|
|
|
314
|
+
fetchMock.get("https://example.com/large-html", {
|
|
315
|
+
body: "<!DOCTYPE html>",
|
|
316
|
+
headers: {
|
|
317
|
+
"Content-Length": String(1024 * 1024 + 1),
|
|
318
|
+
"Content-Type": "text/html; charset=utf-8",
|
|
319
|
+
},
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
await t.test("HTML Content-Length over limit", async () => {
|
|
323
|
+
await rejects(
|
|
324
|
+
() => fetchDocumentLoader("https://example.com/large-html"),
|
|
325
|
+
(error) => {
|
|
326
|
+
ok(error instanceof FetchError);
|
|
327
|
+
ok(
|
|
328
|
+
error.message.includes(
|
|
329
|
+
"HTML document is too large to scan for an ActivityPub alternate link",
|
|
330
|
+
),
|
|
331
|
+
);
|
|
332
|
+
ok(error.response != null);
|
|
333
|
+
deepStrictEqual(error.response.status, 200);
|
|
334
|
+
deepStrictEqual(
|
|
335
|
+
error.response.headers.get("Content-Type"),
|
|
336
|
+
"text/html; charset=utf-8",
|
|
337
|
+
);
|
|
338
|
+
return true;
|
|
339
|
+
},
|
|
340
|
+
);
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
await t.test("HTML Content-Length over limit cancels body", async () => {
|
|
344
|
+
let canceled = false;
|
|
345
|
+
const response = new Response("<!DOCTYPE html>", {
|
|
346
|
+
headers: {
|
|
347
|
+
"Content-Length": String(1024 * 1024 + 1),
|
|
348
|
+
"Content-Type": "text/html; charset=utf-8",
|
|
349
|
+
},
|
|
350
|
+
});
|
|
351
|
+
Object.defineProperty(response, "body", {
|
|
352
|
+
value: {
|
|
353
|
+
cancel: () => {
|
|
354
|
+
canceled = true;
|
|
355
|
+
},
|
|
356
|
+
},
|
|
357
|
+
});
|
|
358
|
+
await rejects(
|
|
359
|
+
() =>
|
|
360
|
+
getRemoteDocument(
|
|
361
|
+
"https://example.com/large-html-cancel",
|
|
362
|
+
response,
|
|
363
|
+
() => {
|
|
364
|
+
throw new Error("unexpected alternate fetch");
|
|
365
|
+
},
|
|
366
|
+
),
|
|
367
|
+
FetchError,
|
|
368
|
+
);
|
|
369
|
+
deepStrictEqual(canceled, true);
|
|
370
|
+
});
|
|
371
|
+
|
|
276
372
|
fetchMock.get("https://example.com/404", { status: 404 });
|
|
277
373
|
|
|
278
374
|
await t.test("not ok", async () => {
|
|
@@ -459,11 +555,11 @@ test("getDocumentLoader()", async (t) => {
|
|
|
459
555
|
|
|
460
556
|
await t.test("ReDoS resistance (CVE-2025-68475)", async () => {
|
|
461
557
|
const start = performance.now();
|
|
462
|
-
// The malicious HTML will fail
|
|
463
|
-
// that it should complete quickly (not hang due to ReDoS)
|
|
558
|
+
// The malicious HTML will fail alternate discovery, but the important
|
|
559
|
+
// thing is that it should complete quickly (not hang due to ReDoS).
|
|
464
560
|
await rejects(
|
|
465
561
|
() => fetchDocumentLoader("https://example.com/redos"),
|
|
466
|
-
|
|
562
|
+
FetchError,
|
|
467
563
|
);
|
|
468
564
|
const elapsed = performance.now() - start;
|
|
469
565
|
|
package/src/docloader.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { UrlError, validatePublicUrl } from "./url.ts";
|
|
|
13
13
|
|
|
14
14
|
const logger = getLogger(["fedify", "runtime", "docloader"]);
|
|
15
15
|
const DEFAULT_MAX_REDIRECTION = 20;
|
|
16
|
+
const MAX_HTML_SIZE = 1024 * 1024; // 1MB
|
|
16
17
|
|
|
17
18
|
/**
|
|
18
19
|
* A remote JSON-LD document and its context fetched by
|
|
@@ -112,6 +113,59 @@ export type AuthenticatedDocumentLoaderFactory = (
|
|
|
112
113
|
options?: DocumentLoaderFactoryOptions,
|
|
113
114
|
) => DocumentLoader;
|
|
114
115
|
|
|
116
|
+
function createResponseMetadata(response: Response): Response {
|
|
117
|
+
return new Response(null, {
|
|
118
|
+
headers: response.headers,
|
|
119
|
+
status: response.status,
|
|
120
|
+
statusText: response.statusText,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async function cancelResponseBody(response: Response): Promise<void> {
|
|
125
|
+
if (response.body != null) {
|
|
126
|
+
await response.body.cancel();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function readBoundedText(
|
|
131
|
+
response: Response,
|
|
132
|
+
maxBytes: number,
|
|
133
|
+
): Promise<{ text: string; size: number; tooLarge: boolean }> {
|
|
134
|
+
const contentLength = response.headers.get("Content-Length");
|
|
135
|
+
if (contentLength != null) {
|
|
136
|
+
const size = Number(contentLength);
|
|
137
|
+
if (size > maxBytes) {
|
|
138
|
+
await cancelResponseBody(response);
|
|
139
|
+
return { text: "", size, tooLarge: true };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (response.body == null) return { text: "", size: 0, tooLarge: false };
|
|
144
|
+
|
|
145
|
+
const reader = response.body.getReader();
|
|
146
|
+
const decoder = new TextDecoder();
|
|
147
|
+
let text = "";
|
|
148
|
+
let size = 0;
|
|
149
|
+
try {
|
|
150
|
+
while (true) {
|
|
151
|
+
const result = await reader.read();
|
|
152
|
+
if (result.done) break;
|
|
153
|
+
const chunkSize = result.value.byteLength;
|
|
154
|
+
if (size + chunkSize > maxBytes) {
|
|
155
|
+
size += chunkSize;
|
|
156
|
+
await reader.cancel();
|
|
157
|
+
return { text: "", size, tooLarge: true };
|
|
158
|
+
}
|
|
159
|
+
size += chunkSize;
|
|
160
|
+
text += decoder.decode(result.value, { stream: true });
|
|
161
|
+
}
|
|
162
|
+
text += decoder.decode();
|
|
163
|
+
return { text, size, tooLarge: false };
|
|
164
|
+
} finally {
|
|
165
|
+
reader.releaseLock();
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
115
169
|
/**
|
|
116
170
|
* Gets a {@link RemoteDocument} from the given response.
|
|
117
171
|
* @param url The URL of the document to load.
|
|
@@ -200,15 +254,19 @@ export async function getRemoteDocument(
|
|
|
200
254
|
contentType === "application/xhtml+xml" ||
|
|
201
255
|
contentType?.startsWith("application/xhtml+xml;"))
|
|
202
256
|
) {
|
|
203
|
-
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
if (html.length > MAX_HTML_SIZE) {
|
|
257
|
+
const errorResponse = createResponseMetadata(response);
|
|
258
|
+
const html = await readBoundedText(response, MAX_HTML_SIZE);
|
|
259
|
+
if (html.tooLarge) {
|
|
207
260
|
logger.warn(
|
|
208
261
|
"HTML response too large, skipping alternate link discovery: {url}",
|
|
209
|
-
{ url: documentUrl, size: html.
|
|
262
|
+
{ url: documentUrl, size: html.size },
|
|
263
|
+
);
|
|
264
|
+
throw new FetchError(
|
|
265
|
+
documentUrl,
|
|
266
|
+
`HTML document is too large to scan for an ActivityPub alternate link ` +
|
|
267
|
+
`(Content-Type: ${contentType})`,
|
|
268
|
+
errorResponse,
|
|
210
269
|
);
|
|
211
|
-
document = JSON.parse(html);
|
|
212
270
|
} else {
|
|
213
271
|
// Safe regex patterns without nested quantifiers to prevent ReDoS
|
|
214
272
|
// (CVE-2025-68475)
|
|
@@ -219,7 +277,7 @@ export async function getRemoteDocument(
|
|
|
219
277
|
/([a-z][a-z:_-]*)=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
220
278
|
|
|
221
279
|
let tagMatch: RegExpExecArray | null;
|
|
222
|
-
while ((tagMatch = tagPattern.exec(html)) !== null) {
|
|
280
|
+
while ((tagMatch = tagPattern.exec(html.text)) !== null) {
|
|
223
281
|
const tagContent = tagMatch[2];
|
|
224
282
|
let attrMatch: RegExpExecArray | null;
|
|
225
283
|
const attribs: Record<string, string> = {};
|
|
@@ -247,7 +305,17 @@ export async function getRemoteDocument(
|
|
|
247
305
|
return await fetch(new URL(attribs.href, docUrl).href);
|
|
248
306
|
}
|
|
249
307
|
}
|
|
250
|
-
|
|
308
|
+
try {
|
|
309
|
+
document = JSON.parse(html.text);
|
|
310
|
+
} catch (error) {
|
|
311
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
312
|
+
throw new FetchError(
|
|
313
|
+
documentUrl,
|
|
314
|
+
`HTML document has no ActivityPub alternate link ` +
|
|
315
|
+
`(Content-Type: ${contentType})`,
|
|
316
|
+
errorResponse,
|
|
317
|
+
);
|
|
318
|
+
}
|
|
251
319
|
}
|
|
252
320
|
} else {
|
|
253
321
|
document = await response.json();
|