@nuxtjs/sitemap 8.2.2 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/module.json +1 -1
- package/dist/module.mjs +8 -3
- package/dist/runtime/server/kit.js +9 -2
- package/dist/runtime/server/plugins/compression.js +57 -13
- package/dist/runtime/server/plugins/stream-transport.d.ts +2 -0
- package/dist/runtime/server/plugins/stream-transport.js +16 -0
- package/dist/runtime/server/routes/__sitemap__/debug.d.ts +1 -0
- package/dist/runtime/server/routes/__zero-runtime/sitemap/[sitemap].xml.d.ts +1 -1
- package/dist/runtime/server/routes/__zero-runtime/sitemap.xml.d.ts +1 -1
- package/dist/runtime/server/routes/__zero-runtime/sitemap_index.xml.d.ts +1 -1
- package/dist/runtime/server/routes/sitemap/[sitemap].xml.d.ts +1 -1
- package/dist/runtime/server/routes/sitemap.xml.d.ts +1 -1
- package/dist/runtime/server/routes/sitemap_index.xml.d.ts +1 -1
- package/dist/runtime/server/sitemap/builder/index-xml.d.ts +13 -0
- package/dist/runtime/server/sitemap/builder/index-xml.js +70 -0
- package/dist/runtime/server/sitemap/builder/sitemap-index.d.ts +1 -4
- package/dist/runtime/server/sitemap/builder/sitemap-index.js +2 -34
- package/dist/runtime/server/sitemap/builder/sitemap.d.ts +1 -1
- package/dist/runtime/server/sitemap/builder/sitemap.js +36 -25
- package/dist/runtime/server/sitemap/builder/xml.d.ts +8 -0
- package/dist/runtime/server/sitemap/builder/xml.js +35 -6
- package/dist/runtime/server/sitemap/event-handlers.d.ts +3 -3
- package/dist/runtime/server/sitemap/event-handlers.js +18 -10
- package/dist/runtime/server/sitemap/nitro.d.ts +3 -1
- package/dist/runtime/server/sitemap/nitro.js +107 -20
- package/dist/runtime/server/sitemap/stream.d.ts +30 -0
- package/dist/runtime/server/sitemap/stream.js +144 -0
- package/dist/runtime/server/sitemap/urlset/normalise.d.ts +6 -1
- package/dist/runtime/server/sitemap/urlset/normalise.js +15 -4
- package/dist/runtime/server/sitemap/urlset/sort.js +12 -3
- package/dist/runtime/server/sitemap/urlset/sources.js +3 -3
- package/dist/runtime/server/utils.d.ts +0 -1
- package/dist/runtime/server/utils.js +12 -12
- package/dist/runtime/types.d.ts +14 -3
- package/dist/runtime/utils-pure.d.ts +5 -4
- package/dist/runtime/utils-pure.js +41 -19
- package/dist/utils.d.mts +30 -2
- package/dist/utils.d.ts +30 -2
- package/dist/utils.mjs +646 -43
- package/package.json +22 -22
package/dist/utils.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { p as parseHtmlExtractSitemapMeta } from './shared/sitemap.BoMnWHOt.mjs';
|
|
2
|
+
import { XMLParser } from 'fast-xml-parser';
|
|
2
3
|
import 'ufo';
|
|
3
4
|
import 'ultrahtml';
|
|
4
5
|
|
|
@@ -8,7 +9,6 @@ function isValidUrl(value) {
|
|
|
8
9
|
async function parseSitemapIndex(xml) {
|
|
9
10
|
if (!xml)
|
|
10
11
|
throw new Error("Empty XML input provided");
|
|
11
|
-
const { XMLParser } = await import('fast-xml-parser');
|
|
12
12
|
const parser = new XMLParser({
|
|
13
13
|
isArray: (tagName) => tagName === "sitemap",
|
|
14
14
|
removeNSPrefix: true,
|
|
@@ -50,6 +50,20 @@ function isSitemapIndex(xml) {
|
|
|
50
50
|
return xml.includes("<sitemapindex") || xml.includes("sitemapindex>");
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
const DEFAULT_MAX_ENTRY_BYTES = 1024 * 1024;
|
|
54
|
+
const ARRAY_TAGS = /* @__PURE__ */ new Set(["url", "image", "video", "link", "tag", "price"]);
|
|
55
|
+
const CHANGE_FREQUENCIES = /* @__PURE__ */ new Set(["always", "hourly", "daily", "weekly", "monthly", "yearly", "never"]);
|
|
56
|
+
const SIMPLE_URL_FIELDS = /* @__PURE__ */ new Set(["loc", "lastmod", "changefreq", "priority"]);
|
|
57
|
+
function createUrlParser() {
|
|
58
|
+
return new XMLParser({
|
|
59
|
+
isArray: (tagName) => ARRAY_TAGS.has(tagName),
|
|
60
|
+
removeNSPrefix: true,
|
|
61
|
+
parseAttributeValue: false,
|
|
62
|
+
ignoreAttributes: false,
|
|
63
|
+
attributeNamePrefix: "",
|
|
64
|
+
trimValues: true
|
|
65
|
+
});
|
|
66
|
+
}
|
|
53
67
|
function isValidString(value) {
|
|
54
68
|
return typeof value === "string" && value.trim().length > 0;
|
|
55
69
|
}
|
|
@@ -85,8 +99,7 @@ function extractUrlFromParsedElement(urlElement, warnings) {
|
|
|
85
99
|
urlObj.lastmod = urlElement.lastmod;
|
|
86
100
|
}
|
|
87
101
|
if (isValidString(urlElement.changefreq)) {
|
|
88
|
-
|
|
89
|
-
if (validFreqs.includes(urlElement.changefreq)) {
|
|
102
|
+
if (CHANGE_FREQUENCIES.has(urlElement.changefreq)) {
|
|
90
103
|
urlObj.changefreq = urlElement.changefreq;
|
|
91
104
|
} else {
|
|
92
105
|
warnings.push({
|
|
@@ -115,21 +128,20 @@ function extractUrlFromParsedElement(urlElement, warnings) {
|
|
|
115
128
|
}
|
|
116
129
|
if (urlElement.image) {
|
|
117
130
|
const images = Array.isArray(urlElement.image) ? urlElement.image : [urlElement.image];
|
|
118
|
-
const validImages =
|
|
119
|
-
|
|
120
|
-
|
|
131
|
+
const validImages = [];
|
|
132
|
+
for (const image of images) {
|
|
133
|
+
if (isValidString(image.loc)) {
|
|
134
|
+
validImages.push({ loc: image.loc });
|
|
121
135
|
} else {
|
|
122
136
|
warnings.push({
|
|
123
137
|
type: "validation",
|
|
124
138
|
message: "Image missing required loc element",
|
|
125
139
|
context: { url: urlElement.loc, field: "image.loc" }
|
|
126
140
|
});
|
|
127
|
-
return null;
|
|
128
141
|
}
|
|
129
|
-
}).filter((img) => img !== null);
|
|
130
|
-
if (validImages.length > 0) {
|
|
131
|
-
urlObj.images = validImages;
|
|
132
142
|
}
|
|
143
|
+
if (validImages.length > 0)
|
|
144
|
+
urlObj.images = validImages;
|
|
133
145
|
}
|
|
134
146
|
if (urlElement.video) {
|
|
135
147
|
const videos = Array.isArray(urlElement.video) ? urlElement.video : [urlElement.video];
|
|
@@ -373,49 +385,640 @@ function extractUrlFromParsedElement(urlElement, warnings) {
|
|
|
373
385
|
});
|
|
374
386
|
}
|
|
375
387
|
}
|
|
376
|
-
return
|
|
377
|
-
Object.entries(urlObj).filter(
|
|
378
|
-
([_, value]) => value != null && (!Array.isArray(value) || value.length > 0)
|
|
379
|
-
)
|
|
380
|
-
);
|
|
388
|
+
return urlObj;
|
|
381
389
|
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
390
|
+
function parseTag(xml, start, end) {
|
|
391
|
+
let cursor = start + 1;
|
|
392
|
+
while (cursor < end && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
393
|
+
cursor++;
|
|
394
|
+
const closing = xml.charCodeAt(cursor) === 47;
|
|
395
|
+
if (closing)
|
|
396
|
+
cursor++;
|
|
397
|
+
if (cursor >= end || xml.charCodeAt(cursor) === 33 || xml.charCodeAt(cursor) === 63)
|
|
398
|
+
return null;
|
|
399
|
+
const nameStart = cursor;
|
|
400
|
+
while (cursor < end) {
|
|
401
|
+
const code = xml.charCodeAt(cursor);
|
|
402
|
+
if (isXmlWhitespace(code) || code === 47)
|
|
403
|
+
break;
|
|
404
|
+
cursor++;
|
|
386
405
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
406
|
+
if (cursor === nameStart)
|
|
407
|
+
return null;
|
|
408
|
+
const qualifiedName = xml.slice(nameStart, cursor);
|
|
409
|
+
const colon = qualifiedName.lastIndexOf(":");
|
|
410
|
+
let tail = end - 1;
|
|
411
|
+
while (tail > start && isXmlWhitespace(xml.charCodeAt(tail)))
|
|
412
|
+
tail--;
|
|
413
|
+
return {
|
|
414
|
+
name: colon === -1 ? qualifiedName : qualifiedName.slice(colon + 1),
|
|
415
|
+
closing,
|
|
416
|
+
selfClosing: !closing && xml.charCodeAt(tail) === 47
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
const URLSET_OPEN = 1;
|
|
420
|
+
const URLSET_CLOSE = 2;
|
|
421
|
+
const URLSET_SELF_CLOSING = 3;
|
|
422
|
+
const URL_OPEN = 4;
|
|
423
|
+
const URL_SELF_CLOSING = 5;
|
|
424
|
+
const URL_CLOSE = 6;
|
|
425
|
+
const SITEMAP_INDEX_OPEN = 7;
|
|
426
|
+
const SITEMAP_INDEX_CLOSE = 8;
|
|
427
|
+
const SITEMAP_INDEX_SELF_CLOSING = 9;
|
|
428
|
+
const SITEMAP_OPEN = 10;
|
|
429
|
+
const SITEMAP_SELF_CLOSING = 11;
|
|
430
|
+
const SITEMAP_CLOSE = 12;
|
|
431
|
+
function parseBoundaryTag(xml, start, end) {
|
|
432
|
+
let cursor = start + 1;
|
|
433
|
+
while (cursor < end && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
434
|
+
cursor++;
|
|
435
|
+
const closing = xml.charCodeAt(cursor) === 47;
|
|
436
|
+
if (closing)
|
|
437
|
+
cursor++;
|
|
438
|
+
if (cursor >= end || xml.charCodeAt(cursor) === 33 || xml.charCodeAt(cursor) === 63)
|
|
439
|
+
return 0;
|
|
440
|
+
let localNameStart = cursor;
|
|
441
|
+
while (cursor < end) {
|
|
442
|
+
const code = xml.charCodeAt(cursor);
|
|
443
|
+
if (isXmlWhitespace(code) || code === 47)
|
|
444
|
+
break;
|
|
445
|
+
if (code === 58)
|
|
446
|
+
localNameStart = cursor + 1;
|
|
447
|
+
cursor++;
|
|
448
|
+
}
|
|
449
|
+
const localNameLength = cursor - localNameStart;
|
|
450
|
+
let tail = end - 1;
|
|
451
|
+
while (tail > start && isXmlWhitespace(xml.charCodeAt(tail)))
|
|
452
|
+
tail--;
|
|
453
|
+
const selfClosing = !closing && xml.charCodeAt(tail) === 47;
|
|
454
|
+
if (localNameLength === 3 && xml.startsWith("url", localNameStart)) {
|
|
455
|
+
if (closing)
|
|
456
|
+
return URL_CLOSE;
|
|
457
|
+
return selfClosing ? URL_SELF_CLOSING : URL_OPEN;
|
|
458
|
+
}
|
|
459
|
+
if (localNameLength === 6 && xml.startsWith("urlset", localNameStart)) {
|
|
460
|
+
if (closing)
|
|
461
|
+
return URLSET_CLOSE;
|
|
462
|
+
return selfClosing ? URLSET_SELF_CLOSING : URLSET_OPEN;
|
|
463
|
+
}
|
|
464
|
+
if (localNameLength === 12 && xml.startsWith("sitemapindex", localNameStart)) {
|
|
465
|
+
if (closing)
|
|
466
|
+
return SITEMAP_INDEX_CLOSE;
|
|
467
|
+
return selfClosing ? SITEMAP_INDEX_SELF_CLOSING : SITEMAP_INDEX_OPEN;
|
|
468
|
+
}
|
|
469
|
+
if (localNameLength === 7 && xml.startsWith("sitemap", localNameStart)) {
|
|
470
|
+
if (closing)
|
|
471
|
+
return SITEMAP_CLOSE;
|
|
472
|
+
return selfClosing ? SITEMAP_SELF_CLOSING : SITEMAP_OPEN;
|
|
473
|
+
}
|
|
474
|
+
return 0;
|
|
475
|
+
}
|
|
476
|
+
function isXmlWhitespace(code) {
|
|
477
|
+
return code === 32 || code === 9 || code === 10 || code === 13;
|
|
478
|
+
}
|
|
479
|
+
function findMarkupEnd(xml, start) {
|
|
480
|
+
if (xml.startsWith("<!--", start)) {
|
|
481
|
+
const end = xml.indexOf("-->", start + 4);
|
|
482
|
+
return end === -1 ? -1 : end + 2;
|
|
483
|
+
}
|
|
484
|
+
if (xml.startsWith("<![CDATA[", start)) {
|
|
485
|
+
const end = xml.indexOf("]]>", start + 9);
|
|
486
|
+
return end === -1 ? -1 : end + 2;
|
|
487
|
+
}
|
|
488
|
+
if (xml.startsWith("<?", start)) {
|
|
489
|
+
const end = xml.indexOf("?>", start + 2);
|
|
490
|
+
return end === -1 ? -1 : end + 1;
|
|
491
|
+
}
|
|
492
|
+
let quote = 0;
|
|
493
|
+
for (let cursor = start + 1; cursor < xml.length; cursor++) {
|
|
494
|
+
const code = xml.charCodeAt(cursor);
|
|
495
|
+
if (quote) {
|
|
496
|
+
if (code === quote)
|
|
497
|
+
quote = 0;
|
|
498
|
+
} else if (code === 34 || code === 39) {
|
|
499
|
+
quote = code;
|
|
500
|
+
} else if (code === 62) {
|
|
501
|
+
return cursor;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
return -1;
|
|
505
|
+
}
|
|
506
|
+
function isAsyncIterable(input) {
|
|
507
|
+
return typeof input === "object" && input !== null && Symbol.asyncIterator in input;
|
|
508
|
+
}
|
|
509
|
+
function isIterable(input) {
|
|
510
|
+
return typeof input === "object" && input !== null && Symbol.iterator in input;
|
|
511
|
+
}
|
|
512
|
+
function isReadableStream(input) {
|
|
513
|
+
return typeof input === "object" && input !== null && "getReader" in input;
|
|
514
|
+
}
|
|
515
|
+
async function* iterateInput(input) {
|
|
516
|
+
if (typeof input === "string" || input instanceof Uint8Array) {
|
|
517
|
+
yield input;
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
if (isReadableStream(input)) {
|
|
521
|
+
const reader = input.getReader();
|
|
522
|
+
let completed = false;
|
|
523
|
+
try {
|
|
524
|
+
while (true) {
|
|
525
|
+
const result = await reader.read();
|
|
526
|
+
if (result.done) {
|
|
527
|
+
completed = true;
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
yield result.value;
|
|
531
|
+
}
|
|
532
|
+
} finally {
|
|
533
|
+
if (!completed)
|
|
534
|
+
await reader.cancel();
|
|
535
|
+
reader.releaseLock();
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
if (isAsyncIterable(input)) {
|
|
539
|
+
yield* input;
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
if (isIterable(input)) {
|
|
543
|
+
yield* input;
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
throw new TypeError("Sitemap XML input must be a string, Uint8Array, iterable, or ReadableStream");
|
|
547
|
+
}
|
|
548
|
+
async function* decodeInput(input) {
|
|
549
|
+
let decoder = new TextDecoder();
|
|
550
|
+
let decodingBytes = false;
|
|
551
|
+
for await (const chunk of iterateInput(input)) {
|
|
552
|
+
if (typeof chunk === "string") {
|
|
553
|
+
if (decodingBytes) {
|
|
554
|
+
const tail = decoder.decode();
|
|
555
|
+
if (tail)
|
|
556
|
+
yield tail;
|
|
557
|
+
decoder = new TextDecoder();
|
|
558
|
+
decodingBytes = false;
|
|
559
|
+
}
|
|
560
|
+
if (chunk)
|
|
561
|
+
yield chunk;
|
|
562
|
+
} else if (chunk instanceof Uint8Array) {
|
|
563
|
+
decodingBytes = true;
|
|
564
|
+
const text = decoder.decode(chunk, { stream: true });
|
|
565
|
+
if (text)
|
|
566
|
+
yield text;
|
|
567
|
+
} else {
|
|
568
|
+
throw new TypeError("Sitemap XML chunks must be strings or Uint8Array values");
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
if (decodingBytes) {
|
|
572
|
+
const tail = decoder.decode();
|
|
573
|
+
if (tail)
|
|
574
|
+
yield tail;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
function utf8ByteLength(value) {
|
|
578
|
+
let bytes = 0;
|
|
579
|
+
for (let index = 0; index < value.length; index++) {
|
|
580
|
+
const code = value.charCodeAt(index);
|
|
581
|
+
if (code < 128) {
|
|
582
|
+
bytes++;
|
|
583
|
+
} else if (code < 2048) {
|
|
584
|
+
bytes += 2;
|
|
585
|
+
} else if (code >= 55296 && code <= 56319 && index + 1 < value.length) {
|
|
586
|
+
const next = value.charCodeAt(index + 1);
|
|
587
|
+
if (next >= 56320 && next <= 57343) {
|
|
588
|
+
bytes += 4;
|
|
589
|
+
index++;
|
|
590
|
+
} else {
|
|
591
|
+
bytes += 3;
|
|
592
|
+
}
|
|
593
|
+
} else {
|
|
594
|
+
bytes += 3;
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
return bytes;
|
|
598
|
+
}
|
|
599
|
+
function exceedsUtf8ByteLimit(value, limit) {
|
|
600
|
+
if (value.length > limit)
|
|
601
|
+
return true;
|
|
602
|
+
return value.length > Math.floor(limit / 3) && utf8ByteLength(value) > limit;
|
|
603
|
+
}
|
|
604
|
+
function resolveMaxEntryBytes(options) {
|
|
605
|
+
const value = options.maxEntryBytes ?? DEFAULT_MAX_ENTRY_BYTES;
|
|
606
|
+
if (!Number.isSafeInteger(value) || value < 1)
|
|
607
|
+
throw new TypeError("maxEntryBytes must be a positive safe integer");
|
|
608
|
+
return value;
|
|
609
|
+
}
|
|
610
|
+
function resolveMaxBufferBytes(options) {
|
|
611
|
+
const value = options.maxBufferBytes ?? DEFAULT_MAX_ENTRY_BYTES;
|
|
612
|
+
if (!Number.isSafeInteger(value) || value < 1)
|
|
613
|
+
throw new TypeError("maxBufferBytes must be a positive safe integer");
|
|
614
|
+
return value;
|
|
615
|
+
}
|
|
616
|
+
function decodeXmlEntities(value) {
|
|
617
|
+
const firstEntity = value.indexOf("&");
|
|
618
|
+
if (firstEntity === -1)
|
|
619
|
+
return value;
|
|
620
|
+
let decoded = value.slice(0, firstEntity);
|
|
621
|
+
let cursor = firstEntity;
|
|
622
|
+
while (cursor < value.length) {
|
|
623
|
+
if (value.charCodeAt(cursor) !== 38) {
|
|
624
|
+
decoded += value[cursor];
|
|
625
|
+
cursor++;
|
|
626
|
+
continue;
|
|
627
|
+
}
|
|
628
|
+
const end = value.indexOf(";", cursor + 1);
|
|
629
|
+
if (end === -1) {
|
|
630
|
+
decoded += value.slice(cursor);
|
|
631
|
+
break;
|
|
632
|
+
}
|
|
633
|
+
const entity = value.slice(cursor + 1, end);
|
|
634
|
+
if (entity === "amp") {
|
|
635
|
+
decoded += "&";
|
|
636
|
+
} else if (entity === "lt") {
|
|
637
|
+
decoded += "<";
|
|
638
|
+
} else if (entity === "gt") {
|
|
639
|
+
decoded += ">";
|
|
640
|
+
} else if (entity === "quot") {
|
|
641
|
+
decoded += '"';
|
|
642
|
+
} else if (entity === "apos") {
|
|
643
|
+
decoded += "'";
|
|
644
|
+
} else if (entity.charCodeAt(0) === 35) {
|
|
645
|
+
const hex = entity.charCodeAt(1) === 120 || entity.charCodeAt(1) === 88;
|
|
646
|
+
const codePoint = Number.parseInt(entity.slice(hex ? 2 : 1), hex ? 16 : 10);
|
|
647
|
+
decoded += Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 1114111 && !(codePoint >= 55296 && codePoint <= 57343) ? String.fromCodePoint(codePoint) : value.slice(cursor, end + 1);
|
|
648
|
+
} else {
|
|
649
|
+
decoded += value.slice(cursor, end + 1);
|
|
650
|
+
}
|
|
651
|
+
cursor = end + 1;
|
|
652
|
+
}
|
|
653
|
+
return decoded;
|
|
654
|
+
}
|
|
655
|
+
function decodeElementContent(value) {
|
|
656
|
+
const firstMarkup = value.indexOf("<");
|
|
657
|
+
if (firstMarkup === -1)
|
|
658
|
+
return decodeXmlEntities(value.trim());
|
|
659
|
+
let decoded = "";
|
|
660
|
+
let cursor = 0;
|
|
661
|
+
while (cursor < value.length) {
|
|
662
|
+
const markupStart = value.indexOf("<", cursor);
|
|
663
|
+
if (markupStart === -1) {
|
|
664
|
+
decoded += decodeXmlEntities(value.slice(cursor));
|
|
665
|
+
break;
|
|
666
|
+
}
|
|
667
|
+
decoded += decodeXmlEntities(value.slice(cursor, markupStart));
|
|
668
|
+
if (value.startsWith("<![CDATA[", markupStart)) {
|
|
669
|
+
const end = value.indexOf("]]>", markupStart + 9);
|
|
670
|
+
if (end === -1)
|
|
671
|
+
return null;
|
|
672
|
+
decoded += value.slice(markupStart + 9, end);
|
|
673
|
+
cursor = end + 3;
|
|
674
|
+
} else if (value.startsWith("<!--", markupStart)) {
|
|
675
|
+
const end = value.indexOf("-->", markupStart + 4);
|
|
676
|
+
if (end === -1)
|
|
677
|
+
return null;
|
|
678
|
+
cursor = end + 3;
|
|
679
|
+
} else {
|
|
680
|
+
return null;
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
return decoded.trim();
|
|
684
|
+
}
|
|
685
|
+
function parseCommonUrlEntry(xml) {
|
|
686
|
+
if (!xml.startsWith("<url>"))
|
|
687
|
+
return null;
|
|
688
|
+
const parsed = {};
|
|
689
|
+
let seenFields = 0;
|
|
690
|
+
let cursor = 5;
|
|
691
|
+
while (cursor < xml.length) {
|
|
692
|
+
while (cursor < xml.length && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
693
|
+
cursor++;
|
|
694
|
+
if (xml.startsWith("</url>", cursor)) {
|
|
695
|
+
cursor += 6;
|
|
696
|
+
while (cursor < xml.length && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
697
|
+
cursor++;
|
|
698
|
+
return cursor === xml.length ? parsed : null;
|
|
699
|
+
}
|
|
700
|
+
let name;
|
|
701
|
+
let fieldBit;
|
|
702
|
+
if (xml.startsWith("<loc>", cursor)) {
|
|
703
|
+
name = "loc";
|
|
704
|
+
fieldBit = 1;
|
|
705
|
+
} else if (xml.startsWith("<lastmod>", cursor)) {
|
|
706
|
+
name = "lastmod";
|
|
707
|
+
fieldBit = 2;
|
|
708
|
+
} else if (xml.startsWith("<changefreq>", cursor)) {
|
|
709
|
+
name = "changefreq";
|
|
710
|
+
fieldBit = 4;
|
|
711
|
+
} else if (xml.startsWith("<priority>", cursor)) {
|
|
712
|
+
name = "priority";
|
|
713
|
+
fieldBit = 8;
|
|
714
|
+
} else {
|
|
715
|
+
return null;
|
|
716
|
+
}
|
|
717
|
+
if (seenFields & fieldBit)
|
|
718
|
+
return null;
|
|
719
|
+
seenFields |= fieldBit;
|
|
720
|
+
const contentStart = cursor + name.length + 2;
|
|
721
|
+
const closingTag = `</${name}>`;
|
|
722
|
+
const contentEnd = xml.indexOf(closingTag, contentStart);
|
|
723
|
+
if (contentEnd === -1 || xml.indexOf("<", contentStart) !== contentEnd)
|
|
724
|
+
return null;
|
|
725
|
+
parsed[name] = decodeXmlEntities(xml.slice(contentStart, contentEnd).trim());
|
|
726
|
+
cursor = contentEnd + closingTag.length;
|
|
727
|
+
}
|
|
728
|
+
return null;
|
|
729
|
+
}
|
|
730
|
+
function parseSimpleUrlEntry(xml) {
|
|
731
|
+
const parsed = {};
|
|
732
|
+
const seenFields = /* @__PURE__ */ new Set();
|
|
733
|
+
let scanIndex = 0;
|
|
734
|
+
let depth = 0;
|
|
735
|
+
let field;
|
|
736
|
+
let contentStart = 0;
|
|
737
|
+
let closedUrl = false;
|
|
738
|
+
while (true) {
|
|
739
|
+
const markupStart = xml.indexOf("<", scanIndex);
|
|
740
|
+
if (markupStart === -1)
|
|
741
|
+
break;
|
|
742
|
+
const markupEnd = findMarkupEnd(xml, markupStart);
|
|
743
|
+
if (markupEnd === -1)
|
|
744
|
+
return null;
|
|
745
|
+
const tag = parseTag(xml, markupStart, markupEnd);
|
|
746
|
+
scanIndex = markupEnd + 1;
|
|
747
|
+
if (!tag)
|
|
748
|
+
continue;
|
|
749
|
+
if (!tag.closing) {
|
|
750
|
+
if (depth === 0 && tag.name === "url") {
|
|
751
|
+
if (tag.selfClosing)
|
|
752
|
+
return {};
|
|
753
|
+
depth = 1;
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
if (depth !== 1 || !SIMPLE_URL_FIELDS.has(tag.name) || seenFields.has(tag.name))
|
|
757
|
+
return null;
|
|
758
|
+
seenFields.add(tag.name);
|
|
759
|
+
if (tag.selfClosing) {
|
|
760
|
+
parsed[tag.name] = "";
|
|
761
|
+
continue;
|
|
762
|
+
}
|
|
763
|
+
field = tag.name;
|
|
764
|
+
contentStart = markupEnd + 1;
|
|
765
|
+
depth = 2;
|
|
766
|
+
continue;
|
|
767
|
+
}
|
|
768
|
+
if (depth === 2 && field === tag.name) {
|
|
769
|
+
const value = decodeElementContent(xml.slice(contentStart, markupStart));
|
|
770
|
+
if (value === null)
|
|
771
|
+
return null;
|
|
772
|
+
parsed[field] = value;
|
|
773
|
+
field = void 0;
|
|
774
|
+
depth = 1;
|
|
775
|
+
} else if (depth === 1 && tag.name === "url") {
|
|
776
|
+
closedUrl = true;
|
|
777
|
+
depth = 0;
|
|
778
|
+
} else {
|
|
779
|
+
return null;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
return closedUrl && depth === 0 ? parsed : null;
|
|
783
|
+
}
|
|
784
|
+
function parseUrlEntry(parser, xml) {
|
|
785
|
+
const simple = parseCommonUrlEntry(xml) ?? parseSimpleUrlEntry(xml);
|
|
786
|
+
if (simple)
|
|
787
|
+
return simple;
|
|
396
788
|
try {
|
|
397
789
|
const parsed = parser.parse(xml);
|
|
398
|
-
|
|
399
|
-
|
|
790
|
+
const value = Array.isArray(parsed?.url) ? parsed.url[0] : parsed?.url;
|
|
791
|
+
return value && typeof value === "object" ? value : {};
|
|
792
|
+
} catch (error) {
|
|
793
|
+
throw new Error(`Failed to parse XML: ${error instanceof Error ? error.message : String(error)}`);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
function parseSitemapEntry(parser, xml, warnings) {
|
|
797
|
+
let parsed;
|
|
798
|
+
try {
|
|
799
|
+
parsed = parser.parse(xml);
|
|
800
|
+
} catch (error) {
|
|
801
|
+
throw new Error(`Failed to parse XML: ${error instanceof Error ? error.message : String(error)}`);
|
|
802
|
+
}
|
|
803
|
+
const loc = typeof parsed.sitemap?.loc === "string" ? parsed.sitemap.loc.trim() : "";
|
|
804
|
+
if (!loc) {
|
|
805
|
+
warnings.push({
|
|
806
|
+
type: "validation",
|
|
807
|
+
message: "Sitemap entry missing required loc element"
|
|
808
|
+
});
|
|
809
|
+
return null;
|
|
810
|
+
}
|
|
811
|
+
if (!URL.canParse(loc)) {
|
|
812
|
+
warnings.push({
|
|
813
|
+
type: "validation",
|
|
814
|
+
message: "Sitemap entry has invalid URL",
|
|
815
|
+
context: { url: loc }
|
|
816
|
+
});
|
|
817
|
+
return null;
|
|
818
|
+
}
|
|
819
|
+
const entry = { loc };
|
|
820
|
+
if (typeof parsed.sitemap?.lastmod === "string" && parsed.sitemap.lastmod.trim())
|
|
821
|
+
entry.lastmod = parsed.sitemap.lastmod.trim();
|
|
822
|
+
return entry;
|
|
823
|
+
}
|
|
824
|
+
async function* parseSitemapStream(input, options = {}) {
|
|
825
|
+
const maxEntryBytes = resolveMaxEntryBytes(options);
|
|
826
|
+
const maxBufferBytes = resolveMaxBufferBytes(options);
|
|
827
|
+
const parser = createUrlParser();
|
|
828
|
+
let buffer = "";
|
|
829
|
+
let scanIndex = 0;
|
|
830
|
+
let entryStart = -1;
|
|
831
|
+
let sawInput = false;
|
|
832
|
+
let kind;
|
|
833
|
+
let insideRoot = false;
|
|
834
|
+
let closedRoot = false;
|
|
835
|
+
let entryCount = 0;
|
|
836
|
+
let validUrlCount = 0;
|
|
837
|
+
for await (const chunk of decodeInput(input)) {
|
|
838
|
+
sawInput = true;
|
|
839
|
+
buffer += chunk;
|
|
840
|
+
while (true) {
|
|
841
|
+
const markupStart = buffer.indexOf("<", scanIndex);
|
|
842
|
+
if (markupStart === -1) {
|
|
843
|
+
scanIndex = buffer.length;
|
|
844
|
+
break;
|
|
845
|
+
}
|
|
846
|
+
const markupEnd = findMarkupEnd(buffer, markupStart);
|
|
847
|
+
if (markupEnd === -1) {
|
|
848
|
+
scanIndex = markupStart;
|
|
849
|
+
break;
|
|
850
|
+
}
|
|
851
|
+
const tag = parseBoundaryTag(buffer, markupStart, markupEnd);
|
|
852
|
+
if (!kind && tag === 0) {
|
|
853
|
+
const root = parseTag(buffer, markupStart, markupEnd);
|
|
854
|
+
if (root && !root.closing)
|
|
855
|
+
throw new Error("XML does not contain a valid sitemap element");
|
|
856
|
+
}
|
|
857
|
+
let record;
|
|
858
|
+
if (!kind && (tag === URLSET_OPEN || tag === URLSET_SELF_CLOSING)) {
|
|
859
|
+
kind = "urlset";
|
|
860
|
+
insideRoot = tag === URLSET_OPEN;
|
|
861
|
+
closedRoot = tag === URLSET_SELF_CLOSING;
|
|
862
|
+
yield { _tag: "kind", kind };
|
|
863
|
+
} else if (!kind && (tag === SITEMAP_INDEX_OPEN || tag === SITEMAP_INDEX_SELF_CLOSING)) {
|
|
864
|
+
kind = "index";
|
|
865
|
+
insideRoot = tag === SITEMAP_INDEX_OPEN;
|
|
866
|
+
closedRoot = tag === SITEMAP_INDEX_SELF_CLOSING;
|
|
867
|
+
yield { _tag: "kind", kind };
|
|
868
|
+
} else if (entryStart === -1 && (kind === "urlset" && tag === URLSET_CLOSE || kind === "index" && tag === SITEMAP_INDEX_CLOSE)) {
|
|
869
|
+
insideRoot = false;
|
|
870
|
+
closedRoot = true;
|
|
871
|
+
}
|
|
872
|
+
const openingEntry = kind === "urlset" ? tag === URL_OPEN || tag === URL_SELF_CLOSING : tag === SITEMAP_OPEN || tag === SITEMAP_SELF_CLOSING;
|
|
873
|
+
const closingEntry = kind === "urlset" ? tag === URL_CLOSE : tag === SITEMAP_CLOSE;
|
|
874
|
+
const selfClosingEntry = tag === URL_SELF_CLOSING || tag === SITEMAP_SELF_CLOSING;
|
|
875
|
+
if (insideRoot && openingEntry && entryStart === -1) {
|
|
876
|
+
entryStart = markupStart;
|
|
877
|
+
if (selfClosingEntry)
|
|
878
|
+
record = buffer.slice(entryStart, markupEnd + 1);
|
|
879
|
+
} else if (closingEntry && entryStart !== -1) {
|
|
880
|
+
record = buffer.slice(entryStart, markupEnd + 1);
|
|
881
|
+
}
|
|
882
|
+
if (!record) {
|
|
883
|
+
scanIndex = markupEnd + 1;
|
|
884
|
+
continue;
|
|
885
|
+
}
|
|
886
|
+
if (exceedsUtf8ByteLimit(record, maxEntryBytes))
|
|
887
|
+
throw new Error(`Sitemap entry exceeds maxEntryBytes of ${maxEntryBytes}`);
|
|
888
|
+
entryCount++;
|
|
889
|
+
const warnings = [];
|
|
890
|
+
scanIndex = markupEnd + 1;
|
|
891
|
+
entryStart = -1;
|
|
892
|
+
const entry = kind === "urlset" ? extractUrlFromParsedElement(parseUrlEntry(parser, record), warnings) : parseSitemapEntry(parser, record, warnings);
|
|
893
|
+
for (const warning of warnings)
|
|
894
|
+
yield { _tag: "warning", warning };
|
|
895
|
+
if (kind === "urlset" && entry) {
|
|
896
|
+
validUrlCount++;
|
|
897
|
+
yield { _tag: "url", url: entry };
|
|
898
|
+
} else if (kind === "index" && entry) {
|
|
899
|
+
yield { _tag: "sitemap", sitemap: entry };
|
|
900
|
+
}
|
|
400
901
|
}
|
|
401
|
-
if (
|
|
402
|
-
|
|
902
|
+
if (entryStart !== -1) {
|
|
903
|
+
if (buffer.length - entryStart > maxEntryBytes)
|
|
904
|
+
throw new Error(`Sitemap entry exceeds maxEntryBytes of ${maxEntryBytes}`);
|
|
905
|
+
if (entryStart > 0) {
|
|
906
|
+
buffer = buffer.slice(entryStart);
|
|
907
|
+
scanIndex -= entryStart;
|
|
908
|
+
entryStart = 0;
|
|
909
|
+
}
|
|
910
|
+
} else if (scanIndex > 0) {
|
|
911
|
+
buffer = buffer.slice(scanIndex);
|
|
912
|
+
scanIndex = 0;
|
|
403
913
|
}
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
914
|
+
if (entryStart === -1 && exceedsUtf8ByteLimit(buffer, maxBufferBytes))
|
|
915
|
+
throw new Error(`Sitemap XML buffer exceeds maxBufferBytes of ${maxBufferBytes}`);
|
|
916
|
+
}
|
|
917
|
+
if (!sawInput)
|
|
918
|
+
throw new Error("Empty XML input provided");
|
|
919
|
+
if (entryStart !== -1)
|
|
920
|
+
throw new Error(`Failed to parse XML: Unclosed ${kind === "index" ? "sitemap" : "url"} element`);
|
|
921
|
+
if (!kind)
|
|
922
|
+
throw new Error("XML does not contain a valid sitemap element");
|
|
923
|
+
if (!closedRoot)
|
|
924
|
+
throw new Error(`Failed to parse XML: Unclosed ${kind === "index" ? "sitemapindex" : "urlset"} element`);
|
|
925
|
+
if (kind === "urlset" && entryCount > 0 && validUrlCount === 0) {
|
|
926
|
+
yield {
|
|
927
|
+
_tag: "warning",
|
|
928
|
+
warning: {
|
|
408
929
|
type: "validation",
|
|
409
930
|
message: "No valid URLs found in sitemap after validation"
|
|
410
|
-
}
|
|
931
|
+
}
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
async function* parseSitemapXmlStream(input, options = {}) {
|
|
936
|
+
for await (const event of parseSitemapStream(input, options)) {
|
|
937
|
+
if (event._tag === "kind") {
|
|
938
|
+
if (event.kind !== "urlset")
|
|
939
|
+
throw new Error("XML does not contain a valid urlset element");
|
|
940
|
+
continue;
|
|
411
941
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
942
|
+
if (event._tag === "sitemap")
|
|
943
|
+
continue;
|
|
944
|
+
yield event;
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
async function* parseSitemapIndexStream(input, options = {}) {
|
|
948
|
+
for await (const event of parseSitemapStream(input, options)) {
|
|
949
|
+
if (event._tag === "kind") {
|
|
950
|
+
if (event.kind !== "index")
|
|
951
|
+
throw new Error("XML does not contain a valid sitemapindex element");
|
|
952
|
+
continue;
|
|
416
953
|
}
|
|
417
|
-
|
|
954
|
+
if (event._tag === "url")
|
|
955
|
+
continue;
|
|
956
|
+
yield event;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
async function parseSitemapXml(xml) {
|
|
960
|
+
if (!xml)
|
|
961
|
+
throw new Error("Empty XML input provided");
|
|
962
|
+
const urls = [];
|
|
963
|
+
const warnings = [];
|
|
964
|
+
const parser = createUrlParser();
|
|
965
|
+
let scanIndex = 0;
|
|
966
|
+
let entryStart = -1;
|
|
967
|
+
let sawUrlset = false;
|
|
968
|
+
let insideUrlset = false;
|
|
969
|
+
let closedUrlset = false;
|
|
970
|
+
let entryCount = 0;
|
|
971
|
+
while (true) {
|
|
972
|
+
const markupStart = xml.indexOf("<", scanIndex);
|
|
973
|
+
if (markupStart === -1)
|
|
974
|
+
break;
|
|
975
|
+
const markupEnd = findMarkupEnd(xml, markupStart);
|
|
976
|
+
if (markupEnd === -1)
|
|
977
|
+
break;
|
|
978
|
+
const tag = parseBoundaryTag(xml, markupStart, markupEnd);
|
|
979
|
+
let record;
|
|
980
|
+
if (tag === URLSET_OPEN) {
|
|
981
|
+
sawUrlset = true;
|
|
982
|
+
insideUrlset = true;
|
|
983
|
+
} else if (tag === URLSET_SELF_CLOSING) {
|
|
984
|
+
sawUrlset = true;
|
|
985
|
+
insideUrlset = false;
|
|
986
|
+
closedUrlset = true;
|
|
987
|
+
} else if (tag === URLSET_CLOSE && entryStart === -1) {
|
|
988
|
+
insideUrlset = false;
|
|
989
|
+
closedUrlset = true;
|
|
990
|
+
}
|
|
991
|
+
if (insideUrlset && (tag === URL_OPEN || tag === URL_SELF_CLOSING) && entryStart === -1) {
|
|
992
|
+
entryStart = markupStart;
|
|
993
|
+
if (tag === URL_SELF_CLOSING)
|
|
994
|
+
record = xml.slice(entryStart, markupEnd + 1);
|
|
995
|
+
} else if (tag === URL_CLOSE && entryStart !== -1) {
|
|
996
|
+
record = xml.slice(entryStart, markupEnd + 1);
|
|
997
|
+
}
|
|
998
|
+
scanIndex = markupEnd + 1;
|
|
999
|
+
if (!record)
|
|
1000
|
+
continue;
|
|
1001
|
+
if (exceedsUtf8ByteLimit(record, DEFAULT_MAX_ENTRY_BYTES))
|
|
1002
|
+
throw new Error(`Sitemap URL entry exceeds maxEntryBytes of ${DEFAULT_MAX_ENTRY_BYTES}`);
|
|
1003
|
+
entryCount++;
|
|
1004
|
+
entryStart = -1;
|
|
1005
|
+
const url = extractUrlFromParsedElement(parseUrlEntry(parser, record), warnings);
|
|
1006
|
+
if (url)
|
|
1007
|
+
urls.push(url);
|
|
1008
|
+
}
|
|
1009
|
+
if (entryStart !== -1)
|
|
1010
|
+
throw new Error("Failed to parse XML: Unclosed url element");
|
|
1011
|
+
if (!sawUrlset)
|
|
1012
|
+
throw new Error("XML does not contain a valid urlset element");
|
|
1013
|
+
if (!closedUrlset)
|
|
1014
|
+
throw new Error("Failed to parse XML: Unclosed urlset element");
|
|
1015
|
+
if (entryCount > 0 && urls.length === 0) {
|
|
1016
|
+
warnings.push({
|
|
1017
|
+
type: "validation",
|
|
1018
|
+
message: "No valid URLs found in sitemap after validation"
|
|
1019
|
+
});
|
|
418
1020
|
}
|
|
1021
|
+
return { urls, warnings };
|
|
419
1022
|
}
|
|
420
1023
|
|
|
421
|
-
export { isSitemapIndex, parseSitemapIndex, parseSitemapXml };
|
|
1024
|
+
export { isSitemapIndex, parseSitemapIndex, parseSitemapIndexStream, parseSitemapStream, parseSitemapXml, parseSitemapXmlStream };
|