@nuxtjs/sitemap 8.3.0 → 8.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content.mjs +11 -8
- package/dist/devtools/lib/sitemap/state.ts +4 -1
- package/dist/devtools/pages/sitemap/index.vue +2 -2
- package/dist/module.json +2 -2
- package/dist/module.mjs +21 -42
- package/dist/runtime/server/routes/__sitemap__/debug-production.d.ts +3 -3
- package/dist/runtime/server/routes/__sitemap__/debug-production.js +43 -27
- package/dist/runtime/server/sitemap/builder/index-xml.js +13 -16
- package/dist/runtime/server/sitemap/builder/sitemap-index.js +13 -26
- package/dist/runtime/server/sitemap/builder/xml.d.ts +2 -2
- package/dist/runtime/server/sitemap/builder/xml.js +16 -19
- package/dist/runtime/server/sitemap/event-handlers.js +3 -16
- package/dist/runtime/server/sitemap/nitro.d.ts +1 -0
- package/dist/runtime/server/sitemap/nitro.js +20 -17
- package/dist/runtime/server/sitemap/urlset/normalise.js +11 -2
- package/dist/runtime/server/sitemap/urlset/sources.js +48 -7
- package/dist/runtime/utils-pure.js +2 -7
- package/dist/utils.d.mts +58 -19
- package/dist/utils.d.ts +58 -19
- package/dist/utils.mjs +171 -955
- package/package.json +25 -27
- package/dist/runtime/server/kit.d.ts +0 -2
- package/dist/runtime/server/kit.js +0 -30
package/dist/utils.mjs
CHANGED
|
@@ -1,928 +1,182 @@
|
|
|
1
|
+
import { parseSitemap } from 'sitemapd/parse';
|
|
2
|
+
export { collectSitemap, parseSitemap } from 'sitemapd/parse';
|
|
1
3
|
export { p as parseHtmlExtractSitemapMeta } from './shared/sitemap.BoMnWHOt.mjs';
|
|
2
|
-
|
|
4
|
+
export { createSitemapReader } from 'sitemapd';
|
|
5
|
+
export { createFetchDocumentLoader } from 'sitemapd/fetch';
|
|
3
6
|
import 'ufo';
|
|
4
7
|
import 'ultrahtml';
|
|
5
8
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const sitemaps = Array.isArray(parsed.sitemapindex.sitemap) ? parsed.sitemapindex.sitemap : [parsed.sitemapindex.sitemap];
|
|
23
|
-
const warnings = [];
|
|
24
|
-
const entries = [];
|
|
25
|
-
for (const s of sitemaps) {
|
|
26
|
-
if (typeof s.loc !== "string" || !s.loc.trim().length) {
|
|
27
|
-
warnings.push({
|
|
28
|
-
type: "validation",
|
|
29
|
-
message: "Sitemap entry missing required loc element"
|
|
30
|
-
});
|
|
31
|
-
continue;
|
|
32
|
-
}
|
|
33
|
-
const loc = s.loc.trim();
|
|
34
|
-
if (!isValidUrl(loc)) {
|
|
35
|
-
warnings.push({
|
|
36
|
-
type: "validation",
|
|
37
|
-
message: "Sitemap entry has invalid URL",
|
|
38
|
-
context: { url: loc }
|
|
39
|
-
});
|
|
40
|
-
continue;
|
|
41
|
-
}
|
|
42
|
-
entries.push({
|
|
43
|
-
loc,
|
|
44
|
-
...s.lastmod && { lastmod: s.lastmod.trim() }
|
|
45
|
-
});
|
|
9
|
+
const CHANGE_FREQUENCIES = /* @__PURE__ */ new Set([
|
|
10
|
+
"always",
|
|
11
|
+
"hourly",
|
|
12
|
+
"daily",
|
|
13
|
+
"weekly",
|
|
14
|
+
"monthly",
|
|
15
|
+
"yearly",
|
|
16
|
+
"never"
|
|
17
|
+
]);
|
|
18
|
+
function legacyWarning(issue, kind) {
|
|
19
|
+
if (issue.code === "missing_loc") {
|
|
20
|
+
return {
|
|
21
|
+
type: "validation",
|
|
22
|
+
message: kind === "index" ? "Sitemap entry missing required loc element" : "URL entry missing required loc element",
|
|
23
|
+
...kind === "urlset" ? { context: { url: "undefined" } } : {}
|
|
24
|
+
};
|
|
46
25
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
const SIMPLE_URL_FIELDS = /* @__PURE__ */ new Set(["loc", "lastmod", "changefreq", "priority"]);
|
|
57
|
-
function createUrlParser() {
|
|
58
|
-
return new XMLParser({
|
|
59
|
-
isArray: (tagName) => ARRAY_TAGS.has(tagName),
|
|
60
|
-
removeNSPrefix: true,
|
|
61
|
-
parseAttributeValue: false,
|
|
62
|
-
ignoreAttributes: false,
|
|
63
|
-
attributeNamePrefix: "",
|
|
64
|
-
trimValues: true
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
function isValidString(value) {
|
|
68
|
-
return typeof value === "string" && value.trim().length > 0;
|
|
69
|
-
}
|
|
70
|
-
function parseNumber(value) {
|
|
71
|
-
if (typeof value === "number")
|
|
72
|
-
return value;
|
|
73
|
-
if (typeof value === "string" && value.trim()) {
|
|
74
|
-
const num = Number.parseFloat(value.trim());
|
|
75
|
-
return Number.isNaN(num) ? void 0 : num;
|
|
26
|
+
if (issue.code === "invalid_loc" && kind === "index") {
|
|
27
|
+
const url = String(issue.value);
|
|
28
|
+
if (URL.canParse(url))
|
|
29
|
+
return void 0;
|
|
30
|
+
return {
|
|
31
|
+
type: "validation",
|
|
32
|
+
message: "Sitemap entry has invalid URL",
|
|
33
|
+
context: { url }
|
|
34
|
+
};
|
|
76
35
|
}
|
|
77
|
-
|
|
36
|
+
if (issue.severity !== "warning" || issue.code === "invalid_loc")
|
|
37
|
+
return void 0;
|
|
38
|
+
return {
|
|
39
|
+
type: "validation",
|
|
40
|
+
message: issue.message,
|
|
41
|
+
...issue.field || issue.value !== void 0 ? {
|
|
42
|
+
context: {
|
|
43
|
+
...issue.field ? { field: issue.field } : {},
|
|
44
|
+
...issue.value !== void 0 ? { value: issue.value } : {}
|
|
45
|
+
}
|
|
46
|
+
} : {}
|
|
47
|
+
};
|
|
78
48
|
}
|
|
79
|
-
function
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
49
|
+
function legacyImages(images) {
|
|
50
|
+
return images?.map((image) => ({
|
|
51
|
+
loc: image.loc,
|
|
52
|
+
...image.caption ? { caption: image.caption } : {},
|
|
53
|
+
...image.geoLocation ? { geo_location: image.geoLocation } : {},
|
|
54
|
+
...image.title ? { title: image.title } : {},
|
|
55
|
+
...image.license ? { license: image.license } : {}
|
|
56
|
+
}));
|
|
87
57
|
}
|
|
88
|
-
function
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
}
|
|
97
|
-
const urlObj = { loc: urlElement.loc };
|
|
98
|
-
if (isValidString(urlElement.lastmod)) {
|
|
99
|
-
urlObj.lastmod = urlElement.lastmod;
|
|
100
|
-
}
|
|
101
|
-
if (isValidString(urlElement.changefreq)) {
|
|
102
|
-
if (CHANGE_FREQUENCIES.has(urlElement.changefreq)) {
|
|
103
|
-
urlObj.changefreq = urlElement.changefreq;
|
|
58
|
+
function legacyUrl(entry) {
|
|
59
|
+
const warnings = [];
|
|
60
|
+
const url = { loc: entry.loc };
|
|
61
|
+
if (entry.lastmod)
|
|
62
|
+
url.lastmod = entry.lastmod;
|
|
63
|
+
if (entry.changefreq) {
|
|
64
|
+
if (CHANGE_FREQUENCIES.has(entry.changefreq)) {
|
|
65
|
+
url.changefreq = entry.changefreq;
|
|
104
66
|
} else {
|
|
105
67
|
warnings.push({
|
|
106
68
|
type: "validation",
|
|
107
69
|
message: "Invalid changefreq value",
|
|
108
|
-
context: { url:
|
|
70
|
+
context: { url: entry.loc, field: "changefreq", value: entry.changefreq }
|
|
109
71
|
});
|
|
110
72
|
}
|
|
111
73
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
if (priority
|
|
74
|
+
if (entry.priority !== void 0) {
|
|
75
|
+
const priority = Number.parseFloat(entry.priority);
|
|
76
|
+
if (Number.isNaN(priority)) {
|
|
115
77
|
warnings.push({
|
|
116
78
|
type: "validation",
|
|
117
|
-
message: "
|
|
118
|
-
context: { url:
|
|
79
|
+
message: "Invalid priority value",
|
|
80
|
+
context: { url: entry.loc, field: "priority", value: entry.priority }
|
|
119
81
|
});
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
} else if (urlElement.priority !== void 0) {
|
|
123
|
-
warnings.push({
|
|
124
|
-
type: "validation",
|
|
125
|
-
message: "Invalid priority value",
|
|
126
|
-
context: { url: urlElement.loc, field: "priority", value: urlElement.priority }
|
|
127
|
-
});
|
|
128
|
-
}
|
|
129
|
-
if (urlElement.image) {
|
|
130
|
-
const images = Array.isArray(urlElement.image) ? urlElement.image : [urlElement.image];
|
|
131
|
-
const validImages = [];
|
|
132
|
-
for (const image of images) {
|
|
133
|
-
if (isValidString(image.loc)) {
|
|
134
|
-
validImages.push({ loc: image.loc });
|
|
135
|
-
} else {
|
|
136
|
-
warnings.push({
|
|
137
|
-
type: "validation",
|
|
138
|
-
message: "Image missing required loc element",
|
|
139
|
-
context: { url: urlElement.loc, field: "image.loc" }
|
|
140
|
-
});
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
if (validImages.length > 0)
|
|
144
|
-
urlObj.images = validImages;
|
|
145
|
-
}
|
|
146
|
-
if (urlElement.video) {
|
|
147
|
-
const videos = Array.isArray(urlElement.video) ? urlElement.video : [urlElement.video];
|
|
148
|
-
const validVideos = videos.map((video) => {
|
|
149
|
-
const missingFields = [];
|
|
150
|
-
if (!isValidString(video.title))
|
|
151
|
-
missingFields.push("title");
|
|
152
|
-
if (!isValidString(video.thumbnail_loc))
|
|
153
|
-
missingFields.push("thumbnail_loc");
|
|
154
|
-
if (!isValidString(video.description))
|
|
155
|
-
missingFields.push("description");
|
|
156
|
-
if (!isValidString(video.content_loc))
|
|
157
|
-
missingFields.push("content_loc");
|
|
158
|
-
if (missingFields.length > 0) {
|
|
159
|
-
warnings.push({
|
|
160
|
-
type: "validation",
|
|
161
|
-
message: `Video missing required fields: ${missingFields.join(", ")}`,
|
|
162
|
-
context: { url: urlElement.loc, field: "video" }
|
|
163
|
-
});
|
|
164
|
-
return null;
|
|
165
|
-
}
|
|
166
|
-
const videoObj = {
|
|
167
|
-
title: video.title,
|
|
168
|
-
thumbnail_loc: video.thumbnail_loc,
|
|
169
|
-
description: video.description,
|
|
170
|
-
content_loc: video.content_loc
|
|
171
|
-
};
|
|
172
|
-
if (isValidString(video.player_loc)) {
|
|
173
|
-
videoObj.player_loc = video.player_loc;
|
|
174
|
-
}
|
|
175
|
-
const duration = parseInteger(video.duration);
|
|
176
|
-
if (duration !== void 0) {
|
|
177
|
-
videoObj.duration = duration;
|
|
178
|
-
} else if (video.duration !== void 0) {
|
|
179
|
-
warnings.push({
|
|
180
|
-
type: "validation",
|
|
181
|
-
message: "Invalid video duration value",
|
|
182
|
-
context: { url: urlElement.loc, field: "video.duration", value: video.duration }
|
|
183
|
-
});
|
|
184
|
-
}
|
|
185
|
-
if (isValidString(video.expiration_date)) {
|
|
186
|
-
videoObj.expiration_date = video.expiration_date;
|
|
187
|
-
}
|
|
188
|
-
const rating = parseNumber(video.rating);
|
|
189
|
-
if (rating !== void 0) {
|
|
190
|
-
if (rating < 0 || rating > 5) {
|
|
191
|
-
warnings.push({
|
|
192
|
-
type: "validation",
|
|
193
|
-
message: "Video rating should be between 0.0 and 5.0",
|
|
194
|
-
context: { url: urlElement.loc, field: "video.rating", value: rating }
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
videoObj.rating = rating;
|
|
198
|
-
} else if (video.rating !== void 0) {
|
|
199
|
-
warnings.push({
|
|
200
|
-
type: "validation",
|
|
201
|
-
message: "Invalid video rating value",
|
|
202
|
-
context: { url: urlElement.loc, field: "video.rating", value: video.rating }
|
|
203
|
-
});
|
|
204
|
-
}
|
|
205
|
-
const viewCount = parseInteger(video.view_count);
|
|
206
|
-
if (viewCount !== void 0) {
|
|
207
|
-
videoObj.view_count = viewCount;
|
|
208
|
-
} else if (video.view_count !== void 0) {
|
|
209
|
-
warnings.push({
|
|
210
|
-
type: "validation",
|
|
211
|
-
message: "Invalid video view_count value",
|
|
212
|
-
context: { url: urlElement.loc, field: "video.view_count", value: video.view_count }
|
|
213
|
-
});
|
|
214
|
-
}
|
|
215
|
-
if (isValidString(video.publication_date)) {
|
|
216
|
-
videoObj.publication_date = video.publication_date;
|
|
217
|
-
}
|
|
218
|
-
if (isValidString(video.family_friendly)) {
|
|
219
|
-
const validValues = ["yes", "no"];
|
|
220
|
-
if (validValues.includes(video.family_friendly)) {
|
|
221
|
-
videoObj.family_friendly = video.family_friendly;
|
|
222
|
-
} else {
|
|
223
|
-
warnings.push({
|
|
224
|
-
type: "validation",
|
|
225
|
-
message: 'Invalid video family_friendly value, should be "yes" or "no"',
|
|
226
|
-
context: { url: urlElement.loc, field: "video.family_friendly", value: video.family_friendly }
|
|
227
|
-
});
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
if (isValidString(video.requires_subscription)) {
|
|
231
|
-
const validValues = ["yes", "no"];
|
|
232
|
-
if (validValues.includes(video.requires_subscription)) {
|
|
233
|
-
videoObj.requires_subscription = video.requires_subscription;
|
|
234
|
-
} else {
|
|
235
|
-
warnings.push({
|
|
236
|
-
type: "validation",
|
|
237
|
-
message: 'Invalid video requires_subscription value, should be "yes" or "no"',
|
|
238
|
-
context: { url: urlElement.loc, field: "video.requires_subscription", value: video.requires_subscription }
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
if (isValidString(video.live)) {
|
|
243
|
-
const validValues = ["yes", "no"];
|
|
244
|
-
if (validValues.includes(video.live)) {
|
|
245
|
-
videoObj.live = video.live;
|
|
246
|
-
} else {
|
|
247
|
-
warnings.push({
|
|
248
|
-
type: "validation",
|
|
249
|
-
message: 'Invalid video live value, should be "yes" or "no"',
|
|
250
|
-
context: { url: urlElement.loc, field: "video.live", value: video.live }
|
|
251
|
-
});
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
if (video.restriction && typeof video.restriction === "object") {
|
|
255
|
-
const restriction = video.restriction;
|
|
256
|
-
if (isValidString(restriction.relationship) && isValidString(restriction["#text"])) {
|
|
257
|
-
const validRelationships = ["allow", "deny"];
|
|
258
|
-
if (validRelationships.includes(restriction.relationship)) {
|
|
259
|
-
videoObj.restriction = {
|
|
260
|
-
relationship: restriction.relationship,
|
|
261
|
-
restriction: restriction["#text"]
|
|
262
|
-
};
|
|
263
|
-
} else {
|
|
264
|
-
warnings.push({
|
|
265
|
-
type: "validation",
|
|
266
|
-
message: 'Invalid video restriction relationship, should be "allow" or "deny"',
|
|
267
|
-
context: { url: urlElement.loc, field: "video.restriction.relationship", value: restriction.relationship }
|
|
268
|
-
});
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
if (video.platform && typeof video.platform === "object") {
|
|
273
|
-
const platform = video.platform;
|
|
274
|
-
if (isValidString(platform.relationship) && isValidString(platform["#text"])) {
|
|
275
|
-
const validRelationships = ["allow", "deny"];
|
|
276
|
-
if (validRelationships.includes(platform.relationship)) {
|
|
277
|
-
videoObj.platform = {
|
|
278
|
-
relationship: platform.relationship,
|
|
279
|
-
platform: platform["#text"]
|
|
280
|
-
};
|
|
281
|
-
} else {
|
|
282
|
-
warnings.push({
|
|
283
|
-
type: "validation",
|
|
284
|
-
message: 'Invalid video platform relationship, should be "allow" or "deny"',
|
|
285
|
-
context: { url: urlElement.loc, field: "video.platform.relationship", value: platform.relationship }
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
if (video.price) {
|
|
291
|
-
const prices = Array.isArray(video.price) ? video.price : [video.price];
|
|
292
|
-
const validPrices = prices.map((price) => {
|
|
293
|
-
const priceValue = price["#text"];
|
|
294
|
-
if (priceValue == null || typeof priceValue !== "string" && typeof priceValue !== "number") {
|
|
295
|
-
warnings.push({
|
|
296
|
-
type: "validation",
|
|
297
|
-
message: "Video price missing value",
|
|
298
|
-
context: { url: urlElement.loc, field: "video.price" }
|
|
299
|
-
});
|
|
300
|
-
return null;
|
|
301
|
-
}
|
|
302
|
-
const validTypes = ["rent", "purchase", "package", "subscription"];
|
|
303
|
-
if (price.type && !validTypes.includes(price.type)) {
|
|
304
|
-
warnings.push({
|
|
305
|
-
type: "validation",
|
|
306
|
-
message: `Invalid video price type "${price.type}", should be one of: ${validTypes.join(", ")}`,
|
|
307
|
-
context: { url: urlElement.loc, field: "video.price.type", value: price.type }
|
|
308
|
-
});
|
|
309
|
-
}
|
|
310
|
-
return {
|
|
311
|
-
price: String(priceValue),
|
|
312
|
-
currency: price.currency,
|
|
313
|
-
type: price.type
|
|
314
|
-
};
|
|
315
|
-
}).filter((p) => p !== null);
|
|
316
|
-
if (validPrices.length > 0) {
|
|
317
|
-
videoObj.price = validPrices;
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
if (video.uploader && typeof video.uploader === "object") {
|
|
321
|
-
const uploader = video.uploader;
|
|
322
|
-
if (isValidString(uploader.info) && isValidString(uploader["#text"])) {
|
|
323
|
-
videoObj.uploader = {
|
|
324
|
-
uploader: uploader["#text"],
|
|
325
|
-
info: uploader.info
|
|
326
|
-
};
|
|
327
|
-
} else {
|
|
328
|
-
warnings.push({
|
|
329
|
-
type: "validation",
|
|
330
|
-
message: "Video uploader missing required info or name",
|
|
331
|
-
context: { url: urlElement.loc, field: "video.uploader" }
|
|
332
|
-
});
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
if (video.tag) {
|
|
336
|
-
const tags = Array.isArray(video.tag) ? video.tag : [video.tag];
|
|
337
|
-
const validTags = tags.filter(isValidString);
|
|
338
|
-
if (validTags.length > 0) {
|
|
339
|
-
videoObj.tag = validTags;
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
return videoObj;
|
|
343
|
-
}).filter((video) => video !== null);
|
|
344
|
-
if (validVideos.length > 0) {
|
|
345
|
-
urlObj.videos = validVideos;
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
if (urlElement.link) {
|
|
349
|
-
const links = Array.isArray(urlElement.link) ? urlElement.link : [urlElement.link];
|
|
350
|
-
const alternatives = links.map((link) => {
|
|
351
|
-
if (link.rel === "alternate" && isValidString(link.hreflang) && isValidString(link.href)) {
|
|
352
|
-
return {
|
|
353
|
-
hreflang: link.hreflang,
|
|
354
|
-
href: link.href
|
|
355
|
-
};
|
|
356
|
-
} else {
|
|
82
|
+
} else {
|
|
83
|
+
if (priority < 0 || priority > 1) {
|
|
357
84
|
warnings.push({
|
|
358
85
|
type: "validation",
|
|
359
|
-
message:
|
|
360
|
-
context: { url:
|
|
86
|
+
message: "Priority value should be between 0.0 and 1.0, clamping to valid range",
|
|
87
|
+
context: { url: entry.loc, field: "priority", value: priority }
|
|
361
88
|
});
|
|
362
|
-
return null;
|
|
363
89
|
}
|
|
364
|
-
|
|
365
|
-
if (alternatives.length > 0) {
|
|
366
|
-
urlObj.alternatives = alternatives;
|
|
90
|
+
url.priority = Math.max(0, Math.min(1, priority));
|
|
367
91
|
}
|
|
368
92
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
publication_date: news.publication_date,
|
|
375
|
-
publication: {
|
|
376
|
-
name: news.publication.name,
|
|
377
|
-
language: news.publication.language
|
|
378
|
-
}
|
|
379
|
-
};
|
|
380
|
-
} else {
|
|
93
|
+
const extensions = entry.extensions;
|
|
94
|
+
if (extensions?.alternatives) {
|
|
95
|
+
url.alternatives = extensions.alternatives.flatMap((alternative) => {
|
|
96
|
+
if ((!alternative.rel || alternative.rel === "alternate") && alternative.hreflang)
|
|
97
|
+
return [{ hreflang: alternative.hreflang, href: alternative.href }];
|
|
381
98
|
warnings.push({
|
|
382
99
|
type: "validation",
|
|
383
|
-
message:
|
|
384
|
-
context: { url:
|
|
100
|
+
message: 'Alternative link missing required rel="alternate", hreflang, or href',
|
|
101
|
+
context: { url: entry.loc, field: "link" }
|
|
385
102
|
});
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
return urlObj;
|
|
389
|
-
}
|
|
390
|
-
function parseTag(xml, start, end) {
|
|
391
|
-
let cursor = start + 1;
|
|
392
|
-
while (cursor < end && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
393
|
-
cursor++;
|
|
394
|
-
const closing = xml.charCodeAt(cursor) === 47;
|
|
395
|
-
if (closing)
|
|
396
|
-
cursor++;
|
|
397
|
-
if (cursor >= end || xml.charCodeAt(cursor) === 33 || xml.charCodeAt(cursor) === 63)
|
|
398
|
-
return null;
|
|
399
|
-
const nameStart = cursor;
|
|
400
|
-
while (cursor < end) {
|
|
401
|
-
const code = xml.charCodeAt(cursor);
|
|
402
|
-
if (isXmlWhitespace(code) || code === 47)
|
|
403
|
-
break;
|
|
404
|
-
cursor++;
|
|
405
|
-
}
|
|
406
|
-
if (cursor === nameStart)
|
|
407
|
-
return null;
|
|
408
|
-
const qualifiedName = xml.slice(nameStart, cursor);
|
|
409
|
-
const colon = qualifiedName.lastIndexOf(":");
|
|
410
|
-
let tail = end - 1;
|
|
411
|
-
while (tail > start && isXmlWhitespace(xml.charCodeAt(tail)))
|
|
412
|
-
tail--;
|
|
413
|
-
return {
|
|
414
|
-
name: colon === -1 ? qualifiedName : qualifiedName.slice(colon + 1),
|
|
415
|
-
closing,
|
|
416
|
-
selfClosing: !closing && xml.charCodeAt(tail) === 47
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
const URLSET_OPEN = 1;
|
|
420
|
-
const URLSET_CLOSE = 2;
|
|
421
|
-
const URLSET_SELF_CLOSING = 3;
|
|
422
|
-
const URL_OPEN = 4;
|
|
423
|
-
const URL_SELF_CLOSING = 5;
|
|
424
|
-
const URL_CLOSE = 6;
|
|
425
|
-
const SITEMAP_INDEX_OPEN = 7;
|
|
426
|
-
const SITEMAP_INDEX_CLOSE = 8;
|
|
427
|
-
const SITEMAP_INDEX_SELF_CLOSING = 9;
|
|
428
|
-
const SITEMAP_OPEN = 10;
|
|
429
|
-
const SITEMAP_SELF_CLOSING = 11;
|
|
430
|
-
const SITEMAP_CLOSE = 12;
|
|
431
|
-
function parseBoundaryTag(xml, start, end) {
|
|
432
|
-
let cursor = start + 1;
|
|
433
|
-
while (cursor < end && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
434
|
-
cursor++;
|
|
435
|
-
const closing = xml.charCodeAt(cursor) === 47;
|
|
436
|
-
if (closing)
|
|
437
|
-
cursor++;
|
|
438
|
-
if (cursor >= end || xml.charCodeAt(cursor) === 33 || xml.charCodeAt(cursor) === 63)
|
|
439
|
-
return 0;
|
|
440
|
-
let localNameStart = cursor;
|
|
441
|
-
while (cursor < end) {
|
|
442
|
-
const code = xml.charCodeAt(cursor);
|
|
443
|
-
if (isXmlWhitespace(code) || code === 47)
|
|
444
|
-
break;
|
|
445
|
-
if (code === 58)
|
|
446
|
-
localNameStart = cursor + 1;
|
|
447
|
-
cursor++;
|
|
448
|
-
}
|
|
449
|
-
const localNameLength = cursor - localNameStart;
|
|
450
|
-
let tail = end - 1;
|
|
451
|
-
while (tail > start && isXmlWhitespace(xml.charCodeAt(tail)))
|
|
452
|
-
tail--;
|
|
453
|
-
const selfClosing = !closing && xml.charCodeAt(tail) === 47;
|
|
454
|
-
if (localNameLength === 3 && xml.startsWith("url", localNameStart)) {
|
|
455
|
-
if (closing)
|
|
456
|
-
return URL_CLOSE;
|
|
457
|
-
return selfClosing ? URL_SELF_CLOSING : URL_OPEN;
|
|
458
|
-
}
|
|
459
|
-
if (localNameLength === 6 && xml.startsWith("urlset", localNameStart)) {
|
|
460
|
-
if (closing)
|
|
461
|
-
return URLSET_CLOSE;
|
|
462
|
-
return selfClosing ? URLSET_SELF_CLOSING : URLSET_OPEN;
|
|
463
|
-
}
|
|
464
|
-
if (localNameLength === 12 && xml.startsWith("sitemapindex", localNameStart)) {
|
|
465
|
-
if (closing)
|
|
466
|
-
return SITEMAP_INDEX_CLOSE;
|
|
467
|
-
return selfClosing ? SITEMAP_INDEX_SELF_CLOSING : SITEMAP_INDEX_OPEN;
|
|
468
|
-
}
|
|
469
|
-
if (localNameLength === 7 && xml.startsWith("sitemap", localNameStart)) {
|
|
470
|
-
if (closing)
|
|
471
|
-
return SITEMAP_CLOSE;
|
|
472
|
-
return selfClosing ? SITEMAP_SELF_CLOSING : SITEMAP_OPEN;
|
|
473
|
-
}
|
|
474
|
-
return 0;
|
|
475
|
-
}
|
|
476
|
-
function isXmlWhitespace(code) {
|
|
477
|
-
return code === 32 || code === 9 || code === 10 || code === 13;
|
|
478
|
-
}
|
|
479
|
-
function findMarkupEnd(xml, start) {
|
|
480
|
-
if (xml.startsWith("<!--", start)) {
|
|
481
|
-
const end = xml.indexOf("-->", start + 4);
|
|
482
|
-
return end === -1 ? -1 : end + 2;
|
|
483
|
-
}
|
|
484
|
-
if (xml.startsWith("<![CDATA[", start)) {
|
|
485
|
-
const end = xml.indexOf("]]>", start + 9);
|
|
486
|
-
return end === -1 ? -1 : end + 2;
|
|
487
|
-
}
|
|
488
|
-
if (xml.startsWith("<?", start)) {
|
|
489
|
-
const end = xml.indexOf("?>", start + 2);
|
|
490
|
-
return end === -1 ? -1 : end + 1;
|
|
491
|
-
}
|
|
492
|
-
let quote = 0;
|
|
493
|
-
for (let cursor = start + 1; cursor < xml.length; cursor++) {
|
|
494
|
-
const code = xml.charCodeAt(cursor);
|
|
495
|
-
if (quote) {
|
|
496
|
-
if (code === quote)
|
|
497
|
-
quote = 0;
|
|
498
|
-
} else if (code === 34 || code === 39) {
|
|
499
|
-
quote = code;
|
|
500
|
-
} else if (code === 62) {
|
|
501
|
-
return cursor;
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
return -1;
|
|
505
|
-
}
|
|
506
|
-
function isAsyncIterable(input) {
|
|
507
|
-
return typeof input === "object" && input !== null && Symbol.asyncIterator in input;
|
|
508
|
-
}
|
|
509
|
-
function isIterable(input) {
|
|
510
|
-
return typeof input === "object" && input !== null && Symbol.iterator in input;
|
|
511
|
-
}
|
|
512
|
-
function isReadableStream(input) {
|
|
513
|
-
return typeof input === "object" && input !== null && "getReader" in input;
|
|
514
|
-
}
|
|
515
|
-
async function* iterateInput(input) {
|
|
516
|
-
if (typeof input === "string" || input instanceof Uint8Array) {
|
|
517
|
-
yield input;
|
|
518
|
-
return;
|
|
519
|
-
}
|
|
520
|
-
if (isReadableStream(input)) {
|
|
521
|
-
const reader = input.getReader();
|
|
522
|
-
let completed = false;
|
|
523
|
-
try {
|
|
524
|
-
while (true) {
|
|
525
|
-
const result = await reader.read();
|
|
526
|
-
if (result.done) {
|
|
527
|
-
completed = true;
|
|
528
|
-
return;
|
|
529
|
-
}
|
|
530
|
-
yield result.value;
|
|
531
|
-
}
|
|
532
|
-
} finally {
|
|
533
|
-
if (!completed)
|
|
534
|
-
await reader.cancel();
|
|
535
|
-
reader.releaseLock();
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
if (isAsyncIterable(input)) {
|
|
539
|
-
yield* input;
|
|
540
|
-
return;
|
|
541
|
-
}
|
|
542
|
-
if (isIterable(input)) {
|
|
543
|
-
yield* input;
|
|
544
|
-
return;
|
|
545
|
-
}
|
|
546
|
-
throw new TypeError("Sitemap XML input must be a string, Uint8Array, iterable, or ReadableStream");
|
|
547
|
-
}
|
|
548
|
-
async function* decodeInput(input) {
|
|
549
|
-
let decoder = new TextDecoder();
|
|
550
|
-
let decodingBytes = false;
|
|
551
|
-
for await (const chunk of iterateInput(input)) {
|
|
552
|
-
if (typeof chunk === "string") {
|
|
553
|
-
if (decodingBytes) {
|
|
554
|
-
const tail = decoder.decode();
|
|
555
|
-
if (tail)
|
|
556
|
-
yield tail;
|
|
557
|
-
decoder = new TextDecoder();
|
|
558
|
-
decodingBytes = false;
|
|
559
|
-
}
|
|
560
|
-
if (chunk)
|
|
561
|
-
yield chunk;
|
|
562
|
-
} else if (chunk instanceof Uint8Array) {
|
|
563
|
-
decodingBytes = true;
|
|
564
|
-
const text = decoder.decode(chunk, { stream: true });
|
|
565
|
-
if (text)
|
|
566
|
-
yield text;
|
|
567
|
-
} else {
|
|
568
|
-
throw new TypeError("Sitemap XML chunks must be strings or Uint8Array values");
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
if (decodingBytes) {
|
|
572
|
-
const tail = decoder.decode();
|
|
573
|
-
if (tail)
|
|
574
|
-
yield tail;
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
function utf8ByteLength(value) {
|
|
578
|
-
let bytes = 0;
|
|
579
|
-
for (let index = 0; index < value.length; index++) {
|
|
580
|
-
const code = value.charCodeAt(index);
|
|
581
|
-
if (code < 128) {
|
|
582
|
-
bytes++;
|
|
583
|
-
} else if (code < 2048) {
|
|
584
|
-
bytes += 2;
|
|
585
|
-
} else if (code >= 55296 && code <= 56319 && index + 1 < value.length) {
|
|
586
|
-
const next = value.charCodeAt(index + 1);
|
|
587
|
-
if (next >= 56320 && next <= 57343) {
|
|
588
|
-
bytes += 4;
|
|
589
|
-
index++;
|
|
590
|
-
} else {
|
|
591
|
-
bytes += 3;
|
|
592
|
-
}
|
|
593
|
-
} else {
|
|
594
|
-
bytes += 3;
|
|
595
|
-
}
|
|
103
|
+
return [];
|
|
104
|
+
});
|
|
596
105
|
}
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
if (
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
}
|
|
610
|
-
function resolveMaxBufferBytes(options) {
|
|
611
|
-
const value = options.maxBufferBytes ?? DEFAULT_MAX_ENTRY_BYTES;
|
|
106
|
+
const images = legacyImages(extensions?.images);
|
|
107
|
+
if (images?.length)
|
|
108
|
+
url.images = images;
|
|
109
|
+
if (extensions?.videos?.length)
|
|
110
|
+
url.videos = extensions.videos;
|
|
111
|
+
if (extensions?.news)
|
|
112
|
+
url.news = extensions.news;
|
|
113
|
+
return { url, warnings };
|
|
114
|
+
}
|
|
115
|
+
function positiveOption(value, name) {
|
|
116
|
+
if (value === void 0)
|
|
117
|
+
return void 0;
|
|
612
118
|
if (!Number.isSafeInteger(value) || value < 1)
|
|
613
|
-
throw new TypeError(
|
|
119
|
+
throw new TypeError(`${name} must be a positive safe integer`);
|
|
614
120
|
return value;
|
|
615
121
|
}
|
|
616
|
-
function
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
if (entity === "amp") {
|
|
635
|
-
decoded += "&";
|
|
636
|
-
} else if (entity === "lt") {
|
|
637
|
-
decoded += "<";
|
|
638
|
-
} else if (entity === "gt") {
|
|
639
|
-
decoded += ">";
|
|
640
|
-
} else if (entity === "quot") {
|
|
641
|
-
decoded += '"';
|
|
642
|
-
} else if (entity === "apos") {
|
|
643
|
-
decoded += "'";
|
|
644
|
-
} else if (entity.charCodeAt(0) === 35) {
|
|
645
|
-
const hex = entity.charCodeAt(1) === 120 || entity.charCodeAt(1) === 88;
|
|
646
|
-
const codePoint = Number.parseInt(entity.slice(hex ? 2 : 1), hex ? 16 : 10);
|
|
647
|
-
decoded += Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 1114111 && !(codePoint >= 55296 && codePoint <= 57343) ? String.fromCodePoint(codePoint) : value.slice(cursor, end + 1);
|
|
648
|
-
} else {
|
|
649
|
-
decoded += value.slice(cursor, end + 1);
|
|
650
|
-
}
|
|
651
|
-
cursor = end + 1;
|
|
652
|
-
}
|
|
653
|
-
return decoded;
|
|
654
|
-
}
|
|
655
|
-
function decodeElementContent(value) {
|
|
656
|
-
const firstMarkup = value.indexOf("<");
|
|
657
|
-
if (firstMarkup === -1)
|
|
658
|
-
return decodeXmlEntities(value.trim());
|
|
659
|
-
let decoded = "";
|
|
660
|
-
let cursor = 0;
|
|
661
|
-
while (cursor < value.length) {
|
|
662
|
-
const markupStart = value.indexOf("<", cursor);
|
|
663
|
-
if (markupStart === -1) {
|
|
664
|
-
decoded += decodeXmlEntities(value.slice(cursor));
|
|
665
|
-
break;
|
|
666
|
-
}
|
|
667
|
-
decoded += decodeXmlEntities(value.slice(cursor, markupStart));
|
|
668
|
-
if (value.startsWith("<![CDATA[", markupStart)) {
|
|
669
|
-
const end = value.indexOf("]]>", markupStart + 9);
|
|
670
|
-
if (end === -1)
|
|
671
|
-
return null;
|
|
672
|
-
decoded += value.slice(markupStart + 9, end);
|
|
673
|
-
cursor = end + 3;
|
|
674
|
-
} else if (value.startsWith("<!--", markupStart)) {
|
|
675
|
-
const end = value.indexOf("-->", markupStart + 4);
|
|
676
|
-
if (end === -1)
|
|
677
|
-
return null;
|
|
678
|
-
cursor = end + 3;
|
|
679
|
-
} else {
|
|
680
|
-
return null;
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
return decoded.trim();
|
|
684
|
-
}
|
|
685
|
-
function parseCommonUrlEntry(xml) {
|
|
686
|
-
if (!xml.startsWith("<url>"))
|
|
687
|
-
return null;
|
|
688
|
-
const parsed = {};
|
|
689
|
-
let seenFields = 0;
|
|
690
|
-
let cursor = 5;
|
|
691
|
-
while (cursor < xml.length) {
|
|
692
|
-
while (cursor < xml.length && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
693
|
-
cursor++;
|
|
694
|
-
if (xml.startsWith("</url>", cursor)) {
|
|
695
|
-
cursor += 6;
|
|
696
|
-
while (cursor < xml.length && isXmlWhitespace(xml.charCodeAt(cursor)))
|
|
697
|
-
cursor++;
|
|
698
|
-
return cursor === xml.length ? parsed : null;
|
|
699
|
-
}
|
|
700
|
-
let name;
|
|
701
|
-
let fieldBit;
|
|
702
|
-
if (xml.startsWith("<loc>", cursor)) {
|
|
703
|
-
name = "loc";
|
|
704
|
-
fieldBit = 1;
|
|
705
|
-
} else if (xml.startsWith("<lastmod>", cursor)) {
|
|
706
|
-
name = "lastmod";
|
|
707
|
-
fieldBit = 2;
|
|
708
|
-
} else if (xml.startsWith("<changefreq>", cursor)) {
|
|
709
|
-
name = "changefreq";
|
|
710
|
-
fieldBit = 4;
|
|
711
|
-
} else if (xml.startsWith("<priority>", cursor)) {
|
|
712
|
-
name = "priority";
|
|
713
|
-
fieldBit = 8;
|
|
714
|
-
} else {
|
|
715
|
-
return null;
|
|
716
|
-
}
|
|
717
|
-
if (seenFields & fieldBit)
|
|
718
|
-
return null;
|
|
719
|
-
seenFields |= fieldBit;
|
|
720
|
-
const contentStart = cursor + name.length + 2;
|
|
721
|
-
const closingTag = `</${name}>`;
|
|
722
|
-
const contentEnd = xml.indexOf(closingTag, contentStart);
|
|
723
|
-
if (contentEnd === -1 || xml.indexOf("<", contentStart) !== contentEnd)
|
|
724
|
-
return null;
|
|
725
|
-
parsed[name] = decodeXmlEntities(xml.slice(contentStart, contentEnd).trim());
|
|
726
|
-
cursor = contentEnd + closingTag.length;
|
|
727
|
-
}
|
|
728
|
-
return null;
|
|
729
|
-
}
|
|
730
|
-
function parseSimpleUrlEntry(xml) {
|
|
731
|
-
const parsed = {};
|
|
732
|
-
const seenFields = /* @__PURE__ */ new Set();
|
|
733
|
-
let scanIndex = 0;
|
|
734
|
-
let depth = 0;
|
|
735
|
-
let field;
|
|
736
|
-
let contentStart = 0;
|
|
737
|
-
let closedUrl = false;
|
|
738
|
-
while (true) {
|
|
739
|
-
const markupStart = xml.indexOf("<", scanIndex);
|
|
740
|
-
if (markupStart === -1)
|
|
741
|
-
break;
|
|
742
|
-
const markupEnd = findMarkupEnd(xml, markupStart);
|
|
743
|
-
if (markupEnd === -1)
|
|
744
|
-
return null;
|
|
745
|
-
const tag = parseTag(xml, markupStart, markupEnd);
|
|
746
|
-
scanIndex = markupEnd + 1;
|
|
747
|
-
if (!tag)
|
|
748
|
-
continue;
|
|
749
|
-
if (!tag.closing) {
|
|
750
|
-
if (depth === 0 && tag.name === "url") {
|
|
751
|
-
if (tag.selfClosing)
|
|
752
|
-
return {};
|
|
753
|
-
depth = 1;
|
|
754
|
-
continue;
|
|
755
|
-
}
|
|
756
|
-
if (depth !== 1 || !SIMPLE_URL_FIELDS.has(tag.name) || seenFields.has(tag.name))
|
|
757
|
-
return null;
|
|
758
|
-
seenFields.add(tag.name);
|
|
759
|
-
if (tag.selfClosing) {
|
|
760
|
-
parsed[tag.name] = "";
|
|
761
|
-
continue;
|
|
762
|
-
}
|
|
763
|
-
field = tag.name;
|
|
764
|
-
contentStart = markupEnd + 1;
|
|
765
|
-
depth = 2;
|
|
766
|
-
continue;
|
|
767
|
-
}
|
|
768
|
-
if (depth === 2 && field === tag.name) {
|
|
769
|
-
const value = decodeElementContent(xml.slice(contentStart, markupStart));
|
|
770
|
-
if (value === null)
|
|
771
|
-
return null;
|
|
772
|
-
parsed[field] = value;
|
|
773
|
-
field = void 0;
|
|
774
|
-
depth = 1;
|
|
775
|
-
} else if (depth === 1 && tag.name === "url") {
|
|
776
|
-
closedUrl = true;
|
|
777
|
-
depth = 0;
|
|
778
|
-
} else {
|
|
779
|
-
return null;
|
|
780
|
-
}
|
|
781
|
-
}
|
|
782
|
-
return closedUrl && depth === 0 ? parsed : null;
|
|
783
|
-
}
|
|
784
|
-
function parseUrlEntry(parser, xml) {
|
|
785
|
-
const simple = parseCommonUrlEntry(xml) ?? parseSimpleUrlEntry(xml);
|
|
786
|
-
if (simple)
|
|
787
|
-
return simple;
|
|
788
|
-
try {
|
|
789
|
-
const parsed = parser.parse(xml);
|
|
790
|
-
const value = Array.isArray(parsed?.url) ? parsed.url[0] : parsed?.url;
|
|
791
|
-
return value && typeof value === "object" ? value : {};
|
|
792
|
-
} catch (error) {
|
|
793
|
-
throw new Error(`Failed to parse XML: ${error instanceof Error ? error.message : String(error)}`);
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
function parseSitemapEntry(parser, xml, warnings) {
|
|
797
|
-
let parsed;
|
|
798
|
-
try {
|
|
799
|
-
parsed = parser.parse(xml);
|
|
800
|
-
} catch (error) {
|
|
801
|
-
throw new Error(`Failed to parse XML: ${error instanceof Error ? error.message : String(error)}`);
|
|
802
|
-
}
|
|
803
|
-
const loc = typeof parsed.sitemap?.loc === "string" ? parsed.sitemap.loc.trim() : "";
|
|
804
|
-
if (!loc) {
|
|
805
|
-
warnings.push({
|
|
806
|
-
type: "validation",
|
|
807
|
-
message: "Sitemap entry missing required loc element"
|
|
808
|
-
});
|
|
809
|
-
return null;
|
|
810
|
-
}
|
|
811
|
-
if (!URL.canParse(loc)) {
|
|
812
|
-
warnings.push({
|
|
813
|
-
type: "validation",
|
|
814
|
-
message: "Sitemap entry has invalid URL",
|
|
815
|
-
context: { url: loc }
|
|
816
|
-
});
|
|
817
|
-
return null;
|
|
818
|
-
}
|
|
819
|
-
const entry = { loc };
|
|
820
|
-
if (typeof parsed.sitemap?.lastmod === "string" && parsed.sitemap.lastmod.trim())
|
|
821
|
-
entry.lastmod = parsed.sitemap.lastmod.trim();
|
|
822
|
-
return entry;
|
|
823
|
-
}
|
|
824
|
-
async function* parseSitemapStream(input, options = {}) {
|
|
825
|
-
const maxEntryBytes = resolveMaxEntryBytes(options);
|
|
826
|
-
const maxBufferBytes = resolveMaxBufferBytes(options);
|
|
827
|
-
const parser = createUrlParser();
|
|
828
|
-
let buffer = "";
|
|
829
|
-
let scanIndex = 0;
|
|
830
|
-
let entryStart = -1;
|
|
831
|
-
let sawInput = false;
|
|
122
|
+
function parserFailure(issue, kind, options) {
|
|
123
|
+
if (issue?.code === "empty")
|
|
124
|
+
return new Error("Empty XML input provided");
|
|
125
|
+
if (issue?.code === "decoded_limit" && options.maxBufferBytes)
|
|
126
|
+
return new Error(`Sitemap XML buffer exceeds maxBufferBytes of ${options.maxBufferBytes}`);
|
|
127
|
+
const entryLimit = issue?.message.match(/^Sitemap entry exceeds (\d+) bytes$/);
|
|
128
|
+
if (entryLimit)
|
|
129
|
+
return new Error(`Sitemap entry exceeds maxEntryBytes of ${entryLimit[1]}`);
|
|
130
|
+
if (issue?.code === "unsupported" || issue?.code === "html") {
|
|
131
|
+
return new Error(
|
|
132
|
+
kind === "index" ? "XML does not contain a valid sitemapindex element" : kind === "urlset" ? "XML does not contain a valid urlset element" : "XML does not contain a valid sitemap element"
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
return new Error(`Failed to parse XML: ${issue?.message || "Malformed sitemap"}`);
|
|
136
|
+
}
|
|
137
|
+
async function* parseSitemapStreamInternal(input, options = {}, expectedKind) {
|
|
138
|
+
const maxEntryBytes = positiveOption(options.maxEntryBytes, "maxEntryBytes");
|
|
139
|
+
const maxBufferBytes = positiveOption(options.maxBufferBytes, "maxBufferBytes");
|
|
832
140
|
let kind;
|
|
833
|
-
let
|
|
834
|
-
let
|
|
835
|
-
let
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
const tag = parseBoundaryTag(buffer, markupStart, markupEnd);
|
|
852
|
-
if (!kind && tag === 0) {
|
|
853
|
-
const root = parseTag(buffer, markupStart, markupEnd);
|
|
854
|
-
if (root && !root.closing)
|
|
855
|
-
throw new Error("XML does not contain a valid sitemap element");
|
|
856
|
-
}
|
|
857
|
-
let record;
|
|
858
|
-
if (!kind && (tag === URLSET_OPEN || tag === URLSET_SELF_CLOSING)) {
|
|
859
|
-
kind = "urlset";
|
|
860
|
-
insideRoot = tag === URLSET_OPEN;
|
|
861
|
-
closedRoot = tag === URLSET_SELF_CLOSING;
|
|
862
|
-
yield { _tag: "kind", kind };
|
|
863
|
-
} else if (!kind && (tag === SITEMAP_INDEX_OPEN || tag === SITEMAP_INDEX_SELF_CLOSING)) {
|
|
864
|
-
kind = "index";
|
|
865
|
-
insideRoot = tag === SITEMAP_INDEX_OPEN;
|
|
866
|
-
closedRoot = tag === SITEMAP_INDEX_SELF_CLOSING;
|
|
867
|
-
yield { _tag: "kind", kind };
|
|
868
|
-
} else if (entryStart === -1 && (kind === "urlset" && tag === URLSET_CLOSE || kind === "index" && tag === SITEMAP_INDEX_CLOSE)) {
|
|
869
|
-
insideRoot = false;
|
|
870
|
-
closedRoot = true;
|
|
871
|
-
}
|
|
872
|
-
const openingEntry = kind === "urlset" ? tag === URL_OPEN || tag === URL_SELF_CLOSING : tag === SITEMAP_OPEN || tag === SITEMAP_SELF_CLOSING;
|
|
873
|
-
const closingEntry = kind === "urlset" ? tag === URL_CLOSE : tag === SITEMAP_CLOSE;
|
|
874
|
-
const selfClosingEntry = tag === URL_SELF_CLOSING || tag === SITEMAP_SELF_CLOSING;
|
|
875
|
-
if (insideRoot && openingEntry && entryStart === -1) {
|
|
876
|
-
entryStart = markupStart;
|
|
877
|
-
if (selfClosingEntry)
|
|
878
|
-
record = buffer.slice(entryStart, markupEnd + 1);
|
|
879
|
-
} else if (closingEntry && entryStart !== -1) {
|
|
880
|
-
record = buffer.slice(entryStart, markupEnd + 1);
|
|
881
|
-
}
|
|
882
|
-
if (!record) {
|
|
883
|
-
scanIndex = markupEnd + 1;
|
|
141
|
+
let lastIssue;
|
|
142
|
+
let invalidUrlEntries = 0;
|
|
143
|
+
let validUrls = 0;
|
|
144
|
+
for await (const event of parseSitemap(input, {
|
|
145
|
+
...maxEntryBytes ? { maxEntryBytes } : {},
|
|
146
|
+
...maxBufferBytes ? { maxDecodedBytes: maxBufferBytes } : {}
|
|
147
|
+
})) {
|
|
148
|
+
if (event._tag === "document") {
|
|
149
|
+
if (event.format !== "xml") {
|
|
150
|
+
throw new Error(
|
|
151
|
+
expectedKind === "index" ? "XML does not contain a valid sitemapindex element" : expectedKind === "urlset" ? "XML does not contain a valid urlset element" : "XML does not contain a valid sitemap element"
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
kind = event.kind;
|
|
155
|
+
yield { _tag: "kind", kind };
|
|
156
|
+
} else if (event._tag === "issue") {
|
|
157
|
+
lastIssue = event.issue;
|
|
158
|
+
if (!kind)
|
|
884
159
|
continue;
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
const warnings = [];
|
|
890
|
-
scanIndex = markupEnd + 1;
|
|
891
|
-
entryStart = -1;
|
|
892
|
-
const entry = kind === "urlset" ? extractUrlFromParsedElement(parseUrlEntry(parser, record), warnings) : parseSitemapEntry(parser, record, warnings);
|
|
893
|
-
for (const warning of warnings)
|
|
160
|
+
if (event.issue.code === "missing_loc" && kind === "urlset")
|
|
161
|
+
invalidUrlEntries++;
|
|
162
|
+
const warning = legacyWarning(event.issue, kind);
|
|
163
|
+
if (warning)
|
|
894
164
|
yield { _tag: "warning", warning };
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
yield { _tag: "
|
|
900
|
-
}
|
|
901
|
-
}
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
scanIndex -= entryStart;
|
|
908
|
-
entryStart = 0;
|
|
909
|
-
}
|
|
910
|
-
} else if (scanIndex > 0) {
|
|
911
|
-
buffer = buffer.slice(scanIndex);
|
|
912
|
-
scanIndex = 0;
|
|
165
|
+
} else if (event._tag === "url") {
|
|
166
|
+
validUrls++;
|
|
167
|
+
const mapped = legacyUrl(event.entry);
|
|
168
|
+
for (const warning of mapped.warnings)
|
|
169
|
+
yield { _tag: "warning", warning };
|
|
170
|
+
yield { _tag: "url", url: mapped.url };
|
|
171
|
+
} else if (event._tag === "sitemap") {
|
|
172
|
+
if (!URL.canParse(event.entry.loc))
|
|
173
|
+
continue;
|
|
174
|
+
yield { _tag: "sitemap", sitemap: event.entry };
|
|
175
|
+
} else if (event.completeness._tag !== "complete") {
|
|
176
|
+
throw parserFailure(lastIssue, kind || expectedKind, options);
|
|
913
177
|
}
|
|
914
|
-
if (entryStart === -1 && exceedsUtf8ByteLimit(buffer, maxBufferBytes))
|
|
915
|
-
throw new Error(`Sitemap XML buffer exceeds maxBufferBytes of ${maxBufferBytes}`);
|
|
916
178
|
}
|
|
917
|
-
if (
|
|
918
|
-
throw new Error("Empty XML input provided");
|
|
919
|
-
if (entryStart !== -1)
|
|
920
|
-
throw new Error(`Failed to parse XML: Unclosed ${kind === "index" ? "sitemap" : "url"} element`);
|
|
921
|
-
if (!kind)
|
|
922
|
-
throw new Error("XML does not contain a valid sitemap element");
|
|
923
|
-
if (!closedRoot)
|
|
924
|
-
throw new Error(`Failed to parse XML: Unclosed ${kind === "index" ? "sitemapindex" : "urlset"} element`);
|
|
925
|
-
if (kind === "urlset" && entryCount > 0 && validUrlCount === 0) {
|
|
179
|
+
if (kind === "urlset" && invalidUrlEntries > 0 && validUrls === 0) {
|
|
926
180
|
yield {
|
|
927
181
|
_tag: "warning",
|
|
928
182
|
warning: {
|
|
@@ -932,93 +186,55 @@ async function* parseSitemapStream(input, options = {}) {
|
|
|
932
186
|
};
|
|
933
187
|
}
|
|
934
188
|
}
|
|
189
|
+
async function* parseSitemapStream(input, options = {}) {
|
|
190
|
+
yield* parseSitemapStreamInternal(input, options);
|
|
191
|
+
}
|
|
935
192
|
async function* parseSitemapXmlStream(input, options = {}) {
|
|
936
|
-
for await (const event of
|
|
193
|
+
for await (const event of parseSitemapStreamInternal(input, options, "urlset")) {
|
|
937
194
|
if (event._tag === "kind") {
|
|
938
195
|
if (event.kind !== "urlset")
|
|
939
196
|
throw new Error("XML does not contain a valid urlset element");
|
|
940
197
|
continue;
|
|
941
198
|
}
|
|
942
|
-
if (event._tag
|
|
943
|
-
|
|
944
|
-
yield event;
|
|
199
|
+
if (event._tag !== "sitemap")
|
|
200
|
+
yield event;
|
|
945
201
|
}
|
|
946
202
|
}
|
|
947
203
|
async function* parseSitemapIndexStream(input, options = {}) {
|
|
948
|
-
for await (const event of
|
|
204
|
+
for await (const event of parseSitemapStreamInternal(input, options, "index")) {
|
|
949
205
|
if (event._tag === "kind") {
|
|
950
206
|
if (event.kind !== "index")
|
|
951
207
|
throw new Error("XML does not contain a valid sitemapindex element");
|
|
952
208
|
continue;
|
|
953
209
|
}
|
|
954
|
-
if (event._tag
|
|
955
|
-
|
|
956
|
-
yield event;
|
|
210
|
+
if (event._tag !== "url")
|
|
211
|
+
yield event;
|
|
957
212
|
}
|
|
958
213
|
}
|
|
959
214
|
async function parseSitemapXml(xml) {
|
|
960
|
-
if (!xml)
|
|
961
|
-
throw new Error("Empty XML input provided");
|
|
962
215
|
const urls = [];
|
|
963
216
|
const warnings = [];
|
|
964
|
-
const
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
let closedUrlset = false;
|
|
970
|
-
let entryCount = 0;
|
|
971
|
-
while (true) {
|
|
972
|
-
const markupStart = xml.indexOf("<", scanIndex);
|
|
973
|
-
if (markupStart === -1)
|
|
974
|
-
break;
|
|
975
|
-
const markupEnd = findMarkupEnd(xml, markupStart);
|
|
976
|
-
if (markupEnd === -1)
|
|
977
|
-
break;
|
|
978
|
-
const tag = parseBoundaryTag(xml, markupStart, markupEnd);
|
|
979
|
-
let record;
|
|
980
|
-
if (tag === URLSET_OPEN) {
|
|
981
|
-
sawUrlset = true;
|
|
982
|
-
insideUrlset = true;
|
|
983
|
-
} else if (tag === URLSET_SELF_CLOSING) {
|
|
984
|
-
sawUrlset = true;
|
|
985
|
-
insideUrlset = false;
|
|
986
|
-
closedUrlset = true;
|
|
987
|
-
} else if (tag === URLSET_CLOSE && entryStart === -1) {
|
|
988
|
-
insideUrlset = false;
|
|
989
|
-
closedUrlset = true;
|
|
990
|
-
}
|
|
991
|
-
if (insideUrlset && (tag === URL_OPEN || tag === URL_SELF_CLOSING) && entryStart === -1) {
|
|
992
|
-
entryStart = markupStart;
|
|
993
|
-
if (tag === URL_SELF_CLOSING)
|
|
994
|
-
record = xml.slice(entryStart, markupEnd + 1);
|
|
995
|
-
} else if (tag === URL_CLOSE && entryStart !== -1) {
|
|
996
|
-
record = xml.slice(entryStart, markupEnd + 1);
|
|
997
|
-
}
|
|
998
|
-
scanIndex = markupEnd + 1;
|
|
999
|
-
if (!record)
|
|
1000
|
-
continue;
|
|
1001
|
-
if (exceedsUtf8ByteLimit(record, DEFAULT_MAX_ENTRY_BYTES))
|
|
1002
|
-
throw new Error(`Sitemap URL entry exceeds maxEntryBytes of ${DEFAULT_MAX_ENTRY_BYTES}`);
|
|
1003
|
-
entryCount++;
|
|
1004
|
-
entryStart = -1;
|
|
1005
|
-
const url = extractUrlFromParsedElement(parseUrlEntry(parser, record), warnings);
|
|
1006
|
-
if (url)
|
|
1007
|
-
urls.push(url);
|
|
1008
|
-
}
|
|
1009
|
-
if (entryStart !== -1)
|
|
1010
|
-
throw new Error("Failed to parse XML: Unclosed url element");
|
|
1011
|
-
if (!sawUrlset)
|
|
1012
|
-
throw new Error("XML does not contain a valid urlset element");
|
|
1013
|
-
if (!closedUrlset)
|
|
1014
|
-
throw new Error("Failed to parse XML: Unclosed urlset element");
|
|
1015
|
-
if (entryCount > 0 && urls.length === 0) {
|
|
1016
|
-
warnings.push({
|
|
1017
|
-
type: "validation",
|
|
1018
|
-
message: "No valid URLs found in sitemap after validation"
|
|
1019
|
-
});
|
|
217
|
+
for await (const event of parseSitemapXmlStream(xml)) {
|
|
218
|
+
if (event._tag === "url")
|
|
219
|
+
urls.push(event.url);
|
|
220
|
+
else
|
|
221
|
+
warnings.push(event.warning);
|
|
1020
222
|
}
|
|
1021
223
|
return { urls, warnings };
|
|
1022
224
|
}
|
|
225
|
+
async function parseSitemapIndex(xml) {
|
|
226
|
+
const entries = [];
|
|
227
|
+
const warnings = [];
|
|
228
|
+
for await (const event of parseSitemapIndexStream(xml)) {
|
|
229
|
+
if (event._tag === "sitemap")
|
|
230
|
+
entries.push(event.sitemap);
|
|
231
|
+
else
|
|
232
|
+
warnings.push(event.warning);
|
|
233
|
+
}
|
|
234
|
+
return { entries, warnings };
|
|
235
|
+
}
|
|
236
|
+
function isSitemapIndex(xml) {
|
|
237
|
+
return xml.includes("<sitemapindex") || xml.includes("sitemapindex>");
|
|
238
|
+
}
|
|
1023
239
|
|
|
1024
240
|
export { isSitemapIndex, parseSitemapIndex, parseSitemapIndexStream, parseSitemapStream, parseSitemapXml, parseSitemapXmlStream };
|