@se-studio/site-check 1.0.12 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +6 -2
- package/dist/cli.js +15 -3
- package/dist/cli.js.map +1 -1
- package/dist/validate.d.ts +7 -1
- package/dist/validate.d.ts.map +1 -1
- package/dist/validate.js +29 -8
- package/dist/validate.js.map +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.0.15
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Bulk version bump: patch for all packages
|
|
8
|
+
|
|
9
|
+
## 1.0.14
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- CLI: `--ignore-missing-child-sitemaps` — when resolving a sitemap index, skip child `<loc>` URLs that return 404 instead of failing validation.
|
|
14
|
+
|
|
15
|
+
## 1.0.13
|
|
16
|
+
|
|
17
|
+
### Patch Changes
|
|
18
|
+
|
|
19
|
+
- Bulk version bump: patch for all packages
|
|
20
|
+
|
|
3
21
|
## 1.0.12
|
|
4
22
|
|
|
5
23
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@ Validate SE marketing sites (sitemap.xml, sitemap-unindexed.xml, llms.txt) and d
|
|
|
5
5
|
## Usage
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
npx @se-studio/site-check <baseUrl> [-o dir] [-H "Name: value"] [--vercel-bypass [secret]] [--check-against <devUrl>] [--check-rewrites]
|
|
8
|
+
npx @se-studio/site-check <baseUrl> [-o dir] [-H "Name: value"] [--vercel-bypass [secret]] [--check-against <devUrl>] [--check-rewrites] [--ignore-missing-child-sitemaps]
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
### Examples
|
|
@@ -31,12 +31,15 @@ npx @se-studio/site-check https://preview.vercel.app --vercel-bypass your-secret
|
|
|
31
31
|
|
|
32
32
|
# Custom headers
|
|
33
33
|
npx @se-studio/site-check https://example.com -H "Authorization: Bearer token" -H "X-Custom: value"
|
|
34
|
+
|
|
35
|
+
# Sitemap index lists a child that does not exist locally (404) — skip it and use remaining children
|
|
36
|
+
npx @se-studio/site-check http://localhost:3016 --ignore-missing-child-sitemaps -o ./out
|
|
34
37
|
```
|
|
35
38
|
|
|
36
39
|
## What it does
|
|
37
40
|
|
|
38
41
|
1. **Validates** the site by fetching:
|
|
39
|
-
- `sitemap.xml` (required) — must return 200 and contain `<loc>` entries; if it is a sitemap index, child sitemaps are fetched and must return valid urlset content
|
|
42
|
+
- `sitemap.xml` (required) — must return 200 and contain `<loc>` entries; if it is a sitemap index, child sitemaps are fetched and must return valid urlset content (or use `--ignore-missing-child-sitemaps` to skip child URLs that return **404**)
|
|
40
43
|
- `sitemap-unindexed.xml` (optional) — if 404, a warning is printed and the run continues. When it exists (returns 200), its page URLs are included in all checks (existence on dev, optional rewrite check, markdown collection).
|
|
41
44
|
- `llms.txt` (optional) — if 404, a warning is printed and the run continues; if it returns 200 it is validated
|
|
42
45
|
|
|
@@ -64,6 +67,7 @@ Use this to ensure a development or staging build has the same pages as producti
|
|
|
64
67
|
| `-H`, `--header "Name: value"` | Add a request header (repeatable). Applied to both production and development requests in compare mode. |
|
|
65
68
|
| `--check-against <devUrl>` | Compare mode: use production sitemap(s) but check that each page exists on development (same path on `<devUrl>`). No markdown download. |
|
|
66
69
|
| `--check-rewrites` | Enable checking for unexpected rewrites (x-nextjs-rewritten-path). Default: off. When set, the run fails if any sitemap page URL rewrites to a different path (locale-only rewrites allowed with `SITE_CHECK_LOCALES`). |
|
|
70
|
+
| `--ignore-missing-child-sitemaps` | When the root sitemap (or `sitemap-unindexed.xml`) is a **sitemap index**, any child `<loc>` that returns **404** is skipped instead of failing validation. Other non-2xx responses still fail. After skipping, at least one page URL must still be collected from the remaining children. |
|
|
67
71
|
| `--vercel-bypass [secret]` | Set `x-vercel-protection-bypass` for [Vercel Deployment Protection](https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/protection-bypass-automation). If `secret` is omitted, uses `VERCEL_AUTOMATION_BYPASS_SECRET` from the environment. |
|
|
68
72
|
|
|
69
73
|
Headers from `-H` override the Vercel bypass header if the same name is used.
|
package/dist/cli.js
CHANGED
|
@@ -24,9 +24,13 @@ function parseArgs(argv) {
|
|
|
24
24
|
let baseUrl = null;
|
|
25
25
|
let checkAgainstUrl = null;
|
|
26
26
|
let checkRewrites = false;
|
|
27
|
+
let ignoreMissingChildSitemaps = false;
|
|
27
28
|
for (let i = 0; i < argv.length; i++) {
|
|
28
29
|
const arg = argv[i];
|
|
29
|
-
if (arg === '--
|
|
30
|
+
if (arg === '--ignore-missing-child-sitemaps') {
|
|
31
|
+
ignoreMissingChildSitemaps = true;
|
|
32
|
+
}
|
|
33
|
+
else if (arg === '--check-rewrites') {
|
|
30
34
|
checkRewrites = true;
|
|
31
35
|
}
|
|
32
36
|
else if (arg === '-o' || arg === '--out') {
|
|
@@ -70,12 +74,13 @@ function parseArgs(argv) {
|
|
|
70
74
|
vercelBypassRequested,
|
|
71
75
|
vercelBypassSecret,
|
|
72
76
|
checkRewrites,
|
|
77
|
+
ignoreMissingChildSitemaps,
|
|
73
78
|
};
|
|
74
79
|
}
|
|
75
80
|
async function main() {
|
|
76
81
|
const args = parseArgs(process.argv.slice(2));
|
|
77
82
|
if (!args.baseUrl) {
|
|
78
|
-
console.error('Usage: site-check <baseUrl> [-o dir] [-H "Name: value"] [--vercel-bypass [secret]] [--check-against <devUrl>] [--check-rewrites]');
|
|
83
|
+
console.error('Usage: site-check <baseUrl> [-o dir] [-H "Name: value"] [--vercel-bypass [secret]] [--check-against <devUrl>] [--check-rewrites] [--ignore-missing-child-sitemaps]');
|
|
79
84
|
console.error('Example: site-check http://localhost:3015 -o ./out');
|
|
80
85
|
console.error('Example (compare prod vs dev): site-check https://example.com --check-against http://localhost:3010');
|
|
81
86
|
console.error('Example (with rewrite check): site-check https://example.com --check-rewrites');
|
|
@@ -92,9 +97,16 @@ async function main() {
|
|
|
92
97
|
console.log(`Comparing: production ${prodUrl} vs development ${args.checkAgainstUrl}`);
|
|
93
98
|
}
|
|
94
99
|
console.log('Validating…');
|
|
95
|
-
const validation = await validate(prodUrl, fetchInit
|
|
100
|
+
const validation = await validate(prodUrl, fetchInit, {
|
|
101
|
+
ignoreMissingChildSitemaps: args.ignoreMissingChildSitemaps,
|
|
102
|
+
});
|
|
96
103
|
const sitemapOk = validation.sitemapXml.ok ? 'OK' : `FAIL (${validation.sitemapXml.status})`;
|
|
97
104
|
console.log(` sitemap.xml … ${sitemapOk}`);
|
|
105
|
+
if (validation.skippedMissingChildSitemaps?.length) {
|
|
106
|
+
for (const u of validation.skippedMissingChildSitemaps) {
|
|
107
|
+
console.log(` skipped missing child sitemap (404): ${u}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
98
110
|
if (validation.sitemapUnindexedXml.warned) {
|
|
99
111
|
console.log(' sitemap-unindexed.xml … not found (optional, continuing)');
|
|
100
112
|
}
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,4EAA4E;AAE5E,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,+BAA+B,EAC/B,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAE3D,MAAM,WAAW,GAAG,mBAAmB,CAAC;AAExC,SAAS,kBAAkB,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,4EAA4E;AAE5E,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,+BAA+B,EAC/B,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAE3D,MAAM,WAAW,GAAG,mBAAmB,CAAC;AAExC,SAAS,kBAAkB,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAaD,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,MAAM,GAAG,WAAW,CAAC;IACzB,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAClC,IAAI,kBAAkB,GAAkB,IAAI,CAAC;IAC7C,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,IAAI,eAAe,GAAkB,IAAI,CAAC;IAC1C,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,IAAI,0BAA0B,GAAG,KAAK,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,GAAG,KAAK,iCAAiC,EAAE,CAAC;YAC9C,0BAA0B,GAAG,IAAI,CAAC;QACpC,CAAC;aAAM,IAAI,GAAG,KAAK,kBAAkB,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;aAAM,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,GAAG,IAAI,CAAC;gBACd,CAAC,EAAE,CAAC;YACN,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,UAAU,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,IAAI,IAAI,EAAE,CAAC;gBACT,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,CAAC,EAAE,CAAC;YACN,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,iBAAiB,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;gBACxE,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC1C,CAAC,EAAE,CAAC;YACN,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,iBAAiB,EAAE,CAAC;YACrC,qBAAqB,GAAG,IAAI,CAAC;YAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,kBAAkB,GAAG,IAAI,CAAC;gBAC1B,CAAC,EAAE,CAAC;YACN,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACnE,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,eAAe;QACf,MAAM;QACN,UAAU;QACV,qBAAqB;QACrB,kBAAkB;QAClB,aAAa;QACb,0BAA0B;KAC3B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CACX,oKAAoK,CACrK,CAAC;QACF,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,KAAK,CACX,qGAAqG,CACtG,CAAC;QACF,OAAO,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;QAC/F,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,iBAAiB,CAAC;QAClC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;QACjD,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;KAC5C,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAE7B,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,yBAAyB,OAAO,mBAAmB,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE;QACpD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;KAC5D,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;IAC7F,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;IAC5C,IAAI,UAAU,CAAC,2BAA2B,EAAE,MAAM,EAAE,CAAC;QACnD,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,2BAA2B,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;IAC5E,CAAC;SAAM,CAAC;QACN,MAAM,WAAW,GAAG,UAAU,CAAC,mBAAmB,CAAC,EAAE;YACnD,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,SAAS,UAAU,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACpF,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,gBAAgB,CAAC,UAAU,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACnE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAyB,CAAC;QAClD,MAAM,YAAY,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;QACtD,MAAM,WAAW,GAAG,oBAAoB,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,OAAO,CAAC,GAAG,CACT,kCAAkC,YAAY,CAAC,MAAM,uBAAuB,WAAW,CAAC,MAAM,kBAAkB,SAAS,EAAE,CAC5H,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAC5D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,YAAY,YAAY,CAAC,MAAM,gCAAgC,CAAC,CAAC;YAC7E,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;YACnE,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;gBACpB,OAAO,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,MAAM,CAAC,MAAM,6CAA6C,CAAC,CAAC;gBAC3F,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;oBACnC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,OAAO,CAAC,CAAC;YACX,CAAC;YACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,YAAY,YAAY,CAAC,MAAM,mCAAmC,CAAC,CAAC;gBAChF,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;gBACnE,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC;oBACtB,OAAO,CAAC,KAAK,CACX,KAAK,aAAa,CAAC,MAAM,CAAC,MAAM,iEAAiE,CAClG,CAAC;oBACF,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;wBACrC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;wBAC5B,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,aAAa,iBAAiB,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;oBACrF,CAAC;oBACD,OAAO,CAAC,KAAK,CACX,yFAAyF,CAC1F,CAAC;oBACF,OAAO,CAAC,CAAC;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAClE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,+BAA+B,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACvE,MAAM,SAAS,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,MAAM,8CAA8C,SAAS,EAAE,CAAC,CAAC;IAE7F,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QACtD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC1D,IAAI,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,YAAY,aAAa,CAAC,MAAM,2CAA2C,CAAC,CAAC;QACzF,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QACpE,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CACX,KAAK,aAAa,CAAC,MAAM,CAAC,MAAM,iEAAiE,CAClG,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;gBACrC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,aAAa,iBAAiB,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;YACrF,CAAC;YACD,OAAO,CAAC,KAAK,CACX,yFAAyF,CAC1F,CAAC;YACF,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,MAAM,kBAAkB,CAAC,CAAC;IACvE,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEjF,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,KAAK,CACX,KAAK,cAAc,CAAC,MAAM,CAAC,MAAM,0DAA0D,CAC5F,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;YACtC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,CAAC,KAAK,CACX,sFAAsF,CACvF,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,cAAc,CAAC,KAAK,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,CAAC;AACX,CAAC;AAED,IAAI,EAAE;KACH,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAClC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACb,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/validate.d.ts
CHANGED
|
@@ -27,7 +27,13 @@ export interface ValidationResult {
|
|
|
27
27
|
sitemapUnindexedPageUrls?: string[];
|
|
28
28
|
/** True when sitemap.xml was a sitemap index (child sitemaps were followed). */
|
|
29
29
|
sitemapWasIndex?: boolean;
|
|
30
|
+
/** Child sitemap URLs that returned 404 and were skipped (only when ignoreMissingChildSitemaps is used). */
|
|
31
|
+
skippedMissingChildSitemaps?: string[];
|
|
30
32
|
}
|
|
31
|
-
export
|
|
33
|
+
export interface ValidateOptions {
|
|
34
|
+
/** When resolving a sitemap index, treat 404 on a child `<loc>` as optional (skip that child). */
|
|
35
|
+
ignoreMissingChildSitemaps?: boolean;
|
|
36
|
+
}
|
|
37
|
+
export declare function validate(baseUrl: string, fetchInit: RequestInit, options?: ValidateOptions): Promise<ValidationResult>;
|
|
32
38
|
export declare function validationFailed(result: ValidationResult): boolean;
|
|
33
39
|
//# sourceMappingURL=validate.d.ts.map
|
package/dist/validate.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3D,mBAAmB,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACtF,OAAO,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAC1E,uFAAuF;IACvF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iGAAiG;IACjG,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,gFAAgF;IAChF,eAAe,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3D,mBAAmB,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACtF,OAAO,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAC1E,uFAAuF;IACvF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iGAAiG;IACjG,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,gFAAgF;IAChF,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4GAA4G;IAC5G,2BAA2B,CAAC,EAAE,MAAM,EAAE,CAAC;CACxC;AAED,MAAM,WAAW,eAAe;IAC9B,kGAAkG;IAClG,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACtC;AA0DD,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,WAAW,EACtB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAgD3B;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAElE"}
|
package/dist/validate.js
CHANGED
|
@@ -19,36 +19,53 @@ function extractLocUrls(xmlBody) {
|
|
|
19
19
|
function isSitemapIndex(body) {
|
|
20
20
|
return /<sitemapindex[\s>]/.test(body) || body.includes('<sitemap>');
|
|
21
21
|
}
|
|
22
|
-
async function resolveSitemapToPageUrls(body, fetchInit) {
|
|
22
|
+
async function resolveSitemapToPageUrls(body, fetchInit, options) {
|
|
23
|
+
const skippedMissingChildSitemaps = [];
|
|
24
|
+
const skip404Children = options?.ignoreMissingChildSitemaps === true;
|
|
23
25
|
if (!isSitemapIndex(body)) {
|
|
24
26
|
const pageUrls = extractLocUrls(body);
|
|
25
|
-
return {
|
|
27
|
+
return {
|
|
28
|
+
ok: pageUrls.length > 0 || hasLocTag(body),
|
|
29
|
+
pageUrls,
|
|
30
|
+
skippedMissingChildSitemaps,
|
|
31
|
+
};
|
|
26
32
|
}
|
|
27
33
|
const childUrls = extractLocUrls(body);
|
|
28
34
|
const allPageUrls = [];
|
|
29
35
|
for (const childUrl of childUrls) {
|
|
30
36
|
const res = await fetch(childUrl, fetchInit);
|
|
31
|
-
if (!res.ok)
|
|
32
|
-
|
|
37
|
+
if (!res.ok) {
|
|
38
|
+
if (skip404Children && res.status === 404) {
|
|
39
|
+
skippedMissingChildSitemaps.push(childUrl);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
return { ok: false, pageUrls: [], skippedMissingChildSitemaps };
|
|
43
|
+
}
|
|
33
44
|
const childBody = await res.text();
|
|
34
45
|
// Empty child sitemaps (0 <loc> entries) are valid; they contribute no URLs.
|
|
35
46
|
allPageUrls.push(...extractLocUrls(childBody));
|
|
36
47
|
}
|
|
37
|
-
return {
|
|
48
|
+
return {
|
|
49
|
+
ok: allPageUrls.length > 0,
|
|
50
|
+
pageUrls: allPageUrls,
|
|
51
|
+
skippedMissingChildSitemaps,
|
|
52
|
+
};
|
|
38
53
|
}
|
|
39
|
-
export async function validate(baseUrl, fetchInit) {
|
|
54
|
+
export async function validate(baseUrl, fetchInit, options) {
|
|
40
55
|
const base = baseUrl.replace(/\/$/, '');
|
|
41
56
|
const result = {
|
|
42
57
|
sitemapXml: { ok: false, status: 0 },
|
|
43
58
|
sitemapUnindexedXml: { ok: false, status: 0 },
|
|
44
59
|
llmsTxt: { ok: false, status: 0 },
|
|
45
60
|
};
|
|
61
|
+
const skippedChildren = [];
|
|
46
62
|
const sitemapRes = await fetch(`${base}/sitemap.xml`, fetchInit);
|
|
47
63
|
result.sitemapXml.status = sitemapRes.status;
|
|
48
64
|
if (sitemapRes.ok) {
|
|
49
65
|
const body = await sitemapRes.text();
|
|
50
66
|
result.sitemapXml.body = body;
|
|
51
|
-
const resolved = await resolveSitemapToPageUrls(body, fetchInit);
|
|
67
|
+
const resolved = await resolveSitemapToPageUrls(body, fetchInit, options);
|
|
68
|
+
skippedChildren.push(...resolved.skippedMissingChildSitemaps);
|
|
52
69
|
result.sitemapXml.ok = resolved.ok;
|
|
53
70
|
result.sitemapPageUrls = resolved.pageUrls;
|
|
54
71
|
result.sitemapWasIndex = isSitemapIndex(body);
|
|
@@ -61,10 +78,14 @@ export async function validate(baseUrl, fetchInit) {
|
|
|
61
78
|
else if (unindexedRes.ok) {
|
|
62
79
|
const body = await unindexedRes.text();
|
|
63
80
|
result.sitemapUnindexedXml.body = body;
|
|
64
|
-
const resolved = await resolveSitemapToPageUrls(body, fetchInit);
|
|
81
|
+
const resolved = await resolveSitemapToPageUrls(body, fetchInit, options);
|
|
82
|
+
skippedChildren.push(...resolved.skippedMissingChildSitemaps);
|
|
65
83
|
result.sitemapUnindexedXml.ok = resolved.ok;
|
|
66
84
|
result.sitemapUnindexedPageUrls = resolved.pageUrls;
|
|
67
85
|
}
|
|
86
|
+
if (skippedChildren.length > 0) {
|
|
87
|
+
result.skippedMissingChildSitemaps = skippedChildren;
|
|
88
|
+
}
|
|
68
89
|
const llmsRes = await fetch(`${base}/llms.txt`, fetchInit);
|
|
69
90
|
result.llmsTxt.status = llmsRes.status;
|
|
70
91
|
if (llmsRes.status === 404) {
|
package/dist/validate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAqBH,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,EAAE,GAAG,sBAAsB,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAChE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,IAAY,EACZ,SAAsB,EACtB,OAAyB;IAEzB,MAAM,2BAA2B,GAAa,EAAE,CAAC;IACjD,MAAM,eAAe,GAAG,OAAO,EAAE,0BAA0B,KAAK,IAAI,CAAC;IAErE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,OAAO;YACL,EAAE,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC;YAC1C,QAAQ;YACR,2BAA2B;SAC5B,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,IAAI,eAAe,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC1C,2BAA2B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,2BAA2B,EAAE,CAAC;QAClE,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACnC,6EAA6E;QAC7E,WAAW,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO;QACL,EAAE,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC;QAC1B,QAAQ,EAAE,WAAW;QACrB,2BAA2B;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,OAAe,EACf,SAAsB,EACtB,OAAyB;IAEzB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,MAAM,GAAqB;QAC/B,UAAU,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;QACpC,mBAAmB,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;QAC7C,OAAO,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;KAClC,CAAC;IACF,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,cAAc,EAAE,SAAS,CAAC,CAAC;IACjE,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAC7C,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACrC,MAAM,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC;QAC9B,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC1E,eAAe,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,2BAA2B,CAAC,CAAC;QAC9D,MAAM,CAAC,UAAU,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,CAAC,eAAe,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAC3C,MAAM,CAAC,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAC7E,MAAM,CAAC,mBAAmB,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IACxD,IAAI,YAAY,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,CAAC,mBAAmB,CAAC,MAAM,GAAG,IAAI,CAAC;IAC3C,CAAC;SAAM,IAAI,YAAY,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;QACvC,MAAM,CAAC,mBAAmB,CAAC,IAAI,GAAG,IAAI,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC1E,eAAe,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,2BAA2B,CAAC,CAAC;QAC9D,MAAM,CAAC,mBAAmB,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,wBAAwB,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACtD,CAAC;IAED,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,2BAA2B,GAAG,eAAe,CAAC;IACvD,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,WAAW,EAAE,SAAS,CAAC,CAAC;IAC3D,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACvC,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAC/B,CAAC;SAAM,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;QACtB,MAAM,CAAC,OAAO,CAAC,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAwB;IACvD,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACjF,CAAC"}
|