afdocs 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -21
- package/dist/checks/authentication/auth-alternative-access.d.ts +2 -0
- package/dist/checks/authentication/auth-alternative-access.d.ts.map +1 -0
- package/dist/checks/authentication/auth-alternative-access.js +17 -0
- package/dist/checks/authentication/auth-alternative-access.js.map +1 -0
- package/dist/checks/authentication/auth-gate-detection.d.ts +2 -0
- package/dist/checks/authentication/auth-gate-detection.d.ts.map +1 -0
- package/dist/checks/authentication/auth-gate-detection.js +17 -0
- package/dist/checks/authentication/auth-gate-detection.js.map +1 -0
- package/dist/checks/index.d.ts +2 -0
- package/dist/checks/index.d.ts.map +1 -1
- package/dist/checks/index.js +3 -0
- package/dist/checks/index.js.map +1 -1
- package/dist/checks/llms-txt/llms-txt-exists.js +83 -6
- package/dist/checks/llms-txt/llms-txt-exists.js.map +1 -1
- package/dist/checks/llms-txt/llms-txt-links-markdown.js +46 -21
- package/dist/checks/llms-txt/llms-txt-links-markdown.js.map +1 -1
- package/dist/checks/llms-txt/llms-txt-links-resolve.js +10 -3
- package/dist/checks/llms-txt/llms-txt-links-resolve.js.map +1 -1
- package/dist/checks/markdown-availability/content-negotiation.js +101 -5
- package/dist/checks/markdown-availability/content-negotiation.js.map +1 -1
- package/dist/checks/markdown-availability/markdown-url-support.js +82 -5
- package/dist/checks/markdown-availability/markdown-url-support.js.map +1 -1
- package/dist/checks/page-size/content-start-position.js +161 -5
- package/dist/checks/page-size/content-start-position.js.map +1 -1
- package/dist/checks/page-size/page-size-html.js +127 -5
- package/dist/checks/page-size/page-size-html.js.map +1 -1
- package/dist/checks/page-size/page-size-markdown.js +183 -5
- package/dist/checks/page-size/page-size-markdown.js.map +1 -1
- package/dist/cli/commands/check.d.ts.map +1 -1
- package/dist/cli/commands/check.js +9 -1
- package/dist/cli/commands/check.js.map +1 -1
- package/dist/cli/formatters/text.d.ts +4 -1
- package/dist/cli/formatters/text.d.ts.map +1 -1
- package/dist/cli/formatters/text.js +94 -1
- package/dist/cli/formatters/text.js.map +1 -1
- package/dist/constants.d.ts +6 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +3 -0
- package/dist/constants.js.map +1 -1
- package/dist/helpers/detect-markdown.d.ts +10 -0
- package/dist/helpers/detect-markdown.d.ts.map +1 -0
- package/dist/helpers/detect-markdown.js +29 -0
- package/dist/helpers/detect-markdown.js.map +1 -0
- package/dist/helpers/get-page-urls.d.ts +39 -0
- package/dist/helpers/get-page-urls.d.ts.map +1 -0
- package/dist/helpers/get-page-urls.js +230 -0
- package/dist/helpers/get-page-urls.js.map +1 -0
- package/dist/helpers/html-to-markdown.d.ts +7 -0
- package/dist/helpers/html-to-markdown.d.ts.map +1 -0
- package/dist/helpers/html-to-markdown.js +11 -0
- package/dist/helpers/html-to-markdown.js.map +1 -0
- package/dist/helpers/index.d.ts +5 -0
- package/dist/helpers/index.d.ts.map +1 -1
- package/dist/helpers/index.js +4 -0
- package/dist/helpers/index.js.map +1 -1
- package/dist/helpers/to-md-urls.d.ts +7 -0
- package/dist/helpers/to-md-urls.d.ts.map +1 -0
- package/dist/helpers/to-md-urls.js +24 -0
- package/dist/helpers/to-md-urls.js.map +1 -0
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +21 -14
- package/dist/runner.js.map +1 -1
- package/dist/types.d.ts +10 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -7,7 +7,12 @@ Test your documentation site against the [Agent-Friendly Documentation Spec](htt
|
|
|
7
7
|
|
|
8
8
|
Agents don't use docs like humans. They hit truncation limits, get walls of CSS instead of content, can't follow cross-host redirects, and don't know about quality-of-life improvements like `llms.txt` or `.md` docs pages that would make life swell. Maybe this is because the industry has lacked guidance - until now.
|
|
9
9
|
|
|
10
|
-
afdocs runs
|
|
10
|
+
afdocs runs 21 checks across 8 categories to evaluate how well your docs serve agent consumers. 10 are fully implemented; the rest return `skip` until completed.
|
|
11
|
+
|
|
12
|
+
> **Status: Early development (0.x)**
|
|
13
|
+
> This project is under active development. Check IDs, CLI flags, and output formats may change between minor versions. Feel free to try it out, but don't build automation against specific output until 1.0.
|
|
14
|
+
>
|
|
15
|
+
> Implements [spec v0.1.0](https://agentdocsspec.com/spec) (2026-02-22).
|
|
11
16
|
|
|
12
17
|
## Quick start
|
|
13
18
|
|
|
@@ -27,8 +32,12 @@ llms-txt
|
|
|
27
32
|
✓ llms-txt-links-resolve: All 50 tested links resolve (177 total links)
|
|
28
33
|
✓ llms-txt-links-markdown: 50/50 links point to markdown content (100%)
|
|
29
34
|
|
|
35
|
+
Markdown Availability
|
|
36
|
+
✗ content-negotiation: Server ignores Accept: text/markdown header (0/50 sampled pages return markdown)
|
|
37
|
+
✗ markdown-url-support: No sampled pages support .md URLs (0/50 tested)
|
|
38
|
+
|
|
30
39
|
Summary
|
|
31
|
-
5 passed, 14 skipped (
|
|
40
|
+
5 passed, 2 failed, 14 skipped (21 total)
|
|
32
41
|
```
|
|
33
42
|
|
|
34
43
|
## Install
|
|
@@ -55,15 +64,16 @@ afdocs check https://docs.example.com --pass-threshold 30000 --fail-threshold 80
|
|
|
55
64
|
|
|
56
65
|
### Options
|
|
57
66
|
|
|
58
|
-
| Option | Default | Description
|
|
59
|
-
| ----------------------- | -------- |
|
|
60
|
-
| `--format <format>` | `text` | Output format: `text` or `json`
|
|
61
|
-
|
|
|
62
|
-
| `--
|
|
63
|
-
| `--
|
|
64
|
-
| `--
|
|
65
|
-
| `--
|
|
66
|
-
| `--
|
|
67
|
+
| Option | Default | Description |
|
|
68
|
+
| ----------------------- | -------- | -------------------------------------------- |
|
|
69
|
+
| `--format <format>` | `text` | Output format: `text` or `json` |
|
|
70
|
+
| `-v, --verbose` | | Show per-page details for checks with issues |
|
|
71
|
+
| `--checks <ids>` | all | Comma-separated list of check IDs |
|
|
72
|
+
| `--max-concurrency <n>` | `3` | Maximum concurrent HTTP requests |
|
|
73
|
+
| `--request-delay <ms>` | `200` | Delay between requests |
|
|
74
|
+
| `--max-links <n>` | `50` | Maximum links to test in link checks |
|
|
75
|
+
| `--pass-threshold <n>` | `50000` | Size pass threshold (characters) |
|
|
76
|
+
| `--fail-threshold <n>` | `100000` | Size fail threshold (characters) |
|
|
67
77
|
|
|
68
78
|
### Exit codes
|
|
69
79
|
|
|
@@ -128,7 +138,7 @@ describe('agent-friendliness', () => {
|
|
|
128
138
|
|
|
129
139
|
## Checks
|
|
130
140
|
|
|
131
|
-
|
|
141
|
+
21 checks across 8 categories. Checks marked with \* are stub implementations that return `skip`.
|
|
132
142
|
|
|
133
143
|
### Category 1: llms.txt
|
|
134
144
|
|
|
@@ -142,18 +152,18 @@ describe('agent-friendliness', () => {
|
|
|
142
152
|
|
|
143
153
|
### Category 2: Markdown Availability
|
|
144
154
|
|
|
145
|
-
| Check
|
|
146
|
-
|
|
|
147
|
-
| `markdown-url-support`
|
|
148
|
-
| `content-negotiation`
|
|
155
|
+
| Check | Description |
|
|
156
|
+
| ---------------------- | ------------------------------------------------- |
|
|
157
|
+
| `markdown-url-support` | Whether `.md` URL variants return markdown |
|
|
158
|
+
| `content-negotiation` | Whether the server honors `Accept: text/markdown` |
|
|
149
159
|
|
|
150
160
|
### Category 3: Page Size and Truncation Risk
|
|
151
161
|
|
|
152
|
-
| Check
|
|
153
|
-
|
|
|
154
|
-
| `page-size-markdown`
|
|
155
|
-
| `page-size-html`
|
|
156
|
-
| `content-start-position`
|
|
162
|
+
| Check | Description |
|
|
163
|
+
| ------------------------ | ------------------------------------------------ |
|
|
164
|
+
| `page-size-markdown` | Character count when served as markdown |
|
|
165
|
+
| `page-size-html` | Character count of HTML and post-conversion size |
|
|
166
|
+
| `content-start-position` | How far into the response actual content begins |
|
|
157
167
|
|
|
158
168
|
### Category 4: Content Structure
|
|
159
169
|
|
|
@@ -184,6 +194,13 @@ describe('agent-friendliness', () => {
|
|
|
184
194
|
| `markdown-content-parity` \* | Whether markdown and HTML versions match |
|
|
185
195
|
| `cache-header-hygiene` \* | Whether cache headers allow timely updates |
|
|
186
196
|
|
|
197
|
+
### Category 8: Authentication and Access
|
|
198
|
+
|
|
199
|
+
| Check | Description |
|
|
200
|
+
| ---------------------------- | -------------------------------------------------------------------- |
|
|
201
|
+
| `auth-gate-detection` \* | Whether documentation pages require authentication to access content |
|
|
202
|
+
| `auth-alternative-access` \* | Whether auth-gated sites provide alternative access paths for agents |
|
|
203
|
+
|
|
187
204
|
## Check dependencies
|
|
188
205
|
|
|
189
206
|
Some checks depend on others. If a dependency doesn't pass, the dependent check is skipped automatically.
|
|
@@ -194,6 +211,7 @@ Some checks depend on others. If a dependency doesn't pass, the dependent check
|
|
|
194
211
|
- `markdown-code-fence-validity` requires `markdown-url-support` or `content-negotiation`
|
|
195
212
|
- `llms-txt-freshness` requires `llms-txt-exists`
|
|
196
213
|
- `markdown-content-parity` requires `markdown-url-support` or `content-negotiation`
|
|
214
|
+
- `auth-alternative-access` requires `auth-gate-detection` (warn or fail)
|
|
197
215
|
|
|
198
216
|
## Responsible use
|
|
199
217
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth-alternative-access.d.ts","sourceRoot":"","sources":["../../../src/checks/authentication/auth-alternative-access.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { registerCheck } from '../registry.js';
|
|
2
|
+
async function check(_ctx) {
|
|
3
|
+
return {
|
|
4
|
+
id: 'auth-alternative-access',
|
|
5
|
+
category: 'authentication',
|
|
6
|
+
status: 'skip',
|
|
7
|
+
message: 'Not yet implemented',
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
registerCheck({
|
|
11
|
+
id: 'auth-alternative-access',
|
|
12
|
+
category: 'authentication',
|
|
13
|
+
description: 'Whether an auth-gated documentation site provides alternative access paths for agents',
|
|
14
|
+
dependsOn: [['auth-gate-detection']],
|
|
15
|
+
run: check,
|
|
16
|
+
});
|
|
17
|
+
//# sourceMappingURL=auth-alternative-access.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth-alternative-access.js","sourceRoot":"","sources":["../../../src/checks/authentication/auth-alternative-access.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,KAAK,UAAU,KAAK,CAAC,IAAkB;IACrC,OAAO;QACL,EAAE,EAAE,yBAAyB;QAC7B,QAAQ,EAAE,gBAAgB;QAC1B,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,qBAAqB;KAC/B,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,yBAAyB;IAC7B,QAAQ,EAAE,gBAAgB;IAC1B,WAAW,EACT,uFAAuF;IACzF,SAAS,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC;IACpC,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth-gate-detection.d.ts","sourceRoot":"","sources":["../../../src/checks/authentication/auth-gate-detection.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { registerCheck } from '../registry.js';
|
|
2
|
+
async function check(_ctx) {
|
|
3
|
+
return {
|
|
4
|
+
id: 'auth-gate-detection',
|
|
5
|
+
category: 'authentication',
|
|
6
|
+
status: 'skip',
|
|
7
|
+
message: 'Not yet implemented',
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
registerCheck({
|
|
11
|
+
id: 'auth-gate-detection',
|
|
12
|
+
category: 'authentication',
|
|
13
|
+
description: 'Whether documentation pages require authentication to access content',
|
|
14
|
+
dependsOn: [],
|
|
15
|
+
run: check,
|
|
16
|
+
});
|
|
17
|
+
//# sourceMappingURL=auth-gate-detection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth-gate-detection.js","sourceRoot":"","sources":["../../../src/checks/authentication/auth-gate-detection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,KAAK,UAAU,KAAK,CAAC,IAAkB;IACrC,OAAO;QACL,EAAE,EAAE,qBAAqB;QACzB,QAAQ,EAAE,gBAAgB;QAC1B,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,qBAAqB;KAC/B,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,qBAAqB;IACzB,QAAQ,EAAE,gBAAgB;IAC1B,WAAW,EAAE,sEAAsE;IACnF,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}
|
package/dist/checks/index.d.ts
CHANGED
|
@@ -17,6 +17,8 @@ import './agent-discoverability/llms-txt-directive.js';
|
|
|
17
17
|
import './observability/llms-txt-freshness.js';
|
|
18
18
|
import './observability/markdown-content-parity.js';
|
|
19
19
|
import './observability/cache-header-hygiene.js';
|
|
20
|
+
import './authentication/auth-gate-detection.js';
|
|
21
|
+
import './authentication/auth-alternative-access.js';
|
|
20
22
|
export { getCheck, getAllChecks, getChecksSorted } from './registry.js';
|
|
21
23
|
export { extractMarkdownLinks } from './llms-txt/llms-txt-valid.js';
|
|
22
24
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAGA,OAAO,+BAA+B,CAAC;AACvC,OAAO,8BAA8B,CAAC;AACtC,OAAO,6BAA6B,CAAC;AACrC,OAAO,sCAAsC,CAAC;AAC9C,OAAO,uCAAuC,CAAC;AAG/C,OAAO,iDAAiD,CAAC;AACzD,OAAO,gDAAgD,CAAC;AAGxD,OAAO,mCAAmC,CAAC;AAC3C,OAAO,+BAA+B,CAAC;AACvC,OAAO,uCAAuC,CAAC;AAG/C,OAAO,qDAAqD,CAAC;AAC7D,OAAO,+CAA+C,CAAC;AACvD,OAAO,qDAAqD,CAAC;AAG7D,OAAO,sCAAsC,CAAC;AAC9C,OAAO,sCAAsC,CAAC;AAG9C,OAAO,+CAA+C,CAAC;AAGvD,OAAO,uCAAuC,CAAC;AAC/C,OAAO,4CAA4C,CAAC;AACpD,OAAO,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAGA,OAAO,+BAA+B,CAAC;AACvC,OAAO,8BAA8B,CAAC;AACtC,OAAO,6BAA6B,CAAC;AACrC,OAAO,sCAAsC,CAAC;AAC9C,OAAO,uCAAuC,CAAC;AAG/C,OAAO,iDAAiD,CAAC;AACzD,OAAO,gDAAgD,CAAC;AAGxD,OAAO,mCAAmC,CAAC;AAC3C,OAAO,+BAA+B,CAAC;AACvC,OAAO,uCAAuC,CAAC;AAG/C,OAAO,qDAAqD,CAAC;AAC7D,OAAO,+CAA+C,CAAC;AACvD,OAAO,qDAAqD,CAAC;AAG7D,OAAO,sCAAsC,CAAC;AAC9C,OAAO,sCAAsC,CAAC;AAG9C,OAAO,+CAA+C,CAAC;AAGvD,OAAO,uCAAuC,CAAC;AAC/C,OAAO,4CAA4C,CAAC;AACpD,OAAO,yCAAyC,CAAC;AAGjD,OAAO,yCAAyC,CAAC;AACjD,OAAO,6CAA6C,CAAC;AAErD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC"}
|
package/dist/checks/index.js
CHANGED
|
@@ -25,6 +25,9 @@ import './agent-discoverability/llms-txt-directive.js';
|
|
|
25
25
|
import './observability/llms-txt-freshness.js';
|
|
26
26
|
import './observability/markdown-content-parity.js';
|
|
27
27
|
import './observability/cache-header-hygiene.js';
|
|
28
|
+
// Category 8: Authentication
|
|
29
|
+
import './authentication/auth-gate-detection.js';
|
|
30
|
+
import './authentication/auth-alternative-access.js';
|
|
28
31
|
export { getCheck, getAllChecks, getChecksSorted } from './registry.js';
|
|
29
32
|
export { extractMarkdownLinks } from './llms-txt/llms-txt-valid.js';
|
|
30
33
|
//# sourceMappingURL=index.js.map
|
package/dist/checks/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAAA,mDAAmD;AAEnD,uBAAuB;AACvB,OAAO,+BAA+B,CAAC;AACvC,OAAO,8BAA8B,CAAC;AACtC,OAAO,6BAA6B,CAAC;AACrC,OAAO,sCAAsC,CAAC;AAC9C,OAAO,uCAAuC,CAAC;AAE/C,oCAAoC;AACpC,OAAO,iDAAiD,CAAC;AACzD,OAAO,gDAAgD,CAAC;AAExD,wBAAwB;AACxB,OAAO,mCAAmC,CAAC;AAC3C,OAAO,+BAA+B,CAAC;AACvC,OAAO,uCAAuC,CAAC;AAE/C,gCAAgC;AAChC,OAAO,qDAAqD,CAAC;AAC7D,OAAO,+CAA+C,CAAC;AACvD,OAAO,qDAAqD,CAAC;AAE7D,4BAA4B;AAC5B,OAAO,sCAAsC,CAAC;AAC9C,OAAO,sCAAsC,CAAC;AAE9C,oCAAoC;AACpC,OAAO,+CAA+C,CAAC;AAEvD,4BAA4B;AAC5B,OAAO,uCAAuC,CAAC;AAC/C,OAAO,4CAA4C,CAAC;AACpD,OAAO,yCAAyC,CAAC;AAEjD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAAA,mDAAmD;AAEnD,uBAAuB;AACvB,OAAO,+BAA+B,CAAC;AACvC,OAAO,8BAA8B,CAAC;AACtC,OAAO,6BAA6B,CAAC;AACrC,OAAO,sCAAsC,CAAC;AAC9C,OAAO,uCAAuC,CAAC;AAE/C,oCAAoC;AACpC,OAAO,iDAAiD,CAAC;AACzD,OAAO,gDAAgD,CAAC;AAExD,wBAAwB;AACxB,OAAO,mCAAmC,CAAC;AAC3C,OAAO,+BAA+B,CAAC;AACvC,OAAO,uCAAuC,CAAC;AAE/C,gCAAgC;AAChC,OAAO,qDAAqD,CAAC;AAC7D,OAAO,+CAA+C,CAAC;AACvD,OAAO,qDAAqD,CAAC;AAE7D,4BAA4B;AAC5B,OAAO,sCAAsC,CAAC;AAC9C,OAAO,sCAAsC,CAAC;AAE9C,oCAAoC;AACpC,OAAO,+CAA+C,CAAC;AAEvD,4BAA4B;AAC5B,OAAO,uCAAuC,CAAC;AAC/C,OAAO,4CAA4C,CAAC;AACpD,OAAO,yCAAyC,CAAC;AAEjD,6BAA6B;AAC7B,OAAO,yCAAyC,CAAC;AACjD,OAAO,6CAA6C,CAAC;AAErD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC"}
|
|
@@ -39,7 +39,8 @@ async function checkLlmsTxtExists(ctx) {
|
|
|
39
39
|
if (response.ok && isText) {
|
|
40
40
|
const content = await response.text();
|
|
41
41
|
// Check it's not an HTML error page
|
|
42
|
-
const
|
|
42
|
+
const trimmed = content.trimStart().toLowerCase();
|
|
43
|
+
const looksLikeHtml = trimmed.startsWith('<!') || trimmed.startsWith('<html');
|
|
43
44
|
if (!looksLikeHtml && content.trim().length > 0) {
|
|
44
45
|
const crossHost = response.redirected && isCrossHostRedirect(url, response.url);
|
|
45
46
|
discovered.push({
|
|
@@ -53,21 +54,97 @@ async function checkLlmsTxtExists(ctx) {
|
|
|
53
54
|
}
|
|
54
55
|
}
|
|
55
56
|
}
|
|
56
|
-
catch {
|
|
57
|
-
checkedUrls.push({
|
|
57
|
+
catch (err) {
|
|
58
|
+
checkedUrls.push({
|
|
59
|
+
url,
|
|
60
|
+
status: 0,
|
|
61
|
+
redirected: false,
|
|
62
|
+
error: err instanceof Error ? err.message : String(err),
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// When no llms.txt found, check if any candidates redirected cross-host.
|
|
67
|
+
// If so, try {redirected_origin}/llms.txt as a fallback.
|
|
68
|
+
const redirectedOrigins = [];
|
|
69
|
+
if (discovered.length === 0) {
|
|
70
|
+
const checkedSet = new Set(checkedUrls.map((u) => u.url));
|
|
71
|
+
const seenOrigins = new Set();
|
|
72
|
+
for (const checked of checkedUrls) {
|
|
73
|
+
if (checked.finalUrl && isCrossHostRedirect(checked.url, checked.finalUrl)) {
|
|
74
|
+
try {
|
|
75
|
+
const redirectedOrigin = new URL(checked.finalUrl).origin;
|
|
76
|
+
const fallbackUrl = `${redirectedOrigin}/llms.txt`;
|
|
77
|
+
if (!checkedSet.has(fallbackUrl) && !seenOrigins.has(redirectedOrigin)) {
|
|
78
|
+
seenOrigins.add(redirectedOrigin);
|
|
79
|
+
checkedSet.add(fallbackUrl);
|
|
80
|
+
redirectedOrigins.push(redirectedOrigin);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
/* malformed URL */
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for (const redirectedOrigin of redirectedOrigins) {
|
|
89
|
+
const fallbackUrl = `${redirectedOrigin}/llms.txt`;
|
|
90
|
+
try {
|
|
91
|
+
const response = await ctx.http.fetch(fallbackUrl);
|
|
92
|
+
const contentType = response.headers.get('content-type') ?? '';
|
|
93
|
+
const isText = contentType.includes('text/');
|
|
94
|
+
checkedUrls.push({
|
|
95
|
+
url: fallbackUrl,
|
|
96
|
+
status: response.status,
|
|
97
|
+
redirected: response.redirected,
|
|
98
|
+
finalUrl: response.redirected ? response.url : undefined,
|
|
99
|
+
});
|
|
100
|
+
if (response.ok && isText) {
|
|
101
|
+
const content = await response.text();
|
|
102
|
+
const trimmed = content.trimStart().toLowerCase();
|
|
103
|
+
const looksLikeHtml = trimmed.startsWith('<!') || trimmed.startsWith('<html');
|
|
104
|
+
if (!looksLikeHtml && content.trim().length > 0) {
|
|
105
|
+
discovered.push({
|
|
106
|
+
url: fallbackUrl,
|
|
107
|
+
content,
|
|
108
|
+
status: response.status,
|
|
109
|
+
redirected: false,
|
|
110
|
+
crossHostRedirect: true,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
checkedUrls.push({
|
|
117
|
+
url: fallbackUrl,
|
|
118
|
+
status: 0,
|
|
119
|
+
redirected: false,
|
|
120
|
+
error: err instanceof Error ? err.message : String(err),
|
|
121
|
+
});
|
|
122
|
+
}
|
|
58
123
|
}
|
|
59
124
|
}
|
|
125
|
+
const fetchErrors = checkedUrls.filter((u) => u.error).length;
|
|
126
|
+
const rateLimited = checkedUrls.filter((u) => u.status === 429).length;
|
|
127
|
+
const suffix = (fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '') +
|
|
128
|
+
(rateLimited > 0 ? `; ${rateLimited} rate-limited (HTTP 429)` : '');
|
|
60
129
|
// Store discovered files for downstream checks
|
|
61
130
|
const details = {
|
|
62
131
|
candidateUrls: checkedUrls,
|
|
63
132
|
discoveredFiles: discovered,
|
|
133
|
+
fetchErrors,
|
|
134
|
+
rateLimited,
|
|
64
135
|
};
|
|
136
|
+
if (redirectedOrigins.length > 0) {
|
|
137
|
+
details.redirectedOrigins = redirectedOrigins;
|
|
138
|
+
}
|
|
65
139
|
if (discovered.length === 0) {
|
|
140
|
+
const redirectNote = redirectedOrigins.length > 0
|
|
141
|
+
? `; candidates redirected cross-host to ${redirectedOrigins.join(', ')} (agents can't follow cross-host redirects)`
|
|
142
|
+
: '';
|
|
66
143
|
return {
|
|
67
144
|
id: 'llms-txt-exists',
|
|
68
145
|
category: 'llms-txt',
|
|
69
146
|
status: 'fail',
|
|
70
|
-
message: `No llms.txt found at any candidate location (${candidates.join(', ')})`,
|
|
147
|
+
message: `No llms.txt found at any candidate location (${candidates.join(', ')})${redirectNote}${suffix}`,
|
|
71
148
|
details,
|
|
72
149
|
};
|
|
73
150
|
}
|
|
@@ -78,7 +155,7 @@ async function checkLlmsTxtExists(ctx) {
|
|
|
78
155
|
id: 'llms-txt-exists',
|
|
79
156
|
category: 'llms-txt',
|
|
80
157
|
status: 'warn',
|
|
81
|
-
message:
|
|
158
|
+
message: `llms.txt found but only reachable via cross-host redirect (agents may not follow it)${suffix}`,
|
|
82
159
|
details,
|
|
83
160
|
};
|
|
84
161
|
}
|
|
@@ -93,7 +170,7 @@ async function checkLlmsTxtExists(ctx) {
|
|
|
93
170
|
id: 'llms-txt-exists',
|
|
94
171
|
category: 'llms-txt',
|
|
95
172
|
status: 'pass',
|
|
96
|
-
message: `llms.txt found at ${discovered.length} location(s)`,
|
|
173
|
+
message: `llms.txt found at ${discovered.length} location(s)${suffix}`,
|
|
97
174
|
details,
|
|
98
175
|
};
|
|
99
176
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llms-txt-exists.js","sourceRoot":"","sources":["../../../src/checks/llms-txt/llms-txt-exists.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,OAAe,EAAE,MAAc;IACvD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,UAAU,CAAC,GAAG,CAAC,GAAG,OAAO,WAAW,CAAC,CAAC;IACtC,UAAU,CAAC,GAAG,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC;IACrC,UAAU,CAAC,GAAG,CAAC,GAAG,MAAM,gBAAgB,CAAC,CAAC;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAmB,EAAE,QAAgB;IAChE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjC,OAAO,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,GAAiB;IACjD,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"llms-txt-exists.js","sourceRoot":"","sources":["../../../src/checks/llms-txt/llms-txt-exists.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,OAAe,EAAE,MAAc;IACvD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,UAAU,CAAC,GAAG,CAAC,GAAG,OAAO,WAAW,CAAC,CAAC;IACtC,UAAU,CAAC,GAAG,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC;IACrC,UAAU,CAAC,GAAG,CAAC,GAAG,MAAM,gBAAgB,CAAC,CAAC;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAmB,EAAE,QAAgB;IAChE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjC,OAAO,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,GAAiB;IACjD,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,MAAM,WAAW,GAMZ,EAAE,CAAC;IAER,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7C,WAAW,CAAC,IAAI,CAAC;gBACf,GAAG;gBACH,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;aACzD,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtC,oCAAoC;gBACpC,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,WAAW,EAAE,CAAC;gBAClD,MAAM,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAC9E,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChD,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,IAAI,mBAAmB,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;oBAChF,UAAU,CAAC,IAAI,CAAC;wBACd,GAAG;wBACH,OAAO;wBACP,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;wBAC/B,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;wBAC3D,iBAAiB,EAAE,SAAS;qBAC7B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,WAAW,CAAC,IAAI,CAAC;gBACf,GAAG;gBACH,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,KAAK;gBACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,yDAAyD;IACzD,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YAClC,IAAI,OAAO,CAAC,QAAQ,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3E,IAAI,CAAC;oBACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;oBAC1D,MAAM,WAAW,GAAG,GAAG,gBAAgB,WAAW,CAAC;oBACnD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;wBACvE,WAAW,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;wBAClC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;wBAC5B,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;oBAC3C,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,mBAAmB;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;YACjD,MAAM,WAAW,GAAG,GAAG,gBAAgB,WAAW,CAAC;YACnD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACnD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;gBAC/D,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAE7C,WAAW,CAAC,IAAI,CAAC;oBACf,GAAG,EAAE,WAAW;oBAChB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;iBACzD,CAAC,CAAC;gBAEH,IAAI,QAAQ,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;oBAC1B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACtC,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,WAAW,EAAE,CAAC;oBAClD,MAAM,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;oBAC9E,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChD,UAAU,CAAC,IAAI,CAAC;4BACd,GAAG,EAAE,WAAW;4BAChB,OAAO;4BACP,MAAM,EAAE,QAAQ,CAAC,MAAM;4BACvB,UAAU,EAAE,KAAK;4BACjB,iBAAiB,EAAE,IAAI;yBACxB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,WAAW,CAAC,IAAI,CAAC;oBACf,GAAG,EAAE,WAAW;oBAChB,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,KAAK;oBACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;IAEvE,MAAM,MAAM,GACV,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,0BAA0B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEtE,+CAA+C;IAC/C,MAAM,OAAO,GAA4B;QACvC,aAAa,EAAE,WAAW;QAC1B,eAAe,EAAE,UAAU;QAC3B,WAAW;QACX,WAAW;KACZ,CAAC;IAEF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAChD,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,YAAY,GAChB,iBAAiB,CAAC,MAAM,GAAG,CAAC;YAC1B,CAAC,CAAC,yCAAyC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,6CAA6C;YACpH,CAAC,CAAC,EAAE,CAAC;QACT,OAAO;YACL,EAAE,EAAE,iBAAiB;YACrB,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,gDAAgD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,GAAG,MAAM,EAAE;YACzG,OAAO;SACR,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAClE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO;YACL,EAAE,EAAE,iBAAiB;YACrB,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uFAAuF,MAAM,EAAE;YACxG,OAAO;SACR,CAAC;IACJ,CAAC;IAED,qDAAqD;IACrD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACjC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC;IAChC,CAAC;IAED,OAAO;QACL,EAAE,EAAE,iBAAiB;QACrB,QAAQ,EAAE,UAAU;QACpB,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,qBAAqB,UAAU,CAAC,MAAM,eAAe,MAAM,EAAE;QACtE,OAAO;KACR,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,iBAAiB;IACrB,QAAQ,EAAE,UAAU;IACpB,WAAW,EAAE,oEAAoE;IACjF,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,kBAAkB;CACxB,CAAC,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { registerCheck } from '../registry.js';
|
|
2
2
|
import { extractMarkdownLinks } from './llms-txt-valid.js';
|
|
3
|
+
import { toMdUrls } from '../../helpers/to-md-urls.js';
|
|
3
4
|
function hasMarkdownExtension(url) {
|
|
4
5
|
try {
|
|
5
6
|
const pathname = new URL(url).pathname;
|
|
@@ -42,7 +43,8 @@ async function checkLlmsTxtLinksMarkdown(ctx) {
|
|
|
42
43
|
// Sample if too many
|
|
43
44
|
let linksToTest = Array.from(allLinks);
|
|
44
45
|
const totalLinks = linksToTest.length;
|
|
45
|
-
|
|
46
|
+
const wasSampled = totalLinks > ctx.options.maxLinksToTest;
|
|
47
|
+
if (wasSampled) {
|
|
46
48
|
for (let i = linksToTest.length - 1; i > 0; i--) {
|
|
47
49
|
const j = Math.floor(Math.random() * (i + 1));
|
|
48
50
|
[linksToTest[i], linksToTest[j]] = [linksToTest[j], linksToTest[i]];
|
|
@@ -66,33 +68,48 @@ async function checkLlmsTxtLinksMarkdown(ctx) {
|
|
|
66
68
|
});
|
|
67
69
|
const contentType = response.headers.get('content-type') ?? '';
|
|
68
70
|
if (contentType.includes('text/markdown')) {
|
|
69
|
-
return {
|
|
71
|
+
return {
|
|
72
|
+
url,
|
|
73
|
+
hasMarkdownExtension: false,
|
|
74
|
+
servesMarkdown: true,
|
|
75
|
+
status: response.status,
|
|
76
|
+
};
|
|
70
77
|
}
|
|
71
|
-
// Try .md variant
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
78
|
+
// Try .md variant candidates
|
|
79
|
+
const candidates = toMdUrls(url);
|
|
80
|
+
for (const mdUrl of candidates) {
|
|
81
|
+
try {
|
|
82
|
+
const mdResponse = await ctx.http.fetch(mdUrl, { method: 'HEAD' });
|
|
83
|
+
if (mdResponse.ok) {
|
|
84
|
+
return {
|
|
85
|
+
url,
|
|
86
|
+
hasMarkdownExtension: false,
|
|
87
|
+
servesMarkdown: false,
|
|
88
|
+
status: response.status,
|
|
89
|
+
mdVariantAvailable: true,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// Try next candidate
|
|
82
95
|
}
|
|
83
|
-
}
|
|
84
|
-
catch {
|
|
85
|
-
// .md variant not available
|
|
86
96
|
}
|
|
87
97
|
return {
|
|
88
98
|
url,
|
|
89
99
|
hasMarkdownExtension: false,
|
|
90
100
|
servesMarkdown: false,
|
|
101
|
+
status: response.status,
|
|
91
102
|
mdVariantAvailable: false,
|
|
92
103
|
};
|
|
93
104
|
}
|
|
94
|
-
catch {
|
|
95
|
-
return {
|
|
105
|
+
catch (err) {
|
|
106
|
+
return {
|
|
107
|
+
url,
|
|
108
|
+
hasMarkdownExtension: false,
|
|
109
|
+
servesMarkdown: false,
|
|
110
|
+
status: 0,
|
|
111
|
+
error: err instanceof Error ? err.message : String(err),
|
|
112
|
+
};
|
|
96
113
|
}
|
|
97
114
|
}));
|
|
98
115
|
results.push(...batchResults);
|
|
@@ -100,20 +117,28 @@ async function checkLlmsTxtLinksMarkdown(ctx) {
|
|
|
100
117
|
const markdownLinks = results.filter((r) => r.hasMarkdownExtension || r.servesMarkdown).length;
|
|
101
118
|
const mdVariantsAvailable = results.filter((r) => r.mdVariantAvailable).length;
|
|
102
119
|
const markdownRate = results.length > 0 ? markdownLinks / results.length : 0;
|
|
120
|
+
const fetchErrors = results.filter((r) => r.error).length;
|
|
121
|
+
const rateLimited = results.filter((r) => r.status === 429).length;
|
|
122
|
+
const linkLabel = wasSampled ? 'sampled links' : 'links';
|
|
123
|
+
const suffix = (fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '') +
|
|
124
|
+
(rateLimited > 0 ? `; ${rateLimited} rate-limited (HTTP 429)` : '');
|
|
103
125
|
const details = {
|
|
104
126
|
totalLinks,
|
|
105
127
|
testedLinks: results.length,
|
|
128
|
+
sampled: wasSampled,
|
|
106
129
|
markdownLinks,
|
|
107
130
|
htmlLinks: results.length - markdownLinks,
|
|
108
131
|
mdVariantsAvailable,
|
|
109
132
|
markdownRate: Math.round(markdownRate * 100),
|
|
133
|
+
fetchErrors,
|
|
134
|
+
rateLimited,
|
|
110
135
|
};
|
|
111
136
|
if (markdownRate >= 0.9) {
|
|
112
137
|
return {
|
|
113
138
|
id: 'llms-txt-links-markdown',
|
|
114
139
|
category: 'llms-txt',
|
|
115
140
|
status: 'pass',
|
|
116
|
-
message: `${markdownLinks}/${results.length}
|
|
141
|
+
message: `${markdownLinks}/${results.length} ${linkLabel} point to markdown content (${Math.round(markdownRate * 100)}%)${suffix}`,
|
|
117
142
|
details,
|
|
118
143
|
};
|
|
119
144
|
}
|
|
@@ -122,7 +147,7 @@ async function checkLlmsTxtLinksMarkdown(ctx) {
|
|
|
122
147
|
id: 'llms-txt-links-markdown',
|
|
123
148
|
category: 'llms-txt',
|
|
124
149
|
status: 'warn',
|
|
125
|
-
message: `Links point to HTML, but ${mdVariantsAvailable} have .md variants available`,
|
|
150
|
+
message: `Links point to HTML, but ${mdVariantsAvailable} have .md variants available${suffix}`,
|
|
126
151
|
details,
|
|
127
152
|
};
|
|
128
153
|
}
|
|
@@ -130,7 +155,7 @@ async function checkLlmsTxtLinksMarkdown(ctx) {
|
|
|
130
155
|
id: 'llms-txt-links-markdown',
|
|
131
156
|
category: 'llms-txt',
|
|
132
157
|
status: 'fail',
|
|
133
|
-
message:
|
|
158
|
+
message: `Links point to HTML and no markdown alternatives detected${suffix}`,
|
|
134
159
|
details,
|
|
135
160
|
};
|
|
136
161
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llms-txt-links-markdown.js","sourceRoot":"","sources":["../../../src/checks/llms-txt/llms-txt-links-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"llms-txt-links-markdown.js","sourceRoot":"","sources":["../../../src/checks/llms-txt/llms-txt-links-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAYvD,SAAS,oBAAoB,CAAC,GAAW;IACvC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACvC,OAAO,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,GAAiB;IACxD,MAAM,YAAY,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,OAAO,EAAE,eAAe,IAAI,EAAE,CAAqB,CAAC;IAEtF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,EAAE,EAAE,yBAAyB;YAC7B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,sCAAsC;YAC/C,SAAS,EAAE,CAAC,iBAAiB,CAAC;SAC/B,CAAC;IACJ,CAAC;IAED,uBAAuB;IACvB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,EAAE,EAAE,yBAAyB;YAC7B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,oCAAoC;SAC9C,CAAC;IACJ,CAAC;IAED,qBAAqB;IACrB,IAAI,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,UAAU,GAAG,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAC3D,IAAI,UAAU,EAAE,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAA+B,EAAE;YACnD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAE3C,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,EAAE,GAAG,EAAE,oBAAoB,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;YACnE,CAAC;YAED,oDAAoD;YACpD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;oBACzC,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;iBACrC,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;gBAC/D,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;oBAC1C,OAAO;wBACL,GAAG;wBACH,oBAAoB,EAAE,KAAK;wBAC3B,cAAc,EAAE,IAAI;wBACpB,MAAM,EAAE,QAAQ,CAAC,MAAM;qBACxB,CAAC;gBACJ,CAAC;gBAED,6BAA6B;gBAC7B,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACjC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,IAAI,CAAC;wBACH,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;wBACnE,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;4BAClB,OAAO;gCACL,GAAG;gCACH,oBAAoB,EAAE,KAAK;gCAC3B,cAAc,EAAE,KAAK;gCACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;gCACvB,kBAAkB,EAAE,IAAI;6BACzB,CAAC;wBACJ,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,qBAAqB;oBACvB,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,GAAG;oBACH,oBAAoB,EAAE,KAAK;oBAC3B,cAAc,EAAE,KAAK;oBACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,kBAAkB,EAAE,KAAK;iBAC1B,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,GAAG;oBACH,oBAAoB,EAAE,KAAK;oBAC3B,cAAc,EAAE,KAAK;oBACrB,MAAM,EAAE,CAAC;oBACT,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC;IAC/F,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC/E,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;IAEnE,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC;IACzD,MAAM,MAAM,GACV,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,0BAA0B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEtE,MAAM,OAAO,GAA4B;QACvC,UAAU;QACV,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,UAAU;QACnB,aAAa;QACb,SAAS,EAAE,OAAO,CAAC,MAAM,GAAG,aAAa;QACzC,mBAAmB;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;QAC5C,WAAW;QACX,WAAW;KACZ,CAAC;IAEF,IAAI,YAAY,IAAI,GAAG,EAAE,CAAC;QACxB,OAAO;YACL,EAAE,EAAE,yBAAyB;YAC7B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,aAAa,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,+BAA+B,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,MAAM,EAAE;YAClI,OAAO;SACR,CAAC;IACJ,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,EAAE,EAAE,yBAAyB;YAC7B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B,mBAAmB,+BAA+B,MAAM,EAAE;YAC/F,OAAO;SACR,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,EAAE,yBAAyB;QAC7B,QAAQ,EAAE,UAAU;QACpB,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,4DAA4D,MAAM,EAAE;QAC7E,OAAO;KACR,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,yBAAyB;IAC7B,QAAQ,EAAE,UAAU;IACpB,WAAW,EAAE,wDAAwD;IACrE,SAAS,EAAE,CAAC,iBAAiB,CAAC;IAC9B,GAAG,EAAE,yBAAyB;CAC/B,CAAC,CAAC"}
|
|
@@ -72,6 +72,11 @@ async function checkLlmsTxtLinksResolve(ctx) {
|
|
|
72
72
|
const resolved = results.filter((r) => r.ok).length;
|
|
73
73
|
const broken = results.filter((r) => !r.ok);
|
|
74
74
|
const resolveRate = resolved / results.length;
|
|
75
|
+
const fetchErrors = results.filter((r) => r.error).length;
|
|
76
|
+
const rateLimited = results.filter((r) => r.status === 429).length;
|
|
77
|
+
const linkLabel = wasSampled ? 'sampled links' : 'links';
|
|
78
|
+
const suffix = (fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '') +
|
|
79
|
+
(rateLimited > 0 ? `; ${rateLimited} rate-limited (HTTP 429)` : '');
|
|
75
80
|
const details = {
|
|
76
81
|
totalLinks,
|
|
77
82
|
testedLinks: results.length,
|
|
@@ -79,13 +84,15 @@ async function checkLlmsTxtLinksResolve(ctx) {
|
|
|
79
84
|
resolved,
|
|
80
85
|
broken: broken.map((b) => ({ url: b.url, status: b.status, error: b.error })),
|
|
81
86
|
resolveRate: Math.round(resolveRate * 100),
|
|
87
|
+
fetchErrors,
|
|
88
|
+
rateLimited,
|
|
82
89
|
};
|
|
83
90
|
if (resolveRate === 1) {
|
|
84
91
|
return {
|
|
85
92
|
id: 'llms-txt-links-resolve',
|
|
86
93
|
category: 'llms-txt',
|
|
87
94
|
status: 'pass',
|
|
88
|
-
message: `All ${results.length} tested
|
|
95
|
+
message: `All ${results.length} tested ${linkLabel} resolve (${totalLinks} total links)${suffix}`,
|
|
89
96
|
details,
|
|
90
97
|
};
|
|
91
98
|
}
|
|
@@ -94,7 +101,7 @@ async function checkLlmsTxtLinksResolve(ctx) {
|
|
|
94
101
|
id: 'llms-txt-links-resolve',
|
|
95
102
|
category: 'llms-txt',
|
|
96
103
|
status: 'warn',
|
|
97
|
-
message: `${resolved}/${results.length}
|
|
104
|
+
message: `${resolved}/${results.length} ${linkLabel} resolve (${Math.round(resolveRate * 100)}%); ${broken.length} broken${suffix}`,
|
|
98
105
|
details,
|
|
99
106
|
};
|
|
100
107
|
}
|
|
@@ -102,7 +109,7 @@ async function checkLlmsTxtLinksResolve(ctx) {
|
|
|
102
109
|
id: 'llms-txt-links-resolve',
|
|
103
110
|
category: 'llms-txt',
|
|
104
111
|
status: 'fail',
|
|
105
|
-
message: `Only ${resolved}/${results.length}
|
|
112
|
+
message: `Only ${resolved}/${results.length} ${linkLabel} resolve (${Math.round(resolveRate * 100)}%); ${broken.length} broken${suffix}`,
|
|
106
113
|
details,
|
|
107
114
|
};
|
|
108
115
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llms-txt-links-resolve.js","sourceRoot":"","sources":["../../../src/checks/llms-txt/llms-txt-links-resolve.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAU3D,KAAK,UAAU,wBAAwB,CAAC,GAAiB;IACvD,MAAM,YAAY,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,OAAO,EAAE,eAAe,IAAI,EAAE,CAAqB,CAAC;IAEtF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,sCAAsC;YAC/C,SAAS,EAAE,CAAC,iBAAiB,CAAC;SAC/B,CAAC;IACJ,CAAC;IAED,uDAAuD;IACvD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,qBAAqB;IACjE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,oCAAoC;SAC9C,CAAC;IACJ,CAAC;IAED,qCAAqC;IACrC,IAAI,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,UAAU,GAAG,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAC3D,IAAI,UAAU,EAAE,CAAC;QACf,4BAA4B;QAC5B,KAAK,IAAI,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACjE,CAAC;IAED,uCAAuC;IACvC,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAA4B,EAAE;YAChD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC/D,oDAAoD;gBACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;gBACjE,CAAC;gBACD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;YAC3D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,GAAG;oBACH,MAAM,EAAE,CAAC;oBACT,EAAE,EAAE,KAAK;oBACT,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"llms-txt-links-resolve.js","sourceRoot":"","sources":["../../../src/checks/llms-txt/llms-txt-links-resolve.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAU3D,KAAK,UAAU,wBAAwB,CAAC,GAAiB;IACvD,MAAM,YAAY,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,OAAO,EAAE,eAAe,IAAI,EAAE,CAAqB,CAAC;IAEtF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,sCAAsC;YAC/C,SAAS,EAAE,CAAC,iBAAiB,CAAC;SAC/B,CAAC;IACJ,CAAC;IAED,uDAAuD;IACvD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,qBAAqB;IACjE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,oCAAoC;SAC9C,CAAC;IACJ,CAAC;IAED,qCAAqC;IACrC,IAAI,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,UAAU,GAAG,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAC3D,IAAI,UAAU,EAAE,CAAC;QACf,4BAA4B;QAC5B,KAAK,IAAI,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACjE,CAAC;IAED,uCAAuC;IACvC,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAA4B,EAAE;YAChD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC/D,oDAAoD;gBACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;gBACjE,CAAC;gBACD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;YAC3D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,GAAG;oBACH,MAAM,EAAE,CAAC;oBACT,EAAE,EAAE,KAAK;oBACT,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;IAEnE,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC;IACzD,MAAM,MAAM,GACV,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,0BAA0B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEtE,MAAM,OAAO,GAA4B;QACvC,UAAU;QACV,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,UAAU;QACnB,QAAQ;QACR,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7E,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC;QAC1C,WAAW;QACX,WAAW;KACZ,CAAC;IAEF,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,OAAO,OAAO,CAAC,MAAM,WAAW,SAAS,aAAa,UAAU,gBAAgB,MAAM,EAAE;YACjG,OAAO;SACR,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,GAAG,sBAAsB,EAAE,CAAC;QACzC,OAAO;YACL,EAAE,EAAE,wBAAwB;YAC5B,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,aAAa,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC,OAAO,MAAM,CAAC,MAAM,UAAU,MAAM,EAAE;YACnI,OAAO;SACR,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,EAAE,wBAAwB;QAC5B,QAAQ,EAAE,UAAU;QACpB,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,QAAQ,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,aAAa,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC,OAAO,MAAM,CAAC,MAAM,UAAU,MAAM,EAAE;QACxI,OAAO;KACR,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,wBAAwB;IAC5B,QAAQ,EAAE,UAAU;IACpB,WAAW,EAAE,sDAAsD;IACnE,SAAS,EAAE,CAAC,iBAAiB,CAAC;IAC9B,GAAG,EAAE,wBAAwB;CAC9B,CAAC,CAAC"}
|