@rankcli/agent-runtime 0.0.15 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/dist/index.d.mts +43 -393
  2. package/dist/index.d.ts +43 -393
  3. package/dist/index.js +306 -6526
  4. package/dist/index.mjs +202 -6412
  5. package/package.json +6 -1
  6. package/scripts/build-deno.ts +0 -134
  7. package/src/analyzers/core-web-vitals-analyzer.test.ts +0 -236
  8. package/src/analyzers/core-web-vitals-analyzer.ts +0 -557
  9. package/src/analyzers/geo-analyzer.test.ts +0 -310
  10. package/src/analyzers/geo-analyzer.ts +0 -814
  11. package/src/analyzers/image-optimization-analyzer.test.ts +0 -145
  12. package/src/analyzers/image-optimization-analyzer.ts +0 -348
  13. package/src/analyzers/index.ts +0 -233
  14. package/src/analyzers/internal-linking-analyzer.test.ts +0 -141
  15. package/src/analyzers/internal-linking-analyzer.ts +0 -419
  16. package/src/analyzers/mobile-seo-analyzer.test.ts +0 -140
  17. package/src/analyzers/mobile-seo-analyzer.ts +0 -455
  18. package/src/analyzers/security-headers-analyzer.test.ts +0 -115
  19. package/src/analyzers/security-headers-analyzer.ts +0 -318
  20. package/src/analyzers/structured-data-analyzer.test.ts +0 -210
  21. package/src/analyzers/structured-data-analyzer.ts +0 -590
  22. package/src/audit/ai/analyzer.ts +0 -347
  23. package/src/audit/ai/index.ts +0 -29
  24. package/src/audit/ai/prompts/content-analysis.ts +0 -271
  25. package/src/audit/ai/types.ts +0 -179
  26. package/src/audit/checks/additional-checks.ts +0 -439
  27. package/src/audit/checks/agent-experience.ts +0 -108
  28. package/src/audit/checks/ai-citation-worthiness.ts +0 -399
  29. package/src/audit/checks/ai-content-structure.ts +0 -325
  30. package/src/audit/checks/ai-readiness.ts +0 -406
  31. package/src/audit/checks/anchor-text.ts +0 -179
  32. package/src/audit/checks/answer-conciseness.ts +0 -322
  33. package/src/audit/checks/asset-minification.ts +0 -270
  34. package/src/audit/checks/bing-optimization.ts +0 -206
  35. package/src/audit/checks/brand-mention-optimization.ts +0 -349
  36. package/src/audit/checks/caching-headers.ts +0 -305
  37. package/src/audit/checks/canonical-advanced.ts +0 -150
  38. package/src/audit/checks/canonical-domain.ts +0 -196
  39. package/src/audit/checks/citation-quality.ts +0 -358
  40. package/src/audit/checks/client-rendering.ts +0 -542
  41. package/src/audit/checks/color-contrast.ts +0 -342
  42. package/src/audit/checks/content-freshness.ts +0 -170
  43. package/src/audit/checks/content-science.ts +0 -589
  44. package/src/audit/checks/conversion-elements.ts +0 -526
  45. package/src/audit/checks/crawlability.ts +0 -220
  46. package/src/audit/checks/directory-listing.ts +0 -172
  47. package/src/audit/checks/dom-analysis.ts +0 -191
  48. package/src/audit/checks/dom-size.ts +0 -246
  49. package/src/audit/checks/duplicate-content.ts +0 -194
  50. package/src/audit/checks/eeat-signals.ts +0 -990
  51. package/src/audit/checks/entity-seo.ts +0 -396
  52. package/src/audit/checks/featured-snippet.ts +0 -473
  53. package/src/audit/checks/freshness-signals.ts +0 -443
  54. package/src/audit/checks/funnel-intent.ts +0 -463
  55. package/src/audit/checks/hreflang.ts +0 -174
  56. package/src/audit/checks/html-compliance.ts +0 -302
  57. package/src/audit/checks/image-dimensions.ts +0 -167
  58. package/src/audit/checks/images.ts +0 -160
  59. package/src/audit/checks/indexnow.ts +0 -275
  60. package/src/audit/checks/interactive-tools.ts +0 -475
  61. package/src/audit/checks/internal-link-graph.ts +0 -436
  62. package/src/audit/checks/keyword-analysis.ts +0 -239
  63. package/src/audit/checks/keyword-cannibalization.ts +0 -385
  64. package/src/audit/checks/keyword-placement.ts +0 -471
  65. package/src/audit/checks/links.ts +0 -273
  66. package/src/audit/checks/llms-txt.ts +0 -224
  67. package/src/audit/checks/local-seo.ts +0 -296
  68. package/src/audit/checks/mobile.ts +0 -167
  69. package/src/audit/checks/modern-images.ts +0 -226
  70. package/src/audit/checks/navboost-signals.ts +0 -395
  71. package/src/audit/checks/on-page.ts +0 -209
  72. package/src/audit/checks/page-resources.ts +0 -285
  73. package/src/audit/checks/pagination.ts +0 -180
  74. package/src/audit/checks/performance.ts +0 -153
  75. package/src/audit/checks/platform-presence.ts +0 -580
  76. package/src/audit/checks/rag-chunk-readiness.test.ts +0 -159
  77. package/src/audit/checks/rag-chunk-readiness.ts +0 -163
  78. package/src/audit/checks/redirect-analysis.ts +0 -153
  79. package/src/audit/checks/redirect-chain.ts +0 -389
  80. package/src/audit/checks/resource-hints.ts +0 -420
  81. package/src/audit/checks/responsive-css.ts +0 -247
  82. package/src/audit/checks/responsive-images.ts +0 -396
  83. package/src/audit/checks/review-ecosystem.ts +0 -415
  84. package/src/audit/checks/robots-validation.ts +0 -373
  85. package/src/audit/checks/security-headers.ts +0 -188
  86. package/src/audit/checks/security.ts +0 -144
  87. package/src/audit/checks/serp-preview.ts +0 -251
  88. package/src/audit/checks/site-maturity.ts +0 -453
  89. package/src/audit/checks/social-meta.test.ts +0 -275
  90. package/src/audit/checks/social-meta.ts +0 -134
  91. package/src/audit/checks/soft-404.ts +0 -151
  92. package/src/audit/checks/structured-data.ts +0 -238
  93. package/src/audit/checks/tech-detection.ts +0 -496
  94. package/src/audit/checks/topical-clusters.ts +0 -435
  95. package/src/audit/checks/tracker-bloat.ts +0 -462
  96. package/src/audit/checks/tracking-verification.test.ts +0 -371
  97. package/src/audit/checks/tracking-verification.ts +0 -636
  98. package/src/audit/checks/url-safety.ts +0 -682
  99. package/src/audit/deno-entry.ts +0 -66
  100. package/src/audit/discovery/index.ts +0 -15
  101. package/src/audit/discovery/link-crawler.ts +0 -233
  102. package/src/audit/discovery/repo-routes.ts +0 -347
  103. package/src/audit/engine.ts +0 -633
  104. package/src/audit/fixes/index.ts +0 -209
  105. package/src/audit/fixes/social-meta-fixes.test.ts +0 -329
  106. package/src/audit/fixes/social-meta-fixes.ts +0 -463
  107. package/src/audit/index.ts +0 -74
  108. package/src/audit/runner.test.ts +0 -299
  109. package/src/audit/runner.ts +0 -130
  110. package/src/audit/types.ts +0 -1987
  111. package/src/content/featured-snippet.ts +0 -367
  112. package/src/content/generator.test.ts +0 -534
  113. package/src/content/generator.ts +0 -501
  114. package/src/content/headline.ts +0 -317
  115. package/src/content/index.ts +0 -62
  116. package/src/content/intent.ts +0 -258
  117. package/src/content/keyword-density.ts +0 -349
  118. package/src/content/readability.ts +0 -262
  119. package/src/executor.ts +0 -336
  120. package/src/fixer/framework-fixes.test.ts +0 -489
  121. package/src/fixer/framework-fixes.ts +0 -5287
  122. package/src/fixer/index.ts +0 -6
  123. package/src/fixer/schemas.ts +0 -971
  124. package/src/fixer.ts +0 -642
  125. package/src/frameworks/detector.test.ts +0 -248
  126. package/src/frameworks/detector.ts +0 -899
  127. package/src/frameworks/index.ts +0 -68
  128. package/src/frameworks/recipes/angular.yaml +0 -171
  129. package/src/frameworks/recipes/astro.yaml +0 -206
  130. package/src/frameworks/recipes/django.yaml +0 -180
  131. package/src/frameworks/recipes/laravel.yaml +0 -137
  132. package/src/frameworks/recipes/nextjs.yaml +0 -268
  133. package/src/frameworks/recipes/nuxt.yaml +0 -175
  134. package/src/frameworks/recipes/rails.yaml +0 -188
  135. package/src/frameworks/recipes/react.yaml +0 -202
  136. package/src/frameworks/recipes/sveltekit.yaml +0 -154
  137. package/src/frameworks/recipes/vue.yaml +0 -137
  138. package/src/frameworks/recipes/wordpress.yaml +0 -209
  139. package/src/frameworks/suggestion-engine.ts +0 -357
  140. package/src/geo/geo-content.test.ts +0 -305
  141. package/src/geo/geo-content.ts +0 -266
  142. package/src/geo/geo-history.test.ts +0 -473
  143. package/src/geo/geo-history.ts +0 -433
  144. package/src/geo/geo-tracker.test.ts +0 -359
  145. package/src/geo/geo-tracker.ts +0 -411
  146. package/src/geo/index.ts +0 -11
  147. package/src/geo/llm-citation-checker.ts +0 -188
  148. package/src/git/commit-helper.test.ts +0 -261
  149. package/src/git/commit-helper.ts +0 -329
  150. package/src/git/index.ts +0 -12
  151. package/src/git/pr-helper.test.ts +0 -284
  152. package/src/git/pr-helper.ts +0 -307
  153. package/src/index.ts +0 -76
  154. package/src/keywords/ai-keyword-engine.ts +0 -1062
  155. package/src/keywords/ai-summarizer.ts +0 -387
  156. package/src/keywords/ci-mode.ts +0 -555
  157. package/src/keywords/engine.ts +0 -359
  158. package/src/keywords/index.ts +0 -151
  159. package/src/keywords/llm-judge.ts +0 -357
  160. package/src/keywords/nlp-analysis.ts +0 -706
  161. package/src/keywords/prioritizer.ts +0 -295
  162. package/src/keywords/site-crawler.ts +0 -342
  163. package/src/keywords/sources/autocomplete.ts +0 -139
  164. package/src/keywords/sources/competitive-search.ts +0 -450
  165. package/src/keywords/sources/competitor-analysis.ts +0 -374
  166. package/src/keywords/sources/dataforseo.ts +0 -206
  167. package/src/keywords/sources/free-sources.ts +0 -294
  168. package/src/keywords/sources/gsc.ts +0 -123
  169. package/src/keywords/topic-grouping.ts +0 -327
  170. package/src/keywords/types.ts +0 -144
  171. package/src/keywords/wizard.ts +0 -457
  172. package/src/loader.ts +0 -40
  173. package/src/ranking/index.ts +0 -5
  174. package/src/ranking/serp-client.ts +0 -348
  175. package/src/ranking/tracker.ts +0 -380
  176. package/src/ranking/types.ts +0 -123
  177. package/src/reports/index.ts +0 -7
  178. package/src/reports/report-generator.test.ts +0 -293
  179. package/src/reports/report-generator.ts +0 -713
  180. package/src/scheduler/alerts.test.ts +0 -458
  181. package/src/scheduler/alerts.ts +0 -328
  182. package/src/scheduler/index.ts +0 -8
  183. package/src/scheduler/scheduled-audit.test.ts +0 -377
  184. package/src/scheduler/scheduled-audit.ts +0 -149
  185. package/src/test/integration-test.ts +0 -325
  186. package/src/tools/analyzer.ts +0 -373
  187. package/src/tools/crawl.ts +0 -293
  188. package/src/tools/files.ts +0 -301
  189. package/src/tools/h1-fixer.ts +0 -249
  190. package/src/tools/index.ts +0 -67
  191. package/src/tracking/github-action.ts +0 -326
  192. package/src/tracking/google-analytics.ts +0 -265
  193. package/src/tracking/index.ts +0 -45
  194. package/src/tracking/report-generator.ts +0 -386
  195. package/src/tracking/search-console.ts +0 -335
  196. package/src/types.ts +0 -148
  197. package/src/utils/http.ts +0 -302
  198. package/src/wasm-adapter.ts +0 -297
  199. package/src/wasm-entry.ts +0 -14
  200. package/tsconfig.json +0 -17
  201. package/tsup.wasm.config.ts +0 -26
  202. package/vitest.config.ts +0 -15
@@ -1,159 +0,0 @@
1
- import { describe, it, expect } from 'vitest';
2
- import { analyzeRAGChunkReadiness } from './rag-chunk-readiness.js';
3
-
4
- const testUrl = 'https://example.com';
5
-
6
- function words(n: number, prefix = 'word'): string {
7
- return Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
8
- }
9
-
10
- describe('analyzeRAGChunkReadiness', () => {
11
- it('returns zero sections and score 0 for a page with no headings', () => {
12
- const html = `<html><body><p>${words(200)}</p></body></html>`;
13
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
14
- expect(data.totalSections).toBe(0);
15
- expect(data.chunkReadinessScore).toBe(0);
16
- });
17
-
18
- it('classifies a well-sized section (150-450 words) as good', () => {
19
- const html = `<html><body><h2>Topic</h2><p>${words(300)}</p></body></html>`;
20
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
21
- expect(data.totalSections).toBe(1);
22
- expect(data.sections[0].wordCount).toBe(300);
23
- expect(data.sections[0].sizeQuality).toBe('good');
24
- });
25
-
26
- it('classifies a very short section as too-short', () => {
27
- const html = `<html><body><h2>Topic</h2><p>${words(20)}</p></body></html>`;
28
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
29
- expect(data.sections[0].sizeQuality).toBe('too-short');
30
- });
31
-
32
- it('classifies a very long section as too-long', () => {
33
- const html = `<html><body><h2>Topic</h2><p>${words(800)}</p></body></html>`;
34
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
35
- expect(data.sections[0].sizeQuality).toBe('too-long');
36
- });
37
-
38
- it('splits content into sections at each h1/h2/h3, regardless of nesting depth', () => {
39
- // Deliberately nested in wrapper divs — a sibling-only DOM walk would
40
- // fail to associate the paragraphs with their heading here; document
41
- // order should still get this right.
42
- const html = `
43
- <html><body>
44
- <div class="section-wrapper">
45
- <div class="inner"><h2>First</h2></div>
46
- <div class="content"><p>${words(200, 'a')}</p></div>
47
- </div>
48
- <div class="section-wrapper">
49
- <div class="inner"><h2>Second</h2></div>
50
- <div class="content"><p>${words(200, 'b')}</p></div>
51
- </div>
52
- </body></html>
53
- `;
54
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
55
- expect(data.totalSections).toBe(2);
56
- expect(data.sections[0].heading).toBe('First');
57
- expect(data.sections[0].wordCount).toBe(200);
58
- expect(data.sections[1].heading).toBe('Second');
59
- expect(data.sections[1].wordCount).toBe(200);
60
- });
61
-
62
- it('accumulates multiple paragraphs and list items under one heading', () => {
63
- const html = `
64
- <html><body>
65
- <h2>Topic</h2>
66
- <p>${words(100, 'a')}</p>
67
- <p>${words(100, 'b')}</p>
68
- <ul><li>${words(50, 'c')}</li><li>${words(50, 'd')}</li></ul>
69
- </body></html>
70
- `;
71
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
72
- expect(data.totalSections).toBe(1);
73
- expect(data.sections[0].wordCount).toBe(300);
74
- });
75
-
76
- it('content before the first heading is not attributed to any section', () => {
77
- const html = `
78
- <html><body>
79
- <p>${words(500)}</p>
80
- <h2>Topic</h2>
81
- <p>${words(200)}</p>
82
- </body></html>
83
- `;
84
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
85
- expect(data.totalSections).toBe(1);
86
- expect(data.sections[0].wordCount).toBe(200);
87
- });
88
-
89
- it('detects a dangling-reference opener', () => {
90
- const html = `<html><body><h2>Topic</h2><p>This approach reduces cost significantly. ${words(150)}</p></body></html>`;
91
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
92
- expect(data.sections[0].startsWithDanglingReference).toBe(true);
93
- });
94
-
95
- it('does not flag a section that names its subject', () => {
96
- const html = `<html><body><h2>Topic</h2><p>Caching reduces cost significantly. ${words(150)}</p></body></html>`;
97
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
98
- expect(data.sections[0].startsWithDanglingReference).toBe(false);
99
- });
100
-
101
- it('raises RAG_CHUNK_SIZE_MISMATCH when most sections are poorly sized', () => {
102
- const html = `
103
- <html><body>
104
- <h2>One</h2><p>${words(20)}</p>
105
- <h2>Two</h2><p>${words(900)}</p>
106
- <h2>Three</h2><p>${words(15)}</p>
107
- </body></html>
108
- `;
109
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
110
- expect(issues.some((i) => i.code === 'RAG_CHUNK_SIZE_MISMATCH')).toBe(true);
111
- });
112
-
113
- it('does not raise RAG_CHUNK_SIZE_MISMATCH when most sections are well-sized', () => {
114
- const html = `
115
- <html><body>
116
- <h2>One</h2><p>${words(200)}</p>
117
- <h2>Two</h2><p>${words(250)}</p>
118
- <h2>Three</h2><p>${words(300)}</p>
119
- </body></html>
120
- `;
121
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
122
- expect(issues.some((i) => i.code === 'RAG_CHUNK_SIZE_MISMATCH')).toBe(false);
123
- });
124
-
125
- it('raises RAG_CHUNK_DANGLING_REFERENCES when over 30% of sections open with one', () => {
126
- const html = `
127
- <html><body>
128
- <h2>One</h2><p>This reduces cost. ${words(150)}</p>
129
- <h2>Two</h2><p>It also improves speed. ${words(150)}</p>
130
- <h2>Three</h2><p>Caching is the mechanism. ${words(150)}</p>
131
- </body></html>
132
- `;
133
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
134
- expect(issues.some((i) => i.code === 'RAG_CHUNK_DANGLING_REFERENCES')).toBe(true);
135
- });
136
-
137
- it('does not raise issues for a page with too few sections to judge (under 3)', () => {
138
- const html = `<html><body><h2>One</h2><p>${words(20)}</p><h2>Two</h2><p>${words(900)}</p></body></html>`;
139
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
140
- expect(issues.length).toBe(0);
141
- });
142
-
143
- it('computes a chunkReadinessScore of 100 when all sections are good and self-contained', () => {
144
- const html = `
145
- <html><body>
146
- <h2>One</h2><p>Caching reduces latency. ${words(200)}</p>
147
- <h2>Two</h2><p>Compression reduces bandwidth. ${words(200)}</p>
148
- </body></html>
149
- `;
150
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
151
- expect(data.chunkReadinessScore).toBe(100);
152
- });
153
-
154
- it('estimates tokens at roughly 1.33 tokens per word', () => {
155
- const html = `<html><body><h2>Topic</h2><p>${words(300)}</p></body></html>`;
156
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
157
- expect(data.sections[0].estimatedTokens).toBe(399); // round(300 * 1.33)
158
- });
159
- });
@@ -1,163 +0,0 @@
1
- /**
2
- * RAG Chunk Readiness
3
- *
4
- * How AI answer engines actually retrieve content: RAG (Retrieval-Augmented
5
- * Generation) systems chunk a page into pieces — the 2026 convergence
6
- * benchmark is roughly 400-512 tokens per chunk, 10-25% overlap — then
7
- * retrieve whichever chunk(s) best match a query, not the whole page. A page
8
- * whose own heading-delimited sections are naturally chunk-sized, with clear
9
- * headings and paragraphs that don't lean on the previous section's context
10
- * to make sense, is less likely to get split mid-thought by a RAG chunker —
11
- * preserving the semantic boundaries the author actually wrote, which is
12
- * what the RAG-chunking literature credits with better retrieval recall.
13
- *
14
- * This is a genuinely different angle from ai-content-structure.ts (tables/
15
- * lists/Q&A formatting): that's about FORMAT, this is about SECTION SIZE
16
- * and SELF-CONTAINMENT. Novel enough that no dedicated site-auditing tool
17
- * for it was found as of this writing — RAG-chunking guidance that exists
18
- * targets people building RAG systems, not people writing web content.
19
- */
20
-
21
- import * as cheerio from 'cheerio';
22
- import type { AuditIssue } from '../types.js';
23
-
24
- export interface ChunkSection {
25
- heading: string;
26
- headingLevel: number;
27
- wordCount: number;
28
- estimatedTokens: number;
29
- sizeQuality: 'too-short' | 'good' | 'too-long';
30
- startsWithDanglingReference: boolean;
31
- }
32
-
33
- export interface RAGChunkReadinessData {
34
- sections: ChunkSection[];
35
- totalSections: number;
36
- wellSizedSectionCount: number;
37
- danglingReferenceCount: number;
38
- chunkReadinessScore: number; // 0-100
39
- }
40
-
41
- // 400-512 tokens is the 2026 RAG-chunking convergence benchmark. English
42
- // prose runs roughly 1.3 words per token (~0.75 tokens/word), so that's
43
- // roughly 300-385 words — widened to 150-450 here since this measures how
44
- // naturally an AUTHOR sectioned their content, not what a chunker itself
45
- // targets; a section in this band is unlikely to need mid-thought splitting
46
- // either way, which is the property that actually matters.
47
- const MIN_GOOD_WORDS = 150;
48
- const MAX_GOOD_WORDS = 450;
49
- const TOKENS_PER_WORD = 1.33;
50
-
51
- // A section opening on a bare pronoun/demonstrative reference ("This
52
- // approach...", "It also...") reads fine in place but loses its antecedent
53
- // the moment a RAG system retrieves it standalone — a real self-containment
54
- // signal, not a grammar nitpick.
55
- const DANGLING_OPENERS = /^(this|it|these|that|they|such|those|the former|the latter)\b/i;
56
-
57
- function classifySize(wordCount: number): ChunkSection['sizeQuality'] {
58
- if (wordCount < MIN_GOOD_WORDS) return 'too-short';
59
- if (wordCount > MAX_GOOD_WORDS) return 'too-long';
60
- return 'good';
61
- }
62
-
63
- /**
64
- * Splits content into heading-delimited sections by walking all
65
- * heading/content elements in DOCUMENT ORDER (a single multi-tag cheerio
66
- * selector), rather than via sibling traversal — sibling-only walks break
67
- * on the wrapper-div-heavy DOM nesting common in modern frameworks, where
68
- * a heading and "its" paragraphs aren't actually direct siblings. Document
69
- * order is robust to arbitrary nesting depth.
70
- */
71
- function extractSections($: cheerio.CheerioAPI): Array<{ heading: string; headingLevel: number; text: string }> {
72
- const elements = $('h1, h2, h3, p, li, blockquote').toArray();
73
- const sections: Array<{ heading: string; headingLevel: number; text: string }> = [];
74
- let current: { heading: string; headingLevel: number; text: string } | null = null;
75
-
76
- for (const el of elements) {
77
- const tag = (el as { tagName?: string }).tagName?.toLowerCase();
78
- if (tag === 'h1' || tag === 'h2' || tag === 'h3') {
79
- if (current) sections.push(current);
80
- current = { heading: $(el).text().trim(), headingLevel: parseInt(tag.slice(1), 10), text: '' };
81
- } else if (current) {
82
- current.text += ' ' + $(el).text();
83
- }
84
- // Content before the first heading has no section to attribute it to —
85
- // intentionally skipped rather than inventing a synthetic "intro" chunk.
86
- }
87
- if (current) sections.push(current);
88
- return sections;
89
- }
90
-
91
- export function analyzeRAGChunkReadiness(
92
- html: string,
93
- url: string
94
- ): { issues: AuditIssue[]; data: RAGChunkReadinessData } {
95
- const issues: AuditIssue[] = [];
96
- const $ = cheerio.load(html);
97
- $('nav, footer, aside, script, style, noscript, header').remove();
98
-
99
- const rawSections = extractSections($);
100
-
101
- const sections: ChunkSection[] = rawSections.map((s) => {
102
- const words = s.text.trim().split(/\s+/).filter(Boolean);
103
- const wordCount = words.length;
104
- const estimatedTokens = Math.round(wordCount * TOKENS_PER_WORD);
105
- const firstSentence = s.text.trim().split(/[.!?]/)[0] || '';
106
- return {
107
- heading: s.heading,
108
- headingLevel: s.headingLevel,
109
- wordCount,
110
- estimatedTokens,
111
- sizeQuality: classifySize(wordCount),
112
- startsWithDanglingReference: DANGLING_OPENERS.test(firstSentence.trim()),
113
- };
114
- });
115
-
116
- const totalSections = sections.length;
117
- const wellSizedSectionCount = sections.filter((s) => s.sizeQuality === 'good').length;
118
- const danglingReferenceCount = sections.filter((s) => s.startsWithDanglingReference).length;
119
-
120
- const chunkReadinessScore =
121
- totalSections === 0
122
- ? 0
123
- : Math.round(
124
- ((wellSizedSectionCount / totalSections) * 0.7 +
125
- ((totalSections - danglingReferenceCount) / totalSections) * 0.3) *
126
- 100
127
- );
128
-
129
- if (totalSections >= 3 && wellSizedSectionCount / totalSections < 0.5) {
130
- const tooLong = sections.filter((s) => s.sizeQuality === 'too-long').length;
131
- const tooShort = sections.filter((s) => s.sizeQuality === 'too-short').length;
132
- issues.push({
133
- code: 'RAG_CHUNK_SIZE_MISMATCH',
134
- severity: 'notice',
135
- category: 'ai-readiness',
136
- title: 'Most sections are poorly sized for AI retrieval chunking',
137
- description: `${wellSizedSectionCount} of ${totalSections} sections fall in a well-sized range for RAG retrieval (roughly 150-450 words); ${tooLong} are too long and likely to get split mid-thought, ${tooShort} are too short and likely to get merged with unrelated neighbors.`,
138
- impact: 'When an AI answer engine retrieves a chunk of this page in response to a query, oversized sections risk being cut off mid-point and undersized ones risk losing standalone context — both reduce the odds the retrieved chunk reads coherently in an AI-generated answer.',
139
- howToFix: 'Break up long sections (450+ words) with an additional H2/H3 subheading roughly every 300-400 words. Merge very short sections (under 150 words) into a neighboring section or expand them with enough context to stand alone.',
140
- affectedUrls: [url],
141
- details: { wellSizedSectionCount, totalSections, tooLong, tooShort },
142
- });
143
- }
144
-
145
- if (totalSections >= 3 && danglingReferenceCount / totalSections > 0.3) {
146
- issues.push({
147
- code: 'RAG_CHUNK_DANGLING_REFERENCES',
148
- severity: 'notice',
149
- category: 'ai-readiness',
150
- title: 'Several sections open with a reference to prior context',
151
- description: `${danglingReferenceCount} of ${totalSections} sections start with a pronoun or demonstrative ("This...", "It...", "These...") that depends on the previous section to make sense.`,
152
- impact: 'A RAG system that retrieves one of these sections on its own — which is exactly how retrieval works, one chunk at a time — surfaces a sentence whose subject is undefined, reading as broken or confusing in an AI-generated answer.',
153
- howToFix: 'Open each section by naming its actual subject instead of referring back to the previous one — e.g. "This approach reduces cost" becomes "Caching reduces cost."',
154
- affectedUrls: [url],
155
- details: { danglingReferenceCount, totalSections },
156
- });
157
- }
158
-
159
- return {
160
- issues,
161
- data: { sections, totalSections, wellSizedSectionCount, danglingReferenceCount, chunkReadinessScore },
162
- };
163
- }
@@ -1,153 +0,0 @@
1
- import { httpHead } from '../../utils/http.js';
2
- import type { AuditIssue } from '../types.js';
3
- import { ISSUE_DEFINITIONS } from '../types.js';
4
-
5
- export interface RedirectData {
6
- hasRedirect: boolean;
7
- redirectChain: RedirectHop[];
8
- isLoop: boolean;
9
- finalUrl?: string;
10
- totalHops: number;
11
- }
12
-
13
- export interface RedirectHop {
14
- url: string;
15
- statusCode: number;
16
- location?: string;
17
- }
18
-
19
- export async function analyzeRedirects(
20
- url: string,
21
- maxHops: number = 10
22
- ): Promise<{ issues: AuditIssue[]; data: RedirectData }> {
23
- const issues: AuditIssue[] = [];
24
- const chain: RedirectHop[] = [];
25
- const visitedUrls = new Set<string>();
26
- let currentUrl = url;
27
- let isLoop = false;
28
-
29
- while (chain.length < maxHops) {
30
- // Check for loop
31
- if (visitedUrls.has(currentUrl)) {
32
- isLoop = true;
33
- break;
34
- }
35
- visitedUrls.add(currentUrl);
36
-
37
- try {
38
- const response = await httpHead(currentUrl, {
39
- timeout: 5000,
40
- maxRedirects: 0,
41
- validateStatus: () => true,
42
- });
43
-
44
- const hop: RedirectHop = {
45
- url: currentUrl,
46
- statusCode: response.status,
47
- };
48
-
49
- if (response.status >= 300 && response.status < 400) {
50
- const location = response.headers['location'];
51
- if (location) {
52
- hop.location = location;
53
- chain.push(hop);
54
- currentUrl = new URL(location, currentUrl).href;
55
- continue;
56
- }
57
- }
58
-
59
- chain.push(hop);
60
- break;
61
- } catch (error) {
62
- chain.push({
63
- url: currentUrl,
64
- statusCode: 0,
65
- });
66
- break;
67
- }
68
- }
69
-
70
- const hasRedirect = chain.length > 1;
71
- const finalUrl = chain.length > 0 ? chain[chain.length - 1].url : url;
72
-
73
- const data: RedirectData = {
74
- hasRedirect,
75
- redirectChain: chain,
76
- isLoop,
77
- finalUrl,
78
- totalHops: chain.length - 1,
79
- };
80
-
81
- // ==================== Issue Detection ====================
82
-
83
- // Redirect loop
84
- if (isLoop) {
85
- issues.push({
86
- ...ISSUE_DEFINITIONS.REDIRECT_LOOP,
87
- affectedUrls: [url],
88
- details: {
89
- chain: chain.map(h => `${h.url} (${h.statusCode})`),
90
- },
91
- });
92
- }
93
-
94
- // Redirect chain (more than one redirect)
95
- if (!isLoop && chain.length > 2) {
96
- issues.push({
97
- ...ISSUE_DEFINITIONS.REDIRECT_CHAIN,
98
- affectedUrls: [url],
99
- details: {
100
- chain: chain.map(h => `${h.url} (${h.statusCode})`),
101
- hops: chain.length - 1,
102
- finalUrl,
103
- },
104
- });
105
- }
106
-
107
- return { issues, data };
108
- }
109
-
110
- // Check internal links for redirect targets
111
- export async function checkInternalRedirects(
112
- internalLinks: string[],
113
- batchSize: number = 10
114
- ): Promise<{ redirectLinks: Array<{ source: string; target: string; statusCode: number }> }> {
115
- const redirectLinks: Array<{ source: string; target: string; statusCode: number }> = [];
116
-
117
- // Process in batches to avoid overwhelming the server
118
- for (let i = 0; i < internalLinks.length; i += batchSize) {
119
- const batch = internalLinks.slice(i, i + batchSize);
120
- const promises = batch.map(async (link) => {
121
- try {
122
- const response = await httpHead(link, {
123
- timeout: 5000,
124
- maxRedirects: 0,
125
- validateStatus: () => true,
126
- });
127
-
128
- if (response.status >= 300 && response.status < 400) {
129
- const location = response.headers['location'];
130
- if (location) {
131
- return {
132
- source: link,
133
- target: new URL(location, link).href,
134
- statusCode: response.status,
135
- };
136
- }
137
- }
138
- return null;
139
- } catch {
140
- return null;
141
- }
142
- });
143
-
144
- const results = await Promise.all(promises);
145
- for (const result of results) {
146
- if (result) {
147
- redirectLinks.push(result);
148
- }
149
- }
150
- }
151
-
152
- return { redirectLinks };
153
- }