@rankcli/agent-runtime 0.0.15 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/README.md +5 -4
  2. package/dist/index.d.mts +44 -393
  3. package/dist/index.d.ts +44 -393
  4. package/dist/index.js +22098 -26131
  5. package/dist/index.mjs +11834 -15857
  6. package/package.json +10 -4
  7. package/scripts/build-deno.ts +0 -134
  8. package/src/analyzers/core-web-vitals-analyzer.test.ts +0 -236
  9. package/src/analyzers/core-web-vitals-analyzer.ts +0 -557
  10. package/src/analyzers/geo-analyzer.test.ts +0 -310
  11. package/src/analyzers/geo-analyzer.ts +0 -814
  12. package/src/analyzers/image-optimization-analyzer.test.ts +0 -145
  13. package/src/analyzers/image-optimization-analyzer.ts +0 -348
  14. package/src/analyzers/index.ts +0 -233
  15. package/src/analyzers/internal-linking-analyzer.test.ts +0 -141
  16. package/src/analyzers/internal-linking-analyzer.ts +0 -419
  17. package/src/analyzers/mobile-seo-analyzer.test.ts +0 -140
  18. package/src/analyzers/mobile-seo-analyzer.ts +0 -455
  19. package/src/analyzers/security-headers-analyzer.test.ts +0 -115
  20. package/src/analyzers/security-headers-analyzer.ts +0 -318
  21. package/src/analyzers/structured-data-analyzer.test.ts +0 -210
  22. package/src/analyzers/structured-data-analyzer.ts +0 -590
  23. package/src/audit/ai/analyzer.ts +0 -347
  24. package/src/audit/ai/index.ts +0 -29
  25. package/src/audit/ai/prompts/content-analysis.ts +0 -271
  26. package/src/audit/ai/types.ts +0 -179
  27. package/src/audit/checks/additional-checks.ts +0 -439
  28. package/src/audit/checks/agent-experience.ts +0 -108
  29. package/src/audit/checks/ai-citation-worthiness.ts +0 -399
  30. package/src/audit/checks/ai-content-structure.ts +0 -325
  31. package/src/audit/checks/ai-readiness.ts +0 -406
  32. package/src/audit/checks/anchor-text.ts +0 -179
  33. package/src/audit/checks/answer-conciseness.ts +0 -322
  34. package/src/audit/checks/asset-minification.ts +0 -270
  35. package/src/audit/checks/bing-optimization.ts +0 -206
  36. package/src/audit/checks/brand-mention-optimization.ts +0 -349
  37. package/src/audit/checks/caching-headers.ts +0 -305
  38. package/src/audit/checks/canonical-advanced.ts +0 -150
  39. package/src/audit/checks/canonical-domain.ts +0 -196
  40. package/src/audit/checks/citation-quality.ts +0 -358
  41. package/src/audit/checks/client-rendering.ts +0 -542
  42. package/src/audit/checks/color-contrast.ts +0 -342
  43. package/src/audit/checks/content-freshness.ts +0 -170
  44. package/src/audit/checks/content-science.ts +0 -589
  45. package/src/audit/checks/conversion-elements.ts +0 -526
  46. package/src/audit/checks/crawlability.ts +0 -220
  47. package/src/audit/checks/directory-listing.ts +0 -172
  48. package/src/audit/checks/dom-analysis.ts +0 -191
  49. package/src/audit/checks/dom-size.ts +0 -246
  50. package/src/audit/checks/duplicate-content.ts +0 -194
  51. package/src/audit/checks/eeat-signals.ts +0 -990
  52. package/src/audit/checks/entity-seo.ts +0 -396
  53. package/src/audit/checks/featured-snippet.ts +0 -473
  54. package/src/audit/checks/freshness-signals.ts +0 -443
  55. package/src/audit/checks/funnel-intent.ts +0 -463
  56. package/src/audit/checks/hreflang.ts +0 -174
  57. package/src/audit/checks/html-compliance.ts +0 -302
  58. package/src/audit/checks/image-dimensions.ts +0 -167
  59. package/src/audit/checks/images.ts +0 -160
  60. package/src/audit/checks/indexnow.ts +0 -275
  61. package/src/audit/checks/interactive-tools.ts +0 -475
  62. package/src/audit/checks/internal-link-graph.ts +0 -436
  63. package/src/audit/checks/keyword-analysis.ts +0 -239
  64. package/src/audit/checks/keyword-cannibalization.ts +0 -385
  65. package/src/audit/checks/keyword-placement.ts +0 -471
  66. package/src/audit/checks/links.ts +0 -273
  67. package/src/audit/checks/llms-txt.ts +0 -224
  68. package/src/audit/checks/local-seo.ts +0 -296
  69. package/src/audit/checks/mobile.ts +0 -167
  70. package/src/audit/checks/modern-images.ts +0 -226
  71. package/src/audit/checks/navboost-signals.ts +0 -395
  72. package/src/audit/checks/on-page.ts +0 -209
  73. package/src/audit/checks/page-resources.ts +0 -285
  74. package/src/audit/checks/pagination.ts +0 -180
  75. package/src/audit/checks/performance.ts +0 -153
  76. package/src/audit/checks/platform-presence.ts +0 -580
  77. package/src/audit/checks/rag-chunk-readiness.test.ts +0 -159
  78. package/src/audit/checks/rag-chunk-readiness.ts +0 -163
  79. package/src/audit/checks/redirect-analysis.ts +0 -153
  80. package/src/audit/checks/redirect-chain.ts +0 -389
  81. package/src/audit/checks/resource-hints.ts +0 -420
  82. package/src/audit/checks/responsive-css.ts +0 -247
  83. package/src/audit/checks/responsive-images.ts +0 -396
  84. package/src/audit/checks/review-ecosystem.ts +0 -415
  85. package/src/audit/checks/robots-validation.ts +0 -373
  86. package/src/audit/checks/security-headers.ts +0 -188
  87. package/src/audit/checks/security.ts +0 -144
  88. package/src/audit/checks/serp-preview.ts +0 -251
  89. package/src/audit/checks/site-maturity.ts +0 -453
  90. package/src/audit/checks/social-meta.test.ts +0 -275
  91. package/src/audit/checks/social-meta.ts +0 -134
  92. package/src/audit/checks/soft-404.ts +0 -151
  93. package/src/audit/checks/structured-data.ts +0 -238
  94. package/src/audit/checks/tech-detection.ts +0 -496
  95. package/src/audit/checks/topical-clusters.ts +0 -435
  96. package/src/audit/checks/tracker-bloat.ts +0 -462
  97. package/src/audit/checks/tracking-verification.test.ts +0 -371
  98. package/src/audit/checks/tracking-verification.ts +0 -636
  99. package/src/audit/checks/url-safety.ts +0 -682
  100. package/src/audit/deno-entry.ts +0 -66
  101. package/src/audit/discovery/index.ts +0 -15
  102. package/src/audit/discovery/link-crawler.ts +0 -233
  103. package/src/audit/discovery/repo-routes.ts +0 -347
  104. package/src/audit/engine.ts +0 -633
  105. package/src/audit/fixes/index.ts +0 -209
  106. package/src/audit/fixes/social-meta-fixes.test.ts +0 -329
  107. package/src/audit/fixes/social-meta-fixes.ts +0 -463
  108. package/src/audit/index.ts +0 -74
  109. package/src/audit/runner.test.ts +0 -299
  110. package/src/audit/runner.ts +0 -130
  111. package/src/audit/types.ts +0 -1987
  112. package/src/content/featured-snippet.ts +0 -367
  113. package/src/content/generator.test.ts +0 -534
  114. package/src/content/generator.ts +0 -501
  115. package/src/content/headline.ts +0 -317
  116. package/src/content/index.ts +0 -62
  117. package/src/content/intent.ts +0 -258
  118. package/src/content/keyword-density.ts +0 -349
  119. package/src/content/readability.ts +0 -262
  120. package/src/executor.ts +0 -336
  121. package/src/fixer/framework-fixes.test.ts +0 -489
  122. package/src/fixer/framework-fixes.ts +0 -5287
  123. package/src/fixer/index.ts +0 -6
  124. package/src/fixer/schemas.ts +0 -971
  125. package/src/fixer.ts +0 -642
  126. package/src/frameworks/detector.test.ts +0 -248
  127. package/src/frameworks/detector.ts +0 -899
  128. package/src/frameworks/index.ts +0 -68
  129. package/src/frameworks/recipes/angular.yaml +0 -171
  130. package/src/frameworks/recipes/astro.yaml +0 -206
  131. package/src/frameworks/recipes/django.yaml +0 -180
  132. package/src/frameworks/recipes/laravel.yaml +0 -137
  133. package/src/frameworks/recipes/nextjs.yaml +0 -268
  134. package/src/frameworks/recipes/nuxt.yaml +0 -175
  135. package/src/frameworks/recipes/rails.yaml +0 -188
  136. package/src/frameworks/recipes/react.yaml +0 -202
  137. package/src/frameworks/recipes/sveltekit.yaml +0 -154
  138. package/src/frameworks/recipes/vue.yaml +0 -137
  139. package/src/frameworks/recipes/wordpress.yaml +0 -209
  140. package/src/frameworks/suggestion-engine.ts +0 -357
  141. package/src/geo/geo-content.test.ts +0 -305
  142. package/src/geo/geo-content.ts +0 -266
  143. package/src/geo/geo-history.test.ts +0 -473
  144. package/src/geo/geo-history.ts +0 -433
  145. package/src/geo/geo-tracker.test.ts +0 -359
  146. package/src/geo/geo-tracker.ts +0 -411
  147. package/src/geo/index.ts +0 -11
  148. package/src/geo/llm-citation-checker.ts +0 -188
  149. package/src/git/commit-helper.test.ts +0 -261
  150. package/src/git/commit-helper.ts +0 -329
  151. package/src/git/index.ts +0 -12
  152. package/src/git/pr-helper.test.ts +0 -284
  153. package/src/git/pr-helper.ts +0 -307
  154. package/src/index.ts +0 -76
  155. package/src/keywords/ai-keyword-engine.ts +0 -1062
  156. package/src/keywords/ai-summarizer.ts +0 -387
  157. package/src/keywords/ci-mode.ts +0 -555
  158. package/src/keywords/engine.ts +0 -359
  159. package/src/keywords/index.ts +0 -151
  160. package/src/keywords/llm-judge.ts +0 -357
  161. package/src/keywords/nlp-analysis.ts +0 -706
  162. package/src/keywords/prioritizer.ts +0 -295
  163. package/src/keywords/site-crawler.ts +0 -342
  164. package/src/keywords/sources/autocomplete.ts +0 -139
  165. package/src/keywords/sources/competitive-search.ts +0 -450
  166. package/src/keywords/sources/competitor-analysis.ts +0 -374
  167. package/src/keywords/sources/dataforseo.ts +0 -206
  168. package/src/keywords/sources/free-sources.ts +0 -294
  169. package/src/keywords/sources/gsc.ts +0 -123
  170. package/src/keywords/topic-grouping.ts +0 -327
  171. package/src/keywords/types.ts +0 -144
  172. package/src/keywords/wizard.ts +0 -457
  173. package/src/loader.ts +0 -40
  174. package/src/ranking/index.ts +0 -5
  175. package/src/ranking/serp-client.ts +0 -348
  176. package/src/ranking/tracker.ts +0 -380
  177. package/src/ranking/types.ts +0 -123
  178. package/src/reports/index.ts +0 -7
  179. package/src/reports/report-generator.test.ts +0 -293
  180. package/src/reports/report-generator.ts +0 -713
  181. package/src/scheduler/alerts.test.ts +0 -458
  182. package/src/scheduler/alerts.ts +0 -328
  183. package/src/scheduler/index.ts +0 -8
  184. package/src/scheduler/scheduled-audit.test.ts +0 -377
  185. package/src/scheduler/scheduled-audit.ts +0 -149
  186. package/src/test/integration-test.ts +0 -325
  187. package/src/tools/analyzer.ts +0 -373
  188. package/src/tools/crawl.ts +0 -293
  189. package/src/tools/files.ts +0 -301
  190. package/src/tools/h1-fixer.ts +0 -249
  191. package/src/tools/index.ts +0 -67
  192. package/src/tracking/github-action.ts +0 -326
  193. package/src/tracking/google-analytics.ts +0 -265
  194. package/src/tracking/index.ts +0 -45
  195. package/src/tracking/report-generator.ts +0 -386
  196. package/src/tracking/search-console.ts +0 -335
  197. package/src/types.ts +0 -148
  198. package/src/utils/http.ts +0 -302
  199. package/src/wasm-adapter.ts +0 -297
  200. package/src/wasm-entry.ts +0 -14
  201. package/tsconfig.json +0 -17
  202. package/tsup.wasm.config.ts +0 -26
  203. package/vitest.config.ts +0 -15
@@ -1,159 +0,0 @@
1
- import { describe, it, expect } from 'vitest';
2
- import { analyzeRAGChunkReadiness } from './rag-chunk-readiness.js';
3
-
4
- const testUrl = 'https://example.com';
5
-
6
- function words(n: number, prefix = 'word'): string {
7
- return Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
8
- }
9
-
10
- describe('analyzeRAGChunkReadiness', () => {
11
- it('returns zero sections and score 0 for a page with no headings', () => {
12
- const html = `<html><body><p>${words(200)}</p></body></html>`;
13
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
14
- expect(data.totalSections).toBe(0);
15
- expect(data.chunkReadinessScore).toBe(0);
16
- });
17
-
18
- it('classifies a well-sized section (150-450 words) as good', () => {
19
- const html = `<html><body><h2>Topic</h2><p>${words(300)}</p></body></html>`;
20
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
21
- expect(data.totalSections).toBe(1);
22
- expect(data.sections[0].wordCount).toBe(300);
23
- expect(data.sections[0].sizeQuality).toBe('good');
24
- });
25
-
26
- it('classifies a very short section as too-short', () => {
27
- const html = `<html><body><h2>Topic</h2><p>${words(20)}</p></body></html>`;
28
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
29
- expect(data.sections[0].sizeQuality).toBe('too-short');
30
- });
31
-
32
- it('classifies a very long section as too-long', () => {
33
- const html = `<html><body><h2>Topic</h2><p>${words(800)}</p></body></html>`;
34
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
35
- expect(data.sections[0].sizeQuality).toBe('too-long');
36
- });
37
-
38
- it('splits content into sections at each h1/h2/h3, regardless of nesting depth', () => {
39
- // Deliberately nested in wrapper divs — a sibling-only DOM walk would
40
- // fail to associate the paragraphs with their heading here; document
41
- // order should still get this right.
42
- const html = `
43
- <html><body>
44
- <div class="section-wrapper">
45
- <div class="inner"><h2>First</h2></div>
46
- <div class="content"><p>${words(200, 'a')}</p></div>
47
- </div>
48
- <div class="section-wrapper">
49
- <div class="inner"><h2>Second</h2></div>
50
- <div class="content"><p>${words(200, 'b')}</p></div>
51
- </div>
52
- </body></html>
53
- `;
54
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
55
- expect(data.totalSections).toBe(2);
56
- expect(data.sections[0].heading).toBe('First');
57
- expect(data.sections[0].wordCount).toBe(200);
58
- expect(data.sections[1].heading).toBe('Second');
59
- expect(data.sections[1].wordCount).toBe(200);
60
- });
61
-
62
- it('accumulates multiple paragraphs and list items under one heading', () => {
63
- const html = `
64
- <html><body>
65
- <h2>Topic</h2>
66
- <p>${words(100, 'a')}</p>
67
- <p>${words(100, 'b')}</p>
68
- <ul><li>${words(50, 'c')}</li><li>${words(50, 'd')}</li></ul>
69
- </body></html>
70
- `;
71
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
72
- expect(data.totalSections).toBe(1);
73
- expect(data.sections[0].wordCount).toBe(300);
74
- });
75
-
76
- it('content before the first heading is not attributed to any section', () => {
77
- const html = `
78
- <html><body>
79
- <p>${words(500)}</p>
80
- <h2>Topic</h2>
81
- <p>${words(200)}</p>
82
- </body></html>
83
- `;
84
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
85
- expect(data.totalSections).toBe(1);
86
- expect(data.sections[0].wordCount).toBe(200);
87
- });
88
-
89
- it('detects a dangling-reference opener', () => {
90
- const html = `<html><body><h2>Topic</h2><p>This approach reduces cost significantly. ${words(150)}</p></body></html>`;
91
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
92
- expect(data.sections[0].startsWithDanglingReference).toBe(true);
93
- });
94
-
95
- it('does not flag a section that names its subject', () => {
96
- const html = `<html><body><h2>Topic</h2><p>Caching reduces cost significantly. ${words(150)}</p></body></html>`;
97
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
98
- expect(data.sections[0].startsWithDanglingReference).toBe(false);
99
- });
100
-
101
- it('raises RAG_CHUNK_SIZE_MISMATCH when most sections are poorly sized', () => {
102
- const html = `
103
- <html><body>
104
- <h2>One</h2><p>${words(20)}</p>
105
- <h2>Two</h2><p>${words(900)}</p>
106
- <h2>Three</h2><p>${words(15)}</p>
107
- </body></html>
108
- `;
109
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
110
- expect(issues.some((i) => i.code === 'RAG_CHUNK_SIZE_MISMATCH')).toBe(true);
111
- });
112
-
113
- it('does not raise RAG_CHUNK_SIZE_MISMATCH when most sections are well-sized', () => {
114
- const html = `
115
- <html><body>
116
- <h2>One</h2><p>${words(200)}</p>
117
- <h2>Two</h2><p>${words(250)}</p>
118
- <h2>Three</h2><p>${words(300)}</p>
119
- </body></html>
120
- `;
121
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
122
- expect(issues.some((i) => i.code === 'RAG_CHUNK_SIZE_MISMATCH')).toBe(false);
123
- });
124
-
125
- it('raises RAG_CHUNK_DANGLING_REFERENCES when over 30% of sections open with one', () => {
126
- const html = `
127
- <html><body>
128
- <h2>One</h2><p>This reduces cost. ${words(150)}</p>
129
- <h2>Two</h2><p>It also improves speed. ${words(150)}</p>
130
- <h2>Three</h2><p>Caching is the mechanism. ${words(150)}</p>
131
- </body></html>
132
- `;
133
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
134
- expect(issues.some((i) => i.code === 'RAG_CHUNK_DANGLING_REFERENCES')).toBe(true);
135
- });
136
-
137
- it('does not raise issues for a page with too few sections to judge (under 3)', () => {
138
- const html = `<html><body><h2>One</h2><p>${words(20)}</p><h2>Two</h2><p>${words(900)}</p></body></html>`;
139
- const { issues } = analyzeRAGChunkReadiness(html, testUrl);
140
- expect(issues.length).toBe(0);
141
- });
142
-
143
- it('computes a chunkReadinessScore of 100 when all sections are good and self-contained', () => {
144
- const html = `
145
- <html><body>
146
- <h2>One</h2><p>Caching reduces latency. ${words(200)}</p>
147
- <h2>Two</h2><p>Compression reduces bandwidth. ${words(200)}</p>
148
- </body></html>
149
- `;
150
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
151
- expect(data.chunkReadinessScore).toBe(100);
152
- });
153
-
154
- it('estimates tokens at roughly 1.33 tokens per word', () => {
155
- const html = `<html><body><h2>Topic</h2><p>${words(300)}</p></body></html>`;
156
- const { data } = analyzeRAGChunkReadiness(html, testUrl);
157
- expect(data.sections[0].estimatedTokens).toBe(399); // round(300 * 1.33)
158
- });
159
- });
@@ -1,163 +0,0 @@
1
- /**
2
- * RAG Chunk Readiness
3
- *
4
- * How AI answer engines actually retrieve content: RAG (Retrieval-Augmented
5
- * Generation) systems chunk a page into pieces — the 2026 convergence
6
- * benchmark is roughly 400-512 tokens per chunk, 10-25% overlap — then
7
- * retrieve whichever chunk(s) best match a query, not the whole page. A page
8
- * whose own heading-delimited sections are naturally chunk-sized, with clear
9
- * headings and paragraphs that don't lean on the previous section's context
10
- * to make sense, is less likely to get split mid-thought by a RAG chunker —
11
- * preserving the semantic boundaries the author actually wrote, which is
12
- * what the RAG-chunking literature credits with better retrieval recall.
13
- *
14
- * This is a genuinely different angle from ai-content-structure.ts (tables/
15
- * lists/Q&A formatting): that's about FORMAT, this is about SECTION SIZE
16
- * and SELF-CONTAINMENT. Novel enough that no dedicated site-auditing tool
17
- * for it was found as of this writing — RAG-chunking guidance that exists
18
- * targets people building RAG systems, not people writing web content.
19
- */
20
-
21
- import * as cheerio from 'cheerio';
22
- import type { AuditIssue } from '../types.js';
23
-
24
- export interface ChunkSection {
25
- heading: string;
26
- headingLevel: number;
27
- wordCount: number;
28
- estimatedTokens: number;
29
- sizeQuality: 'too-short' | 'good' | 'too-long';
30
- startsWithDanglingReference: boolean;
31
- }
32
-
33
- export interface RAGChunkReadinessData {
34
- sections: ChunkSection[];
35
- totalSections: number;
36
- wellSizedSectionCount: number;
37
- danglingReferenceCount: number;
38
- chunkReadinessScore: number; // 0-100
39
- }
40
-
41
- // 400-512 tokens is the 2026 RAG-chunking convergence benchmark. English
42
- // prose runs roughly 1.3 words per token (~0.75 tokens/word), so that's
43
- // roughly 300-385 words — widened to 150-450 here since this measures how
44
- // naturally an AUTHOR sectioned their content, not what a chunker itself
45
- // targets; a section in this band is unlikely to need mid-thought splitting
46
- // either way, which is the property that actually matters.
47
- const MIN_GOOD_WORDS = 150;
48
- const MAX_GOOD_WORDS = 450;
49
- const TOKENS_PER_WORD = 1.33;
50
-
51
- // A section opening on a bare pronoun/demonstrative reference ("This
52
- // approach...", "It also...") reads fine in place but loses its antecedent
53
- // the moment a RAG system retrieves it standalone — a real self-containment
54
- // signal, not a grammar nitpick.
55
- const DANGLING_OPENERS = /^(this|it|these|that|they|such|those|the former|the latter)\b/i;
56
-
57
- function classifySize(wordCount: number): ChunkSection['sizeQuality'] {
58
- if (wordCount < MIN_GOOD_WORDS) return 'too-short';
59
- if (wordCount > MAX_GOOD_WORDS) return 'too-long';
60
- return 'good';
61
- }
62
-
63
- /**
64
- * Splits content into heading-delimited sections by walking all
65
- * heading/content elements in DOCUMENT ORDER (a single multi-tag cheerio
66
- * selector), rather than via sibling traversal — sibling-only walks break
67
- * on the wrapper-div-heavy DOM nesting common in modern frameworks, where
68
- * a heading and "its" paragraphs aren't actually direct siblings. Document
69
- * order is robust to arbitrary nesting depth.
70
- */
71
- function extractSections($: cheerio.CheerioAPI): Array<{ heading: string; headingLevel: number; text: string }> {
72
- const elements = $('h1, h2, h3, p, li, blockquote').toArray();
73
- const sections: Array<{ heading: string; headingLevel: number; text: string }> = [];
74
- let current: { heading: string; headingLevel: number; text: string } | null = null;
75
-
76
- for (const el of elements) {
77
- const tag = (el as { tagName?: string }).tagName?.toLowerCase();
78
- if (tag === 'h1' || tag === 'h2' || tag === 'h3') {
79
- if (current) sections.push(current);
80
- current = { heading: $(el).text().trim(), headingLevel: parseInt(tag.slice(1), 10), text: '' };
81
- } else if (current) {
82
- current.text += ' ' + $(el).text();
83
- }
84
- // Content before the first heading has no section to attribute it to —
85
- // intentionally skipped rather than inventing a synthetic "intro" chunk.
86
- }
87
- if (current) sections.push(current);
88
- return sections;
89
- }
90
-
91
- export function analyzeRAGChunkReadiness(
92
- html: string,
93
- url: string
94
- ): { issues: AuditIssue[]; data: RAGChunkReadinessData } {
95
- const issues: AuditIssue[] = [];
96
- const $ = cheerio.load(html);
97
- $('nav, footer, aside, script, style, noscript, header').remove();
98
-
99
- const rawSections = extractSections($);
100
-
101
- const sections: ChunkSection[] = rawSections.map((s) => {
102
- const words = s.text.trim().split(/\s+/).filter(Boolean);
103
- const wordCount = words.length;
104
- const estimatedTokens = Math.round(wordCount * TOKENS_PER_WORD);
105
- const firstSentence = s.text.trim().split(/[.!?]/)[0] || '';
106
- return {
107
- heading: s.heading,
108
- headingLevel: s.headingLevel,
109
- wordCount,
110
- estimatedTokens,
111
- sizeQuality: classifySize(wordCount),
112
- startsWithDanglingReference: DANGLING_OPENERS.test(firstSentence.trim()),
113
- };
114
- });
115
-
116
- const totalSections = sections.length;
117
- const wellSizedSectionCount = sections.filter((s) => s.sizeQuality === 'good').length;
118
- const danglingReferenceCount = sections.filter((s) => s.startsWithDanglingReference).length;
119
-
120
- const chunkReadinessScore =
121
- totalSections === 0
122
- ? 0
123
- : Math.round(
124
- ((wellSizedSectionCount / totalSections) * 0.7 +
125
- ((totalSections - danglingReferenceCount) / totalSections) * 0.3) *
126
- 100
127
- );
128
-
129
- if (totalSections >= 3 && wellSizedSectionCount / totalSections < 0.5) {
130
- const tooLong = sections.filter((s) => s.sizeQuality === 'too-long').length;
131
- const tooShort = sections.filter((s) => s.sizeQuality === 'too-short').length;
132
- issues.push({
133
- code: 'RAG_CHUNK_SIZE_MISMATCH',
134
- severity: 'notice',
135
- category: 'ai-readiness',
136
- title: 'Most sections are poorly sized for AI retrieval chunking',
137
- description: `${wellSizedSectionCount} of ${totalSections} sections fall in a well-sized range for RAG retrieval (roughly 150-450 words); ${tooLong} are too long and likely to get split mid-thought, ${tooShort} are too short and likely to get merged with unrelated neighbors.`,
138
- impact: 'When an AI answer engine retrieves a chunk of this page in response to a query, oversized sections risk being cut off mid-point and undersized ones risk losing standalone context — both reduce the odds the retrieved chunk reads coherently in an AI-generated answer.',
139
- howToFix: 'Break up long sections (450+ words) with an additional H2/H3 subheading roughly every 300-400 words. Merge very short sections (under 150 words) into a neighboring section or expand them with enough context to stand alone.',
140
- affectedUrls: [url],
141
- details: { wellSizedSectionCount, totalSections, tooLong, tooShort },
142
- });
143
- }
144
-
145
- if (totalSections >= 3 && danglingReferenceCount / totalSections > 0.3) {
146
- issues.push({
147
- code: 'RAG_CHUNK_DANGLING_REFERENCES',
148
- severity: 'notice',
149
- category: 'ai-readiness',
150
- title: 'Several sections open with a reference to prior context',
151
- description: `${danglingReferenceCount} of ${totalSections} sections start with a pronoun or demonstrative ("This...", "It...", "These...") that depends on the previous section to make sense.`,
152
- impact: 'A RAG system that retrieves one of these sections on its own — which is exactly how retrieval works, one chunk at a time — surfaces a sentence whose subject is undefined, reading as broken or confusing in an AI-generated answer.',
153
- howToFix: 'Open each section by naming its actual subject instead of referring back to the previous one — e.g. "This approach reduces cost" becomes "Caching reduces cost."',
154
- affectedUrls: [url],
155
- details: { danglingReferenceCount, totalSections },
156
- });
157
- }
158
-
159
- return {
160
- issues,
161
- data: { sections, totalSections, wellSizedSectionCount, danglingReferenceCount, chunkReadinessScore },
162
- };
163
- }
@@ -1,153 +0,0 @@
1
- import { httpHead } from '../../utils/http.js';
2
- import type { AuditIssue } from '../types.js';
3
- import { ISSUE_DEFINITIONS } from '../types.js';
4
-
5
- export interface RedirectData {
6
- hasRedirect: boolean;
7
- redirectChain: RedirectHop[];
8
- isLoop: boolean;
9
- finalUrl?: string;
10
- totalHops: number;
11
- }
12
-
13
- export interface RedirectHop {
14
- url: string;
15
- statusCode: number;
16
- location?: string;
17
- }
18
-
19
- export async function analyzeRedirects(
20
- url: string,
21
- maxHops: number = 10
22
- ): Promise<{ issues: AuditIssue[]; data: RedirectData }> {
23
- const issues: AuditIssue[] = [];
24
- const chain: RedirectHop[] = [];
25
- const visitedUrls = new Set<string>();
26
- let currentUrl = url;
27
- let isLoop = false;
28
-
29
- while (chain.length < maxHops) {
30
- // Check for loop
31
- if (visitedUrls.has(currentUrl)) {
32
- isLoop = true;
33
- break;
34
- }
35
- visitedUrls.add(currentUrl);
36
-
37
- try {
38
- const response = await httpHead(currentUrl, {
39
- timeout: 5000,
40
- maxRedirects: 0,
41
- validateStatus: () => true,
42
- });
43
-
44
- const hop: RedirectHop = {
45
- url: currentUrl,
46
- statusCode: response.status,
47
- };
48
-
49
- if (response.status >= 300 && response.status < 400) {
50
- const location = response.headers['location'];
51
- if (location) {
52
- hop.location = location;
53
- chain.push(hop);
54
- currentUrl = new URL(location, currentUrl).href;
55
- continue;
56
- }
57
- }
58
-
59
- chain.push(hop);
60
- break;
61
- } catch (error) {
62
- chain.push({
63
- url: currentUrl,
64
- statusCode: 0,
65
- });
66
- break;
67
- }
68
- }
69
-
70
- const hasRedirect = chain.length > 1;
71
- const finalUrl = chain.length > 0 ? chain[chain.length - 1].url : url;
72
-
73
- const data: RedirectData = {
74
- hasRedirect,
75
- redirectChain: chain,
76
- isLoop,
77
- finalUrl,
78
- totalHops: chain.length - 1,
79
- };
80
-
81
- // ==================== Issue Detection ====================
82
-
83
- // Redirect loop
84
- if (isLoop) {
85
- issues.push({
86
- ...ISSUE_DEFINITIONS.REDIRECT_LOOP,
87
- affectedUrls: [url],
88
- details: {
89
- chain: chain.map(h => `${h.url} (${h.statusCode})`),
90
- },
91
- });
92
- }
93
-
94
- // Redirect chain (more than one redirect)
95
- if (!isLoop && chain.length > 2) {
96
- issues.push({
97
- ...ISSUE_DEFINITIONS.REDIRECT_CHAIN,
98
- affectedUrls: [url],
99
- details: {
100
- chain: chain.map(h => `${h.url} (${h.statusCode})`),
101
- hops: chain.length - 1,
102
- finalUrl,
103
- },
104
- });
105
- }
106
-
107
- return { issues, data };
108
- }
109
-
110
- // Check internal links for redirect targets
111
- export async function checkInternalRedirects(
112
- internalLinks: string[],
113
- batchSize: number = 10
114
- ): Promise<{ redirectLinks: Array<{ source: string; target: string; statusCode: number }> }> {
115
- const redirectLinks: Array<{ source: string; target: string; statusCode: number }> = [];
116
-
117
- // Process in batches to avoid overwhelming the server
118
- for (let i = 0; i < internalLinks.length; i += batchSize) {
119
- const batch = internalLinks.slice(i, i + batchSize);
120
- const promises = batch.map(async (link) => {
121
- try {
122
- const response = await httpHead(link, {
123
- timeout: 5000,
124
- maxRedirects: 0,
125
- validateStatus: () => true,
126
- });
127
-
128
- if (response.status >= 300 && response.status < 400) {
129
- const location = response.headers['location'];
130
- if (location) {
131
- return {
132
- source: link,
133
- target: new URL(location, link).href,
134
- statusCode: response.status,
135
- };
136
- }
137
- }
138
- return null;
139
- } catch {
140
- return null;
141
- }
142
- });
143
-
144
- const results = await Promise.all(promises);
145
- for (const result of results) {
146
- if (result) {
147
- redirectLinks.push(result);
148
- }
149
- }
150
- }
151
-
152
- return { redirectLinks };
153
- }