@rankcli/agent-runtime 0.0.13 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +215 -1
- package/dist/index.d.ts +215 -1
- package/dist/index.js +847 -90
- package/dist/index.mjs +813 -60
- package/package.json +2 -1
- package/src/audit/checks/agent-experience.ts +108 -0
- package/src/audit/checks/ai-readiness.ts +67 -0
- package/src/audit/checks/client-rendering.ts +10 -10
- package/src/audit/checks/rag-chunk-readiness.test.ts +159 -0
- package/src/audit/checks/rag-chunk-readiness.ts +163 -0
- package/src/audit/checks/security-headers.ts +18 -2
- package/src/audit/engine.ts +23 -10
- package/src/audit/types.ts +24 -0
- package/src/index.ts +3 -0
- package/src/ranking/index.ts +5 -0
- package/src/ranking/serp-client.ts +348 -0
- package/src/ranking/tracker.ts +380 -0
- package/src/ranking/types.ts +123 -0
package/dist/index.js
CHANGED
|
@@ -239,7 +239,7 @@ async function crawlUrl(params) {
|
|
|
239
239
|
async function extractMeta(params) {
|
|
240
240
|
const { html, url } = params;
|
|
241
241
|
try {
|
|
242
|
-
const $ =
|
|
242
|
+
const $ = cheerio63.load(html);
|
|
243
243
|
const meta = {
|
|
244
244
|
title: $("title").text().trim() || void 0,
|
|
245
245
|
description: $('meta[name="description"]').attr("content")?.trim(),
|
|
@@ -284,7 +284,7 @@ async function extractMeta(params) {
|
|
|
284
284
|
async function analyzeHeadings(params) {
|
|
285
285
|
const { html } = params;
|
|
286
286
|
try {
|
|
287
|
-
const $ =
|
|
287
|
+
const $ = cheerio63.load(html);
|
|
288
288
|
const headings = [];
|
|
289
289
|
$("h1, h2, h3, h4, h5, h6").each((_, el) => {
|
|
290
290
|
const tag = el.tagName.toLowerCase();
|
|
@@ -305,7 +305,7 @@ async function analyzeHeadings(params) {
|
|
|
305
305
|
async function extractImages2(params) {
|
|
306
306
|
const { html } = params;
|
|
307
307
|
try {
|
|
308
|
-
const $ =
|
|
308
|
+
const $ = cheerio63.load(html);
|
|
309
309
|
const images = [];
|
|
310
310
|
$("img").each((_, el) => {
|
|
311
311
|
images.push({
|
|
@@ -327,7 +327,7 @@ async function extractImages2(params) {
|
|
|
327
327
|
async function extractLinks3(params) {
|
|
328
328
|
const { html, baseUrl } = params;
|
|
329
329
|
try {
|
|
330
|
-
const $ =
|
|
330
|
+
const $ = cheerio63.load(html);
|
|
331
331
|
const links = [];
|
|
332
332
|
const baseHostname = new URL(baseUrl).hostname;
|
|
333
333
|
$("a[href]").each((_, el) => {
|
|
@@ -362,7 +362,7 @@ async function extractLinks3(params) {
|
|
|
362
362
|
async function extractSchema2(params) {
|
|
363
363
|
const { html } = params;
|
|
364
364
|
try {
|
|
365
|
-
const $ =
|
|
365
|
+
const $ = cheerio63.load(html);
|
|
366
366
|
const schemas = [];
|
|
367
367
|
$('script[type="application/ld+json"]').each((_, el) => {
|
|
368
368
|
try {
|
|
@@ -431,7 +431,7 @@ async function checkSitemap2(params) {
|
|
|
431
431
|
timeout: 1e4
|
|
432
432
|
});
|
|
433
433
|
if (response.status === 200 && response.data.includes("<?xml")) {
|
|
434
|
-
const $ =
|
|
434
|
+
const $ = cheerio63.load(response.data, { xmlMode: true });
|
|
435
435
|
const urlCount = $("url").length || $("sitemap").length;
|
|
436
436
|
return {
|
|
437
437
|
success: true,
|
|
@@ -460,12 +460,12 @@ async function checkSitemap2(params) {
|
|
|
460
460
|
};
|
|
461
461
|
}
|
|
462
462
|
}
|
|
463
|
-
var
|
|
463
|
+
var cheerio63;
|
|
464
464
|
var init_crawl = __esm({
|
|
465
465
|
"src/tools/crawl.ts"() {
|
|
466
466
|
"use strict";
|
|
467
467
|
init_http();
|
|
468
|
-
|
|
468
|
+
cheerio63 = __toESM(require("cheerio"));
|
|
469
469
|
}
|
|
470
470
|
});
|
|
471
471
|
|
|
@@ -862,7 +862,7 @@ function analyzeRobotsTxtForAI(robotsTxt) {
|
|
|
862
862
|
return { allowed, blocked, recommendations };
|
|
863
863
|
}
|
|
864
864
|
function detectRenderingMode(html) {
|
|
865
|
-
const $ =
|
|
865
|
+
const $ = cheerio64.load(html);
|
|
866
866
|
const signals = [];
|
|
867
867
|
const hasNextData = html.includes("__NEXT_DATA__");
|
|
868
868
|
const hasNuxtData = html.includes("__NUXT__");
|
|
@@ -903,7 +903,7 @@ function detectRenderingMode(html) {
|
|
|
903
903
|
};
|
|
904
904
|
}
|
|
905
905
|
function analyzeContentStructure(html) {
|
|
906
|
-
const $ =
|
|
906
|
+
const $ = cheerio64.load(html);
|
|
907
907
|
const jsonLdScripts = $('script[type="application/ld+json"]');
|
|
908
908
|
let hasStructuredData = jsonLdScripts.length > 0;
|
|
909
909
|
let hasFAQSchema = false;
|
|
@@ -953,7 +953,7 @@ function analyzeContentStructure(html) {
|
|
|
953
953
|
};
|
|
954
954
|
}
|
|
955
955
|
function analyzeCitationReadiness(html) {
|
|
956
|
-
const $ =
|
|
956
|
+
const $ = cheerio64.load(html);
|
|
957
957
|
const bodyText = $("body").text();
|
|
958
958
|
const hasCitations = $('cite, blockquote, [class*="citation"], [class*="reference"]').length > 0 || /\[\d+\]|\(\d{4}\)/.test(bodyText);
|
|
959
959
|
const externalLinks = $('a[href^="http"]').filter((_, el) => {
|
|
@@ -986,7 +986,7 @@ function analyzeCitationReadiness(html) {
|
|
|
986
986
|
};
|
|
987
987
|
}
|
|
988
988
|
function analyzeEntityExtraction(html) {
|
|
989
|
-
const $ =
|
|
989
|
+
const $ = cheerio64.load(html);
|
|
990
990
|
const bodyText = $("body").text();
|
|
991
991
|
const definedTerms = [];
|
|
992
992
|
$("dfn, abbr[title]").each((_, el) => {
|
|
@@ -1022,7 +1022,7 @@ function analyzeEntityExtraction(html) {
|
|
|
1022
1022
|
};
|
|
1023
1023
|
}
|
|
1024
1024
|
function calculateLLMSignals(structure, citation, entity, html) {
|
|
1025
|
-
const $ =
|
|
1025
|
+
const $ = cheerio64.load(html);
|
|
1026
1026
|
let contentClarity = 0;
|
|
1027
1027
|
if (structure.headingHierarchy === "good") contentClarity += 30;
|
|
1028
1028
|
else if (structure.headingHierarchy === "needs-work") contentClarity += 15;
|
|
@@ -1328,11 +1328,11 @@ Allow: /
|
|
|
1328
1328
|
Sitemap: ${siteUrl}/sitemap.xml
|
|
1329
1329
|
`;
|
|
1330
1330
|
}
|
|
1331
|
-
var
|
|
1331
|
+
var cheerio64, AI_CRAWLERS, AI_CRAWLERS_INFO;
|
|
1332
1332
|
var init_geo_analyzer = __esm({
|
|
1333
1333
|
"src/analyzers/geo-analyzer.ts"() {
|
|
1334
1334
|
"use strict";
|
|
1335
|
-
|
|
1335
|
+
cheerio64 = __toESM(require("cheerio"));
|
|
1336
1336
|
AI_CRAWLERS = {
|
|
1337
1337
|
// OpenAI
|
|
1338
1338
|
GPTBot: { userAgent: "GPTBot", company: "OpenAI", purpose: "ChatGPT training & browsing" },
|
|
@@ -1691,7 +1691,7 @@ function generateCWVIssues(lcp, fidInp, cls, ttfb, url) {
|
|
|
1691
1691
|
return issues;
|
|
1692
1692
|
}
|
|
1693
1693
|
function analyzeCoreWebVitals(html, url, headers) {
|
|
1694
|
-
const $ =
|
|
1694
|
+
const $ = cheerio65.load(html);
|
|
1695
1695
|
const lcp = analyzeLCP(html, $);
|
|
1696
1696
|
const fidInp = analyzeFID_INP(html, $);
|
|
1697
1697
|
const cls = analyzeCLS(html, $);
|
|
@@ -1736,11 +1736,11 @@ function analyzeCoreWebVitals(html, url, headers) {
|
|
|
1736
1736
|
issues
|
|
1737
1737
|
};
|
|
1738
1738
|
}
|
|
1739
|
-
var
|
|
1739
|
+
var cheerio65;
|
|
1740
1740
|
var init_core_web_vitals_analyzer = __esm({
|
|
1741
1741
|
"src/analyzers/core-web-vitals-analyzer.ts"() {
|
|
1742
1742
|
"use strict";
|
|
1743
|
-
|
|
1743
|
+
cheerio65 = __toESM(require("cheerio"));
|
|
1744
1744
|
}
|
|
1745
1745
|
});
|
|
1746
1746
|
|
|
@@ -2075,7 +2075,7 @@ function validateSchema(data, type) {
|
|
|
2075
2075
|
};
|
|
2076
2076
|
}
|
|
2077
2077
|
function analyzeStructuredData2(html, url) {
|
|
2078
|
-
const $ =
|
|
2078
|
+
const $ = cheerio66.load(html);
|
|
2079
2079
|
const schemas = [];
|
|
2080
2080
|
const issues = [];
|
|
2081
2081
|
const recommendations = [];
|
|
@@ -2339,11 +2339,11 @@ function generateSchemaTemplate(pageType, options) {
|
|
|
2339
2339
|
}
|
|
2340
2340
|
return JSON.stringify(result, null, 2);
|
|
2341
2341
|
}
|
|
2342
|
-
var
|
|
2342
|
+
var cheerio66, SCHEMA_REQUIREMENTS;
|
|
2343
2343
|
var init_structured_data_analyzer = __esm({
|
|
2344
2344
|
"src/analyzers/structured-data-analyzer.ts"() {
|
|
2345
2345
|
"use strict";
|
|
2346
|
-
|
|
2346
|
+
cheerio66 = __toESM(require("cheerio"));
|
|
2347
2347
|
SCHEMA_REQUIREMENTS = {
|
|
2348
2348
|
"Organization": {
|
|
2349
2349
|
required: ["name", "url"],
|
|
@@ -2459,7 +2459,7 @@ function getImageFormat2(src) {
|
|
|
2459
2459
|
}
|
|
2460
2460
|
}
|
|
2461
2461
|
function analyzeImages3(html, url) {
|
|
2462
|
-
const $ =
|
|
2462
|
+
const $ = cheerio67.load(html);
|
|
2463
2463
|
const issues = [];
|
|
2464
2464
|
const recommendations = [];
|
|
2465
2465
|
let score = 100;
|
|
@@ -2677,11 +2677,11 @@ function generateResponsiveImage(options) {
|
|
|
2677
2677
|
/>
|
|
2678
2678
|
</picture>`;
|
|
2679
2679
|
}
|
|
2680
|
-
var
|
|
2680
|
+
var cheerio67, MODERN_FORMATS3, LEGACY_FORMATS2, MAX_RECOMMENDED_DIMENSION;
|
|
2681
2681
|
var init_image_optimization_analyzer = __esm({
|
|
2682
2682
|
"src/analyzers/image-optimization-analyzer.ts"() {
|
|
2683
2683
|
"use strict";
|
|
2684
|
-
|
|
2684
|
+
cheerio67 = __toESM(require("cheerio"));
|
|
2685
2685
|
MODERN_FORMATS3 = ["webp", "avif"];
|
|
2686
2686
|
LEGACY_FORMATS2 = ["jpg", "jpeg", "png", "gif", "bmp"];
|
|
2687
2687
|
MAX_RECOMMENDED_DIMENSION = 2e3;
|
|
@@ -2722,7 +2722,7 @@ function analyzeAnchorText2(text) {
|
|
|
2722
2722
|
};
|
|
2723
2723
|
}
|
|
2724
2724
|
function analyzeInternalLinking(html, url) {
|
|
2725
|
-
const $ =
|
|
2725
|
+
const $ = cheerio68.load(html);
|
|
2726
2726
|
const issues = [];
|
|
2727
2727
|
const recommendations = [];
|
|
2728
2728
|
let score = 100;
|
|
@@ -2961,11 +2961,11 @@ function suggestInternalLinks(content, availablePages) {
|
|
|
2961
2961
|
}
|
|
2962
2962
|
return suggestions.sort((a, b) => b.relevance - a.relevance).filter((s, i, arr) => arr.findIndex((x) => x.suggestedUrl === s.suggestedUrl) === i).slice(0, 10);
|
|
2963
2963
|
}
|
|
2964
|
-
var
|
|
2964
|
+
var cheerio68, GENERIC_ANCHOR_TEXTS;
|
|
2965
2965
|
var init_internal_linking_analyzer = __esm({
|
|
2966
2966
|
"src/analyzers/internal-linking-analyzer.ts"() {
|
|
2967
2967
|
"use strict";
|
|
2968
|
-
|
|
2968
|
+
cheerio68 = __toESM(require("cheerio"));
|
|
2969
2969
|
GENERIC_ANCHOR_TEXTS = [
|
|
2970
2970
|
"click here",
|
|
2971
2971
|
"read more",
|
|
@@ -3159,7 +3159,7 @@ function analyzeMobileSpecific($) {
|
|
|
3159
3159
|
};
|
|
3160
3160
|
}
|
|
3161
3161
|
function analyzeMobileSEO(html, url) {
|
|
3162
|
-
const $ =
|
|
3162
|
+
const $ = cheerio69.load(html);
|
|
3163
3163
|
const issues = [];
|
|
3164
3164
|
const recommendations = [];
|
|
3165
3165
|
let score = 100;
|
|
@@ -3290,11 +3290,11 @@ function analyzeMobileSEO(html, url) {
|
|
|
3290
3290
|
recommendations
|
|
3291
3291
|
};
|
|
3292
3292
|
}
|
|
3293
|
-
var
|
|
3293
|
+
var cheerio69;
|
|
3294
3294
|
var init_mobile_seo_analyzer = __esm({
|
|
3295
3295
|
"src/analyzers/mobile-seo-analyzer.ts"() {
|
|
3296
3296
|
"use strict";
|
|
3297
|
-
|
|
3297
|
+
cheerio69 = __toESM(require("cheerio"));
|
|
3298
3298
|
}
|
|
3299
3299
|
});
|
|
3300
3300
|
|
|
@@ -3306,9 +3306,12 @@ __export(index_exports, {
|
|
|
3306
3306
|
LOCATION_CODES: () => LOCATION_CODES,
|
|
3307
3307
|
OG_IMAGE_SPECS: () => OG_IMAGE_SPECS,
|
|
3308
3308
|
PRIORITY_WEIGHTS: () => PRIORITY_WEIGHTS,
|
|
3309
|
+
RankTracker: () => RankTracker,
|
|
3309
3310
|
SEO_SCOPES: () => SEO_SCOPES,
|
|
3310
3311
|
SITE_PROFILE_QUESTIONS: () => SITE_PROFILE_QUESTIONS,
|
|
3311
3312
|
Schemas: () => Schemas,
|
|
3313
|
+
SerpClient: () => SerpClient,
|
|
3314
|
+
TIER_LIMITS: () => TIER_LIMITS,
|
|
3312
3315
|
addTrackingResult: () => addTrackingResult,
|
|
3313
3316
|
analyzeAnchorText: () => analyzeAnchorText,
|
|
3314
3317
|
analyzeCanonicalAdvanced: () => analyzeCanonicalAdvanced,
|
|
@@ -4440,6 +4443,24 @@ var ISSUE_DEFINITIONS = {
|
|
|
4440
4443
|
impact: "Your content will not be used for Bard/Gemini AI training (regular search unaffected).",
|
|
4441
4444
|
howToFix: 'Remove "User-agent: Google-Extended Disallow: /" if you want Google AI visibility.'
|
|
4442
4445
|
},
|
|
4446
|
+
CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS: {
|
|
4447
|
+
code: "CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS",
|
|
4448
|
+
severity: "warning",
|
|
4449
|
+
category: "ai-readiness",
|
|
4450
|
+
title: "No explicit AI crawler rules on a Cloudflare-fronted site",
|
|
4451
|
+
description: 'This site appears to be served through Cloudflare but robots.txt has no explicit Allow/Disallow rules for AI crawlers. Cloudflare blocks "mixed-use" AI crawlers by default on ad-hosting zones starting September 15, 2026, and is rolling out Pay Per Crawl / Pay Per Use gating beyond that.',
|
|
4452
|
+
impact: "Without an explicit rule, whether AI crawlers (and future citation opportunities in ChatGPT, Claude, Perplexity, etc.) can reach this site now depends on Cloudflare account-level bot-management defaults, not on this codebase \u2014 a silent, invisible-to-git failure mode.",
|
|
4453
|
+
howToFix: "Add explicit User-agent rules for GPTBot, Claude-Web/ClaudeBot, PerplexityBot, and Google-Extended in robots.txt, and confirm the matching allow/block posture in the Cloudflare dashboard under Bot Management \u2192 AI Crawl Control. If you'd rather charge than block, Cloudflare's Pay Per Crawl (AI Crawl Control \u2192 Payments tab) lets you set a per-crawl price instead of a flat allow/deny \u2014 worth a look if you get meaningful AI-crawler traffic."
|
|
4454
|
+
},
|
|
4455
|
+
NO_AGENT_EXPERIENCE_SURFACE: {
|
|
4456
|
+
code: "NO_AGENT_EXPERIENCE_SURFACE",
|
|
4457
|
+
severity: "notice",
|
|
4458
|
+
category: "ai-readiness",
|
|
4459
|
+
title: "No agent-facing discovery surface found",
|
|
4460
|
+
description: "None of the emerging AI-agent discovery conventions were found: llms-full.txt, SKILL.md, a discoverable MCP server, or an OpenAPI spec. This is a genuinely new, low-adoption category as of 2026 \u2014 most sites have none of these yet \u2014 so this is an opportunity, not a compliance failure.",
|
|
4461
|
+
impact: "AI agents (not just chatbots answering questions, but agents that browse and act on a user's behalf) increasingly prefer sites that expose machine-readable capabilities over ones that require scraping and guessing at HTML structure. Being an early, discoverable site in this space is a low-competition differentiator right now.",
|
|
4462
|
+
howToFix: "Start with whichever fits your site: llms-full.txt (a single Markdown dump of your key content \u2014 measured as fetched roughly 2x more than plain llms.txt), a SKILL.md capability manifest, or an OpenAPI spec at /openapi.json if you already have an API. A discoverable MCP server is the highest-effort, highest-payoff option if your product has one to expose."
|
|
4463
|
+
},
|
|
4443
4464
|
HIGH_JS_RENDERING_RATIO: {
|
|
4444
4465
|
code: "HIGH_JS_RENDERING_RATIO",
|
|
4445
4466
|
severity: "warning",
|
|
@@ -7386,6 +7407,62 @@ async function checkInternalRedirects(internalLinks, batchSize = 10) {
|
|
|
7386
7407
|
// src/audit/checks/ai-readiness.ts
|
|
7387
7408
|
init_http();
|
|
7388
7409
|
var cheerio13 = __toESM(require("cheerio"));
|
|
7410
|
+
|
|
7411
|
+
// src/audit/checks/agent-experience.ts
|
|
7412
|
+
init_http();
|
|
7413
|
+
var SIGNAL_PATHS = [
|
|
7414
|
+
{ path: "/llms-full.txt", description: "Full-site content dump for AI agents \u2014 the fuller sibling of llms.txt, measured as fetched roughly 2x more often" },
|
|
7415
|
+
{ path: "/skill.md", description: "SKILL.md capability manifest (Anthropic Agent Skills convention)" },
|
|
7416
|
+
{ path: "/mcp", description: "MCP (Model Context Protocol) server, discoverable at a conventional path" },
|
|
7417
|
+
{ path: "/.well-known/mcp.json", description: "MCP server descriptor at the .well-known convention" },
|
|
7418
|
+
{ path: "/openapi.json", description: "OpenAPI spec \u2014 lets an agent discover and call your API directly instead of scraping HTML" },
|
|
7419
|
+
{ path: "/.well-known/ai-plugin.json", description: "AI plugin manifest (older but still-referenced convention for agent tool discovery)" }
|
|
7420
|
+
];
|
|
7421
|
+
async function fetchBody(baseUrl, path3) {
|
|
7422
|
+
try {
|
|
7423
|
+
const url = new URL(path3, baseUrl).href;
|
|
7424
|
+
const response = await httpGet(url, {
|
|
7425
|
+
timeout: 8e3,
|
|
7426
|
+
validateStatus: () => true
|
|
7427
|
+
});
|
|
7428
|
+
return { status: response.status, body: String(response.data ?? "") };
|
|
7429
|
+
} catch {
|
|
7430
|
+
return null;
|
|
7431
|
+
}
|
|
7432
|
+
}
|
|
7433
|
+
async function pathExists(baseUrl, path3, fallbackBody) {
|
|
7434
|
+
const result = await fetchBody(baseUrl, path3);
|
|
7435
|
+
if (!result) return false;
|
|
7436
|
+
if (result.status === 404) return false;
|
|
7437
|
+
if (result.status < 200 || result.status >= 400) return false;
|
|
7438
|
+
if (fallbackBody != null && result.body === fallbackBody) return false;
|
|
7439
|
+
return true;
|
|
7440
|
+
}
|
|
7441
|
+
async function checkAgentExperience(baseUrl) {
|
|
7442
|
+
const issues = [];
|
|
7443
|
+
const probePath = `/__rankcli_ax_probe_${Math.random().toString(36).slice(2)}`;
|
|
7444
|
+
const baseline = await fetchBody(baseUrl, probePath);
|
|
7445
|
+
const fallbackBody = baseline && baseline.status >= 200 && baseline.status < 400 ? baseline.body : null;
|
|
7446
|
+
const signals = await Promise.all(
|
|
7447
|
+
SIGNAL_PATHS.map(async ({ path: path3, description }) => ({
|
|
7448
|
+
path: path3,
|
|
7449
|
+
description,
|
|
7450
|
+
present: await pathExists(baseUrl, path3, fallbackBody)
|
|
7451
|
+
}))
|
|
7452
|
+
);
|
|
7453
|
+
const presentCount = signals.filter((s) => s.present).length;
|
|
7454
|
+
const score = Math.round(presentCount / signals.length * 100);
|
|
7455
|
+
if (presentCount === 0) {
|
|
7456
|
+
issues.push({
|
|
7457
|
+
...ISSUE_DEFINITIONS.NO_AGENT_EXPERIENCE_SURFACE,
|
|
7458
|
+
affectedUrls: [baseUrl],
|
|
7459
|
+
details: { checkedPaths: SIGNAL_PATHS.map((s) => s.path) }
|
|
7460
|
+
});
|
|
7461
|
+
}
|
|
7462
|
+
return { issues, data: { signals, score } };
|
|
7463
|
+
}
|
|
7464
|
+
|
|
7465
|
+
// src/audit/checks/ai-readiness.ts
|
|
7389
7466
|
var AI_BOTS = {
|
|
7390
7467
|
GPTBot: "GPTBot",
|
|
7391
7468
|
"ChatGPT-User": "ChatGPT-User",
|
|
@@ -7606,6 +7683,32 @@ function checkJSRenderingRatio(html, url) {
|
|
|
7606
7683
|
data: { ratio, staticWordCount }
|
|
7607
7684
|
};
|
|
7608
7685
|
}
|
|
7686
|
+
async function checkCloudflareAICrawlerGate(baseUrl, botBlocking) {
|
|
7687
|
+
const issues = [];
|
|
7688
|
+
let behindCloudflare = false;
|
|
7689
|
+
try {
|
|
7690
|
+
const response = await httpGet(baseUrl, {
|
|
7691
|
+
timeout: 1e4,
|
|
7692
|
+
validateStatus: () => true
|
|
7693
|
+
});
|
|
7694
|
+
const server = response.headers["server"] || "";
|
|
7695
|
+
behindCloudflare = server.toLowerCase().includes("cloudflare") || "cf-ray" in response.headers;
|
|
7696
|
+
} catch {
|
|
7697
|
+
behindCloudflare = false;
|
|
7698
|
+
}
|
|
7699
|
+
const hasExplicitAIRules = botBlocking.robotsExists && botBlocking.blockedBots.length > 0;
|
|
7700
|
+
const ambiguous = behindCloudflare && !hasExplicitAIRules;
|
|
7701
|
+
if (ambiguous) {
|
|
7702
|
+
issues.push({
|
|
7703
|
+
...ISSUE_DEFINITIONS.CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS,
|
|
7704
|
+
affectedUrls: [baseUrl]
|
|
7705
|
+
});
|
|
7706
|
+
}
|
|
7707
|
+
return {
|
|
7708
|
+
issues,
|
|
7709
|
+
data: { behindCloudflare, hasExplicitAIRules, ambiguous }
|
|
7710
|
+
};
|
|
7711
|
+
}
|
|
7609
7712
|
async function runAIReadinessChecks(baseUrl, html) {
|
|
7610
7713
|
const allIssues = [];
|
|
7611
7714
|
const llmsResult = await checkLlmsTxt(baseUrl);
|
|
@@ -7614,12 +7717,18 @@ async function runAIReadinessChecks(baseUrl, html) {
|
|
|
7614
7717
|
allIssues.push(...botResult.issues);
|
|
7615
7718
|
const jsResult = checkJSRenderingRatio(html, baseUrl);
|
|
7616
7719
|
allIssues.push(...jsResult.issues);
|
|
7720
|
+
const cfGateResult = await checkCloudflareAICrawlerGate(baseUrl, botResult.data);
|
|
7721
|
+
allIssues.push(...cfGateResult.issues);
|
|
7722
|
+
const agentExperienceResult = await checkAgentExperience(baseUrl);
|
|
7723
|
+
allIssues.push(...agentExperienceResult.issues);
|
|
7617
7724
|
return {
|
|
7618
7725
|
issues: allIssues,
|
|
7619
7726
|
data: {
|
|
7620
7727
|
llmsTxt: llmsResult.data,
|
|
7621
7728
|
botBlocking: botResult.data,
|
|
7622
|
-
jsRenderingRatio: jsResult.data.ratio
|
|
7729
|
+
jsRenderingRatio: jsResult.data.ratio,
|
|
7730
|
+
cloudflareAIGate: cfGateResult.data,
|
|
7731
|
+
agentExperience: agentExperienceResult.data
|
|
7623
7732
|
}
|
|
7624
7733
|
};
|
|
7625
7734
|
}
|
|
@@ -8085,7 +8194,13 @@ async function analyzeSecurityHeaders(url) {
|
|
|
8085
8194
|
maxRedirects: 5
|
|
8086
8195
|
});
|
|
8087
8196
|
const headers = response.headers;
|
|
8088
|
-
|
|
8197
|
+
let isHttps = false;
|
|
8198
|
+
try {
|
|
8199
|
+
const parsedUrl = new URL(url);
|
|
8200
|
+
isHttps = parsedUrl.protocol === "https:";
|
|
8201
|
+
} catch {
|
|
8202
|
+
isHttps = url.toLowerCase().startsWith("https://");
|
|
8203
|
+
}
|
|
8089
8204
|
const getHeader = (name) => {
|
|
8090
8205
|
const key = Object.keys(headers).find((k) => k.toLowerCase() === name.toLowerCase());
|
|
8091
8206
|
return key ? headers[key] : null;
|
|
@@ -8157,10 +8272,17 @@ async function analyzeSecurityHeaders(url) {
|
|
|
8157
8272
|
}
|
|
8158
8273
|
};
|
|
8159
8274
|
} catch (error) {
|
|
8275
|
+
let isHttps = false;
|
|
8276
|
+
try {
|
|
8277
|
+
const parsedUrl = new URL(url);
|
|
8278
|
+
isHttps = parsedUrl.protocol === "https:";
|
|
8279
|
+
} catch {
|
|
8280
|
+
isHttps = url.toLowerCase().startsWith("https://");
|
|
8281
|
+
}
|
|
8160
8282
|
return {
|
|
8161
8283
|
issues,
|
|
8162
8284
|
data: {
|
|
8163
|
-
https:
|
|
8285
|
+
https: isHttps,
|
|
8164
8286
|
headers: {
|
|
8165
8287
|
hsts: null,
|
|
8166
8288
|
csp: null,
|
|
@@ -12199,17 +12321,17 @@ function generateRecommendations2(renderingMethod, hasContentInHTML, charCount,
|
|
|
12199
12321
|
);
|
|
12200
12322
|
if (framework === "React") {
|
|
12201
12323
|
recommendations.push(
|
|
12202
|
-
|
|
12324
|
+
"Quick fix: Use Vike (vike.dev) for SSR/SSG - works with Vite, minimal config needed"
|
|
12203
12325
|
);
|
|
12204
12326
|
recommendations.push(
|
|
12205
|
-
"Alternative:
|
|
12327
|
+
"Alternative: Migrate to Next.js or Remix for built-in SSR/SSG support"
|
|
12206
12328
|
);
|
|
12207
12329
|
} else if (framework === "Vue") {
|
|
12208
12330
|
recommendations.push(
|
|
12209
|
-
"Quick fix:
|
|
12331
|
+
"Quick fix: Use Vike (vike.dev) for SSR/SSG - works with Vite, minimal config needed"
|
|
12210
12332
|
);
|
|
12211
12333
|
recommendations.push(
|
|
12212
|
-
"Alternative:
|
|
12334
|
+
"Alternative: Migrate to Nuxt for built-in SSR/SSG support"
|
|
12213
12335
|
);
|
|
12214
12336
|
} else if (framework === "Angular Universal") {
|
|
12215
12337
|
recommendations.push(
|
|
@@ -12217,7 +12339,7 @@ function generateRecommendations2(renderingMethod, hasContentInHTML, charCount,
|
|
|
12217
12339
|
);
|
|
12218
12340
|
} else {
|
|
12219
12341
|
recommendations.push(
|
|
12220
|
-
"Quick fix: Use
|
|
12342
|
+
"Quick fix: Use Vike (vike.dev) for SSR/SSG with any Vite-based framework"
|
|
12221
12343
|
);
|
|
12222
12344
|
}
|
|
12223
12345
|
recommendations.push(
|
|
@@ -12241,7 +12363,7 @@ function analyzeClientRendering(html, url) {
|
|
|
12241
12363
|
const issues = [];
|
|
12242
12364
|
const analysis = analyzeRendering(html);
|
|
12243
12365
|
if (analysis.renderingMethod === "csr" && analysis.confidence !== "low") {
|
|
12244
|
-
const howToFixByFramework = analysis.frameworkDetected === "React" ?
|
|
12366
|
+
const howToFixByFramework = analysis.frameworkDetected === "React" ? "Use Vike (vike.dev) to add SSR/SSG to your Vite React app: npm install vike vike-react, then follow the setup guide. Alternatively, migrate to Next.js or Remix." : analysis.frameworkDetected === "Vue" ? "Use Vike (vike.dev) to add SSR/SSG to your Vite Vue app: npm install vike vike-vue, then follow the setup guide. Alternatively, migrate to Nuxt." : "Use Vike (vike.dev) for SSR/SSG with Vite-based frameworks, or use a framework with built-in SSR support (Next.js, Nuxt, Remix, SvelteKit).";
|
|
12245
12367
|
issues.push({
|
|
12246
12368
|
code: "CLIENT_SIDE_RENDERING",
|
|
12247
12369
|
severity: "error",
|
|
@@ -12282,7 +12404,7 @@ function analyzeClientRendering(html, url) {
|
|
|
12282
12404
|
title: `${analysis.frameworkDetected} detected without SSR markers`,
|
|
12283
12405
|
description: `This appears to be a ${analysis.frameworkDetected} SPA without server-side rendering enabled.`,
|
|
12284
12406
|
impact: "Single Page Applications without SSR have slower time-to-content for search crawlers.",
|
|
12285
|
-
howToFix: analysis.frameworkDetected === "React" ? "
|
|
12407
|
+
howToFix: analysis.frameworkDetected === "React" ? "Add SSR/SSG using Vike (vike.dev): npm install vike vike-react. Alternative: migrate to Next.js or Remix." : "Add SSR/SSG using Vike (vike.dev): npm install vike vike-vue. Alternative: migrate to Nuxt.",
|
|
12286
12408
|
affectedUrls: [url],
|
|
12287
12409
|
details: {
|
|
12288
12410
|
framework: analysis.frameworkDetected,
|
|
@@ -16061,8 +16183,94 @@ function analyzeAIContentStructure(html, url) {
|
|
|
16061
16183
|
};
|
|
16062
16184
|
}
|
|
16063
16185
|
|
|
16064
|
-
// src/audit/checks/
|
|
16186
|
+
// src/audit/checks/rag-chunk-readiness.ts
|
|
16065
16187
|
var cheerio44 = __toESM(require("cheerio"));
|
|
16188
|
+
var MIN_GOOD_WORDS = 150;
|
|
16189
|
+
var MAX_GOOD_WORDS = 450;
|
|
16190
|
+
var TOKENS_PER_WORD = 1.33;
|
|
16191
|
+
var DANGLING_OPENERS = /^(this|it|these|that|they|such|those|the former|the latter)\b/i;
|
|
16192
|
+
function classifySize(wordCount) {
|
|
16193
|
+
if (wordCount < MIN_GOOD_WORDS) return "too-short";
|
|
16194
|
+
if (wordCount > MAX_GOOD_WORDS) return "too-long";
|
|
16195
|
+
return "good";
|
|
16196
|
+
}
|
|
16197
|
+
function extractSections($) {
|
|
16198
|
+
const elements = $("h1, h2, h3, p, li, blockquote").toArray();
|
|
16199
|
+
const sections = [];
|
|
16200
|
+
let current = null;
|
|
16201
|
+
for (const el of elements) {
|
|
16202
|
+
const tag = el.tagName?.toLowerCase();
|
|
16203
|
+
if (tag === "h1" || tag === "h2" || tag === "h3") {
|
|
16204
|
+
if (current) sections.push(current);
|
|
16205
|
+
current = { heading: $(el).text().trim(), headingLevel: parseInt(tag.slice(1), 10), text: "" };
|
|
16206
|
+
} else if (current) {
|
|
16207
|
+
current.text += " " + $(el).text();
|
|
16208
|
+
}
|
|
16209
|
+
}
|
|
16210
|
+
if (current) sections.push(current);
|
|
16211
|
+
return sections;
|
|
16212
|
+
}
|
|
16213
|
+
function analyzeRAGChunkReadiness(html, url) {
|
|
16214
|
+
const issues = [];
|
|
16215
|
+
const $ = cheerio44.load(html);
|
|
16216
|
+
$("nav, footer, aside, script, style, noscript, header").remove();
|
|
16217
|
+
const rawSections = extractSections($);
|
|
16218
|
+
const sections = rawSections.map((s) => {
|
|
16219
|
+
const words = s.text.trim().split(/\s+/).filter(Boolean);
|
|
16220
|
+
const wordCount = words.length;
|
|
16221
|
+
const estimatedTokens = Math.round(wordCount * TOKENS_PER_WORD);
|
|
16222
|
+
const firstSentence = s.text.trim().split(/[.!?]/)[0] || "";
|
|
16223
|
+
return {
|
|
16224
|
+
heading: s.heading,
|
|
16225
|
+
headingLevel: s.headingLevel,
|
|
16226
|
+
wordCount,
|
|
16227
|
+
estimatedTokens,
|
|
16228
|
+
sizeQuality: classifySize(wordCount),
|
|
16229
|
+
startsWithDanglingReference: DANGLING_OPENERS.test(firstSentence.trim())
|
|
16230
|
+
};
|
|
16231
|
+
});
|
|
16232
|
+
const totalSections = sections.length;
|
|
16233
|
+
const wellSizedSectionCount = sections.filter((s) => s.sizeQuality === "good").length;
|
|
16234
|
+
const danglingReferenceCount = sections.filter((s) => s.startsWithDanglingReference).length;
|
|
16235
|
+
const chunkReadinessScore = totalSections === 0 ? 0 : Math.round(
|
|
16236
|
+
(wellSizedSectionCount / totalSections * 0.7 + (totalSections - danglingReferenceCount) / totalSections * 0.3) * 100
|
|
16237
|
+
);
|
|
16238
|
+
if (totalSections >= 3 && wellSizedSectionCount / totalSections < 0.5) {
|
|
16239
|
+
const tooLong = sections.filter((s) => s.sizeQuality === "too-long").length;
|
|
16240
|
+
const tooShort = sections.filter((s) => s.sizeQuality === "too-short").length;
|
|
16241
|
+
issues.push({
|
|
16242
|
+
code: "RAG_CHUNK_SIZE_MISMATCH",
|
|
16243
|
+
severity: "notice",
|
|
16244
|
+
category: "ai-readiness",
|
|
16245
|
+
title: "Most sections are poorly sized for AI retrieval chunking",
|
|
16246
|
+
description: `${wellSizedSectionCount} of ${totalSections} sections fall in a well-sized range for RAG retrieval (roughly 150-450 words); ${tooLong} are too long and likely to get split mid-thought, ${tooShort} are too short and likely to get merged with unrelated neighbors.`,
|
|
16247
|
+
impact: "When an AI answer engine retrieves a chunk of this page in response to a query, oversized sections risk being cut off mid-point and undersized ones risk losing standalone context \u2014 both reduce the odds the retrieved chunk reads coherently in an AI-generated answer.",
|
|
16248
|
+
howToFix: "Break up long sections (450+ words) with an additional H2/H3 subheading roughly every 300-400 words. Merge very short sections (under 150 words) into a neighboring section or expand them with enough context to stand alone.",
|
|
16249
|
+
affectedUrls: [url],
|
|
16250
|
+
details: { wellSizedSectionCount, totalSections, tooLong, tooShort }
|
|
16251
|
+
});
|
|
16252
|
+
}
|
|
16253
|
+
if (totalSections >= 3 && danglingReferenceCount / totalSections > 0.3) {
|
|
16254
|
+
issues.push({
|
|
16255
|
+
code: "RAG_CHUNK_DANGLING_REFERENCES",
|
|
16256
|
+
severity: "notice",
|
|
16257
|
+
category: "ai-readiness",
|
|
16258
|
+
title: "Several sections open with a reference to prior context",
|
|
16259
|
+
description: `${danglingReferenceCount} of ${totalSections} sections start with a pronoun or demonstrative ("This...", "It...", "These...") that depends on the previous section to make sense.`,
|
|
16260
|
+
impact: "A RAG system that retrieves one of these sections on its own \u2014 which is exactly how retrieval works, one chunk at a time \u2014 surfaces a sentence whose subject is undefined, reading as broken or confusing in an AI-generated answer.",
|
|
16261
|
+
howToFix: 'Open each section by naming its actual subject instead of referring back to the previous one \u2014 e.g. "This approach reduces cost" becomes "Caching reduces cost."',
|
|
16262
|
+
affectedUrls: [url],
|
|
16263
|
+
details: { danglingReferenceCount, totalSections }
|
|
16264
|
+
});
|
|
16265
|
+
}
|
|
16266
|
+
return {
|
|
16267
|
+
issues,
|
|
16268
|
+
data: { sections, totalSections, wellSizedSectionCount, danglingReferenceCount, chunkReadinessScore }
|
|
16269
|
+
};
|
|
16270
|
+
}
|
|
16271
|
+
|
|
16272
|
+
// src/audit/checks/citation-quality.ts
|
|
16273
|
+
var cheerio45 = __toESM(require("cheerio"));
|
|
16066
16274
|
var REPUTABLE_SOURCES = {
|
|
16067
16275
|
academic: [
|
|
16068
16276
|
"scholar.google.com",
|
|
@@ -16129,7 +16337,7 @@ var REPUTABLE_SOURCES = {
|
|
|
16129
16337
|
};
|
|
16130
16338
|
function analyzeCitationQuality(html, url) {
|
|
16131
16339
|
const issues = [];
|
|
16132
|
-
const $ =
|
|
16340
|
+
const $ = cheerio45.load(html);
|
|
16133
16341
|
const parsedUrl = new URL(url);
|
|
16134
16342
|
const currentDomain = parsedUrl.hostname;
|
|
16135
16343
|
const $content = $("body").clone();
|
|
@@ -16358,12 +16566,12 @@ function analyzeCitationQuality(html, url) {
|
|
|
16358
16566
|
}
|
|
16359
16567
|
|
|
16360
16568
|
// src/audit/checks/answer-conciseness.ts
|
|
16361
|
-
var
|
|
16569
|
+
var cheerio46 = __toESM(require("cheerio"));
|
|
16362
16570
|
var IDEAL_ANSWER_LENGTH = { min: 40, max: 150 };
|
|
16363
16571
|
var MAX_FIRST_SENTENCE = 200;
|
|
16364
16572
|
function analyzeAnswerConciseness(html, url) {
|
|
16365
16573
|
const issues = [];
|
|
16366
|
-
const $ =
|
|
16574
|
+
const $ = cheerio46.load(html);
|
|
16367
16575
|
$("nav, footer, aside, script, style, noscript, header").remove();
|
|
16368
16576
|
const headings = $("h1, h2, h3, h4, h5, h6");
|
|
16369
16577
|
let totalHeadings = 0;
|
|
@@ -16554,7 +16762,7 @@ function analyzeAnswerConciseness(html, url) {
|
|
|
16554
16762
|
}
|
|
16555
16763
|
|
|
16556
16764
|
// src/audit/checks/brand-mention-optimization.ts
|
|
16557
|
-
var
|
|
16765
|
+
var cheerio47 = __toESM(require("cheerio"));
|
|
16558
16766
|
var BRAND_DEFINITION_PATTERNS = [
|
|
16559
16767
|
/(?:we are|we're|is a|is the|is an)\s+(?:leading|premier|top|best|trusted|innovative|professional)/i,
|
|
16560
16768
|
/(?:our mission|our vision|we help|we provide|we offer|we specialize)/i,
|
|
@@ -16592,7 +16800,7 @@ var REVIEW_PLATFORMS = [
|
|
|
16592
16800
|
];
|
|
16593
16801
|
function analyzeBrandMentionOptimization(html, url) {
|
|
16594
16802
|
const issues = [];
|
|
16595
|
-
const $ =
|
|
16803
|
+
const $ = cheerio47.load(html);
|
|
16596
16804
|
const parsedUrl = new URL(url);
|
|
16597
16805
|
const $content = $("body").clone();
|
|
16598
16806
|
$content.find("nav, footer, script, style, noscript").remove();
|
|
@@ -16784,7 +16992,7 @@ function analyzeBrandMentionOptimization(html, url) {
|
|
|
16784
16992
|
}
|
|
16785
16993
|
|
|
16786
16994
|
// src/audit/checks/ai-citation-worthiness.ts
|
|
16787
|
-
var
|
|
16995
|
+
var cheerio48 = __toESM(require("cheerio"));
|
|
16788
16996
|
var ORIGINAL_DATA_PATTERNS = [
|
|
16789
16997
|
/(?:our data|our research|our study|our analysis|we found|we discovered)/i,
|
|
16790
16998
|
/(?:survey of|surveyed|interviewed|analyzed)\s+\d+/i,
|
|
@@ -16831,7 +17039,7 @@ var STOCK_IMAGE_DOMAINS = [
|
|
|
16831
17039
|
];
|
|
16832
17040
|
function analyzeAICitationWorthiness(html, url) {
|
|
16833
17041
|
const issues = [];
|
|
16834
|
-
const $ =
|
|
17042
|
+
const $ = cheerio48.load(html);
|
|
16835
17043
|
const $content = $("body").clone();
|
|
16836
17044
|
$content.find("nav, footer, script, style, noscript, aside").remove();
|
|
16837
17045
|
const bodyText = $content.text();
|
|
@@ -17056,7 +17264,7 @@ function analyzeAICitationWorthiness(html, url) {
|
|
|
17056
17264
|
}
|
|
17057
17265
|
|
|
17058
17266
|
// src/audit/checks/review-ecosystem.ts
|
|
17059
|
-
var
|
|
17267
|
+
var cheerio49 = __toESM(require("cheerio"));
|
|
17060
17268
|
var REVIEW_PLATFORMS2 = {
|
|
17061
17269
|
general: [
|
|
17062
17270
|
{ name: "Google Business", domain: "google.com/maps", aliases: ["goo.gl/maps", "maps.google"] },
|
|
@@ -17116,7 +17324,7 @@ var TRUST_BADGE_PATTERNS2 = [
|
|
|
17116
17324
|
];
|
|
17117
17325
|
function analyzeReviewEcosystem(html, url) {
|
|
17118
17326
|
const issues = [];
|
|
17119
|
-
const $ =
|
|
17327
|
+
const $ = cheerio49.load(html);
|
|
17120
17328
|
const allLinks = $("a[href]").map((_, a) => $(a).attr("href") || "").get();
|
|
17121
17329
|
const allLinksLower = allLinks.map((l) => l.toLowerCase());
|
|
17122
17330
|
const linkedPlatforms = [];
|
|
@@ -17800,11 +18008,11 @@ function analyzeStructure(content) {
|
|
|
17800
18008
|
}
|
|
17801
18009
|
|
|
17802
18010
|
// src/audit/checks/html-compliance.ts
|
|
17803
|
-
var
|
|
18011
|
+
var cheerio50 = __toESM(require("cheerio"));
|
|
17804
18012
|
init_http();
|
|
17805
18013
|
async function analyzeHtmlCompliance(html, url, headers) {
|
|
17806
18014
|
const issues = [];
|
|
17807
|
-
const $ =
|
|
18015
|
+
const $ = cheerio50.load(html);
|
|
17808
18016
|
const parsedUrl = new URL(url);
|
|
17809
18017
|
const doctypeMatch = html.match(/<!DOCTYPE\s+([^>]+)>/i);
|
|
17810
18018
|
const hasDoctype = doctypeMatch !== null;
|
|
@@ -18370,7 +18578,7 @@ function getAssetType(url) {
|
|
|
18370
18578
|
}
|
|
18371
18579
|
|
|
18372
18580
|
// src/audit/checks/dom-size.ts
|
|
18373
|
-
var
|
|
18581
|
+
var cheerio51 = __toESM(require("cheerio"));
|
|
18374
18582
|
var THRESHOLDS = {
|
|
18375
18583
|
totalElements: {
|
|
18376
18584
|
warning: 1500,
|
|
@@ -18387,7 +18595,7 @@ var THRESHOLDS = {
|
|
|
18387
18595
|
};
|
|
18388
18596
|
function analyzeDomSize(html, url) {
|
|
18389
18597
|
const issues = [];
|
|
18390
|
-
const $ =
|
|
18598
|
+
const $ = cheerio51.load(html);
|
|
18391
18599
|
const allElements = $("*");
|
|
18392
18600
|
const totalElements = allElements.length;
|
|
18393
18601
|
let maxDepth = 0;
|
|
@@ -18543,10 +18751,10 @@ function getDomReductionSuggestions(breakdown) {
|
|
|
18543
18751
|
}
|
|
18544
18752
|
|
|
18545
18753
|
// src/audit/checks/image-dimensions.ts
|
|
18546
|
-
var
|
|
18754
|
+
var cheerio52 = __toESM(require("cheerio"));
|
|
18547
18755
|
function analyzeImageDimensions(html, url) {
|
|
18548
18756
|
const issues = [];
|
|
18549
|
-
const $ =
|
|
18757
|
+
const $ = cheerio52.load(html);
|
|
18550
18758
|
const images = $("img");
|
|
18551
18759
|
const totalImages = images.length;
|
|
18552
18760
|
let withDimensions = 0;
|
|
@@ -18657,7 +18865,7 @@ function truncateSrc(src) {
|
|
|
18657
18865
|
}
|
|
18658
18866
|
|
|
18659
18867
|
// src/audit/checks/color-contrast.ts
|
|
18660
|
-
var
|
|
18868
|
+
var cheerio53 = __toESM(require("cheerio"));
|
|
18661
18869
|
var KNOWN_LOW_CONTRAST_PAIRS = [
|
|
18662
18870
|
{ fg: "#999999", bg: "#ffffff", ratio: 2.85 },
|
|
18663
18871
|
{ fg: "#888888", bg: "#ffffff", ratio: 3.54 },
|
|
@@ -18692,7 +18900,7 @@ var NAMED_COLORS = {
|
|
|
18692
18900
|
};
|
|
18693
18901
|
function analyzeColorContrast(html, url) {
|
|
18694
18902
|
const issues = [];
|
|
18695
|
-
const $ =
|
|
18903
|
+
const $ = cheerio53.load(html);
|
|
18696
18904
|
const potentialIssues = [];
|
|
18697
18905
|
let elementsAnalyzed = 0;
|
|
18698
18906
|
let passedChecks = 0;
|
|
@@ -18891,7 +19099,7 @@ function calculateContrastRatio(fg, bg) {
|
|
|
18891
19099
|
|
|
18892
19100
|
// src/audit/checks/asset-minification.ts
|
|
18893
19101
|
init_http();
|
|
18894
|
-
var
|
|
19102
|
+
var cheerio54 = __toESM(require("cheerio"));
|
|
18895
19103
|
function isMinified(content, type) {
|
|
18896
19104
|
const lines = content.split("\n");
|
|
18897
19105
|
const totalLines = lines.length;
|
|
@@ -18919,7 +19127,7 @@ function isMinified(content, type) {
|
|
|
18919
19127
|
};
|
|
18920
19128
|
}
|
|
18921
19129
|
function extractAssetUrls(html, baseUrl) {
|
|
18922
|
-
const $ =
|
|
19130
|
+
const $ = cheerio54.load(html);
|
|
18923
19131
|
const base = new URL(baseUrl);
|
|
18924
19132
|
const css = [];
|
|
18925
19133
|
const js = [];
|
|
@@ -19065,10 +19273,10 @@ async function analyzeAssetMinification(html, url) {
|
|
|
19065
19273
|
}
|
|
19066
19274
|
|
|
19067
19275
|
// src/audit/checks/page-resources.ts
|
|
19068
|
-
var
|
|
19276
|
+
var cheerio55 = __toESM(require("cheerio"));
|
|
19069
19277
|
function analyzePageResources(html, url) {
|
|
19070
19278
|
const issues = [];
|
|
19071
|
-
const $ =
|
|
19279
|
+
const $ = cheerio55.load(html);
|
|
19072
19280
|
const baseUrl = new URL(url);
|
|
19073
19281
|
const baseHostname = baseUrl.hostname;
|
|
19074
19282
|
const stylesheets = [];
|
|
@@ -19269,7 +19477,7 @@ function analyzePageResources(html, url) {
|
|
|
19269
19477
|
|
|
19270
19478
|
// src/audit/checks/responsive-css.ts
|
|
19271
19479
|
init_http();
|
|
19272
|
-
var
|
|
19480
|
+
var cheerio56 = __toESM(require("cheerio"));
|
|
19273
19481
|
function extractMediaQueries(css) {
|
|
19274
19482
|
const mediaQueries = [];
|
|
19275
19483
|
const regex = /@media\s*([^{]+)/g;
|
|
@@ -19319,7 +19527,7 @@ function classifyBreakpoints(mediaQueries) {
|
|
|
19319
19527
|
}
|
|
19320
19528
|
async function analyzeResponsiveCss(html, url) {
|
|
19321
19529
|
const issues = [];
|
|
19322
|
-
const $ =
|
|
19530
|
+
const $ = cheerio56.load(html);
|
|
19323
19531
|
const baseUrl = new URL(url);
|
|
19324
19532
|
const viewportMeta = $('meta[name="viewport"]').attr("content");
|
|
19325
19533
|
const hasViewport = !!viewportMeta;
|
|
@@ -20057,10 +20265,10 @@ var urlSafetyDatabase = {
|
|
|
20057
20265
|
};
|
|
20058
20266
|
|
|
20059
20267
|
// src/audit/checks/tracking-verification.ts
|
|
20060
|
-
var
|
|
20268
|
+
var cheerio57 = __toESM(require("cheerio"));
|
|
20061
20269
|
function analyzeTrackingVerification(html, url) {
|
|
20062
20270
|
const issues = [];
|
|
20063
|
-
const $ =
|
|
20271
|
+
const $ = cheerio57.load(html);
|
|
20064
20272
|
const ga4Match = html.match(/gtag\s*\(\s*['"]config['"]\s*,\s*['"](G-[A-Z0-9]+)['"]/i) || html.match(/googletagmanager\.com\/gtag\/js\?id=(G-[A-Z0-9]+)/i);
|
|
20065
20273
|
const hasGa4 = !!ga4Match;
|
|
20066
20274
|
const ga4MeasurementId = ga4Match ? ga4Match[1] : void 0;
|
|
@@ -20194,10 +20402,10 @@ async function runFullAudit(options) {
|
|
|
20194
20402
|
const runExtendedChecks = tier !== "free";
|
|
20195
20403
|
const parsedUrl = new URL(url);
|
|
20196
20404
|
const domain = parsedUrl.hostname;
|
|
20197
|
-
console.
|
|
20405
|
+
console.error(`
|
|
20198
20406
|
\u{1F50D} Running comprehensive SEO audit on ${url}...
|
|
20199
20407
|
`);
|
|
20200
|
-
console.
|
|
20408
|
+
console.error("\u{1F4CB} Phase 1: Crawlability checks + page fetch (parallel)...");
|
|
20201
20409
|
const [crawlabilityResult, fetchResult] = await Promise.all([
|
|
20202
20410
|
runCrawlabilityChecks(url).catch((err) => {
|
|
20203
20411
|
console.error("Crawlability check failed:", err);
|
|
@@ -20207,26 +20415,32 @@ async function runFullAudit(options) {
|
|
|
20207
20415
|
timeout: 3e4,
|
|
20208
20416
|
validateStatus: () => true
|
|
20209
20417
|
}).catch((err) => {
|
|
20418
|
+
console.error("Main page fetch failed:", err instanceof Error ? { message: err.message, cause: err.cause } : err);
|
|
20210
20419
|
return { error: err, data: "", headers: {} };
|
|
20211
20420
|
})
|
|
20212
20421
|
]);
|
|
20213
20422
|
allIssues.push(...crawlabilityResult);
|
|
20214
20423
|
if ("error" in fetchResult) {
|
|
20424
|
+
const err = fetchResult.error;
|
|
20425
|
+
const errorMsg = err instanceof Error ? err.message : "Unknown error";
|
|
20426
|
+
const errorCause = err instanceof Error && err.cause ? ` (${String(err.cause)})` : "";
|
|
20215
20427
|
allIssues.push({
|
|
20216
20428
|
code: "FETCH_ERROR",
|
|
20217
20429
|
severity: "error",
|
|
20218
|
-
category: "
|
|
20219
|
-
|
|
20220
|
-
|
|
20221
|
-
|
|
20222
|
-
|
|
20223
|
-
|
|
20430
|
+
category: "indexability",
|
|
20431
|
+
// Not crawlability - robots/sitemap checks may have succeeded
|
|
20432
|
+
title: "Page fetch failed",
|
|
20433
|
+
description: `Could not load page content: ${errorMsg}${errorCause}. Only robots.txt and sitemap checks were performed.`,
|
|
20434
|
+
impact: "Cannot perform full SEO audit without page HTML content.",
|
|
20435
|
+
howToFix: "Verify the URL is accessible, the server is responding, and there are no firewall/geo blocks.",
|
|
20436
|
+
affectedUrls: [url],
|
|
20437
|
+
details: { error: errorMsg, cause: errorCause || void 0 }
|
|
20224
20438
|
});
|
|
20225
20439
|
return createReport(url, domain, allIssues, pages);
|
|
20226
20440
|
}
|
|
20227
20441
|
const html = fetchResult.data;
|
|
20228
20442
|
const headers = fetchResult.headers;
|
|
20229
|
-
console.
|
|
20443
|
+
console.error(`\u{1F4DD} Phase 2: Running synchronous HTML checks (tier: ${tier}, limit: ${checksLimit})...`);
|
|
20230
20444
|
const onPageResult = analyzeOnPage(html, url);
|
|
20231
20445
|
const structuredDataResult = analyzeStructuredData(html, url);
|
|
20232
20446
|
const mobileResult = analyzeMobile(html, url);
|
|
@@ -20266,6 +20480,7 @@ async function runFullAudit(options) {
|
|
|
20266
20480
|
const entityResult = runPremiumChecks ? analyzeEntitySEO(html, url) : { issues: [], data: {} };
|
|
20267
20481
|
const qdfFreshnessResult = runPremiumChecks ? analyzeFreshnessSignals(html, url) : { issues: [], data: {} };
|
|
20268
20482
|
const aiContentStructureResult = runPremiumChecks ? analyzeAIContentStructure(html, url) : { issues: [], data: {} };
|
|
20483
|
+
const ragChunkReadinessResult = runPremiumChecks ? analyzeRAGChunkReadiness(html, url) : { issues: [], data: {} };
|
|
20269
20484
|
const citationQualityResult = runPremiumChecks ? analyzeCitationQuality(html, url) : { issues: [], data: {} };
|
|
20270
20485
|
const answerConcisenessResult = runPremiumChecks ? analyzeAnswerConciseness(html, url) : { issues: [], data: {} };
|
|
20271
20486
|
const brandMentionResult = runPremiumChecks ? analyzeBrandMentionOptimization(html, url) : { issues: [], data: {} };
|
|
@@ -20304,6 +20519,7 @@ async function runFullAudit(options) {
|
|
|
20304
20519
|
...entityResult.issues,
|
|
20305
20520
|
...qdfFreshnessResult.issues,
|
|
20306
20521
|
...aiContentStructureResult.issues,
|
|
20522
|
+
...ragChunkReadinessResult.issues,
|
|
20307
20523
|
...citationQualityResult.issues,
|
|
20308
20524
|
...answerConcisenessResult.issues,
|
|
20309
20525
|
...brandMentionResult.issues,
|
|
@@ -20327,7 +20543,7 @@ async function runFullAudit(options) {
|
|
|
20327
20543
|
};
|
|
20328
20544
|
}
|
|
20329
20545
|
}
|
|
20330
|
-
console.
|
|
20546
|
+
console.error("\u{1F517} Phase 3: Running async checks (parallel)...");
|
|
20331
20547
|
const safeAsync = async (name, fn, timeoutMs = 1e4) => {
|
|
20332
20548
|
try {
|
|
20333
20549
|
const resultPromise = fn();
|
|
@@ -20440,7 +20656,7 @@ async function runFullAudit(options) {
|
|
|
20440
20656
|
loadTime: perfData.loadTime,
|
|
20441
20657
|
issues: allIssues.map((i) => i.code)
|
|
20442
20658
|
});
|
|
20443
|
-
console.
|
|
20659
|
+
console.error("\n\u2705 Audit complete!\n");
|
|
20444
20660
|
return createReport(url, domain, allIssues, pages);
|
|
20445
20661
|
}
|
|
20446
20662
|
function createReport(url, domain, issues, pages) {
|
|
@@ -20601,10 +20817,10 @@ function groupIssuesByCategory(issues) {
|
|
|
20601
20817
|
}
|
|
20602
20818
|
|
|
20603
20819
|
// src/audit/checks/duplicate-content.ts
|
|
20604
|
-
var
|
|
20820
|
+
var cheerio58 = __toESM(require("cheerio"));
|
|
20605
20821
|
var import_crypto = require("crypto");
|
|
20606
20822
|
function extractContentHash(html, url) {
|
|
20607
|
-
const $ =
|
|
20823
|
+
const $ = cheerio58.load(html);
|
|
20608
20824
|
$("script, style, nav, header, footer, aside, .nav, .header, .footer, .sidebar").remove();
|
|
20609
20825
|
const title = $("title").text().trim();
|
|
20610
20826
|
const bodyText = $("body").text().replace(/\s+/g, " ").trim();
|
|
@@ -21832,7 +22048,7 @@ var PRIORITY_WEIGHTS = {
|
|
|
21832
22048
|
};
|
|
21833
22049
|
|
|
21834
22050
|
// src/keywords/engine.ts
|
|
21835
|
-
var
|
|
22051
|
+
var cheerio60 = __toESM(require("cheerio"));
|
|
21836
22052
|
init_http();
|
|
21837
22053
|
|
|
21838
22054
|
// src/keywords/prioritizer.ts
|
|
@@ -22129,7 +22345,7 @@ function enrichKeywordsWithEstimates(keywords) {
|
|
|
22129
22345
|
|
|
22130
22346
|
// src/keywords/sources/free-sources.ts
|
|
22131
22347
|
init_http();
|
|
22132
|
-
var
|
|
22348
|
+
var cheerio59 = __toESM(require("cheerio"));
|
|
22133
22349
|
var USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
|
|
22134
22350
|
async function getPeopleAlsoAsk(query) {
|
|
22135
22351
|
try {
|
|
@@ -22138,7 +22354,7 @@ async function getPeopleAlsoAsk(query) {
|
|
|
22138
22354
|
headers: { "User-Agent": USER_AGENT },
|
|
22139
22355
|
timeout: 1e4
|
|
22140
22356
|
});
|
|
22141
|
-
const $ =
|
|
22357
|
+
const $ = cheerio59.load(response.data);
|
|
22142
22358
|
const questions = [];
|
|
22143
22359
|
$("[data-sgrd]").each((_, el) => {
|
|
22144
22360
|
const text = $(el).text().trim();
|
|
@@ -22164,7 +22380,7 @@ async function getRelatedSearches(query) {
|
|
|
22164
22380
|
headers: { "User-Agent": USER_AGENT },
|
|
22165
22381
|
timeout: 1e4
|
|
22166
22382
|
});
|
|
22167
|
-
const $ =
|
|
22383
|
+
const $ = cheerio59.load(response.data);
|
|
22168
22384
|
const related = [];
|
|
22169
22385
|
$("div[data-ved] a").each((_, el) => {
|
|
22170
22386
|
const href = $(el).attr("href");
|
|
@@ -22254,7 +22470,7 @@ async function analyzeCompetitorTitles(query) {
|
|
|
22254
22470
|
headers: { "User-Agent": USER_AGENT },
|
|
22255
22471
|
timeout: 1e4
|
|
22256
22472
|
});
|
|
22257
|
-
const $ =
|
|
22473
|
+
const $ = cheerio59.load(response.data);
|
|
22258
22474
|
const titles = [];
|
|
22259
22475
|
const keywords = /* @__PURE__ */ new Set();
|
|
22260
22476
|
$("h3").each((_, el) => {
|
|
@@ -22587,7 +22803,7 @@ async function fetchPageMeta(url) {
|
|
|
22587
22803
|
const response = await httpGet(url, {
|
|
22588
22804
|
timeout: 1e4
|
|
22589
22805
|
});
|
|
22590
|
-
const $ =
|
|
22806
|
+
const $ = cheerio60.load(response.data);
|
|
22591
22807
|
return {
|
|
22592
22808
|
url,
|
|
22593
22809
|
title: $("title").text().trim() || void 0,
|
|
@@ -22718,7 +22934,7 @@ async function extractSeedKeywords(url) {
|
|
|
22718
22934
|
const response = await httpGet(url, {
|
|
22719
22935
|
timeout: 1e4
|
|
22720
22936
|
});
|
|
22721
|
-
const $ =
|
|
22937
|
+
const $ = cheerio60.load(response.data);
|
|
22722
22938
|
const seeds = /* @__PURE__ */ new Set();
|
|
22723
22939
|
const title = $("title").text().toLowerCase();
|
|
22724
22940
|
const titleWords = title.split(/[\s\-|:]+/).filter((w) => w.length > 3);
|
|
@@ -22740,7 +22956,7 @@ async function extractSeedKeywords(url) {
|
|
|
22740
22956
|
}
|
|
22741
22957
|
|
|
22742
22958
|
// src/keywords/site-crawler.ts
|
|
22743
|
-
var
|
|
22959
|
+
var cheerio61 = __toESM(require("cheerio"));
|
|
22744
22960
|
init_http();
|
|
22745
22961
|
var EXCLUDED_PATHS = [
|
|
22746
22962
|
"/cdn-cgi/",
|
|
@@ -22853,7 +23069,7 @@ async function crawlPage(url, timeout) {
|
|
|
22853
23069
|
validateStatus: (status) => status === 200
|
|
22854
23070
|
});
|
|
22855
23071
|
const html = response.data;
|
|
22856
|
-
const $ =
|
|
23072
|
+
const $ = cheerio61.load(html);
|
|
22857
23073
|
$('script, style, noscript, iframe, nav, footer, header, aside, [role="navigation"]').remove();
|
|
22858
23074
|
const title = $("title").text().trim();
|
|
22859
23075
|
const description = $('meta[name="description"]').attr("content")?.trim() || "";
|
|
@@ -25511,7 +25727,7 @@ function getDateRange(days = 28) {
|
|
|
25511
25727
|
|
|
25512
25728
|
// src/keywords/sources/competitor-analysis.ts
|
|
25513
25729
|
init_http();
|
|
25514
|
-
var
|
|
25730
|
+
var cheerio62 = __toESM(require("cheerio"));
|
|
25515
25731
|
var USER_AGENT2 = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
|
|
25516
25732
|
async function discoverCompetitorKeywords(yourDomain, seedKeywords, competitors) {
|
|
25517
25733
|
const yourKeywords = /* @__PURE__ */ new Set();
|
|
@@ -25608,7 +25824,7 @@ async function analyzeSERP(query) {
|
|
|
25608
25824
|
headers: { "User-Agent": USER_AGENT2 },
|
|
25609
25825
|
timeout: 1e4
|
|
25610
25826
|
});
|
|
25611
|
-
const $ =
|
|
25827
|
+
const $ = cheerio62.load(response.data);
|
|
25612
25828
|
const results = [];
|
|
25613
25829
|
const relatedSearches = [];
|
|
25614
25830
|
const peopleAlsoAsk = [];
|
|
@@ -37315,6 +37531,544 @@ function getAIVisibilitySummary(results) {
|
|
|
37315
37531
|
};
|
|
37316
37532
|
}
|
|
37317
37533
|
|
|
37534
|
+
// src/ranking/types.ts
|
|
37535
|
+
var TIER_LIMITS = {
|
|
37536
|
+
free: {
|
|
37537
|
+
maxKeywords: 10,
|
|
37538
|
+
checksPerDay: 1,
|
|
37539
|
+
serpFeatures: false,
|
|
37540
|
+
competitorTracking: false,
|
|
37541
|
+
historyDays: 7
|
|
37542
|
+
},
|
|
37543
|
+
solo: {
|
|
37544
|
+
maxKeywords: 100,
|
|
37545
|
+
checksPerDay: 1,
|
|
37546
|
+
serpFeatures: true,
|
|
37547
|
+
competitorTracking: false,
|
|
37548
|
+
historyDays: 30
|
|
37549
|
+
},
|
|
37550
|
+
pro: {
|
|
37551
|
+
maxKeywords: 500,
|
|
37552
|
+
checksPerDay: 2,
|
|
37553
|
+
serpFeatures: true,
|
|
37554
|
+
competitorTracking: true,
|
|
37555
|
+
historyDays: 90
|
|
37556
|
+
},
|
|
37557
|
+
agency: {
|
|
37558
|
+
maxKeywords: 2e3,
|
|
37559
|
+
checksPerDay: 4,
|
|
37560
|
+
serpFeatures: true,
|
|
37561
|
+
competitorTracking: true,
|
|
37562
|
+
historyDays: 365
|
|
37563
|
+
}
|
|
37564
|
+
};
|
|
37565
|
+
|
|
37566
|
+
// src/ranking/serp-client.ts
|
|
37567
|
+
var SerpClient = class {
|
|
37568
|
+
config;
|
|
37569
|
+
constructor(config) {
|
|
37570
|
+
this.config = config;
|
|
37571
|
+
}
|
|
37572
|
+
/**
|
|
37573
|
+
* Check ranking for a single keyword
|
|
37574
|
+
*/
|
|
37575
|
+
async checkRank(options) {
|
|
37576
|
+
const results = [];
|
|
37577
|
+
for (const keyword of options.keywords) {
|
|
37578
|
+
try {
|
|
37579
|
+
const result = await this.checkSingleKeyword({
|
|
37580
|
+
...options,
|
|
37581
|
+
keyword
|
|
37582
|
+
});
|
|
37583
|
+
results.push(result);
|
|
37584
|
+
} catch (error) {
|
|
37585
|
+
console.error(`Error checking rank for "${keyword}":`, error);
|
|
37586
|
+
results.push({
|
|
37587
|
+
keyword,
|
|
37588
|
+
position: null,
|
|
37589
|
+
url: null,
|
|
37590
|
+
serpFeatures: [],
|
|
37591
|
+
topResults: [],
|
|
37592
|
+
checkedAt: /* @__PURE__ */ new Date()
|
|
37593
|
+
});
|
|
37594
|
+
}
|
|
37595
|
+
}
|
|
37596
|
+
return results;
|
|
37597
|
+
}
|
|
37598
|
+
async checkSingleKeyword(options) {
|
|
37599
|
+
switch (this.config.provider) {
|
|
37600
|
+
case "valueserp":
|
|
37601
|
+
return this.checkViaValueSerp(options);
|
|
37602
|
+
case "serpapi":
|
|
37603
|
+
return this.checkViaSerpApi(options);
|
|
37604
|
+
case "direct":
|
|
37605
|
+
default:
|
|
37606
|
+
return this.checkViaDirect(options);
|
|
37607
|
+
}
|
|
37608
|
+
}
|
|
37609
|
+
/**
|
|
37610
|
+
* ValueSERP API implementation
|
|
37611
|
+
* Docs: https://www.valueserp.com/docs
|
|
37612
|
+
*/
|
|
37613
|
+
async checkViaValueSerp(options) {
|
|
37614
|
+
if (!this.config.apiKey) {
|
|
37615
|
+
throw new Error("ValueSERP API key required");
|
|
37616
|
+
}
|
|
37617
|
+
const params = new URLSearchParams({
|
|
37618
|
+
api_key: this.config.apiKey,
|
|
37619
|
+
q: options.keyword,
|
|
37620
|
+
location: options.country || "United States",
|
|
37621
|
+
google_domain: options.country === "US" ? "google.com" : `google.${options.country?.toLowerCase() || "com"}`,
|
|
37622
|
+
gl: options.country || "us",
|
|
37623
|
+
hl: options.language || "en",
|
|
37624
|
+
device: options.device || "desktop",
|
|
37625
|
+
num: "100"
|
|
37626
|
+
// Get top 100 results
|
|
37627
|
+
});
|
|
37628
|
+
const response = await fetch(`https://api.valueserp.com/search?${params}`);
|
|
37629
|
+
if (!response.ok) {
|
|
37630
|
+
throw new Error(`ValueSERP API error: ${response.status}`);
|
|
37631
|
+
}
|
|
37632
|
+
const data = await response.json();
|
|
37633
|
+
return this.parseValueSerpResponse(data, options.domain, options.keyword);
|
|
37634
|
+
}
|
|
37635
|
+
/**
|
|
37636
|
+
* SerpAPI implementation (alternative)
|
|
37637
|
+
* Docs: https://serpapi.com/search-api
|
|
37638
|
+
*/
|
|
37639
|
+
async checkViaSerpApi(options) {
|
|
37640
|
+
if (!this.config.apiKey) {
|
|
37641
|
+
throw new Error("SerpAPI key required");
|
|
37642
|
+
}
|
|
37643
|
+
const params = new URLSearchParams({
|
|
37644
|
+
api_key: this.config.apiKey,
|
|
37645
|
+
q: options.keyword,
|
|
37646
|
+
location: options.country === "US" ? "United States" : options.country || "United States",
|
|
37647
|
+
gl: options.country?.toLowerCase() || "us",
|
|
37648
|
+
hl: options.language || "en",
|
|
37649
|
+
device: options.device || "desktop",
|
|
37650
|
+
num: "100"
|
|
37651
|
+
});
|
|
37652
|
+
const response = await fetch(`https://serpapi.com/search?${params}`);
|
|
37653
|
+
if (!response.ok) {
|
|
37654
|
+
throw new Error(`SerpAPI error: ${response.status}`);
|
|
37655
|
+
}
|
|
37656
|
+
const data = await response.json();
|
|
37657
|
+
return this.parseSerpApiResponse(data, options.domain, options.keyword);
|
|
37658
|
+
}
|
|
37659
|
+
/**
|
|
37660
|
+
* Direct scraping fallback (rate-limited, use with caution)
|
|
37661
|
+
*/
|
|
37662
|
+
async checkViaDirect(options) {
|
|
37663
|
+
const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(options.keyword)}&num=100`;
|
|
37664
|
+
const headers = {
|
|
37665
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
37666
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
37667
|
+
"Accept-Language": options.language || "en-US,en;q=0.9"
|
|
37668
|
+
};
|
|
37669
|
+
const response = await fetch(searchUrl, { headers });
|
|
37670
|
+
if (!response.ok) {
|
|
37671
|
+
throw new Error(`Direct search failed: ${response.status}`);
|
|
37672
|
+
}
|
|
37673
|
+
const html = await response.text();
|
|
37674
|
+
return this.parseDirectSearchResults(html, options.domain, options.keyword);
|
|
37675
|
+
}
|
|
37676
|
+
parseValueSerpResponse(data, domain, keyword) {
|
|
37677
|
+
const topResults = [];
|
|
37678
|
+
const serpFeatures = [];
|
|
37679
|
+
let position = null;
|
|
37680
|
+
let url = null;
|
|
37681
|
+
if (data.organic_results) {
|
|
37682
|
+
for (let i = 0; i < data.organic_results.length; i++) {
|
|
37683
|
+
const result = data.organic_results[i];
|
|
37684
|
+
const resultDomain = this.extractDomain(result.link);
|
|
37685
|
+
topResults.push({
|
|
37686
|
+
position: i + 1,
|
|
37687
|
+
url: result.link,
|
|
37688
|
+
domain: resultDomain,
|
|
37689
|
+
title: result.title
|
|
37690
|
+
});
|
|
37691
|
+
if (this.domainMatches(resultDomain, domain) && position === null) {
|
|
37692
|
+
position = i + 1;
|
|
37693
|
+
url = result.link;
|
|
37694
|
+
}
|
|
37695
|
+
}
|
|
37696
|
+
}
|
|
37697
|
+
if (data.answer_box) {
|
|
37698
|
+
serpFeatures.push({
|
|
37699
|
+
type: "featured_snippet",
|
|
37700
|
+
position: 0,
|
|
37701
|
+
hasOwnSite: this.domainMatches(this.extractDomain(data.answer_box.link || ""), domain)
|
|
37702
|
+
});
|
|
37703
|
+
}
|
|
37704
|
+
if (data.people_also_ask) {
|
|
37705
|
+
serpFeatures.push({
|
|
37706
|
+
type: "people_also_ask",
|
|
37707
|
+
hasOwnSite: data.people_also_ask.some(
|
|
37708
|
+
(paa) => this.domainMatches(this.extractDomain(paa.link || ""), domain)
|
|
37709
|
+
)
|
|
37710
|
+
});
|
|
37711
|
+
}
|
|
37712
|
+
if (data.local_results) {
|
|
37713
|
+
serpFeatures.push({ type: "local_pack" });
|
|
37714
|
+
}
|
|
37715
|
+
if (data.knowledge_graph) {
|
|
37716
|
+
serpFeatures.push({ type: "knowledge_panel" });
|
|
37717
|
+
}
|
|
37718
|
+
return {
|
|
37719
|
+
keyword,
|
|
37720
|
+
position,
|
|
37721
|
+
url,
|
|
37722
|
+
serpFeatures,
|
|
37723
|
+
topResults: topResults.slice(0, 10),
|
|
37724
|
+
checkedAt: /* @__PURE__ */ new Date()
|
|
37725
|
+
};
|
|
37726
|
+
}
|
|
37727
|
+
parseSerpApiResponse(data, domain, keyword) {
|
|
37728
|
+
const topResults = [];
|
|
37729
|
+
const serpFeatures = [];
|
|
37730
|
+
let position = null;
|
|
37731
|
+
let url = null;
|
|
37732
|
+
if (data.organic_results) {
|
|
37733
|
+
for (let i = 0; i < data.organic_results.length; i++) {
|
|
37734
|
+
const result = data.organic_results[i];
|
|
37735
|
+
const resultDomain = this.extractDomain(result.link);
|
|
37736
|
+
topResults.push({
|
|
37737
|
+
position: result.position || i + 1,
|
|
37738
|
+
url: result.link,
|
|
37739
|
+
domain: resultDomain,
|
|
37740
|
+
title: result.title
|
|
37741
|
+
});
|
|
37742
|
+
if (this.domainMatches(resultDomain, domain) && position === null) {
|
|
37743
|
+
position = result.position || i + 1;
|
|
37744
|
+
url = result.link;
|
|
37745
|
+
}
|
|
37746
|
+
}
|
|
37747
|
+
}
|
|
37748
|
+
if (data.answer_box) {
|
|
37749
|
+
serpFeatures.push({
|
|
37750
|
+
type: "featured_snippet",
|
|
37751
|
+
position: 0
|
|
37752
|
+
});
|
|
37753
|
+
}
|
|
37754
|
+
if (data.related_questions) {
|
|
37755
|
+
serpFeatures.push({ type: "people_also_ask" });
|
|
37756
|
+
}
|
|
37757
|
+
return {
|
|
37758
|
+
keyword,
|
|
37759
|
+
position,
|
|
37760
|
+
url,
|
|
37761
|
+
serpFeatures,
|
|
37762
|
+
topResults: topResults.slice(0, 10),
|
|
37763
|
+
checkedAt: /* @__PURE__ */ new Date()
|
|
37764
|
+
};
|
|
37765
|
+
}
|
|
37766
|
+
parseDirectSearchResults(html, domain, keyword) {
|
|
37767
|
+
const topResults = [];
|
|
37768
|
+
let position = null;
|
|
37769
|
+
let url = null;
|
|
37770
|
+
const linkRegex = /<a[^>]+href="\/url\?q=([^"&]+)/g;
|
|
37771
|
+
let match;
|
|
37772
|
+
let index = 0;
|
|
37773
|
+
while ((match = linkRegex.exec(html)) !== null && index < 100) {
|
|
37774
|
+
try {
|
|
37775
|
+
const decodedUrl = decodeURIComponent(match[1]);
|
|
37776
|
+
const resultDomain = this.extractDomain(decodedUrl);
|
|
37777
|
+
if (resultDomain.includes("google.com") || resultDomain.includes("gstatic.com")) {
|
|
37778
|
+
continue;
|
|
37779
|
+
}
|
|
37780
|
+
index++;
|
|
37781
|
+
topResults.push({
|
|
37782
|
+
position: index,
|
|
37783
|
+
url: decodedUrl,
|
|
37784
|
+
domain: resultDomain
|
|
37785
|
+
});
|
|
37786
|
+
if (this.domainMatches(resultDomain, domain) && position === null) {
|
|
37787
|
+
position = index;
|
|
37788
|
+
url = decodedUrl;
|
|
37789
|
+
}
|
|
37790
|
+
} catch {
|
|
37791
|
+
}
|
|
37792
|
+
}
|
|
37793
|
+
return {
|
|
37794
|
+
keyword,
|
|
37795
|
+
position,
|
|
37796
|
+
url,
|
|
37797
|
+
serpFeatures: [],
|
|
37798
|
+
// Direct scraping doesn't easily extract SERP features
|
|
37799
|
+
topResults: topResults.slice(0, 10),
|
|
37800
|
+
checkedAt: /* @__PURE__ */ new Date()
|
|
37801
|
+
};
|
|
37802
|
+
}
|
|
37803
|
+
extractDomain(url) {
|
|
37804
|
+
try {
|
|
37805
|
+
const parsed = new URL(url);
|
|
37806
|
+
return parsed.hostname.replace(/^www\./, "");
|
|
37807
|
+
} catch {
|
|
37808
|
+
return "";
|
|
37809
|
+
}
|
|
37810
|
+
}
|
|
37811
|
+
domainMatches(resultDomain, targetDomain) {
|
|
37812
|
+
const normalizedResult = resultDomain.toLowerCase().replace(/^www\./, "");
|
|
37813
|
+
const normalizedTarget = targetDomain.toLowerCase().replace(/^www\./, "");
|
|
37814
|
+
return normalizedResult === normalizedTarget || normalizedResult.endsWith(`.${normalizedTarget}`);
|
|
37815
|
+
}
|
|
37816
|
+
};
|
|
37817
|
+
|
|
37818
|
+
// src/ranking/tracker.ts
|
|
37819
|
+
var RankTracker = class {
|
|
37820
|
+
supabase;
|
|
37821
|
+
serpClient;
|
|
37822
|
+
constructor(config) {
|
|
37823
|
+
this.supabase = config.supabase;
|
|
37824
|
+
this.serpClient = new SerpClient(config.serpConfig);
|
|
37825
|
+
}
|
|
37826
|
+
/**
|
|
37827
|
+
* Add keywords to track for a project
|
|
37828
|
+
*/
|
|
37829
|
+
async addKeywords(projectId, keywords, options) {
|
|
37830
|
+
const keywordRecords = keywords.map((keyword) => ({
|
|
37831
|
+
project_id: projectId,
|
|
37832
|
+
keyword: keyword.toLowerCase().trim(),
|
|
37833
|
+
search_engine: options?.searchEngine || "google",
|
|
37834
|
+
country: options?.country || "US",
|
|
37835
|
+
language: options?.language || "en",
|
|
37836
|
+
track_url: options?.trackUrl,
|
|
37837
|
+
is_active: true
|
|
37838
|
+
}));
|
|
37839
|
+
const { data, error } = await this.supabase.from("keywords").upsert(keywordRecords, {
|
|
37840
|
+
onConflict: "project_id,keyword",
|
|
37841
|
+
ignoreDuplicates: false
|
|
37842
|
+
}).select();
|
|
37843
|
+
if (error) {
|
|
37844
|
+
throw new Error(`Failed to add keywords: ${error.message}`);
|
|
37845
|
+
}
|
|
37846
|
+
return (data || []).map(this.mapKeyword);
|
|
37847
|
+
}
|
|
37848
|
+
/**
|
|
37849
|
+
* Remove keywords from tracking
|
|
37850
|
+
*/
|
|
37851
|
+
async removeKeywords(projectId, keywords) {
|
|
37852
|
+
const normalizedKeywords = keywords.map((k) => k.toLowerCase().trim());
|
|
37853
|
+
const { error } = await this.supabase.from("keywords").update({ is_active: false }).eq("project_id", projectId).in("keyword", normalizedKeywords);
|
|
37854
|
+
if (error) {
|
|
37855
|
+
throw new Error(`Failed to remove keywords: ${error.message}`);
|
|
37856
|
+
}
|
|
37857
|
+
}
|
|
37858
|
+
/**
|
|
37859
|
+
* Get all tracked keywords for a project
|
|
37860
|
+
*/
|
|
37861
|
+
async getKeywords(projectId, includeInactive = false) {
|
|
37862
|
+
let query = this.supabase.from("keywords").select("*").eq("project_id", projectId);
|
|
37863
|
+
if (!includeInactive) {
|
|
37864
|
+
query = query.eq("is_active", true);
|
|
37865
|
+
}
|
|
37866
|
+
const { data, error } = await query;
|
|
37867
|
+
if (error) {
|
|
37868
|
+
throw new Error(`Failed to get keywords: ${error.message}`);
|
|
37869
|
+
}
|
|
37870
|
+
return (data || []).map(this.mapKeyword);
|
|
37871
|
+
}
|
|
37872
|
+
/**
|
|
37873
|
+
* Check rankings for all active keywords in a project
|
|
37874
|
+
*/
|
|
37875
|
+
async checkRankings(projectId, domain) {
|
|
37876
|
+
const keywords = await this.getKeywords(projectId);
|
|
37877
|
+
if (keywords.length === 0) {
|
|
37878
|
+
return [];
|
|
37879
|
+
}
|
|
37880
|
+
const groups = this.groupKeywords(keywords);
|
|
37881
|
+
const results = [];
|
|
37882
|
+
for (const group of groups) {
|
|
37883
|
+
const checkResults = await this.serpClient.checkRank({
|
|
37884
|
+
keywords: group.keywords.map((k) => k.keyword),
|
|
37885
|
+
domain,
|
|
37886
|
+
searchEngine: group.searchEngine,
|
|
37887
|
+
country: group.country,
|
|
37888
|
+
language: group.language
|
|
37889
|
+
});
|
|
37890
|
+
for (const result of checkResults) {
|
|
37891
|
+
const keyword = group.keywords.find((k) => k.keyword === result.keyword);
|
|
37892
|
+
if (!keyword) continue;
|
|
37893
|
+
await this.saveRanking(keyword.id, result);
|
|
37894
|
+
results.push({
|
|
37895
|
+
keywordId: keyword.id,
|
|
37896
|
+
keyword: result.keyword,
|
|
37897
|
+
position: result.position,
|
|
37898
|
+
url: result.url,
|
|
37899
|
+
serpFeatures: result.serpFeatures,
|
|
37900
|
+
competitorUrls: result.topResults,
|
|
37901
|
+
checkedAt: result.checkedAt
|
|
37902
|
+
});
|
|
37903
|
+
}
|
|
37904
|
+
}
|
|
37905
|
+
await this.supabase.from("projects").update({ last_rank_check_at: (/* @__PURE__ */ new Date()).toISOString() }).eq("id", projectId);
|
|
37906
|
+
return results;
|
|
37907
|
+
}
|
|
37908
|
+
/**
|
|
37909
|
+
* Save a ranking result to the database
|
|
37910
|
+
*/
|
|
37911
|
+
async saveRanking(keywordId, result) {
|
|
37912
|
+
const { data: currentKeyword } = await this.supabase.from("keywords").select("current_position, best_position").eq("id", keywordId).single();
|
|
37913
|
+
const previousPosition = currentKeyword?.current_position;
|
|
37914
|
+
const bestPosition = currentKeyword?.best_position;
|
|
37915
|
+
const newBestPosition = result.position !== null && (bestPosition === null || result.position < bestPosition) ? result.position : bestPosition;
|
|
37916
|
+
const { error: updateError } = await this.supabase.from("keywords").update({
|
|
37917
|
+
previous_position: previousPosition,
|
|
37918
|
+
current_position: result.position,
|
|
37919
|
+
best_position: newBestPosition,
|
|
37920
|
+
last_checked: result.checkedAt.toISOString()
|
|
37921
|
+
}).eq("id", keywordId);
|
|
37922
|
+
if (updateError) {
|
|
37923
|
+
console.error(`Failed to save ranking for ${result.keyword}:`, updateError);
|
|
37924
|
+
}
|
|
37925
|
+
}
|
|
37926
|
+
/**
|
|
37927
|
+
* Get ranking history for a keyword
|
|
37928
|
+
* Note: Full history requires keyword_ranking_history table with keyword_ranking_id
|
|
37929
|
+
* For now, returns current state as single history entry
|
|
37930
|
+
*/
|
|
37931
|
+
async getHistory(keywordId, _days = 30) {
|
|
37932
|
+
const { data, error } = await this.supabase.from("keywords").select("current_position, target_url, last_checked").eq("id", keywordId).single();
|
|
37933
|
+
if (error || !data) {
|
|
37934
|
+
return [];
|
|
37935
|
+
}
|
|
37936
|
+
return [{
|
|
37937
|
+
position: data.current_position,
|
|
37938
|
+
url: data.target_url,
|
|
37939
|
+
serpFeatures: [],
|
|
37940
|
+
recordedAt: data.last_checked ? new Date(data.last_checked) : /* @__PURE__ */ new Date()
|
|
37941
|
+
}];
|
|
37942
|
+
}
|
|
37943
|
+
/**
|
|
37944
|
+
* Get keyword trends for a project
|
|
37945
|
+
*/
|
|
37946
|
+
async getTrends(projectId) {
|
|
37947
|
+
const { data, error } = await this.supabase.from("keyword_rank_trends").select("*").eq("project_id", projectId);
|
|
37948
|
+
if (error) {
|
|
37949
|
+
return this.calculateTrends(projectId);
|
|
37950
|
+
}
|
|
37951
|
+
return (data || []).map((row) => ({
|
|
37952
|
+
keywordId: row.keyword_id,
|
|
37953
|
+
keyword: row.keyword,
|
|
37954
|
+
currentPosition: row.current_position,
|
|
37955
|
+
bestPosition: row.best_position,
|
|
37956
|
+
positionChange: row.position_change || 0,
|
|
37957
|
+
avgPosition: row.avg_position || 0,
|
|
37958
|
+
dataPoints: row.data_points || 0
|
|
37959
|
+
}));
|
|
37960
|
+
}
|
|
37961
|
+
/**
|
|
37962
|
+
* Manual trend calculation fallback
|
|
37963
|
+
*/
|
|
37964
|
+
async calculateTrends(projectId) {
|
|
37965
|
+
const keywords = await this.getKeywords(projectId);
|
|
37966
|
+
const trends = [];
|
|
37967
|
+
for (const keyword of keywords) {
|
|
37968
|
+
const history = await this.getHistory(keyword.id, 30);
|
|
37969
|
+
if (history.length === 0) {
|
|
37970
|
+
trends.push({
|
|
37971
|
+
keywordId: keyword.id,
|
|
37972
|
+
keyword: keyword.keyword,
|
|
37973
|
+
currentPosition: keyword.currentPosition,
|
|
37974
|
+
bestPosition: keyword.bestPosition,
|
|
37975
|
+
positionChange: 0,
|
|
37976
|
+
avgPosition: keyword.currentPosition || 0,
|
|
37977
|
+
dataPoints: 0
|
|
37978
|
+
});
|
|
37979
|
+
continue;
|
|
37980
|
+
}
|
|
37981
|
+
const positions = history.filter((h) => h.position !== null).map((h) => h.position);
|
|
37982
|
+
const avgPosition = positions.length > 0 ? positions.reduce((a, b) => a + b, 0) / positions.length : 0;
|
|
37983
|
+
const positionChange = keyword.previousPosition && keyword.currentPosition ? keyword.previousPosition - keyword.currentPosition : 0;
|
|
37984
|
+
trends.push({
|
|
37985
|
+
keywordId: keyword.id,
|
|
37986
|
+
keyword: keyword.keyword,
|
|
37987
|
+
currentPosition: keyword.currentPosition,
|
|
37988
|
+
bestPosition: keyword.bestPosition,
|
|
37989
|
+
positionChange,
|
|
37990
|
+
avgPosition,
|
|
37991
|
+
dataPoints: history.length
|
|
37992
|
+
});
|
|
37993
|
+
}
|
|
37994
|
+
return trends;
|
|
37995
|
+
}
|
|
37996
|
+
/**
|
|
37997
|
+
* Export ranking data as CSV
|
|
37998
|
+
*/
|
|
37999
|
+
async exportCSV(projectId, days = 30) {
|
|
38000
|
+
const keywords = await this.getKeywords(projectId);
|
|
38001
|
+
const rows = ["Keyword,Current Position,Best Position,Last Checked,Trend"];
|
|
38002
|
+
for (const keyword of keywords) {
|
|
38003
|
+
const history = await this.getHistory(keyword.id, days);
|
|
38004
|
+
const trend = this.calculatePositionTrend(history);
|
|
38005
|
+
rows.push([
|
|
38006
|
+
`"${keyword.keyword}"`,
|
|
38007
|
+
keyword.currentPosition?.toString() || "N/A",
|
|
38008
|
+
keyword.bestPosition?.toString() || "N/A",
|
|
38009
|
+
keyword.lastChecked?.toISOString() || "Never",
|
|
38010
|
+
trend
|
|
38011
|
+
].join(","));
|
|
38012
|
+
}
|
|
38013
|
+
return rows.join("\n");
|
|
38014
|
+
}
|
|
38015
|
+
/**
|
|
38016
|
+
* Calculate position trend from history
|
|
38017
|
+
*/
|
|
38018
|
+
calculatePositionTrend(history) {
|
|
38019
|
+
if (history.length < 2) return "\u2192";
|
|
38020
|
+
const recent = history[0]?.position;
|
|
38021
|
+
const older = history[history.length - 1]?.position;
|
|
38022
|
+
if (recent === null || older === null) return "\u2192";
|
|
38023
|
+
if (recent < older) return "\u2191";
|
|
38024
|
+
if (recent > older) return "\u2193";
|
|
38025
|
+
return "\u2192";
|
|
38026
|
+
}
|
|
38027
|
+
/**
|
|
38028
|
+
* Group keywords by search engine and country
|
|
38029
|
+
*/
|
|
38030
|
+
groupKeywords(keywords) {
|
|
38031
|
+
const groups = /* @__PURE__ */ new Map();
|
|
38032
|
+
for (const keyword of keywords) {
|
|
38033
|
+
const key = `${keyword.searchEngine}:${keyword.country}:${keyword.language}`;
|
|
38034
|
+
if (!groups.has(key)) {
|
|
38035
|
+
groups.set(key, []);
|
|
38036
|
+
}
|
|
38037
|
+
groups.get(key).push(keyword);
|
|
38038
|
+
}
|
|
38039
|
+
return Array.from(groups.entries()).map(([key, keywords2]) => {
|
|
38040
|
+
const [searchEngine, country, language] = key.split(":");
|
|
38041
|
+
return {
|
|
38042
|
+
searchEngine,
|
|
38043
|
+
country,
|
|
38044
|
+
language,
|
|
38045
|
+
keywords: keywords2
|
|
38046
|
+
};
|
|
38047
|
+
});
|
|
38048
|
+
}
|
|
38049
|
+
/**
|
|
38050
|
+
* Map database row to TrackedKeyword
|
|
38051
|
+
*/
|
|
38052
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
38053
|
+
mapKeyword(row) {
|
|
38054
|
+
return {
|
|
38055
|
+
id: row.id,
|
|
38056
|
+
projectId: row.project_id,
|
|
38057
|
+
keyword: row.keyword,
|
|
38058
|
+
searchEngine: row.search_engine || "google",
|
|
38059
|
+
country: row.country || "US",
|
|
38060
|
+
language: row.language || "en",
|
|
38061
|
+
currentPosition: row.current_position,
|
|
38062
|
+
previousPosition: row.previous_position,
|
|
38063
|
+
bestPosition: row.best_position,
|
|
38064
|
+
trackUrl: row.track_url,
|
|
38065
|
+
isActive: row.is_active,
|
|
38066
|
+
lastChecked: row.last_checked ? new Date(row.last_checked) : null,
|
|
38067
|
+
createdAt: new Date(row.created_at)
|
|
38068
|
+
};
|
|
38069
|
+
}
|
|
38070
|
+
};
|
|
38071
|
+
|
|
37318
38072
|
// src/analyzers/index.ts
|
|
37319
38073
|
var analyzers_exports = {};
|
|
37320
38074
|
__export(analyzers_exports, {
|
|
@@ -38506,9 +39260,12 @@ if (typeof globalThis !== "undefined") {
|
|
|
38506
39260
|
LOCATION_CODES,
|
|
38507
39261
|
OG_IMAGE_SPECS,
|
|
38508
39262
|
PRIORITY_WEIGHTS,
|
|
39263
|
+
RankTracker,
|
|
38509
39264
|
SEO_SCOPES,
|
|
38510
39265
|
SITE_PROFILE_QUESTIONS,
|
|
38511
39266
|
Schemas,
|
|
39267
|
+
SerpClient,
|
|
39268
|
+
TIER_LIMITS,
|
|
38512
39269
|
addTrackingResult,
|
|
38513
39270
|
analyzeAnchorText,
|
|
38514
39271
|
analyzeCanonicalAdvanced,
|