@aexol/spectral 0.9.43 → 0.9.45

package/dist/extensions/seo/analyzers/content-optimize.d.ts

@@ -0,0 +1,3 @@
+import type { ContentOptimizationResult, PageData } from "../types.js";
+export declare function optimizeContent(pageData: PageData): ContentOptimizationResult;
+//# sourceMappingURL=content-optimize.d.ts.map

package/dist/extensions/seo/analyzers/content-optimize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-optimize.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/content-optimize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAe,MAAM,aAAa,CAAC;AAepF,wBAAgB,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,yBAAyB,CAyL7E"}

package/dist/extensions/seo/analyzers/content-optimize.js

@@ -0,0 +1,190 @@
+const TITLE_MIN = 30;
+const TITLE_MAX = 60;
+const TITLE_IDEAL_MIN = 50;
+const TITLE_IDEAL_MAX = 60;
+const DESC_MIN = 120;
+const DESC_MAX = 160;
+const DESC_IDEAL_MIN = 150;
+const DESC_IDEAL_MAX = 160;
+const CONTENT_MIN_WORDS = 300;
+const CONTENT_GOOD_WORDS = 1500;
+export function optimizeContent(pageData) {
+    const issues = [];
+    let titleScore = 100;
+    let descriptionScore = 100;
+    let headingStructureScore = 100;
+    let contentLengthScore = 100;
+    // === Title analysis ===
+    const title = pageData.title ?? "";
+    const titleLen = title.length;
+    if (!title) {
+        titleScore = 0;
+        issues.push({
+            category: "Title",
+            severity: "critical",
+            issue: "Missing page title",
+            recommendation: "Add a descriptive <title> tag between 50-60 characters.",
+        });
+    }
+    else if (titleLen < TITLE_MIN) {
+        titleScore -= 30;
+        issues.push({
+            category: "Title",
+            severity: "high",
+            issue: `Title too short (${titleLen} chars, minimum ${TITLE_MIN})`,
+            recommendation: "Expand title to 50-60 characters. Include primary keyword near the beginning.",
+        });
+    }
+    else if (titleLen > TITLE_MAX) {
+        titleScore -= 25;
+        issues.push({
+            category: "Title",
+            severity: "medium",
+            issue: `Title too long (${titleLen} chars, max ${TITLE_MAX})`,
+            recommendation: `Trim title to ${TITLE_MAX} characters. Current title will be truncated in SERP.`,
+        });
+    }
+    else if (titleLen < TITLE_IDEAL_MIN) {
+        titleScore -= 10;
+        issues.push({
+            category: "Title",
+            severity: "low",
+            issue: `Title slightly short (${titleLen} chars, ideal ${TITLE_IDEAL_MIN}-${TITLE_IDEAL_MAX})`,
+            recommendation: "Expand title slightly for better SERP visibility.",
+        });
+    }
+    // === Meta description analysis ===
+    const desc = pageData.metaDescription ?? "";
+    const descLen = desc.length;
+    if (!desc) {
+        descriptionScore = 0;
+        issues.push({
+            category: "Meta Description",
+            severity: "critical",
+            issue: "Missing meta description",
+            recommendation: "Add a meta description between 150-160 characters with a clear call to action.",
+        });
+    }
+    else if (descLen < DESC_MIN) {
+        descriptionScore -= 35;
+        issues.push({
+            category: "Meta Description",
+            severity: "high",
+            issue: `Meta description too short (${descLen} chars, minimum ${DESC_MIN})`,
+            recommendation: "Expand meta description to 150-160 characters. Include primary and secondary keywords naturally.",
+        });
+    }
+    else if (descLen > DESC_MAX) {
+        descriptionScore -= 20;
+        issues.push({
+            category: "Meta Description",
+            severity: "medium",
+            issue: `Meta description too long (${descLen} chars, max ${DESC_MAX})`,
+            recommendation: `Trim description to ${DESC_MAX} characters. Google typically truncates at ~160 chars.`,
+        });
+    }
+    else if (descLen < DESC_IDEAL_MIN) {
+        descriptionScore -= 5;
+        issues.push({
+            category: "Meta Description",
+            severity: "low",
+            issue: `Meta description slightly short (${descLen} chars, ideal ${DESC_IDEAL_MIN}-${DESC_IDEAL_MAX})`,
+            recommendation: "Slightly expand description to fill available SERP space.",
+        });
+    }
+    // Duplicate title/description check
+    if (title === desc) {
+        titleScore -= 40;
+        descriptionScore -= 40;
+        issues.push({
+            category: "Content",
+            severity: "high",
+            issue: "Page title and meta description are identical",
+            recommendation: "Title and description serve different purposes. Make them distinct — title for keyword targeting, description for CTR.",
+        });
+    }
+    // === Heading structure analysis ===
+    if (pageData.h1Count === 0) {
+        headingStructureScore -= 30;
+        issues.push({
+            category: "Heading Structure",
+            severity: "high",
+            issue: "No H1 heading found",
+            recommendation: "Add a single, descriptive H1 that includes your primary keyword.",
+        });
+    }
+    else if (pageData.h1Count > 1) {
+        headingStructureScore -= 20;
+        issues.push({
+            category: "Heading Structure",
+            severity: "medium",
+            issue: `Multiple H1 headings (${pageData.h1Count})`,
+            recommendation: "Use exactly one H1 per page. Convert extra H1s to H2s.",
+        });
+    }
+    if (pageData.h2Count === 0 && pageData.wordCount > 200) {
+        headingStructureScore -= 15;
+        issues.push({
+            category: "Heading Structure",
+            severity: "medium",
+            issue: "No H2 headings for content longer than 200 words",
+            recommendation: "Add H2 headings to break content into scannable sections.",
+        });
+    }
+    // Check heading hierarchy gaps (e.g., H1 → H3 skipping H2)
+    const headingLevels = pageData.headings.map((h) => h.level);
+    for (let i = 1; i < headingLevels.length; i++) {
+        const gap = headingLevels[i] - headingLevels[i - 1];
+        if (gap > 1) {
+            headingStructureScore -= 10;
+            issues.push({
+                category: "Heading Structure",
+                severity: "low",
+                issue: `Heading level skip: H${headingLevels[i - 1]} → H${headingLevels[i]} without intermediate level`,
+                recommendation: `Add an H${headingLevels[i - 1] + 1} between these headings for proper hierarchy.`,
+            });
+            break; // Report once
+        }
+    }
+    // Check suspicious headings
+    const suspiciousHeadings = pageData.headings.filter((h) => h.suspicious);
+    if (suspiciousHeadings.length > 0) {
+        headingStructureScore -= 10;
+        issues.push({
+            category: "Heading Structure",
+            severity: "low",
+            issue: `${suspiciousHeadings.length} suspicious heading(s) found (e.g., "click here", "learn more")`,
+            recommendation: "Use descriptive, keyword-rich headings instead of generic labels.",
+        });
+    }
+    // === Content length analysis ===
+    if (pageData.wordCount < CONTENT_MIN_WORDS) {
+        contentLengthScore -= 40;
+        issues.push({
+            category: "Content Length",
+            severity: "high",
+            issue: `Thin content: ${pageData.wordCount} words (minimum ${CONTENT_MIN_WORDS})`,
+            recommendation: `Expand content to at least ${CONTENT_MIN_WORDS} words. Pages under ${CONTENT_MIN_WORDS} words are considered thin content by Google.`,
+        });
+    }
+    else if (pageData.wordCount < CONTENT_GOOD_WORDS) {
+        contentLengthScore -= 15;
+        issues.push({
+            category: "Content Length",
+            severity: "low",
+            issue: `Content below optimal length: ${pageData.wordCount} words (optimal ${CONTENT_GOOD_WORDS}+)`,
+            recommendation: "Consider expanding to 1500+ words for competitive topics.",
+        });
+    }
+    // === Overall score ===
+    const overallScore = Math.round((titleScore * 0.25 + descriptionScore * 0.20 + headingStructureScore * 0.25 + contentLengthScore * 0.30));
+    return {
+        url: pageData.url,
+        titleScore,
+        descriptionScore,
+        headingStructureScore,
+        contentLengthScore,
+        overallScore,
+        issues,
+    };
+}

package/dist/extensions/seo/analyzers/crawler.d.ts

@@ -0,0 +1,4 @@
+import type { SeoConfig } from "../config.js";
+import type { CrawlConfig, CrawlResult } from "../types.js";
+export declare function crawlSite(startUrl: string, config: SeoConfig, crawlConfig?: Partial<CrawlConfig>): Promise<CrawlResult>;
+//# sourceMappingURL=crawler.d.ts.map

package/dist/extensions/seo/analyzers/crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAEV,WAAW,EACX,WAAW,EAGZ,MAAM,aAAa,CAAC;AAgHrB,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,EACjB,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,WAAW,CAAC,CA+LtB"}

package/dist/extensions/seo/analyzers/crawler.js

@@ -0,0 +1,327 @@
+import { fetchPage, fetchRobotsTxt } from "../utils/fetcher.js";
+import { parsePage } from "../utils/parser.js";
+import { normalizeUrl } from "../utils/url-safety.js";
+const DEFAULT_CRAWL_CONFIG = {
+    maxPages: 200,
+    maxDepth: 5,
+    concurrency: 3,
+    respectRobotsTxt: true,
+    crawlDelayMs: 500,
+    includeExternal: false,
+};
+/**
+ * Strip query parameters and hash for URL identity comparison.
+ * Two URLs that differ only by query params or hash should be considered
+ * the same page for crawling purposes — prevents re-crawling the same page
+ * hundreds of times when query params encode ephemeral UI state.
+ */
+function urlIdentity(url) {
+    try {
+        const u = new URL(url);
+        u.search = "";
+        u.hash = "";
+        if (u.pathname.length > 1 && u.pathname.endsWith("/")) {
+            u.pathname = u.pathname.replace(/\/+$/, "");
+        }
+        return u.href;
+    }
+    catch {
+        return url;
+    }
+}
+function extractInternalLinks(html, baseUrl) {
+    const links = [];
+    const hrefRegex = /<a\s[^>]*href\s*=\s*["']([^"']+)["'][^>]*>/gi;
+    let match;
+    while ((match = hrefRegex.exec(html)) !== null) {
+        try {
+            const resolved = new URL(match[1], baseUrl).href;
+            const base = new URL(baseUrl);
+            const resolvedUrl = new URL(resolved);
+            if (resolvedUrl.hostname === base.hostname &&
+                resolvedUrl.protocol.startsWith("http") &&
+                !resolved.includes("#") &&
+                !resolved.includes("javascript:") &&
+                !resolved.includes("mailto:") &&
+                !resolved.includes("/cdn-cgi/")) {
+                const normalized = normalizeUrl(resolved);
+                links.push(normalized);
+            }
+        }
+        catch {
+            // skip malformed hrefs
+        }
+    }
+    return [...new Set(links)];
+}
+function parseRobotsTxtForDisallowed(content, userAgent) {
+    const lines = content.split("\n").map((l) => l.trim());
+    const disallowed = [];
+    let crawlDelay = null;
+    let currentUserAgents = [];
+    let inRelevantBlock = false;
+    for (const line of lines) {
+        if (!line || line.startsWith("#"))
+            continue;
+        const colonIdx = line.indexOf(":");
+        if (colonIdx === -1)
+            continue;
+        const key = line.slice(0, colonIdx).trim().toLowerCase();
+        const value = line.slice(colonIdx + 1).trim();
+        if (key === "user-agent") {
+            currentUserAgents.push(value.toLowerCase());
+        }
+        else if (inRelevantBlock ||
+            currentUserAgents.some((ua) => ua === "*" || ua === userAgent.toLowerCase())) {
+            inRelevantBlock = true;
+            if (key === "disallow") {
+                if (value)
+                    disallowed.push(value);
+            }
+            else if (key === "crawl-delay") {
+                const delay = Number.parseFloat(value);
+                if (!Number.isNaN(delay))
+                    crawlDelay = delay;
+            }
+        }
+        if (key !== "user-agent") {
+            if (!inRelevantBlock)
+                currentUserAgents = [];
+        }
+    }
+    return { disallowed, crawlDelay };
+}
+function isDisallowed(url, disallowedPaths) {
+    try {
+        const pathname = new URL(url).pathname;
+        for (const pattern of disallowedPaths) {
+            if (pattern === "/")
+                return true;
+            if (pattern && pathname.startsWith(pattern))
+                return true;
+        }
+    }
+    catch {
+        return true; // can't parse → block
+    }
+    return false;
+}
+export async function crawlSite(startUrl, config, crawlConfig) {
+    const cfg = { ...DEFAULT_CRAWL_CONFIG, ...crawlConfig };
+    const normalizedStart = normalizeUrl(startUrl);
+    const baseHost = new URL(normalizedStart).hostname;
+    let disallowedPaths = [];
+    let robotsCrawlDelay = null;
+    if (cfg.respectRobotsTxt) {
+        try {
+            const robotsResult = await fetchRobotsTxt(normalizedStart, config);
+            if (robotsResult.statusCode === 200) {
+                const parsed = parseRobotsTxtForDisallowed(robotsResult.content, config.userAgent);
+                disallowedPaths = parsed.disallowed;
+                robotsCrawlDelay = parsed.crawlDelay;
+            }
+        }
+        catch {
+            // robots.txt not available — crawl freely
+        }
+    }
+    const delayMs = robotsCrawlDelay
+        ? robotsCrawlDelay * 1000
+        : cfg.crawlDelayMs;
+    const visitedIdentities = new Set();
+    const queuedIdentities = new Set();
+    const queue = [];
+    const startIdentity = urlIdentity(normalizedStart);
+    queuedIdentities.add(startIdentity);
+    queue.push({ url: normalizedStart, depth: 0 });
+    const pages = [];
+    const brokenLinks = [];
+    const errors = [];
+    const allDiscoveredLinks = new Set();
+    while (queue.length > 0 &&
+        visitedIdentities.size < cfg.maxPages) {
+        const batch = [];
+        while (batch.length < cfg.concurrency &&
+            queue.length > 0 &&
+            visitedIdentities.size + batch.length < cfg.maxPages) {
+            const next = queue.shift();
+            const identity = urlIdentity(next.url);
+            if (visitedIdentities.has(identity))
+                continue;
+            if (isDisallowed(next.url, disallowedPaths))
+                continue;
+            batch.push(next);
+        }
+        if (batch.length === 0)
+            break;
+        const batchResults = await Promise.allSettled(batch.map(async ({ url, depth }) => {
+            const pageStart = Date.now();
+            try {
+                const fetchResult = await fetchPage(url, config);
+                const pageData = parsePage(fetchResult.content, fetchResult.finalUrl);
+                const internalLinks = extractInternalLinks(fetchResult.content, fetchResult.finalUrl);
+                return {
+                    url: fetchResult.finalUrl,
+                    depth,
+                    statusCode: fetchResult.statusCode,
+                    fetchResult,
+                    pageData,
+                    linksFound: internalLinks,
+                    crawlDuration: Date.now() - pageStart,
+                };
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                errors.push(`Failed to crawl ${url}: ${msg}`);
+                return {
+                    url,
+                    depth,
+                    statusCode: 0,
+                    fetchResult: null,
+                    pageData: null,
+                    linksFound: [],
+                    crawlDuration: Date.now() - pageStart,
+                    error: msg,
+                };
+            }
+        }));
+        for (const result of batchResults) {
+            if (result.status === "rejected") {
+                errors.push(`Batch crawl error: ${String(result.reason)}`);
+                continue;
+            }
+            const page = result.value;
+            const identity = urlIdentity(page.url);
+            visitedIdentities.add(identity);
+            pages.push(page);
+            if (page.pageData) {
+                for (const link of page.pageData.links) {
+                    if (link.type === "external")
+                        continue;
+                    allDiscoveredLinks.add(link.href);
+                    const normalized = normalizeUrl(link.href);
+                    const linkIdentity = urlIdentity(normalized);
+                    if (!visitedIdentities.has(linkIdentity) &&
+                        !queuedIdentities.has(linkIdentity) &&
+                        linkIdentity !== identity) {
+                        try {
+                            const linkUrl = new URL(normalized);
+                            if (linkUrl.hostname === baseHost &&
+                                page.depth < cfg.maxDepth) {
+                                queuedIdentities.add(linkIdentity);
+                                queue.push({
+                                    url: normalized,
+                                    depth: page.depth + 1,
+                                });
+                            }
+                        }
+                        catch {
+                            // skip
+                        }
+                    }
+                }
+            }
+            if (delayMs > 0) {
+                await sleep(delayMs);
+            }
+        }
+    }
+    // Detect broken links (pages crawled with 4xx/5xx status)
+    for (const page of pages) {
+        if (page.statusCode >= 400) {
+            brokenLinks.push({
+                sourcePage: normalizedStart,
+                targetUrl: page.url,
+                statusCode: page.statusCode,
+                linkText: "",
+            });
+        }
+    }
+    const duplicateGroups = detectDuplicates(pages);
+    const linkedPages = new Set();
+    for (const page of pages) {
+        for (const link of page.linksFound) {
+            linkedPages.add(link);
+        }
+    }
+    const orphanPages = pages
+        .filter((p) => p.depth === 0 && !linkedPages.has(p.url))
+        .map((p) => p.url);
+    const linkGraph = pages.map((p) => ({
+        url: p.url,
+        internalInlinks: pages.reduce((count, other) => count + (other.linksFound.includes(p.url) ? 1 : 0), 0),
+        internalOutlinks: p.linksFound.length,
+        externalOutlinks: p.pageData ? p.pageData.externalLinkCount : 0,
+        depth: p.depth,
+        statusCode: p.statusCode,
+    }));
+    return {
+        baseUrl: normalizedStart,
+        pagesCrawled: pages.length,
+        totalPagesFound: allDiscoveredLinks.size,
+        maxDepthReached: pages.reduce((max, p) => Math.max(max, p.depth), 0),
+        brokenLinks,
+        duplicateGroups,
+        orphanPages,
+        pages,
+        linkGraph,
+        errors,
+    };
+}
+function detectDuplicates(pages) {
+    const groups = [];
+    // Use urlIdentity for deduplication so /studio/ and /studio/?pako=...
+    // are counted as the same page
+    const identitySet = new Set();
+    const uniquePages = pages.filter((p) => {
+        const id = urlIdentity(p.url);
+        if (identitySet.has(id))
+            return false;
+        identitySet.add(id);
+        return true;
+    });
+    const titleMap = new Map();
+    const descMap = new Map();
+    const h1Map = new Map();
+    for (const page of uniquePages) {
+        if (!page.pageData)
+            continue;
+        const title = page.pageData.title?.trim().toLowerCase();
+        if (title) {
+            if (!titleMap.has(title))
+                titleMap.set(title, []);
+            titleMap.get(title).push(page.url);
+        }
+        const desc = page.pageData.metaDescription?.trim().toLowerCase();
+        if (desc) {
+            if (!descMap.has(desc))
+                descMap.set(desc, []);
+            descMap.get(desc).push(page.url);
+        }
+        const h1 = page.pageData.headings
+            .filter((h) => h.level === 1)
+            .map((h) => h.text.trim().toLowerCase())
+            .join(" | ");
+        if (h1) {
+            if (!h1Map.has(h1))
+                h1Map.set(h1, []);
+            h1Map.get(h1).push(page.url);
+        }
+    }
+    for (const [value, urls] of titleMap) {
+        if (urls.length > 1)
+            groups.push({ type: "title", value, pageUrls: urls });
+    }
+    for (const [value, urls] of descMap) {
+        if (urls.length > 1)
+            groups.push({ type: "description", value, pageUrls: urls });
+    }
+    for (const [value, urls] of h1Map) {
+        if (urls.length > 1)
+            groups.push({ type: "h1", value, pageUrls: urls });
+    }
+    return groups;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/extensions/seo/analyzers/keywords.d.ts

@@ -0,0 +1,9 @@
+import type { KeywordAnalysisResult } from "../types.js";
+export declare function analyzeKeywords(text: string, url?: string): KeywordAnalysisResult;
+export declare function analyzeKeywordDensity(text: string, targetKeywords: string[]): Array<{
+    keyword: string;
+    count: number;
+    density: number;
+    verdict: string;
+}>;
+//# sourceMappingURL=keywords.d.ts.map

package/dist/extensions/seo/analyzers/keywords.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keywords.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/keywords.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAe,MAAM,aAAa,CAAC;AA4GtE,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,qBAAqB,CAgEjF;AAeD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,EACZ,cAAc,EAAE,MAAM,EAAE,GACvB,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAkB7E"}