@aexol/spectral 0.9.126 → 0.9.128

package/dist/agent/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/agent/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAIX,SAAS,EAET,MAAM,4BAA4B,CAAC;AAUpC,OAAO,EACN,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,KAAK,WAAW,EAAmC,MAAM,aAAa,CAAC;AAEhF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAOrD,OAAO,KAAK,EAAE,gBAAgB,IAAI,oBAAoB,EAAE,MAAM,8CAA8C,CAAC;AA4F7G,wBAAgB,+BAA+B,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpG;AAED,wBAAgB,yBAAyB,CACxC,gBAAgB,EAAE,MAAM,GAAG,SAAS,EACpC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,GAC/C,MAAM,CAKR;AAiHD;;;GAGG;AACH,wBAAgB,oCAAoC,CACnD,cAAc,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,GAChD,IAAI,CAAC,oBAAoB,CAAC,gBAAgB,CAAC,EAAE,WAAW,CAAC,CAK3D;AAED,wBAAgB,oBAAoB,CACnC,SAAS,EAAE,MAAM,EAAE,GAAG,SAAS,EAC/B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,YAAY,EACjB,GAAG,EAAE,gBAAgB,GACnB,SAAS,EAAE,CAqCb;AAuaD,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACxC,GAAG,EAAE,YAAY,EACjB,IAAI,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAA;CAAE,GACpF,IAAI,CAgZN"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/agent/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAIX,SAAS,EAET,MAAM,4BAA4B,CAAC;AAUpC,OAAO,EACN,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,KAAK,WAAW,EAAmC,MAAM,aAAa,CAAC;AAEhF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAOrD,OAAO,KAAK,EAAE,gBAAgB,IAAI,oBAAoB,EAAE,MAAM,8CAA8C,CAAC;AA4F7G,wBAAgB,+BAA+B,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpG;AAED,wBAAgB,yBAAyB,CACxC,gBAAgB,EAAE,MAAM,GAAG,SAAS,EACpC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,GAC/C,MAAM,CAKR;AAiHD;;;GAGG;AACH,wBAAgB,oCAAoC,CACnD,cAAc,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,GAChD,IAAI,CAAC,oBAAoB,CAAC,gBAAgB,CAAC,EAAE,WAAW,CAAC,CAK3D;AAED,wBAAgB,oBAAoB,CACnC,SAAS,EAAE,MAAM,EAAE,GAAG,SAAS,EAC/B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,YAAY,EACjB,GAAG,EAAE,gBAAgB,GACnB,SAAS,EAAE,CAqCb;AA4aD,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACxC,GAAG,EAAE,YAAY,EACjB,IAAI,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAA;CAAE,GACpF,IAAI,CAgZN"}

package/dist/agent/index.js

@@ -494,6 +494,11 @@ async function runSingleAgent(defaultCwd, agents, agentName, task, cwd, step, si
             if (event.type === "tool_execution_end" && event.isError) {
                 emitUpdate();
             }
+            if (event.type === "agent_end" && event.error) {
+                hadError = true;
+                currentResult.errorMessage = event.error;
+                currentResult.stderr = event.error;
+            }
         }
         // Drain the stream result (already consumed by iteration, but ensure cleanup)
         try {

package/dist/extensions/seo/analyzers/content.d.ts

@@ -5,12 +5,35 @@
  * analysis: filler phrase detection, AI-pattern detection, information density,
  * and repetition scoring. No external API calls.
  */
-import type { ContentQualityResult } from "../types.js";
+import type { ContentQualityResult, HeadingInfo, PageData, SchemaBlock } from "../types.js";
+/** Split text into sentences, respecting common abbreviations. */
+export declare function splitSentences(text: string): string[];
+/** Calculate Flesch-Kincaid Grade Level and related readability metrics. */
+export declare function calculateReadability(text: string): {
+    fleschKincaid: number;
+    sentenceCount: number;
+    avgSentenceLength: number;
+    avgWordLength: number;
+};
+/** Extract named entities from text using proper noun heuristics. */
+export declare function extractEntities(text: string, schemaBlocks?: SchemaBlock[]): string[];
+/** Check whether heading keywords appear in body text. */
+export declare function headingContentAlignment(headings: HeadingInfo[], bodyText: string, stopWords?: Set<string>): {
+    score: number;
+    missingKeywords: string[];
+};
+/** Evaluate heading structure depth and completeness. */
+export declare function calculateSemanticDepth(headings: HeadingInfo[]): {
+    score: number;
+    hasH1: boolean;
+    hasH2: boolean;
+    hasH3: boolean;
+    headingCount: number;
+};
 /**
- * Analyze content quality of a text string.
- *
- * All scores are 0-100 where higher is better (except filler and AI pattern
- * scores, where higher means MORE problematic content).
+ * Detect language using word-frequency profiles of common function words.
+ * Returns the ISO 639-1 language code of the best match, or null if confidence is too low.
  */
-export declare function analyzeContentQuality(text: string, url?: string): ContentQualityResult;
+export declare function detectLanguage(text: string): string | null;
+export declare function analyzeContentQuality(text: string, url?: string, pageData?: PageData, customStopWords?: string[]): ContentQualityResult;
 //# sourceMappingURL=content.d.ts.map

package/dist/extensions/seo/analyzers/content.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/content.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AA8HxD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,oBAAoB,CAgDtF"}
+{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/content.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,QAAQ,EAAqB,WAAW,EAAE,MAAM,aAAa,CAAC;AAqJ/G,kEAAkE;AAClE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAcrD;AAaD,4EAA4E;AAC5E,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACvB,CA2BA;AAED,qEAAqE;AACrE,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,WAAW,EAAE,GAAG,MAAM,EAAE,CA0BpF;AAED,0DAA0D;AAC1D,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,WAAW,EAAE,EACvB,QAAQ,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GACtB;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,EAAE,CAAA;CAAE,CAsB9C;AAED,yDAAyD;AACzD,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACtB,CAiBA;AAmBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAgC1D;AAED,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,EACZ,GAAG,CAAC,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,eAAe,CAAC,EAAE,MAAM,EAAE,GACzB,oBAAoB,CAoFtB"}

package/dist/extensions/seo/analyzers/content.js

@@ -1,3 +1,19 @@
+import { analyzeReadability } from "./readability.js";
+/** Common stop words — filtered during tokenization to improve signal. */
+const STOP_WORDS = new Set([
+    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+    "of", "with", "by", "from", "up", "about", "into", "through", "during",
+    "before", "after", "above", "below", "between", "out", "off", "over",
+    "under", "again", "further", "then", "once", "here", "there", "when",
+    "where", "why", "how", "all", "both", "each", "few", "more", "most",
+    "other", "some", "such", "no", "nor", "not", "only", "own", "same",
+    "so", "than", "too", "very", "s", "t", "can", "will", "just", "don",
+    "should", "now", "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "having", "do", "does", "did", "doing", "it",
+    "its", "this", "that", "these", "those", "i", "me", "my", "we", "our",
+    "you", "your", "he", "she", "they", "them", "their", "what", "which",
+    "who", "whom", "would", "could", "may", "might", "shall", "must",
+]);
 /** Common filler phrases that add no information value. */
 const FILLER_PHRASES = [
     "in today's fast-paced world",
@@ -72,11 +88,19 @@ const AI_PATTERNS = [
     "transform your",
     "redefine the way",
 ];
-function tokenize(text) {
+/** Common English abbreviations that should not trigger sentence splits. */
+const ABBREVIATIONS = new Set([
+    "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "ave", "blvd",
+    "etc", "vs", "approx", "dept", "est", "eg", "ie", "inc", "ltd",
+    "co", "corp", "jan", "feb", "mar", "apr", "jun", "jul", "aug",
+    "sep", "oct", "nov", "dec",
+]);
+function tokenize(text, stopWords) {
+    const sw = stopWords ?? STOP_WORDS;
     return text
         .toLowerCase()
         .split(/[^a-z0-9]+/)
-        .filter((t) => t.length > 0);
+        .filter((t) => t.length > 0 && !sw.has(t));
 }
 function countPatternMatches(text, patterns) {
     const lower = text.toLowerCase();
@@ -95,7 +119,6 @@ function countPatternMatches(text, patterns) {
 function calculateRepetition(tokens) {
     if (tokens.length === 0)
         return 0;
-    // Count trigram repetitions.
     const trigrams = new Map();
     for (let i = 0; i < tokens.length - 2; i++) {
         const trigram = `${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`;
@@ -112,21 +135,204 @@ function calculateRepetition(tokens) {
 function calculateInformationDensity(text, tokens) {
     if (tokens.length === 0)
         return 0;
-    // Count entities (capitalized words) and numbers.
     const entities = (text.match(/\b[A-Z][a-z]+\b/g) ?? []).length;
     const numbers = (text.match(/\b\d+([.,]\d+)?\b/g) ?? []).length;
     const informative = entities + numbers;
-    // Density = informative tokens / total tokens, scaled 0-100.
     return Math.min(100, Math.round((informative / tokens.length) * 1000));
 }
+/** Split text into sentences, respecting common abbreviations. */
+export function splitSentences(text) {
+    if (!text)
+        return [];
+    // Replace known abbreviations with placeholders to avoid false splits.
+    let processed = text;
+    for (const abbr of ABBREVIATIONS) {
+        const escaped = abbr.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+        const re = new RegExp(`\\b${escaped}\\.`, "gi");
+        processed = processed.replace(re, (match) => match.replace(".", "##DOT##"));
+    }
+    // Split on .!? followed by whitespace and uppercase letter (or end of string).
+    const sentences = processed.split(/(?<=[.!?])\s+(?=[A-Z])/);
+    return sentences
+        .map((s) => s.replace(/##DOT##/g, ".").trim())
+        .filter((s) => s.length > 0);
+}
+/** Count syllables in a word using a simple vowel-group heuristic. */
+function countSyllables(word) {
+    const cleaned = word.toLowerCase().replace(/[^a-z]/g, "");
+    if (cleaned.length <= 3)
+        return 1;
+    const vowelGroups = cleaned.match(/[aeiouy]+/g);
+    let count = vowelGroups ? vowelGroups.length : 1;
+    // Subtract silent 'e' at end.
+    if (cleaned.endsWith("e") && count > 1)
+        count--;
+    return Math.max(1, count);
+}
+/** Calculate Flesch-Kincaid Grade Level and related readability metrics. */
+export function calculateReadability(text) {
+    const sentences = splitSentences(text);
+    const sentenceCount = sentences.length;
+    if (sentenceCount === 0) {
+        return { fleschKincaid: 0, sentenceCount: 0, avgSentenceLength: 0, avgWordLength: 0 };
+    }
+    const words = text.split(/\s+/).filter(Boolean);
+    const wordCount = words.length;
+    if (wordCount === 0) {
+        return { fleschKincaid: 0, sentenceCount, avgSentenceLength: 0, avgWordLength: 0 };
+    }
+    const avgSentenceLength = wordCount / sentenceCount;
+    let totalSyllables = 0;
+    let totalLetters = 0;
+    for (const word of words) {
+        totalSyllables += countSyllables(word);
+        totalLetters += word.replace(/[^a-zA-Z]/g, "").length;
+    }
+    const avgWordLength = totalLetters / wordCount;
+    // Flesch-Kincaid Grade Level = 0.39*(words/sentences) + 11.8*(syllables/words) - 15.59
+    const fleschKincaid = 0.39 * avgSentenceLength + 11.8 * (totalSyllables / wordCount) - 15.59;
+    return {
+        fleschKincaid: Math.round(fleschKincaid * 10) / 10,
+        sentenceCount,
+        avgSentenceLength: Math.round(avgSentenceLength * 10) / 10,
+        avgWordLength: Math.round(avgWordLength * 10) / 10,
+    };
+}
+/** Extract named entities from text using proper noun heuristics. */
+export function extractEntities(text, schemaBlocks) {
+    const entities = new Set();
+    // Extract consecutive capitalized words (potential proper nouns).
+    const properNouns = text.match(/\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b/g) ?? [];
+    for (const noun of properNouns) {
+        // Only keep multi-word or clearly proper noun single words (not sentence starters).
+        if (noun.includes(" ") || noun.length > 3) {
+            entities.add(noun);
+        }
+    }
+    // Cross-reference with schema @type values.
+    if (schemaBlocks) {
+        for (const block of schemaBlocks) {
+            if (block.type && block.type !== "Unknown" && block.type !== "Invalid" && block.type !== "InvalidJSON") {
+                entities.add(block.type);
+            }
+            // Also check parsed schema for name properties.
+            if (block.parsed && typeof block.parsed === "object") {
+                const parsed = block.parsed;
+                if (typeof parsed.name === "string" && parsed.name.length > 0) {
+                    entities.add(parsed.name);
+                }
+            }
+        }
+    }
+    return [...entities];
+}
+/** Check whether heading keywords appear in body text. */
+export function headingContentAlignment(headings, bodyText, stopWords) {
+    const sw = stopWords ?? STOP_WORDS;
+    const lowerBody = bodyText.toLowerCase();
+    // Extract meaningful keywords from H1/H2 text (words 4+ chars, not stop words).
+    const headingKeywords = new Set();
+    for (const h of headings) {
+        if (h.level > 2)
+            continue; // Only H1/H2.
+        const words = h.text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4 && !sw.has(w));
+        for (const w of words)
+            headingKeywords.add(w);
+    }
+    if (headingKeywords.size === 0)
+        return { score: 100, missingKeywords: [] };
+    const missingKeywords = [];
+    let found = 0;
+    for (const kw of headingKeywords) {
+        if (lowerBody.includes(kw)) {
+            found++;
+        }
+        else {
+            missingKeywords.push(kw);
+        }
+    }
+    const score = Math.round((found / headingKeywords.size) * 100);
+    return { score, missingKeywords };
+}
+/** Evaluate heading structure depth and completeness. */
+export function calculateSemanticDepth(headings) {
+    let hasH1 = false;
+    let hasH2 = false;
+    let hasH3 = false;
+    let headingCount = 0;
+    for (const h of headings) {
+        headingCount++;
+        if (h.level === 1)
+            hasH1 = true;
+        if (h.level === 2)
+            hasH2 = true;
+        if (h.level === 3)
+            hasH3 = true;
+    }
+    let score = 0;
+    if (hasH1)
+        score += 40;
+    if (hasH2)
+        score += 35;
+    if (hasH3)
+        score += 15;
+    if (headingCount >= 5)
+        score += 10;
+    return { score, hasH1, hasH2, hasH3, headingCount };
+}
 /**
  * Analyze content quality of a text string.
  *
  * All scores are 0-100 where higher is better (except filler and AI pattern
  * scores, where higher means MORE problematic content).
  */
-export function analyzeContentQuality(text, url) {
-    const tokens = tokenize(text);
+const LANGUAGE_PROFILES = {
+    en: ["the", "be", "to", "of", "and", "a", "in", "that", "have", "it", "for", "not", "on", "with", "he", "as", "you", "do", "at", "this", "but", "his", "by", "from", "they", "we", "her", "she", "or", "an", "will", "my", "one", "all", "would", "there", "their"],
+    pl: ["i", "w", "się", "na", "nie", "to", "jest", "że", "z", "do", "o", "a", "co", "po", "tak", "jak", "ale", "mnie", "już", "tego", "czy", "jej", "był", "bardzo", "dla", "go", "przy", "przed", "nad", "pod"],
+    de: ["der", "die", "und", "in", "zu", "den", "das", "nicht", "von", "sie", "ist", "des", "sich", "mit", "dem", "dass", "er", "es", "ein", "ich", "auf", "so", "eine", "auch", "als", "an", "nach", "wie", "im", "für"],
+    fr: ["de", "la", "le", "et", "les", "des", "en", "un", "une", "du", "que", "pas", "pour", "dans", "qui", "par", "sur", "au", "avec", "ne", "ce", "plus", "se", "son", "aux", "dont", "comme", "tout", "cette", "même"],
+    es: ["de", "que", "el", "en", "y", "a", "los", "se", "del", "las", "un", "por", "con", "no", "una", "su", "para", "es", "al", "lo", "como", "más", "o", "pero", "sus", "le", "ha", "me", "si", "sin"],
+    it: ["di", "che", "e", "la", "il", "un", "a", "per", "in", "una", "con", "non", "si", "le", "del", "da", "più", "al", "ma", "gli", "ad", "anche", "o", "cui", "suo", "tra", "mio", "questa", "quello", "dalla"],
+    pt: ["de", "que", "e", "do", "a", "o", "não", "um", "da", "se", "com", "uma", "os", "no", "me", "por", "as", "dos", "mas", "ele", "na", "ao", "para", "bem", "seu", "isso", "pela", "lhe", "qual", "já"],
+    nl: ["de", "van", "een", "het", "dat", "in", "en", "is", "niet", "op", "te", "voor", "zijn", "met", "die", "aan", "er", "als", "ook", "om", "dan", "uit", "bij", "of", "kan", "nog", "naar", "al", "worden", "door"],
+};
+/**
+ * Detect language using word-frequency profiles of common function words.
+ * Returns the ISO 639-1 language code of the best match, or null if confidence is too low.
+ */
+export function detectLanguage(text) {
+    const words = text.toLowerCase().split(/[^a-zà-ÿ]+/).filter(w => w.length >= 2);
+    const scores = {};
+    for (const [lang, signalWords] of Object.entries(LANGUAGE_PROFILES)) {
+        const signalSet = new Set(signalWords);
+        let count = 0;
+        for (const word of words) {
+            if (signalSet.has(word))
+                count++;
+        }
+        scores[lang] = count;
+    }
+    let bestLang = null;
+    let bestScore = 0;
+    let secondBest = 0;
+    for (const [lang, score] of Object.entries(scores)) {
+        if (score > bestScore) {
+            secondBest = bestScore;
+            bestScore = score;
+            bestLang = lang;
+        }
+        else if (score > secondBest) {
+            secondBest = score;
+        }
+    }
+    if (bestScore < 3 || (secondBest > 0 && bestScore < secondBest * 2)) {
+        return null;
+    }
+    return bestLang;
+}
+export function analyzeContentQuality(text, url, pageData, customStopWords) {
+    const effectiveStopWords = new Set([...STOP_WORDS, ...((customStopWords ?? []).map(w => w.toLowerCase()))]);
+    const tokens = tokenize(text, effectiveStopWords);
     const wordCount = text.split(/\s+/).filter(Boolean).length;
     const uniqueTokens = new Set(tokens).size;
     // Filler phrase detection.
@@ -157,7 +363,7 @@ export function analyzeContentQuality(text, url) {
         flags.push("low-information-density");
     if (wordCount < 300)
         flags.push("thin-content");
-    return {
+    const result = {
         url,
         wordCount,
         tokens: tokens.length,
@@ -170,5 +376,34 @@ export function analyzeContentQuality(text, url) {
         flags,
         fillerMatches: filler.matches,
         aiPatternMatches: aiPatterns.matches,
+        disclaimer: "Note: This analysis measures content quality heuristics only. It does not measure search rankings, traffic, or search volume data.",
     };
+    // Enriched analysis when pageData is available.
+    if (pageData) {
+        // Readability.
+        const readabilityResult = analyzeReadability(text, url);
+        result.readability = readabilityResult;
+        result.sentenceCount = readabilityResult.sentenceCount;
+        result.avgSentenceLength = readabilityResult.sentenceCount > 0
+            ? Math.round((result.wordCount / readabilityResult.sentenceCount) * 10) / 10
+            : 0;
+        const totalLetters = text.replace(/[^a-zA-Z]/g, "").length;
+        result.avgWordLength = result.wordCount > 0
+            ? Math.round((totalLetters / result.wordCount) * 10) / 10
+            : 0;
+        result.readabilityFleschKincaid = readabilityResult.fleschKincaid;
+        // Heading-content alignment.
+        if (pageData.headings && pageData.headings.length > 0) {
+            result.headingAlignment = headingContentAlignment(pageData.headings, text, effectiveStopWords);
+        }
+        // Semantic depth.
+        if (pageData.headings) {
+            result.semanticDepth = calculateSemanticDepth(pageData.headings);
+        }
+        // Entity extraction.
+        result.entities = extractEntities(text, pageData.schemaBlocks);
+        // Language.
+        result.language = pageData.language ?? detectLanguage(text) ?? undefined;
+    }
+    return result;
 }

package/dist/extensions/seo/analyzers/crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAEV,WAAW,EACX,WAAW,EAGZ,MAAM,aAAa,CAAC;AAgHrB,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,EACjB,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,WAAW,CAAC,CA+LtB"}
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAEV,WAAW,EACX,WAAW,EAGZ,MAAM,aAAa,CAAC;AAsKrB,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,EACjB,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,WAAW,CAAC,CAkOtB"}

package/dist/extensions/seo/analyzers/crawler.js

@@ -30,12 +30,46 @@ function urlIdentity(url) {
     }
 }
 function extractInternalLinks(html, baseUrl) {
-    const links = [];
+    const internalLinks = [];
+    const canonicalUrls = [];
+    const paginationUrls = [];
+    let nofollowCount = 0;
+    let sponsoredCount = 0;
+    let ugcCount = 0;
+    // Extract <link rel="canonical"> and <link rel="next">/<link rel="prev">
+    const linkTagRegex = /<link\s[^>]*\/?>/gi;
+    let linkMatch;
+    while ((linkMatch = linkTagRegex.exec(html)) !== null) {
+        const tag = linkMatch[0];
+        const relMatch = /rel=["']([^"']+)["']/i.exec(tag);
+        const hrefMatch = /href=["']([^"']+)["']/i.exec(tag);
+        if (!relMatch || !hrefMatch)
+            continue;
+        const rel = relMatch[1].toLowerCase();
+        const href = hrefMatch[1];
+        try {
+            const resolved = new URL(href, baseUrl).href;
+            const base = new URL(baseUrl);
+            const resolvedUrl = new URL(resolved);
+            if (resolvedUrl.hostname === base.hostname && resolvedUrl.protocol.startsWith("http")) {
+                const normalized = normalizeUrl(resolved);
+                if (rel === "canonical")
+                    canonicalUrls.push(normalized);
+                if (rel === "next" || rel === "prev")
+                    paginationUrls.push(normalized);
+            }
+        }
+        catch {
+        }
+    }
+    // Extract <a href="..."> links with rel attributes.
     const hrefRegex = /<a\s[^>]*href\s*=\s*["']([^"']+)["'][^>]*>/gi;
     let match;
     while ((match = hrefRegex.exec(html)) !== null) {
+        const tag = match[0];
+        const href = match[1];
         try {
-            const resolved = new URL(match[1], baseUrl).href;
+            const resolved = new URL(href, baseUrl).href;
             const base = new URL(baseUrl);
             const resolvedUrl = new URL(resolved);
             if (resolvedUrl.hostname === base.hostname &&
@@ -45,14 +79,30 @@ function extractInternalLinks(html, baseUrl) {
                 !resolved.includes("mailto:") &&
                 !resolved.includes("/cdn-cgi/")) {
                 const normalized = normalizeUrl(resolved);
-                links.push(normalized);
+                const relMatch = /rel=["']([^"']+)["']/i.exec(tag);
+                const relVals = relMatch ? relMatch[1].toLowerCase().split(/\s+/) : [];
+                if (relVals.includes("nofollow"))
+                    nofollowCount++;
+                if (relVals.includes("sponsored"))
+                    sponsoredCount++;
+                if (relVals.includes("ugc"))
+                    ugcCount++;
+                if (!relVals.includes("nofollow")) {
+                    internalLinks.push(normalized);
+                }
             }
         }
         catch {
-            // skip malformed hrefs
         }
     }
-    return [...new Set(links)];
+    return {
+        internalLinks: [...new Set(internalLinks)],
+        canonicalUrls: [...new Set(canonicalUrls)],
+        paginationUrls: [...new Set(paginationUrls)],
+        nofollowCount,
+        sponsoredCount,
+        ugcCount,
+    };
 }
 function parseRobotsTxtForDisallowed(content, userAgent) {
     const lines = content.split("\n").map((l) => l.trim());
@@ -138,6 +188,12 @@ export async function crawlSite(startUrl, config, crawlConfig) {
     const brokenLinks = [];
     const errors = [];
     const allDiscoveredLinks = new Set();
+    const canonicalUrlSet = new Set();
+    const redirectingInternalLinks = [];
+    let nofollowLinks = 0;
+    let sponsoredLinks = 0;
+    let ugcLinks = 0;
+    let paginationLinks = 0;
     while (queue.length > 0 &&
         visitedIdentities.size < cfg.maxPages) {
         const batch = [];
@@ -159,15 +215,27 @@ export async function crawlSite(startUrl, config, crawlConfig) {
             try {
                 const fetchResult = await fetchPage(url, config);
                 const pageData = parsePage(fetchResult.content, fetchResult.finalUrl);
-                const internalLinks = extractInternalLinks(fetchResult.content, fetchResult.finalUrl);
+                const extracted = extractInternalLinks(fetchResult.content, fetchResult.finalUrl);
+                // Detect 3xx redirects to internal targets.
+                if (fetchResult.statusCode >= 300 && fetchResult.statusCode < 400 && fetchResult.redirectChain.length > 0) {
+                    const lastHop = fetchResult.redirectChain[fetchResult.redirectChain.length - 1];
+                    try {
+                        if (new URL(lastHop.url).hostname === baseHost) {
+                            redirectingInternalLinks.push({ source: fetchResult.url, target: fetchResult.finalUrl });
+                        }
+                    }
+                    catch {
+                    }
+                }
                 return {
                     url: fetchResult.finalUrl,
                     depth,
                     statusCode: fetchResult.statusCode,
                     fetchResult,
                     pageData,
-                    linksFound: internalLinks,
+                    linksFound: [...extracted.internalLinks, ...extracted.paginationUrls, ...extracted.canonicalUrls],
                     crawlDuration: Date.now() - pageStart,
+                    _extracted: extracted,
                 };
             }
             catch (err) {
@@ -194,6 +262,16 @@ export async function crawlSite(startUrl, config, crawlConfig) {
             const identity = urlIdentity(page.url);
             visitedIdentities.add(identity);
             pages.push(page);
+            // Aggregate link counters from the extracted metadata.
+            const ext = page._extracted;
+            if (ext) {
+                nofollowLinks += ext.nofollowCount;
+                sponsoredLinks += ext.sponsoredCount;
+                ugcLinks += ext.ugcCount;
+                paginationLinks += ext.paginationUrls.length;
+                for (const c of ext.canonicalUrls)
+                    canonicalUrlSet.add(c);
+            }
             if (page.pageData) {
                 for (const link of page.pageData.links) {
                     if (link.type === "external")
@@ -255,6 +333,7 @@ export async function crawlSite(startUrl, config, crawlConfig) {
         depth: p.depth,
         statusCode: p.statusCode,
     }));
+    const keywordCannibalization = detectKeywordCannibalization(pages);
     return {
         baseUrl: normalizedStart,
         pagesCrawled: pages.length,
@@ -266,6 +345,13 @@ export async function crawlSite(startUrl, config, crawlConfig) {
         pages,
         linkGraph,
         errors,
+        canonicalUrls: [...canonicalUrlSet],
+        nofollowLinks,
+        sponsoredLinks,
+        ugcLinks,
+        paginationLinks,
+        redirectingInternalLinks,
+        keywordCannibalization,
     };
 }
 function detectDuplicates(pages) {
@@ -322,6 +408,46 @@ function detectDuplicates(pages) {
     }
     return groups;
 }
+function detectKeywordCannibalization(pages) {
+    const NGRAM_MIN = 3;
+    const wordRe = /\b[a-z]{3,}\b/g;
+    const phraseMap = new Map();
+    for (const page of pages) {
+        if (!page.pageData)
+            continue;
+        const title = page.pageData.title?.toLowerCase() ?? "";
+        const h1Texts = page.pageData.headings
+            .filter((h) => h.level === 1)
+            .map((h) => h.text.toLowerCase());
+        const combined = `${title} ${h1Texts.join(" ")}`;
+        const words = combined.match(wordRe);
+        if (!words || words.length < NGRAM_MIN)
+            continue;
+        for (let i = 0; i <= words.length - NGRAM_MIN; i++) {
+            const ngram = words.slice(i, i + NGRAM_MIN).join(" ");
+            if (!phraseMap.has(ngram))
+                phraseMap.set(ngram, []);
+            phraseMap.get(ngram).push(page.url);
+        }
+    }
+    const results = [];
+    const dedupe = new Map();
+    for (const [ngram, urls] of phraseMap) {
+        const uniquePages = [...new Set(urls)];
+        if (uniquePages.length >= 2) {
+            uniquePages.sort();
+            const sig = ngram + "|" + uniquePages.join("|");
+            if (!dedupe.has(sig)) {
+                dedupe.set(sig, uniquePages);
+            }
+        }
+    }
+    for (const [sig, urls] of dedupe) {
+        const keyword = sig.split("|")[0];
+        results.push({ keyword, pages: urls });
+    }
+    return results.sort((a, b) => b.pages.length - a.pages.length).slice(0, 20);
+}
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }

package/dist/extensions/seo/analyzers/drift.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"drift.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/drift.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAqC9C,kEAAkE;AAClE,wBAAgB,YAAY,IAAI,IAAI,CASnC;AAMD,6CAA6C;AAC7C,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,SAAS,GAChB,OAAO,CAAC,aAAa,CAAC,CAwDxB;AAED,6DAA6D;AAC7D,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAWrG;AAuBD,qDAAqD;AACrD,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,SAAS,EACjB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAuE7B;AAED,sCAAsC;AACtC,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,SAAK,GAAG,aAAa,EAAE,CAK9F"}
+{"version":3,"file":"drift.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/analyzers/drift.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AA4D9C,kEAAkE;AAClE,wBAAgB,YAAY,IAAI,IAAI,CASnC;AAMD,6CAA6C;AAC7C,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,SAAS,GAChB,OAAO,CAAC,aAAa,CAAC,CAmExB;AAED,6DAA6D;AAC7D,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAWrG;AAyBD,qDAAqD;AACrD,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,SAAS,EACjB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAuG7B;AAED,sCAAsC;AACtC,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,SAAK,GAAG,aAAa,EAAE,CAK9F"}