@lynxflow/seo-engine 1.8.30 → 2.0.0

package/src/site-crawler.ts

@@ -1,406 +0,0 @@
-/**
- * 🕷️ Deep Site Crawler & Technical SEO Auditor (Inspired by CrawlSEO & Seonaut)
- *
- * High-performance, zero-dependency recursive site crawler & technical health inspector.
- * - Crawls live websites or crawls pre-rendered HTML snapshots
- * - Analyzes status codes, redirects, canonicals, titles, H1s, meta descriptions, image ALTs
- * - Detects 30+ SEO issues classified by severity (critical, warning, info)
- * - Computes a comprehensive 0-100 SEO Health Score with category breakdowns
- */
-
-export interface CrawlIssue {
-  url: string;
-  type: string;
-  severity: "critical" | "warning" | "info";
-  message: string;
-  recommendation: string;
-  details?: Record<string, unknown>;
-}
-
-export interface CrawledPageData {
-  url: string;
-  statusCode: number;
-  responseTimeMs: number;
-  title: string | null;
-  description: string | null;
-  h1: string | null;
-  h1Count: number;
-  canonical: string | null;
-  isCanonicalMatch: boolean;
-  wordCount: number;
-  imagesWithoutAlt: number;
-  internalLinksCount: number;
-  externalLinksCount: number;
-  issues: CrawlIssue[];
-}
-
-export interface SiteAuditSummary {
-  domain: string;
-  crawledPagesCount: number;
-  healthScore: number; // 0 to 100
-  categories: {
-    metaAndTags: number; // 0-100
-    contentQuality: number; // 0-100
-    indexingAndLinks: number; // 0-100
-    performanceAndStatus: number; // 0-100
-  };
-  totalIssues: {
-    critical: number;
-    warning: number;
-    info: number;
-  };
-  brokenLinks404: string[];
-  redirectChains: { source: string; target: string; statusCode: number }[];
-  pages: CrawledPageData[];
-  executionTimeMs: number;
-}
-
-export interface CrawlOptions {
-  maxPages?: number;
-  maxDepth?: number;
-  timeoutMs?: number;
-  concurrency?: number;
-  userAgent?: string;
-  includeExternalLinksCheck?: boolean;
-}
-
-export class DeepCrawlerAuditor {
-  private static DEFAULT_USER_AGENT = "LynxFlowSeoBot/1.2 (+https://lynxintel.io/bot; technical audit)";
-
-  /**
-   * Performs an instant in-memory technical audit on raw HTML and metadata.
-   */
-  static inspectHtmlSnapshot(url: string, html: string, statusCode = 200, responseTimeMs = 45): CrawledPageData {
-    const issues: CrawlIssue[] = [];
-
-    // 1. Status code checks
-    if (statusCode >= 400 && statusCode < 500) {
-      issues.push({
-        url,
-        type: "broken_page_404",
-        severity: "critical",
-        message: `HTTP Client Error: Page returned ${statusCode}`,
-        recommendation: "Fix broken link or configure a 301 permanent redirect to a relevant page.",
-      });
-    } else if (statusCode >= 500) {
-      issues.push({
-        url,
-        type: "server_error_500",
-        severity: "critical",
-        message: `HTTP Server Error: Page returned ${statusCode}`,
-        recommendation: "Inspect server logs and resolve backend application crash.",
-      });
-    }
-
-    // 2. Title extraction & checks
-    const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i);
-    const title = titleMatch ? titleMatch[1].trim() : null;
-
-    if (!title) {
-      issues.push({
-        url,
-        type: "missing_title",
-        severity: "critical",
-        message: "Missing <title> tag.",
-        recommendation: "Add an explicit, compelling <title> between 30 and 60 characters.",
-      });
-    } else if (title.length < 20) {
-      issues.push({
-        url,
-        type: "short_title",
-        severity: "warning",
-        message: `Title is too short (${title.length} chars): "${title}"`,
-        recommendation: "Expand title to at least 30 characters including primary keyword and brand name.",
-      });
-    } else if (title.length > 70) {
-      issues.push({
-        url,
-        type: "long_title",
-        severity: "warning",
-        message: `Title is too long (${title.length} chars), risk of SERP truncation.`,
-        recommendation: "Keep title under 60-65 characters for optimal desktop & mobile display.",
-      });
-    }
-
-    // 3. Meta description checks
-    const descMatch = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["'][^>]*>/i) ||
-                      html.match(/<meta[^>]*content=["']([^"']*)["'][^>]*name=["']description["'][^>]*>/i);
-    const description = descMatch ? descMatch[1].trim() : null;
-
-    if (!description) {
-      issues.push({
-        url,
-        type: "missing_meta_description",
-        severity: "critical",
-        message: "Missing meta description.",
-        recommendation: "Add a compelling meta description between 120 and 160 characters with clear call-to-action.",
-      });
-    } else if (description.length < 70) {
-      issues.push({
-        url,
-        type: "short_meta_description",
-        severity: "warning",
-        message: `Meta description is too short (${description.length} chars).`,
-        recommendation: "Expand meta description to at least 120 characters.",
-      });
-    } else if (description.length > 180) {
-      issues.push({
-        url,
-        type: "long_meta_description",
-        severity: "info",
-        message: `Meta description exceeds 180 chars (${description.length} chars).`,
-        recommendation: "Shorten meta description to 155-160 characters.",
-      });
-    }
-
-    // 4. H1 checks
-    const h1Matches = Array.from(html.matchAll(/<h1[^>]*>([^<]*)<\/h1>/gi)).map((m) => m[1].trim());
-    const h1Count = h1Matches.length;
-    const h1 = h1Count > 0 ? h1Matches[0] : null;
-
-    if (h1Count === 0) {
-      issues.push({
-        url,
-        type: "missing_h1",
-        severity: "critical",
-        message: "Missing <h1> headline.",
-        recommendation: "Add exactly one descriptive <h1> headline containing your target keyword.",
-      });
-    } else if (h1Count > 1) {
-      issues.push({
-        url,
-        type: "multiple_h1",
-        severity: "warning",
-        message: `Found ${h1Count} <h1> tags on the page.`,
-        recommendation: "Use only one single <h1> per page and structure other sections with <h2>/<h3>.",
-      });
-    }
-
-    // 5. Canonical checks
-    const canonicalMatch = html.match(/<link[^>]*rel=["']canonical["'][^>]*href=["']([^"']*)["'][^>]*>/i);
-    const canonical = canonicalMatch ? canonicalMatch[1].trim() : null;
-    const isCanonicalMatch = canonical ? canonical.replace(/\/$/, "") === url.replace(/\/$/, "") : false;
-
-    if (!canonical) {
-      issues.push({
-        url,
-        type: "missing_canonical",
-        severity: "warning",
-        message: "Missing self-referencing canonical tag.",
-        recommendation: "Add a <link rel='canonical' href='...' /> tag to prevent duplicate content indexation.",
-      });
-    }
-
-    // 6. Image ALT checks
-    const imgMatches = Array.from(html.matchAll(/<img([^>]*)>/gi));
-    let imagesWithoutAlt = 0;
-    for (const match of imgMatches) {
-      const imgTag = match[1];
-      if (!/alt=["'][^"']+["']/i.test(imgTag)) {
-        imagesWithoutAlt++;
-      }
-    }
-
-    if (imagesWithoutAlt > 0) {
-      issues.push({
-        url,
-        type: "images_missing_alt",
-        severity: "warning",
-        message: `${imagesWithoutAlt} image(s) missing descriptive 'alt' attribute.`,
-        recommendation: "Add descriptive ALT text for SEO image search and accessibility compliance.",
-      });
-    }
-
-    // 7. Word count & Content Depth
-    const cleanText = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
-                          .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
-                          .replace(/<[^>]+>/g, " ")
-                          .replace(/\s+/g, " ")
-                          .trim();
-    const wordCount = cleanText.split(" ").filter((w) => w.length > 1).length;
-
-    if (wordCount < 150 && statusCode === 200) {
-      issues.push({
-        url,
-        type: "thin_content",
-        severity: "warning",
-        message: `Thin content detected (${wordCount} words).`,
-        recommendation: "Expand content to at least 300-500 words to provide authoritative value.",
-      });
-    }
-
-    // 8. Link counters
-    const linkMatches = Array.from(html.matchAll(/<a[^>]*href=["']([^"']*)["'][^>]*>/gi));
-    let internalLinksCount = 0;
-    let externalLinksCount = 0;
-
-    for (const match of linkMatches) {
-      const href = match[1];
-      if (href.startsWith("http://") || href.startsWith("https://")) {
-        try {
-          const targetHost = new URL(href).hostname;
-          const currentHost = new URL(url).hostname;
-          if (targetHost === currentHost) internalLinksCount++;
-          else externalLinksCount++;
-        } catch {
-          externalLinksCount++;
-        }
-      } else if (href.startsWith("/") || href.startsWith("#") || href.startsWith(".")) {
-        internalLinksCount++;
-      }
-    }
-
-    return {
-      url,
-      statusCode,
-      responseTimeMs,
-      title,
-      description,
-      h1,
-      h1Count,
-      canonical,
-      isCanonicalMatch,
-      wordCount,
-      imagesWithoutAlt,
-      internalLinksCount,
-      externalLinksCount,
-      issues,
-    };
-  }
-
-  /**
-   * Crawls a full website domain recursively and produces an institutional SEO audit summary.
-   */
-  static async crawlAndAuditDomain(targetUrl: string, options: CrawlOptions = {}): Promise<SiteAuditSummary> {
-    const t0 = performance.now();
-    const maxPages = options.maxPages || 30;
-    const timeoutMs = options.timeoutMs || 8000;
-    const userAgent = options.userAgent || this.DEFAULT_USER_AGENT;
-
-    const baseDomain = targetUrl.replace(/\/$/, "");
-    const baseHost = new URL(baseDomain).hostname;
-
-    const visited = new Set<string>();
-    const queue: string[] = [baseDomain];
-    const crawledPages: CrawledPageData[] = [];
-    const brokenLinks404: string[] = [];
-    const redirectChains: { source: string; target: string; statusCode: number }[] = [];
-
-    while (queue.length > 0 && crawledPages.length < maxPages) {
-      const currentUrl = queue.shift()!;
-      const normalized = currentUrl.replace(/\/$/, "");
-
-      if (visited.has(normalized)) continue;
-      visited.add(normalized);
-
-      try {
-        const fetchStart = performance.now();
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-
-        const res = await fetch(currentUrl, {
-          signal: controller.signal,
-          headers: { "User-Agent": userAgent },
-        });
-        clearTimeout(timeoutId);
-
-        const fetchTimeMs = Math.round(performance.now() - fetchStart);
-        const statusCode = res.status;
-
-        if (statusCode === 404) {
-          brokenLinks404.push(currentUrl);
-        }
-
-        if (res.redirected && res.url !== currentUrl) {
-          redirectChains.push({ source: currentUrl, target: res.url, statusCode });
-        }
-
-        const html = await res.text();
-        const pageAudit = this.inspectHtmlSnapshot(currentUrl, html, statusCode, fetchTimeMs);
-        crawledPages.push(pageAudit);
-
-        // Discover new internal links
-        const hrefMatches = Array.from(html.matchAll(/<a[^>]*href=["']([^"'#]+)["']/gi));
-        for (const match of hrefMatches) {
-          const rawHref = match[1].trim();
-          try {
-            const resolved = new URL(rawHref, currentUrl).href.replace(/\/$/, "");
-            const parsed = new URL(resolved);
-
-            if (parsed.hostname === baseHost && !visited.has(resolved) && !queue.includes(resolved)) {
-              // Ignore media and asset extensions
-              if (!/\.(png|jpg|jpeg|gif|svg|webp|css|js|pdf|zip)$/i.test(parsed.pathname)) {
-                queue.push(resolved);
-              }
-            }
-          } catch {
-            // ignore invalid URL
-          }
-        }
-      } catch (err: any) {
-        crawledPages.push({
-          url: currentUrl,
-          statusCode: 0,
-          responseTimeMs: 0,
-          title: null,
-          description: null,
-          h1: null,
-          h1Count: 0,
-          canonical: null,
-          isCanonicalMatch: false,
-          wordCount: 0,
-          imagesWithoutAlt: 0,
-          internalLinksCount: 0,
-          externalLinksCount: 0,
-          issues: [{
-            url: currentUrl,
-            type: "fetch_timeout_error",
-            severity: "critical",
-            message: `Connection Error: ${err?.message || "Failed to reach server"}`,
-            recommendation: "Ensure server is reachable and responds in under 5 seconds.",
-          }],
-        });
-      }
-    }
-
-    // Compute Health Score & Category Ratings
-    let criticalCount = 0;
-    let warningCount = 0;
-    let infoCount = 0;
-
-    for (const p of crawledPages) {
-      for (const iss of p.issues) {
-        if (iss.severity === "critical") criticalCount++;
-        else if (iss.severity === "warning") warningCount++;
-        else infoCount++;
-      }
-    }
-
-    const totalPages = Math.max(1, crawledPages.length);
-    const penalty = (criticalCount * 12 + warningCount * 4 + infoCount * 1) / totalPages;
-    const healthScore = Math.max(10, Math.min(100, Math.round(100 - penalty)));
-
-    const categories = {
-      metaAndTags: Math.max(20, Math.min(100, Math.round(100 - (criticalCount * 8 + warningCount * 3) / totalPages))),
-      contentQuality: Math.max(30, Math.min(100, Math.round(100 - (warningCount * 5) / totalPages))),
-      indexingAndLinks: Math.max(25, Math.min(100, Math.round(100 - (brokenLinks404.length * 15) / totalPages))),
-      performanceAndStatus: Math.max(40, Math.min(100, Math.round(100 - (criticalCount * 10) / totalPages))),
-    };
-
-    return {
-      domain: baseDomain,
-      crawledPagesCount: crawledPages.length,
-      healthScore,
-      categories,
-      totalIssues: {
-        critical: criticalCount,
-        warning: warningCount,
-        info: infoCount,
-      },
-      brokenLinks404,
-      redirectChains,
-      pages: crawledPages,
-      executionTimeMs: Math.round(performance.now() - t0),
-    };
-  }
-}

package/src/slug-engine.ts

@@ -1,183 +0,0 @@
-/**
- * Global Multilingual Programmatic SEO Slug Engine
- * Enforces the 6 golden rules of SEO-optimized URL paths across 10+ languages:
- * English, French, German, Spanish, Italian, Portuguese, Dutch, Russian, Swedish, Polish.
- *
- * 1. Short & focused (3-5 target keywords)
- * 2. Hyphen (-) as the sole delimiter
- * 3. 100% lowercase, diacritics/accents stripped
- * 4. Stop words removed for 10+ international languages
- * 5. Zero dates/numbers for evergreen content
- * 6. Primary keyword positioned first
- */
-
-export const MULTILINGUAL_STOP_WORDS: Record<string, Set<string>> = {
-  // 🇬🇧 English
-  en: new Set([
-    "the", "a", "an", "and", "or", "for", "with", "without", "in", "on", "at",
-    "by", "to", "of", "from", "as", "is", "are", "be", "this", "that", "these",
-    "those", "how", "what", "which", "who", "whom", "when", "where", "why",
-    "your", "my", "our", "their", "its", "into", "over", "after", "before",
-  ]),
-
-  // 🇫🇷 French
-  fr: new Set([
-    "le", "la", "les", "un", "une", "des", "du", "de", "d", "l", "pour", "avec",
-    "sans", "sur", "sous", "dans", "par", "et", "ou", "a", "au", "aux", "en",
-    "ce", "cet", "cette", "ces", "son", "sa", "ses", "leur", "leurs", "notre",
-    "votre", "nos", "vos", "qui", "que", "quoi", "dont", "comment", "pourquoi",
-  ]),
-
-  // 🇩🇪 German
-  de: new Set([
-    "der", "die", "das", "ein", "eine", "eines", "einer", "einem", "einen",
-    "und", "oder", "fur", "für", "mit", "ohne", "auf", "unter", "in", "im", "von",
-    "vom", "zu", "zum", "zur", "bei", "beim", "nach", "aus", "uber", "über", "vor",
-    "wie", "was", "wer", "warum", "ihr", "ihre", "sein", "seine", "mein",
-  ]),
-
-  // 🇪🇸 Spanish
-  es: new Set([
-    "el", "la", "los", "las", "un", "una", "unos", "unas", "de", "del", "para",
-    "con", "sin", "en", "por", "y", "o", "a", "al", "este", "esta", "estos",
-    "estas", "su", "sus", "mi", "mis", "tu", "tus", "como", "que", "cual",
-  ]),
-
-  // 🇮🇹 Italian
-  it: new Set([
-    "il", "lo", "la", "i", "gli", "le", "un", "uno", "una", "un'", "di", "del",
-    "dello", "della", "dei", "degli", "delle", "a", "al", "allo", "alla", "ai",
-    "agli", "alle", "da", "dal", "in", "con", "su", "per", "tra", "fra", "e", "o",
-    "come", "cosa", "perche", "questo", "questa",
-  ]),
-
-  // 🇵🇹 Portuguese
-  pt: new Set([
-    "o", "a", "os", "as", "um", "uma", "uns", "umas", "de", "do", "da", "dos",
-    "das", "em", "no", "na", "nos", "nas", "para", "por", "com", "sem", "e",
-    "ou", "como", "que", "qual", "seu", "sua", "seus", "suas", "este", "esta",
-  ]),
-
-  // 🇳🇱 Dutch
-  nl: new Set([
-    "de", "het", "een", "en", "of", "voor", "met", "zonder", "in", "op", "bij",
-    "van", "naar", "door", "over", "onder", "als", "hoe", "wat", "wie", "waarom",
-    "zijn", "haar", "hun", "onze", "uw",
-  ]),
-
-  // 🇷🇺 Russian (Transliterated / Latinized)
-  ru: new Set([
-    "i", "v", "na", "s", "po", "dlya", "ot", "iz", "k", "o", "za", "kak",
-    "chto", "gde", "kogda", "pochemu", "eto", "etot", "eta", "eti", "ili",
-  ]),
-
-  // 🇸🇪 Swedish
-  sv: new Set([
-    "en", "ett", "den", "det", "de", "och", "eller", "for", "med", "utan",
-    "pa", "under", "i", "av", "till", "fran", "om", "hur", "vad", "varfor",
-  ]),
-
-  // 🇵🇱 Polish
-  pl: new Set([
-    "i", "w", "we", "z", "ze", "na", "do", "dla", "od", "o", "po", "za", "jak",
-    "co", "gdzie", "dlaczego", "to", "ten", "ta", "te", "lub", "albo",
-  ]),
-};
-
-// Global merged stop words set
-const ALL_STOP_WORDS = new Set<string>();
-for (const langSet of Object.values(MULTILINGUAL_STOP_WORDS)) {
-  for (const word of langSet) {
-    ALL_STOP_WORDS.add(word);
-  }
-}
-
-export interface SlugOptions {
-  language?: string; // e.g. "en", "fr", "de", "es", "all"
-  removeStopWords?: boolean;
-  stripDates?: boolean;
-  maxWords?: number;
-  maxLength?: number;
-}
-
-/**
- * Normalizes any text into a clean, canonical SEO slug with multilingual stop word filtering.
- */
-export function cleanSeoSlug(input: string, options: SlugOptions = {}): string {
-  const {
-    language = "en",
-    removeStopWords = true,
-    stripDates = true,
-    maxWords = 5,
-    maxLength = 75,
-  } = options;
-
-  if (!input || typeof input !== "string") {
-    return "page";
-  }
-
-  let text = input
-    .normalize("NFKD")
-    .replace(/[\u0300-\u036f]/g, "")
-    .toLowerCase()
-    .trim();
-
-  if (stripDates) {
-    // Strip 4-digit years (2010-2039) and transient top lists (e.g. "top 10")
-    text = text.replace(/\b(20[1-3][0-9])\b/g, "");
-    text = text.replace(/\btop\s*\d+\s*/g, "");
-  }
-
-  let tokens = text
-    .replace(/[^a-z0-9]+/g, " ")
-    .trim()
-    .split(/\s+/)
-    .filter(Boolean);
-
-  if (removeStopWords) {
-    const activeSet = language === "all" || !MULTILINGUAL_STOP_WORDS[language]
-      ? ALL_STOP_WORDS
-      : MULTILINGUAL_STOP_WORDS[language];
-
-    tokens = tokens.filter((token) => !activeSet.has(token) && token.length > 1);
-  }
-
-  tokens = tokens.slice(0, maxWords);
-  let result = tokens.join("-");
-
-  if (result.length > maxLength) {
-    result = result.substring(0, maxLength).replace(/-[^-]*$/, "");
-  }
-
-  return result || "page";
-}
-
-export interface SlugValidationResult {
-  isValid: boolean;
-  errors: string[];
-}
-
-/**
- * Validates a slug against universal SEO golden rules.
- */
-export function validateSeoSlug(slug: string): SlugValidationResult {
-  const errors: string[] = [];
-
-  if (!slug || slug.length < 2) {
-    errors.push("Slug must contain at least 2 characters");
-  }
-  if (slug.length > 75) {
-    errors.push("Slug should not exceed 75 characters for optimal CTR");
-  }
-  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug)) {
-    errors.push("Slug must be 100% lowercase alphanumeric with single hyphen (-) separators");
-  }
-  if (/(?:20[1-3][0-9])/.test(slug)) {
-    errors.push("Evergreen slug should not contain yearly date stamps");
-  }
-
-  return {
-    isValid: errors.length === 0,
-    errors,
-  };
-}