@houtini/seo-audit-console 0.2.0

package/dist/core/AuditDatabase.js

@@ -0,0 +1,481 @@
+/**
+ * AuditDatabase — one SQLite database per GSC property, colocating:
+ *   - GSC history       (search_analytics)            ← from better-search-console
+ *   - crawl snapshot     (crawl_metadata, pages, links, errors) ← from seo-crawler-mcp
+ *   - audit results      (audit_runs, findings)        ← new
+ *
+ * The join spine is the normalised URL key (`page_key` / `url_key`), written by
+ * url-key.ts on both the GSC and crawl write paths. See plan/architecture §2.
+ *
+ * Never console.log — MCP speaks JSON-RPC over stdio; use console.error for debug.
+ */
+import Database from 'better-sqlite3';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+export class AuditDatabase {
+    db;
+    constructor(dbPath) {
+        mkdirSync(dirname(dbPath), { recursive: true });
+        this.db = new Database(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('synchronous = NORMAL'); // WAL + NORMAL: durable enough, far fewer fsyncs
+        this.db.pragma('busy_timeout = 10000'); // wait for locks instead of throwing SQLITE_BUSY (concurrent access)
+        this.db.pragma('foreign_keys = ON');
+        // Performance: large sites have millions of search_analytics rows and the dashboard runs ~20
+        // GROUP-BY scans over them. A big page cache + memory-mapped I/O keeps the hot table resident
+        // and runs GROUP BY temp b-trees in RAM — big wins on the 1M+ row properties, no accuracy cost.
+        this.db.pragma('cache_size = -262144'); // 256 MB page cache (negative = KiB)
+        this.db.pragma('mmap_size = 536870912'); // 512 MB memory-mapped I/O
+        this.db.pragma('temp_store = MEMORY'); // GROUP BY / ORDER BY scratch in RAM, not on disk
+        this.initializeTables();
+    }
+    initializeTables() {
+        // ── Property + sync/crawl bookkeeping ──────────────────────────────────
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS property_meta (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        site_url TEXT UNIQUE NOT NULL,
+        host_form TEXT,                 -- 'apex' | 'www' | 'asis' (for url-key)
+        permission_level TEXT,
+        dfs_location_code INTEGER,      -- DataForSEO location (e.g. 2036 AU, 2826 UK, 2840 US)
+        dfs_location_name TEXT,         -- or a name, e.g. 'Australia'
+        last_synced_at TEXT,
+        last_crawl_id TEXT,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        // ── GSC history (from better-search-console) + page_key for the join ───
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS search_analytics (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        date TEXT NOT NULL,
+        query TEXT,
+        page TEXT,
+        page_key TEXT,                  -- normalised join key (urlKey(page))
+        device TEXT,
+        country TEXT,
+        search_appearance TEXT,
+        clicks INTEGER NOT NULL DEFAULT 0,
+        impressions INTEGER NOT NULL DEFAULT 0,
+        ctr REAL NOT NULL DEFAULT 0,
+        position REAL NOT NULL DEFAULT 0,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_sa_unique
+        ON search_analytics(date, query, page, device, country);
+      CREATE INDEX IF NOT EXISTS idx_sa_date        ON search_analytics(date);
+      CREATE INDEX IF NOT EXISTS idx_sa_query       ON search_analytics(query);
+      CREATE INDEX IF NOT EXISTS idx_sa_page_key    ON search_analytics(page_key);
+      CREATE INDEX IF NOT EXISTS idx_sa_date_pagekey ON search_analytics(date, page_key, clicks, impressions, position);
+      CREATE INDEX IF NOT EXISTS idx_sa_query_date  ON search_analytics(query, date);
+      CREATE INDEX IF NOT EXISTS idx_sa_date_metrics ON search_analytics(date, clicks, impressions, ctr, position);
+      -- Covering indexes for the windowed GROUP BYs in the merged checks (page_key / query+page_key),
+      -- in group-key order so SQLite can aggregate without a full re-sort on big GSC tables.
+      CREATE INDEX IF NOT EXISTS idx_sa_pk_date ON search_analytics(page_key, date, clicks, impressions, position);
+      CREATE INDEX IF NOT EXISTS idx_sa_q_pk_date ON search_analytics(query, page_key, date, impressions, clicks, position);
+
+      -- Dashboard payload cache: the dashboard runs ~20 GROUP-BY scans over search_analytics, which
+      -- is ~6s on a 1.8M-row property. The result only changes on sync/audit, so we cache the JSON
+      -- keyed by a cheap data-version and serve repeat views/exports instantly (accuracy preserved:
+      -- any data change bumps the version and forces a recompute). Single row (id=1).
+      CREATE TABLE IF NOT EXISTS dashboard_cache (
+        id INTEGER PRIMARY KEY CHECK (id = 1),
+        version TEXT NOT NULL,
+        payload TEXT NOT NULL,
+        created_at TEXT NOT NULL
+      );
+    `);
+        // ── Crawl snapshot (from seo-crawler-mcp), url_key added, redirects captured ─
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS crawl_metadata (
+        crawl_id TEXT PRIMARY KEY,
+        base_url TEXT NOT NULL,
+        base_domain TEXT NOT NULL,
+        status TEXT DEFAULT 'running',  -- running | completed | failed | interrupted | cancelled
+        max_depth INTEGER,
+        max_pages INTEGER,
+        user_agent TEXT,
+        rendered INTEGER DEFAULT 0,      -- 0 = HTTP only, 1 = render tier used
+        urls_discovered INTEGER DEFAULT 0,
+        urls_crawled INTEGER DEFAULT 0,
+        urls_failed INTEGER DEFAULT 0,
+        urls_skipped INTEGER DEFAULT 0,
+        started_at TEXT,
+        finished_at TEXT,
+        duration_ms INTEGER,
+        error TEXT
+      );
+    `);
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS pages (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        crawl_id TEXT NOT NULL,
+        url TEXT NOT NULL,
+        url_key TEXT NOT NULL,           -- urlKey(url) — the join key
+        status_code INTEGER,
+        content_type TEXT,
+        content_encoding TEXT,           -- br/gzip (compression war-story W6)
+        cache_control TEXT,
+        last_modified TEXT,
+        etag TEXT,
+        vary TEXT,                       -- Vary:User-Agent war-story (#54)
+        bytes INTEGER,                   -- transfer/decoded size (excessive-resource check)
+        response_time_ms INTEGER,        -- TTFB proxy
+        depth INTEGER,
+        is_internal INTEGER,
+        indexable INTEGER,               -- derived: 200 + HTML + not noindex + self-canonical
+        indexable_reason TEXT,           -- why NOT indexable: http-404/http-500/noindex-meta/noindex-header/canonicalised/non-html/robots-disallowed/redirect
+        noindex INTEGER,
+        title TEXT,
+        title_length INTEGER,
+        meta_description TEXT,
+        meta_description_length INTEGER,
+        h1 TEXT,
+        h1_count INTEGER,
+        word_count INTEGER,
+        lang TEXT,
+        charset TEXT,
+        canonical_url TEXT,
+        canonical_key TEXT,              -- urlKey(canonical_url) for D5 canonical-conflict
+        robots TEXT,
+        x_robots_tag TEXT,
+        viewport TEXT,
+        json_ld TEXT,                    -- raw JSON-LD blocks (schema validation)
+        og_tags TEXT,
+        hreflang TEXT,
+        redirects TEXT,                  -- captured 3xx chain (crawler bug fix)
+        internal_links INTEGER,
+        external_links INTEGER,
+        image_count INTEGER DEFAULT 0,         -- <img> count (image-alt check)
+        images_without_alt INTEGER DEFAULT 0,  -- <img> lacking non-empty alt
+        images_missing_dimensions INTEGER DEFAULT 0, -- <img> lacking width&height (CLS)
+        canonical_count INTEGER DEFAULT 0,     -- # rel=canonical tags (multiple-canonical)
+        canonical_relative INTEGER DEFAULT 0,  -- canonical declared as a relative URL
+        h2_count INTEGER DEFAULT 0,
+        heading_skips INTEGER DEFAULT 0,       -- skipped heading levels (h1→h3)
+        rel_next INTEGER DEFAULT 0,
+        rel_prev INTEGER DEFAULT 0,
+        rel_amphtml TEXT,                      -- declared AMP version URL
+        mixed_content_count INTEGER DEFAULT 0, -- http subresources on https page
+        twitter_tags TEXT,                     -- JSON of twitter:* meta
+        has_microdata INTEGER DEFAULT 0,
+        has_rdfa INTEGER DEFAULT 0,
+        has_favicon INTEGER,             -- link rel icon present (NULL = crawled before this was captured)
+        has_analytics INTEGER,           -- client-side analytics/tag-manager snippet detected (NULL = pre-feature crawl)
+        body_chunks TEXT,                -- JSON [{heading,level,text}] — content segmented by heading (RAG layer)
+        max_passage_score REAL,          -- cross-encoder best-passage relevance vs top GSC query (score_passages)
+        max_passage_query TEXT,          -- the query that best-passage score was computed against
+        max_passage_impr INTEGER,        -- that query's impressions at scoring time (for priority)
+        max_passage_at TEXT,             -- when scored
+        inlink_count INTEGER DEFAULT 0,  -- in-degree (computed post-crawl: orphans)
+        ipr REAL DEFAULT 0,              -- internal PageRank 0–100 (computed post-crawl)
+        click_depth INTEGER,             -- body-only shortest path from home (post-crawl; null=unreachable via body)
+        rendered INTEGER DEFAULT 0,      -- did this row come from the render tier?
+        render_diff TEXT,                -- raw-vs-rendered diff summary (JSON), if rendered
+        security_headers TEXT,           -- {csp,hsts,xFrame,referrer,...} JSON
+        crawled_at TEXT DEFAULT (datetime('now')),
+        FOREIGN KEY (crawl_id) REFERENCES crawl_metadata(crawl_id)
+      );
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_pages_urlkey ON pages(url_key);
+      CREATE INDEX IF NOT EXISTS idx_pages_status   ON pages(status_code);
+      CREATE INDEX IF NOT EXISTS idx_pages_depth    ON pages(depth);
+      CREATE INDEX IF NOT EXISTS idx_pages_indexable ON pages(indexable);
+      CREATE INDEX IF NOT EXISTS idx_pages_ipr      ON pages(ipr DESC);
+    `);
+        // ── Change-detection (drift): one persistent snapshot per (crawl, URL) of the SEO-salient
+        // fields. `pages` only ever holds the latest crawl (unique url_key), so cross-crawl diffing
+        // needs its own history table. Written once per crawl at audit time (INSERT OR IGNORE). ──
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS page_snapshots (
+        crawl_id TEXT NOT NULL,
+        url_key TEXT NOT NULL,
+        captured_at TEXT NOT NULL,
+        status_code INTEGER,
+        indexable INTEGER,
+        indexable_reason TEXT,
+        title TEXT,
+        meta_description TEXT,
+        h1 TEXT,
+        canonical_key TEXT,
+        robots TEXT,
+        x_robots_tag TEXT,
+        word_count INTEGER,
+        schema_types TEXT,               -- sorted distinct JSON-LD @types, comma-joined
+        PRIMARY KEY (crawl_id, url_key)
+      );
+      CREATE INDEX IF NOT EXISTS idx_snap_urlkey ON page_snapshots(url_key);
+      CREATE INDEX IF NOT EXISTS idx_snap_captured ON page_snapshots(captured_at);
+    `);
+        // Agent-readiness result (one row per origin) — populated by check_agent_readiness, read by
+        // the dashboard. Live HTTP probe, not crawl-derived, so it persists separately.
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS agent_readiness (
+        origin TEXT PRIMARY KEY,
+        score INTEGER,
+        level TEXT,
+        checks TEXT,        -- JSON [{id,category,label,present,detail,fix}]
+        by_category TEXT,   -- JSON [{category,passed,total}]
+        checked_at TEXT
+      );
+    `);
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS links (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        crawl_id TEXT NOT NULL,
+        source_url TEXT NOT NULL,
+        source_key TEXT NOT NULL,
+        target_url TEXT NOT NULL,
+        target_key TEXT NOT NULL,        -- join key both ends → fixes false orphans
+        anchor_text TEXT,
+        is_internal INTEGER,
+        placement TEXT,                  -- navigation | footer | body
+        rel TEXT,
+        FOREIGN KEY (crawl_id) REFERENCES crawl_metadata(crawl_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_links_source  ON links(source_key);
+      CREATE INDEX IF NOT EXISTS idx_links_target  ON links(target_key);
+      CREATE INDEX IF NOT EXISTS idx_links_internal ON links(is_internal);
+      CREATE INDEX IF NOT EXISTS idx_links_internal_placement ON links(is_internal, placement, target_key);
+    `);
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS errors (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        crawl_id TEXT NOT NULL,
+        url TEXT NOT NULL,
+        error_type TEXT,
+        error_message TEXT,
+        created_at TEXT DEFAULT (datetime('now')),
+        FOREIGN KEY (crawl_id) REFERENCES crawl_metadata(crawl_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_errors_type ON errors(error_type);
+    `);
+        // ── GSC URL Inspection (authoritative per-URL index status) ─────────────
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS url_inspection (
+        url_key TEXT PRIMARY KEY,
+        url TEXT NOT NULL,
+        verdict TEXT,
+        coverage_state TEXT,
+        indexing_state TEXT,
+        robots_txt_state TEXT,
+        page_fetch_state TEXT,
+        last_crawl_time TEXT,
+        google_canonical TEXT,           -- Google-selected canonical
+        user_canonical TEXT,             -- declared canonical (mismatch = D5 finding)
+        crawled_as TEXT,
+        mobile_usability TEXT,
+        rich_results TEXT,               -- JSON of detected rich-result types/issues
+        inspected_at TEXT DEFAULT (datetime('now'))
+      );
+      CREATE INDEX IF NOT EXISTS idx_inspection_coverage ON url_inspection(coverage_state);
+    `);
+        // ── DataForSEO rank history (over-time sequence, monthly) ───────────────
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS rank_history (
+        period TEXT PRIMARY KEY,         -- YYYY-MM
+        pos_1_3 INTEGER DEFAULT 0,
+        pos_4_10 INTEGER DEFAULT 0,
+        pos_11_20 INTEGER DEFAULT 0,
+        pos_21_100 INTEGER DEFAULT 0,
+        etv REAL DEFAULT 0,              -- estimated organic traffic value
+        keyword_count INTEGER DEFAULT 0, -- total ranking keywords (not 'count' — clashes with COUNT())
+        source TEXT,
+        fetched_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        // ── Audit results (new) ────────────────────────────────────────────────
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS audit_runs (
+        run_id TEXT PRIMARY KEY,
+        crawl_id TEXT,
+        scope TEXT,                      -- core | full
+        gsc_window_start TEXT,
+        gsc_window_end TEXT,
+        integrity_ok INTEGER,            -- crawl-integrity gates passed?
+        started_at TEXT,
+        finished_at TEXT,
+        finding_count INTEGER DEFAULT 0
+      );
+    `);
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS findings (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        run_id TEXT NOT NULL,
+        check_id TEXT NOT NULL,
+        category TEXT,                   -- integrity|crawlability|indexation|onpage|schema|performance|war-stories|agentic|merged
+        severity TEXT,                   -- crit|high|med|low|info
+        labels TEXT,                     -- JSON array of D|N|L|G|S
+        certainty REAL,                  -- 1.0 deterministic, 0.5 judgement
+        url_key TEXT,                    -- affected page (null = site-wide)
+        evidence TEXT,                   -- JSON proof payload
+        traffic_at_risk TEXT,            -- JSON {clicks,impressions,position,searchVolume}
+        effort TEXT,                     -- JSON {base,scaleModifier,fixType}
+        priority REAL,                   -- P = (S × C × V) / E
+        recommendation TEXT,             -- JSON {text, generated?}
+        FOREIGN KEY (run_id) REFERENCES audit_runs(run_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_findings_run      ON findings(run_id);
+      CREATE INDEX IF NOT EXISTS idx_findings_priority ON findings(priority DESC);
+      CREATE INDEX IF NOT EXISTS idx_findings_check    ON findings(check_id);
+    `);
+        // ── Backlinks (DataForSEO, on-demand) — page-level counts + live HTTP status ──
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS page_backlinks (
+        url_key TEXT PRIMARY KEY,          -- normalised join key (urlKey of the page)
+        url TEXT NOT NULL,                 -- the page address DataForSEO reported
+        backlinks INTEGER DEFAULT 0,
+        referring_domains INTEGER DEFAULT 0,
+        dofollow INTEGER DEFAULT 0,        -- dofollow backlink count
+        status_code INTEGER,              -- live HTTP status of the backlinked page (null = unchecked)
+        fetched_at TEXT DEFAULT (datetime('now'))
+      );
+      CREATE INDEX IF NOT EXISTS idx_pbl_backlinks ON page_backlinks(backlinks DESC);
+    `);
+        // ── Sitemap URLs (for crawl↔sitemap reconciliation) ────────────────────
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS sitemap_urls (
+        url_key TEXT PRIMARY KEY,        -- normalised join key of a URL listed in the XML sitemap(s)
+        url TEXT NOT NULL,
+        lastmod TEXT,                    -- the sitemap's <lastmod> claim (freshness-honesty check)
+        fetched_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        // ── Image weights (post-crawl sample: distinct <img> srcs + content-length) ─
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS image_assets (
+        url TEXT PRIMARY KEY,
+        bytes INTEGER,                   -- content-length (body never downloaded)
+        status INTEGER,
+        content_type TEXT,
+        used_on INTEGER DEFAULT 1,       -- how many crawled pages reference this image
+        fetched_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        // ── DataForSEO on-demand enrichments persisted for the audit (6a) ────────
+        // Keyword intent (Labs search_intent) → powers intent-vs-pagetype-mismatch.
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS keyword_intent (
+        keyword TEXT PRIMARY KEY,
+        intent TEXT,                    -- informational | navigational | commercial | transactional
+        probability REAL,
+        fetched_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        // Per-URL lab Core Web Vitals (On-Page Lighthouse) → powers high-yield-cwv-fail.
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS page_cwv (
+        url_key TEXT PRIMARY KEY,        -- urlKey of the audited URL (join spine)
+        url TEXT NOT NULL,
+        for_mobile INTEGER DEFAULT 1,
+        performance REAL,                -- Lighthouse performance score 0–1
+        lcp_ms REAL,                     -- largest-contentful-paint numericValue (ms)
+        cls REAL,                        -- cumulative-layout-shift numericValue
+        tbt_ms REAL,                     -- total-blocking-time numericValue (ms)
+        fetched_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        // ── Wikidata entity layer (6d) — heuristic H1→QID resolution + relationship edges ──
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS page_entity (
+        url_key TEXT PRIMARY KEY,        -- page resolved to a Wikidata entity
+        qid TEXT NOT NULL,               -- Wikidata QID (e.g. Q42)
+        label TEXT,
+        description TEXT,
+        source TEXT,                     -- 'h1' | 'title' | 'declared' (how we resolved)
+        fetched_at TEXT DEFAULT (datetime('now'))
+      );
+      CREATE INDEX IF NOT EXISTS idx_page_entity_qid ON page_entity(qid);
+      CREATE TABLE IF NOT EXISTS entity_edge (
+        qid TEXT NOT NULL,               -- child / part entity
+        related_qid TEXT NOT NULL,       -- parent / whole entity (P279 subclass-of / P361 part-of target)
+        relation TEXT NOT NULL,          -- 'subclass_of' | 'part_of'
+        PRIMARY KEY (qid, related_qid, relation)
+      );
+      CREATE INDEX IF NOT EXISTS idx_entity_edge_related ON entity_edge(related_qid);
+    `);
+        this.migrate();
+    }
+    /** Idempotent column additions for DBs created before a column existed. */
+    migrate() {
+        const cols = (table) => new Set(this.db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name));
+        const ensure = (table, col, ddl) => {
+            if (!cols(table).has(col))
+                this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${ddl}`);
+        };
+        ensure('pages', 'image_count', 'image_count INTEGER DEFAULT 0');
+        ensure('pages', 'images_without_alt', 'images_without_alt INTEGER DEFAULT 0');
+        ensure('pages', 'images_missing_dimensions', 'images_missing_dimensions INTEGER DEFAULT 0');
+        ensure('pages', 'canonical_count', 'canonical_count INTEGER DEFAULT 0');
+        ensure('pages', 'canonical_relative', 'canonical_relative INTEGER DEFAULT 0');
+        ensure('pages', 'h2_count', 'h2_count INTEGER DEFAULT 0');
+        ensure('pages', 'heading_skips', 'heading_skips INTEGER DEFAULT 0');
+        ensure('pages', 'rel_next', 'rel_next INTEGER DEFAULT 0');
+        ensure('pages', 'rel_prev', 'rel_prev INTEGER DEFAULT 0');
+        ensure('pages', 'rel_amphtml', 'rel_amphtml TEXT');
+        ensure('pages', 'mixed_content_count', 'mixed_content_count INTEGER DEFAULT 0');
+        ensure('pages', 'twitter_tags', 'twitter_tags TEXT');
+        ensure('pages', 'has_microdata', 'has_microdata INTEGER DEFAULT 0');
+        ensure('pages', 'has_rdfa', 'has_rdfa INTEGER DEFAULT 0');
+        ensure('pages', 'body_chunks', 'body_chunks TEXT');
+        ensure('pages', 'has_favicon', 'has_favicon INTEGER');
+        ensure('pages', 'has_analytics', 'has_analytics INTEGER');
+        ensure('pages', 'conditional_304', 'conditional_304 INTEGER'); // 1 = honoured If-Modified-Since/If-None-Match with a 304; 0 = returned 200 anyway; NULL = not probed
+        ensure('sitemap_urls', 'lastmod', 'lastmod TEXT');
+        ensure('pages', 'max_passage_score', 'max_passage_score REAL');
+        ensure('pages', 'max_passage_query', 'max_passage_query TEXT');
+        ensure('pages', 'max_passage_impr', 'max_passage_impr INTEGER');
+        ensure('pages', 'max_passage_at', 'max_passage_at TEXT');
+        ensure('pages', 'click_depth', 'click_depth INTEGER');
+        ensure('pages', 'indexable_reason', 'indexable_reason TEXT');
+        // property-level backlink profile summary (from DataForSEO backlinks/summary)
+        ensure('property_meta', 'total_backlinks', 'total_backlinks INTEGER');
+        ensure('property_meta', 'referring_domains', 'referring_domains INTEGER');
+        ensure('property_meta', 'backlinks_spam_score', 'backlinks_spam_score REAL');
+        ensure('property_meta', 'backlinks_fetched_at', 'backlinks_fetched_at TEXT');
+        // backlinks/summary extras (same paid response): Domain Rank, equity leaks, nofollow share
+        ensure('property_meta', 'backlinks_rank', 'backlinks_rank INTEGER'); // DataForSEO Domain Rank (0–1000)
+        ensure('property_meta', 'broken_backlinks', 'broken_backlinks INTEGER');
+        ensure('property_meta', 'broken_pages', 'broken_pages INTEGER');
+        ensure('property_meta', 'backlinks_nofollow_pct', 'backlinks_nofollow_pct REAL'); // share of referring links marked nofollow (0–1)
+        // rank_history keyword-movement counts + paid-value — same historical_rank_overview response we already pay for
+        ensure('rank_history', 'is_new', 'is_new INTEGER');
+        ensure('rank_history', 'is_up', 'is_up INTEGER');
+        ensure('rank_history', 'is_down', 'is_down INTEGER');
+        ensure('rank_history', 'is_lost', 'is_lost INTEGER');
+        ensure('rank_history', 'estimated_paid_traffic_cost', 'estimated_paid_traffic_cost REAL');
+        // url_inspection: sample of referring pages Google reports for the URL (JSON array of URLs)
+        ensure('url_inspection', 'referring_urls', 'referring_urls TEXT');
+        // GSC search type ('web' | 'discover' | 'googleNews' | 'image' | 'video'). NOTE: the unique
+        // index idx_sa_unique deliberately does NOT include this column (rebuilding it on existing
+        // rows would rewrite months of history) — the table therefore holds ONE search type at a
+        // time, enforced by the mode-guard in GscSync (a type switch wipes, riding the first batch).
+        ensure('search_analytics', 'search_type', "search_type TEXT DEFAULT 'web'");
+    }
+    // ── Minimal property accessors (full CRUD added as modules are wired) ─────
+    upsertProperty(siteUrl, hostForm, permissionLevel) {
+        this.db
+            .prepare(`INSERT INTO property_meta (site_url, host_form, permission_level)
+         VALUES (?, ?, ?)
+         ON CONFLICT(site_url) DO UPDATE SET host_form = excluded.host_form,
+           permission_level = COALESCE(excluded.permission_level, property_meta.permission_level)`)
+            .run(siteUrl, hostForm, permissionLevel ?? null);
+    }
+    getProperty(siteUrl) {
+        return this.db.prepare('SELECT site_url, host_form, last_synced_at FROM property_meta WHERE site_url = ?').get(siteUrl);
+    }
+    /** Persist the DataForSEO target location for a property (string name or numeric code). */
+    setDfsLocation(siteUrl, location) {
+        this.db
+            .prepare('UPDATE property_meta SET dfs_location_code = ?, dfs_location_name = ? WHERE site_url = ?')
+            .run(typeof location === 'number' ? location : null, typeof location === 'string' ? location : null, siteUrl);
+    }
+    /** Resolved DataForSEO location for a property: a code, a name, or undefined. */
+    getDfsLocation(siteUrl) {
+        const r = this.db.prepare('SELECT dfs_location_code c, dfs_location_name n FROM property_meta WHERE site_url = ?').get(siteUrl);
+        return r?.c ?? r?.n ?? undefined;
+    }
+    close() {
+        this.db.close();
+    }
+}
+//# sourceMappingURL=AuditDatabase.js.map

package/dist/core/AuditDatabase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AuditDatabase.js","sourceRoot":"","sources":["../../src/core/AuditDatabase.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,MAAM,OAAO,aAAa;IACf,EAAE,CAAoB;IAE/B,YAAY,MAAc;QACxB,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,iDAAiD;QACzF,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAE,qEAAqE;QAC9G,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QACpC,6FAA6F;QAC7F,8FAA8F;QAC9F,gGAAgG;QAChG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAE,qCAAqC;QAC9E,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,CAAC,2BAA2B;QACpE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAG,kDAAkD;QAC3F,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC1B,CAAC;IAEO,gBAAgB;QACtB,0EAA0E;QAC1E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;KAYZ,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAuCZ,CAAC,CAAC;QAEH,gFAAgF;QAChF,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;KAmBZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA2EZ,CAAC,CAAC;QAEH,2FAA2F;QAC3F,4FAA4F;QAC5F,2FAA2F;QAC3F,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;KAoBZ,CAAC,CAAC;QAEH,4FAA4F;QAC5F,gFAAgF;QAChF,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;KASZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;KAkBZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;KAWZ,CAAC,CAAC;QAEH,2EAA2E;QAC3E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;KAkBZ,CAAC,CAAC;QAEH,2EAA2E;QAC3E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;KAYZ,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;KAYZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;KAoBZ,CAAC,CAAC;QAEH,iFAAiF;QACjF,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;KAWZ,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;KAOZ,CAAC,CAAC;QAEH,+EAA+E;QAC/E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;KASZ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,4EAA4E;QAC5E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;KAOZ,CAAC,CAAC;QACH,iFAAiF;QACjF,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;KAWZ,CAAC,CAAC;QAEH,sFAAsF;QACtF,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;KAiBZ,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED,2EAA2E;IACnE,OAAO;QACb,MAAM,IAAI,GAAG,CAAC,KAAa,EAAe,EAAE,CAC1C,IAAI,GAAG,CAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,GAAG,EAAyB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACzG,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW,EAAQ,EAAE;YAC/D,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,KAAK,eAAe,GAAG,EAAE,CAAC,CAAC;QACpF,CAAC,CAAC;QACF,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,+BAA+B,CAAC,CAAC;QAChE,MAAM,CAAC,OAAO,EAAE,oBAAoB,EAAE,sCAAsC,CAAC,CAAC;QAC9E,MAAM,CAAC,OAAO,EAAE,2BAA2B,EAAE,6CAA6C,CAAC,CAAC;QAC5F,MAAM,CAAC,OAAO,EAAE,iBAAiB,EAAE,mCAAmC,CAAC,CAAC;QACxE,MAAM,CAAC,OAAO,EAAE,oBAAoB,EAAE,sCAAsC,CAAC,CAAC;QAC9E,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,4BAA4B,CAAC,CAAC;QAC1D,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,iCAAiC,CAAC,CAAC;QACpE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,4BAA4B,CAAC,CAAC;QAC1D,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,4BAA4B,CAAC,CAAC;QAC1D,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,kBAAkB,CAAC,CAAC;QACnD,MAAM,CAAC,OAAO,EAAE,qBAAqB,EAAE,uCAAuC,CAAC,CAAC;QAChF,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,mBAAmB,CAAC,CAAC;QACrD,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,iCAAiC,CAAC,CAAC;QACpE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,4BAA4B,CAAC,CAAC;QAC1D,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,kBAAkB,CAAC,CAAC;QACnD,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,qBAAqB,CAAC,CAAC;QACtD,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,uBAAuB,CAAC,CAAC;QAC1D,MAAM,CAAC,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,CAAC,CAAC,CAAC,sGAAsG;QACrK,MAAM,CAAC,cAAc,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;QAClD,MAAM,CAAC,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,CAAC,CAAC;QAC/D,MAAM,CAAC,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,CAAC,CAAC;QAC/D,MAAM,CAAC,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,CAAC,CAAC;QAChE,MAAM,CAAC,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,CAAC,CAAC;QACzD,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,qBAAqB,CAAC,CAAC;QACtD,MAAM,CAAC,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,CAAC,CAAC;QAC7D,8EAA8E;QAC9E,MAAM,CAAC,eAAe,EAAE,iBAAiB,EAAE,yBAAyB,CAAC,CAAC;QACtE,MAAM,CAAC,eAAe,EAAE,mBAAmB,EAAE,2BAA2B,CAAC,CAAC;QAC1E,MAAM,CAAC,eAAe,EAAE,sBAAsB,EAAE,2BAA2B,CAAC,CAAC;QAC7E,MAAM,CAAC,eAAe,EAAE,sBAAsB,EAAE,2BAA2B,CAAC,CAAC;QAC7E,2FAA2F;QAC3F,MAAM,CAAC,eAAe,EAAE,gBAAgB,EAAE,wBAAwB,CAAC,CAAC,CAAW,kCAAkC;QACjH,MAAM,CAAC,eAAe,EAAE,kBAAkB,EAAE,0BAA0B,CAAC,CAAC;QACxE,MAAM,CAAC,eAAe,EAAE,cAAc,EAAE,sBAAsB,CAAC,CAAC;QAChE,MAAM,CAAC,eAAe,EAAE,wBAAwB,EAAE,6BAA6B,CAAC,CAAC,CAAC,iDAAiD;QACnI,gHAAgH;QAChH,MAAM,CAAC,cAAc,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QACnD,MAAM,CAAC,cAAc,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QACjD,MAAM,CAAC,cAAc,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC;QACrD,MAAM,CAAC,cAAc,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC;QACrD,MAAM,CAAC,cAAc,EAAE,6BAA6B,EAAE,kCAAkC,CAAC,CAAC;QAC1F,4FAA4F;QAC5F,MAAM,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,CAAC,CAAC;QAClE,4FAA4F;QAC5F,2FAA2F;QAC3F,yFAAyF;QACzF,6FAA6F;QAC7F,MAAM,CAAC,kBAAkB,EAAE,aAAa,EAAE,gCAAgC,CAAC,CAAC;IAC9E,CAAC;IAED,6EAA6E;IAC7E,cAAc,CAAC,OAAe,EAAE,QAAgB,EAAE,eAAwB;QACxE,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;kGAG0F,CAC3F;aACA,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,WAAW,CAAC,OAAe;QACzB,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kFAAkF,CAAC,CAAC,GAAG,CAAC,OAAO,CAEzG,CAAC;IAChB,CAAC;IAED,2FAA2F;IAC3F,cAAc,CAAC,OAAe,EAAE,QAAyB;QACvD,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,0FAA0F,CAAC;aACnG,GAAG,CAAC,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAClH,CAAC;IAED,iFAAiF;IACjF,cAAc,CAAC,OAAe;QAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uFAAuF,CAAC,CAAC,GAAG,CAAC,OAAO,CAEjH,CAAC;QACd,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,SAAS,CAAC;IACnC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}

package/dist/core/Backlinks.d.ts

@@ -0,0 +1,33 @@
+import { DataForSeoClient } from './DataForSeoClient.js';
+export interface BacklinksResult {
+    siteUrl: string;
+    target: string;
+    pages: number;
+    statusChecked: number;
+    totalBacklinks: number;
+    referringDomains: number;
+    domainRank: number | null;
+    brokenBacklinks: number;
+    brokenPages: number;
+    nofollowPct: number | null;
+    cached: boolean;
+    cost: number;
+}
+/**
+ * On-demand backlink profile via DataForSEO (cheap endpoints): summary/live (overall
+ * profile → property_meta) + domain_pages_summary/live (per-page backlink counts →
+ * page_backlinks). Then resolves each backlinked page's live HTTP status — from the
+ * crawl when present, else a bounded fetch — so backlinks-to-404 is precise.
+ * Never auto-run (the user's rule: backlinks are paid + on-demand); 20-day cached.
+ */
+export declare class Backlinks {
+    private readonly dfs;
+    private readonly dataDir;
+    constructor(dfs: DataForSeoClient, dataDir: string);
+    run(siteUrl: string, opts: {
+        limit?: number;
+        statusLimit?: number;
+        userAgent?: string;
+    }, update: (p: Record<string, unknown>) => void, signal: AbortSignal): Promise<BacklinksResult>;
+}
+//# sourceMappingURL=Backlinks.d.ts.map

package/dist/core/Backlinks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Backlinks.d.ts","sourceRoot":"","sources":["../../src/core/Backlinks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAoBzD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IACtE,cAAc,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAC;IACjD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,GAAG;IAAoB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAA/C,GAAG,EAAE,gBAAgB,EAAmB,OAAO,EAAE,MAAM;IAE9E,GAAG,CACP,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,EAClE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,EAC5C,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,CAAC;CA0F5B"}

package/dist/core/Backlinks.js

@@ -0,0 +1,110 @@
+import { AuditDatabase } from './AuditDatabase.js';
+import { dbPathFor } from './paths.js';
+import { urlKey, hostFormForProperty } from './url-key.js';
+const n = (v) => Number(v) || 0;
+const domainFromProperty = (siteUrl) => siteUrl.replace(/^sc-domain:/, '').replace(/^https?:\/\//, '').replace(/\/+$/, '');
+/** Live HTTP status of a backlinked URL (follow redirects → final status; null on error). */
+async function statusOf(url, ua) {
+    try {
+        const res = await fetch(url, {
+            method: 'GET', redirect: 'follow',
+            headers: { 'user-agent': ua, 'cache-control': 'no-cache' },
+            signal: AbortSignal.timeout(10000),
+        });
+        return res.status;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * On-demand backlink profile via DataForSEO (cheap endpoints): summary/live (overall
+ * profile → property_meta) + domain_pages_summary/live (per-page backlink counts →
+ * page_backlinks). Then resolves each backlinked page's live HTTP status — from the
+ * crawl when present, else a bounded fetch — so backlinks-to-404 is precise.
+ * Never auto-run (the user's rule: backlinks are paid + on-demand); 20-day cached.
+ */
+export class Backlinks {
+    dfs;
+    dataDir;
+    constructor(dfs, dataDir) {
+        this.dfs = dfs;
+        this.dataDir = dataDir;
+    }
+    async run(siteUrl, opts, update, signal) {
+        const target = domainFromProperty(siteUrl);
+        const hostForm = hostFormForProperty(siteUrl);
+        const ua = opts.userAgent ?? 'seo-audit-console';
+        const db = new AuditDatabase(dbPathFor(this.dataDir, siteUrl));
+        try {
+            db.upsertProperty(siteUrl, hostForm ?? 'asis');
+            // 1. Profile summary → property_meta
+            const sum = await this.dfs.backlinksSummary(target);
+            // Surface task-level failures (the Backlinks API is a SEPARATE DataForSEO subscription —
+            // a 40204 here means it isn't activated). Don't silently store 0 backlinks.
+            const task = sum.tasks[0];
+            if (!task || (task.status_code ?? 0) >= 40000) {
+                const msg = task?.status_message ?? 'no task returned';
+                throw new Error(`DataForSEO Backlinks unavailable: ${msg}` +
+                    (task?.status_code === 40204 ? ' — the Backlinks API is a separate DataForSEO subscription; activate it at app.dataforseo.com/backlinks-subscription to use pull_backlinks.' : ''));
+            }
+            const s = task.result?.[0] ?? {};
+            // Summary extras previously dropped from the same paid response: Domain Rank (the authority
+            // number), broken backlinks/pages (equity leaks Google's link graph sees), nofollow share.
+            const nofollowPct = n(s.backlinks) > 0 ? n(s.referring_links_attributes?.nofollow) / n(s.backlinks) : null;
+            db.db.prepare(`UPDATE property_meta SET total_backlinks=?, referring_domains=?, backlinks_spam_score=?,
+           backlinks_rank=?, broken_backlinks=?, broken_pages=?, backlinks_nofollow_pct=?,
+           backlinks_fetched_at=datetime('now') WHERE site_url=?`).run(n(s.backlinks), n(s.referring_domains), s.backlinks_spam_score ?? null, s.rank != null ? n(s.rank) : null, n(s.broken_backlinks), n(s.broken_pages), nofollowPct, siteUrl);
+            update({ phase: 'summary', backlinks: n(s.backlinks), referringDomains: n(s.referring_domains), rank: s.rank ?? null });
+            // 2. Per-page backlink counts → page_backlinks
+            const res = await this.dfs.domainPagesSummary(target, opts.limit ?? 1000);
+            const items = res.tasks[0]?.result?.[0]?.items ?? [];
+            const rows = items
+                .map(it => {
+                const url = it.url ?? it.page_address ?? it.page ?? '';
+                return {
+                    url,
+                    url_key: url ? urlKey(url, { hostForm }) : '',
+                    backlinks: n(it.backlinks),
+                    referring_domains: n(it.referring_domains ?? it.referring_main_domains),
+                    dofollow: n(it.dofollow ?? it.backlinks_dofollow ?? it.referring_links_types?.anchor),
+                };
+            })
+                .filter(r => r.url && r.url_key);
+            // 3. Live status — reuse crawl status where we have it; fetch the rest (bounded, by backlinks).
+            const crawlStatus = new Map(db.db.prepare('SELECT url_key, status_code FROM pages WHERE status_code IS NOT NULL').all()
+                .map(r => [r.url_key, r.status_code]));
+            const toFetch = rows.filter(r => !crawlStatus.has(r.url_key)).sort((a, b) => b.backlinks - a.backlinks).slice(0, opts.statusLimit ?? 150);
+            const fetched = new Map();
+            let i = 0;
+            await Promise.all(Array.from({ length: 8 }, async () => {
+                while (i < toFetch.length && !signal.aborted) {
+                    const r = toFetch[i++];
+                    fetched.set(r.url_key, await statusOf(r.url, ua));
+                    update({ phase: 'status', checked: fetched.size, total: toFetch.length });
+                }
+            }));
+            const upsert = db.db.prepare(`INSERT INTO page_backlinks (url_key, url, backlinks, referring_domains, dofollow, status_code, fetched_at)
+         VALUES (@url_key,@url,@backlinks,@referring_domains,@dofollow,@status_code,datetime('now'))
+         ON CONFLICT(url_key) DO UPDATE SET url=excluded.url, backlinks=excluded.backlinks,
+           referring_domains=excluded.referring_domains, dofollow=excluded.dofollow,
+           status_code=excluded.status_code, fetched_at=datetime('now')`);
+            db.db.transaction(() => {
+                for (const r of rows)
+                    upsert.run({ ...r, status_code: crawlStatus.get(r.url_key) ?? fetched.get(r.url_key) ?? null });
+            })();
+            return {
+                siteUrl, target, pages: rows.length, statusChecked: fetched.size,
+                totalBacklinks: n(s.backlinks), referringDomains: n(s.referring_domains),
+                domainRank: s.rank != null ? n(s.rank) : null,
+                brokenBacklinks: n(s.broken_backlinks), brokenPages: n(s.broken_pages),
+                nofollowPct: nofollowPct != null ? Math.round(nofollowPct * 1000) / 1000 : null,
+                cached: sum.cached && res.cached, cost: sum.cost + res.cost,
+            };
+        }
+        finally {
+            db.close();
+        }
+    }
+}
+//# sourceMappingURL=Backlinks.js.map

package/dist/core/Backlinks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Backlinks.js","sourceRoot":"","sources":["../../src/core/Backlinks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAE3D,MAAM,CAAC,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACjD,MAAM,kBAAkB,GAAG,CAAC,OAAe,EAAU,EAAE,CACrD,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAErF,6FAA6F;AAC7F,KAAK,UAAU,QAAQ,CAAC,GAAW,EAAE,EAAU;IAC7C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ;YACjC,OAAO,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,eAAe,EAAE,UAAU,EAAE;YAC1D,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACnC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC,MAAM,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,IAAI,CAAC;IAAC,CAAC;AAC1B,CAAC;AAWD;;;;;;GAMG;AACH,MAAM,OAAO,SAAS;IACS;IAAwC;IAArE,YAA6B,GAAqB,EAAmB,OAAe;QAAvD,QAAG,GAAH,GAAG,CAAkB;QAAmB,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAExF,KAAK,CAAC,GAAG,CACP,OAAe,EACf,IAAkE,EAClE,MAA4C,EAC5C,MAAmB;QAEnB,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,CAAC;QACjD,MAAM,EAAE,GAAG,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC;YACH,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YACpD,yFAAyF;YACzF,4EAA4E;YAC5E,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;gBAC9C,MAAM,GAAG,GAAG,IAAI,EAAE,cAAc,IAAI,kBAAkB,CAAC;gBACvD,MAAM,IAAI,KAAK,CACb,qCAAqC,GAAG,EAAE;oBAC1C,CAAC,IAAI,EAAE,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,6IAA6I,CAAC,CAAC,CAAC,EAAE,CAAC,CACnL,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACjC,4FAA4F;YAC5F,2FAA2F;YAC3F,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,0BAA0B,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC3G,EAAE,CAAC,EAAE,CAAC,OAAO,CACX;;iEAEyD,CAC1D,CAAC,GAAG,CACH,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,oBAAoB,IAAI,IAAI,EACtE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,WAAW,EACxF,OAAO,CACR,CAAC;YACF,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;YAExH,+CAA+C;YAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;YAC1E,MAAM,KAAK,GAAU,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;YAC5D,MAAM,IAAI,GAAG,KAAK;iBACf,GAAG,CAAC,EAAE,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,YAAY,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACvD,OAAO;oBACL,GAAG;oBACH,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;oBAC7C,SAAS,EAAE,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC;oBAC1B,iBAAiB,EAAE,CAAC,CAAC,EAAE,CAAC,iBAAiB,IAAI,EAAE,CAAC,sBAAsB,CAAC;oBACvE,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,kBAAkB,IAAI,EAAE,CAAC,qBAAqB,EAAE,MAAM,CAAC;iBACtF,CAAC;YACJ,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;YAEnC,gGAAgG;YAChG,MAAM,WAAW,GAAG,IAAI,GAAG,CACxB,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,sEAAsE,CAAC,CAAC,GAAG,EAAiD;iBACxI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CACxC,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC,CAAC;YAC1I,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;YACjD,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;gBACrD,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;oBACvB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;oBAClD,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACH,CAAC,CAAC,CAAC,CAAC;YAEJ,MAAM,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,OAAO,CAC1B;;;;wEAIgE,CACjE,CAAC;YACF,EAAE,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;gBACrB,KAAK,MAAM,CAAC,IAAI,IAAI;oBAAE,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YACxH,CAAC,CAAC,EAAE,CAAC;YAEL,OAAO;gBACL,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,IAAI;gBAChE,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;gBACxE,UAAU,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC7C,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;gBACtE,WAAW,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI;gBAC/E,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI;aAC5D,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}