@crawlith/core 0.1.1 → 0.1.2

package/dist/crawler/metricsRunner.d.ts

@@ -1,3 +1,23 @@
 import { EngineContext } from '../events.js';
 import { Graph } from '../graph/graph.js';
-export declare function runPostCrawlMetrics(snapshotId: number, maxDepth: number, context?: EngineContext, limitReached?: boolean, graphInstance?: Graph): void;
+export interface PostCrawlOptions {
+    context?: EngineContext;
+    limitReached?: boolean;
+    graphInstance?: Graph;
+    clustering?: boolean;
+    clusterThreshold?: number;
+    minClusterSize?: number;
+    health?: boolean;
+    computePagerank?: boolean;
+    computeHits?: boolean;
+    heading?: boolean;
+    orphans?: boolean;
+    orphanSeverity?: 'low' | 'medium' | 'high' | boolean;
+    includeSoftOrphans?: boolean;
+    minInbound?: number;
+    rootOrigin?: string;
+}
+export declare function runPostCrawlMetrics(snapshotId: number, maxDepth: number, options?: PostCrawlOptions): {
+    metrics: any;
+    healthData?: any;
+} | undefined;

package/dist/crawler/metricsRunner.js

@@ -3,11 +3,22 @@ import { loadGraphFromSnapshot } from '../db/graphLoader.js';
 import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
 import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
 import { PageRepository } from '../db/repositories/PageRepository.js';
-import { computePageRank } from '../graph/pagerank.js';
 import { calculateMetrics } from '../graph/metrics.js';
-import { computeHITS } from '../scoring/hits.js';
-import { calculateHealthScore, collectCrawlIssues } from '../scoring/health.js';
-export function runPostCrawlMetrics(snapshotId, maxDepth, context, limitReached = false, graphInstance) {
+import { PageRankService } from '../graph/pagerank.js';
+import { HITSService } from '../graph/hits.js';
+import { TrapDetector } from './trap.js';
+import { ClusteringService } from '../analysis/clustering.js';
+import { DuplicateService } from '../analysis/duplicate.js';
+import { annotateOrphans } from '../analysis/orphan.js';
+import { Soft404Service } from '../analysis/soft404.js';
+import { HeadingHealthService } from '../analysis/heading.js';
+import { HealthService } from '../scoring/health.js';
+import { analyzeContent } from '../analysis/content.js';
+import { load } from 'cheerio';
+export function runPostCrawlMetrics(snapshotId, maxDepth, options = {}) {
+    const context = options.context;
+    const limitReached = options.limitReached || false;
+    const graphInstance = options.graphInstance;
     const db = getDb();
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
@@ -21,62 +32,160 @@ export function runPostCrawlMetrics(snapshotId, maxDepth, context, limitReached
         else {
             if (event.type === 'error')
                 console.error(event.message);
-            else if (event.type !== 'debug')
-                console.log(event.message || event.phase);
+            else if (event.type !== 'debug') {
+                const out = event.message || event.phase;
+                if (out)
+                    console.log(out);
+            }
         }
     };
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
     if (!snapshot) {
         emit({ type: 'error', message: `Snapshot ${snapshotId} not found` });
-        return;
+        return undefined;
     }
     if (!graphInstance) {
         emit({ type: 'metrics:start', phase: 'Loading graph' });
     }
-    emit({ type: 'metrics:start', phase: 'Computing PageRank' });
-    computePageRank(graph);
-    emit({ type: 'metrics:start', phase: 'Computing HITS' });
-    computeHITS(graph);
+    emit({ type: 'metrics:start', phase: 'Running core algorithms' });
+    // 1. Graph Algorithms
+    const prResults = options.computePagerank ? new PageRankService().evaluate(graph) : new Map();
+    const hitsResults = options.computeHits ? new HITSService().evaluate(graph, { iterations: 20 }) : new Map();
+    // 2. Crawler Safety
+    new TrapDetector().analyze(graph);
+    // 3. Analysis / Intelligence
+    if (options.clustering) {
+        const contentClusters = new ClusteringService().detectContentClusters(graph, options.clusterThreshold, options.minClusterSize);
+        if (contentClusters.length > 0) {
+            const insertCluster = db.prepare(`
+                INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
+                VALUES (@id, @snapshot_id, @count, @primary_url, @risk, @shared_path_prefix)
+            `);
+            const insertContentTx = db.transaction((clusters) => {
+                for (const c of clusters) {
+                    insertCluster.run({
+                        id: c.id,
+                        snapshot_id: snapshotId,
+                        count: c.count,
+                        primary_url: c.primaryUrl,
+                        risk: c.risk,
+                        shared_path_prefix: c.sharedPathPrefix ?? null
+                    });
+                }
+            });
+            insertContentTx(contentClusters);
+        }
+    }
+    const duplicateClusters = new DuplicateService().detectDuplicates(graph, { collapse: false });
+    if (duplicateClusters.length > 0) {
+        const insertCluster = db.prepare(`
+            INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
+            VALUES (@id, @snapshot_id, @type, @size, @representative, @severity)
+        `);
+        const insertDuplicateTx = db.transaction((clusters) => {
+            for (const c of clusters) {
+                insertCluster.run({
+                    id: c.id,
+                    snapshot_id: snapshotId,
+                    type: c.type, // valid: 'exact' | 'near' | 'template_heavy'
+                    size: c.size,
+                    representative: c.representative,
+                    severity: c.severity || 'low'
+                });
+            }
+        });
+        insertDuplicateTx(duplicateClusters);
+    }
+    let annotatedNodes = [];
+    if (options.orphans) {
+        const orphanOptions = {
+            enabled: true,
+            severityEnabled: !!options.orphanSeverity || options.orphanSeverity === undefined,
+            includeSoftOrphans: options.includeSoftOrphans ?? true,
+            minInbound: options.minInbound ?? 2
+        };
+        annotatedNodes = annotateOrphans(graph.getNodes(), graph.getEdges(), orphanOptions);
+    }
+    const soft404Service = new Soft404Service();
+    const headingService = new HeadingHealthService();
+    // Pre-calculate heading health for all nodes with HTML
+    let headingPayloads = new Map();
+    if (options.heading) {
+        const result = headingService.evaluateNodes(graph.getNodes());
+        headingPayloads = result.payloadsByUrl;
+    }
+    // Apply signals to nodes
+    for (const node of graph.getNodes()) {
+        const pr = prResults.get(node.url);
+        if (pr)
+            node.pagerankScore = pr.score;
+        const hits = hitsResults.get(node.url);
+        if (hits) {
+            node.authScore = hits.authority_score;
+            node.hubScore = hits.hub_score;
+            node.linkRole = hits.link_role;
+        }
+        if (options.orphans) {
+            const annotated = annotatedNodes.find((n) => n.url === node.url);
+            if (annotated) {
+                node.orphanScore = annotated.orphanSeverity;
+                node.orphanType = annotated.orphanType;
+                node.impactLevel = annotated.impactLevel;
+            }
+        }
+        if (options.heading) {
+            const heading = headingPayloads.get(node.url);
+            if (heading) {
+                node.headingScore = heading.score;
+                node.headingData = JSON.stringify(heading);
+            }
+        }
+        if (node.html) {
+            const soft404 = soft404Service.analyze(node.html, node.outLinks);
+            node.soft404Score = soft404.score;
+            const $ = load(node.html);
+            const content = analyzeContent($);
+            node.wordCount = content.wordCount;
+        }
+    }
     emit({ type: 'metrics:start', phase: 'Updating metrics in DB' });
-    const nodes = graph.getNodes();
     // Pre-fetch all page IDs to avoid N+1 queries
-    // Use getPagesIdentityBySnapshot to avoid loading full page content (HTML) into memory again
-    const pages = pageRepo.getPagesIdentityBySnapshot(snapshotId);
+    const pagesIdentity = pageRepo.getPagesIdentityBySnapshot(snapshotId);
     const urlToId = new Map();
-    for (const p of pages) {
+    for (const p of pagesIdentity) {
         urlToId.set(p.normalized_url, p.id);
     }
-    const clusterStmt = db.prepare(`
-        INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
-        VALUES (?, ?, ?, ?, ?, ?)
-    `);
-    const contentStmt = db.prepare(`
-        INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
-        VALUES (?, ?, ?, ?, ?, ?)
-    `);
+    const metricsToSave = graph.getNodes().map(node => {
+        const pageId = urlToId.get(node.url);
+        if (!pageId)
+            return null;
+        return {
+            snapshot_id: snapshotId,
+            page_id: pageId,
+            crawl_status: node.crawlStatus ?? null,
+            word_count: node.wordCount ?? null,
+            thin_content_score: node.thinContentScore ?? null,
+            external_link_ratio: node.externalLinkRatio ?? null,
+            pagerank_score: node.pagerankScore ?? null,
+            hub_score: node.hubScore ?? null,
+            auth_score: node.authScore ?? null,
+            link_role: node.linkRole ?? null,
+            duplicate_cluster_id: node.duplicateClusterId ?? null,
+            duplicate_type: node.duplicateType ?? null,
+            cluster_id: node.clusterId ?? null,
+            soft404_score: node.soft404Score ?? null,
+            heading_score: node.headingScore ?? null,
+            orphan_score: node.orphanScore ?? null,
+            orphan_type: node.orphanType ?? null,
+            impact_level: node.impactLevel ?? null,
+            heading_data: node.headingData ?? null,
+            is_cluster_primary: node.isClusterPrimary ? 1 : 0
+        };
+    }).filter(m => m !== null);
+    metricsRepo.insertMany(metricsToSave);
+    // Update page-level metadata in transaction
     const tx = db.transaction(() => {
-        for (const node of nodes) {
-            const pageId = urlToId.get(node.url);
-            if (!pageId)
-                continue;
-            metricsRepo.insertMetrics({
-                snapshot_id: snapshotId,
-                page_id: pageId,
-                authority_score: node.authorityScore ?? null,
-                hub_score: node.hubScore ?? null,
-                pagerank: node.pageRank ?? null,
-                pagerank_score: node.pageRankScore ?? null,
-                link_role: node.linkRole ?? null,
-                crawl_status: node.crawlStatus ?? null,
-                word_count: node.wordCount ?? null,
-                thin_content_score: node.thinContentScore ?? null,
-                external_link_ratio: node.externalLinkRatio ?? null,
-                orphan_score: node.orphanScore ?? null,
-                duplicate_cluster_id: node.duplicateClusterId ?? null,
-                duplicate_type: node.duplicateType ?? null,
-                is_cluster_primary: node.isClusterPrimary ? 1 : 0
-            });
-            // Update page-level crawl trap data
+        for (const node of graph.getNodes()) {
             if (node.crawlTrapFlag || node.redirectChain?.length || node.bytesReceived) {
                 pageRepo.upsertPage({
                     site_id: snapshot.site_id,
@@ -90,28 +199,40 @@ export function runPostCrawlMetrics(snapshotId, maxDepth, context, limitReached
                 });
             }
         }
-        // Save duplicate clusters
-        for (const cluster of graph.duplicateClusters) {
-            clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
-        }
-        // Save content clusters
-        for (const cluster of graph.contentClusters) {
-            contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
-        }
     });
     tx();
     emit({ type: 'metrics:start', phase: 'Computing aggregate stats' });
     const metrics = calculateMetrics(graph, maxDepth);
-    // Calculate penalty-based health score (matches CLI)
-    const issues = collectCrawlIssues(graph, metrics);
-    const health = calculateHealthScore(metrics.totalPages, issues);
+    // Compute health score if enabled
+    let healthScore = null;
+    if (options.health) {
+        try {
+            const rootOrigin = options.rootOrigin ?? '';
+            const healthService = new HealthService();
+            const issues = healthService.collectCrawlIssues(graph, metrics, rootOrigin);
+            const breakdown = healthService.calculateHealthScore(metrics.totalPages, issues);
+            healthScore = breakdown.score;
+        }
+        catch (e) {
+            emit({ type: 'error', message: 'Error computing health score', error: e });
+        }
+    }
+    const thinContentCount = graph.getNodes().filter(n => n.wordCount !== undefined && n.wordCount < 200 && n.status === 200).length;
+    const orphanCount = metrics.orphanPages.length;
     snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
         node_count: metrics.totalPages,
         edge_count: metrics.totalEdges,
-        health_score: health.score,
-        orphan_count: issues.orphanPages,
-        thin_content_count: issues.thinContent,
-        limit_reached: limitReached ? 1 : 0
+        limit_reached: limitReached ? 1 : 0,
+        thin_content_count: thinContentCount,
+        orphan_count: orphanCount,
+        ...(healthScore !== null ? { health_score: healthScore } : {})
     });
     emit({ type: 'metrics:complete', durationMs: 0 });
+    return {
+        metrics,
+        healthData: healthScore !== null ? {
+            health: new HealthService().calculateHealthScore(metrics.totalPages, new HealthService().collectCrawlIssues(graph, metrics, options.rootOrigin ?? '')),
+            issues: new HealthService().collectCrawlIssues(graph, metrics, options.rootOrigin ?? '')
+        } : undefined
+    };
 }

package/dist/crawler/normalize.d.ts

@@ -3,5 +3,46 @@
  */
 export interface NormalizeOptions {
     stripQuery?: boolean;
+    toPath?: boolean;
 }
 export declare function normalizeUrl(input: string, base: string, options?: NormalizeOptions): string | null;
+/**
+ * Utility for converting between absolute URLs and relative paths
+ * primarily used for database storage.
+ */
+export declare class UrlUtil {
+    /**
+     * Extract a stable domain key from a URL/domain input.
+     * Examples:
+     *  - "https://www.example.com/a" -> "example.com"
+     *  - "example.com" -> "example.com"
+     */
+    static extractDomain(input: string): string;
+    /**
+     * Resolve a site's absolute origin from persisted site fields.
+     */
+    static resolveSiteOrigin(site: {
+        domain: string;
+        preferred_url?: string | null;
+        ssl?: number | null;
+    }): string;
+    /**
+     * Converts a full URL to a root-relative path if it matches the origin.
+     * If it doesn't match the origin, it's considered external and kept absolute.
+     */
+    static toPath(urlStr: string, origin: string): string;
+    /**
+     * Converts a root-relative path back to an absolute URL relative to the origin.
+     * If the input is already an absolute URL, it is returned as-is.
+     */
+    static toAbsolute(pathOrUrl: string, origin: string): string;
+    /**
+     * Determines if a URL (or path) is internal relative to the origin.
+     */
+    static isInternal(pathOrUrl: string, origin: string): boolean;
+    /**
+     * Build normalized lookup candidates for querying pages table.
+     * Returns path/absolute/original variants in priority order, deduplicated.
+     */
+    static toLookupCandidates(input: string, origin: string): string[];
+}

package/dist/crawler/normalize.js

@@ -10,7 +10,7 @@ const TRACKING_PARAMS = new Set([
 ]);
 const SKIP_EXTENSIONS = new Set([
     '.pdf', '.jpg', '.png', '.svg', '.webp', '.gif',
-    '.zip', '.xml', '.json', '.mp4'
+    '.zip', '.xml', '.json', '.mp4', '.avif', '.ics'
 ]);
 export function normalizeUrl(input, base, options = {}) {
     try {
@@ -71,6 +71,7 @@ export function normalizeUrl(input, base, options = {}) {
             pathname = pathname.slice(0, -1);
         }
         u.pathname = pathname;
+        const finalUrl = u.toString();
         // 9. Skip non-HTML assets by extension
         const lastDotIndex = u.pathname.lastIndexOf('.');
         if (lastDotIndex !== -1) {
@@ -79,10 +80,125 @@ export function normalizeUrl(input, base, options = {}) {
                 return null;
             }
         }
-        // 10. Return final string
-        return u.toString();
+        // 10. Return path if requested
+        if (options.toPath) {
+            return u.pathname + u.search;
+        }
+        // 11. Return final string
+        return finalUrl;
     }
     catch (_e) {
         return null;
     }
 }
+/**
+ * Utility for converting between absolute URLs and relative paths
+ * primarily used for database storage.
+ */
+export class UrlUtil {
+    /**
+     * Extract a stable domain key from a URL/domain input.
+     * Examples:
+     *  - "https://www.example.com/a" -> "example.com"
+     *  - "example.com" -> "example.com"
+     */
+    static extractDomain(input) {
+        const trimmed = input.trim();
+        if (!trimmed)
+            return '';
+        try {
+            const direct = new URL(trimmed);
+            return direct.hostname.toLowerCase().replace(/^www\./, '');
+        }
+        catch {
+            // fall through
+        }
+        try {
+            const withProtocol = new URL(`https://${trimmed}`);
+            return withProtocol.hostname.toLowerCase().replace(/^www\./, '');
+        }
+        catch {
+            return trimmed.toLowerCase().replace(/^www\./, '');
+        }
+    }
+    /**
+     * Resolve a site's absolute origin from persisted site fields.
+     */
+    static resolveSiteOrigin(site) {
+        if (site.preferred_url) {
+            try {
+                return new URL(site.preferred_url).origin;
+            }
+            catch {
+                // fall through to domain+ssl fallback
+            }
+        }
+        const protocol = site.ssl === 0 ? 'http' : 'https';
+        return `${protocol}://${site.domain}`;
+    }
+    /**
+     * Converts a full URL to a root-relative path if it matches the origin.
+     * If it doesn't match the origin, it's considered external and kept absolute.
+     */
+    static toPath(urlStr, origin) {
+        try {
+            const url = new URL(urlStr);
+            const originUrl = new URL(origin);
+            if (url.origin === originUrl.origin) {
+                return url.pathname + url.search;
+            }
+            return urlStr;
+        }
+        catch {
+            return urlStr;
+        }
+    }
+    /**
+     * Converts a root-relative path back to an absolute URL relative to the origin.
+     * If the input is already an absolute URL, it is returned as-is.
+     */
+    static toAbsolute(pathOrUrl, origin) {
+        if (pathOrUrl.startsWith('http://') || pathOrUrl.startsWith('https://')) {
+            return pathOrUrl;
+        }
+        try {
+            return new URL(pathOrUrl, origin).toString();
+        }
+        catch {
+            return pathOrUrl;
+        }
+    }
+    /**
+     * Determines if a URL (or path) is internal relative to the origin.
+     */
+    static isInternal(pathOrUrl, origin) {
+        if (!pathOrUrl.startsWith('http'))
+            return true;
+        try {
+            const url = new URL(pathOrUrl);
+            const originUrl = new URL(origin);
+            return url.origin === originUrl.origin;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Build normalized lookup candidates for querying pages table.
+     * Returns path/absolute/original variants in priority order, deduplicated.
+     */
+    static toLookupCandidates(input, origin) {
+        const candidates = new Set();
+        const raw = input.trim();
+        if (!raw)
+            return [];
+        const absolute = normalizeUrl(raw, origin, { stripQuery: false }) || UrlUtil.toAbsolute(raw, origin);
+        const path = normalizeUrl(raw, origin, { stripQuery: false, toPath: true }) || UrlUtil.toPath(raw, origin);
+        const absolutePath = normalizeUrl(absolute, '', { stripQuery: false, toPath: true }) || UrlUtil.toPath(absolute, origin);
+        candidates.add(path);
+        candidates.add(absolute);
+        candidates.add(absolutePath);
+        candidates.add(raw);
+        return Array.from(candidates).filter(Boolean);
+    }
+}

package/dist/crawler/parser.d.ts

@@ -11,12 +11,10 @@ export interface ParseResult {
     contentHash: string;
     simhash?: string;
     uniqueTokenRatio?: number;
-    soft404Score: number;
-    soft404Signals: string[];
 }
 export declare class Parser {
     /**
      * Parses HTML content to extract metadata and links.
      */
-    parse(html: string, baseUrl: string, status: number): ParseResult;
+    parse(html: string, baseUrl: string, _status: number): ParseResult;
 }

package/dist/crawler/parser.js

@@ -6,7 +6,7 @@ export class Parser {
     /**
      * Parses HTML content to extract metadata and links.
      */
-    parse(html, baseUrl, status) {
+    parse(html, baseUrl, _status) {
         const $ = cheerio.load(html);
         // 1. Robots Meta
         let noindex = false;
@@ -97,51 +97,6 @@ export class Parser {
         const uniqueTokens = new Set(tokens);
         const uniqueTokenRatio = tokens.length > 0 ? (uniqueTokens.size / tokens.length) : 0;
         const simhash = SimHash.generate(tokens).toString();
-        // 5. Soft 404 Detection
-        let soft404Score = 0;
-        const soft404Signals = [];
-        if (status === 200) {
-            const title = $('title').text().toLowerCase();
-            const h1Text = $('h1').first().text().toLowerCase();
-            const bodyText = cleanText.toLowerCase();
-            const errorPatterns = ['404', 'not found', 'error', 'doesn\'t exist', 'unavailable', 'invalid'];
-            // Pattern checks
-            for (const pattern of errorPatterns) {
-                if (title.includes(pattern)) {
-                    soft404Score += 0.4;
-                    soft404Signals.push(`title_pattern_${pattern.replace(/\s+/g, '_')}`);
-                    break;
-                }
-            }
-            for (const pattern of errorPatterns) {
-                if (h1Text.includes(pattern)) {
-                    soft404Score += 0.3;
-                    soft404Signals.push(`h1_pattern_${pattern.replace(/\s+/g, '_')}`);
-                    break;
-                }
-            }
-            if (bodyText.includes('page not found') || bodyText.includes('404 error')) {
-                soft404Score += 0.2;
-                soft404Signals.push('body_error_phrase');
-            }
-            // Content length check (Word count approximation)
-            const words = cleanText.split(/\s+/).filter(w => w.length > 0);
-            if (words.length < 50) {
-                soft404Score += 0.3;
-                soft404Signals.push('very_low_word_count');
-            }
-            else if (words.length < 150) {
-                soft404Score += 0.1;
-                soft404Signals.push('low_word_count');
-            }
-            // Link count check
-            if (links.size === 0) {
-                soft404Score += 0.2;
-                soft404Signals.push('no_outbound_links');
-            }
-            // Cap at 1.0
-            soft404Score = Math.min(1.0, soft404Score);
-        }
         return {
             links: Array.from(links.entries()).map(([url, weight]) => ({ url, weight })),
             html: html, // pass raw HTML for analysis
@@ -150,9 +105,7 @@ export class Parser {
             nofollow,
             contentHash,
             simhash,
-            uniqueTokenRatio,
-            soft404Score,
-            soft404Signals
+            uniqueTokenRatio
         };
     }
 }

package/dist/crawler/resolver.d.ts

@@ -0,0 +1,11 @@
+import { Fetcher } from './fetcher.js';
+import { Site } from '../db/repositories/SiteRepository.js';
+export interface ResolvedUrl {
+    url: string;
+    site: Site;
+}
+export declare class UrlResolver {
+    private siteRepo;
+    constructor();
+    resolve(inputUrl: string, fetcher: Fetcher): Promise<ResolvedUrl>;
+}

package/dist/crawler/resolver.js

@@ -0,0 +1,67 @@
+import { SiteRepository } from '../db/repositories/SiteRepository.js';
+import { getDb } from '../db/index.js';
+export class UrlResolver {
+    siteRepo;
+    constructor() {
+        this.siteRepo = new SiteRepository(getDb());
+    }
+    async resolve(inputUrl, fetcher) {
+        const hasProtocol = inputUrl.startsWith('http://') || inputUrl.startsWith('https://');
+        const workingUrl = hasProtocol ? inputUrl : `https://${inputUrl}`;
+        let hostname;
+        try {
+            hostname = new URL(workingUrl).hostname;
+        }
+        catch {
+            throw new Error(`Invalid URL or domain: ${inputUrl}`);
+        }
+        const domain = hostname.replace(/^www\./, '');
+        let site = this.siteRepo.firstOrCreateSite(domain);
+        // If protocol was omitted, we use our discovery logic or stored preference
+        if (!hasProtocol) {
+            if (site.ssl !== null && site.preferred_url) {
+                return {
+                    url: site.preferred_url,
+                    site
+                };
+            }
+            // No protocol provided and no stored preference: Probe HTTPS first
+            try {
+                const res = await fetcher.fetch(`https://${hostname}/`);
+                if (typeof res.status === 'number' && res.status >= 200 && res.status < 400) {
+                    const isSsl = res.finalUrl.startsWith('https:') ? 1 : 0;
+                    this.siteRepo.updateSitePreference(site.id, { preferred_url: res.finalUrl, ssl: isSsl });
+                    // Refresh site object
+                    site = this.siteRepo.getSiteById(site.id);
+                    return { url: res.finalUrl, site };
+                }
+            }
+            catch {
+                // Fallback to HTTP
+            }
+            // Try HTTP
+            try {
+                const res = await fetcher.fetch(`http://${hostname}/`);
+                if (typeof res.status === 'number' && res.status >= 200 && res.status < 400) {
+                    const isSsl = res.finalUrl.startsWith('https:') ? 1 : 0;
+                    this.siteRepo.updateSitePreference(site.id, { preferred_url: res.finalUrl, ssl: isSsl });
+                    site = this.siteRepo.getSiteById(site.id);
+                    return { url: res.finalUrl, site };
+                }
+            }
+            catch {
+                // If both fail, we still default to the provided input as https
+                return { url: workingUrl, site };
+            }
+        }
+        // Protocol was provided, we just return it but ensure site is in sync if it's the first time
+        if (site.ssl === null) {
+            this.siteRepo.updateSitePreference(site.id, {
+                preferred_url: inputUrl,
+                ssl: inputUrl.startsWith('https:') ? 1 : 0
+            });
+            site = this.siteRepo.getSiteById(site.id);
+        }
+        return { url: inputUrl, site };
+    }
+}