npm - @crawlith/core - Versions diffs - 0.1.0 → 0.1.2 - Mend

@crawlith/core 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (238) hide show

package/LICENSE +201 -0
package/README.md +70 -0
package/dist/analysis/analysis_list.html +35 -0
package/dist/analysis/analysis_page.html +123 -0
package/dist/analysis/analyze.d.ts +40 -5
package/dist/analysis/analyze.js +395 -347
package/dist/analysis/clustering.d.ts +23 -0
package/dist/analysis/clustering.js +206 -0
package/dist/analysis/content.d.ts +1 -1
package/dist/analysis/content.js +11 -5
package/dist/analysis/duplicate.d.ts +34 -0
package/dist/analysis/duplicate.js +305 -0
package/dist/analysis/heading.d.ts +116 -0
package/dist/analysis/heading.js +356 -0
package/dist/analysis/images.d.ts +1 -1
package/dist/analysis/images.js +6 -5
package/dist/analysis/links.d.ts +1 -1
package/dist/analysis/links.js +8 -8
package/dist/{scoring/orphanSeverity.d.ts → analysis/orphan.d.ts} +12 -23
package/dist/{scoring/orphanSeverity.js → analysis/orphan.js} +9 -3
package/dist/analysis/scoring.js +11 -2
package/dist/analysis/seo.d.ts +8 -4
package/dist/analysis/seo.js +41 -30
package/dist/analysis/soft404.d.ts +17 -0
package/dist/analysis/soft404.js +62 -0
package/dist/analysis/structuredData.d.ts +1 -1
package/dist/analysis/structuredData.js +5 -4
package/dist/analysis/templates.d.ts +2 -0
package/dist/analysis/templates.js +7 -0
package/dist/application/index.d.ts +2 -0
package/dist/application/index.js +2 -0
package/dist/application/usecase.d.ts +3 -0
package/dist/application/usecase.js +1 -0
package/dist/application/usecases.d.ts +114 -0
package/dist/application/usecases.js +201 -0
package/dist/audit/index.js +1 -1
package/dist/audit/transport.d.ts +1 -1
package/dist/audit/transport.js +5 -4
package/dist/audit/types.d.ts +1 -0
package/dist/constants.d.ts +17 -0
package/dist/constants.js +23 -0
package/dist/core/scope/scopeManager.js +3 -0
package/dist/core/security/ipGuard.d.ts +11 -0
package/dist/core/security/ipGuard.js +71 -3
package/dist/crawler/crawl.d.ts +4 -22
package/dist/crawler/crawl.js +4 -335
package/dist/crawler/crawler.d.ts +87 -0
package/dist/crawler/crawler.js +683 -0
package/dist/crawler/extract.d.ts +4 -1
package/dist/crawler/extract.js +7 -2
package/dist/crawler/fetcher.d.ts +2 -1
package/dist/crawler/fetcher.js +26 -11
package/dist/crawler/metricsRunner.d.ts +23 -1
package/dist/crawler/metricsRunner.js +202 -72
package/dist/crawler/normalize.d.ts +41 -0
package/dist/crawler/normalize.js +119 -3
package/dist/crawler/parser.d.ts +1 -3
package/dist/crawler/parser.js +2 -49
package/dist/crawler/resolver.d.ts +11 -0
package/dist/crawler/resolver.js +67 -0
package/dist/crawler/sitemap.d.ts +6 -0
package/dist/crawler/sitemap.js +27 -17
package/dist/crawler/trap.d.ts +5 -1
package/dist/crawler/trap.js +23 -2
package/dist/db/CrawlithDB.d.ts +110 -0
package/dist/db/CrawlithDB.js +500 -0
package/dist/db/graphLoader.js +42 -30
package/dist/db/index.d.ts +11 -0
package/dist/db/index.js +41 -29
package/dist/db/migrations.d.ts +2 -0
package/dist/db/{schema.js → migrations.js} +90 -43
package/dist/db/pluginRegistry.d.ts +9 -0
package/dist/db/pluginRegistry.js +19 -0
package/dist/db/repositories/EdgeRepository.d.ts +13 -0
package/dist/db/repositories/EdgeRepository.js +20 -0
package/dist/db/repositories/MetricsRepository.d.ts +16 -8
package/dist/db/repositories/MetricsRepository.js +28 -7
package/dist/db/repositories/PageRepository.d.ts +15 -2
package/dist/db/repositories/PageRepository.js +169 -25
package/dist/db/repositories/SiteRepository.d.ts +9 -0
package/dist/db/repositories/SiteRepository.js +13 -0
package/dist/db/repositories/SnapshotRepository.d.ts +14 -5
package/dist/db/repositories/SnapshotRepository.js +64 -5
package/dist/db/reset.d.ts +9 -0
package/dist/db/reset.js +32 -0
package/dist/db/statements.d.ts +12 -0
package/dist/db/statements.js +40 -0
package/dist/diff/compare.d.ts +0 -5
package/dist/diff/compare.js +0 -12
package/dist/diff/service.d.ts +16 -0
package/dist/diff/service.js +41 -0
package/dist/domain/index.d.ts +4 -0
package/dist/domain/index.js +4 -0
package/dist/events.d.ts +56 -0
package/dist/events.js +1 -0
package/dist/graph/graph.d.ts +36 -42
package/dist/graph/graph.js +26 -17
package/dist/graph/hits.d.ts +23 -0
package/dist/graph/hits.js +111 -0
package/dist/graph/metrics.d.ts +0 -4
package/dist/graph/metrics.js +25 -9
package/dist/graph/pagerank.d.ts +17 -4
package/dist/graph/pagerank.js +126 -91
package/dist/graph/simhash.d.ts +6 -0
package/dist/graph/simhash.js +14 -0
package/dist/index.d.ts +29 -8
package/dist/index.js +29 -8
package/dist/lock/hashKey.js +1 -1
package/dist/lock/lockManager.d.ts +5 -1
package/dist/lock/lockManager.js +38 -13
package/dist/plugin-system/plugin-cli.d.ts +10 -0
package/dist/plugin-system/plugin-cli.js +31 -0
package/dist/plugin-system/plugin-config.d.ts +16 -0
package/dist/plugin-system/plugin-config.js +36 -0
package/dist/plugin-system/plugin-loader.d.ts +17 -0
package/dist/plugin-system/plugin-loader.js +122 -0
package/dist/plugin-system/plugin-registry.d.ts +25 -0
package/dist/plugin-system/plugin-registry.js +167 -0
package/dist/plugin-system/plugin-types.d.ts +205 -0
package/dist/plugin-system/plugin-types.js +1 -0
package/dist/ports/index.d.ts +9 -0
package/dist/ports/index.js +1 -0
package/{src/report/sitegraph_template.ts → dist/report/crawl.html} +330 -81
package/dist/report/crawlExport.d.ts +3 -0
package/dist/report/{sitegraphExport.js → crawlExport.js} +3 -3
package/dist/report/crawl_template.d.ts +1 -0
package/dist/report/crawl_template.js +7 -0
package/dist/report/export.d.ts +3 -0
package/dist/report/export.js +81 -0
package/dist/report/html.js +15 -216
package/dist/report/insight.d.ts +27 -0
package/dist/report/insight.js +103 -0
package/dist/scoring/health.d.ts +56 -0
package/dist/scoring/health.js +213 -0
package/dist/utils/chalk.d.ts +6 -0
package/dist/utils/chalk.js +41 -0
package/dist/utils/secureConfig.d.ts +23 -0
package/dist/utils/secureConfig.js +128 -0
package/package.json +12 -6
package/CHANGELOG.md +0 -7
package/dist/db/schema.d.ts +0 -2
package/dist/graph/cluster.d.ts +0 -6
package/dist/graph/cluster.js +0 -173
package/dist/graph/duplicate.d.ts +0 -10
package/dist/graph/duplicate.js +0 -251
package/dist/report/sitegraphExport.d.ts +0 -3
package/dist/report/sitegraph_template.d.ts +0 -1
package/dist/report/sitegraph_template.js +0 -630
package/dist/scoring/hits.d.ts +0 -9
package/dist/scoring/hits.js +0 -111
package/src/analysis/analyze.ts +0 -548
package/src/analysis/content.ts +0 -62
package/src/analysis/images.ts +0 -28
package/src/analysis/links.ts +0 -41
package/src/analysis/scoring.ts +0 -59
package/src/analysis/seo.ts +0 -82
package/src/analysis/structuredData.ts +0 -62
package/src/audit/dns.ts +0 -49
package/src/audit/headers.ts +0 -98
package/src/audit/index.ts +0 -66
package/src/audit/scoring.ts +0 -232
package/src/audit/transport.ts +0 -258
package/src/audit/types.ts +0 -102
package/src/core/network/proxyAdapter.ts +0 -21
package/src/core/network/rateLimiter.ts +0 -39
package/src/core/network/redirectController.ts +0 -47
package/src/core/network/responseLimiter.ts +0 -34
package/src/core/network/retryPolicy.ts +0 -57
package/src/core/scope/domainFilter.ts +0 -45
package/src/core/scope/scopeManager.ts +0 -52
package/src/core/scope/subdomainPolicy.ts +0 -39
package/src/core/security/ipGuard.ts +0 -92
package/src/crawler/crawl.ts +0 -382
package/src/crawler/extract.ts +0 -34
package/src/crawler/fetcher.ts +0 -233
package/src/crawler/metricsRunner.ts +0 -124
package/src/crawler/normalize.ts +0 -108
package/src/crawler/parser.ts +0 -190
package/src/crawler/sitemap.ts +0 -73
package/src/crawler/trap.ts +0 -96
package/src/db/graphLoader.ts +0 -105
package/src/db/index.ts +0 -70
package/src/db/repositories/EdgeRepository.ts +0 -29
package/src/db/repositories/MetricsRepository.ts +0 -49
package/src/db/repositories/PageRepository.ts +0 -128
package/src/db/repositories/SiteRepository.ts +0 -32
package/src/db/repositories/SnapshotRepository.ts +0 -74
package/src/db/schema.ts +0 -177
package/src/diff/compare.ts +0 -84
package/src/graph/cluster.ts +0 -192
package/src/graph/duplicate.ts +0 -286
package/src/graph/graph.ts +0 -172
package/src/graph/metrics.ts +0 -110
package/src/graph/pagerank.ts +0 -125
package/src/graph/simhash.ts +0 -61
package/src/index.ts +0 -30
package/src/lock/hashKey.ts +0 -51
package/src/lock/lockManager.ts +0 -124
package/src/lock/pidCheck.ts +0 -13
package/src/report/html.ts +0 -227
package/src/report/sitegraphExport.ts +0 -58
package/src/scoring/hits.ts +0 -131
package/src/scoring/orphanSeverity.ts +0 -176
package/src/utils/version.ts +0 -18
package/tests/__snapshots__/orphanSeverity.test.ts.snap +0 -49
package/tests/analysis.unit.test.ts +0 -98
package/tests/analyze.integration.test.ts +0 -98
package/tests/audit/dns.test.ts +0 -31
package/tests/audit/headers.test.ts +0 -45
package/tests/audit/scoring.test.ts +0 -133
package/tests/audit/security.test.ts +0 -12
package/tests/audit/transport.test.ts +0 -112
package/tests/clustering.test.ts +0 -118
package/tests/crawler.test.ts +0 -358
package/tests/db.test.ts +0 -159
package/tests/diff.test.ts +0 -67
package/tests/duplicate.test.ts +0 -110
package/tests/fetcher.test.ts +0 -106
package/tests/fetcher_safety.test.ts +0 -85
package/tests/fixtures/analyze-crawl.json +0 -26
package/tests/hits.test.ts +0 -134
package/tests/html_report.test.ts +0 -58
package/tests/lock/lockManager.test.ts +0 -138
package/tests/metrics.test.ts +0 -196
package/tests/normalize.test.ts +0 -101
package/tests/orphanSeverity.test.ts +0 -160
package/tests/pagerank.test.ts +0 -98
package/tests/parser.test.ts +0 -117
package/tests/proxy_safety.test.ts +0 -57
package/tests/redirect_safety.test.ts +0 -73
package/tests/safety.test.ts +0 -114
package/tests/scope.test.ts +0 -66
package/tests/scoring.test.ts +0 -59
package/tests/sitemap.test.ts +0 -88
package/tests/soft404.test.ts +0 -41
package/tests/trap.test.ts +0 -39
package/tests/visualization_data.test.ts +0 -46
package/tsconfig.json +0 -11

package/dist/analysis/seo.js CHANGED Viewed

@@ -2,9 +2,10 @@ import { load } from 'cheerio';
 function normalizedText(value) {
     return (value ?? '').trim().toLowerCase();
 }
-export function analyzeTitle(html) {
-    const $ = load(html);
-    const title = $('title').first().text().trim();
+export function analyzeTitle($) {
+    const isString = typeof $ === 'string';
+    const cheerioObj = isString ? load($ || '<html></html>') : $;
+    const title = cheerioObj('title').first().text().trim();
     if (!title) {
         return { value: null, length: 0, status: 'missing' };
     }
@@ -14,9 +15,10 @@ export function analyzeTitle(html) {
         return { value: title, length: title.length, status: 'too_long' };
     return { value: title, length: title.length, status: 'ok' };
 }
-export function analyzeMetaDescription(html) {
-    const $ = load(html);
-    const raw = $('meta[name="description"]').attr('content');
+export function analyzeMetaDescription($) {
+    const isString = typeof $ === 'string';
+    const cheerioObj = isString ? load($ || '<html></html>') : $;
+    const raw = cheerioObj('meta[name="description"]').attr('content');
     if (raw === undefined) {
         return { value: null, length: 0, status: 'missing' };
     }
@@ -30,35 +32,44 @@ export function analyzeMetaDescription(html) {
         return { value: description, length: description.length, status: 'too_long' };
     return { value: description, length: description.length, status: 'ok' };
 }
-export function applyDuplicateStatuses(fields) {
-    const counts = new Map();
-    for (const field of fields) {
-        const key = normalizedText(field.value);
-        if (!key)
-            continue;
-        counts.set(key, (counts.get(key) || 0) + 1);
-    }
-    return fields.map((field) => {
-        const key = normalizedText(field.value);
-        if (!key)
-            return field;
-        if ((counts.get(key) || 0) > 1) {
-            return { ...field, status: 'duplicate' };
-        }
-        return field;
-    });
-}
-export function analyzeH1(html, titleValue) {
-    const $ = load(html);
-    const h1Values = $('h1').toArray().map((el) => $(el).text().trim()).filter(Boolean);
+export function analyzeH1($, titleValue) {
+    const isString = typeof $ === 'string';
+    const cheerioObj = isString ? load($ || '<html></html>') : $;
+    const h1Values = cheerioObj('h1').toArray().map((el) => cheerioObj(el).text().trim()).filter(Boolean);
     const count = h1Values.length;
     const first = h1Values[0] || null;
     const matchesTitle = Boolean(first && titleValue && normalizedText(first) === normalizedText(titleValue));
     if (count === 0) {
-        return { count, status: 'critical', matchesTitle };
+        return { count, status: 'critical', matchesTitle, value: null };
     }
     if (count > 1) {
-        return { count, status: 'warning', matchesTitle };
+        return { count, status: 'warning', matchesTitle, value: first };
     }
-    return { count, status: 'ok', matchesTitle };
+    return { count, status: 'ok', matchesTitle, value: first };
+}
+export function applyDuplicateStatuses(items) {
+    const counts = new Map();
+    const normalizedToOriginal = new Map();
+    // First pass: count occurrences of each normalized value
+    for (const item of items) {
+        if (item.value) {
+            const normalized = normalizedText(item.value);
+            if (normalized) {
+                counts.set(normalized, (counts.get(normalized) || 0) + 1);
+                if (!normalizedToOriginal.has(normalized)) {
+                    normalizedToOriginal.set(normalized, item.value);
+                }
+            }
+        }
+    }
+    // Second pass: apply duplicate status
+    return items.map(item => {
+        if (item.value) {
+            const normalized = normalizedText(item.value);
+            if ((counts.get(normalized) || 0) > 1) {
+                return { ...item, status: 'duplicate' };
+            }
+        }
+        return item;
+    });
 }

package/dist/analysis/soft404.d.ts ADDED Viewed

@@ -0,0 +1,17 @@
+export interface Soft404Result {
+    score: number;
+    reason: string;
+}
+/**
+ * Service to analyze HTML content for soft 404 signals.
+ * Extracts signals from title, H1, body content length, and outlinks.
+ */
+export declare class Soft404Service {
+    /**
+     * Analyzes HTML string to determine probability of being a soft 404 page.
+     * @param {string | undefined} html - Raw HTML source code.
+     * @param {number} outLinks - Total number of outbound links extracted during parsing.
+     * @returns {Soft404Result} A calculated score between 0.0 and 1.0, and the matched reasons.
+     */
+    analyze(html: string | undefined, outLinks: number): Soft404Result;
+}

package/dist/analysis/soft404.js ADDED Viewed

@@ -0,0 +1,62 @@
+import * as cheerio from 'cheerio';
+/**
+ * Service to analyze HTML content for soft 404 signals.
+ * Extracts signals from title, H1, body content length, and outlinks.
+ */
+export class Soft404Service {
+    /**
+     * Analyzes HTML string to determine probability of being a soft 404 page.
+     * @param {string | undefined} html - Raw HTML source code.
+     * @param {number} outLinks - Total number of outbound links extracted during parsing.
+     * @returns {Soft404Result} A calculated score between 0.0 and 1.0, and the matched reasons.
+     */
+    analyze(html, outLinks) {
+        if (!html)
+            return { score: 0, reason: '' };
+        let score = 0;
+        const signals = [];
+        const $ = cheerio.load(html);
+        $('script, style, noscript, iframe').remove();
+        const cleanText = $('body').text().replace(/\s+/g, ' ').trim();
+        const title = $('title').text().toLowerCase();
+        const h1Text = $('h1').first().text().toLowerCase();
+        const bodyText = cleanText.toLowerCase();
+        const errorPatterns = ['404', 'not found', 'error', "doesn't exist", 'unavailable', 'invalid'];
+        for (const pattern of errorPatterns) {
+            if (title.includes(pattern)) {
+                score += 0.4;
+                signals.push(`title_contains_${pattern}`);
+                break;
+            }
+        }
+        for (const pattern of errorPatterns) {
+            if (h1Text.includes(pattern)) {
+                score += 0.3;
+                signals.push(`h1_contains_${pattern}`);
+                break;
+            }
+        }
+        if (bodyText.includes('page not found') || bodyText.includes('404 error')) {
+            score += 0.2;
+            signals.push('body_error_phrase');
+        }
+        const words = cleanText.split(/\s+/).filter(w => w.length > 0);
+        if (words.length < 50) {
+            score += 0.3;
+            signals.push('very_low_word_count');
+        }
+        else if (words.length < 150) {
+            score += 0.1;
+            signals.push('low_word_count');
+        }
+        if (outLinks === 0) {
+            score += 0.2;
+            signals.push('no_outbound_links');
+        }
+        score = Math.min(1.0, score);
+        return {
+            score: Number(score.toFixed(2)),
+            reason: signals.join(', ')
+        };
+    }
+}

package/dist/analysis/structuredData.d.ts CHANGED Viewed

@@ -3,4 +3,4 @@ export interface StructuredDataResult {
     types: string[];
     valid: boolean;
 }
-export declare function analyzeStructuredData(html: string): StructuredDataResult;
+export declare function analyzeStructuredData($: any): StructuredDataResult;

package/dist/analysis/structuredData.js CHANGED Viewed

@@ -1,14 +1,15 @@
 import { load } from 'cheerio';
-export function analyzeStructuredData(html) {
-    const $ = load(html);
-    const scripts = $('script[type="application/ld+json"]').toArray();
+export function analyzeStructuredData($) {
+    const isString = typeof $ === 'string';
+    const cheerioObj = isString ? load($ || '<html></html>') : $;
+    const scripts = cheerioObj('script[type="application/ld+json"]').toArray();
     if (scripts.length === 0) {
         return { present: false, types: [], valid: false };
     }
     const types = new Set();
     let valid = true;
     for (const script of scripts) {
-        const raw = $(script).text().trim();
+        const raw = cheerioObj(script).text().trim();
         if (!raw) {
             valid = false;
             continue;

package/dist/analysis/templates.d.ts ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export declare const ANALYSIS_LIST_TEMPLATE: string;
2	+ export declare const ANALYSIS_PAGE_TEMPLATE: string;

package/dist/analysis/templates.js ADDED Viewed

@@ -0,0 +1,7 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+export const ANALYSIS_LIST_TEMPLATE = fs.readFileSync(path.join(__dirname, 'analysis_list.html'), 'utf-8');
+export const ANALYSIS_PAGE_TEMPLATE = fs.readFileSync(path.join(__dirname, 'analysis_page.html'), 'utf-8');

package/dist/application/index.d.ts ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export * from './usecase.js';
2	+ export * from './usecases.js';

package/dist/application/index.js ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export * from './usecase.js';
2	+ export * from './usecases.js';

package/dist/application/usecase.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+export interface UseCase<I, O> {
+    execute(input: I): Promise<O>;
+}

package/dist/application/usecase.js ADDED Viewed

	@@ -0,0 +1 @@
1	+ export {};

package/dist/application/usecases.d.ts ADDED Viewed

@@ -0,0 +1,114 @@
+import { type AnalyzeOptions, type AnalysisResult } from '../analysis/analyze.js';
+import { compareGraphs } from '../diff/compare.js';
+import type { CrawlithPlugin, PluginContext } from '../plugin-system/plugin-types.js';
+import type { UseCase } from './usecase.js';
+import type { Graph } from '../graph/graph.js';
+import type { EngineContext } from '../events.js';
+export interface CrawlSitegraphResult {
+    snapshotId: number;
+    graph: Graph;
+    metrics?: any;
+    healthData?: any;
+}
+export interface SiteCrawlInput {
+    url: string;
+    limit?: number;
+    depth?: number;
+    concurrency?: number;
+    stripQuery?: boolean;
+    ignoreRobots?: boolean;
+    sitemap?: string | boolean;
+    debug?: boolean;
+    detectSoft404?: boolean;
+    detectTraps?: boolean;
+    rate?: number;
+    maxBytes?: number;
+    allowedDomains?: string[];
+    deniedDomains?: string[];
+    includeSubdomains?: boolean;
+    proxyUrl?: string;
+    maxRedirects?: number;
+    userAgent?: string;
+    clustering?: boolean;
+    clusterThreshold?: number;
+    minClusterSize?: number;
+    heading?: boolean;
+    failOnCritical?: boolean;
+    scoreBreakdown?: boolean;
+    computeHits?: boolean;
+    computePagerank?: boolean;
+    orphans?: boolean;
+    orphanSeverity?: 'low' | 'medium' | 'high';
+    includeSoftOrphans?: boolean;
+    minInbound?: number;
+    plugins?: CrawlithPlugin[];
+    context?: PluginContext;
+}
+export declare class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphResult> {
+    execute(input: SiteCrawlInput): Promise<CrawlSitegraphResult>;
+}
+export declare class AnalyzeSnapshot implements UseCase<{
+    url: string;
+    options: AnalyzeOptions;
+    plugins?: CrawlithPlugin[];
+    context?: PluginContext;
+}, AnalysisResult> {
+    execute(input: {
+        url: string;
+        options: AnalyzeOptions;
+        plugins?: CrawlithPlugin[];
+        context?: PluginContext;
+    }): Promise<AnalysisResult>;
+}
+export interface PageAnalysisInput {
+    url: string;
+    live?: boolean;
+    snapshotId?: number;
+    seo?: boolean;
+    content?: boolean;
+    accessibility?: boolean;
+    rate?: number;
+    proxyUrl?: string;
+    userAgent?: string;
+    maxRedirects?: number;
+    maxBytes?: number;
+    clustering?: boolean;
+    clusterThreshold?: number;
+    minClusterSize?: number;
+    debug?: boolean;
+    allPages?: boolean;
+    sitemap?: string | boolean;
+    heading?: boolean;
+    health?: boolean;
+    failOnCritical?: boolean;
+    scoreBreakdown?: boolean;
+    computeHits?: boolean;
+    computePagerank?: boolean;
+    orphans?: boolean;
+    orphanSeverity?: 'low' | 'medium' | 'high';
+    includeSoftOrphans?: boolean;
+    minInbound?: number;
+    plugins?: CrawlithPlugin[];
+    context?: PluginContext;
+}
+export declare class PageAnalysisUseCase implements UseCase<PageAnalysisInput, AnalysisResult> {
+    private readonly context?;
+    constructor(context?: EngineContext | undefined);
+    execute(input: PageAnalysisInput): Promise<AnalysisResult>;
+}
+export declare class ExportReport implements UseCase<{
+    snapshotId: number;
+}, string> {
+    execute(input: {
+        snapshotId: number;
+    }): Promise<string>;
+}
+export declare class DiffSnapshots implements UseCase<{
+    oldSnapshotId: number;
+    newSnapshotId: number;
+}, ReturnType<typeof compareGraphs>> {
+    execute(input: {
+        oldSnapshotId: number;
+        newSnapshotId: number;
+    }): Promise<import("../diff/compare.js").DiffResult>;
+}

package/dist/application/usecases.js ADDED Viewed

@@ -0,0 +1,201 @@
+import { crawl } from '../crawler/crawl.js';
+import { DEFAULTS } from '../constants.js';
+import { runPostCrawlMetrics } from '../crawler/metricsRunner.js';
+import { analyzeSite } from '../analysis/analyze.js';
+import { loadGraphFromSnapshot } from '../db/graphLoader.js';
+import { compareGraphs } from '../diff/compare.js';
+import { SiteRepository } from '../db/repositories/SiteRepository.js';
+import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
+import { PluginRegistry } from '../plugin-system/plugin-registry.js';
+import { getCrawlithDB } from '../db/index.js';
+import { UrlUtil } from '../crawler/normalize.js';
+export class CrawlSitegraph {
+    async execute(input) {
+        const ctx = input.context ?? { command: 'crawl', scope: 'crawl' };
+        ctx.scope = 'crawl';
+        ctx.db = getCrawlithDB();
+        const registry = new PluginRegistry(input.plugins ?? []);
+        await registry.applyStorage(ctx);
+        await registry.runHook('onInit', ctx);
+        if (ctx.terminate)
+            return { snapshotId: 0, graph: undefined };
+        await registry.runHook('onCrawlStart', ctx);
+        if (ctx.terminate)
+            return { snapshotId: 0, graph: undefined };
+        const policy = (ctx.metadata?.crawlPolicy || {});
+        // Map the unified DTO into the underlying CrawlOptions
+        const crawlOpts = {
+            limit: input.limit ?? DEFAULTS.CRAWL_LIMIT,
+            depth: input.depth ?? DEFAULTS.MAX_DEPTH,
+            concurrency: input.concurrency,
+            stripQuery: input.stripQuery,
+            ignoreRobots: policy.ignoreRobots !== undefined ? policy.ignoreRobots : input.ignoreRobots,
+            sitemap: input.sitemap,
+            debug: input.debug,
+            detectSoft404: input.detectSoft404,
+            detectTraps: input.detectTraps,
+            rate: policy.rate !== undefined ? policy.rate : input.rate,
+            maxBytes: policy.maxBytes !== undefined ? policy.maxBytes : input.maxBytes,
+            allowedDomains: policy.allowedDomains?.length ? policy.allowedDomains : input.allowedDomains,
+            deniedDomains: policy.deniedDomains?.length ? policy.deniedDomains : input.deniedDomains,
+            includeSubdomains: policy.includeSubdomains !== undefined ? policy.includeSubdomains : input.includeSubdomains,
+            proxyUrl: policy.proxyUrl !== undefined ? policy.proxyUrl : input.proxyUrl,
+            maxRedirects: policy.maxRedirects !== undefined ? policy.maxRedirects : input.maxRedirects,
+            userAgent: policy.userAgent !== undefined ? policy.userAgent : input.userAgent,
+            registry: registry
+        };
+        // Build an EngineContext from the plugin context's emit so per-page logs reach OutputController
+        const engineContext = ctx.emit
+            ? { emit: (e) => ctx.emit(e) }
+            : undefined;
+        const snapshotId = await crawl(input.url, crawlOpts, engineContext);
+        const graph = loadGraphFromSnapshot(snapshotId);
+        // Ensure plugin hooks that persist data are scoped to the created snapshot.
+        ctx.snapshotId = snapshotId;
+        await registry.runHook('onGraphBuilt', ctx, graph);
+        await registry.runHook('onMetrics', ctx, graph);
+        const db = getCrawlithDB().unsafeGetRawDb();
+        const siteRepo = new SiteRepository(db);
+        const snapshotRepo = new SnapshotRepository(db);
+        const snapshot = snapshotRepo.getSnapshot(snapshotId);
+        let resolvedOrigin = '';
+        if (snapshot) {
+            const site = siteRepo.getSiteById(snapshot.site_id);
+            if (site?.preferred_url) {
+                try {
+                    resolvedOrigin = new URL(site.preferred_url).origin;
+                }
+                catch { /* ignore */ }
+            }
+        }
+        const result = runPostCrawlMetrics(snapshotId, crawlOpts.depth, {
+            context: undefined,
+            limitReached: false,
+            graphInstance: graph,
+            clustering: input.clustering ?? true,
+            clusterThreshold: input.clusterThreshold,
+            minClusterSize: input.minClusterSize,
+            heading: input.heading ?? true,
+            // Always compute and persist health score for full crawls.
+            health: true,
+            computeHits: input.computeHits ?? true,
+            computePagerank: input.computePagerank ?? true,
+            orphans: input.orphans ?? true,
+            orphanSeverity: input.orphanSeverity ?? true,
+            includeSoftOrphans: input.includeSoftOrphans ?? true,
+            minInbound: input.minInbound,
+            rootOrigin: resolvedOrigin || (input.url.startsWith('http://') || input.url.startsWith('https://')
+                ? new URL(input.url).origin
+                : `https://${UrlUtil.extractDomain(input.url)}`)
+        });
+        const metrics = result?.metrics;
+        const healthData = result?.healthData;
+        if (ctx.db) {
+            ctx.db.aggregateScoreProviders(snapshotId, registry.pluginsList);
+        }
+        await registry.runHook('onReport', ctx, { snapshotId, graph, metrics, healthData });
+        return { snapshotId, graph, metrics, healthData };
+    }
+}
+export class AnalyzeSnapshot {
+    async execute(input) {
+        const result = await analyzeSite(input.url, { ...input.options, live: false });
+        if (input.plugins && input.plugins.length > 0) {
+            const registry = new PluginRegistry(input.plugins);
+            const ctx = {
+                command: 'analyze',
+                scope: 'crawl',
+                ...(input.context || {}),
+                db: getCrawlithDB()
+            };
+            await registry.runHook('onInit', ctx);
+            await registry.runHook('onReport', ctx, result);
+        }
+        return result;
+    }
+}
+export class PageAnalysisUseCase {
+    context;
+    constructor(context) {
+        this.context = context;
+    }
+    async execute(input) {
+        // When running a live single-page analysis, enable all content modules
+        // by default (they all work well on a single page). Callers can still
+        // explicitly pass false to disable any of them.
+        const isLive = !!input.live;
+        const seo = input.seo ?? (isLive ? true : undefined);
+        const content = input.content ?? (isLive ? true : undefined);
+        const accessibility = input.accessibility ?? (isLive ? true : undefined);
+        const health = input.health ?? (isLive ? true : undefined);
+        const heading = input.heading ?? (isLive ? true : undefined);
+        const result = await analyzeSite(input.url, {
+            live: input.live,
+            snapshotId: input.snapshotId,
+            seo,
+            content,
+            accessibility,
+            rate: input.rate,
+            proxyUrl: input.proxyUrl,
+            userAgent: input.userAgent,
+            maxRedirects: input.maxRedirects,
+            maxBytes: input.maxBytes,
+            clustering: input.clustering,
+            clusterThreshold: input.clusterThreshold,
+            minClusterSize: input.minClusterSize,
+            sitemap: input.sitemap,
+            heading,
+            health,
+            failOnCritical: input.failOnCritical,
+            scoreBreakdown: input.scoreBreakdown,
+            computeHits: input.computeHits,
+            computePagerank: input.computePagerank,
+            orphans: input.orphans,
+            orphanSeverity: input.orphanSeverity,
+            includeSoftOrphans: input.includeSoftOrphans,
+            minInbound: input.minInbound,
+            debug: input.debug,
+            allPages: input.allPages,
+        }, this.context);
+        // Run plugins with page scope — only onInit + onPage are called
+        if (input.plugins && input.plugins.length > 0) {
+            const { PluginRegistry } = await import('../plugin-system/plugin-registry.js');
+            const registry = new PluginRegistry(input.plugins);
+            const pluginCtx = {
+                command: 'page',
+                scope: 'page',
+                targetUrl: input.url,
+                snapshotId: result.snapshotId,
+                live: input.live || !!(input.context?.flags?.live),
+                ...(input.context || {}),
+                db: getCrawlithDB(),
+            };
+            await registry.applyStorage(pluginCtx);
+            await registry.runHook('onInit', pluginCtx);
+            // Fire onPage once per analyzed page (normally just 1 for the page command)
+            const inputOrigin = new URL(input.url).origin;
+            for (const page of result.pages) {
+                const absoluteUrl = UrlUtil.toAbsolute(page.url, inputOrigin);
+                await registry.runHook('onPage', pluginCtx, {
+                    url: absoluteUrl,
+                    html: page.html ?? '',
+                    status: page.status,
+                });
+            }
+            if (pluginCtx.db && result.snapshotId) {
+                pluginCtx.db.aggregateScoreProviders(result.snapshotId, registry.pluginsList);
+            }
+        }
+        return result;
+    }
+}
+export class ExportReport {
+    async execute(input) {
+        return JSON.stringify(loadGraphFromSnapshot(input.snapshotId).toJSON());
+    }
+}
+export class DiffSnapshots {
+    async execute(input) {
+        return compareGraphs(loadGraphFromSnapshot(input.oldSnapshotId), loadGraphFromSnapshot(input.newSnapshotId));
+    }
+}

package/dist/audit/index.js CHANGED Viewed

@@ -26,7 +26,7 @@ export async function auditUrl(urlStr, options = {}) {
     // We handle transport errors differently as they are fatal for the audit (e.g. connection refused)
     // DNS errors might return partial results but usually if transport works, DNS worked (unless transport used IP)
     const dnsPromise = resolveDns(url.hostname);
-    const transportPromise = analyzeTransport(urlStr, timeout);
+    const transportPromise = analyzeTransport(urlStr, timeout, options.userAgent);
     const [dnsResult, transportResult] = await Promise.all([
         dnsPromise,
         transportPromise

package/dist/audit/transport.d.ts CHANGED Viewed

@@ -1,5 +1,5 @@
 import { TransportDiagnostics, PerformanceMetrics, AuditIssue } from './types.js';
-export declare function analyzeTransport(targetUrl: string, timeout: number): Promise<{
+export declare function analyzeTransport(targetUrl: string, timeout: number, userAgent?: string): Promise<{
     transport: TransportDiagnostics;
     performance: PerformanceMetrics;
     issues: AuditIssue[];

package/dist/audit/transport.js CHANGED Viewed

@@ -3,7 +3,8 @@ import http from 'node:http';
 import tls from 'node:tls';
 import { URL } from 'node:url';
 import { IPGuard } from '../core/security/ipGuard.js';
-export async function analyzeTransport(targetUrl, timeout) {
+import { DEFAULTS } from '../constants.js';
+export async function analyzeTransport(targetUrl, timeout, userAgent) {
     const maxRedirects = 10;
     let currentUrl = targetUrl;
     let redirectCount = 0;
@@ -18,7 +19,7 @@ export async function analyzeTransport(targetUrl, timeout) {
             throw new Error(`Blocked: Redirect to internal/private IP prohibited (${currentUrl})`);
         }
         try {
-            const result = await executeRequest(currentUrl, timeout);
+            const result = await executeRequest(currentUrl, timeout, userAgent);
             if (result.redirectUrl) {
                 redirectCount++;
                 totalRedirectTime += result.timings.total;
@@ -113,7 +114,7 @@ export async function analyzeTransport(targetUrl, timeout) {
     }
     throw new Error(`Too many redirects (limit: ${maxRedirects})`);
 }
-function executeRequest(urlStr, timeout) {
+function executeRequest(urlStr, timeout, userAgent) {
     return new Promise((resolve, reject) => {
         let url;
         try {
@@ -143,7 +144,7 @@ function executeRequest(urlStr, timeout) {
             rejectUnauthorized: false,
             agent: false,
             headers: {
-                'User-Agent': 'Crawlith/Audit',
+                'User-Agent': userAgent || DEFAULTS.USER_AGENT,
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                 'Accept-Encoding': 'gzip, deflate, br'
             }

package/dist/audit/types.d.ts CHANGED Viewed

@@ -85,4 +85,5 @@ export interface AuditOptions {
     timeout?: number;
     verbose?: boolean;
     debug?: boolean;
+    userAgent?: string;
 }

package/dist/constants.d.ts ADDED Viewed

@@ -0,0 +1,17 @@
+export declare const DEFAULTS: {
+    readonly MAX_DEPTH: 5;
+    readonly MAX_DEPTH_LIMIT: 10;
+    readonly CONCURRENCY: 2;
+    readonly CONCURRENCY_LIMIT: 10;
+    readonly CRAWL_LIMIT: 500;
+    readonly USER_AGENT: `crawlith/${string}`;
+    readonly RATE_LIMIT: 10;
+    readonly MAX_BYTES: 2000000;
+    readonly MAX_REDIRECTS: 5;
+    readonly MAX_REDIRECTS_LIMIT: 11;
+    readonly HEADERS_TIMEOUT: 10000;
+    readonly BODY_TIMEOUT: 10000;
+    readonly MAX_SNAPSHOTS: 5;
+    readonly MAX_SINGLE_SNAPSHOTS: 5;
+    readonly GRAPH_PRECISION: 1e-12;
+};

package/dist/constants.js ADDED Viewed

@@ -0,0 +1,23 @@
+import { version } from './utils/version.js';
+export const DEFAULTS = {
+    // Crawler defaults
+    MAX_DEPTH: 5,
+    MAX_DEPTH_LIMIT: 10,
+    CONCURRENCY: 2,
+    CONCURRENCY_LIMIT: 10,
+    CRAWL_LIMIT: 500,
+    // Network/Fetcher defaults
+    USER_AGENT: `crawlith/${version}`,
+    RATE_LIMIT: 10,
+    MAX_BYTES: 2000000, // 2MB
+    MAX_REDIRECTS: 5,
+    MAX_REDIRECTS_LIMIT: 11,
+    // Network timeouts
+    HEADERS_TIMEOUT: 10000,
+    BODY_TIMEOUT: 10000,
+    // Keep only last 5 snapshots
+    MAX_SNAPSHOTS: 5,
+    MAX_SINGLE_SNAPSHOTS: 5,
+    // Graph calculation precision
+    GRAPH_PRECISION: 1e-12
+};

package/dist/core/scope/scopeManager.js CHANGED Viewed

@@ -15,6 +15,9 @@ export class ScopeManager {
         }));
     }
     isUrlEligible(url) {
+        // Root-relative paths (e.g. '/about', '/?q=foo') are always internal
+        if (url.startsWith('/'))
+            return 'allowed';
         let hostname;
         try {
             hostname = new URL(url).hostname.toLowerCase();