@crawlith/core 0.1.0 → 0.1.2

package/dist/plugin-system/plugin-loader.js

@@ -0,0 +1,122 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+export class PluginLoader {
+    plugins = new Map();
+    logger;
+    constructor(logger) {
+        this.logger = logger;
+    }
+    async discover(rootPath) {
+        // 1. Discover Internal Plugins
+        const internalPath = path.resolve(rootPath, 'packages/plugins');
+        if (fs.existsSync(internalPath)) {
+            this.logger?.debug(`[plugin] Scanning internal directory: ${internalPath}`);
+            await this.loadFromDir(internalPath, 'internal');
+        }
+        // 2. Discover External Plugins
+        const nodeModulesPath = path.resolve(rootPath, 'node_modules');
+        if (fs.existsSync(nodeModulesPath)) {
+            this.logger?.debug(`[plugin] Scanning node_modules: ${nodeModulesPath}`);
+            await this.loadFromNodeModules(nodeModulesPath);
+        }
+        return Array.from(this.plugins.values());
+    }
+    async loadFromDir(dirPath, type) {
+        const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.isDirectory()) {
+                const fullPath = path.join(dirPath, entry.name);
+                await this.tryLoadPlugin(fullPath, type);
+            }
+        }
+    }
+    async loadFromNodeModules(nodeModulesPath) {
+        if (!fs.existsSync(nodeModulesPath))
+            return;
+        const entries = fs.readdirSync(nodeModulesPath, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            if (entry.name.startsWith('@')) {
+                // Scoped packages
+                const scopePath = path.join(nodeModulesPath, entry.name);
+                const scopedEntries = fs.readdirSync(scopePath, { withFileTypes: true });
+                for (const scopedEntry of scopedEntries) {
+                    if (scopedEntry.isDirectory()) {
+                        await this.tryLoadPlugin(path.join(scopePath, scopedEntry.name), 'external');
+                    }
+                }
+            }
+            else {
+                await this.tryLoadPlugin(path.join(nodeModulesPath, entry.name), 'external');
+            }
+        }
+    }
+    async tryLoadPlugin(pluginPath, type) {
+        const pkgPath = path.join(pluginPath, 'package.json');
+        if (!fs.existsSync(pkgPath))
+            return;
+        try {
+            const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+            const isCrawlithPlugin = pkg.name?.startsWith('@crawlith/plugin-') ||
+                pkg.name?.startsWith('crawlith-plugin-') ||
+                pkg.crawlith?.type === 'plugin' ||
+                type === 'internal';
+            if (!isCrawlithPlugin)
+                return;
+            let entryPoint = pkg.exports ?? pkg.main ?? 'index.js';
+            if (typeof entryPoint === 'object' && entryPoint !== null) {
+                entryPoint = entryPoint['.'] ?? entryPoint.import ?? entryPoint.default ?? 'index.js';
+            }
+            if (typeof entryPoint !== 'string') {
+                entryPoint = 'index.js';
+            }
+            let fullEntryPoint = path.join(pluginPath, entryPoint);
+            // If we're loading a .ts file as the intended entry point, but a .js one exists in dist, 
+            // prefer the .js one to avoid requiring a TS loader at runtime.
+            if (fullEntryPoint.endsWith('.ts')) {
+                const distJsPath = path.join(pluginPath, 'dist', 'index.js');
+                if (fs.existsSync(distJsPath)) {
+                    fullEntryPoint = distJsPath;
+                }
+            }
+            if (!fs.existsSync(fullEntryPoint) && fs.existsSync(path.join(pluginPath, 'index.ts'))) {
+                fullEntryPoint = path.join(pluginPath, 'index.ts');
+            }
+            if (!fs.existsSync(fullEntryPoint)) {
+                this.logger?.debug(`[plugin] Skipped: ${pkg.name} (entry point not found: ${fullEntryPoint})`);
+                return;
+            }
+            const imported = await import(pathToFileURL(fullEntryPoint).href);
+            const plugin = imported.default || imported;
+            if (this.validatePlugin(plugin, pkg)) {
+                if (this.plugins.has(plugin.name)) {
+                    throw new Error(`Duplicate plugin name: ${plugin.name}`);
+                }
+                this.plugins.set(plugin.name, plugin);
+                this.logger?.debug(`[plugin] Loaded: ${plugin.name} v${plugin.version} (${type})`);
+            }
+        }
+        catch (err) {
+            this.logger?.debug(`[plugin] Failed to load plugin at ${pluginPath}: ${err.message}`);
+        }
+    }
+    validatePlugin(plugin, pkg) {
+        if (!plugin || typeof plugin !== 'object') {
+            this.logger?.debug(`[plugin] Skipped: ${pkg.name} (invalid export)`);
+            return false;
+        }
+        if (!plugin.name) {
+            this.logger?.debug(`[plugin] Skipped: ${pkg.name} (missing name)`);
+            return false;
+        }
+        if (!plugin.version) {
+            plugin.version = pkg.version || '0.0.0';
+        }
+        if (!plugin.description) {
+            plugin.description = pkg.description || '';
+        }
+        return true;
+    }
+}

package/dist/plugin-system/plugin-registry.d.ts

@@ -0,0 +1,25 @@
+import { CrawlithPlugin, PluginContext } from './plugin-types.js';
+export declare class PluginRegistry {
+    private plugins;
+    constructor(plugins?: CrawlithPlugin[]);
+    get pluginsList(): CrawlithPlugin[];
+    private registeredCommands;
+    /**
+     * Registers all plugin CLI flags on the given command.
+     * Handles both declarative `cli` config and legacy `register(cmd)` callbacks.
+     */
+    registerPlugins(program: any): void;
+    /**
+     * Applies declarative `storage` schemas for all plugins.
+     * Called by the core after `getCrawlithDB()` is available, before any hooks run.
+     */
+    applyStorage(context: PluginContext): void;
+    /** Hooks that only make sense during a full site crawl. */
+    private static readonly CRAWL_ONLY_HOOKS;
+    /** Hooks that only make sense during a single-page analysis. */
+    private static readonly PAGE_ONLY_HOOKS;
+    runHook(hookName: string, context: PluginContext, payload?: any): Promise<void>;
+    runSyncBailHook(hookName: string, context: PluginContext, ...args: any[]): any;
+    getPlugins(): CrawlithPlugin[];
+    addPlugin(plugin: CrawlithPlugin): void;
+}

package/dist/plugin-system/plugin-registry.js

@@ -0,0 +1,167 @@
+import { Command } from 'commander';
+import { PluginConfig } from './plugin-config.js';
+export class PluginRegistry {
+    plugins = [];
+    constructor(plugins = []) {
+        this.plugins = plugins;
+    }
+    get pluginsList() {
+        return this.plugins;
+    }
+    registeredCommands = new WeakSet();
+    /**
+     * Registers all plugin CLI flags on the given command.
+     * Handles both declarative `cli` config and legacy `register(cmd)` callbacks.
+     */
+    registerPlugins(program) {
+        if (!(program instanceof Command))
+            return;
+        const traverse = (cmd) => {
+            if (this.registeredCommands.has(cmd))
+                return;
+            this.registeredCommands.add(cmd);
+            const cmdName = cmd.name();
+            for (const plugin of this.plugins) {
+                // Declarative cli registration (preferred)
+                if (plugin.cli) {
+                    const targets = plugin.cli.for ?? ['page', 'crawl'];
+                    if (targets.includes(cmdName)) {
+                        const defaultValue = plugin.cli.defaultValue;
+                        if (defaultValue !== undefined && defaultValue !== null) {
+                            cmd.option(plugin.cli.flag, plugin.cli.description, defaultValue);
+                        }
+                        else {
+                            cmd.option(plugin.cli.flag, plugin.cli.description);
+                        }
+                        for (const opt of plugin.cli.options ?? []) {
+                            const dv = opt.defaultValue;
+                            if (dv !== undefined && dv !== null) {
+                                cmd.option(opt.flag, opt.description, dv);
+                            }
+                            else {
+                                cmd.option(opt.flag, opt.description);
+                            }
+                        }
+                    }
+                }
+                // Legacy imperative registration (backwards compat)
+                if (typeof plugin.register === 'function') {
+                    plugin.register(cmd);
+                }
+            }
+            for (const sub of cmd.commands) {
+                traverse(sub);
+            }
+        };
+        traverse(program);
+    }
+    /**
+     * Applies declarative `storage` schemas for all plugins.
+     * Called by the core after `getCrawlithDB()` is available, before any hooks run.
+     */
+    applyStorage(context) {
+        for (const plugin of this.plugins) {
+            if (!plugin.storage?.perPage?.columns)
+                continue;
+            if (!context.db)
+                continue;
+            try {
+                const scopedDb = context.db.scope(plugin.name, context.snapshotId, {
+                    live: context.live,
+                    fetchMode: plugin.storage.fetchMode
+                });
+                scopedDb.schema.define(plugin.storage.perPage.columns);
+            }
+            catch (err) {
+                context.logger?.error(`[plugin:${plugin.name}] Storage schema failed: ${err.message}`);
+            }
+        }
+    }
+    // ─── Scope guards ─────────────────────────────────────────────────────────
+    /** Hooks that only make sense during a full site crawl. */
+    static CRAWL_ONLY_HOOKS = new Set([
+        'onCrawlStart', 'onPageParsed', 'onGraphBuilt', 'onMetrics', 'onReport', 'shouldEnqueueUrl'
+    ]);
+    /** Hooks that only make sense during a single-page analysis. */
+    static PAGE_ONLY_HOOKS = new Set([
+        'onPage'
+    ]);
+    // ─── Hook runners ─────────────────────────────────────────────────────────
+    async runHook(hookName, context, payload) {
+        // Enforce scope: silently skip hooks that don't belong to this execution context.
+        // Undefined scope (legacy/test) is treated as permissive.
+        if (context.scope === 'page' && PluginRegistry.CRAWL_ONLY_HOOKS.has(hookName))
+            return;
+        if (context.scope === 'crawl' && PluginRegistry.PAGE_ONLY_HOOKS.has(hookName))
+            return;
+        for (const plugin of this.plugins) {
+            const hooks = plugin.hooks;
+            if (hooks && typeof hooks[hookName] === 'function') {
+                const scopedDb = context.db?.scope(plugin.name, context.snapshotId || payload?.snapshotId, {
+                    live: context.live,
+                    fetchMode: plugin.storage?.fetchMode
+                });
+                const scopedConfig = new PluginConfig(plugin.name);
+                // Resolve targetUrl from payload if available (standard result object)
+                const targetUrl = context.targetUrl || payload?.pages?.[0]?.url;
+                // Build a CLIWriter that prefixes plugin name — satisfies both cli and logger
+                const cliWriter = {
+                    info: (m) => context.logger?.info(m),
+                    warn: (m) => context.logger?.warn(m),
+                    error: (m) => context.logger?.error(m),
+                    debug: (m) => context.logger?.debug(m),
+                };
+                const scopedContext = {
+                    ...context,
+                    db: scopedDb,
+                    config: scopedConfig,
+                    targetUrl,
+                    cli: cliWriter,
+                    logger: cliWriter, // keep logger alias in sync
+                };
+                try {
+                    if (payload !== undefined) {
+                        await hooks[hookName](scopedContext, payload);
+                    }
+                    else {
+                        await hooks[hookName](scopedContext);
+                    }
+                }
+                catch (err) {
+                    context.logger?.error(`[plugin:${plugin.name}] Hook ${hookName} failed: ${err.message}`);
+                }
+            }
+        }
+    }
+    runSyncBailHook(hookName, context, ...args) {
+        for (const plugin of this.plugins) {
+            const hooks = plugin.hooks;
+            if (hooks && typeof hooks[hookName] === 'function') {
+                const scopedDb = context.db?.scope(plugin.name, context.snapshotId, {
+                    live: context.live,
+                    fetchMode: plugin.storage?.fetchMode
+                });
+                const scopedConfig = new PluginConfig(plugin.name);
+                const scopedContext = { ...context, db: scopedDb, config: scopedConfig };
+                try {
+                    const result = hooks[hookName](scopedContext, ...args);
+                    if (result !== undefined)
+                        return result;
+                }
+                catch (err) {
+                    context.logger?.error(`[plugin:${plugin.name}] Sync bail hook ${hookName} failed: ${err.message}`);
+                }
+            }
+        }
+        return undefined;
+    }
+    getPlugins() {
+        return this.plugins;
+    }
+    addPlugin(plugin) {
+        if (this.plugins.some(p => p.name === plugin.name)) {
+            throw new Error(`Duplicate plugin name: ${plugin.name}`);
+        }
+        this.plugins.push(plugin);
+    }
+}

package/dist/plugin-system/plugin-types.d.ts

@@ -0,0 +1,205 @@
+import type { CrawlithDB } from '../db/CrawlithDB.js';
+/**
+ * Execution scope — set by the core before any hooks run.
+ * - `page`  → single-URL analysis (crawlith page …)
+ * - `crawl` → full site crawl   (crawlith crawl …)
+ */
+export type PluginScope = 'page' | 'crawl';
+/** Injected into hook contexts to let plugins emit structured CLI output. */
+export interface CLIWriter {
+    info(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+    debug(message: string): void;
+}
+/**
+ * Injected into report-phase hook contexts.
+ * Plugins contribute structured data that the core aggregates into the final output.
+ */
+export interface ReportWriter {
+    /** Attach a named data section to the report output. */
+    addSection(pluginName: string, data: unknown): void;
+    /** Optionally contribute a weighted score component. */
+    contributeScore?(input: {
+        label: string;
+        score: number;
+        weight: number;
+    }): void;
+}
+/** Passed to `onPage` — the single URL being analyzed. */
+export interface PageInput {
+    /** Absolute URL of the page being analyzed. */
+    url: string;
+    /** Raw HTML content of the page. */
+    html: string;
+    /** HTTP status code. */
+    status: number;
+    /** HTTP Response headers. */
+    headers?: Record<string, string | string[]>;
+}
+/** Base context available in every hook. */
+export interface PluginContext {
+    /** Execution scope. Undefined only in legacy / test contexts — treated as permissive. */
+    scope?: PluginScope;
+    command?: string;
+    /** Whether live fallback is allowed (from --live flag). Core-controlled. */
+    live?: boolean;
+    flags?: Record<string, any>;
+    snapshotId?: number;
+    targetUrl?: string;
+    db?: CrawlithDB;
+    config?: {
+        get(key?: string): string;
+        require(key?: string): string;
+        set(value: string): void;
+    };
+    /** CLI writer — populated by the registry before each hook call. */
+    cli?: CLIWriter;
+    /** Legacy logger alias — kept for backwards compatibility. */
+    logger?: CLIWriter;
+    metadata?: Record<string, any>;
+    /** MCP discovery registry available during MCP startup discovery. */
+    mcpDiscovery?: PluginMcpDiscovery;
+    [key: string]: any;
+}
+export interface PluginMcpTool {
+    name: string;
+    description: string;
+    /** Zod-like shape object interpreted by the MCP server package. */
+    inputSchema?: Record<string, unknown>;
+    execute: (input: Record<string, unknown>, ctx: PluginContext) => unknown | Promise<unknown>;
+}
+export interface PluginMcpPrompt {
+    name: string;
+    description: string;
+    /** Zod-like shape object interpreted by the MCP server package. */
+    argumentsSchema?: Record<string, unknown>;
+    buildMessages: (input: Record<string, unknown>, ctx: PluginContext) => {
+        messages: Array<{
+            role: 'user' | 'assistant' | 'system';
+            content: {
+                type: 'text';
+                text: string;
+            };
+        }>;
+    };
+}
+export interface PluginMcpDiscovery {
+    registerTool(tool: PluginMcpTool): void;
+    registerPrompt(prompt: PluginMcpPrompt): void;
+}
+export interface PluginCliOption {
+    /** e.g. '--my-flag <value>' */
+    flag: string;
+    description: string;
+    defaultValue?: unknown;
+}
+export type PluginColumnType = 'INTEGER' | 'REAL' | 'TEXT';
+export interface PluginStorage {
+    /** Whether this plugin fetches data from a network or computes locally. Defaults to 'network'. */
+    fetchMode?: 'local' | 'network';
+    /**
+     * Per-URL columns to add to the plugin's scoped data table.
+     * The core creates the table automatically before `onInit` runs,
+     * so plugins never need to call `ctx.db.schema.define()`.
+     */
+    perPage?: {
+        columns: Record<string, PluginColumnType>;
+    };
+}
+export interface CrawlithPlugin {
+    name: string;
+    version?: string;
+    description?: string;
+    /**
+     * Declarative CLI registration.
+     * The core registers these options on the appropriate commands —
+     * no need to interact with Commander directly.
+     *
+     * `for` controls which commands expose the options:
+     * - `['page', 'crawl']` (default when omitted) — both commands
+     * - `['page']` — page command only
+     * - `['crawl']` — crawl command only
+     */
+    cli?: {
+        flag: string;
+        description: string;
+        defaultValue?: unknown;
+        for?: ('page' | 'crawl')[];
+        options?: PluginCliOption[];
+    };
+    /**
+     * Declarative storage schema.
+     * The core creates the plugin's scoped table before `onInit` runs.
+     * Plugins that don't persist data can omit this entirely.
+     */
+    storage?: PluginStorage;
+    /**
+     * Set to true to declare this plugin as a Score Provider.
+     * The core will automatically aggregate the `score` and `weight` columns
+     * from this plugin's storage table during snapshot aggregation.
+     */
+    scoreProvider?: boolean;
+    /**
+     * Legacy imperative CLI registration — kept for backwards compatibility.
+     * Prefer `cli` for new plugins.
+     * @deprecated Use `cli` instead.
+     */
+    register?: (cli: any) => void;
+    /** Declarative MCP definitions discovered by @crawlith/mcp at startup. */
+    mcp?: {
+        tools?: PluginMcpTool[];
+        prompts?: PluginMcpPrompt[];
+    };
+    hooks?: {
+        /**
+         * Runs on both `page` and `crawl` scopes.
+         * Use for any setup that doesn't depend on the scope —
+         * e.g. initialising in-memory state, reading config.
+         * DB schema is already created by the time this runs (via `storage`).
+         */
+        onInit?: (ctx: PluginContext) => void | Promise<void>;
+        /**
+         * Single-page hook — `page` scope only.
+         * Receives the target URL, its raw HTML, and HTTP status.
+         * Use this for URL-scoped plugins (PageSpeed, heading-health, etc.).
+         */
+        onPage?: (ctx: PluginContext, page: PageInput) => void | Promise<void>;
+        /**
+         * Fired at the very start of a crawl — `crawl` scope only.
+         * Use to initialise crawl-wide state or validate config.
+         */
+        onCrawlStart?: (ctx: PluginContext) => void | Promise<void>;
+        /**
+         * URL enqueue filter — `crawl` scope only.
+         * Return `false` to prevent a URL from being crawled.
+         */
+        shouldEnqueueUrl?: (ctx: PluginContext, url: string, depth: number) => boolean;
+        /**
+         * Fired after each page is fetched and parsed — `crawl` scope only.
+         * Use for real-time per-page processing without waiting for the full graph.
+         */
+        onPageParsed?: (ctx: PluginContext, page: any) => void | Promise<void>;
+        /**
+         * Fired after the full link graph is built — `crawl` scope only.
+         * Graph structure is complete; metrics have not been computed yet.
+         */
+        onGraphBuilt?: (ctx: PluginContext, graph: any) => void | Promise<void>;
+        /**
+         * Graph-level metrics phase — `crawl` scope only.
+         * All pages are available; use for cross-page analysis (duplicate
+         * detection, PageRank, heading structure across the site, etc.).
+         */
+        onMetrics?: (ctx: PluginContext, graph: any) => void | Promise<void>;
+        /**
+         * Final report phase — `crawl` scope only.
+         * Attach snapshot-level summary data to the result object.
+         */
+        onReport?: (ctx: PluginContext, report: any) => void | Promise<void>;
+        /**
+         * MCP discovery phase — executed by @crawlith/mcp server startup.
+         * Plugins can register MCP tools/prompts through `ctx.mcpDiscovery`.
+         */
+        onMcpDiscovery?: (ctx: PluginContext) => void | Promise<void>;
+    };
+}

package/dist/plugin-system/plugin-types.js

@@ -0,0 +1 @@
+export {};

package/dist/ports/index.d.ts

@@ -0,0 +1,9 @@
+export type PageRepositoryPort = Record<string, unknown>;
+export type EdgeRepositoryPort = Record<string, unknown>;
+export type SnapshotRepositoryPort = Record<string, unknown>;
+export type FetcherPort = Record<string, unknown>;
+export interface LoggerPort {
+    info(msg: string): void;
+    warn(msg: string): void;
+    error(msg: string): void;
+}

package/dist/ports/index.js

@@ -0,0 +1 @@
+export {};