docshark 0.1.5 → 0.1.7

package/dist/scraper/fetcher.js

@@ -0,0 +1,129 @@
+import { extractAndConvert } from '../processor/extractor.js';
+const USER_AGENT = 'DocShark/1.0';
+const MIN_CONTENT_LENGTH = 500;
+const MAX_RETRIES = 3;
+/**
+ * Fetch a page and return its HTML.
+ * Supports auto-detection of JS-rendered sites (falls back to puppeteer-core if installed).
+ */
+export async function fetchPage(url, renderer = 'auto') {
+    // Force puppeteer if configured
+    if (renderer === 'puppeteer') {
+        return fetchWithPuppeteer(url);
+    }
+    // Tier 1: Standard fetch
+    const result = await fetchWithRetry(url);
+    if (renderer === 'fetch') {
+        return result;
+    }
+    // Auto mode: check if content is too short (possibly JS-rendered)
+    const { markdown } = extractAndConvert(result.html, url);
+    if (markdown.length >= MIN_CONTENT_LENGTH) {
+        return result;
+    }
+    // Tier 2: Content too short + has <script> tags → likely JS-rendered
+    const looksJsRendered = result.html.includes('<script') && markdown.length < MIN_CONTENT_LENGTH;
+    if (looksJsRendered) {
+        console.warn(`[DocShark] ${url} appears JS-rendered (${markdown.length} chars). Trying puppeteer...`);
+        if (await canUsePuppeteer()) {
+            return fetchWithPuppeteer(url);
+        }
+        console.warn(`[DocShark] puppeteer-core not installed. Run: bun add puppeteer-core\n` +
+            `Or set renderer: "fetch" in the library config to suppress this warning.`);
+    }
+    return result;
+}
+/** Fetch with exponential backoff retry */
+async function fetchWithRetry(url, retries = MAX_RETRIES) {
+    for (let attempt = 1; attempt <= retries; attempt++) {
+        try {
+            const response = await fetch(url, {
+                headers: {
+                    'User-Agent': USER_AGENT,
+                    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                },
+                signal: AbortSignal.timeout(30_000),
+                redirect: 'follow',
+            });
+            const html = await response.text();
+            return {
+                html,
+                renderer: 'fetch',
+                status: response.status,
+                etag: response.headers.get('etag'),
+                lastModified: response.headers.get('last-modified'),
+            };
+        }
+        catch (err) {
+            if (attempt === retries)
+                throw err;
+            // Exponential backoff: 1s, 2s, 4s
+            await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
+        }
+    }
+    throw new Error(`Failed to fetch ${url} after ${retries} attempts`);
+}
+/** Check if puppeteer-core is available (dynamic import) */
+async function canUsePuppeteer() {
+    try {
+        // @ts-ignore — puppeteer-core is an optional dependency
+        await import(/* webpackIgnore: true */ 'puppeteer-core');
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Fetch with puppeteer-core using system Chrome */
+async function fetchWithPuppeteer(url) {
+    // @ts-ignore — puppeteer-core is an optional dependency
+    const puppeteer = await import('puppeteer-core');
+    const { existsSync } = await import('fs');
+    const executablePath = findChrome(existsSync);
+    if (!executablePath) {
+        throw new Error('Chrome not found. Set CHROME_PATH env var or install Chrome.\n' +
+            'Alternatively: npx puppeteer browsers install chrome');
+    }
+    const browser = await puppeteer.default.launch({
+        headless: true,
+        executablePath,
+        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
+    });
+    const page = await browser.newPage();
+    try {
+        await page.setRequestInterception(true);
+        page.on('request', (req) => {
+            const type = req.resourceType();
+            if (['image', 'stylesheet', 'font', 'media'].includes(type)) {
+                req.abort();
+            }
+            else {
+                req.continue();
+            }
+        });
+        await page.goto(url, { waitUntil: 'networkidle2', timeout: 30_000 });
+        const html = await page.content();
+        return { html, renderer: 'puppeteer', status: 200 };
+    }
+    finally {
+        await page.close();
+        await browser.close();
+    }
+}
+function findChrome(existsSync) {
+    const candidates = [
+        process.env.CHROME_PATH,
+        process.env.PUPPETEER_EXECUTABLE_PATH,
+        '/usr/bin/google-chrome',
+        '/usr/bin/google-chrome-stable',
+        '/usr/bin/chromium-browser',
+        '/usr/bin/chromium',
+        '/snap/bin/chromium',
+        '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+    ];
+    for (const path of candidates) {
+        if (path && existsSync(path))
+            return path;
+    }
+    return undefined;
+}

package/dist/scraper/rate-limiter.js

@@ -0,0 +1,18 @@
+// src/scraper/rate-limiter.ts — Configurable rate limiter for polite crawling
+export class RateLimiter {
+    delayMs;
+    lastRequest = 0;
+    constructor(delayMs = 500) {
+        this.delayMs = delayMs;
+    }
+    async wait() {
+        const elapsed = Date.now() - this.lastRequest;
+        if (elapsed < this.delayMs) {
+            await new Promise((resolve) => setTimeout(resolve, this.delayMs - elapsed));
+        }
+        this.lastRequest = Date.now();
+    }
+    setDelay(ms) {
+        this.delayMs = ms;
+    }
+}

package/dist/scraper/robots.js

@@ -0,0 +1,26 @@
+// src/scraper/robots.ts — robots.txt parser
+import robotsParser from 'robots-parser';
+const USER_AGENT = 'DocShark/1.0';
+/** Fetch and parse robots.txt for a given base URL */
+export async function getRobotsParser(baseUrl) {
+    const robotsUrl = new URL('/robots.txt', baseUrl).href;
+    try {
+        const response = await fetch(robotsUrl, {
+            headers: { 'User-Agent': USER_AGENT },
+            signal: AbortSignal.timeout(10_000),
+        });
+        if (!response.ok)
+            return null;
+        const body = await response.text();
+        return robotsParser(robotsUrl, body);
+    }
+    catch {
+        return null;
+    }
+}
+/** Check if a URL is allowed by robots.txt */
+export function isAllowed(robots, url) {
+    if (!robots)
+        return true;
+    return robots.isAllowed(url, USER_AGENT) !== false;
+}

package/dist/server.js

@@ -0,0 +1,154 @@
+// src/server.ts — TMCP McpServer setup + tool registration
+import { McpServer } from 'tmcp';
+import { ValibotJsonSchemaAdapter } from '@tmcp/adapter-valibot';
+import * as v from 'valibot';
+import { tool } from 'tmcp/utils';
+import { Database } from './storage/db.js';
+import { SearchEngine } from './storage/search.js';
+import { LibraryService } from './services/library.js';
+import { JobManager } from './jobs/manager.js';
+import { VERSION } from './version.js';
+import { EventBus } from './jobs/events.js';
+// Initialize core services
+export const db = new Database();
+export const eventBus = new EventBus();
+export const searchEngine = new SearchEngine(db);
+export const jobManager = new JobManager(db, eventBus);
+export const libraryService = new LibraryService(db, jobManager);
+// Create TMCP server
+export const server = new McpServer({
+    name: 'docshark',
+    version: VERSION,
+    description: '🦈 Documentation MCP Server — scrape, index, and search any doc website',
+}, {
+    adapter: new ValibotJsonSchemaAdapter(),
+    capabilities: {
+        tools: { listChanged: true },
+        resources: {},
+    },
+});
+// ──────────────────────────────────────
+// Tool 1: search_docs — Primary search tool
+// ──────────────────────────────────────
+server.tool({
+    name: 'search_docs',
+    description: 'Search through indexed documentation libraries for relevant information. ' +
+        'Returns ranked documentation sections with code examples and source URLs. ' +
+        'Use this when you need to find information about a library, framework, API, ' +
+        'or any technical concept.',
+    schema: v.object({
+        query: v.pipe(v.string(), v.description('Search query. Use natural language.')),
+        library: v.optional(v.pipe(v.string(), v.description('Filter to a specific library.'))),
+        limit: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(20)), 5),
+    }),
+}, async ({ query, library, limit }) => {
+    const results = searchEngine.search(query, { library, limit });
+    if (results.length === 0)
+        return tool.text(`No results found for "${query}".`);
+    const formatted = results
+        .map((r, i) => {
+        let block = `### ${i + 1}. ${r.page_title} — ${r.library_display_name}\n`;
+        block += `**Source:** ${r.page_url}\n`;
+        block += `**Section:** ${r.heading_context}\n\n`;
+        block += r.content;
+        return block;
+    })
+        .join('\n\n---\n\n');
+    return tool.text(`## Results for "${query}"\n\n${formatted}`);
+});
+// ──────────────────────────────────────
+// Tool 2: list_libraries — Discovery tool
+// ──────────────────────────────────────
+server.tool({
+    name: 'list_libraries',
+    description: 'List all documentation libraries currently indexed and available for searching. ' +
+        'Use this to discover what docs are available before running search_docs.',
+    schema: v.object({
+        status: v.optional(v.pipe(v.picklist(['indexed', 'crawling', 'error', 'all']), v.description('Filter by status. Default: "all".')), 'all'),
+    }),
+}, async ({ status }) => {
+    const libraries = db.listLibraries(status);
+    if (libraries.length === 0) {
+        return tool.text('No libraries indexed yet. Use add_library to add a documentation website.');
+    }
+    let output = `## Indexed Libraries (${libraries.length} total)\n\n`;
+    output += '| Library | URL | Pages | Chunks | Status |\n';
+    output += '| ------- | --- | ----- | ------ | ------ |\n';
+    for (const lib of libraries) {
+        output += `| ${lib.name} | ${lib.url} | ${lib.page_count} | ${lib.chunk_count} | ${lib.status} |\n`;
+    }
+    return tool.text(output);
+});
+// ──────────────────────────────────────
+// Tool 3: get_doc_page — Full page read
+// ──────────────────────────────────────
+server.tool({
+    name: 'get_doc_page',
+    description: 'Retrieve the complete content of a specific documentation page as markdown. ' +
+        'Use when search results reference a page and you need full context.',
+    schema: v.object({
+        url: v.optional(v.pipe(v.string(), v.description('The full URL of the documentation page.'))),
+        library: v.optional(v.pipe(v.string(), v.description('Library name to search within.'))),
+        path: v.optional(v.pipe(v.string(), v.description('Relative path within the library.'))),
+    }),
+}, async ({ url, library, path }) => {
+    const page = db.getPage({ url, library, path });
+    if (!page)
+        return tool.text('Page not found. Use search_docs to find the correct page.');
+    return tool.text(`# ${page.title}\n**Source:** ${page.url}\n\n${page.content_markdown}`);
+});
+// ──────────────────────────────────────
+// Tool 4: add_library — Add new doc source
+// ──────────────────────────────────────
+server.tool({
+    name: 'add_library',
+    description: 'Add a new documentation library to be crawled and indexed. ' +
+        'Provide the URL and an optional name. Crawl runs in the background.',
+    schema: v.object({
+        url: v.pipe(v.string(), v.url(), v.description('Base URL of the documentation website.')),
+        name: v.optional(v.pipe(v.string(), v.description('Short identifier (auto-generated if omitted).'))),
+        version: v.optional(v.pipe(v.string(), v.description('Version string.'))),
+        max_depth: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(10)), 3),
+    }),
+}, async ({ url, name, version, max_depth }) => {
+    try {
+        const library = await libraryService.add({ url, name, version, maxDepth: max_depth });
+        return tool.text(`✅ Library "${library.display_name}" added.\n` +
+            `Crawl job ${library.jobId} started. Use list_libraries to check progress.`);
+    }
+    catch (err) {
+        return tool.text(`❌ Failed: ${err.message}`);
+    }
+});
+// ──────────────────────────────────────
+// Tool 5: refresh_library — Re-crawl
+// ──────────────────────────────────────
+server.tool({
+    name: 'refresh_library',
+    description: 'Re-crawl and re-index an existing documentation library to get the latest content.',
+    schema: v.object({
+        library: v.pipe(v.string(), v.description('The library name to refresh.')),
+    }),
+}, async ({ library }) => {
+    const lib = db.getLibraryByName(library);
+    if (!lib)
+        return tool.text(`Library "${library}" not found. Use list_libraries to see available.`);
+    const job = jobManager.startCrawl(lib.id, { incremental: true });
+    return tool.text(`🔄 Refresh started for "${lib.display_name}".\nJob ${job.id}: checking for updated pages...`);
+});
+// ──────────────────────────────────────
+// Tool 6: remove_library — Delete
+// ──────────────────────────────────────
+server.tool({
+    name: 'remove_library',
+    description: 'Remove a documentation library and all its indexed content permanently.',
+    schema: v.object({
+        library: v.pipe(v.string(), v.description('The library name to remove.')),
+    }),
+}, async ({ library }) => {
+    const lib = db.getLibraryByName(library);
+    if (!lib)
+        return tool.text(`Library "${library}" not found.`);
+    db.removeLibrary(lib.id);
+    return tool.text(`🗑️ Library "${lib.display_name}" removed.\nDeleted ${lib.page_count} pages and ${lib.chunk_count} chunks.`);
+});

package/dist/services/library.js

@@ -0,0 +1,66 @@
+// src/services/library.ts — Library management service
+import { nanoid } from 'nanoid';
+export class LibraryService {
+    db;
+    jobManager;
+    constructor(db, jobManager) {
+        this.db = db;
+        this.jobManager = jobManager;
+    }
+    /** Add a new documentation library and start crawling */
+    async add(opts) {
+        const url = normalizeUrl(opts.url);
+        const name = opts.name || generateName(url);
+        const displayName = generateDisplayName(name);
+        // Check if already exists
+        const existing = this.db.getLibraryByName(name);
+        if (existing) {
+            throw new Error(`Library "${name}" already exists. Use refresh_library to re-crawl.`);
+        }
+        const id = nanoid();
+        const crawlConfig = {
+            maxDepth: opts.maxDepth ?? 3,
+            renderer: 'auto',
+        };
+        this.db.addLibrary({
+            id,
+            name,
+            displayName,
+            url,
+            version: opts.version,
+            crawlConfig,
+        });
+        // Start crawl job
+        const job = this.jobManager.startCrawl(id);
+        const library = this.db.getLibraryById(id);
+        return { ...library, jobId: job.id };
+    }
+}
+/** Normalize URL: ensure trailing slash for base docs */
+function normalizeUrl(url) {
+    const parsed = new URL(url);
+    // Remove trailing hash and query for base URL
+    parsed.hash = '';
+    return parsed.href;
+}
+/** Generate a slug name from URL */
+function generateName(url) {
+    const parsed = new URL(url);
+    const host = parsed.hostname.replace(/^www\./, '');
+    const path = parsed.pathname.replace(/\/$/, '').replace(/^\//, '');
+    if (path) {
+        // e.g. svelte.dev/docs → "svelte-docs"
+        const hostPart = host.split('.')[0];
+        const pathPart = path.split('/').slice(0, 2).join('-');
+        return `${hostPart}-${pathPart}`.toLowerCase().replace(/[^a-z0-9-]/g, '-');
+    }
+    // Just the hostname
+    return host.replace(/\./g, '-').toLowerCase();
+}
+/** Generate a display name from the slug */
+function generateDisplayName(name) {
+    return name
+        .split('-')
+        .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+        .join(' ');
+}

package/dist/storage/db.js

@@ -0,0 +1,228 @@
+// src/storage/db.ts — SQLite + FTS5 storage layer (bun:sqlite)
+import { Database as BunDatabase } from 'bun:sqlite';
+import { resolve } from 'path';
+import { mkdirSync } from 'fs';
+import { homedir } from 'os';
+export class Database {
+    db;
+    init() {
+        const dir = process.env.DOCSHARK_DATA_DIR || resolve(homedir(), '.docshark');
+        mkdirSync(dir, { recursive: true });
+        this.db = new BunDatabase(resolve(dir, 'docshark.db'));
+        this.db.run('PRAGMA journal_mode = WAL');
+        this.db.run('PRAGMA foreign_keys = ON');
+        this.migrate();
+    }
+    /** Expose raw DB for search engine direct queries */
+    raw() {
+        return this.db;
+    }
+    migrate() {
+        this.db.run(`
+      CREATE TABLE IF NOT EXISTS libraries (
+        id           TEXT PRIMARY KEY,
+        name         TEXT NOT NULL UNIQUE,
+        display_name TEXT NOT NULL,
+        url          TEXT NOT NULL,
+        version      TEXT,
+        description  TEXT,
+        status       TEXT NOT NULL DEFAULT 'pending',
+        page_count   INTEGER NOT NULL DEFAULT 0,
+        chunk_count  INTEGER NOT NULL DEFAULT 0,
+        crawl_config TEXT,
+        last_crawled_at TEXT,
+        created_at   TEXT NOT NULL DEFAULT (datetime('now')),
+        updated_at   TEXT NOT NULL DEFAULT (datetime('now'))
+      )
+    `);
+        this.db.run(`
+      CREATE TABLE IF NOT EXISTS pages (
+        id              TEXT PRIMARY KEY,
+        library_id      TEXT NOT NULL REFERENCES libraries(id) ON DELETE CASCADE,
+        url             TEXT NOT NULL,
+        path            TEXT NOT NULL,
+        title           TEXT,
+        content_markdown TEXT,
+        content_hash    TEXT,
+        headings        TEXT,
+        http_status     INTEGER,
+        last_modified   TEXT,
+        etag            TEXT,
+        created_at      TEXT NOT NULL DEFAULT (datetime('now')),
+        updated_at      TEXT NOT NULL DEFAULT (datetime('now')),
+        UNIQUE(library_id, url)
+      )
+    `);
+        this.db.run(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id              TEXT PRIMARY KEY,
+        page_id         TEXT NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
+        library_id      TEXT NOT NULL REFERENCES libraries(id) ON DELETE CASCADE,
+        content         TEXT NOT NULL,
+        heading_context TEXT,
+        chunk_index     INTEGER NOT NULL,
+        token_count     INTEGER,
+        has_code_block  INTEGER NOT NULL DEFAULT 0,
+        created_at      TEXT NOT NULL DEFAULT (datetime('now'))
+      )
+    `);
+        this.db.run(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        content,
+        heading_context,
+        content=chunks,
+        content_rowid=rowid,
+        tokenize='porter unicode61 remove_diacritics 2'
+      )
+    `);
+        // FTS5 sync triggers
+        this.db.run(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, content, heading_context)
+        VALUES (NEW.rowid, NEW.content, NEW.heading_context);
+      END
+    `);
+        this.db.run(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content, heading_context)
+        VALUES ('delete', OLD.rowid, OLD.content, OLD.heading_context);
+      END
+    `);
+        this.db.run(`
+      CREATE TABLE IF NOT EXISTS crawl_jobs (
+        id               TEXT PRIMARY KEY,
+        library_id       TEXT NOT NULL REFERENCES libraries(id) ON DELETE CASCADE,
+        status           TEXT NOT NULL DEFAULT 'queued',
+        pages_discovered INTEGER NOT NULL DEFAULT 0,
+        pages_crawled    INTEGER NOT NULL DEFAULT 0,
+        pages_failed     INTEGER NOT NULL DEFAULT 0,
+        chunks_created   INTEGER NOT NULL DEFAULT 0,
+        error_message    TEXT,
+        started_at       TEXT,
+        completed_at     TEXT,
+        created_at       TEXT NOT NULL DEFAULT (datetime('now'))
+      )
+    `);
+    }
+    // ──────────────────────────────────────
+    // Library CRUD
+    // ──────────────────────────────────────
+    addLibrary(lib) {
+        return this.db
+            .prepare(`INSERT INTO libraries (id, name, display_name, url, version, crawl_config)
+         VALUES (?, ?, ?, ?, ?, ?)`)
+            .run(lib.id, lib.name, lib.displayName, lib.url, lib.version ?? null, lib.crawlConfig ? JSON.stringify(lib.crawlConfig) : null);
+    }
+    listLibraries(status) {
+        if (status && status !== 'all') {
+            return this.db.prepare('SELECT * FROM libraries WHERE status = ?').all(status);
+        }
+        return this.db.prepare('SELECT * FROM libraries ORDER BY name').all();
+    }
+    getLibraryByName(name) {
+        return this.db.prepare('SELECT * FROM libraries WHERE name = ?').get(name);
+    }
+    getLibraryById(id) {
+        return this.db.prepare('SELECT * FROM libraries WHERE id = ?').get(id);
+    }
+    removeLibrary(id) {
+        return this.db.prepare('DELETE FROM libraries WHERE id = ?').run(id);
+    }
+    updateLibraryStatus(id, status) {
+        return this.db
+            .prepare('UPDATE libraries SET status = ?, updated_at = datetime("now") WHERE id = ?')
+            .run(status, id);
+    }
+    updateLibraryStats(id, pageCount, chunkCount) {
+        return this.db
+            .prepare(`UPDATE libraries 
+         SET page_count = ?, chunk_count = ?, last_crawled_at = datetime('now'), updated_at = datetime('now')
+         WHERE id = ?`)
+            .run(pageCount, chunkCount, id);
+    }
+    // ──────────────────────────────────────
+    // Page CRUD
+    // ──────────────────────────────────────
+    upsertPage(page) {
+        this.db
+            .prepare(`INSERT INTO pages (id, library_id, url, path, title, content_markdown, content_hash, headings)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(library_id, url) DO UPDATE SET
+           title = excluded.title,
+           content_markdown = excluded.content_markdown,
+           content_hash = excluded.content_hash,
+           headings = excluded.headings,
+           updated_at = datetime('now')`)
+            .run(page.id, page.libraryId, page.url, page.path, page.title, page.contentMarkdown, page.contentHash, JSON.stringify(page.headings));
+        const row = this.db.prepare('SELECT id FROM pages WHERE library_id = ? AND url = ?').get(page.libraryId, page.url);
+        return row.id;
+    }
+    getPage(opts) {
+        if (opts.url) {
+            return this.db.prepare('SELECT * FROM pages WHERE url = ?').get(opts.url);
+        }
+        if (opts.library && opts.path) {
+            return this.db
+                .prepare(`SELECT p.* FROM pages p
+           JOIN libraries l ON p.library_id = l.id
+           WHERE l.name = ? AND p.path = ?`)
+                .get(opts.library, opts.path);
+        }
+        return undefined;
+    }
+    getPagesByLibrary(libraryId) {
+        return this.db
+            .prepare('SELECT * FROM pages WHERE library_id = ? ORDER BY path')
+            .all(libraryId);
+    }
+    // ──────────────────────────────────────
+    // Chunk CRUD
+    // ──────────────────────────────────────
+    insertChunks(chunks) {
+        const insert = this.db.prepare(`INSERT INTO chunks (id, page_id, library_id, content, heading_context, chunk_index, token_count, has_code_block)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
+        const tx = this.db.transaction(() => {
+            for (const c of chunks) {
+                insert.run(c.id, c.pageId, c.libraryId, c.content, c.headingContext, c.chunkIndex, c.tokenCount, c.hasCodeBlock ? 1 : 0);
+            }
+        });
+        tx();
+    }
+    deleteChunksByPage(pageId) {
+        this.db.prepare('DELETE FROM chunks WHERE page_id = ?').run(pageId);
+    }
+    // ──────────────────────────────────────
+    // Crawl Jobs
+    // ──────────────────────────────────────
+    createJob(job) {
+        this.db
+            .prepare('INSERT INTO crawl_jobs (id, library_id) VALUES (?, ?)')
+            .run(job.id, job.libraryId);
+        return this.db.prepare('SELECT * FROM crawl_jobs WHERE id = ?').get(job.id);
+    }
+    getJob(id) {
+        return this.db.prepare('SELECT * FROM crawl_jobs WHERE id = ?').get(id);
+    }
+    updateJob(id, updates) {
+        const sets = [];
+        const values = [];
+        for (const [key, value] of Object.entries(updates)) {
+            sets.push(`${key} = ?`);
+            values.push(value);
+        }
+        if (sets.length === 0)
+            return;
+        values.push(id);
+        this.db.prepare(`UPDATE crawl_jobs SET ${sets.join(', ')} WHERE id = ?`).run(...values);
+    }
+    listJobs(libraryId) {
+        if (libraryId) {
+            return this.db
+                .prepare('SELECT * FROM crawl_jobs WHERE library_id = ? ORDER BY created_at DESC')
+                .all(libraryId);
+        }
+        return this.db
+            .prepare('SELECT * FROM crawl_jobs ORDER BY created_at DESC')
+            .all();
+    }
+}

package/dist/storage/search.js

@@ -0,0 +1,49 @@
+export class SearchEngine {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    search(query, opts = {}) {
+        const limit = opts.limit ?? 5;
+        const ftsQuery = this.sanitizeQuery(query);
+        if (!ftsQuery)
+            return [];
+        try {
+            const stmt = this.db.raw().prepare(`
+        SELECT
+          c.content,
+          c.heading_context,
+          c.has_code_block,
+          c.token_count,
+          p.url   AS page_url,
+          p.title AS page_title,
+          l.name  AS library_name,
+          l.display_name AS library_display_name,
+          bm25(chunks_fts, 1.0, 0.5) AS relevance_score
+        FROM chunks_fts
+        JOIN chunks c ON chunks_fts.rowid = c.rowid
+        JOIN pages p  ON c.page_id = p.id
+        JOIN libraries l ON c.library_id = l.id
+        WHERE chunks_fts MATCH ?
+          AND (? IS NULL OR l.name = ?)
+        ORDER BY relevance_score
+        LIMIT ?
+      `);
+            return stmt.all(ftsQuery, opts.library ?? null, opts.library ?? null, limit);
+        }
+        catch (err) {
+            // FTS5 query might fail with bad syntax — return empty
+            console.warn(`[DocShark] Search failed:`, err.message);
+            return [];
+        }
+    }
+    sanitizeQuery(query) {
+        // Remove FTS5 special operators for safety, wrap terms in quotes
+        return query
+            .replace(/['"]/g, '')
+            .split(/\s+/)
+            .filter(Boolean)
+            .map((term) => `"${term}"`)
+            .join(' OR ');
+    }
+}