docshark 0.1.22 → 0.1.24

package/README.md

@@ -160,6 +160,16 @@ bun run src/cli.ts start --port 6380
 bun run src/cli.ts list
 ```
 
+### Tests
+
+Run the regression suite before merging or publishing changes:
+
+```bash
+bun test scripts/*.test.ts
+```
+
+The suite covers storage and migrations, library management, extraction, chunking, search, crawl helpers, API routes, and MCP tool wrappers.
+
 ## 🔄 Versioning & Changelog
 
 This project uses [Google's Release Please](https://github.com/googleapis/release-please) to automate versioning and changelog generation.

package/dist/api/router.js

@@ -38,6 +38,13 @@ export function createApiRouter(deps) {
                     const results = deps.searchEngine.search(q, { library, limit });
                     return json(results);
                 }
+                // POST /api/search/batch
+                if (method === 'POST' && path === '/search/batch') {
+                    const body = await request.json();
+                    const requests = Array.isArray(body?.requests) ? body.requests : [];
+                    const results = deps.searchEngine.searchMany(requests);
+                    return json(results);
+                }
                 // GET /api/crawls
                 if (method === 'GET' && path === '/crawls') {
                     const libraryId = url.searchParams.get('library_id') || undefined;

package/dist/cli.js

@@ -5,7 +5,7 @@ import { startHttpServer } from "./http.js";
 import { StdioTransport } from "@tmcp/transport-stdio";
 import { server, db, searchEngine, libraryService } from "./server.js";
 import { maybeNotifyAboutUpdate, runUpdateCommand } from "./cli-update.js";
-import { formatSearchResults } from "./search/format-results.js";
+import { formatBatchSearchResults, formatSearchResults, } from "./search/format-results.js";
 import { VERSION } from "./version.js";
 const useColor = process.stdout.isTTY;
 const color = {
@@ -163,6 +163,24 @@ cli
     }
     console.log(`\n${formatSearchResults(query, results)}\n`);
 });
+cli
+    .command("search-batch [...queries]", "Search multiple documentation queries")
+    .option("-l, --library <name>", "Filter all queries by library")
+    .option("-m, --limit <n>", "Max results per query", { default: "5" })
+    .action(async (queries, opts) => {
+    await maybeNotifyForCommand("search-batch");
+    if (!Array.isArray(queries) || queries.length === 0) {
+        console.error("\n❌ Please provide at least one query.\n");
+        process.exit(1);
+    }
+    db.init();
+    const results = searchEngine.searchMany(queries.map((query) => ({
+        query,
+        library: opts.library,
+        limit: parseInt(opts.limit),
+    })));
+    console.log(`\n${formatBatchSearchResults(results)}\n`);
+});
 cli
     .command("list", "List indexed libraries")
     .alias("l")

package/dist/index.d.ts

@@ -2,3 +2,5 @@ export * from "./server.js";
 export * from "./types.js";
 export * from "./version.js";
 export * from "./http.js";
+export * from "./tools/search-docs.js";
+export * from "./tools/search-docs-batch.js";

package/dist/index.js

@@ -2,3 +2,5 @@ export * from "./server.js";
 export * from "./types.js";
 export * from "./version.js";
 export * from "./http.js";
+export * from "./tools/search-docs.js";
+export * from "./tools/search-docs-batch.js";

package/dist/search/format-results.d.ts

@@ -1,2 +1,3 @@
-import type { SearchResult } from './types.js';
+import type { BatchSearchResult, SearchResult } from './types.js';
 export declare function formatSearchResults(query: string, results: SearchResult[]): string;
+export declare function formatBatchSearchResults(results: BatchSearchResult[]): string;

package/dist/search/format-results.js

@@ -5,19 +5,32 @@ function formatReasons(reasons) {
     }
     return `**Why this ranked highly:** ${reasons.join(', ')}\n\n`;
 }
-export function formatSearchResults(query, results) {
-    const formatted = results
+function formatResultBlocks(results) {
+    return results
         .map((result, index) => {
         let block = `### ${index + 1}. ${result.page_title} — ${result.library_display_name}\n`;
         block += `**Source:** ${result.page_url}\n`;
         if (result.heading_context.trim().length > 0) {
             block += `**Section:** ${result.heading_context}\n`;
         }
-        // Sanitize content to prevent prompt injection
         const sanitizedContent = sanitizeDocContent(result.content);
         block += `${formatReasons(result.reasons)}${sanitizedContent}`;
         return block;
     })
         .join('\n\n---\n\n');
-    return `## Results for "${query}"\n\n${formatted}`;
+}
+export function formatSearchResults(query, results) {
+    return `## Results for "${query}"\n\n${formatResultBlocks(results)}`;
+}
+export function formatBatchSearchResults(results) {
+    const formatted = results
+        .map((result, index) => {
+        const librarySuffix = result.library ? ` — ${result.library}` : '';
+        if (result.results.length === 0) {
+            return `### ${index + 1}. ${result.query}${librarySuffix}\n\nNo results found.`;
+        }
+        return `### ${index + 1}. ${result.query}${librarySuffix}\n\n${formatResultBlocks(result.results)}`;
+    })
+        .join('\n\n***\n\n');
+    return `## Batch Search Results\n\n${formatted}`;
 }

package/dist/search/query-planner.d.ts

@@ -2,6 +2,10 @@ import type { SearchPlan } from "./types.js";
 export declare function normalizeSearchText(value: string): string;
 export declare class QueryPlanner {
     build(query: string, library?: string): SearchPlan;
+    private buildDecomposedQueries;
+    private shouldDecompose;
+    private segmentBySeparators;
+    private chunkKeywords;
     private detectIntent;
     private extractVersion;
 }

package/dist/search/query-planner.js

@@ -44,18 +44,60 @@ export class QueryPlanner {
             .map((token) => sanitizeToken(token))
             .filter(Boolean);
         const filteredKeywords = Array.from(new Set(rawTokens.filter((token) => token.length > 1 && !STOP_WORDS.has(token))));
+        const keywords = filteredKeywords.length > 0
+            ? filteredKeywords
+            : Array.from(new Set(rawTokens));
         return {
             original_query: query,
             normalized_query: normalizedQuery,
             intent: this.detectIntent(normalizedQuery),
-            keywords: filteredKeywords.length > 0
-                ? filteredKeywords
-                : Array.from(new Set(rawTokens)),
+            keywords,
             phrases: PHRASE_HINTS.filter((phrase) => normalizedQuery.includes(phrase)),
+            decomposed_queries: this.buildDecomposedQueries(normalizedQuery, keywords),
             requested_library: library,
             requested_version: this.extractVersion(normalizedQuery),
         };
     }
+    buildDecomposedQueries(normalizedQuery, keywords) {
+        if (!this.shouldDecompose(normalizedQuery, keywords)) {
+            return [];
+        }
+        const segmentedQueries = this.segmentBySeparators(normalizedQuery);
+        if (segmentedQueries.length > 1) {
+            return segmentedQueries;
+        }
+        return this.chunkKeywords(keywords);
+    }
+    shouldDecompose(normalizedQuery, keywords) {
+        if (keywords.length < 4) {
+            return false;
+        }
+        if (/[;,]/.test(normalizedQuery) || /\b(and|or|then|plus|with)\b/.test(normalizedQuery)) {
+            return true;
+        }
+        return keywords.length >= 7;
+    }
+    segmentBySeparators(normalizedQuery) {
+        const segments = normalizedQuery
+            .split(/(?:,|;|\band\b|\bor\b|\bthen\b|\bplus\b)/g)
+            .map((segment) => normalizeSearchText(segment))
+            .filter((segment) => segment.split(/\s+/).length >= 2);
+        return Array.from(new Set(segments)).slice(0, 4);
+    }
+    chunkKeywords(keywords) {
+        const targetBranches = Math.min(4, Math.ceil(keywords.length / 2));
+        const chunkSize = Math.min(3, Math.max(2, Math.ceil(keywords.length / targetBranches)));
+        const chunks = [];
+        for (let index = 0; index < keywords.length; index += chunkSize) {
+            const group = keywords.slice(index, index + chunkSize);
+            if (group.length === 1 && chunks.length > 0) {
+                chunks[chunks.length - 1] += ` ${group[0]}`;
+                continue;
+            }
+            chunks.push(group.join(" "));
+        }
+        return Array.from(new Set(chunks)).slice(0, 4);
+    }
     detectIntent(query) {
         if (query.includes("getting started") ||
             query.includes("quickstart") ||

package/dist/search/types.d.ts

@@ -3,12 +3,18 @@ export interface SearchOptions {
     library?: string;
     limit?: number;
 }
+export interface BatchSearchRequest {
+    query: string;
+    library?: string;
+    limit?: number;
+}
 export interface SearchPlan {
     original_query: string;
     normalized_query: string;
     intent: SearchIntent;
     keywords: string[];
     phrases: string[];
+    decomposed_queries: string[];
     requested_version?: string;
     requested_library?: string;
 }
@@ -31,3 +37,9 @@ export interface SearchResult extends SearchCandidate {
     path_type: string;
     version_tag: string | null;
 }
+export interface BatchSearchResult {
+    query: string;
+    library?: string;
+    limit: number;
+    results: SearchResult[];
+}

package/dist/server.js

@@ -5,7 +5,7 @@ import * as v from "valibot";
 import { tool } from "tmcp/utils";
 import { Database } from "./storage/db.js";
 import { SearchEngine } from "./storage/search.js";
-import { formatSearchResults } from "./search/format-results.js";
+import { formatBatchSearchResults, formatSearchResults, } from "./search/format-results.js";
 import { LibraryService } from "./services/library.js";
 import { JobManager } from "./jobs/manager.js";
 import { VERSION } from "./version.js";
@@ -55,6 +55,30 @@ server.tool({
         return tool.text(`❌ Error: ${message}`);
     }
 });
+server.tool({
+    name: "search_docs_batch",
+    description: "Run multiple documentation searches in one call. Use this for repeated or decomposed lookups.",
+    annotations: {
+        readOnlyHint: true,
+        idempotentHint: true,
+    },
+    schema: v.object({
+        requests: v.pipe(v.array(v.object({
+            query: v.pipe(v.string(), v.description("Search query. Use natural language.")),
+            library: v.optional(v.pipe(v.string(), v.description("Filter to a specific library."))),
+            limit: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(20)), 5),
+        })), v.minLength(1), v.maxLength(10)),
+    }),
+}, async ({ requests }) => {
+    try {
+        const results = searchEngine.searchMany(requests);
+        return tool.text(formatBatchSearchResults(results));
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : "Search failed";
+        return tool.text(`❌ Error: ${message}`);
+    }
+});
 function requireValue(value, message) {
     if (value === undefined || value === null || value === "") {
         throw new Error(message);

package/dist/storage/db.d.ts

@@ -2,6 +2,8 @@ import { Database as BunDatabase } from "bun:sqlite";
 import type { Library, Page, CrawlJob } from "../types.js";
 export declare class Database {
     private db;
+    private hasColumn;
+    private ensureColumn;
     init(): void;
     /** Expose raw DB for search engine direct queries */
     raw(): BunDatabase;

package/dist/storage/db.js

@@ -5,6 +5,17 @@ import { mkdirSync } from "fs";
 import { homedir } from "os";
 export class Database {
     db;
+    hasColumn(tableName, columnName) {
+        const columns = this.db
+            .prepare(`PRAGMA table_info(${tableName})`)
+            .all();
+        return columns.some((column) => column.name === columnName);
+    }
+    ensureColumn(tableName, columnName, definition) {
+        if (!this.hasColumn(tableName, columnName)) {
+            this.db.run(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${definition}`);
+        }
+    }
     init() {
         const dir = process.env.DOCSHARK_DATA_DIR || resolve(homedir(), ".docshark");
         mkdirSync(dir, { recursive: true });
@@ -104,6 +115,7 @@ export class Database {
         created_at       TEXT NOT NULL DEFAULT (datetime('now'))
       )
     `);
+        this.ensureColumn("crawl_jobs", "session_id", "TEXT");
     }
     // ──────────────────────────────────────
     // Library CRUD

package/dist/storage/search.d.ts

@@ -1,11 +1,15 @@
 import type { Database } from "./db.js";
-import type { SearchOptions, SearchResult } from "../search/types.js";
-export type { SearchOptions, SearchResult } from "../search/types.js";
+import type { BatchSearchRequest, BatchSearchResult, SearchOptions, SearchResult } from "../search/types.js";
+export type { BatchSearchRequest, BatchSearchResult, SearchOptions, SearchResult, } from "../search/types.js";
 export declare class SearchEngine {
     private db;
     private planner;
     constructor(db: Database);
     search(query: string, opts?: SearchOptions): SearchResult[];
+    searchMany(requests: BatchSearchRequest[]): BatchSearchResult[];
+    private searchWithPlan;
+    private expandPlans;
+    private searchSingle;
     private fetchCandidates;
     private buildFtsQuery;
     private quoteTerm;

package/dist/storage/search.js

@@ -8,16 +8,98 @@ export class SearchEngine {
     search(query, opts = {}) {
         const limit = opts.limit ?? 5;
         const plan = this.planner.build(query, opts.library);
-        const ftsQuery = this.buildFtsQuery(plan);
+        return this.searchWithPlan(plan, opts.library, limit);
+    }
+    searchMany(requests) {
+        return requests.map((request) => {
+            const limit = request.limit ?? 5;
+            return {
+                query: request.query,
+                library: request.library,
+                limit,
+                results: this.search(request.query, {
+                    library: request.library,
+                    limit,
+                }),
+            };
+        });
+    }
+    searchWithPlan(plan, library, limit) {
+        const branchPlans = this.expandPlans(plan, library);
+        if (branchPlans.length === 1) {
+            return this.searchSingle(branchPlans[0], plan, library, limit);
+        }
+        const branchLimit = Math.min(Math.max(limit * 2, 6), 12);
+        const bestByChunk = new Map();
+        for (const [branchIndex, branchPlan] of branchPlans.entries()) {
+            const branchResults = this.searchSingle(branchPlan, plan, library, branchLimit);
+            for (const branchResult of branchResults) {
+                const chunkKey = `${branchResult.page_url}#${branchResult.chunk_index}`;
+                const scoreBoost = branchIndex === 0 ? 0 : 0.03;
+                const adjustedScore = Number((branchResult.rerank_score + scoreBoost).toFixed(6));
+                const existing = bestByChunk.get(chunkKey);
+                if (!existing) {
+                    bestByChunk.set(chunkKey, {
+                        ...branchResult,
+                        rerank_score: adjustedScore,
+                        branch_hits: 1,
+                    });
+                    continue;
+                }
+                existing.branch_hits += 1;
+                if (adjustedScore > existing.rerank_score) {
+                    bestByChunk.set(chunkKey, {
+                        ...branchResult,
+                        rerank_score: adjustedScore,
+                        branch_hits: existing.branch_hits,
+                    });
+                }
+            }
+        }
+        const aggregated = Array.from(bestByChunk.values())
+            .map(({ branch_hits, ...result }) => {
+            const reasons = [...result.reasons];
+            if (branch_hits > 1) {
+                reasons.push("matched multiple focused subqueries");
+            }
+            return {
+                ...result,
+                rerank_score: Number((result.rerank_score + Math.min((branch_hits - 1) * 0.05, 0.15)).toFixed(6)),
+                reasons: Array.from(new Set(reasons)).slice(0, 4),
+            };
+        })
+            .sort((left, right) => {
+            if (right.rerank_score !== left.rerank_score) {
+                return right.rerank_score - left.rerank_score;
+            }
+            return left.lexical_score - right.lexical_score;
+        });
+        return this.collapseDuplicates(plan, aggregated).slice(0, limit);
+    }
+    expandPlans(plan, library) {
+        const plans = [plan];
+        const seen = new Set([plan.normalized_query]);
+        for (const subquery of plan.decomposed_queries) {
+            const subqueryPlan = this.planner.build(subquery, library);
+            if (seen.has(subqueryPlan.normalized_query)) {
+                continue;
+            }
+            seen.add(subqueryPlan.normalized_query);
+            plans.push(subqueryPlan);
+        }
+        return plans;
+    }
+    searchSingle(retrievalPlan, scoringPlan, library, limit) {
+        const ftsQuery = this.buildFtsQuery(retrievalPlan);
         if (!ftsQuery)
             return [];
         try {
-            const candidates = this.fetchCandidates(ftsQuery, opts.library, limit);
+            const candidates = this.fetchCandidates(ftsQuery, library, limit);
             if (candidates.length === 0) {
                 return [];
             }
-            const reranked = this.rerank(plan, candidates);
-            return this.collapseDuplicates(plan, reranked).slice(0, limit);
+            const reranked = this.rerank(scoringPlan, candidates);
+            return this.collapseDuplicates(scoringPlan, reranked).slice(0, limit);
         }
         catch (err) {
             console.warn(`[DocShark] Search failed:`, err.message);

package/dist/tools/search-docs-batch.d.ts

@@ -0,0 +1,35 @@
+import * as v from 'valibot';
+import type { SearchEngine } from '../storage/search.js';
+export declare function createSearchDocsBatchTool(searchEngine: SearchEngine): {
+    definition: {
+        name: "search_docs_batch";
+        description: string;
+        schema: v.ObjectSchema<{
+            readonly requests: v.SchemaWithPipe<readonly [v.ArraySchema<v.ObjectSchema<{
+                readonly query: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.DescriptionAction<string, "The search query. Use natural language or specific terms.">]>;
+                readonly library: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.DescriptionAction<string, "Optional library filter for this query.">]>, undefined>;
+                readonly limit: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 1, undefined>, v.MaxValueAction<number, 20, undefined>, v.DescriptionAction<number, "Max results to return for this query. Default: 5.">]>, 5>;
+            }, undefined>, undefined>, v.MinLengthAction<{
+                query: string;
+                library?: string | undefined;
+                limit: number;
+            }[], 1, undefined>, v.MaxLengthAction<{
+                query: string;
+                library?: string | undefined;
+                limit: number;
+            }[], 10, undefined>]>;
+        }, undefined>;
+    };
+    handler: ({ requests }: {
+        requests: Array<{
+            query: string;
+            library?: string;
+            limit?: number;
+        }>;
+    }) => Promise<{
+        content: {
+            type: "text";
+            text: string;
+        }[];
+    }>;
+};

package/dist/tools/search-docs-batch.js

@@ -0,0 +1,23 @@
+import * as v from 'valibot';
+import { tool } from 'tmcp/utils';
+import { formatBatchSearchResults } from '../search/format-results.js';
+export function createSearchDocsBatchTool(searchEngine) {
+    return {
+        definition: {
+            name: 'search_docs_batch',
+            description: 'Run multiple documentation searches in one call. ' +
+                'Useful when you need several focused lookups against one library or across a small set of libraries.',
+            schema: v.object({
+                requests: v.pipe(v.array(v.object({
+                    query: v.pipe(v.string(), v.description('The search query. Use natural language or specific terms.')),
+                    library: v.optional(v.pipe(v.string(), v.description('Optional library filter for this query.'))),
+                    limit: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(20), v.description('Max results to return for this query. Default: 5.')), 5),
+                })), v.minLength(1), v.maxLength(10)),
+            }),
+        },
+        handler: async ({ requests }) => {
+            const results = searchEngine.searchMany(requests);
+            return tool.text(formatBatchSearchResults(results));
+        },
+    };
+}

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.1.22";
+export declare const VERSION = "0.1.24";

package/dist/version.js

@@ -1,2 +1,2 @@
 // This file is automatically updated by the version sync script.
-export const VERSION = '0.1.22';
+export const VERSION = '0.1.24';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docshark",
-  "version": "0.1.22",
+  "version": "0.1.24",
   "description": "🦈 Documentation MCP Server — scrape, index, and search any doc website",
   "type": "module",
   "main": "./dist/index.js",
@@ -36,6 +36,7 @@
     "dev": "bun run --watch src/cli.ts start",
     "cli": "bun run src/cli.ts",
     "check": "tsc --noEmit",
+    "test": "bun test scripts/*.test.ts",
     "sync:version": "bun run src/scripts/sync-version.ts",
     "build": "bun run sync:version && rm -rf dist && tsc && chmod +x dist/cli.js",
     "prepublishOnly": "bun run build",