@shoalrun/cli 0.0.0-stage → 0.1.0-alpha.11

package/dist/search-handler.js

@@ -0,0 +1,111 @@
+import { validateReaderSelection } from './reader-filters.js';
+import { searchSnippetFromParagraphs } from './search-snippet.js';
+import { identifierRescue, searchIdentifiers } from './search-identifier.js';
+import { createSearchResultSelector } from './search-results.js';
+export function createSearchHandler(options) {
+    const selectResults = createSearchResultSelector(options.retriever);
+    const json = (status, value) => new Response(JSON.stringify(value), { status, headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } });
+    return async (request) => {
+        if (request.method !== 'POST')
+            return json(405, { error: 'Use POST.' });
+        const url = new URL(request.url);
+        if (!options.hosts.includes(url.host) || request.headers.get('origin') !== url.origin)
+            return json(403, { error: 'Search is available only from this preview.' });
+        if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json'))
+            return json(415, { error: 'Send JSON.' });
+        let parsed;
+        try {
+            parsed = await request.json();
+        }
+        catch {
+            return json(400, { error: 'Invalid JSON.' });
+        }
+        if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed))
+            return json(400, { error: 'Invalid search request.' });
+        const body = parsed;
+        const query = body.query;
+        const variant = body.variant;
+        const limit = body.limit === undefined ? 8 : body.limit;
+        if (typeof query !== 'string' || !query.trim() || query.length > 300 || typeof variant !== 'string' || !options.retriever.variants().includes(variant) || !Number.isInteger(limit) || limit < 1 || limit > 20)
+            return json(400, { error: 'Invalid search request.' });
+        let readerSelection;
+        try {
+            if (body.readerSelection !== undefined) {
+                if (!options.readerFilters)
+                    throw new Error('No reader filters configured');
+                readerSelection = validateReaderSelection(body.readerSelection, options.readerFilters, 'readerSelection');
+            }
+        }
+        catch {
+            return json(400, { error: 'Invalid reader view.' });
+        }
+        const searchOptions = { variant, limit: limit, readerSelection };
+        const caller = options.clientKey?.(request) ?? 'shared';
+        if (options.rateLimiter) {
+            const taken = await options.rateLimiter.take(caller);
+            if (!taken.allowed)
+                return new Response(JSON.stringify({ error: 'Too many searches. Wait a moment and try again.' }), { status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store', 'Retry-After': String(taken.retryAfterSeconds) } });
+        }
+        try {
+            let vector;
+            let fallbackReason = 'embeddings-unavailable';
+            if (options.embedQuery && options.retriever.hasVectors(searchOptions)) {
+                const controller = new AbortController();
+                const signal = AbortSignal.any([controller.signal, request.signal]);
+                let timer;
+                try {
+                    vector = await Promise.race([
+                        (async () => {
+                            let allowed;
+                            try {
+                                allowed = !options.budget || await options.budget.reserve(caller);
+                            }
+                            catch (error) {
+                                fallbackReason = 'budget-unavailable';
+                                throw error;
+                            }
+                            signal.throwIfAborted();
+                            if (!allowed) {
+                                fallbackReason = 'budget-denied';
+                                return undefined;
+                            }
+                            fallbackReason = 'provider-error';
+                            return options.embedQuery(query.trim(), signal);
+                        })(),
+                        new Promise((_, reject) => {
+                            timer = setTimeout(() => {
+                                fallbackReason = 'deadline';
+                                const error = new Error('Semantic search timed out');
+                                controller.abort(error);
+                                reject(error);
+                            }, options.semanticTimeoutMs ?? 3000);
+                        })
+                    ]);
+                }
+                catch {
+                    // Provider failures and deadlines must not hide the local lexical results.
+                }
+                finally {
+                    clearTimeout(timer);
+                }
+            }
+            if (!vector && fallbackReason)
+                options.onFallback?.(fallbackReason);
+            // Rank pages after inspecting their passages. A large gallery can otherwise
+            // consume the entire candidate window before page deduplication happens.
+            const retrievalOptions = { ...searchOptions, limit: Number.MAX_SAFE_INTEGER };
+            let hits = vector
+                ? options.retriever.vector(vector, retrievalOptions)
+                : options.retriever.lexical(query.trim(), retrievalOptions);
+            if (vector && limit >= 3 && searchIdentifiers(query).length) {
+                const lexical = options.retriever.lexical(query.trim(), retrievalOptions);
+                hits = identifierRescue(query.trim(), hits, lexical);
+            }
+            const results = selectResults(hits, query.trim(), searchOptions).map(({ chunk, literalSummary }) => ({ url: chunk.url, title: chunk.title, section: chunk.headingPath.at(-1), summary: literalSummary ?? searchSnippetFromParagraphs(chunk.search.paragraphs, query.trim(), options.pageSummaries?.get(`${chunk.variant}:${chunk.docId}`)) }));
+            return json(200, { results, mode: vector ? 'vector' : 'lexical', ...(!vector ? { fallbackReason } : {}) });
+        }
+        catch {
+            return json(503, { error: 'Semantic search is unavailable.' });
+        }
+    };
+}

package/dist/search-identifier.d.ts

@@ -0,0 +1,3 @@
+import type { Hit } from './retrieval.js';
+export declare function searchIdentifiers(query: string): string[];
+export declare function identifierRescue(query: string, semantic: Hit[], lexical: Hit[]): Hit[];

package/dist/search-identifier.js

@@ -0,0 +1,25 @@
+import { searchParagraphs } from './search-snippet.js';
+// Reserve at most one search-box result for a literal configuration key or flag
+// explained in prose. Tables, code listings and identifier prefixes do not qualify.
+export function searchIdentifiers(query) {
+    const identifiers = query.match(/\b[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+\b|--[a-z][\w-]*/g) ?? [];
+    // File names and URLs need ordinary retrieval, not a config-key shortcut.
+    return [...new Set(identifiers.filter(id => !/\.(?:txt|json|ya?ml|mdx?|[cm]?js|tsx?|html?|css|svg|png|jpe?g|com|org|net)$/i.test(id)))];
+}
+export function identifierRescue(query, semantic, lexical) {
+    const identifiers = searchIdentifiers(query);
+    if (!identifiers.length)
+        return semantic;
+    const matches = identifiers.map(id => new RegExp(`(?<![\\w$.-])${id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w$-]|\\.[\\w$])`, 'i'));
+    const candidate = lexical.find(hit => (hit.chunk.search?.paragraphs ?? searchParagraphs(hit.chunk.text)).some(p => matches.every(re => re.test(p))));
+    if (!candidate)
+        return semantic;
+    const pages = [...new Set(semantic.map(h => h.chunk.docId))];
+    if (pages.slice(0, 3).includes(candidate.chunk.docId))
+        return semantic;
+    const remaining = semantic.filter(h => h.chunk.docId !== candidate.chunk.docId);
+    const seen = new Set();
+    const at = remaining.findIndex(hit => { seen.add(hit.chunk.docId); return seen.size === 3; });
+    const position = at < 0 ? remaining.length : at;
+    return [...remaining.slice(0, position), candidate, ...remaining.slice(position)];
+}

package/dist/search-literal.d.ts

@@ -0,0 +1 @@
+export declare function literalSearch(query: string): ((lines: string[]) => string | undefined) | undefined;

package/dist/search-literal.js

@@ -0,0 +1,28 @@
+import { searchIdentifiers } from './search-identifier.js';
+const escape = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const boundary = (value) => new RegExp(`(?<![\\w$.-])${escape(value)}(?![\\w$-]|\\.[\\w$])`, 'i');
+// Only explicit technical syntax gets this exception. Ordinary topic words must
+// not make code comments or link directories outrank explanatory content.
+export function literalSearch(query) {
+    const trimmed = query.trim();
+    const quoted = /^(["'`])[\s\S]+\1$/.test(trimmed);
+    const phrase = (quoted ? trimmed.slice(1, -1) : trimmed).replace(/\s+/g, ' ');
+    const identifiers = searchIdentifiers(phrase);
+    const technical = quoted || identifiers.length || /[a-z][A-Z]|[<>{}=]|->|\b\w+:\w+|(?:^|\s)-[A-Za-z]|^[\w./@-]+\.[A-Za-z]{1,6}$/.test(phrase)
+        || /^(?:npm|npx|pnpm|yarn|bun|node|curl|git|shoalrun)\s+\S/i.test(phrase);
+    if (!technical || !phrase)
+        return undefined;
+    // A complete command is matched as a phrase, even when it contains a flag.
+    const command = /^(?:npm|npx|pnpm|yarn|bun|node|curl|git|shoalrun)\s+\S/i.test(phrase);
+    const matches = (identifiers.length && !command ? identifiers : [phrase]).map(boundary);
+    return lines => {
+        const line = lines.find(line => matches.every(expression => expression.test(line)));
+        if (!line)
+            return undefined;
+        if (line.length <= 180)
+            return line;
+        const at = line.search(matches[0]);
+        const start = Math.max(0, at - 40);
+        return `${start ? '…' : ''}${line.slice(start, start + (start ? 178 : 179)).trim()}…`;
+    };
+}

package/dist/search-query.d.ts

@@ -0,0 +1,54 @@
+export type SearchOptions = {
+    channel?: string;
+    version?: string;
+    language?: string;
+    limit?: number;
+};
+export declare function queryBuiltSite(directory: string, query: string, options?: SearchOptions): Promise<{
+    status: string;
+    query: string;
+    publication: {
+        channel: string;
+        version: string;
+        language: string;
+    };
+    total: number;
+    results: never[];
+    matchMode?: undefined;
+} | {
+    status: string;
+    query: string;
+    publication: {
+        channel: string;
+        version: string;
+        language: string;
+    };
+    total: number;
+    matchMode: string;
+    results: {
+        snippet: string;
+        section?: string | undefined;
+        markdownTokens?: number | undefined;
+        markdownBytes?: number | undefined;
+        markdownTokenEncoding?: string | undefined;
+        score: number;
+        id: string;
+        type: string;
+        title: string;
+        summary: string;
+        url: string;
+        markdownUrl: string;
+        pageMarkdownUrl: string;
+        read: {
+            tool: string;
+            arguments: {
+                maxTokens?: number | undefined;
+                channel: string;
+                version: string;
+                language: string;
+                section?: string | undefined;
+                id: string;
+            };
+        };
+    }[];
+}>;

package/dist/search-query.js

@@ -0,0 +1,120 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { phraseMatches, queryUnits, validateSearchSynonyms } from './search-synonyms.js';
+const fallbackStopWords = new Set('a an the i me my we our you your how do does can could would should will is are was were be to of in on for with please what where when why which some any each several'.split(' '));
+function excerpt(value, terms) {
+    const lower = value.toLocaleLowerCase();
+    const positions = terms.map(term => lower.indexOf(term)).filter(at => at >= 0);
+    if (!positions.length)
+        return value.slice(0, 180).trim();
+    const start = Math.max(0, Math.min(...positions) - 65);
+    const end = Math.min(value.length, start + 180);
+    return `${start ? '…' : ''}${value.slice(start, end).trim()}${end < value.length ? '…' : ''}`;
+}
+export async function queryBuiltSite(directory, query, options = {}) {
+    if (!query.trim())
+        throw new Error('Search query must be nonempty text');
+    const limit = options.limit ?? 8;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100)
+        throw new Error('Search limit must be an integer from 1 to 100');
+    let index;
+    try {
+        index = JSON.parse(await readFile(path.join(directory, 'search-index.json'), 'utf8'));
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            throw new Error(`No search-index.json in ${directory}; build with the lexical provider or custom search index enabled`);
+        throw error;
+    }
+    if (index?.schemaVersion !== 1 || !Array.isArray(index.documents) || index.documents.some(doc => !doc || typeof doc.id !== 'string' || typeof doc.url !== 'string' || typeof doc.markdownUrl !== 'string' || typeof doc.title !== 'string' || typeof doc.summary !== 'string' || typeof doc.type !== 'string' || typeof doc.variant !== 'string' || typeof doc.text !== 'string' || !Array.isArray(doc.headings) || doc.headings.some(heading => !heading || typeof heading.id !== 'string' || typeof heading.title !== 'string')))
+        throw new Error('Unsupported or invalid search-index.json');
+    const synonyms = validateSearchSynonyms(index.synonyms);
+    const matching = index.documents.filter(doc => {
+        const [channel, version, language] = doc.variant.split('|');
+        return (!options.channel || channel === options.channel) && (!options.version || version === options.version) && (!options.language || language === options.language);
+    });
+    const variants = [...new Set(matching.map(doc => doc.variant))].sort();
+    if (!variants.length)
+        throw new Error('No search documents match the selected publication');
+    if (variants.length > 1)
+        throw new Error(`Search matches multiple publications (${variants.join(', ')}); pass --channel, --version, and --language`);
+    const terms = queryUnits(query, synonyms);
+    if (!terms.length) {
+        const [channel, version, language] = variants[0].split('|');
+        return { status: 'ok', query: query.trim(), publication: { channel, version, language }, total: 0, results: [] };
+    }
+    const match = (value, unit) => phraseMatches(value, unit.term) || unit.alternatives.some(term => phraseMatches(value, term));
+    // Page titles are short subject labels, often singular. Accept a plain final
+    // English -s query plural there without broadening body/heading matches.
+    const titleMatches = (value, term) => phraseMatches(value, term)
+        || ((term.split(' ').at(-1)?.length ?? 0) > 3 && /[^sui]s$/.test(term) && phraseMatches(value, term.slice(0, -1)));
+    const matchTitle = (value, unit) => titleMatches(value, unit.term) || unit.alternatives.some(term => titleMatches(value, term));
+    const weight = (value, unit, score, title = false) => {
+        const matches = title ? titleMatches : phraseMatches;
+        return matches(value, unit.term) ? score : unit.alternatives.some(term => matches(value, term)) ? score * 0.5 : 0;
+    };
+    const hits = matching.flatMap(doc => {
+        const title = doc.title.toLocaleLowerCase();
+        const summary = doc.summary.toLocaleLowerCase();
+        const body = doc.text.toLocaleLowerCase();
+        const headings = doc.headings.map(heading => heading.title.toLocaleLowerCase());
+        const matched = terms.filter(term => matchTitle(title, term) || match(summary, term) || match(body, term) || headings.some(heading => match(heading, term)));
+        if (!matched.length)
+            return [];
+        const termScore = (term) => weight(title, term, title.startsWith(term.term) ? 12 : 8, true) || weight(summary, term, 4) || (headings.some(heading => match(heading, term)) ? 3 : 0) || weight(body, term, 1);
+        let score = terms.reduce((total, term) => total + termScore(term), 0);
+        const contentScore = terms.filter(term => !fallbackStopWords.has(term.term)).reduce((total, term) => total + termScore(term), 0);
+        const prominentMatches = terms.filter(term => !fallbackStopWords.has(term.term) && (matchTitle(title, term) || match(summary, term) || headings.some(heading => match(heading, term)))).length;
+        const section = terms.every(term => matchTitle(title, term)) ? undefined : doc.headings.find((heading, at) => terms.every(term => match(headings[at], term)));
+        // Use only query terms with meaning, and require a unique matching section.
+        // Page-title terms may provide context ("search custom provider"). Never pick
+        // a section for a page-title-only query or an ambiguous heading match.
+        const meaningful = terms.filter(term => !fallbackStopWords.has(term.term) && !['use', 'using', 'find', 'read', 'show', 'tell', 'explain', 'configure', 'set', 'up', 'about'].includes(term.term));
+        const specific = meaningful.filter(term => !matchTitle(title, term));
+        const candidates = specific.length ? (doc.sections ?? []).filter(candidate => specific.every(term => match(candidate.title.toLocaleLowerCase(), term))) : [];
+        // Relevance is not context sufficiency. Require an explicit author declaration;
+        // older indexes and unreviewed sections conservatively retain the full page.
+        const selected = candidates.length === 1 && candidates[0].standalone === true ? candidates[0] : undefined;
+        // A complete multiword section-title match outranks incidental page-title words.
+        if (meaningful.length >= 2 && candidates.some(candidate => meaningful.every(term => match(candidate.title.toLocaleLowerCase(), term))))
+            score += 20;
+        const [channel, version, language] = doc.variant.split('|');
+        const tokens = selected?.markdownTokens ?? doc.markdownTokens;
+        const read = { tool: selected ? 'get_section' : 'get_page', arguments: {
+                id: doc.id, ...(selected ? { section: selected.id } : {}), channel, version, language,
+                ...(tokens !== undefined ? { maxTokens: Math.min(100_000, Math.max(4096, tokens * 2 + 1024)) } : {}),
+            } };
+        return [{ score, contentScore, prominentMatches, matched: matched.length, partialMatched: matched.filter(term => !fallbackStopWords.has(term.term)).length, id: doc.id, type: doc.type, title: doc.title, summary: doc.summary, url: section ? `${doc.url}#${section.id}` : doc.url, markdownUrl: selected?.markdownUrl ?? doc.markdownUrl, pageMarkdownUrl: doc.markdownUrl, read, ...(tokens !== undefined ? { markdownTokens: tokens, markdownBytes: selected?.markdownBytes ?? doc.markdownBytes, markdownTokenEncoding: doc.markdownTokenEncoding } : {}), ...(selected || section ? { section: selected?.title ?? section?.title } : {}), snippet: excerpt(doc.text, terms.flatMap(unit => [unit.term, ...unit.alternatives])) }];
+    });
+    const complete = hits.filter(hit => hit.matched === terms.length);
+    // Relax the all-words gate only when complete matches are entirely incidental
+    // body matches. A near-complete result must cover all but one content term and
+    // have a stronger non-filler field score. Title/summary/heading matches on any
+    // complete result preserve the original preference; filler cannot earn promotion.
+    const contentTerms = terms.filter(term => !fallbackStopWords.has(term.term)).length;
+    const bestCompleteContent = Math.max(...complete.map(hit => hit.contentScore));
+    const stronger = complete.length && contentTerms >= 3 && complete.every(hit => hit.prominentMatches === 0)
+        ? hits.filter(hit => hit.matched < terms.length && hit.partialMatched >= contentTerms - 1
+            && hit.prominentMatches > 0 && hit.contentScore > bestCompleteContent) : [];
+    const ranked = complete.length ? [...complete, ...stronger] : hits.filter(hit => hit.partialMatched > 0);
+    // In partial fallback, one extra incidental word must not outweigh a focused
+    // subject match. Compare non-filler field scores within a one-term coverage
+    // band; sparse matches remain below it, ordered by their original coverage.
+    const bestCoverage = Math.max(0, ...ranked.map(hit => hit.partialMatched));
+    const nearCoverage = Math.max(2, bestCoverage - 1);
+    ranked.sort((a, b) => {
+        if (stronger.length)
+            return b.contentScore - a.contentScore || b.score - a.score || a.title.localeCompare(b.title) || a.id.localeCompare(b.id);
+        if (!complete.length) {
+            const aNear = a.partialMatched >= nearCoverage;
+            const bNear = b.partialMatched >= nearCoverage;
+            if (aNear !== bNear)
+                return Number(bNear) - Number(aNear);
+            if (aNear && a.contentScore !== b.contentScore)
+                return b.contentScore - a.contentScore;
+        }
+        return (complete.length ? b.matched - a.matched : b.partialMatched - a.partialMatched) || b.score - a.score || a.title.localeCompare(b.title) || a.id.localeCompare(b.id);
+    });
+    const [channel, version, language] = variants[0].split('|');
+    return { status: 'ok', query: query.trim(), publication: { channel, version, language }, total: ranked.length, matchMode: stronger.length ? 'mixed' : complete.length ? 'all' : 'partial', results: ranked.slice(0, limit).map(({ matched, partialMatched, contentScore, prominentMatches, ...result }) => result) };
+}

package/dist/search-results.d.ts

@@ -0,0 +1,6 @@
+import type { Hit, LexicalOptions, Retriever } from './retrieval.js';
+type SearchHit = Hit & {
+    literalSummary?: string;
+};
+export declare function createSearchResultSelector(retriever: Retriever): (hits: Hit[], query: string, options: LexicalOptions) => SearchHit[];
+export {};

package/dist/search-results.js

@@ -0,0 +1,45 @@
+import { tokenize } from './retrieval.js';
+import { literalSearch } from './search-literal.js';
+// Select landing passages for the search box, without changing passage retrieval
+// used by Ask. A gallery's repeated examples must not crowd other pages out.
+export function createSearchResultSelector(retriever) {
+    retriever.requirePreparedSearch();
+    const normalized = (text) => tokenize(text).join(' ');
+    return (hits, query, options) => {
+        const pages = new Map();
+        const literal = literalSearch(query);
+        const literalPages = new Map();
+        if (literal) {
+            for (const hit of hits) {
+                if (literalPages.has(hit.chunk.docId))
+                    continue;
+                const literalSummary = literal(hit.chunk.search.literalLines);
+                if (literalSummary)
+                    literalPages.set(hit.chunk.docId, { ...hit, literalSummary });
+            }
+        }
+        for (const hit of hits) {
+            if (pages.has(hit.chunk.docId))
+                continue;
+            // Keep the page's retrieved position, but land on the passage that actually
+            // contains the literal. A nearby token table must not hide the correct row.
+            const exactPassage = literalPages.get(hit.chunk.docId);
+            if (exactPassage)
+                pages.set(hit.chunk.docId, { ...exactPassage, score: hit.score });
+            else if (hit.chunk.search.paragraphs.length)
+                pages.set(hit.chunk.docId, hit);
+        }
+        // An exact page-title query is a request for that topic's entry point.
+        // Ordinary semantic queries retain their ranking; incidental words do not qualify.
+        const exact = hits.find(hit => normalized(hit.chunk.title) === normalized(query));
+        if (exact) {
+            const introduction = retriever.pageIntroduction(exact.chunk.docId, options);
+            const landing = introduction ? { chunk: introduction, score: exact.score } : pages.get(exact.chunk.docId);
+            if (landing) {
+                pages.delete(exact.chunk.docId);
+                return [landing, ...pages.values()].slice(0, options.limit ?? 8);
+            }
+        }
+        return [...pages.values()].slice(0, options.limit ?? 8);
+    };
+}

package/dist/search-snippet.d.ts

@@ -0,0 +1,11 @@
+export declare function searchParagraphs(markdown: string): string[];
+export declare function searchPassageParagraphs(markdown: string): string[];
+export declare function searchLiteralLines(markdown: string): string[];
+export type PreparedSearchPassage = {
+    schemaVersion: 1;
+    paragraphs: string[];
+    literalLines: string[];
+};
+export declare function prepareSearchPassage(markdown: string): PreparedSearchPassage;
+export declare function searchSnippet(markdown: string, query: string, pageSummary?: string): string;
+export declare function searchSnippetFromParagraphs(choices: string[], query: string, pageSummary?: string): string;

package/dist/search-snippet.js

@@ -0,0 +1,93 @@
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkGfm from 'remark-gfm';
+const parser = unified().use(remarkParse).use(remarkGfm);
+function inline(node) {
+    if (node.type === 'text' || node.type === 'inlineCode')
+        return node.value ?? '';
+    if (['code', 'html', 'image', 'imageReference', 'table'].includes(node.type))
+        return '';
+    return (node.children ?? []).map(inline).join('');
+}
+function paragraphs(node) {
+    if (node.type === 'paragraph')
+        return [inline(node).replace(/\s+/g, ' ').trim()].filter(Boolean);
+    if (['code', 'html', 'table'].includes(node.type))
+        return [];
+    return (node.children ?? []).flatMap(paragraphs);
+}
+// Visible prose only: ignore code, tables, image labels, and link destinations.
+export function searchParagraphs(markdown) {
+    try {
+        return paragraphs(parser.parse(markdown));
+    }
+    catch {
+        return [];
+    }
+}
+const discovery = /^(?:Documentation index:|Document manifest:|Navigation section:|Breadcrumbs:|Updated:|Verified:|Published:)/i;
+// Link-led lists describe destinations, rather than the page containing them.
+// Do not let related-reading lists become evidence that their host page answers a query.
+function passageParagraphs(node) {
+    if (node.type === 'list' && (node.children ?? []).every(item => item.children?.[0]?.children?.[0]?.type === 'link'))
+        return [];
+    if (node.type === 'paragraph')
+        return paragraphs(node).filter(text => !discovery.test(text));
+    if (['code', 'html', 'table'].includes(node.type))
+        return [];
+    return (node.children ?? []).flatMap(passageParagraphs);
+}
+export function searchPassageParagraphs(markdown) {
+    try {
+        return passageParagraphs(parser.parse(markdown));
+    }
+    catch {
+        return [];
+    }
+}
+// Literal lookups may use visible table rows and code lines. Link destinations,
+// HTML, and surrounding Markdown syntax never become result descriptions.
+function literalLines(node) {
+    if (node.type === 'code')
+        return (node.value ?? '').split('\n');
+    if (node.type === 'tableRow')
+        return [(node.children ?? []).map(inline).join(' — ')];
+    if (['html', 'image'].includes(node.type))
+        return [];
+    return (node.children ?? []).flatMap(literalLines);
+}
+const cleanLines = (lines) => lines.map(line => line.replace(/\s+/g, ' ').trim()).filter(Boolean);
+export function searchLiteralLines(markdown) {
+    try {
+        return cleanLines(literalLines(parser.parse(markdown)));
+    }
+    catch {
+        return [];
+    }
+}
+// Derived once while building chunks.json, never while answering a search.
+// This metadata is not embedded and does not change passage digests.
+export function prepareSearchPassage(markdown) {
+    const tree = parser.parse(markdown);
+    return { schemaVersion: 1, paragraphs: passageParagraphs(tree), literalLines: cleanLines(literalLines(tree)) };
+}
+const stopWords = new Set('a an the i me my we our you your how do does can could would should will is are was were be to of in on for with please what where when why which and or it this that have has after before'.split(' '));
+// Prefer substantive query matches; Markdown boilerplate and common question words
+// must not hide a useful paragraph. Keep a matching term visible in long paragraphs.
+export function searchSnippet(markdown, query, pageSummary = '') {
+    const choices = searchParagraphs(markdown).filter(p => !discovery.test(p));
+    return searchSnippetFromParagraphs(choices, query, pageSummary);
+}
+export function searchSnippetFromParagraphs(choices, query, pageSummary = '') {
+    const terms = [...new Set((query.toLocaleLowerCase().match(/[\p{L}\p{N}_$.-]+/gu) ?? []).map(t => t.replace(/^[.-]+|[.-]+$/g, '')).filter(t => t && !stopWords.has(t)))];
+    const matches = (text) => terms.filter(term => new RegExp(`(?<![\\p{L}\\p{N}_])${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}\\p{N}_])`, 'iu').test(text));
+    const ranked = choices.map((text, at) => ({ text, at, terms: matches(text) })).sort((a, b) => b.terms.length - a.terms.length || a.at - b.at);
+    const selected = ranked[0];
+    const text = (selected?.text ?? pageSummary).replace(/\s+/g, ' ').trim();
+    if (text.length <= 180)
+        return text;
+    const first = selected?.terms.length ? Math.min(...selected.terms.map(t => text.toLocaleLowerCase().indexOf(t))) : 0;
+    const start = Math.max(0, first - 50);
+    const length = 180 - (start ? 1 : 0) - 1;
+    return `${start ? '…' : ''}${text.slice(start, start + length).trim()}…`;
+}

package/dist/search-synonyms.d.ts

@@ -0,0 +1,8 @@
+export type SynonymGroups = string[][];
+export type QueryUnit = {
+    term: string;
+    alternatives: string[];
+};
+export declare function validateSearchSynonyms(value: unknown): SynonymGroups;
+export declare function phraseMatches(value: string, phrase: string): boolean;
+export declare function queryUnits(query: string, groups: SynonymGroups): QueryUnit[];

package/dist/search-synonyms.js

@@ -0,0 +1,46 @@
+// Groups are project data; normalize spacing so a phrase has one stable spelling in every index.
+export function validateSearchSynonyms(value) {
+    if (value === undefined)
+        return [];
+    if (!Array.isArray(value))
+        throw new Error('search.synonyms must be a list of term groups');
+    const used = new Set();
+    return value.map((group, index) => {
+        if (!Array.isArray(group) || group.length < 2)
+            throw new Error(`search.synonyms[${index}] needs at least two terms`);
+        return group.map((term) => {
+            if (typeof term !== 'string' || !/^[\p{L}\p{N}]+(?:\s+[\p{L}\p{N}]+)*$/u.test(term.trim()))
+                throw new Error(`search.synonyms[${index}] accepts words or phrases of words only`);
+            const normalized = term.trim().replace(/\s+/g, ' ').toLocaleLowerCase();
+            if (used.has(normalized))
+                throw new Error(`Duplicate search synonym ${normalized}`);
+            used.add(normalized);
+            return normalized;
+        });
+    });
+}
+// Match complete words, allowing whitespace within a phrase and a plain English final plural.
+export function phraseMatches(value, phrase) {
+    const escaped = phrase.split(' ').map(word => word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('\\s+');
+    return new RegExp(`(?<![\\p{L}\\p{N}])${escaped}s?(?![\\p{L}\\p{N}])`, 'iu').test(value);
+}
+export function queryUnits(query, groups) {
+    const words = query.toLocaleLowerCase().split(/\s+/).map(word => word.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '')).filter(Boolean);
+    const entries = groups.flatMap(group => group.map(term => ({ term, group, words: term.split(' ') }))).sort((a, b) => b.words.length - a.words.length);
+    const units = [];
+    for (let at = 0; at < words.length;) {
+        const found = entries.find(entry => entry.words.every((word, offset) => {
+            const actual = words[at + offset];
+            return actual === word || (offset === entry.words.length - 1 && actual === `${word}s`);
+        }));
+        if (found) {
+            units.push({ term: words.slice(at, at + found.words.length).join(' '), alternatives: found.group.filter(term => term !== found.term) });
+            at += found.words.length;
+        }
+        else {
+            units.push({ term: words[at], alternatives: [] });
+            at++;
+        }
+    }
+    return units;
+}

package/dist/semantic-markdown.d.ts

@@ -0,0 +1,10 @@
+export declare const parse: import("unified").Processor<import("mdast").Root, undefined, undefined, undefined, undefined>;
+export declare function parsePlainMarkdown(value: string): any[];
+export declare function text(value: string): any;
+export declare function paragraph(...children: any[]): any;
+export declare function heading(depth: number, value: string): any;
+export declare function list(items: any[][]): any;
+export declare function link(url: string, value: string): any;
+export declare function code(value: string, lang?: string): any;
+export declare function root(children: any[]): any;
+export declare function renderMarkdown(tree: any): string;

package/dist/semantic-markdown.js

@@ -0,0 +1,39 @@
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkMdx from 'remark-mdx';
+import remarkGfm from 'remark-gfm';
+import remarkStringify from 'remark-stringify';
+export const parse = unified().use(remarkParse).use(remarkGfm).use(remarkMdx);
+const plainMarkdown = unified().use(remarkParse).use(remarkGfm);
+export function parsePlainMarkdown(value) {
+    // OpenAPI prose may contain raw HTML. It is inert in our HTML renderer, but
+    // passing the node through to a Markdown twin would publish active markup.
+    const stripHtml = (node) => node.type === 'html' ? [] : [{ ...node, ...(node.children ? { children: node.children.flatMap(stripHtml) } : {}) }];
+    return plainMarkdown.parse(value).children.flatMap(stripHtml);
+}
+const markdown = unified().use(remarkStringify, { bullet: '-', fences: true }).use(remarkGfm);
+export function text(value) { return { type: 'text', value }; }
+export function paragraph(...children) { return { type: 'paragraph', children }; }
+export function heading(depth, value) { return { type: 'heading', depth, children: [text(value)] }; }
+export function list(items) {
+    return { type: 'list', ordered: false, children: items.map(children => {
+            const properties = children[0]?.data?.hProperties;
+            if (!properties?.id)
+                return { type: 'listItem', spread: false, children };
+            const first = { ...children[0], data: { ...children[0].data, hProperties: undefined } };
+            return { type: 'listItem', spread: true, children: [first, ...children.slice(1)], data: { hProperties: properties } };
+        }) };
+}
+export function link(url, value) { return { type: 'link', url, children: [text(value)] }; }
+export function code(value, lang) { return { type: 'code', lang, value }; }
+export function root(children) { return { type: 'root', children }; }
+export function renderMarkdown(tree) {
+    const flatten = (node) => {
+        if (node.type === 'docApiPathHint')
+            return [];
+        if (['docApiParameter', 'docApiDisclosure', 'docApiVariant', 'docApiInfoCard', 'docApiInfoCards', 'docApiResponse', 'docApiResponseHeader'].includes(node.type))
+            return node.children.flatMap(flatten);
+        return [node.children ? { ...node, children: node.children.flatMap(flatten) } : node];
+    };
+    return String(markdown.stringify({ ...tree, children: tree.children.flatMap(flatten) }));
+}

package/dist/server.d.ts

@@ -0,0 +1,10 @@
+import { type Server, type ServerResponse } from 'node:http';
+export type SiteExtras = {
+    ask?: (request: Request) => Promise<Response>;
+    search?: (request: Request) => Promise<Response>;
+    widget?: string;
+    publicSettings?: Record<string, unknown>;
+    panel?: Record<string, string>;
+};
+export declare function createSiteServer(directory: string, liveClients?: Set<ServerResponse>, extras?: SiteExtras): Promise<Server>;
+export declare function serveSite(directory: string, host: string, port: number, liveClients?: Set<ServerResponse>, extras?: SiteExtras): Promise<Server>;