@shoalrun/cli 0.1.0-alpha.15 → 0.1.0-alpha.17

package/dist/search-results.d.ts

@@ -2,5 +2,5 @@ import type { Hit, LexicalOptions, Retriever } from './retrieval.js';
 type SearchHit = Hit & {
     literalSummary?: string;
 };
-export declare function createSearchResultSelector(retriever: Retriever): (hits: Hit[], query: string, options: LexicalOptions) => SearchHit[];
+export declare function createSearchResultSelector(retriever: Retriever): (hits: Iterable<Hit>, query: string, options: LexicalOptions, rescue?: Hit) => SearchHit[];
 export {};

package/dist/search-results.js

@@ -1,45 +1,59 @@
-import { tokenize } from './retrieval.js';
+import { compareHits, tokenize } from './retrieval.js';
 import { literalSearch } from './search-literal.js';
+import { identifierRescue } from './search-identifier-core.js';
 // Select landing passages for the search box, without changing passage retrieval
 // used by Ask. A gallery's repeated examples must not crowd other pages out.
 export function createSearchResultSelector(retriever) {
     retriever.requirePreparedSearch();
     const normalized = (text) => tokenize(text).join(' ');
-    return (hits, query, options) => {
+    return (hits, query, options, rescue) => {
         const pages = new Map();
+        const normalizedQuery = normalized(query);
         const literal = literalSearch(query);
-        const literalPages = new Map();
-        if (literal) {
-            for (const hit of hits) {
-                if (literalPages.has(hit.chunk.docId))
-                    continue;
-                const literalSummary = literal(hit.chunk.search.literalLines);
-                if (literalSummary)
-                    literalPages.set(hit.chunk.docId, { ...hit, literalSummary });
-            }
-        }
+        let exact;
+        // Retain at most three candidates per page, independent of its passage
+        // count. Input scores need not be sorted or collected into an array.
         for (const hit of hits) {
-            if (pages.has(hit.chunk.docId))
+            const page = pages.get(hit.chunk.docId) ?? { exactTitle: normalized(hit.chunk.title) === normalizedQuery };
+            if (!page.first || compareHits(hit, page.first) < 0)
+                page.first = hit;
+            if (hit.chunk.search.paragraphs.length && (!page.prose || compareHits(hit, page.prose) < 0))
+                page.prose = hit;
+            const literalSummary = literal?.(hit.chunk.search.literalLines);
+            if (literalSummary && (!page.literal || compareHits(hit, page.literal) < 0))
+                page.literal = { ...hit, literalSummary };
+            pages.set(hit.chunk.docId, page);
+            if (page.exactTitle && (!exact || compareHits(hit, exact) < 0))
+                exact = hit;
+        }
+        const limit = options.limit ?? 8;
+        const candidates = [];
+        for (const page of pages.values()) {
+            const candidate = page.literal ? { rank: page.first, hit: { ...page.literal, score: page.first.score } } : page.prose ? { rank: page.prose, hit: page.prose } : undefined;
+            if (!candidate)
                 continue;
-            // Keep the page's retrieved position, but land on the passage that actually
-            // contains the literal. A nearby token table must not hide the correct row.
-            const exactPassage = literalPages.get(hit.chunk.docId);
-            if (exactPassage)
-                pages.set(hit.chunk.docId, { ...exactPassage, score: hit.score });
-            else if (hit.chunk.search.paragraphs.length)
-                pages.set(hit.chunk.docId, hit);
+            const at = candidates.findIndex(other => compareHits(candidate.rank, other.rank) < 0);
+            candidates.splice(at < 0 ? candidates.length : at, 0, candidate);
+            if (candidates.length > limit)
+                candidates.pop();
+        }
+        let selected = candidates.map(item => item.hit);
+        if (rescue) {
+            selected = identifierRescue(query, selected, [rescue]);
+            const landing = pages.get(rescue.chunk.docId)?.literal;
+            if (landing)
+                selected = selected.map(hit => hit.chunk.docId === rescue.chunk.docId ? { ...landing, score: hit.score } : hit);
         }
         // An exact page-title query is a request for that topic's entry point.
         // Ordinary semantic queries retain their ranking; incidental words do not qualify.
-        const exact = hits.find(hit => normalized(hit.chunk.title) === normalized(query));
         if (exact) {
             const introduction = retriever.pageIntroduction(exact.chunk.docId, options);
-            const landing = introduction ? { chunk: introduction, score: exact.score } : pages.get(exact.chunk.docId);
+            const page = pages.get(exact.chunk.docId);
+            const landing = introduction ? { chunk: introduction, score: exact.score } : page?.literal ?? page?.prose;
             if (landing) {
-                pages.delete(exact.chunk.docId);
-                return [landing, ...pages.values()].slice(0, options.limit ?? 8);
+                return [landing, ...selected.filter(hit => hit.chunk.docId !== exact.chunk.docId)].slice(0, options.limit ?? 8);
             }
         }
-        return [...pages.values()].slice(0, options.limit ?? 8);
+        return selected.slice(0, limit);
     };
 }

package/dist/search-snippet.d.ts

@@ -1,3 +1,4 @@
+export { searchSnippetFromParagraphs } from './search-text.js';
 export declare function searchParagraphs(markdown: string): string[];
 export declare function searchPassageParagraphs(markdown: string): string[];
 export declare function searchLiteralLines(markdown: string): string[];
@@ -8,4 +9,3 @@ export type PreparedSearchPassage = {
 };
 export declare function prepareSearchPassage(markdown: string): PreparedSearchPassage;
 export declare function searchSnippet(markdown: string, query: string, pageSummary?: string): string;
-export declare function searchSnippetFromParagraphs(choices: string[], query: string, pageSummary?: string): string;

package/dist/search-snippet.js

@@ -1,3 +1,5 @@
+import { searchSnippetFromParagraphs } from './search-text.js';
+export { searchSnippetFromParagraphs } from './search-text.js';
 import { unified } from 'unified';
 import remarkParse from 'remark-parse';
 import remarkGfm from 'remark-gfm';
@@ -71,23 +73,9 @@ export function prepareSearchPassage(markdown) {
     const tree = parser.parse(markdown);
     return { schemaVersion: 1, paragraphs: passageParagraphs(tree), literalLines: cleanLines(literalLines(tree)) };
 }
-const stopWords = new Set('a an the i me my we our you your how do does can could would should will is are was were be to of in on for with please what where when why which and or it this that have has after before'.split(' '));
 // Prefer substantive query matches; Markdown boilerplate and common question words
 // must not hide a useful paragraph. Keep a matching term visible in long paragraphs.
 export function searchSnippet(markdown, query, pageSummary = '') {
     const choices = searchParagraphs(markdown).filter(p => !discovery.test(p));
     return searchSnippetFromParagraphs(choices, query, pageSummary);
 }
-export function searchSnippetFromParagraphs(choices, query, pageSummary = '') {
-    const terms = [...new Set((query.toLocaleLowerCase().match(/[\p{L}\p{N}_$.-]+/gu) ?? []).map(t => t.replace(/^[.-]+|[.-]+$/g, '')).filter(t => t && !stopWords.has(t)))];
-    const matches = (text) => terms.filter(term => new RegExp(`(?<![\\p{L}\\p{N}_])${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}\\p{N}_])`, 'iu').test(text));
-    const ranked = choices.map((text, at) => ({ text, at, terms: matches(text) })).sort((a, b) => b.terms.length - a.terms.length || a.at - b.at);
-    const selected = ranked[0];
-    const text = (selected?.text ?? pageSummary).replace(/\s+/g, ' ').trim();
-    if (text.length <= 180)
-        return text;
-    const first = selected?.terms.length ? Math.min(...selected.terms.map(t => text.toLocaleLowerCase().indexOf(t))) : 0;
-    const start = Math.max(0, first - 50);
-    const length = 180 - (start ? 1 : 0) - 1;
-    return `${start ? '…' : ''}${text.slice(start, start + length).trim()}…`;
-}

package/dist/search-text.d.ts

@@ -0,0 +1 @@
+export declare function searchSnippetFromParagraphs(choices: string[], query: string, pageSummary?: string): string;

package/dist/search-text.js

@@ -0,0 +1,14 @@
+const stopWords = new Set('a an the i me my we our you your how do does can could would should will is are was were be to of in on for with please what where when why which and or it this that have has after before'.split(' '));
+export function searchSnippetFromParagraphs(choices, query, pageSummary = '') {
+    const terms = [...new Set((query.toLocaleLowerCase().match(/[\p{L}\p{N}_$.-]+/gu) ?? []).map(t => t.replace(/^[.-]+|[.-]+$/g, '')).filter(t => t && !stopWords.has(t)))];
+    const matches = (text) => terms.filter(term => new RegExp(`(?<![\\p{L}\\p{N}_])${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}\\p{N}_])`, 'iu').test(text));
+    const ranked = choices.map((text, at) => ({ text, at, terms: matches(text) })).sort((a, b) => b.terms.length - a.terms.length || a.at - b.at);
+    const selected = ranked[0];
+    const text = (selected?.text ?? pageSummary).replace(/\s+/g, ' ').trim();
+    if (text.length <= 180)
+        return text;
+    const first = selected?.terms.length ? Math.min(...selected.terms.map(t => text.toLocaleLowerCase().indexOf(t))) : 0;
+    const start = Math.max(0, first - 50);
+    const length = 180 - (start ? 1 : 0) - 1;
+    return `${start ? '…' : ''}${text.slice(start, start + length).trim()}…`;
+}

package/dist/vercel-function.js

@@ -100,8 +100,16 @@ export async function addAskFunction(input) {
     await writeFile(path.join(func, 'index.js'), `import { createAskFunction, nodeAdapter } from './lib/ask-function.js';\n\nconst directory = new URL('.', import.meta.url).pathname;\nexport default nodeAdapter(() => createAskFunction(directory));\n`);
     // The answer can take about a minute. Hobby allows up to 300 seconds with Fluid compute, so 120 leaves room.
     await writeFile(path.join(func, '.vc-config.json'), JSON.stringify({ runtime: 'nodejs22.x', handler: 'index.js', launcherType: 'Nodejs', shouldAddHelpers: false, maxDuration: 120, memory: 1024 }, null, 2) + '\n');
-    await cp(func, searchFunc, { recursive: true });
-    await writeFile(path.join(searchFunc, 'index.js'), `import { createHostedSearchFunction, nodeAdapter } from './lib/ask-function.js';\n\nconst directory = new URL('.', import.meta.url).pathname;\nexport default nodeAdapter(() => createHostedSearchFunction(directory));\n`);
+    // Build Search from its own runtime closure and private retrieval data. It
+    // must not inherit chat generation, cards, logging, or compiler dependencies.
+    await mkdir(path.join(searchFunc, 'site', 'ai'), { recursive: true });
+    for (const name of ['chunks.json', 'embeddings.json', 'embeddings.bin'])
+        await cp(path.join(site, 'ai', name), path.join(searchFunc, 'site', 'ai', name));
+    const { chat: _chat, ...searchSettings } = runtime.settings;
+    await writeFile(path.join(searchFunc, 'runtime.json'), JSON.stringify({ ...runtime, settings: searchSettings, searchStore: chat.budget.store }, null, 2) + '\n');
+    await writeFile(path.join(searchFunc, 'package.json'), JSON.stringify({ type: 'module' }) + '\n');
+    await copyFunctionRuntime(dist, 'search-function.js', searchFunc);
+    await writeFile(path.join(searchFunc, 'index.js'), `import { createHostedSearchFunction } from './lib/search-function.js';\nimport { nodeAdapter } from './lib/function-runtime.js';\n\nconst directory = new URL('.', import.meta.url).pathname;\nexport default nodeAdapter(() => createHostedSearchFunction(directory));\n`);
     await writeFile(path.join(searchFunc, '.vc-config.json'), JSON.stringify({ runtime: 'nodejs22.x', handler: 'index.js', launcherType: 'Nodejs', shouldAddHelpers: false, maxDuration: 30, memory: 1024 }, null, 2) + '\n');
     await readdir(func);
     return { added: true, directory: func, chunks: built.index.chunks.length, embedded: true, embeddingCounts: { embedded: embedded.embedded, reused: embedded.reused } };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shoalrun/cli",
-  "version": "0.1.0-alpha.15",
+  "version": "0.1.0-alpha.17",
   "publishConfig": {
     "registry": "https://registry.npmjs.org",
     "access": "public"

package/skills/shoalrun-setup/SKILL.md

@@ -33,4 +33,6 @@ For a new OpenAPI source, read `guide.api-references` and complete its API-sourc
 
 Check the generated HTML, Markdown twins, and `documents.json`. Confirm navigation and API links resolve, and inspect assets inside the components that contain them. Run `shoalrun audit docs.config.yaml` when project checks or API audits are configured; documentation behavior tests can use Doc Detective through `audit.tests` or another external CLI.
 
+For optional semantic search, use `outputs.html.search.provider: semantic` (available since `0.1.0-alpha.16`) with `retrieval.embeddings`; do not generate a custom browser adapter for Shoalrun's own endpoint. Build the site, run `shoalrun index <output> --embed --config docs.config.yaml`, then `shoalrun serve <output> --search --config docs.config.yaml` with the provider key only in the environment. Confirm the endpoint returns `mode: vector`; a visible result or keyword fallback does not prove semantic retrieval. Selecting the browser provider does not provision production hosting, and profiled Vercel chat-function builds remain unsupported.
+
 Report the files created, commands run, and anything still needing an owner decision. Do not deploy, publish a release, or claim factual accuracy from a successful build alone.