@shoalrun/cli 0.1.0-alpha.15 → 0.1.0-alpha.17

package/dist/ask-setup.js

@@ -1,99 +1,11 @@
-import { readFile } from 'node:fs/promises';
-import path from 'node:path';
-import { AnswerCache } from './answer-cache.js';
-import { CardIndex, readCards } from './answer-cards.js';
-import { chatProviderConfig, chatRoutes, validateChat } from './chat-config.js';
-import { createChatProvider, fallbackChat } from './chat.js';
 import { loadConfig } from './config.js';
-import { validateReaderFilters, validateReaderSelection } from './reader-filters.js';
-import { createProvider, readEmbeddings } from './embeddings.js';
-import { DailyBudget, memoryBudgetStore } from './budget.js';
-import { envBudgetStore } from './budget-store.js';
-import { jevAvailable, jevEndpoint } from './jev.js';
-import { queryProviderConfig, validateRetrieval } from './retrieval-config.js';
-import { Retriever } from './retrieval.js';
-// `--reader-view audience=authors,maintainers`, repeatable for more than one filter.
-export function parseReaderView(specs) {
-    if (!specs.length)
-        return undefined;
-    const selection = {};
-    for (const spec of specs) {
-        const match = /^([a-z][a-z0-9-]*)=(.+)$/.exec(spec);
-        if (!match)
-            throw new Error(`--reader-view needs facet=choice[,choice], such as audience=authors; got ${JSON.stringify(spec)}`);
-        if (selection[match[1]])
-            throw new Error(`--reader-view names ${match[1]} twice`);
-        selection[match[1]] = match[2].split(',').map(choice => choice.trim()).filter(Boolean);
-    }
-    return selection;
-}
-// Everything `shoalrun ask` and the answer eval share: read the project's chat and retrieval settings, load the passages and any
-// vectors, and wire up the optional layers. A layer that isn't configured is left out, and one that is configured but can't run
-// (Jev without a key) is left out with a note.
+import { prepareAskRuntime } from './ask-runtime.js';
+export { parseReaderView } from './ask-runtime.js';
+// Local authoring/evaluation accepts a source config. Hosted functions import
+// ask-runtime directly and receive already-resolved settings instead.
 export async function prepareAsk(input) {
     if (!input.settings && !input.configFile)
         throw new Error('prepareAsk needs a config file or settings');
     const settings = input.settings ?? (await loadConfig(input.configFile)).value;
-    const readerFilters = validateReaderFilters(settings.readerFilters);
-    const chatConfig = validateChat(settings.chat, readerFilters);
-    const retrieval = validateRetrieval(settings.retrieval);
-    // --model replaces the main model only. Fallback and escalation models still inherit the config's settings.
-    const mainConfig = chatProviderConfig(chatConfig, { model: input.model });
-    const routes = chatRoutes(chatConfig, chatProviderConfig(chatConfig));
-    const chat = fallbackChat([createChatProvider(mainConfig), ...routes.fallbacks.map(createChatProvider)]);
-    let chunks;
-    try {
-        chunks = JSON.parse(await readFile(path.join(input.siteDirectory, 'ai', 'chunks.json'), 'utf8'));
-    }
-    catch (error) {
-        throw error.code === 'ENOENT' ? new Error(`No ai/chunks.json in ${input.siteDirectory}; run shoalrun index first`) : error;
-    }
-    const embeddings = await readEmbeddings(input.siteDirectory);
-    const retriever = new Retriever(chunks, embeddings);
-    const variants = retriever.variants();
-    const variant = input.channel && input.version && input.language ? `${input.channel}|${input.version}|${input.language}` : variants.length === 1 ? variants[0] : undefined;
-    if (!variant)
-        throw new Error(`The site has several publications (${variants.join(', ')}); pass --channel, --version, and --language`);
-    let embedQuery;
-    let embedTexts;
-    let embedding;
-    if (embeddings) {
-        const queryProvider = createProvider(queryProviderConfig(retrieval, embeddings.meta));
-        if (queryProvider.space !== embeddings.meta.space || (embeddings.meta.backend && queryProvider.backend !== embeddings.meta.backend))
-            throw new Error(`${queryProvider.model} is not in the embedding space or backend of ${embeddings.meta.model}; set retrieval.embeddings.queryModel and baseUrl to compatible values`);
-        embedQuery = async (text, signal) => (await queryProvider.embed([text], 'query', signal))[0];
-        embedTexts = texts => queryProvider.embed(texts, 'query');
-        embedding = { provider: queryProvider.name, model: queryProvider.model, space: queryProvider.space, dimension: queryProvider.dimension };
-    }
-    // A reader view on the command line replaces the config's, so one run can answer for a different audience.
-    if (input.readerView && !readerFilters)
-        throw new Error('--reader-view needs readerFilters.facets in the site config');
-    const readerSelection = input.readerView ? validateReaderSelection(input.readerView, readerFilters, '--reader-view') : chatConfig?.readerView;
-    const notes = [];
-    const rerank = Boolean(chatConfig?.rerank) && !input.noRerank;
-    const scoring = chatConfig?.rerank;
-    const scorer = { ...(scoring?.endpoint ? { endpoint: jevEndpoint(scoring.endpoint) } : {}), ...(scoring?.model ? { model: scoring.model } : {}), ...(scoring?.keyEnv ? { keyEnv: scoring.keyEnv } : {}), ...(scoring?.stateFormat ? { stateFormat: scoring.stateFormat } : {}), ...(input.jevCacheDir ? { cacheDir: input.jevCacheDir } : {}) };
-    const scoringReady = jevAvailable(scoring?.keyEnv);
-    // A scorer that is configured but has no key counts as down, not as absent: it is what keeps off-topic questions from the model.
-    if (rerank && !scoringReady)
-        notes.push(`The relevance scorer has no key: set ${scoring?.keyEnv ?? 'JEV_API_KEY'}. Until then it counts as down, so ${scoring?.onError === 'answer' ? 'the model answers without the gate and the claim check' : 'questions get the closest pages and no model call'}`);
-    // Answer cards need semantic search and Jev. A configured but missing card file is a note, not an error, so the chat still answers.
-    let cards;
-    if (chatConfig?.cards && !input.noCards && rerank && scoringReady) {
-        const file = await readCards(input.siteDirectory);
-        if (!file)
-            notes.push('chat.cards is set but there is no ai/answers.json; run shoalrun answers to make it');
-        else if (!embedding || file.embedding.space !== embedding.space || file.embedding.dimension !== embedding.dimension)
-            notes.push('The answer cards were made with a different embedding model, so they are not used; run shoalrun answers again');
-        else
-            cards = { index: new CardIndex(file), threshold: chatConfig.cards.threshold, confirm: chatConfig.cards.confirm, jev: { ...scorer, maxCalls: 1 } };
-    }
-    // The shared store is for a hosted function. A local run keeps its counters in memory, so it never needs the store's keys.
-    const store = input.sharedStore && chatConfig?.budget?.store ? envBudgetStore(chatConfig.budget.store) : memoryBudgetStore();
-    const reported = new Set();
-    const onBudgetError = (error) => { if (!reported.has(error.message)) {
-        reported.add(error.message);
-        process.stderr.write(`Budget: ${error.message}. Questions get the closest pages, with no model call, until it works.\n`);
-    } };
-    return { ...(chatConfig?.budget ? { budget: new DailyBudget(store, chatConfig.budget, Date.now, onBudgetError), budgetStore: store } : {}), chat, notes, embedTexts, embedding, options: { ...(scoring?.onError ? { onScorerError: scoring.onError } : {}), retriever, chat, variant, readerSelection, embedQuery, passages: input.passages ?? chatConfig?.passages, gate: chatConfig?.gate, ...(input.maxTokens ? { maxTokens: input.maxTokens } : {}), ...(rerank ? { jev: { ...scorer, top: 24, maxCalls: 2 } } : {}), ...(cards ? { cards } : {}), ...(input.cache && chatConfig?.cache ? { cache: new AnswerCache({ maxEntries: chatConfig.cache.maxEntries, ttlMs: chatConfig.cache.ttlMinutes * 60_000 }) } : {}), ...(routes.escalate && !input.noEscalate ? { escalate: { chat: createChatProvider(routes.escalate.config), when: routes.escalate.when } } : {}), ...(rerank && scoringReady && !input.noDirect && (input.direct || chatConfig?.direct) ? { direct: { threshold: input.direct?.threshold ?? chatConfig.direct.threshold, maxChars: input.direct?.maxChars ?? chatConfig?.direct?.maxChars ?? 1200, margin: chatConfig?.direct?.margin ?? 0.1, others: chatConfig?.direct?.others ?? 1, jev: { ...scorer, maxCalls: 2 } } } : {}), ...(rerank && scoringReady && chatConfig?.verify ? { verify: { ...chatConfig.verify, jev: { ...scorer, maxCalls: 3 } } } : {}) } };
+    return prepareAskRuntime({ ...input, settings });
 }

package/dist/cli.js

@@ -238,7 +238,7 @@ try {
                 embedQuery = async (query, signal) => (await provider.embed([query], 'query', signal))[0];
             }
             extras = { search: createSearchHandler({ retriever, embedQuery, hosts, readerFilters: validateReaderFilters(settings.readerFilters), pageSummaries }) };
-            banner = `Search is on: ${embeddings ? 'hybrid retrieval' : 'lexical retrieval; run shoalrun index --embed for semantic results'}.\n`;
+            banner = `Search is on: ${embeddings ? 'semantic retrieval' : 'lexical retrieval; run shoalrun index --embed for semantic results'}.\n`;
         }
         if (ask) {
             // The preview spends your API credits on every question, so it answers only on this machine.
@@ -425,7 +425,11 @@ try {
             const provider = createProvider(embeddingProviderConfig(retrieval, { provider: values['--provider'], model: values['--model'], dimension: values['--dimension'] === undefined ? undefined : Number(values['--dimension']), baseUrl: values['--base-url'], keyEnv: values['--key-env'] }));
             embeddings = await embedChunkIndex(subcommand, result.index, provider, embeddingInput);
         }
-        process.stdout.write(JSON.stringify({ status: 'ok', documents: result.documents, chunks: result.index.chunks.length, output: result.output, ...(embeddings ? { embeddings } : {}) }, null, 2) + '\n');
+        const counts = new Map();
+        for (const chunk of result.index.chunks)
+            counts.set(chunk.variant, (counts.get(chunk.variant) ?? 0) + 1);
+        const publications = [...counts].sort(([a], [b]) => a.localeCompare(b)).map(([variant, passages]) => ({ variant, passages }));
+        process.stdout.write(JSON.stringify({ status: 'ok', documents: result.documents, chunks: result.index.chunks.length, sourceDigest: result.index.sourceDigest, publications, output: result.output, ...(embeddings ? { embeddings } : {}) }, null, 2) + '\n');
     }
     else if (command === 'ask') {
         const usage = 'Usage: shoalrun ask <site-directory> "<question>" [--config FILE] [--model MODEL] [--passages N] [--reader-view FACET=CHOICE[,CHOICE]]... [--channel C --version V --language L] [--no-rerank] [--no-direct] [--no-escalate] [--no-cards] [--json]';

package/dist/compiler.js

@@ -66,6 +66,8 @@ function validateSearch(value, declaringFile) {
         return { provider, index: false };
     if (provider === 'lexical')
         return { provider, module: builtInSearchProvider, index: true };
+    if (provider === 'semantic')
+        return { provider, module: path.resolve(path.dirname(builtInSearchProvider), 'search-semantic.js'), index: true };
     if (provider !== 'custom')
         throw new Error(`Unknown search provider ${provider}`);
     if (typeof config.module !== 'string' || !/\.m?js$/.test(config.module))

package/dist/embeddings.d.ts

@@ -1,4 +1,4 @@
-import { type Chunk, type ChunkIndex } from './index-build.js';
+import type { Chunk, ChunkIndex } from './index-build.js';
 import { type Transport } from './provider-http.js';
 export type EmbedInputType = 'document' | 'query';
 export type EmbeddingProvider = {

package/dist/embeddings.js

@@ -1,7 +1,7 @@
 import { readFile, realpath, writeFile } from 'node:fs/promises';
 import { createHash } from 'node:crypto';
 import path from 'node:path';
-import { aiDirectory } from './index-build.js';
+import { aiDirectory } from './ai-directory.js';
 import { postJson, throwIfCancelled } from './provider-http.js';
 const hash = (value) => createHash('sha256').update(JSON.stringify(value)).digest('hex');
 function normalize(vector) {

package/dist/function-package.js

@@ -55,4 +55,9 @@ export async function copyFunctionRuntime(dist, entry, target) {
     const done = new Set();
     for (const name of packages)
         await copyPackage(name, dist, target, done);
+    // Preserve the runtime author's license as well as the licenses retained in
+    // copied dependency packages. Custom runtime directories may have no parent license.
+    const license = path.join(dist, '..', 'LICENSE');
+    if (await stat(license).then(info => info.isFile(), () => false))
+        await cp(license, path.join(target, 'LICENSE.shoalrun'));
 }

package/dist/function-runtime.d.ts

@@ -0,0 +1,23 @@
+import type { IncomingMessage, ServerResponse } from 'node:http';
+export type FunctionRuntime = {
+    schemaVersion: 1;
+    settings: Record<string, unknown>;
+    variant?: {
+        channel: string;
+        version: string;
+        language: string;
+    };
+    hosts?: string[];
+    callerHeader?: string;
+    maxRequestsPerMinute?: number;
+    searchStore?: {
+        provider: 'upstash';
+        urlEnv: string;
+        tokenEnv: string;
+    };
+};
+export type AskFunction = (request: Request) => Promise<Response>;
+export declare function allowedHosts(runtime: FunctionRuntime, env: NodeJS.ProcessEnv): string[];
+export declare function nodeAdapter(load: () => Promise<AskFunction>, limits?: {
+    bodyBytes: number;
+}): (req: IncomingMessage, res: ServerResponse) => Promise<void>;

package/dist/function-runtime.js

@@ -0,0 +1,45 @@
+export function allowedHosts(runtime, env) {
+    return [...new Set([...(runtime.hosts ?? []), env.VERCEL_URL, env.VERCEL_BRANCH_URL, env.VERCEL_PROJECT_PRODUCTION_URL, ...(env.SHOALRUN_ASK_HOSTS?.split(',') ?? [])].map(host => host?.trim().toLowerCase()).filter((host) => Boolean(host)))];
+}
+const failure = (status, message) => new Response(JSON.stringify({ error: message }), { status, headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } });
+// Vercel's Node runtime calls `(req, res)`. This turns that into a Web request and back. Starting up happens on the first request, and a
+// failed start is tried again on the next one. The reader never sees why it failed; the reason goes to the function log.
+export function nodeAdapter(load, limits = { bodyBytes: 32_768 }) {
+    let ready;
+    const write = async (res, response) => {
+        res.statusCode = response.status;
+        response.headers.forEach((value, name) => { if (name !== 'set-cookie')
+            res.setHeader(name, value); });
+        const cookies = response.headers.getSetCookie();
+        if (cookies.length)
+            res.setHeader('set-cookie', cookies);
+        res.end(Buffer.from(await response.arrayBuffer()));
+    };
+    return async (req, res) => {
+        try {
+            const handler = await (ready ??= load().catch(error => { ready = undefined; throw error; }));
+            const chunks = [];
+            let size = 0;
+            for await (const chunk of req) {
+                size += chunk.length;
+                if (size > limits.bodyBytes) {
+                    await write(res, failure(413, 'The request is too large.'));
+                    return;
+                }
+                chunks.push(chunk);
+            }
+            const headers = new Headers();
+            for (const [name, value] of Object.entries(req.headers))
+                for (const item of Array.isArray(value) ? value : value === undefined ? [] : [value])
+                    headers.append(name, item);
+            const method = req.method ?? 'GET';
+            const request = new Request(`https://${req.headers.host ?? 'localhost'}${req.url ?? '/'}`, { method, headers, ...(method === 'GET' || method === 'HEAD' ? {} : { body: Buffer.concat(chunks) }) });
+            await write(res, await handler(request));
+        }
+        catch (error) {
+            console.error(`The answer function failed: ${error.message}`);
+            if (!res.headersSent)
+                await write(res, failure(503, 'The chat is unavailable right now.'));
+        }
+    };
+}

package/dist/index-build.d.ts

@@ -1,3 +1,4 @@
+export { aiDirectory } from './ai-directory.js';
 import { type PreparedSearchPassage } from './search-snippet.js';
 export type Chunk = {
     id: string;
@@ -46,10 +47,8 @@ export declare function chunkMarkdown(markdown: string, document: ChunkSource, a
     id: string;
     title: string;
 }>, options?: ChunkOptions): Chunk[];
-export declare function aiDirectory(directory: string): Promise<string>;
 export declare function buildChunkIndex(directory: string, options?: ChunkOptions): Promise<{
     index: ChunkIndex;
     output: string;
     documents: number;
 }>;
-export {};

package/dist/index-build.js

@@ -1,5 +1,7 @@
+import { aiDirectory } from './ai-directory.js';
+export { aiDirectory } from './ai-directory.js';
 import { createHash } from 'node:crypto';
-import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises';
+import { readFile, realpath, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { validateSearchSynonyms } from './search-synonyms.js';
 import { prepareSearchPassage } from './search-snippet.js';
@@ -164,7 +166,9 @@ export function chunkMarkdown(markdown, document, anchors = [], options = {}) {
     return chunks;
 }
 function makeChunk(document, headingPath, anchor, text, used, part = 0) {
-    const base = `${document.id}#${slug(headingPath.join(' ')) || 'intro'}${part ? `~${part + 1}` : ''}`;
+    // Logical page IDs repeat across releases. Stored passage IDs must not: vector
+    // rows and answer citations are keyed globally within this site's index.
+    const base = `${encodeURIComponent(document.variant)}:${encodeURIComponent(document.id)}#${slug(headingPath.join(' ')) || 'intro'}${part ? `~${part + 1}` : ''}`;
     const seen = used.get(base) ?? 0;
     used.set(base, seen + 1);
     const chunk = { id: seen ? `${base}-${seen + 1}` : base, docId: document.id, title: document.title, type: document.type, variant: document.variant, headingPath, url: anchor ? `${document.url}#${anchor}` : document.url, markdownUrl: document.markdownUrl, text };
@@ -185,16 +189,6 @@ async function readJson(file) {
 }
 // The directory that holds the passage and vector files. It is created on demand, and it must stay inside the site even if
 // `ai` is a symlink.
-export async function aiDirectory(directory) {
-    const root = path.resolve(directory);
-    const target = path.join(root, 'ai');
-    await mkdir(target, { recursive: true });
-    const real = await realpath(root);
-    const inside = await realpath(target);
-    if (inside !== real && !inside.startsWith(real + path.sep))
-        throw new Error('The ai directory leaves the site directory');
-    return target;
-}
 export async function buildChunkIndex(directory, options = {}) {
     const root = path.resolve(directory);
     const manifest = await readJson(path.join(root, 'documents.json'));
@@ -219,8 +213,10 @@ export async function buildChunkIndex(directory, options = {}) {
         const { channel, version, language } = document.variant;
         chunks.push(...chunkMarkdown(markdown, { ...document, variant: `${channel}|${version}|${language}` }, anchors.get(document.url), options));
     }
-    const source = manifest.documents.map(document => `${document.id}:${document.digest}`).sort().join('\n');
-    const index = { schemaVersion: 1, sourceDigest: sha(source + (synonyms.length ? `\n${JSON.stringify(synonyms)}` : '')), chunks, ...(synonyms.length ? { synonyms } : {}) };
+    // Generation identity includes publication membership, routes, reader marks,
+    // and prepared search evidence. Embedding digests remain content-only.
+    const source = chunks.map(({ id, variant, digest, url, markdownUrl, readerFilters, search }) => ({ id, variant, digest, url, markdownUrl, readerFilters: Object.fromEntries(Object.entries(readerFilters ?? {}).sort(([a], [b]) => a.localeCompare(b)).map(([facet, values]) => [facet, [...values].sort()])), search })).sort((a, b) => a.id.localeCompare(b.id));
+    const index = { schemaVersion: 1, sourceDigest: sha(JSON.stringify({ identityVersion: 2, source, synonyms })), chunks, ...(synonyms.length ? { synonyms } : {}) };
     const output = path.join(await aiDirectory(root), 'chunks.json');
     await writeFile(output, JSON.stringify(index) + '\n');
     return { index, output, documents: manifest.documents.length };

package/dist/publication-render.d.ts

@@ -52,7 +52,7 @@ export declare function renderPublicationPages({ rendered, docs, relationships,
     header: HeaderConfig;
     footer?: FooterConfig;
     search: {
-        provider: 'lexical' | 'custom' | 'none';
+        provider: 'lexical' | 'semantic' | 'custom' | 'none';
     };
     readerFilters?: ReaderFiltersConfig;
     variants: Variant[];

package/dist/retrieval-runtime.d.ts

@@ -0,0 +1,5 @@
+import { Retriever } from './retrieval.js';
+export declare function prepareSearchRetrieval(directory: string, settings: Record<string, unknown>): Promise<{
+    retriever: Retriever;
+    embedQuery: ((query: string, signal?: AbortSignal) => Promise<Float32Array>) | undefined;
+}>;

package/dist/retrieval-runtime.js

@@ -0,0 +1,21 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { createProvider, readEmbeddings } from './embeddings.js';
+import { queryProviderConfig, validateRetrieval } from './retrieval-config.js';
+import { Retriever } from './retrieval.js';
+// Search needs only passage retrieval and query embeddings, never chat models,
+// answer cards, source configuration, rendering, or component compilation.
+export async function prepareSearchRetrieval(directory, settings) {
+    const retrieval = validateRetrieval(settings.retrieval);
+    const chunks = JSON.parse(await readFile(path.join(directory, 'ai', 'chunks.json'), 'utf8'));
+    const embeddings = await readEmbeddings(directory);
+    const retriever = new Retriever(chunks, embeddings);
+    let embedQuery;
+    if (embeddings) {
+        const provider = createProvider(queryProviderConfig(retrieval, embeddings.meta));
+        if (provider.space !== embeddings.meta.space || (embeddings.meta.backend && provider.backend !== embeddings.meta.backend))
+            throw new Error(`${provider.model} is not in the embedding space or backend of ${embeddings.meta.model}`);
+        embedQuery = async (query, signal) => (await provider.embed([query], 'query', signal))[0];
+    }
+    return { retriever, embedQuery };
+}

package/dist/retrieval.d.ts

@@ -21,6 +21,7 @@ export declare class LexicalIndex {
     private readonly average;
     constructor(chunks: Chunk[], synonyms?: SynonymGroups);
     search(query: string, limit?: number, keep?: Keep): Hit[];
+    scan(query: string, keep?: Keep): IterableIterator<Hit>;
 }
 type VectorRow = {
     chunk: Chunk;
@@ -30,7 +31,9 @@ export declare class VectorIndex {
     private readonly rows;
     constructor(rows: VectorRow[]);
     search(query: Float32Array, limit?: number, keep?: Keep): Hit[];
+    scan(query: Float32Array, keep?: Keep): IterableIterator<Hit>;
 }
+export declare const compareHits: (left: Hit, right: Hit) => number;
 export declare function fuse(lists: Hit[][], limit?: number, k?: number): Hit[];
 export type Coverage = {
     total: number;
@@ -53,6 +56,7 @@ export declare class Retriever {
     private variant;
     pageIntroduction(docId: string, options?: LexicalOptions): Chunk | undefined;
     lexical(query: string, options?: LexicalOptions): Hit[];
+    searchPassages(query: string, queryVector: Float32Array | undefined, options?: LexicalOptions): Iterable<Hit>;
     vector(queryVector: Float32Array, options?: LexicalOptions): Hit[];
     hasVectors(options?: LexicalOptions): boolean;
     hybrid(query: string, queryVector: Float32Array | undefined, options?: LexicalOptions): Hit[];

package/dist/retrieval.js

@@ -67,10 +67,12 @@ export class LexicalIndex {
         this.average = this.entries.reduce((sum, entry) => sum + entry.length, 0) / (this.entries.length || 1);
     }
     search(query, limit = 8, keep) {
+        return [...this.scan(query, keep)].sort(compareHits).slice(0, limit);
+    }
+    *scan(query, keep) {
         const terms = [...new Set(tokenize(query))];
         const units = queryUnits(query, this.synonyms).filter(unit => unit.alternatives.length);
         const total = this.entries.length;
-        const hits = [];
         for (const entry of this.entries) {
             if (keep && !keep(entry.chunk))
                 continue;
@@ -93,9 +95,8 @@ export class LexicalIndex {
                 }
             }
             if (score > 0)
-                hits.push({ chunk: entry.chunk, score });
+                yield { chunk: entry.chunk, score };
         }
-        return hits.sort((left, right) => right.score - left.score || left.chunk.id.localeCompare(right.chunk.id)).slice(0, limit);
     }
 }
 function unit(vector) {
@@ -113,8 +114,10 @@ export class VectorIndex {
         this.rows = rows;
     }
     search(query, limit = 8, keep) {
+        return [...this.scan(query, keep)].sort(compareHits).slice(0, limit);
+    }
+    *scan(query, keep) {
         const q = unit(query);
-        const hits = [];
         for (const row of this.rows) {
             if (keep && !keep(row.chunk))
                 continue;
@@ -123,11 +126,11 @@ export class VectorIndex {
             let score = 0;
             for (let at = 0; at < q.length; at++)
                 score += q[at] * row.vector[at];
-            hits.push({ chunk: row.chunk, score });
+            yield { chunk: row.chunk, score };
         }
-        return hits.sort((left, right) => right.score - left.score || left.chunk.id.localeCompare(right.chunk.id)).slice(0, limit);
     }
 }
+export const compareHits = (left, right) => right.score - left.score || left.chunk.id.localeCompare(right.chunk.id);
 // Reciprocal rank fusion. Lexical and vector scores are not comparable, so only ranks are combined.
 export function fuse(lists, limit = 8, k = 60) {
     const scores = new Map();
@@ -149,10 +152,16 @@ export class Retriever {
     constructor(index, embeddings) {
         if (index?.schemaVersion !== 1 || !Array.isArray(index.chunks))
             throw new Error('Unsupported or invalid chunks.json');
+        if (new Set(index.chunks.map(chunk => chunk.id)).size !== index.chunks.length)
+            throw new Error('Ambiguous passage IDs in chunks.json; run shoalrun index <site-directory> to rebuild publication-qualified IDs. Existing passage embeddings can be reused.');
         const synonyms = validateSearchSynonyms(index.synonyms);
         const groups = new Map();
-        for (const chunk of index.chunks)
-            groups.set(chunk.variant, [...(groups.get(chunk.variant) ?? []), chunk]);
+        for (const chunk of index.chunks) {
+            let passages = groups.get(chunk.variant);
+            if (!passages)
+                groups.set(chunk.variant, passages = []);
+            passages.push(chunk);
+        }
         this.counts = { total: index.chunks.length, embedded: 0, stale: 0, ...(embeddings ? { model: embeddings.meta.model, dimension: embeddings.meta.dimension } : {}) };
         const rowOf = new Map((embeddings?.meta.ids ?? []).map((id, row) => [id, row]));
         for (const chunk of index.chunks)
@@ -209,6 +218,17 @@ export class Retriever {
     lexical(query, options = {}) {
         return this.byVariant.get(this.variant(options)).search(query, options.limit, keeping(options));
     }
+    // File-backed search still scans its selected publication, but streams scores
+    // into page selection instead of sorting/materializing every passage first.
+    searchPassages(query, queryVector, options = {}) {
+        const variant = this.variant(options);
+        if (!queryVector)
+            return this.byVariant.get(variant).scan(query, keeping(options));
+        const index = this.vectors.get(variant);
+        if (!index)
+            throw new Error('No current embeddings for this publication; run shoalrun index --embed');
+        return index.scan(queryVector, keeping(options));
+    }
     vector(queryVector, options = {}) {
         const index = this.vectors.get(this.variant(options));
         if (!index)

package/dist/search-function.d.ts

@@ -0,0 +1,2 @@
+import { type AskFunction } from './function-runtime.js';
+export declare function createHostedSearchFunction(directory: string, env?: NodeJS.ProcessEnv): Promise<AskFunction>;

package/dist/search-function.js

@@ -0,0 +1,30 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { DailyBudget, SharedRateLimit } from './budget.js';
+import { envBudgetStore } from './budget-store.js';
+import { validateReaderFilters } from './reader-filters.js';
+import { createSearchHandler } from './search-handler.js';
+import { validateSearch } from './search-config.js';
+import { prepareSearchRetrieval } from './retrieval-runtime.js';
+import { allowedHosts } from './function-runtime.js';
+export async function createHostedSearchFunction(directory, env = process.env) {
+    const runtime = JSON.parse(await readFile(path.join(directory, 'runtime.json'), 'utf8'));
+    if (runtime?.schemaVersion !== 1 || !runtime.settings)
+        throw new Error('runtime.json is not a Shoalrun search function file');
+    const filters = validateReaderFilters(runtime.settings.readerFilters);
+    // Accept old function metadata while new builds omit the chat configuration.
+    const legacy = runtime.settings.chat;
+    const settings = runtime.searchStore ?? legacy?.budget?.store;
+    if (!settings || typeof settings !== 'object' || !('provider' in settings) || settings.provider !== 'upstash' || !('urlEnv' in settings) || typeof settings.urlEnv !== 'string' || !('tokenEnv' in settings) || typeof settings.tokenEnv !== 'string' || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(settings.urlEnv) || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(settings.tokenEnv))
+        throw new Error('Hosted semantic search needs a shared budget store');
+    const setup = await prepareSearchRetrieval(path.join(directory, 'site'), runtime.settings);
+    const store = envBudgetStore({ urlEnv: settings.urlEnv, tokenEnv: settings.tokenEnv }, env);
+    const { limits } = validateSearch(runtime.settings.search);
+    return createSearchHandler({
+        ...setup, readerFilters: filters, hosts: allowedHosts(runtime, env),
+        onFallback: reason => console.warn(`Search fallback: ${reason}`),
+        rateLimiter: new SharedRateLimit(store, limits.perCallerPerMinute, Date.now, error => console.error(`Search rate limit: ${error.message}`), 'search-rate'),
+        budget: new DailyBudget(store, { perCaller: limits.perCallerPerDay, perDay: limits.perDay }, Date.now, error => console.error(`Search budget: ${error.message}`), 'search-budget'),
+        clientKey: request => request.headers.get(runtime.callerHeader ?? 'x-vercel-forwarded-for')?.split(',')[0].trim() || 'unknown'
+    });
+}

package/dist/search-handler.js

@@ -1,6 +1,7 @@
+import { compareHits } from './retrieval.js';
 import { validateReaderSelection } from './reader-filters.js';
-import { searchSnippetFromParagraphs } from './search-snippet.js';
-import { identifierRescue, searchIdentifiers } from './search-identifier.js';
+import { searchSnippetFromParagraphs } from './search-text.js';
+import { identifierMatch } from './search-identifier-core.js';
 import { createSearchResultSelector } from './search-results.js';
 export function createSearchHandler(options) {
     const selectResults = createSearchResultSelector(options.retriever);
@@ -91,17 +92,16 @@ export function createSearchHandler(options) {
             }
             if (!vector && fallbackReason)
                 options.onFallback?.(fallbackReason);
-            // Rank pages after inspecting their passages. A large gallery can otherwise
-            // consume the entire candidate window before page deduplication happens.
-            const retrievalOptions = { ...searchOptions, limit: Number.MAX_SAFE_INTEGER };
-            let hits = vector
-                ? options.retriever.vector(vector, retrievalOptions)
-                : options.retriever.lexical(query.trim(), retrievalOptions);
-            if (vector && limit >= 3 && searchIdentifiers(query).length) {
-                const lexical = options.retriever.lexical(query.trim(), retrievalOptions);
-                hits = identifierRescue(query.trim(), hits, lexical);
+            let candidate;
+            const matchesIdentifier = identifierMatch(query);
+            if (vector && limit >= 3 && matchesIdentifier) {
+                for (const hit of options.retriever.searchPassages(query.trim(), undefined, searchOptions)) {
+                    if (matchesIdentifier(hit) && (!candidate || compareHits(hit, candidate) < 0))
+                        candidate = hit;
+                }
             }
-            const results = selectResults(hits, query.trim(), searchOptions).map(({ chunk, literalSummary }) => ({ url: chunk.url, title: chunk.title, section: chunk.headingPath.at(-1), summary: literalSummary ?? searchSnippetFromParagraphs(chunk.search.paragraphs, query.trim(), options.pageSummaries?.get(`${chunk.variant}:${chunk.docId}`)) }));
+            const hits = selectResults(options.retriever.searchPassages(query.trim(), vector, searchOptions), query.trim(), searchOptions, candidate);
+            const results = hits.map(({ chunk, literalSummary }) => ({ url: chunk.url, title: chunk.title, section: chunk.headingPath.at(-1), summary: literalSummary ?? searchSnippetFromParagraphs(chunk.search.paragraphs, query.trim(), options.pageSummaries?.get(`${chunk.variant}:${chunk.docId}`)) }));
             return json(200, { results, mode: vector ? 'vector' : 'lexical', ...(!vector ? { fallbackReason } : {}) });
         }
         catch {

package/dist/search-identifier-core.d.ts

@@ -0,0 +1,4 @@
+import type { Hit } from './retrieval.js';
+export declare function searchIdentifiers(query: string): string[];
+export declare function identifierRescue<T extends Hit>(query: string, semantic: T[], lexical: T[], paragraphs?: (hit: Hit) => string[]): T[];
+export declare function identifierMatch(query: string, paragraphs?: (hit: Hit) => string[]): ((hit: Hit) => boolean) | undefined;

package/dist/search-identifier-core.js

@@ -0,0 +1,30 @@
+// Reserve at most one search-box result for a literal configuration key or flag
+// explained in prose. Tables, code listings and identifier prefixes do not qualify.
+export function searchIdentifiers(query) {
+    const identifiers = query.match(/\b[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+\b|--[a-z][\w-]*/g) ?? [];
+    // File names and URLs need ordinary retrieval, not a config-key shortcut.
+    return [...new Set(identifiers.filter(id => !/\.(?:txt|json|ya?ml|mdx?|[cm]?js|tsx?|html?|css|svg|png|jpe?g|com|org|net)$/i.test(id)))];
+}
+export function identifierRescue(query, semantic, lexical, paragraphs) {
+    const matches = identifierMatch(query, paragraphs);
+    if (!matches)
+        return semantic;
+    const candidate = lexical.find(matches);
+    if (!candidate)
+        return semantic;
+    const pages = [...new Set(semantic.map(h => h.chunk.docId))];
+    if (pages.slice(0, 3).includes(candidate.chunk.docId))
+        return semantic;
+    const remaining = semantic.filter(h => h.chunk.docId !== candidate.chunk.docId);
+    const seen = new Set();
+    const at = remaining.findIndex(hit => { seen.add(hit.chunk.docId); return seen.size === 3; });
+    const position = at < 0 ? remaining.length : at;
+    return [...remaining.slice(0, position), candidate, ...remaining.slice(position)];
+}
+export function identifierMatch(query, paragraphs = hit => hit.chunk.search?.paragraphs ?? []) {
+    const identifiers = searchIdentifiers(query);
+    if (!identifiers.length)
+        return undefined;
+    const matches = identifiers.map(id => new RegExp(`(?<![\\w$.-])${id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w$-]|\\.[\\w$])`, 'i'));
+    return hit => paragraphs(hit).some(p => matches.every(re => re.test(p)));
+}

package/dist/search-identifier.d.ts

@@ -1,3 +1,4 @@
 import type { Hit } from './retrieval.js';
-export declare function searchIdentifiers(query: string): string[];
-export declare function identifierRescue(query: string, semantic: Hit[], lexical: Hit[]): Hit[];
+export { searchIdentifiers } from './search-identifier-core.js';
+export declare const identifierMatch: (query: string) => ((hit: Hit) => boolean) | undefined;
+export declare const identifierRescue: <T extends Hit>(query: string, semantic: T[], lexical: T[]) => T[];

package/dist/search-identifier.js

@@ -1,25 +1,8 @@
 import { searchParagraphs } from './search-snippet.js';
-// Reserve at most one search-box result for a literal configuration key or flag
-// explained in prose. Tables, code listings and identifier prefixes do not qualify.
-export function searchIdentifiers(query) {
-    const identifiers = query.match(/\b[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+\b|--[a-z][\w-]*/g) ?? [];
-    // File names and URLs need ordinary retrieval, not a config-key shortcut.
-    return [...new Set(identifiers.filter(id => !/\.(?:txt|json|ya?ml|mdx?|[cm]?js|tsx?|html?|css|svg|png|jpe?g|com|org|net)$/i.test(id)))];
-}
-export function identifierRescue(query, semantic, lexical) {
-    const identifiers = searchIdentifiers(query);
-    if (!identifiers.length)
-        return semantic;
-    const matches = identifiers.map(id => new RegExp(`(?<![\\w$.-])${id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w$-]|\\.[\\w$])`, 'i'));
-    const candidate = lexical.find(hit => (hit.chunk.search?.paragraphs ?? searchParagraphs(hit.chunk.text)).some(p => matches.every(re => re.test(p))));
-    if (!candidate)
-        return semantic;
-    const pages = [...new Set(semantic.map(h => h.chunk.docId))];
-    if (pages.slice(0, 3).includes(candidate.chunk.docId))
-        return semantic;
-    const remaining = semantic.filter(h => h.chunk.docId !== candidate.chunk.docId);
-    const seen = new Set();
-    const at = remaining.findIndex(hit => { seen.add(hit.chunk.docId); return seen.size === 3; });
-    const position = at < 0 ? remaining.length : at;
-    return [...remaining.slice(0, position), candidate, ...remaining.slice(position)];
-}
+import { identifierMatch as preparedMatch, identifierRescue as preparedRescue } from './search-identifier-core.js';
+export { searchIdentifiers } from './search-identifier-core.js';
+// Keep the older unprepared-passage API usable by local callers. Hosted search
+// imports the prepared core and never needs a Markdown parser at request time.
+const paragraphs = (hit) => hit.chunk.search?.paragraphs ?? searchParagraphs(hit.chunk.text);
+export const identifierMatch = (query) => preparedMatch(query, paragraphs);
+export const identifierRescue = (query, semantic, lexical) => preparedRescue(query, semantic, lexical, paragraphs);

package/dist/search-literal.js

@@ -1,4 +1,4 @@
-import { searchIdentifiers } from './search-identifier.js';
+import { searchIdentifiers } from './search-identifier-core.js';
 const escape = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 const boundary = (value) => new RegExp(`(?<![\\w$.-])${escape(value)}(?![\\w$-]|\\.[\\w$])`, 'i');
 // Only explicit technical syntax gets this exception. Ordinary topic words must