@shoalrun/cli 0.0.0-stage → 0.1.0-alpha.11

package/dist/retrieval-config.d.ts

@@ -0,0 +1,30 @@
+import type { ProviderConfig } from './embeddings.js';
+export type EmbeddingConfig = {
+    provider: 'voyage' | 'openai';
+    model: string;
+    dimension?: number;
+    queryModel?: string;
+    baseUrl?: string;
+    keyEnv?: string;
+};
+export type RetrievalConfig = {
+    embeddings?: EmbeddingConfig;
+};
+export declare function validateRetrieval(value: unknown): RetrievalConfig | undefined;
+export type EmbeddingFlags = {
+    provider?: string;
+    model?: string;
+    dimension?: number;
+    baseUrl?: string;
+    keyEnv?: string;
+};
+export declare function embeddingProviderConfig(config: RetrievalConfig | undefined, flags?: EmbeddingFlags): ProviderConfig;
+export declare function queryProviderConfig(config: RetrievalConfig | undefined, stored: {
+    provider: string;
+    model: string;
+    dimension: number;
+}, flags?: {
+    model?: string;
+    baseUrl?: string;
+    keyEnv?: string;
+}): ProviderConfig;

package/dist/retrieval-config.js

@@ -0,0 +1,83 @@
+const mapping = (value) => !!value && typeof value === 'object' && !Array.isArray(value);
+const providers = ['voyage', 'openai'];
+const embeddingKeys = ['provider', 'model', 'dimension', 'queryModel', 'baseUrl', 'keyEnv'];
+function text(value, setting) {
+    if (typeof value !== 'string' || !value.trim())
+        throw new Error(`${setting} must be a nonempty string`);
+    return value.trim();
+}
+export function validateRetrieval(value) {
+    if (value === undefined)
+        return undefined;
+    if (!mapping(value))
+        throw new Error('retrieval must be a mapping');
+    for (const key of Object.keys(value))
+        if (key !== 'embeddings')
+            throw new Error(`retrieval has an unknown setting ${key}; it accepts only embeddings`);
+    if (value.embeddings === undefined)
+        return {};
+    const embeddings = value.embeddings;
+    if (!mapping(embeddings))
+        throw new Error('retrieval.embeddings must be a mapping');
+    for (const key of Object.keys(embeddings)) {
+        if (embeddingKeys.includes(key))
+            continue;
+        throw new Error(/key|secret|token|password/i.test(key)
+            ? `retrieval.embeddings.${key} is not a setting. The config never holds a key; name the environment variable that does with keyEnv`
+            : `retrieval.embeddings has an unknown setting ${key}; it accepts ${embeddingKeys.join(', ')}`);
+    }
+    const provider = text(embeddings.provider, 'retrieval.embeddings.provider');
+    if (!providers.includes(provider))
+        throw new Error(`retrieval.embeddings.provider must be one of ${providers.join(', ')}`);
+    const result = { provider: provider, model: text(embeddings.model, 'retrieval.embeddings.model') };
+    if (embeddings.dimension !== undefined) {
+        if (typeof embeddings.dimension !== 'number' || !Number.isSafeInteger(embeddings.dimension) || embeddings.dimension < 1)
+            throw new Error('retrieval.embeddings.dimension must be a positive integer');
+        result.dimension = embeddings.dimension;
+    }
+    if (embeddings.queryModel !== undefined)
+        result.queryModel = text(embeddings.queryModel, 'retrieval.embeddings.queryModel');
+    if (embeddings.keyEnv !== undefined) {
+        const name = text(embeddings.keyEnv, 'retrieval.embeddings.keyEnv');
+        if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name))
+            throw new Error('retrieval.embeddings.keyEnv names an environment variable, such as OPENAI_API_KEY, not the key itself');
+        result.keyEnv = name;
+    }
+    if (embeddings.baseUrl !== undefined) {
+        if (provider !== 'openai')
+            throw new Error('retrieval.embeddings.baseUrl applies only to the openai provider');
+        const url = text(embeddings.baseUrl, 'retrieval.embeddings.baseUrl');
+        let parsed;
+        try {
+            parsed = new URL(url);
+        }
+        catch {
+            throw new Error('retrieval.embeddings.baseUrl must be an http or https URL');
+        }
+        if (!['http:', 'https:'].includes(parsed.protocol))
+            throw new Error('retrieval.embeddings.baseUrl must be an http or https URL');
+        if (parsed.username || parsed.password)
+            throw new Error('retrieval.embeddings.baseUrl must not contain credentials; name the key variable with keyEnv');
+        result.baseUrl = url;
+    }
+    return { embeddings: result };
+}
+// Command-line flags override the config, so a one-off experiment never needs a config edit. Both must together name a provider
+// and a model, and neither has a default.
+export function embeddingProviderConfig(config, flags = {}) {
+    const base = config?.embeddings;
+    const provider = flags.provider ?? base?.provider;
+    const model = flags.model ?? base?.model;
+    if (!provider || !model)
+        throw new Error('Choose an embedding provider and model: set retrieval.embeddings in docs.config.yaml, or pass --provider and --model');
+    // A flag that names a different provider drops the config's provider-specific settings, which would not apply to it.
+    const same = provider === base?.provider;
+    return { provider, model, dimension: flags.dimension ?? (same ? base?.dimension : undefined), baseUrl: flags.baseUrl ?? (same ? base?.baseUrl : undefined), keyEnv: flags.keyEnv ?? (same ? base?.keyEnv : undefined) };
+}
+// Queries use the config's queryModel when there is one. It must be in the passage model's embedding space, which the
+// provider check enforces when vectors are compared.
+export function queryProviderConfig(config, stored, flags = {}) {
+    const base = config?.embeddings;
+    const same = base?.provider === stored.provider;
+    return { provider: stored.provider, model: flags.model ?? (same ? base?.queryModel : undefined) ?? stored.model, dimension: stored.dimension, baseUrl: flags.baseUrl ?? (same ? base?.baseUrl : undefined), keyEnv: flags.keyEnv ?? (same ? base?.keyEnv : undefined) };
+}

package/dist/retrieval.d.ts

@@ -0,0 +1,60 @@
+import type { Chunk, ChunkIndex } from './index-build.js';
+import type { StoredEmbeddings } from './embeddings.js';
+import { type SynonymGroups } from './search-synonyms.js';
+export type Hit = {
+    chunk: Chunk;
+    score: number;
+};
+export type ReaderSelection = Record<string, string[]>;
+export type LexicalOptions = {
+    limit?: number;
+    variant?: string;
+    readerSelection?: ReaderSelection;
+};
+export declare function readerMatches(marks: Record<string, string[]> | undefined, selection: ReaderSelection | undefined): boolean;
+type Keep = ((chunk: Chunk) => boolean) | undefined;
+export declare function tokenize(value: string): string[];
+export declare class LexicalIndex {
+    private readonly synonyms;
+    private readonly entries;
+    private readonly frequency;
+    private readonly average;
+    constructor(chunks: Chunk[], synonyms?: SynonymGroups);
+    search(query: string, limit?: number, keep?: Keep): Hit[];
+}
+type VectorRow = {
+    chunk: Chunk;
+    vector: Float32Array;
+};
+export declare class VectorIndex {
+    private readonly rows;
+    constructor(rows: VectorRow[]);
+    search(query: Float32Array, limit?: number, keep?: Keep): Hit[];
+}
+export declare function fuse(lists: Hit[][], limit?: number, k?: number): Hit[];
+export type Coverage = {
+    total: number;
+    embedded: number;
+    stale: number;
+    model?: string;
+    dimension?: number;
+};
+export declare class Retriever {
+    private readonly passages;
+    private readonly byVariant;
+    private readonly vectors;
+    private readonly counts;
+    private readonly digests;
+    constructor(index: ChunkIndex, embeddings?: StoredEmbeddings);
+    variants(): string[];
+    digestOf(id: string): string | undefined;
+    coverage(): Coverage;
+    requirePreparedSearch(): void;
+    private variant;
+    pageIntroduction(docId: string, options?: LexicalOptions): Chunk | undefined;
+    lexical(query: string, options?: LexicalOptions): Hit[];
+    vector(queryVector: Float32Array, options?: LexicalOptions): Hit[];
+    hasVectors(options?: LexicalOptions): boolean;
+    hybrid(query: string, queryVector: Float32Array | undefined, options?: LexicalOptions): Hit[];
+}
+export {};

package/dist/retrieval.js

@@ -0,0 +1,228 @@
+import { phraseMatches, queryUnits, validateSearchSynonyms } from './search-synonyms.js';
+// The rule the search box applies (assets/search-lexical.js), so search and chat agree on who a page is for. A page with no mark
+// for a facet is eligible for everyone. A marked page must share at least one choice with the selection for that facet.
+export function readerMatches(marks, selection) {
+    if (!selection || !Object.keys(selection).length)
+        return true;
+    return Object.entries(marks ?? {}).every(([facet, values]) => !selection[facet]?.length || values.some(value => selection[facet].includes(value)));
+}
+const keeping = (options) => options.readerSelection && Object.keys(options.readerSelection).length ? chunk => readerMatches(chunk.readerFilters, options.readerSelection) : undefined;
+// Weights for a term found in the page title, the heading path, or the passage text.
+const weights = { title: 2, heading: 3, text: 1 };
+const k1 = 1.2;
+const b = 0.75;
+// Folds plain English plurals so `pages` matches `page` and `classes` matches `class`. Words ending in -ss, -us, or -is are
+// left alone (`class`, `status`, `access`), and anything that isn't a plain word, such as an identifier, is never touched.
+function singular(token) {
+    if (token.length <= 3 || !/^\p{L}+$/u.test(token) || /(ss|us|is)$/.test(token))
+        return token;
+    if (token.endsWith('ies'))
+        return `${token.slice(0, -3)}y`;
+    if (/(ss|us|x|z|ch|sh)es$/.test(token))
+        return token.slice(0, -2);
+    return token.endsWith('s') ? token.slice(0, -1) : token;
+}
+// Keeps identifiers whole (`docs.config.yaml`, `$include`, `/publish/audit`) and also indexes their parts, so both an exact
+// identifier and a loose mention match.
+export function tokenize(value) {
+    const tokens = [];
+    for (const raw of value.toLocaleLowerCase().match(/[\p{L}\p{N}_$@:./-]+/gu) ?? []) {
+        const whole = raw.replace(/^[.:/-]+|[.:/-]+$/g, '');
+        if (!whole)
+            continue;
+        const parts = whole.split(/[._$@:/-]+/).filter(Boolean);
+        for (const token of new Set([whole, ...parts]))
+            tokens.push(singular(token));
+    }
+    return tokens;
+}
+function counts(tokens) {
+    const result = new Map();
+    for (const token of tokens)
+        result.set(token, (result.get(token) ?? 0) + 1);
+    return result;
+}
+// BM25 over passages with the title and heading path weighted above body text. One instance covers one publication, so
+// document frequencies never mix v1 with v2 or sandbox with production.
+export class LexicalIndex {
+    synonyms;
+    entries;
+    frequency = new Map();
+    average;
+    constructor(chunks, synonyms = []) {
+        this.synonyms = synonyms;
+        this.entries = chunks.map(chunk => {
+            const frequency = new Map();
+            const add = (tokens, weight) => { for (const [token, count] of counts(tokens))
+                frequency.set(token, (frequency.get(token) ?? 0) + count * weight); };
+            const body = tokenize(chunk.text);
+            add(body, weights.text);
+            add(tokenize(chunk.headingPath.join(' ')), weights.heading);
+            add(tokenize(chunk.title), weights.title);
+            return { chunk, frequency, length: body.length + 1 };
+        });
+        for (const entry of this.entries)
+            for (const token of entry.frequency.keys())
+                this.frequency.set(token, (this.frequency.get(token) ?? 0) + 1);
+        this.average = this.entries.reduce((sum, entry) => sum + entry.length, 0) / (this.entries.length || 1);
+    }
+    search(query, limit = 8, keep) {
+        const terms = [...new Set(tokenize(query))];
+        const units = queryUnits(query, this.synonyms).filter(unit => unit.alternatives.length);
+        const total = this.entries.length;
+        const hits = [];
+        for (const entry of this.entries) {
+            if (keep && !keep(entry.chunk))
+                continue;
+            let score = 0;
+            for (const term of terms) {
+                const tf = entry.frequency.get(term);
+                if (!tf)
+                    continue;
+                const df = this.frequency.get(term) ?? 0;
+                score += Math.log(1 + (total - df + 0.5) / (df + 0.5)) * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * entry.length / this.average));
+            }
+            for (const unit of units) {
+                const fields = [entry.chunk.title, entry.chunk.headingPath.join(' '), entry.chunk.text];
+                const fieldWeight = [4, 5, 2];
+                for (let at = 0; at < fields.length; at++) {
+                    if (phraseMatches(fields[at], unit.term))
+                        score += fieldWeight[at];
+                    else if (unit.alternatives.some(term => phraseMatches(fields[at], term)))
+                        score += fieldWeight[at] * 0.5;
+                }
+            }
+            if (score > 0)
+                hits.push({ chunk: entry.chunk, score });
+        }
+        return hits.sort((left, right) => right.score - left.score || left.chunk.id.localeCompare(right.chunk.id)).slice(0, limit);
+    }
+}
+function unit(vector) {
+    let sum = 0;
+    for (const value of vector)
+        sum += value * value;
+    const length = Math.sqrt(sum);
+    return length ? vector.map(value => value / length) : vector;
+}
+// Brute-force cosine similarity. A docs site has hundreds to a few thousand passages, so this is fast enough without an
+// approximate index. Stored vectors are unit length, and the query is normalized here so callers can pass any vector.
+export class VectorIndex {
+    rows;
+    constructor(rows) {
+        this.rows = rows;
+    }
+    search(query, limit = 8, keep) {
+        const q = unit(query);
+        const hits = [];
+        for (const row of this.rows) {
+            if (keep && !keep(row.chunk))
+                continue;
+            if (row.vector.length !== q.length)
+                throw new Error(`Query vector has ${q.length} dimensions but stored vectors have ${row.vector.length}`);
+            let score = 0;
+            for (let at = 0; at < q.length; at++)
+                score += q[at] * row.vector[at];
+            hits.push({ chunk: row.chunk, score });
+        }
+        return hits.sort((left, right) => right.score - left.score || left.chunk.id.localeCompare(right.chunk.id)).slice(0, limit);
+    }
+}
+// Reciprocal rank fusion. Lexical and vector scores are not comparable, so only ranks are combined.
+export function fuse(lists, limit = 8, k = 60) {
+    const scores = new Map();
+    for (const list of lists)
+        list.forEach((hit, rank) => {
+            const seen = scores.get(hit.chunk.id);
+            scores.set(hit.chunk.id, { chunk: hit.chunk, score: (seen?.score ?? 0) + 1 / (k + rank + 1) });
+        });
+    return [...scores.values()].sort((left, right) => right.score - left.score || left.chunk.id.localeCompare(right.chunk.id)).slice(0, limit);
+}
+// One index per publication. A query names its publication; nothing is searched across publications implicitly. A stored
+// vector is used only when its digest matches the passage's, so a passage edited after embedding falls back to lexical.
+export class Retriever {
+    passages = new Map();
+    byVariant = new Map();
+    vectors = new Map();
+    counts;
+    digests = new Map();
+    constructor(index, embeddings) {
+        if (index?.schemaVersion !== 1 || !Array.isArray(index.chunks))
+            throw new Error('Unsupported or invalid chunks.json');
+        const synonyms = validateSearchSynonyms(index.synonyms);
+        const groups = new Map();
+        for (const chunk of index.chunks)
+            groups.set(chunk.variant, [...(groups.get(chunk.variant) ?? []), chunk]);
+        this.counts = { total: index.chunks.length, embedded: 0, stale: 0, ...(embeddings ? { model: embeddings.meta.model, dimension: embeddings.meta.dimension } : {}) };
+        const rowOf = new Map((embeddings?.meta.ids ?? []).map((id, row) => [id, row]));
+        for (const chunk of index.chunks)
+            this.digests.set(chunk.id, chunk.digest);
+        for (const [variant, chunks] of groups) {
+            this.passages.set(variant, chunks);
+            this.byVariant.set(variant, new LexicalIndex(chunks, synonyms));
+            if (!embeddings)
+                continue;
+            const rows = [];
+            for (const chunk of chunks) {
+                const row = rowOf.get(chunk.id);
+                if (row === undefined)
+                    continue;
+                if (embeddings.meta.digests[row] !== chunk.digest) {
+                    this.counts.stale++;
+                    continue;
+                }
+                rows.push({ chunk, vector: embeddings.vectors[row] });
+            }
+            this.counts.embedded += rows.length;
+            if (rows.length)
+                this.vectors.set(variant, new VectorIndex(rows));
+        }
+    }
+    variants() { return [...this.byVariant.keys()].sort(); }
+    // The current digest of a passage, so a stored answer can tell whether the text behind it has changed.
+    digestOf(id) { return this.digests.get(id); }
+    coverage() { return { ...this.counts }; }
+    requirePreparedSearch() {
+        for (const chunks of this.passages.values())
+            for (const chunk of chunks) {
+                const search = chunk.search;
+                if (search?.schemaVersion !== 1 || !Array.isArray(search.paragraphs) || !search.paragraphs.every(text => typeof text === 'string') || !Array.isArray(search.literalLines) || !search.literalLines.every(text => typeof text === 'string')) {
+                    throw new Error('Search index needs prepared passage data; run shoalrun index <site-directory> before serving search. Existing passage embeddings can be reused.');
+                }
+            }
+    }
+    variant(options) {
+        const variants = this.variants();
+        const variant = options.variant ?? (variants.length === 1 ? variants[0] : undefined);
+        if (!variant)
+            throw new Error(`Search matches multiple publications (${variants.join(', ')}); pass a variant`);
+        if (!this.byVariant.has(variant))
+            throw new Error(`No indexed passages for publication ${variant}`);
+        return variant;
+    }
+    // A retrieved page's introduction can explain prerequisites that a matching subsection omits.
+    // It must pass the same publication and reader filters as the original search.
+    pageIntroduction(docId, options = {}) {
+        const keep = keeping(options);
+        return this.passages.get(this.variant(options))?.find(chunk => chunk.docId === docId && !chunk.headingPath.length && (!keep || keep(chunk)));
+    }
+    lexical(query, options = {}) {
+        return this.byVariant.get(this.variant(options)).search(query, options.limit, keeping(options));
+    }
+    vector(queryVector, options = {}) {
+        const index = this.vectors.get(this.variant(options));
+        if (!index)
+            throw new Error('No current embeddings for this publication; run shoalrun index --embed');
+        return index.search(queryVector, options.limit, keeping(options));
+    }
+    // Whether this publication has current vectors. Callers use it to say truthfully which mode answered.
+    hasVectors(options = {}) { return this.vectors.has(this.variant(options)); }
+    // Semantic search is optional, so hybrid degrades to lexical when there is no query vector or no current vectors for the
+    // publication. Passages without a current vector still compete through the lexical list.
+    hybrid(query, queryVector, options = {}) {
+        if (!queryVector || !this.hasVectors(options))
+            return this.lexical(query, options);
+        const depth = Math.max(50, (options.limit ?? 8) * 4);
+        return fuse([this.lexical(query, { ...options, limit: depth }), this.vector(queryVector, { ...options, limit: depth })], options.limit ?? 8);
+    }
+}

package/dist/scaffold.d.ts

@@ -0,0 +1,39 @@
+export type ScaffoldInput = {
+    id: string;
+    title: string;
+    summary: string;
+    slug: string;
+    operations?: string[];
+    channel?: string;
+    version?: string;
+    language?: string;
+};
+export declare const builtinAuthoringTemplates: Readonly<Record<string, {
+    type: string;
+    body: string;
+    required?: string[];
+}>>;
+export declare function scaffold(configFile: string, templateId: string, targetFile: string, input: ScaffoldInput): Promise<{
+    status: string;
+    template: string;
+    id: string;
+    file: string;
+    publication: {
+        channel: any;
+        version: any;
+        language: string;
+    };
+    access: string;
+}>;
+export declare function scaffoldFromPath(configFile: string, templateId: string, targetFile: string, overrides?: Partial<ScaffoldInput>): Promise<{
+    status: string;
+    template: string;
+    id: string;
+    file: string;
+    publication: {
+        channel: any;
+        version: any;
+        language: string;
+    };
+    access: string;
+}>;

package/dist/scaffold.js

@@ -0,0 +1,167 @@
+import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import YAML from 'yaml';
+import { explainSegments, loadConfig } from './config.js';
+import { defaultLanguage, resolveFrontmatter } from './frontmatter.js';
+import { routeFor } from './content.js';
+const object = (value) => !!value && typeof value === 'object' && !Array.isArray(value);
+async function canonicalPath(value) {
+    const resolved = path.resolve(value);
+    try {
+        return await realpath(resolved);
+    }
+    catch {
+        const parent = path.dirname(resolved);
+        if (parent === resolved)
+            return resolved;
+        return path.join(await canonicalPath(parent), path.basename(resolved));
+    }
+}
+export const builtinAuthoringTemplates = {
+    reference: { type: 'reference', body: 'reference.mdx' },
+    'api-reference': { type: 'api-reference', body: 'api-reference.mdx', required: ['operations'] },
+    'how-to': { type: 'how-to', body: 'how-to.mdx' },
+    explanation: { type: 'explanation', body: 'explanation.mdx' },
+    recipe: { type: 'recipe', body: 'recipe.mdx' },
+    ingredient: { type: 'ingredient', body: 'ingredient.mdx' },
+    cookbook: { type: 'cookbook', body: 'cookbook.mdx' },
+    guide: { type: 'guide', body: 'guide.mdx' },
+    tutorial: { type: 'tutorial', body: 'tutorial.mdx' },
+    troubleshooting: { type: 'troubleshooting', body: 'troubleshooting.mdx' }
+};
+// A template may open with commented frontmatter. Scaffold keeps its comments and extra fields, and sets the fields it owns.
+function splitTemplate(source, bodyFile) {
+    const match = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(source);
+    if (!match) {
+        if (/^---/.test(source))
+            throw new Error(`${bodyFile}: template frontmatter is not closed by a --- line`);
+        return { body: source };
+    }
+    const header = YAML.parseDocument(match[1], { uniqueKeys: true });
+    if (header.errors.length)
+        throw new Error(`${bodyFile}: template frontmatter ${header.errors.map(error => error.message).join('; ')}`);
+    if (!YAML.isMap(header.contents))
+        throw new Error(`${bodyFile}: template frontmatter must be a mapping`);
+    return { header, body: source.slice(match[0].length) };
+}
+function fillTemplateHeader(header, owned) {
+    const map = header.contents;
+    for (const [key, value] of Object.entries(owned)) {
+        const existing = map.items.find(pair => YAML.isScalar(pair.key) && pair.key.value === key);
+        const node = header.createNode(value);
+        if (existing) {
+            if (YAML.isNode(existing.value)) {
+                node.comment = existing.value.comment;
+                node.commentBefore = existing.value.commentBefore;
+                if (YAML.isCollection(node) && YAML.isCollection(existing.value))
+                    node.flow = existing.value.flow;
+            }
+            existing.value = node;
+        }
+        else
+            map.add(header.createPair(key, node));
+    }
+    if (!('operations' in owned))
+        map.delete('operations');
+    return header.toString({ lineWidth: 0 });
+}
+export async function scaffold(configFile, templateId, targetFile, input) {
+    const loaded = await loadConfig(configFile);
+    const config = loaded.value;
+    if (config.templates !== undefined && !object(config.templates))
+        throw new Error('templates must be a mapping');
+    const localTemplate = config.templates?.[templateId];
+    const template = localTemplate ?? builtinAuthoringTemplates[templateId];
+    if (!object(template))
+        throw new Error(`Unknown template ${templateId}`);
+    for (const key of Object.keys(template))
+        if (!['type', 'body', 'required'].includes(key))
+            throw new Error(`Unknown template setting ${templateId}.${key}`);
+    if (typeof template.type !== 'string' || !template.type)
+        throw new Error(`Template ${templateId} needs a document type`);
+    if (typeof template.body !== 'string' || !/\.mdx?$/.test(template.body))
+        throw new Error(`Template ${templateId} needs a .md or .mdx body`);
+    if (template.required !== undefined && (!Array.isArray(template.required) || template.required.some((field) => field !== 'operations')))
+        throw new Error(`Template ${templateId}.required only supports operations in this slice`);
+    for (const field of ['id', 'title', 'summary', 'slug'])
+        if (typeof input[field] !== 'string' || !input[field].trim())
+            throw new Error(`Scaffold needs ${field}`);
+    if (!input.slug.startsWith('/') || input.slug.includes('..'))
+        throw new Error('Scaffold slug must be an absolute path without traversal');
+    if (input.operations !== undefined && (!Array.isArray(input.operations) || input.operations.some(item => typeof item !== 'string' || !item)))
+        throw new Error('Scaffold operations must be stable IDs');
+    if (template.required?.includes('operations') && !input.operations?.length)
+        throw new Error(`Template ${templateId} requires operations`);
+    if (template.type === 'api-reference' && input.operations?.length !== 1)
+        throw new Error(`Template ${templateId} needs exactly one operation for an api-reference page`);
+    const knownOperations = new Set(Object.values(config.apis ?? {}).flatMap(api => (api.operations ?? []).map((item) => item.id)));
+    for (const operation of input.operations ?? [])
+        if (!knownOperations.has(operation))
+            throw new Error(`Unknown scaffold operation ${operation}`);
+    const defaults = config.publications?.defaults;
+    const applicability = resolveFrontmatter(Object.fromEntries(Object.entries({ channel: input.channel, version: input.version, language: input.language }).filter(([, value]) => value !== undefined)), config.frontmatter);
+    const channel = applicability.channel ?? defaults?.channel;
+    const version = applicability.version ?? defaults?.version;
+    const language = applicability.language ?? defaultLanguage(config);
+    if (!channel || !version || !language)
+        throw new Error('Scaffold needs channel, version, and language');
+    if (!config.publications?.enabled?.some((item) => item.channel === channel && item.version === version && (item.language ?? defaultLanguage(config)) === language))
+        throw new Error(`Scaffold publication ${channel}/${version}/${language} is not enabled`);
+    const root = path.dirname(path.resolve(configFile));
+    const contentDir = await canonicalPath(path.resolve(root, config.site?.contentDir ?? 'content'));
+    const target = await canonicalPath(targetFile);
+    const relative = path.relative(contentDir, target);
+    if (!relative || relative.startsWith('..') || path.isAbsolute(relative) || !/\.mdx?$/.test(target))
+        throw new Error(`Scaffold target must be a .md or .mdx file inside ${contentDir}`);
+    const bodyFile = localTemplate
+        ? path.resolve(path.dirname(explainSegments(loaded, ['templates', templateId, 'body']).source.file), template.body)
+        : path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../templates', template.body);
+    const { header, body } = splitTemplate(await readFile(bodyFile, 'utf8'), bodyFile);
+    const frontmatter = { id: input.id, title: input.title, summary: input.summary, type: template.type, slug: input.slug, channel, version, language, ...(input.operations?.length ? { operations: input.operations } : {}), access: { policy: 'draft' } };
+    routeFor(frontmatter, { channel, version, language }, config);
+    const written = header ? fillTemplateHeader(header, frontmatter) : YAML.stringify(frontmatter);
+    const content = `---\n${written.replace(/\s*$/, '\n')}---\n\n${body.trimStart().replace(/\s*$/, '\n')}`;
+    await mkdir(path.dirname(target), { recursive: true });
+    await writeFile(target, content, { flag: 'wx' });
+    return { status: 'ok', template: templateId, id: input.id, file: target, publication: { channel, version, language }, access: 'draft' };
+}
+export async function scaffoldFromPath(configFile, templateId, targetFile, overrides = {}) {
+    const loaded = await loadConfig(configFile);
+    const config = loaded.value;
+    const contentDir = await canonicalPath(path.resolve(path.dirname(path.resolve(configFile)), config.site?.contentDir ?? 'content'));
+    const relative = path.relative(contentDir, await canonicalPath(targetFile));
+    if (!relative || relative.startsWith('..') || path.isAbsolute(relative) || !/\.mdx?$/.test(relative))
+        throw new Error(`Scaffold target must be a .md or .mdx file inside ${contentDir}`);
+    const parts = relative.replace(/\.mdx?$/, '').split(path.sep).map(part => part.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''));
+    if (parts.some(part => !part))
+        throw new Error('Scaffold target needs a usable filename');
+    const title = parts.at(-1).split('-').map(word => word[0].toUpperCase() + word.slice(1)).join(' ');
+    const selected = resolveFrontmatter(Object.fromEntries(Object.entries({ channel: overrides.channel, version: overrides.version, language: overrides.language }).filter(([, value]) => value !== undefined)), config.frontmatter);
+    const channel = selected.channel ?? config.publications?.defaults?.channel;
+    const version = selected.version ?? config.publications?.defaults?.version;
+    const language = selected.language ?? defaultLanguage(config);
+    const prefixGroups = [
+        language === defaultLanguage(config) ? '' : language,
+        config.publications?.channels?.[channel]?.pathPrefix ?? '',
+        config.publications?.versions?.[version]?.pathPrefix ?? ''
+    ].map(value => value.split('/').filter(Boolean));
+    const prefixCandidates = [];
+    for (let start = 0; start < prefixGroups.length; start++) {
+        for (let end = start + 1; end <= prefixGroups.length; end++) {
+            const candidate = prefixGroups.slice(start, end).flat();
+            if (candidate.length)
+                prefixCandidates.push(candidate);
+        }
+    }
+    prefixCandidates.sort((a, b) => b.length - a.length);
+    const matchingPrefix = prefixCandidates.find(candidate => parts.length > candidate.length && candidate.every((part, index) => parts[index] === part));
+    const slugParts = matchingPrefix ? parts.slice(matchingPrefix.length) : parts;
+    return scaffold(configFile, templateId, targetFile, {
+        id: `${templateId}.${parts.join('.')}`,
+        title,
+        summary: `Draft guide for ${title.toLowerCase()}. Replace this summary before publication.`,
+        slug: `/${slugParts.join('/')}`,
+        ...overrides
+    });
+}

package/dist/search-config.d.ts

@@ -0,0 +1,8 @@
+export type SearchLimits = {
+    perCallerPerDay: number;
+    perDay: number;
+    perCallerPerMinute: number;
+};
+export declare function validateSearch(value: unknown): {
+    limits: SearchLimits;
+};

package/dist/search-config.js

@@ -0,0 +1,26 @@
+import { validateSearchSynonyms } from './search-synonyms.js';
+// Hosted search has its own limits; chat budgets do not change these defaults.
+export function validateSearch(value) {
+    const limits = { perCallerPerDay: 100, perDay: 1000, perCallerPerMinute: 10 };
+    if (value === undefined)
+        return { limits };
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error('search must be an object');
+    const config = value;
+    for (const key of Object.keys(config))
+        if (!['synonyms', 'limits'].includes(key))
+            throw new Error(`Unknown search setting: ${key}`);
+    validateSearchSynonyms(config.synonyms);
+    if (config.limits === undefined)
+        return { limits };
+    if (!config.limits || typeof config.limits !== 'object' || Array.isArray(config.limits))
+        throw new Error('search.limits must be an object');
+    for (const [key, limit] of Object.entries(config.limits)) {
+        if (!Object.hasOwn(limits, key))
+            throw new Error(`Unknown search.limits setting: ${key}`);
+        if (typeof limit !== 'number' || !Number.isSafeInteger(limit) || limit < 1)
+            throw new Error(`search.limits.${key} must be a positive whole number`);
+        limits[key] = limit;
+    }
+    return { limits };
+}

package/dist/search-handler.d.ts

@@ -0,0 +1,16 @@
+import type { Retriever } from './retrieval.js';
+import type { ReaderFiltersConfig } from './reader-filters.js';
+import type { DailyBudget, SharedRateLimit } from './budget.js';
+export type SearchFallbackReason = 'embeddings-unavailable' | 'budget-unavailable' | 'budget-denied' | 'provider-error' | 'deadline';
+export declare function createSearchHandler(options: {
+    retriever: Retriever;
+    embedQuery?: (query: string, signal?: AbortSignal) => Promise<Float32Array>;
+    semanticTimeoutMs?: number;
+    onFallback?: (reason: SearchFallbackReason) => void;
+    hosts: string[];
+    readerFilters?: ReaderFiltersConfig;
+    pageSummaries?: Map<string, string>;
+    rateLimiter?: Pick<SharedRateLimit, 'take'>;
+    budget?: Pick<DailyBudget, 'reserve'>;
+    clientKey?: (request: Request) => string;
+}): (request: Request) => Promise<Response>;