@shoalrun/cli 0.0.0-stage → 0.1.0-alpha.11

package/dist/function-package.js

@@ -0,0 +1,58 @@
+import { cp, mkdir, readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+const importPattern = /(?:import|export)\s[^'"]*?from\s*['"]([^'"]+)['"]|import\s*\(\s*['"]([^'"]+)['"]\s*\)|^import\s*['"]([^'"]+)['"]/gm;
+// The compiled files a module needs, followed through its relative imports, and the packages it names.
+async function closure(dist, entry) {
+    const files = new Set();
+    const packages = new Set();
+    const visit = async (file) => {
+        if (path.isAbsolute(file) || file.split(path.sep).includes('..'))
+            throw new Error('Server adapter relative imports must stay inside its module directory');
+        if (files.has(file))
+            return;
+        files.add(file);
+        const source = await readFile(path.join(dist, file), 'utf8');
+        for (const match of source.matchAll(importPattern)) {
+            const specifier = match[1] ?? match[2] ?? match[3];
+            if (!specifier || specifier.startsWith('node:'))
+                continue;
+            if (specifier.startsWith('.'))
+                await visit(path.normalize(path.join(path.dirname(file), specifier)));
+            else
+                packages.add(specifier.startsWith('@') ? specifier.split('/').slice(0, 2).join('/') : specifier.split('/')[0]);
+        }
+    };
+    await visit(entry);
+    return { files: [...files], packages: [...packages] };
+}
+// A package and the packages it depends on, copied whole.
+async function copyPackage(name, from, target, done) {
+    if (done.has(name))
+        return;
+    done.add(name);
+    let directory;
+    for (let at = from; !directory; at = path.dirname(at)) {
+        const candidate = path.join(at, 'node_modules', name);
+        if (await stat(path.join(candidate, 'package.json')).then(() => true, () => false))
+            directory = candidate;
+        else if (path.dirname(at) === at)
+            break;
+    }
+    if (!directory)
+        throw new Error(`The server function needs the package ${name}, and it is not installed`);
+    await cp(directory, path.join(target, 'node_modules', name), { recursive: true, dereference: true });
+    const manifest = JSON.parse(await readFile(path.join(directory, 'package.json'), 'utf8'));
+    const requiredPeers = Object.keys(manifest.peerDependencies ?? {}).filter(name => !manifest.peerDependenciesMeta?.[name]?.optional);
+    for (const dependency of new Set([...Object.keys(manifest.dependencies ?? {}), ...requiredPeers]))
+        await copyPackage(dependency, path.dirname(directory), target, done);
+}
+export async function copyFunctionRuntime(dist, entry, target) {
+    const { files, packages } = await closure(dist, entry);
+    for (const file of files) {
+        await mkdir(path.dirname(path.join(target, 'lib', file)), { recursive: true });
+        await cp(path.join(dist, file), path.join(target, 'lib', file));
+    }
+    const done = new Set();
+    for (const name of packages)
+        await copyPackage(name, dist, target, done);
+}

package/dist/highlight.d.ts

@@ -0,0 +1,14 @@
+export type HighlightToken = {
+    text: string;
+    color?: string;
+};
+export type CodeHighlighter = {
+    highlight: (code: string, language?: string) => HighlightToken[];
+    background?: string;
+    foreground?: string;
+    dark?: boolean;
+    dependency?: string;
+};
+export declare function createCodeHighlighter(config?: any, declaringFile?: string): Promise<CodeHighlighter>;
+export declare function highlightCode(highlighter?: CodeHighlighter): (tree: any) => void;
+export declare function highlightedExample(value: string, language: string, highlighter?: CodeHighlighter): string;

package/dist/highlight.js

@@ -0,0 +1,114 @@
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { unified } from 'unified';
+import rehypeStringify from 'rehype-stringify';
+const plain = { highlight: text => [{ text }] };
+const stringify = unified().use(rehypeStringify);
+const hex = /^#(?:[\da-f]{3}|[\da-f]{4}|[\da-f]{6}|[\da-f]{8})$/i;
+// The site theme colors tokens with CSS variables that each palette sets.
+const siteColor = /^var\(--code-(?:foreground|token-[a-z-]+)\)$/;
+export async function createCodeHighlighter(config = {}, declaringFile) {
+    if (!config || typeof config !== 'object' || Array.isArray(config))
+        throw new Error('outputs.html.highlighting must be a mapping');
+    const provider = config.provider ?? 'shiki';
+    const allowed = provider === 'shiki' ? ['provider', 'theme'] : provider === 'custom' ? ['provider', 'module'] : ['provider'];
+    for (const key of Object.keys(config))
+        if (!allowed.includes(key))
+            throw new Error(`Unsupported highlighting setting ${key} for ${provider}`);
+    if (provider === 'none')
+        return plain;
+    if (provider === 'custom') {
+        if (typeof config.module !== 'string' || !config.module || !declaringFile)
+            throw new Error('Custom highlighting needs a local module path');
+        const file = path.resolve(path.dirname(declaringFile), config.module);
+        const adapter = await import(pathToFileURL(file).href);
+        if (typeof adapter.highlight !== 'function')
+            throw new Error(`${file}: export a synchronous highlight(code, language) function`);
+        return { highlight: adapter.highlight, dependency: file };
+    }
+    if (provider !== 'shiki')
+        throw new Error(`Unknown highlighting provider ${provider}`);
+    // Load only when selected. Shiki caches its build-time instance and grammars.
+    const { getSingletonHighlighter, bundledThemes, createCssVariablesTheme } = await import('shiki');
+    const theme = config.theme ?? 'site';
+    if (typeof theme !== 'string' || (theme !== 'site' && !Object.hasOwn(bundledThemes, theme)))
+        throw new Error(`Unknown Shiki theme ${theme}; use site or a bundled Shiki theme name`);
+    const highlighter = await getSingletonHighlighter({
+        themes: [theme === 'site' ? createCssVariablesTheme({ name: 'site', variablePrefix: '--code-' }) : theme],
+        langs: ['json', 'yaml', 'bash', 'javascript', 'typescript', 'python', 'html', 'css', 'mdx', 'markdown', 'http', 'xml', 'sql', 'go', 'java', 'ruby', 'csharp', 'php', 'rust']
+    });
+    const languages = new Set(highlighter.getLoadedLanguages());
+    const colors = highlighter.getTheme(theme);
+    return {
+        ...(theme === 'site' ? {} : { background: colors.bg, foreground: colors.fg, dark: colors.type === 'dark' }),
+        highlight(value, language) {
+            const lang = language?.toLowerCase();
+            if (!lang || !languages.has(lang))
+                return [{ text: value }];
+            const breaks = value.match(/\r\n|\r|\n/g) ?? [];
+            return highlighter.codeToTokens(value, { lang: lang, theme }).tokens.flatMap((line, index) => [
+                ...(index ? [{ text: breaks[index - 1] }] : []),
+                ...line.map(token => ({ text: token.content, color: token.color }))
+            ]);
+        }
+    };
+}
+function tokens(value, language, highlighter) {
+    const result = highlighter.highlight(value, language);
+    if (!Array.isArray(result) || result.some(token => !token || typeof token.text !== 'string' || (token.color !== undefined && (typeof token.color !== 'string' || !(hex.test(token.color) || siteColor.test(token.color))))) || result.map(token => token.text).join('') !== value) {
+        throw new Error('Highlighter must return text tokens with optional hex colors and preserve the exact code text');
+    }
+    return result.map(token => token.color ? {
+        type: 'element', tagName: 'span', properties: { style: `color:${token.color}` }, children: [{ type: 'text', value: token.text }]
+    } : { type: 'text', value: token.text });
+}
+// A dark theme leaves the background to the palette so the block matches its toolbar.
+function preProperties(highlighter) {
+    if (!highlighter.background || !highlighter.foreground)
+        return {};
+    return { style: highlighter.dark ? `color:${highlighter.foreground}` : `background-color:${highlighter.background};color:${highlighter.foreground}` };
+}
+export function highlightCode(highlighter = plain) {
+    return (tree) => {
+        const visit = (node) => {
+            if (node.children)
+                node.children = node.children.map((child) => visit(child));
+            if (node.tagName === 'pre') {
+                const code = node.children?.find((child) => child.tagName === 'code');
+                if (code && code.children.every((child) => child.type === 'text')) {
+                    const language = code.properties?.className?.find((name) => name.startsWith('language-'))?.slice(9);
+                    code.children = tokens(code.children.map((child) => child.value).join(''), language, highlighter);
+                    node.properties = { ...node.properties, ...preProperties(highlighter) };
+                }
+                const metadata = new Map();
+                for (const match of String(code?.data?.meta ?? '').matchAll(/(?:^|\s)(title|filename)=(?:"([^"]+)"|'([^']+)'|([^\s]+))(?=\s|$)/g))
+                    metadata.set(match[1], match[2] ?? match[3] ?? match[4]);
+                const title = metadata.get('title');
+                const filename = metadata.get('filename');
+                if (title || filename)
+                    return {
+                        type: 'element', tagName: 'div', properties: { className: ['code-block'] }, children: [
+                            { type: 'element', tagName: 'div', properties: { className: ['code-toolbar'] }, children: [
+                                    { type: 'element', tagName: 'span', properties: { className: ['code-block-meta'] }, children: [
+                                            ...(title ? [{ type: 'element', tagName: 'span', properties: { className: ['code-block-title'] }, children: [{ type: 'text', value: title }] }] : []),
+                                            ...(filename ? [{ type: 'element', tagName: 'span', properties: { className: ['code-block-filename'] }, children: [{ type: 'text', value: filename }] }] : [])
+                                        ] }
+                                ] },
+                            node
+                        ]
+                    };
+            }
+            return node;
+        };
+        visit(tree);
+    };
+}
+export function highlightedExample(value, language, highlighter = plain) {
+    return stringify.stringify({
+        type: 'root', children: [{
+                type: 'element', tagName: 'pre', properties: preProperties(highlighter), children: [{
+                        type: 'element', tagName: 'code', properties: { className: [`language-${language}`] }, children: tokens(value, language, highlighter)
+                    }]
+            }]
+    });
+}

package/dist/html-renderer.d.ts

@@ -0,0 +1,10 @@
+import { type CodeHighlighter } from './highlight.js';
+export declare function renderHtmlWithHeadings(tree: any, highlighter?: CodeHighlighter, reservedIds?: Iterable<string>): {
+    html: string;
+    headings: Array<{
+        depth: number;
+        id: string;
+        title: string;
+    }>;
+};
+export declare function renderHtml(tree: any, highlighter?: CodeHighlighter): string;

package/dist/html-renderer.js

@@ -0,0 +1,63 @@
+import { unified } from 'unified';
+import remarkRehype from 'remark-rehype';
+import rehypeStringify from 'rehype-stringify';
+import { highlightCode } from './highlight.js';
+import { walkthroughCode } from './walkthrough.js';
+export function renderHtmlWithHeadings(tree, highlighter, reservedIds = []) {
+    const math = new Map();
+    const collectMath = (node) => {
+        if (node.data?.mathMarkup)
+            math.set(node.value, node.data.mathMarkup);
+        node.children?.forEach(collectMath);
+    };
+    collectMath(tree);
+    const html = unified().use(remarkRehype).use(highlightCode, highlighter).use(walkthroughCode).use(rehypeStringify);
+    const hast = html.runSync(tree);
+    const used = new Set(reservedIds);
+    const headings = [];
+    const textOf = (node) => node.type === 'text' ? node.value : (node.children ?? []).map(textOf).join('');
+    const checkUrl = (value) => {
+        if (typeof value !== 'string')
+            return;
+        const protocol = new URL(value, 'https://shoalrun.invalid/').protocol;
+        if (!['https:', 'http:', 'mailto:'].includes(protocol))
+            throw new Error(`Unsafe URL in rendered content: ${value}`);
+    };
+    const collectIds = (node) => {
+        if (node.tagName === 'a')
+            checkUrl(node.properties?.href);
+        if (node.tagName === 'img')
+            checkUrl(node.properties?.src);
+        if (typeof node.properties?.id === 'string')
+            used.add(node.properties.id);
+        node.children?.forEach(collectIds);
+    };
+    collectIds(hast);
+    const visit = (node) => {
+        const match = /^h([2-6])$/.exec(node.tagName ?? '');
+        if (match && !node.properties?.className?.some((name) => ['tab-fallback-heading', 'walkthrough-step-title'].includes(name))) {
+            const title = textOf(node).trim();
+            const depth = Number(match[1]);
+            if (title) {
+                let id = node.properties?.id;
+                if (typeof id !== 'string' || !id) {
+                    const stem = title.normalize('NFKD').toLowerCase().replace(/[\u0300-\u036f]/g, '').replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'section';
+                    id = stem;
+                    for (let suffix = 2; used.has(id); suffix++)
+                        id = `${stem}-${suffix}`;
+                    node.properties = { ...node.properties, id };
+                    used.add(id);
+                }
+                if (!node.properties?.className?.includes('doc-step-heading-toc-ignore'))
+                    headings.push({ depth, id, title });
+            }
+        }
+        node.children?.forEach(visit);
+    };
+    visit(hast);
+    let rendered = String(html.stringify(hast));
+    for (const [marker, markup] of math)
+        rendered = rendered.replaceAll(marker, markup);
+    return { html: rendered, headings };
+}
+export function renderHtml(tree, highlighter) { return renderHtmlWithHeadings(tree, highlighter).html; }

package/dist/icons.d.ts

@@ -0,0 +1,8 @@
+export declare const iconLicenseFile: string;
+export declare function validateCardIcon(name: string, file: string): void;
+export declare function validateLucideIcon(name: string, file: string, field: string): void;
+export declare function cardIcon(name: string): any;
+export declare function calloutIcon(name: string): any;
+export declare function treeIcon(name: string): any;
+export declare function badgeIcon(name: string): any;
+export declare function stepIcon(name: string): any;

package/dist/icons.js

@@ -0,0 +1,40 @@
+import { icons } from 'lucide';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+export const iconLicenseFile = path.join(path.dirname(fileURLToPath(import.meta.resolve('lucide/package.json'))), 'LICENSE');
+function iconNodes(name) {
+    if (!/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/.test(name))
+        return undefined;
+    const key = name.split('-').map(part => part[0].toUpperCase() + part.slice(1)).join('');
+    return Object.hasOwn(icons, key) ? icons[key] : undefined;
+}
+export function validateCardIcon(name, file) {
+    validateLucideIcon(name, file, 'card.icon');
+}
+export function validateLucideIcon(name, file, field) {
+    if (!iconNodes(name))
+        throw new Error(`${file}: unknown Lucide ${field} "${name}"; use a lowercase icon name such as book-open`);
+}
+function lucideIcon(name, className, type) {
+    const nodes = iconNodes(name);
+    if (!nodes)
+        throw new Error(`Unknown Lucide icon ${name}`);
+    return {
+        type, value: '', children: [],
+        data: {
+            htmlOnly: true,
+            hName: 'svg',
+            hProperties: {
+                xmlns: 'http://www.w3.org/2000/svg', viewBox: '0 0 24 24', width: 24, height: 24,
+                fill: 'none', stroke: 'currentColor', strokeWidth: 1.75, strokeLinecap: 'round', strokeLinejoin: 'round',
+                ariaHidden: 'true', focusable: 'false', className: [className], 'data-lucide': name
+            },
+            hChildren: nodes.map(([tagName, properties]) => ({ type: 'element', tagName, properties: { ...properties }, children: [] }))
+        }
+    };
+}
+export function cardIcon(name) { return lucideIcon(name, 'doc-card-icon', 'paragraph'); }
+export function calloutIcon(name) { return lucideIcon(name, 'callout-icon', 'text'); }
+export function treeIcon(name) { return lucideIcon(name, 'file-tree-icon', 'text'); }
+export function badgeIcon(name) { return lucideIcon(name, 'doc-badge-icon', 'text'); }
+export function stepIcon(name) { return lucideIcon(name, 'doc-step-icon', 'text'); }

package/dist/index-build.d.ts

@@ -0,0 +1,55 @@
+import { type PreparedSearchPassage } from './search-snippet.js';
+export type Chunk = {
+    id: string;
+    docId: string;
+    title: string;
+    type: string;
+    variant: string;
+    headingPath: string[];
+    url: string;
+    markdownUrl: string;
+    text: string;
+    digest: string;
+    readerFilters?: Record<string, string[]>;
+    search?: PreparedSearchPassage;
+};
+export type ChunkIndex = {
+    schemaVersion: 1;
+    sourceDigest: string;
+    chunks: Chunk[];
+    synonyms?: string[][];
+};
+export type ChunkOptions = {
+    maxChars?: number;
+    minChars?: number;
+};
+type ChunkSource = Pick<ManifestDocument, 'id' | 'title' | 'type' | 'url' | 'markdownUrl' | 'readerFilters'> & {
+    variant: string;
+};
+type ManifestDocument = {
+    id: string;
+    type: string;
+    title: string;
+    url: string;
+    markdownUrl: string;
+    digest: string;
+    readerFilters?: Record<string, string[]>;
+    variant: {
+        channel: string;
+        version: string;
+        language: string;
+    };
+};
+export declare function embeddingInput(chunk: Pick<Chunk, 'title' | 'headingPath' | 'text'>): string;
+export declare function splitBlocks(text: string): string[];
+export declare function chunkMarkdown(markdown: string, document: ChunkSource, anchors?: Array<{
+    id: string;
+    title: string;
+}>, options?: ChunkOptions): Chunk[];
+export declare function aiDirectory(directory: string): Promise<string>;
+export declare function buildChunkIndex(directory: string, options?: ChunkOptions): Promise<{
+    index: ChunkIndex;
+    output: string;
+    documents: number;
+}>;
+export {};

package/dist/index-build.js

@@ -0,0 +1,227 @@
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { validateSearchSynonyms } from './search-synonyms.js';
+import { prepareSearchPassage } from './search-snippet.js';
+const fence = /^\s{0,3}(`{3,}|~{3,})/;
+const heading = /^(#{1,6})\s+(.+?)\s*#*\s*$/;
+const sha = (value) => createHash('sha256').update(value).digest('hex');
+const slug = (value) => value.toLocaleLowerCase().replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, '');
+// Headings in the Markdown twin keep inline formatting; the search index stores plain text.
+const plain = (value) => value.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1').replace(/[`*_]/g, '').replace(/\s+/g, ' ').trim().toLocaleLowerCase();
+// The text a chunk is embedded and lexically scored as: page title and heading path give a passage its context.
+export function embeddingInput(chunk) {
+    return `${chunk.title}${chunk.headingPath.length ? ` > ${chunk.headingPath.join(' > ')}` : ''}\n\n${chunk.text}`;
+}
+function stripFrontmatter(markdown) {
+    const match = /^---\r?\n[\s\S]*?\r?\n---\r?\n/.exec(markdown);
+    return match ? markdown.slice(match[0].length) : markdown;
+}
+// Split at h2 and h3. Fenced code stays whole, and lower headings stay inside their section's text.
+function sections(body) {
+    const result = [];
+    const path = [];
+    let current = { level: 1, title: '', path: [], blocks: [] };
+    let block = [];
+    let open;
+    const flush = () => { if (block.length)
+        current.blocks.push(block.join('\n')); block = []; };
+    for (const line of body.split(/\r?\n/)) {
+        const marker = fence.exec(line)?.[1];
+        if (open) {
+            block.push(line);
+            if (marker && marker[0] === open[0] && marker.length >= open.length && !line.replace(fence, '').trim())
+                open = undefined;
+            continue;
+        }
+        if (marker) {
+            open = marker;
+            block.push(line);
+            continue;
+        }
+        const found = heading.exec(line);
+        if (found && found[1].length <= 3) {
+            flush();
+            result.push(current);
+            const level = found[1].length;
+            if (level === 1) {
+                current = { level: 1, title: '', path: [], blocks: [] };
+                continue;
+            }
+            path.splice(level - 2);
+            path.push(found[2]);
+            current = { level, title: found[2], path: [...path], blocks: [] };
+            continue;
+        }
+        if (!line.trim())
+            flush();
+        else
+            block.push(line);
+    }
+    flush();
+    result.push(current);
+    return result.filter(section => section.blocks.length || section.title);
+}
+// Paragraphs, lists, tables, and fenced code as separate blocks, in order. A fence stays whole even when it contains blank lines.
+export function splitBlocks(text) {
+    const blocks = [];
+    let block = [];
+    let open;
+    const flush = () => { if (block.length)
+        blocks.push(block.join('\n')); block = []; };
+    for (const line of text.split(/\r?\n/)) {
+        const marker = fence.exec(line)?.[1];
+        if (open) {
+            block.push(line);
+            if (marker && marker[0] === open[0] && marker.length >= open.length && !line.replace(fence, '').trim()) {
+                open = undefined;
+                flush();
+            }
+            continue;
+        }
+        if (marker) {
+            flush();
+            open = marker;
+            block.push(line);
+            continue;
+        }
+        if (!line.trim())
+            flush();
+        else
+            block.push(line);
+    }
+    flush();
+    return blocks;
+}
+// A table larger than the limit is cut between rows, and each piece repeats the header so a row keeps its column names.
+function splitTable(block, maxChars) {
+    const lines = block.split('\n');
+    if (block.length <= maxChars || lines.length < 3 || !lines.every(line => line.trimStart().startsWith('|')) || !/^\s*\|[\s:|-]+\|?\s*$/.test(lines[1]))
+        return [block];
+    const header = lines.slice(0, 2).join('\n');
+    const parts = [];
+    let rows = [];
+    for (const row of lines.slice(2)) {
+        if (rows.length && header.length + rows.join('\n').length + row.length + 2 > maxChars) {
+            parts.push(`${header}\n${rows.join('\n')}`);
+            rows = [];
+        }
+        rows.push(row);
+    }
+    if (rows.length)
+        parts.push(`${header}\n${rows.join('\n')}`);
+    return parts;
+}
+// Greedy-pack blocks up to maxChars. A code sample larger than the limit stays whole.
+function pack(blocks, maxChars) {
+    const parts = [];
+    let current = '';
+    for (const block of blocks.flatMap(item => splitTable(item, maxChars))) {
+        if (current && current.length + block.length + 2 > maxChars) {
+            parts.push(current);
+            current = '';
+        }
+        current = current ? `${current}\n\n${block}` : block;
+    }
+    if (current)
+        parts.push(current);
+    return parts;
+}
+export function chunkMarkdown(markdown, document, anchors = [], options = {}) {
+    const maxChars = options.maxChars ?? 2000;
+    const minChars = options.minChars ?? 200;
+    const chunks = [];
+    const used = new Map();
+    let cursor = 0;
+    // A section too short to stand alone rides along with its first child section, which reads better than a fragment.
+    let carry;
+    for (const section of sections(stripFrontmatter(markdown))) {
+        let anchor;
+        if (section.title) {
+            const wanted = plain(section.title);
+            const at = anchors.findIndex((item, index) => index >= cursor && plain(item.title) === wanted);
+            if (at >= 0) {
+                anchor = anchors[at].id;
+                cursor = at + 1;
+            }
+        }
+        const child = carry && section.path.length > carry.path.length && carry.path.every((part, index) => section.path[index] === part);
+        if (carry && !child)
+            chunks.push(makeChunk(document, carry.path, carry.anchor, carry.text, used));
+        const blocks = carry && child ? [carry.text, ...section.blocks] : section.blocks;
+        carry = undefined;
+        const text = blocks.join('\n\n');
+        if (!text.trim())
+            continue;
+        if (text.length < minChars && section.path.length && blocks === section.blocks) {
+            carry = { text, path: section.path, anchor };
+            continue;
+        }
+        pack(blocks, maxChars).forEach((part, index) => chunks.push(makeChunk(document, section.path, anchor, part, used, index)));
+    }
+    if (carry)
+        chunks.push(makeChunk(document, carry.path, carry.anchor, carry.text, used));
+    return chunks;
+}
+function makeChunk(document, headingPath, anchor, text, used, part = 0) {
+    const base = `${document.id}#${slug(headingPath.join(' ')) || 'intro'}${part ? `~${part + 1}` : ''}`;
+    const seen = used.get(base) ?? 0;
+    used.set(base, seen + 1);
+    const chunk = { id: seen ? `${base}-${seen + 1}` : base, docId: document.id, title: document.title, type: document.type, variant: document.variant, headingPath, url: anchor ? `${document.url}#${anchor}` : document.url, markdownUrl: document.markdownUrl, text };
+    // The page's reader marks (audience, product, and so on) travel with each passage so retrieval can filter by them. They are not part
+    // of the embedding input, so changing a page's audience never forces a re-embed.
+    const marks = document.readerFilters && Object.keys(document.readerFilters).length ? { readerFilters: document.readerFilters } : {};
+    return { ...chunk, ...marks, digest: sha(embeddingInput(chunk)), search: prepareSearchPassage(text) };
+}
+async function readJson(file) {
+    try {
+        return JSON.parse(await readFile(file, 'utf8'));
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return undefined;
+        throw error;
+    }
+}
+// The directory that holds the passage and vector files. It is created on demand, and it must stay inside the site even if
+// `ai` is a symlink.
+export async function aiDirectory(directory) {
+    const root = path.resolve(directory);
+    const target = path.join(root, 'ai');
+    await mkdir(target, { recursive: true });
+    const real = await realpath(root);
+    const inside = await realpath(target);
+    if (inside !== real && !inside.startsWith(real + path.sep))
+        throw new Error('The ai directory leaves the site directory');
+    return target;
+}
+export async function buildChunkIndex(directory, options = {}) {
+    const root = path.resolve(directory);
+    const manifest = await readJson(path.join(root, 'documents.json'));
+    if (!manifest || !Array.isArray(manifest.documents))
+        throw new Error(`No documents.json in ${directory}; build the site first`);
+    const synonyms = validateSearchSynonyms(manifest.searchSynonyms);
+    const search = await readJson(path.join(root, 'search-index.json'));
+    const anchors = new Map((search?.documents ?? []).map(item => [item.url, item.headings]));
+    const chunks = [];
+    const realRoot = await realpath(root);
+    const inside = (file) => file === realRoot || file.startsWith(realRoot + path.sep);
+    for (const document of manifest.documents) {
+        if (typeof document?.markdownUrl !== 'string')
+            throw new Error('documents.json has a page without a markdownUrl');
+        const file = path.resolve(root, `.${document.markdownUrl}`);
+        // The path check catches `..`; the real path catches a symlink that points out of the site directory.
+        if (file !== root && !file.startsWith(root + path.sep))
+            throw new Error(`Markdown path for ${document.id} leaves the site directory`);
+        if (!inside(await realpath(file)))
+            throw new Error(`Markdown path for ${document.id} leaves the site directory`);
+        const markdown = await readFile(file, 'utf8');
+        const { channel, version, language } = document.variant;
+        chunks.push(...chunkMarkdown(markdown, { ...document, variant: `${channel}|${version}|${language}` }, anchors.get(document.url), options));
+    }
+    const source = manifest.documents.map(document => `${document.id}:${document.digest}`).sort().join('\n');
+    const index = { schemaVersion: 1, sourceDigest: sha(source + (synonyms.length ? `\n${JSON.stringify(synonyms)}` : '')), chunks, ...(synonyms.length ? { synonyms } : {}) };
+    const output = path.join(await aiDirectory(root), 'chunks.json');
+    await writeFile(output, JSON.stringify(index) + '\n');
+    return { index, output, documents: manifest.documents.length };
+}

package/dist/ingredient-draft.d.ts

@@ -0,0 +1,17 @@
+export declare function renderIngredientDraft(configFile: string, specFile: string): Promise<{
+    content: string;
+    source: string;
+    sourceDigest: string;
+}>;
+export declare function writeIngredientDraft(configFile: string, specFile: string, outputFile: string, checkOnly?: boolean): Promise<{
+    content: string;
+    source: string;
+    sourceDigest: string;
+    status: string;
+    output: string;
+} | {
+    status: string;
+    output: string;
+    source: string;
+    sourceDigest: string;
+}>;