@shoalrun/cli 0.0.0-stage → 0.1.0-alpha.11

package/dist/ask-handler.js

@@ -0,0 +1,157 @@
+import { randomBytes } from 'node:crypto';
+import { validLogReceipt } from './ask-log-kv.js';
+// A `clientKey` that reads the caller's address from a header the host sets, such as `x-vercel-forwarded-for` on Vercel or
+// `cf-connecting-ip` on Cloudflare. Use only a header your host sets itself. On a host that passes the header through, a caller can
+// forge it and start a fresh personal cap, though the site-wide cap still holds. A missing header shares one key.
+export const clientKeyFromHeader = (name) => (request) => request.headers.get(name)?.split(',')[0].trim() || 'unknown';
+export const askLimits = { requestBytes: 16_384, question: 500, historyMessages: 10, historyMessage: 2000 };
+class AnswerTimeout extends Error {
+}
+const json = (status, body, headers = {}) => new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store', ...headers } });
+export function createAskHandler(options) {
+    const limit = options.maxRequestsPerMinute ?? 20;
+    const now = options.now ?? Date.now;
+    const seen = new Map();
+    return async (request) => {
+        if (request.method !== 'POST' && !(request.method === 'DELETE' && options.deleteRecord))
+            return json(405, { error: 'Use POST.' }, { Allow: options.deleteRecord ? 'POST, DELETE' : 'POST' });
+        const hosts = new Set(options.hosts.map(item => item.toLowerCase()));
+        const host = request.headers.get('host')?.toLowerCase();
+        const origin = request.headers.get('origin');
+        let originHost;
+        try {
+            originHost = origin ? new URL(origin).host.toLowerCase() : undefined;
+        }
+        catch {
+            return json(403, { error: 'This request is not allowed.' });
+        }
+        // A request from a browser always carries an Origin, and one from curl carries none. A cross-site page can send a text/plain
+        // POST without a preflight, so the JSON content type below is a second check, not the only one.
+        if (!host || !hosts.has(host) || (originHost !== undefined && !hosts.has(originHost)))
+            return json(403, { error: 'This request is not allowed.' });
+        if (!/^application\/json\b/i.test(request.headers.get('content-type') ?? ''))
+            return json(415, { error: 'Send application/json.' });
+        if (request.method === 'DELETE') {
+            const raw = await request.text();
+            if (Buffer.byteLength(raw) > 1024)
+                return json(413, { error: 'The request is too large.' });
+            let body;
+            try {
+                body = JSON.parse(raw);
+            }
+            catch {
+                return json(400, { error: 'The request body is not valid JSON.' });
+            }
+            if (!body || typeof body.receipt !== 'string' || !validLogReceipt(body.receipt))
+                return json(400, { error: 'A valid deletion receipt is required.' });
+            if (options.rateLimiter) {
+                const taken = await options.rateLimiter.take(options.clientKey?.(request) ?? 'shared');
+                if (!taken.allowed)
+                    return json(429, { error: 'Too many requests. Wait a moment and try again.' }, { 'Retry-After': String(taken.retryAfterSeconds) });
+            }
+            try {
+                return json(200, { deleted: await options.deleteRecord(body.receipt) });
+            }
+            catch {
+                return json(503, { error: 'The question could not be deleted right now. Try again later.' });
+            }
+        }
+        if (options.paused && await options.paused().catch(() => true))
+            return json(503, { error: 'The chat is paused right now. Please try again later.', paused: true }, { 'Retry-After': '300' });
+        const declared = Number(request.headers.get('content-length') ?? 0);
+        if (declared > askLimits.requestBytes)
+            return json(413, { error: 'The request is too large.' });
+        const raw = await request.text();
+        if (Buffer.byteLength(raw) > askLimits.requestBytes)
+            return json(413, { error: 'The request is too large.' });
+        let body;
+        try {
+            body = JSON.parse(raw);
+        }
+        catch {
+            return json(400, { error: 'The request body is not valid JSON.' });
+        }
+        if (!body || typeof body !== 'object' || Array.isArray(body))
+            return json(400, { error: 'The request body must be an object.' });
+        const question = typeof body.question === 'string' ? body.question.trim() : '';
+        if (!question)
+            return json(400, { error: 'Ask a question.' });
+        if (question.length > askLimits.question)
+            return json(400, { error: `Keep the question under ${askLimits.question} characters.` });
+        const history = [];
+        if (body.history !== undefined) {
+            if (!Array.isArray(body.history) || body.history.length > askLimits.historyMessages)
+                return json(400, { error: `History holds at most ${askLimits.historyMessages} messages.` });
+            for (const message of body.history) {
+                if (!message || (message.role !== 'user' && message.role !== 'assistant') || typeof message.content !== 'string')
+                    return json(400, { error: 'Each history message needs a user or assistant role and text.' });
+                // An earlier answer can be long. Only its start is kept as context, so a follow-up is never refused for the length of what came before.
+                history.push({ role: message.role, content: message.content.slice(0, askLimits.historyMessage) });
+            }
+        }
+        const key = options.clientKey?.(request) ?? 'shared';
+        let passHeaders = {};
+        const held = Boolean(options.session?.valid(request.headers.get('cookie'), key));
+        if (options.challenge && !held && (!options.challengeNeeded || await options.challengeNeeded(key).catch(() => true))) {
+            const passed = await options.challenge({ token: typeof body.turnstileToken === 'string' ? body.turnstileToken : undefined, ip: key, request }).catch(() => false);
+            if (passed && options.session)
+                passHeaders = { 'Set-Cookie': options.session.header(key, new URL(request.url).protocol === 'https:') };
+            if (!passed)
+                return json(403, { error: 'The check did not pass. Reload the page and try again.', challenge: true });
+        }
+        // From here the visitor has passed, so every reply carries their pass when they were just given one.
+        const reply = (status, body, headers = {}) => json(status, body, { ...passHeaders, ...headers });
+        const current = now();
+        if (options.rateLimiter) {
+            const taken = await options.rateLimiter.take(key);
+            if (!taken.allowed)
+                return reply(429, { error: 'Too many questions. Wait a moment and try again.' }, { 'Retry-After': String(taken.retryAfterSeconds) });
+        }
+        const recent = options.rateLimiter ? [] : (seen.get(key) ?? []).filter(time => current - time < 60_000);
+        if (!options.rateLimiter && recent.length >= limit) {
+            seen.set(key, recent);
+            return reply(429, { error: 'Too many questions. Wait a moment and try again.' }, { 'Retry-After': String(Math.max(1, Math.ceil((60_000 - (current - recent[0])) / 1000))) });
+        }
+        if (!options.rateLimiter) {
+            seen.set(key, [...recent, current]);
+            // Callers who stopped asking would otherwise stay in the map for the life of the process.
+            if (seen.size > 10_000)
+                for (const [caller, times] of seen)
+                    if (!times.some(time => current - time < 60_000))
+                        seen.delete(caller);
+        }
+        let timer;
+        const controller = new AbortController();
+        try {
+            // A slot is taken before the question is answered, and given back if no model call was needed. If answering fails, the slot stays
+            // taken, because the model may already have been paid.
+            const reservation = options.budget ? await options.budget.reserveWithReceipt(key) : undefined;
+            const answering = options.answer(question, history, { modelAllowed: !options.budget || Boolean(reservation), signal: controller.signal });
+            let result;
+            if (options.answerTimeoutMs) {
+                answering.catch(() => { });
+                result = await Promise.race([answering, new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(new AnswerTimeout()); reject(new AnswerTimeout()); }, options.answerTimeoutMs); })]);
+            }
+            else
+                result = await answering;
+            if (options.budget && reservation && result.usage.chatCalls === 0)
+                await options.budget.releaseReservation(reservation);
+            const recordedAt = new Date(now());
+            const receipt = options.record && options.response === 'public' ? `${recordedAt.toISOString().slice(0, 10)}.${randomBytes(16).toString('hex')}` : undefined;
+            if (options.record)
+                await Promise.resolve().then(() => options.record({ id: receipt, question, history, result, elapsedMs: now() - current, now: () => recordedAt })).catch(() => { });
+            if (options.response === 'public')
+                return reply(200, { answer: result.answer, decision: result.decision, sources: result.sources, uncited: result.uncited, unsupportedCode: result.unsupportedCode, unsupportedClaims: result.unsupportedClaims.map(claim => ({ text: claim.text })), ...(receipt ? { deletionReceipt: receipt } : {}) });
+            return reply(200, { answer: result.answer, decision: result.decision, mode: result.mode, sources: result.sources, notes: result.notes, jevBest: result.jevBest, standalone: result.standalone, uncited: result.uncited, unsupportedCode: result.unsupportedCode, verified: result.verified, unsupportedClaims: result.unsupportedClaims, routing: result.routing, cached: result.cached, usage: result.usage, timings: result.timings });
+        }
+        catch (error) {
+            if (error instanceof AnswerTimeout)
+                return reply(504, { error: 'The answer took too long. Try again in a moment.' });
+            return reply(502, { error: options.exposeErrors ? error.message : 'The assistant is unavailable right now. Try again shortly.' });
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    };
+}

package/dist/ask-log-kv.d.ts

@@ -0,0 +1,14 @@
+import { type AskEvent, type LogRecord } from './ask-log.js';
+import type { RestRedis } from './budget-store.js';
+export type KvLogOptions = {
+    retentionDays: number;
+    maxPerDay: number;
+    answers?: boolean;
+    now?: () => Date;
+    onError?: (error: Error) => void;
+};
+export declare const logKey: (day: string) => string;
+export declare function validLogReceipt(receipt: string): boolean;
+export declare function deleteKvLogRecord(redis: Pick<RestRedis, 'command'>, receipt: string): Promise<boolean>;
+export declare function kvLogSink(redis: Pick<RestRedis, 'pipeline'>, options: KvLogOptions): (event: AskEvent) => Promise<void>;
+export declare function readKvLog(redis: Pick<RestRedis, 'pipeline'>, days: number, now?: () => Date): Promise<LogRecord[]>;

package/dist/ask-log-kv.js

@@ -0,0 +1,69 @@
+import { toRecord } from './ask-log.js';
+export const logKey = (day) => `asklog:${day}`;
+const dayOf = (date) => date.toISOString().slice(0, 10);
+const receiptPattern = /^([0-9]{4}-[0-9]{2}-[0-9]{2})\.([a-f0-9]{32})$/;
+// A receipt is a bearer capability. It names one daily list and one random record, without retaining an address or identity.
+export function validLogReceipt(receipt) {
+    const match = receiptPattern.exec(receipt);
+    return Boolean(match && !Number.isNaN(Date.parse(`${match[1]}T00:00:00Z`)));
+}
+export async function deleteKvLogRecord(redis, receipt) {
+    if (!validLogReceipt(receipt))
+        throw new Error('Invalid question deletion receipt');
+    const day = receipt.slice(0, 10);
+    const lines = await redis.command(['LRANGE', logKey(day), '0', '-1']);
+    if (!Array.isArray(lines))
+        throw new Error('The question log could not be read');
+    const line = lines.find(raw => {
+        try {
+            return JSON.parse(String(raw)).id === receipt;
+        }
+        catch {
+            return false;
+        }
+    });
+    if (line === undefined)
+        return false;
+    return Number(await redis.command(['LREM', logKey(day), '1', String(line)])) === 1;
+}
+export function kvLogSink(redis, options) {
+    if (!Number.isSafeInteger(options.retentionDays) || options.retentionDays < 1)
+        throw new Error('retentionDays must be a whole number of 1 or more');
+    if (!Number.isSafeInteger(options.maxPerDay) || options.maxPerDay < 1)
+        throw new Error('maxPerDay must be a whole number of 1 or more');
+    const now = options.now ?? (() => new Date());
+    return async (event) => {
+        try {
+            const at = event.now?.() ?? now();
+            const record = toRecord({ ...event, now: () => at }, { answers: options.answers });
+            const key = logKey(dayOf(at));
+            // Newest first, trimmed to the cap, with the list's lifetime set to the retention period from this write.
+            await redis.pipeline([['LPUSH', key, JSON.stringify(record)], ['LTRIM', key, '0', String(options.maxPerDay - 1)], ['EXPIRE', key, String(options.retentionDays * 86_400)]]);
+        }
+        catch (error) {
+            options.onError?.(error);
+        }
+    };
+}
+// The records for the last `days` days, oldest first. A line that cannot be read is skipped.
+export async function readKvLog(redis, days, now = () => new Date()) {
+    if (!Number.isSafeInteger(days) || days < 1 || days > 366)
+        throw new Error('days must be a whole number from 1 to 366');
+    const today = now();
+    const keys = Array.from({ length: days }, (_, back) => logKey(dayOf(new Date(today.getTime() - back * 86_400_000)))).reverse();
+    const lists = await redis.pipeline(keys.map(key => ['LRANGE', key, '0', '-1']));
+    const records = [];
+    for (const list of lists) {
+        if (!Array.isArray(list))
+            continue;
+        for (const line of [...list].reverse()) {
+            try {
+                const record = JSON.parse(String(line));
+                if (record?.v === 1 && typeof record.question === 'string')
+                    records.push(record);
+            }
+            catch { /* skip */ }
+        }
+    }
+    return records;
+}

package/dist/ask-log.d.ts

@@ -0,0 +1,106 @@
+import { type AskResult, type Decision } from './ask.js';
+export type LogRecord = {
+    v: 1;
+    id?: string;
+    at: string;
+    question: string;
+    followUp?: boolean;
+    standalone?: string;
+    decision: Decision;
+    mode: string;
+    cached?: boolean;
+    card?: string;
+    jevBest?: number;
+    model?: string;
+    chatCalls: number;
+    tokens: {
+        in: number;
+        out: number;
+    };
+    sources: Array<{
+        id: string;
+        title: string;
+        url: string;
+    }>;
+    closest?: Array<{
+        title: string;
+        url: string;
+    }>;
+    flags: {
+        uncited?: boolean;
+        invalidCitations?: number;
+        unsupportedCode?: number;
+        unsupportedClaims?: number;
+    };
+    verified: boolean;
+    notes: string[];
+    answer: string;
+    ms: number;
+};
+export type AskEvent = {
+    id?: string;
+    question: string;
+    history: unknown[];
+    result: AskResult;
+    elapsedMs: number;
+    now?: () => Date;
+};
+export declare function redact(text: string): string;
+export declare function toRecord(event: AskEvent, options?: {
+    answers?: boolean;
+}): LogRecord;
+export declare function checkLogPath(file: string, siteDirectory: string): Promise<string>;
+export declare class AskLog {
+    readonly file: string;
+    private readonly options;
+    private queue;
+    constructor(file: string, options?: {
+        maxBytes?: number;
+        answers?: boolean;
+    });
+    write(event: AskEvent): Promise<void>;
+}
+export declare function readLog(file: string): Promise<LogRecord[]>;
+export type Gap = {
+    question: string;
+    count: number;
+    decisions: string[];
+    bestScore?: number;
+    offTopic: boolean;
+    closest: Array<{
+        title: string;
+        url: string;
+    }>;
+};
+export type CardCandidate = {
+    question: string;
+    count: number;
+    pages: string[];
+};
+export type Flagged = {
+    question: string;
+    count: number;
+    flags: string[];
+    model?: string;
+};
+export type LogReport = {
+    total: number;
+    from?: string;
+    to?: string;
+    byDecision: Record<string, number>;
+    modelCalls: number;
+    modelShare: number;
+    tokens: {
+        in: number;
+        out: number;
+    };
+    medianMs: number;
+    gaps: Gap[];
+    cards: CardCandidate[];
+    flagged: Flagged[];
+    paused: number;
+};
+export declare function analyzeLog(records: LogRecord[], options?: {
+    minRepeats?: number;
+}): LogReport;
+export declare function formatReport(report: LogReport, limit?: number): string;

package/dist/ask-log.js

@@ -0,0 +1,159 @@
+import { appendFile, mkdir, readFile, realpath, rename, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { normalizeQuestion } from './answer-cache.js';
+import { closestFloor } from './ask.js';
+// Best effort, not a guarantee: it removes the common ways a question carries a person's details or a credential. It leaves ordinary words alone.
+export function redact(text) {
+    return text
+        .replace(/https?:\/\/[^\s)>\]]+/gi, url => url.replace(/[?#].*$/, ''))
+        .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '[email]')
+        .replace(/\b(?:sk|pk|rk|pa|lsk|ghp|gho|xox[bpas])[-_][A-Za-z0-9_-]{12,}/g, '[secret]')
+        .replace(/\b[A-Za-z0-9_-]{32,}\b/g, '[secret]')
+        .replace(/\+?\d[\d\s().-]{7,}\d/g, match => match.replace(/\D/g, '').length >= 9 ? '[number]' : match);
+}
+const trim = (text, limit) => text.length > limit ? `${text.slice(0, limit)}…` : text;
+export function toRecord(event, options = {}) {
+    const { result } = event;
+    const closest = result.decision === 'declined' ? [...result.answer.matchAll(/^- (.+?): (\/\S*)$/gm)].map(match => ({ title: redact(match[1]), url: match[2] })) : undefined;
+    return {
+        v: 1, ...(event.id ? { id: event.id } : {}), at: (event.now ?? (() => new Date()))().toISOString(), question: redact(event.question),
+        ...(event.history.length ? { followUp: true } : {}), ...(result.standalone ? { standalone: redact(result.standalone) } : {}),
+        decision: result.decision, mode: result.mode, ...(result.cached ? { cached: true } : {}), ...(result.card ? { card: result.card.id } : {}),
+        ...(result.jevBest !== undefined ? { jevBest: Number(result.jevBest.toFixed(3)) } : {}),
+        ...(result.routing?.answeredBy ? { model: result.routing.answeredBy } : {}),
+        chatCalls: result.usage.chatCalls, tokens: { in: result.usage.promptTokens, out: result.usage.completionTokens },
+        sources: result.sources.map(source => ({ id: source.id, title: source.title, url: source.url })),
+        ...(closest?.length ? { closest } : {}),
+        flags: { ...(result.uncited ? { uncited: true } : {}), ...(result.invalidCitations.length ? { invalidCitations: result.invalidCitations.length } : {}), ...(result.unsupportedCode.length ? { unsupportedCode: result.unsupportedCode.length } : {}), ...(result.unsupportedClaims.length ? { unsupportedClaims: result.unsupportedClaims.length } : {}) },
+        verified: result.verified, notes: result.notes.map(note => trim(redact(note), 300)),
+        answer: options.answers === false ? '' : trim(redact(result.answer), 4000), ms: Math.round(event.elapsedMs)
+    };
+}
+// The log must not sit where a deploy could publish it. This resolves symlinks in the part of the path that exists.
+export async function checkLogPath(file, siteDirectory) {
+    const target = path.resolve(file);
+    let existing = path.dirname(target);
+    const rest = [path.basename(target)];
+    for (;;) {
+        try {
+            existing = await realpath(existing);
+            break;
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+            rest.unshift(path.basename(existing));
+            const parent = path.dirname(existing);
+            if (parent === existing)
+                break;
+            existing = parent;
+        }
+    }
+    const real = path.join(existing, ...rest);
+    const site = await realpath(path.resolve(siteDirectory));
+    if (real === site || real.startsWith(site + path.sep))
+        throw new Error('The log file must be outside the site directory, because a deploy could publish it');
+    return target;
+}
+export class AskLog {
+    file;
+    options;
+    queue = Promise.resolve();
+    constructor(file, options = {}) {
+        this.file = file;
+        this.options = options;
+    }
+    // Writes are queued so lines never interleave. A failure never reaches the reader who asked.
+    write(event) {
+        const line = `${JSON.stringify(toRecord(event, this.options))}\n`;
+        const next = this.queue.then(async () => {
+            await mkdir(path.dirname(this.file), { recursive: true, mode: 0o700 });
+            const size = await stat(this.file).then(info => info.size, () => 0);
+            if (size + line.length > (this.options.maxBytes ?? 5_000_000))
+                await rename(this.file, `${this.file}.1`);
+            await appendFile(this.file, line, { mode: 0o600 });
+        });
+        this.queue = next.catch(() => { });
+        return next.catch(() => { });
+    }
+}
+export async function readLog(file) {
+    const records = [];
+    for (const line of (await readFile(file, 'utf8')).split('\n')) {
+        if (!line.trim())
+            continue;
+        try {
+            const record = JSON.parse(line);
+            if (record?.v === 1 && typeof record.question === 'string')
+                records.push(record);
+        }
+        catch { /* a line cut off by a crash */ }
+    }
+    return records;
+}
+const median = (values) => { const sorted = [...values].sort((a, b) => a - b); return sorted.length ? sorted[Math.floor(sorted.length / 2)] : 0; };
+const operational = (record) => record.decision === 'search' && record.notes.some(note => /daily budget|scorer that checks/.test(note));
+// Turns the log into three lists. Gaps are questions the docs did not answer, most asked first. Card candidates are questions asked more than once
+// that the model answered cleanly and no card covers, which would be free to serve from a card. Flagged answers are worth reading for the eval set.
+// Questions are grouped by their normalized text, so a paraphrase counts as a separate question.
+export function analyzeLog(records, options = {}) {
+    const minRepeats = options.minRepeats ?? 2;
+    const groups = new Map();
+    for (const record of records) {
+        const key = normalizeQuestion(record.standalone ?? record.question);
+        groups.set(key, [...(groups.get(key) ?? []), record]);
+    }
+    const byDecision = {};
+    for (const record of records)
+        byDecision[record.decision] = (byDecision[record.decision] ?? 0) + 1;
+    const modelCalls = records.reduce((total, record) => total + record.chatCalls, 0);
+    const gaps = [];
+    const cards = [];
+    const flagged = [];
+    for (const [question, group] of groups) {
+        const missed = group.filter(record => (record.decision === 'declined' || record.decision === 'hedged'));
+        if (missed.length) {
+            const scores = missed.map(record => record.jevBest).filter((score) => score !== undefined);
+            const closest = new Map();
+            for (const record of missed)
+                for (const page of record.closest ?? record.sources)
+                    closest.set(page.url, { title: page.title, url: page.url });
+            gaps.push({ question, count: missed.length, decisions: [...new Set(missed.map(record => record.decision))], ...(scores.length ? { bestScore: Math.max(...scores) } : {}), offTopic: scores.length > 0 && Math.max(...scores) < closestFloor && missed.every(record => record.decision === 'declined'), closest: [...closest.values()].slice(0, 3) });
+        }
+        const covered = group.some(record => record.decision === 'card');
+        const clean = group.filter(record => record.decision === 'answered' && record.chatCalls > 0 && !Object.keys(record.flags).length && !record.followUp);
+        const demand = group.filter(record => !record.followUp && ['answered', 'excerpt', 'card'].includes(record.decision)).length;
+        if (!covered && clean.length && demand >= minRepeats)
+            cards.push({ question, count: demand, pages: [...new Set(clean.flatMap(record => record.sources.map(source => source.url.split('#')[0])))] });
+        const bad = group.filter(record => Object.keys(record.flags).length);
+        if (bad.length)
+            flagged.push({ question, count: bad.length, flags: [...new Set(bad.flatMap(record => Object.keys(record.flags)))], ...(bad[0].model ? { model: bad[0].model } : {}) });
+    }
+    const byCount = (items) => items.sort((a, b) => b.count - a.count);
+    return {
+        total: records.length, ...(records.length ? { from: records[0].at, to: records[records.length - 1].at } : {}), byDecision, modelCalls,
+        modelShare: records.length ? records.filter(record => record.chatCalls > 0).length / records.length : 0,
+        tokens: { in: records.reduce((total, record) => total + record.tokens.in, 0), out: records.reduce((total, record) => total + record.tokens.out, 0) },
+        medianMs: median(records.map(record => record.ms)), gaps: byCount(gaps), cards: byCount(cards), flagged: byCount(flagged), paused: records.filter(operational).length
+    };
+}
+export function formatReport(report, limit = 15) {
+    const lines = [];
+    const pct = (value) => `${Math.round(value * 100)}%`;
+    lines.push(`# Ask log: ${report.total} question${report.total === 1 ? '' : 's'}${report.from ? `, ${report.from.slice(0, 10)} to ${report.to?.slice(0, 10)}` : ''}`, '');
+    if (!report.total)
+        return `${lines.join('\n')}\nThe log is empty.\n`;
+    lines.push(`- How they were answered: ${Object.entries(report.byDecision).sort(([, a], [, b]) => b - a).map(([name, count]) => `${count} ${name}`).join(', ')}`);
+    lines.push(`- ${pct(report.modelShare)} of questions called the model (${report.modelCalls} calls, ${report.tokens.in} in and ${report.tokens.out} out tokens). Median time ${(report.medianMs / 1000).toFixed(1)}s.`);
+    if (report.paused)
+        lines.push(`- ${report.paused} question${report.paused === 1 ? '' : 's'} got only a list of pages, because the scorer was down or the daily budget was spent.`);
+    const section = (title, note, rows) => { lines.push('', `## ${title}`, note, ''); lines.push(...(rows.length ? rows : ['None.'])); };
+    const near = report.gaps.filter(gap => !gap.offTopic);
+    const off = report.gaps.filter(gap => gap.offTopic);
+    section('Gaps in the docs', 'Questions the docs came close to answering but did not, most asked first. Each is a page or a paragraph to write, or a question the docs should say they do not cover.', near.slice(0, limit).map(gap => `- ${gap.count}× "${gap.question}" (${gap.decisions.join(', ')}${gap.bestScore !== undefined ? `, best score ${gap.bestScore}` : ''})${gap.closest.length ? `. Closest: ${gap.closest.map(page => `${page.title} ${page.url}`).join('; ')}` : ''}`));
+    if (off.length)
+        lines.push('', `Off topic: ${off.reduce((total, gap) => total + gap.count, 0)} question${off.reduce((total, gap) => total + gap.count, 0) === 1 ? '' : 's'} that no page came close to answering, such as ${off.slice(0, 3).map(gap => `"${gap.question}"`).join(', ')}. They are probably not gaps.`);
+    section('Questions worth answering ahead of time', 'Asked more than once, answered cleanly by the model, and not served by a card. `--cards-out FILE` writes them in the format `shoalrun answers` reads.', report.cards.slice(0, limit).map(card => `- ${card.count}× "${card.question}"${card.pages.length ? ` (${card.pages.join(', ')})` : ''}`));
+    section('Flagged answers', 'Answers a check flagged. Read them, and add the good ones to the eval set.', report.flagged.slice(0, limit).map(item => `- ${item.count}× "${item.question}" (${item.flags.join(', ')}${item.model ? `, ${item.model}` : ''})`));
+    return `${lines.join('\n')}\n`;
+}

package/dist/ask-setup.d.ts

@@ -0,0 +1,43 @@
+import type { AskOptions } from './ask.js';
+import { type ChatProvider } from './chat.js';
+import { type ReaderSelection } from './reader-filters.js';
+import { DailyBudget, type BudgetStore } from './budget.js';
+export declare function parseReaderView(specs: string[]): ReaderSelection | undefined;
+export type AskSetupInput = {
+    sharedStore?: boolean;
+    siteDirectory: string;
+    configFile?: string;
+    settings?: Record<string, unknown>;
+    model?: string;
+    passages?: number;
+    channel?: string;
+    version?: string;
+    language?: string;
+    noRerank?: boolean;
+    noDirect?: boolean;
+    noEscalate?: boolean;
+    noCards?: boolean;
+    cache?: boolean;
+    direct?: {
+        threshold: number;
+        maxChars?: number;
+    };
+    jevCacheDir?: string;
+    readerView?: ReaderSelection;
+    maxTokens?: number;
+};
+export type AskSetup = {
+    budget?: DailyBudget;
+    budgetStore?: BudgetStore;
+    chat: ChatProvider;
+    options: Omit<AskOptions, 'question'>;
+    notes: string[];
+    embedTexts?: (texts: string[]) => Promise<Float32Array[]>;
+    embedding?: {
+        provider: string;
+        model: string;
+        space: string;
+        dimension: number;
+    };
+};
+export declare function prepareAsk(input: AskSetupInput): Promise<AskSetup>;

package/dist/ask-setup.js

@@ -0,0 +1,99 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { AnswerCache } from './answer-cache.js';
+import { CardIndex, readCards } from './answer-cards.js';
+import { chatProviderConfig, chatRoutes, validateChat } from './chat-config.js';
+import { createChatProvider, fallbackChat } from './chat.js';
+import { loadConfig } from './config.js';
+import { validateReaderFilters, validateReaderSelection } from './reader-filters.js';
+import { createProvider, readEmbeddings } from './embeddings.js';
+import { DailyBudget, memoryBudgetStore } from './budget.js';
+import { envBudgetStore } from './budget-store.js';
+import { jevAvailable, jevEndpoint } from './jev.js';
+import { queryProviderConfig, validateRetrieval } from './retrieval-config.js';
+import { Retriever } from './retrieval.js';
+// `--reader-view audience=authors,maintainers`, repeatable for more than one filter.
+export function parseReaderView(specs) {
+    if (!specs.length)
+        return undefined;
+    const selection = {};
+    for (const spec of specs) {
+        const match = /^([a-z][a-z0-9-]*)=(.+)$/.exec(spec);
+        if (!match)
+            throw new Error(`--reader-view needs facet=choice[,choice], such as audience=authors; got ${JSON.stringify(spec)}`);
+        if (selection[match[1]])
+            throw new Error(`--reader-view names ${match[1]} twice`);
+        selection[match[1]] = match[2].split(',').map(choice => choice.trim()).filter(Boolean);
+    }
+    return selection;
+}
+// Everything `shoalrun ask` and the answer eval share: read the project's chat and retrieval settings, load the passages and any
+// vectors, and wire up the optional layers. A layer that isn't configured is left out, and one that is configured but can't run
+// (Jev without a key) is left out with a note.
+export async function prepareAsk(input) {
+    if (!input.settings && !input.configFile)
+        throw new Error('prepareAsk needs a config file or settings');
+    const settings = input.settings ?? (await loadConfig(input.configFile)).value;
+    const readerFilters = validateReaderFilters(settings.readerFilters);
+    const chatConfig = validateChat(settings.chat, readerFilters);
+    const retrieval = validateRetrieval(settings.retrieval);
+    // --model replaces the main model only. Fallback and escalation models still inherit the config's settings.
+    const mainConfig = chatProviderConfig(chatConfig, { model: input.model });
+    const routes = chatRoutes(chatConfig, chatProviderConfig(chatConfig));
+    const chat = fallbackChat([createChatProvider(mainConfig), ...routes.fallbacks.map(createChatProvider)]);
+    let chunks;
+    try {
+        chunks = JSON.parse(await readFile(path.join(input.siteDirectory, 'ai', 'chunks.json'), 'utf8'));
+    }
+    catch (error) {
+        throw error.code === 'ENOENT' ? new Error(`No ai/chunks.json in ${input.siteDirectory}; run shoalrun index first`) : error;
+    }
+    const embeddings = await readEmbeddings(input.siteDirectory);
+    const retriever = new Retriever(chunks, embeddings);
+    const variants = retriever.variants();
+    const variant = input.channel && input.version && input.language ? `${input.channel}|${input.version}|${input.language}` : variants.length === 1 ? variants[0] : undefined;
+    if (!variant)
+        throw new Error(`The site has several publications (${variants.join(', ')}); pass --channel, --version, and --language`);
+    let embedQuery;
+    let embedTexts;
+    let embedding;
+    if (embeddings) {
+        const queryProvider = createProvider(queryProviderConfig(retrieval, embeddings.meta));
+        if (queryProvider.space !== embeddings.meta.space || (embeddings.meta.backend && queryProvider.backend !== embeddings.meta.backend))
+            throw new Error(`${queryProvider.model} is not in the embedding space or backend of ${embeddings.meta.model}; set retrieval.embeddings.queryModel and baseUrl to compatible values`);
+        embedQuery = async (text, signal) => (await queryProvider.embed([text], 'query', signal))[0];
+        embedTexts = texts => queryProvider.embed(texts, 'query');
+        embedding = { provider: queryProvider.name, model: queryProvider.model, space: queryProvider.space, dimension: queryProvider.dimension };
+    }
+    // A reader view on the command line replaces the config's, so one run can answer for a different audience.
+    if (input.readerView && !readerFilters)
+        throw new Error('--reader-view needs readerFilters.facets in the site config');
+    const readerSelection = input.readerView ? validateReaderSelection(input.readerView, readerFilters, '--reader-view') : chatConfig?.readerView;
+    const notes = [];
+    const rerank = Boolean(chatConfig?.rerank) && !input.noRerank;
+    const scoring = chatConfig?.rerank;
+    const scorer = { ...(scoring?.endpoint ? { endpoint: jevEndpoint(scoring.endpoint) } : {}), ...(scoring?.model ? { model: scoring.model } : {}), ...(scoring?.keyEnv ? { keyEnv: scoring.keyEnv } : {}), ...(scoring?.stateFormat ? { stateFormat: scoring.stateFormat } : {}), ...(input.jevCacheDir ? { cacheDir: input.jevCacheDir } : {}) };
+    const scoringReady = jevAvailable(scoring?.keyEnv);
+    // A scorer that is configured but has no key counts as down, not as absent: it is what keeps off-topic questions from the model.
+    if (rerank && !scoringReady)
+        notes.push(`The relevance scorer has no key: set ${scoring?.keyEnv ?? 'JEV_API_KEY'}. Until then it counts as down, so ${scoring?.onError === 'answer' ? 'the model answers without the gate and the claim check' : 'questions get the closest pages and no model call'}`);
+    // Answer cards need semantic search and Jev. A configured but missing card file is a note, not an error, so the chat still answers.
+    let cards;
+    if (chatConfig?.cards && !input.noCards && rerank && scoringReady) {
+        const file = await readCards(input.siteDirectory);
+        if (!file)
+            notes.push('chat.cards is set but there is no ai/answers.json; run shoalrun answers to make it');
+        else if (!embedding || file.embedding.space !== embedding.space || file.embedding.dimension !== embedding.dimension)
+            notes.push('The answer cards were made with a different embedding model, so they are not used; run shoalrun answers again');
+        else
+            cards = { index: new CardIndex(file), threshold: chatConfig.cards.threshold, confirm: chatConfig.cards.confirm, jev: { ...scorer, maxCalls: 1 } };
+    }
+    // The shared store is for a hosted function. A local run keeps its counters in memory, so it never needs the store's keys.
+    const store = input.sharedStore && chatConfig?.budget?.store ? envBudgetStore(chatConfig.budget.store) : memoryBudgetStore();
+    const reported = new Set();
+    const onBudgetError = (error) => { if (!reported.has(error.message)) {
+        reported.add(error.message);
+        process.stderr.write(`Budget: ${error.message}. Questions get the closest pages, with no model call, until it works.\n`);
+    } };
+    return { ...(chatConfig?.budget ? { budget: new DailyBudget(store, chatConfig.budget, Date.now, onBudgetError), budgetStore: store } : {}), chat, notes, embedTexts, embedding, options: { ...(scoring?.onError ? { onScorerError: scoring.onError } : {}), retriever, chat, variant, readerSelection, embedQuery, passages: input.passages ?? chatConfig?.passages, gate: chatConfig?.gate, ...(input.maxTokens ? { maxTokens: input.maxTokens } : {}), ...(rerank ? { jev: { ...scorer, top: 24, maxCalls: 2 } } : {}), ...(cards ? { cards } : {}), ...(input.cache && chatConfig?.cache ? { cache: new AnswerCache({ maxEntries: chatConfig.cache.maxEntries, ttlMs: chatConfig.cache.ttlMinutes * 60_000 }) } : {}), ...(routes.escalate && !input.noEscalate ? { escalate: { chat: createChatProvider(routes.escalate.config), when: routes.escalate.when } } : {}), ...(rerank && scoringReady && !input.noDirect && (input.direct || chatConfig?.direct) ? { direct: { threshold: input.direct?.threshold ?? chatConfig.direct.threshold, maxChars: input.direct?.maxChars ?? chatConfig?.direct?.maxChars ?? 1200, margin: chatConfig?.direct?.margin ?? 0.1, others: chatConfig?.direct?.others ?? 1, jev: { ...scorer, maxCalls: 2 } } } : {}), ...(rerank && scoringReady && chatConfig?.verify ? { verify: { ...chatConfig.verify, jev: { ...scorer, maxCalls: 3 } } } : {}) } };
+}