crewhaus 0.1.8 → 0.2.0

package/dist/scaffold-evals.d.ts

@@ -0,0 +1,138 @@
+/**
+ * Item 13 — `crewhaus scaffold-evals` + `init --with-evals`: day-one eval
+ * assets generated FROM the spec itself, so a fresh harness can run
+ * `crewhaus eval` before its first user rating ever lands.
+ *
+ * Two generation modes share one pipeline:
+ *
+ * - **Model mode** (credentials for the spec's model — or an explicit
+ *   `--model` — are present): ONE model call through the same model-router
+ *   path `optimize --mutator claude` uses (`resolveModel`) asks for N
+ *   realistic user prompts. The call itself lives in the CLI entry file
+ *   behind an injectable seam; this module owns only the pure halves — the
+ *   meta-prompt ({@link buildSampleGenerationPrompt}) and the tolerant
+ *   response parser ({@link parseModelSampleInputs}) — so the model path is
+ *   unit-testable without a provider.
+ * - **Template mode** (offline, and ALWAYS for `init --with-evals` — init
+ *   must never require credentials): {@link taskPhrasesFromInstructions}
+ *   parses the instruction sentences into task phrases and
+ *   {@link templateSampleInputs} wraps them in deterministic prompt
+ *   templates. Same inputs → same outputs, byte for byte.
+ *
+ * Graders reuse feedback.ts's synthesis machinery as a spec-driven
+ * front-end: online mode emits ONE `llm_judge` rubric with all five anchors
+ * pre-filled from the spec's stated goals; offline mode emits the same safe
+ * non-empty-answer floor grader `distill` falls back to
+ * (`synthesizeGraders([])`). Either way exactly one grader is emitted —
+ * stacking graders hard-ANDs their scores (the eval-grader `all(...)`
+ * min-collapse), exactly the gotcha distill's output already documents.
+ *
+ * Kept in a side-effect-free module (the CLI entry file runs an argv switch
+ * on import) mirroring `feedback.ts` / `datasets.ts`; all filesystem access
+ * and the actual model call live in `apps/cli/src/index.ts`.
+ */
+import type { Sample } from "@crewhaus/eval-dataset";
+import { type GradersConfigObject } from "./feedback";
+/** Thrown on a spec this scaffold cannot derive eval assets from (and on
+ *  invalid flag values). The CLI entry file routes it through `die()`;
+ *  tests assert on `.message` without the process exiting. */
+export declare class ScaffoldEvalsError extends Error {
+    readonly name = "ScaffoldEvalsError";
+}
+/** Default number of Sample stubs `scaffold-evals` generates. */
+export declare const DEFAULT_SCAFFOLD_SAMPLES = 8;
+/** Header for the scaffolded graders.yaml (threaded through feedback.ts's
+ *  `gradersConfigToYaml`, replacing its distill-specific default). */
+export declare const SCAFFOLD_GRADERS_HEADER: ReadonlyArray<string>;
+/** The spec fields the scaffold generates from. */
+export type ScaffoldInfo = {
+    readonly name: string;
+    readonly target: string;
+    /** agent.instructions (cli/channel/… agent-block shapes), or the joined
+     *  step instructions for workflow specs. */
+    readonly instructions: string;
+    /** The spec-level tools list, verbatim spec names (e.g. `webSearch`). */
+    readonly tools: ReadonlyArray<string>;
+    /** agent.model / workflow model — the model whose credentials decide
+     *  online vs offline mode when `--model` is not given. */
+    readonly model?: string;
+    /** True when the spec already declares a `feedback:` block. */
+    readonly hasFeedback: boolean;
+};
+/**
+ * Tolerantly extract the scaffold source from a spec. Supports every shape
+ * with an `agent.instructions` block (cli, channel, voice, browser, …) plus
+ * workflow specs (step instructions joined — each step is a stated goal).
+ * Throws ScaffoldEvalsError for shapes carrying no instructions at all;
+ * parse failures propagate as SpecParseError (a CrewhausError) so the CLI
+ * reports the real YAML problem.
+ */
+export declare function extractScaffoldInfo(yamlText: string): ScaffoldInfo;
+export declare function taskPhrasesFromInstructions(instructions: string): string[];
+/**
+ * Deterministic template mode: `n` sample-input prompts derived from the
+ * instructions' task phrases (cycling phrase × template), or from generic
+ * task-shaped stubs when no phrase survives the filter (e.g. the bare
+ * `crewhaus init` placeholder spec). Pure — same info + n always yields the
+ * identical list.
+ */
+export declare function templateSampleInputs(info: ScaffoldInfo, n: number): string[];
+/**
+ * The runtime tool names a sample input obviously implies, drawn ONLY from
+ * the spec's own tools list. Builtin spec names map to their runtime
+ * PascalCase form (`webSearch` → `WebSearch` — the casing tool graders
+ * compare against); unknown/custom tools (e.g. `mcp__*`) are implied only
+ * when the input mentions them verbatim and pass through unchanged.
+ */
+export declare function impliedTools(input: string, specTools: ReadonlyArray<string>): string[];
+export type ScaffoldGenerator = "template" | "model";
+/**
+ * Wrap generated inputs into SampleSchema-valid stubs: zero-padded stable
+ * ids, `expected_tools` where the spec's tools list is obviously implied,
+ * and metadata marking them as scaffold stubs to edit.
+ */
+export declare function buildScaffoldSamples(info: ScaffoldInfo, inputs: ReadonlyArray<string>, generator: ScaffoldGenerator): Sample[];
+/**
+ * The starter graders config — a spec-driven front-end over feedback.ts's
+ * synthesis machinery, single-grader in both modes (hard-AND-safe, exactly
+ * like distill emits):
+ *
+ * - `online: true` → one `llm_judge` rubric whose five anchors are
+ *   pre-filled against the spec's stated goals (the task phrases parsed
+ *   from its instructions). Deterministic text — no model call is needed to
+ *   WRITE the rubric; credentials are needed to RUN it, which is why this
+ *   mode keys on their presence.
+ * - `online: false` → `synthesizeGraders([])`'s safe non-empty-answer floor
+ *   grader, so a credential-free `crewhaus eval` still runs end to end.
+ */
+export declare function buildScaffoldGraders(info: ScaffoldInfo, opts: {
+    online: boolean;
+    model?: string;
+}): GradersConfigObject;
+/** System block for the one-shot sample-input generation call. */
+export declare const SCAFFOLD_GENERATION_SYSTEM = "You write starter eval datasets for AI agent harnesses. You will receive an agent's instructions and tool list. Produce realistic, distinct user prompts that exercise the agent's stated tasks \u2014 including at least one edge case (ambiguous, adversarial, or out-of-scope request).\n\nHard rules:\n- Output exactly one JSON object: {\"inputs\": [\"...\", \"...\"]} with the requested number of entries. No text outside the JSON.\n- Each entry is one user prompt, phrased as a real user would type it.\n- Do NOT include expected answers, numbering, or commentary.";
+/** The user message for the generation call — the spec's instructions
+ *  (clipped) + tools + the requested sample count. */
+export declare function buildSampleGenerationPrompt(info: ScaffoldInfo, n: number): string;
+/**
+ * Tolerant parse of the generation response: first balanced `{…}` substring
+ * (```json fences and leading prose tolerated), `inputs` must be an array of
+ * non-empty strings. Returns the trimmed, deduped prompts clipped to `max`;
+ * `[]` on ANY failure — the caller falls back to (or tops up from) the
+ * deterministic template.
+ */
+export declare function parseModelSampleInputs(raw: string, max: number): string[];
+/** Top up `primary` (model inputs) from `fallback` (template inputs) to `n`
+ *  entries, skipping duplicates — a short model response still yields a full
+ *  dataset. */
+export declare function mergeInputs(primary: ReadonlyArray<string>, fallback: ReadonlyArray<string>, n: number): string[];
+/**
+ * Never overwrite existing eval assets without `--force`. Returns the error
+ * message when blocked (the CLI routes it through `die`), undefined when the
+ * write may proceed. `exists` is injected so the guard is unit-testable.
+ */
+export declare function checkNoOverwrite(paths: ReadonlyArray<string>, exists: (path: string) => boolean, force: boolean): string | undefined;
+/** The `feedback:` block suggestion printed (never auto-applied) when the
+ *  spec lacks one — ratings are what upgrade these scaffolds into distilled
+ *  datasets later. */
+export declare function feedbackBlockSuggestion(): string;

package/dist/scaffold-evals.js

@@ -0,0 +1,410 @@
+import { parseSpec } from "@crewhaus/spec";
+import { synthesizeGraders } from "./feedback";
+/** Thrown on a spec this scaffold cannot derive eval assets from (and on
+ *  invalid flag values). The CLI entry file routes it through `die()`;
+ *  tests assert on `.message` without the process exiting. */
+export class ScaffoldEvalsError extends Error {
+    name = "ScaffoldEvalsError";
+}
+/** Default number of Sample stubs `scaffold-evals` generates. */
+export const DEFAULT_SCAFFOLD_SAMPLES = 8;
+/** Header for the scaffolded graders.yaml (threaded through feedback.ts's
+ *  `gradersConfigToYaml`, replacing its distill-specific default). */
+export const SCAFFOLD_GRADERS_HEADER = [
+    "# Scaffolded by `crewhaus scaffold-evals` from the spec's stated goals.",
+    "# Exactly one grader — stacking graders hard-ANDs their scores (see eval-grader `all`).",
+];
+/**
+ * Tolerantly extract the scaffold source from a spec. Supports every shape
+ * with an `agent.instructions` block (cli, channel, voice, browser, …) plus
+ * workflow specs (step instructions joined — each step is a stated goal).
+ * Throws ScaffoldEvalsError for shapes carrying no instructions at all;
+ * parse failures propagate as SpecParseError (a CrewhausError) so the CLI
+ * reports the real YAML problem.
+ */
+export function extractScaffoldInfo(yamlText) {
+    const spec = parseSpec(yamlText);
+    const hasFeedback = spec.feedback !== undefined;
+    if (typeof spec.agent?.instructions === "string") {
+        return {
+            name: spec.name,
+            target: spec.target,
+            instructions: spec.agent.instructions,
+            tools: spec.tools ?? [],
+            ...(spec.agent.model !== undefined ? { model: spec.agent.model } : {}),
+            hasFeedback,
+        };
+    }
+    if (Array.isArray(spec.steps) && spec.steps.length > 0) {
+        const tools = [...new Set(spec.steps.flatMap((s) => s.tools ?? []))];
+        return {
+            name: spec.name,
+            target: spec.target,
+            instructions: spec.steps.map((s) => s.instructions).join("\n"),
+            tools,
+            ...(spec.model !== undefined ? { model: spec.model } : {}),
+            hasFeedback,
+        };
+    }
+    throw new ScaffoldEvalsError(`target "${spec.target}" carries no agent.instructions — scaffold-evals supports agent-block shapes (cli, channel, …) and workflow steps`);
+}
+// -------- template mode: instructions → task phrases → prompts --------
+/** Sentences that describe the agent rather than a task it performs. The
+ *  `init` placeholder line is filtered too so `init --with-evals` on a fresh
+ *  scaffold falls through to the generic stubs instead of echoing it. */
+const PERSONA_PATTERNS = [
+    /^you are\b/i,
+    /^your name\b/i,
+    /^act as\b/i,
+    /replace these instructions/i,
+];
+/** Constraint sentences ("never …", "do not …") are guardrails, not tasks —
+ *  a deterministic template can't invert them into a useful probe. */
+const CONSTRAINT_PATTERNS = [/^never\b/i, /^do not\b/i, /^don'?t\b/i];
+const MODAL_PREFIX = /^you (?:must|should|will|can|typically|generally|always)\s+/i;
+/**
+ * Parse instruction sentences into task phrases — the deterministic half of
+ * sample-input generation. Persona/placeholder sentences and negative
+ * constraints are dropped; "You must/should/will/can X" and bare "You X"
+ * collapse to "X"; leading "Always" is stripped; short fragments (< 3 words)
+ * are discarded. Order follows the instructions; duplicates collapse.
+ */
+const BULLET_PREFIX = /^(?:[-*•]|\d+[.)])\s+/;
+/** Reflow instruction text into segments: YAML block scalars wrap sentences
+ *  across lines, so plain continuation lines rejoin the previous one, while
+ *  bullet markers and blank lines start a new segment. */
+function reflowSegments(instructions) {
+    const segments = [];
+    let current = "";
+    const flush = () => {
+        if (current.trim() !== "")
+            segments.push(current.trim());
+        current = "";
+    };
+    for (const line of instructions.split("\n")) {
+        const trimmed = line.trim();
+        if (trimmed === "") {
+            flush();
+            continue;
+        }
+        if (BULLET_PREFIX.test(trimmed)) {
+            flush();
+            current = trimmed.replace(BULLET_PREFIX, "");
+            continue;
+        }
+        current = current === "" ? trimmed : `${current} ${trimmed}`;
+    }
+    flush();
+    return segments;
+}
+export function taskPhrasesFromInstructions(instructions) {
+    const phrases = [];
+    const seen = new Set();
+    for (const segment of reflowSegments(instructions)) {
+        for (const raw of segment.split(/(?<=[.!?])\s+/)) {
+            let s = raw.replace(/\s+/g, " ").trim();
+            if (s === "")
+                continue;
+            if (PERSONA_PATTERNS.some((p) => p.test(s)))
+                continue;
+            if (CONSTRAINT_PATTERNS.some((p) => p.test(s)))
+                continue;
+            s = s.replace(/^always\s+/i, "");
+            s = s.replace(MODAL_PREFIX, "");
+            s = s.replace(/^you\s+/i, "");
+            s = s.replace(/[.!?]+$/, "").trim();
+            if (s.split(" ").length < 3 || s.length < 12)
+                continue;
+            const key = s.toLowerCase();
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            phrases.push(s);
+        }
+    }
+    return phrases;
+}
+const capitalize = (s) => (s === "" ? s : s[0]?.toUpperCase() + s.slice(1));
+const uncapitalize = (s) => (s === "" ? s : s[0]?.toLowerCase() + s.slice(1));
+const TASK_TEMPLATES = [
+    (t) => `${capitalize(t)}.`,
+    (t) => `Please ${uncapitalize(t)}.`,
+    (t) => `Can you ${uncapitalize(t)}?`,
+    (t) => `Here is a request for you: ${uncapitalize(t)}. Walk through it step by step.`,
+];
+const GENERIC_TEMPLATES = [
+    (n) => `Handle a typical ${n} request end to end, stating any assumptions you make.`,
+    (n) => `A user brings you the most common task ${n} exists for. Complete it and explain your steps.`,
+    () => "The request is ambiguous. Ask one clarifying question, then answer for the most likely interpretation.",
+    (n) => `Summarize what you can do as ${n}, then demonstrate the most useful capability on a small example.`,
+];
+/**
+ * Deterministic template mode: `n` sample-input prompts derived from the
+ * instructions' task phrases (cycling phrase × template), or from generic
+ * task-shaped stubs when no phrase survives the filter (e.g. the bare
+ * `crewhaus init` placeholder spec). Pure — same info + n always yields the
+ * identical list.
+ */
+export function templateSampleInputs(info, n) {
+    const phrases = taskPhrasesFromInstructions(info.instructions);
+    const inputs = [];
+    if (phrases.length > 0) {
+        for (let i = 0; i < n; i += 1) {
+            const phrase = phrases[i % phrases.length];
+            const tier = Math.floor(i / phrases.length);
+            const template = TASK_TEMPLATES[tier % TASK_TEMPLATES.length];
+            const variation = tier >= TASK_TEMPLATES.length ? ` (variation ${tier})` : "";
+            inputs.push(`${template(phrase)}${variation}`);
+        }
+        return inputs;
+    }
+    for (let i = 0; i < n; i += 1) {
+        const template = GENERIC_TEMPLATES[i % GENERIC_TEMPLATES.length];
+        const tier = Math.floor(i / GENERIC_TEMPLATES.length);
+        const variation = tier >= 1 ? ` (variation ${tier + 1})` : "";
+        inputs.push(`${template(info.name)}${variation}`);
+    }
+    return inputs;
+}
+// -------- tool implication --------
+/**
+ * Spec tool name → the RUNTIME (PascalCase) tool name recorded in trace
+ * events and expected by tool graders, plus the input keywords that
+ * "obviously imply" the tool. Mirrors `loadToolMap` in index.ts /
+ * target-cli's BUILTIN_TOOL_MAP — keep the name column in sync. Keyword
+ * sets are deliberately conservative: a missed implication is a stub the
+ * user fills in; a false one is a failing grader they must debug.
+ */
+const TOOL_IMPLICATIONS = {
+    read: { runtime: "Read", keywords: ["read the file", "open the file", "file contents"] },
+    write: { runtime: "Write", keywords: ["write a file", "create a file", "save to a file"] },
+    edit: { runtime: "Edit", keywords: ["edit", "modify the file", "refactor"] },
+    glob: { runtime: "Glob", keywords: ["find files", "list files", "matching files"] },
+    grep: {
+        runtime: "Grep",
+        keywords: ["grep", "search the code", "search the codebase", "find references"],
+    },
+    bash: {
+        runtime: "Bash",
+        keywords: ["run the", "execute", "shell", "command", "install", "the tests"],
+    },
+    todoWrite: { runtime: "TodoWrite", keywords: ["todo", "task list", "plan out"] },
+    webFetch: { runtime: "WebFetch", keywords: ["fetch", "url", "web page", "website"] },
+    webSearch: {
+        runtime: "WebSearch",
+        keywords: ["search the web", "search online", "look up online", "web search", "latest news"],
+    },
+    readImage: { runtime: "ReadImage", keywords: ["image", "screenshot", "photo"] },
+    fetch: { runtime: "Fetch", keywords: ["fetch", "http", "api", "endpoint", "url"] },
+    imageGenerate: {
+        runtime: "ImageGenerate",
+        keywords: ["generate an image", "draw", "illustration"],
+    },
+    ingestDocument: { runtime: "IngestDocument", keywords: ["pdf", "document", "docx"] },
+};
+function keywordImplied(input, keyword) {
+    // Word-boundary match so "read" can't fire inside "already". Keywords are
+    // hand-authored lowercase alphanumeric phrases, so no escaping is needed.
+    return new RegExp(`\\b${keyword}\\b`, "i").test(input);
+}
+/**
+ * The runtime tool names a sample input obviously implies, drawn ONLY from
+ * the spec's own tools list. Builtin spec names map to their runtime
+ * PascalCase form (`webSearch` → `WebSearch` — the casing tool graders
+ * compare against); unknown/custom tools (e.g. `mcp__*`) are implied only
+ * when the input mentions them verbatim and pass through unchanged.
+ */
+export function impliedTools(input, specTools) {
+    const out = [];
+    for (const tool of specTools) {
+        const known = TOOL_IMPLICATIONS[tool];
+        if (known !== undefined) {
+            if (known.keywords.some((k) => keywordImplied(input, k)))
+                out.push(known.runtime);
+        }
+        else if (input.toLowerCase().includes(tool.toLowerCase())) {
+            out.push(tool);
+        }
+    }
+    return [...new Set(out)];
+}
+/**
+ * Wrap generated inputs into SampleSchema-valid stubs: zero-padded stable
+ * ids, `expected_tools` where the spec's tools list is obviously implied,
+ * and metadata marking them as scaffold stubs to edit.
+ */
+export function buildScaffoldSamples(info, inputs, generator) {
+    const width = Math.max(2, String(inputs.length).length);
+    return inputs.map((input, i) => {
+        const tools = impliedTools(input, info.tools);
+        return {
+            id: `scaffold_${String(i + 1).padStart(width, "0")}`,
+            input,
+            ...(tools.length > 0 ? { expected_tools: tools } : {}),
+            metadata: {
+                source: "scaffold-evals",
+                generator,
+                note: "starter stub — edit into a real task (and add expected_output for gold answers)",
+            },
+        };
+    });
+}
+// -------- graders --------
+/** Clip + quote-strip a goals summary for a rubric description (mirrors
+ *  feedback.ts's summarizeComments bounds). */
+function summarizeGoals(goals) {
+    const cleaned = goals.map((g) => g.trim().replace(/'/g, "")).filter((g) => g !== "");
+    if (cleaned.length === 0)
+        return undefined;
+    const joined = cleaned.slice(0, 4).join("; ");
+    return joined.length > 400 ? `${joined.slice(0, 400)}…` : joined;
+}
+/**
+ * The starter graders config — a spec-driven front-end over feedback.ts's
+ * synthesis machinery, single-grader in both modes (hard-AND-safe, exactly
+ * like distill emits):
+ *
+ * - `online: true` → one `llm_judge` rubric whose five anchors are
+ *   pre-filled against the spec's stated goals (the task phrases parsed
+ *   from its instructions). Deterministic text — no model call is needed to
+ *   WRITE the rubric; credentials are needed to RUN it, which is why this
+ *   mode keys on their presence.
+ * - `online: false` → `synthesizeGraders([])`'s safe non-empty-answer floor
+ *   grader, so a credential-free `crewhaus eval` still runs end to end.
+ */
+export function buildScaffoldGraders(info, opts) {
+    if (!opts.online) {
+        // The floor path of feedback.ts's synthesis (no positive turns yet); the
+        // scaffold prints its own offline note, so the distill-flavoured warning
+        // is discarded.
+        return synthesizeGraders([], []);
+    }
+    const goals = summarizeGoals(taskPhrasesFromInstructions(info.instructions));
+    const goalsNote = goals !== undefined
+        ? `: '${goals}'`
+        : ` of the "${info.name}" harness (see its crewhaus.yaml instructions)`;
+    return {
+        graders: [
+            {
+                name: "spec_goal_alignment",
+                type: "llm_judge",
+                rubric: {
+                    criteria: [
+                        {
+                            name: "spec_goal_alignment",
+                            description: `Judge how well the response fulfils the spec's stated goals${goalsNote}. Score strictly against the anchors.`,
+                            anchors: {
+                                "1": "Off-task or contradicts the stated goals; ignores the harness's purpose.",
+                                "2": "Barely on-task; most of the stated goals are unmet.",
+                                "3": "Partially fulfils the stated goals; noticeable gaps or constraint violations.",
+                                "4": "Fulfils the stated goals with only minor gaps.",
+                                "5": "Fully fulfils the stated goals and respects the instructions' constraints.",
+                            },
+                        },
+                    ],
+                    passing_score: 3,
+                },
+                ...(opts.model !== undefined ? { model: opts.model } : {}),
+            },
+        ],
+    };
+}
+// -------- model mode (pure halves; the call lives in index.ts) --------
+/** System block for the one-shot sample-input generation call. */
+export const SCAFFOLD_GENERATION_SYSTEM = `You write starter eval datasets for AI agent harnesses. You will receive an agent's instructions and tool list. Produce realistic, distinct user prompts that exercise the agent's stated tasks — including at least one edge case (ambiguous, adversarial, or out-of-scope request).
+
+Hard rules:
+- Output exactly one JSON object: {"inputs": ["...", "..."]} with the requested number of entries. No text outside the JSON.
+- Each entry is one user prompt, phrased as a real user would type it.
+- Do NOT include expected answers, numbering, or commentary.`;
+const MAX_INSTRUCTIONS_IN_PROMPT = 6000;
+/** The user message for the generation call — the spec's instructions
+ *  (clipped) + tools + the requested sample count. */
+export function buildSampleGenerationPrompt(info, n) {
+    const instructions = info.instructions.length > MAX_INSTRUCTIONS_IN_PROMPT
+        ? `${info.instructions.slice(0, MAX_INSTRUCTIONS_IN_PROMPT)}…`
+        : info.instructions;
+    const tools = info.tools.length > 0 ? info.tools.join(", ") : "(none declared)";
+    return [
+        `HARNESS: ${info.name} (target: ${info.target})`,
+        "",
+        "AGENT INSTRUCTIONS:",
+        instructions,
+        "",
+        `TOOLS: ${tools}`,
+        "",
+        `Produce exactly ${n} eval input prompts as {"inputs": [...]}.`,
+    ].join("\n");
+}
+/**
+ * Tolerant parse of the generation response: first balanced `{…}` substring
+ * (```json fences and leading prose tolerated), `inputs` must be an array of
+ * non-empty strings. Returns the trimmed, deduped prompts clipped to `max`;
+ * `[]` on ANY failure — the caller falls back to (or tops up from) the
+ * deterministic template.
+ */
+export function parseModelSampleInputs(raw, max) {
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (jsonMatch === null)
+        return [];
+    let parsed;
+    try {
+        parsed = JSON.parse(jsonMatch[0]);
+    }
+    catch {
+        return [];
+    }
+    const inputs = parsed.inputs;
+    if (!Array.isArray(inputs))
+        return [];
+    const out = [];
+    const seen = new Set();
+    for (const entry of inputs) {
+        if (typeof entry !== "string")
+            continue;
+        const trimmed = entry.trim();
+        if (trimmed === "" || seen.has(trimmed))
+            continue;
+        seen.add(trimmed);
+        out.push(trimmed);
+        if (out.length >= max)
+            break;
+    }
+    return out;
+}
+/** Top up `primary` (model inputs) from `fallback` (template inputs) to `n`
+ *  entries, skipping duplicates — a short model response still yields a full
+ *  dataset. */
+export function mergeInputs(primary, fallback, n) {
+    const out = primary.slice(0, n);
+    const seen = new Set(out);
+    for (const f of fallback) {
+        if (out.length >= n)
+            break;
+        if (seen.has(f))
+            continue;
+        seen.add(f);
+        out.push(f);
+    }
+    return out;
+}
+// -------- overwrite guard + suggestions --------
+/**
+ * Never overwrite existing eval assets without `--force`. Returns the error
+ * message when blocked (the CLI routes it through `die`), undefined when the
+ * write may proceed. `exists` is injected so the guard is unit-testable.
+ */
+export function checkNoOverwrite(paths, exists, force) {
+    if (force)
+        return undefined;
+    const existing = paths.filter(exists);
+    if (existing.length === 0)
+        return undefined;
+    return `refusing to overwrite existing eval asset(s): ${existing.join(", ")} — re-run with --force to replace them`;
+}
+/** The `feedback:` block suggestion printed (never auto-applied) when the
+ *  spec lacks one — ratings are what upgrade these scaffolds into distilled
+ *  datasets later. */
+export function feedbackBlockSuggestion() {
+    return ["feedback:", "  modality: binary", "  autoDistill: true"].join("\n");
+}