crewhaus 0.1.8 → 0.2.0

package/dist/graders-suggest.d.ts

@@ -0,0 +1,186 @@
+/**
+ * Item 4 — `crewhaus graders suggest`: draft grader suites from the failure
+ * rationale that already accumulates on disk, instead of hand-authoring
+ * gotcha-prone graders.yaml files from scratch.
+ *
+ * Evidence sources (all read-side, none mutated):
+ * - Per-sample grader rationale from recent eval run dirs (the grades.json
+ *   the runner persists next to results.json), located via the item-3
+ *   run-history index for the cwd spec.
+ * - Judge criterionScores where present in grades.json (GradeResult carries
+ *   no structured output today, so this is a defensive parse that lights up
+ *   as soon as a judge grader persists them).
+ * - User feedback comments (`extractFeedbackRecords` rows) — down-rated
+ *   comments join the failure evidence; up-rated turns join the pass
+ *   exemplars.
+ *
+ * Pipeline: failure texts cluster into themes DETERMINISTICALLY (normalized
+ * token overlap — no model call needed to cluster); each theme then gets a
+ * deterministic draft grader (tool_call_sequence / json_path / regex /
+ * contains, in that priority order) derived from the observed up-rated
+ * outputs; with `--model`/credentials one complete llm_judge rubric (all
+ * five anchors) is additionally drafted from real good/bad exemplars — the
+ * model call itself lives in the CLI entry file, this module owns the pure
+ * prompt builder + response parser.
+ *
+ * The output is a REVIEW file (never auto-applied) whose header documents
+ * the hard-AND collapse: `crewhaus eval` combines stacked graders with
+ * eval-grader's `all(...)` (min-score, every-grader-must-pass), so the file
+ * tells the user to adopt ONE grader rather than the whole stack.
+ *
+ * Kept in a side-effect-free module (the CLI entry file runs an argv switch
+ * on import) mirroring `feedback.ts` / `triage.ts`.
+ */
+import type { LoadedRun } from "@crewhaus/eval-report";
+import { type FeedbackRecord, type GraderSpecObject, type GradersConfigObject, type SessionTurn } from "./feedback";
+/** Thrown on malformed flags / unusable evidence. The CLI entry file routes
+ *  it through `die()`; tests assert on `.message`. */
+export declare class GradersSuggestError extends Error {
+    readonly name = "GradersSuggestError";
+}
+/** How many recent runs (for the cwd spec) feed the suggestion by default. */
+export declare const DEFAULT_SUGGEST_RUNS = 10;
+/** Default output path for the review file. */
+export declare const DEFAULT_SUGGESTED_GRADERS_FILE = "graders-suggested.yaml";
+/** One-liner printed by `crewhaus distill` when it falls back to the floor
+ *  grader (see {@link isFloorGraderConfig}). */
+export declare const FLOOR_GRADER_HINT = "hint: draft sharper graders from recent eval failures and ratings with `crewhaus graders suggest`";
+/** True when a synthesized config is exactly the non-empty-answer floor
+ *  grader `synthesizeGraders` emits when the ratings carry no signal — the
+ *  trigger for the {@link FLOOR_GRADER_HINT} suggestion hook. */
+export declare function isFloorGraderConfig(config: GradersConfigObject): boolean;
+export type RunsSelector = {
+    kind: "dir";
+    dir: string;
+} | {
+    kind: "last";
+    n: number;
+};
+/** Parse `--runs`: `last:N` / `last N` selects the N most recent indexed
+ *  runs; anything else is a run directory path. */
+export declare function parseRunsFlag(value: string): RunsSelector;
+/** One failing grader verdict (or down-rated comment), joined to its sample. */
+export type FailureEvidence = {
+    readonly sampleId: string;
+    /** runId, or "feedback" for rating comments. */
+    readonly runId: string;
+    /** Grader name, `<grader>:criterion`, or "user_feedback". */
+    readonly source: string;
+    /** Cleaned rationale/comment text — the clustering signal. */
+    readonly text: string;
+    /** The failing output, when known (discriminative-token drafting). */
+    readonly output?: string;
+};
+/** One up-rated output — a passing eval sample or a thumbs-up turn. */
+export type PassExemplar = {
+    readonly sampleId: string;
+    readonly runId: string;
+    readonly output: string;
+    readonly toolNames: ReadonlyArray<string>;
+};
+export type SuggestEvidence = {
+    readonly failures: ReadonlyArray<FailureEvidence>;
+    readonly passes: ReadonlyArray<PassExemplar>;
+};
+/** Mirror of eval-runner's per-sample dir sanitizer — the key LoadedRun's
+ *  perSample map uses. Keep in sync with run-sample.ts. */
+export declare function sanitizeSampleId(id: string): string;
+/** Unique tool names (first-seen order) from a per-sample events.jsonl —
+ *  `tool_call_end` trace events carry the runtime tool name. Tolerant of
+ *  torn/corrupt lines. */
+export declare function toolNamesFromEventsJsonl(text: string): string[];
+/**
+ * Strip runner/judge boilerplate from a grader rationale so clustering sees
+ * the substance: the `[name: ✗]` combinator markers, the `judge=N (need
+ * ≥M):` prefix. Infra noise ("grader threw: …", "agent invocation error:
+ * …") returns "" — an outage is not a failure theme.
+ */
+export declare function cleanRationale(rationale: string): string;
+/** Defensive parse of a raw grades.json for judge criterionScores (not part
+ *  of GradeResult today — see the module doc). Criteria scored ≤ 2 of 5
+ *  become failure evidence in their own right. */
+export declare function criterionEvidence(gradesText: string, sampleId: string, runId: string): FailureEvidence[];
+/**
+ * Extract failure evidence + pass exemplars from one loaded eval run.
+ * Samples are visited in sampleId order (determinism), errored/grader-threw
+ * samples are skipped wholesale (infra noise, not a failure theme — the
+ * same posture as triage's pin guards).
+ */
+export declare function evidenceFromRun(run: LoadedRun): SuggestEvidence;
+/**
+ * Fold user ratings into the evidence: down-rated comments become failure
+ * evidence, up-rated turns become pass exemplars (their outputs + observed
+ * tool names). Positive comments are returned separately — they season the
+ * llm_judge rubric prompt, not the clustering.
+ */
+export declare function evidenceFromFeedback(turns: ReadonlyArray<SessionTurn>, records: ReadonlyArray<FeedbackRecord>, minScore: number): SuggestEvidence & {
+    positiveComments: string[];
+};
+/** Normalized token list for one evidence text: lowercase, alphanumeric,
+ *  length ≥ 3, non-stopword, unique, in first-seen order. */
+export declare function normalizeEvidenceTokens(text: string): string[];
+export type FailureTheme = {
+    /** Top tokens joined — the human handle for the theme. */
+    readonly label: string;
+    readonly tokens: ReadonlyArray<string>;
+    readonly items: ReadonlyArray<FailureEvidence>;
+    readonly sampleIds: ReadonlyArray<string>;
+    readonly runIds: ReadonlyArray<string>;
+    readonly sources: ReadonlyArray<string>;
+};
+/**
+ * Deterministic greedy clustering by normalized token overlap — no model
+ * call. Items are visited in a stable sort order (runId, sampleId, source,
+ * text) and joined to the best existing cluster when the Jaccard overlap
+ * with its SEED item's tokens reaches `threshold` (seed comparison keeps a
+ * cluster from drifting as it grows); otherwise they seed a new cluster.
+ * Themes come back largest-first, labelled by their most frequent tokens.
+ */
+export declare function clusterFailures(items: ReadonlyArray<FailureEvidence>, opts?: {
+    threshold?: number;
+}): FailureTheme[];
+export type SuggestedGrader = {
+    readonly spec: GraderSpecObject;
+    /** Comment lines (without the leading `# `) citing the evidence. */
+    readonly evidence: ReadonlyArray<string>;
+};
+export type DraftResult = {
+    readonly suggestions: ReadonlyArray<SuggestedGrader>;
+    /** Themes no deterministic grader could be drafted for. */
+    readonly undrafted: ReadonlyArray<FailureTheme>;
+};
+/**
+ * Draft one deterministic grader per failure theme from the observed
+ * up-rated outputs, in priority order: tool_call_sequence (theme mentions
+ * tools + passes share tools) → json_path (all pass outputs are JSON
+ * objects with a common key) → regex (shared output prefix) → contains
+ * (discriminative pass token). Themes with no deterministic signal come
+ * back in `undrafted` so the review file can note them.
+ */
+export declare function draftGradersForThemes(themes: ReadonlyArray<FailureTheme>, passes: ReadonlyArray<PassExemplar>): DraftResult;
+/** System block for the one-shot rubric drafting call. */
+export declare const RUBRIC_SUGGESTION_SYSTEM = "You design eval rubrics for AI agent harnesses. You will receive real GOOD exemplars (outputs users or graders rated up), real BAD evidence (grader rationales / user complaints), and the clustered failure themes. Write one llm_judge rubric criterion that separates the good from the bad.\n\nHard rules:\n- Output exactly one JSON object: {\"name\": \"...\", \"description\": \"...\", \"anchors\": {\"1\": \"...\", \"2\": \"...\", \"3\": \"...\", \"4\": \"...\", \"5\": \"...\"}, \"passing_score\": 3}. No text outside the JSON.\n- \"name\" is a short snake_case identifier.\n- The five anchors must be concrete, observable behaviors grounded in the exemplars \u2014 1 is the worst observed failure mode, 5 is the best observed behavior.\n- Do not quote any exemplar verbatim for more than a short phrase.";
+/** The user message for the rubric call: clipped good/bad exemplars + the
+ *  deterministic theme labels. */
+export declare function buildRubricSuggestionPrompt(goodOutputs: ReadonlyArray<string>, badEvidence: ReadonlyArray<FailureEvidence>, themes: ReadonlyArray<FailureTheme>): string;
+/**
+ * Tolerant parse of the rubric response into a complete single-criterion
+ * `llm_judge` GraderSpecObject (all five anchors required). Returns
+ * undefined on ANY shape failure — the caller keeps the deterministic
+ * suggestions and notes the skipped rubric.
+ */
+export declare function parseRubricSuggestion(raw: string, model?: string): GraderSpecObject | undefined;
+export type RenderSuggestionsOptions = {
+    readonly specName?: string;
+    readonly runsSeen: number;
+    readonly failureCount: number;
+    readonly feedbackCount: number;
+    readonly undraftedLabels: ReadonlyArray<string>;
+};
+/**
+ * Render the review file: a hard-AND-aware header, then a valid graders.yaml
+ * (parseable by `parseGradersConfig`, so a reviewed grader can be copied —
+ * or the file trimmed to one grader and used directly), with each grader
+ * preceded by a `# evidence:` comment citing what it was drafted from.
+ */
+export declare function renderSuggestedGradersYaml(suggestions: ReadonlyArray<SuggestedGrader>, opts: RenderSuggestionsOptions): string;