crewhaus 0.1.6 → 0.1.8

package/dist/feedback.d.ts

@@ -0,0 +1,239 @@
+/**
+ * Response-feedback core — the pure, side-effect-free half of the
+ * `crewhaus rate` / `crewhaus feedback` / `crewhaus distill` subcommands.
+ *
+ * A `FeedbackRecord` is one human rating placed on one assistant turn (a
+ * thumbs up/down, a 1–5 star vote, an arbitrary scale vote, and/or a
+ * free-text comment or correction). The capture surfaces write it durably as
+ * a `user_feedback` event-log line (payload = FeedbackRecord); the web-UI host
+ * writes bare records to `.crewhaus/feedback/*.jsonl`. `distill` reads both,
+ * pairs each rating to a conversation turn, and emits the two artifacts the
+ * R-eval stack already consumes: a `Sample[]` dataset (`@crewhaus/eval-dataset`)
+ * and a `graders.yaml` (`@crewhaus/eval-grader`).
+ *
+ * Everything here is pure so it is unit-testable; all filesystem access lives
+ * in `apps/cli/src/index.ts` (mirrors `doctor-checks.ts` / `scope-audit.ts`).
+ *
+ * turnNumber convention: the 1-based ordinal of USER-TEXT turns in a session.
+ * A `user_message` whose content is a string (or an array containing a `text`
+ * block and no `tool_result` block) starts a new turn; a `user_message` that
+ * carries only `tool_result` blocks is a tool-result echo, and one marked
+ * `synthetic: true` (a runtime-injected loop/continue/tombstone nudge) is NOT a
+ * turn. This is the same count the CLI `--turn` flag, the web-UI (prompts
+ * sent), and the runtime's runContext.turnNumber all compute, so a rating keys
+ * back to the exact exchange it rated even after a recovery fires mid-session.
+ */
+/** The event-log EventKind under which a FeedbackRecord is persisted. */
+export declare const FEEDBACK_EVENT_KIND: "user_feedback";
+export declare const FEEDBACK_SCHEMA_VERSION: 1;
+export type FeedbackModality = "binary" | "stars" | "scale" | "comment";
+export type FeedbackSource = "user" | "ui" | "channel" | "cli";
+export type FeedbackRating = {
+    thumbs?: "up" | "down";
+    stars?: number;
+    scale?: {
+        value: number;
+        min: number;
+        max: number;
+    };
+};
+export type FeedbackRecord = {
+    schemaVersion: 1;
+    id: string;
+    sessionId: string;
+    runId?: string;
+    /** 1-based ordinal of the rated user-text turn. */
+    turnNumber: number;
+    /** spanId of the rated model_response, when the capture surface knows it. */
+    targetSpanId?: string;
+    modality: FeedbackModality;
+    rating: FeedbackRating;
+    comment?: string;
+    /** A user-supplied better answer — becomes `expected_output` at distill. */
+    correction?: string;
+    source: FeedbackSource;
+    rater?: string;
+    /** ISO 8601 timestamp. */
+    ts: string;
+};
+/** Max stored length for a free-text comment/correction. Untrusted input (a
+ *  hand-edited or web-UI-supplied `.crewhaus/feedback` line) flows into
+ *  expected_output/metadata and the optimizer meta-prompt, so it is bounded at
+ *  ingestion and stripped of control chars (newline/tab kept). */
+export declare const MAX_FEEDBACK_TEXT = 8192;
+export declare function clipFeedbackText(s: string): string;
+/** Narrow an arbitrary parsed JSON value to a FeedbackRecord. Tolerant of the
+ *  optional fields; rejects anything missing the required core OR carrying a
+ *  malformed rating (so a corrupt/crafted line can't coerce a NaN/spurious
+ *  score downstream). */
+export declare function isFeedbackRecord(value: unknown): value is FeedbackRecord;
+export type BuildFeedbackInput = {
+    id: string;
+    sessionId: string;
+    turnNumber: number;
+    ts: string;
+    source: FeedbackSource;
+    runId?: string;
+    targetSpanId?: string;
+    rater?: string;
+    thumbs?: "up" | "down";
+    stars?: number;
+    /** A [0,1] scalar score (stored as scale {min:0,max:1}). */
+    score?: number;
+    comment?: string;
+    correction?: string;
+};
+/**
+ * Assemble + validate a FeedbackRecord from capture-surface inputs. The
+ * modality is inferred from which rating fields are present (thumbs → binary,
+ * stars → stars, score → scale, else → comment). Throws a plain Error (the CLI
+ * routes it through `die`) when no rating signal at all is supplied.
+ */
+export declare function buildFeedbackRecord(input: BuildFeedbackInput): FeedbackRecord;
+/**
+ * Normalize a rating to [0,1] — the same convention `GradeResult.score` uses
+ * ((n-1)/4 for a 1–5 judge score). Returns `undefined` for a comment-only
+ * record (no numeric signal). Thumbs: up→1, down→0. Stars n→(n-1)/4. Scale
+ * value→(value-min)/(max-min), clamped.
+ */
+export declare function normalizeRating(r: FeedbackRecord): number | undefined;
+/** A parsed event-log line: `{ ts, version, kind, payload }`. */
+export type LoggedEvent = {
+    kind?: string;
+    payload?: unknown;
+};
+export type DerivedTurn = {
+    turnNumber: number;
+    /** The user's text prompt for this turn. */
+    input: string;
+    /** The assistant's final text answer for this turn (what was rated). */
+    output: string;
+    /** Tool names the assistant called this turn, verbatim (PascalCase), unique. */
+    toolNames: string[];
+};
+/** A DerivedTurn tagged with the session it came from — the join key for
+ *  `distill`, so ratings across many sessions never collide on turnNumber. */
+export type SessionTurn = DerivedTurn & {
+    sessionId: string;
+};
+/**
+ * Reconstruct conversation turns from a session transcript. Each user-text
+ * message opens a turn; the assistant's messages until the next user-text
+ * message form its response. `output` is the LAST assistant text (the answer);
+ * `toolNames` are the unique tool names the assistant called, in first-seen
+ * order.
+ */
+export declare function deriveTurns(events: ReadonlyArray<LoggedEvent>): DerivedTurn[];
+/** Extract FeedbackRecords from parsed JSON objects. Accepts both event-log
+ *  envelopes (`{ kind: "user_feedback", payload }`) and bare records (the
+ *  web-UI host's `.crewhaus/feedback/*.jsonl`). */
+export declare function extractFeedbackRecords(objects: ReadonlyArray<unknown>): FeedbackRecord[];
+/**
+ * Collapse records to one per (sessionId, turnNumber). Rather than pure
+ * last-write-wins — which would let a later comment-only `feedback` erase an
+ * earlier `rate` on the same turn — fields are merged chronologically: the
+ * newest value of each field wins, but a later comment-only record keeps the
+ * earlier record's rating. So `rate --thumbs up` then `feedback --text "…"`
+ * yields one record carrying BOTH.
+ */
+export declare function mergeFeedback(records: ReadonlyArray<FeedbackRecord>): FeedbackRecord[];
+type Sample = {
+    id: string;
+    input: string;
+    expected_output?: string;
+    expected_tools?: string[];
+    metadata?: Record<string, unknown>;
+};
+export type RubricAnchors = {
+    "1": string;
+    "2": string;
+    "3": string;
+    "4": string;
+    "5": string;
+};
+export type GraderSpecObject = {
+    name: string;
+    type: "tool_call_sequence";
+    expected: string[];
+    mode: "set" | "subseq" | "exact";
+} | {
+    name: string;
+    type: "contains";
+    substring: string;
+    case_insensitive?: boolean;
+} | {
+    name: string;
+    type: "regex";
+    pattern: string;
+} | {
+    name: string;
+    type: "llm_judge";
+    rubric: {
+        criteria: Array<{
+            name: string;
+            description: string;
+            anchors: RubricAnchors;
+        }>;
+        passing_score?: number;
+    };
+    model?: string;
+};
+export type GradersConfigObject = {
+    graders: GraderSpecObject[];
+};
+export type DistillOptions = {
+    /** Normalized score at/above which a turn is a positive (gold) sample. */
+    minScore: number;
+    /** Emit a single `llm_judge` grader seeded from comment themes instead of the
+     *  deterministic grader. Opt-in (each eval sample then costs a judge call). */
+    judge?: boolean;
+    /** Judge model baked into the emitted llm_judge grader (else the runner's
+     *  `--judge-model` applies at eval time). */
+    judgeModel?: string;
+};
+export type DistillStats = {
+    totalFeedback: number;
+    matchedTurns: number;
+    unmatchedFeedback: number;
+    positives: number;
+    negatives: number;
+};
+export type DistillResult = {
+    samples: Sample[];
+    graders: GradersConfigObject;
+    stats: DistillStats;
+    /** Non-fatal warnings (e.g. a rating whose turn is missing from the log). */
+    warnings: string[];
+};
+/**
+ * Tag-all distillation. Every rated turn becomes a Sample; positively-rated
+ * turns (normalized ≥ minScore) — and any turn with a `correction` — become
+ * GOLD (expected_output set), the rest carry a low `metadata.user_rating` and
+ * no expected_output so they feed the optimizer's failure channel. A single
+ * deterministic grader is synthesized from the positive turns' behavior
+ * (exactly one, to avoid the hard-AND collapse in `all(...)`).
+ */
+export declare function distill(turns: ReadonlyArray<SessionTurn>, feedback: ReadonlyArray<FeedbackRecord>, opts: DistillOptions): DistillResult;
+/**
+ * Pick the single deterministic grader that best captures the up-rated
+ * behavior. Prefers a `tool_call_sequence` over the tools every tool-using
+ * positive turn shared (falling back to the most common tool); if the positive
+ * turns used no tools, a `contains` over a distinctive token common to their
+ * answers; failing that, a non-empty-answer floor (with a warning).
+ */
+export declare function synthesizeGraders(positiveTurns: ReadonlyArray<DerivedTurn>, warnings?: string[]): GradersConfigObject;
+/**
+ * Build a single `llm_judge` grader seeded from the aggregated comment themes.
+ * One criterion ("user_preference") with fixed 1–5 anchors and a description
+ * folding a short, quoted, clipped summary of what users praised vs criticized.
+ * Emitted as the SOLE grader so it is never hard-ANDed with a deterministic one
+ * (the `all(...)` min-collapse). The untrusted comment text is bounded and
+ * quoted-as-data; the rubric is trusted context sent verbatim to the judge.
+ */
+export declare function buildJudgeRubricGrader(positiveComments: ReadonlyArray<string>, negativeComments: ReadonlyArray<string>, model?: string): GraderSpecObject;
+/** One SampleSchema-valid JSON object per line (round-trips through loadJsonl). */
+export declare function samplesToJsonl(samples: ReadonlyArray<Sample>): string;
+/** Emit a GradersConfigSchema-valid graders.yaml. Scalars are JSON-encoded so
+ *  arbitrary substrings/tool names stay YAML-safe (JSON is a YAML subset). */
+export declare function gradersConfigToYaml(config: GradersConfigObject): string;
+export {};