crewhaus 0.1.7 → 0.2.0

package/dist/faq.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Item #55 — distill repeated user questions into an auto-discovered FAQ skill.
+ *
+ * `crewhaus faq distill` clusters recurring `user_message` questions across
+ * sessions (deterministic token clustering — the same greedy-jaccard approach
+ * `graders suggest` uses, no model call), pairs each cluster with its best-rated
+ * answer (the up-rated turn's output), and emits a SKILL.md FAQ skill into
+ * `.crewhaus/skills/faq/` so `@crewhaus/skills-registry` auto-discovers it —
+ * its name + description go into the prompt and the body loads on demand when
+ * the model invokes the `Skill` tool (the registry is lazy, not eager).
+ *
+ * Everything here is pure + deterministic so it is unit-testable; the FS access
+ * and PII/secret redaction wiring live in `apps/cli/src/index.ts`. Both the
+ * harvested ANSWER and the representative QUESTION are user/model text that may
+ * carry pasted credentials, so the caller redacts BOTH (via
+ * `SYNTHESIZE_PII_DETECTORS`) before they land in the skill body (#55 F4).
+ *
+ * The emitted SKILL.md is guaranteed valid per skills-registry's `parseSkillFile`
+ * (leading `---` frontmatter with `name` + `description`, then the FAQ body).
+ */
+import type { FeedbackRecord, LoggedEvent, SessionTurn } from "./feedback";
+/** One recurring-question cluster paired with its best answer. */
+export type FaqEntry = {
+    /** The representative (most-frequent-token, then shortest) question. */
+    readonly question: string;
+    /** The best-rated answer for the cluster (already redacted by the caller). */
+    readonly answer: string;
+    /** How many distinct sessions asked a question in this cluster. */
+    readonly sessionCount: number;
+    /** How many questions total fell into this cluster. */
+    readonly occurrences: number;
+};
+export type ClusterOptions = {
+    /** Jaccard overlap at/above which a question joins a cluster's seed. Default 0.5. */
+    readonly threshold?: number;
+    /** Minimum number of DISTINCT sessions that must ask a clustered question
+     *  for it to become an FAQ entry. Default 2 (recurring across sessions — a
+     *  one-off, or a single session that repeats itself, is not an FAQ). */
+    readonly minOccurrences?: number;
+    /** Normalized rating at/above which an answer is "good". Default 0.7. */
+    readonly minScore?: number;
+    /** Redactor for answers (async). Identity by default. */
+    readonly redact?: (text: string) => Promise<string>;
+};
+/**
+ * Build the FAQ entries from per-session turns + feedback. Deterministic:
+ * questions are clustered greedily by normalized-token Jaccard overlap in a
+ * stable visit order; each cluster keeps its best-rated answer (highest score,
+ * ties broken by session/turn order). Only clusters asked in ≥ `minOccurrences`
+ * DISTINCT sessions (#55 F9) AND with a qualifying answer (score ≥ `minScore`)
+ * become entries. Entries come back most-recurring-first.
+ */
+export declare function distillFaq(turnsBySession: ReadonlyArray<SessionTurn>, feedback: ReadonlyArray<FeedbackRecord>, opts?: ClusterOptions): Promise<FaqEntry[]>;
+/** Derive `SessionTurn`s from many sessions' event logs. Convenience wrapper so
+ *  the CLI and tests build the clustering input the same way. */
+export declare function turnsFromSessions(perSession: ReadonlyArray<{
+    sessionId: string;
+    events: ReadonlyArray<LoggedEvent>;
+}>): SessionTurn[];
+/**
+ * Render a valid SKILL.md for the FAQ. Frontmatter carries `name` +
+ * `description` (the required pair skills-registry enforces) plus `triggers`;
+ * the body is the Q→A list. Guaranteed to parse via `parseSkillFile`.
+ */
+export declare function buildFaqSkill(entries: ReadonlyArray<FaqEntry>, opts?: {
+    name?: string;
+    harnessName?: string;
+}): string;

package/dist/faq.js

@@ -0,0 +1,168 @@
+/**
+ * Item #55 — distill repeated user questions into an auto-discovered FAQ skill.
+ *
+ * `crewhaus faq distill` clusters recurring `user_message` questions across
+ * sessions (deterministic token clustering — the same greedy-jaccard approach
+ * `graders suggest` uses, no model call), pairs each cluster with its best-rated
+ * answer (the up-rated turn's output), and emits a SKILL.md FAQ skill into
+ * `.crewhaus/skills/faq/` so `@crewhaus/skills-registry` auto-discovers it —
+ * its name + description go into the prompt and the body loads on demand when
+ * the model invokes the `Skill` tool (the registry is lazy, not eager).
+ *
+ * Everything here is pure + deterministic so it is unit-testable; the FS access
+ * and PII/secret redaction wiring live in `apps/cli/src/index.ts`. Both the
+ * harvested ANSWER and the representative QUESTION are user/model text that may
+ * carry pasted credentials, so the caller redacts BOTH (via
+ * `SYNTHESIZE_PII_DETECTORS`) before they land in the skill body (#55 F4).
+ *
+ * The emitted SKILL.md is guaranteed valid per skills-registry's `parseSkillFile`
+ * (leading `---` frontmatter with `name` + `description`, then the FAQ body).
+ */
+import { deriveTurns, mergeFeedback, normalizeRating } from "./feedback";
+import { normalizeEvidenceTokens } from "./graders-suggest";
+function jaccard(a, b) {
+    let intersection = 0;
+    for (const t of a)
+        if (b.has(t))
+            intersection += 1;
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+/**
+ * Build the FAQ entries from per-session turns + feedback. Deterministic:
+ * questions are clustered greedily by normalized-token Jaccard overlap in a
+ * stable visit order; each cluster keeps its best-rated answer (highest score,
+ * ties broken by session/turn order). Only clusters asked in ≥ `minOccurrences`
+ * DISTINCT sessions (#55 F9) AND with a qualifying answer (score ≥ `minScore`)
+ * become entries. Entries come back most-recurring-first.
+ */
+export async function distillFaq(turnsBySession, feedback, opts = {}) {
+    const threshold = opts.threshold ?? 0.5;
+    const minOccurrences = opts.minOccurrences ?? 2;
+    const minScore = opts.minScore ?? 0.7;
+    const redact = opts.redact ?? (async (t) => t);
+    // Rating lookup keyed by session#turn.
+    const scoreByKey = new Map();
+    for (const fb of mergeFeedback(feedback)) {
+        const s = normalizeRating(fb);
+        if (s !== undefined)
+            scoreByKey.set(`${fb.sessionId}#${fb.turnNumber}`, s);
+    }
+    const records = turnsBySession
+        .filter((t) => t.input.trim() !== "")
+        .map((t) => ({
+        sessionId: t.sessionId,
+        turnNumber: t.turnNumber,
+        question: t.input.trim(),
+        answer: t.output,
+        score: scoreByKey.get(`${t.sessionId}#${t.turnNumber}`),
+    }))
+        .sort((a, b) => a.sessionId.localeCompare(b.sessionId) ||
+        a.turnNumber - b.turnNumber ||
+        a.question.localeCompare(b.question));
+    const clusters = [];
+    for (const rec of records) {
+        const tokens = new Set(normalizeEvidenceTokens(rec.question));
+        if (tokens.size === 0)
+            continue;
+        let bestIdx = -1;
+        let bestOverlap = 0;
+        for (let i = 0; i < clusters.length; i += 1) {
+            const overlap = jaccard(tokens, clusters[i].seed);
+            if (overlap > bestOverlap) {
+                bestOverlap = overlap;
+                bestIdx = i;
+            }
+        }
+        if (bestIdx >= 0 && bestOverlap >= threshold) {
+            const c = clusters[bestIdx];
+            c.items.push(rec);
+            c.tokenSets.push(tokens);
+        }
+        else {
+            clusters.push({ seed: tokens, items: [rec], tokenSets: [tokens] });
+        }
+    }
+    const entries = [];
+    for (const c of clusters) {
+        // #55 F9 — "recurring" means asked across DISTINCT sessions, not the same
+        // session twice. Gate on the distinct-session count so two questions from
+        // one session don't manufacture an FAQ entry at minOccurrences:2.
+        const sessionCount = new Set(c.items.map((i) => i.sessionId)).size;
+        if (sessionCount < minOccurrences)
+            continue;
+        // Best answer: highest-scored qualifying turn (stable tie-break by order).
+        const qualifying = c.items
+            .filter((i) => i.score !== undefined && i.score >= minScore && i.answer.trim() !== "")
+            .sort((a, b) => (b.score ?? 0) - (a.score ?? 0) ||
+            a.sessionId.localeCompare(b.sessionId) ||
+            a.turnNumber - b.turnNumber);
+        const best = qualifying[0];
+        if (best === undefined)
+            continue;
+        // Representative question: the shortest question in the cluster (most
+        // canonical phrasing), stable tie-break. #55 F4 — the representative
+        // question is a raw user turn rendered as a Markdown heading in SKILL.md,
+        // so it is redacted with the SAME redactor as the answer (a question can
+        // paste a secret/email just as an answer can).
+        const rawQuestion = [...c.items]
+            .map((i) => i.question)
+            .sort((a, b) => a.length - b.length || a.localeCompare(b))[0];
+        const question = (await redact(rawQuestion)).trim();
+        const answer = (await redact(best.answer)).trim();
+        if (question === "" || answer === "")
+            continue;
+        entries.push({
+            question,
+            answer,
+            sessionCount,
+            occurrences: c.items.length,
+        });
+    }
+    return entries.sort((a, b) => b.occurrences - a.occurrences || a.question.localeCompare(b.question));
+}
+/** Derive `SessionTurn`s from many sessions' event logs. Convenience wrapper so
+ *  the CLI and tests build the clustering input the same way. */
+export function turnsFromSessions(perSession) {
+    const turns = [];
+    for (const { sessionId, events } of perSession) {
+        for (const t of deriveTurns(events))
+            turns.push({ ...t, sessionId });
+    }
+    return turns;
+}
+/** YAML-escape a scalar for the SKILL.md frontmatter (JSON is a YAML subset). */
+function yamlScalar(s) {
+    return JSON.stringify(s);
+}
+/**
+ * Render a valid SKILL.md for the FAQ. Frontmatter carries `name` +
+ * `description` (the required pair skills-registry enforces) plus `triggers`;
+ * the body is the Q→A list. Guaranteed to parse via `parseSkillFile`.
+ */
+export function buildFaqSkill(entries, opts = {}) {
+    const name = opts.name ?? "faq";
+    const label = opts.harnessName ?? "this harness";
+    const description = `Frequently asked questions for ${label}, distilled from recurring user questions and their best-rated answers. Consult before answering common questions.`;
+    const triggers = ["faq", "frequently asked", "common question", "how do i"];
+    const bodyLines = [
+        `# FAQ — ${label}`,
+        "",
+        "These are recurring user questions paired with the answers users rated best. Prefer these answers for matching questions.",
+        "",
+    ];
+    entries.forEach((e, i) => {
+        bodyLines.push(`## Q${i + 1}: ${e.question}`);
+        bodyLines.push("");
+        bodyLines.push(e.answer);
+        bodyLines.push("");
+    });
+    const frontmatter = [
+        "---",
+        `name: ${yamlScalar(name)}`,
+        `description: ${yamlScalar(description)}`,
+        `triggers: ${JSON.stringify(triggers)}`,
+        "---",
+    ].join("\n");
+    return `${frontmatter}\n${bodyLines.join("\n")}`;
+}

package/dist/feedback.d.ts

@@ -0,0 +1,246 @@
+/**
+ * Response-feedback core — the pure, side-effect-free half of the
+ * `crewhaus rate` / `crewhaus feedback` / `crewhaus distill` subcommands.
+ *
+ * A `FeedbackRecord` is one human rating placed on one assistant turn (a
+ * thumbs up/down, a 1–5 star vote, an arbitrary scale vote, and/or a
+ * free-text comment or correction). The capture surfaces write it durably as
+ * a `user_feedback` event-log line (payload = FeedbackRecord); the web-UI host
+ * writes bare records to `.crewhaus/feedback/*.jsonl`. `distill` reads both,
+ * pairs each rating to a conversation turn, and emits the two artifacts the
+ * R-eval stack already consumes: a `Sample[]` dataset (`@crewhaus/eval-dataset`)
+ * and a `graders.yaml` (`@crewhaus/eval-grader`).
+ *
+ * Everything here is pure so it is unit-testable; all filesystem access lives
+ * in `apps/cli/src/index.ts` (mirrors `doctor-checks.ts` / `scope-audit.ts`).
+ *
+ * turnNumber convention: the 1-based ordinal of USER-TEXT turns in a session.
+ * A `user_message` whose content is a string (or an array containing a `text`
+ * block and no `tool_result` block) starts a new turn; a `user_message` that
+ * carries only `tool_result` blocks is a tool-result echo, and one marked
+ * `synthetic: true` (a runtime-injected loop/continue/tombstone nudge) is NOT a
+ * turn. This is the same count the CLI `--turn` flag, the web-UI (prompts
+ * sent), and the runtime's runContext.turnNumber all compute, so a rating keys
+ * back to the exact exchange it rated even after a recovery fires mid-session.
+ */
+/** The event-log EventKind under which a FeedbackRecord is persisted. */
+export declare const FEEDBACK_EVENT_KIND: "user_feedback";
+export declare const FEEDBACK_SCHEMA_VERSION: 1;
+export type FeedbackModality = "binary" | "stars" | "scale" | "comment";
+export type FeedbackSource = "user" | "ui" | "channel" | "cli";
+export type FeedbackRating = {
+    thumbs?: "up" | "down";
+    stars?: number;
+    scale?: {
+        value: number;
+        min: number;
+        max: number;
+    };
+};
+export type FeedbackRecord = {
+    schemaVersion: 1;
+    id: string;
+    sessionId: string;
+    runId?: string;
+    /** 1-based ordinal of the rated user-text turn. */
+    turnNumber: number;
+    /** spanId of the rated model_response, when the capture surface knows it. */
+    targetSpanId?: string;
+    modality: FeedbackModality;
+    rating: FeedbackRating;
+    comment?: string;
+    /** A user-supplied better answer — becomes `expected_output` at distill. */
+    correction?: string;
+    source: FeedbackSource;
+    rater?: string;
+    /** ISO 8601 timestamp. */
+    ts: string;
+};
+/** Max stored length for a free-text comment/correction. Untrusted input (a
+ *  hand-edited or web-UI-supplied `.crewhaus/feedback` line) flows into
+ *  expected_output/metadata and the optimizer meta-prompt, so it is bounded at
+ *  ingestion and stripped of control chars (newline/tab kept). */
+export declare const MAX_FEEDBACK_TEXT = 8192;
+export declare function clipFeedbackText(s: string): string;
+/** Narrow an arbitrary parsed JSON value to a FeedbackRecord. Tolerant of the
+ *  optional fields; rejects anything missing the required core OR carrying a
+ *  malformed rating (so a corrupt/crafted line can't coerce a NaN/spurious
+ *  score downstream). */
+export declare function isFeedbackRecord(value: unknown): value is FeedbackRecord;
+export type BuildFeedbackInput = {
+    id: string;
+    sessionId: string;
+    turnNumber: number;
+    ts: string;
+    source: FeedbackSource;
+    runId?: string;
+    targetSpanId?: string;
+    rater?: string;
+    thumbs?: "up" | "down";
+    stars?: number;
+    /** A [0,1] scalar score (stored as scale {min:0,max:1}). */
+    score?: number;
+    comment?: string;
+    correction?: string;
+};
+/**
+ * Assemble + validate a FeedbackRecord from capture-surface inputs. The
+ * modality is inferred from which rating fields are present (thumbs → binary,
+ * stars → stars, score → scale, else → comment). Throws a plain Error (the CLI
+ * routes it through `die`) when no rating signal at all is supplied.
+ */
+export declare function buildFeedbackRecord(input: BuildFeedbackInput): FeedbackRecord;
+/**
+ * Normalize a rating to [0,1] — the same convention `GradeResult.score` uses
+ * ((n-1)/4 for a 1–5 judge score). Returns `undefined` for a comment-only
+ * record (no numeric signal). Thumbs: up→1, down→0. Stars n→(n-1)/4. Scale
+ * value→(value-min)/(max-min), clamped.
+ */
+export declare function normalizeRating(r: FeedbackRecord): number | undefined;
+/** A parsed event-log line: `{ ts, version, kind, payload }`. */
+export type LoggedEvent = {
+    kind?: string;
+    payload?: unknown;
+};
+export type DerivedTurn = {
+    turnNumber: number;
+    /** The user's text prompt for this turn. */
+    input: string;
+    /** The assistant's final text answer for this turn (what was rated). */
+    output: string;
+    /** Tool names the assistant called this turn, verbatim (PascalCase), unique. */
+    toolNames: string[];
+};
+/** A DerivedTurn tagged with the session it came from — the join key for
+ *  `distill`, so ratings across many sessions never collide on turnNumber. */
+export type SessionTurn = DerivedTurn & {
+    sessionId: string;
+};
+/**
+ * Reconstruct conversation turns from a session transcript. Each user-text
+ * message opens a turn; the assistant's messages until the next user-text
+ * message form its response. `output` is the LAST assistant text (the answer);
+ * `toolNames` are the unique tool names the assistant called, in first-seen
+ * order.
+ */
+export declare function deriveTurns(events: ReadonlyArray<LoggedEvent>): DerivedTurn[];
+/** Extract FeedbackRecords from parsed JSON objects. Accepts both event-log
+ *  envelopes (`{ kind: "user_feedback", payload }`) and bare records (the
+ *  web-UI host's `.crewhaus/feedback/*.jsonl`). */
+export declare function extractFeedbackRecords(objects: ReadonlyArray<unknown>): FeedbackRecord[];
+/**
+ * Collapse records to one per (sessionId, turnNumber). Rather than pure
+ * last-write-wins — which would let a later comment-only `feedback` erase an
+ * earlier `rate` on the same turn — fields are merged chronologically: the
+ * newest value of each field wins, but a later comment-only record keeps the
+ * earlier record's rating. So `rate --thumbs up` then `feedback --text "…"`
+ * yields one record carrying BOTH.
+ */
+export declare function mergeFeedback(records: ReadonlyArray<FeedbackRecord>): FeedbackRecord[];
+type Sample = {
+    id: string;
+    input: string;
+    expected_output?: string;
+    expected_tools?: string[];
+    metadata?: Record<string, unknown>;
+};
+export type RubricAnchors = {
+    "1": string;
+    "2": string;
+    "3": string;
+    "4": string;
+    "5": string;
+};
+export type GraderSpecObject = {
+    name: string;
+    type: "tool_call_sequence";
+    expected: string[];
+    mode: "set" | "subseq" | "exact";
+} | {
+    name: string;
+    type: "contains";
+    substring: string;
+    case_insensitive?: boolean;
+} | {
+    name: string;
+    type: "regex";
+    pattern: string;
+} | {
+    name: string;
+    type: "json_path";
+    path: string;
+    expected?: unknown;
+} | {
+    name: string;
+    type: "llm_judge";
+    rubric: {
+        criteria: Array<{
+            name: string;
+            description: string;
+            anchors: RubricAnchors;
+        }>;
+        passing_score?: number;
+    };
+    model?: string;
+};
+export type GradersConfigObject = {
+    graders: GraderSpecObject[];
+};
+export type DistillOptions = {
+    /** Normalized score at/above which a turn is a positive (gold) sample. */
+    minScore: number;
+    /** Emit a single `llm_judge` grader seeded from comment themes instead of the
+     *  deterministic grader. Opt-in (each eval sample then costs a judge call). */
+    judge?: boolean;
+    /** Judge model baked into the emitted llm_judge grader (else the runner's
+     *  `--judge-model` applies at eval time). */
+    judgeModel?: string;
+};
+export type DistillStats = {
+    totalFeedback: number;
+    matchedTurns: number;
+    unmatchedFeedback: number;
+    positives: number;
+    negatives: number;
+};
+export type DistillResult = {
+    samples: Sample[];
+    graders: GradersConfigObject;
+    stats: DistillStats;
+    /** Non-fatal warnings (e.g. a rating whose turn is missing from the log). */
+    warnings: string[];
+};
+/**
+ * Tag-all distillation. Every rated turn becomes a Sample; positively-rated
+ * turns (normalized ≥ minScore) — and any turn with a `correction` — become
+ * GOLD (expected_output set), the rest carry a low `metadata.user_rating` and
+ * no expected_output so they feed the optimizer's failure channel. A single
+ * deterministic grader is synthesized from the positive turns' behavior
+ * (exactly one, to avoid the hard-AND collapse in `all(...)`).
+ */
+export declare function distill(turns: ReadonlyArray<SessionTurn>, feedback: ReadonlyArray<FeedbackRecord>, opts: DistillOptions): DistillResult;
+/**
+ * Pick the single deterministic grader that best captures the up-rated
+ * behavior. Prefers a `tool_call_sequence` over the tools every tool-using
+ * positive turn shared (falling back to the most common tool); if the positive
+ * turns used no tools, a `contains` over a distinctive token common to their
+ * answers; failing that, a non-empty-answer floor (with a warning).
+ */
+export declare function synthesizeGraders(positiveTurns: ReadonlyArray<DerivedTurn>, warnings?: string[]): GradersConfigObject;
+/**
+ * Build a single `llm_judge` grader seeded from the aggregated comment themes.
+ * One criterion ("user_preference") with fixed 1–5 anchors and a description
+ * folding a short, quoted, clipped summary of what users praised vs criticized.
+ * Emitted as the SOLE grader so it is never hard-ANDed with a deterministic one
+ * (the `all(...)` min-collapse). The untrusted comment text is bounded and
+ * quoted-as-data; the rubric is trusted context sent verbatim to the judge.
+ */
+export declare function buildJudgeRubricGrader(positiveComments: ReadonlyArray<string>, negativeComments: ReadonlyArray<string>, model?: string): GraderSpecObject;
+/** One SampleSchema-valid JSON object per line (round-trips through loadJsonl). */
+export declare function samplesToJsonl(samples: ReadonlyArray<Sample>): string;
+/** Emit a GradersConfigSchema-valid graders.yaml. Scalars are JSON-encoded so
+ *  arbitrary substrings/tool names stay YAML-safe (JSON is a YAML subset).
+ *  `headerLines` replaces the distill provenance header (each line must be a
+ *  full `# …` YAML comment). */
+export declare function gradersConfigToYaml(config: GradersConfigObject, headerLines?: ReadonlyArray<string>): string;
+export {};