crewhaus 0.1.8 → 0.2.1

package/dist/advise-rules.d.ts

@@ -0,0 +1,348 @@
+/**
+ * Item 14 — `crewhaus advise` rule library: the pure, side-effect-free half
+ * of the trace-mining spec advisor. Aggregates session JSONLs (including the
+ * advisor kinds runtime-core now persists — `recovery`, `tool_stats`,
+ * `permission`, `model_meta`) plus `.crewhaus/audit` records into an
+ * `AdviceContext`, then runs a library of threshold rules over it. Each rule
+ * returns typed findings whose suggestion is either a pre-validated
+ * `SpecPatch` (consumable by a future `optimize --from-advice`) or free-form
+ * advice text.
+ *
+ * Everything here is pure so it is unit-testable; all filesystem access
+ * lives in `apps/cli/src/index.ts` (mirrors `feedback.ts` / `eval-history.ts`).
+ *
+ * Old-vintage logs (written before the advisor kinds existed) build an empty
+ * context slice for each missing kind and simply produce no findings — the
+ * rules only fire where the data now exists.
+ */
+import type { Spec } from "@crewhaus/spec";
+import { type SpecPatch } from "@crewhaus/spec-patch";
+export type AdviceSeverity = "info" | "warn";
+export type AdviceSuggestion = {
+    readonly kind: "spec-patch";
+    readonly patch: SpecPatch;
+} | {
+    readonly kind: "advice";
+    readonly text: string;
+};
+export type AdviceFinding = {
+    /** `<rule>` or `<rule>:<subject>` (e.g. `repeated-tool-failures:Fetch`). */
+    readonly id: string;
+    readonly severity: AdviceSeverity;
+    /** One-line human summary of what tripped. */
+    readonly summary: string;
+    /** Human-readable evidence lines (counts, rates, example subjects). */
+    readonly evidence: ReadonlyArray<string>;
+    /** The numeric aggregates the rule fired on. */
+    readonly counts: Readonly<Record<string, number>>;
+    readonly suggestion: AdviceSuggestion;
+};
+export type ToolCallStats = {
+    calls: number;
+    errors: number;
+    totalDurationMs: number;
+};
+export type AskStats = {
+    asks: number;
+    approved: number;
+    denied: number;
+};
+/** Item 19 — per-errorName recovery aggregate: total occurrences plus the
+ *  distribution of recovery actions that error drove. */
+export type ErrorRecoveryStats = {
+    count: number;
+    /** action (`retry`/`compact`/`continue`/`tombstone`/`fail`) → occurrences. */
+    actions: Map<string, number>;
+};
+/**
+ * Session-derived aggregates the rules consume. Built by
+ * `buildAdviceContext` from parsed session-JSONL objects (tolerant of
+ * old-vintage logs, malformed lines, and unknown kinds) plus optional
+ * `.crewhaus/audit` records.
+ */
+export type AdviceContext = {
+    readonly sessionIds: ReadonlyArray<string>;
+    /** Per-tool aggregate over `tool_stats` lines. */
+    readonly toolStats: ReadonlyMap<string, ToolCallStats>;
+    /** `recovery` lines grouped by action (`retry`/`compact`/`continue`/…). */
+    readonly recoveriesByAction: ReadonlyMap<string, number>;
+    /**
+     * Item 19 — `recovery` lines grouped by `errorName`, each carrying the
+     * action distribution that error drove. The failure-taxonomy rule clusters
+     * these into draft `failure_taxonomy` entries (error class → recovery
+     * action). Old-vintage logs (no recovery lines) leave this empty.
+     */
+    readonly recoveriesByErrorName: ReadonlyMap<string, ErrorRecoveryStats>;
+    /**
+     * `action: "continue"` recoveries — emitted exclusively for
+     * `max_output_tokens` truncations (recovery-engine's taxonomy), so this
+     * IS the truncation-pressure count.
+     */
+    readonly truncationContinues: number;
+    /**
+     * Item 19 — synthetic `[runtime] possible loop detected` nudges mined from
+     * `user_message` lines, grouped by tool signature. `matchNamedFailure`
+     * matches error.message only, so a loop (which is NOT an error — no
+     * exception is thrown) can never be caught by a taxonomy entry; the
+     * loop-break rule turns recurring signatures into instructions-addendum
+     * advice instead.
+     */
+    readonly loopSignatures: ReadonlyMap<string, number>;
+    /** `compaction` lines per session (any vintage — this kind predates item 14). */
+    readonly compactionsBySession: ReadonlyMap<string, number>;
+    /** Resolved permission asks per tool, from `permission` lines. */
+    readonly asksByTool: ReadonlyMap<string, AskStats>;
+    /** Stop-reason distribution over `model_meta` lines. */
+    readonly stopReasons: ReadonlyMap<string, number>;
+    readonly modelResponses: number;
+    /**
+     * Item 21 — per-tool total byte spend (tool_use input JSON + the correlated
+     * tool_result content), keyed by the PascalCase runtime tool name. The
+     * sub-agent-split rule uses the dominance of one tool's byte spend as the
+     * per-turn-token-pressure proxy (a tool that dominates context is the one
+     * to move behind its own window). Empty on old-vintage logs.
+     */
+    readonly perToolBytes: ReadonlyMap<string, number>;
+    /** Item 21 — total observed tool byte spend across all tools (the
+     *  denominator for the dominance ratio). */
+    readonly totalToolBytes: number;
+    /** `.crewhaus/audit` record kinds → counts (report context + future rules). */
+    readonly auditKindCounts: ReadonlyMap<string, number>;
+};
+export type SessionEvents = {
+    readonly sessionId: string;
+    readonly objects: ReadonlyArray<unknown>;
+};
+/** Parse a JSONL blob into objects, skipping blank/malformed lines —
+ *  a session log must never abort mining over one corrupt line. */
+export declare function parseJsonlObjects(text: string): unknown[];
+/** The fixed prefix the runtime stamps on a loop-detection nudge (a
+ *  `user_message` with `synthetic: true`). Exported so a fixture/test can
+ *  build a realistic nudge without hard-coding the string twice. */
+export declare const LOOP_NUDGE_PREFIX = "[runtime] possible loop detected:";
+/**
+ * Item 19 — extract a loop signature (the looping tool name) from a
+ * `user_message` payload's `content` when it is a runtime loop-detection
+ * nudge, else undefined. The nudge content is the fixed string
+ * `[runtime] possible loop detected: tool "<name>" has been called …`.
+ * Returns `tool:<name>` so recurring loops on the same tool cluster; falls
+ * back to a generic `loop` signature when the tool name can't be parsed.
+ */
+export declare function loopSignatureOf(content: unknown): string | undefined;
+/** Tolerantly read a session-JSONL line's payload when its `kind` matches —
+ *  the canonical defensive accessor shared with `context-pressure.ts`
+ *  (`doctor --context-pressure` folds the same persisted events). */
+export declare function payloadOf(obj: unknown, kind: string): Record<string, unknown> | undefined;
+/**
+ * Fold parsed session objects (and optional audit records) into the
+ * aggregates the rules consume. Every field access is defensive: missing
+ * payloads, wrong-typed fields, and unknown kinds are skipped, so
+ * old-vintage logs (no advisor kinds) and future kinds both pass through
+ * cleanly.
+ */
+export declare function buildAdviceContext(sessions: ReadonlyArray<SessionEvents>, auditObjects?: ReadonlyArray<unknown>): AdviceContext;
+export type AdviceThresholds = {
+    /** Minimum calls before a tool's error rate is judged at all. */
+    toolFailureMinCalls: number;
+    /** Error rate (errors/calls) at/above which the failure rule fires. */
+    toolFailureRate: number;
+    /** `continue` recoveries at/above which truncation pressure fires. */
+    truncationContinues: number;
+    /** Compactions in a single session at/above which thrash fires. */
+    compactionsPerSession: number;
+    /** Resolved asks per tool at/above which churn is judged. */
+    asksPerTool: number;
+    /** Minimum model responses before stop reasons are judged at all. */
+    stopMinResponses: number;
+    /** Anomalous-stop rate at/above which the stop rule fires. */
+    stopAnomalyRate: number;
+    /** Item 19 — occurrences of one error CLASS at/above which a
+     *  failure_taxonomy entry is drafted. */
+    recoveryClusterMin: number;
+    /** Item 19 — occurrences of one loop SIGNATURE at/above which loop-break
+     *  instructions advice fires. */
+    loopSignatureMin: number;
+    /** Item 21 — sessions that must each compact ≥ compactionsPerSession
+     *  before a sub-agent restructure is suggested (chronic, not one-off). */
+    chronicCompactionSessions: number;
+    /** Item 21 — fraction of total tool byte-spend one tool must dominate for a
+     *  sub-agent split to be suggested on the dominance signal. */
+    toolByteDominance: number;
+    /** Item 21 — minimum total tool bytes before dominance is judged at all
+     *  (a handful of tiny calls shouldn't trip a restructure). */
+    toolByteMinTotal: number;
+};
+/**
+ * Canonical rule thresholds. runtime-core's in-run digest
+ * (observability.ts DIGEST_THRESHOLDS) mirrors these — it cannot import
+ * them (apps/cli depends on runtime-core, not vice versa), so keep the two
+ * in sync when tuning.
+ */
+export declare const DEFAULT_ADVICE_THRESHOLDS: AdviceThresholds;
+/** Stop reasons that are part of healthy operation; everything else
+ *  (max_tokens, refusal, pause_turn, …) counts as anomalous. Mirrored by
+ *  runtime-core's digest tally. */
+export declare const CLEAN_STOP_REASONS: ReadonlySet<string>;
+export type RuleOptions = {
+    /** The cwd spec, when one exists — enables patch suggestions. */
+    readonly spec?: Spec;
+    readonly thresholds?: Partial<AdviceThresholds>;
+};
+export type AdviceRule = (ctx: AdviceContext, opts?: RuleOptions) => AdviceFinding[];
+/**
+ * Repeated tool failures: a tool with ≥ minCalls whose error rate crosses
+ * the threshold. Advice-only by design — `tool_config` is not in
+ * OPTIMIZABLE_PATHS (the whitelist is the autotune safety floor), so the
+ * suggestion points at the spec's `tool_config` block when one exists for
+ * the tool instead of emitting a patch. NOTE: `tool_stats.isError` includes
+ * permission-denied results; the churn rule covers the ask side of that.
+ */
+export declare const ruleRepeatedToolFailures: AdviceRule;
+/**
+ * Truncation pressure: `continue` recoveries (always max_output_tokens) at
+ * or above the threshold. Suggests a SpecPatch bumping `agent.max_tokens`
+ * (whitelisted in OPTIMIZABLE_PATHS): double the spec's current value, or
+ * add 16384 (2× the runtime's 8192 default) when the spec doesn't set one.
+ */
+export declare const ruleTruncationPressure: AdviceRule;
+/**
+ * Compaction thrash: any single session compacting ≥ threshold times.
+ * ADVICE-ONLY — no auto-applicable SpecPatch. `compaction.curate` is listed
+ * in OPTIMIZABLE_PATHS and lowers to IR (`packages/ir/src/index.ts`,
+ * `IrCompaction.curate`), but no target emitter or runtime-core path
+ * actually consumes it yet (see the IR field's doc comment). A patch that
+ * flips a runtime no-op would pass the `optimize --from-advice` eval gate
+ * trivially (inert change → equal pass rate) and get written into the
+ * user's spec while changing nothing, so this rule surfaces the finding as
+ * a warning with concrete alternatives instead of proposing `curate: true`.
+ * `compaction.threshold` is ALSO whitelisted, but the strict spec schema
+ * carries no such key today, so a threshold patch could never apply either.
+ */
+export declare const ruleCompactionThrash: AdviceRule;
+/**
+ * Permission churn: a tool prompting ≥ threshold times with 100% approved
+ * outcomes. NEVER a patch — permissions are excluded from OPTIMIZABLE_PATHS
+ * by design (an optimizer must not be able to widen its own permissions);
+ * the human gets the alwaysAllow one-liner with the evidence instead.
+ */
+export declare const rulePermissionChurn: AdviceRule;
+/**
+ * Stop-reason anomalies: a high rate of stops outside CLEAN_STOP_REASONS
+ * (end_turn / tool_use / stop_sequence are healthy; max_tokens, refusal,
+ * pause_turn, … are not). Advice-only — the right fix depends on which
+ * reason dominates.
+ */
+export declare const ruleStopReasonAnomalies: AdviceRule;
+/** One drafted failure_taxonomy entry (shape = the spec's entry schema:
+ *  `{ class, pattern, recovery, hint? }`). */
+export type FailureTaxonomyEntry = {
+    readonly class: string;
+    readonly pattern: string;
+    readonly recovery: string;
+    readonly hint?: string;
+};
+/**
+ * Item 19 — specificity floor for a drafted taxonomy `pattern`. The pattern is
+ * `errorName` verbatim, matched by the recovery engine as a case-insensitive
+ * SUBSTRING of `error.message`. A very short or generic name (e.g. "Error")
+ * drafts a rule that fires on almost every failure — worse than no entry. Such
+ * clusters are surfaced as ADVICE (a human should hand-write a precise pattern)
+ * rather than drafted into a patch.
+ */
+export declare const TAXONOMY_MIN_PATTERN_LEN = 4;
+/** Generic error tokens too broad to auto-draft, regardless of length. */
+export declare const TAXONOMY_GENERIC_TOKENS: ReadonlySet<string>;
+/** True when `errorName` is too broad to draft a taxonomy pattern for
+ *  (too short, or a generic token). Case-insensitive. */
+export declare function taxonomyPatternTooBroad(errorName: string): boolean;
+/**
+ * Item 19 — failure-taxonomy learning. Clusters recurring `recovery` events
+ * by `errorName`; each class seen ≥ recoveryClusterMin times (and NOT already
+ * covered by the spec's taxonomy) becomes a draft `failure_taxonomy` entry
+ * whose `recovery` action is the one the runtime actually took most for that
+ * class, and whose `pattern` is a case-insensitive substring of the error
+ * name.
+ *
+ * `failure_taxonomy` IS whitelisted in OPTIMIZABLE_PATHS for every target, so
+ * the emitted patch rides the eval-gated `optimize --from-advice` apply. The
+ * patch REPLACES the whole `failure_taxonomy` array (existing entries +
+ * drafts) when the spec already has one, else ADDS the array — a whole-block
+ * edit the whitelist's prefix match allows. Budget caps aren't a field on the
+ * strict entry schema (`{ class, pattern, recovery, hint? }`), so the drafted
+ * `hint` carries the observed count + the "matches error.message" caveat and
+ * the recovery-engine's per-turn budget note.
+ */
+export declare const ruleFailureTaxonomy: AdviceRule;
+/**
+ * Item 19 — loop-break learning. A `[runtime] possible loop detected` nudge
+ * is NOT an error (no exception is thrown), so `matchNamedFailure` — which
+ * matches error.message only — can never catch it; a taxonomy entry is the
+ * wrong tool. This rule instead drafts an instructions-ADDENDUM as advice
+ * text (never a patch: `agent.instructions` IS optimizable, but a
+ * mechanically-appended nudge is a prompt edit a human should own, and the
+ * optimizer already rewrites instructions on its own signal). Fires per
+ * recurring loop signature (same tool looped in ≥ loopSignatureMin sessions/
+ * turns).
+ */
+export declare const ruleLoopBreak: AdviceRule;
+/**
+ * Targets whose AGENT block carries a `sub_agents:` map, so the drafted
+ * fragment (which sits under `agent:`) pastes in verbatim: cli and channel.
+ * Crew ALSO has sub_agents, but under each ROLE rather than a single `agent`
+ * block, so the paste location differs — it's excluded here rather than emit a
+ * fragment that lands in the wrong place. Any other target has no sub_agents
+ * block at all, so the rule stays silent (nowhere to paste).
+ */
+export declare const SUB_AGENT_TARGETS: ReadonlySet<string>;
+/** Draft a ready-to-paste `sub_agents:` YAML fragment moving `toolName`'s
+ *  concern into its own sub-agent with its own context window. Two-space
+ *  indented to sit under the spec's `agent:` block (cli/channel) — the CLI
+ *  prints it verbatim in the advice text. */
+export declare function subAgentFragment(toolName: string): string;
+/**
+ * Item 21 — suggest a sub-agent split when context pressure is chronic:
+ * either the mined sessions ROUTINELY compact (≥ chronicCompactionSessions
+ * sessions each compacting ≥ compactionsPerSession times) OR one tool/concern
+ * DOMINATES tool byte-spend (≥ toolByteDominance of ≥ toolByteMinTotal total
+ * bytes). Moving the heavy concern behind a sub-agent's own context window
+ * relieves the parent.
+ *
+ * STRICTLY suggest+scaffold — NEVER a patch: a structural change (adding a
+ * sub-agent + rewriting instructions) isn't eval-safe to auto-apply, so this
+ * emits advice text carrying a ready-to-paste `sub_agents:` fragment (the
+ * correct spec key — NOT `agents:`). Gated per target: only cli/channel/crew
+ * schemas have a `sub_agents` block; on any other target (or with no spec) the
+ * rule stays silent, since there's nowhere to paste it.
+ */
+export declare const ruleSubAgentSplit: AdviceRule;
+export declare const ADVICE_RULES: ReadonlyArray<AdviceRule>;
+/**
+ * Run every rule and rank the findings: warn before info, then by the
+ * magnitude of the primary count, then by id (deterministic output). Each
+ * emitted patch was already pre-validated by its rule via `patchOrAdvice`.
+ */
+export declare function runAdviceRules(ctx: AdviceContext, opts?: RuleOptions): AdviceFinding[];
+export type SuggestionsFile = {
+    readonly generatedAt: string;
+    readonly sessionIds: ReadonlyArray<string>;
+    /** Only the findings whose suggestion is a validated SpecPatch. */
+    readonly suggestions: ReadonlyArray<{
+        readonly findingId: string;
+        readonly severity: AdviceSeverity;
+        readonly summary: string;
+        readonly patch: SpecPatch;
+    }>;
+};
+/** The `suggestions.json` payload — the SpecPatch list a future
+ *  `optimize --from-advice` consumes. Advice-only findings are report-only. */
+export declare function buildSuggestionsFile(findings: ReadonlyArray<AdviceFinding>, sessionIds: ReadonlyArray<string>, generatedAt: string): SuggestionsFile;
+export declare function escapeHtml(s: string): string;
+/** Self-contained `report.html` for `crewhaus advise` — ranked findings with
+ *  evidence and the suggested patch/advice per finding. */
+export declare function renderAdviceHtml(input: {
+    readonly findings: ReadonlyArray<AdviceFinding>;
+    readonly sessionIds: ReadonlyArray<string>;
+    readonly generatedAt: string;
+}): string;
+/** One line per finding for the CLI's text mode. */
+export declare function formatFindingLines(f: AdviceFinding): string[];