@phnx-labs/agents-cli 1.22.52 → 1.22.53

package/dist/lib/traces/insights.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Cross-session failure clustering + time-wasted attribution for the traces
+ * insight engine (PHNX-3141) — the piece that turns per-tool error counts
+ * into "here is your #1 systemic problem and what it cost."
+ *
+ * Pure and SQL-shaped: it consumes the same `SyncRow[]` + `tool_calls` rows
+ * `buildIndexShard` already loads (no re-parsing of transcripts), so cost
+ * stays proportional to this sync's row count, never the full corpus.
+ *
+ * `tool_calls` rows carry `ordinal`/`timestamp` per call within a session
+ * (`db.ts`'s `idx_tool_calls_session ON tool_calls(session_id, ordinal)`), which
+ * is enough to reconstruct per-session call order and inter-call gaps without a
+ * full `SessionTrajectory` — that is what makes this incremental at scale.
+ *
+ * Scope note: `FailureSignature` does not yet carry a `phenotype`
+ * (false-termination / out-of-order / …, `phenotype.ts`) — classifying that
+ * needs the full derived trajectory (turns, ordered steps, gaps), which is
+ * only ever materialized per-session during upload, not cached the way
+ * `InsightFacets` is. Folding it in is a real, scoped follow-up (see
+ * `cli/AGENTS.md`), not a silent omission.
+ */
+import { type TraceFailureCause } from './classify.js';
+import { type LatencyInsight } from './segments.js';
+import { type SyncRow, type ToolCallRow, type TracesIndexShard } from './sync.js';
+export interface FailureSignature {
+    tool: string;
+    cause: TraceFailureCause;
+    /** Normalized error text — volatile tokens (ids, counts, countdowns) stripped so instances fold together. */
+    key: string;
+}
+export interface FailurePattern {
+    /** Stable hash of the signature — deep-linkable, unaffected by row order. */
+    id: string;
+    label: string;
+    signature: FailureSignature;
+    /** Distinct sessions this pattern occurred in. */
+    sessions: number;
+    /** Total failing calls matching this signature. */
+    occurrences: number;
+    /** Estimated ms of retry/stall time attributable to this pattern (see attribution rule below). */
+    wastedMs: number;
+    /** Bounded example session ids for drill-down. */
+    exampleSessionIds: string[];
+    /** Movement vs the same pattern id in the previous shard. */
+    drift: 'up' | 'flat' | 'down';
+}
+export interface ComputedInsights {
+    /** Top-K patterns ranked by wastedMs (impact) — a rare 1-occurrence/8h loop still surfaces. */
+    failurePatterns: FailurePattern[];
+    /** Sum of wastedMs across every cluster found this sync, not just the top-K rows above. */
+    wastedMsTotal: number;
+    latency: LatencyInsight;
+}
+/** Strip volatile tokens from a failure's evidence text so repeat instances hash identically. */
+export declare function normalizeErrorKey(desc: string, raw: string | null): string;
+/**
+ * Cluster failed tool calls into ranked patterns and estimate the wasted time
+ * behind each, plus device-wide time-to-first-tool latency.
+ *
+ * wastedMs attribution: the gap between a failed call and the NEXT call in the
+ * same session counts as wasted when either (a) the next call repeats the same
+ * signature (a retry loop) or (b) the gap itself is a stall (≥60s) — an idle
+ * gap unrelated to a nearby failure is never counted. This is an estimate, not
+ * ground truth (a stall could be legitimate user think-time); it is not
+ * inflated by folding in ordinary processing time between unrelated calls.
+ */
+export declare function computeInsights(rows: readonly SyncRow[], calls: readonly ToolCallRow[], prevShard?: TracesIndexShard | null): ComputedInsights;

package/dist/lib/traces/insights.js

@@ -0,0 +1,178 @@
+/**
+ * Cross-session failure clustering + time-wasted attribution for the traces
+ * insight engine (PHNX-3141) — the piece that turns per-tool error counts
+ * into "here is your #1 systemic problem and what it cost."
+ *
+ * Pure and SQL-shaped: it consumes the same `SyncRow[]` + `tool_calls` rows
+ * `buildIndexShard` already loads (no re-parsing of transcripts), so cost
+ * stays proportional to this sync's row count, never the full corpus.
+ *
+ * `tool_calls` rows carry `ordinal`/`timestamp` per call within a session
+ * (`db.ts`'s `idx_tool_calls_session ON tool_calls(session_id, ordinal)`), which
+ * is enough to reconstruct per-session call order and inter-call gaps without a
+ * full `SessionTrajectory` — that is what makes this incremental at scale.
+ *
+ * Scope note: `FailureSignature` does not yet carry a `phenotype`
+ * (false-termination / out-of-order / …, `phenotype.ts`) — classifying that
+ * needs the full derived trajectory (turns, ordered steps, gaps), which is
+ * only ever materialized per-session during upload, not cached the way
+ * `InsightFacets` is. Folding it in is a real, scoped follow-up (see
+ * `cli/AGENTS.md`), not a silent omission.
+ */
+import { classifyCause } from './classify.js';
+import { computeLatency } from './segments.js';
+import { failureDescription } from './sync.js';
+// ---------------------------------------------------------------------------
+// Tunables
+// ---------------------------------------------------------------------------
+/** Bounded shard size — patterns, not sessions, so 738 or 738k render identically. */
+const TOP_K_PATTERNS = 25;
+const MAX_EXAMPLE_SESSIONS = 5;
+/** A gap this long right after a failure reads as an idle stall, not think-time (matches sync.ts's own "stalled Xm" threshold). */
+const STALL_MS = 60_000;
+// ---------------------------------------------------------------------------
+// Signature normalization — fold volatile per-instance text together
+// ---------------------------------------------------------------------------
+const VOLATILE_TOKEN_PATTERNS = [
+    { pattern: /\bfor user [\w.-]+/gi, replacement: 'for user _' },
+    { pattern: /\btry again in [\w.]+s?\b/gi, replacement: 'try again in _s' },
+    { pattern: /\b[0-9a-f]{7,40}\b/gi, replacement: '_sha_' },
+    { pattern: /\b[\w.-]+@[\w.-]+\.\w+\b/gi, replacement: '_email_' },
+    { pattern: /\b\d+\b/g, replacement: '_n_' },
+];
+/** Strip volatile tokens from a failure's evidence text so repeat instances hash identically. */
+export function normalizeErrorKey(desc, raw) {
+    let text = (raw && raw.trim().length > 0 ? raw : desc).toLowerCase();
+    for (const { pattern, replacement } of VOLATILE_TOKEN_PATTERNS) {
+        text = text.replace(pattern, replacement);
+    }
+    return text.replace(/\s+/g, ' ').trim().slice(0, 160);
+}
+function hashSignature(tool, cause, key) {
+    const input = `${tool} ${cause} ${key}`;
+    let hash = 5381;
+    for (let i = 0; i < input.length; i++) {
+        hash = ((hash << 5) + hash + input.charCodeAt(i)) >>> 0;
+    }
+    return hash.toString(16).padStart(8, '0');
+}
+// ---------------------------------------------------------------------------
+// Label rules — a table, not an if/else-by-name chain (matches segments.ts's TASK_TYPE_RULES)
+// ---------------------------------------------------------------------------
+const LABEL_RULES = [
+    { pattern: /rate limit/i, label: 'rate limit back-off loop' },
+    { pattern: /permission denied/i, label: 'permission denied' },
+    { pattern: /not found|no such file/i, label: 'missing resource' },
+    { pattern: /timed? ?out/i, label: 'timeout' },
+    { pattern: /econnrefused|connection refused|network/i, label: 'network error' },
+    { pattern: /conflict|diverged/i, label: 'git conflict' },
+];
+function labelFor(tool, cause, key) {
+    if (cause === 'guard')
+        return `${tool}: git guard denial`;
+    if (cause === 'hook')
+        return `${tool}: hook denial`;
+    const rule = LABEL_RULES.find((row) => row.pattern.test(key));
+    return `${tool}: ${rule ? rule.label : key.slice(0, 48)}`;
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Cluster failed tool calls into ranked patterns and estimate the wasted time
+ * behind each, plus device-wide time-to-first-tool latency.
+ *
+ * wastedMs attribution: the gap between a failed call and the NEXT call in the
+ * same session counts as wasted when either (a) the next call repeats the same
+ * signature (a retry loop) or (b) the gap itself is a stall (≥60s) — an idle
+ * gap unrelated to a nearby failure is never counted. This is an estimate, not
+ * ground truth (a stall could be legitimate user think-time); it is not
+ * inflated by folding in ordinary processing time between unrelated calls.
+ */
+export function computeInsights(rows, calls, prevShard) {
+    const bySession = new Map();
+    for (const call of calls) {
+        const list = bySession.get(call.session_id);
+        if (list)
+            list.push(call);
+        else
+            bySession.set(call.session_id, [call]);
+    }
+    const groups = new Map();
+    for (const [sessionId, sessionCalls] of bySession) {
+        const ordered = [...sessionCalls].sort((a, b) => a.ordinal - b.ordinal);
+        for (let i = 0; i < ordered.length; i++) {
+            const call = ordered[i];
+            if (call.outcome !== 'error')
+                continue;
+            const cause = classifyCause(call);
+            const key = normalizeErrorKey(failureDescription(call, cause), call.error);
+            const groupKey = `${call.tool} ${cause} ${key}`;
+            let group = groups.get(groupKey);
+            if (!group) {
+                group = { tool: call.tool, cause, key, sessions: new Set(), occurrences: 0, wastedMs: 0, examples: [] };
+                groups.set(groupKey, group);
+            }
+            group.occurrences++;
+            group.sessions.add(sessionId);
+            if (group.examples.length < MAX_EXAMPLE_SESSIONS && !group.examples.includes(sessionId)) {
+                group.examples.push(sessionId);
+            }
+            const next = ordered[i + 1];
+            if (!next)
+                continue;
+            const gapMs = Date.parse(next.timestamp) - Date.parse(call.timestamp);
+            if (!Number.isFinite(gapMs) || gapMs <= 0)
+                continue;
+            const nextIsSameFailure = next.outcome === 'error' &&
+                next.tool === call.tool &&
+                classifyCause(next) === cause &&
+                normalizeErrorKey(failureDescription(next, cause), next.error) === key;
+            if (nextIsSameFailure || gapMs >= STALL_MS) {
+                group.wastedMs += gapMs;
+            }
+        }
+    }
+    const prevById = new Map((prevShard?.failurePatterns ?? []).map((p) => [p.id, p]));
+    const allPatterns = [...groups.values()].map((group) => {
+        const id = hashSignature(group.tool, group.cause, group.key);
+        const prev = prevById.get(id);
+        const drift = !prev
+            ? 'up'
+            : group.occurrences > prev.occurrences
+                ? 'up'
+                : group.occurrences < prev.occurrences
+                    ? 'down'
+                    : 'flat';
+        return {
+            id,
+            label: labelFor(group.tool, group.cause, group.key),
+            signature: { tool: group.tool, cause: group.cause, key: group.key },
+            sessions: group.sessions.size,
+            occurrences: group.occurrences,
+            wastedMs: group.wastedMs,
+            exampleSessionIds: group.examples,
+            drift,
+        };
+    });
+    const wastedMsTotal = allPatterns.reduce((sum, p) => sum + p.wastedMs, 0);
+    const failurePatterns = [...allPatterns]
+        .sort((a, b) => b.wastedMs - a.wastedMs || b.occurrences - a.occurrences || a.id.localeCompare(b.id))
+        .slice(0, TOP_K_PATTERNS);
+    const latency = computeLatency(firstToolSegments(rows, bySession));
+    return { failurePatterns, wastedMsTotal, latency };
+}
+/** Synthesize one-step SegmentSessions carrying only the time-to-first-tool offset, for computeLatency() reuse. */
+function firstToolSegments(rows, bySession) {
+    return rows.flatMap((row) => {
+        const sessionCalls = bySession.get(row.id);
+        if (!sessionCalls || sessionCalls.length === 0)
+            return [];
+        const first = sessionCalls.reduce((min, call) => (call.ordinal < min.ordinal ? call : min));
+        const sessionStartMs = Date.parse(row.timestamp);
+        const firstCallMs = Date.parse(first.timestamp);
+        if (!Number.isFinite(sessionStartMs) || !Number.isFinite(firstCallMs))
+            return [];
+        return [{ steps: [{ startMs: Math.max(0, firstCallMs - sessionStartMs) }] }];
+    });
+}

package/dist/lib/traces/phenotype.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Failure phenotype classifier + outcome taxonomy for the traces insight engine.
+ *
+ * Both functions are pure: they take a redacted {@link SessionDetail} (the same
+ * shape the traces sync writes to `sessions/<id>.json`) and return a decision
+ * derived only from the already-derived step/gap/meta signal. They never read
+ * raw transcript text and never fabricate a signal that is not in the data.
+ *
+ * The rubrics are expressed as data-driven tables of conditions, not as
+ * if/else-by-name chains. Each table row is a named phenotype/outcome with a
+ * declarative predicate; the classifier walks the table in priority order and
+ * returns the first match, or the honest lower-confidence default when no
+ * high-confidence signal is present.
+ */
+import type { SessionDetail } from './sync.js';
+/** A failure mode detectable from the derived trajectory of a session. */
+export type FailurePhenotype = 'false-termination' | 'premature-completion' | 'out-of-order' | 'failure-to-act';
+/**
+ * The coarse outcome of a session's work.
+ *
+ * - `merged`       : explicit PR/branch merge signal in the steps (high confidence).
+ * - `tests-green`  : explicit test command returned ok with no later failure (high confidence).
+ * - `partial`      : progress was made but no landing/test signal is present (low confidence default).
+ * - `abandoned`    : errored, stalled, and unresolved.
+ * - `human-takeover`: the final substantive action was a human-facing ask/wait.
+ * - `invalid-env`  : environment/setup failures dominated the session.
+ */
+export type TraceOutcome = 'merged' | 'tests-green' | 'partial' | 'abandoned' | 'human-takeover' | 'invalid-env';
+export interface PhenotypeResult {
+    phenotype: FailurePhenotype | null;
+    reason: string;
+}
+export interface OutcomeResult {
+    outcome: TraceOutcome;
+    confidence: 'high' | 'medium' | 'low';
+    reason: string;
+}
+/**
+ * Classify the failure phenotype of a session from its derived trajectory.
+ *
+ * Definitions (from agent-failure research):
+ * - `false-termination`  — stopped with an unresolved error.
+ * - `premature-completion` — declared done while tests were failing or no
+ *   verification step ran for the engineering work.
+ * - `out-of-order` — a write/edit step occurred before any read/plan of the
+ *   target.
+ * - `failure-to-act` — stalled or produced no meaningful tool use.
+ *
+ * Returns `null` when none of the failure phenotypes apply.
+ */
+export declare function classifyPhenotype(session: SessionDetail): FailurePhenotype | null;
+/** Detailed phenotype result with a human-readable reason. */
+export declare function classifyPhenotypeDetailed(session: SessionDetail): PhenotypeResult;
+/**
+ * Derive the coarse outcome of a session from its end state + tool signals.
+ *
+ * `merged` and `tests-green` are high-confidence only when an explicit signal
+ * is present in the derived steps. When that signal is genuinely not in the
+ * data, the function returns the honest lower-confidence value (`partial` for
+ * completed work without a landing signal, `abandoned` for errored/unresolved
+ * work, `invalid-env` for setup-dominant failures, `human-takeover` when the
+ * session ends on a human-facing ask).
+ */
+export declare function deriveOutcome(session: SessionDetail): TraceOutcome;
+/** Detailed outcome result with confidence and a human-readable reason. */
+export declare function deriveOutcomeDetailed(session: SessionDetail): OutcomeResult;
+export type { SessionDetail } from './sync.js';