@phnx-labs/agents-cli 1.22.52 → 1.22.53

package/dist/lib/traces/phenotype.js

@@ -0,0 +1,437 @@
+/**
+ * Failure phenotype classifier + outcome taxonomy for the traces insight engine.
+ *
+ * Both functions are pure: they take a redacted {@link SessionDetail} (the same
+ * shape the traces sync writes to `sessions/<id>.json`) and return a decision
+ * derived only from the already-derived step/gap/meta signal. They never read
+ * raw transcript text and never fabricate a signal that is not in the data.
+ *
+ * The rubrics are expressed as data-driven tables of conditions, not as
+ * if/else-by-name chains. Each table row is a named phenotype/outcome with a
+ * declarative predicate; the classifier walks the table in priority order and
+ * returns the first match, or the honest lower-confidence default when no
+ * high-confidence signal is present.
+ */
+// ---------------------------------------------------------------------------
+// Tool / program category tables
+// ---------------------------------------------------------------------------
+const READ_PLAN_TOOLS = new Set([
+    'Read',
+    'read_file',
+    'grep',
+    'list_dir',
+    'search',
+    'codebase_search',
+    'ToolSearch',
+    'web_search',
+    'web_fetch',
+    'WebSearch',
+    'FetchURL',
+    'TaskCreate',
+    'todo_write',
+]);
+const WRITE_EDIT_TOOLS = new Set([
+    'Edit',
+    'Write',
+    'search_replace',
+    'write',
+    'notebookedit',
+    'multiedit',
+]);
+const HUMAN_FACING_TOOLS = new Set(['AskUserQuestion', 'SendMessage', 'wait']);
+const SHELL_TOOLS = new Set([
+    'Bash',
+    'run_terminal_command',
+    'exec_command',
+    'exec',
+    'shell',
+    'Execute',
+]);
+const OUTCOME_SIGNALS = [
+    { type: 'merge', pattern: /\bgh pr merge\b|\bmerged?\b.*\b(PR|pull request|branch)\b|\brebase-?merge\b/i },
+    { type: 'merge', pattern: /\bmerge\b.*\bsucceeded\b|\bsuccessfully\s+merged\b/i },
+    { type: 'test', pattern: /\b(bun test|npm test|yarn test|pnpm test|pytest|jest|vitest|cargo test|go test)\b/i },
+    { type: 'test', pattern: /\btsc\s+--noEmit\b|\blint\b|\btest\.sh\b/i },
+    { type: 'env', pattern: /\bbun install\b|\bnpm install\b|\byarn install\b|\bpnpm install\b|\bpip install\b/i },
+    { type: 'env', pattern: /\bnode_modules\b|\bmissing\b|\bnot found\b|\bpermission denied\b|\bcommand not found\b/i },
+    { type: 'env', pattern: /\bssh.*key\b|\bclone\b.*\bfailed\b/i },
+    { type: 'revert', pattern: /\brevert\b|\bgit checkout\b|\breset\s+--hard\b/i },
+];
+// ---------------------------------------------------------------------------
+// Step helpers
+// ---------------------------------------------------------------------------
+function isToolStep(step) {
+    return step.kind === 'tool';
+}
+function substantiveSteps(session) {
+    return session.steps.filter((s) => isToolStep(s) && !HUMAN_FACING_TOOLS.has(s.tool ?? s.lane));
+}
+function firstStepOrdinalOf(session, predicate) {
+    for (const step of session.steps) {
+        if (predicate(step))
+            return step.ordinal;
+    }
+    return undefined;
+}
+function lastStepOrdinalOf(session, predicate) {
+    let last;
+    for (const step of session.steps) {
+        if (predicate(step))
+            last = step.ordinal;
+    }
+    return last;
+}
+function stepText(step) {
+    return `${step.label ?? ''} ${step.detail ?? ''}`.trim().toLowerCase();
+}
+function isShellStep(step) {
+    return SHELL_TOOLS.has(step.tool ?? step.lane);
+}
+function hasSignal(session, type, toolFilter) {
+    for (const step of session.steps) {
+        if (toolFilter && !toolFilter.has(step.tool ?? step.lane))
+            continue;
+        const text = stepText(step);
+        for (const signal of OUTCOME_SIGNALS) {
+            if (signal.type === type && signal.pattern.test(text))
+                return true;
+        }
+    }
+    return false;
+}
+function signalCounts(session) {
+    const counts = { merge: 0, test: 0, env: 0, revert: 0 };
+    for (const step of session.steps) {
+        const text = stepText(step);
+        for (const signal of OUTCOME_SIGNALS) {
+            // Merge/test signals must come from an actual shell/exec step, otherwise
+            // a grep pattern or read_file detail false-positives as a landing signal.
+            if ((signal.type === 'merge' || signal.type === 'test') && !isShellStep(step))
+                continue;
+            if (signal.pattern.test(text))
+                counts[signal.type]++;
+        }
+    }
+    return counts;
+}
+// ---------------------------------------------------------------------------
+// Phenotype predicates
+// ---------------------------------------------------------------------------
+/**
+ * Failure-to-act: the agent stalled or produced no meaningful tool use.
+ *
+ * Rubric:
+ * - zero tool calls and ≤2 turns, or
+ * - all substantive steps are `thinking`/human-facing, or
+ * - the session ended on a human-facing ask with no follow-up tool work.
+ */
+function isFailureToAct(session) {
+    if (session.meta.tools === 0 && session.meta.turns <= 2)
+        return true;
+    const substantive = substantiveSteps(session);
+    return substantive.length === 0;
+}
+function reasonFailureToAct(session) {
+    if (session.meta.tools === 0 && session.meta.turns <= 2) {
+        return `no tool use (${session.meta.tools} tools, ${session.meta.turns} turns)`;
+    }
+    return 'no substantive tool steps';
+}
+/**
+ * Out-of-order: a write/edit happened before any read/plan of the target.
+ *
+ * Rubric: the first substantive step is a write/edit and no earlier read/plan
+ * or search step exists.
+ */
+function isOutOfOrder(session) {
+    const firstWrite = firstStepOrdinalOf(session, (s) => WRITE_EDIT_TOOLS.has(s.tool ?? s.lane));
+    if (firstWrite === undefined)
+        return false;
+    const firstReadPlan = firstStepOrdinalOf(session, (s) => READ_PLAN_TOOLS.has(s.tool ?? s.lane));
+    if (firstReadPlan === undefined)
+        return true;
+    return firstWrite < firstReadPlan;
+}
+function reasonOutOfOrder(session) {
+    const tool = session.steps.find((s) => WRITE_EDIT_TOOLS.has(s.tool ?? s.lane));
+    return `write/edit step ${tool?.tool ?? tool?.lane ?? ''} preceded any read/plan`;
+}
+/**
+ * False-termination: the session stopped with an unresolved error.
+ *
+ * Rubric:
+ * - meta outcome is `errored`, and
+ * - at least one step outcome is `error`, and
+ * - the last non-thinking step is an error or is followed by no successful recovery.
+ */
+function isFalseTermination(session) {
+    if (session.meta.outcome !== 'errored')
+        return false;
+    if (session.meta.errorCount === 0)
+        return false;
+    const substantive = substantiveSteps(session);
+    if (substantive.length === 0)
+        return true;
+    const last = substantive[substantive.length - 1];
+    if (last.outcome === 'error')
+        return true;
+    const lastErrorOrdinal = lastStepOrdinalOf(session, (s) => s.outcome === 'error');
+    if (lastErrorOrdinal === undefined)
+        return false;
+    const recoveryAfter = substantive.some((s) => s.ordinal > lastErrorOrdinal && s.outcome === 'ok' && !HUMAN_FACING_TOOLS.has(s.tool ?? s.lane));
+    return !recoveryAfter;
+}
+function reasonFalseTermination(session) {
+    const substantive = substantiveSteps(session);
+    if (substantive.length === 0)
+        return 'errored with no substantive steps';
+    const last = substantive[substantive.length - 1];
+    if (last.outcome === 'error') {
+        return `last substantive step ${last.tool ?? last.lane} ended in error`;
+    }
+    return 'errored with no successful recovery after the last error';
+}
+/**
+ * Premature-completion: declared done while tests were failing or no verification
+ * step ran for an engineering task.
+ *
+ * Rubric:
+ * - meta outcome is `completed`, and
+ * - the session performed write/edit work, and
+ * - either errors were present or no test/build/lint verification step ran.
+ */
+function isPrematureCompletion(session) {
+    if (session.meta.outcome !== 'completed')
+        return false;
+    const didWriteEdit = session.steps.some((s) => WRITE_EDIT_TOOLS.has(s.tool ?? s.lane));
+    if (!didWriteEdit)
+        return false;
+    if (session.meta.errorCount > 0)
+        return true;
+    const verified = session.steps.some((s) => {
+        if (!SHELL_TOOLS.has(s.tool ?? s.lane))
+            return false;
+        const text = stepText(s);
+        return /\b(bun test|npm test|yarn test|pnpm test|pytest|jest|vitest|cargo test|go test|tsc\s+--noEmit|lint|build|verify)\b/.test(text);
+    });
+    return !verified;
+}
+function reasonPrematureCompletion(session) {
+    if (session.meta.errorCount > 0) {
+        return `declared completed with ${session.meta.errorCount} unresolved error(s)`;
+    }
+    return 'engineering work completed without a test/build/lint verification step';
+}
+/** Ordered rubric: the first matching phenotype wins. */
+const PHENOTYPE_RULES = [
+    { key: 'failure-to-act', score: isFailureToAct, reason: reasonFailureToAct },
+    { key: 'out-of-order', score: isOutOfOrder, reason: reasonOutOfOrder },
+    { key: 'false-termination', score: isFalseTermination, reason: reasonFalseTermination },
+    { key: 'premature-completion', score: isPrematureCompletion, reason: reasonPrematureCompletion },
+];
+// ---------------------------------------------------------------------------
+// Outcome predicates
+// ---------------------------------------------------------------------------
+/** High-confidence merge when explicit merge command/signal exists and session completed. */
+function isMerged(session) {
+    // Only a shell/exec step can produce a real merge signal; grep patterns and
+    // read_file details must not count.
+    return session.meta.outcome === 'completed' && hasSignal(session, 'merge', SHELL_TOOLS);
+}
+function mergedReason(session) {
+    return 'explicit merge signal present and session completed';
+}
+/**
+ * Tests-green when a test/build/lint step returned ok, or when the session
+ * completed cleanly with test steps whose individual outcomes are unknown
+ * (e.g. Grok-derived sessions that record spanMs=0 and never pair results).
+ */
+function isTestsGreen(session) {
+    if (session.meta.outcome !== 'completed')
+        return false;
+    const testSteps = session.steps.filter((s) => {
+        if (!isShellStep(s))
+            return false;
+        const text = stepText(s);
+        return OUTCOME_SIGNALS.some((sig) => sig.type === 'test' && sig.pattern.test(text));
+    });
+    if (testSteps.length === 0)
+        return false;
+    const lastTest = testSteps[testSteps.length - 1];
+    if (lastTest.outcome === 'ok')
+        return true;
+    // Infer from a clean completion when step-level outcomes are not available.
+    return session.meta.errorCount === 0;
+}
+function testsGreenConfidence(session) {
+    const testSteps = session.steps.filter((s) => {
+        if (!isShellStep(s))
+            return false;
+        const text = stepText(s);
+        return OUTCOME_SIGNALS.some((sig) => sig.type === 'test' && sig.pattern.test(text));
+    });
+    const lastTest = testSteps[testSteps.length - 1];
+    return lastTest?.outcome === 'ok' ? 'high' : 'medium';
+}
+function testsGreenReason(session) {
+    const confidence = testsGreenConfidence(session);
+    return confidence === 'high'
+        ? 'last test/build/lint step returned ok'
+        : 'completed cleanly with test steps (step outcomes unavailable)';
+}
+/** Human-takeover when the final tool action asks the human. */
+function isHumanTakeover(session) {
+    const tools = session.steps.filter((s) => isToolStep(s));
+    if (tools.length === 0)
+        return false;
+    const last = tools[tools.length - 1];
+    return HUMAN_FACING_TOOLS.has(last.tool ?? last.lane);
+}
+function humanTakeoverReason(session) {
+    const tools = session.steps.filter((s) => isToolStep(s));
+    const last = tools[tools.length - 1];
+    return `final tool step is ${last.tool ?? last.lane}`;
+}
+/**
+ * Invalid-env when setup/environment failures dominate the session.
+ *
+ * Rubric:
+ * - ≥2 env errors and they make up at least half of all errors, or
+ * - ≥3 env steps with ≥50% error rate in an errored session, or
+ * - the first substantive error is an env/setup error and no recovery follows it.
+ */
+function isInvalidEnv(session) {
+    const envSteps = session.steps.filter((s) => {
+        const text = stepText(s);
+        return OUTCOME_SIGNALS.some((sig) => sig.type === 'env' && sig.pattern.test(text));
+    });
+    const envErrors = envSteps.filter((s) => s.outcome === 'error').length;
+    const totalErrors = session.meta.errorCount;
+    if (totalErrors > 0 && envErrors >= 2 && envErrors / totalErrors >= 0.5)
+        return true;
+    if (session.meta.outcome === 'errored' && envSteps.length >= 3 && envErrors / envSteps.length >= 0.5)
+        return true;
+    const substantive = substantiveSteps(session);
+    const firstError = substantive.find((s) => s.outcome === 'error');
+    if (session.meta.outcome === 'errored' &&
+        firstError &&
+        OUTCOME_SIGNALS.some((sig) => sig.type === 'env' && sig.pattern.test(stepText(firstError)))) {
+        const firstErrorIndex = substantive.indexOf(firstError);
+        const after = substantive.slice(firstErrorIndex + 1);
+        const recovered = after.some((s) => s.outcome === 'ok' && !HUMAN_FACING_TOOLS.has(s.tool ?? s.lane));
+        if (!recovered)
+            return true;
+    }
+    return false;
+}
+function invalidEnvReason(session) {
+    const envSteps = session.steps.filter((s) => {
+        const text = stepText(s);
+        return OUTCOME_SIGNALS.some((sig) => sig.type === 'env' && sig.pattern.test(text));
+    });
+    const envErrors = envSteps.filter((s) => s.outcome === 'error').length;
+    const totalErrors = session.meta.errorCount;
+    if (totalErrors > 0 && envErrors >= 2 && envErrors / totalErrors >= 0.5) {
+        return `${envErrors}/${totalErrors} errors are environment/setup related`;
+    }
+    return 'environment/setup failures dominated the early session and blocked recovery';
+}
+/** Abandoned when the session errored and stalled without recovery. */
+function isAbandoned(session) {
+    if (session.meta.outcome !== 'errored')
+        return false;
+    const hasStall = session.gaps.some((g) => g.durationMs >= 120_000);
+    const substantive = substantiveSteps(session);
+    if (substantive.length === 0)
+        return hasStall || session.meta.errorCount > 0;
+    const lastErrorOrdinal = lastStepOrdinalOf(session, (s) => s.outcome === 'error');
+    if (lastErrorOrdinal === undefined)
+        return false;
+    const recoveryAfter = substantive.some((s) => s.ordinal > lastErrorOrdinal && s.outcome === 'ok' && !HUMAN_FACING_TOOLS.has(s.tool ?? s.lane));
+    return !recoveryAfter || hasStall;
+}
+function abandonedReason(session) {
+    const hasStall = session.gaps.some((g) => g.durationMs >= 120_000);
+    return hasStall ? 'errored with a long stall and no recovery' : 'errored with no successful recovery';
+}
+/** Partial: progress was made but no landing or test signal is present. */
+function isPartial(_session) {
+    return true;
+}
+function partialReason(session) {
+    const counts = signalCounts(session);
+    if (session.meta.outcome === 'completed') {
+        if (counts.test > 0)
+            return 'tests ran but final test step did not return ok';
+        if (counts.merge > 0)
+            return 'merge-related activity but no confirmed merge';
+        return 'completed without a landing or test signal';
+    }
+    return 'incomplete work with no higher-confidence outcome signal';
+}
+function ruleConfidence(rule, session) {
+    return typeof rule.confidence === 'function' ? rule.confidence(session) : rule.confidence;
+}
+/**
+ * Ordered rubric: the first matching outcome wins. `partial` is the
+ * lower-confidence default when no high/medium-confidence signal is present.
+ */
+const OUTCOME_RULES = [
+    { key: 'merged', confidence: 'high', score: isMerged, reason: mergedReason },
+    { key: 'tests-green', confidence: testsGreenConfidence, score: isTestsGreen, reason: testsGreenReason },
+    { key: 'human-takeover', confidence: 'medium', score: isHumanTakeover, reason: humanTakeoverReason },
+    { key: 'invalid-env', confidence: 'medium', score: isInvalidEnv, reason: invalidEnvReason },
+    { key: 'abandoned', confidence: 'medium', score: isAbandoned, reason: abandonedReason },
+    { key: 'partial', confidence: 'low', score: isPartial, reason: partialReason },
+];
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Classify the failure phenotype of a session from its derived trajectory.
+ *
+ * Definitions (from agent-failure research):
+ * - `false-termination`  — stopped with an unresolved error.
+ * - `premature-completion` — declared done while tests were failing or no
+ *   verification step ran for the engineering work.
+ * - `out-of-order` — a write/edit step occurred before any read/plan of the
+ *   target.
+ * - `failure-to-act` — stalled or produced no meaningful tool use.
+ *
+ * Returns `null` when none of the failure phenotypes apply.
+ */
+export function classifyPhenotype(session) {
+    return classifyPhenotypeDetailed(session).phenotype;
+}
+/** Detailed phenotype result with a human-readable reason. */
+export function classifyPhenotypeDetailed(session) {
+    for (const rule of PHENOTYPE_RULES) {
+        if (rule.score(session)) {
+            return { phenotype: rule.key, reason: rule.reason(session) };
+        }
+    }
+    return { phenotype: null, reason: 'no failure phenotype matched' };
+}
+/**
+ * Derive the coarse outcome of a session from its end state + tool signals.
+ *
+ * `merged` and `tests-green` are high-confidence only when an explicit signal
+ * is present in the derived steps. When that signal is genuinely not in the
+ * data, the function returns the honest lower-confidence value (`partial` for
+ * completed work without a landing signal, `abandoned` for errored/unresolved
+ * work, `invalid-env` for setup-dominant failures, `human-takeover` when the
+ * session ends on a human-facing ask).
+ */
+export function deriveOutcome(session) {
+    return deriveOutcomeDetailed(session).outcome;
+}
+/** Detailed outcome result with confidence and a human-readable reason. */
+export function deriveOutcomeDetailed(session) {
+    for (const rule of OUTCOME_RULES) {
+        if (rule.score(session)) {
+            return { outcome: rule.key, confidence: ruleConfidence(rule, session), reason: rule.reason(session) };
+        }
+    }
+    // `partial` is the exhaustive default; this line is unreachable but keeps TS happy.
+    return { outcome: 'partial', confidence: 'low', reason: 'no outcome signal matched' };
+}

package/dist/lib/traces/segments.d.ts

@@ -0,0 +1,133 @@
+/**
+ * Group-by dimensions + time-to-first-tool latency for the traces insight
+ * engine (the console's Issue bar).
+ *
+ * Pure functions, no I/O. Classifiers are DATA-DRIVEN TABLES — a new task type
+ * or timing bucket is a row, not a new if/else-by-name arm. The integrator
+ * (`insights.ts` / `sync.ts`) tags each session; this file does not write the
+ * shard.
+ *
+ * Input is the redacted `SessionDetail` from `agents traces sync` plus the
+ * SyncRow fields the detail strips (prompt, gitBranch, files). Structural —
+ * `SessionDetail` is a valid `SegmentSession`.
+ */
+export declare const TASK_TYPES: readonly ["bugfix", "feature", "refactor", "test", "chore", "other"];
+export type TaskType = (typeof TASK_TYPES)[number];
+export declare const FAILURE_TIMINGS: readonly ["early", "mid", "late"];
+export type FailureTiming = (typeof FAILURE_TIMINGS)[number];
+/** The #1 group-by axis: compare the pair, never the model alone. */
+export interface SegmentAgent {
+    model: string;
+    harness: string;
+}
+export interface SegmentFile {
+    path: string;
+    /** Write/create vs Edit/patch. */
+    action: 'new' | 'edit';
+}
+export interface SegmentStep {
+    startMs: number;
+    durationMs?: number;
+    outcome?: 'ok' | 'error' | 'unknown' | string;
+    tool?: string;
+    kind?: 'tool' | 'thinking' | string;
+    label?: string;
+}
+/**
+ * Session shape the classifiers read. `SessionDetail` (sessions/<id>.json)
+ * satisfies this; callers may also pass SyncRow fields (`prompt`, `gitBranch`,
+ * `agent`, `model`, `files`) alongside steps.
+ */
+export interface SegmentSession {
+    id?: string;
+    prompt?: string | null;
+    gitBranch?: string | null;
+    topic?: string | null;
+    label?: string | null;
+    mode?: string | null;
+    agent?: string | null;
+    harness?: string | null;
+    model?: string | null;
+    spanMs?: number | null;
+    files?: readonly SegmentFile[] | null;
+    meta?: {
+        agent?: string | null;
+        model?: string | null;
+        spanMs?: number | null;
+        repo?: string | null;
+        tools?: number | null;
+    } | null;
+    steps?: readonly SegmentStep[] | null;
+    whereItWentWrong?: string | null;
+}
+export interface Percentiles {
+    p50: number;
+    p90: number;
+    p99: number;
+    max: number;
+}
+/** Time-to-first-tool latency — `steps[0].startMs` over tool-using sessions. */
+export interface LatencyInsight {
+    firstToolMs: Percentiles;
+}
+export interface SegmentDimensions {
+    agent: SegmentAgent;
+    taskType: TaskType;
+    failureTiming: FailureTiming | null;
+}
+/**
+ * Tool name → file action. Lowercased. Write-family is a new file; Edit-family
+ * is a patch. Anything else is ignored for diff-shape.
+ */
+export declare const FILE_ACTION_TOOLS: ReadonlyArray<{
+    action: SegmentFile['action'];
+    tools: readonly string[];
+}>;
+type DiffShape = {
+    files: readonly SegmentFile[];
+    testOnly: boolean;
+    newOnly: boolean;
+    choreOnly: boolean;
+};
+/**
+ * Task-type rules, first match wins. Prompt/branch keywords outrank diff shape
+ * except where `shape` is set (test-only files, chore-only files, all-new files).
+ *
+ * Order is the product priority: test-only work is not a "feature"; a `fix/`
+ * branch is a bugfix even if files are new; chore lockfile edits are not features.
+ */
+export declare const TASK_TYPE_RULES: ReadonlyArray<{
+    type: Exclude<TaskType, 'other'>;
+    patterns: readonly RegExp[];
+    shape?: keyof Pick<DiffShape, 'testOnly' | 'newOnly' | 'choreOnly'>;
+}>;
+/**
+ * Normalized position of the first failing step inside the session span.
+ * Early failures cascade — highest-signal bucket. Exclusive upper bound.
+ */
+export declare const FAILURE_TIMING_BUCKETS: ReadonlyArray<{
+    timing: FailureTiming;
+    maxExclusive: number;
+}>;
+/** Model × harness unit — the #1 group-by axis. */
+export declare function deriveAgent(session: SegmentSession): SegmentAgent;
+/**
+ * Classify the session's work from the opening prompt + diff shape (files
+ * touched, new vs edit, test-only). First matching table row wins.
+ */
+export declare function classifyTaskType(session: SegmentSession): TaskType;
+/**
+ * Bucket the FIRST failing step by its normalized position in the session
+ * span (`firstFailure.startMs / spanMs`). Sessions with no failing step
+ * return null — they do not belong on the failure-timing axis.
+ */
+export declare function failureTiming(session: SegmentSession): FailureTiming | null;
+/**
+ * Time-to-first-tool latency from `steps[0].startMs` across sessions that
+ * recorded at least one step. Nearest-rank percentiles (same formula as the
+ * traces index stats) so p99 on `/tmp/traces-real` lands at ~128s.
+ */
+export declare function computeLatency(sessions: readonly SegmentSession[]): LatencyInsight;
+/** All three group-by dimensions for one session. */
+export declare function deriveDimensions(session: SegmentSession): SegmentDimensions;
+export {};