@yagni-app/code-staging 1.1.0-staging.1328.1 → 1.1.0-staging.1329.1

package/dist/extension/index.js

@@ -49,6 +49,7 @@ import { makeDecisionCapture } from "./decisionCapture.js";
 import { registerAmbientRecall } from "./recall.js";
 import { resilientFetch } from "./resilientFetch.js";
 import { installUncaughtExceptionMonitor, makeCrashReporter, runningUnderTest } from "./crashReport.js";
+import { createToolOutcomeBatcher } from "./toolOutcomes.js";
 import { flushSpool as defaultFlushSpool } from "./spool.js";
 import { makeAuthedFetch, makeTokenProvider } from "./tokenProvider.js";
 import { attributionHeaders, fetchCatalog as defaultFetchCatalog, fetchContextBrief as defaultFetchContextBrief, getToken, getTokenExpiresAt as defaultGetTokenExpiresAt, getWorkspaceId as defaultGetWorkspaceId, isDriverCaller, resolveBaseUrl, tokenExpiryNotice, } from "./config.js";
@@ -430,6 +431,30 @@ export async function registerYagni(pi, deps = {}) {
             previousMode = m;
         });
     }
+    // Tool-outcome counts for the fleet dashboards (toolOutcomes.ts): which
+    // tool failures are the model's normal business and which are the tool
+    // machinery breaking. Counts and a closed reason vocabulary only; off in
+    // eval mode and under the crash-report opt-out; posted on a timer and at
+    // session shutdown, fail-soft throughout.
+    const toolOutcomes = createToolOutcomeBatcher({
+        baseUrl,
+        getToken: getTokenFn,
+        headers: attributionHeaders(deps.env),
+        fetchImpl: deps.fetchImpl,
+        env: deps.env,
+        enabled: !evalMode,
+    });
+    pi.on("tool_execution_start", (event) => {
+        if (event?.toolCallId && event?.toolName)
+            toolOutcomes.toolStart(event.toolCallId, event.toolName);
+    });
+    pi.on("tool_execution_end", (event) => {
+        if (event?.toolCallId)
+            toolOutcomes.toolEnd(event.toolCallId, { isError: !!event.isError, result: event.result });
+    });
+    pi.on("session_shutdown", async () => {
+        await toolOutcomes.close();
+    });
     const footerInvalidateHandle = { invalidateGit: () => { }, requestRender: () => { } };
     const guardianState = makeGuardianState();
     // Disabled by the local env override OR the workspace kill switch

package/dist/extension/toolOutcomes.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Tool-call outcome telemetry: counts per (tool family, outcome, reason),
+ * batched and posted to the backend's /api/yagni-code/tool-outcomes so the
+ * fleet dashboards can tell a NORMAL tool failure (the model ran a command
+ * that exited 1, read a file that is not there, tried an edit that did not
+ * match) from the tool machinery actually breaking (an MCP transport, an
+ * internal exception, a timeout).
+ *
+ * Content never leaves the machine: the classifier reads the result text
+ * locally and emits only a closed reason vocabulary, and tool names collapse
+ * to a closed family list HERE, before buffering, so an MCP server name never
+ * reaches the wire. The payload is family, outcome, reason, count, summed
+ * duration. Opt-out and test suppression follow the crash reporter
+ * (`YAGNI_DISABLE_CRASH_REPORTS=1` turns both off), eval mode is off like
+ * every other external side effect, and everything is fail-soft: one
+ * attempt, short timeout, never throws, never blocks a turn. A failed post is
+ * counted and written to the local error trail (source `telemetry`) so "why
+ * is the dashboard empty" has something to read.
+ */
+export type ToolOutcome = "ok" | "expected_error" | "real_error";
+export type ToolOutcomeReason = "ok" | "exit_nonzero" | "not_found" | "no_match" | "denied" | "cancelled" | "invalid_input" | "timeout" | "mcp_transport" | "internal" | "unknown";
+export declare const TOOL_FAMILIES: readonly ["bash", "read", "edit", "write", "grep", "find", "ls", "mcp", "subagent", "web", "other"];
+export type ToolFamily = (typeof TOOL_FAMILIES)[number];
+/** Collapse a tool name to its family. Never returns anything outside TOOL_FAMILIES. */
+export declare function toolFamilyOf(toolName: string): ToolFamily;
+export interface ToolOutcomeClass {
+    outcome: ToolOutcome;
+    reason: ToolOutcomeReason;
+}
+/** Best-effort text from a pi tool result (string, content blocks, or an Error). */
+export declare function toolResultText(result: unknown): string;
+/**
+ * Classify one tool call. The order matters: transport and internal faults
+ * win over the softer patterns because a stack trace can mention a file.
+ */
+export declare function classifyToolOutcome(input: {
+    toolName: string;
+    isError: boolean;
+    result?: unknown;
+}): ToolOutcomeClass;
+export interface ToolOutcomeSample {
+    /** A TOOL_FAMILIES value, never the raw tool name. */
+    tool: ToolFamily;
+    outcome: ToolOutcome;
+    reason: ToolOutcomeReason;
+    count: number;
+    durationMsSum: number;
+}
+export interface ToolOutcomeBatcherOpts {
+    baseUrl: string;
+    getToken: () => string | undefined;
+    /** Attribution headers (`x-yagni-caller` / session / run) from config.ts. */
+    headers?: Record<string, string>;
+    fetchImpl?: typeof fetch;
+    env?: NodeJS.ProcessEnv;
+    /** Flush cadence; the interval timer is unref'd so it never holds the process. */
+    flushIntervalMs?: number;
+    timeoutMs?: number;
+    now?: () => number;
+    /** Disabled entirely (eval mode, tests). */
+    enabled?: boolean;
+    /** Local trail sink for a failed post (defaults to the unified error sink). */
+    logSink?: (event: {
+        event: string;
+        fields: Record<string, unknown>;
+    }) => void;
+}
+export interface ToolOutcomeBatcher {
+    toolStart(toolCallId: string, toolName: string): void;
+    toolEnd(toolCallId: string, outcome: {
+        isError: boolean;
+        result?: unknown;
+    }): void;
+    /** Post whatever is buffered. Resolves on every outcome; never throws. */
+    flush(): Promise<void>;
+    /** Stop the timer and flush once. */
+    close(): Promise<void>;
+    /** Test/introspection seam: the buffered samples. */
+    pending(): ToolOutcomeSample[];
+    /** Batches that failed to post (network, non-2xx, no token) since start. */
+    dropped(): number;
+}
+export declare const TOOL_OUTCOME_FLUSH_INTERVAL_MS = 60000;
+export declare const TOOL_OUTCOME_TIMEOUT_MS = 2000;
+export declare function createToolOutcomeBatcher(opts: ToolOutcomeBatcherOpts): ToolOutcomeBatcher;
+//# sourceMappingURL=toolOutcomes.d.ts.map

package/dist/extension/toolOutcomes.js

@@ -0,0 +1,240 @@
+/**
+ * Tool-call outcome telemetry: counts per (tool family, outcome, reason),
+ * batched and posted to the backend's /api/yagni-code/tool-outcomes so the
+ * fleet dashboards can tell a NORMAL tool failure (the model ran a command
+ * that exited 1, read a file that is not there, tried an edit that did not
+ * match) from the tool machinery actually breaking (an MCP transport, an
+ * internal exception, a timeout).
+ *
+ * Content never leaves the machine: the classifier reads the result text
+ * locally and emits only a closed reason vocabulary, and tool names collapse
+ * to a closed family list HERE, before buffering, so an MCP server name never
+ * reaches the wire. The payload is family, outcome, reason, count, summed
+ * duration. Opt-out and test suppression follow the crash reporter
+ * (`YAGNI_DISABLE_CRASH_REPORTS=1` turns both off), eval mode is off like
+ * every other external side effect, and everything is fail-soft: one
+ * attempt, short timeout, never throws, never blocks a turn. A failed post is
+ * counted and written to the local error trail (source `telemetry`) so "why
+ * is the dashboard empty" has something to read.
+ */
+import { crashReportsSuppressed } from "./crashReport.js";
+import { logEvent } from "./errorSink.js";
+import { isDesktopSurface } from "./surface.js";
+export const TOOL_FAMILIES = ["bash", "read", "edit", "write", "grep", "find", "ls", "mcp", "subagent", "web", "other"];
+/** Collapse a tool name to its family. Never returns anything outside TOOL_FAMILIES. */
+export function toolFamilyOf(toolName) {
+    const name = toolName.toLowerCase();
+    if (name.startsWith("mcp__") || name.startsWith("mcp:"))
+        return "mcp";
+    if (name === "bash" || name === "shell" || name === "exec")
+        return "bash";
+    if (name === "read" || name === "read_file" || name === "view")
+        return "read";
+    if (name === "edit" || name === "multiedit" || name === "str_replace")
+        return "edit";
+    if (name === "write" || name === "write_file" || name === "create")
+        return "write";
+    if (name === "grep" || name === "search")
+        return "grep";
+    if (name === "find" || name === "glob")
+        return "find";
+    if (name === "ls" || name === "list")
+        return "ls";
+    if (name.includes("subagent") || name === "agent" || name === "task")
+        return "subagent";
+    if (name.startsWith("web") || name.includes("fetch") || name.includes("browser"))
+        return "web";
+    return "other";
+}
+/** Only this much of a result is inspected; the classifier is pattern-based. */
+const CLASSIFY_TEXT_CAP = 4_000;
+/** Best-effort text from a pi tool result (string, content blocks, or an Error). */
+export function toolResultText(result) {
+    if (typeof result === "string")
+        return result.slice(0, CLASSIFY_TEXT_CAP);
+    if (result instanceof Error)
+        return `${result.name}: ${result.message}`.slice(0, CLASSIFY_TEXT_CAP);
+    if (typeof result !== "object" || result === null)
+        return "";
+    const r = result;
+    if (typeof r.text === "string")
+        return r.text.slice(0, CLASSIFY_TEXT_CAP);
+    if (typeof r.error === "string")
+        return r.error.slice(0, CLASSIFY_TEXT_CAP);
+    if (typeof r.message === "string")
+        return r.message.slice(0, CLASSIFY_TEXT_CAP);
+    if (Array.isArray(r.content)) {
+        const parts = [];
+        let size = 0;
+        for (const block of r.content) {
+            const text = typeof block === "string" ? block : block?.text;
+            if (typeof text !== "string")
+                continue;
+            parts.push(text);
+            size += text.length;
+            if (size >= CLASSIFY_TEXT_CAP)
+                break;
+        }
+        return parts.join("\n").slice(0, CLASSIFY_TEXT_CAP);
+    }
+    return "";
+}
+const NOT_FOUND_RE = /no such file|not found|does not exist|enoent|cannot find|unknown file|no matches? found/i;
+const NO_MATCH_RE = /old_string|did not match|not unique|could not find the (?:string|text)|no occurrences|nothing to replace/i;
+const DENIED_RE = /permission denied|denied by|blocked by|not allowed|not permitted|refused|guardian|plan mode|requires approval|eacces|eperm/i;
+const CANCELLED_RE = /cancel+ed|aborted|interrupted|user (?:declined|rejected|stopped)/i;
+const INVALID_RE = /invalid (?:argument|input|json|parameter)|missing required|expected .* to be|is required|malformed|validation/i;
+const TIMEOUT_RE = /timed? ?out|deadline exceeded|etimedout/i;
+const MCP_RE = /mcp|transport|econnrefused|econnreset|socket hang up|server (?:closed|disconnected|unavailable)|jsonrpc|connection (?:closed|lost|refused)/i;
+const INTERNAL_RE = /^(?:type|reference|range|syntax)error\b|internal error|unhandled|stack trace|cannot read propert|is not a function|undefined is not/i;
+const EXIT_RE = /exit(?:ed)? (?:with )?(?:code|status)[: ]+(\d+)|command failed|non-zero exit|\bexit code\b/i;
+/**
+ * Classify one tool call. The order matters: transport and internal faults
+ * win over the softer patterns because a stack trace can mention a file.
+ */
+export function classifyToolOutcome(input) {
+    if (!input.isError)
+        return { outcome: "ok", reason: "ok" };
+    const text = toolResultText(input.result);
+    // Accepts a raw tool name or an already-collapsed family (the batcher
+    // classifies by family).
+    const mcp = input.toolName === "mcp" || toolFamilyOf(input.toolName) === "mcp";
+    if (INTERNAL_RE.test(text))
+        return { outcome: "real_error", reason: "internal" };
+    if (mcp && MCP_RE.test(text))
+        return { outcome: "real_error", reason: "mcp_transport" };
+    if (TIMEOUT_RE.test(text))
+        return { outcome: "real_error", reason: "timeout" };
+    if (CANCELLED_RE.test(text))
+        return { outcome: "expected_error", reason: "cancelled" };
+    if (DENIED_RE.test(text))
+        return { outcome: "expected_error", reason: "denied" };
+    if (NO_MATCH_RE.test(text))
+        return { outcome: "expected_error", reason: "no_match" };
+    if (NOT_FOUND_RE.test(text))
+        return { outcome: "expected_error", reason: "not_found" };
+    if (INVALID_RE.test(text))
+        return { outcome: "expected_error", reason: "invalid_input" };
+    if (EXIT_RE.test(text) || input.toolName === "bash")
+        return { outcome: "expected_error", reason: "exit_nonzero" };
+    if (!mcp && MCP_RE.test(text))
+        return { outcome: "real_error", reason: "mcp_transport" };
+    return { outcome: "real_error", reason: "unknown" };
+}
+export const TOOL_OUTCOME_FLUSH_INTERVAL_MS = 60_000;
+export const TOOL_OUTCOME_TIMEOUT_MS = 2_000;
+const MAX_TRACKED_STARTS = 512;
+export function createToolOutcomeBatcher(opts) {
+    const env = opts.env ?? process.env;
+    const now = opts.now ?? Date.now;
+    const enabled = (opts.enabled ?? true) && !crashReportsSuppressed(env);
+    const logSink = opts.logSink ??
+        ((e) => logEvent({ source: "telemetry", level: "warn", event: e.event, fields: e.fields, sessionId: env.YAGNI_SESSION_ID }));
+    const starts = new Map();
+    const buffer = new Map();
+    let timer;
+    let closed = false;
+    let droppedBatches = 0;
+    const key = (tool, outcome, reason) => `${tool}|${outcome}|${reason}`;
+    const record = (tool, cls, durationMs) => {
+        const k = key(tool, cls.outcome, cls.reason);
+        const existing = buffer.get(k);
+        if (existing) {
+            existing.count += 1;
+            existing.durationMsSum += durationMs;
+            return;
+        }
+        buffer.set(k, { tool, outcome: cls.outcome, reason: cls.reason, count: 1, durationMsSum: durationMs });
+    };
+    const drop = (samples, fields) => {
+        droppedBatches += 1;
+        try {
+            logSink({ event: "tool_outcomes_post_failed", fields: { ...fields, samples: samples.length, droppedBatches } });
+        }
+        catch {
+            // the trail is best-effort
+        }
+    };
+    const flush = async () => {
+        if (!enabled || buffer.size === 0)
+            return;
+        const samples = [...buffer.values()];
+        buffer.clear();
+        try {
+            // The token getter is fail-soft too: a throwing provider must not
+            // reject the timer's `void flush()` or surface at shutdown.
+            const token = opts.getToken();
+            if (!token) {
+                drop(samples, { kind: "no_token" });
+                return;
+            }
+            const fetchImpl = opts.fetchImpl ?? fetch;
+            const controller = new AbortController();
+            const t = setTimeout(() => controller.abort(), opts.timeoutMs ?? TOOL_OUTCOME_TIMEOUT_MS);
+            t.unref?.();
+            try {
+                const res = await fetchImpl(`${opts.baseUrl.replace(/\/$/, "")}/api/yagni-code/tool-outcomes`, {
+                    method: "POST",
+                    headers: {
+                        "content-type": "application/json",
+                        authorization: `Bearer ${token}`,
+                        ...(opts.headers ?? {}),
+                    },
+                    body: JSON.stringify({
+                        client: isDesktopSurface() ? "desktop" : "cli",
+                        clientVersion: env.YAGNI_CODE_VERSION?.trim() || "unknown",
+                        samples,
+                    }),
+                    signal: controller.signal,
+                });
+                if (!res.ok)
+                    drop(samples, { kind: "http", status: res.status });
+            }
+            finally {
+                clearTimeout(t);
+            }
+        }
+        catch (err) {
+            drop(samples, { kind: "network", error: err instanceof Error ? err.name : "unknown" });
+        }
+    };
+    if (enabled) {
+        timer = setInterval(() => { void flush(); }, opts.flushIntervalMs ?? TOOL_OUTCOME_FLUSH_INTERVAL_MS);
+        timer.unref?.();
+    }
+    return {
+        toolStart(toolCallId, toolName) {
+            if (!enabled || closed)
+                return;
+            if (starts.size >= MAX_TRACKED_STARTS)
+                starts.clear();
+            starts.set(toolCallId, { family: toolFamilyOf(toolName), startedAt: now() });
+        },
+        toolEnd(toolCallId, outcome) {
+            if (!enabled || closed)
+                return;
+            const slot = starts.get(toolCallId);
+            starts.delete(toolCallId);
+            const family = slot?.family ?? "other";
+            const durationMs = slot ? Math.max(0, now() - slot.startedAt) : 0;
+            try {
+                record(family, classifyToolOutcome({ toolName: family, isError: outcome.isError, result: outcome.result }), durationMs);
+            }
+            catch {
+                // classification must never break a tool result
+            }
+        },
+        flush,
+        async close() {
+            if (closed)
+                return;
+            closed = true;
+            if (timer)
+                clearInterval(timer);
+            await flush();
+        },
+        pending: () => [...buffer.values()],
+        dropped: () => droppedBatches,
+    };
+}
+//# sourceMappingURL=toolOutcomes.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yagni-app/code-staging",
-  "version": "1.1.0-staging.1328.1",
+  "version": "1.1.0-staging.1329.1",
   "description": "YAGNI Code: a terminal coding agent that already knows your company. One YAGNI login routes the model and grounds the agent in your team's context.",
   "license": "SEE LICENSE IN LICENSE.md",
   "author": "YAGNI, Inc. <jack@yagni.app> (https://yagni.app)",
@@ -58,5 +58,5 @@
     "turndown": "^7.2.4",
     "typebox": "^1.3.15"
   },
-  "yagniSourceSha": "34b64c98d42aa85f408a4f85de7ef103722038f7"
+  "yagniSourceSha": "36a0344047224a979e0714451f6e710658d1454b"
 }