@yagni-app/code-staging 1.1.0-staging.1327.1 → 1.1.0-staging.1329.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/extension/index.js
CHANGED
|
@@ -49,6 +49,7 @@ import { makeDecisionCapture } from "./decisionCapture.js";
|
|
|
49
49
|
import { registerAmbientRecall } from "./recall.js";
|
|
50
50
|
import { resilientFetch } from "./resilientFetch.js";
|
|
51
51
|
import { installUncaughtExceptionMonitor, makeCrashReporter, runningUnderTest } from "./crashReport.js";
|
|
52
|
+
import { createToolOutcomeBatcher } from "./toolOutcomes.js";
|
|
52
53
|
import { flushSpool as defaultFlushSpool } from "./spool.js";
|
|
53
54
|
import { makeAuthedFetch, makeTokenProvider } from "./tokenProvider.js";
|
|
54
55
|
import { attributionHeaders, fetchCatalog as defaultFetchCatalog, fetchContextBrief as defaultFetchContextBrief, getToken, getTokenExpiresAt as defaultGetTokenExpiresAt, getWorkspaceId as defaultGetWorkspaceId, isDriverCaller, resolveBaseUrl, tokenExpiryNotice, } from "./config.js";
|
|
@@ -430,6 +431,30 @@ export async function registerYagni(pi, deps = {}) {
|
|
|
430
431
|
previousMode = m;
|
|
431
432
|
});
|
|
432
433
|
}
|
|
434
|
+
// Tool-outcome counts for the fleet dashboards (toolOutcomes.ts): which
|
|
435
|
+
// tool failures are the model's normal business and which are the tool
|
|
436
|
+
// machinery breaking. Counts and a closed reason vocabulary only; off in
|
|
437
|
+
// eval mode and under the crash-report opt-out; posted on a timer and at
|
|
438
|
+
// session shutdown, fail-soft throughout.
|
|
439
|
+
const toolOutcomes = createToolOutcomeBatcher({
|
|
440
|
+
baseUrl,
|
|
441
|
+
getToken: getTokenFn,
|
|
442
|
+
headers: attributionHeaders(deps.env),
|
|
443
|
+
fetchImpl: deps.fetchImpl,
|
|
444
|
+
env: deps.env,
|
|
445
|
+
enabled: !evalMode,
|
|
446
|
+
});
|
|
447
|
+
pi.on("tool_execution_start", (event) => {
|
|
448
|
+
if (event?.toolCallId && event?.toolName)
|
|
449
|
+
toolOutcomes.toolStart(event.toolCallId, event.toolName);
|
|
450
|
+
});
|
|
451
|
+
pi.on("tool_execution_end", (event) => {
|
|
452
|
+
if (event?.toolCallId)
|
|
453
|
+
toolOutcomes.toolEnd(event.toolCallId, { isError: !!event.isError, result: event.result });
|
|
454
|
+
});
|
|
455
|
+
pi.on("session_shutdown", async () => {
|
|
456
|
+
await toolOutcomes.close();
|
|
457
|
+
});
|
|
433
458
|
const footerInvalidateHandle = { invalidateGit: () => { }, requestRender: () => { } };
|
|
434
459
|
const guardianState = makeGuardianState();
|
|
435
460
|
// Disabled by the local env override OR the workspace kill switch
|
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Pure: no I/O, no network, no model. Loads at startup and classifies
|
|
6
6
|
* synchronously. The curated default set auto-allows read-only commands
|
|
7
|
-
* (ls, cat, rg, git status/log/diff), forbids destructive ones (rm
|
|
8
|
-
* git reset --hard, git push --force, pipe-to-shell
|
|
9
|
-
* ambiguous middle band (
|
|
7
|
+
* (ls, cat, rg, git status/log/diff), forbids destructive ones (recursive rm,
|
|
8
|
+
* git reset --hard, git push --force, pipe-to-shell, bare-interpreter
|
|
9
|
+
* pipes), and prompts for the ambiguous middle band (non-recursive rm -f,
|
|
10
|
+
* inline-code interpreter pipes, npm install, git commit, curl, …).
|
|
10
11
|
*
|
|
11
12
|
* The `prompt` band is what the Guardian arbitrates — see guardian.ts.
|
|
12
13
|
*
|
|
@@ -126,8 +127,10 @@ export declare function tokenize(command: string): string[];
|
|
|
126
127
|
* each is classified independently; the strictest decision wins (forbidden >
|
|
127
128
|
* prompt > allow). Commands with shell constructs (substitution, redirects,
|
|
128
129
|
* background &) have a floor of `prompt`, and their substitution inner text
|
|
129
|
-
* is danger-scanned against the forbidden rules.
|
|
130
|
-
*
|
|
130
|
+
* is danger-scanned against the forbidden rules. Pipes into shells and
|
|
131
|
+
* network relays, or into a bare interpreter (stdin executed as the program),
|
|
132
|
+
* are always forbidden; an interpreter carrying inline code (-c/-e) falls to
|
|
133
|
+
* the prompt band — the code rides the command string the Guardian can read.
|
|
131
134
|
*/
|
|
132
135
|
export declare function classifyCommand(command: string, policy: ExecPolicy): ExecClassification;
|
|
133
136
|
/** Curated default rules — the shipped safety floor. */
|
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Pure: no I/O, no network, no model. Loads at startup and classifies
|
|
6
6
|
* synchronously. The curated default set auto-allows read-only commands
|
|
7
|
-
* (ls, cat, rg, git status/log/diff), forbids destructive ones (rm
|
|
8
|
-
* git reset --hard, git push --force, pipe-to-shell
|
|
9
|
-
* ambiguous middle band (
|
|
7
|
+
* (ls, cat, rg, git status/log/diff), forbids destructive ones (recursive rm,
|
|
8
|
+
* git reset --hard, git push --force, pipe-to-shell, bare-interpreter
|
|
9
|
+
* pipes), and prompts for the ambiguous middle band (non-recursive rm -f,
|
|
10
|
+
* inline-code interpreter pipes, npm install, git commit, curl, …).
|
|
10
11
|
*
|
|
11
12
|
* The `prompt` band is what the Guardian arbitrates — see guardian.ts.
|
|
12
13
|
*
|
|
@@ -391,11 +392,28 @@ export function extractSubstitutions(command) {
|
|
|
391
392
|
}
|
|
392
393
|
return found;
|
|
393
394
|
}
|
|
394
|
-
/**
|
|
395
|
+
/**
|
|
396
|
+
* Words that, when piped into, are always forbidden: shells (execute stdin)
|
|
397
|
+
* and network relays (remote-shell class). Interpreters are separate — see
|
|
398
|
+
* INTERPRETER_INLINE_FLAGS — because with an inline-code flag (-c/-e) the
|
|
399
|
+
* piped data is just input to a script the Guardian can read on the command
|
|
400
|
+
* line; without one, stdin IS the program (download-and-execute).
|
|
401
|
+
*/
|
|
395
402
|
const PIPE_TO_SHELL = new Set([
|
|
396
|
-
"sh", "bash", "zsh", "fish", "nc", "ncat", "socat",
|
|
403
|
+
"sh", "bash", "zsh", "fish", "dash", "ksh", "nc", "ncat", "socat",
|
|
404
|
+
]);
|
|
405
|
+
/** Interpreters that execute piped stdin as a program unless given inline code. */
|
|
406
|
+
const PIPE_INTERPRETERS = new Set([
|
|
397
407
|
"python", "python3", "perl", "ruby", "node",
|
|
398
408
|
]);
|
|
409
|
+
/** Per-interpreter flags that mean "the code is on the command line". */
|
|
410
|
+
const INTERPRETER_INLINE_FLAGS = {
|
|
411
|
+
python: new Set(["-c"]),
|
|
412
|
+
python3: new Set(["-c"]),
|
|
413
|
+
perl: new Set(["-e"]),
|
|
414
|
+
ruby: new Set(["-e"]),
|
|
415
|
+
node: new Set(["-e", "-p"]),
|
|
416
|
+
};
|
|
399
417
|
/** Wrapper words that forward to another command (`sudo rm …` runs rm). */
|
|
400
418
|
const WRAPPER_WORDS = new Set(["sudo", "env", "command", "builtin", "exec", "nohup", "time", "nice"]);
|
|
401
419
|
/** Shell reserved words that can precede a command inside control flow. */
|
|
@@ -710,23 +728,83 @@ function classifySegmentTokens(rawTokens, policy, opts) {
|
|
|
710
728
|
// No rule matched → prompt (fail toward review, not toward allow)
|
|
711
729
|
return { decision: "prompt", justification: `no policy rule matched for "${tokens[0]}"` };
|
|
712
730
|
}
|
|
713
|
-
/**
|
|
731
|
+
/**
|
|
732
|
+
* Check if any segment pipes into a known shell/network interpreter, or into
|
|
733
|
+
* a bare interpreter (no inline-code flag — stdin is executed as the program).
|
|
734
|
+
* Interpreters carrying an inline-code flag WITH its code argument
|
|
735
|
+
* (`… | python3 -c '…'`) are NOT caught here: they fall through to normal
|
|
736
|
+
* segment classification (prompt band), because the code rides the command
|
|
737
|
+
* string the Guardian can read. The pipe target is resolved through the SAME
|
|
738
|
+
* stripLeadingTokens walk used for command words, so wrapper spellings
|
|
739
|
+
* (`… | env sh`, `… | env -i python3`, `… | /usr/bin/env python3 -c '…'`)
|
|
740
|
+
* resolve identically — one resolver, no drift between the two paths.
|
|
741
|
+
*/
|
|
714
742
|
function isPipeToShell(command) {
|
|
715
743
|
const parsed = shellParse(command);
|
|
716
744
|
for (let i = 0; i < parsed.length - 1; i++) {
|
|
717
745
|
const t = parsed[i];
|
|
718
|
-
if (typeof t
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
746
|
+
if (typeof t !== "object" || t.op !== "pipe")
|
|
747
|
+
continue;
|
|
748
|
+
// Collect the pipe target's leading plain tokens (up to the next op),
|
|
749
|
+
// then resolve the program word through the shared wrapper strip.
|
|
750
|
+
const target = [];
|
|
751
|
+
let j = i + 1;
|
|
752
|
+
while (j < parsed.length && typeof parsed[j] === "string") {
|
|
753
|
+
target.push(parsed[j]);
|
|
754
|
+
j++;
|
|
755
|
+
}
|
|
756
|
+
const { tokens: resolved, stripped: targetStripped } = stripLeadingTokens(target);
|
|
757
|
+
if (resolved.length === 0)
|
|
758
|
+
continue;
|
|
759
|
+
// Candidate program words: the resolved word, plus — when wrappers were
|
|
760
|
+
// stripped — every non-flag token after it. stripLeadingTokens stops at
|
|
761
|
+
// the first non-flag token after a wrapper, which for flags that TAKE a
|
|
762
|
+
// value (`env -u FOO python3`) makes the value the resolved word and can
|
|
763
|
+
// let a bare interpreter slip past. Checking every candidate is a
|
|
764
|
+
// deliberate conservative tradeoff: it can only ADD forbidden verdicts,
|
|
765
|
+
// never remove them, so a benign target like `… | env grep python3`
|
|
766
|
+
// (grep is the program, python3 an argument) is hard-forbidden instead
|
|
767
|
+
// of prompted — a false positive we accept to keep bare interpreters
|
|
768
|
+
// from escaping the floor through wrapper-flag spellings.
|
|
769
|
+
const candidates = [basenameToken(resolved[0])];
|
|
770
|
+
if (targetStripped) {
|
|
771
|
+
for (let k = 1; k < resolved.length; k++) {
|
|
772
|
+
if (!resolved[k].startsWith("-"))
|
|
773
|
+
candidates.push(basenameToken(resolved[k]));
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
for (const word of candidates) {
|
|
723
777
|
if (PIPE_TO_SHELL.has(word))
|
|
724
778
|
return true;
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
779
|
+
}
|
|
780
|
+
if (candidates.some((word) => PIPE_INTERPRETERS.has(word))) {
|
|
781
|
+
if (!hasInlineCodeArg(resolved))
|
|
782
|
+
return true;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
return false;
|
|
786
|
+
}
|
|
787
|
+
/**
|
|
788
|
+
* Does a resolved pipe-target token list contain an interpreter carrying an
|
|
789
|
+
* inline-code flag WITH its code argument? An inline flag only counts when
|
|
790
|
+
* its argument follows in the same segment: argumentless `… | python3 -c`
|
|
791
|
+
* still leaves stdin as the program, so it stays forbidden. ANY interpreter
|
|
792
|
+
* in the segment satisfying flag+arg is enough — multi-interpreter segments
|
|
793
|
+
* (`… | python3 -c - python3 -c '…'`) downgrade to prompt when any word has
|
|
794
|
+
* its code on the command line, so the scan never early-returns on the first
|
|
795
|
+
* flag-without-arg.
|
|
796
|
+
*/
|
|
797
|
+
function hasInlineCodeArg(resolved) {
|
|
798
|
+
for (let k = 0; k < resolved.length; k++) {
|
|
799
|
+
const w = basenameToken(resolved[k]);
|
|
800
|
+
if (!PIPE_INTERPRETERS.has(w))
|
|
801
|
+
continue;
|
|
802
|
+
const inlineFlags = INTERPRETER_INLINE_FLAGS[w] ?? new Set();
|
|
803
|
+
for (let m = k + 1; m < resolved.length; m++) {
|
|
804
|
+
if (inlineFlags.has(resolved[m])) {
|
|
805
|
+
if (m + 1 < resolved.length && !resolved[m + 1].startsWith("-"))
|
|
729
806
|
return true;
|
|
807
|
+
break;
|
|
730
808
|
}
|
|
731
809
|
}
|
|
732
810
|
}
|
|
@@ -814,8 +892,10 @@ function dangerScanSubstitutions(command, policy, depth) {
|
|
|
814
892
|
* each is classified independently; the strictest decision wins (forbidden >
|
|
815
893
|
* prompt > allow). Commands with shell constructs (substitution, redirects,
|
|
816
894
|
* background &) have a floor of `prompt`, and their substitution inner text
|
|
817
|
-
* is danger-scanned against the forbidden rules.
|
|
818
|
-
*
|
|
895
|
+
* is danger-scanned against the forbidden rules. Pipes into shells and
|
|
896
|
+
* network relays, or into a bare interpreter (stdin executed as the program),
|
|
897
|
+
* are always forbidden; an interpreter carrying inline code (-c/-e) falls to
|
|
898
|
+
* the prompt band — the code rides the command string the Guardian can read.
|
|
819
899
|
*/
|
|
820
900
|
export function classifyCommand(command, policy) {
|
|
821
901
|
// Pipe-to-shell is always forbidden regardless of other rules.
|
|
@@ -858,11 +938,16 @@ export const DEFAULT_EXEC_POLICY = {
|
|
|
858
938
|
// --- forbidden: position-independent dangerous-flag rules (checked first;
|
|
859
939
|
// GNU getopt permutes flags, so `rm x -rf` and `git push origin
|
|
860
940
|
// --force` carry the flag after positional args) ---
|
|
941
|
+
// Recursive deletion only: any -r/-R-bearing flag bundle. Combined
|
|
942
|
+
// bundles where r is not first (-fr, -fR) are listed explicitly because
|
|
943
|
+
// the trailing-star globs only anchor at the leading char. Exotic bundles
|
|
944
|
+
// (-fir) miss this rule and land on the -f prompt rule below — fail-closed
|
|
945
|
+
// toward Guardian review, never auto-allowed.
|
|
861
946
|
{
|
|
862
947
|
pattern: ["rm"],
|
|
863
|
-
flagsAnywhere: ["-r*", "-
|
|
948
|
+
flagsAnywhere: ["-r*", "-R*", "-fr", "-fR", "--recursive*"],
|
|
864
949
|
decision: "forbidden",
|
|
865
|
-
justification: "recursive
|
|
950
|
+
justification: "recursive deletion is destructive and irreversible",
|
|
866
951
|
},
|
|
867
952
|
{
|
|
868
953
|
// Exact --force/-f only: --force-with-lease is the guarded variant and
|
|
@@ -900,9 +985,9 @@ export const DEFAULT_EXEC_POLICY = {
|
|
|
900
985
|
},
|
|
901
986
|
// --- forbidden: destructive commands (positional) ---
|
|
902
987
|
{
|
|
903
|
-
pattern: ["rm", ["-rf", "-fr", "-r", "-
|
|
988
|
+
pattern: ["rm", ["-rf", "-fr", "-r", "-R", "--recursive"]],
|
|
904
989
|
decision: "forbidden",
|
|
905
|
-
justification: "recursive
|
|
990
|
+
justification: "recursive deletion is destructive and irreversible",
|
|
906
991
|
},
|
|
907
992
|
{ pattern: ["git", "reset", "--hard"], decision: "forbidden", justification: "hard reset discards uncommitted changes irreversibly" },
|
|
908
993
|
{ pattern: ["git", "checkout", "--"], decision: "forbidden", justification: "discards working tree changes" },
|
|
@@ -1049,6 +1134,14 @@ export const DEFAULT_EXEC_POLICY = {
|
|
|
1049
1134
|
{ pattern: ["printenv"], decision: "allow", justification: "print environment variables (read-only)" },
|
|
1050
1135
|
{ pattern: ["npm", ["view", "info"]], decision: "allow", justification: "read package metadata from registry" },
|
|
1051
1136
|
// --- prompt: potentially destructive but context-dependent ---
|
|
1137
|
+
// Non-recursive forced deletion — Guardian-reviewable, single file.
|
|
1138
|
+
// Must precede the generic rm prompt rule; ordered before it in this list.
|
|
1139
|
+
{
|
|
1140
|
+
pattern: ["rm"],
|
|
1141
|
+
flagsAnywhere: ["-f*", "--force*"],
|
|
1142
|
+
decision: "prompt",
|
|
1143
|
+
justification: "forced deletion of a single file — review the target",
|
|
1144
|
+
},
|
|
1052
1145
|
{ pattern: ["rm"], decision: "prompt", justification: "file deletion — review the target" },
|
|
1053
1146
|
{ pattern: ["git", "commit"], decision: "prompt", justification: "creates a commit — confirm intent" },
|
|
1054
1147
|
{ pattern: ["git", "push"], decision: "prompt", justification: "pushes to remote — confirm intent" },
|
|
@@ -1079,6 +1172,19 @@ export const DEFAULT_EXEC_POLICY = {
|
|
|
1079
1172
|
{ pattern: ["mkdir"], decision: "prompt", justification: "creates directories" },
|
|
1080
1173
|
{ pattern: ["touch"], decision: "prompt", justification: "creates or updates file timestamps" },
|
|
1081
1174
|
{ pattern: ["tar"], decision: "prompt", justification: "archive operation" },
|
|
1175
|
+
// Interpreters with inline code — the code is on the command line where
|
|
1176
|
+
// the Guardian can read it. Bare interpreters (program from stdin or a
|
|
1177
|
+
// file) never reach these: the pipe-to-interpreter check forbids the piped
|
|
1178
|
+
// form, and the file form is covered by the prompt rules below. Ordered
|
|
1179
|
+
// AFTER the `node --version`/`node -v` allow rules so version checks stay
|
|
1180
|
+
// auto-allowed.
|
|
1181
|
+
{ pattern: ["python"], flagsAnywhere: ["-c"], decision: "prompt", justification: "python runs inline code — review the script" },
|
|
1182
|
+
{ pattern: ["python3"], flagsAnywhere: ["-c"], decision: "prompt", justification: "python runs inline code — review the script" },
|
|
1183
|
+
{ pattern: ["node", ["-e", "-p"]], decision: "prompt", justification: "node runs inline code — review the script" },
|
|
1184
|
+
{ pattern: ["perl", "-e"], decision: "prompt", justification: "perl runs inline code — review the script" },
|
|
1185
|
+
{ pattern: ["ruby", "-e"], decision: "prompt", justification: "ruby runs inline code — review the script" },
|
|
1186
|
+
{ pattern: ["python", "-m"], decision: "prompt", justification: "python runs a module — review the module and args" },
|
|
1187
|
+
{ pattern: ["python3", "-m"], decision: "prompt", justification: "python runs a module — review the module and args" },
|
|
1082
1188
|
{ pattern: ["zip"], decision: "prompt", justification: "archive operation" },
|
|
1083
1189
|
{ pattern: ["unzip"], decision: "prompt", justification: "archive operation" },
|
|
1084
1190
|
{ pattern: ["kill"], decision: "prompt", justification: "sends a signal to a process" },
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool-call outcome telemetry: counts per (tool family, outcome, reason),
|
|
3
|
+
* batched and posted to the backend's /api/yagni-code/tool-outcomes so the
|
|
4
|
+
* fleet dashboards can tell a NORMAL tool failure (the model ran a command
|
|
5
|
+
* that exited 1, read a file that is not there, tried an edit that did not
|
|
6
|
+
* match) from the tool machinery actually breaking (an MCP transport, an
|
|
7
|
+
* internal exception, a timeout).
|
|
8
|
+
*
|
|
9
|
+
* Content never leaves the machine: the classifier reads the result text
|
|
10
|
+
* locally and emits only a closed reason vocabulary, and tool names collapse
|
|
11
|
+
* to a closed family list HERE, before buffering, so an MCP server name never
|
|
12
|
+
* reaches the wire. The payload is family, outcome, reason, count, summed
|
|
13
|
+
* duration. Opt-out and test suppression follow the crash reporter
|
|
14
|
+
* (`YAGNI_DISABLE_CRASH_REPORTS=1` turns both off), eval mode is off like
|
|
15
|
+
* every other external side effect, and everything is fail-soft: one
|
|
16
|
+
* attempt, short timeout, never throws, never blocks a turn. A failed post is
|
|
17
|
+
* counted and written to the local error trail (source `telemetry`) so "why
|
|
18
|
+
* is the dashboard empty" has something to read.
|
|
19
|
+
*/
|
|
20
|
+
export type ToolOutcome = "ok" | "expected_error" | "real_error";
|
|
21
|
+
export type ToolOutcomeReason = "ok" | "exit_nonzero" | "not_found" | "no_match" | "denied" | "cancelled" | "invalid_input" | "timeout" | "mcp_transport" | "internal" | "unknown";
|
|
22
|
+
export declare const TOOL_FAMILIES: readonly ["bash", "read", "edit", "write", "grep", "find", "ls", "mcp", "subagent", "web", "other"];
|
|
23
|
+
export type ToolFamily = (typeof TOOL_FAMILIES)[number];
|
|
24
|
+
/** Collapse a tool name to its family. Never returns anything outside TOOL_FAMILIES. */
|
|
25
|
+
export declare function toolFamilyOf(toolName: string): ToolFamily;
|
|
26
|
+
export interface ToolOutcomeClass {
|
|
27
|
+
outcome: ToolOutcome;
|
|
28
|
+
reason: ToolOutcomeReason;
|
|
29
|
+
}
|
|
30
|
+
/** Best-effort text from a pi tool result (string, content blocks, or an Error). */
|
|
31
|
+
export declare function toolResultText(result: unknown): string;
|
|
32
|
+
/**
|
|
33
|
+
* Classify one tool call. The order matters: transport and internal faults
|
|
34
|
+
* win over the softer patterns because a stack trace can mention a file.
|
|
35
|
+
*/
|
|
36
|
+
export declare function classifyToolOutcome(input: {
|
|
37
|
+
toolName: string;
|
|
38
|
+
isError: boolean;
|
|
39
|
+
result?: unknown;
|
|
40
|
+
}): ToolOutcomeClass;
|
|
41
|
+
export interface ToolOutcomeSample {
|
|
42
|
+
/** A TOOL_FAMILIES value, never the raw tool name. */
|
|
43
|
+
tool: ToolFamily;
|
|
44
|
+
outcome: ToolOutcome;
|
|
45
|
+
reason: ToolOutcomeReason;
|
|
46
|
+
count: number;
|
|
47
|
+
durationMsSum: number;
|
|
48
|
+
}
|
|
49
|
+
export interface ToolOutcomeBatcherOpts {
|
|
50
|
+
baseUrl: string;
|
|
51
|
+
getToken: () => string | undefined;
|
|
52
|
+
/** Attribution headers (`x-yagni-caller` / session / run) from config.ts. */
|
|
53
|
+
headers?: Record<string, string>;
|
|
54
|
+
fetchImpl?: typeof fetch;
|
|
55
|
+
env?: NodeJS.ProcessEnv;
|
|
56
|
+
/** Flush cadence; the interval timer is unref'd so it never holds the process. */
|
|
57
|
+
flushIntervalMs?: number;
|
|
58
|
+
timeoutMs?: number;
|
|
59
|
+
now?: () => number;
|
|
60
|
+
/** Disabled entirely (eval mode, tests). */
|
|
61
|
+
enabled?: boolean;
|
|
62
|
+
/** Local trail sink for a failed post (defaults to the unified error sink). */
|
|
63
|
+
logSink?: (event: {
|
|
64
|
+
event: string;
|
|
65
|
+
fields: Record<string, unknown>;
|
|
66
|
+
}) => void;
|
|
67
|
+
}
|
|
68
|
+
export interface ToolOutcomeBatcher {
|
|
69
|
+
toolStart(toolCallId: string, toolName: string): void;
|
|
70
|
+
toolEnd(toolCallId: string, outcome: {
|
|
71
|
+
isError: boolean;
|
|
72
|
+
result?: unknown;
|
|
73
|
+
}): void;
|
|
74
|
+
/** Post whatever is buffered. Resolves on every outcome; never throws. */
|
|
75
|
+
flush(): Promise<void>;
|
|
76
|
+
/** Stop the timer and flush once. */
|
|
77
|
+
close(): Promise<void>;
|
|
78
|
+
/** Test/introspection seam: the buffered samples. */
|
|
79
|
+
pending(): ToolOutcomeSample[];
|
|
80
|
+
/** Batches that failed to post (network, non-2xx, no token) since start. */
|
|
81
|
+
dropped(): number;
|
|
82
|
+
}
|
|
83
|
+
export declare const TOOL_OUTCOME_FLUSH_INTERVAL_MS = 60000;
|
|
84
|
+
export declare const TOOL_OUTCOME_TIMEOUT_MS = 2000;
|
|
85
|
+
export declare function createToolOutcomeBatcher(opts: ToolOutcomeBatcherOpts): ToolOutcomeBatcher;
|
|
86
|
+
//# sourceMappingURL=toolOutcomes.d.ts.map
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool-call outcome telemetry: counts per (tool family, outcome, reason),
|
|
3
|
+
* batched and posted to the backend's /api/yagni-code/tool-outcomes so the
|
|
4
|
+
* fleet dashboards can tell a NORMAL tool failure (the model ran a command
|
|
5
|
+
* that exited 1, read a file that is not there, tried an edit that did not
|
|
6
|
+
* match) from the tool machinery actually breaking (an MCP transport, an
|
|
7
|
+
* internal exception, a timeout).
|
|
8
|
+
*
|
|
9
|
+
* Content never leaves the machine: the classifier reads the result text
|
|
10
|
+
* locally and emits only a closed reason vocabulary, and tool names collapse
|
|
11
|
+
* to a closed family list HERE, before buffering, so an MCP server name never
|
|
12
|
+
* reaches the wire. The payload is family, outcome, reason, count, summed
|
|
13
|
+
* duration. Opt-out and test suppression follow the crash reporter
|
|
14
|
+
* (`YAGNI_DISABLE_CRASH_REPORTS=1` turns both off), eval mode is off like
|
|
15
|
+
* every other external side effect, and everything is fail-soft: one
|
|
16
|
+
* attempt, short timeout, never throws, never blocks a turn. A failed post is
|
|
17
|
+
* counted and written to the local error trail (source `telemetry`) so "why
|
|
18
|
+
* is the dashboard empty" has something to read.
|
|
19
|
+
*/
|
|
20
|
+
import { crashReportsSuppressed } from "./crashReport.js";
|
|
21
|
+
import { logEvent } from "./errorSink.js";
|
|
22
|
+
import { isDesktopSurface } from "./surface.js";
|
|
23
|
+
export const TOOL_FAMILIES = ["bash", "read", "edit", "write", "grep", "find", "ls", "mcp", "subagent", "web", "other"];
|
|
24
|
+
/** Collapse a tool name to its family. Never returns anything outside TOOL_FAMILIES. */
|
|
25
|
+
export function toolFamilyOf(toolName) {
|
|
26
|
+
const name = toolName.toLowerCase();
|
|
27
|
+
if (name.startsWith("mcp__") || name.startsWith("mcp:"))
|
|
28
|
+
return "mcp";
|
|
29
|
+
if (name === "bash" || name === "shell" || name === "exec")
|
|
30
|
+
return "bash";
|
|
31
|
+
if (name === "read" || name === "read_file" || name === "view")
|
|
32
|
+
return "read";
|
|
33
|
+
if (name === "edit" || name === "multiedit" || name === "str_replace")
|
|
34
|
+
return "edit";
|
|
35
|
+
if (name === "write" || name === "write_file" || name === "create")
|
|
36
|
+
return "write";
|
|
37
|
+
if (name === "grep" || name === "search")
|
|
38
|
+
return "grep";
|
|
39
|
+
if (name === "find" || name === "glob")
|
|
40
|
+
return "find";
|
|
41
|
+
if (name === "ls" || name === "list")
|
|
42
|
+
return "ls";
|
|
43
|
+
if (name.includes("subagent") || name === "agent" || name === "task")
|
|
44
|
+
return "subagent";
|
|
45
|
+
if (name.startsWith("web") || name.includes("fetch") || name.includes("browser"))
|
|
46
|
+
return "web";
|
|
47
|
+
return "other";
|
|
48
|
+
}
|
|
49
|
+
/** Only this much of a result is inspected; the classifier is pattern-based. */
|
|
50
|
+
const CLASSIFY_TEXT_CAP = 4_000;
|
|
51
|
+
/** Best-effort text from a pi tool result (string, content blocks, or an Error). */
|
|
52
|
+
export function toolResultText(result) {
|
|
53
|
+
if (typeof result === "string")
|
|
54
|
+
return result.slice(0, CLASSIFY_TEXT_CAP);
|
|
55
|
+
if (result instanceof Error)
|
|
56
|
+
return `${result.name}: ${result.message}`.slice(0, CLASSIFY_TEXT_CAP);
|
|
57
|
+
if (typeof result !== "object" || result === null)
|
|
58
|
+
return "";
|
|
59
|
+
const r = result;
|
|
60
|
+
if (typeof r.text === "string")
|
|
61
|
+
return r.text.slice(0, CLASSIFY_TEXT_CAP);
|
|
62
|
+
if (typeof r.error === "string")
|
|
63
|
+
return r.error.slice(0, CLASSIFY_TEXT_CAP);
|
|
64
|
+
if (typeof r.message === "string")
|
|
65
|
+
return r.message.slice(0, CLASSIFY_TEXT_CAP);
|
|
66
|
+
if (Array.isArray(r.content)) {
|
|
67
|
+
const parts = [];
|
|
68
|
+
let size = 0;
|
|
69
|
+
for (const block of r.content) {
|
|
70
|
+
const text = typeof block === "string" ? block : block?.text;
|
|
71
|
+
if (typeof text !== "string")
|
|
72
|
+
continue;
|
|
73
|
+
parts.push(text);
|
|
74
|
+
size += text.length;
|
|
75
|
+
if (size >= CLASSIFY_TEXT_CAP)
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
return parts.join("\n").slice(0, CLASSIFY_TEXT_CAP);
|
|
79
|
+
}
|
|
80
|
+
return "";
|
|
81
|
+
}
|
|
82
|
+
const NOT_FOUND_RE = /no such file|not found|does not exist|enoent|cannot find|unknown file|no matches? found/i;
|
|
83
|
+
const NO_MATCH_RE = /old_string|did not match|not unique|could not find the (?:string|text)|no occurrences|nothing to replace/i;
|
|
84
|
+
const DENIED_RE = /permission denied|denied by|blocked by|not allowed|not permitted|refused|guardian|plan mode|requires approval|eacces|eperm/i;
|
|
85
|
+
const CANCELLED_RE = /cancel+ed|aborted|interrupted|user (?:declined|rejected|stopped)/i;
|
|
86
|
+
const INVALID_RE = /invalid (?:argument|input|json|parameter)|missing required|expected .* to be|is required|malformed|validation/i;
|
|
87
|
+
const TIMEOUT_RE = /timed? ?out|deadline exceeded|etimedout/i;
|
|
88
|
+
const MCP_RE = /mcp|transport|econnrefused|econnreset|socket hang up|server (?:closed|disconnected|unavailable)|jsonrpc|connection (?:closed|lost|refused)/i;
|
|
89
|
+
const INTERNAL_RE = /^(?:type|reference|range|syntax)error\b|internal error|unhandled|stack trace|cannot read propert|is not a function|undefined is not/i;
|
|
90
|
+
const EXIT_RE = /exit(?:ed)? (?:with )?(?:code|status)[: ]+(\d+)|command failed|non-zero exit|\bexit code\b/i;
|
|
91
|
+
/**
|
|
92
|
+
* Classify one tool call. The order matters: transport and internal faults
|
|
93
|
+
* win over the softer patterns because a stack trace can mention a file.
|
|
94
|
+
*/
|
|
95
|
+
export function classifyToolOutcome(input) {
|
|
96
|
+
if (!input.isError)
|
|
97
|
+
return { outcome: "ok", reason: "ok" };
|
|
98
|
+
const text = toolResultText(input.result);
|
|
99
|
+
// Accepts a raw tool name or an already-collapsed family (the batcher
|
|
100
|
+
// classifies by family).
|
|
101
|
+
const mcp = input.toolName === "mcp" || toolFamilyOf(input.toolName) === "mcp";
|
|
102
|
+
if (INTERNAL_RE.test(text))
|
|
103
|
+
return { outcome: "real_error", reason: "internal" };
|
|
104
|
+
if (mcp && MCP_RE.test(text))
|
|
105
|
+
return { outcome: "real_error", reason: "mcp_transport" };
|
|
106
|
+
if (TIMEOUT_RE.test(text))
|
|
107
|
+
return { outcome: "real_error", reason: "timeout" };
|
|
108
|
+
if (CANCELLED_RE.test(text))
|
|
109
|
+
return { outcome: "expected_error", reason: "cancelled" };
|
|
110
|
+
if (DENIED_RE.test(text))
|
|
111
|
+
return { outcome: "expected_error", reason: "denied" };
|
|
112
|
+
if (NO_MATCH_RE.test(text))
|
|
113
|
+
return { outcome: "expected_error", reason: "no_match" };
|
|
114
|
+
if (NOT_FOUND_RE.test(text))
|
|
115
|
+
return { outcome: "expected_error", reason: "not_found" };
|
|
116
|
+
if (INVALID_RE.test(text))
|
|
117
|
+
return { outcome: "expected_error", reason: "invalid_input" };
|
|
118
|
+
if (EXIT_RE.test(text) || input.toolName === "bash")
|
|
119
|
+
return { outcome: "expected_error", reason: "exit_nonzero" };
|
|
120
|
+
if (!mcp && MCP_RE.test(text))
|
|
121
|
+
return { outcome: "real_error", reason: "mcp_transport" };
|
|
122
|
+
return { outcome: "real_error", reason: "unknown" };
|
|
123
|
+
}
|
|
124
|
+
export const TOOL_OUTCOME_FLUSH_INTERVAL_MS = 60_000;
|
|
125
|
+
export const TOOL_OUTCOME_TIMEOUT_MS = 2_000;
|
|
126
|
+
const MAX_TRACKED_STARTS = 512;
|
|
127
|
+
export function createToolOutcomeBatcher(opts) {
|
|
128
|
+
const env = opts.env ?? process.env;
|
|
129
|
+
const now = opts.now ?? Date.now;
|
|
130
|
+
const enabled = (opts.enabled ?? true) && !crashReportsSuppressed(env);
|
|
131
|
+
const logSink = opts.logSink ??
|
|
132
|
+
((e) => logEvent({ source: "telemetry", level: "warn", event: e.event, fields: e.fields, sessionId: env.YAGNI_SESSION_ID }));
|
|
133
|
+
const starts = new Map();
|
|
134
|
+
const buffer = new Map();
|
|
135
|
+
let timer;
|
|
136
|
+
let closed = false;
|
|
137
|
+
let droppedBatches = 0;
|
|
138
|
+
const key = (tool, outcome, reason) => `${tool}|${outcome}|${reason}`;
|
|
139
|
+
const record = (tool, cls, durationMs) => {
|
|
140
|
+
const k = key(tool, cls.outcome, cls.reason);
|
|
141
|
+
const existing = buffer.get(k);
|
|
142
|
+
if (existing) {
|
|
143
|
+
existing.count += 1;
|
|
144
|
+
existing.durationMsSum += durationMs;
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
buffer.set(k, { tool, outcome: cls.outcome, reason: cls.reason, count: 1, durationMsSum: durationMs });
|
|
148
|
+
};
|
|
149
|
+
const drop = (samples, fields) => {
|
|
150
|
+
droppedBatches += 1;
|
|
151
|
+
try {
|
|
152
|
+
logSink({ event: "tool_outcomes_post_failed", fields: { ...fields, samples: samples.length, droppedBatches } });
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
// the trail is best-effort
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
const flush = async () => {
|
|
159
|
+
if (!enabled || buffer.size === 0)
|
|
160
|
+
return;
|
|
161
|
+
const samples = [...buffer.values()];
|
|
162
|
+
buffer.clear();
|
|
163
|
+
try {
|
|
164
|
+
// The token getter is fail-soft too: a throwing provider must not
|
|
165
|
+
// reject the timer's `void flush()` or surface at shutdown.
|
|
166
|
+
const token = opts.getToken();
|
|
167
|
+
if (!token) {
|
|
168
|
+
drop(samples, { kind: "no_token" });
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
172
|
+
const controller = new AbortController();
|
|
173
|
+
const t = setTimeout(() => controller.abort(), opts.timeoutMs ?? TOOL_OUTCOME_TIMEOUT_MS);
|
|
174
|
+
t.unref?.();
|
|
175
|
+
try {
|
|
176
|
+
const res = await fetchImpl(`${opts.baseUrl.replace(/\/$/, "")}/api/yagni-code/tool-outcomes`, {
|
|
177
|
+
method: "POST",
|
|
178
|
+
headers: {
|
|
179
|
+
"content-type": "application/json",
|
|
180
|
+
authorization: `Bearer ${token}`,
|
|
181
|
+
...(opts.headers ?? {}),
|
|
182
|
+
},
|
|
183
|
+
body: JSON.stringify({
|
|
184
|
+
client: isDesktopSurface() ? "desktop" : "cli",
|
|
185
|
+
clientVersion: env.YAGNI_CODE_VERSION?.trim() || "unknown",
|
|
186
|
+
samples,
|
|
187
|
+
}),
|
|
188
|
+
signal: controller.signal,
|
|
189
|
+
});
|
|
190
|
+
if (!res.ok)
|
|
191
|
+
drop(samples, { kind: "http", status: res.status });
|
|
192
|
+
}
|
|
193
|
+
finally {
|
|
194
|
+
clearTimeout(t);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
catch (err) {
|
|
198
|
+
drop(samples, { kind: "network", error: err instanceof Error ? err.name : "unknown" });
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
if (enabled) {
|
|
202
|
+
timer = setInterval(() => { void flush(); }, opts.flushIntervalMs ?? TOOL_OUTCOME_FLUSH_INTERVAL_MS);
|
|
203
|
+
timer.unref?.();
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
toolStart(toolCallId, toolName) {
|
|
207
|
+
if (!enabled || closed)
|
|
208
|
+
return;
|
|
209
|
+
if (starts.size >= MAX_TRACKED_STARTS)
|
|
210
|
+
starts.clear();
|
|
211
|
+
starts.set(toolCallId, { family: toolFamilyOf(toolName), startedAt: now() });
|
|
212
|
+
},
|
|
213
|
+
toolEnd(toolCallId, outcome) {
|
|
214
|
+
if (!enabled || closed)
|
|
215
|
+
return;
|
|
216
|
+
const slot = starts.get(toolCallId);
|
|
217
|
+
starts.delete(toolCallId);
|
|
218
|
+
const family = slot?.family ?? "other";
|
|
219
|
+
const durationMs = slot ? Math.max(0, now() - slot.startedAt) : 0;
|
|
220
|
+
try {
|
|
221
|
+
record(family, classifyToolOutcome({ toolName: family, isError: outcome.isError, result: outcome.result }), durationMs);
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
// classification must never break a tool result
|
|
225
|
+
}
|
|
226
|
+
},
|
|
227
|
+
flush,
|
|
228
|
+
async close() {
|
|
229
|
+
if (closed)
|
|
230
|
+
return;
|
|
231
|
+
closed = true;
|
|
232
|
+
if (timer)
|
|
233
|
+
clearInterval(timer);
|
|
234
|
+
await flush();
|
|
235
|
+
},
|
|
236
|
+
pending: () => [...buffer.values()],
|
|
237
|
+
dropped: () => droppedBatches,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
//# sourceMappingURL=toolOutcomes.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yagni-app/code-staging",
|
|
3
|
-
"version": "1.1.0-staging.
|
|
3
|
+
"version": "1.1.0-staging.1329.1",
|
|
4
4
|
"description": "YAGNI Code: a terminal coding agent that already knows your company. One YAGNI login routes the model and grounds the agent in your team's context.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE.md",
|
|
6
6
|
"author": "YAGNI, Inc. <jack@yagni.app> (https://yagni.app)",
|
|
@@ -58,5 +58,5 @@
|
|
|
58
58
|
"turndown": "^7.2.4",
|
|
59
59
|
"typebox": "^1.3.15"
|
|
60
60
|
},
|
|
61
|
-
"yagniSourceSha": "
|
|
61
|
+
"yagniSourceSha": "36a0344047224a979e0714451f6e710658d1454b"
|
|
62
62
|
}
|