noctrace 0.7.4 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/client/index.html
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
|
8
8
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
|
9
9
|
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;600;700&display=swap" rel="stylesheet" />
|
|
10
|
-
<script type="module" crossorigin src="/assets/index-
|
|
10
|
+
<script type="module" crossorigin src="/assets/index-9-y1oErw.js"></script>
|
|
11
11
|
<link rel="stylesheet" crossorigin href="/assets/index-DyjeNSzP.css">
|
|
12
12
|
</head>
|
|
13
13
|
<body>
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/** Collect all rows (including nested children) into a flat list. */
|
|
2
|
+
function flattenRows(rows) {
|
|
3
|
+
const result = [];
|
|
4
|
+
const walk = (list) => {
|
|
5
|
+
for (const row of list) {
|
|
6
|
+
result.push(row);
|
|
7
|
+
if (row.children.length > 0)
|
|
8
|
+
walk(row.children);
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
walk(rows);
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Extract the first plausible file path from a label string.
|
|
16
|
+
* Matches Unix-style absolute paths or relative paths containing a `/`.
|
|
17
|
+
* Returns null when no path-like token is found.
|
|
18
|
+
*/
|
|
19
|
+
function extractFilePath(label) {
|
|
20
|
+
// Match tokens that look like file paths:
|
|
21
|
+
// - absolute: /foo/bar.ts
|
|
22
|
+
// - relative with directory: src/foo.ts
|
|
23
|
+
// We split on spaces/colons/parens and look for tokens containing '/'
|
|
24
|
+
const tokens = label.split(/[\s:()[\],"']+/);
|
|
25
|
+
for (const token of tokens) {
|
|
26
|
+
if (token.length === 0)
|
|
27
|
+
continue;
|
|
28
|
+
// Must contain a slash and at least one dot (heuristic for real paths)
|
|
29
|
+
if (token.includes('/') && token.includes('.')) {
|
|
30
|
+
return token;
|
|
31
|
+
}
|
|
32
|
+
// Also accept absolute paths without a dot (e.g. /usr/local/bin/node)
|
|
33
|
+
if (token.startsWith('/') && token.includes('/')) {
|
|
34
|
+
return token;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Compute per-tool and session-level reliability statistics.
|
|
41
|
+
*
|
|
42
|
+
* Rows with status 'running' are excluded from all counts — they have not
|
|
43
|
+
* yet completed and would skew success/error ratios.
|
|
44
|
+
*
|
|
45
|
+
* @param rows - Top-level WaterfallRow array from the session parser.
|
|
46
|
+
* @returns {@link SessionReliability} with per-tool and aggregate metrics.
|
|
47
|
+
*/
|
|
48
|
+
export function computeReliability(rows) {
|
|
49
|
+
const flat = flattenRows(rows);
|
|
50
|
+
// Only consider completed rows (skip running)
|
|
51
|
+
const completed = flat.filter((r) => r.status !== 'running');
|
|
52
|
+
if (completed.length === 0) {
|
|
53
|
+
return {
|
|
54
|
+
totalCalls: 0,
|
|
55
|
+
successCount: 0,
|
|
56
|
+
errorCount: 0,
|
|
57
|
+
failureCount: 0,
|
|
58
|
+
overallReliability: 100,
|
|
59
|
+
errorDensity: 0,
|
|
60
|
+
recoveryAttempts: 0,
|
|
61
|
+
recoverySuccesses: 0,
|
|
62
|
+
recoveryRate: 0,
|
|
63
|
+
avgErrorsBeforeFix: 0,
|
|
64
|
+
toolReliability: [],
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
// -------------------------------------------------------------------------
|
|
68
|
+
// Overall counts
|
|
69
|
+
// -------------------------------------------------------------------------
|
|
70
|
+
let successCount = 0;
|
|
71
|
+
let errorCount = 0;
|
|
72
|
+
let failureCount = 0;
|
|
73
|
+
for (const row of completed) {
|
|
74
|
+
if (row.isFailure) {
|
|
75
|
+
failureCount++;
|
|
76
|
+
}
|
|
77
|
+
else if (row.status === 'error') {
|
|
78
|
+
errorCount++;
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
successCount++;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const totalCalls = completed.length;
|
|
85
|
+
const overallReliability = (successCount / totalCalls) * 100;
|
|
86
|
+
// -------------------------------------------------------------------------
|
|
87
|
+
// Error density: errors per 10 calls
|
|
88
|
+
// -------------------------------------------------------------------------
|
|
89
|
+
const errorDensity = totalCalls > 0 ? (errorCount / totalCalls) * 10 : 0;
|
|
90
|
+
// -------------------------------------------------------------------------
|
|
91
|
+
// Recovery rate: error→same-tool-success sequences
|
|
92
|
+
// -------------------------------------------------------------------------
|
|
93
|
+
// Build a map from toolName (lowercased) to the ordered list of completed rows
|
|
94
|
+
const toolSequences = new Map();
|
|
95
|
+
for (const row of completed) {
|
|
96
|
+
const key = row.toolName.toLowerCase();
|
|
97
|
+
if (!toolSequences.has(key))
|
|
98
|
+
toolSequences.set(key, []);
|
|
99
|
+
toolSequences.get(key).push(row);
|
|
100
|
+
}
|
|
101
|
+
let recoveryAttempts = 0;
|
|
102
|
+
let recoverySuccesses = 0;
|
|
103
|
+
for (const [, sequence] of toolSequences) {
|
|
104
|
+
for (let i = 0; i < sequence.length - 1; i++) {
|
|
105
|
+
if (sequence[i].status === 'error') {
|
|
106
|
+
// Look for the immediately next row in the same-tool sequence
|
|
107
|
+
recoveryAttempts++;
|
|
108
|
+
if (sequence[i + 1].status === 'success') {
|
|
109
|
+
recoverySuccesses++;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
const recoveryRate = recoveryAttempts > 0 ? (recoverySuccesses / recoveryAttempts) * 100 : 0;
|
|
115
|
+
// -------------------------------------------------------------------------
|
|
116
|
+
// Error-to-fix ratio: avg errors before first success, per file
|
|
117
|
+
// -------------------------------------------------------------------------
|
|
118
|
+
// Group rows by extracted file path
|
|
119
|
+
const fileRows = new Map();
|
|
120
|
+
for (const row of completed) {
|
|
121
|
+
const filePath = extractFilePath(row.label);
|
|
122
|
+
if (filePath === null)
|
|
123
|
+
continue;
|
|
124
|
+
if (!fileRows.has(filePath))
|
|
125
|
+
fileRows.set(filePath, []);
|
|
126
|
+
fileRows.get(filePath).push(row);
|
|
127
|
+
}
|
|
128
|
+
const errorsBeforeFixCounts = [];
|
|
129
|
+
for (const [, fileSeq] of fileRows) {
|
|
130
|
+
// Find the index of the first success
|
|
131
|
+
const firstSuccessIdx = fileSeq.findIndex((r) => r.status === 'success');
|
|
132
|
+
if (firstSuccessIdx <= 0)
|
|
133
|
+
continue; // no success, or success was first — skip
|
|
134
|
+
// Count errors before the first success
|
|
135
|
+
const errorsBeforeSuccess = fileSeq
|
|
136
|
+
.slice(0, firstSuccessIdx)
|
|
137
|
+
.filter((r) => r.status === 'error').length;
|
|
138
|
+
if (errorsBeforeSuccess === 0)
|
|
139
|
+
continue; // no errors before success — skip
|
|
140
|
+
errorsBeforeFixCounts.push(errorsBeforeSuccess);
|
|
141
|
+
}
|
|
142
|
+
const avgErrorsBeforeFix = errorsBeforeFixCounts.length > 0
|
|
143
|
+
? errorsBeforeFixCounts.reduce((a, b) => a + b, 0) / errorsBeforeFixCounts.length
|
|
144
|
+
: 0;
|
|
145
|
+
// -------------------------------------------------------------------------
|
|
146
|
+
// Per-tool reliability
|
|
147
|
+
// -------------------------------------------------------------------------
|
|
148
|
+
const toolGroups = new Map();
|
|
149
|
+
for (const row of completed) {
|
|
150
|
+
const key = row.toolName.toLowerCase();
|
|
151
|
+
if (!toolGroups.has(key)) {
|
|
152
|
+
toolGroups.set(key, { originalName: row.toolName, success: 0, errors: 0, failures: 0 });
|
|
153
|
+
}
|
|
154
|
+
const entry = toolGroups.get(key);
|
|
155
|
+
if (row.isFailure) {
|
|
156
|
+
entry.failures++;
|
|
157
|
+
}
|
|
158
|
+
else if (row.status === 'error') {
|
|
159
|
+
entry.errors++;
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
entry.success++;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
const toolReliability = [];
|
|
166
|
+
for (const [, { originalName, success, errors, failures }] of toolGroups) {
|
|
167
|
+
const total = success + errors + failures;
|
|
168
|
+
const reliability = total > 0 ? (success / total) * 100 : 100;
|
|
169
|
+
toolReliability.push({ toolName: originalName, total, success, errors, failures, reliability });
|
|
170
|
+
}
|
|
171
|
+
// Sort ascending by reliability — worst tools first
|
|
172
|
+
toolReliability.sort((a, b) => a.reliability - b.reliability);
|
|
173
|
+
return {
|
|
174
|
+
totalCalls,
|
|
175
|
+
successCount,
|
|
176
|
+
errorCount,
|
|
177
|
+
failureCount,
|
|
178
|
+
overallReliability,
|
|
179
|
+
errorDensity,
|
|
180
|
+
recoveryAttempts,
|
|
181
|
+
recoverySuccesses,
|
|
182
|
+
recoveryRate,
|
|
183
|
+
avgErrorsBeforeFix,
|
|
184
|
+
toolReliability,
|
|
185
|
+
};
|
|
186
|
+
}
|