@cyanautomation/kaseki-agent 1.61.0 → 1.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/execution-time-aggregator.d.ts +58 -0
- package/dist/execution-time-aggregator.d.ts.map +1 -0
- package/dist/execution-time-aggregator.js +92 -0
- package/dist/execution-time-aggregator.js.map +1 -0
- package/dist/lib/event-timestamp-helpers.d.ts +3 -0
- package/dist/lib/event-timestamp-helpers.d.ts.map +1 -1
- package/dist/pi-event-filter.js +106 -3
- package/dist/pi-event-filter.js.map +1 -1
- package/dist/tool-reliability-aggregator.d.ts +58 -0
- package/dist/tool-reliability-aggregator.d.ts.map +1 -0
- package/dist/tool-reliability-aggregator.js +116 -0
- package/dist/tool-reliability-aggregator.js.map +1 -0
- package/package.json +1 -1
- package/scripts/kaseki-setup-host.sh +111 -55
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* execution-time-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Tracks API time vs tool execution time to identify performance bottlenecks.
|
|
5
|
+
* Useful for understanding whether kaseki runs are bottlenecked by LLM API calls
|
|
6
|
+
* or external tool execution (git, npm, validation commands).
|
|
7
|
+
*/
|
|
8
|
+
export interface ExecutionTimeSummary {
|
|
9
|
+
api_time_seconds: number;
|
|
10
|
+
tool_time_seconds: number;
|
|
11
|
+
total_time_seconds: number;
|
|
12
|
+
api_percent: number;
|
|
13
|
+
tool_percent: number;
|
|
14
|
+
}
|
|
15
|
+
export interface ExecutionStats {
|
|
16
|
+
[identifier: string]: {
|
|
17
|
+
calls: number;
|
|
18
|
+
total_seconds: number;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* ExecutionTimeAggregator tracks wall time spent in two categories:
|
|
23
|
+
* 1. API calls (Pi agent invocations, scouting, goal-check, etc.)
|
|
24
|
+
* 2. Tool execution (npm, git, bash, validation commands)
|
|
25
|
+
*
|
|
26
|
+
* This breakdown helps identify if the bottleneck is API latency or tool execution.
|
|
27
|
+
*/
|
|
28
|
+
export declare class ExecutionTimeAggregator {
|
|
29
|
+
private apiTimeSeconds;
|
|
30
|
+
private toolTimeSeconds;
|
|
31
|
+
private apiStats;
|
|
32
|
+
private toolStats;
|
|
33
|
+
/**
|
|
34
|
+
* Record an API call duration.
|
|
35
|
+
* @param apiName - Identifier for the API call (e.g., 'pi-agent', 'pi-scouting')
|
|
36
|
+
* @param durationSeconds - Duration in seconds
|
|
37
|
+
*/
|
|
38
|
+
recordApiCall(apiName: string, durationSeconds: number): void;
|
|
39
|
+
/**
|
|
40
|
+
* Record a tool execution duration.
|
|
41
|
+
* @param toolName - Identifier for the tool (e.g., 'npm test', 'git clone')
|
|
42
|
+
* @param durationSeconds - Duration in seconds
|
|
43
|
+
*/
|
|
44
|
+
recordToolExecution(toolName: string, durationSeconds: number): void;
|
|
45
|
+
/**
|
|
46
|
+
* Get the overall execution time summary.
|
|
47
|
+
*/
|
|
48
|
+
getSummary(): ExecutionTimeSummary;
|
|
49
|
+
/**
|
|
50
|
+
* Get per-API statistics.
|
|
51
|
+
*/
|
|
52
|
+
getApiStats(): ExecutionStats;
|
|
53
|
+
/**
|
|
54
|
+
* Get per-tool statistics.
|
|
55
|
+
*/
|
|
56
|
+
getToolStats(): ExecutionStats;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=execution-time-aggregator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execution-time-aggregator.d.ts","sourceRoot":"","sources":["../src/execution-time-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,UAAU,EAAE,MAAM,GAAG;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED;;;;;;GAMG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,eAAe,CAAK;IAG5B,OAAO,CAAC,QAAQ,CACJ;IAGZ,OAAO,CAAC,SAAS,CACL;IAEZ;;;;OAIG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI;IAY7D;;;;OAIG;IACH,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI;IAYpE;;OAEG;IACH,UAAU,IAAI,oBAAoB;IAoBlC;;OAEG;IACH,WAAW,IAAI,cAAc;IAU7B;;OAEG;IACH,YAAY,IAAI,cAAc;CAS/B"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* execution-time-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Tracks API time vs tool execution time to identify performance bottlenecks.
|
|
5
|
+
* Useful for understanding whether kaseki runs are bottlenecked by LLM API calls
|
|
6
|
+
* or external tool execution (git, npm, validation commands).
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* ExecutionTimeAggregator tracks wall time spent in two categories:
|
|
10
|
+
* 1. API calls (Pi agent invocations, scouting, goal-check, etc.)
|
|
11
|
+
* 2. Tool execution (npm, git, bash, validation commands)
|
|
12
|
+
*
|
|
13
|
+
* This breakdown helps identify if the bottleneck is API latency or tool execution.
|
|
14
|
+
*/
|
|
15
|
+
export class ExecutionTimeAggregator {
|
|
16
|
+
apiTimeSeconds = 0;
|
|
17
|
+
toolTimeSeconds = 0;
|
|
18
|
+
// Per-API tracking: apiName -> { calls, total_seconds }
|
|
19
|
+
apiStats = new Map();
|
|
20
|
+
// Per-tool tracking: toolName -> { calls, total_seconds }
|
|
21
|
+
toolStats = new Map();
|
|
22
|
+
/**
|
|
23
|
+
* Record an API call duration.
|
|
24
|
+
* @param apiName - Identifier for the API call (e.g., 'pi-agent', 'pi-scouting')
|
|
25
|
+
* @param durationSeconds - Duration in seconds
|
|
26
|
+
*/
|
|
27
|
+
recordApiCall(apiName, durationSeconds) {
|
|
28
|
+
this.apiTimeSeconds += durationSeconds;
|
|
29
|
+
const existing = this.apiStats.get(apiName) || {
|
|
30
|
+
calls: 0,
|
|
31
|
+
total_seconds: 0,
|
|
32
|
+
};
|
|
33
|
+
existing.calls += 1;
|
|
34
|
+
existing.total_seconds += durationSeconds;
|
|
35
|
+
this.apiStats.set(apiName, existing);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Record a tool execution duration.
|
|
39
|
+
* @param toolName - Identifier for the tool (e.g., 'npm test', 'git clone')
|
|
40
|
+
* @param durationSeconds - Duration in seconds
|
|
41
|
+
*/
|
|
42
|
+
recordToolExecution(toolName, durationSeconds) {
|
|
43
|
+
this.toolTimeSeconds += durationSeconds;
|
|
44
|
+
const existing = this.toolStats.get(toolName) || {
|
|
45
|
+
calls: 0,
|
|
46
|
+
total_seconds: 0,
|
|
47
|
+
};
|
|
48
|
+
existing.calls += 1;
|
|
49
|
+
existing.total_seconds += durationSeconds;
|
|
50
|
+
this.toolStats.set(toolName, existing);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Get the overall execution time summary.
|
|
54
|
+
*/
|
|
55
|
+
getSummary() {
|
|
56
|
+
const totalSeconds = this.apiTimeSeconds + this.toolTimeSeconds;
|
|
57
|
+
const apiPercent = totalSeconds === 0
|
|
58
|
+
? 0
|
|
59
|
+
: Math.round((this.apiTimeSeconds / totalSeconds) * 10000) / 100;
|
|
60
|
+
const toolPercent = totalSeconds === 0
|
|
61
|
+
? 0
|
|
62
|
+
: Math.round((this.toolTimeSeconds / totalSeconds) * 10000) / 100;
|
|
63
|
+
return {
|
|
64
|
+
api_time_seconds: this.apiTimeSeconds,
|
|
65
|
+
tool_time_seconds: this.toolTimeSeconds,
|
|
66
|
+
total_time_seconds: totalSeconds,
|
|
67
|
+
api_percent: apiPercent,
|
|
68
|
+
tool_percent: toolPercent,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Get per-API statistics.
|
|
73
|
+
*/
|
|
74
|
+
getApiStats() {
|
|
75
|
+
const result = {};
|
|
76
|
+
for (const [apiName, stats] of this.apiStats.entries()) {
|
|
77
|
+
result[apiName] = { ...stats };
|
|
78
|
+
}
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Get per-tool statistics.
|
|
83
|
+
*/
|
|
84
|
+
getToolStats() {
|
|
85
|
+
const result = {};
|
|
86
|
+
for (const [toolName, stats] of this.toolStats.entries()) {
|
|
87
|
+
result[toolName] = { ...stats };
|
|
88
|
+
}
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=execution-time-aggregator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execution-time-aggregator.js","sourceRoot":"","sources":["../src/execution-time-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAiBH;;;;;;GAMG;AACH,MAAM,OAAO,uBAAuB;IAC1B,cAAc,GAAG,CAAC,CAAC;IACnB,eAAe,GAAG,CAAC,CAAC;IAE5B,wDAAwD;IAChD,QAAQ,GACd,IAAI,GAAG,EAAE,CAAC;IAEZ,0DAA0D;IAClD,SAAS,GACf,IAAI,GAAG,EAAE,CAAC;IAEZ;;;;OAIG;IACH,aAAa,CAAC,OAAe,EAAE,eAAuB;QACpD,IAAI,CAAC,cAAc,IAAI,eAAe,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI;YAC7C,KAAK,EAAE,CAAC;YACR,aAAa,EAAE,CAAC;SACjB,CAAC;QACF,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,aAAa,IAAI,eAAe,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACH,mBAAmB,CAAC,QAAgB,EAAE,eAAuB;QAC3D,IAAI,CAAC,eAAe,IAAI,eAAe,CAAC;QAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI;YAC/C,KAAK,EAAE,CAAC;YACR,aAAa,EAAE,CAAC;SACjB,CAAC;QACF,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,aAAa,IAAI,eAAe,CAAC;QAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,UAAU;QACR,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC;QAChE,MAAM,UAAU,GACd,YAAY,KAAK,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;QACrE,MAAM,WAAW,GACf,YAAY,KAAK,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,GAAG,YAAY,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;QAEtE,OAAO;YACL,gBAAgB,EAAE,IAAI,CAAC,cAAc;YACrC,iBAAiB,EAAE,IAAI,CAAC,eAAe;YACvC,kBAAkB,EAAE,YAAY;YAChC,WAAW,EAAE,UAAU;YACvB,YAAY,EAAE,WAAW;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,WAAW;QACT,MAAM,MAAM,GAAmB,EAAE,CAAC;QAElC,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,MAAM,GAAmB,EAAE,CAAC;QAElC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAClC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -17,6 +17,7 @@ export interface PiEvent {
|
|
|
17
17
|
timestamp?: string | number;
|
|
18
18
|
content?: Array<{
|
|
19
19
|
type: string;
|
|
20
|
+
text?: string;
|
|
20
21
|
}>;
|
|
21
22
|
};
|
|
22
23
|
assistantMessageEvent?: {
|
|
@@ -27,6 +28,7 @@ export interface PiEvent {
|
|
|
27
28
|
timestamp?: string | number;
|
|
28
29
|
content?: Array<{
|
|
29
30
|
type: string;
|
|
31
|
+
text?: string;
|
|
30
32
|
}>;
|
|
31
33
|
};
|
|
32
34
|
partial?: {
|
|
@@ -35,6 +37,7 @@ export interface PiEvent {
|
|
|
35
37
|
timestamp?: string | number;
|
|
36
38
|
content?: Array<{
|
|
37
39
|
type: string;
|
|
40
|
+
text?: string;
|
|
38
41
|
}>;
|
|
39
42
|
};
|
|
40
43
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event-timestamp-helpers.d.ts","sourceRoot":"","sources":["../../src/lib/event-timestamp-helpers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"event-timestamp-helpers.d.ts","sourceRoot":"","sources":["../../src/lib/event-timestamp-helpers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClD,CAAC;IACF,qBAAqB,CAAC,EAAE;QACtB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE;YACR,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,GAAG,CAAC,EAAE,MAAM,CAAC;YACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;YAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;SAClD,CAAC;QACF,OAAO,CAAC,EAAE;YACR,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,GAAG,CAAC,EAAE,MAAM,CAAC;YACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;YAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;SAClD,CAAC;KACH,CAAC;CACH;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAgBnE;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,GAAG,SAAS,CAiB3G;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAG3D"}
|
package/dist/pi-event-filter.js
CHANGED
|
@@ -3,6 +3,8 @@ import fs from 'node:fs';
|
|
|
3
3
|
import { once } from 'node:events';
|
|
4
4
|
import readline from 'node:readline';
|
|
5
5
|
import { EventCounterAggregator } from './event-aggregator.js';
|
|
6
|
+
import { ToolReliabilityAggregator } from './tool-reliability-aggregator.js';
|
|
7
|
+
import { ExecutionTimeAggregator } from './execution-time-aggregator.js';
|
|
6
8
|
import { TimestampTracker } from './timestamp-tracker.js';
|
|
7
9
|
import { extractEventTimestamp } from './lib/event-timestamp-helpers.js';
|
|
8
10
|
const inputPath = process.argv[2] ?? '/tmp/pi-events.raw.jsonl';
|
|
@@ -47,14 +49,90 @@ function sanitize(event) {
|
|
|
47
49
|
}
|
|
48
50
|
return copy;
|
|
49
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Extract tool name from a Pi event.
|
|
54
|
+
* Handles both tool_execution events and tool_call events.
|
|
55
|
+
*/
|
|
56
|
+
function extractToolName(event) {
|
|
57
|
+
// Handle tool_call events (e.g., hashline_edit)
|
|
58
|
+
if (event.tool_name) {
|
|
59
|
+
return event.tool_name;
|
|
60
|
+
}
|
|
61
|
+
// Could extract from message content in future if needed
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Extract message content from a tool_execution_end event for error detection.
|
|
66
|
+
* Aggregates output_text parts and visible content.
|
|
67
|
+
*/
|
|
68
|
+
function extractToolOutput(event) {
|
|
69
|
+
const parts = [];
|
|
70
|
+
if (event.message?.content && Array.isArray(event.message.content)) {
|
|
71
|
+
for (const part of event.message.content) {
|
|
72
|
+
if (part?.type === 'output_text' && part.text) {
|
|
73
|
+
parts.push(part.text);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return parts.join(' ');
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Extract Unix timestamp in seconds from event timestamp.
|
|
81
|
+
* Handles both ISO strings and Unix epoch numbers.
|
|
82
|
+
*/
|
|
83
|
+
function extractTimestampSeconds(event) {
|
|
84
|
+
const timestamp = event.timestamp;
|
|
85
|
+
if (!timestamp)
|
|
86
|
+
return null;
|
|
87
|
+
if (typeof timestamp === 'number') {
|
|
88
|
+
// Already a Unix timestamp (ms or seconds)
|
|
89
|
+
if (timestamp > 1e10) {
|
|
90
|
+
// Likely milliseconds
|
|
91
|
+
return timestamp / 1000;
|
|
92
|
+
}
|
|
93
|
+
return timestamp;
|
|
94
|
+
}
|
|
95
|
+
if (typeof timestamp === 'string') {
|
|
96
|
+
// ISO 8601 string
|
|
97
|
+
const ms = new Date(timestamp).getTime();
|
|
98
|
+
return isNaN(ms) ? null : ms / 1000;
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Detect if this event marks the start of an agent invocation.
|
|
104
|
+
*/
|
|
105
|
+
function isAgentStart(event) {
|
|
106
|
+
return event.type === 'agent_start' || event.type === 'agentstart';
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Detect if this event marks the end of an agent invocation.
|
|
110
|
+
*/
|
|
111
|
+
function isAgentEnd(event) {
|
|
112
|
+
return event.type === 'agent_end' || event.type === 'agentend';
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Extract the phase name from an agent event (e.g., from assistantMessageEvent context).
|
|
116
|
+
*/
|
|
117
|
+
function extractPhase(event) {
|
|
118
|
+
// Try to infer phase from event context
|
|
119
|
+
// This is a heuristic; actual phase names come from kaseki-agent.sh
|
|
120
|
+
const context = event.context || event.phase || 'unknown';
|
|
121
|
+
return typeof context === 'string' ? context : 'unknown';
|
|
122
|
+
}
|
|
50
123
|
async function main() {
|
|
51
124
|
startRssSampler();
|
|
52
125
|
const input = fs.createReadStream(inputPath, { encoding: 'utf8' });
|
|
53
126
|
const output = fs.createWriteStream(filteredPath, { encoding: 'utf8' });
|
|
54
127
|
const lines = readline.createInterface({ input, crlfDelay: Infinity });
|
|
55
128
|
const aggregator = new EventCounterAggregator();
|
|
129
|
+
const toolReliability = new ToolReliabilityAggregator();
|
|
130
|
+
const executionTime = new ExecutionTimeAggregator();
|
|
56
131
|
const tracker = new TimestampTracker();
|
|
57
132
|
let invalidJsonLines = 0;
|
|
133
|
+
// Track agent phase timing
|
|
134
|
+
let agentPhaseStart = null;
|
|
135
|
+
let lastPhase = 'unknown';
|
|
58
136
|
for await (const line of lines) {
|
|
59
137
|
if (!line.trim())
|
|
60
138
|
continue;
|
|
@@ -80,11 +158,31 @@ async function main() {
|
|
|
80
158
|
// Record assistant event type
|
|
81
159
|
const assistantType = event.assistantMessageEvent?.type;
|
|
82
160
|
aggregator.recordAssistantEventType(assistantType);
|
|
83
|
-
// Track
|
|
84
|
-
|
|
161
|
+
// Track agent timing (API invocation time)
|
|
162
|
+
const timestampSecs = extractTimestampSeconds(event);
|
|
163
|
+
if (isAgentStart(event)) {
|
|
164
|
+
const phase = extractPhase(event);
|
|
165
|
+
lastPhase = phase;
|
|
166
|
+
agentPhaseStart = timestampSecs;
|
|
167
|
+
}
|
|
168
|
+
else if (isAgentEnd(event) && agentPhaseStart !== null && timestampSecs !== null) {
|
|
169
|
+
const duration = timestampSecs - agentPhaseStart;
|
|
170
|
+
if (duration >= 0) {
|
|
171
|
+
executionTime.recordApiCall(lastPhase, duration);
|
|
172
|
+
}
|
|
173
|
+
agentPhaseStart = null;
|
|
174
|
+
}
|
|
175
|
+
// Track tool executions with reliability metrics
|
|
176
|
+
if (event.type === 'tool_execution_start') {
|
|
85
177
|
aggregator.recordToolStart();
|
|
86
|
-
|
|
178
|
+
const toolName = extractToolName(event);
|
|
179
|
+
toolReliability.recordToolStart(toolName);
|
|
180
|
+
}
|
|
181
|
+
if (event.type === 'tool_execution_end') {
|
|
87
182
|
aggregator.recordToolEnd();
|
|
183
|
+
const toolOutput = extractToolOutput(event);
|
|
184
|
+
toolReliability.recordToolEnd(toolOutput);
|
|
185
|
+
}
|
|
88
186
|
// Write event if it should be kept
|
|
89
187
|
if (shouldKeep(event)) {
|
|
90
188
|
const canContinue = output.write(`${JSON.stringify(sanitize(event))}\n`);
|
|
@@ -100,6 +198,11 @@ async function main() {
|
|
|
100
198
|
invalid_json_lines: invalidJsonLines,
|
|
101
199
|
first_event_at: tracker.firstEpochMs() !== null ? new Date(tracker.firstEpochMs()).toISOString() : tracker.firstTimestamp(),
|
|
102
200
|
last_event_at: tracker.lastEpochMs() !== null ? new Date(tracker.lastEpochMs()).toISOString() : tracker.lastTimestamp(),
|
|
201
|
+
tool_reliability: toolReliability.getSummary(),
|
|
202
|
+
tool_stats: toolReliability.getToolStats(),
|
|
203
|
+
execution_time: executionTime.getSummary(),
|
|
204
|
+
execution_api_stats: executionTime.getApiStats(),
|
|
205
|
+
execution_tool_stats: executionTime.getToolStats(),
|
|
103
206
|
};
|
|
104
207
|
fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
|
|
105
208
|
stopRssSampler();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pi-event-filter.js","sourceRoot":"","sources":["../src/pi-event-filter.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,QAAQ,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAW,MAAM,kCAAkC,CAAC;
|
|
1
|
+
{"version":3,"file":"pi-event-filter.js","sourceRoot":"","sources":["../src/pi-event-filter.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,QAAQ,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAqC,MAAM,kCAAkC,CAAC;AAChH,OAAO,EAAE,uBAAuB,EAAwC,MAAM,gCAAgC,CAAC;AAC/G,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAW,MAAM,kCAAkC,CAAC;AAuBlF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,0BAA0B,CAAC;AAChE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,0BAA0B,CAAC;AACnE,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,0BAA0B,CAAC;AAElE,IAAI,UAAU,GAA0B,IAAI,CAAC;AAC7C,IAAI,WAAW,GAAG,CAAC,CAAC;AAEpB,SAAS,eAAe;IACtB,IAAI,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,GAAG;QAAE,OAAO;IAC1D,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;IACxC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;QAC5B,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;IACjE,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,UAAU,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC;AAED,SAAS,cAAc;IACrB,IAAI,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,GAAG;QAAE,OAAO;IAC1D,IAAI,UAAU,EAAE,CAAC;QACf,aAAa,CAAC,UAAU,CAAC,CAAC;QAC1B,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;IACD,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,WAAW;CAClD,CAAC,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,MAAM,aAAa,GAAG,KAAK,CAAC,qBAAqB,EAAE,IAAI,CAAC;IACxD,IAAI,aAAa,EAAE,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IACzD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAY,CAAC;IAC1D,IAAI,IAAI,CAAC,qBAAqB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO;YACxC,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAC/C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,UAAU,CACpC,CAAC;IACN,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAChD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,UAAU,CACpC,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,KAAc;IACrC,gDAAgD;IAChD,IAAK,KAAa,CAAC,SAAS,EAAE,CAAC;QAC7B,OAAQ,KAAa,CAAC,SAAS,CAAC;IAClC,CAAC;IACD,yDAAyD;IACzD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,KAAc;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACzC,IAAI,IAAI,EAAE,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAAC,KAAc;IAC7C,MAAM,SAAS,GAAI,KAAa,CAAC,SAAS,CAAC;IAC3C,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAE5B,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,2CAA2C;QAC3C,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;YACrB,sBAAsB;YACtB,OAAO,SAAS,GAAG,IAAI,CAAC;QAC1B,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,kBAAkB;QAClB,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACzC,OAAO,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC;IACtC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;AACrE,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC;AACjE,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,wCAAwC;IACxC,oEAAoE;IACpE,MAAM,OAAO,GAAI,KAAa,CAAC,OAAO,IAAK,KAAa,CAAC,KAAK,IAAI,SAAS,CAAC;IAC5E,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,eAAe,EAAE,CAAC;IAClB,MAAM,KAAK,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,EAAE,CAAC,iBAAiB,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEvE,MAAM,UAAU,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAChD,MAAM,eAAe,GAAG,IAAI,yBAAyB,EAAE,CAAC;IACxD,MAAM,aAAa,GAAG,IAAI,uBAAuB,EAAE,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACvC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,2BAA2B;IAC3B,IAAI,eAAe,GAAkB,IAAI,CAAC;IAC1C,IAAI,SAAS,GAAG,SAAS,CAAC;IAE1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,IAAI,KAAc,CAAC;QACnB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB,EAAE,CAAC;YACnB,SAAS;QACX,CAAC;QAED,oBAAoB;QACpB,UAAU,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvC,kBAAkB;QAClB,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;QAED,oCAAoC;QACpC,UAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,UAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QACnE,UAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAEnE,8BAA8B;QAC9B,MAAM,aAAa,GAAG,KAAK,CAAC,qBAAqB,EAAE,IAAI,CAAC;QACxD,UAAU,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;QAEnD,2CAA2C;QAC3C,MAAM,aAAa,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAClC,SAAS,GAAG,KAAK,CAAC;YAClB,eAAe,GAAG,aAAa,CAAC;QAClC,CAAC;aAAM,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;YACnF,MAAM,QAAQ,GAAG,aAAa,GAAG,eAAe,CAAC;YACjD,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAClB,aAAa,CAAC,aAAa,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACnD,CAAC;YACD,eAAe,GAAG,IAAI,CAAC;QACzB,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YAC1C,UAAU,CAAC,eAAe,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YACxC,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YACxC,UAAU,CAAC,aAAa,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAC5C,eAAe,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;QAED,mCAAmC;QACnC,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;YACzE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAE1D,mBAAmB;IACnB,MAAM,OAAO,GAAY;QACvB,GAAG,UAAU,CAAC,OAAO,EAAE;QACvB,kBAAkB,EAAE,gBAAgB;QACpC,cAAc,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE;QAC5H,aAAa,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE;QACxH,gBAAgB,EAAE,eAAe,CAAC,UAAU,EAAE;QAC9C,UAAU,EAAE,eAAe,CAAC,YAAY,EAAE;QAC1C,cAAc,EAAE,aAAa,CAAC,UAAU,EAAE;QAC1C,mBAAmB,EAAE,aAAa,CAAC,WAAW,EAAE;QAChD,oBAAoB,EAAE,aAAa,CAAC,YAAY,EAAE;KACnD,CAAC;IAEF,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACvE,cAAc,EAAE,CAAC;AACnB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;IAC5B,cAAc,EAAE,CAAC;IACjB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-reliability-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Aggregates tool call success/failure metrics from Pi event streams.
|
|
5
|
+
* Detects failures using error pattern matching (same patterns as pi-progress-summarizer).
|
|
6
|
+
*/
|
|
7
|
+
export interface ToolReliabilitySummary {
|
|
8
|
+
total_tool_calls: number;
|
|
9
|
+
successful_tool_calls: number;
|
|
10
|
+
failed_tool_calls: number;
|
|
11
|
+
success_rate_percent: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ToolStats {
|
|
14
|
+
[toolName: string]: {
|
|
15
|
+
total: number;
|
|
16
|
+
successful: number;
|
|
17
|
+
failed: number;
|
|
18
|
+
success_rate_percent: number;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* ToolReliabilityAggregator tracks the success/failure rate of tool calls.
|
|
23
|
+
*
|
|
24
|
+
* It uses pattern-based error detection to classify tool outcomes:
|
|
25
|
+
* - Success: tool_execution_end with no error patterns in message
|
|
26
|
+
* - Failure: tool_execution_end with one or more error patterns detected
|
|
27
|
+
*
|
|
28
|
+
* Per-tool statistics are tracked separately from overall summary.
|
|
29
|
+
*/
|
|
30
|
+
export declare class ToolReliabilityAggregator {
|
|
31
|
+
private totalToolCalls;
|
|
32
|
+
private successfulToolCalls;
|
|
33
|
+
private failedToolCalls;
|
|
34
|
+
private toolStats;
|
|
35
|
+
private currentToolName;
|
|
36
|
+
/**
|
|
37
|
+
* Detect error patterns in a message string (case-insensitive).
|
|
38
|
+
*/
|
|
39
|
+
private detectError;
|
|
40
|
+
/**
|
|
41
|
+
* Record the start of a tool execution (with optional tool name).
|
|
42
|
+
*/
|
|
43
|
+
recordToolStart(toolName?: string | null): void;
|
|
44
|
+
/**
|
|
45
|
+
* Record the end of a tool execution with its output message.
|
|
46
|
+
* Analyzes the message for error patterns to determine success/failure.
|
|
47
|
+
*/
|
|
48
|
+
recordToolEnd(message: string | null | undefined): void;
|
|
49
|
+
/**
|
|
50
|
+
* Get the overall success/failure summary.
|
|
51
|
+
*/
|
|
52
|
+
getSummary(): ToolReliabilitySummary;
|
|
53
|
+
/**
|
|
54
|
+
* Get per-tool success rates.
|
|
55
|
+
*/
|
|
56
|
+
getToolStats(): ToolStats;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=tool-reliability-aggregator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-reliability-aggregator.d.ts","sourceRoot":"","sources":["../src/tool-reliability-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,oBAAoB,EAAE,MAAM,CAAC;KAC9B,CAAC;CACH;AAkBD;;;;;;;;GAQG;AACH,qBAAa,yBAAyB;IACpC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,eAAe,CAAK;IAG5B,OAAO,CAAC,SAAS,CAGH;IAEd,OAAO,CAAC,eAAe,CAAuB;IAE9C;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,eAAe,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,IAAI;IAIrD;;;OAGG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI;IA6BvD;;OAEG;IACH,UAAU,IAAI,sBAAsB;IAgBpC;;OAEG;IACH,YAAY,IAAI,SAAS;CAmB1B"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-reliability-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Aggregates tool call success/failure metrics from Pi event streams.
|
|
5
|
+
* Detects failures using error pattern matching (same patterns as pi-progress-summarizer).
|
|
6
|
+
*/
|
|
7
|
+
const ERROR_PATTERNS = [
|
|
8
|
+
'error',
|
|
9
|
+
'failed',
|
|
10
|
+
'failure',
|
|
11
|
+
'exception',
|
|
12
|
+
'cannot',
|
|
13
|
+
'unable to',
|
|
14
|
+
'invalid',
|
|
15
|
+
'undefined',
|
|
16
|
+
'null',
|
|
17
|
+
'not found',
|
|
18
|
+
'does not exist',
|
|
19
|
+
'exit code',
|
|
20
|
+
'exited with code',
|
|
21
|
+
];
|
|
22
|
+
/**
|
|
23
|
+
* ToolReliabilityAggregator tracks the success/failure rate of tool calls.
|
|
24
|
+
*
|
|
25
|
+
* It uses pattern-based error detection to classify tool outcomes:
|
|
26
|
+
* - Success: tool_execution_end with no error patterns in message
|
|
27
|
+
* - Failure: tool_execution_end with one or more error patterns detected
|
|
28
|
+
*
|
|
29
|
+
* Per-tool statistics are tracked separately from overall summary.
|
|
30
|
+
*/
|
|
31
|
+
export class ToolReliabilityAggregator {
|
|
32
|
+
totalToolCalls = 0;
|
|
33
|
+
successfulToolCalls = 0;
|
|
34
|
+
failedToolCalls = 0;
|
|
35
|
+
// Per-tool tracking: toolName -> { total, successful, failed }
|
|
36
|
+
toolStats = new Map();
|
|
37
|
+
currentToolName = null;
|
|
38
|
+
/**
|
|
39
|
+
* Detect error patterns in a message string (case-insensitive).
|
|
40
|
+
*/
|
|
41
|
+
detectError(message) {
|
|
42
|
+
if (!message || typeof message !== 'string')
|
|
43
|
+
return false;
|
|
44
|
+
const lowerMessage = message.toLowerCase();
|
|
45
|
+
return ERROR_PATTERNS.some((pattern) => lowerMessage.includes(pattern));
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Record the start of a tool execution (with optional tool name).
|
|
49
|
+
*/
|
|
50
|
+
recordToolStart(toolName = null) {
|
|
51
|
+
this.currentToolName = toolName || `tool_${this.totalToolCalls}`;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Record the end of a tool execution with its output message.
|
|
55
|
+
* Analyzes the message for error patterns to determine success/failure.
|
|
56
|
+
*/
|
|
57
|
+
recordToolEnd(message) {
|
|
58
|
+
const toolName = this.currentToolName || `tool_${this.totalToolCalls}`;
|
|
59
|
+
const isError = this.detectError(message);
|
|
60
|
+
this.totalToolCalls += 1;
|
|
61
|
+
if (isError) {
|
|
62
|
+
this.failedToolCalls += 1;
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
this.successfulToolCalls += 1;
|
|
66
|
+
}
|
|
67
|
+
// Track per-tool statistics
|
|
68
|
+
const existing = this.toolStats.get(toolName) || {
|
|
69
|
+
total: 0,
|
|
70
|
+
successful: 0,
|
|
71
|
+
failed: 0,
|
|
72
|
+
};
|
|
73
|
+
existing.total += 1;
|
|
74
|
+
if (isError) {
|
|
75
|
+
existing.failed += 1;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
existing.successful += 1;
|
|
79
|
+
}
|
|
80
|
+
this.toolStats.set(toolName, existing);
|
|
81
|
+
this.currentToolName = null;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Get the overall success/failure summary.
|
|
85
|
+
*/
|
|
86
|
+
getSummary() {
|
|
87
|
+
const successRate = this.totalToolCalls === 0
|
|
88
|
+
? 0
|
|
89
|
+
: Math.round((this.successfulToolCalls / this.totalToolCalls) * 10000) / 100;
|
|
90
|
+
return {
|
|
91
|
+
total_tool_calls: this.totalToolCalls,
|
|
92
|
+
successful_tool_calls: this.successfulToolCalls,
|
|
93
|
+
failed_tool_calls: this.failedToolCalls,
|
|
94
|
+
success_rate_percent: successRate,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Get per-tool success rates.
|
|
99
|
+
*/
|
|
100
|
+
getToolStats() {
|
|
101
|
+
const result = {};
|
|
102
|
+
for (const [toolName, stats] of this.toolStats.entries()) {
|
|
103
|
+
const successRate = stats.total === 0
|
|
104
|
+
? 0
|
|
105
|
+
: Math.round((stats.successful / stats.total) * 10000) / 100;
|
|
106
|
+
result[toolName] = {
|
|
107
|
+
total: stats.total,
|
|
108
|
+
successful: stats.successful,
|
|
109
|
+
failed: stats.failed,
|
|
110
|
+
success_rate_percent: successRate,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
return result;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=tool-reliability-aggregator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-reliability-aggregator.js","sourceRoot":"","sources":["../src/tool-reliability-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAkBH,MAAM,cAAc,GAAG;IACrB,OAAO;IACP,QAAQ;IACR,SAAS;IACT,WAAW;IACX,QAAQ;IACR,WAAW;IACX,SAAS;IACT,WAAW;IACX,MAAM;IACN,WAAW;IACX,gBAAgB;IAChB,WAAW;IACX,kBAAkB;CACnB,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,OAAO,yBAAyB;IAC5B,cAAc,GAAG,CAAC,CAAC;IACnB,mBAAmB,GAAG,CAAC,CAAC;IACxB,eAAe,GAAG,CAAC,CAAC;IAE5B,+DAA+D;IACvD,SAAS,GAGb,IAAI,GAAG,EAAE,CAAC;IAEN,eAAe,GAAkB,IAAI,CAAC;IAE9C;;OAEG;IACK,WAAW,CAAC,OAAkC;QACpD,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC1D,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAC3C,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,WAA0B,IAAI;QAC5C,IAAI,CAAC,eAAe,GAAG,QAAQ,IAAI,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,OAAkC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,IAAI,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QAEzB,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAChC,CAAC;QAED,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI;YAC/C,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,MAAM,EAAE,CAAC;SACV,CAAC;QACF,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpB,IAAI,OAAO,EAAE,CAAC;YACZ,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,UAAU;QACR,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,KAAK,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,KAAK,CACV,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,KAAK,CACzD,GAAG,GAAG,CAAC;QAEZ,OAAO;YACL,gBAAgB,EAAE,IAAI,CAAC,cAAc;YACrC,qBAAqB,EAAE,IAAI,CAAC,mBAAmB;YAC/C,iBAAiB,EAAE,IAAI,CAAC,eAAe;YACvC,oBAAoB,EAAE,WAAW;SAClC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,MAAM,GAAc,EAAE,CAAC;QAE7B,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,WAAW,GACf,KAAK,CAAC,KAAK,KAAK,CAAC;gBACf,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;YAEjE,MAAM,CAAC,QAAQ,CAAC,GAAG;gBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,oBAAoB,EAAE,WAAW;aAClC,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cyanautomation/kaseki-agent",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.62.0",
|
|
4
4
|
"description": "Admin/helper/doctor toolbox and local API client for Kaseki diagnostics, setup, and API-backed coding-agent task workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -213,30 +213,6 @@ normalize_secrets_dir() {
|
|
|
213
213
|
done
|
|
214
214
|
}
|
|
215
215
|
|
|
216
|
-
verify_permission_changes() {
|
|
217
|
-
local dir="$1" expected_owner="$2" expected_mode="$3"
|
|
218
|
-
local actual_owner actual_mode
|
|
219
|
-
|
|
220
|
-
if [ ! -d "$dir" ]; then
|
|
221
|
-
return 1
|
|
222
|
-
fi
|
|
223
|
-
|
|
224
|
-
actual_owner=$(stat -c '%U:%G' "$dir" 2>/dev/null || stat -f '%Su:%Sg' "$dir" 2>/dev/null || echo "unknown:unknown")
|
|
225
|
-
actual_mode=$(stat -c '%a' "$dir" 2>/dev/null || stat -f '%OLp' "$dir" 2>/dev/null | sed 's/^.*\([0-9]\{3\}\)$/\1/' || echo "unknown")
|
|
226
|
-
|
|
227
|
-
if [ "$actual_owner" != "$expected_owner" ]; then
|
|
228
|
-
printf 'warning: ownership mismatch for %s (actual: %s, expected: %s). May be on read-only mount.\n' "$dir" "$actual_owner" "$expected_owner" >&2
|
|
229
|
-
return 1
|
|
230
|
-
fi
|
|
231
|
-
|
|
232
|
-
if [ "$actual_mode" != "$expected_mode" ]; then
|
|
233
|
-
printf 'warning: permission mismatch for %s (actual: %s, expected: %s). May be on read-only mount.\n' "$dir" "$actual_mode" "$expected_mode" >&2
|
|
234
|
-
return 1
|
|
235
|
-
fi
|
|
236
|
-
|
|
237
|
-
return 0
|
|
238
|
-
}
|
|
239
|
-
|
|
240
216
|
run_checkout_freshness_probe() {
|
|
241
217
|
local checkout_dir="$1"
|
|
242
218
|
local probe_status="skipped"
|
|
@@ -277,22 +253,14 @@ run_checkout_freshness_probe() {
|
|
|
277
253
|
resolved_user_name="$(resolve_uid_to_name "$KASEKI_CONTAINER_UID")"
|
|
278
254
|
resolved_group_name="$(resolve_gid_to_name "$KASEKI_CONTAINER_GID")"
|
|
279
255
|
|
|
280
|
-
# Phase
|
|
281
|
-
#
|
|
256
|
+
# Phase 4: Use parallel privilege tool testing when running as root
|
|
257
|
+
# Runs setpriv, runuser, and sudo in parallel; returns on first success
|
|
258
|
+
# This reduces probe time from ~6 seconds (sequential 3×2s timeouts) to ~2 seconds (first success)
|
|
282
259
|
if [ "$(id -u)" -eq "$KASEKI_CONTAINER_UID" ] && [ "$(id -g)" -eq "$KASEKI_CONTAINER_GID" ]; then
|
|
283
260
|
"${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
284
|
-
elif [ "$(id -u)" -eq 0 ]
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" runuser -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
288
|
-
elif [ "$(id -u)" -eq 0 ] && command -v sudo >/dev/null 2>&1; then
|
|
289
|
-
if [ -n "$resolved_user_name" ] && [ -n "$resolved_group_name" ]; then
|
|
290
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
291
|
-
elif [ -n "$resolved_user_name" ]; then
|
|
292
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
293
|
-
else
|
|
294
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "#${KASEKI_CONTAINER_UID}" -g "#${KASEKI_CONTAINER_GID}" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
295
|
-
fi
|
|
261
|
+
elif [ "$(id -u)" -eq 0 ]; then
|
|
262
|
+
# Phase 4: Parallel privilege tool testing
|
|
263
|
+
run_privilege_tools_parallel "$checkout_dir" "${probe_command[@]}" "$stderr_file" "$resolved_user_name" "$resolved_group_name" || true
|
|
296
264
|
else
|
|
297
265
|
"${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
298
266
|
fi
|
|
@@ -375,26 +343,99 @@ write_host_state() {
|
|
|
375
343
|
printf 'ok: state file written to %s\n' "$state_file"
|
|
376
344
|
}
|
|
377
345
|
|
|
378
|
-
# Phase
|
|
379
|
-
|
|
380
|
-
|
|
346
|
+
# Phase 4: Parallel privilege tool testing helper
|
|
347
|
+
# Runs privilege tools in parallel and returns success on first success
|
|
348
|
+
run_privilege_tools_parallel() {
|
|
349
|
+
local checkout_dir="$1"
|
|
350
|
+
local probe_command=("${2[@]}")
|
|
351
|
+
local stderr_file="$3"
|
|
352
|
+
local resolved_user_name="$4"
|
|
353
|
+
local resolved_group_name="$5"
|
|
381
354
|
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
355
|
+
local temp_dir
|
|
356
|
+
temp_dir=$(mktemp -d)
|
|
357
|
+
local success_marker="$temp_dir/success"
|
|
358
|
+
local pids=()
|
|
359
|
+
|
|
360
|
+
cleanup_parallel() {
|
|
361
|
+
rm -rf "$temp_dir"
|
|
362
|
+
}
|
|
363
|
+
trap cleanup_parallel EXIT
|
|
364
|
+
|
|
365
|
+
# Test 1: setpriv (fastest, preferred)
|
|
366
|
+
if [ "$(id -u)" -eq 0 ] && command -v setpriv >/dev/null 2>&1; then
|
|
367
|
+
(
|
|
368
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" setpriv --reuid "$KASEKI_CONTAINER_UID" --regid "$KASEKI_CONTAINER_GID" --clear-groups -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" && touch "$success_marker"
|
|
369
|
+
) &
|
|
370
|
+
pids+=("$!")
|
|
371
|
+
fi
|
|
372
|
+
|
|
373
|
+
# Test 2: runuser (if resolved user/group available)
|
|
374
|
+
if [ "$(id -u)" -eq 0 ] && command -v runuser >/dev/null 2>&1 && [ -n "$resolved_user_name" ] && [ -n "$resolved_group_name" ]; then
|
|
375
|
+
(
|
|
376
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" runuser -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" && touch "$success_marker"
|
|
377
|
+
) &
|
|
378
|
+
pids+=("$!")
|
|
379
|
+
fi
|
|
380
|
+
|
|
381
|
+
# Test 3: sudo (fallback, slowest)
|
|
382
|
+
if [ "$(id -u)" -eq 0 ] && command -v sudo >/dev/null 2>&1; then
|
|
383
|
+
(
|
|
384
|
+
if [ -n "$resolved_user_name" ] && [ -n "$resolved_group_name" ]; then
|
|
385
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file"
|
|
386
|
+
elif [ -n "$resolved_user_name" ]; then
|
|
387
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file"
|
|
388
|
+
else
|
|
389
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "#${KASEKI_CONTAINER_UID}" -g "#${KASEKI_CONTAINER_GID}" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file"
|
|
390
|
+
fi && touch "$success_marker"
|
|
391
|
+
) &
|
|
392
|
+
pids+=("$!")
|
|
393
|
+
fi
|
|
394
|
+
|
|
395
|
+
# Wait for any process to succeed (check success marker while processes run)
|
|
396
|
+
local timeout_elapsed=0
|
|
397
|
+
local max_wait=$((KASEKI_PRIV_TOOL_TIMEOUT + 1))
|
|
398
|
+
while [ $timeout_elapsed -lt $max_wait ]; do
|
|
399
|
+
if [ -f "$success_marker" ]; then
|
|
400
|
+
# Kill remaining background processes
|
|
401
|
+
for pid in "${pids[@]}"; do
|
|
402
|
+
kill "$pid" 2>/dev/null || true
|
|
403
|
+
done
|
|
404
|
+
return 0
|
|
405
|
+
fi
|
|
406
|
+
sleep 0.1
|
|
407
|
+
timeout_elapsed=$(( timeout_elapsed + 1 ))
|
|
408
|
+
done
|
|
409
|
+
|
|
410
|
+
# Wait for all processes to complete
|
|
411
|
+
for pid in "${pids[@]}"; do
|
|
412
|
+
wait "$pid" 2>/dev/null || true
|
|
413
|
+
done
|
|
414
|
+
|
|
415
|
+
# Check if any succeeded
|
|
416
|
+
[ -f "$success_marker" ] && return 0
|
|
417
|
+
return 1
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
# Phase 4: Performance tracking helpers
|
|
421
|
+
track_stage_start() {
|
|
422
|
+
local stage="$1"
|
|
423
|
+
export "${stage}_start=$(date +%s%N)"
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
track_stage_end() {
|
|
427
|
+
local stage="$1"
|
|
428
|
+
local start_var="${stage}_start"
|
|
429
|
+
local start_time="${!start_var:-0}"
|
|
430
|
+
if [ "$start_time" -gt 0 ]; then
|
|
431
|
+
local end_time
|
|
432
|
+
end_time=$(date +%s%N)
|
|
433
|
+
local elapsed_ms=$(( (end_time - start_time) / 1000000 ))
|
|
434
|
+
export "${stage}_duration=$elapsed_ms"
|
|
394
435
|
fi
|
|
395
436
|
}
|
|
396
437
|
|
|
397
|
-
# Phase 3: Enhanced setup results with structured output
|
|
438
|
+
# Phase 3: Enhanced setup results with structured output (with Phase 4 timing)
|
|
398
439
|
write_setup_results_enhanced() {
|
|
399
440
|
local home_dir="$1"
|
|
400
441
|
local exit_code="$2"
|
|
@@ -417,6 +458,15 @@ write_setup_results_enhanced() {
|
|
|
417
458
|
local temp_file="${results_file}.tmp"
|
|
418
459
|
local status_name="ok"
|
|
419
460
|
[ "$exit_code" != "0" ] && status_name="failed"
|
|
461
|
+
|
|
462
|
+
# Phase 4: Collect timing metrics from stage tracking
|
|
463
|
+
local timing_obj="{}"
|
|
464
|
+
if [ -n "${STAGE_1_duration:-}" ]; then
|
|
465
|
+
timing_obj=$(echo "$timing_obj" | jq --arg k "stage_1_ms" --arg v "${STAGE_1_duration}" '. + {($k): ($v | tonumber)}' 2>/dev/null || echo "{}")
|
|
466
|
+
fi
|
|
467
|
+
if [ -n "${STAGE_6_duration:-}" ]; then
|
|
468
|
+
timing_obj=$(echo "$timing_obj" | jq --arg k "probe_duration_ms" --arg v "${STAGE_6_duration}" '. + {($k): ($v | tonumber)}' 2>/dev/null || echo "{}")
|
|
469
|
+
fi
|
|
420
470
|
|
|
421
471
|
jq -n \
|
|
422
472
|
--arg timestamp "$timestamp" \
|
|
@@ -427,6 +477,7 @@ write_setup_results_enhanced() {
|
|
|
427
477
|
--arg version "2" \
|
|
428
478
|
--arg probe_status "$probe_status" \
|
|
429
479
|
--arg template_status "$template_status" \
|
|
480
|
+
--argjson timing "$timing_obj" \
|
|
430
481
|
'{
|
|
431
482
|
timestamp: $timestamp,
|
|
432
483
|
mode: $mode,
|
|
@@ -437,7 +488,8 @@ write_setup_results_enhanced() {
|
|
|
437
488
|
checks: {
|
|
438
489
|
checkout_freshness_probe: $probe_status,
|
|
439
490
|
template_ready: $template_status
|
|
440
|
-
}
|
|
491
|
+
},
|
|
492
|
+
performance: $timing
|
|
441
493
|
}' > "$temp_file"
|
|
442
494
|
|
|
443
495
|
chmod 0644 "$temp_file"
|
|
@@ -643,7 +695,9 @@ status=0
|
|
|
643
695
|
|
|
644
696
|
# Phase 1: Host Prerequisites validation
|
|
645
697
|
log_info "Stage 1: Host Prerequisites"
|
|
698
|
+
track_stage_start "STAGE_1"
|
|
646
699
|
validate_host_prerequisites || status=$?
|
|
700
|
+
track_stage_end "STAGE_1"
|
|
647
701
|
echo ""
|
|
648
702
|
|
|
649
703
|
# Early exit if prerequisites fail in check-only mode
|
|
@@ -685,7 +739,9 @@ fi
|
|
|
685
739
|
|
|
686
740
|
# Checkout freshness probe (Phase 2: used to conditionally run bootstrap)
|
|
687
741
|
log_info "Stage 6: Checkout freshness probe"
|
|
742
|
+
track_stage_start "STAGE_6"
|
|
688
743
|
probe_payload="$(run_checkout_freshness_probe "$KASEKI_CHECKOUT_DIR")"
|
|
744
|
+
track_stage_end "STAGE_6"
|
|
689
745
|
IFS="|" read -r checkout_probe_status checkout_probe_detail checkout_probe_remediation <<< "$probe_payload"
|
|
690
746
|
printf "checkout-freshness-probe: %s\n" "$checkout_probe_status"
|
|
691
747
|
printf "%s\n" "$checkout_probe_detail"
|