@cyanautomation/kaseki-agent 1.61.0 → 1.62.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/execution-time-aggregator.d.ts +58 -0
- package/dist/execution-time-aggregator.d.ts.map +1 -0
- package/dist/execution-time-aggregator.js +92 -0
- package/dist/execution-time-aggregator.js.map +1 -0
- package/dist/lib/event-timestamp-helpers.d.ts +3 -0
- package/dist/lib/event-timestamp-helpers.d.ts.map +1 -1
- package/dist/pi-event-filter.js +106 -3
- package/dist/pi-event-filter.js.map +1 -1
- package/dist/tool-reliability-aggregator.d.ts +58 -0
- package/dist/tool-reliability-aggregator.d.ts.map +1 -0
- package/dist/tool-reliability-aggregator.js +116 -0
- package/dist/tool-reliability-aggregator.js.map +1 -0
- package/package.json +1 -1
- package/scripts/kaseki-setup-host.sh +112 -55
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* execution-time-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Tracks API time vs tool execution time to identify performance bottlenecks.
|
|
5
|
+
* Useful for understanding whether kaseki runs are bottlenecked by LLM API calls
|
|
6
|
+
* or external tool execution (git, npm, validation commands).
|
|
7
|
+
*/
|
|
8
|
+
export interface ExecutionTimeSummary {
|
|
9
|
+
api_time_seconds: number;
|
|
10
|
+
tool_time_seconds: number;
|
|
11
|
+
total_time_seconds: number;
|
|
12
|
+
api_percent: number;
|
|
13
|
+
tool_percent: number;
|
|
14
|
+
}
|
|
15
|
+
export interface ExecutionStats {
|
|
16
|
+
[identifier: string]: {
|
|
17
|
+
calls: number;
|
|
18
|
+
total_seconds: number;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* ExecutionTimeAggregator tracks wall time spent in two categories:
|
|
23
|
+
* 1. API calls (Pi agent invocations, scouting, goal-check, etc.)
|
|
24
|
+
* 2. Tool execution (npm, git, bash, validation commands)
|
|
25
|
+
*
|
|
26
|
+
* This breakdown helps identify if the bottleneck is API latency or tool execution.
|
|
27
|
+
*/
|
|
28
|
+
export declare class ExecutionTimeAggregator {
|
|
29
|
+
private apiTimeSeconds;
|
|
30
|
+
private toolTimeSeconds;
|
|
31
|
+
private apiStats;
|
|
32
|
+
private toolStats;
|
|
33
|
+
/**
|
|
34
|
+
* Record an API call duration.
|
|
35
|
+
* @param apiName - Identifier for the API call (e.g., 'pi-agent', 'pi-scouting')
|
|
36
|
+
* @param durationSeconds - Duration in seconds
|
|
37
|
+
*/
|
|
38
|
+
recordApiCall(apiName: string, durationSeconds: number): void;
|
|
39
|
+
/**
|
|
40
|
+
* Record a tool execution duration.
|
|
41
|
+
* @param toolName - Identifier for the tool (e.g., 'npm test', 'git clone')
|
|
42
|
+
* @param durationSeconds - Duration in seconds
|
|
43
|
+
*/
|
|
44
|
+
recordToolExecution(toolName: string, durationSeconds: number): void;
|
|
45
|
+
/**
|
|
46
|
+
* Get the overall execution time summary.
|
|
47
|
+
*/
|
|
48
|
+
getSummary(): ExecutionTimeSummary;
|
|
49
|
+
/**
|
|
50
|
+
* Get per-API statistics.
|
|
51
|
+
*/
|
|
52
|
+
getApiStats(): ExecutionStats;
|
|
53
|
+
/**
|
|
54
|
+
* Get per-tool statistics.
|
|
55
|
+
*/
|
|
56
|
+
getToolStats(): ExecutionStats;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=execution-time-aggregator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execution-time-aggregator.d.ts","sourceRoot":"","sources":["../src/execution-time-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,UAAU,EAAE,MAAM,GAAG;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED;;;;;;GAMG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,eAAe,CAAK;IAG5B,OAAO,CAAC,QAAQ,CACJ;IAGZ,OAAO,CAAC,SAAS,CACL;IAEZ;;;;OAIG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI;IAY7D;;;;OAIG;IACH,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI;IAYpE;;OAEG;IACH,UAAU,IAAI,oBAAoB;IAoBlC;;OAEG;IACH,WAAW,IAAI,cAAc;IAU7B;;OAEG;IACH,YAAY,IAAI,cAAc;CAS/B"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* execution-time-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Tracks API time vs tool execution time to identify performance bottlenecks.
|
|
5
|
+
* Useful for understanding whether kaseki runs are bottlenecked by LLM API calls
|
|
6
|
+
* or external tool execution (git, npm, validation commands).
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* ExecutionTimeAggregator tracks wall time spent in two categories:
|
|
10
|
+
* 1. API calls (Pi agent invocations, scouting, goal-check, etc.)
|
|
11
|
+
* 2. Tool execution (npm, git, bash, validation commands)
|
|
12
|
+
*
|
|
13
|
+
* This breakdown helps identify if the bottleneck is API latency or tool execution.
|
|
14
|
+
*/
|
|
15
|
+
export class ExecutionTimeAggregator {
|
|
16
|
+
apiTimeSeconds = 0;
|
|
17
|
+
toolTimeSeconds = 0;
|
|
18
|
+
// Per-API tracking: apiName -> { calls, total_seconds }
|
|
19
|
+
apiStats = new Map();
|
|
20
|
+
// Per-tool tracking: toolName -> { calls, total_seconds }
|
|
21
|
+
toolStats = new Map();
|
|
22
|
+
/**
|
|
23
|
+
* Record an API call duration.
|
|
24
|
+
* @param apiName - Identifier for the API call (e.g., 'pi-agent', 'pi-scouting')
|
|
25
|
+
* @param durationSeconds - Duration in seconds
|
|
26
|
+
*/
|
|
27
|
+
recordApiCall(apiName, durationSeconds) {
|
|
28
|
+
this.apiTimeSeconds += durationSeconds;
|
|
29
|
+
const existing = this.apiStats.get(apiName) || {
|
|
30
|
+
calls: 0,
|
|
31
|
+
total_seconds: 0,
|
|
32
|
+
};
|
|
33
|
+
existing.calls += 1;
|
|
34
|
+
existing.total_seconds += durationSeconds;
|
|
35
|
+
this.apiStats.set(apiName, existing);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Record a tool execution duration.
|
|
39
|
+
* @param toolName - Identifier for the tool (e.g., 'npm test', 'git clone')
|
|
40
|
+
* @param durationSeconds - Duration in seconds
|
|
41
|
+
*/
|
|
42
|
+
recordToolExecution(toolName, durationSeconds) {
|
|
43
|
+
this.toolTimeSeconds += durationSeconds;
|
|
44
|
+
const existing = this.toolStats.get(toolName) || {
|
|
45
|
+
calls: 0,
|
|
46
|
+
total_seconds: 0,
|
|
47
|
+
};
|
|
48
|
+
existing.calls += 1;
|
|
49
|
+
existing.total_seconds += durationSeconds;
|
|
50
|
+
this.toolStats.set(toolName, existing);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Get the overall execution time summary.
|
|
54
|
+
*/
|
|
55
|
+
getSummary() {
|
|
56
|
+
const totalSeconds = this.apiTimeSeconds + this.toolTimeSeconds;
|
|
57
|
+
const apiPercent = totalSeconds === 0
|
|
58
|
+
? 0
|
|
59
|
+
: Math.round((this.apiTimeSeconds / totalSeconds) * 10000) / 100;
|
|
60
|
+
const toolPercent = totalSeconds === 0
|
|
61
|
+
? 0
|
|
62
|
+
: Math.round((this.toolTimeSeconds / totalSeconds) * 10000) / 100;
|
|
63
|
+
return {
|
|
64
|
+
api_time_seconds: this.apiTimeSeconds,
|
|
65
|
+
tool_time_seconds: this.toolTimeSeconds,
|
|
66
|
+
total_time_seconds: totalSeconds,
|
|
67
|
+
api_percent: apiPercent,
|
|
68
|
+
tool_percent: toolPercent,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Get per-API statistics.
|
|
73
|
+
*/
|
|
74
|
+
getApiStats() {
|
|
75
|
+
const result = {};
|
|
76
|
+
for (const [apiName, stats] of this.apiStats.entries()) {
|
|
77
|
+
result[apiName] = { ...stats };
|
|
78
|
+
}
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Get per-tool statistics.
|
|
83
|
+
*/
|
|
84
|
+
getToolStats() {
|
|
85
|
+
const result = {};
|
|
86
|
+
for (const [toolName, stats] of this.toolStats.entries()) {
|
|
87
|
+
result[toolName] = { ...stats };
|
|
88
|
+
}
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=execution-time-aggregator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execution-time-aggregator.js","sourceRoot":"","sources":["../src/execution-time-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAiBH;;;;;;GAMG;AACH,MAAM,OAAO,uBAAuB;IAC1B,cAAc,GAAG,CAAC,CAAC;IACnB,eAAe,GAAG,CAAC,CAAC;IAE5B,wDAAwD;IAChD,QAAQ,GACd,IAAI,GAAG,EAAE,CAAC;IAEZ,0DAA0D;IAClD,SAAS,GACf,IAAI,GAAG,EAAE,CAAC;IAEZ;;;;OAIG;IACH,aAAa,CAAC,OAAe,EAAE,eAAuB;QACpD,IAAI,CAAC,cAAc,IAAI,eAAe,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI;YAC7C,KAAK,EAAE,CAAC;YACR,aAAa,EAAE,CAAC;SACjB,CAAC;QACF,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,aAAa,IAAI,eAAe,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACH,mBAAmB,CAAC,QAAgB,EAAE,eAAuB;QAC3D,IAAI,CAAC,eAAe,IAAI,eAAe,CAAC;QAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI;YAC/C,KAAK,EAAE,CAAC;YACR,aAAa,EAAE,CAAC;SACjB,CAAC;QACF,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,aAAa,IAAI,eAAe,CAAC;QAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,UAAU;QACR,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC;QAChE,MAAM,UAAU,GACd,YAAY,KAAK,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;QACrE,MAAM,WAAW,GACf,YAAY,KAAK,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,GAAG,YAAY,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;QAEtE,OAAO;YACL,gBAAgB,EAAE,IAAI,CAAC,cAAc;YACrC,iBAAiB,EAAE,IAAI,CAAC,eAAe;YACvC,kBAAkB,EAAE,YAAY;YAChC,WAAW,EAAE,UAAU;YACvB,YAAY,EAAE,WAAW;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,WAAW;QACT,MAAM,MAAM,GAAmB,EAAE,CAAC;QAElC,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,MAAM,GAAmB,EAAE,CAAC;QAElC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAClC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -17,6 +17,7 @@ export interface PiEvent {
|
|
|
17
17
|
timestamp?: string | number;
|
|
18
18
|
content?: Array<{
|
|
19
19
|
type: string;
|
|
20
|
+
text?: string;
|
|
20
21
|
}>;
|
|
21
22
|
};
|
|
22
23
|
assistantMessageEvent?: {
|
|
@@ -27,6 +28,7 @@ export interface PiEvent {
|
|
|
27
28
|
timestamp?: string | number;
|
|
28
29
|
content?: Array<{
|
|
29
30
|
type: string;
|
|
31
|
+
text?: string;
|
|
30
32
|
}>;
|
|
31
33
|
};
|
|
32
34
|
partial?: {
|
|
@@ -35,6 +37,7 @@ export interface PiEvent {
|
|
|
35
37
|
timestamp?: string | number;
|
|
36
38
|
content?: Array<{
|
|
37
39
|
type: string;
|
|
40
|
+
text?: string;
|
|
38
41
|
}>;
|
|
39
42
|
};
|
|
40
43
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event-timestamp-helpers.d.ts","sourceRoot":"","sources":["../../src/lib/event-timestamp-helpers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"event-timestamp-helpers.d.ts","sourceRoot":"","sources":["../../src/lib/event-timestamp-helpers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClD,CAAC;IACF,qBAAqB,CAAC,EAAE;QACtB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE;YACR,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,GAAG,CAAC,EAAE,MAAM,CAAC;YACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;YAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;SAClD,CAAC;QACF,OAAO,CAAC,EAAE;YACR,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,GAAG,CAAC,EAAE,MAAM,CAAC;YACb,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;YAC5B,OAAO,CAAC,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;SAClD,CAAC;KACH,CAAC;CACH;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAgBnE;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,GAAG,SAAS,CAiB3G;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAG3D"}
|
package/dist/pi-event-filter.js
CHANGED
|
@@ -3,6 +3,8 @@ import fs from 'node:fs';
|
|
|
3
3
|
import { once } from 'node:events';
|
|
4
4
|
import readline from 'node:readline';
|
|
5
5
|
import { EventCounterAggregator } from './event-aggregator.js';
|
|
6
|
+
import { ToolReliabilityAggregator } from './tool-reliability-aggregator.js';
|
|
7
|
+
import { ExecutionTimeAggregator } from './execution-time-aggregator.js';
|
|
6
8
|
import { TimestampTracker } from './timestamp-tracker.js';
|
|
7
9
|
import { extractEventTimestamp } from './lib/event-timestamp-helpers.js';
|
|
8
10
|
const inputPath = process.argv[2] ?? '/tmp/pi-events.raw.jsonl';
|
|
@@ -47,14 +49,90 @@ function sanitize(event) {
|
|
|
47
49
|
}
|
|
48
50
|
return copy;
|
|
49
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Extract tool name from a Pi event.
|
|
54
|
+
* Handles both tool_execution events and tool_call events.
|
|
55
|
+
*/
|
|
56
|
+
function extractToolName(event) {
|
|
57
|
+
// Handle tool_call events (e.g., hashline_edit)
|
|
58
|
+
if (event.tool_name) {
|
|
59
|
+
return event.tool_name;
|
|
60
|
+
}
|
|
61
|
+
// Could extract from message content in future if needed
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Extract message content from a tool_execution_end event for error detection.
|
|
66
|
+
* Aggregates output_text parts and visible content.
|
|
67
|
+
*/
|
|
68
|
+
function extractToolOutput(event) {
|
|
69
|
+
const parts = [];
|
|
70
|
+
if (event.message?.content && Array.isArray(event.message.content)) {
|
|
71
|
+
for (const part of event.message.content) {
|
|
72
|
+
if (part?.type === 'output_text' && part.text) {
|
|
73
|
+
parts.push(part.text);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return parts.join(' ');
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Extract Unix timestamp in seconds from event timestamp.
|
|
81
|
+
* Handles both ISO strings and Unix epoch numbers.
|
|
82
|
+
*/
|
|
83
|
+
function extractTimestampSeconds(event) {
|
|
84
|
+
const timestamp = event.timestamp;
|
|
85
|
+
if (!timestamp)
|
|
86
|
+
return null;
|
|
87
|
+
if (typeof timestamp === 'number') {
|
|
88
|
+
// Already a Unix timestamp (ms or seconds)
|
|
89
|
+
if (timestamp > 1e10) {
|
|
90
|
+
// Likely milliseconds
|
|
91
|
+
return timestamp / 1000;
|
|
92
|
+
}
|
|
93
|
+
return timestamp;
|
|
94
|
+
}
|
|
95
|
+
if (typeof timestamp === 'string') {
|
|
96
|
+
// ISO 8601 string
|
|
97
|
+
const ms = new Date(timestamp).getTime();
|
|
98
|
+
return isNaN(ms) ? null : ms / 1000;
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Detect if this event marks the start of an agent invocation.
|
|
104
|
+
*/
|
|
105
|
+
function isAgentStart(event) {
|
|
106
|
+
return event.type === 'agent_start' || event.type === 'agentstart';
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Detect if this event marks the end of an agent invocation.
|
|
110
|
+
*/
|
|
111
|
+
function isAgentEnd(event) {
|
|
112
|
+
return event.type === 'agent_end' || event.type === 'agentend';
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Extract the phase name from an agent event (e.g., from assistantMessageEvent context).
|
|
116
|
+
*/
|
|
117
|
+
function extractPhase(event) {
|
|
118
|
+
// Try to infer phase from event context
|
|
119
|
+
// This is a heuristic; actual phase names come from kaseki-agent.sh
|
|
120
|
+
const context = event.context || event.phase || 'unknown';
|
|
121
|
+
return typeof context === 'string' ? context : 'unknown';
|
|
122
|
+
}
|
|
50
123
|
async function main() {
|
|
51
124
|
startRssSampler();
|
|
52
125
|
const input = fs.createReadStream(inputPath, { encoding: 'utf8' });
|
|
53
126
|
const output = fs.createWriteStream(filteredPath, { encoding: 'utf8' });
|
|
54
127
|
const lines = readline.createInterface({ input, crlfDelay: Infinity });
|
|
55
128
|
const aggregator = new EventCounterAggregator();
|
|
129
|
+
const toolReliability = new ToolReliabilityAggregator();
|
|
130
|
+
const executionTime = new ExecutionTimeAggregator();
|
|
56
131
|
const tracker = new TimestampTracker();
|
|
57
132
|
let invalidJsonLines = 0;
|
|
133
|
+
// Track agent phase timing
|
|
134
|
+
let agentPhaseStart = null;
|
|
135
|
+
let lastPhase = 'unknown';
|
|
58
136
|
for await (const line of lines) {
|
|
59
137
|
if (!line.trim())
|
|
60
138
|
continue;
|
|
@@ -80,11 +158,31 @@ async function main() {
|
|
|
80
158
|
// Record assistant event type
|
|
81
159
|
const assistantType = event.assistantMessageEvent?.type;
|
|
82
160
|
aggregator.recordAssistantEventType(assistantType);
|
|
83
|
-
// Track
|
|
84
|
-
|
|
161
|
+
// Track agent timing (API invocation time)
|
|
162
|
+
const timestampSecs = extractTimestampSeconds(event);
|
|
163
|
+
if (isAgentStart(event)) {
|
|
164
|
+
const phase = extractPhase(event);
|
|
165
|
+
lastPhase = phase;
|
|
166
|
+
agentPhaseStart = timestampSecs;
|
|
167
|
+
}
|
|
168
|
+
else if (isAgentEnd(event) && agentPhaseStart !== null && timestampSecs !== null) {
|
|
169
|
+
const duration = timestampSecs - agentPhaseStart;
|
|
170
|
+
if (duration >= 0) {
|
|
171
|
+
executionTime.recordApiCall(lastPhase, duration);
|
|
172
|
+
}
|
|
173
|
+
agentPhaseStart = null;
|
|
174
|
+
}
|
|
175
|
+
// Track tool executions with reliability metrics
|
|
176
|
+
if (event.type === 'tool_execution_start') {
|
|
85
177
|
aggregator.recordToolStart();
|
|
86
|
-
|
|
178
|
+
const toolName = extractToolName(event);
|
|
179
|
+
toolReliability.recordToolStart(toolName);
|
|
180
|
+
}
|
|
181
|
+
if (event.type === 'tool_execution_end') {
|
|
87
182
|
aggregator.recordToolEnd();
|
|
183
|
+
const toolOutput = extractToolOutput(event);
|
|
184
|
+
toolReliability.recordToolEnd(toolOutput);
|
|
185
|
+
}
|
|
88
186
|
// Write event if it should be kept
|
|
89
187
|
if (shouldKeep(event)) {
|
|
90
188
|
const canContinue = output.write(`${JSON.stringify(sanitize(event))}\n`);
|
|
@@ -100,6 +198,11 @@ async function main() {
|
|
|
100
198
|
invalid_json_lines: invalidJsonLines,
|
|
101
199
|
first_event_at: tracker.firstEpochMs() !== null ? new Date(tracker.firstEpochMs()).toISOString() : tracker.firstTimestamp(),
|
|
102
200
|
last_event_at: tracker.lastEpochMs() !== null ? new Date(tracker.lastEpochMs()).toISOString() : tracker.lastTimestamp(),
|
|
201
|
+
tool_reliability: toolReliability.getSummary(),
|
|
202
|
+
tool_stats: toolReliability.getToolStats(),
|
|
203
|
+
execution_time: executionTime.getSummary(),
|
|
204
|
+
execution_api_stats: executionTime.getApiStats(),
|
|
205
|
+
execution_tool_stats: executionTime.getToolStats(),
|
|
103
206
|
};
|
|
104
207
|
fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
|
|
105
208
|
stopRssSampler();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pi-event-filter.js","sourceRoot":"","sources":["../src/pi-event-filter.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,QAAQ,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAW,MAAM,kCAAkC,CAAC;
|
|
1
|
+
{"version":3,"file":"pi-event-filter.js","sourceRoot":"","sources":["../src/pi-event-filter.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,QAAQ,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAqC,MAAM,kCAAkC,CAAC;AAChH,OAAO,EAAE,uBAAuB,EAAwC,MAAM,gCAAgC,CAAC;AAC/G,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAW,MAAM,kCAAkC,CAAC;AAuBlF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,0BAA0B,CAAC;AAChE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,0BAA0B,CAAC;AACnE,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,0BAA0B,CAAC;AAElE,IAAI,UAAU,GAA0B,IAAI,CAAC;AAC7C,IAAI,WAAW,GAAG,CAAC,CAAC;AAEpB,SAAS,eAAe;IACtB,IAAI,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,GAAG;QAAE,OAAO;IAC1D,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;IACxC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;QAC5B,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;IACjE,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,UAAU,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC;AAED,SAAS,cAAc;IACrB,IAAI,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,GAAG;QAAE,OAAO;IAC1D,IAAI,UAAU,EAAE,CAAC;QACf,aAAa,CAAC,UAAU,CAAC,CAAC;QAC1B,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;IACD,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,WAAW;CAClD,CAAC,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,MAAM,aAAa,GAAG,KAAK,CAAC,qBAAqB,EAAE,IAAI,CAAC;IACxD,IAAI,aAAa,EAAE,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IACzD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAY,CAAC;IAC1D,IAAI,IAAI,CAAC,qBAAqB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO;YACxC,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAC/C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,UAAU,CACpC,CAAC;IACN,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAChD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,UAAU,CACpC,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,KAAc;IACrC,gDAAgD;IAChD,IAAK,KAAa,CAAC,SAAS,EAAE,CAAC;QAC7B,OAAQ,KAAa,CAAC,SAAS,CAAC;IAClC,CAAC;IACD,yDAAyD;IACzD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,KAAc;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACzC,IAAI,IAAI,EAAE,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAAC,KAAc;IAC7C,MAAM,SAAS,GAAI,KAAa,CAAC,SAAS,CAAC;IAC3C,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAE5B,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,2CAA2C;QAC3C,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;YACrB,sBAAsB;YACtB,OAAO,SAAS,GAAG,IAAI,CAAC;QAC1B,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,kBAAkB;QAClB,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACzC,OAAO,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC;IACtC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;AACrE,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC;AACjE,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,wCAAwC;IACxC,oEAAoE;IACpE,MAAM,OAAO,GAAI,KAAa,CAAC,OAAO,IAAK,KAAa,CAAC,KAAK,IAAI,SAAS,CAAC;IAC5E,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,eAAe,EAAE,CAAC;IAClB,MAAM,KAAK,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,EAAE,CAAC,iBAAiB,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEvE,MAAM,UAAU,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAChD,MAAM,eAAe,GAAG,IAAI,yBAAyB,EAAE,CAAC;IACxD,MAAM,aAAa,GAAG,IAAI,uBAAuB,EAAE,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACvC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,2BAA2B;IAC3B,IAAI,eAAe,GAAkB,IAAI,CAAC;IAC1C,IAAI,SAAS,GAAG,SAAS,CAAC;IAE1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,IAAI,KAAc,CAAC;QACnB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB,EAAE,CAAC;YACnB,SAAS;QACX,CAAC;QAED,oBAAoB;QACpB,UAAU,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvC,kBAAkB;QAClB,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;QAED,oCAAoC;QACpC,UAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,UAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QACnE,UAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAEnE,8BAA8B;QAC9B,MAAM,aAAa,GAAG,KAAK,CAAC,qBAAqB,EAAE,IAAI,CAAC;QACxD,UAAU,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;QAEnD,2CAA2C;QAC3C,MAAM,aAAa,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAClC,SAAS,GAAG,KAAK,CAAC;YAClB,eAAe,GAAG,aAAa,CAAC;QAClC,CAAC;aAAM,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;YACnF,MAAM,QAAQ,GAAG,aAAa,GAAG,eAAe,CAAC;YACjD,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAClB,aAAa,CAAC,aAAa,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACnD,CAAC;YACD,eAAe,GAAG,IAAI,CAAC;QACzB,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YAC1C,UAAU,CAAC,eAAe,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YACxC,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YACxC,UAAU,CAAC,aAAa,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAC5C,eAAe,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;QAED,mCAAmC;QACnC,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;YACzE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAE1D,mBAAmB;IACnB,MAAM,OAAO,GAAY;QACvB,GAAG,UAAU,CAAC,OAAO,EAAE;QACvB,kBAAkB,EAAE,gBAAgB;QACpC,cAAc,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE;QAC5H,aAAa,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE;QACxH,gBAAgB,EAAE,eAAe,CAAC,UAAU,EAAE;QAC9C,UAAU,EAAE,eAAe,CAAC,YAAY,EAAE;QAC1C,cAAc,EAAE,aAAa,CAAC,UAAU,EAAE;QAC1C,mBAAmB,EAAE,aAAa,CAAC,WAAW,EAAE;QAChD,oBAAoB,EAAE,aAAa,CAAC,YAAY,EAAE;KACnD,CAAC;IAEF,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACvE,cAAc,EAAE,CAAC;AACnB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;IAC5B,cAAc,EAAE,CAAC;IACjB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-reliability-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Aggregates tool call success/failure metrics from Pi event streams.
|
|
5
|
+
* Detects failures using error pattern matching (same patterns as pi-progress-summarizer).
|
|
6
|
+
*/
|
|
7
|
+
export interface ToolReliabilitySummary {
|
|
8
|
+
total_tool_calls: number;
|
|
9
|
+
successful_tool_calls: number;
|
|
10
|
+
failed_tool_calls: number;
|
|
11
|
+
success_rate_percent: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ToolStats {
|
|
14
|
+
[toolName: string]: {
|
|
15
|
+
total: number;
|
|
16
|
+
successful: number;
|
|
17
|
+
failed: number;
|
|
18
|
+
success_rate_percent: number;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* ToolReliabilityAggregator tracks the success/failure rate of tool calls.
|
|
23
|
+
*
|
|
24
|
+
* It uses pattern-based error detection to classify tool outcomes:
|
|
25
|
+
* - Success: tool_execution_end with no error patterns in message
|
|
26
|
+
* - Failure: tool_execution_end with one or more error patterns detected
|
|
27
|
+
*
|
|
28
|
+
* Per-tool statistics are tracked separately from overall summary.
|
|
29
|
+
*/
|
|
30
|
+
export declare class ToolReliabilityAggregator {
|
|
31
|
+
private totalToolCalls;
|
|
32
|
+
private successfulToolCalls;
|
|
33
|
+
private failedToolCalls;
|
|
34
|
+
private toolStats;
|
|
35
|
+
private currentToolName;
|
|
36
|
+
/**
|
|
37
|
+
* Detect error patterns in a message string (case-insensitive).
|
|
38
|
+
*/
|
|
39
|
+
private detectError;
|
|
40
|
+
/**
|
|
41
|
+
* Record the start of a tool execution (with optional tool name).
|
|
42
|
+
*/
|
|
43
|
+
recordToolStart(toolName?: string | null): void;
|
|
44
|
+
/**
|
|
45
|
+
* Record the end of a tool execution with its output message.
|
|
46
|
+
* Analyzes the message for error patterns to determine success/failure.
|
|
47
|
+
*/
|
|
48
|
+
recordToolEnd(message: string | null | undefined): void;
|
|
49
|
+
/**
|
|
50
|
+
* Get the overall success/failure summary.
|
|
51
|
+
*/
|
|
52
|
+
getSummary(): ToolReliabilitySummary;
|
|
53
|
+
/**
|
|
54
|
+
* Get per-tool success rates.
|
|
55
|
+
*/
|
|
56
|
+
getToolStats(): ToolStats;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=tool-reliability-aggregator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-reliability-aggregator.d.ts","sourceRoot":"","sources":["../src/tool-reliability-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,oBAAoB,EAAE,MAAM,CAAC;KAC9B,CAAC;CACH;AAkBD;;;;;;;;GAQG;AACH,qBAAa,yBAAyB;IACpC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,eAAe,CAAK;IAG5B,OAAO,CAAC,SAAS,CAGH;IAEd,OAAO,CAAC,eAAe,CAAuB;IAE9C;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,eAAe,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,IAAI;IAIrD;;;OAGG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI;IA6BvD;;OAEG;IACH,UAAU,IAAI,sBAAsB;IAgBpC;;OAEG;IACH,YAAY,IAAI,SAAS;CAmB1B"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-reliability-aggregator.ts
|
|
3
|
+
*
|
|
4
|
+
* Aggregates tool call success/failure metrics from Pi event streams.
|
|
5
|
+
* Detects failures using error pattern matching (same patterns as pi-progress-summarizer).
|
|
6
|
+
*/
|
|
7
|
+
const ERROR_PATTERNS = [
|
|
8
|
+
'error',
|
|
9
|
+
'failed',
|
|
10
|
+
'failure',
|
|
11
|
+
'exception',
|
|
12
|
+
'cannot',
|
|
13
|
+
'unable to',
|
|
14
|
+
'invalid',
|
|
15
|
+
'undefined',
|
|
16
|
+
'null',
|
|
17
|
+
'not found',
|
|
18
|
+
'does not exist',
|
|
19
|
+
'exit code',
|
|
20
|
+
'exited with code',
|
|
21
|
+
];
|
|
22
|
+
/**
|
|
23
|
+
* ToolReliabilityAggregator tracks the success/failure rate of tool calls.
|
|
24
|
+
*
|
|
25
|
+
* It uses pattern-based error detection to classify tool outcomes:
|
|
26
|
+
* - Success: tool_execution_end with no error patterns in message
|
|
27
|
+
* - Failure: tool_execution_end with one or more error patterns detected
|
|
28
|
+
*
|
|
29
|
+
* Per-tool statistics are tracked separately from overall summary.
|
|
30
|
+
*/
|
|
31
|
+
export class ToolReliabilityAggregator {
|
|
32
|
+
totalToolCalls = 0;
|
|
33
|
+
successfulToolCalls = 0;
|
|
34
|
+
failedToolCalls = 0;
|
|
35
|
+
// Per-tool tracking: toolName -> { total, successful, failed }
|
|
36
|
+
toolStats = new Map();
|
|
37
|
+
currentToolName = null;
|
|
38
|
+
/**
|
|
39
|
+
* Detect error patterns in a message string (case-insensitive).
|
|
40
|
+
*/
|
|
41
|
+
detectError(message) {
|
|
42
|
+
if (!message || typeof message !== 'string')
|
|
43
|
+
return false;
|
|
44
|
+
const lowerMessage = message.toLowerCase();
|
|
45
|
+
return ERROR_PATTERNS.some((pattern) => lowerMessage.includes(pattern));
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Record the start of a tool execution (with optional tool name).
|
|
49
|
+
*/
|
|
50
|
+
recordToolStart(toolName = null) {
|
|
51
|
+
this.currentToolName = toolName || `tool_${this.totalToolCalls}`;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Record the end of a tool execution with its output message.
|
|
55
|
+
* Analyzes the message for error patterns to determine success/failure.
|
|
56
|
+
*/
|
|
57
|
+
recordToolEnd(message) {
|
|
58
|
+
const toolName = this.currentToolName || `tool_${this.totalToolCalls}`;
|
|
59
|
+
const isError = this.detectError(message);
|
|
60
|
+
this.totalToolCalls += 1;
|
|
61
|
+
if (isError) {
|
|
62
|
+
this.failedToolCalls += 1;
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
this.successfulToolCalls += 1;
|
|
66
|
+
}
|
|
67
|
+
// Track per-tool statistics
|
|
68
|
+
const existing = this.toolStats.get(toolName) || {
|
|
69
|
+
total: 0,
|
|
70
|
+
successful: 0,
|
|
71
|
+
failed: 0,
|
|
72
|
+
};
|
|
73
|
+
existing.total += 1;
|
|
74
|
+
if (isError) {
|
|
75
|
+
existing.failed += 1;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
existing.successful += 1;
|
|
79
|
+
}
|
|
80
|
+
this.toolStats.set(toolName, existing);
|
|
81
|
+
this.currentToolName = null;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Get the overall success/failure summary.
|
|
85
|
+
*/
|
|
86
|
+
getSummary() {
|
|
87
|
+
const successRate = this.totalToolCalls === 0
|
|
88
|
+
? 0
|
|
89
|
+
: Math.round((this.successfulToolCalls / this.totalToolCalls) * 10000) / 100;
|
|
90
|
+
return {
|
|
91
|
+
total_tool_calls: this.totalToolCalls,
|
|
92
|
+
successful_tool_calls: this.successfulToolCalls,
|
|
93
|
+
failed_tool_calls: this.failedToolCalls,
|
|
94
|
+
success_rate_percent: successRate,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Get per-tool success rates.
|
|
99
|
+
*/
|
|
100
|
+
getToolStats() {
|
|
101
|
+
const result = {};
|
|
102
|
+
for (const [toolName, stats] of this.toolStats.entries()) {
|
|
103
|
+
const successRate = stats.total === 0
|
|
104
|
+
? 0
|
|
105
|
+
: Math.round((stats.successful / stats.total) * 10000) / 100;
|
|
106
|
+
result[toolName] = {
|
|
107
|
+
total: stats.total,
|
|
108
|
+
successful: stats.successful,
|
|
109
|
+
failed: stats.failed,
|
|
110
|
+
success_rate_percent: successRate,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
return result;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=tool-reliability-aggregator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-reliability-aggregator.js","sourceRoot":"","sources":["../src/tool-reliability-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAkBH,MAAM,cAAc,GAAG;IACrB,OAAO;IACP,QAAQ;IACR,SAAS;IACT,WAAW;IACX,QAAQ;IACR,WAAW;IACX,SAAS;IACT,WAAW;IACX,MAAM;IACN,WAAW;IACX,gBAAgB;IAChB,WAAW;IACX,kBAAkB;CACnB,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,OAAO,yBAAyB;IAC5B,cAAc,GAAG,CAAC,CAAC;IACnB,mBAAmB,GAAG,CAAC,CAAC;IACxB,eAAe,GAAG,CAAC,CAAC;IAE5B,+DAA+D;IACvD,SAAS,GAGb,IAAI,GAAG,EAAE,CAAC;IAEN,eAAe,GAAkB,IAAI,CAAC;IAE9C;;OAEG;IACK,WAAW,CAAC,OAAkC;QACpD,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC1D,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAC3C,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,WAA0B,IAAI;QAC5C,IAAI,CAAC,eAAe,GAAG,QAAQ,IAAI,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,OAAkC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,IAAI,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QAEzB,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAChC,CAAC;QAED,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI;YAC/C,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,MAAM,EAAE,CAAC;SACV,CAAC;QACF,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpB,IAAI,OAAO,EAAE,CAAC;YACZ,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,UAAU;QACR,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,KAAK,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,KAAK,CACV,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,KAAK,CACzD,GAAG,GAAG,CAAC;QAEZ,OAAO;YACL,gBAAgB,EAAE,IAAI,CAAC,cAAc;YACrC,qBAAqB,EAAE,IAAI,CAAC,mBAAmB;YAC/C,iBAAiB,EAAE,IAAI,CAAC,eAAe;YACvC,oBAAoB,EAAE,WAAW;SAClC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,MAAM,GAAc,EAAE,CAAC;QAE7B,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,WAAW,GACf,KAAK,CAAC,KAAK,KAAK,CAAC;gBACf,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;YAEjE,MAAM,CAAC,QAAQ,CAAC,GAAG;gBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,oBAAoB,EAAE,WAAW;aAClC,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cyanautomation/kaseki-agent",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.62.1",
|
|
4
4
|
"description": "Admin/helper/doctor toolbox and local API client for Kaseki diagnostics, setup, and API-backed coding-agent task workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -213,30 +213,6 @@ normalize_secrets_dir() {
|
|
|
213
213
|
done
|
|
214
214
|
}
|
|
215
215
|
|
|
216
|
-
verify_permission_changes() {
|
|
217
|
-
local dir="$1" expected_owner="$2" expected_mode="$3"
|
|
218
|
-
local actual_owner actual_mode
|
|
219
|
-
|
|
220
|
-
if [ ! -d "$dir" ]; then
|
|
221
|
-
return 1
|
|
222
|
-
fi
|
|
223
|
-
|
|
224
|
-
actual_owner=$(stat -c '%U:%G' "$dir" 2>/dev/null || stat -f '%Su:%Sg' "$dir" 2>/dev/null || echo "unknown:unknown")
|
|
225
|
-
actual_mode=$(stat -c '%a' "$dir" 2>/dev/null || stat -f '%OLp' "$dir" 2>/dev/null | sed 's/^.*\([0-9]\{3\}\)$/\1/' || echo "unknown")
|
|
226
|
-
|
|
227
|
-
if [ "$actual_owner" != "$expected_owner" ]; then
|
|
228
|
-
printf 'warning: ownership mismatch for %s (actual: %s, expected: %s). May be on read-only mount.\n' "$dir" "$actual_owner" "$expected_owner" >&2
|
|
229
|
-
return 1
|
|
230
|
-
fi
|
|
231
|
-
|
|
232
|
-
if [ "$actual_mode" != "$expected_mode" ]; then
|
|
233
|
-
printf 'warning: permission mismatch for %s (actual: %s, expected: %s). May be on read-only mount.\n' "$dir" "$actual_mode" "$expected_mode" >&2
|
|
234
|
-
return 1
|
|
235
|
-
fi
|
|
236
|
-
|
|
237
|
-
return 0
|
|
238
|
-
}
|
|
239
|
-
|
|
240
216
|
run_checkout_freshness_probe() {
|
|
241
217
|
local checkout_dir="$1"
|
|
242
218
|
local probe_status="skipped"
|
|
@@ -277,22 +253,14 @@ run_checkout_freshness_probe() {
|
|
|
277
253
|
resolved_user_name="$(resolve_uid_to_name "$KASEKI_CONTAINER_UID")"
|
|
278
254
|
resolved_group_name="$(resolve_gid_to_name "$KASEKI_CONTAINER_GID")"
|
|
279
255
|
|
|
280
|
-
# Phase
|
|
281
|
-
#
|
|
256
|
+
# Phase 4: Use parallel privilege tool testing when running as root
|
|
257
|
+
# Runs setpriv, runuser, and sudo in parallel; returns on first success
|
|
258
|
+
# This reduces probe time from ~6 seconds (sequential 3×2s timeouts) to ~2 seconds (first success)
|
|
282
259
|
if [ "$(id -u)" -eq "$KASEKI_CONTAINER_UID" ] && [ "$(id -g)" -eq "$KASEKI_CONTAINER_GID" ]; then
|
|
283
260
|
"${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
284
|
-
elif [ "$(id -u)" -eq 0 ]
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" runuser -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
288
|
-
elif [ "$(id -u)" -eq 0 ] && command -v sudo >/dev/null 2>&1; then
|
|
289
|
-
if [ -n "$resolved_user_name" ] && [ -n "$resolved_group_name" ]; then
|
|
290
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
291
|
-
elif [ -n "$resolved_user_name" ]; then
|
|
292
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
293
|
-
else
|
|
294
|
-
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "#${KASEKI_CONTAINER_UID}" -g "#${KASEKI_CONTAINER_GID}" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
295
|
-
fi
|
|
261
|
+
elif [ "$(id -u)" -eq 0 ]; then
|
|
262
|
+
# Phase 4: Parallel privilege tool testing
|
|
263
|
+
run_privilege_tools_parallel "$checkout_dir" "$stderr_file" "$resolved_user_name" "$resolved_group_name" "${probe_command[@]}" || true
|
|
296
264
|
else
|
|
297
265
|
"${probe_command[@]}" >/dev/null 2>"$stderr_file" || true
|
|
298
266
|
fi
|
|
@@ -375,26 +343,100 @@ write_host_state() {
|
|
|
375
343
|
printf 'ok: state file written to %s\n' "$state_file"
|
|
376
344
|
}
|
|
377
345
|
|
|
378
|
-
# Phase
|
|
379
|
-
|
|
380
|
-
|
|
346
|
+
# Phase 4: Parallel privilege tool testing helper
|
|
347
|
+
# Runs privilege tools in parallel and returns success on first success
|
|
348
|
+
run_privilege_tools_parallel() {
|
|
349
|
+
local checkout_dir="$1"
|
|
350
|
+
local stderr_file="$2"
|
|
351
|
+
local resolved_user_name="$3"
|
|
352
|
+
local resolved_group_name="$4"
|
|
353
|
+
shift 4
|
|
354
|
+
local probe_command=("$@")
|
|
381
355
|
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
356
|
+
local temp_dir
|
|
357
|
+
temp_dir=$(mktemp -d)
|
|
358
|
+
local success_marker="$temp_dir/success"
|
|
359
|
+
local pids=()
|
|
360
|
+
|
|
361
|
+
cleanup_parallel() {
|
|
362
|
+
rm -rf "$temp_dir"
|
|
363
|
+
}
|
|
364
|
+
trap cleanup_parallel EXIT
|
|
365
|
+
|
|
366
|
+
# Test 1: setpriv (fastest, preferred)
|
|
367
|
+
if [ "$(id -u)" -eq 0 ] && command -v setpriv >/dev/null 2>&1; then
|
|
368
|
+
(
|
|
369
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" setpriv --reuid "$KASEKI_CONTAINER_UID" --regid "$KASEKI_CONTAINER_GID" --clear-groups -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" && touch "$success_marker"
|
|
370
|
+
) &
|
|
371
|
+
pids+=("$!")
|
|
372
|
+
fi
|
|
373
|
+
|
|
374
|
+
# Test 2: runuser (if resolved user/group available)
|
|
375
|
+
if [ "$(id -u)" -eq 0 ] && command -v runuser >/dev/null 2>&1 && [ -n "$resolved_user_name" ] && [ -n "$resolved_group_name" ]; then
|
|
376
|
+
(
|
|
377
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" runuser -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file" && touch "$success_marker"
|
|
378
|
+
) &
|
|
379
|
+
pids+=("$!")
|
|
380
|
+
fi
|
|
381
|
+
|
|
382
|
+
# Test 3: sudo (fallback, slowest)
|
|
383
|
+
if [ "$(id -u)" -eq 0 ] && command -v sudo >/dev/null 2>&1; then
|
|
384
|
+
(
|
|
385
|
+
if [ -n "$resolved_user_name" ] && [ -n "$resolved_group_name" ]; then
|
|
386
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -g "$resolved_group_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file"
|
|
387
|
+
elif [ -n "$resolved_user_name" ]; then
|
|
388
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "$resolved_user_name" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file"
|
|
389
|
+
else
|
|
390
|
+
timeout "$KASEKI_PRIV_TOOL_TIMEOUT" sudo -u "#${KASEKI_CONTAINER_UID}" -g "#${KASEKI_CONTAINER_GID}" -- "${probe_command[@]}" >/dev/null 2>"$stderr_file"
|
|
391
|
+
fi && touch "$success_marker"
|
|
392
|
+
) &
|
|
393
|
+
pids+=("$!")
|
|
394
|
+
fi
|
|
395
|
+
|
|
396
|
+
# Wait for any process to succeed (check success marker while processes run)
|
|
397
|
+
local timeout_elapsed=0
|
|
398
|
+
local max_wait=$((KASEKI_PRIV_TOOL_TIMEOUT + 1))
|
|
399
|
+
while [ $timeout_elapsed -lt $max_wait ]; do
|
|
400
|
+
if [ -f "$success_marker" ]; then
|
|
401
|
+
# Kill remaining background processes
|
|
402
|
+
for pid in "${pids[@]}"; do
|
|
403
|
+
kill "$pid" 2>/dev/null || true
|
|
404
|
+
done
|
|
405
|
+
return 0
|
|
406
|
+
fi
|
|
407
|
+
sleep 0.1
|
|
408
|
+
timeout_elapsed=$(( timeout_elapsed + 1 ))
|
|
409
|
+
done
|
|
410
|
+
|
|
411
|
+
# Wait for all processes to complete
|
|
412
|
+
for pid in "${pids[@]}"; do
|
|
413
|
+
wait "$pid" 2>/dev/null || true
|
|
414
|
+
done
|
|
415
|
+
|
|
416
|
+
# Check if any succeeded
|
|
417
|
+
[ -f "$success_marker" ] && return 0
|
|
418
|
+
return 1
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
# Phase 4: Performance tracking helpers
|
|
422
|
+
track_stage_start() {
|
|
423
|
+
local stage="$1"
|
|
424
|
+
export "${stage}_start=$(date +%s%N)"
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
track_stage_end() {
|
|
428
|
+
local stage="$1"
|
|
429
|
+
local start_var="${stage}_start"
|
|
430
|
+
local start_time="${!start_var:-0}"
|
|
431
|
+
if [ "$start_time" -gt 0 ]; then
|
|
432
|
+
local end_time
|
|
433
|
+
end_time=$(date +%s%N)
|
|
434
|
+
local elapsed_ms=$(( (end_time - start_time) / 1000000 ))
|
|
435
|
+
export "${stage}_duration=$elapsed_ms"
|
|
394
436
|
fi
|
|
395
437
|
}
|
|
396
438
|
|
|
397
|
-
# Phase 3: Enhanced setup results with structured output
|
|
439
|
+
# Phase 3: Enhanced setup results with structured output (with Phase 4 timing)
|
|
398
440
|
write_setup_results_enhanced() {
|
|
399
441
|
local home_dir="$1"
|
|
400
442
|
local exit_code="$2"
|
|
@@ -417,6 +459,15 @@ write_setup_results_enhanced() {
|
|
|
417
459
|
local temp_file="${results_file}.tmp"
|
|
418
460
|
local status_name="ok"
|
|
419
461
|
[ "$exit_code" != "0" ] && status_name="failed"
|
|
462
|
+
|
|
463
|
+
# Phase 4: Collect timing metrics from stage tracking
|
|
464
|
+
local timing_obj="{}"
|
|
465
|
+
if [ -n "${STAGE_1_duration:-}" ]; then
|
|
466
|
+
timing_obj=$(echo "$timing_obj" | jq --arg k "stage_1_ms" --arg v "${STAGE_1_duration}" '. + {($k): ($v | tonumber)}' 2>/dev/null || echo "{}")
|
|
467
|
+
fi
|
|
468
|
+
if [ -n "${STAGE_6_duration:-}" ]; then
|
|
469
|
+
timing_obj=$(echo "$timing_obj" | jq --arg k "probe_duration_ms" --arg v "${STAGE_6_duration}" '. + {($k): ($v | tonumber)}' 2>/dev/null || echo "{}")
|
|
470
|
+
fi
|
|
420
471
|
|
|
421
472
|
jq -n \
|
|
422
473
|
--arg timestamp "$timestamp" \
|
|
@@ -427,6 +478,7 @@ write_setup_results_enhanced() {
|
|
|
427
478
|
--arg version "2" \
|
|
428
479
|
--arg probe_status "$probe_status" \
|
|
429
480
|
--arg template_status "$template_status" \
|
|
481
|
+
--argjson timing "$timing_obj" \
|
|
430
482
|
'{
|
|
431
483
|
timestamp: $timestamp,
|
|
432
484
|
mode: $mode,
|
|
@@ -437,7 +489,8 @@ write_setup_results_enhanced() {
|
|
|
437
489
|
checks: {
|
|
438
490
|
checkout_freshness_probe: $probe_status,
|
|
439
491
|
template_ready: $template_status
|
|
440
|
-
}
|
|
492
|
+
},
|
|
493
|
+
performance: $timing
|
|
441
494
|
}' > "$temp_file"
|
|
442
495
|
|
|
443
496
|
chmod 0644 "$temp_file"
|
|
@@ -643,7 +696,9 @@ status=0
|
|
|
643
696
|
|
|
644
697
|
# Phase 1: Host Prerequisites validation
|
|
645
698
|
log_info "Stage 1: Host Prerequisites"
|
|
699
|
+
track_stage_start "STAGE_1"
|
|
646
700
|
validate_host_prerequisites || status=$?
|
|
701
|
+
track_stage_end "STAGE_1"
|
|
647
702
|
echo ""
|
|
648
703
|
|
|
649
704
|
# Early exit if prerequisites fail in check-only mode
|
|
@@ -685,7 +740,9 @@ fi
|
|
|
685
740
|
|
|
686
741
|
# Checkout freshness probe (Phase 2: used to conditionally run bootstrap)
|
|
687
742
|
log_info "Stage 6: Checkout freshness probe"
|
|
743
|
+
track_stage_start "STAGE_6"
|
|
688
744
|
probe_payload="$(run_checkout_freshness_probe "$KASEKI_CHECKOUT_DIR")"
|
|
745
|
+
track_stage_end "STAGE_6"
|
|
689
746
|
IFS="|" read -r checkout_probe_status checkout_probe_detail checkout_probe_remediation <<< "$probe_payload"
|
|
690
747
|
printf "checkout-freshness-probe: %s\n" "$checkout_probe_status"
|
|
691
748
|
printf "%s\n" "$checkout_probe_detail"
|