claudify-agent 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/QUICK-REFERENCE.md +213 -213
- package/.claude/cost-dashboard.md +162 -162
- package/.claude/docs/COST-OPTIMIZATION.md +402 -403
- package/.claude/docs/MCP-INTEGRATION-GUIDE.md +348 -348
- package/.claude/docs/PLAN-MODE-OPTIMIZATION.md +835 -836
- package/.claude/hooks/auto-learning-review.ts +200 -195
- package/.claude/hooks/auto-lint-fix.ts +60 -104
- package/.claude/hooks/auto-security-review.ts +314 -354
- package/.claude/hooks/auto-validate.ts +249 -348
- package/.claude/hooks/lib/claude-hook.ts +168 -0
- package/.claude/hooks/plan-quality-tracker.ts +598 -719
- package/.claude/rules/anti-patterns.md +22 -22
- package/.claude/rules/context-tier-1.md +41 -41
- package/.claude/rules/context-tier-2.md +285 -285
- package/.claude/rules/general.md +25 -25
- package/.claude/rules/mcp-auto-use.md +46 -46
- package/.claude/rules/parallel-exploration-auto.md +30 -31
- package/.claude/rules/plan-mode-execution.md +289 -289
- package/.claude/rules/plan-mode-research.md +34 -34
- package/.claude/rules/plan-mode-structure.md +166 -166
- package/.claude/rules/plan-mode-validation.md +129 -130
- package/.claude/rules/security-critical.md +19 -19
- package/.claude/rules/tool-auto-selection.md +45 -45
- package/.claude/rules/tool-strategy.md +22 -22
- package/.claude/scripts/security-scan.js +1 -1
- package/.claude/security-dashboard.md +142 -142
- package/.claude/settings.json +50 -0
- package/.claude/skills/debug-optimizer/SKILL.md +420 -399
- package/.claude/skills/learning-loop/SKILL.md +313 -308
- package/.claude/skills/mcp-mastery/SKILL.md +218 -214
- package/.claude/skills/nextjs-guide/SKILL.md +687 -687
- package/.claude/skills/parallel-exploration/SKILL.md +197 -193
- package/.claude/skills/plan-mode-mastery/SKILL.md +268 -263
- package/.claude/skills/python-guide/SKILL.md +822 -822
- package/.claude/skills/react-guide/SKILL.md +762 -762
- package/.claude/skills/{security.md → security-guide/SKILL.md} +10 -0
- package/.claude/skills/solidity-guide/SKILL.md +5 -0
- package/.claude/skills/tool-selection/SKILL.md +433 -428
- package/.claude/skills/vite-guide/SKILL.md +445 -445
- package/CLAUDE.md +1 -0
- package/README.md +74 -35
- package/README.vi.md +106 -0
- package/bin/cli.js +13 -4
- package/package.json +9 -1
- package/.claude/skills/debug-optimizer.md +0 -12
- package/.claude/skills/nextjs.md +0 -13
- package/.claude/skills/react.md +0 -12
|
@@ -1,719 +1,598 @@
|
|
|
1
|
-
// .
|
|
2
|
-
//
|
|
3
|
-
|
|
4
|
-
import { writeFileSync, readFileSync, existsSync } from "fs";
|
|
5
|
-
import { join } from "path";
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
interface
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
const
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
:
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
if (
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
{
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
{
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
if (
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
const
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
if (
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
const
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
let avgParallelSubagents = 0;
|
|
600
|
-
let avgCost = 0;
|
|
601
|
-
|
|
602
|
-
if (planTrackingConfig.trackMetrics && allMetrics.length > 0) {
|
|
603
|
-
const recentMetrics = allMetrics.slice(-10);
|
|
604
|
-
avgAccuracy = recentMetrics.reduce((sum, m) => sum + m.plan_accuracy, 0) / recentMetrics.length;
|
|
605
|
-
avgIterations = recentMetrics.reduce((sum, m) => sum + m.iteration_count, 0) / recentMetrics.length;
|
|
606
|
-
avgToolEfficiency = recentMetrics.reduce((sum, m) => sum + (m.tool_efficiency || 75), 0) / recentMetrics.length;
|
|
607
|
-
avgParallelSubagents = recentMetrics.reduce((sum, m) => sum + (m.parallel_subagents || 0), 0) / recentMetrics.length;
|
|
608
|
-
|
|
609
|
-
if (planTrackingConfig.enableCostTracking) {
|
|
610
|
-
avgCost = recentMetrics.reduce((sum, m) => sum + (m.cost_estimate || 0), 0) / recentMetrics.length;
|
|
611
|
-
}
|
|
612
|
-
|
|
613
|
-
console.error(`[Plan Tracker] Rolling averages (last 10 plans):`);
|
|
614
|
-
console.error(` - Accuracy: ${avgAccuracy.toFixed(1)}%`);
|
|
615
|
-
console.error(` - Avg iterations: ${avgIterations.toFixed(1)}`);
|
|
616
|
-
console.error(` - Tool efficiency: ${avgToolEfficiency.toFixed(1)}`);
|
|
617
|
-
console.error(` - Avg parallel subagents: ${avgParallelSubagents.toFixed(1)}`);
|
|
618
|
-
if (planTrackingConfig.enableCostTracking) {
|
|
619
|
-
console.error(` - Avg estimated cost: ${avgCost.toFixed(0)} tokens`);
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
// Analyze MCP effectiveness
|
|
624
|
-
const currentAccuracy = calculatePlanAccuracy(input, conversation);
|
|
625
|
-
|
|
626
|
-
if (Object.keys(mcpUsage).length > 0) {
|
|
627
|
-
console.error(`[Plan Tracker] MCP tools used in this session: ${mcpToolsUsed.join(', ')}`);
|
|
628
|
-
|
|
629
|
-
// Check correlation with success
|
|
630
|
-
if (currentAccuracy > 85 && Object.keys(mcpUsage).length > 0) {
|
|
631
|
-
console.error("[Plan Tracker] ✓ High accuracy with MCP tools - good tool selection");
|
|
632
|
-
}
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
// Analyze parallel subagent usage
|
|
636
|
-
if (parallelSubagents > 0) {
|
|
637
|
-
console.error(`[Plan Tracker] Used ${parallelSubagents} parallel subagent(s)`);
|
|
638
|
-
|
|
639
|
-
if (parallelSubagents > 2 && currentAccuracy > 80) {
|
|
640
|
-
console.error("[Plan Tracker] ✓ Parallel exploration appears effective");
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
// Provide feedback based on metrics (if enabled)
|
|
645
|
-
if (planTrackingConfig.provideFeedback) {
|
|
646
|
-
if (input.status === "completed") {
|
|
647
|
-
if (input.loop_count === 0) {
|
|
648
|
-
console.error("[Plan Tracker] ✓ Perfect execution (no iterations needed)");
|
|
649
|
-
} else if (input.loop_count <= 2) {
|
|
650
|
-
console.error("[Plan Tracker] ✓ Good execution (minimal iterations)");
|
|
651
|
-
} else if (input.loop_count > planTrackingConfig.maxIterations) {
|
|
652
|
-
console.error("[Plan Tracker] ⚠ High iteration count - plan accuracy may need improvement");
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
// Suggest improvements if accuracy is low
|
|
656
|
-
if (avgAccuracy < planTrackingConfig.accuracyThreshold) {
|
|
657
|
-
const output: StopHookOutput = {
|
|
658
|
-
followup_message: `Plan accuracy has been below ${planTrackingConfig.accuracyThreshold}% recently. Consider:\n1. More thorough pre-plan research\n2. Better validation before execution\n3. More specific tool selection`,
|
|
659
|
-
};
|
|
660
|
-
console.log(JSON.stringify(output));
|
|
661
|
-
process.exit(0);
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
// Suggest tool efficiency improvements
|
|
665
|
-
if (avgToolEfficiency < planTrackingConfig.efficiencyThreshold) {
|
|
666
|
-
const output: StopHookOutput = {
|
|
667
|
-
followup_message: "Tool efficiency has been low. Consider:\n1. Using simpler tools when possible (Tier 1: Shell, Read, Write)\n2. Avoiding expensive MCP calls for simple tasks\n3. Using 'fast' model for subagents",
|
|
668
|
-
};
|
|
669
|
-
console.log(JSON.stringify(output));
|
|
670
|
-
process.exit(0);
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
if (input.status === "error") {
|
|
675
|
-
console.error("[Plan Tracker] ✗ Execution failed - analyzing patterns");
|
|
676
|
-
|
|
677
|
-
// Check for recurring failures
|
|
678
|
-
if (planTrackingConfig.trackMetrics) {
|
|
679
|
-
const recentFailures = allMetrics.slice(-10).filter(m => m.blocker_count > 0).length;
|
|
680
|
-
if (recentFailures > 2) {
|
|
681
|
-
const output: StopHookOutput = {
|
|
682
|
-
followup_message: "Multiple execution failures detected. Common causes:\n1. Insufficient research phase\n2. Missing validation steps\n3. Unrealistic time estimates\n\nWould you like to review the plan approach?",
|
|
683
|
-
};
|
|
684
|
-
console.log(JSON.stringify(output));
|
|
685
|
-
process.exit(0);
|
|
686
|
-
}
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
// Check for tool-related failures
|
|
690
|
-
if (failedTools.length > 0) {
|
|
691
|
-
console.error(`[Plan Tracker] Failed tools: ${failedTools.join(', ')}`);
|
|
692
|
-
const output: StopHookOutput = {
|
|
693
|
-
followup_message: `Execution failed. Failed tools: ${failedTools.join(', ')}. Consider:\n1. Verifying tool prerequisites\n2. Using alternative tools\n3. Checking tool configuration`,
|
|
694
|
-
};
|
|
695
|
-
console.log(JSON.stringify(output));
|
|
696
|
-
process.exit(0);
|
|
697
|
-
}
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
// Handle aborted execution (from plan-mode-monitor merge)
|
|
701
|
-
if (input.status === "aborted") {
|
|
702
|
-
console.error("[Plan Tracker] ⚠ Plan execution aborted by user");
|
|
703
|
-
|
|
704
|
-
// Gather feedback
|
|
705
|
-
if (input.loop_count < 2) {
|
|
706
|
-
const output: StopHookOutput = {
|
|
707
|
-
followup_message: "Plan execution aborted. What was the main issue?\n1. Plan was inaccurate\n2. Requirements changed\n3. Found better approach\n4. Other",
|
|
708
|
-
};
|
|
709
|
-
console.log(JSON.stringify(output));
|
|
710
|
-
process.exit(0);
|
|
711
|
-
}
|
|
712
|
-
}
|
|
713
|
-
} else {
|
|
714
|
-
console.error("[Plan Tracker] ⚠ Feedback disabled in config");
|
|
715
|
-
}
|
|
716
|
-
|
|
717
|
-
// Default: stop without followup
|
|
718
|
-
console.log(JSON.stringify({}));
|
|
719
|
-
process.exit(0);
|
|
1
|
+
// .claude/hooks/plan-quality-tracker.ts
|
|
2
|
+
// Stop hook — plan metrics tracking from the real tool usage in the Claude Code transcript
|
|
3
|
+
|
|
4
|
+
import { writeFileSync, readFileSync, existsSync } from "fs";
|
|
5
|
+
import { join } from "path";
|
|
6
|
+
import {
|
|
7
|
+
readHookInput,
|
|
8
|
+
loadCurrentTurn,
|
|
9
|
+
mcpServer,
|
|
10
|
+
stateDir,
|
|
11
|
+
notify,
|
|
12
|
+
done,
|
|
13
|
+
SUBAGENT_TOOLS,
|
|
14
|
+
type Turn,
|
|
15
|
+
} from "./lib/claude-hook.ts";
|
|
16
|
+
|
|
17
|
+
interface AntiPattern {
|
|
18
|
+
id: string;
|
|
19
|
+
name: string;
|
|
20
|
+
description: string;
|
|
21
|
+
detectionRule: (metrics: PlanMetrics[]) => boolean;
|
|
22
|
+
suggestion: string;
|
|
23
|
+
severity: "critical" | "high" | "medium" | "low";
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
interface PlanMetrics {
|
|
27
|
+
plan_accuracy: number;
|
|
28
|
+
time_variance: number;
|
|
29
|
+
tool_usage: Record<string, number>;
|
|
30
|
+
iteration_count: number;
|
|
31
|
+
blocker_count: number;
|
|
32
|
+
timestamp: string;
|
|
33
|
+
plan_type: string;
|
|
34
|
+
mcp_usage: Record<string, number>;
|
|
35
|
+
tool_efficiency: number;
|
|
36
|
+
parallel_subagents: number;
|
|
37
|
+
mcp_tools_used: string[];
|
|
38
|
+
failed_tools: string[];
|
|
39
|
+
conversation_duration_ms?: number;
|
|
40
|
+
cost_estimate?: number;
|
|
41
|
+
anti_patterns_detected?: string[];
|
|
42
|
+
repeated_mistakes?: string[];
|
|
43
|
+
session_id?: string;
|
|
44
|
+
turn_id?: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface PatternAnalysis {
|
|
48
|
+
detectedPatterns: AntiPattern[];
|
|
49
|
+
repeatedMistakes: string[];
|
|
50
|
+
improvementSuggestions: string[];
|
|
51
|
+
trendDirection: "improving" | "stable" | "declining";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface ToolCall {
|
|
55
|
+
name: string;
|
|
56
|
+
execution_time_ms?: number;
|
|
57
|
+
success: boolean;
|
|
58
|
+
mcp_server?: string;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
interface HooksConfig {
|
|
62
|
+
planTracking?: {
|
|
63
|
+
trackMetrics?: boolean;
|
|
64
|
+
accuracyThreshold?: number;
|
|
65
|
+
efficiencyThreshold?: number;
|
|
66
|
+
maxIterations?: number;
|
|
67
|
+
provideFeedback?: boolean;
|
|
68
|
+
enableCostTracking?: boolean;
|
|
69
|
+
enablePatternAnalysis?: boolean;
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const input = await readHookInput();
|
|
74
|
+
|
|
75
|
+
const METRICS_FILE = join(stateDir(input), "plan-metrics.json");
|
|
76
|
+
|
|
77
|
+
// Load configuration
|
|
78
|
+
const configPath = join(stateDir(input), 'hooks.config.json');
|
|
79
|
+
const config: HooksConfig = existsSync(configPath)
|
|
80
|
+
? JSON.parse(readFileSync(configPath, 'utf-8'))
|
|
81
|
+
: {};
|
|
82
|
+
|
|
83
|
+
const planTrackingConfig = {
|
|
84
|
+
trackMetrics: config.planTracking?.trackMetrics !== false,
|
|
85
|
+
accuracyThreshold: config.planTracking?.accuracyThreshold || 70,
|
|
86
|
+
efficiencyThreshold: config.planTracking?.efficiencyThreshold || 60,
|
|
87
|
+
maxIterations: config.planTracking?.maxIterations || 5,
|
|
88
|
+
provideFeedback: config.planTracking?.provideFeedback !== false,
|
|
89
|
+
enableCostTracking: config.planTracking?.enableCostTracking !== false,
|
|
90
|
+
enablePatternAnalysis: config.planTracking?.enablePatternAnalysis !== false
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
console.error(`[Plan Tracker] Processing execution data for session ${input.session_id}...`);
|
|
94
|
+
|
|
95
|
+
// Helper functions
|
|
96
|
+
function extractToolCalls(turn: Turn): ToolCall[] {
|
|
97
|
+
return turn.toolUses.map(t => ({
|
|
98
|
+
name: t.name,
|
|
99
|
+
success: !t.isError,
|
|
100
|
+
mcp_server: mcpServer(t.name)
|
|
101
|
+
}));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function countParallelSubagents(turn: Turn): number {
|
|
105
|
+
return turn.toolUses.filter(t => SUBAGENT_TOOLS.includes(t.name)).length;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function detectPlanType(turn: Turn): string {
|
|
109
|
+
if (turn.toolUses.some(t => t.name === 'ExitPlanMode')) return 'plan-mode';
|
|
110
|
+
|
|
111
|
+
const conversationText = JSON.stringify(turn.entries).toLowerCase();
|
|
112
|
+
|
|
113
|
+
if (conversationText.includes('plan mode')) return 'plan-mode';
|
|
114
|
+
if (conversationText.includes('browser') || conversationText.includes('web')) return 'browser-automation';
|
|
115
|
+
if (conversationText.includes('github') || conversationText.includes('pr') || conversationText.includes('issue')) return 'github-ops';
|
|
116
|
+
if (conversationText.includes('test')) return 'testing';
|
|
117
|
+
if (conversationText.includes('refactor')) return 'refactoring';
|
|
118
|
+
if (conversationText.includes('bug') || conversationText.includes('fix')) return 'bug-fix';
|
|
119
|
+
if (conversationText.includes('feature') || conversationText.includes('implement')) return 'feature-development';
|
|
120
|
+
|
|
121
|
+
return 'general';
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Iterations = failed tool calls that had to be retried (failed tests, lint, edits...)
|
|
125
|
+
function calculatePlanAccuracy(iterations: number): number {
|
|
126
|
+
// Fewer iterations = higher accuracy
|
|
127
|
+
if (iterations === 0) return 100;
|
|
128
|
+
if (iterations <= 2) return 85;
|
|
129
|
+
if (iterations <= 5) return 70;
|
|
130
|
+
return 50;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function calculateToolEfficiency(tools: ToolCall[]): number {
|
|
134
|
+
if (tools.length === 0) return 75; // Neutral score if no tools
|
|
135
|
+
|
|
136
|
+
// Tier 1: Simple tools (Bash, Read, Edit, Write, Glob, Grep)
|
|
137
|
+
// Tier 2: Analysis (LSP)
|
|
138
|
+
// Tier 3: Complex (Agent/Task subagents, WebSearch, WebFetch)
|
|
139
|
+
// Tier 4: MCP tools (mcp__*)
|
|
140
|
+
|
|
141
|
+
const tierScores: Record<string, number> = {
|
|
142
|
+
'Bash': 100, 'Read': 100, 'Edit': 100, 'MultiEdit': 100, 'Write': 100, 'Glob': 100, 'Grep': 100,
|
|
143
|
+
'LSP': 80,
|
|
144
|
+
'Agent': 60, 'Task': 60, 'WebSearch': 60, 'WebFetch': 60,
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
let totalScore = 0;
|
|
148
|
+
let toolCount = 0;
|
|
149
|
+
|
|
150
|
+
for (const tool of tools) {
|
|
151
|
+
if (tool.mcp_server) {
|
|
152
|
+
totalScore += 40; // MCP tools are most expensive
|
|
153
|
+
toolCount++;
|
|
154
|
+
} else {
|
|
155
|
+
const tierScore = tierScores[tool.name] || 75; // Default to middle tier
|
|
156
|
+
totalScore += tierScore;
|
|
157
|
+
toolCount++;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return toolCount > 0 ? Math.round(totalScore / toolCount) : 75;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function estimateCost(tools: ToolCall[]): number {
|
|
165
|
+
// Estimated token cost per tool type
|
|
166
|
+
const costEstimates: Record<string, number> = {
|
|
167
|
+
'Bash': 100, 'Read': 500, 'Edit': 800, 'MultiEdit': 1000, 'Write': 1000, 'Glob': 200, 'Grep': 300,
|
|
168
|
+
'LSP': 800,
|
|
169
|
+
'Agent': 3000, 'Task': 3000, 'WebSearch': 2000, 'WebFetch': 2500,
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
let totalCost = 0;
|
|
173
|
+
for (const tool of tools) {
|
|
174
|
+
if (tool.mcp_server) {
|
|
175
|
+
totalCost += 5000; // MCP tools estimated higher cost
|
|
176
|
+
} else {
|
|
177
|
+
totalCost += costEstimates[tool.name] || 1000;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
return totalCost;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Anti-pattern definitions based on AGENT-INSTRUCTION-BEST-PRACTICES.md
|
|
185
|
+
const ANTI_PATTERNS: AntiPattern[] = [
|
|
186
|
+
{
|
|
187
|
+
id: "over-engineering",
|
|
188
|
+
name: "Over-Engineering",
|
|
189
|
+
description: "Using expensive tools when cheaper alternatives would work",
|
|
190
|
+
detectionRule: (metrics: PlanMetrics[]) => {
|
|
191
|
+
if (metrics.length < 3) return false;
|
|
192
|
+
const recent = metrics.slice(-3);
|
|
193
|
+
const avgEfficiency = recent.reduce((sum, m) => sum + (m.tool_efficiency || 75), 0) / recent.length;
|
|
194
|
+
return avgEfficiency < 50;
|
|
195
|
+
},
|
|
196
|
+
suggestion: "Start with Tier 1 tools (Read, Grep, Glob) before escalating to Tier 3-4",
|
|
197
|
+
severity: "high"
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
id: "no-research",
|
|
201
|
+
name: "Planning Without Research",
|
|
202
|
+
description: "Creating plans without using Grep/Glob or Explore subagents",
|
|
203
|
+
detectionRule: (metrics: PlanMetrics[]) => {
|
|
204
|
+
if (metrics.length < 3) return false;
|
|
205
|
+
const recent = metrics.slice(-3);
|
|
206
|
+
return recent.every(m => !m.tool_usage['Grep'] && !m.tool_usage['Glob'] && m.parallel_subagents === 0);
|
|
207
|
+
},
|
|
208
|
+
suggestion: "Always research with Grep/Glob and parallel Explore subagents before planning complex features",
|
|
209
|
+
severity: "critical"
|
|
210
|
+
},
|
|
211
|
+
{
|
|
212
|
+
id: "repeated-failures",
|
|
213
|
+
name: "Repeated Execution Failures",
|
|
214
|
+
description: "Multiple consecutive plans with low accuracy or high iterations",
|
|
215
|
+
detectionRule: (metrics: PlanMetrics[]) => {
|
|
216
|
+
if (metrics.length < 3) return false;
|
|
217
|
+
const recent = metrics.slice(-3);
|
|
218
|
+
const lowAccuracy = recent.filter(m => m.plan_accuracy < 60).length;
|
|
219
|
+
return lowAccuracy >= 2;
|
|
220
|
+
},
|
|
221
|
+
suggestion: "Review plan validation process, add more thorough feasibility checks",
|
|
222
|
+
severity: "critical"
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
id: "mcp-overuse",
|
|
226
|
+
name: "MCP Overuse",
|
|
227
|
+
description: "Using MCP tools for tasks that don't require them",
|
|
228
|
+
detectionRule: (metrics: PlanMetrics[]) => {
|
|
229
|
+
if (metrics.length < 3) return false;
|
|
230
|
+
const recent = metrics.slice(-3);
|
|
231
|
+
const highMCPUsage = recent.filter(m => Object.keys(m.mcp_usage).length > 3).length;
|
|
232
|
+
const lowAccuracy = recent.filter(m => m.plan_accuracy < 70).length;
|
|
233
|
+
return highMCPUsage >= 2 && lowAccuracy >= 2;
|
|
234
|
+
},
|
|
235
|
+
suggestion: "Use MCP tools only when essential (visual verification, git ops, docs)",
|
|
236
|
+
severity: "medium"
|
|
237
|
+
},
|
|
238
|
+
{
|
|
239
|
+
id: "no-validation",
|
|
240
|
+
name: "Skipping Validation",
|
|
241
|
+
description: "Not running lint/typecheck or tests after code changes",
|
|
242
|
+
detectionRule: (metrics: PlanMetrics[]) => {
|
|
243
|
+
if (metrics.length < 3) return false;
|
|
244
|
+
const recent = metrics.slice(-3);
|
|
245
|
+
return recent.every(m => !m.tool_usage['Bash']);
|
|
246
|
+
},
|
|
247
|
+
suggestion: "Always run lint/typecheck (Bash) after edits and tests after substantive changes",
|
|
248
|
+
severity: "high"
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
id: "context-bloat",
|
|
252
|
+
name: "Context Bloat",
|
|
253
|
+
description: "Loading too much context, violating minimalism principle",
|
|
254
|
+
detectionRule: (metrics: PlanMetrics[]) => {
|
|
255
|
+
// Detect via low efficiency + high iteration pattern
|
|
256
|
+
if (metrics.length < 3) return false;
|
|
257
|
+
const recent = metrics.slice(-3);
|
|
258
|
+
return recent.every(m => m.iteration_count > 3 && m.tool_efficiency < 60);
|
|
259
|
+
},
|
|
260
|
+
suggestion: "Apply tiered injection, use references over content, keep rules under 20 lines",
|
|
261
|
+
severity: "medium"
|
|
262
|
+
}
|
|
263
|
+
];
|
|
264
|
+
|
|
265
|
+
function detectAntiPatterns(metrics: PlanMetrics[]): AntiPattern[] {
|
|
266
|
+
const detected: AntiPattern[] = [];
|
|
267
|
+
|
|
268
|
+
for (const pattern of ANTI_PATTERNS) {
|
|
269
|
+
if (pattern.detectionRule(metrics)) {
|
|
270
|
+
detected.push(pattern);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return detected;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function identifyRepeatedMistakes(metrics: PlanMetrics[]): string[] {
|
|
278
|
+
const mistakes: string[] = [];
|
|
279
|
+
|
|
280
|
+
if (metrics.length < 5) return mistakes;
|
|
281
|
+
|
|
282
|
+
const recent = metrics.slice(-5);
|
|
283
|
+
|
|
284
|
+
// Check for repeated low accuracy
|
|
285
|
+
const lowAccuracyCount = recent.filter(m => m.plan_accuracy < 70).length;
|
|
286
|
+
if (lowAccuracyCount >= 3) {
|
|
287
|
+
mistakes.push("Consistently low plan accuracy (<70%)");
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Check for repeated high iterations
|
|
291
|
+
const highIterationCount = recent.filter(m => m.iteration_count > 3).length;
|
|
292
|
+
if (highIterationCount >= 3) {
|
|
293
|
+
mistakes.push("Frequently requiring many iterations (>3)");
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Check for repeated tool failures
|
|
297
|
+
const toolFailureCount = recent.filter(m => m.failed_tools && m.failed_tools.length > 0).length;
|
|
298
|
+
if (toolFailureCount >= 3) {
|
|
299
|
+
mistakes.push("Recurring tool failures");
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// Check for declining performance
|
|
303
|
+
const firstHalf = recent.slice(0, Math.floor(recent.length / 2));
|
|
304
|
+
const secondHalf = recent.slice(Math.floor(recent.length / 2));
|
|
305
|
+
const firstAvg = firstHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / firstHalf.length;
|
|
306
|
+
const secondAvg = secondHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / secondHalf.length;
|
|
307
|
+
|
|
308
|
+
if (secondAvg < firstAvg - 10) {
|
|
309
|
+
mistakes.push("Performance declining over time");
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
return mistakes;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function generateImprovementSuggestions(antiPatterns: AntiPattern[], mistakes: string[]): string[] {
|
|
316
|
+
const suggestions: string[] = [];
|
|
317
|
+
|
|
318
|
+
// Add suggestions from detected anti-patterns
|
|
319
|
+
for (const pattern of antiPatterns) {
|
|
320
|
+
suggestions.push(`[${pattern.severity.toUpperCase()}] ${pattern.suggestion}`);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Add suggestions for repeated mistakes
|
|
324
|
+
if (mistakes.includes("Consistently low plan accuracy (<70%)")) {
|
|
325
|
+
suggestions.push("Improve pre-plan research phase - spend 2-3 min on Grep/Glob and Explore subagents");
|
|
326
|
+
suggestions.push("Add validation subagent before presenting plans");
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (mistakes.includes("Frequently requiring many iterations (>3)")) {
|
|
330
|
+
suggestions.push("Add more thorough feasibility validation");
|
|
331
|
+
suggestions.push("Create checkpoints before risky changes");
|
|
332
|
+
suggestions.push("Use the fix-and-retest loop strategically (max 5 iterations)");
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (mistakes.includes("Recurring tool failures")) {
|
|
336
|
+
suggestions.push("Verify tool prerequisites before use");
|
|
337
|
+
suggestions.push("Follow tool escalation protocol (Tier 1 → Tier 4)");
|
|
338
|
+
suggestions.push("Check MCP server availability");
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
if (mistakes.includes("Performance declining over time")) {
|
|
342
|
+
suggestions.push("Review recent changes to workflow or rules");
|
|
343
|
+
suggestions.push("Analyze failing plans for common patterns");
|
|
344
|
+
suggestions.push("Consider updating anti-pattern detection rules");
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
return suggestions;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function analyzeTrend(metrics: PlanMetrics[]): "improving" | "stable" | "declining" {
|
|
351
|
+
if (metrics.length < 5) return "stable";
|
|
352
|
+
|
|
353
|
+
const recent = metrics.slice(-5);
|
|
354
|
+
const firstHalf = recent.slice(0, Math.floor(recent.length / 2));
|
|
355
|
+
const secondHalf = recent.slice(Math.floor(recent.length / 2));
|
|
356
|
+
|
|
357
|
+
const firstAvg = firstHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / firstHalf.length;
|
|
358
|
+
const secondAvg = secondHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / secondHalf.length;
|
|
359
|
+
|
|
360
|
+
if (secondAvg > firstAvg + 5) return "improving";
|
|
361
|
+
if (secondAvg < firstAvg - 5) return "declining";
|
|
362
|
+
return "stable";
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
function performPatternAnalysis(metrics: PlanMetrics[]): PatternAnalysis {
|
|
366
|
+
const antiPatterns = detectAntiPatterns(metrics);
|
|
367
|
+
const mistakes = identifyRepeatedMistakes(metrics);
|
|
368
|
+
const suggestions = generateImprovementSuggestions(antiPatterns, mistakes);
|
|
369
|
+
const trend = analyzeTrend(metrics);
|
|
370
|
+
|
|
371
|
+
return {
|
|
372
|
+
detectedPatterns: antiPatterns,
|
|
373
|
+
repeatedMistakes: mistakes,
|
|
374
|
+
improvementSuggestions: suggestions,
|
|
375
|
+
trendDirection: trend
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// Load existing metrics
|
|
380
|
+
let allMetrics: PlanMetrics[] = [];
|
|
381
|
+
if (existsSync(METRICS_FILE)) {
|
|
382
|
+
try {
|
|
383
|
+
const content = readFileSync(METRICS_FILE, "utf-8");
|
|
384
|
+
allMetrics = JSON.parse(content);
|
|
385
|
+
} catch (error) {
|
|
386
|
+
console.error("[Plan Tracker] Error loading metrics:", error);
|
|
387
|
+
allMetrics = [];
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// Load the current turn from the transcript; skip turns without tool use (plain Q&A)
|
|
392
|
+
const turn = loadCurrentTurn(input.transcript_path);
|
|
393
|
+
if (!turn || turn.toolUses.length === 0) done();
|
|
394
|
+
|
|
395
|
+
const toolCalls = extractToolCalls(turn);
|
|
396
|
+
const parallelSubagents = countParallelSubagents(turn);
|
|
397
|
+
const planType = detectPlanType(turn);
|
|
398
|
+
|
|
399
|
+
// Track MCP usage and effectiveness
|
|
400
|
+
const mcpUsage: Record<string, number> = {};
|
|
401
|
+
const mcpToolsUsed: string[] = [];
|
|
402
|
+
const failedTools: string[] = [];
|
|
403
|
+
|
|
404
|
+
for (const tool of toolCalls) {
|
|
405
|
+
if (tool.mcp_server) {
|
|
406
|
+
mcpUsage[tool.mcp_server] = (mcpUsage[tool.mcp_server] || 0) + 1;
|
|
407
|
+
if (!mcpToolsUsed.includes(tool.name)) {
|
|
408
|
+
mcpToolsUsed.push(tool.name);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
if (!tool.success) {
|
|
413
|
+
failedTools.push(tool.name);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// MCP Effectiveness Analysis
|
|
418
|
+
if (Object.keys(mcpUsage).length > 0) {
|
|
419
|
+
const mcpServers = Object.keys(mcpUsage).join(', ');
|
|
420
|
+
console.error(`[Plan Tracker] MCP servers used: ${mcpServers}`);
|
|
421
|
+
|
|
422
|
+
// Track MCP effectiveness per server
|
|
423
|
+
const mcpEffectiveness: Record<string, { usage: number; success: number }> = {};
|
|
424
|
+
toolCalls.forEach(tool => {
|
|
425
|
+
if (tool.mcp_server) {
|
|
426
|
+
if (!mcpEffectiveness[tool.mcp_server]) {
|
|
427
|
+
mcpEffectiveness[tool.mcp_server] = { usage: 0, success: 0 };
|
|
428
|
+
}
|
|
429
|
+
mcpEffectiveness[tool.mcp_server].usage++;
|
|
430
|
+
if (tool.success) mcpEffectiveness[tool.mcp_server].success++;
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
// Report effectiveness
|
|
435
|
+
Object.entries(mcpEffectiveness).forEach(([server, data]) => {
|
|
436
|
+
const successRate = Math.round((data.success / data.usage) * 100);
|
|
437
|
+
console.error(`[Plan Tracker] ${server}: ${data.usage} calls, ${successRate}% success rate`);
|
|
438
|
+
|
|
439
|
+
if (successRate < 50) {
|
|
440
|
+
console.error(`[Plan Tracker] ⚠ ${server} has low success rate (<50%). Consider disabling or reviewing usage.`);
|
|
441
|
+
}
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// Track general tool usage
|
|
446
|
+
const toolUsage: Record<string, number> = {};
|
|
447
|
+
for (const tool of toolCalls) {
|
|
448
|
+
toolUsage[tool.name] = (toolUsage[tool.name] || 0) + 1;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
// Estimate cost
|
|
452
|
+
const estimatedCost = planTrackingConfig.enableCostTracking ? estimateCost(toolCalls) : undefined;
|
|
453
|
+
|
|
454
|
+
// Perform pattern analysis (if enabled)
|
|
455
|
+
let detectedAntiPatterns: string[] = [];
|
|
456
|
+
let repeatedMistakes: string[] = [];
|
|
457
|
+
|
|
458
|
+
if (planTrackingConfig.enablePatternAnalysis && allMetrics.length >= 3) {
|
|
459
|
+
const analysis = performPatternAnalysis(allMetrics);
|
|
460
|
+
detectedAntiPatterns = analysis.detectedPatterns.map(p => p.id);
|
|
461
|
+
repeatedMistakes = analysis.repeatedMistakes;
|
|
462
|
+
|
|
463
|
+
if (analysis.detectedPatterns.length > 0) {
|
|
464
|
+
console.error(`[Plan Tracker] ⚠ Detected ${analysis.detectedPatterns.length} anti-pattern(s):`);
|
|
465
|
+
for (const pattern of analysis.detectedPatterns) {
|
|
466
|
+
console.error(` - [${pattern.severity.toUpperCase()}] ${pattern.name}: ${pattern.suggestion}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
if (analysis.repeatedMistakes.length > 0) {
|
|
471
|
+
console.error(`[Plan Tracker] ⚠ Repeated mistakes detected:`);
|
|
472
|
+
for (const mistake of analysis.repeatedMistakes) {
|
|
473
|
+
console.error(` - ${mistake}`);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
console.error(`[Plan Tracker] Trend: ${analysis.trendDirection}`);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// Create metric entry (used whether tracking is enabled or not)
|
|
481
|
+
const iterations = failedTools.length;
|
|
482
|
+
const currentAccuracy = calculatePlanAccuracy(iterations);
|
|
483
|
+
|
|
484
|
+
const newMetric: PlanMetrics = {
|
|
485
|
+
plan_accuracy: currentAccuracy,
|
|
486
|
+
time_variance: 0,
|
|
487
|
+
tool_usage: toolUsage,
|
|
488
|
+
iteration_count: iterations,
|
|
489
|
+
blocker_count: 0,
|
|
490
|
+
timestamp: new Date().toISOString(),
|
|
491
|
+
plan_type: planType,
|
|
492
|
+
mcp_usage: mcpUsage,
|
|
493
|
+
tool_efficiency: calculateToolEfficiency(toolCalls),
|
|
494
|
+
parallel_subagents: parallelSubagents,
|
|
495
|
+
mcp_tools_used: mcpToolsUsed,
|
|
496
|
+
failed_tools: failedTools,
|
|
497
|
+
conversation_duration_ms: undefined,
|
|
498
|
+
cost_estimate: estimatedCost,
|
|
499
|
+
anti_patterns_detected: detectedAntiPatterns,
|
|
500
|
+
repeated_mistakes: repeatedMistakes,
|
|
501
|
+
session_id: input.session_id,
|
|
502
|
+
turn_id: turn.id
|
|
503
|
+
};
|
|
504
|
+
|
|
505
|
+
// Save metric entry if tracking is enabled
|
|
506
|
+
if (planTrackingConfig.trackMetrics) {
|
|
507
|
+
// Add to metrics; a Stop after a stop-hook continuation updates the same turn's entry
|
|
508
|
+
const last = allMetrics[allMetrics.length - 1];
|
|
509
|
+
if (last && last.session_id === input.session_id && last.turn_id === turn.id) {
|
|
510
|
+
allMetrics[allMetrics.length - 1] = newMetric;
|
|
511
|
+
} else {
|
|
512
|
+
allMetrics.push(newMetric);
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// Save updated metrics
|
|
516
|
+
try {
|
|
517
|
+
writeFileSync(METRICS_FILE, JSON.stringify(allMetrics, null, 2));
|
|
518
|
+
console.error("[Plan Tracker] ✓ Metrics saved");
|
|
519
|
+
} catch (error) {
|
|
520
|
+
console.error("[Plan Tracker] Error saving metrics:", error);
|
|
521
|
+
}
|
|
522
|
+
} else {
|
|
523
|
+
console.error("[Plan Tracker] ⚠ Metrics tracking disabled in config");
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// Calculate rolling averages (only if tracking is enabled)
|
|
527
|
+
let avgAccuracy = 75;
|
|
528
|
+
let avgToolEfficiency = 75;
|
|
529
|
+
let avgCost = 0;
|
|
530
|
+
|
|
531
|
+
if (planTrackingConfig.trackMetrics && allMetrics.length > 0) {
|
|
532
|
+
const recentMetrics = allMetrics.slice(-10);
|
|
533
|
+
avgAccuracy = recentMetrics.reduce((sum, m) => sum + m.plan_accuracy, 0) / recentMetrics.length;
|
|
534
|
+
const avgIterations = recentMetrics.reduce((sum, m) => sum + m.iteration_count, 0) / recentMetrics.length;
|
|
535
|
+
avgToolEfficiency = recentMetrics.reduce((sum, m) => sum + (m.tool_efficiency || 75), 0) / recentMetrics.length;
|
|
536
|
+
const avgParallelSubagents = recentMetrics.reduce((sum, m) => sum + (m.parallel_subagents || 0), 0) / recentMetrics.length;
|
|
537
|
+
|
|
538
|
+
if (planTrackingConfig.enableCostTracking) {
|
|
539
|
+
avgCost = recentMetrics.reduce((sum, m) => sum + (m.cost_estimate || 0), 0) / recentMetrics.length;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
console.error(`[Plan Tracker] Rolling averages (last 10 plans):`);
|
|
543
|
+
console.error(` - Accuracy: ${avgAccuracy.toFixed(1)}%`);
|
|
544
|
+
console.error(` - Avg iterations: ${avgIterations.toFixed(1)}`);
|
|
545
|
+
console.error(` - Tool efficiency: ${avgToolEfficiency.toFixed(1)}`);
|
|
546
|
+
console.error(` - Avg parallel subagents: ${avgParallelSubagents.toFixed(1)}`);
|
|
547
|
+
if (planTrackingConfig.enableCostTracking) {
|
|
548
|
+
console.error(` - Avg estimated cost: ${avgCost.toFixed(0)} tokens`);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// Analyze MCP effectiveness
|
|
553
|
+
|
|
554
|
+
if (Object.keys(mcpUsage).length > 0) {
|
|
555
|
+
console.error(`[Plan Tracker] MCP tools used in this session: ${mcpToolsUsed.join(', ')}`);
|
|
556
|
+
|
|
557
|
+
// Check correlation with success
|
|
558
|
+
if (currentAccuracy > 85 && Object.keys(mcpUsage).length > 0) {
|
|
559
|
+
console.error("[Plan Tracker] ✓ High accuracy with MCP tools - good tool selection");
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// Analyze parallel subagent usage
|
|
564
|
+
if (parallelSubagents > 0) {
|
|
565
|
+
console.error(`[Plan Tracker] Used ${parallelSubagents} parallel subagent(s)`);
|
|
566
|
+
|
|
567
|
+
if (parallelSubagents > 2 && currentAccuracy > 80) {
|
|
568
|
+
console.error("[Plan Tracker] ✓ Parallel exploration appears effective");
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
// Provide feedback based on metrics (if enabled) — advisory, shown to the user
|
|
573
|
+
if (planTrackingConfig.provideFeedback) {
|
|
574
|
+
if (iterations === 0) {
|
|
575
|
+
console.error("[Plan Tracker] ✓ Perfect execution (no iterations needed)");
|
|
576
|
+
} else if (iterations <= 2) {
|
|
577
|
+
console.error("[Plan Tracker] ✓ Good execution (minimal iterations)");
|
|
578
|
+
} else if (iterations > planTrackingConfig.maxIterations) {
|
|
579
|
+
console.error("[Plan Tracker] ⚠ High iteration count - plan accuracy may need improvement");
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// Suggest improvements if accuracy is low
|
|
583
|
+
if (avgAccuracy < planTrackingConfig.accuracyThreshold) {
|
|
584
|
+
notify(`Plan accuracy has been below ${planTrackingConfig.accuracyThreshold}% recently. Consider:\n1. More thorough pre-plan research\n2. Better validation before execution\n3. More specific tool selection`);
|
|
585
|
+
process.exit(0);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
// Suggest tool efficiency improvements
|
|
589
|
+
if (avgToolEfficiency < planTrackingConfig.efficiencyThreshold) {
|
|
590
|
+
notify("Tool efficiency has been low. Consider:\n1. Using simpler tools when possible (Tier 1: Bash, Read, Edit, Grep)\n2. Avoiding expensive MCP calls for simple tasks\n3. Using `model: haiku` for Explore subagents");
|
|
591
|
+
process.exit(0);
|
|
592
|
+
}
|
|
593
|
+
} else {
|
|
594
|
+
console.error("[Plan Tracker] ⚠ Feedback disabled in config");
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
// Default: stop without followup
|
|
598
|
+
done();
|