claudify-agent 1.0.1 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/.claude/QUICK-REFERENCE.md +213 -213
  2. package/.claude/cost-dashboard.md +162 -162
  3. package/.claude/docs/COST-OPTIMIZATION.md +402 -403
  4. package/.claude/docs/MCP-INTEGRATION-GUIDE.md +348 -348
  5. package/.claude/docs/PLAN-MODE-OPTIMIZATION.md +835 -836
  6. package/.claude/hooks/auto-learning-review.ts +200 -195
  7. package/.claude/hooks/auto-lint-fix.ts +60 -104
  8. package/.claude/hooks/auto-security-review.ts +314 -354
  9. package/.claude/hooks/auto-validate.ts +249 -348
  10. package/.claude/hooks/lib/claude-hook.ts +168 -0
  11. package/.claude/hooks/plan-quality-tracker.ts +598 -719
  12. package/.claude/rules/anti-patterns.md +22 -22
  13. package/.claude/rules/context-tier-1.md +41 -41
  14. package/.claude/rules/context-tier-2.md +285 -285
  15. package/.claude/rules/general.md +25 -25
  16. package/.claude/rules/mcp-auto-use.md +46 -46
  17. package/.claude/rules/parallel-exploration-auto.md +30 -31
  18. package/.claude/rules/plan-mode-execution.md +289 -289
  19. package/.claude/rules/plan-mode-research.md +34 -34
  20. package/.claude/rules/plan-mode-structure.md +166 -166
  21. package/.claude/rules/plan-mode-validation.md +129 -130
  22. package/.claude/rules/security-critical.md +19 -19
  23. package/.claude/rules/tool-auto-selection.md +45 -45
  24. package/.claude/rules/tool-strategy.md +22 -22
  25. package/.claude/scripts/security-scan.js +1 -1
  26. package/.claude/security-dashboard.md +142 -142
  27. package/.claude/settings.json +50 -0
  28. package/.claude/skills/debug-optimizer/SKILL.md +420 -399
  29. package/.claude/skills/learning-loop/SKILL.md +313 -308
  30. package/.claude/skills/mcp-mastery/SKILL.md +218 -214
  31. package/.claude/skills/nextjs-guide/SKILL.md +687 -687
  32. package/.claude/skills/parallel-exploration/SKILL.md +197 -193
  33. package/.claude/skills/plan-mode-mastery/SKILL.md +268 -263
  34. package/.claude/skills/python-guide/SKILL.md +822 -822
  35. package/.claude/skills/react-guide/SKILL.md +762 -762
  36. package/.claude/skills/{security.md → security-guide/SKILL.md} +10 -0
  37. package/.claude/skills/solidity-guide/SKILL.md +5 -0
  38. package/.claude/skills/tool-selection/SKILL.md +433 -428
  39. package/.claude/skills/vite-guide/SKILL.md +445 -445
  40. package/CLAUDE.md +1 -0
  41. package/README.md +74 -35
  42. package/README.vi.md +106 -0
  43. package/bin/cli.js +13 -4
  44. package/package.json +9 -1
  45. package/.claude/skills/debug-optimizer.md +0 -12
  46. package/.claude/skills/nextjs.md +0 -13
  47. package/.claude/skills/react.md +0 -12
@@ -1,719 +1,598 @@
1
- // .cursor/hooks/plan-quality-tracker.ts
2
- // Unified plan metrics tracking with real tool usage and accuracy calculations
3
-
4
- import { writeFileSync, readFileSync, existsSync } from "fs";
5
- import { join } from "path";
6
-
7
- export {};
8
-
9
- interface StopHookInput {
10
- conversation_id: string;
11
- status: "completed" | "aborted" | "error";
12
- loop_count: number;
13
- }
14
-
15
- interface StopHookOutput {
16
- followup_message?: string;
17
- }
18
-
19
- interface AntiPattern {
20
- id: string;
21
- name: string;
22
- description: string;
23
- detectionRule: (metrics: PlanMetrics[]) => boolean;
24
- suggestion: string;
25
- severity: "critical" | "high" | "medium" | "low";
26
- }
27
-
28
- interface PlanMetrics {
29
- plan_accuracy: number;
30
- time_variance: number;
31
- tool_usage: Record<string, number>;
32
- iteration_count: number;
33
- blocker_count: number;
34
- timestamp: string;
35
- plan_type: string;
36
- mcp_usage: Record<string, number>;
37
- tool_efficiency: number;
38
- parallel_subagents: number;
39
- mcp_tools_used: string[];
40
- failed_tools: string[];
41
- conversation_duration_ms?: number;
42
- cost_estimate?: number;
43
- anti_patterns_detected?: string[];
44
- repeated_mistakes?: string[];
45
- }
46
-
47
- interface MCPEffectiveness {
48
- usage: number;
49
- success_rate: number;
50
- }
51
-
52
- interface PatternAnalysis {
53
- detectedPatterns: AntiPattern[];
54
- repeatedMistakes: string[];
55
- improvementSuggestions: string[];
56
- trendDirection: "improving" | "stable" | "declining";
57
- }
58
-
59
- interface ToolCall {
60
- name: string;
61
- execution_time_ms?: number;
62
- success: boolean;
63
- mcp_server?: string;
64
- }
65
-
66
- interface HooksConfig {
67
- planTracking?: {
68
- trackMetrics?: boolean;
69
- accuracyThreshold?: number;
70
- efficiencyThreshold?: number;
71
- maxIterations?: number;
72
- provideFeedback?: boolean;
73
- enableCostTracking?: boolean;
74
- enablePatternAnalysis?: boolean;
75
- };
76
- }
77
-
78
- const METRICS_FILE = join(process.cwd(), ".cursor/plan-metrics.json");
79
- const CONVERSATIONS_DIR = join(process.cwd(), ".cursor/conversations");
80
-
81
- // Load configuration
82
- const configPath = join(process.cwd(), '.cursor/hooks.config.json');
83
- const config: HooksConfig = existsSync(configPath)
84
- ? JSON.parse(readFileSync(configPath, 'utf-8'))
85
- : {};
86
-
87
- const planTrackingConfig = {
88
- trackMetrics: config.planTracking?.trackMetrics !== false,
89
- accuracyThreshold: config.planTracking?.accuracyThreshold || 70,
90
- efficiencyThreshold: config.planTracking?.efficiencyThreshold || 60,
91
- maxIterations: config.planTracking?.maxIterations || 5,
92
- provideFeedback: config.planTracking?.provideFeedback !== false,
93
- enableCostTracking: config.planTracking?.enableCostTracking !== false,
94
- enablePatternAnalysis: config.planTracking?.enablePatternAnalysis !== false
95
- };
96
-
97
- // Read input from stdin
98
- const stdin = await new Promise<string>((resolve) => {
99
- let data = "";
100
- process.stdin.on("data", (chunk) => (data += chunk));
101
- process.stdin.on("end", () => resolve(data));
102
- });
103
-
104
- const input: StopHookInput = JSON.parse(stdin);
105
-
106
- console.error(`[Plan Tracker] Processing execution data for conversation ${input.conversation_id}...`);
107
-
108
- // Helper functions
109
- function loadConversation(conversationId: string): any {
110
- const conversationPath = join(CONVERSATIONS_DIR, `${conversationId}.json`);
111
- if (!existsSync(conversationPath)) {
112
- console.error("[Plan Tracker] ⚠ Conversation file not found");
113
- return null;
114
- }
115
-
116
- try {
117
- return JSON.parse(readFileSync(conversationPath, 'utf-8'));
118
- } catch (error) {
119
- console.error("[Plan Tracker] Error loading conversation:", error);
120
- return null;
121
- }
122
- }
123
-
124
- function extractToolCalls(conversation: any): ToolCall[] {
125
- const tools: ToolCall[] = [];
126
-
127
- if (!conversation?.messages) return tools;
128
-
129
- for (const message of conversation.messages) {
130
- if (message.tool_calls) {
131
- for (const toolCall of message.tool_calls) {
132
- const toolName = toolCall.function?.name || 'unknown';
133
-
134
- // Determine if it's an MCP tool and which server
135
- let mcpServer: string | undefined;
136
- if (toolName.startsWith('browser_')) mcpServer = 'cursor-ide-browser';
137
- else if (toolName.startsWith('github_')) mcpServer = 'user-github';
138
- else if (toolName.includes('eslint')) mcpServer = 'user-ESLint';
139
- else if (toolName.includes('context7')) mcpServer = 'user-context7';
140
- else if (toolName.includes('chrome')) mcpServer = 'user-chrome-devtools';
141
- else if (toolName.includes('playwright')) mcpServer = 'user-playwright';
142
- else if (toolName.includes('selenium')) mcpServer = 'user-selenium';
143
- else if (toolName.includes('dart')) mcpServer = 'user-dart';
144
- else if (toolName.includes('stitch')) mcpServer = 'user-stitch';
145
- else if (toolName.includes('shadcn')) mcpServer = 'user-shadcn';
146
-
147
- tools.push({
148
- name: toolName,
149
- success: true, // Simplified - would check actual results in production
150
- mcp_server: mcpServer
151
- });
152
- }
153
- }
154
- }
155
-
156
- return tools;
157
- }
158
-
159
- function countParallelSubagents(conversation: any): number {
160
- if (!conversation?.messages) return 0;
161
-
162
- let count = 0;
163
- for (const message of conversation.messages) {
164
- if (message.tool_calls) {
165
- for (const toolCall of message.tool_calls) {
166
- if (toolCall.function?.name === 'Task' ||
167
- toolCall.function?.name?.includes('subagent')) {
168
- count++;
169
- }
170
- }
171
- }
172
- }
173
-
174
- return count;
175
- }
176
-
177
- function detectPlanType(conversation: any): string {
178
- if (!conversation?.messages) return 'unknown';
179
-
180
- const conversationText = JSON.stringify(conversation.messages).toLowerCase();
181
-
182
- if (conversationText.includes('plan mode')) return 'plan-mode';
183
- if (conversationText.includes('browser') || conversationText.includes('web')) return 'browser-automation';
184
- if (conversationText.includes('github') || conversationText.includes('pr') || conversationText.includes('issue')) return 'github-ops';
185
- if (conversationText.includes('test')) return 'testing';
186
- if (conversationText.includes('refactor')) return 'refactoring';
187
- if (conversationText.includes('bug') || conversationText.includes('fix')) return 'bug-fix';
188
- if (conversationText.includes('feature') || conversationText.includes('implement')) return 'feature-development';
189
-
190
- return 'general';
191
- }
192
-
193
- function calculatePlanAccuracy(input: StopHookInput, conversation: any): number {
194
- if (input.status !== "completed") return 0;
195
-
196
- // Base accuracy on loop count (fewer iterations = higher accuracy)
197
- let baseAccuracy: number;
198
- if (input.loop_count === 0) baseAccuracy = 100;
199
- else if (input.loop_count <= 2) baseAccuracy = 85;
200
- else if (input.loop_count <= 5) baseAccuracy = 70;
201
- else baseAccuracy = 50;
202
-
203
- // Adjust based on tool usage patterns
204
- const tools = extractToolCalls(conversation);
205
- const failedTools = tools.filter(t => !t.success).length;
206
-
207
- if (failedTools > 0) {
208
- baseAccuracy -= (failedTools * 5); // Penalty for failed tools
209
- }
210
-
211
- return Math.max(0, Math.min(100, baseAccuracy));
212
- }
213
-
214
- function calculateToolEfficiency(tools: ToolCall[]): number {
215
- if (tools.length === 0) return 75; // Neutral score if no tools
216
-
217
- // Tier 1: Simple tools (Shell, Read, Write, Glob, Grep)
218
- // Tier 2: Analysis (ReadLints, SemanticSearch)
219
- // Tier 3: Complex (Task, WebSearch, WebFetch)
220
- // Tier 4: MCP tools
221
-
222
- const tierScores: Record<string, number> = {
223
- 'Shell': 100, 'Read': 100, 'Write': 100, 'Glob': 100, 'Grep': 100,
224
- 'ReadLints': 80, 'SemanticSearch': 80,
225
- 'Task': 60, 'WebSearch': 60, 'WebFetch': 60,
226
- };
227
-
228
- let totalScore = 0;
229
- let toolCount = 0;
230
-
231
- for (const tool of tools) {
232
- if (tool.mcp_server) {
233
- totalScore += 40; // MCP tools are most expensive
234
- toolCount++;
235
- } else {
236
- const tierScore = tierScores[tool.name] || 75; // Default to middle tier
237
- totalScore += tierScore;
238
- toolCount++;
239
- }
240
- }
241
-
242
- return toolCount > 0 ? Math.round(totalScore / toolCount) : 75;
243
- }
244
-
245
- function estimateCost(tools: ToolCall[]): number {
246
- // Estimated token cost per tool type
247
- const costEstimates: Record<string, number> = {
248
- 'Shell': 100, 'Read': 500, 'Write': 1000, 'Glob': 200, 'Grep': 300,
249
- 'ReadLints': 800, 'SemanticSearch': 1500,
250
- 'Task': 3000, 'WebSearch': 2000, 'WebFetch': 2500,
251
- };
252
-
253
- let totalCost = 0;
254
- for (const tool of tools) {
255
- if (tool.mcp_server) {
256
- totalCost += 5000; // MCP tools estimated higher cost
257
- } else {
258
- totalCost += costEstimates[tool.name] || 1000;
259
- }
260
- }
261
-
262
- return totalCost;
263
- }
264
-
265
- // Anti-pattern definitions based on AGENT-INSTRUCTION-BEST-PRACTICES.md
266
- const ANTI_PATTERNS: AntiPattern[] = [
267
- {
268
- id: "over-engineering",
269
- name: "Over-Engineering",
270
- description: "Using expensive tools when cheaper alternatives would work",
271
- detectionRule: (metrics: PlanMetrics[]) => {
272
- if (metrics.length < 3) return false;
273
- const recent = metrics.slice(-3);
274
- const avgEfficiency = recent.reduce((sum, m) => sum + (m.tool_efficiency || 75), 0) / recent.length;
275
- return avgEfficiency < 50;
276
- },
277
- suggestion: "Start with Tier 1 tools (Read, Grep, Glob) before escalating to Tier 3-4",
278
- severity: "high"
279
- },
280
- {
281
- id: "no-research",
282
- name: "Planning Without Research",
283
- description: "Creating plans without using SemanticSearch or exploration tools",
284
- detectionRule: (metrics: PlanMetrics[]) => {
285
- if (metrics.length < 3) return false;
286
- const recent = metrics.slice(-3);
287
- return recent.every(m => !m.tool_usage['SemanticSearch'] && m.parallel_subagents === 0);
288
- },
289
- suggestion: "Always use SemanticSearch and parallel subagents before planning complex features",
290
- severity: "critical"
291
- },
292
- {
293
- id: "repeated-failures",
294
- name: "Repeated Execution Failures",
295
- description: "Multiple consecutive plans with low accuracy or high iterations",
296
- detectionRule: (metrics: PlanMetrics[]) => {
297
- if (metrics.length < 3) return false;
298
- const recent = metrics.slice(-3);
299
- const lowAccuracy = recent.filter(m => m.plan_accuracy < 60).length;
300
- return lowAccuracy >= 2;
301
- },
302
- suggestion: "Review plan validation process, add more thorough feasibility checks",
303
- severity: "critical"
304
- },
305
- {
306
- id: "mcp-overuse",
307
- name: "MCP Overuse",
308
- description: "Using MCP tools for tasks that don't require them",
309
- detectionRule: (metrics: PlanMetrics[]) => {
310
- if (metrics.length < 3) return false;
311
- const recent = metrics.slice(-3);
312
- const highMCPUsage = recent.filter(m => Object.keys(m.mcp_usage).length > 3).length;
313
- const lowAccuracy = recent.filter(m => m.plan_accuracy < 70).length;
314
- return highMCPUsage >= 2 && lowAccuracy >= 2;
315
- },
316
- suggestion: "Use MCP tools only when essential (visual verification, git ops, docs)",
317
- severity: "medium"
318
- },
319
- {
320
- id: "no-validation",
321
- name: "Skipping Validation",
322
- description: "Not running ReadLints or tests after code changes",
323
- detectionRule: (metrics: PlanMetrics[]) => {
324
- if (metrics.length < 3) return false;
325
- const recent = metrics.slice(-3);
326
- return recent.every(m => !m.tool_usage['ReadLints'] && !m.tool_usage['Shell']);
327
- },
328
- suggestion: "Always run ReadLints after edits and tests after substantive changes",
329
- severity: "high"
330
- },
331
- {
332
- id: "context-bloat",
333
- name: "Context Bloat",
334
- description: "Loading too much context, violating minimalism principle",
335
- detectionRule: (metrics: PlanMetrics[]) => {
336
- // Detect via low efficiency + high iteration pattern
337
- if (metrics.length < 3) return false;
338
- const recent = metrics.slice(-3);
339
- return recent.every(m => m.iteration_count > 3 && m.tool_efficiency < 60);
340
- },
341
- suggestion: "Apply tiered injection, use references over content, keep rules under 20 lines",
342
- severity: "medium"
343
- }
344
- ];
345
-
346
- function detectAntiPatterns(metrics: PlanMetrics[]): AntiPattern[] {
347
- const detected: AntiPattern[] = [];
348
-
349
- for (const pattern of ANTI_PATTERNS) {
350
- if (pattern.detectionRule(metrics)) {
351
- detected.push(pattern);
352
- }
353
- }
354
-
355
- return detected;
356
- }
357
-
358
- function identifyRepeatedMistakes(metrics: PlanMetrics[]): string[] {
359
- const mistakes: string[] = [];
360
-
361
- if (metrics.length < 5) return mistakes;
362
-
363
- const recent = metrics.slice(-5);
364
-
365
- // Check for repeated low accuracy
366
- const lowAccuracyCount = recent.filter(m => m.plan_accuracy < 70).length;
367
- if (lowAccuracyCount >= 3) {
368
- mistakes.push("Consistently low plan accuracy (<70%)");
369
- }
370
-
371
- // Check for repeated high iterations
372
- const highIterationCount = recent.filter(m => m.iteration_count > 3).length;
373
- if (highIterationCount >= 3) {
374
- mistakes.push("Frequently requiring many iterations (>3)");
375
- }
376
-
377
- // Check for repeated tool failures
378
- const toolFailureCount = recent.filter(m => m.failed_tools && m.failed_tools.length > 0).length;
379
- if (toolFailureCount >= 3) {
380
- mistakes.push("Recurring tool failures");
381
- }
382
-
383
- // Check for declining performance
384
- const firstHalf = recent.slice(0, Math.floor(recent.length / 2));
385
- const secondHalf = recent.slice(Math.floor(recent.length / 2));
386
- const firstAvg = firstHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / firstHalf.length;
387
- const secondAvg = secondHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / secondHalf.length;
388
-
389
- if (secondAvg < firstAvg - 10) {
390
- mistakes.push("Performance declining over time");
391
- }
392
-
393
- return mistakes;
394
- }
395
-
396
- function generateImprovementSuggestions(antiPatterns: AntiPattern[], mistakes: string[]): string[] {
397
- const suggestions: string[] = [];
398
-
399
- // Add suggestions from detected anti-patterns
400
- for (const pattern of antiPatterns) {
401
- suggestions.push(`[${pattern.severity.toUpperCase()}] ${pattern.suggestion}`);
402
- }
403
-
404
- // Add suggestions for repeated mistakes
405
- if (mistakes.includes("Consistently low plan accuracy (<70%)")) {
406
- suggestions.push("Improve pre-plan research phase - spend 2-3 min on SemanticSearch");
407
- suggestions.push("Add validation subagent before presenting plans");
408
- }
409
-
410
- if (mistakes.includes("Frequently requiring many iterations (>3)")) {
411
- suggestions.push("Add more thorough feasibility validation");
412
- suggestions.push("Create checkpoints before risky changes");
413
- suggestions.push("Use /grind loop strategically (max 5 iterations)");
414
- }
415
-
416
- if (mistakes.includes("Recurring tool failures")) {
417
- suggestions.push("Verify tool prerequisites before use");
418
- suggestions.push("Follow tool escalation protocol (Tier 1 → Tier 4)");
419
- suggestions.push("Check MCP server availability");
420
- }
421
-
422
- if (mistakes.includes("Performance declining over time")) {
423
- suggestions.push("Review recent changes to workflow or rules");
424
- suggestions.push("Analyze failing plans for common patterns");
425
- suggestions.push("Consider updating anti-pattern detection rules");
426
- }
427
-
428
- return suggestions;
429
- }
430
-
431
- function analyzeTrend(metrics: PlanMetrics[]): "improving" | "stable" | "declining" {
432
- if (metrics.length < 5) return "stable";
433
-
434
- const recent = metrics.slice(-5);
435
- const firstHalf = recent.slice(0, Math.floor(recent.length / 2));
436
- const secondHalf = recent.slice(Math.floor(recent.length / 2));
437
-
438
- const firstAvg = firstHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / firstHalf.length;
439
- const secondAvg = secondHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / secondHalf.length;
440
-
441
- if (secondAvg > firstAvg + 5) return "improving";
442
- if (secondAvg < firstAvg - 5) return "declining";
443
- return "stable";
444
- }
445
-
446
- function performPatternAnalysis(metrics: PlanMetrics[]): PatternAnalysis {
447
- const antiPatterns = detectAntiPatterns(metrics);
448
- const mistakes = identifyRepeatedMistakes(metrics);
449
- const suggestions = generateImprovementSuggestions(antiPatterns, mistakes);
450
- const trend = analyzeTrend(metrics);
451
-
452
- return {
453
- detectedPatterns: antiPatterns,
454
- repeatedMistakes: mistakes,
455
- improvementSuggestions: suggestions,
456
- trendDirection: trend
457
- };
458
- }
459
-
460
- // Load existing metrics
461
- let allMetrics: PlanMetrics[] = [];
462
- if (existsSync(METRICS_FILE)) {
463
- try {
464
- const content = readFileSync(METRICS_FILE, "utf-8");
465
- allMetrics = JSON.parse(content);
466
- } catch (error) {
467
- console.error("[Plan Tracker] Error loading metrics:", error);
468
- allMetrics = [];
469
- }
470
- }
471
-
472
- // Load conversation and extract data
473
- const conversation = loadConversation(input.conversation_id);
474
- const toolCalls = conversation ? extractToolCalls(conversation) : [];
475
- const parallelSubagents = conversation ? countParallelSubagents(conversation) : 0;
476
- const planType = conversation ? detectPlanType(conversation) : 'unknown';
477
-
478
- // Track MCP usage and effectiveness
479
- const mcpUsage: Record<string, number> = {};
480
- const mcpToolsUsed: string[] = [];
481
- const failedTools: string[] = [];
482
-
483
- for (const tool of toolCalls) {
484
- if (tool.mcp_server) {
485
- mcpUsage[tool.mcp_server] = (mcpUsage[tool.mcp_server] || 0) + 1;
486
- if (!mcpToolsUsed.includes(tool.name)) {
487
- mcpToolsUsed.push(tool.name);
488
- }
489
- }
490
-
491
- if (!tool.success) {
492
- failedTools.push(tool.name);
493
- }
494
- }
495
-
496
- // MCP Effectiveness Analysis
497
- if (Object.keys(mcpUsage).length > 0) {
498
- const mcpServers = Object.keys(mcpUsage).join(', ');
499
- console.error(`[Plan Tracker] MCP servers used: ${mcpServers}`);
500
-
501
- // Track MCP effectiveness per server
502
- const mcpEffectiveness: Record<string, { usage: number; success: number }> = {};
503
- toolCalls.forEach(tool => {
504
- if (tool.mcp_server) {
505
- if (!mcpEffectiveness[tool.mcp_server]) {
506
- mcpEffectiveness[tool.mcp_server] = { usage: 0, success: 0 };
507
- }
508
- mcpEffectiveness[tool.mcp_server].usage++;
509
- if (tool.success) mcpEffectiveness[tool.mcp_server].success++;
510
- }
511
- });
512
-
513
- // Report effectiveness
514
- Object.entries(mcpEffectiveness).forEach(([server, data]) => {
515
- const successRate = Math.round((data.success / data.usage) * 100);
516
- console.error(`[Plan Tracker] ${server}: ${data.usage} calls, ${successRate}% success rate`);
517
-
518
- if (successRate < 50) {
519
- console.error(`[Plan Tracker] ⚠ ${server} has low success rate (<50%). Consider disabling or reviewing usage.`);
520
- }
521
- });
522
- }
523
-
524
- // Track general tool usage
525
- const toolUsage: Record<string, number> = {};
526
- for (const tool of toolCalls) {
527
- toolUsage[tool.name] = (toolUsage[tool.name] || 0) + 1;
528
- }
529
-
530
- // Estimate cost
531
- const estimatedCost = planTrackingConfig.enableCostTracking ? estimateCost(toolCalls) : undefined;
532
-
533
- // Perform pattern analysis (if enabled)
534
- let detectedAntiPatterns: string[] = [];
535
- let repeatedMistakes: string[] = [];
536
-
537
- if (planTrackingConfig.enablePatternAnalysis && allMetrics.length >= 3) {
538
- const analysis = performPatternAnalysis(allMetrics);
539
- detectedAntiPatterns = analysis.detectedPatterns.map(p => p.id);
540
- repeatedMistakes = analysis.repeatedMistakes;
541
-
542
- if (analysis.detectedPatterns.length > 0) {
543
- console.error(`[Plan Tracker] ⚠ Detected ${analysis.detectedPatterns.length} anti-pattern(s):`);
544
- for (const pattern of analysis.detectedPatterns) {
545
- console.error(` - [${pattern.severity.toUpperCase()}] ${pattern.name}: ${pattern.suggestion}`);
546
- }
547
- }
548
-
549
- if (analysis.repeatedMistakes.length > 0) {
550
- console.error(`[Plan Tracker] ⚠ Repeated mistakes detected:`);
551
- for (const mistake of analysis.repeatedMistakes) {
552
- console.error(` - ${mistake}`);
553
- }
554
- }
555
-
556
- console.error(`[Plan Tracker] Trend: ${analysis.trendDirection}`);
557
- }
558
-
559
- // Create metric entry (used whether tracking is enabled or not)
560
- const newMetric: PlanMetrics = {
561
- plan_accuracy: calculatePlanAccuracy(input, conversation),
562
- time_variance: 0,
563
- tool_usage: toolUsage,
564
- iteration_count: input.loop_count,
565
- blocker_count: input.status === "error" ? 1 : 0,
566
- timestamp: new Date().toISOString(),
567
- plan_type: planType,
568
- mcp_usage: mcpUsage,
569
- tool_efficiency: calculateToolEfficiency(toolCalls),
570
- parallel_subagents: parallelSubagents,
571
- mcp_tools_used: mcpToolsUsed,
572
- failed_tools: failedTools,
573
- conversation_duration_ms: undefined,
574
- cost_estimate: estimatedCost,
575
- anti_patterns_detected: detectedAntiPatterns,
576
- repeated_mistakes: repeatedMistakes
577
- };
578
-
579
- // Save metric entry if tracking is enabled
580
- if (planTrackingConfig.trackMetrics) {
581
- // Add to metrics
582
- allMetrics.push(newMetric);
583
-
584
- // Save updated metrics
585
- try {
586
- writeFileSync(METRICS_FILE, JSON.stringify(allMetrics, null, 2));
587
- console.error("[Plan Tracker] ✓ Metrics saved");
588
- } catch (error) {
589
- console.error("[Plan Tracker] Error saving metrics:", error);
590
- }
591
- } else {
592
- console.error("[Plan Tracker] ⚠ Metrics tracking disabled in config");
593
- }
594
-
595
- // Calculate rolling averages (only if tracking is enabled)
596
- let avgAccuracy = 75;
597
- let avgIterations = 2;
598
- let avgToolEfficiency = 75;
599
- let avgParallelSubagents = 0;
600
- let avgCost = 0;
601
-
602
- if (planTrackingConfig.trackMetrics && allMetrics.length > 0) {
603
- const recentMetrics = allMetrics.slice(-10);
604
- avgAccuracy = recentMetrics.reduce((sum, m) => sum + m.plan_accuracy, 0) / recentMetrics.length;
605
- avgIterations = recentMetrics.reduce((sum, m) => sum + m.iteration_count, 0) / recentMetrics.length;
606
- avgToolEfficiency = recentMetrics.reduce((sum, m) => sum + (m.tool_efficiency || 75), 0) / recentMetrics.length;
607
- avgParallelSubagents = recentMetrics.reduce((sum, m) => sum + (m.parallel_subagents || 0), 0) / recentMetrics.length;
608
-
609
- if (planTrackingConfig.enableCostTracking) {
610
- avgCost = recentMetrics.reduce((sum, m) => sum + (m.cost_estimate || 0), 0) / recentMetrics.length;
611
- }
612
-
613
- console.error(`[Plan Tracker] Rolling averages (last 10 plans):`);
614
- console.error(` - Accuracy: ${avgAccuracy.toFixed(1)}%`);
615
- console.error(` - Avg iterations: ${avgIterations.toFixed(1)}`);
616
- console.error(` - Tool efficiency: ${avgToolEfficiency.toFixed(1)}`);
617
- console.error(` - Avg parallel subagents: ${avgParallelSubagents.toFixed(1)}`);
618
- if (planTrackingConfig.enableCostTracking) {
619
- console.error(` - Avg estimated cost: ${avgCost.toFixed(0)} tokens`);
620
- }
621
- }
622
-
623
- // Analyze MCP effectiveness
624
- const currentAccuracy = calculatePlanAccuracy(input, conversation);
625
-
626
- if (Object.keys(mcpUsage).length > 0) {
627
- console.error(`[Plan Tracker] MCP tools used in this session: ${mcpToolsUsed.join(', ')}`);
628
-
629
- // Check correlation with success
630
- if (currentAccuracy > 85 && Object.keys(mcpUsage).length > 0) {
631
- console.error("[Plan Tracker] ✓ High accuracy with MCP tools - good tool selection");
632
- }
633
- }
634
-
635
- // Analyze parallel subagent usage
636
- if (parallelSubagents > 0) {
637
- console.error(`[Plan Tracker] Used ${parallelSubagents} parallel subagent(s)`);
638
-
639
- if (parallelSubagents > 2 && currentAccuracy > 80) {
640
- console.error("[Plan Tracker] ✓ Parallel exploration appears effective");
641
- }
642
- }
643
-
644
- // Provide feedback based on metrics (if enabled)
645
- if (planTrackingConfig.provideFeedback) {
646
- if (input.status === "completed") {
647
- if (input.loop_count === 0) {
648
- console.error("[Plan Tracker] ✓ Perfect execution (no iterations needed)");
649
- } else if (input.loop_count <= 2) {
650
- console.error("[Plan Tracker] ✓ Good execution (minimal iterations)");
651
- } else if (input.loop_count > planTrackingConfig.maxIterations) {
652
- console.error("[Plan Tracker] ⚠ High iteration count - plan accuracy may need improvement");
653
- }
654
-
655
- // Suggest improvements if accuracy is low
656
- if (avgAccuracy < planTrackingConfig.accuracyThreshold) {
657
- const output: StopHookOutput = {
658
- followup_message: `Plan accuracy has been below ${planTrackingConfig.accuracyThreshold}% recently. Consider:\n1. More thorough pre-plan research\n2. Better validation before execution\n3. More specific tool selection`,
659
- };
660
- console.log(JSON.stringify(output));
661
- process.exit(0);
662
- }
663
-
664
- // Suggest tool efficiency improvements
665
- if (avgToolEfficiency < planTrackingConfig.efficiencyThreshold) {
666
- const output: StopHookOutput = {
667
- followup_message: "Tool efficiency has been low. Consider:\n1. Using simpler tools when possible (Tier 1: Shell, Read, Write)\n2. Avoiding expensive MCP calls for simple tasks\n3. Using 'fast' model for subagents",
668
- };
669
- console.log(JSON.stringify(output));
670
- process.exit(0);
671
- }
672
- }
673
-
674
- if (input.status === "error") {
675
- console.error("[Plan Tracker] ✗ Execution failed - analyzing patterns");
676
-
677
- // Check for recurring failures
678
- if (planTrackingConfig.trackMetrics) {
679
- const recentFailures = allMetrics.slice(-10).filter(m => m.blocker_count > 0).length;
680
- if (recentFailures > 2) {
681
- const output: StopHookOutput = {
682
- followup_message: "Multiple execution failures detected. Common causes:\n1. Insufficient research phase\n2. Missing validation steps\n3. Unrealistic time estimates\n\nWould you like to review the plan approach?",
683
- };
684
- console.log(JSON.stringify(output));
685
- process.exit(0);
686
- }
687
- }
688
-
689
- // Check for tool-related failures
690
- if (failedTools.length > 0) {
691
- console.error(`[Plan Tracker] Failed tools: ${failedTools.join(', ')}`);
692
- const output: StopHookOutput = {
693
- followup_message: `Execution failed. Failed tools: ${failedTools.join(', ')}. Consider:\n1. Verifying tool prerequisites\n2. Using alternative tools\n3. Checking tool configuration`,
694
- };
695
- console.log(JSON.stringify(output));
696
- process.exit(0);
697
- }
698
- }
699
-
700
- // Handle aborted execution (from plan-mode-monitor merge)
701
- if (input.status === "aborted") {
702
- console.error("[Plan Tracker] ⚠ Plan execution aborted by user");
703
-
704
- // Gather feedback
705
- if (input.loop_count < 2) {
706
- const output: StopHookOutput = {
707
- followup_message: "Plan execution aborted. What was the main issue?\n1. Plan was inaccurate\n2. Requirements changed\n3. Found better approach\n4. Other",
708
- };
709
- console.log(JSON.stringify(output));
710
- process.exit(0);
711
- }
712
- }
713
- } else {
714
- console.error("[Plan Tracker] ⚠ Feedback disabled in config");
715
- }
716
-
717
- // Default: stop without followup
718
- console.log(JSON.stringify({}));
719
- process.exit(0);
1
+ // .claude/hooks/plan-quality-tracker.ts
2
+ // Stop hook — plan metrics tracking from the real tool usage in the Claude Code transcript
3
+
4
+ import { writeFileSync, readFileSync, existsSync } from "fs";
5
+ import { join } from "path";
6
+ import {
7
+ readHookInput,
8
+ loadCurrentTurn,
9
+ mcpServer,
10
+ stateDir,
11
+ notify,
12
+ done,
13
+ SUBAGENT_TOOLS,
14
+ type Turn,
15
+ } from "./lib/claude-hook.ts";
16
+
17
+ interface AntiPattern {
18
+ id: string;
19
+ name: string;
20
+ description: string;
21
+ detectionRule: (metrics: PlanMetrics[]) => boolean;
22
+ suggestion: string;
23
+ severity: "critical" | "high" | "medium" | "low";
24
+ }
25
+
26
+ interface PlanMetrics {
27
+ plan_accuracy: number;
28
+ time_variance: number;
29
+ tool_usage: Record<string, number>;
30
+ iteration_count: number;
31
+ blocker_count: number;
32
+ timestamp: string;
33
+ plan_type: string;
34
+ mcp_usage: Record<string, number>;
35
+ tool_efficiency: number;
36
+ parallel_subagents: number;
37
+ mcp_tools_used: string[];
38
+ failed_tools: string[];
39
+ conversation_duration_ms?: number;
40
+ cost_estimate?: number;
41
+ anti_patterns_detected?: string[];
42
+ repeated_mistakes?: string[];
43
+ session_id?: string;
44
+ turn_id?: string;
45
+ }
46
+
47
+ interface PatternAnalysis {
48
+ detectedPatterns: AntiPattern[];
49
+ repeatedMistakes: string[];
50
+ improvementSuggestions: string[];
51
+ trendDirection: "improving" | "stable" | "declining";
52
+ }
53
+
54
+ interface ToolCall {
55
+ name: string;
56
+ execution_time_ms?: number;
57
+ success: boolean;
58
+ mcp_server?: string;
59
+ }
60
+
61
+ interface HooksConfig {
62
+ planTracking?: {
63
+ trackMetrics?: boolean;
64
+ accuracyThreshold?: number;
65
+ efficiencyThreshold?: number;
66
+ maxIterations?: number;
67
+ provideFeedback?: boolean;
68
+ enableCostTracking?: boolean;
69
+ enablePatternAnalysis?: boolean;
70
+ };
71
+ }
72
+
73
+ const input = await readHookInput();
74
+
75
+ const METRICS_FILE = join(stateDir(input), "plan-metrics.json");
76
+
77
+ // Load configuration
78
+ const configPath = join(stateDir(input), 'hooks.config.json');
79
+ const config: HooksConfig = existsSync(configPath)
80
+ ? JSON.parse(readFileSync(configPath, 'utf-8'))
81
+ : {};
82
+
83
+ const planTrackingConfig = {
84
+ trackMetrics: config.planTracking?.trackMetrics !== false,
85
+ accuracyThreshold: config.planTracking?.accuracyThreshold || 70,
86
+ efficiencyThreshold: config.planTracking?.efficiencyThreshold || 60,
87
+ maxIterations: config.planTracking?.maxIterations || 5,
88
+ provideFeedback: config.planTracking?.provideFeedback !== false,
89
+ enableCostTracking: config.planTracking?.enableCostTracking !== false,
90
+ enablePatternAnalysis: config.planTracking?.enablePatternAnalysis !== false
91
+ };
92
+
93
+ console.error(`[Plan Tracker] Processing execution data for session ${input.session_id}...`);
94
+
95
+ // Helper functions
96
+ function extractToolCalls(turn: Turn): ToolCall[] {
97
+ return turn.toolUses.map(t => ({
98
+ name: t.name,
99
+ success: !t.isError,
100
+ mcp_server: mcpServer(t.name)
101
+ }));
102
+ }
103
+
104
+ function countParallelSubagents(turn: Turn): number {
105
+ return turn.toolUses.filter(t => SUBAGENT_TOOLS.includes(t.name)).length;
106
+ }
107
+
108
+ function detectPlanType(turn: Turn): string {
109
+ if (turn.toolUses.some(t => t.name === 'ExitPlanMode')) return 'plan-mode';
110
+
111
+ const conversationText = JSON.stringify(turn.entries).toLowerCase();
112
+
113
+ if (conversationText.includes('plan mode')) return 'plan-mode';
114
+ if (conversationText.includes('browser') || conversationText.includes('web')) return 'browser-automation';
115
+ if (conversationText.includes('github') || conversationText.includes('pr') || conversationText.includes('issue')) return 'github-ops';
116
+ if (conversationText.includes('test')) return 'testing';
117
+ if (conversationText.includes('refactor')) return 'refactoring';
118
+ if (conversationText.includes('bug') || conversationText.includes('fix')) return 'bug-fix';
119
+ if (conversationText.includes('feature') || conversationText.includes('implement')) return 'feature-development';
120
+
121
+ return 'general';
122
+ }
123
+
124
+ // Iterations = failed tool calls that had to be retried (failed tests, lint, edits...)
125
+ function calculatePlanAccuracy(iterations: number): number {
126
+ // Fewer iterations = higher accuracy
127
+ if (iterations === 0) return 100;
128
+ if (iterations <= 2) return 85;
129
+ if (iterations <= 5) return 70;
130
+ return 50;
131
+ }
132
+
133
+ function calculateToolEfficiency(tools: ToolCall[]): number {
134
+ if (tools.length === 0) return 75; // Neutral score if no tools
135
+
136
+ // Tier 1: Simple tools (Bash, Read, Edit, Write, Glob, Grep)
137
+ // Tier 2: Analysis (LSP)
138
+ // Tier 3: Complex (Agent/Task subagents, WebSearch, WebFetch)
139
+ // Tier 4: MCP tools (mcp__*)
140
+
141
+ const tierScores: Record<string, number> = {
142
+ 'Bash': 100, 'Read': 100, 'Edit': 100, 'MultiEdit': 100, 'Write': 100, 'Glob': 100, 'Grep': 100,
143
+ 'LSP': 80,
144
+ 'Agent': 60, 'Task': 60, 'WebSearch': 60, 'WebFetch': 60,
145
+ };
146
+
147
+ let totalScore = 0;
148
+ let toolCount = 0;
149
+
150
+ for (const tool of tools) {
151
+ if (tool.mcp_server) {
152
+ totalScore += 40; // MCP tools are most expensive
153
+ toolCount++;
154
+ } else {
155
+ const tierScore = tierScores[tool.name] || 75; // Default to middle tier
156
+ totalScore += tierScore;
157
+ toolCount++;
158
+ }
159
+ }
160
+
161
+ return toolCount > 0 ? Math.round(totalScore / toolCount) : 75;
162
+ }
163
+
164
+ function estimateCost(tools: ToolCall[]): number {
165
+ // Estimated token cost per tool type
166
+ const costEstimates: Record<string, number> = {
167
+ 'Bash': 100, 'Read': 500, 'Edit': 800, 'MultiEdit': 1000, 'Write': 1000, 'Glob': 200, 'Grep': 300,
168
+ 'LSP': 800,
169
+ 'Agent': 3000, 'Task': 3000, 'WebSearch': 2000, 'WebFetch': 2500,
170
+ };
171
+
172
+ let totalCost = 0;
173
+ for (const tool of tools) {
174
+ if (tool.mcp_server) {
175
+ totalCost += 5000; // MCP tools estimated higher cost
176
+ } else {
177
+ totalCost += costEstimates[tool.name] || 1000;
178
+ }
179
+ }
180
+
181
+ return totalCost;
182
+ }
183
+
184
+ // Anti-pattern definitions based on AGENT-INSTRUCTION-BEST-PRACTICES.md
185
+ const ANTI_PATTERNS: AntiPattern[] = [
186
+ {
187
+ id: "over-engineering",
188
+ name: "Over-Engineering",
189
+ description: "Using expensive tools when cheaper alternatives would work",
190
+ detectionRule: (metrics: PlanMetrics[]) => {
191
+ if (metrics.length < 3) return false;
192
+ const recent = metrics.slice(-3);
193
+ const avgEfficiency = recent.reduce((sum, m) => sum + (m.tool_efficiency || 75), 0) / recent.length;
194
+ return avgEfficiency < 50;
195
+ },
196
+ suggestion: "Start with Tier 1 tools (Read, Grep, Glob) before escalating to Tier 3-4",
197
+ severity: "high"
198
+ },
199
+ {
200
+ id: "no-research",
201
+ name: "Planning Without Research",
202
+ description: "Creating plans without using Grep/Glob or Explore subagents",
203
+ detectionRule: (metrics: PlanMetrics[]) => {
204
+ if (metrics.length < 3) return false;
205
+ const recent = metrics.slice(-3);
206
+ return recent.every(m => !m.tool_usage['Grep'] && !m.tool_usage['Glob'] && m.parallel_subagents === 0);
207
+ },
208
+ suggestion: "Always research with Grep/Glob and parallel Explore subagents before planning complex features",
209
+ severity: "critical"
210
+ },
211
+ {
212
+ id: "repeated-failures",
213
+ name: "Repeated Execution Failures",
214
+ description: "Multiple consecutive plans with low accuracy or high iterations",
215
+ detectionRule: (metrics: PlanMetrics[]) => {
216
+ if (metrics.length < 3) return false;
217
+ const recent = metrics.slice(-3);
218
+ const lowAccuracy = recent.filter(m => m.plan_accuracy < 60).length;
219
+ return lowAccuracy >= 2;
220
+ },
221
+ suggestion: "Review plan validation process, add more thorough feasibility checks",
222
+ severity: "critical"
223
+ },
224
+ {
225
+ id: "mcp-overuse",
226
+ name: "MCP Overuse",
227
+ description: "Using MCP tools for tasks that don't require them",
228
+ detectionRule: (metrics: PlanMetrics[]) => {
229
+ if (metrics.length < 3) return false;
230
+ const recent = metrics.slice(-3);
231
+ const highMCPUsage = recent.filter(m => Object.keys(m.mcp_usage).length > 3).length;
232
+ const lowAccuracy = recent.filter(m => m.plan_accuracy < 70).length;
233
+ return highMCPUsage >= 2 && lowAccuracy >= 2;
234
+ },
235
+ suggestion: "Use MCP tools only when essential (visual verification, git ops, docs)",
236
+ severity: "medium"
237
+ },
238
+ {
239
+ id: "no-validation",
240
+ name: "Skipping Validation",
241
+ description: "Not running lint/typecheck or tests after code changes",
242
+ detectionRule: (metrics: PlanMetrics[]) => {
243
+ if (metrics.length < 3) return false;
244
+ const recent = metrics.slice(-3);
245
+ return recent.every(m => !m.tool_usage['Bash']);
246
+ },
247
+ suggestion: "Always run lint/typecheck (Bash) after edits and tests after substantive changes",
248
+ severity: "high"
249
+ },
250
+ {
251
+ id: "context-bloat",
252
+ name: "Context Bloat",
253
+ description: "Loading too much context, violating minimalism principle",
254
+ detectionRule: (metrics: PlanMetrics[]) => {
255
+ // Detect via low efficiency + high iteration pattern
256
+ if (metrics.length < 3) return false;
257
+ const recent = metrics.slice(-3);
258
+ return recent.every(m => m.iteration_count > 3 && m.tool_efficiency < 60);
259
+ },
260
+ suggestion: "Apply tiered injection, use references over content, keep rules under 20 lines",
261
+ severity: "medium"
262
+ }
263
+ ];
264
+
265
+ function detectAntiPatterns(metrics: PlanMetrics[]): AntiPattern[] {
266
+ const detected: AntiPattern[] = [];
267
+
268
+ for (const pattern of ANTI_PATTERNS) {
269
+ if (pattern.detectionRule(metrics)) {
270
+ detected.push(pattern);
271
+ }
272
+ }
273
+
274
+ return detected;
275
+ }
276
+
277
+ function identifyRepeatedMistakes(metrics: PlanMetrics[]): string[] {
278
+ const mistakes: string[] = [];
279
+
280
+ if (metrics.length < 5) return mistakes;
281
+
282
+ const recent = metrics.slice(-5);
283
+
284
+ // Check for repeated low accuracy
285
+ const lowAccuracyCount = recent.filter(m => m.plan_accuracy < 70).length;
286
+ if (lowAccuracyCount >= 3) {
287
+ mistakes.push("Consistently low plan accuracy (<70%)");
288
+ }
289
+
290
+ // Check for repeated high iterations
291
+ const highIterationCount = recent.filter(m => m.iteration_count > 3).length;
292
+ if (highIterationCount >= 3) {
293
+ mistakes.push("Frequently requiring many iterations (>3)");
294
+ }
295
+
296
+ // Check for repeated tool failures
297
+ const toolFailureCount = recent.filter(m => m.failed_tools && m.failed_tools.length > 0).length;
298
+ if (toolFailureCount >= 3) {
299
+ mistakes.push("Recurring tool failures");
300
+ }
301
+
302
+ // Check for declining performance
303
+ const firstHalf = recent.slice(0, Math.floor(recent.length / 2));
304
+ const secondHalf = recent.slice(Math.floor(recent.length / 2));
305
+ const firstAvg = firstHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / firstHalf.length;
306
+ const secondAvg = secondHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / secondHalf.length;
307
+
308
+ if (secondAvg < firstAvg - 10) {
309
+ mistakes.push("Performance declining over time");
310
+ }
311
+
312
+ return mistakes;
313
+ }
314
+
315
+ function generateImprovementSuggestions(antiPatterns: AntiPattern[], mistakes: string[]): string[] {
316
+ const suggestions: string[] = [];
317
+
318
+ // Add suggestions from detected anti-patterns
319
+ for (const pattern of antiPatterns) {
320
+ suggestions.push(`[${pattern.severity.toUpperCase()}] ${pattern.suggestion}`);
321
+ }
322
+
323
+ // Add suggestions for repeated mistakes
324
+ if (mistakes.includes("Consistently low plan accuracy (<70%)")) {
325
+ suggestions.push("Improve pre-plan research phase - spend 2-3 min on Grep/Glob and Explore subagents");
326
+ suggestions.push("Add validation subagent before presenting plans");
327
+ }
328
+
329
+ if (mistakes.includes("Frequently requiring many iterations (>3)")) {
330
+ suggestions.push("Add more thorough feasibility validation");
331
+ suggestions.push("Create checkpoints before risky changes");
332
+ suggestions.push("Use the fix-and-retest loop strategically (max 5 iterations)");
333
+ }
334
+
335
+ if (mistakes.includes("Recurring tool failures")) {
336
+ suggestions.push("Verify tool prerequisites before use");
337
+ suggestions.push("Follow tool escalation protocol (Tier 1 → Tier 4)");
338
+ suggestions.push("Check MCP server availability");
339
+ }
340
+
341
+ if (mistakes.includes("Performance declining over time")) {
342
+ suggestions.push("Review recent changes to workflow or rules");
343
+ suggestions.push("Analyze failing plans for common patterns");
344
+ suggestions.push("Consider updating anti-pattern detection rules");
345
+ }
346
+
347
+ return suggestions;
348
+ }
349
+
350
+ function analyzeTrend(metrics: PlanMetrics[]): "improving" | "stable" | "declining" {
351
+ if (metrics.length < 5) return "stable";
352
+
353
+ const recent = metrics.slice(-5);
354
+ const firstHalf = recent.slice(0, Math.floor(recent.length / 2));
355
+ const secondHalf = recent.slice(Math.floor(recent.length / 2));
356
+
357
+ const firstAvg = firstHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / firstHalf.length;
358
+ const secondAvg = secondHalf.reduce((sum, m) => sum + m.plan_accuracy, 0) / secondHalf.length;
359
+
360
+ if (secondAvg > firstAvg + 5) return "improving";
361
+ if (secondAvg < firstAvg - 5) return "declining";
362
+ return "stable";
363
+ }
364
+
365
+ function performPatternAnalysis(metrics: PlanMetrics[]): PatternAnalysis {
366
+ const antiPatterns = detectAntiPatterns(metrics);
367
+ const mistakes = identifyRepeatedMistakes(metrics);
368
+ const suggestions = generateImprovementSuggestions(antiPatterns, mistakes);
369
+ const trend = analyzeTrend(metrics);
370
+
371
+ return {
372
+ detectedPatterns: antiPatterns,
373
+ repeatedMistakes: mistakes,
374
+ improvementSuggestions: suggestions,
375
+ trendDirection: trend
376
+ };
377
+ }
378
+
379
+ // Load existing metrics
380
+ let allMetrics: PlanMetrics[] = [];
381
+ if (existsSync(METRICS_FILE)) {
382
+ try {
383
+ const content = readFileSync(METRICS_FILE, "utf-8");
384
+ allMetrics = JSON.parse(content);
385
+ } catch (error) {
386
+ console.error("[Plan Tracker] Error loading metrics:", error);
387
+ allMetrics = [];
388
+ }
389
+ }
390
+
391
+ // Load the current turn from the transcript; skip turns without tool use (plain Q&A)
392
+ const turn = loadCurrentTurn(input.transcript_path);
393
+ if (!turn || turn.toolUses.length === 0) done();
394
+
395
+ const toolCalls = extractToolCalls(turn);
396
+ const parallelSubagents = countParallelSubagents(turn);
397
+ const planType = detectPlanType(turn);
398
+
399
+ // Track MCP usage and effectiveness
400
+ const mcpUsage: Record<string, number> = {};
401
+ const mcpToolsUsed: string[] = [];
402
+ const failedTools: string[] = [];
403
+
404
+ for (const tool of toolCalls) {
405
+ if (tool.mcp_server) {
406
+ mcpUsage[tool.mcp_server] = (mcpUsage[tool.mcp_server] || 0) + 1;
407
+ if (!mcpToolsUsed.includes(tool.name)) {
408
+ mcpToolsUsed.push(tool.name);
409
+ }
410
+ }
411
+
412
+ if (!tool.success) {
413
+ failedTools.push(tool.name);
414
+ }
415
+ }
416
+
417
+ // MCP Effectiveness Analysis
418
+ if (Object.keys(mcpUsage).length > 0) {
419
+ const mcpServers = Object.keys(mcpUsage).join(', ');
420
+ console.error(`[Plan Tracker] MCP servers used: ${mcpServers}`);
421
+
422
+ // Track MCP effectiveness per server
423
+ const mcpEffectiveness: Record<string, { usage: number; success: number }> = {};
424
+ toolCalls.forEach(tool => {
425
+ if (tool.mcp_server) {
426
+ if (!mcpEffectiveness[tool.mcp_server]) {
427
+ mcpEffectiveness[tool.mcp_server] = { usage: 0, success: 0 };
428
+ }
429
+ mcpEffectiveness[tool.mcp_server].usage++;
430
+ if (tool.success) mcpEffectiveness[tool.mcp_server].success++;
431
+ }
432
+ });
433
+
434
+ // Report effectiveness
435
+ Object.entries(mcpEffectiveness).forEach(([server, data]) => {
436
+ const successRate = Math.round((data.success / data.usage) * 100);
437
+ console.error(`[Plan Tracker] ${server}: ${data.usage} calls, ${successRate}% success rate`);
438
+
439
+ if (successRate < 50) {
440
+ console.error(`[Plan Tracker] ⚠ ${server} has low success rate (<50%). Consider disabling or reviewing usage.`);
441
+ }
442
+ });
443
+ }
444
+
445
+ // Track general tool usage
446
+ const toolUsage: Record<string, number> = {};
447
+ for (const tool of toolCalls) {
448
+ toolUsage[tool.name] = (toolUsage[tool.name] || 0) + 1;
449
+ }
450
+
451
+ // Estimate cost
452
+ const estimatedCost = planTrackingConfig.enableCostTracking ? estimateCost(toolCalls) : undefined;
453
+
454
+ // Perform pattern analysis (if enabled)
455
+ let detectedAntiPatterns: string[] = [];
456
+ let repeatedMistakes: string[] = [];
457
+
458
+ if (planTrackingConfig.enablePatternAnalysis && allMetrics.length >= 3) {
459
+ const analysis = performPatternAnalysis(allMetrics);
460
+ detectedAntiPatterns = analysis.detectedPatterns.map(p => p.id);
461
+ repeatedMistakes = analysis.repeatedMistakes;
462
+
463
+ if (analysis.detectedPatterns.length > 0) {
464
+ console.error(`[Plan Tracker] ⚠ Detected ${analysis.detectedPatterns.length} anti-pattern(s):`);
465
+ for (const pattern of analysis.detectedPatterns) {
466
+ console.error(` - [${pattern.severity.toUpperCase()}] ${pattern.name}: ${pattern.suggestion}`);
467
+ }
468
+ }
469
+
470
+ if (analysis.repeatedMistakes.length > 0) {
471
+ console.error(`[Plan Tracker] ⚠ Repeated mistakes detected:`);
472
+ for (const mistake of analysis.repeatedMistakes) {
473
+ console.error(` - ${mistake}`);
474
+ }
475
+ }
476
+
477
+ console.error(`[Plan Tracker] Trend: ${analysis.trendDirection}`);
478
+ }
479
+
480
+ // Create metric entry (used whether tracking is enabled or not)
481
+ const iterations = failedTools.length;
482
+ const currentAccuracy = calculatePlanAccuracy(iterations);
483
+
484
+ const newMetric: PlanMetrics = {
485
+ plan_accuracy: currentAccuracy,
486
+ time_variance: 0,
487
+ tool_usage: toolUsage,
488
+ iteration_count: iterations,
489
+ blocker_count: 0,
490
+ timestamp: new Date().toISOString(),
491
+ plan_type: planType,
492
+ mcp_usage: mcpUsage,
493
+ tool_efficiency: calculateToolEfficiency(toolCalls),
494
+ parallel_subagents: parallelSubagents,
495
+ mcp_tools_used: mcpToolsUsed,
496
+ failed_tools: failedTools,
497
+ conversation_duration_ms: undefined,
498
+ cost_estimate: estimatedCost,
499
+ anti_patterns_detected: detectedAntiPatterns,
500
+ repeated_mistakes: repeatedMistakes,
501
+ session_id: input.session_id,
502
+ turn_id: turn.id
503
+ };
504
+
505
+ // Save metric entry if tracking is enabled
506
+ if (planTrackingConfig.trackMetrics) {
507
+ // Add to metrics; a Stop after a stop-hook continuation updates the same turn's entry
508
+ const last = allMetrics[allMetrics.length - 1];
509
+ if (last && last.session_id === input.session_id && last.turn_id === turn.id) {
510
+ allMetrics[allMetrics.length - 1] = newMetric;
511
+ } else {
512
+ allMetrics.push(newMetric);
513
+ }
514
+
515
+ // Save updated metrics
516
+ try {
517
+ writeFileSync(METRICS_FILE, JSON.stringify(allMetrics, null, 2));
518
+ console.error("[Plan Tracker] ✓ Metrics saved");
519
+ } catch (error) {
520
+ console.error("[Plan Tracker] Error saving metrics:", error);
521
+ }
522
+ } else {
523
+ console.error("[Plan Tracker] ⚠ Metrics tracking disabled in config");
524
+ }
525
+
526
+ // Calculate rolling averages (only if tracking is enabled)
527
+ let avgAccuracy = 75;
528
+ let avgToolEfficiency = 75;
529
+ let avgCost = 0;
530
+
531
+ if (planTrackingConfig.trackMetrics && allMetrics.length > 0) {
532
+ const recentMetrics = allMetrics.slice(-10);
533
+ avgAccuracy = recentMetrics.reduce((sum, m) => sum + m.plan_accuracy, 0) / recentMetrics.length;
534
+ const avgIterations = recentMetrics.reduce((sum, m) => sum + m.iteration_count, 0) / recentMetrics.length;
535
+ avgToolEfficiency = recentMetrics.reduce((sum, m) => sum + (m.tool_efficiency || 75), 0) / recentMetrics.length;
536
+ const avgParallelSubagents = recentMetrics.reduce((sum, m) => sum + (m.parallel_subagents || 0), 0) / recentMetrics.length;
537
+
538
+ if (planTrackingConfig.enableCostTracking) {
539
+ avgCost = recentMetrics.reduce((sum, m) => sum + (m.cost_estimate || 0), 0) / recentMetrics.length;
540
+ }
541
+
542
+ console.error(`[Plan Tracker] Rolling averages (last 10 plans):`);
543
+ console.error(` - Accuracy: ${avgAccuracy.toFixed(1)}%`);
544
+ console.error(` - Avg iterations: ${avgIterations.toFixed(1)}`);
545
+ console.error(` - Tool efficiency: ${avgToolEfficiency.toFixed(1)}`);
546
+ console.error(` - Avg parallel subagents: ${avgParallelSubagents.toFixed(1)}`);
547
+ if (planTrackingConfig.enableCostTracking) {
548
+ console.error(` - Avg estimated cost: ${avgCost.toFixed(0)} tokens`);
549
+ }
550
+ }
551
+
552
+ // Analyze MCP effectiveness
553
+
554
+ if (Object.keys(mcpUsage).length > 0) {
555
+ console.error(`[Plan Tracker] MCP tools used in this session: ${mcpToolsUsed.join(', ')}`);
556
+
557
+ // Check correlation with success
558
+ if (currentAccuracy > 85 && Object.keys(mcpUsage).length > 0) {
559
+ console.error("[Plan Tracker] ✓ High accuracy with MCP tools - good tool selection");
560
+ }
561
+ }
562
+
563
+ // Analyze parallel subagent usage
564
+ if (parallelSubagents > 0) {
565
+ console.error(`[Plan Tracker] Used ${parallelSubagents} parallel subagent(s)`);
566
+
567
+ if (parallelSubagents > 2 && currentAccuracy > 80) {
568
+ console.error("[Plan Tracker] ✓ Parallel exploration appears effective");
569
+ }
570
+ }
571
+
572
+ // Provide feedback based on metrics (if enabled) — advisory, shown to the user
573
+ if (planTrackingConfig.provideFeedback) {
574
+ if (iterations === 0) {
575
+ console.error("[Plan Tracker] ✓ Perfect execution (no iterations needed)");
576
+ } else if (iterations <= 2) {
577
+ console.error("[Plan Tracker] ✓ Good execution (minimal iterations)");
578
+ } else if (iterations > planTrackingConfig.maxIterations) {
579
+ console.error("[Plan Tracker] ⚠ High iteration count - plan accuracy may need improvement");
580
+ }
581
+
582
+ // Suggest improvements if accuracy is low
583
+ if (avgAccuracy < planTrackingConfig.accuracyThreshold) {
584
+ notify(`Plan accuracy has been below ${planTrackingConfig.accuracyThreshold}% recently. Consider:\n1. More thorough pre-plan research\n2. Better validation before execution\n3. More specific tool selection`);
585
+ process.exit(0);
586
+ }
587
+
588
+ // Suggest tool efficiency improvements
589
+ if (avgToolEfficiency < planTrackingConfig.efficiencyThreshold) {
590
+ notify("Tool efficiency has been low. Consider:\n1. Using simpler tools when possible (Tier 1: Bash, Read, Edit, Grep)\n2. Avoiding expensive MCP calls for simple tasks\n3. Using `model: haiku` for Explore subagents");
591
+ process.exit(0);
592
+ }
593
+ } else {
594
+ console.error("[Plan Tracker] ⚠ Feedback disabled in config");
595
+ }
596
+
597
+ // Default: stop without followup
598
+ done();