@agent-relay/resiliency 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/context-persistence.d.ts +140 -0
  2. package/dist/context-persistence.d.ts.map +1 -0
  3. package/dist/context-persistence.js +397 -0
  4. package/dist/context-persistence.js.map +1 -0
  5. package/dist/crash-insights.d.ts +156 -0
  6. package/dist/crash-insights.d.ts.map +1 -0
  7. package/dist/crash-insights.js +492 -0
  8. package/dist/crash-insights.js.map +1 -0
  9. package/dist/gossip-health.d.ts +137 -0
  10. package/dist/gossip-health.d.ts.map +1 -0
  11. package/dist/gossip-health.js +241 -0
  12. package/dist/gossip-health.js.map +1 -0
  13. package/dist/health-monitor.d.ts +97 -0
  14. package/dist/health-monitor.d.ts.map +1 -0
  15. package/dist/health-monitor.js +291 -0
  16. package/dist/health-monitor.js.map +1 -0
  17. package/dist/index.d.ts +68 -0
  18. package/dist/index.d.ts.map +1 -0
  19. package/dist/index.js +68 -0
  20. package/dist/index.js.map +1 -0
  21. package/dist/leader-watchdog.d.ts +109 -0
  22. package/dist/leader-watchdog.d.ts.map +1 -0
  23. package/dist/leader-watchdog.js +189 -0
  24. package/dist/leader-watchdog.js.map +1 -0
  25. package/dist/logger.d.ts +114 -0
  26. package/dist/logger.d.ts.map +1 -0
  27. package/dist/logger.js +250 -0
  28. package/dist/logger.js.map +1 -0
  29. package/dist/memory-monitor.d.ts +172 -0
  30. package/dist/memory-monitor.d.ts.map +1 -0
  31. package/dist/memory-monitor.js +599 -0
  32. package/dist/memory-monitor.js.map +1 -0
  33. package/dist/metrics.d.ts +115 -0
  34. package/dist/metrics.d.ts.map +1 -0
  35. package/dist/metrics.js +239 -0
  36. package/dist/metrics.js.map +1 -0
  37. package/dist/provider-context.d.ts +100 -0
  38. package/dist/provider-context.d.ts.map +1 -0
  39. package/dist/provider-context.js +362 -0
  40. package/dist/provider-context.js.map +1 -0
  41. package/dist/stateless-lead.d.ts +149 -0
  42. package/dist/stateless-lead.d.ts.map +1 -0
  43. package/dist/stateless-lead.js +308 -0
  44. package/dist/stateless-lead.js.map +1 -0
  45. package/dist/supervisor.d.ts +147 -0
  46. package/dist/supervisor.d.ts.map +1 -0
  47. package/dist/supervisor.js +459 -0
  48. package/dist/supervisor.js.map +1 -0
  49. package/package.json +28 -0
@@ -0,0 +1,156 @@
1
+ /**
2
+ * Crash Insights Service
3
+ *
4
+ * Captures and analyzes agent crashes to provide actionable insights:
5
+ * - Memory state at crash time
6
+ * - Crash history and patterns
7
+ * - Root cause analysis
8
+ * - Recommendations for prevention
9
+ */
10
+ import { EventEmitter } from 'events';
11
+ import { AgentMemoryMonitor, CrashMemoryContext } from './memory-monitor.js';
12
+ export interface CrashRecord {
13
+ id: string;
14
+ agentName: string;
15
+ pid: number;
16
+ crashTime: Date;
17
+ exitCode: number | null;
18
+ signal: string | null;
19
+ reason: string;
20
+ memoryContext: CrashMemoryContext;
21
+ stackTrace?: string;
22
+ lastOutput?: string;
23
+ environment: {
24
+ nodeVersion: string;
25
+ platform: string;
26
+ arch: string;
27
+ systemMemory: {
28
+ total: number;
29
+ free: number;
30
+ };
31
+ uptime: number;
32
+ };
33
+ analysis: CrashAnalysis;
34
+ }
35
+ export interface CrashAnalysis {
36
+ likelyCause: 'oom' | 'memory_leak' | 'sudden_spike' | 'signal' | 'error' | 'unknown';
37
+ confidence: 'high' | 'medium' | 'low';
38
+ summary: string;
39
+ details: string[];
40
+ recommendations: string[];
41
+ relatedCrashes: string[];
42
+ }
43
+ export interface CrashPattern {
44
+ pattern: string;
45
+ occurrences: number;
46
+ lastSeen: Date;
47
+ affectedAgents: string[];
48
+ avgMemoryAtCrash: number;
49
+ commonCause: string;
50
+ }
51
+ export interface CrashStats {
52
+ totalCrashes: number;
53
+ crashesByAgent: Record<string, number>;
54
+ crashesByCause: Record<string, number>;
55
+ avgTimeBetweenCrashes: number;
56
+ mostCrashProne: {
57
+ agent: string;
58
+ count: number;
59
+ } | null;
60
+ recentCrashes: CrashRecord[];
61
+ patterns: CrashPattern[];
62
+ }
63
+ export declare class CrashInsightsService extends EventEmitter {
64
+ private crashes;
65
+ private memoryMonitor;
66
+ private persistPath;
67
+ private maxCrashHistory;
68
+ constructor(memoryMonitor?: AgentMemoryMonitor);
69
+ /**
70
+ * Set the memory monitor instance
71
+ */
72
+ setMemoryMonitor(monitor: AgentMemoryMonitor): void;
73
+ /**
74
+ * Record a crash event
75
+ */
76
+ recordCrash(params: {
77
+ agentName: string;
78
+ pid: number;
79
+ exitCode: number | null;
80
+ signal: string | null;
81
+ reason: string;
82
+ stackTrace?: string;
83
+ lastOutput?: string;
84
+ }): CrashRecord;
85
+ /**
86
+ * Get crash history for an agent
87
+ */
88
+ getCrashHistory(agentName?: string, limit?: number): CrashRecord[];
89
+ /**
90
+ * Get a specific crash record
91
+ */
92
+ getCrash(id: string): CrashRecord | undefined;
93
+ /**
94
+ * Get crash statistics
95
+ */
96
+ getStats(): CrashStats;
97
+ /**
98
+ * Get insights and recommendations
99
+ */
100
+ getInsights(): {
101
+ summary: string;
102
+ topIssues: Array<{
103
+ issue: string;
104
+ severity: 'high' | 'medium' | 'low';
105
+ recommendation: string;
106
+ }>;
107
+ healthScore: number;
108
+ trends: Array<{
109
+ metric: string;
110
+ trend: 'improving' | 'stable' | 'degrading';
111
+ details: string;
112
+ }>;
113
+ };
114
+ /**
115
+ * Analyze a crash and determine likely cause
116
+ */
117
+ private analyzeCrash;
118
+ /**
119
+ * Find related crashes
120
+ */
121
+ private findRelatedCrashes;
122
+ /**
123
+ * Detect crash patterns
124
+ */
125
+ private detectPatterns;
126
+ /**
127
+ * Generate crash summary
128
+ */
129
+ private generateCrashSummary;
130
+ /**
131
+ * Generate overall summary
132
+ */
133
+ private generateSummary;
134
+ /**
135
+ * Create empty memory context when no monitor available
136
+ */
137
+ private createEmptyMemoryContext;
138
+ /**
139
+ * Load crashes from disk
140
+ */
141
+ private loadCrashes;
142
+ /**
143
+ * Save crashes to disk
144
+ */
145
+ private saveCrashes;
146
+ /**
147
+ * Clear all crash history
148
+ */
149
+ clear(): void;
150
+ /**
151
+ * Structured logging
152
+ */
153
+ private log;
154
+ }
155
+ export declare function getCrashInsights(memoryMonitor?: AgentMemoryMonitor): CrashInsightsService;
156
+ //# sourceMappingURL=crash-insights.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"crash-insights.d.ts","sourceRoot":"","sources":["../src/crash-insights.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAItC,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAEnB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,IAAI,CAAC;IAChB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,kBAAkB,CAAC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE;QACX,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,YAAY,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9C,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,KAAK,GAAG,aAAa,GAAG,cAAc,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;IACrF,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,IAAI,CAAC;IACf,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACxD,aAAa,EAAE,WAAW,EAAE,CAAC;IAC7B,QAAQ,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,qBAAa,oBAAqB,SAAQ,YAAY;IACpD,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,aAAa,CAAmC;IACxD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,eAAe,CAAQ;gBAEnB,aAAa,CAAC,EAAE,kBAAkB;IAc9C;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,kBAAkB,GAAG,IAAI;IAInD;;OAEG;IACH,WAAW,CAAC,MAAM,EAAE;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,WAAW;IA8Df;;OAEG;IACH,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,WAAW,EAAE;IAQ9D;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAI7C;;OAEG;IACH,QAAQ,IAAI,UAAU;IAqDtB;;OAEG;IACH,WAAW,IAAI;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,KAAK,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;YAAC,cAAc,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACjG,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACjG;IAoFD;;OAEG;IACH,OAAO,CAAC,YAAY;IAkGpB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,OAAO,CAAC,cAAc;IAmCtB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAiB5B;;OAEG;IACH,OAAO,CAAC,eAAe;IAsBvB;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAmBhC;;OAEG;IACH,OAAO,CAAC,WAAW;IA+BnB;;OAEG;IACH,OAAO,CAAC,WAAW;IAenB;;OAEG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,OAAO,CAAC,GAAG;CA4BZ;AAKD,wBAAgB,gBAAgB,CAC9B,aAAa,CAAC,EAAE,kBAAkB,GACjC,oBAAoB,CAKtB"}
@@ -0,0 +1,492 @@
1
+ /**
2
+ * Crash Insights Service
3
+ *
4
+ * Captures and analyzes agent crashes to provide actionable insights:
5
+ * - Memory state at crash time
6
+ * - Crash history and patterns
7
+ * - Root cause analysis
8
+ * - Recommendations for prevention
9
+ */
10
+ import { EventEmitter } from 'events';
11
+ import * as fs from 'fs';
12
+ import * as path from 'path';
13
+ import * as os from 'os';
14
+ import { formatBytes, } from './memory-monitor.js';
15
+ export class CrashInsightsService extends EventEmitter {
16
+ crashes = [];
17
+ memoryMonitor = null;
18
+ persistPath;
19
+ maxCrashHistory = 1000;
20
+ constructor(memoryMonitor) {
21
+ super();
22
+ this.memoryMonitor = memoryMonitor || null;
23
+ // Set up persistence path
24
+ const dataDir = process.env.AGENT_RELAY_DATA_DIR ||
25
+ path.join(os.homedir(), '.local', 'share', 'agent-relay');
26
+ this.persistPath = path.join(dataDir, 'crash-insights.json');
27
+ // Load existing crash history
28
+ this.loadCrashes();
29
+ }
30
+ /**
31
+ * Set the memory monitor instance
32
+ */
33
+ setMemoryMonitor(monitor) {
34
+ this.memoryMonitor = monitor;
35
+ }
36
+ /**
37
+ * Record a crash event
38
+ */
39
+ recordCrash(params) {
40
+ const id = `crash-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
41
+ const crashTime = new Date();
42
+ // Get memory context from memory monitor
43
+ const memoryContext = this.memoryMonitor
44
+ ? this.memoryMonitor.getCrashContext(params.agentName)
45
+ : this.createEmptyMemoryContext(params.agentName, params.pid, crashTime);
46
+ // Analyze the crash
47
+ const analysis = this.analyzeCrash({
48
+ ...params,
49
+ memoryContext,
50
+ });
51
+ const record = {
52
+ id,
53
+ agentName: params.agentName,
54
+ pid: params.pid,
55
+ crashTime,
56
+ exitCode: params.exitCode,
57
+ signal: params.signal,
58
+ reason: params.reason,
59
+ memoryContext,
60
+ stackTrace: params.stackTrace,
61
+ lastOutput: params.lastOutput?.slice(-2000), // Keep last 2KB
62
+ environment: {
63
+ nodeVersion: process.version,
64
+ platform: process.platform,
65
+ arch: process.arch,
66
+ systemMemory: {
67
+ total: os.totalmem(),
68
+ free: os.freemem(),
69
+ },
70
+ uptime: process.uptime(),
71
+ },
72
+ analysis,
73
+ };
74
+ // Add to history
75
+ this.crashes.unshift(record);
76
+ // Trim history
77
+ if (this.crashes.length > this.maxCrashHistory) {
78
+ this.crashes = this.crashes.slice(0, this.maxCrashHistory);
79
+ }
80
+ // Persist
81
+ this.saveCrashes();
82
+ // Emit event
83
+ this.emit('crash', record);
84
+ this.log('error', `Crash recorded for ${params.agentName}`, {
85
+ id,
86
+ cause: analysis.likelyCause,
87
+ confidence: analysis.confidence,
88
+ });
89
+ return record;
90
+ }
91
+ /**
92
+ * Get crash history for an agent
93
+ */
94
+ getCrashHistory(agentName, limit = 50) {
95
+ let history = this.crashes;
96
+ if (agentName) {
97
+ history = history.filter((c) => c.agentName === agentName);
98
+ }
99
+ return history.slice(0, limit);
100
+ }
101
+ /**
102
+ * Get a specific crash record
103
+ */
104
+ getCrash(id) {
105
+ return this.crashes.find((c) => c.id === id);
106
+ }
107
+ /**
108
+ * Get crash statistics
109
+ */
110
+ getStats() {
111
+ const crashesByAgent = {};
112
+ const crashesByCause = {};
113
+ const agentCrashTimes = {};
114
+ for (const crash of this.crashes) {
115
+ crashesByAgent[crash.agentName] = (crashesByAgent[crash.agentName] || 0) + 1;
116
+ crashesByCause[crash.analysis.likelyCause] =
117
+ (crashesByCause[crash.analysis.likelyCause] || 0) + 1;
118
+ if (!agentCrashTimes[crash.agentName]) {
119
+ agentCrashTimes[crash.agentName] = [];
120
+ }
121
+ agentCrashTimes[crash.agentName].push(crash.crashTime.getTime());
122
+ }
123
+ // Find most crash-prone agent
124
+ let mostCrashProne = null;
125
+ for (const [agent, count] of Object.entries(crashesByAgent)) {
126
+ if (!mostCrashProne || count > mostCrashProne.count) {
127
+ mostCrashProne = { agent, count };
128
+ }
129
+ }
130
+ // Calculate average time between crashes
131
+ let totalIntervals = 0;
132
+ let intervalCount = 0;
133
+ for (const times of Object.values(agentCrashTimes)) {
134
+ if (times.length > 1) {
135
+ const sorted = times.sort((a, b) => a - b);
136
+ for (let i = 1; i < sorted.length; i++) {
137
+ totalIntervals += sorted[i] - sorted[i - 1];
138
+ intervalCount++;
139
+ }
140
+ }
141
+ }
142
+ const avgTimeBetweenCrashes = intervalCount > 0 ? totalIntervals / intervalCount : 0;
143
+ // Detect patterns
144
+ const patterns = this.detectPatterns();
145
+ return {
146
+ totalCrashes: this.crashes.length,
147
+ crashesByAgent,
148
+ crashesByCause,
149
+ avgTimeBetweenCrashes,
150
+ mostCrashProne,
151
+ recentCrashes: this.crashes.slice(0, 10),
152
+ patterns,
153
+ };
154
+ }
155
+ /**
156
+ * Get insights and recommendations
157
+ */
158
+ getInsights() {
159
+ const stats = this.getStats();
160
+ const issues = [];
161
+ const trends = [];
162
+ // Analyze OOM crashes
163
+ const oomCrashes = stats.crashesByCause['oom'] || 0;
164
+ if (oomCrashes > 0) {
165
+ issues.push({
166
+ issue: `${oomCrashes} crash${oomCrashes > 1 ? 'es' : ''} caused by out of memory`,
167
+ severity: 'high',
168
+ recommendation: 'Increase memory limits or optimize agent memory usage',
169
+ });
170
+ }
171
+ // Analyze memory leaks
172
+ const leakCrashes = stats.crashesByCause['memory_leak'] || 0;
173
+ if (leakCrashes > 0) {
174
+ issues.push({
175
+ issue: `${leakCrashes} crash${leakCrashes > 1 ? 'es' : ''} likely caused by memory leaks`,
176
+ severity: 'high',
177
+ recommendation: 'Investigate agent code for memory leaks, consider periodic restarts',
178
+ });
179
+ }
180
+ // Check crash frequency
181
+ const recentCrashes = this.crashes.filter((c) => Date.now() - c.crashTime.getTime() < 24 * 60 * 60 * 1000).length;
182
+ if (recentCrashes > 5) {
183
+ issues.push({
184
+ issue: `${recentCrashes} crashes in the last 24 hours`,
185
+ severity: recentCrashes > 10 ? 'high' : 'medium',
186
+ recommendation: 'Investigate root cause, consider rolling back recent changes',
187
+ });
188
+ }
189
+ // Check repeat offenders
190
+ if (stats.mostCrashProne && stats.mostCrashProne.count > 5) {
191
+ issues.push({
192
+ issue: `Agent "${stats.mostCrashProne.agent}" has crashed ${stats.mostCrashProne.count} times`,
193
+ severity: 'medium',
194
+ recommendation: 'Investigate why this agent is unstable',
195
+ });
196
+ }
197
+ // Calculate health score (0-100)
198
+ let healthScore = 100;
199
+ healthScore -= oomCrashes * 10;
200
+ healthScore -= leakCrashes * 8;
201
+ healthScore -= recentCrashes * 3;
202
+ healthScore = Math.max(0, Math.min(100, healthScore));
203
+ // Analyze trends
204
+ const last24h = this.crashes.filter((c) => Date.now() - c.crashTime.getTime() < 24 * 60 * 60 * 1000).length;
205
+ const prev24h = this.crashes.filter((c) => Date.now() - c.crashTime.getTime() >= 24 * 60 * 60 * 1000 &&
206
+ Date.now() - c.crashTime.getTime() < 48 * 60 * 60 * 1000).length;
207
+ let crashTrend = 'stable';
208
+ if (last24h < prev24h * 0.7)
209
+ crashTrend = 'improving';
210
+ else if (last24h > prev24h * 1.3)
211
+ crashTrend = 'degrading';
212
+ trends.push({
213
+ metric: 'Crash frequency',
214
+ trend: crashTrend,
215
+ details: `${last24h} crashes in last 24h vs ${prev24h} in previous 24h`,
216
+ });
217
+ return {
218
+ summary: this.generateSummary(stats),
219
+ topIssues: issues.sort((a, b) => {
220
+ const severityOrder = { high: 0, medium: 1, low: 2 };
221
+ return severityOrder[a.severity] - severityOrder[b.severity];
222
+ }),
223
+ healthScore,
224
+ trends,
225
+ };
226
+ }
227
+ /**
228
+ * Analyze a crash and determine likely cause
229
+ */
230
+ analyzeCrash(params) {
231
+ const details = [];
232
+ const recommendations = [];
233
+ let likelyCause = 'unknown';
234
+ let confidence = 'low';
235
+ // Check memory-based causes first
236
+ if (params.memoryContext.likelyCause !== 'unknown') {
237
+ likelyCause = params.memoryContext.likelyCause;
238
+ confidence = 'high';
239
+ details.push(...params.memoryContext.analysisNotes);
240
+ }
241
+ // Check signal
242
+ if (params.signal) {
243
+ details.push(`Process received signal: ${params.signal}`);
244
+ if (params.signal === 'SIGKILL') {
245
+ if (likelyCause === 'unknown') {
246
+ likelyCause = 'oom';
247
+ confidence = 'medium';
248
+ }
249
+ details.push('SIGKILL often indicates OOM killer intervention');
250
+ recommendations.push('Check system logs for OOM killer activity');
251
+ }
252
+ else if (params.signal === 'SIGSEGV') {
253
+ likelyCause = 'error';
254
+ confidence = 'high';
255
+ details.push('Segmentation fault - memory access violation');
256
+ recommendations.push('Check for native module issues or memory corruption');
257
+ }
258
+ }
259
+ // Check exit code
260
+ if (params.exitCode !== null) {
261
+ details.push(`Exit code: ${params.exitCode}`);
262
+ if (params.exitCode === 137) {
263
+ // 128 + 9 (SIGKILL)
264
+ if (likelyCause === 'unknown') {
265
+ likelyCause = 'oom';
266
+ confidence = 'high';
267
+ }
268
+ details.push('Exit code 137 typically indicates OOM kill');
269
+ }
270
+ }
271
+ // Check stack trace for clues
272
+ if (params.stackTrace) {
273
+ if (params.stackTrace.includes('FATAL ERROR: CALL_AND_RETRY_LAST')) {
274
+ likelyCause = 'oom';
275
+ confidence = 'high';
276
+ details.push('V8 heap allocation failure detected');
277
+ recommendations.push('Increase Node.js memory limit with --max-old-space-size');
278
+ }
279
+ if (params.stackTrace.includes('RangeError: Invalid array length')) {
280
+ likelyCause = 'memory_leak';
281
+ confidence = 'medium';
282
+ details.push('Array grew too large - possible unbounded growth');
283
+ recommendations.push('Review array handling code for unbounded growth');
284
+ }
285
+ }
286
+ // Add memory-specific recommendations
287
+ if (likelyCause === 'oom' || likelyCause === 'memory_leak') {
288
+ recommendations.push('Review agent memory usage patterns');
289
+ recommendations.push('Consider implementing memory limits or checkpoints');
290
+ if (params.memoryContext.peakMemory > 1024 * 1024 * 1024) {
291
+ recommendations.push(`Peak memory was ${formatBytes(params.memoryContext.peakMemory)} - consider memory profiling`);
292
+ }
293
+ }
294
+ // Find related crashes
295
+ const relatedCrashes = this.findRelatedCrashes(params.agentName, likelyCause);
296
+ // Generate summary
297
+ const summary = this.generateCrashSummary(likelyCause, confidence, params);
298
+ return {
299
+ likelyCause,
300
+ confidence,
301
+ summary,
302
+ details,
303
+ recommendations: recommendations.length > 0
304
+ ? recommendations
305
+ : ['Monitor agent for recurrence', 'Check logs for additional context'],
306
+ relatedCrashes,
307
+ };
308
+ }
309
+ /**
310
+ * Find related crashes
311
+ */
312
+ findRelatedCrashes(agentName, cause) {
313
+ return this.crashes
314
+ .filter((c) => (c.agentName === agentName || c.analysis.likelyCause === cause) &&
315
+ Date.now() - c.crashTime.getTime() < 7 * 24 * 60 * 60 * 1000 // Last 7 days
316
+ )
317
+ .slice(0, 5)
318
+ .map((c) => c.id);
319
+ }
320
+ /**
321
+ * Detect crash patterns
322
+ */
323
+ detectPatterns() {
324
+ const patterns = [];
325
+ const causeGroups = {};
326
+ // Group by cause
327
+ for (const crash of this.crashes) {
328
+ const cause = crash.analysis.likelyCause;
329
+ if (!causeGroups[cause]) {
330
+ causeGroups[cause] = [];
331
+ }
332
+ causeGroups[cause].push(crash);
333
+ }
334
+ // Create patterns for significant groups
335
+ for (const [cause, crashes] of Object.entries(causeGroups)) {
336
+ if (crashes.length >= 3) {
337
+ const agents = [...new Set(crashes.map((c) => c.agentName))];
338
+ const avgMemory = crashes.reduce((sum, c) => sum + (c.memoryContext.peakMemory || 0), 0) /
339
+ crashes.length;
340
+ patterns.push({
341
+ pattern: `${cause}_pattern`,
342
+ occurrences: crashes.length,
343
+ lastSeen: crashes[0].crashTime,
344
+ affectedAgents: agents,
345
+ avgMemoryAtCrash: avgMemory,
346
+ commonCause: cause,
347
+ });
348
+ }
349
+ }
350
+ return patterns;
351
+ }
352
+ /**
353
+ * Generate crash summary
354
+ */
355
+ generateCrashSummary(cause, confidence, params) {
356
+ const causeDescriptions = {
357
+ oom: 'ran out of memory',
358
+ memory_leak: 'experienced a memory leak',
359
+ sudden_spike: 'had a sudden memory spike',
360
+ signal: 'was terminated by a signal',
361
+ error: 'encountered an error',
362
+ unknown: 'crashed for unknown reasons',
363
+ };
364
+ return `Agent "${params.agentName}" ${causeDescriptions[cause] || 'crashed'} (${confidence} confidence). ${params.reason}`;
365
+ }
366
+ /**
367
+ * Generate overall summary
368
+ */
369
+ generateSummary(stats) {
370
+ if (stats.totalCrashes === 0) {
371
+ return 'No crashes recorded. System is stable.';
372
+ }
373
+ const parts = [];
374
+ parts.push(`${stats.totalCrashes} total crash${stats.totalCrashes > 1 ? 'es' : ''} recorded.`);
375
+ if (stats.mostCrashProne) {
376
+ parts.push(`Most unstable: "${stats.mostCrashProne.agent}" (${stats.mostCrashProne.count} crashes).`);
377
+ }
378
+ const topCause = Object.entries(stats.crashesByCause).sort((a, b) => b[1] - a[1])[0];
379
+ if (topCause) {
380
+ parts.push(`Primary cause: ${topCause[0]} (${topCause[1]} occurrences).`);
381
+ }
382
+ return parts.join(' ');
383
+ }
384
+ /**
385
+ * Create empty memory context when no monitor available
386
+ */
387
+ createEmptyMemoryContext(agentName, pid, crashTime) {
388
+ return {
389
+ agentName,
390
+ pid,
391
+ crashTime,
392
+ lastKnownMemory: null,
393
+ peakMemory: 0,
394
+ averageMemory: 0,
395
+ memoryTrend: 'unknown',
396
+ recentHistory: [],
397
+ likelyCause: 'unknown',
398
+ analysisNotes: ['Memory monitoring was not enabled'],
399
+ };
400
+ }
401
+ /**
402
+ * Load crashes from disk
403
+ */
404
+ loadCrashes() {
405
+ try {
406
+ if (fs.existsSync(this.persistPath)) {
407
+ const data = fs.readFileSync(this.persistPath, 'utf-8');
408
+ const parsed = JSON.parse(data);
409
+ this.crashes = parsed.crashes.map((c) => ({
410
+ ...c,
411
+ crashTime: new Date(c.crashTime),
412
+ memoryContext: {
413
+ ...c.memoryContext,
414
+ crashTime: new Date(c.memoryContext.crashTime),
415
+ lastKnownMemory: c.memoryContext.lastKnownMemory
416
+ ? {
417
+ ...c.memoryContext.lastKnownMemory,
418
+ timestamp: new Date(c.memoryContext.lastKnownMemory.timestamp),
419
+ }
420
+ : null,
421
+ recentHistory: c.memoryContext.recentHistory.map((h) => ({
422
+ ...h,
423
+ timestamp: new Date(h.timestamp),
424
+ })),
425
+ },
426
+ }));
427
+ this.log('info', `Loaded ${this.crashes.length} crash records`);
428
+ }
429
+ }
430
+ catch (error) {
431
+ this.log('warn', 'Failed to load crash history', { error: String(error) });
432
+ this.crashes = [];
433
+ }
434
+ }
435
+ /**
436
+ * Save crashes to disk
437
+ */
438
+ saveCrashes() {
439
+ try {
440
+ const dir = path.dirname(this.persistPath);
441
+ if (!fs.existsSync(dir)) {
442
+ fs.mkdirSync(dir, { recursive: true });
443
+ }
444
+ fs.writeFileSync(this.persistPath, JSON.stringify({ crashes: this.crashes }, null, 2));
445
+ }
446
+ catch (error) {
447
+ this.log('error', 'Failed to save crash history', { error: String(error) });
448
+ }
449
+ }
450
+ /**
451
+ * Clear all crash history
452
+ */
453
+ clear() {
454
+ this.crashes = [];
455
+ this.saveCrashes();
456
+ this.emit('cleared');
457
+ }
458
+ /**
459
+ * Structured logging
460
+ */
461
+ log(level, message, context) {
462
+ const entry = {
463
+ timestamp: new Date().toISOString(),
464
+ level,
465
+ component: 'crash-insights',
466
+ message,
467
+ ...context,
468
+ };
469
+ this.emit('log', entry);
470
+ const prefix = `[crash-insights]`;
471
+ switch (level) {
472
+ case 'info':
473
+ console.log(prefix, message, context ? JSON.stringify(context) : '');
474
+ break;
475
+ case 'warn':
476
+ console.warn(prefix, message, context ? JSON.stringify(context) : '');
477
+ break;
478
+ case 'error':
479
+ console.error(prefix, message, context ? JSON.stringify(context) : '');
480
+ break;
481
+ }
482
+ }
483
+ }
484
+ // Singleton instance
485
+ let _crashInsights = null;
486
+ export function getCrashInsights(memoryMonitor) {
487
+ if (!_crashInsights) {
488
+ _crashInsights = new CrashInsightsService(memoryMonitor);
489
+ }
490
+ return _crashInsights;
491
+ }
492
+ //# sourceMappingURL=crash-insights.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"crash-insights.js","sourceRoot":"","sources":["../src/crash-insights.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAGL,WAAW,GACZ,MAAM,qBAAqB,CAAC;AAmD7B,MAAM,OAAO,oBAAqB,SAAQ,YAAY;IAC5C,OAAO,GAAkB,EAAE,CAAC;IAC5B,aAAa,GAA8B,IAAI,CAAC;IAChD,WAAW,CAAS;IACpB,eAAe,GAAG,IAAI,CAAC;IAE/B,YAAY,aAAkC;QAC5C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,IAAI,CAAC;QAE3C,0BAA0B;QAC1B,MAAM,OAAO,GACX,OAAO,CAAC,GAAG,CAAC,oBAAoB;YAChC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;QAE7D,8BAA8B;QAC9B,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,OAA2B;QAC1C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,MAQX;QACC,MAAM,EAAE,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC5E,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,yCAAyC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC;YACtD,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE3E,oBAAoB;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;YACjC,GAAG,MAAM;YACT,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,MAAM,GAAgB;YAC1B,EAAE;YACF,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS;YACT,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa;YACb,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,gBAAgB;YAC7D,WAAW,EAAE;gBACX,WAAW,EAAE,OAAO,CAAC,OAAO;gBAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,YAAY,EAAE;oBACZ,KAAK,EAAE,EAAE,CAAC,QAAQ,EAAE;oBACpB,IAAI,EAAE,EAAE,CAAC,OAAO,EAAE;iBACnB;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;aACzB;YACD,QAAQ;SACT,CAAC;QAEF,iBAAiB;QACjB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE7B,eAAe;QACf,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YAC/C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,CAAC;QAED,UAAU;QACV,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,aAAa;QACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAE3B,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,sBAAsB,MAAM,CAAC,SAAS,EAAE,EAAE;YAC1D,EAAE;YACF,KAAK,EAAE,QAAQ,CAAC,WAAW;YAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;SAChC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,SAAkB,EAAE,KAAK,GAAG,EAAE;QAC5C,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3B,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,EAAU;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,MAAM,eAAe,GAA6B,EAAE,CAAC;QAErD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC7E,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACxC,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAExD,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;gBACtC,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;YACxC,CAAC;YACD,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,8BAA8B;QAC9B,IAAI,cAAc,GAA4C,IAAI,CAAC;QACnE,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAC5D,IAAI,CAAC,cAAc,IAAI,KAAK,GAAG,cAAc,CAAC,KAAK,EAAE,CAAC;gBACpD,cAAc,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACpC,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,cAAc,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC5C,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,qBAAqB,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QAErF,kBAAkB;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEvC,OAAO;YACL,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YACjC,cAAc;YACd,cAAc;YACd,qBAAqB;YACrB,cAAc;YACd,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACxC,QAAQ;SACT,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,WAAW;QAMT,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAA0F,EAAE,CAAC;QACzG,MAAM,MAAM,GAA4F,EAAE,CAAC;QAE3G,sBAAsB;QACtB,MAAM,UAAU,GAAG,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,GAAG,UAAU,SAAS,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,0BAA0B;gBACjF,QAAQ,EAAE,MAAM;gBAChB,cAAc,EAAE,uDAAuD;aACxE,CAAC,CAAC;QACL,CAAC;QAED,uBAAuB;QACvB,MAAM,WAAW,GAAG,KAAK,CAAC,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,GAAG,WAAW,SAAS,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,gCAAgC;gBACzF,QAAQ,EAAE,MAAM;gBAChB,cAAc,EAAE,qEAAqE;aACtF,CAAC,CAAC;QACL,CAAC;QAED,wBAAwB;QACxB,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAChE,CAAC,MAAM,CAAC;QACT,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,GAAG,aAAa,+BAA+B;gBACtD,QAAQ,EAAE,aAAa,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;gBAChD,cAAc,EAAE,8DAA8D;aAC/E,CAAC,CAAC;QACL,CAAC;QAED,yBAAyB;QACzB,IAAI,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,cAAc,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,UAAU,KAAK,CAAC,cAAc,CAAC,KAAK,iBAAiB,KAAK,CAAC,cAAc,CAAC,KAAK,QAAQ;gBAC9F,QAAQ,EAAE,QAAQ;gBAClB,cAAc,EAAE,wCAAwC;aACzD,CAAC,CAAC;QACL,CAAC;QAED,iCAAiC;QACjC,IAAI,WAAW,GAAG,GAAG,CAAC;QACtB,WAAW,IAAI,UAAU,GAAG,EAAE,CAAC;QAC/B,WAAW,IAAI,WAAW,GAAG,CAAC,CAAC;QAC/B,WAAW,IAAI,aAAa,GAAG,CAAC,CAAC;QACjC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAEtD,iBAAiB;QACjB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAChE,CAAC,MAAM,CAAC;QACT,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,EAAE,CACJ,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;YACzD,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAC3D,CAAC,MAAM,CAAC;QAET,IAAI,UAAU,GAAyC,QAAQ,CAAC;QAChE,IAAI,OAAO,GAAG,OAAO,GAAG,GAAG;YAAE,UAAU,GAAG,WAAW,CAAC;aACjD,IAAI,OAAO,GAAG,OAAO,GAAG,GAAG;YAAE,UAAU,GAAG,WAAW,CAAC;QAE3D,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,iBAAiB;YACzB,KAAK,EAAE,UAAU;YACjB,OAAO,EAAE,GAAG,OAAO,2BAA2B,OAAO,kBAAkB;SACxE,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YACpC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC9B,MAAM,aAAa,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;gBACrD,OAAO,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC/D,CAAC,CAAC;YACF,WAAW;YACX,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,MAQpB;QACC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,eAAe,GAAa,EAAE,CAAC;QACrC,IAAI,WAAW,GAAiC,SAAS,CAAC;QAC1D,IAAI,UAAU,GAAgC,KAAK,CAAC;QAEpD,kCAAkC;QAClC,IAAI,MAAM,CAAC,aAAa,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACnD,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC;YAC/C,UAAU,GAAG,MAAM,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QACtD,CAAC;QAED,eAAe;QACf,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1D,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;oBAC9B,WAAW,GAAG,KAAK,CAAC;oBACpB,UAAU,GAAG,QAAQ,CAAC;gBACxB,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;gBAChE,eAAe,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;YACpE,CAAC;iBAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvC,WAAW,GAAG,OAAO,CAAC;gBACtB,UAAU,GAAG,MAAM,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;gBAC7D,eAAe,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9C,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,EAAE,CAAC;gBAC5B,oBAAoB;gBACpB,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;oBAC9B,WAAW,GAAG,KAAK,CAAC;oBACpB,UAAU,GAAG,MAAM,CAAC;gBACtB,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE,CAAC;gBACnE,WAAW,GAAG,KAAK,CAAC;gBACpB,UAAU,GAAG,MAAM,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;gBACpD,eAAe,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;YAClF,CAAC;YACD,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE,CAAC;gBACnE,WAAW,GAAG,aAAa,CAAC;gBAC5B,UAAU,GAAG,QAAQ,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;gBACjE,eAAe,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,WAAW,KAAK,KAAK,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;YAC3D,eAAe,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAC3D,eAAe,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YAC3E,IAAI,MAAM,CAAC,aAAa,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;gBACzD,eAAe,CAAC,IAAI,CAClB,mBAAmB,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,8BAA8B,CAC9F,CAAC;YACJ,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAE9E,mBAAmB;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QAE3E,OAAO;YACL,WAAW;YACX,UAAU;YACV,OAAO;YACP,OAAO;YACP,eAAe,EACb,eAAe,CAAC,MAAM,GAAG,CAAC;gBACxB,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,CAAC,8BAA8B,EAAE,mCAAmC,CAAC;YAC3E,cAAc;SACf,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAiB,EAAE,KAAa;QACzD,OAAO,IAAI,CAAC,OAAO;aAChB,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,KAAK,KAAK,CAAC;YAC/D,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc;SAC9E;aACA,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,MAAM,WAAW,GAAkC,EAAE,CAAC;QAEtD,iBAAiB;QACjB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YACzC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC1B,CAAC;YACD,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAED,yCAAyC;QACzC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3D,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC7D,MAAM,SAAS,GACb,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBACtE,OAAO,CAAC,MAAM,CAAC;gBAEjB,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,GAAG,KAAK,UAAU;oBAC3B,WAAW,EAAE,OAAO,CAAC,MAAM;oBAC3B,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;oBAC9B,cAAc,EAAE,MAAM;oBACtB,gBAAgB,EAAE,SAAS;oBAC3B,WAAW,EAAE,KAAK;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAC1B,KAAa,EACb,UAAkB,EAClB,MAA6C;QAE7C,MAAM,iBAAiB,GAA2B;YAChD,GAAG,EAAE,mBAAmB;YACxB,WAAW,EAAE,2BAA2B;YACxC,YAAY,EAAE,2BAA2B;YACzC,MAAM,EAAE,4BAA4B;YACpC,KAAK,EAAE,sBAAsB;YAC7B,OAAO,EAAE,6BAA6B;SACvC,CAAC;QAEF,OAAO,UAAU,MAAM,CAAC,SAAS,KAAK,iBAAiB,CAAC,KAAK,CAAC,IAAI,SAAS,KAAK,UAAU,iBAAiB,MAAM,CAAC,MAAM,EAAE,CAAC;IAC7H,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAiB;QACvC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,wCAAwC,CAAC;QAClD,CAAC;QAED,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,YAAY,eAAe,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAE/F,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CACR,mBAAmB,KAAK,CAAC,cAAc,CAAC,KAAK,MAAM,KAAK,CAAC,cAAc,CAAC,KAAK,YAAY,CAC1F,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,kBAAkB,QAAQ,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,wBAAwB,CAC9B,SAAiB,EACjB,GAAW,EACX,SAAe;QAEf,OAAO;YACL,SAAS;YACT,GAAG;YACH,SAAS;YACT,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,CAAC;YACb,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,SAAS;YACtB,aAAa,EAAE,EAAE;YACjB,WAAW,EAAE,SAAS;YACtB,aAAa,EAAE,CAAC,mCAAmC,CAAC;SACrD,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,WAAW;QACjB,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBACpC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;gBACxD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;oBAC7C,GAAG,CAAC;oBACJ,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;oBAChC,aAAa,EAAE;wBACb,GAAG,CAAC,CAAC,aAAa;wBAClB,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC;wBAC9C,eAAe,EAAE,CAAC,CAAC,aAAa,CAAC,eAAe;4BAC9C,CAAC,CAAC;gCACE,GAAG,CAAC,CAAC,aAAa,CAAC,eAAe;gCAClC,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,eAAe,CAAC,SAAS,CAAC;6BAC/D;4BACH,CAAC,CAAC,IAAI;wBACR,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;4BAC5D,GAAG,CAAC;4BACJ,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;yBACjC,CAAC,CAAC;qBACJ;iBACF,CAAC,CAAC,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,gBAAgB,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,8BAA8B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW;QACjB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC3C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YACD,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACnD,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,8BAA8B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,GAAG,CACT,KAAgC,EAChC,OAAe,EACf,OAAiC;QAEjC,MAAM,KAAK,GAAG;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK;YACL,SAAS,EAAE,gBAAgB;YAC3B,OAAO;YACP,GAAG,OAAO;SACX,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,kBAAkB,CAAC;QAClC,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,MAAM;gBACT,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrE,MAAM;YACR,KAAK,MAAM;gBACT,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACtE,MAAM;YACR,KAAK,OAAO;gBACV,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;QACV,CAAC;IACH,CAAC;CACF;AAED,qBAAqB;AACrB,IAAI,cAAc,GAAgC,IAAI,CAAC;AAEvD,MAAM,UAAU,gBAAgB,CAC9B,aAAkC;IAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,cAAc,GAAG,IAAI,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC"}