@push.rocks/smartproxy 19.6.13 → 19.6.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist_ts/core/utils/log-deduplicator.d.ts +36 -0
  2. package/dist_ts/core/utils/log-deduplicator.js +224 -0
  3. package/dist_ts/core/utils/shared-security-manager.d.ts +2 -1
  4. package/dist_ts/core/utils/shared-security-manager.js +22 -2
  5. package/dist_ts/proxies/http-proxy/http-proxy.d.ts +1 -0
  6. package/dist_ts/proxies/http-proxy/http-proxy.js +94 -9
  7. package/dist_ts/proxies/http-proxy/models/types.d.ts +2 -0
  8. package/dist_ts/proxies/http-proxy/models/types.js +1 -1
  9. package/dist_ts/proxies/http-proxy/security-manager.d.ts +42 -1
  10. package/dist_ts/proxies/http-proxy/security-manager.js +121 -2
  11. package/dist_ts/proxies/smart-proxy/connection-manager.d.ts +14 -0
  12. package/dist_ts/proxies/smart-proxy/connection-manager.js +74 -26
  13. package/dist_ts/proxies/smart-proxy/http-proxy-bridge.js +5 -1
  14. package/dist_ts/proxies/smart-proxy/models/interfaces.d.ts +1 -0
  15. package/dist_ts/proxies/smart-proxy/route-connection-handler.js +24 -8
  16. package/dist_ts/proxies/smart-proxy/security-manager.d.ts +9 -0
  17. package/dist_ts/proxies/smart-proxy/security-manager.js +63 -1
  18. package/dist_ts/proxies/smart-proxy/smart-proxy.js +4 -1
  19. package/package.json +1 -1
  20. package/readme.hints.md +80 -1
  21. package/readme.plan.md +34 -353
  22. package/ts/core/utils/log-deduplicator.ts +280 -0
  23. package/ts/core/utils/shared-security-manager.ts +24 -1
  24. package/ts/proxies/http-proxy/http-proxy.ts +129 -9
  25. package/ts/proxies/http-proxy/models/types.ts +4 -0
  26. package/ts/proxies/http-proxy/security-manager.ts +136 -1
  27. package/ts/proxies/smart-proxy/connection-manager.ts +93 -27
  28. package/ts/proxies/smart-proxy/http-proxy-bridge.ts +5 -0
  29. package/ts/proxies/smart-proxy/models/interfaces.ts +1 -0
  30. package/ts/proxies/smart-proxy/route-connection-handler.ts +45 -14
  31. package/ts/proxies/smart-proxy/security-manager.ts +76 -1
  32. package/ts/proxies/smart-proxy/smart-proxy.ts +4 -0
package/readme.plan.md CHANGED
@@ -1,364 +1,45 @@
1
- # SmartProxy Metrics Improvement Plan
1
+ # SmartProxy Connection Limiting Improvements Plan
2
2
 
3
- ## Overview
3
+ Command to re-read CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`
4
4
 
5
- The current `getThroughputRate()` implementation calculates cumulative throughput over a 60-second window rather than providing an actual rate, making metrics misleading for monitoring systems. This plan outlines a comprehensive redesign of the metrics system to provide accurate, time-series based metrics suitable for production monitoring.
5
+ ## Issues Identified
6
6
 
7
- ## 1. Core Issues with Current Implementation
7
+ 1. **HttpProxy Bypass**: Connections forwarded to HttpProxy for TLS termination only check global limits, not per-IP limits
8
+ 2. **Missing Route-Level Connection Enforcement**: Routes can define `security.maxConnections` but it's never enforced
9
+ 3. **Cleanup Queue Race Condition**: New connections can be added to cleanup queue while processing
10
+ 4. **IP Tracking Memory Optimization**: IP entries remain in map even without active connections
8
11
 
9
- - **Cumulative vs Rate**: Current method accumulates all bytes from connections in the last minute rather than calculating actual throughput rate
10
- - **No Time-Series Data**: Cannot track throughput changes over time
11
- - **Inaccurate Estimates**: Attempting to estimate rates for older connections is fundamentally flawed
12
- - **No Sliding Windows**: Cannot provide different time window views (1s, 10s, 60s, etc.)
13
- - **Limited Granularity**: Only provides a single 60-second view
12
+ ## Implementation Steps
14
13
 
15
- ## 2. Proposed Architecture
14
+ ### 1. Fix HttpProxy Per-IP Validation ✓
15
+ - [x] Pass IP information to HttpProxy when forwarding connections
16
+ - [x] Add per-IP validation in HttpProxy connection handler
17
+ - [x] Ensure connection tracking is consistent between SmartProxy and HttpProxy
16
18
 
17
- ### A. Time-Series Throughput Tracking
19
+ ### 2. Implement Route-Level Connection Limits ✓
20
+ - [x] Add connection count tracking per route in ConnectionManager
21
+ - [x] Update SharedSecurityManager.isAllowed() to check route-specific maxConnections
22
+ - [x] Add route connection limit validation in route-connection-handler.ts
18
23
 
19
- ```typescript
20
- interface IThroughputSample {
21
- timestamp: number;
22
- bytesIn: number;
23
- bytesOut: number;
24
- }
24
+ ### 3. Fix Cleanup Queue Race Condition ✓
25
+ - [x] Implement proper queue snapshotting before processing
26
+ - [x] Ensure new connections added during processing aren't missed
27
+ - [x] Add proper synchronization for cleanup operations
25
28
 
26
- class ThroughputTracker {
27
- private samples: IThroughputSample[] = [];
28
- private readonly MAX_SAMPLES = 3600; // 1 hour at 1 sample/second
29
- private lastSampleTime: number = 0;
30
- private accumulatedBytesIn: number = 0;
31
- private accumulatedBytesOut: number = 0;
32
-
33
- // Called on every data transfer
34
- public recordBytes(bytesIn: number, bytesOut: number): void {
35
- this.accumulatedBytesIn += bytesIn;
36
- this.accumulatedBytesOut += bytesOut;
37
- }
38
-
39
- // Called periodically (every second)
40
- public takeSample(): void {
41
- const now = Date.now();
42
-
43
- // Record accumulated bytes since last sample
44
- this.samples.push({
45
- timestamp: now,
46
- bytesIn: this.accumulatedBytesIn,
47
- bytesOut: this.accumulatedBytesOut
48
- });
49
-
50
- // Reset accumulators
51
- this.accumulatedBytesIn = 0;
52
- this.accumulatedBytesOut = 0;
53
-
54
- // Trim old samples
55
- const cutoff = now - 3600000; // 1 hour
56
- this.samples = this.samples.filter(s => s.timestamp > cutoff);
57
- }
58
-
59
- // Get rate over specified window
60
- public getRate(windowSeconds: number): { bytesInPerSec: number; bytesOutPerSec: number } {
61
- const now = Date.now();
62
- const windowStart = now - (windowSeconds * 1000);
63
-
64
- const relevantSamples = this.samples.filter(s => s.timestamp > windowStart);
65
-
66
- if (relevantSamples.length === 0) {
67
- return { bytesInPerSec: 0, bytesOutPerSec: 0 };
68
- }
69
-
70
- const totalBytesIn = relevantSamples.reduce((sum, s) => sum + s.bytesIn, 0);
71
- const totalBytesOut = relevantSamples.reduce((sum, s) => sum + s.bytesOut, 0);
72
-
73
- const actualWindow = (now - relevantSamples[0].timestamp) / 1000;
74
-
75
- return {
76
- bytesInPerSec: Math.round(totalBytesIn / actualWindow),
77
- bytesOutPerSec: Math.round(totalBytesOut / actualWindow)
78
- };
79
- }
80
- }
81
- ```
29
+ ### 4. Optimize IP Tracking Memory Usage ✓
30
+ - [x] Add periodic cleanup for IPs with no active connections
31
+ - [x] Implement expiry for rate limit timestamps
32
+ - [x] Add memory-efficient data structures for IP tracking
82
33
 
83
- ### B. Connection-Level Byte Tracking
34
+ ### 5. Add Comprehensive Tests ✓
35
+ - [x] Test per-IP limits with HttpProxy forwarding
36
+ - [x] Test route-level connection limits
37
+ - [x] Test cleanup queue edge cases
38
+ - [x] Test memory usage with many unique IPs
84
39
 
85
- ```typescript
86
- // In ConnectionRecord, add:
87
- interface IConnectionRecord {
88
- // ... existing fields ...
89
-
90
- // Byte counters with timestamps
91
- bytesReceivedHistory: Array<{ timestamp: number; bytes: number }>;
92
- bytesSentHistory: Array<{ timestamp: number; bytes: number }>;
93
-
94
- // For efficiency, could use circular buffer
95
- lastBytesReceivedUpdate: number;
96
- lastBytesSentUpdate: number;
97
- }
98
- ```
40
+ ## Notes
99
41
 
100
- ### C. Enhanced Metrics Interface
101
-
102
- ```typescript
103
- interface IMetrics {
104
- // Connection metrics
105
- connections: {
106
- active(): number;
107
- total(): number;
108
- byRoute(): Map<string, number>;
109
- byIP(): Map<string, number>;
110
- topIPs(limit?: number): Array<{ ip: string; count: number }>;
111
- };
112
-
113
- // Throughput metrics (bytes per second)
114
- throughput: {
115
- instant(): { in: number; out: number }; // Last 1 second
116
- recent(): { in: number; out: number }; // Last 10 seconds
117
- average(): { in: number; out: number }; // Last 60 seconds
118
- custom(seconds: number): { in: number; out: number };
119
- history(seconds: number): Array<{ timestamp: number; in: number; out: number }>;
120
- byRoute(windowSeconds?: number): Map<string, { in: number; out: number }>;
121
- byIP(windowSeconds?: number): Map<string, { in: number; out: number }>;
122
- };
123
-
124
- // Request metrics
125
- requests: {
126
- perSecond(): number;
127
- perMinute(): number;
128
- total(): number;
129
- };
130
-
131
- // Cumulative totals
132
- totals: {
133
- bytesIn(): number;
134
- bytesOut(): number;
135
- connections(): number;
136
- };
137
-
138
- // Performance metrics
139
- percentiles: {
140
- connectionDuration(): { p50: number; p95: number; p99: number };
141
- bytesTransferred(): {
142
- in: { p50: number; p95: number; p99: number };
143
- out: { p50: number; p95: number; p99: number };
144
- };
145
- };
146
- }
147
- ```
148
-
149
- ## 3. Implementation Plan
150
-
151
- ### Current Status
152
- - **Phase 1**: ~90% complete (core functionality implemented, tests need fixing)
153
- - **Phase 2**: ~60% complete (main features done, percentiles pending)
154
- - **Phase 3**: ~40% complete (basic optimizations in place)
155
- - **Phase 4**: 0% complete (export formats not started)
156
-
157
- ### Phase 1: Core Throughput Tracking (Week 1)
158
- - [x] Implement `ThroughputTracker` class
159
- - [x] Integrate byte recording into socket data handlers
160
- - [x] Add periodic sampling (1-second intervals)
161
- - [x] Update `getThroughputRate()` to use time-series data (replaced with new clean API)
162
- - [ ] Add unit tests for throughput tracking
163
-
164
- ### Phase 2: Enhanced Metrics (Week 2)
165
- - [x] Add configurable time windows (1s, 10s, 60s, 5m, etc.)
166
- - [ ] Implement percentile calculations
167
- - [x] Add route-specific and IP-specific throughput tracking
168
- - [x] Create historical data access methods
169
- - [ ] Add integration tests
170
-
171
- ### Phase 3: Performance Optimization (Week 3)
172
- - [x] Use circular buffers for efficiency
173
- - [ ] Implement data aggregation for longer time windows
174
- - [x] Add configurable retention periods
175
- - [ ] Optimize memory usage
176
- - [ ] Add performance benchmarks
177
-
178
- ### Phase 4: Export Formats (Week 4)
179
- - [ ] Add Prometheus metric format with proper metric types
180
- - [ ] Add StatsD format support
181
- - [ ] Add JSON export with metadata
182
- - [ ] Create OpenMetrics compatibility
183
- - [ ] Add documentation and examples
184
-
185
- ## 4. Key Design Decisions
186
-
187
- ### A. Sampling Strategy
188
- - **1-second samples** for fine-grained data
189
- - **Aggregate to 1-minute** for longer retention
190
- - **Keep 1 hour** of second-level data
191
- - **Keep 24 hours** of minute-level data
192
-
193
- ### B. Memory Management
194
- - **Circular buffers** for fixed memory usage
195
- - **Configurable retention** periods
196
- - **Lazy aggregation** for older data
197
- - **Efficient data structures** (typed arrays for samples)
198
-
199
- ### C. Performance Considerations
200
- - **Batch updates** during high throughput
201
- - **Debounced calculations** for expensive metrics
202
- - **Cached results** with TTL
203
- - **Worker thread** option for heavy calculations
204
-
205
- ## 5. Configuration Options
206
-
207
- ```typescript
208
- interface IMetricsConfig {
209
- enabled: boolean;
210
-
211
- // Sampling configuration
212
- sampleIntervalMs: number; // Default: 1000 (1 second)
213
- retentionSeconds: number; // Default: 3600 (1 hour)
214
-
215
- // Performance tuning
216
- enableDetailedTracking: boolean; // Per-connection byte history
217
- enablePercentiles: boolean; // Calculate percentiles
218
- cacheResultsMs: number; // Cache expensive calculations
219
-
220
- // Export configuration
221
- prometheusEnabled: boolean;
222
- prometheusPath: string; // Default: /metrics
223
- prometheusPrefix: string; // Default: smartproxy_
224
- }
225
- ```
226
-
227
- ## 6. Example Usage
228
-
229
- ```typescript
230
- const proxy = new SmartProxy({
231
- metrics: {
232
- enabled: true,
233
- sampleIntervalMs: 1000,
234
- enableDetailedTracking: true
235
- }
236
- });
237
-
238
- // Get metrics instance
239
- const metrics = proxy.getMetrics();
240
-
241
- // Connection metrics
242
- console.log(`Active connections: ${metrics.connections.active()}`);
243
- console.log(`Total connections: ${metrics.connections.total()}`);
244
-
245
- // Throughput metrics
246
- const instant = metrics.throughput.instant();
247
- console.log(`Current: ${instant.in} bytes/sec in, ${instant.out} bytes/sec out`);
248
-
249
- const recent = metrics.throughput.recent(); // Last 10 seconds
250
- const average = metrics.throughput.average(); // Last 60 seconds
251
-
252
- // Custom time window
253
- const custom = metrics.throughput.custom(30); // Last 30 seconds
254
-
255
- // Historical data for graphing
256
- const history = metrics.throughput.history(300); // Last 5 minutes
257
- history.forEach(point => {
258
- console.log(`${new Date(point.timestamp)}: ${point.in} bytes/sec in, ${point.out} bytes/sec out`);
259
- });
260
-
261
- // Top routes by throughput
262
- const routeThroughput = metrics.throughput.byRoute(60);
263
- routeThroughput.forEach((stats, route) => {
264
- console.log(`Route ${route}: ${stats.in} bytes/sec in, ${stats.out} bytes/sec out`);
265
- });
266
-
267
- // Request metrics
268
- console.log(`RPS: ${metrics.requests.perSecond()}`);
269
- console.log(`RPM: ${metrics.requests.perMinute()}`);
270
-
271
- // Totals
272
- console.log(`Total bytes in: ${metrics.totals.bytesIn()}`);
273
- console.log(`Total bytes out: ${metrics.totals.bytesOut()}`);
274
- ```
275
-
276
- ## 7. Prometheus Export Example
277
-
278
- ```
279
- # HELP smartproxy_throughput_bytes_per_second Current throughput in bytes per second
280
- # TYPE smartproxy_throughput_bytes_per_second gauge
281
- smartproxy_throughput_bytes_per_second{direction="in",window="1s"} 1234567
282
- smartproxy_throughput_bytes_per_second{direction="out",window="1s"} 987654
283
- smartproxy_throughput_bytes_per_second{direction="in",window="10s"} 1134567
284
- smartproxy_throughput_bytes_per_second{direction="out",window="10s"} 887654
285
-
286
- # HELP smartproxy_bytes_total Total bytes transferred
287
- # TYPE smartproxy_bytes_total counter
288
- smartproxy_bytes_total{direction="in"} 123456789
289
- smartproxy_bytes_total{direction="out"} 98765432
290
-
291
- # HELP smartproxy_active_connections Current number of active connections
292
- # TYPE smartproxy_active_connections gauge
293
- smartproxy_active_connections 42
294
-
295
- # HELP smartproxy_connection_duration_seconds Connection duration in seconds
296
- # TYPE smartproxy_connection_duration_seconds histogram
297
- smartproxy_connection_duration_seconds_bucket{le="0.1"} 100
298
- smartproxy_connection_duration_seconds_bucket{le="1"} 500
299
- smartproxy_connection_duration_seconds_bucket{le="10"} 800
300
- smartproxy_connection_duration_seconds_bucket{le="+Inf"} 850
301
- smartproxy_connection_duration_seconds_sum 4250
302
- smartproxy_connection_duration_seconds_count 850
303
- ```
304
-
305
- ## 8. Migration Strategy
306
-
307
- ### Breaking Changes
308
- - Completely replace the old metrics API with the new clean design
309
- - Remove all `get*` prefixed methods in favor of grouped properties
310
- - Use simple `{ in, out }` objects instead of verbose property names
311
- - Provide clear migration guide in documentation
312
-
313
- ### Implementation Approach
314
- 1. ✅ Create new `ThroughputTracker` class for time-series data
315
- 2. ✅ Implement new `IMetrics` interface with clean API
316
- 3. ✅ Replace `MetricsCollector` implementation entirely
317
- 4. ✅ Update all references to use new API
318
- 5. ⚠️ Add comprehensive tests for accuracy validation (partial)
319
-
320
- ### Additional Refactoring Completed
321
- - Refactored all SmartProxy components to use cleaner dependency pattern
322
- - Components now receive only `SmartProxy` instance instead of individual dependencies
323
- - Access to other components via `this.smartProxy.componentName`
324
- - Significantly simplified constructor signatures across the codebase
325
-
326
- ## 9. Success Metrics
327
-
328
- - **Accuracy**: Throughput metrics accurate within 1% of actual
329
- - **Performance**: < 1% CPU overhead for metrics collection
330
- - **Memory**: < 10MB memory usage for 1 hour of data
331
- - **Latency**: < 1ms to retrieve any metric
332
- - **Reliability**: No metrics data loss under load
333
-
334
- ## 10. Future Enhancements
335
-
336
- ### Phase 5: Advanced Analytics
337
- - Anomaly detection for traffic patterns
338
- - Predictive analytics for capacity planning
339
- - Correlation analysis between routes
340
- - Real-time alerting integration
341
-
342
- ### Phase 6: Distributed Metrics
343
- - Metrics aggregation across multiple proxies
344
- - Distributed time-series storage
345
- - Cross-proxy analytics
346
- - Global dashboard support
347
-
348
- ## 11. Risks and Mitigations
349
-
350
- ### Risk: Memory Usage
351
- - **Mitigation**: Circular buffers and configurable retention
352
- - **Monitoring**: Track memory usage per metric type
353
-
354
- ### Risk: Performance Impact
355
- - **Mitigation**: Efficient data structures and caching
356
- - **Testing**: Load test with metrics enabled/disabled
357
-
358
- ### Risk: Data Accuracy
359
- - **Mitigation**: Atomic operations and proper synchronization
360
- - **Validation**: Compare with external monitoring tools
361
-
362
- ## Conclusion
363
-
364
- This plan transforms SmartProxy's metrics from a basic cumulative system to a comprehensive, time-series based monitoring solution suitable for production environments. The phased approach ensures minimal disruption while delivering immediate value through accurate throughput measurements.
42
+ - All connection limiting is now consistent across SmartProxy and HttpProxy
43
+ - Route-level limits provide additional granular control
44
+ - Memory usage is optimized for high-traffic scenarios
45
+ - Comprehensive test coverage ensures reliability
@@ -0,0 +1,280 @@
1
+ import { logger } from './logger.js';
2
+
3
+ interface ILogEvent {
4
+ level: 'info' | 'warn' | 'error' | 'debug';
5
+ message: string;
6
+ data?: any;
7
+ count: number;
8
+ firstSeen: number;
9
+ lastSeen: number;
10
+ }
11
+
12
+ interface IAggregatedEvent {
13
+ key: string;
14
+ events: Map<string, ILogEvent>;
15
+ flushTimer?: NodeJS.Timeout;
16
+ }
17
+
18
+ /**
19
+ * Log deduplication utility to reduce log spam for repetitive events
20
+ */
21
+ export class LogDeduplicator {
22
+ private globalFlushTimer?: NodeJS.Timeout;
23
+ private aggregatedEvents: Map<string, IAggregatedEvent> = new Map();
24
+ private flushInterval: number = 5000; // 5 seconds
25
+ private maxBatchSize: number = 100;
26
+
27
+ constructor(flushInterval?: number) {
28
+ if (flushInterval) {
29
+ this.flushInterval = flushInterval;
30
+ }
31
+
32
+ // Set up global periodic flush to ensure logs are emitted regularly
33
+ this.globalFlushTimer = setInterval(() => {
34
+ this.flushAll();
35
+ }, this.flushInterval * 2); // Flush everything every 2x the normal interval
36
+
37
+ if (this.globalFlushTimer.unref) {
38
+ this.globalFlushTimer.unref();
39
+ }
40
+ }
41
+
42
+ /**
43
+ * Log a deduplicated event
44
+ * @param key - Aggregation key (e.g., 'connection-rejected', 'cleanup-batch')
45
+ * @param level - Log level
46
+ * @param message - Log message template
47
+ * @param data - Additional data
48
+ * @param dedupeKey - Deduplication key within the aggregation (e.g., IP address, reason)
49
+ */
50
+ public log(
51
+ key: string,
52
+ level: 'info' | 'warn' | 'error' | 'debug',
53
+ message: string,
54
+ data?: any,
55
+ dedupeKey?: string
56
+ ): void {
57
+ const eventKey = dedupeKey || message;
58
+ const now = Date.now();
59
+
60
+ if (!this.aggregatedEvents.has(key)) {
61
+ this.aggregatedEvents.set(key, {
62
+ key,
63
+ events: new Map(),
64
+ flushTimer: undefined
65
+ });
66
+ }
67
+
68
+ const aggregated = this.aggregatedEvents.get(key)!;
69
+
70
+ if (aggregated.events.has(eventKey)) {
71
+ const event = aggregated.events.get(eventKey)!;
72
+ event.count++;
73
+ event.lastSeen = now;
74
+ if (data) {
75
+ event.data = { ...event.data, ...data };
76
+ }
77
+ } else {
78
+ aggregated.events.set(eventKey, {
79
+ level,
80
+ message,
81
+ data,
82
+ count: 1,
83
+ firstSeen: now,
84
+ lastSeen: now
85
+ });
86
+ }
87
+
88
+ // Check if we should flush due to size
89
+ if (aggregated.events.size >= this.maxBatchSize) {
90
+ this.flush(key);
91
+ } else if (!aggregated.flushTimer) {
92
+ // Schedule flush
93
+ aggregated.flushTimer = setTimeout(() => {
94
+ this.flush(key);
95
+ }, this.flushInterval);
96
+
97
+ if (aggregated.flushTimer.unref) {
98
+ aggregated.flushTimer.unref();
99
+ }
100
+ }
101
+ }
102
+
103
+ /**
104
+ * Flush aggregated events for a specific key
105
+ */
106
+ public flush(key: string): void {
107
+ const aggregated = this.aggregatedEvents.get(key);
108
+ if (!aggregated || aggregated.events.size === 0) {
109
+ return;
110
+ }
111
+
112
+ if (aggregated.flushTimer) {
113
+ clearTimeout(aggregated.flushTimer);
114
+ aggregated.flushTimer = undefined;
115
+ }
116
+
117
+ // Emit aggregated log based on the key
118
+ switch (key) {
119
+ case 'connection-rejected':
120
+ this.flushConnectionRejections(aggregated);
121
+ break;
122
+ case 'connection-cleanup':
123
+ this.flushConnectionCleanups(aggregated);
124
+ break;
125
+ case 'ip-rejected':
126
+ this.flushIPRejections(aggregated);
127
+ break;
128
+ default:
129
+ this.flushGeneric(aggregated);
130
+ }
131
+
132
+ // Clear events
133
+ aggregated.events.clear();
134
+ }
135
+
136
+ /**
137
+ * Flush all pending events
138
+ */
139
+ public flushAll(): void {
140
+ for (const key of this.aggregatedEvents.keys()) {
141
+ this.flush(key);
142
+ }
143
+ }
144
+
145
+ private flushConnectionRejections(aggregated: IAggregatedEvent): void {
146
+ const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
147
+ const byReason = new Map<string, number>();
148
+
149
+ for (const [, event] of aggregated.events) {
150
+ const reason = event.data?.reason || 'unknown';
151
+ byReason.set(reason, (byReason.get(reason) || 0) + event.count);
152
+ }
153
+
154
+ const reasonSummary = Array.from(byReason.entries())
155
+ .sort((a, b) => b[1] - a[1])
156
+ .map(([reason, count]) => `${reason}: ${count}`)
157
+ .join(', ');
158
+
159
+ logger.log('warn', `Rejected ${totalCount} connections`, {
160
+ reasons: reasonSummary,
161
+ uniqueIPs: aggregated.events.size,
162
+ duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
163
+ component: 'connection-dedup'
164
+ });
165
+ }
166
+
167
+ private flushConnectionCleanups(aggregated: IAggregatedEvent): void {
168
+ const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
169
+ const byReason = new Map<string, number>();
170
+
171
+ for (const [, event] of aggregated.events) {
172
+ const reason = event.data?.reason || 'normal';
173
+ byReason.set(reason, (byReason.get(reason) || 0) + event.count);
174
+ }
175
+
176
+ const reasonSummary = Array.from(byReason.entries())
177
+ .sort((a, b) => b[1] - a[1])
178
+ .slice(0, 5) // Top 5 reasons
179
+ .map(([reason, count]) => `${reason}: ${count}`)
180
+ .join(', ');
181
+
182
+ logger.log('info', `Cleaned up ${totalCount} connections`, {
183
+ reasons: reasonSummary,
184
+ duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
185
+ component: 'connection-dedup'
186
+ });
187
+ }
188
+
189
+ private flushIPRejections(aggregated: IAggregatedEvent): void {
190
+ const byIP = new Map<string, { count: number; reasons: Set<string> }>();
191
+
192
+ for (const [ip, event] of aggregated.events) {
193
+ if (!byIP.has(ip)) {
194
+ byIP.set(ip, { count: 0, reasons: new Set() });
195
+ }
196
+ const ipData = byIP.get(ip)!;
197
+ ipData.count += event.count;
198
+ if (event.data?.reason) {
199
+ ipData.reasons.add(event.data.reason);
200
+ }
201
+ }
202
+
203
+ // Log top offenders
204
+ const topOffenders = Array.from(byIP.entries())
205
+ .sort((a, b) => b[1].count - a[1].count)
206
+ .slice(0, 10)
207
+ .map(([ip, data]) => `${ip} (${data.count}x, ${Array.from(data.reasons).join('/')})`)
208
+ .join(', ');
209
+
210
+ const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
211
+
212
+ logger.log('warn', `Rejected ${totalRejections} connections from ${byIP.size} IPs`, {
213
+ topOffenders,
214
+ duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
215
+ component: 'ip-dedup'
216
+ });
217
+ }
218
+
219
+ private flushGeneric(aggregated: IAggregatedEvent): void {
220
+ const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
221
+ const level = aggregated.events.values().next().value?.level || 'info';
222
+
223
+ // Special handling for IP cleanup events
224
+ if (aggregated.key === 'ip-cleanup') {
225
+ const totalCleaned = Array.from(aggregated.events.values()).reduce((sum, e) => {
226
+ return sum + (e.data?.cleanedIPs || 0) + (e.data?.cleanedRateLimits || 0);
227
+ }, 0);
228
+
229
+ if (totalCleaned > 0) {
230
+ logger.log(level as any, `IP tracking cleanup: removed ${totalCleaned} entries across ${totalCount} cleanup cycles`, {
231
+ duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
232
+ component: 'log-dedup'
233
+ });
234
+ }
235
+ } else {
236
+ logger.log(level as any, `${aggregated.key}: ${totalCount} events`, {
237
+ uniqueEvents: aggregated.events.size,
238
+ duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
239
+ component: 'log-dedup'
240
+ });
241
+ }
242
+ }
243
+
244
+ /**
245
+ * Cleanup and stop deduplication
246
+ */
247
+ public cleanup(): void {
248
+ this.flushAll();
249
+
250
+ if (this.globalFlushTimer) {
251
+ clearInterval(this.globalFlushTimer);
252
+ this.globalFlushTimer = undefined;
253
+ }
254
+
255
+ for (const aggregated of this.aggregatedEvents.values()) {
256
+ if (aggregated.flushTimer) {
257
+ clearTimeout(aggregated.flushTimer);
258
+ }
259
+ }
260
+ this.aggregatedEvents.clear();
261
+ }
262
+ }
263
+
264
+ // Global instance for connection-related log deduplication
265
+ export const connectionLogDeduplicator = new LogDeduplicator(5000); // 5 second batches
266
+
267
+ // Ensure logs are flushed on process exit
268
+ process.on('beforeExit', () => {
269
+ connectionLogDeduplicator.flushAll();
270
+ });
271
+
272
+ process.on('SIGINT', () => {
273
+ connectionLogDeduplicator.cleanup();
274
+ process.exit(0);
275
+ });
276
+
277
+ process.on('SIGTERM', () => {
278
+ connectionLogDeduplicator.cleanup();
279
+ process.exit(0);
280
+ });