@push.rocks/smartproxy 19.6.13 → 19.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/core/utils/log-deduplicator.d.ts +36 -0
- package/dist_ts/core/utils/log-deduplicator.js +224 -0
- package/dist_ts/core/utils/shared-security-manager.d.ts +2 -1
- package/dist_ts/core/utils/shared-security-manager.js +22 -2
- package/dist_ts/proxies/http-proxy/http-proxy.d.ts +1 -0
- package/dist_ts/proxies/http-proxy/http-proxy.js +94 -9
- package/dist_ts/proxies/http-proxy/models/types.d.ts +2 -0
- package/dist_ts/proxies/http-proxy/models/types.js +1 -1
- package/dist_ts/proxies/http-proxy/security-manager.d.ts +42 -1
- package/dist_ts/proxies/http-proxy/security-manager.js +121 -2
- package/dist_ts/proxies/smart-proxy/connection-manager.d.ts +14 -0
- package/dist_ts/proxies/smart-proxy/connection-manager.js +74 -26
- package/dist_ts/proxies/smart-proxy/http-proxy-bridge.js +5 -1
- package/dist_ts/proxies/smart-proxy/models/interfaces.d.ts +1 -0
- package/dist_ts/proxies/smart-proxy/route-connection-handler.js +24 -8
- package/dist_ts/proxies/smart-proxy/security-manager.d.ts +9 -0
- package/dist_ts/proxies/smart-proxy/security-manager.js +63 -1
- package/dist_ts/proxies/smart-proxy/smart-proxy.js +4 -1
- package/package.json +1 -1
- package/readme.hints.md +80 -1
- package/readme.plan.md +34 -353
- package/ts/core/utils/log-deduplicator.ts +280 -0
- package/ts/core/utils/shared-security-manager.ts +24 -1
- package/ts/proxies/http-proxy/http-proxy.ts +129 -9
- package/ts/proxies/http-proxy/models/types.ts +4 -0
- package/ts/proxies/http-proxy/security-manager.ts +136 -1
- package/ts/proxies/smart-proxy/connection-manager.ts +93 -27
- package/ts/proxies/smart-proxy/http-proxy-bridge.ts +5 -0
- package/ts/proxies/smart-proxy/models/interfaces.ts +1 -0
- package/ts/proxies/smart-proxy/route-connection-handler.ts +45 -14
- package/ts/proxies/smart-proxy/security-manager.ts +76 -1
- package/ts/proxies/smart-proxy/smart-proxy.ts +4 -0
package/readme.plan.md
CHANGED
|
@@ -1,364 +1,45 @@
|
|
|
1
|
-
# SmartProxy
|
|
1
|
+
# SmartProxy Connection Limiting Improvements Plan
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Command to re-read CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## Issues Identified
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
1. **HttpProxy Bypass**: Connections forwarded to HttpProxy for TLS termination only check global limits, not per-IP limits
|
|
8
|
+
2. **Missing Route-Level Connection Enforcement**: Routes can define `security.maxConnections` but it's never enforced
|
|
9
|
+
3. **Cleanup Queue Race Condition**: New connections can be added to cleanup queue while processing
|
|
10
|
+
4. **IP Tracking Memory Optimization**: IP entries remain in map even without active connections
|
|
8
11
|
|
|
9
|
-
|
|
10
|
-
- **No Time-Series Data**: Cannot track throughput changes over time
|
|
11
|
-
- **Inaccurate Estimates**: Attempting to estimate rates for older connections is fundamentally flawed
|
|
12
|
-
- **No Sliding Windows**: Cannot provide different time window views (1s, 10s, 60s, etc.)
|
|
13
|
-
- **Limited Granularity**: Only provides a single 60-second view
|
|
12
|
+
## Implementation Steps
|
|
14
13
|
|
|
15
|
-
|
|
14
|
+
### 1. Fix HttpProxy Per-IP Validation ✓
|
|
15
|
+
- [x] Pass IP information to HttpProxy when forwarding connections
|
|
16
|
+
- [x] Add per-IP validation in HttpProxy connection handler
|
|
17
|
+
- [x] Ensure connection tracking is consistent between SmartProxy and HttpProxy
|
|
16
18
|
|
|
17
|
-
###
|
|
19
|
+
### 2. Implement Route-Level Connection Limits ✓
|
|
20
|
+
- [x] Add connection count tracking per route in ConnectionManager
|
|
21
|
+
- [x] Update SharedSecurityManager.isAllowed() to check route-specific maxConnections
|
|
22
|
+
- [x] Add route connection limit validation in route-connection-handler.ts
|
|
18
23
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
bytesOut: number;
|
|
24
|
-
}
|
|
24
|
+
### 3. Fix Cleanup Queue Race Condition ✓
|
|
25
|
+
- [x] Implement proper queue snapshotting before processing
|
|
26
|
+
- [x] Ensure new connections added during processing aren't missed
|
|
27
|
+
- [x] Add proper synchronization for cleanup operations
|
|
25
28
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
private accumulatedBytesIn: number = 0;
|
|
31
|
-
private accumulatedBytesOut: number = 0;
|
|
32
|
-
|
|
33
|
-
// Called on every data transfer
|
|
34
|
-
public recordBytes(bytesIn: number, bytesOut: number): void {
|
|
35
|
-
this.accumulatedBytesIn += bytesIn;
|
|
36
|
-
this.accumulatedBytesOut += bytesOut;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
// Called periodically (every second)
|
|
40
|
-
public takeSample(): void {
|
|
41
|
-
const now = Date.now();
|
|
42
|
-
|
|
43
|
-
// Record accumulated bytes since last sample
|
|
44
|
-
this.samples.push({
|
|
45
|
-
timestamp: now,
|
|
46
|
-
bytesIn: this.accumulatedBytesIn,
|
|
47
|
-
bytesOut: this.accumulatedBytesOut
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
// Reset accumulators
|
|
51
|
-
this.accumulatedBytesIn = 0;
|
|
52
|
-
this.accumulatedBytesOut = 0;
|
|
53
|
-
|
|
54
|
-
// Trim old samples
|
|
55
|
-
const cutoff = now - 3600000; // 1 hour
|
|
56
|
-
this.samples = this.samples.filter(s => s.timestamp > cutoff);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
// Get rate over specified window
|
|
60
|
-
public getRate(windowSeconds: number): { bytesInPerSec: number; bytesOutPerSec: number } {
|
|
61
|
-
const now = Date.now();
|
|
62
|
-
const windowStart = now - (windowSeconds * 1000);
|
|
63
|
-
|
|
64
|
-
const relevantSamples = this.samples.filter(s => s.timestamp > windowStart);
|
|
65
|
-
|
|
66
|
-
if (relevantSamples.length === 0) {
|
|
67
|
-
return { bytesInPerSec: 0, bytesOutPerSec: 0 };
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
const totalBytesIn = relevantSamples.reduce((sum, s) => sum + s.bytesIn, 0);
|
|
71
|
-
const totalBytesOut = relevantSamples.reduce((sum, s) => sum + s.bytesOut, 0);
|
|
72
|
-
|
|
73
|
-
const actualWindow = (now - relevantSamples[0].timestamp) / 1000;
|
|
74
|
-
|
|
75
|
-
return {
|
|
76
|
-
bytesInPerSec: Math.round(totalBytesIn / actualWindow),
|
|
77
|
-
bytesOutPerSec: Math.round(totalBytesOut / actualWindow)
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
```
|
|
29
|
+
### 4. Optimize IP Tracking Memory Usage ✓
|
|
30
|
+
- [x] Add periodic cleanup for IPs with no active connections
|
|
31
|
+
- [x] Implement expiry for rate limit timestamps
|
|
32
|
+
- [x] Add memory-efficient data structures for IP tracking
|
|
82
33
|
|
|
83
|
-
###
|
|
34
|
+
### 5. Add Comprehensive Tests ✓
|
|
35
|
+
- [x] Test per-IP limits with HttpProxy forwarding
|
|
36
|
+
- [x] Test route-level connection limits
|
|
37
|
+
- [x] Test cleanup queue edge cases
|
|
38
|
+
- [x] Test memory usage with many unique IPs
|
|
84
39
|
|
|
85
|
-
|
|
86
|
-
// In ConnectionRecord, add:
|
|
87
|
-
interface IConnectionRecord {
|
|
88
|
-
// ... existing fields ...
|
|
89
|
-
|
|
90
|
-
// Byte counters with timestamps
|
|
91
|
-
bytesReceivedHistory: Array<{ timestamp: number; bytes: number }>;
|
|
92
|
-
bytesSentHistory: Array<{ timestamp: number; bytes: number }>;
|
|
93
|
-
|
|
94
|
-
// For efficiency, could use circular buffer
|
|
95
|
-
lastBytesReceivedUpdate: number;
|
|
96
|
-
lastBytesSentUpdate: number;
|
|
97
|
-
}
|
|
98
|
-
```
|
|
40
|
+
## Notes
|
|
99
41
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
// Connection metrics
|
|
105
|
-
connections: {
|
|
106
|
-
active(): number;
|
|
107
|
-
total(): number;
|
|
108
|
-
byRoute(): Map<string, number>;
|
|
109
|
-
byIP(): Map<string, number>;
|
|
110
|
-
topIPs(limit?: number): Array<{ ip: string; count: number }>;
|
|
111
|
-
};
|
|
112
|
-
|
|
113
|
-
// Throughput metrics (bytes per second)
|
|
114
|
-
throughput: {
|
|
115
|
-
instant(): { in: number; out: number }; // Last 1 second
|
|
116
|
-
recent(): { in: number; out: number }; // Last 10 seconds
|
|
117
|
-
average(): { in: number; out: number }; // Last 60 seconds
|
|
118
|
-
custom(seconds: number): { in: number; out: number };
|
|
119
|
-
history(seconds: number): Array<{ timestamp: number; in: number; out: number }>;
|
|
120
|
-
byRoute(windowSeconds?: number): Map<string, { in: number; out: number }>;
|
|
121
|
-
byIP(windowSeconds?: number): Map<string, { in: number; out: number }>;
|
|
122
|
-
};
|
|
123
|
-
|
|
124
|
-
// Request metrics
|
|
125
|
-
requests: {
|
|
126
|
-
perSecond(): number;
|
|
127
|
-
perMinute(): number;
|
|
128
|
-
total(): number;
|
|
129
|
-
};
|
|
130
|
-
|
|
131
|
-
// Cumulative totals
|
|
132
|
-
totals: {
|
|
133
|
-
bytesIn(): number;
|
|
134
|
-
bytesOut(): number;
|
|
135
|
-
connections(): number;
|
|
136
|
-
};
|
|
137
|
-
|
|
138
|
-
// Performance metrics
|
|
139
|
-
percentiles: {
|
|
140
|
-
connectionDuration(): { p50: number; p95: number; p99: number };
|
|
141
|
-
bytesTransferred(): {
|
|
142
|
-
in: { p50: number; p95: number; p99: number };
|
|
143
|
-
out: { p50: number; p95: number; p99: number };
|
|
144
|
-
};
|
|
145
|
-
};
|
|
146
|
-
}
|
|
147
|
-
```
|
|
148
|
-
|
|
149
|
-
## 3. Implementation Plan
|
|
150
|
-
|
|
151
|
-
### Current Status
|
|
152
|
-
- **Phase 1**: ~90% complete (core functionality implemented, tests need fixing)
|
|
153
|
-
- **Phase 2**: ~60% complete (main features done, percentiles pending)
|
|
154
|
-
- **Phase 3**: ~40% complete (basic optimizations in place)
|
|
155
|
-
- **Phase 4**: 0% complete (export formats not started)
|
|
156
|
-
|
|
157
|
-
### Phase 1: Core Throughput Tracking (Week 1)
|
|
158
|
-
- [x] Implement `ThroughputTracker` class
|
|
159
|
-
- [x] Integrate byte recording into socket data handlers
|
|
160
|
-
- [x] Add periodic sampling (1-second intervals)
|
|
161
|
-
- [x] Update `getThroughputRate()` to use time-series data (replaced with new clean API)
|
|
162
|
-
- [ ] Add unit tests for throughput tracking
|
|
163
|
-
|
|
164
|
-
### Phase 2: Enhanced Metrics (Week 2)
|
|
165
|
-
- [x] Add configurable time windows (1s, 10s, 60s, 5m, etc.)
|
|
166
|
-
- [ ] Implement percentile calculations
|
|
167
|
-
- [x] Add route-specific and IP-specific throughput tracking
|
|
168
|
-
- [x] Create historical data access methods
|
|
169
|
-
- [ ] Add integration tests
|
|
170
|
-
|
|
171
|
-
### Phase 3: Performance Optimization (Week 3)
|
|
172
|
-
- [x] Use circular buffers for efficiency
|
|
173
|
-
- [ ] Implement data aggregation for longer time windows
|
|
174
|
-
- [x] Add configurable retention periods
|
|
175
|
-
- [ ] Optimize memory usage
|
|
176
|
-
- [ ] Add performance benchmarks
|
|
177
|
-
|
|
178
|
-
### Phase 4: Export Formats (Week 4)
|
|
179
|
-
- [ ] Add Prometheus metric format with proper metric types
|
|
180
|
-
- [ ] Add StatsD format support
|
|
181
|
-
- [ ] Add JSON export with metadata
|
|
182
|
-
- [ ] Create OpenMetrics compatibility
|
|
183
|
-
- [ ] Add documentation and examples
|
|
184
|
-
|
|
185
|
-
## 4. Key Design Decisions
|
|
186
|
-
|
|
187
|
-
### A. Sampling Strategy
|
|
188
|
-
- **1-second samples** for fine-grained data
|
|
189
|
-
- **Aggregate to 1-minute** for longer retention
|
|
190
|
-
- **Keep 1 hour** of second-level data
|
|
191
|
-
- **Keep 24 hours** of minute-level data
|
|
192
|
-
|
|
193
|
-
### B. Memory Management
|
|
194
|
-
- **Circular buffers** for fixed memory usage
|
|
195
|
-
- **Configurable retention** periods
|
|
196
|
-
- **Lazy aggregation** for older data
|
|
197
|
-
- **Efficient data structures** (typed arrays for samples)
|
|
198
|
-
|
|
199
|
-
### C. Performance Considerations
|
|
200
|
-
- **Batch updates** during high throughput
|
|
201
|
-
- **Debounced calculations** for expensive metrics
|
|
202
|
-
- **Cached results** with TTL
|
|
203
|
-
- **Worker thread** option for heavy calculations
|
|
204
|
-
|
|
205
|
-
## 5. Configuration Options
|
|
206
|
-
|
|
207
|
-
```typescript
|
|
208
|
-
interface IMetricsConfig {
|
|
209
|
-
enabled: boolean;
|
|
210
|
-
|
|
211
|
-
// Sampling configuration
|
|
212
|
-
sampleIntervalMs: number; // Default: 1000 (1 second)
|
|
213
|
-
retentionSeconds: number; // Default: 3600 (1 hour)
|
|
214
|
-
|
|
215
|
-
// Performance tuning
|
|
216
|
-
enableDetailedTracking: boolean; // Per-connection byte history
|
|
217
|
-
enablePercentiles: boolean; // Calculate percentiles
|
|
218
|
-
cacheResultsMs: number; // Cache expensive calculations
|
|
219
|
-
|
|
220
|
-
// Export configuration
|
|
221
|
-
prometheusEnabled: boolean;
|
|
222
|
-
prometheusPath: string; // Default: /metrics
|
|
223
|
-
prometheusPrefix: string; // Default: smartproxy_
|
|
224
|
-
}
|
|
225
|
-
```
|
|
226
|
-
|
|
227
|
-
## 6. Example Usage
|
|
228
|
-
|
|
229
|
-
```typescript
|
|
230
|
-
const proxy = new SmartProxy({
|
|
231
|
-
metrics: {
|
|
232
|
-
enabled: true,
|
|
233
|
-
sampleIntervalMs: 1000,
|
|
234
|
-
enableDetailedTracking: true
|
|
235
|
-
}
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
// Get metrics instance
|
|
239
|
-
const metrics = proxy.getMetrics();
|
|
240
|
-
|
|
241
|
-
// Connection metrics
|
|
242
|
-
console.log(`Active connections: ${metrics.connections.active()}`);
|
|
243
|
-
console.log(`Total connections: ${metrics.connections.total()}`);
|
|
244
|
-
|
|
245
|
-
// Throughput metrics
|
|
246
|
-
const instant = metrics.throughput.instant();
|
|
247
|
-
console.log(`Current: ${instant.in} bytes/sec in, ${instant.out} bytes/sec out`);
|
|
248
|
-
|
|
249
|
-
const recent = metrics.throughput.recent(); // Last 10 seconds
|
|
250
|
-
const average = metrics.throughput.average(); // Last 60 seconds
|
|
251
|
-
|
|
252
|
-
// Custom time window
|
|
253
|
-
const custom = metrics.throughput.custom(30); // Last 30 seconds
|
|
254
|
-
|
|
255
|
-
// Historical data for graphing
|
|
256
|
-
const history = metrics.throughput.history(300); // Last 5 minutes
|
|
257
|
-
history.forEach(point => {
|
|
258
|
-
console.log(`${new Date(point.timestamp)}: ${point.in} bytes/sec in, ${point.out} bytes/sec out`);
|
|
259
|
-
});
|
|
260
|
-
|
|
261
|
-
// Top routes by throughput
|
|
262
|
-
const routeThroughput = metrics.throughput.byRoute(60);
|
|
263
|
-
routeThroughput.forEach((stats, route) => {
|
|
264
|
-
console.log(`Route ${route}: ${stats.in} bytes/sec in, ${stats.out} bytes/sec out`);
|
|
265
|
-
});
|
|
266
|
-
|
|
267
|
-
// Request metrics
|
|
268
|
-
console.log(`RPS: ${metrics.requests.perSecond()}`);
|
|
269
|
-
console.log(`RPM: ${metrics.requests.perMinute()}`);
|
|
270
|
-
|
|
271
|
-
// Totals
|
|
272
|
-
console.log(`Total bytes in: ${metrics.totals.bytesIn()}`);
|
|
273
|
-
console.log(`Total bytes out: ${metrics.totals.bytesOut()}`);
|
|
274
|
-
```
|
|
275
|
-
|
|
276
|
-
## 7. Prometheus Export Example
|
|
277
|
-
|
|
278
|
-
```
|
|
279
|
-
# HELP smartproxy_throughput_bytes_per_second Current throughput in bytes per second
|
|
280
|
-
# TYPE smartproxy_throughput_bytes_per_second gauge
|
|
281
|
-
smartproxy_throughput_bytes_per_second{direction="in",window="1s"} 1234567
|
|
282
|
-
smartproxy_throughput_bytes_per_second{direction="out",window="1s"} 987654
|
|
283
|
-
smartproxy_throughput_bytes_per_second{direction="in",window="10s"} 1134567
|
|
284
|
-
smartproxy_throughput_bytes_per_second{direction="out",window="10s"} 887654
|
|
285
|
-
|
|
286
|
-
# HELP smartproxy_bytes_total Total bytes transferred
|
|
287
|
-
# TYPE smartproxy_bytes_total counter
|
|
288
|
-
smartproxy_bytes_total{direction="in"} 123456789
|
|
289
|
-
smartproxy_bytes_total{direction="out"} 98765432
|
|
290
|
-
|
|
291
|
-
# HELP smartproxy_active_connections Current number of active connections
|
|
292
|
-
# TYPE smartproxy_active_connections gauge
|
|
293
|
-
smartproxy_active_connections 42
|
|
294
|
-
|
|
295
|
-
# HELP smartproxy_connection_duration_seconds Connection duration in seconds
|
|
296
|
-
# TYPE smartproxy_connection_duration_seconds histogram
|
|
297
|
-
smartproxy_connection_duration_seconds_bucket{le="0.1"} 100
|
|
298
|
-
smartproxy_connection_duration_seconds_bucket{le="1"} 500
|
|
299
|
-
smartproxy_connection_duration_seconds_bucket{le="10"} 800
|
|
300
|
-
smartproxy_connection_duration_seconds_bucket{le="+Inf"} 850
|
|
301
|
-
smartproxy_connection_duration_seconds_sum 4250
|
|
302
|
-
smartproxy_connection_duration_seconds_count 850
|
|
303
|
-
```
|
|
304
|
-
|
|
305
|
-
## 8. Migration Strategy
|
|
306
|
-
|
|
307
|
-
### Breaking Changes
|
|
308
|
-
- Completely replace the old metrics API with the new clean design
|
|
309
|
-
- Remove all `get*` prefixed methods in favor of grouped properties
|
|
310
|
-
- Use simple `{ in, out }` objects instead of verbose property names
|
|
311
|
-
- Provide clear migration guide in documentation
|
|
312
|
-
|
|
313
|
-
### Implementation Approach
|
|
314
|
-
1. ✅ Create new `ThroughputTracker` class for time-series data
|
|
315
|
-
2. ✅ Implement new `IMetrics` interface with clean API
|
|
316
|
-
3. ✅ Replace `MetricsCollector` implementation entirely
|
|
317
|
-
4. ✅ Update all references to use new API
|
|
318
|
-
5. ⚠️ Add comprehensive tests for accuracy validation (partial)
|
|
319
|
-
|
|
320
|
-
### Additional Refactoring Completed
|
|
321
|
-
- Refactored all SmartProxy components to use cleaner dependency pattern
|
|
322
|
-
- Components now receive only `SmartProxy` instance instead of individual dependencies
|
|
323
|
-
- Access to other components via `this.smartProxy.componentName`
|
|
324
|
-
- Significantly simplified constructor signatures across the codebase
|
|
325
|
-
|
|
326
|
-
## 9. Success Metrics
|
|
327
|
-
|
|
328
|
-
- **Accuracy**: Throughput metrics accurate within 1% of actual
|
|
329
|
-
- **Performance**: < 1% CPU overhead for metrics collection
|
|
330
|
-
- **Memory**: < 10MB memory usage for 1 hour of data
|
|
331
|
-
- **Latency**: < 1ms to retrieve any metric
|
|
332
|
-
- **Reliability**: No metrics data loss under load
|
|
333
|
-
|
|
334
|
-
## 10. Future Enhancements
|
|
335
|
-
|
|
336
|
-
### Phase 5: Advanced Analytics
|
|
337
|
-
- Anomaly detection for traffic patterns
|
|
338
|
-
- Predictive analytics for capacity planning
|
|
339
|
-
- Correlation analysis between routes
|
|
340
|
-
- Real-time alerting integration
|
|
341
|
-
|
|
342
|
-
### Phase 6: Distributed Metrics
|
|
343
|
-
- Metrics aggregation across multiple proxies
|
|
344
|
-
- Distributed time-series storage
|
|
345
|
-
- Cross-proxy analytics
|
|
346
|
-
- Global dashboard support
|
|
347
|
-
|
|
348
|
-
## 11. Risks and Mitigations
|
|
349
|
-
|
|
350
|
-
### Risk: Memory Usage
|
|
351
|
-
- **Mitigation**: Circular buffers and configurable retention
|
|
352
|
-
- **Monitoring**: Track memory usage per metric type
|
|
353
|
-
|
|
354
|
-
### Risk: Performance Impact
|
|
355
|
-
- **Mitigation**: Efficient data structures and caching
|
|
356
|
-
- **Testing**: Load test with metrics enabled/disabled
|
|
357
|
-
|
|
358
|
-
### Risk: Data Accuracy
|
|
359
|
-
- **Mitigation**: Atomic operations and proper synchronization
|
|
360
|
-
- **Validation**: Compare with external monitoring tools
|
|
361
|
-
|
|
362
|
-
## Conclusion
|
|
363
|
-
|
|
364
|
-
This plan transforms SmartProxy's metrics from a basic cumulative system to a comprehensive, time-series based monitoring solution suitable for production environments. The phased approach ensures minimal disruption while delivering immediate value through accurate throughput measurements.
|
|
42
|
+
- All connection limiting is now consistent across SmartProxy and HttpProxy
|
|
43
|
+
- Route-level limits provide additional granular control
|
|
44
|
+
- Memory usage is optimized for high-traffic scenarios
|
|
45
|
+
- Comprehensive test coverage ensures reliability
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
import { logger } from './logger.js';
|
|
2
|
+
|
|
3
|
+
interface ILogEvent {
|
|
4
|
+
level: 'info' | 'warn' | 'error' | 'debug';
|
|
5
|
+
message: string;
|
|
6
|
+
data?: any;
|
|
7
|
+
count: number;
|
|
8
|
+
firstSeen: number;
|
|
9
|
+
lastSeen: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
interface IAggregatedEvent {
|
|
13
|
+
key: string;
|
|
14
|
+
events: Map<string, ILogEvent>;
|
|
15
|
+
flushTimer?: NodeJS.Timeout;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Log deduplication utility to reduce log spam for repetitive events
|
|
20
|
+
*/
|
|
21
|
+
export class LogDeduplicator {
|
|
22
|
+
private globalFlushTimer?: NodeJS.Timeout;
|
|
23
|
+
private aggregatedEvents: Map<string, IAggregatedEvent> = new Map();
|
|
24
|
+
private flushInterval: number = 5000; // 5 seconds
|
|
25
|
+
private maxBatchSize: number = 100;
|
|
26
|
+
|
|
27
|
+
constructor(flushInterval?: number) {
|
|
28
|
+
if (flushInterval) {
|
|
29
|
+
this.flushInterval = flushInterval;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Set up global periodic flush to ensure logs are emitted regularly
|
|
33
|
+
this.globalFlushTimer = setInterval(() => {
|
|
34
|
+
this.flushAll();
|
|
35
|
+
}, this.flushInterval * 2); // Flush everything every 2x the normal interval
|
|
36
|
+
|
|
37
|
+
if (this.globalFlushTimer.unref) {
|
|
38
|
+
this.globalFlushTimer.unref();
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Log a deduplicated event
|
|
44
|
+
* @param key - Aggregation key (e.g., 'connection-rejected', 'cleanup-batch')
|
|
45
|
+
* @param level - Log level
|
|
46
|
+
* @param message - Log message template
|
|
47
|
+
* @param data - Additional data
|
|
48
|
+
* @param dedupeKey - Deduplication key within the aggregation (e.g., IP address, reason)
|
|
49
|
+
*/
|
|
50
|
+
public log(
|
|
51
|
+
key: string,
|
|
52
|
+
level: 'info' | 'warn' | 'error' | 'debug',
|
|
53
|
+
message: string,
|
|
54
|
+
data?: any,
|
|
55
|
+
dedupeKey?: string
|
|
56
|
+
): void {
|
|
57
|
+
const eventKey = dedupeKey || message;
|
|
58
|
+
const now = Date.now();
|
|
59
|
+
|
|
60
|
+
if (!this.aggregatedEvents.has(key)) {
|
|
61
|
+
this.aggregatedEvents.set(key, {
|
|
62
|
+
key,
|
|
63
|
+
events: new Map(),
|
|
64
|
+
flushTimer: undefined
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const aggregated = this.aggregatedEvents.get(key)!;
|
|
69
|
+
|
|
70
|
+
if (aggregated.events.has(eventKey)) {
|
|
71
|
+
const event = aggregated.events.get(eventKey)!;
|
|
72
|
+
event.count++;
|
|
73
|
+
event.lastSeen = now;
|
|
74
|
+
if (data) {
|
|
75
|
+
event.data = { ...event.data, ...data };
|
|
76
|
+
}
|
|
77
|
+
} else {
|
|
78
|
+
aggregated.events.set(eventKey, {
|
|
79
|
+
level,
|
|
80
|
+
message,
|
|
81
|
+
data,
|
|
82
|
+
count: 1,
|
|
83
|
+
firstSeen: now,
|
|
84
|
+
lastSeen: now
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Check if we should flush due to size
|
|
89
|
+
if (aggregated.events.size >= this.maxBatchSize) {
|
|
90
|
+
this.flush(key);
|
|
91
|
+
} else if (!aggregated.flushTimer) {
|
|
92
|
+
// Schedule flush
|
|
93
|
+
aggregated.flushTimer = setTimeout(() => {
|
|
94
|
+
this.flush(key);
|
|
95
|
+
}, this.flushInterval);
|
|
96
|
+
|
|
97
|
+
if (aggregated.flushTimer.unref) {
|
|
98
|
+
aggregated.flushTimer.unref();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Flush aggregated events for a specific key
|
|
105
|
+
*/
|
|
106
|
+
public flush(key: string): void {
|
|
107
|
+
const aggregated = this.aggregatedEvents.get(key);
|
|
108
|
+
if (!aggregated || aggregated.events.size === 0) {
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (aggregated.flushTimer) {
|
|
113
|
+
clearTimeout(aggregated.flushTimer);
|
|
114
|
+
aggregated.flushTimer = undefined;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Emit aggregated log based on the key
|
|
118
|
+
switch (key) {
|
|
119
|
+
case 'connection-rejected':
|
|
120
|
+
this.flushConnectionRejections(aggregated);
|
|
121
|
+
break;
|
|
122
|
+
case 'connection-cleanup':
|
|
123
|
+
this.flushConnectionCleanups(aggregated);
|
|
124
|
+
break;
|
|
125
|
+
case 'ip-rejected':
|
|
126
|
+
this.flushIPRejections(aggregated);
|
|
127
|
+
break;
|
|
128
|
+
default:
|
|
129
|
+
this.flushGeneric(aggregated);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Clear events
|
|
133
|
+
aggregated.events.clear();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Flush all pending events
|
|
138
|
+
*/
|
|
139
|
+
public flushAll(): void {
|
|
140
|
+
for (const key of this.aggregatedEvents.keys()) {
|
|
141
|
+
this.flush(key);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private flushConnectionRejections(aggregated: IAggregatedEvent): void {
|
|
146
|
+
const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
|
|
147
|
+
const byReason = new Map<string, number>();
|
|
148
|
+
|
|
149
|
+
for (const [, event] of aggregated.events) {
|
|
150
|
+
const reason = event.data?.reason || 'unknown';
|
|
151
|
+
byReason.set(reason, (byReason.get(reason) || 0) + event.count);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const reasonSummary = Array.from(byReason.entries())
|
|
155
|
+
.sort((a, b) => b[1] - a[1])
|
|
156
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
157
|
+
.join(', ');
|
|
158
|
+
|
|
159
|
+
logger.log('warn', `Rejected ${totalCount} connections`, {
|
|
160
|
+
reasons: reasonSummary,
|
|
161
|
+
uniqueIPs: aggregated.events.size,
|
|
162
|
+
duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
|
|
163
|
+
component: 'connection-dedup'
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
private flushConnectionCleanups(aggregated: IAggregatedEvent): void {
|
|
168
|
+
const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
|
|
169
|
+
const byReason = new Map<string, number>();
|
|
170
|
+
|
|
171
|
+
for (const [, event] of aggregated.events) {
|
|
172
|
+
const reason = event.data?.reason || 'normal';
|
|
173
|
+
byReason.set(reason, (byReason.get(reason) || 0) + event.count);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const reasonSummary = Array.from(byReason.entries())
|
|
177
|
+
.sort((a, b) => b[1] - a[1])
|
|
178
|
+
.slice(0, 5) // Top 5 reasons
|
|
179
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
180
|
+
.join(', ');
|
|
181
|
+
|
|
182
|
+
logger.log('info', `Cleaned up ${totalCount} connections`, {
|
|
183
|
+
reasons: reasonSummary,
|
|
184
|
+
duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
|
|
185
|
+
component: 'connection-dedup'
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
private flushIPRejections(aggregated: IAggregatedEvent): void {
|
|
190
|
+
const byIP = new Map<string, { count: number; reasons: Set<string> }>();
|
|
191
|
+
|
|
192
|
+
for (const [ip, event] of aggregated.events) {
|
|
193
|
+
if (!byIP.has(ip)) {
|
|
194
|
+
byIP.set(ip, { count: 0, reasons: new Set() });
|
|
195
|
+
}
|
|
196
|
+
const ipData = byIP.get(ip)!;
|
|
197
|
+
ipData.count += event.count;
|
|
198
|
+
if (event.data?.reason) {
|
|
199
|
+
ipData.reasons.add(event.data.reason);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Log top offenders
|
|
204
|
+
const topOffenders = Array.from(byIP.entries())
|
|
205
|
+
.sort((a, b) => b[1].count - a[1].count)
|
|
206
|
+
.slice(0, 10)
|
|
207
|
+
.map(([ip, data]) => `${ip} (${data.count}x, ${Array.from(data.reasons).join('/')})`)
|
|
208
|
+
.join(', ');
|
|
209
|
+
|
|
210
|
+
const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
|
|
211
|
+
|
|
212
|
+
logger.log('warn', `Rejected ${totalRejections} connections from ${byIP.size} IPs`, {
|
|
213
|
+
topOffenders,
|
|
214
|
+
duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
|
|
215
|
+
component: 'ip-dedup'
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
private flushGeneric(aggregated: IAggregatedEvent): void {
|
|
220
|
+
const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
|
|
221
|
+
const level = aggregated.events.values().next().value?.level || 'info';
|
|
222
|
+
|
|
223
|
+
// Special handling for IP cleanup events
|
|
224
|
+
if (aggregated.key === 'ip-cleanup') {
|
|
225
|
+
const totalCleaned = Array.from(aggregated.events.values()).reduce((sum, e) => {
|
|
226
|
+
return sum + (e.data?.cleanedIPs || 0) + (e.data?.cleanedRateLimits || 0);
|
|
227
|
+
}, 0);
|
|
228
|
+
|
|
229
|
+
if (totalCleaned > 0) {
|
|
230
|
+
logger.log(level as any, `IP tracking cleanup: removed ${totalCleaned} entries across ${totalCount} cleanup cycles`, {
|
|
231
|
+
duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
|
|
232
|
+
component: 'log-dedup'
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
} else {
|
|
236
|
+
logger.log(level as any, `${aggregated.key}: ${totalCount} events`, {
|
|
237
|
+
uniqueEvents: aggregated.events.size,
|
|
238
|
+
duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
|
|
239
|
+
component: 'log-dedup'
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Cleanup and stop deduplication
|
|
246
|
+
*/
|
|
247
|
+
public cleanup(): void {
|
|
248
|
+
this.flushAll();
|
|
249
|
+
|
|
250
|
+
if (this.globalFlushTimer) {
|
|
251
|
+
clearInterval(this.globalFlushTimer);
|
|
252
|
+
this.globalFlushTimer = undefined;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
for (const aggregated of this.aggregatedEvents.values()) {
|
|
256
|
+
if (aggregated.flushTimer) {
|
|
257
|
+
clearTimeout(aggregated.flushTimer);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
this.aggregatedEvents.clear();
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Global instance for connection-related log deduplication
|
|
265
|
+
export const connectionLogDeduplicator = new LogDeduplicator(5000); // 5 second batches
|
|
266
|
+
|
|
267
|
+
// Ensure logs are flushed on process exit
|
|
268
|
+
process.on('beforeExit', () => {
|
|
269
|
+
connectionLogDeduplicator.flushAll();
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
process.on('SIGINT', () => {
|
|
273
|
+
connectionLogDeduplicator.cleanup();
|
|
274
|
+
process.exit(0);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
process.on('SIGTERM', () => {
|
|
278
|
+
connectionLogDeduplicator.cleanup();
|
|
279
|
+
process.exit(0);
|
|
280
|
+
});
|